Browse Source

working on the ingress gateway

master
forest 2 years ago
parent
commit
4e6fc20c52
  1. 0
      ansible-roles/threshold-client-config-deprecated/files/ReadMe.md
  2. 0
      ansible-roles/threshold-client-config-deprecated/files/test.sh
  3. 0
      ansible-roles/threshold-client-config-deprecated/tasks/main.yml
  4. 0
      ansible-roles/threshold-client-config-deprecated/templates/config.j2
  5. 1
      application-modules/hello-world/docker-compose.yml
  6. 2
      application-modules/servergarden-ingress/docker-compose.yml
  7. 91
      automation/docker.go
  8. 29
      automation/dockerCompose.go
  9. 330
      automation/ingressController.go
  10. 3
      configuration/configuration.go
  11. 15
      main.go
  12. 105
      pki/pki.go
  13. 12
      terraform-modules/ansible-threshold-client/main.tf
  14. 8
      terraform-modules/ansible-threshold-client/playbook.yml

0
ansible-roles/threshold-client-config/files/ReadMe.md → ansible-roles/threshold-client-config-deprecated/files/ReadMe.md

0
ansible-roles/threshold-client-config/files/test.sh → ansible-roles/threshold-client-config-deprecated/files/test.sh

0
ansible-roles/threshold-client-config/tasks/main.yml → ansible-roles/threshold-client-config-deprecated/tasks/main.yml

0
ansible-roles/threshold-client-config/templates/config.j2 → ansible-roles/threshold-client-config-deprecated/templates/config.j2

1
application-modules/hello-world/docker-compose.yml

@ -7,7 +7,6 @@ services:
labels:
servergarden-ingress-80-public-port: 443
servergarden-ingress-80-public-protocol: https
servergarden-ingress-80-public-subdomain: ""
servergarden-ingress-80-container-protocol: http
networks:

2
application-modules/servergarden-ingress/docker-compose.yml

@ -1,7 +1,7 @@
version: "3.3"
services:
threshold:
image: sequentialread/threshold:0.0.1
image: sequentialread/threshold:0.0.5
command: ["-mode", "client", "-configFile", "/threshold/config/config.json"]
volumes:
- type: bind

91
automation/docker.go

@ -0,0 +1,91 @@
package automation
import (
"encoding/json"
"fmt"
"net/http"
errors "git.sequentialread.com/forest/pkg-errors"
"git.sequentialread.com/forest/rootsystem/configuration"
)
type DockerContainer struct {
Id string
Names []string
Labels map[string]string
NetworkSettings DockerContainerNetworkSettings
}
type DockerContainerNetworkSettings struct {
Networks map[string]DockerContainerNetwork
}
type DockerContainerNetwork struct {
NetworkID string
IPAddress string
}
type DockerNetwork struct {
Id string
Name string
}
func (container DockerContainer) GetDisplayName() string {
if container.Names != nil && len(container.Names) > 0 {
return fmt.Sprintf("%s (%s)", container.Names[0], container.Id)
} else {
return container.Id
}
}
func (container DockerContainer) GetShortName() string {
if container.Names != nil && len(container.Names) > 0 {
return container.Names[0]
} else {
return container.Id
}
}
func myDockerGet(endpoint string) ([]byte, error) {
response, bytes, err := unixHTTP("GET", configuration.DOCKER_SOCKET, fmt.Sprintf("/v1.40/%s", endpoint), nil)
if err != nil {
return nil, errors.Wrap(err, "can't talk to docker api")
}
if response.StatusCode != http.StatusOK {
return nil, fmt.Errorf(
"docker api (%s) returned HTTP %d: %s",
endpoint, response.StatusCode, string(bytes))
}
return bytes, nil
}
//https://docs.docker.com/engine/api/v1.40/#tag/Container
func ListDockerContainers() ([]DockerContainer, error) {
bytes, err := myDockerGet("containers/json?all=true")
if err != nil {
return nil, err
}
var containers []DockerContainer
err = json.Unmarshal(bytes, &containers)
if err != nil {
return nil, errors.Wrap(err, "docker API json parse error")
}
return containers, nil
}
//https://docs.docker.com/engine/api/v1.40/#tag/Network
func ListDockerNetworks() ([]DockerNetwork, error) {
bytes, err := myDockerGet("networks")
if err != nil {
return nil, err
}
var networks []DockerNetwork
err = json.Unmarshal(bytes, &networks)
if err != nil {
return nil, errors.Wrap(err, "docker API json parse error")
}
return networks, nil
}

29
automation/dockerCompose.go

@ -1,13 +1,10 @@
package automation
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net"
"net/http"
"os"
"os/exec"
"path"
@ -22,11 +19,6 @@ import (
composetypes "github.com/docker/cli/cli/compose/types"
)
type DockerContainer struct {
ID string
Labels map[string]string
}
func DockerComposeUp(
config *configuration.Configuration,
workingDirectory string,
@ -345,24 +337,7 @@ func dockerComposePlan(
tasks := []func() TaskResult{
func() TaskResult {
unixHTTPClient := http.Client{
Transport: &http.Transport{
DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
return net.Dial("unix", configuration.DOCKER_SOCKET)
},
},
}
var bytes []byte
var containers []DockerContainer
response, err := unixHTTPClient.Get("http://localhost/v1.40/containers/json?all=true")
if err == nil {
bytes, err = ioutil.ReadAll(response.Body)
}
if err == nil {
err = json.Unmarshal(bytes, &containers)
}
containers, err := ListDockerContainers()
if err != nil {
return TaskResult{
@ -481,7 +456,7 @@ func dockerComposePlan(
for _, service := range config.Services {
container := containersByModuleService[fmt.Sprintf("%s_%s", moduleName, service.Name)]
plan := "none"
if container.ID != "" {
if container.Id != "" {
existingConfigHash := container.Labels["com.docker.compose.config-hash"]
getConfigHashTaskName := fmt.Sprintf("%s_get_compose_config_hash", moduleName)
newConfigHash := results[getConfigHashTaskName].Result.(map[string]string)[service.Name]

330
automation/ingressController.go

@ -0,0 +1,330 @@
package automation
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net"
"net/http"
"regexp"
"strconv"
"strings"
errors "git.sequentialread.com/forest/pkg-errors"
"git.sequentialread.com/forest/rootsystem/configuration"
)
type ListenerConfig struct {
HaProxyProxyProtocol bool
ListenAddress string
ListenHostnameGlob string
ListenPort int
BackEndService string
ClientIdentifier string
}
type ThresholdLiveConfig struct {
Listeners []ListenerConfig
ServiceToLocalAddrMap map[string]string
}
type ContainerConfig struct {
PublicPort int
PublicProtocol string
PublicHostnameGlob string
ContainerProtocol string
ContainerAddress string
ContainerName string
HaProxyProxyProtocol bool
}
type CaddyApp struct {
Servers map[string]*CaddyServer `json:"servers"`
}
type CaddyServer struct {
Listen []string `json:"listen"`
Routes []CaddyRoute `json:"routes"`
}
type CaddyRoute struct {
Handle []CaddyHandler `json:"handle"`
Match []CaddyMatch `json:"match"`
Terminal bool `json:"terminal"`
}
// https://caddyserver.com/docs/json/apps/http/servers/routes/handle/
type CaddyHandler struct {
Handler string `json:"handler"`
Routes []CaddyRoute `json:"routes"`
Upstreams []CaddyUpstream `json:"upstreams"`
}
// https://caddyserver.com/docs/json/apps/http/servers/routes/handle/reverse_proxy/
type CaddyUpstream struct {
Dial string `json:"dial"`
}
// https://caddyserver.com/docs/json/apps/http/servers/routes/match/
type CaddyMatch struct {
Host []string `json:"host"`
}
func IngressConfig(config *configuration.Configuration) error {
// i think the network name is the key of the networks map on the container object so this is not needed
// networks, err := ListDockerNetworks()
// if err != nil {
// return errors.Wrap(err, "can't list docker networks")
// }
// ingressNetworkId := ""
// for _, network := range networks {
// if network.Name == "servergarden-ingress_default" {
// ingressNetworkId = network.Id
// }
// }
// if ingressNetworkId == "" {
// return errors.New("ingress docker network was not found")
// }
containers, err := ListDockerContainers()
if err != nil {
return errors.Wrap(err, "can't list docker containers")
}
// servergarden-ingress-80-public-port: 443
// servergarden-ingress-80-public-protocol: https
// servergarden-ingress-80-public-subdomain: ""
// servergarden-ingress-80-container-protocol: http
ingressLabelRegexp := regexp.MustCompile("servergarden-ingress-([0-9]+)-((public-port)|(public-protocol)|(public-hostname-glob)|(container-protocol)|(haproxy-proxy-protocol))")
thresholdConfig := ThresholdLiveConfig{
Listeners: []ListenerConfig{},
ServiceToLocalAddrMap: map[string]string{},
}
containerConfigs := map[string]*ContainerConfig{}
caddyIpAddress := ""
for _, container := range containers {
ipAddress := ""
for networkName, containerNetwork := range container.NetworkSettings.Networks {
if networkName == "servergarden-ingress_default" {
ipAddress = containerNetwork.IPAddress
}
}
for _, name := range container.Names {
if strings.TrimPrefix(name, "/") == "servergarden-ingress_caddy_1" {
caddyIpAddress = ipAddress
}
}
for key, value := range container.Labels {
matches := ingressLabelRegexp.FindAllStringSubmatch(key, -1)
if strings.HasPrefix(key, "servergarden-ingress") && len(matches) == 0 {
return errors.Wrapf(
err, "failed to parse container %s ingress label '%s'. please refer to the documentation for valid label formats (TODO include a link here)",
container.GetDisplayName(), key,
)
}
if len(matches) > 0 {
port, _ := strconv.Atoi(matches[0][1])
labelType := matches[0][2]
if ipAddress == "" {
return fmt.Errorf(
"container %s has an ingress label '%s' but it doesn't have an IP address on the ingress network",
container.GetDisplayName(), key,
)
}
if _, has := containerConfigs[container.Id]; !has {
containerConfigs[container.Id] = &ContainerConfig{
ContainerAddress: fmt.Sprintf("%s:%d", ipAddress, port),
ContainerName: container.GetShortName(),
}
}
if labelType == "public-protocol" {
containerConfigs[container.Id].PublicProtocol = value
}
if labelType == "public-port" {
port, err := strconv.Atoi(value)
if err != nil {
return errors.Wrapf(
err, "container %s public-port ingress label must be an integer ('%s' was given)",
container.GetDisplayName(), value,
)
}
containerConfigs[container.Id].PublicPort = port
}
if labelType == "public-hostname-glob" {
containerConfigs[container.Id].PublicHostnameGlob = value
}
if labelType == "container-protocol" {
containerConfigs[container.Id].ContainerProtocol = value
}
if labelType == "haproxy-proxy-protocol" {
lowerValue := strings.ToLower(value)
if lowerValue == "t" || lowerValue == "true" || lowerValue == "yes" || lowerValue == "1" {
containerConfigs[container.Id].HaProxyProxyProtocol = true
}
}
}
}
}
if caddyIpAddress == "" {
return errors.New("unable to obtain ip address for caddy container")
}
caddyConfig := map[string]*CaddyApp{}
publicProtocols := map[string][]*ContainerConfig{}
for _, x := range containerConfigs {
if x.PublicPort == 0 {
return fmt.Errorf(
"found ingress label for container %s but it is missing the required public-port label",
x.ContainerName,
)
}
serviceName := x.ContainerName
if x.PublicProtocol == "https" {
serviceName = "https"
thresholdConfig.ServiceToLocalAddrMap[serviceName] = fmt.Sprintf("%s:%d", caddyIpAddress, 443)
} else {
thresholdConfig.ServiceToLocalAddrMap[serviceName] = x.ContainerAddress
}
thresholdConfig.Listeners = append(thresholdConfig.Listeners, ListenerConfig{
HaProxyProxyProtocol: x.HaProxyProxyProtocol,
ListenAddress: "0.0.0.0",
ListenHostnameGlob: x.PublicHostnameGlob,
ListenPort: x.PublicPort,
BackEndService: serviceName,
ClientIdentifier: config.Host.Name, // TODO failover stuff
})
if _, has := publicProtocols[x.PublicProtocol]; !has {
publicProtocols[x.PublicProtocol] = []*ContainerConfig{}
}
publicProtocols[x.PublicProtocol] = append(publicProtocols[x.PublicProtocol], x)
}
for protocol, containerConfigs := range publicProtocols {
if protocol == "https" {
caddyConfig["http"] = &CaddyApp{
Servers: map[string]*CaddyServer{
"srv0": {
Listen: []string{":443"},
Routes: []CaddyRoute{},
},
},
}
for _, container := range containerConfigs {
newRoute := CaddyRoute{
Handle: []CaddyHandler{
CaddyHandler{
Handler: "reverse_proxy",
Upstreams: []CaddyUpstream{
CaddyUpstream{
Dial: container.ContainerAddress,
},
},
},
},
Match: []CaddyMatch{
CaddyMatch{
Host: []string{
config.Terraform.Variables["domain_name"],
},
},
},
Terminal: true,
}
if container.PublicHostnameGlob != "" {
newRoute.Match = []CaddyMatch{
CaddyMatch{
Host: []string{
container.PublicHostnameGlob,
},
},
}
}
caddyConfig["http"].Servers["srv0"].Routes = append(
caddyConfig["http"].Servers["srv0"].Routes,
newRoute,
)
}
} else {
// TODO support TCP and TLS (udp??)
return fmt.Errorf(
"unsupported public-protocol %s on container '%s'. currently only https is supported",
protocol, containerConfigs[0].ContainerName,
)
}
}
thresholdConfigBytes, _ := json.MarshalIndent(thresholdConfig, "", " ")
caddyConfigBytes, _ := json.MarshalIndent(caddyConfig, "", " ")
log.Printf("thresholdConfigBytes: %s\n\n", string(thresholdConfigBytes))
log.Printf("caddyConfigBytes: %s\n\n", string(caddyConfigBytes))
thresholdResponse, thresholdResponseBytes, err := unixHTTP(
"PUT", configuration.THRESHOLD_SOCKET, "/liveconfig",
thresholdConfigBytes,
)
if err != nil {
return errors.Wrap(err, "failed to call threshold liveconfig api")
}
if thresholdResponse.StatusCode != http.StatusOK {
return fmt.Errorf("threshold liveconfig api returned HTTP %d: %s", thresholdResponse.StatusCode, string(thresholdResponseBytes))
}
caddyResponse, caddyResponseBytes, err := unixHTTP(
"POST", configuration.CADDY_SOCKET, "/config/apps",
caddyConfigBytes,
)
if err != nil {
return errors.Wrap(err, "failed to call caddy admin api")
}
if thresholdResponse.StatusCode != http.StatusOK {
return fmt.Errorf("caddy admin api returned HTTP %d: %s", caddyResponse.StatusCode, string(caddyResponseBytes))
}
return nil
}
func unixHTTP(method, socketFile, endpoint string, body []byte) (*http.Response, []byte, error) {
unixHTTPClient := http.Client{
Transport: &http.Transport{
DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
return net.Dial("unix", socketFile)
},
},
}
request, err := http.NewRequest(method, fmt.Sprintf("http://localhost%s", endpoint), bytes.NewReader(body))
if err != nil {
return nil, nil, errors.Wrapf(err, "unixHTTP could not create request object (%s)", socketFile)
}
if body != nil {
request.Header.Add("content-type", "application/json")
}
response, err := unixHTTPClient.Do(request)
if err != nil {
return nil, nil, errors.Wrapf(err, "unixHTTP failed (%s)", socketFile)
}
bytes, err := ioutil.ReadAll(response.Body)
if err != nil {
return nil, nil, errors.Wrapf(err, "unixHTTP read error (%s)", socketFile)
}
return response, bytes, nil
}

3
configuration/configuration.go

@ -65,6 +65,9 @@ const ANSIBLE_PLAYBOOK_FILE_NAME = "playbook.yml"
const TERRAFORM_PLAN_FILE_NAME = "terraform-plan-file"
const ANSIBLE_WRAPPER_PATH = "ansible-wrapper"
const DOCKER_SOCKET = "/var/run/docker.sock"
const THRESHOLD_SOCKET = "/var/run/servergarden/threshold/threshold.sock"
const CADDY_SOCKET = "/var/run/servergarden/caddy/caddy.sock"
const CADDY_DATA = "/var/lib/servergarden/caddy/data/"
const TERRAFORM_APPLY_STATUS_UPDATE_INTERVAL_SECONDS = 5
func GET_ANSIBLE_WRAPPER_FILES() []string {

15
main.go

@ -6,8 +6,10 @@ import (
"log"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"time"
errors "git.sequentialread.com/forest/pkg-errors"
"git.sequentialread.com/forest/rootsystem/automation"
@ -219,9 +221,10 @@ func main() {
} else if !success {
log.Printf("rootsystem %s build failed", projectName)
} else {
os.MkdirAll("/var/run/servergarden/threshold/", 0o700)
os.MkdirAll("/var/run/servergarden/caddy/", 0o700)
os.MkdirAll("/var/lib/servergarden/caddy/data/", 0o700)
os.MkdirAll(filepath.Dir(configuration.THRESHOLD_SOCKET), 0o700)
os.MkdirAll(filepath.Dir(configuration.CADDY_SOCKET), 0o700)
os.MkdirAll(configuration.CADDY_DATA, 0o700)
svg, statusChannel, err := automation.DockerComposeUp(config, workingDirectory)
if err != nil {
@ -247,6 +250,12 @@ func main() {
log.Printf("rootsystem docker-compose errored out (exception): %+v", err)
} else if !success {
log.Printf("rootsystem docker-compose failed")
} else {
time.Sleep(5 * time.Second)
err = automation.IngressConfig(config)
if err != nil {
log.Printf("rootsystem IngressConfig failed: %+v", err)
}
}
}
}

105
pki/pki.go

@ -6,6 +6,7 @@ import (
"encoding/pem"
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"time"
@ -61,7 +62,9 @@ func BuildTLSCertsForThreshold(
}
}
domainCACertFile, notFound, err := storage.Get(fmt.Sprintf("threshold/%s.crt", domainCA))
objectStorageKey := fmt.Sprintf("threshold/%s.crt", domainCA)
domainCACertFile, notFound, err := storage.Get(objectStorageKey)
if err != nil && !notFound {
return errors.Wrap(err, "BuildTLSCertsForThreshold(): failed trying to get domain CA from object storage:")
}
@ -69,9 +72,13 @@ func BuildTLSCertsForThreshold(
domainKeyLocalPath := filepath.Join(thresholdServerConfigRole, fmt.Sprintf("%s.key", domain))
domainCertLocalPath := filepath.Join(thresholdServerConfigRole, fmt.Sprintf("%s.crt", domain))
log.Printf("BuildTLSCertsForThreshold(): object storage file '%s' exists: %t\n", objectStorageKey, !notFound)
// if the file was not already uploaded to object storage, that must mean
// we are the first node to run -- therefore we must create and upload it
if notFound {
log.Println("BuildTLSCertsForThreshold(): creating threshold server CA")
// Create a CA for the server's key/cert
err = pki.Sign(
nil,
@ -158,6 +165,8 @@ func BuildTLSCertsForThreshold(
// since it runs on cloud, on someone elses computer
} else {
log.Println("BuildTLSCertsForThreshold(): using existing threshold server CA")
err = ioutil.WriteFile(domainCACertLocalPath, domainCACertFile.Content, 0600)
if err != nil {
return errors.Wrap(err, "BuildTLSCertsForThreshold(): writing domainCA file to ansible role failed")
@ -182,54 +191,64 @@ func BuildTLSCertsForThreshold(
}
}
// Create A CA for this client
err = pki.Sign(
nil,
&easypki.Request{
Name: clientCA,
Template: &x509.Certificate{
NotAfter: time.Now().Add(time.Hour * 24 * 720),
IsCA: true,
MaxPathLen: -1,
Subject: getSubject(clientCA),
// Create A CA for this client if it doesn't already exist
clientCAFilePath := filepath.Join(thresholdRegisterClientWithServerConfigRole, fmt.Sprintf("%s.crt", clientCA))
_, statErr := os.Stat(clientCAFilePath)
log.Printf("BuildTLSCertsForThreshold(): file '%s' exists: %t\n", clientCAFilePath, !os.IsNotExist(statErr))
if os.IsNotExist(statErr) {
log.Println("BuildTLSCertsForThreshold(): creating threshold client CA")
err = pki.Sign(
nil,
&easypki.Request{
Name: clientCA,
Template: &x509.Certificate{
NotAfter: time.Now().Add(time.Hour * 24 * 720),
IsCA: true,
MaxPathLen: -1,
Subject: getSubject(clientCA),
},
},
},
)
if err != nil {
return errors.Wrap(err, "BuildTLSCertsForThreshold(): failed creating CA")
}
)
if err != nil {
return errors.Wrap(err, "BuildTLSCertsForThreshold(): failed creating CA")
}
err = saveBundle(pki, clientCA, clientCA, false, thresholdRegisterClientWithServerConfigRole)
if err != nil {
return errors.Wrap(err, "BuildTLSCertsForThreshold(): saveBundle():")
}
err = saveBundle(pki, clientCA, clientCA, false, thresholdRegisterClientWithServerConfigRole)
if err != nil {
return errors.Wrap(err, "BuildTLSCertsForThreshold(): saveBundle():")
}
clientCASigner, err := pki.GetCA(clientCA)
if err != nil {
return errors.Wrap(err, "BuildTLSCertsForThreshold(): signer named \"CA\" was not found")
}
clientCASigner, err := pki.GetCA(clientCA)
if err != nil {
return errors.Wrap(err, "BuildTLSCertsForThreshold(): signer named \"CA\" was not found")
}
// Create Threshold client certificates
clientEmail := fmt.Sprintf("%s@%s", clientId, domain)
err = pki.Sign(
clientCASigner,
&easypki.Request{
Name: clientEmail,
Template: &x509.Certificate{
NotAfter: time.Now().Add(time.Hour * 24 * 720),
IsCA: false,
Subject: getSubject(clientEmail),
EmailAddresses: []string{clientEmail},
// Create Threshold client certificates
clientEmail := fmt.Sprintf("%s@%s", clientId, domain)
err = pki.Sign(
clientCASigner,
&easypki.Request{
Name: clientEmail,
Template: &x509.Certificate{
NotAfter: time.Now().Add(time.Hour * 24 * 720),
IsCA: false,
Subject: getSubject(clientEmail),
EmailAddresses: []string{clientEmail},
},
},
},
)
if err != nil {
return errors.Wrap(err, "BuildTLSCertsForThreshold(): failed creating Threshold client certificate")
}
)
if err != nil {
return errors.Wrap(err, "BuildTLSCertsForThreshold(): failed creating Threshold client certificate")
}
err = saveBundle(pki, clientCA, clientEmail, true, thresholdClientConfigMount)
if err != nil {
return errors.Wrap(err, "BuildTLSCertsForThreshold(): saveBundle():")
err = saveBundle(pki, clientCA, clientEmail, true, thresholdClientConfigMount)
if err != nil {
return errors.Wrap(err, "BuildTLSCertsForThreshold(): saveBundle():")
}
} else {
log.Println("BuildTLSCertsForThreshold(): using existing threshold client CA")
}
return nil

12
terraform-modules/ansible-threshold-client/main.tf

@ -27,11 +27,19 @@ variable "ingress_host_list" {
resource "null_resource" "ansible_playbook" {
count = length(var.ingress_host_list)
// always run this for now because I can't be bothered to figure out what's the correct way to trigger it.
// this should not be required, but in some edge cases it is required to run again. it doesn't hurt much to
// just run it every time.
triggers = {
id = var.ingress_host_list[count.index].known_hosts_file_name
domain = var.domain_name
always_run = timestamp()
}
// triggers = {
// id = var.ingress_host_list[count.index].known_hosts_file_name
// domain = var.domain_name
// }
// now that the servers in the ingress_host_list have had thier host keys added to known_hosts,
// we can proceed with runnning ansible (ssh to the server and install things).
// the ansible-playbook-wrapper as well as the ansible config & roles folder will be linked into this directory

8
terraform-modules/ansible-threshold-client/playbook.yml

@ -4,11 +4,3 @@
gather_facts: no
roles:
- threshold-register-client-with-server
- name: install threshold in client mode on localhost
hosts: localhost
gather_facts: no
roles:
- threshold
- threshold-client-config

Loading…
Cancel
Save