rootsystem/automation/ingressController.go

package automation

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"strings"

	errors "git.sequentialread.com/forest/pkg-errors"
	"git.sequentialread.com/forest/rootsystem/configuration"
)

type ListenerConfig struct {
	HaProxyProxyProtocol bool
	ListenAddress        string
	ListenHostnameGlob   string
	ListenPort           int
	BackEndService       string
	ClientId             string
}

type ThresholdLiveConfig struct {
	Listeners             []ListenerConfig
	ServiceToLocalAddrMap map[string]string
}

type ContainerConfig struct {
	PublicPort           int
	PublicProtocol       string
	PublicHostnameGlob   string
	ContainerProtocol    string
	ContainerAddress     string
	ContainerName        string
	HaProxyProxyProtocol bool
}

type CaddyApp struct {
	Servers map[string]*CaddyServer `json:"servers"`
}

type CaddyServer struct {
	Listen []string     `json:"listen"`
	Routes []CaddyRoute `json:"routes"`
}

type CaddyRoute struct {
	Handle   []CaddyHandler `json:"handle,omitempty"`
	Match    []CaddyMatch   `json:"match,omitempty"`
	Terminal bool           `json:"terminal"`
}

// https://caddyserver.com/docs/json/apps/http/servers/routes/handle/
type CaddyHandler struct {
	Handler   string          `json:"handler"`
	Routes    []CaddyRoute    `json:"routes,omitempty"`
	Upstreams []CaddyUpstream `json:"upstreams,omitempty"`
}

// https://caddyserver.com/docs/json/apps/http/servers/routes/handle/reverse_proxy/
type CaddyUpstream struct {
	Dial string `json:"dial"`
}

// https://caddyserver.com/docs/json/apps/http/servers/routes/match/
type CaddyMatch struct {
	Host []string `json:"host"`
}

func IngressConfig(config *configuration.Configuration) error {

	// i think the network name is the key of the networks map on the container object so this is not needed
	// networks, err := ListDockerNetworks()
	// if err != nil {
	// 	return errors.Wrap(err, "can't list docker networks")
	// }
	// ingressNetworkId := ""
	// for _, network := range networks {
	// 	if network.Name == "servergarden-ingress_default" {
	// 		ingressNetworkId = network.Id
	// 	}
	// }
	// if ingressNetworkId == "" {
	// 	return errors.New("ingress docker network was not found")
	// }

	containers, err := ListDockerContainers()
	if err != nil {
		return errors.Wrap(err, "can't list docker containers")
	}

	// servergarden-ingress-80-public-port: 443
	// servergarden-ingress-80-public-protocol: https
	// servergarden-ingress-80-public-subdomain: ""
	// servergarden-ingress-80-container-protocol: http
	ingressLabelRegexp := regexp.MustCompile("servergarden-ingress-([0-9]+)-((public-port)|(public-protocol)|(public-hostname-glob)|(container-protocol)|(haproxy-proxy-protocol))")

	thresholdConfig := ThresholdLiveConfig{
		Listeners:             []ListenerConfig{},
		ServiceToLocalAddrMap: map[string]string{},
	}

	containerConfigs := map[string]*ContainerConfig{}
	caddyIpAddress := ""

	for _, container := range containers {
		ipAddress := ""
		for networkName, containerNetwork := range container.NetworkSettings.Networks {
			if networkName == "servergarden-ingress_default" {
				ipAddress = containerNetwork.IPAddress
			}
		}
		for _, name := range container.Names {
			if strings.TrimPrefix(name, "/") == "servergarden-ingress_caddy_1" {
				caddyIpAddress = ipAddress
			}
		}
		for key, value := range container.Labels {
			matches := ingressLabelRegexp.FindAllStringSubmatch(key, -1)
			if strings.HasPrefix(key, "servergarden-ingress") && len(matches) == 0 {
				return errors.Wrapf(
					err, "failed to parse container %s ingress label '%s'. please refer to the documentation for valid label formats (TODO include a link here)",
					container.GetDisplayName(), key,
				)
			}
			if len(matches) > 0 {
				port, _ := strconv.Atoi(matches[0][1])
				labelType := matches[0][2]

				if ipAddress == "" {
					return fmt.Errorf(
						"container %s has an ingress label '%s' but it doesn't have an IP address on the ingress network",
						container.GetDisplayName(), key,
					)
				}

				if _, has := containerConfigs[container.Id]; !has {
					containerConfigs[container.Id] = &ContainerConfig{
						ContainerAddress: fmt.Sprintf("%s:%d", ipAddress, port),
						ContainerName:    container.GetShortName(),
					}
				}

				if labelType == "public-protocol" {
					containerConfigs[container.Id].PublicProtocol = value
				}
				if labelType == "public-port" {
					port, err := strconv.Atoi(value)
					if err != nil {
						return errors.Wrapf(
							err, "container %s public-port ingress label must be an integer ('%s' was given)",
							container.GetDisplayName(), value,
						)
					}
					containerConfigs[container.Id].PublicPort = port
				}
				if labelType == "public-hostname-glob" {
					containerConfigs[container.Id].PublicHostnameGlob = value
				}
				if labelType == "container-protocol" {
					containerConfigs[container.Id].ContainerProtocol = value
				}
				if labelType == "haproxy-proxy-protocol" {
					lowerValue := strings.ToLower(value)
					if lowerValue == "t" || lowerValue == "true" || lowerValue == "yes" || lowerValue == "1" {
						containerConfigs[container.Id].HaProxyProxyProtocol = true
					}
				}
			}
		}
	}

	if caddyIpAddress == "" {
		return errors.New("unable to obtain ip address for caddy container")
	}

	caddyConfig := map[string]*CaddyApp{}

	publicProtocols := map[string][]*ContainerConfig{}

	for _, x := range containerConfigs {
		if x.PublicPort == 0 {
			return fmt.Errorf(
				"found ingress label for container %s but it is missing the required public-port label",
				x.ContainerName,
			)
		}

		serviceName := x.ContainerName
		if x.PublicProtocol == "https" {
			serviceName = "https"
			thresholdConfig.ServiceToLocalAddrMap[serviceName] = fmt.Sprintf("%s:%d", caddyIpAddress, 443)
		} else {
			thresholdConfig.ServiceToLocalAddrMap[serviceName] = x.ContainerAddress
		}

		thresholdConfig.Listeners = append(thresholdConfig.Listeners, ListenerConfig{
			HaProxyProxyProtocol: x.HaProxyProxyProtocol,
			ListenAddress:        "0.0.0.0",
			ListenHostnameGlob:   x.PublicHostnameGlob,
			ListenPort:           x.PublicPort,
			BackEndService:       serviceName,
			ClientId:             config.Host.Name, // TODO failover stuff
		})

		if _, has := publicProtocols[x.PublicProtocol]; !has {
			publicProtocols[x.PublicProtocol] = []*ContainerConfig{}
		}
		publicProtocols[x.PublicProtocol] = append(publicProtocols[x.PublicProtocol], x)
	}

	for protocol, containerConfigs := range publicProtocols {
		if protocol == "https" {
			caddyConfig["http"] = &CaddyApp{
				Servers: map[string]*CaddyServer{
					"srv0": {
						Listen: []string{":443"},
						Routes: []CaddyRoute{},
					},
				},
			}

			for _, container := range containerConfigs {
				newRoute := CaddyRoute{
					Handle: []CaddyHandler{
						CaddyHandler{
							Handler: "reverse_proxy",
							Upstreams: []CaddyUpstream{
								CaddyUpstream{
									Dial: container.ContainerAddress,
								},
							},
						},
					},
					Match: []CaddyMatch{
						CaddyMatch{
							Host: []string{
								config.Terraform.Variables["domain_name"],
							},
						},
					},
					Terminal: true,
				}

				if container.PublicHostnameGlob != "" {
					newRoute.Match = []CaddyMatch{
						CaddyMatch{
							Host: []string{
								container.PublicHostnameGlob,
							},
						},
					}
				}
				caddyConfig["http"].Servers["srv0"].Routes = append(
					caddyConfig["http"].Servers["srv0"].Routes,
					newRoute,
				)
			}
		} else {
			// TODO support TCP and TLS (udp??)
			return fmt.Errorf(
				"unsupported public-protocol %s on container '%s'. currently only https is supported",
				protocol, containerConfigs[0].ContainerName,
			)
		}
	}

	thresholdConfigBytes, _ := json.MarshalIndent(thresholdConfig, "", "  ")
	caddyConfigBytes, _ := json.MarshalIndent(caddyConfig, "", "  ")

	log.Printf("thresholdConfigBytes: %s\n\n", string(thresholdConfigBytes))
	log.Printf("caddyConfigBytes: %s\n\n", string(caddyConfigBytes))

	thresholdResponse, thresholdResponseBytes, err := unixHTTP(
		"PUT", configuration.THRESHOLD_SOCKET, "/liveconfig",
		thresholdConfigBytes,
	)
	if err != nil {
		return errors.Wrap(err, "failed to call threshold liveconfig api")
	}
	if thresholdResponse.StatusCode != http.StatusOK {
		return fmt.Errorf("threshold liveconfig api returned HTTP %d: %s", thresholdResponse.StatusCode, string(thresholdResponseBytes))
	}

	caddyResponse, caddyResponseBytes, err := unixHTTP(
		"POST", configuration.CADDY_SOCKET, "/config/apps",
		caddyConfigBytes,
	)
	if err != nil {
		return errors.Wrap(err, "failed to call caddy admin api")
	}
	if caddyResponse.StatusCode != http.StatusOK {
		return fmt.Errorf("caddy admin api returned HTTP %d: %s", caddyResponse.StatusCode, string(caddyResponseBytes))
	}

	return nil
}

func unixHTTP(method, socketFile, endpoint string, body []byte) (*http.Response, []byte, error) {
	unixHTTPClient := http.Client{
		Transport: &http.Transport{
			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
				return net.Dial("unix", socketFile)
			},
		},
	}

	request, err := http.NewRequest(method, fmt.Sprintf("http://localhost%s", endpoint), bytes.NewReader(body))
	if err != nil {
		return nil, nil, errors.Wrapf(err, "unixHTTP could not create request object (%s)", socketFile)
	}
	if body != nil {
		request.Header.Add("content-type", "application/json")
	}
	response, err := unixHTTPClient.Do(request)
	if err != nil {
		return nil, nil, errors.Wrapf(err, "unixHTTP failed (%s)", socketFile)
	}
	bytes, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, nil, errors.Wrapf(err, "unixHTTP read error (%s)", socketFile)
	}
	return response, bytes, nil
}