Browse Source

docker-compose automation continued

master
forest 2 years ago
parent
commit
9d54f00b87
  1. 564
      automation/dockerCompose.go
  2. 30
      automation/terraformActions.go
  3. 459
      automation/terraformSvg.go
  4. 70
      main.go

564
automation/dockerCompose.go

@ -1,16 +1,23 @@
package automation
import (
"context"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path"
"strings"
"time"
"git.sequentialread.com/forest/rootsystem/configuration"
composeloader "github.com/docker/cli/cli/compose/loader"
composeschema "github.com/docker/cli/cli/compose/schema"
composetypes "github.com/docker/cli/cli/compose/types"
dockertypes "github.com/docker/docker/api/types"
dockerclient "github.com/docker/docker/client"
"github.com/pkg/errors"
)
@ -67,7 +74,7 @@ func DockerComposeUp(
configs := map[string]*composetypes.Config{}
moduleByNetwork := map[string]string{}
moduleDependencies := map[string][]string{}
connectionsMap := map[string]map[string]SimplifiedTerraformConnection{}
for _, moduleName := range config.ApplicationModules {
config, err := composeloader.Load(*(applicationModulesMap[moduleName]))
@ -108,16 +115,19 @@ func DockerComposeUp(
for _, network := range config.Networks {
dependsOnModule := moduleByNetwork[network.External.Name]
if network.External.External && dependsOnModule != "" {
if moduleDependencies[moduleName] == nil {
moduleDependencies[moduleName] = []string{dependsOnModule}
} else {
moduleDependencies[moduleName] = append(moduleDependencies[moduleName], dependsOnModule)
if connectionsMap[moduleName] == nil {
connectionsMap[moduleName] = map[string]SimplifiedTerraformConnection{}
}
connectionsMap[moduleName][dependsOnModule] = SimplifiedTerraformConnection{
From: dependsOnModule,
To: moduleName,
DisplayName: network.External.Name,
}
}
}
//TODO external volumes as well..?
//TODO depends_on.. ?
//TODO dis-allow links?
//TODO dis-allow links? other validation?
}
sortedApplicationModulesMap := map[string]bool{}
@ -129,9 +139,9 @@ func DockerComposeUp(
for _, moduleName := range config.ApplicationModules {
if !sortedApplicationModulesMap[moduleName] {
allDependenciesMet := true
dependencies := moduleDependencies[moduleName]
dependencies := connectionsMap[moduleName]
if dependencies != nil {
for _, dependsOnModule := range dependencies {
for dependsOnModule := range dependencies {
if !sortedApplicationModulesMap[dependsOnModule] {
allDependenciesMet = false
}
@ -151,98 +161,488 @@ func DockerComposeUp(
)
}
svg, err := makeSVGFromDockerCompose(moduleDependencies, configs)
connections := []SimplifiedTerraformConnection{}
for _, dependsOn := range connectionsMap {
for _, connection := range dependsOn {
connections = append(connections, connection)
}
}
simpleStatus, err := dockerComposePlan(configs, applicationModulesMap, connections)
if err != nil {
return nil, nil, errors.Wrap(err, "can't create docker-compose plan")
}
svgBytes, err := makeSVGFromSimpleStatus(simpleStatus)
//svg, err := makeSVGFromDockerCompose(moduleDependencies, applicationModulesMap, configs)
if err != nil {
return nil, nil, errors.Wrap(err, "can't create svg diagram from docker-compose plan document")
}
outputChannel := make(chan TerraformApplyResult)
go (func() {
logLinesChannel := make(chan string)
logLines := []string{}
dockerComposeIsRunning := true
go (func() {
for logLine := range logLinesChannel {
logLines = append(logLines, logLine)
}
})()
abort := func(err error) {
dockerComposeIsRunning = false
outputChannel <- TerraformApplyResult{
Error: err,
Complete: true,
Success: false,
Log: strings.Join(logLines, "\n"),
Status: simpleStatus,
}
close(outputChannel)
}
go (func() {
for dockerComposeIsRunning {
time.Sleep(time.Second * configuration.TERRAFORM_APPLY_STATUS_UPDATE_INTERVAL_SECONDS)
if dockerComposeIsRunning {
outputChannel <- TerraformApplyResult{
Error: nil,
Log: strings.Join(logLines, "\n"),
Status: simpleStatus,
}
}
}
})()
dockerComposeSuccess := true
for _, moduleName := range sortedApplicationModules {
// TODO handle panics in here with abort
details := applicationModulesMap[moduleName]
process := exec.Command("docker-compose", "--no-ansi", "up", "-d")
logLinesChannel <- fmt.Sprintf("\n(%s) $ docker-compose --no-ansi up -d\n", details.WorkingDir)
process.Dir = details.WorkingDir
stdoutPipe, err := process.StdoutPipe()
if err != nil {
abort(errors.Wrap(err, "can't DockerComposeUp because can't process.StdoutPipe() docker-compose process"))
return
}
stderrPipe, err := process.StderrPipe()
if err != nil {
abort(errors.Wrap(err, "can't DockerComposeUp because can't process.StderrPipe() docker-compose process"))
return
}
err = process.Start()
if err != nil {
abort(errors.Wrap(err, "can't DockerComposeUp because can't process.Start() docker-compose process"))
return
}
go scanAllOutput(stdoutPipe, logLinesChannel)
go scanAllOutput(stderrPipe, logLinesChannel)
err = process.Wait()
_, isExitError := err.(*exec.ExitError)
if err != nil && !isExitError {
abort(errors.Wrap(err, "can't DockerComposeUp, error occurred while waiting for docker-compose to finish"))
return
}
if process.ProcessState.ExitCode() != 0 {
dockerComposeSuccess = false
break
}
}
dockerComposeIsRunning = false
return nil, nil, nil
outputChannel <- TerraformApplyResult{
Error: nil,
Complete: true,
Success: dockerComposeSuccess,
Log: strings.Join(logLines, "\n"),
Status: simpleStatus,
}
close(outputChannel)
})()
return svgBytes, outputChannel, nil
}
func makeSVGFromDockerCompose(
moduleDependencies map[string][]string,
func dockerComposePlan(
configs map[string]*composetypes.Config,
) ([]byte, error) {
configDetails map[string]*composetypes.ConfigDetails,
connections []SimplifiedTerraformConnection,
) (*SimplifiedTerraformStatus, error) {
moduleDots := []string{}
for moduleName := range configs {
serviceDots := []string{}
config := configs[moduleName]
tasks := []func() taskResult{
func() taskResult {
for _, service := range config.Services {
id := fmt.Sprintf(`%s_%s`, moduleName, service.Name)
id = strings.ReplaceAll(strings.ReplaceAll(id, ".", "_"), "-", "_")
serviceDot := fmt.Sprintf(`
"%s" [label = "%s", tooltip="%s", margin = 0.1, shape = "box3d]`,
id, padStringForDot(service.Name), id,
)
serviceDots = append(serviceDots, serviceDot)
cli, err := dockerclient.NewClientWithOpts(dockerclient.FromEnv)
if err != nil {
return taskResult{
Name: "list_containers_task",
Err: err,
}
}
containers, err := cli.ContainerList(context.Background(), dockertypes.ContainerListOptions{All: true})
return taskResult{
Name: "list_containers_task",
Result: containers,
Err: err,
}
},
}
for moduleName, details := range configDetails {
taskName := fmt.Sprintf("%s_get_compose_config_hash", moduleName)
tasks = append(tasks, func() taskResult {
exitCode, stdout, stderr, err := shellExec(details.WorkingDir, "docker-compose", "config", "--hash=*")
err = errorFromShellExecResult("docker-compose config --hash=*", exitCode, stdout, stderr, err)
if err != nil {
return taskResult{
Name: taskName,
Err: err,
}
}
// $ sudo docker-compose config --hash=*
// threshold 5f6deb1fa7dfc76fa0dc04e96d22cb4a2cc05a02060a4a3e77d38b87ff1d7f82
// caddy 95b3516b900089a3d04376b2635bfbde5596ec3769c50f12ce53b65f2d2e62a0
lines := strings.Split(string(stdout), "\n")
configHashByServiceName := map[string]string{}
for _, line := range lines {
elements := strings.Split(line, " ")
if len(elements) == 2 {
configHashByServiceName[elements[0]] = elements[1]
}
}
return taskResult{
Name: taskName,
Result: configHashByServiceName,
}
})
}
results := doInParallel(tasks...)
for _, result := range results {
if result.Err != nil {
return nil, errors.Wrapf(result.Err, "can't dockerComposePlan because %s failed", result.Name)
}
i := 0
for _, service := range config.Services {
if i != 0 {
id0 := fmt.Sprintf(`%s_%s`, moduleName, config.Services[i-1].Name)
id0 = strings.ReplaceAll(strings.ReplaceAll(id0, ".", "_"), "-", "_")
id1 := fmt.Sprintf(`%s_%s`, moduleName, service.Name)
id1 = strings.ReplaceAll(strings.ReplaceAll(id1, ".", "_"), "-", "_")
serviceDot := fmt.Sprintf(`
"%s" -> "%s" [style=invis]`, id0, id1,
)
serviceDots = append(serviceDots, serviceDot)
}
i++
}
moduleDot := fmt.Sprintf(`
subgraph cluster_%s {
bgcolor = lightgrey;
tooltip = "%s";
label = "%s";
%s
}`,
strings.ReplaceAll(strings.ReplaceAll(moduleName, ".", "_"), "-", "_"),
moduleName,
padStringForDot(moduleName),
strings.Join(serviceDots, ""),
)
moduleDots = append(moduleDots, moduleDot)
}
dependencyDots := []string{}
for to, fromList := range moduleDependencies {
for _, from := range fromList {
fromService := configs[from].Services[len(configs[from].Services)-1].Name
toService := configs[to].Services[0].Name
containers := results["list_containers_task"].Result.([]dockertypes.Container)
containersByModuleService := map[string]dockertypes.Container{}
planModules := map[string]*SimplifiedTerraformModule{}
dependencyDot := fmt.Sprintf(`
"%s" -> "%s" [ ltail="cluster_%s", lhead="cluster_%s" ];`,
fromService,
toService,
strings.ReplaceAll(strings.ReplaceAll(from, ".", "_"), "-", "_"),
strings.ReplaceAll(strings.ReplaceAll(to, ".", "_"), "-", "_"),
// First we go through all the containers returned by `docker ps -a` api command.
// we fill out the containersByModuleService map, and
// mark to be deleted any docker-compose containers that aren't in the current docker-compose configs.
for _, container := range containers {
containersModuleName := container.Labels["com.docker.compose.project"]
containersServiceName := container.Labels["com.docker.compose.service"]
if containersModuleName == "" {
continue
}
containersConfig, has := configs[containersModuleName]
foundInConfig := false
if has {
if containersServiceName != "" {
containersByModuleService[fmt.Sprintf("%s_%s", containersModuleName, containersServiceName)] = container
}
var containersService composetypes.ServiceConfig
for _, service := range containersConfig.Services {
if service.Name == containersServiceName {
containersService = service
}
}
if containersService.Name != "" {
foundInConfig = true
}
}
if !foundInConfig {
if _, has := planModules[containersModuleName]; !has {
planModules[containersModuleName] = &SimplifiedTerraformModule{
DisplayName: containersModuleName,
Resources: []*SimplifiedTerraformResource{},
}
}
planModules[containersModuleName].Resources = append(
planModules[containersModuleName].Resources,
&SimplifiedTerraformResource{
DisplayName: containersServiceName,
Plan: "delete",
},
)
}
}
dependencyDots = append(dependencyDots, dependencyDot)
// Now we go through the current docker-compose configs and mark any services
// that have no containers as to be created,
// plus mark any that have differing configHash as to be updated
for moduleName, config := range configs {
if _, has := planModules[moduleName]; !has {
planModules[moduleName] = &SimplifiedTerraformModule{
DisplayName: moduleName,
Resources: []*SimplifiedTerraformResource{},
}
}
for _, service := range config.Services {
container := containersByModuleService[fmt.Sprintf("%s_%s", moduleName, service.Name)]
plan := "none"
if container.ID != "" {
existingConfigHash := container.Labels["com.docker.compose.config-hash"]
getConfigHashTaskName := fmt.Sprintf("%s_get_compose_config_hash", moduleName)
newConfigHash := results[getConfigHashTaskName].Result.(map[string]string)[service.Name]
if existingConfigHash != newConfigHash {
plan = "recreate"
}
} else {
plan = "create"
}
planModules[moduleName].Resources = append(
planModules[moduleName].Resources,
&SimplifiedTerraformResource{
DisplayName: service.Name,
Plan: plan,
},
)
}
}
dot := fmt.Sprintf(`
digraph {
compound = "true"
newrank = "true"
ranksep = 0.1;
return &SimplifiedTerraformStatus{
Modules: planModules,
Variables: map[string]string{},
Connections: connections,
}, nil
%s
%s
}`,
strings.Join(moduleDots, ""),
strings.Join(dependencyDots, ""),
)
}
exitCode, dotStdout, dotStderr, err := shellExecInputPipe(".", &dot, "dot", "-Tsvg")
err = errorFromShellExecResult("dot -Tsvg", exitCode, dotStdout, dotStderr, err)
if err != nil {
return []byte{}, err
func doInParallel(actions ...func() taskResult) map[string]taskResult {
resultsChannel := make(chan taskResult)
results := map[string]taskResult{}
for _, action := range actions {
go (func() {
result := action()
resultsChannel <- result
if result.Err != nil {
close(resultsChannel)
}
})()
}
// svgString := string(dotStdout)
for range actions {
select {
case result := <-resultsChannel:
results[result.Name] = result
log.Printf("task '%s' completed", result.Name)
default:
// nothing to do.
}
}
return results
}
return dotStdout, nil
type taskResult struct {
Name string
Err error
Result interface{}
}
// func makeSVGFromDockerCompose(
// moduleDependencies map[string][]string,
// configs map[string]*composetypes.Config,
// ) ([]byte, error) {
// moduleDots := []string{}
// for moduleName := range configs {
// serviceDots := []string{}
// config := configs[moduleName]
// for _, service := range config.Services {
// id := fmt.Sprintf(`service_%s_%s`, moduleName, service.Name)
// id = strings.ReplaceAll(strings.ReplaceAll(id, ".", "_"), "-", "_")
// serviceDot := fmt.Sprintf(`
// "%s" [label = "%s", tooltip="%s", margin = 0.1, shape = "box3d]`,
// id, padStringForDot(service.Name), id,
// )
// serviceDots = append(serviceDots, serviceDot)
// }
// i := 0
// for _, service := range config.Services {
// if i != 0 {
// id0 := fmt.Sprintf(`service_%s_%s`, moduleName, config.Services[i-1].Name)
// id0 = strings.ReplaceAll(strings.ReplaceAll(id0, ".", "_"), "-", "_")
// id1 := fmt.Sprintf(`service_%s_%s`, moduleName, service.Name)
// id1 = strings.ReplaceAll(strings.ReplaceAll(id1, ".", "_"), "-", "_")
// serviceDot := fmt.Sprintf(`
// "%s" -> "%s" [style=invis]`, id0, id1,
// )
// serviceDots = append(serviceDots, serviceDot)
// }
// i++
// }
// moduleDot := fmt.Sprintf(`
// subgraph cluster_%s {
// bgcolor = lightgrey;
// tooltip = "%s";
// label = "%s";
// %s
// }`,
// strings.ReplaceAll(strings.ReplaceAll(moduleName, ".", "_"), "-", "_"),
// moduleName,
// padStringForDot(moduleName),
// strings.Join(serviceDots, ""),
// )
// moduleDots = append(moduleDots, moduleDot)
// }
// dependencyDots := []string{}
// for to, fromList := range moduleDependencies {
// for _, from := range fromList {
// fromServiceName := configs[from].Services[len(configs[from].Services)-1].Name
// toServiceName := configs[to].Services[0].Name
// fromService := fmt.Sprintf(`service_%s_%s`, from, fromServiceName)
// toService := fmt.Sprintf(`service_%s_%s`, to, toServiceName)
// dependencyDot := fmt.Sprintf(`
// "%s" -> "%s" [ ltail="cluster_%s", lhead="cluster_%s" ];`,
// strings.ReplaceAll(strings.ReplaceAll(fromService, ".", "_"), "-", "_"),
// strings.ReplaceAll(strings.ReplaceAll(toService, ".", "_"), "-", "_"),
// strings.ReplaceAll(strings.ReplaceAll(from, ".", "_"), "-", "_"),
// strings.ReplaceAll(strings.ReplaceAll(to, ".", "_"), "-", "_"),
// )
// dependencyDots = append(dependencyDots, dependencyDot)
// }
// }
// dot := fmt.Sprintf(`
// digraph {
// compound = "true"
// newrank = "true"
// ranksep = 0.1;
// %s
// %s
// }`,
// strings.Join(moduleDots, ""),
// strings.Join(dependencyDots, ""),
// )
// exitCode, dotStdout, dotStderr, err := shellExecInputPipe(".", &dot, "dot", "-Tsvg")
// err = errorFromShellExecResult("dot -Tsvg", exitCode, dotStdout, dotStderr, err)
// if err != nil {
// return []byte{}, err
// }
// svgBytes, err := modifyDotSVGAsXML(dotStdout, func(svgDoc *XMLNode) error {
// // correctly set the dot property for services
// svgDoc.WithQuerySelector(
// // 'g.node > title'
// []XMLQuery{XMLQuery{NodeType: "g", Class: "node"}, XMLQuery{NodeType: "title", DirectChild: true}},
// func(node *XMLNode) {
// if node.Parent != nil {
// title := html.UnescapeString(string(node.Content))
// node.Parent.SetAttr("data-dot", title)
// if strings.HasPrefix(title, "service_") {
// node.Parent.SetAttr("class", "resource")
// }
// }
// },
// )
// // correctly set the dot property for the "cluster"s (aka modules)
// moduleNames := []string{}
// svgDoc.WithQuerySelector(
// // 'a[xlink:title]'
// []XMLQuery{XMLQuery{NodeType: "a", Attr: "xlink:title"}},
// func(node *XMLNode) {
// if node.Parent != nil && node.Parent.Parent != nil {
// title := html.UnescapeString(node.GetAttr("xlink:title"))
// title = regexp.MustCompile(`[.-]`).ReplaceAllString(title, "_")
// if node.Parent.Parent.GetAttr("data-dot") == "" {
// moduleNames = append(moduleNames, title)
// node.Parent.Parent.SetAttr("data-dot", title)
// node.Parent.Parent.SetAttr("class", "module")
// }
// }
// },
// )
// // correctly set the dot property for edges
// // in order to get dot to render it correctly, the connections are actually between resources,
// // not between modules. So we have to trim the resource name off of the dot attribute
// // on the connections to make them match what we have in the status JSON object.
// svgDoc.WithQuerySelector(
// // 'g.edge title'
// []XMLQuery{XMLQuery{NodeType: "g", Class: "edge"}, XMLQuery{NodeType: "title"}},
// func(node *XMLNode) {
// if node.Parent != nil {
// title := html.UnescapeString(string(node.Content))
// fromTo := strings.Split(title, "->")
// if len(fromTo) != 2 {
// fmt.Printf("malformed dot string on svg edge: fromTo.length != 2: %s", title)
// }
// // if this edge starts or ends at a resource inside a module, override the dot string
// // to make it start or end at that module instead.
// for _, moduleName := range moduleNames {
// if strings.HasPrefix(strings.TrimPrefix(fromTo[0], "service_"), moduleName) {
// fromTo[0] = moduleName
// }
// if strings.HasPrefix(strings.TrimPrefix(fromTo[1], "service_"), moduleName) {
// fromTo[1] = moduleName
// }
// }
// node.Parent.SetAttr("data-dot", html.EscapeString(fmt.Sprintf("%s->%s", fromTo[0], fromTo[1])))
// }
// },
// )
// // two last things to do:
// // 1. center each resource horizontally within the bounding box of its parent module
// // because dot is dumb and it just kinda throws them wherever
// // 2. create little create/delete/modify icons next to each changed resource
// // similar to how terraform displays -, +, +/-, or ~ on each line-item in the plan
// for moduleName, config := range configs {
// moduleId := regexp.MustCompile(`[.-]`).ReplaceAllString(moduleName, "_")
// svgDoc.WithQuerySelector(
// []XMLQuery{XMLQuery{Attr: "data-dot", AttrValue: moduleId}},
// func(node *XMLNode) {
// moduleRect := node.GetBoundingBox()
// for _, service := range config.Services {
// resourceId := fmt.Sprintf("%s_%s", moduleId, regexp.MustCompile(`[.-]`).ReplaceAllString(service.Name, "_"))
// svgDoc.WithQuerySelector(
// // '[data-dot=module_dns_gandi_gandi_livedns_record_dns_entries]' for example
// []XMLQuery{XMLQuery{Attr: "data-dot", AttrValue: resourceId}},
// func(node *XMLNode) {
// // TODO pass the plan here
// centerResourceAndAddPlannedAction(moduleRect, node, resourceId, "create")
// },
// )
// }
// },
// )
// }
// return nil
// })
// return svgBytes, err
// }

30
automation/terraformActions.go

@ -295,21 +295,8 @@ func TerraformPlanAndApply(
return []byte{}, nil, errors.Wrap(err, "can't TerraformPlanAndApply because can't process.Start() terraform apply process")
}
scanAllOutput := func(reader io.ReadCloser) {
defer func() {
if err := recover(); err != nil {
log.Println("scanAllOutput panic:")
log.Println(err)
}
}()
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
logLinesChannel <- scanner.Text()
}
}
go scanAllOutput(stdoutPipe)
go scanAllOutput(stderrPipe)
go scanAllOutput(stdoutPipe, logLinesChannel)
go scanAllOutput(stderrPipe, logLinesChannel)
toReturn := make(chan TerraformApplyResult)
@ -329,6 +316,19 @@ func TerraformPlanAndApply(
return svg, toReturn, nil
}
func scanAllOutput(reader io.ReadCloser, logLinesChannel chan string) {
defer func() {
if err := recover(); err != nil {
log.Println("scanAllOutput panic:")
log.Println(err)
}
}()
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
logLinesChannel <- scanner.Text()
}
}
func monitorTerraformApplyProgress(
workingDirectory string,
terraformProject string,

459
automation/terraformSvg.go

@ -265,29 +265,146 @@ func makeSVGFromSimpleStatus(simpleStatus *SimplifiedTerraformStatus) ([]byte, e
return []byte{}, err
}
svgString := string(dotStdout)
// the svg output from dot has html comments that contain metadata preceeding each entity.
// for example:
// <!-- param_ssh_private_key_filepath -->
// <g id="node16" class="node">
// We will attach that metadata to the svg elements as a data property
// Note that this will not work perfectly for the "cluster"s (aka modules)
// they are handled separately after the SVG has been inserted.
// Yes parsing XML with regex is "evil" but keep in mind, this XML was generated by a program, not written by hand.
// So it is a lot more predictable. Also, I couldn't figure out how to parse the comments with the go std lib XML parser.
commentRegex := regexp.MustCompile(`<!--\s*(.+)\s*-->\s*\n\s*(<[a-z]+\s)`)
svgString = commentRegex.ReplaceAllStringFunc(svgString, func(matched string) string {
matches := commentRegex.FindStringSubmatch(matched)
return fmt.Sprintf("%s dot=\"%s\" ", matches[2], strings.Trim(html.UnescapeString(matches[1]), " "))
svgBytes, err := modifyDotSVGAsXML(dotStdout, func(svgDoc *XMLNode) error {
// correctly set the dot property for resources, variables, and parameters
svgDoc.WithQuerySelector(
// 'g.node > title'
[]XMLQuery{XMLQuery{NodeType: "g", Class: "node"}, XMLQuery{NodeType: "title", DirectChild: true}},
func(node *XMLNode) {
if node.Parent != nil {
title := html.UnescapeString(string(node.Content))
node.Parent.SetAttr("data-dot", title)
if strings.HasPrefix(title, "var_") || strings.HasPrefix(title, "data_") {
node.Parent.SetAttr("class", "variable")
}
if strings.HasPrefix(title, "param_") {
node.Parent.SetAttr("class", "parameter")
// the parameters start out as boxes
//but lets squeeze them into keystone shape to differentiate them a bit
node.Parent.WithQuerySelector([]XMLQuery{XMLQuery{NodeType: "polygon"}}, func(polygonNode *XMLNode) {
vertices, err := polygonNode.GetSVGPolygonVertices()
if err == nil {
center := []float64{0, 0}
for _, vertex := range vertices {
center[0] += vertex[0]
center[1] += vertex[1]
}
center[0] = center[0] / float64(len(vertices))
center[1] = center[1] / float64(len(vertices))
newVertices := [][]float64{}
for _, vertex := range vertices {
x := vertex[0]
y := vertex[1]
if y > center[1] {
if x > center[0] {
x -= 10
} else {
x += 10
}
}
newVertices = append(newVertices, []float64{x, y})
}
polygonNode.SetSVGPolygonVertices(newVertices)
}
})
}
if strings.HasPrefix(title, "module_") {
node.Parent.SetAttr("class", "resource")
}
}
},
)
// correctly set the dot property for the "cluster"s (aka modules)
moduleNames := []string{}
svgDoc.WithQuerySelector(
// 'a[xlink:title]'
[]XMLQuery{XMLQuery{NodeType: "a", Attr: "xlink:title"}},
func(node *XMLNode) {
if node.Parent != nil && node.Parent.Parent != nil {
title := html.UnescapeString(node.GetAttr("xlink:title"))
title = regexp.MustCompile(`[.-]`).ReplaceAllString(title, "_")
if node.Parent.Parent.GetAttr("data-dot") == "" {
moduleNames = append(moduleNames, title)
node.Parent.Parent.SetAttr("data-dot", title)
node.Parent.Parent.SetAttr("class", "module")
}
}
},
)
// correctly set the dot property for edges
// in order to get dot to render it correctly, the connections are actually between resources,
// not between modules. So we have to trim the resource name off of the dot attribute
// on the connections to make them match what we have in the status JSON object.
svgDoc.WithQuerySelector(
// 'g.edge title'
[]XMLQuery{XMLQuery{NodeType: "g", Class: "edge"}, XMLQuery{NodeType: "title"}},
func(node *XMLNode) {
if node.Parent != nil {
title := html.UnescapeString(string(node.Content))
fromTo := strings.Split(title, "->")
if len(fromTo) != 2 {
fmt.Printf("malformed dot string on svg edge: fromTo.length != 2: %s", title)
}
// if this edge starts or ends at a resource inside a module, override the dot string
// to make it start or end at that module instead.
for _, moduleName := range moduleNames {
if strings.HasPrefix(fromTo[0], moduleName) {
node.Parent.SetAttr("data-dot", html.EscapeString(fmt.Sprintf("%s->%s", moduleName, fromTo[1])))
}
if strings.HasPrefix(fromTo[1], moduleName) {
node.Parent.SetAttr("data-dot", html.EscapeString(fmt.Sprintf("%s->%s", fromTo[0], moduleName)))
}
}
}
},
)
// two last things to do:
// 1. center each resource horizontally within the bounding box of its parent module
// because dot is dumb and it just kinda throws them wherever
// 2. create little create/delete/modify icons next to each changed resource
// similar to how terraform displays -, +, +/-, or ~ on each line-item in the plan
for moduleName, module := range simpleStatus.Modules {
moduleId := regexp.MustCompile(`[.-]`).ReplaceAllString(moduleName, "_")
svgDoc.WithQuerySelector(
[]XMLQuery{XMLQuery{Attr: "data-dot", AttrValue: moduleId}},
func(node *XMLNode) {
moduleRect := node.GetBoundingBox()
for _, resource := range module.Resources {
resourceId := fmt.Sprintf("%s_%s", moduleId, regexp.MustCompile(`[.-]`).ReplaceAllString(resource.DisplayName, "_"))
svgDoc.WithQuerySelector(
// '[data-dot=module_dns_gandi_gandi_livedns_record_dns_entries]' for example
[]XMLQuery{XMLQuery{Attr: "data-dot", AttrValue: resourceId}},
func(node *XMLNode) {
centerResourceAndAddPlannedAction(moduleRect, node, resourceId, resource.Plan)
},
)
}
},
)
}
return nil
})
//ioutil.WriteFile("./test-gen.svg", []byte(svgString), 0777)
return svgBytes, err
}
func modifyDotSVGAsXML(svgInputBytes []byte, action func(*XMLNode) error) ([]byte, error) {
var svgDoc XMLNode
err = xml.Unmarshal(dotStdout, &svgDoc)
err := xml.Unmarshal(svgInputBytes, &svgDoc)
if err != nil {
panic(err)
return nil, err
}
var setParentRecurse func(*XMLNode)
@ -308,106 +425,6 @@ func makeSVGFromSimpleStatus(simpleStatus *SimplifiedTerraformStatus) ([]byte, e
},
)
// correctly set the dot property for resources, variables, and parameters
svgDoc.WithQuerySelector(
// 'g.node > title'
[]XMLQuery{XMLQuery{NodeType: "g", Class: "node"}, XMLQuery{NodeType: "title", DirectChild: true}},
func(node *XMLNode) {
if node.Parent != nil {
title := html.UnescapeString(string(node.Content))
node.Parent.SetAttr("data-dot", title)
if strings.HasPrefix(title, "var_") || strings.HasPrefix(title, "data_") {
node.Parent.SetAttr("class", "variable")
}
if strings.HasPrefix(title, "param_") {
node.Parent.SetAttr("class", "parameter")
// the parameters start out as boxes
//but lets squeeze them into keystone shape to differentiate them a bit
node.Parent.WithQuerySelector([]XMLQuery{XMLQuery{NodeType: "polygon"}}, func(polygonNode *XMLNode) {
vertices, err := polygonNode.GetSVGPolygonVertices()
if err == nil {
center := []float64{0, 0}
for _, vertex := range vertices {
center[0] += vertex[0]
center[1] += vertex[1]
}
center[0] = center[0] / float64(len(vertices))
center[1] = center[1] / float64(len(vertices))
newVertices := [][]float64{}
for _, vertex := range vertices {
x := vertex[0]
y := vertex[1]
if y > center[1] {
if x > center[0] {
x -= 10
} else {
x += 10
}
}
newVertices = append(newVertices, []float64{x, y})
}
polygonNode.SetSVGPolygonVertices(newVertices)
}
})
}
if strings.HasPrefix(title, "module_") {
node.Parent.SetAttr("class", "resource")
}
}
},
)
// correctly set the dot property for the "cluster"s (aka modules)
moduleNames := []string{}
svgDoc.WithQuerySelector(
// 'a[xlink:title]'
[]XMLQuery{XMLQuery{NodeType: "a", Attr: "xlink:title"}},
func(node *XMLNode) {
if node.Parent != nil && node.Parent.Parent != nil {
title := html.UnescapeString(node.GetAttr("xlink:title"))
title = regexp.MustCompile(`[.-]`).ReplaceAllString(title, "_")
if node.Parent.Parent.GetAttr("data-dot") == "" {
moduleNames = append(moduleNames, title)
node.Parent.Parent.SetAttr("data-dot", title)
node.Parent.Parent.SetAttr("class", "module")
}
}
},
)
// correctly set the dot property for edges
// in order to get dot to render it correctly, the connections are actually between resources,
// not between modules. So we have to trim the resource name off of the dot attribute
// on the connections to make them match what we have in the status JSON object.
svgDoc.WithQuerySelector(
// 'g.edge title'
[]XMLQuery{XMLQuery{NodeType: "g", Class: "edge"}, XMLQuery{NodeType: "title"}},
func(node *XMLNode) {
if node.Parent != nil {
title := html.UnescapeString(string(node.Content))
fromTo := strings.Split(title, "->")
if len(fromTo) != 2 {
fmt.Printf("malformed dot string on svg edge: fromTo.length != 2: %s", title)
}
// if this edge starts or ends at a resource inside a module, override the dot string
// to make it start or end at that module instead.
for _, moduleName := range moduleNames {
if strings.HasPrefix(fromTo[0], moduleName) {
node.Parent.SetAttr("data-dot", html.EscapeString(fmt.Sprintf("%s->%s", moduleName, fromTo[1])))
}
if strings.HasPrefix(fromTo[1], moduleName) {
node.Parent.SetAttr("data-dot", html.EscapeString(fmt.Sprintf("%s->%s", fromTo[0], moduleName)))
}
}
}
},
)
svgDoc.WithQuerySelector(
// '[font-family]'
[]XMLQuery{XMLQuery{Attr: "font-family"}},
@ -416,120 +433,9 @@ func makeSVGFromSimpleStatus(simpleStatus *SimplifiedTerraformStatus) ([]byte, e
},
)
// two last things to do:
// 1. center each resource horizontally within the bounding box of its parent module
// because dot is dumb and it just kinda throws them wherever
// 2. create little create/delete/modify icons next to each changed resource
// similar to how terraform displays -, +, +/-, or ~ on each line-item in the plan
for moduleName, module := range simpleStatus.Modules {
moduleId := regexp.MustCompile(`[.-]`).ReplaceAllString(moduleName, "_")
svgDoc.WithQuerySelector(
[]XMLQuery{XMLQuery{Attr: "data-dot", AttrValue: moduleId}},
func(node *XMLNode) {
moduleRect := node.GetBoundingBox()
for _, resource := range module.Resources {
resourceId := fmt.Sprintf("%s_%s", moduleId, regexp.MustCompile(`[.-]`).ReplaceAllString(resource.DisplayName, "_"))
svgDoc.WithQuerySelector(
// '[data-dot=module_dns_gandi_gandi_livedns_record_dns_entries]' for example
[]XMLQuery{XMLQuery{Attr: "data-dot", AttrValue: resourceId}},
func(node *XMLNode) {
resourceRect := node.GetBoundingBox()
// Step 1 center the resources
nudgeX := ((moduleRect.Left + moduleRect.Right) - (resourceRect.Left + resourceRect.Right)) * float64(0.5)
node.SetAttr("transform", fmt.Sprintf("scale(1 1) rotate(0) translate(%d %d)", int(nudgeX), 0))
// Step 2 add the terraform action display if there is a planned action
if resource.Plan != "none" && resource.Plan != "" {
actionX := resourceRect.Left + nudgeX + float64(terraformActionOffsetX)
actionY := resourceRect.Top + float64(terraformActionOffsetY)
// start with a circle element
actionContents := []*XMLNode{
&XMLNode{
XMLName: xml.Name{Local: "circle", Space: "http://www.w3.org/2000/svg"},
Attrs: []*XMLAttr{
&XMLAttr{
Local: "cx",
Value: fmt.Sprintf("%.4f", actionX),
},
&XMLAttr{
Local: "fill",
Value: "#ffffff",
},
&XMLAttr{
Local: "cy",
Value: fmt.Sprintf("%.4f", actionY),
},
&XMLAttr{
Local: "r",
Value: strconv.Itoa(terraformActionCircleRadius),
},
},
},
}
// macro to add text to the inside of the circle
createText := func(str, class string, offset int) {
actionContents = append(actionContents, &XMLNode{
XMLName: xml.Name{Local: "text", Space: "http://www.w3.org/2000/svg"},
Attrs: []*XMLAttr{
&XMLAttr{Local: "text-anchor", Value: "middle"},
&XMLAttr{Local: "font-family", Value: terraformSvgFontFamily},
&XMLAttr{Local: "text-size", Value: strconv.Itoa(terraformActionFontSize)},
&XMLAttr{Local: "class", Value: class},
&XMLAttr{Local: "x", Value: fmt.Sprintf("%.4f", actionX+float64(offset*terraformActionTextSpacingX))},
&XMLAttr{Local: "y", Value: fmt.Sprintf("%.4f", actionY+float64(terraformActionTextOffsetY))},
},
Content: []byte(str),
})
}
// add text to the circle depending on the planned action
if resource.Plan == "recreate" {
createText("-", terraformActionRedClass, -1)
createText("/", "", 0)
createText("+", terraformActionGreenClass, 1)
}
if resource.Plan == "update" {
createText("~", terraformActionYellowClass, 0)
}
if resource.Plan == "create" {
createText("+", terraformActionGreenClass, 0)
}
if resource.Plan == "delete" {
createText("-", terraformActionRedClass, 0)
}
// Finally, add all the action content inside a single <g> tag & append to the svg
node.Parent.Children = append(node.Parent.Children, &XMLNode{
XMLName: xml.Name{Local: "g", Space: "http://www.w3.org/2000/svg"},
Attrs: []*XMLAttr{
&XMLAttr{
Local: "class",
Value: "action-circle",
},
&XMLAttr{
Local: "data-dot",
Value: fmt.Sprintf("%s_action_circle", resourceId),
},
},
Children: actionContents,
})
}
},
)
}
},
)
err = action(&svgDoc)
if err != nil {
return nil, err
}
var cleanUpSvgRecurse func(*XMLNode)
@ -574,6 +480,95 @@ func makeSVGFromSimpleStatus(simpleStatus *SimplifiedTerraformStatus) ([]byte, e
return svgBytes, nil
}
func centerResourceAndAddPlannedAction(parent *Rect2D, node *XMLNode, nodeId string, action string) {
resourceRect := node.GetBoundingBox()
// Step 1 center the resources
nudgeX := ((parent.Left + parent.Right) - (resourceRect.Left + resourceRect.Right)) * float64(0.5)
node.SetAttr("transform", fmt.Sprintf("scale(1 1) rotate(0) translate(%d %d)", int(nudgeX), 0))
// Step 2 add the terraform action display if there is a planned action
if action != "none" && action != "" {
actionX := resourceRect.Left + nudgeX + float64(terraformActionOffsetX)
actionY := resourceRect.Top + float64(terraformActionOffsetY)
// start with a circle element
actionContents := []*XMLNode{
&XMLNode{
XMLName: xml.Name{Local: "circle", Space: "http://www.w3.org/2000/svg"},
Attrs: []*XMLAttr{
&XMLAttr{
Local: "cx",
Value: fmt.Sprintf("%.4f", actionX),
},
&XMLAttr{
Local: "fill",
Value: "#ffffff",
},
&XMLAttr{
Local: "cy",
Value: fmt.Sprintf("%.4f", actionY),
},
&XMLAttr{
Local: "r",
Value: strconv.Itoa(terraformActionCircleRadius),
},
},
},
}
// macro to add text to the inside of the circle
createText := func(str, class string, offset int) {
actionContents = append(actionContents, &XMLNode{
XMLName: xml.Name{Local: "text", Space: "http://www.w3.org/2000/svg"},
Attrs: []*XMLAttr{
&XMLAttr{Local: "text-anchor", Value: "middle"},
&XMLAttr{Local: "font-family", Value: terraformSvgFontFamily},
&XMLAttr{Local: "text-size", Value: strconv.Itoa(terraformActionFontSize)},
&XMLAttr{Local: "class", Value: class},
&XMLAttr{Local: "x", Value: fmt.Sprintf("%.4f", actionX+float64(offset*terraformActionTextSpacingX))},
&XMLAttr{Local: "y", Value: fmt.Sprintf("%.4f", actionY+float64(terraformActionTextOffsetY))},
},
Content: []byte(str),
})
}
// add text to the circle depending on the planned action
if action == "recreate" {
createText("-", terraformActionRedClass, -1)
createText("/", "", 0)
createText("+", terraformActionGreenClass, 1)
}
if action == "update" {
createText("~", terraformActionYellowClass, 0)
}
if action == "create" {
createText("+", terraformActionGreenClass, 0)
}
if action == "delete" {
createText("-", terraformActionRedClass, 0)
}
// Finally, add all the action content inside a single <g> tag & append to the svg
node.Parent.Children = append(node.Parent.Children, &XMLNode{
XMLName: xml.Name{Local: "g", Space: "http://www.w3.org/2000/svg"},
Attrs: []*XMLAttr{
&XMLAttr{
Local: "class",
Value: "action-circle",
},
&XMLAttr{
Local: "data-dot",
Value: fmt.Sprintf("%s_action_circle", nodeId),
},
},
Children: actionContents,
})
}
}
func padStringForDot(str string) string {
pad := strings.Repeat(" ", int(float32(len(str)+dotStringCharactersMinPadding)*dotStringLinearPadding))
return fmt.Sprintf("%s%s%s", pad, str, pad)

70
main.go

@ -20,12 +20,6 @@ type applicationState struct {
storage objectStorage.ObjectStorager
}
type taskResult struct {
Name string
Err error
Result interface{}
}
var global applicationState
func main() {
@ -43,8 +37,8 @@ func main() {
go terraformStateServer()
results := doStartupActionsInParallel(
func() taskResult {
results := automation.doInParallel(
func() automation.taskResult {
// This creates an access key that the gateway cloud instance can use to upload its SSH public key
// to our object storage. the host-key-poller will download this SSH host public key and add it to our known_hosts
// so that we can SSH to the gateway instance securely
@ -58,18 +52,18 @@ func main() {
}
knownHostsCredentials, err := global.storage.CreateAccessKeyIfNotExists(hostKeysAccessSpec)
if err != nil {
return taskResult{
return automation.taskResult{
Name: "knownHostsCredentials",
Err: errors.Wrap(err, "can't create object storage access key for known_hosts"),
}
}
return taskResult{
return automation.taskResult{
Name: "knownHostsCredentials",
Result: knownHostsCredentials,
}
},
func() taskResult {
func() automation.taskResult {
// BuildTLSCertsForThreshold fills in the CAs, Keys, and Certificates in the Threshold ansible roles.
// So when terraform invokes ansible to install threshold client/server, it will install working
// certificates and keys
@ -80,20 +74,20 @@ func main() {
global.storage,
)
if err != nil {
return taskResult{
return automation.taskResult{
Name: "buildTLSCertsForThreshold",
Err: errors.Wrap(err, "can't create certs for threshold"),
}
}
return taskResult{Name: "knownHostsCredentials"}
return automation.taskResult{Name: "knownHostsCredentials"}
},
func() taskResult {
func() automation.taskResult {
sshPort := 2201
hostSSHPortFilename := fmt.Sprintf("rootsystem/ssh/%s.txt", config.Host.Name)
file, notFound, err := global.storage.Get(hostSSHPortFilename)
if err != nil && !notFound {
return taskResult{
return automation.taskResult{
Name: "sshPort",
Err: errors.Wrapf(err, "can't download %s", hostSSHPortFilename),
}
@ -101,7 +95,7 @@ func main() {
if !notFound {
sshPort, err = strconv.Atoi(string(file.Content))
if err != nil {
return taskResult{
return automation.taskResult{
Name: "sshPort",
Err: errors.Wrapf(err, "can't read %s as a number", hostSSHPortFilename),
}
@ -109,7 +103,7 @@ func main() {
} else {
file, notFound, err := global.storage.Get("rootsystem/ssh/next-port.txt")
if err != nil && !notFound {
return taskResult{
return automation.taskResult{
Name: "sshPort",
Err: errors.Wrap(err, "can't download next-port.txt"),
}
@ -123,31 +117,31 @@ func main() {
}
err = global.storage.Put(hostSSHPortFilename, []byte(strconv.Itoa(sshPort)))
if err != nil {
return taskResult{
return automation.taskResult{
Name: "sshPort",
Err: errors.Wrapf(err, "can't can't upload %s", hostSSHPortFilename),
}
}
err = global.storage.Put("rootsystem/ssh/next-port.txt", []byte(strconv.Itoa(sshPort+1)))
if err != nil {
return taskResult{
return automation.taskResult{
Name: "sshPort",
Err: errors.Wrap(err, "can't can't upload next-port.txt"),
}
}
}
return taskResult{
return automation.taskResult{
Name: "sshPort",
Result: sshPort,
}
},
func() taskResult {
func() automation.taskResult {
// Add 1 to the build number, each time rootsystem runs is a different build.
file, notFound, err := global.storage.Get("rootsystem/terraform-logs/build-number.txt")
if err != nil && !notFound {
return taskResult{
return automation.taskResult{
Name: "buildNumber",
Err: errors.Wrap(err, "can't download build-number.txt"),
}
@ -163,12 +157,12 @@ func main() {
}
err = global.storage.Put("rootsystem/terraform-logs/build-number.txt", []byte(strconv.Itoa(buildNumber)))
if err != nil {
return taskResult{
return automation.taskResult{
Name: "buildNumber",
Err: errors.Wrap(err, "can't can't upload build-number.txt"),
}
}
return taskResult{
return automation.taskResult{
Name: "buildNumber",
Result: buildNumber,
}
@ -323,31 +317,3 @@ func terraformStateServer() error {
return server.ListenAndServe()
}
func doStartupActionsInParallel(actions ...func() taskResult) map[string]taskResult {
resultsChannel := make(chan taskResult)
results := map[string]taskResult{}
for _, action := range actions {
go (func() {
result := action()
resultsChannel <- result
if result.Err != nil {
close(resultsChannel)
}
})()
}
for range actions {
select {
case result := <-resultsChannel:
results[result.Name] = result
log.Printf("task '%s' completed", result.Name)
default:
// nothing to do.
}
}
return results
}

Loading…
Cancel
Save