docker-compose automation continued

automation/dockerCompose.go

@@ -1,16 +1,23 @@
 package automation
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"os"
+	"os/exec"
 	"path"
 	"strings"
+	"time"
 
 	"git.sequentialread.com/forest/rootsystem/configuration"
 	composeloader "github.com/docker/cli/cli/compose/loader"
 	composeschema "github.com/docker/cli/cli/compose/schema"
 	composetypes "github.com/docker/cli/cli/compose/types"
+
+	dockertypes "github.com/docker/docker/api/types"
+	dockerclient "github.com/docker/docker/client"
 	"github.com/pkg/errors"
 )
 
@@ -67,7 +74,7 @@ func DockerComposeUp(
 
 	configs := map[string]*composetypes.Config{}
 	moduleByNetwork := map[string]string{}
-	moduleDependencies := map[string][]string{}
+	connectionsMap := map[string]map[string]SimplifiedTerraformConnection{}
 
 	for _, moduleName := range config.ApplicationModules {
 		config, err := composeloader.Load(*(applicationModulesMap[moduleName]))
@@ -108,16 +115,19 @@ func DockerComposeUp(
 		for _, network := range config.Networks {
 			dependsOnModule := moduleByNetwork[network.External.Name]
 			if network.External.External && dependsOnModule != "" {
-				if moduleDependencies[moduleName] == nil {
-					moduleDependencies[moduleName] = []string{dependsOnModule}
-				} else {
-					moduleDependencies[moduleName] = append(moduleDependencies[moduleName], dependsOnModule)
+				if connectionsMap[moduleName] == nil {
+					connectionsMap[moduleName] = map[string]SimplifiedTerraformConnection{}
+				}
+				connectionsMap[moduleName][dependsOnModule] = SimplifiedTerraformConnection{
+					From:        dependsOnModule,
+					To:          moduleName,
+					DisplayName: network.External.Name,
 				}
 			}
 		}
 		//TODO external volumes as well..?
 		//TODO depends_on.. ?
-		//TODO dis-allow links?
+		//TODO dis-allow links? other validation?
 	}
 
 	sortedApplicationModulesMap := map[string]bool{}
@@ -129,9 +139,9 @@ func DockerComposeUp(
 		for _, moduleName := range config.ApplicationModules {
 			if !sortedApplicationModulesMap[moduleName] {
 				allDependenciesMet := true
-				dependencies := moduleDependencies[moduleName]
+				dependencies := connectionsMap[moduleName]
 				if dependencies != nil {
-					for _, dependsOnModule := range dependencies {
+					for dependsOnModule := range dependencies {
 						if !sortedApplicationModulesMap[dependsOnModule] {
 							allDependenciesMet = false
 						}
@@ -151,98 +161,488 @@ func DockerComposeUp(
 		)
 	}
 
-	svg, err := makeSVGFromDockerCompose(moduleDependencies, configs)
+	connections := []SimplifiedTerraformConnection{}
+	for _, dependsOn := range connectionsMap {
+		for _, connection := range dependsOn {
+			connections = append(connections, connection)
+		}
+	}
+
+	simpleStatus, err := dockerComposePlan(configs, applicationModulesMap, connections)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "can't create docker-compose plan")
+	}
+
+	svgBytes, err := makeSVGFromSimpleStatus(simpleStatus)
+	//svg, err := makeSVGFromDockerCompose(moduleDependencies, applicationModulesMap, configs)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "can't create svg diagram from docker-compose plan document")
+	}
+
+	outputChannel := make(chan TerraformApplyResult)
+
+	go (func() {
+		logLinesChannel := make(chan string)
+		logLines := []string{}
+		dockerComposeIsRunning := true
+		go (func() {
+			for logLine := range logLinesChannel {
+				logLines = append(logLines, logLine)
+			}
+		})()
+
+		abort := func(err error) {
+			dockerComposeIsRunning = false
+			outputChannel <- TerraformApplyResult{
+				Error:    err,
+				Complete: true,
+				Success:  false,
+				Log:      strings.Join(logLines, "\n"),
+				Status:   simpleStatus,
+			}
+			close(outputChannel)
+		}
+
+		go (func() {
+			for dockerComposeIsRunning {
+				time.Sleep(time.Second * configuration.TERRAFORM_APPLY_STATUS_UPDATE_INTERVAL_SECONDS)
+				if dockerComposeIsRunning {
+					outputChannel <- TerraformApplyResult{
+						Error:  nil,
+						Log:    strings.Join(logLines, "\n"),
+						Status: simpleStatus,
+					}
+				}
+			}
+		})()
+
+		dockerComposeSuccess := true
+
+		for _, moduleName := range sortedApplicationModules {
+			// TODO handle panics in here with abort
+			details := applicationModulesMap[moduleName]
+
+			process := exec.Command("docker-compose", "--no-ansi", "up", "-d")
+			logLinesChannel <- fmt.Sprintf("\n(%s) $ docker-compose --no-ansi up -d\n", details.WorkingDir)
+			process.Dir = details.WorkingDir
+
+			stdoutPipe, err := process.StdoutPipe()
+			if err != nil {
+				abort(errors.Wrap(err, "can't DockerComposeUp because can't process.StdoutPipe() docker-compose process"))
+				return
+			}
+			stderrPipe, err := process.StderrPipe()
+			if err != nil {
+				abort(errors.Wrap(err, "can't DockerComposeUp because can't process.StderrPipe() docker-compose process"))
+				return
+			}
+
+			err = process.Start()
+			if err != nil {
+				abort(errors.Wrap(err, "can't DockerComposeUp because can't process.Start() docker-compose process"))
+				return
+			}
+
+			go scanAllOutput(stdoutPipe, logLinesChannel)
+			go scanAllOutput(stderrPipe, logLinesChannel)
+
+			err = process.Wait()
+			_, isExitError := err.(*exec.ExitError)
+			if err != nil && !isExitError {
+				abort(errors.Wrap(err, "can't DockerComposeUp, error occurred while waiting for docker-compose to finish"))
+				return
+			}
+			if process.ProcessState.ExitCode() != 0 {
+				dockerComposeSuccess = false
+				break
+			}
+		}
+		dockerComposeIsRunning = false
 
-	return nil, nil, nil
+		outputChannel <- TerraformApplyResult{
+			Error:    nil,
+			Complete: true,
+			Success:  dockerComposeSuccess,
+			Log:      strings.Join(logLines, "\n"),
+			Status:   simpleStatus,
+		}
+
+		close(outputChannel)
+
+	})()
+
+	return svgBytes, outputChannel, nil
 }
 
-func makeSVGFromDockerCompose(
-	moduleDependencies map[string][]string,
+func dockerComposePlan(
 	configs map[string]*composetypes.Config,
-) ([]byte, error) {
+	configDetails map[string]*composetypes.ConfigDetails,
+	connections []SimplifiedTerraformConnection,
+) (*SimplifiedTerraformStatus, error) {
 
-	moduleDots := []string{}
-	for moduleName := range configs {
-		serviceDots := []string{}
-		config := configs[moduleName]
+	tasks := []func() taskResult{
+		func() taskResult {
 
-		for _, service := range config.Services {
-			id := fmt.Sprintf(`%s_%s`, moduleName, service.Name)
-			id = strings.ReplaceAll(strings.ReplaceAll(id, ".", "_"), "-", "_")
-			serviceDot := fmt.Sprintf(`
-	      "%s" [label = "%s", tooltip="%s", margin = 0.1, shape = "box3d]`,
-				id, padStringForDot(service.Name), id,
-			)
-			serviceDots = append(serviceDots, serviceDot)
+			cli, err := dockerclient.NewClientWithOpts(dockerclient.FromEnv)
+			if err != nil {
+				return taskResult{
+					Name: "list_containers_task",
+					Err:  err,
+				}
+			}
+			containers, err := cli.ContainerList(context.Background(), dockertypes.ContainerListOptions{All: true})
+
+			return taskResult{
+				Name:   "list_containers_task",
+				Result: containers,
+				Err:    err,
+			}
+		},
+	}
+
+	for moduleName, details := range configDetails {
+		taskName := fmt.Sprintf("%s_get_compose_config_hash", moduleName)
+		tasks = append(tasks, func() taskResult {
+			exitCode, stdout, stderr, err := shellExec(details.WorkingDir, "docker-compose", "config", "--hash=*")
+			err = errorFromShellExecResult("docker-compose config --hash=*", exitCode, stdout, stderr, err)
+			if err != nil {
+				return taskResult{
+					Name: taskName,
+					Err:  err,
+				}
+			}
+			// $ sudo docker-compose config --hash=*
+			// threshold 5f6deb1fa7dfc76fa0dc04e96d22cb4a2cc05a02060a4a3e77d38b87ff1d7f82
+			// caddy 95b3516b900089a3d04376b2635bfbde5596ec3769c50f12ce53b65f2d2e62a0
+			lines := strings.Split(string(stdout), "\n")
+			configHashByServiceName := map[string]string{}
+			for _, line := range lines {
+				elements := strings.Split(line, " ")
+				if len(elements) == 2 {
+					configHashByServiceName[elements[0]] = elements[1]
+				}
+			}
+			return taskResult{
+				Name:   taskName,
+				Result: configHashByServiceName,
+			}
+		})
+	}
+
+	results := doInParallel(tasks...)
+
+	for _, result := range results {
+		if result.Err != nil {
+			return nil, errors.Wrapf(result.Err, "can't dockerComposePlan because %s failed", result.Name)
 		}
-		i := 0
-		for _, service := range config.Services {
-			if i != 0 {
-				id0 := fmt.Sprintf(`%s_%s`, moduleName, config.Services[i-1].Name)
-				id0 = strings.ReplaceAll(strings.ReplaceAll(id0, ".", "_"), "-", "_")
-				id1 := fmt.Sprintf(`%s_%s`, moduleName, service.Name)
-				id1 = strings.ReplaceAll(strings.ReplaceAll(id1, ".", "_"), "-", "_")
-				serviceDot := fmt.Sprintf(`
-	        "%s" -> "%s" [style=invis]`, id0, id1,
-				)
-				serviceDots = append(serviceDots, serviceDot)
-			}
-			i++
-		}
-
-		moduleDot := fmt.Sprintf(`
-	    subgraph cluster_%s {
-	      bgcolor = lightgrey;
-	      tooltip = "%s";
-	      label = "%s";
-	      %s
-	    }`,
-			strings.ReplaceAll(strings.ReplaceAll(moduleName, ".", "_"), "-", "_"),
-			moduleName,
-			padStringForDot(moduleName),
-			strings.Join(serviceDots, ""),
-		)
-		moduleDots = append(moduleDots, moduleDot)
 	}
 
-	dependencyDots := []string{}
-	for to, fromList := range moduleDependencies {
-		for _, from := range fromList {
-			fromService := configs[from].Services[len(configs[from].Services)-1].Name
-			toService := configs[to].Services[0].Name
+	containers := results["list_containers_task"].Result.([]dockertypes.Container)
+	containersByModuleService := map[string]dockertypes.Container{}
+	planModules := map[string]*SimplifiedTerraformModule{}
 
-			dependencyDot := fmt.Sprintf(`
-				"%s" -> "%s" [ ltail="cluster_%s", lhead="cluster_%s" ];`,
-				fromService,
-				toService,
-				strings.ReplaceAll(strings.ReplaceAll(from, ".", "_"), "-", "_"),
-				strings.ReplaceAll(strings.ReplaceAll(to, ".", "_"), "-", "_"),
+	// First we go through all the containers returned by `docker ps -a` api command.
+	// we fill out the containersByModuleService map, and
+	// mark to be deleted any docker-compose containers that aren't in the current docker-compose configs.
+	for _, container := range containers {
+		containersModuleName := container.Labels["com.docker.compose.project"]
+		containersServiceName := container.Labels["com.docker.compose.service"]
+		if containersModuleName == "" {
+			continue
+		}
+		containersConfig, has := configs[containersModuleName]
+		foundInConfig := false
+		if has {
+			if containersServiceName != "" {
+				containersByModuleService[fmt.Sprintf("%s_%s", containersModuleName, containersServiceName)] = container
+			}
+			var containersService composetypes.ServiceConfig
+			for _, service := range containersConfig.Services {
+				if service.Name == containersServiceName {
+					containersService = service
+				}
+			}
+			if containersService.Name != "" {
+				foundInConfig = true
+			}
+		}
+		if !foundInConfig {
+			if _, has := planModules[containersModuleName]; !has {
+				planModules[containersModuleName] = &SimplifiedTerraformModule{
+					DisplayName: containersModuleName,
+					Resources:   []*SimplifiedTerraformResource{},
+				}
+			}
+			planModules[containersModuleName].Resources = append(
+				planModules[containersModuleName].Resources,
+				&SimplifiedTerraformResource{
+					DisplayName: containersServiceName,
+					Plan:        "delete",
+				},
 			)
+		}
+	}
 
-			dependencyDots = append(dependencyDots, dependencyDot)
+	// Now we go through the current docker-compose configs and mark any services
+	// that have no containers as to be created,
+	// plus mark any that have differing configHash as to be updated
+	for moduleName, config := range configs {
+		if _, has := planModules[moduleName]; !has {
+			planModules[moduleName] = &SimplifiedTerraformModule{
+				DisplayName: moduleName,
+				Resources:   []*SimplifiedTerraformResource{},
+			}
+		}
+		for _, service := range config.Services {
+			container := containersByModuleService[fmt.Sprintf("%s_%s", moduleName, service.Name)]
+			plan := "none"
+			if container.ID != "" {
+				existingConfigHash := container.Labels["com.docker.compose.config-hash"]
+				getConfigHashTaskName := fmt.Sprintf("%s_get_compose_config_hash", moduleName)
+				newConfigHash := results[getConfigHashTaskName].Result.(map[string]string)[service.Name]
+				if existingConfigHash != newConfigHash {
+					plan = "recreate"
+				}
+			} else {
+				plan = "create"
+			}
+			planModules[moduleName].Resources = append(
+				planModules[moduleName].Resources,
+				&SimplifiedTerraformResource{
+					DisplayName: service.Name,
+					Plan:        plan,
+				},
+			)
 		}
 	}
 
-	dot := fmt.Sprintf(`
-	  digraph {
-	    compound = "true"
-	    newrank = "true"
-	    ranksep = 0.1;
+	return &SimplifiedTerraformStatus{
+		Modules:     planModules,
+		Variables:   map[string]string{},
+		Connections: connections,
+	}, nil
 
-	    %s
-	    %s
-	  }`,
-		strings.Join(moduleDots, ""),
-		strings.Join(dependencyDots, ""),
-	)
+}
 
-	exitCode, dotStdout, dotStderr, err := shellExecInputPipe(".", &dot, "dot", "-Tsvg")
-	err = errorFromShellExecResult("dot -Tsvg", exitCode, dotStdout, dotStderr, err)
-	if err != nil {
-		return []byte{}, err
+func doInParallel(actions ...func() taskResult) map[string]taskResult {
+
+	resultsChannel := make(chan taskResult)
+	results := map[string]taskResult{}
+
+	for _, action := range actions {
+		go (func() {
+			result := action()
+			resultsChannel <- result
+			if result.Err != nil {
+				close(resultsChannel)
+			}
+		})()
 	}
 
-	// svgString := string(dotStdout)
+	for range actions {
+		select {
+		case result := <-resultsChannel:
+			results[result.Name] = result
+			log.Printf("task '%s' completed", result.Name)
+		default:
+			// nothing to do.
+		}
+	}
+
+	return results
+}
 
-	return dotStdout, nil
+type taskResult struct {
+	Name   string
+	Err    error
+	Result interface{}
 }
+
+// func makeSVGFromDockerCompose(
+// 	moduleDependencies map[string][]string,
+// 	configs map[string]*composetypes.Config,
+// ) ([]byte, error) {
+
+// 	moduleDots := []string{}
+// 	for moduleName := range configs {
+// 		serviceDots := []string{}
+// 		config := configs[moduleName]
+
+// 		for _, service := range config.Services {
+// 			id := fmt.Sprintf(`service_%s_%s`, moduleName, service.Name)
+// 			id = strings.ReplaceAll(strings.ReplaceAll(id, ".", "_"), "-", "_")
+// 			serviceDot := fmt.Sprintf(`
+// 	      "%s" [label = "%s", tooltip="%s", margin = 0.1, shape = "box3d]`,
+// 				id, padStringForDot(service.Name), id,
+// 			)
+// 			serviceDots = append(serviceDots, serviceDot)
+// 		}
+// 		i := 0
+// 		for _, service := range config.Services {
+// 			if i != 0 {
+// 				id0 := fmt.Sprintf(`service_%s_%s`, moduleName, config.Services[i-1].Name)
+// 				id0 = strings.ReplaceAll(strings.ReplaceAll(id0, ".", "_"), "-", "_")
+// 				id1 := fmt.Sprintf(`service_%s_%s`, moduleName, service.Name)
+// 				id1 = strings.ReplaceAll(strings.ReplaceAll(id1, ".", "_"), "-", "_")
+// 				serviceDot := fmt.Sprintf(`
+// 	        "%s" -> "%s" [style=invis]`, id0, id1,
+// 				)
+// 				serviceDots = append(serviceDots, serviceDot)
+// 			}
+// 			i++
+// 		}
+
+// 		moduleDot := fmt.Sprintf(`
+// 	    subgraph cluster_%s {
+// 	      bgcolor = lightgrey;
+// 	      tooltip = "%s";
+// 	      label = "%s";
+// 	      %s
+// 	    }`,
+// 			strings.ReplaceAll(strings.ReplaceAll(moduleName, ".", "_"), "-", "_"),
+// 			moduleName,
+// 			padStringForDot(moduleName),
+// 			strings.Join(serviceDots, ""),
+// 		)
+// 		moduleDots = append(moduleDots, moduleDot)
+// 	}
+
+// 	dependencyDots := []string{}
+// 	for to, fromList := range moduleDependencies {
+// 		for _, from := range fromList {
+// 			fromServiceName := configs[from].Services[len(configs[from].Services)-1].Name
+// 			toServiceName := configs[to].Services[0].Name
+
+// 			fromService := fmt.Sprintf(`service_%s_%s`, from, fromServiceName)
+// 			toService := fmt.Sprintf(`service_%s_%s`, to, toServiceName)
+
+// 			dependencyDot := fmt.Sprintf(`
+// 				"%s" -> "%s" [ ltail="cluster_%s", lhead="cluster_%s" ];`,
+// 				strings.ReplaceAll(strings.ReplaceAll(fromService, ".", "_"), "-", "_"),
+// 				strings.ReplaceAll(strings.ReplaceAll(toService, ".", "_"), "-", "_"),
+// 				strings.ReplaceAll(strings.ReplaceAll(from, ".", "_"), "-", "_"),
+// 				strings.ReplaceAll(strings.ReplaceAll(to, ".", "_"), "-", "_"),
+// 			)
+
+// 			dependencyDots = append(dependencyDots, dependencyDot)
+// 		}
+// 	}
+
+// 	dot := fmt.Sprintf(`
+// 	  digraph {
+// 	    compound = "true"
+// 	    newrank = "true"
+// 	    ranksep = 0.1;
+
+// 	    %s
+// 	    %s
+// 	  }`,
+// 		strings.Join(moduleDots, ""),
+// 		strings.Join(dependencyDots, ""),
+// 	)
+
+// 	exitCode, dotStdout, dotStderr, err := shellExecInputPipe(".", &dot, "dot", "-Tsvg")
+// 	err = errorFromShellExecResult("dot -Tsvg", exitCode, dotStdout, dotStderr, err)
+// 	if err != nil {
+// 		return []byte{}, err
+// 	}
+
+// 	svgBytes, err := modifyDotSVGAsXML(dotStdout, func(svgDoc *XMLNode) error {
+
+// 		// correctly set the dot property for services
+// 		svgDoc.WithQuerySelector(
+// 			// 'g.node > title'
+// 			[]XMLQuery{XMLQuery{NodeType: "g", Class: "node"}, XMLQuery{NodeType: "title", DirectChild: true}},
+// 			func(node *XMLNode) {
+// 				if node.Parent != nil {
+// 					title := html.UnescapeString(string(node.Content))
+// 					node.Parent.SetAttr("data-dot", title)
+// 					if strings.HasPrefix(title, "service_") {
+// 						node.Parent.SetAttr("class", "resource")
+// 					}
+// 				}
+// 			},
+// 		)
+
+// 		// correctly set the dot property for the "cluster"s (aka modules)
+// 		moduleNames := []string{}
+// 		svgDoc.WithQuerySelector(
+// 			// 'a[xlink:title]'
+// 			[]XMLQuery{XMLQuery{NodeType: "a", Attr: "xlink:title"}},
+// 			func(node *XMLNode) {
+// 				if node.Parent != nil && node.Parent.Parent != nil {
+// 					title := html.UnescapeString(node.GetAttr("xlink:title"))
+// 					title = regexp.MustCompile(`[.-]`).ReplaceAllString(title, "_")
+// 					if node.Parent.Parent.GetAttr("data-dot") == "" {
+// 						moduleNames = append(moduleNames, title)
+// 						node.Parent.Parent.SetAttr("data-dot", title)
+// 						node.Parent.Parent.SetAttr("class", "module")
+// 					}
+
+// 				}
+// 			},
+// 		)
+
+// 		// correctly set the dot property for edges
+// 		// in order to get dot to render it correctly, the connections are actually between resources,
+// 		// not between modules. So we have to trim the resource name off of the dot attribute
+// 		// on the connections to make them match what we have in the status JSON object.
+// 		svgDoc.WithQuerySelector(
+// 			// 'g.edge title'
+// 			[]XMLQuery{XMLQuery{NodeType: "g", Class: "edge"}, XMLQuery{NodeType: "title"}},
+// 			func(node *XMLNode) {
+// 				if node.Parent != nil {
+// 					title := html.UnescapeString(string(node.Content))
+// 					fromTo := strings.Split(title, "->")
+// 					if len(fromTo) != 2 {
+// 						fmt.Printf("malformed dot string on svg edge: fromTo.length != 2: %s", title)
+// 					}
+// 					// if this edge starts or ends at a resource inside a module, override the dot string
+// 					// to make it start or end at that module instead.
+// 					for _, moduleName := range moduleNames {
+
+// 						if strings.HasPrefix(strings.TrimPrefix(fromTo[0], "service_"), moduleName) {
+// 							fromTo[0] = moduleName
+// 						}
+// 						if strings.HasPrefix(strings.TrimPrefix(fromTo[1], "service_"), moduleName) {
+// 							fromTo[1] = moduleName
+// 						}
+// 					}
+// 					node.Parent.SetAttr("data-dot", html.EscapeString(fmt.Sprintf("%s->%s", fromTo[0], fromTo[1])))
+// 				}
+// 			},
+// 		)
+
+// 		// two last things to do:
+// 		// 1. center each resource horizontally within the bounding box of its parent module
+// 		//    because dot is dumb and it just kinda throws them wherever
+// 		// 2. create little create/delete/modify icons next to each changed resource
+// 		//    similar to how terraform displays -, +, +/-, or ~ on each line-item in the plan
+
+// 		for moduleName, config := range configs {
+// 			moduleId := regexp.MustCompile(`[.-]`).ReplaceAllString(moduleName, "_")
+// 			svgDoc.WithQuerySelector(
+// 				[]XMLQuery{XMLQuery{Attr: "data-dot", AttrValue: moduleId}},
+// 				func(node *XMLNode) {
+// 					moduleRect := node.GetBoundingBox()
+
+// 					for _, service := range config.Services {
+// 						resourceId := fmt.Sprintf("%s_%s", moduleId, regexp.MustCompile(`[.-]`).ReplaceAllString(service.Name, "_"))
+// 						svgDoc.WithQuerySelector(
+// 							// '[data-dot=module_dns_gandi_gandi_livedns_record_dns_entries]' for example
+// 							[]XMLQuery{XMLQuery{Attr: "data-dot", AttrValue: resourceId}},
+// 							func(node *XMLNode) {
+// 								// TODO pass the plan here
+// 								centerResourceAndAddPlannedAction(moduleRect, node, resourceId, "create")
+// 							},
+// 						)
+// 					}
+// 				},
+// 			)
+// 		}
+
+// 		return nil
+// 	})
+
+// 	return svgBytes, err
+// }

automation/terraformActions.go

@@ -295,21 +295,8 @@ func TerraformPlanAndApply(
 		return []byte{}, nil, errors.Wrap(err, "can't TerraformPlanAndApply because can't process.Start() terraform apply process")
 	}
 
-	scanAllOutput := func(reader io.ReadCloser) {
-		defer func() {
-			if err := recover(); err != nil {
-				log.Println("scanAllOutput panic:")
-				log.Println(err)
-			}
-		}()
-		scanner := bufio.NewScanner(reader)
-		for scanner.Scan() {
-			logLinesChannel <- scanner.Text()
-		}
-	}
-
-	go scanAllOutput(stdoutPipe)
-	go scanAllOutput(stderrPipe)
+	go scanAllOutput(stdoutPipe, logLinesChannel)
+	go scanAllOutput(stderrPipe, logLinesChannel)
 
 	toReturn := make(chan TerraformApplyResult)
 
@@ -329,6 +316,19 @@ func TerraformPlanAndApply(
 	return svg, toReturn, nil
 }
 
+func scanAllOutput(reader io.ReadCloser, logLinesChannel chan string) {
+	defer func() {
+		if err := recover(); err != nil {
+			log.Println("scanAllOutput panic:")
+			log.Println(err)
+		}
+	}()
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		logLinesChannel <- scanner.Text()
+	}
+}
+
 func monitorTerraformApplyProgress(
 	workingDirectory string,
 	terraformProject string,

automation/terraformSvg.go

@@ -265,29 +265,146 @@ func makeSVGFromSimpleStatus(simpleStatus *SimplifiedTerraformStatus) ([]byte, e
 		return []byte{}, err
 	}
 
-	svgString := string(dotStdout)
-
-	// the svg output from dot has html comments that contain metadata preceeding each entity.
-	// for example:
-	// <!-- param_ssh_private_key_filepath -->
-	// <g id="node16" class="node">
-	// We will attach that metadata to the svg elements as a data property
-	// Note that this will not work perfectly for the "cluster"s (aka modules)
-	// they are handled separately after the SVG has been inserted.
-	// Yes parsing XML with regex is "evil"  but keep in mind, this XML was generated by a program, not written by hand.
-	// So it is a lot more predictable. Also, I couldn't figure out how to parse the comments with the go std lib XML parser.
-	commentRegex := regexp.MustCompile(`<!--\s*(.+)\s*-->\s*\n\s*(<[a-z]+\s)`)
-	svgString = commentRegex.ReplaceAllStringFunc(svgString, func(matched string) string {
-		matches := commentRegex.FindStringSubmatch(matched)
-		return fmt.Sprintf("%s dot=\"%s\" ", matches[2], strings.Trim(html.UnescapeString(matches[1]), " "))
+	svgBytes, err := modifyDotSVGAsXML(dotStdout, func(svgDoc *XMLNode) error {
+		// correctly set the dot property for resources, variables, and parameters
+		svgDoc.WithQuerySelector(
+			// 'g.node > title'
+			[]XMLQuery{XMLQuery{NodeType: "g", Class: "node"}, XMLQuery{NodeType: "title", DirectChild: true}},
+			func(node *XMLNode) {
+				if node.Parent != nil {
+					title := html.UnescapeString(string(node.Content))
+					node.Parent.SetAttr("data-dot", title)
+					if strings.HasPrefix(title, "var_") || strings.HasPrefix(title, "data_") {
+						node.Parent.SetAttr("class", "variable")
+					}
+					if strings.HasPrefix(title, "param_") {
+						node.Parent.SetAttr("class", "parameter")
+
+						// the parameters start out as boxes
+						//but lets squeeze them into keystone shape to differentiate them a bit
+						node.Parent.WithQuerySelector([]XMLQuery{XMLQuery{NodeType: "polygon"}}, func(polygonNode *XMLNode) {
+							vertices, err := polygonNode.GetSVGPolygonVertices()
+							if err == nil {
+								center := []float64{0, 0}
+								for _, vertex := range vertices {
+									center[0] += vertex[0]
+									center[1] += vertex[1]
+								}
+								center[0] = center[0] / float64(len(vertices))
+								center[1] = center[1] / float64(len(vertices))
+
+								newVertices := [][]float64{}
+								for _, vertex := range vertices {
+									x := vertex[0]
+									y := vertex[1]
+									if y > center[1] {
+										if x > center[0] {
+											x -= 10
+										} else {
+											x += 10
+										}
+									}
+									newVertices = append(newVertices, []float64{x, y})
+								}
+								polygonNode.SetSVGPolygonVertices(newVertices)
+							}
+
+						})
+					}
+					if strings.HasPrefix(title, "module_") {
+						node.Parent.SetAttr("class", "resource")
+					}
+				}
+			},
+		)
+
+		// correctly set the dot property for the "cluster"s (aka modules)
+		moduleNames := []string{}
+		svgDoc.WithQuerySelector(
+			// 'a[xlink:title]'
+			[]XMLQuery{XMLQuery{NodeType: "a", Attr: "xlink:title"}},
+			func(node *XMLNode) {
+				if node.Parent != nil && node.Parent.Parent != nil {
+					title := html.UnescapeString(node.GetAttr("xlink:title"))
+					title = regexp.MustCompile(`[.-]`).ReplaceAllString(title, "_")
+					if node.Parent.Parent.GetAttr("data-dot") == "" {
+						moduleNames = append(moduleNames, title)
+						node.Parent.Parent.SetAttr("data-dot", title)
+						node.Parent.Parent.SetAttr("class", "module")
+					}
+
+				}
+			},
+		)
+
+		// correctly set the dot property for edges
+		// in order to get dot to render it correctly, the connections are actually between resources,
+		// not between modules. So we have to trim the resource name off of the dot attribute
+		// on the connections to make them match what we have in the status JSON object.
+		svgDoc.WithQuerySelector(
+			// 'g.edge title'
+			[]XMLQuery{XMLQuery{NodeType: "g", Class: "edge"}, XMLQuery{NodeType: "title"}},
+			func(node *XMLNode) {
+				if node.Parent != nil {
+					title := html.UnescapeString(string(node.Content))
+					fromTo := strings.Split(title, "->")
+					if len(fromTo) != 2 {
+						fmt.Printf("malformed dot string on svg edge: fromTo.length != 2: %s", title)
+					}
+
+					// if this edge starts or ends at a resource inside a module, override the dot string
+					// to make it start or end at that module instead.
+					for _, moduleName := range moduleNames {
+						if strings.HasPrefix(fromTo[0], moduleName) {
+							node.Parent.SetAttr("data-dot", html.EscapeString(fmt.Sprintf("%s->%s", moduleName, fromTo[1])))
+						}
+						if strings.HasPrefix(fromTo[1], moduleName) {
+							node.Parent.SetAttr("data-dot", html.EscapeString(fmt.Sprintf("%s->%s", fromTo[0], moduleName)))
+						}
+					}
+				}
+			},
+		)
+
+		// two last things to do:
+		// 1. center each resource horizontally within the bounding box of its parent module
+		//    because dot is dumb and it just kinda throws them wherever
+		// 2. create little create/delete/modify icons next to each changed resource
+		//    similar to how terraform displays -, +, +/-, or ~ on each line-item in the plan
+
+		for moduleName, module := range simpleStatus.Modules {
+			moduleId := regexp.MustCompile(`[.-]`).ReplaceAllString(moduleName, "_")
+			svgDoc.WithQuerySelector(
+				[]XMLQuery{XMLQuery{Attr: "data-dot", AttrValue: moduleId}},
+				func(node *XMLNode) {
+					moduleRect := node.GetBoundingBox()
+
+					for _, resource := range module.Resources {
+						resourceId := fmt.Sprintf("%s_%s", moduleId, regexp.MustCompile(`[.-]`).ReplaceAllString(resource.DisplayName, "_"))
+						svgDoc.WithQuerySelector(
+							// '[data-dot=module_dns_gandi_gandi_livedns_record_dns_entries]' for example
+							[]XMLQuery{XMLQuery{Attr: "data-dot", AttrValue: resourceId}},
+							func(node *XMLNode) {
+								centerResourceAndAddPlannedAction(moduleRect, node, resourceId, resource.Plan)
+							},
+						)
+					}
+				},
+			)
+		}
+
+		return nil
 	})
 
-	//ioutil.WriteFile("./test-gen.svg", []byte(svgString), 0777)
+	return svgBytes, err
+}
+
+func modifyDotSVGAsXML(svgInputBytes []byte, action func(*XMLNode) error) ([]byte, error) {
 
 	var svgDoc XMLNode
-	err = xml.Unmarshal(dotStdout, &svgDoc)
+	err := xml.Unmarshal(svgInputBytes, &svgDoc)
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
 
 	var setParentRecurse func(*XMLNode)
@@ -308,106 +425,6 @@ func makeSVGFromSimpleStatus(simpleStatus *SimplifiedTerraformStatus) ([]byte, e
 		},
 	)
 
-	// correctly set the dot property for resources, variables, and parameters
-	svgDoc.WithQuerySelector(
-		// 'g.node > title'
-		[]XMLQuery{XMLQuery{NodeType: "g", Class: "node"}, XMLQuery{NodeType: "title", DirectChild: true}},
-		func(node *XMLNode) {
-			if node.Parent != nil {
-				title := html.UnescapeString(string(node.Content))
-				node.Parent.SetAttr("data-dot", title)
-				if strings.HasPrefix(title, "var_") || strings.HasPrefix(title, "data_") {
-					node.Parent.SetAttr("class", "variable")
-				}
-				if strings.HasPrefix(title, "param_") {
-					node.Parent.SetAttr("class", "parameter")
-
-					// the parameters start out as boxes
-					//but lets squeeze them into keystone shape to differentiate them a bit
-					node.Parent.WithQuerySelector([]XMLQuery{XMLQuery{NodeType: "polygon"}}, func(polygonNode *XMLNode) {
-						vertices, err := polygonNode.GetSVGPolygonVertices()
-						if err == nil {
-							center := []float64{0, 0}
-							for _, vertex := range vertices {
-								center[0] += vertex[0]
-								center[1] += vertex[1]
-							}
-							center[0] = center[0] / float64(len(vertices))
-							center[1] = center[1] / float64(len(vertices))
-
-							newVertices := [][]float64{}
-							for _, vertex := range vertices {
-								x := vertex[0]
-								y := vertex[1]
-								if y > center[1] {
-									if x > center[0] {
-										x -= 10
-									} else {
-										x += 10
-									}
-								}
-								newVertices = append(newVertices, []float64{x, y})
-							}
-							polygonNode.SetSVGPolygonVertices(newVertices)
-						}
-
-					})
-				}
-				if strings.HasPrefix(title, "module_") {
-					node.Parent.SetAttr("class", "resource")
-				}
-			}
-		},
-	)
-
-	// correctly set the dot property for the "cluster"s (aka modules)
-	moduleNames := []string{}
-	svgDoc.WithQuerySelector(
-		// 'a[xlink:title]'
-		[]XMLQuery{XMLQuery{NodeType: "a", Attr: "xlink:title"}},
-		func(node *XMLNode) {
-			if node.Parent != nil && node.Parent.Parent != nil {
-				title := html.UnescapeString(node.GetAttr("xlink:title"))
-				title = regexp.MustCompile(`[.-]`).ReplaceAllString(title, "_")
-				if node.Parent.Parent.GetAttr("data-dot") == "" {
-					moduleNames = append(moduleNames, title)
-					node.Parent.Parent.SetAttr("data-dot", title)
-					node.Parent.Parent.SetAttr("class", "module")
-				}
-
-			}
-		},
-	)
-
-	// correctly set the dot property for edges
-	// in order to get dot to render it correctly, the connections are actually between resources,
-	// not between modules. So we have to trim the resource name off of the dot attribute
-	// on the connections to make them match what we have in the status JSON object.
-	svgDoc.WithQuerySelector(
-		// 'g.edge title'
-		[]XMLQuery{XMLQuery{NodeType: "g", Class: "edge"}, XMLQuery{NodeType: "title"}},
-		func(node *XMLNode) {
-			if node.Parent != nil {
-				title := html.UnescapeString(string(node.Content))
-				fromTo := strings.Split(title, "->")
-				if len(fromTo) != 2 {
-					fmt.Printf("malformed dot string on svg edge: fromTo.length != 2: %s", title)
-				}
-
-				// if this edge starts or ends at a resource inside a module, override the dot string
-				// to make it start or end at that module instead.
-				for _, moduleName := range moduleNames {
-					if strings.HasPrefix(fromTo[0], moduleName) {
-						node.Parent.SetAttr("data-dot", html.EscapeString(fmt.Sprintf("%s->%s", moduleName, fromTo[1])))
-					}
-					if strings.HasPrefix(fromTo[1], moduleName) {
-						node.Parent.SetAttr("data-dot", html.EscapeString(fmt.Sprintf("%s->%s", fromTo[0], moduleName)))
-					}
-				}
-			}
-		},
-	)
-
 	svgDoc.WithQuerySelector(
 		// '[font-family]'
 		[]XMLQuery{XMLQuery{Attr: "font-family"}},
@@ -416,120 +433,9 @@ func makeSVGFromSimpleStatus(simpleStatus *SimplifiedTerraformStatus) ([]byte, e
 		},
 	)
 
-	// two last things to do:
-	// 1. center each resource horizontally within the bounding box of its parent module
-	//    because dot is dumb and it just kinda throws them wherever
-	// 2. create little create/delete/modify icons next to each changed resource
-	//    similar to how terraform displays -, +, +/-, or ~ on each line-item in the plan
-
-	for moduleName, module := range simpleStatus.Modules {
-
-		moduleId := regexp.MustCompile(`[.-]`).ReplaceAllString(moduleName, "_")
-		svgDoc.WithQuerySelector(
-			[]XMLQuery{XMLQuery{Attr: "data-dot", AttrValue: moduleId}},
-			func(node *XMLNode) {
-				moduleRect := node.GetBoundingBox()
-
-				for _, resource := range module.Resources {
-					resourceId := fmt.Sprintf("%s_%s", moduleId, regexp.MustCompile(`[.-]`).ReplaceAllString(resource.DisplayName, "_"))
-					svgDoc.WithQuerySelector(
-						// '[data-dot=module_dns_gandi_gandi_livedns_record_dns_entries]' for example
-						[]XMLQuery{XMLQuery{Attr: "data-dot", AttrValue: resourceId}},
-						func(node *XMLNode) {
-							resourceRect := node.GetBoundingBox()
-
-							// Step 1 center the resources
-							nudgeX := ((moduleRect.Left + moduleRect.Right) - (resourceRect.Left + resourceRect.Right)) * float64(0.5)
-							node.SetAttr("transform", fmt.Sprintf("scale(1 1) rotate(0) translate(%d %d)", int(nudgeX), 0))
-
-							// Step 2 add the terraform action display if there is a planned action
-							if resource.Plan != "none" && resource.Plan != "" {
-
-								actionX := resourceRect.Left + nudgeX + float64(terraformActionOffsetX)
-								actionY := resourceRect.Top + float64(terraformActionOffsetY)
-
-								// start with a circle element
-								actionContents := []*XMLNode{
-									&XMLNode{
-										XMLName: xml.Name{Local: "circle", Space: "http://www.w3.org/2000/svg"},
-										Attrs: []*XMLAttr{
-											&XMLAttr{
-												Local: "cx",
-												Value: fmt.Sprintf("%.4f", actionX),
-											},
-											&XMLAttr{
-												Local: "fill",
-												Value: "#ffffff",
-											},
-											&XMLAttr{
-												Local: "cy",
-												Value: fmt.Sprintf("%.4f", actionY),
-											},
-											&XMLAttr{
-												Local: "r",
-												Value: strconv.Itoa(terraformActionCircleRadius),
-											},
-										},
-									},
-								}
-
-								// macro to add text to the inside of the circle
-								createText := func(str, class string, offset int) {
-									actionContents = append(actionContents, &XMLNode{
-										XMLName: xml.Name{Local: "text", Space: "http://www.w3.org/2000/svg"},
-										Attrs: []*XMLAttr{
-											&XMLAttr{Local: "text-anchor", Value: "middle"},
-											&XMLAttr{Local: "font-family", Value: terraformSvgFontFamily},
-											&XMLAttr{Local: "text-size", Value: strconv.Itoa(terraformActionFontSize)},
-											&XMLAttr{Local: "class", Value: class},
-											&XMLAttr{Local: "x", Value: fmt.Sprintf("%.4f", actionX+float64(offset*terraformActionTextSpacingX))},
-											&XMLAttr{Local: "y", Value: fmt.Sprintf("%.4f", actionY+float64(terraformActionTextOffsetY))},
-										},
-										Content: []byte(str),
-									})
-								}
-
-								// add text to the circle depending on the planned action
-								if resource.Plan == "recreate" {
-									createText("-", terraformActionRedClass, -1)
-									createText("/", "", 0)
-									createText("+", terraformActionGreenClass, 1)
-								}
-								if resource.Plan == "update" {
-									createText("~", terraformActionYellowClass, 0)
-								}
-								if resource.Plan == "create" {
-									createText("+", terraformActionGreenClass, 0)
-								}
-								if resource.Plan == "delete" {
-									createText("-", terraformActionRedClass, 0)
-								}
-
-								// Finally, add all the action content inside a single <g> tag & append to the svg
-								node.Parent.Children = append(node.Parent.Children, &XMLNode{
-									XMLName: xml.Name{Local: "g", Space: "http://www.w3.org/2000/svg"},
-									Attrs: []*XMLAttr{
-										&XMLAttr{
-											Local: "class",
-											Value: "action-circle",
-										},
-										&XMLAttr{
-											Local: "data-dot",
-											Value: fmt.Sprintf("%s_action_circle", resourceId),
-										},
-									},
-									Children: actionContents,
-								})
-
-							}
-						},
-					)
-
-				}
-
-			},
-		)
-
+	err = action(&svgDoc)
+	if err != nil {
+		return nil, err
 	}
 
 	var cleanUpSvgRecurse func(*XMLNode)
@@ -574,6 +480,95 @@ func makeSVGFromSimpleStatus(simpleStatus *SimplifiedTerraformStatus) ([]byte, e
 	return svgBytes, nil
 }
 
+func centerResourceAndAddPlannedAction(parent *Rect2D, node *XMLNode, nodeId string, action string) {
+	resourceRect := node.GetBoundingBox()
+
+	// Step 1 center the resources
+	nudgeX := ((parent.Left + parent.Right) - (resourceRect.Left + resourceRect.Right)) * float64(0.5)
+	node.SetAttr("transform", fmt.Sprintf("scale(1 1) rotate(0) translate(%d %d)", int(nudgeX), 0))
+
+	// Step 2 add the terraform action display if there is a planned action
+	if action != "none" && action != "" {
+
+		actionX := resourceRect.Left + nudgeX + float64(terraformActionOffsetX)
+		actionY := resourceRect.Top + float64(terraformActionOffsetY)
+
+		// start with a circle element
+		actionContents := []*XMLNode{
+			&XMLNode{
+				XMLName: xml.Name{Local: "circle", Space: "http://www.w3.org/2000/svg"},
+				Attrs: []*XMLAttr{
+					&XMLAttr{
+						Local: "cx",
+						Value: fmt.Sprintf("%.4f", actionX),
+					},
+					&XMLAttr{
+						Local: "fill",
+						Value: "#ffffff",
+					},
+					&XMLAttr{
+						Local: "cy",
+						Value: fmt.Sprintf("%.4f", actionY),
+					},
+					&XMLAttr{
+						Local: "r",
+						Value: strconv.Itoa(terraformActionCircleRadius),
+					},
+				},
+			},
+		}
+
+		// macro to add text to the inside of the circle
+		createText := func(str, class string, offset int) {
+			actionContents = append(actionContents, &XMLNode{
+				XMLName: xml.Name{Local: "text", Space: "http://www.w3.org/2000/svg"},
+				Attrs: []*XMLAttr{
+					&XMLAttr{Local: "text-anchor", Value: "middle"},
+					&XMLAttr{Local: "font-family", Value: terraformSvgFontFamily},
+					&XMLAttr{Local: "text-size", Value: strconv.Itoa(terraformActionFontSize)},
+					&XMLAttr{Local: "class", Value: class},
+					&XMLAttr{Local: "x", Value: fmt.Sprintf("%.4f", actionX+float64(offset*terraformActionTextSpacingX))},
+					&XMLAttr{Local: "y", Value: fmt.Sprintf("%.4f", actionY+float64(terraformActionTextOffsetY))},
+				},
+				Content: []byte(str),
+			})
+		}
+
+		// add text to the circle depending on the planned action
+		if action == "recreate" {
+			createText("-", terraformActionRedClass, -1)
+			createText("/", "", 0)
+			createText("+", terraformActionGreenClass, 1)
+		}
+		if action == "update" {
+			createText("~", terraformActionYellowClass, 0)
+		}
+		if action == "create" {
+			createText("+", terraformActionGreenClass, 0)
+		}
+		if action == "delete" {
+			createText("-", terraformActionRedClass, 0)
+		}
+
+		// Finally, add all the action content inside a single <g> tag & append to the svg
+		node.Parent.Children = append(node.Parent.Children, &XMLNode{
+			XMLName: xml.Name{Local: "g", Space: "http://www.w3.org/2000/svg"},
+			Attrs: []*XMLAttr{
+				&XMLAttr{
+					Local: "class",
+					Value: "action-circle",
+				},
+				&XMLAttr{
+					Local: "data-dot",
+					Value: fmt.Sprintf("%s_action_circle", nodeId),
+				},
+			},
+			Children: actionContents,
+		})
+
+	}
+}
+
 func padStringForDot(str string) string {
 	pad := strings.Repeat(" ", int(float32(len(str)+dotStringCharactersMinPadding)*dotStringLinearPadding))
 	return fmt.Sprintf("%s%s%s", pad, str, pad)

main.go

@@ -20,12 +20,6 @@ type applicationState struct {
 	storage          objectStorage.ObjectStorager
 }
 
-type taskResult struct {
-	Name   string
-	Err    error
-	Result interface{}
-}
-
 var global applicationState
 
 func main() {
@@ -43,8 +37,8 @@ func main() {
 
 	go terraformStateServer()
 
-	results := doStartupActionsInParallel(
-		func() taskResult {
+	results := automation.doInParallel(
+		func() automation.taskResult {
 			// This creates an access key that the gateway cloud instance can use to upload its SSH public key
 			// to our object storage. the host-key-poller will download this SSH host public key and add it to our known_hosts
 			// so that we can SSH to the gateway instance securely
@@ -58,18 +52,18 @@ func main() {
 			}
 			knownHostsCredentials, err := global.storage.CreateAccessKeyIfNotExists(hostKeysAccessSpec)
 			if err != nil {
-				return taskResult{
+				return automation.taskResult{
 					Name: "knownHostsCredentials",
 					Err:  errors.Wrap(err, "can't create object storage access key for known_hosts"),
 				}
 			}
-			return taskResult{
+			return automation.taskResult{
 				Name:   "knownHostsCredentials",
 				Result: knownHostsCredentials,
 			}
 		},
 
-		func() taskResult {
+		func() automation.taskResult {
 			// BuildTLSCertsForThreshold fills in the CAs, Keys, and Certificates in the Threshold ansible roles.
 			// So when terraform invokes ansible to install threshold client/server, it will install working
 			// certificates and keys
@@ -80,20 +74,20 @@ func main() {
 				global.storage,
 			)
 			if err != nil {
-				return taskResult{
+				return automation.taskResult{
 					Name: "buildTLSCertsForThreshold",
 					Err:  errors.Wrap(err, "can't create certs for threshold"),
 				}
 			}
-			return taskResult{Name: "knownHostsCredentials"}
+			return automation.taskResult{Name: "knownHostsCredentials"}
 		},
 
-		func() taskResult {
+		func() automation.taskResult {
 			sshPort := 2201
 			hostSSHPortFilename := fmt.Sprintf("rootsystem/ssh/%s.txt", config.Host.Name)
 			file, notFound, err := global.storage.Get(hostSSHPortFilename)
 			if err != nil && !notFound {
-				return taskResult{
+				return automation.taskResult{
 					Name: "sshPort",
 					Err:  errors.Wrapf(err, "can't download %s", hostSSHPortFilename),
 				}
@@ -101,7 +95,7 @@ func main() {
 			if !notFound {
 				sshPort, err = strconv.Atoi(string(file.Content))
 				if err != nil {
-					return taskResult{
+					return automation.taskResult{
 						Name: "sshPort",
 						Err:  errors.Wrapf(err, "can't read %s as a number", hostSSHPortFilename),
 					}
@@ -109,7 +103,7 @@ func main() {
 			} else {
 				file, notFound, err := global.storage.Get("rootsystem/ssh/next-port.txt")
 				if err != nil && !notFound {
-					return taskResult{
+					return automation.taskResult{
 						Name: "sshPort",
 						Err:  errors.Wrap(err, "can't download next-port.txt"),
 					}
@@ -123,31 +117,31 @@ func main() {
 				}
 				err = global.storage.Put(hostSSHPortFilename, []byte(strconv.Itoa(sshPort)))
 				if err != nil {
-					return taskResult{
+					return automation.taskResult{
 						Name: "sshPort",
 						Err:  errors.Wrapf(err, "can't can't upload %s", hostSSHPortFilename),
 					}
 				}
 				err = global.storage.Put("rootsystem/ssh/next-port.txt", []byte(strconv.Itoa(sshPort+1)))
 				if err != nil {
-					return taskResult{
+					return automation.taskResult{
 						Name: "sshPort",
 						Err:  errors.Wrap(err, "can't can't upload next-port.txt"),
 					}
 				}
 			}
-			return taskResult{
+			return automation.taskResult{
 				Name:   "sshPort",
 				Result: sshPort,
 			}
 
 		},
 
-		func() taskResult {
+		func() automation.taskResult {
 			// Add 1 to the build number, each time rootsystem runs is a different build.
 			file, notFound, err := global.storage.Get("rootsystem/terraform-logs/build-number.txt")
 			if err != nil && !notFound {
-				return taskResult{
+				return automation.taskResult{
 					Name: "buildNumber",
 					Err:  errors.Wrap(err, "can't download build-number.txt"),
 				}
@@ -163,12 +157,12 @@ func main() {
 			}
 			err = global.storage.Put("rootsystem/terraform-logs/build-number.txt", []byte(strconv.Itoa(buildNumber)))
 			if err != nil {
-				return taskResult{
+				return automation.taskResult{
 					Name: "buildNumber",
 					Err:  errors.Wrap(err, "can't can't upload build-number.txt"),
 				}
 			}
-			return taskResult{
+			return automation.taskResult{
 				Name:   "buildNumber",
 				Result: buildNumber,
 			}
@@ -323,31 +317,3 @@ func terraformStateServer() error {
 
 	return server.ListenAndServe()
 }
-
-func doStartupActionsInParallel(actions ...func() taskResult) map[string]taskResult {
-
-	resultsChannel := make(chan taskResult)
-	results := map[string]taskResult{}
-
-	for _, action := range actions {
-		go (func() {
-			result := action()
-			resultsChannel <- result
-			if result.Err != nil {
-				close(resultsChannel)
-			}
-		})()
-	}
-
-	for range actions {
-		select {
-		case result := <-resultsChannel:
-			results[result.Name] = result
-			log.Printf("task '%s' completed", result.Name)
-		default:
-			// nothing to do.
-		}
-	}
-
-	return results
-}