rootsystem/automation/terraformActions.go

package automation

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	errors "git.sequentialread.com/forest/pkg-errors"
	"git.sequentialread.com/forest/rootsystem/configuration"
)

type TerraformShow struct {
	PlannedValues   TerraformShowPlannedValues     `json:"planned_values"`
	Configuraton    TerraformShowConfiguration     `json:"configuration"`
	ResourceChanges []*TerraformShowResourceChange `json:"resource_changes"`
}

type TerraformShowPlannedValues struct {
	RootModule *TerraformShowPlannedValuesRoot `json:"root_module"`
}

type TerraformShowPlannedValuesRoot struct {
	ChildModules []*TerraformShowModule `json:"child_modules"`
}

type TerraformShowConfiguration struct {
	RootModule TerraformShowRootModule `json:"root_module"`
}

type TerraformShowRootModule struct {
	ModuleCalls map[string]*TerraformShowModuleCall `json:"module_calls"`
}

type TerraformShowModule struct {
	Resources []*TerraformShowResource `json:"resources"`
	Address   string                   `json:"address"`
}

type TerraformShowResource struct {
	Address string                 `json:"address"`
	Type    string                 `json:"type"`
	Name    string                 `json:"name"`
	Index   int                    `json:"index"`
	Values  map[string]interface{} `json:"values"`
}

type ResourceChangeFilter struct {
	ResourceType string
	Action       string
	NotAction    string
}

type TerraformShowResourceChange struct {
	TerraformShowResource
	ModuleAddress string                  `json:"module_address"`
	Change        TerraformShowChangeSpec `json:"change"`
}

type TerraformShowChangeSpec struct {
	Actions []string `json:"actions"`
}

type TerraformShowModuleCall struct {
	Expressions map[string]*TerraformShowExpression `json:"expressions"`
	Module      *TerraformShowModule                `json:"module"`
}

type TerraformShowExpression struct {
	References []string `json:"references"`
}

type SimplifiedTerraformStatus struct {
	Modules     map[string]*SimplifiedTerraformModule
	Variables   map[string]string
	Connections []SimplifiedTerraformConnection
}

type SimplifiedTerraformModule struct {
	DisplayName string
	IsAnsible   bool
	Resources   []*SimplifiedTerraformResource
}

type SimplifiedTerraformResource struct {
	ResourceType  string
	DisplayName   string
	Plan          string
	State         string
	Progress      int
	ProgressTotal int
}

type SimplifiedTerraformConnection struct {
	From        string
	To          string
	DisplayName string
}

type AnsibleTaskResult struct {
	Name    string
	Success bool
	Skipped bool
	Mode    string
	Role    string
	Changed bool
}

type TerraformApplyResult struct {
	Error    error
	Success  bool
	Log      string
	Complete bool
	Status   *SimplifiedTerraformStatus
}

const data_template_file = "data.template_file"

func TerraformPlanAndApply(
	config *configuration.Configuration,
	workingDirectory string,
	terraformProject string,
) ([]byte, chan TerraformApplyResult, error) {
	terraformDirectory := filepath.Join(workingDirectory, terraformProject)

	// Under normal conditions you would not init every time. But I ran into some issues and decided doing this
	// every time was the best course of action.
	exitCode, initStdout, initStderr, err := shellExec(terraformDirectory, "terraform", "init")
	err = errorFromShellExecResult("terraform init", exitCode, initStdout, initStderr, err)
	if err != nil {
		return []byte{}, nil, err
	}

	// Convenience function so we can plan multiple times if needed
	doPlan := func(terraformDirectory string) (*SimplifiedTerraformStatus, *TerraformShow, string, error) {

		exitCode, planStdout, planStderr, err := shellExec(terraformDirectory, "terraform", "plan", "-out", configuration.TERRAFORM_PLAN_FILE_NAME)
		err = errorFromShellExecResult("terraform plan", exitCode, planStdout, planStderr, err)
		if err != nil {
			return nil, nil, "", err
		}

		exitCode, tfJson, showJsonStderr, err := shellExec(terraformDirectory, "terraform", "show", "-json", configuration.TERRAFORM_PLAN_FILE_NAME)
		err = errorFromShellExecResult("terraform show", exitCode, tfJson, showJsonStderr, err)
		if err != nil {
			return nil, nil, "", err
		}

		// log.Println("--------------------------")
		// log.Println(string(tfJson))
		// log.Println("--------------------------")

		var tfShow TerraformShow
		err = json.Unmarshal([]byte(tfJson), &tfShow)
		if err != nil {
			return nil, nil, "", errors.Wrap(err, "can't TerraformPlanAndApply because can't json.Unmarshal the output of `terraform show`")
		}

		// json, err := json.MarshalIndent(tfShow, "", "  ")
		// if err != nil {
		// 	return nil, nil, "", errors.Wrap(err, "can't GenerateTerraformPlan because can't json.Marshal the output of `terraform show`")
		// }
		// log.Println(string(json))
		// log.Println("--------------------------")

		// Copy some values over from PlannedValues to ResourceChanges for convenience access later
		changedResources := map[string]*TerraformShowResource{}
		for _, module := range tfShow.PlannedValues.RootModule.ChildModules {
			for _, resource := range module.Resources {
				changedResources[resource.Address] = resource
			}
		}
		for _, changedResource := range tfShow.ResourceChanges {
			resourceWithValues := changedResources[changedResource.Address]
			if resourceWithValues != nil {
				changedResource.Index = resourceWithValues.Index
				changedResource.Type = resourceWithValues.Type
				changedResource.Name = resourceWithValues.Name
				changedResource.Values = resourceWithValues.Values
			} else {
				// TODO this can happen when a resource is deleted ( like an extra ssh key being deleted )
				// for example:
				// "planned_values": { "root_module": { "child_modules": [
				// 		...,
				// 		{
				// 			"resources": [ { "address": "module.ssh-keys-digitalocean.digitalocean_ssh_key.default[0]", ... } ],
				// 			"address": "module.ssh-keys-digitalocean"
				// 		}
				//  ] } }
				// "resource_changes": [
				// 	...
				// 	{
				// 		"address": "module.ssh-keys-digitalocean.digitalocean_ssh_key.default[0]",
				// 		"change": { "actions": [ "no-op" ] }
				// 	},
				// 	{
				// 		"address": "module.ssh-keys-digitalocean.digitalocean_ssh_key.default[1]",
				// 		"change": { "actions": [ "delete" ] }
				// 	}
				// ]
				// do we care??
				log.Printf(
					"TerraformPlanAndApply(): doPlan(): found tfShow.ResourceChanges entry %s which is not in changedResources\n",
					changedResource.Address,
				)
			}
		}

		simpleStatus, err := makeSimplifiedTerraformStatus(config, workingDirectory, tfShow)
		if err != nil {
			return nil, nil, "", errors.Wrap(err, "can't TerraformPlanAndApply because can't makeSimplifiedTerraformStatus")
		}

		// json, err := json.MarshalIndent(simpleStatus, "", "  ")
		// if err != nil {
		//   return nil, errors.Wrap(err, "can't GenerateTerraformPlan because can't json.Marshal the simpleStatus")
		// }
		// log.Println(string(json))

		return &simpleStatus, &tfShow, string(planStdout), nil
	}

	simpleStatus, tfShow, planStdout, err := doPlan(terraformDirectory)
	if err != nil {
		return []byte{}, nil, errors.Wrap(err, "can't TerraformPlanAndApply because can't doPlan()")
	}

	// After plan but before apply, check over the plan and fix any known issues with cloud providers
	stateModified := false

	// DigitalOcean
	// TODO remove/import any orphaned server.garden tagged instances?
	mod, err := handleDigitalOceanSSHKeyAlreadyExists(config, workingDirectory, terraformDirectory, tfShow)
	if mod {
		stateModified = true
	}

	// Gandi.net
	//
	mod, err = handleGandiLiveDNSRecordAlreadyExists(config, workingDirectory, terraformDirectory, tfShow)
	if mod {
		stateModified = true
	}

	// After we try to fix any known issues, we may have to plan again if the terraform state was changed
	// for example, with terraform import or state commands.
	if stateModified {
		simpleStatus, tfShow, planStdout, err = doPlan(terraformDirectory)

		if err != nil {
			return []byte{}, nil, errors.Wrap(err, "can't TerraformPlanAndApply because can't doPlan()")
		}
	}

	svg, err := makeSVGFromSimpleStatus(simpleStatus)
	if err != nil {
		return []byte{}, nil, errors.Wrap(err, "can't TerraformPlanAndApply because can't makeSVGFromSimpleStatus")
	}

	for moduleName, module := range simpleStatus.Modules {
		if module.IsAnsible {
			err := linkAnsibleWrapperToModule(strings.TrimPrefix(moduleName, "module."), workingDirectory, terraformProject, true)
			if err != nil {
				return []byte{}, nil, errors.Wrap(err, "can't TerraformPlanAndApply because can't linkAnsibleWrapperToModule")
			}
		}
	}

	process := exec.Command("terraform", "apply", "-auto-approve", configuration.TERRAFORM_PLAN_FILE_NAME)
	process.Dir = terraformDirectory

	logLinesChannel := make(chan string)
	stdoutPipe, err := process.StdoutPipe()
	if err != nil {
		return []byte{}, nil, errors.Wrap(err, "can't TerraformPlanAndApply because can't process.StdoutPipe() terraform apply process")
	}
	stderrPipe, err := process.StderrPipe()
	if err != nil {
		return []byte{}, nil, errors.Wrap(err, "can't TerraformPlanAndApply because can't process.StderrPipe() terraform apply process")
	}

	err = process.Start()
	if err != nil {
		return []byte{}, nil, errors.Wrap(err, "can't TerraformPlanAndApply because can't process.Start() terraform apply process")
	}

	scanAllOutput := func(reader io.ReadCloser) {
		defer func() {
			if err := recover(); err != nil {
				log.Println("scanAllOutput panic:")
				log.Println(err)
			}
		}()
		scanner := bufio.NewScanner(reader)
		for scanner.Scan() {
			logLinesChannel <- scanner.Text()
		}
	}

	go scanAllOutput(stdoutPipe)
	go scanAllOutput(stderrPipe)

	toReturn := make(chan TerraformApplyResult)

	logSoFar := strings.Join(
		[]string{
			fmt.Sprintf("\n(%s) $ terraform init\n", terraformDirectory),
			string(initStdout),
			fmt.Sprintf("\n(%s) $ terraform plan -out %s\n", terraformDirectory, configuration.TERRAFORM_PLAN_FILE_NAME),
			planStdout,
			fmt.Sprintf("\n(%s) $ terraform apply -auto-approve %s\n", terraformDirectory, configuration.TERRAFORM_PLAN_FILE_NAME),
		},
		"\n",
	)

	go monitorTerraformApplyProgress(workingDirectory, terraformProject, simpleStatus, process, logSoFar, logLinesChannel, toReturn)

	return svg, toReturn, nil
}

func monitorTerraformApplyProgress(
	workingDirectory string,
	terraformProject string,
	simpleStatus *SimplifiedTerraformStatus,
	applyProcess *exec.Cmd,
	logSoFar string,
	logLinesChannel chan string,
	outputChannel chan TerraformApplyResult,
) {

	//ansibleModules := map[string]bool
	logLines := []string{}
	logLinesWithoutAnsiEscapes := []string{}
	terraformIsRunning := true
	terraformDirectory := filepath.Join(workingDirectory, terraformProject)
	ansibleModuleLevelErrors := map[string]bool{}
	ansibleModulesThatHaveAtLeastOneSuccessfulTask := map[string]bool{}

	// https://github.com/acarl005/stripansi/blob/master/stripansi.go
	ansiEscapeRegex := regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))")

	go (func() {
		terraformApplyLogRegex := regexp.MustCompile(`^(([a-zA-Z0-9_-]+\.)+[a-zA-Z0-9_-]+)(\[([0-9]+)\])?( \([a-zA-Z0-9_-]+\))?: (.*)$`)
		terraformExecErrorRegex := regexp.MustCompile(`Error running command '(.*)': exit status [0-9]+\. Output:.*$`)
		for logLine := range logLinesChannel {
			logLines = append(logLines, logLine)
			lineWithoutAnsiEscapes := ansiEscapeRegex.ReplaceAllString(logLine, "")
			logLinesWithoutAnsiEscapes = append(logLinesWithoutAnsiEscapes, lineWithoutAnsiEscapes)
			matches := terraformApplyLogRegex.FindStringSubmatch(lineWithoutAnsiEscapes)
			if matches != nil {
				address := matches[1]
				//index := matches[4]
				message := matches[6]

				// address = strings.Trim(address, ".")
				// address = regexp.MustCompile(`\[[0-9]+\]$`).ReplaceAllString(address, "")

				foundModule := false
				for moduleName, module := range simpleStatus.Modules {
					if strings.HasPrefix(address, moduleName) {
						foundModule = true
						if module.IsAnsible {
							// TODO test this strings.HasPrefix(message, "Executing:") instead
							//if strings.HasPrefix(message, "Creating...") || strings.HasPrefix(message, "Modifying...") {
							if strings.HasPrefix(message, "Executing:") {
								//ansibleModules[moduleName] = true

								//fmt.Printf("__INCLUDE_ANSIBLE__%s\n", module.DisplayName)
								logLines = append(logLines, fmt.Sprintf("__INCLUDE_ANSIBLE__%s", module.DisplayName))
							}
						} else {
							addressInsideModule := strings.TrimPrefix(address, moduleName)
							addressInsideModule = strings.Trim(addressInsideModule, ".")
							foundResource := false
							for _, resource := range module.Resources {
								if addressInsideModule == resource.DisplayName {
									foundResource = true
									// https://github.com/hashicorp/terraform/blob/5d0b75df7ae65e9f6a8314961c253ca1085c6534/command/hook_ui.go#L78
									// https://github.com/hashicorp/terraform/blob/5d0b75df7ae65e9f6a8314961c253ca1085c6534/command/hook_ui.go#L243
									if strings.HasPrefix(message, "Creating...") {
										resource.State = "creating"
									}
									if strings.HasPrefix(message, "Creation complete") {
										resource.State = "ok"
									}
									if strings.HasPrefix(message, "Modifying...") {
										resource.State = "modifying"
									}
									if strings.HasPrefix(message, "Modifications complete") {
										resource.State = "ok"
									}
									if strings.HasPrefix(message, "Destroying...") {
										resource.State = "destroying"
									}
									if strings.HasPrefix(message, "Destruction complete") {
										resource.State = "destroyed"
									}
									//fmt.Printf("non-ansible: %s %s %s %s\n", moduleName, resource.DisplayName, resource.State, message)
								}
							}
							if !foundResource {
								//fmt.Printf("!foundResource %s %s: %s\n", address, addressInsideModule, message)
							}
						}
					}
				}
				if !foundModule {
					//fmt.Printf("!foundModule %s: %s\n", address, message)
				}
			} else {
				//fmt.Printf("NOT PARSE %s\n", lineWithoutAnsiEscapes)

				// Check if there is a line which contains an exec error.
				// If there is such a line, it won't contain the module that failed, just the command that failed.
				// so we have to look back in the log to find the last time that command was mentioned, which should hit the
				// line which has the module / resource name on it.
				// So for example we could find a line:
				//
				// Error: Error running command './ansible-playbook-wrapper --private-key '/usr/lib/rootsystem/ssh/servergarden_builtin_ed22519' -i '167.71.175.207,' -u root -e 'domain=greenhouseusers.com arch=amd64' playbook.yml': exit status 4. Output:
				//
				// Then we look back in the log for any line which contains that command, and we find:
				//
				// module.ansible-threshold-server.null_resource.ansible_playbook[0] (local-exec): Executing: ["/bin/sh" "-c" "./ansible-playbook-wrapper --private-key '/usr/lib/rootsystem/ssh/servergarden_builtin_ed22519' -i '167.71.175.207,' -u root -e 'domain=greenhouseusers.com arch=amd64' playbook.yml"]
				//
				matches := terraformExecErrorRegex.FindStringSubmatch(lineWithoutAnsiEscapes)
				if matches != nil {
					commandThatFailed := matches[1]
					log.Printf("terraformExecErrorRegex matched %s", lineWithoutAnsiEscapes)
					log.Printf("terraformExecErrorRegex matched %s", commandThatFailed)
					previousLineContainingCommandThatFailed := ""
					for i, previousLine := range logLinesWithoutAnsiEscapes {
						if i < len(logLinesWithoutAnsiEscapes)-2 && strings.Contains(previousLine, commandThatFailed) {
							previousLineContainingCommandThatFailed = previousLine
						}
					}
					if previousLineContainingCommandThatFailed != "" {
						log.Printf("previousLineContainingCommandThatFailed: %s", previousLineContainingCommandThatFailed)
						matches = terraformApplyLogRegex.FindStringSubmatch(previousLineContainingCommandThatFailed)
						if matches != nil {
							address := matches[1]
							log.Printf("address of previousLineContainingCommandThatFailed: %s", address)
							for moduleName, module := range simpleStatus.Modules {
								if strings.HasPrefix(address, moduleName) {

									if module.IsAnsible {
										if !ansibleModulesThatHaveAtLeastOneSuccessfulTask[module.DisplayName] {
											log.Printf("setting ansible module %s resources to state error because ansible-playbook failed to start/connect", module.DisplayName)
											ansibleModuleLevelErrors[module.DisplayName] = true
											for _, resource := range module.Resources {
												resource.State = "error"
											}
										} else {
											log.Printf("at least one task succeeded in ansible module %s, skipping cuz ansible-playbook error handling will take care of it.", module.DisplayName)
										}

									} else {
										addressInsideModule := strings.TrimPrefix(address, moduleName)
										addressInsideModule = strings.Trim(addressInsideModule, ".")
										log.Printf("looking for module %s resource %s...", moduleName, addressInsideModule)
										for _, resource := range module.Resources {
											if addressInsideModule == resource.DisplayName {
												log.Printf("setting module %s resource %s to state error", moduleName, addressInsideModule)
												resource.State = "error"
											}
										}
									}
								}
							}
						}
					}
				}

			}

		}
	})()

	joinAnsibleLogsAndUpdateAnsibleStatus := func() string {
		processedLogLines := []string{}
		for _, line := range logLines {
			if strings.HasPrefix(line, "__INCLUDE_ANSIBLE__") {
				moduleName := strings.TrimPrefix(line, "__INCLUDE_ANSIBLE__")

				logBytes, err := ioutil.ReadFile(filepath.Join(terraformDirectory, "modules", moduleName, "ansible.log"))
				if err == nil {
					processedLogLines = append(processedLogLines, string(logBytes))
				} else {
					fmt.Printf("error trying to read ansible log: %s\n", err)
				}

				// if the entire ansible playbook errored out (like, it couldn't connect or something)
				// then the status of the individual roles has already been set to error, we don't want to overwrite that.
				// so just exit early
				if ansibleModuleLevelErrors[moduleName] {
					continue
				}
				jsonBytes, err := ioutil.ReadFile(filepath.Join(terraformDirectory, "modules", moduleName, "ansible-log.json"))
				if err == nil {
					var ansibleLog []AnsibleTaskResult
					//log.Printf("%s\n", string(jsonBytes))
					err = json.Unmarshal(jsonBytes, &ansibleLog)
					if err == nil {
						module, has := simpleStatus.Modules[fmt.Sprintf("module.%s", moduleName)]
						if has {
							ansibleRoles := map[string]int{}
							ansibleRoleErrors := map[string]int{}
							//ansibleRolesErrors := map[string]int{}
							for _, ansibleResult := range ansibleLog {
								if ansibleResult.Role == "" {
									continue
								}
								if ansibleResult.Success && !ansibleResult.Skipped {
									ansibleModulesThatHaveAtLeastOneSuccessfulTask[moduleName] = true
								}
								if ansibleResult.Success || ansibleResult.Skipped {
									ansibleRoles[ansibleResult.Role]++
								}
								if !ansibleResult.Success && !ansibleResult.Skipped {
									ansibleRoleErrors[ansibleResult.Role]++
								}
							}
							for _, resource := range module.Resources {
								resource.Progress = ansibleRoles[resource.DisplayName]
								if ansibleRoleErrors[resource.DisplayName] > 0 {
									resource.State = "error"
								} else if resource.Progress < resource.ProgressTotal {
									resource.State = "creating"
								} else {
									resource.State = "ok"
								}

								// fmt.Printf(
								//   "ansible %s %s %s (%d/%d)\n",
								//   fmt.Sprintf("module.%s", module.DisplayName), resource.DisplayName, resource.State, resource.Progress, resource.ProgressTotal,
								// )
							}
						} else {
							//fmt.Printf("module %s not found\n", fmt.Sprintf("module.%s", module))
						}
					}
				} else {
					//fmt.Printf("ansible-log.json: %s\n", err)
				}
			} else {
				processedLogLines = append(processedLogLines, line)
			}
		}

		return fmt.Sprintf("%s%s", logSoFar, strings.Join(processedLogLines, "\n"))
	}

	go (func() {
		for terraformIsRunning {
			time.Sleep(time.Second * configuration.TERRAFORM_APPLY_STATUS_UPDATE_INTERVAL_SECONDS)
			if terraformIsRunning {
				log := joinAnsibleLogsAndUpdateAnsibleStatus()
				outputChannel <- TerraformApplyResult{
					Error:  nil,
					Log:    log,
					Status: simpleStatus,
				}
			}
		}
	})()

	err := applyProcess.Wait()
	_, isExitError := err.(*exec.ExitError)
	if err != nil && !isExitError {
		err = errors.Wrap(err, "error waiting for terraform to finish running: ")
	} else {
		err = nil
	}

	// once upon a time I got an error
	// panic: send on closed channel @ terraformActions.go:204 (logLinesChannel <- blahblah; in scanAllOutput())
	// I assumed it was a race condition so I put this sleep here to try to fix it, lets see if it ever happens again...
	go (func() {
		time.Sleep(time.Millisecond * 100)
		close(logLinesChannel)
	})()

	terraformIsRunning = false
	terraformSuccess := true

	if err == nil && applyProcess.ProcessState.ExitCode() != 0 {
		// If the apply fails, its not really an exceptional case, we don't want to exit the application.
		terraformSuccess = false
		//err = errors.New(fmt.Sprintf("terraform apply failed: exit code %d", applyProcess.ProcessState.ExitCode()))
	}

	log := joinAnsibleLogsAndUpdateAnsibleStatus()

	outputChannel <- TerraformApplyResult{
		Error:    err,
		Complete: true,
		Success:  terraformSuccess,
		Log:      log,
		Status:   simpleStatus,
	}

	close(outputChannel)

	//  cleanUpSymlinksInTerraformProjects
	for moduleName, module := range simpleStatus.Modules {
		if module.IsAnsible {
			linkAnsibleWrapperToModule(strings.TrimPrefix(moduleName, "module."), workingDirectory, terraformProject, false)
			rolesFolder := filepath.Join(workingDirectory, configuration.TERRAFORM_MODULES, moduleName, "roles")
			os.Remove(rolesFolder)
		}
	}

}

func makeSimplifiedTerraformStatus(
	config *configuration.Configuration,
	workingDirectory string,
	tfShow TerraformShow,
) (SimplifiedTerraformStatus, error) {

	omitFromDiagram := []string{
		post_to_object_storage_shell_script,
		data_template_file,
	}

	shouldOmit := func(name string) bool {
		toReturn := false
		for _, omit := range omitFromDiagram {
			if strings.Contains(name, omit) {
				toReturn = true
			}
		}
		return toReturn
	}

	simpleModules := map[string]*SimplifiedTerraformModule{}
	for name, module := range tfShow.Configuraton.RootModule.ModuleCalls {
		if shouldOmit(name) {
			continue
		}

		resources := []*SimplifiedTerraformResource{}
		isAnsible := strings.HasPrefix(name, "ansible-")

		if isAnsible {
			rolesMap, err := getAnsibleRolesFromModule(name, workingDirectory)
			if err != nil {
				return SimplifiedTerraformStatus{}, errors.Wrapf(err, "cant getAnsibleRoles(%s) because", name)
			}
			for roleName, tasks := range rolesMap {
				if shouldOmit(roleName) {
					continue
				}
				resource := SimplifiedTerraformResource{
					ResourceType:  "ansible_role",
					DisplayName:   roleName,
					Plan:          "none",
					State:         "ok",
					Progress:      0,
					ProgressTotal: len(tasks),
				}
				resources = append(resources, &resource)
			}
		} else {
			for _, resource := range module.Module.Resources {
				if shouldOmit(resource.Address) {
					continue
				}
				resource := SimplifiedTerraformResource{
					ResourceType: resource.Type,
					DisplayName:  resource.Address,
					Plan:         "none",
					State:        "ok",
				}
				resources = append(resources, &resource)
			}
		}

		if len(resources) == 0 {
			resource := SimplifiedTerraformResource{
				DisplayName: "none",
			}
			resources = append(resources, &resource)
		}
		simpleModules[fmt.Sprintf("module.%s", name)] = &SimplifiedTerraformModule{
			DisplayName: name,
			IsAnsible:   isAnsible,
			Resources:   resources,
		}
	}

	resourceIndexRegexp := regexp.MustCompile(`\[([0-9]+)\]$`)
	for _, resourceChange := range tfShow.ResourceChanges {
		// resourceChange.Values should have come from the corresponding resource.
		// if there is no corresponding resource, I think it should be nil and that means it was
		// a deposed resource or something... lets omit it
		if resourceChange.Values != nil {
			continue
		}

		module, has := simpleModules[resourceChange.ModuleAddress]
		if has {
			address := strings.TrimPrefix(resourceChange.Address, resourceChange.ModuleAddress)
			address = strings.Trim(address, ".")

			// TODO it looks like the simplified terraform status wraps up repeated resources into one
			// and it takes the "Plan" and "State" from whichever was the last one
			// Is that ok? do we need to change that ?
			//indexMatches = resourceIndexRegexp.FindStringSubmatch(address)
			address = resourceIndexRegexp.ReplaceAllString(address, "")

			if shouldOmit(address) {
				continue
			}

			foundResource := false
			for _, resource := range module.Resources {
				if resource.DisplayName == address || module.IsAnsible {
					foundResource = true

					create := false
					delete := false
					update := false
					for _, action := range resourceChange.Change.Actions {

						if action == "create" {
							create = true
						}
						if action == "delete" {
							delete = true
						}
						if action == "update" {
							update = true
						}
					}
					if create && delete {
						resource.Plan = "recreate"
						resource.State = "tainted"
					} else if create {
						resource.Plan = "create"
						resource.State = "planned"
					} else if delete {
						resource.Plan = "delete"
						resource.State = "tainted"
					} else if update {
						resource.Plan = "update"
						resource.State = "tainted"
					}

					//fmt.Printf("init: %s %s %s %s\n", resourceChange.ModuleAddress, address, strings.Join(resourceChange.Change.Actions, ","), resource.State)
				}
			}
			if !foundResource {
				err := fmt.Errorf("no resource \"%s\" found in module \"%s\" when processing a resource change", address, resourceChange.ModuleAddress)
				return SimplifiedTerraformStatus{}, err
			}
		} else {
			err := fmt.Errorf("no module \"%s\" found when processing a resource change", resourceChange.ModuleAddress)
			return SimplifiedTerraformStatus{}, err
		}
	}

	simpleConnectionsMap := map[string]*SimplifiedTerraformConnection{}
	for moduleName, module := range tfShow.Configuraton.RootModule.ModuleCalls {
		if shouldOmit(moduleName) {
			continue
		}
		for expressionName, expression := range module.Expressions {
			if shouldOmit(expressionName) {
				continue
			}
			for _, reference := range expression.References {
				if shouldOmit(reference) {
					continue
				}
				refSlice := strings.Split(reference, ".")
				if len(refSlice) > 1 {
					reference = strings.Join(refSlice[:2], ".")
					simpleConnectionsMap[fmt.Sprintf("%s%s%s", reference, moduleName, expressionName)] = &SimplifiedTerraformConnection{
						From:        reference,
						To:          fmt.Sprintf("module.%s", moduleName),
						DisplayName: expressionName,
					}
				}
			}
		}
	}
	simpleConnections := []SimplifiedTerraformConnection{}
	for _, simpleConnection := range simpleConnectionsMap {
		simpleConnections = append(simpleConnections, *simpleConnection)
	}

	simpleVariables := map[string]string{}
	simpleVariables[ssh_public_keys] = "<secure shell public keys>"
	simpleVariables[ssh_private_key_filepath] = ssh_private_key_filepath_value
	for variableName, value := range config.Terraform.Variables {
		if shouldOmit(variableName) {
			continue
		}
		simpleVariables[variableName] = value
	}

	return SimplifiedTerraformStatus{
		Modules:     simpleModules,
		Connections: simpleConnections,
		Variables:   simpleVariables,
	}, nil
}

func linkAnsibleWrapperToModule(moduleName, workingDirectory, terraformProject string, createLink bool) error {
	ansibleDirectory := filepath.Join(workingDirectory, terraformProject, "modules", moduleName)

	for _, toLink := range configuration.GET_ANSIBLE_WRAPPER_FILES() {
		inModule := filepath.Join(ansibleDirectory, toLink)
		inAnsibleWrapper := filepath.Join(workingDirectory, configuration.ANSIBLE_WRAPPER_PATH, toLink)
		os.Remove(inModule)
		if createLink {
			err := os.Symlink(inAnsibleWrapper, inModule)
			if err != nil {
				return errors.Wrapf(err, "could not create symbolic link for ansible wrapper")
			}
		}

	}
	return nil
}

func getAnsibleRolesFromModule(moduleName, workingDirectory string) (map[string][]string, error) {
	ansibleDirectory := filepath.Join(workingDirectory, configuration.TERRAFORM_MODULES, moduleName)

	rolesFolder := filepath.Join(ansibleDirectory, "roles")

	os.Remove(rolesFolder)
	err := os.Symlink(
		filepath.Join(workingDirectory, configuration.ANSIBLE_ROLES),
		rolesFolder,
	)
	if err != nil {
		return nil, errors.Wrapf(err, "could not create symbolic link to ansible roles folder")
	}

	exitCode, ansibleStdout, ansibleStderr, err := shellExec(ansibleDirectory, "ansible-playbook", "--list-tasks", configuration.ANSIBLE_PLAYBOOK_FILE_NAME)
	err = errorFromShellExecResult("ansible-playbook --list-tasks", exitCode, ansibleStdout, ansibleStderr, err)
	if err != nil {
		return nil, err
	}

	// ansibleStdout looks like:
	// playbook: provision.yml
	//
	// play #1 (localhost): This is a hello-world example    TAGS: []
	//   tasks:
	//     test-role : Create a file called '/tmp/testfile.txt' with the content 'hello world'.        TAGS: []
	//     .....

	matchTask := regexp.MustCompile(`\n\s+(( *[^ :]+)+) : (.*?)\s*TAGS: \[.+`)
	taskMatches := matchTask.FindAllStringSubmatch(string(ansibleStdout), -1)
	ansibleRoles := map[string][]string{}

	for _, match := range taskMatches {
		role := match[1]
		task := match[3]
		if _, has := ansibleRoles[role]; !has {
			ansibleRoles[role] = []string{}
		}
		ansibleRoles[role] = append(ansibleRoles[role], task)
	}

	return ansibleRoles, nil
}

// TODO which user to run this command as ??
// Dont run as root. (this process probably runs as root)
func shellExec(workingDirectory string, executable string, arguments ...string) (int, []byte, []byte, error) {
	return shellExecInputPipe(workingDirectory, nil, executable, arguments...)
}

func shellExecInputPipe(workingDirectory string, input *string, executable string, arguments ...string) (int, []byte, []byte, error) {
	process := exec.Command(executable, arguments...)
	process.Dir = workingDirectory

	if input != nil {
		stdin, err := process.StdinPipe()
		if err != nil {
			return -1, []byte{}, []byte{}, errors.Wrap(err, "process.StdinPipe() returned")
		}

		go func() {
			defer stdin.Close()
			io.WriteString(stdin, *input)
		}()
	}

	var processStdoutBuffer, processStderrBuffer bytes.Buffer
	process.Stdout = &processStdoutBuffer
	process.Stderr = &processStderrBuffer
	err := process.Start()
	if err != nil {
		err = errors.Wrapf(err, "can't ShellExec(%s %s), process.Start() returned", executable, strings.Join(arguments, " "))
		return process.ProcessState.ExitCode(), []byte(""), []byte(""), err
	}

	err = process.Wait()
	if err != nil {
		err = errors.Wrapf(err, "can't ShellExec(%s %s), process.Wait() returned", executable, strings.Join(arguments, " "))
	}

	return process.ProcessState.ExitCode(), processStdoutBuffer.Bytes(), processStderrBuffer.Bytes(), err
}

func errorFromShellExecResult(command string, exitCode int, stdout []byte, stderr []byte, err error) error {
	if exitCode != 0 || err != nil {
		errorString := "nil"
		if err != nil {
			errorString = err.Error()
		}
		return fmt.Errorf(
			"%s failed with exit code %d, stdout: \n----\n%s\n----\nstderr: \n----\n%s\n----\nerror: %s",
			command, exitCode, stdout, stderr, errorString,
		)
	}
	return nil
}