ansible-playbook error handling is working

ansible-roles/threshold/tasks/main.yml

@@ -24,9 +24,11 @@
     path: /opt/threshold/threshold
     checksum_algorithm: sha256
   register: threshold_binary
+
 - name: log the checksum
   debug:
     var: threshold_binary.stat.checksum
+  when: threshold_binary.stat.checksum is defined
 
 - name: Download & validate the Threshold tar.gz file (if reinstall is desired)
   get_url:

ansible-wrapper/main.go

@@ -94,7 +94,7 @@ func main() {
 				err = writeLogLine(line)
 			}
 			if err != nil {
-				panic(err)
+				panic(fmt.Sprintf("%+v", err))
 			}
 		}
 	})()
@@ -103,14 +103,20 @@ func main() {
 		for stderrScanner.Scan() {
 			err := writeLogLine(stderrScanner.Text())
 			if err != nil {
-				panic(err)
+				panic(fmt.Sprintf("%+v", err))
 			}
 		}
 	})()
 
 	err = process.Wait()
-	if err != nil {
-		err = errors.Wrapf(err, "can't ShellExec(ansible-playbook %s), process.Wait() returned", strings.Join(arguments, " "))
+	_, isExitError := err.(*exec.ExitError)
+	if err != nil && !isExitError {
+		err = errors.Wrapf(err, "can't ShellExec(ansible-playbook %s), process.Wait() returned: ", strings.Join(arguments, " "))
+		panic(fmt.Sprintf("%+v", err))
+	}
+
+	if process.ProcessState.ExitCode() != 0 {
+		fmt.Fprintf(os.Stderr, "ansible-playbook process exited with error code %d", process.ProcessState.ExitCode())
 	}
 
 	os.Exit(process.ProcessState.ExitCode())

automation/terraformActions.go

@@ -343,6 +343,7 @@ func monitorTerraformApplyProgress(
 	terraformIsRunning := true
 	terraformDirectory := filepath.Join(workingDirectory, terraformProject)
 	ansibleModuleLevelErrors := map[string]bool{}
+	ansibleModulesThatHaveAtLeastOneSuccessfulTask := map[string]bool{}
 
 	// https://github.com/acarl005/stripansi/blob/master/stripansi.go
 	ansiEscapeRegex := regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))")
@@ -449,12 +450,18 @@ func monitorTerraformApplyProgress(
 							log.Printf("address of previousLineContainingCommandThatFailed: %s", address)
 							for moduleName, module := range simpleStatus.Modules {
 								if strings.HasPrefix(address, moduleName) {
+
 									if module.IsAnsible {
-										log.Printf("setting ansible module %s resources to state error", module.DisplayName)
-										ansibleModuleLevelErrors[module.DisplayName] = true
-										for _, resource := range module.Resources {
-											resource.State = "error"
+										if !ansibleModulesThatHaveAtLeastOneSuccessfulTask[module.DisplayName] {
+											log.Printf("setting ansible module %s resources to state error because ansible-playbook failed to start/connect", module.DisplayName)
+											ansibleModuleLevelErrors[module.DisplayName] = true
+											for _, resource := range module.Resources {
+												resource.State = "error"
+											}
+										} else {
+											log.Printf("at least one task succeeded in ansible module %s, skipping cuz ansible-playbook error handling will take care of it.", module.DisplayName)
 										}
+
 									} else {
 										addressInsideModule := strings.TrimPrefix(address, moduleName)
 										addressInsideModule = strings.Trim(addressInsideModule, ".")
@@ -481,9 +488,9 @@ func monitorTerraformApplyProgress(
 		processedLogLines := []string{}
 		for _, line := range logLines {
 			if strings.HasPrefix(line, "__INCLUDE_ANSIBLE__") {
-				module := strings.TrimPrefix(line, "__INCLUDE_ANSIBLE__")
+				moduleName := strings.TrimPrefix(line, "__INCLUDE_ANSIBLE__")
 
-				logBytes, err := ioutil.ReadFile(filepath.Join(terraformDirectory, "modules", module, "ansible.log"))
+				logBytes, err := ioutil.ReadFile(filepath.Join(terraformDirectory, "modules", moduleName, "ansible.log"))
 				if err == nil {
 					processedLogLines = append(processedLogLines, string(logBytes))
 				} else {
@@ -493,28 +500,39 @@ func monitorTerraformApplyProgress(
 				// if the entire ansible playbook errored out (like, it couldn't connect or something)
 				// then the status of the individual roles has already been set to error, we don't want to overwrite that.
 				// so just exit early
-				if ansibleModuleLevelErrors[module] {
+				if ansibleModuleLevelErrors[moduleName] {
 					continue
 				}
-				jsonBytes, err := ioutil.ReadFile(filepath.Join(terraformDirectory, "modules", module, "ansible-log.json"))
+				jsonBytes, err := ioutil.ReadFile(filepath.Join(terraformDirectory, "modules", moduleName, "ansible-log.json"))
 				if err == nil {
 					var ansibleLog []AnsibleTaskResult
 					//log.Printf("%s\n", string(jsonBytes))
 					err = json.Unmarshal(jsonBytes, &ansibleLog)
 					if err == nil {
-						module, has := simpleStatus.Modules[fmt.Sprintf("module.%s", module)]
+						module, has := simpleStatus.Modules[fmt.Sprintf("module.%s", moduleName)]
 						if has {
 							ansibleRoles := map[string]int{}
+							ansibleRoleErrors := map[string]int{}
 							//ansibleRolesErrors := map[string]int{}
 							for _, ansibleResult := range ansibleLog {
-								//if ansibleResult.
-								if ansibleResult.Role != "" && ansibleResult.Success {
+								if ansibleResult.Role == "" {
+									continue
+								}
+								if ansibleResult.Success && !ansibleResult.Skipped {
+									ansibleModulesThatHaveAtLeastOneSuccessfulTask[moduleName] = true
+								}
+								if ansibleResult.Success || ansibleResult.Skipped {
 									ansibleRoles[ansibleResult.Role]++
 								}
+								if !ansibleResult.Success && !ansibleResult.Skipped {
+									ansibleRoleErrors[ansibleResult.Role]++
+								}
 							}
 							for _, resource := range module.Resources {
 								resource.Progress = ansibleRoles[resource.DisplayName]
-								if resource.Progress < resource.ProgressTotal {
+								if ansibleRoleErrors[resource.DisplayName] > 0 {
+									resource.State = "error"
+								} else if resource.Progress < resource.ProgressTotal {
 									resource.State = "creating"
 								} else {
 									resource.State = "ok"

main.go

@@ -148,11 +148,11 @@ func terraformBuild(
 			return []string{}, false, err
 		}
 
-		newLog := strings.TrimPrefix(lastLog, status.Log)
+		newLog := strings.TrimPrefix(status.Log, lastLog)
 		lastLog = status.Log
 		log.Println(newLog)
 
-		log.Println(string(json))
+		// log.Println(string(json))
 
 		err = global.storage.Put(fmt.Sprintf("rootsystem/terraform/%s/status.json", terraformConfig.TerraformProject), json)
 		if err != nil {