From 8b1ff6386e667a0d7a49db53ea086870469dd3d3 Mon Sep 17 00:00:00 2001 From: forest Date: Tue, 22 Sep 2020 13:32:03 -0500 Subject: [PATCH] ansible-playbook error handling is working --- ansible-roles/threshold/tasks/main.yml | 2 ++ ansible-wrapper/main.go | 14 ++++++--- automation/terraformActions.go | 42 ++++++++++++++++++-------- main.go | 4 +-- 4 files changed, 44 insertions(+), 18 deletions(-) diff --git a/ansible-roles/threshold/tasks/main.yml b/ansible-roles/threshold/tasks/main.yml index 1dc7d72..912fe7d 100644 --- a/ansible-roles/threshold/tasks/main.yml +++ b/ansible-roles/threshold/tasks/main.yml @@ -24,9 +24,11 @@ path: /opt/threshold/threshold checksum_algorithm: sha256 register: threshold_binary + - name: log the checksum debug: var: threshold_binary.stat.checksum + when: threshold_binary.stat.checksum is defined - name: Download & validate the Threshold tar.gz file (if reinstall is desired) get_url: diff --git a/ansible-wrapper/main.go b/ansible-wrapper/main.go index f6e324b..1a90a96 100644 --- a/ansible-wrapper/main.go +++ b/ansible-wrapper/main.go @@ -94,7 +94,7 @@ func main() { err = writeLogLine(line) } if err != nil { - panic(err) + panic(fmt.Sprintf("%+v", err)) } } })() @@ -103,14 +103,20 @@ func main() { for stderrScanner.Scan() { err := writeLogLine(stderrScanner.Text()) if err != nil { - panic(err) + panic(fmt.Sprintf("%+v", err)) } } })() err = process.Wait() - if err != nil { - err = errors.Wrapf(err, "can't ShellExec(ansible-playbook %s), process.Wait() returned", strings.Join(arguments, " ")) + _, isExitError := err.(*exec.ExitError) + if err != nil && !isExitError { + err = errors.Wrapf(err, "can't ShellExec(ansible-playbook %s), process.Wait() returned: ", strings.Join(arguments, " ")) + panic(fmt.Sprintf("%+v", err)) + } + + if process.ProcessState.ExitCode() != 0 { + fmt.Fprintf(os.Stderr, "ansible-playbook process exited with error code %d", process.ProcessState.ExitCode()) } os.Exit(process.ProcessState.ExitCode()) diff --git a/automation/terraformActions.go b/automation/terraformActions.go index 993c061..8df90de 100644 --- a/automation/terraformActions.go +++ b/automation/terraformActions.go @@ -343,6 +343,7 @@ func monitorTerraformApplyProgress( terraformIsRunning := true terraformDirectory := filepath.Join(workingDirectory, terraformProject) ansibleModuleLevelErrors := map[string]bool{} + ansibleModulesThatHaveAtLeastOneSuccessfulTask := map[string]bool{} // https://github.com/acarl005/stripansi/blob/master/stripansi.go ansiEscapeRegex := regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))") @@ -449,12 +450,18 @@ func monitorTerraformApplyProgress( log.Printf("address of previousLineContainingCommandThatFailed: %s", address) for moduleName, module := range simpleStatus.Modules { if strings.HasPrefix(address, moduleName) { + if module.IsAnsible { - log.Printf("setting ansible module %s resources to state error", module.DisplayName) - ansibleModuleLevelErrors[module.DisplayName] = true - for _, resource := range module.Resources { - resource.State = "error" + if !ansibleModulesThatHaveAtLeastOneSuccessfulTask[module.DisplayName] { + log.Printf("setting ansible module %s resources to state error because ansible-playbook failed to start/connect", module.DisplayName) + ansibleModuleLevelErrors[module.DisplayName] = true + for _, resource := range module.Resources { + resource.State = "error" + } + } else { + log.Printf("at least one task succeeded in ansible module %s, skipping cuz ansible-playbook error handling will take care of it.", module.DisplayName) } + } else { addressInsideModule := strings.TrimPrefix(address, moduleName) addressInsideModule = strings.Trim(addressInsideModule, ".") @@ -481,9 +488,9 @@ func monitorTerraformApplyProgress( processedLogLines := []string{} for _, line := range logLines { if strings.HasPrefix(line, "__INCLUDE_ANSIBLE__") { - module := strings.TrimPrefix(line, "__INCLUDE_ANSIBLE__") + moduleName := strings.TrimPrefix(line, "__INCLUDE_ANSIBLE__") - logBytes, err := ioutil.ReadFile(filepath.Join(terraformDirectory, "modules", module, "ansible.log")) + logBytes, err := ioutil.ReadFile(filepath.Join(terraformDirectory, "modules", moduleName, "ansible.log")) if err == nil { processedLogLines = append(processedLogLines, string(logBytes)) } else { @@ -493,28 +500,39 @@ func monitorTerraformApplyProgress( // if the entire ansible playbook errored out (like, it couldn't connect or something) // then the status of the individual roles has already been set to error, we don't want to overwrite that. // so just exit early - if ansibleModuleLevelErrors[module] { + if ansibleModuleLevelErrors[moduleName] { continue } - jsonBytes, err := ioutil.ReadFile(filepath.Join(terraformDirectory, "modules", module, "ansible-log.json")) + jsonBytes, err := ioutil.ReadFile(filepath.Join(terraformDirectory, "modules", moduleName, "ansible-log.json")) if err == nil { var ansibleLog []AnsibleTaskResult //log.Printf("%s\n", string(jsonBytes)) err = json.Unmarshal(jsonBytes, &ansibleLog) if err == nil { - module, has := simpleStatus.Modules[fmt.Sprintf("module.%s", module)] + module, has := simpleStatus.Modules[fmt.Sprintf("module.%s", moduleName)] if has { ansibleRoles := map[string]int{} + ansibleRoleErrors := map[string]int{} //ansibleRolesErrors := map[string]int{} for _, ansibleResult := range ansibleLog { - //if ansibleResult. - if ansibleResult.Role != "" && ansibleResult.Success { + if ansibleResult.Role == "" { + continue + } + if ansibleResult.Success && !ansibleResult.Skipped { + ansibleModulesThatHaveAtLeastOneSuccessfulTask[moduleName] = true + } + if ansibleResult.Success || ansibleResult.Skipped { ansibleRoles[ansibleResult.Role]++ } + if !ansibleResult.Success && !ansibleResult.Skipped { + ansibleRoleErrors[ansibleResult.Role]++ + } } for _, resource := range module.Resources { resource.Progress = ansibleRoles[resource.DisplayName] - if resource.Progress < resource.ProgressTotal { + if ansibleRoleErrors[resource.DisplayName] > 0 { + resource.State = "error" + } else if resource.Progress < resource.ProgressTotal { resource.State = "creating" } else { resource.State = "ok" diff --git a/main.go b/main.go index 0e54b8b..3101c98 100644 --- a/main.go +++ b/main.go @@ -148,11 +148,11 @@ func terraformBuild( return []string{}, false, err } - newLog := strings.TrimPrefix(lastLog, status.Log) + newLog := strings.TrimPrefix(status.Log, lastLog) lastLog = status.Log log.Println(newLog) - log.Println(string(json)) + // log.Println(string(json)) err = global.storage.Put(fmt.Sprintf("rootsystem/terraform/%s/status.json", terraformConfig.TerraformProject), json) if err != nil {