Browse Source

ansible-playbook error handling is working

master
forest 2 years ago
parent
commit
8b1ff6386e
  1. 2
      ansible-roles/threshold/tasks/main.yml
  2. 14
      ansible-wrapper/main.go
  3. 42
      automation/terraformActions.go
  4. 4
      main.go

2
ansible-roles/threshold/tasks/main.yml

@ -24,9 +24,11 @@
path: /opt/threshold/threshold
checksum_algorithm: sha256
register: threshold_binary
- name: log the checksum
debug:
var: threshold_binary.stat.checksum
when: threshold_binary.stat.checksum is defined
- name: Download & validate the Threshold tar.gz file (if reinstall is desired)
get_url:

14
ansible-wrapper/main.go

@ -94,7 +94,7 @@ func main() {
err = writeLogLine(line)
}
if err != nil {
panic(err)
panic(fmt.Sprintf("%+v", err))
}
}
})()
@ -103,14 +103,20 @@ func main() {
for stderrScanner.Scan() {
err := writeLogLine(stderrScanner.Text())
if err != nil {
panic(err)
panic(fmt.Sprintf("%+v", err))
}
}
})()
err = process.Wait()
if err != nil {
err = errors.Wrapf(err, "can't ShellExec(ansible-playbook %s), process.Wait() returned", strings.Join(arguments, " "))
_, isExitError := err.(*exec.ExitError)
if err != nil && !isExitError {
err = errors.Wrapf(err, "can't ShellExec(ansible-playbook %s), process.Wait() returned: ", strings.Join(arguments, " "))
panic(fmt.Sprintf("%+v", err))
}
if process.ProcessState.ExitCode() != 0 {
fmt.Fprintf(os.Stderr, "ansible-playbook process exited with error code %d", process.ProcessState.ExitCode())
}
os.Exit(process.ProcessState.ExitCode())

42
automation/terraformActions.go

@ -343,6 +343,7 @@ func monitorTerraformApplyProgress(
terraformIsRunning := true
terraformDirectory := filepath.Join(workingDirectory, terraformProject)
ansibleModuleLevelErrors := map[string]bool{}
ansibleModulesThatHaveAtLeastOneSuccessfulTask := map[string]bool{}
// https://github.com/acarl005/stripansi/blob/master/stripansi.go
ansiEscapeRegex := regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))")
@ -449,12 +450,18 @@ func monitorTerraformApplyProgress(
log.Printf("address of previousLineContainingCommandThatFailed: %s", address)
for moduleName, module := range simpleStatus.Modules {
if strings.HasPrefix(address, moduleName) {
if module.IsAnsible {
log.Printf("setting ansible module %s resources to state error", module.DisplayName)
ansibleModuleLevelErrors[module.DisplayName] = true
for _, resource := range module.Resources {
resource.State = "error"
if !ansibleModulesThatHaveAtLeastOneSuccessfulTask[module.DisplayName] {
log.Printf("setting ansible module %s resources to state error because ansible-playbook failed to start/connect", module.DisplayName)
ansibleModuleLevelErrors[module.DisplayName] = true
for _, resource := range module.Resources {
resource.State = "error"
}
} else {
log.Printf("at least one task succeeded in ansible module %s, skipping cuz ansible-playbook error handling will take care of it.", module.DisplayName)
}
} else {
addressInsideModule := strings.TrimPrefix(address, moduleName)
addressInsideModule = strings.Trim(addressInsideModule, ".")
@ -481,9 +488,9 @@ func monitorTerraformApplyProgress(
processedLogLines := []string{}
for _, line := range logLines {
if strings.HasPrefix(line, "__INCLUDE_ANSIBLE__") {
module := strings.TrimPrefix(line, "__INCLUDE_ANSIBLE__")
moduleName := strings.TrimPrefix(line, "__INCLUDE_ANSIBLE__")
logBytes, err := ioutil.ReadFile(filepath.Join(terraformDirectory, "modules", module, "ansible.log"))
logBytes, err := ioutil.ReadFile(filepath.Join(terraformDirectory, "modules", moduleName, "ansible.log"))
if err == nil {
processedLogLines = append(processedLogLines, string(logBytes))
} else {
@ -493,28 +500,39 @@ func monitorTerraformApplyProgress(
// if the entire ansible playbook errored out (like, it couldn't connect or something)
// then the status of the individual roles has already been set to error, we don't want to overwrite that.
// so just exit early
if ansibleModuleLevelErrors[module] {
if ansibleModuleLevelErrors[moduleName] {
continue
}
jsonBytes, err := ioutil.ReadFile(filepath.Join(terraformDirectory, "modules", module, "ansible-log.json"))
jsonBytes, err := ioutil.ReadFile(filepath.Join(terraformDirectory, "modules", moduleName, "ansible-log.json"))
if err == nil {
var ansibleLog []AnsibleTaskResult
//log.Printf("%s\n", string(jsonBytes))
err = json.Unmarshal(jsonBytes, &ansibleLog)
if err == nil {
module, has := simpleStatus.Modules[fmt.Sprintf("module.%s", module)]
module, has := simpleStatus.Modules[fmt.Sprintf("module.%s", moduleName)]
if has {
ansibleRoles := map[string]int{}
ansibleRoleErrors := map[string]int{}
//ansibleRolesErrors := map[string]int{}
for _, ansibleResult := range ansibleLog {
//if ansibleResult.
if ansibleResult.Role != "" && ansibleResult.Success {
if ansibleResult.Role == "" {
continue
}
if ansibleResult.Success && !ansibleResult.Skipped {
ansibleModulesThatHaveAtLeastOneSuccessfulTask[moduleName] = true
}
if ansibleResult.Success || ansibleResult.Skipped {
ansibleRoles[ansibleResult.Role]++
}
if !ansibleResult.Success && !ansibleResult.Skipped {
ansibleRoleErrors[ansibleResult.Role]++
}
}
for _, resource := range module.Resources {
resource.Progress = ansibleRoles[resource.DisplayName]
if resource.Progress < resource.ProgressTotal {
if ansibleRoleErrors[resource.DisplayName] > 0 {
resource.State = "error"
} else if resource.Progress < resource.ProgressTotal {
resource.State = "creating"
} else {
resource.State = "ok"

4
main.go

@ -148,11 +148,11 @@ func terraformBuild(
return []string{}, false, err
}
newLog := strings.TrimPrefix(lastLog, status.Log)
newLog := strings.TrimPrefix(status.Log, lastLog)
lastLog = status.Log
log.Println(newLog)
log.Println(string(json))
// log.Println(string(json))
err = global.storage.Put(fmt.Sprintf("rootsystem/terraform/%s/status.json", terraformConfig.TerraformProject), json)
if err != nil {

Loading…
Cancel
Save