Browse Source

I just realized that gofmt forces tabs instead of spaces... wow

https://github.com/golang/go/issues/7101

> cmd/gofmt: remove -tabs and -tabwidth flags #7101

https://stackoverflow.blog/2017/06/15/developers-use-spaces-make-money-use-tabs/

> Developers Who Use Spaces Make More Money Than Those Who Use Tabs

googles trying to take all our money guys
master
forest 2 years ago
parent
commit
2b035a0750
  1. 214
      ansible-wrapper/main.go
  2. 196
      automation/patchDigitalOcean.go
  3. 270
      automation/patchGandi.go
  4. 124
      automation/patchHelpers.go
  5. 52
      automation/terraformActions.go
  6. 1270
      automation/terraformCodeGeneration.go
  7. 60
      automation/terraformSvg.go
  8. 218
      configuration/configuration.go
  9. 236
      host-key-poller/main.go
  10. 460
      lock.go
  11. 1076
      objectStorage/backblazeb2.go
  12. 476
      objectStorage/e2eeObjectStorage.go
  13. 240
      objectStorage/initialization.go
  14. 48
      objectStorage/objectStorager.go
  15. 390
      objectStorage/redundantObjectStorage.go
  16. 314
      objectStorage/s3Compatible.go
  17. 574
      pki/pki.go
  18. 110
      terraformStateHandler.go

214
ansible-wrapper/main.go

@ -1,117 +1,117 @@
package main
import (
"bufio"
"encoding/json"
"fmt"
"os"
"os/exec"
"strings"
errors "git.sequentialread.com/forest/pkg-errors"
"bufio"
"encoding/json"
"fmt"
"os"
"os/exec"
"strings"
errors "git.sequentialread.com/forest/pkg-errors"
)
type AnsibleTaskResult struct {
Name string
Success bool
Skipped bool
Mode string
Role string
Changed bool
Name string
Success bool
Skipped bool
Mode string
Role string
Changed bool
}
func main() {
arguments := os.Args[1:]
process := exec.Command("ansible-playbook", arguments...)
stdoutPipe, err := process.StdoutPipe()
if err != nil {
fmt.Fprint(os.Stderr, "can't connect stdout pipe to ansible-playbook process")
os.Exit(1)
}
stderrPipe, err := process.StderrPipe()
if err != nil {
fmt.Fprint(os.Stderr, "can't connect stderr pipe to ansible-playbook process")
os.Exit(1)
}
err = process.Start()
if err != nil {
err = errors.Wrapf(err, "can't ShellExec(ansible-playbook %s), process.Start() returned", strings.Join(arguments, " "))
os.Exit(1)
}
stdoutScanner := bufio.NewScanner(stdoutPipe)
stdoutScanner.Split(bufio.ScanLines)
stderrScanner := bufio.NewScanner(stderrPipe)
stderrScanner.Split(bufio.ScanLines)
_ = os.Remove("ansible.log")
_ = os.Remove("ansible-log.json")
ansibleTaskResults := []AnsibleTaskResult{}
writeLogLine := func(text string) error {
file, err := os.OpenFile("ansible.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
return err
}
defer file.Close()
if _, err = file.WriteString(fmt.Sprintln(text)); err != nil {
return err
}
return nil
}
writeTaskResult := func(taskResult AnsibleTaskResult) error {
ansibleTaskResults = append(ansibleTaskResults, taskResult)
file, err := os.OpenFile("ansible-log.json", os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
return err
}
defer file.Close()
jsonBytes, err := json.MarshalIndent(ansibleTaskResults, "", " ")
if err != nil {
return err
}
_, err = file.Write(jsonBytes)
if err != nil {
return err
}
return nil
}
go (func() {
for stdoutScanner.Scan() {
line := stdoutScanner.Text()
var taskResult AnsibleTaskResult
err := json.Unmarshal([]byte(line), &taskResult)
if err == nil && taskResult.Name != "" {
err = writeTaskResult(taskResult)
} else {
err = writeLogLine(fmt.Sprintf("stdout: %s", line))
}
if err != nil {
panic(err)
}
}
})()
go (func() {
for stderrScanner.Scan() {
err := writeLogLine(fmt.Sprintf("stderr: %s", stderrScanner.Text()))
if err != nil {
panic(err)
}
}
})()
err = process.Wait()
if err != nil {
err = errors.Wrapf(err, "can't ShellExec(ansible-playbook %s), process.Wait() returned", strings.Join(arguments, " "))
}
os.Exit(process.ProcessState.ExitCode())
arguments := os.Args[1:]
process := exec.Command("ansible-playbook", arguments...)
stdoutPipe, err := process.StdoutPipe()
if err != nil {
fmt.Fprint(os.Stderr, "can't connect stdout pipe to ansible-playbook process")
os.Exit(1)
}
stderrPipe, err := process.StderrPipe()
if err != nil {
fmt.Fprint(os.Stderr, "can't connect stderr pipe to ansible-playbook process")
os.Exit(1)
}
err = process.Start()
if err != nil {
err = errors.Wrapf(err, "can't ShellExec(ansible-playbook %s), process.Start() returned", strings.Join(arguments, " "))
os.Exit(1)
}
stdoutScanner := bufio.NewScanner(stdoutPipe)
stdoutScanner.Split(bufio.ScanLines)
stderrScanner := bufio.NewScanner(stderrPipe)
stderrScanner.Split(bufio.ScanLines)
_ = os.Remove("ansible.log")
_ = os.Remove("ansible-log.json")
ansibleTaskResults := []AnsibleTaskResult{}
writeLogLine := func(text string) error {
file, err := os.OpenFile("ansible.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
return err
}
defer file.Close()
if _, err = file.WriteString(fmt.Sprintln(text)); err != nil {
return err
}
return nil
}
writeTaskResult := func(taskResult AnsibleTaskResult) error {
ansibleTaskResults = append(ansibleTaskResults, taskResult)
file, err := os.OpenFile("ansible-log.json", os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
return err
}
defer file.Close()
jsonBytes, err := json.MarshalIndent(ansibleTaskResults, "", " ")
if err != nil {
return err
}
_, err = file.Write(jsonBytes)
if err != nil {
return err
}
return nil
}
go (func() {
for stdoutScanner.Scan() {
line := stdoutScanner.Text()
var taskResult AnsibleTaskResult
err := json.Unmarshal([]byte(line), &taskResult)
if err == nil && taskResult.Name != "" {
err = writeTaskResult(taskResult)
} else {
err = writeLogLine(line)
}
if err != nil {
panic(err)
}
}
})()
go (func() {
for stderrScanner.Scan() {
err := writeLogLine(stderrScanner.Text())
if err != nil {
panic(err)
}
}
})()
err = process.Wait()
if err != nil {
err = errors.Wrapf(err, "can't ShellExec(ansible-playbook %s), process.Wait() returned", strings.Join(arguments, " "))
}
os.Exit(process.ProcessState.ExitCode())
}

196
automation/patchDigitalOcean.go

@ -1,113 +1,113 @@
package automation
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strconv"
"strings"
errors "git.sequentialread.com/forest/pkg-errors"
"git.sequentialread.com/forest/rootsystem/configuration"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strconv"
"strings"
errors "git.sequentialread.com/forest/pkg-errors"
"git.sequentialread.com/forest/rootsystem/configuration"
)
type digialOceanSSHKeysResponse struct {
SSHKeys []digitalOceanSSHKey `json:"ssh_keys"`
SSHKeys []digitalOceanSSHKey `json:"ssh_keys"`
}
type digitalOceanSSHKey struct {
Id int `json:"id"`
Fingerprint string `json:"fingerprint"`
PublicKey string `json:"public_key"`
Pame string `json:"name"`
Id int `json:"id"`
Fingerprint string `json:"fingerprint"`
PublicKey string `json:"public_key"`
Pame string `json:"name"`
}
func handleDigitalOceanSSHKeyAlreadyExists(
config *configuration.Configuration,
workingDirectory string,
terraformDirectory string,
tfShow *TerraformShow,
config *configuration.Configuration,
workingDirectory string,
terraformDirectory string,
tfShow *TerraformShow,
) (bool, error) {
var digitaloceanCredential *configuration.Credential = nil
for _, cred := range config.Credentials {
if cred.Type == configuration.DIGITALOCEAN {
digitaloceanCredential = &cred
break
}
}
createSSHKeys := filterResourceChanges(
tfShow,
ResourceChangeFilter{ResourceType: "digitalocean_ssh_key", Action: "create", NotAction: "delete"},
)
if digitaloceanCredential == nil || len(createSSHKeys) == 0 {
return false, nil
}
// createAnySSHKeys
// for _, module := range tfShow.PlannedValues.RootModule.ChildModules {
// for _, resource := range module.Resources {
// if resource.Type == "digitalocean_ssh_key" {
// }
// }
// }
// Get the ssh public keys from digital ocean API
httpClient := http.Client{}
request, err := http.NewRequest("GET", fmt.Sprintf("%s%s", configuration.DIGITALOCEAN_API_URL, "/v2/account/keys"), nil)
request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", digitaloceanCredential.Password))
response, err := httpClient.Do(request)
if err != nil {
return false, err
}
if response.StatusCode != 200 {
return false, fmt.Errorf("HTTP %d when calling /v2/account/keys on digitalocean API", response.StatusCode)
}
bytes, err := ioutil.ReadAll(response.Body)
if err != nil {
return false, errors.Wrap(err, "HTTP read error when calling /v2/account/keys on digitalocean API")
}
var responseObject digialOceanSSHKeysResponse
err = json.Unmarshal(bytes, &responseObject)
if err != nil {
return false, errors.Wrap(err, "JSON parse error when calling /v2/account/keys on digitalocean API")
}
digitalOceanIdByKeyContent := map[string]int{}
for _, sshKey := range responseObject.SSHKeys {
digitalOceanIdByKeyContent[strings.TrimSpace(sshKey.PublicKey)] = sshKey.Id
}
importedAnySSHKeys := false
// for each local SSH key which tf plans to create, check if it already exists in DO
for _, resource := range createSSHKeys {
sshKeyContent, hasContent := getStringValue(resource, "public_key")
if !hasContent {
return false, fmt.Errorf("digitalocean_ssh_key %s (from terraform planned_values) is missing string value \"public_key\" ", resource.Address)
}
digitalOceanId, has := digitalOceanIdByKeyContent[strings.TrimSpace(sshKeyContent)]
if has {
fmt.Println("---------------------------------------")
fmt.Printf(
"Terraform plans to add the local ssh public key %s to digitalocean, but digitalocean already has that key \n",
resource.Values["name"],
)
exitCode := removeAndImportResource(terraformDirectory, resource.Address, strconv.Itoa(digitalOceanId))
if exitCode != 0 {
return false, errors.New("terraform import returned a non-zero exit code")
}
importedAnySSHKeys = true
fmt.Println("---------------------------------------")
}
}
return importedAnySSHKeys, nil
var digitaloceanCredential *configuration.Credential = nil
for _, cred := range config.Credentials {
if cred.Type == configuration.DIGITALOCEAN {
digitaloceanCredential = &cred
break
}
}
createSSHKeys := filterResourceChanges(
tfShow,
ResourceChangeFilter{ResourceType: "digitalocean_ssh_key", Action: "create", NotAction: "delete"},
)
if digitaloceanCredential == nil || len(createSSHKeys) == 0 {
return false, nil
}
// createAnySSHKeys
// for _, module := range tfShow.PlannedValues.RootModule.ChildModules {
// for _, resource := range module.Resources {
// if resource.Type == "digitalocean_ssh_key" {
// }
// }
// }
// Get the ssh public keys from digital ocean API
httpClient := http.Client{}
request, err := http.NewRequest("GET", fmt.Sprintf("%s%s", configuration.DIGITALOCEAN_API_URL, "/v2/account/keys"), nil)
request.Header.Add("Authorization", fmt.Sprintf("Bearer %s", digitaloceanCredential.Password))
response, err := httpClient.Do(request)
if err != nil {
return false, err
}
if response.StatusCode != 200 {
return false, fmt.Errorf("HTTP %d when calling /v2/account/keys on digitalocean API", response.StatusCode)
}
bytes, err := ioutil.ReadAll(response.Body)
if err != nil {
return false, errors.Wrap(err, "HTTP read error when calling /v2/account/keys on digitalocean API")
}
var responseObject digialOceanSSHKeysResponse
err = json.Unmarshal(bytes, &responseObject)
if err != nil {
return false, errors.Wrap(err, "JSON parse error when calling /v2/account/keys on digitalocean API")
}
digitalOceanIdByKeyContent := map[string]int{}
for _, sshKey := range responseObject.SSHKeys {
digitalOceanIdByKeyContent[strings.TrimSpace(sshKey.PublicKey)] = sshKey.Id
}
importedAnySSHKeys := false
// for each local SSH key which tf plans to create, check if it already exists in DO
for _, resource := range createSSHKeys {
sshKeyContent, hasContent := getStringValue(resource, "public_key")
if !hasContent {
return false, fmt.Errorf("digitalocean_ssh_key %s (from terraform planned_values) is missing string value \"public_key\" ", resource.Address)
}
digitalOceanId, has := digitalOceanIdByKeyContent[strings.TrimSpace(sshKeyContent)]
if has {
fmt.Println("---------------------------------------")
fmt.Printf(
"Terraform plans to add the local ssh public key %s to digitalocean, but digitalocean already has that key \n",
resource.Values["name"],
)
exitCode := removeAndImportResource(terraformDirectory, resource.Address, strconv.Itoa(digitalOceanId))
if exitCode != 0 {
return false, errors.New("terraform import returned a non-zero exit code")
}
importedAnySSHKeys = true
fmt.Println("---------------------------------------")
}
}
return importedAnySSHKeys, nil
}

270
automation/patchGandi.go

@ -1,149 +1,149 @@
package automation
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
errors "git.sequentialread.com/forest/pkg-errors"
"git.sequentialread.com/forest/rootsystem/configuration"
errors "git.sequentialread.com/forest/pkg-errors"
"git.sequentialread.com/forest/rootsystem/configuration"
)
// https://api.gandi.net/docs/livedns/
type gandiLiveDNSRecord struct {
Name string `json:"rrset_name"`
Type string `json:"rrset_type"`
Values []string `json:"rrset_values"`
Name string `json:"rrset_name"`
Type string `json:"rrset_type"`
Values []string `json:"rrset_values"`
}
func handleGandiLiveDNSRecordAlreadyExists(
config *configuration.Configuration,
workingDirectory string,
terraformDirectory string,
tfShow *TerraformShow,
config *configuration.Configuration,
workingDirectory string,
terraformDirectory string,
tfShow *TerraformShow,
) (bool, error) {
var gandiCredential *configuration.Credential = nil
for _, cred := range config.Credentials {
if cred.Type == configuration.GANDI {
gandiCredential = &cred
break
}
}
createLiveDNSRecords := filterResourceChanges(
tfShow,
ResourceChangeFilter{ResourceType: "gandi_livedns_record", Action: "create", NotAction: "delete"},
)
domainName, hasDomainName := config.Terraform.Variables["domain_name"]
if gandiCredential == nil || len(createLiveDNSRecords) == 0 || !hasDomainName {
return false, nil
}
// Get the list of DNS records from the Gandi API
httpClient := http.Client{}
gandiURL := fmt.Sprintf(
"%s/%s/%s/%s",
configuration.GANDI_API_URL, "v5/livedns/domains", domainName, "records",
)
request, err := http.NewRequest("GET", gandiURL, nil)
request.Header.Add("Authorization", fmt.Sprintf("Apikey %s", gandiCredential.Password))
response, err := httpClient.Do(request)
if err != nil {
return false, err
}
if response.StatusCode != 200 {
return false, fmt.Errorf("HTTP %d when calling %s ", gandiURL, response.StatusCode)
}
bytes, err := ioutil.ReadAll(response.Body)
if err != nil {
return false, errors.Wrapf(err, "HTTP read error when calling %s ", gandiURL)
}
var recordsFromAPI []gandiLiveDNSRecord
err = json.Unmarshal(bytes, &recordsFromAPI)
if err != nil {
return false, errors.Wrapf(err, "JSON parse error when calling %s ", gandiURL)
}
recordsFromAPIById := map[string]gandiLiveDNSRecord{}
for _, record := range recordsFromAPI {
recordsFromAPIById[fmt.Sprintf("%s/%s/%s", domainName, record.Name, record.Type)] = record
}
importedAnyRecords := false
// for each record terraform plans to create, check if it already exists in Gandi
for _, resource := range createLiveDNSRecords {
recordName, hasName := getStringValue(resource, "name")
recordType, hasType := getStringValue(resource, "type")
//recordName, hasName := getStringValue(resource, "type")
if !hasName {
return false, fmt.Errorf("gandi_livedns_record %s (from terraform planned_values) is missing string value \"name\" ", resource.Address)
}
if !hasType {
return false, fmt.Errorf("gandi_livedns_record %s (from terraform planned_values) is missing string value \"type\" ", resource.Address)
}
recordId := fmt.Sprintf("%s/%s/%s", domainName, recordName, recordType)
_, gandiAlreadyHasRecord := recordsFromAPIById[recordId]
if gandiAlreadyHasRecord {
fmt.Println("---------------------------------------")
fmt.Printf(
"Terraform plans to add the DNS record %s %s to gandi, but gandi already has a record like that \n",
recordType, domainName,
)
exitCode := removeAndImportResource(terraformDirectory, resource.Address, recordId)
if exitCode != 0 {
return false, errors.New("terraform import returned a non-zero exit code")
}
importedAnyRecords = true
fmt.Println("---------------------------------------")
} else {
altType := ""
if recordType == "CNAME" {
altType = "A"
}
if recordType == "A" {
altType = "CNAME"
}
altRecordId := fmt.Sprintf("%s/%s/%s", domainName, recordName, altType)
_, gandiHasAltRecord := recordsFromAPIById[altRecordId]
if gandiHasAltRecord {
fmt.Printf(
"Terraform plans to add the DNS record [%s %s] to gandi, but gandi has a conflicting record [%s %s]. Deleting the conflicting record...\n",
recordType, domainName, altType, domainName,
)
gandiURL := fmt.Sprintf(
"%s/%s/%s/%s/%s/%s",
configuration.GANDI_API_URL, "v5/livedns/domains", domainName, "records", recordName, altType,
)
request, err := http.NewRequest("DELETE", gandiURL, nil)
request.Header.Add("Authorization", fmt.Sprintf("Apikey %s", gandiCredential.Password))
response, err := httpClient.Do(request)
if err != nil {
return false, err
}
if response.StatusCode != 200 {
return false, fmt.Errorf("HTTP %d when calling DELETE %s ", gandiURL, response.StatusCode)
}
}
}
}
return importedAnyRecords, nil
var gandiCredential *configuration.Credential = nil
for _, cred := range config.Credentials {
if cred.Type == configuration.GANDI {
gandiCredential = &cred
break
}
}
createLiveDNSRecords := filterResourceChanges(
tfShow,
ResourceChangeFilter{ResourceType: "gandi_livedns_record", Action: "create", NotAction: "delete"},
)
domainName, hasDomainName := config.Terraform.Variables["domain_name"]
if gandiCredential == nil || len(createLiveDNSRecords) == 0 || !hasDomainName {
return false, nil
}
// Get the list of DNS records from the Gandi API
httpClient := http.Client{}
gandiURL := fmt.Sprintf(
"%s/%s/%s/%s",
configuration.GANDI_API_URL, "v5/livedns/domains", domainName, "records",
)
request, err := http.NewRequest("GET", gandiURL, nil)
request.Header.Add("Authorization", fmt.Sprintf("Apikey %s", gandiCredential.Password))
response, err := httpClient.Do(request)
if err != nil {
return false, err
}
if response.StatusCode != 200 {
return false, fmt.Errorf("HTTP %d when calling %s ", gandiURL, response.StatusCode)
}
bytes, err := ioutil.ReadAll(response.Body)
if err != nil {
return false, errors.Wrapf(err, "HTTP read error when calling %s ", gandiURL)
}
var recordsFromAPI []gandiLiveDNSRecord
err = json.Unmarshal(bytes, &recordsFromAPI)
if err != nil {
return false, errors.Wrapf(err, "JSON parse error when calling %s ", gandiURL)
}
recordsFromAPIById := map[string]gandiLiveDNSRecord{}
for _, record := range recordsFromAPI {
recordsFromAPIById[fmt.Sprintf("%s/%s/%s", domainName, record.Name, record.Type)] = record
}
importedAnyRecords := false
// for each record terraform plans to create, check if it already exists in Gandi
for _, resource := range createLiveDNSRecords {
recordName, hasName := getStringValue(resource, "name")
recordType, hasType := getStringValue(resource, "type")
//recordName, hasName := getStringValue(resource, "type")
if !hasName {
return false, fmt.Errorf("gandi_livedns_record %s (from terraform planned_values) is missing string value \"name\" ", resource.Address)
}
if !hasType {
return false, fmt.Errorf("gandi_livedns_record %s (from terraform planned_values) is missing string value \"type\" ", resource.Address)
}
recordId := fmt.Sprintf("%s/%s/%s", domainName, recordName, recordType)
_, gandiAlreadyHasRecord := recordsFromAPIById[recordId]
if gandiAlreadyHasRecord {
fmt.Println("---------------------------------------")
fmt.Printf(
"Terraform plans to add the DNS record %s %s to gandi, but gandi already has a record like that \n",
recordType, domainName,
)
exitCode := removeAndImportResource(terraformDirectory, resource.Address, recordId)
if exitCode != 0 {
return false, errors.New("terraform import returned a non-zero exit code")
}
importedAnyRecords = true
fmt.Println("---------------------------------------")
} else {
altType := ""
if recordType == "CNAME" {
altType = "A"
}
if recordType == "A" {
altType = "CNAME"
}
altRecordId := fmt.Sprintf("%s/%s/%s", domainName, recordName, altType)
_, gandiHasAltRecord := recordsFromAPIById[altRecordId]
if gandiHasAltRecord {
fmt.Printf(
"Terraform plans to add the DNS record [%s %s] to gandi, but gandi has a conflicting record [%s %s]. Deleting the conflicting record...\n",
recordType, domainName, altType, domainName,
)
gandiURL := fmt.Sprintf(
"%s/%s/%s/%s/%s/%s",
configuration.GANDI_API_URL, "v5/livedns/domains", domainName, "records", recordName, altType,
)
request, err := http.NewRequest("DELETE", gandiURL, nil)
request.Header.Add("Authorization", fmt.Sprintf("Apikey %s", gandiCredential.Password))
response, err := httpClient.Do(request)
if err != nil {
return false, err
}
if response.StatusCode != 200 {
return false, fmt.Errorf("HTTP %d when calling DELETE %s ", gandiURL, response.StatusCode)
}
}
}
}
return importedAnyRecords, nil
}

124
automation/patchHelpers.go

@ -1,81 +1,81 @@
package automation
import (
"fmt"
"fmt"
)
func getStringValue(change *TerraformShowResourceChange, key string) (string, bool) {
interfaceValue, has := change.Values[key]
if !has {
return "", false
}
switch stringValue := interfaceValue.(type) {
case string:
return stringValue, true
default:
return "", false
}
interfaceValue, has := change.Values[key]
if !has {
return "", false
}
switch stringValue := interfaceValue.(type) {
case string:
return stringValue, true
default:
return "", false
}
}
func getStringSliceValue(change *TerraformShowResourceChange, key string) ([]string, bool) {
interfaceValue, has := change.Values[key]
if !has {
return []string{}, false
}
switch stringSliceValue := interfaceValue.(type) {
case []string:
return stringSliceValue, true
default:
return []string{}, false
}
interfaceValue, has := change.Values[key]
if !has {
return []string{}, false
}
switch stringSliceValue := interfaceValue.(type) {
case []string:
return stringSliceValue, true
default:
return []string{}, false
}
}
func removeAndImportResource(terraformDirectory, address, id string) int {
// fullyQualifiedResourcePath := fmt.Sprintf("%s[%d]", tfResourcePath, index)
fmt.Printf(
"to fix this, I will run: \n(%s) $ terraform state rm %s\n(%s) $ terraform import %s %s\n\n",
terraformDirectory, address, terraformDirectory, address, id,
)
exitCode, stdout, stderr, _ := shellExec(terraformDirectory, "terraform", "state", "rm", address)
fmt.Printf(
"exitCode: %d\n\nstdout:\n%s\n\nstderr:\n%s\n\n",
exitCode, stdout, stderr,
)
exitCode, stdout, stderr, err := shellExec(terraformDirectory, "terraform", "import", address, id)
fmt.Printf(
"exitCode: %d\n\nstdout:\n%s\n\nstderr:\n%s\n\n",
exitCode, stdout, stderr,
)
if err != nil {
fmt.Println("shellExec failed! aborting.")
return exitCode
}
return exitCode
// fullyQualifiedResourcePath := fmt.Sprintf("%s[%d]", tfResourcePath, index)
fmt.Printf(
"to fix this, I will run: \n(%s) $ terraform state rm %s\n(%s) $ terraform import %s %s\n\n",
terraformDirectory, address, terraformDirectory, address, id,
)
exitCode, stdout, stderr, _ := shellExec(terraformDirectory, "terraform", "state", "rm", address)
fmt.Printf(
"exitCode: %d\n\nstdout:\n%s\n\nstderr:\n%s\n\n",
exitCode, stdout, stderr,
)
exitCode, stdout, stderr, err := shellExec(terraformDirectory, "terraform", "import", address, id)
fmt.Printf(
"exitCode: %d\n\nstdout:\n%s\n\nstderr:\n%s\n\n",
exitCode, stdout, stderr,
)
if err != nil {
fmt.Println("shellExec failed! aborting.")
return exitCode
}
return exitCode
}
func filterResourceChanges(tfShow *TerraformShow, filter ResourceChangeFilter) []*TerraformShowResourceChange {
toReturn := []*TerraformShowResourceChange{}
toReturn := []*TerraformShowResourceChange{}
for _, resource := range tfShow.ResourceChanges {
hasAction := filter.Action == ""
hasNotAction := false
for _, action := range resource.Change.Actions {
if action == filter.Action {
hasAction = true
}
if action == filter.NotAction {
hasNotAction = true
}
}
// fmt.Printf(
// "resource.Type: %s == filter.ResourceType: %s, hasAction %t, !hasNotAction %t \n",
// resource.Type, filter.ResourceType, hasAction, !hasNotAction,
// )
if resource.Type == filter.ResourceType && hasAction && !hasNotAction {
toReturn = append(toReturn, resource)
}
for _, resource := range tfShow.ResourceChanges {
hasAction := filter.Action == ""
hasNotAction := false
for _, action := range resource.Change.Actions {
if action == filter.Action {
hasAction = true
}
if action == filter.NotAction {
hasNotAction = true
}
}
// fmt.Printf(
// "resource.Type: %s == filter.ResourceType: %s, hasAction %t, !hasNotAction %t \n",
// resource.Type, filter.ResourceType, hasAction, !hasNotAction,
// )
if resource.Type == filter.ResourceType && hasAction && !hasNotAction {
toReturn = append(toReturn, resource)
}
}
return toReturn
}
return toReturn
}

52
automation/terraformActions.go

@ -292,15 +292,15 @@ func TerraformPlanAndApply(
return []byte{}, nil, errors.Wrap(err, "can't TerraformPlanAndApply because can't process.Start() terraform apply process")
}
scanAllOutput := func(prefix string, reader io.ReadCloser) {
scanAllOutput := func(reader io.ReadCloser) {
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
logLinesChannel <- fmt.Sprintf("%s%s", prefix, scanner.Text())
logLinesChannel <- scanner.Text()
}
}
go scanAllOutput("stdout: ", stdoutPipe)
go scanAllOutput("stderr: ", stderrPipe)
go scanAllOutput(stdoutPipe)
go scanAllOutput(stderrPipe)
toReturn := make(chan TerraformApplyResult)
@ -332,6 +332,7 @@ func monitorTerraformApplyProgress(
//ansibleModules := map[string]bool
logLines := []string{}
logLinesWithoutAnsiEscapes := []string{}
terraformIsRunning := true
terraformDirectory := filepath.Join(workingDirectory, terraformProject)
@ -339,10 +340,12 @@ func monitorTerraformApplyProgress(
ansiEscapeRegex := regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))")
go (func() {
terraformApplyLogRegex := regexp.MustCompile(`^stdout: (([a-zA-Z0-9_-]+\.)+[a-zA-Z0-9_-]+)(\[([0-9]+)\])?( \([a-zA-Z0-9_-]+\))?: (.*)$`)
terraformApplyLogRegex := regexp.MustCompile(`^(([a-zA-Z0-9_-]+\.)+[a-zA-Z0-9_-]+)(\[([0-9]+)\])?( \([a-zA-Z0-9_-]+\))?: (.*)$`)
terraformExecErrorRegex := regexp.MustCompile(`Error running command '(.*)': exit status [0-9]+\. Output:.*$`)
for logLine := range logLinesChannel {
logLines = append(logLines, logLine)
lineWithoutAnsiEscapes := ansiEscapeRegex.ReplaceAllString(logLine, "")
logLinesWithoutAnsiEscapes = append(logLinesWithoutAnsiEscapes, lineWithoutAnsiEscapes)
matches := terraformApplyLogRegex.FindStringSubmatch(lineWithoutAnsiEscapes)
if matches != nil {
address := matches[1]
@ -357,7 +360,9 @@ func monitorTerraformApplyProgress(
if strings.HasPrefix(address, moduleName) {
foundModule = true
if module.IsAnsible {
if strings.HasPrefix(message, "Creating...") || strings.HasPrefix(message, "Modifying...") {
// TODO test this strings.HasPrefix(message, "Executing:") instead
//if strings.HasPrefix(message, "Creating...") || strings.HasPrefix(message, "Modifying...") {
if strings.HasPrefix(message, "Executing:") {
//ansibleModules[moduleName] = true
//fmt.Printf("__INCLUDE_ANSIBLE__%s\n", module.DisplayName)
@ -404,6 +409,41 @@ func monitorTerraformApplyProgress(
}
} else {
//fmt.Printf("NOT PARSE %s\n", lineWithoutAnsiEscapes)
matches := terraformExecErrorRegex.FindStringSubmatch(lineWithoutAnsiEscapes)
if matches != nil {
commandThatFailed := matches[1]
previousLineContainingCommandThatFailed := ""
for i, previousLine := range logLinesWithoutAnsiEscapes {
if i < len(logLinesWithoutAnsiEscapes.length-2) && strings.Contains(previousLine, commandThatFailed) {
previousLineContainingCommandThatFailed = previousLine
}
}
if previousLineContainingCommandThatFailed != "" {
matches = terraformApplyLogRegex.FindStringSubmatch(previousLineContainingCommandThatFailed)
if matches != nil {
address := matches[1]
for moduleName, module := range simpleStatus.Modules {
if strings.HasPrefix(address, moduleName) {
if module.IsAnsible {
for _, resource := range module.Resources {
resource.State = "error"
}
} else {
addressInsideModule := strings.TrimPrefix(address, moduleName)
addressInsideModule = strings.Trim(addressInsideModule, ".")
for _, resource := range module.Resources {
if addressInsideModule == resource.DisplayName {
resource.State = "error"
}
}
}
}
}
}
}
}
}
}

1270
automation/terraformCodeGeneration.go

File diff suppressed because it is too large Load Diff

60
automation/terraformSvg.go

@ -128,8 +128,8 @@ func makeSVGFromSimpleStatus(simpleStatus *SimplifiedTerraformStatus) ([]byte, e
tooltip = fmt.Sprintf(`, tooltip = "%s.%s"`, moduleName, resource.DisplayName)
}
resourceDot := fmt.Sprintf(`
"%s" [label = "%s"%s%s]`,
id, padStringForDot(resource.DisplayName), tooltip, resourceStyle,
"%s" [label = "%s"%s%s]`,
id, padStringForDot(resource.DisplayName, 0.16), tooltip, resourceStyle,
)
resourceDots = append(resourceDots, resourceDot)
}
@ -141,22 +141,22 @@ func makeSVGFromSimpleStatus(simpleStatus *SimplifiedTerraformStatus) ([]byte, e
id1 := fmt.Sprintf(`%s_%s`, moduleName, resource.DisplayName)
id1 = strings.ReplaceAll(strings.ReplaceAll(id1, ".", "_"), "-", "_")
resourceDot := fmt.Sprintf(`
"%s" -> "%s" [style=invis]`, id0, id1,
"%s" -> "%s" [style=invis]`, id0, id1,
)
resourceDots = append(resourceDots, resourceDot)
}
i++
}
moduleDot := fmt.Sprintf(`
subgraph cluster_%s {
bgcolor = lightgrey;
tooltip = "%s";
label = "%s";
%s
}`,
subgraph cluster_%s {
bgcolor = lightgrey;
tooltip = "%s";
label = "%s";
%s
}`,
strings.ReplaceAll(strings.ReplaceAll(moduleName, ".", "_"), "-", "_"),
moduleName,
padStringForDot(moduleName),
padStringForDot(moduleName, 0.16),
strings.Join(resourceDots, ""),
)
//fmt.Println(moduleName)
@ -166,8 +166,8 @@ func makeSVGFromSimpleStatus(simpleStatus *SimplifiedTerraformStatus) ([]byte, e
variableDots := []string{}
for variableName, value := range simpleStatus.Variables {
variableDot := fmt.Sprintf(`
"var_%s" [label = "%s", tooltip = "%s", margin = 0.1, shape = "note"];`,
strings.ReplaceAll(variableName, "-", "_"), padStringForDot(value), fmt.Sprintf("var.%s", variableName),
"var_%s" [label = "%s", tooltip = "%s", margin = 0.1, shape = "note"];`,
strings.ReplaceAll(variableName, "-", "_"), padStringForDot(value, 0.16), fmt.Sprintf("var.%s", variableName),
)
variableDots = append(variableDots, variableDot)
}
@ -200,7 +200,7 @@ func makeSVGFromSimpleStatus(simpleStatus *SimplifiedTerraformStatus) ([]byte, e
paramName := strings.ReplaceAll(strings.ReplaceAll(connection.DisplayName, ".", "_"), "-", "_")
sourceEdge := fmt.Sprintf(`
"%s" -> "param_%s" [%sarrowhead=none];`,
"%s" -> "param_%s" [%sarrowhead=none];`,
from, paramName, tailLocation,
)
if !edgeMap[sourceEdge] {
@ -209,7 +209,7 @@ func makeSVGFromSimpleStatus(simpleStatus *SimplifiedTerraformStatus) ([]byte, e
sourceEdge = ""
}
destEdge := fmt.Sprintf(`
"param_%s" -> "%s" [%s];`,
"param_%s" -> "%s" [%s];`,
paramName, to, headLocation,
)
if !edgeMap[destEdge] {
@ -219,22 +219,22 @@ func makeSVGFromSimpleStatus(simpleStatus *SimplifiedTerraformStatus) ([]byte, e
}
connectionDot := fmt.Sprintf(`
"param_%s" [label = "%s", shape = "invhouse", style=filled, color=lightgrey]; %s %s`,
"param_%s" [label = "%s", shape = "invhouse", style=filled, color=lightgrey]; %s %s`,
paramName, connection.DisplayName, sourceEdge, destEdge,
)
connectionDots = append(connectionDots, connectionDot)
}
dot := fmt.Sprintf(`
digraph {
compound = "true"
newrank = "true"
ranksep = 0.1;
%s
%s
%s
}`,
digraph {
compound = "true"
newrank = "true"
ranksep = 0.1;
%s
%s
%s
}`,
strings.Join(moduleDots, ""),
strings.Join(variableDots, ""),
strings.Join(connectionDots, ""),
@ -343,13 +343,15 @@ func makeSVGFromSimpleStatus(simpleStatus *SimplifiedTerraformStatus) ([]byte, e
if len(fromTo) != 2 {
fmt.Printf("malformed dot string on svg edge: fromTo.length != 2: %s", title)
}
// note that we also replaced "->" with "_to_" to avoid html escape confusion
// if this edge starts or ends at a resource inside a module, override the dot string
// to make it start or end at that module instead.
for _, moduleName := range moduleNames {
if strings.HasPrefix(fromTo[0], moduleName) {
node.Parent.SetAttr("data-dot", fmt.Sprintf("%s_to_%s", moduleName, fromTo[1]))
node.Parent.SetAttr("data-dot", html.EscapeString(fmt.Sprintf("%s->%s", moduleName, fromTo[1])))
}
if strings.HasPrefix(fromTo[1], moduleName) {
node.Parent.SetAttr("data-dot", fmt.Sprintf("%s_to_%s", fromTo[0], moduleName))
node.Parent.SetAttr("data-dot", html.EscapeString(fmt.Sprintf("%s->%s", fromTo[0], moduleName)))
}
}
}
@ -522,8 +524,8 @@ func makeSVGFromSimpleStatus(simpleStatus *SimplifiedTerraformStatus) ([]byte, e
return svgBytes, nil
}
func padStringForDot(str string) string {
pad := strings.Repeat(" ", int(float32(len(str))/4))
func padStringForDot(str string, padAmount float32) string {
pad := strings.Repeat(" ", int(float32(len(str))/(float32(1)/padAmount)))
return fmt.Sprintf("%s%s%s", pad, str, pad)
}

218
configuration/configuration.go

@ -1,51 +1,51 @@
package configuration
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"runtime"
"syscall"
errors "git.sequentialread.com/forest/pkg-errors"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"runtime"
"syscall"
errors "git.sequentialread.com/forest/pkg-errors"
)
type Configuration struct {
Host HostConfiguration
Terraform TerraformConfiguration
ObjectStorage ObjectStorageConfiguration
Credentials []Credential
Host HostConfiguration
Terraform TerraformConfiguration
ObjectStorage ObjectStorageConfiguration
Credentials []Credential
}
type HostConfiguration struct {
Name string
Name string
}
type TerraformConfiguration struct {
GlobalModules []string
LocalModules []string
Variables map[string]string
GlobalModules []string
LocalModules []string
Variables map[string]string
}
type ObjectStorageConfiguration struct {
Encryption string
Backends []ObjectStorageBackend
Encryption string
Backends []ObjectStorageBackend
}
type ObjectStorageBackend struct {
Provider string
Region string
Name string
Provider string
Region string
Name string
}
type Credential struct {
Type string
SSID string
Username string
Password string
Type string
SSID string
Username string
Password string
}
const GANDI = "Gandi"
@ -65,11 +65,11 @@ const ANSIBLE_WRAPPER_PATH = "ansible-wrapper"
const TERRAFORM_APPLY_STATUS_UPDATE_INTERVAL_SECONDS = 5
func GET_ANSIBLE_WRAPPER_FILES() []string {
return []string{
"ansible-playbook-wrapper",
"callback_plugins",
"ansible.cfg",
}
return []string{
"ansible-playbook-wrapper",
"callback_plugins",
"ansible.cfg",
}
}
const SSH_KEYS_PATH = "ssh"
@ -77,91 +77,91 @@ const TERRAFORM_STATE_SERVER_PORT_NUMBER = 6471
func LoadConfiguration() (*Configuration, string, error) {
if runtime.GOOS == "windows" {
return nil, "", errors.New("windows operating system is not supported.")
}
workingDirectory, err := os.Getwd()
if err != nil {
return nil, "", errors.Wrap(err, "can't os.Getwd()")
}
executableDirectory, err := getCurrentExecDir()
if err != nil {
return nil, "", errors.Wrap(err, "can't getCurrentExecDir()")
}
configFileLocation1 := filepath.Join(executableDirectory, "config.json")
configFileLocation2 := filepath.Join(workingDirectory, "config.json")
configFileLocation := configFileLocation1
configFileStat, err := os.Stat(configFileLocation)
workingDirectoryToReturn := executableDirectory
if err != nil || !configFileStat.Mode().IsRegular() {
configFileLocation = configFileLocation2
configFileStat, err = os.Stat(configFileLocation)
workingDirectoryToReturn = workingDirectory
}
if err != nil || !configFileStat.Mode().IsRegular() {
return nil, workingDirectoryToReturn, fmt.Errorf("no config file. checked %s and %s", configFileLocation1, configFileLocation2)
}
// configFileUid, err := getUID(configFileStat)
// if err != nil {
// return nil, errors.Wrapf(err, "can't getUID() config file %s", configFileLocation)
// }
// if configFileUid != 0 {
// return nil, fmt.Errorf("can't start rootsystem: the config file %s is not owned by root.", configFileLocation)
// }
// ownerReadWriteOnlyPermissionsOctal := "600"
// configFilePermissionsOctal := fmt.Sprintf("%o", configFileStat.Mode().Perm())
// if configFilePermissionsOctal != ownerReadWriteOnlyPermissionsOctal {
// return nil, fmt.Errorf(
// "can't start rootsystem: the config file %s had permissions %s. expected %s. %s",
// configFileLocation,
// configFilePermissionsOctal,
// ownerReadWriteOnlyPermissionsOctal,
// "(config file should only be readable and writable by the owner, aka the root user)",
// )
// }
jsonBytes, err := ioutil.ReadFile(configFileLocation)
if err != nil {
return nil, workingDirectoryToReturn, errors.Wrap(err, "can't read config file")
}
var config Configuration
err = json.Unmarshal(jsonBytes, &config)
if err != nil {
return nil, workingDirectoryToReturn, errors.Wrap(err, "can't json.Unmarshal config file")
}
return &config, workingDirectoryToReturn, nil
if runtime.GOOS == "windows" {
return nil, "", errors.New("windows operating system is not supported.")
}
workingDirectory, err := os.Getwd()
if err != nil {
return nil, "", errors.Wrap(err, "can't os.Getwd()")
}
executableDirectory, err := getCurrentExecDir()
if err != nil {
return nil, "", errors.Wrap(err, "can't getCurrentExecDir()")
}
configFileLocation1 := filepath.Join(executableDirectory, "config.json")
configFileLocation2 := filepath.Join(workingDirectory, "config.json")
configFileLocation := configFileLocation1
configFileStat, err := os.Stat(configFileLocation)
workingDirectoryToReturn := executableDirectory
if err != nil || !configFileStat.Mode().IsRegular() {
configFileLocation = configFileLocation2
configFileStat, err = os.Stat(configFileLocation)
workingDirectoryToReturn = workingDirectory
}
if err != nil || !configFileStat.Mode().IsRegular() {
return nil, workingDirectoryToReturn, fmt.Errorf("no config file. checked %s and %s", configFileLocation1, configFileLocation2)
}
// configFileUid, err := getUID(configFileStat)
// if err != nil {
// return nil, errors.Wrapf(err, "can't getUID() config file %s", configFileLocation)
// }
// if configFileUid != 0 {
// return nil, fmt.Errorf("can't start rootsystem: the config file %s is not owned by root.", configFileLocation)
// }
// ownerReadWriteOnlyPermissionsOctal := "600"
// configFilePermissionsOctal := fmt.Sprintf("%o", configFileStat.Mode().Perm())
// if configFilePermissionsOctal != ownerReadWriteOnlyPermissionsOctal {
// return nil, fmt.Errorf(
// "can't start rootsystem: the config file %s had permissions %s. expected %s. %s",
// configFileLocation,
// configFilePermissionsOctal,
// ownerReadWriteOnlyPermissionsOctal,
// "(config file should only be readable and writable by the owner, aka the root user)",
// )
// }
jsonBytes, err := ioutil.ReadFile(configFileLocation)
if err != nil {
return nil, workingDirectoryToReturn, errors.Wrap(err, "can't read config file")
}
var config Configuration
err = json.Unmarshal(jsonBytes, &config)
if err != nil {
return nil, workingDirectoryToReturn, errors.Wrap(err, "can't json.Unmarshal config file")
}
return &config, workingDirectoryToReturn, nil
}
func getUID(fileInfo os.FileInfo) (int, error) {
stat, ok := fileInfo.Sys().(*syscall.Stat_t)
if !ok {
return -1, fmt.Errorf("can't cast os.Stat(\"%s\").Sys() to *syscall.Stat_t")
}
stat, ok := fileInfo.Sys().(*syscall.Stat_t)
if !ok {
return -1, fmt.Errorf("can't cast os.Stat(\"%s\").Sys() to *syscall.Stat_t")
}
return int(stat.Uid), nil
return int(stat.Uid), nil
}
func getCurrentExecDir() (dir string, err error) {
path, err := exec<