rootsystem/automation/terraformCodeGeneration.go

package automation

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strings"

	errors "git.sequentialread.com/forest/pkg-errors"
	"git.sequentialread.com/forest/rootsystem/configuration"
)

type TerraformConfiguration struct {
	BuildNumber                      int
	TargetedModules                  []string
	TerraformProject                 string
	RemoteState                      string
	RemoteStateVariables             []string
	HostKeysObjectStorageCredentials []configuration.Credential
}

type tfProvider struct {
	Name                 string
	Version              string
	CredentialType       string
	UsernamePropertyName string
	PasswordPropertyName string
}

type tfModule struct {
	Name       string
	Providers  []string
	Arguments  []*tfVariable
	Attributes []*tfVariable
}

type tfVariable struct {
	Name       string
	IsList     bool
	ProvidedBy []*tfProvidedBy
}

type tfProvidedBy struct {
	Module            string
	Attribute         string
	Variable          string
	RemoteStateOutput string
}

const ssh_public_keys = "ssh_public_keys"
const ssh_private_key_filepath = "ssh_private_key_filepath"
const node_id = "node_id"
const node_arch = "node_arch"
const post_to_object_storage_shell_script = "post_to_object_storage_shell_script"
const ssh_private_key_filepath_value = "ssh/servergarden_builtin_ed22519"
const remote_state_placeholder_text = "<previous build>"
const ssh_public_keys_placeholder_text = "<secure shell public keys>"
const post_to_object_storage_shell_script_placeholder_text = "<long shell script>"

func setHardcodedVariables(config *configuration.Configuration, variables map[string]string) {
	variables[ssh_public_keys] = ssh_public_keys_placeholder_text
	variables[node_id] = config.Host.Name
	variables[node_arch] = runtime.GOARCH
	variables[ssh_private_key_filepath] = ssh_private_key_filepath_value
	variables[post_to_object_storage_shell_script] = post_to_object_storage_shell_script_placeholder_text
}

func WriteTerraformCodeForTargetedModules(
	config *configuration.Configuration,
	workingDirectory string,
	terraformConfig TerraformConfiguration,
) ([]string, error) {

	providers := map[string]tfProvider{
		"digitalocean": tfProvider{
			Name:                 "digitalocean",
			Version:              "1.15",
			CredentialType:       configuration.DIGITALOCEAN,
			PasswordPropertyName: "token",
		},
		"gandi": tfProvider{
			Name:                 "gandi",
			CredentialType:       configuration.GANDI,
			PasswordPropertyName: "key",
		},
	}

	modules, err := parseModulesFolder(providers, workingDirectory)
	if err != nil {
		return []string{}, errors.Wrap(err, "can't WriteTerraformCodeForTargetedModules because can't parseModulesFolder()")
	}

	usedProviders := make(map[string]tfProvider)
	for _, provider := range providers {
		for _, credential := range config.Credentials {
			if provider.CredentialType == credential.Type {
				usedProviders[provider.Name] = provider
			}
		}
	}

	usedModules := make(map[string]tfModule, 0)
	for _, module := range modules {
		missingOne := false
		for _, providerName := range module.Providers {
			if _, has := usedProviders[providerName]; !has {
				missingOne = true
			}
		}
		if !missingOne {
			usedModules[module.Name] = module
		}
	}

	// these variables don't have to have the real values, but all used variables  do have to be
	// included in this map so we can wire up the modules correctly.
	allVariables := map[string]string{}
	for k, v := range config.Terraform.Variables {
		allVariables[k] = v
	}
	setHardcodedVariables(config, allVariables)

	err = fillOutProvidedByOnAttributes(usedModules, allVariables, terraformConfig.RemoteStateVariables)
	if err != nil {
		return []string{}, errors.Wrap(err, "can't WriteTerraformCodeForTargetedModules because can't fillOutProvidedByOnAttributes")
	}

	modulesToCreate := map[string]bool{}
	for _, target := range terraformConfig.TargetedModules {
		module, has := usedModules[target]
		if !has {
			return []string{}, errors.Wrapf(
				err,
				`can't WriteTerraformCodeForTargetedModules because the TargetedModule \"%s\" 
	      was not found in the list of modules that we have authenticated providers for. 
	      Missing credentials or incorrect module name.`,
				target,
			)
		}

		dependencies, err := getDependencies(module, usedModules, nil)
		if !has {
			return []string{}, errors.Wrapf(err, `can't WriteTerraformCodeForTargetedModules because cant getDependencies for module "%s"`, module.Name)
		}

		modulesToCreate[module.Name] = true
		for _, moduleName := range dependencies {
			modulesToCreate[moduleName] = true
		}
	}

	providerStanzas := make([]string, 0)
	for _, provider := range usedProviders {
		for _, credential := range config.Credentials {
			if provider.CredentialType == credential.Type {
				versionProperty := ""
				usernameProperty := ""
				passwordProperty := ""
				if provider.UsernamePropertyName != "" {
					usernameProperty = fmt.Sprintf(
						`
						  %s = "%s"
						`,
						provider.UsernamePropertyName,
						credential.Username,
					)
				}
				if provider.PasswordPropertyName != "" {
					passwordProperty = fmt.Sprintf(
						`
						  %s = "%s"
						`,
						provider.PasswordPropertyName,
						credential.Password,
					)
				}
				if provider.Version != "" {
					versionProperty = fmt.Sprintf(
						`
						  version = "%s"
						`,
						provider.Version,
					)
				}
				providerStanzas = append(providerStanzas, fmt.Sprintf(
					`
	        provider "%s" {%s%s%s}
	        `,
					provider.Name,
					versionProperty,
					usernameProperty,
					passwordProperty,
				))
			}
		}
	}

	sshPublicKeyObjectStrings := make([]string, 0)

	sshPublicKeys, err := getSSHPublicKeys(workingDirectory)
	if err != nil {
		return []string{}, errors.Wrap(err, "can't WriteTerraformCodeForTargetedModules because can't getSSHPublicKeys")
	}

	for _, sshPublicKey := range sshPublicKeys {
		sshPublicKeyObjectStrings = append(sshPublicKeyObjectStrings, fmt.Sprintf(
			`{
	      name = "%s"
	      filepath = "%s"
	    }`,
			sshPublicKey[1],
			sshPublicKey[0],
		))
	}

	// we can't use the allVariables here like we used above, because we have to handle variable values
	// that are a list of objects, not just a string.
	// so we special case it below this loop.
	variableStanzas := make([]string, 0)
	for key, value := range config.Terraform.Variables {
		if strings.Contains(value, "\n") {
			variableStanzas = append(variableStanzas, fmt.Sprintf(`
	      variable "%s" { 
	        default = <<EOT
	        %s
EOT
	      }
	      `, key, value,
			))
		} else {
			variableStanzas = append(variableStanzas, fmt.Sprintf(`
	      variable "%s" { default = "%s" }
	      `, key, value,
			))
		}
	}

	postToObjectStorageShellScript := ""
	if terraformConfig.HostKeysObjectStorageCredentials != nil {
		postToObjectStorageShellScript, err = getPostToObjectStorageShellScript(
			config,
			terraformConfig.HostKeysObjectStorageCredentials,
		)
		if err != nil {
			return []string{}, errors.Wrap(err, "can't WriteTerraformCodeForTargetedModules because")
		}
	}

	variableStanzas = append(variableStanzas,
		fmt.Sprintf(`
	    variable "%s" {
	      type = list(object({
	         name = string
	         filepath = string
	      }))
	      default = [%s]
	    }
	    `, ssh_public_keys, strings.Join(sshPublicKeyObjectStrings, ",\n"),
		),
		fmt.Sprintf(`
	    variable "%s" { default = "%s" }
	    `, node_id, config.Host.Name,
		),

		fmt.Sprintf(`
	    variable "%s" { default = "%s" }
	    `, node_arch, runtime.GOARCH,
		),
		fmt.Sprintf(`
	    variable "%s" { default = "%s" }
	    `, ssh_private_key_filepath, filepath.Join(workingDirectory, ssh_private_key_filepath_value),
		),
		fmt.Sprintf(`
	    variable "%s" { 
	      default = <<EOT
	      %s
EOT
	    }
	    `, post_to_object_storage_shell_script, postToObjectStorageShellScript,
		),
	)

	moduleStanzas := make([]string, 0)
	for moduleName := range modulesToCreate {
		module := usedModules[moduleName]
		argumentLines := []string{}

		providedByToString := func(provision *tfProvidedBy) string {
			if provision.Variable != "" {
				return fmt.Sprintf("var.%s", provision.Variable)
			} else if provision.RemoteStateOutput != "" {
				return fmt.Sprintf(
					"data.terraform_remote_state.%s.outputs.%s",
					terraformConfig.RemoteState, provision.RemoteStateOutput,
				)
			} else {
				return fmt.Sprintf("module.%s.%s", provision.Module, provision.Attribute)
			}
		}

		for _, argument := range module.Arguments {
			if !argument.IsList {
				var usedProvision *tfProvidedBy = nil
				providedByVariable := false
				providedByModules := []string{}
				for _, provision := range argument.ProvidedBy {
					if provision.Variable != "" {
						providedByVariable = true
						usedProvision = provision
					}
					if provision.RemoteStateOutput != "" {
						usedProvision = provision
					}
					if provision.Module != "" {
						if usedProvision == nil {
							usedProvision = provision
						}
						providedByModules = append(providedByModules, provision.Module)
					}
				}
				if providedByVariable && len(providedByModules) > 0 {
					return []string{},
						fmt.Errorf(
							"TODO: what do do in this case? a non-list argument %s is provided by both a variable and a module (%s)",
							argument.Name,
							providedByModules[0],
						)
				}
				if len(providedByModules) > 1 {
					return []string{},
						fmt.Errorf(
							"TODO: what do do in this case? a non-list argument %s is provided by more than one module: [%s]",
							argument.Name,
							strings.Join(providedByModules, ", "),
						)
				}
				if usedProvision == nil {
					return []string{}, fmt.Errorf("argument %s is not provided by any variables or modules", argument.Name)
				}
				argumentLines = append(argumentLines, fmt.Sprintf(
					`
						%s = %s
	        `,
					argument.Name,
					providedByToString(usedProvision),
				))
			} else {
				fixedProvidedBy := removeRedundantModuleProvision(argument.ProvidedBy)
				provisionStrings := []string{}
				for _, provision := range fixedProvidedBy {
					provisionStrings = append(provisionStrings, providedByToString(provision))
				}
				sort.Strings(provisionStrings)
				argumentLines = append(argumentLines, fmt.Sprintf(
					`
						%s = [
						  %s
						]
	        `,
					argument.Name,
					strings.Join(provisionStrings, ",\n						  "),
				))
			}
		}

		moduleStanzas = append(moduleStanzas, fmt.Sprintf(
			`
	    module "%s" { 
	      source = "./modules/%s" 
%s
	    }
	    `,
			module.Name,
			module.Name,
			strings.Join(argumentLines, "\n"),
		))
	}

	outputStanzas := []string{}
	outputVariables := []string{}
	for moduleName := range modulesToCreate {
		module := usedModules[moduleName]
		for _, attribute := range module.Attributes {
			outputVariables = append(outputVariables, attribute.Name)
			outputStanzas = append(outputStanzas, fmt.Sprintf(`
	    output "%s" {
	      value = module.%s.%s
	    }
	    `, attribute.Name, module.Name, attribute.Name))
		}
	}

	terraformFolder := filepath.Join(workingDirectory, terraformConfig.TerraformProject)
	if _, err := os.Stat(terraformFolder); os.IsNotExist(err) {
		err = os.Mkdir(terraformFolder, 0700)
		if err != nil {
			return []string{}, errors.Wrapf(err, "can't initializeTerraformProject because can't os.Mkdir(\"%s\")", terraformFolder)
		}
	}
	terraformCodeFilepath := filepath.Join(terraformFolder, "main.tf")
	terraformCodeFile, err := os.OpenFile(terraformCodeFilepath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0700)
	if err != nil {
		return []string{}, errors.Wrapf(err, "can't initializeTerraformProject because can't os.OpenFile(\"%s\")", terraformCodeFilepath)
	}

	terraformModulesFolder := filepath.Join(terraformFolder, "modules")

	os.Remove(terraformModulesFolder)
	err = os.Symlink(
		filepath.Join(workingDirectory, configuration.TERRAFORM_MODULES),
		terraformModulesFolder,
	)
	if err != nil {
		return []string{}, errors.Wrapf(err, "could not create symbolic link to terraform modules folder")
	}

	remoteState := ""
	if terraformConfig.RemoteState != "" {
		remoteState = fmt.Sprintf(`
	    data "terraform_remote_state" "%s" {
	      backend = "http"
	      config = {
	        address = "http://localhost:6471/%s"
	      }
	    }
	    `,
			terraformConfig.RemoteState,
			terraformConfig.RemoteState,
		)
	}

	fmt.Fprintf(
		terraformCodeFile,
		`
	  terraform {
	    backend "http" {
	      address = "http://localhost:%d/%s"
	    }
	  }
%s
%s
%s
%s
%s
	  `,
		configuration.TERRAFORM_STATE_SERVER_PORT_NUMBER,
		terraformConfig.TerraformProject,
		remoteState,
		strings.Join(providerStanzas, "\n"),
		strings.Join(variableStanzas, "\n"),
		strings.Join(moduleStanzas, "\n"),
		strings.Join(outputStanzas, "\n"),
	)

	return outputVariables, nil
}

func removeRedundantModuleProvision(provisions []*tfProvidedBy) []*tfProvidedBy {
	provisionMap := map[string]*tfProvidedBy{}
	for _, provision := range provisions {
		name := provision.Attribute
		if name == "" {
			name = provision.Variable
		}
		if name == "" {
			name = provision.RemoteStateOutput
		}
		_, has := provisionMap[name]
		if !has || provision.Variable != "" || provision.RemoteStateOutput != "" {
			provisionMap[name] = provision
		}
	}
	toReturn := []*tfProvidedBy{}
	for _, provision := range provisionMap {
		toReturn = append(toReturn, provision)
	}
	return toReturn
}

func provides(attributeName string, argument *tfVariable) bool {
	argumentNamePrefix := regexp.MustCompile("[-_]list$").ReplaceAllString(argument.Name, "")
	listMatch := argument.IsList && strings.HasPrefix(attributeName, argumentNamePrefix)
	return attributeName == argument.Name || listMatch
}

func fillOutProvidedByOnAttributes(
	modules map[string]tfModule,
	variables map[string]string,
	remoteStateOutputs []string,
) error {
	for _, module := range modules {
		for _, argument := range module.Arguments {
			argument.ProvidedBy = make([]*tfProvidedBy, 0)

			for variableName := range variables {
				if provides(variableName, argument) {
					argument.ProvidedBy = append(argument.ProvidedBy, &tfProvidedBy{Variable: variableName})
				}
			}

			for _, outputName := range remoteStateOutputs {
				if provides(outputName, argument) {
					argument.ProvidedBy = append(argument.ProvidedBy, &tfProvidedBy{RemoteStateOutput: outputName})
				}
			}

			for _, otherModule := range modules {
				if otherModule.Name != module.Name {
					for _, otherModuleAttribute := range otherModule.Attributes {
						if provides(otherModuleAttribute.Name, argument) {
							argument.ProvidedBy = append(argument.ProvidedBy, &tfProvidedBy{
								Module:    otherModule.Name,
								Attribute: otherModuleAttribute.Name,
							})
						}
					}
				}
			}

			if len(argument.ProvidedBy) == 0 {
				return fmt.Errorf(
					`can't find any variables or module attributes to provide argument %s on module %s. 
Are you missing a variable, or credentials for a provider that is required for one of your modules or one of its dependencies?`,
					argument.Name,
					module.Name,
				)
			}
		}
	}

	return nil
}

func getDependencies(
	module tfModule,
	modules map[string]tfModule,
	visited map[string]bool,
) ([]string, error) {

	if visited == nil {
		visited = map[string]bool{}
		visited[module.Name] = true
	}
	dependencies := make([]string, 0)

	for _, argument := range module.Arguments {

		fixedProvidedBy := removeRedundantModuleProvision(argument.ProvidedBy)

		for _, provision := range fixedProvidedBy {
			requiresModule := provision.RemoteStateOutput == "" && provision.Variable == ""
			if requiresModule && provision.Module != "" && !visited[provision.Module] {
				visited[provision.Module] = true
				dependencies = append(dependencies, provision.Module)
				otherModule, has := modules[provision.Module]
				if !has {
					return nil, fmt.Errorf(
						"can't get transitiveDependencies for module \"%s\": other module \"%s\" does not exist",
						module.Name,
						provision.Module,
					)
				}
				transitiveDependencies, err := getDependencies(otherModule, modules, visited)
				if err != nil {
					return nil, errors.Wrapf(err, "can't get transitiveDependencies for module %s", provision.Module)
				}
				dependencies = append(dependencies, transitiveDependencies...)
			}
		}
	}

	return dependencies, nil
}

func parseModulesFolder(providers map[string]tfProvider, workingDirectory string) ([]tfModule, error) {

	modules := make([]tfModule, 0)

	// TODO make sure modules folder and all the code inside is owned by root and only writable by root
	modulesFolder := filepath.Join(workingDirectory, configuration.TERRAFORM_MODULES)
	modulesFileInfos, err := ioutil.ReadDir(modulesFolder)
	if err != nil {
		return nil, errors.Wrapf(err, "can't parseModulesFolder because can't ioutil.ReadDir(\"%s\")", modulesFolder)
	}

	for _, fileInfo := range modulesFileInfos {
		if fileInfo.IsDir() {
			moduleName := fileInfo.Name()

			module := tfModule{
				Name:       moduleName,
				Providers:  make([]string, 0),
				Arguments:  make([]*tfVariable, 0),
				Attributes: make([]*tfVariable, 0),
			}

			for _, provider := range providers {
				if strings.Contains(moduleName, provider.Name) {
					module.Providers = append(module.Providers, provider.Name)
				}
			}

			moduleFolder := filepath.Join(modulesFolder, moduleName)
			moduleCodeFileInfos, err := ioutil.ReadDir(moduleFolder)
			if err != nil {
				return nil, errors.Wrapf(err, "can't parseModulesFolder because can't ioutil.ReadDir(\"%s\")", moduleFolder)
			}

			for _, fileInfo := range moduleCodeFileInfos {
				filepath := filepath.Join(moduleFolder, fileInfo.Name())
				if strings.HasSuffix(filepath, ".tf") {
					bytes, err := ioutil.ReadFile(filepath)
					if err != nil {
						return nil, errors.Wrapf(err, "can't parseModulesFolder because can't ioutil.ReadFile(\"%s\")", filepath)
					}

					// Variables / Arguments
					// TODO replace this with a proper lexer so we can tell whether the var has a default or not
					matches := regexp.MustCompile("variable\\s+\"([^\"]+)\"\\s*{?").FindAllSubmatch(bytes, -1)
					for _, match := range matches {
						if len(match) < 2 {
							return nil, errors.New("can't parseModulesFolder because something unexpected happened parsing terraform code")
						}
						variableName := string(match[1])
						module.Arguments = append(module.Arguments, &tfVariable{
							Name:   variableName,
							IsList: strings.HasSuffix(variableName, "list"),
						})
					}

					// Outputs / Attributes
					matches = regexp.MustCompile("output\\s+\"([^\"]+)\"\\s*{?").FindAllSubmatch(bytes, -1)
					for _, match := range matches {
						if len(match) < 2 {
							return nil, errors.New("can't parseModulesFolder because something unexpected happened parsing terraform code")
						}
						variableName := string(match[1])
						module.Attributes = append(module.Attributes, &tfVariable{
							Name:   variableName,
							IsList: strings.HasSuffix(variableName, "list"),
						})
					}
				}
			}

			modules = append(modules, module)

		}
	}

	return modules, nil
}

func getPostToObjectStorageShellScript(
	config *configuration.Configuration,
	credentials []configuration.Credential,
) (string, error) {

	backblazeTemplate := `
BUCKET_NAME="%s"
AUTH_JSON="$(curl -sS -u "%s:%s" https://api.backblazeb2.com/b2api/v2/b2_authorize_account)"

API_URL="$(echo "$AUTH_JSON" | grep -E -o '"apiUrl": "([^"]+)"' | sed -E 's|"apiUrl": "([^"]+)"|\1|')"
ACCOUNT_ID="$(echo "$AUTH_JSON" | grep -E -o '"accountId": "([^"]+)"' | sed -E 's|"accountId": "([^"]+)"|\1|')"
AUTH_TOKEN="$(echo "$AUTH_JSON" | grep -E -o '"authorizationToken": "([^"]+)"' | sed -E 's|"authorizationToken": "([^"]+)"|\1|')"

LIST_BUCKETS_JSON="$(curl -sS -H "Authorization: $AUTH_TOKEN" "$API_URL/b2api/v2/b2_list_buckets?accountId=$ACCOUNT_ID&bucketName=$BUCKET_NAME" )"
BUCKET_ID="$(echo "$LIST_BUCKETS_JSON" | grep -E -o '"bucketId": "([^"]+)"' | sed -E 's|"bucketId": "([^"]+)"|\1|')"

UPLOAD_URL_JSON="$(curl -sS -H "Authorization: $AUTH_TOKEN" "$API_URL/b2api/v2/b2_get_upload_url?bucketId=$BUCKET_ID" )"

UPLOAD_URL="$(echo "$UPLOAD_URL_JSON" | grep -E -o '"uploadUrl": "([^"]+)"' | sed -E 's|"uploadUrl": "([^"]+)"|\1|')"
AUTH_TOKEN="$(echo "$UPLOAD_URL_JSON" | grep -E -o '"authorizationToken": "([^"]+)"' | sed -E 's|"authorizationToken": "([^"]+)"|\1|')"

CONTENT_SHA1="$(echo -n "$CONTENT" | sha1sum | awk '{ print $1 }')"

curl -sS -X POST \
	-H "Authorization: $AUTH_TOKEN" \
	-H "X-Bz-File-Name: $FILE_PATH" \
	-H "X-Bz-Content-Sha1: $CONTENT_SHA1" \
	-H "Content-Type: text/plain" \
	"$UPLOAD_URL" -d "$CONTENT"
`
	scripts := []string{}

	for _, backend := range config.ObjectStorage.Backends {
		for _, credential := range credentials {
			if credential.Type == backend.Provider {
				if backend.Provider == configuration.BACKBLAZE_B2 {
					scripts = append(scripts, fmt.Sprintf(backblazeTemplate, backend.Name, credential.Username, credential.Password))
				}
				if backend.Provider == configuration.AMAZON_S3 {
					// TODO
					return "", errors.New("getPostToObjectStorageShellScript not implemented yet for S3-compatible")
				}
			}
		}
	}

	if len(scripts) == 0 {
		return "", errors.New("No credentials were passed to getPostToObjectStorageShellScript")
	}

	return strings.Join(scripts, "\n"), nil
}

func getSSHPublicKeys(workingDirectory string) ([][]string, error) {
	// TODO Make sure these keys only readable by root
	sshDirectory := filepath.Join(workingDirectory, configuration.SSH_KEYS_PATH)
	sshFileInfos, err := ioutil.ReadDir(sshDirectory)
	if err != nil {
		return [][]string{}, err
	}
	toReturn := [][]string{}
	for _, fileInfo := range sshFileInfos {
		filepath := filepath.Join(sshDirectory, fileInfo.Name())
		if !fileInfo.IsDir() && strings.HasSuffix(filepath, ".pub") {

			// todo scrape the name from the key comment ??
			name := strings.TrimSuffix(fileInfo.Name(), ".pub")
			toReturn = append(toReturn, []string{filepath, name})
		}
	}
	return toReturn, nil
}