package bootstrap import ( "bytes" "context" "crypto/rand" "encoding/base64" "encoding/json" "errors" "fmt" "io" "os" "os/exec" "path/filepath" "reflect" "sort" "strings" "time" "github.com/Pingu-Studio/MaidnCLI/internal/config" "github.com/Pingu-Studio/MaidnCLI/internal/forgejo" ghrepo "github.com/Pingu-Studio/MaidnCLI/internal/github" "github.com/Pingu-Studio/MaidnCLI/internal/openbao" "github.com/Pingu-Studio/MaidnCLI/internal/utils" "gopkg.in/yaml.v3" ) type Mode string const ( Reconcile Mode = "reconcile" Rebuild Mode = "rebuild" ) type Runner struct { Config config.Config Mode Mode ConfirmRebuild bool RegisterWebhook bool } type operationalSecrets struct { Secrets map[string]map[string]string `yaml:"secrets"` } var initializeOpenBao = openbao.Initialize var readOpenBaoRecovery = openbao.ReadRecoveryMaterial var decryptGeneratedSOPS = decryptSOPSFile var writeGeneratedSOPS = writeSOPSEncryptedFile var ensureForgejoWebhook = func(cfg config.Config, repo, webhookURL, authorization string) error { manager := forgejo.NewRepoManager(cfg.Git.BaseURL, cfg.Git.Token, cfg.Git.Owner, cfg.Git.Username, cfg.Flux.ManifestsRepo, cfg.Flux.RepoName, cfg.Flux.Branch, "maidn/bootstrap-"+cfg.ClusterID) return manager.EnsureWebhook(repo, webhookURL, authorization) } var runWebhookCommand = utils.RunCommandQuietOutputInDir var preflight = config.Preflight var runGit = func(dir string, args ...string) ([]byte, error) { command := exec.Command("git", args...) command.Dir = dir return command.Output() } var webhookTargetTimeout = 70 * time.Minute var webhookTargetPollInterval = 2 * time.Second var templateBaseComponents = []string{"snapshot-crds", "democratic-csi", "cert-manager", "cluster-issuers", "gateway-api", "gateway", "monitoring", "openbao", "external-secrets", "external-dns", "tekton", "tekton-triggers"} var requiredClusterKustomizations = []string{"snapshot-crds-kustomization.yaml", "democratic-csi-kustomization.yaml", "cert-manager-kustomization.yaml", "cluster-issuers-kustomization.yaml", "gateway-api-kustomization.yaml", "cilium-kustomization.yaml", "cilium-config-kustomization.yaml", "openbao-kustomization.yaml", "external-secrets-kustomization.yaml", "cnpg-kustomization.yaml", "gateway-kustomization.yaml", "external-dns-kustomization.yaml", "cloudflare-tunnel-kustomization.yaml", "monitoring-kustomization.yaml", "tekton-kustomization.yaml", "tekton-triggers-kustomization.yaml", "cicd-manifests-repo.yaml"} func (r Runner) Run() error { resolved, err := config.Resolve(r.Config) if err != nil { return err } r.Config = resolved if err := preflight(r.Config); err != nil { return fmt.Errorf("preflight: %w", err) } if _, err := os.Stat(r.Config.SOPS.AgeKeyPath); err != nil { return fmt.Errorf("preflight SOPS age identity: %w", err) } if r.Config.Talos.AutoBootstrapFlux { if _, err := os.Stat(r.Config.SOPS.OperationalSecretsPath); err != nil { return fmt.Errorf("preflight operational SOPS secrets: %w", err) } if r.Config.SOPS.RecoveryRecipient == "" { return errors.New("SOPS recoveryRecipient is required") } } if r.Mode == "" { r.Mode = Reconcile } if r.Mode != Reconcile && r.Mode != Rebuild { return fmt.Errorf("unsupported bootstrap mode %q", r.Mode) } if r.Mode == Rebuild && !r.ConfirmRebuild { return fmt.Errorf("rebuild is destructive; rerun with --mode=rebuild --yes") } if r.RegisterWebhook { return r.reconcileWebhook(filepath.Join(r.Config.Git.CloneParent, r.Config.Talos.RepoDirName, r.Config.Talos.GeneratedDir)) } if r.Config.SOPS.RecoveryIdentityPath == "" || r.Config.SOPS.RecoveryBundlePath == "" { return errors.New("SOPS recoveryIdentityPath and recoveryBundlePath are required") } if _, err := os.Stat(r.Config.SOPS.RecoveryIdentityPath); err != nil { return fmt.Errorf("preflight OpenBao recovery identity: %w", err) } if _, err := os.Stat(r.Config.SOPS.RecoveryBundlePath); err != nil { return fmt.Errorf("preflight OpenBao recovery bundle: %w", err) } workspace := r.Config.WorkspaceDir if err := EnsureTemplateRevisions(r.Config); err != nil { return err } if err := config.WriteRedacted(filepath.Join(workspace, "maidn-bootstrap.resolved.yaml"), r.Config); err != nil { return err } manifestsURL := forgejo.CloneURL(r.Config.Git.BaseURL, r.Config.Git.Owner, r.Config.Flux.ManifestsRepo) fluxConfig := ghrepo.BuildFluxConfig(manifestsURL, r.Config.Flux.ManifestsRepo, r.Config.Flux.Branch) cicdTemplateDir := filepath.Join(workspace, "maidn-cicd-cluster-template") manifestsTemplateDir := filepath.Join(workspace, "cicd-deployment-manifests-template") manager := forgejo.NewRepoManager(r.Config.Git.BaseURL, r.Config.Git.Token, r.Config.Git.Owner, r.Config.Git.Username, r.Config.Flux.ManifestsRepo, r.Config.Flux.RepoName, r.Config.Flux.Branch, "maidn/bootstrap-"+r.Config.ClusterID) if err := manager.InitializeAll( func(dir string) error { if err := copyDir(manifestsTemplateDir, dir, false); err != nil { return err } return ensureManifestsKustomizations(dir) }, func(dir string) error { clusterDir := filepath.Join(dir, strings.TrimPrefix(r.Config.Flux.ClusterPath, "./")) if err := copyDir(filepath.Join(cicdTemplateDir, "base"), filepath.Join(dir, "base"), false); err != nil { return err } if err := copyTemplateBaseComponents(cicdTemplateDir, dir); err != nil { return err } if err := copyClusterTemplate(filepath.Join(cicdTemplateDir, "clusters", "template"), clusterDir); err != nil { return err } for _, name := range []string{"external-secrets", "cnpg", "cloudflare-tunnel", "external-dns", "monitoring", "tekton"} { content, err := os.ReadFile(filepath.Join(cicdTemplateDir, "clusters", "template", name+"-kustomization.yaml")) if err != nil { return err } if err := os.WriteFile(filepath.Join(clusterDir, name+"-kustomization.yaml"), content, 0644); err != nil { return err } } csiKustomization, err := os.ReadFile(filepath.Join(cicdTemplateDir, "clusters", "template", "democratic-csi-kustomization.yaml")) if err != nil { return err } if err := os.WriteFile(filepath.Join(clusterDir, "democratic-csi-kustomization.yaml"), csiKustomization, 0644); err != nil { return err } if err := copyDir(filepath.Join(cicdTemplateDir, "base", "cilium"), filepath.Join(dir, "base", "cilium"), true); err != nil { return err } if err := copyDir(filepath.Join(cicdTemplateDir, "base", "cilium-config"), filepath.Join(dir, "base", "cilium-config"), true); err != nil { return err } if err := copyDir(filepath.Join(cicdTemplateDir, "base", "democratic-csi"), filepath.Join(dir, "base", "democratic-csi"), true); err != nil { return err } if err := renderCiliumConfig(filepath.Join(dir, "base", "cilium"), r.Config); err != nil { return err } if err := renderCiliumConfig(filepath.Join(dir, "base", "cilium-config"), r.Config); err != nil { return err } if err := copyAndRenderDeliveryBases(cicdTemplateDir, dir, r.Config); err != nil { return err } if err := writeDemocraticCSISecret(filepath.Join(dir, "base", "democratic-csi", "secret.sops.yaml"), r.Config.DemocraticCSI, r.Config.SOPS.AgeKeyPath); err != nil { return err } openbaoDir := filepath.Join(dir, "base", "openbao") if err := writeOpenBaoUnsealSecret(filepath.Join(openbaoDir, "unseal.sops.yaml"), r.Config.SOPS.RecoveryIdentityPath, r.Config.SOPS.RecoveryBundlePath, r.Config.SOPS.AgeKeyPath); err != nil { return err } if err := ensureOpenBaoUnsealKustomization(filepath.Join(openbaoDir, "kustomization.yaml")); err != nil { return err } if err := ensureClusterKustomizations(clusterDir); err != nil { return err } return ghrepo.WriteFluxStructure(dir, r.Config.Flux.RepoName, r.Config.Flux.ClusterPath, fluxConfig) }, ); err != nil { return err } if manager.MigrationPending { return errors.New("existing repository migration PR created; merge and rerun bootstrap before infrastructure changes") } repoDir := filepath.Join(r.Config.Git.CloneParent, r.Config.Talos.RepoDirName) terraformDir := filepath.Join(repoDir, r.Config.Talos.TerraformDir) generatedDir := filepath.Join(repoDir, r.Config.Talos.GeneratedDir) if err := os.MkdirAll(generatedDir, 0755); err != nil { return err } if err := ensureLifecycleIdentity(terraformDir, r.Config); err != nil { return err } if err := os.WriteFile(filepath.Join(terraformDir, r.Config.Talos.ConfigFileName), []byte(renderTerraformTFVars(r.Config)), 0600); err != nil { return err } if r.Config.Talos.AutoRunTerraform { if err := r.reconcileTerraform(terraformDir); err != nil { return err } } if err := ensureTalosConfig(generatedDir, r.Config); err != nil { return err } if r.Config.Talos.AutoBootstrap { if err := applyTalosConfigs(generatedDir, r.Config); err != nil { return err } if err := bootstrapEtcdIfNeeded(generatedDir, r.Config); err != nil { return err } if err := utils.RunCommandInDir(generatedDir, "talosctl", "kubeconfig", "--talosconfig=./clusterconfig/talosconfig", "--nodes="+r.Config.Talos.KubeconfigNode, "."); err != nil { return err } if err := waitForKubernetesAPI(generatedDir); err != nil { return err } } if r.Config.Talos.AutoBootstrapFlux { if err := installCilium(generatedDir, r.Config); err != nil { return err } if err := utils.RunCommandInDirEnv(generatedDir, []string{"GIT_PASSWORD=" + r.Config.Git.Token}, "flux", "bootstrap", "git", "--url="+forgejo.CloneURL(r.Config.Git.BaseURL, r.Config.Git.Owner, r.Config.Flux.RepoName), "--branch="+r.Config.Flux.Branch, "--path="+r.Config.Flux.ClusterPath, "--cluster-domain="+r.Config.Flux.ClusterDomain, "--username="+r.Config.Git.Username, "--token-auth", "--kubeconfig=kubeconfig"); err != nil { return err } if err := installSOPSKey(generatedDir, r.Config); err != nil { return err } if err := configureFluxSOPS(generatedDir); err != nil { return err } return r.reconcileWebhook(generatedDir) } return nil } func (r Runner) reconcileWebhook(generatedDir string) error { operationalSecrets, err := initializeOpenBao(filepath.Join(generatedDir, "kubeconfig"), r.Config.SOPS.RecoveryRecipient, r.Config.SOPS.RecoveryIdentityPath, r.Config.SOPS.RecoveryBundlePath, r.Config.SOPS.AgeKeyPath, r.Config.SOPS.OperationalSecretsPath) if err != nil { return fmt.Errorf("initialize OpenBao: %w", err) } authorization := operationalSecrets["cicd/forgejo-webhook"]["authorization"] if authorization == "" { return errors.New("operational SOPS secrets requires cicd/forgejo-webhook.authorization") } if err := waitForWebhookTargets(generatedDir, r.Config, authorization); err != nil { return err } if err := ensureForgejoWebhook(r.Config, r.Config.Delivery.AppName, r.Config.Delivery.WebhookURL(), authorization); err != nil { return fmt.Errorf("reconcile Forgejo webhook: %w", err) } return nil } func renderCiliumConfig(dir string, cfg config.Config) error { replacements := strings.NewReplacer( "${CILIUM_K8S_SERVICE_HOST}", cfg.Talos.KubeconfigEndpoint, "${CILIUM_TRAFFIC_INTERFACE}", cfg.Cilium.TrafficInterface, "${CILIUM_LB_START}", cfg.Cilium.LoadBalancerStart, "${CILIUM_LB_END}", cfg.Cilium.LoadBalancerEnd, ) return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil || info.IsDir() { return err } content, err := os.ReadFile(path) if err != nil { return err } return os.WriteFile(path, []byte(replacements.Replace(string(content))), info.Mode()) }) } func renderDeliveryConfig(dir string, cfg config.Config) error { replacements := strings.NewReplacer( "${APP_NAME}", cfg.Delivery.AppName, "${APP_REPO_URL}", cfg.Delivery.AppRepoURL, "${APP_REPO_REF}", cfg.Delivery.AppRepoRef, "${IMAGE_REPOSITORY}", cfg.Delivery.ImageRepository, "${FORGEJO_BASE_URL}", cfg.Git.BaseURL, "${CLUSTER_DOMAIN}", cfg.Flux.ClusterDomain, "${TEKTON_CATALOG_REPO_URL}", cfg.Templates.TektonCatalogRepoURL, "${TEKTON_CATALOG_REPO_REF}", cfg.Templates.TektonCatalogRepoRef, "${WEBHOOK_HOSTNAME}", cfg.Delivery.WebhookHostname, "${WEBHOOK_PATH}", cfg.Delivery.WebhookPath, "${TEKTON_CATALOG_REPO_URL}", cfg.Templates.TektonCatalogRepoURL, "${TEKTON_CATALOG_REPO_REF}", cfg.Templates.TektonCatalogRepoRef, ) return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil || info.IsDir() { return err } content, err := os.ReadFile(path) if err != nil { return err } return os.WriteFile(path, []byte(replacements.Replace(string(content))), info.Mode()) }) } func copyAndRenderDeliveryBases(templateDir, repoDir string, cfg config.Config) error { bases := []string{"gateway", "tekton", "tekton-triggers"} for _, base := range bases { baseDir := filepath.Join(repoDir, "base", base) if err := copyDir(filepath.Join(templateDir, "base", base), baseDir, true); err != nil { return err } } for _, base := range bases { baseDir := filepath.Join(repoDir, "base", base) if err := renderDeliveryConfig(baseDir, cfg); err != nil { return err } } return nil } func writeDemocraticCSISecret(path string, csi config.DemocraticCSIConfig, ageKeyPath string) error { plaintext, err := renderDemocraticCSISecret(csi) if err != nil { return err } if existing, err := decryptSOPSFile(path, ageKeyPath); err == nil && sameYAML(existing, plaintext) { return nil } return writeSOPSEncryptedFile(path, ageKeyPath, plaintext) } func writeOpenBaoUnsealSecret(path, recoveryIdentityPath, recoveryBundlePath, ageKeyPath string) error { material, err := readOpenBaoRecovery(recoveryIdentityPath, recoveryBundlePath) if err != nil { return fmt.Errorf("read OpenBao recovery material: %w", err) } plaintext, err := renderOpenBaoUnsealSecret(material) if err != nil { return err } if _, err := os.Stat(path); err == nil { existing, err := decryptGeneratedSOPS(path, ageKeyPath) if err != nil { return fmt.Errorf("decrypt existing OpenBao unseal secret: %w", err) } if sameYAML(existing, plaintext) { return nil } } else if !os.IsNotExist(err) { return err } return writeGeneratedSOPS(path, ageKeyPath, plaintext) } func renderOpenBaoUnsealSecret(material openbao.RecoveryMaterial) ([]byte, error) { if material.RootToken == "" || material.UnsealThreshold < 1 || len(material.UnsealKeysB64) < material.UnsealThreshold { return nil, errors.New("OpenBao recovery material is incomplete") } data := map[string]string{"root-token": material.RootToken} for index, share := range material.UnsealKeysB64 { if share == "" { return nil, errors.New("OpenBao recovery material contains an invalid unseal key") } data[fmt.Sprintf("unseal-%d", index+1)] = share } return yaml.Marshal(struct { APIVersion string `yaml:"apiVersion"` Kind string `yaml:"kind"` Metadata map[string]string `yaml:"metadata"` Type string `yaml:"type"` StringData map[string]string `yaml:"stringData"` }{ APIVersion: "v1", Kind: "Secret", Metadata: map[string]string{"name": "openbao-unseal", "namespace": "openbao"}, Type: "Opaque", StringData: data, }) } func ensureOpenBaoUnsealKustomization(path string) error { content, err := os.ReadFile(path) if err != nil { return err } var document struct { APIVersion string `yaml:"apiVersion"` Kind string `yaml:"kind"` Resources []string `yaml:"resources"` } decoder := yaml.NewDecoder(bytes.NewReader(content)) if err := decoder.Decode(&document); err != nil { return fmt.Errorf("parse OpenBao Kustomization: %w", err) } if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { return errors.New("OpenBao Kustomization must contain one YAML document") } if document.APIVersion != "kustomize.config.k8s.io/v1beta1" || document.Kind != "Kustomization" || len(document.Resources) == 0 { return errors.New("OpenBao Kustomization must define resources") } count := 0 for _, resource := range document.Resources { if resource == "" { return errors.New("OpenBao Kustomization contains an empty resource") } if resource == "unseal.sops.yaml" { count++ } } if count > 1 { return errors.New("OpenBao Kustomization references unseal.sops.yaml more than once") } if count == 1 { return nil } if !bytes.HasSuffix(content, []byte("\n")) { content = append(content, '\n') } return os.WriteFile(path, append(content, []byte(" - unseal.sops.yaml\n")...), 0644) } func WriteOperationalSecrets(path, ageKeyPath string, secrets map[string]map[string]string) error { plaintext, err := yaml.Marshal(operationalSecrets{Secrets: secrets}) if err != nil { return err } return writeSOPSEncrypted(path, ageKeyPath, plaintext, "^(secrets)$") } func UpsertOperationalSecret(path, ageKeyPath, secretPath, key, value string) error { secrets, err := ReadOperationalSecrets(path, ageKeyPath) if err != nil { return err } values := secrets[secretPath] if values == nil { values = map[string]string{} secrets[secretPath] = values } values[key] = value return WriteOperationalSecrets(path, ageKeyPath, secrets) } func ReadOperationalSecrets(path, ageKeyPath string) (map[string]map[string]string, error) { plaintext, err := decryptSOPSFile(path, ageKeyPath) if err != nil { return nil, fmt.Errorf("decrypt operational SOPS secrets: %w", err) } decoder := yaml.NewDecoder(bytes.NewReader(plaintext)) decoder.KnownFields(true) var document operationalSecrets if err := decoder.Decode(&document); err != nil { return nil, fmt.Errorf("parse operational SOPS secrets: %w", err) } if err := decoder.Decode(&operationalSecrets{}); !errors.Is(err, io.EOF) { return nil, errors.New("operational SOPS secrets must contain one YAML document") } if document.Secrets == nil { return nil, errors.New("operational SOPS secrets requires a secrets mapping") } for secretPath, values := range document.Secrets { if values == nil { return nil, fmt.Errorf("operational SOPS secret %q must be a mapping", secretPath) } } return document.Secrets, nil } func ForgejoRegistryDockerConfig(imageRepository, username, token string) (string, error) { registryHost := strings.Split(strings.TrimSpace(imageRepository), "/")[0] if registryHost == "" || username == "" || token == "" { return "", errors.New("delivery registry host, Forgejo username, and token are required") } config, err := json.Marshal(map[string]map[string]map[string]string{ "auths": { registryHost: {"auth": base64.StdEncoding.EncodeToString([]byte(username + ":" + token))}, }, }) if err != nil { return "", err } return string(config), nil } func InitializeOpenBao(cfg config.Config) error { kubeconfig := filepath.Join(cfg.Git.CloneParent, cfg.Talos.RepoDirName, cfg.Talos.GeneratedDir, "kubeconfig") if _, err := initializeOpenBao(kubeconfig, cfg.SOPS.RecoveryRecipient, cfg.SOPS.RecoveryIdentityPath, cfg.SOPS.RecoveryBundlePath, cfg.SOPS.AgeKeyPath, cfg.SOPS.OperationalSecretsPath); err != nil { return fmt.Errorf("initialize OpenBao: %w", err) } return nil } func NewWebhookAuthorization() (string, error) { value := make([]byte, 32) if _, err := rand.Read(value); err != nil { return "", err } return "Bearer " + base64.RawURLEncoding.EncodeToString(value), nil } func writeSOPSEncryptedFile(path, ageKeyPath string, plaintext []byte) error { return writeSOPSEncrypted(path, ageKeyPath, plaintext, "^(data|stringData)$") } func writeSOPSEncrypted(path, ageKeyPath string, plaintext []byte, encryptedRegex string) error { if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { return err } recipient, err := exec.Command("age-keygen", "-y", ageKeyPath).Output() if err != nil { return fmt.Errorf("derive SOPS age recipient: %w", err) } command := exec.Command("sops", "encrypt", "--age", strings.TrimSpace(string(recipient)), "--encrypted-regex", encryptedRegex, "--input-type", "yaml", "--output-type", "yaml", "--filename-override", filepath.Base(path)) command.Stdin = bytes.NewReader(plaintext) encrypted, err := command.Output() if err != nil { return fmt.Errorf("encrypt SOPS file: %w", err) } return os.WriteFile(path, encrypted, 0600) } func decryptSOPSFile(path, ageKeyPath string) ([]byte, error) { command := exec.Command("sops", "--decrypt", "--output-type", "yaml", path) command.Env = append(os.Environ(), "SOPS_AGE_KEY_FILE="+ageKeyPath) return command.Output() } func sameYAML(left, right []byte) bool { var leftValue any var rightValue any return yaml.Unmarshal(left, &leftValue) == nil && yaml.Unmarshal(right, &rightValue) == nil && reflect.DeepEqual(leftValue, rightValue) } func renderDemocraticCSISecret(csi config.DemocraticCSIConfig) ([]byte, error) { return yaml.Marshal(struct { APIVersion string `yaml:"apiVersion"` Kind string `yaml:"kind"` Metadata map[string]any `yaml:"metadata"` Type string `yaml:"type"` StringData map[string]string `yaml:"stringData"` }{ APIVersion: "v1", Kind: "Secret", Metadata: map[string]any{ "name": "democratic-csi-secrets", "namespace": "democratic-storage", "labels": map[string]string{ "reconcile.fluxcd.io/watch": "Enabled", }, }, Type: "Opaque", StringData: map[string]string{ "truenas-api-key": csi.TrueNASAPIKey, "truenas-host": csi.TrueNASHost, "target-portal": csi.TargetPortal, "share-host": csi.ShareHost, "dataset-parent-nfs": csi.DatasetParentNFS, "dataset-snapshots-nfs": csi.DatasetSnapshotsNFS, "allowed-networks": csi.AllowedNetworks, "name-suffix": csi.NameSuffix, "portal-group": csi.PortalGroup, "initiator-group": csi.InitiatorGroup, }, }) } func waitForWebhookTargets(dir string, cfg config.Config, authorization string) error { if err := waitForWebhookAuthorization(dir, authorization); err != nil { return err } resources := []string{"deployment/el-" + cfg.Delivery.AppName, "pipeline/" + cfg.Delivery.AppName} for _, resource := range resources { deadline := time.Now().Add(webhookTargetTimeout) for time.Now().Before(deadline) { if _, err := runWebhookCommand(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err == nil { break } time.Sleep(webhookTargetPollInterval) } if _, err := runWebhookCommand(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err != nil { return fmt.Errorf("wait for %s before registering Forgejo webhook", resource) } } return nil } func waitForWebhookAuthorization(dir, authorization string) error { deadline := time.Now().Add(webhookTargetTimeout) for { output, err := runWebhookCommand(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", "secret/forgejo-webhook", "-o=jsonpath={.data.authorization}") if err == nil { observed, decodeErr := base64.StdEncoding.DecodeString(strings.TrimSpace(string(output))) if decodeErr == nil && string(observed) == authorization { return nil } } if !time.Now().Before(deadline) { return errors.New("ExternalSecret target Secret forgejo-webhook did not refresh within the timeout; Forgejo webhook was not updated. Wait for External Secrets to recover, then safely rerun cicd-tool bootstrap --config --register-webhook") } time.Sleep(webhookTargetPollInterval) } } func ensureClusterKustomizations(clusterDir string) error { path := filepath.Join(clusterDir, "kustomization.yaml") content, err := os.ReadFile(path) if os.IsNotExist(err) { // Existing Flux roots may intentionally use recursive discovery. Do not // introduce a partial root Kustomization that could prune its resources. return nil } if err != nil { return err } updated := string(content) updated = strings.ReplaceAll(updated, " - bootstrap-secrets.sops.yaml\n", "") for _, resource := range requiredClusterKustomizations { if !strings.Contains(updated, resource) { updated += " - " + resource + "\n" } } if updated == string(content) { return nil } return os.WriteFile(path, []byte(updated), 0644) } func copyTemplateBaseComponents(templateDir, repoDir string) error { for _, component := range templateBaseComponents { if err := copyDir(filepath.Join(templateDir, "base", component), filepath.Join(repoDir, "base", component), true); err != nil { return err } } return nil } func copyClusterTemplate(source, destination string) error { entries, err := os.ReadDir(destination) if err != nil && !os.IsNotExist(err) { return err } if len(entries) == 0 { return copyDir(source, destination, false) } if _, err := os.Stat(filepath.Join(destination, "kustomization.yaml")); err == nil { return copyDir(source, destination, false) } else if !os.IsNotExist(err) { return err } return copyDirExcept(source, destination, false, map[string]bool{"kustomization.yaml": true}) } func ensureManifestsKustomizations(dir string) error { for _, environment := range []string{"previews", "staging", "production"} { path := filepath.Join(dir, "apps", environment, "kustomization.yaml") content, err := os.ReadFile(path) if err == nil { if strings.Contains(string(content), "resources:") { continue } if err := os.WriteFile(path, append(content, []byte("resources:\n")...), 0644); err != nil { return err } continue } else if !os.IsNotExist(err) { return err } if err := os.WriteFile(path, []byte("apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nresources:\n"), 0644); err != nil { return err } } return nil } func installCilium(dir string, cfg config.Config) error { helmDir := filepath.Join(dir, ".helm") if err := os.MkdirAll(helmDir, 0755); err != nil { return err } return utils.RunCommandInDir(dir, "helm", "upgrade", "--install", "cilium", "cilium", "--repo=https://helm.cilium.io", "--version=1.19.6", "--repository-config="+filepath.Join(helmDir, "repositories.yaml"), "--repository-cache="+helmDir, "--namespace=kube-system", "--create-namespace", "--kubeconfig=kubeconfig", "--wait", "--timeout=5m", "--set=kubeProxyReplacement=true", "--set=ipam.mode=kubernetes", "--set=k8sServiceHost=localhost", "--set=k8sServicePort=7445", "--set=cgroup.autoMount.enabled=false", "--set=cgroup.hostRoot=/sys/fs/cgroup", "--set=bpf.hostLegacyRouting=true", "--set=securityContext.capabilities.ciliumAgent={CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}", "--set=securityContext.capabilities.cleanCiliumState={NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}", "--set=envoy.enabled=true", "--set=gatewayAPI.enabled=true", "--set=l2announcements.enabled=true", "--set=rollOutCiliumPods=true", "--set=operator.replicas=1", "--set=operator.rollOutPods=true") } func copyDir(source, destination string, overwrite bool) error { return copyDirExcept(source, destination, overwrite, nil) } func copyDirExcept(source, destination string, overwrite bool, excluded map[string]bool) error { return filepath.Walk(source, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if info.IsDir() && info.Name() == ".git" { return filepath.SkipDir } relative, err := filepath.Rel(source, path) if err != nil || relative == "." { return err } if excluded[filepath.ToSlash(relative)] { if info.IsDir() { return filepath.SkipDir } return nil } target := filepath.Join(destination, relative) if info.IsDir() { return os.MkdirAll(target, 0755) } if _, err := os.Stat(target); err == nil && !overwrite { return nil } input, err := os.Open(path) if err != nil { return err } flags := os.O_WRONLY | os.O_CREATE | os.O_EXCL if overwrite { flags = os.O_WRONLY | os.O_CREATE | os.O_TRUNC } output, err := os.OpenFile(target, flags, info.Mode()) if err != nil { _ = input.Close() return err } _, copyErr := io.Copy(output, input) closeInputErr := input.Close() closeOutputErr := output.Close() if copyErr != nil { return copyErr } if closeInputErr != nil { return closeInputErr } return closeOutputErr }) } func applyTalosConfigs(dir string, cfg config.Config) error { for _, node := range cfg.Talos.Nodes { configFile := filepath.Join("clusterconfig", fmt.Sprintf("%s-%s.yaml", cfg.Talos.Cluster.Name, node.Name)) nodeAddress := node.Networks[0].IP secureArgs := []string{"apply-config", "--talosconfig=./clusterconfig/talosconfig", "--nodes=" + nodeAddress, "--endpoints=" + cfg.Talos.BootstrapEndpoint, "--file=" + configFile} if _, err := runTalosctlOutput(dir, "get", "machinestatus", "--talosconfig=./clusterconfig/talosconfig", "--nodes="+nodeAddress, "--endpoints="+cfg.Talos.BootstrapEndpoint, "--output=json"); err == nil { if err := utils.RunCommandInDir(dir, "talosctl", secureArgs...); err != nil { return fmt.Errorf("apply Talos config to %s: %w", node.Name, err) } } else { if _, maintenanceErr := runTalosctlOutput(dir, "get", "machinestatus", "--insecure", "--nodes="+nodeAddress, "--endpoints="+nodeAddress, "--output=json"); maintenanceErr == nil { if err := utils.RunCommandInDir(dir, "talosctl", "apply-config", "--insecure", "--nodes="+nodeAddress, "--endpoints="+nodeAddress, "--file="+configFile); err != nil { return fmt.Errorf("apply initial Talos config to %s: %w", node.Name, err) } } else { if err := waitForTalosAPI(dir, cfg, nodeAddress); err != nil { return fmt.Errorf("determine Talos state for %s: %w", node.Name, err) } if err := utils.RunCommandInDir(dir, "talosctl", secureArgs...); err != nil { return fmt.Errorf("apply Talos config to %s: %w", node.Name, err) } } } if err := waitForTalosReboot(dir, cfg, nodeAddress); err != nil { return fmt.Errorf("wait for %s: %w", node.Name, err) } } return nil } func bootstrapEtcdIfNeeded(dir string, cfg config.Config) error { if _, err := runTalosctlOutput(dir, "etcd", "status", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode); err == nil { return nil } if err := waitForTalosAPI(dir, cfg, cfg.Talos.BootstrapNode); err != nil { return err } ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() command := exec.CommandContext(ctx, "talosctl", "bootstrap", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode) command.Dir = dir command.Stdout = os.Stdout command.Stderr = os.Stderr if err := command.Run(); err != nil { if ctx.Err() != nil { return errors.New("etcd bootstrap request timed out") } return fmt.Errorf("bootstrap etcd: %w", err) } deadline := time.Now().Add(2 * time.Minute) for time.Now().Before(deadline) { if _, err := runTalosctlOutput(dir, "etcd", "status", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode); err == nil { return nil } time.Sleep(2 * time.Second) } return errors.New("etcd did not become healthy after bootstrap") } func waitForTalosReboot(dir string, cfg config.Config, node string) error { return waitForTalosAPI(dir, cfg, node) } func waitForTalosAPI(dir string, cfg config.Config, node string) error { deadline := time.Now().Add(5 * time.Minute) for time.Now().Before(deadline) { if _, err := runTalosctlOutput(dir, "get", "machinestatus", "--output=json", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+node, "--nodes="+node); err == nil { return nil } time.Sleep(2 * time.Second) } return fmt.Errorf("Talos API did not return after applying its machine configuration") } func runTalosctlOutput(dir string, args ...string) ([]byte, error) { ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() command := exec.CommandContext(ctx, "talosctl", args...) command.Dir = dir return command.Output() } func waitForKubernetesAPI(dir string) error { deadline := time.Now().Add(5 * time.Minute) for time.Now().Before(deadline) { if _, err := utils.RunCommandQuietOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "get", "--raw=/readyz"); err == nil { return nil } time.Sleep(2 * time.Second) } return errors.New("Kubernetes API did not become ready after Talos bootstrap") } type lifecycle struct { ClusterID string `yaml:"clusterId"` ClusterName string `yaml:"clusterName"` } func ensureLifecycleIdentity(terraformDir string, cfg config.Config) error { statePath := filepath.Join(terraformDir, "terraform.tfstate") lifecyclePath := filepath.Join(terraformDir, ".maidn", "lifecycle.yaml") data, err := os.ReadFile(lifecyclePath) if err == nil { var current lifecycle if err := yaml.Unmarshal(data, ¤t); err != nil { return fmt.Errorf("read lifecycle metadata: %w", err) } if current.ClusterID != cfg.ClusterID || current.ClusterName != cfg.Talos.Cluster.Name { return fmt.Errorf("terraform checkout belongs to cluster %q; use its matching configuration", current.ClusterID) } return nil } if !os.IsNotExist(err) { return err } if state, err := os.ReadFile(statePath); err == nil && !strings.Contains(string(state), cfg.Talos.Cluster.Name) { return errors.New("terraform state exists but does not match clusterName; use an explicit recovery checkout") } else if err != nil && !os.IsNotExist(err) { return err } if err := os.MkdirAll(filepath.Dir(lifecyclePath), 0700); err != nil { return err } data, err = yaml.Marshal(lifecycle{ClusterID: cfg.ClusterID, ClusterName: cfg.Talos.Cluster.Name}) if err != nil { return err } return os.WriteFile(lifecyclePath, data, 0600) } func (r Runner) reconcileTerraform(terraformDir string) error { environment := []string{"TF_VAR_proxmox_api_token=" + r.Config.Talos.Proxmox.APITokenID + "=" + r.Config.Talos.Proxmox.APITokenSecret} if err := utils.RunCommandInDirEnv(terraformDir, environment, "terraform", "init", "-input=false"); err != nil { return err } if r.Mode == Rebuild { if err := utils.RunCommandInDirEnv(terraformDir, environment, "terraform", "destroy", "-auto-approve", "-input=false", "-target=proxmox_virtual_environment_vm.vm"); err != nil { return err } } planPath, err := terraformPlanPath(terraformDir, r.Config.ClusterID) if err != nil { return err } defer os.Remove(planPath) if err := utils.RunCommandInDirEnv(terraformDir, environment, "terraform", "plan", "-input=false", "-out="+planPath); err != nil { return err } return utils.RunCommandInDirEnv(terraformDir, environment, "terraform", "apply", "-input=false", "-auto-approve", planPath) } func terraformPlanPath(terraformDir, clusterID string) (string, error) { return filepath.Abs(filepath.Join(terraformDir, clusterID+".tfplan")) } func installSOPSKey(dir string, cfg config.Config) error { ageKeyPath, err := filepath.Abs(cfg.SOPS.AgeKeyPath) if err != nil { return fmt.Errorf("resolve SOPS age identity: %w", err) } if _, err := os.Stat(ageKeyPath); err != nil { return fmt.Errorf("read SOPS age identity: %w", err) } manifest, err := utils.RunCommandOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "flux-system", "create", "secret", "generic", "sops-age", "--from-file=age.agekey="+ageKeyPath, "--dry-run=client", "-o", "yaml") if err != nil { return err } return utils.RunCommandInputInDir(dir, manifest, "kubectl", "--kubeconfig=kubeconfig", "apply", "-f", "-") } func configureFluxSOPS(dir string) error { patch := `{"spec":{"decryption":{"provider":"sops","secretRef":{"name":"sops-age"}}}}` return utils.RunCommandInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "flux-system", "patch", "kustomization", "flux-system", "--type=merge", "-p", patch) } type templateRevision struct { Repository string `yaml:"repository"` Ref string `yaml:"ref"` Commit string `yaml:"commit"` } type templateRevisionLock struct { Version int `yaml:"version"` CICD templateRevision `yaml:"cicd"` Manifests templateRevision `yaml:"manifests"` Talos templateRevision `yaml:"talos"` } type templateCheckout struct { Dir string Repository string Ref string } func ensureTemplateRevisions(workspace string, cfg config.Config) error { lockPath := filepath.Join(workspace, "maidn-template-revisions.yaml") checkouts := []templateCheckout{ {filepath.Join(workspace, "maidn-cicd-cluster-template"), cfg.Templates.CICDRepoURL, cfg.Templates.CICDRepoRef}, {filepath.Join(workspace, "cicd-deployment-manifests-template"), cfg.Templates.ManifestsRepoURL, cfg.Templates.ManifestsRepoRef}, {filepath.Join(cfg.Git.CloneParent, cfg.Talos.RepoDirName), cfg.Templates.TalosRepoURL, cfg.Templates.TalosRepoRef}, } lock, err := readTemplateRevisionLock(lockPath) if err == nil { revisions := []templateRevision{lock.CICD, lock.Manifests, lock.Talos} for index, checkout := range checkouts { if !sameTemplateSource(revisions[index], checkout) { return errors.New("configured template source or ref differs from its workspace revision lock; use a new empty workspaceDir to intentionally refresh templates") } if _, err := checkoutTemplateRevision(checkout, revisions[index].Commit); err != nil { return errors.New("locked template revision cannot be resolved; restore the locked commit or use a new empty workspaceDir to intentionally refresh templates") } } return nil } if !os.IsNotExist(err) { return errors.New("template revision lock is invalid; use a new empty workspaceDir to intentionally refresh templates") } revisions := make([]templateRevision, len(checkouts)) for index, checkout := range checkouts { if config.RedactURL(checkout.Repository) == "" { return errors.New("configured template source cannot be safely recorded; use a standard repository URL without embedded query credentials") } commit, err := checkoutTemplateRevision(checkout, "") if err != nil { return errors.New("configured template revision cannot be resolved; correct the template source or ref, then rerun bootstrap") } revisions[index] = templateRevision{Repository: config.RedactURL(checkout.Repository), Ref: checkout.Ref, Commit: commit} } return writeTemplateRevisionLock(lockPath, templateRevisionLock{Version: 1, CICD: revisions[0], Manifests: revisions[1], Talos: revisions[2]}) } // EnsureTemplateRevisions records or restores the workspace's template commits. func EnsureTemplateRevisions(cfg config.Config) error { if err := os.MkdirAll(cfg.WorkspaceDir, 0755); err != nil { return err } if err := os.MkdirAll(cfg.Git.CloneParent, 0755); err != nil { return err } return ensureTemplateRevisions(cfg.WorkspaceDir, cfg) } func readTemplateRevisionLock(path string) (templateRevisionLock, error) { var lock templateRevisionLock data, err := os.ReadFile(path) if err != nil { return lock, err } decoder := yaml.NewDecoder(bytes.NewReader(data)) decoder.KnownFields(true) if err := decoder.Decode(&lock); err != nil { return lock, err } if err := decoder.Decode(&templateRevisionLock{}); !errors.Is(err, io.EOF) { return lock, errors.New("multiple YAML documents") } if lock.Version != 1 || !validTemplateRevision(lock.CICD) || !validTemplateRevision(lock.Manifests) || !validTemplateRevision(lock.Talos) { return lock, errors.New("invalid template revision lock") } return lock, nil } func writeTemplateRevisionLock(path string, lock templateRevisionLock) error { data, err := yaml.Marshal(lock) if err != nil { return err } file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } if _, err := file.Write(data); err != nil { _ = file.Close() return err } return file.Close() } func sameTemplateSource(revision templateRevision, checkout templateCheckout) bool { return validTemplateRevision(revision) && revision.Repository == config.RedactURL(checkout.Repository) && revision.Ref == checkout.Ref } func validTemplateRevision(revision templateRevision) bool { if revision.Repository == "" || revision.Repository == "" || revision.Ref == "" || len(revision.Commit) < 40 || len(revision.Commit) > 64 { return false } for _, character := range revision.Commit { if !((character >= '0' && character <= '9') || (character >= 'a' && character <= 'f')) { return false } } return true } func checkoutTemplateRevision(checkout templateCheckout, lockedCommit string) (string, error) { repository := config.RedactURL(checkout.Repository) if repository == "" { return "", errors.New("template source cannot be safely used") } if info, err := os.Stat(checkout.Dir); os.IsNotExist(err) { if _, err := runGit("", "clone", "--no-checkout", repository, checkout.Dir); err != nil { return "", err } } else if err != nil || !info.IsDir() { return "", errors.New("template checkout is not a directory") } else { inside, err := runGit(checkout.Dir, "rev-parse", "--is-inside-work-tree") if err != nil || strings.TrimSpace(string(inside)) != "true" { return "", errors.New("template checkout is not a Git work tree") } origin, err := runGit(checkout.Dir, "remote", "get-url", "origin") if err != nil || config.RedactURL(strings.TrimSpace(string(origin))) != repository { return "", errors.New("template checkout source does not match configuration") } status, err := runGit(checkout.Dir, "status", "--porcelain") if err != nil || strings.TrimSpace(string(status)) != "" { return "", errors.New("template checkout has uncommitted changes") } } target := checkout.Ref if lockedCommit != "" { target = lockedCommit } if _, err := runGit(checkout.Dir, "fetch", "origin", target); err != nil { return "", err } commit, err := runGit(checkout.Dir, "rev-parse", "--verify", "FETCH_HEAD^{commit}") if err != nil { return "", err } commitID := strings.TrimSpace(string(commit)) if !validTemplateRevision(templateRevision{Repository: "source", Ref: "ref", Commit: commitID}) || (lockedCommit != "" && commitID != lockedCommit) { return "", errors.New("template ref did not resolve to the expected commit") } if _, err := runGit(checkout.Dir, "checkout", "--detach", commitID); err != nil { return "", err } head, err := runGit(checkout.Dir, "rev-parse", "--verify", "HEAD^{commit}") if err != nil || strings.TrimSpace(string(head)) != commitID { return "", errors.New("template checkout did not reach the expected commit") } return commitID, nil } func ensureTalosConfig(generatedDir string, cfg config.Config) error { talhelperPath := "talhelper" secretPath := filepath.Join(generatedDir, "talsecret.yaml") if _, err := os.Stat(secretPath); os.IsNotExist(err) { legacyPath := filepath.Join(generatedDir, "talsecret.sops.yaml") if _, legacyErr := os.Stat(legacyPath); legacyErr == nil { secretPath = legacyPath } else { secrets, err := utils.RunCommandOutputInDir(generatedDir, talhelperPath, "gensecret") if err != nil { return err } if err := os.WriteFile(secretPath, secrets, 0600); err != nil { return err } } } return utils.RunCommandInDir(generatedDir, talhelperPath, "genconfig", "--config-file", "talconfig.yaml", "--secret-file", filepath.Base(secretPath)) } func renderTerraformTFVars(cfg config.Config) string { var builder strings.Builder builder.WriteString(fmt.Sprintf("proxmox_api_url = %q\n", cfg.Talos.Proxmox.APIURL)) builder.WriteString(fmt.Sprintf("proxmox_node = %q\n", cfg.Talos.Proxmox.DefaultNode)) builder.WriteString(fmt.Sprintf("proxmox_pool = %q\n", cfg.Talos.Proxmox.Pool)) builder.WriteString(fmt.Sprintf("cluster_name = %q\n", cfg.Talos.Cluster.Name)) builder.WriteString(fmt.Sprintf("cluster_domain = %q\n", cfg.Talos.Cluster.Domain)) builder.WriteString(fmt.Sprintf("talos_factory_schematic_id = %q\n", cfg.Talos.Image.SchematicID)) builder.WriteString(fmt.Sprintf("talos_version = %q\n", cfg.Talos.Image.TalosVersion)) builder.WriteString(fmt.Sprintf("kubernetes_version = %q\n", cfg.Talos.Image.KubernetesVersion)) builder.WriteString("cni_name = \"none\"\n") isoStorage := cfg.Talos.Image.Storage if isoStorage == cfg.Talos.Cluster.DiskStorage { isoStorage = "local" } isoFilename := fmt.Sprintf("talos-%s-%s-%s.iso", strings.TrimPrefix(cfg.Talos.Image.TalosVersion, "v"), cfg.Talos.Image.SchematicID[:12], cfg.Talos.Image.Architecture) builder.WriteString(fmt.Sprintf("talos_iso_file = %q\n", fmt.Sprintf("%s:iso/%s", isoStorage, isoFilename))) builder.WriteString(fmt.Sprintf("talos_image_update_mode = %q\n", cfg.Talos.Image.UpdateMode)) builder.WriteString(fmt.Sprintf("talos_image_storage = %q\n", isoStorage)) builder.WriteString(fmt.Sprintf("talos_image_architecture = %q\n", cfg.Talos.Image.Architecture)) builder.WriteString(fmt.Sprintf("disk_storage = %q\n", cfg.Talos.Cluster.DiskStorage)) builder.WriteString(fmt.Sprintf("additional_disk_storage = %q\n", cfg.Talos.Cluster.AdditionalStorage)) builder.WriteString(fmt.Sprintf("dns_servers = [%s]\n", quoteList(cfg.Talos.Cluster.DNSServers))) builder.WriteString(fmt.Sprintf("control_plane_vip = %q\n", cfg.Talos.Cluster.ControlPlaneVIP)) builder.WriteString("node_interfaces = {\n") for _, node := range sortedKeys(cfg.Talos.Proxmox.NodeInterfaces) { iface := cfg.Talos.Proxmox.NodeInterfaces[node] builder.WriteString(fmt.Sprintf(" %q = %q\n", node, iface)) } builder.WriteString("}\n") builder.WriteString("node_addresses = {\n") for _, node := range sortedKeys(cfg.Talos.Proxmox.NodeAddresses) { addr := cfg.Talos.Proxmox.NodeAddresses[node] builder.WriteString(fmt.Sprintf(" %q = %q\n", node, addr)) } builder.WriteString("}\n") builder.WriteString(fmt.Sprintf("create_vlan_interface = %t\n", cfg.Talos.Cluster.CreateVLANInterface)) builder.WriteString(fmt.Sprintf("manage_network_bridges = %t\n", cfg.Talos.Cluster.ManageNetworkBridges)) builder.WriteString("image_cache_proxy = { enabled = false, ip = \"\", port = 3128 }\n") builder.WriteString("nodes = [\n") for _, node := range cfg.Talos.Nodes { builder.WriteString(" {\n") builder.WriteString(fmt.Sprintf(" name = %q\n", node.Name)) builder.WriteString(fmt.Sprintf(" vmid = %d\n", node.VMID)) builder.WriteString(fmt.Sprintf(" role = %q\n", node.Role)) builder.WriteString(fmt.Sprintf(" cores = %d\n", node.Cores)) builder.WriteString(fmt.Sprintf(" memory = %d\n", node.Memory)) builder.WriteString(fmt.Sprintf(" disk_size = %q\n", node.DiskSize)) if node.AdditionalDiskSize != "" { builder.WriteString(fmt.Sprintf(" additional_disk_size = %q\n", node.AdditionalDiskSize)) } builder.WriteString(fmt.Sprintf(" tags = [%s]\n", quoteList(node.Tags))) builder.WriteString(fmt.Sprintf(" proxmox_node = %q\n", node.ProxmoxNode)) builder.WriteString(" networks = [\n") for _, net := range node.Networks { builder.WriteString(" {\n") builder.WriteString(fmt.Sprintf(" mac_address = %q\n", net.MACAddress)) builder.WriteString(fmt.Sprintf(" cidr = %q\n", net.CIDR)) if net.IP != "" { builder.WriteString(fmt.Sprintf(" ip = %q\n", net.IP)) } if net.Gateway != "" { builder.WriteString(fmt.Sprintf(" gateway = %q\n", net.Gateway)) } builder.WriteString(fmt.Sprintf(" vlan_id = %d\n", net.VLANID)) builder.WriteString(" },\n") } builder.WriteString(" ]\n") builder.WriteString(" },\n") } builder.WriteString("]\n") return builder.String() } func quoteList(values []string) string { quoted := make([]string, 0, len(values)) for _, value := range values { quoted = append(quoted, fmt.Sprintf("%q", value)) } return strings.Join(quoted, ", ") } func sortedKeys(values map[string]string) []string { keys := make([]string, 0, len(values)) for key := range values { keys = append(keys, key) } sort.Strings(keys) return keys }