727 lines
29 KiB
Go
727 lines
29 KiB
Go
package bootstrap
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/rand"
|
|
"encoding/base64"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"reflect"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/Pingu-Studio/MaidnCLI/internal/config"
|
|
"github.com/Pingu-Studio/MaidnCLI/internal/forgejo"
|
|
ghrepo "github.com/Pingu-Studio/MaidnCLI/internal/github"
|
|
"github.com/Pingu-Studio/MaidnCLI/internal/openbao"
|
|
"github.com/Pingu-Studio/MaidnCLI/internal/utils"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
type Mode string
|
|
|
|
const (
|
|
Reconcile Mode = "reconcile"
|
|
Rebuild Mode = "rebuild"
|
|
)
|
|
|
|
type Runner struct {
|
|
Config config.Config
|
|
Mode Mode
|
|
ConfirmRebuild bool
|
|
}
|
|
|
|
func (r Runner) Run() error {
|
|
resolved, err := config.Resolve(r.Config)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
r.Config = resolved
|
|
if err := config.Preflight(r.Config); err != nil {
|
|
return fmt.Errorf("preflight: %w", err)
|
|
}
|
|
if _, err := os.Stat(r.Config.SOPS.AgeKeyPath); err != nil {
|
|
return fmt.Errorf("preflight SOPS age identity: %w", err)
|
|
}
|
|
if r.Config.Talos.AutoBootstrapFlux {
|
|
if _, err := os.Stat(r.Config.SOPS.OperationalSecretsPath); err != nil {
|
|
return fmt.Errorf("preflight operational SOPS secrets: %w", err)
|
|
}
|
|
if r.Config.SOPS.RecoveryRecipient == "" || r.Config.SOPS.RecoveryIdentityPath == "" || r.Config.SOPS.RecoveryBundlePath == "" {
|
|
return errors.New("SOPS recoveryRecipient, recoveryIdentityPath, and recoveryBundlePath are required")
|
|
}
|
|
}
|
|
if r.Mode == "" {
|
|
r.Mode = Reconcile
|
|
}
|
|
if r.Mode != Reconcile && r.Mode != Rebuild {
|
|
return fmt.Errorf("unsupported bootstrap mode %q", r.Mode)
|
|
}
|
|
if r.Mode == Rebuild && !r.ConfirmRebuild {
|
|
return fmt.Errorf("rebuild is destructive; rerun with --mode=rebuild --yes")
|
|
}
|
|
|
|
workspace := r.Config.WorkspaceDir
|
|
if err := os.MkdirAll(workspace, 0755); err != nil {
|
|
return err
|
|
}
|
|
if err := config.WriteRedacted(filepath.Join(workspace, "maidn-bootstrap.resolved.yaml"), r.Config); err != nil {
|
|
return err
|
|
}
|
|
if err := os.MkdirAll(r.Config.Git.CloneParent, 0755); err != nil {
|
|
return err
|
|
}
|
|
|
|
manifestsURL := forgejo.CloneURL(r.Config.Git.BaseURL, r.Config.Git.Owner, r.Config.Flux.ManifestsRepo)
|
|
fluxConfig := ghrepo.BuildFluxConfig(manifestsURL, r.Config.Flux.ManifestsRepo, r.Config.Flux.Branch)
|
|
cicdTemplateDir := filepath.Join(workspace, "maidn-cicd-cluster-template")
|
|
if err := ensureRepo(cicdTemplateDir, r.Config.Templates.CICDRepoURL, r.Config.Templates.CICDRepoRef); err != nil {
|
|
return err
|
|
}
|
|
manifestsTemplateDir := filepath.Join(workspace, "cicd-deployment-manifests-template")
|
|
if err := ensureRepo(manifestsTemplateDir, r.Config.Templates.ManifestsRepoURL, r.Config.Templates.ManifestsRepoRef); err != nil {
|
|
return err
|
|
}
|
|
manager := forgejo.NewRepoManager(r.Config.Git.BaseURL, r.Config.Git.Token, r.Config.Git.Owner, r.Config.Git.Username, r.Config.Flux.ManifestsRepo, r.Config.Flux.RepoName, r.Config.Flux.Branch, "maidn/bootstrap-"+r.Config.ClusterID)
|
|
if err := manager.InitializeAll(
|
|
func(dir string) error {
|
|
if err := copyDir(manifestsTemplateDir, dir, false); err != nil {
|
|
return err
|
|
}
|
|
return ensureManifestsKustomizations(dir)
|
|
},
|
|
func(dir string) error {
|
|
clusterDir := filepath.Join(dir, strings.TrimPrefix(r.Config.Flux.ClusterPath, "./"))
|
|
if err := copyDir(filepath.Join(cicdTemplateDir, "base"), filepath.Join(dir, "base"), false); err != nil {
|
|
return err
|
|
}
|
|
if err := copyClusterTemplate(filepath.Join(cicdTemplateDir, "clusters", "template"), clusterDir); err != nil {
|
|
return err
|
|
}
|
|
if err := copyDir(filepath.Join(cicdTemplateDir, "base", "cilium"), filepath.Join(dir, "base", "cilium"), true); err != nil {
|
|
return err
|
|
}
|
|
if err := copyDir(filepath.Join(cicdTemplateDir, "base", "cilium-config"), filepath.Join(dir, "base", "cilium-config"), true); err != nil {
|
|
return err
|
|
}
|
|
if err := copyDir(filepath.Join(cicdTemplateDir, "base", "democratic-csi"), filepath.Join(dir, "base", "democratic-csi"), true); err != nil {
|
|
return err
|
|
}
|
|
if err := renderCiliumConfig(filepath.Join(dir, "base", "cilium"), r.Config); err != nil {
|
|
return err
|
|
}
|
|
if err := renderCiliumConfig(filepath.Join(dir, "base", "cilium-config"), r.Config); err != nil {
|
|
return err
|
|
}
|
|
if err := renderDeliveryConfig(filepath.Join(dir, "base", "tekton"), r.Config); err != nil {
|
|
return err
|
|
}
|
|
if err := renderDeliveryConfig(filepath.Join(dir, "base", "tekton-triggers"), r.Config); err != nil {
|
|
return err
|
|
}
|
|
if err := writeDemocraticCSISecret(filepath.Join(dir, "base", "democratic-csi", "secret.sops.yaml"), r.Config.DemocraticCSI, r.Config.SOPS.AgeKeyPath); err != nil {
|
|
return err
|
|
}
|
|
if err := ensureClusterKustomizations(clusterDir); err != nil {
|
|
return err
|
|
}
|
|
return ghrepo.WriteFluxStructure(dir, r.Config.Flux.RepoName, r.Config.Flux.ClusterPath, fluxConfig)
|
|
},
|
|
); err != nil {
|
|
return err
|
|
}
|
|
if manager.MigrationPending {
|
|
return errors.New("existing repository migration PR created; merge and rerun bootstrap before infrastructure changes")
|
|
}
|
|
|
|
repoDir := filepath.Join(r.Config.Git.CloneParent, r.Config.Talos.RepoDirName)
|
|
if err := ensureRepo(repoDir, r.Config.Templates.TalosRepoURL, r.Config.Templates.TalosRepoRef); err != nil {
|
|
return err
|
|
}
|
|
|
|
terraformDir := filepath.Join(repoDir, r.Config.Talos.TerraformDir)
|
|
generatedDir := filepath.Join(repoDir, r.Config.Talos.GeneratedDir)
|
|
if err := os.MkdirAll(generatedDir, 0755); err != nil {
|
|
return err
|
|
}
|
|
if err := ensureLifecycleIdentity(terraformDir, r.Config); err != nil {
|
|
return err
|
|
}
|
|
if err := os.WriteFile(filepath.Join(terraformDir, r.Config.Talos.ConfigFileName), []byte(renderTerraformTFVars(r.Config)), 0600); err != nil {
|
|
return err
|
|
}
|
|
|
|
if r.Config.Talos.AutoRunTerraform {
|
|
if err := r.reconcileTerraform(terraformDir); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := ensureTalosConfig(generatedDir, r.Config); err != nil {
|
|
return err
|
|
}
|
|
if r.Config.Talos.AutoBootstrap {
|
|
if err := applyTalosConfigs(generatedDir, r.Config); err != nil {
|
|
return err
|
|
}
|
|
if err := bootstrapEtcdIfNeeded(generatedDir, r.Config); err != nil {
|
|
return err
|
|
}
|
|
if err := utils.RunCommandInDir(generatedDir, "talosctl", "kubeconfig", "--talosconfig=./clusterconfig/talosconfig", "--nodes="+r.Config.Talos.KubeconfigNode, "."); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if r.Config.Talos.AutoBootstrapFlux {
|
|
if err := installCilium(generatedDir, r.Config); err != nil {
|
|
return err
|
|
}
|
|
if err := utils.RunCommandInDirEnv(generatedDir, []string{"GIT_PASSWORD=" + r.Config.Git.Token}, "flux", "bootstrap", "git", "--url="+forgejo.CloneURL(r.Config.Git.BaseURL, r.Config.Git.Owner, r.Config.Flux.RepoName), "--branch="+r.Config.Flux.Branch, "--path="+r.Config.Flux.ClusterPath, "--cluster-domain="+r.Config.Flux.ClusterDomain, "--username="+r.Config.Git.Username, "--token-auth", "--kubeconfig=kubeconfig"); err != nil {
|
|
return err
|
|
}
|
|
if err := installSOPSKey(generatedDir, r.Config); err != nil {
|
|
return err
|
|
}
|
|
if err := configureFluxSOPS(generatedDir); err != nil {
|
|
return err
|
|
}
|
|
operationalSecrets, err := openbao.Initialize(filepath.Join(generatedDir, "kubeconfig"), r.Config.SOPS.RecoveryRecipient, r.Config.SOPS.RecoveryIdentityPath, r.Config.SOPS.RecoveryBundlePath, r.Config.SOPS.AgeKeyPath, r.Config.SOPS.OperationalSecretsPath)
|
|
if err != nil {
|
|
return fmt.Errorf("initialize OpenBao: %w", err)
|
|
}
|
|
authorization := operationalSecrets["cicd/forgejo-webhook"]["authorization"]
|
|
if authorization == "" {
|
|
return errors.New("operational SOPS secrets requires cicd/forgejo-webhook.authorization")
|
|
}
|
|
if err := waitForWebhookTargets(generatedDir, r.Config); err != nil {
|
|
return err
|
|
}
|
|
if err := manager.EnsureWebhook(r.Config.Delivery.AppName, r.Config.Delivery.WebhookURL(), authorization); err != nil {
|
|
return fmt.Errorf("reconcile Forgejo webhook: %w", err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func renderCiliumConfig(dir string, cfg config.Config) error {
|
|
replacements := strings.NewReplacer(
|
|
"${CILIUM_K8S_SERVICE_HOST}", cfg.Talos.KubeconfigEndpoint,
|
|
"${CILIUM_TRAFFIC_INTERFACE}", cfg.Cilium.TrafficInterface,
|
|
"${CILIUM_LB_START}", cfg.Cilium.LoadBalancerStart,
|
|
"${CILIUM_LB_END}", cfg.Cilium.LoadBalancerEnd,
|
|
)
|
|
return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil || info.IsDir() {
|
|
return err
|
|
}
|
|
content, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(path, []byte(replacements.Replace(string(content))), info.Mode())
|
|
})
|
|
}
|
|
|
|
func renderDeliveryConfig(dir string, cfg config.Config) error {
|
|
replacements := strings.NewReplacer(
|
|
"${APP_NAME}", cfg.Delivery.AppName,
|
|
"${APP_REPO_URL}", cfg.Delivery.AppRepoURL,
|
|
"${APP_REPO_REF}", cfg.Delivery.AppRepoRef,
|
|
"${IMAGE_REPOSITORY}", cfg.Delivery.ImageRepository,
|
|
"${FORGEJO_BASE_URL}", cfg.Git.BaseURL,
|
|
"${WEBHOOK_HOSTNAME}", cfg.Delivery.WebhookHostname,
|
|
"${WEBHOOK_PATH}", cfg.Delivery.WebhookPath,
|
|
)
|
|
return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil || info.IsDir() {
|
|
return err
|
|
}
|
|
content, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(path, []byte(replacements.Replace(string(content))), info.Mode())
|
|
})
|
|
}
|
|
|
|
func writeDemocraticCSISecret(path string, csi config.DemocraticCSIConfig, ageKeyPath string) error {
|
|
plaintext, err := renderDemocraticCSISecret(csi)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if existing, err := decryptSOPSFile(path, ageKeyPath); err == nil && sameYAML(existing, plaintext) {
|
|
return nil
|
|
}
|
|
return writeSOPSEncryptedFile(path, ageKeyPath, plaintext)
|
|
}
|
|
|
|
func WriteOperationalSecrets(path, ageKeyPath string, secrets map[string]map[string]string) error {
|
|
plaintext, err := yaml.Marshal(struct {
|
|
Secrets map[string]map[string]string `yaml:"secrets"`
|
|
}{Secrets: secrets})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return writeSOPSEncryptedFile(path, ageKeyPath, plaintext)
|
|
}
|
|
|
|
func NewWebhookAuthorization() (string, error) {
|
|
value := make([]byte, 32)
|
|
if _, err := rand.Read(value); err != nil {
|
|
return "", err
|
|
}
|
|
return "Bearer " + base64.RawURLEncoding.EncodeToString(value), nil
|
|
}
|
|
|
|
func writeSOPSEncryptedFile(path, ageKeyPath string, plaintext []byte) error {
|
|
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
|
return err
|
|
}
|
|
recipient, err := exec.Command("age-keygen", "-y", ageKeyPath).Output()
|
|
if err != nil {
|
|
return fmt.Errorf("derive SOPS age recipient: %w", err)
|
|
}
|
|
command := exec.Command("sops", "encrypt", "--age", strings.TrimSpace(string(recipient)), "--encrypted-regex", "^(data|stringData)$", "--input-type", "yaml", "--output-type", "yaml", "--filename-override", filepath.Base(path))
|
|
command.Stdin = bytes.NewReader(plaintext)
|
|
encrypted, err := command.Output()
|
|
if err != nil {
|
|
return fmt.Errorf("encrypt Democratic CSI secret: %w", err)
|
|
}
|
|
return os.WriteFile(path, encrypted, 0600)
|
|
}
|
|
|
|
func decryptSOPSFile(path, ageKeyPath string) ([]byte, error) {
|
|
command := exec.Command("sops", "--decrypt", "--output-type", "yaml", path)
|
|
command.Env = append(os.Environ(), "SOPS_AGE_KEY_FILE="+ageKeyPath)
|
|
return command.Output()
|
|
}
|
|
|
|
func sameYAML(left, right []byte) bool {
|
|
var leftValue any
|
|
var rightValue any
|
|
return yaml.Unmarshal(left, &leftValue) == nil && yaml.Unmarshal(right, &rightValue) == nil && reflect.DeepEqual(leftValue, rightValue)
|
|
}
|
|
|
|
func renderDemocraticCSISecret(csi config.DemocraticCSIConfig) ([]byte, error) {
|
|
return yaml.Marshal(struct {
|
|
APIVersion string `yaml:"apiVersion"`
|
|
Kind string `yaml:"kind"`
|
|
Metadata map[string]string `yaml:"metadata"`
|
|
Type string `yaml:"type"`
|
|
StringData map[string]string `yaml:"stringData"`
|
|
}{
|
|
APIVersion: "v1",
|
|
Kind: "Secret",
|
|
Metadata: map[string]string{"name": "democratic-csi-secrets", "namespace": "democratic-storage"},
|
|
Type: "Opaque",
|
|
StringData: map[string]string{
|
|
"truenas-api-key": csi.TrueNASAPIKey,
|
|
"truenas-host": csi.TrueNASHost,
|
|
"target-portal": csi.TargetPortal,
|
|
"share-host": csi.ShareHost,
|
|
"dataset-parent-nfs": csi.DatasetParentNFS,
|
|
"dataset-snapshots-nfs": csi.DatasetSnapshotsNFS,
|
|
"allowed-networks": csi.AllowedNetworks,
|
|
"name-suffix": csi.NameSuffix,
|
|
"portal-group": csi.PortalGroup,
|
|
"initiator-group": csi.InitiatorGroup,
|
|
},
|
|
})
|
|
}
|
|
|
|
func waitForWebhookTargets(dir string, cfg config.Config) error {
|
|
resources := []string{
|
|
"secret/forgejo-webhook",
|
|
"deployment/el-" + cfg.Delivery.AppName,
|
|
"pipeline/" + cfg.Delivery.AppName,
|
|
}
|
|
for _, resource := range resources {
|
|
deadline := time.Now().Add(10 * time.Minute)
|
|
for time.Now().Before(deadline) {
|
|
if _, err := utils.RunCommandQuietOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err == nil {
|
|
break
|
|
}
|
|
time.Sleep(2 * time.Second)
|
|
}
|
|
if _, err := utils.RunCommandQuietOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err != nil {
|
|
return fmt.Errorf("wait for %s before registering Forgejo webhook", resource)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ensureClusterKustomizations(clusterDir string) error {
|
|
path := filepath.Join(clusterDir, "kustomization.yaml")
|
|
content, err := os.ReadFile(path)
|
|
if os.IsNotExist(err) {
|
|
// Existing Flux roots may intentionally use recursive discovery. Do not
|
|
// introduce a partial root Kustomization that could prune its resources.
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
updated := string(content)
|
|
updated = strings.ReplaceAll(updated, " - bootstrap-secrets.sops.yaml\n", "")
|
|
for _, resource := range []string{"cilium-kustomization.yaml", "cilium-config-kustomization.yaml", "cicd-manifests-repo.yaml", "tekton-triggers-kustomization.yaml"} {
|
|
if !strings.Contains(updated, resource) {
|
|
updated += " - " + resource + "\n"
|
|
}
|
|
}
|
|
if updated == string(content) {
|
|
return nil
|
|
}
|
|
return os.WriteFile(path, []byte(updated), 0644)
|
|
}
|
|
|
|
func copyClusterTemplate(source, destination string) error {
|
|
entries, err := os.ReadDir(destination)
|
|
if err != nil && !os.IsNotExist(err) {
|
|
return err
|
|
}
|
|
if len(entries) == 0 {
|
|
return copyDir(source, destination, false)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(destination, "kustomization.yaml")); err == nil {
|
|
return copyDir(source, destination, false)
|
|
} else if !os.IsNotExist(err) {
|
|
return err
|
|
}
|
|
return copyDirExcept(source, destination, false, map[string]bool{"kustomization.yaml": true})
|
|
}
|
|
|
|
func ensureManifestsKustomizations(dir string) error {
|
|
for _, environment := range []string{"previews", "staging", "production"} {
|
|
path := filepath.Join(dir, "apps", environment, "kustomization.yaml")
|
|
if _, err := os.Stat(path); err == nil {
|
|
continue
|
|
} else if !os.IsNotExist(err) {
|
|
return err
|
|
}
|
|
if err := os.WriteFile(path, []byte("apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nresources:\n"), 0644); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func installCilium(dir string, cfg config.Config) error {
|
|
helmDir := filepath.Join(dir, ".helm")
|
|
if err := os.MkdirAll(helmDir, 0755); err != nil {
|
|
return err
|
|
}
|
|
return utils.RunCommandInDir(dir, "helm", "upgrade", "--install", "cilium", "cilium", "--repo=https://helm.cilium.io", "--version=1.19.6", "--repository-config="+filepath.Join(helmDir, "repositories.yaml"), "--repository-cache="+helmDir, "--namespace=kube-system", "--create-namespace", "--kubeconfig=kubeconfig", "--wait", "--timeout=5m", "--set=kubeProxyReplacement=true", "--set=ipam.mode=kubernetes", "--set=k8sServiceHost=localhost", "--set=k8sServicePort=7445", "--set=cgroup.autoMount.enabled=false", "--set=cgroup.hostRoot=/sys/fs/cgroup", "--set=bpf.hostLegacyRouting=true", "--set=securityContext.capabilities.ciliumAgent={CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}", "--set=securityContext.capabilities.cleanCiliumState={NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}", "--set=gatewayAPI.enabled=false", "--set=l2announcements.enabled=true", "--set=operator.replicas=1")
|
|
}
|
|
|
|
func copyDir(source, destination string, overwrite bool) error {
|
|
return copyDirExcept(source, destination, overwrite, nil)
|
|
}
|
|
|
|
func copyDirExcept(source, destination string, overwrite bool, excluded map[string]bool) error {
|
|
return filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.IsDir() && info.Name() == ".git" {
|
|
return filepath.SkipDir
|
|
}
|
|
relative, err := filepath.Rel(source, path)
|
|
if err != nil || relative == "." {
|
|
return err
|
|
}
|
|
if excluded[filepath.ToSlash(relative)] {
|
|
if info.IsDir() {
|
|
return filepath.SkipDir
|
|
}
|
|
return nil
|
|
}
|
|
target := filepath.Join(destination, relative)
|
|
if info.IsDir() {
|
|
return os.MkdirAll(target, 0755)
|
|
}
|
|
if _, err := os.Stat(target); err == nil && !overwrite {
|
|
return nil
|
|
}
|
|
input, err := os.Open(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
flags := os.O_WRONLY | os.O_CREATE | os.O_EXCL
|
|
if overwrite {
|
|
flags = os.O_WRONLY | os.O_CREATE | os.O_TRUNC
|
|
}
|
|
output, err := os.OpenFile(target, flags, info.Mode())
|
|
if err != nil {
|
|
_ = input.Close()
|
|
return err
|
|
}
|
|
_, copyErr := io.Copy(output, input)
|
|
closeInputErr := input.Close()
|
|
closeOutputErr := output.Close()
|
|
if copyErr != nil {
|
|
return copyErr
|
|
}
|
|
if closeInputErr != nil {
|
|
return closeInputErr
|
|
}
|
|
return closeOutputErr
|
|
})
|
|
}
|
|
|
|
func applyTalosConfigs(dir string, cfg config.Config) error {
|
|
for _, node := range cfg.Talos.Nodes {
|
|
configFile := filepath.Join("clusterconfig", fmt.Sprintf("%s-%s.yaml", cfg.Talos.Cluster.Name, node.Name))
|
|
nodeAddress := node.Networks[0].IP
|
|
secureArgs := []string{"apply-config", "--talosconfig=./clusterconfig/talosconfig", "--nodes=" + nodeAddress, "--endpoints=" + cfg.Talos.BootstrapEndpoint, "--file=" + configFile}
|
|
if _, err := utils.RunCommandQuietOutputInDir(dir, "talosctl", "get", "machinestatus", "--talosconfig=./clusterconfig/talosconfig", "--nodes="+nodeAddress, "--endpoints="+cfg.Talos.BootstrapEndpoint, "--output=json"); err == nil {
|
|
if err := utils.RunCommandInDir(dir, "talosctl", secureArgs...); err != nil {
|
|
return fmt.Errorf("apply Talos config to %s: %w", node.Name, err)
|
|
}
|
|
} else {
|
|
if _, maintenanceErr := utils.RunCommandQuietOutputInDir(dir, "talosctl", "get", "machinestatus", "--insecure", "--nodes="+nodeAddress, "--endpoints="+nodeAddress, "--output=json"); maintenanceErr == nil {
|
|
if err := utils.RunCommandInDir(dir, "talosctl", "apply-config", "--insecure", "--nodes="+nodeAddress, "--endpoints="+nodeAddress, "--file="+configFile); err != nil {
|
|
return fmt.Errorf("apply initial Talos config to %s: %w", node.Name, err)
|
|
}
|
|
} else {
|
|
if err := waitForTalosAPI(dir, cfg, nodeAddress); err != nil {
|
|
return fmt.Errorf("determine Talos state for %s: %w", node.Name, err)
|
|
}
|
|
if err := utils.RunCommandInDir(dir, "talosctl", secureArgs...); err != nil {
|
|
return fmt.Errorf("apply Talos config to %s: %w", node.Name, err)
|
|
}
|
|
}
|
|
}
|
|
if err := waitForTalosReboot(dir, cfg, nodeAddress); err != nil {
|
|
return fmt.Errorf("wait for %s: %w", node.Name, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func bootstrapEtcdIfNeeded(dir string, cfg config.Config) error {
|
|
if _, err := utils.RunCommandQuietOutputInDir(dir, "talosctl", "etcd", "status", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode); err == nil {
|
|
return nil
|
|
}
|
|
return utils.RunCommandInDir(dir, "talosctl", "bootstrap", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode)
|
|
}
|
|
|
|
func waitForTalosReboot(dir string, cfg config.Config, node string) error {
|
|
return waitForTalosAPI(dir, cfg, node)
|
|
}
|
|
|
|
func waitForTalosAPI(dir string, cfg config.Config, node string) error {
|
|
deadline := time.Now().Add(5 * time.Minute)
|
|
for time.Now().Before(deadline) {
|
|
if _, err := utils.RunCommandQuietOutputInDir(dir, "talosctl", "get", "machinestatus", "--output=json", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+node, "--nodes="+node); err == nil {
|
|
return nil
|
|
}
|
|
time.Sleep(2 * time.Second)
|
|
}
|
|
return fmt.Errorf("Talos API did not return after applying its machine configuration")
|
|
}
|
|
|
|
type lifecycle struct {
|
|
ClusterID string `yaml:"clusterId"`
|
|
ClusterName string `yaml:"clusterName"`
|
|
}
|
|
|
|
func ensureLifecycleIdentity(terraformDir string, cfg config.Config) error {
|
|
statePath := filepath.Join(terraformDir, "terraform.tfstate")
|
|
lifecyclePath := filepath.Join(terraformDir, ".maidn", "lifecycle.yaml")
|
|
data, err := os.ReadFile(lifecyclePath)
|
|
if err == nil {
|
|
var current lifecycle
|
|
if err := yaml.Unmarshal(data, ¤t); err != nil {
|
|
return fmt.Errorf("read lifecycle metadata: %w", err)
|
|
}
|
|
if current.ClusterID != cfg.ClusterID || current.ClusterName != cfg.Talos.Cluster.Name {
|
|
return fmt.Errorf("terraform checkout belongs to cluster %q; use its matching configuration", current.ClusterID)
|
|
}
|
|
return nil
|
|
}
|
|
if !os.IsNotExist(err) {
|
|
return err
|
|
}
|
|
if state, err := os.ReadFile(statePath); err == nil && !strings.Contains(string(state), cfg.Talos.Cluster.Name) {
|
|
return errors.New("terraform state exists but does not match clusterName; use an explicit recovery checkout")
|
|
} else if err != nil && !os.IsNotExist(err) {
|
|
return err
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(lifecyclePath), 0700); err != nil {
|
|
return err
|
|
}
|
|
data, err = yaml.Marshal(lifecycle{ClusterID: cfg.ClusterID, ClusterName: cfg.Talos.Cluster.Name})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(lifecyclePath, data, 0600)
|
|
}
|
|
|
|
func (r Runner) reconcileTerraform(terraformDir string) error {
|
|
environment := []string{"TF_VAR_proxmox_api_token=" + r.Config.Talos.Proxmox.APITokenID + "=" + r.Config.Talos.Proxmox.APITokenSecret}
|
|
if err := utils.RunCommandInDirEnv(terraformDir, environment, "terraform", "init", "-input=false"); err != nil {
|
|
return err
|
|
}
|
|
if r.Mode == Rebuild {
|
|
if err := utils.RunCommandInDirEnv(terraformDir, environment, "terraform", "destroy", "-auto-approve", "-input=false", "-target=proxmox_virtual_environment_vm.vm"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
planPath, err := terraformPlanPath(terraformDir, r.Config.ClusterID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer os.Remove(planPath)
|
|
if err := utils.RunCommandInDirEnv(terraformDir, environment, "terraform", "plan", "-input=false", "-out="+planPath); err != nil {
|
|
return err
|
|
}
|
|
return utils.RunCommandInDirEnv(terraformDir, environment, "terraform", "apply", "-input=false", "-auto-approve", planPath)
|
|
}
|
|
|
|
func terraformPlanPath(terraformDir, clusterID string) (string, error) {
|
|
return filepath.Abs(filepath.Join(terraformDir, clusterID+".tfplan"))
|
|
}
|
|
|
|
func installSOPSKey(dir string, cfg config.Config) error {
|
|
if _, err := os.Stat(cfg.SOPS.AgeKeyPath); err != nil {
|
|
return fmt.Errorf("read SOPS age identity: %w", err)
|
|
}
|
|
manifest, err := utils.RunCommandOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "flux-system", "create", "secret", "generic", "sops-age", "--from-file=age.agekey="+cfg.SOPS.AgeKeyPath, "--dry-run=client", "-o", "yaml")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return utils.RunCommandInputInDir(dir, manifest, "kubectl", "--kubeconfig=kubeconfig", "apply", "-f", "-")
|
|
}
|
|
|
|
func configureFluxSOPS(dir string) error {
|
|
patch := `{"spec":{"decryption":{"provider":"sops","secretRef":{"name":"sops-age"}}}}`
|
|
return utils.RunCommandInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "flux-system", "patch", "kustomization", "flux-system", "--type=merge", "-p", patch)
|
|
}
|
|
|
|
func ensureRepo(dir, repoURL, ref string) error {
|
|
if _, err := os.Stat(dir); os.IsNotExist(err) {
|
|
return utils.RunCommand("git", "clone", "--branch", ref, repoURL, dir)
|
|
}
|
|
if err := utils.RunCommandInDir(dir, "git", "fetch", "origin"); err != nil {
|
|
return err
|
|
}
|
|
if err := utils.RunCommandInDir(dir, "git", "checkout", ref); err != nil {
|
|
return err
|
|
}
|
|
return utils.RunCommandInDir(dir, "git", "pull", "--ff-only", "origin", ref)
|
|
}
|
|
|
|
func ensureTalosConfig(generatedDir string, cfg config.Config) error {
|
|
talhelperPath := "talhelper"
|
|
secretPath := filepath.Join(generatedDir, "talsecret.yaml")
|
|
if _, err := os.Stat(secretPath); os.IsNotExist(err) {
|
|
legacyPath := filepath.Join(generatedDir, "talsecret.sops.yaml")
|
|
if _, legacyErr := os.Stat(legacyPath); legacyErr == nil {
|
|
secretPath = legacyPath
|
|
} else {
|
|
secrets, err := utils.RunCommandOutputInDir(generatedDir, talhelperPath, "gensecret")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := os.WriteFile(secretPath, secrets, 0600); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return utils.RunCommandInDir(generatedDir, talhelperPath, "genconfig", "--config-file", "talconfig.yaml", "--secret-file", filepath.Base(secretPath))
|
|
}
|
|
|
|
func renderTerraformTFVars(cfg config.Config) string {
|
|
var builder strings.Builder
|
|
builder.WriteString(fmt.Sprintf("proxmox_api_url = %q\n", cfg.Talos.Proxmox.APIURL))
|
|
builder.WriteString(fmt.Sprintf("proxmox_node = %q\n", cfg.Talos.Proxmox.DefaultNode))
|
|
builder.WriteString(fmt.Sprintf("proxmox_pool = %q\n", cfg.Talos.Proxmox.Pool))
|
|
builder.WriteString(fmt.Sprintf("cluster_name = %q\n", cfg.Talos.Cluster.Name))
|
|
builder.WriteString(fmt.Sprintf("cluster_domain = %q\n", cfg.Talos.Cluster.Domain))
|
|
builder.WriteString(fmt.Sprintf("talos_factory_schematic_id = %q\n", cfg.Talos.Image.SchematicID))
|
|
builder.WriteString(fmt.Sprintf("talos_version = %q\n", cfg.Talos.Image.TalosVersion))
|
|
builder.WriteString(fmt.Sprintf("kubernetes_version = %q\n", cfg.Talos.Image.KubernetesVersion))
|
|
builder.WriteString("cni_name = \"none\"\n")
|
|
isoStorage := cfg.Talos.Image.Storage
|
|
if isoStorage == cfg.Talos.Cluster.DiskStorage {
|
|
isoStorage = "local"
|
|
}
|
|
isoFilename := fmt.Sprintf("talos-%s-%s-%s.iso", strings.TrimPrefix(cfg.Talos.Image.TalosVersion, "v"), cfg.Talos.Image.SchematicID[:12], cfg.Talos.Image.Architecture)
|
|
builder.WriteString(fmt.Sprintf("talos_iso_file = %q\n", fmt.Sprintf("%s:iso/%s", isoStorage, isoFilename)))
|
|
builder.WriteString(fmt.Sprintf("talos_image_update_mode = %q\n", cfg.Talos.Image.UpdateMode))
|
|
builder.WriteString(fmt.Sprintf("talos_image_storage = %q\n", isoStorage))
|
|
builder.WriteString(fmt.Sprintf("talos_image_architecture = %q\n", cfg.Talos.Image.Architecture))
|
|
builder.WriteString(fmt.Sprintf("disk_storage = %q\n", cfg.Talos.Cluster.DiskStorage))
|
|
builder.WriteString(fmt.Sprintf("additional_disk_storage = %q\n", cfg.Talos.Cluster.AdditionalStorage))
|
|
builder.WriteString(fmt.Sprintf("dns_servers = [%s]\n", quoteList(cfg.Talos.Cluster.DNSServers)))
|
|
builder.WriteString(fmt.Sprintf("control_plane_vip = %q\n", cfg.Talos.Cluster.ControlPlaneVIP))
|
|
builder.WriteString("node_interfaces = {\n")
|
|
for _, node := range sortedKeys(cfg.Talos.Proxmox.NodeInterfaces) {
|
|
iface := cfg.Talos.Proxmox.NodeInterfaces[node]
|
|
builder.WriteString(fmt.Sprintf(" %q = %q\n", node, iface))
|
|
}
|
|
builder.WriteString("}\n")
|
|
builder.WriteString("node_addresses = {\n")
|
|
for _, node := range sortedKeys(cfg.Talos.Proxmox.NodeAddresses) {
|
|
addr := cfg.Talos.Proxmox.NodeAddresses[node]
|
|
builder.WriteString(fmt.Sprintf(" %q = %q\n", node, addr))
|
|
}
|
|
builder.WriteString("}\n")
|
|
builder.WriteString(fmt.Sprintf("create_vlan_interface = %t\n", cfg.Talos.Cluster.CreateVLANInterface))
|
|
builder.WriteString(fmt.Sprintf("manage_network_bridges = %t\n", cfg.Talos.Cluster.ManageNetworkBridges))
|
|
builder.WriteString("image_cache_proxy = { enabled = false, ip = \"\", port = 3128 }\n")
|
|
builder.WriteString("nodes = [\n")
|
|
for _, node := range cfg.Talos.Nodes {
|
|
builder.WriteString(" {\n")
|
|
builder.WriteString(fmt.Sprintf(" name = %q\n", node.Name))
|
|
builder.WriteString(fmt.Sprintf(" vmid = %d\n", node.VMID))
|
|
builder.WriteString(fmt.Sprintf(" role = %q\n", node.Role))
|
|
builder.WriteString(fmt.Sprintf(" cores = %d\n", node.Cores))
|
|
builder.WriteString(fmt.Sprintf(" memory = %d\n", node.Memory))
|
|
builder.WriteString(fmt.Sprintf(" disk_size = %q\n", node.DiskSize))
|
|
if node.AdditionalDiskSize != "" {
|
|
builder.WriteString(fmt.Sprintf(" additional_disk_size = %q\n", node.AdditionalDiskSize))
|
|
}
|
|
builder.WriteString(fmt.Sprintf(" tags = [%s]\n", quoteList(node.Tags)))
|
|
builder.WriteString(fmt.Sprintf(" proxmox_node = %q\n", node.ProxmoxNode))
|
|
builder.WriteString(" networks = [\n")
|
|
for _, net := range node.Networks {
|
|
builder.WriteString(" {\n")
|
|
builder.WriteString(fmt.Sprintf(" mac_address = %q\n", net.MACAddress))
|
|
builder.WriteString(fmt.Sprintf(" cidr = %q\n", net.CIDR))
|
|
if net.IP != "" {
|
|
builder.WriteString(fmt.Sprintf(" ip = %q\n", net.IP))
|
|
}
|
|
if net.Gateway != "" {
|
|
builder.WriteString(fmt.Sprintf(" gateway = %q\n", net.Gateway))
|
|
}
|
|
builder.WriteString(fmt.Sprintf(" vlan_id = %d\n", net.VLANID))
|
|
builder.WriteString(" },\n")
|
|
}
|
|
builder.WriteString(" ]\n")
|
|
builder.WriteString(" },\n")
|
|
}
|
|
builder.WriteString("]\n")
|
|
return builder.String()
|
|
}
|
|
|
|
func quoteList(values []string) string {
|
|
quoted := make([]string, 0, len(values))
|
|
for _, value := range values {
|
|
quoted = append(quoted, fmt.Sprintf("%q", value))
|
|
}
|
|
return strings.Join(quoted, ", ")
|
|
}
|
|
|
|
func sortedKeys(values map[string]string) []string {
|
|
keys := make([]string, 0, len(values))
|
|
for key := range values {
|
|
keys = append(keys, key)
|
|
}
|
|
sort.Strings(keys)
|
|
return keys
|
|
}
|