diff --git a/MaidnCLI b/MaidnCLI deleted file mode 160000 index cc8fab6..0000000 --- a/MaidnCLI +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cc8fab6832a3a51bd072155f90f91354e440077a diff --git a/README.md b/README.md index d04122a..63cc3bd 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,14 @@ in powershell run go install github.com/go-delve/delve/cmd/dlv@latest dlv version ``` - ## Commands - `cicd-tool repo init --org --flux-repo ` creates the manifests and Flux repos - `cicd-tool bootstrap` runs a shorter Forgejo-first wizard, asks for a Forgejo PAT, asks where local repos should be cloned, discovers Proxmox nodes/storage/networks, retries without losing entered answers when discovery fails, shows the latest Talos version, derives the standardized Talos factory URL, schematic, and required extensions automatically from the chosen version, writes `terraform.tfvars`, stages Talos images on Proxmox, and can execute Terraform, Talos bootstrap, and Flux bootstrap - `cicd-tool bootstrap --config maidn-bootstrap.yaml` skips the wizard and uses the saved config +See `docs/operations.md` for the authorized operating and verification runbook. + ## Forgejo setup For `https://git.pingu.pw` you need: @@ -45,15 +46,3 @@ talos: ip: vlanId: ``` - - -https://192.168.0.15:8006 -root@pam!maidn-test-key -2ce7bff1-ac98-4a45-8db4-186d49ae4159 - -test-org-test-key:147878348db3b8ab660b1af2799c3842705a31bc -maidn-dev-work:fca448f2677362cc165eebb7859ce8f9f5e3735c - -https://192.168.0.13:8006 -root@pam!maidn-test-key -3773033f-552b-4572-83e9-cd16e8ac2e13 \ No newline at end of file diff --git a/cmd/bootstrap.go b/cmd/bootstrap.go index f412708..ad141fb 100644 --- a/cmd/bootstrap.go +++ b/cmd/bootstrap.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "path/filepath" "github.com/Pingu-Studio/MaidnCLI/internal/bootstrap" "github.com/Pingu-Studio/MaidnCLI/internal/config" @@ -20,6 +21,8 @@ var bootstrapPromptOperationalSecrets bool var bootstrapInitializeOpenBaoRecovery bool var bootstrapInitializeOpenBao bool var bootstrapCreateForgejoRegistryToken bool +var bootstrapRegisterWebhook bool +var bootstrapRotateWebhookAuthorization bool var bootstrapPublishAppFrom string var bootstrapMergeBootstrapPR bool var bootstrapManageNetworkBridges bool @@ -41,6 +44,8 @@ func init() { bootstrapCmd.Flags().BoolVar(&bootstrapInitializeOpenBaoRecovery, "initialize-openbao-recovery", false, "Create and save a separate OpenBao recovery age identity for --config") bootstrapCmd.Flags().BoolVar(&bootstrapInitializeOpenBao, "initialize-openbao", false, "Initialize OpenBao and seed encrypted operational secrets for --config") bootstrapCmd.Flags().BoolVar(&bootstrapCreateForgejoRegistryToken, "create-forgejo-registry-token", false, "Create a least-privilege Forgejo package registry token and seed it through OpenBao") + bootstrapCmd.Flags().BoolVar(&bootstrapRegisterWebhook, "register-webhook", false, "Seed OpenBao secrets and register the Forgejo webhook") + bootstrapCmd.Flags().BoolVar(&bootstrapRotateWebhookAuthorization, "rotate-webhook-authorization", false, "Replace the Forgejo webhook authorization and reconcile it through OpenBao") bootstrapCmd.Flags().StringVar(&bootstrapPublishAppFrom, "publish-app-from", "", "Push this app checkout's current branch and create a Forgejo delivery PR") bootstrapCmd.Flags().BoolVar(&bootstrapMergeBootstrapPR, "merge-bootstrap-pr", false, "Merge the generated Flux repository migration PR before bootstrapping") bootstrapCmd.Flags().BoolVar(&bootstrapManageNetworkBridges, "manage-network-bridges", false, "Persist Terraform management for existing Talos network bridges") @@ -59,6 +64,23 @@ func runBootstrap(cmd *cobra.Command, args []string) error { } return createForgejoRegistryToken(cfg) } + if bootstrapRotateWebhookAuthorization { + if bootstrapConfigPath == "" { + return fmt.Errorf("--rotate-webhook-authorization requires --config") + } + cfg, err = config.Load(bootstrapConfigPath) + if err != nil { + return err + } + authorization, err := bootstrap.NewWebhookAuthorization() + if err != nil { + return fmt.Errorf("generate Forgejo webhook authorization: %w", err) + } + if err := bootstrap.UpsertOperationalSecret(cfg.SOPS.OperationalSecretsPath, cfg.SOPS.AgeKeyPath, "cicd/forgejo-webhook", "authorization", authorization); err != nil { + return fmt.Errorf("save Forgejo webhook authorization: %w", err) + } + return bootstrap.Runner{Config: cfg, RegisterWebhook: true}.Run() + } if bootstrapInitializeOpenBao { if bootstrapConfigPath == "" { return fmt.Errorf("--initialize-openbao requires --config") @@ -117,6 +139,18 @@ func runBootstrap(cmd *cobra.Command, args []string) error { } return manager.CreatePullRequest(repo, "feat: migrate delivery to Tekton", branch, cfg.Delivery.AppRepoRef) } + if bootstrapInitializeOpenBao { + if bootstrapConfigPath == "" { + return fmt.Errorf("--initialize-openbao requires --config") + } + cfg, err = config.Load(bootstrapConfigPath) + if err != nil { + return err + } + generatedDir := filepath.Join(cfg.Git.CloneParent, cfg.Talos.RepoDirName, cfg.Talos.GeneratedDir) + _, err = openbao.Initialize(filepath.Join(generatedDir, "kubeconfig"), cfg.SOPS.RecoveryRecipient, cfg.SOPS.RecoveryIdentityPath, cfg.SOPS.RecoveryBundlePath, cfg.SOPS.AgeKeyPath, cfg.SOPS.OperationalSecretsPath) + return err + } if bootstrapConfigPath != "" { if bootstrapPromptDemocraticCSI || bootstrapPromptOperationalSecrets || bootstrapInitializeOpenBaoRecovery || bootstrapManageNetworkBridges { @@ -171,7 +205,7 @@ func runBootstrap(cmd *cobra.Command, args []string) error { return err } - runner := bootstrap.Runner{Config: cfg, Mode: bootstrap.Mode(bootstrapMode), ConfirmRebuild: bootstrapYes} + runner := bootstrap.Runner{Config: cfg, Mode: bootstrap.Mode(bootstrapMode), ConfirmRebuild: bootstrapYes, RegisterWebhook: bootstrapRegisterWebhook} return runner.Run() } diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..574bdf1 --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,142 @@ +# Operations Runbook + +Use this runbook from the MaidnCLI checkout. Bootstrap YAML, age identities, +OpenBao recovery material, Terraform state, and generated workspaces are +secret-bearing local inputs. Do not commit or print them. + +## Normal Reconciliation + +Run after a merged GitOps migration or to recover ordinary drift: + +```powershell +go run . bootstrap --config --mode=reconcile +``` + +This is the only regular lifecycle command. Do not use direct `kubectl apply`, +`flux reconcile`, Helm upgrades, or mutating `talosctl` commands. + +## Rebuild + +Use only when an authorized recovery requires recreating the Talos VM: + +```powershell +go run . bootstrap --config --mode=rebuild --yes +``` + +The rebuild replaces Terraform-managed Talos VMs only. It does not manage or +delete TrueNAS datasets or unrelated infrastructure. + +## OpenBao And Webhooks + +After a rebuild or an OpenBao restart, refresh Kubernetes auth and reseed the +encrypted operational values: + +```powershell +go run . bootstrap --config --initialize-openbao +``` + +When the Pipeline and EventListener are already Ready, register or update the +Forgejo webhook without re-running the full lifecycle: + +```powershell +go run . bootstrap --config --register-webhook +``` + +The webhook authorization value stays in encrypted operational secrets and is +never supplied on the command line. + +If that authorization value is exposed, replace it and reconcile both OpenBao +and the Forgejo hook in one command: + +```powershell +go run . bootstrap --config --rotate-webhook-authorization +``` + +## Read-Only Verification + +Set `KUBECONFIG` to the generated kubeconfig for the configured cluster, then +check the control plane and delivery chain: + +```powershell +kubectl -n flux-system get kustomizations +kubectl get clustersecretstores +kubectl -n flux-system get externalsecrets +kubectl -n tekton-pipelines get pipelines,eventlisteners,externalsecrets +``` + +Expected state: + +- Flux Kustomizations are `READY=True`. +- `ClusterSecretStore/openbao` is `READY=True`. +- Forgejo credential ExternalSecrets are `SecretSynced`. +- The application Pipeline exists and the EventListener is available. + +If OpenBao authentication is invalid after a rebuild, run +`--initialize-openbao`, then allow the controllers to retry. Do not recreate +the ClusterSecretStore or Secrets manually. + +## External DNS + +Webhook delivery requires the configured `tekton.` hostname to +resolve through Pi-hole to the Cilium Gateway address. ExternalDNS uses the +Pi-hole provider with Gateway API routes, an `upsert-only` policy, and no +ownership registry. + +```powershell +kubectl -n external-dns get pods,externalsecrets +kubectl -n external-dns logs deployment/external-dns --tail=100 +``` + +If `ExternalSecret/pihole-credentials` is not `SecretSynced`, rerun +`--prompt-operational-secrets` to enter the Pi-hole server and password, then +run `--initialize-openbao` and `--register-webhook`. Do not create or edit the +provider Secret directly. + +## Webhook TLS + +The public Gateway terminates HTTPS with a cert-manager certificate. Its +Cloudflare DNS-01 token is used only to issue the `nid3.com` certificate; +Pi-hole remains the ExternalDNS provider. Check certificate readiness with: + +```powershell +kubectl -n cert-manager get externalsecret cloudflare-api-token +kubectl -n gateway-system get certificate webhook-tls +``` + +Enter the Pi-hole values, Cloudflare DNS-01 token, and Tunnel token through +`--prompt-operational-secrets`, then run `--initialize-openbao`. Do not put the +Cloudflare token in the cluster repository. + +## Internal Platform UIs + +Pi-hole resolves these HTTPS names to the Cilium Gateway only on the LAN: + +- `https://grafana./` for Grafana. Authenticate with Grafana. +- `https://openbao./` for OpenBao. Authenticate with an OpenBao token. + +Hubble UI is enabled for in-cluster troubleshooting but has no LAN route because +it does not provide authentication. Add an authenticated proxy before exposing +it outside the cluster. + +## Webhook Smoke Test + +Use Forgejo's hook test endpoint against an existing non-`main` ref. It emits +a real push delivery, runs the Node build, and pushes a SHA-tagged registry +image, but skips the main-only staging manifest update: + +```powershell +# Discover the hook ID and choose an existing non-main branch or tag. +Invoke-RestMethod -Headers @{ Authorization = "token $env:FORGEJO_TOKEN" } ` + -Uri "https:///api/v1/repos///hooks" + +Invoke-WebRequest -Method Post -Headers @{ Authorization = "token $env:FORGEJO_TOKEN" } ` + -Uri "https:///api/v1/repos///hooks//tests?ref=" +``` + +Forgejo returns `204` after accepting the delivery. Confirm the resulting +PipelineRun instead of treating `204` as a successful build: + +```powershell +kubectl -n tekton-pipelines get pipelineruns +kubectl -n tekton-pipelines describe pipelinerun +``` diff --git a/docs/secrets.md b/docs/secrets.md index 25959af..df40045 100644 --- a/docs/secrets.md +++ b/docs/secrets.md @@ -17,6 +17,9 @@ secrets: dockerconfigjson: encrypted-value cicd/forgejo-webhook: authorization: encrypted-value + platform/pihole: + server: encrypted-value + password: encrypted-value platform/cloudflare: api-token: encrypted-value platform/cloudflare-tunnel: @@ -34,8 +37,10 @@ For an existing configuration, run `bootstrap --config For a new operational-secret input, run `bootstrap --config --prompt-operational-secrets`. It derives Forgejo Git and registry credentials -from the configured Forgejo account, prompts for Cloudflare credentials with -input masked, and generates the webhook authorization value. +from the configured Forgejo account, prompts for the Pi-hole server and masked +password, masked Cloudflare DNS-01 and Tunnel tokens, and generates the webhook +authorization value. The DNS-01 token issues the Gateway certificate; it is not +used by ExternalDNS. `cicd/forgejo-webhook.authorization` is required for delivery bootstrap. The CLI supplies it as the Forgejo webhook Authorization header and Tekton compares diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index dde298d..3d8e8c1 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -2,6 +2,7 @@ package bootstrap import ( "bytes" + "context" "crypto/rand" "encoding/base64" "encoding/json" @@ -32,9 +33,10 @@ const ( ) type Runner struct { - Config config.Config - Mode Mode - ConfirmRebuild bool + Config config.Config + Mode Mode + ConfirmRebuild bool + RegisterWebhook bool } type operationalSecrets struct { @@ -72,6 +74,9 @@ func (r Runner) Run() error { if r.Mode == Rebuild && !r.ConfirmRebuild { return fmt.Errorf("rebuild is destructive; rerun with --mode=rebuild --yes") } + if r.RegisterWebhook { + return r.reconcileWebhook(filepath.Join(r.Config.Git.CloneParent, r.Config.Talos.RepoDirName, r.Config.Talos.GeneratedDir)) + } workspace := r.Config.WorkspaceDir if err := os.MkdirAll(workspace, 0755); err != nil { @@ -107,9 +112,30 @@ func (r Runner) Run() error { if err := copyDir(filepath.Join(cicdTemplateDir, "base"), filepath.Join(dir, "base"), false); err != nil { return err } + for _, component := range []string{"snapshot-crds", "democratic-csi", "cert-manager", "cluster-issuers", "gateway-api", "gateway", "monitoring", "openbao", "external-secrets", "external-secrets-config", "external-dns", "tekton", "tekton-triggers"} { + if err := copyDir(filepath.Join(cicdTemplateDir, "base", component), filepath.Join(dir, "base", component), true); err != nil { + return err + } + } if err := copyClusterTemplate(filepath.Join(cicdTemplateDir, "clusters", "template"), clusterDir); err != nil { return err } + for _, name := range []string{"external-secrets", "cnpg", "cloudflare-tunnel", "external-dns", "monitoring", "tekton"} { + content, err := os.ReadFile(filepath.Join(cicdTemplateDir, "clusters", "template", name+"-kustomization.yaml")) + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(clusterDir, name+"-kustomization.yaml"), content, 0644); err != nil { + return err + } + } + csiKustomization, err := os.ReadFile(filepath.Join(cicdTemplateDir, "clusters", "template", "democratic-csi-kustomization.yaml")) + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(clusterDir, "democratic-csi-kustomization.yaml"), csiKustomization, 0644); err != nil { + return err + } if err := copyDir(filepath.Join(cicdTemplateDir, "base", "cilium"), filepath.Join(dir, "base", "cilium"), true); err != nil { return err } @@ -178,6 +204,9 @@ func (r Runner) Run() error { if err := utils.RunCommandInDir(generatedDir, "talosctl", "kubeconfig", "--talosconfig=./clusterconfig/talosconfig", "--nodes="+r.Config.Talos.KubeconfigNode, "."); err != nil { return err } + if err := waitForKubernetesAPI(generatedDir); err != nil { + return err + } } if r.Config.Talos.AutoBootstrapFlux { if err := installCilium(generatedDir, r.Config); err != nil { @@ -192,20 +221,26 @@ func (r Runner) Run() error { if err := configureFluxSOPS(generatedDir); err != nil { return err } - operationalSecrets, err := openbao.Initialize(filepath.Join(generatedDir, "kubeconfig"), r.Config.SOPS.RecoveryRecipient, r.Config.SOPS.RecoveryIdentityPath, r.Config.SOPS.RecoveryBundlePath, r.Config.SOPS.AgeKeyPath, r.Config.SOPS.OperationalSecretsPath) - if err != nil { - return fmt.Errorf("initialize OpenBao: %w", err) - } - authorization := operationalSecrets["cicd/forgejo-webhook"]["authorization"] - if authorization == "" { - return errors.New("operational SOPS secrets requires cicd/forgejo-webhook.authorization") - } - if err := waitForWebhookTargets(generatedDir, r.Config); err != nil { - return err - } - if err := manager.EnsureWebhook(r.Config.Delivery.AppName, r.Config.Delivery.WebhookURL(), authorization); err != nil { - return fmt.Errorf("reconcile Forgejo webhook: %w", err) - } + return r.reconcileWebhook(generatedDir) + } + return nil +} + +func (r Runner) reconcileWebhook(generatedDir string) error { + operationalSecrets, err := openbao.Initialize(filepath.Join(generatedDir, "kubeconfig"), r.Config.SOPS.RecoveryRecipient, r.Config.SOPS.RecoveryIdentityPath, r.Config.SOPS.RecoveryBundlePath, r.Config.SOPS.AgeKeyPath, r.Config.SOPS.OperationalSecretsPath) + if err != nil { + return fmt.Errorf("initialize OpenBao: %w", err) + } + authorization := operationalSecrets["cicd/forgejo-webhook"]["authorization"] + if authorization == "" { + return errors.New("operational SOPS secrets requires cicd/forgejo-webhook.authorization") + } + if err := waitForWebhookTargets(generatedDir, r.Config); err != nil { + return err + } + manager := forgejo.NewRepoManager(r.Config.Git.BaseURL, r.Config.Git.Token, r.Config.Git.Owner, r.Config.Git.Username, r.Config.Flux.ManifestsRepo, r.Config.Flux.RepoName, r.Config.Flux.Branch, "maidn/bootstrap-"+r.Config.ClusterID) + if err := manager.EnsureWebhook(r.Config.Delivery.AppName, r.Config.Delivery.WebhookURL(), authorization); err != nil { + return fmt.Errorf("reconcile Forgejo webhook: %w", err) } return nil } @@ -236,6 +271,9 @@ func renderDeliveryConfig(dir string, cfg config.Config) error { "${APP_REPO_REF}", cfg.Delivery.AppRepoRef, "${IMAGE_REPOSITORY}", cfg.Delivery.ImageRepository, "${FORGEJO_BASE_URL}", cfg.Git.BaseURL, + "${CLUSTER_DOMAIN}", cfg.Flux.ClusterDomain, + "${TEKTON_CATALOG_REPO_URL}", cfg.Templates.TektonCatalogRepoURL, + "${TEKTON_CATALOG_REPO_REF}", cfg.Templates.TektonCatalogRepoRef, "${WEBHOOK_HOSTNAME}", cfg.Delivery.WebhookHostname, "${WEBHOOK_PATH}", cfg.Delivery.WebhookPath, "${TEKTON_CATALOG_REPO_URL}", cfg.Templates.TektonCatalogRepoURL, @@ -397,14 +435,20 @@ func renderDemocraticCSISecret(csi config.DemocraticCSIConfig) ([]byte, error) { return yaml.Marshal(struct { APIVersion string `yaml:"apiVersion"` Kind string `yaml:"kind"` - Metadata map[string]string `yaml:"metadata"` + Metadata map[string]any `yaml:"metadata"` Type string `yaml:"type"` StringData map[string]string `yaml:"stringData"` }{ APIVersion: "v1", Kind: "Secret", - Metadata: map[string]string{"name": "democratic-csi-secrets", "namespace": "democratic-storage"}, - Type: "Opaque", + Metadata: map[string]any{ + "name": "democratic-csi-secrets", + "namespace": "democratic-storage", + "labels": map[string]string{ + "reconcile.fluxcd.io/watch": "Enabled", + }, + }, + Type: "Opaque", StringData: map[string]string{ "truenas-api-key": csi.TrueNASAPIKey, "truenas-host": csi.TrueNASHost, @@ -454,7 +498,7 @@ func ensureClusterKustomizations(clusterDir string) error { } updated := string(content) updated = strings.ReplaceAll(updated, " - bootstrap-secrets.sops.yaml\n", "") - for _, resource := range []string{"cilium-kustomization.yaml", "cilium-config-kustomization.yaml", "cicd-manifests-repo.yaml", "tekton-triggers-kustomization.yaml"} { + for _, resource := range []string{"snapshot-crds-kustomization.yaml", "democratic-csi-kustomization.yaml", "cert-manager-kustomization.yaml", "cluster-issuers-kustomization.yaml", "gateway-api-kustomization.yaml", "gateway-kustomization.yaml", "cilium-kustomization.yaml", "cilium-config-kustomization.yaml", "cicd-manifests-repo.yaml", "external-secrets-config-kustomization.yaml", "external-dns-kustomization.yaml", "monitoring-kustomization.yaml", "tekton-kustomization.yaml", "tekton-triggers-kustomization.yaml"} { if !strings.Contains(updated, resource) { updated += " - " + resource + "\n" } @@ -484,7 +528,14 @@ func copyClusterTemplate(source, destination string) error { func ensureManifestsKustomizations(dir string) error { for _, environment := range []string{"previews", "staging", "production"} { path := filepath.Join(dir, "apps", environment, "kustomization.yaml") - if _, err := os.Stat(path); err == nil { + content, err := os.ReadFile(path) + if err == nil { + if strings.Contains(string(content), "resources:") { + continue + } + if err := os.WriteFile(path, append(content, []byte("resources:\n")...), 0644); err != nil { + return err + } continue } else if !os.IsNotExist(err) { return err @@ -501,7 +552,7 @@ func installCilium(dir string, cfg config.Config) error { if err := os.MkdirAll(helmDir, 0755); err != nil { return err } - return utils.RunCommandInDir(dir, "helm", "upgrade", "--install", "cilium", "cilium", "--repo=https://helm.cilium.io", "--version=1.19.6", "--repository-config="+filepath.Join(helmDir, "repositories.yaml"), "--repository-cache="+helmDir, "--namespace=kube-system", "--create-namespace", "--kubeconfig=kubeconfig", "--wait", "--timeout=5m", "--set=kubeProxyReplacement=true", "--set=ipam.mode=kubernetes", "--set=k8sServiceHost=localhost", "--set=k8sServicePort=7445", "--set=cgroup.autoMount.enabled=false", "--set=cgroup.hostRoot=/sys/fs/cgroup", "--set=bpf.hostLegacyRouting=true", "--set=securityContext.capabilities.ciliumAgent={CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}", "--set=securityContext.capabilities.cleanCiliumState={NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}", "--set=gatewayAPI.enabled=false", "--set=l2announcements.enabled=true", "--set=operator.replicas=1") + return utils.RunCommandInDir(dir, "helm", "upgrade", "--install", "cilium", "cilium", "--repo=https://helm.cilium.io", "--version=1.19.6", "--repository-config="+filepath.Join(helmDir, "repositories.yaml"), "--repository-cache="+helmDir, "--namespace=kube-system", "--create-namespace", "--kubeconfig=kubeconfig", "--wait", "--timeout=5m", "--set=kubeProxyReplacement=true", "--set=ipam.mode=kubernetes", "--set=k8sServiceHost=localhost", "--set=k8sServicePort=7445", "--set=cgroup.autoMount.enabled=false", "--set=cgroup.hostRoot=/sys/fs/cgroup", "--set=bpf.hostLegacyRouting=true", "--set=securityContext.capabilities.ciliumAgent={CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}", "--set=securityContext.capabilities.cleanCiliumState={NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}", "--set=envoy.enabled=true", "--set=gatewayAPI.enabled=true", "--set=l2announcements.enabled=true", "--set=rollOutCiliumPods=true", "--set=operator.replicas=1", "--set=operator.rollOutPods=true") } func copyDir(source, destination string, overwrite bool) error { @@ -564,12 +615,12 @@ func applyTalosConfigs(dir string, cfg config.Config) error { configFile := filepath.Join("clusterconfig", fmt.Sprintf("%s-%s.yaml", cfg.Talos.Cluster.Name, node.Name)) nodeAddress := node.Networks[0].IP secureArgs := []string{"apply-config", "--talosconfig=./clusterconfig/talosconfig", "--nodes=" + nodeAddress, "--endpoints=" + cfg.Talos.BootstrapEndpoint, "--file=" + configFile} - if _, err := utils.RunCommandQuietOutputInDir(dir, "talosctl", "get", "machinestatus", "--talosconfig=./clusterconfig/talosconfig", "--nodes="+nodeAddress, "--endpoints="+cfg.Talos.BootstrapEndpoint, "--output=json"); err == nil { + if _, err := runTalosctlOutput(dir, "get", "machinestatus", "--talosconfig=./clusterconfig/talosconfig", "--nodes="+nodeAddress, "--endpoints="+cfg.Talos.BootstrapEndpoint, "--output=json"); err == nil { if err := utils.RunCommandInDir(dir, "talosctl", secureArgs...); err != nil { return fmt.Errorf("apply Talos config to %s: %w", node.Name, err) } } else { - if _, maintenanceErr := utils.RunCommandQuietOutputInDir(dir, "talosctl", "get", "machinestatus", "--insecure", "--nodes="+nodeAddress, "--endpoints="+nodeAddress, "--output=json"); maintenanceErr == nil { + if _, maintenanceErr := runTalosctlOutput(dir, "get", "machinestatus", "--insecure", "--nodes="+nodeAddress, "--endpoints="+nodeAddress, "--output=json"); maintenanceErr == nil { if err := utils.RunCommandInDir(dir, "talosctl", "apply-config", "--insecure", "--nodes="+nodeAddress, "--endpoints="+nodeAddress, "--file="+configFile); err != nil { return fmt.Errorf("apply initial Talos config to %s: %w", node.Name, err) } @@ -590,10 +641,32 @@ func applyTalosConfigs(dir string, cfg config.Config) error { } func bootstrapEtcdIfNeeded(dir string, cfg config.Config) error { - if _, err := utils.RunCommandQuietOutputInDir(dir, "talosctl", "etcd", "status", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode); err == nil { + if _, err := runTalosctlOutput(dir, "etcd", "status", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode); err == nil { return nil } - return utils.RunCommandInDir(dir, "talosctl", "bootstrap", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode) + if err := waitForTalosAPI(dir, cfg, cfg.Talos.BootstrapNode); err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + command := exec.CommandContext(ctx, "talosctl", "bootstrap", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode) + command.Dir = dir + command.Stdout = os.Stdout + command.Stderr = os.Stderr + if err := command.Run(); err != nil { + if ctx.Err() != nil { + return errors.New("etcd bootstrap request timed out") + } + return fmt.Errorf("bootstrap etcd: %w", err) + } + deadline := time.Now().Add(2 * time.Minute) + for time.Now().Before(deadline) { + if _, err := runTalosctlOutput(dir, "etcd", "status", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode); err == nil { + return nil + } + time.Sleep(2 * time.Second) + } + return errors.New("etcd did not become healthy after bootstrap") } func waitForTalosReboot(dir string, cfg config.Config, node string) error { @@ -603,7 +676,7 @@ func waitForTalosReboot(dir string, cfg config.Config, node string) error { func waitForTalosAPI(dir string, cfg config.Config, node string) error { deadline := time.Now().Add(5 * time.Minute) for time.Now().Before(deadline) { - if _, err := utils.RunCommandQuietOutputInDir(dir, "talosctl", "get", "machinestatus", "--output=json", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+node, "--nodes="+node); err == nil { + if _, err := runTalosctlOutput(dir, "get", "machinestatus", "--output=json", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+node, "--nodes="+node); err == nil { return nil } time.Sleep(2 * time.Second) @@ -611,6 +684,25 @@ func waitForTalosAPI(dir string, cfg config.Config, node string) error { return fmt.Errorf("Talos API did not return after applying its machine configuration") } +func runTalosctlOutput(dir string, args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + command := exec.CommandContext(ctx, "talosctl", args...) + command.Dir = dir + return command.Output() +} + +func waitForKubernetesAPI(dir string) error { + deadline := time.Now().Add(5 * time.Minute) + for time.Now().Before(deadline) { + if _, err := utils.RunCommandQuietOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "get", "--raw=/readyz"); err == nil { + return nil + } + time.Sleep(2 * time.Second) + } + return errors.New("Kubernetes API did not become ready after Talos bootstrap") +} + type lifecycle struct { ClusterID string `yaml:"clusterId"` ClusterName string `yaml:"clusterName"` @@ -674,10 +766,14 @@ func terraformPlanPath(terraformDir, clusterID string) (string, error) { } func installSOPSKey(dir string, cfg config.Config) error { - if _, err := os.Stat(cfg.SOPS.AgeKeyPath); err != nil { + ageKeyPath, err := filepath.Abs(cfg.SOPS.AgeKeyPath) + if err != nil { + return fmt.Errorf("resolve SOPS age identity: %w", err) + } + if _, err := os.Stat(ageKeyPath); err != nil { return fmt.Errorf("read SOPS age identity: %w", err) } - manifest, err := utils.RunCommandOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "flux-system", "create", "secret", "generic", "sops-age", "--from-file=age.agekey="+cfg.SOPS.AgeKeyPath, "--dry-run=client", "-o", "yaml") + manifest, err := utils.RunCommandOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "flux-system", "create", "secret", "generic", "sops-age", "--from-file=age.agekey="+ageKeyPath, "--dry-run=client", "-o", "yaml") if err != nil { return err } diff --git a/internal/bootstrap/bootstrap_test.go b/internal/bootstrap/bootstrap_test.go index cf9e9bc..a17aaa6 100644 --- a/internal/bootstrap/bootstrap_test.go +++ b/internal/bootstrap/bootstrap_test.go @@ -106,7 +106,7 @@ func TestRenderDemocraticCSISecret(t *testing.T) { if err != nil { t.Fatal(err) } - if !strings.Contains(string(secret), "name: democratic-csi-secrets") || !strings.Contains(string(secret), "dataset-parent-nfs: pool/kubernetes/nfs/v") { + if !strings.Contains(string(secret), "name: democratic-csi-secrets") || !strings.Contains(string(secret), "labels:\n reconcile.fluxcd.io/watch: Enabled") || !strings.Contains(string(secret), "dataset-parent-nfs: pool/kubernetes/nfs/v") { t.Fatalf("Democratic CSI secret was not rendered: %s", secret) } } @@ -275,6 +275,9 @@ func TestEnsureManifestsKustomizations(t *testing.T) { t.Fatal(err) } } + if err := os.WriteFile(filepath.Join(dir, "apps", "staging", "kustomization.yaml"), []byte("apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n"), 0644); err != nil { + t.Fatal(err) + } if err := ensureManifestsKustomizations(dir); err != nil { t.Fatal(err) } @@ -282,6 +285,25 @@ func TestEnsureManifestsKustomizations(t *testing.T) { if err != nil || !strings.Contains(string(content), "resources:") { t.Fatalf("preview Kustomization was not created: %q, %v", content, err) } + content, err = os.ReadFile(filepath.Join(dir, "apps", "staging", "kustomization.yaml")) + if err != nil || !strings.Contains(string(content), "resources:") { + t.Fatalf("staging Kustomization was not repaired: %q, %v", content, err) + } +} + +func TestEnsureClusterKustomizationsAddsStorageDependencies(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "kustomization.yaml") + if err := os.WriteFile(path, []byte("resources:\n"), 0644); err != nil { + t.Fatal(err) + } + if err := ensureClusterKustomizations(dir); err != nil { + t.Fatal(err) + } + content, err := os.ReadFile(path) + if err != nil || !strings.Contains(string(content), "snapshot-crds-kustomization.yaml") || !strings.Contains(string(content), "democratic-csi-kustomization.yaml") || !strings.Contains(string(content), "gateway-api-kustomization.yaml") || !strings.Contains(string(content), "gateway-kustomization.yaml") || !strings.Contains(string(content), "external-secrets-config-kustomization.yaml") || !strings.Contains(string(content), "external-dns-kustomization.yaml") || !strings.Contains(string(content), "monitoring-kustomization.yaml") || !strings.Contains(string(content), "tekton-kustomization.yaml") { + t.Fatalf("cluster Kustomization was not updated: %q, %v", content, err) + } } func TestRenderTerraformTFVarsIsStableAndRedactsToken(t *testing.T) { diff --git a/internal/openbao/bootstrap.go b/internal/openbao/bootstrap.go index a3acb94..a9c173b 100644 --- a/internal/openbao/bootstrap.go +++ b/internal/openbao/bootstrap.go @@ -88,7 +88,7 @@ func Initialize(kubeconfig, recipient, identityPath, bundlePath, ageKeyPath, ope return nil, err } } - reviewerToken, err := kubectlOutput(kubeconfig, "-n", "openbao", "create", "token", "openbao-auth") + reviewerToken, err := kubectlOutput(kubeconfig, "-n", "openbao", "create", "token", "openbao-auth", "--duration=8760h") if err != nil { return nil, fmt.Errorf("create OpenBao Kubernetes token reviewer token: %w", err) } @@ -179,10 +179,8 @@ func writeSecret(kubeconfig, rootToken, secretPath string, values map[string]str func waitForPod(kubeconfig string) error { deadline := time.Now().Add(10 * time.Minute) for time.Now().Before(deadline) { - if _, err := kubectlOutput(kubeconfig, "-n", "openbao", "get", "pod", "openbao-0"); err == nil { - if output, _ := execInPod(kubeconfig, nil, "bao", "status", "-format=json"); len(output) > 0 { - return nil - } + if _, err := getStatus(kubeconfig); err == nil { + return nil } time.Sleep(2 * time.Second) } @@ -190,8 +188,9 @@ func waitForPod(kubeconfig string) error { } func getStatus(kubeconfig string) (status, error) { - output, err := execInPod(kubeconfig, nil, "bao", "status", "-format=json") - if err != nil && len(output) == 0 { + command := []string{"--kubeconfig", kubeconfig, "-n", "openbao", "exec", "openbao-0", "--", "bao", "status", "-format=json"} + output, err := exec.Command("kubectl", command...).Output() + if err != nil && !json.Valid(output) { return status{}, fmt.Errorf("get OpenBao status: %w", err) } var current status diff --git a/internal/ui/wizard.go b/internal/ui/wizard.go index c0a2636..a4a4ce2 100644 --- a/internal/ui/wizard.go +++ b/internal/ui/wizard.go @@ -130,10 +130,12 @@ func PromptOperationalSecrets(cfg config.Config) (map[string]map[string]string, return nil, errors.New("git username, token, and delivery imageRepository are required") } reader := bufio.NewReader(os.Stdin) - cloudflareAPIToken := promptSecret(reader, "Cloudflare API token", "") + piholeServer := prompt(reader, "Pi-hole server", "") + piholePassword := promptSecret(reader, "Pi-hole password", "") + cloudflareAPIToken := promptSecret(reader, "Cloudflare DNS-01 API token", "") cloudflareTunnelToken := promptSecret(reader, "Cloudflare Tunnel token", "") - if cloudflareAPIToken == "" || cloudflareTunnelToken == "" { - return nil, errors.New("Cloudflare API and Tunnel tokens are required") + if piholeServer == "" || piholePassword == "" || cloudflareAPIToken == "" || cloudflareTunnelToken == "" { + return nil, errors.New("Pi-hole server, password, Cloudflare DNS-01 API token, and Cloudflare Tunnel token are required") } registryHost := strings.Split(cfg.Delivery.ImageRepository, "/")[0] dockerConfig, err := json.Marshal(map[string]map[string]map[string]string{ @@ -147,6 +149,7 @@ func PromptOperationalSecrets(cfg config.Config) (map[string]map[string]string, return map[string]map[string]string{ "cicd/forgejo": {"username": cfg.Git.Username, "token": cfg.Git.Token}, "cicd/forgejo-registry": {"dockerconfigjson": string(dockerConfig)}, + "platform/pihole": {"server": piholeServer, "password": piholePassword}, "platform/cloudflare": {"api-token": cloudflareAPIToken}, "platform/cloudflare-tunnel": {"token": cloudflareTunnelToken}, }, nil