MaidnCLI

@@ -1 +0,0 @@
-Subproject commit cc8fab6832a3a51bd072155f90f91354e440077a

README.md

@@ -11,13 +11,14 @@ in powershell run
 go install github.com/go-delve/delve/cmd/dlv@latest
 dlv version
 ```
-
 ## Commands
 
 - `cicd-tool repo init --org <org> --flux-repo <repo>` creates the manifests and Flux repos
 - `cicd-tool bootstrap` runs a shorter Forgejo-first wizard, asks for a Forgejo PAT, asks where local repos should be cloned, discovers Proxmox nodes/storage/networks, retries without losing entered answers when discovery fails, shows the latest Talos version, derives the standardized Talos factory URL, schematic, and required extensions automatically from the chosen version, writes `terraform.tfvars`, stages Talos images on Proxmox, and can execute Terraform, Talos bootstrap, and Flux bootstrap
 - `cicd-tool bootstrap --config maidn-bootstrap.yaml` skips the wizard and uses the saved config
 
+See `docs/operations.md` for the authorized operating and verification runbook.
+
 ## Forgejo setup
 
 For `https://git.pingu.pw` you need:
@@ -45,15 +46,3 @@ talos:
           ip: <node-traffic-address>
           vlanId: <opnsense-traffic-vlan>
 ```
-
-
-https://192.168.0.15:8006
-root@pam!maidn-test-key
-2ce7bff1-ac98-4a45-8db4-186d49ae4159
-
-test-org-test-key:147878348db3b8ab660b1af2799c3842705a31bc
-maidn-dev-work:fca448f2677362cc165eebb7859ce8f9f5e3735c
-
-https://192.168.0.13:8006
-root@pam!maidn-test-key
-3773033f-552b-4572-83e9-cd16e8ac2e13

cmd/bootstrap.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"path/filepath"
 
 	"github.com/Pingu-Studio/MaidnCLI/internal/bootstrap"
 	"github.com/Pingu-Studio/MaidnCLI/internal/config"
@@ -20,6 +21,8 @@ var bootstrapPromptOperationalSecrets bool
 var bootstrapInitializeOpenBaoRecovery bool
 var bootstrapInitializeOpenBao bool
 var bootstrapCreateForgejoRegistryToken bool
+var bootstrapRegisterWebhook bool
+var bootstrapRotateWebhookAuthorization bool
 var bootstrapPublishAppFrom string
 var bootstrapMergeBootstrapPR bool
 var bootstrapManageNetworkBridges bool
@@ -41,6 +44,8 @@ func init() {
 	bootstrapCmd.Flags().BoolVar(&bootstrapInitializeOpenBaoRecovery, "initialize-openbao-recovery", false, "Create and save a separate OpenBao recovery age identity for --config")
 	bootstrapCmd.Flags().BoolVar(&bootstrapInitializeOpenBao, "initialize-openbao", false, "Initialize OpenBao and seed encrypted operational secrets for --config")
 	bootstrapCmd.Flags().BoolVar(&bootstrapCreateForgejoRegistryToken, "create-forgejo-registry-token", false, "Create a least-privilege Forgejo package registry token and seed it through OpenBao")
+	bootstrapCmd.Flags().BoolVar(&bootstrapRegisterWebhook, "register-webhook", false, "Seed OpenBao secrets and register the Forgejo webhook")
+	bootstrapCmd.Flags().BoolVar(&bootstrapRotateWebhookAuthorization, "rotate-webhook-authorization", false, "Replace the Forgejo webhook authorization and reconcile it through OpenBao")
 	bootstrapCmd.Flags().StringVar(&bootstrapPublishAppFrom, "publish-app-from", "", "Push this app checkout's current branch and create a Forgejo delivery PR")
 	bootstrapCmd.Flags().BoolVar(&bootstrapMergeBootstrapPR, "merge-bootstrap-pr", false, "Merge the generated Flux repository migration PR before bootstrapping")
 	bootstrapCmd.Flags().BoolVar(&bootstrapManageNetworkBridges, "manage-network-bridges", false, "Persist Terraform management for existing Talos network bridges")
@@ -59,6 +64,23 @@ func runBootstrap(cmd *cobra.Command, args []string) error {
 		}
 		return createForgejoRegistryToken(cfg)
 	}
+	if bootstrapRotateWebhookAuthorization {
+		if bootstrapConfigPath == "" {
+			return fmt.Errorf("--rotate-webhook-authorization requires --config")
+		}
+		cfg, err = config.Load(bootstrapConfigPath)
+		if err != nil {
+			return err
+		}
+		authorization, err := bootstrap.NewWebhookAuthorization()
+		if err != nil {
+			return fmt.Errorf("generate Forgejo webhook authorization: %w", err)
+		}
+		if err := bootstrap.UpsertOperationalSecret(cfg.SOPS.OperationalSecretsPath, cfg.SOPS.AgeKeyPath, "cicd/forgejo-webhook", "authorization", authorization); err != nil {
+			return fmt.Errorf("save Forgejo webhook authorization: %w", err)
+		}
+		return bootstrap.Runner{Config: cfg, RegisterWebhook: true}.Run()
+	}
 	if bootstrapInitializeOpenBao {
 		if bootstrapConfigPath == "" {
 			return fmt.Errorf("--initialize-openbao requires --config")
@@ -117,6 +139,18 @@ func runBootstrap(cmd *cobra.Command, args []string) error {
 		}
 		return manager.CreatePullRequest(repo, "feat: migrate delivery to Tekton", branch, cfg.Delivery.AppRepoRef)
 	}
+	if bootstrapInitializeOpenBao {
+		if bootstrapConfigPath == "" {
+			return fmt.Errorf("--initialize-openbao requires --config")
+		}
+		cfg, err = config.Load(bootstrapConfigPath)
+		if err != nil {
+			return err
+		}
+		generatedDir := filepath.Join(cfg.Git.CloneParent, cfg.Talos.RepoDirName, cfg.Talos.GeneratedDir)
+		_, err = openbao.Initialize(filepath.Join(generatedDir, "kubeconfig"), cfg.SOPS.RecoveryRecipient, cfg.SOPS.RecoveryIdentityPath, cfg.SOPS.RecoveryBundlePath, cfg.SOPS.AgeKeyPath, cfg.SOPS.OperationalSecretsPath)
+		return err
+	}
 
 	if bootstrapConfigPath != "" {
 		if bootstrapPromptDemocraticCSI || bootstrapPromptOperationalSecrets || bootstrapInitializeOpenBaoRecovery || bootstrapManageNetworkBridges {
@@ -171,7 +205,7 @@ func runBootstrap(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	runner := bootstrap.Runner{Config: cfg, Mode: bootstrap.Mode(bootstrapMode), ConfirmRebuild: bootstrapYes}
+	runner := bootstrap.Runner{Config: cfg, Mode: bootstrap.Mode(bootstrapMode), ConfirmRebuild: bootstrapYes, RegisterWebhook: bootstrapRegisterWebhook}
 	return runner.Run()
 }
 

docs/operations.md

@@ -0,0 +1,142 @@
+# Operations Runbook
+
+Use this runbook from the MaidnCLI checkout. Bootstrap YAML, age identities,
+OpenBao recovery material, Terraform state, and generated workspaces are
+secret-bearing local inputs. Do not commit or print them.
+
+## Normal Reconciliation
+
+Run after a merged GitOps migration or to recover ordinary drift:
+
+```powershell
+go run . bootstrap --config <private-bootstrap-config> --mode=reconcile
+```
+
+This is the only regular lifecycle command. Do not use direct `kubectl apply`,
+`flux reconcile`, Helm upgrades, or mutating `talosctl` commands.
+
+## Rebuild
+
+Use only when an authorized recovery requires recreating the Talos VM:
+
+```powershell
+go run . bootstrap --config <private-bootstrap-config> --mode=rebuild --yes
+```
+
+The rebuild replaces Terraform-managed Talos VMs only. It does not manage or
+delete TrueNAS datasets or unrelated infrastructure.
+
+## OpenBao And Webhooks
+
+After a rebuild or an OpenBao restart, refresh Kubernetes auth and reseed the
+encrypted operational values:
+
+```powershell
+go run . bootstrap --config <private-bootstrap-config> --initialize-openbao
+```
+
+When the Pipeline and EventListener are already Ready, register or update the
+Forgejo webhook without re-running the full lifecycle:
+
+```powershell
+go run . bootstrap --config <private-bootstrap-config> --register-webhook
+```
+
+The webhook authorization value stays in encrypted operational secrets and is
+never supplied on the command line.
+
+If that authorization value is exposed, replace it and reconcile both OpenBao
+and the Forgejo hook in one command:
+
+```powershell
+go run . bootstrap --config <private-bootstrap-config> --rotate-webhook-authorization
+```
+
+## Read-Only Verification
+
+Set `KUBECONFIG` to the generated kubeconfig for the configured cluster, then
+check the control plane and delivery chain:
+
+```powershell
+kubectl -n flux-system get kustomizations
+kubectl get clustersecretstores
+kubectl -n flux-system get externalsecrets
+kubectl -n tekton-pipelines get pipelines,eventlisteners,externalsecrets
+```
+
+Expected state:
+
+- Flux Kustomizations are `READY=True`.
+- `ClusterSecretStore/openbao` is `READY=True`.
+- Forgejo credential ExternalSecrets are `SecretSynced`.
+- The application Pipeline exists and the EventListener is available.
+
+If OpenBao authentication is invalid after a rebuild, run
+`--initialize-openbao`, then allow the controllers to retry. Do not recreate
+the ClusterSecretStore or Secrets manually.
+
+## External DNS
+
+Webhook delivery requires the configured `tekton.<cluster-domain>` hostname to
+resolve through Pi-hole to the Cilium Gateway address. ExternalDNS uses the
+Pi-hole provider with Gateway API routes, an `upsert-only` policy, and no
+ownership registry.
+
+```powershell
+kubectl -n external-dns get pods,externalsecrets
+kubectl -n external-dns logs deployment/external-dns --tail=100
+```
+
+If `ExternalSecret/pihole-credentials` is not `SecretSynced`, rerun
+`--prompt-operational-secrets` to enter the Pi-hole server and password, then
+run `--initialize-openbao` and `--register-webhook`. Do not create or edit the
+provider Secret directly.
+
+## Webhook TLS
+
+The public Gateway terminates HTTPS with a cert-manager certificate. Its
+Cloudflare DNS-01 token is used only to issue the `nid3.com` certificate;
+Pi-hole remains the ExternalDNS provider. Check certificate readiness with:
+
+```powershell
+kubectl -n cert-manager get externalsecret cloudflare-api-token
+kubectl -n gateway-system get certificate webhook-tls
+```
+
+Enter the Pi-hole values, Cloudflare DNS-01 token, and Tunnel token through
+`--prompt-operational-secrets`, then run `--initialize-openbao`. Do not put the
+Cloudflare token in the cluster repository.
+
+## Internal Platform UIs
+
+Pi-hole resolves these HTTPS names to the Cilium Gateway only on the LAN:
+
+- `https://grafana.<cluster-domain>/` for Grafana. Authenticate with Grafana.
+- `https://openbao.<cluster-domain>/` for OpenBao. Authenticate with an OpenBao token.
+
+Hubble UI is enabled for in-cluster troubleshooting but has no LAN route because
+it does not provide authentication. Add an authenticated proxy before exposing
+it outside the cluster.
+
+## Webhook Smoke Test
+
+Use Forgejo's hook test endpoint against an existing non-`main` ref. It emits
+a real push delivery, runs the Node build, and pushes a SHA-tagged registry
+image, but skips the main-only staging manifest update:
+
+```powershell
+# Discover the hook ID and choose an existing non-main branch or tag.
+Invoke-RestMethod -Headers @{ Authorization = "token $env:FORGEJO_TOKEN" } `
+  -Uri "https://<forgejo>/api/v1/repos/<owner>/<repo>/hooks"
+
+Invoke-WebRequest -Method Post -Headers @{ Authorization = "token $env:FORGEJO_TOKEN" } `
+  -Uri "https://<forgejo>/api/v1/repos/<owner>/<repo>/hooks/<hook-id>/tests?ref=<non-main-ref>"
+```
+
+Forgejo returns `204` after accepting the delivery. Confirm the resulting
+PipelineRun instead of treating `204` as a successful build:
+
+```powershell
+kubectl -n tekton-pipelines get pipelineruns
+kubectl -n tekton-pipelines describe pipelinerun <name>
+```

docs/secrets.md

@@ -17,6 +17,9 @@ secrets:
     dockerconfigjson: encrypted-value
   cicd/forgejo-webhook:
     authorization: encrypted-value
+  platform/pihole:
+    server: encrypted-value
+    password: encrypted-value
   platform/cloudflare:
     api-token: encrypted-value
   platform/cloudflare-tunnel:
@@ -34,8 +37,10 @@ For an existing configuration, run `bootstrap --config <path>
 
 For a new operational-secret input, run `bootstrap --config <path>
 --prompt-operational-secrets`. It derives Forgejo Git and registry credentials
-from the configured Forgejo account, prompts for Cloudflare credentials with
+from the configured Forgejo account, prompts for the Pi-hole server and masked
-input masked, and generates the webhook authorization value.
+password, masked Cloudflare DNS-01 and Tunnel tokens, and generates the webhook
+authorization value. The DNS-01 token issues the Gateway certificate; it is not
+used by ExternalDNS.
 
 `cicd/forgejo-webhook.authorization` is required for delivery bootstrap. The
 CLI supplies it as the Forgejo webhook Authorization header and Tekton compares

internal/bootstrap/bootstrap.go

@@ -2,6 +2,7 @@ package bootstrap
 
 import (
 	"bytes"
+	"context"
 	"crypto/rand"
 	"encoding/base64"
 	"encoding/json"
@@ -35,6 +36,7 @@ type Runner struct {
 	Config          config.Config
 	Mode            Mode
 	ConfirmRebuild  bool
+	RegisterWebhook bool
 }
 
 type operationalSecrets struct {
@@ -72,6 +74,9 @@ func (r Runner) Run() error {
 	if r.Mode == Rebuild && !r.ConfirmRebuild {
 		return fmt.Errorf("rebuild is destructive; rerun with --mode=rebuild --yes")
 	}
+	if r.RegisterWebhook {
+		return r.reconcileWebhook(filepath.Join(r.Config.Git.CloneParent, r.Config.Talos.RepoDirName, r.Config.Talos.GeneratedDir))
+	}
 
 	workspace := r.Config.WorkspaceDir
 	if err := os.MkdirAll(workspace, 0755); err != nil {
@@ -107,9 +112,30 @@ func (r Runner) Run() error {
 			if err := copyDir(filepath.Join(cicdTemplateDir, "base"), filepath.Join(dir, "base"), false); err != nil {
 				return err
 			}
+			for _, component := range []string{"snapshot-crds", "democratic-csi", "cert-manager", "cluster-issuers", "gateway-api", "gateway", "monitoring", "openbao", "external-secrets", "external-secrets-config", "external-dns", "tekton", "tekton-triggers"} {
+				if err := copyDir(filepath.Join(cicdTemplateDir, "base", component), filepath.Join(dir, "base", component), true); err != nil {
+					return err
+				}
+			}
 			if err := copyClusterTemplate(filepath.Join(cicdTemplateDir, "clusters", "template"), clusterDir); err != nil {
 				return err
 			}
+			for _, name := range []string{"external-secrets", "cnpg", "cloudflare-tunnel", "external-dns", "monitoring", "tekton"} {
+				content, err := os.ReadFile(filepath.Join(cicdTemplateDir, "clusters", "template", name+"-kustomization.yaml"))
+				if err != nil {
+					return err
+				}
+				if err := os.WriteFile(filepath.Join(clusterDir, name+"-kustomization.yaml"), content, 0644); err != nil {
+					return err
+				}
+			}
+			csiKustomization, err := os.ReadFile(filepath.Join(cicdTemplateDir, "clusters", "template", "democratic-csi-kustomization.yaml"))
+			if err != nil {
+				return err
+			}
+			if err := os.WriteFile(filepath.Join(clusterDir, "democratic-csi-kustomization.yaml"), csiKustomization, 0644); err != nil {
+				return err
+			}
 			if err := copyDir(filepath.Join(cicdTemplateDir, "base", "cilium"), filepath.Join(dir, "base", "cilium"), true); err != nil {
 				return err
 			}
@@ -178,6 +204,9 @@ func (r Runner) Run() error {
 		if err := utils.RunCommandInDir(generatedDir, "talosctl", "kubeconfig", "--talosconfig=./clusterconfig/talosconfig", "--nodes="+r.Config.Talos.KubeconfigNode, "."); err != nil {
 			return err
 		}
+		if err := waitForKubernetesAPI(generatedDir); err != nil {
+			return err
+		}
 	}
 	if r.Config.Talos.AutoBootstrapFlux {
 		if err := installCilium(generatedDir, r.Config); err != nil {
@@ -192,6 +221,12 @@ func (r Runner) Run() error {
 		if err := configureFluxSOPS(generatedDir); err != nil {
 			return err
 		}
+		return r.reconcileWebhook(generatedDir)
+	}
+	return nil
+}
+
+func (r Runner) reconcileWebhook(generatedDir string) error {
 	operationalSecrets, err := openbao.Initialize(filepath.Join(generatedDir, "kubeconfig"), r.Config.SOPS.RecoveryRecipient, r.Config.SOPS.RecoveryIdentityPath, r.Config.SOPS.RecoveryBundlePath, r.Config.SOPS.AgeKeyPath, r.Config.SOPS.OperationalSecretsPath)
 	if err != nil {
 		return fmt.Errorf("initialize OpenBao: %w", err)
@@ -203,10 +238,10 @@ func (r Runner) Run() error {
 	if err := waitForWebhookTargets(generatedDir, r.Config); err != nil {
 		return err
 	}
+	manager := forgejo.NewRepoManager(r.Config.Git.BaseURL, r.Config.Git.Token, r.Config.Git.Owner, r.Config.Git.Username, r.Config.Flux.ManifestsRepo, r.Config.Flux.RepoName, r.Config.Flux.Branch, "maidn/bootstrap-"+r.Config.ClusterID)
 	if err := manager.EnsureWebhook(r.Config.Delivery.AppName, r.Config.Delivery.WebhookURL(), authorization); err != nil {
 		return fmt.Errorf("reconcile Forgejo webhook: %w", err)
 	}
-	}
 	return nil
 }
 
@@ -236,6 +271,9 @@ func renderDeliveryConfig(dir string, cfg config.Config) error {
 		"${APP_REPO_REF}", cfg.Delivery.AppRepoRef,
 		"${IMAGE_REPOSITORY}", cfg.Delivery.ImageRepository,
 		"${FORGEJO_BASE_URL}", cfg.Git.BaseURL,
+		"${CLUSTER_DOMAIN}", cfg.Flux.ClusterDomain,
+		"${TEKTON_CATALOG_REPO_URL}", cfg.Templates.TektonCatalogRepoURL,
+		"${TEKTON_CATALOG_REPO_REF}", cfg.Templates.TektonCatalogRepoRef,
 		"${WEBHOOK_HOSTNAME}", cfg.Delivery.WebhookHostname,
 		"${WEBHOOK_PATH}", cfg.Delivery.WebhookPath,
 		"${TEKTON_CATALOG_REPO_URL}", cfg.Templates.TektonCatalogRepoURL,
@@ -397,13 +435,19 @@ func renderDemocraticCSISecret(csi config.DemocraticCSIConfig) ([]byte, error) {
 	return yaml.Marshal(struct {
 		APIVersion string            `yaml:"apiVersion"`
 		Kind       string            `yaml:"kind"`
-		Metadata   map[string]string `yaml:"metadata"`
+		Metadata   map[string]any    `yaml:"metadata"`
 		Type       string            `yaml:"type"`
 		StringData map[string]string `yaml:"stringData"`
 	}{
 		APIVersion: "v1",
 		Kind:       "Secret",
-		Metadata:   map[string]string{"name": "democratic-csi-secrets", "namespace": "democratic-storage"},
+		Metadata: map[string]any{
+			"name":      "democratic-csi-secrets",
+			"namespace": "democratic-storage",
+			"labels": map[string]string{
+				"reconcile.fluxcd.io/watch": "Enabled",
+			},
+		},
 		Type: "Opaque",
 		StringData: map[string]string{
 			"truenas-api-key":       csi.TrueNASAPIKey,
@@ -454,7 +498,7 @@ func ensureClusterKustomizations(clusterDir string) error {
 	}
 	updated := string(content)
 	updated = strings.ReplaceAll(updated, "  - bootstrap-secrets.sops.yaml\n", "")
-	for _, resource := range []string{"cilium-kustomization.yaml", "cilium-config-kustomization.yaml", "cicd-manifests-repo.yaml", "tekton-triggers-kustomization.yaml"} {
+	for _, resource := range []string{"snapshot-crds-kustomization.yaml", "democratic-csi-kustomization.yaml", "cert-manager-kustomization.yaml", "cluster-issuers-kustomization.yaml", "gateway-api-kustomization.yaml", "gateway-kustomization.yaml", "cilium-kustomization.yaml", "cilium-config-kustomization.yaml", "cicd-manifests-repo.yaml", "external-secrets-config-kustomization.yaml", "external-dns-kustomization.yaml", "monitoring-kustomization.yaml", "tekton-kustomization.yaml", "tekton-triggers-kustomization.yaml"} {
 		if !strings.Contains(updated, resource) {
 			updated += "  - " + resource + "\n"
 		}
@@ -484,7 +528,14 @@ func copyClusterTemplate(source, destination string) error {
 func ensureManifestsKustomizations(dir string) error {
 	for _, environment := range []string{"previews", "staging", "production"} {
 		path := filepath.Join(dir, "apps", environment, "kustomization.yaml")
-		if _, err := os.Stat(path); err == nil {
+		content, err := os.ReadFile(path)
+		if err == nil {
+			if strings.Contains(string(content), "resources:") {
+				continue
+			}
+			if err := os.WriteFile(path, append(content, []byte("resources:\n")...), 0644); err != nil {
+				return err
+			}
 			continue
 		} else if !os.IsNotExist(err) {
 			return err
@@ -501,7 +552,7 @@ func installCilium(dir string, cfg config.Config) error {
 	if err := os.MkdirAll(helmDir, 0755); err != nil {
 		return err
 	}
-	return utils.RunCommandInDir(dir, "helm", "upgrade", "--install", "cilium", "cilium", "--repo=https://helm.cilium.io", "--version=1.19.6", "--repository-config="+filepath.Join(helmDir, "repositories.yaml"), "--repository-cache="+helmDir, "--namespace=kube-system", "--create-namespace", "--kubeconfig=kubeconfig", "--wait", "--timeout=5m", "--set=kubeProxyReplacement=true", "--set=ipam.mode=kubernetes", "--set=k8sServiceHost=localhost", "--set=k8sServicePort=7445", "--set=cgroup.autoMount.enabled=false", "--set=cgroup.hostRoot=/sys/fs/cgroup", "--set=bpf.hostLegacyRouting=true", "--set=securityContext.capabilities.ciliumAgent={CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}", "--set=securityContext.capabilities.cleanCiliumState={NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}", "--set=gatewayAPI.enabled=false", "--set=l2announcements.enabled=true", "--set=operator.replicas=1")
+	return utils.RunCommandInDir(dir, "helm", "upgrade", "--install", "cilium", "cilium", "--repo=https://helm.cilium.io", "--version=1.19.6", "--repository-config="+filepath.Join(helmDir, "repositories.yaml"), "--repository-cache="+helmDir, "--namespace=kube-system", "--create-namespace", "--kubeconfig=kubeconfig", "--wait", "--timeout=5m", "--set=kubeProxyReplacement=true", "--set=ipam.mode=kubernetes", "--set=k8sServiceHost=localhost", "--set=k8sServicePort=7445", "--set=cgroup.autoMount.enabled=false", "--set=cgroup.hostRoot=/sys/fs/cgroup", "--set=bpf.hostLegacyRouting=true", "--set=securityContext.capabilities.ciliumAgent={CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}", "--set=securityContext.capabilities.cleanCiliumState={NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}", "--set=envoy.enabled=true", "--set=gatewayAPI.enabled=true", "--set=l2announcements.enabled=true", "--set=rollOutCiliumPods=true", "--set=operator.replicas=1", "--set=operator.rollOutPods=true")
 }
 
 func copyDir(source, destination string, overwrite bool) error {
@@ -564,12 +615,12 @@ func applyTalosConfigs(dir string, cfg config.Config) error {
 		configFile := filepath.Join("clusterconfig", fmt.Sprintf("%s-%s.yaml", cfg.Talos.Cluster.Name, node.Name))
 		nodeAddress := node.Networks[0].IP
 		secureArgs := []string{"apply-config", "--talosconfig=./clusterconfig/talosconfig", "--nodes=" + nodeAddress, "--endpoints=" + cfg.Talos.BootstrapEndpoint, "--file=" + configFile}
-		if _, err := utils.RunCommandQuietOutputInDir(dir, "talosctl", "get", "machinestatus", "--talosconfig=./clusterconfig/talosconfig", "--nodes="+nodeAddress, "--endpoints="+cfg.Talos.BootstrapEndpoint, "--output=json"); err == nil {
+		if _, err := runTalosctlOutput(dir, "get", "machinestatus", "--talosconfig=./clusterconfig/talosconfig", "--nodes="+nodeAddress, "--endpoints="+cfg.Talos.BootstrapEndpoint, "--output=json"); err == nil {
 			if err := utils.RunCommandInDir(dir, "talosctl", secureArgs...); err != nil {
 				return fmt.Errorf("apply Talos config to %s: %w", node.Name, err)
 			}
 		} else {
-			if _, maintenanceErr := utils.RunCommandQuietOutputInDir(dir, "talosctl", "get", "machinestatus", "--insecure", "--nodes="+nodeAddress, "--endpoints="+nodeAddress, "--output=json"); maintenanceErr == nil {
+			if _, maintenanceErr := runTalosctlOutput(dir, "get", "machinestatus", "--insecure", "--nodes="+nodeAddress, "--endpoints="+nodeAddress, "--output=json"); maintenanceErr == nil {
 				if err := utils.RunCommandInDir(dir, "talosctl", "apply-config", "--insecure", "--nodes="+nodeAddress, "--endpoints="+nodeAddress, "--file="+configFile); err != nil {
 					return fmt.Errorf("apply initial Talos config to %s: %w", node.Name, err)
 				}
@@ -590,10 +641,32 @@ func applyTalosConfigs(dir string, cfg config.Config) error {
 }
 
 func bootstrapEtcdIfNeeded(dir string, cfg config.Config) error {
-	if _, err := utils.RunCommandQuietOutputInDir(dir, "talosctl", "etcd", "status", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode); err == nil {
+	if _, err := runTalosctlOutput(dir, "etcd", "status", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode); err == nil {
 		return nil
 	}
-	return utils.RunCommandInDir(dir, "talosctl", "bootstrap", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode)
+	if err := waitForTalosAPI(dir, cfg, cfg.Talos.BootstrapNode); err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	defer cancel()
+	command := exec.CommandContext(ctx, "talosctl", "bootstrap", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode)
+	command.Dir = dir
+	command.Stdout = os.Stdout
+	command.Stderr = os.Stderr
+	if err := command.Run(); err != nil {
+		if ctx.Err() != nil {
+			return errors.New("etcd bootstrap request timed out")
+		}
+		return fmt.Errorf("bootstrap etcd: %w", err)
+	}
+	deadline := time.Now().Add(2 * time.Minute)
+	for time.Now().Before(deadline) {
+		if _, err := runTalosctlOutput(dir, "etcd", "status", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode); err == nil {
+			return nil
+		}
+		time.Sleep(2 * time.Second)
+	}
+	return errors.New("etcd did not become healthy after bootstrap")
 }
 
 func waitForTalosReboot(dir string, cfg config.Config, node string) error {
@@ -603,7 +676,7 @@ func waitForTalosReboot(dir string, cfg config.Config, node string) error {
 func waitForTalosAPI(dir string, cfg config.Config, node string) error {
 	deadline := time.Now().Add(5 * time.Minute)
 	for time.Now().Before(deadline) {
-		if _, err := utils.RunCommandQuietOutputInDir(dir, "talosctl", "get", "machinestatus", "--output=json", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+node, "--nodes="+node); err == nil {
+		if _, err := runTalosctlOutput(dir, "get", "machinestatus", "--output=json", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+node, "--nodes="+node); err == nil {
 			return nil
 		}
 		time.Sleep(2 * time.Second)
@@ -611,6 +684,25 @@ func waitForTalosAPI(dir string, cfg config.Config, node string) error {
 	return fmt.Errorf("Talos API did not return after applying its machine configuration")
 }
 
+func runTalosctlOutput(dir string, args ...string) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+	command := exec.CommandContext(ctx, "talosctl", args...)
+	command.Dir = dir
+	return command.Output()
+}
+
+func waitForKubernetesAPI(dir string) error {
+	deadline := time.Now().Add(5 * time.Minute)
+	for time.Now().Before(deadline) {
+		if _, err := utils.RunCommandQuietOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "get", "--raw=/readyz"); err == nil {
+			return nil
+		}
+		time.Sleep(2 * time.Second)
+	}
+	return errors.New("Kubernetes API did not become ready after Talos bootstrap")
+}
+
 type lifecycle struct {
 	ClusterID   string `yaml:"clusterId"`
 	ClusterName string `yaml:"clusterName"`
@@ -674,10 +766,14 @@ func terraformPlanPath(terraformDir, clusterID string) (string, error) {
 }
 
 func installSOPSKey(dir string, cfg config.Config) error {
-	if _, err := os.Stat(cfg.SOPS.AgeKeyPath); err != nil {
+	ageKeyPath, err := filepath.Abs(cfg.SOPS.AgeKeyPath)
+	if err != nil {
+		return fmt.Errorf("resolve SOPS age identity: %w", err)
+	}
+	if _, err := os.Stat(ageKeyPath); err != nil {
 		return fmt.Errorf("read SOPS age identity: %w", err)
 	}
-	manifest, err := utils.RunCommandOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "flux-system", "create", "secret", "generic", "sops-age", "--from-file=age.agekey="+cfg.SOPS.AgeKeyPath, "--dry-run=client", "-o", "yaml")
+	manifest, err := utils.RunCommandOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "flux-system", "create", "secret", "generic", "sops-age", "--from-file=age.agekey="+ageKeyPath, "--dry-run=client", "-o", "yaml")
 	if err != nil {
 		return err
 	}

internal/bootstrap/bootstrap_test.go

@@ -106,7 +106,7 @@ func TestRenderDemocraticCSISecret(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !strings.Contains(string(secret), "name: democratic-csi-secrets") || !strings.Contains(string(secret), "dataset-parent-nfs: pool/kubernetes/nfs/v") {
+	if !strings.Contains(string(secret), "name: democratic-csi-secrets") || !strings.Contains(string(secret), "labels:\n        reconcile.fluxcd.io/watch: Enabled") || !strings.Contains(string(secret), "dataset-parent-nfs: pool/kubernetes/nfs/v") {
 		t.Fatalf("Democratic CSI secret was not rendered: %s", secret)
 	}
 }
@@ -275,6 +275,9 @@ func TestEnsureManifestsKustomizations(t *testing.T) {
 			t.Fatal(err)
 		}
 	}
+	if err := os.WriteFile(filepath.Join(dir, "apps", "staging", "kustomization.yaml"), []byte("apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n"), 0644); err != nil {
+		t.Fatal(err)
+	}
 	if err := ensureManifestsKustomizations(dir); err != nil {
 		t.Fatal(err)
 	}
@@ -282,6 +285,25 @@ func TestEnsureManifestsKustomizations(t *testing.T) {
 	if err != nil || !strings.Contains(string(content), "resources:") {
 		t.Fatalf("preview Kustomization was not created: %q, %v", content, err)
 	}
+	content, err = os.ReadFile(filepath.Join(dir, "apps", "staging", "kustomization.yaml"))
+	if err != nil || !strings.Contains(string(content), "resources:") {
+		t.Fatalf("staging Kustomization was not repaired: %q, %v", content, err)
+	}
+}
+
+func TestEnsureClusterKustomizationsAddsStorageDependencies(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "kustomization.yaml")
+	if err := os.WriteFile(path, []byte("resources:\n"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	if err := ensureClusterKustomizations(dir); err != nil {
+		t.Fatal(err)
+	}
+	content, err := os.ReadFile(path)
+	if err != nil || !strings.Contains(string(content), "snapshot-crds-kustomization.yaml") || !strings.Contains(string(content), "democratic-csi-kustomization.yaml") || !strings.Contains(string(content), "gateway-api-kustomization.yaml") || !strings.Contains(string(content), "gateway-kustomization.yaml") || !strings.Contains(string(content), "external-secrets-config-kustomization.yaml") || !strings.Contains(string(content), "external-dns-kustomization.yaml") || !strings.Contains(string(content), "monitoring-kustomization.yaml") || !strings.Contains(string(content), "tekton-kustomization.yaml") {
+		t.Fatalf("cluster Kustomization was not updated: %q, %v", content, err)
+	}
 }
 
 func TestRenderTerraformTFVarsIsStableAndRedactsToken(t *testing.T) {

internal/openbao/bootstrap.go

@@ -88,7 +88,7 @@ func Initialize(kubeconfig, recipient, identityPath, bundlePath, ageKeyPath, ope
 			return nil, err
 		}
 	}
-	reviewerToken, err := kubectlOutput(kubeconfig, "-n", "openbao", "create", "token", "openbao-auth")
+	reviewerToken, err := kubectlOutput(kubeconfig, "-n", "openbao", "create", "token", "openbao-auth", "--duration=8760h")
 	if err != nil {
 		return nil, fmt.Errorf("create OpenBao Kubernetes token reviewer token: %w", err)
 	}
@@ -179,19 +179,18 @@ func writeSecret(kubeconfig, rootToken, secretPath string, values map[string]str
 func waitForPod(kubeconfig string) error {
 	deadline := time.Now().Add(10 * time.Minute)
 	for time.Now().Before(deadline) {
-		if _, err := kubectlOutput(kubeconfig, "-n", "openbao", "get", "pod", "openbao-0"); err == nil {
+		if _, err := getStatus(kubeconfig); err == nil {
-			if output, _ := execInPod(kubeconfig, nil, "bao", "status", "-format=json"); len(output) > 0 {
 			return nil
 		}
-		}
 		time.Sleep(2 * time.Second)
 	}
 	return fmt.Errorf("OpenBao pod did not become ready")
 }
 
 func getStatus(kubeconfig string) (status, error) {
-	output, err := execInPod(kubeconfig, nil, "bao", "status", "-format=json")
+	command := []string{"--kubeconfig", kubeconfig, "-n", "openbao", "exec", "openbao-0", "--", "bao", "status", "-format=json"}
-	if err != nil && len(output) == 0 {
+	output, err := exec.Command("kubectl", command...).Output()
+	if err != nil && !json.Valid(output) {
 		return status{}, fmt.Errorf("get OpenBao status: %w", err)
 	}
 	var current status

internal/ui/wizard.go

@@ -130,10 +130,12 @@ func PromptOperationalSecrets(cfg config.Config) (map[string]map[string]string,
 		return nil, errors.New("git username, token, and delivery imageRepository are required")
 	}
 	reader := bufio.NewReader(os.Stdin)
-	cloudflareAPIToken := promptSecret(reader, "Cloudflare API token", "")
+	piholeServer := prompt(reader, "Pi-hole server", "")
+	piholePassword := promptSecret(reader, "Pi-hole password", "")
+	cloudflareAPIToken := promptSecret(reader, "Cloudflare DNS-01 API token", "")
 	cloudflareTunnelToken := promptSecret(reader, "Cloudflare Tunnel token", "")
-	if cloudflareAPIToken == "" || cloudflareTunnelToken == "" {
+	if piholeServer == "" || piholePassword == "" || cloudflareAPIToken == "" || cloudflareTunnelToken == "" {
-		return nil, errors.New("Cloudflare API and Tunnel tokens are required")
+		return nil, errors.New("Pi-hole server, password, Cloudflare DNS-01 API token, and Cloudflare Tunnel token are required")
 	}
 	registryHost := strings.Split(cfg.Delivery.ImageRepository, "/")[0]
 	dockerConfig, err := json.Marshal(map[string]map[string]map[string]string{
@@ -147,6 +149,7 @@ func PromptOperationalSecrets(cfg config.Config) (map[string]map[string]string,
 	return map[string]map[string]string{
 		"cicd/forgejo":               {"username": cfg.Git.Username, "token": cfg.Git.Token},
 		"cicd/forgejo-registry":      {"dockerconfigjson": string(dockerConfig)},
+		"platform/pihole":            {"server": piholeServer, "password": piholePassword},
 		"platform/cloudflare":        {"api-token": cloudflareAPIToken},
 		"platform/cloudflare-tunnel": {"token": cloudflareTunnelToken},
 	}, nil