diff --git a/README.md b/README.md index d04122a..63cc3bd 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,14 @@ in powershell run go install github.com/go-delve/delve/cmd/dlv@latest dlv version ``` - ## Commands - `cicd-tool repo init --org --flux-repo ` creates the manifests and Flux repos - `cicd-tool bootstrap` runs a shorter Forgejo-first wizard, asks for a Forgejo PAT, asks where local repos should be cloned, discovers Proxmox nodes/storage/networks, retries without losing entered answers when discovery fails, shows the latest Talos version, derives the standardized Talos factory URL, schematic, and required extensions automatically from the chosen version, writes `terraform.tfvars`, stages Talos images on Proxmox, and can execute Terraform, Talos bootstrap, and Flux bootstrap - `cicd-tool bootstrap --config maidn-bootstrap.yaml` skips the wizard and uses the saved config +See `docs/operations.md` for the authorized operating and verification runbook. + ## Forgejo setup For `https://git.pingu.pw` you need: @@ -45,15 +46,3 @@ talos: ip: vlanId: ``` - - -https://192.168.0.15:8006 -root@pam!maidn-test-key -2ce7bff1-ac98-4a45-8db4-186d49ae4159 - -test-org-test-key:147878348db3b8ab660b1af2799c3842705a31bc -maidn-dev-work:fca448f2677362cc165eebb7859ce8f9f5e3735c - -https://192.168.0.13:8006 -root@pam!maidn-test-key -3773033f-552b-4572-83e9-cd16e8ac2e13 \ No newline at end of file diff --git a/cmd/bootstrap.go b/cmd/bootstrap.go index f412708..564c5a2 100644 --- a/cmd/bootstrap.go +++ b/cmd/bootstrap.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "path/filepath" "github.com/Pingu-Studio/MaidnCLI/internal/bootstrap" "github.com/Pingu-Studio/MaidnCLI/internal/config" @@ -20,10 +21,14 @@ var bootstrapPromptOperationalSecrets bool var bootstrapInitializeOpenBaoRecovery bool var bootstrapInitializeOpenBao bool var bootstrapCreateForgejoRegistryToken bool +var bootstrapRegisterWebhook bool +var bootstrapRotateWebhookAuthorization bool var bootstrapPublishAppFrom string var bootstrapMergeBootstrapPR bool var bootstrapManageNetworkBridges bool +var upsertOperationalSecret = bootstrap.UpsertOperationalSecret + var bootstrapCmd = &cobra.Command{ Use: "bootstrap", Short: "Bootstrap Talos and Flux from config or an interactive wizard.", @@ -41,6 +46,8 @@ func init() { bootstrapCmd.Flags().BoolVar(&bootstrapInitializeOpenBaoRecovery, "initialize-openbao-recovery", false, "Create and save a separate OpenBao recovery age identity for --config") bootstrapCmd.Flags().BoolVar(&bootstrapInitializeOpenBao, "initialize-openbao", false, "Initialize OpenBao and seed encrypted operational secrets for --config") bootstrapCmd.Flags().BoolVar(&bootstrapCreateForgejoRegistryToken, "create-forgejo-registry-token", false, "Create a least-privilege Forgejo package registry token and seed it through OpenBao") + bootstrapCmd.Flags().BoolVar(&bootstrapRegisterWebhook, "register-webhook", false, "Seed OpenBao secrets and register the Forgejo webhook") + bootstrapCmd.Flags().BoolVar(&bootstrapRotateWebhookAuthorization, "rotate-webhook-authorization", false, "Replace the Forgejo webhook authorization and reconcile it through OpenBao") bootstrapCmd.Flags().StringVar(&bootstrapPublishAppFrom, "publish-app-from", "", "Push this app checkout's current branch and create a Forgejo delivery PR") bootstrapCmd.Flags().BoolVar(&bootstrapMergeBootstrapPR, "merge-bootstrap-pr", false, "Merge the generated Flux repository migration PR before bootstrapping") bootstrapCmd.Flags().BoolVar(&bootstrapManageNetworkBridges, "manage-network-bridges", false, "Persist Terraform management for existing Talos network bridges") @@ -57,8 +64,28 @@ func runBootstrap(cmd *cobra.Command, args []string) error { if err != nil { return err } + if err := bootstrap.EnsureTemplateRevisions(cfg); err != nil { + return err + } return createForgejoRegistryToken(cfg) } + if bootstrapRotateWebhookAuthorization { + if bootstrapConfigPath == "" { + return fmt.Errorf("--rotate-webhook-authorization requires --config") + } + cfg, err = config.Load(bootstrapConfigPath) + if err != nil { + return err + } + authorization, err := bootstrap.NewWebhookAuthorization() + if err != nil { + return fmt.Errorf("generate Forgejo webhook authorization: %w", err) + } + if err := bootstrap.UpsertOperationalSecret(cfg.SOPS.OperationalSecretsPath, cfg.SOPS.AgeKeyPath, "cicd/forgejo-webhook", "authorization", authorization); err != nil { + return fmt.Errorf("save Forgejo webhook authorization: %w", err) + } + return bootstrap.Runner{Config: cfg, RegisterWebhook: true}.Run() + } if bootstrapInitializeOpenBao { if bootstrapConfigPath == "" { return fmt.Errorf("--initialize-openbao requires --config") @@ -67,6 +94,9 @@ func runBootstrap(cmd *cobra.Command, args []string) error { if err != nil { return err } + if err := bootstrap.EnsureTemplateRevisions(cfg); err != nil { + return err + } return bootstrap.InitializeOpenBao(cfg) } if bootstrapMergeBootstrapPR { @@ -77,6 +107,9 @@ func runBootstrap(cmd *cobra.Command, args []string) error { if err != nil { return err } + if err := bootstrap.EnsureTemplateRevisions(cfg); err != nil { + return err + } manager := forgejo.NewRepoManager(cfg.Git.BaseURL, cfg.Git.Token, cfg.Git.Owner, cfg.Git.Username, "", "", cfg.Flux.Branch, "") if err := manager.MergePullRequest(cfg.Flux.RepoName, "maidn/bootstrap-"+cfg.ClusterID); err != nil { return err @@ -90,6 +123,9 @@ func runBootstrap(cmd *cobra.Command, args []string) error { if err != nil { return err } + if err := bootstrap.EnsureTemplateRevisions(cfg); err != nil { + return err + } branch, err := forgejo.CurrentBranch(bootstrapPublishAppFrom) if err != nil { return err @@ -117,6 +153,18 @@ func runBootstrap(cmd *cobra.Command, args []string) error { } return manager.CreatePullRequest(repo, "feat: migrate delivery to Tekton", branch, cfg.Delivery.AppRepoRef) } + if bootstrapInitializeOpenBao { + if bootstrapConfigPath == "" { + return fmt.Errorf("--initialize-openbao requires --config") + } + cfg, err = config.Load(bootstrapConfigPath) + if err != nil { + return err + } + generatedDir := filepath.Join(cfg.Git.CloneParent, cfg.Talos.RepoDirName, cfg.Talos.GeneratedDir) + _, err = openbao.Initialize(filepath.Join(generatedDir, "kubeconfig"), cfg.SOPS.RecoveryRecipient, cfg.SOPS.RecoveryIdentityPath, cfg.SOPS.RecoveryBundlePath, cfg.SOPS.AgeKeyPath, cfg.SOPS.OperationalSecretsPath) + return err + } if bootstrapConfigPath != "" { if bootstrapPromptDemocraticCSI || bootstrapPromptOperationalSecrets || bootstrapInitializeOpenBaoRecovery || bootstrapManageNetworkBridges { @@ -170,11 +218,26 @@ func runBootstrap(cmd *cobra.Command, args []string) error { if err != nil { return err } + if bootstrapRegisterWebhook { + if err := seedForgejoOperationalCredentials(cfg); err != nil { + return err + } + } - runner := bootstrap.Runner{Config: cfg, Mode: bootstrap.Mode(bootstrapMode), ConfirmRebuild: bootstrapYes} + runner := bootstrap.Runner{Config: cfg, Mode: bootstrap.Mode(bootstrapMode), ConfirmRebuild: bootstrapYes, RegisterWebhook: bootstrapRegisterWebhook} return runner.Run() } +func seedForgejoOperationalCredentials(cfg config.Config) error { + if err := upsertOperationalSecret(cfg.SOPS.OperationalSecretsPath, cfg.SOPS.AgeKeyPath, "cicd/forgejo", "username", cfg.Git.Username); err != nil { + return fmt.Errorf("save Forgejo username for webhook registration: %w", err) + } + if err := upsertOperationalSecret(cfg.SOPS.OperationalSecretsPath, cfg.SOPS.AgeKeyPath, "cicd/forgejo", "token", cfg.Git.Token); err != nil { + return fmt.Errorf("save Forgejo token for webhook registration: %w", err) + } + return nil +} + func createForgejoRegistryToken(cfg config.Config) error { if _, err := bootstrap.ReadOperationalSecrets(cfg.SOPS.OperationalSecretsPath, cfg.SOPS.AgeKeyPath); err != nil { return err diff --git a/cmd/bootstrap_test.go b/cmd/bootstrap_test.go index ef50bf3..3342ea5 100644 --- a/cmd/bootstrap_test.go +++ b/cmd/bootstrap_test.go @@ -1,6 +1,12 @@ package cmd -import "testing" +import ( + "errors" + "strings" + "testing" + + "github.com/Pingu-Studio/MaidnCLI/internal/config" +) func TestCreateForgejoRegistryTokenRequiresConfig(t *testing.T) { originalConfigPath, originalCreate := bootstrapConfigPath, bootstrapCreateForgejoRegistryToken @@ -14,3 +20,45 @@ func TestCreateForgejoRegistryTokenRequiresConfig(t *testing.T) { t.Fatal("--create-forgejo-registry-token accepted a missing --config") } } + +func TestSeedForgejoOperationalCredentialsUsesEncryptedUpsertBoundary(t *testing.T) { + original := upsertOperationalSecret + t.Cleanup(func() { upsertOperationalSecret = original }) + cfg := config.Config{ + Git: config.GitConfig{Username: "webhook-bot", Token: "test-pat"}, + SOPS: config.SOPSConfig{OperationalSecretsPath: "secrets.sops.yaml", AgeKeyPath: "age-key.txt"}, + } + type call struct{ path, key, value string } + var calls []call + upsertOperationalSecret = func(secretsPath, ageKeyPath, path, key, value string) error { + if secretsPath != cfg.SOPS.OperationalSecretsPath || ageKeyPath != cfg.SOPS.AgeKeyPath { + t.Fatal("credential upsert used unexpected secret paths") + } + calls = append(calls, call{path, key, value}) + return nil + } + + if err := seedForgejoOperationalCredentials(cfg); err != nil { + t.Fatal(err) + } + if len(calls) != 2 || calls[0] != (call{"cicd/forgejo", "username", cfg.Git.Username}) || calls[1] != (call{"cicd/forgejo", "token", cfg.Git.Token}) { + t.Fatal("configured Forgejo credentials were not passed only to the encrypted upsert boundary") + } +} + +func TestSeedForgejoOperationalCredentialsHidesTokenOnUpsertFailure(t *testing.T) { + original := upsertOperationalSecret + t.Cleanup(func() { upsertOperationalSecret = original }) + cfg := config.Config{Git: config.GitConfig{Username: "webhook-bot", Token: "test-pat"}} + upsertOperationalSecret = func(_, _, _, key, _ string) error { + if key == "token" { + return errors.New("unavailable") + } + return nil + } + + err := seedForgejoOperationalCredentials(cfg) + if err == nil || !strings.Contains(err.Error(), "save Forgejo token for webhook registration") || strings.Contains(err.Error(), cfg.Git.Token) { + t.Fatal("credential upsert failure was not clear and token-safe") + } +} diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..d177efd --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,150 @@ +# Operations Runbook + +Use this runbook from the MaidnCLI checkout. Bootstrap YAML, age identities, +OpenBao recovery material, Terraform state, and generated workspaces are +secret-bearing local inputs. Do not commit or print them. + +## Normal Reconciliation + +Run after a merged GitOps migration or to recover ordinary drift: + +```powershell +go run . bootstrap --config --mode=reconcile +``` + +This is the only regular lifecycle command. Do not use direct `kubectl apply`, +`flux reconcile`, Helm upgrades, or mutating `talosctl` commands. + +Bootstrap records the CICD, manifests, and Talos template commits in the +workspace's `maidn-template-revisions.yaml` before it creates migration PRs or +touches infrastructure. Later runs require the same template sources and refs +and check out those commits, even when a configured branch advances. To accept +new template revisions intentionally, use a new empty `workspaceDir` (and a +fresh `cloneParent` when it is configured separately) and keep the prior +secret-bearing workspace intact for recovery. + +## Rebuild + +Use only when an authorized recovery requires recreating the Talos VM: + +```powershell +go run . bootstrap --config --mode=rebuild --yes +``` + +The rebuild replaces Terraform-managed Talos VMs only. It does not manage or +delete TrueNAS datasets or unrelated infrastructure. + +## OpenBao And Webhooks + +After a rebuild or an OpenBao restart, refresh Kubernetes auth and reseed the +encrypted operational values: + +```powershell +go run . bootstrap --config --initialize-openbao +``` + +When the Pipeline and EventListener are already Ready, register or update the +Forgejo webhook without re-running the full lifecycle: + +```powershell +go run . bootstrap --config --register-webhook +``` + +The webhook authorization value stays in encrypted operational secrets and is +never supplied on the command line. + +If that authorization value is exposed, replace it and reconcile both OpenBao +and the Forgejo hook in one command: + +```powershell +go run . bootstrap --config --rotate-webhook-authorization +``` + +## Read-Only Verification + +Set `KUBECONFIG` to the generated kubeconfig for the configured cluster, then +check the control plane and delivery chain: + +```powershell +kubectl -n flux-system get kustomizations +kubectl get clustersecretstores +kubectl -n flux-system get externalsecrets +kubectl -n tekton-pipelines get pipelines,eventlisteners,externalsecrets +``` + +Expected state: + +- Flux Kustomizations are `READY=True`. +- `ClusterSecretStore/openbao` is `READY=True`. +- Forgejo credential ExternalSecrets are `SecretSynced`. +- The application Pipeline exists and the EventListener is available. + +If OpenBao authentication is invalid after a rebuild, run +`--initialize-openbao`, then allow the controllers to retry. Do not recreate +the ClusterSecretStore or Secrets manually. + +## External DNS + +Webhook delivery requires the configured `tekton.` hostname to +resolve through Pi-hole to the Cilium Gateway address. ExternalDNS uses the +Pi-hole provider with Gateway API routes, an `upsert-only` policy, and no +ownership registry. + +```powershell +kubectl -n external-dns get pods,externalsecrets +kubectl -n external-dns logs deployment/external-dns --tail=100 +``` + +If `ExternalSecret/pihole-credentials` is not `SecretSynced`, rerun +`--prompt-operational-secrets` to enter the Pi-hole server and password, then +run `--initialize-openbao` and `--register-webhook`. Do not create or edit the +provider Secret directly. + +## Webhook TLS + +The public Gateway terminates HTTPS with a cert-manager certificate. Its +Cloudflare DNS-01 token is used only to issue the `nid3.com` certificate; +Pi-hole remains the ExternalDNS provider. Check certificate readiness with: + +```powershell +kubectl -n cert-manager get externalsecret cloudflare-api-token +kubectl -n gateway-system get certificate webhook-tls +``` + +Enter the Pi-hole values, Cloudflare DNS-01 token, and Tunnel token through +`--prompt-operational-secrets`, then run `--initialize-openbao`. Do not put the +Cloudflare token in the cluster repository. + +## Internal Platform UIs + +Pi-hole resolves these HTTPS names to the Cilium Gateway only on the LAN: + +- `https://grafana./` for Grafana. Authenticate with Grafana. +- `https://openbao./` for OpenBao. Authenticate with an OpenBao token. + +Hubble UI is enabled for in-cluster troubleshooting but has no LAN route because +it does not provide authentication. Add an authenticated proxy before exposing +it outside the cluster. + +## Webhook Smoke Test + +Use Forgejo's hook test endpoint against an existing non-`main` ref. It emits +a real push delivery, runs the Node build, and pushes a SHA-tagged registry +image, but skips the main-only staging manifest update: + +```powershell +# Discover the hook ID and choose an existing non-main branch or tag. +Invoke-RestMethod -Headers @{ Authorization = "token $env:FORGEJO_TOKEN" } ` + -Uri "https:///api/v1/repos///hooks" + +Invoke-WebRequest -Method Post -Headers @{ Authorization = "token $env:FORGEJO_TOKEN" } ` + -Uri "https:///api/v1/repos///hooks//tests?ref=" +``` + +Forgejo returns `204` after accepting the delivery. Confirm the resulting +PipelineRun instead of treating `204` as a successful build: + +```powershell +kubectl -n tekton-pipelines get pipelineruns +kubectl -n tekton-pipelines describe pipelinerun +``` diff --git a/docs/secrets.md b/docs/secrets.md index 25959af..df40045 100644 --- a/docs/secrets.md +++ b/docs/secrets.md @@ -17,6 +17,9 @@ secrets: dockerconfigjson: encrypted-value cicd/forgejo-webhook: authorization: encrypted-value + platform/pihole: + server: encrypted-value + password: encrypted-value platform/cloudflare: api-token: encrypted-value platform/cloudflare-tunnel: @@ -34,8 +37,10 @@ For an existing configuration, run `bootstrap --config For a new operational-secret input, run `bootstrap --config --prompt-operational-secrets`. It derives Forgejo Git and registry credentials -from the configured Forgejo account, prompts for Cloudflare credentials with -input masked, and generates the webhook authorization value. +from the configured Forgejo account, prompts for the Pi-hole server and masked +password, masked Cloudflare DNS-01 and Tunnel tokens, and generates the webhook +authorization value. The DNS-01 token issues the Gateway certificate; it is not +used by ExternalDNS. `cicd/forgejo-webhook.authorization` is required for delivery bootstrap. The CLI supplies it as the Forgejo webhook Authorization header and Tekton compares diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index dde298d..910dd44 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -2,6 +2,7 @@ package bootstrap import ( "bytes" + "context" "crypto/rand" "encoding/base64" "encoding/json" @@ -32,9 +33,10 @@ const ( ) type Runner struct { - Config config.Config - Mode Mode - ConfirmRebuild bool + Config config.Config + Mode Mode + ConfirmRebuild bool + RegisterWebhook bool } type operationalSecrets struct { @@ -43,13 +45,36 @@ type operationalSecrets struct { var initializeOpenBao = openbao.Initialize +var ensureForgejoWebhook = func(cfg config.Config, repo, webhookURL, authorization string) error { + manager := forgejo.NewRepoManager(cfg.Git.BaseURL, cfg.Git.Token, cfg.Git.Owner, cfg.Git.Username, cfg.Flux.ManifestsRepo, cfg.Flux.RepoName, cfg.Flux.Branch, "maidn/bootstrap-"+cfg.ClusterID) + return manager.EnsureWebhook(repo, webhookURL, authorization) +} + +var runWebhookCommand = utils.RunCommandQuietOutputInDir + +var preflight = config.Preflight + +var runGit = func(dir string, args ...string) ([]byte, error) { + command := exec.Command("git", args...) + command.Dir = dir + return command.Output() +} + +var webhookTargetTimeout = 70 * time.Minute + +var webhookTargetPollInterval = 2 * time.Second + +var templateBaseComponents = []string{"snapshot-crds", "democratic-csi", "cert-manager", "cluster-issuers", "gateway-api", "gateway", "monitoring", "openbao", "external-secrets", "external-dns", "tekton", "tekton-triggers"} + +var requiredClusterKustomizations = []string{"snapshot-crds-kustomization.yaml", "democratic-csi-kustomization.yaml", "cert-manager-kustomization.yaml", "cluster-issuers-kustomization.yaml", "gateway-api-kustomization.yaml", "cilium-kustomization.yaml", "cilium-config-kustomization.yaml", "openbao-kustomization.yaml", "external-secrets-kustomization.yaml", "cnpg-kustomization.yaml", "gateway-kustomization.yaml", "external-dns-kustomization.yaml", "cloudflare-tunnel-kustomization.yaml", "monitoring-kustomization.yaml", "tekton-kustomization.yaml", "tekton-triggers-kustomization.yaml", "cicd-manifests-repo.yaml"} + func (r Runner) Run() error { resolved, err := config.Resolve(r.Config) if err != nil { return err } r.Config = resolved - if err := config.Preflight(r.Config); err != nil { + if err := preflight(r.Config); err != nil { return fmt.Errorf("preflight: %w", err) } if _, err := os.Stat(r.Config.SOPS.AgeKeyPath); err != nil { @@ -72,28 +97,20 @@ func (r Runner) Run() error { if r.Mode == Rebuild && !r.ConfirmRebuild { return fmt.Errorf("rebuild is destructive; rerun with --mode=rebuild --yes") } - + if r.RegisterWebhook { + return r.reconcileWebhook(filepath.Join(r.Config.Git.CloneParent, r.Config.Talos.RepoDirName, r.Config.Talos.GeneratedDir)) + } workspace := r.Config.WorkspaceDir - if err := os.MkdirAll(workspace, 0755); err != nil { + if err := EnsureTemplateRevisions(r.Config); err != nil { return err } if err := config.WriteRedacted(filepath.Join(workspace, "maidn-bootstrap.resolved.yaml"), r.Config); err != nil { return err } - if err := os.MkdirAll(r.Config.Git.CloneParent, 0755); err != nil { - return err - } - manifestsURL := forgejo.CloneURL(r.Config.Git.BaseURL, r.Config.Git.Owner, r.Config.Flux.ManifestsRepo) fluxConfig := ghrepo.BuildFluxConfig(manifestsURL, r.Config.Flux.ManifestsRepo, r.Config.Flux.Branch) cicdTemplateDir := filepath.Join(workspace, "maidn-cicd-cluster-template") - if err := ensureRepo(cicdTemplateDir, r.Config.Templates.CICDRepoURL, r.Config.Templates.CICDRepoRef); err != nil { - return err - } manifestsTemplateDir := filepath.Join(workspace, "cicd-deployment-manifests-template") - if err := ensureRepo(manifestsTemplateDir, r.Config.Templates.ManifestsRepoURL, r.Config.Templates.ManifestsRepoRef); err != nil { - return err - } manager := forgejo.NewRepoManager(r.Config.Git.BaseURL, r.Config.Git.Token, r.Config.Git.Owner, r.Config.Git.Username, r.Config.Flux.ManifestsRepo, r.Config.Flux.RepoName, r.Config.Flux.Branch, "maidn/bootstrap-"+r.Config.ClusterID) if err := manager.InitializeAll( func(dir string) error { @@ -107,9 +124,28 @@ func (r Runner) Run() error { if err := copyDir(filepath.Join(cicdTemplateDir, "base"), filepath.Join(dir, "base"), false); err != nil { return err } + if err := copyTemplateBaseComponents(cicdTemplateDir, dir); err != nil { + return err + } if err := copyClusterTemplate(filepath.Join(cicdTemplateDir, "clusters", "template"), clusterDir); err != nil { return err } + for _, name := range []string{"external-secrets", "cnpg", "cloudflare-tunnel", "external-dns", "monitoring", "tekton"} { + content, err := os.ReadFile(filepath.Join(cicdTemplateDir, "clusters", "template", name+"-kustomization.yaml")) + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(clusterDir, name+"-kustomization.yaml"), content, 0644); err != nil { + return err + } + } + csiKustomization, err := os.ReadFile(filepath.Join(cicdTemplateDir, "clusters", "template", "democratic-csi-kustomization.yaml")) + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(clusterDir, "democratic-csi-kustomization.yaml"), csiKustomization, 0644); err != nil { + return err + } if err := copyDir(filepath.Join(cicdTemplateDir, "base", "cilium"), filepath.Join(dir, "base", "cilium"), true); err != nil { return err } @@ -144,9 +180,6 @@ func (r Runner) Run() error { } repoDir := filepath.Join(r.Config.Git.CloneParent, r.Config.Talos.RepoDirName) - if err := ensureRepo(repoDir, r.Config.Templates.TalosRepoURL, r.Config.Templates.TalosRepoRef); err != nil { - return err - } terraformDir := filepath.Join(repoDir, r.Config.Talos.TerraformDir) generatedDir := filepath.Join(repoDir, r.Config.Talos.GeneratedDir) @@ -178,6 +211,9 @@ func (r Runner) Run() error { if err := utils.RunCommandInDir(generatedDir, "talosctl", "kubeconfig", "--talosconfig=./clusterconfig/talosconfig", "--nodes="+r.Config.Talos.KubeconfigNode, "."); err != nil { return err } + if err := waitForKubernetesAPI(generatedDir); err != nil { + return err + } } if r.Config.Talos.AutoBootstrapFlux { if err := installCilium(generatedDir, r.Config); err != nil { @@ -192,20 +228,25 @@ func (r Runner) Run() error { if err := configureFluxSOPS(generatedDir); err != nil { return err } - operationalSecrets, err := openbao.Initialize(filepath.Join(generatedDir, "kubeconfig"), r.Config.SOPS.RecoveryRecipient, r.Config.SOPS.RecoveryIdentityPath, r.Config.SOPS.RecoveryBundlePath, r.Config.SOPS.AgeKeyPath, r.Config.SOPS.OperationalSecretsPath) - if err != nil { - return fmt.Errorf("initialize OpenBao: %w", err) - } - authorization := operationalSecrets["cicd/forgejo-webhook"]["authorization"] - if authorization == "" { - return errors.New("operational SOPS secrets requires cicd/forgejo-webhook.authorization") - } - if err := waitForWebhookTargets(generatedDir, r.Config); err != nil { - return err - } - if err := manager.EnsureWebhook(r.Config.Delivery.AppName, r.Config.Delivery.WebhookURL(), authorization); err != nil { - return fmt.Errorf("reconcile Forgejo webhook: %w", err) - } + return r.reconcileWebhook(generatedDir) + } + return nil +} + +func (r Runner) reconcileWebhook(generatedDir string) error { + operationalSecrets, err := initializeOpenBao(filepath.Join(generatedDir, "kubeconfig"), r.Config.SOPS.RecoveryRecipient, r.Config.SOPS.RecoveryIdentityPath, r.Config.SOPS.RecoveryBundlePath, r.Config.SOPS.AgeKeyPath, r.Config.SOPS.OperationalSecretsPath) + if err != nil { + return fmt.Errorf("initialize OpenBao: %w", err) + } + authorization := operationalSecrets["cicd/forgejo-webhook"]["authorization"] + if authorization == "" { + return errors.New("operational SOPS secrets requires cicd/forgejo-webhook.authorization") + } + if err := waitForWebhookTargets(generatedDir, r.Config, authorization); err != nil { + return err + } + if err := ensureForgejoWebhook(r.Config, r.Config.Delivery.AppName, r.Config.Delivery.WebhookURL(), authorization); err != nil { + return fmt.Errorf("reconcile Forgejo webhook: %w", err) } return nil } @@ -236,6 +277,9 @@ func renderDeliveryConfig(dir string, cfg config.Config) error { "${APP_REPO_REF}", cfg.Delivery.AppRepoRef, "${IMAGE_REPOSITORY}", cfg.Delivery.ImageRepository, "${FORGEJO_BASE_URL}", cfg.Git.BaseURL, + "${CLUSTER_DOMAIN}", cfg.Flux.ClusterDomain, + "${TEKTON_CATALOG_REPO_URL}", cfg.Templates.TektonCatalogRepoURL, + "${TEKTON_CATALOG_REPO_REF}", cfg.Templates.TektonCatalogRepoRef, "${WEBHOOK_HOSTNAME}", cfg.Delivery.WebhookHostname, "${WEBHOOK_PATH}", cfg.Delivery.WebhookPath, "${TEKTON_CATALOG_REPO_URL}", cfg.Templates.TektonCatalogRepoURL, @@ -397,14 +441,20 @@ func renderDemocraticCSISecret(csi config.DemocraticCSIConfig) ([]byte, error) { return yaml.Marshal(struct { APIVersion string `yaml:"apiVersion"` Kind string `yaml:"kind"` - Metadata map[string]string `yaml:"metadata"` + Metadata map[string]any `yaml:"metadata"` Type string `yaml:"type"` StringData map[string]string `yaml:"stringData"` }{ APIVersion: "v1", Kind: "Secret", - Metadata: map[string]string{"name": "democratic-csi-secrets", "namespace": "democratic-storage"}, - Type: "Opaque", + Metadata: map[string]any{ + "name": "democratic-csi-secrets", + "namespace": "democratic-storage", + "labels": map[string]string{ + "reconcile.fluxcd.io/watch": "Enabled", + }, + }, + Type: "Opaque", StringData: map[string]string{ "truenas-api-key": csi.TrueNASAPIKey, "truenas-host": csi.TrueNASHost, @@ -420,27 +470,43 @@ func renderDemocraticCSISecret(csi config.DemocraticCSIConfig) ([]byte, error) { }) } -func waitForWebhookTargets(dir string, cfg config.Config) error { - resources := []string{ - "secret/forgejo-webhook", - "deployment/el-" + cfg.Delivery.AppName, - "pipeline/" + cfg.Delivery.AppName, +func waitForWebhookTargets(dir string, cfg config.Config, authorization string) error { + if err := waitForWebhookAuthorization(dir, authorization); err != nil { + return err } + resources := []string{"deployment/el-" + cfg.Delivery.AppName, "pipeline/" + cfg.Delivery.AppName} for _, resource := range resources { - deadline := time.Now().Add(10 * time.Minute) + deadline := time.Now().Add(webhookTargetTimeout) for time.Now().Before(deadline) { - if _, err := utils.RunCommandQuietOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err == nil { + if _, err := runWebhookCommand(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err == nil { break } - time.Sleep(2 * time.Second) + time.Sleep(webhookTargetPollInterval) } - if _, err := utils.RunCommandQuietOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err != nil { + if _, err := runWebhookCommand(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err != nil { return fmt.Errorf("wait for %s before registering Forgejo webhook", resource) } } return nil } +func waitForWebhookAuthorization(dir, authorization string) error { + deadline := time.Now().Add(webhookTargetTimeout) + for { + output, err := runWebhookCommand(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", "secret/forgejo-webhook", "-o=jsonpath={.data.authorization}") + if err == nil { + observed, decodeErr := base64.StdEncoding.DecodeString(strings.TrimSpace(string(output))) + if decodeErr == nil && string(observed) == authorization { + return nil + } + } + if !time.Now().Before(deadline) { + return errors.New("ExternalSecret target Secret forgejo-webhook did not refresh within the timeout; Forgejo webhook was not updated. Wait for External Secrets to recover, then safely rerun cicd-tool bootstrap --config --register-webhook") + } + time.Sleep(webhookTargetPollInterval) + } +} + func ensureClusterKustomizations(clusterDir string) error { path := filepath.Join(clusterDir, "kustomization.yaml") content, err := os.ReadFile(path) @@ -454,7 +520,7 @@ func ensureClusterKustomizations(clusterDir string) error { } updated := string(content) updated = strings.ReplaceAll(updated, " - bootstrap-secrets.sops.yaml\n", "") - for _, resource := range []string{"cilium-kustomization.yaml", "cilium-config-kustomization.yaml", "cicd-manifests-repo.yaml", "tekton-triggers-kustomization.yaml"} { + for _, resource := range requiredClusterKustomizations { if !strings.Contains(updated, resource) { updated += " - " + resource + "\n" } @@ -465,6 +531,15 @@ func ensureClusterKustomizations(clusterDir string) error { return os.WriteFile(path, []byte(updated), 0644) } +func copyTemplateBaseComponents(templateDir, repoDir string) error { + for _, component := range templateBaseComponents { + if err := copyDir(filepath.Join(templateDir, "base", component), filepath.Join(repoDir, "base", component), true); err != nil { + return err + } + } + return nil +} + func copyClusterTemplate(source, destination string) error { entries, err := os.ReadDir(destination) if err != nil && !os.IsNotExist(err) { @@ -484,7 +559,14 @@ func copyClusterTemplate(source, destination string) error { func ensureManifestsKustomizations(dir string) error { for _, environment := range []string{"previews", "staging", "production"} { path := filepath.Join(dir, "apps", environment, "kustomization.yaml") - if _, err := os.Stat(path); err == nil { + content, err := os.ReadFile(path) + if err == nil { + if strings.Contains(string(content), "resources:") { + continue + } + if err := os.WriteFile(path, append(content, []byte("resources:\n")...), 0644); err != nil { + return err + } continue } else if !os.IsNotExist(err) { return err @@ -501,7 +583,7 @@ func installCilium(dir string, cfg config.Config) error { if err := os.MkdirAll(helmDir, 0755); err != nil { return err } - return utils.RunCommandInDir(dir, "helm", "upgrade", "--install", "cilium", "cilium", "--repo=https://helm.cilium.io", "--version=1.19.6", "--repository-config="+filepath.Join(helmDir, "repositories.yaml"), "--repository-cache="+helmDir, "--namespace=kube-system", "--create-namespace", "--kubeconfig=kubeconfig", "--wait", "--timeout=5m", "--set=kubeProxyReplacement=true", "--set=ipam.mode=kubernetes", "--set=k8sServiceHost=localhost", "--set=k8sServicePort=7445", "--set=cgroup.autoMount.enabled=false", "--set=cgroup.hostRoot=/sys/fs/cgroup", "--set=bpf.hostLegacyRouting=true", "--set=securityContext.capabilities.ciliumAgent={CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}", "--set=securityContext.capabilities.cleanCiliumState={NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}", "--set=gatewayAPI.enabled=false", "--set=l2announcements.enabled=true", "--set=operator.replicas=1") + return utils.RunCommandInDir(dir, "helm", "upgrade", "--install", "cilium", "cilium", "--repo=https://helm.cilium.io", "--version=1.19.6", "--repository-config="+filepath.Join(helmDir, "repositories.yaml"), "--repository-cache="+helmDir, "--namespace=kube-system", "--create-namespace", "--kubeconfig=kubeconfig", "--wait", "--timeout=5m", "--set=kubeProxyReplacement=true", "--set=ipam.mode=kubernetes", "--set=k8sServiceHost=localhost", "--set=k8sServicePort=7445", "--set=cgroup.autoMount.enabled=false", "--set=cgroup.hostRoot=/sys/fs/cgroup", "--set=bpf.hostLegacyRouting=true", "--set=securityContext.capabilities.ciliumAgent={CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}", "--set=securityContext.capabilities.cleanCiliumState={NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}", "--set=envoy.enabled=true", "--set=gatewayAPI.enabled=true", "--set=l2announcements.enabled=true", "--set=rollOutCiliumPods=true", "--set=operator.replicas=1", "--set=operator.rollOutPods=true") } func copyDir(source, destination string, overwrite bool) error { @@ -564,12 +646,12 @@ func applyTalosConfigs(dir string, cfg config.Config) error { configFile := filepath.Join("clusterconfig", fmt.Sprintf("%s-%s.yaml", cfg.Talos.Cluster.Name, node.Name)) nodeAddress := node.Networks[0].IP secureArgs := []string{"apply-config", "--talosconfig=./clusterconfig/talosconfig", "--nodes=" + nodeAddress, "--endpoints=" + cfg.Talos.BootstrapEndpoint, "--file=" + configFile} - if _, err := utils.RunCommandQuietOutputInDir(dir, "talosctl", "get", "machinestatus", "--talosconfig=./clusterconfig/talosconfig", "--nodes="+nodeAddress, "--endpoints="+cfg.Talos.BootstrapEndpoint, "--output=json"); err == nil { + if _, err := runTalosctlOutput(dir, "get", "machinestatus", "--talosconfig=./clusterconfig/talosconfig", "--nodes="+nodeAddress, "--endpoints="+cfg.Talos.BootstrapEndpoint, "--output=json"); err == nil { if err := utils.RunCommandInDir(dir, "talosctl", secureArgs...); err != nil { return fmt.Errorf("apply Talos config to %s: %w", node.Name, err) } } else { - if _, maintenanceErr := utils.RunCommandQuietOutputInDir(dir, "talosctl", "get", "machinestatus", "--insecure", "--nodes="+nodeAddress, "--endpoints="+nodeAddress, "--output=json"); maintenanceErr == nil { + if _, maintenanceErr := runTalosctlOutput(dir, "get", "machinestatus", "--insecure", "--nodes="+nodeAddress, "--endpoints="+nodeAddress, "--output=json"); maintenanceErr == nil { if err := utils.RunCommandInDir(dir, "talosctl", "apply-config", "--insecure", "--nodes="+nodeAddress, "--endpoints="+nodeAddress, "--file="+configFile); err != nil { return fmt.Errorf("apply initial Talos config to %s: %w", node.Name, err) } @@ -590,10 +672,32 @@ func applyTalosConfigs(dir string, cfg config.Config) error { } func bootstrapEtcdIfNeeded(dir string, cfg config.Config) error { - if _, err := utils.RunCommandQuietOutputInDir(dir, "talosctl", "etcd", "status", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode); err == nil { + if _, err := runTalosctlOutput(dir, "etcd", "status", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode); err == nil { return nil } - return utils.RunCommandInDir(dir, "talosctl", "bootstrap", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode) + if err := waitForTalosAPI(dir, cfg, cfg.Talos.BootstrapNode); err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + command := exec.CommandContext(ctx, "talosctl", "bootstrap", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode) + command.Dir = dir + command.Stdout = os.Stdout + command.Stderr = os.Stderr + if err := command.Run(); err != nil { + if ctx.Err() != nil { + return errors.New("etcd bootstrap request timed out") + } + return fmt.Errorf("bootstrap etcd: %w", err) + } + deadline := time.Now().Add(2 * time.Minute) + for time.Now().Before(deadline) { + if _, err := runTalosctlOutput(dir, "etcd", "status", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+cfg.Talos.BootstrapEndpoint, "--nodes="+cfg.Talos.BootstrapNode); err == nil { + return nil + } + time.Sleep(2 * time.Second) + } + return errors.New("etcd did not become healthy after bootstrap") } func waitForTalosReboot(dir string, cfg config.Config, node string) error { @@ -603,7 +707,7 @@ func waitForTalosReboot(dir string, cfg config.Config, node string) error { func waitForTalosAPI(dir string, cfg config.Config, node string) error { deadline := time.Now().Add(5 * time.Minute) for time.Now().Before(deadline) { - if _, err := utils.RunCommandQuietOutputInDir(dir, "talosctl", "get", "machinestatus", "--output=json", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+node, "--nodes="+node); err == nil { + if _, err := runTalosctlOutput(dir, "get", "machinestatus", "--output=json", "--talosconfig=./clusterconfig/talosconfig", "--endpoints="+node, "--nodes="+node); err == nil { return nil } time.Sleep(2 * time.Second) @@ -611,6 +715,25 @@ func waitForTalosAPI(dir string, cfg config.Config, node string) error { return fmt.Errorf("Talos API did not return after applying its machine configuration") } +func runTalosctlOutput(dir string, args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + command := exec.CommandContext(ctx, "talosctl", args...) + command.Dir = dir + return command.Output() +} + +func waitForKubernetesAPI(dir string) error { + deadline := time.Now().Add(5 * time.Minute) + for time.Now().Before(deadline) { + if _, err := utils.RunCommandQuietOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "get", "--raw=/readyz"); err == nil { + return nil + } + time.Sleep(2 * time.Second) + } + return errors.New("Kubernetes API did not become ready after Talos bootstrap") +} + type lifecycle struct { ClusterID string `yaml:"clusterId"` ClusterName string `yaml:"clusterName"` @@ -674,10 +797,14 @@ func terraformPlanPath(terraformDir, clusterID string) (string, error) { } func installSOPSKey(dir string, cfg config.Config) error { - if _, err := os.Stat(cfg.SOPS.AgeKeyPath); err != nil { + ageKeyPath, err := filepath.Abs(cfg.SOPS.AgeKeyPath) + if err != nil { + return fmt.Errorf("resolve SOPS age identity: %w", err) + } + if _, err := os.Stat(ageKeyPath); err != nil { return fmt.Errorf("read SOPS age identity: %w", err) } - manifest, err := utils.RunCommandOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "flux-system", "create", "secret", "generic", "sops-age", "--from-file=age.agekey="+cfg.SOPS.AgeKeyPath, "--dry-run=client", "-o", "yaml") + manifest, err := utils.RunCommandOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "flux-system", "create", "secret", "generic", "sops-age", "--from-file=age.agekey="+ageKeyPath, "--dry-run=client", "-o", "yaml") if err != nil { return err } @@ -689,17 +816,175 @@ func configureFluxSOPS(dir string) error { return utils.RunCommandInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "flux-system", "patch", "kustomization", "flux-system", "--type=merge", "-p", patch) } -func ensureRepo(dir, repoURL, ref string) error { - if _, err := os.Stat(dir); os.IsNotExist(err) { - return utils.RunCommand("git", "clone", "--branch", ref, repoURL, dir) +type templateRevision struct { + Repository string `yaml:"repository"` + Ref string `yaml:"ref"` + Commit string `yaml:"commit"` +} + +type templateRevisionLock struct { + Version int `yaml:"version"` + CICD templateRevision `yaml:"cicd"` + Manifests templateRevision `yaml:"manifests"` + Talos templateRevision `yaml:"talos"` +} + +type templateCheckout struct { + Dir string + Repository string + Ref string +} + +func ensureTemplateRevisions(workspace string, cfg config.Config) error { + lockPath := filepath.Join(workspace, "maidn-template-revisions.yaml") + checkouts := []templateCheckout{ + {filepath.Join(workspace, "maidn-cicd-cluster-template"), cfg.Templates.CICDRepoURL, cfg.Templates.CICDRepoRef}, + {filepath.Join(workspace, "cicd-deployment-manifests-template"), cfg.Templates.ManifestsRepoURL, cfg.Templates.ManifestsRepoRef}, + {filepath.Join(cfg.Git.CloneParent, cfg.Talos.RepoDirName), cfg.Templates.TalosRepoURL, cfg.Templates.TalosRepoRef}, } - if err := utils.RunCommandInDir(dir, "git", "fetch", "origin"); err != nil { + lock, err := readTemplateRevisionLock(lockPath) + if err == nil { + revisions := []templateRevision{lock.CICD, lock.Manifests, lock.Talos} + for index, checkout := range checkouts { + if !sameTemplateSource(revisions[index], checkout) { + return errors.New("configured template source or ref differs from its workspace revision lock; use a new empty workspaceDir to intentionally refresh templates") + } + if _, err := checkoutTemplateRevision(checkout, revisions[index].Commit); err != nil { + return errors.New("locked template revision cannot be resolved; restore the locked commit or use a new empty workspaceDir to intentionally refresh templates") + } + } + return nil + } + if !os.IsNotExist(err) { + return errors.New("template revision lock is invalid; use a new empty workspaceDir to intentionally refresh templates") + } + + revisions := make([]templateRevision, len(checkouts)) + for index, checkout := range checkouts { + if config.RedactURL(checkout.Repository) == "" { + return errors.New("configured template source cannot be safely recorded; use a standard repository URL without embedded query credentials") + } + commit, err := checkoutTemplateRevision(checkout, "") + if err != nil { + return errors.New("configured template revision cannot be resolved; correct the template source or ref, then rerun bootstrap") + } + revisions[index] = templateRevision{Repository: config.RedactURL(checkout.Repository), Ref: checkout.Ref, Commit: commit} + } + return writeTemplateRevisionLock(lockPath, templateRevisionLock{Version: 1, CICD: revisions[0], Manifests: revisions[1], Talos: revisions[2]}) +} + +// EnsureTemplateRevisions records or restores the workspace's template commits. +func EnsureTemplateRevisions(cfg config.Config) error { + if err := os.MkdirAll(cfg.WorkspaceDir, 0755); err != nil { return err } - if err := utils.RunCommandInDir(dir, "git", "checkout", ref); err != nil { + if err := os.MkdirAll(cfg.Git.CloneParent, 0755); err != nil { return err } - return utils.RunCommandInDir(dir, "git", "pull", "--ff-only", "origin", ref) + return ensureTemplateRevisions(cfg.WorkspaceDir, cfg) +} + +func readTemplateRevisionLock(path string) (templateRevisionLock, error) { + var lock templateRevisionLock + data, err := os.ReadFile(path) + if err != nil { + return lock, err + } + decoder := yaml.NewDecoder(bytes.NewReader(data)) + decoder.KnownFields(true) + if err := decoder.Decode(&lock); err != nil { + return lock, err + } + if err := decoder.Decode(&templateRevisionLock{}); !errors.Is(err, io.EOF) { + return lock, errors.New("multiple YAML documents") + } + if lock.Version != 1 || !validTemplateRevision(lock.CICD) || !validTemplateRevision(lock.Manifests) || !validTemplateRevision(lock.Talos) { + return lock, errors.New("invalid template revision lock") + } + return lock, nil +} + +func writeTemplateRevisionLock(path string, lock templateRevisionLock) error { + data, err := yaml.Marshal(lock) + if err != nil { + return err + } + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + if _, err := file.Write(data); err != nil { + _ = file.Close() + return err + } + return file.Close() +} + +func sameTemplateSource(revision templateRevision, checkout templateCheckout) bool { + return validTemplateRevision(revision) && revision.Repository == config.RedactURL(checkout.Repository) && revision.Ref == checkout.Ref +} + +func validTemplateRevision(revision templateRevision) bool { + if revision.Repository == "" || revision.Repository == "" || revision.Ref == "" || len(revision.Commit) < 40 || len(revision.Commit) > 64 { + return false + } + for _, character := range revision.Commit { + if !((character >= '0' && character <= '9') || (character >= 'a' && character <= 'f')) { + return false + } + } + return true +} + +func checkoutTemplateRevision(checkout templateCheckout, lockedCommit string) (string, error) { + repository := config.RedactURL(checkout.Repository) + if repository == "" { + return "", errors.New("template source cannot be safely used") + } + if info, err := os.Stat(checkout.Dir); os.IsNotExist(err) { + if _, err := runGit("", "clone", "--no-checkout", repository, checkout.Dir); err != nil { + return "", err + } + } else if err != nil || !info.IsDir() { + return "", errors.New("template checkout is not a directory") + } else { + inside, err := runGit(checkout.Dir, "rev-parse", "--is-inside-work-tree") + if err != nil || strings.TrimSpace(string(inside)) != "true" { + return "", errors.New("template checkout is not a Git work tree") + } + origin, err := runGit(checkout.Dir, "remote", "get-url", "origin") + if err != nil || config.RedactURL(strings.TrimSpace(string(origin))) != repository { + return "", errors.New("template checkout source does not match configuration") + } + status, err := runGit(checkout.Dir, "status", "--porcelain") + if err != nil || strings.TrimSpace(string(status)) != "" { + return "", errors.New("template checkout has uncommitted changes") + } + } + + target := checkout.Ref + if lockedCommit != "" { + target = lockedCommit + } + if _, err := runGit(checkout.Dir, "fetch", "origin", target); err != nil { + return "", err + } + commit, err := runGit(checkout.Dir, "rev-parse", "--verify", "FETCH_HEAD^{commit}") + if err != nil { + return "", err + } + commitID := strings.TrimSpace(string(commit)) + if !validTemplateRevision(templateRevision{Repository: "source", Ref: "ref", Commit: commitID}) || (lockedCommit != "" && commitID != lockedCommit) { + return "", errors.New("template ref did not resolve to the expected commit") + } + if _, err := runGit(checkout.Dir, "checkout", "--detach", commitID); err != nil { + return "", err + } + head, err := runGit(checkout.Dir, "rev-parse", "--verify", "HEAD^{commit}") + if err != nil || strings.TrimSpace(string(head)) != commitID { + return "", errors.New("template checkout did not reach the expected commit") + } + return commitID, nil } func ensureTalosConfig(generatedDir string, cfg config.Config) error { diff --git a/internal/bootstrap/bootstrap_test.go b/internal/bootstrap/bootstrap_test.go index cf9e9bc..d970873 100644 --- a/internal/bootstrap/bootstrap_test.go +++ b/internal/bootstrap/bootstrap_test.go @@ -4,13 +4,17 @@ import ( "encoding/base64" "encoding/json" "errors" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" "strings" "testing" + "time" "github.com/Pingu-Studio/MaidnCLI/internal/config" + "github.com/Pingu-Studio/MaidnCLI/internal/forgejo" ) func TestRenderCiliumConfig(t *testing.T) { @@ -101,12 +105,41 @@ func TestCopyAndRenderDeliveryBasesOverwritesExistingMigrationOutput(t *testing. } } +func TestCopyTemplateBaseComponentsCopiesExternalSecretsConfig(t *testing.T) { + templateDir := t.TempDir() + repoDir := t.TempDir() + for _, component := range templateBaseComponents { + if err := os.MkdirAll(filepath.Join(templateDir, "base", component), 0755); err != nil { + t.Fatal(err) + } + } + source := filepath.Join(templateDir, "base", "external-secrets", "config.yaml") + destination := filepath.Join(repoDir, "base", "external-secrets", "config.yaml") + if err := os.WriteFile(source, []byte("store: openbao\n"), 0644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(destination), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(destination, []byte("store: stale\n"), 0644); err != nil { + t.Fatal(err) + } + + if err := copyTemplateBaseComponents(templateDir, repoDir); err != nil { + t.Fatal(err) + } + content, err := os.ReadFile(destination) + if err != nil || string(content) != "store: openbao\n" { + t.Fatalf("external-secrets configuration was not copied: %q, %v", content, err) + } +} + func TestRenderDemocraticCSISecret(t *testing.T) { secret, err := renderDemocraticCSISecret(config.DemocraticCSIConfig{TrueNASAPIKey: "api-key", TrueNASHost: "truenas.example.test", TargetPortal: "truenas.example.test:3260", ShareHost: "truenas.example.test", DatasetParentNFS: "pool/kubernetes/nfs/v", DatasetSnapshotsNFS: "pool/kubernetes/nfs/s", AllowedNetworks: "192.168.45.0/24", NameSuffix: "-test", PortalGroup: "1", InitiatorGroup: "1"}) if err != nil { t.Fatal(err) } - if !strings.Contains(string(secret), "name: democratic-csi-secrets") || !strings.Contains(string(secret), "dataset-parent-nfs: pool/kubernetes/nfs/v") { + if !strings.Contains(string(secret), "name: democratic-csi-secrets") || !strings.Contains(string(secret), "labels:\n reconcile.fluxcd.io/watch: Enabled") || !strings.Contains(string(secret), "dataset-parent-nfs: pool/kubernetes/nfs/v") { t.Fatalf("Democratic CSI secret was not rendered: %s", secret) } } @@ -275,6 +308,9 @@ func TestEnsureManifestsKustomizations(t *testing.T) { t.Fatal(err) } } + if err := os.WriteFile(filepath.Join(dir, "apps", "staging", "kustomization.yaml"), []byte("apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\n"), 0644); err != nil { + t.Fatal(err) + } if err := ensureManifestsKustomizations(dir); err != nil { t.Fatal(err) } @@ -282,6 +318,216 @@ func TestEnsureManifestsKustomizations(t *testing.T) { if err != nil || !strings.Contains(string(content), "resources:") { t.Fatalf("preview Kustomization was not created: %q, %v", content, err) } + content, err = os.ReadFile(filepath.Join(dir, "apps", "staging", "kustomization.yaml")) + if err != nil || !strings.Contains(string(content), "resources:") { + t.Fatalf("staging Kustomization was not repaired: %q, %v", content, err) + } +} + +func TestEnsureClusterKustomizationsUsesTemplateExternalSecretsResource(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "kustomization.yaml") + if err := os.WriteFile(path, []byte("resources:\n"), 0644); err != nil { + t.Fatal(err) + } + if err := ensureClusterKustomizations(dir); err != nil { + t.Fatal(err) + } + content, err := os.ReadFile(path) + if err != nil || !strings.Contains(string(content), "snapshot-crds-kustomization.yaml") || !strings.Contains(string(content), "democratic-csi-kustomization.yaml") || !strings.Contains(string(content), "openbao-kustomization.yaml") || !strings.Contains(string(content), "external-secrets-kustomization.yaml") || !strings.Contains(string(content), "cnpg-kustomization.yaml") || !strings.Contains(string(content), "cloudflare-tunnel-kustomization.yaml") || !strings.Contains(string(content), "tekton-kustomization.yaml") || strings.Contains(string(content), "external-secrets-config-kustomization.yaml") { + t.Fatalf("cluster Kustomization was not updated: %q, %v", content, err) + } +} + +func TestWebhookTargetTimeoutExceedsExternalSecretRefreshInterval(t *testing.T) { + if webhookTargetTimeout <= time.Hour { + t.Fatal("webhook target timeout must exceed the one-hour ExternalSecret refresh interval") + } +} + +func TestRunnerRegisterWebhookSkipsTemplateRevisions(t *testing.T) { + originalPreflight := preflight + originalGit := runGit + originalInitialize := initializeOpenBao + originalCommand := runWebhookCommand + originalWebhook := ensureForgejoWebhook + t.Cleanup(func() { + preflight = originalPreflight + runGit = originalGit + initializeOpenBao = originalInitialize + runWebhookCommand = originalCommand + ensureForgejoWebhook = originalWebhook + }) + + workspace := t.TempDir() + ageKeyPath := filepath.Join(workspace, "age-key.txt") + if err := os.WriteFile(ageKeyPath, nil, 0600); err != nil { + t.Fatal(err) + } + preflight = func(config.Config) error { return nil } + runGit = func(string, ...string) ([]byte, error) { + t.Fatal("webhook-only reconciliation must not access template repositories") + return nil, nil + } + authorization := "Bearer test-webhook-authorization" + initializeOpenBao = func(string, string, string, string, string, string) (map[string]map[string]string, error) { + return map[string]map[string]string{"cicd/forgejo-webhook": {"authorization": authorization}}, nil + } + runWebhookCommand = func(_ string, _ string, args ...string) ([]byte, error) { + if strings.Contains(strings.Join(args, " "), "secret/forgejo-webhook") { + return []byte(base64.StdEncoding.EncodeToString([]byte(authorization))), nil + } + return nil, nil + } + ensureForgejoWebhook = func(_ config.Config, _, _, observedAuthorization string) error { + if observedAuthorization != authorization { + t.Fatal("webhook reconciliation used the wrong authorization") + } + return nil + } + + cfg := config.Config{ + ClusterID: "test-cluster", + WorkspaceDir: workspace, + Git: config.GitConfig{Provider: "forgejo", BaseURL: "https://git.example.test", Username: "bot", Token: "token", Owner: "test-org", CloneParent: filepath.Join(workspace, "checkouts")}, + Flux: config.FluxConfig{RepoName: "cluster", Branch: "main", ClusterPath: "./clusters/test", ClusterDomain: "example.test", ManifestsRepo: "manifests"}, + Templates: config.TemplateConfig{TalosRepoURL: "https://git.example.test/talos.git", TalosRepoRef: "main", CICDRepoURL: "https://git.example.test/template.git", CICDRepoRef: "main", ManifestsRepoURL: "https://git.example.test/manifests.git", ManifestsRepoRef: "main"}, + Cilium: config.CiliumConfig{TrafficInterface: "eth1", LoadBalancerStart: "192.168.45.19", LoadBalancerEnd: "192.168.45.30"}, + DemocraticCSI: config.DemocraticCSIConfig{TrueNASAPIKey: "key", TrueNASHost: "truenas.example.test", TargetPortal: "truenas.example.test:3260", ShareHost: "truenas.example.test", DatasetParentNFS: "pool/kubernetes/nfs/v", DatasetSnapshotsNFS: "pool/kubernetes/nfs/s", AllowedNetworks: "192.168.45.0/24", NameSuffix: "-test", PortalGroup: "1", InitiatorGroup: "1"}, + Delivery: config.DeliveryConfig{AppName: "app", AppRepoURL: "https://git.example.test/app.git", AppRepoRef: "main", ImageRepository: "registry.example.test/test/app", WebhookHostname: "tekton.example.test", WebhookPath: "/"}, + SOPS: config.SOPSConfig{AgeKeyPath: ageKeyPath}, + Talos: config.TalosConfig{ + RepoDirName: "talos", TerraformDir: "terraform", GeneratedDir: "generated", ConfigFileName: "terraform.tfvars", + Proxmox: config.TalosProxmoxConfig{APIURL: "https://proxmox.example.test:8006", APITokenID: "id", APITokenSecret: "secret"}, + Cluster: config.TalosClusterConfig{Name: "test-cluster", Domain: "example.test"}, + Image: config.TalosImageConfig{TalosVersion: "v1.13.6", KubernetesVersion: "v1.33.4", SchematicID: "abcdefghijkl"}, + Nodes: []config.TalosNode{{Name: "cp-01", VMID: 100, Role: "controlplane", Networks: []config.TalosNetwork{{IP: "192.168.45.3", CIDR: "192.168.45.0/28", Gateway: "192.168.45.1", VLANID: 45}, {IP: "192.168.45.18", CIDR: "192.168.45.16/28", VLANID: 451}}}}, + }, + } + if err := (Runner{Config: cfg, RegisterWebhook: true}).Run(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(workspace, "maidn-template-revisions.yaml")); !os.IsNotExist(err) { + t.Fatal("webhook-only reconciliation created a template revision lock") + } +} + +func TestReconcileWebhookWaitsForRefreshedTargetSecret(t *testing.T) { + originalInitialize := initializeOpenBao + originalCommand := runWebhookCommand + originalWebhook := ensureForgejoWebhook + originalTimeout := webhookTargetTimeout + originalInterval := webhookTargetPollInterval + t.Cleanup(func() { + initializeOpenBao = originalInitialize + runWebhookCommand = originalCommand + ensureForgejoWebhook = originalWebhook + webhookTargetTimeout = originalTimeout + webhookTargetPollInterval = originalInterval + }) + + authorization := "Bearer test-webhook-authorization" + staleTarget := base64.StdEncoding.EncodeToString([]byte("Bearer stale-webhook-authorization")) + refreshedTarget := base64.StdEncoding.EncodeToString([]byte(authorization)) + targetChecks := 0 + refreshedObserved := false + runWebhookCommand = func(_ string, name string, args ...string) ([]byte, error) { + if name != "kubectl" || strings.Contains(strings.Join(args, " "), authorization) { + t.Fatal("webhook target probe used an unexpected command") + } + if strings.Contains(strings.Join(args, " "), "secret/forgejo-webhook") { + targetChecks++ + if targetChecks == 1 { + return []byte(staleTarget), nil + } + refreshedObserved = true + return []byte(refreshedTarget), nil + } + return nil, nil + } + webhookTargetPollInterval = 0 + + patches := 0 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if !refreshedObserved { + t.Error("Forgejo was called before the refreshed webhook Secret was observed") + writer.WriteHeader(http.StatusInternalServerError) + return + } + switch request.Method { + case http.MethodGet: + _ = json.NewEncoder(writer).Encode([]map[string]any{{"id": 7, "url": "https://tekton.example.test/"}}) + case http.MethodPatch: + var body struct { + AuthorizationHeader string `json:"authorization_header"` + } + if err := json.NewDecoder(request.Body).Decode(&body); err != nil || body.AuthorizationHeader != authorization { + t.Error("Forgejo webhook update did not use the observed authorization") + writer.WriteHeader(http.StatusBadRequest) + return + } + patches++ + writer.WriteHeader(http.StatusOK) + default: + t.Error("unexpected Forgejo request") + writer.WriteHeader(http.StatusMethodNotAllowed) + } + })) + defer server.Close() + ensureForgejoWebhook = func(_ config.Config, repo, webhookURL, authorization string) error { + manager := forgejo.NewRepoManager(server.URL, "test-token", "owner", "user", "manifests", "flux", "main", "migration") + manager.HTTPClient = server.Client() + return manager.EnsureWebhook(repo, webhookURL, authorization) + } + initializeOpenBao = func(string, string, string, string, string, string) (map[string]map[string]string, error) { + return map[string]map[string]string{"cicd/forgejo-webhook": {"authorization": authorization}}, nil + } + + cfg := config.Config{Delivery: config.DeliveryConfig{AppName: "app", WebhookHostname: "tekton.example.test", WebhookPath: "/"}} + if err := (Runner{Config: cfg}).reconcileWebhook(t.TempDir()); err != nil { + t.Fatal(err) + } + if targetChecks != 2 || patches != 1 { + t.Fatal("Forgejo webhook was not updated after the target Secret refreshed") + } +} + +func TestReconcileWebhookTimeoutDoesNotUpdateForgejo(t *testing.T) { + originalInitialize := initializeOpenBao + originalCommand := runWebhookCommand + originalWebhook := ensureForgejoWebhook + originalTimeout := webhookTargetTimeout + originalInterval := webhookTargetPollInterval + t.Cleanup(func() { + initializeOpenBao = originalInitialize + runWebhookCommand = originalCommand + ensureForgejoWebhook = originalWebhook + webhookTargetTimeout = originalTimeout + webhookTargetPollInterval = originalInterval + }) + + authorization := "Bearer test-webhook-authorization" + initializeOpenBao = func(string, string, string, string, string, string) (map[string]map[string]string, error) { + return map[string]map[string]string{"cicd/forgejo-webhook": {"authorization": authorization}}, nil + } + runWebhookCommand = func(_ string, _ string, _ ...string) ([]byte, error) { + return []byte(base64.StdEncoding.EncodeToString([]byte("Bearer stale-webhook-authorization"))), nil + } + webhookTargetTimeout = -time.Nanosecond + webhookTargetPollInterval = 0 + webhookUpdated := false + ensureForgejoWebhook = func(config.Config, string, string, string) error { + webhookUpdated = true + return nil + } + + err := (Runner{Config: config.Config{Delivery: config.DeliveryConfig{AppName: "app"}}}).reconcileWebhook(t.TempDir()) + if err == nil || !strings.Contains(err.Error(), "Forgejo webhook was not updated") || !strings.Contains(err.Error(), "--register-webhook") { + t.Fatal("webhook refresh timeout did not return a safe rerun error") + } + if webhookUpdated { + t.Fatal("Forgejo webhook update was attempted before the target Secret refreshed") + } } func TestRenderTerraformTFVarsIsStableAndRedactsToken(t *testing.T) { diff --git a/internal/bootstrap/template_revisions_test.go b/internal/bootstrap/template_revisions_test.go new file mode 100644 index 0000000..531e5f3 --- /dev/null +++ b/internal/bootstrap/template_revisions_test.go @@ -0,0 +1,202 @@ +package bootstrap + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Pingu-Studio/MaidnCLI/internal/config" +) + +func TestEnsureTemplateRevisionsLocksFirstCheckout(t *testing.T) { + workspace, cfg, git := templateRevisionTestConfig(t) + original := runGit + runGit = git.run + t.Cleanup(func() { runGit = original }) + + if err := ensureTemplateRevisions(workspace, cfg); err != nil { + t.Fatal(err) + } + lock, err := readTemplateRevisionLock(filepath.Join(workspace, "maidn-template-revisions.yaml")) + if err != nil { + t.Fatal(err) + } + if lock.CICD.Commit != git.commits["cicd"] || lock.Manifests.Commit != git.commits["manifests"] || lock.Talos.Commit != git.commits["talos"] { + t.Fatalf("lock did not record checked-out commits: %#v", lock) + } + data, err := os.ReadFile(filepath.Join(workspace, "maidn-template-revisions.yaml")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(data), "template-password") || strings.Contains(git.commands(), "template-password") { + t.Fatal("template credentials reached the lock or Git command arguments") + } +} + +func TestEnsureTemplateRevisionsReusesLockedCommitAfterBranchDrift(t *testing.T) { + workspace, cfg, git := templateRevisionTestConfig(t) + original := runGit + runGit = git.run + t.Cleanup(func() { runGit = original }) + + if err := ensureTemplateRevisions(workspace, cfg); err != nil { + t.Fatal(err) + } + locked := git.commits["cicd"] + git.commits["cicd"] = strings.Repeat("d", 40) + git.resetCalls() + if err := ensureTemplateRevisions(workspace, cfg); err != nil { + t.Fatal(err) + } + if git.fetched["cicd"] != locked || git.checkedOut["cicd"] != locked { + t.Fatalf("branch drift changed locked CICD revision: fetched %q, checked out %q", git.fetched["cicd"], git.checkedOut["cicd"]) + } + if strings.Contains(git.commands(), "fetch origin main") { + t.Fatal("later run fetched a mutable branch instead of the lock commit") + } +} + +func TestEnsureTemplateRevisionsRejectsChangedRefWithoutGit(t *testing.T) { + workspace, cfg, git := templateRevisionTestConfig(t) + original := runGit + runGit = git.run + t.Cleanup(func() { runGit = original }) + + if err := ensureTemplateRevisions(workspace, cfg); err != nil { + t.Fatal(err) + } + git.resetCalls() + cfg.Templates.CICDRepoRef = "release" + err := ensureTemplateRevisions(workspace, cfg) + if err == nil || !strings.Contains(err.Error(), "workspace revision lock") || !strings.Contains(err.Error(), "new empty workspaceDir") { + t.Fatalf("changed ref error was not safe and actionable: %v", err) + } + if git.commands() != "" { + t.Fatal("changed ref touched Git before rejecting the lock mismatch") + } +} + +func TestEnsureTemplateRevisionsHidesSourceWhenLockedCommitIsUnavailable(t *testing.T) { + workspace, cfg, git := templateRevisionTestConfig(t) + original := runGit + runGit = git.run + t.Cleanup(func() { runGit = original }) + + if err := ensureTemplateRevisions(workspace, cfg); err != nil { + t.Fatal(err) + } + git.failFetch = true + err := ensureTemplateRevisions(workspace, cfg) + if err == nil || !strings.Contains(err.Error(), "locked template revision cannot be resolved") || !strings.Contains(err.Error(), "new empty workspaceDir") || strings.Contains(err.Error(), "template-password") { + t.Fatalf("locked revision failure exposed source details or lacked recovery guidance: %v", err) + } +} + +func templateRevisionTestConfig(t *testing.T) (string, config.Config, *fakeTemplateGit) { + t.Helper() + workspace := t.TempDir() + cloneParent := filepath.Join(workspace, "checkouts") + if err := os.MkdirAll(cloneParent, 0755); err != nil { + t.Fatal(err) + } + git := &fakeTemplateGit{ + commits: map[string]string{ + "cicd": strings.Repeat("a", 40), + "manifests": strings.Repeat("b", 40), + "talos": strings.Repeat("c", 40), + }, + origins: map[string]string{}, + fetched: map[string]string{}, + checkedOut: map[string]string{}, + } + return workspace, config.Config{ + WorkspaceDir: workspace, + Git: config.GitConfig{CloneParent: cloneParent}, + Talos: config.TalosConfig{RepoDirName: "talos"}, + Templates: config.TemplateConfig{ + CICDRepoURL: "https://reader:template-password@git.example.test/templates/cicd.git", + CICDRepoRef: "main", + ManifestsRepoURL: "https://git.example.test/templates/manifests.git", + ManifestsRepoRef: "main", + TalosRepoURL: "https://git.example.test/templates/talos.git", + TalosRepoRef: "main", + }, + }, git +} + +type fakeTemplateGit struct { + commits map[string]string + origins map[string]string + fetched map[string]string + checkedOut map[string]string + calls []string + failFetch bool +} + +func (git *fakeTemplateGit) run(dir string, args ...string) ([]byte, error) { + git.calls = append(git.calls, strings.Join(args, " ")) + if len(args) == 0 { + return nil, errors.New("missing Git command") + } + switch args[0] { + case "clone": + dir = args[len(args)-1] + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, err + } + git.origins[dir] = args[len(args)-2] + return nil, nil + case "remote": + return []byte(git.origins[dir] + "\n"), nil + case "status": + return nil, nil + case "fetch": + if git.failFetch { + return nil, errors.New("not found") + } + target := args[len(args)-1] + name := git.templateName(dir) + if len(target) == 40 { + git.fetched[name] = target + } else { + git.fetched[name] = git.commits[name] + } + return nil, nil + case "checkout": + git.checkedOut[git.templateName(dir)] = args[len(args)-1] + return nil, nil + case "rev-parse": + if len(args) == 2 && args[1] == "--is-inside-work-tree" { + return []byte("true\n"), nil + } + target := args[len(args)-1] + if target == "FETCH_HEAD^{commit}" { + return []byte(git.fetched[git.templateName(dir)] + "\n"), nil + } + if target == "HEAD^{commit}" { + return []byte(git.checkedOut[git.templateName(dir)] + "\n"), nil + } + } + return nil, errors.New("unexpected Git command") +} + +func (git *fakeTemplateGit) templateName(dir string) string { + switch filepath.Base(dir) { + case "maidn-cicd-cluster-template": + return "cicd" + case "cicd-deployment-manifests-template": + return "manifests" + default: + return "talos" + } +} + +func (git *fakeTemplateGit) resetCalls() { + git.calls = nil +} + +func (git *fakeTemplateGit) commands() string { + return strings.Join(git.calls, "\n") +} diff --git a/internal/config/config.go b/internal/config/config.go index f84167d..c7ad921 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/netip" + "net/url" "os" "path/filepath" "regexp" @@ -55,8 +56,16 @@ func Save(path string, cfg Config) error { func WriteRedacted(path string, cfg Config) error { redacted := cfg redacted.Git.Token = "" + redacted.Git.BaseURL = RedactURL(redacted.Git.BaseURL) redacted.Talos.Proxmox.APITokenSecret = "" + redacted.Talos.Proxmox.APIURL = RedactURL(redacted.Talos.Proxmox.APIURL) redacted.DemocraticCSI.TrueNASAPIKey = "" + redacted.Templates.TalosRepoURL = RedactURL(redacted.Templates.TalosRepoURL) + redacted.Templates.CICDRepoURL = RedactURL(redacted.Templates.CICDRepoURL) + redacted.Templates.ManifestsRepoURL = RedactURL(redacted.Templates.ManifestsRepoURL) + redacted.Templates.TektonCatalogRepoURL = RedactURL(redacted.Templates.TektonCatalogRepoURL) + redacted.Delivery.AppRepoURL = RedactURL(redacted.Delivery.AppRepoURL) + redacted.Delivery.ImageRepository = RedactURL(redacted.Delivery.ImageRepository) data, err := yaml.Marshal(redacted) if err != nil { return err @@ -64,6 +73,28 @@ func WriteRedacted(path string, cfg Config) error { return os.WriteFile(path, data, 0600) } +// RedactURL removes credential-bearing URL components before persistent output. +func RedactURL(value string) string { + if strings.Contains(value, "://") { + parsed, err := url.Parse(value) + if err != nil { + return "" + } + parsed.User = nil + parsed.RawQuery = "" + parsed.ForceQuery = false + parsed.Fragment = "" + return parsed.String() + } + if at := strings.Index(value, "@"); at > 0 && !strings.Contains(value[:at], "/") && strings.Contains(value[at+1:], ":") { + return value[at+1:] + } + if strings.ContainsAny(value, "@?#") { + return "" + } + return value +} + func Resolve(cfg Config) (Config, error) { applyDefaults(&cfg) return cfg, Validate(cfg) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 3f3e126..5d72fda 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -62,6 +62,9 @@ func TestLoadRawAllowsCompletionBeforeValidation(t *testing.T) { func TestWriteRedactedOmitsCredentials(t *testing.T) { cfg := validConfig(t) + cfg.Git.BaseURL = "https://reader:git-token@git.example.test" + cfg.Templates.CICDRepoURL = "https://reader:template-token@git.example.test/template.git?access_token=query-token" + cfg.Delivery.ImageRepository = "reader:registry-token@registry.example.test/team/app" path := filepath.Join(t.TempDir(), "resolved.yaml") if err := WriteRedacted(path, cfg); err != nil { t.Fatal(err) @@ -70,8 +73,8 @@ func TestWriteRedactedOmitsCredentials(t *testing.T) { if err != nil { t.Fatal(err) } - if strings.Contains(string(data), "token: token") || strings.Contains(string(data), "apiTokenSecret: secret") || strings.Contains(string(data), "truenasApiKey: api-key") { - t.Fatalf("redacted config contains credentials: %s", data) + if strings.Contains(string(data), "token: token") || strings.Contains(string(data), "apiTokenSecret: secret") || strings.Contains(string(data), "truenasApiKey: api-key") || strings.Contains(string(data), "git-token") || strings.Contains(string(data), "template-token") || strings.Contains(string(data), "query-token") || strings.Contains(string(data), "registry-token") { + t.Fatal("redacted config contains credentials") } } diff --git a/internal/openbao/bootstrap.go b/internal/openbao/bootstrap.go index a3acb94..a9c173b 100644 --- a/internal/openbao/bootstrap.go +++ b/internal/openbao/bootstrap.go @@ -88,7 +88,7 @@ func Initialize(kubeconfig, recipient, identityPath, bundlePath, ageKeyPath, ope return nil, err } } - reviewerToken, err := kubectlOutput(kubeconfig, "-n", "openbao", "create", "token", "openbao-auth") + reviewerToken, err := kubectlOutput(kubeconfig, "-n", "openbao", "create", "token", "openbao-auth", "--duration=8760h") if err != nil { return nil, fmt.Errorf("create OpenBao Kubernetes token reviewer token: %w", err) } @@ -179,10 +179,8 @@ func writeSecret(kubeconfig, rootToken, secretPath string, values map[string]str func waitForPod(kubeconfig string) error { deadline := time.Now().Add(10 * time.Minute) for time.Now().Before(deadline) { - if _, err := kubectlOutput(kubeconfig, "-n", "openbao", "get", "pod", "openbao-0"); err == nil { - if output, _ := execInPod(kubeconfig, nil, "bao", "status", "-format=json"); len(output) > 0 { - return nil - } + if _, err := getStatus(kubeconfig); err == nil { + return nil } time.Sleep(2 * time.Second) } @@ -190,8 +188,9 @@ func waitForPod(kubeconfig string) error { } func getStatus(kubeconfig string) (status, error) { - output, err := execInPod(kubeconfig, nil, "bao", "status", "-format=json") - if err != nil && len(output) == 0 { + command := []string{"--kubeconfig", kubeconfig, "-n", "openbao", "exec", "openbao-0", "--", "bao", "status", "-format=json"} + output, err := exec.Command("kubectl", command...).Output() + if err != nil && !json.Valid(output) { return status{}, fmt.Errorf("get OpenBao status: %w", err) } var current status diff --git a/internal/ui/wizard.go b/internal/ui/wizard.go index c0a2636..a4a4ce2 100644 --- a/internal/ui/wizard.go +++ b/internal/ui/wizard.go @@ -130,10 +130,12 @@ func PromptOperationalSecrets(cfg config.Config) (map[string]map[string]string, return nil, errors.New("git username, token, and delivery imageRepository are required") } reader := bufio.NewReader(os.Stdin) - cloudflareAPIToken := promptSecret(reader, "Cloudflare API token", "") + piholeServer := prompt(reader, "Pi-hole server", "") + piholePassword := promptSecret(reader, "Pi-hole password", "") + cloudflareAPIToken := promptSecret(reader, "Cloudflare DNS-01 API token", "") cloudflareTunnelToken := promptSecret(reader, "Cloudflare Tunnel token", "") - if cloudflareAPIToken == "" || cloudflareTunnelToken == "" { - return nil, errors.New("Cloudflare API and Tunnel tokens are required") + if piholeServer == "" || piholePassword == "" || cloudflareAPIToken == "" || cloudflareTunnelToken == "" { + return nil, errors.New("Pi-hole server, password, Cloudflare DNS-01 API token, and Cloudflare Tunnel token are required") } registryHost := strings.Split(cfg.Delivery.ImageRepository, "/")[0] dockerConfig, err := json.Marshal(map[string]map[string]map[string]string{ @@ -147,6 +149,7 @@ func PromptOperationalSecrets(cfg config.Config) (map[string]map[string]string, return map[string]map[string]string{ "cicd/forgejo": {"username": cfg.Git.Username, "token": cfg.Git.Token}, "cicd/forgejo-registry": {"dockerconfigjson": string(dockerConfig)}, + "platform/pihole": {"server": piholeServer, "password": piholePassword}, "platform/cloudflare": {"api-token": cloudflareAPIToken}, "platform/cloudflare-tunnel": {"token": cloudflareTunnelToken}, }, nil