From 676e89de405588b4a672be5c21b8009065e4d9ae Mon Sep 17 00:00:00 2001 From: eding Date: Sat, 1 Aug 2026 22:49:24 +0200 Subject: [PATCH] fix: wait for webhook secret refresh --- internal/bootstrap/bootstrap.go | 73 +++++++++--- internal/bootstrap/bootstrap_test.go | 161 ++++++++++++++++++++++++++- 2 files changed, 214 insertions(+), 20 deletions(-) diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 3d8e8c1..ad0d5b0 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -45,6 +45,21 @@ type operationalSecrets struct { var initializeOpenBao = openbao.Initialize +var ensureForgejoWebhook = func(cfg config.Config, repo, webhookURL, authorization string) error { + manager := forgejo.NewRepoManager(cfg.Git.BaseURL, cfg.Git.Token, cfg.Git.Owner, cfg.Git.Username, cfg.Flux.ManifestsRepo, cfg.Flux.RepoName, cfg.Flux.Branch, "maidn/bootstrap-"+cfg.ClusterID) + return manager.EnsureWebhook(repo, webhookURL, authorization) +} + +var runWebhookCommand = utils.RunCommandQuietOutputInDir + +var webhookTargetTimeout = 70 * time.Minute + +var webhookTargetPollInterval = 2 * time.Second + +var templateBaseComponents = []string{"snapshot-crds", "democratic-csi", "cert-manager", "cluster-issuers", "gateway-api", "gateway", "monitoring", "openbao", "external-secrets", "external-dns", "tekton", "tekton-triggers"} + +var requiredClusterKustomizations = []string{"snapshot-crds-kustomization.yaml", "democratic-csi-kustomization.yaml", "cert-manager-kustomization.yaml", "cluster-issuers-kustomization.yaml", "gateway-api-kustomization.yaml", "cilium-kustomization.yaml", "cilium-config-kustomization.yaml", "openbao-kustomization.yaml", "external-secrets-kustomization.yaml", "cnpg-kustomization.yaml", "gateway-kustomization.yaml", "external-dns-kustomization.yaml", "cloudflare-tunnel-kustomization.yaml", "monitoring-kustomization.yaml", "tekton-kustomization.yaml", "tekton-triggers-kustomization.yaml", "cicd-manifests-repo.yaml"} + func (r Runner) Run() error { resolved, err := config.Resolve(r.Config) if err != nil { @@ -112,10 +127,8 @@ func (r Runner) Run() error { if err := copyDir(filepath.Join(cicdTemplateDir, "base"), filepath.Join(dir, "base"), false); err != nil { return err } - for _, component := range []string{"snapshot-crds", "democratic-csi", "cert-manager", "cluster-issuers", "gateway-api", "gateway", "monitoring", "openbao", "external-secrets", "external-secrets-config", "external-dns", "tekton", "tekton-triggers"} { - if err := copyDir(filepath.Join(cicdTemplateDir, "base", component), filepath.Join(dir, "base", component), true); err != nil { - return err - } + if err := copyTemplateBaseComponents(cicdTemplateDir, dir); err != nil { + return err } if err := copyClusterTemplate(filepath.Join(cicdTemplateDir, "clusters", "template"), clusterDir); err != nil { return err @@ -227,7 +240,7 @@ func (r Runner) Run() error { } func (r Runner) reconcileWebhook(generatedDir string) error { - operationalSecrets, err := openbao.Initialize(filepath.Join(generatedDir, "kubeconfig"), r.Config.SOPS.RecoveryRecipient, r.Config.SOPS.RecoveryIdentityPath, r.Config.SOPS.RecoveryBundlePath, r.Config.SOPS.AgeKeyPath, r.Config.SOPS.OperationalSecretsPath) + operationalSecrets, err := initializeOpenBao(filepath.Join(generatedDir, "kubeconfig"), r.Config.SOPS.RecoveryRecipient, r.Config.SOPS.RecoveryIdentityPath, r.Config.SOPS.RecoveryBundlePath, r.Config.SOPS.AgeKeyPath, r.Config.SOPS.OperationalSecretsPath) if err != nil { return fmt.Errorf("initialize OpenBao: %w", err) } @@ -235,11 +248,10 @@ func (r Runner) reconcileWebhook(generatedDir string) error { if authorization == "" { return errors.New("operational SOPS secrets requires cicd/forgejo-webhook.authorization") } - if err := waitForWebhookTargets(generatedDir, r.Config); err != nil { + if err := waitForWebhookTargets(generatedDir, r.Config, authorization); err != nil { return err } - manager := forgejo.NewRepoManager(r.Config.Git.BaseURL, r.Config.Git.Token, r.Config.Git.Owner, r.Config.Git.Username, r.Config.Flux.ManifestsRepo, r.Config.Flux.RepoName, r.Config.Flux.Branch, "maidn/bootstrap-"+r.Config.ClusterID) - if err := manager.EnsureWebhook(r.Config.Delivery.AppName, r.Config.Delivery.WebhookURL(), authorization); err != nil { + if err := ensureForgejoWebhook(r.Config, r.Config.Delivery.AppName, r.Config.Delivery.WebhookURL(), authorization); err != nil { return fmt.Errorf("reconcile Forgejo webhook: %w", err) } return nil @@ -464,27 +476,43 @@ func renderDemocraticCSISecret(csi config.DemocraticCSIConfig) ([]byte, error) { }) } -func waitForWebhookTargets(dir string, cfg config.Config) error { - resources := []string{ - "secret/forgejo-webhook", - "deployment/el-" + cfg.Delivery.AppName, - "pipeline/" + cfg.Delivery.AppName, +func waitForWebhookTargets(dir string, cfg config.Config, authorization string) error { + if err := waitForWebhookAuthorization(dir, authorization); err != nil { + return err } + resources := []string{"deployment/el-" + cfg.Delivery.AppName, "pipeline/" + cfg.Delivery.AppName} for _, resource := range resources { - deadline := time.Now().Add(10 * time.Minute) + deadline := time.Now().Add(webhookTargetTimeout) for time.Now().Before(deadline) { - if _, err := utils.RunCommandQuietOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err == nil { + if _, err := runWebhookCommand(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err == nil { break } - time.Sleep(2 * time.Second) + time.Sleep(webhookTargetPollInterval) } - if _, err := utils.RunCommandQuietOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err != nil { + if _, err := runWebhookCommand(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err != nil { return fmt.Errorf("wait for %s before registering Forgejo webhook", resource) } } return nil } +func waitForWebhookAuthorization(dir, authorization string) error { + deadline := time.Now().Add(webhookTargetTimeout) + for { + output, err := runWebhookCommand(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", "secret/forgejo-webhook", "-o=jsonpath={.data.authorization}") + if err == nil { + observed, decodeErr := base64.StdEncoding.DecodeString(strings.TrimSpace(string(output))) + if decodeErr == nil && string(observed) == authorization { + return nil + } + } + if !time.Now().Before(deadline) { + return errors.New("ExternalSecret target Secret forgejo-webhook did not refresh within the timeout; Forgejo webhook was not updated. Wait for External Secrets to recover, then safely rerun cicd-tool bootstrap --config --register-webhook") + } + time.Sleep(webhookTargetPollInterval) + } +} + func ensureClusterKustomizations(clusterDir string) error { path := filepath.Join(clusterDir, "kustomization.yaml") content, err := os.ReadFile(path) @@ -498,7 +526,7 @@ func ensureClusterKustomizations(clusterDir string) error { } updated := string(content) updated = strings.ReplaceAll(updated, " - bootstrap-secrets.sops.yaml\n", "") - for _, resource := range []string{"snapshot-crds-kustomization.yaml", "democratic-csi-kustomization.yaml", "cert-manager-kustomization.yaml", "cluster-issuers-kustomization.yaml", "gateway-api-kustomization.yaml", "gateway-kustomization.yaml", "cilium-kustomization.yaml", "cilium-config-kustomization.yaml", "cicd-manifests-repo.yaml", "external-secrets-config-kustomization.yaml", "external-dns-kustomization.yaml", "monitoring-kustomization.yaml", "tekton-kustomization.yaml", "tekton-triggers-kustomization.yaml"} { + for _, resource := range requiredClusterKustomizations { if !strings.Contains(updated, resource) { updated += " - " + resource + "\n" } @@ -509,6 +537,15 @@ func ensureClusterKustomizations(clusterDir string) error { return os.WriteFile(path, []byte(updated), 0644) } +func copyTemplateBaseComponents(templateDir, repoDir string) error { + for _, component := range templateBaseComponents { + if err := copyDir(filepath.Join(templateDir, "base", component), filepath.Join(repoDir, "base", component), true); err != nil { + return err + } + } + return nil +} + func copyClusterTemplate(source, destination string) error { entries, err := os.ReadDir(destination) if err != nil && !os.IsNotExist(err) { diff --git a/internal/bootstrap/bootstrap_test.go b/internal/bootstrap/bootstrap_test.go index a17aaa6..9c8a0cb 100644 --- a/internal/bootstrap/bootstrap_test.go +++ b/internal/bootstrap/bootstrap_test.go @@ -4,13 +4,17 @@ import ( "encoding/base64" "encoding/json" "errors" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" "strings" "testing" + "time" "github.com/Pingu-Studio/MaidnCLI/internal/config" + "github.com/Pingu-Studio/MaidnCLI/internal/forgejo" ) func TestRenderCiliumConfig(t *testing.T) { @@ -101,6 +105,35 @@ func TestCopyAndRenderDeliveryBasesOverwritesExistingMigrationOutput(t *testing. } } +func TestCopyTemplateBaseComponentsCopiesExternalSecretsConfig(t *testing.T) { + templateDir := t.TempDir() + repoDir := t.TempDir() + for _, component := range templateBaseComponents { + if err := os.MkdirAll(filepath.Join(templateDir, "base", component), 0755); err != nil { + t.Fatal(err) + } + } + source := filepath.Join(templateDir, "base", "external-secrets", "config.yaml") + destination := filepath.Join(repoDir, "base", "external-secrets", "config.yaml") + if err := os.WriteFile(source, []byte("store: openbao\n"), 0644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(destination), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(destination, []byte("store: stale\n"), 0644); err != nil { + t.Fatal(err) + } + + if err := copyTemplateBaseComponents(templateDir, repoDir); err != nil { + t.Fatal(err) + } + content, err := os.ReadFile(destination) + if err != nil || string(content) != "store: openbao\n" { + t.Fatalf("external-secrets configuration was not copied: %q, %v", content, err) + } +} + func TestRenderDemocraticCSISecret(t *testing.T) { secret, err := renderDemocraticCSISecret(config.DemocraticCSIConfig{TrueNASAPIKey: "api-key", TrueNASHost: "truenas.example.test", TargetPortal: "truenas.example.test:3260", ShareHost: "truenas.example.test", DatasetParentNFS: "pool/kubernetes/nfs/v", DatasetSnapshotsNFS: "pool/kubernetes/nfs/s", AllowedNetworks: "192.168.45.0/24", NameSuffix: "-test", PortalGroup: "1", InitiatorGroup: "1"}) if err != nil { @@ -291,7 +324,7 @@ func TestEnsureManifestsKustomizations(t *testing.T) { } } -func TestEnsureClusterKustomizationsAddsStorageDependencies(t *testing.T) { +func TestEnsureClusterKustomizationsUsesTemplateExternalSecretsResource(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "kustomization.yaml") if err := os.WriteFile(path, []byte("resources:\n"), 0644); err != nil { @@ -301,11 +334,135 @@ func TestEnsureClusterKustomizationsAddsStorageDependencies(t *testing.T) { t.Fatal(err) } content, err := os.ReadFile(path) - if err != nil || !strings.Contains(string(content), "snapshot-crds-kustomization.yaml") || !strings.Contains(string(content), "democratic-csi-kustomization.yaml") || !strings.Contains(string(content), "gateway-api-kustomization.yaml") || !strings.Contains(string(content), "gateway-kustomization.yaml") || !strings.Contains(string(content), "external-secrets-config-kustomization.yaml") || !strings.Contains(string(content), "external-dns-kustomization.yaml") || !strings.Contains(string(content), "monitoring-kustomization.yaml") || !strings.Contains(string(content), "tekton-kustomization.yaml") { + if err != nil || !strings.Contains(string(content), "snapshot-crds-kustomization.yaml") || !strings.Contains(string(content), "democratic-csi-kustomization.yaml") || !strings.Contains(string(content), "openbao-kustomization.yaml") || !strings.Contains(string(content), "external-secrets-kustomization.yaml") || !strings.Contains(string(content), "cnpg-kustomization.yaml") || !strings.Contains(string(content), "cloudflare-tunnel-kustomization.yaml") || !strings.Contains(string(content), "tekton-kustomization.yaml") || strings.Contains(string(content), "external-secrets-config-kustomization.yaml") { t.Fatalf("cluster Kustomization was not updated: %q, %v", content, err) } } +func TestWebhookTargetTimeoutExceedsExternalSecretRefreshInterval(t *testing.T) { + if webhookTargetTimeout <= time.Hour { + t.Fatal("webhook target timeout must exceed the one-hour ExternalSecret refresh interval") + } +} + +func TestReconcileWebhookWaitsForRefreshedTargetSecret(t *testing.T) { + originalInitialize := initializeOpenBao + originalCommand := runWebhookCommand + originalWebhook := ensureForgejoWebhook + originalTimeout := webhookTargetTimeout + originalInterval := webhookTargetPollInterval + t.Cleanup(func() { + initializeOpenBao = originalInitialize + runWebhookCommand = originalCommand + ensureForgejoWebhook = originalWebhook + webhookTargetTimeout = originalTimeout + webhookTargetPollInterval = originalInterval + }) + + authorization := "Bearer test-webhook-authorization" + staleTarget := base64.StdEncoding.EncodeToString([]byte("Bearer stale-webhook-authorization")) + refreshedTarget := base64.StdEncoding.EncodeToString([]byte(authorization)) + targetChecks := 0 + refreshedObserved := false + runWebhookCommand = func(_ string, name string, args ...string) ([]byte, error) { + if name != "kubectl" || strings.Contains(strings.Join(args, " "), authorization) { + t.Fatal("webhook target probe used an unexpected command") + } + if strings.Contains(strings.Join(args, " "), "secret/forgejo-webhook") { + targetChecks++ + if targetChecks == 1 { + return []byte(staleTarget), nil + } + refreshedObserved = true + return []byte(refreshedTarget), nil + } + return nil, nil + } + webhookTargetPollInterval = 0 + + patches := 0 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if !refreshedObserved { + t.Error("Forgejo was called before the refreshed webhook Secret was observed") + writer.WriteHeader(http.StatusInternalServerError) + return + } + switch request.Method { + case http.MethodGet: + _ = json.NewEncoder(writer).Encode([]map[string]any{{"id": 7, "url": "https://tekton.example.test/"}}) + case http.MethodPatch: + var body struct { + AuthorizationHeader string `json:"authorization_header"` + } + if err := json.NewDecoder(request.Body).Decode(&body); err != nil || body.AuthorizationHeader != authorization { + t.Error("Forgejo webhook update did not use the observed authorization") + writer.WriteHeader(http.StatusBadRequest) + return + } + patches++ + writer.WriteHeader(http.StatusOK) + default: + t.Error("unexpected Forgejo request") + writer.WriteHeader(http.StatusMethodNotAllowed) + } + })) + defer server.Close() + ensureForgejoWebhook = func(_ config.Config, repo, webhookURL, authorization string) error { + manager := forgejo.NewRepoManager(server.URL, "test-token", "owner", "user", "manifests", "flux", "main", "migration") + manager.HTTPClient = server.Client() + return manager.EnsureWebhook(repo, webhookURL, authorization) + } + initializeOpenBao = func(string, string, string, string, string, string) (map[string]map[string]string, error) { + return map[string]map[string]string{"cicd/forgejo-webhook": {"authorization": authorization}}, nil + } + + cfg := config.Config{Delivery: config.DeliveryConfig{AppName: "app", WebhookHostname: "tekton.example.test", WebhookPath: "/"}} + if err := (Runner{Config: cfg}).reconcileWebhook(t.TempDir()); err != nil { + t.Fatal(err) + } + if targetChecks != 2 || patches != 1 { + t.Fatal("Forgejo webhook was not updated after the target Secret refreshed") + } +} + +func TestReconcileWebhookTimeoutDoesNotUpdateForgejo(t *testing.T) { + originalInitialize := initializeOpenBao + originalCommand := runWebhookCommand + originalWebhook := ensureForgejoWebhook + originalTimeout := webhookTargetTimeout + originalInterval := webhookTargetPollInterval + t.Cleanup(func() { + initializeOpenBao = originalInitialize + runWebhookCommand = originalCommand + ensureForgejoWebhook = originalWebhook + webhookTargetTimeout = originalTimeout + webhookTargetPollInterval = originalInterval + }) + + authorization := "Bearer test-webhook-authorization" + initializeOpenBao = func(string, string, string, string, string, string) (map[string]map[string]string, error) { + return map[string]map[string]string{"cicd/forgejo-webhook": {"authorization": authorization}}, nil + } + runWebhookCommand = func(_ string, _ string, _ ...string) ([]byte, error) { + return []byte(base64.StdEncoding.EncodeToString([]byte("Bearer stale-webhook-authorization"))), nil + } + webhookTargetTimeout = -time.Nanosecond + webhookTargetPollInterval = 0 + webhookUpdated := false + ensureForgejoWebhook = func(config.Config, string, string, string) error { + webhookUpdated = true + return nil + } + + err := (Runner{Config: config.Config{Delivery: config.DeliveryConfig{AppName: "app"}}}).reconcileWebhook(t.TempDir()) + if err == nil || !strings.Contains(err.Error(), "Forgejo webhook was not updated") || !strings.Contains(err.Error(), "--register-webhook") { + t.Fatal("webhook refresh timeout did not return a safe rerun error") + } + if webhookUpdated { + t.Fatal("Forgejo webhook update was attempted before the target Secret refreshed") + } +} + func TestRenderTerraformTFVarsIsStableAndRedactsToken(t *testing.T) { cfg := config.Config{ ClusterID: "test",