fix: wait for webhook secret refresh

internal/bootstrap/bootstrap.go

@@ -45,6 +45,21 @@ type operationalSecrets struct {
 
 var initializeOpenBao = openbao.Initialize
 
+var ensureForgejoWebhook = func(cfg config.Config, repo, webhookURL, authorization string) error {
+	manager := forgejo.NewRepoManager(cfg.Git.BaseURL, cfg.Git.Token, cfg.Git.Owner, cfg.Git.Username, cfg.Flux.ManifestsRepo, cfg.Flux.RepoName, cfg.Flux.Branch, "maidn/bootstrap-"+cfg.ClusterID)
+	return manager.EnsureWebhook(repo, webhookURL, authorization)
+}
+
+var runWebhookCommand = utils.RunCommandQuietOutputInDir
+
+var webhookTargetTimeout = 70 * time.Minute
+
+var webhookTargetPollInterval = 2 * time.Second
+
+var templateBaseComponents = []string{"snapshot-crds", "democratic-csi", "cert-manager", "cluster-issuers", "gateway-api", "gateway", "monitoring", "openbao", "external-secrets", "external-dns", "tekton", "tekton-triggers"}
+
+var requiredClusterKustomizations = []string{"snapshot-crds-kustomization.yaml", "democratic-csi-kustomization.yaml", "cert-manager-kustomization.yaml", "cluster-issuers-kustomization.yaml", "gateway-api-kustomization.yaml", "cilium-kustomization.yaml", "cilium-config-kustomization.yaml", "openbao-kustomization.yaml", "external-secrets-kustomization.yaml", "cnpg-kustomization.yaml", "gateway-kustomization.yaml", "external-dns-kustomization.yaml", "cloudflare-tunnel-kustomization.yaml", "monitoring-kustomization.yaml", "tekton-kustomization.yaml", "tekton-triggers-kustomization.yaml", "cicd-manifests-repo.yaml"}
+
 func (r Runner) Run() error {
 	resolved, err := config.Resolve(r.Config)
 	if err != nil {
@@ -112,11 +127,9 @@ func (r Runner) Run() error {
 			if err := copyDir(filepath.Join(cicdTemplateDir, "base"), filepath.Join(dir, "base"), false); err != nil {
 				return err
 			}
-			for _, component := range []string{"snapshot-crds", "democratic-csi", "cert-manager", "cluster-issuers", "gateway-api", "gateway", "monitoring", "openbao", "external-secrets", "external-secrets-config", "external-dns", "tekton", "tekton-triggers"} {
+			if err := copyTemplateBaseComponents(cicdTemplateDir, dir); err != nil {
-				if err := copyDir(filepath.Join(cicdTemplateDir, "base", component), filepath.Join(dir, "base", component), true); err != nil {
 				return err
 			}
-			}
 			if err := copyClusterTemplate(filepath.Join(cicdTemplateDir, "clusters", "template"), clusterDir); err != nil {
 				return err
 			}
@@ -227,7 +240,7 @@ func (r Runner) Run() error {
 }
 
 func (r Runner) reconcileWebhook(generatedDir string) error {
-	operationalSecrets, err := openbao.Initialize(filepath.Join(generatedDir, "kubeconfig"), r.Config.SOPS.RecoveryRecipient, r.Config.SOPS.RecoveryIdentityPath, r.Config.SOPS.RecoveryBundlePath, r.Config.SOPS.AgeKeyPath, r.Config.SOPS.OperationalSecretsPath)
+	operationalSecrets, err := initializeOpenBao(filepath.Join(generatedDir, "kubeconfig"), r.Config.SOPS.RecoveryRecipient, r.Config.SOPS.RecoveryIdentityPath, r.Config.SOPS.RecoveryBundlePath, r.Config.SOPS.AgeKeyPath, r.Config.SOPS.OperationalSecretsPath)
 	if err != nil {
 		return fmt.Errorf("initialize OpenBao: %w", err)
 	}
@@ -235,11 +248,10 @@ func (r Runner) reconcileWebhook(generatedDir string) error {
 	if authorization == "" {
 		return errors.New("operational SOPS secrets requires cicd/forgejo-webhook.authorization")
 	}
-	if err := waitForWebhookTargets(generatedDir, r.Config); err != nil {
+	if err := waitForWebhookTargets(generatedDir, r.Config, authorization); err != nil {
 		return err
 	}
-	manager := forgejo.NewRepoManager(r.Config.Git.BaseURL, r.Config.Git.Token, r.Config.Git.Owner, r.Config.Git.Username, r.Config.Flux.ManifestsRepo, r.Config.Flux.RepoName, r.Config.Flux.Branch, "maidn/bootstrap-"+r.Config.ClusterID)
+	if err := ensureForgejoWebhook(r.Config, r.Config.Delivery.AppName, r.Config.Delivery.WebhookURL(), authorization); err != nil {
-	if err := manager.EnsureWebhook(r.Config.Delivery.AppName, r.Config.Delivery.WebhookURL(), authorization); err != nil {
 		return fmt.Errorf("reconcile Forgejo webhook: %w", err)
 	}
 	return nil
@@ -464,27 +476,43 @@ func renderDemocraticCSISecret(csi config.DemocraticCSIConfig) ([]byte, error) {
 	})
 }
 
-func waitForWebhookTargets(dir string, cfg config.Config) error {
+func waitForWebhookTargets(dir string, cfg config.Config, authorization string) error {
-	resources := []string{
+	if err := waitForWebhookAuthorization(dir, authorization); err != nil {
-		"secret/forgejo-webhook",
+		return err
-		"deployment/el-" + cfg.Delivery.AppName,
-		"pipeline/" + cfg.Delivery.AppName,
 	}
+	resources := []string{"deployment/el-" + cfg.Delivery.AppName, "pipeline/" + cfg.Delivery.AppName}
 	for _, resource := range resources {
-		deadline := time.Now().Add(10 * time.Minute)
+		deadline := time.Now().Add(webhookTargetTimeout)
 		for time.Now().Before(deadline) {
-			if _, err := utils.RunCommandQuietOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err == nil {
+			if _, err := runWebhookCommand(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err == nil {
 				break
 			}
-			time.Sleep(2 * time.Second)
+			time.Sleep(webhookTargetPollInterval)
 		}
-		if _, err := utils.RunCommandQuietOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err != nil {
+		if _, err := runWebhookCommand(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err != nil {
 			return fmt.Errorf("wait for %s before registering Forgejo webhook", resource)
 		}
 	}
 	return nil
 }
 
+func waitForWebhookAuthorization(dir, authorization string) error {
+	deadline := time.Now().Add(webhookTargetTimeout)
+	for {
+		output, err := runWebhookCommand(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", "secret/forgejo-webhook", "-o=jsonpath={.data.authorization}")
+		if err == nil {
+			observed, decodeErr := base64.StdEncoding.DecodeString(strings.TrimSpace(string(output)))
+			if decodeErr == nil && string(observed) == authorization {
+				return nil
+			}
+		}
+		if !time.Now().Before(deadline) {
+			return errors.New("ExternalSecret target Secret forgejo-webhook did not refresh within the timeout; Forgejo webhook was not updated. Wait for External Secrets to recover, then safely rerun cicd-tool bootstrap --config <config> --register-webhook")
+		}
+		time.Sleep(webhookTargetPollInterval)
+	}
+}
+
 func ensureClusterKustomizations(clusterDir string) error {
 	path := filepath.Join(clusterDir, "kustomization.yaml")
 	content, err := os.ReadFile(path)
@@ -498,7 +526,7 @@ func ensureClusterKustomizations(clusterDir string) error {
 	}
 	updated := string(content)
 	updated = strings.ReplaceAll(updated, "  - bootstrap-secrets.sops.yaml\n", "")
-	for _, resource := range []string{"snapshot-crds-kustomization.yaml", "democratic-csi-kustomization.yaml", "cert-manager-kustomization.yaml", "cluster-issuers-kustomization.yaml", "gateway-api-kustomization.yaml", "gateway-kustomization.yaml", "cilium-kustomization.yaml", "cilium-config-kustomization.yaml", "cicd-manifests-repo.yaml", "external-secrets-config-kustomization.yaml", "external-dns-kustomization.yaml", "monitoring-kustomization.yaml", "tekton-kustomization.yaml", "tekton-triggers-kustomization.yaml"} {
+	for _, resource := range requiredClusterKustomizations {
 		if !strings.Contains(updated, resource) {
 			updated += "  - " + resource + "\n"
 		}
@@ -509,6 +537,15 @@ func ensureClusterKustomizations(clusterDir string) error {
 	return os.WriteFile(path, []byte(updated), 0644)
 }
 
+func copyTemplateBaseComponents(templateDir, repoDir string) error {
+	for _, component := range templateBaseComponents {
+		if err := copyDir(filepath.Join(templateDir, "base", component), filepath.Join(repoDir, "base", component), true); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func copyClusterTemplate(source, destination string) error {
 	entries, err := os.ReadDir(destination)
 	if err != nil && !os.IsNotExist(err) {

internal/bootstrap/bootstrap_test.go

@@ -4,13 +4,17 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"errors"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/Pingu-Studio/MaidnCLI/internal/config"
+	"github.com/Pingu-Studio/MaidnCLI/internal/forgejo"
 )
 
 func TestRenderCiliumConfig(t *testing.T) {
@@ -101,6 +105,35 @@ func TestCopyAndRenderDeliveryBasesOverwritesExistingMigrationOutput(t *testing.
 	}
 }
 
+func TestCopyTemplateBaseComponentsCopiesExternalSecretsConfig(t *testing.T) {
+	templateDir := t.TempDir()
+	repoDir := t.TempDir()
+	for _, component := range templateBaseComponents {
+		if err := os.MkdirAll(filepath.Join(templateDir, "base", component), 0755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	source := filepath.Join(templateDir, "base", "external-secrets", "config.yaml")
+	destination := filepath.Join(repoDir, "base", "external-secrets", "config.yaml")
+	if err := os.WriteFile(source, []byte("store: openbao\n"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.MkdirAll(filepath.Dir(destination), 0755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(destination, []byte("store: stale\n"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := copyTemplateBaseComponents(templateDir, repoDir); err != nil {
+		t.Fatal(err)
+	}
+	content, err := os.ReadFile(destination)
+	if err != nil || string(content) != "store: openbao\n" {
+		t.Fatalf("external-secrets configuration was not copied: %q, %v", content, err)
+	}
+}
+
 func TestRenderDemocraticCSISecret(t *testing.T) {
 	secret, err := renderDemocraticCSISecret(config.DemocraticCSIConfig{TrueNASAPIKey: "api-key", TrueNASHost: "truenas.example.test", TargetPortal: "truenas.example.test:3260", ShareHost: "truenas.example.test", DatasetParentNFS: "pool/kubernetes/nfs/v", DatasetSnapshotsNFS: "pool/kubernetes/nfs/s", AllowedNetworks: "192.168.45.0/24", NameSuffix: "-test", PortalGroup: "1", InitiatorGroup: "1"})
 	if err != nil {
@@ -291,7 +324,7 @@ func TestEnsureManifestsKustomizations(t *testing.T) {
 	}
 }
 
-func TestEnsureClusterKustomizationsAddsStorageDependencies(t *testing.T) {
+func TestEnsureClusterKustomizationsUsesTemplateExternalSecretsResource(t *testing.T) {
 	dir := t.TempDir()
 	path := filepath.Join(dir, "kustomization.yaml")
 	if err := os.WriteFile(path, []byte("resources:\n"), 0644); err != nil {
@@ -301,11 +334,135 @@ func TestEnsureClusterKustomizationsAddsStorageDependencies(t *testing.T) {
 		t.Fatal(err)
 	}
 	content, err := os.ReadFile(path)
-	if err != nil || !strings.Contains(string(content), "snapshot-crds-kustomization.yaml") || !strings.Contains(string(content), "democratic-csi-kustomization.yaml") || !strings.Contains(string(content), "gateway-api-kustomization.yaml") || !strings.Contains(string(content), "gateway-kustomization.yaml") || !strings.Contains(string(content), "external-secrets-config-kustomization.yaml") || !strings.Contains(string(content), "external-dns-kustomization.yaml") || !strings.Contains(string(content), "monitoring-kustomization.yaml") || !strings.Contains(string(content), "tekton-kustomization.yaml") {
+	if err != nil || !strings.Contains(string(content), "snapshot-crds-kustomization.yaml") || !strings.Contains(string(content), "democratic-csi-kustomization.yaml") || !strings.Contains(string(content), "openbao-kustomization.yaml") || !strings.Contains(string(content), "external-secrets-kustomization.yaml") || !strings.Contains(string(content), "cnpg-kustomization.yaml") || !strings.Contains(string(content), "cloudflare-tunnel-kustomization.yaml") || !strings.Contains(string(content), "tekton-kustomization.yaml") || strings.Contains(string(content), "external-secrets-config-kustomization.yaml") {
 		t.Fatalf("cluster Kustomization was not updated: %q, %v", content, err)
 	}
 }
 
+func TestWebhookTargetTimeoutExceedsExternalSecretRefreshInterval(t *testing.T) {
+	if webhookTargetTimeout <= time.Hour {
+		t.Fatal("webhook target timeout must exceed the one-hour ExternalSecret refresh interval")
+	}
+}
+
+func TestReconcileWebhookWaitsForRefreshedTargetSecret(t *testing.T) {
+	originalInitialize := initializeOpenBao
+	originalCommand := runWebhookCommand
+	originalWebhook := ensureForgejoWebhook
+	originalTimeout := webhookTargetTimeout
+	originalInterval := webhookTargetPollInterval
+	t.Cleanup(func() {
+		initializeOpenBao = originalInitialize
+		runWebhookCommand = originalCommand
+		ensureForgejoWebhook = originalWebhook
+		webhookTargetTimeout = originalTimeout
+		webhookTargetPollInterval = originalInterval
+	})
+
+	authorization := "Bearer test-webhook-authorization"
+	staleTarget := base64.StdEncoding.EncodeToString([]byte("Bearer stale-webhook-authorization"))
+	refreshedTarget := base64.StdEncoding.EncodeToString([]byte(authorization))
+	targetChecks := 0
+	refreshedObserved := false
+	runWebhookCommand = func(_ string, name string, args ...string) ([]byte, error) {
+		if name != "kubectl" || strings.Contains(strings.Join(args, " "), authorization) {
+			t.Fatal("webhook target probe used an unexpected command")
+		}
+		if strings.Contains(strings.Join(args, " "), "secret/forgejo-webhook") {
+			targetChecks++
+			if targetChecks == 1 {
+				return []byte(staleTarget), nil
+			}
+			refreshedObserved = true
+			return []byte(refreshedTarget), nil
+		}
+		return nil, nil
+	}
+	webhookTargetPollInterval = 0
+
+	patches := 0
+	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		if !refreshedObserved {
+			t.Error("Forgejo was called before the refreshed webhook Secret was observed")
+			writer.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		switch request.Method {
+		case http.MethodGet:
+			_ = json.NewEncoder(writer).Encode([]map[string]any{{"id": 7, "url": "https://tekton.example.test/"}})
+		case http.MethodPatch:
+			var body struct {
+				AuthorizationHeader string `json:"authorization_header"`
+			}
+			if err := json.NewDecoder(request.Body).Decode(&body); err != nil || body.AuthorizationHeader != authorization {
+				t.Error("Forgejo webhook update did not use the observed authorization")
+				writer.WriteHeader(http.StatusBadRequest)
+				return
+			}
+			patches++
+			writer.WriteHeader(http.StatusOK)
+		default:
+			t.Error("unexpected Forgejo request")
+			writer.WriteHeader(http.StatusMethodNotAllowed)
+		}
+	}))
+	defer server.Close()
+	ensureForgejoWebhook = func(_ config.Config, repo, webhookURL, authorization string) error {
+		manager := forgejo.NewRepoManager(server.URL, "test-token", "owner", "user", "manifests", "flux", "main", "migration")
+		manager.HTTPClient = server.Client()
+		return manager.EnsureWebhook(repo, webhookURL, authorization)
+	}
+	initializeOpenBao = func(string, string, string, string, string, string) (map[string]map[string]string, error) {
+		return map[string]map[string]string{"cicd/forgejo-webhook": {"authorization": authorization}}, nil
+	}
+
+	cfg := config.Config{Delivery: config.DeliveryConfig{AppName: "app", WebhookHostname: "tekton.example.test", WebhookPath: "/"}}
+	if err := (Runner{Config: cfg}).reconcileWebhook(t.TempDir()); err != nil {
+		t.Fatal(err)
+	}
+	if targetChecks != 2 || patches != 1 {
+		t.Fatal("Forgejo webhook was not updated after the target Secret refreshed")
+	}
+}
+
+func TestReconcileWebhookTimeoutDoesNotUpdateForgejo(t *testing.T) {
+	originalInitialize := initializeOpenBao
+	originalCommand := runWebhookCommand
+	originalWebhook := ensureForgejoWebhook
+	originalTimeout := webhookTargetTimeout
+	originalInterval := webhookTargetPollInterval
+	t.Cleanup(func() {
+		initializeOpenBao = originalInitialize
+		runWebhookCommand = originalCommand
+		ensureForgejoWebhook = originalWebhook
+		webhookTargetTimeout = originalTimeout
+		webhookTargetPollInterval = originalInterval
+	})
+
+	authorization := "Bearer test-webhook-authorization"
+	initializeOpenBao = func(string, string, string, string, string, string) (map[string]map[string]string, error) {
+		return map[string]map[string]string{"cicd/forgejo-webhook": {"authorization": authorization}}, nil
+	}
+	runWebhookCommand = func(_ string, _ string, _ ...string) ([]byte, error) {
+		return []byte(base64.StdEncoding.EncodeToString([]byte("Bearer stale-webhook-authorization"))), nil
+	}
+	webhookTargetTimeout = -time.Nanosecond
+	webhookTargetPollInterval = 0
+	webhookUpdated := false
+	ensureForgejoWebhook = func(config.Config, string, string, string) error {
+		webhookUpdated = true
+		return nil
+	}
+
+	err := (Runner{Config: config.Config{Delivery: config.DeliveryConfig{AppName: "app"}}}).reconcileWebhook(t.TempDir())
+	if err == nil || !strings.Contains(err.Error(), "Forgejo webhook was not updated") || !strings.Contains(err.Error(), "--register-webhook") {
+		t.Fatal("webhook refresh timeout did not return a safe rerun error")
+	}
+	if webhookUpdated {
+		t.Fatal("Forgejo webhook update was attempted before the target Secret refreshed")
+	}
+}
+
 func TestRenderTerraformTFVarsIsStableAndRedactsToken(t *testing.T) {
 	cfg := config.Config{
 		ClusterID: "test",