fix: wait for webhook secret refresh
This commit is contained in:
parent
979a950d33
commit
676e89de40
|
|
@ -45,6 +45,21 @@ type operationalSecrets struct {
|
|||
|
||||
var initializeOpenBao = openbao.Initialize
|
||||
|
||||
var ensureForgejoWebhook = func(cfg config.Config, repo, webhookURL, authorization string) error {
|
||||
manager := forgejo.NewRepoManager(cfg.Git.BaseURL, cfg.Git.Token, cfg.Git.Owner, cfg.Git.Username, cfg.Flux.ManifestsRepo, cfg.Flux.RepoName, cfg.Flux.Branch, "maidn/bootstrap-"+cfg.ClusterID)
|
||||
return manager.EnsureWebhook(repo, webhookURL, authorization)
|
||||
}
|
||||
|
||||
var runWebhookCommand = utils.RunCommandQuietOutputInDir
|
||||
|
||||
var webhookTargetTimeout = 70 * time.Minute
|
||||
|
||||
var webhookTargetPollInterval = 2 * time.Second
|
||||
|
||||
var templateBaseComponents = []string{"snapshot-crds", "democratic-csi", "cert-manager", "cluster-issuers", "gateway-api", "gateway", "monitoring", "openbao", "external-secrets", "external-dns", "tekton", "tekton-triggers"}
|
||||
|
||||
var requiredClusterKustomizations = []string{"snapshot-crds-kustomization.yaml", "democratic-csi-kustomization.yaml", "cert-manager-kustomization.yaml", "cluster-issuers-kustomization.yaml", "gateway-api-kustomization.yaml", "cilium-kustomization.yaml", "cilium-config-kustomization.yaml", "openbao-kustomization.yaml", "external-secrets-kustomization.yaml", "cnpg-kustomization.yaml", "gateway-kustomization.yaml", "external-dns-kustomization.yaml", "cloudflare-tunnel-kustomization.yaml", "monitoring-kustomization.yaml", "tekton-kustomization.yaml", "tekton-triggers-kustomization.yaml", "cicd-manifests-repo.yaml"}
|
||||
|
||||
func (r Runner) Run() error {
|
||||
resolved, err := config.Resolve(r.Config)
|
||||
if err != nil {
|
||||
|
|
@ -112,11 +127,9 @@ func (r Runner) Run() error {
|
|||
if err := copyDir(filepath.Join(cicdTemplateDir, "base"), filepath.Join(dir, "base"), false); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, component := range []string{"snapshot-crds", "democratic-csi", "cert-manager", "cluster-issuers", "gateway-api", "gateway", "monitoring", "openbao", "external-secrets", "external-secrets-config", "external-dns", "tekton", "tekton-triggers"} {
|
||||
if err := copyDir(filepath.Join(cicdTemplateDir, "base", component), filepath.Join(dir, "base", component), true); err != nil {
|
||||
if err := copyTemplateBaseComponents(cicdTemplateDir, dir); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := copyClusterTemplate(filepath.Join(cicdTemplateDir, "clusters", "template"), clusterDir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -227,7 +240,7 @@ func (r Runner) Run() error {
|
|||
}
|
||||
|
||||
func (r Runner) reconcileWebhook(generatedDir string) error {
|
||||
operationalSecrets, err := openbao.Initialize(filepath.Join(generatedDir, "kubeconfig"), r.Config.SOPS.RecoveryRecipient, r.Config.SOPS.RecoveryIdentityPath, r.Config.SOPS.RecoveryBundlePath, r.Config.SOPS.AgeKeyPath, r.Config.SOPS.OperationalSecretsPath)
|
||||
operationalSecrets, err := initializeOpenBao(filepath.Join(generatedDir, "kubeconfig"), r.Config.SOPS.RecoveryRecipient, r.Config.SOPS.RecoveryIdentityPath, r.Config.SOPS.RecoveryBundlePath, r.Config.SOPS.AgeKeyPath, r.Config.SOPS.OperationalSecretsPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("initialize OpenBao: %w", err)
|
||||
}
|
||||
|
|
@ -235,11 +248,10 @@ func (r Runner) reconcileWebhook(generatedDir string) error {
|
|||
if authorization == "" {
|
||||
return errors.New("operational SOPS secrets requires cicd/forgejo-webhook.authorization")
|
||||
}
|
||||
if err := waitForWebhookTargets(generatedDir, r.Config); err != nil {
|
||||
if err := waitForWebhookTargets(generatedDir, r.Config, authorization); err != nil {
|
||||
return err
|
||||
}
|
||||
manager := forgejo.NewRepoManager(r.Config.Git.BaseURL, r.Config.Git.Token, r.Config.Git.Owner, r.Config.Git.Username, r.Config.Flux.ManifestsRepo, r.Config.Flux.RepoName, r.Config.Flux.Branch, "maidn/bootstrap-"+r.Config.ClusterID)
|
||||
if err := manager.EnsureWebhook(r.Config.Delivery.AppName, r.Config.Delivery.WebhookURL(), authorization); err != nil {
|
||||
if err := ensureForgejoWebhook(r.Config, r.Config.Delivery.AppName, r.Config.Delivery.WebhookURL(), authorization); err != nil {
|
||||
return fmt.Errorf("reconcile Forgejo webhook: %w", err)
|
||||
}
|
||||
return nil
|
||||
|
|
@ -464,27 +476,43 @@ func renderDemocraticCSISecret(csi config.DemocraticCSIConfig) ([]byte, error) {
|
|||
})
|
||||
}
|
||||
|
||||
func waitForWebhookTargets(dir string, cfg config.Config) error {
|
||||
resources := []string{
|
||||
"secret/forgejo-webhook",
|
||||
"deployment/el-" + cfg.Delivery.AppName,
|
||||
"pipeline/" + cfg.Delivery.AppName,
|
||||
func waitForWebhookTargets(dir string, cfg config.Config, authorization string) error {
|
||||
if err := waitForWebhookAuthorization(dir, authorization); err != nil {
|
||||
return err
|
||||
}
|
||||
resources := []string{"deployment/el-" + cfg.Delivery.AppName, "pipeline/" + cfg.Delivery.AppName}
|
||||
for _, resource := range resources {
|
||||
deadline := time.Now().Add(10 * time.Minute)
|
||||
deadline := time.Now().Add(webhookTargetTimeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, err := utils.RunCommandQuietOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err == nil {
|
||||
if _, err := runWebhookCommand(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
time.Sleep(webhookTargetPollInterval)
|
||||
}
|
||||
if _, err := utils.RunCommandQuietOutputInDir(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err != nil {
|
||||
if _, err := runWebhookCommand(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", resource); err != nil {
|
||||
return fmt.Errorf("wait for %s before registering Forgejo webhook", resource)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func waitForWebhookAuthorization(dir, authorization string) error {
|
||||
deadline := time.Now().Add(webhookTargetTimeout)
|
||||
for {
|
||||
output, err := runWebhookCommand(dir, "kubectl", "--kubeconfig=kubeconfig", "-n", "tekton-pipelines", "get", "secret/forgejo-webhook", "-o=jsonpath={.data.authorization}")
|
||||
if err == nil {
|
||||
observed, decodeErr := base64.StdEncoding.DecodeString(strings.TrimSpace(string(output)))
|
||||
if decodeErr == nil && string(observed) == authorization {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if !time.Now().Before(deadline) {
|
||||
return errors.New("ExternalSecret target Secret forgejo-webhook did not refresh within the timeout; Forgejo webhook was not updated. Wait for External Secrets to recover, then safely rerun cicd-tool bootstrap --config <config> --register-webhook")
|
||||
}
|
||||
time.Sleep(webhookTargetPollInterval)
|
||||
}
|
||||
}
|
||||
|
||||
func ensureClusterKustomizations(clusterDir string) error {
|
||||
path := filepath.Join(clusterDir, "kustomization.yaml")
|
||||
content, err := os.ReadFile(path)
|
||||
|
|
@ -498,7 +526,7 @@ func ensureClusterKustomizations(clusterDir string) error {
|
|||
}
|
||||
updated := string(content)
|
||||
updated = strings.ReplaceAll(updated, " - bootstrap-secrets.sops.yaml\n", "")
|
||||
for _, resource := range []string{"snapshot-crds-kustomization.yaml", "democratic-csi-kustomization.yaml", "cert-manager-kustomization.yaml", "cluster-issuers-kustomization.yaml", "gateway-api-kustomization.yaml", "gateway-kustomization.yaml", "cilium-kustomization.yaml", "cilium-config-kustomization.yaml", "cicd-manifests-repo.yaml", "external-secrets-config-kustomization.yaml", "external-dns-kustomization.yaml", "monitoring-kustomization.yaml", "tekton-kustomization.yaml", "tekton-triggers-kustomization.yaml"} {
|
||||
for _, resource := range requiredClusterKustomizations {
|
||||
if !strings.Contains(updated, resource) {
|
||||
updated += " - " + resource + "\n"
|
||||
}
|
||||
|
|
@ -509,6 +537,15 @@ func ensureClusterKustomizations(clusterDir string) error {
|
|||
return os.WriteFile(path, []byte(updated), 0644)
|
||||
}
|
||||
|
||||
func copyTemplateBaseComponents(templateDir, repoDir string) error {
|
||||
for _, component := range templateBaseComponents {
|
||||
if err := copyDir(filepath.Join(templateDir, "base", component), filepath.Join(repoDir, "base", component), true); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyClusterTemplate(source, destination string) error {
|
||||
entries, err := os.ReadDir(destination)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
|
|
|
|||
|
|
@ -4,13 +4,17 @@ import (
|
|||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Pingu-Studio/MaidnCLI/internal/config"
|
||||
"github.com/Pingu-Studio/MaidnCLI/internal/forgejo"
|
||||
)
|
||||
|
||||
func TestRenderCiliumConfig(t *testing.T) {
|
||||
|
|
@ -101,6 +105,35 @@ func TestCopyAndRenderDeliveryBasesOverwritesExistingMigrationOutput(t *testing.
|
|||
}
|
||||
}
|
||||
|
||||
func TestCopyTemplateBaseComponentsCopiesExternalSecretsConfig(t *testing.T) {
|
||||
templateDir := t.TempDir()
|
||||
repoDir := t.TempDir()
|
||||
for _, component := range templateBaseComponents {
|
||||
if err := os.MkdirAll(filepath.Join(templateDir, "base", component), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
source := filepath.Join(templateDir, "base", "external-secrets", "config.yaml")
|
||||
destination := filepath.Join(repoDir, "base", "external-secrets", "config.yaml")
|
||||
if err := os.WriteFile(source, []byte("store: openbao\n"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(destination), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(destination, []byte("store: stale\n"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := copyTemplateBaseComponents(templateDir, repoDir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
content, err := os.ReadFile(destination)
|
||||
if err != nil || string(content) != "store: openbao\n" {
|
||||
t.Fatalf("external-secrets configuration was not copied: %q, %v", content, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderDemocraticCSISecret(t *testing.T) {
|
||||
secret, err := renderDemocraticCSISecret(config.DemocraticCSIConfig{TrueNASAPIKey: "api-key", TrueNASHost: "truenas.example.test", TargetPortal: "truenas.example.test:3260", ShareHost: "truenas.example.test", DatasetParentNFS: "pool/kubernetes/nfs/v", DatasetSnapshotsNFS: "pool/kubernetes/nfs/s", AllowedNetworks: "192.168.45.0/24", NameSuffix: "-test", PortalGroup: "1", InitiatorGroup: "1"})
|
||||
if err != nil {
|
||||
|
|
@ -291,7 +324,7 @@ func TestEnsureManifestsKustomizations(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestEnsureClusterKustomizationsAddsStorageDependencies(t *testing.T) {
|
||||
func TestEnsureClusterKustomizationsUsesTemplateExternalSecretsResource(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "kustomization.yaml")
|
||||
if err := os.WriteFile(path, []byte("resources:\n"), 0644); err != nil {
|
||||
|
|
@ -301,11 +334,135 @@ func TestEnsureClusterKustomizationsAddsStorageDependencies(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil || !strings.Contains(string(content), "snapshot-crds-kustomization.yaml") || !strings.Contains(string(content), "democratic-csi-kustomization.yaml") || !strings.Contains(string(content), "gateway-api-kustomization.yaml") || !strings.Contains(string(content), "gateway-kustomization.yaml") || !strings.Contains(string(content), "external-secrets-config-kustomization.yaml") || !strings.Contains(string(content), "external-dns-kustomization.yaml") || !strings.Contains(string(content), "monitoring-kustomization.yaml") || !strings.Contains(string(content), "tekton-kustomization.yaml") {
|
||||
if err != nil || !strings.Contains(string(content), "snapshot-crds-kustomization.yaml") || !strings.Contains(string(content), "democratic-csi-kustomization.yaml") || !strings.Contains(string(content), "openbao-kustomization.yaml") || !strings.Contains(string(content), "external-secrets-kustomization.yaml") || !strings.Contains(string(content), "cnpg-kustomization.yaml") || !strings.Contains(string(content), "cloudflare-tunnel-kustomization.yaml") || !strings.Contains(string(content), "tekton-kustomization.yaml") || strings.Contains(string(content), "external-secrets-config-kustomization.yaml") {
|
||||
t.Fatalf("cluster Kustomization was not updated: %q, %v", content, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebhookTargetTimeoutExceedsExternalSecretRefreshInterval(t *testing.T) {
|
||||
if webhookTargetTimeout <= time.Hour {
|
||||
t.Fatal("webhook target timeout must exceed the one-hour ExternalSecret refresh interval")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileWebhookWaitsForRefreshedTargetSecret(t *testing.T) {
|
||||
originalInitialize := initializeOpenBao
|
||||
originalCommand := runWebhookCommand
|
||||
originalWebhook := ensureForgejoWebhook
|
||||
originalTimeout := webhookTargetTimeout
|
||||
originalInterval := webhookTargetPollInterval
|
||||
t.Cleanup(func() {
|
||||
initializeOpenBao = originalInitialize
|
||||
runWebhookCommand = originalCommand
|
||||
ensureForgejoWebhook = originalWebhook
|
||||
webhookTargetTimeout = originalTimeout
|
||||
webhookTargetPollInterval = originalInterval
|
||||
})
|
||||
|
||||
authorization := "Bearer test-webhook-authorization"
|
||||
staleTarget := base64.StdEncoding.EncodeToString([]byte("Bearer stale-webhook-authorization"))
|
||||
refreshedTarget := base64.StdEncoding.EncodeToString([]byte(authorization))
|
||||
targetChecks := 0
|
||||
refreshedObserved := false
|
||||
runWebhookCommand = func(_ string, name string, args ...string) ([]byte, error) {
|
||||
if name != "kubectl" || strings.Contains(strings.Join(args, " "), authorization) {
|
||||
t.Fatal("webhook target probe used an unexpected command")
|
||||
}
|
||||
if strings.Contains(strings.Join(args, " "), "secret/forgejo-webhook") {
|
||||
targetChecks++
|
||||
if targetChecks == 1 {
|
||||
return []byte(staleTarget), nil
|
||||
}
|
||||
refreshedObserved = true
|
||||
return []byte(refreshedTarget), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
webhookTargetPollInterval = 0
|
||||
|
||||
patches := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if !refreshedObserved {
|
||||
t.Error("Forgejo was called before the refreshed webhook Secret was observed")
|
||||
writer.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
switch request.Method {
|
||||
case http.MethodGet:
|
||||
_ = json.NewEncoder(writer).Encode([]map[string]any{{"id": 7, "url": "https://tekton.example.test/"}})
|
||||
case http.MethodPatch:
|
||||
var body struct {
|
||||
AuthorizationHeader string `json:"authorization_header"`
|
||||
}
|
||||
if err := json.NewDecoder(request.Body).Decode(&body); err != nil || body.AuthorizationHeader != authorization {
|
||||
t.Error("Forgejo webhook update did not use the observed authorization")
|
||||
writer.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
patches++
|
||||
writer.WriteHeader(http.StatusOK)
|
||||
default:
|
||||
t.Error("unexpected Forgejo request")
|
||||
writer.WriteHeader(http.StatusMethodNotAllowed)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
ensureForgejoWebhook = func(_ config.Config, repo, webhookURL, authorization string) error {
|
||||
manager := forgejo.NewRepoManager(server.URL, "test-token", "owner", "user", "manifests", "flux", "main", "migration")
|
||||
manager.HTTPClient = server.Client()
|
||||
return manager.EnsureWebhook(repo, webhookURL, authorization)
|
||||
}
|
||||
initializeOpenBao = func(string, string, string, string, string, string) (map[string]map[string]string, error) {
|
||||
return map[string]map[string]string{"cicd/forgejo-webhook": {"authorization": authorization}}, nil
|
||||
}
|
||||
|
||||
cfg := config.Config{Delivery: config.DeliveryConfig{AppName: "app", WebhookHostname: "tekton.example.test", WebhookPath: "/"}}
|
||||
if err := (Runner{Config: cfg}).reconcileWebhook(t.TempDir()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if targetChecks != 2 || patches != 1 {
|
||||
t.Fatal("Forgejo webhook was not updated after the target Secret refreshed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileWebhookTimeoutDoesNotUpdateForgejo(t *testing.T) {
|
||||
originalInitialize := initializeOpenBao
|
||||
originalCommand := runWebhookCommand
|
||||
originalWebhook := ensureForgejoWebhook
|
||||
originalTimeout := webhookTargetTimeout
|
||||
originalInterval := webhookTargetPollInterval
|
||||
t.Cleanup(func() {
|
||||
initializeOpenBao = originalInitialize
|
||||
runWebhookCommand = originalCommand
|
||||
ensureForgejoWebhook = originalWebhook
|
||||
webhookTargetTimeout = originalTimeout
|
||||
webhookTargetPollInterval = originalInterval
|
||||
})
|
||||
|
||||
authorization := "Bearer test-webhook-authorization"
|
||||
initializeOpenBao = func(string, string, string, string, string, string) (map[string]map[string]string, error) {
|
||||
return map[string]map[string]string{"cicd/forgejo-webhook": {"authorization": authorization}}, nil
|
||||
}
|
||||
runWebhookCommand = func(_ string, _ string, _ ...string) ([]byte, error) {
|
||||
return []byte(base64.StdEncoding.EncodeToString([]byte("Bearer stale-webhook-authorization"))), nil
|
||||
}
|
||||
webhookTargetTimeout = -time.Nanosecond
|
||||
webhookTargetPollInterval = 0
|
||||
webhookUpdated := false
|
||||
ensureForgejoWebhook = func(config.Config, string, string, string) error {
|
||||
webhookUpdated = true
|
||||
return nil
|
||||
}
|
||||
|
||||
err := (Runner{Config: config.Config{Delivery: config.DeliveryConfig{AppName: "app"}}}).reconcileWebhook(t.TempDir())
|
||||
if err == nil || !strings.Contains(err.Error(), "Forgejo webhook was not updated") || !strings.Contains(err.Error(), "--register-webhook") {
|
||||
t.Fatal("webhook refresh timeout did not return a safe rerun error")
|
||||
}
|
||||
if webhookUpdated {
|
||||
t.Fatal("Forgejo webhook update was attempted before the target Secret refreshed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderTerraformTFVarsIsStableAndRedactsToken(t *testing.T) {
|
||||
cfg := config.Config{
|
||||
ClusterID: "test",
|
||||
|
|
|
|||
Loading…
Reference in a new issue