Merge pull request 'feat: refresh template revisions during reconcile' (#53) from feat/template-revision-refresh into main

Reviewed-on: #53

cmd/bootstrap.go

@@ -34,6 +34,7 @@ var bootstrapMergeBootstrapPR bool
 var bootstrapManageNetworkBridges bool
 var bootstrapEnableDelivery bool
 var bootstrapDestroyDemocraticCSIStorage bool
+var bootstrapRefreshTemplateRevisions bool
 
 var upsertOperationalSecret = bootstrap.UpsertOperationalSecret
 var readOperationalSecrets = bootstrap.ReadOperationalSecrets
@@ -68,9 +69,16 @@ func init() {
 	bootstrapCmd.Flags().BoolVar(&bootstrapManageNetworkBridges, "manage-network-bridges", false, "Persist Terraform management for existing Talos network bridges")
 	bootstrapCmd.Flags().BoolVar(&bootstrapEnableDelivery, "enable-delivery", false, "Resolve delivery defaults and reconcile the configured app delivery source")
 	bootstrapCmd.Flags().BoolVar(&bootstrapDestroyDemocraticCSIStorage, "destroy-democratic-csi-storage", false, "Delete only TrueNAS datasets under this cluster's configured Democratic CSI parent during rebuild")
+	bootstrapCmd.Flags().BoolVar(&bootstrapRefreshTemplateRevisions, "refresh-template-revisions", false, "Refresh existing template revision locks from configured branch heads during reconcile")
 }
 
 func runBootstrap(cmd *cobra.Command, args []string) error {
+	if err := validateTemplateRevisionRefresh(bootstrapRefreshTemplateRevisions, bootstrap.Mode(bootstrapMode), cmd != nil && cmd.Flags().Changed("mode")); err != nil {
+		return err
+	}
+	if bootstrapRefreshTemplateRevisions && (bootstrapProvisionAppSecretIdentities || bootstrapCreateForgejoRegistryToken || bootstrapCreateForgejoDeliveryStatusToken || bootstrapRotateWebhookAuthorization || bootstrapInitializeOpenBao || bootstrapMergeBootstrapPR) {
+		return errors.New("--refresh-template-revisions is available only for the bootstrap lifecycle")
+	}
 	var cfg config.Config
 	var err error
 	if bootstrapProvisionAppSecretIdentities {
@@ -246,10 +254,17 @@ func runBootstrap(cmd *cobra.Command, args []string) error {
 		}
 	}
 
-	runner := bootstrap.Runner{Config: cfg, Mode: bootstrap.Mode(bootstrapMode), ConfirmRebuild: bootstrapYes, RegisterWebhook: bootstrapRegisterWebhook, EnableDelivery: bootstrapEnableDelivery, DestroyDemocraticCSIStorage: bootstrapDestroyDemocraticCSIStorage}
+	runner := bootstrap.Runner{Config: cfg, Mode: bootstrap.Mode(bootstrapMode), ConfirmRebuild: bootstrapYes, RegisterWebhook: bootstrapRegisterWebhook, EnableDelivery: bootstrapEnableDelivery, DestroyDemocraticCSIStorage: bootstrapDestroyDemocraticCSIStorage, RefreshTemplateRevisions: bootstrapRefreshTemplateRevisions}
 	return runner.Run()
 }
 
+func validateTemplateRevisionRefresh(refresh bool, mode bootstrap.Mode, modeExplicit bool) error {
+	if refresh && (!modeExplicit || mode != bootstrap.Reconcile) {
+		return errors.New("--refresh-template-revisions requires --mode=reconcile")
+	}
+	return nil
+}
+
 func seedForgejoOperationalCredentials(cfg config.Config) error {
 	if err := upsertOperationalSecret(cfg.SOPS.OperationalSecretsPath, cfg.SOPS.AgeKeyPath, "cicd/forgejo", "username", cfg.Git.Username); err != nil {
 		return fmt.Errorf("save Forgejo username for webhook registration: %w", err)

cmd/bootstrap_test.go

@@ -5,9 +5,31 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/Pingu-Studio/MaidnCLI/internal/bootstrap"
 	"github.com/Pingu-Studio/MaidnCLI/internal/config"
 )
 
+func TestValidateTemplateRevisionRefreshRequiresExplicitReconcileMode(t *testing.T) {
+	tests := []struct {
+		name         string
+		mode         bootstrap.Mode
+		modeExplicit bool
+		wantErr      bool
+	}{
+		{name: "reconcile", mode: bootstrap.Reconcile, modeExplicit: true},
+		{name: "rebuild", mode: bootstrap.Rebuild, modeExplicit: true, wantErr: true},
+		{name: "missing mode", mode: bootstrap.Reconcile, wantErr: true},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			err := validateTemplateRevisionRefresh(true, test.mode, test.modeExplicit)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("validateTemplateRevisionRefresh() error = %v, wantErr %t", err, test.wantErr)
+			}
+		})
+	}
+}
+
 func TestCreateForgejoRegistryTokenRequiresConfig(t *testing.T) {
 	originalConfigPath, originalCreate := bootstrapConfigPath, bootstrapCreateForgejoRegistryToken
 	defer func() {

internal/bootstrap/bootstrap.go

@@ -49,6 +49,7 @@ type Runner struct {
 	SkipDeliveryScaffolding     bool
 	AutoMergeBootstrapMigration bool
 	DestroyDemocraticCSIStorage bool
+	RefreshTemplateRevisions    bool
 }
 
 type operationalSecrets struct {
@@ -173,6 +174,15 @@ func (r Runner) Run() error {
 		}
 		r.Config = resolvedDelivery
 	}
+	if r.Mode, err = resolveLifecycleMode(r.Mode, r.ConfirmRebuild); err != nil {
+		return err
+	}
+	if r.RefreshTemplateRevisions && r.Mode != Reconcile {
+		return errors.New("--refresh-template-revisions requires --mode=reconcile")
+	}
+	if r.DestroyDemocraticCSIStorage && r.Mode != Rebuild {
+		return errors.New("--destroy-democratic-csi-storage requires --mode=rebuild --yes")
+	}
 	if err := preflight(r.Config); err != nil {
 		return fmt.Errorf("preflight: %w", err)
 	}
@@ -187,13 +197,12 @@ func (r Runner) Run() error {
 			return errors.New("SOPS recoveryRecipient is required")
 		}
 	}
-	if r.Mode, err = resolveLifecycleMode(r.Mode, r.ConfirmRebuild); err != nil {
+	if r.RegisterWebhook {
+		if r.RefreshTemplateRevisions {
+			if err := RefreshTemplateRevisions(r.Config); err != nil {
 				return err
 			}
-	if r.DestroyDemocraticCSIStorage && r.Mode != Rebuild {
-		return errors.New("--destroy-democratic-csi-storage requires --mode=rebuild --yes")
 		}
-	if r.RegisterWebhook {
 		return r.reconcileWebhook(filepath.Join(r.Config.Git.CloneParent, r.Config.Talos.RepoDirName, r.Config.Talos.GeneratedDir))
 	}
 	if r.Config.SOPS.RecoveryIdentityPath == "" || r.Config.SOPS.RecoveryBundlePath == "" {
@@ -206,9 +215,15 @@ func (r Runner) Run() error {
 		return fmt.Errorf("preflight OpenBao recovery bundle: %w", err)
 	}
 	workspace := r.Config.WorkspaceDir
+	if r.RefreshTemplateRevisions {
+		if err := RefreshTemplateRevisions(r.Config); err != nil {
+			return err
+		}
+	} else {
 		if err := EnsureTemplateRevisions(r.Config); err != nil {
 			return err
 		}
+	}
 	catalogManager := forgejo.NewRepoManager(r.Config.Git.BaseURL, r.Config.Git.Token, r.Config.Git.Owner, r.Config.Git.Username, "", "", r.Config.Templates.TektonCatalogRepoRef, "")
 	if _, err := catalogManager.EnsureRepositoryCopy(r.Config.Flux.TektonCatalogRepo, "User-managed Tekton pipeline catalog", r.Config.Templates.TektonCatalogRepoURL); err != nil {
 		return fmt.Errorf("initialize Tekton catalog repository: %w", err)
@@ -1719,6 +1734,42 @@ func EnsureTemplateRevisions(cfg config.Config) error {
 	return ensureTemplateRevisions(cfg.WorkspaceDir, cfg)
 }
 
+// RefreshTemplateRevisions replaces an existing template lock with configured ref heads.
+func RefreshTemplateRevisions(cfg config.Config) error {
+	return refreshTemplateRevisions(cfg.WorkspaceDir, cfg)
+}
+
+func refreshTemplateRevisions(workspace string, cfg config.Config) error {
+	lockPath := filepath.Join(workspace, "maidn-template-revisions.yaml")
+	lock, err := readTemplateRevisionLock(lockPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return errors.New("template revision lock does not exist; run bootstrap without --refresh-template-revisions first")
+		}
+		return errors.New("template revision lock is invalid")
+	}
+	checkouts := []templateCheckout{
+		{Dir: filepath.Join(workspace, "maidn-cicd-cluster-template"), Repository: cfg.Templates.CICDRepoURL, Ref: cfg.Templates.CICDRepoRef},
+		{Dir: filepath.Join(workspace, "cicd-deployment-manifests-template"), Repository: cfg.Templates.ManifestsRepoURL, Ref: cfg.Templates.ManifestsRepoRef},
+		{Dir: filepath.Join(cfg.Git.CloneParent, cfg.Talos.RepoDirName), Repository: cfg.Templates.TalosRepoURL, Ref: cfg.Templates.TalosRepoRef, GeneratedDir: cfg.Talos.GeneratedDir},
+	}
+	revisions := []templateRevision{lock.CICD, lock.Manifests, lock.Talos}
+	for index, checkout := range checkouts {
+		if !sameTemplateSource(revisions[index], checkout) {
+			return errors.New("configured template source or ref differs from its workspace revision lock")
+		}
+		commit, err := checkoutTemplateRevision(cfg, checkout, "")
+		if err != nil {
+			return errors.New("configured template revision cannot be resolved; correct the template source or ref, then rerun bootstrap")
+		}
+		revisions[index].Commit = commit
+	}
+	if err := replaceTemplateRevisionLock(lockPath, templateRevisionLock{Version: 1, CICD: revisions[0], Manifests: revisions[1], Talos: revisions[2]}); err != nil {
+		return errors.New("cannot update template revision lock")
+	}
+	return nil
+}
+
 func readTemplateRevisionLock(path string) (templateRevisionLock, error) {
 	var lock templateRevisionLock
 	data, err := os.ReadFile(path)
@@ -1755,6 +1806,27 @@ func writeTemplateRevisionLock(path string, lock templateRevisionLock) error {
 	return file.Close()
 }
 
+func replaceTemplateRevisionLock(path string, lock templateRevisionLock) error {
+	data, err := yaml.Marshal(lock)
+	if err != nil {
+		return err
+	}
+	file, err := os.CreateTemp(filepath.Dir(path), ".maidn-template-revisions-")
+	if err != nil {
+		return err
+	}
+	temporaryPath := file.Name()
+	defer os.Remove(temporaryPath)
+	if _, err := file.Write(data); err != nil {
+		_ = file.Close()
+		return err
+	}
+	if err := file.Close(); err != nil {
+		return err
+	}
+	return os.Rename(temporaryPath, path)
+}
+
 func sameTemplateSource(revision templateRevision, checkout templateCheckout) bool {
 	return validTemplateRevision(revision) && revision.Repository == config.RedactURL(checkout.Repository) && revision.Ref == checkout.Ref
 }

internal/bootstrap/template_revisions_test.go

@@ -40,20 +40,61 @@ func TestEnsureTemplateRevisionsReusesLockedCommitAfterBranchDrift(t *testing.T)
 	if err := ensureTemplateRevisions(workspace, cfg); err != nil {
 		t.Fatal(err)
 	}
-	locked := git.commits["cicd"]
-	git.commits["cicd"] = strings.Repeat("d", 40)
+	locked := map[string]string{}
+	for name, commit := range git.commits {
+		locked[name] = commit
+	}
+	git.commits = map[string]string{"cicd": strings.Repeat("d", 40), "manifests": strings.Repeat("e", 40), "talos": strings.Repeat("f", 40)}
 	git.resetCalls()
 	if err := ensureTemplateRevisions(workspace, cfg); err != nil {
 		t.Fatal(err)
 	}
-	if git.fetched["cicd"] != locked || git.checkedOut["cicd"] != locked {
-		t.Fatalf("branch drift changed locked CICD revision: fetched %q, checked out %q", git.fetched["cicd"], git.checkedOut["cicd"])
+	lock, err := readTemplateRevisionLock(filepath.Join(workspace, "maidn-template-revisions.yaml"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for name, commit := range locked {
+		if git.fetched[name] != commit || git.checkedOut[name] != commit {
+			t.Fatalf("branch drift changed locked %s revision: fetched %q, checked out %q", name, git.fetched[name], git.checkedOut[name])
+		}
+	}
+	if lock.CICD.Commit != locked["cicd"] || lock.Manifests.Commit != locked["manifests"] || lock.Talos.Commit != locked["talos"] {
+		t.Fatalf("normal bootstrap rewrote template lock: %#v", lock)
 	}
 	if strings.Contains(git.commands(), "fetch origin main") {
 		t.Fatal("later run fetched a mutable branch instead of the lock commit")
 	}
 }
 
+func TestRefreshTemplateRevisionsUpdatesExistingLockToConfiguredHeads(t *testing.T) {
+	workspace, cfg, git := templateRevisionTestConfig(t)
+	useTemplateRevisionGit(t, git)
+	if err := ensureTemplateRevisions(workspace, cfg); err != nil {
+		t.Fatal(err)
+	}
+	git.commits = map[string]string{"cicd": strings.Repeat("d", 40), "manifests": strings.Repeat("e", 40), "talos": strings.Repeat("f", 40)}
+	git.resetCalls()
+
+	if err := refreshTemplateRevisions(workspace, cfg); err != nil {
+		t.Fatal(err)
+	}
+	lock, err := readTemplateRevisionLock(filepath.Join(workspace, "maidn-template-revisions.yaml"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for name, commit := range git.commits {
+		if git.fetched[name] != commit || git.checkedOut[name] != commit {
+			t.Fatalf("refresh did not use configured %s branch head: fetched %q, checked out %q", name, git.fetched[name], git.checkedOut[name])
+		}
+	}
+	if lock.CICD.Commit != git.commits["cicd"] || lock.Manifests.Commit != git.commits["manifests"] || lock.Talos.Commit != git.commits["talos"] {
+		t.Fatalf("refresh did not update template lock: %#v", lock)
+	}
+	if !strings.Contains(git.commands(), "fetch origin main") || strings.Contains(git.commands(), "template-password") {
+		t.Fatal("refresh did not safely fetch configured branch heads")
+	}
+}
+
 func TestEnsureTemplateRevisionsRejectsChangedRefWithoutGit(t *testing.T) {
 	workspace, cfg, git := templateRevisionTestConfig(t)
 	useTemplateRevisionGit(t, git)