Merge pull request 'fix: recover externally removed Talos VMs' (#29) from fix/rebuild-absent-vm into main

Reviewed-on: #29

cmd/bootstrap.go

@@ -143,13 +143,6 @@ func runBootstrap(cmd *cobra.Command, args []string) error {
 		if err := ensurePublishAppCheckoutClean(bootstrapPublishAppFrom); err != nil {
 			return err
 		}
-		origin, err := forgejo.CheckoutOrigin(bootstrapPublishAppFrom)
-		if err != nil {
-			return err
-		}
-		if config.RedactURL(origin) != cfg.Delivery.AppRepoURL {
-			return fmt.Errorf("--publish-app-from origin does not match delivery appRepoUrl")
-		}
 		branch, err := forgejo.CurrentBranch(bootstrapPublishAppFrom)
 		if err != nil {
 			return err

internal/bootstrap/bootstrap.go

@@ -40,12 +40,12 @@ const (
 )
 
 type Runner struct {
-	Config                   config.Config
-	Mode                     Mode
-	ConfirmRebuild           bool
-	RegisterWebhook          bool
-	EnableDelivery           bool
-	SkipDeliveryScaffolding  bool
+	Config                      config.Config
+	Mode                        Mode
+	ConfirmRebuild              bool
+	RegisterWebhook             bool
+	EnableDelivery              bool
+	SkipDeliveryScaffolding     bool
 	AutoMergeBootstrapMigration bool
 }
 
@@ -1406,6 +1406,10 @@ func terraformReconcileTargets(cfg config.Config) []string {
 
 func (r Runner) rebuildTalosVMs(terraformDir string, environment []string) error {
 	if err := r.importConfiguredTalosVMs(terraformDir, environment); err != nil {
+		var apiErr proxmox.APIError
+		if errors.As(err, &apiErr) && (apiErr.StatusCode == 404 || apiErr.StatusCode == 500 && apiErr.Message == `{"data":null}` && strings.HasSuffix(apiErr.Path, "/config")) {
+			return nil
+		}
 		return err
 	}
 	return destroyTalosVMs(terraformDir, environment)

internal/bootstrap/bootstrap_test.go

@@ -4,6 +4,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -17,6 +18,7 @@ import (
 	"github.com/Pingu-Studio/MaidnCLI/internal/config"
 	"github.com/Pingu-Studio/MaidnCLI/internal/forgejo"
 	"github.com/Pingu-Studio/MaidnCLI/internal/openbao"
+	"github.com/Pingu-Studio/MaidnCLI/internal/proxmox"
 	"gopkg.in/yaml.v3"
 )
 
@@ -1278,6 +1280,64 @@ func TestRebuildTerraformRetainsFullTalosVMLifecycle(t *testing.T) {
 	}
 }
 
+func TestRebuildTerraformSkipsAlreadyAbsentTalosVM(t *testing.T) {
+	originalVerify := verifyTalosVMs
+	originalRun := runTerraform
+	originalDestroy := destroyTalosVMs
+	t.Cleanup(func() {
+		verifyTalosVMs = originalVerify
+		runTerraform = originalRun
+		destroyTalosVMs = originalDestroy
+	})
+
+	cfg := config.Config{ClusterID: "test-cluster", Talos: config.TalosConfig{
+		Nodes: []config.TalosNode{{Name: "cp-01", ProxmoxNode: "pve", VMID: 100, Role: "controlplane"}},
+	}}
+	var events []string
+	for _, apiErr := range []proxmox.APIError{{StatusCode: 404}, {StatusCode: 500, Message: `{"data":null}`, Path: "/nodes/pve/qemu/100/config"}} {
+		verifyTalosVMs = func(config.Config) error {
+			return fmt.Errorf("inspect configured Talos VM: %w", apiErr)
+		}
+		destroyTalosVMs = func(string, []string) error {
+			t.Fatal("destroy ran for an already-absent VM")
+			return nil
+		}
+		runTerraform = func(_ string, _ []string, name string, args ...string) error {
+			if name != "terraform" {
+				t.Fatalf("unexpected command %q", name)
+			}
+			switch args[0] {
+			case "init", "plan", "apply":
+				events = append(events, args[0])
+			default:
+				t.Fatalf("unexpected Terraform operation %q", args[0])
+			}
+			return nil
+		}
+
+		if err := (Runner{Config: cfg, Mode: Rebuild}).reconcileTerraform(t.TempDir()); err != nil {
+			t.Fatal(err)
+		}
+		if strings.Join(events, ",") != "init,plan,apply" {
+			t.Fatalf("Terraform phase order = %q", events)
+		}
+		events = nil
+	}
+}
+
+func TestRebuildTerraformRejectsUnexpectedProxmoxError(t *testing.T) {
+	originalVerify := verifyTalosVMs
+	t.Cleanup(func() { verifyTalosVMs = originalVerify })
+	verifyTalosVMs = func(config.Config) error {
+		return fmt.Errorf("inspect configured Talos VM: %w", proxmox.APIError{StatusCode: 500, Message: "server unavailable", Path: "/nodes/pve/qemu/100/config"})
+	}
+
+	err := (Runner{Config: config.Config{Talos: config.TalosConfig{Nodes: []config.TalosNode{{Name: "cp-01", ProxmoxNode: "pve", VMID: 100}}}}, Mode: Rebuild}).rebuildTalosVMs(t.TempDir(), nil)
+	if err == nil || !strings.Contains(err.Error(), "server unavailable") {
+		t.Fatalf("unexpected Proxmox error was accepted: %v", err)
+	}
+}
+
 func TestEnsureLifecycleIdentityStoresMetadataUnderGeneratedDirectory(t *testing.T) {
 	repo := t.TempDir()
 	terraformDir := filepath.Join(repo, "terraform")

internal/forgejo/repo.go

@@ -465,9 +465,6 @@ func (rm *RepoManager) PublishDeliveryBranch(sourceDir, sourceBranch, repoURL, d
 	if err := runGit("", os.Environ(), "clone", "--no-local", "--branch", sourceBranch, sourceDir, temporary); err != nil {
 		return err
 	}
-	if err := generate(temporary); err != nil {
-		return err
-	}
 	cleanupAskPass, environment, err := rm.gitEnvironment()
 	if err != nil {
 		return err
@@ -476,6 +473,12 @@ func (rm *RepoManager) PublishDeliveryBranch(sourceDir, sourceBranch, repoURL, d
 	if err := runGit(temporary, environment, "checkout", "-B", deliveryBranch); err != nil {
 		return err
 	}
+	if err := os.RemoveAll(filepath.Join(temporary, ".tekton")); err != nil {
+		return err
+	}
+	if err := generate(temporary); err != nil {
+		return err
+	}
 	if err := runGit(temporary, environment, "add", ".tekton"); err != nil {
 		return err
 	}

internal/openbao/bootstrap.go

@@ -330,6 +330,13 @@ func refreshExternalSecrets(kubeconfig string) error {
 	if err != nil {
 		return fmt.Errorf("refresh OpenBao secret store after seed: %w", err)
 	}
+	webhook, err := kubectlOutput(kubeconfig, "get", "externalsecret", "forgejo-webhook", "-n", "tekton-pipelines", "--ignore-not-found", "-o=name")
+	if err != nil {
+		return fmt.Errorf("check Forgejo webhook ExternalSecret after OpenBao seed: %w", err)
+	}
+	if strings.TrimSpace(string(webhook)) == "" {
+		return nil
+	}
 	_, err = kubectlOutput(kubeconfig, externalSecretRefreshArgs(timestamp)...)
 	if err != nil {
 		return fmt.Errorf("refresh ExternalSecrets after OpenBao seed: %w", err)

internal/openbao/bootstrap_test.go

@@ -84,13 +84,32 @@ func TestRefreshExternalSecretsIsReadyGatedAndScoped(t *testing.T) {
 		if len(calls) == 1 {
 			return []byte("True"), nil
 		}
+		if len(calls) == 3 {
+			return []byte("externalsecret.external-secrets.io/forgejo-webhook"), nil
+		}
 		return nil, nil
 	}
-	if err := refreshExternalSecrets("kubeconfig"); err != nil || len(calls) != 3 || !strings.Contains(calls[0], "get deployment/external-secrets") || !strings.Contains(calls[1], "annotate clustersecretstore openbao") || !strings.Contains(calls[2], "annotate externalsecret forgejo-webhook") || strings.Contains(calls[1], "--all") || strings.Contains(calls[2], "--all") {
+	if err := refreshExternalSecrets("kubeconfig"); err != nil || len(calls) != 4 || !strings.Contains(calls[0], "get deployment/external-secrets") || !strings.Contains(calls[1], "annotate clustersecretstore openbao") || !strings.Contains(calls[2], "get externalsecret forgejo-webhook") || !strings.Contains(calls[3], "annotate externalsecret forgejo-webhook") || strings.Contains(calls[1], "--all") || strings.Contains(calls[3], "--all") {
 		t.Fatalf("ExternalSecret refresh was not readiness-gated and scoped: %q, %v", calls, err)
 	}
 }
 
+func TestRefreshExternalSecretsSkipsWebhookBeforeTekton(t *testing.T) {
+	original := kubectlOutput
+	t.Cleanup(func() { kubectlOutput = original })
+	var calls []string
+	kubectlOutput = func(_ string, args ...string) ([]byte, error) {
+		calls = append(calls, strings.Join(args, " "))
+		if len(calls) == 1 {
+			return []byte("True"), nil
+		}
+		return nil, nil
+	}
+	if err := refreshExternalSecrets("kubeconfig"); err != nil || len(calls) != 3 || !strings.Contains(calls[2], "--ignore-not-found") {
+		t.Fatalf("missing webhook ExternalSecret was not safely skipped: %q, %v", calls, err)
+	}
+}
+
 func TestReadRecoveryMaterialDecryptsAndValidatesBundle(t *testing.T) {
 	original := decryptRecovery
 	t.Cleanup(func() { decryptRecovery = original })