diff --git a/README.md b/README.md index 02a6906..be2ee5d 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ dlv version - `cicd-tool bootstrap` runs a shorter Forgejo-first wizard, asks for a Forgejo PAT, asks where local repos should be cloned, discovers Proxmox nodes/storage/networks, retries without losing entered answers when discovery fails, shows the latest Talos version, derives the standardized Talos factory URL, schematic, and required extensions automatically from the chosen version, writes `terraform.tfvars`, stages Talos images on Proxmox, and can execute Terraform, Talos bootstrap, and Flux bootstrap - `cicd-tool bootstrap --config maidn-bootstrap.yaml` skips the wizard and uses the saved config - `maidn bootstrap init --config --organization --create-organization --enable-delivery` locks an isolated workspace, initializes the Forgejo repositories, then runs the non-destructive bootstrap reconcile lifecycle; use `--mode=rebuild --yes` for an authorized rebuild. Delivery scaffolding requires `--enable-delivery`. -- `maidn app onboard --config --from ` validates a clean configured checkout and adds its `.tekton` delivery contract. +- `maidn app onboard --config --from ` opens reviewed source-delivery and cluster-registration PRs; see `docs/operations.md`. - `cicd-tool e2e` runs bounded, read-only Flux, ExternalSecret, PipelineRun, preview, and promotion-PR checks with JSON output. See `docs/e2e.md`. See `docs/operations.md` for the authorized operating and verification runbook. diff --git a/cmd/fresh.go b/cmd/fresh.go index 1494fe0..04ef9f0 100644 --- a/cmd/fresh.go +++ b/cmd/fresh.go @@ -9,7 +9,7 @@ import ( ) var freshConfigPath, freshOrganization, onboardConfigPath, onboardFrom string -var onboardAppName, onboardAppRepoURL, onboardImageRepository, onboardBuildStrategy string +var onboardAppName, onboardAppRepoURL, onboardImageRepository, onboardBuildStrategy, onboardBuildOutputDirectory, onboardBuildConfiguration string var freshCreateOrganization, freshEnableDelivery, freshYes bool var freshMode string @@ -55,6 +55,8 @@ func init() { appOnboardCmd.Flags().StringVar(&onboardAppRepoURL, "app-repo-url", "", "Application repository URL override") appOnboardCmd.Flags().StringVar(&onboardImageRepository, "image-repository", "", "OCI image repository override") appOnboardCmd.Flags().StringVar(&onboardBuildStrategy, "build-strategy", "", "Build strategy override: static or runtime") + appOnboardCmd.Flags().StringVar(&onboardBuildOutputDirectory, "build-output-directory", "", "Static build output directory override") + appOnboardCmd.Flags().StringVar(&onboardBuildConfiguration, "build-configuration", "", "Static build configuration override") _ = appOnboardCmd.MarkFlagRequired("config") _ = appOnboardCmd.MarkFlagRequired("from") } @@ -91,6 +93,12 @@ func runAppOnboard(_ *cobra.Command, _ []string) error { if onboardBuildStrategy != "" { cfg.Delivery.BuildStrategy = onboardBuildStrategy } + if onboardBuildOutputDirectory != "" { + cfg.Delivery.BuildOutputDirectory = onboardBuildOutputDirectory + } + if onboardBuildConfiguration != "" { + cfg.Delivery.BuildConfiguration = onboardBuildConfiguration + } cfg, err = resolveAppOnboarding(cfg) if err != nil { return err diff --git a/cmd/fresh_test.go b/cmd/fresh_test.go index 7891895..76004af 100644 --- a/cmd/fresh_test.go +++ b/cmd/fresh_test.go @@ -65,11 +65,11 @@ func TestBootstrapInitAppliesFluxDefaultsBeforeFreshValidation(t *testing.T) { func TestAppOnboardValidatesConfigBeforeExternalWork(t *testing.T) { originalConfig, originalResolve, originalOnboard := loadAppOnboardConfig, resolveAppOnboarding, onboardApp originalConfigPath, originalFrom := onboardConfigPath, onboardFrom - originalName, originalRepo, originalImage, originalBuildStrategy := onboardAppName, onboardAppRepoURL, onboardImageRepository, onboardBuildStrategy + originalName, originalRepo, originalImage, originalBuildStrategy, originalBuildOutput, originalBuildConfiguration := onboardAppName, onboardAppRepoURL, onboardImageRepository, onboardBuildStrategy, onboardBuildOutputDirectory, onboardBuildConfiguration t.Cleanup(func() { loadAppOnboardConfig, resolveAppOnboarding, onboardApp = originalConfig, originalResolve, originalOnboard onboardConfigPath, onboardFrom = originalConfigPath, originalFrom - onboardAppName, onboardAppRepoURL, onboardImageRepository, onboardBuildStrategy = originalName, originalRepo, originalImage, originalBuildStrategy + onboardAppName, onboardAppRepoURL, onboardImageRepository, onboardBuildStrategy, onboardBuildOutputDirectory, onboardBuildConfiguration = originalName, originalRepo, originalImage, originalBuildStrategy, originalBuildOutput, originalBuildConfiguration }) loadAppOnboardConfig = func(string) (config.Config, error) { return config.Config{}, nil } resolveAppOnboarding = func(config.Config) (config.Config, error) { return config.Config{}, errors.New("incomplete delivery") } @@ -78,7 +78,7 @@ func TestAppOnboardValidatesConfigBeforeExternalWork(t *testing.T) { return nil } onboardConfigPath, onboardFrom = "private.yaml", "app-checkout" - onboardAppName, onboardAppRepoURL, onboardImageRepository, onboardBuildStrategy = "", "", "", "" + onboardAppName, onboardAppRepoURL, onboardImageRepository, onboardBuildStrategy, onboardBuildOutputDirectory, onboardBuildConfiguration = "", "", "", "", "", "" if err := runAppOnboard(nil, nil); err == nil { t.Fatal("onboarding accepted invalid configuration") } @@ -87,25 +87,25 @@ func TestAppOnboardValidatesConfigBeforeExternalWork(t *testing.T) { func TestAppOnboardPassesOnlyValidatedConfigAndCheckout(t *testing.T) { originalConfig, originalResolve, originalOnboard := loadAppOnboardConfig, resolveAppOnboarding, onboardApp originalConfigPath, originalFrom := onboardConfigPath, onboardFrom - originalName, originalRepo, originalImage, originalBuildStrategy := onboardAppName, onboardAppRepoURL, onboardImageRepository, onboardBuildStrategy + originalName, originalRepo, originalImage, originalBuildStrategy, originalBuildOutput, originalBuildConfiguration := onboardAppName, onboardAppRepoURL, onboardImageRepository, onboardBuildStrategy, onboardBuildOutputDirectory, onboardBuildConfiguration t.Cleanup(func() { loadAppOnboardConfig, resolveAppOnboarding, onboardApp = originalConfig, originalResolve, originalOnboard onboardConfigPath, onboardFrom = originalConfigPath, originalFrom - onboardAppName, onboardAppRepoURL, onboardImageRepository, onboardBuildStrategy = originalName, originalRepo, originalImage, originalBuildStrategy + onboardAppName, onboardAppRepoURL, onboardImageRepository, onboardBuildStrategy, onboardBuildOutputDirectory, onboardBuildConfiguration = originalName, originalRepo, originalImage, originalBuildStrategy, originalBuildOutput, originalBuildConfiguration }) cfg := config.Config{Delivery: config.DeliveryConfig{AppRepoURL: "https://git.example.test/new-org/app.git", AppRepoRef: "main"}} loadAppOnboardConfig = func(string) (config.Config, error) { return cfg, nil } resolveAppOnboarding = func(got config.Config) (config.Config, error) { return got, nil } calls := 0 onboardApp = func(got config.Config, checkout string) error { - if checkout != "app-checkout" || got.Delivery.AppName != "fixture" || got.Delivery.AppRepoURL != "https://git.example.test/new-org/fixture.git" || got.Delivery.ImageRepository != "registry.example.test/new-org/fixture" || got.Delivery.BuildStrategy != "runtime" { + if checkout != "app-checkout" || got.Delivery.AppName != "fixture" || got.Delivery.AppRepoURL != "https://git.example.test/new-org/fixture.git" || got.Delivery.ImageRepository != "registry.example.test/new-org/fixture" || got.Delivery.BuildStrategy != "runtime" || got.Delivery.BuildOutputDirectory != "dist/fixture" || got.Delivery.BuildConfiguration != "ci" { t.Fatal("onboarding used the wrong checkout or config") } calls++ return nil } onboardConfigPath, onboardFrom = "private.yaml", "app-checkout" - onboardAppName, onboardAppRepoURL, onboardImageRepository, onboardBuildStrategy = "fixture", "https://git.example.test/new-org/fixture.git", "registry.example.test/new-org/fixture", "runtime" + onboardAppName, onboardAppRepoURL, onboardImageRepository, onboardBuildStrategy, onboardBuildOutputDirectory, onboardBuildConfiguration = "fixture", "https://git.example.test/new-org/fixture.git", "registry.example.test/new-org/fixture", "runtime", "dist/fixture", "ci" if err := runAppOnboard(nil, nil); err != nil || calls != 1 { t.Fatalf("runAppOnboard() = %v, calls = %d", err, calls) } diff --git a/docs/operations.md b/docs/operations.md index 9cb31c0..eef58cf 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -23,31 +23,77 @@ new template revisions intentionally, use a new empty `workspaceDir` (and a fresh `cloneParent` when it is configured separately) and keep the prior secret-bearing workspace intact for recovery. -## Platform Initialization And App Onboarding +## Standard Delivery Workflow -Initialize the shared delivery platform before onboarding any application. This +Open and merge reviewed platform-source PRs before reconciling. When a merged +template source must replace an existing revision lock, run the refresh by +itself. It creates migration PRs; review and merge them, then rerun normal +reconciliation: + +```powershell +go run . bootstrap --config --mode=reconcile --refresh-template-revisions +go run . bootstrap --config --mode=reconcile +``` + +Initialize the shared delivery platform before onboarding an application. This creates the Gateway, Tekton, and Tekton Triggers platform resources using -`tekton.` and the configured Forgejo owner; it does not render -an application Pipeline or register an application webhook: +`tekton.` and the configured Forgejo owner: ```powershell -bootstrap init --config --organization --create-organization +go run . bootstrap init --config --organization --create-organization ``` -Import a clean checkout into the configured owner with a separate command. The -checkout may originate in another Forgejo organization, but `delivery.appRepoUrl` -must target `/.git` and its current branch must be -`delivery.appRepoRef`: +Use a clean checkout on `delivery.appRepoRef`. The app repository URL must be +the canonical source owner, such as `Maidn/.git`; `test-org-2` is only the +execution owner. Set per-app static build values with command-line overrides, +not by rewriting the private default: ```powershell -app onboard --config --from +go run . app onboard --config --from ` + --app-name --app-repo-url https:///Maidn/.git ` + --image-repository // --build-strategy static ` + --build-output-directory --build-configuration ``` -Onboarding copies the current branch to the target repository, replaces only -the two Maidn-generated `.tekton` files, protects the production branch, and -auto-merges the app delivery and cluster registration PRs. The registration is -stored in `base/tekton/apps/.yaml` in the configured cluster repository; -the generic EventListener dispatches by the Forgejo repository name. +For a runtime build, set `--build-strategy runtime`; static-only output options +remain harmless. Onboarding opens, but never merges, a source delivery PR and a +cluster registration PR. It registers the Forgejo hook but does not emit a test +delivery. Review and merge the source delivery PR first, then the cluster +registration PR. The registration is stored in +`base/tekton/apps/.yaml`; Flux loads the source-owned `.tekton` path and +the generic EventListener dispatches by Forgejo repository name. + +Existing registrations are never overwritten. Move a legacy source registration +to its canonical source in a separate reviewed cluster-repository PR before +onboarding that application. + +After Flux reports the app Kustomization Ready, use Forgejo's hook test endpoint +against a non-`main` ref and inspect the resulting PipelineRun. The command and +read-only checks are in [Webhook Smoke Test](#webhook-smoke-test). + +### Shared Services And Add-ons + +Environment databases are platform-owned shared services. Staging workloads use +the CNPG-generated `staging-postgres-app` Secret and production workloads use +`production-postgres-app`; applications must not declare their own CNPG Cluster +by default. Shared credentials are appropriate only for the shared environment +database. Use a dedicated service only when isolation, lifecycle, or storage +requirements demand it. + +An application can carry reviewed dedicated resources in +`.maidn/kustomization.yaml`. Onboarding registers that path as the app's +`-addons` Flux Kustomization without changing the generated `.tekton` +files. Runtime secret grants also generate the app's `.maidn/secret-access.yaml` +there. Add-ons must declare their namespace explicitly and contain references, +never credential values. Preview namespaces do not receive staging or production +runtime credentials; preview-safe configuration is the application chart's +responsibility. + +Declare runtime secret access in the private configuration and run normal +bootstrap reconciliation to create its policy and role. Set values only with +`app secret set` using stdin, `--file`, or `--generate`; provision the scoped +E2E identity with `bootstrap --provision-app-secret-identities --e2e-app ` +only when a probe needs it. See [secret-grants.md](secret-grants.md). `delivery.productionBranch` is the application production branch. A production delivery PR targets the configured generic diff --git a/internal/bootstrap/onboard.go b/internal/bootstrap/onboard.go index 02d1123..9db629a 100644 --- a/internal/bootstrap/onboard.go +++ b/internal/bootstrap/onboard.go @@ -22,15 +22,13 @@ type onboardingRepoManager interface { EnsurePullRequest(string, string, string, string) error PublishRepositoryPullRequest(string, string, string, string, func(string) error) (bool, error) EnsureWebhook(string, string, string) error - TriggerWebhookTest(string, string, string) error } var newOnboardingRepoManager = func(baseURL, token, owner, username, manifestsRepo, fluxRepo, branch, migrationBranch string) onboardingRepoManager { return forgejo.NewRepoManager(baseURL, token, owner, username, manifestsRepo, fluxRepo, branch, migrationBranch) } -// OnboardApp imports one clean checkout into its source Forgejo owner, -// publishes its delivery branch, registers it with Flux, then adds its webhook. +// OnboardApp opens reviewed source-delivery and cluster-registration pull requests. func OnboardApp(cfg config.Config, sourceDir string) error { resolved, err := config.ResolveAppOnboarding(cfg) if err != nil { @@ -46,6 +44,10 @@ func OnboardApp(cfg config.Config, sourceDir string) error { if sourceBranch != resolved.Delivery.AppRepoRef { return errors.New("--from branch must match delivery appRepoRef") } + hasAddons, err := hasMaidnAddons(sourceDir) + if err != nil { + return err + } owner, repository, err := forgejo.RepositoryFromURL(resolved.Delivery.AppRepoURL) if err != nil { return err @@ -82,7 +84,7 @@ func OnboardApp(cfg config.Config, sourceDir string) error { registrationBranch := "maidn/register-" + resolved.Delivery.AppName clusterManager := newOnboardingRepoManager(resolved.Git.BaseURL, resolved.Git.Token, resolved.Git.Owner, resolved.Git.Username, "", "", resolved.Flux.Branch, "") if _, err := clusterManager.PublishRepositoryPullRequest(resolved.Flux.RepoName, "feat: register "+resolved.Delivery.AppName+" delivery", registrationBranch, resolved.Flux.Branch, func(dir string) error { - return RegisterAppInCluster(dir, resolved) + return registerAppInCluster(dir, resolved, hasAddons || hasRuntimeSecretGrant(resolved)) }); err != nil { return fmt.Errorf("register app in cluster repository: %w", err) } @@ -102,12 +104,17 @@ func OnboardApp(cfg config.Config, sourceDir string) error { if err := sourceManager.EnsureWebhook(repository, webhookURL, authorization); err != nil { return fmt.Errorf("register Forgejo webhook: %w", err) } - if err := sourceManager.TriggerWebhookTest(repository, webhookURL, resolved.Delivery.AppRepoRef); err != nil { - return fmt.Errorf("trigger Forgejo webhook test: %w", err) - } return nil } +func hasMaidnAddons(dir string) (bool, error) { + _, err := readRegularFile(filepath.Join(dir, ".maidn", "kustomization.yaml")) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + return err == nil, err +} + // publishInitialAppBranches establishes the immutable source baseline before delivery setup. func publishInitialAppBranches(manager onboardingRepoManager, sourceDir, targetURL, sourceBranch, targetBranch, productionBranch string) error { sourceRevision, err := forgejo.BranchRevision(sourceDir, sourceBranch) @@ -150,7 +157,11 @@ func publishInitialAppBranches(manager onboardingRepoManager, sourceDir, targetU // RegisterAppInCluster writes only the managed Flux registration for one app. func RegisterAppInCluster(dir string, cfg config.Config) error { - content, err := renderAppRegistration(cfg) + return registerAppInCluster(dir, cfg, hasRuntimeSecretGrant(cfg)) +} + +func registerAppInCluster(dir string, cfg config.Config, hasAddons bool) error { + content, err := renderAppRegistrationWithAddons(cfg, hasAddons) if err != nil { return err } @@ -208,6 +219,10 @@ func RegisterAppInCluster(dir string, cfg config.Config) error { } func renderAppRegistration(cfg config.Config) ([]byte, error) { + return renderAppRegistrationWithAddons(cfg, hasRuntimeSecretGrant(cfg)) +} + +func renderAppRegistrationWithAddons(cfg config.Config, hasAddons bool) ([]byte, error) { if err := config.ValidateDelivery(cfg); err != nil { return nil, err } @@ -242,15 +257,15 @@ spec: dependsOn: - name: tekton-catalog sourceRef: - kind: GitRepository - name: %s + kind: GitRepository + name: %s `, cfg.Delivery.AppName, cfg.Delivery.AppRepoURL, branch, cfg.Delivery.AppName, cfg.Delivery.AppName) - if hasRuntimeSecretGrant(cfg) { + if hasAddons { content += fmt.Sprintf(`--- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: - name: %s-secrets + name: %s-addons namespace: flux-system spec: interval: 5m diff --git a/internal/bootstrap/onboard_test.go b/internal/bootstrap/onboard_test.go index e52929e..7683c29 100644 --- a/internal/bootstrap/onboard_test.go +++ b/internal/bootstrap/onboard_test.go @@ -98,6 +98,17 @@ func TestRegisterAppInClusterRendersManagedFluxSource(t *testing.T) { t.Fatalf("Kustomization %s does not include %s: %q, %v", path, resource, content, err) } } + decoder := yaml.NewDecoder(bytes.NewReader(registration)) + for { + var document yaml.Node + err := decoder.Decode(&document) + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("registration YAML: %v", err) + } + } if err := os.WriteFile(filepath.Join(tektonDir, "apps", "web-ui.yaml"), bytes.ReplaceAll(registration, []byte("\n"), []byte("\r\n")), 0644); err != nil { t.Fatal(err) } @@ -134,11 +145,34 @@ func TestGenerateAppSecretAccessRendersOnlyDeclaredRuntimeSecrets(t *testing.T) t.Fatalf("secret Kustomization = %q, %v", kustomization, err) } registration, err := renderAppRegistration(cfg) - if err != nil || !strings.Contains(string(registration), "name: web-ui-secrets") || !strings.Contains(string(registration), "path: ./.maidn") { + if err != nil || !strings.Contains(string(registration), "name: web-ui-addons") || !strings.Contains(string(registration), "path: ./.maidn") { t.Fatalf("secret registration = %q, %v", registration, err) } } +func TestRenderAppRegistrationIncludesSourceAddons(t *testing.T) { + registration, err := renderAppRegistrationWithAddons(onboardingConfig(), true) + if err != nil || !strings.Contains(string(registration), "name: web-ui-addons") || !strings.Contains(string(registration), "path: ./.maidn") || !strings.Contains(string(registration), "external-secrets-config") { + t.Fatalf("addon registration = %q, %v", registration, err) + } +} + +func TestHasMaidnAddons(t *testing.T) { + dir := t.TempDir() + if hasAddons, err := hasMaidnAddons(dir); err != nil || hasAddons { + t.Fatalf("absent add-ons = %t, %v", hasAddons, err) + } + if err := os.Mkdir(filepath.Join(dir, ".maidn"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, ".maidn", "kustomization.yaml"), []byte("apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nresources:\n"), 0644); err != nil { + t.Fatal(err) + } + if hasAddons, err := hasMaidnAddons(dir); err != nil || !hasAddons { + t.Fatalf("present add-ons = %t, %v", hasAddons, err) + } +} + func TestPublishInitialAppBranchesCreatesAndPreservesProduction(t *testing.T) { source := filepath.Join(t.TempDir(), "source") target := filepath.Join(t.TempDir(), "target.git") diff --git a/internal/forgejo/repo.go b/internal/forgejo/repo.go index 1ef08f6..15f1355 100644 --- a/internal/forgejo/repo.go +++ b/internal/forgejo/repo.go @@ -623,14 +623,11 @@ func (rm *RepoManager) PublishRepositoryPullRequest(repo, title, branch, base st if !open { return false, nil } - return true, rm.MergePullRequest(repo, branch) + return true, nil } if err := rm.EnsurePullRequest(repo, title, branch, base); err != nil { return false, err } - if err := rm.MergePullRequest(repo, branch); err != nil { - return false, err - } return true, nil }