feat: bootstrap cilium traffic plane

README.md

@@ -27,5 +27,21 @@ For `https://git.pingu.pw` you need:
 - git/ssh access from the machine running the CLI if you want SSH later
 - Flux bootstrap credentials for the repo URL that gets created
 
-root@pam!maidn-test-key
-2ce7bff1-ac98-4a45-8db4-186d49ae4159
+## Cilium traffic network
+
+Every Talos node needs a second static network for Cilium L2 announcements. It has no gateway; the primary network remains the default route. Configure the matching VLAN and a unique MAC address for each node:
+
+```yaml
+cilium:
+  trafficInterface: eth1
+  loadBalancerStart: <first-reserved-address>
+  loadBalancerEnd: <last-reserved-address>
+talos:
+  nodes:
+    - networks:
+        - # Primary management network
+        - macAddress: <unique-mac>
+          cidr: <traffic-subnet>
+          ip: <node-traffic-address>
+          vlanId: <opnsense-traffic-vlan>
+```

internal/bootstrap/bootstrap.go

@@ -2,6 +2,7 @@ package bootstrap
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
@@ -31,10 +32,30 @@ func (r Runner) Run() error {
 
 	manifestsURL := forgejo.CloneURL(r.Config.Git.BaseURL, r.Config.Git.Owner, r.Config.Flux.ManifestsRepo)
 	fluxConfig := ghrepo.BuildFluxConfig(strings.ToLower(r.Config.Git.Owner), manifestsURL, r.Config.Flux.ManifestsRepo)
+	cicdTemplateDir := filepath.Join(workspace, "maidn-cicd-cluster-template")
+	if err := ensureRepo(cicdTemplateDir, r.Config.Templates.CICDRepoURL, r.Config.Templates.CICDRepoRef); err != nil {
+		return err
+	}
 	manager := forgejo.NewRepoManager(r.Config.Git.BaseURL, r.Config.Git.Token, r.Config.Git.Owner, r.Config.Git.Username, r.Config.Flux.ManifestsRepo, r.Config.Flux.RepoName)
 	if err := manager.InitializeAll(
 		func(dir string) error { return ghrepo.WriteManifestsStructure(dir, r.Config.Flux.ManifestsRepo) },
 		func(dir string) error {
+			clusterDir := filepath.Join(dir, strings.TrimPrefix(r.Config.Flux.ClusterPath, "./"))
+			if err := copyDir(filepath.Join(cicdTemplateDir, "base"), filepath.Join(dir, "base")); err != nil {
+				return err
+			}
+			if err := copyDir(filepath.Join(cicdTemplateDir, "clusters", "template"), clusterDir); err != nil {
+				return err
+			}
+			if err := renderCiliumConfig(filepath.Join(dir, "base", "cilium"), r.Config); err != nil {
+				return err
+			}
+			if err := renderCiliumConfig(filepath.Join(dir, "base", "cilium-config"), r.Config); err != nil {
+				return err
+			}
+			if err := ensureCiliumKustomizations(clusterDir); err != nil {
+				return err
+			}
 			return ghrepo.WriteFluxStructure(dir, r.Config.Flux.RepoName, r.Config.Flux.ClusterPath, fluxConfig)
 		},
 	); err != nil {
@@ -85,6 +106,9 @@ func (r Runner) Run() error {
 		}
 	}
 	if r.Config.Talos.AutoBootstrapFlux {
+		if err := installCilium(generatedDir, r.Config); err != nil {
+			return err
+		}
 		if err := utils.RunCommandInDir(generatedDir, "flux", "bootstrap", "git", "--url="+forgejo.CloneURL(r.Config.Git.BaseURL, r.Config.Git.Owner, r.Config.Flux.RepoName), "--branch="+r.Config.Flux.Branch, "--path="+r.Config.Flux.ClusterPath, "--cluster-domain="+r.Config.Flux.ClusterDomain, "--username="+r.Config.Git.Username, "--password="+r.Config.Git.Token, "--token-auth", "--kubeconfig=kubeconfig"); err != nil {
 			return err
 		}
@@ -92,6 +116,82 @@ func (r Runner) Run() error {
 	return nil
 }
 
+func renderCiliumConfig(dir string, cfg config.Config) error {
+	replacements := strings.NewReplacer(
+		"${CILIUM_K8S_SERVICE_HOST}", cfg.Talos.KubeconfigEndpoint,
+		"${CILIUM_TRAFFIC_INTERFACE}", cfg.Cilium.TrafficInterface,
+		"${CILIUM_LB_START}", cfg.Cilium.LoadBalancerStart,
+		"${CILIUM_LB_END}", cfg.Cilium.LoadBalancerEnd,
+	)
+	return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil || info.IsDir() {
+			return err
+		}
+		content, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		return os.WriteFile(path, []byte(replacements.Replace(string(content))), info.Mode())
+	})
+}
+
+func ensureCiliumKustomizations(clusterDir string) error {
+	path := filepath.Join(clusterDir, "kustomization.yaml")
+	content, err := os.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	updated := string(content)
+	for _, resource := range []string{"cilium-kustomization.yaml", "cilium-config-kustomization.yaml"} {
+		if !strings.Contains(updated, resource) {
+			updated += "  - " + resource + "\n"
+		}
+	}
+	if updated == string(content) {
+		return nil
+	}
+	return os.WriteFile(path, []byte(updated), 0644)
+}
+
+func installCilium(dir string, cfg config.Config) error {
+	helmDir := filepath.Join(dir, ".helm")
+	if err := os.MkdirAll(helmDir, 0755); err != nil {
+		return err
+	}
+	return utils.RunCommandInDir(dir, "helm", "upgrade", "--install", "cilium", "cilium", "--repo=https://helm.cilium.io", "--version=1.19.6", "--repository-config="+filepath.Join(helmDir, "repositories.yaml"), "--repository-cache="+helmDir, "--namespace=kube-system", "--create-namespace", "--kubeconfig=kubeconfig", "--wait", "--timeout=5m", "--set=kubeProxyReplacement=true", "--set=ipam.mode=kubernetes", "--set=k8sServiceHost="+cfg.Talos.KubeconfigEndpoint, "--set=k8sServicePort=6443", "--set=gatewayAPI.enabled=true", "--set=l2announcements.enabled=true")
+}
+
+func copyDir(source, destination string) error {
+	return filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		relative, err := filepath.Rel(source, path)
+		if err != nil || relative == "." {
+			return err
+		}
+		target := filepath.Join(destination, relative)
+		if info.IsDir() {
+			return os.MkdirAll(target, 0755)
+		}
+		if _, err := os.Stat(target); err == nil {
+			return nil
+		}
+		input, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer input.Close()
+		output, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, info.Mode())
+		if err != nil {
+			return err
+		}
+		defer output.Close()
+		_, err = io.Copy(output, input)
+		return err
+	})
+}
+
 func waitForTalosReboot(dir string, cfg config.Config) error {
 	deadline := time.Now().Add(5 * time.Minute)
 	offline := false
@@ -170,7 +270,7 @@ func renderTerraformTFVars(cfg config.Config) string {
 	builder.WriteString(fmt.Sprintf("cluster_domain = %q\n", cfg.Talos.Cluster.Domain))
 	builder.WriteString(fmt.Sprintf("talos_factory_schematic_id = %q\n", cfg.Talos.Image.SchematicID))
 	builder.WriteString(fmt.Sprintf("talos_version = %q\n", cfg.Talos.Image.TalosVersion))
-	builder.WriteString("cni_name = \"flannel\"\n")
+	builder.WriteString("cni_name = \"none\"\n")
 	isoStorage := cfg.Talos.Image.Storage
 	if isoStorage == cfg.Talos.Cluster.DiskStorage {
 		isoStorage = "local"

internal/bootstrap/bootstrap_test.go

@@ -0,0 +1,29 @@
+package bootstrap
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/Pingu-Studio/MaidnCLI/internal/config"
+)
+
+func TestRenderCiliumConfig(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "values.yaml")
+	if err := os.WriteFile(path, []byte("host: ${CILIUM_K8S_SERVICE_HOST}\ninterface: ${CILIUM_TRAFFIC_INTERFACE}\nstart: ${CILIUM_LB_START}\nend: ${CILIUM_LB_END}\n"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	cfg := config.Config{Talos: config.TalosConfig{KubeconfigEndpoint: "192.168.45.3"}, Cilium: config.CiliumConfig{TrafficInterface: "eth1", LoadBalancerStart: "192.168.45.19", LoadBalancerEnd: "192.168.45.30"}}
+	if err := renderCiliumConfig(dir, cfg); err != nil {
+		t.Fatal(err)
+	}
+	content, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(string(content), "${") || !strings.Contains(string(content), "192.168.45.30") {
+		t.Fatalf("Cilium configuration was not rendered: %s", content)
+	}
+}

internal/config/config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"errors"
+	"net/netip"
 	"os"
 
 	"github.com/Pingu-Studio/MaidnCLI/internal/talos"
@@ -58,6 +59,15 @@ func applyDefaults(cfg *Config) {
 	if cfg.Templates.TalosRepoRef == "" {
 		cfg.Templates.TalosRepoRef = "main"
 	}
+	if cfg.Templates.CICDRepoURL == "" {
+		cfg.Templates.CICDRepoURL = "https://git.pingu.pw/Maidn/maidn-cicd-cluster-template.git"
+	}
+	if cfg.Templates.CICDRepoRef == "" {
+		cfg.Templates.CICDRepoRef = "main"
+	}
+	if cfg.Cilium.TrafficInterface == "" {
+		cfg.Cilium.TrafficInterface = "eth1"
+	}
 	if cfg.Talos.RepoDirName == "" {
 		cfg.Talos.RepoDirName = "maidn-talos-proxmox"
 	}
@@ -121,5 +131,33 @@ func Validate(cfg Config) error {
 	if len(cfg.Talos.Nodes) == 0 {
 		return errors.New("at least one talos node is required")
 	}
+	if cfg.Cilium.LoadBalancerStart == "" || cfg.Cilium.LoadBalancerEnd == "" {
+		return errors.New("cilium loadBalancerStart and loadBalancerEnd are required")
+	}
+	start, err := netip.ParseAddr(cfg.Cilium.LoadBalancerStart)
+	if err != nil {
+		return errors.New("cilium loadBalancerStart must be an IP address")
+	}
+	end, err := netip.ParseAddr(cfg.Cilium.LoadBalancerEnd)
+	if err != nil || start.BitLen() != end.BitLen() || start.Compare(end) > 0 {
+		return errors.New("cilium loadBalancerEnd must be an IP address after loadBalancerStart")
+	}
+	for _, node := range cfg.Talos.Nodes {
+		if len(node.Networks) < 2 {
+			return errors.New("cilium requires a second static traffic network on every talos node")
+		}
+		traffic := node.Networks[1]
+		prefix, err := netip.ParsePrefix(traffic.CIDR)
+		if err != nil || traffic.IP == "" || traffic.Gateway != "" {
+			return errors.New("cilium traffic networks require a static IP, valid CIDR, and no gateway")
+		}
+		nodeIP, err := netip.ParseAddr(traffic.IP)
+		if err != nil || !prefix.Contains(nodeIP) || nodeIP == start || nodeIP == end {
+			return errors.New("cilium traffic node IP must belong to its CIDR and not use the LoadBalancer range")
+		}
+		if !prefix.Contains(start) || !prefix.Contains(end) {
+			return errors.New("cilium LoadBalancer range must belong to every node traffic network")
+		}
+	}
 	return nil
 }

internal/config/types.go

@@ -8,6 +8,7 @@ type Config struct {
 	Flux         FluxConfig     `yaml:"flux"`
 	Talos        TalosConfig    `yaml:"talos"`
 	Templates    TemplateConfig `yaml:"templates"`
+	Cilium       CiliumConfig   `yaml:"cilium"`
 }
 
 type GitConfig struct {
@@ -32,6 +33,14 @@ type FluxConfig struct {
 type TemplateConfig struct {
 	TalosRepoURL string `yaml:"talosRepoUrl"`
 	TalosRepoRef string `yaml:"talosRepoRef"`
+	CICDRepoURL  string `yaml:"cicdRepoUrl"`
+	CICDRepoRef  string `yaml:"cicdRepoRef"`
+}
+
+type CiliumConfig struct {
+	TrafficInterface  string `yaml:"trafficInterface"`
+	LoadBalancerStart string `yaml:"loadBalancerStart"`
+	LoadBalancerEnd   string `yaml:"loadBalancerEnd"`
 }
 
 type TalosConfig struct {

internal/ui/wizard.go

@@ -31,8 +31,10 @@ func RunBootstrapWizard(initial config.Config) (config.Config, error) {
 	cfg.Flux.Branch = "main"
 	cfg.Flux.ClusterPath = "./clusters/maidn-cd-0"
 	cfg.Templates.TalosRepoURL = prompt(reader, "Talos template repo URL", fallback(cfg.Templates.TalosRepoURL, "https://git.pingu.pw/Maidn/maidn-talos-proxmox.git"))
+	cfg.Templates.CICDRepoURL = prompt(reader, "CI/CD template repo URL", fallback(cfg.Templates.CICDRepoURL, "https://git.pingu.pw/Maidn/maidn-cicd-cluster-template.git"))
 	fmt.Printf("Template repos are cloned under: %s\n", cfg.Git.CloneParent)
 	cfg.Templates.TalosRepoRef = "main"
+	cfg.Templates.CICDRepoRef = "main"
 	cfg.Talos.RepoDirName = "maidn-talos-proxmox"
 	cfg.Talos.TerraformDir = "terraform"
 	cfg.Talos.GeneratedDir = "generated"
@@ -89,6 +91,9 @@ func RunBootstrapWizard(initial config.Config) (config.Config, error) {
 
 	nodePreset := prompt(reader, "Cluster size preset (single/ha/custom)", "single")
 	cfg.Talos.Nodes = buildNodes(reader, cfg, discovered, nodePreset)
+	cfg.Cilium.TrafficInterface = prompt(reader, "Cilium traffic interface", fallback(cfg.Cilium.TrafficInterface, "eth1"))
+	cfg.Cilium.LoadBalancerStart = prompt(reader, "Cilium LoadBalancer range start", cfg.Cilium.LoadBalancerStart)
+	cfg.Cilium.LoadBalancerEnd = prompt(reader, "Cilium LoadBalancer range end", cfg.Cilium.LoadBalancerEnd)
 	cfg.Talos.BootstrapNode = cfg.Talos.Nodes[0].Networks[0].IP
 	cfg.Talos.BootstrapEndpoint = cfg.Talos.BootstrapNode
 	cfg.Talos.KubeconfigNode = cfg.Talos.Nodes[0].Networks[0].IP