diff --git a/README.md b/README.md index c300564..b901114 100644 --- a/README.md +++ b/README.md @@ -27,5 +27,21 @@ For `https://git.pingu.pw` you need: - git/ssh access from the machine running the CLI if you want SSH later - Flux bootstrap credentials for the repo URL that gets created -root@pam!maidn-test-key -2ce7bff1-ac98-4a45-8db4-186d49ae4159 \ No newline at end of file +## Cilium traffic network + +Every Talos node needs a second static network for Cilium L2 announcements. It has no gateway; the primary network remains the default route. Configure the matching VLAN and a unique MAC address for each node: + +```yaml +cilium: + trafficInterface: eth1 + loadBalancerStart: + loadBalancerEnd: +talos: + nodes: + - networks: + - # Primary management network + - macAddress: + cidr: + ip: + vlanId: +``` diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index ca40a17..58d9d9f 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -2,6 +2,7 @@ package bootstrap import ( "fmt" + "io" "os" "path/filepath" "strings" @@ -31,10 +32,30 @@ func (r Runner) Run() error { manifestsURL := forgejo.CloneURL(r.Config.Git.BaseURL, r.Config.Git.Owner, r.Config.Flux.ManifestsRepo) fluxConfig := ghrepo.BuildFluxConfig(strings.ToLower(r.Config.Git.Owner), manifestsURL, r.Config.Flux.ManifestsRepo) + cicdTemplateDir := filepath.Join(workspace, "maidn-cicd-cluster-template") + if err := ensureRepo(cicdTemplateDir, r.Config.Templates.CICDRepoURL, r.Config.Templates.CICDRepoRef); err != nil { + return err + } manager := forgejo.NewRepoManager(r.Config.Git.BaseURL, r.Config.Git.Token, r.Config.Git.Owner, r.Config.Git.Username, r.Config.Flux.ManifestsRepo, r.Config.Flux.RepoName) if err := manager.InitializeAll( func(dir string) error { return ghrepo.WriteManifestsStructure(dir, r.Config.Flux.ManifestsRepo) }, func(dir string) error { + clusterDir := filepath.Join(dir, strings.TrimPrefix(r.Config.Flux.ClusterPath, "./")) + if err := copyDir(filepath.Join(cicdTemplateDir, "base"), filepath.Join(dir, "base")); err != nil { + return err + } + if err := copyDir(filepath.Join(cicdTemplateDir, "clusters", "template"), clusterDir); err != nil { + return err + } + if err := renderCiliumConfig(filepath.Join(dir, "base", "cilium"), r.Config); err != nil { + return err + } + if err := renderCiliumConfig(filepath.Join(dir, "base", "cilium-config"), r.Config); err != nil { + return err + } + if err := ensureCiliumKustomizations(clusterDir); err != nil { + return err + } return ghrepo.WriteFluxStructure(dir, r.Config.Flux.RepoName, r.Config.Flux.ClusterPath, fluxConfig) }, ); err != nil { @@ -85,6 +106,9 @@ func (r Runner) Run() error { } } if r.Config.Talos.AutoBootstrapFlux { + if err := installCilium(generatedDir, r.Config); err != nil { + return err + } if err := utils.RunCommandInDir(generatedDir, "flux", "bootstrap", "git", "--url="+forgejo.CloneURL(r.Config.Git.BaseURL, r.Config.Git.Owner, r.Config.Flux.RepoName), "--branch="+r.Config.Flux.Branch, "--path="+r.Config.Flux.ClusterPath, "--cluster-domain="+r.Config.Flux.ClusterDomain, "--username="+r.Config.Git.Username, "--password="+r.Config.Git.Token, "--token-auth", "--kubeconfig=kubeconfig"); err != nil { return err } @@ -92,6 +116,82 @@ func (r Runner) Run() error { return nil } +func renderCiliumConfig(dir string, cfg config.Config) error { + replacements := strings.NewReplacer( + "${CILIUM_K8S_SERVICE_HOST}", cfg.Talos.KubeconfigEndpoint, + "${CILIUM_TRAFFIC_INTERFACE}", cfg.Cilium.TrafficInterface, + "${CILIUM_LB_START}", cfg.Cilium.LoadBalancerStart, + "${CILIUM_LB_END}", cfg.Cilium.LoadBalancerEnd, + ) + return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() { + return err + } + content, err := os.ReadFile(path) + if err != nil { + return err + } + return os.WriteFile(path, []byte(replacements.Replace(string(content))), info.Mode()) + }) +} + +func ensureCiliumKustomizations(clusterDir string) error { + path := filepath.Join(clusterDir, "kustomization.yaml") + content, err := os.ReadFile(path) + if err != nil { + return err + } + updated := string(content) + for _, resource := range []string{"cilium-kustomization.yaml", "cilium-config-kustomization.yaml"} { + if !strings.Contains(updated, resource) { + updated += " - " + resource + "\n" + } + } + if updated == string(content) { + return nil + } + return os.WriteFile(path, []byte(updated), 0644) +} + +func installCilium(dir string, cfg config.Config) error { + helmDir := filepath.Join(dir, ".helm") + if err := os.MkdirAll(helmDir, 0755); err != nil { + return err + } + return utils.RunCommandInDir(dir, "helm", "upgrade", "--install", "cilium", "cilium", "--repo=https://helm.cilium.io", "--version=1.19.6", "--repository-config="+filepath.Join(helmDir, "repositories.yaml"), "--repository-cache="+helmDir, "--namespace=kube-system", "--create-namespace", "--kubeconfig=kubeconfig", "--wait", "--timeout=5m", "--set=kubeProxyReplacement=true", "--set=ipam.mode=kubernetes", "--set=k8sServiceHost="+cfg.Talos.KubeconfigEndpoint, "--set=k8sServicePort=6443", "--set=gatewayAPI.enabled=true", "--set=l2announcements.enabled=true") +} + +func copyDir(source, destination string) error { + return filepath.Walk(source, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + relative, err := filepath.Rel(source, path) + if err != nil || relative == "." { + return err + } + target := filepath.Join(destination, relative) + if info.IsDir() { + return os.MkdirAll(target, 0755) + } + if _, err := os.Stat(target); err == nil { + return nil + } + input, err := os.Open(path) + if err != nil { + return err + } + defer input.Close() + output, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, info.Mode()) + if err != nil { + return err + } + defer output.Close() + _, err = io.Copy(output, input) + return err + }) +} + func waitForTalosReboot(dir string, cfg config.Config) error { deadline := time.Now().Add(5 * time.Minute) offline := false @@ -170,7 +270,7 @@ func renderTerraformTFVars(cfg config.Config) string { builder.WriteString(fmt.Sprintf("cluster_domain = %q\n", cfg.Talos.Cluster.Domain)) builder.WriteString(fmt.Sprintf("talos_factory_schematic_id = %q\n", cfg.Talos.Image.SchematicID)) builder.WriteString(fmt.Sprintf("talos_version = %q\n", cfg.Talos.Image.TalosVersion)) - builder.WriteString("cni_name = \"flannel\"\n") + builder.WriteString("cni_name = \"none\"\n") isoStorage := cfg.Talos.Image.Storage if isoStorage == cfg.Talos.Cluster.DiskStorage { isoStorage = "local" diff --git a/internal/bootstrap/bootstrap_test.go b/internal/bootstrap/bootstrap_test.go new file mode 100644 index 0000000..bc21c3d --- /dev/null +++ b/internal/bootstrap/bootstrap_test.go @@ -0,0 +1,29 @@ +package bootstrap + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Pingu-Studio/MaidnCLI/internal/config" +) + +func TestRenderCiliumConfig(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "values.yaml") + if err := os.WriteFile(path, []byte("host: ${CILIUM_K8S_SERVICE_HOST}\ninterface: ${CILIUM_TRAFFIC_INTERFACE}\nstart: ${CILIUM_LB_START}\nend: ${CILIUM_LB_END}\n"), 0644); err != nil { + t.Fatal(err) + } + cfg := config.Config{Talos: config.TalosConfig{KubeconfigEndpoint: "192.168.45.3"}, Cilium: config.CiliumConfig{TrafficInterface: "eth1", LoadBalancerStart: "192.168.45.19", LoadBalancerEnd: "192.168.45.30"}} + if err := renderCiliumConfig(dir, cfg); err != nil { + t.Fatal(err) + } + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(content), "${") || !strings.Contains(string(content), "192.168.45.30") { + t.Fatalf("Cilium configuration was not rendered: %s", content) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 7c41e5f..9a3fee1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2,6 +2,7 @@ package config import ( "errors" + "net/netip" "os" "github.com/Pingu-Studio/MaidnCLI/internal/talos" @@ -58,6 +59,15 @@ func applyDefaults(cfg *Config) { if cfg.Templates.TalosRepoRef == "" { cfg.Templates.TalosRepoRef = "main" } + if cfg.Templates.CICDRepoURL == "" { + cfg.Templates.CICDRepoURL = "https://git.pingu.pw/Maidn/maidn-cicd-cluster-template.git" + } + if cfg.Templates.CICDRepoRef == "" { + cfg.Templates.CICDRepoRef = "main" + } + if cfg.Cilium.TrafficInterface == "" { + cfg.Cilium.TrafficInterface = "eth1" + } if cfg.Talos.RepoDirName == "" { cfg.Talos.RepoDirName = "maidn-talos-proxmox" } @@ -121,5 +131,33 @@ func Validate(cfg Config) error { if len(cfg.Talos.Nodes) == 0 { return errors.New("at least one talos node is required") } + if cfg.Cilium.LoadBalancerStart == "" || cfg.Cilium.LoadBalancerEnd == "" { + return errors.New("cilium loadBalancerStart and loadBalancerEnd are required") + } + start, err := netip.ParseAddr(cfg.Cilium.LoadBalancerStart) + if err != nil { + return errors.New("cilium loadBalancerStart must be an IP address") + } + end, err := netip.ParseAddr(cfg.Cilium.LoadBalancerEnd) + if err != nil || start.BitLen() != end.BitLen() || start.Compare(end) > 0 { + return errors.New("cilium loadBalancerEnd must be an IP address after loadBalancerStart") + } + for _, node := range cfg.Talos.Nodes { + if len(node.Networks) < 2 { + return errors.New("cilium requires a second static traffic network on every talos node") + } + traffic := node.Networks[1] + prefix, err := netip.ParsePrefix(traffic.CIDR) + if err != nil || traffic.IP == "" || traffic.Gateway != "" { + return errors.New("cilium traffic networks require a static IP, valid CIDR, and no gateway") + } + nodeIP, err := netip.ParseAddr(traffic.IP) + if err != nil || !prefix.Contains(nodeIP) || nodeIP == start || nodeIP == end { + return errors.New("cilium traffic node IP must belong to its CIDR and not use the LoadBalancer range") + } + if !prefix.Contains(start) || !prefix.Contains(end) { + return errors.New("cilium LoadBalancer range must belong to every node traffic network") + } + } return nil } diff --git a/internal/config/types.go b/internal/config/types.go index fc42338..be9dfe1 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -8,6 +8,7 @@ type Config struct { Flux FluxConfig `yaml:"flux"` Talos TalosConfig `yaml:"talos"` Templates TemplateConfig `yaml:"templates"` + Cilium CiliumConfig `yaml:"cilium"` } type GitConfig struct { @@ -32,6 +33,14 @@ type FluxConfig struct { type TemplateConfig struct { TalosRepoURL string `yaml:"talosRepoUrl"` TalosRepoRef string `yaml:"talosRepoRef"` + CICDRepoURL string `yaml:"cicdRepoUrl"` + CICDRepoRef string `yaml:"cicdRepoRef"` +} + +type CiliumConfig struct { + TrafficInterface string `yaml:"trafficInterface"` + LoadBalancerStart string `yaml:"loadBalancerStart"` + LoadBalancerEnd string `yaml:"loadBalancerEnd"` } type TalosConfig struct { diff --git a/internal/ui/wizard.go b/internal/ui/wizard.go index 7d4d6e8..4885aed 100644 --- a/internal/ui/wizard.go +++ b/internal/ui/wizard.go @@ -31,8 +31,10 @@ func RunBootstrapWizard(initial config.Config) (config.Config, error) { cfg.Flux.Branch = "main" cfg.Flux.ClusterPath = "./clusters/maidn-cd-0" cfg.Templates.TalosRepoURL = prompt(reader, "Talos template repo URL", fallback(cfg.Templates.TalosRepoURL, "https://git.pingu.pw/Maidn/maidn-talos-proxmox.git")) + cfg.Templates.CICDRepoURL = prompt(reader, "CI/CD template repo URL", fallback(cfg.Templates.CICDRepoURL, "https://git.pingu.pw/Maidn/maidn-cicd-cluster-template.git")) fmt.Printf("Template repos are cloned under: %s\n", cfg.Git.CloneParent) cfg.Templates.TalosRepoRef = "main" + cfg.Templates.CICDRepoRef = "main" cfg.Talos.RepoDirName = "maidn-talos-proxmox" cfg.Talos.TerraformDir = "terraform" cfg.Talos.GeneratedDir = "generated" @@ -89,6 +91,9 @@ func RunBootstrapWizard(initial config.Config) (config.Config, error) { nodePreset := prompt(reader, "Cluster size preset (single/ha/custom)", "single") cfg.Talos.Nodes = buildNodes(reader, cfg, discovered, nodePreset) + cfg.Cilium.TrafficInterface = prompt(reader, "Cilium traffic interface", fallback(cfg.Cilium.TrafficInterface, "eth1")) + cfg.Cilium.LoadBalancerStart = prompt(reader, "Cilium LoadBalancer range start", cfg.Cilium.LoadBalancerStart) + cfg.Cilium.LoadBalancerEnd = prompt(reader, "Cilium LoadBalancer range end", cfg.Cilium.LoadBalancerEnd) cfg.Talos.BootstrapNode = cfg.Talos.Nodes[0].Networks[0].IP cfg.Talos.BootstrapEndpoint = cfg.Talos.BootstrapNode cfg.Talos.KubeconfigNode = cfg.Talos.Nodes[0].Networks[0].IP