From 5280128ae5a57304223ff8da46e40094bb8c3593 Mon Sep 17 00:00:00 2001 From: eding Date: Wed, 15 Jul 2026 20:39:39 +0200 Subject: [PATCH] feat: extract talos proxmox stack --- .gitignore | 2 + README.md | 46 +++++- docs/maintenance.md | 44 ++++++ terraform/.gitignore | 61 ++++++++ terraform/.terraform.lock.hcl | 65 ++++++++ terraform/cluster.auto.tfvars.example | 41 +++++ terraform/locals.tf | 85 +++++++++++ terraform/main.tf | 136 +++++++++++++++++ terraform/scripts/stage_talos_image.py | 60 ++++++++ terraform/talconfig_generator.tf | 79 ++++++++++ terraform/templates/talconfig.yaml.tpl | 198 +++++++++++++++++++++++++ terraform/terraform.examples.tfvars | 120 +++++++++++++++ terraform/variables.tf | 190 ++++++++++++++++++++++++ terraform/version.auto.tfvars.example | 3 + 14 files changed, 1129 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 docs/maintenance.md create mode 100644 terraform/.gitignore create mode 100644 terraform/.terraform.lock.hcl create mode 100644 terraform/cluster.auto.tfvars.example create mode 100644 terraform/locals.tf create mode 100644 terraform/main.tf create mode 100644 terraform/scripts/stage_talos_image.py create mode 100644 terraform/talconfig_generator.tf create mode 100644 terraform/templates/talconfig.yaml.tpl create mode 100644 terraform/terraform.examples.tfvars create mode 100644 terraform/variables.tf create mode 100644 terraform/version.auto.tfvars.example diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e88963b --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/.dump +/generated diff --git a/README.md b/README.md index 6ebc59a..7b16e5d 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,47 @@ # maidn-talos-proxmox -Standalone Talos on Proxmox IaC for Maidn \ No newline at end of file +Standalone Talos-on-Proxmox IaC for Maidn. + +## Layout + +- `terraform/` provisions Proxmox networking and VMs and generates `generated/talconfig.yaml` +- `generated/` holds local generated Talos artifacts and should stay uncommitted + +## Usage + +1. Copy `terraform/terraform.examples.tfvars` to `terraform/terraform.tfvars` +2. Set Proxmox, cluster, and node values +3. Run: + +```bash +cd terraform +terraform init +terraform apply +``` + +4. Generate secrets/config outside Terraform: + +```bash +cd ../generated +talhelper gensecret > talsecret.sops.yaml +sops -e -i talsecret.sops.yaml +talhelper genconfig +``` + +5. Bootstrap Talos and Flux: + +```bash +talosctl bootstrap --talosconfig=./clusterconfig/talosconfig --nodes= +talosctl kubeconfig --talosconfig=./clusterconfig/talosconfig --nodes= . +flux bootstrap git --url= --branch=main --path=./clusters/ +``` + +## Image updates + +Keep Talos version and factory schematic in config. A repo automation job can resolve the matching installer/raw/iso artifacts and open a PR updating pins. + +If `talos_image_update_mode = "download"`, Terraform also stages the required Talos ISO onto every Proxmox node used by the cluster before VM creation. + +## Maintenance + +See `docs/maintenance.md` for upgrades, certificate rotation, health checks, and backup basics. diff --git a/docs/maintenance.md b/docs/maintenance.md new file mode 100644 index 0000000..7172cc9 --- /dev/null +++ b/docs/maintenance.md @@ -0,0 +1,44 @@ +# Talos maintenance + +## Regular checks + +- `talosctl --talosconfig=./generated/clusterconfig/talosconfig health` +- `kubectl get nodes` +- `flux get all -A` +- `talosctl -n version` + +## Upgrade flow + +1. Update `talos_factory_schematic_id`, `talos_version`, and `talos_iso_file` in Terraform inputs. +2. If `talos_image_update_mode = "download"`, run `terraform apply` to stage the ISO on every target Proxmox node. +3. Regenerate machine config from `generated/talconfig.yaml`: + - `talhelper genconfig` +4. Upgrade one control plane node at a time: + - `talosctl upgrade --nodes --image factory.talos.dev/installer//` + - wait for `talosctl health` +5. Upgrade workers one at a time. +6. Refresh kubeconfig if needed: + - `talosctl kubeconfig --talosconfig=./generated/clusterconfig/talosconfig --nodes= .` + +## Certificate rotation + +- Check cert expiry: + - `talosctl -n get kubeapicerts,k8saggregatorca,etcdcerts` +- Rotate before expiry during a maintenance window: + - `talosctl -n rotate-ca` + - `talosctl -n rotate-certs` +- After rotation, verify API and kubelet health on all nodes. + +## Disaster recovery basics + +- Keep `generated/talsecret.sops.yaml` encrypted and backed up. +- Keep the rendered `generated/clusterconfig/talosconfig` in a secure backup. +- Capture an etcd snapshot before major upgrades: + - `talosctl -n etcd snapshot ./etcd.snapshot` + +## What to automate next + +- PR automation that bumps Talos version + factory schematic outputs together. +- Scheduled `talosctl health` and cert expiry checks. +- Optional automated image staging to all Proxmox nodes before upgrades. +- Optional upgrade orchestration that rolls one node at a time with health gates. diff --git a/terraform/.gitignore b/terraform/.gitignore new file mode 100644 index 0000000..2ab921d --- /dev/null +++ b/terraform/.gitignore @@ -0,0 +1,61 @@ +# Terraform General +# ================================================================= +# Ignore local Terraform state files and backups. +# These files contain the state of your infrastructure and can include secrets. +# They should be managed remotely (e.g., using Terraform Cloud, S3, etc.). +*.tfstate +*.tfstate.backup + +# Ignore crash log files. +crash.log +crash.*.log + +# Ignore local Terraform configuration files that can contain sensitive data. +#terraform.tfvars + +# Ignore override files as they are typically used for local testing. +# These should not be checked in to source control. +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Ignore CLI configuration files. +.terraformrc +terraform.rc + + +# Terraform Provider Binaries and Lock Files +# ================================================================= +# Ignore the directory where Terraform downloads provider plugins. +# This directory is automatically managed by `terraform init`. +.terraform/ + +# Ignore the dependency lock file. This file ensures that the same provider +# versions are used across all environments. It's best practice for modules, +# but can be ignored for root configurations if you want to allow flexibility. +# For consistency, it's often better to COMMIT this file. I've included it here +# as an option if you choose to ignore it. +# .terraform.lock.hcl + + +# Terraform Plan and Output Files +# ================================================================= +# Ignore plan files. These can contain the full configuration and secrets +# in plain text and should be handled securely. +*.tfplan + + +# Other Common Files +# ================================================================= +# Ignore temporary files from text editors. +.idea/ +.vscode/ +*.swp +*~ + +# Ignore operating system files. +.DS_Store +Thumbs.db +/.secrets +/test \ No newline at end of file diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..90bb3c3 --- /dev/null +++ b/terraform/.terraform.lock.hcl @@ -0,0 +1,65 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/bpg/proxmox" { + version = "0.83.1" + constraints = "0.83.1" + hashes = [ + "h1:R1OKyousY2wQML/cnFsIwP9ctymlbNc3hORpEVgA1Dg=", + "zh:0572dcbf8e214bf6b1cf7f17f26929e59d1d4a136becac39f577262762d6d2a1", + "zh:0bf9fae2da3d4bdf22b8610f9c091bf90a874a3e5304dbc628599a11415ccc17", + "zh:29df0391f6baf3f439c4267323fab17ca5a45233e0ffd4a8cf6bbaa50bd91472", + "zh:3be786d09e0b41f89a9198aca4a5c5acb6d4fa652648708d6908cfd59f19c602", + "zh:69f52e4484b8837659f255ab1ee80424634fde69f2708ee6b401aa2a5923b5d4", + "zh:78f8c2f32d90be49f0bdcc1bf9e3cab511627f0467743c4e612f090e29aaab31", + "zh:81b22476a789786f0435bf919337541faa5cc3013017f59f5f6f8044fd280b7b", + "zh:8fc2bed97af911e55bbc14aa1b0ee165c90b32d3c398f5d11421abb91b9abc56", + "zh:9ce6030058ae410f95e8fca00c4075b201d4e2af66d2a29d80821efa217aff31", + "zh:9eec11a45ad966b680880244e52db89f89a1a96304fc1e4c5df4fbaae3108486", + "zh:ba854227762e513c08fcfde86195457ccc572811ddfccce9a682f10c3c18483a", + "zh:bc6bb583cf470283e86aa42ccd834f523beb43270292376f774072ac446bcb96", + "zh:d2cb5b64a09d23e99537a0d9d6fc1f5f39b81d943eb242cc1cbd51d251eae77c", + "zh:d89023a2164de21da5bb953d6767e1730d204f43c25c46ffde91fed32e00ebae", + "zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597", + ] +} + +provider "registry.terraform.io/hashicorp/local" { + version = "2.9.0" + hashes = [ + "h1:px3Hpv/tL288wzu5knHywTTBcrydLnnGEiF/NIQBaRs=", + "zh:0baa4566cf77f1ff52f4293d1c8536202dd23edc197c3196413a28343c3ac3a0", + "zh:16b5559c3c07088ddad11a9bb9e9c0799999363c2958e9a5be2bcbbf2cd9ca64", + "zh:197c79015a10d1cce904a8ea722cbc750c42aeae2da53f44a6a0751d9fd1aa90", + "zh:29d0b03e5343a80677ebfeb2e2c31cbe4b1f65e736e53417454a4277fec2544c", + "zh:4896bfa6cf1d2fd562b47ef2e87f47862ae92a04f8ad5d764380f0c6653473b8", + "zh:531f8529cbca49f681883e57761a05a8398afaef6d1ab0d205d26bf12f4428e8", + "zh:6aaf5011d83161c86d2bfb80c0923ec934e578288758da2f37acb7aec129004b", + "zh:7430275253d3d3c40aa6179e0ec0d63212874dbbc06c5a51b9d07ec590f9756c", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:be17dc611e95e26cdf6cad79dfccf1064f0e32032a2efeb939a9bbe7fb1cbfe9", + "zh:f0e3b0aa644202e1d79d2000dca91f6019425da71e9800fa23f27e51c034f195", + "zh:f62bae4519e4ead49182ddc8afe8cf61e2a4c3ba3973b0fbba967736a2696aa3", + "zh:fcafa360a5b0b96244f26f4e3a6d642b716a376557142c2442ff2fb12d11da18", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.4" + constraints = "3.2.4" + hashes = [ + "h1:+Ag4hSb4qQjNtAS6gj2+gsGl7v0iB/Bif6zZZU8lXsw=", + "zh:59f6b52ab4ff35739647f9509ee6d93d7c032985d9f8c6237d1f8a59471bbbe2", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:795c897119ff082133150121d39ff26cb5f89a730a2c8c26f3a9c1abf81a9c43", + "zh:7b9c7b16f118fbc2b05a983817b8ce2f86df125857966ad356353baf4bff5c0a", + "zh:85e33ab43e0e1726e5f97a874b8e24820b6565ff8076523cc2922ba671492991", + "zh:9d32ac3619cfc93eb3c4f423492a8e0f79db05fec58e449dee9b2d5873d5f69f", + "zh:9e15c3c9dd8e0d1e3731841d44c34571b6c97f5b95e8296a45318b94e5287a6e", + "zh:b4c2ab35d1b7696c30b64bf2c0f3a62329107bd1a9121ce70683dec58af19615", + "zh:c43723e8cc65bcdf5e0c92581dcbbdcbdcf18b8d2037406a5f2033b1e22de442", + "zh:ceb5495d9c31bfb299d246ab333f08c7fb0d67a4f82681fbf47f2a21c3e11ab5", + "zh:e171026b3659305c558d9804062762d168f50ba02b88b231d20ec99578a6233f", + "zh:ed0fe2acdb61330b01841fa790be00ec6beaac91d41f311fb8254f74eb6a711f", + ] +} diff --git a/terraform/cluster.auto.tfvars.example b/terraform/cluster.auto.tfvars.example new file mode 100644 index 0000000..5752e44 --- /dev/null +++ b/terraform/cluster.auto.tfvars.example @@ -0,0 +1,41 @@ +# Copy to cluster.auto.tfvars (git-ignored) and fill in real values. +# This file is an example; Terraform will NOT auto-load it until you remove the .example suffix. + +# --- Proxmox Connection (credentials.auto.tfvars already suggested for secrets) --- +# proxmox_api_url = "https://192.168.10.13:8006/api2/json" +# proxmox_node = "proxmox-threadripper" +# proxmox_api_token = "root@pam!iac=" # keep secret (place in credentials.auto.tfvars ideally) +# proxmox_pool = "" # optional +# proxmox_ssh_password = "" # or set via environment TF_VAR_proxmox_ssh_password + +# --- Talos ISO (uploaded manually to Proxmox ISO storage) --- +# Use factory.talos.dev output OR official Talos release ISO +# Path pattern: :iso/ +talos_iso_file = "local:iso/talos-1.10.6.iso" + +# --- Storage Selection --- +# Primary system disk datastore +# Choose one visible on your node: e.g. local, local-zfs, zfs1, zfs2, zfs3 +# Run in Proxmox host: pvesh get /nodes//storage + +disk_storage = "local" + +# Datastore for any additional data disks (only used when nodes declare additional_disk_size) +additional_disk_storage = "zfs1" + +# --- Network --- +# Bridge for VM NICs (check Proxmox node network config) +network_bridge = "vmbr0" + +# --- Node Shape (override var.nodes entirely if you want custom layout) --- +# You can supply a custom nodes list by creating a tfvars entry like below. +# Omit this block to use the default nodes in variables.tf. +# Example: +# nodes = [ +# { name = "talos-master-00", vmid = 400, role = "controlplane", ip = "192.168.10.100", cores = 6, memory = 16000, disk_size = "48G", mac_address = "BC:24:11:A4:B2:97" }, +# { name = "talos-worker-01", vmid = 411, role = "worker", ip = "192.168.10.201", cores = 8, memory = 18000, disk_size = "64G", additional_disk_size = "712G", mac_address = "BC:24:11:4C:99:A2" } +# ] + +# After populating required values run: +# terraform plan -out .tfplan +# terraform apply .tfplan diff --git a/terraform/locals.tf b/terraform/locals.tf new file mode 100644 index 0000000..9bcb0cc --- /dev/null +++ b/terraform/locals.tf @@ -0,0 +1,85 @@ +locals { + # --- Primary Network Identification --- + control_plane_nodes_source = [for node in var.nodes : node if node.role == "controlplane"] + primary_network_cidr = local.control_plane_nodes_source[0].networks[0].cidr + primary_gateway = local.control_plane_nodes_source[0].networks[0].gateway + cluster_endpoint = length(local.control_plane_nodes_source) > 1 ? "https://${var.control_plane_vip}:6443" : "https://${local.control_plane_nodes_source[0].networks[0].ip}:6443" + + # --- Proxmox Network Resource Generation --- + + # 1. Flatten the list of all network assignments from all nodes. + all_networks_flat = flatten([ + for node in var.nodes : [ + for network in node.networks : { + proxmox_node = lookup(node, "proxmox_node", var.proxmox_node) + vlan_id = network.vlan_id + } + ] + ]) + + # 2. CORRECTED LOGIC: Group all network assignments by a unique key. + # The "..." syntax groups all items with the same key into a list. + # This creates a map where each value is a LIST of networks for that key. + grouped_bridges = { + for net in local.all_networks_flat : + "${net.proxmox_node}-${net.vlan_id}" => net... + } + + # 3. Build the final unique map for the resources to iterate over. + # We now loop over the de-duplicated, grouped map. We only need the first item ([0]) + # from each group to get the data we need to create the bridge. + unique_bridges = { + for key, group in local.grouped_bridges : key => { + proxmox_node = group[0].proxmox_node + vlan_id = group[0].vlan_id + bridge_name = "vmbr${group[0].vlan_id}" + phys_iface = lookup(var.node_interfaces, group[0].proxmox_node, var.default_physical_interface) + } + } + + talos_iso_download_url = "https://factory.talos.dev/image/${var.talos_factory_schematic_id}/${var.talos_version}/nocloud-${var.talos_image_architecture}.iso" + + image_target_nodes = toset(distinct([for node in var.nodes : lookup(node, "proxmox_node", var.proxmox_node)])) + + # --- VM Resource Transformation (No changes needed below this line) --- + all_nodes_transformed = { + for node in var.nodes : node.name => { + vmid = node.vmid + name = node.name + cores = node.cores + memory = node.memory + role = node.role + proxmox_node = lookup(node, "proxmox_node", var.proxmox_node) + disk_size = node.disk_size + disk_storage = var.disk_storage + tags = lookup(node, "tags", [node.role]) + iso_file = var.talos_iso_file + additional_disk_size = lookup(node, "additional_disk_size", null) + additional_disk_storage = lookup(node, "additional_disk_size", null) != null ? var.additional_disk_storage : null + + network_devices = [ + for net in node.networks : { + mac_address = net.mac_address + bridge = "vmbr${net.vlan_id}" + model = "virtio" + } + ] + + # Data for Talos config and outputs + ip = node.networks[0].ip + gateway = node.networks[0].gateway + network_cidr = node.networks[0].cidr + } + } + + node_configs = { + controlplane = { + cpu_type = "host", bios = "seabios", boot_order = ["scsi0", "ide2"], + description = "Talos Control Plane Node - Managed by Terraform" + } + worker = { + cpu_type = "host", bios = "seabios", boot_order = ["scsi0", "ide2"], + description = "Talos Worker Node - Managed by Terraform" + } + } +} \ No newline at end of file diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..1414ce2 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,136 @@ +terraform { + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "0.83.1" + } + null = { + source = "hashicorp/null" + version = "3.2.4" + } + } +} + +provider "proxmox" { + endpoint = var.proxmox_api_url + api_token = var.proxmox_api_token + insecure = true +} + +resource "null_resource" "stage_talos_image" { + for_each = var.talos_image_update_mode == "download" ? local.image_target_nodes : [] + + triggers = { + node_name = each.value + image_url = local.talos_iso_download_url + storage = var.talos_image_storage + talos_version = var.talos_version + } + + provisioner "local-exec" { + command = "python \"${path.module}/scripts/stage_talos_image.py\" --api-url \"${var.proxmox_api_url}\" --api-token \"${var.proxmox_api_token}\" --node \"${each.value}\" --storage \"${var.talos_image_storage}\" --image-url \"${local.talos_iso_download_url}\"" + } +} + +# Create all required VLAN interfaces based on the unique bridges map +resource "proxmox_virtual_environment_network_linux_vlan" "cluster_vlan" { + for_each = var.create_vlan_interface ? local.unique_bridges : {} + + node_name = each.value.proxmox_node + name = "${each.value.phys_iface}.${each.value.vlan_id}" +} + +# Create all required network bridges +resource "proxmox_virtual_environment_network_linux_bridge" "cluster_bridge" { + for_each = local.unique_bridges + depends_on = [proxmox_virtual_environment_network_linux_vlan.cluster_vlan] + + node_name = each.value.proxmox_node + name = each.value.bridge_name + comment = "Auto-created bridge for Talos VLAN ${each.value.vlan_id} on ${each.value.proxmox_node}" + ports = var.create_vlan_interface ? ["${each.value.phys_iface}.${each.value.vlan_id}"] : [each.value.phys_iface] + vlan_aware = false # The VLAN tagging is handled by the interface +} + +# Create the Virtual Machines +resource "proxmox_virtual_environment_vm" "vm" { + for_each = local.all_nodes_transformed + + name = each.value.name + node_name = each.value.proxmox_node + description = local.node_configs[each.value.role].description + tags = each.value.tags + vm_id = each.value.vmid + + started = true + bios = local.node_configs[each.value.role].bios + + # Enable QEMU Guest Agent for better VM management + agent { + enabled = true + } + + # Boot the disk first, then the ISO as fallback + boot_order = local.node_configs[each.value.role].boot_order + + cpu { + cores = each.value.cores + sockets = 1 + type = local.node_configs[each.value.role].cpu_type + } + + memory { + dedicated = each.value.memory + } + + # --- DYNAMIC NETWORK DEVICES --- + # This block iterates over the network_devices list for each node + dynamic "network_device" { + for_each = each.value.network_devices + content { + bridge = network_device.value.bridge + model = network_device.value.model + mac_address = network_device.value.mac_address + } + } + + # Primary disk + disk { + interface = "scsi0" + datastore_id = each.value.disk_storage + size = tonumber(trimsuffix(each.value.disk_size, "G")) + cache = "none" + discard = "ignore" + ssd = false + } + + # Additional disk for Rook Ceph storage (only if specified) + # This creates a raw passthrough disk that Rook can manage directly + dynamic "disk" { + for_each = each.value.additional_disk_size != null ? [1] : [] + content { + interface = "scsi1" + datastore_id = each.value.additional_disk_storage + size = tonumber(trimsuffix(each.value.additional_disk_size, "G")) + cache = "none" + discard = "ignore" + ssd = false + file_format = "raw" # Raw format for direct storage access + } + } + + # Attach Talos ISO + cdrom { + interface = "ide2" + file_id = each.value.iso_file + } + + # Ensure bridges are created before VMs + depends_on = [ + proxmox_virtual_environment_network_linux_bridge.cluster_bridge, + null_resource.stage_talos_image, + ] +} + +# (Your output definitions should still work, but you may want to update +# vm_details to show the list of MAC addresses and bridges) \ No newline at end of file diff --git a/terraform/scripts/stage_talos_image.py b/terraform/scripts/stage_talos_image.py new file mode 100644 index 0000000..610b356 --- /dev/null +++ b/terraform/scripts/stage_talos_image.py @@ -0,0 +1,60 @@ +import argparse +import json +import os +import sys +import urllib.error +import urllib.parse +import urllib.request + + +def api_request(base_url, token, method, path, data=None): + url = base_url.rstrip("/") + path + req = urllib.request.Request(url, method=method) + req.add_header("Authorization", f"PVEAPIToken={token}") + if data is not None: + payload = urllib.parse.urlencode(data).encode() + req.add_header("Content-Type", "application/x-www-form-urlencoded") + else: + payload = None + with urllib.request.urlopen(req, data=payload) as response: + return json.loads(response.read().decode()) + + +def ensure_image(base_url, token, node, storage, image_url): + filename = os.path.basename(urllib.parse.urlparse(image_url).path) + content_path = f"/api2/json/nodes/{node}/storage/{storage}/content" + response = api_request(base_url, token, "GET", content_path) + for item in response.get("data", []): + if item.get("volid", "").endswith(filename): + return filename, False + + download_path = f"/api2/json/nodes/{node}/storage/{storage}/download-url" + api_request(base_url, token, "POST", download_path, { + "content": "iso", + "filename": filename, + "url": image_url, + }) + return filename, True + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--api-url", required=True) + parser.add_argument("--api-token", required=True) + parser.add_argument("--node", required=True) + parser.add_argument("--storage", required=True) + parser.add_argument("--image-url", required=True) + args = parser.parse_args() + + try: + filename, downloaded = ensure_image(args.api_url, args.api_token, args.node, args.storage, args.image_url) + except urllib.error.HTTPError as exc: + sys.stderr.write(exc.read().decode()) + raise + + action = "downloaded" if downloaded else "already present" + print(f"{filename} {action} on {args.node}/{args.storage}") + + +if __name__ == "__main__": + main() diff --git a/terraform/talconfig_generator.tf b/terraform/talconfig_generator.tf new file mode 100644 index 0000000..96a5488 --- /dev/null +++ b/terraform/talconfig_generator.tf @@ -0,0 +1,79 @@ +# Generate talconfig.yaml from your terraform variables and locals +resource "local_file" "talconfig" { + filename = "${path.module}/../generated/talconfig.yaml" + content = templatefile("${path.module}/templates/talconfig.yaml.tpl", { + # Cluster configuration + cluster_name = var.cluster_name + cluster_endpoint = local.cluster_endpoint + cluster_domain = var.cluster_domain + talos_version = var.talos_version + talos_factory_schematic_id = var.talos_factory_schematic_id + control_plane_vip = var.control_plane_vip + + # Network configuration + dns_servers = var.dns_servers + gateway = local.primary_gateway + network_cidr = local.primary_network_cidr + network_mask = split("/", local.primary_network_cidr)[1] + + control_plane_nodes = [for node in var.nodes : { + name = node.name, + role = node.role, + networks = node.networks, + tags = node.tags, + proxmox_node = node.proxmox_node + } if node.role == "controlplane"] + + compute_workers = [for node in var.nodes : { + name = node.name, + role = node.role, + networks = node.networks, + tags = node.tags, + proxmox_node = node.proxmox_node + } if node.role == "worker" && contains(lookup(node, "tags", []), "compute")] + + storage_workers = [for node in var.nodes : { + name = node.name, + role = node.role, + networks = node.networks, + tags = node.tags, + proxmox_node = node.proxmox_node, + additional_disk_size = lookup(node, "additional_disk_size", "") + } if node.role == "worker" && contains(lookup(node, "tags", []), "storage")] + + gpu_workers = [for node in var.nodes : { + name = node.name, + role = node.role, + networks = node.networks, + tags = node.tags, + proxmox_node = node.proxmox_node + } if node.role == "worker" && contains(lookup(node, "tags", []), "gpu")] + + # Other variables + cluster_pod_nets = var.cluster_pod_nets + cluster_svc_nets = var.cluster_svc_nets + cni_name = var.cni_name + + # --- Pass the proxy config --- + image_cache_proxy = var.image_cache_proxy + }) +} + +# (The output block is already correct from the last fix) +output "node_organization" { + description = "How your nodes are organized for Talos configuration" + value = { + control_plane_nodes = [for node in var.nodes : { + name = node.name, ip = node.networks[0].ip, tags = lookup(node, "tags", []) + } if node.role == "controlplane"] + compute_workers = [for node in var.nodes : { + name = node.name, ip = node.networks[0].ip, tags = lookup(node, "tags", []) + } if node.role == "worker" && contains(lookup(node, "tags", []), "compute")] + storage_workers = [for node in var.nodes : { + name = node.name, ip = node.networks[0].ip, tags = lookup(node, "tags", []), additional_disk = lookup(node, "additional_disk_size", null) + } if node.role == "worker" && contains(lookup(node, "tags", []), "storage")] + gpu_workers = [for node in var.nodes : { + name = node.name, ip = node.networks[0].ip, tags = lookup(node, "tags", []) + } if node.role == "worker" && contains(lookup(node, "tags", []), "gpu")] + } +} \ No newline at end of file diff --git a/terraform/templates/talconfig.yaml.tpl b/terraform/templates/talconfig.yaml.tpl new file mode 100644 index 0000000..35f6ea2 --- /dev/null +++ b/terraform/templates/talconfig.yaml.tpl @@ -0,0 +1,198 @@ +clusterName: ${cluster_name} +endpoint: ${cluster_endpoint} +domain: ${cluster_domain} +talosVersion: ${talos_version} +allowSchedulingOnMasters: false + +# Network configuration +clusterPodNets: +%{ for net in cluster_pod_nets ~} + - ${net} +%{ endfor ~} +clusterSvcNets: +%{ for net in cluster_svc_nets ~} + - ${net} +%{ endfor ~} + +# Node definitions +nodes: +%{ for node in control_plane_nodes ~} + # Control Plane: ${node.name} + - hostname: ${node.name} + ipAddress: ${node.networks[0].ip} + controlPlane: true + installDisk: /dev/sda + machineSpec: + mode: metal + arch: amd64 + talosImageURL: "factory.talos.dev/installer/${talos_factory_schematic_id}/${talos_version}" + nodeLabels: + node-role.kubernetes.io/control-plane: "" + cluster.local/node-pool: "control-plane" + cluster.local/proxmox-node: "${lookup(node, "proxmox_node", "unknown")}" +%{ for tag in lookup(node, "tags", []) ~} + cluster.local/tag-${tag}: "true" +%{ endfor ~} + networkInterfaces: +%{ for i, net in node.networks ~} + - interface: eth${i} +%{ if i == 0 ~} + addresses: + - "${net.ip}/${split("/", net.cidr)[1]}" + routes: + - network: 0.0.0.0/0 + gateway: ${net.gateway} + vip: + ip: ${control_plane_vip} +%{ else ~} + dhcp: true +%{ endif ~} +%{ endfor ~} +%{ endfor ~} + +%{ for node in compute_workers ~} + # Compute Worker: ${node.name} + - hostname: ${node.name} + ipAddress: ${node.networks[0].ip} + controlPlane: false + installDisk: /dev/sda + machineSpec: + mode: metal + arch: amd64 + talosImageURL: "factory.talos.dev/installer/${talos_factory_schematic_id}/${talos_version}" + nodeLabels: + node-role.kubernetes.io/worker: "" + cluster.local/node-pool: "compute" + cluster.local/workload-type: "cpu-intensive" + cluster.local/proxmox-node: "${lookup(node, "proxmox_node", "unknown")}" +%{ for tag in lookup(node, "tags", []) ~} + cluster.local/tag-${tag}: "true" +%{ endfor ~} + networkInterfaces: +%{ for i, net in node.networks ~} + - interface: eth${i} +%{ if i == 0 ~} + addresses: + - "${net.ip}/${split("/", net.cidr)[1]}" + routes: + - network: 0.0.0.0/0 + gateway: ${net.gateway} +%{ else ~} + dhcp: true +%{ endif ~} +%{ endfor ~} +%{ endfor ~} + +%{ for node in storage_workers ~} + # Storage Worker: ${node.name} + - hostname: ${node.name} + ipAddress: ${node.networks[0].ip} + controlPlane: false + installDisk: /dev/sda + machineSpec: + mode: metal + arch: amd64 + talosImageURL: "factory.talos.dev/installer/${talos_factory_schematic_id}/${talos_version}" + nodeLabels: + node-role.kubernetes.io/worker: "" + cluster.local/node-pool: "storage" + cluster.local/workload-type: "storage-intensive" + cluster.local/proxmox-node: "${lookup(node, "proxmox_node", "unknown")}" +%{ for tag in lookup(node, "tags", []) ~} + cluster.local/tag-${tag}: "true" +%{ endfor ~} + networkInterfaces: +%{ for i, net in node.networks ~} + - interface: eth${i} +%{ if i == 0 ~} + addresses: + - "${net.ip}/${split("/", net.cidr)[1]}" + routes: + - network: 0.0.0.0/0 + gateway: ${net.gateway} +%{ else ~} + dhcp: true +%{ endif ~} +%{ endfor ~} +%{ endfor ~} + +%{ for node in gpu_workers ~} + # GPU Worker: ${node.name} + - hostname: ${node.name} + ipAddress: ${node.networks[0].ip} + controlPlane: false + installDisk: /dev/sda + machineSpec: + mode: metal + arch: amd64 + talosImageURL: "factory.talos.dev/installer/${talos_factory_schematic_id}/${talos_version}" + nodeLabels: + node-role.kubernetes.io/worker: "" + cluster.local/node-pool: "gpu" + cluster.local/workload-type: "gpu-intensive" + cluster.local/proxmox-node: "${lookup(node, "proxmox_node", "unknown")}" +%{ for tag in lookup(node, "tags", []) ~} + cluster.local/tag-${tag}: "true" +%{ endfor ~} + networkInterfaces: +%{ for i, net in node.networks ~} + - interface: eth${i} +%{ if i == 0 ~} + addresses: + - "${net.ip}/${split("/", net.cidr)[1]}" + routes: + - network: 0.0.0.0/0 + gateway: ${net.gateway} +%{ else ~} + dhcp: true +%{ endif ~} +%{ endfor ~} +%{ endfor ~} + +# Global patches applied to ALL nodes +patches: + - |- + machine: + network: + nameservers: +%{ for dns in dns_servers ~} + - "${dns}" +%{ endfor ~} + +%{ if image_cache_proxy.enabled } + # Registry Proxy Configuration + - |- + machine: + registries: + mirrors: + docker.io: + endpoints: + - http://${image_cache_proxy.ip}:${image_cache_proxy.port} + quay.io: + endpoints: + - http://${image_cache_proxy.ip}:${image_cache_proxy.port} + gcr.io: + endpoints: + - http://${image_cache_proxy.ip}:${image_cache_proxy.port} + ghcr.io: + endpoints: + - http://${image_cache_proxy.ip}:${image_cache_proxy.port} + k8s.gcr.io: + endpoints: + - http://${image_cache_proxy.ip}:${image_cache_proxy.port} + registry.k8s.io: + endpoints: + - http://${image_cache_proxy.ip}:${image_cache_proxy.port} +%{ endif } + +# Control plane specific configuration +controlPlane: + patches: + - |- + cluster: + etcd: + advertisedSubnets: + - ${network_cidr} + proxy: {} + discovery: + enabled: true \ No newline at end of file diff --git a/terraform/terraform.examples.tfvars b/terraform/terraform.examples.tfvars new file mode 100644 index 0000000..ecd2845 --- /dev/null +++ b/terraform/terraform.examples.tfvars @@ -0,0 +1,120 @@ +# --- Proxmox Connection --- +proxmox_api_url = "https://192.168.0.9:8006/api2/json" #! Change me +proxmox_node = "proxmox-node-name" #! Default/fallback node +proxmox_api_token = "root@pam!talos-iac=xxx-xxx-xxx-xxx" #! Change me +proxmox_pool = "" + +# --- Cluster Configuration --- +cluster_name = "proxmox-talos-cluster" #! Change me +cluster_domain = "example.com" #! Change me +talos_factory_schematic_id = "ddf38e55e30aa9b2bb0b765054ed63444dc00244ab8bb4ad4ad92486602285f8" +talos_version = "v1.11.1" +cni_name = "flannel" +talos_iso_file = "local:iso/talos-1.11.1.iso" +talos_image_update_mode = "manual" +talos_image_storage = "local" + +# --- Storage Configuration --- +disk_storage = "local-lvm" +additional_disk_storage = "local-lvm" + +# --- Network Configuration --- +dns_servers = ["192.168.43.1", "192.168.0.222", "1.1.1.1"] +control_plane_vip = "192.168.43.7" +node_interfaces = { + "proxmox-node-name" = "enp6s0" + # "pingu5" = "ens18" +} +create_vlan_interface = true + +# --- TrueNAS Image Cache Proxy --- +image_cache_proxy = { + enabled = true + ip = "192.168.43.100" + port = 3128 +} + +# --- Cluster Node Definitions --- +nodes = [ + { + name = "talos-master-01" + vmid = 1050 + role = "controlplane" + cores = 2 + memory = 4096 + disk_size = "48G" + tags = ["talos", "controlplane"] + proxmox_node = "pingu4" + networks = [ + { + mac_address = "BC:24:11:A4:B2:10" + cidr = "192.168.43.0/28" + ip = "192.168.43.2" + gateway = "192.168.43.1" + vlan_id = 43 + }, + { + mac_address = "BC:24:11:A4:B3:10" + cidr = "192.168.43.16/28" + vlan_id = 431 + } + ] + }, + { + name = "talos-master-02" + vmid = 1051 + role = "controlplane" + cores = 2 + memory = 4096 + disk_size = "48G" + tags = ["talos", "controlplane"] + proxmox_node = "pingu5" + networks = [ + { + mac_address = "BC:24:11:A4:B2:11" + cidr = "192.168.43.0/28" + ip = "192.168.43.3" + gateway = "192.168.43.1" + vlan_id = 43 + } + ] + }, + { + name = "talos-master-03" + vmid = 1052 + role = "controlplane" + cores = 2 + memory = 4096 + disk_size = "48G" + tags = ["talos", "controlplane"] + proxmox_node = "pingu6" + networks = [ + { + mac_address = "BC:24:11:A4:B2:12" + cidr = "192.168.43.0/28" + ip = "192.168.43.4" + gateway = "192.168.43.1" + vlan_id = 43 + } + ] + }, + { + name = "talos-worker-01" + vmid = 1060 + role = "worker" + cores = 4 + memory = 8192 + disk_size = "64G" + tags = ["talos", "worker", "compute"] + proxmox_node = "pingu4" + networks = [ + { + mac_address = "BC:24:11:A4:B2:20" + cidr = "192.168.43.0/28" + ip = "192.168.43.5" + gateway = "192.168.43.1" + vlan_id = 43 + } + ] + } +] diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000..fe2b837 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,190 @@ +# --- Proxmox Configuration --- +variable "proxmox_api_url" { + description = "Proxmox VE API URL" + type = string +} + +variable "proxmox_api_token" { + description = "Proxmox VE API token" + type = string + sensitive = true +} + +variable "proxmox_node" { + description = "Default Proxmox node name (can be overridden per node)" + type = string +} + +variable "proxmox_pool" { + description = "Proxmox resource pool" + type = string + default = "" +} + +# --- Cluster Configuration --- +variable "cluster_name" { + description = "The name of the Talos cluster." + type = string + default = "proxmox-talos-cluster" +} + +variable "cluster_domain" { + description = "The domain name of the Talos cluster." + type = string + default = "cluster.local" +} + +variable "talos_version" { + description = "The version of Talos Linux to use." + type = string +} + +variable "talos_factory_schematic_id" { + description = "Talos Factory schematic ID used for installer image URLs." + type = string +} + +variable "cni_name" { + description = "The name of the CNI to configure (e.g., 'flannel', 'cilium')." + type = string + default = "flannel" +} + +# --- Global Network Configuration --- +variable "dns_servers" { + description = "Global DNS servers for the cluster" + type = list(string) + default = ["1.1.1.1", "1.0.0.1"] +} + +variable "control_plane_vip" { + description = "Virtual IP for HA control plane (only used when multiple control plane nodes)" + type = string +} + +variable "cluster_pod_nets" { + description = "The CIDR blocks for Kubernetes pods." + type = list(string) + default = ["10.244.0.0/16"] +} + +variable "cluster_svc_nets" { + description = "The CIDR blocks for Kubernetes services." + type = list(string) + default = ["10.96.0.0/12"] +} + +# --- Storage Configuration --- +variable "disk_storage" { + description = "Proxmox storage for primary disk" + type = string +} + +variable "additional_disk_storage" { + description = "Proxmox storage for additional disks" + type = string +} + +variable "talos_iso_file" { + description = "Talos ISO file in Proxmox storage" + type = string +} + +variable "talos_image_update_mode" { + description = "How the Talos image should be staged on Proxmox nodes." + type = string + default = "manual" + + validation { + condition = contains(["manual", "download"], var.talos_image_update_mode) + error_message = "talos_image_update_mode must be either 'manual' or 'download'." + } +} + +variable "talos_image_storage" { + description = "Proxmox storage target for downloaded Talos ISO images." + type = string + default = "local" +} + +variable "talos_image_architecture" { + description = "Talos image architecture used for factory downloads." + type = string + default = "amd64" +} + +# --- Node Configuration --- +variable "nodes" { + description = "List of cluster nodes, each with its own network configurations." + type = list(object({ + name = string + vmid = number + role = string # "controlplane" or "worker" + cores = number + memory = number + disk_size = string + tags = optional(list(string)) + additional_disk_size = optional(string) + proxmox_node = optional(string) + + # Define one or more networks per node. + # The first network in the list is considered the primary for IP and gateway settings. + networks = list(object({ + mac_address = string + cidr = string + vlan_id = number + ip = optional(string) # Static IP, required for control plane, optional for workers + gateway = optional(string) # Should only be defined on the primary interface + })) + })) + + validation { + condition = alltrue([for node in var.nodes : length(node.networks) >= 1]) + error_message = "Each node must have at least one network defined." + } + + validation { + condition = alltrue([for node in var.nodes : node.role == "controlplane" ? node.networks[0].ip != null : true]) + error_message = "The first network for a 'controlplane' node must have a static IP address." + } +} + +variable "default_physical_interface" { + description = "Default physical interface for nodes (fallback)" + type = string + default = "enp6s0" +} + +variable "node_interfaces" { + description = "Map of Proxmox node names to their physical interfaces" + type = map(string) + default = {} + + # Example: + # { + # "pve1" = "enp1s0" + # "pve2" = "ens18" + # "pve3" = "enp6s0" + # } +} + +variable "create_vlan_interface" { + description = "Whether to create VLAN interfaces (true for tagged VLANs, false for untagged)" + type = bool + default = true +} + +# --- Registry Proxy Configuration --- +variable "image_cache_proxy" { + description = "Configuration for the TrueNAS container image registry proxy/mirror." + type = object({ + enabled = bool + ip = string + port = number + }) + default = { + enabled = false + ip = "" + port = 3128 + } +} \ No newline at end of file diff --git a/terraform/version.auto.tfvars.example b/terraform/version.auto.tfvars.example new file mode 100644 index 0000000..28597f9 --- /dev/null +++ b/terraform/version.auto.tfvars.example @@ -0,0 +1,3 @@ +talos_factory_schematic_id = "ddf38e55e30aa9b2bb0b765054ed63444dc00244ab8bb4ad4ad92486602285f8" +talos_version = "v1.11.1" +talos_iso_file = "local:iso/talos-1.11.1.iso"