Add static IPv6 default gateway configuration

00-infrastructure/01-tofu/talos/config.tf

@@ -51,7 +51,7 @@ data "talos_machine_configuration" "this" {
   cluster_name = var.cluster.name
   # This is the Kubernetes API Server endpoint.
   # ref - https://www.talos.dev/latest/introduction/prodnotes/#decide-the-kubernetes-endpoint
-  cluster_endpoint = "https://${local.kubernetes_endpoint}:6443"
+  cluster_endpoint = "https://[${local.kubernetes_endpoint}]:6443"
   # @formatter:off
   talos_version = var.cluster.talos_machine_config_version != null ? var.cluster.talos_machine_config_version : (each.value.update == true ? var.image.update_version : var.image.version)
   # @formatter:on
@@ -60,26 +60,45 @@ data "talos_machine_configuration" "this" {
   config_patches = [
     templatefile("${path.module}/machine-config/common.yaml.tftpl", {
       node_name          = each.value.host_node
+      ipv4_address       = each.value.ip
+      ipv6_address       = each.value.ipv6
       cluster_name       = var.cluster.proxmox_cluster
       kubernetes_version = var.cluster.kubernetes_version
       http_proxy         = var.cluster.http_proxy
       no_proxy           = var.cluster.no_proxy
       ntp_servers        = var.cluster.ntp_servers
+      name_servers       = var.cluster.name_servers
       registry_mirrors   = var.cluster.registry_mirrors
       hostname           = each.key
       kubelet            = var.cluster.kubelet
+      pod_subnets        = var.cluster.pod_subnets
+      service_subnets    = var.cluster.service_subnets
     }), each.value.machine_type == "controlplane" ?
     templatefile("${path.module}/machine-config/control_plane.yaml.tftpl", {
       mac_address      = lower(each.value.mac_address)
       vip              = var.cluster.vip
+      ipv4_address     = each.value.ip
+      ipv6_address     = each.value.ipv6
+      ipv4_gateway     = var.cluster.default4_gateway
+      ipv6_gateway     = var.cluster.default6_gateway
+      subnet_mask      = var.cluster.subnet_mask
+      subnet6_mask     = var.cluster.subnet6_mask
       cluster_mtu      = var.cluster.mtu
       extra_manifests  = jsonencode(local.extra_manifests)
       api_server       = var.cluster.api_server
+      pod_subnets      = var.cluster.pod_subnets
+      service_subnets  = var.cluster.service_subnets
       inline_manifests = jsonencode(terraform_data.bootstrap_inline_manifests.output)
     }) :
     templatefile("${path.module}/machine-config/worker.yaml.tftpl", {
-      mac_address = lower(each.value.mac_address)
-      cluster_mtu      = var.cluster.mtu
+      mac_address     = lower(each.value.mac_address)
+      ipv4_address    = each.value.ip
+      ipv6_address    = each.value.ipv6
+      ipv4_gateway    = var.cluster.default4_gateway
+      ipv6_gateway    = var.cluster.default6_gateway
+      subnet_mask     = var.cluster.subnet_mask
+      subnet6_mask    = var.cluster.subnet6_mask
+      cluster_mtu     = var.cluster.mtu
     })
   ]
 }
@@ -105,30 +124,30 @@ resource "talos_machine_bootstrap" "this" {
   client_configuration = talos_machine_secrets.this.client_configuration
 }
 
-data "talos_cluster_health" "this" {
-  depends_on = [
-    talos_machine_configuration_apply.this,
-    talos_machine_bootstrap.this
-  ]
-  skip_kubernetes_checks = false
-  client_configuration   = data.talos_client_configuration.this.client_configuration
-  control_plane_nodes    = [for k, v in var.nodes : v.ip if v.machine_type == "controlplane"]
-  worker_nodes           = [for k, v in var.nodes : v.ip if v.machine_type == "worker"]
-  endpoints              = data.talos_client_configuration.this.endpoints
-  timeouts = {
-    read = "10m"
-  }
-}
+# data "talos_cluster_health" "this" {
+#   depends_on = [
+#     talos_machine_configuration_apply.this,
+#     talos_machine_bootstrap.this
+#   ]
+#   skip_kubernetes_checks = false
+#   client_configuration   = data.talos_client_configuration.this.client_configuration
+#   control_plane_nodes    = [for k, v in var.nodes : v.ip if v.machine_type == "controlplane"]
+#   worker_nodes           = [for k, v in var.nodes : v.ip if v.machine_type == "worker"]
+#   endpoints              = data.talos_client_configuration.this.endpoints
+#   timeouts = {
+#     read = "10m"
+#   }
+# }
 
-resource "talos_cluster_kubeconfig" "this" {
-  depends_on = [
-    talos_machine_bootstrap.this,
-    data.talos_cluster_health.this
-  ]
-  # The kubeconfig endpoint will be populated from the talos_machine_configuration cluster_endpoint
-  node                 = local.first_control_plane_node_ip
-  client_configuration = talos_machine_secrets.this.client_configuration
-  timeouts = {
-    read = "1m"
-  }
-}
+# resource "talos_cluster_kubeconfig" "this" {
+#   depends_on = [
+#     talos_machine_bootstrap.this,
+#     data.talos_cluster_health.this
+#   ]
+#   # The kubeconfig endpoint will be populated from the talos_machine_configuration cluster_endpoint
+#   node                 = local.first_control_plane_node_ip
+#   client_configuration = talos_machine_secrets.this.client_configuration
+#   timeouts = {
+#     read = "1m"
+#   }
+# }

00-infrastructure/01-tofu/talos/machine-config/common.yaml.tftpl

@@ -17,6 +17,13 @@ machine:
       forwardKubeDNSToHost: false
   kubelet:
     image: ghcr.io/siderolabs/kubelet:${kubernetes_version}
+    extraArgs:
+      address: "::"
+      v: 1
+    nodeIP:
+      validSubnets:
+        - 10.0.0.0/8
+        - 2a13:fc80:1::/48
 %{if kubelet != ""}
     ${indent(4, kubelet)}
 %{endif}
@@ -25,6 +32,12 @@ machine:
     topology.kubernetes.io/zone: ${node_name}
   network:
     hostname: ${hostname}
+%{if length(name_servers) > 0 }
+    nameservers:
+%{ for addr in name_servers ~}
+      - ${addr}
+%{ endfor ~}
+%{endif}
 %{if length(ntp_servers) > 0 }
   time:
     servers:
@@ -57,16 +70,33 @@ machine:
 cluster:
   apiServer:
     image: registry.k8s.io/kube-apiserver:${kubernetes_version}
+    extraArgs:
+      bind-address: "::"
+      advertise-address: "${ipv6_address}"
+      service-cluster-ip-range: "${join(",", service_subnets)}"
   controllerManager:
     image: registry.k8s.io/kube-controller-manager:${kubernetes_version}
+    extraArgs:
+      bind-address: "::"
+      node-cidr-mask-size-ipv4: "24"
+      node-cidr-mask-size-ipv6: "112"
+      cluster-cidr: "${join(",", pod_subnets)}"
+      service-cluster-ip-range: "${join(",", service_subnets)}"
   proxy:
     image: registry.k8s.io/kube-proxy:${kubernetes_version}
+    extraArgs:
+      bind-address: "::"
+      cluster-cidr: "${join(",", pod_subnets)}"
   scheduler:
     image: registry.k8s.io/kube-scheduler:${kubernetes_version}
+    extraArgs:
+      bind-address: "::"
   network:
     podSubnets:
-      - 10.244.0.0/16
-#      - 2a13:fc80:1:d200::/64
+%{ for subnet in pod_subnets ~}
+      - ${subnet}
+%{ endfor ~}
     serviceSubnets:
-      - 10.96.0.0/16
-#      - 2a13:fc80:1:d201::/64
+%{ for subnet in service_subnets ~}
+      - ${subnet}
+%{ endfor ~}

00-infrastructure/01-tofu/talos/machine-config/control_plane.yaml.tftpl

@@ -4,8 +4,15 @@ machine:
     interfaces:
       - deviceSelector:
           hardwareAddr: ${mac_address}
-        dhcp: true
+        addresses:
+          - ${ipv4_address}/${subnet_mask}
+          - ${ipv6_address}/${subnet6_mask}
         mtu: ${cluster_mtu}
+        routes:
+          - network: 0.0.0.0/0
+            gateway: ${ipv4_gateway}
+          - network: ::/0
+            gateway: ${ipv6_gateway}
 %{ if vip != null }
         vip:
           ip: ${vip}
@@ -19,15 +26,13 @@ cluster:
 %{endif}    
   controllerManager:
     extraArgs:
-      bind-address: 0.0.0.0
       node-cidr-mask-size-ipv4: "24"
-#      node-cidr-mask-size-ipv6: "112"
+      node-cidr-mask-size-ipv6: "118"
+      cluster-cidr: "${join(",", pod_subnets)}"
+      service-cluster-ip-range: "${join(",", service_subnets)}"
   etcd:
     extraArgs:
-      listen-metrics-urls: http://0.0.0.0:2381
-  scheduler:
-    extraArgs:
-      bind-address: 0.0.0.0
+      listen-metrics-urls: http://[::]:2381
   # Let's go with the default network
   network:
     cni:

00-infrastructure/01-tofu/talos/machine-config/worker.yaml.tftpl

@@ -3,5 +3,12 @@ machine:
     interfaces:
       - deviceSelector:
           hardwareAddr: ${mac_address}
-        dhcp: true
-        mtu: ${cluster_mtu}
+        addresses:
+          - ${ipv4_address}/${subnet_mask}
+          - ${ipv6_address}/${subnet6_mask}
+        mtu: ${cluster_mtu}
+        routes:
+          - network: 0.0.0.0/0
+            gateway: ${ipv4_gateway}
+          - network: ::/0
+            gateway: ${ipv6_gateway}

00-infrastructure/01-tofu/talos/variables.tf

@@ -19,12 +19,18 @@ variable "cluster" {
     vip                          = optional(string)
     mtu                          = optional(number, 1500)
     subnet_mask                  = optional(string, "24")
+    subnet6_mask                 = optional(string, "64")
     talos_machine_config_version = optional(string)
+    default4_gateway             = string
+    default6_gateway             = string
     proxmox_cluster              = string
     kubernetes_version           = string
     gateway_api_version          = string
+    pod_subnets                  = list(string)
+    service_subnets              = list(string)
     node_network_vlan            = optional(number)
     ntp_servers                  = optional(list(string), [])
+    name_servers                 = optional(list(string), [])
     registry_mirrors             = map(string)
     http_proxy                   = optional(string, "")
     no_proxy                     = optional(string, "")
@@ -48,6 +54,7 @@ variable "nodes" {
     machine_type  = string
     datastore_id  = optional(string, "ZFS")
     ip            = string
+    ipv6          = string
     dns           = optional(list(string))
     mac_address   = string
     vm_id         = number

00-infrastructure/01-tofu/talos_cluster.auto.tfvars

@@ -5,14 +5,33 @@ talos_cluster_config  = {
     ntp_servers         = [
         "2a13:fc80:1:f000::1"
     ]
+    name_servers        = [
+        "2a13:fc80:1:f000::1",
+        "10.51.255.1"
+    ]
     # http_proxy          = "http://100.64.0.1:3128"
     # no_proxy            = "10.0.0.0/8"
 
     mtu                 = 9000
-    vip                 = "10.51.10.100"
+    default4_gateway    = "10.51.10.1"
+    default6_gateway    = "2a13:fc80:1:a::1"
+    subnet_mask         = "23"
+    subnet6_mask        = "64"
+    vip                 = "2a13:fc80:1:a:ffff::0"
     kubernetes_version  = "v1.34.0"
     gateway_api_version = "v1.3.0"
 
+    # order matters, first address family defined priamry address family
+    pod_subnets = [
+      "2a13:fc80:1:d200::/64",
+      "10.244.0.0/16"
+    ]
+    # order matters, first address family defined priamry address family
+    service_subnets = [
+      "2a13:fc80:1:d201::/112",
+      "10.96.0.0/16"
+    ]
+
     cilium = {
         bootstrap_manifest_path = "talos/inline-manifests/cilium_install.yaml"
         values_file_path        = "../../02-k8s/infra/network/cilium/values.yaml"

00-infrastructure/01-tofu/talos_nodes.auto.tfvars

@@ -4,6 +4,7 @@ talos_nodes = {
         host_node       = "pve01"
         machine_type    = "controlplane"
         ip              = "10.51.10.101"
+        ipv6            = "2a13:fc80:1:a:ffff::1"
         mac_address     = "BC:24:11:7B:76:3E"
         vm_id           = 301
         cpu             = 2
@@ -13,6 +14,7 @@ talos_nodes = {
         host_node       = "pve02"
         machine_type    = "controlplane"
         ip              = "10.51.10.102"
+        ipv6            = "2a13:fc80:1:a:ffff::2"
         mac_address     = "BC:24:11:16:85:7D"
         vm_id           = 302
         cpu             = 2
@@ -22,6 +24,7 @@ talos_nodes = {
         host_node       = "pve-oberon"
         machine_type    = "controlplane"
         ip              = "10.51.10.103"
+        ipv6            = "2a13:fc80:1:a:ffff::3"
         mac_address     = "BC:24:11:B8:B6:6F"
         vm_id           = 303
         cpu             = 2
@@ -32,6 +35,7 @@ talos_nodes = {
         host_node       = "pve01"
         machine_type    = "worker"
         ip              = "10.51.11.1"
+        ipv6            = "2a13:fc80:1:a:fffe::1"
         mac_address     = "BC:24:11:E1:E9:AE"
         vm_id           = 311
         cpu             = 4
@@ -41,6 +45,7 @@ talos_nodes = {
         host_node       = "pve02"
         machine_type    = "worker"
         ip              = "10.51.11.2"
+        ipv6            = "2a13:fc80:1:a:fffe::2"
         mac_address     = "BC:24:11:63:3A:85"
         vm_id           = 312
         cpu             = 4
@@ -50,6 +55,7 @@ talos_nodes = {
         host_node       = "pve-oberon"
         machine_type    = "worker"
         ip              = "10.51.11.3"
+        ipv6            = "2a13:fc80:1:a:fffe::3"
         mac_address     = "BC:24:11:8E:75:0E"
         vm_id           = 313
         cpu             = 4

00-infrastructure/01-tofu/variables.tf

@@ -35,15 +35,23 @@ variable "talos_cluster_config" {
     name                         = string
     vip                          = optional(string)
     mtu                          = optional(number, 1500)
+    # network mask of the talos node
     subnet_mask                  = optional(string, "24")
+    # network mask of the talos node
+    subnet6_mask                 = optional(string, "64")
     talos_machine_config_version = optional(string)
+    default4_gateway             = string
+    default6_gateway             = string
     proxmox_cluster              = string
     kubernetes_version           = string
     gateway_api_version          = string
+    pod_subnets                  = list(string)
+    service_subnets              = list(string)
     node_network_vlan            = optional(number)
     http_proxy                   = optional(string)
     no_proxy                     = optional(string)
     ntp_servers                  = optional(list(string), [])
+    name_servers                 = optional(list(string), []) 
     registry_mirrors             = map(string)
     extra_manifests              = optional(list(string), [])
     kubelet                      = optional(string, "")
@@ -64,6 +72,7 @@ variable "talos_nodes" {
       host_node     = string
       machine_type  = string
       ip            = string
+      ipv6          = string
       dns           = optional(list(string))
       mac_address   = string
       vm_id         = number

02-k8s/infra/network/cilium/values.yaml

@@ -28,14 +28,14 @@ bpf:
 ipam:
   mode: kubernetes
 
-# k8s:
-#   requireIPv4PodCIDR: true
-#   requireIPv6PodCIDR: false
+k8s:
+  requireIPv4PodCIDR: true
+  requireIPv6PodCIDR: true
 
 ipv4:
   enabled: true
 ipv6:
-  enabled: false
+  enabled: true
 
 # Avoid encapsulation for direct access
 routingMode: native
@@ -54,8 +54,8 @@ bgpControlPlane:
 # auto-direct-node-routes: true
 # direct-routing-skip-unreachable: true
 # The whole internet is directly reachable from each pod
-# ipv6-native-routing-cidr: ::/0
-ipv4-native-routing-cidr: 10.0.0.0/8
+ipv6-native-routing-cidr: ::/0
+ipv4-native-routing-cidr: 0.0.0.0/0
 
 # Disabling DNAT
 enableIPv4Masquerade: false