Enable IPv4 BGP networking (#10)

- IPv4 networking with BGP peering to upstream router - Ethernet interface MTU configurable (9000 bytes) Co-authored-by: Marcel Straub <m@straubs.eu> Reviewed-on: #10
2025-09-08 21:54:03 +02:00
parent 700db1be08
commit 18941f9eec
12 changed files with 175 additions and 30 deletions
--- a/00-infrastructure/01-tofu/talos/config.tf
+++ b/00-infrastructure/01-tofu/talos/config.tf
@@ -72,12 +72,14 @@ data "talos_machine_configuration" "this" {
    templatefile("${path.module}/machine-config/control_plane.yaml.tftpl", {
      mac_address      = lower(each.value.mac_address)
      vip              = var.cluster.vip
+      cluster_mtu      = var.cluster.mtu
      extra_manifests  = jsonencode(local.extra_manifests)
      api_server       = var.cluster.api_server
      inline_manifests = jsonencode(terraform_data.bootstrap_inline_manifests.output)
    }) :
    templatefile("${path.module}/machine-config/worker.yaml.tftpl", {
      mac_address = lower(each.value.mac_address)
+      cluster_mtu      = var.cluster.mtu
    })
  ]
 }
--- a/00-infrastructure/01-tofu/talos/machine-config/common.yaml.tftpl
+++ b/00-infrastructure/01-tofu/talos/machine-config/common.yaml.tftpl
@@ -56,4 +56,11 @@ cluster:
  proxy:
    image: registry.k8s.io/kube-proxy:${kubernetes_version}
  scheduler:
-    image: registry.k8s.io/kube-scheduler:${kubernetes_version}
+    image: registry.k8s.io/kube-scheduler:${kubernetes_version}
+  network:
+    podSubnets:
+      - 10.244.0.0/16
+#      - 2a13:fc80:1:d200::/64
+    serviceSubnets:
+      - 10.96.0.0/12
+#      - 2a13:fc80:1:d201::/64
--- a/00-infrastructure/01-tofu/talos/machine-config/control_plane.yaml.tftpl
+++ b/00-infrastructure/01-tofu/talos/machine-config/control_plane.yaml.tftpl
@@ -5,6 +5,7 @@ machine:
      - deviceSelector:
          hardwareAddr: ${mac_address}
        dhcp: true
+        mtu: ${cluster_mtu}
 %{ if vip != null }
        vip:
          ip: ${vip}
@@ -19,13 +20,15 @@ cluster:
  controllerManager:
    extraArgs:
      bind-address: 0.0.0.0
+#      node-cidr-mask-size-ipv4: "24"
+#      node-cidr-mask-size-ipv6: "112"
  etcd:
    extraArgs:
      listen-metrics-urls: http://0.0.0.0:2381
  scheduler:
    extraArgs:
      bind-address: 0.0.0.0
-# Let's go with the default network
+  # Let's go with the default network
  network:
    cni:
      name: none
--- a/00-infrastructure/01-tofu/talos/machine-config/worker.yaml.tftpl
+++ b/00-infrastructure/01-tofu/talos/machine-config/worker.yaml.tftpl
@@ -3,4 +3,5 @@ machine:
    interfaces:
      - deviceSelector:
          hardwareAddr: ${mac_address}
-        dhcp: true
+        dhcp: true
+        mtu: ${cluster_mtu}
--- a/00-infrastructure/01-tofu/talos/variables.tf
+++ b/00-infrastructure/01-tofu/talos/variables.tf
@@ -17,6 +17,7 @@ variable "cluster" {
  type = object({
    name                         = string
    vip                          = optional(string)
+    mtu                          = optional(number, 1500)
    subnet_mask                  = optional(string, "24")
    talos_machine_config_version = optional(string)
    proxmox_cluster              = string
--- a/00-infrastructure/01-tofu/talos_cluster.auto.tfvars
+++ b/00-infrastructure/01-tofu/talos_cluster.auto.tfvars
@@ -8,6 +8,7 @@ talos_cluster_config  = {
    # http_proxy          = "http://100.64.0.1:3128"
    # no_proxy            = "10.0.0.0/8"

+    mtu                 = 9000
    vip                 = "10.51.10.100"
    kubernetes_version  = "v1.33.3"
    gateway_api_version = "v1.3.0"
--- a/00-infrastructure/01-tofu/variables.tf
+++ b/00-infrastructure/01-tofu/variables.tf
@@ -34,6 +34,7 @@ variable "talos_cluster_config" {
  type = object({
    name                         = string
    vip                          = optional(string)
+    mtu                          = optional(number, 1500)
    subnet_mask                  = optional(string, "24")
    talos_machine_config_version = optional(string)
    proxmox_cluster              = string
--- a/02-k8s/infra/network/cilium/bgp.yaml
+++ b/02-k8s/infra/network/cilium/bgp.yaml
@@ -0,0 +1,101 @@
+---
+apiVersion: cilium.io/v2
+kind: CiliumBGPClusterConfig
+metadata:
+  name: vy-eis-mk8-de-bgp
+spec:
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/os: linux
+  bgpInstances:
+    - name: "ASN65001"
+      localASN: 65001
+      peers:
+        - name: "vy-eis-mk8-de-1-v6"
+          peerASN: 65000
+          peerAddress: 2a13:fc80:1:a::1
+          peerConfigRef:
+            name: "vy-eis-mk8-de-bgp-1-peer-config"
+        - name: "vy-eis-mk8-de-1-v4"
+          peerASN: 65000
+          peerAddress: 10.51.10.1
+          peerConfigRef:
+            name: "vy-eis-mk8-de-bgp-1-peer-config"
+---
+apiVersion: cilium.io/v2
+kind: CiliumBGPPeerConfig
+metadata:
+  name: vy-eis-mk8-de-bgp-1-peer-config
+spec:
+
+  # Optimized timers for datacenter environments
+  timers:
+    connectRetryTimeSeconds: 5
+    holdTimeSeconds: 9
+    keepAliveTimeSeconds: 3
+  
+  # Enable graceful restart for zero-downtime operations
+  gracefulRestart:
+    enabled: true
+    restartTimeSeconds: 15
+  
+  # Custom transport configuration
+  transport:
+    peerPort: 179
+  
+  # Address family configuration with advertisements
+  families:
+  - afi: ipv4
+    safi: unicast
+    advertisements:
+      matchLabels:
+        advertise: "datacenter-bgp"
+  - afi: ipv6
+    safi: unicast
+    advertisements:
+      matchLabels:
+        advertise: "datacenter-bgp"
+---
+apiVersion: cilium.io/v2
+kind: CiliumBGPAdvertisement
+metadata:
+  name: pod-cidr-advertisements
+  labels:
+    advertise: "datacenter-bgp"
+spec:
+  advertisements:
+  - advertisementType: "PodCIDR"
+    attributes:
+      communities:
+        standard: ["65001:100", "65001:200"]
+        large: ["65001:100:1"]
+        wellKnown: ["no-export"]
+      localPreference: 200
+
+  # LoadBalancer services
+  - advertisementType: "Service"
+    service:
+      addresses:
+      - LoadBalancerIP
+    selector:
+      matchLabels:
+        service-type: "external"
+    attributes:
+      communities:
+        standard: ["65001:300"]
+      localPreference: 150
+  
+  # ClusterIP services for internal access
+  - advertisementType: "Service"
+    service:
+      addresses:
+      - ClusterIP
+    selector:
+      matchExpressions:
+      - key: "internal-bgp"
+        operator: "In"
+        values: ["enabled"]
+    attributes:
+      communities:
+        standard: ["65001:400"]
+      localPreference: 100
--- a/02-k8s/infra/network/cilium/ip-pool.yaml
+++ b/02-k8s/infra/network/cilium/ip-pool.yaml
@@ -1,8 +1,15 @@
 apiVersion: cilium.io/v2
 kind: CiliumLoadBalancerIPPool
 metadata:
-  name: internal-v4
+  name: public-v4-01
 spec:
  blocks:
-    - start: 10.51.10.64
-      stop: 10.51.10.71
+    - cidr: 185.83.87.48/29
+---
+apiVersion: cilium.io/v2
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: public-v6-01
+spec:
+  blocks:
+    - cidr: 2a13:fc80:0001:d201::/64
--- a/02-k8s/infra/network/cilium/kustomization.yaml
+++ b/02-k8s/infra/network/cilium/kustomization.yaml
@@ -4,6 +4,7 @@ kind: Kustomization
 resources:
  - announce.yaml
  - ip-pool.yaml
+  - bgp.yaml

 helmCharts:
  - name: cilium
--- a/02-k8s/infra/network/cilium/values.yaml
+++ b/02-k8s/infra/network/cilium/values.yaml
@@ -26,34 +26,34 @@ bpf:
 # https://docs.cilium.io/en/stable/network/concepts/ipam/
 ipam:
  mode: kubernetes
-  multiPoolPreAllocation: ""
-# mode: multi-pool
-# operator:
-#   autoCreateCiliumPodIPPools:
-#     default:
-#       ipv6:
-#         cidrs:
-#           - 2a13:fc80:0001:d200::/64
-#         maskSize: 120
-    
-# # TODO ########!!!!!!!!!!!!!!!!!!!!!!!!!!!!%%%%%%%%%%%%%%%%%%%%%%%%%%%55555
-# # The service subnet CIDR.
-# serviceSubnets:
-#   - 10.96.0.0/12
-#   - 2a13:fc80:0001:d201::/64

-# routingMode: native
+k8s:
+  requireIPv4PodCIDR: true
+  requireIPv6PodCIDR: false

-# bgpControlPlane:
-#   enabled: true
+ipv4:
+  enabled: true
+ipv6:
+  enabled: true

-# ipv4:
-#   enabled: true
-# ipv6:
-#   enabled: true
+# Avoid encapsulation for direct access
+routingMode: native
+
+#Route distribution gets managed by BGP
+bgpControlPlane:
+  enabled: true
+
+# Only BGP manages the routes
+# auto-direct-node-routes: true
+# direct-routing-skip-unreachable: true
+# The whole internet is directly reachable from each pod
+# ipv6-native-routing-cidr: ::/0
+ipv4-native-routing-cidr: 10.0.0.0/8
+
+# Disabling DNAT
+enableIPv4Masquerade: false
+enableIPv6Masquerade: false

-# enableIPv4Masquerade: false
-# enableIPv6Masquerade: false

 operator:
  rollOutPods: true
--- a/README.md
+++ b/README.md
@@ -8,6 +8,8 @@

 ### Network

+#### Node Network
+
 - IPv4 configuration uses DHCP with static MAC binding for easy bring-up
 - IPv6 addresses are manually assigned
 - DNS Zone: prod.k8.eis-mk8.de.s5b.org
@@ -16,6 +18,24 @@
 |--|--|--|
 |210|10.51.10.0/23|2a13:fc80:1:a::/64|

+#### Pod Network
+
+- IPv6 only
+- Prefix: 2a13:fc80:0001:d200::/64
+
+#### Service Network
+
+- Dual-Stack
+- IPv6 Prefix: 2a13:fc80:0001:d201::/64
+- IPv4 Prefix: 185.83.87.48/28
+
+#### BGP
+
+|ASN|Who|
+|--|--|
+|65000|Upstream Router|
+|65001|Cluster|
+
 ## How to use

 ### Prerequisites