fix: Make Cilium IPv4 BGP network fully functional including proof-of-concept application (#11)

- Upgraded to
  - Talos v1.11 (newest)
  - Kubernetes v1.34
- Added test application for LB-IPAM
- Fixed IPv4 stack:
  - BGP advertisement of Services (ClusterIP, ExternalIP and LoadBalancerIP)
  - Fixed CoreDNS (integration between Talos HostDNS and Cilium BPF)

Co-authored-by: Marcel Straub <m@straubs.eu>
Reviewed-on: #11

00-infrastructure/01-tofu/talos/machine-config/common.yaml.tftpl

@@ -9,6 +9,12 @@ machine:
         overridePath: true
 %{endfor}
 %{endif}
+  # https://github.com/siderolabs/talos/issues/10002
+  # Incompatibility with Cilium BPF routing and Talos Host DNS
+  features:
+    hostDNS:
+      enabled: true
+      forwardKubeDNSToHost: false
   kubelet:
     image: ghcr.io/siderolabs/kubelet:${kubernetes_version}
 %{if kubelet != ""}
@@ -62,5 +68,5 @@ cluster:
       - 10.244.0.0/16
 #      - 2a13:fc80:1:d200::/64
     serviceSubnets:
-      - 10.96.0.0/12
+      - 10.96.0.0/16
 #      - 2a13:fc80:1:d201::/64

00-infrastructure/01-tofu/talos/machine-config/control_plane.yaml.tftpl

@@ -20,7 +20,7 @@ cluster:
   controllerManager:
     extraArgs:
       bind-address: 0.0.0.0
-#      node-cidr-mask-size-ipv4: "24"
+      node-cidr-mask-size-ipv4: "24"
 #      node-cidr-mask-size-ipv6: "112"
   etcd:
     extraArgs:

00-infrastructure/01-tofu/talos_cluster.auto.tfvars

@@ -10,7 +10,7 @@ talos_cluster_config  = {
 
     mtu                 = 9000
     vip                 = "10.51.10.100"
-    kubernetes_version  = "v1.33.3"
+    kubernetes_version  = "v1.34.0"
     gateway_api_version = "v1.3.0"
 
     cilium = {

00-infrastructure/01-tofu/talos_image.auto.tfvars

@@ -1,6 +1,6 @@
 talos_image = {
-    version         = "v1.10.7"
-    update_version  = "v1.10.7"
+    version         = "v1.11.0"
+    update_version  = "v1.11.0"
     schematic_path  = "talos/image/schematic.yaml"
     #update_schematic_path = "talos/image/schematic.yaml"
 }

02-k8s/infra/network/cilium/bgp.yaml

@@ -16,11 +16,11 @@ spec:
           peerAddress: 2a13:fc80:1:a::1
           peerConfigRef:
             name: "vy-eis-mk8-de-bgp-1-peer-config"
-        - name: "vy-eis-mk8-de-1-v4"
-          peerASN: 65000
-          peerAddress: 10.51.10.1
-          peerConfigRef:
-            name: "vy-eis-mk8-de-bgp-1-peer-config"
+        # - name: "vy-eis-mk8-de-1-v4"
+        #   peerASN: 65000
+        #   peerAddress: 10.51.10.1
+        #   peerConfigRef:
+        #     name: "vy-eis-mk8-de-bgp-1-peer-config"
 ---
 apiVersion: cilium.io/v2
 kind: CiliumBGPPeerConfig
@@ -72,30 +72,49 @@ spec:
         wellKnown: ["no-export"]
       localPreference: 200
 
-  # LoadBalancer services
   - advertisementType: "Service"
     service:
       addresses:
-      - LoadBalancerIP
-    selector:
-      matchLabels:
-        service-type: "external"
-    attributes:
-      communities:
-        standard: ["65001:300"]
-      localPreference: 150
-  
-  # ClusterIP services for internal access
-  - advertisementType: "Service"
-    service:
-      addresses:
-      - ClusterIP
-    selector:
+        - ClusterIP
+        - ExternalIP
+        - LoadBalancerIP
+    selector:             # select all services by a dummy expression always matching
       matchExpressions:
-      - key: "internal-bgp"
-        operator: "In"
-        values: ["enabled"]
-    attributes:
-      communities:
-        standard: ["65001:400"]
-      localPreference: 100
+        - {key: somekey, operator: NotIn, values: ['never-used-value']}
+
+  # That at least works for the PodCIDR
+  # - advertisementType: "PodCIDR"
+  #   attributes:
+  #     communities:
+  #       standard: ["65001:100", "65001:200"]
+  #       large: ["65001:100:1"]
+  #       wellKnown: ["no-export"]
+  #     localPreference: 200
+
+  # # LoadBalancer services
+  # - advertisementType: "Service"
+  #   service:
+  #     addresses:
+  #     - LoadBalancerIP
+  #   # selector:
+  #   #   matchLabels:
+  #   #     service-type: "external"
+  #   attributes:
+  #     communities:
+  #       standard: ["65001:300"]
+  #     localPreference: 150
+  
+  # # ClusterIP services for internal access
+  # - advertisementType: "Service"
+  #   service:
+  #     addresses:
+  #     - ClusterIP
+  #   # selector:
+  #   #   matchExpressions:
+  #   #   - key: "internal-bgp"
+  #   #     operator: "In"
+  #   #     values: ["enabled"]
+  #   attributes:
+  #     communities:
+  #       standard: ["65001:400"]
+  #     localPreference: 100

02-k8s/infra/network/cilium/values.yaml

@@ -18,23 +18,24 @@ cgroup:
     enabled: false
   hostRoot: /sys/fs/cgroup
 
+bpf:
+  lbExternalClusterIP: true
 # https://www.talos.dev/latest/talos-guides/network/host-dns/#forwarding-kube-dns-to-host-dns
 # https://docs.cilium.io/en/stable/operations/performance/tuning/#ebpf-host-routing
-bpf:
-  hostLegacyRouting: true
+#   hostLegacyRouting: true
 
 # https://docs.cilium.io/en/stable/network/concepts/ipam/
 ipam:
   mode: kubernetes
 
-k8s:
-  requireIPv4PodCIDR: true
-  requireIPv6PodCIDR: false
+# k8s:
+#   requireIPv4PodCIDR: true
+#   requireIPv6PodCIDR: false
 
 ipv4:
   enabled: true
 ipv6:
-  enabled: true
+  enabled: false
 
 # Avoid encapsulation for direct access
 routingMode: native
@@ -43,6 +44,12 @@ routingMode: native
 bgpControlPlane:
   enabled: true
 
+# enable instead of bgpControlPlane
+# l2announcements:
+#   enabled: false
+# externalIPs:
+#   enabled: false
+
 # Only BGP manages the routes
 # auto-direct-node-routes: true
 # direct-routing-skip-unreachable: true
@@ -54,6 +61,14 @@ ipv4-native-routing-cidr: 10.0.0.0/8
 enableIPv4Masquerade: false
 enableIPv6Masquerade: false
 
+enableIPv6BIGTCP: true
+
+bandwidthManager:
+  enabled: true
+  bbr: true
+
+#debug:
+#  enabled: true
 
 operator:
   rollOutPods: true
@@ -83,20 +98,11 @@ resources:
     cpu: 200m
     memory: 512Mi
 
-#debug:
-#  enabled: true
-
 # Increase rate limit when doing L2 announcements
 k8sClientRateLimit:
   qps: 20
   burst: 100
 
-l2announcements:
-  enabled: true
-
-externalIPs:
-  enabled: true
-
 loadBalancer:
   # https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/#maglev-consistent-hashing
   algorithm: maglev

90-debugging/test_application.yaml

@@ -0,0 +1,49 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: load-balancer-example
+  name: hello-world
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: load-balancer-example
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: load-balancer-example
+    spec:
+      containers:
+      - image: gcr.io/google-samples/hello-app:2.0
+        name: hello-world
+        ports:
+          - containerPort: 8080
+            name: http-web-svc
+        securityContext:
+          seccompProfile:
+            type: "RuntimeDefault"
+          allowPrivilegeEscalation: false
+          runAsNonRoot: true
+          readOnlyRootFilesystem: true
+          runAsUser: 1000
+          capabilities:
+            drop: ["ALL"]
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: hello-world-service
+  namespace: default
+  labels:
+    app.kubernetes.io/name: load-balancer-example
+spec:
+  selector:
+    app.kubernetes.io/name: load-balancer-example
+  type: LoadBalancer
+  loadBalancerClass: io.cilium/bgp-control-plane
+  # ipFamilyPolicy: RequireDualStack
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: http-web-svc

README.md

@@ -20,8 +20,13 @@
 
 #### Pod Network
 
-- IPv6 only
-- Prefix: 2a13:fc80:0001:d200::/64
+- IPv4 only
+- IPv4 prefix: 10.244.0.0/16
+- Potential IPv6 prefix: 2a13:fc80:0001:d200::/64
+
+##### Important side-notes
+
+- DNS resolver (Vyos) these networks must be whitelisted to allow recursive DNS
 
 #### Service Network