From eb00ba2c5a549a674353ceb063f97df2a2865663 Mon Sep 17 00:00:00 2001 From: Marcel Straub Date: Sat, 13 Sep 2025 23:28:37 +0200 Subject: [PATCH] fix: Make Cilium IPv4 BGP network fully functional including proof-of-concept application (#11) - Upgraded to - Talos v1.11 (newest) - Kubernetes v1.34 - Added test application for LB-IPAM - Fixed IPv4 stack: - BGP advertisement of Services (ClusterIP, ExternalIP and LoadBalancerIP) - Fixed CoreDNS (integration between Talos HostDNS and Cilium BPF) Co-authored-by: Marcel Straub Reviewed-on: https://git.straubintra.net/s5b-public/k8s/pulls/11 --- .../talos/machine-config/common.yaml.tftpl | 8 +- .../machine-config/control_plane.yaml.tftpl | 2 +- .../01-tofu/talos_cluster.auto.tfvars | 2 +- .../01-tofu/talos_image.auto.tfvars | 4 +- 02-k8s/infra/network/cilium/bgp.yaml | 75 ++++++++++++------- 02-k8s/infra/network/cilium/values.yaml | 36 +++++---- 90-debugging/test_application.yaml | 49 ++++++++++++ README.md | 9 ++- 8 files changed, 135 insertions(+), 50 deletions(-) create mode 100644 90-debugging/test_application.yaml diff --git a/00-infrastructure/01-tofu/talos/machine-config/common.yaml.tftpl b/00-infrastructure/01-tofu/talos/machine-config/common.yaml.tftpl index b52b6ae..7595fe8 100644 --- a/00-infrastructure/01-tofu/talos/machine-config/common.yaml.tftpl +++ b/00-infrastructure/01-tofu/talos/machine-config/common.yaml.tftpl @@ -9,6 +9,12 @@ machine: overridePath: true %{endfor} %{endif} + # https://github.com/siderolabs/talos/issues/10002 + # Incompatibility with Cilium BPF routing and Talos Host DNS + features: + hostDNS: + enabled: true + forwardKubeDNSToHost: false kubelet: image: ghcr.io/siderolabs/kubelet:${kubernetes_version} %{if kubelet != ""} @@ -62,5 +68,5 @@ cluster: - 10.244.0.0/16 # - 2a13:fc80:1:d200::/64 serviceSubnets: - - 10.96.0.0/12 + - 10.96.0.0/16 # - 2a13:fc80:1:d201::/64 \ No newline at end of file diff --git a/00-infrastructure/01-tofu/talos/machine-config/control_plane.yaml.tftpl b/00-infrastructure/01-tofu/talos/machine-config/control_plane.yaml.tftpl index f099161..725119a 100644 --- a/00-infrastructure/01-tofu/talos/machine-config/control_plane.yaml.tftpl +++ b/00-infrastructure/01-tofu/talos/machine-config/control_plane.yaml.tftpl @@ -20,7 +20,7 @@ cluster: controllerManager: extraArgs: bind-address: 0.0.0.0 -# node-cidr-mask-size-ipv4: "24" + node-cidr-mask-size-ipv4: "24" # node-cidr-mask-size-ipv6: "112" etcd: extraArgs: diff --git a/00-infrastructure/01-tofu/talos_cluster.auto.tfvars b/00-infrastructure/01-tofu/talos_cluster.auto.tfvars index 78e38e9..488dd5b 100644 --- a/00-infrastructure/01-tofu/talos_cluster.auto.tfvars +++ b/00-infrastructure/01-tofu/talos_cluster.auto.tfvars @@ -10,7 +10,7 @@ talos_cluster_config = { mtu = 9000 vip = "10.51.10.100" - kubernetes_version = "v1.33.3" + kubernetes_version = "v1.34.0" gateway_api_version = "v1.3.0" cilium = { diff --git a/00-infrastructure/01-tofu/talos_image.auto.tfvars b/00-infrastructure/01-tofu/talos_image.auto.tfvars index b267206..b4ffed1 100644 --- a/00-infrastructure/01-tofu/talos_image.auto.tfvars +++ b/00-infrastructure/01-tofu/talos_image.auto.tfvars @@ -1,6 +1,6 @@ talos_image = { - version = "v1.10.7" - update_version = "v1.10.7" + version = "v1.11.0" + update_version = "v1.11.0" schematic_path = "talos/image/schematic.yaml" #update_schematic_path = "talos/image/schematic.yaml" } \ No newline at end of file diff --git a/02-k8s/infra/network/cilium/bgp.yaml b/02-k8s/infra/network/cilium/bgp.yaml index 5129b6c..08c828e 100644 --- a/02-k8s/infra/network/cilium/bgp.yaml +++ b/02-k8s/infra/network/cilium/bgp.yaml @@ -16,11 +16,11 @@ spec: peerAddress: 2a13:fc80:1:a::1 peerConfigRef: name: "vy-eis-mk8-de-bgp-1-peer-config" - - name: "vy-eis-mk8-de-1-v4" - peerASN: 65000 - peerAddress: 10.51.10.1 - peerConfigRef: - name: "vy-eis-mk8-de-bgp-1-peer-config" + # - name: "vy-eis-mk8-de-1-v4" + # peerASN: 65000 + # peerAddress: 10.51.10.1 + # peerConfigRef: + # name: "vy-eis-mk8-de-bgp-1-peer-config" --- apiVersion: cilium.io/v2 kind: CiliumBGPPeerConfig @@ -72,30 +72,49 @@ spec: wellKnown: ["no-export"] localPreference: 200 - # LoadBalancer services - advertisementType: "Service" service: addresses: - - LoadBalancerIP - selector: - matchLabels: - service-type: "external" - attributes: - communities: - standard: ["65001:300"] - localPreference: 150 - - # ClusterIP services for internal access - - advertisementType: "Service" - service: - addresses: - - ClusterIP - selector: + - ClusterIP + - ExternalIP + - LoadBalancerIP + selector: # select all services by a dummy expression always matching matchExpressions: - - key: "internal-bgp" - operator: "In" - values: ["enabled"] - attributes: - communities: - standard: ["65001:400"] - localPreference: 100 \ No newline at end of file + - {key: somekey, operator: NotIn, values: ['never-used-value']} + + # That at least works for the PodCIDR + # - advertisementType: "PodCIDR" + # attributes: + # communities: + # standard: ["65001:100", "65001:200"] + # large: ["65001:100:1"] + # wellKnown: ["no-export"] + # localPreference: 200 + + # # LoadBalancer services + # - advertisementType: "Service" + # service: + # addresses: + # - LoadBalancerIP + # # selector: + # # matchLabels: + # # service-type: "external" + # attributes: + # communities: + # standard: ["65001:300"] + # localPreference: 150 + + # # ClusterIP services for internal access + # - advertisementType: "Service" + # service: + # addresses: + # - ClusterIP + # # selector: + # # matchExpressions: + # # - key: "internal-bgp" + # # operator: "In" + # # values: ["enabled"] + # attributes: + # communities: + # standard: ["65001:400"] + # localPreference: 100 \ No newline at end of file diff --git a/02-k8s/infra/network/cilium/values.yaml b/02-k8s/infra/network/cilium/values.yaml index 69b1129..c5e738e 100644 --- a/02-k8s/infra/network/cilium/values.yaml +++ b/02-k8s/infra/network/cilium/values.yaml @@ -18,23 +18,24 @@ cgroup: enabled: false hostRoot: /sys/fs/cgroup +bpf: + lbExternalClusterIP: true # https://www.talos.dev/latest/talos-guides/network/host-dns/#forwarding-kube-dns-to-host-dns # https://docs.cilium.io/en/stable/operations/performance/tuning/#ebpf-host-routing -bpf: - hostLegacyRouting: true +# hostLegacyRouting: true # https://docs.cilium.io/en/stable/network/concepts/ipam/ ipam: mode: kubernetes -k8s: - requireIPv4PodCIDR: true - requireIPv6PodCIDR: false +# k8s: +# requireIPv4PodCIDR: true +# requireIPv6PodCIDR: false ipv4: enabled: true ipv6: - enabled: true + enabled: false # Avoid encapsulation for direct access routingMode: native @@ -43,6 +44,12 @@ routingMode: native bgpControlPlane: enabled: true +# enable instead of bgpControlPlane +# l2announcements: +# enabled: false +# externalIPs: +# enabled: false + # Only BGP manages the routes # auto-direct-node-routes: true # direct-routing-skip-unreachable: true @@ -54,6 +61,14 @@ ipv4-native-routing-cidr: 10.0.0.0/8 enableIPv4Masquerade: false enableIPv6Masquerade: false +enableIPv6BIGTCP: true + +bandwidthManager: + enabled: true + bbr: true + +#debug: +# enabled: true operator: rollOutPods: true @@ -83,20 +98,11 @@ resources: cpu: 200m memory: 512Mi -#debug: -# enabled: true - # Increase rate limit when doing L2 announcements k8sClientRateLimit: qps: 20 burst: 100 -l2announcements: - enabled: true - -externalIPs: - enabled: true - loadBalancer: # https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/#maglev-consistent-hashing algorithm: maglev diff --git a/90-debugging/test_application.yaml b/90-debugging/test_application.yaml new file mode 100644 index 0000000..baad7da --- /dev/null +++ b/90-debugging/test_application.yaml @@ -0,0 +1,49 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: load-balancer-example + name: hello-world +spec: + replicas: 3 + selector: + matchLabels: + app.kubernetes.io/name: load-balancer-example + template: + metadata: + labels: + app.kubernetes.io/name: load-balancer-example + spec: + containers: + - image: gcr.io/google-samples/hello-app:2.0 + name: hello-world + ports: + - containerPort: 8080 + name: http-web-svc + securityContext: + seccompProfile: + type: "RuntimeDefault" + allowPrivilegeEscalation: false + runAsNonRoot: true + readOnlyRootFilesystem: true + runAsUser: 1000 + capabilities: + drop: ["ALL"] +--- +apiVersion: v1 +kind: Service +metadata: + name: hello-world-service + namespace: default + labels: + app.kubernetes.io/name: load-balancer-example +spec: + selector: + app.kubernetes.io/name: load-balancer-example + type: LoadBalancer + loadBalancerClass: io.cilium/bgp-control-plane + # ipFamilyPolicy: RequireDualStack + ports: + - protocol: TCP + port: 80 + targetPort: http-web-svc diff --git a/README.md b/README.md index dd215a7..429c296 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,13 @@ #### Pod Network -- IPv6 only -- Prefix: 2a13:fc80:0001:d200::/64 +- IPv4 only +- IPv4 prefix: 10.244.0.0/16 +- Potential IPv6 prefix: 2a13:fc80:0001:d200::/64 + +##### Important side-notes + +- DNS resolver (Vyos) these networks must be whitelisted to allow recursive DNS #### Service Network