link to cilium documentation

- Upgraded to
  - Talos v1.11 (newest)
  - Kubernetes v1.34
- Added test application for LB-IPAM
- Fixed IPv4 stack:
  - BGP advertisement of Services (ClusterIP, ExternalIP and LoadBalancerIP)
  - Fixed CoreDNS (integration between Talos HostDNS and Cilium BPF)

Co-authored-by: Marcel Straub <m@straubs.eu>
Reviewed-on: #11

00-infrastructure/01-tofu/talos/config.tf

@@ -72,12 +72,14 @@ data "talos_machine_configuration" "this" {
     templatefile("${path.module}/machine-config/control_plane.yaml.tftpl", {
       mac_address      = lower(each.value.mac_address)
       vip              = var.cluster.vip
+      cluster_mtu      = var.cluster.mtu
       extra_manifests  = jsonencode(local.extra_manifests)
       api_server       = var.cluster.api_server
       inline_manifests = jsonencode(terraform_data.bootstrap_inline_manifests.output)
     }) :
     templatefile("${path.module}/machine-config/worker.yaml.tftpl", {
       mac_address = lower(each.value.mac_address)
+      cluster_mtu      = var.cluster.mtu
     })
   ]
 }

00-infrastructure/01-tofu/talos/machine-config/common.yaml.tftpl

@@ -9,6 +9,12 @@ machine:
         overridePath: true
 %{endfor}
 %{endif}
+  # https://github.com/siderolabs/talos/issues/10002
+  # Incompatibility with Cilium BPF routing and Talos Host DNS
+  features:
+    hostDNS:
+      enabled: true
+      forwardKubeDNSToHost: false
   kubelet:
     image: ghcr.io/siderolabs/kubelet:${kubernetes_version}
 %{if kubelet != ""}
@@ -56,4 +62,11 @@ cluster:
   proxy:
     image: registry.k8s.io/kube-proxy:${kubernetes_version}
   scheduler:
-    image: registry.k8s.io/kube-scheduler:${kubernetes_version}
+    image: registry.k8s.io/kube-scheduler:${kubernetes_version}
+  network:
+    podSubnets:
+      - 10.244.0.0/16
+#      - 2a13:fc80:1:d200::/64
+    serviceSubnets:
+      - 10.96.0.0/16
+#      - 2a13:fc80:1:d201::/64

00-infrastructure/01-tofu/talos/machine-config/control_plane.yaml.tftpl

@@ -5,6 +5,7 @@ machine:
       - deviceSelector:
           hardwareAddr: ${mac_address}
         dhcp: true
+        mtu: ${cluster_mtu}
 %{ if vip != null }
         vip:
           ip: ${vip}
@@ -19,13 +20,15 @@ cluster:
   controllerManager:
     extraArgs:
       bind-address: 0.0.0.0
+      node-cidr-mask-size-ipv4: "24"
+#      node-cidr-mask-size-ipv6: "112"
   etcd:
     extraArgs:
       listen-metrics-urls: http://0.0.0.0:2381
   scheduler:
     extraArgs:
       bind-address: 0.0.0.0
-# Let's go with the default network
+  # Let's go with the default network
   network:
     cni:
       name: none

00-infrastructure/01-tofu/talos/machine-config/worker.yaml.tftpl

@@ -3,4 +3,5 @@ machine:
     interfaces:
       - deviceSelector:
           hardwareAddr: ${mac_address}
-        dhcp: true
+        dhcp: true
+        mtu: ${cluster_mtu}

00-infrastructure/01-tofu/talos/variables.tf

@@ -17,6 +17,7 @@ variable "cluster" {
   type = object({
     name                         = string
     vip                          = optional(string)
+    mtu                          = optional(number, 1500)
     subnet_mask                  = optional(string, "24")
     talos_machine_config_version = optional(string)
     proxmox_cluster              = string

00-infrastructure/01-tofu/talos_cluster.auto.tfvars

@@ -8,8 +8,9 @@ talos_cluster_config  = {
     # http_proxy          = "http://100.64.0.1:3128"
     # no_proxy            = "10.0.0.0/8"
 
+    mtu                 = 9000
     vip                 = "10.51.10.100"
-    kubernetes_version  = "v1.33.3"
+    kubernetes_version  = "v1.34.0"
     gateway_api_version = "v1.3.0"
 
     cilium = {

00-infrastructure/01-tofu/talos_image.auto.tfvars

@@ -1,6 +1,6 @@
 talos_image = {
-    version         = "v1.10.7"
-    update_version  = "v1.10.7"
+    version         = "v1.11.0"
+    update_version  = "v1.11.0"
     schematic_path  = "talos/image/schematic.yaml"
     #update_schematic_path = "talos/image/schematic.yaml"
 }

00-infrastructure/01-tofu/variables.tf

@@ -34,6 +34,7 @@ variable "talos_cluster_config" {
   type = object({
     name                         = string
     vip                          = optional(string)
+    mtu                          = optional(number, 1500)
     subnet_mask                  = optional(string, "24")
     talos_machine_config_version = optional(string)
     proxmox_cluster              = string

00-infrastructure/README.md

@@ -37,6 +37,48 @@ deb-src [signed-by=/etc/apt/keyrings/opentofu.gpg,/etc/apt/keyrings/opentofu-rep
 sudo apt install tofu
 ```
 
+### Deploy Talos cluster
+
+The initial deployment of the Talos cluster consists of two steps:
+
+1. Install Talos
+2. Configure Cilium networking
+3. Configure Argo CD
+
+For the future, it is planned to fully automate step 2 and 3 and integrate it into the initial ``tofu apply``.
+
+#### Setup the bare Talos kubernetes
+
+- Go to directory ``01-tofu``
+- Execute ``tofu apply``
+- Configure Kubernetes Cluster access by executing
+
+```shell
+talosctl -n 10.51.10.101 --endpoints 10.51.10.100 --talosconfig=output/talos-config.yaml kubeconfig
+```
+
+#### Configure Cilium Networking
+
+Before we can configure any other services, the K8s network must be operational.
+
+Go to directory ``../02-k8s`` and execute:
+
+```shell
+kustomize build --enable-helm infra/network/cilium | kubectl apply -f -
+```
+
+Check that the cluster now gets into a healthy state:
+
+```shell
+talosctl -n 10.51.10.101 --endpoints 10.51.10.100 --talosconfig=output/talos-config.yaml health
+```
+
+How to configure the router etc.pp. can be found at [Cilium/README.MD](../02-k8s/infra/network/cilium/README.md).
+
+#### Configure ArgoCD
+
+[Follow K8s documentation](../02-k8s/README.md).
+
 ## Important tasks
 
 ### Reuse machine secrets
@@ -54,11 +96,11 @@ kubectl run -i --tty --rm debug --image=harbor.prod.eis-mk8.de.s5b.org/internal/
 ```
 
 If the container is not automatically removed, use
+
 ```shell
 kubectl delete pod debug
 ```
 
-
 ## Literature
 - [Talos Kubernetes on Proxmox using OpenTofu](https://blog.stonegarden.dev/articles/2024/08/talos-proxmox-tofu/)
 - [Talos on Proxmox with Terraform (multiple node pools)](https://github.com/sergelogvinov/terraform-talos/tree/main/proxmox)

02-k8s/.gitignore

@@ -0,0 +1,3 @@
+# Kube Seal backup
+kubeseal.cert
+kubeseal.key

02-k8s/README.md

@@ -14,6 +14,12 @@ Deployment of ArgoCD itself:
 kustomize build --enable-helm infra/controllers/argocd | kubectl apply -f -
 ```
 
+You can uninstall it with the command
+
+```shell
+kustomize build --enable-helm infra/controllers/argocd | kubectl delete -f -
+```
+
 Configure ArgoCD to pull our setup from this repo:
 
 ```shell
@@ -34,4 +40,19 @@ kubectl port-forward svc/argocd-server -n argocd 8080:443
 1. CLI Login with previously retrieved password and username ``admin``:
 ```shell
 argocd login localhost:8080 --insecure
+```
+
+## Other
+
+### What's keepign a namespace in state deleting?
+
+```shell
+kubectl api-resources --verbs=list --namespaced -o name \
+  | xargs -n 1 kubectl get --show-kind --ignore-not-found -n argocd
+```
+
+### Delete ArgoCD applications stuck in dependencies
+
+```shell
+kubectl -nargocd patch applications/controllers  -p '{"metadata": {"finalizers": null}}' --type merge
 ```

02-k8s/fetch_kubeseal_certs.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/bash
+
+rm kubeseal.cert kubeseal.key 2> /dev/null
+
+# public cert
+(umask 0077 && kubeseal --controller-namespace=sealed-secrets --fetch-cert > kubeseal.cert)
+
+# full cert backup
+(umask 0077 && kubectl get secret -n sealed-secrets -l sealedsecrets.bitnami.com/sealed-secrets-key -o yaml > kubeseal.key)

02-k8s/infra/application-set.yaml

@@ -4,7 +4,7 @@ metadata:
   name: infrastructure
   namespace: argocd
   labels:
-    dev.stonegarden: infrastructure
+    s5b.org: infrastructure
 spec:
   generators:
     - git:
@@ -16,7 +16,7 @@ spec:
     metadata:
       name: '{{ path.basename }}'
       labels:
-        dev.stonegarden: infrastructure
+        s5b.org: infrastructure
     spec:
       project: infrastructure
       source:

02-k8s/infra/controllers/argocd/values.yaml

@@ -7,12 +7,9 @@ configs:
     create: true
     application.resourceTrackingMethod: annotation+label
     admin.enabled: true
-    kustomize.buildOptions: --enable-helm
+    kustomize.buildOptions: "--enable-helm --load-restrictor LoadRestrictionsNone --enable-alpha-plugins --enable-exec"
     # url: https://gitops.prod.k8.eis-mk8.de.s5b.org
-      
 
-  cmp:
-    create: true
   params:
     controller.diff.server.side: true
     server.insecure: false
@@ -84,12 +81,6 @@ repoServer:
       enabled: true
   containerSecurityContext:
     readOnlyRootFilesystem: true
-  volumes:
-    - name: cmp-kustomize-build-with-helm
-      configMap:
-        name: argocd-cmp-cm
-    - name: cmp-tmp
-      emptyDir: { }
   resources:
     requests:
       cpu: 100m

02-k8s/infra/controllers/sealed-secrets/kustomization.yaml

@@ -1,10 +1,13 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
+resources:
+  - ns.yaml
+
 helmCharts:
   - name: sealed-secrets
     repo: oci://registry-1.docker.io/bitnamicharts
-    version: 2.17.4
+    version: 2.5.19
     releaseName: sealed-secrets-controller
     namespace: sealed-secrets
     includeCRDs: true

02-k8s/infra/controllers/sealed-secrets/ns.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: sealed-secrets

02-k8s/infra/network/application-set.yaml

@@ -0,0 +1,34 @@
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: network
+  namespace: argocd
+  labels:
+    s5b.org: network
+spec:
+  generators:
+    - git:
+        repoURL: https://git.straubintra.net/s5b-public/k8s.git
+        revision: HEAD
+        directories:
+          - path: 02-k8s/infra/network/*
+  template:
+    metadata:
+      name: '{{ path.basename }}'
+      labels:
+        s5b.org: network
+    spec:
+      project: network
+      source:
+        repoURL: https://git.straubintra.net/s5b-public/k8s.git
+        targetRevision: HEAD
+        path: '{{ path }}'
+      destination:
+        name: in-cluster
+        namespace: argocd
+      syncPolicy:
+        automated:
+          selfHeal: true
+          prune: true
+        syncOptions:
+          - ServerSideApply=true

02-k8s/infra/network/cilium/README.md

@@ -0,0 +1,56 @@
+# Cilium Configuration
+
+## Architecture
+
+- Use native routing
+- Announce IP addresses with BGP to routers
+  - Router:
+    - ASN: 65000
+    - Address: 2a13:fc80:1:a::1
+  - K8S cluster:
+    - ASN: 65001
+    - Loadbalance IP Pools:
+      - ``public-v4-01``: 185.83.87.48/29
+      - ``public-v6-01``: 2a13:fc80:1:d201::/64
+    - Export:
+      - PodCIDR
+      - Service (ClusterIP, ExternalIP, LoadBalanceIP)
+
+## Prepare router for BGP setup
+
+The network router is assumed to be a VyOS router. The configuration assumes that the Kubernetes network is on interface ``eth1.210``.
+
+```vyos
+ bgp {
+     listen {
+         range 10.51.10.0/23 {
+             peer-group k8s-prod
+         }
+         range 2a13:fc80:1:a::/64 {
+             peer-group k8s-prod
+         }
+     }
+     neighbor eth1.210 {
+         description "K8s - Production Cluster"
+         interface {
+             peer-group k8s-prod
+         }
+     }
+     peer-group k8s-prod {
+         address-family {
+             ipv4-unicast {
+                 route-map {
+                     import ASN65001
+                 }
+             }
+             ipv6-unicast {
+                 route-map {
+                     import ASN65001
+                 }
+             }
+         }
+         remote-as 65001
+     }
+     system-as 65000
+ }
+```

02-k8s/infra/network/cilium/announce.yaml

@@ -0,0 +1,8 @@
+apiVersion: cilium.io/v2alpha1
+kind: CiliumL2AnnouncementPolicy
+metadata:
+  name: default-l2-announcement-policy
+  namespace: kube-system
+spec:
+  externalIPs: true
+  loadBalancerIPs: true

02-k8s/infra/network/cilium/bgp.yaml

@@ -0,0 +1,120 @@
+---
+apiVersion: cilium.io/v2
+kind: CiliumBGPClusterConfig
+metadata:
+  name: vy-eis-mk8-de-bgp
+spec:
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/os: linux
+  bgpInstances:
+    - name: "ASN65001"
+      localASN: 65001
+      peers:
+        - name: "vy-eis-mk8-de-1-v6"
+          peerASN: 65000
+          peerAddress: 2a13:fc80:1:a::1
+          peerConfigRef:
+            name: "vy-eis-mk8-de-bgp-1-peer-config"
+        # - name: "vy-eis-mk8-de-1-v4"
+        #   peerASN: 65000
+        #   peerAddress: 10.51.10.1
+        #   peerConfigRef:
+        #     name: "vy-eis-mk8-de-bgp-1-peer-config"
+---
+apiVersion: cilium.io/v2
+kind: CiliumBGPPeerConfig
+metadata:
+  name: vy-eis-mk8-de-bgp-1-peer-config
+spec:
+
+  # Optimized timers for datacenter environments
+  timers:
+    connectRetryTimeSeconds: 5
+    holdTimeSeconds: 9
+    keepAliveTimeSeconds: 3
+  
+  # Enable graceful restart for zero-downtime operations
+  gracefulRestart:
+    enabled: true
+    restartTimeSeconds: 15
+  
+  # Custom transport configuration
+  transport:
+    peerPort: 179
+  
+  # Address family configuration with advertisements
+  families:
+  - afi: ipv4
+    safi: unicast
+    advertisements:
+      matchLabels:
+        advertise: "datacenter-bgp"
+  - afi: ipv6
+    safi: unicast
+    advertisements:
+      matchLabels:
+        advertise: "datacenter-bgp"
+---
+apiVersion: cilium.io/v2
+kind: CiliumBGPAdvertisement
+metadata:
+  name: pod-cidr-advertisements
+  labels:
+    advertise: "datacenter-bgp"
+spec:
+  advertisements:
+  - advertisementType: "PodCIDR"
+    attributes:
+      communities:
+        standard: ["65001:100", "65001:200"]
+        large: ["65001:100:1"]
+        wellKnown: ["no-export"]
+      localPreference: 200
+
+  - advertisementType: "Service"
+    service:
+      addresses:
+        - ClusterIP
+        - ExternalIP
+        - LoadBalancerIP
+    selector:             # select all services by a dummy expression always matching
+      matchExpressions:
+        - {key: somekey, operator: NotIn, values: ['never-used-value']}
+
+  # That at least works for the PodCIDR
+  # - advertisementType: "PodCIDR"
+  #   attributes:
+  #     communities:
+  #       standard: ["65001:100", "65001:200"]
+  #       large: ["65001:100:1"]
+  #       wellKnown: ["no-export"]
+  #     localPreference: 200
+
+  # # LoadBalancer services
+  # - advertisementType: "Service"
+  #   service:
+  #     addresses:
+  #     - LoadBalancerIP
+  #   # selector:
+  #   #   matchLabels:
+  #   #     service-type: "external"
+  #   attributes:
+  #     communities:
+  #       standard: ["65001:300"]
+  #     localPreference: 150
+  
+  # # ClusterIP services for internal access
+  # - advertisementType: "Service"
+  #   service:
+  #     addresses:
+  #     - ClusterIP
+  #   # selector:
+  #   #   matchExpressions:
+  #   #   - key: "internal-bgp"
+  #   #     operator: "In"
+  #   #     values: ["enabled"]
+  #   attributes:
+  #     communities:
+  #       standard: ["65001:400"]
+  #     localPreference: 100

02-k8s/infra/network/cilium/ip-pool.yaml

@@ -0,0 +1,15 @@
+apiVersion: cilium.io/v2
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: public-v4-01
+spec:
+  blocks:
+    - cidr: 185.83.87.48/29
+---
+apiVersion: cilium.io/v2
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: public-v6-01
+spec:
+  blocks:
+    - cidr: 2a13:fc80:0001:d201::/64

02-k8s/infra/network/cilium/kustomization.yaml

@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - announce.yaml
+  - ip-pool.yaml
+  - bgp.yaml
+
+helmCharts:
+  - name: cilium
+    repo: https://helm.cilium.io
+    version: 1.18.0 # renovate: github-releases=cilium/cilium
+    releaseName: "cilium"
+    includeCRDs: true
+    namespace: kube-system
+    valuesFile: values.yaml

02-k8s/infra/network/cilium/values.yaml

@@ -18,42 +18,57 @@ cgroup:
     enabled: false
   hostRoot: /sys/fs/cgroup
 
+bpf:
+  lbExternalClusterIP: true
 # https://www.talos.dev/latest/talos-guides/network/host-dns/#forwarding-kube-dns-to-host-dns
 # https://docs.cilium.io/en/stable/operations/performance/tuning/#ebpf-host-routing
-bpf:
-  hostLegacyRouting: true
+#   hostLegacyRouting: true
 
 # https://docs.cilium.io/en/stable/network/concepts/ipam/
 ipam:
   mode: kubernetes
-  multiPoolPreAllocation: ""
-# mode: multi-pool
-# operator:
-#   autoCreateCiliumPodIPPools:
-#     default:
-#       ipv6:
-#         cidrs:
-#           - 2a13:fc80:0001:d200::/64
-#         maskSize: 120
-    
-# # TODO ########!!!!!!!!!!!!!!!!!!!!!!!!!!!!%%%%%%%%%%%%%%%%%%%%%%%%%%%55555
-# # The service subnet CIDR.
-# serviceSubnets:
-#   - 10.96.0.0/12
-#   - 2a13:fc80:0001:d201::/64
 
-# routingMode: native
+# k8s:
+#   requireIPv4PodCIDR: true
+#   requireIPv6PodCIDR: false
 
-# bgpControlPlane:
-#   enabled: true
+ipv4:
+  enabled: true
+ipv6:
+  enabled: false
 
-# ipv4:
-#   enabled: true
-# ipv6:
-#   enabled: true
+# Avoid encapsulation for direct access
+routingMode: native
 
-# enableIPv4Masquerade: false
-# enableIPv6Masquerade: false
+#Route distribution gets managed by BGP
+bgpControlPlane:
+  enabled: true
+
+# enable instead of bgpControlPlane
+# l2announcements:
+#   enabled: false
+# externalIPs:
+#   enabled: false
+
+# Only BGP manages the routes
+# auto-direct-node-routes: true
+# direct-routing-skip-unreachable: true
+# The whole internet is directly reachable from each pod
+# ipv6-native-routing-cidr: ::/0
+ipv4-native-routing-cidr: 10.0.0.0/8
+
+# Disabling DNAT
+enableIPv4Masquerade: false
+enableIPv6Masquerade: false
+
+enableIPv6BIGTCP: true
+
+bandwidthManager:
+  enabled: true
+  bbr: true
+
+#debug:
+#  enabled: true
 
 operator:
   rollOutPods: true
@@ -83,20 +98,11 @@ resources:
     cpu: 200m
     memory: 512Mi
 
-#debug:
-#  enabled: true
-
 # Increase rate limit when doing L2 announcements
 k8sClientRateLimit:
   qps: 20
   burst: 100
 
-l2announcements:
-  enabled: true
-
-externalIPs:
-  enabled: true
-
 loadBalancer:
   # https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/#maglev-consistent-hashing
   algorithm: maglev

02-k8s/infra/network/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+commonLabels:
+  s5b.org: network
+  app.kubernetes.io/managed-by: argocd
+
+resources:
+  - project.yaml
+  - application-set.yaml

02-k8s/infra/network/project.yaml

@@ -0,0 +1,18 @@
+apiVersion: argoproj.io/v1alpha1
+kind: AppProject
+metadata:
+  name: network
+  namespace: argocd
+spec:
+  sourceRepos:
+    - 'https://git.straubintra.net/s5b-public/k8s.git'
+  destinations:
+    - namespace: 'argocd'
+      server: '*'
+    - namespace: 'kube-system'
+      server: '*'
+    - namespace: 'cilium-secrets'
+      server: '*'
+  clusterResourceWhitelist:
+    - group: '*'
+      kind: '*'

90-debugging/test_application.yaml

@@ -0,0 +1,49 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: load-balancer-example
+  name: hello-world
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: load-balancer-example
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: load-balancer-example
+    spec:
+      containers:
+      - image: gcr.io/google-samples/hello-app:2.0
+        name: hello-world
+        ports:
+          - containerPort: 8080
+            name: http-web-svc
+        securityContext:
+          seccompProfile:
+            type: "RuntimeDefault"
+          allowPrivilegeEscalation: false
+          runAsNonRoot: true
+          readOnlyRootFilesystem: true
+          runAsUser: 1000
+          capabilities:
+            drop: ["ALL"]
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: hello-world-service
+  namespace: default
+  labels:
+    app.kubernetes.io/name: load-balancer-example
+spec:
+  selector:
+    app.kubernetes.io/name: load-balancer-example
+  type: LoadBalancer
+  loadBalancerClass: io.cilium/bgp-control-plane
+  # ipFamilyPolicy: RequireDualStack
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: http-web-svc

README.md

@@ -8,6 +8,8 @@
 
 ### Network
 
+#### Node Network
+
 - IPv4 configuration uses DHCP with static MAC binding for easy bring-up
 - IPv6 addresses are manually assigned
 - DNS Zone: prod.k8.eis-mk8.de.s5b.org
@@ -16,6 +18,29 @@
 |--|--|--|
 |210|10.51.10.0/23|2a13:fc80:1:a::/64|
 
+#### Pod Network
+
+- IPv4 only
+- IPv4 prefix: 10.244.0.0/16
+- Potential IPv6 prefix: 2a13:fc80:0001:d200::/64
+
+##### Important side-notes
+
+- DNS resolver (Vyos) these networks must be whitelisted to allow recursive DNS
+
+#### Service Network
+
+- Dual-Stack
+- IPv6 Prefix: 2a13:fc80:0001:d201::/64
+- IPv4 Prefix: 185.83.87.48/28
+
+#### BGP
+
+|ASN|Who|
+|--|--|
+|65000|Upstream Router|
+|65001|Cluster|
+
 ## How to use
 
 ### Prerequisites