configure argocd project for infra and infra/controllers

00-infrastructure/01-tofu/talos/config.tf

@@ -72,14 +72,12 @@ data "talos_machine_configuration" "this" {
     templatefile("${path.module}/machine-config/control_plane.yaml.tftpl", {
       mac_address      = lower(each.value.mac_address)
       vip              = var.cluster.vip
-      cluster_mtu      = var.cluster.mtu
       extra_manifests  = jsonencode(local.extra_manifests)
       api_server       = var.cluster.api_server
       inline_manifests = jsonencode(terraform_data.bootstrap_inline_manifests.output)
     }) :
     templatefile("${path.module}/machine-config/worker.yaml.tftpl", {
       mac_address = lower(each.value.mac_address)
-      cluster_mtu      = var.cluster.mtu
     })
   ]
 }

00-infrastructure/01-tofu/talos/machine-config/common.yaml.tftpl

@@ -9,12 +9,6 @@ machine:
         overridePath: true
 %{endfor}
 %{endif}
-  # https://github.com/siderolabs/talos/issues/10002
-  # Incompatibility with Cilium BPF routing and Talos Host DNS
-  features:
-    hostDNS:
-      enabled: true
-      forwardKubeDNSToHost: false
   kubelet:
     image: ghcr.io/siderolabs/kubelet:${kubernetes_version}
 %{if kubelet != ""}
@@ -62,11 +56,4 @@ cluster:
   proxy:
     image: registry.k8s.io/kube-proxy:${kubernetes_version}
   scheduler:
-    image: registry.k8s.io/kube-scheduler:${kubernetes_version}
-  network:
-    podSubnets:
-      - 10.244.0.0/16
-#      - 2a13:fc80:1:d200::/64
-    serviceSubnets:
-      - 10.96.0.0/16
-#      - 2a13:fc80:1:d201::/64
+    image: registry.k8s.io/kube-scheduler:${kubernetes_version}

00-infrastructure/01-tofu/talos/machine-config/control_plane.yaml.tftpl

@@ -5,7 +5,6 @@ machine:
       - deviceSelector:
           hardwareAddr: ${mac_address}
         dhcp: true
-        mtu: ${cluster_mtu}
 %{ if vip != null }
         vip:
           ip: ${vip}
@@ -20,15 +19,13 @@ cluster:
   controllerManager:
     extraArgs:
       bind-address: 0.0.0.0
-      node-cidr-mask-size-ipv4: "24"
-#      node-cidr-mask-size-ipv6: "112"
   etcd:
     extraArgs:
       listen-metrics-urls: http://0.0.0.0:2381
   scheduler:
     extraArgs:
       bind-address: 0.0.0.0
-  # Let's go with the default network
+# Let's go with the default network
   network:
     cni:
       name: none

00-infrastructure/01-tofu/talos/machine-config/worker.yaml.tftpl

@@ -3,5 +3,4 @@ machine:
     interfaces:
       - deviceSelector:
           hardwareAddr: ${mac_address}
-        dhcp: true
-        mtu: ${cluster_mtu}
+        dhcp: true

00-infrastructure/01-tofu/talos/variables.tf

@@ -17,7 +17,6 @@ variable "cluster" {
   type = object({
     name                         = string
     vip                          = optional(string)
-    mtu                          = optional(number, 1500)
     subnet_mask                  = optional(string, "24")
     talos_machine_config_version = optional(string)
     proxmox_cluster              = string

00-infrastructure/01-tofu/talos_cluster.auto.tfvars

@@ -8,9 +8,8 @@ talos_cluster_config  = {
     # http_proxy          = "http://100.64.0.1:3128"
     # no_proxy            = "10.0.0.0/8"
 
-    mtu                 = 9000
     vip                 = "10.51.10.100"
-    kubernetes_version  = "v1.34.0"
+    kubernetes_version  = "v1.33.3"
     gateway_api_version = "v1.3.0"
 
     cilium = {

00-infrastructure/01-tofu/talos_image.auto.tfvars

@@ -1,6 +1,6 @@
 talos_image = {
-    version         = "v1.11.0"
-    update_version  = "v1.11.0"
+    version         = "v1.10.7"
+    update_version  = "v1.10.7"
     schematic_path  = "talos/image/schematic.yaml"
     #update_schematic_path = "talos/image/schematic.yaml"
 }

00-infrastructure/01-tofu/variables.tf

@@ -34,7 +34,6 @@ variable "talos_cluster_config" {
   type = object({
     name                         = string
     vip                          = optional(string)
-    mtu                          = optional(number, 1500)
     subnet_mask                  = optional(string, "24")
     talos_machine_config_version = optional(string)
     proxmox_cluster              = string

02-k8s/.gitignore

@@ -1,3 +0,0 @@
-# Kube Seal backup
-kubeseal.cert
-kubeseal.key

02-k8s/README.md

@@ -8,51 +8,6 @@ Follow documentation at [Deploy Kubernetes Cluster](../00-infrastructure/01-tofu
 
 ### 2. Deploy Argo CD
 
-Deployment of ArgoCD itself:
-
 ```shell
 kustomize build --enable-helm infra/controllers/argocd | kubectl apply -f -
-```
-
-You can uninstall it with the command
-
-```shell
-kustomize build --enable-helm infra/controllers/argocd | kubectl delete -f -
-```
-
-Configure ArgoCD to pull our setup from this repo:
-
-```shell
-kubectl apply -k infra
-```
-
-#### Temporary legacy admin user
-
-Configure Admin users:
-1. Get random password:
-```shell
-argocd admin initial-password -n argocd
-```
-1. Establish UI forward:
-```shell
-kubectl port-forward svc/argocd-server -n argocd 8080:443
-```
-1. CLI Login with previously retrieved password and username ``admin``:
-```shell
-argocd login localhost:8080 --insecure
-```
-
-## Other
-
-### What's keepign a namespace in state deleting?
-
-```shell
-kubectl api-resources --verbs=list --namespaced -o name \
-  | xargs -n 1 kubectl get --show-kind --ignore-not-found -n argocd
-```
-
-### Delete ArgoCD applications stuck in dependencies
-
-```shell
-kubectl -nargocd patch applications/controllers  -p '{"metadata": {"finalizers": null}}' --type merge
 ```

02-k8s/fetch_kubeseal_certs.sh

@@ -1,9 +0,0 @@
-#!/usr/bin/bash
-
-rm kubeseal.cert kubeseal.key 2> /dev/null
-
-# public cert
-(umask 0077 && kubeseal --controller-namespace=sealed-secrets --fetch-cert > kubeseal.cert)
-
-# full cert backup
-(umask 0077 && kubectl get secret -n sealed-secrets -l sealedsecrets.bitnami.com/sealed-secrets-key -o yaml > kubeseal.key)

02-k8s/infra/application-set.yaml

@@ -4,11 +4,11 @@ metadata:
   name: infrastructure
   namespace: argocd
   labels:
-    s5b.org: infrastructure
+    dev.stonegarden: infrastructure
 spec:
   generators:
     - git:
-        repoURL: https://git.straubintra.net/s5b-public/k8s.git
+        repoURL: https://git.straubintra.net/s5b-private/k8s/
         revision: HEAD
         directories:
           - path: 02-k8s/infra/*
@@ -16,11 +16,11 @@ spec:
     metadata:
       name: '{{ path.basename }}'
       labels:
-        s5b.org: infrastructure
+        dev.stonegarden: infrastructure
     spec:
       project: infrastructure
       source:
-        repoURL: https://git.straubintra.net/s5b-public/k8s.git
+        repoURL: https://git.straubintra.net/s5b-private/k8s/
         targetRevision: HEAD
         path: '{{ path }}'
       destination:

02-k8s/infra/controllers/application-set.yaml

@@ -8,7 +8,7 @@ metadata:
 spec:
   generators:
     - git:
-        repoURL: https://git.straubintra.net/s5b-public/k8s.git
+        repoURL: https://git.straubintra.net/s5b-private/k8s/
         revision: HEAD
         directories:
           - path: 02-k8s/infra/controllers/*
@@ -20,7 +20,7 @@ spec:
     spec:
       project: controllers
       source:
-        repoURL: https://git.straubintra.net/s5b-public/k8s.git
+        repoURL: https://git.straubintra.net/s5b-private/k8s/
         targetRevision: HEAD
         path: '{{ path }}'
       destination:

02-k8s/infra/controllers/argocd/values.yaml

@@ -7,9 +7,12 @@ configs:
     create: true
     application.resourceTrackingMethod: annotation+label
     admin.enabled: true
-    kustomize.buildOptions: "--enable-helm --load-restrictor LoadRestrictionsNone --enable-alpha-plugins --enable-exec"
+    kustomize.buildOptions: --enable-helm
     # url: https://gitops.prod.k8.eis-mk8.de.s5b.org
+      
 
+  cmp:
+    create: true
   params:
     controller.diff.server.side: true
     server.insecure: false
@@ -81,6 +84,12 @@ repoServer:
       enabled: true
   containerSecurityContext:
     readOnlyRootFilesystem: true
+  volumes:
+    - name: cmp-kustomize-build-with-helm
+      configMap:
+        name: argocd-cmp-cm
+    - name: cmp-tmp
+      emptyDir: { }
   resources:
     requests:
       cpu: 100m

02-k8s/infra/controllers/project.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: argocd
 spec:
   sourceRepos:
-    - 'https://git.straubintra.net/s5b-public/k8s.git'
+    - 'https://git.straubintra.net/s5b-private/k8s/'
   destinations:
     - namespace: 'argocd'
       server: '*'

02-k8s/infra/controllers/sealed-secrets/kustomization.yaml

@@ -1,13 +1,10 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-resources:
-  - ns.yaml
-
 helmCharts:
   - name: sealed-secrets
     repo: oci://registry-1.docker.io/bitnamicharts
-    version: 2.5.19
+    version: 2.17.4
     releaseName: sealed-secrets-controller
     namespace: sealed-secrets
     includeCRDs: true

02-k8s/infra/controllers/sealed-secrets/ns.yaml

@@ -1,4 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: sealed-secrets

02-k8s/infra/network/application-set.yaml

@@ -1,34 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: ApplicationSet
-metadata:
-  name: network
-  namespace: argocd
-  labels:
-    s5b.org: network
-spec:
-  generators:
-    - git:
-        repoURL: https://git.straubintra.net/s5b-public/k8s.git
-        revision: HEAD
-        directories:
-          - path: 02-k8s/infra/network/*
-  template:
-    metadata:
-      name: '{{ path.basename }}'
-      labels:
-        s5b.org: network
-    spec:
-      project: network
-      source:
-        repoURL: https://git.straubintra.net/s5b-public/k8s.git
-        targetRevision: HEAD
-        path: '{{ path }}'
-      destination:
-        name: in-cluster
-        namespace: argocd
-      syncPolicy:
-        automated:
-          selfHeal: true
-          prune: true
-        syncOptions:
-          - ServerSideApply=true

02-k8s/infra/network/cilium/announce.yaml

@@ -1,8 +0,0 @@
-apiVersion: cilium.io/v2alpha1
-kind: CiliumL2AnnouncementPolicy
-metadata:
-  name: default-l2-announcement-policy
-  namespace: kube-system
-spec:
-  externalIPs: true
-  loadBalancerIPs: true

02-k8s/infra/network/cilium/bgp.yaml

@@ -1,120 +0,0 @@
----
-apiVersion: cilium.io/v2
-kind: CiliumBGPClusterConfig
-metadata:
-  name: vy-eis-mk8-de-bgp
-spec:
-  nodeSelector:
-    matchLabels:
-      kubernetes.io/os: linux
-  bgpInstances:
-    - name: "ASN65001"
-      localASN: 65001
-      peers:
-        - name: "vy-eis-mk8-de-1-v6"
-          peerASN: 65000
-          peerAddress: 2a13:fc80:1:a::1
-          peerConfigRef:
-            name: "vy-eis-mk8-de-bgp-1-peer-config"
-        # - name: "vy-eis-mk8-de-1-v4"
-        #   peerASN: 65000
-        #   peerAddress: 10.51.10.1
-        #   peerConfigRef:
-        #     name: "vy-eis-mk8-de-bgp-1-peer-config"
----
-apiVersion: cilium.io/v2
-kind: CiliumBGPPeerConfig
-metadata:
-  name: vy-eis-mk8-de-bgp-1-peer-config
-spec:
-
-  # Optimized timers for datacenter environments
-  timers:
-    connectRetryTimeSeconds: 5
-    holdTimeSeconds: 9
-    keepAliveTimeSeconds: 3
-  
-  # Enable graceful restart for zero-downtime operations
-  gracefulRestart:
-    enabled: true
-    restartTimeSeconds: 15
-  
-  # Custom transport configuration
-  transport:
-    peerPort: 179
-  
-  # Address family configuration with advertisements
-  families:
-  - afi: ipv4
-    safi: unicast
-    advertisements:
-      matchLabels:
-        advertise: "datacenter-bgp"
-  - afi: ipv6
-    safi: unicast
-    advertisements:
-      matchLabels:
-        advertise: "datacenter-bgp"
----
-apiVersion: cilium.io/v2
-kind: CiliumBGPAdvertisement
-metadata:
-  name: pod-cidr-advertisements
-  labels:
-    advertise: "datacenter-bgp"
-spec:
-  advertisements:
-  - advertisementType: "PodCIDR"
-    attributes:
-      communities:
-        standard: ["65001:100", "65001:200"]
-        large: ["65001:100:1"]
-        wellKnown: ["no-export"]
-      localPreference: 200
-
-  - advertisementType: "Service"
-    service:
-      addresses:
-        - ClusterIP
-        - ExternalIP
-        - LoadBalancerIP
-    selector:             # select all services by a dummy expression always matching
-      matchExpressions:
-        - {key: somekey, operator: NotIn, values: ['never-used-value']}
-
-  # That at least works for the PodCIDR
-  # - advertisementType: "PodCIDR"
-  #   attributes:
-  #     communities:
-  #       standard: ["65001:100", "65001:200"]
-  #       large: ["65001:100:1"]
-  #       wellKnown: ["no-export"]
-  #     localPreference: 200
-
-  # # LoadBalancer services
-  # - advertisementType: "Service"
-  #   service:
-  #     addresses:
-  #     - LoadBalancerIP
-  #   # selector:
-  #   #   matchLabels:
-  #   #     service-type: "external"
-  #   attributes:
-  #     communities:
-  #       standard: ["65001:300"]
-  #     localPreference: 150
-  
-  # # ClusterIP services for internal access
-  # - advertisementType: "Service"
-  #   service:
-  #     addresses:
-  #     - ClusterIP
-  #   # selector:
-  #   #   matchExpressions:
-  #   #   - key: "internal-bgp"
-  #   #     operator: "In"
-  #   #     values: ["enabled"]
-  #   attributes:
-  #     communities:
-  #       standard: ["65001:400"]
-  #     localPreference: 100

02-k8s/infra/network/cilium/ip-pool.yaml

@@ -1,15 +0,0 @@
-apiVersion: cilium.io/v2
-kind: CiliumLoadBalancerIPPool
-metadata:
-  name: public-v4-01
-spec:
-  blocks:
-    - cidr: 185.83.87.48/29
----
-apiVersion: cilium.io/v2
-kind: CiliumLoadBalancerIPPool
-metadata:
-  name: public-v6-01
-spec:
-  blocks:
-    - cidr: 2a13:fc80:0001:d201::/64

02-k8s/infra/network/cilium/kustomization.yaml

@@ -1,16 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - announce.yaml
-  - ip-pool.yaml
-  - bgp.yaml
-
-helmCharts:
-  - name: cilium
-    repo: https://helm.cilium.io
-    version: 1.18.0 # renovate: github-releases=cilium/cilium
-    releaseName: "cilium"
-    includeCRDs: true
-    namespace: kube-system
-    valuesFile: values.yaml

02-k8s/infra/network/cilium/values.yaml

@@ -18,57 +18,42 @@ cgroup:
     enabled: false
   hostRoot: /sys/fs/cgroup
 
-bpf:
-  lbExternalClusterIP: true
 # https://www.talos.dev/latest/talos-guides/network/host-dns/#forwarding-kube-dns-to-host-dns
 # https://docs.cilium.io/en/stable/operations/performance/tuning/#ebpf-host-routing
-#   hostLegacyRouting: true
+bpf:
+  hostLegacyRouting: true
 
 # https://docs.cilium.io/en/stable/network/concepts/ipam/
 ipam:
   mode: kubernetes
+  multiPoolPreAllocation: ""
+# mode: multi-pool
+# operator:
+#   autoCreateCiliumPodIPPools:
+#     default:
+#       ipv6:
+#         cidrs:
+#           - 2a13:fc80:0001:d200::/64
+#         maskSize: 120
+    
+# # TODO ########!!!!!!!!!!!!!!!!!!!!!!!!!!!!%%%%%%%%%%%%%%%%%%%%%%%%%%%55555
+# # The service subnet CIDR.
+# serviceSubnets:
+#   - 10.96.0.0/12
+#   - 2a13:fc80:0001:d201::/64
 
-# k8s:
-#   requireIPv4PodCIDR: true
-#   requireIPv6PodCIDR: false
+# routingMode: native
 
-ipv4:
-  enabled: true
-ipv6:
-  enabled: false
+# bgpControlPlane:
+#   enabled: true
 
-# Avoid encapsulation for direct access
-routingMode: native
+# ipv4:
+#   enabled: true
+# ipv6:
+#   enabled: true
 
-#Route distribution gets managed by BGP
-bgpControlPlane:
-  enabled: true
-
-# enable instead of bgpControlPlane
-# l2announcements:
-#   enabled: false
-# externalIPs:
-#   enabled: false
-
-# Only BGP manages the routes
-# auto-direct-node-routes: true
-# direct-routing-skip-unreachable: true
-# The whole internet is directly reachable from each pod
-# ipv6-native-routing-cidr: ::/0
-ipv4-native-routing-cidr: 10.0.0.0/8
-
-# Disabling DNAT
-enableIPv4Masquerade: false
-enableIPv6Masquerade: false
-
-enableIPv6BIGTCP: true
-
-bandwidthManager:
-  enabled: true
-  bbr: true
-
-#debug:
-#  enabled: true
+# enableIPv4Masquerade: false
+# enableIPv6Masquerade: false
 
 operator:
   rollOutPods: true
@@ -98,11 +83,20 @@ resources:
     cpu: 200m
     memory: 512Mi
 
+#debug:
+#  enabled: true
+
 # Increase rate limit when doing L2 announcements
 k8sClientRateLimit:
   qps: 20
   burst: 100
 
+l2announcements:
+  enabled: true
+
+externalIPs:
+  enabled: true
+
 loadBalancer:
   # https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/#maglev-consistent-hashing
   algorithm: maglev

02-k8s/infra/network/kustomization.yaml

@@ -1,9 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-commonLabels:
-  s5b.org: network
-  app.kubernetes.io/managed-by: argocd
-
-resources:
-  - project.yaml
-  - application-set.yaml

02-k8s/infra/network/project.yaml

@@ -1,18 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: AppProject
-metadata:
-  name: network
-  namespace: argocd
-spec:
-  sourceRepos:
-    - 'https://git.straubintra.net/s5b-public/k8s.git'
-  destinations:
-    - namespace: 'argocd'
-      server: '*'
-    - namespace: 'kube-system'
-      server: '*'
-    - namespace: 'cilium-secrets'
-      server: '*'
-  clusterResourceWhitelist:
-    - group: '*'
-      kind: '*'

02-k8s/infra/project.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: argocd
 spec:
   sourceRepos:
-    - 'https://git.straubintra.net/s5b-public/k8s.git'
+    - 'https://git.straubintra.net/s5b-private/k8s/'
   destinations:
     - namespace: 'kube-system'
       server: '*'

90-debugging/test_application.yaml

@@ -1,49 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app.kubernetes.io/name: load-balancer-example
-  name: hello-world
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: load-balancer-example
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: load-balancer-example
-    spec:
-      containers:
-      - image: gcr.io/google-samples/hello-app:2.0
-        name: hello-world
-        ports:
-          - containerPort: 8080
-            name: http-web-svc
-        securityContext:
-          seccompProfile:
-            type: "RuntimeDefault"
-          allowPrivilegeEscalation: false
-          runAsNonRoot: true
-          readOnlyRootFilesystem: true
-          runAsUser: 1000
-          capabilities:
-            drop: ["ALL"]
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: hello-world-service
-  namespace: default
-  labels:
-    app.kubernetes.io/name: load-balancer-example
-spec:
-  selector:
-    app.kubernetes.io/name: load-balancer-example
-  type: LoadBalancer
-  loadBalancerClass: io.cilium/bgp-control-plane
-  # ipFamilyPolicy: RequireDualStack
-  ports:
-    - protocol: TCP
-      port: 80
-      targetPort: http-web-svc

README.md

@@ -8,8 +8,6 @@
 
 ### Network
 
-#### Node Network
-
 - IPv4 configuration uses DHCP with static MAC binding for easy bring-up
 - IPv6 addresses are manually assigned
 - DNS Zone: prod.k8.eis-mk8.de.s5b.org
@@ -18,29 +16,6 @@
 |--|--|--|
 |210|10.51.10.0/23|2a13:fc80:1:a::/64|
 
-#### Pod Network
-
-- IPv4 only
-- IPv4 prefix: 10.244.0.0/16
-- Potential IPv6 prefix: 2a13:fc80:0001:d200::/64
-
-##### Important side-notes
-
-- DNS resolver (Vyos) these networks must be whitelisted to allow recursive DNS
-
-#### Service Network
-
-- Dual-Stack
-- IPv6 Prefix: 2a13:fc80:0001:d201::/64
-- IPv4 Prefix: 185.83.87.48/28
-
-#### BGP
-
-|ASN|Who|
-|--|--|
-|65000|Upstream Router|
-|65001|Cluster|
-
 ## How to use
 
 ### Prerequisites