diff --git a/README.md b/README.md index e966d8e..8dfe7f0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,30 @@ # TALOS Linux Tests +## Install Kubectl + +Follow the installation guide at [kubernetes.io](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/). + +```bash +$ curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" +$ curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl.sha256" +$ echo "$(cat kubectl.sha256) kubectl" | sha256sum --check +$ chmod +x kubectl +$ mv kubectl ~/.local/bin/ +``` + +## Install Helm + +Follow [official documentation](https://helm.sh/docs/intro/install/). The way for Debian based distros: + +```bash +curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null +sudo apt-get install apt-transport-https --yes +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list +sudo apt-get update +sudo apt-get install helm +``` + + ## Install Talos Linux CLI On Ubuntu 22.04: @@ -17,18 +42,6 @@ talosctl completion zsh > ~/.oh-my-zsh/custom/plugins/talosctl/_talosctl Add ``talosctl`` to ``plugins`` within your ``~/.zshrc``. -### Install Kubectl - -Follow the installation guide at [kubernetes.io](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/). - -```bash -$ curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" -$ curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl.sha256" -$ echo "$(cat kubectl.sha256) kubectl" | sha256sum --check -$ chmod +x kubectl -$ mv kubectl ~/.local/bin/ -``` - ## Install Talos from ISO > **WARNING**: At least until v1.9.0 use IPv4 for creating the cluster. Clusters build up on IPv6 show several errors starting from etcd not starting and so on. @@ -183,4 +196,14 @@ $ talosctl apply-config --nodes 10.51.10.101 --endpoints 10.51.10.101 --taloscon ```bash $ talosctl --nodes 10.51.10.101 --endpoints 10.51.10.101 --talosconfig=./talosconfig get address +``` + +## Install Cilium + +- [Documentation for Cilium on TALOS Linux](https://www.talos.dev/v1.9/kubernetes-guides/network/deploying-cilium/#method-2-helm-manifests-install) + +- Add Cilium helm repo: +```bash +helm repo add cilium https://helm.cilium.io/ +helm repo update ``` \ No newline at end of file diff --git a/apply_all.sh b/apply_all.sh index cc302a8..7a179f8 100755 --- a/apply_all.sh +++ b/apply_all.sh @@ -23,6 +23,7 @@ for i in ctl-1 ctl-2 ctl-3; do -p @patches/10_control_plane.yaml \ -p @patches/machines/$i.yaml \ --output transient/$i.yaml + # -p @patches/11_cilium.yaml \ talosctl apply-config \ --nodes $i \ --endpoints $ENDPOINTS \ @@ -30,18 +31,18 @@ for i in ctl-1 ctl-2 ctl-3; do --file transient/$i.yaml done -for i in node-1 node-2 node-3; do - echo "Configuring WORKER host $i" - talosctl machineconfig patch \ - worker.yaml \ - -p @patches/00_all.yaml \ - -p @patches/20_worker.yaml \ - -p @patches/machines/$i.yaml \ - --output transient/$i.yaml +# for i in node-1 node-2 node-3; do +# echo "Configuring WORKER host $i" +# talosctl machineconfig patch \ +# worker.yaml \ +# -p @patches/00_all.yaml \ +# -p @patches/20_worker.yaml \ +# -p @patches/machines/$i.yaml \ +# --output transient/$i.yaml - talosctl apply-config \ - --nodes $i \ - --endpoints $ENDPOINTS \ - --talosconfig=./talosconfig \ - --file transient/$i.yaml -done \ No newline at end of file +# # talosctl apply-config \ +# # --nodes $i \ +# # --endpoints $ENDPOINTS \ +# # --talosconfig=./talosconfig \ +# # --file transient/$i.yaml +# done \ No newline at end of file diff --git a/boot-assets.yml b/boot-assets.yml new file mode 100644 index 0000000..253c7f3 --- /dev/null +++ b/boot-assets.yml @@ -0,0 +1,15 @@ +customization: + systemExtensions: + officialExtensions: + - siderolabs/amd-ucode + - siderolabs/amdgpu + - siderolabs/fuse3 + - siderolabs/intel-ice-firmware + - siderolabs/intel-ucode + - siderolabs/iscsi-tools + - siderolabs/lldpd + - siderolabs/nonfree-kmod-nvidia-lts + - siderolabs/nvidia-container-toolkit-lts + - siderolabs/nvidia-open-gpu-kernel-modules-lts + - siderolabs/nvme-cli + - siderolabs/qemu-guest-agent \ No newline at end of file diff --git a/cilium.yaml b/cilium.yaml new file mode 100644 index 0000000..62e05c7 --- /dev/null +++ b/cilium.yaml @@ -0,0 +1,2037 @@ +--- +# Source: cilium/templates/cilium-secrets-namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: "cilium-secrets" +--- +# Source: cilium/templates/cilium-agent/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-envoy/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-envoy" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/cilium-ca-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: cilium-ca + namespace: kube-system +data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQUk2M1ZNSmJXcFNRcTBCWmVDdk9uL0l3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTURFeE9URTNOREF4TVZvWERUSTRNREV4T1RFMwpOREF4TVZvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUF3YmFaQk5OR1ZEVHJYU0JEQXdiNGVnUkMrTlZHQjdEcHN5a25Ma0ZZZ2JMaTE5Ty8KN2gxT1BJVkJzMmJLQmhwOGFDdW0zdG1VdVZZc0NMUDFocmxrTzQwUzMyaEFuOGY0VzhiRzk5TjJZc0tuY21mRAoxbXdSN1JVeG4zVjh4WmZoUkt4aUZCTHNoSkdGZWpuM1NaQUVqdU5PaVZtUHhORWkyWHAvOVc0SWVhaHVKdXltCnlJdk96Q1VrTFFIZHhHajlTUzhNSWs1RlRLTlNnRjlPcVM2WjcwUzhjYkxBamhPZHNIbDlIb1cxbXJKRm1GSG8KSDhiKy8xb1BTazlmT2k1NzdUWjI4RkJ3SlZqSW1oN2FibU9tTUYxVUtjVzVFOXIxS0tkb1lENEZlblN0NG05OAppNnNJYXpYTWR6UVdxUXlEMGYrWmozQ3BPdFE1ekovZER4T1lkd0lEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGQklYZVpFazB2WXZuWmF2aUU1cm43bTFqOFlkTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ1l2RmNicWlmZzhUb0JJM3J6MDJGTU93M01KUXYxdlpNZE85by9mUE9HM0tpazV4TkhaUXJOClhsRHJ5Z3VHYy91eVQwRjB1Q1lJMTYyWjhiUXR2UVRya0xSNXRRK0ZEakwvelozYThzVHNpbUtFS2s3dEpoKzQKdXdpTHNCVXR2QzBFWVNZQS9BQTc2MmhqRGU2NmxoVmlXa0RTSHNIZmErNTNZc1RrVUtHNXVDNDhxQjltUU9tUApOUzlyaGQxY0xJVWlJYXg5WXlLZXdXTHhzeExxcC8vcTE1aTFFZ1V1UDNjd1ZtZ0ZGa2xCOTUvVlFROVRwejB3CjhhcUVzdHFLMGpPZm53dmtkRTRteXlCQ1hDUWpHUWVTcS9naHpBenRpWGR1dDQ0bFlKcWVHd3FHQ0JhbFk5L2wKLy9FN3dHUWtqaGNVRS9vWUloU2l6TEo3Lzl2Skt6NTcKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBd2JhWkJOTkdWRFRyWFNCREF3YjRlZ1JDK05WR0I3RHBzeWtuTGtGWWdiTGkxOU8vCjdoMU9QSVZCczJiS0JocDhhQ3VtM3RtVXVWWXNDTFAxaHJsa080MFMzMmhBbjhmNFc4Ykc5OU4yWXNLbmNtZkQKMW13UjdSVXhuM1Y4eFpmaFJLeGlGQkxzaEpHRmVqbjNTWkFFanVOT2lWbVB4TkVpMlhwLzlXNEllYWh1SnV5bQp5SXZPekNVa0xRSGR4R2o5U1M4TUlrNUZUS05TZ0Y5T3FTNlo3MFM4Y2JMQWpoT2RzSGw5SG9XMW1ySkZtRkhvCkg4YisvMW9QU2s5Zk9pNTc3VFoyOEZCd0pWakltaDdhYm1PbU1GMVVLY1c1RTlyMUtLZG9ZRDRGZW5TdDRtOTgKaTZzSWF6WE1kelFXcVF5RDBmK1pqM0NwT3RRNXpKL2REeE9ZZHdJREFRQUJBb0lCQUVZRGdFVHRNOE1scVZmcwpYQ3dRN2VmeitGekZnTjB3eFJiYkxlMGE4eWxWaFlXcVkvbGxJQ3pob1FCOWkxRXZhMzJmSVYzMFdEdXdlSlBnCnIwSTFhWlBZaGxTMjhueU5veWRRcmdOT3E4QkFTSk1pQy9kSThpVzFma3o3QXRETDkwQ0I2YVVGdmZTZ1FCVVEKTkYrakxQS1VGNE92R2paeHZGM01WOGxXTHJ3QlVWeXI4Y2pUUVBIV2p5SXlDem1PTzVtSStkdjBJRlBza1YrQwp0SGVLWmN0YUphRUVxRlBpSEZ4aDd5dTljR0xZb1ppYWNyK2dnd1JnR3pyTm56K3FzcHBXaDVib3JrTWFOZWp3Cnc4ZmFScDNTQitHcm5KdC9ZNlBMWVViVUJlSXRBeHlYS1lGdDdUVUJFNjRHbmhPNHg5MHkwTmRwSTM3T1l2YloKaENHZklLa0NnWUVBeG9RdWpyc3BFakpDTFBZcFN3eFNnaWtCQW9uTkgwWWg5UFBRTjhUVXJISWtNUUZzNGdhcQpVMW1pZjlKRXEwZGdTVTFiaDRvUFVwajI2TDFBWUNZbW14THpXQzRjUEdNWmsrR2hLYUtGMDYxbkgyVGtOdmpqCjBvd1c4ZFgzNHBmalJWeFQvR1FFOTlPTDlVRkhBN2JzdkpydzMwTlBNVEZlN1NCOTJudUYrMjBDZ1lFQStjNWUKblZMUU95M09FeXBXOCsxWHdkMHk1WXJTZTg0TzY1MUVSTDc5cGcyYW5lMjVnY3N6NWV1YWU0QjkwaTFFTmpyYgp6Q3lBcFJoZ2FPT2VUZ2thMThDUGJEWkpIUEk5Q0pIbWh3bzBSellPL2NBdlg4U3k4czY0NU1FQzVQZFBGdlBICnVBdWtPSHZ0azA0blRlaVNYWER2anVGNGVLSElNa2NUa2tCWHNQTUNnWUFUVGVWTklld2poZG1qajZpQ1VNZHgKT3RXTFRxVXd4bHdrK3RTanA3TmF0WnJKWFM4ZVo3R0dlMi91NkI2ZlhVaUN1RkFQUlNDcVVvK29JRWN6VW1KcwpzQklUNENBRXkwcEFkQzk4NGY0TGtKQTNvRk1FK1gwR0MzTnJ2dmZsQkFvb3c0cXpQM0t3T3RVZ1cyR3hmY2RWCnErZUNmb1FqNzM0Z3B3azJCczhIblFLQmdHNnVCZk8wbG5neDhrVkJsQ2NkZUsxM2dTNTgxSWR3SStJMjJ1R08Kc2JtWiszQ1B2SXFLYXVNckFWWkk5dlpoZ2dCNUVadWorQmV0OWtGaTRHZjQxc0RURzNpNWtLbFhscjB1VDdKegpUZmhqTmo4dG9JL1JDZXhVdFV6OVMzZ0JMelNmTEZVYUsxZXZhUExFUk52YStNYVgzRHcxSC9ocXpkV2c4a1h4Clppa0xBb0dBQ1pQbkdiZmxUUEFpaHVzVGJLcXB5Zzg3RDhjMUhnZHRvaDExWnRaTXIyclljUFdhdFlTeFJpaGQKd0dsTkxzTlJoVFpocGhUeFdHd3RRVDYxd29XemFRaWlqd3QrVGVqa0w1UU5qak9qSE5Td0VZMVNZMWRBak52SgpNT1RyMEFQSzZ2b2ZINHBJYVBHWTIrQk55UlVZM3dsYmFJZ2p0ZDJGMGFzWWtBcGdFY1U9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== +--- +# Source: cilium/templates/hubble/tls-helm/server-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: hubble-server-certs + namespace: kube-system +type: kubernetes.io/tls +data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQUk2M1ZNSmJXcFNRcTBCWmVDdk9uL0l3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTURFeE9URTNOREF4TVZvWERUSTRNREV4T1RFMwpOREF4TVZvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUF3YmFaQk5OR1ZEVHJYU0JEQXdiNGVnUkMrTlZHQjdEcHN5a25Ma0ZZZ2JMaTE5Ty8KN2gxT1BJVkJzMmJLQmhwOGFDdW0zdG1VdVZZc0NMUDFocmxrTzQwUzMyaEFuOGY0VzhiRzk5TjJZc0tuY21mRAoxbXdSN1JVeG4zVjh4WmZoUkt4aUZCTHNoSkdGZWpuM1NaQUVqdU5PaVZtUHhORWkyWHAvOVc0SWVhaHVKdXltCnlJdk96Q1VrTFFIZHhHajlTUzhNSWs1RlRLTlNnRjlPcVM2WjcwUzhjYkxBamhPZHNIbDlIb1cxbXJKRm1GSG8KSDhiKy8xb1BTazlmT2k1NzdUWjI4RkJ3SlZqSW1oN2FibU9tTUYxVUtjVzVFOXIxS0tkb1lENEZlblN0NG05OAppNnNJYXpYTWR6UVdxUXlEMGYrWmozQ3BPdFE1ekovZER4T1lkd0lEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGQklYZVpFazB2WXZuWmF2aUU1cm43bTFqOFlkTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ1l2RmNicWlmZzhUb0JJM3J6MDJGTU93M01KUXYxdlpNZE85by9mUE9HM0tpazV4TkhaUXJOClhsRHJ5Z3VHYy91eVQwRjB1Q1lJMTYyWjhiUXR2UVRya0xSNXRRK0ZEakwvelozYThzVHNpbUtFS2s3dEpoKzQKdXdpTHNCVXR2QzBFWVNZQS9BQTc2MmhqRGU2NmxoVmlXa0RTSHNIZmErNTNZc1RrVUtHNXVDNDhxQjltUU9tUApOUzlyaGQxY0xJVWlJYXg5WXlLZXdXTHhzeExxcC8vcTE1aTFFZ1V1UDNjd1ZtZ0ZGa2xCOTUvVlFROVRwejB3CjhhcUVzdHFLMGpPZm53dmtkRTRteXlCQ1hDUWpHUWVTcS9naHpBenRpWGR1dDQ0bFlKcWVHd3FHQ0JhbFk5L2wKLy9FN3dHUWtqaGNVRS9vWUloU2l6TEo3Lzl2Skt6NTcKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWekNDQWorZ0F3SUJBZ0lSQU9RVEFLRys4c2YvejVZeW8vREszNzR3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTURFeE9URTNOREF4TVZvWERUSTJNREV4T1RFMwpOREF4TVZvd0tqRW9NQ1lHQTFVRUF3d2ZLaTVrWldaaGRXeDBMbWgxWW1Kc1pTMW5jbkJqTG1OcGJHbDFiUzVwCmJ6Q0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQU5nUUZBT2lFKytheVljOHVpanMKL2JCamNuZExwbkRLOGZTOUVGbENDaDVwSVV2UnoycGJFS24vaTM4WHk4R09DMUNxK3JHcmVWOTNtVlZoZkd3bQovWTNoTjg4dXV5NnEvRGJXTk84eXI4TXRJZ0pyMHpVbGtKcHZ5T1drMEx2ZWdPb1RycmZ4c3Rzd3RybEN3QUJECk9jZENFR2x2eHNrQVZZbzJsODJpVDdKeDRldmo3bEFOMnZNQzFUSnF0dmxQS2tYOG9YSlZPOW9nN250TDd6bC8KbzZvdHNSWGZFMjBVNHJoRFBuUkI5azNSVFB5cURCS1ZqbFpZTTQ5dlVuQzlSTWZrMTRHNVdKR1hEQVhYWnJacQpDdkpPckY1OUZRU2FNSjFxd2ZWbklLNUEzaVhWRUZkZTlOeC9rbXdTMHVGOHNId1ZYQ3VUeitGcGVPQnB3TXlPClFMY0NBd0VBQWFPQmpUQ0JpakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdIUVlEVlIwbEJCWXdGQVlJS3dZQkJRVUgKQXdFR0NDc0dBUVVGQndNQ01Bd0dBMVVkRXdFQi93UUNNQUF3SHdZRFZSMGpCQmd3Rm9BVUVoZDVrU1RTOWkrZApscStJVG11ZnViV1B4aDB3S2dZRFZSMFJCQ013SVlJZktpNWtaV1poZFd4MExtaDFZbUpzWlMxbmNuQmpMbU5wCmJHbDFiUzVwYnpBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQVVUKys3KzBsR0JzMmFIRnZwT0dCcWI3MEtUTGIKMFNSOGRTZlVPVzIwRmtqeEhuWFR5RHlyOWkvZytvT2F4RVp6c2psSG44MlJuWmIyczI4NzZSbVIvWE9aVXR6TgpEQUVIcXdpZXdTUzhReDViTGdKa1I5SzNBUHhZM3RCS29ENi9YM3hyRkZLQ0J1aHcyZytVU2dDSGEyQUgwb3h6ClVYRlVEK1dVZGZ1SW5Ed0s1VzgyanBITUR6cFpwUDhSaTRWZGR1WC93M1hFKzlIMUxaRlhkd1FMTU9aVmdWL0cKVWFieGNVNlc1MHpaaUdHelBLeUlZTk1WNVNMa1RINmVCekQyNWxCTUhJZFNaUEtIdUlPNmFIUStxQVdiVUU2QwpUZUd1ejdvcTAwbGVKcUdjdnJvemZhVW1DQk9wZ2hEd29hTjh1bXFmSEZKUFA1ZmtySmlWZEJzU0Z3PT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcFFJQkFBS0NBUUVBMkJBVUE2SVQ3NXJKaHp5NktPejlzR055ZDB1bWNNcng5TDBRV1VJS0hta2hTOUhQCmFsc1FxZitMZnhmTHdZNExVS3I2c2F0NVgzZVpWV0Y4YkNiOWplRTN6eTY3THFyOE50WTA3ekt2d3kwaUFtdlQKTlNXUW1tL0k1YVRRdTk2QTZoT3V0L0d5MnpDMnVVTEFBRU01eDBJUWFXL0d5UUJWaWphWHphSlBzbkhoNitQdQpVQTNhOHdMVk1tcTIrVThxUmZ5aGNsVTcyaUR1ZTB2dk9YK2pxaTJ4RmQ4VGJSVGl1RU0rZEVIMlRkRk0vS29NCkVwV09WbGd6ajI5U2NMMUV4K1RYZ2JsWWtaY01CZGRtdG1vSzhrNnNYbjBWQkpvd25XckI5V2NncmtEZUpkVVEKVjE3MDNIK1NiQkxTNFh5d2ZCVmNLNVBQNFdsNDRHbkF6STVBdHdJREFRQUJBb0lCQVFDZ2ZEalJoZ2IyMTJ0UApwMXpEWDEwRU1xbm9VR205cUJEV2I4Ui8rQnlhQUE2NlNZcWQvaXcvaVUvdFJCaVZtVk53UkwvWFRuM2wybWtyClVURGhDSWtYWksxV2wraHpQby8rajFkY1FMTHE3MUF1cWphRWFIV2pNTnJ4TUYzVzNldTFjbzg1bEM3OHZvOUsKeG5qaytRb21wanlZaEcwTUNkSi95aE14cTVrMCtpR2NvalU2SWY3VU56bXNLV1ZVM1REMzNJVForSVplWElYUgpoelRVaUZKS0p4dUJoTlhnSThOQTRqRzd0VWZJT3lGQm81dVFva084K1lldUx1eWxOK0RUWFB1YzFlRC9GQW0zCnFXVmJ0ZTQrY0F6SmI3WDdhcHV4aDhnTEtET0prcGM0MmpROWRaNlBuTEZqS3YyTWJRVGxpOHZRa1c0M1RhNW8KRUNNM2daVkpBb0dCQVA0ci9sbloyVmtZYkV2dVAwT2RWMG1pc3htTnNrT2h6b2gwQVlnMWhDeDdTQVRjd2thMApnMmZLOWc4MVVrYTgwcllRVW00ZC9yV0lpMWpVcnU1WkRBbHFmN1daUlErTFVkSHhJb3k2RnBYRE16Y1R3MU5yCjhwOXpkTFJzZkd1d3lHSDJ1Z0J2bG03SEIvQUN2NFNCWGl0WjBoa0JzdGN0ZGRiN1BXVFdNcVc5QW9HQkFObWQKNmhwMU1GKzQ0M0dMNHJONjYyNHluMU9qUUExcUJQbTRackE0clJPanM0L2xLQzduMFFiMEVVRno2NGRXaXJUSwpPbXh1UXRGTVV0djdqZVZCdEFlSUM4c3ZERFRZbmd2VWY5dG5yalluVnBnQ1JVTmk1UUcxdEZtUy9kM3VvbTUyCkNwU0M2WXRubWFJcmxWMStJN2d0N2I2c1ZLOWpzZzRQL1J5R2pjV0RBb0dCQU5Xd3NSQmptbHRRc0VRS1V3QUsKdVljOEZhMm5OcmVyd3hOdnpKWHl3R1FSWmRSSXNjRjJKSUJQNEIyYks4Z3pHa2hkWXFBWDhidzN4MGZ3ei8ydAp1ZlAwcU5oSnV3azlqWldmVVBmVndBVTdic1NnaGZpTjZoYnFuN1RNL0xJaGwrYmM0elpZbXJZWitXOXRNQTA4CklxcjV3aXlxQSt6cTJJYmRCWm5RaUJaVkFvR0FiaWlBOCttMGxoQUttbFRjYWt4ajF3R3JJM1IxSFJVNDlZeEIKOXh5SmlIS0tsK0hCaUpUeWU5bWF6bzQyZ09qdkIvdUN2WmJISGhuTHErM1g4MkJpR0pubWxuRDNkZHpkakZHQwpjM2MxZWRaS0FHTE4yd0c1dVRiWTE2c1A5WU15R0VhS3JvamlYaDRRYUVZMXN3RUNaZDVpQmU1SVpBeE1lcy9mCktjeDNBYThDZ1lFQTYyQWcwWWpTTnh1ZmJiMVJZQUEvRFNjU3JYeTBQMVNLVzZtck9WQ3lWRjVqYkxGNVhjVWEKZ20wV1FKWnFhQVRrdGMwQlgvVGV5Q2xPdVFkM2E5ZG1mNlJEcm9uM1p5bXVaT09ZeFlHc3pHN1Y3V1pTc0NyVQoxS2trcVdxNFpWcnlWUU5QTTJsTUI3T1BjMXRBSjJvd0o2SVVubHRkSXRIb0k5U0pwZ1VZdTYwPQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= +--- +# Source: cilium/templates/cilium-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-config + namespace: kube-system +data: + + # Identity allocation mode selects how identities are shared between cilium + # nodes by setting how they are stored. The options are "crd" or "kvstore". + # - "crd" stores identities in kubernetes as CRDs (custom resource definition). + # These can be queried with: + # kubectl get ciliumid + # - "kvstore" stores identities in an etcd kvstore, that is + # configured below. Cilium versions before 1.6 supported only the kvstore + # backend. Upgrades from these older cilium versions should continue using + # the kvstore by commenting out the identity-allocation-mode below, or + # setting it to "kvstore". + identity-allocation-mode: crd + identity-heartbeat-timeout: "30m0s" + identity-gc-interval: "15m0s" + cilium-endpoint-gc-interval: "5m0s" + nodes-gc-interval: "5m0s" + + # If you want to run cilium in debug mode change this value to true + debug: "false" + debug-verbose: "" + # The agent can be put into the following three policy enforcement modes + # default, always and never. + # https://docs.cilium.io/en/latest/security/policy/intro/#policy-enforcement-modes + enable-policy: "default" + policy-cidr-match-mode: "" + # If you want metrics enabled in cilium-operator, set the port for + # which the Cilium Operator will have their metrics exposed. + # NOTE that this will open the port on the nodes where Cilium operator pod + # is scheduled. + operator-prometheus-serve-addr: ":9963" + enable-metrics: "true" + enable-envoy-config: "true" + envoy-config-retry-interval: "15s" + enable-gateway-api: "true" + enable-gateway-api-secrets-sync: "true" + enable-gateway-api-proxy-protocol: "false" + enable-gateway-api-app-protocol: "true" + enable-gateway-api-alpn: "true" + gateway-api-xff-num-trusted-hops: "0" + gateway-api-service-externaltrafficpolicy: "Cluster" + gateway-api-secrets-namespace: "cilium-secrets" + gateway-api-hostnetwork-enabled: "false" + gateway-api-hostnetwork-nodelabelselector: "" + + # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 + # address. + enable-ipv4: "true" + + # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 + # address. + enable-ipv6: "true" + # Users who wish to specify their own custom CNI configuration file must set + # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. + custom-cni-conf: "false" + enable-bpf-clock-probe: "false" + # If you want cilium monitor to aggregate tracing for packets, set this level + # to "low", "medium", or "maximum". The higher the level, the less packets + # that will be seen in monitor output. + monitor-aggregation: medium + + # The monitor aggregation interval governs the typical time between monitor + # notification events for each allowed connection. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-interval: "5s" + + # The monitor aggregation flags determine which TCP flags which, upon the + # first observation, cause monitor notifications to be generated. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-flags: all + # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic + # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. + bpf-map-dynamic-size-ratio: "0.0025" + # bpf-policy-map-max specifies the maximum number of entries in endpoint + # policy map (per endpoint) + bpf-policy-map-max: "16384" + # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, + # backend and affinity maps. + bpf-lb-map-max: "65536" + bpf-lb-external-clusterip: "false" + + bpf-events-drop-enabled: "true" + bpf-events-policy-verdict-enabled: "true" + bpf-events-trace-enabled: "true" + + # Pre-allocation of map entries allows per-packet latency to be reduced, at + # the expense of up-front memory allocation for the entries in the maps. The + # default value below will minimize memory usage in the default installation; + # users who are sensitive to latency may consider setting this to "true". + # + # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore + # this option and behave as though it is set to "true". + # + # If this value is modified, then during the next Cilium startup the restore + # of existing endpoints and tracking of ongoing connections may be disrupted. + # As a result, reply packets may be dropped and the load-balancing decisions + # for established connections may change. + # + # If this option is set to "false" during an upgrade from 1.3 or earlier to + # 1.4 or later, then it may cause one-time disruptions during the upgrade. + preallocate-bpf-maps: "false" + + # Name of the cluster. Only relevant when building a mesh of clusters. + cluster-name: default + # Unique ID of the cluster. Must be unique across all conneted clusters and + # in the range of 1 and 255. Only relevant when building a mesh of clusters. + cluster-id: "0" + + # Encapsulation mode for communication between nodes + # Possible values: + # - disabled + # - vxlan (default) + # - geneve + # Default case + routing-mode: "tunnel" + tunnel-protocol: "vxlan" + service-no-backend-response: "reject" + + + # Enables L7 proxy for L7 policy enforcement and visibility + enable-l7-proxy: "true" + + enable-ipv4-masquerade: "true" + enable-ipv4-big-tcp: "false" + enable-ipv6-big-tcp: "false" + enable-ipv6-masquerade: "true" + enable-tcx: "true" + datapath-mode: "veth" + enable-masquerade-to-route-source: "false" + + enable-xt-socket-fallback: "true" + install-no-conntrack-iptables-rules: "false" + + auto-direct-node-routes: "false" + direct-routing-skip-unreachable: "false" + enable-local-redirect-policy: "false" + enable-runtime-device-detection: "true" + + kube-proxy-replacement: "true" + kube-proxy-replacement-healthz-bind-address: "" + bpf-lb-sock: "false" + bpf-lb-sock-terminate-pod-connections: "false" + nodeport-addresses: "" + enable-health-check-nodeport: "true" + enable-health-check-loadbalancer-ip: "false" + node-port-bind-protection: "true" + enable-auto-protect-node-port-range: "true" + bpf-lb-acceleration: "disabled" + enable-svc-source-range-check: "true" + enable-l2-neigh-discovery: "true" + arping-refresh-period: "30s" + k8s-require-ipv4-pod-cidr: "false" + k8s-require-ipv6-pod-cidr: "false" + enable-k8s-networkpolicy: "true" + # Tell the agent to generate and write a CNI configuration file + write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist + cni-exclusive: "true" + cni-log-file: "/var/run/cilium/cilium-cni.log" + enable-endpoint-health-checking: "true" + enable-health-checking: "true" + enable-well-known-identities: "false" + enable-node-selector-labels: "false" + synchronize-k8s-nodes: "true" + operator-api-serve-addr: "127.0.0.1:9234" + # Enable Hubble gRPC service. + enable-hubble: "true" + # UNIX domain socket for Hubble server to listen to. + hubble-socket-path: "/var/run/cilium/hubble.sock" + hubble-export-file-max-size-mb: "10" + hubble-export-file-max-backups: "5" + # An additional address for Hubble server to listen to (e.g. ":4244"). + hubble-listen-address: ":4244" + hubble-disable-tls: "false" + hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt + hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key + hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt + hubble-prefer-ipv6: "true" + ipam: "kubernetes" + ipam-cilium-node-update-rate: "15s" + egress-gateway-reconciliation-trigger-interval: "1s" + enable-vtep: "false" + vtep-endpoint: "" + vtep-cidr: "" + vtep-mask: "" + vtep-mac: "" + enable-bgp-control-plane: "true" + bgp-secrets-namespace: "kube-system" + procfs: "/host/proc" + bpf-root: "/sys/fs/bpf" + cgroup-root: "/sys/fs/cgroup" + enable-k8s-terminating-endpoint: "true" + enable-sctp: "false" + + k8s-client-qps: "10" + k8s-client-burst: "20" + remove-cilium-node-taints: "true" + set-cilium-node-taints: "true" + set-cilium-is-up-condition: "true" + unmanaged-pod-watcher-interval: "15" + # default DNS proxy to transparent mode in non-chaining modes + dnsproxy-enable-transparent-mode: "true" + dnsproxy-socket-linger-timeout: "10" + tofqdns-dns-reject-response-code: "refused" + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "50" + tofqdns-idle-connection-grace-period: "0s" + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-proxy-response-max-delay: "100ms" + agent-not-ready-taint-key: "node.cilium.io/agent-not-ready" + + mesh-auth-enabled: "true" + mesh-auth-queue-size: "1024" + mesh-auth-rotated-identities-queue-size: "1024" + mesh-auth-gc-interval: "5m0s" + + proxy-xff-num-trusted-hops-ingress: "0" + proxy-xff-num-trusted-hops-egress: "0" + proxy-connect-timeout: "2" + proxy-initial-fetch-timeout: "30" + proxy-max-requests-per-connection: "0" + proxy-max-connection-duration-seconds: "0" + proxy-idle-timeout-seconds: "60" + + external-envoy-proxy: "true" + envoy-base-id: "0" + + envoy-keep-cap-netbindservice: "false" + max-connected-clusters: "255" + clustermesh-enable-endpoint-sync: "false" + clustermesh-enable-mcs-api: "false" + + nat-map-stats-entries: "32" + nat-map-stats-interval: "30s" + +# Extra config allows adding arbitrary properties to the cilium config. +# By putting it at the end of the ConfigMap, it's also possible to override existing properties. +--- +# Source: cilium/templates/cilium-envoy/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-envoy-config + namespace: kube-system +data: + bootstrap-config.json: | + { + "node": { + "id": "host~127.0.0.1~no-id~localdomain", + "cluster": "ingress-cluster" + }, + "staticResources": { + "listeners": [ + { + "name": "envoy-prometheus-metrics-listener", + "address": { + "socket_address": { + "address": "0.0.0.0", + "port_value": 9964 + } + }, + "filter_chains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "envoy-prometheus-metrics-listener", + "route_config": { + "virtual_hosts": [ + { + "name": "prometheus_metrics_route", + "domains": [ + "*" + ], + "routes": [ + { + "name": "prometheus_metrics_route", + "match": { + "prefix": "/metrics" + }, + "route": { + "cluster": "/envoy-admin", + "prefix_rewrite": "/stats/prometheus" + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ], + "internal_address_config": { + "cidr_ranges": [ + { + "address_prefix": "10.0.0.0", + "prefix_len": 8 + }, + { + "address_prefix": "172.16.0.0", + "prefix_len": 12 + }, + { + "address_prefix": "192.168.0.0", + "prefix_len": 16 + }, + { + "address_prefix": "127.0.0.1", + "prefix_len": 32 + }, + { + "address_prefix": "::1", + "prefix_len": 128 + } + ] + }, + "stream_idle_timeout": "0s" + } + } + ] + } + ] + }, + { + "name": "envoy-health-listener", + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 9878 + } + }, + "additional_addresses": [ + { + "address": { + "socket_address": { + "address": "::1", + "port_value": 9878 + } + } + } + ], + "filter_chains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "envoy-health-listener", + "route_config": { + "virtual_hosts": [ + { + "name": "health", + "domains": [ + "*" + ], + "routes": [ + { + "name": "health", + "match": { + "prefix": "/healthz" + }, + "route": { + "cluster": "/envoy-admin", + "prefix_rewrite": "/ready" + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ], + "internal_address_config": { + "cidr_ranges": [ + { + "address_prefix": "10.0.0.0", + "prefix_len": 8 + }, + { + "address_prefix": "172.16.0.0", + "prefix_len": 12 + }, + { + "address_prefix": "192.168.0.0", + "prefix_len": 16 + }, + { + "address_prefix": "127.0.0.1", + "prefix_len": 32 + }, + { + "address_prefix": "::1", + "prefix_len": 128 + } + ] + }, + "stream_idle_timeout": "0s" + } + } + ] + } + ] + } + ], + "clusters": [ + { + "name": "ingress-cluster", + "type": "ORIGINAL_DST", + "connectTimeout": "2s", + "lbPolicy": "CLUSTER_PROVIDED", + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "commonHttpProtocolOptions": { + "idleTimeout": "60s", + "maxConnectionDuration": "0s", + "maxRequestsPerConnection": 0 + }, + "useDownstreamProtocolConfig": {} + } + }, + "cleanupInterval": "2.500s" + }, + { + "name": "egress-cluster-tls", + "type": "ORIGINAL_DST", + "connectTimeout": "2s", + "lbPolicy": "CLUSTER_PROVIDED", + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "commonHttpProtocolOptions": { + "idleTimeout": "60s", + "maxConnectionDuration": "0s", + "maxRequestsPerConnection": 0 + }, + "upstreamHttpProtocolOptions": {}, + "useDownstreamProtocolConfig": {} + } + }, + "cleanupInterval": "2.500s", + "transportSocket": { + "name": "cilium.tls_wrapper", + "typedConfig": { + "@type": "type.googleapis.com/cilium.UpstreamTlsWrapperContext" + } + } + }, + { + "name": "egress-cluster", + "type": "ORIGINAL_DST", + "connectTimeout": "2s", + "lbPolicy": "CLUSTER_PROVIDED", + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "commonHttpProtocolOptions": { + "idleTimeout": "60s", + "maxConnectionDuration": "0s", + "maxRequestsPerConnection": 0 + }, + "useDownstreamProtocolConfig": {} + } + }, + "cleanupInterval": "2.500s" + }, + { + "name": "ingress-cluster-tls", + "type": "ORIGINAL_DST", + "connectTimeout": "2s", + "lbPolicy": "CLUSTER_PROVIDED", + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "commonHttpProtocolOptions": { + "idleTimeout": "60s", + "maxConnectionDuration": "0s", + "maxRequestsPerConnection": 0 + }, + "upstreamHttpProtocolOptions": {}, + "useDownstreamProtocolConfig": {} + } + }, + "cleanupInterval": "2.500s", + "transportSocket": { + "name": "cilium.tls_wrapper", + "typedConfig": { + "@type": "type.googleapis.com/cilium.UpstreamTlsWrapperContext" + } + } + }, + { + "name": "xds-grpc-cilium", + "type": "STATIC", + "connectTimeout": "2s", + "loadAssignment": { + "clusterName": "xds-grpc-cilium", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "pipe": { + "path": "/var/run/cilium/envoy/sockets/xds.sock" + } + } + } + } + ] + } + ] + }, + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "explicitHttpConfig": { + "http2ProtocolOptions": {} + } + } + } + }, + { + "name": "/envoy-admin", + "type": "STATIC", + "connectTimeout": "2s", + "loadAssignment": { + "clusterName": "/envoy-admin", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "pipe": { + "path": "/var/run/cilium/envoy/sockets/admin.sock" + } + } + } + } + ] + } + ] + } + } + ] + }, + "dynamicResources": { + "ldsConfig": { + "initialFetchTimeout": "30s", + "apiConfigSource": { + "apiType": "GRPC", + "transportApiVersion": "V3", + "grpcServices": [ + { + "envoyGrpc": { + "clusterName": "xds-grpc-cilium" + } + } + ], + "setNodeOnFirstMessageOnly": true + }, + "resourceApiVersion": "V3" + }, + "cdsConfig": { + "initialFetchTimeout": "30s", + "apiConfigSource": { + "apiType": "GRPC", + "transportApiVersion": "V3", + "grpcServices": [ + { + "envoyGrpc": { + "clusterName": "xds-grpc-cilium" + } + } + ], + "setNodeOnFirstMessageOnly": true + }, + "resourceApiVersion": "V3" + } + }, + "bootstrapExtensions": [ + { + "name": "envoy.bootstrap.internal_listener", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener" + } + } + ], + "overload_manager": { + "resource_monitors": [ + { + "name": "envoy.resource_monitors.global_downstream_max_connections", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig", + "max_active_downstream_connections": "50000" + } + } + ] + }, + "admin": { + "address": { + "pipe": { + "path": "/var/run/cilium/envoy/sockets/admin.sock" + } + } + } + } +--- +# Source: cilium/templates/cilium-agent/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - pods + - endpoints + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch + # This is used when validating policies in preflight. This will need to stay + # until we figure out how to avoid "get" inside the preflight, and then + # should be removed ideally. + - get +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumbgppeeringpolicies + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs + - ciliumclusterwideenvoyconfigs + - ciliumclusterwidenetworkpolicies + - ciliumegressgatewaypolicies + - ciliumendpoints + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumidentities + - ciliumlocalredirectpolicies + - ciliumnetworkpolicies + - ciliumnodes + - ciliumnodeconfigs + - ciliumcidrgroups + - ciliuml2announcementpolicies + - ciliumpodippools + verbs: + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + - ciliumendpoints + - ciliumnodes + verbs: + - create +- apiGroups: + - cilium.io + # To synchronize garbage collection of such resources + resources: + - ciliumidentities + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + verbs: + - delete + - get +- apiGroups: + - cilium.io + resources: + - ciliumnodes + - ciliumnodes/status + verbs: + - get + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints/status + - ciliumendpoints + - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status + verbs: + - patch +--- +# Source: cilium/templates/cilium-operator/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-operator + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium + - delete +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - cilium-config + verbs: + # allow patching of the configmap to set annotations + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + # To remove node taints + - nodes + # To set NetworkUnavailable false on startup + - nodes/status + verbs: + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform LB IP allocation for BGP + - services/status + verbs: + - update + - patch +- apiGroups: + - "" + resources: + # to check apiserver connectivity + - namespaces + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform the translation of a CNP that contains `ToGroup` to its endpoints + - services + - endpoints + verbs: + - get + - list + - watch + - create + - update + - delete + - patch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumclusterwidenetworkpolicies + verbs: + # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups' + - create + - update + - deletecollection + # To update the status of the CNPs and CCNPs + - patch + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + verbs: + # Update the auto-generated CNPs and CCNPs status. + - patch + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + - ciliumidentities + verbs: + # To perform garbage collection of such resources + - delete + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + verbs: + # To synchronize garbage collection of such resources + - update +- apiGroups: + - cilium.io + resources: + - ciliumnodes + verbs: + - create + - update + - get + - list + - watch + # To perform CiliumNode garbage collector + - delete +- apiGroups: + - cilium.io + resources: + - ciliumnodes/status + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumbgppeerconfigs + - ciliumbgpadvertisements + - ciliumbgpnodeconfigs + verbs: + - create + - update + - get + - list + - watch + - delete + - patch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - update + resourceNames: + - ciliumloadbalancerippools.cilium.io + - ciliumbgppeeringpolicies.cilium.io + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io + - ciliumclusterwideenvoyconfigs.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumegressgatewaypolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumendpointslices.cilium.io + - ciliumenvoyconfigs.cilium.io + - ciliumexternalworkloads.cilium.io + - ciliumidentities.cilium.io + - ciliumlocalredirectpolicies.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodes.cilium.io + - ciliumnodeconfigs.cilium.io + - ciliumcidrgroups.cilium.io + - ciliuml2announcementpolicies.cilium.io + - ciliumpodippools.cilium.io +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumpodippools + - ciliumbgppeeringpolicies + - ciliumbgpclusterconfigs + - ciliumbgpnodeconfigoverrides + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumpodippools + verbs: + - create +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools/status + verbs: + - patch +# For cilium-operator running in HA mode. +# +# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election +# between multiple running instances. +# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less +# common and fewer objects in the cluster watch "all Leases". +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + - gateways + - tlsroutes + - httproutes + - grpcroutes + - referencegrants + - referencepolicies + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses/status + - gateways/status + - httproutes/status + - grpcroutes/status + - tlsroutes/status + verbs: + - update + - patch +- apiGroups: + - multicluster.x-k8s.io + resources: + - serviceimports + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/cilium-agent/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-operator + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/cilium-agent/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-config-agent + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/cilium-agent/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-gateway-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/cilium-agent/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-bgp-control-plane-secrets + namespace: "kube-system" + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/cilium-operator/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-operator-gateway-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - update + - patch +--- +# Source: cilium/templates/cilium-agent/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-config-agent + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-config-agent +subjects: + - kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-agent/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-gateway-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-gateway-secrets +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-agent/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-bgp-control-plane-secrets + namespace: "kube-system" + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-bgp-control-plane-secrets +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-operator-gateway-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-operator-gateway-secrets +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/cilium-envoy/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: cilium-envoy + namespace: kube-system + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9964" + labels: + k8s-app: cilium-envoy + app.kubernetes.io/name: cilium-envoy + app.kubernetes.io/part-of: cilium + io.cilium/app: proxy +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: cilium-envoy + ports: + - name: envoy-metrics + port: 9964 + protocol: TCP + targetPort: envoy-metrics +--- +# Source: cilium/templates/hubble/peer-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: hubble-peer + namespace: kube-system + labels: + k8s-app: cilium + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: hubble-peer +spec: + selector: + k8s-app: cilium + ports: + - name: peer-service + port: 443 + protocol: TCP + targetPort: 4244 + internalTrafficPolicy: Local +--- +# Source: cilium/templates/cilium-agent/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium + namespace: kube-system + labels: + k8s-app: cilium + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-agent +spec: + selector: + matchLabels: + k8s-app: cilium + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + labels: + k8s-app: cilium + app.kubernetes.io/name: cilium-agent + app.kubernetes.io/part-of: cilium + spec: + securityContext: + appArmorProfile: + type: Unconfined + containers: + - name: cilium-agent + image: "quay.io/cilium/cilium:v1.16.5@sha256:758ca0793f5995bb938a2fa219dcce63dc0b3fa7fc4ce5cc851125281fb7361d" + imagePullPolicy: IfNotPresent + command: + - cilium-agent + args: + - --config-dir=/tmp/cilium/config-map + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + failureThreshold: 105 + periodSeconds: 2 + successThreshold: 1 + initialDelaySeconds: 5 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: GOMEMLIMIT + valueFrom: + resourceFieldRef: + resource: limits.memory + divisor: '1' + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "7445" + lifecycle: + postStart: + exec: + command: + - "bash" + - "-c" + - | + set -o errexit + set -o pipefail + set -o nounset + + # When running in AWS ENI mode, it's likely that 'aws-node' has + # had a chance to install SNAT iptables rules. These can result + # in dropped traffic, so we should attempt to remove them. + # We do it using a 'postStart' hook since this may need to run + # for nodes which might have already been init'ed but may still + # have dangling rules. This is safe because there are no + # dependencies on anything that is part of the startup script + # itself, and can be safely run multiple times per node (e.g. in + # case of a restart). + if [[ "$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; + then + echo 'Deleting iptables rules created by the AWS CNI VPC plugin' + iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore + fi + echo 'Done!' + + preStop: + exec: + command: + - /cni-uninstall.sh + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: envoy-sockets + mountPath: /var/run/cilium/envoy/sockets + readOnly: false + # Unprivileged containers need to mount /proc/sys/net from the host + # to have write access + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + # Unprivileged containers need to mount /proc/sys/kernel from the host + # to have write access + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - name: bpf-maps + mountPath: /sys/fs/bpf + # Unprivileged containers can't set mount propagation to bidirectional + # in this case we will mount the bpf fs from an init container that + # is privileged and set the mount propagation from host to container + # in Cilium. + mountPropagation: HostToContainer + # Check for duplicate mounts before mounting + - name: cilium-cgroup + mountPath: /sys/fs/cgroup + - name: cilium-run + mountPath: /var/run/cilium + - name: etc-cni-netd + mountPath: /host/etc/cni/net.d + - name: clustermesh-secrets + mountPath: /var/lib/cilium/clustermesh + readOnly: true + # Needed to be able to load kernel modules + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + - name: hubble-tls + mountPath: /var/lib/cilium/tls/hubble + readOnly: true + - name: tmp + mountPath: /tmp + initContainers: + - name: config + image: "quay.io/cilium/cilium:v1.16.5@sha256:758ca0793f5995bb938a2fa219dcce63dc0b3fa7fc4ce5cc851125281fb7361d" + imagePullPolicy: IfNotPresent + command: + - cilium-dbg + - build-config + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "7445" + volumeMounts: + - name: tmp + mountPath: /tmp + terminationMessagePolicy: FallbackToLogsOnError + - name: apply-sysctl-overwrites + image: "quay.io/cilium/cilium:v1.16.5@sha256:758ca0793f5995bb938a2fa219dcce63dc0b3fa7fc4ce5cc851125281fb7361d" + imagePullPolicy: IfNotPresent + env: + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + # Mount the bpf fs if it is not mounted. We will perform this task + # from a privileged container because the mount propagation bidirectional + # only works from privileged containers. + - name: mount-bpf-fs + image: "quay.io/cilium/cilium:v1.16.5@sha256:758ca0793f5995bb938a2fa219dcce63dc0b3fa7fc4ce5cc851125281fb7361d" + imagePullPolicy: IfNotPresent + args: + - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' + command: + - /bin/bash + - -c + - -- + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + privileged: true + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + - name: clean-cilium-state + image: "quay.io/cilium/cilium:v1.16.5@sha256:758ca0793f5995bb938a2fa219dcce63dc0b3fa7fc4ce5cc851125281fb7361d" + imagePullPolicy: IfNotPresent + command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-state + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-bpf-state + optional: true + - name: WRITE_CNI_CONF_WHEN_READY + valueFrom: + configMapKeyRef: + name: cilium-config + key: write-cni-conf-when-ready + optional: true + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "7445" + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - NET_ADMIN + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + # Required to mount cgroup filesystem from the host to cilium agent pod + - name: cilium-cgroup + mountPath: /sys/fs/cgroup + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium # wait-for-kube-proxy + # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent + - name: install-cni-binaries + image: "quay.io/cilium/cilium:v1.16.5@sha256:758ca0793f5995bb938a2fa219dcce63dc0b3fa7fc4ce5cc851125281fb7361d" + imagePullPolicy: IfNotPresent + command: + - "/install-plugin.sh" + resources: + requests: + cpu: 100m + memory: 10Mi + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cni-path + mountPath: /host/opt/cni/bin # .Values.cni.install + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccountName: "cilium" + automountServiceAccountToken: true + terminationGracePeriodSeconds: 1 + hostNetwork: true + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + # For sharing configuration between the "config" initContainer and the agent + - name: tmp + emptyDir: {} + # To keep state between restarts / upgrades + - name: cilium-run + hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + # To mount cgroup2 filesystem on the host or apply sysctlfix + - name: hostproc + hostPath: + path: /proc + type: Directory + # To keep state between restarts / upgrades for cgroup2 filesystem + - name: cilium-cgroup + hostPath: + path: /sys/fs/cgroup + type: DirectoryOrCreate + # To install cilium cni plugin in the host + - name: cni-path + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + # To install cilium cni configuration in the host + - name: etc-cni-netd + hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + # To be able to load kernel modules + - name: lib-modules + hostPath: + path: /lib/modules + # To access iptables concurrently with other processes (e.g. kube-proxy) + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Sharing socket with Cilium Envoy on the same node by using a host path + - name: envoy-sockets + hostPath: + path: "/var/run/cilium/envoy/sockets" + type: DirectoryOrCreate + # To read the clustermesh configuration + - name: clustermesh-secrets + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: cilium-clustermesh + optional: true + # note: items are not explicitly listed here, since the entries of this secret + # depend on the peers configured, and that would cause a restart of all agents + # at every addition/removal. Leaving the field empty makes each secret entry + # to be automatically projected into the volume as a file whose name is the key. + - secret: + name: clustermesh-apiserver-remote-cert + optional: true + items: + - key: tls.key + path: common-etcd-client.key + - key: tls.crt + path: common-etcd-client.crt + - key: ca.crt + path: common-etcd-client-ca.crt + # note: we configure the volume for the kvstoremesh-specific certificate + # regardless of whether KVStoreMesh is enabled or not, so that it can be + # automatically mounted in case KVStoreMesh gets subsequently enabled, + # without requiring an agent restart. + - secret: + name: clustermesh-apiserver-local-cert + optional: true + items: + - key: tls.key + path: local-etcd-client.key + - key: tls.crt + path: local-etcd-client.crt + - key: ca.crt + path: local-etcd-client-ca.crt + - name: host-proc-sys-net + hostPath: + path: /proc/sys/net + type: Directory + - name: host-proc-sys-kernel + hostPath: + path: /proc/sys/kernel + type: Directory + - name: hubble-tls + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: hubble-server-certs + optional: true + items: + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key + - key: ca.crt + path: client-ca.crt +--- +# Source: cilium/templates/cilium-envoy/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium-envoy + namespace: kube-system + labels: + k8s-app: cilium-envoy + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-envoy + name: cilium-envoy +spec: + selector: + matchLabels: + k8s-app: cilium-envoy + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + labels: + k8s-app: cilium-envoy + name: cilium-envoy + app.kubernetes.io/name: cilium-envoy + app.kubernetes.io/part-of: cilium + spec: + securityContext: + appArmorProfile: + type: Unconfined + containers: + - name: cilium-envoy + image: "quay.io/cilium/cilium-envoy:v1.30.8-1733837904-eaae5aca0fb988583e5617170a65ac5aa51c0aa8@sha256:709c08ade3d17d52da4ca2af33f431360ec26268d288d9a6cd1d98acc9a1dced" + imagePullPolicy: IfNotPresent + command: + - /usr/bin/cilium-envoy-starter + args: + - '--' + - '-c /var/run/cilium/envoy/bootstrap-config.json' + - '--base-id 0' + - '--log-level info' + - '--log-format [%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v' + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + failureThreshold: 105 + periodSeconds: 2 + successThreshold: 1 + initialDelaySeconds: 5 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "7445" + ports: + - name: envoy-metrics + containerPort: 9964 + hostPort: 9964 + protocol: TCP + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - NET_ADMIN + - SYS_ADMIN + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: envoy-sockets + mountPath: /var/run/cilium/envoy/sockets + readOnly: false + - name: envoy-artifacts + mountPath: /var/run/cilium/envoy/artifacts + readOnly: true + - name: envoy-config + mountPath: /var/run/cilium/envoy/ + readOnly: true + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: HostToContainer + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccountName: "cilium-envoy" + automountServiceAccountToken: true + terminationGracePeriodSeconds: 1 + hostNetwork: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cilium.io/no-schedule + operator: NotIn + values: + - "true" + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium-envoy + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + - name: envoy-sockets + hostPath: + path: "/var/run/cilium/envoy/sockets" + type: DirectoryOrCreate + - name: envoy-artifacts + hostPath: + path: "/var/run/cilium/envoy/artifacts" + type: DirectoryOrCreate + - name: envoy-config + configMap: + name: cilium-envoy-config + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + items: + - key: bootstrap-config.json + path: bootstrap-config.json + # To keep state between restarts / upgrades + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate +--- +# Source: cilium/templates/cilium-operator/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cilium-operator + namespace: kube-system + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator +spec: + # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go + # for more details. + replicas: 2 + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + # ensure operator update on single node k8s clusters, by using rolling update with maxUnavailable=100% in case + # of one replica and no user configured Recreate strategy. + # otherwise an update might get stuck due to the default maxUnavailable=50% in combination with the + # podAntiAffinity which prevents deployments of multiple operator replicas on the same node. + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 50% + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9963" + prometheus.io/scrape: "true" + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator + spec: + containers: + - name: cilium-operator + image: "quay.io/cilium/operator-generic:v1.16.5@sha256:f7884848483bbcd7b1e0ccfd34ba4546f258b460cb4b7e2f06a1bcc96ef88039" + imagePullPolicy: IfNotPresent + command: + - cilium-operator-generic + args: + - --config-dir=/tmp/cilium/config-map + - --debug=$(CILIUM_DEBUG) + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "7445" + ports: + - name: prometheus + containerPort: 9963 + hostPort: 9963 + protocol: TCP + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 5 + volumeMounts: + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + terminationMessagePolicy: FallbackToLogsOnError + hostNetwork: true + restartPolicy: Always + priorityClassName: system-cluster-critical + serviceAccountName: "cilium-operator" + automountServiceAccountToken: true + # In HA mode, cilium-operator pods must not be scheduled on the same + # node as they will clash with each other. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + io.cilium/app: operator + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config diff --git a/cluster_state.sh b/cluster_state.sh new file mode 100755 index 0000000..95e47f7 --- /dev/null +++ b/cluster_state.sh @@ -0,0 +1,10 @@ +#!/usr/bin/bash +NODES="-n 10.51.10.101 -n 10.51.10.103 -n 10.51.10.102" +ENDPOINTS="--endpoints 10.51.10.101" + +PARAMETERS="${NODES} ${ENDPOINTS} --talosconfig=./talosconfig" +echo " === Talos Cluster Members ===" +talosctl ${PARAMETERS} get members +echo +echo " === etcd status ===" +talosctl ${PARAMETERS} etcd status \ No newline at end of file diff --git a/controlplane.yaml b/controlplane.yaml index ad33a94..524bd41 100644 --- a/controlplane.yaml +++ b/controlplane.yaml @@ -454,7 +454,8 @@ cluster: image: registry.k8s.io/kube-apiserver:v1.32.0 # The container image used in the API server manifest. # Extra certificate subject alternative names for the API server's certificate. certSANs: - - ctl.prod.k8.eis-mk8.de.s5b.org + - ctl.prod.k8.eis-mk8.de. + - 2a13:fc80:1:a:ffff::0 disablePodSecurityPolicy: true # Disable PodSecurityPolicy in the API server and default manifests. # Configure the API server admission plugins. admissionControl: diff --git a/patches/00_all.yaml b/patches/00_all.yaml index b33b100..1732f29 100644 --- a/patches/00_all.yaml +++ b/patches/00_all.yaml @@ -1,5 +1,11 @@ # Cluster level patches for all machines machine: + install: + disk: /dev/sda # The disk used for installations. + # image: ghcr.io/siderolabs/installer:v1.9.1 + image: factory.talos.dev/installer/ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515:v1.9.2 + wipe: false # Indicates if the installation disk should be wiped at installation time. + kubelet: nodeIP: validSubnets: @@ -26,17 +32,19 @@ cluster: # Provides cluster specific network configuration options. network: - cni: - name: none + # cni: + # name: none dnsDomain: cluster.local # The domain used by Kubernetes DNS. # The pod subnet CIDR. podSubnets: - 10.244.0.0/16 + - 2a13:fc80:0001:d200::/64 # The service subnet CIDR. serviceSubnets: - 10.96.0.0/12 + - 2a13:fc80:0001:d201::/64 - # kube-proxy - proxy: - disabled: true # we are using Cillium for all networking stuff \ No newline at end of file + # # kube-proxy + # proxy: + # disabled: false # we are using Cillium for all networking stuff \ No newline at end of file diff --git a/patches/11_cilium.yaml b/patches/11_cilium.yaml new file mode 100644 index 0000000..125044c --- /dev/null +++ b/patches/11_cilium.yaml @@ -0,0 +1,2041 @@ +cluster: + inlineManifests: + - name: cilium + contents: | + --- + # Source: cilium/templates/cilium-secrets-namespace.yaml + apiVersion: v1 + kind: Namespace + metadata: + name: "cilium-secrets" + --- + # Source: cilium/templates/cilium-agent/serviceaccount.yaml + apiVersion: v1 + kind: ServiceAccount + metadata: + name: "cilium" + namespace: kube-system + --- + # Source: cilium/templates/cilium-envoy/serviceaccount.yaml + apiVersion: v1 + kind: ServiceAccount + metadata: + name: "cilium-envoy" + namespace: kube-system + --- + # Source: cilium/templates/cilium-operator/serviceaccount.yaml + apiVersion: v1 + kind: ServiceAccount + metadata: + name: "cilium-operator" + namespace: kube-system + --- + # Source: cilium/templates/cilium-ca-secret.yaml + apiVersion: v1 + kind: Secret + metadata: + name: cilium-ca + namespace: kube-system + data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQUk2M1ZNSmJXcFNRcTBCWmVDdk9uL0l3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTURFeE9URTNOREF4TVZvWERUSTRNREV4T1RFMwpOREF4TVZvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUF3YmFaQk5OR1ZEVHJYU0JEQXdiNGVnUkMrTlZHQjdEcHN5a25Ma0ZZZ2JMaTE5Ty8KN2gxT1BJVkJzMmJLQmhwOGFDdW0zdG1VdVZZc0NMUDFocmxrTzQwUzMyaEFuOGY0VzhiRzk5TjJZc0tuY21mRAoxbXdSN1JVeG4zVjh4WmZoUkt4aUZCTHNoSkdGZWpuM1NaQUVqdU5PaVZtUHhORWkyWHAvOVc0SWVhaHVKdXltCnlJdk96Q1VrTFFIZHhHajlTUzhNSWs1RlRLTlNnRjlPcVM2WjcwUzhjYkxBamhPZHNIbDlIb1cxbXJKRm1GSG8KSDhiKy8xb1BTazlmT2k1NzdUWjI4RkJ3SlZqSW1oN2FibU9tTUYxVUtjVzVFOXIxS0tkb1lENEZlblN0NG05OAppNnNJYXpYTWR6UVdxUXlEMGYrWmozQ3BPdFE1ekovZER4T1lkd0lEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGQklYZVpFazB2WXZuWmF2aUU1cm43bTFqOFlkTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ1l2RmNicWlmZzhUb0JJM3J6MDJGTU93M01KUXYxdlpNZE85by9mUE9HM0tpazV4TkhaUXJOClhsRHJ5Z3VHYy91eVQwRjB1Q1lJMTYyWjhiUXR2UVRya0xSNXRRK0ZEakwvelozYThzVHNpbUtFS2s3dEpoKzQKdXdpTHNCVXR2QzBFWVNZQS9BQTc2MmhqRGU2NmxoVmlXa0RTSHNIZmErNTNZc1RrVUtHNXVDNDhxQjltUU9tUApOUzlyaGQxY0xJVWlJYXg5WXlLZXdXTHhzeExxcC8vcTE1aTFFZ1V1UDNjd1ZtZ0ZGa2xCOTUvVlFROVRwejB3CjhhcUVzdHFLMGpPZm53dmtkRTRteXlCQ1hDUWpHUWVTcS9naHpBenRpWGR1dDQ0bFlKcWVHd3FHQ0JhbFk5L2wKLy9FN3dHUWtqaGNVRS9vWUloU2l6TEo3Lzl2Skt6NTcKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBd2JhWkJOTkdWRFRyWFNCREF3YjRlZ1JDK05WR0I3RHBzeWtuTGtGWWdiTGkxOU8vCjdoMU9QSVZCczJiS0JocDhhQ3VtM3RtVXVWWXNDTFAxaHJsa080MFMzMmhBbjhmNFc4Ykc5OU4yWXNLbmNtZkQKMW13UjdSVXhuM1Y4eFpmaFJLeGlGQkxzaEpHRmVqbjNTWkFFanVOT2lWbVB4TkVpMlhwLzlXNEllYWh1SnV5bQp5SXZPekNVa0xRSGR4R2o5U1M4TUlrNUZUS05TZ0Y5T3FTNlo3MFM4Y2JMQWpoT2RzSGw5SG9XMW1ySkZtRkhvCkg4YisvMW9QU2s5Zk9pNTc3VFoyOEZCd0pWakltaDdhYm1PbU1GMVVLY1c1RTlyMUtLZG9ZRDRGZW5TdDRtOTgKaTZzSWF6WE1kelFXcVF5RDBmK1pqM0NwT3RRNXpKL2REeE9ZZHdJREFRQUJBb0lCQUVZRGdFVHRNOE1scVZmcwpYQ3dRN2VmeitGekZnTjB3eFJiYkxlMGE4eWxWaFlXcVkvbGxJQ3pob1FCOWkxRXZhMzJmSVYzMFdEdXdlSlBnCnIwSTFhWlBZaGxTMjhueU5veWRRcmdOT3E4QkFTSk1pQy9kSThpVzFma3o3QXRETDkwQ0I2YVVGdmZTZ1FCVVEKTkYrakxQS1VGNE92R2paeHZGM01WOGxXTHJ3QlVWeXI4Y2pUUVBIV2p5SXlDem1PTzVtSStkdjBJRlBza1YrQwp0SGVLWmN0YUphRUVxRlBpSEZ4aDd5dTljR0xZb1ppYWNyK2dnd1JnR3pyTm56K3FzcHBXaDVib3JrTWFOZWp3Cnc4ZmFScDNTQitHcm5KdC9ZNlBMWVViVUJlSXRBeHlYS1lGdDdUVUJFNjRHbmhPNHg5MHkwTmRwSTM3T1l2YloKaENHZklLa0NnWUVBeG9RdWpyc3BFakpDTFBZcFN3eFNnaWtCQW9uTkgwWWg5UFBRTjhUVXJISWtNUUZzNGdhcQpVMW1pZjlKRXEwZGdTVTFiaDRvUFVwajI2TDFBWUNZbW14THpXQzRjUEdNWmsrR2hLYUtGMDYxbkgyVGtOdmpqCjBvd1c4ZFgzNHBmalJWeFQvR1FFOTlPTDlVRkhBN2JzdkpydzMwTlBNVEZlN1NCOTJudUYrMjBDZ1lFQStjNWUKblZMUU95M09FeXBXOCsxWHdkMHk1WXJTZTg0TzY1MUVSTDc5cGcyYW5lMjVnY3N6NWV1YWU0QjkwaTFFTmpyYgp6Q3lBcFJoZ2FPT2VUZ2thMThDUGJEWkpIUEk5Q0pIbWh3bzBSellPL2NBdlg4U3k4czY0NU1FQzVQZFBGdlBICnVBdWtPSHZ0azA0blRlaVNYWER2anVGNGVLSElNa2NUa2tCWHNQTUNnWUFUVGVWTklld2poZG1qajZpQ1VNZHgKT3RXTFRxVXd4bHdrK3RTanA3TmF0WnJKWFM4ZVo3R0dlMi91NkI2ZlhVaUN1RkFQUlNDcVVvK29JRWN6VW1KcwpzQklUNENBRXkwcEFkQzk4NGY0TGtKQTNvRk1FK1gwR0MzTnJ2dmZsQkFvb3c0cXpQM0t3T3RVZ1cyR3hmY2RWCnErZUNmb1FqNzM0Z3B3azJCczhIblFLQmdHNnVCZk8wbG5neDhrVkJsQ2NkZUsxM2dTNTgxSWR3SStJMjJ1R08Kc2JtWiszQ1B2SXFLYXVNckFWWkk5dlpoZ2dCNUVadWorQmV0OWtGaTRHZjQxc0RURzNpNWtLbFhscjB1VDdKegpUZmhqTmo4dG9JL1JDZXhVdFV6OVMzZ0JMelNmTEZVYUsxZXZhUExFUk52YStNYVgzRHcxSC9ocXpkV2c4a1h4Clppa0xBb0dBQ1pQbkdiZmxUUEFpaHVzVGJLcXB5Zzg3RDhjMUhnZHRvaDExWnRaTXIyclljUFdhdFlTeFJpaGQKd0dsTkxzTlJoVFpocGhUeFdHd3RRVDYxd29XemFRaWlqd3QrVGVqa0w1UU5qak9qSE5Td0VZMVNZMWRBak52SgpNT1RyMEFQSzZ2b2ZINHBJYVBHWTIrQk55UlVZM3dsYmFJZ2p0ZDJGMGFzWWtBcGdFY1U9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== + --- + # Source: cilium/templates/hubble/tls-helm/server-secret.yaml + apiVersion: v1 + kind: Secret + metadata: + name: hubble-server-certs + namespace: kube-system + type: kubernetes.io/tls + data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQUk2M1ZNSmJXcFNRcTBCWmVDdk9uL0l3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTURFeE9URTNOREF4TVZvWERUSTRNREV4T1RFMwpOREF4TVZvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUF3YmFaQk5OR1ZEVHJYU0JEQXdiNGVnUkMrTlZHQjdEcHN5a25Ma0ZZZ2JMaTE5Ty8KN2gxT1BJVkJzMmJLQmhwOGFDdW0zdG1VdVZZc0NMUDFocmxrTzQwUzMyaEFuOGY0VzhiRzk5TjJZc0tuY21mRAoxbXdSN1JVeG4zVjh4WmZoUkt4aUZCTHNoSkdGZWpuM1NaQUVqdU5PaVZtUHhORWkyWHAvOVc0SWVhaHVKdXltCnlJdk96Q1VrTFFIZHhHajlTUzhNSWs1RlRLTlNnRjlPcVM2WjcwUzhjYkxBamhPZHNIbDlIb1cxbXJKRm1GSG8KSDhiKy8xb1BTazlmT2k1NzdUWjI4RkJ3SlZqSW1oN2FibU9tTUYxVUtjVzVFOXIxS0tkb1lENEZlblN0NG05OAppNnNJYXpYTWR6UVdxUXlEMGYrWmozQ3BPdFE1ekovZER4T1lkd0lEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGQklYZVpFazB2WXZuWmF2aUU1cm43bTFqOFlkTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ1l2RmNicWlmZzhUb0JJM3J6MDJGTU93M01KUXYxdlpNZE85by9mUE9HM0tpazV4TkhaUXJOClhsRHJ5Z3VHYy91eVQwRjB1Q1lJMTYyWjhiUXR2UVRya0xSNXRRK0ZEakwvelozYThzVHNpbUtFS2s3dEpoKzQKdXdpTHNCVXR2QzBFWVNZQS9BQTc2MmhqRGU2NmxoVmlXa0RTSHNIZmErNTNZc1RrVUtHNXVDNDhxQjltUU9tUApOUzlyaGQxY0xJVWlJYXg5WXlLZXdXTHhzeExxcC8vcTE1aTFFZ1V1UDNjd1ZtZ0ZGa2xCOTUvVlFROVRwejB3CjhhcUVzdHFLMGpPZm53dmtkRTRteXlCQ1hDUWpHUWVTcS9naHpBenRpWGR1dDQ0bFlKcWVHd3FHQ0JhbFk5L2wKLy9FN3dHUWtqaGNVRS9vWUloU2l6TEo3Lzl2Skt6NTcKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWekNDQWorZ0F3SUJBZ0lSQU9RVEFLRys4c2YvejVZeW8vREszNzR3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTURFeE9URTNOREF4TVZvWERUSTJNREV4T1RFMwpOREF4TVZvd0tqRW9NQ1lHQTFVRUF3d2ZLaTVrWldaaGRXeDBMbWgxWW1Kc1pTMW5jbkJqTG1OcGJHbDFiUzVwCmJ6Q0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQU5nUUZBT2lFKytheVljOHVpanMKL2JCamNuZExwbkRLOGZTOUVGbENDaDVwSVV2UnoycGJFS24vaTM4WHk4R09DMUNxK3JHcmVWOTNtVlZoZkd3bQovWTNoTjg4dXV5NnEvRGJXTk84eXI4TXRJZ0pyMHpVbGtKcHZ5T1drMEx2ZWdPb1RycmZ4c3Rzd3RybEN3QUJECk9jZENFR2x2eHNrQVZZbzJsODJpVDdKeDRldmo3bEFOMnZNQzFUSnF0dmxQS2tYOG9YSlZPOW9nN250TDd6bC8KbzZvdHNSWGZFMjBVNHJoRFBuUkI5azNSVFB5cURCS1ZqbFpZTTQ5dlVuQzlSTWZrMTRHNVdKR1hEQVhYWnJacQpDdkpPckY1OUZRU2FNSjFxd2ZWbklLNUEzaVhWRUZkZTlOeC9rbXdTMHVGOHNId1ZYQ3VUeitGcGVPQnB3TXlPClFMY0NBd0VBQWFPQmpUQ0JpakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdIUVlEVlIwbEJCWXdGQVlJS3dZQkJRVUgKQXdFR0NDc0dBUVVGQndNQ01Bd0dBMVVkRXdFQi93UUNNQUF3SHdZRFZSMGpCQmd3Rm9BVUVoZDVrU1RTOWkrZApscStJVG11ZnViV1B4aDB3S2dZRFZSMFJCQ013SVlJZktpNWtaV1poZFd4MExtaDFZbUpzWlMxbmNuQmpMbU5wCmJHbDFiUzVwYnpBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQVVUKys3KzBsR0JzMmFIRnZwT0dCcWI3MEtUTGIKMFNSOGRTZlVPVzIwRmtqeEhuWFR5RHlyOWkvZytvT2F4RVp6c2psSG44MlJuWmIyczI4NzZSbVIvWE9aVXR6TgpEQUVIcXdpZXdTUzhReDViTGdKa1I5SzNBUHhZM3RCS29ENi9YM3hyRkZLQ0J1aHcyZytVU2dDSGEyQUgwb3h6ClVYRlVEK1dVZGZ1SW5Ed0s1VzgyanBITUR6cFpwUDhSaTRWZGR1WC93M1hFKzlIMUxaRlhkd1FMTU9aVmdWL0cKVWFieGNVNlc1MHpaaUdHelBLeUlZTk1WNVNMa1RINmVCekQyNWxCTUhJZFNaUEtIdUlPNmFIUStxQVdiVUU2QwpUZUd1ejdvcTAwbGVKcUdjdnJvemZhVW1DQk9wZ2hEd29hTjh1bXFmSEZKUFA1ZmtySmlWZEJzU0Z3PT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcFFJQkFBS0NBUUVBMkJBVUE2SVQ3NXJKaHp5NktPejlzR055ZDB1bWNNcng5TDBRV1VJS0hta2hTOUhQCmFsc1FxZitMZnhmTHdZNExVS3I2c2F0NVgzZVpWV0Y4YkNiOWplRTN6eTY3THFyOE50WTA3ekt2d3kwaUFtdlQKTlNXUW1tL0k1YVRRdTk2QTZoT3V0L0d5MnpDMnVVTEFBRU01eDBJUWFXL0d5UUJWaWphWHphSlBzbkhoNitQdQpVQTNhOHdMVk1tcTIrVThxUmZ5aGNsVTcyaUR1ZTB2dk9YK2pxaTJ4RmQ4VGJSVGl1RU0rZEVIMlRkRk0vS29NCkVwV09WbGd6ajI5U2NMMUV4K1RYZ2JsWWtaY01CZGRtdG1vSzhrNnNYbjBWQkpvd25XckI5V2NncmtEZUpkVVEKVjE3MDNIK1NiQkxTNFh5d2ZCVmNLNVBQNFdsNDRHbkF6STVBdHdJREFRQUJBb0lCQVFDZ2ZEalJoZ2IyMTJ0UApwMXpEWDEwRU1xbm9VR205cUJEV2I4Ui8rQnlhQUE2NlNZcWQvaXcvaVUvdFJCaVZtVk53UkwvWFRuM2wybWtyClVURGhDSWtYWksxV2wraHpQby8rajFkY1FMTHE3MUF1cWphRWFIV2pNTnJ4TUYzVzNldTFjbzg1bEM3OHZvOUsKeG5qaytRb21wanlZaEcwTUNkSi95aE14cTVrMCtpR2NvalU2SWY3VU56bXNLV1ZVM1REMzNJVForSVplWElYUgpoelRVaUZKS0p4dUJoTlhnSThOQTRqRzd0VWZJT3lGQm81dVFva084K1lldUx1eWxOK0RUWFB1YzFlRC9GQW0zCnFXVmJ0ZTQrY0F6SmI3WDdhcHV4aDhnTEtET0prcGM0MmpROWRaNlBuTEZqS3YyTWJRVGxpOHZRa1c0M1RhNW8KRUNNM2daVkpBb0dCQVA0ci9sbloyVmtZYkV2dVAwT2RWMG1pc3htTnNrT2h6b2gwQVlnMWhDeDdTQVRjd2thMApnMmZLOWc4MVVrYTgwcllRVW00ZC9yV0lpMWpVcnU1WkRBbHFmN1daUlErTFVkSHhJb3k2RnBYRE16Y1R3MU5yCjhwOXpkTFJzZkd1d3lHSDJ1Z0J2bG03SEIvQUN2NFNCWGl0WjBoa0JzdGN0ZGRiN1BXVFdNcVc5QW9HQkFObWQKNmhwMU1GKzQ0M0dMNHJONjYyNHluMU9qUUExcUJQbTRackE0clJPanM0L2xLQzduMFFiMEVVRno2NGRXaXJUSwpPbXh1UXRGTVV0djdqZVZCdEFlSUM4c3ZERFRZbmd2VWY5dG5yalluVnBnQ1JVTmk1UUcxdEZtUy9kM3VvbTUyCkNwU0M2WXRubWFJcmxWMStJN2d0N2I2c1ZLOWpzZzRQL1J5R2pjV0RBb0dCQU5Xd3NSQmptbHRRc0VRS1V3QUsKdVljOEZhMm5OcmVyd3hOdnpKWHl3R1FSWmRSSXNjRjJKSUJQNEIyYks4Z3pHa2hkWXFBWDhidzN4MGZ3ei8ydAp1ZlAwcU5oSnV3azlqWldmVVBmVndBVTdic1NnaGZpTjZoYnFuN1RNL0xJaGwrYmM0elpZbXJZWitXOXRNQTA4CklxcjV3aXlxQSt6cTJJYmRCWm5RaUJaVkFvR0FiaWlBOCttMGxoQUttbFRjYWt4ajF3R3JJM1IxSFJVNDlZeEIKOXh5SmlIS0tsK0hCaUpUeWU5bWF6bzQyZ09qdkIvdUN2WmJISGhuTHErM1g4MkJpR0pubWxuRDNkZHpkakZHQwpjM2MxZWRaS0FHTE4yd0c1dVRiWTE2c1A5WU15R0VhS3JvamlYaDRRYUVZMXN3RUNaZDVpQmU1SVpBeE1lcy9mCktjeDNBYThDZ1lFQTYyQWcwWWpTTnh1ZmJiMVJZQUEvRFNjU3JYeTBQMVNLVzZtck9WQ3lWRjVqYkxGNVhjVWEKZ20wV1FKWnFhQVRrdGMwQlgvVGV5Q2xPdVFkM2E5ZG1mNlJEcm9uM1p5bXVaT09ZeFlHc3pHN1Y3V1pTc0NyVQoxS2trcVdxNFpWcnlWUU5QTTJsTUI3T1BjMXRBSjJvd0o2SVVubHRkSXRIb0k5U0pwZ1VZdTYwPQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= + --- + # Source: cilium/templates/cilium-configmap.yaml + apiVersion: v1 + kind: ConfigMap + metadata: + name: cilium-config + namespace: kube-system + data: + + # Identity allocation mode selects how identities are shared between cilium + # nodes by setting how they are stored. The options are "crd" or "kvstore". + # - "crd" stores identities in kubernetes as CRDs (custom resource definition). + # These can be queried with: + # kubectl get ciliumid + # - "kvstore" stores identities in an etcd kvstore, that is + # configured below. Cilium versions before 1.6 supported only the kvstore + # backend. Upgrades from these older cilium versions should continue using + # the kvstore by commenting out the identity-allocation-mode below, or + # setting it to "kvstore". + identity-allocation-mode: crd + identity-heartbeat-timeout: "30m0s" + identity-gc-interval: "15m0s" + cilium-endpoint-gc-interval: "5m0s" + nodes-gc-interval: "5m0s" + + # If you want to run cilium in debug mode change this value to true + debug: "false" + debug-verbose: "" + # The agent can be put into the following three policy enforcement modes + # default, always and never. + # https://docs.cilium.io/en/latest/security/policy/intro/#policy-enforcement-modes + enable-policy: "default" + policy-cidr-match-mode: "" + # If you want metrics enabled in cilium-operator, set the port for + # which the Cilium Operator will have their metrics exposed. + # NOTE that this will open the port on the nodes where Cilium operator pod + # is scheduled. + operator-prometheus-serve-addr: ":9963" + enable-metrics: "true" + enable-envoy-config: "true" + envoy-config-retry-interval: "15s" + enable-gateway-api: "true" + enable-gateway-api-secrets-sync: "true" + enable-gateway-api-proxy-protocol: "false" + enable-gateway-api-app-protocol: "true" + enable-gateway-api-alpn: "true" + gateway-api-xff-num-trusted-hops: "0" + gateway-api-service-externaltrafficpolicy: "Cluster" + gateway-api-secrets-namespace: "cilium-secrets" + gateway-api-hostnetwork-enabled: "false" + gateway-api-hostnetwork-nodelabelselector: "" + + # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 + # address. + enable-ipv4: "true" + + # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 + # address. + enable-ipv6: "true" + # Users who wish to specify their own custom CNI configuration file must set + # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. + custom-cni-conf: "false" + enable-bpf-clock-probe: "false" + # If you want cilium monitor to aggregate tracing for packets, set this level + # to "low", "medium", or "maximum". The higher the level, the less packets + # that will be seen in monitor output. + monitor-aggregation: medium + + # The monitor aggregation interval governs the typical time between monitor + # notification events for each allowed connection. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-interval: "5s" + + # The monitor aggregation flags determine which TCP flags which, upon the + # first observation, cause monitor notifications to be generated. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-flags: all + # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic + # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. + bpf-map-dynamic-size-ratio: "0.0025" + # bpf-policy-map-max specifies the maximum number of entries in endpoint + # policy map (per endpoint) + bpf-policy-map-max: "16384" + # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, + # backend and affinity maps. + bpf-lb-map-max: "65536" + bpf-lb-external-clusterip: "false" + + bpf-events-drop-enabled: "true" + bpf-events-policy-verdict-enabled: "true" + bpf-events-trace-enabled: "true" + + # Pre-allocation of map entries allows per-packet latency to be reduced, at + # the expense of up-front memory allocation for the entries in the maps. The + # default value below will minimize memory usage in the default installation; + # users who are sensitive to latency may consider setting this to "true". + # + # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore + # this option and behave as though it is set to "true". + # + # If this value is modified, then during the next Cilium startup the restore + # of existing endpoints and tracking of ongoing connections may be disrupted. + # As a result, reply packets may be dropped and the load-balancing decisions + # for established connections may change. + # + # If this option is set to "false" during an upgrade from 1.3 or earlier to + # 1.4 or later, then it may cause one-time disruptions during the upgrade. + preallocate-bpf-maps: "false" + + # Name of the cluster. Only relevant when building a mesh of clusters. + cluster-name: default + # Unique ID of the cluster. Must be unique across all conneted clusters and + # in the range of 1 and 255. Only relevant when building a mesh of clusters. + cluster-id: "0" + + # Encapsulation mode for communication between nodes + # Possible values: + # - disabled + # - vxlan (default) + # - geneve + # Default case + routing-mode: "tunnel" + tunnel-protocol: "vxlan" + service-no-backend-response: "reject" + + + # Enables L7 proxy for L7 policy enforcement and visibility + enable-l7-proxy: "true" + + enable-ipv4-masquerade: "true" + enable-ipv4-big-tcp: "false" + enable-ipv6-big-tcp: "false" + enable-ipv6-masquerade: "true" + enable-tcx: "true" + datapath-mode: "veth" + enable-masquerade-to-route-source: "false" + + enable-xt-socket-fallback: "true" + install-no-conntrack-iptables-rules: "false" + + auto-direct-node-routes: "false" + direct-routing-skip-unreachable: "false" + enable-local-redirect-policy: "false" + enable-runtime-device-detection: "true" + + kube-proxy-replacement: "true" + kube-proxy-replacement-healthz-bind-address: "" + bpf-lb-sock: "false" + bpf-lb-sock-terminate-pod-connections: "false" + nodeport-addresses: "" + enable-health-check-nodeport: "true" + enable-health-check-loadbalancer-ip: "false" + node-port-bind-protection: "true" + enable-auto-protect-node-port-range: "true" + bpf-lb-acceleration: "disabled" + enable-svc-source-range-check: "true" + enable-l2-neigh-discovery: "true" + arping-refresh-period: "30s" + k8s-require-ipv4-pod-cidr: "false" + k8s-require-ipv6-pod-cidr: "false" + enable-k8s-networkpolicy: "true" + # Tell the agent to generate and write a CNI configuration file + write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist + cni-exclusive: "true" + cni-log-file: "/var/run/cilium/cilium-cni.log" + enable-endpoint-health-checking: "true" + enable-health-checking: "true" + enable-well-known-identities: "false" + enable-node-selector-labels: "false" + synchronize-k8s-nodes: "true" + operator-api-serve-addr: "127.0.0.1:9234" + # Enable Hubble gRPC service. + enable-hubble: "true" + # UNIX domain socket for Hubble server to listen to. + hubble-socket-path: "/var/run/cilium/hubble.sock" + hubble-export-file-max-size-mb: "10" + hubble-export-file-max-backups: "5" + # An additional address for Hubble server to listen to (e.g. ":4244"). + hubble-listen-address: ":4244" + hubble-disable-tls: "false" + hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt + hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key + hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt + hubble-prefer-ipv6: "true" + ipam: "kubernetes" + ipam-cilium-node-update-rate: "15s" + egress-gateway-reconciliation-trigger-interval: "1s" + enable-vtep: "false" + vtep-endpoint: "" + vtep-cidr: "" + vtep-mask: "" + vtep-mac: "" + enable-bgp-control-plane: "true" + bgp-secrets-namespace: "kube-system" + procfs: "/host/proc" + bpf-root: "/sys/fs/bpf" + cgroup-root: "/sys/fs/cgroup" + enable-k8s-terminating-endpoint: "true" + enable-sctp: "false" + + k8s-client-qps: "10" + k8s-client-burst: "20" + remove-cilium-node-taints: "true" + set-cilium-node-taints: "true" + set-cilium-is-up-condition: "true" + unmanaged-pod-watcher-interval: "15" + # default DNS proxy to transparent mode in non-chaining modes + dnsproxy-enable-transparent-mode: "true" + dnsproxy-socket-linger-timeout: "10" + tofqdns-dns-reject-response-code: "refused" + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "50" + tofqdns-idle-connection-grace-period: "0s" + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-proxy-response-max-delay: "100ms" + agent-not-ready-taint-key: "node.cilium.io/agent-not-ready" + + mesh-auth-enabled: "true" + mesh-auth-queue-size: "1024" + mesh-auth-rotated-identities-queue-size: "1024" + mesh-auth-gc-interval: "5m0s" + + proxy-xff-num-trusted-hops-ingress: "0" + proxy-xff-num-trusted-hops-egress: "0" + proxy-connect-timeout: "2" + proxy-initial-fetch-timeout: "30" + proxy-max-requests-per-connection: "0" + proxy-max-connection-duration-seconds: "0" + proxy-idle-timeout-seconds: "60" + + external-envoy-proxy: "true" + envoy-base-id: "0" + + envoy-keep-cap-netbindservice: "false" + max-connected-clusters: "255" + clustermesh-enable-endpoint-sync: "false" + clustermesh-enable-mcs-api: "false" + + nat-map-stats-entries: "32" + nat-map-stats-interval: "30s" + + # Extra config allows adding arbitrary properties to the cilium config. + # By putting it at the end of the ConfigMap, it's also possible to override existing properties. + --- + # Source: cilium/templates/cilium-envoy/configmap.yaml + apiVersion: v1 + kind: ConfigMap + metadata: + name: cilium-envoy-config + namespace: kube-system + data: + bootstrap-config.json: | + { + "node": { + "id": "host~127.0.0.1~no-id~localdomain", + "cluster": "ingress-cluster" + }, + "staticResources": { + "listeners": [ + { + "name": "envoy-prometheus-metrics-listener", + "address": { + "socket_address": { + "address": "0.0.0.0", + "port_value": 9964 + } + }, + "filter_chains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "envoy-prometheus-metrics-listener", + "route_config": { + "virtual_hosts": [ + { + "name": "prometheus_metrics_route", + "domains": [ + "*" + ], + "routes": [ + { + "name": "prometheus_metrics_route", + "match": { + "prefix": "/metrics" + }, + "route": { + "cluster": "/envoy-admin", + "prefix_rewrite": "/stats/prometheus" + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ], + "internal_address_config": { + "cidr_ranges": [ + { + "address_prefix": "10.0.0.0", + "prefix_len": 8 + }, + { + "address_prefix": "172.16.0.0", + "prefix_len": 12 + }, + { + "address_prefix": "192.168.0.0", + "prefix_len": 16 + }, + { + "address_prefix": "127.0.0.1", + "prefix_len": 32 + }, + { + "address_prefix": "::1", + "prefix_len": 128 + } + ] + }, + "stream_idle_timeout": "0s" + } + } + ] + } + ] + }, + { + "name": "envoy-health-listener", + "address": { + "socket_address": { + "address": "127.0.0.1", + "port_value": 9878 + } + }, + "additional_addresses": [ + { + "address": { + "socket_address": { + "address": "::1", + "port_value": 9878 + } + } + } + ], + "filter_chains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "envoy-health-listener", + "route_config": { + "virtual_hosts": [ + { + "name": "health", + "domains": [ + "*" + ], + "routes": [ + { + "name": "health", + "match": { + "prefix": "/healthz" + }, + "route": { + "cluster": "/envoy-admin", + "prefix_rewrite": "/ready" + } + } + ] + } + ] + }, + "http_filters": [ + { + "name": "envoy.filters.http.router", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + } + } + ], + "internal_address_config": { + "cidr_ranges": [ + { + "address_prefix": "10.0.0.0", + "prefix_len": 8 + }, + { + "address_prefix": "172.16.0.0", + "prefix_len": 12 + }, + { + "address_prefix": "192.168.0.0", + "prefix_len": 16 + }, + { + "address_prefix": "127.0.0.1", + "prefix_len": 32 + }, + { + "address_prefix": "::1", + "prefix_len": 128 + } + ] + }, + "stream_idle_timeout": "0s" + } + } + ] + } + ] + } + ], + "clusters": [ + { + "name": "ingress-cluster", + "type": "ORIGINAL_DST", + "connectTimeout": "2s", + "lbPolicy": "CLUSTER_PROVIDED", + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "commonHttpProtocolOptions": { + "idleTimeout": "60s", + "maxConnectionDuration": "0s", + "maxRequestsPerConnection": 0 + }, + "useDownstreamProtocolConfig": {} + } + }, + "cleanupInterval": "2.500s" + }, + { + "name": "egress-cluster-tls", + "type": "ORIGINAL_DST", + "connectTimeout": "2s", + "lbPolicy": "CLUSTER_PROVIDED", + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "commonHttpProtocolOptions": { + "idleTimeout": "60s", + "maxConnectionDuration": "0s", + "maxRequestsPerConnection": 0 + }, + "upstreamHttpProtocolOptions": {}, + "useDownstreamProtocolConfig": {} + } + }, + "cleanupInterval": "2.500s", + "transportSocket": { + "name": "cilium.tls_wrapper", + "typedConfig": { + "@type": "type.googleapis.com/cilium.UpstreamTlsWrapperContext" + } + } + }, + { + "name": "egress-cluster", + "type": "ORIGINAL_DST", + "connectTimeout": "2s", + "lbPolicy": "CLUSTER_PROVIDED", + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "commonHttpProtocolOptions": { + "idleTimeout": "60s", + "maxConnectionDuration": "0s", + "maxRequestsPerConnection": 0 + }, + "useDownstreamProtocolConfig": {} + } + }, + "cleanupInterval": "2.500s" + }, + { + "name": "ingress-cluster-tls", + "type": "ORIGINAL_DST", + "connectTimeout": "2s", + "lbPolicy": "CLUSTER_PROVIDED", + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "commonHttpProtocolOptions": { + "idleTimeout": "60s", + "maxConnectionDuration": "0s", + "maxRequestsPerConnection": 0 + }, + "upstreamHttpProtocolOptions": {}, + "useDownstreamProtocolConfig": {} + } + }, + "cleanupInterval": "2.500s", + "transportSocket": { + "name": "cilium.tls_wrapper", + "typedConfig": { + "@type": "type.googleapis.com/cilium.UpstreamTlsWrapperContext" + } + } + }, + { + "name": "xds-grpc-cilium", + "type": "STATIC", + "connectTimeout": "2s", + "loadAssignment": { + "clusterName": "xds-grpc-cilium", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "pipe": { + "path": "/var/run/cilium/envoy/sockets/xds.sock" + } + } + } + } + ] + } + ] + }, + "typedExtensionProtocolOptions": { + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": { + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions", + "explicitHttpConfig": { + "http2ProtocolOptions": {} + } + } + } + }, + { + "name": "/envoy-admin", + "type": "STATIC", + "connectTimeout": "2s", + "loadAssignment": { + "clusterName": "/envoy-admin", + "endpoints": [ + { + "lbEndpoints": [ + { + "endpoint": { + "address": { + "pipe": { + "path": "/var/run/cilium/envoy/sockets/admin.sock" + } + } + } + } + ] + } + ] + } + } + ] + }, + "dynamicResources": { + "ldsConfig": { + "initialFetchTimeout": "30s", + "apiConfigSource": { + "apiType": "GRPC", + "transportApiVersion": "V3", + "grpcServices": [ + { + "envoyGrpc": { + "clusterName": "xds-grpc-cilium" + } + } + ], + "setNodeOnFirstMessageOnly": true + }, + "resourceApiVersion": "V3" + }, + "cdsConfig": { + "initialFetchTimeout": "30s", + "apiConfigSource": { + "apiType": "GRPC", + "transportApiVersion": "V3", + "grpcServices": [ + { + "envoyGrpc": { + "clusterName": "xds-grpc-cilium" + } + } + ], + "setNodeOnFirstMessageOnly": true + }, + "resourceApiVersion": "V3" + } + }, + "bootstrapExtensions": [ + { + "name": "envoy.bootstrap.internal_listener", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener" + } + } + ], + "overload_manager": { + "resource_monitors": [ + { + "name": "envoy.resource_monitors.global_downstream_max_connections", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig", + "max_active_downstream_connections": "50000" + } + } + ] + }, + "admin": { + "address": { + "pipe": { + "path": "/var/run/cilium/envoy/sockets/admin.sock" + } + } + } + } + --- + # Source: cilium/templates/cilium-agent/clusterrole.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: cilium + labels: + app.kubernetes.io/part-of: cilium + rules: + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - namespaces + - services + - pods + - endpoints + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch + # This is used when validating policies in preflight. This will need to stay + # until we figure out how to avoid "get" inside the preflight, and then + # should be removed ideally. + - get + - apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumbgppeeringpolicies + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs + - ciliumclusterwideenvoyconfigs + - ciliumclusterwidenetworkpolicies + - ciliumegressgatewaypolicies + - ciliumendpoints + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumidentities + - ciliumlocalredirectpolicies + - ciliumnetworkpolicies + - ciliumnodes + - ciliumnodeconfigs + - ciliumcidrgroups + - ciliuml2announcementpolicies + - ciliumpodippools + verbs: + - list + - watch + - apiGroups: + - cilium.io + resources: + - ciliumidentities + - ciliumendpoints + - ciliumnodes + verbs: + - create + - apiGroups: + - cilium.io + # To synchronize garbage collection of such resources + resources: + - ciliumidentities + verbs: + - update + - apiGroups: + - cilium.io + resources: + - ciliumendpoints + verbs: + - delete + - get + - apiGroups: + - cilium.io + resources: + - ciliumnodes + - ciliumnodes/status + verbs: + - get + - update + - apiGroups: + - cilium.io + resources: + - ciliumendpoints/status + - ciliumendpoints + - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status + verbs: + - patch + --- + # Source: cilium/templates/cilium-operator/clusterrole.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: cilium-operator + labels: + app.kubernetes.io/part-of: cilium + rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium + - delete + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - cilium-config + verbs: + # allow patching of the configmap to set annotations + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - apiGroups: + - "" + resources: + # To remove node taints + - nodes + # To set NetworkUnavailable false on startup + - nodes/status + verbs: + - patch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + # to perform LB IP allocation for BGP + - services/status + verbs: + - update + - patch + - apiGroups: + - "" + resources: + # to check apiserver connectivity + - namespaces + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + # to perform the translation of a CNP that contains `ToGroup` to its endpoints + - services + - endpoints + verbs: + - get + - list + - watch + - create + - update + - delete + - patch + - apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumclusterwidenetworkpolicies + verbs: + # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups' + - create + - update + - deletecollection + # To update the status of the CNPs and CCNPs + - patch + - get + - list + - watch + - apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + verbs: + # Update the auto-generated CNPs and CCNPs status. + - patch + - update + - apiGroups: + - cilium.io + resources: + - ciliumendpoints + - ciliumidentities + verbs: + # To perform garbage collection of such resources + - delete + - list + - watch + - apiGroups: + - cilium.io + resources: + - ciliumidentities + verbs: + # To synchronize garbage collection of such resources + - update + - apiGroups: + - cilium.io + resources: + - ciliumnodes + verbs: + - create + - update + - get + - list + - watch + # To perform CiliumNode garbage collector + - delete + - apiGroups: + - cilium.io + resources: + - ciliumnodes/status + verbs: + - update + - apiGroups: + - cilium.io + resources: + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumbgppeerconfigs + - ciliumbgpadvertisements + - ciliumbgpnodeconfigs + verbs: + - create + - update + - get + - list + - watch + - delete + - patch + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - update + resourceNames: + - ciliumloadbalancerippools.cilium.io + - ciliumbgppeeringpolicies.cilium.io + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io + - ciliumclusterwideenvoyconfigs.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumegressgatewaypolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumendpointslices.cilium.io + - ciliumenvoyconfigs.cilium.io + - ciliumexternalworkloads.cilium.io + - ciliumidentities.cilium.io + - ciliumlocalredirectpolicies.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodes.cilium.io + - ciliumnodeconfigs.cilium.io + - ciliumcidrgroups.cilium.io + - ciliuml2announcementpolicies.cilium.io + - ciliumpodippools.cilium.io + - apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumpodippools + - ciliumbgppeeringpolicies + - ciliumbgpclusterconfigs + - ciliumbgpnodeconfigoverrides + verbs: + - get + - list + - watch + - apiGroups: + - cilium.io + resources: + - ciliumpodippools + verbs: + - create + - apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools/status + verbs: + - patch + # For cilium-operator running in HA mode. + # + # Cilium operator running in HA mode requires the use of ResourceLock for Leader Election + # between multiple running instances. + # The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less + # common and fewer objects in the cluster watch "all Leases". + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update + - apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + - gateways + - tlsroutes + - httproutes + - grpcroutes + - referencegrants + - referencepolicies + verbs: + - get + - list + - watch + - apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses/status + - gateways/status + - httproutes/status + - grpcroutes/status + - tlsroutes/status + verbs: + - update + - patch + - apiGroups: + - multicluster.x-k8s.io + resources: + - serviceimports + verbs: + - get + - list + - watch + --- + # Source: cilium/templates/cilium-agent/clusterrolebinding.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cilium + labels: + app.kubernetes.io/part-of: cilium + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium + subjects: + - kind: ServiceAccount + name: "cilium" + namespace: kube-system + --- + # Source: cilium/templates/cilium-operator/clusterrolebinding.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: cilium-operator + labels: + app.kubernetes.io/part-of: cilium + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator + subjects: + - kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system + --- + # Source: cilium/templates/cilium-agent/role.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: cilium-config-agent + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium + rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + --- + # Source: cilium/templates/cilium-agent/role.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: cilium-gateway-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium + rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + --- + # Source: cilium/templates/cilium-agent/role.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: cilium-bgp-control-plane-secrets + namespace: "kube-system" + labels: + app.kubernetes.io/part-of: cilium + rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + --- + # Source: cilium/templates/cilium-operator/role.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: cilium-operator-gateway-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium + rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - update + - patch + --- + # Source: cilium/templates/cilium-agent/rolebinding.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cilium-config-agent + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-config-agent + subjects: + - kind: ServiceAccount + name: "cilium" + namespace: kube-system + --- + # Source: cilium/templates/cilium-agent/rolebinding.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cilium-gateway-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-gateway-secrets + subjects: + - kind: ServiceAccount + name: "cilium" + namespace: kube-system + --- + # Source: cilium/templates/cilium-agent/rolebinding.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cilium-bgp-control-plane-secrets + namespace: "kube-system" + labels: + app.kubernetes.io/part-of: cilium + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-bgp-control-plane-secrets + subjects: + - kind: ServiceAccount + name: "cilium" + namespace: kube-system + --- + # Source: cilium/templates/cilium-operator/rolebinding.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cilium-operator-gateway-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-operator-gateway-secrets + subjects: + - kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system + --- + # Source: cilium/templates/cilium-envoy/service.yaml + apiVersion: v1 + kind: Service + metadata: + name: cilium-envoy + namespace: kube-system + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9964" + labels: + k8s-app: cilium-envoy + app.kubernetes.io/name: cilium-envoy + app.kubernetes.io/part-of: cilium + io.cilium/app: proxy + spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: cilium-envoy + ports: + - name: envoy-metrics + port: 9964 + protocol: TCP + targetPort: envoy-metrics + --- + # Source: cilium/templates/hubble/peer-service.yaml + apiVersion: v1 + kind: Service + metadata: + name: hubble-peer + namespace: kube-system + labels: + k8s-app: cilium + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: hubble-peer + spec: + selector: + k8s-app: cilium + ports: + - name: peer-service + port: 443 + protocol: TCP + targetPort: 4244 + internalTrafficPolicy: Local + --- + # Source: cilium/templates/cilium-agent/daemonset.yaml + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cilium + namespace: kube-system + labels: + k8s-app: cilium + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-agent + spec: + selector: + matchLabels: + k8s-app: cilium + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + labels: + k8s-app: cilium + app.kubernetes.io/name: cilium-agent + app.kubernetes.io/part-of: cilium + spec: + securityContext: + appArmorProfile: + type: Unconfined + containers: + - name: cilium-agent + image: "quay.io/cilium/cilium:v1.16.5@sha256:758ca0793f5995bb938a2fa219dcce63dc0b3fa7fc4ce5cc851125281fb7361d" + imagePullPolicy: IfNotPresent + command: + - cilium-agent + args: + - --config-dir=/tmp/cilium/config-map + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + failureThreshold: 105 + periodSeconds: 2 + successThreshold: 1 + initialDelaySeconds: 5 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9879 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: GOMEMLIMIT + valueFrom: + resourceFieldRef: + resource: limits.memory + divisor: '1' + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "7445" + lifecycle: + postStart: + exec: + command: + - "bash" + - "-c" + - | + set -o errexit + set -o pipefail + set -o nounset + + # When running in AWS ENI mode, it's likely that 'aws-node' has + # had a chance to install SNAT iptables rules. These can result + # in dropped traffic, so we should attempt to remove them. + # We do it using a 'postStart' hook since this may need to run + # for nodes which might have already been init'ed but may still + # have dangling rules. This is safe because there are no + # dependencies on anything that is part of the startup script + # itself, and can be safely run multiple times per node (e.g. in + # case of a restart). + if [[ "$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; + then + echo 'Deleting iptables rules created by the AWS CNI VPC plugin' + iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore + fi + echo 'Done!' + + preStop: + exec: + command: + - /cni-uninstall.sh + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: envoy-sockets + mountPath: /var/run/cilium/envoy/sockets + readOnly: false + # Unprivileged containers need to mount /proc/sys/net from the host + # to have write access + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + # Unprivileged containers need to mount /proc/sys/kernel from the host + # to have write access + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - name: bpf-maps + mountPath: /sys/fs/bpf + # Unprivileged containers can't set mount propagation to bidirectional + # in this case we will mount the bpf fs from an init container that + # is privileged and set the mount propagation from host to container + # in Cilium. + mountPropagation: HostToContainer + # Check for duplicate mounts before mounting + - name: cilium-cgroup + mountPath: /sys/fs/cgroup + - name: cilium-run + mountPath: /var/run/cilium + - name: etc-cni-netd + mountPath: /host/etc/cni/net.d + - name: clustermesh-secrets + mountPath: /var/lib/cilium/clustermesh + readOnly: true + # Needed to be able to load kernel modules + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + - name: hubble-tls + mountPath: /var/lib/cilium/tls/hubble + readOnly: true + - name: tmp + mountPath: /tmp + initContainers: + - name: config + image: "quay.io/cilium/cilium:v1.16.5@sha256:758ca0793f5995bb938a2fa219dcce63dc0b3fa7fc4ce5cc851125281fb7361d" + imagePullPolicy: IfNotPresent + command: + - cilium-dbg + - build-config + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "7445" + volumeMounts: + - name: tmp + mountPath: /tmp + terminationMessagePolicy: FallbackToLogsOnError + - name: apply-sysctl-overwrites + image: "quay.io/cilium/cilium:v1.16.5@sha256:758ca0793f5995bb938a2fa219dcce63dc0b3fa7fc4ce5cc851125281fb7361d" + imagePullPolicy: IfNotPresent + env: + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + # Mount the bpf fs if it is not mounted. We will perform this task + # from a privileged container because the mount propagation bidirectional + # only works from privileged containers. + - name: mount-bpf-fs + image: "quay.io/cilium/cilium:v1.16.5@sha256:758ca0793f5995bb938a2fa219dcce63dc0b3fa7fc4ce5cc851125281fb7361d" + imagePullPolicy: IfNotPresent + args: + - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' + command: + - /bin/bash + - -c + - -- + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + privileged: true + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + - name: clean-cilium-state + image: "quay.io/cilium/cilium:v1.16.5@sha256:758ca0793f5995bb938a2fa219dcce63dc0b3fa7fc4ce5cc851125281fb7361d" + imagePullPolicy: IfNotPresent + command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-state + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-bpf-state + optional: true + - name: WRITE_CNI_CONF_WHEN_READY + valueFrom: + configMapKeyRef: + name: cilium-config + key: write-cni-conf-when-ready + optional: true + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "7445" + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - NET_ADMIN + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + # Required to mount cgroup filesystem from the host to cilium agent pod + - name: cilium-cgroup + mountPath: /sys/fs/cgroup + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium # wait-for-kube-proxy + # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent + - name: install-cni-binaries + image: "quay.io/cilium/cilium:v1.16.5@sha256:758ca0793f5995bb938a2fa219dcce63dc0b3fa7fc4ce5cc851125281fb7361d" + imagePullPolicy: IfNotPresent + command: + - "/install-plugin.sh" + resources: + requests: + cpu: 100m + memory: 10Mi + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cni-path + mountPath: /host/opt/cni/bin # .Values.cni.install + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccountName: "cilium" + automountServiceAccountToken: true + terminationGracePeriodSeconds: 1 + hostNetwork: true + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + # For sharing configuration between the "config" initContainer and the agent + - name: tmp + emptyDir: {} + # To keep state between restarts / upgrades + - name: cilium-run + hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + # To mount cgroup2 filesystem on the host or apply sysctlfix + - name: hostproc + hostPath: + path: /proc + type: Directory + # To keep state between restarts / upgrades for cgroup2 filesystem + - name: cilium-cgroup + hostPath: + path: /sys/fs/cgroup + type: DirectoryOrCreate + # To install cilium cni plugin in the host + - name: cni-path + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + # To install cilium cni configuration in the host + - name: etc-cni-netd + hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + # To be able to load kernel modules + - name: lib-modules + hostPath: + path: /lib/modules + # To access iptables concurrently with other processes (e.g. kube-proxy) + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Sharing socket with Cilium Envoy on the same node by using a host path + - name: envoy-sockets + hostPath: + path: "/var/run/cilium/envoy/sockets" + type: DirectoryOrCreate + # To read the clustermesh configuration + - name: clustermesh-secrets + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: cilium-clustermesh + optional: true + # note: items are not explicitly listed here, since the entries of this secret + # depend on the peers configured, and that would cause a restart of all agents + # at every addition/removal. Leaving the field empty makes each secret entry + # to be automatically projected into the volume as a file whose name is the key. + - secret: + name: clustermesh-apiserver-remote-cert + optional: true + items: + - key: tls.key + path: common-etcd-client.key + - key: tls.crt + path: common-etcd-client.crt + - key: ca.crt + path: common-etcd-client-ca.crt + # note: we configure the volume for the kvstoremesh-specific certificate + # regardless of whether KVStoreMesh is enabled or not, so that it can be + # automatically mounted in case KVStoreMesh gets subsequently enabled, + # without requiring an agent restart. + - secret: + name: clustermesh-apiserver-local-cert + optional: true + items: + - key: tls.key + path: local-etcd-client.key + - key: tls.crt + path: local-etcd-client.crt + - key: ca.crt + path: local-etcd-client-ca.crt + - name: host-proc-sys-net + hostPath: + path: /proc/sys/net + type: Directory + - name: host-proc-sys-kernel + hostPath: + path: /proc/sys/kernel + type: Directory + - name: hubble-tls + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: hubble-server-certs + optional: true + items: + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key + - key: ca.crt + path: client-ca.crt + --- + # Source: cilium/templates/cilium-envoy/daemonset.yaml + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: cilium-envoy + namespace: kube-system + labels: + k8s-app: cilium-envoy + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-envoy + name: cilium-envoy + spec: + selector: + matchLabels: + k8s-app: cilium-envoy + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + labels: + k8s-app: cilium-envoy + name: cilium-envoy + app.kubernetes.io/name: cilium-envoy + app.kubernetes.io/part-of: cilium + spec: + securityContext: + appArmorProfile: + type: Unconfined + containers: + - name: cilium-envoy + image: "quay.io/cilium/cilium-envoy:v1.30.8-1733837904-eaae5aca0fb988583e5617170a65ac5aa51c0aa8@sha256:709c08ade3d17d52da4ca2af33f431360ec26268d288d9a6cd1d98acc9a1dced" + imagePullPolicy: IfNotPresent + command: + - /usr/bin/cilium-envoy-starter + args: + - '--' + - '-c /var/run/cilium/envoy/bootstrap-config.json' + - '--base-id 0' + - '--log-level info' + - '--log-format [%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v' + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + failureThreshold: 105 + periodSeconds: 2 + successThreshold: 1 + initialDelaySeconds: 5 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "7445" + ports: + - name: envoy-metrics + containerPort: 9964 + hostPort: 9964 + protocol: TCP + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - NET_ADMIN + - SYS_ADMIN + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: envoy-sockets + mountPath: /var/run/cilium/envoy/sockets + readOnly: false + - name: envoy-artifacts + mountPath: /var/run/cilium/envoy/artifacts + readOnly: true + - name: envoy-config + mountPath: /var/run/cilium/envoy/ + readOnly: true + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: HostToContainer + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccountName: "cilium-envoy" + automountServiceAccountToken: true + terminationGracePeriodSeconds: 1 + hostNetwork: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cilium.io/no-schedule + operator: NotIn + values: + - "true" + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium-envoy + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + - name: envoy-sockets + hostPath: + path: "/var/run/cilium/envoy/sockets" + type: DirectoryOrCreate + - name: envoy-artifacts + hostPath: + path: "/var/run/cilium/envoy/artifacts" + type: DirectoryOrCreate + - name: envoy-config + configMap: + name: cilium-envoy-config + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + items: + - key: bootstrap-config.json + path: bootstrap-config.json + # To keep state between restarts / upgrades + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + --- + # Source: cilium/templates/cilium-operator/deployment.yaml + apiVersion: apps/v1 + kind: Deployment + metadata: + name: cilium-operator + namespace: kube-system + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator + spec: + # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go + # for more details. + replicas: 2 + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + # ensure operator update on single node k8s clusters, by using rolling update with maxUnavailable=100% in case + # of one replica and no user configured Recreate strategy. + # otherwise an update might get stuck due to the default maxUnavailable=50% in combination with the + # podAntiAffinity which prevents deployments of multiple operator replicas on the same node. + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 50% + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9963" + prometheus.io/scrape: "true" + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator + spec: + containers: + - name: cilium-operator + image: "quay.io/cilium/operator-generic:v1.16.5@sha256:f7884848483bbcd7b1e0ccfd34ba4546f258b460cb4b7e2f06a1bcc96ef88039" + imagePullPolicy: IfNotPresent + command: + - cilium-operator-generic + args: + - --config-dir=/tmp/cilium/config-map + - --debug=$(CILIUM_DEBUG) + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "7445" + ports: + - name: prometheus + containerPort: 9963 + hostPort: 9963 + protocol: TCP + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 5 + volumeMounts: + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + terminationMessagePolicy: FallbackToLogsOnError + hostNetwork: true + restartPolicy: Always + priorityClassName: system-cluster-critical + serviceAccountName: "cilium-operator" + automountServiceAccountToken: true + # In HA mode, cilium-operator pods must not be scheduled on the same + # node as they will clash with each other. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + io.cilium/app: operator + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config diff --git a/patches/machines/ctl-1.yaml b/patches/machines/ctl-1.yaml index cdf71be..d40cbe2 100644 --- a/patches/machines/ctl-1.yaml +++ b/patches/machines/ctl-1.yaml @@ -7,6 +7,8 @@ machine: addresses: - 10.51.10.101/23 - 2a13:fc80:1:a:ffff::1/64 + vip: + ip: 2a13:fc80:1:a:ffff::0 mtu: 9000 routes: - network: 0.0.0.0/0 diff --git a/patches/machines/ctl-2.yaml b/patches/machines/ctl-2.yaml index fd58df1..13f9b64 100644 --- a/patches/machines/ctl-2.yaml +++ b/patches/machines/ctl-2.yaml @@ -7,6 +7,8 @@ machine: addresses: - 10.51.10.102/23 - 2a13:fc80:1:a:ffff::2/64 + vip: + ip: 2a13:fc80:1:a:ffff::0 mtu: 9000 routes: - network: 0.0.0.0/0 diff --git a/patches/machines/ctl-3.yaml b/patches/machines/ctl-3.yaml index ea84885..83e45a2 100644 --- a/patches/machines/ctl-3.yaml +++ b/patches/machines/ctl-3.yaml @@ -7,6 +7,8 @@ machine: addresses: - 10.51.10.103/23 - 2a13:fc80:1:a:ffff::3/64 + vip: + ip: 2a13:fc80:1:a:ffff::0 mtu: 9000 routes: - network: 0.0.0.0/0 diff --git a/worker.yaml b/worker.yaml index ae224df..c0003a0 100644 --- a/worker.yaml +++ b/worker.yaml @@ -189,7 +189,7 @@ machine: # Used to provide instructions for installations. install: disk: /dev/sda # The disk used for installations. - image: factory.talos.dev/installer/ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515:v1.9.2 + image: ghcr.io/siderolabs/installer:v1.9.1 wipe: false # Indicates if the installation disk should be wiped at installation time. # # Look up disk using disk attributes like model, size, serial and others. diff --git a/xx.sh b/xx.sh new file mode 100644 index 0000000..8df723a --- /dev/null +++ b/xx.sh @@ -0,0 +1,22 @@ +#!/usr/bin/bash + +helm template \ + cilium \ + cilium/cilium \ + --version 1.16.5 \ + --namespace kube-system \ + --set bgpControlPlane.enabled=true \ + --set hubble.preferIpv6=true \ + --set ipv6.enabled=true \ + --set ipam.mode=kubernetes \ + --set kubeProxyReplacement=true \ + --set securityContext.capabilities.ciliumAgent="{CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}" \ + --set securityContext.capabilities.cleanCiliumState="{NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}" \ + --set cgroup.autoMount.enabled=false \ + --set cgroup.hostRoot=/sys/fs/cgroup \ + --set k8sServiceHost=localhost \ + --set k8sServicePort=7445 \ + --set gatewayAPI.enabled=true \ + --set gatewayAPI.enableAlpn=true \ + --set gatewayAPI.enableAppProtocol=true \ + > cilium.yaml