From 99efafb55a553a9ab962d56c20dab54ba65b7ae0 Mon Sep 17 00:00:00 2001 From: Clint Chester Date: Mon, 16 Dec 2024 04:58:30 +0800 Subject: [PATCH] feat: Confluent Connector Resource Health Checker - #17695 (#17697) * Adding Synergy as a ArgoCD user Signed-off-by: GitHub * Health checking Kafka Connector resources Signed-off-by: Clint Chester * Includes Kafka Connect Task Failures Signed-off-by: Clint Chester --------- Signed-off-by: GitHub Signed-off-by: Clint Chester --- USERS.md | 1 + .../Connector/health.lua | 25 +++++++++++++ .../Connector/health_test.yaml | 13 +++++++ .../Connector/testdata/connector-failure.yaml | 27 ++++++++++++++ .../Connector/testdata/connector-healthy.yaml | 31 ++++++++++++++++ .../testdata/connector-task-failure.yaml | 37 +++++++++++++++++++ 6 files changed, 134 insertions(+) create mode 100644 resource_customizations/platform.confluent.io/Connector/health.lua create mode 100644 resource_customizations/platform.confluent.io/Connector/health_test.yaml create mode 100644 resource_customizations/platform.confluent.io/Connector/testdata/connector-failure.yaml create mode 100644 resource_customizations/platform.confluent.io/Connector/testdata/connector-healthy.yaml create mode 100644 resource_customizations/platform.confluent.io/Connector/testdata/connector-task-failure.yaml diff --git a/USERS.md b/USERS.md index 8b3130548f..ef200ecb3a 100644 --- a/USERS.md +++ b/USERS.md @@ -335,6 +335,7 @@ Currently, the following organizations are **officially** using Argo CD: 1. [Swisscom](https://www.swisscom.ch) 1. [Swissquote](https://github.com/swissquote) 1. [Syncier](https://syncier.com/) +1. [Synergy](https://synergy.net.au) 1. [Syself](https://syself.com) 1. [TableCheck](https://tablecheck.com/) 1. [Tailor Brands](https://www.tailorbrands.com) diff --git a/resource_customizations/platform.confluent.io/Connector/health.lua b/resource_customizations/platform.confluent.io/Connector/health.lua new file mode 100644 index 0000000000..a800a33179 --- /dev/null +++ b/resource_customizations/platform.confluent.io/Connector/health.lua @@ -0,0 +1,25 @@ +hs = {} +if obj.status ~= nil and obj.status.state ~= nil then + if obj.status.state == "CREATED" and obj.status.connectorState == "RUNNING" and obj.status.failedTasksCount == nil then + hs.status = "Healthy" + hs.message = "Connector running" + return hs + end + if obj.status.state == "ERROR" then + hs.status = "Degraded" + if obj.status.conditions and #obj.status.conditions > 0 then + hs.message = obj.status.conditions[1].message -- Kafka Connector only has one condition and nests the issues in the error message here + else + hs.message = "No conditions available" + end + return hs + end + if obj.status.failedTasksCount ~= nil and obj.status.failedTasksCount > 0 then + hs.status = "Degraded" + hs.message = "Connector has failed tasks" + return hs + end +end +hs.status = "Progressing" +hs.message = "Waiting for Kafka Connector" +return hs \ No newline at end of file diff --git a/resource_customizations/platform.confluent.io/Connector/health_test.yaml b/resource_customizations/platform.confluent.io/Connector/health_test.yaml new file mode 100644 index 0000000000..c449ba9819 --- /dev/null +++ b/resource_customizations/platform.confluent.io/Connector/health_test.yaml @@ -0,0 +1,13 @@ +tests: + - healthStatus: + status: Healthy + message: 'Connector running' + inputPath: testdata/connector-healthy.yaml + - healthStatus: + status: Degraded + message: 'connect Rest API request failed: Connector configuration is invalid and contains the following 1 error(s): Could not read properties from file' + inputPath: testdata/connector-failure.yaml + - healthStatus: + status: Degraded + message: 'Connector has failed tasks' + inputPath: testdata/connector-task-failure.yaml \ No newline at end of file diff --git a/resource_customizations/platform.confluent.io/Connector/testdata/connector-failure.yaml b/resource_customizations/platform.confluent.io/Connector/testdata/connector-failure.yaml new file mode 100644 index 0000000000..b34a297bba --- /dev/null +++ b/resource_customizations/platform.confluent.io/Connector/testdata/connector-failure.yaml @@ -0,0 +1,27 @@ +apiVersion: platform.confluent.io/v1beta1 +kind: Connector +metadata: + finalizers: + - connect.finalizers.platform.confluent.io + generation: 1 + name: connect + namespace: confluent +spec: + class: io.confluent.connect.sftp.SftpSinkConnector + configs: + topics: test-topic + connectClusterRef: + name: connect + name: test-sftp-connector + taskMax: 3 +status: + appState: Failed + conditions: + - lastProbeTime: '2024-04-02T07:43:35Z' + lastTransitionTime: '2024-04-02T07:43:35Z' + message: >- + connect Rest API request failed: Connector configuration is invalid and contains the following 1 error(s): Could not read properties from file + reason: CreateFailed + status: 'False' + type: platform.confluent.io/app-ready + state: ERROR \ No newline at end of file diff --git a/resource_customizations/platform.confluent.io/Connector/testdata/connector-healthy.yaml b/resource_customizations/platform.confluent.io/Connector/testdata/connector-healthy.yaml new file mode 100644 index 0000000000..10df701b7c --- /dev/null +++ b/resource_customizations/platform.confluent.io/Connector/testdata/connector-healthy.yaml @@ -0,0 +1,31 @@ +apiVersion: platform.confluent.io/v1beta1 +kind: Connector +metadata: + finalizers: + - connect.finalizers.platform.confluent.io + generation: 1 + name: connect + namespace: confluent +spec: + class: io.confluent.connect.sftp.SftpSinkConnector + configs: + topics: test-topic + connectClusterRef: + name: connect + name: test-sftp-connector + taskMax: 3 +status: + appState: Created + conditions: + - lastProbeTime: '2024-04-02T07:43:35Z' + lastTransitionTime: '2024-04-02T07:43:35Z' + message: Application is created + reason: Created + status: 'True' + type: platform.confluent.io/app-ready + connectorState: RUNNING + restartPolicy: + maxRetry: 10 + type: OnFailure + state: CREATED + tasksReady: 3/3 \ No newline at end of file diff --git a/resource_customizations/platform.confluent.io/Connector/testdata/connector-task-failure.yaml b/resource_customizations/platform.confluent.io/Connector/testdata/connector-task-failure.yaml new file mode 100644 index 0000000000..09134a48f4 --- /dev/null +++ b/resource_customizations/platform.confluent.io/Connector/testdata/connector-task-failure.yaml @@ -0,0 +1,37 @@ +apiVersion: platform.confluent.io/v1beta1 +kind: Connector +metadata: + finalizers: + - connect.finalizers.platform.confluent.io + generation: 1 + name: connect + namespace: confluent +spec: + class: io.confluent.connect.sftp.SftpSinkConnector + configs: + topics: test-topic + connectClusterRef: + name: connect + name: test-sftp-connector + taskMax: 3 +status: + appState: Unknown + conditions: + - lastProbeTime: '2024-01-19T06:42:40Z' + lastTransitionTime: '2024-01-19T06:42:40Z' + message: Application is created + reason: Created + status: 'True' + type: platform.confluent.io/app-ready + connectorState: RUNNING + failedTasks: + task-0: + id: 0 + retryCount: 10 + failedTasksCount: 1 + observedGeneration: 1 + restartPolicy: + maxRetry: 10 + type: OnFailure + state: CREATED + tasksReady: 0/1 \ No newline at end of file