fix: Allow the ISVC to be healthy when the Stopped Condition is False (cherry-pick #25312 for 3.2) (#25318)

Signed-off-by: Hannah DeFazio <h2defazio@gmail.com>
Co-authored-by: Hannah DeFazio <h2defazio@gmail.com>

resource_customizations/serving.kserve.io/InferenceService/health.lua

@@ -24,14 +24,23 @@ if obj.status ~= nil then
   
   if obj.status.conditions ~= nil then
     for i, condition in pairs(obj.status.conditions) do
+      -- Check if the InferenceService is Stopped
+      if condition.type == "Stopped" and condition.status == "True" then 
+        health_status.status = "Suspended"
+        health_status.message = "InferenceService is Stopped"
+        return health_status
+      end
 
+      -- Check for unhealthy statuses
+      -- Note: The Stopped condition's healthy status is False
       if condition.status == "Unknown" then
         status_unknown = status_unknown + 1
-      elseif condition.status == "False" then
+      elseif condition.status == "False" and condition.type ~= "Stopped" then
         status_false = status_false + 1
       end
 
-      if condition.status ~= "True" then
+      -- Add the error messages if the status is unhealthy
+      if condition.status ~= "True" and condition.type ~= "Stopped" then
         msg = msg .. " | " .. i .. ": " .. condition.type .. " | " .. condition.status
         if condition.reason ~= nil and condition.reason ~= "" then
           msg = msg .. " | " .. condition.reason

resource_customizations/serving.kserve.io/InferenceService/health_test.yaml

@@ -23,6 +23,10 @@ tests:
     status: Degraded
     message: "0: transitionStatus | BlockedByFailedLoad"
   inputPath: testdata/degraded_modelmesh.yaml
+- healthStatus:
+    status: Suspended
+    message: InferenceService is Stopped
+  inputPath: testdata/stopped.yaml
 - healthStatus:
     status: Healthy
     message: InferenceService is healthy.

resource_customizations/serving.kserve.io/InferenceService/testdata/healthy.yaml

@@ -23,3 +23,7 @@ status:
   - lastTransitionTime: "2023-06-20T22:44:51Z"
     status: "True"
     type: Ready
+  - lastTransitionTime: "2023-06-20T22:44:51Z"
+    severity: Info
+    status: 'False'
+    type: Stopped

resource_customizations/serving.kserve.io/InferenceService/testdata/healthy_ocp.yaml

@@ -31,5 +31,9 @@ status:
       severity: Info
       status: 'True'
       type: RoutesReady
+    - lastTransitionTime: '2024-05-30T22:14:31Z'
+      severity: Info
+      status: 'False'
+      type: Stopped
   modelStatus:
     transitionStatus: UpToDate

resource_customizations/serving.kserve.io/InferenceService/testdata/healthy_raw.yaml

@@ -17,3 +17,7 @@ status:
   - lastTransitionTime: '2024-05-16T18:48:56Z'
     status: 'True'
     type: Ready
+  - lastTransitionTime: '2024-05-16T18:48:56Z'
+    severity: Info
+    status: 'False'
+    type: Stopped

resource_customizations/serving.kserve.io/InferenceService/testdata/stopped.yaml

@@ -0,0 +1,23 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  name: helloworld
+  namespace: default
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+    serving.kserve.io/stop: 'true'
+spec: {}
+status:
+  conditions:
+  - lastTransitionTime: '2024-05-16T18:48:56Z'
+    reason: Stopped
+    status: 'False'
+    type: PredictorReady
+  - lastTransitionTime: '2024-05-16T18:48:56Z'
+    reason: Stopped
+    status: 'False'
+    type: Ready
+  - lastTransitionTime: '2024-05-16T18:48:56Z'
+    severity: Info
+    status: 'True'
+    type: Stopped