chore: bump gitops-engine with force+replace test (#24508)

Signed-off-by: Alexandre Gaudreault <alexandre_gaudreault@intuit.com>

test/e2e/fixture/fixture.go

@@ -14,6 +14,7 @@ import (
 	"testing"
 	"time"
 
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 
@@ -21,10 +22,12 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/yaml"
 
 	"github.com/argoproj/argo-cd/v3/common"
@@ -67,6 +70,10 @@ const (
 
 	E2ETestPrefix = "e2e-test-"
 
+	// finalizer to add to resources during tests. Make sure that the resource Kind of your object
+	// is included in the test EnsureCleanState code
+	TestFinalizer = TestingLabel + "/finalizer"
+
 	// Account for batch events processing (set to 1ms in e2e tests)
 	WhenThenSleepInterval = 5 * time.Millisecond
 )
@@ -661,6 +668,45 @@ func EnsureCleanState(t *testing.T, opts ...TestOption) {
 	start := time.Now()
 	policy := metav1.DeletePropagationBackground
 
+	deleteNamespaces := func(namespaces []corev1.Namespace, wait bool) error {
+		args := []string{"delete", "ns", "--ignore-not-found=true", fmt.Sprintf("--wait=%t", wait)}
+		for _, namespace := range namespaces {
+			args = append(args, namespace.Name)
+		}
+		_, err := Run("", "kubectl", args...)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+
+	deleteResourceWithTestFinalizer := func(namespaces []corev1.Namespace, gvrs []schema.GroupVersionResource) error {
+		for _, namespace := range namespaces {
+			for _, gvr := range gvrs {
+				objects, err := DynamicClientset.Resource(gvr).Namespace(namespace.GetName()).List(t.Context(), metav1.ListOptions{})
+				if err != nil {
+					return err
+				}
+				for i := range objects.Items {
+					obj := &objects.Items[i]
+					updated := controllerutil.RemoveFinalizer(obj, TestFinalizer)
+					if updated {
+						log.WithFields(log.Fields{
+							"namespace": namespace.GetName(),
+							"resource":  gvr,
+							"name":      obj.GetName(),
+						}).Info("removing test finalizer")
+						_, err := DynamicClientset.Resource(gvr).Namespace(namespace.GetName()).Update(t.Context(), obj, metav1.UpdateOptions{})
+						if err != nil {
+							return err
+						}
+					}
+				}
+			}
+		}
+		return nil
+	}
+
 	RunFunctionsInParallelAndCheckErrors(t, []func() error{
 		func() error {
 			// kubectl delete apps ...
@@ -720,18 +766,33 @@ func EnsureCleanState(t *testing.T, opts ...TestOption) {
 				t.Context(),
 				metav1.ListOptions{
 					LabelSelector: TestingLabel + "=true",
-					FieldSelector: "status.phase=Active",
 				},
 			)
 			if err != nil {
 				return err
 			}
 			if len(namespaces.Items) > 0 {
-				args := []string{"delete", "ns", "--wait=false"}
-				for _, namespace := range namespaces.Items {
-					args = append(args, namespace.Name)
+				err = deleteNamespaces(namespaces.Items, false)
+				if err != nil {
+					return err
 				}
-				_, err := Run("", "kubectl", args...)
+			}
+
+			// Get all namespaces stuck in Terminating state
+			terminatingNamespaces, err := KubeClientset.CoreV1().Namespaces().List(
+				t.Context(),
+				metav1.ListOptions{
+					LabelSelector: TestingLabel + "=true",
+					FieldSelector: "status.phase=Terminating",
+				})
+			if err != nil {
+				return err
+			}
+			if len(terminatingNamespaces.Items) > 0 {
+				err = deleteResourceWithTestFinalizer(terminatingNamespaces.Items, []schema.GroupVersionResource{
+					// If finalizers are added to new resource kinds, they must be added here for a proper cleanup
+					appsv1.SchemeGroupVersion.WithResource("deployments"),
+				})
 				if err != nil {
 					return err
 				}
@@ -741,16 +802,14 @@ func EnsureCleanState(t *testing.T, opts ...TestOption) {
 			if err != nil {
 				return err
 			}
-			testNamespaceNames := []string{}
+			testNamespaceNames := []corev1.Namespace{}
 			for _, namespace := range namespaces.Items {
 				if strings.HasPrefix(namespace.Name, E2ETestPrefix) {
-					testNamespaceNames = append(testNamespaceNames, namespace.Name)
+					testNamespaceNames = append(testNamespaceNames, namespace)
 				}
 			}
 			if len(testNamespaceNames) > 0 {
-				args := []string{"delete", "ns"}
-				args = append(args, testNamespaceNames...)
-				_, err := Run("", "kubectl", args...)
+				err = deleteNamespaces(testNamespaceNames, true)
 				if err != nil {
 					return err
 				}

test/e2e/sync_options_test.go

@@ -199,6 +199,12 @@ func TestSyncWithSkipHook(t *testing.T) {
 }
 
 func TestSyncWithForceReplace(t *testing.T) {
+	t.Cleanup(func() {
+		// remove finalizer to ensure easy cleanup
+		_, err := Run("", "kubectl", "patch", "deployment", "guestbook-ui", "-n", DeploymentNamespace(), "-p", `{"metadata":{"finalizers":[]}}`, "--type=merge")
+		require.NoError(t, err)
+	})
+
 	Given(t).
 		Path(guestbookPath).
 		When().
@@ -207,14 +213,18 @@ func TestSyncWithForceReplace(t *testing.T) {
 		Then().
 		Expect(SyncStatusIs(SyncStatusCodeSynced)).
 		// app having `Replace=true` and `Force=true` annotation should sync succeed if change in immutable field
+		// The finalizers allow us to validate that the existing resource (no finalizers) is just deleted once
+		// and does not get stuck in terminating state
 		When().
+		PatchFile("guestbook-ui-deployment.yaml", fmt.Sprintf(`[{ "op": "add", "path": "/metadata/finalizers", "value": [%q]}]`, TestFinalizer)).
 		PatchFile("guestbook-ui-deployment.yaml", `[{ "op": "add", "path": "/metadata/annotations", "value": { "argocd.argoproj.io/sync-options": "Force=true,Replace=true" }}]`).
 		PatchFile("guestbook-ui-deployment.yaml", `[{ "op": "add", "path": "/spec/selector/matchLabels/env", "value": "e2e" }, { "op": "add", "path": "/spec/template/metadata/labels/env", "value": "e2e" }]`).
-		PatchFile("guestbook-ui-deployment.yaml", `[{ "op": "replace", "path": "/spec/replicas", "value": 1 }]`).
+		PatchFile("guestbook-ui-deployment.yaml", `[{ "op": "replace", "path": "/spec/replicas", "value": 2 }]`).
 		Refresh(RefreshTypeNormal).
 		Sync().
 		Then().
-		Expect(SyncStatusIs(SyncStatusCodeSynced))
+		Expect(SyncStatusIs(SyncStatusCodeSynced)).
+		Expect(HealthIs(health.HealthStatusHealthy))
 }
 
 // Given application is set with --sync-option CreateNamespace=true and --sync-option ServerSideApply=true