fix: optimize cli server-side diff with parallel dynamic batching (#25550)

Signed-off-by: Arnaud Farbos <afarbos@nvidia.com>

cmd/argocd/commands/app.go

@@ -17,6 +17,8 @@ import (
 	"time"
 	"unicode/utf8"
 
+	"golang.org/x/sync/errgroup"
+
 	"github.com/argoproj/gitops-engine/pkg/health"
 	"github.com/argoproj/gitops-engine/pkg/sync/common"
 	"github.com/argoproj/gitops-engine/pkg/sync/hook"
@@ -1275,24 +1277,32 @@ type objKeyLiveTarget struct {
 	target *unstructured.Unstructured
 }
 
+// addServerSideDiffPerfFlags adds server-side diff performance tuning flags to a command
+func addServerSideDiffPerfFlags(command *cobra.Command, serverSideDiffConcurrency *int, serverSideDiffMaxBatchKB *int) {
+	command.Flags().IntVar(serverSideDiffConcurrency, "server-side-diff-concurrency", -1, "Max concurrent batches for server-side diff. -1 = unlimited, 1 = sequential, 2+ = concurrent (0 = invalid)")
+	command.Flags().IntVar(serverSideDiffMaxBatchKB, "server-side-diff-max-batch-kb", 250, "Max batch size in KB for server-side diff. Smaller values are safer for proxies")
+}
+
 // NewApplicationDiffCommand returns a new instance of an `argocd app diff` command
 func NewApplicationDiffCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command {
 	var (
-		refresh              bool
-		hardRefresh          bool
-		exitCode             bool
-		diffExitCode         int
-		local                string
-		revision             string
-		localRepoRoot        string
-		serverSideGenerate   bool
-		serverSideDiff       bool
-		localIncludes        []string
-		appNamespace         string
-		revisions            []string
-		sourcePositions      []int64
-		sourceNames          []string
-		ignoreNormalizerOpts normalizers.IgnoreNormalizerOpts
+		refresh                   bool
+		hardRefresh               bool
+		exitCode                  bool
+		diffExitCode              int
+		local                     string
+		revision                  string
+		localRepoRoot             string
+		serverSideGenerate        bool
+		serverSideDiff            bool
+		serverSideDiffConcurrency int
+		serverSideDiffMaxBatchKB  int
+		localIncludes             []string
+		appNamespace              string
+		revisions                 []string
+		sourcePositions           []int64
+		sourceNames               []string
+		ignoreNormalizerOpts      normalizers.IgnoreNormalizerOpts
 	)
 	shortDesc := "Perform a diff against the target and live state."
 	command := &cobra.Command{
@@ -1423,7 +1433,7 @@ func NewApplicationDiffCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co
 			}
 			proj := getProject(ctx, c, clientOpts, app.Spec.Project)
 
-			foundDiffs := findAndPrintDiff(ctx, app, proj.Project, resources, argoSettings, diffOption, ignoreNormalizerOpts, serverSideDiff, appIf, app.GetName(), app.GetNamespace())
+			foundDiffs := findAndPrintDiff(ctx, app, proj.Project, resources, argoSettings, diffOption, ignoreNormalizerOpts, serverSideDiff, appIf, app.GetName(), app.GetNamespace(), serverSideDiffConcurrency, serverSideDiffMaxBatchKB)
 			if foundDiffs && exitCode {
 				os.Exit(diffExitCode)
 			}
@@ -1438,6 +1448,7 @@ func NewApplicationDiffCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co
 	command.Flags().StringVar(&localRepoRoot, "local-repo-root", "/", "Path to the repository root. Used together with --local allows setting the repository root")
 	command.Flags().BoolVar(&serverSideGenerate, "server-side-generate", false, "Used with --local, this will send your manifests to the server for diffing")
 	command.Flags().BoolVar(&serverSideDiff, "server-side-diff", false, "Use server-side diff to calculate the diff. This will default to true if the ServerSideDiff annotation is set on the application.")
+	addServerSideDiffPerfFlags(command, &serverSideDiffConcurrency, &serverSideDiffMaxBatchKB)
 	command.Flags().StringArrayVar(&localIncludes, "local-include", []string{"*.yaml", "*.yml", "*.json"}, "Used with --server-side-generate, specify patterns of filenames to send. Matching is based on filename and not path.")
 	command.Flags().StringVarP(&appNamespace, "app-namespace", "N", "", "Only render the difference in namespace")
 	command.Flags().StringArrayVar(&revisions, "revisions", []string{}, "Show manifests at specific revisions for source position in source-positions")
@@ -1454,7 +1465,14 @@ func printResourceDiff(group, kind, namespace, name string, live, target *unstru
 }
 
 // findAndPrintServerSideDiff performs a server-side diff by making requests to the api server and prints the response
-func findAndPrintServerSideDiff(ctx context.Context, app *argoappv1.Application, items []objKeyLiveTarget, resources *application.ManagedResourcesResponse, appIf application.ApplicationServiceClient, appName, appNs string) bool {
+func findAndPrintServerSideDiff(ctx context.Context, app *argoappv1.Application, items []objKeyLiveTarget, resources *application.ManagedResourcesResponse, appIf application.ApplicationServiceClient, appName, appNs string, maxConcurrency int, maxBatchSizeKB int) bool {
+	if maxConcurrency == 0 {
+		errors.CheckError(stderrors.New("invalid value for --server-side-diff-concurrency: 0 is not allowed (use -1 for unlimited, or a positive number to limit concurrency)"))
+	}
+
+	liveResources := make([]*argoappv1.ResourceDiff, 0, len(items))
+	targetManifests := make([]string, 0, len(items))
+
 	// Process each item for server-side diff
 	foundDiffs := false
 	for _, item := range items {
@@ -1488,6 +1506,7 @@ func findAndPrintServerSideDiff(ctx context.Context, app *argoappv1.Application,
 				Modified:    true,
 			}
 		}
+		liveResources = append(liveResources, liveResource)
 
 		if item.target != nil {
 			jsonBytes, err := json.Marshal(item.target)
@@ -1496,23 +1515,63 @@ func findAndPrintServerSideDiff(ctx context.Context, app *argoappv1.Application,
 			}
 			targetManifest = string(jsonBytes)
 		}
+		targetManifests = append(targetManifests, targetManifest)
+	}
 
-		// Call server-side diff for this individual resource
-		serverSideDiffQuery := &application.ApplicationServerSideDiffQuery{
-			AppName:         &appName,
-			AppNamespace:    &appNs,
-			Project:         &app.Spec.Project,
-			LiveResources:   []*argoappv1.ResourceDiff{liveResource},
-			TargetManifests: []string{targetManifest},
+	if len(liveResources) == 0 {
+		return false
+	}
+
+	// Batch by size to avoid proxy limits
+	maxBatchSize := maxBatchSizeKB * 1024
+	var batches []struct{ start, end int }
+	for i := 0; i < len(liveResources); {
+		start := i
+		size := 0
+		for i < len(liveResources) {
+			resourceSize := len(liveResources[i].LiveState) + len(targetManifests[i])
+			if size+resourceSize > maxBatchSize && i > start {
+				break
+			}
+			size += resourceSize
+			i++
 		}
+		batches = append(batches, struct{ start, end int }{start, i})
+	}
 
-		serverSideDiffRes, err := appIf.ServerSideDiff(ctx, serverSideDiffQuery)
-		if err != nil {
-			errors.CheckError(err)
-		}
+	// Process batches in parallel
+	g, errGroupCtx := errgroup.WithContext(ctx)
+	g.SetLimit(maxConcurrency)
 
-		// Extract diff for this resource
-		for _, resultItem := range serverSideDiffRes.Items {
+	results := make([][]*argoappv1.ResourceDiff, len(batches))
+
+	for idx, batch := range batches {
+		i := idx
+		b := batch
+		g.Go(func() error {
+			// Call server-side diff for this batch of resources
+			serverSideDiffQuery := &application.ApplicationServerSideDiffQuery{
+				AppName:         &appName,
+				AppNamespace:    &appNs,
+				Project:         &app.Spec.Project,
+				LiveResources:   liveResources[b.start:b.end],
+				TargetManifests: targetManifests[b.start:b.end],
+			}
+			serverSideDiffRes, err := appIf.ServerSideDiff(errGroupCtx, serverSideDiffQuery)
+			if err != nil {
+				return err
+			}
+			results[i] = serverSideDiffRes.Items
+			return nil
+		})
+	}
+
+	if err := g.Wait(); err != nil {
+		errors.CheckError(err)
+	}
+
+	for _, items := range results {
+		for _, resultItem := range items {
 			if resultItem.Hook || (!resultItem.Modified && resultItem.TargetState != "" && resultItem.LiveState != "") {
 				continue
 			}
@@ -1522,13 +1581,12 @@ func findAndPrintServerSideDiff(ctx context.Context, app *argoappv1.Application,
 
 				if resultItem.TargetState != "" && resultItem.TargetState != "null" {
 					target = &unstructured.Unstructured{}
-					err = json.Unmarshal([]byte(resultItem.TargetState), target)
+					err := json.Unmarshal([]byte(resultItem.TargetState), target)
 					errors.CheckError(err)
 				}
-
 				if resultItem.LiveState != "" && resultItem.LiveState != "null" {
 					live = &unstructured.Unstructured{}
-					err = json.Unmarshal([]byte(resultItem.LiveState), live)
+					err := json.Unmarshal([]byte(resultItem.LiveState), live)
 					errors.CheckError(err)
 				}
 
@@ -1554,14 +1612,14 @@ type DifferenceOption struct {
 }
 
 // findAndPrintDiff ... Prints difference between application current state and state stored in git or locally, returns boolean as true if difference is found else returns false
-func findAndPrintDiff(ctx context.Context, app *argoappv1.Application, proj *argoappv1.AppProject, resources *application.ManagedResourcesResponse, argoSettings *settings.Settings, diffOptions *DifferenceOption, ignoreNormalizerOpts normalizers.IgnoreNormalizerOpts, useServerSideDiff bool, appIf application.ApplicationServiceClient, appName, appNs string) bool {
+func findAndPrintDiff(ctx context.Context, app *argoappv1.Application, proj *argoappv1.AppProject, resources *application.ManagedResourcesResponse, argoSettings *settings.Settings, diffOptions *DifferenceOption, ignoreNormalizerOpts normalizers.IgnoreNormalizerOpts, useServerSideDiff bool, appIf application.ApplicationServiceClient, appName, appNs string, serverSideDiffConcurrency int, serverSideDiffMaxBatchKB int) bool {
 	var foundDiffs bool
 
 	items, err := prepareObjectsForDiff(ctx, app, proj, resources, argoSettings, diffOptions)
 	errors.CheckError(err)
 
 	if useServerSideDiff {
-		return findAndPrintServerSideDiff(ctx, app, items, resources, appIf, appName, appNs)
+		return findAndPrintServerSideDiff(ctx, app, items, resources, appIf, appName, appNs, serverSideDiffConcurrency, serverSideDiffMaxBatchKB)
 	}
 
 	for _, item := range items {
@@ -2075,36 +2133,38 @@ func printTreeViewDetailed(nodeMapping map[string]argoappv1.ResourceNode, parent
 // NewApplicationSyncCommand returns a new instance of an `argocd app sync` command
 func NewApplicationSyncCommand(clientOpts *argocdclient.ClientOptions) *cobra.Command {
 	var (
-		revision                string
-		revisions               []string
-		sourcePositions         []int64
-		sourceNames             []string
-		resources               []string
-		labels                  []string
-		selector                string
-		prune                   bool
-		dryRun                  bool
-		timeout                 uint
-		strategy                string
-		force                   bool
-		replace                 bool
-		serverSideApply         bool
-		applyOutOfSyncOnly      bool
-		async                   bool
-		retryLimit              int64
-		retryRefresh            bool
-		retryBackoffDuration    time.Duration
-		retryBackoffMaxDuration time.Duration
-		retryBackoffFactor      int64
-		local                   string
-		localRepoRoot           string
-		infos                   []string
-		diffChanges             bool
-		diffChangesConfirm      bool
-		projects                []string
-		output                  string
-		appNamespace            string
-		ignoreNormalizerOpts    normalizers.IgnoreNormalizerOpts
+		revision                  string
+		revisions                 []string
+		sourcePositions           []int64
+		sourceNames               []string
+		resources                 []string
+		labels                    []string
+		selector                  string
+		prune                     bool
+		dryRun                    bool
+		timeout                   uint
+		strategy                  string
+		force                     bool
+		replace                   bool
+		serverSideApply           bool
+		applyOutOfSyncOnly        bool
+		async                     bool
+		retryLimit                int64
+		retryRefresh              bool
+		retryBackoffDuration      time.Duration
+		retryBackoffMaxDuration   time.Duration
+		retryBackoffFactor        int64
+		local                     string
+		localRepoRoot             string
+		infos                     []string
+		diffChanges               bool
+		diffChangesConfirm        bool
+		projects                  []string
+		output                    string
+		appNamespace              string
+		ignoreNormalizerOpts      normalizers.IgnoreNormalizerOpts
+		serverSideDiffConcurrency int
+		serverSideDiffMaxBatchKB  int
 	)
 	command := &cobra.Command{
 		Use:   "sync [APPNAME... | -l selector | --project project-name]",
@@ -2393,7 +2453,7 @@ func NewApplicationSyncCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co
 					// Check if application has ServerSideDiff annotation
 					serverSideDiff := resourceutil.HasAnnotationOption(app, argocommon.AnnotationCompareOptions, "ServerSideDiff=true")
 
-					foundDiffs = findAndPrintDiff(ctx, app, proj.Project, resources, argoSettings, diffOption, ignoreNormalizerOpts, serverSideDiff, appIf, appName, appNs)
+					foundDiffs = findAndPrintDiff(ctx, app, proj.Project, resources, argoSettings, diffOption, ignoreNormalizerOpts, serverSideDiff, appIf, appName, appNs, serverSideDiffConcurrency, serverSideDiffMaxBatchKB)
 					if !foundDiffs {
 						fmt.Printf("====== No Differences found ======\n")
 						// if no differences found, then no need to sync
@@ -2458,6 +2518,7 @@ func NewApplicationSyncCommand(clientOpts *argocdclient.ClientOptions) *cobra.Co
 	command.Flags().StringArrayVar(&revisions, "revisions", []string{}, "Show manifests at specific revisions for source position in source-positions")
 	command.Flags().Int64SliceVar(&sourcePositions, "source-positions", []int64{}, "List of source positions. Default is empty array. Counting start at 1.")
 	command.Flags().StringArrayVar(&sourceNames, "source-names", []string{}, "List of source names. Default is an empty array.")
+	addServerSideDiffPerfFlags(command, &serverSideDiffConcurrency, &serverSideDiffMaxBatchKB)
 	return command
 }