marcel/argo-cd
1
0
Fork 0
mirror of https://github.com/argoproj/argo-cd.git synced 2026-02-20 01:28:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix: configurable CMP tar exclusions (#9675) (#9789)

Browse Source 
* feat: configurable CMP tar exclusions

Signed-off-by: notfromstatefarm <86763948+notfromstatefarm@users.noreply.github.com>

* rename cmp to plugin

Signed-off-by: notfromstatefarm <86763948+notfromstatefarm@users.noreply.github.com>

* add test for tar stream exclusions

Signed-off-by: notfromstatefarm <86763948+notfromstatefarm@users.noreply.github.com>

* fix tests

Signed-off-by: notfromstatefarm <86763948+notfromstatefarm@users.noreply.github.com>

* update cmp guide

Signed-off-by: notfromstatefarm <86763948+notfromstatefarm@users.noreply.github.com>

* fully parameterize

Signed-off-by: notfromstatefarm <86763948+notfromstatefarm@users.noreply.github.com>
This commit is contained in:
Jake
2022-06-29 16:27:02 -04:00
committed by GitHub
parent 2ac7bd3039
commit 355025240c
20 changed files with 120 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
USERS.md
View File

@@ -37,6 +37,7 @@ Currently, the following organizations are **officially** using Argo CD:
1. [Chargetrip](https://chargetrip.com)
1. [Chime](https://www.chime.com)
1. [Cisco ET&I](https://eti.cisco.com/)
1. [Cobalt](https://www.cobalt.io/)
1. [Codefresh](https://www.codefresh.io/)
1. [Codility](https://www.codility.com/)
1. [Commonbond](https://commonbond.co/)

3
cmd/argocd-repo-server/commands/argocd_repo_server.go
View File

@@ -81,6 +81,7 @@ func NewCommand() *cobra.Command {
redisClient *redis.Client
disableTLS bool
maxCombinedDirectoryManifestsSize string
cmpTarExcludedGlobs []string
)
var command = cobra.Command{
Use: cliName,
@@ -121,6 +122,7 @@ func NewCommand() *cobra.Command {
PauseGenerationOnFailureForRequests: getPauseGenerationOnFailureForRequests(),
SubmoduleEnabled: getSubmoduleEnabled(),
MaxCombinedDirectoryManifestsSize: maxCombinedDirectoryManifestsQuantity,
CMPTarExcludedGlobs: cmpTarExcludedGlobs,
}, askPassServer)
errors.CheckError(err)
@@ -197,6 +199,7 @@ func NewCommand() *cobra.Command {
command.Flags().StringVar(&otlpAddress, "otlp-address", env.StringFromEnv("ARGOCD_REPO_SERVER_OTLP_ADDRESS", ""), "OpenTelemetry collector address to send traces to")
command.Flags().BoolVar(&disableTLS, "disable-tls", env.ParseBoolFromEnv("ARGOCD_REPO_SERVER_DISABLE_TLS", false), "Disable TLS on the gRPC endpoint")
command.Flags().StringVar(&maxCombinedDirectoryManifestsSize, "max-combined-directory-manifests-size", env.StringFromEnv("ARGOCD_REPO_SERVER_MAX_COMBINED_DIRECTORY_MANIFESTS_SIZE", "10M"), "Max combined size of manifest files in a directory-type Application")
command.Flags().StringArrayVar(&cmpTarExcludedGlobs, "plugin-tar-exclude", env.StringsFromEnv("ARGOCD_REPO_SERVER_PLUGIN_TAR_EXCLUSIONS", []string{}, ";"), "Globs to filter when sending tarballs to plugins.")
tlsConfigCustomizerSrc = tls.AddTLSFlagsToCmd(&command)
cacheSrc = reposervercache.AddCacheFlagsToCmd(&command, func(client *redis.Client) {

2
docs/operator-manual/argocd-cmd-params-cm.yaml
View File

@@ -111,3 +111,5 @@ data:
# for 300x memory expansion and N Applications running at the same time.
# (example 10M max * 300 expansion * 10 Apps = 30G max theoretical memory usage).
reposerver.max.combined.directory.manifests.size: '10M'
# Paths to be excluded from the tarball streamed to plugins. Separate with ;
reposerver.plugin.tar.exclusions: ""

1
docs/operator-manual/server-commands/argocd-repo-server.md
View File

@@ -22,6 +22,7 @@ argocd-repo-server [flags]
--metrics-port int Start metrics server on given port (default 8084)
--otlp-address string OpenTelemetry collector address to send traces to
--parallelismlimit int Limit on number of concurrent manifests generate requests. Any value less the 1 means no limit.
--plugin-tar-exclude stringArray Globs to filter when sending tarballs to plugins.
--port int Listen on given port for incoming connections (default 8081)
--redis string Redis server hostname and port (e.g. argocd-redis:6379).
--redis-ca-certificate string Path to Redis server CA certificate (e.g. /etc/certs/redis/ca.crt). If not specified, system trusted CAs will be used for server certificate validation.

14
docs/user-guide/config-management-plugins.md
View File

@@ -234,3 +234,17 @@ If you don't need to set any environment variables, you can set an empty plugin
Each CMP command will also independently timeout on the `ARGOCD_EXEC_TIMEOUT` set for the CMP sidecar. The default
is 90s. So if you increase the repo server timeout greater than 90s, be sure to set `ARGOCD_EXEC_TIMEOUT` on the
sidecar.
## Tarball stream filtering
In order to increase the speed of manifest generation, certain files and folders can be excluded from being sent to your
plugin. We recommend excluding your `.git` folder if it isn't necessary. Use Go's
[filepatch.Match](https://pkg.go.dev/path/filepath#Match) syntax.
You can set it one of three ways:
1. The `--plugin-tar-exclude` argument on the repo server.
2. The `reposerver.plugin.tar.exclusions` key if you are using `argocd-cmd-params-cm`
3. Directly setting 'ARGOCD_REPO_SERVER_PLUGIN_TAR_EXCLUSIONS' environment variable on the repo server.
For option 1, the flag can be repeated multiple times. For option 2 and 3, you can specify multiple globs by separating
them with semicolons.

6
manifests/base/repo-server/argocd-repo-server-deployment.yaml
View File

@@ -107,6 +107,12 @@ spec:
name: argocd-cmd-params-cm
key: reposerver.max.combined.directory.manifests.size
optional: true
- name: ARGOCD_REPO_SERVER_PLUGIN_TAR_EXCLUSIONS
valueFrom:
configMapKeyRef:
name: argocd-cmd-params-cm
key: reposerver.plugin.tar.exclusions
optional: true
- name: HELM_CACHE_HOME
value: /helm-working-dir
- name: HELM_CONFIG_HOME

6
manifests/core-install.yaml
View File

@@ -9757,6 +9757,12 @@ spec:
key: reposerver.max.combined.directory.manifests.size
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_REPO_SERVER_PLUGIN_TAR_EXCLUSIONS
valueFrom:
configMapKeyRef:
key: reposerver.plugin.tar.exclusions
name: argocd-cmd-params-cm
optional: true
- name: HELM_CACHE_HOME
value: /helm-working-dir
- name: HELM_CONFIG_HOME

6
manifests/ha/install.yaml
View File

@@ -10864,6 +10864,12 @@ spec:
key: reposerver.max.combined.directory.manifests.size
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_REPO_SERVER_PLUGIN_TAR_EXCLUSIONS
valueFrom:
configMapKeyRef:
key: reposerver.plugin.tar.exclusions
name: argocd-cmd-params-cm
optional: true
- name: HELM_CACHE_HOME
value: /helm-working-dir
- name: HELM_CONFIG_HOME

6
manifests/ha/namespace-install.yaml
View File

@@ -1638,6 +1638,12 @@ spec:
key: reposerver.max.combined.directory.manifests.size
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_REPO_SERVER_PLUGIN_TAR_EXCLUSIONS
valueFrom:
configMapKeyRef:
key: reposerver.plugin.tar.exclusions
name: argocd-cmd-params-cm
optional: true
- name: HELM_CACHE_HOME
value: /helm-working-dir
- name: HELM_CONFIG_HOME

6
manifests/install.yaml
View File

@@ -10206,6 +10206,12 @@ spec:
key: reposerver.max.combined.directory.manifests.size
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_REPO_SERVER_PLUGIN_TAR_EXCLUSIONS
valueFrom:
configMapKeyRef:
key: reposerver.plugin.tar.exclusions
name: argocd-cmd-params-cm
optional: true
- name: HELM_CACHE_HOME
value: /helm-working-dir
- name: HELM_CONFIG_HOME

6
manifests/namespace-install.yaml
View File

@@ -980,6 +980,12 @@ spec:
key: reposerver.max.combined.directory.manifests.size
name: argocd-cmd-params-cm
optional: true
- name: ARGOCD_REPO_SERVER_PLUGIN_TAR_EXCLUSIONS
valueFrom:
configMapKeyRef:
key: reposerver.plugin.tar.exclusions
name: argocd-cmd-params-cm
optional: true
- name: HELM_CACHE_HOME
value: /helm-working-dir
- name: HELM_CONFIG_HOME

38
reposerver/repository/repository.go
View File

@@ -103,6 +103,7 @@ type RepoServerInitConstants struct {
PauseGenerationOnFailureForRequests int
SubmoduleEnabled bool
MaxCombinedDirectoryManifestsSize resource.Quantity
CMPTarExcludedGlobs []string
}
// NewService returns a new instance of the Manifest service
@@ -213,7 +214,7 @@ func (s *Service) ListApps(ctx context.Context, q *apiclient.ListAppsRequest) (*
}
defer io.Close(closer)
apps, err := discovery.Discover(ctx, gitClient.Root(), q.EnabledSourceTypes)
apps, err := discovery.Discover(ctx, gitClient.Root(), q.EnabledSourceTypes, s.initConstants.CMPTarExcludedGlobs)
if err != nil {
return nil, err
}
@@ -465,7 +466,7 @@ func (s *Service) runManifestGenAsync(ctx context.Context, repoRoot, commitSHA,
var manifestGenResult *apiclient.ManifestResponse
opContext, err := opContextSrc()
if err == nil {
manifestGenResult, err = GenerateManifests(ctx, opContext.appPath, repoRoot, commitSHA, q, false, s.gitCredsStore, s.initConstants.MaxCombinedDirectoryManifestsSize, WithCMPTarDoneChannel(ch.tarDoneCh))
manifestGenResult, err = GenerateManifests(ctx, opContext.appPath, repoRoot, commitSHA, q, false, s.gitCredsStore, s.initConstants.MaxCombinedDirectoryManifestsSize, WithCMPTarDoneChannel(ch.tarDoneCh), WithCMPTarExcludedGlobs(s.initConstants.CMPTarExcludedGlobs))
}
if err != nil {
// If manifest generation error caching is enabled
@@ -872,7 +873,8 @@ func getRepoCredential(repoCredentials []*v1alpha1.RepoCreds, repoURL string) *v
type GenerateManifestOpt func(*generateManifestOpt)
type generateManifestOpt struct {
cmpTarDoneCh chan<- bool
cmpTarDoneCh chan<- bool
cmpTarExcludedGlobs []string
}
func newGenerateManifestOpt(opts ...GenerateManifestOpt) *generateManifestOpt {
@@ -892,6 +894,14 @@ func WithCMPTarDoneChannel(ch chan<- bool) GenerateManifestOpt {
}
}
// WithCMPTarExcludedGlobs defines globs for files to filter out when streaming the tarball
// to a CMP sidecar.
func WithCMPTarExcludedGlobs(excludedGlobs []string) GenerateManifestOpt {
return func(o *generateManifestOpt) {
o.cmpTarExcludedGlobs = excludedGlobs
}
}
// GenerateManifests generates manifests from a path. Overrides are applied as a side effect on the given ApplicationSource.
func GenerateManifests(ctx context.Context, appPath, repoRoot, revision string, q *apiclient.ManifestRequest, isLocal bool, gitCredsStore git.CredsStore, maxCombinedManifestQuantity resource.Quantity, opts ...GenerateManifestOpt) (*apiclient.ManifestResponse, error) {
opt := newGenerateManifestOpt(opts...)
@@ -900,7 +910,7 @@ func GenerateManifests(ctx context.Context, appPath, repoRoot, revision string,
resourceTracking := argo.NewResourceTracking()
appSourceType, err := GetAppSourceType(ctx, q.ApplicationSource, appPath, q.AppName, q.EnabledSourceTypes)
appSourceType, err := GetAppSourceType(ctx, q.ApplicationSource, appPath, q.AppName, q.EnabledSourceTypes, opt.cmpTarExcludedGlobs)
if err != nil {
return nil, err
}
@@ -924,7 +934,7 @@ func GenerateManifests(ctx context.Context, appPath, repoRoot, revision string,
if q.ApplicationSource.Plugin != nil && q.ApplicationSource.Plugin.Name != "" {
targetObjs, err = runConfigManagementPlugin(appPath, repoRoot, env, q, q.Repo.GetGitCreds(gitCredsStore))
} else {
targetObjs, err = runConfigManagementPluginSidecars(ctx, appPath, repoRoot, env, q, q.Repo.GetGitCreds(gitCredsStore), opt.cmpTarDoneCh)
targetObjs, err = runConfigManagementPluginSidecars(ctx, appPath, repoRoot, env, q, q.Repo.GetGitCreds(gitCredsStore), opt.cmpTarDoneCh, opt.cmpTarExcludedGlobs)
if err != nil {
err = fmt.Errorf("plugin sidecar failed. %s", err.Error())
}
@@ -1058,7 +1068,7 @@ func mergeSourceParameters(source *v1alpha1.ApplicationSource, path, appName str
}
// GetAppSourceType returns explicit application source type or examines a directory and determines its application source type
func GetAppSourceType(ctx context.Context, source *v1alpha1.ApplicationSource, path, appName string, enableGenerateManifests map[string]bool) (v1alpha1.ApplicationSourceType, error) {
func GetAppSourceType(ctx context.Context, source *v1alpha1.ApplicationSource, path, appName string, enableGenerateManifests map[string]bool, tarExcludedGlobs []string) (v1alpha1.ApplicationSourceType, error) {
err := mergeSourceParameters(source, path, appName)
if err != nil {
return "", fmt.Errorf("error while parsing source parameters: %v", err)
@@ -1075,7 +1085,7 @@ func GetAppSourceType(ctx context.Context, source *v1alpha1.ApplicationSource, p
}
return *appSourceType, nil
}
appType, err := discovery.AppType(ctx, path, enableGenerateManifests)
appType, err := discovery.AppType(ctx, path, enableGenerateManifests, tarExcludedGlobs)
if err != nil {
return "", err
}
@@ -1479,7 +1489,7 @@ func getPluginEnvs(envVars *v1alpha1.Env, q *apiclient.ManifestRequest, creds gi
return env, nil
}
func runConfigManagementPluginSidecars(ctx context.Context, appPath, repoPath string, envVars *v1alpha1.Env, q *apiclient.ManifestRequest, creds git.Creds, tarDoneCh chan<- bool) ([]*unstructured.Unstructured, error) {
func runConfigManagementPluginSidecars(ctx context.Context, appPath, repoPath string, envVars *v1alpha1.Env, q *apiclient.ManifestRequest, creds git.Creds, tarDoneCh chan<- bool, tarExcludedGlobs []string) ([]*unstructured.Unstructured, error) {
// compute variables.
env, err := getPluginEnvs(envVars, q, creds, true)
if err != nil {
@@ -1487,14 +1497,14 @@ func runConfigManagementPluginSidecars(ctx context.Context, appPath, repoPath st
}
// detect config management plugin server (sidecar)
conn, cmpClient, err := discovery.DetectConfigManagementPlugin(ctx, appPath, env)
conn, cmpClient, err := discovery.DetectConfigManagementPlugin(ctx, appPath, env, tarExcludedGlobs)
if err != nil {
return nil, err
}
defer io.Close(conn)
// generate manifests using commands provided in plugin config file in detected cmp-server sidecar
cmpManifests, err := generateManifestsCMP(ctx, appPath, repoPath, env, cmpClient, tarDoneCh)
cmpManifests, err := generateManifestsCMP(ctx, appPath, repoPath, env, cmpClient, tarDoneCh, tarExcludedGlobs)
if err != nil {
return nil, fmt.Errorf("error generating manifests in cmp: %s", err)
}
@@ -1512,7 +1522,7 @@ func runConfigManagementPluginSidecars(ctx context.Context, appPath, repoPath st
// generateManifestsCMP will send the appPath files to the cmp-server over a gRPC stream.
// The cmp-server will generate the manifests. Returns a response object with the generated
// manifests.
func generateManifestsCMP(ctx context.Context, appPath, repoPath string, env []string, cmpClient pluginclient.ConfigManagementPluginServiceClient, tarDoneCh chan<- bool) (*pluginclient.ManifestResponse, error) {
func generateManifestsCMP(ctx context.Context, appPath, repoPath string, env []string, cmpClient pluginclient.ConfigManagementPluginServiceClient, tarDoneCh chan<- bool, tarExcludedGlobs []string) (*pluginclient.ManifestResponse, error) {
generateManifestStream, err := cmpClient.GenerateManifest(ctx, grpc_retry.Disable())
if err != nil {
return nil, fmt.Errorf("error getting generateManifestStream: %s", err)
@@ -1520,7 +1530,7 @@ func generateManifestsCMP(ctx context.Context, appPath, repoPath string, env []s
opts := []cmp.SenderOption{
cmp.WithTarDoneChan(tarDoneCh),
}
err = cmp.SendRepoStream(generateManifestStream.Context(), appPath, repoPath, generateManifestStream, env, opts...)
err = cmp.SendRepoStream(generateManifestStream.Context(), appPath, repoPath, generateManifestStream, env, tarExcludedGlobs, opts...)
if err != nil {
return nil, fmt.Errorf("error sending file to cmp-server: %s", err)
}
@@ -1538,7 +1548,7 @@ func (s *Service) GetAppDetails(ctx context.Context, q *apiclient.RepoServerAppD
return err
}
appSourceType, err := GetAppSourceType(ctx, q.Source, opContext.appPath, q.AppName, q.EnabledSourceTypes)
appSourceType, err := GetAppSourceType(ctx, q.Source, opContext.appPath, q.AppName, q.EnabledSourceTypes, s.initConstants.CMPTarExcludedGlobs)
if err != nil {
return err
}
@@ -1653,7 +1663,7 @@ func loadFileIntoIfExists(path pathutil.ResolvedFilePath, destination *string) e
info, err := os.Stat(stringPath)
if err == nil && !info.IsDir() {
bytes, err := ioutil.ReadFile(stringPath);
bytes, err := ioutil.ReadFile(stringPath)
if err != nil {
return err
}

8
reposerver/repository/repository_test.go
View File

@@ -1049,15 +1049,15 @@ func TestGenerateNullList(t *testing.T) {
}
func TestIdentifyAppSourceTypeByAppDirWithKustomizations(t *testing.T) {
sourceType, err := GetAppSourceType(context.Background(), &argoappv1.ApplicationSource{}, "./testdata/kustomization_yaml", "testapp", map[string]bool{})
sourceType, err := GetAppSourceType(context.Background(), &argoappv1.ApplicationSource{}, "./testdata/kustomization_yaml", "testapp", map[string]bool{}, []string{})
assert.Nil(t, err)
assert.Equal(t, argoappv1.ApplicationSourceTypeKustomize, sourceType)
sourceType, err = GetAppSourceType(context.Background(), &argoappv1.ApplicationSource{}, "./testdata/kustomization_yml", "testapp", map[string]bool{})
sourceType, err = GetAppSourceType(context.Background(), &argoappv1.ApplicationSource{}, "./testdata/kustomization_yml", "testapp", map[string]bool{}, []string{})
assert.Nil(t, err)
assert.Equal(t, argoappv1.ApplicationSourceTypeKustomize, sourceType)
sourceType, err = GetAppSourceType(context.Background(), &argoappv1.ApplicationSource{}, "./testdata/Kustomization", "testapp", map[string]bool{})
sourceType, err = GetAppSourceType(context.Background(), &argoappv1.ApplicationSource{}, "./testdata/Kustomization", "testapp", map[string]bool{}, []string{})
assert.Nil(t, err)
assert.Equal(t, argoappv1.ApplicationSourceTypeKustomize, sourceType)
}
@@ -1582,7 +1582,7 @@ func TestGenerateManifestsWithAppParameterFile(t *testing.T) {
source := &argoappv1.ApplicationSource{
Path: path,
}
sourceCopy := source.DeepCopy() // make a copy in case GenerateManifest mutates it.
sourceCopy := source.DeepCopy() // make a copy in case GenerateManifest mutates it.
_, err := service.GenerateManifest(context.Background(), &apiclient.ManifestRequest{
Repo: &argoappv1.Repository{},
ApplicationSource: sourceCopy,

16
util/app/discovery/discovery.go
View File

@@ -29,11 +29,11 @@ func IsManifestGenerationEnabled(sourceType v1alpha1.ApplicationSourceType, enab
return enabled
}
func Discover(ctx context.Context, repoPath string, enableGenerateManifests map[string]bool) (map[string]string, error) {
func Discover(ctx context.Context, repoPath string, enableGenerateManifests map[string]bool, tarExcludedGlobs []string) (map[string]string, error) {
apps := make(map[string]string)
// Check if it is CMP
conn, _, err := DetectConfigManagementPlugin(ctx, repoPath, []string{})
conn, _, err := DetectConfigManagementPlugin(ctx, repoPath, []string{}, tarExcludedGlobs)
if err == nil {
// Found CMP
io.Close(conn)
@@ -65,8 +65,8 @@ func Discover(ctx context.Context, repoPath string, enableGenerateManifests map[
return apps, err
}
func AppType(ctx context.Context, path string, enableGenerateManifests map[string]bool) (string, error) {
apps, err := Discover(ctx, path, enableGenerateManifests)
func AppType(ctx context.Context, path string, enableGenerateManifests map[string]bool, tarExcludedGlobs []string) (string, error) {
apps, err := Discover(ctx, path, enableGenerateManifests, tarExcludedGlobs)
if err != nil {
return "", err
}
@@ -82,7 +82,7 @@ func AppType(ctx context.Context, path string, enableGenerateManifests map[strin
// 3. check isSupported(path)?
// 4.a if no then close connection
// 4.b if yes then return conn for detected plugin
func DetectConfigManagementPlugin(ctx context.Context, repoPath string, env []string) (io.Closer, pluginclient.ConfigManagementPluginServiceClient, error) {
func DetectConfigManagementPlugin(ctx context.Context, repoPath string, env []string, tarExcludedGlobs []string) (io.Closer, pluginclient.ConfigManagementPluginServiceClient, error) {
var conn io.Closer
var cmpClient pluginclient.ConfigManagementPluginServiceClient
@@ -106,7 +106,7 @@ func DetectConfigManagementPlugin(ctx context.Context, repoPath string, env []st
continue
}
isSupported, err := matchRepositoryCMP(ctx, repoPath, cmpClient, env)
isSupported, err := matchRepositoryCMP(ctx, repoPath, cmpClient, env, tarExcludedGlobs)
if err != nil {
log.Errorf("repository %s is not the match because %v", repoPath, err)
continue
@@ -131,13 +131,13 @@ func DetectConfigManagementPlugin(ctx context.Context, repoPath string, env []st
// matchRepositoryCMP will send the repoPath to the cmp-server. The cmp-server will
// inspect the files and return true if the repo is supported for manifest generation.
// Will return false otherwise.
func matchRepositoryCMP(ctx context.Context, repoPath string, client pluginclient.ConfigManagementPluginServiceClient, env []string) (bool, error) {
func matchRepositoryCMP(ctx context.Context, repoPath string, client pluginclient.ConfigManagementPluginServiceClient, env []string, tarExcludedGlobs []string) (bool, error) {
matchRepoStream, err := client.MatchRepository(ctx, grpc_retry.Disable())
if err != nil {
return false, fmt.Errorf("error getting stream client: %s", err)
}
err = cmp.SendRepoStream(ctx, repoPath, repoPath, matchRepoStream, env)
err = cmp.SendRepoStream(ctx, repoPath, repoPath, matchRepoStream, env, tarExcludedGlobs)
if err != nil {
return false, fmt.Errorf("error sending stream: %s", err)
}

14
util/app/discovery/discovery_test.go
View File

@@ -10,7 +10,7 @@ import (
)
func TestDiscover(t *testing.T) {
apps, err := Discover(context.Background(), "./testdata", map[string]bool{})
apps, err := Discover(context.Background(), "./testdata", map[string]bool{}, []string{})
assert.NoError(t, err)
assert.Equal(t, map[string]string{
"foo": "Kustomize",
@@ -19,15 +19,15 @@ func TestDiscover(t *testing.T) {
}
func TestAppType(t *testing.T) {
appType, err := AppType(context.Background(), "./testdata/foo", map[string]bool{})
appType, err := AppType(context.Background(), "./testdata/foo", map[string]bool{}, []string{})
assert.NoError(t, err)
assert.Equal(t, "Kustomize", appType)
appType, err = AppType(context.Background(), "./testdata/baz", map[string]bool{})
appType, err = AppType(context.Background(), "./testdata/baz", map[string]bool{}, []string{})
assert.NoError(t, err)
assert.Equal(t, "Helm", appType)
appType, err = AppType(context.Background(), "./testdata", map[string]bool{})
appType, err = AppType(context.Background(), "./testdata", map[string]bool{}, []string{})
assert.NoError(t, err)
assert.Equal(t, "Directory", appType)
}
@@ -37,15 +37,15 @@ func TestAppType_Disabled(t *testing.T) {
string(v1alpha1.ApplicationSourceTypeKustomize): false,
string(v1alpha1.ApplicationSourceTypeHelm): false,
}
appType, err := AppType(context.Background(), "./testdata/foo", enableManifestGeneration)
appType, err := AppType(context.Background(), "./testdata/foo", enableManifestGeneration, []string{})
assert.NoError(t, err)
assert.Equal(t, "Directory", appType)
appType, err = AppType(context.Background(), "./testdata/baz", enableManifestGeneration)
appType, err = AppType(context.Background(), "./testdata/baz", enableManifestGeneration, []string{})
assert.NoError(t, err)
assert.Equal(t, "Directory", appType)
appType, err = AppType(context.Background(), "./testdata", enableManifestGeneration)
appType, err = AppType(context.Background(), "./testdata", enableManifestGeneration, []string{})
assert.NoError(t, err)
assert.Equal(t, "Directory", appType)
}

13
util/cmp/stream.go
View File

@@ -84,11 +84,11 @@ func WithTarDoneChan(ch chan<- bool) SenderOption {
// SendRepoStream will compress the files under the given repoPath and send
// them using the plugin stream sender.
func SendRepoStream(ctx context.Context, appPath, repoPath string, sender StreamSender, env []string, opts ...SenderOption) error {
func SendRepoStream(ctx context.Context, appPath, repoPath string, sender StreamSender, env []string, excludedGlobs []string, opts ...SenderOption) error {
opt := newSenderOption(opts...)
// compress all files in appPath in tgz
tgz, checksum, err := compressFiles(repoPath)
tgz, checksum, err := compressFiles(repoPath, excludedGlobs)
if err != nil {
return fmt.Errorf("error compressing repo files: %w", err)
}
@@ -162,11 +162,10 @@ func closeAndDelete(f *os.File) {
}
// compressFiles will create a tgz file with all contents of appPath
// directory excluding the .git folder. Returns the file alongside
// its sha256 hash to be used as checksum. It is the responsibility
// of the caller to close the file.
func compressFiles(appPath string) (*os.File, string, error) {
excluded := []string{".git"}
// directory excluding globs in the excluded array. Returns the file
// alongside its sha256 hash to be used as checksum. It is the
// responsibility of the caller to close the file.
func compressFiles(appPath string, excluded []string) (*os.File, string, error) {
appName := filepath.Base(appPath)
tempDir, err := files.CreateTempDir(os.TempDir())
if err != nil {

8
util/cmp/stream_test.go
View File

@@ -60,7 +60,7 @@ func TestReceiveApplicationStream(t *testing.T) {
close(streamMock.messages)
os.RemoveAll(workdir)
}()
go streamMock.sendFile(context.Background(), t, appDir, streamMock, []string{"env1", "env2"})
go streamMock.sendFile(context.Background(), t, appDir, streamMock, []string{"env1", "env2"}, []string{"DUMMY.md", "dum*"})
// when
env, err := cmp.ReceiveRepoStream(context.Background(), streamMock, workdir)
@@ -77,16 +77,18 @@ func TestReceiveApplicationStream(t *testing.T) {
}
assert.Contains(t, names, "README.md")
assert.Contains(t, names, "applicationset")
assert.NotContains(t, names, "DUMMY.md")
assert.NotContains(t, names, "dummy")
assert.NotNil(t, env)
})
}
func (m *streamMock) sendFile(ctx context.Context, t *testing.T, basedir string, sender cmp.StreamSender, env []string) {
func (m *streamMock) sendFile(ctx context.Context, t *testing.T, basedir string, sender cmp.StreamSender, env []string, excludedGlobs []string) {
t.Helper()
defer func() {
m.done <- true
}()
err := cmp.SendRepoStream(ctx, basedir, basedir, sender, env)
err := cmp.SendRepoStream(ctx, basedir, basedir, sender, env, excludedGlobs)
require.NoError(t, err)
}

1
util/cmp/testdata/app/DUMMY.md vendored Normal file
View File

@@ -0,0 +1 @@
This file is used to test the tar stream exclusions.

1
util/cmp/testdata/app/dummy/DUMMY2.md vendored Normal file
View File

@@ -0,0 +1 @@
This is another file that should get excluded because the entire directory is excluded.

7
util/env/env.go vendored
View File

@@ -126,6 +126,13 @@ func StringFromEnv(env string, defaultValue string) string {
return defaultValue
}
func StringsFromEnv(env string, defaultValue []string, separator string) []string {
if str := os.Getenv(env); str != "" {
return strings.Split(str, separator)
}
return defaultValue
}
// ParseBoolFromEnv retrieves a boolean value from given environment envVar.
// Returns default value if envVar is not set.
//