chore: portable and simpler toolchain install (#7920)

Signed-off-by: Jesse Suen <jesse@akuity.io>

hack/generate-proto.sh

@@ -9,24 +9,16 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+PROJECT_ROOT=$(cd $(dirname ${BASH_SOURCE})/..; pwd)
+PATH="${PROJECT_ROOT}/dist:${PATH}"
+
 # output tool versions
 protoc --version
 swagger version
 jq --version
 
-PROJECT_ROOT=$(cd $(dirname ${BASH_SOURCE})/..; pwd)
-CODEGEN_PKG=${CODEGEN_PKG:-$(cd ${PROJECT_ROOT}; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ../code-generator)}
-PATH="${PROJECT_ROOT}/dist:${PATH}"
-MOD_ROOT=${GOPATH}/pkg/mod
-
-. ${PROJECT_ROOT}/hack/versions.sh
-
 export GO111MODULE=off
 
-# protobuf tooling required to build .proto files from go annotations from k8s-like api types
-go build -o dist/go-to-protobuf ./vendor/k8s.io/code-generator/cmd/go-to-protobuf
-go build -o dist/protoc-gen-gogo ./vendor/k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo
-
 # Generate pkg/apis/<group>/<apiversion>/(generated.proto,generated.pb.go)
 # NOTE: any dependencies of our types to the k8s.io apimachinery types should be added to the
 # --apimachinery-packages= option so that go-to-protobuf can locate the types, but prefixed with a
@@ -56,26 +48,20 @@ ${PROJECT_ROOT}/dist/go-to-protobuf \
 # server/*/<service>.pb.go from .proto files. golang/protobuf and gogo/protobuf can be used
 # interchangeably. The difference in the options are:
 # 1. protoc-gen-go - official golang/protobuf
-#go build -o dist/protoc-gen-go ./vendor/github.com/golang/protobuf/protoc-gen-go
 #GOPROTOBINARY=go
 # 2. protoc-gen-gofast - fork of golang golang/protobuf. Faster code generation
-#go build -o dist/protoc-gen-gofast ./vendor/github.com/gogo/protobuf/protoc-gen-gofast
 #GOPROTOBINARY=gofast
 # 3. protoc-gen-gogofast - faster code generation and gogo extensions and flexibility in controlling
 # the generated go code (e.g. customizing field names, nullable fields)
-go build -o dist/protoc-gen-gogofast ./vendor/github.com/gogo/protobuf/protoc-gen-gogofast
 GOPROTOBINARY=gogofast
 
-# protoc-gen-grpc-gateway is used to build <service>.pb.gw.go files from from .proto files
-go build -o dist/protoc-gen-grpc-gateway ./vendor/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
-# protoc-gen-swagger is used to build swagger.json
-go build -o dist/protoc-gen-swagger ./vendor/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger
-
 # Generate server/<service>/(<service>.pb.go|<service>.pb.gw.go)
+MOD_ROOT=${GOPATH}/pkg/mod
+grpc_gateway_version=$(go list -m github.com/grpc-ecosystem/grpc-gateway | awk '{print $NF}' | head -1)
+GOOGLE_PROTO_API_PATH=${MOD_ROOT}/github.com/grpc-ecosystem/grpc-gateway@${grpc_gateway_version}/third_party/googleapis
+GOGO_PROTOBUF_PATH=${PROJECT_ROOT}/vendor/github.com/gogo/protobuf
 PROTO_FILES=$(find $PROJECT_ROOT \( -name "*.proto" -and -path '*/server/*' -or -path '*/reposerver/*' -and -name "*.proto" -or -path '*/cmpserver/*' -and -name "*.proto" \) | sort)
 for i in ${PROTO_FILES}; do
-    GOOGLE_PROTO_API_PATH=${MOD_ROOT}/github.com/grpc-ecosystem/grpc-gateway@${grpc_gateway_version}/third_party/googleapis
-    GOGO_PROTOBUF_PATH=${PROJECT_ROOT}/vendor/github.com/gogo/protobuf
     protoc \
         -I${PROJECT_ROOT} \
         -I/usr/local/include \

hack/install.sh

@@ -9,6 +9,7 @@ mkdir -p $DOWNLOADS
 ARCHITECTURE=""
 case $(uname -m) in
     x86_64)                     ARCHITECTURE="amd64" ;;
+    arm64)                      ARCHITECTURE="arm64" ;;
     arm|armv7l|armv8l|aarch64)  dpkg --print-architecture | grep -q "arm64" && ARCHITECTURE="arm64" || ARCHITECTURE="arm" ;;
 esac
 

hack/installers/compare-chksum.sh

@@ -23,4 +23,4 @@ if ! grep -q "${TARGET_FILE}" ${CHKSUM_FILE}; then
 	exit 1
 fi
 
-sha256sum -c ${CHKSUM_FILE}
+shasum -a 256 -c ${CHKSUM_FILE}

hack/installers/install-codegen-go-tools.sh

@@ -1,9 +1,52 @@
 #!/bin/bash
 set -eux -o pipefail
 
-GO111MODULE=on go get github.com/gogo/protobuf/gogoproto@v1.3.1
+SRCROOT="$( CDPATH='' cd -- "$(dirname "$0")/../.." && pwd -P )"
-GO111MODULE=on go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1
+
-GO111MODULE=on go get github.com/golang/protobuf/protoc-gen-go@v1.4.2
+# This script installs all our golang-based codegen utility CLIs necessary for codegen.
-GO111MODULE=on go get github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway@v1.12.2
+# Some dependencies are vendored in go.mod (ones which are actually imported in our codebase).
-GO111MODULE=on go get github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger@v1.12.2
+# Other dependencies are only used as a CLI and do not need vendoring in go.mod (doing so adds
-GO111MODULE=on go get golang.org/x/tools/cmd/goimports@v0.0.0-20190627203933-19ff4fff8850
+# unecessary dependencies to go.mod). We want to maintain a single source of truth for versioning
+# our binaries (either go.mod or go install <pkg>@<version>), so we use two techniques to install
+# our CLIs:
+# 1. For CLIs which are NOT vendored in go.mod, we can run `go install` with an explicit version
+# 2. For packages which we *do* vendor in go.mod, we can run `go install` from the vendor directory
+go_mod_install() {
+    go install -mod=vendor ./vendor/$1
+}
+
+# All binaries are compiled into the argo-cd/dist directory, which is added to the PATH during codegen
+export GOBIN="${SRCROOT}/dist"
+mkdir -p $GOBIN
+
+# protoc-gen-go* is used to generate <service>.pb.go from .proto files
+#go_mod_install github.com/golang/protobuf/protoc-gen-go
+#go_mod_install github.com/gogo/protobuf/protoc-gen-gogo
+go_mod_install github.com/gogo/protobuf/protoc-gen-gogofast
+
+# protoc-gen-grpc-gateway is used to build <service>.pb.gw.go files from from .proto files
+go_mod_install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
+
+# # protoc-gen-swagger is used to build swagger.json
+go_mod_install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger
+
+# k8s tools to codegen .proto files, client libraries, and helpers from types.go
+go_mod_install k8s.io/code-generator/cmd/go-to-protobuf
+go_mod_install k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo
+go_mod_install k8s.io/code-generator/cmd/client-gen
+go_mod_install k8s.io/code-generator/cmd/deepcopy-gen
+go_mod_install k8s.io/code-generator/cmd/defaulter-gen
+go_mod_install k8s.io/code-generator/cmd/informer-gen
+go_mod_install k8s.io/code-generator/cmd/lister-gen
+
+# We still install openapi-gen from go.mod since upstream does not utilize release tags
+go_mod_install k8s.io/kube-openapi/cmd/openapi-gen
+
+# controller-gen is run by ./hack/gen-crd-spec to generate the CRDs
+go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1
+
+# swagger cli is used to generate swagger docs
+go install github.com/go-swagger/go-swagger/cmd/swagger@v0.28.0
+
+# goimports is used to auto-format generated code
+go install golang.org/x/tools/cmd/goimports@v0.1.8

hack/installers/install-codegen-tools.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 set -eux -o pipefail
 
-KUSTOMIZE_VERSION=4.2.0 "$(dirname $0)/../install.sh" helm2-linux jq-linux kustomize-linux protoc-linux swagger-linux
+KUSTOMIZE_VERSION=4.2.0 "$(dirname $0)/../install.sh" helm2-linux jq-linux kustomize-linux protoc-linux

hack/installers/install-dep-linux.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-set -eux -o pipefail
-
-[ -e $DOWNLOADS/dep ] || curl -sLf --retry 3 -o $DOWNLOADS/dep https://github.com/golang/dep/releases/download/v0.5.3/dep-linux-$ARCHITECTURE
-cp $DOWNLOADS/dep $BIN/
-chmod +x $BIN/dep
-dep version

hack/installers/install-swagger-linux.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-set -eux -o pipefail
-
-. $(dirname $0)/../tool-versions.sh
-
-export TARGET_FILE=swagger_linux_${ARCHITECTURE}_${swagger_version}
-[ -e $DOWNLOADS/${TARGET_FILE} ] || curl -sLf --retry 3 -o $DOWNLOADS/${TARGET_FILE} https://github.com/go-swagger/go-swagger/releases/download/v${swagger_version}/swagger_linux_$ARCHITECTURE
-$(dirname $0)/compare-chksum.sh
-sudo install -m 0755 $DOWNLOADS/${TARGET_FILE} $BIN/swagger
-swagger version

hack/tool-versions.sh

@@ -16,4 +16,3 @@ kubectl_version=1.17.8
 kubectx_version=0.6.3
 kustomize4_version=4.2.0
 protoc_version=3.7.1
-swagger_version=0.19.0

hack/tools.go

@@ -1,19 +1,28 @@
+//go:build tools
 // +build tools
 
 package tools
 
 import (
+	// gogo/protobuf is vendored because the generated *.pb.go code imports it.
+	// Also, we need the gogo/protobuf/gogoproto/gogo.proto file
 	_ "github.com/gogo/protobuf/protoc-gen-gogofast"
+
+	// grpc-ecosystem/grpc-gateway is vendored because the generated *.pb.gw.go code imports it.
+	// Also, we need the .proto files under grpc-gateway/third_party/googleapis
 	_ "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway"
 	_ "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger"
+
+	// k8s.io/code-generator is vendored to get generate-groups.sh, and k8s codegen utilities
+	_ "k8s.io/code-generator"
 	_ "k8s.io/code-generator/cmd/client-gen"
 	_ "k8s.io/code-generator/cmd/deepcopy-gen"
 	_ "k8s.io/code-generator/cmd/defaulter-gen"
-	_ "k8s.io/code-generator/cmd/informer-gen"
-	_ "k8s.io/code-generator/cmd/lister-gen"
 	_ "k8s.io/code-generator/cmd/go-to-protobuf"
 	_ "k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo"
-	_ "k8s.io/code-generator/pkg/util"
+	_ "k8s.io/code-generator/cmd/informer-gen"
-	_ "k8s.io/code-generator/third_party/forked/golang/reflect"
+	_ "k8s.io/code-generator/cmd/lister-gen"
+
+	// openapi-gen is vendored because upstream does not have tagged releases
 	_ "k8s.io/kube-openapi/cmd/openapi-gen"
 )

hack/update-codegen.sh

@@ -19,25 +19,14 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
+PROJECT_ROOT=$(cd $(dirname ${BASH_SOURCE})/..; pwd)
-. ${SCRIPT_ROOT}/hack/versions.sh
+
-CODEGEN_PKG=$GOPATH/pkg/mod/k8s.io/code-generator@${kube_version}
 TARGET_SCRIPT=/tmp/generate-groups.sh
-
+sed -e '/go install/d' ${PROJECT_ROOT}/vendor/k8s.io/code-generator/generate-groups.sh > ${TARGET_SCRIPT}
-(
-  cd $CODEGEN_PKG
-  go install ./cmd/{defaulter-gen,client-gen,lister-gen,informer-gen,deepcopy-gen}
-)
-
-export GO111MODULE=off
-
-sed -e '/go install/d' ${CODEGEN_PKG}/generate-groups.sh > ${TARGET_SCRIPT}
-
-export GO111MODULE=on
 
 [ -e ./v2 ] || ln -s . v2
-bash -x ${TARGET_SCRIPT} "deepcopy,client,informer,lister" \
+GOBIN=${PROJECT_ROOT}/dist bash -x ${TARGET_SCRIPT} "deepcopy,client,informer,lister" \
   github.com/argoproj/argo-cd/v2/pkg/client github.com/argoproj/argo-cd/v2/pkg/apis \
   "application:v1alpha1" \
-  --go-header-file ${SCRIPT_ROOT}/hack/custom-boilerplate.go.txt
+  --go-header-file ${PROJECT_ROOT}/hack/custom-boilerplate.go.txt
 [ -e ./v2 ] && rm -rf v2

hack/update-openapi.sh

@@ -6,14 +6,9 @@ set -o nounset
 set -o pipefail
 
 PROJECT_ROOT=$(cd $(dirname "$0")/.. ; pwd)
-CODEGEN_PKG=${PROJECT_ROOT}/vendor/k8s.io/kube-openapi
+PATH="${PROJECT_ROOT}/dist:${PATH}"
 VERSION="v1alpha1"
 
-export GO111MODULE=off
-go build -o dist/openapi-gen ${CODEGEN_PKG}/cmd/openapi-gen
-
-export GO111MODULE=on
-
 [ -e ./v2 ] || ln -s . v2
 ./dist/openapi-gen \
   --go-header-file ${PROJECT_ROOT}/hack/custom-boilerplate.go.txt \

hack/versions.sh

@@ -1,7 +0,0 @@
-#!/bin/sh
-# Update required versions of dependencies here whenever you change them in 
-# go.mod
-kube_version=v0.22.2
-grpc_version=v1.26.0
-protobuf_version=v1.3.2
-grpc_gateway_version=v1.16.0

test/container/Dockerfile

@@ -38,8 +38,7 @@ ADD ./hack/tool-versions.sh .
 ADD ./hack/install.sh .
 ADD ./hack/installers installers
 
-RUN ./install.sh dep-linux && \
+RUN ./install.sh ksonnet-linux && \
-    ./install.sh ksonnet-linux && \
     ./install.sh helm2-linux && \
     ./install.sh helm-linux && \
     ./install.sh kubectl-linux && \