diff --git a/hack/generate-proto.sh b/hack/generate-proto.sh index 0898df19ec..0084ad5922 100755 --- a/hack/generate-proto.sh +++ b/hack/generate-proto.sh @@ -9,24 +9,16 @@ set -o errexit set -o nounset set -o pipefail +PROJECT_ROOT=$(cd $(dirname ${BASH_SOURCE})/..; pwd) +PATH="${PROJECT_ROOT}/dist:${PATH}" + # output tool versions protoc --version swagger version jq --version -PROJECT_ROOT=$(cd $(dirname ${BASH_SOURCE})/..; pwd) -CODEGEN_PKG=${CODEGEN_PKG:-$(cd ${PROJECT_ROOT}; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ../code-generator)} -PATH="${PROJECT_ROOT}/dist:${PATH}" -MOD_ROOT=${GOPATH}/pkg/mod - -. ${PROJECT_ROOT}/hack/versions.sh - export GO111MODULE=off -# protobuf tooling required to build .proto files from go annotations from k8s-like api types -go build -o dist/go-to-protobuf ./vendor/k8s.io/code-generator/cmd/go-to-protobuf -go build -o dist/protoc-gen-gogo ./vendor/k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo - # Generate pkg/apis///(generated.proto,generated.pb.go) # NOTE: any dependencies of our types to the k8s.io apimachinery types should be added to the # --apimachinery-packages= option so that go-to-protobuf can locate the types, but prefixed with a @@ -56,26 +48,20 @@ ${PROJECT_ROOT}/dist/go-to-protobuf \ # server/*/.pb.go from .proto files. golang/protobuf and gogo/protobuf can be used # interchangeably. The difference in the options are: # 1. protoc-gen-go - official golang/protobuf -#go build -o dist/protoc-gen-go ./vendor/github.com/golang/protobuf/protoc-gen-go #GOPROTOBINARY=go # 2. protoc-gen-gofast - fork of golang golang/protobuf. Faster code generation -#go build -o dist/protoc-gen-gofast ./vendor/github.com/gogo/protobuf/protoc-gen-gofast #GOPROTOBINARY=gofast # 3. protoc-gen-gogofast - faster code generation and gogo extensions and flexibility in controlling # the generated go code (e.g. customizing field names, nullable fields) -go build -o dist/protoc-gen-gogofast ./vendor/github.com/gogo/protobuf/protoc-gen-gogofast GOPROTOBINARY=gogofast -# protoc-gen-grpc-gateway is used to build .pb.gw.go files from from .proto files -go build -o dist/protoc-gen-grpc-gateway ./vendor/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway -# protoc-gen-swagger is used to build swagger.json -go build -o dist/protoc-gen-swagger ./vendor/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger - # Generate server//(.pb.go|.pb.gw.go) +MOD_ROOT=${GOPATH}/pkg/mod +grpc_gateway_version=$(go list -m github.com/grpc-ecosystem/grpc-gateway | awk '{print $NF}' | head -1) +GOOGLE_PROTO_API_PATH=${MOD_ROOT}/github.com/grpc-ecosystem/grpc-gateway@${grpc_gateway_version}/third_party/googleapis +GOGO_PROTOBUF_PATH=${PROJECT_ROOT}/vendor/github.com/gogo/protobuf PROTO_FILES=$(find $PROJECT_ROOT \( -name "*.proto" -and -path '*/server/*' -or -path '*/reposerver/*' -and -name "*.proto" -or -path '*/cmpserver/*' -and -name "*.proto" \) | sort) for i in ${PROTO_FILES}; do - GOOGLE_PROTO_API_PATH=${MOD_ROOT}/github.com/grpc-ecosystem/grpc-gateway@${grpc_gateway_version}/third_party/googleapis - GOGO_PROTOBUF_PATH=${PROJECT_ROOT}/vendor/github.com/gogo/protobuf protoc \ -I${PROJECT_ROOT} \ -I/usr/local/include \ diff --git a/hack/install.sh b/hack/install.sh index 15ef7f9dd1..a5da4f1da3 100755 --- a/hack/install.sh +++ b/hack/install.sh @@ -9,6 +9,7 @@ mkdir -p $DOWNLOADS ARCHITECTURE="" case $(uname -m) in x86_64) ARCHITECTURE="amd64" ;; + arm64) ARCHITECTURE="arm64" ;; arm|armv7l|armv8l|aarch64) dpkg --print-architecture | grep -q "arm64" && ARCHITECTURE="arm64" || ARCHITECTURE="arm" ;; esac diff --git a/hack/installers/compare-chksum.sh b/hack/installers/compare-chksum.sh index e68e2ab484..e69a2fe71f 100755 --- a/hack/installers/compare-chksum.sh +++ b/hack/installers/compare-chksum.sh @@ -23,4 +23,4 @@ if ! grep -q "${TARGET_FILE}" ${CHKSUM_FILE}; then exit 1 fi -sha256sum -c ${CHKSUM_FILE} \ No newline at end of file +shasum -a 256 -c ${CHKSUM_FILE} \ No newline at end of file diff --git a/hack/installers/install-codegen-go-tools.sh b/hack/installers/install-codegen-go-tools.sh index c696b10643..fab4c0a109 100755 --- a/hack/installers/install-codegen-go-tools.sh +++ b/hack/installers/install-codegen-go-tools.sh @@ -1,9 +1,52 @@ #!/bin/bash set -eux -o pipefail -GO111MODULE=on go get github.com/gogo/protobuf/gogoproto@v1.3.1 -GO111MODULE=on go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1 -GO111MODULE=on go get github.com/golang/protobuf/protoc-gen-go@v1.4.2 -GO111MODULE=on go get github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway@v1.12.2 -GO111MODULE=on go get github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger@v1.12.2 -GO111MODULE=on go get golang.org/x/tools/cmd/goimports@v0.0.0-20190627203933-19ff4fff8850 +SRCROOT="$( CDPATH='' cd -- "$(dirname "$0")/../.." && pwd -P )" + +# This script installs all our golang-based codegen utility CLIs necessary for codegen. +# Some dependencies are vendored in go.mod (ones which are actually imported in our codebase). +# Other dependencies are only used as a CLI and do not need vendoring in go.mod (doing so adds +# unecessary dependencies to go.mod). We want to maintain a single source of truth for versioning +# our binaries (either go.mod or go install @), so we use two techniques to install +# our CLIs: +# 1. For CLIs which are NOT vendored in go.mod, we can run `go install` with an explicit version +# 2. For packages which we *do* vendor in go.mod, we can run `go install` from the vendor directory +go_mod_install() { + go install -mod=vendor ./vendor/$1 +} + +# All binaries are compiled into the argo-cd/dist directory, which is added to the PATH during codegen +export GOBIN="${SRCROOT}/dist" +mkdir -p $GOBIN + +# protoc-gen-go* is used to generate .pb.go from .proto files +#go_mod_install github.com/golang/protobuf/protoc-gen-go +#go_mod_install github.com/gogo/protobuf/protoc-gen-gogo +go_mod_install github.com/gogo/protobuf/protoc-gen-gogofast + +# protoc-gen-grpc-gateway is used to build .pb.gw.go files from from .proto files +go_mod_install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway + +# # protoc-gen-swagger is used to build swagger.json +go_mod_install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger + +# k8s tools to codegen .proto files, client libraries, and helpers from types.go +go_mod_install k8s.io/code-generator/cmd/go-to-protobuf +go_mod_install k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo +go_mod_install k8s.io/code-generator/cmd/client-gen +go_mod_install k8s.io/code-generator/cmd/deepcopy-gen +go_mod_install k8s.io/code-generator/cmd/defaulter-gen +go_mod_install k8s.io/code-generator/cmd/informer-gen +go_mod_install k8s.io/code-generator/cmd/lister-gen + +# We still install openapi-gen from go.mod since upstream does not utilize release tags +go_mod_install k8s.io/kube-openapi/cmd/openapi-gen + +# controller-gen is run by ./hack/gen-crd-spec to generate the CRDs +go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1 + +# swagger cli is used to generate swagger docs +go install github.com/go-swagger/go-swagger/cmd/swagger@v0.28.0 + +# goimports is used to auto-format generated code +go install golang.org/x/tools/cmd/goimports@v0.1.8 diff --git a/hack/installers/install-codegen-tools.sh b/hack/installers/install-codegen-tools.sh index 46bcec2be6..a0e3d14ec4 100755 --- a/hack/installers/install-codegen-tools.sh +++ b/hack/installers/install-codegen-tools.sh @@ -1,4 +1,4 @@ #!/bin/bash set -eux -o pipefail -KUSTOMIZE_VERSION=4.2.0 "$(dirname $0)/../install.sh" helm2-linux jq-linux kustomize-linux protoc-linux swagger-linux +KUSTOMIZE_VERSION=4.2.0 "$(dirname $0)/../install.sh" helm2-linux jq-linux kustomize-linux protoc-linux diff --git a/hack/installers/install-dep-linux.sh b/hack/installers/install-dep-linux.sh deleted file mode 100755 index 989cb47e83..0000000000 --- a/hack/installers/install-dep-linux.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -eux -o pipefail - -[ -e $DOWNLOADS/dep ] || curl -sLf --retry 3 -o $DOWNLOADS/dep https://github.com/golang/dep/releases/download/v0.5.3/dep-linux-$ARCHITECTURE -cp $DOWNLOADS/dep $BIN/ -chmod +x $BIN/dep -dep version diff --git a/hack/installers/install-swagger-linux.sh b/hack/installers/install-swagger-linux.sh deleted file mode 100755 index 603f8a26d6..0000000000 --- a/hack/installers/install-swagger-linux.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -eux -o pipefail - -. $(dirname $0)/../tool-versions.sh - -export TARGET_FILE=swagger_linux_${ARCHITECTURE}_${swagger_version} -[ -e $DOWNLOADS/${TARGET_FILE} ] || curl -sLf --retry 3 -o $DOWNLOADS/${TARGET_FILE} https://github.com/go-swagger/go-swagger/releases/download/v${swagger_version}/swagger_linux_$ARCHITECTURE -$(dirname $0)/compare-chksum.sh -sudo install -m 0755 $DOWNLOADS/${TARGET_FILE} $BIN/swagger -swagger version diff --git a/hack/tool-versions.sh b/hack/tool-versions.sh index 40c4b6e162..d1c455acea 100644 --- a/hack/tool-versions.sh +++ b/hack/tool-versions.sh @@ -16,4 +16,3 @@ kubectl_version=1.17.8 kubectx_version=0.6.3 kustomize4_version=4.2.0 protoc_version=3.7.1 -swagger_version=0.19.0 diff --git a/hack/tools.go b/hack/tools.go index a3d9f228e2..63c05678ec 100644 --- a/hack/tools.go +++ b/hack/tools.go @@ -1,19 +1,28 @@ +//go:build tools // +build tools package tools import ( + // gogo/protobuf is vendored because the generated *.pb.go code imports it. + // Also, we need the gogo/protobuf/gogoproto/gogo.proto file _ "github.com/gogo/protobuf/protoc-gen-gogofast" + + // grpc-ecosystem/grpc-gateway is vendored because the generated *.pb.gw.go code imports it. + // Also, we need the .proto files under grpc-gateway/third_party/googleapis _ "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway" _ "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger" + + // k8s.io/code-generator is vendored to get generate-groups.sh, and k8s codegen utilities + _ "k8s.io/code-generator" _ "k8s.io/code-generator/cmd/client-gen" _ "k8s.io/code-generator/cmd/deepcopy-gen" _ "k8s.io/code-generator/cmd/defaulter-gen" - _ "k8s.io/code-generator/cmd/informer-gen" - _ "k8s.io/code-generator/cmd/lister-gen" _ "k8s.io/code-generator/cmd/go-to-protobuf" _ "k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo" - _ "k8s.io/code-generator/pkg/util" - _ "k8s.io/code-generator/third_party/forked/golang/reflect" + _ "k8s.io/code-generator/cmd/informer-gen" + _ "k8s.io/code-generator/cmd/lister-gen" + + // openapi-gen is vendored because upstream does not have tagged releases _ "k8s.io/kube-openapi/cmd/openapi-gen" ) diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index 1c44b8b830..2d2e0e1270 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -19,25 +19,14 @@ set -o errexit set -o nounset set -o pipefail -SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. -. ${SCRIPT_ROOT}/hack/versions.sh -CODEGEN_PKG=$GOPATH/pkg/mod/k8s.io/code-generator@${kube_version} +PROJECT_ROOT=$(cd $(dirname ${BASH_SOURCE})/..; pwd) + TARGET_SCRIPT=/tmp/generate-groups.sh - -( - cd $CODEGEN_PKG - go install ./cmd/{defaulter-gen,client-gen,lister-gen,informer-gen,deepcopy-gen} -) - -export GO111MODULE=off - -sed -e '/go install/d' ${CODEGEN_PKG}/generate-groups.sh > ${TARGET_SCRIPT} - -export GO111MODULE=on +sed -e '/go install/d' ${PROJECT_ROOT}/vendor/k8s.io/code-generator/generate-groups.sh > ${TARGET_SCRIPT} [ -e ./v2 ] || ln -s . v2 -bash -x ${TARGET_SCRIPT} "deepcopy,client,informer,lister" \ +GOBIN=${PROJECT_ROOT}/dist bash -x ${TARGET_SCRIPT} "deepcopy,client,informer,lister" \ github.com/argoproj/argo-cd/v2/pkg/client github.com/argoproj/argo-cd/v2/pkg/apis \ "application:v1alpha1" \ - --go-header-file ${SCRIPT_ROOT}/hack/custom-boilerplate.go.txt + --go-header-file ${PROJECT_ROOT}/hack/custom-boilerplate.go.txt [ -e ./v2 ] && rm -rf v2 \ No newline at end of file diff --git a/hack/update-openapi.sh b/hack/update-openapi.sh index b22531c48c..21b983e92f 100755 --- a/hack/update-openapi.sh +++ b/hack/update-openapi.sh @@ -6,14 +6,9 @@ set -o nounset set -o pipefail PROJECT_ROOT=$(cd $(dirname "$0")/.. ; pwd) -CODEGEN_PKG=${PROJECT_ROOT}/vendor/k8s.io/kube-openapi +PATH="${PROJECT_ROOT}/dist:${PATH}" VERSION="v1alpha1" -export GO111MODULE=off -go build -o dist/openapi-gen ${CODEGEN_PKG}/cmd/openapi-gen - -export GO111MODULE=on - [ -e ./v2 ] || ln -s . v2 ./dist/openapi-gen \ --go-header-file ${PROJECT_ROOT}/hack/custom-boilerplate.go.txt \ diff --git a/hack/versions.sh b/hack/versions.sh deleted file mode 100644 index d97392b640..0000000000 --- a/hack/versions.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -# Update required versions of dependencies here whenever you change them in -# go.mod -kube_version=v0.22.2 -grpc_version=v1.26.0 -protobuf_version=v1.3.2 -grpc_gateway_version=v1.16.0 diff --git a/test/container/Dockerfile b/test/container/Dockerfile index 6f0ca5e35f..066c35bbde 100644 --- a/test/container/Dockerfile +++ b/test/container/Dockerfile @@ -38,8 +38,7 @@ ADD ./hack/tool-versions.sh . ADD ./hack/install.sh . ADD ./hack/installers installers -RUN ./install.sh dep-linux && \ - ./install.sh ksonnet-linux && \ +RUN ./install.sh ksonnet-linux && \ ./install.sh helm2-linux && \ ./install.sh helm-linux && \ ./install.sh kubectl-linux && \