From 7101aecc53f49402b725a8081f7005c15839cb43 Mon Sep 17 00:00:00 2001 From: Guilherme Gallo Date: Wed, 24 Apr 2024 16:40:18 -0300 Subject: [PATCH] ci: Use id_tokens for JWT auth Fixes: #9180 Signed-off-by: Guilherme Gallo Part-of: --- .gitlab-ci.yml | 15 +++++++-------- .gitlab-ci/bare-metal/rootfs-setup.sh | 2 +- .gitlab-ci/common/generate-env.sh | 2 +- .gitlab-ci/common/init-stage2.sh | 2 +- .gitlab-ci/container/lava_build.sh | 4 ++-- .gitlab-ci/lava/lava-submit.sh | 4 ++-- .gitlab-ci/lava/utils/lava_job_definition.py | 2 +- .gitlab-ci/piglit/piglit-traces.sh | 2 +- .gitlab-ci/prepare-artifacts.sh | 2 +- .gitlab-ci/test/gitlab-ci.yml | 2 +- src/amd/ci/gitlab-ci.yml | 2 +- src/freedreno/ci/gitlab-ci.yml | 2 +- src/gallium/drivers/zink/ci/gitlab-ci.yml | 2 +- 13 files changed, 21 insertions(+), 22 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 815cad9687f..63a6c05b103 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -72,7 +72,7 @@ variables: bash download-git-cache.sh rm download-git-cache.sh set +o xtrace - CI_JOB_JWT_FILE: /minio_jwt + S3_JWT_FILE: /s3_jwt S3_HOST: s3.freedesktop.org # per-pipeline artifact storage on MinIO PIPELINE_ARTIFACTS_BASE: ${S3_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID} @@ -101,8 +101,8 @@ default: export SCRIPTS_DIR=$(mktemp -d) && curl -L -s --retry 4 -f --retry-all-errors --retry-delay 60 -O --output-dir "${SCRIPTS_DIR}" "${CI_PROJECT_URL}/-/raw/${CI_COMMIT_SHA}/.gitlab-ci/setup-test-env.sh" && . ${SCRIPTS_DIR}/setup-test-env.sh && - echo -n "${CI_JOB_JWT}" > "${CI_JOB_JWT_FILE}" && - unset CI_JOB_JWT # Unsetting vulnerable env variables + echo -n "${S3_JWT}" > "${S3_JWT_FILE}" && + unset CI_JOB_JWT S3_JWT # Unsetting vulnerable env variables after_script: # Work around https://gitlab.com/gitlab-org/gitlab/-/issues/20338 @@ -111,9 +111,9 @@ default: - > set +x - test -e "${CI_JOB_JWT_FILE}" && - export CI_JOB_JWT="$(<${CI_JOB_JWT_FILE})" && - rm "${CI_JOB_JWT_FILE}" + test -e "${S3_JWT_FILE}" && + export S3_JWT="$(<${S3_JWT_FILE})" && + rm "${S3_JWT_FILE}" # Retry when job fails. Failed jobs can be found in the Mesa CI Daily Reports: # https://gitlab.freedesktop.org/mesa/mesa/-/issues/?sort=created_date&state=opened&label_name%5B%5D=CI%20daily @@ -266,8 +266,7 @@ make git archive: # compress the current folder - tar -cvzf ../$CI_PROJECT_NAME.tar.gz . - - ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ../$CI_PROJECT_NAME.tar.gz https://$S3_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz - + - ci-fairy s3cp --token-file "${S3_JWT_FILE}" ../$CI_PROJECT_NAME.tar.gz https://$S3_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz # Sanity checks of MR settings and commit logs sanity: diff --git a/.gitlab-ci/bare-metal/rootfs-setup.sh b/.gitlab-ci/bare-metal/rootfs-setup.sh index 6d33dd0a249..882ddb964c2 100644 --- a/.gitlab-ci/bare-metal/rootfs-setup.sh +++ b/.gitlab-ci/bare-metal/rootfs-setup.sh @@ -13,7 +13,7 @@ date +'%F %T' # Make JWT token available as file in the bare-metal storage to enable access # to MinIO -cp "${CI_JOB_JWT_FILE}" "${rootfs_dst}${CI_JOB_JWT_FILE}" +cp "${S3_JWT_FILE}" "${rootfs_dst}${S3_JWT_FILE}" date +'%F %T' diff --git a/.gitlab-ci/common/generate-env.sh b/.gitlab-ci/common/generate-env.sh index 1c2a110d73d..fe23365d516 100755 --- a/.gitlab-ci/common/generate-env.sh +++ b/.gitlab-ci/common/generate-env.sh @@ -10,7 +10,7 @@ VARS=( CI_COMMIT_REF_NAME CI_COMMIT_TITLE CI_JOB_ID - CI_JOB_JWT_FILE + S3_JWT_FILE CI_JOB_STARTED_AT CI_JOB_NAME CI_JOB_URL diff --git a/.gitlab-ci/common/init-stage2.sh b/.gitlab-ci/common/init-stage2.sh index 8e65ec63906..ca41530fb94 100755 --- a/.gitlab-ci/common/init-stage2.sh +++ b/.gitlab-ci/common/init-stage2.sh @@ -217,7 +217,7 @@ cleanup # upload artifacts if [ -n "$S3_RESULTS_UPLOAD" ]; then tar --zstd -cf results.tar.zst results/; - ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" results.tar.zst https://"$S3_RESULTS_UPLOAD"/results.tar.zst; + ci-fairy s3cp --token-file "${S3_JWT_FILE}" results.tar.zst https://"$S3_RESULTS_UPLOAD"/results.tar.zst; fi # We still need to echo the hwci: mesa message, as some scripts rely on it, such diff --git a/.gitlab-ci/container/lava_build.sh b/.gitlab-ci/container/lava_build.sh index d082cd63efd..474a271da2c 100755 --- a/.gitlab-ci/container/lava_build.sh +++ b/.gitlab-ci/container/lava_build.sh @@ -365,8 +365,8 @@ popd . .gitlab-ci/container/container_post_build.sh -ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/"${ROOTFSTAR}" \ +ci-fairy s3cp --token-file "${S3_JWT_FILE}" /lava-files/"${ROOTFSTAR}" \ https://${S3_PATH}/"${ROOTFSTAR}" touch /lava-files/done -ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/done https://${S3_PATH}/done +ci-fairy s3cp --token-file "${S3_JWT_FILE}" /lava-files/done https://${S3_PATH}/done diff --git a/.gitlab-ci/lava/lava-submit.sh b/.gitlab-ci/lava/lava-submit.sh index 41bdc86ac66..3531437f7dd 100755 --- a/.gitlab-ci/lava/lava-submit.sh +++ b/.gitlab-ci/lava/lava-submit.sh @@ -30,7 +30,7 @@ artifacts/ci-common/generate-env.sh | tee results/job-rootfs-overlay/set-job-env section_end variables tar zcf job-rootfs-overlay.tar.gz -C results/job-rootfs-overlay/ . -ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" job-rootfs-overlay.tar.gz "https://${JOB_ROOTFS_OVERLAY_PATH}" +ci-fairy s3cp --token-file "${S3_JWT_FILE}" job-rootfs-overlay.tar.gz "https://${JOB_ROOTFS_OVERLAY_PATH}" ARTIFACT_URL="${FDO_HTTP_CACHE_URI:-}https://${PIPELINE_ARTIFACTS_BASE}/${S3_ARTIFACT_NAME:?}.tar.zst" @@ -50,7 +50,7 @@ PYTHONPATH=artifacts/ artifacts/lava/lava_job_submitter.py \ --ci-project-dir "${CI_PROJECT_DIR}" \ --device-type "${DEVICE_TYPE}" \ --dtb-filename "${DTB}" \ - --jwt-file "${CI_JOB_JWT_FILE}" \ + --jwt-file "${S3_JWT_FILE}" \ --kernel-image-name "${KERNEL_IMAGE_NAME}" \ --kernel-image-type "${KERNEL_IMAGE_TYPE}" \ --boot-method "${BOOT_METHOD}" \ diff --git a/.gitlab-ci/lava/utils/lava_job_definition.py b/.gitlab-ci/lava/utils/lava_job_definition.py index 1a75df0b5af..1227297d064 100644 --- a/.gitlab-ci/lava/utils/lava_job_definition.py +++ b/.gitlab-ci/lava/utils/lava_job_definition.py @@ -193,7 +193,7 @@ class LAVAJobDefinition: "set +x # HIDE_START", f'echo -n "{jwt_file.read()}" > "{self.job_submitter.jwt_file}"', "set -x # HIDE_END", - f'echo "export CI_JOB_JWT_FILE={self.job_submitter.jwt_file}" >> /set-job-env-vars.sh', + f'echo "export S3_JWT_FILE={self.job_submitter.jwt_file}" >> /set-job-env-vars.sh', ] else: download_steps += [ diff --git a/.gitlab-ci/piglit/piglit-traces.sh b/.gitlab-ci/piglit/piglit-traces.sh index 6ea4872db90..18a4b543eb5 100755 --- a/.gitlab-ci/piglit/piglit-traces.sh +++ b/.gitlab-ci/piglit/piglit-traces.sh @@ -8,7 +8,7 @@ set -ex export PAGER=cat # FIXME: export everywhere INSTALL=$(realpath -s "$PWD"/install) -S3_ARGS="--token-file ${CI_JOB_JWT_FILE}" +S3_ARGS="--token-file ${S3_JWT_FILE}" RESULTS=$(realpath -s "$PWD"/results) mkdir -p "$RESULTS" diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh index a58cf705535..2709c03f988 100755 --- a/.gitlab-ci/prepare-artifacts.sh +++ b/.gitlab-ci/prepare-artifacts.sh @@ -60,7 +60,7 @@ if [ -n "$S3_ARTIFACT_NAME" ]; then # Pass needed files to the test stage S3_ARTIFACT_NAME="$S3_ARTIFACT_NAME.tar.zst" zstd artifacts/install.tar -o ${S3_ARTIFACT_NAME} - ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ${S3_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${S3_ARTIFACT_NAME} + ci-fairy s3cp --token-file "${S3_JWT_FILE}" ${S3_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${S3_ARTIFACT_NAME} fi section_end prepare-artifacts diff --git a/.gitlab-ci/test/gitlab-ci.yml b/.gitlab-ci/test/gitlab-ci.yml index 9d10d36b408..bb4dd92cc8b 100644 --- a/.gitlab-ci/test/gitlab-ci.yml +++ b/.gitlab-ci/test/gitlab-ci.yml @@ -158,7 +158,7 @@ python-test: exclude: - results/*.shader_cache variables: - PIGLIT_REPLAY_EXTRA_ARGS: --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_bucket=mesa-tracie-public --jwt-file=${CI_JOB_JWT_FILE} + PIGLIT_REPLAY_EXTRA_ARGS: --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_bucket=mesa-tracie-public --jwt-file=${S3_JWT_FILE} # until we overcome Infrastructure issues, give traces extra 5 min before timeout DEVICE_HANGING_TIMEOUT_SEC: 600 script: diff --git a/src/amd/ci/gitlab-ci.yml b/src/amd/ci/gitlab-ci.yml index ead3bc4d368..dcf8cd93052 100644 --- a/src/amd/ci/gitlab-ci.yml +++ b/src/amd/ci/gitlab-ci.yml @@ -89,7 +89,7 @@ radv-raven-traces-restricted:x86_64: PIGLIT_REPLAY_ANGLE_TAG: "2023-02-10-1" PIGLIT_TRACES_FILE: restricted-traces-amd.yml PIGLIT_REPLAY_DEVICE_NAME: "vk-${GPU_VERSION}" - PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --minio_bucket=mesa-tracie-private --jwt-file=${CI_JOB_JWT_FILE} + PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --minio_bucket=mesa-tracie-private --jwt-file=${S3_JWT_FILE} FDO_CI_CONCURRENT: 10 radeonsi-raven-piglit-quick_gl:x86_64: diff --git a/src/freedreno/ci/gitlab-ci.yml b/src/freedreno/ci/gitlab-ci.yml index 9345c9e9fd2..99c9ad12ea6 100644 --- a/src/freedreno/ci/gitlab-ci.yml +++ b/src/freedreno/ci/gitlab-ci.yml @@ -268,7 +268,7 @@ a630-traces-restricted: - .google-freedreno-rules-restricted variables: PIGLIT_TRACES_FILE: restricted-traces-freedreno.yml - PIGLIT_REPLAY_EXTRA_ARGS: "--download-caching-proxy-url=http://10.42.0.1:8888/cache/?uri= --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_bucket=mesa-tracie-private --jwt-file=${CI_JOB_JWT_FILE}" + PIGLIT_REPLAY_EXTRA_ARGS: "--download-caching-proxy-url=http://10.42.0.1:8888/cache/?uri= --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_bucket=mesa-tracie-private --jwt-file=${S3_JWT_FILE}" allow_failure: true a630-traces-performance: diff --git a/src/gallium/drivers/zink/ci/gitlab-ci.yml b/src/gallium/drivers/zink/ci/gitlab-ci.yml index 139853dd6f9..e775e71b758 100644 --- a/src/gallium/drivers/zink/ci/gitlab-ci.yml +++ b/src/gallium/drivers/zink/ci/gitlab-ci.yml @@ -78,7 +78,7 @@ zink-anv-tgl-traces-restricted: - .zink-anv-rules-restricted variables: PIGLIT_TRACES_FILE: traces-zink-restricted.yml - PIGLIT_REPLAY_EXTRA_ARGS: --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_bucket=mesa-tracie-private --jwt-file=${CI_JOB_JWT_FILE} + PIGLIT_REPLAY_EXTRA_ARGS: --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_bucket=mesa-tracie-private --jwt-file=${S3_JWT_FILE} allow_failure: true zink-tu-a618: