diff --git a/.gitlab-ci/build/gitlab-ci.yml b/.gitlab-ci/build/gitlab-ci.yml index ad29b369e8b..729a1968287 100644 --- a/.gitlab-ci/build/gitlab-ci.yml +++ b/.gitlab-ci/build/gitlab-ci.yml @@ -80,6 +80,8 @@ optional: true - job: windows-msvc optional: true + - job: python-test + optional: true debian-testing: @@ -754,3 +756,18 @@ debian-ppc64el: VULKAN_DRIVERS: "amd,swrast" DRI_LOADERS: -D glvnd=disabled + +# This job tests our Python scripts, and also emits our scripts into +# artifacts, so they can be reused for job submission to hardware devices. +python-test: + stage: build-for-tests + extends: + - .use-debian/x86_64_pyutils + - .build-common + variables: + GIT_STRATEGY: fetch + S3_ARTIFACT_NAME: mesa-python-test + timeout: 10m + script: + - .gitlab-ci/run-pytest.sh + - .gitlab-ci/prepare-artifacts-python.sh diff --git a/.gitlab-ci/container/debian/x86_64_pyutils.sh b/.gitlab-ci/container/debian/x86_64_pyutils.sh new file mode 100644 index 00000000000..156daabd411 --- /dev/null +++ b/.gitlab-ci/container/debian/x86_64_pyutils.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2086 # we want word splitting + +# When changing this file, you need to bump the following +# .gitlab-ci/image-tags.yml tags: +# DEBIAN_PYUTILS_TAG + +set -e +set -o xtrace + +export DEBIAN_FRONTEND=noninteractive + +apt-get install -y ca-certificates +sed -i -e 's/http:\/\/deb/https:\/\/deb/g' /etc/apt/sources.list.d/* +echo "deb [trusted=yes] https://gitlab.freedesktop.org/gfx-ci/ci-deb-repo/-/raw/${PKG_REPO_REV}/ ${FDO_DISTRIBUTION_VERSION%-*} main" | tee /etc/apt/sources.list.d/gfx-ci_.list + +# Ephemeral packages (installed for this script and removed again at +# the end) +EPHEMERAL=( + binutils + build-essential + cpp + dpkg-dev + g++ + gcc + libc6-dev + python3-dev +) + +DEPS=( + apt-utils + curl + file + findutils + git + python3-mako + python3-pil + python3-pip + python3-ply + python3-setuptools + python3-venv + python3-yaml + shellcheck + xz-utils + yamllint + zstd +) + +apt-get update + +apt-get install -y --no-remove --no-install-recommends "${DEPS[@]}" "${EPHEMERAL[@]}" \ + "${EXTRA_LOCAL_PACKAGES:-}" + +# Needed for ci-fairy, this revision is able to upload files to S3 +pip3 install --break-system-packages git+http://gitlab.freedesktop.org/freedesktop/ci-templates@ffe4d1b10aab7534489f0c4bbc4c5899df17d3f2 + +pip3 install --break-system-packages -r bin/ci/test/requirements.txt + +############### Uninstall ephemeral packages + +apt-get purge -y "${EPHEMERAL[@]}" +apt-get autoremove --purge -y + +. .gitlab-ci/container/container_post_build.sh diff --git a/.gitlab-ci/container/gitlab-ci.yml b/.gitlab-ci/container/gitlab-ci.yml index 7d40013fc89..17188fc9cc1 100644 --- a/.gitlab-ci/container/gitlab-ci.yml +++ b/.gitlab-ci/container/gitlab-ci.yml @@ -72,10 +72,6 @@ debian/x86_64_build-base: - .debian-container variables: MESA_IMAGE_TAG: &debian-x86_64_build-base "${DEBIAN_BASE_TAG}--${PKG_REPO_REV}" - rules: - # python-test requires debian/x86_64_build, which requires this job - - !reference [python-test, rules] - - !reference [.container, rules] .use-debian/x86_64_build-base: extends: @@ -95,10 +91,6 @@ debian/x86_64_build: - .use-debian/x86_64_build-base variables: MESA_IMAGE_TAG: &debian-x86_64_build ${DEBIAN_BUILD_TAG} - rules: - # python-test requires this job - - !reference [python-test, rules] - - !reference [.use-debian/x86_64_build-base, rules] .use-debian/x86_64_build: extends: @@ -269,6 +261,28 @@ debian/x86_64_test-vk: needs: - debian/x86_64_test-android +# Debian-based x86_64 image to run Python utilities +debian/x86_64_pyutils: + extends: + - .debian-container + variables: + MESA_IMAGE_TAG: &debian-x86_64_pyutils "${DEBIAN_PYUTILS_TAG}" + rules: + # python-test requires this job + - !reference [python-test, rules] + - !reference [.container, rules] + +.use-debian/x86_64_pyutils: + extends: + - .fdo.container-build@debian + - .debian-container-version + - .set-image + variables: + MESA_IMAGE_PATH: ${DEBIAN_PYUTILS_IMAGE} + MESA_IMAGE_TAG: *debian-x86_64_pyutils + needs: + - debian/x86_64_pyutils + # Debian based aarch64 test image for GL debian/arm64_test-gl: tags: diff --git a/.gitlab-ci/image-tags.yml b/.gitlab-ci/image-tags.yml index bbb97cf05cd..2bb1a792275 100644 --- a/.gitlab-ci/image-tags.yml +++ b/.gitlab-ci/image-tags.yml @@ -32,6 +32,9 @@ variables: DEBIAN_TEST_VK_TAG: "20240909-deqp-oom" KERNEL_ROOTFS_TAG: "20240909-pytest-deqp" + DEBIAN_PYUTILS_IMAGE: "debian/x86_64_pyutils" + DEBIAN_PYUTILS_TAG: "20240912-hello-to-you" + ALPINE_X86_64_BUILD_TAG: "20240814-var" ALPINE_X86_64_LAVA_SSH_TAG: "20240401-wlproto" FEDORA_X86_64_BUILD_TAG: "20240717-pyyaml" diff --git a/.gitlab-ci/prepare-artifacts-python.sh b/.gitlab-ci/prepare-artifacts-python.sh new file mode 100755 index 00000000000..eb7ee94817d --- /dev/null +++ b/.gitlab-ci/prepare-artifacts-python.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2038 # TODO: rewrite the find +# shellcheck disable=SC2086 # we want word splitting + +section_switch prepare-artifacts "artifacts: prepare" + +set -e +set -o xtrace + +mkdir -p artifacts/ + +# Test runs don't pull down the git tree, so put the dEQP helper +# script and associated bits there. +echo "$(cat VERSION) (git-$(git rev-parse HEAD | cut -b -10))" > artifacts/VERSION +cp -Rp .gitlab-ci/report-flakes.py artifacts/ +cp -Rp .gitlab-ci/setup-test-env.sh artifacts/ +cp -Rp .gitlab-ci/common artifacts/ci-common +cp -Rp .gitlab-ci/b2c artifacts/ +cp -Rp .gitlab-ci/bare-metal artifacts/ +cp -Rp .gitlab-ci/lava artifacts/ +cp -Rp .gitlab-ci/bin/*_logger.py artifacts/ + +mapfile -t duplicate_files < <( + find src/ -path '*/ci/*' \ + \( \ + -name '*.txt' \ + -o -name '*.toml' \ + -o -name '*traces*.yml' \ + \) \ + -exec basename -a {} + | sort | uniq -d +) +if [ ${#duplicate_files[@]} -gt 0 ]; then + echo 'Several files with the same name in various ci/ folders:' + printf -- ' %s\n' "${duplicate_files[@]}" + exit 1 +fi + +find src/ -path '*/ci/*' \ + \( \ + -name '*.txt' \ + -o -name '*.toml' \ + -o -name '*traces*.yml' \ + \) \ + -exec cp -p {} artifacts/ \; +cp -Rp .gitlab-ci/*.txt artifacts/ + +if [ -n "$S3_ARTIFACT_NAME" ]; then + # Pass needed files to the test stage + S3_ARTIFACT_TAR="$S3_ARTIFACT_NAME.tar.zst" + tar cv artifacts/ | zstd -o "${S3_ARTIFACT_TAR}" + ci-fairy s3cp --token-file "${S3_JWT_FILE}" "${S3_ARTIFACT_TAR}" "https://${PIPELINE_ARTIFACTS_BASE}/${S3_ARTIFACT_TAR}" + rm "${S3_ARTIFACT_TAR}" +fi + +section_end prepare-artifacts diff --git a/.gitlab-ci/test/gitlab-ci.yml b/.gitlab-ci/test/gitlab-ci.yml index ed17e04c315..bb20a70012a 100644 --- a/.gitlab-ci/test/gitlab-ci.yml +++ b/.gitlab-ci/test/gitlab-ci.yml @@ -42,33 +42,6 @@ rustfmt: - rustfmt --version - rustfmt --verbose src/**/lib.rs -python-test: - # Cancel job if a newer commit is pushed to the same branch - interruptible: true - stage: code-validation - extends: - - .use-debian/x86_64_build - variables: - GIT_STRATEGY: fetch - timeout: 10m - script: - - .gitlab-ci/run-pytest.sh - rules: - - !reference [.disable-farm-mr-rules, rules] - - !reference [.never-post-merge-rules, rules] - - if: $CI_PIPELINE_SOURCE == "schedule" - when: on_success - - if: $CI_PIPELINE_SOURCE == "push" && $CI_PROJECT_NAMESPACE == "mesa" && $GITLAB_USER_LOGIN != "marge-bot" - when: on_success - - if: $GITLAB_USER_LOGIN == "marge-bot" - changes: &bin_ci_files - - .gitlab-ci.yml - - .gitlab-ci/**/* - - bin/ci/**/* - when: on_success - - changes: *bin_ci_files - when: manual - yaml-toml-shell-test: extends: - python-test