ci: Add pyutils container

Currently, our jobs which only want to run a little bit of python -
python-test and the LAVA jobs - pull the entire x86_64-build image,
which is both massive, and massively unnecessary.

Create a separate image which only carries what we need to run our
Python tests and utilities, and switch python-test to using that.

Signed-off-by: Daniel Stone <daniels@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/31151>

.gitlab-ci/build/gitlab-ci.yml

@@ -80,6 +80,8 @@
     optional: true
   - job: windows-msvc
     optional: true
+  - job: python-test
+    optional: true
 
 
 debian-testing:
@@ -754,3 +756,18 @@ debian-ppc64el:
     VULKAN_DRIVERS: "amd,swrast"
     DRI_LOADERS:
       -D glvnd=disabled
+
+# This job tests our Python scripts, and also emits our scripts into
+# artifacts, so they can be reused for job submission to hardware devices.
+python-test:
+  stage: build-for-tests 
+  extends:
+    - .use-debian/x86_64_pyutils
+    - .build-common
+  variables:
+    GIT_STRATEGY: fetch
+    S3_ARTIFACT_NAME: mesa-python-test
+  timeout: 10m
+  script:
+    - .gitlab-ci/run-pytest.sh
+    - .gitlab-ci/prepare-artifacts-python.sh

.gitlab-ci/container/debian/x86_64_pyutils.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2086 # we want word splitting
+
+# When changing this file, you need to bump the following
+# .gitlab-ci/image-tags.yml tags:
+# DEBIAN_PYUTILS_TAG
+
+set -e
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get install -y ca-certificates
+sed -i -e 's/http:\/\/deb/https:\/\/deb/g' /etc/apt/sources.list.d/*
+echo "deb [trusted=yes] https://gitlab.freedesktop.org/gfx-ci/ci-deb-repo/-/raw/${PKG_REPO_REV}/ ${FDO_DISTRIBUTION_VERSION%-*} main" | tee /etc/apt/sources.list.d/gfx-ci_.list
+
+# Ephemeral packages (installed for this script and removed again at
+# the end)
+EPHEMERAL=(
+    binutils
+    build-essential
+    cpp
+    dpkg-dev
+    g++
+    gcc
+    libc6-dev
+    python3-dev
+)
+
+DEPS=(
+    apt-utils
+    curl
+    file
+    findutils
+    git
+    python3-mako
+    python3-pil
+    python3-pip
+    python3-ply
+    python3-setuptools
+    python3-venv
+    python3-yaml
+    shellcheck
+    xz-utils
+    yamllint
+    zstd
+)
+
+apt-get update
+
+apt-get install -y --no-remove --no-install-recommends "${DEPS[@]}" "${EPHEMERAL[@]}" \
+        "${EXTRA_LOCAL_PACKAGES:-}"
+
+# Needed for ci-fairy, this revision is able to upload files to S3
+pip3 install --break-system-packages git+http://gitlab.freedesktop.org/freedesktop/ci-templates@ffe4d1b10aab7534489f0c4bbc4c5899df17d3f2
+
+pip3 install --break-system-packages -r bin/ci/test/requirements.txt
+
+############### Uninstall ephemeral packages
+
+apt-get purge -y "${EPHEMERAL[@]}"
+apt-get autoremove --purge -y
+
+. .gitlab-ci/container/container_post_build.sh

.gitlab-ci/container/gitlab-ci.yml

@@ -72,10 +72,6 @@ debian/x86_64_build-base:
     - .debian-container
   variables:
     MESA_IMAGE_TAG: &debian-x86_64_build-base "${DEBIAN_BASE_TAG}--${PKG_REPO_REV}"
-  rules:
-    # python-test requires debian/x86_64_build, which requires this job
-    - !reference [python-test, rules]
-    - !reference [.container, rules]
 
 .use-debian/x86_64_build-base:
   extends:
@@ -95,10 +91,6 @@ debian/x86_64_build:
     - .use-debian/x86_64_build-base
   variables:
     MESA_IMAGE_TAG: &debian-x86_64_build ${DEBIAN_BUILD_TAG}
-  rules:
-    # python-test requires this job
-    - !reference [python-test, rules]
-    - !reference [.use-debian/x86_64_build-base, rules]
 
 .use-debian/x86_64_build:
   extends:
@@ -269,6 +261,28 @@ debian/x86_64_test-vk:
   needs:
     - debian/x86_64_test-android
 
+# Debian-based x86_64 image to run Python utilities
+debian/x86_64_pyutils:
+  extends:
+    - .debian-container
+  variables:
+    MESA_IMAGE_TAG: &debian-x86_64_pyutils "${DEBIAN_PYUTILS_TAG}"
+  rules:
+    # python-test requires this job
+    - !reference [python-test, rules]
+    - !reference [.container, rules]
+
+.use-debian/x86_64_pyutils:
+  extends:
+    - .fdo.container-build@debian
+    - .debian-container-version
+    - .set-image
+  variables:
+    MESA_IMAGE_PATH: ${DEBIAN_PYUTILS_IMAGE}
+    MESA_IMAGE_TAG: *debian-x86_64_pyutils
+  needs:
+    - debian/x86_64_pyutils
+
 # Debian based aarch64 test image for GL
 debian/arm64_test-gl:
   tags:

.gitlab-ci/image-tags.yml

@@ -32,6 +32,9 @@ variables:
    DEBIAN_TEST_VK_TAG: "20240909-deqp-oom"
    KERNEL_ROOTFS_TAG: "20240909-pytest-deqp"
 
+   DEBIAN_PYUTILS_IMAGE: "debian/x86_64_pyutils"
+   DEBIAN_PYUTILS_TAG: "20240912-hello-to-you"
+
    ALPINE_X86_64_BUILD_TAG: "20240814-var"
    ALPINE_X86_64_LAVA_SSH_TAG: "20240401-wlproto"
    FEDORA_X86_64_BUILD_TAG: "20240717-pyyaml"

.gitlab-ci/prepare-artifacts-python.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2038 # TODO: rewrite the find
+# shellcheck disable=SC2086 # we want word splitting
+
+section_switch prepare-artifacts "artifacts: prepare"
+
+set -e
+set -o xtrace
+
+mkdir -p artifacts/
+
+# Test runs don't pull down the git tree, so put the dEQP helper
+# script and associated bits there.
+echo "$(cat VERSION) (git-$(git rev-parse HEAD | cut -b -10))" > artifacts/VERSION
+cp -Rp .gitlab-ci/report-flakes.py artifacts/
+cp -Rp .gitlab-ci/setup-test-env.sh artifacts/
+cp -Rp .gitlab-ci/common artifacts/ci-common
+cp -Rp .gitlab-ci/b2c artifacts/
+cp -Rp .gitlab-ci/bare-metal artifacts/
+cp -Rp .gitlab-ci/lava artifacts/
+cp -Rp .gitlab-ci/bin/*_logger.py artifacts/
+
+mapfile -t duplicate_files < <(
+  find src/ -path '*/ci/*' \
+    \( \
+      -name '*.txt' \
+      -o -name '*.toml' \
+      -o -name '*traces*.yml' \
+    \) \
+    -exec basename -a {} + | sort | uniq -d
+)
+if [ ${#duplicate_files[@]} -gt 0 ]; then
+  echo 'Several files with the same name in various ci/ folders:'
+  printf -- '  %s\n' "${duplicate_files[@]}"
+  exit 1
+fi
+
+find src/ -path '*/ci/*' \
+  \( \
+    -name '*.txt' \
+    -o -name '*.toml' \
+    -o -name '*traces*.yml' \
+  \) \
+  -exec cp -p {} artifacts/ \;
+cp -Rp .gitlab-ci/*.txt artifacts/
+
+if [ -n "$S3_ARTIFACT_NAME" ]; then
+    # Pass needed files to the test stage
+    S3_ARTIFACT_TAR="$S3_ARTIFACT_NAME.tar.zst"
+    tar cv artifacts/ | zstd -o "${S3_ARTIFACT_TAR}"
+    ci-fairy s3cp --token-file "${S3_JWT_FILE}" "${S3_ARTIFACT_TAR}" "https://${PIPELINE_ARTIFACTS_BASE}/${S3_ARTIFACT_TAR}"
+    rm "${S3_ARTIFACT_TAR}"
+fi
+
+section_end prepare-artifacts

.gitlab-ci/test/gitlab-ci.yml

@@ -42,33 +42,6 @@ rustfmt:
     - rustfmt --version
     - rustfmt --verbose src/**/lib.rs
 
-python-test:
-  # Cancel job if a newer commit is pushed to the same branch
-  interruptible: true
-  stage: code-validation
-  extends:
-    - .use-debian/x86_64_build
-  variables:
-    GIT_STRATEGY: fetch
-  timeout: 10m
-  script:
-    - .gitlab-ci/run-pytest.sh
-  rules:
-    - !reference [.disable-farm-mr-rules, rules]
-    - !reference [.never-post-merge-rules, rules]
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "push" && $CI_PROJECT_NAMESPACE == "mesa" && $GITLAB_USER_LOGIN != "marge-bot"
-      when: on_success
-    - if: $GITLAB_USER_LOGIN == "marge-bot"
-      changes: &bin_ci_files
-        - .gitlab-ci.yml
-        - .gitlab-ci/**/*
-        - bin/ci/**/*
-      when: on_success
-    - changes: *bin_ci_files
-      when: manual
-
 yaml-toml-shell-test:
   extends:
     - python-test