ci: inject gfx-ci/linux S3 artifacts without rebuilding containers

We need update kernel often. We need test kernel changes often.

Introduced `KERNEL_EXTERNAL_TAG` to differ between `KERNEL_TAG` which is
also used to rebuild the containers. We don't need rebuild containers
for the external kernel, so this way we don't have to.

Updating kernel goes wruuuuuum.

Signed-off-by: David Heidelberg <david.heidelberg@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/23563>

.gitlab-ci.yml

@@ -6,20 +6,29 @@ workflow:
     # merge pipeline
     - if: $GITLAB_USER_LOGIN == "marge-bot" && $CI_COMMIT_BRANCH == null
       variables:
+        KERNEL_IMAGE_BASE: https://${S3_HOST}/mesa-lava/${KERNEL_REPO}/${KERNEL_TAG}
         MESA_CI_PERFORMANCE_ENABLED: 1
         VALVE_INFRA_VANGOGH_JOB_PRIORITY: ""  # Empty tags are ignored by gitlab
     # post-merge pipeline
     - if: $GITLAB_USER_LOGIN == "marge-bot" && $CI_COMMIT_BRANCH
       variables:
+        KERNEL_IMAGE_BASE: https://${S3_HOST}/mesa-lava/${KERNEL_REPO}/${KERNEL_TAG}
         JOB_PRIORITY: 40
         VALVE_INFRA_VANGOGH_JOB_PRIORITY: priority:low
     # any other pipeline
-    - if: $GITLAB_USER_LOGIN != "marge-bot"
+    - if: $GITLAB_USER_LOGIN != "marge-bot" && $FORCE_KERNEL_TAG != null
       variables:
+        KERNEL_IMAGE_BASE: https://${S3_HOST}/mesa-lava/${KERNEL_REPO}/${FORCE_KERNEL_TAG}
+        JOB_PRIORITY: 50
+        VALVE_INFRA_VANGOGH_JOB_PRIORITY: priority:low
+    - if: $GITLAB_USER_LOGIN != "marge-bot" && $FORCE_KERNEL_TAG == null
+      variables:
+        KERNEL_IMAGE_BASE: https://${S3_HOST}/mesa-lava/${KERNEL_REPO}/${KERNEL_TAG}
         JOB_PRIORITY: 50
         VALVE_INFRA_VANGOGH_JOB_PRIORITY: priority:low
     - when: always
 
+
 variables:
   FDO_UPSTREAM_REPO: mesa/mesa
   MESA_TEMPLATES_COMMIT: &ci-templates-commit d5aa3941aa03c2f716595116354fb81eb8012acb
@@ -35,7 +44,6 @@ variables:
   PIPELINE_ARTIFACTS_BASE: ${S3_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}
   # per-job artifact storage on MinIO
   JOB_ARTIFACTS_BASE: ${PIPELINE_ARTIFACTS_BASE}/${CI_JOB_ID}
-  KERNEL_IMAGE_BASE: https://${S3_HOST}/mesa-lava/gfx-ci/linux/${KERNEL_TAG}
   # reference images stored for traces
   PIGLIT_REPLAY_REFERENCE_IMAGES_BASE: "${S3_HOST}/mesa-tracie-results/$FDO_UPSTREAM_REPO"
   # For individual CI farm status see .ci-farms folder

.gitlab-ci/bare-metal/cros-servo.sh

@@ -85,6 +85,13 @@ rm -rf /tftp/*
 if echo "$BM_KERNEL" | grep -q http; then
   curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
       $BM_KERNEL -o /tftp/vmlinuz
+elif [ -n "${FORCE_KERNEL_TAG}" ]; then
+  curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
+    "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/${BM_KERNEL}" -o /tftp/vmlinuz
+  curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
+    "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/modules.tar.zst" -o modules.tar.zst
+  tar --keep-directory-symlink --zstd -xf modules.tar.zst -C "/nfs/"
+  rm modules.tar.zst &
 else
   cp /baremetal-files/"$BM_KERNEL" /tftp/vmlinuz
 fi

.gitlab-ci/bare-metal/fastboot.sh

@@ -96,22 +96,30 @@ else
   popd
 fi
 
-# Make the combined kernel image and dtb for passing to fastboot.  For normal
-# Mesa development, we build the kernel and store it in the docker container
-# that this script is running in.
-#
-# However, container builds are expensive, so when you're hacking on the
-# kernel, it's nice to be able to skip the half hour container build and plus
-# moving that container to the runner.  So, if BM_KERNEL+BM_DTB are URLs,
-# fetch them instead of looking in the container.
 if echo "$BM_KERNEL $BM_DTB" | grep -q http; then
   curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
       "$BM_KERNEL" -o kernel
+  # FIXME: modules should be supplied too
   curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
       "$BM_DTB" -o dtb
 
   cat kernel dtb > Image.gz-dtb
+
+elif [ -n "${FORCE_KERNEL_TAG}" ]; then
+  curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
+      "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/${BM_KERNEL}" -o kernel
+  curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
+      "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/modules.tar.zst" -o modules.tar.zst
+
+  if [ -n "$BM_DTB" ]; then
+    curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
+	"${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/${BM_DTB}.dtb" -o dtb
+  fi
+
+  cat kernel dtb > Image.gz-dtb || echo "No DTB available, using pure kernel."
   rm kernel
+  tar --keep-directory-symlink --zstd -xf modules.tar.zst -C "$BM_ROOTFS/"
+  rm modules.tar.zst &
 else
   cat /baremetal-files/"$BM_KERNEL" /baremetal-files/"$BM_DTB".dtb > Image.gz-dtb
   cp /baremetal-files/"$BM_DTB".dtb dtb

.gitlab-ci/bare-metal/poe-powered.sh

@@ -60,8 +60,8 @@ if [ -z "$BM_ROOTFS" ]; then
   exit 1
 fi
 
-if [ -z "$BM_BOOTFS" ]; then
+if [ -z "$BM_BOOTFS" ] && { [ -z "$BM_KERNEL" ] || [ -z "$BM_DTB" ]; } ; then
-  echo "Must set /boot files for the TFTP boot in the job's variables"
+  echo "Must set /boot files for the TFTP boot in the job's variables or set kernel and dtb"
   exit 1
 fi
 
@@ -99,23 +99,50 @@ fi
 date +'%F %T'
 
 # If BM_BOOTFS is a file, assume it is a tarball and uncompress it
-if [ -f $BM_BOOTFS ]; then
+if [ -f "${BM_BOOTFS}" ]; then
   mkdir -p /tmp/bootfs
   tar xf $BM_BOOTFS -C /tmp/bootfs
   BM_BOOTFS=/tmp/bootfs
 fi
 
+# If BM_KERNEL and BM_DTS is present
+if [ -n "${FORCE_KERNEL_TAG}" ]; then
+  if [ -z "${BM_KERNEL}" ] || [ -z "${BM_DTB}" ]; then
+    echo "This machine cannot be tested with external kernel since BM_KERNEL or BM_DTB missing!"
+    exit 1
+  fi
+
+  curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
+      "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/${BM_KERNEL}" -o "${BM_KERNEL}"
+  curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
+      "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/${BM_DTB}.dtb" -o "${BM_DTB}.dtb"
+  curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
+      "${FDO_HTTP_CACHE_URI:-}${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/modules.tar.zst" -o modules.tar.zst
+fi
+
 date +'%F %T'
 
 # Install kernel modules (it could be either in /lib/modules or
 # /usr/lib/modules, but we want to install in the latter)
-[ -d $BM_BOOTFS/usr/lib/modules ] && rsync -a $BM_BOOTFS/usr/lib/modules/ /nfs/usr/lib/modules/
+if [ -n "${FORCE_KERNEL_TAG}" ]; then
-[ -d $BM_BOOTFS/lib/modules ] && rsync -a $BM_BOOTFS/lib/modules/ /nfs/lib/modules/
+  tar --keep-directory-symlink --zstd -xf modules.tar.zst -C /nfs/
+  rm modules.tar.zst &
+elif [ -n "${BM_BOOTFS}" ]; then
+  [ -d $BM_BOOTFS/usr/lib/modules ] && rsync -a $BM_BOOTFS/usr/lib/modules/ /nfs/usr/lib/modules/
+  [ -d $BM_BOOTFS/lib/modules ] && rsync -a $BM_BOOTFS/lib/modules/ /nfs/lib/modules/
+else
+  echo "No modules!"
+fi
+
 
 date +'%F %T'
 
 # Install kernel image + bootloader files
-rsync -aL --delete $BM_BOOTFS/boot/ /tftp/
+if [ -n "${FORCE_KERNEL_TAG}" ] || [ -z "$BM_BOOTFS" ]; then
+  mv "${BM_KERNEL}" "${BM_DTB}.dtb" /tftp/
+else  # BM_BOOTFS
+  rsync -aL --delete $BM_BOOTFS/boot/ /tftp/
+fi
 
 date +'%F %T'
 
@@ -147,7 +174,6 @@ LABEL primary
 EOF
 
 # Create the rootfs in the NFS directory
-mkdir -p /nfs/results
 . $BM/rootfs-setup.sh /nfs
 
 date +'%F %T'

.gitlab-ci/image-tags.yml

@@ -27,6 +27,7 @@ variables:
    FEDORA_X86_64_BUILD_TAG: "2023-10-30-ci-improv"
    KERNEL_ROOTFS_TAG: "2023-11-04-version-log"
    KERNEL_TAG: "v6.4.12-for-mesa-ci-f6b4ad45f48d"
+   KERNEL_REPO: "gfx-ci/linux"
 
    WINDOWS_X64_VS_PATH: "windows/x64_vs"
    WINDOWS_X64_VS_TAG: "2022-10-20-upgrade-zlib"

.gitlab-ci/lava/lava-submit.sh

@@ -42,6 +42,7 @@ PYTHONPATH=artifacts/ artifacts/lava/lava_job_submitter.py \
 	--pipeline-info "$CI_JOB_NAME: $CI_PIPELINE_URL on $CI_COMMIT_REF_NAME ${CI_NODE_INDEX}/${CI_NODE_TOTAL}" \
 	--rootfs-url-prefix "https://${BASE_SYSTEM_HOST_PATH}" \
 	--kernel-url-prefix "${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}" \
+	--kernel-external "${FORCE_KERNEL_TAG}" \
 	--build-url "${ARTIFACT_URL}" \
 	--job-rootfs-overlay-url "${FDO_HTTP_CACHE_URI:-}https://${JOB_ROOTFS_OVERLAY_PATH}" \
 	--job-timeout-min ${JOB_TIMEOUT:-30} \

.gitlab-ci/lava/lava_job_submitter.py

@@ -370,6 +370,7 @@ class LAVAJobSubmitter(PathResolver):
     kernel_image_name: str = None
     kernel_image_type: str = ""
     kernel_url_prefix: str = None
+    kernel_external: str = None
     lava_tags: str = ""  # Comma-separated LAVA tags for the job
     mesa_job_name: str = "mesa_ci_job"
     pipeline_info: str = ""

.gitlab-ci/lava/utils/lava_job_definition.py

@@ -153,6 +153,13 @@ class LAVAJobDefinition:
                 f"{self.job_submitter.dtb_filename}.dtb"
             }
 
+    def attach_external_modules(self, deploy_field):
+        if self.job_submitter.kernel_external:
+            deploy_field["modules"] = {
+                "url": f"{self.job_submitter.kernel_url_prefix}/modules.tar.zst",
+                "compression": "zstd"
+            }
+
     def artifact_download_steps(self):
         """
         This function is responsible for setting up the SSH server in the DUT and to

.gitlab-ci/lava/utils/uart_job_definition.py

@@ -60,6 +60,7 @@ def fastboot_deploy_actions(
     # URLs to our kernel rootfs to boot from, both generated by the base
     # container build
     job_definition.attach_kernel_and_dtb(fastboot_deploy_prepare["images"])
+    job_definition.attach_external_modules(fastboot_deploy_nfs)
 
     return (fastboot_deploy_nfs, fastboot_deploy_prepare, fastboot_deploy)
 
@@ -76,6 +77,7 @@ def tftp_deploy_actions(job_definition: "LAVAJobDefinition", nfsrootfs) -> tuple
         "nfsrootfs": nfsrootfs,
     }
     job_definition.attach_kernel_and_dtb(tftp_deploy)
+    job_definition.attach_external_modules(tftp_deploy)
 
     return (tftp_deploy,)
 

.gitlab-ci/test/gitlab-ci.yml

@@ -280,6 +280,9 @@ clang-format:
   # built as part of the CI in the boot2container project.
   image: registry.freedesktop.org/gfx-ci/ci-tron/mesa-trigger:2023-06-02.1
   timeout: 1h 40m
+  rules:
+    - if: $FORCE_KERNEL_TAG != null
+      when: never
   variables:
     # No need by default to pull the whole repo
     GIT_STRATEGY: none

src/broadcom/ci/gitlab-ci.yml

@@ -20,6 +20,8 @@
 .vc4-rules:
   stage: broadcom
   rules:
+    - if: $FORCE_KERNEL_TAG != null
+      when: never
     - !reference [.never-post-merge-rules, rules]
     - !reference [.igalia-farm-rules, rules]
     - !reference [.gl-rules, rules]
@@ -43,6 +45,8 @@
 .v3d-rules:
   stage: broadcom
   rules:
+    - if: $FORCE_KERNEL_TAG != null
+      when: never
     - !reference [.never-post-merge-rules, rules]
     - !reference [.igalia-farm-rules, rules]
     - !reference [.gl-rules, rules]
@@ -69,6 +73,8 @@
 .v3dv-rules:
   stage: broadcom
   rules:
+    - if: $FORCE_KERNEL_TAG != null
+      when: never
     - !reference [.never-post-merge-rules, rules]
     - !reference [.igalia-farm-rules, rules]
     - !reference [.vulkan-rules, rules]

src/gallium/drivers/llvmpipe/ci/gitlab-ci-inc.yml

@@ -1,6 +1,8 @@
 .llvmpipe-rules:
   stage: software-renderer
   rules:
+    - if: $FORCE_KERNEL_TAG != null
+      when: never
     - !reference [.gl-rules, rules]
     - changes: &llvmpipe_file_list
       - src/gallium/drivers/llvmpipe/**/*

src/gallium/drivers/softpipe/ci/gitlab-ci-inc.yml

@@ -1,6 +1,8 @@
 .softpipe-rules:
   stage: software-renderer
   rules:
+    - if: $FORCE_KERNEL_TAG != null
+      when: never
     - !reference [.gl-rules, rules]
     - changes: &softpipe_file_list
       - src/gallium/drivers/softpipe/**/*

src/gallium/drivers/virgl/ci/gitlab-ci-inc.yml

@@ -1,6 +1,8 @@
 .virgl-rules:
   stage: layered-backends
   rules:
+    - if: $FORCE_KERNEL_TAG != null
+      when: never
     - !reference [.gl-rules, rules]
     - !reference [.llvmpipe-rules, rules]
     - changes: &virgl_file_list

src/gallium/frontends/lavapipe/ci/gitlab-ci-inc.yml

@@ -1,6 +1,8 @@
 .lavapipe-rules:
   stage: software-renderer
   rules:
+    - if: $FORCE_KERNEL_TAG != null
+      when: never
     - !reference [.vulkan-rules, rules]
     - !reference [.gallium-core-rules, rules]
     - changes: &lavapipe_file_list