ci: replace gzip usage with zstd where posible

v2: added missing zstd to arm_build.sh

Reviewed-by: Emma Anholt <emma@anholt.net>
Signed-off-by: David Heidelberg <david.heidelberg@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/17776>

.gitlab-ci/common/init-stage2.sh

@@ -149,9 +149,9 @@ cleanup
 
 # upload artifacts
 if [ -n "$MINIO_RESULTS_UPLOAD" ]; then
-  tar -czf results.tar.gz results/;
+  tar --zstd -cf results.tar.zst results/;
   ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}";
-  ci-fairy minio cp results.tar.gz minio://"$MINIO_RESULTS_UPLOAD"/results.tar.gz;
+  ci-fairy minio cp results.tar.zst minio://"$MINIO_RESULTS_UPLOAD"/results.tar.zst;
 fi
 
 # We still need to echo the hwci: mesa message, as some scripts rely on it, such

.gitlab-ci/container/create-rootfs.sh

@@ -92,7 +92,8 @@ apt-get -y install --no-install-recommends \
     waffle-utils \
     wget \
     xinit \
-    xserver-xorg-core
+    xserver-xorg-core \
+    zstd
 
 # Needed for ci-fairy, this revision is able to upload files to
 # MinIO and doesn't depend on git

.gitlab-ci/container/debian/arm_build.sh

@@ -54,7 +54,8 @@ apt-get -y install \
 	u-boot-tools \
 	wget \
 	xz-utils \
-	zlib1g-dev
+	zlib1g-dev \
+	zstd
 
 # Not available anymore in bullseye
 apt-get install -y --no-remove -t buster \

.gitlab-ci/container/debian/x86_build-base.sh

@@ -73,7 +73,8 @@ apt-get install -y --no-remove \
         x11proto-gl-dev \
         x11proto-randr-dev \
         xz-utils \
-        zlib1g-dev
+        zlib1g-dev \
+	zstd
 
 # Needed for ci-fairy, this revision is able to upload files to MinIO
 pip3 install git+http://gitlab.freedesktop.org/freedesktop/ci-templates@34f4ade99434043f88e164933f570301fd18b125

.gitlab-ci/container/debian/x86_build.sh

@@ -48,7 +48,8 @@ apt-get install -y --no-remove \
       procps \
       spirv-tools \
       strace \
-      time
+      time \
+      zstd
 
 
 . .gitlab-ci/container/container_pre_build.sh

.gitlab-ci/container/debian/x86_test-base.sh

@@ -57,7 +57,8 @@ apt-get install -y --no-remove \
       waffle-utils \
       xauth \
       xvfb \
-      zlib1g
+      zlib1g \
+      zstd
 
 apt-get install -y --no-install-recommends \
       $STABLE_EPHEMERAL

.gitlab-ci/image-tags.yml

@@ -1,21 +1,21 @@
 variables:
    DEBIAN_X86_BUILD_BASE_IMAGE: "debian/x86_build-base"
-   DEBIAN_BASE_TAG: "2022-07-01-bb-llvm13"
+   DEBIAN_BASE_TAG: "2022-08-04-zstd"
 
    DEBIAN_X86_BUILD_IMAGE_PATH: "debian/x86_build"
-   DEBIAN_BUILD_TAG: "2022-07-14-directx-headers"
+   DEBIAN_BUILD_TAG: "2022-08-04-zstd"
 
    DEBIAN_X86_BUILD_MINGW_IMAGE_PATH: "debian/x86_build-mingw"
-   DEBIAN_BUILD_MINGW_TAG: "2022-07-14-directx-headers"
+   DEBIAN_BUILD_MINGW_TAG: "2022-08-04-zstd"
 
    DEBIAN_X86_TEST_BASE_IMAGE: "debian/x86_test-base"
 
    DEBIAN_X86_TEST_IMAGE_PATH: "debian/x86_test-gl"
-   DEBIAN_X86_TEST_GL_TAG: "2022-07-06-virgl-update"
-   DEBIAN_X86_TEST_VK_TAG: "2022-07-18-apitrace-11-1"
+   DEBIAN_X86_TEST_GL_TAG: "2022-08-04-zstd"
+   DEBIAN_X86_TEST_VK_TAG: "2022-08-04-zstd"
 
    FEDORA_X86_BUILD_TAG: "2022-04-24-spirv-tools-5"
-   KERNEL_ROOTFS_TAG: "2022-07-27-rootfs-zstd"
+   KERNEL_ROOTFS_TAG: "2022-08-04-zstd"
 
    WINDOWS_X64_VS_PATH: "windows/x64_vs"
    WINDOWS_X64_VS_TAG: "2022-06-15-vs-winsdk"

.gitlab-ci/lava/lava-gitlab-ci.yml

@@ -12,9 +12,9 @@
     BASE_SYSTEM_MAINLINE_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${FDO_UPSTREAM_REPO}/${DISTRIBUTION_TAG}/${ARCH}"
     BASE_SYSTEM_FORK_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${CI_PROJECT_PATH}/${DISTRIBUTION_TAG}/${ARCH}"
     # per-job build artifacts
-    BUILD_PATH: "${PIPELINE_ARTIFACTS_BASE}/${CI_PROJECT_NAME}-${ARCH}.tar.gz"
+    BUILD_PATH: "${PIPELINE_ARTIFACTS_BASE}/${CI_PROJECT_NAME}-${ARCH}.tar.zst"
     JOB_ROOTFS_OVERLAY_PATH: "${JOB_ARTIFACTS_BASE}/job-rootfs-overlay.tar.gz"
-    JOB_RESULTS_PATH: "${JOB_ARTIFACTS_BASE}/results.tar.gz"
+    JOB_RESULTS_PATH: "${JOB_ARTIFACTS_BASE}/results.tar.zst"
     MINIO_RESULTS_UPLOAD: "${JOB_ARTIFACTS_BASE}"
     PIGLIT_NO_WINDOW: 1
     VISIBILITY_GROUP: "Collabora+fdo"
@@ -30,7 +30,7 @@
   tags:
     - $RUNNER_TAG
   after_script:
-    - wget -q "https://${JOB_RESULTS_PATH}" -O- | tar -xz
+    - wget -q "https://${JOB_RESULTS_PATH}" -O- | tar --zstd -x
 
 .lava-test:armhf:
   variables:

.gitlab-ci/lava/lava_job_submitter.py

@@ -166,7 +166,7 @@ def generate_lava_yaml(args):
 
     run_steps += [
       'mkdir -p {}'.format(args.ci_project_dir),
-      'wget -S --progress=dot:giga -O- {} | tar -xz -C {}'.format(args.build_url, args.ci_project_dir),
+      'wget -S --progress=dot:giga -O- {} | tar --zstd -x -C {}'.format(args.build_url, args.ci_project_dir),
       'wget -S --progress=dot:giga -O- {} | tar -xz -C /'.format(args.job_rootfs_overlay_url),
 
       # Sleep a bit to give time for bash to dump shell xtrace messages into

.gitlab-ci/prepare-artifacts.sh

@@ -52,8 +52,8 @@ cp -Rp .gitlab-ci/b2c artifacts/
 
 if [ -n "$MINIO_ARTIFACT_NAME" ]; then
     # Pass needed files to the test stage
-    MINIO_ARTIFACT_NAME="$MINIO_ARTIFACT_NAME.tar.gz"
-    gzip -c artifacts/install.tar > ${MINIO_ARTIFACT_NAME}
+    MINIO_ARTIFACT_NAME="$MINIO_ARTIFACT_NAME.tar.zst"
+    zstd artifacts/install.tar -o ${MINIO_ARTIFACT_NAME}
     ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"
     ci-fairy minio cp ${MINIO_ARTIFACT_NAME} minio://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME}
 fi

.gitlab-ci/test/gitlab-ci.yml

@@ -124,7 +124,7 @@
     # improve it even more (see https://docs.mesa3d.org/ci/bare-metal.html for
     # setup).
     - echo -e "\e[0Ksection_start:$(date +%s):artifacts_download[collapsed=true]\r\e[0KDownloading artifacts from minio"
-    - wget ${FDO_HTTP_CACHE_URI:-}https://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME}.tar.gz -S --progress=dot:giga -O- | tar -xz
+    - wget ${FDO_HTTP_CACHE_URI:-}https://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME}.tar.zst -S --progress=dot:giga -O- | tar --zstd -x
     - echo -e "\e[0Ksection_end:$(date +%s):artifacts_download\r\e[0K"
   artifacts:
     when: always