ci: build kernel in gfx-ci/linux and just use binaries in Mesa3D CI

This bring visible speedup while preparing the rootfs and containers.

Acked-by: Guilherme Gallo <guilherme.gallo@collabora.com>
Acked-by: Erico Nunes <nunes.erico@gmail.com>
Signed-off-by: David Heidelberg <david.heidelberg@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/24079>
This commit is contained in:
David Heidelberg
2023-06-10 05:16:56 +02:00
parent 9bcdc45ee7
commit 23158e9a3f
11 changed files with 79 additions and 107 deletions

View File

@@ -32,6 +32,7 @@ variables:
PIPELINE_ARTIFACTS_BASE: ${S3_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}
# per-job artifact storage on MinIO
JOB_ARTIFACTS_BASE: ${PIPELINE_ARTIFACTS_BASE}/${CI_JOB_ID}
KERNEL_IMAGE_BASE: https://${S3_HOST}/mesa-lava/gfx-ci/linux/${KERNEL_TAG}
# reference images stored for traces
PIGLIT_REPLAY_REFERENCE_IMAGES_BASE: "${S3_HOST}/mesa-tracie-results/$FDO_UPSTREAM_REPO"
# For individual CI farm status see .ci-farms folder

View File

@@ -69,7 +69,7 @@ for var in \
JOB_ARTIFACTS_BASE \
JOB_RESULTS_PATH \
JOB_ROOTFS_OVERLAY_PATH \
KERNEL_IMAGE_BASE_URL \
KERNEL_IMAGE_BASE \
KERNEL_IMAGE_NAME \
LD_LIBRARY_PATH \
LP_NUM_THREADS \

View File

@@ -86,7 +86,7 @@ if [ "$HWCI_KVM" = "true" ]; then
mkdir -p /lava-files
curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-o "/lava-files/${KERNEL_IMAGE_NAME}" \
"${KERNEL_IMAGE_BASE_URL}/${KERNEL_IMAGE_NAME}"
"${KERNEL_IMAGE_BASE}/amd64/${KERNEL_IMAGE_NAME}"
fi
# Fix prefix confusion: the build installs to $CI_PROJECT_DIR, but we expect

View File

@@ -24,11 +24,11 @@ if [[ $arch == "arm64" ]]; then
pushd /baremetal-files
curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-O "${ARTIFACTS_URL}"/Image
-O "${KERNEL_IMAGE_BASE}"/arm64/Image
curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-O "${ARTIFACTS_URL}"/Image.gz
-O "${KERNEL_IMAGE_BASE}"/arm64/Image.gz
curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-O "${ARTIFACTS_URL}"/cheza-kernel
-O "${KERNEL_IMAGE_BASE}"/arm64/cheza-kernel
DEVICE_TREES=""
DEVICE_TREES="$DEVICE_TREES apq8016-sbc.dtb"
@@ -38,7 +38,7 @@ if [[ $arch == "arm64" ]]; then
for DTB in $DEVICE_TREES; do
curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-O "${ARTIFACTS_URL}/$DTB"
-O "${KERNEL_IMAGE_BASE}/arm64/$DTB"
done
popd
@@ -47,7 +47,7 @@ elif [[ $arch == "armhf" ]]; then
pushd /baremetal-files
curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-O "${ARTIFACTS_URL}"/zImage
-O "${KERNEL_IMAGE_BASE}"/armhf/zImage
DEVICE_TREES=""
DEVICE_TREES="$DEVICE_TREES imx6q-cubox-i.dtb"
@@ -55,7 +55,7 @@ elif [[ $arch == "armhf" ]]; then
for DTB in $DEVICE_TREES; do
curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-O "${ARTIFACTS_URL}/$DTB"
-O "${KERNEL_IMAGE_BASE}/armhf/$DTB"
done
popd

View File

@@ -1,54 +1,31 @@
#!/bin/bash
#!/usr/bin/env bash
# shellcheck disable=SC2086 # we want word splitting
# shellcheck disable=SC2153
set -ex
mkdir -p kernel
curl -L -s --retry 4 -f --retry-all-errors --retry-delay 60 ${KERNEL_URL} \
| tar -xj --strip-components=1 -C kernel
pushd kernel
# The kernel doesn't like the gold linker (or the old lld in our debians).
# Sneak in some override symlinks during kernel build until we can update
# debian (they'll get blown away by the rm of the kernel dir at the end).
mkdir -p ld-links
for i in /usr/bin/*-ld /usr/bin/ld; do
i=$(basename $i)
ln -sf /usr/bin/$i.bfd ld-links/$i
done
NEWPATH=$(pwd)/ld-links
export PATH=$NEWPATH:$PATH
KERNEL_FILENAME=$(basename $KERNEL_URL)
export LOCALVERSION="$KERNEL_FILENAME"
./scripts/kconfig/merge_config.sh ${DEFCONFIG} ../.gitlab-ci/container/${KERNEL_ARCH}.config
make ${KERNEL_IMAGE_NAME}
for image in ${KERNEL_IMAGE_NAME}; do
cp arch/${KERNEL_ARCH}/boot/${image} /lava-files/.
done
if [[ -n ${DEVICE_TREES} ]]; then
make dtbs
cp ${DEVICE_TREES} /lava-files/.
fi
make modules
INSTALL_MOD_PATH=/lava-files/rootfs-${DEBIAN_ARCH}/ make modules_install
if [[ ${DEBIAN_ARCH} = "arm64" ]]; then
make Image.lzma
mkimage \
-f auto \
-A arm \
-O linux \
-d arch/arm64/boot/Image.lzma \
-C lzma\
-b arch/arm64/boot/dts/qcom/sdm845-cheza-r3.dtb \
/lava-files/cheza-kernel
KERNEL_IMAGE_NAME+=" cheza-kernel"
fi
for image in ${KERNEL_IMAGE_NAME}; do
curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-o "/lava-files/${image}" "${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/${image}"
done
for dtb in ${DEVICE_TREES}; do
curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-o "/lava-files/${dtb}" "${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/${dtb}"
done
mkdir -p "/lava-files/rootfs-${DEBIAN_ARCH}"
curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \
-O "${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}/modules.tar.zst"
tar --keep-directory-symlink --zstd -xf modules.tar.zst -C "/lava-files/rootfs-${DEBIAN_ARCH}/"
popd
rm -rf kernel

View File

@@ -192,8 +192,7 @@ debian/android_build:
debian/x86_64_test-base:
extends: debian/x86_64_build-base
variables:
KERNEL_URL: &kernel-rootfs-url "https://gitlab.freedesktop.org/gfx-ci/linux/-/archive/v6.3-for-mesa-ci-43c973a8ff91/linux-v6.3-for-mesa-ci-6fc749a2a59a.tar.bz2"
MESA_IMAGE_TAG: &debian-x86_64_test-base ${DEBIAN_BASE_TAG}
MESA_IMAGE_TAG: &debian-x86_64_test-base "${DEBIAN_BASE_TAG}--${KERNEL_TAG}"
.use-debian/x86_64_test-base:
extends:
@@ -331,9 +330,8 @@ fedora/x86_64_build:
stage: container
variables:
GIT_STRATEGY: fetch
KERNEL_URL: *kernel-rootfs-url
MESA_ROOTFS_TAG: &kernel-rootfs ${KERNEL_ROOTFS_TAG}
DISTRIBUTION_TAG: &distribution-tag-arm "${MESA_ROOTFS_TAG}--${MESA_ARTIFACTS_TAG}--${MESA_TEMPLATES_COMMIT}"
DISTRIBUTION_TAG: &distribution-tag-arm "${MESA_ROOTFS_TAG}--${KERNEL_TAG}--${MESA_ARTIFACTS_TAG}--${MESA_TEMPLATES_COMMIT}"
script:
- .gitlab-ci/container/lava_build.sh
@@ -344,7 +342,7 @@ kernel+rootfs_x86_64:
image: "$FDO_BASE_IMAGE"
variables:
DEBIAN_ARCH: "amd64"
DISTRIBUTION_TAG: &distribution-tag-x86_64 "${MESA_ROOTFS_TAG}--${MESA_ARTIFACTS_BASE_TAG}--${MESA_TEMPLATES_COMMIT}"
DISTRIBUTION_TAG: &distribution-tag-x86_64 "${MESA_ROOTFS_TAG}--${KERNEL_TAG}--${MESA_ARTIFACTS_BASE_TAG}--${MESA_TEMPLATES_COMMIT}"
kernel+rootfs_arm64:
extends:
@@ -380,9 +378,9 @@ kernel+rootfs_arm32:
# Don't want the .container rules
- .build-rules
variables:
FDO_DISTRIBUTION_TAG: "${MESA_IMAGE_TAG}--${MESA_ROOTFS_TAG}--${MESA_TEMPLATES_COMMIT}"
FDO_DISTRIBUTION_TAG: "${MESA_IMAGE_TAG}--${MESA_ROOTFS_TAG}--${KERNEL_TAG}--${MESA_TEMPLATES_COMMIT}"
ARTIFACTS_PREFIX: "https://${S3_HOST}/mesa-lava"
ARTIFACTS_SUFFIX: "${MESA_ROOTFS_TAG}--${MESA_ARTIFACTS_TAG}--${MESA_TEMPLATES_COMMIT}"
ARTIFACTS_SUFFIX: "${MESA_ROOTFS_TAG}--${KERNEL_TAG}--${MESA_ARTIFACTS_TAG}--${MESA_TEMPLATES_COMMIT}"
MESA_ARTIFACTS_TAG: *debian-arm64_build
MESA_ROOTFS_TAG: *kernel-rootfs
@@ -407,7 +405,7 @@ debian/arm64_test:
MESA_ROOTFS_TAG: *kernel-rootfs
.use-debian/arm32_test:
image: "$CI_REGISTRY_IMAGE/${MESA_IMAGE_PATH}:${MESA_IMAGE_TAG}--${MESA_ROOTFS_TAG}--${MESA_TEMPLATES_COMMIT}"
image: "$CI_REGISTRY_IMAGE/${MESA_IMAGE_PATH}:${MESA_IMAGE_TAG}--${MESA_ROOTFS_TAG}--${KERNEL_TAG}--${MESA_TEMPLATES_COMMIT}"
extends:
- .use-debian/arm_test
variables:
@@ -417,7 +415,7 @@ debian/arm64_test:
- debian/arm_test
.use-debian/arm64_test:
image: "$CI_REGISTRY_IMAGE/${MESA_IMAGE_PATH}:${MESA_IMAGE_TAG}--${MESA_ROOTFS_TAG}--${MESA_TEMPLATES_COMMIT}"
image: "$CI_REGISTRY_IMAGE/${MESA_IMAGE_PATH}:${MESA_IMAGE_TAG}--${MESA_ROOTFS_TAG}--${KERNEL_TAG}--${MESA_TEMPLATES_COMMIT}"
extends:
- .use-debian/arm_test
variables:

View File

@@ -35,20 +35,20 @@ if [[ "$DEBIAN_ARCH" = "arm64" ]]; then
KERNEL_ARCH="arm64"
SKQP_ARCH="arm64"
DEFCONFIG="arch/arm64/configs/defconfig"
DEVICE_TREES="arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dtb"
DEVICE_TREES+=" arch/arm64/boot/dts/amlogic/meson-g12b-a311d-khadas-vim3.dtb"
DEVICE_TREES+=" arch/arm64/boot/dts/amlogic/meson-gxl-s805x-libretech-ac.dtb"
DEVICE_TREES+=" arch/arm64/boot/dts/amlogic/meson-gxm-khadas-vim2.dtb"
DEVICE_TREES+=" arch/arm64/boot/dts/allwinner/sun50i-h6-pine-h64.dtb"
DEVICE_TREES+=" arch/arm64/boot/dts/freescale/imx8mq-nitrogen.dtb"
DEVICE_TREES+=" arch/arm64/boot/dts/mediatek/mt8192-asurada-spherion-r0.dtb"
DEVICE_TREES+=" arch/arm64/boot/dts/mediatek/mt8183-kukui-jacuzzi-juniper-sku16.dtb"
DEVICE_TREES+=" arch/arm64/boot/dts/nvidia/tegra210-p3450-0000.dtb"
DEVICE_TREES+=" arch/arm64/boot/dts/qcom/apq8016-sbc.dtb"
DEVICE_TREES+=" arch/arm64/boot/dts/qcom/apq8096-db820c.dtb"
DEVICE_TREES+=" arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-limozeen-nots-r5.dtb"
DEVICE_TREES+=" arch/arm64/boot/dts/qcom/sc7180-trogdor-kingoftown-r1.dtb"
DEVICE_TREES+=" arch/arm64/boot/dts/qcom/sm8350-hdk.dtb"
DEVICE_TREES="rk3399-gru-kevin.dtb"
DEVICE_TREES+=" meson-g12b-a311d-khadas-vim3.dtb"
DEVICE_TREES+=" meson-gxl-s805x-libretech-ac.dtb"
DEVICE_TREES+=" meson-gxm-khadas-vim2.dtb"
DEVICE_TREES+=" sun50i-h6-pine-h64.dtb"
DEVICE_TREES+=" imx8mq-nitrogen.dtb"
DEVICE_TREES+=" mt8192-asurada-spherion-r0.dtb"
DEVICE_TREES+=" mt8183-kukui-jacuzzi-juniper-sku16.dtb"
DEVICE_TREES+=" tegra210-p3450-0000.dtb"
DEVICE_TREES+=" apq8016-sbc.dtb"
DEVICE_TREES+=" apq8096-db820c.dtb"
DEVICE_TREES+=" sc7180-trogdor-lazor-limozeen-nots-r5.dtb"
DEVICE_TREES+=" sc7180-trogdor-kingoftown-r1.dtb"
DEVICE_TREES+=" sm8350-hdk.dtb"
KERNEL_IMAGE_NAME="Image"
elif [[ "$DEBIAN_ARCH" = "armhf" ]]; then
@@ -56,10 +56,10 @@ elif [[ "$DEBIAN_ARCH" = "armhf" ]]; then
KERNEL_ARCH="arm"
SKQP_ARCH="arm"
DEFCONFIG="arch/arm/configs/multi_v7_defconfig"
DEVICE_TREES="arch/arm/boot/dts/rk3288-veyron-jaq.dtb"
DEVICE_TREES+=" arch/arm/boot/dts/sun8i-h3-libretech-all-h3-cc.dtb"
DEVICE_TREES+=" arch/arm/boot/dts/imx6q-cubox-i.dtb"
DEVICE_TREES+=" arch/arm/boot/dts/tegra124-jetson-tk1.dtb"
DEVICE_TREES="rk3288-veyron-jaq.dtb"
DEVICE_TREES+=" sun8i-h3-libretech-all-h3-cc.dtb"
DEVICE_TREES+=" imx6q-cubox-i.dtb"
DEVICE_TREES+=" tegra124-jetson-tk1.dtb"
KERNEL_IMAGE_NAME="zImage"
. .gitlab-ci/container/create-cross-file.sh armhf
else
@@ -324,25 +324,16 @@ if [ "${DEBIAN_ARCH}" = "arm64" ]; then
KERNEL_IMAGE_NAME+=" Image.gz"
fi
ROOTFSTAR="lava-rootfs.tar.zst"
du -ah "$ROOTFS" | sort -h | tail -100
pushd $ROOTFS
tar --zstd -cf /lava-files/lava-rootfs.tar.zst .
tar --zstd -cf /lava-files/${ROOTFSTAR} .
popd
. .gitlab-ci/container/container_post_build.sh
############### Upload the files!
FILES_TO_UPLOAD="lava-rootfs.tar.zst \
$KERNEL_IMAGE_NAME"
if [[ -n $DEVICE_TREES ]]; then
FILES_TO_UPLOAD="$FILES_TO_UPLOAD $(basename -a $DEVICE_TREES)"
fi
for f in $FILES_TO_UPLOAD; do
ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/$f \
https://${S3_PATH}/$f
done
ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/"${ROOTFSTAR}" \
https://${S3_PATH}/"${ROOTFSTAR}"
touch /lava-files/done
ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/done https://${S3_PATH}/done

View File

@@ -1,6 +1,6 @@
variables:
DEBIAN_X86_64_BUILD_BASE_IMAGE: "debian/x86_64_build-base"
DEBIAN_BASE_TAG: "2023-07-10-virglrenderer"
DEBIAN_BASE_TAG: "2023-07-11-dekernelize"
DEBIAN_X86_64_BUILD_IMAGE_PATH: "debian/x86_64_build"
DEBIAN_BUILD_TAG: "2023-06-24-agility-711"
@@ -21,7 +21,8 @@ variables:
ALPINE_X86_64_BUILD_TAG: "2023-05-01-3.18-bump-1"
ALPINE_X86_64_LAVA_SSH_TAG: "2023-06-26-first-version"
FEDORA_X86_64_BUILD_TAG: "2023-05-05-ccache-on"
KERNEL_ROOTFS_TAG: "2023-07-10-virglrenderer"
KERNEL_ROOTFS_TAG: "2023-07-11-dekernelize"
KERNEL_TAG: "v6.3-rc6-for-mesa-ci-a8016765452e"
WINDOWS_X64_VS_PATH: "windows/x64_vs"
WINDOWS_X64_VS_TAG: "2022-10-20-upgrade-zlib"

View File

@@ -25,7 +25,6 @@ cp "$SCRIPTS_DIR"/setup-test-env.sh results/job-rootfs-overlay/
# Prepare env vars for upload.
section_start variables "Variables passed through:"
KERNEL_IMAGE_BASE_URL="https://${BASE_SYSTEM_HOST_PATH}" \
artifacts/ci-common/generate-env.sh | tee results/job-rootfs-overlay/set-job-env-vars.sh
section_end variables
@@ -41,7 +40,7 @@ PYTHONPATH=artifacts/ artifacts/lava/lava_job_submitter.py \
--dump-yaml \
--pipeline-info "$CI_JOB_NAME: $CI_PIPELINE_URL on $CI_COMMIT_REF_NAME ${CI_NODE_INDEX}/${CI_NODE_TOTAL}" \
--rootfs-url-prefix "https://${BASE_SYSTEM_HOST_PATH}" \
--kernel-url-prefix "https://${BASE_SYSTEM_HOST_PATH}" \
--kernel-url-prefix "${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}" \
--build-url "${ARTIFACT_URL}" \
--job-rootfs-overlay-url "${FDO_HTTP_CACHE_URI:-}https://${JOB_ROOTFS_OVERLAY_PATH}" \
--job-timeout-min ${JOB_TIMEOUT:-30} \

View File

@@ -56,6 +56,8 @@ clang-format:
- debian/x86_64_test-gl
- debian-testing
- !reference [.required-for-hardware-jobs, needs]
variables:
DEBIAN_ARCH: amd64
.test-vk:
extends:
@@ -65,6 +67,8 @@ clang-format:
- debian-testing
- debian/x86_64_test-vk
- !reference [.required-for-hardware-jobs, needs]
variables:
DEBIAN_ARCH: amd64
.test-cl:
extends:
@@ -177,6 +181,8 @@ clang-format:
- rm -rf install
- (set -x; curl -L --retry 4 -f --retry-all-errors --retry-delay 60 ${FDO_HTTP_CACHE_URI:-}https://${PIPELINE_ARTIFACTS_BASE}/${S3_ARTIFACT_NAME}.tar.zst | tar --zstd -x)
- section_end artifacts_download
variables:
BM_ROOTFS: /rootfs-${DEBIAN_ARCH}
artifacts:
when: always
name: "mesa_${CI_JOB_NAME}"
@@ -194,7 +200,7 @@ clang-format:
- .baremetal-test
- .use-debian/arm32_test
variables:
BM_ROOTFS: /rootfs-armhf
DEBIAN_ARCH: armhf
S3_ARTIFACT_NAME: mesa-arm32-default-debugoptimized
needs:
- debian/arm32_test
@@ -208,7 +214,7 @@ clang-format:
- .baremetal-test
- .use-debian/arm64_test
variables:
BM_ROOTFS: /rootfs-arm64
DEBIAN_ARCH: arm64
S3_ARTIFACT_NAME: mesa-arm64-default-debugoptimized
needs:
- debian/arm64_test

View File

@@ -18,7 +18,7 @@ Linux mainline, that is why Mesa has its own kernel version which should be used
as the base for newer kernels.
So, one should base the kernel uprev from the last tag used in the Mesa CI,
please refer to ``.gitlab-ci/container/gitlab-ci.yml`` ``KERNEL_URL`` variable.
please refer to ``.gitlab-ci/image-tags.yml`` ``KERNEL_TAG`` variable.
Every tag has a standard naming: ``vX.YZ-for-mesa-ci-<commit_short_SHA>``, which
can be created via the command:
@@ -27,8 +27,7 @@ can be created via the command:
Building Kernel
---------------
When Mesa CI generates a new rootfs image, the Linux Kernel is built based on
the script located at ``.gitlab-ci/container/build-kernel.sh``.
The kernel files are loaded from the artifacts uploaded to S3 from gfx-ci/linux.
Updating Kconfigs
^^^^^^^^^^^^^^^^^
@@ -41,15 +40,15 @@ defconfig* made via ``merge_config.sh`` script located at Linux Kernel tree.
Kconfigs location
"""""""""""""""""
+------------+--------------------------------------------+-------------------------------------+
+------------+-----------------------------------------------------+-------------------------------------+
| Platform | Mesa CI Kconfig location | Linux tree defconfig |
+============+============================================+=====================================+
| arm | .gitlab-ci/container/arm.config | arch/arm/configs/multi_v7_defconfig |
+------------+--------------------------------------------+-------------------------------------+
| arm64 | .gitlab-ci/container/arm64.config | arch/arm64/configs/defconfig |
+------------+--------------------------------------------+-------------------------------------+
| x86-64 | .gitlab-ci/container/x86_64.config | arch/x86/configs/x86_64_defconfig |
+------------+--------------------------------------------+-------------------------------------+
+============+=====================================================+=====================================+
| arm | kernel/configs/mesa3d-ci_arm.config@gfx-ci/linux | arch/arm/configs/multi_v7_defconfig |
+------------+-----------------------------------------------------+-------------------------------------+
| arm64 | kernel/configs/mesa3d-ci_arm64.config@gfx-ci/linux | arch/arm64/configs/defconfig |
+------------+-----------------------------------------------------+-------------------------------------+
| x86-64 | kernel/configs/mesa3d-ci_x86_64.config@gfx-ci/linux | arch/x86/configs/x86_64_defconfig |
+------------+-----------------------------------------------------+-------------------------------------+
Updating image tags
-------------------