diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7e99ee91132..b5f9c883351 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -72,6 +72,8 @@ variables: echo -n '${S3_JWT}' > '${S3_JWT_FILE}' && unset CI_JOB_JWT S3_JWT # Unsetting vulnerable env variables S3_HOST: s3.freedesktop.org + # This bucket is used to fetch ANDROID prebuilts and images + S3_ANDROID_BUCKET: mesa-rootfs # This bucket is used to fetch the kernel image S3_KERNEL_BUCKET: mesa-rootfs # Bucket for git cache diff --git a/.gitlab-ci/container/build-android-x86_64-llvm.sh b/.gitlab-ci/container/build-android-x86_64-llvm.sh new file mode 100755 index 00000000000..4dc3b62e6a3 --- /dev/null +++ b/.gitlab-ci/container/build-android-x86_64-llvm.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +set -exu + +# If CI vars are not set, assign an empty value, this prevents -u to fail +: "${CI:=}" +: "${CI_PROJECT_PATH:=}" + +# Early check for required env variables, relies on `set -u` +: "$ANDROID_SDK_VERSION" +: "$ANDROID_NDK" +: "$LLVM_VERSION" +: "$LLVM_ARTIFACT_NAME" +: "$S3_JWT_FILE" +: "$S3_HOST" +: "$S3_ANDROID_BUCKET" + +# Check if the auth file used later on for CI is non-empty +if [ -n "$CI" ] && [ ! -s "${S3_JWT_FILE}" ]; then + echo "Error: ${S3_JWT_FILE} is empty." 1>&2 + exit 1 +fi + +if curl -s -o /dev/null -I -L -f --retry 4 --retry-delay 15 "https://${S3_HOST}/${S3_ANDROID_BUCKET}/${CI_PROJECT_PATH}/${LLVM_ARTIFACT_NAME}.tar.zst"; then + echo "Artifact ${LLVM_ARTIFACT_NAME}.tar.zst already exists, skip re-building." + exit +fi + +# Install some dependencies needed to build LLVM +EPHEMERAL=( + ninja-build + unzip +) + +apt-get update +apt-get install -y --no-install-recommends --no-remove "${EPHEMERAL[@]}" + +ANDROID_NDK_ROOT="/${ANDROID_NDK}" +if [ ! -d "$ANDROID_NDK_ROOT" ]; +then + curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \ + -o "${ANDROID_NDK}.zip" \ + "https://dl.google.com/android/repository/${ANDROID_NDK}-linux.zip" + unzip -d / "${ANDROID_NDK}.zip" "$ANDROID_NDK/source.properties" "$ANDROID_NDK/build/cmake/*" "$ANDROID_NDK/toolchains/llvm/*" + rm "${ANDROID_NDK}.zip" +fi + +if [ ! -d "/llvm-project" ]; +then + mkdir "/llvm-project" + pushd "/llvm-project" + git init + git remote add origin https://github.com/llvm/llvm-project.git + git fetch --depth 1 origin "$LLVM_VERSION" + git checkout FETCH_HEAD + popd +fi + +pushd "/llvm-project" + +# Checkout again the intended version, just in case of a pre-existing full clone +git checkout "$LLVM_VERSION" || true + +LLVM_INSTALL_PREFIX="/${LLVM_ARTIFACT_NAME}" + +rm -rf build/ +cmake -GNinja -S llvm -B build/ \ + -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake" \ + -DANDROID_ABI=x86_64 \ + -DANDROID_PLATFORM="android-${ANDROID_SDK_VERSION}" \ + -DANDROID_NDK="${ANDROID_NDK_ROOT}" \ + -DCMAKE_ANDROID_ARCH_ABI=x86_64 \ + -DCMAKE_ANDROID_NDK="${ANDROID_NDK_ROOT}" \ + -DCMAKE_BUILD_TYPE=MinSizeRel \ + -DCMAKE_SYSTEM_NAME=Android \ + -DCMAKE_SYSTEM_VERSION="${ANDROID_SDK_VERSION}" \ + -DCMAKE_INSTALL_PREFIX="${LLVM_INSTALL_PREFIX}" \ + -DCMAKE_CXX_FLAGS="-march=x86-64 --target=x86_64-linux-android${ANDROID_SDK_VERSION} -fno-rtti" \ + -DLLVM_HOST_TRIPLE="x86_64-linux-android${ANDROID_SDK_VERSION}" \ + -DLLVM_TARGETS_TO_BUILD=X86 \ + -DLLVM_BUILD_LLVM_DYLIB=OFF \ + -DLLVM_BUILD_TESTS=OFF \ + -DLLVM_BUILD_EXAMPLES=OFF \ + -DLLVM_BUILD_DOCS=OFF \ + -DLLVM_BUILD_TOOLS=OFF \ + -DLLVM_ENABLE_RTTI=OFF \ + -DLLVM_BUILD_INSTRUMENTED_COVERAGE=OFF \ + -DLLVM_NATIVE_TOOL_DIR="${ANDROID_NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/bin" \ + -DLLVM_ENABLE_PIC=False \ + -DLLVM_OPTIMIZED_TABLEGEN=ON + +ninja "-j${FDO_CI_CONCURRENT:-4}" -C build/ install + +popd + +rm -rf /llvm-project + +tar --zstd -cf "${LLVM_ARTIFACT_NAME}.tar.zst" "$LLVM_INSTALL_PREFIX" + +# If run in CI upload the tar.zst archive to S3 to avoid rebuilding it if the +# version does not change, and delete it. +# The file is not deleted for non-CI because it can be useful in local runs. +if [ -n "$CI" ]; then + ci-fairy s3cp --token-file "${S3_JWT_FILE}" "${LLVM_ARTIFACT_NAME}.tar.zst" "https://${S3_HOST}/${S3_ANDROID_BUCKET}/${CI_PROJECT_PATH}/${LLVM_ARTIFACT_NAME}.tar.zst" + rm "${LLVM_ARTIFACT_NAME}.tar.zst" +fi + +rm -rf "$LLVM_INSTALL_PREFIX" + +apt-get purge -y "${EPHEMERAL[@]}" diff --git a/.gitlab-ci/container/debian/android_build.sh b/.gitlab-ci/container/debian/android_build.sh old mode 100644 new mode 100755 index a8b15b0a465..724804289f0 --- a/.gitlab-ci/container/debian/android_build.sh +++ b/.gitlab-ci/container/debian/android_build.sh @@ -23,7 +23,7 @@ apt-get install -y --no-remove "${EPHEMERAL[@]}" ndk=$ANDROID_NDK curl -L --retry 4 -f --retry-all-errors --retry-delay 60 \ -o $ndk.zip https://dl.google.com/android/repository/$ndk-linux.zip -unzip -d / $ndk.zip "$ndk/toolchains/llvm/*" +unzip -d / $ndk.zip "$ndk/source.properties" "$ndk/build/cmake/*" "$ndk/toolchains/llvm/*" rm $ndk.zip # Since it was packed as a zip file, symlinks/hardlinks got turned into # duplicate files. Turn them into hardlinks to save on container space. @@ -100,4 +100,9 @@ done cd .. rm -rf $LIBELF_VERSION + +# Build LLVM libraries for Android only if necessary, uploading a copy to S3 +# to avoid rebuilding it in a future run if the version does not change. +bash .gitlab-ci/container/build-android-x86_64-llvm.sh + apt-get purge -y "${EPHEMERAL[@]}" diff --git a/.gitlab-ci/container/gitlab-ci.yml b/.gitlab-ci/container/gitlab-ci.yml index 7e82abd13f7..d6ef4a8ee0e 100644 --- a/.gitlab-ci/container/gitlab-ci.yml +++ b/.gitlab-ci/container/gitlab-ci.yml @@ -161,6 +161,8 @@ debian/android_build: MESA_IMAGE_TAG: &debian-android_build ${DEBIAN_BUILD_TAG} ANDROID_SDK_VERSION: 33 ANDROID_NDK: android-ndk-r25b + LLVM_VERSION: llvmorg-18.1.8 + LLVM_ARTIFACT_NAME: android-x86_64-llvm-20241219 .use-debian/android_build: extends: diff --git a/.gitlab-ci/image-tags.yml b/.gitlab-ci/image-tags.yml index a1a65bcaa2b..b14b4f46228 100644 --- a/.gitlab-ci/image-tags.yml +++ b/.gitlab-ci/image-tags.yml @@ -16,7 +16,7 @@ variables: DEBIAN_BASE_TAG: "20241125-crosvm" DEBIAN_X86_64_BUILD_IMAGE_PATH: "debian/x86_64_build" - DEBIAN_BUILD_TAG: "20241122-sections" + DEBIAN_BUILD_TAG: "20241219-android" DEBIAN_X86_64_TEST_BASE_IMAGE: "debian/x86_64_test-base" DEBIAN_ARM64_TEST_BASE_IMAGE: "debian/arm64_test-base"