OpenHarmony/third_party_mesa3d
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ci: Consistently use -j4 across x86 build jobs and -j8 on ARM.

Browse Source 
Our shared runners are set up for concurrent jobs ~= CPUs / 4 (x86) or 8
(ARM).  If you use more build processes than that, then jobs may be
fighting each other for shared system resources, possibly to the point of
failure (we've seen one of the runners OOM on some jobs before, though I'm
not sure if this was the cause).

To try to systematically prevent the problem, we make a ninja wrapper in
the containers that passes the -j flags, and set MAKEFLAGS in the
container builds.  This doesn't cover make in non-container builds, but I
believe we don't have any of those.

Reviewed-by: Michel Dänzer <mdaenzer@redhat.com>
Tested-by: Marge Bot <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3782>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3782>
This commit is contained in:
Eric Anholt
2020-02-11 15:44:56 -08:00
committed by Marge Bot
parent 2f424c83e0
commit c1e7e83d52
14 changed files with 43 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
.gitlab-ci.yml
View File

@@ -121,7 +121,7 @@ x86_build:
- .fdo.container-build@debian
- .container
variables:
FDO_DISTRIBUTION_TAG: &x86_build "2020-03-13"
FDO_DISTRIBUTION_TAG: &x86_build "2020-03-18-jflags"
.use-x86_build:
variables:
@@ -134,19 +134,19 @@ x86_build:
x86_test-gl:
extends: x86_build
variables:
FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-03-13"
FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-03-18-jflags"
# Debian 10 based x86 test image for VK
x86_test-vk:
extends: x86_build
variables:
FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-03-27"
FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-04-01-jflags"
# Debian 9 based x86 build image (old LLVM)
x86_build_old:
extends: x86_build
variables:
FDO_DISTRIBUTION_TAG: &x86_build_old "2019-09-18"
FDO_DISTRIBUTION_TAG: &x86_build_old "2019-03-18-jflags"
FDO_DISTRIBUTION_VERSION: stretch-slim
.use-x86_build_old:
@@ -162,7 +162,7 @@ arm_build:
- .fdo.container-build@debian@arm64v8
- .container
variables:
FDO_DISTRIBUTION_TAG: &arm_build "2020-03-24"
FDO_DISTRIBUTION_TAG: &arm_build "2020-04-01-jflags"
.use-arm_build:
variables:
@@ -175,7 +175,7 @@ arm_build:
arm_test:
extends: arm_build
variables:
FDO_DISTRIBUTION_TAG: &arm_test "2020-01-30"
FDO_DISTRIBUTION_TAG: &arm_test "2020-03-18-jflags"
.use-arm_test:
variables:

2
.gitlab-ci/build-apitrace.sh
View File

@@ -20,7 +20,7 @@ git clone https://github.com/apitrace/apitrace.git --single-branch --no-checkout
pushd /apitrace
git checkout "$APITRACE_VERSION"
cmake -G Ninja -B_build -H. -DCMAKE_BUILD_TYPE=Release -DENABLE_GUI=False -DENABLE_WAFFLE=on -DWaffle_DIR=/usr/local/lib/cmake/Waffle/
ninja -C _build -j4
ninja -C _build
mkdir build
cp _build/apitrace build
cp _build/glretrace build

2
.gitlab-ci/build-cts-runner.sh
View File

@@ -5,6 +5,6 @@ set -ex
git clone https://gitlab.freedesktop.org/mesa/parallel-deqp-runner.git --depth 1 -b mesa-ci-2019-12-17 /parallel-deqp-runner
pushd /parallel-deqp-runner
meson build/ $EXTRA_MESON_ARGS
ninja -C build -j4 install
ninja -C build install
popd
rm -rf /parallel-deqp-runner

2
.gitlab-ci/build-deqp-vk.sh
View File

@@ -23,7 +23,7 @@ cmake -G Ninja \
-DDEQP_TARGET=x11_glx \
-DCMAKE_BUILD_TYPE=Release \
/VK-GL-CTS
ninja -j4
ninja
# Copy out the mustpass list we want.
mkdir /deqp/mustpass

2
.gitlab-ci/build-fossilize.sh
View File

@@ -9,6 +9,6 @@ git submodule update --init
mkdir build
cd build
cmake .. -DCMAKE_BUILD_TYPE=Release -G Ninja
ninja -C . -j4 install
ninja -C . install
cd ../..
rm -rf Fossilize

2
.gitlab-ci/build-gfxreconstruct.sh
View File

@@ -11,7 +11,7 @@ git checkout "$GFXRECONSTRUCT_VERSION"
git submodule update --init
git submodule update
cmake -G Ninja -B_build -H. -DCMAKE_BUILD_TYPE=Release
ninja -C _build -j4 gfxrecon-replay
ninja -C _build gfxrecon-replay
mkdir -p build/bin
install _build/tools/replay/gfxrecon-replay build/bin
strip build/bin/*

2
.gitlab-ci/build-piglit.sh
View File

@@ -7,7 +7,7 @@ pushd /piglit
git checkout 8771c3860505db2bcf4877216221d774bf90af6b
patch -p1 <$OLDPWD/.gitlab-ci/piglit/disable-vs_in.diff
cmake -G Ninja -DCMAKE_BUILD_TYPE=Release
ninja -j4
ninja
find -name .git -o -name '*ninja*' -o -iname '*cmake*' -o -name '*.[chao]' | xargs rm -rf
rm -rf target_api
popd

2
.gitlab-ci/build-renderdoc.sh
View File

@@ -8,7 +8,7 @@ git clone https://github.com/baldurk/renderdoc.git --single-branch --no-checkout
pushd /renderdoc
git checkout "$RENDERDOC_VERSION"
cmake -G Ninja -B_build -H. -DENABLE_QRENDERDOC=false -DCMAKE_BUILD_TYPE=Release
ninja -C _build -j4
ninja -C _build
mkdir -p build/lib
cp _build/lib/renderdoc.so build/lib
cp _build/lib/librenderdoc.so build/lib

2
.gitlab-ci/build-vulkantools.sh
View File

@@ -19,7 +19,7 @@ cmake -G Ninja -B_build -H. \
-DBUILD_VIA=OFF \
-DBUILD_VKTRACE_REPLAY=OFF \
-C_build/helper.cmake
ninja -C _build -j4 VkLayer_screenshot VkLayer_screenshot-staging-json
ninja -C _build VkLayer_screenshot VkLayer_screenshot-staging-json
mkdir -p build/etc/vulkan/explicit_layer.d
mkdir build/lib
install _build/layersvt/staging-json/VkLayer_screenshot.json build/etc/vulkan/explicit_layer.d

2
.gitlab-ci/container/arm_build.sh
View File

@@ -59,7 +59,7 @@ export LIBDRM_VERSION=libdrm-2.4.100
wget https://dri.freedesktop.org/libdrm/$LIBDRM_VERSION.tar.bz2
tar -xvf $LIBDRM_VERSION.tar.bz2 && rm $LIBDRM_VERSION.tar.bz2
cd $LIBDRM_VERSION; meson build -D vc4=true -D freedreno=true -D etnaviv=true; ninja -j4 -C build install; cd ..
cd $LIBDRM_VERSION; meson build -D vc4=true -D freedreno=true -D etnaviv=true; ninja -C build install; cd ..
rm -rf $LIBDRM_VERSION
############### Generate cross build file for Meson

16
.gitlab-ci/container/container_pre_build.sh
View File

@@ -1,5 +1,7 @@
#!/bin/sh
# Common setup among container builds before we get to building code.
export CCACHE_COMPILERCHECK=content
export CCACHE_COMPRESS=true
export CCACHE_DIR=/cache/mesa/ccache
@@ -11,3 +13,17 @@ export CC="/usr/lib/ccache/gcc"
export CXX="/usr/lib/ccache/g++"
ccache --show-stats
if uname -m | grep -q arm || uname -m | grep -q aarch64; then
export JFLAGS=-j8
else
export JFLAGS=-j4
fi
# Make a wrapper script for ninja to always include the -j flags
echo /usr/bin/ninja $JFLAGS '"$@"' > /usr/local/bin/ninja
chmod +x /usr/local/bin/ninja
# Set MAKEFLAGS so that all make invocations in container builds include the
# flags (doesn't apply to non-container builds, but we don't run make there)
export MAKEFLAGS=$JFLAGS

2
.gitlab-ci/container/lava_arm.sh
View File

@@ -45,7 +45,7 @@ mkdir -p kernel
wget -qO- ${KERNEL_URL} | tar -xz --strip-components=1 -C kernel
pushd kernel
./scripts/kconfig/merge_config.sh ${DEFCONFIG} ../.gitlab-ci/${KERNEL_ARCH}.config
make -j12 ${KERNEL_IMAGE_NAME} dtbs
make ${KERNEL_IMAGE_NAME} dtbs
cp arch/${KERNEL_ARCH}/boot/${KERNEL_IMAGE_NAME} /lava-files/.
cp ${DEVICE_TREES} /lava-files/.
popd

20
.gitlab-ci/container/x86_build.sh
View File

@@ -170,36 +170,36 @@ export WAYLAND_PROTOCOLS_VERSION=wayland-protocols-1.12
wget $XORG_RELEASES/util/$XORGMACROS_VERSION.tar.bz2
tar -xvf $XORGMACROS_VERSION.tar.bz2 && rm $XORGMACROS_VERSION.tar.bz2
cd $XORGMACROS_VERSION; ./configure; make -j4 install; cd ..
cd $XORGMACROS_VERSION; ./configure; make install; cd ..
rm -rf $XORGMACROS_VERSION
wget $XCB_RELEASES/$XCBPROTO_VERSION.tar.bz2
tar -xvf $XCBPROTO_VERSION.tar.bz2 && rm $XCBPROTO_VERSION.tar.bz2
cd $XCBPROTO_VERSION; ./configure; make -j4 install; cd ..
cd $XCBPROTO_VERSION; ./configure; make install; cd ..
rm -rf $XCBPROTO_VERSION
wget $XCB_RELEASES/$LIBXCB_VERSION.tar.bz2
tar -xvf $LIBXCB_VERSION.tar.bz2 && rm $LIBXCB_VERSION.tar.bz2
cd $LIBXCB_VERSION; ./configure; make -j4 install; cd ..
cd $LIBXCB_VERSION; ./configure; make install; cd ..
rm -rf $LIBXCB_VERSION
wget https://dri.freedesktop.org/libdrm/$LIBDRM_VERSION.tar.bz2
tar -xvf $LIBDRM_VERSION.tar.bz2 && rm $LIBDRM_VERSION.tar.bz2
cd $LIBDRM_VERSION
meson build -D vc4=true -D freedreno=true -D etnaviv=true -D libdir=lib/x86_64-linux-gnu; ninja -j4 -C build install
rm -rf build; meson --cross-file=/cross_file-ppc64el.txt build -D libdir=lib/powerpc64le-linux-gnu; ninja -j4 -C build install
rm -rf build; meson --cross-file=/cross_file-i386.txt build -D libdir=lib/i386-linux-gnu; ninja -j4 -C build install
meson build -D vc4=true -D freedreno=true -D etnaviv=true -D libdir=lib/x86_64-linux-gnu; ninja -C build install
rm -rf build; meson --cross-file=/cross_file-ppc64el.txt build -D libdir=lib/powerpc64le-linux-gnu; ninja -C build install
rm -rf build; meson --cross-file=/cross_file-i386.txt build -D libdir=lib/i386-linux-gnu; ninja -C build install
cd ..
rm -rf $LIBDRM_VERSION
wget $WAYLAND_RELEASES/$LIBWAYLAND_VERSION.tar.xz
tar -xvf $LIBWAYLAND_VERSION.tar.xz && rm $LIBWAYLAND_VERSION.tar.xz
cd $LIBWAYLAND_VERSION; ./configure --enable-libraries --without-host-scanner --disable-documentation --disable-dtd-validation; make -j4 install; cd ..
cd $LIBWAYLAND_VERSION; ./configure --enable-libraries --without-host-scanner --disable-documentation --disable-dtd-validation; make install; cd ..
rm -rf $LIBWAYLAND_VERSION
wget $WAYLAND_RELEASES/$WAYLAND_PROTOCOLS_VERSION.tar.xz
tar -xvf $WAYLAND_PROTOCOLS_VERSION.tar.xz && rm $WAYLAND_PROTOCOLS_VERSION.tar.xz
cd $WAYLAND_PROTOCOLS_VERSION; ./configure; make -j4 install; cd ..
cd $WAYLAND_PROTOCOLS_VERSION; ./configure; make install; cd ..
rm -rf $WAYLAND_PROTOCOLS_VERSION
@@ -209,7 +209,7 @@ rm -rf $WAYLAND_PROTOCOLS_VERSION
GLVND_VERSION=1.2.0
wget https://gitlab.freedesktop.org/glvnd/libglvnd/-/archive/v$GLVND_VERSION/libglvnd-v$GLVND_VERSION.tar.gz
tar -xvf libglvnd-v$GLVND_VERSION.tar.gz && rm libglvnd-v$GLVND_VERSION.tar.gz
pushd libglvnd-v$GLVND_VERSION; ./autogen.sh; ./configure; make -j4 install; popd
pushd libglvnd-v$GLVND_VERSION; ./autogen.sh; ./configure; make install; popd
rm -rf libglvnd-v$GLVND_VERSION
@@ -217,7 +217,7 @@ pushd /usr/local
git clone https://gitlab.freedesktop.org/mesa/shader-db.git --depth 1
rm -rf shader-db/.git
cd shader-db
make -j4
make
popd

2
.gitlab-ci/meson-build.sh
View File

@@ -57,7 +57,7 @@ meson _build --native-file=native.file \
${EXTRA_OPTION}
cd _build
meson configure
ninja -j4
ninja
LC_ALL=C.UTF-8 ninja test
ninja install
cd ..