From e814e23f59a80a1ee092d5cf94cc0ea98235d7db Mon Sep 17 00:00:00 2001 From: "Juan A. Suarez Romero" Date: Fri, 12 Feb 2021 18:20:00 +0100 Subject: [PATCH] ci/piglit: allow parallel piglit jobs This allows to split a piglit job in several parallel jobs, to speed up the execution. Due piglit restrictions, this only works for single profiles. Otherwise an error will be shown in the runner. Also, a new gitlab job variable `PIGLIT_TESTS` is introduced that contains the excluded/included tests with `-x` or `-n`. The rest of the piglit options go to `PIGLIT_OPTIONS` (like `--timeout n`). v2 (Andres): - Replay profile is supported in parallel jobs. - Bail out inmediately if parallel jobs is tried with multiple profiles. - Use testlist only when doing parallel jobs. - Do not drop pass tests when filtering executed tests. - Get rid of PIGLIT_FRACTION. v4: - uncommit unrelated change (Andres). Signed-off-by: Juan A. Suarez Romero Reviewed-by: Andres Gomez Part-of: --- .gitlab-ci.yml | 2 +- .gitlab-ci/bare-metal/rootfs-setup.sh | 1 + .gitlab-ci/piglit/run.sh | 66 +++++++++++++++---- .gitlab-ci/piglit/run_cl.sh | 42 ++++++++++-- .gitlab-ci/windows/piglit_run.ps1 | 2 +- src/broadcom/ci/gitlab-ci.yml | 6 +- src/freedreno/ci/gitlab-ci.yml | 4 +- src/gallium/drivers/llvmpipe/ci/gitlab-ci.yml | 3 +- src/gallium/drivers/softpipe/ci/gitlab-ci.yml | 2 +- src/gallium/drivers/zink/ci/gitlab-ci.yml | 2 +- 10 files changed, 102 insertions(+), 28 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3fa967f7070..eaf56173152 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -905,7 +905,7 @@ test-d3d12-windows: GIT_STRATEGY: none # testing doesn't build anything from source GALLIUM_DRIVER: d3d12 PIGLIT_PROFILE: quick_gl - PIGLIT_OPTIONS: -x nv_copy_depth_to_color -x repeat-wait -x arb_timer_query@timestamp-get + PIGLIT_TESTS: -x nv_copy_depth_to_color -x repeat-wait -x arb_timer_query@timestamp-get script: - . _install/piglit_run.ps1 artifacts: diff --git a/.gitlab-ci/bare-metal/rootfs-setup.sh b/.gitlab-ci/bare-metal/rootfs-setup.sh index fe027ac5c98..c97d6463d5c 100644 --- a/.gitlab-ci/bare-metal/rootfs-setup.sh +++ b/.gitlab-ci/bare-metal/rootfs-setup.sh @@ -71,6 +71,7 @@ for var in \ PIGLIT_REPLAY_REFERENCE_IMAGES_BASE_URL \ PIGLIT_REPLAY_UPLOAD_TO_MINIO \ PIGLIT_RESULTS \ + PIGLIT_TESTS \ TEST_LD_PRELOAD \ TU_DEBUG \ VK_CPU \ diff --git a/.gitlab-ci/piglit/run.sh b/.gitlab-ci/piglit/run.sh index 42242e49b46..06bba6c01b3 100755 --- a/.gitlab-ci/piglit/run.sh +++ b/.gitlab-ci/piglit/run.sh @@ -113,23 +113,21 @@ else fi fi -SANITY_MESA_VERSION_CMD="$SANITY_MESA_VERSION_CMD | tee /tmp/version.txt | grep \"Mesa $MESA_VERSION\(\s\|$\)\"" - if [ "$ZINK_USE_LAVAPIPE" ]; then export VK_ICD_FILENAMES="$INSTALL/share/vulkan/icd.d/lvp_icd.x86_64.json" fi -rm -rf results -cd /piglit +# If the job is parallel at the gitlab job level, will take the corresponding +# fraction of the caselist. +if [ -n "$CI_NODE_INDEX" ]; then -PIGLIT_OPTIONS=$(printf "%s" "$PIGLIT_OPTIONS") + if [ "$PIGLIT_PROFILES" != "${PIGLIT_PROFILES% *}" ]; then + FAILURE_MESSAGE=$(printf "%s" "Can't parallelize piglit with multiple profiles") + quiet print_red printf "%s\n" "$FAILURE_MESSAGE" + exit 1 + fi -PIGLIT_CMD="./piglit run -j${FDO_CI_CONCURRENT:-4} $PIGLIT_OPTIONS $PIGLIT_PROFILES "$(/usr/bin/printf "%q" "$RESULTS") - -RUN_CMD="export LD_LIBRARY_PATH=$__LD_LIBRARY_PATH; $SANITY_MESA_VERSION_CMD && $PIGLIT_CMD" - -if [ "$RUN_CMD_WRAPPER" ]; then - RUN_CMD="set +e; $RUN_CMD_WRAPPER "$(/usr/bin/printf "%q" "$RUN_CMD")"; set -e" + USE_CASELIST=1 fi print_red() { @@ -183,6 +181,35 @@ replay_minio_upload_images() { done } +SANITY_MESA_VERSION_CMD="$SANITY_MESA_VERSION_CMD | tee /tmp/version.txt | grep \"Mesa $MESA_VERSION\(\s\|$\)\"" + +rm -rf results +cd /piglit + +if [ -n "$USE_CASELIST" ]; then + PIGLIT_TESTS=$(printf "%s" "$PIGLIT_TESTS") + PIGLIT_GENTESTS="./piglit print-cmd $PIGLIT_TESTS $PIGLIT_PROFILES --format \"{name}\" > /tmp/case-list.txt" + RUN_GENTESTS="export LD_LIBRARY_PATH=$__LD_LIBRARY_PATH; $PIGLIT_GENTESTS" + + eval $RUN_GENTESTS + + sed -ni $CI_NODE_INDEX~$CI_NODE_TOTAL"p" /tmp/case-list.txt + + PIGLIT_TESTS="--test-list /tmp/case-list.txt" +fi + +PIGLIT_OPTIONS=$(printf "%s" "$PIGLIT_OPTIONS") + +PIGLIT_TESTS=$(printf "%s" "$PIGLIT_TESTS") + +PIGLIT_CMD="./piglit run -j${FDO_CI_CONCURRENT:-4} $PIGLIT_OPTIONS $PIGLIT_TESTS $PIGLIT_PROFILES "$(/usr/bin/printf "%q" "$RESULTS") + +RUN_CMD="export LD_LIBRARY_PATH=$__LD_LIBRARY_PATH; $SANITY_MESA_VERSION_CMD && $PIGLIT_CMD" + +if [ "$RUN_CMD_WRAPPER" ]; then + RUN_CMD="set +e; $RUN_CMD_WRAPPER "$(/usr/bin/printf "%q" "$RUN_CMD")"; set -e" +fi + FAILURE_MESSAGE=$(printf "%s" "Unexpected change in results:") eval $RUN_CMD @@ -223,8 +250,21 @@ if [ "x$PIGLIT_PROFILES" = "xreplay" ] \ "minio://${MINIO_HOST}${__MINIO_PATH}/${__MINIO_TRACES_PREFIX}/junit.xml" fi -cp "$INSTALL/$PIGLIT_RESULTS.txt" \ - ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline" +if [ -n "$USE_CASELIST" ]; then + # Just filter the expected results based on the tests that were actually + # executed, and switch to the version with no summary + cat $RESULTSFILE | head -n -16 > ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.new" + RESULTSFILE=".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.new" + cat ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.orig" | head -n -17 | rev \ + | cut -f2- -d: | rev | sed "s/$/:/g" > /tmp/executed.txt + + grep -F -f /tmp/executed.txt "$INSTALL/$PIGLIT_RESULTS.txt" \ + > ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline" || true +else + cp "$INSTALL/$PIGLIT_RESULTS.txt" \ + ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline" +fi + if diff -q ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline" $RESULTSFILE; then exit 0 fi diff --git a/.gitlab-ci/piglit/run_cl.sh b/.gitlab-ci/piglit/run_cl.sh index 95d70787001..d1c675d7af6 100755 --- a/.gitlab-ci/piglit/run_cl.sh +++ b/.gitlab-ci/piglit/run_cl.sh @@ -10,12 +10,31 @@ cd /piglit export OCL_ICD_VENDORS=$OLDPWD/install/etc/OpenCL/vendors/ -PIGLIT_OPTIONS=$(echo $PIGLIT_OPTIONS | head -n 1) set +e unset DISPLAY export LD_LIBRARY_PATH=$OLDPWD/install/lib clinfo -./piglit run -c -j${FDO_CI_CONCURRENT:-4} $PIGLIT_OPTIONS $PIGLIT_PROFILES $OLDPWD/results + +# If the job is parallel at the gitlab job level, will take the corresponding +# fraction of the caselist. +if [ -n "$CI_NODE_INDEX" ]; then + + if [ "$PIGLIT_PROFILES" != "${PIGLIT_PROFILES% *}" ]; then + echo "Can't parallelize piglit with multiple profiles" + exit 1 + fi + USE_CASELIST=1 +fi + +if [ -n "$USE_CASELIST" ]; then + ./piglit print-cmd $PIGLIT_TESTS $PIGLIT_PROFILES --format "{name}" > /tmp/case-list.txt + + sed -ni $CI_NODE_INDEX~$CI_NODE_TOTAL"p" /tmp/case-list.txt + + PIGLIT_TESTS="--test-list /tmp/case-list.txt" +fi + +./piglit run -c -j${FDO_CI_CONCURRENT:-4} $PIGLIT_OPTIONS $PIGLIT_TESTS $PIGLIT_PROFILES $OLDPWD/results retVal=$? if [ $retVal -ne 0 ]; then echo "Found $(cat /tmp/version.txt), expected $VERSION" @@ -24,15 +43,26 @@ set -e PIGLIT_RESULTS=${PIGLIT_RESULTS:-$PIGLIT_PROFILES} mkdir -p .gitlab-ci/piglit -cp $OLDPWD/install/$PIGLIT_RESULTS.txt .gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline -./piglit summary console $OLDPWD/results | head -n -1 | grep -v ": pass" >.gitlab-ci/piglit/$PIGLIT_RESULTS.txt +./piglit summary console $OLDPWD/results | head -n -1 | grep -v ": pass" > .gitlab-ci/piglit/$PIGLIT_RESULTS.txt -if diff -q .gitlab-ci/piglit/$PIGLIT_RESULTS.txt{.baseline,}; then +if [ -n "$USE_CASELIST" ]; then + # Just filter the expected results based on the tests that were actually + # executed, and switch to the version with no summary + cat .gitlab-ci/piglit/$PIGLIT_RESULTS.txt | head -n -16 | tee .gitlab-ci/piglit/$PIGLIT_RESULTS.txt.new \ + | rev | cut -f2- -d: | rev | sed "s/$/:/g" > /tmp/executed.txt + grep -F -f /tmp/executed.txt $OLDPWD/install/$PIGLIT_RESULTS.txt \ + > .gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline || true +else + cp $OLDPWD/install/$PIGLIT_RESULTS.txt .gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline + cp .gitlab-ci/piglit/$PIGLIT_RESULTS.txt .gitlab-ci/piglit/$PIGLIT_RESULTS.txt.new +fi + +if diff -q .gitlab-ci/piglit/$PIGLIT_RESULTS.txt.{baseline,new}; then exit 0 fi ./piglit summary html --exclude-details=pass $OLDPWD/summary $OLDPWD/results echo Unexpected change in results: -diff -u .gitlab-ci/piglit/$PIGLIT_RESULTS.txt{.baseline,} +diff -u .gitlab-ci/piglit/$PIGLIT_RESULTS.txt.{baseline,new} exit 1 diff --git a/.gitlab-ci/windows/piglit_run.ps1 b/.gitlab-ci/windows/piglit_run.ps1 index f62bb2c12c7..d965f51d0c7 100644 --- a/.gitlab-ci/windows/piglit_run.ps1 +++ b/.gitlab-ci/windows/piglit_run.ps1 @@ -3,7 +3,7 @@ $env:PIGLIT_NO_FAST_SKIP = 1 Copy-Item -Path _install\bin\opengl32.dll -Destination C:\Piglit\lib\piglit\bin\opengl32.dll # Run this using VsDevCmd.bat to ensure DXIL.dll is in %PATH% -cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64 && py -3 C:\Piglit\bin\piglit.py run `"$env:PIGLIT_PROFILE`" $env:PIGLIT_OPTIONS .\results" +cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64 && py -3 C:\Piglit\bin\piglit.py run `"$env:PIGLIT_PROFILE`" $env:PIGLIT_OPTIONS $env:PIGLIT_TESTS .\results" py -3 C:\Piglit\bin\piglit.py summary console .\results | Select -SkipLast 1 | Select-String -NotMatch -Pattern ': pass' | Set-Content -Path .\result.txt diff --git a/src/broadcom/ci/gitlab-ci.yml b/src/broadcom/ci/gitlab-ci.yml index 39cce154fa7..1cc78a43f4e 100644 --- a/src/broadcom/ci/gitlab-ci.yml +++ b/src/broadcom/ci/gitlab-ci.yml @@ -40,6 +40,8 @@ vc4-rpi3-gles2:armhf: BM_POE_TIMEOUT: 180 PIGLIT_OPTIONS: >- --timeout 60 + PIGLIT_PLATFORM: gbm + PIGLIT_TESTS: >- -x .tesc -x .tese -x arb_gpu_shader_fp64 @@ -83,7 +85,6 @@ vc4-rpi3-gles2:armhf: -x ppgtt_memory_alignment -x streaming-texture-leak -x texturesize - PIGLIT_PLATFORM: gbm vc4-rpi3-piglit-quick_gl:armhf: extends: @@ -154,6 +155,8 @@ v3d-rpi4-gles2:armhf: BARE_METAL_TEST_SCRIPT: "/install/piglit/run.sh" PIGLIT_OPTIONS: >- --timeout 60 + PIGLIT_PLATFORM: gbm + PIGLIT_TESTS: >- -x .tesc -x .tese -x arb_gpu_shader_fp64 @@ -182,7 +185,6 @@ v3d-rpi4-gles2:armhf: -x maxsize -x streaming-texture-leak -x tex-miplevel-selection - PIGLIT_PLATFORM: gbm v3d-rpi4-piglit-quick_gl:armhf: extends: diff --git a/src/freedreno/ci/gitlab-ci.yml b/src/freedreno/ci/gitlab-ci.yml index 04c7aa6363a..56a531f945b 100644 --- a/src/freedreno/ci/gitlab-ci.yml +++ b/src/freedreno/ci/gitlab-ci.yml @@ -112,7 +112,7 @@ a530_gles31: # Also excluding some particularly slow tests (gl-1.3-texture-env) # # Check fastboot.sh for more piglit tests being excluded! - PIGLIT_OPTIONS: >- + PIGLIT_TESTS: >- -x atomic_counters -x gl-1.0-blend-func -x glsl-1.30@execution@clipping @@ -277,7 +277,7 @@ a630_vk_sysmem: # # vs-clip-distance/vertex-enables skipped because they seem to be flaky in # parallel with other tests (reliably fails on its own) - PIGLIT_OPTIONS: >- + PIGLIT_TESTS: >- -x vs-output-array-vec2-index-wr-before-gs -x fixed-clip-enables -x vs-clip-distance-enables diff --git a/src/gallium/drivers/llvmpipe/ci/gitlab-ci.yml b/src/gallium/drivers/llvmpipe/ci/gitlab-ci.yml index 28e0dd066ec..73c321a6fac 100644 --- a/src/gallium/drivers/llvmpipe/ci/gitlab-ci.yml +++ b/src/gallium/drivers/llvmpipe/ci/gitlab-ci.yml @@ -16,7 +16,7 @@ llvmpipe-piglit-cl: LP_NUM_THREADS: 1 PIGLIT_PROFILES: cl PIGLIT_RESULTS: "llvmpipe-cl" - PIGLIT_OPTIONS: > + PIGLIT_TESTS: > -x bswap -x phatk -x clz-optimizations script: - install/piglit/run_cl.sh @@ -31,6 +31,7 @@ llvmpipe-piglit-quick_gl: LP_NUM_THREADS: 0 PIGLIT_OPTIONS: > --process-isolation false + PIGLIT_TESTS: > -x egl_ext_device_ -x egl_ext_platform_device -x ext_timer_query@time-elapsed diff --git a/src/gallium/drivers/softpipe/ci/gitlab-ci.yml b/src/gallium/drivers/softpipe/ci/gitlab-ci.yml index 4b5fb2eb5c9..68ef54f2000 100644 --- a/src/gallium/drivers/softpipe/ci/gitlab-ci.yml +++ b/src/gallium/drivers/softpipe/ci/gitlab-ci.yml @@ -39,7 +39,7 @@ softpipe-piglit-quick: # rasterpos skipped until we uprev piglit (commit ff2a7650be7349) # triangle-rasterization skipped due to flakes thanks to it using a time-based random seed. # glx-multithread-texture is flaky, undiagnosed - PIGLIT_OPTIONS: > + PIGLIT_TESTS: > -x rasterpos -x triangle-rasterization -x glx-multithread-texture diff --git a/src/gallium/drivers/zink/ci/gitlab-ci.yml b/src/gallium/drivers/zink/ci/gitlab-ci.yml index 3ad090b3710..092f92e2f9d 100644 --- a/src/gallium/drivers/zink/ci/gitlab-ci.yml +++ b/src/gallium/drivers/zink/ci/gitlab-ci.yml @@ -15,7 +15,7 @@ zink-piglit-quick_gl: - .piglit-test - .zink-lvp-test variables: - PIGLIT_OPTIONS: > + PIGLIT_TESTS: > -x arb_map_buffer_alignment@arb_map_buffer_alignment-map-invalidate-range -x glx-make-current -x arb_timer_query -x arb_sample_shading@builtin-gl-sample-mask PIGLIT_PROFILES: quick_gl