From 463dbbffa8cc3d55715ab0a886ff542057c23236 Mon Sep 17 00:00:00 2001 From: Eric Anholt Date: Thu, 17 Dec 2020 11:08:42 -0800 Subject: [PATCH] ci/deqp: Make sure that we pull in all board-specific xfail/skip/flake files. When introducing/removing these files, it's easy to forget to update the yml to point to them. Instead of requiring the separate update, just have the runner script pick the right one from a single per-gpu variable. As a result, we now pick up the new deqp-lvp-skips.txt that was added but not conected. This also required moving some bypass flakes from the shared a630 flakes list to a separate list, which is a feature because now we'd notice the introduction of flakes to the gmem path. Fixes: ab79e6b8e336 ("ci: skip failing test on lavapipe") Acked-by: Daniel Stone Reviewed-by: Juan A. Suarez Reviewed-by: Christian Gmeiner Part-of: --- .gitlab-ci.yml | 40 ++++++------------- .../bare-metal/arm64_a630_gles_others.sh | 2 +- .gitlab-ci/bare-metal/rootfs-setup.sh | 4 +- .../deqp-freedreno-a630-bypass-flakes.txt | 11 +++++ .gitlab-ci/deqp-freedreno-a630-flakes.txt | 8 ---- .gitlab-ci/deqp-runner.sh | 21 +++++----- .gitlab-ci/lava-deqp.yml.jinja2 | 3 +- .gitlab-ci/lava-gitlab-ci.yml | 2 +- 8 files changed, 40 insertions(+), 51 deletions(-) create mode 100644 .gitlab-ci/deqp-freedreno-a630-bypass-flakes.txt diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 401e7894afa..4485c922a61 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1063,8 +1063,6 @@ llvmpipe-piglit-quick_shader: PIGLIT_PROFILES: quick_shader .deqp-test: - variables: - DEQP_SKIPS: deqp-default-skips.txt script: - ./install/deqp-runner.sh @@ -1096,7 +1094,7 @@ llvmpipe-gles2: # Don't use threads inside llvmpipe, we've already got all cores # busy at the deqp-runner level. LP_NUM_THREADS: 0 - DEQP_EXPECTED_FAILS: deqp-llvmpipe-fails.txt + GPU_VERSION: llvmpipe LIBGL_ALWAYS_SOFTWARE: "true" DEQP_EXPECTED_RENDERER: llvmpipe extends: @@ -1108,9 +1106,7 @@ softpipe-gles2: - llvmpipe-gles2 - .softpipe-rules variables: - DEQP_EXPECTED_FAILS: deqp-softpipe-fails.txt - DEQP_SKIPS: deqp-softpipe-skips.txt - DEQP_FLAKES: deqp-softpipe-flakes.txt + GPU_VERSION: softpipe GALLIUM_DRIVER: "softpipe" DEQP_EXPECTED_RENDERER: softpipe @@ -1138,11 +1134,10 @@ virgl-gles2-on-gl: variables: DEQP_VER: gles2 DEQP_NO_SAVE_RESULTS: 1 - DEQP_SKIPS: deqp-virgl-gl-skips.txt + GPU_VERSION: virgl-gl # Don't use threads inside llvmpipe, we've already got all cores # busy at the deqp-runner level. LP_NUM_THREADS: 0 - DEQP_EXPECTED_FAILS: deqp-virgl-gl-fails.txt DEQP_OPTIONS: "--deqp-log-images=disable" LIBGL_ALWAYS_SOFTWARE: "true" GALLIUM_DRIVER: "virpipe" @@ -1200,7 +1195,7 @@ virgl-gl32-on-gl: virgl-gles2-on-gles: variables: VIRGL_HOST_API: GLES - DEQP_EXPECTED_FAILS: deqp-virgl-gles-fails.txt + GPU_VERSION: virgl-gles extends: - virgl-gles2-on-gl - .test-manual @@ -1208,7 +1203,7 @@ virgl-gles2-on-gles: virgl-gles3-on-gles: variables: VIRGL_HOST_API: GLES - DEQP_EXPECTED_FAILS: deqp-virgl-gles-fails.txt + GPU_VERSION: virgl-gles extends: - virgl-gles3-on-gl - .test-manual @@ -1216,7 +1211,7 @@ virgl-gles3-on-gles: virgl-gles31-on-gles: variables: VIRGL_HOST_API: GLES - DEQP_EXPECTED_FAILS: deqp-virgl-gles-fails.txt + GPU_VERSION: virgl-gles extends: - virgl-gles31-on-gl - .test-manual @@ -1227,9 +1222,7 @@ arm64_a630_gles2: variables: BM_KERNEL: /lava-files/cheza-kernel BM_CMDLINE: "ip=dhcp console=ttyMSM0,115200n8 root=/dev/nfs rw nfsrootdebug nfsroot=,tcp,nfsvers=4.2 init=/init" - DEQP_EXPECTED_FAILS: deqp-freedreno-a630-fails.txt - DEQP_FLAKES: deqp-freedreno-a630-flakes.txt - DEQP_SKIPS: deqp-freedreno-a630-skips.txt + GPU_VERSION: freedreno-a630 GIT_STRATEGY: none DEQP_EXPECTED_RENDERER: FD630 VK_DRIVER: freedreno @@ -1297,7 +1290,7 @@ arm64_a630_vk_sysmem: DEQP_VER: vk DEQP_FRACTION: 15 DEQP_CASELIST_FILTER: "dEQP-VK.renderpass.*" - DEQP_EXPECTED_FAILS: deqp-freedreno-a630-bypass-fails.txt + GPU_VERSION: freedreno-a630-bypass TU_DEBUG: sysmem .baremetal-test: @@ -1326,9 +1319,7 @@ arm64_a306_gles2: BM_CMDLINE: "ip=dhcp console=ttyMSM0,115200n8" FLAKES_CHANNEL: "#freedreno-ci" BARE_METAL_TEST_SCRIPT: "/install/deqp-runner.sh" - DEQP_EXPECTED_FAILS: deqp-freedreno-a307-fails.txt - DEQP_SKIPS: deqp-default-skips.txt - DEQP_FLAKES: deqp-freedreno-a307-flakes.txt + GPU_VERSION: freedreno-a307 DEQP_PARALLEL: 0 # Default to number of CPUs DEQP_VER: gles2 DEQP_EXPECTED_RENDERER: FD307 @@ -1368,8 +1359,7 @@ arm64_a306_gles3_options: # Disable SMP because only CPU 0 is at a freq higher than 19mhz on # current upstream kernel. BM_CMDLINE: "ip=dhcp console=ttyMSM0,115200n8 nosmp" - DEQP_EXPECTED_FAILS: deqp-freedreno-a530-fails.txt - DEQP_FLAKES: deqp-freedreno-a530-flakes.txt + GPU_VERSION: freedreno-a530 DEQP_EXPECTED_RENDERER: FD530 tags: - google-freedreno-db820c @@ -1406,10 +1396,8 @@ vc4-rpi3-gles2:armhf: BM_KERNEL_MODULES: vc4 BM_ROOTFS: /lava-files/rootfs-armhf BM_ROOTFS_EXTRA: /modules/armhf - DEQP_EXPECTED_FAILS: deqp-vc4-rpi3-fails.txt + GPU_VERSION: vc4-rpi3 DEQP_EXPECTED_RENDERER: VC4 - DEQP_FLAKES: deqp-vc4-rpi3-flakes.txt - DEQP_SKIPS: deqp-vc4-rpi3-skips.txt DEQP_VER: gles2 script: - ./install/bare-metal/poe-powered.sh @@ -1425,9 +1413,7 @@ lavapipe-vk: - .llvmpipe-rules - .deqp-test-vk variables: - DEQP_EXPECTED_FAILS: deqp-lvp-fails.txt - DEQP_FLAKES: deqp-lvp-flakes.txt - DEQP_SKIPS: deqp-lvp-skips.txt + GPU_VERSION: lvp VK_DRIVER: lvp DEQP_FRACTION: 10 @@ -1447,7 +1433,7 @@ radv_polaris10_vkcts: - .test-radv - .test-manual variables: - DEQP_SKIPS: deqp-radv-polaris10-skips.txt + GPU_VERSION: radv-polaris10 tags: - polaris10 diff --git a/.gitlab-ci/bare-metal/arm64_a630_gles_others.sh b/.gitlab-ci/bare-metal/arm64_a630_gles_others.sh index d2bb08be7c0..6e2399c8b55 100755 --- a/.gitlab-ci/bare-metal/arm64_a630_gles_others.sh +++ b/.gitlab-ci/bare-metal/arm64_a630_gles_others.sh @@ -23,7 +23,7 @@ if ! env \ DEQP_VER=gles31 \ DEQP_FRACTION=5 \ FD_MESA_DEBUG=nogmem \ - DEQP_EXPECTED_FAILS=deqp-freedreno-a630-bypass-fails.txt \ + GPU_VERSION=freedreno-a630-bypass \ /install/deqp-runner.sh; then EXIT=1 fi diff --git a/.gitlab-ci/bare-metal/rootfs-setup.sh b/.gitlab-ci/bare-metal/rootfs-setup.sh index 8438ac0dc98..26fcfe1833e 100644 --- a/.gitlab-ci/bare-metal/rootfs-setup.sh +++ b/.gitlab-ci/bare-metal/rootfs-setup.sh @@ -30,15 +30,12 @@ for var in \ CI_RUNNER_DESCRIPTION \ DEQP_CASELIST_FILTER \ DEQP_CONFIG \ - DEQP_EXPECTED_FAILS \ DEQP_EXPECTED_RENDERER \ DEQP_FRACTION \ DEQP_HEIGHT \ DEQP_NO_SAVE_RESULTS \ - DEQP_FLAKES \ DEQP_PARALLEL \ DEQP_RESULTS_DIR \ - DEQP_SKIPS \ DEQP_VARIANT \ DEQP_VER \ DEQP_WIDTH \ @@ -46,6 +43,7 @@ for var in \ DRIVER_NAME \ FD_MESA_DEBUG \ FLAKES_CHANNEL \ + GPU_VERSION \ IR3_SHADER_DEBUG \ MESA_GL_VERSION_OVERRIDE \ MESA_GLSL_VERSION_OVERRIDE \ diff --git a/.gitlab-ci/deqp-freedreno-a630-bypass-flakes.txt b/.gitlab-ci/deqp-freedreno-a630-bypass-flakes.txt new file mode 100644 index 00000000000..a1d91cdc93a --- /dev/null +++ b/.gitlab-ci/deqp-freedreno-a630-bypass-flakes.txt @@ -0,0 +1,11 @@ +# Note: flakes lists for CI are just a list of lines that, when +# non-zero-length and not starting with '#', will regex match to +# delete lines from the test list. Be careful. + +# At least some of the separate_channels tests fail on sysmem due to an +# interaction of use of a UBWC buffer as both a render target and a +# texture. Stores are done through both paths in separate channels, +# and the UBWC updates don't get synced. The current a650 blob also +# fails these tests and qcom apparently noted the failure at one point +# https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/2017 +dEQP-VK.renderpass.*separate_channels.* diff --git a/.gitlab-ci/deqp-freedreno-a630-flakes.txt b/.gitlab-ci/deqp-freedreno-a630-flakes.txt index 88ab1c1ea5a..ff7f2672a5a 100644 --- a/.gitlab-ci/deqp-freedreno-a630-flakes.txt +++ b/.gitlab-ci/deqp-freedreno-a630-flakes.txt @@ -28,14 +28,6 @@ dEQP-VK.pipeline.spec_constant.compute.composite.matrix.mat3x2 # Fails NIR_VALIDATE so probably flaky dEQP-VK.memory_model.write_after_read.core11.u32.coherent.fence_fence.atomicwrite.workgroup.payload_nonlocal.workgroup.guard_local.buffer.comp -# At least some of the separate_channels tests fail on sysmem due to an -# interaction of use of a UBWC buffer as both a render target and a -# texture. Stores are done through both paths in separate channels, -# and the UBWC updates don't get synced. The current a650 blob also -# fails these tests and qcom apparently noted the failure at one point -# https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/2017 -dEQP-VK.renderpass.*separate_channels.* - # Undiagnosed flakes appearing more than once in the last 2 months as # of 2020-08-19, in descending order of frequency. dEQP-GLES3.functional.fbo.msaa.2_samples.stencil_index8 diff --git a/.gitlab-ci/deqp-runner.sh b/.gitlab-ci/deqp-runner.sh index 2bb89d92145..129e4416559 100755 --- a/.gitlab-ci/deqp-runner.sh +++ b/.gitlab-ci/deqp-runner.sh @@ -24,8 +24,8 @@ if [ "$DEQP_VER" = "vk" ]; then fi fi -if [ -z "$DEQP_SKIPS" ]; then - echo 'DEQP_SKIPS must be set to something like "deqp-default-skips.txt"' +if [ -z "$GPU_VERSION" ]; then + echo 'GPU_VERSION must be set to something like "llvmpipe" or "freedreno-a630" (the name used in .gitlab-ci/deqp-gpu-version-*.txt)' exit 1 fi @@ -79,12 +79,18 @@ if [ ! -s /tmp/case-list.txt ]; then exit 1 fi -if [ -n "$DEQP_EXPECTED_FAILS" ]; then - BASELINE="--baseline $INSTALL/$DEQP_EXPECTED_FAILS" +if [ -e "$INSTALL/deqp-$GPU_VERSION-fails.txt" ]; then + DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --baseline $INSTALL/deqp-$GPU_VERSION-fails.txt" fi -if [ -n "$DEQP_FLAKES" ]; then - FLAKES="--flakes $INSTALL/$DEQP_FLAKES" +if [ -e "$INSTALL/deqp-$GPU_VERSION-flakes.txt" ]; then + DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt" +fi + +if [ -e "$INSTALL/deqp-$GPU_VERSION-skips.txt" ]; then + DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --skips $INSTALL/deqp-$GPU_VERSION-skips.txt" +else + DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --skips $INSTALL/deqp-default-skips.txt" fi set +e @@ -114,9 +120,6 @@ run_cts() { --deqp $deqp \ --output $RESULTS \ --caselist $caselist \ - --skips $INSTALL/$DEQP_SKIPS \ - $BASELINE \ - $FLAKES \ $JOB \ $SUMMARY_LIMIT \ $DEQP_RUNNER_OPTIONS \ diff --git a/.gitlab-ci/lava-deqp.yml.jinja2 b/.gitlab-ci/lava-deqp.yml.jinja2 index 8fa95ff1f81..be3a6024312 100644 --- a/.gitlab-ci/lava-deqp.yml.jinja2 +++ b/.gitlab-ci/lava-deqp.yml.jinja2 @@ -95,8 +95,7 @@ actions: - wget -S --progress=dot:giga -O- {{ mesa_url }} | tar -xz - export DEQP_NO_SAVE_RESULTS=1 - - export DEQP_EXPECTED_FAILS=deqp-{{ gpu_version }}-fails.txt - - export DEQP_SKIPS=deqp-{{ gpu_version }}-skips.txt + - export GPU_VERSION={{ gpu_version }} - export DEQP_VER={{ deqp_version }} - export LIBGL_DRIVERS_PATH=`pwd`/install/lib/dri diff --git a/.gitlab-ci/lava-gitlab-ci.yml b/.gitlab-ci/lava-gitlab-ci.yml index 13827b55d65..7c334ea722f 100644 --- a/.gitlab-ci/lava-gitlab-ci.yml +++ b/.gitlab-ci/lava-gitlab-ci.yml @@ -170,7 +170,7 @@ panfrost-t720-gles2:arm64: DEVICE_TYPE: sun50i-h6-pine-h64 DTB: ${DEVICE_TYPE} GPU_VERSION: panfrost-t720 - ENV_VARS: "PAN_MESA_DEBUG=gles3 DEQP_PARALLEL=6 DEQP_EXPECTED_RENDERER=T720 DEQP_FLAKES=deqp-panfrost-t720-flakes.txt" + ENV_VARS: "PAN_MESA_DEBUG=gles3 DEQP_PARALLEL=6 DEQP_EXPECTED_RENDERER=T720" tags: - mesa-ci-aarch64-lava-collabora