From 302b4f772445cbfd2dd3f3ad69e4e293f5768fc9 Mon Sep 17 00:00:00 2001 From: Eric Anholt Date: Fri, 18 Dec 2020 16:47:51 -0800 Subject: [PATCH] ci/deqp: Upgrade the runner, enable junit output. I moved QPA-to-XML conversion to the runner, so Mesa CI (and developers!) don't need to do quite so much in bash. I also made it clean up caselist .qpa files since nobody ever wants them and we deleted them anyway. This cleans up a ton of the job log output. Additionally, I added a subcommend to turn the .csv into a junit output that we can expose to gitlab. Now, the pipeline's status page will report the failed testcases, and the "detail" button will give you a link to the .XML to view for the failure. (We don't report all testcases because it's too much load for the gitlab server). Note that this will 404 for the LAVA runners for now, as they don't retain artifacts in gitlab (the plan is to eventually have them minio upload the artifacts). This uprev also includes a deqp output parsing fix, resulting in us catching a couple more failures in some drivers. Reviewed-by: Christian Gmeiner Part-of: --- .gitlab-ci.yml | 13 ++++-- .gitlab-ci/build-deqp-runner.sh | 2 +- .gitlab-ci/deqp-freedreno-a530-fails.txt | 1 + .gitlab-ci/deqp-freedreno-a630-fails.txt | 11 +++++ .gitlab-ci/deqp-freedreno-a630-skips.txt | 1 + .gitlab-ci/deqp-lvp-fails.txt | 1 + .gitlab-ci/deqp-runner.sh | 59 +++++++----------------- .gitlab-ci/deqp-virgl-gl-fails.txt | 2 +- .gitlab-ci/lava-gitlab-ci.yml | 2 +- 9 files changed, 43 insertions(+), 49 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2d6c4db49a7..35300f2c5b1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -362,13 +362,13 @@ x86_test-base: x86_test-gl: extends: .use-x86_test-base variables: - MESA_IMAGE_TAG: &x86_test-gl "2020-12-18-piglit-replayer" + MESA_IMAGE_TAG: &x86_test-gl "2020-12-22-runner" # Debian 10 based x86 test image for VK x86_test-vk: extends: .use-x86_test-base variables: - MESA_IMAGE_TAG: &x86_test-vk "2020-12-19-build-piglit" + MESA_IMAGE_TAG: &x86_test-vk "2020-12-22-runner" # Debian 10 based ARM build image arm_build: @@ -408,7 +408,7 @@ arm64_test: extends: - .use-arm_test-base variables: - MESA_IMAGE_TAG: &arm64_test "2020-12-19-build-piglit" + MESA_IMAGE_TAG: &arm64_test "2020-12-22-runner" .use-arm64_test: variables: @@ -422,7 +422,7 @@ armhf_test: extends: - .use-arm_test-base variables: - MESA_IMAGE_TAG: &armhf_test "2020-12-19-build-piglit" + MESA_IMAGE_TAG: &armhf_test "2020-12-22-runner" .use-armhf_test: variables: @@ -1179,6 +1179,9 @@ radv-raven-traces: .deqp-test: script: - ./install/deqp-runner.sh + artifacts: + reports: + junit: results/junit.xml .deqp-test-vk: extends: @@ -1412,6 +1415,8 @@ arm64_a630_vk_sysmem: - serial*.txt exclude: - results/*.shader_cache + reports: + junit: results/junit.xml .freedreno-test: extends: diff --git a/.gitlab-ci/build-deqp-runner.sh b/.gitlab-ci/build-deqp-runner.sh index 99915182dce..c8b8298da49 100644 --- a/.gitlab-ci/build-deqp-runner.sh +++ b/.gitlab-ci/build-deqp-runner.sh @@ -4,6 +4,6 @@ set -ex cargo install deqp-runner \ -j ${FDO_CI_CONCURRENT:-4} \ - --version 0.1.5 \ + --version 0.4.0 \ --root /usr/local \ $EXTRA_CARGO_ARGS diff --git a/.gitlab-ci/deqp-freedreno-a530-fails.txt b/.gitlab-ci/deqp-freedreno-a530-fails.txt index 9b86f26f0e8..13e8d0d12ff 100644 --- a/.gitlab-ci/deqp-freedreno-a530-fails.txt +++ b/.gitlab-ci/deqp-freedreno-a530-fails.txt @@ -39,6 +39,7 @@ dEQP-GLES3.functional.transform_feedback.random.interleaved.lines.3,Fail dEQP-GLES3.functional.transform_feedback.random.separate.points.3,Fail dEQP-GLES3.functional.transform_feedback.random_full_array_capture.separate.triangles.3,Fail dEQP-GLES31.functional.image_load_store.cube.format_reinterpret.r32i_rgba8,Fail +dEQP-GLES31.functional.image_load_store.cube.format_reinterpret.rgba32f_rgba32ui,Fail dEQP-GLES31.functional.image_load_store.cube.format_reinterpret.rgba8_snorm_r32ui,Fail dEQP-GLES31.functional.image_load_store.cube.format_reinterpret.rgba8i_r32f,Fail dEQP-GLES31.functional.image_load_store.cube.load_store.r32f_single_layer,Fail diff --git a/.gitlab-ci/deqp-freedreno-a630-fails.txt b/.gitlab-ci/deqp-freedreno-a630-fails.txt index c7d0899436e..42b142076cb 100644 --- a/.gitlab-ci/deqp-freedreno-a630-fails.txt +++ b/.gitlab-ci/deqp-freedreno-a630-fails.txt @@ -1,8 +1,14 @@ KHR-GL30.transform_feedback.api_errors_test,Fail +KHR-GL30.transform_feedback.capture_vertex_interleaved_test,Fail +KHR-GL30.transform_feedback.capture_vertex_separate_test,Fail +KHR-GL30.transform_feedback.discard_vertex_test,Fail KHR-GL30.transform_feedback.draw_xfb_feedbackk_test,Crash KHR-GL30.transform_feedback.draw_xfb_instanced_test,Crash KHR-GL30.transform_feedback.draw_xfb_stream_instanced_test,Crash KHR-GL30.transform_feedback.draw_xfb_test,Crash +KHR-GL30.transform_feedback.get_xfb_varying,Fail +KHR-GL30.transform_feedback.query_vertex_interleaved_test,Fail +KHR-GL30.transform_feedback.query_vertex_separate_test,Fail dEQP-VK.api.object_management.single_alloc_callbacks.compute_pipeline,Fail dEQP-VK.compute.indirect_dispatch.gen_in_compute.multiple_groups_multiple_invocations,Fail dEQP-VK.compute.indirect_dispatch.upload_buffer.multiple_groups,Fail @@ -29,6 +35,7 @@ dEQP-VK.multiview.renderpass2.masks.max_multi_view_view_count,Fail dEQP-VK.pipeline.extended_dynamic_state.after_pipelines.depth_compare_greater_equal_greater,Fail dEQP-VK.pipeline.extended_dynamic_state.before_draw.depth_compare_always_greater,Fail dEQP-VK.pipeline.multisample.alpha_to_coverage_unused_attachment.samples_4.alpha_invisible,Fail +dEQP-VK.pipeline.push_descriptor.compute.binding3_numcalls2_sampler,Crash dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_16_64_6.samples_2.d24_unorm_s8_uint.depth_zero,Fail dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_16_64_6.samples_4.x8_d24_unorm_pack32.depth_zero,Fail dEQP-VK.renderpass2.suballocation.attachment_allocation.input_output.7,Fail @@ -38,11 +45,15 @@ dEQP-VK.spirv_assembly.instruction.graphics.opquantize.negative_too_small_tesse, dEQP-VK.spirv_assembly.instruction.graphics.opquantize.round_to_inf_tesse,Fail dEQP-VK.spirv_assembly.instruction.graphics.opquantize.spec_const_carry_to_exponent_tesse,Fail dEQP-VK.spirv_assembly.instruction.graphics.opquantize.spec_const_negative_round_up_or_round_down_tesse,Fail +dEQP-VK.tessellation.invariance.inner_triangle_set.triangles_equal_spacing,Fail dEQP-VK.tessellation.invariance.outer_edge_division.triangles_fractional_even_spacing,Fail dEQP-VK.tessellation.invariance.outer_edge_index_independence.triangles_equal_spacing_ccw,Fail +dEQP-VK.tessellation.invariance.outer_edge_index_independence.triangles_fractional_even_spacing_cw,Fail +dEQP-VK.tessellation.invariance.outer_edge_index_independence.quads_fractional_even_spacing_ccw,Fail dEQP-VK.tessellation.invariance.outer_edge_symmetry.isolines_equal_spacing_cw,Fail dEQP-VK.tessellation.invariance.outer_edge_symmetry.quads_fractional_odd_spacing_ccw,Fail dEQP-VK.tessellation.invariance.outer_edge_symmetry.triangles_fractional_odd_spacing_cw,Fail +dEQP-VK.tessellation.invariance.outer_triangle_set.quads_fractional_odd_spacing,Fail dEQP-VK.tessellation.invariance.primitive_set.isolines_fractional_odd_spacing_ccw,Fail dEQP-VK.tessellation.invariance.primitive_set.quads_fractional_odd_spacing_cw,Fail dEQP-VK.tessellation.invariance.primitive_set.triangles_fractional_even_spacing_ccw,Fail diff --git a/.gitlab-ci/deqp-freedreno-a630-skips.txt b/.gitlab-ci/deqp-freedreno-a630-skips.txt index 588b657f356..07deb2b5405 100644 --- a/.gitlab-ci/deqp-freedreno-a630-skips.txt +++ b/.gitlab-ci/deqp-freedreno-a630-skips.txt @@ -18,6 +18,7 @@ dEQP-VK.spirv_assembly.instruction.graphics.spirv_ids_abuse.lots_ids_tesse dEQP-VK.tessellation.invariance.outer_edge_division.quads_fractional_odd_spacing # Timeout (VK-GL-CTS 1.2.5.0) +dEQP-VK.geometry.layered.cube_array.36_36_12.readback dEQP-VK.geometry.layered.cube_array.64_64_12.readback # Crashes likely caused by https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/2701 diff --git a/.gitlab-ci/deqp-lvp-fails.txt b/.gitlab-ci/deqp-lvp-fails.txt index 14bb70d2992..0150b2520b6 100644 --- a/.gitlab-ci/deqp-lvp-fails.txt +++ b/.gitlab-ci/deqp-lvp-fails.txt @@ -4,6 +4,7 @@ dEQP-VK.api.copy_and_blit.core.blit_image.all_formats.color.2d.r8g8_unorm.r16g16 dEQP-VK.api.copy_and_blit.core.blit_image.all_formats.color.2d.r8g8b8a8_unorm.a2b10g10r10_unorm_pack32.optimal_optimal_linear,Fail dEQP-VK.api.copy_and_blit.core.blit_image.all_formats.color.2d.r8g8b8a8_unorm.r16g16_unorm.general_optimal_linear,Fail dEQP-VK.api.copy_and_blit.core.blit_image.all_formats.color.2d.r8g8b8a8_unorm.r16g16b16a16_unorm.linear_general_linear,Fail +dEQP-VK.api.copy_and_blit.core.blit_image.all_formats.color.2d.r8_unorm.r16g16b16a16_unorm.general_optimal_linear,Fail dEQP-VK.glsl.builtin.precision.pow.highp.vec3,Fail dEQP-VK.glsl.texture_functions.query.texturequerylod.sampler1d_fixed_fragment,Fail dEQP-VK.glsl.texture_functions.query.texturequerylod.sampler2darray_fixed_fragment,Fail diff --git a/.gitlab-ci/deqp-runner.sh b/.gitlab-ci/deqp-runner.sh index 42f6f730e46..6d04fdbb90a 100755 --- a/.gitlab-ci/deqp-runner.sh +++ b/.gitlab-ci/deqp-runner.sh @@ -120,6 +120,7 @@ run_cts() { --deqp $deqp \ --output $RESULTS \ --caselist $caselist \ + --testlog-to-xml /deqp/executor/testlog-to-xml \ $JOB \ $SUMMARY_LIMIT \ $DEQP_RUNNER_OPTIONS \ @@ -170,31 +171,6 @@ report_flakes() { } -# Generate junit results -generate_junit() { - results=$1 - echo "" - echo "" - echo "" - while read line; do - testcase=${line%,*} - result=${line#*,} - # avoid counting Skip's in the # of tests: - if [ "$result" = "Skip" ]; then - continue; - fi - echo "" - if [ "$result" != "Pass" ]; then - echo "" - echo "$result: See $CI_JOB_URL/artifacts/results/$testcase.xml" - echo "" - fi - echo "" - done < $results - echo "" - echo "" -} - parse_renderer() { RENDERER=`grep -A1 TestCaseResult.\*info.renderer $RESULTS/deqp-info.qpa | grep '||g' | sed 's|||g'` VERSION=`grep -A1 TestCaseResult.\*info.version $RESULTS/deqp-info.qpa | grep '||g' | sed 's|||g'` @@ -266,25 +242,24 @@ DEQP_EXITCODE=$? echo "System load: $(cut -d' ' -f1-3 < /proc/loadavg)" echo "# of CPU cores: $(cat /proc/cpuinfo | grep processor | wc -l)" -# junit is disabled, because it overloads gitlab.freedesktop.org to parse it. -# quiet generate_junit $RESULTS_CSV > $RESULTS/results.xml +# Remove all but the first 50 individual XML files uploaded as artifacts, to +# save fd.o space when you break everything. +find $RESULTS -name \*.xml | \ + sort -n | + sed -n '1,+49!p' | \ + xargs rm -f -# Turn up to the first 50 individual test QPA files from failures or flakes into -# XML results you can view from the browser. -qpas=`find $RESULTS -name \*.qpa -a ! -name deqp-info.qpa` -if [ -n "$qpas" ]; then - shard_qpas=`echo "$qpas" | grep dEQP- | head -n 50` - for qpa in $shard_qpas; do - xml=`echo $qpa | sed 's|\.qpa|.xml|'` - /deqp/executor/testlog-to-xml $qpa $xml - done +# If any QPA XMLs are there, then include the XSL/CSS in our artifacts. +find $RESULTS -name \*.xml \ + -exec cp /deqp/testlog.css /deqp/testlog.xsl "$RESULTS/" ";" \ + -quit - cp /deqp/testlog.css "$RESULTS/" - cp /deqp/testlog.xsl "$RESULTS/" - - # Remove all the QPA files (extracted or not) now that we have the XML we want. - echo $qpas | xargs rm -f -fi +deqp-runner junit \ + --testsuite $DEQP_VER \ + --results $RESULTS/failures.csv \ + --output $RESULTS/junit.xml \ + --limit 50 \ + --template "See https://$CI_PROJECT_NAMESPACE.pages.freedesktop.org/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/results/{{testcase}}.xml" # Report the flakes to the IRC channel for monitoring (if configured): quiet report_flakes $RESULTS_CSV diff --git a/.gitlab-ci/deqp-virgl-gl-fails.txt b/.gitlab-ci/deqp-virgl-gl-fails.txt index eeebd7eca15..0aa551ba4f4 100644 --- a/.gitlab-ci/deqp-virgl-gl-fails.txt +++ b/.gitlab-ci/deqp-virgl-gl-fails.txt @@ -4678,7 +4678,7 @@ KHR-GL30.transform_feedback.query_vertex_separate_test,Fail KHR-GL31.transform_feedback.capture_vertex_interleaved_test,Fail KHR-GL31.transform_feedback.capture_vertex_separate_test,Fail KHR-GL31.transform_feedback.discard_vertex_test,Fail -KHR-GL31.transform_feedback.draw_xfb_instanced_test,Fail +KHR-GL31.transform_feedback.draw_xfb_instanced_test,Crash KHR-GL31.transform_feedback.draw_xfb_stream_instanced_test,Crash KHR-GL31.transform_feedback.query_vertex_interleaved_test,Fail KHR-GL31.transform_feedback.query_vertex_separate_test,Fail diff --git a/.gitlab-ci/lava-gitlab-ci.yml b/.gitlab-ci/lava-gitlab-ci.yml index 0d7a76582c4..45a92e788c6 100644 --- a/.gitlab-ci/lava-gitlab-ci.yml +++ b/.gitlab-ci/lava-gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DISTRIBUTION_TAG: "2020-12-19-build-piglit" + DISTRIBUTION_TAG: "2020-12-22-runner" .kernel+rootfs: stage: container-2