ci/vkd3d: add support for *-flakes.txt list files

We still print them in the job log to not completely ignore them, but
they don't break the jobs anymore.

Additionally, we use the same format as deqp-runner for reporting
flakes, so that existing tools can pick up this new information without
requiring any change.

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/30413>

.gitlab-ci/vkd3d-runner.sh

@@ -57,6 +57,35 @@ else
     touch "$EXPECTATIONFILE"
 fi
 
+if [ -f "$INSTALL/$GPU_VERSION-vkd3d-flakes.txt" ]; then
+  mapfile -t flakes < <(grep -vE '^#|^$' "$INSTALL/$GPU_VERSION-vkd3d-flakes.txt")
+else
+  flakes=()
+fi
+
+# Some sanity checks before we start
+mapfile -t flakes_dups < <(
+  [ ${#flakes[@]} -eq 0 ] ||
+  printf '%s\n' "${flakes[@]}" | sort | uniq -d
+)
+if [ ${#flakes_dups[@]} -gt 0 ]; then
+  echo >&2 'Duplicate flakes lines:'
+  printf >&2 '  %s\n' "${flakes_dups[@]}"
+  exit 1
+fi
+
+flakes_in_baseline=()
+for flake in "${flakes[@]}"; do
+  if grep -qF "$flake" "$EXPECTATIONFILE"; then
+    flakes_in_baseline+=("$flake")
+  fi
+done
+if [ ${#flakes_in_baseline[@]} -gt 0 ]; then
+  echo >&2 "Flakes found in $EXPECTATIONFILE:"
+  printf >&2 '  %s\n' "${flakes_in_baseline[@]}"
+  exit 1
+fi
+
 printf "%s\n" "Running vkd3d-proton testsuite..."
 
 if ! /vkd3d-proton-tests/x64/bin/d3d12 &> "$RESULTS/vkd3d-proton-log.txt"; then
@@ -78,11 +107,32 @@ if ! /vkd3d-proton-tests/x64/bin/d3d12 &> "$RESULTS/vkd3d-proton-log.txt"; then
       exit 1
     fi
 
+    # Ignore flakes when comparing
+    STABLERESULTSFILE="$RESULTS/$GPU_VERSION-results-minus-flakes.txt"
+    cp "$RESULTSFILE" "$STABLERESULTSFILE"
+    for flake in "${flakes[@]}"; do
+      grep -vF "$flake" "$STABLERESULTSFILE" > tmp && mv tmp "$STABLERESULTSFILE"
+    done
+
     # Make sure that the failures found in this run match the current expectation
-    if ! diff --color=always -u "$EXPECTATIONFILE" "$RESULTSFILE"; then
+    if ! diff --color=always -u "$EXPECTATIONFILE" "$STABLERESULTSFILE"; then
         error "Changes found, see ${ARTIFACTS_BASE_URL}/results/vkd3d-proton-log.txt"
         exit 1
     fi
+
+    # Print list of flakes seen this time
+    flakes_seen=()
+    for flake in "${flakes[@]}"; do
+      if grep -qF "$flake" "$RESULTSFILE"; then
+        flakes_seen+=("$flake")
+      fi
+    done
+    if [ ${#flakes_seen[@]} -gt 0 ]; then
+      # Keep this string and output format in line with the corresponding
+      # deqp-runner message
+      echo >&2 'Some known flakes found:'
+      printf >&2 '  %s\n' "${flakes_seen[@]}"
+    fi
 fi
 
 exit 0

src/amd/ci/gitlab-ci-inc.yml

@@ -15,6 +15,7 @@
         - src/amd/ci/$GPU_VERSION-skips.txt
         - src/amd/ci/$PIGLIT_TRACES_FILE
         - src/amd/ci/$GPU_VERSION-vkd3d-fails.txt
+        - src/amd/ci/$GPU_VERSION-vkd3d-flakes.txt
         - src/amd/ci/$GPU_VERSION-vkd3d-skips.txt
         - src/amd/common/**/*
         - src/amd/llvm/**/*