From 057b77d2a383d08547e6ef529a6d83701ee4a937 Mon Sep 17 00:00:00 2001
From: Eric Engestrom <eric@igalia.com>
Date: Mon, 15 Jul 2024 15:15:47 +0200
Subject: [PATCH] ci/vkd3d: add support for *-flakes.txt list files

We still print them in the job log to not completely ignore them, but
they don't break the jobs anymore.

Additionally, we use the same format as deqp-runner for reporting
flakes, so that existing tools can pick up this new information without
requiring any change.

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/30413>
---
 .gitlab-ci/vkd3d-runner.sh   | 52 +++++++++++++++++++++++++++++++++++-
 src/amd/ci/gitlab-ci-inc.yml |  1 +
 2 files changed, 52 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci/vkd3d-runner.sh b/.gitlab-ci/vkd3d-runner.sh
index 3f2bfae2a46..175dcd65890 100755
--- a/.gitlab-ci/vkd3d-runner.sh
+++ b/.gitlab-ci/vkd3d-runner.sh
@@ -57,6 +57,35 @@ else
     touch "$EXPECTATIONFILE"
 fi
 
+if [ -f "$INSTALL/$GPU_VERSION-vkd3d-flakes.txt" ]; then
+  mapfile -t flakes < <(grep -vE '^#|^$' "$INSTALL/$GPU_VERSION-vkd3d-flakes.txt")
+else
+  flakes=()
+fi
+
+# Some sanity checks before we start
+mapfile -t flakes_dups < <(
+  [ ${#flakes[@]} -eq 0 ] ||
+  printf '%s\n' "${flakes[@]}" | sort | uniq -d
+)
+if [ ${#flakes_dups[@]} -gt 0 ]; then
+  echo >&2 'Duplicate flakes lines:'
+  printf >&2 '  %s\n' "${flakes_dups[@]}"
+  exit 1
+fi
+
+flakes_in_baseline=()
+for flake in "${flakes[@]}"; do
+  if grep -qF "$flake" "$EXPECTATIONFILE"; then
+    flakes_in_baseline+=("$flake")
+  fi
+done
+if [ ${#flakes_in_baseline[@]} -gt 0 ]; then
+  echo >&2 "Flakes found in $EXPECTATIONFILE:"
+  printf >&2 '  %s\n' "${flakes_in_baseline[@]}"
+  exit 1
+fi
+
 printf "%s\n" "Running vkd3d-proton testsuite..."
 
 if ! /vkd3d-proton-tests/x64/bin/d3d12 &> "$RESULTS/vkd3d-proton-log.txt"; then
@@ -78,11 +107,32 @@ if ! /vkd3d-proton-tests/x64/bin/d3d12 &> "$RESULTS/vkd3d-proton-log.txt"; then
       exit 1
     fi
 
+    # Ignore flakes when comparing
+    STABLERESULTSFILE="$RESULTS/$GPU_VERSION-results-minus-flakes.txt"
+    cp "$RESULTSFILE" "$STABLERESULTSFILE"
+    for flake in "${flakes[@]}"; do
+      grep -vF "$flake" "$STABLERESULTSFILE" > tmp && mv tmp "$STABLERESULTSFILE"
+    done
+
     # Make sure that the failures found in this run match the current expectation
-    if ! diff --color=always -u "$EXPECTATIONFILE" "$RESULTSFILE"; then
+    if ! diff --color=always -u "$EXPECTATIONFILE" "$STABLERESULTSFILE"; then
         error "Changes found, see ${ARTIFACTS_BASE_URL}/results/vkd3d-proton-log.txt"
         exit 1
     fi
+
+    # Print list of flakes seen this time
+    flakes_seen=()
+    for flake in "${flakes[@]}"; do
+      if grep -qF "$flake" "$RESULTSFILE"; then
+        flakes_seen+=("$flake")
+      fi
+    done
+    if [ ${#flakes_seen[@]} -gt 0 ]; then
+      # Keep this string and output format in line with the corresponding
+      # deqp-runner message
+      echo >&2 'Some known flakes found:'
+      printf >&2 '  %s\n' "${flakes_seen[@]}"
+    fi
 fi
 
 exit 0
diff --git a/src/amd/ci/gitlab-ci-inc.yml b/src/amd/ci/gitlab-ci-inc.yml
index 2f3c98c54aa..10a9c86ae09 100644
--- a/src/amd/ci/gitlab-ci-inc.yml
+++ b/src/amd/ci/gitlab-ci-inc.yml
@@ -15,6 +15,7 @@
         - src/amd/ci/$GPU_VERSION-skips.txt
         - src/amd/ci/$PIGLIT_TRACES_FILE
         - src/amd/ci/$GPU_VERSION-vkd3d-fails.txt
+        - src/amd/ci/$GPU_VERSION-vkd3d-flakes.txt
         - src/amd/ci/$GPU_VERSION-vkd3d-skips.txt
         - src/amd/common/**/*
         - src/amd/llvm/**/*