ci/deqp-runner: implement max-fails

If we get more than 40 unexpected fails/crashes in a merge pipeline,
then stop early to preserve pipeline resources.

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/31429>

.gitlab-ci.yml

@@ -36,6 +36,8 @@ workflow:
         KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${KERNEL_TAG}
         MESA_CI_PERFORMANCE_ENABLED: 1
         VALVE_INFRA_VANGOGH_JOB_PRIORITY: ""  # Empty tags are ignored by gitlab
+        # fast-fail in merge pipelines: stop early if we get this many unexpected fails/crashes
+        DEQP_RUNNER_MAX_FAILS: 40
     # post-merge pipeline
     - if: &is-post-merge $GITLAB_USER_LOGIN == "marge-bot" && $CI_PIPELINE_SOURCE == "push"
     # nightly pipeline

.gitlab-ci/common/generate-env.sh

@@ -35,6 +35,7 @@ VARS=(
     DEQP_EXPECTED_RENDERER
     DEQP_FRACTION
     DEQP_HEIGHT
+    DEQP_RUNNER_MAX_FAILS
     DEQP_RUNNER_OPTIONS
     DEQP_SUITE
     DEQP_TEMP_DIR

.gitlab-ci/deqp-runner.sh

@@ -128,6 +128,10 @@ if [ -n "$VK_DRIVER" ] && [ -z "$DEQP_SUITE" ]; then
     DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --tests-per-group ${DEQP_RUNNER_TESTS_PER_GROUP:-5000}"
 fi
 
+if [ -n "${DEQP_RUNNER_MAX_FAILS:-}" ]; then
+    DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --max-fails ${DEQP_RUNNER_MAX_FAILS}"
+fi
+
 # Set the path to VK validation layer settings (in case it ends up getting loaded)
 # Note: If you change the format of this filename, look through the rest of the
 # tree for other places that need to be kept in sync (e.g.