ci: implement farms handling trough files inside .ci-farms

- Disable farm with `git mv .ci-farm{,-disabled}/$farm_name`.
 - Re-enable farm with `git mv .ci-farm{-disabled,}/$farm_name`.

NEVER MIX FARM MAINTENANCE WITH ANY OTHER CHANGE IN THE SAME MERGE REQUEST!

Reviewed-by: Martin Roukala (né Peres) <martin.roukala@mupuf.org>
Acked-by: Daniel Stone <daniels@collabora.com>
Signed-off-by: David Heidelberg <david.heidelberg@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/23629>

.gitlab-ci.yml

@@ -28,16 +28,10 @@ variables:
   JOB_ARTIFACTS_BASE: ${PIPELINE_ARTIFACTS_BASE}/${CI_JOB_ID}
   # reference images stored for traces
   PIGLIT_REPLAY_REFERENCE_IMAGES_BASE: "${S3_HOST}/mesa-tracie-results/$FDO_UPSTREAM_REPO"
-  # Individual CI farm status, set to "offline" to disable jobs
+  # For individual CI farm status see .ci-farms folder
-  # running on a particular CI farm (ie. for outages, etc):
+  # Disable farm with   `git mv .ci-farms{,-disabled}/$farm_name`
-  FD_FARM: "online"
+  # Re-enable farm with `git mv .ci-farms{-disabled,}/$farm_name`
-  COLLABORA_FARM: "online"
+  # NEVER MIX FARM MAINTENANCE WITH ANY OTHER CHANGE IN THE SAME MERGE REQUEST!
-  MICROSOFT_FARM: "online"
-  LIMA_FARM: "offline"
-  IGALIA_FARM: "online"
-  ANHOLT_FARM: "online"
-  VALVE_FARM: "online"
-  AUSTRIANCODER_FARM: "online"  # only etnaviv GPUs
 
 default:
   before_script:
@@ -165,6 +159,7 @@ stages:
       # Source code
       - include/**/*
       - src/**/*
+      - .ci-farms/*
       when: on_success
     # Otherwise, build/test jobs won't run because no rule matched.
 
@@ -209,6 +204,16 @@ stages:
       when: manual
     # Otherwise, container jobs won't run because no rule matched.
 
+.microsoft-farm-container-rules:
+  rules:
+    # Allow triggering jobs manually in other cases if any files affecting the
+    # pipeline were changed
+    - exists: [ .ci-farms-disabled/microsoft ]
+      when: never
+    - changes: [ .ci-farms-disabled/microsoft ]
+      when: manual
+    - !reference [.microsoft-farm-rules, rules]
+
 # Git archive
 
 make git archive:

.gitlab-ci/container/gitlab-ci.yml

@@ -437,7 +437,7 @@ debian/arm64_test:
     - .container
     - .windows-docker-vs2019
   rules:
-    - !reference [.microsoft-farm-rules, rules]
+    - !reference [.microsoft-farm-container-rules, rules]
     - !reference [.container-rules, rules]
   variables:
     GIT_STRATEGY: fetch # we do actually need the full repository though

.gitlab-ci/test-source-dep.yml

@@ -268,41 +268,81 @@
       - src/gallium/frontends/rusticl/**/*
       when: on_success
 
+# Farm rules
+.microsoft-farm-rules:
+  rules:
+    - exists: [ .ci-farms-disabled/microsoft ]  # 1. Is disabled, never run
+      when: never
+    - changes: [ .ci-farms-disabled/microsoft ]  # 2. Removed from disabled, run
+      when: on_success
+    - changes: [ .ci-farms-disabled/* ]  # 3. We touched other farms in MR, do not run
+      when: never
+    # 4. Fall-through (other rules or on_success)
+
 .collabora-farm-rules:
   rules:
-    - if: '$COLLABORA_FARM == "offline" && $RUNNER_TAG =~ /^mesa-ci-x86-64-lava-/'
+    - if: '$RUNNER_TAG =~ /^mesa-ci-x86-64-lava-/'
+      exists: [ .ci-farms-disabled/collabora ]
+      when: never
+    - if: '$RUNNER_TAG =~ /^mesa-ci-x86-64-lava-/'
+      changes: [ .ci-farms-disabled/collabora ]
+      when: on_success
+    - if: '$RUNNER_TAG =~ /^mesa-ci-x86-64-lava-/'
+      changes: [ .ci-farms-disabled/* ]
       when: never
 
 .igalia-farm-rules:
   rules:
-    - if: '$IGALIA_FARM == "offline"'
+    - exists: [ .ci-farms-disabled/igalia ]
+      when: never
+    - changes: [ .ci-farms-disabled/igalia ]
+      when: on_success
+    - changes: [ .ci-farms-disabled/* ]
+      when: never
+
+.lima-farm-rules:
+  rules:
+    - exists: [ .ci-farms-disabled/lima ]
+      when: never
+    - changes: [ .ci-farms-disabled/lima ]
+      when: on_success
+    - changes: [ .ci-farms-disabled/* ]
       when: never
 
 .anholt-farm-rules:
   rules:
-    - if: '$ANHOLT_FARM == "offline"'
+    - exists: [ .ci-farms-disabled/anholt ]
+      when: never
+    - changes: [ .ci-farms-disabled/anholt ]
+      when: on_success
+    - changes: [ .ci-farms-disabled/* ]
       when: never
 
 .valve-farm-rules:
   rules:
-    - if: '$VALVE_FARM == "offline"'
+    - exists: [ .ci-farms-disabled/valve ]
+      when: never
+    - changes: [ .ci-farms-disabled/valve ]
+      when: on_success
+    - changes: [ .ci-farms-disabled/* ]
       when: never
 
 .austriancoder-farm-rules:
   rules:
-    - if: '$AUSTRIANCODER_FARM == "offline"'
+    - exists: [ .ci-farms-disabled/austriancoder ]
+      when: never
+    - changes: [ .ci-farms-disabled/austriancoder ]
+      when: on_success
+    - changes: [ .ci-farms-disabled/* ]
       when: never
 
-# Skips freedreno jobs if either of the farms we use are offline.
 .freedreno-farm-rules:
   rules:
-    - if: '$FD_FARM == "offline"'
+    - exists: [ .ci-farms-disabled/freedreno ]
       when: never
-    - !reference [.collabora-farm-rules, rules]
+    - changes: [ .ci-farms-disabled/freedreno ]
-
+      when: on_success
-.microsoft-farm-rules:
+    - changes: [ .ci-farms-disabled/* ]
-  rules:
-    - if: '$MICROSOFT_FARM == "offline"'
       when: never
 
 # Rules for changes that impact either freedreno or turnip.
@@ -555,8 +595,7 @@
 .lima-rules:
   stage: arm
   rules:
-    - if: '$LIMA_FARM == "offline"'
+    - !reference [.lima-farm-rules, rules]
-      when: never
     - !reference [.gl-rules, rules]
     - changes:
       - src/gallium/drivers/lima/**/*

docs/ci/index.rst

@@ -52,6 +52,23 @@ The three GitLab CI systems currently integrated are:
    LAVA
    docker
 
+Farm management
+---------------
+
+.. note::
+   Never mix farm maintenance with any other change in the same merge request!
+
+When the farm starts failing for any reason (power, network, out-of-space), it needs to be disabled by pushing separate MR with
+.. code-block:: console
+ 
+   git mv .ci-farm{,-disabled}/$farm_name`
+   
+After farm restore functionality can be enabled by pushing a new merge request, which contains
+.. code-block:: console
+
+   git mv .ci-farm{-disabled,}/$farm_name`
+
+
 Application traces replay
 -------------------------