ci: introduce tool for comparing nightly runs

nightly_compare.
Compare the two latest scheduled pipelines and provide information
about the jobs you're interested in.

The job part reports:
 - RED previously passing jobs
 - YELLOW jobs which failed before, but continue to fail
   here is also available link to the previous failed run
 - If no job failing, program exits.

The test part reports:
 - everything in lovely table

If any failing job is found, after this phase, commit list between these
two scheduled run is printed (you can also use the WebUI link).

Example: I care about all Adreno jobs passing and one Radeon (r300)
with gallium-nine tests.

```
./bin/ci/nightly_compare.py --target "a[3-7][0-9][0-9].*|r300-rv530-nine"
```

Co-authored-by: Sergi Blanch Torne <sergi.blanch.torne@collabora.com>
Signed-off-by: David Heidelberg <david@ixit.cz>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/29392>

bin/ci/nightly_compare.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+# Copyright © 2020 - 2024 Collabora Ltd.
+# Authors:
+#   David Heidelberg <david.heidelberg@collabora.com>
+#   Sergi Blanch Torne <sergi.blanch.torne@collabora.com>
+# SPDX-License-Identifier: MIT
+
+"""
+Compare the two latest scheduled pipelines and provide information
+about the jobs you're interested in.
+"""
+
+import argparse
+import csv
+import re
+import requests
+import io
+from tabulate import tabulate
+
+import gitlab
+from colorama import Fore, Style
+from gitlab_common import read_token
+
+
+MARGE_BOT_USER_ID = 9716
+
+def print_failures_csv(id):
+    url = 'https://gitlab.freedesktop.org/mesa/mesa/-/jobs/' + str(id) + '/artifacts/raw/results/failures.csv'
+    missing: int = 0
+    MAX_MISS: int = 20
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        csv_content = io.StringIO(response.text)
+        csv_reader = csv.reader(csv_content)
+        data = list(csv_reader)
+
+        for line in data[:]:
+            if line[1] == "UnexpectedImprovement(Pass)":
+                line[1] = Fore.GREEN + line[1] + Style.RESET_ALL
+            elif line[1] == "UnexpectedImprovement(Fail)":
+                line[1] = Fore.YELLOW + line[1] + Style.RESET_ALL
+            elif line[1] == "Crash" or line[1] == "Fail":
+                line[1] = Fore.RED + line[1] + Style.RESET_ALL
+            elif line[1] == "Missing":
+                if missing > MAX_MISS:
+                    data.remove(line)
+                    continue
+                missing += 1
+                line[1] = Fore.YELLOW + line[1] + Style.RESET_ALL
+            elif line[1] == "Fail":
+                line[1] = Fore.RED + line[1] + Style.RESET_ALL
+            else:
+                line[1] = Fore.WHITE + line[1] + Style.RESET_ALL
+
+        if missing > MAX_MISS:
+            data.append([Fore.RED + f"... more than {MAX_MISS} missing tests, something crashed?", "Missing" + Style.RESET_ALL])
+        headers = ["Test                                                                           ", "Result"]
+        print(tabulate(data, headers, tablefmt="plain"))
+    except Exception:
+        pass
+
+
+def job_failed_before(old_jobs, job):
+    for old_job in old_jobs:
+        if job.name == old_job.name:
+            return old_job
+
+
+def parse_args() -> None:
+    """Parse args"""
+    parser = argparse.ArgumentParser(
+        description="Tool to show merge requests assigned to the marge-bot",
+    )
+    parser.add_argument(
+        "--target",
+        metavar="target-job",
+        help="Target job regex. For multiple targets, pass multiple values, "
+        "eg. `--target foo bar`.",
+        required=False,
+        nargs=argparse.ONE_OR_MORE,
+    )
+    parser.add_argument(
+        "--token",
+        metavar="token",
+        help="force GitLab token, otherwise it's read from ~/.config/gitlab-token",
+    )
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    token = read_token(args.token)
+    gl = gitlab.Gitlab(url="https://gitlab.freedesktop.org", private_token=token)
+
+    project = gl.projects.get("mesa/mesa")
+
+    print(
+        "\u001b]8;;https://gitlab.freedesktop.org/mesa/mesa/-/pipelines?page=1&scope=all&source=schedule\u001b\\Scheduled pipelines overview\u001b]8;;\u001b\\"
+    )
+    pipelines = project.pipelines.list(
+        source="schedule", ordered_by="created_at", sort="desc", page=1, per_page=2
+    )
+    print(
+        f"Old pipeline: {pipelines[1].created_at}\t\u001b]8;;{pipelines[1].web_url}\u001b\\{pipelines[1].status}\u001b]8;;\u001b\\\t{pipelines[1].sha}"
+    )
+    print(
+        f"New pipeline: {pipelines[0].created_at}\t\u001b]8;;{pipelines[0].web_url}\u001b\\{pipelines[0].status}\u001b]8;;\u001b\\\t{pipelines[0].sha}"
+    )
+    print(
+        f"\nWebUI visual compare: https://gitlab.freedesktop.org/mesa/mesa/-/compare/{pipelines[1].sha}...{pipelines[0].sha}\n"
+    )
+
+    # regex part
+    if args.target:
+        target = "|".join(args.target)
+        target = target.strip()
+        print("🞋 jobs: " + Fore.BLUE + target + Style.RESET_ALL)
+
+        target = f"({target})" + r"( \d+/\d+)?"
+    else:
+        target = ".*"
+
+    target_jobs_regex: re.Pattern = re.compile(target)
+
+    old_failed_jobs = []
+    for job in pipelines[1].jobs.list(all=True):
+        if (
+            job.status != "failed"
+            or target_jobs_regex
+            and not target_jobs_regex.fullmatch(job.name)
+        ):
+            continue
+        old_failed_jobs.append(job)
+
+    job_failed = False
+    for job in pipelines[0].jobs.list(all=True):
+        if (
+            job.status != "failed"
+            or target_jobs_regex
+            and not target_jobs_regex.fullmatch(job.name)
+        ):
+            continue
+
+        job_failed = True
+
+        previously_failed_job = job_failed_before(old_failed_jobs, job)
+        if previously_failed_job:
+            print(
+                Fore.YELLOW
+                + f":: \u001b]8;;{job.web_url}\u001b\\{job.name}\u001b]8;;\u001b\\"
+                + Fore.MAGENTA
+                + f" \u001b]8;;{previously_failed_job.web_url}\u001b\\(previous run)\u001b]8;;\u001b\\"
+                + Style.RESET_ALL
+            )
+        else:
+            print(
+                Fore.RED
+                + f":: \u001b]8;;{job.web_url}\u001b\\{job.name}\u001b]8;;\u001b\\"
+                + Style.RESET_ALL
+            )
+        print_failures_csv(job.id)
+
+    if not job_failed:
+        exit(0)
+
+    print("Commits between nightly pipelines:")
+    commit = project.commits.get(pipelines[0].sha)
+    while True:
+        print(
+            f"{commit.id}  \u001b]8;;{commit.web_url}\u001b\\{commit.title}\u001b]8;;\u001b\\"
+        )
+        if commit.id == pipelines[1].sha:
+            break
+        commit = project.commits.get(commit.parent_ids[0])

bin/ci/nightly_compare.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -eu
+
+this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
+readonly this_dir
+
+exec \
+  "$this_dir/../python-venv.sh" \
+  "$this_dir/requirements.txt" \
+  "$this_dir/nightly_compare.py" "$@"
+

bin/ci/requirements.txt

@@ -9,3 +9,4 @@ python-gitlab==3.5.0
 PyYAML==6.0.1
 ruamel.yaml.clib==0.2.8
 ruamel.yaml==0.17.21
+tabulate==0.9.0