bin/ci: crnm: Deal with created status

Jobs that we want to run can get the `created` status when its
dependencies are still running.

So let's gather this information and ensure that we will wait these jobs
to reach the `manual` or `running` status before jumping to monitor
target jobs trace.

Closes: #11517

Signed-off-by: Guilherme Gallo <guilherme.gallo@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/30361>

bin/ci/ci_run_n_monitor.py

@@ -133,11 +133,23 @@ def monitor_pipeline(
                 stress_status_counter[job.name][job.status] += 1
                 execution_times[job.name][job.id] = (job_duration(job), job.status, job.web_url)
 
+    # jobs_waiting is a list of job names that are waiting for status update.
+    # It occurs when a job that we want to run depends on another job that is not yet finished.
+    jobs_waiting = []
+    # FIXME: This function has too many parameters, consider refactoring.
+    enable_job_fn = partial(
+        enable_job,
+        project=project,
+        pipeline=pipeline,
+        force_manual=force_manual,
+        job_name_field_pad=name_field_pad,
+        jobs_waiting=jobs_waiting,
+    )
     while True:
         deps_failed = []
         to_cancel = []
+        jobs_waiting.clear()
         for job in sorted(pipeline.jobs.list(all=True), key=lambda j: j.name):
-            # target jobs
             if target_jobs_regex.fullmatch(job.name):
                 target_id = job.id
                 target_status = job.status
@@ -149,10 +161,10 @@ def monitor_pipeline(
                     ):
                         stress_status_counter[job.name][target_status] += 1
                         execution_times[job.name][job.id] = (job_duration(job), target_status, job.web_url)
-                        job = enable_job(project, pipeline, job, "retry", force_manual, name_field_pad)
+                        job = enable_job_fn(job=job, action_type="retry")
                 else:
                     execution_times[job.name][job.id] = (job_duration(job), target_status, job.web_url)
-                    job = enable_job(project, pipeline, job, "target", force_manual, name_field_pad)
+                    job = enable_job_fn(job=job, action_type="target")
 
                 print_job_status(job, target_status not in target_statuses[job.name], name_field_pad)
                 target_statuses[job.name] = target_status
@@ -165,7 +177,7 @@ def monitor_pipeline(
 
             # run dependencies and cancel the rest
             if job.name in dependencies:
-                job = enable_job(project, pipeline, job, "dep", True, name_field_pad)
+                job = enable_job_fn(job=job, action_type="dep")
                 if job.status == "failed":
                     deps_failed.append(job.name)
             else:
@@ -191,6 +203,15 @@ def monitor_pipeline(
 
         print("---------------------------------", flush=False)
 
+        if jobs_waiting:
+            print(
+                f"{Fore.YELLOW}Waiting for jobs to update status:",
+                ", ".join(jobs_waiting),
+                Fore.RESET,
+            )
+            pretty_wait(REFRESH_WAIT_JOBS)
+            continue
+
         if len(target_statuses) == 1 and RUNNING_STATUSES.intersection(
             target_statuses.values()
         ):
@@ -235,8 +256,14 @@ def enable_job(
     action_type: Literal["target", "dep", "retry"],
     force_manual: bool,
     job_name_field_pad: int = 0,
+    jobs_waiting: list[str] = [],
 ) -> gitlab.v4.objects.ProjectPipelineJob:
-    """enable job"""
+    # We want to run this job, but it is not ready to run yet, so let's try again in the next
+    # iteration.
+    if job.status == "created":
+        jobs_waiting.append(job.name)
+        return job
+
     if (
         (job.status in COMPLETED_STATUSES and action_type != "retry")
         or (job.status == "manual" and not force_manual)