From 3975366a88711b986a283f1b7ea56782506280b2 Mon Sep 17 00:00:00 2001 From: Guilherme Gallo Date: Thu, 25 Jul 2024 00:34:31 -0300 Subject: [PATCH] bin/ci: crnm: Deal with `created` status Jobs that we want to run can get the `created` status when its dependencies are still running. So let's gather this information and ensure that we will wait these jobs to reach the `manual` or `running` status before jumping to monitor target jobs trace. Closes: #11517 Signed-off-by: Guilherme Gallo Part-of: --- bin/ci/ci_run_n_monitor.py | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/bin/ci/ci_run_n_monitor.py b/bin/ci/ci_run_n_monitor.py index 80340cb13cf..bf7f7d1da90 100755 --- a/bin/ci/ci_run_n_monitor.py +++ b/bin/ci/ci_run_n_monitor.py @@ -133,11 +133,23 @@ def monitor_pipeline( stress_status_counter[job.name][job.status] += 1 execution_times[job.name][job.id] = (job_duration(job), job.status, job.web_url) + # jobs_waiting is a list of job names that are waiting for status update. + # It occurs when a job that we want to run depends on another job that is not yet finished. + jobs_waiting = [] + # FIXME: This function has too many parameters, consider refactoring. + enable_job_fn = partial( + enable_job, + project=project, + pipeline=pipeline, + force_manual=force_manual, + job_name_field_pad=name_field_pad, + jobs_waiting=jobs_waiting, + ) while True: deps_failed = [] to_cancel = [] + jobs_waiting.clear() for job in sorted(pipeline.jobs.list(all=True), key=lambda j: j.name): - # target jobs if target_jobs_regex.fullmatch(job.name): target_id = job.id target_status = job.status @@ -149,10 +161,10 @@ def monitor_pipeline( ): stress_status_counter[job.name][target_status] += 1 execution_times[job.name][job.id] = (job_duration(job), target_status, job.web_url) - job = enable_job(project, pipeline, job, "retry", force_manual, name_field_pad) + job = enable_job_fn(job=job, action_type="retry") else: execution_times[job.name][job.id] = (job_duration(job), target_status, job.web_url) - job = enable_job(project, pipeline, job, "target", force_manual, name_field_pad) + job = enable_job_fn(job=job, action_type="target") print_job_status(job, target_status not in target_statuses[job.name], name_field_pad) target_statuses[job.name] = target_status @@ -165,7 +177,7 @@ def monitor_pipeline( # run dependencies and cancel the rest if job.name in dependencies: - job = enable_job(project, pipeline, job, "dep", True, name_field_pad) + job = enable_job_fn(job=job, action_type="dep") if job.status == "failed": deps_failed.append(job.name) else: @@ -191,6 +203,15 @@ def monitor_pipeline( print("---------------------------------", flush=False) + if jobs_waiting: + print( + f"{Fore.YELLOW}Waiting for jobs to update status:", + ", ".join(jobs_waiting), + Fore.RESET, + ) + pretty_wait(REFRESH_WAIT_JOBS) + continue + if len(target_statuses) == 1 and RUNNING_STATUSES.intersection( target_statuses.values() ): @@ -235,8 +256,14 @@ def enable_job( action_type: Literal["target", "dep", "retry"], force_manual: bool, job_name_field_pad: int = 0, + jobs_waiting: list[str] = [], ) -> gitlab.v4.objects.ProjectPipelineJob: - """enable job""" + # We want to run this job, but it is not ready to run yet, so let's try again in the next + # iteration. + if job.status == "created": + jobs_waiting.append(job.name) + return job + if ( (job.status in COMPLETED_STATUSES and action_type != "retry") or (job.status == "manual" and not force_manual)