From bbdbf0862c2202606e170ee34fc00017d9967f05 Mon Sep 17 00:00:00 2001 From: Guilherme Gallo Date: Mon, 9 Jan 2023 16:46:28 -0300 Subject: [PATCH] ci/lava: Update lavacli version - Use new YAML loader derived from ruamel.yaml - Remove PyYAML dependency from LAVA job submitter package Signed-off-by: Guilherme Gallo Part-of: --- .gitlab-ci/container/debian/x86_build.sh | 2 +- .gitlab-ci/image-tags.yml | 2 +- .gitlab-ci/lava/lava_job_submitter.py | 16 +++++++++------- .gitlab-ci/lava/utils/log_follower.py | 2 +- .gitlab-ci/tests/lava/helpers.py | 13 ++++++++++--- .gitlab-ci/tests/test_lava_job_submitter.py | 4 ++-- .gitlab-ci/tests/utils/test_lava_log.py | 7 +++---- 7 files changed, 27 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci/container/debian/x86_build.sh b/.gitlab-ci/container/debian/x86_build.sh index 2932f983e51..ea268cb22ad 100644 --- a/.gitlab-ci/container/debian/x86_build.sh +++ b/.gitlab-ci/container/debian/x86_build.sh @@ -92,7 +92,7 @@ ninja install popd rm -rf DirectX-Headers -pip3 install git+https://git.lavasoftware.org/lava/lavacli@3db3ddc45e5358908bc6a17448059ea2340492b7 +pip3 install lavacli==1.5.2 # install bindgen RUSTFLAGS='-L native=/usr/local/lib' cargo install \ diff --git a/.gitlab-ci/image-tags.yml b/.gitlab-ci/image-tags.yml index 859cab63909..1ea1bb41e00 100644 --- a/.gitlab-ci/image-tags.yml +++ b/.gitlab-ci/image-tags.yml @@ -3,7 +3,7 @@ variables: DEBIAN_BASE_TAG: "2023-01-06-virglrenderer" DEBIAN_X86_BUILD_IMAGE_PATH: "debian/x86_build" - DEBIAN_BUILD_TAG: "2023-01-06-mold-1_9" + DEBIAN_BUILD_TAG: "2023-01-09-lavacli" DEBIAN_X86_BUILD_MINGW_IMAGE_PATH: "debian/x86_build-mingw" DEBIAN_BUILD_MINGW_TAG: "2023-01-03-ci-libva-2.17" diff --git a/.gitlab-ci/lava/lava_job_submitter.py b/.gitlab-ci/lava/lava_job_submitter.py index c9b58c84459..5c70a74fd81 100755 --- a/.gitlab-ci/lava/lava_job_submitter.py +++ b/.gitlab-ci/lava/lava_job_submitter.py @@ -20,11 +20,11 @@ import traceback import urllib.parse import xmlrpc.client from datetime import datetime, timedelta +from io import StringIO from os import getenv from typing import Any, Optional import lavacli -import yaml from lava.exceptions import ( MesaCIException, MesaCIKnownIssueException, @@ -42,7 +42,7 @@ from lava.utils import ( hide_sensitive_data, print_log, ) -from lavacli.utils import loader +from lavacli.utils import flow_yaml as lava_yaml # Timeout in seconds to decide if the device from the dispatched LAVA job has # hung or not due to the lack of new log output. @@ -62,7 +62,7 @@ NUMBER_OF_RETRIES_TIMEOUT_DETECTION = int(getenv("LAVA_NUMBER_OF_RETRIES_TIMEOUT NUMBER_OF_ATTEMPTS_LAVA_BOOT = int(getenv("LAVA_NUMBER_OF_ATTEMPTS_LAVA_BOOT", 3)) -def generate_lava_yaml(args): +def generate_lava_yaml_payload(args) -> dict[str, Any]: # General metadata and permissions, plus also inexplicably kernel arguments values = { 'job_name': 'mesa: {}'.format(args.pipeline_info), @@ -189,7 +189,7 @@ def generate_lava_yaml(args): { 'test': test }, ] - return yaml.dump(values, width=10000000) + return values def setup_lava_proxy(): @@ -281,7 +281,7 @@ class LAVAJob: # Let's extract the data data = data.data # When there is no new log data, the YAML is empty - if loaded_lines := yaml.load(data, Loader=loader(False)): + if loaded_lines := lava_yaml.load(data): lines = loaded_lines self.last_log_line += len(lines) return lines @@ -346,7 +346,7 @@ def find_exception_from_metadata(metadata, job_id): def find_lava_error(job) -> None: # Look for infrastructure errors and retry if we see them. results_yaml = _call_proxy(job.proxy.results.get_testjob_results_yaml, job.job_id) - results = yaml.load(results_yaml, Loader=loader(False)) + results = lava_yaml.load(results_yaml) for res in results: metadata = res["metadata"] find_exception_from_metadata(metadata, job.job_id) @@ -513,7 +513,9 @@ def main(args): # script section timeout with a reasonable delay. GL_SECTION_TIMEOUTS[LogSectionType.TEST_CASE] = timedelta(minutes=args.job_timeout) - job_definition = generate_lava_yaml(args) + job_definition_stream = StringIO() + lava_yaml.dump(generate_lava_yaml_payload(args), job_definition_stream) + job_definition = job_definition_stream.getvalue() if args.dump_yaml: with GitlabSection( diff --git a/.gitlab-ci/lava/utils/log_follower.py b/.gitlab-ci/lava/utils/log_follower.py index 7ea34ffd307..36892dce0c7 100644 --- a/.gitlab-ci/lava/utils/log_follower.py +++ b/.gitlab-ci/lava/utils/log_follower.py @@ -224,5 +224,5 @@ def fatal_err(msg): sys.exit(1) -def hide_sensitive_data(yaml_data, hide_tag="HIDEME"): +def hide_sensitive_data(yaml_data: str, hide_tag: str ="HIDEME"): return "".join(line for line in yaml_data.splitlines(True) if hide_tag not in line) diff --git a/.gitlab-ci/tests/lava/helpers.py b/.gitlab-ci/tests/lava/helpers.py index a7b4526392b..d78aa1c10d1 100644 --- a/.gitlab-ci/tests/lava/helpers.py +++ b/.gitlab-ci/tests/lava/helpers.py @@ -1,15 +1,22 @@ from contextlib import nullcontext as does_not_raise from datetime import datetime +from io import StringIO from itertools import cycle -from typing import Callable, Generator, Iterable, Optional, Tuple, Union +from typing import Any, Callable, Generator, Iterable, Optional, Tuple, Union -import yaml from freezegun import freeze_time from lava.utils.log_section import ( DEFAULT_GITLAB_SECTION_TIMEOUTS, FALLBACK_GITLAB_SECTION_TIMEOUT, LogSectionType, ) +from lavacli.utils import flow_yaml as lava_yaml + + +def yaml_dump(data: dict[str, Any]) -> str: + stream = StringIO() + lava_yaml.dump(data, stream) + return stream.getvalue() def section_timeout(section_type: LogSectionType) -> int: @@ -46,7 +53,7 @@ def jobs_logs_response( logs = [timed_msg] if msg is None else msg - return finished, yaml.safe_dump(logs) + return finished, yaml_dump(logs) def section_aware_message_generator( diff --git a/.gitlab-ci/tests/test_lava_job_submitter.py b/.gitlab-ci/tests/test_lava_job_submitter.py index e9f73d8bf5c..88b253fd071 100644 --- a/.gitlab-ci/tests/test_lava_job_submitter.py +++ b/.gitlab-ci/tests/test_lava_job_submitter.py @@ -300,7 +300,7 @@ def test_parse_job_result_from_log(message, expectation, mock_proxy): def test_full_yaml_log(mock_proxy, frozen_time): import random - import yaml + from lavacli.utils import flow_yaml as lava_yaml def time_travel_from_log_chunk(data_chunk): if not data_chunk: @@ -319,7 +319,7 @@ def test_full_yaml_log(mock_proxy, frozen_time): # the same of from the job submitter execution with open("/tmp/log.yaml", "r") as f: first_log = f.readline() - first_log_time = yaml.safe_load(first_log)[0]["dt"] + first_log_time = lava_yaml.load(first_log)[0]["dt"] frozen_time.move_to(first_log_time) def load_lines() -> list: diff --git a/.gitlab-ci/tests/utils/test_lava_log.py b/.gitlab-ci/tests/utils/test_lava_log.py index e04e84635b6..1979d59a1e8 100644 --- a/.gitlab-ci/tests/utils/test_lava_log.py +++ b/.gitlab-ci/tests/utils/test_lava_log.py @@ -8,7 +8,6 @@ from datetime import datetime, timedelta import pytest -import yaml from lava.exceptions import MesaCIKnownIssueException, MesaCITimeoutError from lava.utils import ( GitlabSection, @@ -18,7 +17,7 @@ from lava.utils import ( hide_sensitive_data, ) -from ..lava.helpers import create_lava_yaml_msg, does_not_raise +from ..lava.helpers import create_lava_yaml_msg, does_not_raise, lava_yaml, yaml_dump GITLAB_SECTION_SCENARIOS = { "start collapsed": ( @@ -157,9 +156,9 @@ SENSITIVE_DATA_SCENARIOS = { ids=SENSITIVE_DATA_SCENARIOS.keys(), ) def test_hide_sensitive_data(input, expectation, tag): - yaml_data = yaml.safe_dump(input) + yaml_data = yaml_dump(input) yaml_result = hide_sensitive_data(yaml_data, tag) - result = yaml.safe_load(yaml_result) + result = lava_yaml.load(yaml_result) assert result == expectation