ci/lava: Update lavacli version

- Use new YAML loader derived from ruamel.yaml
- Remove PyYAML dependency from LAVA job submitter package

Signed-off-by: Guilherme Gallo <guilherme.gallo@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/20596>

.gitlab-ci/container/debian/x86_build.sh

@@ -92,7 +92,7 @@ ninja install
 popd
 rm -rf DirectX-Headers
 
-pip3 install git+https://git.lavasoftware.org/lava/lavacli@3db3ddc45e5358908bc6a17448059ea2340492b7
+pip3 install lavacli==1.5.2
 
 # install bindgen
 RUSTFLAGS='-L native=/usr/local/lib' cargo install \

.gitlab-ci/image-tags.yml

@@ -3,7 +3,7 @@ variables:
    DEBIAN_BASE_TAG: "2023-01-06-virglrenderer"
 
    DEBIAN_X86_BUILD_IMAGE_PATH: "debian/x86_build"
-   DEBIAN_BUILD_TAG: "2023-01-06-mold-1_9"
+   DEBIAN_BUILD_TAG: "2023-01-09-lavacli"
 
    DEBIAN_X86_BUILD_MINGW_IMAGE_PATH: "debian/x86_build-mingw"
    DEBIAN_BUILD_MINGW_TAG: "2023-01-03-ci-libva-2.17"

.gitlab-ci/lava/lava_job_submitter.py

@@ -20,11 +20,11 @@ import traceback
 import urllib.parse
 import xmlrpc.client
 from datetime import datetime, timedelta
+from io import StringIO
 from os import getenv
 from typing import Any, Optional
 
 import lavacli
-import yaml
 from lava.exceptions import (
     MesaCIException,
     MesaCIKnownIssueException,
@@ -42,7 +42,7 @@ from lava.utils import (
     hide_sensitive_data,
     print_log,
 )
-from lavacli.utils import loader
+from lavacli.utils import flow_yaml as lava_yaml
 
 # Timeout in seconds to decide if the device from the dispatched LAVA job has
 # hung or not due to the lack of new log output.
@@ -62,7 +62,7 @@ NUMBER_OF_RETRIES_TIMEOUT_DETECTION = int(getenv("LAVA_NUMBER_OF_RETRIES_TIMEOUT
 NUMBER_OF_ATTEMPTS_LAVA_BOOT = int(getenv("LAVA_NUMBER_OF_ATTEMPTS_LAVA_BOOT", 3))
 
 
-def generate_lava_yaml(args):
+def generate_lava_yaml_payload(args) -> dict[str, Any]:
     # General metadata and permissions, plus also inexplicably kernel arguments
     values = {
         'job_name': 'mesa: {}'.format(args.pipeline_info),
@@ -189,7 +189,7 @@ def generate_lava_yaml(args):
       { 'test': test },
     ]
 
-    return yaml.dump(values, width=10000000)
+    return values
 
 
 def setup_lava_proxy():
@@ -281,7 +281,7 @@ class LAVAJob:
             # Let's extract the data
             data = data.data
         # When there is no new log data, the YAML is empty
-        if loaded_lines := yaml.load(data, Loader=loader(False)):
+        if loaded_lines := lava_yaml.load(data):
             lines = loaded_lines
             self.last_log_line += len(lines)
         return lines
@@ -346,7 +346,7 @@ def find_exception_from_metadata(metadata, job_id):
 def find_lava_error(job) -> None:
     # Look for infrastructure errors and retry if we see them.
     results_yaml = _call_proxy(job.proxy.results.get_testjob_results_yaml, job.job_id)
-    results = yaml.load(results_yaml, Loader=loader(False))
+    results = lava_yaml.load(results_yaml)
     for res in results:
         metadata = res["metadata"]
         find_exception_from_metadata(metadata, job.job_id)
@@ -513,7 +513,9 @@ def main(args):
     # script section timeout with a reasonable delay.
     GL_SECTION_TIMEOUTS[LogSectionType.TEST_CASE] = timedelta(minutes=args.job_timeout)
 
-    job_definition = generate_lava_yaml(args)
+    job_definition_stream = StringIO()
+    lava_yaml.dump(generate_lava_yaml_payload(args), job_definition_stream)
+    job_definition = job_definition_stream.getvalue()
 
     if args.dump_yaml:
         with GitlabSection(

.gitlab-ci/lava/utils/log_follower.py

@@ -224,5 +224,5 @@ def fatal_err(msg):
     sys.exit(1)
 
 
-def hide_sensitive_data(yaml_data, hide_tag="HIDEME"):
+def hide_sensitive_data(yaml_data: str, hide_tag: str ="HIDEME"):
     return "".join(line for line in yaml_data.splitlines(True) if hide_tag not in line)

.gitlab-ci/tests/lava/helpers.py

@@ -1,15 +1,22 @@
 from contextlib import nullcontext as does_not_raise
 from datetime import datetime
+from io import StringIO
 from itertools import cycle
-from typing import Callable, Generator, Iterable, Optional, Tuple, Union
+from typing import Any, Callable, Generator, Iterable, Optional, Tuple, Union
 
-import yaml
 from freezegun import freeze_time
 from lava.utils.log_section import (
     DEFAULT_GITLAB_SECTION_TIMEOUTS,
     FALLBACK_GITLAB_SECTION_TIMEOUT,
     LogSectionType,
 )
+from lavacli.utils import flow_yaml as lava_yaml
+
+
+def yaml_dump(data: dict[str, Any]) -> str:
+    stream = StringIO()
+    lava_yaml.dump(data, stream)
+    return stream.getvalue()
 
 
 def section_timeout(section_type: LogSectionType) -> int:
@@ -46,7 +53,7 @@ def jobs_logs_response(
 
     logs = [timed_msg] if msg is None else msg
 
-    return finished, yaml.safe_dump(logs)
+    return finished, yaml_dump(logs)
 
 
 def section_aware_message_generator(

.gitlab-ci/tests/test_lava_job_submitter.py

@@ -300,7 +300,7 @@ def test_parse_job_result_from_log(message, expectation, mock_proxy):
 def test_full_yaml_log(mock_proxy, frozen_time):
     import random
 
-    import yaml
+    from lavacli.utils import flow_yaml as lava_yaml
 
     def time_travel_from_log_chunk(data_chunk):
         if not data_chunk:
@@ -319,7 +319,7 @@ def test_full_yaml_log(mock_proxy, frozen_time):
         # the same of from the job submitter execution
         with open("/tmp/log.yaml", "r") as f:
             first_log = f.readline()
-            first_log_time = yaml.safe_load(first_log)[0]["dt"]
+            first_log_time = lava_yaml.load(first_log)[0]["dt"]
             frozen_time.move_to(first_log_time)
 
     def load_lines() -> list:

.gitlab-ci/tests/utils/test_lava_log.py

@@ -8,7 +8,6 @@
 from datetime import datetime, timedelta
 
 import pytest
-import yaml
 from lava.exceptions import MesaCIKnownIssueException, MesaCITimeoutError
 from lava.utils import (
     GitlabSection,
@@ -18,7 +17,7 @@ from lava.utils import (
     hide_sensitive_data,
 )
 
-from ..lava.helpers import create_lava_yaml_msg, does_not_raise
+from ..lava.helpers import create_lava_yaml_msg, does_not_raise, lava_yaml, yaml_dump
 
 GITLAB_SECTION_SCENARIOS = {
     "start collapsed": (
@@ -157,9 +156,9 @@ SENSITIVE_DATA_SCENARIOS = {
     ids=SENSITIVE_DATA_SCENARIOS.keys(),
 )
 def test_hide_sensitive_data(input, expectation, tag):
-    yaml_data = yaml.safe_dump(input)
+    yaml_data = yaml_dump(input)
     yaml_result = hide_sensitive_data(yaml_data, tag)
-    result = yaml.safe_load(yaml_result)
+    result = lava_yaml.load(yaml_result)
 
     assert result == expectation