ci/lava: Feed yaml.load with raw bytes data

LAVA uses XMLRPC to send jobs information and control, more specifically
it sends device logs via YAML dumps encoded in UTF-8 bytes.

In Python, we have xmlrpc.client.Binary class as the serializer
protocol, we get the logs wrapped by this class, which encodes the data
as UTF-8 bytes data.

We were converting the encoded data to a string via the `str` function,
but this led the loaded YAML data to use single quotes instead of double
quotes for string values that made special characters, such as `\x1b` to
be escaped as `\\x1b`.

With this fix, we can now drop one of the hacks that fixed the bash
colors.

Signed-off-by: Guilherme Gallo <guilherme.gallo@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/20051>

.gitlab-ci/lava/lava_job_submitter.py

@@ -272,8 +272,12 @@ class LAVAJob:
 
     def _load_log_from_data(self, data) -> list[str]:
         lines = []
+        if isinstance(data, xmlrpc.client.Binary):
+            # We are dealing with xmlrpc.client.Binary
+            # Let's extract the data
+            data = data.data
         # When there is no new log data, the YAML is empty
-        if loaded_lines := yaml.load(str(data), Loader=loader(False)):
+        if loaded_lines := yaml.load(data, Loader=loader(False)):
             lines = loaded_lines
             self.last_log_line += len(lines)
         return lines

.gitlab-ci/lava/utils/__init__.py

@@ -3,7 +3,6 @@ from .gitlab_section import GitlabSection
 from .log_follower import (
     LogFollower,
     fatal_err,
-    fix_lava_color_log,
     fix_lava_gitlab_section_log,
     hide_sensitive_data,
     print_log,

.gitlab-ci/lava/utils/log_follower.py

@@ -150,19 +150,6 @@ class LogFollower:
         self._buffer = []
         return buffer
 
-
-def fix_lava_color_log(line):
-    """This function is a temporary solution for the color escape codes mangling
-    problem. There is some problem in message passing between the LAVA
-    dispatcher and the device under test (DUT). Here \x1b character is missing
-    before `[:digit::digit:?:digit:?m` ANSI TTY color codes, or the more
-    complicated ones with number values for text format before background and
-    foreground colors.
-    When this problem is fixed on the LAVA side, one should remove this function.
-    """
-    line["msg"] = re.sub(r"(\[(\d+;){0,2}\d{1,3}m)", "\x1b" + r"\1", line["msg"])
-
-
 def fix_lava_gitlab_section_log(line):
     """This function is a temporary solution for the Gitlab section markers
     mangling problem. Gitlab parses the following lines to define a collapsible
@@ -195,7 +182,6 @@ def parse_lava_line(line) -> Optional[str]:
         prefix = "$ "
         suffix = ""
     elif line["lvl"] == "target":
-        fix_lava_color_log(line)
         fix_lava_gitlab_section_log(line)
 
     return f'{prefix}{line["msg"]}{suffix}'

.gitlab-ci/tests/test_lava_job_submitter.py

@@ -298,9 +298,7 @@ def test_parse_job_result_from_log(message, expectation, mock_proxy):
     reason="Slow and sketchy test. Needs a LAVA log raw file at /tmp/log.yaml"
 )
 def test_full_yaml_log(mock_proxy, frozen_time):
-    import itertools
     import random
-    from datetime import datetime
 
     import yaml
 
@@ -326,21 +324,23 @@ def test_full_yaml_log(mock_proxy, frozen_time):
 
     def load_lines() -> list:
         with open("/tmp/log.yaml", "r") as f:
-            data = yaml.safe_load(f)
-            chain = itertools.chain(data)
+            # data = yaml.safe_load(f)
+            data = f.readlines()
+            stream = chain(data)
             try:
                 while True:
-                    data_chunk = [next(chain) for _ in range(random.randint(0, 50))]
+                    data_chunk = [next(stream) for _ in range(random.randint(0, 50))]
+                    serial_message = "".join(data_chunk)
                     # Suppose that the first message timestamp is the same of
                     # log fetch RPC call
                     time_travel_from_log_chunk(data_chunk)
-                    yield False, []
+                    yield False, "[]"
                     # Travel to the same datetime of the last fetched log line
                     # in the chunk
                     time_travel_from_log_chunk(data_chunk)
-                    yield False, data_chunk
+                    yield False, serial_message
             except StopIteration:
-                yield True, data_chunk
+                yield True, serial_message
                 return
 
     proxy = mock_proxy()

.gitlab-ci/tests/utils/test_lava_log.py

@@ -14,7 +14,6 @@ from lava.utils import (
     GitlabSection,
     LogFollower,
     LogSectionType,
-    fix_lava_color_log,
     fix_lava_gitlab_section_log,
     hide_sensitive_data,
 )
@@ -165,50 +164,6 @@ def test_hide_sensitive_data(input, expectation, tag):
     assert result == expectation
 
 
-COLOR_MANGLED_SCENARIOS = {
-    "Mangled error message at target level": (
-        create_lava_yaml_msg(msg="[0m[0m[31mERROR - dEQP error: ", lvl="target"),
-        "\x1b[0m\x1b[0m\x1b[31mERROR - dEQP error: ",
-    ),
-    "Mangled pass message at target level": (
-        create_lava_yaml_msg(
-            msg="[0mPass: 26718, ExpectedFail: 95, Skip: 25187, Duration: 8:18, Remaining: 13",
-            lvl="target",
-        ),
-        "\x1b[0mPass: 26718, ExpectedFail: 95, Skip: 25187, Duration: 8:18, Remaining: 13",
-    ),
-    "Mangled error message with bold formatting at target level": (
-        create_lava_yaml_msg(msg="[1;31mReview the image changes...", lvl="target"),
-        "\x1b[1;31mReview the image changes...",
-    ),
-    "Mangled error message with high intensity background at target level": (
-        create_lava_yaml_msg(msg="[100mReview the image changes...", lvl="target"),
-        "\x1b[100mReview the image changes...",
-    ),
-    "Mangled error message with underline+bg color+fg color at target level": (
-        create_lava_yaml_msg(msg="[4;41;97mReview the image changes...", lvl="target"),
-        "\x1b[4;41;97mReview the image changes...",
-    ),
-    "Bad input for color code.": (
-        create_lava_yaml_msg(
-            msg="[4;97 This message is missing the `m`.", lvl="target"
-        ),
-        "[4;97 This message is missing the `m`.",
-    ),
-}
-
-
-@pytest.mark.parametrize(
-    "message, fixed_message",
-    COLOR_MANGLED_SCENARIOS.values(),
-    ids=COLOR_MANGLED_SCENARIOS.keys(),
-)
-def test_fix_lava_color_log(message, fixed_message):
-    fix_lava_color_log(message)
-
-    assert message["msg"] == fixed_message
-
-
 GITLAB_SECTION_MANGLED_SCENARIOS = {
     "Mangled section_start at target level": (
         create_lava_yaml_msg(