diff --git a/.gitlab-ci/lava/lava_job_submitter.py b/.gitlab-ci/lava/lava_job_submitter.py index 9c4af85cfde..c338cc8cd95 100755 --- a/.gitlab-ci/lava/lava_job_submitter.py +++ b/.gitlab-ci/lava/lava_job_submitter.py @@ -272,8 +272,12 @@ class LAVAJob: def _load_log_from_data(self, data) -> list[str]: lines = [] + if isinstance(data, xmlrpc.client.Binary): + # We are dealing with xmlrpc.client.Binary + # Let's extract the data + data = data.data # When there is no new log data, the YAML is empty - if loaded_lines := yaml.load(str(data), Loader=loader(False)): + if loaded_lines := yaml.load(data, Loader=loader(False)): lines = loaded_lines self.last_log_line += len(lines) return lines diff --git a/.gitlab-ci/lava/utils/__init__.py b/.gitlab-ci/lava/utils/__init__.py index 0be88266d35..18bb459c1a4 100644 --- a/.gitlab-ci/lava/utils/__init__.py +++ b/.gitlab-ci/lava/utils/__init__.py @@ -3,7 +3,6 @@ from .gitlab_section import GitlabSection from .log_follower import ( LogFollower, fatal_err, - fix_lava_color_log, fix_lava_gitlab_section_log, hide_sensitive_data, print_log, diff --git a/.gitlab-ci/lava/utils/log_follower.py b/.gitlab-ci/lava/utils/log_follower.py index c61a9aa4e60..506cb7b8a3b 100644 --- a/.gitlab-ci/lava/utils/log_follower.py +++ b/.gitlab-ci/lava/utils/log_follower.py @@ -150,19 +150,6 @@ class LogFollower: self._buffer = [] return buffer - -def fix_lava_color_log(line): - """This function is a temporary solution for the color escape codes mangling - problem. There is some problem in message passing between the LAVA - dispatcher and the device under test (DUT). Here \x1b character is missing - before `[:digit::digit:?:digit:?m` ANSI TTY color codes, or the more - complicated ones with number values for text format before background and - foreground colors. - When this problem is fixed on the LAVA side, one should remove this function. - """ - line["msg"] = re.sub(r"(\[(\d+;){0,2}\d{1,3}m)", "\x1b" + r"\1", line["msg"]) - - def fix_lava_gitlab_section_log(line): """This function is a temporary solution for the Gitlab section markers mangling problem. Gitlab parses the following lines to define a collapsible @@ -195,7 +182,6 @@ def parse_lava_line(line) -> Optional[str]: prefix = "$ " suffix = "" elif line["lvl"] == "target": - fix_lava_color_log(line) fix_lava_gitlab_section_log(line) return f'{prefix}{line["msg"]}{suffix}' diff --git a/.gitlab-ci/tests/test_lava_job_submitter.py b/.gitlab-ci/tests/test_lava_job_submitter.py index 716cb60bb8c..e9f73d8bf5c 100644 --- a/.gitlab-ci/tests/test_lava_job_submitter.py +++ b/.gitlab-ci/tests/test_lava_job_submitter.py @@ -298,9 +298,7 @@ def test_parse_job_result_from_log(message, expectation, mock_proxy): reason="Slow and sketchy test. Needs a LAVA log raw file at /tmp/log.yaml" ) def test_full_yaml_log(mock_proxy, frozen_time): - import itertools import random - from datetime import datetime import yaml @@ -326,21 +324,23 @@ def test_full_yaml_log(mock_proxy, frozen_time): def load_lines() -> list: with open("/tmp/log.yaml", "r") as f: - data = yaml.safe_load(f) - chain = itertools.chain(data) + # data = yaml.safe_load(f) + data = f.readlines() + stream = chain(data) try: while True: - data_chunk = [next(chain) for _ in range(random.randint(0, 50))] + data_chunk = [next(stream) for _ in range(random.randint(0, 50))] + serial_message = "".join(data_chunk) # Suppose that the first message timestamp is the same of # log fetch RPC call time_travel_from_log_chunk(data_chunk) - yield False, [] + yield False, "[]" # Travel to the same datetime of the last fetched log line # in the chunk time_travel_from_log_chunk(data_chunk) - yield False, data_chunk + yield False, serial_message except StopIteration: - yield True, data_chunk + yield True, serial_message return proxy = mock_proxy() diff --git a/.gitlab-ci/tests/utils/test_lava_log.py b/.gitlab-ci/tests/utils/test_lava_log.py index 1219cc9f830..69fa149d657 100644 --- a/.gitlab-ci/tests/utils/test_lava_log.py +++ b/.gitlab-ci/tests/utils/test_lava_log.py @@ -14,7 +14,6 @@ from lava.utils import ( GitlabSection, LogFollower, LogSectionType, - fix_lava_color_log, fix_lava_gitlab_section_log, hide_sensitive_data, ) @@ -165,50 +164,6 @@ def test_hide_sensitive_data(input, expectation, tag): assert result == expectation -COLOR_MANGLED_SCENARIOS = { - "Mangled error message at target level": ( - create_lava_yaml_msg(msg="[0m[0m[31mERROR - dEQP error: ", lvl="target"), - "\x1b[0m\x1b[0m\x1b[31mERROR - dEQP error: ", - ), - "Mangled pass message at target level": ( - create_lava_yaml_msg( - msg="[0mPass: 26718, ExpectedFail: 95, Skip: 25187, Duration: 8:18, Remaining: 13", - lvl="target", - ), - "\x1b[0mPass: 26718, ExpectedFail: 95, Skip: 25187, Duration: 8:18, Remaining: 13", - ), - "Mangled error message with bold formatting at target level": ( - create_lava_yaml_msg(msg="[1;31mReview the image changes...", lvl="target"), - "\x1b[1;31mReview the image changes...", - ), - "Mangled error message with high intensity background at target level": ( - create_lava_yaml_msg(msg="[100mReview the image changes...", lvl="target"), - "\x1b[100mReview the image changes...", - ), - "Mangled error message with underline+bg color+fg color at target level": ( - create_lava_yaml_msg(msg="[4;41;97mReview the image changes...", lvl="target"), - "\x1b[4;41;97mReview the image changes...", - ), - "Bad input for color code.": ( - create_lava_yaml_msg( - msg="[4;97 This message is missing the `m`.", lvl="target" - ), - "[4;97 This message is missing the `m`.", - ), -} - - -@pytest.mark.parametrize( - "message, fixed_message", - COLOR_MANGLED_SCENARIOS.values(), - ids=COLOR_MANGLED_SCENARIOS.keys(), -) -def test_fix_lava_color_log(message, fixed_message): - fix_lava_color_log(message) - - assert message["msg"] == fixed_message - - GITLAB_SECTION_MANGLED_SCENARIOS = { "Mangled section_start at target level": ( create_lava_yaml_msg(