OpenHarmony/third_party_mesa3d
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ci/lava: Create LogFollower and move logging methods

Browse Source 
- Create LogFollower to capture LAVA log and process it adding some
- GitlabSection and color treatment to it
- Break logs further, make new gitlab sections between testcases
- Implement LogFollower as ContextManager to deal with incomplete LAVA
  jobs.
- Use template method to simplify gitlab log sections management
- Fix sections timestamps

Signed-off-by: Guilherme Gallo <guilherme.gallo@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/16323>
This commit is contained in:
Guilherme Gallo
2022-06-23 21:43:00 -03:00
committed by Marge Bot
parent c86ba3640f
commit 2569d7d7df
9 changed files with 512 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
.gitlab-ci/lava/__init__.py Normal file
View File

113
.gitlab-ci/lava/lava_job_submitter.py
View File

@@ -46,7 +46,14 @@ from lava.exceptions import (
MesaCIRetryError,
MesaCITimeoutError,
)
from lava.utils.lava_log import GitlabSection
from lava.utils.lava_log import (
CONSOLE_LOG,
GitlabSection,
fatal_err,
hide_sensitive_data,
parse_lava_lines,
print_log,
)
from lavacli.utils import loader
# Timeout in seconds to decide if the device from the dispatched LAVA job has
@@ -66,26 +73,6 @@ NUMBER_OF_RETRIES_TIMEOUT_DETECTION = int(getenv("LAVA_NUMBER_OF_RETRIES_TIMEOUT
# How many attempts should be made when a timeout happen during LAVA device boot.
NUMBER_OF_ATTEMPTS_LAVA_BOOT = int(getenv("LAVA_NUMBER_OF_ATTEMPTS_LAVA_BOOT", 3))
# Helper constants to colorize the job output
CONSOLE_LOG_COLOR_GREEN = "\x1b[1;32;5;197m"
CONSOLE_LOG_COLOR_RED = "\x1b[1;38;5;197m"
CONSOLE_LOG_COLOR_RESET = "\x1b[0m"
def print_log(msg):
# Reset color from timestamp, since `msg` can tint the terminal color
print(f"{CONSOLE_LOG_COLOR_RESET}{datetime.now()}: {msg}")
def fatal_err(msg):
print_log(msg)
sys.exit(1)
def hide_sensitive_data(yaml_data, hide_tag="HIDEME"):
return "".join(line for line in yaml_data.splitlines(True) if hide_tag not in line)
def generate_lava_yaml(args):
# General metadata and permissions, plus also inexplicably kernel arguments
values = {
@@ -247,7 +234,7 @@ def _call_proxy(fn, *args):
class LAVAJob:
color_status_map = {"pass": CONSOLE_LOG_COLOR_GREEN}
COLOR_STATUS_MAP = {"pass": CONSOLE_LOG["COLOR_GREEN"]}
def __init__(self, proxy, definition):
self.job_id = None
@@ -322,11 +309,13 @@ class LAVAJob:
if result := re.search(r"hwci: mesa: (pass|fail)", line):
self.is_finished = True
self.status = result.group(1)
color = LAVAJob.color_status_map.get(self.status, CONSOLE_LOG_COLOR_RED)
color = LAVAJob.COLOR_STATUS_MAP.get(
self.status, CONSOLE_LOG["COLOR_RED"]
)
print_log(
f"{color}"
f"LAVA Job finished with result: {self.status}"
f"{CONSOLE_LOG_COLOR_RESET}"
f"{CONSOLE_LOG['RESET']}"
)
# We reached the log end here. hwci script has finished.
@@ -377,82 +366,6 @@ def show_job_data(job):
print("{}\t: {}".format(field, value))
def fix_lava_color_log(line):
"""This function is a temporary solution for the color escape codes mangling
problem. There is some problem in message passing between the LAVA
dispatcher and the device under test (DUT). Here \x1b character is missing
before `[:digit::digit:?:digit:?m` ANSI TTY color codes, or the more
complicated ones with number values for text format before background and
foreground colors.
When this problem is fixed on the LAVA side, one should remove this function.
"""
line["msg"] = re.sub(r"(\[(\d+;){0,2}\d{1,3}m)", "\x1b" + r"\1", line["msg"])
def fix_lava_gitlab_section_log(line):
"""This function is a temporary solution for the Gitlab section markers
mangling problem. Gitlab parses the following lines to define a collapsible
gitlab section in their log:
- \x1b[0Ksection_start:timestamp:section_id[collapsible=true/false]\r\x1b[0Ksection_header
- \x1b[0Ksection_end:timestamp:section_id\r\x1b[0K
There is some problem in message passing between the LAVA dispatcher and the
device under test (DUT), that digests \x1b and \r control characters
incorrectly. When this problem is fixed on the LAVA side, one should remove
this function.
"""
if match := re.match(r"\[0K(section_\w+):(\d+):(\S+)\[0K([\S ]+)?", line["msg"]):
marker, timestamp, id_collapsible, header = match.groups()
# The above regex serves for both section start and end lines.
# When the header is None, it means we are dealing with `section_end` line
header = header or ""
line["msg"] = f"\x1b[0K{marker}:{timestamp}:{id_collapsible}\r\x1b[0K{header}"
def filter_debug_messages(line: dict[str, str]) -> bool:
"""Filter some LAVA debug messages that does not add much information to the
developer and may clutter the trace log."""
if line["lvl"] != "debug":
return False
# line["msg"] can be a list[str] when there is a kernel dump
if not isinstance(line["msg"], str):
return False
if re.match(
# Sometimes LAVA dumps this messages lots of times when the LAVA job is
# reaching the end.
r"^Listened to connection for namespace",
line["msg"],
):
return True
return False
def parse_lava_lines(new_lines) -> list[str]:
parsed_lines: list[str] = []
for line in new_lines:
prefix = ""
suffix = ""
if line["lvl"] in ["results", "feedback"]:
continue
elif line["lvl"] in ["warning", "error"]:
prefix = CONSOLE_LOG_COLOR_RED
suffix = CONSOLE_LOG_COLOR_RESET
elif filter_debug_messages(line):
continue
elif line["lvl"] == "input":
prefix = "$ "
suffix = ""
elif line["lvl"] == "target":
fix_lava_color_log(line)
fix_lava_gitlab_section_log(line)
line = f'{prefix}{line["msg"]}{suffix}'
parsed_lines.append(line)
return parsed_lines
def fetch_logs(job, max_idle_time) -> None:
# Poll to check for new logs, assuming that a prolonged period of
# silence means that the device has died and we should try it again

0
.gitlab-ci/lava/utils/__init__.py Normal file
View File

195
.gitlab-ci/lava/utils/lava_log.py
View File

@@ -27,8 +27,12 @@ Some utilities to analyse logs, create gitlab sections and other quality of life
improvements
"""
from dataclasses import dataclass
import logging
import re
import sys
from dataclasses import dataclass, field
from datetime import datetime
from typing import Optional, Pattern, Union
# Helper constants to colorize the job output
CONSOLE_LOG = {
@@ -68,15 +72,190 @@ class GitlabSection:
return f"{before_header}{header_wrapper}"
def __enter__(self):
print(self.start())
return self
def __exit__(self, exc_type, exc_val, exc_tb):
print(self.end())
def start(self) -> str:
return self.section(marker="start", header=self.header)
def end(self) -> str:
return self.section(marker="end", header="")
@dataclass(frozen=True)
class LogSection:
regex: Union[Pattern, str]
level: str
section_id: str
section_header: str
collapsed: bool = False
def from_log_line_to_section(
self, lava_log_line: dict[str, str]
) -> Optional[GitlabSection]:
if lava_log_line["lvl"] == self.level:
if match := re.match(self.regex, lava_log_line["msg"]):
section_id = self.section_id.format(*match.groups())
section_header = self.section_header.format(*match.groups())
return GitlabSection(
id=section_id,
header=section_header,
start_collapsed=self.collapsed,
)
LOG_SECTIONS = (
LogSection(
regex=re.compile(r".*<STARTTC> (.*)"),
level="debug",
section_id="{}",
section_header="test_case {}",
),
LogSection(
regex=re.compile(r".*<STARTRUN> (\S*)"),
level="debug",
section_id="{}",
section_header="test_suite {}",
),
LogSection(
regex=re.compile(r"^<LAVA_SIGNAL_ENDTC ([^>]+)"),
level="target",
section_id="post-{}",
section_header="Post test_case {}",
collapsed=True,
),
)
@dataclass
class LogFollower:
current_section: Optional[GitlabSection] = None
sections: list[str] = field(default_factory=list)
collapsed_sections: tuple[str] = ("setup",)
_buffer: list[str] = field(default_factory=list, init=False)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""Cleanup existing buffer if this object gets out from the context"""
self.clear_current_section()
last_lines = self.flush()
for line in last_lines:
print(line)
def clear_current_section(self):
if self.current_section:
self._buffer.append(self.current_section.end())
self.current_section = None
def update_section(self, new_section: GitlabSection):
self.clear_current_section()
self.current_section = new_section
self._buffer.append(new_section.start())
def manage_gl_sections(self, line):
if isinstance(line["msg"], list):
logging.debug("Ignoring messages as list. Kernel dumps.")
return
for log_section in LOG_SECTIONS:
if new_section := log_section.from_log_line_to_section(line):
self.update_section(new_section)
def feed(self, new_lines: list[dict[str, str]]) -> None:
for line in new_lines:
self.manage_gl_sections(line)
self._buffer.append(line)
def flush(self) -> list[str]:
buffer = self._buffer
self._buffer = []
return buffer
def fix_lava_color_log(line):
"""This function is a temporary solution for the color escape codes mangling
problem. There is some problem in message passing between the LAVA
dispatcher and the device under test (DUT). Here \x1b character is missing
before `[:digit::digit:?:digit:?m` ANSI TTY color codes, or the more
complicated ones with number values for text format before background and
foreground colors.
When this problem is fixed on the LAVA side, one should remove this function.
"""
line["msg"] = re.sub(r"(\[(\d+;){0,2}\d{1,3}m)", "\x1b" + r"\1", line["msg"])
def fix_lava_gitlab_section_log(line):
"""This function is a temporary solution for the Gitlab section markers
mangling problem. Gitlab parses the following lines to define a collapsible
gitlab section in their log:
- \x1b[0Ksection_start:timestamp:section_id[collapsible=true/false]\r\x1b[0Ksection_header
- \x1b[0Ksection_end:timestamp:section_id\r\x1b[0K
There is some problem in message passing between the LAVA dispatcher and the
device under test (DUT), that digests \x1b and \r control characters
incorrectly. When this problem is fixed on the LAVA side, one should remove
this function.
"""
if match := re.match(r"\[0K(section_\w+):(\d+):(\S+)\[0K([\S ]+)?", line["msg"]):
marker, timestamp, id_collapsible, header = match.groups()
# The above regex serves for both section start and end lines.
# When the header is None, it means we are dealing with `section_end` line
header = header or ""
line["msg"] = f"\x1b[0K{marker}:{timestamp}:{id_collapsible}\r\x1b[0K{header}"
def filter_debug_messages(line: dict[str, str]) -> bool:
"""Filter some LAVA debug messages that does not add much information to the
developer and may clutter the trace log."""
if line["lvl"] != "debug":
return False
# line["msg"] can be a list[str] when there is a kernel dump
if not isinstance(line["msg"], str):
return False
if re.match(
# Sometimes LAVA dumps this messages lots of times when the LAVA job is
# reaching the end.
r"^Listened to connection for namespace",
line["msg"],
):
return True
return False
def parse_lava_lines(new_lines) -> list[str]:
parsed_lines: list[str] = []
for line in new_lines:
prefix = ""
suffix = ""
if line["lvl"] in ["results", "feedback"]:
continue
elif line["lvl"] in ["warning", "error"]:
prefix = CONSOLE_LOG["COLOR_RED"]
suffix = CONSOLE_LOG["RESET"]
elif filter_debug_messages(line):
continue
elif line["lvl"] == "input":
prefix = "$ "
suffix = ""
elif line["lvl"] == "target":
fix_lava_color_log(line)
fix_lava_gitlab_section_log(line)
line = f'{prefix}{line["msg"]}{suffix}'
parsed_lines.append(line)
return parsed_lines
def print_log(msg):
# Reset color from timestamp, since `msg` can tint the terminal color
print(f"{CONSOLE_LOG['RESET']}{datetime.now()}: {msg}")
def fatal_err(msg):
print_log(msg)
sys.exit(1)
def hide_sensitive_data(yaml_data, hide_tag="HIDEME"):
return "".join(line for line in yaml_data.splitlines(True) if hide_tag not in line)