ci/lava: Fix the integration test

During the development of this fix, I utilized the `test_full_yaml_log`
test, which is marked as slow. This test is excellent for reproducing
past job submissions. It can be executed using the following commands:

```
lavacli jobs logs --raw 12496073 > /tmp/log.yaml
pytest .gitlab-ci/tests/test_lava_job_submitter.py -m slow -s
```

Here, `12496073` is the LAVA job ID from this specific job:
https://gitlab.freedesktop.org/mesa/mesa/-/jobs/53546660

The logs were not functioning as expected due to a few mistakes I made
with generators, such as:
- Not reusing the `time_traveller_gen` generator to call `next` more
  than once.
- Forgetting to parse the YAML inside `time_travel_from_log_chunk`.

Additionally:
- Added some statistics at the end of the test to aid in diagnosing
  whether the test was reproduced accurately.
- Renamed some variables for clarity.

Signed-off-by: Guilherme Gallo <guilherme.gallo@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/26996>
This commit is contained in:
Guilherme Gallo
2024-01-10 18:18:42 -03:00
committed by Marge Bot
parent 2b86e49393
commit f3850c97d1

View File

@@ -9,8 +9,9 @@ import os
import xmlrpc.client import xmlrpc.client
from contextlib import nullcontext as does_not_raise from contextlib import nullcontext as does_not_raise
from datetime import datetime from datetime import datetime
from itertools import chain, repeat from itertools import islice, repeat
from pathlib import Path from pathlib import Path
from typing import Generator
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
import pytest import pytest
@@ -341,13 +342,15 @@ def test_full_yaml_log(mock_proxy, frozen_time, lava_job_submitter):
if not data_chunk: if not data_chunk:
return return
first_log_time = data_chunk[0]["dt"] first_log = lava_yaml.load(data_chunk[0])[0]
first_log_time = first_log["dt"]
frozen_time.move_to(first_log_time) frozen_time.move_to(first_log_time)
yield yield
last_log_time = data_chunk[-1]["dt"] last_log = lava_yaml.load(data_chunk[-1])[0]
last_log_time = last_log["dt"]
frozen_time.move_to(last_log_time) frozen_time.move_to(last_log_time)
return yield
def time_travel_to_test_time(): def time_travel_to_test_time():
# Suppose that the first message timestamp of the entire LAVA job log is # Suppose that the first message timestamp of the entire LAVA job log is
@@ -357,22 +360,31 @@ def test_full_yaml_log(mock_proxy, frozen_time, lava_job_submitter):
first_log_time = lava_yaml.load(first_log)[0]["dt"] first_log_time = lava_yaml.load(first_log)[0]["dt"]
frozen_time.move_to(first_log_time) frozen_time.move_to(first_log_time)
def load_lines() -> list: def load_lines() -> Generator[tuple[bool, str], None, None]:
with open("/tmp/log.yaml", "r") as f: with open("/tmp/log.yaml", "r") as f:
# data = yaml.safe_load(f) # data = yaml.safe_load(f)
data = f.readlines() log_lines = f.readlines()
stream = chain(data) serial_message: str = ""
chunk_start_line = 0
chunk_end_line = 0
chunk_max_size = 100
try: try:
while True: while True:
data_chunk = [next(stream) for _ in range(random.randint(0, 50))] chunk_end_line = chunk_start_line + random.randint(1, chunk_max_size)
serial_message = "".join(data_chunk) # split the log in chunks of random size
log_chunk = list(islice(log_lines, chunk_start_line, chunk_end_line))
chunk_start_line = chunk_end_line + 1
serial_message = "".join(log_chunk)
# time_traveller_gen will make the time trave according to the timestamp from
# the message
time_traveller_gen = time_travel_from_log_chunk(log_chunk)
# Suppose that the first message timestamp is the same of # Suppose that the first message timestamp is the same of
# log fetch RPC call # log fetch RPC call
time_travel_from_log_chunk(data_chunk) next(time_traveller_gen)
yield False, "[]" yield False, "[]"
# Travel to the same datetime of the last fetched log line # Travel to the same datetime of the last fetched log line
# in the chunk # in the chunk
time_travel_from_log_chunk(data_chunk) next(time_traveller_gen)
yield False, serial_message yield False, serial_message
except StopIteration: except StopIteration:
yield True, serial_message yield True, serial_message
@@ -384,10 +396,20 @@ def test_full_yaml_log(mock_proxy, frozen_time, lava_job_submitter):
proxy.scheduler.jobs.logs.side_effect = load_lines() proxy.scheduler.jobs.logs.side_effect = load_lines()
proxy.scheduler.jobs.submit = reset_logs proxy.scheduler.jobs.submit = reset_logs
with pytest.raises(MesaCIRetryError): start_time = datetime.now()
try:
time_travel_to_test_time() time_travel_to_test_time()
retriable_follow_job(proxy, "") retriable_follow_job(proxy, "")
print(lava_job_submitter.structured_log_file.read_text()) finally:
try:
# If the job fails, maybe there will be no structured log
print(lava_job_submitter.structured_log_file.read_text())
finally:
end_time = datetime.now()
print("---- Reproduction log stats ----")
print(f"Start time: {start_time}")
print(f"End time: {end_time}")
print(f"Duration: {end_time - start_time}")
@pytest.mark.parametrize( @pytest.mark.parametrize(