From d24ffd870615f0918c19c35b428251ba86bb49d7 Mon Sep 17 00:00:00 2001 From: Alexandros Frantzis Date: Thu, 3 Sep 2020 16:29:05 +0300 Subject: [PATCH] tracie: Produce JUnit XML results Write tracie results to the 'results/junit.xml' file using the JUnit XML format. Among other uses, this file can be picked up by gitlab to display more useful information to the user. Signed-off-by: Alexandros Frantzis Reviewed-by: Eric Anholt Reviewed-by: Pierre-Eric Pelloux-Prayer Part-of: --- .gitlab-ci/tracie/tests/test.py | 36 ++++++++++++++++++++++++ .gitlab-ci/tracie/tracie.py | 50 +++++++++++++++++++++++++++++---- 2 files changed, 81 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci/tracie/tests/test.py b/.gitlab-ci/tracie/tests/test.py index ff4780cbe3b..ac531a3aef6 100644 --- a/.gitlab-ci/tracie/tests/test.py +++ b/.gitlab-ci/tracie/tests/test.py @@ -2,6 +2,7 @@ import logging import pytest import re import shutil +import xml.etree.ElementTree as ET from os import environ, chdir from os.path import dirname, exists, realpath @@ -10,6 +11,7 @@ import tracie RESULTS_YAML = "results/results.yml" +JUNIT_XML = "results/junit.xml" TRACE_LOG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace.log" TRACE_LOG_TEST2 = "results/trace2/test/vk-test-device/olive.testtrace.log" TRACE_PNG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace-0.png" @@ -215,3 +217,37 @@ def test_tracie_stores_images_on_request(): assert run_tracie() assert exists(TRACE_PNG_TEST1) assert exists(TRACE_PNG_TEST2) + +def test_tracie_writes_junit_xml(): + assert run_tracie() + junit_xml = ET.parse(JUNIT_XML) + assert junit_xml.getroot().tag == 'testsuites' + testsuites = junit_xml.findall("./testsuite") + testcases_gl = junit_xml.findall("./testsuite[@name='traces.yml:gl-test-device']/testcase") + testcases_vk = junit_xml.findall("./testsuite[@name='traces.yml:vk-test-device']/testcase") + + assert len(testsuites) == 2 + assert len(testcases_gl) == 1 + assert len(testcases_vk) == 1 + assert testcases_gl[0].get("name") == "trace1/magenta.testtrace" + assert testcases_gl[0].get("classname") == "traces.yml:gl-test-device" + assert testcases_vk[0].get("name") == "trace2/olive.testtrace" + assert testcases_vk[0].get("classname") == "traces.yml:vk-test-device" + +def test_tracie_writes_dashboard_url_in_junit_xml_failure_tag(): + filename = "./tests/traces.yml" + content = read_from(filename) + content = content.replace("5efda83854befe0155ff8517a58d5b51", + "8e0a801367e1714463475a824dab363b") + write_to(content, filename) + + assert not run_tracie() + + junit_xml = ET.parse(JUNIT_XML) + failures_gl = junit_xml.findall("./testsuite[@name='traces.yml:gl-test-device']/testcase/failure") + failures_vk = junit_xml.findall("./testsuite[@name='traces.yml:vk-test-device']/testcase/failure") + + assert len(failures_gl) == 0 + assert len(failures_vk) == 1 + dashboard_url = "https://tracie.freedesktop.org/dashboard/imagediff/test-project/42/trace2/olive.testtrace" + assert dashboard_url in failures_vk[0].text diff --git a/.gitlab-ci/tracie/tracie.py b/.gitlab-ci/tracie/tracie.py index 170400d9d54..30468078868 100644 --- a/.gitlab-ci/tracie/tracie.py +++ b/.gitlab-ci/tracie/tracie.py @@ -138,6 +138,12 @@ def ensure_reference_image(file_name, checksum): return upload_to_minio(file_name, resource, 'image/png') +def image_diff_url(trace_path): + return "%s/imagediff/%s/%s/%s" % (DASHBOARD_URL, + os.environ.get('CI_PROJECT_PATH'), + os.environ.get('CI_JOB_ID'), + trace_path) + def gitlab_check_trace(project_url, device_name, trace, expectation): gitlab_ensure_trace(project_url, trace) @@ -158,11 +164,7 @@ def gitlab_check_trace(project_url, device_name, trace, expectation): (trace['path'], expectation['checksum'], checksum)) print("[check_image] For more information see " "https://gitlab.freedesktop.org/mesa/mesa/blob/master/.gitlab-ci/tracie/README.md") - image_diff_url = "%s/imagediff/%s/%s/%s" % (DASHBOARD_URL, - os.environ['CI_PROJECT_PATH'], - os.environ['CI_JOB_ID'], - trace['path']) - print("[check_image] %s" % image_diff_url) + print("[check_image] %s" % image_diff_url(trace['path'])) ok = False trace_dir = os.path.split(trace['path'])[0] @@ -185,6 +187,39 @@ def gitlab_check_trace(project_url, device_name, trace, expectation): return ok, result +def write_junit_xml(junit_xml_path, traces_filename, device_name, results): + tests = len(results) + failures = sum(1 for r in results.values() if r["actual"] != r["expected"]) + + try: + testsuites = ET.parse(junit_xml_path).getroot() + except: + test_name = os.environ.get('CI_PROJECT_PATH') + "/" + \ + os.environ.get('CI_PIPELINE_ID') + "/" + \ + os.environ.get('CI_JOB_ID') + testsuites = ET.Element('testsuites', name=test_name) + + testsuites.set('tests', str(int(testsuites.get('tests', 0)) + tests)) + testsuites.set('failures', str(int(testsuites.get('failures', 0)) + failures)) + + testsuite_name = os.path.basename(traces_filename) + ":" + device_name + + testsuite = ET.SubElement(testsuites, 'testsuite', + name=testsuite_name, + tests=str(tests), failures=str(failures)) + + for (path, result) in results.items(): + testcase = ET.SubElement(testsuite, 'testcase', name=path, + classname=testsuite_name) + if result["actual"] != result["expected"]: + failure = ET.SubElement(testcase, 'failure') + failure.text = \ + ("Images differ (expected: %s, actual: %s).\n" + \ + "To view the image differences visit:\n%s") % \ + (result["expected"], result["actual"], image_diff_url(path)) + + ET.ElementTree(testsuites).write(junit_xml_path) + def run(filename, device_name): with open(filename, 'r') as f: @@ -210,8 +245,13 @@ def run(filename, device_name): os.makedirs(RESULTS_PATH, exist_ok=True) with open(os.path.join(RESULTS_PATH, 'results.yml'), 'w') as f: yaml.safe_dump(results, f, default_flow_style=False) + + junit_xml_path = os.path.join(RESULTS_PATH, "junit.xml") + write_junit_xml(junit_xml_path, filename, device_name, results) + if os.environ.get('TRACIE_UPLOAD_TO_MINIO', '0') == '1': upload_artifact(os.path.join(RESULTS_PATH, 'results.yml'), 'traces/results.yml', 'text/yaml') + upload_artifact(junit_xml_path, 'traces/junit.xml', 'text/xml') return all_ok