tracie: Produce JUnit XML results

Write tracie results to the 'results/junit.xml' file using the JUnit XML
format. Among other uses, this file can be picked up by gitlab to
display more useful information to the user.

Signed-off-by: Alexandros Frantzis <alexandros.frantzis@collabora.com>
Reviewed-by: Eric Anholt <eric@anholt.net>
Reviewed-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/6598>

.gitlab-ci/tracie/tests/test.py

@@ -2,6 +2,7 @@ import logging
 import pytest
 import re
 import shutil
+import xml.etree.ElementTree as ET
 
 from os import environ, chdir
 from os.path import dirname, exists, realpath
@@ -10,6 +11,7 @@ import tracie
 
 
 RESULTS_YAML = "results/results.yml"
+JUNIT_XML = "results/junit.xml"
 TRACE_LOG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace.log"
 TRACE_LOG_TEST2 = "results/trace2/test/vk-test-device/olive.testtrace.log"
 TRACE_PNG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace-0.png"
@@ -215,3 +217,37 @@ def test_tracie_stores_images_on_request():
     assert run_tracie()
     assert exists(TRACE_PNG_TEST1)
     assert exists(TRACE_PNG_TEST2)
+
+def test_tracie_writes_junit_xml():
+    assert run_tracie()
+    junit_xml = ET.parse(JUNIT_XML)
+    assert junit_xml.getroot().tag == 'testsuites'
+    testsuites = junit_xml.findall("./testsuite")
+    testcases_gl = junit_xml.findall("./testsuite[@name='traces.yml:gl-test-device']/testcase")
+    testcases_vk = junit_xml.findall("./testsuite[@name='traces.yml:vk-test-device']/testcase")
+
+    assert len(testsuites) == 2
+    assert len(testcases_gl) == 1
+    assert len(testcases_vk) == 1
+    assert testcases_gl[0].get("name") == "trace1/magenta.testtrace"
+    assert testcases_gl[0].get("classname") == "traces.yml:gl-test-device"
+    assert testcases_vk[0].get("name") == "trace2/olive.testtrace"
+    assert testcases_vk[0].get("classname") == "traces.yml:vk-test-device"
+
+def test_tracie_writes_dashboard_url_in_junit_xml_failure_tag():
+    filename = "./tests/traces.yml"
+    content = read_from(filename)
+    content = content.replace("5efda83854befe0155ff8517a58d5b51",
+                              "8e0a801367e1714463475a824dab363b")
+    write_to(content, filename)
+
+    assert not run_tracie()
+
+    junit_xml = ET.parse(JUNIT_XML)
+    failures_gl = junit_xml.findall("./testsuite[@name='traces.yml:gl-test-device']/testcase/failure")
+    failures_vk = junit_xml.findall("./testsuite[@name='traces.yml:vk-test-device']/testcase/failure")
+
+    assert len(failures_gl) == 0
+    assert len(failures_vk) == 1
+    dashboard_url = "https://tracie.freedesktop.org/dashboard/imagediff/test-project/42/trace2/olive.testtrace"
+    assert dashboard_url in failures_vk[0].text

.gitlab-ci/tracie/tracie.py

@@ -138,6 +138,12 @@ def ensure_reference_image(file_name, checksum):
         return
     upload_to_minio(file_name, resource, 'image/png')
 
+def image_diff_url(trace_path):
+    return "%s/imagediff/%s/%s/%s" % (DASHBOARD_URL,
+                                      os.environ.get('CI_PROJECT_PATH'),
+                                      os.environ.get('CI_JOB_ID'),
+                                      trace_path)
+
 def gitlab_check_trace(project_url, device_name, trace, expectation):
     gitlab_ensure_trace(project_url, trace)
 
@@ -158,11 +164,7 @@ def gitlab_check_trace(project_url, device_name, trace, expectation):
                 (trace['path'], expectation['checksum'], checksum))
         print("[check_image] For more information see "
                 "https://gitlab.freedesktop.org/mesa/mesa/blob/master/.gitlab-ci/tracie/README.md")
-        image_diff_url = "%s/imagediff/%s/%s/%s" % (DASHBOARD_URL,
+        print("[check_image] %s" % image_diff_url(trace['path']))
-                                                    os.environ['CI_PROJECT_PATH'],
-                                                    os.environ['CI_JOB_ID'],
-                                                    trace['path'])
-        print("[check_image] %s" % image_diff_url)
         ok = False
 
     trace_dir = os.path.split(trace['path'])[0]
@@ -185,6 +187,39 @@ def gitlab_check_trace(project_url, device_name, trace, expectation):
 
     return ok, result
 
+def write_junit_xml(junit_xml_path, traces_filename, device_name, results):
+    tests = len(results)
+    failures = sum(1 for r in results.values() if r["actual"] != r["expected"])
+
+    try:
+        testsuites = ET.parse(junit_xml_path).getroot()
+    except:
+        test_name = os.environ.get('CI_PROJECT_PATH') + "/" + \
+                    os.environ.get('CI_PIPELINE_ID') + "/" + \
+                    os.environ.get('CI_JOB_ID')
+        testsuites = ET.Element('testsuites', name=test_name)
+
+    testsuites.set('tests', str(int(testsuites.get('tests', 0)) + tests))
+    testsuites.set('failures', str(int(testsuites.get('failures', 0)) + failures))
+
+    testsuite_name = os.path.basename(traces_filename) + ":" + device_name
+
+    testsuite = ET.SubElement(testsuites, 'testsuite',
+                              name=testsuite_name,
+                              tests=str(tests), failures=str(failures))
+
+    for (path, result) in results.items():
+        testcase = ET.SubElement(testsuite, 'testcase', name=path,
+                                 classname=testsuite_name)
+        if result["actual"] != result["expected"]:
+            failure = ET.SubElement(testcase, 'failure')
+            failure.text = \
+                ("Images differ (expected: %s, actual: %s).\n" + \
+                 "To view the image differences visit:\n%s") % \
+                (result["expected"], result["actual"], image_diff_url(path))
+
+    ET.ElementTree(testsuites).write(junit_xml_path)
+
 def run(filename, device_name):
 
     with open(filename, 'r') as f:
@@ -210,8 +245,13 @@ def run(filename, device_name):
     os.makedirs(RESULTS_PATH, exist_ok=True)
     with open(os.path.join(RESULTS_PATH, 'results.yml'), 'w') as f:
         yaml.safe_dump(results, f, default_flow_style=False)
+
+    junit_xml_path = os.path.join(RESULTS_PATH, "junit.xml")
+    write_junit_xml(junit_xml_path, filename, device_name, results)
+
     if os.environ.get('TRACIE_UPLOAD_TO_MINIO', '0') == '1':
         upload_artifact(os.path.join(RESULTS_PATH, 'results.yml'), 'traces/results.yml', 'text/yaml')
+        upload_artifact(junit_xml_path, 'traces/junit.xml', 'text/xml')
 
     return all_ok