OpenHarmony/third_party_mesa3d
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tracie: Produce JUnit XML results

Browse Source 
Write tracie results to the 'results/junit.xml' file using the JUnit XML
format. Among other uses, this file can be picked up by gitlab to
display more useful information to the user.

Signed-off-by: Alexandros Frantzis <alexandros.frantzis@collabora.com>
Reviewed-by: Eric Anholt <eric@anholt.net>
Reviewed-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/6598>
This commit is contained in:
Alexandros Frantzis
2020-09-03 16:29:05 +03:00
parent dd471928c9
commit d24ffd8706
2 changed files with 81 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
.gitlab-ci/tracie/tests/test.py
View File

@@ -2,6 +2,7 @@ import logging
import pytest
import re
import shutil
import xml.etree.ElementTree as ET
from os import environ, chdir
from os.path import dirname, exists, realpath
@@ -10,6 +11,7 @@ import tracie
RESULTS_YAML = "results/results.yml"
JUNIT_XML = "results/junit.xml"
TRACE_LOG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace.log"
TRACE_LOG_TEST2 = "results/trace2/test/vk-test-device/olive.testtrace.log"
TRACE_PNG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace-0.png"
@@ -215,3 +217,37 @@ def test_tracie_stores_images_on_request():
assert run_tracie()
assert exists(TRACE_PNG_TEST1)
assert exists(TRACE_PNG_TEST2)
def test_tracie_writes_junit_xml():
assert run_tracie()
junit_xml = ET.parse(JUNIT_XML)
assert junit_xml.getroot().tag == 'testsuites'
testsuites = junit_xml.findall("./testsuite")
testcases_gl = junit_xml.findall("./testsuite[@name='traces.yml:gl-test-device']/testcase")
testcases_vk = junit_xml.findall("./testsuite[@name='traces.yml:vk-test-device']/testcase")
assert len(testsuites) == 2
assert len(testcases_gl) == 1
assert len(testcases_vk) == 1
assert testcases_gl[0].get("name") == "trace1/magenta.testtrace"
assert testcases_gl[0].get("classname") == "traces.yml:gl-test-device"
assert testcases_vk[0].get("name") == "trace2/olive.testtrace"
assert testcases_vk[0].get("classname") == "traces.yml:vk-test-device"
def test_tracie_writes_dashboard_url_in_junit_xml_failure_tag():
filename = "./tests/traces.yml"
content = read_from(filename)
content = content.replace("5efda83854befe0155ff8517a58d5b51",
"8e0a801367e1714463475a824dab363b")
write_to(content, filename)
assert not run_tracie()
junit_xml = ET.parse(JUNIT_XML)
failures_gl = junit_xml.findall("./testsuite[@name='traces.yml:gl-test-device']/testcase/failure")
failures_vk = junit_xml.findall("./testsuite[@name='traces.yml:vk-test-device']/testcase/failure")
assert len(failures_gl) == 0
assert len(failures_vk) == 1
dashboard_url = "https://tracie.freedesktop.org/dashboard/imagediff/test-project/42/trace2/olive.testtrace"
assert dashboard_url in failures_vk[0].text

50
.gitlab-ci/tracie/tracie.py
View File

@@ -138,6 +138,12 @@ def ensure_reference_image(file_name, checksum):
return
upload_to_minio(file_name, resource, 'image/png')
def image_diff_url(trace_path):
return "%s/imagediff/%s/%s/%s" % (DASHBOARD_URL,
os.environ.get('CI_PROJECT_PATH'),
os.environ.get('CI_JOB_ID'),
trace_path)
def gitlab_check_trace(project_url, device_name, trace, expectation):
gitlab_ensure_trace(project_url, trace)
@@ -158,11 +164,7 @@ def gitlab_check_trace(project_url, device_name, trace, expectation):
(trace['path'], expectation['checksum'], checksum))
print("[check_image] For more information see "
"https://gitlab.freedesktop.org/mesa/mesa/blob/master/.gitlab-ci/tracie/README.md")
image_diff_url = "%s/imagediff/%s/%s/%s" % (DASHBOARD_URL,
os.environ['CI_PROJECT_PATH'],
os.environ['CI_JOB_ID'],
trace['path'])
print("[check_image] %s" % image_diff_url)
print("[check_image] %s" % image_diff_url(trace['path']))
ok = False
trace_dir = os.path.split(trace['path'])[0]
@@ -185,6 +187,39 @@ def gitlab_check_trace(project_url, device_name, trace, expectation):
return ok, result
def write_junit_xml(junit_xml_path, traces_filename, device_name, results):
tests = len(results)
failures = sum(1 for r in results.values() if r["actual"] != r["expected"])
try:
testsuites = ET.parse(junit_xml_path).getroot()
except:
test_name = os.environ.get('CI_PROJECT_PATH') + "/" + \
os.environ.get('CI_PIPELINE_ID') + "/" + \
os.environ.get('CI_JOB_ID')
testsuites = ET.Element('testsuites', name=test_name)
testsuites.set('tests', str(int(testsuites.get('tests', 0)) + tests))
testsuites.set('failures', str(int(testsuites.get('failures', 0)) + failures))
testsuite_name = os.path.basename(traces_filename) + ":" + device_name
testsuite = ET.SubElement(testsuites, 'testsuite',
name=testsuite_name,
tests=str(tests), failures=str(failures))
for (path, result) in results.items():
testcase = ET.SubElement(testsuite, 'testcase', name=path,
classname=testsuite_name)
if result["actual"] != result["expected"]:
failure = ET.SubElement(testcase, 'failure')
failure.text = \
("Images differ (expected: %s, actual: %s).\n" + \
"To view the image differences visit:\n%s") % \
(result["expected"], result["actual"], image_diff_url(path))
ET.ElementTree(testsuites).write(junit_xml_path)
def run(filename, device_name):
with open(filename, 'r') as f:
@@ -210,8 +245,13 @@ def run(filename, device_name):
os.makedirs(RESULTS_PATH, exist_ok=True)
with open(os.path.join(RESULTS_PATH, 'results.yml'), 'w') as f:
yaml.safe_dump(results, f, default_flow_style=False)
junit_xml_path = os.path.join(RESULTS_PATH, "junit.xml")
write_junit_xml(junit_xml_path, filename, device_name, results)
if os.environ.get('TRACIE_UPLOAD_TO_MINIO', '0') == '1':
upload_artifact(os.path.join(RESULTS_PATH, 'results.yml'), 'traces/results.yml', 'text/yaml')
upload_artifact(junit_xml_path, 'traces/junit.xml', 'text/xml')
return all_ok