diff --git a/.gitlab-ci/traces.yml b/.gitlab-ci/traces.yml index 1ec7603407f..87db9e94856 100644 --- a/.gitlab-ci/traces.yml +++ b/.gitlab-ci/traces.yml @@ -1,6 +1,5 @@ traces-db: - gitlab-project-url: "https://gitlab.freedesktop.org/gfx-ci/tracie/traces-db" - commit: "d3b1efe0cd69ef6ae40a29a14ed733ee0ba0cb4c" + download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/" traces: - path: glmark2/desktop-blur-radius=5:effect=blur:passes=1:separable=true:windows=4.rdc diff --git a/.gitlab-ci/tracie/tracie.py b/.gitlab-ci/tracie/tracie.py index 59016cae164..b09bf4a9014 100644 --- a/.gitlab-ci/tracie/tracie.py +++ b/.gitlab-ci/tracie/tracie.py @@ -36,55 +36,9 @@ def replay(trace_path, device_name): log_file = files[0] return hashlib.md5(Image.open(image_file).tobytes()).hexdigest(), image_file, log_file -def gitlab_download_metadata(project_url, repo_commit, trace_path): - url = parse.urlparse(project_url) - - url_path = url.path - if url_path.startswith("/"): - url_path = url_path[1:] - - gitlab_api_url = url.scheme + "://" + url.netloc + "/api/v4/projects/" + parse.quote_plus(url_path) - - r = requests.get(gitlab_api_url + "/repository/files/%s/raw?ref=%s" % (parse.quote_plus(trace_path), repo_commit)) - metadata_raw = r.text.strip().split('\n') - metadata = dict(line.split(' ', 1) for line in metadata_raw[1:]) - oid = metadata["oid"][7:] if metadata["oid"].startswith('sha256:') else metadata["oid"] - size = int(metadata['size']) - - return oid, size - -def gitlfs_download_trace(repo_url, repo_commit, trace_path, oid, size): - headers = { - "Accept": "application/vnd.git-lfs+json", - "Content-Type": "application/vnd.git-lfs+json" - } - json = { - "operation": "download", - "transfers": [ "basic" ], - "ref": { "name": "refs/heads/%s" % repo_commit }, - "objects": [ - { - "oid": oid, - "size": size - } - ] - } - - r = requests.post(repo_url + "/info/lfs/objects/batch", headers=headers, json=json) - url = r.json()["objects"][0]["actions"]["download"]["href"] - open(TRACES_DB_PATH + trace_path, "wb").write(requests.get(url).content) - -def checksum(filename, hash_factory=hashlib.sha256, chunk_num_blocks=128): - h = hash_factory() - with open(filename,'rb') as f: - for chunk in iter(lambda: f.read(chunk_num_blocks*h.block_size), b''): - h.update(chunk) - return h.hexdigest() - -def gitlab_ensure_trace(project_url, repo_commit, trace): +def gitlab_ensure_trace(project_url, trace): trace_path = TRACES_DB_PATH + trace['path'] if project_url is None: - assert(repo_commit is None) if not os.path.exists(trace_path): print("{} missing".format(trace_path)) sys.exit(1) @@ -93,18 +47,16 @@ def gitlab_ensure_trace(project_url, repo_commit, trace): os.makedirs(os.path.dirname(trace_path), exist_ok=True) if os.path.exists(trace_path): - local_oid = checksum(trace_path) + return - remote_oid, size = gitlab_download_metadata(project_url, repo_commit, trace['path']) + print("[check_image] Downloading trace %s" % (trace['path']), end=" ", flush=True) + download_time = time.time() + r = requests.get(project_url + trace['path']) + open(trace_path, "wb").write(r.content) + print("took %ds." % (time.time() - download_time), flush=True) - if not os.path.exists(trace_path) or local_oid != remote_oid: - print("[check_image] Downloading trace %s" % (trace['path']), end=" ", flush=True) - download_time = time.time() - gitlfs_download_trace(project_url + ".git", repo_commit, trace['path'], remote_oid, size) - print("took %ds." % (time.time() - download_time), flush=True) - -def gitlab_check_trace(project_url, repo_commit, device_name, trace, expectation): - gitlab_ensure_trace(project_url, repo_commit, trace) +def gitlab_check_trace(project_url, device_name, trace, expectation): + gitlab_ensure_trace(project_url, trace) result = {} result[trace['path']] = {} @@ -145,11 +97,9 @@ def run(filename, device_name): y = yaml.safe_load(f) if "traces-db" in y: - project_url = y["traces-db"]["gitlab-project-url"] - commit_id = y["traces-db"]["commit"] + project_url = y["traces-db"]["download-url"] else: project_url = None - commit_id = None traces = y['traces'] or [] all_ok = True @@ -157,7 +107,7 @@ def run(filename, device_name): for trace in traces: for expectation in trace['expectations']: if expectation['device'] == device_name: - ok, result = gitlab_check_trace(project_url, commit_id, + ok, result = gitlab_check_trace(project_url, device_name, trace, expectation) all_ok = all_ok and ok