OpenHarmony/third_party_mesa3d
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

gitlab-ci: Download traces from MinIO

Browse Source 
Downloading the traces directly from git causes very high egress from
GCE, which is expensive.

So we can expand trace testing further, we are going to keep a cache in
freedesktop.org's MinIO instance. This commit implements downloading
from it.

Signed-off-by: Tomeu Vizoso <tomeu.vizoso@collabora.com>
Reviewed-by: Daniel Stone <daniels@collabora.com>
Reviewed-By: Rohan Garg <rohan.garg@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/5472>
This commit is contained in:
Tomeu Vizoso
2020-07-09 12:42:02 +02:00
committed by Marge Bot
parent 087be7e322
commit 20507f8b17
2 changed files with 12 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
.gitlab-ci/tracie/tracie.py
View File

@@ -36,55 +36,9 @@ def replay(trace_path, device_name):
log_file = files[0]
return hashlib.md5(Image.open(image_file).tobytes()).hexdigest(), image_file, log_file
def gitlab_download_metadata(project_url, repo_commit, trace_path):
url = parse.urlparse(project_url)
url_path = url.path
if url_path.startswith("/"):
url_path = url_path[1:]
gitlab_api_url = url.scheme + "://" + url.netloc + "/api/v4/projects/" + parse.quote_plus(url_path)
r = requests.get(gitlab_api_url + "/repository/files/%s/raw?ref=%s" % (parse.quote_plus(trace_path), repo_commit))
metadata_raw = r.text.strip().split('\n')
metadata = dict(line.split(' ', 1) for line in metadata_raw[1:])
oid = metadata["oid"][7:] if metadata["oid"].startswith('sha256:') else metadata["oid"]
size = int(metadata['size'])
return oid, size
def gitlfs_download_trace(repo_url, repo_commit, trace_path, oid, size):
headers = {
"Accept": "application/vnd.git-lfs+json",
"Content-Type": "application/vnd.git-lfs+json"
}
json = {
"operation": "download",
"transfers": [ "basic" ],
"ref": { "name": "refs/heads/%s" % repo_commit },
"objects": [
{
"oid": oid,
"size": size
}
]
}
r = requests.post(repo_url + "/info/lfs/objects/batch", headers=headers, json=json)
url = r.json()["objects"][0]["actions"]["download"]["href"]
open(TRACES_DB_PATH + trace_path, "wb").write(requests.get(url).content)
def checksum(filename, hash_factory=hashlib.sha256, chunk_num_blocks=128):
h = hash_factory()
with open(filename,'rb') as f:
for chunk in iter(lambda: f.read(chunk_num_blocks*h.block_size), b''):
h.update(chunk)
return h.hexdigest()
def gitlab_ensure_trace(project_url, repo_commit, trace):
def gitlab_ensure_trace(project_url, trace):
trace_path = TRACES_DB_PATH + trace['path']
if project_url is None:
assert(repo_commit is None)
if not os.path.exists(trace_path):
print("{} missing".format(trace_path))
sys.exit(1)
@@ -93,18 +47,16 @@ def gitlab_ensure_trace(project_url, repo_commit, trace):
os.makedirs(os.path.dirname(trace_path), exist_ok=True)
if os.path.exists(trace_path):
local_oid = checksum(trace_path)
return
remote_oid, size = gitlab_download_metadata(project_url, repo_commit, trace['path'])
print("[check_image] Downloading trace %s" % (trace['path']), end=" ", flush=True)
download_time = time.time()
r = requests.get(project_url + trace['path'])
open(trace_path, "wb").write(r.content)
print("took %ds." % (time.time() - download_time), flush=True)
if not os.path.exists(trace_path) or local_oid != remote_oid:
print("[check_image] Downloading trace %s" % (trace['path']), end=" ", flush=True)
download_time = time.time()
gitlfs_download_trace(project_url + ".git", repo_commit, trace['path'], remote_oid, size)
print("took %ds." % (time.time() - download_time), flush=True)
def gitlab_check_trace(project_url, repo_commit, device_name, trace, expectation):
gitlab_ensure_trace(project_url, repo_commit, trace)
def gitlab_check_trace(project_url, device_name, trace, expectation):
gitlab_ensure_trace(project_url, trace)
result = {}
result[trace['path']] = {}
@@ -145,11 +97,9 @@ def run(filename, device_name):
y = yaml.safe_load(f)
if "traces-db" in y:
project_url = y["traces-db"]["gitlab-project-url"]
commit_id = y["traces-db"]["commit"]
project_url = y["traces-db"]["download-url"]
else:
project_url = None
commit_id = None
traces = y['traces'] or []
all_ok = True
@@ -157,7 +107,7 @@ def run(filename, device_name):
for trace in traces:
for expectation in trace['expectations']:
if expectation['device'] == device_name:
ok, result = gitlab_check_trace(project_url, commit_id,
ok, result = gitlab_check_trace(project_url,
device_name, trace,
expectation)
all_ok = all_ok and ok