From a439db58441980a22dbcb578763bc3ac2b76615d Mon Sep 17 00:00:00 2001 From: Daniel Stone Date: Fri, 11 Jun 2021 20:04:38 +0100 Subject: [PATCH] ci/lava: Generate YAML from Python, not Jinja This makes it much easier to use the common init scripts ... which we also do here. Signed-off-by: Daniel Stone Acked-by: Martin Peres Acked-by: Emma Anholt Reviewed-by: Tomeu Vizoso Part-of: --- .gitlab-ci/common/init-stage1.sh | 2 + .gitlab-ci/lava/lava-submit.sh | 3 +- .gitlab-ci/lava/lava.yml.jinja2 | 84 ----------------- .gitlab-ci/lava/lava_job_submitter.py | 126 +++++++++++++++++++++----- 4 files changed, 107 insertions(+), 108 deletions(-) delete mode 100644 .gitlab-ci/lava/lava.yml.jinja2 diff --git a/.gitlab-ci/common/init-stage1.sh b/.gitlab-ci/common/init-stage1.sh index 60b1e28366e..648c37a2f90 100755 --- a/.gitlab-ci/common/init-stage1.sh +++ b/.gitlab-ci/common/init-stage1.sh @@ -5,6 +5,8 @@ set -ex +cd / + mount -t proc none /proc mount -t sysfs none /sys mount -t devtmpfs none /dev || echo possibly already mounted diff --git a/.gitlab-ci/lava/lava-submit.sh b/.gitlab-ci/lava/lava-submit.sh index 32d30174ac3..ecd2bba1cf7 100755 --- a/.gitlab-ci/lava/lava-submit.sh +++ b/.gitlab-ci/lava/lava-submit.sh @@ -29,12 +29,13 @@ touch results/lava.log tail -f results/lava.log & artifacts/lava/lava_job_submitter.py \ --dump-yaml \ - --template artifacts/lava/lava.yml.jinja2 \ --pipeline-info "$CI_JOB_NAME: $CI_PIPELINE_URL on $CI_COMMIT_REF_NAME ${CI_NODE_INDEX}/${CI_NODE_TOTAL}" \ --base-system-url-prefix "https://${BASE_SYSTEM_HOST_PATH}" \ --mesa-build-url "${FDO_HTTP_CACHE_URI:-}https://${MESA_BUILD_PATH}" \ --job-rootfs-overlay-url "${FDO_HTTP_CACHE_URI:-}https://${JOB_ROOTFS_OVERLAY_PATH}" \ --job-artifacts-base ${JOB_ARTIFACTS_BASE} \ + --first-stage-init artifacts/ci-common/init-stage1.sh \ + --ci-project-dir ${CI_PROJECT_DIR} \ --device-type ${DEVICE_TYPE} \ --dtb ${DTB} \ --jwt "${CI_JOB_JWT}" \ diff --git a/.gitlab-ci/lava/lava.yml.jinja2 b/.gitlab-ci/lava/lava.yml.jinja2 deleted file mode 100644 index ec71a96e4e7..00000000000 --- a/.gitlab-ci/lava/lava.yml.jinja2 +++ /dev/null @@ -1,84 +0,0 @@ -job_name: "mesa: {{ pipeline_info }}" -device_type: {{ device_type }} -context: - extra_nfsroot_args: " init=/init rootwait minio_results={{ job_artifacts_base }}" -timeouts: - job: - minutes: 30 -priority: 75 -visibility: - group: - - "Collabora+fdo" -{% if tags %} -{% set lavatags = tags.split(',') %} -tags: -{% for tag in lavatags %} - - {{ tag }} -{% endfor %} -{% endif %} -actions: -- deploy: - timeout: - minutes: 10 - to: tftp - kernel: - url: {{ base_system_url_prefix }}/{{ kernel_image_name }} -{% if kernel_image_type %} - type: {{ kernel_image_type }} -{% endif %} - nfsrootfs: - url: {{ base_system_url_prefix }}/lava-rootfs.tgz - compression: gz -{% if dtb %} - dtb: - url: {{ base_system_url_prefix }}/{{ dtb }}.dtb -{% endif %} - os: oe -- boot: - timeout: - minutes: 25 - method: {{ boot_method }} - commands: nfs - prompts: - - 'lava-shell:' -- test: - timeout: - minutes: 30 - failure_retry: 1 - definitions: - - repository: - metadata: - format: Lava-Test Test Definition 1.0 - name: mesa - description: "Mesa test plan" - os: - - oe - scope: - - functional - run: - steps: - # A bunch of setup we have to do before we can pull anything - - cd / - - mount -t proc none /proc - - mount -t sysfs none /sys - - mount -t devtmpfs none /dev || echo possibly already mounted - - mkdir -p /dev/pts - - mount -t devpts devpts /dev/pts - - mkdir -p /dev/shm - - mount -t tmpfs tmpfs /dev/shm - - mount -t tmpfs tmpfs /tmp - - - echo "nameserver 8.8.8.8" > /etc/resolv.conf - - echo "$NFS_SERVER_IP caching-proxy" >> /etc/hosts - - for i in 1 2 3; do sntp -sS pool.ntp.org && break || sleep 2; done - - - mkdir -p $CI_PROJECT_DIR - - wget -S --progress=dot:giga -O- {{ mesa_build_url }} | tar -xz -C $CI_PROJECT_DIR - - wget -S --progress=dot:giga -O- {{ job_rootfs_overlay_url }} | tar -xz -C / - - export CI_JOB_JWT="{{ jwt }}" - - exec /init-stage2.sh - parse: - pattern: 'hwci: (?P\S*):\s+(?P(pass|fail))' - from: inline - name: mesa - path: inline/mesa.yaml diff --git a/.gitlab-ci/lava/lava_job_submitter.py b/.gitlab-ci/lava/lava_job_submitter.py index c74e16150a1..917701a7747 100755 --- a/.gitlab-ci/lava/lava_job_submitter.py +++ b/.gitlab-ci/lava/lava_job_submitter.py @@ -25,7 +25,6 @@ """Send a job to LAVA, track it and collect log back""" import argparse -import jinja2 import lavacli import os import sys @@ -47,31 +46,106 @@ def fatal_err(msg): sys.exit(1) def generate_lava_yaml(args): - env = jinja2.Environment(loader = jinja2.FileSystemLoader(os.path.dirname(args.template)), trim_blocks=True, lstrip_blocks=True) - template = env.get_template(os.path.basename(args.template)) + # General metadata and permissions, plus also inexplicably kernel arguments + values = { + 'job_name': 'mesa: {}'.format(args.pipeline_info), + 'device_type': args.device_type, + 'visibility': { 'group': [ 'Collabora+fdo'] }, + 'priority': 75, + 'context': { + 'extra_nfsroot_args': ' init=/init rootwait minio_results={}'.format(args.job_artifacts_base) + }, + 'timeouts': { + 'job': { + 'minutes': 30 + } + }, + } - values = {} - values['pipeline_info'] = args.pipeline_info - values['base_system_url_prefix'] = args.base_system_url_prefix - values['mesa_build_url'] = args.mesa_build_url - values['job_rootfs_overlay_url'] = args.job_rootfs_overlay_url - values['job_artifacts_base'] = args.job_artifacts_base - values['device_type'] = args.device_type - values['dtb'] = args.dtb - values['kernel_image_name'] = args.kernel_image_name - values['kernel_image_type'] = args.kernel_image_type - values['boot_method'] = args.boot_method - values['tags'] = args.lava_tags + if args.lava_tags: + values['tags'] = args.lava_tags.split(',') - if args.dump_yaml: - dump_values = values - dump_values['jwt'] = 'xxx' - print(template.render(dump_values)) + # URLs to our kernel rootfs to boot from, both generated by the base + # container build + deploy = { + 'timeout': { 'minutes': 10 }, + 'to': 'tftp', + 'os': 'oe', + 'kernel': { + 'url': '{}/{}'.format(args.base_system_url_prefix, args.kernel_image_name), + }, + 'nfsrootfs': { + 'url': '{}/lava-rootfs.tgz'.format(args.base_system_url_prefix), + 'compression': 'gz', + } + } + if args.kernel_image_type: + deploy['kernel']['type'] = args.kernel_image_type + if args.dtb: + deploy['dtb'] = { + 'url': '{}/{}.dtb'.format(args.base_system_url_prefix, args.dtb) + } - values['jwt'] = args.jwt - yaml = template.render(values) + # always boot over NFS + boot = { + 'timeout': { 'minutes': 25 }, + 'method': args.boot_method, + 'commands': 'nfs', + 'prompts': ['lava-shell:'], + } - return yaml + # skeleton test definition: only declaring each job as a single 'test' + # since LAVA's test parsing is not useful to us + test = { + 'timeout': { 'minutes': 30 }, + 'failure_retry': 1, + 'definitions': [ { + 'name': 'mesa', + 'from': 'inline', + 'path': 'inline/mesa.yaml', + 'repository': { + 'metadata': { + 'name': 'mesa', + 'description': 'Mesa test plan', + 'os': [ 'oe' ], + 'scope': [ 'functional' ], + 'format': 'Lava-Test Test Definition 1.0', + }, + 'parse': { + 'pattern': 'hwci: (?P\S*):\s+(?P(pass|fail))' + }, + 'run': { + }, + }, + } ], + } + + # job execution script: + # - inline .gitlab-ci/common/init-stage1.sh + # - fetch and unpack per-pipeline build artifacts from build job + # - fetch and unpack per-job environment from lava-submit.sh + # - exec .gitlab-ci/common/init-stage2.sh + init_lines = [] + with open(args.first_stage_init, 'r') as init_sh: + init_lines += [ x.rstrip() for x in init_sh if not x.startswith('#') and x.rstrip() ] + init_lines += [ + 'mkdir -p {}'.format(args.ci_project_dir), + 'wget -S --progress=dot:giga -O- {} | tar -xz -C {}'.format(args.mesa_build_url, args.ci_project_dir), + 'wget -S --progress=dot:giga -O- {} | tar -xz -C /'.format(args.job_rootfs_overlay_url), + 'set +x', + 'export CI_JOB_JWT="{}"'.format(args.jwt), + 'set -x', + 'exec /init-stage2.sh', + ] + test['definitions'][0]['repository']['run']['steps'] = init_lines + + values['actions'] = [ + { 'deploy': deploy }, + { 'boot': boot }, + { 'test': test }, + ] + + return yaml.dump(values, width=10000000) def setup_lava_proxy(): @@ -171,6 +245,11 @@ def main(args): yaml_file = generate_lava_yaml(args) + if args.dump_yaml: + censored_args = args + censored_args.jwt = "jwt-hidden" + print(generate_lava_yaml(censored_args)) + if args.validate_only: ret = validate_job(proxy, yaml_file) if not ret: @@ -200,12 +279,13 @@ if __name__ == '__main__': sys.stderr.reconfigure(line_buffering=True) parser = argparse.ArgumentParser("LAVA job submitter") - parser.add_argument("--template") parser.add_argument("--pipeline-info") parser.add_argument("--base-system-url-prefix") parser.add_argument("--mesa-build-url") parser.add_argument("--job-rootfs-overlay-url") parser.add_argument("--job-artifacts-base") + parser.add_argument("--first-stage-init") + parser.add_argument("--ci-project-dir") parser.add_argument("--device-type") parser.add_argument("--dtb", nargs='?', default="") parser.add_argument("--kernel-image-name")