ci/lava: Generate YAML from Python, not Jinja

This makes it much easier to use the common init scripts ... which we
also do here.

Signed-off-by: Daniel Stone <daniels@collabora.com>
Acked-by: Martin Peres <martin.peres@mupuf.org>
Acked-by: Emma Anholt <emma@anholt.net>
Reviewed-by: Tomeu Vizoso <tomeu.vizoso@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/11337>

.gitlab-ci/common/init-stage1.sh

@@ -5,6 +5,8 @@
 
 set -ex
 
+cd /
+
 mount -t proc none /proc
 mount -t sysfs none /sys
 mount -t devtmpfs none /dev || echo possibly already mounted

.gitlab-ci/lava/lava-submit.sh

@@ -29,12 +29,13 @@ touch results/lava.log
 tail -f results/lava.log &
 artifacts/lava/lava_job_submitter.py \
 	--dump-yaml \
-	--template artifacts/lava/lava.yml.jinja2 \
 	--pipeline-info "$CI_JOB_NAME: $CI_PIPELINE_URL on $CI_COMMIT_REF_NAME ${CI_NODE_INDEX}/${CI_NODE_TOTAL}" \
 	--base-system-url-prefix "https://${BASE_SYSTEM_HOST_PATH}" \
 	--mesa-build-url "${FDO_HTTP_CACHE_URI:-}https://${MESA_BUILD_PATH}" \
 	--job-rootfs-overlay-url "${FDO_HTTP_CACHE_URI:-}https://${JOB_ROOTFS_OVERLAY_PATH}" \
 	--job-artifacts-base ${JOB_ARTIFACTS_BASE} \
+	--first-stage-init artifacts/ci-common/init-stage1.sh \
+	--ci-project-dir ${CI_PROJECT_DIR} \
 	--device-type ${DEVICE_TYPE} \
 	--dtb ${DTB} \
 	--jwt "${CI_JOB_JWT}" \

.gitlab-ci/lava/lava.yml.jinja2

@@ -1,84 +0,0 @@
-job_name: "mesa: {{ pipeline_info }}"
-device_type: {{ device_type }}
-context:
-  extra_nfsroot_args: " init=/init rootwait minio_results={{ job_artifacts_base }}"
-timeouts:
-  job:
-    minutes: 30
-priority: 75
-visibility:
-    group:
-      - "Collabora+fdo"
-{% if tags %}
-{% set lavatags = tags.split(',') %}
-tags:
-{% for tag in lavatags %}
-  - {{ tag }}
-{% endfor %}
-{% endif %}
-actions:
-- deploy:
-    timeout:
-      minutes: 10
-    to: tftp
-    kernel:
-      url: {{ base_system_url_prefix }}/{{ kernel_image_name }}
-{% if kernel_image_type %}
-      type: {{ kernel_image_type }}
-{% endif %}
-    nfsrootfs:
-      url: {{ base_system_url_prefix }}/lava-rootfs.tgz
-      compression: gz
-{% if dtb %}
-    dtb:
-      url: {{ base_system_url_prefix }}/{{ dtb }}.dtb
-{% endif %}
-    os: oe
-- boot:
-    timeout:
-      minutes: 25
-    method: {{ boot_method }}
-    commands: nfs
-    prompts:
-      - 'lava-shell:'
-- test:
-    timeout:
-      minutes: 30
-    failure_retry: 1
-    definitions:
-    - repository:
-        metadata:
-          format: Lava-Test Test Definition 1.0
-          name: mesa
-          description: "Mesa test plan"
-          os:
-          - oe
-          scope:
-          - functional
-        run:
-          steps:
-          # A bunch of setup we have to do before we can pull anything
-          - cd /
-          - mount -t proc none /proc
-          - mount -t sysfs none /sys
-          - mount -t devtmpfs none /dev || echo possibly already mounted
-          - mkdir -p /dev/pts
-          - mount -t devpts devpts /dev/pts
-          - mkdir -p /dev/shm
-          - mount -t tmpfs tmpfs /dev/shm
-          - mount -t tmpfs tmpfs /tmp
-
-          - echo "nameserver 8.8.8.8" > /etc/resolv.conf
-          - echo "$NFS_SERVER_IP caching-proxy" >> /etc/hosts
-          - for i in 1 2 3; do sntp -sS pool.ntp.org && break || sleep 2; done
-
-          - mkdir -p $CI_PROJECT_DIR
-          - wget -S --progress=dot:giga -O- {{ mesa_build_url }} | tar -xz -C $CI_PROJECT_DIR
-          - wget -S --progress=dot:giga -O- {{ job_rootfs_overlay_url }} | tar -xz -C /
-          - export CI_JOB_JWT="{{ jwt }}"
-          - exec /init-stage2.sh
-        parse:
-          pattern: 'hwci: (?P<test_case_id>\S*):\s+(?P<result>(pass|fail))'
-      from: inline
-      name: mesa
-      path: inline/mesa.yaml

.gitlab-ci/lava/lava_job_submitter.py

@@ -25,7 +25,6 @@
 """Send a job to LAVA, track it and collect log back"""
 
 import argparse
-import jinja2
 import lavacli
 import os
 import sys
@@ -47,31 +46,106 @@ def fatal_err(msg):
     sys.exit(1)
 
 def generate_lava_yaml(args):
-    env = jinja2.Environment(loader = jinja2.FileSystemLoader(os.path.dirname(args.template)), trim_blocks=True, lstrip_blocks=True)
-    template = env.get_template(os.path.basename(args.template))
+    # General metadata and permissions, plus also inexplicably kernel arguments
+    values = {
+        'job_name': 'mesa: {}'.format(args.pipeline_info),
+        'device_type': args.device_type,
+        'visibility': { 'group': [ 'Collabora+fdo'] },
+        'priority': 75,
+        'context': {
+            'extra_nfsroot_args': ' init=/init rootwait minio_results={}'.format(args.job_artifacts_base)
+        },
+        'timeouts': {
+            'job': {
+                'minutes': 30
+            }
+        },
+    }
 
-    values = {}
-    values['pipeline_info'] = args.pipeline_info
-    values['base_system_url_prefix'] = args.base_system_url_prefix
-    values['mesa_build_url'] = args.mesa_build_url
-    values['job_rootfs_overlay_url'] = args.job_rootfs_overlay_url
-    values['job_artifacts_base'] = args.job_artifacts_base
-    values['device_type'] = args.device_type
-    values['dtb'] = args.dtb
-    values['kernel_image_name'] = args.kernel_image_name
-    values['kernel_image_type'] = args.kernel_image_type
-    values['boot_method'] = args.boot_method
-    values['tags'] = args.lava_tags
+    if args.lava_tags:
+        values['tags'] = args.lava_tags.split(',')
 
-    if args.dump_yaml:
-        dump_values = values
-        dump_values['jwt'] = 'xxx'
-        print(template.render(dump_values))
+    # URLs to our kernel rootfs to boot from, both generated by the base
+    # container build
+    deploy = {
+      'timeout': { 'minutes': 10 },
+      'to': 'tftp',
+      'os': 'oe',
+      'kernel': {
+        'url': '{}/{}'.format(args.base_system_url_prefix, args.kernel_image_name),
+      },
+      'nfsrootfs': {
+        'url': '{}/lava-rootfs.tgz'.format(args.base_system_url_prefix),
+        'compression': 'gz',
+      }
+    }
+    if args.kernel_image_type:
+        deploy['kernel']['type'] = args.kernel_image_type
+    if args.dtb:
+        deploy['dtb'] = {
+          'url': '{}/{}.dtb'.format(args.base_system_url_prefix, args.dtb)
+        }
 
-    values['jwt'] = args.jwt
-    yaml = template.render(values)
+    # always boot over NFS
+    boot = {
+      'timeout': { 'minutes': 25 },
+      'method': args.boot_method,
+      'commands': 'nfs',
+      'prompts': ['lava-shell:'],
+    }
 
-    return yaml
+    # skeleton test definition: only declaring each job as a single 'test'
+    # since LAVA's test parsing is not useful to us
+    test = {
+      'timeout': { 'minutes': 30 },
+      'failure_retry': 1,
+      'definitions': [ {
+        'name': 'mesa',
+        'from': 'inline',
+        'path': 'inline/mesa.yaml',
+        'repository': {
+          'metadata': {
+            'name': 'mesa',
+            'description': 'Mesa test plan',
+            'os': [ 'oe' ],
+            'scope': [ 'functional' ],
+            'format': 'Lava-Test Test Definition 1.0',
+          },
+          'parse': {
+            'pattern': 'hwci: (?P<test_case_id>\S*):\s+(?P<result>(pass|fail))'
+          },
+          'run': {
+          },
+        },
+      } ],
+    }
+
+    # job execution script:
+    #   - inline .gitlab-ci/common/init-stage1.sh
+    #   - fetch and unpack per-pipeline build artifacts from build job
+    #   - fetch and unpack per-job environment from lava-submit.sh
+    #   - exec .gitlab-ci/common/init-stage2.sh 
+    init_lines = []
+    with open(args.first_stage_init, 'r') as init_sh:
+      init_lines += [ x.rstrip() for x in init_sh if not x.startswith('#') and x.rstrip() ]
+    init_lines += [
+      'mkdir -p {}'.format(args.ci_project_dir),
+      'wget -S --progress=dot:giga -O- {} | tar -xz -C {}'.format(args.mesa_build_url, args.ci_project_dir),
+      'wget -S --progress=dot:giga -O- {} | tar -xz -C /'.format(args.job_rootfs_overlay_url),
+      'set +x',
+      'export CI_JOB_JWT="{}"'.format(args.jwt),
+      'set -x',
+      'exec /init-stage2.sh',
+    ]
+    test['definitions'][0]['repository']['run']['steps'] = init_lines
+
+    values['actions'] = [
+      { 'deploy': deploy },
+      { 'boot': boot },
+      { 'test': test },
+    ]
+
+    return yaml.dump(values, width=10000000)
 
 
 def setup_lava_proxy():
@@ -171,6 +245,11 @@ def main(args):
 
     yaml_file = generate_lava_yaml(args)
 
+    if args.dump_yaml:
+        censored_args = args
+        censored_args.jwt = "jwt-hidden"
+        print(generate_lava_yaml(censored_args))
+
     if args.validate_only:
         ret = validate_job(proxy, yaml_file)
         if not ret:
@@ -200,12 +279,13 @@ if __name__ == '__main__':
     sys.stderr.reconfigure(line_buffering=True)
     parser = argparse.ArgumentParser("LAVA job submitter")
 
-    parser.add_argument("--template")
     parser.add_argument("--pipeline-info")
     parser.add_argument("--base-system-url-prefix")
     parser.add_argument("--mesa-build-url")
     parser.add_argument("--job-rootfs-overlay-url")
     parser.add_argument("--job-artifacts-base")
+    parser.add_argument("--first-stage-init")
+    parser.add_argument("--ci-project-dir")
     parser.add_argument("--device-type")
     parser.add_argument("--dtb", nargs='?', default="")
     parser.add_argument("--kernel-image-name")