ci/lava: Generate YAML from Python, not Jinja

This makes it much easier to use the common init scripts ... which we
also do here.

Signed-off-by: Daniel Stone <daniels@collabora.com>
Acked-by: Martin Peres <martin.peres@mupuf.org>
Acked-by: Emma Anholt <emma@anholt.net>
Reviewed-by: Tomeu Vizoso <tomeu.vizoso@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/11337>

.gitlab-ci/lava/lava_job_submitter.py

@@ -25,7 +25,6 @@
 """Send a job to LAVA, track it and collect log back"""
 
 import argparse
-import jinja2
 import lavacli
 import os
 import sys
@@ -47,31 +46,106 @@ def fatal_err(msg):
     sys.exit(1)
 
 def generate_lava_yaml(args):
-    env = jinja2.Environment(loader = jinja2.FileSystemLoader(os.path.dirname(args.template)), trim_blocks=True, lstrip_blocks=True)
-    template = env.get_template(os.path.basename(args.template))
+    # General metadata and permissions, plus also inexplicably kernel arguments
+    values = {
+        'job_name': 'mesa: {}'.format(args.pipeline_info),
+        'device_type': args.device_type,
+        'visibility': { 'group': [ 'Collabora+fdo'] },
+        'priority': 75,
+        'context': {
+            'extra_nfsroot_args': ' init=/init rootwait minio_results={}'.format(args.job_artifacts_base)
+        },
+        'timeouts': {
+            'job': {
+                'minutes': 30
+            }
+        },
+    }
 
-    values = {}
-    values['pipeline_info'] = args.pipeline_info
-    values['base_system_url_prefix'] = args.base_system_url_prefix
-    values['mesa_build_url'] = args.mesa_build_url
-    values['job_rootfs_overlay_url'] = args.job_rootfs_overlay_url
-    values['job_artifacts_base'] = args.job_artifacts_base
-    values['device_type'] = args.device_type
-    values['dtb'] = args.dtb
-    values['kernel_image_name'] = args.kernel_image_name
-    values['kernel_image_type'] = args.kernel_image_type
-    values['boot_method'] = args.boot_method
-    values['tags'] = args.lava_tags
+    if args.lava_tags:
+        values['tags'] = args.lava_tags.split(',')
 
-    if args.dump_yaml:
-        dump_values = values
-        dump_values['jwt'] = 'xxx'
-        print(template.render(dump_values))
+    # URLs to our kernel rootfs to boot from, both generated by the base
+    # container build
+    deploy = {
+      'timeout': { 'minutes': 10 },
+      'to': 'tftp',
+      'os': 'oe',
+      'kernel': {
+        'url': '{}/{}'.format(args.base_system_url_prefix, args.kernel_image_name),
+      },
+      'nfsrootfs': {
+        'url': '{}/lava-rootfs.tgz'.format(args.base_system_url_prefix),
+        'compression': 'gz',
+      }
+    }
+    if args.kernel_image_type:
+        deploy['kernel']['type'] = args.kernel_image_type
+    if args.dtb:
+        deploy['dtb'] = {
+          'url': '{}/{}.dtb'.format(args.base_system_url_prefix, args.dtb)
+        }
 
-    values['jwt'] = args.jwt
-    yaml = template.render(values)
+    # always boot over NFS
+    boot = {
+      'timeout': { 'minutes': 25 },
+      'method': args.boot_method,
+      'commands': 'nfs',
+      'prompts': ['lava-shell:'],
+    }
 
-    return yaml
+    # skeleton test definition: only declaring each job as a single 'test'
+    # since LAVA's test parsing is not useful to us
+    test = {
+      'timeout': { 'minutes': 30 },
+      'failure_retry': 1,
+      'definitions': [ {
+        'name': 'mesa',
+        'from': 'inline',
+        'path': 'inline/mesa.yaml',
+        'repository': {
+          'metadata': {
+            'name': 'mesa',
+            'description': 'Mesa test plan',
+            'os': [ 'oe' ],
+            'scope': [ 'functional' ],
+            'format': 'Lava-Test Test Definition 1.0',
+          },
+          'parse': {
+            'pattern': 'hwci: (?P<test_case_id>\S*):\s+(?P<result>(pass|fail))'
+          },
+          'run': {
+          },
+        },
+      } ],
+    }
+
+    # job execution script:
+    #   - inline .gitlab-ci/common/init-stage1.sh
+    #   - fetch and unpack per-pipeline build artifacts from build job
+    #   - fetch and unpack per-job environment from lava-submit.sh
+    #   - exec .gitlab-ci/common/init-stage2.sh 
+    init_lines = []
+    with open(args.first_stage_init, 'r') as init_sh:
+      init_lines += [ x.rstrip() for x in init_sh if not x.startswith('#') and x.rstrip() ]
+    init_lines += [
+      'mkdir -p {}'.format(args.ci_project_dir),
+      'wget -S --progress=dot:giga -O- {} | tar -xz -C {}'.format(args.mesa_build_url, args.ci_project_dir),
+      'wget -S --progress=dot:giga -O- {} | tar -xz -C /'.format(args.job_rootfs_overlay_url),
+      'set +x',
+      'export CI_JOB_JWT="{}"'.format(args.jwt),
+      'set -x',
+      'exec /init-stage2.sh',
+    ]
+    test['definitions'][0]['repository']['run']['steps'] = init_lines
+
+    values['actions'] = [
+      { 'deploy': deploy },
+      { 'boot': boot },
+      { 'test': test },
+    ]
+
+    return yaml.dump(values, width=10000000)
 
 
 def setup_lava_proxy():
@@ -171,6 +245,11 @@ def main(args):
 
     yaml_file = generate_lava_yaml(args)
 
+    if args.dump_yaml:
+        censored_args = args
+        censored_args.jwt = "jwt-hidden"
+        print(generate_lava_yaml(censored_args))
+
     if args.validate_only:
         ret = validate_job(proxy, yaml_file)
         if not ret:
@@ -200,12 +279,13 @@ if __name__ == '__main__':
     sys.stderr.reconfigure(line_buffering=True)
     parser = argparse.ArgumentParser("LAVA job submitter")
 
-    parser.add_argument("--template")
     parser.add_argument("--pipeline-info")
     parser.add_argument("--base-system-url-prefix")
     parser.add_argument("--mesa-build-url")
     parser.add_argument("--job-rootfs-overlay-url")
     parser.add_argument("--job-artifacts-base")
+    parser.add_argument("--first-stage-init")
+    parser.add_argument("--ci-project-dir")
     parser.add_argument("--device-type")
     parser.add_argument("--dtb", nargs='?', default="")
     parser.add_argument("--kernel-image-name")