ci: add testing for VC4 drivers (Raspberry Pi 3)

This tests OpenGL ES 2.0 CTS suite with VC4 drivers, through baremetal
Raspberry Pi 3 devices.

The devices are connected to a switch that supports Power over Ethernet
(PoE), so the devices can be started/stopped through the switch, and
also to a host that runs the GitLab runner through serial-to-USB cables,
to monitor the devices to know when the testing finishes.

The Raspberries uses a network boot, using NFS and TFTP. For the root
filesystem, they use the one created in the armhf container. For the
kernel/modules case, this is handled externally. Currently it is using
the same kernel/modules that come with the Raspberry Pi OS. In future we
could build them in the same armhf container.

At this moment we only test armhf architecture, as this is the default
one suggested by the Raspberry Pi Foundation. In future we could also
add testing for arm64 architecture.

Finally, for the very rare ocassions where the Raspberry Pi 3 device is
booted but no data is received, it retries the testing for a second
time, powering off and on the device in the process.

v2:
 - Remove commit that exists capture devcoredump (Eric)
 - Squash remaining commits in one (Andres)

v3:
 - Add missing boot timeout check (Juan)

v4:
 - Use locks when running the PoE on/off script (Eric)
 - Use a timeout for serial read (Eric)

v5:
 - Rename stage to "raspberrypi" (Eric)
 - Bump up arm64_test tag (Eric)

v6:
 - Make serial buffer timeout optional (Juan)

Reviewed-by: Eric Anholt <eric@anholt.net>
Signed-off-by: Juan A. Suarez Romero <jasuarez@igalia.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/7628>

.gitlab-ci.yml

@@ -28,6 +28,7 @@ stages:
   - amd
   - arm
   - freedreno
+  - raspberrypi
   - software-renderer
   - layered-backends
   - success
@@ -383,7 +384,7 @@ arm_test-base:
     - .fdo.container-build@debian
     - .container
   variables:
-    FDO_DISTRIBUTION_TAG: &arm_test-base "2020-10-06-clang10-2"
+    FDO_DISTRIBUTION_TAG: &arm_test-base "2020-12-03-expect"
 
 .use-arm_test-base:
   extends:
@@ -401,7 +402,7 @@ arm64_test:
   extends:
     - .use-arm_test-base
   variables:
-    FDO_DISTRIBUTION_TAG: &arm64_test "2020-11-09-db410c-net"
+    FDO_DISTRIBUTION_TAG: &arm64_test "2020-12-03-expect"
 
 .use-arm64_test:
   variables:
@@ -410,6 +411,20 @@ arm64_test:
   needs:
     - arm64_test
 
+# x86 image with armhf rootfs for baremetal testing
+armhf_test:
+  extends:
+    - .use-arm_test-base
+  variables:
+    FDO_DISTRIBUTION_TAG: &armhf_test "2020-12-03-expect"
+
+.use-armhf_test:
+  variables:
+    TAG: *armhf_test
+  image: "$CI_REGISTRY_IMAGE/debian/armhf_test:$TAG"
+  needs:
+    - armhf_test
+
 # Native Windows docker builds
 #
 # Unlike the above Linux-based builds - including MinGW/SCons builds which
@@ -1380,6 +1395,31 @@ arm64_a530_gles31:
     DEQP_VER: gles31
     DEQP_FRACTION: 10
 
+vc4-rpi3-gles2:armhf:
+  extends:
+    - .baremetal-test
+    - .use-armhf_test
+    - .vc4-rules
+  parallel: 4
+  variables:
+    BARE_METAL_TEST_SCRIPT: "/install/deqp-runner.sh"
+    BM_BOOTFS: /boot/armhf
+    BM_KERNEL_MODULES: vc4
+    BM_ROOTFS: /lava-files/rootfs-armhf
+    BM_ROOTFS_EXTRA: /modules/armhf
+    DEQP_EXPECTED_FAILS: deqp-vc4-rpi3-fails.txt
+    DEQP_EXPECTED_RENDERER: VC4
+    DEQP_FLAKES: deqp-vc4-rpi3-flakes.txt
+    DEQP_SKIPS: deqp-vc4-rpi3-skips.txt
+    DEQP_VER: gles2
+  script:
+    - ./install/bare-metal/poe-powered.sh
+  needs:
+    - armhf_test
+    - meson-armhf
+  tags:
+    - igalia-rpi3
+
 # RADV CI
 .test-radv:
   extends: .radv-rules

.gitlab-ci/bare-metal/init.sh

@@ -11,6 +11,8 @@ mount -t tmpfs tmpfs /tmp
 
 . /set-job-env-vars.sh
 
+[ -z "$BM_KERNEL_MODULES" ] || modprobe "$BM_KERNEL_MODULES"
+
 # Store Mesa's disk cache under /tmp, rather than sending it out over NFS.
 export XDG_CACHE_HOME=/tmp
 

.gitlab-ci/bare-metal/poe-off

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [ -z "$BM_POE_INTERFACE" ]; then
+    echo "Must supply the PoE Interface to power off"
+    exit 1
+fi
+
+flock /var/run/poe.lock -c "$CI_PROJECT_DIR/install/bare-metal/poe-set $BM_POE_INTERFACE off"

.gitlab-ci/bare-metal/poe-on

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [ -z "$BM_POE_INTERFACE" ]; then
+    echo "Must supply the PoE Interface to power up"
+    exit 1
+fi
+
+flock /var/run/poe.lock -c "$CI_PROJECT_DIR/install/bare-metal/poe-set $BM_POE_INTERFACE reset"

.gitlab-ci/bare-metal/poe-powered.sh

@@ -0,0 +1,118 @@
+#!/bin/bash
+
+# Boot script for devices attached to a PoE switch, using NFS for the root
+# filesystem.
+
+# We're run from the root of the repo, make a helper var for our paths
+BM=$CI_PROJECT_DIR/install/bare-metal
+
+# Runner config checks
+if [ -z "$BM_SERIAL" ]; then
+  echo "Must set BM_SERIAL in your gitlab-runner config.toml [[runners]] environment"
+  echo "This is the serial port to listen the device."
+  exit 1
+fi
+
+if [ -z "$BM_POE_ADDRESS" ]; then
+  echo "Must set BM_POE_ADDRESS in your gitlab-runner config.toml [[runners]] environment"
+  echo "This is the PoE switch address to connect for powering up/down devices."
+  exit 1
+fi
+
+if [ -z "$BM_POE_USERNAME" ]; then
+  echo "Must set BM_POE_USERNAME in your gitlab-runner config.toml [[runners]] environment"
+  echo "This is the PoE switch username."
+  exit 1
+fi
+
+if [ -z "$BM_POE_PASSWORD" ]; then
+  echo "Must set BM_POE_PASSWORD in your gitlab-runner config.toml [[runners]] environment"
+  echo "This is the PoE switch password."
+  exit 1
+fi
+
+if [ -z "$BM_POE_INTERFACE" ]; then
+  echo "Must set BM_POE_INTERFACE in your gitlab-runner config.toml [[runners]] environment"
+  echo "This is the PoE switch interface where the device is connected."
+  exit 1
+fi
+
+if [ -z "$BM_POWERUP" ]; then
+  echo "Must set BM_POWERUP in your gitlab-runner config.toml [[runners]] environment"
+  echo "This is a shell script that should power up the device and begin its boot sequence."
+  exit 1
+fi
+
+if [ -z "$BM_POWERDOWN" ]; then
+  echo "Must set BM_POWERDOWN in your gitlab-runner config.toml [[runners]] environment"
+  echo "This is a shell script that should power off the device."
+  exit 1
+fi
+
+if [ ! -d /nfs ]; then
+  echo "NFS rootfs directory needs to be mounted at /nfs by the gitlab runner"
+  exit 1
+fi
+
+if [ ! -d /tftp ]; then
+  echo "TFTP directory for this board needs to be mounted at /tftp by the gitlab runner"
+  exit 1
+fi
+
+# job config checks
+if [ -z "$BM_ROOTFS" ]; then
+  echo "Must set BM_ROOTFS to your board's rootfs directory in the job's variables"
+  exit 1
+fi
+
+if [ -z "$BM_BOOTFS" ]; then
+  echo "Must set /boot files for the TFTP boot in the job's variables"
+  exit 1
+fi
+
+if [ -z "$BM_CMDLINE" ]; then
+  echo "Must set BM_CMDLINE to your board's kernel command line arguments"
+  exit 1
+fi
+
+set -ex
+
+# Clear out any previous run's artifacts.
+rm -rf results/
+mkdir -p results
+
+# Create the rootfs in the NFS directory.  rm to make sure it's in a pristine
+# state, since it's volume-mounted on the host.
+rsync -a --delete $BM_ROOTFS/ /nfs/
+
+[ -z $BM_ROOTFS_EXTRA ] || rsync -a $BM_ROOTFS_EXTRA/ /nfs/
+
+mkdir -p /nfs/results
+. $BM/rootfs-setup.sh /nfs
+
+rsync -a --delete $BM_BOOTFS/ /tftp/
+
+echo "$BM_CMDLINE" > /tftp/cmdline.txt
+
+set +e
+ATTEMPTS=2
+while [ $((ATTEMPTS--)) -gt 0 ]; do
+  python3 $BM/poe_run.py \
+          --dev="$BM_SERIAL" \
+          --powerup="$BM_POWERUP" \
+          --powerdown="$BM_POWERDOWN"
+  ret=$?
+
+  if [ $ret -eq 2 ]; then
+    echo "Did not detect boot sequence, retrying..."
+  else
+    ATTEMPTS=0
+  fi
+done
+set -e
+
+# Bring artifacts back from the NFS dir to the build dir where gitlab-runner
+# will look for them.
+cp -Rp /nfs/results/. results/
+
+exit $ret

.gitlab-ci/bare-metal/poe-set

@@ -0,0 +1,42 @@
+#!/usr/bin/expect -f
+set SWITCHSERVER $env(BM_POE_ADDRESS)
+set USERNAME $env(BM_POE_USERNAME)
+set PASSWORD $env(BM_POE_PASSWORD)
+
+set PORTNUMBER [lindex $argv 0]
+set POESTATUS [lindex $argv 1]
+
+log_user 0
+
+spawn telnet $SWITCHSERVER
+expect "Login"
+sleep 1
+send "$USERNAME\t$PASSWORD\r"
+expect "Menu"
+send "\x01"
+expect ">"
+send "lcli\r"
+expect "Name:"
+send "$USERNAME\r"
+expect "Password:"
+send "$PASSWORD\r"
+expect "#"
+send "configure\r"
+expect "(config)#"
+send "interface GE $PORTNUMBER\r"
+expect "(config-if)#"
+if { "$POESTATUS" == "off" } {
+   send "power inline never\r"
+} elseif { "$POESTATUS" == "on" } {
+   send "power inline auto\r"
+} elseif { "$POESTATUS" == "reset" } {
+   send "power inline never\r"
+   send "power inline auto\r"
+}
+expect "(config-if)#"
+send "exit\r"
+expect "(config)#"
+send "exit\r"
+expect "$#"
+send "exit\r"
+expect eof

.gitlab-ci/bare-metal/poe_run.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+#
+# Copyright © 2020 Igalia, S.L.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+import argparse
+import os
+import re
+from serial_buffer import SerialBuffer
+import sys
+import threading
+
+class PoERun:
+    def __init__(self, args):
+        self.powerup = args.powerup
+        self.powerdown = args.powerdown
+        self.ser = SerialBuffer(args.dev, "results/serial-output.txt", "", 60)
+
+    def logged_system(self, cmd):
+        print("Running '{}'".format(cmd))
+        return os.system(cmd)
+
+    def run(self):
+        if self.logged_system(self.powerup) != 0:
+            return 1
+
+        boot_detected = False
+        for line in self.ser.lines():
+            if re.search("Booting Linux", line):
+                boot_detected = True
+                break
+
+        if not boot_detected:
+            print("Something wrong; couldn't detect the boot start up sequence")
+            self.logged_system(self.powerdown)
+            return 2
+
+        for line in self.ser.lines():
+            if re.search("---. end Kernel panic", line):
+                return 1
+
+            # Binning memory problems
+            if re.search("binner overflow mem", line):
+                print("Memory overflow in the binner; GPU hang")
+                return 1
+
+            result = re.search("bare-metal result: (\S*)", line)
+            if result:
+                if result.group(1) == "pass":
+                    return 0
+                else:
+                    return 1
+
+        print("Reached the end of the CPU serial log without finding a result")
+        return 1
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--dev', type=str, help='Serial device to monitor', required=True)
+    parser.add_argument('--powerup', type=str, help='shell command for rebooting', required=True)
+    parser.add_argument('--powerdown', type=str, help='shell command for powering off', required=True)
+    args = parser.parse_args()
+
+    poe = PoERun(args)
+    retval = poe.run()
+
+    poe.logged_system(args.powerdown)
+
+    sys.exit(retval)
+
+if __name__ == '__main__':
+    main()

.gitlab-ci/bare-metal/rootfs-setup.sh

@@ -15,6 +15,7 @@ touch $rootfs_dst/set-job-env-vars.sh
 chmod +x $rootfs_dst/set-job-env-vars.sh
 for var in \
     BARE_METAL_TEST_SCRIPT \
+    BM_KERNEL_MODULES \
     CI_COMMIT_BRANCH \
     CI_COMMIT_TITLE \
     CI_JOB_JWT \

.gitlab-ci/bare-metal/serial_buffer.py

@@ -30,19 +30,20 @@ import time
 
 
 class SerialBuffer:
-    def __init__(self, dev, filename, prefix):
+    def __init__(self, dev, filename, prefix, timeout = None):
         self.filename = filename
         self.dev = dev
 
         if dev:
             self.f = open(filename, "wb+")
-            self.serial = serial.Serial(dev, 115200, timeout=10)
+            self.serial = serial.Serial(dev, 115200, timeout=timeout if timeout else 10)
         else:
             self.f = open(filename, "rb")
 
         self.byte_queue = queue.Queue()
         self.line_queue = queue.Queue()
         self.prefix = prefix
+        self.timeout = timeout
         self.sentinel = object()
 
         if self.dev:
@@ -58,14 +59,19 @@ class SerialBuffer:
         self.lines_thread.start()
 
     # Thread that just reads the bytes from the serial device to try to keep from
-    # buffer overflowing it.
+    # buffer overflowing it. If nothing is received in 1 minute, it finalizes.
     def serial_read_thread_loop(self):
         greet = "Serial thread reading from %s\n" % self.dev
         self.byte_queue.put(greet.encode())
 
         while True:
             try:
-                self.byte_queue.put(self.serial.read())
+                b = self.serial.read()
+                if len(b) > 0:
+                    self.byte_queue.put(b)
+                elif self.timeout:
+                    self.byte_queue.put(self.sentinel)
+                    break
             except Exception as err:
                 print(self.prefix + str(err))
                 self.byte_queue.put(self.sentinel)

.gitlab-ci/container/arm_test-base.sh

@@ -19,6 +19,7 @@ apt-get install -y --no-remove \
         cmake \
         cpio \
         debootstrap \
+        expect \
         fastboot \
         flex \
         g++ \
@@ -32,6 +33,7 @@ apt-get install -y --no-remove \
         python3-serial \
         python3.7 \
         rsync \
+        telnet \
         u-boot-tools \
         unzip
 

.gitlab-ci/container/armhf_test.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+arch=armhf
+
+. .gitlab-ci/container/baremetal_build.sh

.gitlab-ci/deqp-vc4-rpi3-fails.txt

@@ -0,0 +1,23 @@
+dEQP-GLES2.functional.clipping.line.wide_line_clip_viewport_center,Fail
+dEQP-GLES2.functional.clipping.line.wide_line_clip_viewport_corner,Fail
+dEQP-GLES2.functional.depth_stencil_clear.depth_stencil_masked,Fail
+dEQP-GLES2.functional.draw.draw_arrays.line_loop.multiple_attributes,Fail
+dEQP-GLES2.functional.draw.draw_arrays.line_loop.single_attribute,Fail
+dEQP-GLES2.functional.fbo.completeness.size.distinct,Fail
+dEQP-GLES2.functional.fbo.render.texsubimage.after_render_tex2d_rgba,Fail
+dEQP-GLES2.functional.fbo.render.texsubimage.between_render_tex2d_rgba,Fail
+dEQP-GLES2.functional.negative_api.shader.uniform_matrixfv_invalid_transpose,Fail
+dEQP-GLES2.functional.negative_api.texture.generatemipmap_zero_level_array_compressed,Fail
+dEQP-GLES2.functional.negative_api.vertex_array.vertex_attrib,Fail
+dEQP-GLES2.functional.negative_api.vertex_array.vertex_attribv,Fail
+dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_linear_mirror_rgba8888,Fail
+dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_linear_linear_repeat_rgba8888,Fail
+dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_nearest_linear_mirror_rgba8888,Fail
+dEQP-GLES2.functional.texture.filtering.2d.nearest_mipmap_nearest_linear_repeat_rgba8888,Fail
+dEQP-GLES2.functional.texture.mipmap.2d.basic.linear_linear_repeat_non_square,Fail
+dEQP-GLES2.functional.texture.mipmap.2d.basic.nearest_linear_clamp_non_square,Fail
+dEQP-GLES2.functional.texture.mipmap.2d.basic.nearest_linear_mirror_non_square,Fail
+dEQP-GLES2.functional.texture.mipmap.2d.basic.nearest_linear_repeat_non_square,Fail
+dEQP-GLES2.functional.texture.specification.basic_copytexsubimage2d.2d_rgba,Fail
+dEQP-GLES2.functional.texture.specification.basic_copytexsubimage2d.cube_rgba,Fail
+dEQP-GLES2.functional.texture.wrap.clamp_clamp_nearest_npot_etc1,Fail

.gitlab-ci/deqp-vc4-rpi3-flakes.txt

@@ -0,0 +1,30 @@
+dEQP-GLES2.functional.clipping.triangle_vertex.clip_three.clip_neg_x_and_pos_x_and_neg_x_neg_y_neg_z
+dEQP-GLES2.functional.clipping.triangle_vertex.clip_three.clip_neg_x_and_pos_x_and_pos_y_pos_z
+dEQP-GLES2.functional.clipping.triangle_vertex.clip_three.clip_neg_x_and_pos_x_neg_y_pos_z_and_neg_x_pos_y_neg_z
+dEQP-GLES2.functional.clipping.triangle_vertex.clip_three.clip_pos_x_and_neg_x_neg_y_pos_z_and_neg_x_pos_y_neg_z
+dEQP-GLES2.functional.draw.random.51
+dEQP-GLES2.functional.fragment_ops.blend.rgb_func_alpha_func.src.one_minus_src_alpha_constant_color
+dEQP-GLES2.functional.shaders.indexing.vector_subscript.vec4_direct_write_dynamic_loop_subscript_read_vertex
+dEQP-GLES2.functional.shaders.loops.do_while_dynamic_iterations.basic_mediump_int_vertex
+dEQP-GLES2.functional.shaders.loops.do_while_dynamic_iterations.conditional_continue_vertex
+dEQP-GLES2.functional.shaders.loops.do_while_dynamic_iterations.function_call_inout_vertex
+dEQP-GLES2.functional.shaders.loops.do_while_dynamic_iterations.function_call_return_vertex
+dEQP-GLES2.functional.shaders.loops.do_while_dynamic_iterations.nested_sequence_vertex
+dEQP-GLES2.functional.shaders.loops.while_constant_iterations.select_iteration_count_vertex
+dEQP-GLES2.functional.shaders.loops.while_dynamic_iterations.function_call_return_vertex
+dEQP-GLES2.functional.shaders.loops.while_dynamic_iterations.infinite_with_conditional_break_vertex
+dEQP-GLES2.functional.shaders.loops.while_dynamic_iterations.post_increment_vertex
+dEQP-GLES2.functional.shaders.loops.while_dynamic_iterations.single_iteration_vertex
+dEQP-GLES2.functional.shaders.operator.unary_operator.pre_decrement_result.mediump_vec3_fragment
+dEQP-GLES2.functional.shaders.random.exponential.fragment.51
+dEQP-GLES2.functional.shaders.random.texture.fragment.129
+dEQP-GLES2.functional.shaders.return.output_write_in_func_never_vertex
+dEQP-GLES2.functional.texture.filtering.2d.linear_linear_clamp_rgb888_pot
+dEQP-GLES2.functional.texture.filtering.cube.linear_mipmap_linear_nearest_mirror_rgba8888
+dEQP-GLES2.functional.texture.filtering.cube.nearest_linear_mirror_rgba8888_pot
+dEQP-GLES2.functional.texture.filtering.cube.nearest_mipmap_linear_linear_clamp_rgba8888
+dEQP-GLES2.functional.texture.filtering.cube.nearest_mipmap_linear_nearest_repeat_l8
+dEQP-GLES2.functional.texture.filtering.cube.nearest_mipmap_nearest_linear_clamp_rgba8888
+dEQP-GLES2.functional.texture.filtering.cube.nearest_mipmap_nearest_linear_mirror_rgba8888
+dEQP-GLES2.functional.texture.mipmap.cube.generate.rgb565_fastest
+dEQP-GLES2.functional.texture.size.cube.256x256_rgb888

.gitlab-ci/deqp-vc4-rpi3-skips.txt

@@ -0,0 +1,17 @@
+# Note: skips lists for CI are just a list of lines that, when
+# non-zero-length and not starting with '#', will regex match to
+# delete lines from the test list.  Be careful.
+
+# Skip the perf/stress tests to keep runtime manageable
+dEQP-GLES[0-9]*.performance.*
+dEQP-GLES[0-9]*.stress.*
+
+# These are really slow on tiling architectures (including llvmpipe).
+dEQP-GLES[0-9]*.functional.flush_finish.*
+
+# This is causing a binning memory overflow problem
+dEQP-GLES2.functional.fragment_ops.scissor.outside_render_line
+
+# These are very slow
+dEQP-GLES2.functional.uniform_api.random.3
+dEQP-GLES2.functional.uniform_api.random.79

.gitlab-ci/test-source-dep.yml

@@ -147,6 +147,24 @@
       when: on_success
     - when: never
 
+.vc4-rules:
+  stage: raspberrypi
+  rules:
+    - *ignore_scheduled_pipelines
+    - changes:
+        *mesa_core_file_list
+      when: on_success
+    - changes:
+        *gallium_core_file_list
+      when: on_success
+    - changes:
+      - src/broadcom/**/*
+      - src/gallium/drivers/vc4/**/*
+      - src/gallium/winsys/kmsro/**/*
+      - src/gallium/winsys/vc4/**/*
+      when: on_success
+    - when: never
+
 .lima-rules:
   stage: arm
   rules: