virgl/ci: Run each dEQP instance in its own VM

Currently we run deqp-runner inside a single VM, which makes very poor
use of the available CPUs because Virgl has a bottleneck in the VMM that
serializes everything.

With this change, we can run several Crosvm instances in a runner and
make full use of the CPUs. Getting the same coverage with 3 runners
instead of 6.

Signed-off-by: Tomeu Vizoso <tomeu.vizoso@collabora.com>
Reviewed-by: Emma Anholt <emma@anholt.net>
Reviewed-by: Corentin Noël <corentin.noel@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/12828>

.gitlab-ci/crosvm-init.sh

@@ -1,27 +1,29 @@
 #!/bin/sh
 
-set -ex
+set -e
+
+export DEQP_TEMP_DIR=$1
 
 mount -t proc none /proc
 mount -t sysfs none /sys
-mount -t devtmpfs none /dev || echo possibly already mounted
 mkdir -p /dev/pts
 mount -t devpts devpts /dev/pts
 mount -t tmpfs tmpfs /tmp
 
-. /crosvm-env.sh
+. $DEQP_TEMP_DIR/crosvm-env.sh
 
-# / is ro
-export PIGLIT_REPLAY_EXTRA_ARGS="$PIGLIT_REPLAY_EXTRA_ARGS --db-path /tmp/replayer-db"
+cd $PWD
 
-if sh $CROSVM_TEST_SCRIPT; then
-    touch /results/success
-fi
+dmesg --level crit,err,warn -w >> $DEQP_TEMP_DIR/stderr &
 
-sleep 5   # Leave some time to get the last output flushed out
+set +e
+stdbuf -oL sh $DEQP_TEMP_DIR/crosvm-script.sh 2>> $DEQP_TEMP_DIR/stderr >> $DEQP_TEMP_DIR/stdout
+echo $? > $DEQP_TEMP_DIR/exit_code
+set -e
+
+sync
+sleep 1
 
 poweroff -d -n -f || true
 
-sleep 10   # Just in case init would exit before the kernel shuts down the VM
-
-exit 1
+sleep 1   # Just in case init would exit before the kernel shuts down the VM

.gitlab-ci/crosvm-runner.sh

@@ -1,25 +1,20 @@
 #!/bin/sh
 
-set -x
+set -e
 
-ln -sf $CI_PROJECT_DIR/install /install
+# This script can be called concurrently, pass arguments and env in a per-instance tmp dir
+export DEQP_TEMP_DIR=`mktemp -d /tmp.XXXXXXXXXX`
 
-export LD_LIBRARY_PATH=$CI_PROJECT_DIR/install/lib/
-export EGL_PLATFORM=surfaceless
+# The dEQP binary needs to run from the directory it's in
+if [ -z "${1##*"deqp"*}" ]; then
+  PWD=`dirname $1`
+fi
 
-export -p > /crosvm-env.sh
-export GALLIUM_DRIVER="$CROSVM_GALLIUM_DRIVER"
-export GALLIVM_PERF="nopt"
-export LIBGL_ALWAYS_SOFTWARE="true"
+export -p > $DEQP_TEMP_DIR/crosvm-env.sh
 
-CROSVM_KERNEL_ARGS="root=my_root rw rootfstype=virtiofs loglevel=3 init=$CI_PROJECT_DIR/install/crosvm-init.sh ip=192.168.30.2::192.168.30.1:255.255.255.0:crosvm:eth0"
+CROSVM_KERNEL_ARGS="console=null root=my_root rw rootfstype=virtiofs init=$CI_PROJECT_DIR/install/crosvm-init.sh ip=192.168.30.2::192.168.30.1:255.255.255.0:crosvm:eth0 -- $DEQP_TEMP_DIR"
 
-# Temporary results dir because from the guest we cannot write to /
-mkdir -p /results
-mount -t tmpfs tmpfs /results
-
-mkdir -p /piglit/.gitlab-ci/piglit
-mount -t tmpfs tmpfs /piglit/.gitlab-ci/piglit
+echo $@ > $DEQP_TEMP_DIR/crosvm-script.sh
 
 unset DISPLAY
 unset XDG_RUNTIME_DIR
@@ -27,20 +22,25 @@ unset XDG_RUNTIME_DIR
 /usr/sbin/iptables-legacy  -t nat -A POSTROUTING -o eth0 -j MASQUERADE
 echo 1 > /proc/sys/net/ipv4/ip_forward
 
-# We aren't testing LLVMPipe here, so we don't need to validate NIR on the host
-export NIR_VALIDATE=0
+# Send output from guest to host
+touch $DEQP_TEMP_DIR/stderr $DEQP_TEMP_DIR/stdout
+tail -f $DEQP_TEMP_DIR/stderr > /dev/stderr &
+ERR_TAIL_PID=$!
+tail -f $DEQP_TEMP_DIR/stdout > /dev/stdout &
+OUT_TAIL_PID=$!
 
-crosvm run \
+trap "exit \$exit_code" INT TERM
+trap "exit_code=\$?; kill $ERR_TAIL_PID $OUT_TAIL_PID" EXIT
+
+# We aren't testing LLVMPipe here, so we don't need to validate NIR on the host
+NIR_VALIDATE=0 LIBGL_ALWAYS_SOFTWARE="true" GALLIUM_DRIVER="$CROSVM_GALLIUM_DRIVER" stdbuf -oL crosvm run \
   --gpu "$CROSVM_GPU_ARGS" \
   -m 4096 \
-  -c $((FDO_CI_CONCURRENT > 1 ? FDO_CI_CONCURRENT - 1 : 1)) \
+  -c 2 \
   --disable-sandbox \
   --shared-dir /:my_root:type=fs:writeback=true:timeout=60:cache=always \
   --host_ip=192.168.30.1 --netmask=255.255.255.0 --mac "AA:BB:CC:00:00:12" \
   -p "$CROSVM_KERNEL_ARGS" \
-  /lava-files/bzImage
+  /lava-files/bzImage >> $DEQP_TEMP_DIR/stderr > /dev/null
 
-mkdir -p $CI_PROJECT_DIR/results
-mv /results/* $CI_PROJECT_DIR/results/.
-
-test -f $CI_PROJECT_DIR/results/success
+exit `cat $DEQP_TEMP_DIR/exit_code`