ci/bare-metal: Use a new serial buffer tool.

This one uses python threads to move some of our logic from shell
pipelines to python, and opens the door to doing better serial output
tracking in the future (the SerialBuffer.lines() method)

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/6398>

.gitlab-ci/bare-metal/cros-servo.sh

@@ -65,8 +65,15 @@ cp $BM_KERNEL /tftp/vmlinuz
 echo "$BM_CMDLINE" > /tftp/cmdline
 
 # Start watching serials, and power up the device.
-$BM/serial-buffer.py $BM_SERIAL_EC | tee serial-ec-output.txt | sed -u 's|^|SERIAL-EC> |g' &
-$BM/serial-buffer.py $BM_SERIAL | tee serial-output.txt | sed -u 's|^|SERIAL-CPU> |g'  &
+python3 $BM/serial_buffer.py \
+  --dev $BM_SERIAL_EC \
+  --file serial-ec-output.txt \
+  --prefix "SERIAL-EC> " &
+python3 $BM/serial_buffer.py \
+  --dev $BM_SERIAL \
+  --file serial-output.txt \
+  --prefix "SERIAL-CPU> " &
+
 while [ ! -e serial-output.txt ]; do
   sleep 1
 done

.gitlab-ci/bare-metal/fastboot.sh

@@ -92,7 +92,10 @@ fi
 
 # Start watching serial, and power up the device.
 if [ -n "$BM_SERIAL" ]; then
-  $BM/serial-buffer.py $BM_SERIAL | tee artifacts/serial-output.txt &
+  python3 $BM/serial_buffer.py \
+    --dev $BM_SERIAL \
+    --file artifacts/serial-output.txt \
+    --prefix "SERIAL> " &
 else
   PATH=$BM:$PATH $BM_SERIAL_SCRIPT | tee artifacts/serial-output.txt &
 fi

.gitlab-ci/bare-metal/serial-buffer.py

@@ -1,46 +0,0 @@
-#!/usr/bin/python3
-
-# Copyright © 2020 Google LLC
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the "Software"),
-# to deal in the Software without restriction, including without limitation
-# the rights to use, copy, modify, merge, publish, distribute, sublicense,
-# and/or sell copies of the Software, and to permit persons to whom the
-# Software is furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice (including the next
-# paragraph) shall be included in all copies or substantial portions of the
-# Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
-# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-
-# Tiny script to read bytes from serial, and write the output to stdout, with a
-# buffer in between so we don't lose serial output from its buffer.
-#
-# We don't use 'cu' because it requires stdin to be hooked up and I never
-# managed to make that work without getting blocked somewhere.  We don't use
-# 'conserver' because it's non-free.
-
-import sys
-import serial
-import select
-import os
-import posix
-
-dev=sys.argv[1]
-
-ser = serial.Serial(dev, 115200, timeout=10)
-
-while True:
-    bytes = ser.read()
-    sys.stdout.buffer.write(bytes)
-    sys.stdout.flush()
-
-ser.close()

.gitlab-ci/bare-metal/serial_buffer.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+#
+# Copyright © 2020 Google LLC
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+import argparse
+import queue
+import serial
+import threading
+
+class SerialBuffer:
+    def __init__(self, dev, filename, prefix):
+        self.f = open(filename, "wb+")
+        self.dev = dev
+        self.serial = serial.Serial(dev, 115200, timeout=10)
+        self.byte_queue = queue.Queue()
+        self.line_queue = queue.Queue()
+        self.prefix = prefix
+        self.sentinel = object()
+
+        self.read_thread = threading.Thread(target=self.serial_read_thread_loop, daemon=True)
+        self.read_thread.start()
+        self.lines_thread = threading.Thread(target=self.serial_lines_thread_loop, daemon=True)
+        self.lines_thread.start()
+
+    # Thread that just reads the bytes from the serial device to try to keep from
+    # buffer overflowing it.
+    def serial_read_thread_loop(self):
+        greet = "Serial thread reading from %s\n" % self.dev
+        self.byte_queue.put(greet.encode())
+
+        while True:
+            try:
+                self.byte_queue.put(self.serial.read())
+            except Exception as err:
+                print(self.prefix + str(err))
+                self.byte_queue.put(self.sentinel)
+                break
+
+    # Thread that processes the stream of bytes to 1) log to stdout, 2) log to
+    # file, 3) add to the queue of lines to be read by program logic
+
+    def serial_lines_thread_loop(self):
+        line = bytearray()
+        while True:
+            bytes = self.byte_queue.get(block=True)
+
+            if bytes == self.sentinel:
+                self.read_thread.join()
+                self.line_queue.put(self.sentinel)
+                break;
+
+            # Write our data to the output file
+            self.f.write(bytes)
+            self.f.flush()
+
+            for b in bytes:
+                line.append(b)
+                if b == b'\n'[0]:
+                    line = line.decode(errors="replace")
+                    print(self.prefix + line, flush=True, end='')
+                    self.line_queue.put(line)
+                    line = bytearray()
+
+    def get_line(self):
+        line = self.line_queue.get()
+        if line == self.sentinel:
+            self.lines_thread.join()
+        return line
+
+    def lines(self):
+        return iter(self.get_line, self.sentinel)
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('--dev', type=str, help='Serial device', required=True)
+    parser.add_argument('--file', type=str, help='Filename to output our serial data to')
+    parser.add_argument('--prefix', type=str, help='Prefix for logging serial to stdout', nargs='?')
+
+    args = parser.parse_args()
+
+    ser = SerialBuffer(args.dev, args.file, args.prefix or "")
+    for line in ser.lines():
+        # We're just using this as a logger, so eat the produced lines and drop
+        # them
+        pass
+
+if __name__ == '__main__':
+    main()