ci: bare-metal: poe: Create strutured logs

Use the CustomLogger class and CLI tool to create strutured logs for poe scripts which are used by broadcom and nouveau jobs. Renamed stage lint to code-validation and added python-test job which runs the tests for structured and customer logger to ci. Signed-off-by: Vignesh Raman <vignesh.raman@collabora.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/25179>
2023-12-15 17:30:04 +05:30
parent 2a530cb188
commit 2487e18d4e
10 changed files with 83 additions and 9 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -85,6 +85,8 @@ variables:
  # Re-enable farm with `git mv .ci-farms{-disabled,}/$farm_name`
  # NEVER MIX FARM MAINTENANCE WITH ANY OTHER CHANGE IN THE SAME MERGE REQUEST!
  ARTIFACTS_BASE_URL: https://${CI_PROJECT_ROOT_NAMESPACE}.${CI_PAGES_DOMAIN}/-/${CI_PROJECT_NAME}/-/jobs/${CI_JOB_ID}/artifacts
+  # Python scripts for structured logger
+  PYTHONPATH: "$PYTHONPATH:$CI_PROJECT_DIR/install"

 default:
  before_script:
@@ -127,7 +129,7 @@ stages:
  - git-archive
  - build-x86_64
  - build-misc
-  - lint
+  - code-validation
  - amd
  - intel
  - nouveau
--- a/.gitlab-ci/bare-metal/poe-powered.sh
+++ b/.gitlab-ci/bare-metal/poe-powered.sh
@@ -12,6 +12,7 @@
 # We're run from the root of the repo, make a helper var for our paths
 BM=$CI_PROJECT_DIR/install/bare-metal
 CI_COMMON=$CI_PROJECT_DIR/install/common
+CI_INSTALL=$CI_PROJECT_DIR/install

 # Runner config checks
 if [ -z "$BM_SERIAL" ]; then
@@ -181,8 +182,19 @@ if [ -n "$BM_BOOTCONFIG" ]; then
 fi

 set +e
+STRUCTURED_LOG_FILE=job_detail.json
+python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --update dut_job_type "${DEVICE_TYPE}"
+python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --update farm "${FARM}"
 ATTEMPTS=3
+first_attempt=True
 while [ $((ATTEMPTS--)) -gt 0 ]; do
+  python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --create-dut-job dut_name "${CI_RUNNER_DESCRIPTION}"
+  # Update subtime time to CI_JOB_STARTED_AT only for the first run
+  if [ "$first_attempt" = "True" ]; then
+    python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --update-dut-time submit "${CI_JOB_STARTED_AT}"
+  else
+    python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --update-dut-time submit
+  fi
  python3 $BM/poe_run.py \
          --dev="$BM_SERIAL" \
          --powerup="$BM_POWERUP" \
@@ -192,10 +204,14 @@ while [ $((ATTEMPTS--)) -gt 0 ]; do

  if [ $ret -eq 2 ]; then
    echo "Did not detect boot sequence, retrying..."
+    python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --close-dut-job
+    first_attempt=False
  else
    ATTEMPTS=0
  fi
 done
+python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --close-dut-job
+python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --close
 set -e

 date +'%F %T'
@@ -203,6 +219,10 @@ date +'%F %T'
 # Bring artifacts back from the NFS dir to the build dir where gitlab-runner
 # will look for them.
 cp -Rp /nfs/results/. results/
+if [ -f "${STRUCTURED_LOG_FILE}" ]; then
+  cp -p ${STRUCTURED_LOG_FILE} results/
+  echo "Structured log file is available at ${ARTIFACTS_BASE_URL}/results/${STRUCTURED_LOG_FILE}"
+fi

 date +'%F %T'

--- a/.gitlab-ci/bare-metal/poe_run.py
+++ b/.gitlab-ci/bare-metal/poe_run.py
@@ -24,23 +24,26 @@
 import argparse
 import os
 import re
-from serial_buffer import SerialBuffer
 import sys
 import threading

+from custom_logger import CustomLogger
+from serial_buffer import SerialBuffer

 class PoERun:
-    def __init__(self, args, test_timeout):
+    def __init__(self, args, test_timeout, logger):
        self.powerup = args.powerup
        self.powerdown = args.powerdown
        self.ser = SerialBuffer(
            args.dev, "results/serial-output.txt", "")
        self.test_timeout = test_timeout
+        self.logger = logger

    def print_error(self, message):
        RED = '\033[0;31m'
        NO_COLOR = '\033[0m'
        print(RED + message + NO_COLOR)
+        self.logger.update_status_fail(message)

    def logged_system(self, cmd):
        print("Running '{}'".format(cmd))
@@ -48,9 +51,11 @@ class PoERun:

    def run(self):
        if self.logged_system(self.powerup) != 0:
+            self.logger.update_status_fail("powerup failed")
            return 1

        boot_detected = False
+        self.logger.create_job_phase("boot")
        for line in self.ser.lines(timeout=5 * 60, phase="bootloader"):
            if re.search("Booting Linux", line):
                boot_detected = True
@@ -61,8 +66,10 @@ class PoERun:
                "Something wrong; couldn't detect the boot start up sequence")
            return 1

+        self.logger.create_job_phase("test")
        for line in self.ser.lines(timeout=self.test_timeout, phase="test"):
            if re.search("---. end Kernel panic", line):
+                self.logger.update_status_fail("kernel panic")
                return 1

            # Binning memory problems
@@ -82,8 +89,10 @@ class PoERun:
            result = re.search("hwci: mesa: (\S*)", line)
            if result:
                if result.group(1) == "pass":
+                    self.logger.update_dut_job("status", "pass")
                    return 0
                else:
+                    self.logger.update_status_fail("test fail")
                    return 1

        self.print_error(
@@ -103,10 +112,13 @@ def main():
        '--test-timeout', type=int, help='Test phase timeout (minutes)', required=True)
    args = parser.parse_args()

-    poe = PoERun(args, args.test_timeout * 60)
+    logger = CustomLogger("job_detail.json")
+    logger.update_dut_time("start", None)
+    poe = PoERun(args, args.test_timeout * 60, logger)
    retval = poe.run()

    poe.logged_system(args.powerdown)
+    logger.update_dut_time("end", None)

    sys.exit(retval)

--- a/.gitlab-ci/container/debian/arm_test.sh
+++ b/.gitlab-ci/container/debian/arm_test.sh
@@ -17,6 +17,8 @@ DEPS=(
    openssh-server
    procps
    python3-distutils
+    python3-filelock
+    python3-fire
    python3-minimal
    python3-serial
    rsync
--- a/.gitlab-ci/image-tags.yml
+++ b/.gitlab-ci/image-tags.yml
@@ -18,14 +18,14 @@ variables:
   DEBIAN_X86_64_TEST_IMAGE_VK_PATH: "debian/x86_64_test-vk"
   DEBIAN_X86_64_TEST_ANDROID_IMAGE_PATH: "debian/x86_64_test-android"

-   DEBIAN_X86_64_TEST_ANDROID_TAG: "2023-11-18-deqp-runner"
-   DEBIAN_X86_64_TEST_GL_TAG: "2023-11-06-deqp-fixes"
-   DEBIAN_X86_64_TEST_VK_TAG: "2023-12-08-vkd3d"
+   DEBIAN_X86_64_TEST_ANDROID_TAG: "2023-12-14-structured-logs"
+   DEBIAN_X86_64_TEST_GL_TAG: "2023-12-14-structured-logs"
+   DEBIAN_X86_64_TEST_VK_TAG: "2023-12-14-structured-logs"

   ALPINE_X86_64_BUILD_TAG: "2023-10-30-ci-improv"
   ALPINE_X86_64_LAVA_SSH_TAG: "2023-06-26-first-version"
   FEDORA_X86_64_BUILD_TAG: "2023-12-11-meson-bump"
-   KERNEL_ROOTFS_TAG: "2023-12-08-firmware"
+   KERNEL_ROOTFS_TAG: "2023-12-14-structured-logs"
   KERNEL_TAG: "v6.6.4-for-mesa-ci-e4f4c500f7fb"
   KERNEL_REPO: "gfx-ci/linux"

--- a/.gitlab-ci/test/gitlab-ci.yml
+++ b/.gitlab-ci/test/gitlab-ci.yml
@@ -22,7 +22,7 @@
 .formatting-check:
  # Cancel job if a newer commit is pushed to the same branch
  interruptible: true
-  stage: lint
+  stage: code-validation
  extends:
    - .use-debian/x86_64_build
  variables:
@@ -57,6 +57,29 @@ clang-format:
    - clang-format-${LLVM_VERSION} --version
    - ninja -C build clang-format

+.test-check:
+  # Cancel job if a newer commit is pushed to the same branch
+  interruptible: true
+  stage: code-validation
+  extends:
+    - .use-debian/x86_64_build
+  variables:
+    GIT_STRATEGY: fetch
+  timeout: 10m
+
+python-test:
+  extends:
+    - .test-check
+  script:
+    - cd bin/ci
+    - pip install --break-system-packages -r test/requirements.txt
+    - PYTHONPATH=. pytest -v
+  rules:
+    - !reference [.disable-farm-mr-rules, rules]
+    - !reference [.never-post-merge-rules, rules]
+    - changes:
+        - bin/ci/**/*
+
 .test-gl:
  extends:
    - .test
--- a/bin/ci/test/requirements.txt
+++ b/bin/ci/test/requirements.txt
@@ -0,0 +1,5 @@
+filelock==3.12.4
+fire==0.5.0
+mock==5.1.0
+polars==0.19.3
+pytest==7.4.2
--- a/src/broadcom/ci/gitlab-ci-inc.yml
+++ b/src/broadcom/ci/gitlab-ci-inc.yml
@@ -107,6 +107,7 @@
  variables:
    HWCI_TEST_SCRIPT: "/install/deqp-runner.sh"
    FLAKES_CHANNEL: "#videocore-ci"
+    FARM: igalia
  timeout: 20m

 .broadcom-test:arm64:
--- a/src/broadcom/ci/gitlab-ci.yml
+++ b/src/broadcom/ci/gitlab-ci.yml
@@ -14,6 +14,7 @@ vc4-rpi3-gl:arm32:
    HWCI_KERNEL_MODULES: vc4
    DEQP_SUITE: broadcom-rpi3
    HWCI_START_WESTON: 1
+    DEVICE_TYPE: rpi3

 vc4-rpi3-gl-piglit-full:arm32:
  extends:
@@ -24,6 +25,7 @@ vc4-rpi3-gl-piglit-full:arm32:
    - igalia-fullrun
  variables:
    DEQP_SUITE: broadcom-rpi3-piglit-full
+    DEVICE_TYPE: rpi3


 v3d-rpi4-gl:arm64:
@@ -39,6 +41,7 @@ v3d-rpi4-gl:arm64:
    HWCI_START_WESTON: 1
    DEQP_SUITE: broadcom-rpi4
    DEQP_FRACTION: 2
+    DEVICE_TYPE: rpi4

 v3d-rpi4-gl-full:arm64:
  extends:
@@ -52,6 +55,7 @@ v3d-rpi4-gl-full:arm64:
  variables:
    TEST_PHASE_TIMEOUT: 40
    DEQP_FRACTION: 1
+    DEVICE_TYPE: rpi4


 v3d-rpi4-traces:arm64:
@@ -66,6 +70,7 @@ v3d-rpi4-traces:arm64:
    PIGLIT_TRACES_FILE: traces-broadcom.yml
    PIGLIT_REPLAY_DEVICE_NAME: "broadcom-rpi4"
    PIGLIT_RESULTS: "broadcom-rpi4-replay"
+    DEVICE_TYPE: rpi4
  tags:
    - igalia-rpi4

@@ -84,6 +89,7 @@ v3dv-rpi4-vk:arm64:
    GPU_VERSION: broadcom-rpi4
    HWCI_KERNEL_MODULES: v3d,vc4
    VK_DRIVER: broadcom
+    DEVICE_TYPE: rpi4
  tags:
    - igalia-rpi4

@@ -99,3 +105,4 @@ v3dv-rpi4-vk-full:arm64:
  variables:
    TEST_PHASE_TIMEOUT: 80
    DEQP_FRACTION: 1
+    DEVICE_TYPE: rpi4
--- a/src/gallium/drivers/nouveau/ci/gitlab-ci.yml
+++ b/src/gallium/drivers/nouveau/ci/gitlab-ci.yml
@@ -10,3 +10,5 @@ gk20a-gles:
  variables:
    HWCI_TEST_SCRIPT: "/install/deqp-runner.sh"
    DEQP_SUITE: nouveau-gk20a
+    FARM : anholt
+    DEVICE_TYPE: anholt-jetson