vulkan/wsi: implement missing wsi_register_device_event

These changes implement vkRegisterDeviceEventEXT and detection of
monitor hotplug. Wsi launches a thread that listens to udev events and
signals the appropriate device fences when hotplug hapens.

v2: use wsi fences instead of syncobj api (Jason Ekstrand)
v3: refactor + cleanups, create thread on demand (Samuel Pitoiset)
v4: bring back syncobj support from initial version for radv
v5: make libudev dependency optional, check for poll errors (Simon Ser)
v6: change matching mechanism to use udev device node instead of path
v7: remove the matching mechanism
v8: fix a race with thread creation + use single mutex + other cleanups
    (Jason Ekstrand)

Fixes:
   dEQP-VK.wsi.display_control.register_device_event

Signed-off-by: Tapani Pälli <tapani.palli@intel.com>
Reviewed-by: Jason Ekstrand <jason@jlekstrand.net>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/12305>

meson.build

@@ -1620,6 +1620,11 @@ if dep_libdrm.found()
   endif
 endif
 
+dep_libudev = dependency('libudev', required : false)
+if dep_libudev.found()
+  pre_args += '-DHAVE_LIBUDEV'
+endif
+
 llvm_modules = ['bitwriter', 'engine', 'mcdisassembler', 'mcjit', 'core', 'executionengine', 'scalaropts', 'transformutils', 'instcombine']
 llvm_optional_modules = ['coroutines']
 if with_amd_vk or with_gallium_radeonsi or with_gallium_r600

src/vulkan/wsi/meson.build

@@ -60,7 +60,7 @@ libvulkan_wsi = static_library(
   [files_vulkan_wsi, wsi_entrypoints],
   include_directories : [inc_include, inc_src],
   dependencies : [
-    vulkan_wsi_deps, dep_libdrm, idep_vulkan_util_headers,
+    vulkan_wsi_deps, dep_libdrm, dep_libudev, idep_vulkan_util_headers,
     idep_vulkan_runtime_headers, idep_xmlconfig,
   ],
   c_args : [vulkan_wsi_args],

src/vulkan/wsi/wsi_common.c

@@ -75,6 +75,8 @@ wsi_device_init(struct wsi_device *wsi,
    GetPhysicalDeviceMemoryProperties(pdevice, &wsi->memory_props);
    GetPhysicalDeviceQueueFamilyProperties(pdevice, &wsi->queue_family_count, NULL);
 
+   list_inithead(&wsi->hotplug_fences);
+
 #define WSI_GET_CB(func) \
    wsi->func = (PFN_vk##func)proc_addr(pdevice, "vk" #func)
    WSI_GET_CB(AllocateMemory);

src/vulkan/wsi/wsi_common.h

@@ -37,6 +37,8 @@ extern const struct vk_physical_device_entrypoint_table wsi_physical_device_entr
 extern const struct vk_device_entrypoint_table wsi_device_entrypoints;
 #endif
 
+#include <util/list.h>
+
 /* This is guaranteed to not collide with anything because it's in the
  * VK_KHR_swapchain namespace but not actually used by the extension.
  */
@@ -114,6 +116,9 @@ struct wsi_device {
     * available. Not all window systems might support this. */
    bool enable_adaptive_sync;
 
+   /* List of fences to signal when hotplug event happens. */
+   struct list_head hotplug_fences;
+
    struct {
       /* Override the minimum number of images on the swapchain.
        * 0 = no override */

src/vulkan/wsi/wsi_common_display.c

@@ -23,6 +23,7 @@
 #include "util/macros.h"
 #include <stdlib.h>
 #include <stdio.h>
+#include <sys/stat.h>
 #include <unistd.h>
 #include <errno.h>
 #include <string.h>
@@ -32,6 +33,9 @@
 #include <math.h>
 #include <xf86drm.h>
 #include <xf86drmMode.h>
+#ifdef HAVE_LIBUDEV
+#include <libudev.h>
+#endif
 #include "drm-uapi/drm_fourcc.h"
 #ifdef VK_USE_PLATFORM_XLIB_XRANDR_EXT
 #include <xcb/randr.h>
@@ -103,6 +107,9 @@ struct wsi_display {
    pthread_cond_t               wait_cond;
    pthread_t                    wait_thread;
 
+   pthread_cond_t               hotplug_cond;
+   pthread_t                    hotplug_thread;
+
    struct list_head             connectors; /* list of all discovered connectors */
 };
 
@@ -142,10 +149,12 @@ struct wsi_display_swapchain {
 
 struct wsi_display_fence {
    struct wsi_fence             base;
+   struct list_head             link;
    bool                         event_received;
    bool                         destroyed;
    uint32_t                     syncobj; /* syncobj to signal on event */
    uint64_t                     sequence;
+   bool                         device_event; /* fence is used for device events */
 };
 
 static uint64_t fence_sequence;
@@ -1254,6 +1263,21 @@ wsi_display_stop_wait_thread(struct wsi_display *wsi)
    pthread_mutex_unlock(&wsi->wait_mutex);
 }
 
+static int
+cond_timedwait_ns(pthread_cond_t *cond,
+                  pthread_mutex_t *mutex,
+                  uint64_t timeout_ns)
+{
+   struct timespec abs_timeout = {
+      .tv_sec = timeout_ns / 1000000000ULL,
+      .tv_nsec = timeout_ns % 1000000000ULL,
+   };
+
+   int ret = pthread_cond_timedwait(cond, mutex, &abs_timeout);
+   wsi_display_debug("%9ld done waiting for event %d\n", pthread_self(), ret);
+   return ret;
+}
+
 /*
  * Wait for at least one event from the kernel to be processed.
  * Call with wait_mutex held
@@ -1262,23 +1286,22 @@ static int
 wsi_display_wait_for_event(struct wsi_display *wsi,
                            uint64_t timeout_ns)
 {
-   int ret;
+   int ret = wsi_display_start_wait_thread(wsi);
-
-   ret = wsi_display_start_wait_thread(wsi);
 
    if (ret)
       return ret;
 
-   struct timespec abs_timeout = {
+   return cond_timedwait_ns(&wsi->wait_cond, &wsi->wait_mutex, timeout_ns);
-      .tv_sec = timeout_ns / 1000000000ULL,
+}
-      .tv_nsec = timeout_ns % 1000000000ULL,
-   };
 
-   ret = pthread_cond_timedwait(&wsi->wait_cond, &wsi->wait_mutex,
+/* Wait for device event to be processed.
-                                &abs_timeout);
+ * Call with wait_mutex held
-
+ */
-   wsi_display_debug("%9ld done waiting for event %d\n", pthread_self(), ret);
+static int
-   return ret;
+wsi_device_wait_for_event(struct wsi_display *wsi,
+                          uint64_t timeout_ns)
+{
+   return cond_timedwait_ns(&wsi->hotplug_cond, &wsi->wait_mutex, timeout_ns);
 }
 
 static VkResult
@@ -1515,7 +1538,10 @@ wsi_display_fence_wait(struct wsi_fence *fence_wsi, uint64_t timeout)
          break;
       }
 
-      ret = wsi_display_wait_for_event(wsi, timeout);
+      if (fence->device_event)
+         ret = wsi_device_wait_for_event(wsi, timeout);
+      else
+         ret = wsi_display_wait_for_event(wsi, timeout);
 
       if (ret && ret != ETIMEDOUT) {
          wsi_display_debug("%9lu fence %lu error\n",
@@ -1558,6 +1584,18 @@ wsi_display_fence_destroy(struct wsi_fence *fence_wsi)
 {
    struct wsi_display_fence *fence = (struct wsi_display_fence *) fence_wsi;
 
+   const struct wsi_device *wsi_device = fence->base.wsi_device;
+   struct wsi_display *wsi =
+      (struct wsi_display *) wsi_device->wsi[VK_ICD_WSI_PLATFORM_DISPLAY];
+
+   /* Destroy hotplug fence list. */
+   if (fence->device_event) {
+      mtx_lock(&wsi->wait_mutex);
+      list_del(&fence->link);
+      mtx_unlock(&wsi->wait_mutex);
+      fence->event_received = true;
+   }
+
    assert(!fence->destroyed);
    fence->destroyed = true;
    wsi_display_fence_check_free(fence);
@@ -1912,6 +1950,88 @@ local_drmIsMaster(int fd)
    return drmAuthMagic(fd, 0) != -EACCES;
 }
 
+#ifdef HAVE_LIBUDEV
+static void *
+udev_event_listener_thread(void *data)
+{
+   struct wsi_device *wsi_device = data;
+   struct wsi_display *wsi =
+      (struct wsi_display *) wsi_device->wsi[VK_ICD_WSI_PLATFORM_DISPLAY];
+
+   struct udev *u = udev_new();
+   if (!u)
+      goto fail;
+
+   struct udev_monitor *mon =
+      udev_monitor_new_from_netlink(u, "udev");
+   if (!mon)
+      goto fail_udev;
+
+   int ret =
+      udev_monitor_filter_add_match_subsystem_devtype(mon, "drm", "drm_minor");
+   if (ret < 0)
+      goto fail_udev_monitor;
+
+   ret = udev_monitor_enable_receiving(mon);
+   if (ret < 0)
+      goto fail_udev_monitor;
+
+   int udev_fd = udev_monitor_get_fd(mon);
+
+   pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
+
+   for (;;) {
+      nfds_t nfds = 1;
+      struct pollfd fds[1] =  {
+         {
+            .fd = udev_fd,
+            .events = POLLIN,
+         },
+      };
+
+      int ret = poll(fds, nfds, -1);
+      if (ret > 0) {
+         if (fds[0].revents & POLLIN) {
+            struct udev_device *dev = udev_monitor_receive_device(mon);
+
+            /* Ignore event if it is not a hotplug event */
+            if (!atoi(udev_device_get_property_value(dev, "HOTPLUG")))
+               continue;
+
+            /* Note, this supports both drmSyncobjWait for fence->syncobj
+             * and wsi_display_wait_for_event.
+             */
+            mtx_lock(&wsi->wait_mutex);
+            pthread_cond_broadcast(&wsi->hotplug_cond);
+            list_for_each_entry(struct wsi_display_fence, fence,
+                                &wsi_device->hotplug_fences, link) {
+               if (fence->syncobj)
+                  drmSyncobjSignal(wsi->syncobj_fd, &fence->syncobj, 1);
+               fence->event_received = true;
+            }
+            mtx_unlock(&wsi->wait_mutex);
+            udev_device_unref(dev);
+         }
+      } else if (ret < 0) {
+         goto fail;
+      }
+   }
+
+   udev_monitor_unref(mon);
+   udev_unref(u);
+
+   return 0;
+
+fail_udev_monitor:
+   udev_monitor_unref(mon);
+fail_udev:
+   udev_unref(u);
+fail:
+   wsi_display_debug("critical hotplug thread error\n");
+   return 0;
+}
+#endif
+
 VkResult
 wsi_display_init_wsi(struct wsi_device *wsi_device,
                      const VkAllocationCallbacks *alloc,
@@ -1947,6 +2067,11 @@ wsi_display_init_wsi(struct wsi_device *wsi_device,
       goto fail_cond;
    }
 
+   if (!wsi_init_pthread_cond_monotonic(&wsi->hotplug_cond)) {
+      result = VK_ERROR_OUT_OF_HOST_MEMORY;
+      goto fail_hotplug_cond;
+   }
+
    wsi->base.get_support = wsi_display_surface_get_support;
    wsi->base.get_capabilities2 = wsi_display_surface_get_capabilities2;
    wsi->base.get_formats = wsi_display_surface_get_formats;
@@ -1959,6 +2084,8 @@ wsi_display_init_wsi(struct wsi_device *wsi_device,
 
    return VK_SUCCESS;
 
+fail_hotplug_cond:
+   pthread_cond_destroy(&wsi->wait_cond);
 fail_cond:
    pthread_mutex_destroy(&wsi->wait_mutex);
 fail_mutex:
@@ -1983,8 +2110,15 @@ wsi_display_finish_wsi(struct wsi_device *wsi_device,
       }
 
       wsi_display_stop_wait_thread(wsi);
+
+      if (wsi->hotplug_thread) {
+         pthread_cancel(wsi->hotplug_thread);
+         pthread_join(wsi->hotplug_thread, NULL);
+      }
+
       pthread_mutex_destroy(&wsi->wait_mutex);
       pthread_cond_destroy(&wsi->wait_cond);
+      pthread_cond_destroy(&wsi->hotplug_cond);
 
       vk_free(alloc, wsi);
    }
@@ -2541,7 +2675,41 @@ wsi_register_device_event(VkDevice device,
                           struct wsi_fence **fence_p,
                           int sync_fd)
 {
-   return VK_ERROR_FEATURE_NOT_PRESENT;
+   struct wsi_display *wsi =
+      (struct wsi_display *) wsi_device->wsi[VK_ICD_WSI_PLATFORM_DISPLAY];
+
+#ifdef HAVE_LIBUDEV
+   /* Start listening for output change notifications. */
+   mtx_lock(&wsi->wait_mutex);
+   if (!wsi->hotplug_thread) {
+      if (pthread_create(&wsi->hotplug_thread, NULL, udev_event_listener_thread,
+                         wsi_device))
+         return VK_ERROR_OUT_OF_HOST_MEMORY;
+   }
+   mtx_unlock(&wsi->wait_mutex);
+#endif
+
+   struct wsi_display_fence *fence;
+   assert(device_event_info->deviceEvent ==
+          VK_DEVICE_EVENT_TYPE_DISPLAY_HOTPLUG_EXT);
+
+   fence = wsi_display_fence_alloc(device, wsi_device, 0, allocator, sync_fd);
+
+   if (!fence)
+      return VK_ERROR_OUT_OF_HOST_MEMORY;
+
+   if (fence_p)
+      *fence_p = &fence->base;
+   else
+      fence->base.destroy(&fence->base);
+
+   fence->device_event = true;
+
+   mtx_lock(&wsi->wait_mutex);
+   list_addtail(&fence->link, &wsi_device->hotplug_fences);
+   mtx_unlock(&wsi->wait_mutex);
+
+   return VK_SUCCESS;
 }
 
 VKAPI_ATTR VkResult VKAPI_CALL