v3dv: Implement VK_EXT_pipeline_creation_cache_control

Reviewed-by: Alejandro Piñeiro <apinheiro@igalia.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/12381>

docs/features.txt

@@ -534,7 +534,7 @@ Khronos extensions that are not part of any Vulkan version:
   VK_EXT_multi_draw                                     DONE (anv, lvp, radv)
   VK_EXT_pci_bus_info                                   DONE (anv, radv)
   VK_EXT_physical_device_drm                            DONE (anv, radv, v3dv)
-  VK_EXT_pipeline_creation_cache_control                DONE (anv, radv)
+  VK_EXT_pipeline_creation_cache_control                DONE (anv, radv, v3dv)
   VK_EXT_pipeline_creation_feedback                     DONE (anv, radv)
   VK_EXT_post_depth_coverage                            DONE (anv/gfx10+, lvp, radv)
   VK_EXT_private_data                                   DONE (anv, lvp, radv, tu, v3dv)

src/broadcom/vulkan/v3dv_device.c

@@ -144,6 +144,7 @@ get_device_extensions(const struct v3dv_physical_device *device,
       .EXT_external_memory_dma_buf         = true,
       .EXT_index_type_uint8                = true,
       .EXT_physical_device_drm             = true,
+      .EXT_pipeline_creation_cache_control = true,
       .EXT_private_data                    = true,
    };
 }
@@ -1108,6 +1109,12 @@ v3dv_GetPhysicalDeviceFeatures2(VkPhysicalDevice physicalDevice,
           break;
       }
 
+      case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_CREATION_CACHE_CONTROL_FEATURES_EXT: {
+         VkPhysicalDevicePipelineCreationCacheControlFeaturesEXT *features = (void *) ext;
+         features->pipelineCreationCacheControl = true;
+         break;
+      }
+
       /* Vulkan 1.1 */
       case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES: {
          VkPhysicalDeviceVulkan11Features *features =
@@ -1758,7 +1765,7 @@ v3dv_CreateDevice(VkPhysicalDevice physicalDevice,
 #endif
    init_device_meta(device);
    v3dv_bo_cache_init(device);
-   v3dv_pipeline_cache_init(&device->default_pipeline_cache, device,
+   v3dv_pipeline_cache_init(&device->default_pipeline_cache, device, 0,
                             device->instance->default_pipeline_cache_enabled);
    device->default_attribute_float =
       v3dv_pipeline_create_default_attribute_values(device, NULL);

src/broadcom/vulkan/v3dv_pipeline.c

@@ -2423,6 +2423,9 @@ pipeline_compile_graphics(struct v3dv_pipeline *pipeline,
       goto success;
    }
 
+   if (pCreateInfo->flags & VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT_EXT)
+      return VK_PIPELINE_COMPILE_REQUIRED_EXT;
+
    /* Otherwise we try to get the NIR shaders (either from the original SPIR-V
     * shader or the pipeline cache) and compile.
     */
@@ -2930,6 +2933,8 @@ graphics_pipeline_create(VkDevice _device,
 
    if (result != VK_SUCCESS) {
       v3dv_destroy_pipeline(pipeline, device, pAllocator);
+      if (result == VK_PIPELINE_COMPILE_REQUIRED_EXT)
+         *pPipeline = VK_NULL_HANDLE;
       return result;
    }
 
@@ -2952,7 +2957,8 @@ v3dv_CreateGraphicsPipelines(VkDevice _device,
    if (unlikely(V3D_DEBUG & V3D_DEBUG_SHADERS))
       mtx_lock(&device->pdevice->mutex);
 
-   for (uint32_t i = 0; i < count; i++) {
+   uint32_t i = 0;
+   for (; i < count; i++) {
       VkResult local_result;
 
       local_result = graphics_pipeline_create(_device,
@@ -2964,9 +2970,16 @@ v3dv_CreateGraphicsPipelines(VkDevice _device,
       if (local_result != VK_SUCCESS) {
          result = local_result;
          pPipelines[i] = VK_NULL_HANDLE;
+
+         if (pCreateInfos[i].flags &
+             VK_PIPELINE_CREATE_EARLY_RETURN_ON_FAILURE_BIT_EXT)
+            break;
       }
    }
 
+   for (; i < count; i++)
+      pPipelines[i] = VK_NULL_HANDLE;
+
    if (unlikely(V3D_DEBUG & V3D_DEBUG_SHADERS))
       mtx_unlock(&device->pdevice->mutex);
 
@@ -3045,6 +3058,9 @@ pipeline_compile_compute(struct v3dv_pipeline *pipeline,
       goto success;
    }
 
+   if (info->flags & VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT_EXT)
+      return VK_PIPELINE_COMPILE_REQUIRED_EXT;
+
    pipeline->shared_data = v3dv_pipeline_shared_data_new_empty(pipeline_sha1,
                                                                pipeline,
                                                                false);
@@ -3128,6 +3144,8 @@ compute_pipeline_create(VkDevice _device,
                                   pCreateInfo, pAllocator);
    if (result != VK_SUCCESS) {
       v3dv_destroy_pipeline(pipeline, device, pAllocator);
+      if (result == VK_PIPELINE_COMPILE_REQUIRED_EXT)
+         *pPipeline = VK_NULL_HANDLE;
       return result;
    }
 
@@ -3150,7 +3168,8 @@ v3dv_CreateComputePipelines(VkDevice _device,
    if (unlikely(V3D_DEBUG & V3D_DEBUG_SHADERS))
       mtx_lock(&device->pdevice->mutex);
 
-   for (uint32_t i = 0; i < createInfoCount; i++) {
+   uint32_t i = 0;
+   for (; i < createInfoCount; i++) {
       VkResult local_result;
       local_result = compute_pipeline_create(_device,
                                               pipelineCache,
@@ -3161,9 +3180,16 @@ v3dv_CreateComputePipelines(VkDevice _device,
       if (local_result != VK_SUCCESS) {
          result = local_result;
          pPipelines[i] = VK_NULL_HANDLE;
+
+         if (pCreateInfos[i].flags &
+             VK_PIPELINE_CREATE_EARLY_RETURN_ON_FAILURE_BIT_EXT)
+            break;
       }
    }
 
+   for (; i < createInfoCount; i++)
+      pPipelines[i] = VK_NULL_HANDLE;
+
    if (unlikely(V3D_DEBUG & V3D_DEBUG_SHADERS))
       mtx_unlock(&device->pdevice->mutex);
 

src/broadcom/vulkan/v3dv_pipeline_cache.c

@@ -63,6 +63,20 @@ cache_dump_stats(struct v3dv_pipeline_cache *cache)
    fprintf(stderr, "  cache hit  count:   %d\n", cache->stats.hit);
 }
 
+static void
+pipeline_cache_lock(struct v3dv_pipeline_cache *cache)
+{
+   if (!cache->externally_synchronized)
+      pthread_mutex_lock(&cache->mutex);
+}
+
+static void
+pipeline_cache_unlock(struct v3dv_pipeline_cache *cache)
+{
+   if (!cache->externally_synchronized)
+      pthread_mutex_unlock(&cache->mutex);
+}
+
 void
 v3dv_pipeline_cache_upload_nir(struct v3dv_pipeline *pipeline,
                                struct v3dv_pipeline_cache *cache,
@@ -75,10 +89,10 @@ v3dv_pipeline_cache_upload_nir(struct v3dv_pipeline *pipeline,
    if (cache->nir_stats.count > V3DV_MAX_PIPELINE_CACHE_ENTRIES)
       return;
 
-   pthread_mutex_lock(&cache->mutex);
+   pipeline_cache_lock(cache);
    struct hash_entry *entry =
       _mesa_hash_table_search(cache->nir_cache, sha1_key);
-   pthread_mutex_unlock(&cache->mutex);
+   pipeline_cache_unlock(cache);
    if (entry)
       return;
 
@@ -91,7 +105,7 @@ v3dv_pipeline_cache_upload_nir(struct v3dv_pipeline *pipeline,
       return;
    }
 
-   pthread_mutex_lock(&cache->mutex);
+   pipeline_cache_lock(cache);
    /* Because ralloc isn't thread-safe, we have to do all this inside the
     * lock.  We could unlock for the big memcpy but it's probably not worth
     * the hassle.
@@ -99,7 +113,7 @@ v3dv_pipeline_cache_upload_nir(struct v3dv_pipeline *pipeline,
    entry = _mesa_hash_table_search(cache->nir_cache, sha1_key);
    if (entry) {
       blob_finish(&blob);
-      pthread_mutex_unlock(&cache->mutex);
+      pipeline_cache_unlock(cache);
       return;
    }
 
@@ -122,7 +136,7 @@ v3dv_pipeline_cache_upload_nir(struct v3dv_pipeline *pipeline,
 
    _mesa_hash_table_insert(cache->nir_cache, snir->sha1_key, snir);
 
-   pthread_mutex_unlock(&cache->mutex);
+   pipeline_cache_unlock(cache);
 }
 
 nir_shader*
@@ -143,12 +157,12 @@ v3dv_pipeline_cache_search_for_nir(struct v3dv_pipeline *pipeline,
 
    const struct serialized_nir *snir = NULL;
 
-   pthread_mutex_lock(&cache->mutex);
+   pipeline_cache_lock(cache);
    struct hash_entry *entry =
       _mesa_hash_table_search(cache->nir_cache, sha1_key);
    if (entry)
       snir = entry->data;
-   pthread_mutex_unlock(&cache->mutex);
+   pipeline_cache_unlock(cache);
 
    if (snir) {
       struct blob_reader blob;
@@ -185,6 +199,7 @@ v3dv_pipeline_cache_search_for_nir(struct v3dv_pipeline *pipeline,
 void
 v3dv_pipeline_cache_init(struct v3dv_pipeline_cache *cache,
                          struct v3dv_device *device,
+                         VkPipelineCacheCreateFlags flags,
                          bool cache_enabled)
 {
    cache->device = device;
@@ -202,6 +217,9 @@ v3dv_pipeline_cache_init(struct v3dv_pipeline_cache *cache,
       cache->stats.miss = 0;
       cache->stats.hit = 0;
       cache->stats.count = 0;
+
+      cache->externally_synchronized = flags &
+         VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT_EXT;
    } else {
       cache->nir_cache = NULL;
       cache->cache = NULL;
@@ -241,7 +259,7 @@ v3dv_pipeline_cache_search_for_pipeline(struct v3dv_pipeline_cache *cache,
       fprintf(stderr, "pipeline cache %p, search pipeline with key %s\n", cache, sha1buf);
    }
 
-   pthread_mutex_lock(&cache->mutex);
+   pipeline_cache_lock(cache);
 
    struct hash_entry *entry =
       _mesa_hash_table_search(cache->cache, sha1_key);
@@ -261,7 +279,7 @@ v3dv_pipeline_cache_search_for_pipeline(struct v3dv_pipeline_cache *cache,
 
       v3dv_pipeline_shared_data_ref(cache_entry);
 
-      pthread_mutex_unlock(&cache->mutex);
+      pipeline_cache_unlock(cache);
 
       return cache_entry;
    }
@@ -273,7 +291,7 @@ v3dv_pipeline_cache_search_for_pipeline(struct v3dv_pipeline_cache *cache,
          cache_dump_stats(cache);
    }
 
-   pthread_mutex_unlock(&cache->mutex);
+   pipeline_cache_unlock(cache);
 
 #ifdef ENABLE_SHADER_CACHE
    struct v3dv_device *device = cache->device;
@@ -404,12 +422,12 @@ pipeline_cache_upload_shared_data(struct v3dv_pipeline_cache *cache,
    if (cache->stats.count > V3DV_MAX_PIPELINE_CACHE_ENTRIES)
       return;
 
-   pthread_mutex_lock(&cache->mutex);
+   pipeline_cache_lock(cache);
    struct hash_entry *entry =
       _mesa_hash_table_search(cache->cache, shared_data->sha1_key);
 
    if (entry) {
-      pthread_mutex_unlock(&cache->mutex);
+      pipeline_cache_unlock(cache);
       return;
    }
 
@@ -426,7 +444,7 @@ pipeline_cache_upload_shared_data(struct v3dv_pipeline_cache *cache,
          cache_dump_stats(cache);
    }
 
-   pthread_mutex_unlock(&cache->mutex);
+   pipeline_cache_unlock(cache);
 
 #ifdef ENABLE_SHADER_CACHE
    /* If we are being called from a on-disk-cache hit, we can skip writing to
@@ -669,7 +687,6 @@ v3dv_CreatePipelineCache(VkDevice _device,
    struct v3dv_pipeline_cache *cache;
 
    assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO);
-   assert(pCreateInfo->flags == 0);
 
    cache = vk_object_zalloc(&device->vk, pAllocator,
                             sizeof(*cache),
@@ -678,7 +695,7 @@ v3dv_CreatePipelineCache(VkDevice _device,
    if (cache == NULL)
       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
 
-   v3dv_pipeline_cache_init(cache, device,
+   v3dv_pipeline_cache_init(cache, device, pCreateInfo->flags,
                             device->instance->pipeline_cache_enabled);
 
    if (pCreateInfo->initialDataSize > 0) {
@@ -918,7 +935,7 @@ v3dv_GetPipelineCacheData(VkDevice _device,
    struct v3dv_physical_device *pdevice = &device->instance->physicalDevice;
    VkResult result = VK_INCOMPLETE;
 
-   pthread_mutex_lock(&cache->mutex);
+   pipeline_cache_lock(cache);
 
    struct vk_pipeline_cache_header header = {
       .header_size = sizeof(struct vk_pipeline_cache_header),
@@ -995,7 +1012,7 @@ v3dv_GetPipelineCacheData(VkDevice _device,
  done:
    blob_finish(&blob);
 
-   pthread_mutex_unlock(&cache->mutex);
+   pipeline_cache_unlock(cache);
 
    return result;
 }

src/broadcom/vulkan/v3dv_private.h

@@ -396,6 +396,9 @@ struct v3dv_pipeline_cache {
 
    struct hash_table *cache;
    struct v3dv_pipeline_cache_stats stats;
+
+   /* For VK_EXT_pipeline_creation_cache_control. */
+   bool externally_synchronized;
 };
 
 struct v3dv_device {
@@ -2017,6 +2020,7 @@ v3dv_immutable_samplers(const struct v3dv_descriptor_set_layout *set,
 
 void v3dv_pipeline_cache_init(struct v3dv_pipeline_cache *cache,
                               struct v3dv_device *device,
+                              VkPipelineCacheCreateFlags,
                               bool cache_enabled);
 
 void v3dv_pipeline_cache_finish(struct v3dv_pipeline_cache *cache);