radv: allow shaders caching with RADV_DEBUG=hang and the trap handler

When debugging a game that compiles a ton of shaders at beginning, it
can be very very slow because shaders cache was disabled by default
with RADV_DEBUG=hang.

To make debugging such a game faster, let's cache shaders with
RADV_DEBUG=hang. Note that only the backend IR (ACO or LLVM) and the
disassembly are stored in the cache. To get SPIR-V and NIR, you might
need RADV_DEBUG=hang,nocache.

This also handles the trap handler.

Signed-off-by: Samuel Pitoiset <samuel.pitoiset@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/32246>

docs/drivers/amd/hang-debugging.rst

@@ -50,6 +50,10 @@ there are a couple of files:
   including register values.
 * ``vm_fault.log``: The page fault address if a page fault occurred.
 
+Note: By default, the backend IR (ACO or LLVM) and the disassembly should be
+dumped to ``pipeline.log``. But due to shaders caching, you might need
+``RADV_DEBUG=hang,nocache`` to get SPIR-V and NIR in the GPU hang report.
+
 Debugging Steam games
 ---------------------
 

src/amd/vulkan/radv_device.c

@@ -848,8 +848,11 @@ static void
 radv_device_init_cache_key(struct radv_device *device)
 {
    const struct radv_physical_device *pdev = radv_device_physical(device);
+   const struct radv_instance *instance = radv_physical_device_instance(pdev);
    struct radv_device_cache_key *key = &device->cache_key;
 
+   key->keep_shader_info = device->keep_shader_info;
+   key->trap_excp_flags = device->trap_handler_shader && instance->trap_excp_flags;
    key->disable_trunc_coord = device->disable_trunc_coord;
    key->image_2d_view_of_3d = device->vk.enabled_features.image2DViewOf3D && pdev->info.gfx_level == GFX9;
    key->mesh_shader_queries = device->vk.enabled_features.meshShaderQueries;

src/amd/vulkan/radv_device.h

@@ -53,10 +53,12 @@ struct radv_layer_dispatch_tables {
 };
 
 struct radv_device_cache_key {
+   uint32_t keep_shader_info : 1;
    uint32_t disable_trunc_coord : 1;
    uint32_t image_2d_view_of_3d : 1;
    uint32_t mesh_shader_queries : 1;
    uint32_t primitives_generated_query : 1;
+   uint32_t trap_excp_flags : 4;
 };
 
 enum radv_force_vrs {

src/amd/vulkan/radv_pipeline.c

@@ -73,13 +73,17 @@ radv_pipeline_capture_shader_stats(const struct radv_device *device, VkPipelineC
 bool
 radv_pipeline_skip_shaders_cache(const struct radv_device *device, const struct radv_pipeline *pipeline)
 {
-   const bool keep_executable_info = radv_pipeline_capture_shaders(device, pipeline->create_flags);
+   const struct radv_physical_device *pdev = radv_device_physical(device);
+   const struct radv_instance *instance = radv_physical_device_instance(pdev);
 
    /* Skip the shaders cache when any of the below are true:
-    * - shaders are captured because it's for debugging purposes
-    * - binaries are captured for later uses
+    * - shaders are dumped for debugging (RADV_DEBUG=shaders)
+    * - shaders IR are captured (NIR, backend IR and ASM)
+    * - binaries are captured (driver shouldn't store data to an internal cache)
     */
-   return keep_executable_info || (pipeline->create_flags & VK_PIPELINE_CREATE_2_CAPTURE_DATA_BIT_KHR);
+   return (instance->debug_flags & RADV_DEBUG_DUMP_SHADERS) ||
+          (pipeline->create_flags &
+           (VK_PIPELINE_CREATE_2_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR | VK_PIPELINE_CREATE_2_CAPTURE_DATA_BIT_KHR));
 }
 
 void