diff --git a/src/amd/vulkan/radv_pipeline.c b/src/amd/vulkan/radv_pipeline.c index 689e75a7cae..e343403bc5e 100644 --- a/src/amd/vulkan/radv_pipeline.c +++ b/src/amd/vulkan/radv_pipeline.c @@ -3215,7 +3215,8 @@ non_uniform_access_callback(const nir_src *src, void *_) VkResult -radv_upload_shaders(struct radv_device *device, struct radv_pipeline *pipeline) +radv_upload_shaders(struct radv_device *device, struct radv_pipeline *pipeline, + struct radv_shader_binary **binaries, struct radv_shader_binary *gs_copy_binary) { uint32_t code_size = 0; @@ -3258,7 +3259,7 @@ radv_upload_shaders(struct radv_device *device, struct radv_pipeline *pipeline) shader->va = slab_va + slab_offset; void *dest_ptr = slab_ptr + slab_offset; - if (!radv_shader_binary_upload(device, shader->binary, shader, dest_ptr)) + if (!radv_shader_binary_upload(device, binaries[i], shader, dest_ptr)) return VK_ERROR_OUT_OF_HOST_MEMORY; slab_offset += align(shader->code_size, RADV_SHADER_ALLOC_ALIGNMENT); @@ -3268,8 +3269,7 @@ radv_upload_shaders(struct radv_device *device, struct radv_pipeline *pipeline) pipeline->gs_copy_shader->va = slab_va + slab_offset; void *dest_ptr = slab_ptr + slab_offset; - if (!radv_shader_binary_upload(device, pipeline->gs_copy_shader->binary, - pipeline->gs_copy_shader, dest_ptr)) + if (!radv_shader_binary_upload(device, gs_copy_binary, pipeline->gs_copy_shader, dest_ptr)) return VK_ERROR_OUT_OF_HOST_MEMORY; } @@ -3634,7 +3634,8 @@ radv_pipeline_create_gs_copy_shader(struct radv_pipeline *pipeline, struct radv_pipeline_stage *stages, const struct radv_pipeline_key *pipeline_key, const struct radv_pipeline_layout *pipeline_layout, - bool keep_executable_info, bool keep_statistic_info) + bool keep_executable_info, bool keep_statistic_info, + struct radv_shader_binary **gs_copy_binary) { struct radv_device *device = pipeline->device; struct radv_shader_info info = {0}; @@ -3663,7 +3664,7 @@ radv_pipeline_create_gs_copy_shader(struct radv_pipeline *pipeline, info.inline_push_constant_mask = gs_copy_args.ac.inline_push_const_mask; return radv_create_gs_copy_shader(device, stages[MESA_SHADER_GEOMETRY].nir, &info, &gs_copy_args, - keep_executable_info, keep_statistic_info, + gs_copy_binary, keep_executable_info, keep_statistic_info, pipeline_key->optimisations_disabled); } @@ -3672,7 +3673,9 @@ radv_pipeline_nir_to_asm(struct radv_pipeline *pipeline, struct radv_pipeline_st const struct radv_pipeline_key *pipeline_key, const struct radv_pipeline_layout *pipeline_layout, bool keep_executable_info, bool keep_statistic_info, - gl_shader_stage last_vgt_api_stage) + gl_shader_stage last_vgt_api_stage, + struct radv_shader_binary **binaries, + struct radv_shader_binary **gs_copy_binary) { struct radv_device *device = pipeline->device; unsigned active_stages = 0; @@ -3688,7 +3691,8 @@ radv_pipeline_nir_to_asm(struct radv_pipeline *pipeline, struct radv_pipeline_st if (stages[MESA_SHADER_GEOMETRY].nir && !pipeline_has_ngg) { pipeline->gs_copy_shader = radv_pipeline_create_gs_copy_shader(pipeline, stages, pipeline_key, pipeline_layout, - keep_executable_info, keep_statistic_info); + keep_executable_info, keep_statistic_info, + gs_copy_binary); } for (int s = MESA_VULKAN_SHADER_STAGES - 1; s >= 0; s--) { @@ -3718,7 +3722,7 @@ radv_pipeline_nir_to_asm(struct radv_pipeline *pipeline, struct radv_pipeline_st pipeline->shaders[s] = radv_shader_nir_to_asm(device, &stages[s], shaders, shader_count, pipeline_key, keep_executable_info, - keep_statistic_info); + keep_statistic_info, &binaries[s]); stages[s].feedback.duration += os_time_get_nano() - stage_start; @@ -3993,6 +3997,8 @@ radv_create_shaders(struct radv_pipeline *pipeline, struct radv_pipeline_layout gl_shader_stage *last_vgt_api_stage) { const char *noop_fs_entrypoint = "noop_fs"; + struct radv_shader_binary *binaries[MESA_VULKAN_SHADER_STAGES] = {NULL}; + struct radv_shader_binary *gs_copy_binary = NULL; unsigned char hash[20]; bool keep_executable_info = (flags & VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR) || @@ -4176,7 +4182,7 @@ radv_create_shaders(struct radv_pipeline *pipeline, struct radv_pipeline_layout /* Compile NIR shaders to AMD assembly. */ radv_pipeline_nir_to_asm(pipeline, stages, pipeline_key, pipeline_layout, keep_executable_info, - keep_statistic_info, *last_vgt_api_stage); + keep_statistic_info, *last_vgt_api_stage, binaries, &gs_copy_binary); if (keep_executable_info) { for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { @@ -4210,35 +4216,29 @@ radv_create_shaders(struct radv_pipeline *pipeline, struct radv_pipeline_layout } /* Upload shader binaries. */ - radv_upload_shaders(device, pipeline); + radv_upload_shaders(device, pipeline, binaries, gs_copy_binary); if (!keep_executable_info) { if (pipeline->gs_copy_shader) { - assert(!pipeline->shaders[MESA_SHADER_COMPUTE]); + assert(!binaries[MESA_SHADER_COMPUTE] && !pipeline->shaders[MESA_SHADER_COMPUTE]); + binaries[MESA_SHADER_COMPUTE] = gs_copy_binary; pipeline->shaders[MESA_SHADER_COMPUTE] = pipeline->gs_copy_shader; } - radv_pipeline_cache_insert_shaders(device, cache, hash, pipeline, + radv_pipeline_cache_insert_shaders(device, cache, hash, pipeline, binaries, stack_sizes ? *stack_sizes : NULL, num_stack_sizes ? *num_stack_sizes : 0); if (pipeline->gs_copy_shader) { pipeline->gs_copy_shader = pipeline->shaders[MESA_SHADER_COMPUTE]; pipeline->shaders[MESA_SHADER_COMPUTE] = NULL; + binaries[MESA_SHADER_COMPUTE] = NULL; } } - if (pipeline->gs_copy_shader) { - free(pipeline->gs_copy_shader->binary); - pipeline->gs_copy_shader->binary = NULL; - } - + free(gs_copy_binary); for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { - if (pipeline->shaders[i]) { - free(pipeline->shaders[i]->binary); - pipeline->shaders[i]->binary = NULL; - } - + free(binaries[i]); if (stages[i].nir) { if (radv_can_dump_shader_stats(device, stages[i].nir) && pipeline->shaders[i]) { radv_dump_shader_stats(device, pipeline, i, stderr); diff --git a/src/amd/vulkan/radv_pipeline_cache.c b/src/amd/vulkan/radv_pipeline_cache.c index 06b825b98b6..bb8b5ed6b9f 100644 --- a/src/amd/vulkan/radv_pipeline_cache.c +++ b/src/amd/vulkan/radv_pipeline_cache.c @@ -372,6 +372,8 @@ radv_create_shaders_from_pipeline_cache( } } + struct radv_shader_binary *binaries[MESA_VULKAN_SHADER_STAGES] = {NULL}; + struct radv_shader_binary *gs_copy_binary = NULL; bool needs_upload = false; char *p = entry->code; for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { @@ -383,6 +385,7 @@ radv_create_shaders_from_pipeline_cache( entry->shaders[i] = radv_shader_create(device, binary, false, true, NULL); needs_upload = true; + binaries[i] = binary; } else if (entry->binary_sizes[i]) { p += entry->binary_sizes[i]; } @@ -395,22 +398,17 @@ radv_create_shaders_from_pipeline_cache( /* For the GS copy shader, RADV uses the compute shader slot to avoid a new cache entry. */ pipeline->gs_copy_shader = pipeline->shaders[MESA_SHADER_COMPUTE]; pipeline->shaders[MESA_SHADER_COMPUTE] = NULL; + gs_copy_binary = binaries[MESA_SHADER_COMPUTE]; } if (needs_upload) { - result = radv_upload_shaders(device, pipeline); + result = radv_upload_shaders(device, pipeline, binaries, gs_copy_binary); for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { - if (pipeline->shaders[i]) { - free(pipeline->shaders[i]->binary); - pipeline->shaders[i]->binary = NULL; - } - } - - if (pipeline->gs_copy_shader) { - free(pipeline->gs_copy_shader->binary); - pipeline->gs_copy_shader->binary = NULL; + if (pipeline->shaders[i]) + free(binaries[i]); } + free(gs_copy_binary); if (result != VK_SUCCESS) { radv_pipeline_cache_unlock(cache); @@ -452,6 +450,7 @@ radv_create_shaders_from_pipeline_cache( void radv_pipeline_cache_insert_shaders(struct radv_device *device, struct radv_pipeline_cache *cache, const unsigned char *sha1, struct radv_pipeline *pipeline, + struct radv_shader_binary *const *binaries, const struct radv_pipeline_shader_stack_size *stack_sizes, uint32_t num_stack_sizes) { @@ -489,14 +488,9 @@ radv_pipeline_cache_insert_shaders(struct radv_device *device, struct radv_pipel } size_t size = sizeof(*entry) + sizeof(*stack_sizes) * num_stack_sizes; - for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { - struct radv_shader *shader = pipeline->shaders[i]; - if (!shader) - continue; - - size += shader->binary->total_size; - } - + for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) + if (pipeline->shaders[i]) + size += binaries[i]->total_size; const size_t size_without_align = size; size = align(size_without_align, alignof(struct cache_entry)); @@ -512,15 +506,13 @@ radv_pipeline_cache_insert_shaders(struct radv_device *device, struct radv_pipel char *p = entry->code; for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) { - struct radv_shader *shader = pipeline->shaders[i]; - - if (!shader) + if (!pipeline->shaders[i]) continue; - entry->binary_sizes[i] = shader->binary->total_size; + entry->binary_sizes[i] = binaries[i]->total_size; - memcpy(p, shader->binary, shader->binary->total_size); - p += shader->binary->total_size; + memcpy(p, binaries[i], binaries[i]->total_size); + p += binaries[i]->total_size; } if (num_stack_sizes) { diff --git a/src/amd/vulkan/radv_private.h b/src/amd/vulkan/radv_private.h index e95a7b57809..56efccf6255 100644 --- a/src/amd/vulkan/radv_private.h +++ b/src/amd/vulkan/radv_private.h @@ -392,10 +392,12 @@ bool radv_create_shaders_from_pipeline_cache( void radv_pipeline_cache_insert_shaders( struct radv_device *device, struct radv_pipeline_cache *cache, const unsigned char *sha1, - struct radv_pipeline *pipeline, const struct radv_pipeline_shader_stack_size *stack_sizes, - uint32_t num_stack_sizes); + struct radv_pipeline *pipeline, struct radv_shader_binary *const *binaries, + const struct radv_pipeline_shader_stack_size *stack_sizes, uint32_t num_stack_sizes); -VkResult radv_upload_shaders(struct radv_device *device, struct radv_pipeline *pipeline); +VkResult radv_upload_shaders(struct radv_device *device, struct radv_pipeline *pipeline, + struct radv_shader_binary **binaries, + struct radv_shader_binary *gs_copy_binary); enum radv_blit_ds_layout { RADV_BLIT_DS_LAYOUT_TILE_ENABLE, diff --git a/src/amd/vulkan/radv_shader.c b/src/amd/vulkan/radv_shader.c index e31c128c4f9..c9151247bf2 100644 --- a/src/amd/vulkan/radv_shader.c +++ b/src/amd/vulkan/radv_shader.c @@ -2111,7 +2111,7 @@ radv_shader_binary_upload(struct radv_device *device, const struct radv_shader_b } struct radv_shader * -radv_shader_create(struct radv_device *device, struct radv_shader_binary *binary, +radv_shader_create(struct radv_device *device, const struct radv_shader_binary *binary, bool keep_shader_info, bool from_cache, const struct radv_shader_args *args) { struct ac_shader_config config = {0}; @@ -2120,7 +2120,6 @@ radv_shader_create(struct radv_device *device, struct radv_shader_binary *binary return NULL; shader->ref_count = 1; - shader->binary = binary; if (binary->type == RADV_BINARY_TYPE_RTLD) { #if !defined(USE_LIBELF) @@ -2340,7 +2339,8 @@ static struct radv_shader * shader_compile(struct radv_device *device, struct nir_shader *const *shaders, int shader_count, gl_shader_stage stage, const struct radv_shader_info *info, const struct radv_shader_args *args, struct radv_nir_compiler_options *options, bool gs_copy_shader, - bool trap_handler_shader, bool keep_shader_info, bool keep_statistic_info) + bool trap_handler_shader, bool keep_shader_info, bool keep_statistic_info, + struct radv_shader_binary **binary_out) { enum radeon_family chip_family = device->physical_device->rad_info.family; struct radv_shader_binary *binary = NULL; @@ -2406,6 +2406,7 @@ shader_compile(struct radv_device *device, struct nir_shader *const *shaders, in /* Copy the shader binary configuration to store it in the cache. */ memcpy(&binary->config, &shader->config, sizeof(binary->config)); + *binary_out = binary; return shader; } @@ -2413,7 +2414,7 @@ struct radv_shader * radv_shader_nir_to_asm(struct radv_device *device, struct radv_pipeline_stage *pl_stage, struct nir_shader *const *shaders, int shader_count, const struct radv_pipeline_key *key, bool keep_shader_info, - bool keep_statistic_info) + bool keep_statistic_info, struct radv_shader_binary **binary_out) { gl_shader_stage stage = shaders[shader_count - 1]->info.stage; struct radv_nir_compiler_options options = {0}; @@ -2426,14 +2427,14 @@ radv_shader_nir_to_asm(struct radv_device *device, struct radv_pipeline_stage *p return shader_compile(device, shaders, shader_count, stage, &pl_stage->info, &pl_stage->args, &options, false, false, keep_shader_info, - keep_statistic_info); + keep_statistic_info, binary_out); } struct radv_shader * radv_create_gs_copy_shader(struct radv_device *device, struct nir_shader *shader, const struct radv_shader_info *info, const struct radv_shader_args *args, - bool keep_shader_info, bool keep_statistic_info, - bool disable_optimizations) + struct radv_shader_binary **binary_out, bool keep_shader_info, + bool keep_statistic_info, bool disable_optimizations) { struct radv_nir_compiler_options options = {0}; gl_shader_stage stage = MESA_SHADER_VERTEX; @@ -2441,7 +2442,7 @@ radv_create_gs_copy_shader(struct radv_device *device, struct nir_shader *shader options.key.optimisations_disabled = disable_optimizations; return shader_compile(device, &shader, 1, stage, info, args, &options, true, false, - keep_shader_info, keep_statistic_info); + keep_shader_info, keep_statistic_info, binary_out); } struct radv_trap_handler_shader * @@ -2449,6 +2450,7 @@ radv_create_trap_handler_shader(struct radv_device *device) { struct radv_nir_compiler_options options = {0}; struct radv_shader *shader = NULL; + struct radv_shader_binary *binary = NULL; struct radv_shader_info info = {0}; struct radv_pipeline_key key = {0}; struct radv_trap_handler_shader *trap; @@ -2469,19 +2471,19 @@ radv_create_trap_handler_shader(struct radv_device *device) MESA_SHADER_COMPUTE, false, MESA_SHADER_VERTEX, &args); shader = shader_compile(device, &b.shader, 1, MESA_SHADER_COMPUTE, &info, &args, &options, - false, true, false, false); + false, true, false, false, &binary); trap->alloc = radv_alloc_shader_memory(device, shader->code_size, NULL); trap->bo = trap->alloc->arena->bo; char *dest_ptr = trap->alloc->arena->ptr + trap->alloc->offset; - struct radv_shader_binary_legacy *bin = (struct radv_shader_binary_legacy *)shader->binary; + struct radv_shader_binary_legacy *bin = (struct radv_shader_binary_legacy *)binary; memcpy(dest_ptr, bin->data, bin->code_size); ralloc_free(b.shader); - free(shader->binary); free(shader); + free(binary); return trap; } @@ -2684,7 +2686,6 @@ radv_shader_destroy(struct radv_device *device, struct radv_shader *shader) { assert(shader->ref_count == 0); - free(shader->binary); free(shader->spirv); free(shader->nir_string); free(shader->disasm_string); diff --git a/src/amd/vulkan/radv_shader.h b/src/amd/vulkan/radv_shader.h index 1688916d424..a0d75fd96f3 100644 --- a/src/amd/vulkan/radv_shader.h +++ b/src/amd/vulkan/radv_shader.h @@ -493,7 +493,6 @@ struct radv_shader { uint32_t code_size; uint32_t exec_size; struct radv_shader_info info; - struct radv_shader_binary *binary; /* debug only */ char *spirv; @@ -571,12 +570,13 @@ VkResult radv_create_shaders(struct radv_pipeline *pipeline, struct radv_shader_args; struct radv_shader *radv_shader_create(struct radv_device *device, - struct radv_shader_binary *binary, + const struct radv_shader_binary *binary, bool keep_shader_info, bool from_cache, const struct radv_shader_args *args); struct radv_shader *radv_shader_nir_to_asm( struct radv_device *device, struct radv_pipeline_stage *stage, struct nir_shader *const *shaders, - int shader_count, const struct radv_pipeline_key *key, bool keep_shader_info, bool keep_statistic_info); + int shader_count, const struct radv_pipeline_key *key, bool keep_shader_info, bool keep_statistic_info, + struct radv_shader_binary **binary_out); bool radv_shader_binary_upload(struct radv_device *device, const struct radv_shader_binary *binary, struct radv_shader *shader, void *dest_ptr); @@ -590,6 +590,7 @@ void radv_free_shader_memory(struct radv_device *device, union radv_shader_arena struct radv_shader * radv_create_gs_copy_shader(struct radv_device *device, struct nir_shader *nir, const struct radv_shader_info *info, const struct radv_shader_args *args, + struct radv_shader_binary **binary_out, bool keep_shader_info, bool keep_statistic_info, bool disable_optimizations);