From 2f2a9f253e973f38a37a0491f87db64dc1026dc5 Mon Sep 17 00:00:00 2001 From: Rhys Perry Date: Mon, 12 Oct 2020 15:07:39 +0100 Subject: [PATCH] radv,ac/nir: implement non-uniform get_ssbo_size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rhys Perry Reviewed-by: Daniel Schürmann Gitlab: https://gitlab.freedesktop.org/mesa/mesa/-/issues/3711 Part-of: --- src/amd/llvm/ac_nir_to_llvm.c | 12 ++++++------ src/amd/llvm/ac_shader_abi.h | 3 ++- src/amd/vulkan/radv_nir_to_llvm.c | 15 ++++++++++++--- .../drivers/radeonsi/si_shader_llvm_resources.c | 2 +- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/amd/llvm/ac_nir_to_llvm.c b/src/amd/llvm/ac_nir_to_llvm.c index b6c768f4b84..3dac927a6cc 100644 --- a/src/amd/llvm/ac_nir_to_llvm.c +++ b/src/amd/llvm/ac_nir_to_llvm.c @@ -1618,9 +1618,9 @@ static LLVMValueRef visit_load_push_constant(struct ac_nir_context *ctx, nir_int static LLVMValueRef visit_get_ssbo_size(struct ac_nir_context *ctx, const nir_intrinsic_instr *instr) { - LLVMValueRef index = get_src(ctx, instr->src[0]); - - return get_buffer_size(ctx, ctx->abi->load_ssbo(ctx->abi, index, false), false); + bool non_uniform = nir_intrinsic_access(instr) & ACCESS_NON_UNIFORM; + LLVMValueRef rsrc = ctx->abi->load_ssbo(ctx->abi, get_src(ctx, instr->src[0]), false, non_uniform); + return get_buffer_size(ctx, rsrc, false); } static LLVMValueRef extract_vector_range(struct ac_llvm_context *ctx, LLVMValueRef src, @@ -1693,7 +1693,7 @@ static void visit_store_ssbo(struct ac_nir_context *ctx, nir_intrinsic_instr *in struct waterfall_context wctx; LLVMValueRef rsrc_base = enter_waterfall_ssbo(ctx, &wctx, instr, instr->src[1]); - LLVMValueRef rsrc = ctx->abi->load_ssbo(ctx->abi, rsrc_base, true); + LLVMValueRef rsrc = ctx->abi->load_ssbo(ctx->abi, rsrc_base, true, false); LLVMValueRef base_data = src_data; base_data = ac_trim_vector(&ctx->ac, base_data, instr->num_components); LLVMValueRef base_offset = get_src(ctx, instr->src[2]); @@ -1889,7 +1889,7 @@ static LLVMValueRef visit_atomic_ssbo(struct ac_nir_context *ctx, nir_intrinsic_ abort(); } - descriptor = ctx->abi->load_ssbo(ctx->abi, rsrc_base, true); + descriptor = ctx->abi->load_ssbo(ctx->abi, rsrc_base, true, false); if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap && return_type == ctx->ac.i64) { result = emit_ssbo_comp_swap_64(ctx, descriptor, get_src(ctx, instr->src[1]), @@ -1940,7 +1940,7 @@ static LLVMValueRef visit_load_buffer(struct ac_nir_context *ctx, nir_intrinsic_ unsigned cache_policy = get_cache_policy(ctx, access, false, false); LLVMValueRef offset = get_src(ctx, instr->src[1]); - LLVMValueRef rsrc = ctx->abi->load_ssbo(ctx->abi, rsrc_base, false); + LLVMValueRef rsrc = ctx->abi->load_ssbo(ctx->abi, rsrc_base, false, false); LLVMValueRef vindex = ctx->ac.i32_0; LLVMTypeRef def_type = get_def_type(ctx, &instr->dest.ssa); diff --git a/src/amd/llvm/ac_shader_abi.h b/src/amd/llvm/ac_shader_abi.h index 8f904b522a2..aba8559b20a 100644 --- a/src/amd/llvm/ac_shader_abi.h +++ b/src/amd/llvm/ac_shader_abi.h @@ -110,8 +110,9 @@ struct ac_shader_abi { * \param buffer the buffer as presented in NIR: this is the descriptor * in Vulkan, and the buffer index in OpenGL/Gallium * \param write whether buffer contents will be written + * \param non_uniform whether the buffer descriptor is not assumed to be uniform */ - LLVMValueRef (*load_ssbo)(struct ac_shader_abi *abi, LLVMValueRef buffer, bool write); + LLVMValueRef (*load_ssbo)(struct ac_shader_abi *abi, LLVMValueRef buffer, bool write, bool non_uniform); /** * Load a descriptor associated to a sampler. diff --git a/src/amd/vulkan/radv_nir_to_llvm.c b/src/amd/vulkan/radv_nir_to_llvm.c index 3818a41f047..b16bc53e930 100644 --- a/src/amd/vulkan/radv_nir_to_llvm.c +++ b/src/amd/vulkan/radv_nir_to_llvm.c @@ -374,7 +374,6 @@ radv_load_resource(struct ac_shader_abi *abi, LLVMValueRef index, desc_ptr = LLVMBuildGEP(ctx->ac.builder, desc_ptr, &offset, 1, ""); desc_ptr = ac_cast_ptr(&ctx->ac, desc_ptr, ctx->ac.v4i32); - LLVMSetMetadata(desc_ptr, ctx->ac.uniform_md_kind, ctx->ac.empty_md); return desc_ptr; } @@ -869,12 +868,22 @@ static LLVMValueRef radv_load_base_vertex(struct ac_shader_abi *abi, bool non_in } static LLVMValueRef radv_load_ssbo(struct ac_shader_abi *abi, - LLVMValueRef buffer_ptr, bool write) + LLVMValueRef buffer_ptr, bool write, bool non_uniform) { struct radv_shader_context *ctx = radv_shader_context_from_abi(abi); LLVMValueRef result; - LLVMSetMetadata(buffer_ptr, ctx->ac.uniform_md_kind, ctx->ac.empty_md); + if (!non_uniform) + LLVMSetMetadata(buffer_ptr, ctx->ac.uniform_md_kind, ctx->ac.empty_md); + + if (non_uniform && LLVMGetPointerAddressSpace(LLVMTypeOf(buffer_ptr)) == AC_ADDR_SPACE_CONST_32BIT) { + /* 32-bit seems to always use SMEM. addrspacecast from 32-bit -> 64-bit is broken. */ + buffer_ptr = LLVMBuildPtrToInt(ctx->ac.builder, buffer_ptr, ctx->ac.i32, ""), + buffer_ptr = LLVMBuildZExt(ctx->ac.builder, buffer_ptr, ctx->ac.i64, ""); + uint64_t hi = (uint64_t)ctx->args->options->address32_hi << 32; + buffer_ptr = LLVMBuildOr(ctx->ac.builder, buffer_ptr, LLVMConstInt(ctx->ac.i64, hi, false), ""); + buffer_ptr = LLVMBuildIntToPtr(ctx->ac.builder, buffer_ptr, LLVMPointerType(ctx->ac.v4i32, AC_ADDR_SPACE_CONST), ""); + } result = LLVMBuildLoad(ctx->ac.builder, buffer_ptr, ""); LLVMSetMetadata(result, ctx->ac.invariant_load_md_kind, ctx->ac.empty_md); diff --git a/src/gallium/drivers/radeonsi/si_shader_llvm_resources.c b/src/gallium/drivers/radeonsi/si_shader_llvm_resources.c index 6cf26ac9fdb..260fafff5a2 100644 --- a/src/gallium/drivers/radeonsi/si_shader_llvm_resources.c +++ b/src/gallium/drivers/radeonsi/si_shader_llvm_resources.c @@ -106,7 +106,7 @@ static LLVMValueRef load_ubo(struct ac_shader_abi *abi, return ac_build_load_to_sgpr(&ctx->ac, ptr, index); } -static LLVMValueRef load_ssbo(struct ac_shader_abi *abi, LLVMValueRef index, bool write) +static LLVMValueRef load_ssbo(struct ac_shader_abi *abi, LLVMValueRef index, bool write, bool non_uniform) { struct si_shader_context *ctx = si_shader_context_from_abi(abi);