diff --git a/src/amd/llvm/ac_llvm_build.c b/src/amd/llvm/ac_llvm_build.c index 7b0390d7ca3..4c09ed8feea 100644 --- a/src/amd/llvm/ac_llvm_build.c +++ b/src/amd/llvm/ac_llvm_build.c @@ -2820,20 +2820,24 @@ void ac_init_exec_full_mask(struct ac_llvm_context *ctx) void ac_declare_lds_as_pointer(struct ac_llvm_context *ctx) { unsigned lds_size = ctx->gfx_level >= GFX7 ? 65536 : 32768; - ctx->lds = LLVMBuildIntToPtr( - ctx->builder, ctx->i32_0, - LLVMPointerType(LLVMArrayType(ctx->i32, lds_size / 4), AC_ADDR_SPACE_LDS), "lds"); + LLVMTypeRef type = LLVMArrayType(ctx->i32, lds_size / 4); + ctx->lds = (struct ac_llvm_pointer) { + .value = LLVMBuildIntToPtr(ctx->builder, ctx->i32_0, + LLVMPointerType(type, AC_ADDR_SPACE_LDS), "lds"), + .pointee_type = type + }; } LLVMValueRef ac_lds_load(struct ac_llvm_context *ctx, LLVMValueRef dw_addr) { - return LLVMBuildLoad2(ctx->builder, ctx->i32, ac_build_gep0(ctx, ctx->lds, dw_addr), ""); + LLVMValueRef v = ac_build_gep0_2(ctx, ctx->lds.t, ctx->lds.v, dw_addr); + return LLVMBuildLoad2(ctx->builder, ctx->i32, v, ""); } void ac_lds_store(struct ac_llvm_context *ctx, LLVMValueRef dw_addr, LLVMValueRef value) { value = ac_to_integer(ctx, value); - ac_build_indexed_store(ctx, ctx->lds, dw_addr, value); + ac_build_indexed_store2(ctx, ctx->lds.t, ctx->lds.v, dw_addr, value); } LLVMValueRef ac_find_lsb(struct ac_llvm_context *ctx, LLVMTypeRef dst_type, LLVMValueRef src0) diff --git a/src/amd/llvm/ac_llvm_build.h b/src/amd/llvm/ac_llvm_build.h index dee1bce44de..aeee00ee517 100644 --- a/src/amd/llvm/ac_llvm_build.h +++ b/src/amd/llvm/ac_llvm_build.h @@ -156,7 +156,7 @@ struct ac_llvm_context { unsigned float_mode; - LLVMValueRef lds; + struct ac_llvm_pointer lds; }; void ac_llvm_context_init(struct ac_llvm_context *ctx, struct ac_llvm_compiler *compiler, diff --git a/src/amd/llvm/ac_nir_to_llvm.c b/src/amd/llvm/ac_nir_to_llvm.c index e9fc7f6d1d7..240abab6ebc 100644 --- a/src/amd/llvm/ac_nir_to_llvm.c +++ b/src/amd/llvm/ac_nir_to_llvm.c @@ -84,7 +84,7 @@ static LLVMValueRef get_memory_ptr(struct ac_nir_context *ctx, nir_src src, unsi LLVMValueRef ptr = get_src(ctx, src); ptr = LLVMBuildAdd(ctx->ac.builder, ptr, LLVMConstInt(ctx->ac.i32, c_off, 0), ""); /* LDS is used here as a i8 pointer. */ - return LLVMBuildGEP2(ctx->ac.builder, ctx->ac.i8, ctx->ac.lds, &ptr, 1, ""); + return LLVMBuildGEP2(ctx->ac.builder, ctx->ac.i8, ctx->ac.lds.value, &ptr, 1, ""); } static LLVMBasicBlockRef get_block(struct ac_nir_context *nir, const struct nir_block *b) @@ -5373,7 +5373,7 @@ static void setup_constant_data(struct ac_nir_context *ctx, struct nir_shader *s static void setup_shared(struct ac_nir_context *ctx, struct nir_shader *nir) { - if (ctx->ac.lds) + if (ctx->ac.lds.value) return; LLVMTypeRef type = LLVMArrayType(ctx->ac.i8, nir->info.shared_size); @@ -5382,7 +5382,10 @@ static void setup_shared(struct ac_nir_context *ctx, struct nir_shader *nir) LLVMAddGlobalInAddressSpace(ctx->ac.module, type, "compute_lds", AC_ADDR_SPACE_LDS); LLVMSetAlignment(lds, 64 * 1024); - ctx->ac.lds = lds; + ctx->ac.lds = (struct ac_llvm_pointer) { + .value = lds, + .pointee_type = type + }; } static void setup_gds(struct ac_nir_context *ctx, nir_function_impl *impl) diff --git a/src/gallium/drivers/radeonsi/si_shader_llvm.c b/src/gallium/drivers/radeonsi/si_shader_llvm.c index 03b4f10353e..41fabf19476 100644 --- a/src/gallium/drivers/radeonsi/si_shader_llvm.c +++ b/src/gallium/drivers/radeonsi/si_shader_llvm.c @@ -232,9 +232,12 @@ void si_llvm_create_main_func(struct si_shader_context *ctx, bool ngg_cull_shade * the shader (currently none, unless LLVM decides to do its * own LDS-based lowering). */ - ctx->ac.lds = LLVMAddGlobalInAddressSpace(ctx->ac.module, LLVMArrayType(ctx->ac.i32, 0), - "__lds_end", AC_ADDR_SPACE_LDS); - LLVMSetAlignment(ctx->ac.lds, 256); + ctx->ac.lds = (struct ac_llvm_pointer) { + .value = LLVMAddGlobalInAddressSpace(ctx->ac.module, LLVMArrayType(ctx->ac.i32, 0), + "__lds_end", AC_ADDR_SPACE_LDS), + .pointee_type = LLVMArrayType(ctx->ac.i32, 0) + }; + LLVMSetAlignment(ctx->ac.lds.value, 256); } /* Unlike radv, we override these arguments in the prolog, so to the @@ -407,13 +410,17 @@ static void si_llvm_declare_compute_memory(struct si_shader_context *ctx) LLVMTypeRef i8p = LLVMPointerType(ctx->ac.i8, AC_ADDR_SPACE_LDS); LLVMValueRef var; - assert(!ctx->ac.lds); + assert(!ctx->ac.lds.value); - var = LLVMAddGlobalInAddressSpace(ctx->ac.module, LLVMArrayType(ctx->ac.i8, lds_size), + LLVMTypeRef type = LLVMArrayType(ctx->ac.i8, lds_size); + var = LLVMAddGlobalInAddressSpace(ctx->ac.module, type, "compute_lds", AC_ADDR_SPACE_LDS); LLVMSetAlignment(var, 64 * 1024); - ctx->ac.lds = LLVMBuildBitCast(ctx->ac.builder, var, i8p, ""); + ctx->ac.lds = (struct ac_llvm_pointer) { + .value = LLVMBuildBitCast(ctx->ac.builder, var, i8p, ""), + .pointee_type = type, + }; } /** diff --git a/src/gallium/drivers/radeonsi/si_shader_llvm_gs.c b/src/gallium/drivers/radeonsi/si_shader_llvm_gs.c index 9da73718b5d..4634fdf4c8d 100644 --- a/src/gallium/drivers/radeonsi/si_shader_llvm_gs.c +++ b/src/gallium/drivers/radeonsi/si_shader_llvm_gs.c @@ -290,7 +290,10 @@ void si_preload_esgs_ring(struct si_shader_context *ctx) } else { /* Declare the ESGS ring as an explicit LDS symbol. */ si_llvm_declare_esgs_ring(ctx); - ctx->ac.lds = ctx->esgs_ring; + ctx->ac.lds = (struct ac_llvm_pointer) { + .value = ctx->esgs_ring, + .pointee_type = LLVMArrayType(ctx->ac.i32, 0), + }; } }