radeonsi/gfx11: fix alpha-to-coverage with stencil or samplemask export

We can't use UINT16_ABGR for the alpha channel. Always use 32_ABGR.

Reviewed-by: Mihai Preda <mhpreda@gmail.com>
Reviewed-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/16509>

src/amd/common/ac_shader_util.c

@@ -31,11 +31,15 @@
 #include <stdlib.h>
 #include <string.h>
 
-unsigned ac_get_spi_shader_z_format(bool writes_z, bool writes_stencil, bool writes_samplemask)
+unsigned ac_get_spi_shader_z_format(bool writes_z, bool writes_stencil, bool writes_samplemask,
+                                    bool writes_mrt0_alpha)
 {
-   if (writes_z) {
+   /* If writes_mrt0_alpha is true, one other flag must be true too. */
+   assert(!writes_mrt0_alpha || writes_z || writes_stencil || writes_samplemask);
+
+   if (writes_z || writes_mrt0_alpha) {
       /* Z needs 32 bits. */
-      if (writes_samplemask)
+      if (writes_samplemask || writes_mrt0_alpha)
          return V_028710_SPI_SHADER_32_ABGR;
       else if (writes_stencil)
          return V_028710_SPI_SHADER_32_GR;

src/amd/common/ac_shader_util.h

@@ -90,7 +90,8 @@ enum ac_descriptor_type
    AC_DESC_PLANE_2,
 };
 
-unsigned ac_get_spi_shader_z_format(bool writes_z, bool writes_stencil, bool writes_samplemask);
+unsigned ac_get_spi_shader_z_format(bool writes_z, bool writes_stencil, bool writes_samplemask,
+                                    bool writes_mrt0_alpha);
 
 unsigned ac_get_cb_shader_mask(unsigned spi_shader_col_format);
 

src/amd/llvm/ac_llvm_build.c

@@ -4290,11 +4290,12 @@ LLVMValueRef ac_build_call(struct ac_llvm_context *ctx, LLVMValueRef func, LLVMV
 }
 
 void ac_export_mrt_z(struct ac_llvm_context *ctx, LLVMValueRef depth, LLVMValueRef stencil,
-                     LLVMValueRef samplemask, LLVMValueRef mrtz_alpha, bool is_last,
+                     LLVMValueRef samplemask, LLVMValueRef mrt0_alpha, bool is_last,
                      struct ac_export_args *args)
 {
    unsigned mask = 0;
-   unsigned format = ac_get_spi_shader_z_format(depth != NULL, stencil != NULL, samplemask != NULL);
+   unsigned format = ac_get_spi_shader_z_format(depth != NULL, stencil != NULL, samplemask != NULL,
+                                                mrt0_alpha != NULL);
 
    assert(depth || stencil || samplemask);
 
@@ -4330,17 +4331,6 @@ void ac_export_mrt_z(struct ac_llvm_context *ctx, LLVMValueRef depth, LLVMValueR
          args->out[1] = samplemask;
          mask |= ctx->gfx_level >= GFX11 ? 0x2 : 0xc;
       }
-      if (mrtz_alpha) {
-         /* MRT0 alpha should be in Y[31:16] if alpha-to-coverage is enabled and MRTZ is present. */
-         assert(ctx->gfx_level >= GFX11);
-         mrtz_alpha = LLVMBuildFPTrunc(ctx->builder, mrtz_alpha, ctx->f16, "");
-         mrtz_alpha = ac_to_integer(ctx, mrtz_alpha);
-         mrtz_alpha = LLVMBuildZExt(ctx->builder, mrtz_alpha, ctx->i32, "");
-         mrtz_alpha = LLVMBuildShl(ctx->builder, mrtz_alpha, LLVMConstInt(ctx->i32, 16, 0), "");
-         args->out[1] = LLVMBuildOr(ctx->builder, ac_to_integer(ctx, args->out[1]), mrtz_alpha, "");
-         args->out[1] = ac_to_float(ctx, args->out[1]);
-         mask |= 0x2;
-      }
    } else {
       if (depth) {
          args->out[0] = depth;
@@ -4354,8 +4344,8 @@ void ac_export_mrt_z(struct ac_llvm_context *ctx, LLVMValueRef depth, LLVMValueR
          args->out[2] = samplemask;
          mask |= 0x4;
       }
-      if (mrtz_alpha) {
+      if (mrt0_alpha) {
-         args->out[3] = mrtz_alpha;
+         args->out[3] = mrt0_alpha;
          mask |= 0x8;
       }
    }

src/amd/llvm/ac_llvm_build.h

@@ -556,7 +556,7 @@ LLVMValueRef ac_build_atomic_cmp_xchg(struct ac_llvm_context *ctx, LLVMValueRef
                                       LLVMValueRef cmp, LLVMValueRef val, const char *sync_scope);
 
 void ac_export_mrt_z(struct ac_llvm_context *ctx, LLVMValueRef depth, LLVMValueRef stencil,
-                     LLVMValueRef samplemask, LLVMValueRef mrtz_alpha, bool is_last,
+                     LLVMValueRef samplemask, LLVMValueRef mrt0_alpha, bool is_last,
                      struct ac_export_args *args);
 
 void ac_build_sendmsg_gs_alloc_req(struct ac_llvm_context *ctx, LLVMValueRef wave_id,

src/amd/vulkan/radv_pipeline.c

@@ -6208,7 +6208,7 @@ radv_pipeline_generate_fragment_shader(struct radeon_cmdbuf *ctx_cs, struct rade
    radeon_set_context_reg(
       ctx_cs, R_028710_SPI_SHADER_Z_FORMAT,
       ac_get_spi_shader_z_format(ps->info.ps.writes_z, ps->info.ps.writes_stencil,
-                                 ps->info.ps.writes_sample_mask));
+                                 ps->info.ps.writes_sample_mask, false));
 }
 
 static void

src/gallium/drivers/radeonsi/si_state_shaders.cpp

@@ -1943,7 +1943,8 @@ static void si_shader_ps(struct si_screen *sscreen, struct si_shader *shader)
    shader->ctx_reg.ps.spi_baryc_cntl = spi_baryc_cntl;
    shader->ctx_reg.ps.spi_ps_in_control = spi_ps_in_control;
    shader->ctx_reg.ps.spi_shader_z_format =
-      ac_get_spi_shader_z_format(info->writes_z, info->writes_stencil, info->writes_samplemask);
+      ac_get_spi_shader_z_format(info->writes_z, info->writes_stencil, info->writes_samplemask,
+                                 shader->key.ps.part.epilog.alpha_to_coverage_via_mrtz);
    shader->ctx_reg.ps.spi_shader_col_format = spi_shader_col_format;
    shader->ctx_reg.ps.cb_shader_mask = cb_shader_mask;