diff --git a/src/amd/vulkan/nir/radv_nir.h b/src/amd/vulkan/nir/radv_nir.h index c49657dad46..796a71040f7 100644 --- a/src/amd/vulkan/nir/radv_nir.h +++ b/src/amd/vulkan/nir/radv_nir.h @@ -36,34 +36,35 @@ extern "C" { typedef struct nir_shader nir_shader; struct radeon_info; struct radv_pipeline_layout; -struct radv_pipeline_key; struct radv_shader_stage; struct radv_shader_info; struct radv_shader_args; struct radv_shader_layout; struct radv_device; +struct radv_graphics_state_key; void radv_nir_apply_pipeline_layout(nir_shader *shader, struct radv_device *device, const struct radv_shader_stage *stage); void radv_nir_lower_abi(nir_shader *shader, enum amd_gfx_level gfx_level, const struct radv_shader_stage *stage, - const struct radv_pipeline_key *pl_key, uint32_t address32_hi); + const struct radv_graphics_state_key *gfx_state, uint32_t address32_hi); bool radv_nir_lower_hit_attrib_derefs(nir_shader *shader); bool radv_nir_lower_ray_queries(nir_shader *shader, struct radv_device *device); bool radv_nir_lower_vs_inputs(nir_shader *shader, const struct radv_shader_stage *vs_stage, - const struct radv_pipeline_key *pl_key, const struct radeon_info *rad_info); + const struct radv_graphics_state_key *gfx_state, const struct radeon_info *rad_info); bool radv_nir_lower_primitive_shading_rate(nir_shader *nir, enum amd_gfx_level gfx_level); bool radv_nir_lower_fs_intrinsics(nir_shader *nir, const struct radv_shader_stage *fs_stage, - const struct radv_pipeline_key *key); + const struct radv_graphics_state_key *gfx_state); -bool radv_nir_lower_fs_barycentric(nir_shader *shader, const struct radv_pipeline_key *key, unsigned rast_prim); +bool radv_nir_lower_fs_barycentric(nir_shader *shader, const struct radv_graphics_state_key *gfx_state, + unsigned rast_prim); -bool radv_nir_lower_intrinsics_early(nir_shader *nir, const struct radv_pipeline_key *key); +bool radv_nir_lower_intrinsics_early(nir_shader *nir, const struct radv_graphics_state_key *gfx_state); bool radv_nir_lower_view_index(nir_shader *nir, bool per_primitive); @@ -77,7 +78,7 @@ void radv_nir_lower_io(struct radv_device *device, nir_shader *nir); bool radv_nir_lower_io_to_mem(struct radv_device *device, struct radv_shader_stage *stage); -void radv_nir_lower_poly_line_smooth(nir_shader *nir, const struct radv_pipeline_key *key); +void radv_nir_lower_poly_line_smooth(nir_shader *nir, const struct radv_graphics_state_key *gfx_state); bool radv_nir_lower_cooperative_matrix(nir_shader *shader, unsigned wave_size); diff --git a/src/amd/vulkan/nir/radv_nir_lower_abi.c b/src/amd/vulkan/nir/radv_nir_lower_abi.c index 95ff340e0f9..1f536fe3264 100644 --- a/src/amd/vulkan/nir/radv_nir_lower_abi.c +++ b/src/amd/vulkan/nir/radv_nir_lower_abi.c @@ -37,7 +37,7 @@ typedef struct { enum amd_gfx_level gfx_level; const struct radv_shader_args *args; const struct radv_shader_info *info; - const struct radv_pipeline_key *pl_key; + const struct radv_graphics_state_key *gfx_state; uint32_t address32_hi; nir_def *gsvs_ring[4]; } lower_abi_state; @@ -157,8 +157,8 @@ lower_abi_instr(nir_builder *b, nir_intrinsic_instr *intrin, void *state) break; case nir_intrinsic_load_patch_vertices_in: if (stage == MESA_SHADER_TESS_CTRL) { - if (s->pl_key->ts.patch_control_points) { - replacement = nir_imm_int(b, s->pl_key->ts.patch_control_points); + if (s->gfx_state->ts.patch_control_points) { + replacement = nir_imm_int(b, s->gfx_state->ts.patch_control_points); } else { replacement = GET_SGPR_FIELD_NIR(s->args->tcs_offchip_layout, TCS_OFFCHIP_LAYOUT_PATCH_CONTROL_POINTS); } @@ -348,20 +348,20 @@ lower_abi_instr(nir_builder *b, nir_intrinsic_instr *intrin, void *state) break; } case nir_intrinsic_load_rasterization_samples_amd: - if (s->pl_key->dynamic_rasterization_samples) { + if (s->gfx_state->dynamic_rasterization_samples) { replacement = GET_SGPR_FIELD_NIR(s->args->ps_state, PS_STATE_NUM_SAMPLES); } else { - replacement = nir_imm_int(b, s->pl_key->ms.rasterization_samples); + replacement = nir_imm_int(b, s->gfx_state->ms.rasterization_samples); } break; case nir_intrinsic_load_provoking_vtx_in_prim_amd: { - if (s->pl_key->dynamic_provoking_vtx_mode) { + if (s->gfx_state->dynamic_provoking_vtx_mode) { replacement = ac_nir_load_arg(b, &s->args->ac, s->args->ngg_provoking_vtx); } else { unsigned provoking_vertex = 0; - if (s->pl_key->rs.provoking_vtx_last) { + if (s->gfx_state->rs.provoking_vtx_last) { if (stage == MESA_SHADER_VERTEX) { - provoking_vertex = radv_get_num_vertices_per_prim(s->pl_key) - 1; + provoking_vertex = radv_get_num_vertices_per_prim(s->gfx_state) - 1; } else if (stage == MESA_SHADER_GEOMETRY) { provoking_vertex = b->shader->info.gs.vertices_in - 1; } else { @@ -436,7 +436,7 @@ lower_abi_instr(nir_builder *b, nir_intrinsic_instr *intrin, void *state) if (s->info->vs.dynamic_num_verts_per_prim) { replacement = ac_nir_load_arg(b, &s->args->ac, s->args->num_verts_per_prim); } else { - replacement = nir_imm_int(b, radv_get_num_vertices_per_prim(s->pl_key)); + replacement = nir_imm_int(b, radv_get_num_vertices_per_prim(s->gfx_state)); } } else if (stage == MESA_SHADER_TESS_EVAL) { if (s->info->tes.point_mode) { @@ -485,11 +485,11 @@ lower_abi_instr(nir_builder *b, nir_intrinsic_instr *intrin, void *state) break; } case nir_intrinsic_load_poly_line_smooth_enabled: - if (s->pl_key->dynamic_line_rast_mode) { + if (s->gfx_state->dynamic_line_rast_mode) { nir_def *line_rast_mode = GET_SGPR_FIELD_NIR(s->args->ps_state, PS_STATE_LINE_RAST_MODE); replacement = nir_ieq_imm(b, line_rast_mode, VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT); } else { - replacement = nir_imm_bool(b, s->pl_key->rs.line_smooth_enabled); + replacement = nir_imm_bool(b, s->gfx_state->rs.line_smooth_enabled); } break; case nir_intrinsic_load_initial_edgeflags_amd: @@ -499,7 +499,7 @@ lower_abi_instr(nir_builder *b, nir_intrinsic_instr *intrin, void *state) replacement = ac_nir_load_arg(b, &s->args->ac, s->args->ac.load_provoking_vtx); break; case nir_intrinsic_load_rasterization_primitive_amd: - assert(s->pl_key->unknown_rast_prim); + assert(s->gfx_state->unknown_rast_prim); /* Load the primitive topology from an user SGPR when it's unknown at compile time (GPL). */ replacement = GET_SGPR_FIELD_NIR(s->args->ps_state, PS_STATE_RAST_PRIM); break; @@ -548,13 +548,13 @@ load_gsvs_ring(nir_builder *b, lower_abi_state *s, unsigned stream_id) void radv_nir_lower_abi(nir_shader *shader, enum amd_gfx_level gfx_level, const struct radv_shader_stage *stage, - const struct radv_pipeline_key *pl_key, uint32_t address32_hi) + const struct radv_graphics_state_key *gfx_state, uint32_t address32_hi) { lower_abi_state state = { .gfx_level = gfx_level, .info = &stage->info, .args = &stage->args, - .pl_key = pl_key, + .gfx_state = gfx_state, .address32_hi = address32_hi, }; diff --git a/src/amd/vulkan/nir/radv_nir_lower_fs_barycentric.c b/src/amd/vulkan/nir/radv_nir_lower_fs_barycentric.c index 41a86b5b856..50aa5aed32f 100644 --- a/src/amd/vulkan/nir/radv_nir_lower_fs_barycentric.c +++ b/src/amd/vulkan/nir/radv_nir_lower_fs_barycentric.c @@ -257,7 +257,7 @@ lower_load_barycentric_coord(nir_builder *b, lower_fs_barycentric_state *state, } bool -radv_nir_lower_fs_barycentric(nir_shader *shader, const struct radv_pipeline_key *key, unsigned rast_prim) +radv_nir_lower_fs_barycentric(nir_shader *shader, const struct radv_graphics_state_key *gfx_state, unsigned rast_prim) { nir_function_impl *impl = nir_shader_get_entrypoint(shader); bool progress = false; @@ -265,8 +265,8 @@ radv_nir_lower_fs_barycentric(nir_shader *shader, const struct radv_pipeline_key nir_builder b; lower_fs_barycentric_state state = { - .dynamic_rasterization_samples = key->dynamic_rasterization_samples, - .num_rasterization_samples = key->ms.rasterization_samples, + .dynamic_rasterization_samples = gfx_state->dynamic_rasterization_samples, + .num_rasterization_samples = gfx_state->ms.rasterization_samples, .rast_prim = rast_prim, }; diff --git a/src/amd/vulkan/nir/radv_nir_lower_fs_intrinsics.c b/src/amd/vulkan/nir/radv_nir_lower_fs_intrinsics.c index 8686972deb2..8eb1f98d9ff 100644 --- a/src/amd/vulkan/nir/radv_nir_lower_fs_intrinsics.c +++ b/src/amd/vulkan/nir/radv_nir_lower_fs_intrinsics.c @@ -30,7 +30,7 @@ bool radv_nir_lower_fs_intrinsics(nir_shader *nir, const struct radv_shader_stage *fs_stage, - const struct radv_pipeline_key *key) + const struct radv_graphics_state_key *gfx_state) { const struct radv_shader_info *info = &fs_stage->info; const struct radv_shader_args *args = &fs_stage->args; @@ -52,7 +52,7 @@ radv_nir_lower_fs_intrinsics(nir_shader *nir, const struct radv_shader_stage *fs nir_def *sample_coverage = nir_load_vector_arg_amd(&b, 1, .base = args->ac.sample_coverage.arg_index); nir_def *def = NULL; - if (info->ps.uses_sample_shading || key->ms.sample_shading_enable) { + if (info->ps.uses_sample_shading || gfx_state->ms.sample_shading_enable) { /* gl_SampleMaskIn[0] = (SampleCoverage & (PsIterMask << gl_SampleID)). */ nir_def *ps_state = nir_load_scalar_arg_amd(&b, 1, .base = args->ps_state.arg_index); nir_def *ps_iter_mask = @@ -70,7 +70,7 @@ radv_nir_lower_fs_intrinsics(nir_shader *nir, const struct radv_shader_stage *fs break; } case nir_intrinsic_load_frag_coord: { - if (!key->adjust_frag_coord_z) + if (!gfx_state->adjust_frag_coord_z) continue; if (!(nir_def_components_read(&intrin->def) & (1 << 2))) @@ -100,7 +100,7 @@ radv_nir_lower_fs_intrinsics(nir_shader *nir, const struct radv_shader_stage *fs nir_def *num_samples = nir_load_rasterization_samples_amd(&b); nir_def *new_dest; - if (key->dynamic_rasterization_samples) { + if (gfx_state->dynamic_rasterization_samples) { nir_def *res1, *res2; nir_push_if(&b, nir_ieq_imm(&b, num_samples, 1)); @@ -121,7 +121,7 @@ radv_nir_lower_fs_intrinsics(nir_shader *nir, const struct radv_shader_stage *fs new_dest = nir_if_phi(&b, res1, res2); } else { - if (!key->ms.rasterization_samples) { + if (!gfx_state->ms.rasterization_samples) { new_dest = nir_load_barycentric_pixel(&b, 32, .interp_mode = nir_intrinsic_interp_mode(intrin)); } else { nir_def *sample_pos = nir_load_sample_positions_amd(&b, 32, intrin->src[0].ssa, num_samples); diff --git a/src/amd/vulkan/nir/radv_nir_lower_intrinsics_early.c b/src/amd/vulkan/nir/radv_nir_lower_intrinsics_early.c index 6e33689741f..b7868bd2b9e 100644 --- a/src/amd/vulkan/nir/radv_nir_lower_intrinsics_early.c +++ b/src/amd/vulkan/nir/radv_nir_lower_intrinsics_early.c @@ -29,7 +29,7 @@ #include "radv_private.h" bool -radv_nir_lower_intrinsics_early(nir_shader *nir, const struct radv_pipeline_key *key) +radv_nir_lower_intrinsics_early(nir_shader *nir, const struct radv_graphics_state_key *gfx_state) { nir_function_impl *entry = nir_shader_get_entrypoint(nir); bool progress = false; @@ -52,7 +52,7 @@ radv_nir_lower_intrinsics_early(nir_shader *nir, const struct radv_pipeline_key def = nir_ior(&b, intrin->src[0].ssa, intrin->src[1].ssa); break; case nir_intrinsic_load_view_index: - if (key->has_multiview_view_index) + if (gfx_state->has_multiview_view_index) continue; def = nir_imm_zero(&b, 1, 32); break; diff --git a/src/amd/vulkan/nir/radv_nir_lower_poly_line_smooth.c b/src/amd/vulkan/nir/radv_nir_lower_poly_line_smooth.c index b9939e0938b..a1cc6a4e17d 100644 --- a/src/amd/vulkan/nir/radv_nir_lower_poly_line_smooth.c +++ b/src/amd/vulkan/nir/radv_nir_lower_poly_line_smooth.c @@ -27,11 +27,11 @@ #include "radv_private.h" static bool -radv_should_lower_poly_line_smooth(nir_shader *nir, const struct radv_pipeline_key *key) +radv_should_lower_poly_line_smooth(nir_shader *nir, const struct radv_graphics_state_key *gfx_state) { nir_function_impl *impl = nir_shader_get_entrypoint(nir); - if (!key->rs.line_smooth_enabled && !key->dynamic_line_rast_mode) + if (!gfx_state->rs.line_smooth_enabled && !gfx_state->dynamic_line_rast_mode) return false; nir_foreach_block (block, impl) { @@ -53,11 +53,11 @@ radv_should_lower_poly_line_smooth(nir_shader *nir, const struct radv_pipeline_k } void -radv_nir_lower_poly_line_smooth(nir_shader *nir, const struct radv_pipeline_key *key) +radv_nir_lower_poly_line_smooth(nir_shader *nir, const struct radv_graphics_state_key *gfx_state) { bool progress = false; - if (!radv_should_lower_poly_line_smooth(nir, key)) + if (!radv_should_lower_poly_line_smooth(nir, gfx_state)) return; NIR_PASS(progress, nir, nir_lower_poly_line_smooth, RADV_NUM_SMOOTH_AA_SAMPLES); diff --git a/src/amd/vulkan/nir/radv_nir_lower_vs_inputs.c b/src/amd/vulkan/nir/radv_nir_lower_vs_inputs.c index 11f06e3de24..79de5b44e44 100644 --- a/src/amd/vulkan/nir/radv_nir_lower_vs_inputs.c +++ b/src/amd/vulkan/nir/radv_nir_lower_vs_inputs.c @@ -33,7 +33,7 @@ typedef struct { const struct radv_shader_args *args; const struct radv_shader_info *info; - const struct radv_pipeline_key *pl_key; + const struct radv_graphics_state_key *gfx_state; const struct radeon_info *rad_info; } lower_vs_inputs_state; @@ -81,7 +81,7 @@ lower_load_vs_input_from_prolog(nir_builder *b, nir_intrinsic_instr *intrin, low static nir_def * calc_vs_input_index_instance_rate(nir_builder *b, unsigned location, lower_vs_inputs_state *s) { - const uint32_t divisor = s->pl_key->vi.instance_rate_divisors[location]; + const uint32_t divisor = s->gfx_state->vi.instance_rate_divisors[location]; nir_def *start_instance = nir_load_base_instance(b); if (divisor == 0) @@ -94,7 +94,7 @@ calc_vs_input_index_instance_rate(nir_builder *b, unsigned location, lower_vs_in static nir_def * calc_vs_input_index(nir_builder *b, unsigned location, lower_vs_inputs_state *s) { - if (s->pl_key->vi.instance_rate_inputs & BITFIELD_BIT(location)) + if (s->gfx_state->vi.instance_rate_inputs & BITFIELD_BIT(location)) return calc_vs_input_index_instance_rate(b, location, s); return nir_iadd(b, nir_load_first_vertex(b), nir_load_vertex_id_zero_base(b)); @@ -233,10 +233,10 @@ lower_load_vs_input(nir_builder *b, nir_intrinsic_instr *intrin, lower_vs_inputs if (!dest_use_mask) return nir_undef(b, dest_num_components, bit_size); - const uint32_t attrib_binding = s->pl_key->vi.vertex_attribute_bindings[location]; - const uint32_t attrib_offset = s->pl_key->vi.vertex_attribute_offsets[location]; - const uint32_t attrib_stride = s->pl_key->vi.vertex_attribute_strides[location]; - const enum pipe_format attrib_format = s->pl_key->vi.vertex_attribute_formats[location]; + const uint32_t attrib_binding = s->gfx_state->vi.vertex_attribute_bindings[location]; + const uint32_t attrib_offset = s->gfx_state->vi.vertex_attribute_offsets[location]; + const uint32_t attrib_stride = s->gfx_state->vi.vertex_attribute_strides[location]; + const enum pipe_format attrib_format = s->gfx_state->vi.vertex_attribute_formats[location]; const struct util_format_description *f = util_format_description(attrib_format); const struct ac_vtx_format_info *vtx_info = ac_get_vtx_format_info(s->rad_info->gfx_level, s->rad_info->family, attrib_format); @@ -328,7 +328,7 @@ lower_load_vs_input(nir_builder *b, nir_intrinsic_instr *intrin, lower_vs_inputs loads[num_loads++] = nir_load_buffer_amd(b, channels, bit_size, descriptor, zero, zero, index, .base = const_off, .memory_modes = nir_var_shader_in); } else { - const unsigned align_mul = MAX2(1, s->pl_key->vi.vertex_binding_align[attrib_binding]); + const unsigned align_mul = MAX2(1, s->gfx_state->vi.vertex_binding_align[attrib_binding]); const unsigned align_offset = const_off % align_mul; loads[num_loads++] = nir_load_typed_buffer_amd( @@ -411,14 +411,14 @@ lower_vs_input_instr(nir_builder *b, nir_intrinsic_instr *intrin, void *state) bool radv_nir_lower_vs_inputs(nir_shader *shader, const struct radv_shader_stage *vs_stage, - const struct radv_pipeline_key *pl_key, const struct radeon_info *rad_info) + const struct radv_graphics_state_key *gfx_state, const struct radeon_info *rad_info) { assert(shader->info.stage == MESA_SHADER_VERTEX); lower_vs_inputs_state state = { .info = &vs_stage->info, .args = &vs_stage->args, - .pl_key = pl_key, + .gfx_state = gfx_state, .rad_info = rad_info, }; diff --git a/src/amd/vulkan/radv_pipeline.c b/src/amd/vulkan/radv_pipeline.c index 8a9309ecd22..95b6a389b40 100644 --- a/src/amd/vulkan/radv_pipeline.c +++ b/src/amd/vulkan/radv_pipeline.c @@ -468,7 +468,7 @@ non_uniform_access_callback(const nir_src *src, void *_) } void -radv_postprocess_nir(struct radv_device *device, const struct radv_pipeline_key *pipeline_key, +radv_postprocess_nir(struct radv_device *device, const struct radv_graphics_state_key *gfx_state, struct radv_shader_stage *stage) { enum amd_gfx_level gfx_level = device->physical_device->rad_info.gfx_level; @@ -481,7 +481,7 @@ radv_postprocess_nir(struct radv_device *device, const struct radv_pipeline_key if (!stage->key.optimisations_disabled) { NIR_PASS(_, stage->nir, nir_opt_cse); } - NIR_PASS(_, stage->nir, radv_nir_lower_fs_intrinsics, stage, pipeline_key); + NIR_PASS(_, stage->nir, radv_nir_lower_fs_intrinsics, stage, gfx_state); } enum nir_lower_non_uniform_access_type lower_non_uniform_access_types = @@ -591,7 +591,7 @@ radv_postprocess_nir(struct radv_device *device, const struct radv_pipeline_key * load_input can be reordered, but buffer loads can't. */ if (stage->stage == MESA_SHADER_VERTEX) { - NIR_PASS(_, stage->nir, radv_nir_lower_vs_inputs, stage, pipeline_key, &device->physical_device->rad_info); + NIR_PASS(_, stage->nir, radv_nir_lower_vs_inputs, stage, gfx_state, &device->physical_device->rad_info); } /* Lower I/O intrinsics to memory instructions. */ @@ -599,7 +599,7 @@ radv_postprocess_nir(struct radv_device *device, const struct radv_pipeline_key bool io_to_mem = radv_nir_lower_io_to_mem(device, stage); bool lowered_ngg = stage->info.is_ngg && is_last_vgt_stage; if (lowered_ngg) { - radv_lower_ngg(device, stage, pipeline_key); + radv_lower_ngg(device, stage, gfx_state); } else if (is_last_vgt_stage) { if (stage->stage != MESA_SHADER_GEOMETRY) { NIR_PASS_V(stage->nir, ac_nir_lower_legacy_vs, gfx_level, @@ -633,17 +633,17 @@ radv_postprocess_nir(struct radv_device *device, const struct radv_pipeline_key }; if (!options.no_color_export) { - options.dual_src_blend_swizzle = pipeline_key->ps.epilog.mrt0_is_dual_src && gfx_level >= GFX11; - options.color_is_int8 = pipeline_key->ps.epilog.color_is_int8; - options.color_is_int10 = pipeline_key->ps.epilog.color_is_int10; + options.dual_src_blend_swizzle = gfx_state->ps.epilog.mrt0_is_dual_src && gfx_level >= GFX11; + options.color_is_int8 = gfx_state->ps.epilog.color_is_int8; + options.color_is_int10 = gfx_state->ps.epilog.color_is_int10; options.enable_mrt_output_nan_fixup = - pipeline_key->ps.epilog.enable_mrt_output_nan_fixup && !stage->nir->info.internal; + gfx_state->ps.epilog.enable_mrt_output_nan_fixup && !stage->nir->info.internal; /* Need to filter out unwritten color slots. */ - options.spi_shader_col_format = pipeline_key->ps.epilog.spi_shader_col_format & stage->info.ps.colors_written; + options.spi_shader_col_format = gfx_state->ps.epilog.spi_shader_col_format & stage->info.ps.colors_written; } if (!options.no_depth_export) { - /* Compared to radv_pipeline_key.ps.alpha_to_coverage_via_mrtz, + /* Compared to gfx_state.ps.alpha_to_coverage_via_mrtz, * radv_shader_info.ps.writes_mrt0_alpha need any depth/stencil/sample_mask exist. * ac_nir_lower_ps() require this field to reflect whether alpha via mrtz is really * present. @@ -675,7 +675,7 @@ radv_postprocess_nir(struct radv_device *device, const struct radv_pipeline_key NIR_PASS(_, stage->nir, ac_nir_lower_global_access); NIR_PASS_V(stage->nir, ac_nir_lower_intrinsics_to_args, gfx_level, radv_select_hw_stage(&stage->info, gfx_level), &stage->args.ac); - NIR_PASS_V(stage->nir, radv_nir_lower_abi, gfx_level, stage, pipeline_key, + NIR_PASS_V(stage->nir, radv_nir_lower_abi, gfx_level, stage, gfx_state, device->physical_device->rad_info.address32_hi); radv_optimize_nir_algebraic( stage->nir, io_to_mem || lowered_ngg || stage->stage == MESA_SHADER_COMPUTE || stage->stage == MESA_SHADER_TASK); diff --git a/src/amd/vulkan/radv_pipeline_graphics.c b/src/amd/vulkan/radv_pipeline_graphics.c index f385bf703de..f7e1711a6a1 100644 --- a/src/amd/vulkan/radv_pipeline_graphics.c +++ b/src/amd/vulkan/radv_pipeline_graphics.c @@ -1176,19 +1176,19 @@ get_vs_output_info(const struct radv_graphics_pipeline *pipeline) } static bool -radv_should_export_multiview(const struct radv_shader_stage *stage, const struct radv_pipeline_key *pipeline_key) +radv_should_export_multiview(const struct radv_shader_stage *stage, const struct radv_graphics_state_key *gfx_state) { /* Export the layer in the last VGT stage if multiview is used. When the next stage is unknown * (with graphics pipeline library), the layer is exported unconditionally. */ - return pipeline_key->has_multiview_view_index && + return gfx_state->has_multiview_view_index && (stage->info.next_stage == MESA_SHADER_FRAGMENT || - !(pipeline_key->lib_flags & VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT)) && + !(gfx_state->lib_flags & VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT)) && !(stage->nir->info.outputs_written & VARYING_BIT_LAYER); } static void -radv_remove_point_size(const struct radv_pipeline_key *pipeline_key, nir_shader *producer, nir_shader *consumer) +radv_remove_point_size(const struct radv_graphics_state_key *gfx_state, nir_shader *producer, nir_shader *consumer) { if ((consumer->info.inputs_read & VARYING_BIT_PSIZ) || !(producer->info.outputs_written & VARYING_BIT_PSIZ)) return; @@ -1218,13 +1218,13 @@ radv_remove_point_size(const struct radv_pipeline_key *pipeline_key, nir_shader } static void -radv_remove_color_exports(const struct radv_pipeline_key *pipeline_key, nir_shader *nir) +radv_remove_color_exports(const struct radv_graphics_state_key *gfx_state, nir_shader *nir) { bool fixup_derefs = false; /* Do not remove color exports when a PS epilog is used because the format isn't known and the color write mask can * be dynamic. */ - if (pipeline_key->ps.has_epilog) + if (gfx_state->ps.has_epilog) return; nir_foreach_shader_out_variable (var, nir) { @@ -1234,7 +1234,7 @@ radv_remove_color_exports(const struct radv_pipeline_key *pipeline_key, nir_shad if (idx < 0) continue; - unsigned col_format = (pipeline_key->ps.epilog.spi_shader_col_format >> (4 * idx)) & 0xf; + unsigned col_format = (gfx_state->ps.epilog.spi_shader_col_format >> (4 * idx)) & 0xf; if (col_format == V_028714_SPI_SHADER_ZERO) { /* Remove the color export if it's unused or in presence of holes. */ @@ -1294,7 +1294,7 @@ merge_tess_info(struct shader_info *tes_info, struct shader_info *tcs_info) static void radv_link_shaders(const struct radv_device *device, struct radv_shader_stage *producer_stage, - struct radv_shader_stage *consumer_stage, const struct radv_pipeline_key *pipeline_key) + struct radv_shader_stage *consumer_stage, const struct radv_graphics_state_key *gfx_state) { const enum amd_gfx_level gfx_level = device->physical_device->rad_info.gfx_level; nir_shader *producer = producer_stage->nir; @@ -1336,8 +1336,8 @@ radv_link_shaders(const struct radv_device *device, struct radv_shader_stage *pr /* Remove PSIZ from shaders when it's not needed. * This is typically produced by translation layers like Zink or D9VK. */ - if (pipeline_key->enable_remove_point_size) - radv_remove_point_size(pipeline_key, producer, consumer); + if (gfx_state->enable_remove_point_size) + radv_remove_point_size(gfx_state, producer, consumer); if (nir_link_opt_varyings(producer, consumer)) { nir_validate_shader(producer, "after nir_link_opt_varyings"); @@ -1418,11 +1418,11 @@ static const gl_shader_stage graphics_shader_order[] = { static void radv_link_vs(const struct radv_device *device, struct radv_shader_stage *vs_stage, struct radv_shader_stage *next_stage, - const struct radv_pipeline_key *pipeline_key) + const struct radv_graphics_state_key *gfx_state) { assert(vs_stage->nir->info.stage == MESA_SHADER_VERTEX); - if (radv_should_export_multiview(vs_stage, pipeline_key)) { + if (radv_should_export_multiview(vs_stage, gfx_state)) { NIR_PASS(_, vs_stage->nir, radv_nir_export_multiview); } @@ -1431,7 +1431,7 @@ radv_link_vs(const struct radv_device *device, struct radv_shader_stage *vs_stag next_stage->nir->info.stage == MESA_SHADER_GEOMETRY || next_stage->nir->info.stage == MESA_SHADER_FRAGMENT); - radv_link_shaders(device, vs_stage, next_stage, pipeline_key); + radv_link_shaders(device, vs_stage, next_stage, gfx_state); } nir_foreach_shader_in_variable (var, vs_stage->nir) { @@ -1463,7 +1463,7 @@ radv_link_vs(const struct radv_device *device, struct radv_shader_stage *vs_stag static void radv_link_tcs(const struct radv_device *device, struct radv_shader_stage *tcs_stage, - struct radv_shader_stage *tes_stage, const struct radv_pipeline_key *pipeline_key) + struct radv_shader_stage *tes_stage, const struct radv_graphics_state_key *gfx_state) { if (!tes_stage) return; @@ -1471,7 +1471,7 @@ radv_link_tcs(const struct radv_device *device, struct radv_shader_stage *tcs_st assert(tcs_stage->nir->info.stage == MESA_SHADER_TESS_CTRL); assert(tes_stage->nir->info.stage == MESA_SHADER_TESS_EVAL); - radv_link_shaders(device, tcs_stage, tes_stage, pipeline_key); + radv_link_shaders(device, tcs_stage, tes_stage, gfx_state); /* Copy TCS info into the TES info */ merge_tess_info(&tes_stage->nir->info, &tcs_stage->nir->info); @@ -1488,11 +1488,11 @@ radv_link_tcs(const struct radv_device *device, struct radv_shader_stage *tcs_st static void radv_link_tes(const struct radv_device *device, struct radv_shader_stage *tes_stage, - struct radv_shader_stage *next_stage, const struct radv_pipeline_key *pipeline_key) + struct radv_shader_stage *next_stage, const struct radv_graphics_state_key *gfx_state) { assert(tes_stage->nir->info.stage == MESA_SHADER_TESS_EVAL); - if (radv_should_export_multiview(tes_stage, pipeline_key)) { + if (radv_should_export_multiview(tes_stage, gfx_state)) { NIR_PASS(_, tes_stage->nir, radv_nir_export_multiview); } @@ -1500,7 +1500,7 @@ radv_link_tes(const struct radv_device *device, struct radv_shader_stage *tes_st assert(next_stage->nir->info.stage == MESA_SHADER_GEOMETRY || next_stage->nir->info.stage == MESA_SHADER_FRAGMENT); - radv_link_shaders(device, tes_stage, next_stage, pipeline_key); + radv_link_shaders(device, tes_stage, next_stage, gfx_state); } if (next_stage && next_stage->nir->info.stage == MESA_SHADER_GEOMETRY) { @@ -1520,18 +1520,18 @@ radv_link_tes(const struct radv_device *device, struct radv_shader_stage *tes_st static void radv_link_gs(const struct radv_device *device, struct radv_shader_stage *gs_stage, struct radv_shader_stage *fs_stage, - const struct radv_pipeline_key *pipeline_key) + const struct radv_graphics_state_key *gfx_state) { assert(gs_stage->nir->info.stage == MESA_SHADER_GEOMETRY); - if (radv_should_export_multiview(gs_stage, pipeline_key)) { + if (radv_should_export_multiview(gs_stage, gfx_state)) { NIR_PASS(_, gs_stage->nir, radv_nir_export_multiview); } if (fs_stage) { assert(fs_stage->nir->info.stage == MESA_SHADER_FRAGMENT); - radv_link_shaders(device, gs_stage, fs_stage, pipeline_key); + radv_link_shaders(device, gs_stage, fs_stage, gfx_state); } nir_foreach_shader_out_variable (var, gs_stage->nir) { @@ -1541,18 +1541,18 @@ radv_link_gs(const struct radv_device *device, struct radv_shader_stage *gs_stag static void radv_link_task(const struct radv_device *device, struct radv_shader_stage *task_stage, - struct radv_shader_stage *mesh_stage, const struct radv_pipeline_key *pipeline_key) + struct radv_shader_stage *mesh_stage, const struct radv_graphics_state_key *gfx_state) { assert(task_stage->nir->info.stage == MESA_SHADER_TASK); assert(mesh_stage->nir->info.stage == MESA_SHADER_MESH); /* Linking task and mesh shaders shouldn't do anything for now but keep it for consistency. */ - radv_link_shaders(device, task_stage, mesh_stage, pipeline_key); + radv_link_shaders(device, task_stage, mesh_stage, gfx_state); } static void radv_link_mesh(const struct radv_device *device, struct radv_shader_stage *mesh_stage, - struct radv_shader_stage *fs_stage, const struct radv_pipeline_key *pipeline_key) + struct radv_shader_stage *fs_stage, const struct radv_graphics_state_key *gfx_state) { assert(mesh_stage->nir->info.stage == MESA_SHADER_MESH); @@ -1567,7 +1567,7 @@ radv_link_mesh(const struct radv_device *device, struct radv_shader_stage *mesh_ } } - radv_link_shaders(device, mesh_stage, fs_stage, pipeline_key); + radv_link_shaders(device, mesh_stage, fs_stage, gfx_state); } /* ac_nir_lower_ngg ignores driver locations for mesh shaders, but set them to all zero just to @@ -1579,11 +1579,11 @@ radv_link_mesh(const struct radv_device *device, struct radv_shader_stage *mesh_ } static void -radv_link_fs(struct radv_shader_stage *fs_stage, const struct radv_pipeline_key *pipeline_key) +radv_link_fs(struct radv_shader_stage *fs_stage, const struct radv_graphics_state_key *gfx_state) { assert(fs_stage->nir->info.stage == MESA_SHADER_FRAGMENT); - radv_remove_color_exports(pipeline_key, fs_stage->nir); + radv_remove_color_exports(gfx_state, fs_stage->nir); nir_foreach_shader_out_variable (var, fs_stage->nir) { var->data.driver_location = var->data.location + var->data.index; @@ -1591,14 +1591,14 @@ radv_link_fs(struct radv_shader_stage *fs_stage, const struct radv_pipeline_key } static bool -radv_pipeline_needs_noop_fs(struct radv_graphics_pipeline *pipeline, const struct radv_pipeline_key *pipeline_key) +radv_pipeline_needs_noop_fs(struct radv_graphics_pipeline *pipeline, const struct radv_graphics_state_key *gfx_state) { if (pipeline->base.type == RADV_PIPELINE_GRAPHICS && !(radv_pipeline_to_graphics(&pipeline->base)->active_stages & VK_SHADER_STAGE_FRAGMENT_BIT)) return true; if (pipeline->base.type == RADV_PIPELINE_GRAPHICS_LIB && - (pipeline_key->lib_flags & VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT) && + (gfx_state->lib_flags & VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT) && !(radv_pipeline_to_graphics_lib(&pipeline->base)->base.active_stages & VK_SHADER_STAGE_FRAGMENT_BIT)) return true; @@ -1637,7 +1637,7 @@ radv_remove_varyings(nir_shader *nir) } static void -radv_graphics_shaders_link(const struct radv_device *device, const struct radv_pipeline_key *pipeline_key, +radv_graphics_shaders_link(const struct radv_device *device, const struct radv_graphics_state_key *gfx_state, struct radv_shader_stage *stages) { /* Walk backwards to link */ @@ -1649,25 +1649,25 @@ radv_graphics_shaders_link(const struct radv_device *device, const struct radv_p switch (s) { case MESA_SHADER_VERTEX: - radv_link_vs(device, &stages[s], next_stage, pipeline_key); + radv_link_vs(device, &stages[s], next_stage, gfx_state); break; case MESA_SHADER_TESS_CTRL: - radv_link_tcs(device, &stages[s], next_stage, pipeline_key); + radv_link_tcs(device, &stages[s], next_stage, gfx_state); break; case MESA_SHADER_TESS_EVAL: - radv_link_tes(device, &stages[s], next_stage, pipeline_key); + radv_link_tes(device, &stages[s], next_stage, gfx_state); break; case MESA_SHADER_GEOMETRY: - radv_link_gs(device, &stages[s], next_stage, pipeline_key); + radv_link_gs(device, &stages[s], next_stage, gfx_state); break; case MESA_SHADER_TASK: - radv_link_task(device, &stages[s], next_stage, pipeline_key); + radv_link_task(device, &stages[s], next_stage, gfx_state); break; case MESA_SHADER_MESH: - radv_link_mesh(device, &stages[s], next_stage, pipeline_key); + radv_link_mesh(device, &stages[s], next_stage, gfx_state); break; case MESA_SHADER_FRAGMENT: - radv_link_fs(&stages[s], pipeline_key); + radv_link_fs(&stages[s], gfx_state); break; default: unreachable("Invalid graphics shader stage"); @@ -1796,17 +1796,15 @@ radv_pipeline_generate_ps_epilog_key(const struct radv_device *device, const str return radv_generate_ps_epilog_key(device, &ps_epilog); } -static struct radv_pipeline_key -radv_generate_graphics_pipeline_key(const struct radv_device *device, const struct radv_graphics_pipeline *pipeline, - const VkGraphicsPipelineCreateInfo *pCreateInfo, - const struct vk_graphics_pipeline_state *state, - VkGraphicsPipelineLibraryFlagBitsEXT lib_flags) +static struct radv_graphics_state_key +radv_generate_graphics_state_key(const struct radv_device *device, const struct radv_graphics_pipeline *pipeline, + const struct vk_graphics_pipeline_state *state, + VkGraphicsPipelineLibraryFlagBitsEXT lib_flags) { const struct radv_physical_device *pdevice = device->physical_device; - struct radv_pipeline_key key = radv_generate_pipeline_key(device, pCreateInfo->pStages, pCreateInfo->stageCount, - pipeline->base.create_flags, pCreateInfo->pNext); + struct radv_graphics_state_key key; - key.shader_version = device->instance->drirc.override_graphics_shader_version; + memset(&key, 0, sizeof(key)); key.lib_flags = lib_flags; key.has_multiview_view_index = state->rp ? !!state->rp->view_mask : 0; @@ -1964,6 +1962,22 @@ radv_generate_graphics_pipeline_key(const struct radv_device *device, const stru return key; } +static struct radv_pipeline_key +radv_generate_graphics_pipeline_key(const struct radv_device *device, const struct radv_graphics_pipeline *pipeline, + const VkGraphicsPipelineCreateInfo *pCreateInfo, + const struct vk_graphics_pipeline_state *state, + VkGraphicsPipelineLibraryFlagBitsEXT lib_flags) +{ + struct radv_pipeline_key key = radv_generate_pipeline_key(device, pCreateInfo->pStages, pCreateInfo->stageCount, + pipeline->base.create_flags, pCreateInfo->pNext); + + key.shader_version = device->instance->drirc.override_graphics_shader_version; + + key.gfx_state = radv_generate_graphics_state_key(device, pipeline, state, lib_flags); + + return key; +} + static void radv_fill_shader_info_ngg(struct radv_device *device, struct radv_shader_stage *stages, VkShaderStageFlagBits active_nir_stages) @@ -2020,10 +2034,10 @@ radv_fill_shader_info_ngg(struct radv_device *device, struct radv_shader_stage * } static bool -radv_consider_force_vrs(const struct radv_pipeline_key *pipeline_key, const struct radv_shader_stage *last_vgt_stage, +radv_consider_force_vrs(const struct radv_graphics_state_key *gfx_state, const struct radv_shader_stage *last_vgt_stage, const struct radv_shader_stage *fs_stage) { - if (!pipeline_key->ps.force_vrs_enabled) + if (!gfx_state->ps.force_vrs_enabled) return false; /* Mesh shaders aren't considered. */ @@ -2093,7 +2107,7 @@ radv_get_next_stage(gl_shader_stage stage, VkShaderStageFlagBits active_nir_stag static void radv_fill_shader_info(struct radv_device *device, const enum radv_pipeline_type pipeline_type, - const struct radv_pipeline_key *pipeline_key, struct radv_shader_stage *stages, + const struct radv_graphics_state_key *gfx_state, struct radv_shader_stage *stages, VkShaderStageFlagBits active_nir_stages) { radv_foreach_stage(i, active_nir_stages) @@ -2101,24 +2115,24 @@ radv_fill_shader_info(struct radv_device *device, const enum radv_pipeline_type bool consider_force_vrs = false; if (radv_is_last_vgt_stage(&stages[i])) { - consider_force_vrs = radv_consider_force_vrs(pipeline_key, &stages[i], &stages[MESA_SHADER_FRAGMENT]); + consider_force_vrs = radv_consider_force_vrs(gfx_state, &stages[i], &stages[MESA_SHADER_FRAGMENT]); } - radv_nir_shader_info_pass(device, stages[i].nir, &stages[i].layout, &stages[i].key, pipeline_key, pipeline_type, + radv_nir_shader_info_pass(device, stages[i].nir, &stages[i].layout, &stages[i].key, gfx_state, pipeline_type, consider_force_vrs, &stages[i].info); } - radv_nir_shader_info_link(device, pipeline_key, stages); + radv_nir_shader_info_link(device, gfx_state, stages); } static void radv_declare_pipeline_args(struct radv_device *device, struct radv_shader_stage *stages, - const struct radv_pipeline_key *pipeline_key, VkShaderStageFlagBits active_nir_stages) + const struct radv_graphics_state_key *gfx_state, VkShaderStageFlagBits active_nir_stages) { enum amd_gfx_level gfx_level = device->physical_device->rad_info.gfx_level; if (gfx_level >= GFX9 && stages[MESA_SHADER_TESS_CTRL].nir) { - radv_declare_shader_args(device, pipeline_key, &stages[MESA_SHADER_TESS_CTRL].info, MESA_SHADER_TESS_CTRL, + radv_declare_shader_args(device, gfx_state, &stages[MESA_SHADER_TESS_CTRL].info, MESA_SHADER_TESS_CTRL, MESA_SHADER_VERTEX, &stages[MESA_SHADER_TESS_CTRL].args); stages[MESA_SHADER_TESS_CTRL].info.user_sgprs_locs = stages[MESA_SHADER_TESS_CTRL].args.user_sgprs_locs; stages[MESA_SHADER_TESS_CTRL].info.inline_push_constant_mask = @@ -2135,8 +2149,8 @@ radv_declare_pipeline_args(struct radv_device *device, struct radv_shader_stage if (gfx_level >= GFX9 && stages[MESA_SHADER_GEOMETRY].nir) { gl_shader_stage pre_stage = stages[MESA_SHADER_TESS_EVAL].nir ? MESA_SHADER_TESS_EVAL : MESA_SHADER_VERTEX; - radv_declare_shader_args(device, pipeline_key, &stages[MESA_SHADER_GEOMETRY].info, MESA_SHADER_GEOMETRY, - pre_stage, &stages[MESA_SHADER_GEOMETRY].args); + radv_declare_shader_args(device, gfx_state, &stages[MESA_SHADER_GEOMETRY].info, MESA_SHADER_GEOMETRY, pre_stage, + &stages[MESA_SHADER_GEOMETRY].args); stages[MESA_SHADER_GEOMETRY].info.user_sgprs_locs = stages[MESA_SHADER_GEOMETRY].args.user_sgprs_locs; stages[MESA_SHADER_GEOMETRY].info.inline_push_constant_mask = stages[MESA_SHADER_GEOMETRY].args.ac.inline_push_const_mask; @@ -2149,7 +2163,7 @@ radv_declare_pipeline_args(struct radv_device *device, struct radv_shader_stage } u_foreach_bit (i, active_nir_stages) { - radv_declare_shader_args(device, pipeline_key, &stages[i].info, i, MESA_SHADER_NONE, &stages[i].args); + radv_declare_shader_args(device, gfx_state, &stages[i].info, i, MESA_SHADER_NONE, &stages[i].args); stages[i].info.user_sgprs_locs = stages[i].args.user_sgprs_locs; stages[i].info.inline_push_constant_mask = stages[i].args.ac.inline_push_const_mask; } @@ -2157,7 +2171,7 @@ radv_declare_pipeline_args(struct radv_device *device, struct radv_shader_stage static struct radv_shader * radv_create_gs_copy_shader(struct radv_device *device, struct vk_pipeline_cache *cache, - struct radv_shader_stage *gs_stage, const struct radv_pipeline_key *pipeline_key, + struct radv_shader_stage *gs_stage, const struct radv_graphics_state_key *gfx_state, bool keep_executable_info, bool keep_statistic_info, struct radv_shader_binary **gs_copy_binary) { @@ -2183,8 +2197,8 @@ radv_create_gs_copy_shader(struct radv_device *device, struct vk_pipeline_cache }, }; radv_nir_shader_info_init(gs_copy_stage.stage, MESA_SHADER_FRAGMENT, &gs_copy_stage.info); - radv_nir_shader_info_pass(device, nir, &gs_stage->layout, &gs_stage->key, pipeline_key, RADV_PIPELINE_GRAPHICS, - false, &gs_copy_stage.info); + radv_nir_shader_info_pass(device, nir, &gs_stage->layout, &gs_stage->key, gfx_state, RADV_PIPELINE_GRAPHICS, false, + &gs_copy_stage.info); gs_copy_stage.info.wave_size = 64; /* Wave32 not supported. */ gs_copy_stage.info.workgroup_size = 64; /* HW VS: separate waves, no workgroups */ gs_copy_stage.info.so = gs_info->so; @@ -2192,21 +2206,21 @@ radv_create_gs_copy_shader(struct radv_device *device, struct vk_pipeline_cache gs_copy_stage.info.force_vrs_per_vertex = gs_info->force_vrs_per_vertex; gs_copy_stage.info.type = RADV_SHADER_TYPE_GS_COPY; - radv_declare_shader_args(device, pipeline_key, &gs_copy_stage.info, MESA_SHADER_VERTEX, MESA_SHADER_NONE, + radv_declare_shader_args(device, gfx_state, &gs_copy_stage.info, MESA_SHADER_VERTEX, MESA_SHADER_NONE, &gs_copy_stage.args); gs_copy_stage.info.user_sgprs_locs = gs_copy_stage.args.user_sgprs_locs; gs_copy_stage.info.inline_push_constant_mask = gs_copy_stage.args.ac.inline_push_const_mask; NIR_PASS_V(nir, ac_nir_lower_intrinsics_to_args, device->physical_device->rad_info.gfx_level, AC_HW_VERTEX_SHADER, &gs_copy_stage.args.ac); - NIR_PASS_V(nir, radv_nir_lower_abi, device->physical_device->rad_info.gfx_level, &gs_copy_stage, pipeline_key, + NIR_PASS_V(nir, radv_nir_lower_abi, device->physical_device->rad_info.gfx_level, &gs_copy_stage, gfx_state, device->physical_device->rad_info.address32_hi); struct radv_pipeline_key key = {0}; bool dump_shader = radv_can_dump_shader(device, nir, true); - *gs_copy_binary = - radv_shader_nir_to_asm(device, &gs_copy_stage, &nir, 1, &key, keep_executable_info, keep_statistic_info); + *gs_copy_binary = radv_shader_nir_to_asm(device, &gs_copy_stage, &nir, 1, &key.gfx_state, keep_executable_info, + keep_statistic_info); struct radv_shader *copy_shader = radv_shader_create(device, cache, *gs_copy_binary, keep_executable_info || dump_shader); if (copy_shader) @@ -2217,7 +2231,7 @@ radv_create_gs_copy_shader(struct radv_device *device, struct vk_pipeline_cache static void radv_graphics_shaders_nir_to_asm(struct radv_device *device, struct vk_pipeline_cache *cache, - struct radv_shader_stage *stages, const struct radv_pipeline_key *pipeline_key, + struct radv_shader_stage *stages, const struct radv_graphics_state_key *gfx_state, bool keep_executable_info, bool keep_statistic_info, VkShaderStageFlagBits active_nir_stages, struct radv_shader **shaders, struct radv_shader_binary **binaries, struct radv_shader **gs_copy_shader, @@ -2250,14 +2264,14 @@ radv_graphics_shaders_nir_to_asm(struct radv_device *device, struct vk_pipeline_ bool dump_shader = radv_can_dump_shader(device, nir_shaders[0], false); - binaries[s] = radv_shader_nir_to_asm(device, &stages[s], nir_shaders, shader_count, pipeline_key, + binaries[s] = radv_shader_nir_to_asm(device, &stages[s], nir_shaders, shader_count, gfx_state, keep_executable_info, keep_statistic_info); shaders[s] = radv_shader_create(device, cache, binaries[s], keep_executable_info || dump_shader); radv_shader_generate_debug_info(device, dump_shader, keep_executable_info, binaries[s], shaders[s], nir_shaders, shader_count, &stages[s].info); if (s == MESA_SHADER_GEOMETRY && !stages[s].info.is_ngg) { - *gs_copy_shader = radv_create_gs_copy_shader(device, cache, &stages[MESA_SHADER_GEOMETRY], pipeline_key, + *gs_copy_shader = radv_create_gs_copy_shader(device, cache, &stages[MESA_SHADER_GEOMETRY], gfx_state, keep_executable_info, keep_statistic_info, gs_copy_binary); } @@ -2364,11 +2378,11 @@ radv_pipeline_load_retained_shaders(const struct radv_device *device, struct rad } static unsigned -radv_get_rasterization_prim(const struct radv_shader_stage *stages, const struct radv_pipeline_key *pipeline_key) +radv_get_rasterization_prim(const struct radv_shader_stage *stages, const struct radv_graphics_state_key *gfx_state) { unsigned rast_prim; - if (pipeline_key->unknown_rast_prim) + if (gfx_state->unknown_rast_prim) return -1; if (stages[MESA_SHADER_GEOMETRY].nir) { @@ -2382,7 +2396,7 @@ radv_get_rasterization_prim(const struct radv_shader_stage *stages, const struct } else if (stages[MESA_SHADER_MESH].nir) { rast_prim = radv_conv_gl_prim_to_gs_out(stages[MESA_SHADER_MESH].nir->info.mesh.primitive_type); } else { - rast_prim = radv_conv_prim_to_gs_out(pipeline_key->ia.topology, false); + rast_prim = radv_conv_prim_to_gs_out(gfx_state->ia.topology, false); } return rast_prim; @@ -2434,7 +2448,7 @@ radv_skip_graphics_pipeline_compile(const struct radv_device *device, const stru static void radv_graphics_shaders_compile(struct radv_device *device, struct vk_pipeline_cache *cache, - struct radv_shader_stage *stages, const struct radv_pipeline_key *pipeline_key, + struct radv_shader_stage *stages, const struct radv_graphics_state_key *gfx_state, bool keep_executable_info, bool keep_statistic_info, bool is_internal, struct radv_retained_shaders *retained_shaders, bool noop_fs, struct radv_shader **shaders, struct radv_shader_binary **binaries, @@ -2448,7 +2462,7 @@ radv_graphics_shaders_compile(struct radv_device *device, struct vk_pipeline_cac /* NIR might already have been imported from a library. */ if (!stages[s].nir) { - stages[s].nir = radv_shader_spirv_to_nir(device, &stages[s], pipeline_key, is_internal); + stages[s].nir = radv_shader_spirv_to_nir(device, &stages[s], gfx_state, is_internal); } stages[s].feedback.duration += os_time_get_nano() - stage_start; @@ -2516,12 +2530,12 @@ radv_graphics_shaders_compile(struct radv_device *device, struct vk_pipeline_cac } } - radv_graphics_shaders_link(device, pipeline_key, stages); + radv_graphics_shaders_link(device, gfx_state, stages); if (stages[MESA_SHADER_FRAGMENT].nir) { - unsigned rast_prim = radv_get_rasterization_prim(stages, pipeline_key); + unsigned rast_prim = radv_get_rasterization_prim(stages, gfx_state); - NIR_PASS(_, stages[MESA_SHADER_FRAGMENT].nir, radv_nir_lower_fs_barycentric, pipeline_key, rast_prim); + NIR_PASS(_, stages[MESA_SHADER_FRAGMENT].nir, radv_nir_lower_fs_barycentric, gfx_state, rast_prim); } radv_foreach_stage(i, active_nir_stages) @@ -2538,18 +2552,18 @@ radv_graphics_shaders_compile(struct radv_device *device, struct vk_pipeline_cac } if (stages[MESA_SHADER_FRAGMENT].nir) { - radv_nir_lower_poly_line_smooth(stages[MESA_SHADER_FRAGMENT].nir, pipeline_key); + radv_nir_lower_poly_line_smooth(stages[MESA_SHADER_FRAGMENT].nir, gfx_state); } - radv_fill_shader_info(device, RADV_PIPELINE_GRAPHICS, pipeline_key, stages, active_nir_stages); + radv_fill_shader_info(device, RADV_PIPELINE_GRAPHICS, gfx_state, stages, active_nir_stages); - radv_declare_pipeline_args(device, stages, pipeline_key, active_nir_stages); + radv_declare_pipeline_args(device, stages, gfx_state, active_nir_stages); radv_foreach_stage(i, active_nir_stages) { int64_t stage_start = os_time_get_nano(); - radv_postprocess_nir(device, pipeline_key, &stages[i]); + radv_postprocess_nir(device, gfx_state, &stages[i]); stages[i].feedback.duration += os_time_get_nano() - stage_start; @@ -2558,7 +2572,7 @@ radv_graphics_shaders_compile(struct radv_device *device, struct vk_pipeline_cac } /* Compile NIR shaders to AMD assembly. */ - radv_graphics_shaders_nir_to_asm(device, cache, stages, pipeline_key, keep_executable_info, keep_statistic_info, + radv_graphics_shaders_nir_to_asm(device, cache, stages, gfx_state, keep_executable_info, keep_statistic_info, active_nir_stages, shaders, binaries, gs_copy_shader, gs_copy_binary); if (keep_executable_info) { @@ -2690,11 +2704,11 @@ radv_graphics_pipeline_compile(struct radv_graphics_pipeline *pipeline, const Vk retained_shaders = &gfx_pipeline_lib->retained_shaders; } - const bool noop_fs = radv_pipeline_needs_noop_fs(pipeline, pipeline_key); + const bool noop_fs = radv_pipeline_needs_noop_fs(pipeline, &pipeline_key->gfx_state); - radv_graphics_shaders_compile(device, cache, stages, pipeline_key, keep_executable_info, keep_statistic_info, - pipeline->base.is_internal, retained_shaders, noop_fs, pipeline->base.shaders, - binaries, &pipeline->base.gs_copy_shader, &gs_copy_binary); + radv_graphics_shaders_compile(device, cache, stages, &pipeline_key->gfx_state, keep_executable_info, + keep_statistic_info, pipeline->base.is_internal, retained_shaders, noop_fs, + pipeline->base.shaders, binaries, &pipeline->base.gs_copy_shader, &gs_copy_binary); if (!skip_shaders_cache) { radv_pipeline_cache_insert(device, cache, &pipeline->base, hash); diff --git a/src/amd/vulkan/radv_private.h b/src/amd/vulkan/radv_private.h index c3b66a2d67b..c51a600d5c4 100644 --- a/src/amd/vulkan/radv_private.h +++ b/src/amd/vulkan/radv_private.h @@ -3172,9 +3172,9 @@ radv_primitive_topology_is_line_list(unsigned primitive_topology) } static inline unsigned -radv_get_num_vertices_per_prim(const struct radv_pipeline_key *pipeline_key) +radv_get_num_vertices_per_prim(const struct radv_graphics_state_key *gfx_state) { - if (pipeline_key->ia.topology == V_008958_DI_PT_NONE) { + if (gfx_state->ia.topology == V_008958_DI_PT_NONE) { /* When the topology is unknown (with graphics pipeline library), return the maximum number of * vertices per primitives for VS. This is used to lower NGG (the HW will ignore the extra * bits for points/lines) and also to enable NGG culling unconditionally (it will be disabled @@ -3183,7 +3183,7 @@ radv_get_num_vertices_per_prim(const struct radv_pipeline_key *pipeline_key) return 3; } else { /* Need to add 1, because: V_028A6C_POINTLIST=0, V_028A6C_LINESTRIP=1, V_028A6C_TRISTRIP=2, etc. */ - return radv_conv_prim_to_gs_out(pipeline_key->ia.topology, false) + 1; + return radv_conv_prim_to_gs_out(gfx_state->ia.topology, false) + 1; } } diff --git a/src/amd/vulkan/radv_shader.c b/src/amd/vulkan/radv_shader.c index 492a0ee362c..0c0662bc78c 100644 --- a/src/amd/vulkan/radv_shader.c +++ b/src/amd/vulkan/radv_shader.c @@ -350,7 +350,7 @@ fix_dual_src_mrt1_export(nir_shader *nir) nir_shader * radv_shader_spirv_to_nir(struct radv_device *device, const struct radv_shader_stage *stage, - const struct radv_pipeline_key *key, bool is_internal) + const struct radv_graphics_state_key *gfx_state, bool is_internal) { unsigned subgroup_size = 64, ballot_bit_size = 64; const unsigned required_subgroup_size = stage->key.subgroup_required_size * 32; @@ -541,7 +541,7 @@ radv_shader_spirv_to_nir(struct radv_device *device, const struct radv_shader_st NIR_PASS(_, nir, nir_remove_dead_variables, nir_var_shader_in | nir_var_shader_out | nir_var_system_value | nir_var_mem_shared, &dead_vars_opts); - if (nir->info.stage == MESA_SHADER_FRAGMENT && key->ps.epilog.mrt0_is_dual_src && + if (nir->info.stage == MESA_SHADER_FRAGMENT && gfx_state->ps.epilog.mrt0_is_dual_src && device->instance->drirc.dual_color_blend_by_location) fix_dual_src_mrt1_export(nir); @@ -697,7 +697,7 @@ radv_shader_spirv_to_nir(struct radv_device *device, const struct radv_shader_st NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_ubo | nir_var_mem_ssbo, nir_address_format_vec2_index_32bit_offset); - NIR_PASS(_, nir, radv_nir_lower_intrinsics_early, key); + NIR_PASS(_, nir, radv_nir_lower_intrinsics_early, gfx_state); /* Lower deref operations for compute shared memory. */ if (nir->info.stage == MESA_SHADER_COMPUTE || nir->info.stage == MESA_SHADER_TASK || @@ -845,7 +845,8 @@ setup_ngg_lds_layout(struct radv_device *device, nir_shader *nir, struct radv_sh } void -radv_lower_ngg(struct radv_device *device, struct radv_shader_stage *ngg_stage, const struct radv_pipeline_key *pl_key) +radv_lower_ngg(struct radv_device *device, struct radv_shader_stage *ngg_stage, + const struct radv_graphics_state_key *gfx_state) { const struct radv_shader_info *info = &ngg_stage->info; nir_shader *nir = ngg_stage->nir; @@ -868,10 +869,10 @@ radv_lower_ngg(struct radv_device *device, struct radv_shader_stage *ngg_stage, BITSET_SET(nir->info.system_values_read, SYSTEM_VALUE_PRIMITIVE_ID); } else if (nir->info.stage == MESA_SHADER_VERTEX) { - num_vertices_per_prim = radv_get_num_vertices_per_prim(pl_key); + num_vertices_per_prim = radv_get_num_vertices_per_prim(gfx_state); /* Manually mark the instance ID used, so the shader can repack it. */ - if (pl_key->vi.instance_rate_inputs) + if (gfx_state->vi.instance_rate_inputs) BITSET_SET(nir->info.system_values_read, SYSTEM_VALUE_INSTANCE_ID); } else if (nir->info.stage == MESA_SHADER_GEOMETRY) { @@ -916,7 +917,7 @@ radv_lower_ngg(struct radv_device *device, struct radv_shader_stage *ngg_stage, options.early_prim_export = info->has_ngg_early_prim_export; options.passthrough = info->is_ngg_passthrough; options.export_primitive_id = info->outinfo.export_prim_id; - options.instance_rate_inputs = pl_key->vi.instance_rate_inputs << VERT_ATTRIB_GENERIC0; + options.instance_rate_inputs = gfx_state->vi.instance_rate_inputs << VERT_ATTRIB_GENERIC0; NIR_PASS_V(nir, ac_nir_lower_ngg_nogs, &options); @@ -935,7 +936,8 @@ radv_lower_ngg(struct radv_device *device, struct radv_shader_stage *ngg_stage, bool scratch_ring = false; NIR_PASS_V(nir, ac_nir_lower_ngg_ms, options.gfx_level, options.clipdist_enable_mask, options.vs_output_param_offset, options.has_param_exports, &scratch_ring, info->wave_size, - hw_workgroup_size, pl_key->has_multiview_view_index, info->ms.has_query, device->mesh_fast_launch_2); + hw_workgroup_size, gfx_state->has_multiview_view_index, info->ms.has_query, + device->mesh_fast_launch_2); ngg_stage->info.ms.needs_ms_scratch_ring = scratch_ring; } else { unreachable("invalid SW stage passed to radv_lower_ngg"); @@ -2410,8 +2412,9 @@ radv_aco_build_shader_binary(void **bin, const struct ac_shader_config *config, static void radv_fill_nir_compiler_options(struct radv_nir_compiler_options *options, struct radv_device *device, - const struct radv_pipeline_key *key, bool should_use_wgp, bool can_dump_shader, - bool is_meta_shader, bool keep_shader_info, bool keep_statistic_info) + const struct radv_graphics_state_key *gfx_state, bool should_use_wgp, + bool can_dump_shader, bool is_meta_shader, bool keep_shader_info, + bool keep_statistic_info) { /* robust_buffer_access_llvm here used by LLVM only, pipeline robustness is not exposed there. */ options->robust_buffer_access_llvm = device->buffer_robustness >= RADV_BUFFER_ROBUSTNESS_1; @@ -2422,7 +2425,7 @@ radv_fill_nir_compiler_options(struct radv_nir_compiler_options *options, struct options->record_ir = keep_shader_info; options->record_stats = keep_statistic_info; options->check_ir = device->instance->debug_flags & RADV_DEBUG_CHECKIR; - options->enable_mrt_output_nan_fixup = key ? key->ps.epilog.enable_mrt_output_nan_fixup : false; + options->enable_mrt_output_nan_fixup = gfx_state ? gfx_state->ps.epilog.enable_mrt_output_nan_fixup : false; } static void @@ -2509,14 +2512,14 @@ shader_compile(struct radv_device *device, struct nir_shader *const *shaders, in struct radv_shader_binary * radv_shader_nir_to_asm(struct radv_device *device, struct radv_shader_stage *pl_stage, - struct nir_shader *const *shaders, int shader_count, const struct radv_pipeline_key *key, - bool keep_shader_info, bool keep_statistic_info) + struct nir_shader *const *shaders, int shader_count, + const struct radv_graphics_state_key *gfx_state, bool keep_shader_info, bool keep_statistic_info) { gl_shader_stage stage = shaders[shader_count - 1]->info.stage; struct radv_shader_info *info = &pl_stage->info; struct radv_nir_compiler_options options = {0}; - radv_fill_nir_compiler_options(&options, device, key, radv_should_use_wgp_mode(device, stage, info), + radv_fill_nir_compiler_options(&options, device, gfx_state, radv_should_use_wgp_mode(device, stage, info), radv_can_dump_shader(device, shaders[0], false), is_meta_shader(shaders[0]), keep_shader_info, keep_statistic_info); @@ -2549,9 +2552,8 @@ radv_create_trap_handler_shader(struct radv_device *device) gl_shader_stage stage = MESA_SHADER_COMPUTE; struct radv_shader_stage_key stage_key = {0}; struct radv_shader_info info = {0}; - struct radv_pipeline_key key = {0}; struct radv_nir_compiler_options options = {0}; - radv_fill_nir_compiler_options(&options, device, &key, radv_should_use_wgp_mode(device, stage, &info), false, false, + radv_fill_nir_compiler_options(&options, device, NULL, radv_should_use_wgp_mode(device, stage, &info), false, false, false, false); nir_builder b = radv_meta_init_shader(device, stage, "meta_trap_handler"); @@ -2560,7 +2562,7 @@ radv_create_trap_handler_shader(struct radv_device *device) info.type = RADV_SHADER_TYPE_TRAP_HANDLER; struct radv_shader_args args; - radv_declare_shader_args(device, &key, &info, stage, MESA_SHADER_NONE, &args); + radv_declare_shader_args(device, NULL, &info, stage, MESA_SHADER_NONE, &args); struct radv_shader_binary *binary = shader_compile(device, &b.shader, 1, stage, &info, &args, &stage_key, &options); struct radv_shader *shader; @@ -2623,9 +2625,7 @@ radv_create_rt_prolog(struct radv_device *device) for (unsigned i = 0; i < 3; i++) info.cs.uses_block_id[i] = true; - struct radv_pipeline_key pipeline_key = {0}; - - radv_declare_shader_args(device, &pipeline_key, &info, MESA_SHADER_COMPUTE, MESA_SHADER_NONE, &in_args); + radv_declare_shader_args(device, NULL, &info, MESA_SHADER_COMPUTE, MESA_SHADER_NONE, &in_args); radv_declare_rt_shader_args(options.info->gfx_level, &out_args); info.user_sgprs_locs = in_args.user_sgprs_locs; @@ -2685,9 +2685,9 @@ radv_create_vs_prolog(struct radv_device *device, const struct radv_vs_prolog_ke info.vs.as_ls = key->as_ls; info.is_ngg = key->is_ngg; - struct radv_pipeline_key pipeline_key = {0}; + struct radv_graphics_state_key gfx_state = {0}; - radv_declare_shader_args(device, &pipeline_key, &info, key->next_stage, + radv_declare_shader_args(device, &gfx_state, &info, key->next_stage, key->next_stage != MESA_SHADER_VERTEX ? MESA_SHADER_VERTEX : MESA_SHADER_NONE, &args); info.user_sgprs_locs = args.user_sgprs_locs; @@ -2937,7 +2937,7 @@ radv_get_shader_name(const struct radv_shader_info *info, gl_shader_stage stage) } unsigned -radv_compute_spi_ps_input(const struct radv_pipeline_key *pipeline_key, const struct radv_shader_info *info) +radv_compute_spi_ps_input(const struct radv_graphics_state_key *gfx_state, const struct radv_shader_info *info) { unsigned spi_ps_input; @@ -2958,7 +2958,7 @@ radv_compute_spi_ps_input(const struct radv_pipeline_key *pipeline_key, const st spi_ps_input |= S_0286CC_POS_X_FLOAT_ENA(1) << i; } - if (pipeline_key->adjust_frag_coord_z && info->ps.reads_frag_coord_mask & (1 << 2)) { + if (gfx_state->adjust_frag_coord_z && info->ps.reads_frag_coord_mask & (1 << 2)) { spi_ps_input |= S_0286CC_ANCILLARY_ENA(1); } } diff --git a/src/amd/vulkan/radv_shader.h b/src/amd/vulkan/radv_shader.h index c254c442e18..07b3a736f08 100644 --- a/src/amd/vulkan/radv_shader.h +++ b/src/amd/vulkan/radv_shader.h @@ -114,7 +114,7 @@ struct radv_ps_epilog_key { bool alpha_to_coverage_via_mrtz; }; -struct radv_pipeline_key { +struct radv_graphics_state_key { uint32_t lib_flags : 4; /* VkGraphicsPipelineLibraryFlagBitsEXT */ uint32_t has_multiview_view_index : 1; @@ -126,13 +126,6 @@ struct radv_pipeline_key { uint32_t enable_remove_point_size : 1; uint32_t unknown_rast_prim : 1; - uint32_t keep_statistic_info : 1; - - /* Pipeline shader version (up to 8) to force re-compilation when RADV_BUILD_ID_OVERRIDE is enabled. */ - uint32_t shader_version : 3; - - struct radv_shader_stage_key stage_info[MESA_VULKAN_SHADER_STAGES]; - struct { uint8_t topology; } ia; @@ -174,6 +167,17 @@ struct radv_pipeline_key { } ps; }; +struct radv_pipeline_key { + struct radv_graphics_state_key gfx_state; + + uint32_t keep_statistic_info : 1; + + /* Pipeline shader version (up to 8) to force re-compilation when RADV_BUILD_ID_OVERRIDE is enabled. */ + uint32_t shader_version : 3; + + struct radv_shader_stage_key stage_info[MESA_VULKAN_SHADER_STAGES]; +}; + struct radv_nir_compiler_options { bool robust_buffer_access_llvm; bool dump_shader; @@ -716,7 +720,7 @@ struct radv_shader_stage; void radv_optimize_nir(struct nir_shader *shader, bool optimize_conservatively); void radv_optimize_nir_algebraic(nir_shader *shader, bool opt_offsets); -void radv_postprocess_nir(struct radv_device *device, const struct radv_pipeline_key *pipeline_key, +void radv_postprocess_nir(struct radv_device *device, const struct radv_graphics_state_key *gfx_state, struct radv_shader_stage *stage); bool radv_shader_should_clear_lds(const struct radv_device *device, const nir_shader *shader); @@ -733,7 +737,7 @@ void radv_nir_lower_rt_abi(nir_shader *shader, const VkRayTracingPipelineCreateI struct radv_shader_stage; nir_shader *radv_shader_spirv_to_nir(struct radv_device *device, const struct radv_shader_stage *stage, - const struct radv_pipeline_key *key, bool is_internal); + const struct radv_graphics_state_key *gfx_state, bool is_internal); void radv_init_shader_arenas(struct radv_device *device); void radv_destroy_shader_arenas(struct radv_device *device); @@ -751,8 +755,8 @@ VkResult radv_shader_create_uncached(struct radv_device *device, const struct ra struct radv_shader_binary *radv_shader_nir_to_asm(struct radv_device *device, struct radv_shader_stage *pl_stage, struct nir_shader *const *shaders, int shader_count, - const struct radv_pipeline_key *key, bool keep_shader_info, - bool keep_statistic_info); + const struct radv_graphics_state_key *gfx_state, + bool keep_shader_info, bool keep_statistic_info); void radv_shader_generate_debug_info(struct radv_device *device, bool dump_shader, bool keep_shader_info, struct radv_shader_binary *binary, struct radv_shader *shader, @@ -814,7 +818,8 @@ unsigned radv_get_max_scratch_waves(const struct radv_device *device, struct rad const char *radv_get_shader_name(const struct radv_shader_info *info, gl_shader_stage stage); -unsigned radv_compute_spi_ps_input(const struct radv_pipeline_key *pipeline_key, const struct radv_shader_info *info); +unsigned radv_compute_spi_ps_input(const struct radv_graphics_state_key *gfx_state, + const struct radv_shader_info *info); bool radv_can_dump_shader(struct radv_device *device, nir_shader *nir, bool meta_shader); @@ -953,7 +958,7 @@ get_tcs_num_patches(unsigned tcs_num_input_vertices, unsigned tcs_num_output_ver } void radv_lower_ngg(struct radv_device *device, struct radv_shader_stage *ngg_stage, - const struct radv_pipeline_key *pl_key); + const struct radv_graphics_state_key *gfx_state); bool radv_consider_culling(const struct radv_physical_device *pdevice, struct nir_shader *nir, uint64_t ps_inputs_read, unsigned num_vertices_per_primitive, const struct radv_shader_info *info); @@ -995,13 +1000,13 @@ enum radv_pipeline_type; void radv_nir_shader_info_pass(struct radv_device *device, const struct nir_shader *nir, const struct radv_shader_layout *layout, const struct radv_shader_stage_key *stage_key, - const struct radv_pipeline_key *pipeline_key, + const struct radv_graphics_state_key *gfx_state, const enum radv_pipeline_type pipeline_type, bool consider_force_vrs, struct radv_shader_info *info); void radv_nir_shader_info_init(gl_shader_stage stage, gl_shader_stage next_stage, struct radv_shader_info *info); -void radv_nir_shader_info_link(struct radv_device *device, const struct radv_pipeline_key *pipeline_key, +void radv_nir_shader_info_link(struct radv_device *device, const struct radv_graphics_state_key *gfx_state, struct radv_shader_stage *stages); #endif diff --git a/src/amd/vulkan/radv_shader_args.c b/src/amd/vulkan/radv_shader_args.c index c188af7f5b3..0bccf27b934 100644 --- a/src/amd/vulkan/radv_shader_args.c +++ b/src/amd/vulkan/radv_shader_args.c @@ -358,10 +358,10 @@ radv_declare_rt_shader_args(enum amd_gfx_level gfx_level, struct radv_shader_arg } static bool -radv_tcs_needs_state_sgpr(const struct radv_shader_info *info, const struct radv_pipeline_key *key) +radv_tcs_needs_state_sgpr(const struct radv_shader_info *info, const struct radv_graphics_state_key *gfx_state) { /* Some values are loaded from a SGPR when dynamic states are used or when the shader is unlinked. */ - return !key->ts.patch_control_points || !info->num_tess_patches || !info->inputs_linked; + return !gfx_state->ts.patch_control_points || !info->num_tess_patches || !info->inputs_linked; } static bool @@ -372,26 +372,26 @@ radv_tes_needs_state_sgpr(const struct radv_shader_info *info) } static bool -radv_ps_needs_state_sgpr(const struct radv_shader_info *info, const struct radv_pipeline_key *key) +radv_ps_needs_state_sgpr(const struct radv_shader_info *info, const struct radv_graphics_state_key *gfx_state) { - if (info->ps.needs_sample_positions && key->dynamic_rasterization_samples) + if (info->ps.needs_sample_positions && gfx_state->dynamic_rasterization_samples) return true; - if (key->dynamic_line_rast_mode) + if (gfx_state->dynamic_line_rast_mode) return true; - if (info->ps.reads_sample_mask_in && (info->ps.uses_sample_shading || key->ms.sample_shading_enable)) + if (info->ps.reads_sample_mask_in && (info->ps.uses_sample_shading || gfx_state->ms.sample_shading_enable)) return true; /* For computing barycentrics when the primitive topology is unknown at compile time (GPL). */ - if (info->ps.load_rasterization_prim && key->unknown_rast_prim) + if (info->ps.load_rasterization_prim && gfx_state->unknown_rast_prim) return true; return false; } static void -declare_shader_args(const struct radv_device *device, const struct radv_pipeline_key *key, +declare_shader_args(const struct radv_device *device, const struct radv_graphics_state_key *gfx_state, const struct radv_shader_info *info, gl_shader_stage stage, gl_shader_stage previous_stage, struct radv_shader_args *args, struct user_sgpr_info *user_sgpr_info) { @@ -401,7 +401,7 @@ declare_shader_args(const struct radv_device *device, const struct radv_pipeline (stage == MESA_SHADER_MESH && info->ms.has_query) || (stage == MESA_SHADER_TASK && info->cs.has_query); bool has_ngg_provoking_vtx = - (stage == MESA_SHADER_VERTEX || stage == MESA_SHADER_GEOMETRY) && key->dynamic_provoking_vtx_mode; + (stage == MESA_SHADER_VERTEX || stage == MESA_SHADER_GEOMETRY) && gfx_state->dynamic_provoking_vtx_mode; if (gfx_level >= GFX10 && info->is_ngg && stage != MESA_SHADER_GEOMETRY) { /* Handle all NGG shaders as GS to simplify the code here. */ @@ -536,7 +536,7 @@ declare_shader_args(const struct radv_device *device, const struct radv_pipeline add_ud_arg(args, 1, AC_ARG_INT, &args->ac.view_index, AC_UD_VIEW_INDEX); } - if (radv_tcs_needs_state_sgpr(info, key)) { + if (radv_tcs_needs_state_sgpr(info, gfx_state)) { add_ud_arg(args, 1, AC_ARG_INT, &args->tcs_offchip_layout, AC_UD_TCS_OFFCHIP_LAYOUT); } @@ -582,7 +582,7 @@ declare_shader_args(const struct radv_device *device, const struct radv_pipeline add_ud_arg(args, 1, AC_ARG_INT, &args->ac.view_index, AC_UD_VIEW_INDEX); } - if (radv_tcs_needs_state_sgpr(info, key)) { + if (radv_tcs_needs_state_sgpr(info, gfx_state)) { add_ud_arg(args, 1, AC_ARG_INT, &args->tcs_offchip_layout, AC_UD_TCS_OFFCHIP_LAYOUT); } @@ -758,7 +758,7 @@ declare_shader_args(const struct radv_device *device, const struct radv_pipeline add_ud_arg(args, 1, AC_ARG_INT, &args->ps_epilog_pc, AC_UD_PS_EPILOG_PC); } - if (radv_ps_needs_state_sgpr(info, key)) + if (radv_ps_needs_state_sgpr(info, gfx_state)) add_ud_arg(args, 1, AC_ARG_INT, &args->ps_state, AC_UD_PS_STATE); ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->ac.prim_mask); @@ -783,11 +783,11 @@ declare_shader_args(const struct radv_device *device, const struct radv_pipeline } void -radv_declare_shader_args(const struct radv_device *device, const struct radv_pipeline_key *key, +radv_declare_shader_args(const struct radv_device *device, const struct radv_graphics_state_key *gfx_state, const struct radv_shader_info *info, gl_shader_stage stage, gl_shader_stage previous_stage, struct radv_shader_args *args) { - declare_shader_args(device, key, info, stage, previous_stage, args, NULL); + declare_shader_args(device, gfx_state, info, stage, previous_stage, args, NULL); if (gl_shader_stage_is_rt(stage)) return; @@ -816,7 +816,7 @@ radv_declare_shader_args(const struct radv_device *device, const struct radv_pip if (!info->merged_shader_compiled_separately) allocate_inline_push_consts(info, &user_sgpr_info); - declare_shader_args(device, key, info, stage, previous_stage, args, &user_sgpr_info); + declare_shader_args(device, gfx_state, info, stage, previous_stage, args, &user_sgpr_info); } void diff --git a/src/amd/vulkan/radv_shader_args.h b/src/amd/vulkan/radv_shader_args.h index 08107d6b989..5327affbc3a 100644 --- a/src/amd/vulkan/radv_shader_args.h +++ b/src/amd/vulkan/radv_shader_args.h @@ -106,10 +106,10 @@ radv_shader_args_from_ac(struct ac_shader_args *args) return container_of(args, struct radv_shader_args, ac); } -struct radv_pipeline_key; +struct radv_graphics_state_key; struct radv_shader_info; -void radv_declare_shader_args(const struct radv_device *device, const struct radv_pipeline_key *key, +void radv_declare_shader_args(const struct radv_device *device, const struct radv_graphics_state_key *gfx_state, const struct radv_shader_info *info, gl_shader_stage stage, gl_shader_stage previous_stage, struct radv_shader_args *args); diff --git a/src/amd/vulkan/radv_shader_info.c b/src/amd/vulkan/radv_shader_info.c index 7d54db7fb58..ca28ac1bcc9 100644 --- a/src/amd/vulkan/radv_shader_info.c +++ b/src/amd/vulkan/radv_shader_info.c @@ -384,10 +384,10 @@ radv_compute_esgs_itemsize(const struct radv_device *device, uint32_t num_varyin static void gather_info_input_decl_vs(const nir_shader *nir, unsigned location, const struct glsl_type *type, - const struct radv_pipeline_key *key, struct radv_shader_info *info) + const struct radv_graphics_state_key *gfx_state, struct radv_shader_info *info) { if (glsl_type_is_scalar(type) || glsl_type_is_vector(type)) { - if (key->vi.instance_rate_inputs & BITFIELD_BIT(location)) { + if (gfx_state->vi.instance_rate_inputs & BITFIELD_BIT(location)) { info->vs.needs_instance_id = true; info->vs.needs_base_instance = true; } @@ -395,7 +395,7 @@ gather_info_input_decl_vs(const nir_shader *nir, unsigned location, const struct if (info->vs.use_per_attribute_vb_descs) info->vs.vb_desc_usage_mask |= BITFIELD_BIT(location); else - info->vs.vb_desc_usage_mask |= BITFIELD_BIT(key->vi.vertex_attribute_bindings[location]); + info->vs.vb_desc_usage_mask |= BITFIELD_BIT(gfx_state->vi.vertex_attribute_bindings[location]); info->vs.input_slot_usage_mask |= BITFIELD_RANGE(location, glsl_count_attribute_slots(type, false)); } else if (glsl_type_is_matrix(type) || glsl_type_is_array(type)) { @@ -403,23 +403,24 @@ gather_info_input_decl_vs(const nir_shader *nir, unsigned location, const struct unsigned stride = glsl_count_attribute_slots(elem, false); for (unsigned i = 0; i < glsl_get_length(type); ++i) - gather_info_input_decl_vs(nir, location + i * stride, elem, key, info); + gather_info_input_decl_vs(nir, location + i * stride, elem, gfx_state, info); } else { assert(glsl_type_is_struct_or_ifc(type)); for (unsigned i = 0; i < glsl_get_length(type); i++) { const struct glsl_type *field = glsl_get_struct_field(type, i); - gather_info_input_decl_vs(nir, location, field, key, info); + gather_info_input_decl_vs(nir, location, field, gfx_state, info); location += glsl_count_attribute_slots(field, false); } } } static void -gather_shader_info_vs(struct radv_device *device, const nir_shader *nir, const struct radv_pipeline_key *pipeline_key, - const struct radv_shader_stage_key *stage_key, struct radv_shader_info *info) +gather_shader_info_vs(struct radv_device *device, const nir_shader *nir, + const struct radv_graphics_state_key *gfx_state, const struct radv_shader_stage_key *stage_key, + struct radv_shader_info *info) { - if (pipeline_key->vs.has_prolog && nir->info.inputs_read) { + if (gfx_state->vs.has_prolog && nir->info.inputs_read) { info->vs.has_prolog = true; info->vs.dynamic_inputs = true; } @@ -435,7 +436,7 @@ gather_shader_info_vs(struct radv_device *device, const nir_shader *nir, const s info->vs.needs_draw_id |= info->vs.has_prolog; nir_foreach_shader_in_variable (var, nir) - gather_info_input_decl_vs(nir, var->data.location - VERT_ATTRIB_GENERIC0, var->type, pipeline_key, info); + gather_info_input_decl_vs(nir, var->data.location - VERT_ATTRIB_GENERIC0, var->type, gfx_state, info); if (info->vs.dynamic_inputs) info->vs.vb_desc_usage_mask = BITFIELD_MASK(util_last_bit(info->vs.vb_desc_usage_mask)); @@ -444,8 +445,7 @@ gather_shader_info_vs(struct radv_device *device, const nir_shader *nir, const s * through a user SGPR for NGG streamout with VS. Otherwise, the XFB offset is incorrectly * computed because using the maximum number of vertices can't work. */ - info->vs.dynamic_num_verts_per_prim = - pipeline_key->ia.topology == V_008958_DI_PT_NONE && info->is_ngg && nir->xfb_info; + info->vs.dynamic_num_verts_per_prim = gfx_state->ia.topology == V_008958_DI_PT_NONE && info->is_ngg && nir->xfb_info; if (!info->outputs_linked) info->vs.num_linked_outputs = util_last_bit64(nir->info.outputs_written); @@ -459,8 +459,8 @@ gather_shader_info_vs(struct radv_device *device, const nir_shader *nir, const s } static void -gather_shader_info_tcs(struct radv_device *device, const nir_shader *nir, const struct radv_pipeline_key *pipeline_key, - struct radv_shader_info *info) +gather_shader_info_tcs(struct radv_device *device, const nir_shader *nir, + const struct radv_graphics_state_key *gfx_state, struct radv_shader_info *info) { info->tcs.tcs_vertices_out = nir->info.tess.tcs_vertices_out; info->tcs.tes_inputs_read = ~0ULL; @@ -473,17 +473,17 @@ gather_shader_info_tcs(struct radv_device *device, const nir_shader *nir, const info->tcs.num_linked_patch_outputs = util_last_bit64(nir->info.patch_outputs_written); } - if (!(pipeline_key->dynamic_patch_control_points)) { + if (!(gfx_state->dynamic_patch_control_points)) { /* Number of tessellation patches per workgroup processed by the current pipeline. */ info->num_tess_patches = - get_tcs_num_patches(pipeline_key->ts.patch_control_points, nir->info.tess.tcs_vertices_out, + get_tcs_num_patches(gfx_state->ts.patch_control_points, nir->info.tess.tcs_vertices_out, info->tcs.num_linked_inputs, info->tcs.num_linked_outputs, info->tcs.num_linked_patch_outputs, device->physical_device->hs.tess_offchip_block_dw_size, device->physical_device->rad_info.gfx_level, device->physical_device->rad_info.family); /* LDS size used by VS+TCS for storing TCS inputs and outputs. */ info->tcs.num_lds_blocks = - calculate_tess_lds_size(device->physical_device->rad_info.gfx_level, pipeline_key->ts.patch_control_points, + calculate_tess_lds_size(device->physical_device->rad_info.gfx_level, gfx_state->ts.patch_control_points, nir->info.tess.tcs_vertices_out, info->tcs.num_linked_inputs, info->num_tess_patches, info->tcs.num_linked_outputs, info->tcs.num_linked_patch_outputs); } @@ -681,8 +681,8 @@ gather_shader_info_gs(struct radv_device *device, const nir_shader *nir, struct } static void -gather_shader_info_mesh(struct radv_device *device, const nir_shader *nir, const struct radv_pipeline_key *pipeline_key, - struct radv_shader_info *info) +gather_shader_info_mesh(struct radv_device *device, const nir_shader *nir, + const struct radv_graphics_state_key *gfx_state, struct radv_shader_info *info) { struct gfx10_ngg_info *ngg_info = &info->ngg_info; @@ -744,7 +744,7 @@ calc_mesh_workgroup_size(const struct radv_device *device, const nir_shader *nir static void gather_shader_info_fs(const struct radv_device *device, const nir_shader *nir, - const struct radv_pipeline_key *pipeline_key, struct radv_shader_info *info) + const struct radv_graphics_state_key *gfx_state, struct radv_shader_info *info) { uint64_t per_primitive_input_mask = nir->info.inputs_read & nir->info.per_primitive_inputs; unsigned num_per_primitive_inputs = util_bitcount64(per_primitive_input_mask); @@ -802,23 +802,23 @@ gather_shader_info_fs(const struct radv_device *device, const nir_shader *nir, info->ps.pops_is_per_sample = info->ps.pops && (nir->info.fs.sample_interlock_ordered || nir->info.fs.sample_interlock_unordered); - info->ps.spi_ps_input = radv_compute_spi_ps_input(pipeline_key, info); + info->ps.spi_ps_input = radv_compute_spi_ps_input(gfx_state, info); - info->has_epilog = pipeline_key->ps.has_epilog && info->ps.colors_written; + info->has_epilog = gfx_state->ps.has_epilog && info->ps.colors_written; if (!info->has_epilog) { - info->ps.mrt0_is_dual_src = pipeline_key->ps.epilog.mrt0_is_dual_src; - info->ps.spi_shader_col_format = pipeline_key->ps.epilog.spi_shader_col_format; + info->ps.mrt0_is_dual_src = gfx_state->ps.epilog.mrt0_is_dual_src; + info->ps.spi_shader_col_format = gfx_state->ps.epilog.spi_shader_col_format; } const bool export_alpha_and_mrtz = (info->ps.color0_written & 0x8) && (info->ps.writes_z || info->ps.writes_stencil || info->ps.writes_sample_mask); info->ps.exports_mrtz_via_epilog = - info->has_epilog && pipeline_key->ps.exports_mrtz_via_epilog && export_alpha_and_mrtz; + info->has_epilog && gfx_state->ps.exports_mrtz_via_epilog && export_alpha_and_mrtz; if (!info->ps.exports_mrtz_via_epilog) { - info->ps.writes_mrt0_alpha = pipeline_key->ms.alpha_to_coverage_via_mrtz && export_alpha_and_mrtz; + info->ps.writes_mrt0_alpha = gfx_state->ms.alpha_to_coverage_via_mrtz && export_alpha_and_mrtz; } nir_foreach_shader_in_variable (var, nir) { @@ -1077,7 +1077,7 @@ radv_nir_shader_info_init(gl_shader_stage stage, gl_shader_stage next_stage, str void radv_nir_shader_info_pass(struct radv_device *device, const struct nir_shader *nir, const struct radv_shader_layout *layout, const struct radv_shader_stage_key *stage_key, - const struct radv_pipeline_key *pipeline_key, const enum radv_pipeline_type pipeline_type, + const struct radv_graphics_state_key *gfx_state, const enum radv_pipeline_type pipeline_type, bool consider_force_vrs, struct radv_shader_info *info) { struct nir_function *func = (struct nir_function *)exec_list_get_head_const(&nir->functions); @@ -1107,7 +1107,7 @@ radv_nir_shader_info_pass(struct radv_device *device, const struct nir_shader *n uint64_t per_vtx_mask = nir->info.outputs_written & ~nir->info.per_primitive_outputs & ~special_mask; /* Mesh multivew is only lowered in ac_nir_lower_ngg, so we have to fake it here. */ - if (nir->info.stage == MESA_SHADER_MESH && pipeline_key->has_multiview_view_index) { + if (nir->info.stage == MESA_SHADER_MESH && gfx_state->has_multiview_view_index) { per_prim_mask |= VARYING_BIT_LAYER; info->uses_view_index = true; } @@ -1199,7 +1199,7 @@ radv_nir_shader_info_pass(struct radv_device *device, const struct nir_shader *n gather_shader_info_task(device, nir, stage_key, info); break; case MESA_SHADER_FRAGMENT: - gather_shader_info_fs(device, nir, pipeline_key, info); + gather_shader_info_fs(device, nir, gfx_state, info); break; case MESA_SHADER_GEOMETRY: gather_shader_info_gs(device, nir, info); @@ -1208,13 +1208,13 @@ radv_nir_shader_info_pass(struct radv_device *device, const struct nir_shader *n gather_shader_info_tes(device, nir, info); break; case MESA_SHADER_TESS_CTRL: - gather_shader_info_tcs(device, nir, pipeline_key, info); + gather_shader_info_tcs(device, nir, gfx_state, info); break; case MESA_SHADER_VERTEX: - gather_shader_info_vs(device, nir, pipeline_key, stage_key, info); + gather_shader_info_vs(device, nir, gfx_state, stage_key, info); break; case MESA_SHADER_MESH: - gather_shader_info_mesh(device, nir, pipeline_key, info); + gather_shader_info_mesh(device, nir, gfx_state, info); break; default: if (gl_shader_stage_is_rt(nir->info.stage)) @@ -1518,7 +1518,7 @@ gfx10_get_ngg_info(const struct radv_device *device, struct radv_shader_stage *e static void gfx10_get_ngg_query_info(const struct radv_device *device, struct radv_shader_stage *es_stage, - struct radv_shader_stage *gs_stage, const struct radv_pipeline_key *pipeline_key) + struct radv_shader_stage *gs_stage, const struct radv_graphics_state_key *gfx_state) { struct radv_shader_info *info = gs_stage ? &gs_stage->info : &es_stage->info; @@ -1529,7 +1529,7 @@ gfx10_get_ngg_query_info(const struct radv_device *device, struct radv_shader_st static void radv_determine_ngg_settings(struct radv_device *device, struct radv_shader_stage *es_stage, - struct radv_shader_stage *fs_stage, const struct radv_pipeline_key *pipeline_key) + struct radv_shader_stage *fs_stage, const struct radv_graphics_state_key *gfx_state) { assert(es_stage->stage == MESA_SHADER_VERTEX || es_stage->stage == MESA_SHADER_TESS_EVAL); assert(!fs_stage || fs_stage->stage == MESA_SHADER_FRAGMENT); @@ -1538,7 +1538,7 @@ radv_determine_ngg_settings(struct radv_device *device, struct radv_shader_stage unsigned num_vertices_per_prim = 0; if (es_stage->stage == MESA_SHADER_VERTEX) { - num_vertices_per_prim = radv_get_num_vertices_per_prim(pipeline_key); + num_vertices_per_prim = radv_get_num_vertices_per_prim(gfx_state); } else if (es_stage->stage == MESA_SHADER_TESS_EVAL) { num_vertices_per_prim = es_stage->nir->info.tess.point_mode ? 1 : es_stage->nir->info.tess._primitive_mode == TESS_PRIMITIVE_ISOLINES ? 2 @@ -1562,13 +1562,13 @@ radv_determine_ngg_settings(struct radv_device *device, struct radv_shader_stage static void radv_link_shaders_info(struct radv_device *device, struct radv_shader_stage *producer, - struct radv_shader_stage *consumer, const struct radv_pipeline_key *pipeline_key) + struct radv_shader_stage *consumer, const struct radv_graphics_state_key *gfx_state) { /* Export primitive ID and clip/cull distances if read by the FS, or export unconditionally when * the next stage is unknown (with graphics pipeline library). */ if (producer->info.next_stage == MESA_SHADER_FRAGMENT || - !(pipeline_key->lib_flags & VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT)) { + !(gfx_state->lib_flags & VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT)) { struct radv_vs_output_info *outinfo = &producer->info.outinfo; const bool ps_prim_id_in = !consumer || consumer->info.ps.prim_id_input; const bool ps_clip_dists_in = !consumer || !!consumer->info.ps.num_input_clips_culls; @@ -1595,11 +1595,11 @@ radv_link_shaders_info(struct radv_device *device, struct radv_shader_stage *pro struct radv_shader_stage *gs_stage = consumer && consumer->stage == MESA_SHADER_GEOMETRY ? consumer : NULL; gfx10_get_ngg_info(device, producer, gs_stage); - gfx10_get_ngg_query_info(device, producer, gs_stage, pipeline_key); + gfx10_get_ngg_query_info(device, producer, gs_stage, gfx_state); /* Determine other NGG settings like culling for VS or TES without GS. */ if (!gs_stage) { - radv_determine_ngg_settings(device, producer, consumer, pipeline_key); + radv_determine_ngg_settings(device, producer, consumer, gfx_state); } } else if (consumer && consumer->stage == MESA_SHADER_GEOMETRY) { struct radv_shader_info *gs_info = &consumer->info; @@ -1620,7 +1620,7 @@ radv_link_shaders_info(struct radv_device *device, struct radv_shader_stage *pro struct radv_shader_stage *vs_stage = producer; struct radv_shader_stage *tcs_stage = consumer; - if (pipeline_key->dynamic_patch_control_points) { + if (gfx_state->dynamic_patch_control_points) { /* Set the workgroup size to the maximum possible value to ensure that compilers don't * optimize barriers. */ @@ -1629,11 +1629,11 @@ radv_link_shaders_info(struct radv_device *device, struct radv_shader_stage *pro } else { vs_stage->info.workgroup_size = ac_compute_lshs_workgroup_size( device->physical_device->rad_info.gfx_level, MESA_SHADER_VERTEX, tcs_stage->info.num_tess_patches, - pipeline_key->ts.patch_control_points, tcs_stage->info.tcs.tcs_vertices_out); + gfx_state->ts.patch_control_points, tcs_stage->info.tcs.tcs_vertices_out); tcs_stage->info.workgroup_size = ac_compute_lshs_workgroup_size( device->physical_device->rad_info.gfx_level, MESA_SHADER_TESS_CTRL, tcs_stage->info.num_tess_patches, - pipeline_key->ts.patch_control_points, tcs_stage->info.tcs.tcs_vertices_out); + gfx_state->ts.patch_control_points, tcs_stage->info.tcs.tcs_vertices_out); if (!radv_use_llvm_for_stage(device, MESA_SHADER_VERTEX)) { /* When the number of TCS input and output vertices are the same (typically 3): @@ -1647,7 +1647,7 @@ radv_link_shaders_info(struct radv_device *device, struct radv_shader_stage *pro */ vs_stage->info.vs.tcs_in_out_eq = device->physical_device->rad_info.gfx_level >= GFX9 && - pipeline_key->ts.patch_control_points == tcs_stage->info.tcs.tcs_vertices_out && + gfx_state->ts.patch_control_points == tcs_stage->info.tcs.tcs_vertices_out && vs_stage->nir->info.float_controls_execution_mode == tcs_stage->nir->info.float_controls_execution_mode; if (vs_stage->info.vs.tcs_in_out_eq) @@ -1669,7 +1669,7 @@ radv_link_shaders_info(struct radv_device *device, struct radv_shader_stage *pro tcs_stage->info.tcs.tes_inputs_read = tes_stage->nir->info.inputs_read; tcs_stage->info.tcs.tes_patch_inputs_read = tes_stage->nir->info.patch_inputs_read; - if (!pipeline_key->dynamic_patch_control_points) + if (!gfx_state->dynamic_patch_control_points) tes_stage->info.num_tess_patches = tcs_stage->info.num_tess_patches; } @@ -1716,7 +1716,7 @@ static const gl_shader_stage graphics_shader_order[] = { }; void -radv_nir_shader_info_link(struct radv_device *device, const struct radv_pipeline_key *pipeline_key, +radv_nir_shader_info_link(struct radv_device *device, const struct radv_graphics_state_key *gfx_state, struct radv_shader_stage *stages) { /* Walk backwards to link */ @@ -1727,7 +1727,7 @@ radv_nir_shader_info_link(struct radv_device *device, const struct radv_pipeline if (!stages[s].nir) continue; - radv_link_shaders_info(device, &stages[s], next_stage, pipeline_key); + radv_link_shaders_info(device, &stages[s], next_stage, gfx_state); next_stage = &stages[s]; }