From f95665cfebbd943ca3795c026e3dabf4af3f262c Mon Sep 17 00:00:00 2001 From: Jason Ekstrand Date: Thu, 5 Nov 2020 23:09:14 -0600 Subject: [PATCH] nir/lower_bit_size: Add support for lowering subgroup ops Reviewed-by: Kenneth Graunke Part-of: --- src/compiler/nir/nir_lower_bit_size.c | 106 +++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 4 deletions(-) diff --git a/src/compiler/nir/nir_lower_bit_size.c b/src/compiler/nir/nir_lower_bit_size.c index a53090a8760..6e7b26bff45 100644 --- a/src/compiler/nir/nir_lower_bit_size.c +++ b/src/compiler/nir/nir_lower_bit_size.c @@ -95,6 +95,95 @@ lower_alu_instr(nir_builder *bld, nir_alu_instr *alu, unsigned bit_size) } } +static void +lower_intrinsic_instr(nir_builder *b, nir_intrinsic_instr *intrin, + unsigned bit_size) +{ + switch (intrin->intrinsic) { + case nir_intrinsic_read_invocation: + case nir_intrinsic_read_first_invocation: + case nir_intrinsic_vote_feq: + case nir_intrinsic_vote_ieq: + case nir_intrinsic_shuffle: + case nir_intrinsic_shuffle_xor: + case nir_intrinsic_shuffle_up: + case nir_intrinsic_shuffle_down: + case nir_intrinsic_quad_broadcast: + case nir_intrinsic_quad_swap_horizontal: + case nir_intrinsic_quad_swap_vertical: + case nir_intrinsic_quad_swap_diagonal: + case nir_intrinsic_reduce: + case nir_intrinsic_inclusive_scan: + case nir_intrinsic_exclusive_scan: { + assert(intrin->src[0].is_ssa && intrin->dest.is_ssa); + const unsigned old_bit_size = intrin->dest.ssa.bit_size; + assert(old_bit_size < bit_size); + + nir_alu_type type = nir_type_uint; + if (nir_intrinsic_has_reduction_op(intrin)) + type = nir_op_infos[nir_intrinsic_reduction_op(intrin)].input_types[0]; + else if (intrin->intrinsic == nir_intrinsic_vote_feq) + type = nir_type_float; + + b->cursor = nir_before_instr(&intrin->instr); + nir_intrinsic_instr *new_intrin = + nir_instr_as_intrinsic(nir_instr_clone(b->shader, &intrin->instr)); + + nir_ssa_def *new_src = nir_convert_to_bit_size(b, intrin->src[0].ssa, + type, bit_size); + new_intrin->src[0] = nir_src_for_ssa(new_src); + + if (intrin->intrinsic == nir_intrinsic_vote_feq || + intrin->intrinsic == nir_intrinsic_vote_ieq) { + /* These return a Boolean; it's always 1-bit */ + assert(new_intrin->dest.ssa.bit_size == 1); + } else { + /* These return the same bit size as the source; we need to adjust + * the size and then we'll have to emit a down-cast. + */ + assert(intrin->src[0].ssa->bit_size == intrin->dest.ssa.bit_size); + new_intrin->dest.ssa.bit_size = bit_size; + } + + nir_builder_instr_insert(b, &new_intrin->instr); + + nir_ssa_def *res = &new_intrin->dest.ssa; + if (intrin->intrinsic == nir_intrinsic_exclusive_scan) { + /* For exclusive scan, we have to be careful because the identity + * value for the higher bit size may get added into the mix by + * disabled channels. For some cases (imin/imax in particular), + * this value won't convert to the right identity value when we + * down-cast so we have to clamp it. + */ + switch (nir_intrinsic_reduction_op(intrin)) { + case nir_op_imin: { + int64_t int_max = (1ull << (old_bit_size - 1)) - 1; + res = nir_imin(b, res, nir_imm_intN_t(b, int_max, bit_size)); + break; + } + case nir_op_imax: { + int64_t int_min = -(int64_t)(1ull << (old_bit_size - 1)); + res = nir_imax(b, res, nir_imm_intN_t(b, int_min, bit_size)); + break; + } + default: + break; + } + } + + if (intrin->intrinsic != nir_intrinsic_vote_feq && + intrin->intrinsic != nir_intrinsic_vote_ieq) + res = nir_u2u(b, res, old_bit_size); + + nir_ssa_def_rewrite_uses(&intrin->dest.ssa, nir_src_for_ssa(res)); + break; + } + + default: + unreachable("Unsupported instruction"); + } +} + static bool lower_impl(nir_function_impl *impl, nir_lower_bit_size_callback callback, @@ -106,14 +195,23 @@ lower_impl(nir_function_impl *impl, nir_foreach_block(block, impl) { nir_foreach_instr_safe(instr, block) { - if (instr->type != nir_instr_type_alu) - continue; - unsigned lower_bit_size = callback(instr, callback_data); if (lower_bit_size == 0) continue; - lower_alu_instr(&b, nir_instr_as_alu(instr), lower_bit_size); + switch (instr->type) { + case nir_instr_type_alu: + lower_alu_instr(&b, nir_instr_as_alu(instr), lower_bit_size); + break; + + case nir_instr_type_intrinsic: + lower_intrinsic_instr(&b, nir_instr_as_intrinsic(instr), + lower_bit_size); + break; + + default: + unreachable("Unsupported instruction type"); + } progress = true; } }