diff --git a/src/compiler/nir/meson.build b/src/compiler/nir/meson.build index 3efb187f7af..df4a8a59919 100644 --- a/src/compiler/nir/meson.build +++ b/src/compiler/nir/meson.build @@ -263,6 +263,7 @@ files_libnir = files( 'nir_range_analysis.h', 'nir_remove_dead_variables.c', 'nir_repair_ssa.c', + 'nir_scale_fdiv.c', 'nir_schedule.c', 'nir_schedule.h', 'nir_search.c', diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index 45ca5789825..543dad8c165 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -4722,6 +4722,8 @@ bool nir_lower_alu(nir_shader *shader); bool nir_lower_flrp(nir_shader *shader, unsigned lowering_mask, bool always_precise); +bool nir_scale_fdiv(nir_shader *shader); + bool nir_lower_alu_to_scalar(nir_shader *shader, nir_instr_filter_cb cb, const void *data); bool nir_lower_bool_to_bitsize(nir_shader *shader); bool nir_lower_bool_to_float(nir_shader *shader); diff --git a/src/compiler/nir/nir_scale_fdiv.c b/src/compiler/nir/nir_scale_fdiv.c new file mode 100644 index 00000000000..9ffe998b8a3 --- /dev/null +++ b/src/compiler/nir/nir_scale_fdiv.c @@ -0,0 +1,75 @@ +/* + * Copyright © 2020 Microsoft Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "nir.h" +#include "nir_builder.h" + +static bool +nir_scale_fdiv_instr(nir_builder *b, nir_instr *instr, UNUSED void *_data) +{ + if (instr->type != nir_instr_type_alu) + return false; + + nir_alu_instr *alu = nir_instr_as_alu(instr); + if (alu->op != nir_op_fdiv || alu->src[0].src.ssa->bit_size != 32) + return false; + + b->cursor = nir_before_instr(&alu->instr); + + nir_ssa_def *fabs = nir_fabs(b, alu->src[1].src.ssa); + nir_ssa_def *big = nir_flt(b, nir_imm_int(b, 0x7e800000), fabs); + nir_ssa_def *small = nir_flt(b, fabs, nir_imm_int(b, 0x00800000)); + + nir_ssa_def *scaled_down_a = nir_fmul_imm(b, alu->src[0].src.ssa, 0.25); + nir_ssa_def *scaled_down_b = nir_fmul_imm(b, alu->src[1].src.ssa, 0.25); + nir_ssa_def *scaled_up_a = nir_fmul_imm(b, alu->src[0].src.ssa, 16777216.0); + nir_ssa_def *scaled_up_b = nir_fmul_imm(b, alu->src[1].src.ssa, 16777216.0); + + nir_ssa_def *final_a = + nir_bcsel(b, big, scaled_down_a, + (nir_bcsel(b, small, scaled_up_a, alu->src[0].src.ssa))); + nir_ssa_def *final_b = + nir_bcsel(b, big, scaled_down_b, + (nir_bcsel(b, small, scaled_up_b, alu->src[1].src.ssa))); + + nir_instr_rewrite_src_ssa(instr, &alu->src[0].src, final_a); + nir_instr_rewrite_src_ssa(instr, &alu->src[1].src, final_b); + + return true; +} + +/** Scale both sides of an fdiv if needed to prevent denorm flushing + * + * This may be needed to satisfy the precision requirements of OpenCL. When + * fdiv is lowered to frcp+fmul, denorm flushing may cause the frcp to return + * zero even for finite floats. This multiplies both sides of an fdiv by a + * constant, if needed, to prevent such flushing. + */ +bool +nir_scale_fdiv(nir_shader *shader) +{ + return nir_shader_instructions_pass(shader, nir_scale_fdiv_instr, + nir_metadata_block_index | + nir_metadata_dominance, + NULL); +} diff --git a/src/microsoft/clc/clc_compiler.c b/src/microsoft/clc/clc_compiler.c index 819095396b6..ef8d725ee15 100644 --- a/src/microsoft/clc/clc_compiler.c +++ b/src/microsoft/clc/clc_compiler.c @@ -725,49 +725,6 @@ static bool shader_has_double(nir_shader *nir) return false; } -static bool -scale_fdiv(nir_shader *nir) -{ - bool progress = false; - nir_foreach_function(func, nir) { - if (!func->impl) - continue; - nir_builder b; - nir_builder_init(&b, func->impl); - nir_foreach_block(block, func->impl) { - nir_foreach_instr(instr, block) { - if (instr->type != nir_instr_type_alu) - continue; - nir_alu_instr *alu = nir_instr_as_alu(instr); - if (alu->op != nir_op_fdiv || alu->src[0].src.ssa->bit_size != 32) - continue; - - b.cursor = nir_before_instr(instr); - nir_ssa_def *fabs = nir_fabs(&b, alu->src[1].src.ssa); - nir_ssa_def *big = nir_flt(&b, nir_imm_int(&b, 0x7e800000), fabs); - nir_ssa_def *small = nir_flt(&b, fabs, nir_imm_int(&b, 0x00800000)); - - nir_ssa_def *scaled_down_a = nir_fmul_imm(&b, alu->src[0].src.ssa, 0.25); - nir_ssa_def *scaled_down_b = nir_fmul_imm(&b, alu->src[1].src.ssa, 0.25); - nir_ssa_def *scaled_up_a = nir_fmul_imm(&b, alu->src[0].src.ssa, 16777216.0); - nir_ssa_def *scaled_up_b = nir_fmul_imm(&b, alu->src[1].src.ssa, 16777216.0); - - nir_ssa_def *final_a = - nir_bcsel(&b, big, scaled_down_a, - (nir_bcsel(&b, small, scaled_up_a, alu->src[0].src.ssa))); - nir_ssa_def *final_b = - nir_bcsel(&b, big, scaled_down_b, - (nir_bcsel(&b, small, scaled_up_b, alu->src[1].src.ssa))); - - nir_instr_rewrite_src(instr, &alu->src[0].src, nir_src_for_ssa(final_a)); - nir_instr_rewrite_src(instr, &alu->src[1].src, nir_src_for_ssa(final_b)); - progress = true; - } - } - } - return progress; -} - struct clc_libclc * clc_libclc_new_dxil(const struct clc_logger *logger, const struct clc_libclc_dxil_options *options) @@ -918,7 +875,7 @@ clc_spirv_to_dxil(struct clc_libclc *lib, } while (progress); } - NIR_PASS_V(nir, scale_fdiv); + NIR_PASS_V(nir, nir_scale_fdiv); dxil_wrap_sampler_state int_sampler_states[PIPE_MAX_SHADER_SAMPLER_VIEWS] = { {{0}} }; unsigned sampler_id = 0;