diff --git a/src/compiler/Makefile.sources b/src/compiler/Makefile.sources index 9b9d8af7212..706a181c7ab 100644 --- a/src/compiler/Makefile.sources +++ b/src/compiler/Makefile.sources @@ -275,6 +275,7 @@ NIR_FILES = \ nir/nir_lower_io_to_scalar.c \ nir/nir_lower_io_to_vector.c \ nir/nir_lower_mediump_outputs.c \ + nir/nir_lower_memcpy.c \ nir/nir_lower_memory_model.c \ nir/nir_lower_non_uniform_access.c \ nir/nir_lower_packing.c \ diff --git a/src/compiler/nir/meson.build b/src/compiler/nir/meson.build index e563bd2f270..af633d87061 100644 --- a/src/compiler/nir/meson.build +++ b/src/compiler/nir/meson.build @@ -155,6 +155,7 @@ files_libnir = files( 'nir_lower_io_to_scalar.c', 'nir_lower_io_to_vector.c', 'nir_lower_mediump_outputs.c', + 'nir_lower_memcpy.c', 'nir_lower_memory_model.c', 'nir_lower_non_uniform_access.c', 'nir_lower_packing.c', diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index 368e7be7e2c..fddd8060d44 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -4089,6 +4089,8 @@ void nir_lower_deref_copy_instr(struct nir_builder *b, nir_intrinsic_instr *copy); bool nir_lower_var_copies(nir_shader *shader); +bool nir_lower_memcpy(nir_shader *shader); + void nir_fixup_deref_modes(nir_shader *shader); bool nir_lower_global_vars_to_local(nir_shader *shader); diff --git a/src/compiler/nir/nir_lower_memcpy.c b/src/compiler/nir/nir_lower_memcpy.c new file mode 100644 index 00000000000..ab2914485f6 --- /dev/null +++ b/src/compiler/nir/nir_lower_memcpy.c @@ -0,0 +1,195 @@ +/* + * Copyright © 2020 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "nir_builder.h" + +#include + +/** Returns the type to use for a copy of the given size. + * + * The actual type doesn't matter here all that much as we're just going to do + * a load/store on it and never any arithmetic. + */ +static const struct glsl_type * +copy_type_for_byte_size(unsigned size) +{ + switch (size) { + case 1: return glsl_vector_type(GLSL_TYPE_UINT8, 1); + case 2: return glsl_vector_type(GLSL_TYPE_UINT16, 1); + case 4: return glsl_vector_type(GLSL_TYPE_UINT, 1); + case 8: return glsl_vector_type(GLSL_TYPE_UINT, 2); + case 16: return glsl_vector_type(GLSL_TYPE_UINT, 4); + default: + unreachable("Unsupported size"); + } +} + +static nir_ssa_def * +memcpy_load_deref_elem(nir_builder *b, nir_deref_instr *parent, + nir_ssa_def *index) +{ + nir_deref_instr *deref; + + index = nir_i2i(b, index, nir_dest_bit_size(parent->dest)); + assert(parent->deref_type == nir_deref_type_cast); + deref = nir_build_deref_ptr_as_array(b, parent, index); + + return nir_load_deref(b, deref); +} + +static nir_ssa_def * +memcpy_load_deref_elem_imm(nir_builder *b, nir_deref_instr *parent, + uint64_t index) +{ + nir_ssa_def *idx = nir_imm_intN_t(b, index, parent->dest.ssa.bit_size); + return memcpy_load_deref_elem(b, parent, idx); +} + +static void +memcpy_store_deref_elem(nir_builder *b, nir_deref_instr *parent, + nir_ssa_def *index, nir_ssa_def *value) +{ + nir_deref_instr *deref; + + index = nir_i2i(b, index, nir_dest_bit_size(parent->dest)); + assert(parent->deref_type == nir_deref_type_cast); + deref = nir_build_deref_ptr_as_array(b, parent, index); + nir_store_deref(b, deref, value, 1); +} + +static void +memcpy_store_deref_elem_imm(nir_builder *b, nir_deref_instr *parent, + uint64_t index, nir_ssa_def *value) +{ + nir_ssa_def *idx = nir_imm_intN_t(b, index, parent->dest.ssa.bit_size); + memcpy_store_deref_elem(b, parent, idx, value); +} + +static bool +lower_memcpy_impl(nir_function_impl *impl) +{ + nir_builder b; + nir_builder_init(&b, impl); + + bool found_const_memcpy = false, found_non_const_memcpy = false; + + nir_foreach_block_safe(block, impl) { + nir_foreach_instr_safe(instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + + nir_intrinsic_instr *cpy = nir_instr_as_intrinsic(instr); + if (cpy->intrinsic != nir_intrinsic_memcpy_deref) + continue; + + b.cursor = nir_instr_remove(&cpy->instr); + + nir_deref_instr *dst = nir_src_as_deref(cpy->src[0]); + nir_deref_instr *src = nir_src_as_deref(cpy->src[1]); + if (nir_src_is_const(cpy->src[2])) { + found_const_memcpy = true; + uint64_t size = nir_src_as_uint(cpy->src[2]); + uint64_t offset = 0; + while (offset < size) { + uint64_t remaining = offset - size; + /* For our chunk size, we choose the largest power-of-two that + * divides size with a maximum of 16B (a vec4). + */ + unsigned copy_size = 1u << MIN2(ffsll(remaining) - 1, 4); + const struct glsl_type *copy_type = + copy_type_for_byte_size(copy_size); + + nir_deref_instr *copy_dst = + nir_build_deref_cast(&b, &dst->dest.ssa, dst->mode, + copy_type, copy_size); + nir_deref_instr *copy_src = + nir_build_deref_cast(&b, &src->dest.ssa, src->mode, + copy_type, copy_size); + + uint64_t index = offset / copy_size; + nir_ssa_def *value = + memcpy_load_deref_elem_imm(&b, copy_src, index); + memcpy_store_deref_elem_imm(&b, copy_dst, index, value); + offset += copy_size; + } + } else { + found_non_const_memcpy = true; + assert(cpy->src[2].is_ssa); + nir_ssa_def *size = cpy->src[2].ssa; + + /* In this case, we don't have any idea what the size is so we + * emit a loop which copies one byte at a time. + */ + nir_deref_instr *copy_dst = + nir_build_deref_cast(&b, &dst->dest.ssa, dst->mode, + glsl_uint8_t_type(), 1); + nir_deref_instr *copy_src = + nir_build_deref_cast(&b, &src->dest.ssa, src->mode, + glsl_uint8_t_type(), 1); + + nir_variable *i = nir_local_variable_create(impl, + glsl_uintN_t_type(size->bit_size), NULL); + nir_store_var(&b, i, nir_imm_intN_t(&b, 0, size->bit_size), ~0); + nir_push_loop(&b); + { + nir_ssa_def *index = nir_load_var(&b, i); + nir_push_if(&b, nir_uge(&b, index, size)); + { + nir_jump(&b, nir_jump_break); + } + nir_pop_if(&b, NULL); + + nir_ssa_def *value = + memcpy_load_deref_elem(&b, copy_src, index); + memcpy_store_deref_elem(&b, copy_dst, index, value); + nir_store_var(&b, i, nir_iadd_imm(&b, index, 1), ~0); + } + nir_pop_loop(&b, NULL); + } + } + } + + if (found_non_const_memcpy) { + nir_metadata_preserve(impl, nir_metadata_none); + } else if (found_const_memcpy) { + nir_metadata_preserve(impl, nir_metadata_block_index | + nir_metadata_dominance); + } else { + nir_metadata_preserve(impl, nir_metadata_all); + } + + return found_const_memcpy || found_non_const_memcpy; +} + +bool +nir_lower_memcpy(nir_shader *shader) +{ + bool progress = false; + + nir_foreach_function(function, shader) { + if (function->impl && lower_memcpy_impl(function->impl)) + progress = true; + } + + return progress; +}