diff --git a/src/amd/llvm/ac_nir_to_llvm.c b/src/amd/llvm/ac_nir_to_llvm.c index 03e5e991a0b..3d47ab263ee 100644 --- a/src/amd/llvm/ac_nir_to_llvm.c +++ b/src/amd/llvm/ac_nir_to_llvm.c @@ -4574,7 +4574,7 @@ static void visit_deref(struct ac_nir_context *ctx, nir_deref_instr *instr) unsigned address_space; - switch (instr->mode) { + switch (instr->modes) { case nir_var_mem_shared: address_space = AC_ADDR_SPACE_LDS; break; diff --git a/src/compiler/glsl/gl_nir_link_uniforms.c b/src/compiler/glsl/gl_nir_link_uniforms.c index a8bc48fdd88..f8a6cc8728e 100644 --- a/src/compiler/glsl/gl_nir_link_uniforms.c +++ b/src/compiler/glsl/gl_nir_link_uniforms.c @@ -484,7 +484,7 @@ add_var_use_deref(nir_deref_instr *deref, struct hash_table *live, util_dynarray_append(ainfo->deref_list, nir_deref_instr *, deref); } - assert(deref->mode == deref->var->data.mode); + assert(deref->modes == deref->var->data.mode); _mesa_hash_table_insert(live, deref->var->name, ainfo); } diff --git a/src/compiler/glsl/gl_nir_lower_buffers.c b/src/compiler/glsl/gl_nir_lower_buffers.c index dcbe783f77c..a9a74ffce0a 100644 --- a/src/compiler/glsl/gl_nir_lower_buffers.c +++ b/src/compiler/glsl/gl_nir_lower_buffers.c @@ -209,7 +209,7 @@ lower_buffer_interface_derefs_impl(nir_function_impl *impl, break; } - nir_deref_instr *cast = nir_build_deref_cast(&b, ptr, deref->mode, + nir_deref_instr *cast = nir_build_deref_cast(&b, ptr, deref->modes, deref->type, 0); /* Set the alignment on the cast so that we get good alignment out * of nir_lower_explicit_io. Our offset to the start of the UBO diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index cb23c9999bb..71c8bad62b2 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -1443,12 +1443,16 @@ typedef struct { /** The type of this deref instruction */ nir_deref_type deref_type; - /** The mode of the underlying variable + /** Bitmask what modes the underlying variable might be + * + * For OpenCL-style generic pointers, we may not know exactly what mode it + * is at any given point in time in the compile process. This bitfield + * contains the set of modes which it MAY be. * * Generally, this field should not be accessed directly. Use one of the * nir_deref_mode_ helpers instead. */ - nir_variable_mode mode; + nir_variable_mode modes; /** The dereferenced type of the resulting pointer value */ const struct glsl_type *type; @@ -1493,8 +1497,8 @@ static inline bool nir_deref_mode_may_be(const nir_deref_instr *deref, nir_variable_mode modes) { assert(!(modes & ~nir_var_all)); - assert(deref->mode != 0); - return deref->mode & modes; + assert(deref->modes != 0); + return deref->modes & modes; } /** Returns true if deref must have one of the given modes @@ -1509,8 +1513,8 @@ static inline bool nir_deref_mode_must_be(const nir_deref_instr *deref, nir_variable_mode modes) { assert(!(modes & ~nir_var_all)); - assert(deref->mode != 0); - return !(deref->mode & ~modes); + assert(deref->modes != 0); + return !(deref->modes & ~modes); } /** Returns true if deref has the given mode @@ -1526,16 +1530,17 @@ static inline bool nir_deref_mode_is(const nir_deref_instr *deref, nir_variable_mode mode) { assert(util_bitcount(mode) == 1 && (mode & nir_var_all)); + assert(deref->modes != 0); /* This is only for "simple" cases so, if modes might interact with this * deref then the deref has to have a single mode. */ if (nir_deref_mode_may_be(deref, mode)) { - assert(util_bitcount(deref->mode) == 1); - assert(deref->mode == mode); + assert(util_bitcount(deref->modes) == 1); + assert(deref->modes == mode); } - return deref->mode == mode; + return deref->modes == mode; } /** Returns true if deref has one of the given modes @@ -1552,7 +1557,7 @@ nir_deref_mode_is_one_of(const nir_deref_instr *deref, nir_variable_mode modes) * deref then the deref has to have a single mode. */ if (nir_deref_mode_may_be(deref, modes)) { - assert(util_bitcount(deref->mode) == 1); + assert(util_bitcount(deref->modes) == 1); assert(nir_deref_mode_must_be(deref, modes)); } diff --git a/src/compiler/nir/nir_builder.h b/src/compiler/nir/nir_builder.h index c92b4053151..96c89c3822d 100644 --- a/src/compiler/nir/nir_builder.h +++ b/src/compiler/nir/nir_builder.h @@ -1084,7 +1084,7 @@ nir_build_deref_var(nir_builder *build, nir_variable *var) nir_deref_instr *deref = nir_deref_instr_create(build->shader, nir_deref_type_var); - deref->mode = (nir_variable_mode)var->data.mode; + deref->modes = (nir_variable_mode)var->data.mode; deref->type = var->type; deref->var = var; @@ -1109,7 +1109,7 @@ nir_build_deref_array(nir_builder *build, nir_deref_instr *parent, nir_deref_instr *deref = nir_deref_instr_create(build->shader, nir_deref_type_array); - deref->mode = parent->mode; + deref->modes = parent->modes; deref->type = glsl_get_array_element(parent->type); deref->parent = nir_src_for_ssa(&parent->dest.ssa); deref->arr.index = nir_src_for_ssa(index); @@ -1147,7 +1147,7 @@ nir_build_deref_ptr_as_array(nir_builder *build, nir_deref_instr *parent, nir_deref_instr *deref = nir_deref_instr_create(build->shader, nir_deref_type_ptr_as_array); - deref->mode = parent->mode; + deref->modes = parent->modes; deref->type = parent->type; deref->parent = nir_src_for_ssa(&parent->dest.ssa); deref->arr.index = nir_src_for_ssa(index); @@ -1170,7 +1170,7 @@ nir_build_deref_array_wildcard(nir_builder *build, nir_deref_instr *parent) nir_deref_instr *deref = nir_deref_instr_create(build->shader, nir_deref_type_array_wildcard); - deref->mode = parent->mode; + deref->modes = parent->modes; deref->type = glsl_get_array_element(parent->type); deref->parent = nir_src_for_ssa(&parent->dest.ssa); @@ -1192,7 +1192,7 @@ nir_build_deref_struct(nir_builder *build, nir_deref_instr *parent, nir_deref_instr *deref = nir_deref_instr_create(build->shader, nir_deref_type_struct); - deref->mode = parent->mode; + deref->modes = parent->modes; deref->type = glsl_get_struct_field(parent->type, index); deref->parent = nir_src_for_ssa(&parent->dest.ssa); deref->strct.index = index; @@ -1208,13 +1208,13 @@ nir_build_deref_struct(nir_builder *build, nir_deref_instr *parent, static inline nir_deref_instr * nir_build_deref_cast(nir_builder *build, nir_ssa_def *parent, - nir_variable_mode mode, const struct glsl_type *type, + nir_variable_mode modes, const struct glsl_type *type, unsigned ptr_stride) { nir_deref_instr *deref = nir_deref_instr_create(build->shader, nir_deref_type_cast); - deref->mode = mode; + deref->modes = modes; deref->type = type; deref->parent = nir_src_for_ssa(parent); deref->cast.ptr_stride = ptr_stride; @@ -1234,7 +1234,7 @@ nir_alignment_deref_cast(nir_builder *build, nir_deref_instr *parent, nir_deref_instr *deref = nir_deref_instr_create(build->shader, nir_deref_type_cast); - deref->mode = parent->mode; + deref->modes = parent->modes; deref->type = parent->type; deref->parent = nir_src_for_ssa(&parent->dest.ssa); deref->cast.ptr_stride = nir_deref_instr_array_stride(deref); diff --git a/src/compiler/nir/nir_clone.c b/src/compiler/nir/nir_clone.c index 5caf3fc3939..eca80fc9308 100644 --- a/src/compiler/nir/nir_clone.c +++ b/src/compiler/nir/nir_clone.c @@ -312,7 +312,7 @@ clone_deref_instr(clone_state *state, const nir_deref_instr *deref) __clone_dst(state, &nderef->instr, &nderef->dest, &deref->dest); - nderef->mode = deref->mode; + nderef->modes = deref->modes; nderef->type = deref->type; if (deref->deref_type == nir_deref_type_var) { diff --git a/src/compiler/nir/nir_deref.c b/src/compiler/nir/nir_deref.c index 55209299671..6af7580dd83 100644 --- a/src/compiler/nir/nir_deref.c +++ b/src/compiler/nir/nir_deref.c @@ -33,7 +33,7 @@ is_trivial_deref_cast(nir_deref_instr *cast) if (!parent) return false; - return cast->mode == parent->mode && + return cast->modes == parent->modes && cast->type == parent->type && cast->dest.ssa.num_components == parent->dest.ssa.num_components && cast->dest.ssa.bit_size == parent->dest.ssa.bit_size; @@ -410,17 +410,17 @@ nir_fixup_deref_modes(nir_shader *shader) if (deref->deref_type == nir_deref_type_cast) continue; - nir_variable_mode parent_mode; + nir_variable_mode parent_modes; if (deref->deref_type == nir_deref_type_var) { - parent_mode = deref->var->data.mode; + parent_modes = deref->var->data.mode; } else { assert(deref->parent.is_ssa); nir_deref_instr *parent = nir_instr_as_deref(deref->parent.ssa->parent_instr); - parent_mode = parent->mode; + parent_modes = parent->modes; } - deref->mode = parent_mode; + deref->modes = parent_modes; } } } @@ -430,17 +430,12 @@ static bool modes_may_alias(nir_variable_mode a, nir_variable_mode b) { /* Generic pointers can alias with SSBOs */ - if ((a == nir_var_mem_ssbo || a == nir_var_mem_global) && - (b == nir_var_mem_ssbo || b == nir_var_mem_global)) + if ((a & (nir_var_mem_ssbo | nir_var_mem_global)) && + (b & (nir_var_mem_ssbo | nir_var_mem_global))) return true; - /* In the general case, pointers can only alias if they have the same mode. - * - * NOTE: In future, with things like OpenCL generic pointers, this may not - * be true and will have to be re-evaluated. However, with graphics only, - * it should be safe. - */ - return a == b; + /* Pointers can only alias if they share a mode. */ + return a & b; } static bool @@ -469,7 +464,7 @@ nir_deref_compare_result nir_compare_deref_paths(nir_deref_path *a_path, nir_deref_path *b_path) { - if (!modes_may_alias(b_path->path[0]->mode, a_path->path[0]->mode)) + if (!modes_may_alias(b_path->path[0]->modes, a_path->path[0]->modes)) return nir_derefs_do_not_alias; if (a_path->path[0]->deref_type != b_path->path[0]->deref_type) @@ -482,8 +477,8 @@ nir_compare_deref_paths(nir_deref_path *a_path, */ static const nir_variable_mode temp_var_modes = nir_var_shader_temp | nir_var_function_temp; - if ((a_path->path[0]->mode & temp_var_modes) || - (b_path->path[0]->mode & temp_var_modes)) + if (!(a_path->path[0]->modes & ~temp_var_modes) || + !(b_path->path[0]->modes & ~temp_var_modes)) return nir_derefs_do_not_alias; /* If they are both declared coherent or have coherent somewhere in @@ -670,7 +665,7 @@ rematerialize_deref_in_block(nir_deref_instr *deref, nir_builder *b = &state->builder; nir_deref_instr *new_deref = nir_deref_instr_create(b->shader, deref->deref_type); - new_deref->mode = deref->mode; + new_deref->modes = deref->modes; new_deref->type = deref->type; if (deref->deref_type == nir_deref_type_var) { diff --git a/src/compiler/nir/nir_instr_set.c b/src/compiler/nir/nir_instr_set.c index c30d7317b94..dce7b2ac352 100644 --- a/src/compiler/nir/nir_instr_set.c +++ b/src/compiler/nir/nir_instr_set.c @@ -152,7 +152,7 @@ static uint32_t hash_deref(uint32_t hash, const nir_deref_instr *instr) { hash = HASH(hash, instr->deref_type); - hash = HASH(hash, instr->mode); + hash = HASH(hash, instr->modes); hash = HASH(hash, instr->type); if (instr->deref_type == nir_deref_type_var) @@ -602,7 +602,7 @@ nir_instrs_equal(const nir_instr *instr1, const nir_instr *instr2) nir_deref_instr *deref2 = nir_instr_as_deref(instr2); if (deref1->deref_type != deref2->deref_type || - deref1->mode != deref2->mode || + deref1->modes != deref2->modes || deref1->type != deref2->type) return false; diff --git a/src/compiler/nir/nir_lower_io.c b/src/compiler/nir/nir_lower_io.c index f376c211b54..89b027fa866 100644 --- a/src/compiler/nir/nir_lower_io.c +++ b/src/compiler/nir/nir_lower_io.c @@ -1097,7 +1097,7 @@ build_explicit_io_load(nir_builder *b, nir_intrinsic_instr *intrin, unsigned num_components) { nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]); - nir_variable_mode mode = deref->mode; + nir_variable_mode mode = deref->modes; nir_intrinsic_op op; switch (mode) { @@ -1241,7 +1241,7 @@ build_explicit_io_store(nir_builder *b, nir_intrinsic_instr *intrin, uint32_t align_mul, uint32_t align_offset, nir_ssa_def *value, nir_component_mask_t write_mask) { - nir_variable_mode mode = nir_src_as_deref(intrin->src[0])->mode; + nir_variable_mode mode = nir_src_as_deref(intrin->src[0])->modes; nir_intrinsic_op op; switch (mode) { @@ -1329,7 +1329,7 @@ static nir_ssa_def * build_explicit_io_atomic(nir_builder *b, nir_intrinsic_instr *intrin, nir_ssa_def *addr, nir_address_format addr_format) { - nir_variable_mode mode = nir_src_as_deref(intrin->src[0])->mode; + nir_variable_mode mode = nir_src_as_deref(intrin->src[0])->modes; const unsigned num_data_srcs = nir_intrinsic_infos[intrin->intrinsic].num_srcs - 1; diff --git a/src/compiler/nir/nir_lower_memcpy.c b/src/compiler/nir/nir_lower_memcpy.c index 9ffb3bfa1d5..b7a3f1752cb 100644 --- a/src/compiler/nir/nir_lower_memcpy.c +++ b/src/compiler/nir/nir_lower_memcpy.c @@ -120,10 +120,10 @@ lower_memcpy_impl(nir_function_impl *impl) copy_type_for_byte_size(copy_size); nir_deref_instr *copy_dst = - nir_build_deref_cast(&b, &dst->dest.ssa, dst->mode, + nir_build_deref_cast(&b, &dst->dest.ssa, dst->modes, copy_type, copy_size); nir_deref_instr *copy_src = - nir_build_deref_cast(&b, &src->dest.ssa, src->mode, + nir_build_deref_cast(&b, &src->dest.ssa, src->modes, copy_type, copy_size); uint64_t index = offset / copy_size; @@ -141,10 +141,10 @@ lower_memcpy_impl(nir_function_impl *impl) * emit a loop which copies one byte at a time. */ nir_deref_instr *copy_dst = - nir_build_deref_cast(&b, &dst->dest.ssa, dst->mode, + nir_build_deref_cast(&b, &dst->dest.ssa, dst->modes, glsl_uint8_t_type(), 1); nir_deref_instr *copy_src = - nir_build_deref_cast(&b, &src->dest.ssa, src->mode, + nir_build_deref_cast(&b, &src->dest.ssa, src->modes, glsl_uint8_t_type(), 1); nir_variable *i = nir_local_variable_create(impl, diff --git a/src/compiler/nir/nir_lower_memory_model.c b/src/compiler/nir/nir_lower_memory_model.c index d2ba829c89c..e29d97f3242 100644 --- a/src/compiler/nir/nir_lower_memory_model.c +++ b/src/compiler/nir/nir_lower_memory_model.c @@ -31,16 +31,16 @@ #include "shader_enums.h" static bool -get_intrinsic_info(nir_intrinsic_instr *intrin, nir_variable_mode *mode, +get_intrinsic_info(nir_intrinsic_instr *intrin, nir_variable_mode *modes, bool *reads, bool *writes) { switch (intrin->intrinsic) { case nir_intrinsic_image_deref_load: - *mode = nir_src_as_deref(intrin->src[0])->mode; + *modes = nir_src_as_deref(intrin->src[0])->modes; *reads = true; break; case nir_intrinsic_image_deref_store: - *mode = nir_src_as_deref(intrin->src[0])->mode; + *modes = nir_src_as_deref(intrin->src[0])->modes; *writes = true; break; case nir_intrinsic_image_deref_atomic_add: @@ -53,16 +53,16 @@ get_intrinsic_info(nir_intrinsic_instr *intrin, nir_variable_mode *mode, case nir_intrinsic_image_deref_atomic_xor: case nir_intrinsic_image_deref_atomic_exchange: case nir_intrinsic_image_deref_atomic_comp_swap: - *mode = nir_src_as_deref(intrin->src[0])->mode; + *modes = nir_src_as_deref(intrin->src[0])->modes; *reads = true; *writes = true; break; case nir_intrinsic_load_ssbo: - *mode = nir_var_mem_ssbo; + *modes = nir_var_mem_ssbo; *reads = true; break; case nir_intrinsic_store_ssbo: - *mode = nir_var_mem_ssbo; + *modes = nir_var_mem_ssbo; *writes = true; break; case nir_intrinsic_ssbo_atomic_add: @@ -75,16 +75,16 @@ get_intrinsic_info(nir_intrinsic_instr *intrin, nir_variable_mode *mode, case nir_intrinsic_ssbo_atomic_xor: case nir_intrinsic_ssbo_atomic_exchange: case nir_intrinsic_ssbo_atomic_comp_swap: - *mode = nir_var_mem_ssbo; + *modes = nir_var_mem_ssbo; *reads = true; *writes = true; break; case nir_intrinsic_load_global: - *mode = nir_var_mem_global; + *modes = nir_var_mem_global; *reads = true; break; case nir_intrinsic_store_global: - *mode = nir_var_mem_global; + *modes = nir_var_mem_global; *writes = true; break; case nir_intrinsic_global_atomic_add: @@ -97,16 +97,16 @@ get_intrinsic_info(nir_intrinsic_instr *intrin, nir_variable_mode *mode, case nir_intrinsic_global_atomic_xor: case nir_intrinsic_global_atomic_exchange: case nir_intrinsic_global_atomic_comp_swap: - *mode = nir_var_mem_global; + *modes = nir_var_mem_global; *reads = true; *writes = true; break; case nir_intrinsic_load_deref: - *mode = nir_src_as_deref(intrin->src[0])->mode; + *modes = nir_src_as_deref(intrin->src[0])->modes; *reads = true; break; case nir_intrinsic_store_deref: - *mode = nir_src_as_deref(intrin->src[0])->mode; + *modes = nir_src_as_deref(intrin->src[0])->modes; *writes = true; break; case nir_intrinsic_deref_atomic_add: @@ -119,7 +119,7 @@ get_intrinsic_info(nir_intrinsic_instr *intrin, nir_variable_mode *mode, case nir_intrinsic_deref_atomic_xor: case nir_intrinsic_deref_atomic_exchange: case nir_intrinsic_deref_atomic_comp_swap: - *mode = nir_src_as_deref(intrin->src[0])->mode; + *modes = nir_src_as_deref(intrin->src[0])->modes; *reads = true; *writes = true; break; @@ -149,9 +149,9 @@ visit_instr(nir_instr *instr, uint32_t *cur_modes, unsigned vis_avail_sem) if (!*cur_modes) return false; /* early exit */ - nir_variable_mode mode; + nir_variable_mode modes; bool reads = false, writes = false; - if (!get_intrinsic_info(intrin, &mode, &reads, &writes)) + if (!get_intrinsic_info(intrin, &modes, &reads, &writes)) return false; if (!reads && vis_avail_sem == NIR_MEMORY_MAKE_VISIBLE) @@ -167,7 +167,7 @@ visit_instr(nir_instr *instr, uint32_t *cur_modes, unsigned vis_avail_sem) if (access & (ACCESS_NON_READABLE | ACCESS_NON_WRITEABLE | ACCESS_CAN_REORDER | ACCESS_COHERENT)) return false; - if (*cur_modes & mode) { + if (*cur_modes & modes) { nir_intrinsic_set_access(intrin, access | ACCESS_COHERENT); return true; } diff --git a/src/compiler/nir/nir_opt_load_store_vectorize.c b/src/compiler/nir/nir_opt_load_store_vectorize.c index 8f1074de38d..68296940931 100644 --- a/src/compiler/nir/nir_opt_load_store_vectorize.c +++ b/src/compiler/nir/nir_opt_load_store_vectorize.c @@ -513,8 +513,8 @@ get_variable_mode(struct entry *entry) { if (entry->info->mode) return entry->info->mode; - assert(entry->deref); - return entry->deref->mode; + assert(entry->deref && util_bitcount(entry->deref->modes) == 1); + return entry->deref->modes; } static unsigned @@ -622,7 +622,7 @@ cast_deref(nir_builder *b, unsigned num_components, unsigned bit_size, nir_deref if (deref->type == type) return deref; - return nir_build_deref_cast(b, &deref->dest.ssa, deref->mode, type, 0); + return nir_build_deref_cast(b, &deref->dest.ssa, deref->modes, type, 0); } /* Return true if "new_bit_size" is a usable bit size for a vectorized load/store @@ -697,7 +697,7 @@ static nir_deref_instr *subtract_deref(nir_builder *b, nir_deref_instr *deref, i } - deref = nir_build_deref_cast(b, &deref->dest.ssa, deref->mode, + deref = nir_build_deref_cast(b, &deref->dest.ssa, deref->modes, glsl_scalar_type(GLSL_TYPE_UINT8), 1); return nir_build_deref_ptr_as_array( b, deref, nir_imm_intN_t(b, -offset, deref->dest.ssa.bit_size)); @@ -1263,7 +1263,7 @@ process_block(nir_function_impl *impl, struct vectorize_ctx *ctx, nir_block *blo nir_variable_mode mode = info->mode; if (!mode) - mode = nir_src_as_deref(intrin->src[info->deref_src])->mode; + mode = nir_src_as_deref(intrin->src[info->deref_src])->modes; if (!(mode & aliasing_modes(ctx->modes))) continue; unsigned mode_index = mode_to_index(mode); diff --git a/src/compiler/nir/nir_opt_peephole_select.c b/src/compiler/nir/nir_opt_peephole_select.c index 590fec82405..be0f3081fad 100644 --- a/src/compiler/nir/nir_opt_peephole_select.c +++ b/src/compiler/nir/nir_opt_peephole_select.c @@ -72,7 +72,7 @@ block_check_for_allowed_instrs(nir_block *block, unsigned *count, case nir_intrinsic_load_deref: { nir_deref_instr *const deref = nir_src_as_deref(intrin->src[0]); - switch (deref->mode) { + switch (deref->modes) { case nir_var_shader_in: case nir_var_uniform: /* Don't try to remove flow control around an indirect load diff --git a/src/compiler/nir/nir_print.c b/src/compiler/nir/nir_print.c index 0790d02ca4a..44aa645b59d 100644 --- a/src/compiler/nir/nir_print.c +++ b/src/compiler/nir/nir_print.c @@ -724,7 +724,7 @@ print_deref_instr(nir_deref_instr *instr, print_state *state) print_deref_link(instr, false, state); fprintf(fp, " (%s %s) ", - get_variable_mode_str(instr->mode, true), + get_variable_mode_str(instr->modes, true), glsl_get_type_name(instr->type)); if (instr->deref_type != nir_deref_type_var && diff --git a/src/compiler/nir/nir_remove_dead_variables.c b/src/compiler/nir/nir_remove_dead_variables.c index 3d1df3a58c9..01ea18cc37e 100644 --- a/src/compiler/nir/nir_remove_dead_variables.c +++ b/src/compiler/nir/nir_remove_dead_variables.c @@ -108,17 +108,17 @@ remove_dead_var_writes(nir_shader *shader, struct set *live) !nir_deref_instr_parent(deref)) continue; - nir_variable_mode parent_mode; + nir_variable_mode parent_modes; if (deref->deref_type == nir_deref_type_var) - parent_mode = deref->var->data.mode; + parent_modes = deref->var->data.mode; else - parent_mode = nir_deref_instr_parent(deref)->mode; + parent_modes = nir_deref_instr_parent(deref)->modes; /* If the parent mode is 0, then it references a dead variable. * Flag this deref as dead and remove it. */ - if (parent_mode == 0) { - deref->mode = 0; + if (parent_modes == 0) { + deref->modes = 0; nir_instr_remove(&deref->instr); } break; @@ -130,7 +130,7 @@ remove_dead_var_writes(nir_shader *shader, struct set *live) intrin->intrinsic != nir_intrinsic_store_deref) break; - if (nir_src_as_deref(intrin->src[0])->mode == 0) + if (nir_src_as_deref(intrin->src[0])->modes == 0) nir_instr_remove(instr); break; } diff --git a/src/compiler/nir/nir_repair_ssa.c b/src/compiler/nir/nir_repair_ssa.c index ce5465ebf77..2c9073ffc1e 100644 --- a/src/compiler/nir/nir_repair_ssa.c +++ b/src/compiler/nir/nir_repair_ssa.c @@ -125,7 +125,7 @@ repair_ssa_def(nir_ssa_def *def, void *void_state) nir_deref_type_cast); nir_deref_instr *deref = nir_instr_as_deref(def->parent_instr); - cast->mode = deref->mode; + cast->modes = deref->modes; cast->type = deref->type; cast->parent = nir_src_for_ssa(block_def); cast->cast.ptr_stride = nir_deref_instr_array_stride(deref); diff --git a/src/compiler/nir/nir_serialize.c b/src/compiler/nir/nir_serialize.c index e2f99a2f20c..2617a731f9e 100644 --- a/src/compiler/nir/nir_serialize.c +++ b/src/compiler/nir/nir_serialize.c @@ -637,7 +637,7 @@ union packed_instr { unsigned instr_type:4; unsigned deref_type:3; unsigned cast_type_same_as_last:1; - unsigned mode:12; /* deref_var redefines this */ + unsigned modes:12; /* deref_var redefines this */ unsigned packed_src_ssa_16bit:1; /* deref_var redefines this */ unsigned _pad:3; /* deref_var redefines this */ unsigned dest:8; @@ -984,7 +984,7 @@ static void write_deref(write_ctx *ctx, const nir_deref_instr *deref) { assert(deref->deref_type < 8); - assert(deref->mode < (1 << 12)); + assert(deref->modes < (1 << 12)); union packed_instr header; header.u32 = 0; @@ -993,7 +993,7 @@ write_deref(write_ctx *ctx, const nir_deref_instr *deref) header.deref.deref_type = deref->deref_type; if (deref->deref_type == nir_deref_type_cast) { - header.deref.mode = deref->mode; + header.deref.modes = deref->modes; header.deref.cast_type_same_as_last = deref->type == ctx->last_type; } @@ -1127,12 +1127,12 @@ read_deref(read_ctx *ctx, union packed_instr header) } if (deref_type == nir_deref_type_var) { - deref->mode = deref->var->data.mode; + deref->modes = deref->var->data.mode; } else if (deref->deref_type == nir_deref_type_cast) { - deref->mode = header.deref.mode; + deref->modes = header.deref.modes; } else { assert(deref->parent.is_ssa); - deref->mode = nir_instr_as_deref(deref->parent.ssa->parent_instr)->mode; + deref->modes = nir_instr_as_deref(deref->parent.ssa->parent_instr)->modes; } return deref; diff --git a/src/compiler/nir/nir_to_lcssa.c b/src/compiler/nir/nir_to_lcssa.c index de2c7b600bd..527cdca4120 100644 --- a/src/compiler/nir/nir_to_lcssa.c +++ b/src/compiler/nir/nir_to_lcssa.c @@ -250,7 +250,7 @@ convert_loop_exit_for_ssa(nir_ssa_def *def, void *void_state) nir_deref_instr_create(state->shader, nir_deref_type_cast); nir_deref_instr *instr = nir_instr_as_deref(def->parent_instr); - cast->mode = instr->mode; + cast->modes = instr->modes; cast->type = instr->type; cast->parent = nir_src_for_ssa(&phi->dest.ssa); cast->cast.ptr_stride = nir_deref_instr_array_stride(instr); diff --git a/src/compiler/nir/nir_validate.c b/src/compiler/nir/nir_validate.c index f3d90facead..ee1c5c13c9b 100644 --- a/src/compiler/nir/nir_validate.c +++ b/src/compiler/nir/nir_validate.c @@ -407,9 +407,11 @@ validate_var_use(nir_variable *var, validate_state *state) static void validate_deref_instr(nir_deref_instr *instr, validate_state *state) { + validate_assert(state, util_bitcount(instr->modes) == 1); + if (instr->deref_type == nir_deref_type_var) { /* Variable dereferences are stupid simple. */ - validate_assert(state, instr->mode == instr->var->data.mode); + validate_assert(state, instr->modes == instr->var->data.mode); validate_assert(state, instr->type == instr->var->type); validate_var_use(instr->var, state); } else if (instr->deref_type == nir_deref_type_cast) { @@ -419,7 +421,7 @@ validate_deref_instr(nir_deref_instr *instr, validate_state *state) validate_src(&instr->parent, state, 0, 0); /* We just validate that the type and mode are there */ - validate_assert(state, instr->mode); + validate_assert(state, instr->modes); validate_assert(state, instr->type); if (instr->cast.align_mul > 0) { validate_assert(state, util_is_power_of_two_nonzero(instr->cast.align_mul)); @@ -444,7 +446,7 @@ validate_deref_instr(nir_deref_instr *instr, validate_state *state) nir_deref_instr *parent = nir_instr_as_deref(parent_instr); - validate_assert(state, instr->mode == parent->mode); + validate_assert(state, instr->modes == parent->modes); switch (instr->deref_type) { case nir_deref_type_struct: @@ -457,11 +459,9 @@ validate_deref_instr(nir_deref_instr *instr, validate_state *state) case nir_deref_type_array: case nir_deref_type_array_wildcard: - if (instr->mode == nir_var_mem_ubo || - instr->mode == nir_var_mem_ssbo || - instr->mode == nir_var_mem_shared || - instr->mode == nir_var_mem_global || - instr->mode == nir_var_mem_push_const) { + if (instr->modes & (nir_var_mem_ubo | nir_var_mem_ssbo | + nir_var_mem_shared | nir_var_mem_global | + nir_var_mem_push_const)) { /* Shared variables and UBO/SSBOs have a bit more relaxed rules * because we need to be able to handle array derefs on vectors. * Fortunately, nir_lower_io handles these just fine. @@ -518,10 +518,10 @@ validate_deref_instr(nir_deref_instr *instr, validate_state *state) */ nir_foreach_use(use, &instr->dest.ssa) { if (use->parent_instr->type == nir_instr_type_phi) { - validate_assert(state, instr->mode != nir_var_shader_in && - instr->mode != nir_var_shader_out && - instr->mode != nir_var_shader_out && - instr->mode != nir_var_uniform); + validate_assert(state, !(instr->modes & (nir_var_shader_in | + nir_var_shader_out | + nir_var_shader_out | + nir_var_uniform))); } } } @@ -570,7 +570,7 @@ validate_intrinsic_instr(nir_intrinsic_instr *instr, validate_state *state) nir_deref_instr *src = nir_src_as_deref(instr->src[0]); assert(src); validate_assert(state, glsl_type_is_vector_or_scalar(src->type) || - (src->mode == nir_var_uniform && + (src->modes == nir_var_uniform && glsl_get_base_type(src->type) == GLSL_TYPE_SUBROUTINE)); validate_assert(state, instr->num_components == glsl_get_vector_elements(src->type)); diff --git a/src/compiler/spirv/vtn_variables.c b/src/compiler/spirv/vtn_variables.c index 1a8cad5a7b7..672d0872c30 100644 --- a/src/compiler/spirv/vtn_variables.c +++ b/src/compiler/spirv/vtn_variables.c @@ -430,7 +430,7 @@ vtn_pointer_dereference(struct vtn_builder *b, /* We start with a deref cast to get the stride. Hopefully, we'll be * able to delete that cast eventually. */ - tail = nir_build_deref_cast(&b->nb, &tail->dest.ssa, tail->mode, + tail = nir_build_deref_cast(&b->nb, &tail->dest.ssa, tail->modes, tail->type, base->ptr_type->stride); nir_ssa_def *index = vtn_access_link_as_ssa(b, deref_chain->link[0], 1, diff --git a/src/gallium/auxiliary/gallivm/lp_bld_nir.c b/src/gallium/auxiliary/gallivm/lp_bld_nir.c index cc0f993d849..915c9d96919 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_nir.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_nir.c @@ -1129,7 +1129,8 @@ static void visit_load_var(struct lp_build_nir_context *bld_base, { nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr); nir_variable *var = nir_deref_instr_get_variable(deref); - nir_variable_mode mode = deref->mode; + assert(util_bitcount(deref->modes) == 1); + nir_variable_mode mode = deref->modes; unsigned const_index; LLVMValueRef indir_index; LLVMValueRef indir_vertex_index = NULL; @@ -1162,7 +1163,8 @@ visit_store_var(struct lp_build_nir_context *bld_base, { nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr); nir_variable *var = nir_deref_instr_get_variable(deref); - nir_variable_mode mode = deref->mode; + assert(util_bitcount(deref->modes) == 1); + nir_variable_mode mode = deref->modes; int writemask = instr->const_index[0]; unsigned bit_size = nir_src_bit_size(instr->src[1]); LLVMValueRef src = get_src(bld_base, instr->src[1]); diff --git a/src/gallium/drivers/r600/sfn/sfn_shader_base.cpp b/src/gallium/drivers/r600/sfn/sfn_shader_base.cpp index ec18b11fb22..60594448059 100644 --- a/src/gallium/drivers/r600/sfn/sfn_shader_base.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_shader_base.cpp @@ -311,7 +311,8 @@ void ShaderFromNirProcessor::set_var_address(nir_deref_instr *instr) { auto& dest = instr->dest; unsigned index = dest.is_ssa ? dest.ssa.index : dest.reg.reg->index; - m_var_mode[instr->var] = instr->mode; + assert(util_bitcount(instr->modes) == 1); + m_var_mode[instr->var] = instr->modes; m_var_derefs[index] = instr->var; sfn_log << SfnLog::io << "Add var deref:" << index