nir/lower_io: Add support for nir_var_mem_constant

This commit adds support for nir_var_mem_constant various places.  It
also adds a pass similar to nir_lower_vars_to_explicit_types except it
also scrapes out the constants and stuffs them into constant_data.

Reviewed-by: Eric Anholt <eric@anholt.net>
Reviewed-by: Jesse Natalie <jenatali@microsoft.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/6379>

src/compiler/nir/nir.h

@@ -4152,6 +4152,9 @@ nir_lower_vars_to_explicit_types(nir_shader *shader,
                                  nir_variable_mode modes,
                                  glsl_type_size_align_func type_info);
 
+bool nir_lower_mem_constant_vars(nir_shader *shader,
+                                 glsl_type_size_align_func type_info);
+
 typedef enum {
    /**
     * An address format which is a simple 32-bit global GPU address.

src/compiler/nir/nir_intrinsics.py

@@ -624,6 +624,7 @@ system_value("line_width", 1)
 system_value("aa_line_width", 1)
 # BASE=0 for global/shader, BASE=1 for local/function
 system_value("scratch_base_ptr", 0, bit_sizes=[32,64], indices=[BASE])
+system_value("constant_base_ptr", 0, bit_sizes=[32,64])
 
 # Driver-specific viewport scale/offset parameters.
 #

src/compiler/nir/nir_lower_io.c

@@ -807,7 +807,8 @@ build_addr_for_var(nir_builder *b, nir_variable *var,
                    nir_address_format addr_format)
 {
    assert(var->data.mode & (nir_var_uniform | nir_var_mem_shared |
-                            nir_var_shader_temp | nir_var_function_temp));
+                            nir_var_shader_temp | nir_var_function_temp |
+                            nir_var_mem_constant));
 
    const unsigned num_comps = nir_address_format_num_components(addr_format);
    const unsigned bit_size = nir_address_format_bit_size(addr_format);
@@ -825,6 +826,10 @@ build_addr_for_var(nir_builder *b, nir_variable *var,
          base_addr = nir_load_scratch_base_ptr(b, 1, num_comps, bit_size);
          break;
 
+      case nir_var_mem_constant:
+         base_addr = nir_load_constant_base_ptr(b, num_comps, bit_size);
+         break;
+
       default:
          unreachable("Unsupported variable mode");
       }
@@ -984,6 +989,14 @@ build_explicit_io_load(nir_builder *b, nir_intrinsic_instr *intrin,
          op = nir_intrinsic_load_global;
       }
       break;
+   case nir_var_mem_constant:
+      if (addr_format_is_offset(addr_format)) {
+         op = nir_intrinsic_load_constant;
+      } else {
+         assert(addr_format_is_global(addr_format));
+         op = nir_intrinsic_load_global_constant;
+      }
+      break;
    default:
       unreachable("Unsupported explicit IO variable mode");
    }
@@ -1003,6 +1016,11 @@ build_explicit_io_load(nir_builder *b, nir_intrinsic_instr *intrin,
    if (nir_intrinsic_has_access(load))
       nir_intrinsic_set_access(load, nir_intrinsic_access(intrin));
 
+   if (op == nir_intrinsic_load_constant) {
+      nir_intrinsic_set_base(load, 0);
+      nir_intrinsic_set_range(load, b->shader->constant_data_size);
+   }
+
    unsigned bit_size = intrin->dest.ssa.bit_size;
    if (bit_size == 1) {
       /* TODO: Make the native bool bit_size an option. */
@@ -1581,6 +1599,9 @@ lower_vars_to_explicit(nir_shader *shader,
    case nir_var_mem_shared:
       offset = 0;
       break;
+   case nir_var_mem_constant:
+      offset = shader->constant_data_size;
+      break;
    default:
       unreachable("Unsupported mode");
    }
@@ -1592,13 +1613,12 @@ lower_vars_to_explicit(nir_shader *shader,
       const struct glsl_type *explicit_type =
          glsl_get_explicit_type_for_size_align(var->type, type_info, &size, &align);
 
-      if (explicit_type != var->type) {
-         progress = true;
+      if (explicit_type != var->type)
          var->type = explicit_type;
-      }
 
       var->data.driver_location = ALIGN_POT(offset, align);
       offset = var->data.driver_location + size;
+      progress = true;
    }
 
    switch (mode) {
@@ -1610,6 +1630,9 @@ lower_vars_to_explicit(nir_shader *shader,
       shader->info.cs.shared_size = offset;
       shader->shared_size = offset;
       break;
+   case nir_var_mem_constant:
+      shader->constant_data_size = offset;
+      break;
    default:
       unreachable("Unsupported mode");
    }
@@ -1650,6 +1673,83 @@ nir_lower_vars_to_explicit_types(nir_shader *shader,
    return progress;
 }
 
+static void
+write_constant(void *dst, const nir_constant *c, const struct glsl_type *type)
+{
+   if (glsl_type_is_vector_or_scalar(type)) {
+      const unsigned num_components = glsl_get_vector_elements(type);
+      const unsigned bit_size = glsl_get_bit_size(type);
+      if (bit_size == 1) {
+         /* Booleans are special-cased to be 32-bit
+          *
+          * TODO: Make the native bool bit_size an option.
+          */
+         for (unsigned i = 0; i < num_components; i++) {
+            int32_t b32 = -(int)c->values[i].b;
+            memcpy((char *)dst + i * 4, &b32, 4);
+         }
+      } else {
+         assert(bit_size >= 8 && bit_size % 8 == 0);
+         const unsigned byte_size = bit_size / 8;
+         for (unsigned i = 0; i < num_components; i++) {
+            /* Annoyingly, thanks to packed structs, we can't make any
+             * assumptions about the alignment of dst.  To avoid any strange
+             * issues with unaligned writes, we always use memcpy.
+             */
+            memcpy((char *)dst + i * byte_size, &c->values[i], byte_size);
+         }
+      }
+   } else if (glsl_type_is_array_or_matrix(type)) {
+      const unsigned array_len = glsl_get_length(type);
+      const unsigned stride = glsl_get_explicit_stride(type);
+      assert(stride > 0);
+      const struct glsl_type *elem_type = glsl_get_array_element(type);
+      for (unsigned i = 0; i < array_len; i++)
+         write_constant((char *)dst + i * stride, c->elements[i], elem_type);
+   } else {
+      assert(glsl_type_is_struct_or_ifc(type));
+      const unsigned num_fields = glsl_get_length(type);
+      for (unsigned i = 0; i < num_fields; i++) {
+         const int field_offset = glsl_get_struct_field_offset(type, i);
+         assert(field_offset >= 0);
+         const struct glsl_type *field_type = glsl_get_struct_field(type, i);
+         write_constant((char *)dst + field_offset, c->elements[i], field_type);
+      }
+   }
+}
+
+bool
+nir_lower_mem_constant_vars(nir_shader *shader,
+                            glsl_type_size_align_func type_info)
+{
+   unsigned old_constant_data_size = shader->constant_data_size;
+   if (!lower_vars_to_explicit(shader, &shader->variables,
+                               nir_var_mem_constant, type_info)) {
+      nir_shader_preserve_all_metadata(shader);
+      return false;
+   }
+
+   shader->constant_data = rerzalloc_size(shader, shader->constant_data,
+                                          old_constant_data_size,
+                                          shader->constant_data_size);
+
+   nir_foreach_variable_with_modes(var, shader, nir_var_mem_constant) {
+      write_constant((char *)shader->constant_data + var->data.driver_location,
+                     var->constant_initializer, var->type);
+   }
+
+   nir_foreach_function(function, shader) {
+      if (!function->impl)
+         continue;
+
+      nir_lower_vars_to_explicit_types_impl(function->impl,
+                                            nir_var_mem_constant,
+                                            type_info);
+   }
+
+   return true;
+}
+
 /**
  * Return the offset source for a load/store intrinsic.
  */