nir/lower_io: Introduce nir_store_per_vertex_output intrinsics.

Similar to nir_load_per_vertex_input, but for outputs.  This is not
useful in geometry shaders, but will be useful in tessellation shaders.

v2: Change stage_uses_per_vertex_outputs() to is_per_vertex_output(),
    taking a nir_variable (requested by Jason Ekstrand).

Signed-off-by: Kenneth Graunke <kenneth@whitecape.org>
Reviewed-by: Jason Ekstrand <jason.ekstrand@intel.com>

src/glsl/nir/nir_intrinsics.h

@@ -272,6 +272,7 @@ LOAD(ssbo, 1, 1, NIR_INTRINSIC_CAN_ELIMINATE)
              false, 0, 0, 1 + extra_indices, flags)
 
 STORE(output, 0, 0, 0, 0)
+STORE(per_vertex_output, 1, 1, 0, 0)
 STORE(ssbo, 1, 1, 1, 0)
 
 LAST_INTRINSIC(store_ssbo_indirect)

src/glsl/nir/nir_lower_io.c

@@ -78,6 +78,14 @@ is_per_vertex_input(struct lower_io_state *state, nir_variable *var)
            stage == MESA_SHADER_GEOMETRY);
 }
 
+static bool
+is_per_vertex_output(struct lower_io_state *state, nir_variable *var)
+{
+   gl_shader_stage stage = state->builder.shader->stage;
+   return var->data.mode == nir_var_shader_out && !var->data.patch &&
+          stage == MESA_SHADER_TESS_CTRL;
+}
+
 static unsigned
 get_io_offset(nir_deref_var *deref, nir_instr *instr,
               nir_ssa_def **vertex_index,
@@ -237,16 +245,23 @@ nir_lower_io_block(nir_block *block, void *void_state)
             continue;
 
          nir_ssa_def *indirect;
+         nir_ssa_def *vertex_index;
+
+         bool per_vertex =
+            is_per_vertex_output(state, intrin->variables[0]->var);
 
          unsigned offset = get_io_offset(intrin->variables[0], &intrin->instr,
-                                         NULL, &indirect, state);
+                                         per_vertex ? &vertex_index : NULL,
+                                         &indirect, state);
          offset += intrin->variables[0]->var->data.driver_location;
 
          nir_intrinsic_op store_op;
-         if (indirect) {
-            store_op = nir_intrinsic_store_output_indirect;
+         if (per_vertex) {
+            store_op = indirect ? nir_intrinsic_store_per_vertex_output_indirect
+                                : nir_intrinsic_store_per_vertex_output;
          } else {
-            store_op = nir_intrinsic_store_output;
+            store_op = indirect ? nir_intrinsic_store_output_indirect
+                                : nir_intrinsic_store_output;
          }
 
          nir_intrinsic_instr *store = nir_intrinsic_instr_create(state->mem_ctx,
@@ -256,8 +271,11 @@ nir_lower_io_block(nir_block *block, void *void_state)
 
          nir_src_copy(&store->src[0], &intrin->src[0], store);
 
+         if (per_vertex)
+            store->src[1] = nir_src_for_ssa(vertex_index);
+
          if (indirect)
-            store->src[1] = nir_src_for_ssa(indirect);
+            store->src[per_vertex ? 2 : 1] = nir_src_for_ssa(indirect);
 
          nir_instr_insert_before(&intrin->instr, &store->instr);
          nir_instr_remove(&intrin->instr);

src/glsl/nir/nir_print.c

@@ -450,6 +450,8 @@ print_intrinsic_instr(nir_intrinsic_instr *instr, print_state *state)
       break;
    case nir_intrinsic_store_output:
    case nir_intrinsic_store_output_indirect:
+   case nir_intrinsic_store_per_vertex_output:
+   case nir_intrinsic_store_per_vertex_output_indirect:
       var_list = &state->shader->outputs;
       break;
    default: