nir: Add nir_intrinsic_terminate and nir_intrinsic_terminate_if

Reviewed-by: Jason Ekstrand <jason@jlekstrand.net>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/7150>

src/compiler/nir/nir_gather_info.c

@@ -337,6 +337,12 @@ gather_intrinsic_info(nir_intrinsic_instr *instr, nir_shader *shader,
          shader->info.fs.uses_discard = true;
       break;
 
+   case nir_intrinsic_terminate:
+   case nir_intrinsic_terminate_if:
+      assert(shader->info.stage == MESA_SHADER_FRAGMENT);
+      shader->info.fs.uses_discard = true;
+      break;
+
    case nir_intrinsic_interp_deref_at_centroid:
    case nir_intrinsic_interp_deref_at_sample:
    case nir_intrinsic_interp_deref_at_offset:

src/compiler/nir/nir_intrinsics.py

@@ -228,6 +228,9 @@ barrier("discard")
 barrier("demote")
 intrinsic("is_helper_invocation", dest_comp=1, flags=[CAN_ELIMINATE])
 
+# SpvOpTerminateInvocation from SPIR-V.  Essentially a discard "for real".
+barrier("terminate")
+
 # A workgroup-level control barrier.  Any thread which hits this barrier will
 # pause until all threads within the current workgroup have also hit the
 # barrier.  For compute shaders, the workgroup is defined as the local group.
@@ -288,9 +291,10 @@ barrier("end_invocation_interlock")
 # Memory barrier for synchronizing TCS patch outputs
 barrier("memory_barrier_tcs_patch")
 
-# A conditional discard/demote, with a single boolean source.
+# A conditional discard/demote/terminate, with a single boolean source.
 intrinsic("discard_if", src_comp=[1])
 intrinsic("demote_if", src_comp=[1])
+intrinsic("terminate_if", src_comp=[1])
 
 # ARB_shader_group_vote intrinsics
 intrinsic("vote_any", src_comp=[1], dest_comp=1, flags=[CAN_ELIMINATE])

src/compiler/nir/nir_opt_conditional_discard.c

@@ -29,6 +29,7 @@
  * Handles optimization of lowering of
  *  - if (cond) discard to discard_if(cond) and
  *  - if (cond) demote to demote_if(cond)
+ *  - if (cond) terminate to terminate_if(cond)
  */
 
 static bool
@@ -94,8 +95,12 @@ nir_opt_conditional_discard_block(nir_builder *b, nir_block *block)
    case nir_intrinsic_demote:
       op = nir_intrinsic_demote_if;
       break;
+   case nir_intrinsic_terminate:
+      op = nir_intrinsic_terminate_if;
+      break;
    case nir_intrinsic_discard_if:
    case nir_intrinsic_demote_if:
+   case nir_intrinsic_terminate_if:
       assert(intrin->src[0].is_ssa);
       cond = nir_iand(b, cond, intrin->src[0].ssa);
       break;

src/compiler/nir/nir_opt_constant_folding.c

@@ -186,13 +186,25 @@ try_fold_intrinsic(nir_builder *b, nir_intrinsic_instr *instr,
    bool progress = false;
 
    if ((instr->intrinsic == nir_intrinsic_demote_if ||
-        instr->intrinsic == nir_intrinsic_discard_if) &&
+        instr->intrinsic == nir_intrinsic_discard_if ||
+        instr->intrinsic == nir_intrinsic_terminate_if) &&
        nir_src_is_const(instr->src[0])) {
       if (nir_src_as_bool(instr->src[0])) {
          b->cursor = nir_before_instr(&instr->instr);
-         nir_intrinsic_op op = instr->intrinsic == nir_intrinsic_discard_if ?
+         nir_intrinsic_op op;
-                               nir_intrinsic_discard :
+         switch (instr->intrinsic) {
-                               nir_intrinsic_demote;
+         case nir_intrinsic_discard_if:
+            op = nir_intrinsic_discard;
+            break;
+         case nir_intrinsic_demote_if:
+            op = nir_intrinsic_demote;
+            break;
+         case nir_intrinsic_terminate_if:
+            op = nir_intrinsic_terminate;
+            break;
+         default:
+            unreachable("invalid intrinsic");
+         }
          nir_intrinsic_instr *new_instr =
             nir_intrinsic_instr_create(b->shader, op);
          nir_builder_instr_insert(b, &new_instr->instr);

src/compiler/nir/nir_opt_load_store_vectorize.c

@@ -1170,6 +1170,8 @@ handle_barrier(struct vectorize_ctx *ctx, bool *progress, nir_function_impl *imp
       /* prevent speculative loads/stores */
       case nir_intrinsic_discard_if:
       case nir_intrinsic_discard:
+      case nir_intrinsic_terminate_if:
+      case nir_intrinsic_terminate:
          modes = nir_var_all;
          break;
       case nir_intrinsic_demote_if:

src/compiler/nir/nir_schedule.c

@@ -357,6 +357,8 @@ nir_schedule_intrinsic_deps(nir_deps_state *state,
    case nir_intrinsic_discard_if:
    case nir_intrinsic_demote:
    case nir_intrinsic_demote_if:
+   case nir_intrinsic_terminate:
+   case nir_intrinsic_terminate_if:
       /* We are adding two dependencies:
        *
        * * A individual one that we could use to add a read_dep while handling