agx: Insert waits post-RA

This is the first step towards reducing stalling.

Signed-off-by: Alyssa Rosenzweig <alyssa@rosenzweig.io>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/20446>

src/asahi/compiler/agx_compile.c

@@ -524,7 +524,6 @@ agx_emit_load_global(agx_builder *b, agx_index dest, nir_intrinsic_instr *instr)
    agx_device_load_to(b, dest, addr, offset, fmt,
                       BITFIELD_MASK(nir_dest_num_components(instr->dest)), 0,
                       0);
-   agx_wait(b, 0);
    agx_emit_cached_split(b, dest, nir_dest_num_components(instr->dest));
 }
 
@@ -543,7 +542,6 @@ agx_emit_load(agx_builder *b, agx_index dest, nir_intrinsic_instr *instr)
    agx_device_load_to(b, dest, addr, offset, fmt,
                       BITFIELD_MASK(nir_dest_num_components(instr->dest)),
                       shift, 0);
-   agx_wait(b, 0);
    agx_emit_cached_split(b, dest, nir_dest_num_components(instr->dest));
 }
 
@@ -1246,8 +1244,6 @@ agx_emit_tex(agx_builder *b, nir_tex_instr *instr)
    if (txf)
       I->op = AGX_OPCODE_TEXTURE_LOAD;
 
-   agx_wait(b, 0);
-
    agx_index packed_channels[4] = {agx_null()};
    agx_index unpacked_channels[4] = {agx_null()};
 
@@ -1872,11 +1868,12 @@ agx_compile_function_nir(nir_shader *nir, nir_function_impl *impl,
    if (ctx->stage == MESA_SHADER_VERTEX)
       agx_set_st_vary_final(ctx);
 
+   agx_lower_pseudo(ctx);
+   agx_insert_waits(ctx);
+
    if (agx_should_dump(nir, AGX_DBG_SHADERS))
       agx_print_shader(ctx, stdout);
 
-   agx_lower_pseudo(ctx);
-
    /* Pad binary */
    if (binary->size % AGX_CODE_ALIGN) {
       unsigned ngrow = AGX_CODE_ALIGN - (binary->size % AGX_CODE_ALIGN);

src/asahi/compiler/agx_compiler.h

@@ -793,6 +793,7 @@ void agx_opt_cse(agx_context *ctx);
 void agx_dce(agx_context *ctx);
 void agx_ra(agx_context *ctx);
 void agx_lower_64bit_postra(agx_context *ctx);
+void agx_insert_waits(agx_context *ctx);
 void agx_pack_binary(agx_context *ctx, struct util_dynarray *emission);
 
 #ifndef NDEBUG

src/asahi/compiler/agx_insert_waits.c

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2022 Alyssa Rosenzweig
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "agx_builder.h"
+#include "agx_compiler.h"
+
+/*
+ * Returns whether an instruction is asynchronous and needs a scoreboard slot
+ */
+static bool
+instr_is_async(agx_instr *I)
+{
+   return agx_opcodes_info[I->op].immediates & AGX_IMMEDIATE_SCOREBOARD;
+}
+
+/*
+ * Insert waits within a block to stall after every async instruction. Useful
+ * for debugging.
+ */
+static void
+agx_insert_waits_trivial(agx_context *ctx, agx_block *block)
+{
+   agx_foreach_instr_in_block_safe(block, I) {
+      if (instr_is_async(I)) {
+         agx_builder b = agx_init_builder(ctx, agx_after_instr(I));
+         agx_wait(&b, I->scoreboard);
+      }
+   }
+}
+
+/*
+ * Assign scoreboard slots to asynchronous instructions and insert waits for the
+ * appropriate hazard tracking.
+ */
+void
+agx_insert_waits(agx_context *ctx)
+{
+   agx_foreach_block(ctx, block) {
+      agx_insert_waits_trivial(ctx, block);
+   }
+}

src/asahi/compiler/meson.build

@@ -23,6 +23,7 @@ libasahi_agx_files = files(
   'agx_compile.c',
   'agx_dce.c',
   'agx_liveness.c',
+  'agx_insert_waits.c',
   'agx_nir_lower_zs_emit.c',
   'agx_nir_lower_texture.c',
   'agx_nir_lower_load_mask.c',