From 243ced4eb2072d074e85c0f2b8d328c8d31408d9 Mon Sep 17 00:00:00 2001
From: Lionel Landwerlin <lionel.g.landwerlin@intel.com>
Date: Fri, 7 Jun 2024 09:56:05 +0300
Subject: [PATCH] mi-builder: add a write check parameter
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

All the MI_SDI currently have forced write checks (meaning the command
streamer will stall until completion) on Gfx12.0+.

Now on Gfx12.0/12.5, the read commands have implicit waits on previous
writes (BSpec ). So if we're only dealing with CS writes & reads, we
don't need forced write checks.

In the few cases where CS is writing data for other bits of HW, we
need the forced write checks. This change adds an API that will let
the driver decide when to enable forced write checks.

Signed-off-by: Lionel Landwerlin <lionel.g.landwerlin@intel.com>
Reviewed-by: José Roberto de Souza <jose.souza@intel.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/29571>
---
 src/intel/common/mi_builder.h | 46 +++++++++++++++++++++++++++++++++--
 1 file changed, 44 insertions(+), 2 deletions(-)

diff --git a/src/intel/common/mi_builder.h b/src/intel/common/mi_builder.h
index 6c0df95b09c..912be0b4f72 100644
--- a/src/intel/common/mi_builder.h
+++ b/src/intel/common/mi_builder.h
@@ -39,6 +39,10 @@
 #define MI_BUILDER_NUM_ALLOC_GPRS 16
 #endif
 
+#ifndef MI_BUILDER_DEFAULT_WRITE_CHECK
+#define MI_BUILDER_DEFAULT_WRITE_CHECK true
+#endif
+
 /** These must be defined by the user of the builder
  *
  * void *__gen_get_batch_dwords(__gen_user_data *user_data,
@@ -147,6 +151,10 @@ struct mi_builder {
 #if GFX_VERx10 >= 125
    uint32_t mocs;
 #endif
+
+#if GFX_VER >= 12
+   bool write_check;
+#endif
 };
 
 static inline void
@@ -158,6 +166,9 @@ mi_builder_init(struct mi_builder *b,
    b->devinfo = devinfo;
    b->user_data = user_data;
 
+#if GFX_VER >= 12
+   b->write_check = MI_BUILDER_DEFAULT_WRITE_CHECK;
+#endif
 #if GFX_VERx10 >= 75
    b->gprs = 0;
    b->num_math_dwords = 0;
@@ -200,6 +211,30 @@ mi_builder_set_mocs(UNUSED struct mi_builder *b, UNUSED uint32_t mocs)
 #endif
 }
 
+/**
+ * Set write checks on immediate writes
+ *
+ * This ensures that the next memory write will complete only when all emitted
+ * previously emitted memory write are .
+ */
+static inline void
+mi_builder_set_write_check(UNUSED struct mi_builder *b, UNUSED bool check)
+{
+#if GFX_VER >= 12
+   b->write_check = check;
+#endif
+}
+
+static inline bool
+mi_builder_write_checked(UNUSED struct mi_builder *b)
+{
+#if GFX_VER >= 12
+   return b->write_check;
+#else
+   return false;
+#endif
+}
+
 #define _MI_BUILDER_GPR_BASE 0x2600
 /* The actual hardware limit on GPRs */
 #define _MI_BUILDER_NUM_HW_GPRS 16
@@ -459,7 +494,7 @@ _mi_copy_no_unref(struct mi_builder *b,
                sdi.StoreQword = true;
                sdi.Address = dst.addr;
 #if GFX_VER >= 12
-               sdi.ForceWriteCompletionCheck = true;
+               sdi.ForceWriteCompletionCheck = b->write_check;
 #endif
             }
             dw[3] = src.imm;
@@ -497,7 +532,7 @@ _mi_copy_no_unref(struct mi_builder *b,
          mi_builder_emit(b, GENX(MI_STORE_DATA_IMM), sdi) {
             sdi.Address = dst.addr;
 #if GFX_VER >= 12
-            sdi.ForceWriteCompletionCheck = true;
+            sdi.ForceWriteCompletionCheck = b->write_check;
 #endif
             sdi.ImmediateData = src.imm;
          }
@@ -650,10 +685,17 @@ mi_memcpy(struct mi_builder *b, __gen_address_type dst,
    assert(b->num_math_dwords == 0);
 #endif
 
+   /* Hold off write checks until the last write. */
+   bool write_check = mi_builder_write_checked(b);
+   mi_builder_set_write_check(b, false);
+
    /* This memcpy operates in units of dwords. */
    assert(size % 4 == 0);
 
    for (uint32_t i = 0; i < size; i += 4) {
+      if (i == size - 4)
+         mi_builder_set_write_check(b, write_check);
+
       struct mi_value dst_val = mi_mem32(__gen_address_offset(dst, i));
       struct mi_value src_val = mi_mem32(__gen_address_offset(src, i));
 #if GFX_VERx10 >= 75