intel/clflush: Add support for clflushopt instruction

Rework:
 * Split clflushopt into a separate file as recommended by Ken.
   If we enable -mclflush on all driver source compilation, then
   gcc may insert uses of it on processors that don't support it.
 * Add uintptr_t casting to cpu_caps->cacheline usage

Signed-off-by: Jordan Justen <jordan.l.justen@intel.com>
Reviewed-by: Lionel Landwerlin <lionel.g.landwerlin@intel.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/22379>

src/intel/common/intel_clflushopt.c

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2023 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "intel_mem.h"
+#include "util/u_cpu_detect.h"
+
+#ifndef HAVE___BUILTIN_IA32_CLFLUSHOPT
+#error "Compiler doesn't support clflushopt!"
+#endif
+
+void intel_clflushopt_range(void *start, size_t size);
+
+void
+intel_clflushopt_range(void *start, size_t size)
+{
+   const struct util_cpu_caps_t *cpu_caps = util_get_cpu_caps();
+   assert(cpu_caps->has_clflushopt);
+   assert(cpu_caps->cacheline > 0);
+   void *p = (void *) (((uintptr_t) start) &
+                       ~((uintptr_t)cpu_caps->cacheline - 1));
+   void *end = start + size;
+
+   while (p < end) {
+      __builtin_ia32_clflushopt(p);
+      p += cpu_caps->cacheline;
+   }
+}

src/intel/common/intel_mem.c

@@ -22,10 +22,19 @@
  */
 
 #include "intel_mem.h"
+#include "util/u_cpu_detect.h"
 
 #include <stdint.h>
 
+#define CACHELINE_SIZE 64
+#define CACHELINE_MASK 63
+
 #ifdef SUPPORT_INTEL_INTEGRATED_GPUS
+
+#ifdef HAVE___BUILTIN_IA32_CLFLUSHOPT
+void intel_clflushopt_range(void *start, size_t size);
+#endif
+
 static void
 intel_clflush_range(void *start, size_t size)
 {
@@ -41,6 +50,13 @@ intel_clflush_range(void *start, size_t size)
 void
 intel_flush_range_no_fence(void *start, size_t size)
 {
+#ifdef HAVE___BUILTIN_IA32_CLFLUSHOPT
+   const struct util_cpu_caps_t *cpu_caps = util_get_cpu_caps();
+   if (cpu_caps->has_clflushopt) {
+      intel_clflushopt_range(start, size);
+      return;
+   }
+#endif
    intel_clflush_range(start, size);
 }
 
@@ -49,6 +65,11 @@ intel_flush_range(void *start, size_t size)
 {
    __builtin_ia32_mfence();
    intel_flush_range_no_fence(start, size);
+#ifdef HAVE___BUILTIN_IA32_CLFLUSHOPT
+   /* clflushopt doesn't include an mfence like clflush */
+   if (util_get_cpu_caps()->has_clflushopt)
+      __builtin_ia32_mfence();
+#endif
 }
 
 void

src/intel/common/intel_mem.h

@@ -30,9 +30,6 @@
 extern "C" {
 #endif
 
-#define CACHELINE_SIZE 64
-#define CACHELINE_MASK 63
-
 #ifdef SUPPORT_INTEL_INTEGRATED_GPUS
 void intel_flush_range(void *start, size_t size);
 void intel_flush_range_no_fence(void *start, size_t size);

src/intel/common/meson.build

@@ -60,6 +60,19 @@ files_batch_decoder = files(
   'intel_decoder.c',
 )
 
+libintel_common_links = [libisl]
+
+if with_clflushopt
+  libintel_clflushopt = static_library(
+    'intel_clflushopt',
+    ['intel_clflushopt.c'],
+    include_directories : [inc_include, inc_src],
+    c_args : [no_override_init_args] + clflushopt_args,
+    gnu_symbol_visibility : 'hidden',
+  )
+  libintel_common_links += libintel_clflushopt
+endif
+
 batch_decoder_dependencies = []
 if not dep_expat.found()
   files_libintel_common += 'intel_batch_decoder_stub.c'
@@ -74,7 +87,7 @@ libintel_common = static_library(
   include_directories : [inc_include, inc_src, inc_intel],
   c_args : [no_override_init_args],
   gnu_symbol_visibility : 'hidden',
-  link_with : [libisl],
+  link_with : libintel_common_links,
   dependencies : [batch_decoder_dependencies, dep_libdrm, dep_thread, idep_genxml, idep_mesautil, idep_intel_dev],
 )