vk: clflush all state for non-LLC GPUs

src/vulkan/anv_batch_chain.c

@@ -724,6 +724,7 @@ anv_cmd_buffer_add_secondary(struct anv_cmd_buffer *primary,
       struct anv_batch_bo *this_bbo = anv_cmd_buffer_current_batch_bo(primary);
       assert(primary->batch.start == this_bbo->bo.map);
       uint32_t offset = primary->batch.next - primary->batch.start;
+      const uint32_t inst_size = GEN8_MI_BATCH_BUFFER_START_length * 4;
 
       /* Roll back the previous MI_BATCH_BUFFER_START and its relocation so we
        * can emit a new command and relocation for the current splice.  In
@@ -732,9 +733,25 @@ anv_cmd_buffer_add_secondary(struct anv_cmd_buffer *primary,
        * here.
        */
       last_bbo->relocs.num_relocs--;
-      secondary->batch.next -= GEN8_MI_BATCH_BUFFER_START_length * 4;
+      secondary->batch.next -= inst_size;
       emit_batch_buffer_start(secondary, &this_bbo->bo, offset);
       anv_cmd_buffer_add_seen_bbos(primary, &secondary->batch_bos);
+
+      /* After patching up the secondary buffer, we need to clflush the
+       * modified instruction in case we're on a !llc platform. We use a
+       * little loop to handle the case where the instruction crosses a cache
+       * line boundary.
+       */
+      if (!primary->device->info.has_llc) {
+         void *inst = secondary->batch.next - inst_size;
+         void *p = (void *) (((uintptr_t) inst) & ~CACHELINE_MASK);
+         __builtin_ia32_sfence();
+         while (p < secondary->batch.next) {
+            __builtin_ia32_clflush(p);
+            p += CACHELINE_SIZE;
+         }
+      }
+
       break;
    }
    case ANV_CMD_BUFFER_EXEC_MODE_COPY_AND_CHAIN: {
@@ -886,6 +903,11 @@ adjust_relocations_from_block_pool(struct anv_block_pool *pool,
        */
       assert(relocs->relocs[i].offset < pool->state.end);
       uint32_t *reloc_data = pool->map + relocs->relocs[i].offset;
+
+      /* We're reading back the relocated value from potentially incoherent
+       * memory here. However, any change to the value will be from the kernel
+       * writing out relocations, which will keep the CPU cache up to date.
+       */
       relocs->relocs[i].presumed_offset = *reloc_data - relocs->relocs[i].delta;
 
       /* All of the relocations from this block pool to other BO's should
@@ -994,6 +1016,14 @@ anv_cmd_buffer_prepare_execbuf(struct anv_cmd_buffer *cmd_buffer)
 
    anv_cmd_buffer_process_relocs(cmd_buffer, &cmd_buffer->surface_relocs);
 
+   if (!cmd_buffer->device->info.has_llc) {
+      __builtin_ia32_sfence();
+      anv_vector_foreach(bbo, &cmd_buffer->seen_bbos) {
+         for (uint32_t i = 0; i < (*bbo)->length; i += CACHELINE_SIZE)
+            __builtin_ia32_clflush((*bbo)->bo.map + i);
+      }
+   }
+
    cmd_buffer->execbuf2.execbuf = (struct drm_i915_gem_execbuffer2) {
       .buffers_ptr = (uintptr_t) cmd_buffer->execbuf2.objects,
       .buffer_count = cmd_buffer->execbuf2.bo_count,

src/vulkan/anv_cmd_buffer.c

@@ -666,7 +666,7 @@ anv_cmd_buffer_emit_binding_table(struct anv_cmd_buffer *cmd_buffer,
    }
 
    if (layout == NULL)
-      return VK_SUCCESS;
+      goto out;
 
    for (uint32_t s = 0; s < layout->stage[stage].surface_count; s++) {
       struct anv_pipeline_binding *binding =
@@ -698,6 +698,10 @@ anv_cmd_buffer_emit_binding_table(struct anv_cmd_buffer *cmd_buffer,
                                               surface_state.map,
                                               stage, desc->type,
                                               bo_offset, desc->range);
+
+         if (!cmd_buffer->device->info.has_llc)
+            anv_state_clflush(surface_state);
+
          break;
       }
 
@@ -724,6 +728,10 @@ anv_cmd_buffer_emit_binding_table(struct anv_cmd_buffer *cmd_buffer,
       add_surface_state_reloc(cmd_buffer, surface_state, bo, bo_offset);
    }
 
+ out:
+   if (!cmd_buffer->device->info.has_llc)
+      anv_state_clflush(*bt_state);
+
    return VK_SUCCESS;
 }
 
@@ -772,20 +780,25 @@ anv_cmd_buffer_emit_samplers(struct anv_cmd_buffer *cmd_buffer,
              sampler->state, sizeof(sampler->state));
    }
 
+   if (!cmd_buffer->device->info.has_llc)
+      anv_state_clflush(*state);
+
    return VK_SUCCESS;
 }
 
 struct anv_state
 anv_cmd_buffer_emit_dynamic(struct anv_cmd_buffer *cmd_buffer,
-                             uint32_t *a, uint32_t dwords, uint32_t alignment)
+                            const void *data, uint32_t size, uint32_t alignment)
 {
    struct anv_state state;
 
-   state = anv_cmd_buffer_alloc_dynamic_state(cmd_buffer,
+   state = anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, size, alignment);
-                                              dwords * 4, alignment);
+   memcpy(state.map, data, size);
-   memcpy(state.map, a, dwords * 4);
 
-   VG(VALGRIND_CHECK_MEM_IS_DEFINED(state.map, dwords * 4));
+   if (!cmd_buffer->device->info.has_llc)
+      anv_state_clflush(state);
+
+   VG(VALGRIND_CHECK_MEM_IS_DEFINED(state.map, size));
 
    return state;
 }
@@ -804,6 +817,9 @@ anv_cmd_buffer_merge_dynamic(struct anv_cmd_buffer *cmd_buffer,
    for (uint32_t i = 0; i < dwords; i++)
       p[i] = a[i] | b[i];
 
+   if (!cmd_buffer->device->info.has_llc)
+      anv_state_clflush(state);
+
    VG(VALGRIND_CHECK_MEM_IS_DEFINED(p, dwords * 4));
 
    return state;
@@ -881,6 +897,9 @@ anv_cmd_buffer_push_constants(struct anv_cmd_buffer *cmd_buffer,
       u32_map[i] = *(uint32_t *)((uint8_t *)data + offset);
    }
 
+   if (!cmd_buffer->device->info.has_llc)
+      anv_state_clflush(state);
+
    return state;
 }
 

src/vulkan/anv_device.c

@@ -599,6 +599,20 @@ anv_queue_finish(struct anv_queue *queue)
 {
 }
 
+static struct anv_state
+anv_state_pool_emit_data(struct anv_state_pool *pool, size_t size, size_t align, const void *p)
+{
+   struct anv_state state;
+
+   state = anv_state_pool_alloc(pool, size, align);
+   memcpy(state.map, p, size);
+
+   if (!pool->block_pool->device->info.has_llc)
+      anv_state_clflush(state);
+
+   return state;
+}
+
 static void
 anv_device_init_border_colors(struct anv_device *device)
 {
@@ -611,10 +625,8 @@ anv_device_init_border_colors(struct anv_device *device)
       [VK_BORDER_COLOR_INT_OPAQUE_WHITE] =         { .uint32 = { 1, 1, 1, 1 } },
    };
 
-   device->border_colors =
+   device->border_colors = anv_state_pool_emit_data(&device->dynamic_state_pool,
-      anv_state_pool_alloc(&device->dynamic_state_pool,
+                                                    sizeof(border_colors), 32, border_colors);
-                           sizeof(border_colors), 32);
-   memcpy(device->border_colors.map, border_colors, sizeof(border_colors));
 }
 
 VkResult anv_CreateDevice(
@@ -885,6 +897,9 @@ VkResult anv_DeviceWaitIdle(
    anv_batch_emit(&batch, GEN7_MI_BATCH_BUFFER_END);
    anv_batch_emit(&batch, GEN7_MI_NOOP);
 
+   if (!device->info.has_llc)
+      anv_state_clflush(state);
+
    exec2_objects[0].handle = bo->gem_handle;
    exec2_objects[0].relocation_count = 0;
    exec2_objects[0].relocs_ptr = 0;
@@ -1219,6 +1234,13 @@ VkResult anv_CreateFence(
    anv_batch_emit(&batch, GEN7_MI_BATCH_BUFFER_END);
    anv_batch_emit(&batch, GEN7_MI_NOOP);
 
+   if (!device->info.has_llc) {
+      assert(((uintptr_t) fence->bo.map & CACHELINE_MASK) == 0);
+      assert(batch.next - fence->bo.map <= CACHELINE_SIZE);
+      __builtin_ia32_sfence();
+      __builtin_ia32_clflush(fence->bo.map);
+   }
+
    fence->exec2_objects[0].handle = fence->bo.gem_handle;
    fence->exec2_objects[0].relocation_count = 0;
    fence->exec2_objects[0].relocs_ptr = 0;

src/vulkan/anv_meta.c

@@ -478,6 +478,8 @@ meta_emit_blit(struct anv_cmd_buffer *cmd_buffer,
       },
    };
 
+   anv_state_clflush(vb_state);
+
    struct anv_buffer vertex_buffer = {
       .device = device,
       .size = vb_size,

src/vulkan/anv_meta_clear.c

@@ -316,8 +316,7 @@ emit_load_color_clear(struct anv_cmd_buffer *cmd_buffer,
    };
 
    struct anv_state state =
-      anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, sizeof(vertex_data), 16);
+      anv_cmd_buffer_emit_dynamic(cmd_buffer, vertex_data, sizeof(vertex_data), 16);
-   memcpy(state.map, vertex_data, sizeof(vertex_data));
 
    struct anv_buffer vertex_buffer = {
       .device = device,
@@ -485,8 +484,7 @@ emit_load_depthstencil_clear(struct anv_cmd_buffer *cmd_buffer,
    };
 
    struct anv_state state =
-      anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, sizeof(vertex_data), 16);
+      anv_cmd_buffer_emit_dynamic(cmd_buffer, vertex_data, sizeof(vertex_data), 16);
-   memcpy(state.map, vertex_data, sizeof(vertex_data));
 
    struct anv_buffer vertex_buffer = {
       .device = device,

src/vulkan/anv_pipeline.c

@@ -373,8 +373,12 @@ anv_pipeline_upload_kernel(struct anv_pipeline *pipeline,
 
    memcpy(state.map, data, size);
 
+   if (!pipeline->device->info.has_llc)
+      anv_state_clflush(state);
+
    return state.offset;
 }
+
 static void
 anv_pipeline_add_compiled_stage(struct anv_pipeline *pipeline,
                                 gl_shader_stage stage,

src/vulkan/anv_private.h

@@ -394,6 +394,22 @@ struct anv_state_stream {
 #define CACHELINE_SIZE 64
 #define CACHELINE_MASK 63
 
+static void inline
+anv_state_clflush(struct anv_state state)
+{
+   /* state.map may not be cacheline aligned, so round down the start pointer
+    * to a cacheline boundary so we flush all pages that contain the state.
+    */
+   void *end = state.map + state.alloc_size;
+   void *p = (void *) (((uintptr_t) state.map) & ~CACHELINE_MASK);
+
+   __builtin_ia32_sfence();
+   while (p < end) {
+      __builtin_ia32_clflush(p);
+      p += CACHELINE_SIZE;
+   }
+}
+
 void anv_block_pool_init(struct anv_block_pool *pool,
                          struct anv_device *device, uint32_t block_size);
 void anv_block_pool_finish(struct anv_block_pool *pool);
@@ -721,6 +737,20 @@ __gen_combine_address(struct anv_batch *batch, void *location,
       VG(VALGRIND_CHECK_MEM_IS_DEFINED(dw, ARRAY_SIZE(dwords0) * 4));\
    } while (0)
 
+#define anv_state_pool_emit(pool, cmd, align, ...) ({                   \
+      const uint32_t __size = __anv_cmd_length(cmd) * 4;                \
+      struct anv_state __state =                                        \
+         anv_state_pool_alloc((pool), __size, align);                   \
+      struct cmd __template = {                                         \
+         __VA_ARGS__                                                    \
+      };                                                                \
+      __anv_cmd_pack(cmd)(NULL, __state.map, &__template);              \
+      VG(VALGRIND_CHECK_MEM_IS_DEFINED(__state.map, __anv_cmd_length(cmd) * 4)); \
+      if (!(pool)->block_pool->device->info.has_llc)                    \
+         anv_state_clflush(__state);                                    \
+      __state;                                                          \
+   })
+
 #define GEN7_MOCS (struct GEN7_MEMORY_OBJECT_CONTROL_STATE) {  \
    .GraphicsDataTypeGFDT                        = 0,           \
    .LLCCacheabilityControlLLCCC                 = 0,           \
@@ -1104,8 +1134,7 @@ VkResult anv_cmd_buffer_emit_samplers(struct anv_cmd_buffer *cmd_buffer,
 void gen7_cmd_buffer_flush_descriptor_sets(struct anv_cmd_buffer *cmd_buffer);
 
 struct anv_state anv_cmd_buffer_emit_dynamic(struct anv_cmd_buffer *cmd_buffer,
-                                             uint32_t *a, uint32_t dwords,
+                                             const void *data, uint32_t size, uint32_t alignment);
-                                             uint32_t alignment);
 struct anv_state anv_cmd_buffer_merge_dynamic(struct anv_cmd_buffer *cmd_buffer,
                                               uint32_t *a, uint32_t *b,
                                               uint32_t dwords, uint32_t alignment);

src/vulkan/gen7_cmd_buffer.c

@@ -201,6 +201,9 @@ emit_scissor_state(struct anv_cmd_buffer *cmd_buffer,
 
    anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_SCISSOR_STATE_POINTERS,
                   .ScissorRectPointer = scissor_state.offset);
+
+   if (!cmd_buffer->device->info.has_llc)
+      anv_state_clflush(scissor_state);
 }
 
 GENX_FUNC(GEN7, GEN7) void
@@ -266,19 +269,15 @@ flush_compute_descriptor_set(struct anv_cmd_buffer *cmd_buffer)
    if (result != VK_SUCCESS)
       return result;
 
-   struct GEN7_INTERFACE_DESCRIPTOR_DATA desc = {
+   struct anv_state state =
+      anv_state_pool_emit(&device->dynamic_state_pool,
+                          GEN7_INTERFACE_DESCRIPTOR_DATA, 64,
                           .KernelStartPointer = pipeline->cs_simd,
                           .BindingTablePointer = surfaces.offset,
                           .SamplerStatePointer = samplers.offset,
-      .NumberofThreadsinGPGPUThreadGroup = 0 /* FIXME: Really? */
+                          .NumberofThreadsinGPGPUThreadGroup = 0);
-   };
-
-   uint32_t size = GEN7_INTERFACE_DESCRIPTOR_DATA_length * sizeof(uint32_t);
-   struct anv_state state =
-      anv_state_pool_alloc(&device->dynamic_state_pool, size, 64);
-
-   GEN7_INTERFACE_DESCRIPTOR_DATA_pack(NULL, state.map, &desc);
 
+   const uint32_t size = GEN7_INTERFACE_DESCRIPTOR_DATA_length * sizeof(uint32_t);
    anv_batch_emit(&cmd_buffer->batch, GEN7_MEDIA_INTERFACE_DESCRIPTOR_LOAD,
                   .InterfaceDescriptorTotalLength = size,
                   .InterfaceDescriptorDataStartAddress = state.offset);
@@ -441,6 +440,8 @@ cmd_buffer_flush_state(struct anv_cmd_buffer *cmd_buffer)
             cmd_buffer->state.dynamic.stencil_reference.back,
       };
       GEN7_COLOR_CALC_STATE_pack(NULL, cc_state.map, &cc);
+      if (!cmd_buffer->device->info.has_llc)
+         anv_state_clflush(cc_state);
 
       anv_batch_emit(&cmd_buffer->batch,
                      GEN7_3DSTATE_CC_STATE_POINTERS,

src/vulkan/gen7_pipeline.c

@@ -253,27 +253,24 @@ gen7_emit_cb_state(struct anv_pipeline *pipeline,
 {
    struct anv_device *device = pipeline->device;
 
-   uint32_t num_dwords = GEN7_BLEND_STATE_length;
-   pipeline->blend_state =
-      anv_state_pool_alloc(&device->dynamic_state_pool, num_dwords * 4, 64);
-
    if (info->pAttachments == NULL) {
-      struct GEN7_BLEND_STATE blend_state = {
+      pipeline->blend_state =
+         anv_state_pool_emit(&device->dynamic_state_pool,
+            GEN7_BLEND_STATE, 64,
             .ColorBufferBlendEnable = false,
             .WriteDisableAlpha = false,
             .WriteDisableRed = false,
             .WriteDisableGreen = false,
-         .WriteDisableBlue = false,
+            .WriteDisableBlue = false);
-      };
-
-      GEN7_BLEND_STATE_pack(NULL, pipeline->blend_state.map, &blend_state);
    } else {
       /* FIXME-GEN7: All render targets share blend state settings on gen7, we
        * can't implement this.
        */
       const VkPipelineColorBlendAttachmentState *a = &info->pAttachments[0];
+      pipeline->blend_state =
+         anv_state_pool_emit(&device->dynamic_state_pool,
+            GEN7_BLEND_STATE, 64,
 
-      struct GEN7_BLEND_STATE blend_state = {
             .ColorBufferBlendEnable = a->blendEnable,
             .IndependentAlphaBlendEnable = true, /* FIXME: yes? */
             .AlphaBlendFunction = vk_to_gen_blend_op[a->alphaBlendOp],
@@ -309,9 +306,7 @@ gen7_emit_cb_state(struct anv_pipeline *pipeline,
             bool                                PreBlendColorClampEnable;
             bool                                PostBlendColorClampEnable;
 #     endif
-      };
+            );
-
-      GEN7_BLEND_STATE_pack(NULL, pipeline->blend_state.map, &blend_state);
     }
 
    anv_batch_emit(&pipeline->batch, GEN7_3DSTATE_BLEND_STATE_POINTERS,

src/vulkan/gen7_state.c

@@ -308,6 +308,9 @@ genX(image_view_init)(struct anv_image_view *iview,
 
       GENX(RENDER_SURFACE_STATE_pack)(NULL, iview->nonrt_surface_state.map,
                                       &surface_state);
+
+      if (!device->info.has_llc)
+         anv_state_clflush(iview->nonrt_surface_state);
    }
 
    if (image->needs_color_rt_surface_state) {
@@ -326,5 +329,7 @@ genX(image_view_init)(struct anv_image_view *iview,
 
       GENX(RENDER_SURFACE_STATE_pack)(NULL, iview->color_rt_surface_state.map,
                                       &surface_state);
+      if (!device->info.has_llc)
+         anv_state_clflush(iview->color_rt_surface_state);
    }
 }

src/vulkan/gen8_cmd_buffer.c

@@ -107,6 +107,11 @@ emit_viewport_state(struct anv_cmd_buffer *cmd_buffer,
       GENX(CC_VIEWPORT_pack)(NULL, cc_state.map + i * 32, &cc_viewport);
    }
 
+   if (!cmd_buffer->device->info.has_llc) {
+      anv_state_clflush(sf_clip_state);
+      anv_state_clflush(cc_state);
+   }
+
    anv_batch_emit(&cmd_buffer->batch,
                   GENX(3DSTATE_VIEWPORT_STATE_POINTERS_CC),
                   .CCViewportPointer = cc_state.offset);
@@ -270,6 +275,9 @@ cmd_buffer_flush_state(struct anv_cmd_buffer *cmd_buffer)
       };
       GEN8_COLOR_CALC_STATE_pack(NULL, cc_state.map, &cc);
 
+      if (!cmd_buffer->device->info.has_llc)
+         anv_state_clflush(cc_state);
+
       anv_batch_emit(&cmd_buffer->batch,
                      GEN8_3DSTATE_CC_STATE_POINTERS,
                      .ColorCalcStatePointer = cc_state.offset,
@@ -317,6 +325,9 @@ cmd_buffer_flush_state(struct anv_cmd_buffer *cmd_buffer)
       };
       GEN9_COLOR_CALC_STATE_pack(NULL, cc_state.map, &cc);
 
+      if (!cmd_buffer->device->info.has_llc)
+         anv_state_clflush(cc_state);
+
       anv_batch_emit(&cmd_buffer->batch,
                      GEN9_3DSTATE_CC_STATE_POINTERS,
                      .ColorCalcStatePointer = cc_state.offset,
@@ -500,22 +511,18 @@ flush_compute_descriptor_set(struct anv_cmd_buffer *cmd_buffer)
    if (result != VK_SUCCESS)
       return result;
 
-   struct GENX(INTERFACE_DESCRIPTOR_DATA) desc = {
+   struct anv_state state =
+      anv_state_pool_emit(&device->dynamic_state_pool,
+                          GENX(INTERFACE_DESCRIPTOR_DATA), 64,
                           .KernelStartPointer = pipeline->cs_simd,
                           .KernelStartPointerHigh = 0,
                           .BindingTablePointer = surfaces.offset,
                           .BindingTableEntryCount = 0,
                           .SamplerStatePointer = samplers.offset,
                           .SamplerCount = 0,
-      .NumberofThreadsinGPGPUThreadGroup = 0 /* FIXME: Really? */
+                          .NumberofThreadsinGPGPUThreadGroup = 0);
-   };
 
    uint32_t size = GENX(INTERFACE_DESCRIPTOR_DATA_length) * sizeof(uint32_t);
-   struct anv_state state =
-      anv_state_pool_alloc(&device->dynamic_state_pool, size, 64);
-
-   GENX(INTERFACE_DESCRIPTOR_DATA_pack)(NULL, state.map, &desc);
-
    anv_batch_emit(&cmd_buffer->batch, GENX(MEDIA_INTERFACE_DESCRIPTOR_LOAD),
                   .InterfaceDescriptorTotalLength = size,
                   .InterfaceDescriptorDataStartAddress = state.offset);

src/vulkan/gen8_pipeline.c

@@ -259,6 +259,8 @@ emit_cb_state(struct anv_pipeline *pipeline,
    }
 
    GENX(BLEND_STATE_pack)(NULL, pipeline->blend_state.map, &blend_state);
+   if (!device->info.has_llc)
+      anv_state_clflush(pipeline->blend_state);
 
    anv_batch_emit(&pipeline->batch, GENX(3DSTATE_BLEND_STATE_POINTERS),
                   .BlendStatePointer = pipeline->blend_state.offset,

src/vulkan/gen8_state.c

@@ -280,6 +280,8 @@ genX(image_view_init)(struct anv_image_view *iview,
 
       GENX(RENDER_SURFACE_STATE_pack)(NULL, iview->nonrt_surface_state.map,
                                       &surface_state);
+      if (!device->info.has_llc)
+         anv_state_clflush(iview->nonrt_surface_state);
    }
 
    if (image->needs_color_rt_surface_state) {
@@ -297,6 +299,8 @@ genX(image_view_init)(struct anv_image_view *iview,
 
       GENX(RENDER_SURFACE_STATE_pack)(NULL, iview->color_rt_surface_state.map,
                                       &surface_state);
+      if (!device->info.has_llc)
+         anv_state_clflush(iview->color_rt_surface_state);
    }
 }