zink: start adding C++ draw templates

templated draw functions enable moving some checks/calculations/code
into template conditionals, which are resolved in advance of the draw call,
reducing draw overhead

Reviewed-by: Dave Airlie <airlied@redhat.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/11964>

src/gallium/drivers/zink/meson.build

@@ -29,7 +29,7 @@ files_libzink = files(
   'zink_context.c',
   'zink_descriptors.c',
   'zink_descriptors_lazy.c',
-  'zink_draw.c',
+  'zink_draw.cpp',
   'zink_fence.c',
   'zink_format.c',
   'zink_framebuffer.c',

src/gallium/drivers/zink/zink_context.c

@@ -36,6 +36,7 @@
 #include "zink_screen.h"
 #include "zink_state.h"
 #include "zink_surface.h"
+#include "zink_inlines.h"
 
 #include "util/u_blitter.h"
 #include "util/u_debug.h"
@@ -3420,6 +3421,8 @@ zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags)
    ctx->compute_pipeline_state.dirty = true;
    ctx->fb_changed = ctx->rp_changed = true;
 
+   zink_init_draw_functions(ctx);
+
    ctx->base.screen = pscreen;
    ctx->base.priv = priv;
 
@@ -3463,7 +3466,6 @@ zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags)
    ctx->base.clear_render_target = zink_clear_render_target;
    ctx->base.clear_depth_stencil = zink_clear_depth_stencil;
 
-   ctx->base.draw_vbo = zink_draw_vbo;
    ctx->base.launch_grid = zink_launch_grid;
    ctx->base.fence_server_sync = zink_fence_server_sync;
    ctx->base.flush = zink_flush;
@@ -3567,6 +3569,8 @@ zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags)
    }
    p_atomic_inc(&screen->base.num_contexts);
 
+   zink_select_draw_vbo(ctx);
+
    if (!(flags & PIPE_CONTEXT_PREFER_THREADED) || flags & PIPE_CONTEXT_COMPUTE_ONLY) {
       return &ctx->base;
    }
@@ -3579,6 +3583,7 @@ zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags)
    if (tc && (struct zink_context*)tc != ctx) {
       tc->bytes_mapped_limit = screen->total_mem / 4;
       ctx->base.set_context_param = zink_set_context_param;
+      ctx->multidraw = screen->info.have_EXT_multi_draw;
    }
 
    return (struct pipe_context*)tc;

src/gallium/drivers/zink/zink_context.h

@@ -136,6 +136,19 @@ struct zink_descriptor_surface {
    bool is_buffer;
 };
 
+typedef void (*pipe_draw_vbo_func)(struct pipe_context *pipe,
+                                   const struct pipe_draw_info *info,
+                                   unsigned drawid_offset,
+                                   const struct pipe_draw_indirect_info *indirect,
+                                   const struct pipe_draw_start_count_bias *draws,
+                                   unsigned num_draws);
+
+
+typedef enum {
+   ZINK_NO_MULTIDRAW,
+   ZINK_MULTIDRAW,
+} zink_multidraw;
+
 struct zink_context {
    struct pipe_context base;
    struct threaded_context *tc;
@@ -143,6 +156,9 @@ struct zink_context {
    struct slab_child_pool transfer_pool_unsync;
    struct blitter_context *blitter;
 
+   zink_multidraw multidraw;
+   pipe_draw_vbo_func draw_vbo[2]; //multidraw
+
    struct pipe_device_reset_callback reset;
 
    uint32_t curr_batch; //the current batch id
@@ -362,6 +378,11 @@ zink_pipeline_flags_from_pipe_stage(enum pipe_shader_type pstage)
    }
 }
 
+void
+zink_init_draw_functions(struct zink_context *ctx);
+
+void
+zink_launch_grid(struct pipe_context *pctx, const struct pipe_grid_info *info);
 #ifdef __cplusplus
 }
 #endif
@@ -408,17 +429,6 @@ zink_resource_rebind(struct zink_context *ctx, struct zink_resource *res);
 void
 zink_rebind_framebuffer(struct zink_context *ctx, struct zink_resource *res);
 
-void
-zink_draw_vbo(struct pipe_context *pctx,
-              const struct pipe_draw_info *dinfo,
-              unsigned drawid_offset,
-              const struct pipe_draw_indirect_info *indirect,
-              const struct pipe_draw_start_count_bias *draws,
-              unsigned num_draws);
-
-void
-zink_launch_grid(struct pipe_context *pctx, const struct pipe_grid_info *info);
-
 void
 zink_copy_buffer(struct zink_context *ctx, struct zink_batch *batch, struct zink_resource *dst, struct zink_resource *src,
                  unsigned dst_offset, unsigned src_offset, unsigned size);
@@ -448,8 +458,4 @@ zink_buffer_view_reference(struct zink_screen *screen,
 }
 #endif
 
-#ifdef __cplusplus
-}
-#endif
-
 #endif

src/gallium/drivers/zink/zink_draw.cpp

@@ -270,6 +270,7 @@ draw_indexed_need_index_buffer_unref(struct zink_context *ctx,
    }
 }
 
+template <zink_multidraw HAS_MULTIDRAW>
 ALWAYS_INLINE static void
 draw_indexed(struct zink_context *ctx,
              const struct pipe_draw_info *dinfo,
@@ -290,12 +291,12 @@ draw_indexed(struct zink_context *ctx,
    } else {
       if (needs_drawid)
          update_drawid(ctx, draw_id);
-      if (zink_screen(ctx->base.screen)->info.have_EXT_multi_draw)
+      if (HAS_MULTIDRAW) {
          zink_screen(ctx->base.screen)->vk.CmdDrawMultiIndexedEXT(cmdbuf, num_draws, (const VkMultiDrawIndexedInfoEXT*)draws,
                                                                    dinfo->instance_count,
                                                                    dinfo->start_instance, sizeof(struct pipe_draw_start_count_bias),
                                                                    dinfo->index_bias_varies ? NULL : &draws[0].index_bias);
-      else {
+      } else {
          for (unsigned i = 0; i < num_draws; i++)
             vkCmdDrawIndexed(cmdbuf,
                draws[i].count, dinfo->instance_count,
@@ -304,6 +305,7 @@ draw_indexed(struct zink_context *ctx,
    }
 }
 
+template <zink_multidraw HAS_MULTIDRAW>
 ALWAYS_INLINE static void
 draw(struct zink_context *ctx,
      const struct pipe_draw_info *dinfo,
@@ -322,7 +324,7 @@ draw(struct zink_context *ctx,
    } else {
       if (needs_drawid)
          update_drawid(ctx, draw_id);
-      if (zink_screen(ctx->base.screen)->info.have_EXT_multi_draw)
+      if (HAS_MULTIDRAW)
          zink_screen(ctx->base.screen)->vk.CmdDrawMultiEXT(cmdbuf, num_draws, (const VkMultiDrawInfoEXT*)draws,
                                                             dinfo->instance_count, dinfo->start_instance,
                                                             sizeof(struct pipe_draw_start_count_bias));
@@ -392,6 +394,7 @@ update_barriers(struct zink_context *ctx, bool is_compute)
    }
 }
 
+template <zink_multidraw HAS_MULTIDRAW>
 void
 zink_draw_vbo(struct pipe_context *pctx,
               const struct pipe_draw_info *dinfo,
@@ -422,8 +425,8 @@ zink_draw_vbo(struct pipe_context *pctx,
    if (!dindirect || !dindirect->buffer)
       ctx->drawid_broken = BITSET_TEST(ctx->gfx_stages[PIPE_SHADER_VERTEX]->nir->info.system_values_read, SYSTEM_VALUE_DRAW_ID) &&
                            (drawid_offset != 0 ||
-                           (!screen->info.have_EXT_multi_draw && num_draws > 1) ||
-                           (screen->info.have_EXT_multi_draw && num_draws > 1 && !dinfo->increment_draw_id));
+                           (!HAS_MULTIDRAW && num_draws > 1) ||
+                           (HAS_MULTIDRAW && num_draws > 1 && !dinfo->increment_draw_id));
    if (drawid_broken != ctx->drawid_broken)
       ctx->dirty_shader_stages |= BITFIELD_BIT(PIPE_SHADER_VERTEX);
    ctx->gfx_pipeline_state.vertices_per_patch = dinfo->vertices_per_patch;
@@ -692,7 +695,7 @@ zink_draw_vbo(struct pipe_context *pctx,
          if (need_index_buffer_unref)
             draw_indexed_need_index_buffer_unref(ctx, dinfo, draws, num_draws, draw_id, needs_drawid);
          else
-            draw_indexed(ctx, dinfo, draws, num_draws, draw_id, needs_drawid);
+            draw_indexed<HAS_MULTIDRAW>(ctx, dinfo, draws, num_draws, draw_id, needs_drawid);
       }
    } else {
       if (so_target && screen->info.tf_props.transformFeedbackDraw) {
@@ -718,7 +721,7 @@ zink_draw_vbo(struct pipe_context *pctx,
          } else
             vkCmdDrawIndirect(batch->state->cmdbuf, indirect->obj->buffer, dindirect->offset, dindirect->draw_count, dindirect->stride);
       } else {
-         draw(ctx, dinfo, draws, num_draws, draw_id, needs_drawid);
+         draw<HAS_MULTIDRAW>(ctx, dinfo, draws, num_draws, draw_id, needs_drawid);
       }
    }
 
@@ -764,7 +767,7 @@ zink_launch_grid(struct pipe_context *pctx, const struct pipe_grid_info *info)
       vkCmdBindPipeline(batch->state->cmdbuf, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
    ctx->pipeline_changed[1] = false;
 
-   if (BITSET_TEST(ctx->curr_compute->shader->nir->info.system_values_read, SYSTEM_VALUE_WORK_DIM))
+   if (BITSET_TEST(ctx->compute_stage->nir->info.system_values_read, SYSTEM_VALUE_WORK_DIM))
       vkCmdPushConstants(batch->state->cmdbuf, ctx->curr_compute->base.layout, VK_SHADER_STAGE_COMPUTE_BIT,
                          offsetof(struct zink_cs_push_constant, work_dim), sizeof(uint32_t),
                          &info->work_dim);
@@ -779,3 +782,39 @@ zink_launch_grid(struct pipe_context *pctx, const struct pipe_grid_info *info)
    /* check memory usage and flush/stall as needed to avoid oom */
    zink_maybe_flush_or_stall(ctx);
 }
+
+template <zink_multidraw HAS_MULTIDRAW>
+static void
+init_multidraw_functions(struct zink_context *ctx)
+{
+   ctx->draw_vbo[HAS_MULTIDRAW] = zink_draw_vbo<HAS_MULTIDRAW>;
+}
+
+static void
+init_all_draw_functions(struct zink_context *ctx)
+{
+   init_multidraw_functions<ZINK_NO_MULTIDRAW>(ctx);
+   init_multidraw_functions<ZINK_MULTIDRAW>(ctx);
+}
+
+static void
+zink_invalid_draw_vbo(struct pipe_context *pipe,
+                      const struct pipe_draw_info *dinfo,
+                      unsigned drawid_offset,
+                      const struct pipe_draw_indirect_info *dindirect,
+                      const struct pipe_draw_start_count_bias *draws,
+                      unsigned num_draws)
+{
+   unreachable("vertex shader not bound");
+}
+
+extern "C"
+void
+zink_init_draw_functions(struct zink_context *ctx)
+{
+   init_all_draw_functions(ctx);
+   /* Bind a fake draw_vbo, so that draw_vbo isn't NULL, which would skip
+    * initialization of callbacks in upper layers (such as u_threaded_context).
+    */
+   ctx->base.draw_vbo = zink_invalid_draw_vbo;
+}

src/gallium/drivers/zink/zink_inlines.h

@@ -0,0 +1,12 @@
+#ifndef ZINK_INLINES_H
+#define ZINK_INLINES_H
+
+/* these go here to avoid include hell */
+static inline void
+zink_select_draw_vbo(struct zink_context *ctx)
+{
+   ctx->base.draw_vbo = ctx->draw_vbo[ctx->multidraw];
+   assert(ctx->base.draw_vbo);
+}
+
+#endif

src/gallium/drivers/zink/zink_program.c

@@ -30,6 +30,7 @@
 #include "zink_resource.h"
 #include "zink_screen.h"
 #include "zink_state.h"
+#include "zink_inlines.h"
 
 #include "util/hash_table.h"
 #include "util/set.h"