nvc0: add support for bindless textures on kepler+

This keeps a list of resident textures (per context), and dumps that
list into the active buffer list when submitting. We also treat bindless
texture fetches slightly differently, wrt the meaning of indirect, and
not requiring the SAMPLER file to be used.

Signed-off-by: Ilia Mirkin <imirkin@alum.mit.edu>

src/gallium/drivers/nouveau/codegen/nv50_ir.h

@@ -1024,6 +1024,7 @@ public:
       bool liveOnly; // only execute on live pixels of a quad (optimization)
       bool levelZero;
       bool derivAll;
+      bool bindless;
 
       int8_t useOffsets; // 0, 1, or 4 for textureGatherOffsets
       int8_t offset[3]; // only used on nv50

src/gallium/drivers/nouveau/codegen/nv50_ir_from_tgsi.cpp

@@ -2206,6 +2206,16 @@ Converter::setTexRS(TexInstruction *tex, unsigned int& s, int R, int S)
 {
    unsigned rIdx = 0, sIdx = 0;
 
+   if (R >= 0 && tgsi.getSrc(R).getFile() != TGSI_FILE_SAMPLER) {
+      // This is the bindless case. We have to get the actual value and pass
+      // it in. This will be the complete handle.
+      tex->tex.rIndirectSrc = s;
+      tex->setSrc(s++, fetchSrc(R, 0));
+      tex->setTexture(tgsi.getTexture(code, R), 0xff, 0x1f);
+      tex->tex.bindless = true;
+      return;
+   }
+
    if (R >= 0)
       rIdx = tgsi.getSrc(R).getIndex(0);
    if (S >= 0)

src/gallium/drivers/nouveau/codegen/nv50_ir_lowering_nvc0.cpp

@@ -882,10 +882,12 @@ NVC0LoweringPass::handleTEX(TexInstruction *i)
       if (i->tex.rIndirectSrc >= 0 || i->tex.sIndirectSrc >= 0) {
          // XXX this ignores tsc, and assumes a 1:1 mapping
          assert(i->tex.rIndirectSrc >= 0);
-         Value *hnd = loadTexHandle(i->getIndirectR(), i->tex.r);
-         i->tex.r = 0xff;
-         i->tex.s = 0x1f;
-         i->setIndirectR(hnd);
+         if (!i->tex.bindless) {
+            Value *hnd = loadTexHandle(i->getIndirectR(), i->tex.r);
+            i->tex.r = 0xff;
+            i->tex.s = 0x1f;
+            i->setIndirectR(hnd);
+         }
          i->setIndirectS(NULL);
       } else if (i->tex.r == i->tex.s || i->op == OP_TXF) {
          if (i->tex.r == 0xffff)

src/gallium/drivers/nouveau/nv50/nv50_stateobj_tex.h

@@ -20,6 +20,7 @@ struct nv50_tic_entry {
    struct pipe_sampler_view pipe;
    int id;
    uint32_t tic[8];
+   uint32_t bindless;
 };
 
 static inline struct nv50_tic_entry *

src/gallium/drivers/nouveau/nvc0/nvc0_context.c

@@ -206,6 +206,16 @@ nvc0_destroy(struct pipe_context *pipe)
    nvc0_context_unreference_resources(nvc0);
    nvc0_blitctx_destroy(nvc0);
 
+   list_for_each_entry_safe(struct nvc0_resident, pos, &nvc0->tex_head, list) {
+      list_del(&pos->list);
+      free(pos);
+   }
+
+   list_for_each_entry_safe(struct nvc0_resident, pos, &nvc0->img_head, list) {
+      list_del(&pos->list);
+      free(pos);
+   }
+
    nouveau_context_destroy(&nvc0->base);
 }
 
@@ -401,6 +411,11 @@ nvc0_create(struct pipe_screen *pscreen, void *priv, unsigned ctxflags)
    nvc0_init_state_functions(nvc0);
    nvc0_init_transfer_functions(nvc0);
    nvc0_init_resource_functions(pipe);
+   if (nvc0->screen->base.class_3d >= NVE4_3D_CLASS)
+      nvc0_init_bindless_functions(pipe);
+
+   list_inithead(&nvc0->tex_head);
+   list_inithead(&nvc0->img_head);
 
    nvc0->base.invalidate_resource_storage = nvc0_invalidate_resource_storage;
 

src/gallium/drivers/nouveau/nvc0/nvc0_context.h

@@ -5,6 +5,7 @@
 #include "pipe/p_defines.h"
 #include "pipe/p_state.h"
 
+#include "util/list.h"
 #include "util/u_memory.h"
 #include "util/u_math.h"
 #include "util/u_inlines.h"
@@ -81,6 +82,7 @@
 #define NVC0_BIND_3D_SUF         245
 #define NVC0_BIND_3D_BUF         246
 #define NVC0_BIND_3D_SCREEN      247
+#define NVC0_BIND_3D_BINDLESS    248
 #define NVC0_BIND_3D_TLS         249
 #define NVC0_BIND_3D_TEXT        250
 #define NVC0_BIND_3D_COUNT       251
@@ -95,7 +97,8 @@
 #define NVC0_BIND_CP_QUERY       52
 #define NVC0_BIND_CP_BUF         53
 #define NVC0_BIND_CP_TEXT        54
-#define NVC0_BIND_CP_COUNT       55
+#define NVC0_BIND_CP_BINDLESS    55
+#define NVC0_BIND_CP_COUNT       56
 
 /* bufctx for other operations */
 #define NVC0_BIND_2D            0
@@ -151,6 +154,13 @@ struct nvc0_blitctx;
 bool nvc0_blitctx_create(struct nvc0_context *);
 void nvc0_blitctx_destroy(struct nvc0_context *);
 
+struct nvc0_resident {
+   struct list_head list;
+   uint64_t handle;
+   struct nv04_resource *buf;
+   uint32_t flags;
+};
+
 struct nvc0_context {
    struct nouveau_context base;
 
@@ -212,6 +222,9 @@ struct nvc0_context {
 
    uint32_t tex_handles[6][PIPE_MAX_SAMPLERS]; /* for nve4 */
 
+   struct list_head tex_head;
+   struct list_head img_head;
+
    struct pipe_framebuffer_state framebuffer;
    struct pipe_blend_color blend_colour;
    struct pipe_stencil_ref stencil_ref;
@@ -362,6 +375,8 @@ struct pipe_sampler_view *
 gm107_create_texture_view_from_image(struct pipe_context *,
                                      const struct pipe_image_view *);
 
+void nvc0_init_bindless_functions(struct pipe_context *);
+
 /* nvc0_transfer.c */
 void
 nvc0_init_transfer_functions(struct nvc0_context *);

src/gallium/drivers/nouveau/nvc0/nvc0_screen.h

@@ -193,6 +193,8 @@ extern const struct nvc0_vertex_format nvc0_vertex_format[];
 static inline void
 nvc0_screen_tic_unlock(struct nvc0_screen *screen, struct nv50_tic_entry *tic)
 {
+   if (tic->bindless)
+      return;
    if (tic->id >= 0)
       screen->tic.lock[tic->id / 32] &= ~(1 << (tic->id % 32));
 }

src/gallium/drivers/nouveau/nvc0/nvc0_tex.c

@@ -90,6 +90,7 @@ gm107_create_texture_view(struct pipe_context *pipe,
    view->pipe.context = pipe;
 
    view->id = -1;
+   view->bindless = 0;
 
    pipe_resource_reference(&view->pipe.texture, texture);
 
@@ -302,6 +303,7 @@ gf100_create_texture_view(struct pipe_context *pipe,
    view->pipe.context = pipe;
 
    view->id = -1;
+   view->bindless = 0;
 
    pipe_resource_reference(&view->pipe.texture, texture);
 
@@ -762,6 +764,124 @@ nve4_set_tex_handles(struct nvc0_context *nvc0)
    }
 }
 
+static uint64_t
+nve4_create_texture_handle(struct pipe_context *pipe,
+                           struct pipe_sampler_view *view,
+                           const struct pipe_sampler_state *sampler)
+{
+   /* We have to create persistent handles that won't change for these objects
+    * That means that we have to upload them into place and lock them so that
+    * they can't be kicked out later.
+    */
+   struct nvc0_context *nvc0 = nvc0_context(pipe);
+   struct nouveau_pushbuf *push = nvc0->base.pushbuf;
+   struct nv50_tic_entry *tic = nv50_tic_entry(view);
+   struct nv50_tsc_entry *tsc = pipe->create_sampler_state(pipe, sampler);
+   struct pipe_sampler_view *v = NULL;
+
+   tsc->id = nvc0_screen_tsc_alloc(nvc0->screen, tsc);
+   if (tsc->id < 0)
+      goto fail;
+
+   if (tic->id < 0) {
+      tic->id = nvc0_screen_tic_alloc(nvc0->screen, tic);
+      if (tic->id < 0)
+         goto fail;
+
+      nve4_p2mf_push_linear(&nvc0->base, nvc0->screen->txc, tic->id * 32,
+                            NV_VRAM_DOMAIN(&nvc0->screen->base), 32,
+                            tic->tic);
+
+      IMMED_NVC0(push, NVC0_3D(TIC_FLUSH), 0);
+   }
+
+   nve4_p2mf_push_linear(&nvc0->base, nvc0->screen->txc,
+                         65536 + tsc->id * 32,
+                         NV_VRAM_DOMAIN(&nvc0->screen->base),
+                         32, tsc->tsc);
+
+   IMMED_NVC0(push, NVC0_3D(TSC_FLUSH), 0);
+
+   // Add an extra reference to this sampler view effectively held by this
+   // texture handle. This is to deal with the sampler view being dereferenced
+   // before the handle is. However we need the view to still be live until the
+   // handle to it is deleted.
+   pipe_sampler_view_reference(&v, view);
+   p_atomic_inc(&tic->bindless);
+
+   nvc0->screen->tic.lock[tic->id / 32] |= 1 << (tic->id % 32);
+   nvc0->screen->tsc.lock[tsc->id / 32] |= 1 << (tsc->id % 32);
+
+   return 0x100000000ULL | (tsc->id << 20) | tic->id;
+
+fail:
+   pipe->delete_sampler_state(pipe, tsc);
+   return 0;
+}
+
+static bool
+view_bound(struct nvc0_context *nvc0, struct pipe_sampler_view *view) {
+   for (int s = 0; s < 6; s++) {
+      for (int i = 0; i < nvc0->num_textures[s]; i++)
+         if (nvc0->textures[s][i] == view)
+            return true;
+   }
+   return false;
+}
+
+static void
+nve4_delete_texture_handle(struct pipe_context *pipe, uint64_t handle)
+{
+   struct nvc0_context *nvc0 = nvc0_context(pipe);
+   uint32_t tic = handle & NVE4_TIC_ENTRY_INVALID;
+   uint32_t tsc = (handle & NVE4_TSC_ENTRY_INVALID) >> 20;
+   struct nv50_tic_entry *entry = nvc0->screen->tic.entries[tic];
+
+   if (entry) {
+      struct pipe_sampler_view *view = &entry->pipe;
+      assert(entry->bindless);
+      p_atomic_dec(&entry->bindless);
+      if (!view_bound(nvc0, view))
+         nvc0_screen_tic_unlock(nvc0->screen, entry);
+      pipe_sampler_view_reference(&view, NULL);
+   }
+
+   pipe->delete_sampler_state(pipe, nvc0->screen->tsc.entries[tsc]);
+}
+
+static void
+nve4_make_texture_handle_resident(struct pipe_context *pipe,
+                                  uint64_t handle, bool resident)
+{
+   struct nvc0_context *nvc0 = nvc0_context(pipe);
+   if (resident) {
+      struct nvc0_resident *res = calloc(1, sizeof(struct nvc0_resident));
+      struct nv50_tic_entry *tic =
+         nvc0->screen->tic.entries[handle & NVE4_TIC_ENTRY_INVALID];
+      assert(tic);
+      assert(tic->bindless);
+
+      res->handle = handle;
+      res->buf = nv04_resource(tic->pipe.texture);
+      res->flags = NOUVEAU_BO_RD;
+      list_add(&res->list, &nvc0->tex_head);
+   } else {
+      list_for_each_entry_safe(struct nvc0_resident, pos, &nvc0->tex_head, list) {
+         if (pos->handle == handle) {
+            list_del(&pos->list);
+            free(pos);
+            break;
+         }
+      }
+   }
+}
+
+void
+nvc0_init_bindless_functions(struct pipe_context *pipe) {
+   pipe->create_texture_handle = nve4_create_texture_handle;
+   pipe->delete_texture_handle = nve4_delete_texture_handle;
+   pipe->make_texture_handle_resident = nve4_make_texture_handle_resident;
+}
 
 static const uint8_t nve4_su_format_map[PIPE_FORMAT_COUNT];
 static const uint16_t nve4_su_format_aux_map[PIPE_FORMAT_COUNT];

src/gallium/drivers/nouveau/nvc0/nvc0_vbo.c

@@ -972,6 +972,11 @@ nvc0_draw_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info)
       BCTX_REFN(nvc0->bufctx_3d, 3D_IDX, buf, RD);
    }
 
+   list_for_each_entry(struct nvc0_resident, resident, &nvc0->tex_head, list) {
+      nvc0_add_resident(nvc0->bufctx_3d, NVC0_BIND_3D_BINDLESS, resident->buf,
+                        resident->flags);
+   }
+
    nvc0_state_validate_3d(nvc0, ~0);
 
    if (nvc0->vertprog->vp.need_draw_parameters && !info->indirect) {
@@ -1083,4 +1088,5 @@ cleanup:
    nouveau_pushbuf_bufctx(push, NULL);
 
    nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_IDX);
+   nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_BINDLESS);
 }

src/gallium/drivers/nouveau/nvc0/nve4_compute.c

@@ -681,6 +681,11 @@ nve4_launch_grid(struct pipe_context *pipe, const struct pipe_grid_info *info)
    BCTX_REFN_bo(nvc0->bufctx_cp, CP_DESC, NOUVEAU_BO_GART | NOUVEAU_BO_RD,
                 desc_bo);
 
+   list_for_each_entry(struct nvc0_resident, resident, &nvc0->tex_head, list) {
+      nvc0_add_resident(nvc0->bufctx_cp, NVC0_BIND_CP_BINDLESS, resident->buf,
+                        resident->flags);
+   }
+
    ret = !nve4_state_validate_cp(nvc0, ~0);
    if (ret)
       goto out;
@@ -742,6 +747,7 @@ out:
       NOUVEAU_ERR("Failed to launch grid !\n");
    nouveau_scratch_done(&nvc0->base);
    nouveau_bufctx_reset(nvc0->bufctx_cp, NVC0_BIND_CP_DESC);
+   nouveau_bufctx_reset(nvc0->bufctx_cp, NVC0_BIND_CP_BINDLESS);
 }