panvk: Use memory pools for internal GPU data attached to vulkan objects

Some panvk objects need to allocate GPU memory but don't have Pool
objects to get this memory from. Use device-wide mempools with
.owns_bos=false, such that small allocations don't have to pay the 4k
granularity price of private BO allocations.

Signed-off-by: Boris Brezillon <boris.brezillon@collabora.com>
Reviewed-by: Mary Guillemard <mary.guillemard@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/29161>

src/panfrost/vulkan/bifrost/panvk_pipeline.h

@@ -27,11 +27,11 @@
 #define MAX_RTS 8
 
 struct panvk_pipeline_shader {
-   mali_ptr code;
-   mali_ptr rsd;
+   struct panvk_priv_mem code;
+   struct panvk_priv_mem rsd;
 
    struct {
-      mali_ptr attribs;
+      struct panvk_priv_mem attribs;
       unsigned buf_strides[PANVK_VARY_BUF_MAX];
    } varyings;
 
@@ -49,7 +49,7 @@ struct panvk_pipeline_shader {
          uint32_t count;
       } dyn_ssbos;
       struct {
-         mali_ptr map;
+         struct panvk_priv_mem map;
          uint32_t count[PANVK_BIFROST_DESC_TABLE_COUNT];
       } others;
    } desc_info;
@@ -65,9 +65,6 @@ struct panvk_pipeline {
    enum panvk_pipeline_type type;
 
    const struct vk_pipeline_layout *layout;
-
-   struct panvk_pool bin_pool;
-   struct panvk_pool desc_pool;
 };
 
 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_pipeline, base, VkPipeline,

src/panfrost/vulkan/bifrost/panvk_vX_meta_desc_copy.c

@@ -301,7 +301,7 @@ panvk_per_arch(meta_get_copy_desc_job)(
    const struct panvk_descriptor_state *desc_state,
    const struct panvk_shader_desc_state *shader_desc_state)
 {
-   mali_ptr copy_table = shader->desc_info.others.map;
+   mali_ptr copy_table = panvk_priv_mem_dev_addr(shader->desc_info.others.map);
    if (!copy_table)
       return (struct panfrost_ptr){0};
 

src/panfrost/vulkan/bifrost/panvk_vX_pipeline.c

@@ -69,10 +69,10 @@ init_pipeline_shader(struct panvk_pipeline *pipeline,
    unsigned shader_sz = util_dynarray_num_elements(&shader->binary, uint8_t);
 
    if (shader_sz) {
-      pshader->code = pan_pool_upload_aligned(&pipeline->bin_pool.base,
-                                              shader_data, shader_sz, 128);
+      pshader->code = panvk_pool_upload_aligned(&dev->mempools.exec,
+                                                shader_data, shader_sz, 128);
    } else {
-      pshader->code = 0;
+      pshader->code = (struct panvk_priv_mem){0};
    }
 
    pshader->info = shader->info;
@@ -85,8 +85,8 @@ init_pipeline_shader(struct panvk_pipeline *pipeline,
    }
 
    if (copy_count) {
-      pshader->desc_info.others.map = pan_pool_upload_aligned(
-         &pipeline->desc_pool.base, shader->desc_info.others[0].map,
+      pshader->desc_info.others.map = panvk_pool_upload_aligned(
+         &dev->mempools.rw, shader->desc_info.others[0].map,
          copy_count * sizeof(uint32_t), sizeof(uint32_t));
    }
 
@@ -111,20 +111,29 @@ init_pipeline_shader(struct panvk_pipeline *pipeline,
    }
 
    if (stage_info->stage != VK_SHADER_STAGE_FRAGMENT_BIT) {
-      struct panfrost_ptr rsd =
-         pan_pool_alloc_desc(&pipeline->desc_pool.base, RENDERER_STATE);
+      pshader->rsd = panvk_pool_alloc_desc(&dev->mempools.rw, RENDERER_STATE);
 
-      pan_pack(rsd.cpu, RENDERER_STATE, cfg) {
-         pan_shader_prepare_rsd(&pshader->info, pshader->code, &cfg);
+      pan_pack(panvk_priv_mem_host_addr(pshader->rsd), RENDERER_STATE, cfg) {
+         pan_shader_prepare_rsd(&pshader->info,
+                                panvk_priv_mem_dev_addr(pshader->code), &cfg);
       }
-
-      pshader->rsd = rsd.gpu;
    }
 
    panvk_per_arch(shader_destroy)(dev, shader, alloc);
    return VK_SUCCESS;
 }
 
+static void
+cleanup_pipeline_shader(struct panvk_pipeline *pipeline,
+                        struct panvk_pipeline_shader *pshader)
+{
+   struct panvk_device *dev = to_panvk_device(pipeline->base.device);
+
+   panvk_pool_free_mem(&dev->mempools.exec, pshader->code);
+   panvk_pool_free_mem(&dev->mempools.rw, pshader->rsd);
+   panvk_pool_free_mem(&dev->mempools.rw, pshader->desc_info.others.map);
+}
+
 static mali_pixel_format
 get_varying_format(gl_shader_stage stage, gl_varying_slot loc,
                    enum pipe_format pfmt)
@@ -204,16 +213,16 @@ varying_format(gl_varying_slot loc, enum pipe_format pfmt)
    }
 }
 
-static mali_ptr
-emit_varying_attrs(struct pan_pool *desc_pool,
+static struct panvk_priv_mem
+emit_varying_attrs(struct panvk_pool *desc_pool,
                    const struct pan_shader_varying *varyings,
                    unsigned varying_count, const struct varyings_info *info,
                    unsigned *buf_offsets)
 {
    unsigned attr_count = BITSET_COUNT(info->active);
-   struct panfrost_ptr ptr =
-      pan_pool_alloc_desc_array(desc_pool, attr_count, ATTRIBUTE);
-   struct mali_attribute_packed *attrs = ptr.cpu;
+   struct panvk_priv_mem mem =
+      panvk_pool_alloc_desc_array(desc_pool, attr_count, ATTRIBUTE);
+   struct mali_attribute_packed *attrs = panvk_priv_mem_host_addr(mem);
    unsigned attr_idx = 0;
 
    for (unsigned i = 0; i < varying_count; i++) {
@@ -238,7 +247,7 @@ emit_varying_attrs(struct pan_pool *desc_pool,
       }
    }
 
-   return ptr.gpu;
+   return mem;
 }
 
 static void
@@ -246,6 +255,7 @@ link_shaders(struct panvk_graphics_pipeline *pipeline,
              struct panvk_pipeline_shader *stage,
              struct panvk_pipeline_shader *next_stage)
 {
+   struct panvk_device *dev = to_panvk_device(pipeline->base.base.device);
    BITSET_DECLARE(active_attrs, VARYING_SLOT_MAX) = {0};
    unsigned buf_strides[PANVK_VARY_BUF_MAX] = {0};
    unsigned buf_offsets[VARYING_SLOT_MAX] = {0};
@@ -310,10 +320,10 @@ link_shaders(struct panvk_graphics_pipeline *pipeline,
    }
 
    stage->varyings.attribs = emit_varying_attrs(
-      &pipeline->base.desc_pool.base, stage->info.varyings.output,
+      &dev->mempools.rw, stage->info.varyings.output,
       stage->info.varyings.output_count, &out_vars, buf_offsets);
    next_stage->varyings.attribs = emit_varying_attrs(
-      &pipeline->base.desc_pool.base, next_stage->info.varyings.input,
+      &dev->mempools.rw, next_stage->info.varyings.input,
       next_stage->info.varyings.input_count, &in_vars, buf_offsets);
    memcpy(stage->varyings.buf_strides, buf_strides,
           sizeof(stage->varyings.buf_strides));
@@ -352,26 +362,6 @@ panvk_graphics_pipeline_create(struct panvk_device *dev,
    vk_dynamic_graphics_state_fill(&gfx_pipeline->state.dynamic, &state);
    gfx_pipeline->state.rp = *state.rp;
 
-   struct panvk_pool_properties bin_pool_props = {
-      .create_flags = PAN_KMOD_BO_FLAG_EXECUTABLE,
-      .slab_size = 4096,
-      .label = "Pipeline shader binaries",
-      .prealloc = false,
-      .owns_bos = true,
-      .needs_locking = false,
-   };
-   panvk_pool_init(&gfx_pipeline->base.bin_pool, dev, NULL, &bin_pool_props);
-
-   struct panvk_pool_properties desc_pool_props = {
-      .create_flags = 0,
-      .slab_size = 4096,
-      .label = "Pipeline static state",
-      .prealloc = false,
-      .owns_bos = true,
-      .needs_locking = false,
-   };
-   panvk_pool_init(&gfx_pipeline->base.desc_pool, dev, NULL, &desc_pool_props);
-
    /* Make sure the stage info is correct even if no stage info is provided for
     * this stage in pStages.
     */
@@ -451,28 +441,6 @@ panvk_compute_pipeline_create(struct panvk_device *dev,
    compute_pipeline->base.layout = layout;
    compute_pipeline->base.type = PANVK_PIPELINE_COMPUTE;
 
-   struct panvk_pool_properties bin_pool_props = {
-      .create_flags = PAN_KMOD_BO_FLAG_EXECUTABLE,
-      .slab_size = 4096,
-      .label = "Pipeline shader binaries",
-      .prealloc = false,
-      .owns_bos = true,
-      .needs_locking = false,
-   };
-   panvk_pool_init(&compute_pipeline->base.bin_pool, dev, NULL,
-                   &bin_pool_props);
-
-   struct panvk_pool_properties desc_pool_props = {
-      .create_flags = 0,
-      .slab_size = 4096,
-      .label = "Pipeline static state",
-      .prealloc = false,
-      .owns_bos = true,
-      .needs_locking = false,
-   };
-   panvk_pool_init(&compute_pipeline->base.desc_pool, dev, NULL,
-                   &desc_pool_props);
-
    VkResult result =
       init_pipeline_shader(&compute_pipeline->base, &create_info->stage, alloc,
                            &compute_pipeline->cs);
@@ -518,7 +486,18 @@ panvk_per_arch(DestroyPipeline)(VkDevice _device, VkPipeline _pipeline,
    VK_FROM_HANDLE(panvk_device, device, _device);
    VK_FROM_HANDLE(panvk_pipeline, pipeline, _pipeline);
 
-   panvk_pool_cleanup(&pipeline->bin_pool);
-   panvk_pool_cleanup(&pipeline->desc_pool);
+   if (pipeline->type == PANVK_PIPELINE_GRAPHICS) {
+      struct panvk_graphics_pipeline *gfx_pipeline =
+         panvk_pipeline_to_graphics_pipeline(pipeline);
+
+      cleanup_pipeline_shader(pipeline, &gfx_pipeline->vs);
+      cleanup_pipeline_shader(pipeline, &gfx_pipeline->fs);
+   } else {
+      struct panvk_compute_pipeline *compute_pipeline =
+         panvk_pipeline_to_compute_pipeline(pipeline);
+
+      cleanup_pipeline_shader(pipeline, &compute_pipeline->cs);
+   }
+
    vk_object_free(&device->vk, pAllocator, pipeline);
 }

src/panfrost/vulkan/jm/panvk_vX_cmd_dispatch.c

@@ -111,7 +111,7 @@ panvk_per_arch(CmdDispatch)(VkCommandBuffer commandBuffer, uint32_t x,
    }
 
    pan_section_pack(job.cpu, COMPUTE_JOB, DRAW, cfg) {
-      cfg.state = pipeline->cs.rsd;
+      cfg.state = panvk_priv_mem_dev_addr(pipeline->cs.rsd);
       cfg.attributes = cs_desc_state->img_attrib_table;
       cfg.attribute_buffers =
          cs_desc_state->tables[PANVK_BIFROST_DESC_TABLE_IMG];

src/panfrost/vulkan/jm/panvk_vX_cmd_draw.c

@@ -349,18 +349,18 @@ panvk_draw_prepare_fs_rsd(struct panvk_cmd_buffer *cmdbuf,
       PAN_DESC_ARRAY(bd_count, BLEND));
    struct mali_renderer_state_packed *rsd = ptr.cpu;
    struct mali_blend_packed *bds = ptr.cpu + pan_size(RENDERER_STATE);
+   mali_ptr fs_code = panvk_priv_mem_dev_addr(pipeline->fs.code);
 
    panvk_per_arch(blend_emit_descs)(
       dev, cb, cmdbuf->state.gfx.render.color_attachments.fmts,
-      cmdbuf->state.gfx.render.color_attachments.samples, fs_info,
-      pipeline->fs.code, bds, &blend_reads_dest,
-      &blend_shader_loads_blend_const);
+      cmdbuf->state.gfx.render.color_attachments.samples, fs_info, fs_code, bds,
+      &blend_reads_dest, &blend_shader_loads_blend_const);
 
    pan_pack(rsd, RENDERER_STATE, cfg) {
       bool alpha_to_coverage = dyns->ms.alpha_to_coverage_enable;
 
       if (needs_fs) {
-         pan_shader_prepare_rsd(fs_info, pipeline->fs.code, &cfg);
+         pan_shader_prepare_rsd(fs_info, fs_code, &cfg);
 
          if (blend_shader_loads_blend_const) {
             /* Preload the blend constant if the blend shader depends on it. */
@@ -543,8 +543,8 @@ panvk_draw_prepare_varyings(struct panvk_cmd_buffer *cmdbuf,
       draw->line_width = 1.0f;
 
    draw->varying_bufs = bufs.gpu;
-   draw->vs.varyings = pipeline->vs.varyings.attribs;
-   draw->fs.varyings = pipeline->fs.varyings.attribs;
+   draw->vs.varyings = panvk_priv_mem_dev_addr(pipeline->vs.varyings.attribs);
+   draw->fs.varyings = panvk_priv_mem_dev_addr(pipeline->fs.varyings.attribs);
 }
 
 static void
@@ -805,7 +805,7 @@ panvk_draw_prepare_vertex_job(struct panvk_cmd_buffer *cmdbuf,
    }
 
    pan_section_pack(ptr.cpu, COMPUTE_JOB, DRAW, cfg) {
-      cfg.state = pipeline->vs.rsd;
+      cfg.state = panvk_priv_mem_dev_addr(pipeline->vs.rsd);
       cfg.attributes = draw->vs.attributes;
       cfg.attribute_buffers = draw->vs.attribute_bufs;
       cfg.varyings = draw->vs.varyings;

src/panfrost/vulkan/panvk_buffer_view.h

@@ -12,15 +12,15 @@
 
 #include <stdint.h>
 
+#include "panvk_mempool.h"
+
 #include "vk_buffer_view.h"
 
 #include "genxml/gen_macros.h"
 
-struct panvk_priv_bo;
-
 struct panvk_buffer_view {
    struct vk_buffer_view vk;
-   struct panvk_priv_bo *bo;
+   struct panvk_priv_mem mem;
 
    struct {
       struct mali_texture_packed tex;

src/panfrost/vulkan/panvk_device.h

@@ -44,6 +44,11 @@ struct panvk_device {
       struct panvk_priv_bo *rsd_bo;
    } desc_copy;
 
+   struct {
+      struct panvk_pool rw;
+      struct panvk_pool exec;
+   } mempools;
+
    struct vk_device_dispatch_table cmd_dispatch;
 
    struct panvk_queue *queues[PANVK_MAX_QUEUE_FAMILIES];

src/panfrost/vulkan/panvk_image_view.h

@@ -25,7 +25,7 @@ struct panvk_image_view {
 
    struct pan_image_view pview;
 
-   struct panvk_priv_bo *bo;
+   struct panvk_priv_mem mem;
 
    struct {
       struct mali_texture_packed tex;

src/panfrost/vulkan/panvk_vX_buffer_view.c

@@ -92,14 +92,16 @@ panvk_per_arch(CreateBufferView)(VkDevice _device,
 
       pan_image_layout_init(arch, &plane.layout, NULL);
 
-      unsigned bo_size = GENX(panfrost_estimate_texture_payload_size)(&pview);
+      struct panvk_pool_alloc_info alloc_info = {
+         .alignment = pan_alignment(TEXTURE),
+         .size = GENX(panfrost_estimate_texture_payload_size)(&pview),
+      };
 
-      view->bo = panvk_priv_bo_create(device, bo_size, 0,
-                                      VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
+      view->mem = panvk_pool_alloc_mem(&device->mempools.rw, alloc_info);
 
       struct panfrost_ptr ptr = {
-         .gpu = view->bo->addr.dev,
-         .cpu = view->bo->addr.host,
+         .gpu = panvk_priv_mem_dev_addr(view->mem),
+         .cpu = panvk_priv_mem_host_addr(view->mem),
       };
 
       GENX(panfrost_new_texture)(&pview, view->descs.tex.opaque, &ptr);
@@ -151,6 +153,6 @@ panvk_per_arch(DestroyBufferView)(VkDevice _device, VkBufferView bufferView,
    if (!view)
       return;
 
-   panvk_priv_bo_unref(view->bo);
+   panvk_pool_free_mem(&device->mempools.rw, view->mem);
    vk_buffer_view_destroy(&device->vk, pAllocator, &view->vk);
 }

src/panfrost/vulkan/panvk_vX_device.c

@@ -47,6 +47,39 @@ panvk_kmod_free(const struct pan_kmod_allocator *allocator, void *data)
    return vk_free(vkalloc, data);
 }
 
+static void
+panvk_device_init_mempools(struct panvk_device *dev)
+{
+   struct panvk_pool_properties rw_pool_props = {
+      .create_flags = 0,
+      .slab_size = 16 * 1024,
+      .label = "Device RW memory pool",
+      .owns_bos = false,
+      .needs_locking = true,
+      .prealloc = false,
+   };
+
+   panvk_pool_init(&dev->mempools.rw, dev, NULL, &rw_pool_props);
+
+   struct panvk_pool_properties exec_pool_props = {
+      .create_flags = PAN_KMOD_BO_FLAG_EXECUTABLE,
+      .slab_size = 16 * 1024,
+      .label = "Device executable memory pool (shaders)",
+      .owns_bos = false,
+      .needs_locking = true,
+      .prealloc = false,
+   };
+
+   panvk_pool_init(&dev->mempools.exec, dev, NULL, &exec_pool_props);
+}
+
+static void
+panvk_device_cleanup_mempools(struct panvk_device *dev)
+{
+   panvk_pool_cleanup(&dev->mempools.rw);
+   panvk_pool_cleanup(&dev->mempools.exec);
+}
+
 /* Always reserve the lower 32MB. */
 #define PANVK_VA_RESERVE_BOTTOM 0x2000000ull
 
@@ -128,6 +161,8 @@ panvk_per_arch(create_device)(struct panvk_physical_device *physical_device,
       pan_kmod_vm_create(device->kmod.dev, PAN_KMOD_VM_FLAG_AUTO_VA,
                          user_va_start, user_va_end - user_va_start);
 
+   panvk_device_init_mempools(device);
+
    device->tiler_heap = panvk_priv_bo_create(
       device, 128 * 1024 * 1024,
       PAN_KMOD_BO_FLAG_NO_MMAP | PAN_KMOD_BO_FLAG_ALLOC_ON_FAULT,
@@ -184,6 +219,7 @@ fail:
    panvk_per_arch(blend_shader_cache_cleanup)(device);
    panvk_priv_bo_unref(device->tiler_heap);
    panvk_priv_bo_unref(device->sample_positions);
+   panvk_device_cleanup_mempools(device);
    pan_kmod_vm_destroy(device->kmod.vm);
    pan_kmod_dev_destroy(device->kmod.dev);
 

src/panfrost/vulkan/panvk_vX_image_view.c

@@ -144,14 +144,16 @@ panvk_per_arch(CreateImageView)(VkDevice _device,
          util_format_compose_swizzles(r001, view->pview.swizzle, pview.swizzle);
       }
 
-      unsigned bo_size = GENX(panfrost_estimate_texture_payload_size)(&pview);
+      struct panvk_pool_alloc_info alloc_info = {
+         .alignment = pan_alignment(TEXTURE),
+         .size = GENX(panfrost_estimate_texture_payload_size)(&pview),
+      };
 
-      view->bo = panvk_priv_bo_create(device, bo_size, 0,
-                                      VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
+      view->mem = panvk_pool_alloc_mem(&device->mempools.rw, alloc_info);
 
       struct panfrost_ptr ptr = {
-         .gpu = view->bo->addr.dev,
-         .cpu = view->bo->addr.host,
+         .gpu = panvk_priv_mem_dev_addr(view->mem),
+         .cpu = panvk_priv_mem_host_addr(view->mem),
       };
 
       GENX(panfrost_new_texture)(&pview, view->descs.tex.opaque, &ptr);
@@ -221,6 +223,6 @@ panvk_per_arch(DestroyImageView)(VkDevice _device, VkImageView _view,
    if (!view)
       return;
 
-   panvk_priv_bo_unref(view->bo);
+   panvk_pool_free_mem(&device->mempools.rw, view->mem);
    vk_image_view_destroy(&device->vk, pAllocator, &view->vk);
 }