lavapipe: Implement sparse buffers and images

Reviewed-By: Mike Blumenkrantz <michael.blumenkrantz@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/29408>

src/gallium/auxiliary/gallivm/lp_bld_sample.c

@@ -127,6 +127,9 @@ lp_sampler_static_texture_state(struct lp_static_texture_state *state,
    state->pot_height = util_is_power_of_two_or_zero(texture->height0);
    state->pot_depth = util_is_power_of_two_or_zero(texture->depth0);
    state->level_zero_only = !view->u.tex.last_level;
+   state->tiled = !!(texture->flags & PIPE_RESOURCE_FLAG_SPARSE);
+   if (state->tiled)
+      state->tiled_samples = texture->nr_samples;
 
    /*
     * the layer / element / level parameters are all either dynamic
@@ -167,6 +170,9 @@ lp_sampler_static_texture_state_image(struct lp_static_texture_state *state,
    state->pot_height = util_is_power_of_two_or_zero(resource->height0);
    state->pot_depth = util_is_power_of_two_or_zero(resource->depth0);
    state->level_zero_only = view->u.tex.level == 0;
+   state->tiled = !!(resource->flags & PIPE_RESOURCE_FLAG_SPARSE);
+   if (state->tiled)
+      state->tiled_samples = resource->nr_samples;
 
    /*
     * the layer / element / level parameters are all either dynamic
@@ -2196,6 +2202,122 @@ lp_build_sample_offset(struct lp_build_context *bld,
 }
 
 
+
+void
+lp_build_tiled_sample_offset(struct lp_build_context *bld,
+                             enum pipe_format format,
+                             const struct lp_static_texture_state *static_texture_state,
+                             LLVMValueRef x,
+                             LLVMValueRef y,
+                             LLVMValueRef z,
+                             LLVMValueRef width,
+                             LLVMValueRef height,
+                             LLVMValueRef z_stride,
+                             LLVMValueRef *out_offset,
+                             LLVMValueRef *out_i,
+                             LLVMValueRef *out_j)
+{
+   struct gallivm_state *gallivm = bld->gallivm;
+   LLVMBuilderRef builder = gallivm->builder;
+
+   assert(static_texture_state->tiled);
+
+   uint32_t dimensions = 1;
+   switch (static_texture_state->target) {
+   case PIPE_TEXTURE_2D:
+   case PIPE_TEXTURE_CUBE:
+   case PIPE_TEXTURE_RECT:
+   case PIPE_TEXTURE_2D_ARRAY:
+      dimensions = 2;
+      break;
+   case PIPE_TEXTURE_3D:
+      dimensions = 3;
+      break;
+   default:
+      break;
+   }
+
+   uint32_t block_size[3] = {
+      util_format_get_blockwidth(format),
+      util_format_get_blockheight(format),
+      util_format_get_blockdepth(format),
+   };
+
+   uint32_t sparse_tile_size[3] = {
+      util_format_get_tilesize(format, dimensions, static_texture_state->tiled_samples, 0) * block_size[0],
+      util_format_get_tilesize(format, dimensions, static_texture_state->tiled_samples, 1) * block_size[1],
+      util_format_get_tilesize(format, dimensions, static_texture_state->tiled_samples, 2) * block_size[2],
+   };
+
+   LLVMValueRef sparse_tile_size_log2[3] = {
+      lp_build_const_vec(gallivm, bld->type, util_logbase2(sparse_tile_size[0])),
+      lp_build_const_vec(gallivm, bld->type, util_logbase2(sparse_tile_size[1])),
+      lp_build_const_vec(gallivm, bld->type, util_logbase2(sparse_tile_size[2])),
+   };
+
+   LLVMValueRef tile_index = LLVMBuildLShr(builder, x, sparse_tile_size_log2[0], "");
+
+   if (y && dimensions > 1) {
+      LLVMValueRef x_tile_count = lp_build_add(bld, width, lp_build_const_vec(gallivm, bld->type, sparse_tile_size[0] - 1));
+      x_tile_count = LLVMBuildLShr(builder, x_tile_count, sparse_tile_size_log2[0], "");
+      LLVMValueRef y_tile = LLVMBuildLShr(builder, y, sparse_tile_size_log2[1], "");
+      tile_index = lp_build_add(bld, tile_index, lp_build_mul(bld, y_tile, x_tile_count));
+
+      if (z && dimensions > 2) {
+         LLVMValueRef y_tile_count = lp_build_add(bld, height, lp_build_const_vec(gallivm, bld->type, sparse_tile_size[1] - 1));
+         y_tile_count = LLVMBuildLShr(builder, y_tile_count, sparse_tile_size_log2[1], "");
+         LLVMValueRef z_tile = LLVMBuildLShr(builder, z, sparse_tile_size_log2[2], "");
+         tile_index = lp_build_add(bld, tile_index, lp_build_mul(bld, z_tile, lp_build_mul(bld, x_tile_count, y_tile_count)));
+      }
+   }
+
+   LLVMValueRef offset = LLVMBuildShl(builder, tile_index, lp_build_const_vec(gallivm, bld->type, 16), "");
+
+   LLVMValueRef sparse_tile_masks[3] = {
+      lp_build_const_vec(gallivm, bld->type, sparse_tile_size[0] - 1),
+      lp_build_const_vec(gallivm, bld->type, sparse_tile_size[1] - 1),
+      lp_build_const_vec(gallivm, bld->type, sparse_tile_size[2] - 1),
+   };
+
+   x = LLVMBuildAnd(builder, x, sparse_tile_masks[0], "");
+   LLVMValueRef x_stride = lp_build_const_vec(gallivm, bld->type, util_format_get_blocksize(format));
+
+   LLVMValueRef x_offset;
+   lp_build_sample_partial_offset(bld, block_size[0],
+                                  x, x_stride, &x_offset, out_i);
+   offset = lp_build_add(bld, offset, x_offset);
+
+   if (y && dimensions > 1) {
+      y = LLVMBuildAnd(builder, y, sparse_tile_masks[1], "");
+      LLVMValueRef y_stride = lp_build_const_vec(gallivm, bld->type, util_format_get_blocksize(format) *
+                                                 sparse_tile_size[0] / block_size[0]);
+
+      LLVMValueRef y_offset;
+      lp_build_sample_partial_offset(bld, block_size[1],
+                                     y, y_stride, &y_offset, out_j);
+      offset = lp_build_add(bld, offset, y_offset);
+   } else {
+      *out_j = bld->zero;
+   }
+
+   if (z && (z_stride || dimensions > 2)) {
+      if (dimensions > 2) {
+         z = LLVMBuildAnd(builder, z, sparse_tile_masks[2], "");
+         z_stride = lp_build_const_vec(gallivm, bld->type, util_format_get_blocksize(format) *
+                                       sparse_tile_size[0] / block_size[0] *
+                                       sparse_tile_size[1] / block_size[1]);
+      }
+
+      LLVMValueRef z_offset;
+      LLVMValueRef k;
+      lp_build_sample_partial_offset(bld, 1, z, z_stride, &z_offset, &k);
+      offset = lp_build_add(bld, offset, z_offset);
+   }
+
+   *out_offset = offset;
+}
+
+
 static LLVMValueRef
 lp_build_sample_min(struct lp_build_context *bld,
                     LLVMValueRef x,

src/gallium/auxiliary/gallivm/lp_bld_sample.h

@@ -198,6 +198,8 @@ struct lp_static_texture_state
    unsigned pot_height:1;
    unsigned pot_depth:1;
    unsigned level_zero_only:1;
+   unsigned tiled:1;
+   unsigned tiled_samples:5;
 };
 
 
@@ -738,6 +740,21 @@ lp_build_sample_offset(struct lp_build_context *bld,
                        LLVMValueRef *out_j);
 
 
+void
+lp_build_tiled_sample_offset(struct lp_build_context *bld,
+                             enum pipe_format format,
+                             const struct lp_static_texture_state *static_texture_state,
+                             LLVMValueRef x,
+                             LLVMValueRef y,
+                             LLVMValueRef z,
+                             LLVMValueRef width,
+                             LLVMValueRef height,
+                             LLVMValueRef z_stride,
+                             LLVMValueRef *out_offset,
+                             LLVMValueRef *out_i,
+                             LLVMValueRef *out_j);
+
+
 void
 lp_build_sample_soa_code(struct gallivm_state *gallivm,
                          const struct lp_static_texture_state *static_texture_state,

src/gallium/auxiliary/gallivm/lp_bld_sample_soa.c

@@ -138,10 +138,19 @@ lp_build_sample_texel_soa(struct lp_build_sample_context *bld,
    }
 
    /* convert x,y,z coords to linear offset from start of texture, in bytes */
-   lp_build_sample_offset(&bld->int_coord_bld,
+   if (bld->static_texture_state->tiled) {
-                          bld->format_desc,
+      lp_build_tiled_sample_offset(&bld->int_coord_bld,
-                          x, y, z, y_stride, z_stride,
+                                   bld->format_desc->format,
-                          &offset, &i, &j);
+                                   bld->static_texture_state,
+                                   x, y, z, width, height, z_stride,
+                                   &offset, &i, &j);
+   } else {
+      lp_build_sample_offset(&bld->int_coord_bld,
+                             bld->format_desc,
+                             x, y, z, y_stride, z_stride,
+                             &offset, &i, &j);
+   }
+
    if (mipoffsets) {
       offset = lp_build_add(&bld->int_coord_bld, offset, mipoffsets);
    }
@@ -3248,10 +3257,18 @@ lp_build_fetch_texel(struct lp_build_sample_context *bld,
       }
    }
 
-   lp_build_sample_offset(int_coord_bld,
+   if (bld->static_texture_state->tiled) {
-                          bld->format_desc,
+      lp_build_tiled_sample_offset(&bld->int_coord_bld,
-                          x, y, z, row_stride_vec, img_stride_vec,
+                                   bld->format_desc->format,
-                          &offset, &i, &j);
+                                   bld->static_texture_state,
+                                   x, y, z, width, height, img_stride_vec,
+                                   &offset, &i, &j);
+   } else {
+      lp_build_sample_offset(int_coord_bld,
+                             bld->format_desc,
+                             x, y, z, row_stride_vec, img_stride_vec,
+                             &offset, &i, &j);
+   }
 
    if (bld->static_texture_state->target != PIPE_BUFFER) {
       offset = lp_build_add(int_coord_bld, offset,
@@ -3767,6 +3784,8 @@ lp_build_sample_soa_code(struct gallivm_state *gallivm,
                  derived_sampler_state.min_img_filter ==
                     derived_sampler_state.mag_img_filter;
 
+      use_aos &= !static_texture_state->tiled;
+
       if (gallivm_perf & GALLIVM_PERF_NO_AOS_SAMPLING) {
          use_aos = 0;
       }
@@ -4957,10 +4976,18 @@ lp_build_img_op_soa(const struct lp_static_texture_state *static_texture_state,
    }
 
    LLVMValueRef offset, i, j;
-   lp_build_sample_offset(&int_coord_bld,
+   if (static_texture_state->tiled) {
-                          format_desc,
+      lp_build_tiled_sample_offset(&int_coord_bld,
-                          x, y, z, row_stride_vec, img_stride_vec,
+                                   format_desc->format,
-                          &offset, &i, &j);
+                                   static_texture_state,
+                                   x, y, z, width, height, img_stride_vec,
+                                   &offset, &i, &j);
+   } else {
+      lp_build_sample_offset(&int_coord_bld,
+                             format_desc,
+                             x, y, z, row_stride_vec, img_stride_vec,
+                             &offset, &i, &j);
+   }
 
    if (params->ms_index && static_texture_state->level_zero_only) {
       LLVMValueRef num_samples = dynamic_state->last_level(gallivm,

src/gallium/drivers/llvmpipe/lp_texture.c

@@ -116,15 +116,38 @@ llvmpipe_texture_layout(struct llvmpipe_screen *screen,
     * for the virgl driver when host uses llvmpipe, causing Qemu and crosvm to
     * bail out on the KVM error.
     */
-   if (lpr->base.flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT)
+   if (lpr->base.flags & PIPE_RESOURCE_FLAG_SPARSE)
+      mip_align = 64 * 1024;
+   else if (lpr->base.flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT)
       os_get_page_size(&mip_align);
 
    assert(LP_MAX_TEXTURE_2D_LEVELS <= LP_MAX_TEXTURE_LEVELS);
    assert(LP_MAX_TEXTURE_3D_LEVELS <= LP_MAX_TEXTURE_LEVELS);
 
+   uint32_t dimensions = 1;
+   switch (pt->target) {
+   case PIPE_TEXTURE_2D:
+   case PIPE_TEXTURE_CUBE:
+   case PIPE_TEXTURE_RECT:
+   case PIPE_TEXTURE_2D_ARRAY:
+      dimensions = 2;
+      break;
+   case PIPE_TEXTURE_3D:
+      dimensions = 3;
+      break;
+   default:
+      break;
+   }
+
+   uint32_t sparse_tile_size[3] = {
+      util_format_get_tilesize(pt->format, dimensions, pt->nr_samples, 0),
+      util_format_get_tilesize(pt->format, dimensions, pt->nr_samples, 1),
+      util_format_get_tilesize(pt->format, dimensions, pt->nr_samples, 2),
+   };
+
    for (unsigned level = 0; level <= pt->last_level; level++) {
       uint64_t mipsize;
-      unsigned align_x, align_y, nblocksx, nblocksy, block_size, num_slices;
+      unsigned align_x, align_y, align_z, nblocksx, nblocksy, block_size, num_slices;
 
       /* Row stride and image stride */
 
@@ -145,6 +168,7 @@ llvmpipe_texture_layout(struct llvmpipe_screen *screen,
          else
             align_y = LP_RASTER_BLOCK_SIZE;
       }
+      align_z = 1;
 
       nblocksx = util_format_get_nblocksx(pt->format,
                                           align(width, align_x));
@@ -152,6 +176,12 @@ llvmpipe_texture_layout(struct llvmpipe_screen *screen,
                                           align(height, align_y));
       block_size = util_format_get_blocksize(pt->format);
 
+      if (pt->flags & PIPE_RESOURCE_FLAG_SPARSE) {
+         nblocksx = align(nblocksx, sparse_tile_size[0]);
+         nblocksy = align(nblocksy, sparse_tile_size[1]);
+         align_z = MAX2(align_z, sparse_tile_size[2]);
+      }
+
       if (util_format_is_compressed(pt->format))
          lpr->row_stride[level] = nblocksx * block_size;
       else
@@ -166,7 +196,7 @@ llvmpipe_texture_layout(struct llvmpipe_screen *screen,
       }
 
       if (lpr->base.target == PIPE_TEXTURE_3D)
-         num_slices = depth;
+         num_slices = align(depth, align_z);
       else if (lpr->base.target == PIPE_TEXTURE_1D_ARRAY ||
                lpr->base.target == PIPE_TEXTURE_2D_ARRAY ||
                lpr->base.target == PIPE_TEXTURE_CUBE ||
@@ -201,6 +231,11 @@ llvmpipe_texture_layout(struct llvmpipe_screen *screen,
          memset(lpr->tex_data, 0, total_size);
       }
    }
+   if (lpr->base.flags & PIPE_RESOURCE_FLAG_SPARSE) {
+      uint64_t page_align;
+      os_get_page_size(&page_align);
+      lpr->size_required = align64(lpr->size_required, page_align);
+   }
 
    return true;
 
@@ -285,6 +320,14 @@ llvmpipe_resource_create_all(struct pipe_screen *_screen,
          /* texture map */
          if (!llvmpipe_texture_layout(screen, lpr, alloc_backing))
             goto fail;
+
+         if (templat->flags & PIPE_RESOURCE_FLAG_SPARSE) {
+#if DETECT_OS_LINUX
+            lpr->tex_data = os_mmap(NULL, lpr->size_required, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_SHARED,
+                                    -1, 0);
+            madvise(lpr->tex_data, lpr->size_required, MADV_DONTNEED);
+#endif
+         }
       }
    } else {
       /* other data (vertex buffer, const buffer, etc) */
@@ -309,9 +352,8 @@ llvmpipe_resource_create_all(struct pipe_screen *_screen,
       if (!(templat->flags & PIPE_RESOURCE_FLAG_DONT_OVER_ALLOCATE))
          lpr->size_required += (LP_RASTER_BLOCK_SIZE - 1) * 4 * sizeof(float);
 
+      uint64_t alignment = sizeof(uint64_t) * 16;
       if (alloc_backing) {
-         uint64_t alignment = sizeof(uint64_t) * 16;
-
          if (templat->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT)
             os_get_page_size(&alignment);
 
@@ -321,6 +363,16 @@ llvmpipe_resource_create_all(struct pipe_screen *_screen,
             goto fail;
          memset(lpr->data, 0, bytes);
       }
+
+      if (templat->flags & PIPE_RESOURCE_FLAG_SPARSE) {
+         os_get_page_size(&alignment);
+         lpr->size_required = align64(lpr->size_required, alignment);
+#if DETECT_OS_LINUX
+         lpr->data = os_mmap(NULL, lpr->size_required, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_SHARED,
+                             -1, 0);
+         madvise(lpr->data, lpr->size_required, MADV_DONTNEED);
+#endif
+      }
    }
 
    lpr->id = id_counter++;
@@ -517,6 +569,15 @@ llvmpipe_resource_destroy(struct pipe_screen *pscreen,
       pscreen->free_memory_fd(pscreen, (struct pipe_memory_allocation*)lpr->dmabuf_alloc);
 #endif
 
+   if (lpr->base.flags & PIPE_RESOURCE_FLAG_SPARSE) {
+#if DETECT_OS_LINUX
+      if (llvmpipe_resource_is_texture(pt))
+         munmap(lpr->tex_data, lpr->size_required);
+      else
+         munmap(lpr->data, lpr->size_required);
+#endif
+   }
+
 #if MESA_DEBUG
    simple_mtx_lock(&resource_list_mutex);
    if (!list_is_empty(&lpr->list))
@@ -914,6 +975,48 @@ llvmpipe_transfer_map_ms(struct pipe_context *pipe,
 
    format = lpr->base.format;
 
+   if (llvmpipe_resource_is_texture(resource) && (resource->flags & PIPE_RESOURCE_FLAG_SPARSE)) {
+      map = llvmpipe_resource_map(resource, 0, 0, tex_usage);
+
+      lpt->block_box = (struct pipe_box) {
+         .x = box->x / util_format_get_blockwidth(format),
+         .width = DIV_ROUND_UP(box->x + box->width, util_format_get_blockwidth(format)),
+         .y = box->y / util_format_get_blockheight(format),
+         .height = DIV_ROUND_UP(box->y + box->height, util_format_get_blockheight(format)),
+         .z = box->z / util_format_get_blockdepth(format),
+         .depth = DIV_ROUND_UP(box->z + box->depth, util_format_get_blockdepth(format)),
+      };
+
+      lpt->block_box.width -= lpt->block_box.x;
+      lpt->block_box.height -= lpt->block_box.y;
+      lpt->block_box.depth -= lpt->block_box.z;
+
+      uint32_t block_stride = util_format_get_blocksize(format);
+      pt->stride = lpt->block_box.width * block_stride;
+      pt->layer_stride = pt->stride * lpt->block_box.height;
+
+      uint8_t *staging_map = malloc(pt->layer_stride * lpt->block_box.depth);
+      lpt->map = staging_map;
+
+      if (usage & PIPE_MAP_READ) {
+         for (uint32_t z = 0; z < lpt->block_box.depth; z++) {
+            for (uint32_t y = 0; y < lpt->block_box.height; y++) {
+               for (uint32_t x = 0; x < lpt->block_box.width; x++) {
+                  memcpy(staging_map,
+                         map + llvmpipe_get_texel_offset(resource, level,
+                                                         lpt->block_box.x + x,
+                                                         lpt->block_box.y + y,
+                                                         lpt->block_box.z + z),
+                         block_stride);
+                  staging_map += block_stride;
+               }
+            }
+         }
+      }
+
+      return lpt->map;
+   }
+
    map = llvmpipe_resource_map(resource, level, box->z, tex_usage);
 
 
@@ -934,6 +1037,60 @@ llvmpipe_transfer_map_ms(struct pipe_context *pipe,
    return map;
 }
 
+uint32_t
+llvmpipe_get_texel_offset(struct pipe_resource *resource,
+                          uint32_t level, uint32_t x,
+                          uint32_t y, uint32_t z)
+{
+   struct llvmpipe_resource *lpr = llvmpipe_resource(resource);
+
+   uint32_t layer = 0;
+   if (resource->target != PIPE_TEXTURE_3D) {
+      layer = z;
+      z = 0;
+   }
+
+   uint32_t dimensions = 1;
+   switch (resource->target) {
+   case PIPE_TEXTURE_2D:
+   case PIPE_TEXTURE_CUBE:
+   case PIPE_TEXTURE_RECT:
+   case PIPE_TEXTURE_2D_ARRAY:
+      dimensions = 2;
+      break;
+   case PIPE_TEXTURE_3D:
+      dimensions = 3;
+      break;
+   default:
+      break;
+   }
+
+   uint32_t sparse_tile_size[3] = {
+      util_format_get_tilesize(resource->format, dimensions, resource->nr_samples, 0),
+      util_format_get_tilesize(resource->format, dimensions, resource->nr_samples, 1),
+      util_format_get_tilesize(resource->format, dimensions, resource->nr_samples, 2),
+   };
+
+   uint32_t num_tiles_x = DIV_ROUND_UP(u_minify(resource->width0, level),
+                                       sparse_tile_size[0] * util_format_get_blockwidth(resource->format));
+   uint32_t num_tiles_y = DIV_ROUND_UP(u_minify(resource->height0, level),
+                                       sparse_tile_size[1] * util_format_get_blockheight(resource->format));
+
+   uint32_t offset = (
+      x / sparse_tile_size[0] +
+      y / sparse_tile_size[1] * num_tiles_x +
+      z / sparse_tile_size[2] * num_tiles_x * num_tiles_y
+   ) * 64 * 1024;
+
+   offset += (
+      x % sparse_tile_size[0] + 
+      (y % sparse_tile_size[1]) * sparse_tile_size[0] +
+      (z % sparse_tile_size[2]) * sparse_tile_size[0] * sparse_tile_size[1]
+   ) * util_format_get_blocksize(resource->format);
+
+   return offset + lpr->mip_offsets[level] + lpr->img_stride[level] * layer;   
+}
+
 
 static void *
 llvmpipe_transfer_map(struct pipe_context *pipe,
@@ -952,18 +1109,39 @@ static void
 llvmpipe_transfer_unmap(struct pipe_context *pipe,
                         struct pipe_transfer *transfer)
 {
-   assert(transfer->resource);
+   struct llvmpipe_transfer *lpt = (struct llvmpipe_transfer *)transfer;
+   struct pipe_resource *resource = transfer->resource;
+   struct llvmpipe_resource *lpr = llvmpipe_resource(resource);
 
-   llvmpipe_resource_unmap(transfer->resource,
+   assert(resource);
+
+   if (llvmpipe_resource_is_texture(resource) && (resource->flags & PIPE_RESOURCE_FLAG_SPARSE) &&
+       (transfer->usage & PIPE_MAP_WRITE)) {
+      uint32_t block_stride = util_format_get_blocksize(resource->format);
+
+      const uint8_t *src = lpt->map;
+      uint8_t *dst = lpr->tex_data;
+
+      for (uint32_t z = 0; z < lpt->block_box.depth; z++) {
+         for (uint32_t y = 0; y < lpt->block_box.height; y++) {
+            for (uint32_t x = 0; x < lpt->block_box.width; x++) {
+               memcpy(dst + llvmpipe_get_texel_offset(resource, transfer->level,
+                                                      lpt->block_box.x + x,
+                                                      lpt->block_box.y + y,
+                                                      lpt->block_box.z + z),
+                      src, block_stride);
+               src += block_stride;
+            }
+         }
+      }
+   }
+
+   llvmpipe_resource_unmap(resource,
                            transfer->level,
                            transfer->box.z);
 
-   /* Effectively do the texture_update work here - if texture images
+   pipe_resource_reference(&resource, NULL);
-    * needed post-processing to put them into hardware layout, this is
+   free(lpt->map);
-    * where it would happen.  For llvmpipe, nothing to do.
-    */
-   assert (transfer->resource);
-   pipe_resource_reference(&transfer->resource, NULL);
    FREE(transfer);
 }
 
@@ -1379,6 +1557,31 @@ llvmpipe_resource_bind_backing(struct pipe_screen *pscreen,
    if (!lpr->backable)
       return false;
 
+   if ((lpr->base.flags & PIPE_RESOURCE_FLAG_SPARSE) && offset < lpr->size_required) {
+#if DETECT_OS_LINUX
+      struct llvmpipe_memory_allocation *mem = (struct llvmpipe_memory_allocation *)pmem;
+      if (mem) {
+         if (llvmpipe_resource_is_texture(&lpr->base)) {
+            mmap((char *)lpr->tex_data + offset, size, PROT_READ|PROT_WRITE,
+                 MAP_SHARED|MAP_FIXED, mem->fd, mem->offset + fd_offset);
+         } else {
+            mmap((char *)lpr->data + offset, size, PROT_READ|PROT_WRITE,
+                 MAP_SHARED|MAP_FIXED, mem->fd, mem->offset + fd_offset);
+         }
+      } else {
+         if (llvmpipe_resource_is_texture(&lpr->base)) {
+            mmap((char *)lpr->tex_data + offset, size, PROT_READ|PROT_WRITE,
+                 MAP_SHARED|MAP_FIXED|MAP_ANONYMOUS, -1, 0);
+         } else {
+            mmap((char *)lpr->data + offset, size, PROT_READ|PROT_WRITE,
+                 MAP_SHARED|MAP_FIXED|MAP_ANONYMOUS, -1, 0);
+         }
+      }
+#endif
+
+      return true;
+   }
+
    addr = llvmpipe_map_memory(pscreen, pmem);
 
    if (llvmpipe_resource_is_texture(&lpr->base)) {

src/gallium/drivers/llvmpipe/lp_texture.h

@@ -122,6 +122,8 @@ struct llvmpipe_resource
 struct llvmpipe_transfer
 {
    struct pipe_transfer base;
+   void *map;
+   struct pipe_box block_box;
 };
 
 struct llvmpipe_memory_allocation
@@ -300,4 +302,9 @@ llvmpipe_transfer_map_ms(struct pipe_context *pipe,
                          const struct pipe_box *box,
                          struct pipe_transfer **transfer);
 
+uint32_t
+llvmpipe_get_texel_offset(struct pipe_resource *resource,
+                          uint32_t level, uint32_t x,
+                          uint32_t y, uint32_t z);
+
 #endif /* LP_TEXTURE_H */

src/gallium/drivers/llvmpipe/lp_texture_handle.c

@@ -122,7 +122,7 @@ llvmpipe_create_image_handle(struct pipe_context *pctx, const struct pipe_image_
    if (view->u.tex.first_layer == view->u.tex.last_layer) {
       if (state.target == PIPE_TEXTURE_1D_ARRAY)
          state.target = PIPE_TEXTURE_1D;
-      else if (state.target == PIPE_TEXTURE_2D_ARRAY || state.target == PIPE_TEXTURE_3D)
+      else if (state.target == PIPE_TEXTURE_2D_ARRAY || (state.target == PIPE_TEXTURE_3D && !state.tiled))
          state.target = PIPE_TEXTURE_2D;
       else if (state.target == PIPE_TEXTURE_CUBE_ARRAY)
          state.target = PIPE_TEXTURE_CUBE;

src/gallium/drivers/zink/ci/zink-lvp-fails.txt

@@ -140,3 +140,6 @@ spec@!opengl 1.0@gl-1.0-dlist-beginend,Crash
 spec@nv_texture_barrier@blending-in-shader,Crash
 
 spec@arb_viewport_array@display-list,Fail
+
+# passes locally
+KHR-GL46.sparse_texture_tests.SparseTextureCommitment,Fail

src/gallium/frontends/lavapipe/ci/lvp-asan-fails.txt

@@ -40,3 +40,6 @@ dEQP-VK.pipeline.shader_object_linked_binary.multisample.multisampled_render_to_
 dEQP-VK.pipeline.shader_object_linked_spirv.multisample.multisampled_render_to_single_sampled.dynamic_rendering.multi_renderpass.r8g8b8a8_unorm_r16g16b16a16_sfloat_r16g16b16a16_sint_d32_sfloat_s8_uint.random_119,Fail
 dEQP-VK.pipeline.shader_object_linked_spirv.multisample.multisampled_render_to_single_sampled.multi_renderpass.r8g8b8a8_unorm_r16g16b16a16_sfloat_r16g16b16a16_sint_d32_sfloat_s8_uint.random_119,Fail
 dEQP-VK.pipeline.shader_object_linked_spirv.multisample.multisampled_render_to_single_sampled.multi_subpass.r8g8b8a8_unorm_r16g16b16a16_sfloat_r16g16b16a16_sint_d32_sfloat_s8_uint.random_119,Fail
+
+# The test has a loop that exceeds LP_MAX_TGSI_LOOP_ITERATIONS
+dEQP-VK.sparse_resources.buffer.ssbo.sparse_residency.buffer_size_2_24

src/gallium/frontends/lavapipe/ci/lvp-asan-skips.txt

@@ -9,3 +9,6 @@ dEQP-VK.subgroups.ballot_broadcast.compute.subgroupbroadcast.*
 dEQP-VK.ssbo.layout.3_level_unsized_array.scalar.*
 dEQP-VK.ssbo.layout.3_level_array.scalar.*
 dEQP-VK.ssbo.phys.layout.unsized_struct_array.per_block_buffer.*_instance_array_comp_access_store_cols
+
+# https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/5117
+dEQP-VK.image.host_image_copy.*

src/gallium/frontends/lavapipe/ci/lvp-fails.txt

@@ -5,6 +5,9 @@ dEQP-VK.rasterization.provoking_vertex.transform_feedback.per_pipeline.triangle_
 
 dEQP-VK.api.version_check.unavailable_entry_points,Fail
 
+# The test has a loop that exceeds LP_MAX_TGSI_LOOP_ITERATIONS
+dEQP-VK.sparse_resources.buffer.ssbo.sparse_residency.buffer_size_2_24,Fail
+
 # New failure since CTS 1.3.8.0
 dEQP-VK.api.get_device_proc_addr.non_enabled,Fail
 

src/gallium/frontends/lavapipe/ci/lvp-skips.txt

@@ -47,3 +47,6 @@ dEQP-VK.binding_model.descriptor_buffer.multiple.graphics_vert_buffers8_sets1
 
 # New timeout since CTS 1.3.8.0
 dEQP-VK.mesh_shader.ext.misc.many_task_work_groups_z
+
+# https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/5117
+dEQP-VK.image.host_image_copy.*

src/gallium/frontends/lavapipe/lvp_device.c

@@ -47,6 +47,7 @@
 
 #if DETECT_OS_LINUX
 #include <sys/mman.h>
+#include <sys/resource.h>
 #endif
 
 #if defined(VK_USE_PLATFORM_WAYLAND_KHR) || \
@@ -343,6 +344,11 @@ lvp_get_features(const struct lvp_physical_device *pdevice,
       .shaderInt16                              = (min_shader_param(pdevice->pscreen, PIPE_SHADER_CAP_INT16) == 1),
       .variableMultisampleRate                  = false,
       .inheritedQueries                         = false,
+      .sparseBinding                            = DETECT_OS_LINUX,
+      .sparseResidencyBuffer                    = DETECT_OS_LINUX,
+      .sparseResidencyImage2D                   = DETECT_OS_LINUX,
+      .sparseResidencyImage3D                   = DETECT_OS_LINUX,
+      .sparseResidencyAliased                   = DETECT_OS_LINUX,
 
       /* Vulkan 1.1 */
       .storageBuffer16BitAccess            = true,
@@ -628,8 +634,8 @@ lvp_get_features(const struct lvp_physical_device *pdevice,
       .shaderSharedFloat64AtomicAdd =  false,
       .shaderImageFloat32Atomics =     true,
       .shaderImageFloat32AtomicAdd =   true,
-      .sparseImageFloat32Atomics =     false,
+      .sparseImageFloat32Atomics =     DETECT_OS_LINUX,
-      .sparseImageFloat32AtomicAdd =   false,
+      .sparseImageFloat32AtomicAdd =   DETECT_OS_LINUX,
 
       /* VK_EXT_shader_atomic_float2 */
       .shaderBufferFloat16Atomics      = false,
@@ -773,7 +779,7 @@ lvp_get_properties(const struct lvp_physical_device *device, struct vk_propertie
       .maxMemoryAllocationCount                 = UINT32_MAX,
       .maxSamplerAllocationCount                = 32 * 1024,
       .bufferImageGranularity                   = 64, /* A cache line */
-      .sparseAddressSpaceSize                   = 0,
+      .sparseAddressSpaceSize                   = 2UL*1024*1024*1024,
       .maxBoundDescriptorSets                   = MAX_SETS,
       .maxPerStageDescriptorSamplers            = MAX_DESCRIPTORS,
       .maxPerStageDescriptorUniformBuffers      = MAX_DESCRIPTORS,
@@ -871,6 +877,9 @@ lvp_get_properties(const struct lvp_physical_device *device, struct vk_propertie
       .optimalBufferCopyOffsetAlignment         = 128,
       .optimalBufferCopyRowPitchAlignment       = 128,
       .nonCoherentAtomSize                      = 64,
+      .sparseResidencyStandard2DBlockShape      = true,
+      .sparseResidencyStandard2DMultisampleBlockShape = true,
+      .sparseResidencyStandard3DBlockShape      = true,
 
       /* Vulkan 1.1 */
       /* The LUID is for Windows. */
@@ -1418,7 +1427,8 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetPhysicalDeviceQueueFamilyProperties2(
       p->queueFamilyProperties = (VkQueueFamilyProperties) {
          .queueFlags = VK_QUEUE_GRAPHICS_BIT |
          VK_QUEUE_COMPUTE_BIT |
-         VK_QUEUE_TRANSFER_BIT,
+         VK_QUEUE_TRANSFER_BIT |
+         (DETECT_OS_LINUX ? VK_QUEUE_SPARSE_BINDING_BIT : 0),
          .queueCount = 1,
          .timestampValidBits = 64,
          .minImageTransferGranularity = (VkExtent3D) { 1, 1, 1 },
@@ -1535,6 +1545,24 @@ lvp_queue_submit(struct vk_queue *vk_queue,
 
    simple_mtx_lock(&queue->lock);
 
+   for (uint32_t i = 0; i < submit->buffer_bind_count; i++) {
+      VkSparseBufferMemoryBindInfo *bind = &submit->buffer_binds[i];
+
+      lvp_buffer_bind_sparse(queue->device, queue, bind);
+   }
+
+   for (uint32_t i = 0; i < submit->image_opaque_bind_count; i++) {
+      VkSparseImageOpaqueMemoryBindInfo *bind = &submit->image_opaque_binds[i];
+
+      lvp_image_bind_opaque_sparse(queue->device, queue, bind);
+   }
+
+   for (uint32_t i = 0; i < submit->image_bind_count; i++) {
+      VkSparseImageMemoryBindInfo *bind = &submit->image_binds[i];
+
+      lvp_image_bind_sparse(queue->device, queue, bind);
+   }
+
    for (uint32_t i = 0; i < submit->command_buffer_count; i++) {
       struct lvp_cmd_buffer *cmd_buffer =
          container_of(submit->command_buffers[i], struct lvp_cmd_buffer, vk);
@@ -1993,6 +2021,12 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetDeviceBufferMemoryRequirements(
 {
    pMemoryRequirements->memoryRequirements.memoryTypeBits = 1;
    pMemoryRequirements->memoryRequirements.alignment = 64;
+
+   if (pInfo->pCreateInfo->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) {
+      uint64_t alignment;
+      os_get_page_size(&alignment);
+      pMemoryRequirements->memoryRequirements.alignment = alignment;
+   }
    pMemoryRequirements->memoryRequirements.size = 0;
 
    VkBuffer _buffer;
@@ -2003,15 +2037,6 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetDeviceBufferMemoryRequirements(
    lvp_DestroyBuffer(_device, _buffer, NULL);
 }
 
-VKAPI_ATTR void VKAPI_CALL lvp_GetDeviceImageSparseMemoryRequirements(
-    VkDevice                                    device,
-    const VkDeviceImageMemoryRequirements*      pInfo,
-    uint32_t*                                   pSparseMemoryRequirementCount,
-    VkSparseImageMemoryRequirements2*           pSparseMemoryRequirements)
-{
-   stub();
-}
-
 VKAPI_ATTR void VKAPI_CALL lvp_GetDeviceImageMemoryRequirements(
     VkDevice                                    _device,
     const VkDeviceImageMemoryRequirements*     pInfo,
@@ -2037,6 +2062,12 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetBufferMemoryRequirements(
 {
    LVP_FROM_HANDLE(lvp_buffer, buffer, _buffer);
 
+   pMemoryRequirements->alignment = 64;
+   if (buffer->vk.create_flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) {
+      uint64_t alignment;
+      os_get_page_size(&alignment);
+      pMemoryRequirements->alignment = alignment;
+   }
    /* The Vulkan spec (git aaed022) says:
     *
     *    memoryTypeBits is a bitfield and contains one bit set for every
@@ -2049,7 +2080,6 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetBufferMemoryRequirements(
    pMemoryRequirements->memoryTypeBits = 1;
 
    pMemoryRequirements->size = buffer->total_size;
-   pMemoryRequirements->alignment = 64;
 }
 
 VKAPI_ATTR void VKAPI_CALL lvp_GetBufferMemoryRequirements2(
@@ -2109,24 +2139,6 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetImageMemoryRequirements2(
    }
 }
 
-VKAPI_ATTR void VKAPI_CALL lvp_GetImageSparseMemoryRequirements(
-   VkDevice                                    device,
-   VkImage                                     image,
-   uint32_t*                                   pSparseMemoryRequirementCount,
-   VkSparseImageMemoryRequirements*            pSparseMemoryRequirements)
-{
-   stub();
-}
-
-VKAPI_ATTR void VKAPI_CALL lvp_GetImageSparseMemoryRequirements2(
-   VkDevice                                    device,
-   const VkImageSparseMemoryRequirementsInfo2* pInfo,
-   uint32_t* pSparseMemoryRequirementCount,
-   VkSparseImageMemoryRequirements2* pSparseMemoryRequirements)
-{
-   stub();
-}
-
 VKAPI_ATTR void VKAPI_CALL lvp_GetDeviceMemoryCommitment(
    VkDevice                                    device,
    VkDeviceMemory                              memory,
@@ -2291,15 +2303,6 @@ lvp_GetMemoryFdPropertiesKHR(VkDevice _device,
 
 #endif
 
-VKAPI_ATTR VkResult VKAPI_CALL lvp_QueueBindSparse(
-   VkQueue                                     queue,
-   uint32_t                                    bindInfoCount,
-   const VkBindSparseInfo*                     pBindInfo,
-   VkFence                                     fence)
-{
-   stub_return(VK_ERROR_INCOMPATIBLE_DRIVER);
-}
-
 VKAPI_ATTR VkResult VKAPI_CALL lvp_CreateEvent(
    VkDevice                                    _device,
    const VkEventCreateInfo*                    pCreateInfo,

src/gallium/frontends/lavapipe/lvp_formats.c

@@ -508,18 +508,20 @@ VKAPI_ATTR VkResult VKAPI_CALL lvp_GetPhysicalDeviceImageFormatProperties2(
    return VK_SUCCESS;
 }
 
-VKAPI_ATTR void VKAPI_CALL lvp_GetPhysicalDeviceSparseImageFormatProperties(
+static void
-    VkPhysicalDevice                            physicalDevice,
+fill_sparse_image_format_properties(struct lvp_physical_device *pdev, VkImageType type,
-    VkFormat                                    format,
+                                    VkFormat format, VkSampleCountFlagBits samples,
-    VkImageType                                 type,
+                                    VkSparseImageFormatProperties *prop)
-    VkSampleCountFlagBits                       samples,
-    VkImageUsageFlags                           usage,
-    VkImageTiling                               tiling,
-    uint32_t*                                   pNumProperties,
-    VkSparseImageFormatProperties*              pProperties)
 {
-   /* Sparse images are not yet supported. */
+   enum pipe_format pformat = vk_format_to_pipe_format(format);
-   *pNumProperties = 0;
+
+   prop->aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
+   prop->flags = 0;
+   prop->imageGranularity = (VkExtent3D){
+      .width = util_format_get_tilesize(pformat, type + 1, samples, 0) * util_format_get_blockwidth(pformat),
+      .height = util_format_get_tilesize(pformat, type + 1, samples, 1) * util_format_get_blockheight(pformat),
+      .depth = util_format_get_tilesize(pformat, type + 1, samples, 2) * util_format_get_blockdepth(pformat),
+   };
 }
 
 VKAPI_ATTR void VKAPI_CALL lvp_GetPhysicalDeviceSparseImageFormatProperties2(
@@ -528,10 +530,98 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetPhysicalDeviceSparseImageFormatProperties2(
         uint32_t                                   *pPropertyCount,
         VkSparseImageFormatProperties2             *pProperties)
 {
-        /* Sparse images are not yet supported. */
+   LVP_FROM_HANDLE(lvp_physical_device, physical_device, physicalDevice);
-        *pPropertyCount = 0;
+   VkResult result;
+
+   if (pFormatInfo->samples > VK_SAMPLE_COUNT_1_BIT) {
+      *pPropertyCount = 0;
+      return;
+   }
+   const VkPhysicalDeviceImageFormatInfo2 fmt_info = {
+      .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
+      .format = pFormatInfo->format,
+      .type = pFormatInfo->type,
+      .tiling = pFormatInfo->tiling,
+      .usage = pFormatInfo->usage,
+      .flags = VK_IMAGE_CREATE_SPARSE_BINDING_BIT | VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT};
+
+   VkImageFormatProperties fmt_props;
+   result = lvp_get_image_format_properties(physical_device, &fmt_info,
+                                            &fmt_props);
+   if (result != VK_SUCCESS) {
+      *pPropertyCount = 0;
+      return;
+   }
+
+   VK_OUTARRAY_MAKE_TYPED(VkSparseImageFormatProperties2, out, pProperties, pPropertyCount);
+
+   vk_outarray_append_typed(VkSparseImageFormatProperties2, &out, prop)
+   {
+      fill_sparse_image_format_properties(physical_device, pFormatInfo->type, pFormatInfo->format,
+                                          pFormatInfo->samples, &prop->properties);
+   };
 }
 
+VKAPI_ATTR void VKAPI_CALL lvp_GetDeviceImageSparseMemoryRequirements(
+    VkDevice                                    _device,
+    const VkDeviceImageMemoryRequirements*      pInfo,
+    uint32_t*                                   pSparseMemoryRequirementCount,
+    VkSparseImageMemoryRequirements2*           pSparseMemoryRequirements)
+{
+   LVP_FROM_HANDLE(lvp_device, device, _device);
+
+   if (!(pInfo->pCreateInfo->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT)) {
+      *pSparseMemoryRequirementCount = 0;
+      return;
+   }
+
+   VK_OUTARRAY_MAKE_TYPED(VkSparseImageMemoryRequirements2, out, pSparseMemoryRequirements,
+                          pSparseMemoryRequirementCount);
+
+   vk_outarray_append_typed(VkSparseImageMemoryRequirements2, &out, req)
+   {
+      fill_sparse_image_format_properties(device->physical_device, pInfo->pCreateInfo->imageType,
+                                          pInfo->pCreateInfo->format, pInfo->pCreateInfo->samples,
+                                          &req->memoryRequirements.formatProperties);
+
+      req->memoryRequirements.imageMipTailFirstLod = pInfo->pCreateInfo->mipLevels;
+      req->memoryRequirements.imageMipTailSize = 0;
+      req->memoryRequirements.imageMipTailOffset = 0;
+      req->memoryRequirements.imageMipTailStride = 0;
+   };
+}
+
+VKAPI_ATTR void VKAPI_CALL lvp_GetImageSparseMemoryRequirements2(
+   VkDevice                                    _device,
+   const VkImageSparseMemoryRequirementsInfo2* pInfo,
+   uint32_t* pSparseMemoryRequirementCount,
+   VkSparseImageMemoryRequirements2* pSparseMemoryRequirements)
+{
+   LVP_FROM_HANDLE(lvp_device, device, _device);
+   LVP_FROM_HANDLE(lvp_image, image, pInfo->image);
+
+   if (!(image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT)) {
+      *pSparseMemoryRequirementCount = 0;
+      return;
+   }
+
+   VK_OUTARRAY_MAKE_TYPED(VkSparseImageMemoryRequirements2, out, pSparseMemoryRequirements,
+                          pSparseMemoryRequirementCount);
+
+   vk_outarray_append_typed(VkSparseImageMemoryRequirements2, &out, req)
+   {
+      fill_sparse_image_format_properties(device->physical_device, image->vk.image_type,
+                                          image->vk.format, image->vk.samples,
+                                          &req->memoryRequirements.formatProperties);
+
+      req->memoryRequirements.imageMipTailFirstLod = image->vk.mip_levels;
+      req->memoryRequirements.imageMipTailSize = 0;
+      req->memoryRequirements.imageMipTailOffset = 0;
+      req->memoryRequirements.imageMipTailStride = 0;
+   };
+}
+
+
 VKAPI_ATTR void VKAPI_CALL lvp_GetPhysicalDeviceExternalBufferProperties(
    VkPhysicalDevice                            physicalDevice,
    const VkPhysicalDeviceExternalBufferInfo    *pExternalBufferInfo,

src/gallium/frontends/lavapipe/lvp_image.c

@@ -66,6 +66,9 @@ lvp_image_create(VkDevice _device,
       return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
 
    image->alignment = 64;
+   if (image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT)
+      image->alignment = 64 * 1024;
+
    image->plane_count = vk_format_get_plane_count(pCreateInfo->format);
    image->disjoint = image->plane_count > 1 &&
                      (pCreateInfo->flags & VK_IMAGE_CREATE_DISJOINT_BIT);
@@ -126,6 +129,9 @@ lvp_image_create(VkDevice _device,
                                 VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT))
          template.bind |= PIPE_BIND_SHADER_IMAGE;
 
+      if (pCreateInfo->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT)
+         template.flags |= PIPE_RESOURCE_FLAG_SPARSE;
+
       template.width0 = pCreateInfo->extent.width / width_scale;
       template.height0 = pCreateInfo->extent.height / height_scale;
       template.depth0 = pCreateInfo->extent.depth;
@@ -162,6 +168,8 @@ lvp_image_create(VkDevice _device,
       if (!image->planes[p].bo)
          return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
 
+      image->planes[p].size = align64(image->planes[p].size, image->alignment);
+
       image->size += image->planes[p].size;
    }
    *pImage = lvp_image_to_handle(image);
@@ -536,6 +544,8 @@ VKAPI_ATTR VkResult VKAPI_CALL lvp_CreateBuffer(
       if (buffer->vk.usage & VK_BUFFER_USAGE_2_STORAGE_TEXEL_BUFFER_BIT_KHR)
          template.bind |= PIPE_BIND_SHADER_IMAGE;
       template.flags = PIPE_RESOURCE_FLAG_DONT_OVER_ALLOCATE;
+      if (pCreateInfo->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT)
+         template.flags |= PIPE_RESOURCE_FLAG_SPARSE;
       buffer->bo = device->pscreen->resource_create_unbacked(device->pscreen,
                                                              &template,
                                                              &buffer->total_size);
@@ -543,6 +553,12 @@ VKAPI_ATTR VkResult VKAPI_CALL lvp_CreateBuffer(
          vk_free2(&device->vk.alloc, pAllocator, buffer);
          return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY);
       }
+
+      if (pCreateInfo->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) {
+         buffer->map = device->queue.ctx->buffer_map(device->queue.ctx, buffer->bo, 0,
+                                                     PIPE_MAP_READ | PIPE_MAP_WRITE | PIPE_MAP_PERSISTENT,
+                                                     &(struct pipe_box){ 0 }, &buffer->transfer);
+      }
    }
    *pBuffer = lvp_buffer_to_handle(buffer);
 
@@ -566,6 +582,9 @@ VKAPI_ATTR void VKAPI_CALL lvp_DestroyBuffer(
       if (he)
          _mesa_hash_table_remove(&device->bda, he);
       simple_mtx_unlock(&device->bda_lock);
+
+      if (buffer->bo->flags & PIPE_RESOURCE_FLAG_SPARSE)
+         device->queue.ctx->buffer_unmap(device->queue.ctx, buffer->transfer);
    }
    pipe_resource_reference(&buffer->bo, NULL);
    vk_buffer_destroy(&device->vk, pAllocator, &buffer->vk);
@@ -836,3 +855,134 @@ lvp_TransitionImageLayoutEXT(VkDevice device, uint32_t transitionCount, const Vk
    /* no-op */
    return VK_SUCCESS;
 }
+
+VkResult
+lvp_buffer_bind_sparse(struct lvp_device *device,
+                       struct lvp_queue *queue,
+                       VkSparseBufferMemoryBindInfo *bind)
+{
+   LVP_FROM_HANDLE(lvp_buffer, buffer, bind->buffer);
+
+   for (uint32_t i = 0; i < bind->bindCount; i++) {
+      LVP_FROM_HANDLE(lvp_device_memory, mem, bind->pBinds[i].memory);
+      device->pscreen->resource_bind_backing(device->pscreen,
+                                             buffer->bo,
+                                             mem ? mem->pmem : NULL,
+                                             bind->pBinds[i].memoryOffset,
+                                             bind->pBinds[i].size,
+                                             bind->pBinds[i].resourceOffset);
+   }
+
+   return VK_SUCCESS;
+}
+
+VkResult
+lvp_image_bind_opaque_sparse(struct lvp_device *device,
+                             struct lvp_queue *queue,
+                             VkSparseImageOpaqueMemoryBindInfo *bind_info)
+{
+   LVP_FROM_HANDLE(lvp_image, image, bind_info->image);
+
+   for (uint32_t i = 0; i < bind_info->bindCount; i++) {
+      const VkSparseMemoryBind *bind = &bind_info->pBinds[i];
+      LVP_FROM_HANDLE(lvp_device_memory, mem, bind->memory);
+
+      uint32_t plane_index;
+      uint32_t offset;
+      if (bind->resourceOffset < image->planes[0].size) {
+         plane_index = 0;
+         offset = bind->resourceOffset;
+      } else if (bind->resourceOffset < image->planes[0].size + image->planes[1].size) {
+         plane_index = 1;
+         offset = bind->resourceOffset - image->planes[0].size;
+      } else {
+         plane_index = 2;
+         offset = bind->resourceOffset - image->planes[0].size - image->planes[1].size;
+      }
+
+      device->pscreen->resource_bind_backing(device->pscreen,
+                                             image->planes[plane_index].bo,
+                                             mem ? mem->pmem : NULL,
+                                             bind->memoryOffset,
+                                             bind->size,
+                                             offset);
+   }
+
+   return VK_SUCCESS;
+}
+
+VkResult
+lvp_image_bind_sparse(struct lvp_device *device,
+                      struct lvp_queue *queue,
+                      VkSparseImageMemoryBindInfo *bind_info)
+{
+   LVP_FROM_HANDLE(lvp_image, image, bind_info->image);
+
+   enum pipe_format format = vk_format_to_pipe_format(image->vk.format);
+
+   for (uint32_t i = 0; i < bind_info->bindCount; i++) {
+      const VkSparseImageMemoryBind *bind = &bind_info->pBinds[i];
+      LVP_FROM_HANDLE(lvp_device_memory, mem, bind->memory);
+
+      uint8_t plane = lvp_image_aspects_to_plane(image, bind->subresource.aspectMask);
+
+      uint32_t depth = 1;
+      uint32_t z = 0;
+      uint32_t dimensions = 2;
+      switch (image->planes[plane].bo->target) {
+      case PIPE_TEXTURE_CUBE:
+      case PIPE_TEXTURE_CUBE_ARRAY:
+      case PIPE_TEXTURE_2D_ARRAY:
+      case PIPE_TEXTURE_1D_ARRAY:
+         /* these use layer */
+         z = bind->subresource.arrayLayer;
+         break;
+      case PIPE_TEXTURE_3D:
+         /* this uses depth */
+         z = bind->offset.z;
+         depth = bind->extent.depth;
+         dimensions = 3;
+         break;
+      default:
+         break;
+      }
+
+      uint32_t sparse_tile_size[3] = {
+         util_format_get_tilesize(format, dimensions, image->vk.samples, 0),
+         util_format_get_tilesize(format, dimensions, image->vk.samples, 1),
+         util_format_get_tilesize(format, dimensions, image->vk.samples, 2),
+      };
+
+      uint32_t sparse_block_base[3] = {
+         bind->offset.x / (sparse_tile_size[0] * util_format_get_blockwidth(format)),
+         bind->offset.y / (sparse_tile_size[1] * util_format_get_blockheight(format)),
+         z / (sparse_tile_size[2] * util_format_get_blockdepth(format)),
+      };
+
+      uint32_t sparse_block_counts[3] = {
+         DIV_ROUND_UP(bind->extent.width, sparse_tile_size[0] * util_format_get_blockwidth(format)),
+         DIV_ROUND_UP(bind->extent.height, sparse_tile_size[1] * util_format_get_blockheight(format)),
+         DIV_ROUND_UP(depth, sparse_tile_size[2] * util_format_get_blockdepth(format)),
+      };
+
+      uint32_t sparse_block_count = sparse_block_counts[0] * sparse_block_counts[1] * sparse_block_counts[2];
+
+      for (uint32_t block = 0; block < sparse_block_count; block++) {
+         uint32_t start_x = (sparse_block_base[0] + block % sparse_block_counts[0]) * sparse_tile_size[0];
+         uint32_t start_y = (sparse_block_base[1] + (block / sparse_block_counts[0]) % sparse_block_counts[1]) *
+                            sparse_tile_size[1];
+         uint32_t start_z = (sparse_block_base[2] + (block / sparse_block_counts[0] / sparse_block_counts[1]) % sparse_block_counts[2]) *
+                            sparse_tile_size[2];
+
+         uint64_t offset = llvmpipe_get_texel_offset(image->planes[plane].bo, bind->subresource.mipLevel, start_x, start_y, start_z);
+         device->pscreen->resource_bind_backing(device->pscreen,
+                                                image->planes[plane].bo,
+                                                mem ? mem->pmem : NULL,
+                                                bind->memoryOffset + block * 64 * 1024,
+                                                64 * 1024,
+                                                offset);
+      }
+   }
+
+   return VK_SUCCESS;
+}

src/gallium/frontends/lavapipe/lvp_private.h

@@ -582,6 +582,7 @@ struct lvp_buffer {
    uint64_t total_size;
    uint64_t offset;
    void *map;
+   struct pipe_transfer *transfer;
 };
 
 struct lvp_buffer_view {
@@ -687,6 +688,16 @@ VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_indirect_command_layout_nv, base, VkIndirectC
 
 void lvp_add_enqueue_cmd_entrypoints(struct vk_device_dispatch_table *disp);
 
+VkResult lvp_buffer_bind_sparse(struct lvp_device *device,
+                                struct lvp_queue *queue,
+                                VkSparseBufferMemoryBindInfo *bind);
+VkResult lvp_image_bind_opaque_sparse(struct lvp_device *device,
+                                      struct lvp_queue *queue,
+                                      VkSparseImageOpaqueMemoryBindInfo *bind);
+VkResult lvp_image_bind_sparse(struct lvp_device *device,
+                               struct lvp_queue *queue,
+                               VkSparseImageMemoryBindInfo *bind);
+
 VkResult lvp_execute_cmds(struct lvp_device *device,
                           struct lvp_queue *queue,
                           struct lvp_cmd_buffer *cmd_buffer);