diff --git a/src/gallium/auxiliary/gallivm/lp_bld_sample.c b/src/gallium/auxiliary/gallivm/lp_bld_sample.c index 5c4b5149463..fcc925a19af 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_sample.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_sample.c @@ -127,6 +127,9 @@ lp_sampler_static_texture_state(struct lp_static_texture_state *state, state->pot_height = util_is_power_of_two_or_zero(texture->height0); state->pot_depth = util_is_power_of_two_or_zero(texture->depth0); state->level_zero_only = !view->u.tex.last_level; + state->tiled = !!(texture->flags & PIPE_RESOURCE_FLAG_SPARSE); + if (state->tiled) + state->tiled_samples = texture->nr_samples; /* * the layer / element / level parameters are all either dynamic @@ -167,6 +170,9 @@ lp_sampler_static_texture_state_image(struct lp_static_texture_state *state, state->pot_height = util_is_power_of_two_or_zero(resource->height0); state->pot_depth = util_is_power_of_two_or_zero(resource->depth0); state->level_zero_only = view->u.tex.level == 0; + state->tiled = !!(resource->flags & PIPE_RESOURCE_FLAG_SPARSE); + if (state->tiled) + state->tiled_samples = resource->nr_samples; /* * the layer / element / level parameters are all either dynamic @@ -2196,6 +2202,122 @@ lp_build_sample_offset(struct lp_build_context *bld, } + +void +lp_build_tiled_sample_offset(struct lp_build_context *bld, + enum pipe_format format, + const struct lp_static_texture_state *static_texture_state, + LLVMValueRef x, + LLVMValueRef y, + LLVMValueRef z, + LLVMValueRef width, + LLVMValueRef height, + LLVMValueRef z_stride, + LLVMValueRef *out_offset, + LLVMValueRef *out_i, + LLVMValueRef *out_j) +{ + struct gallivm_state *gallivm = bld->gallivm; + LLVMBuilderRef builder = gallivm->builder; + + assert(static_texture_state->tiled); + + uint32_t dimensions = 1; + switch (static_texture_state->target) { + case PIPE_TEXTURE_2D: + case PIPE_TEXTURE_CUBE: + case PIPE_TEXTURE_RECT: + case PIPE_TEXTURE_2D_ARRAY: + dimensions = 2; + break; + case PIPE_TEXTURE_3D: + dimensions = 3; + break; + default: + break; + } + + uint32_t block_size[3] = { + util_format_get_blockwidth(format), + util_format_get_blockheight(format), + util_format_get_blockdepth(format), + }; + + uint32_t sparse_tile_size[3] = { + util_format_get_tilesize(format, dimensions, static_texture_state->tiled_samples, 0) * block_size[0], + util_format_get_tilesize(format, dimensions, static_texture_state->tiled_samples, 1) * block_size[1], + util_format_get_tilesize(format, dimensions, static_texture_state->tiled_samples, 2) * block_size[2], + }; + + LLVMValueRef sparse_tile_size_log2[3] = { + lp_build_const_vec(gallivm, bld->type, util_logbase2(sparse_tile_size[0])), + lp_build_const_vec(gallivm, bld->type, util_logbase2(sparse_tile_size[1])), + lp_build_const_vec(gallivm, bld->type, util_logbase2(sparse_tile_size[2])), + }; + + LLVMValueRef tile_index = LLVMBuildLShr(builder, x, sparse_tile_size_log2[0], ""); + + if (y && dimensions > 1) { + LLVMValueRef x_tile_count = lp_build_add(bld, width, lp_build_const_vec(gallivm, bld->type, sparse_tile_size[0] - 1)); + x_tile_count = LLVMBuildLShr(builder, x_tile_count, sparse_tile_size_log2[0], ""); + LLVMValueRef y_tile = LLVMBuildLShr(builder, y, sparse_tile_size_log2[1], ""); + tile_index = lp_build_add(bld, tile_index, lp_build_mul(bld, y_tile, x_tile_count)); + + if (z && dimensions > 2) { + LLVMValueRef y_tile_count = lp_build_add(bld, height, lp_build_const_vec(gallivm, bld->type, sparse_tile_size[1] - 1)); + y_tile_count = LLVMBuildLShr(builder, y_tile_count, sparse_tile_size_log2[1], ""); + LLVMValueRef z_tile = LLVMBuildLShr(builder, z, sparse_tile_size_log2[2], ""); + tile_index = lp_build_add(bld, tile_index, lp_build_mul(bld, z_tile, lp_build_mul(bld, x_tile_count, y_tile_count))); + } + } + + LLVMValueRef offset = LLVMBuildShl(builder, tile_index, lp_build_const_vec(gallivm, bld->type, 16), ""); + + LLVMValueRef sparse_tile_masks[3] = { + lp_build_const_vec(gallivm, bld->type, sparse_tile_size[0] - 1), + lp_build_const_vec(gallivm, bld->type, sparse_tile_size[1] - 1), + lp_build_const_vec(gallivm, bld->type, sparse_tile_size[2] - 1), + }; + + x = LLVMBuildAnd(builder, x, sparse_tile_masks[0], ""); + LLVMValueRef x_stride = lp_build_const_vec(gallivm, bld->type, util_format_get_blocksize(format)); + + LLVMValueRef x_offset; + lp_build_sample_partial_offset(bld, block_size[0], + x, x_stride, &x_offset, out_i); + offset = lp_build_add(bld, offset, x_offset); + + if (y && dimensions > 1) { + y = LLVMBuildAnd(builder, y, sparse_tile_masks[1], ""); + LLVMValueRef y_stride = lp_build_const_vec(gallivm, bld->type, util_format_get_blocksize(format) * + sparse_tile_size[0] / block_size[0]); + + LLVMValueRef y_offset; + lp_build_sample_partial_offset(bld, block_size[1], + y, y_stride, &y_offset, out_j); + offset = lp_build_add(bld, offset, y_offset); + } else { + *out_j = bld->zero; + } + + if (z && (z_stride || dimensions > 2)) { + if (dimensions > 2) { + z = LLVMBuildAnd(builder, z, sparse_tile_masks[2], ""); + z_stride = lp_build_const_vec(gallivm, bld->type, util_format_get_blocksize(format) * + sparse_tile_size[0] / block_size[0] * + sparse_tile_size[1] / block_size[1]); + } + + LLVMValueRef z_offset; + LLVMValueRef k; + lp_build_sample_partial_offset(bld, 1, z, z_stride, &z_offset, &k); + offset = lp_build_add(bld, offset, z_offset); + } + + *out_offset = offset; +} + + static LLVMValueRef lp_build_sample_min(struct lp_build_context *bld, LLVMValueRef x, diff --git a/src/gallium/auxiliary/gallivm/lp_bld_sample.h b/src/gallium/auxiliary/gallivm/lp_bld_sample.h index e8c1a54b1be..9336955ec93 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_sample.h +++ b/src/gallium/auxiliary/gallivm/lp_bld_sample.h @@ -198,6 +198,8 @@ struct lp_static_texture_state unsigned pot_height:1; unsigned pot_depth:1; unsigned level_zero_only:1; + unsigned tiled:1; + unsigned tiled_samples:5; }; @@ -738,6 +740,21 @@ lp_build_sample_offset(struct lp_build_context *bld, LLVMValueRef *out_j); +void +lp_build_tiled_sample_offset(struct lp_build_context *bld, + enum pipe_format format, + const struct lp_static_texture_state *static_texture_state, + LLVMValueRef x, + LLVMValueRef y, + LLVMValueRef z, + LLVMValueRef width, + LLVMValueRef height, + LLVMValueRef z_stride, + LLVMValueRef *out_offset, + LLVMValueRef *out_i, + LLVMValueRef *out_j); + + void lp_build_sample_soa_code(struct gallivm_state *gallivm, const struct lp_static_texture_state *static_texture_state, diff --git a/src/gallium/auxiliary/gallivm/lp_bld_sample_soa.c b/src/gallium/auxiliary/gallivm/lp_bld_sample_soa.c index 4887a58cda2..3e9b0cb6f34 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_sample_soa.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_sample_soa.c @@ -138,10 +138,19 @@ lp_build_sample_texel_soa(struct lp_build_sample_context *bld, } /* convert x,y,z coords to linear offset from start of texture, in bytes */ - lp_build_sample_offset(&bld->int_coord_bld, - bld->format_desc, - x, y, z, y_stride, z_stride, - &offset, &i, &j); + if (bld->static_texture_state->tiled) { + lp_build_tiled_sample_offset(&bld->int_coord_bld, + bld->format_desc->format, + bld->static_texture_state, + x, y, z, width, height, z_stride, + &offset, &i, &j); + } else { + lp_build_sample_offset(&bld->int_coord_bld, + bld->format_desc, + x, y, z, y_stride, z_stride, + &offset, &i, &j); + } + if (mipoffsets) { offset = lp_build_add(&bld->int_coord_bld, offset, mipoffsets); } @@ -3248,10 +3257,18 @@ lp_build_fetch_texel(struct lp_build_sample_context *bld, } } - lp_build_sample_offset(int_coord_bld, - bld->format_desc, - x, y, z, row_stride_vec, img_stride_vec, - &offset, &i, &j); + if (bld->static_texture_state->tiled) { + lp_build_tiled_sample_offset(&bld->int_coord_bld, + bld->format_desc->format, + bld->static_texture_state, + x, y, z, width, height, img_stride_vec, + &offset, &i, &j); + } else { + lp_build_sample_offset(int_coord_bld, + bld->format_desc, + x, y, z, row_stride_vec, img_stride_vec, + &offset, &i, &j); + } if (bld->static_texture_state->target != PIPE_BUFFER) { offset = lp_build_add(int_coord_bld, offset, @@ -3767,6 +3784,8 @@ lp_build_sample_soa_code(struct gallivm_state *gallivm, derived_sampler_state.min_img_filter == derived_sampler_state.mag_img_filter; + use_aos &= !static_texture_state->tiled; + if (gallivm_perf & GALLIVM_PERF_NO_AOS_SAMPLING) { use_aos = 0; } @@ -4957,10 +4976,18 @@ lp_build_img_op_soa(const struct lp_static_texture_state *static_texture_state, } LLVMValueRef offset, i, j; - lp_build_sample_offset(&int_coord_bld, - format_desc, - x, y, z, row_stride_vec, img_stride_vec, - &offset, &i, &j); + if (static_texture_state->tiled) { + lp_build_tiled_sample_offset(&int_coord_bld, + format_desc->format, + static_texture_state, + x, y, z, width, height, img_stride_vec, + &offset, &i, &j); + } else { + lp_build_sample_offset(&int_coord_bld, + format_desc, + x, y, z, row_stride_vec, img_stride_vec, + &offset, &i, &j); + } if (params->ms_index && static_texture_state->level_zero_only) { LLVMValueRef num_samples = dynamic_state->last_level(gallivm, diff --git a/src/gallium/drivers/llvmpipe/lp_texture.c b/src/gallium/drivers/llvmpipe/lp_texture.c index a5713230f16..4563b84b3b1 100644 --- a/src/gallium/drivers/llvmpipe/lp_texture.c +++ b/src/gallium/drivers/llvmpipe/lp_texture.c @@ -116,15 +116,38 @@ llvmpipe_texture_layout(struct llvmpipe_screen *screen, * for the virgl driver when host uses llvmpipe, causing Qemu and crosvm to * bail out on the KVM error. */ - if (lpr->base.flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) + if (lpr->base.flags & PIPE_RESOURCE_FLAG_SPARSE) + mip_align = 64 * 1024; + else if (lpr->base.flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) os_get_page_size(&mip_align); assert(LP_MAX_TEXTURE_2D_LEVELS <= LP_MAX_TEXTURE_LEVELS); assert(LP_MAX_TEXTURE_3D_LEVELS <= LP_MAX_TEXTURE_LEVELS); + uint32_t dimensions = 1; + switch (pt->target) { + case PIPE_TEXTURE_2D: + case PIPE_TEXTURE_CUBE: + case PIPE_TEXTURE_RECT: + case PIPE_TEXTURE_2D_ARRAY: + dimensions = 2; + break; + case PIPE_TEXTURE_3D: + dimensions = 3; + break; + default: + break; + } + + uint32_t sparse_tile_size[3] = { + util_format_get_tilesize(pt->format, dimensions, pt->nr_samples, 0), + util_format_get_tilesize(pt->format, dimensions, pt->nr_samples, 1), + util_format_get_tilesize(pt->format, dimensions, pt->nr_samples, 2), + }; + for (unsigned level = 0; level <= pt->last_level; level++) { uint64_t mipsize; - unsigned align_x, align_y, nblocksx, nblocksy, block_size, num_slices; + unsigned align_x, align_y, align_z, nblocksx, nblocksy, block_size, num_slices; /* Row stride and image stride */ @@ -145,6 +168,7 @@ llvmpipe_texture_layout(struct llvmpipe_screen *screen, else align_y = LP_RASTER_BLOCK_SIZE; } + align_z = 1; nblocksx = util_format_get_nblocksx(pt->format, align(width, align_x)); @@ -152,6 +176,12 @@ llvmpipe_texture_layout(struct llvmpipe_screen *screen, align(height, align_y)); block_size = util_format_get_blocksize(pt->format); + if (pt->flags & PIPE_RESOURCE_FLAG_SPARSE) { + nblocksx = align(nblocksx, sparse_tile_size[0]); + nblocksy = align(nblocksy, sparse_tile_size[1]); + align_z = MAX2(align_z, sparse_tile_size[2]); + } + if (util_format_is_compressed(pt->format)) lpr->row_stride[level] = nblocksx * block_size; else @@ -166,7 +196,7 @@ llvmpipe_texture_layout(struct llvmpipe_screen *screen, } if (lpr->base.target == PIPE_TEXTURE_3D) - num_slices = depth; + num_slices = align(depth, align_z); else if (lpr->base.target == PIPE_TEXTURE_1D_ARRAY || lpr->base.target == PIPE_TEXTURE_2D_ARRAY || lpr->base.target == PIPE_TEXTURE_CUBE || @@ -201,6 +231,11 @@ llvmpipe_texture_layout(struct llvmpipe_screen *screen, memset(lpr->tex_data, 0, total_size); } } + if (lpr->base.flags & PIPE_RESOURCE_FLAG_SPARSE) { + uint64_t page_align; + os_get_page_size(&page_align); + lpr->size_required = align64(lpr->size_required, page_align); + } return true; @@ -285,6 +320,14 @@ llvmpipe_resource_create_all(struct pipe_screen *_screen, /* texture map */ if (!llvmpipe_texture_layout(screen, lpr, alloc_backing)) goto fail; + + if (templat->flags & PIPE_RESOURCE_FLAG_SPARSE) { +#if DETECT_OS_LINUX + lpr->tex_data = os_mmap(NULL, lpr->size_required, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_SHARED, + -1, 0); + madvise(lpr->tex_data, lpr->size_required, MADV_DONTNEED); +#endif + } } } else { /* other data (vertex buffer, const buffer, etc) */ @@ -309,9 +352,8 @@ llvmpipe_resource_create_all(struct pipe_screen *_screen, if (!(templat->flags & PIPE_RESOURCE_FLAG_DONT_OVER_ALLOCATE)) lpr->size_required += (LP_RASTER_BLOCK_SIZE - 1) * 4 * sizeof(float); + uint64_t alignment = sizeof(uint64_t) * 16; if (alloc_backing) { - uint64_t alignment = sizeof(uint64_t) * 16; - if (templat->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) os_get_page_size(&alignment); @@ -321,6 +363,16 @@ llvmpipe_resource_create_all(struct pipe_screen *_screen, goto fail; memset(lpr->data, 0, bytes); } + + if (templat->flags & PIPE_RESOURCE_FLAG_SPARSE) { + os_get_page_size(&alignment); + lpr->size_required = align64(lpr->size_required, alignment); +#if DETECT_OS_LINUX + lpr->data = os_mmap(NULL, lpr->size_required, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_SHARED, + -1, 0); + madvise(lpr->data, lpr->size_required, MADV_DONTNEED); +#endif + } } lpr->id = id_counter++; @@ -517,6 +569,15 @@ llvmpipe_resource_destroy(struct pipe_screen *pscreen, pscreen->free_memory_fd(pscreen, (struct pipe_memory_allocation*)lpr->dmabuf_alloc); #endif + if (lpr->base.flags & PIPE_RESOURCE_FLAG_SPARSE) { +#if DETECT_OS_LINUX + if (llvmpipe_resource_is_texture(pt)) + munmap(lpr->tex_data, lpr->size_required); + else + munmap(lpr->data, lpr->size_required); +#endif + } + #if MESA_DEBUG simple_mtx_lock(&resource_list_mutex); if (!list_is_empty(&lpr->list)) @@ -914,6 +975,48 @@ llvmpipe_transfer_map_ms(struct pipe_context *pipe, format = lpr->base.format; + if (llvmpipe_resource_is_texture(resource) && (resource->flags & PIPE_RESOURCE_FLAG_SPARSE)) { + map = llvmpipe_resource_map(resource, 0, 0, tex_usage); + + lpt->block_box = (struct pipe_box) { + .x = box->x / util_format_get_blockwidth(format), + .width = DIV_ROUND_UP(box->x + box->width, util_format_get_blockwidth(format)), + .y = box->y / util_format_get_blockheight(format), + .height = DIV_ROUND_UP(box->y + box->height, util_format_get_blockheight(format)), + .z = box->z / util_format_get_blockdepth(format), + .depth = DIV_ROUND_UP(box->z + box->depth, util_format_get_blockdepth(format)), + }; + + lpt->block_box.width -= lpt->block_box.x; + lpt->block_box.height -= lpt->block_box.y; + lpt->block_box.depth -= lpt->block_box.z; + + uint32_t block_stride = util_format_get_blocksize(format); + pt->stride = lpt->block_box.width * block_stride; + pt->layer_stride = pt->stride * lpt->block_box.height; + + uint8_t *staging_map = malloc(pt->layer_stride * lpt->block_box.depth); + lpt->map = staging_map; + + if (usage & PIPE_MAP_READ) { + for (uint32_t z = 0; z < lpt->block_box.depth; z++) { + for (uint32_t y = 0; y < lpt->block_box.height; y++) { + for (uint32_t x = 0; x < lpt->block_box.width; x++) { + memcpy(staging_map, + map + llvmpipe_get_texel_offset(resource, level, + lpt->block_box.x + x, + lpt->block_box.y + y, + lpt->block_box.z + z), + block_stride); + staging_map += block_stride; + } + } + } + } + + return lpt->map; + } + map = llvmpipe_resource_map(resource, level, box->z, tex_usage); @@ -934,6 +1037,60 @@ llvmpipe_transfer_map_ms(struct pipe_context *pipe, return map; } +uint32_t +llvmpipe_get_texel_offset(struct pipe_resource *resource, + uint32_t level, uint32_t x, + uint32_t y, uint32_t z) +{ + struct llvmpipe_resource *lpr = llvmpipe_resource(resource); + + uint32_t layer = 0; + if (resource->target != PIPE_TEXTURE_3D) { + layer = z; + z = 0; + } + + uint32_t dimensions = 1; + switch (resource->target) { + case PIPE_TEXTURE_2D: + case PIPE_TEXTURE_CUBE: + case PIPE_TEXTURE_RECT: + case PIPE_TEXTURE_2D_ARRAY: + dimensions = 2; + break; + case PIPE_TEXTURE_3D: + dimensions = 3; + break; + default: + break; + } + + uint32_t sparse_tile_size[3] = { + util_format_get_tilesize(resource->format, dimensions, resource->nr_samples, 0), + util_format_get_tilesize(resource->format, dimensions, resource->nr_samples, 1), + util_format_get_tilesize(resource->format, dimensions, resource->nr_samples, 2), + }; + + uint32_t num_tiles_x = DIV_ROUND_UP(u_minify(resource->width0, level), + sparse_tile_size[0] * util_format_get_blockwidth(resource->format)); + uint32_t num_tiles_y = DIV_ROUND_UP(u_minify(resource->height0, level), + sparse_tile_size[1] * util_format_get_blockheight(resource->format)); + + uint32_t offset = ( + x / sparse_tile_size[0] + + y / sparse_tile_size[1] * num_tiles_x + + z / sparse_tile_size[2] * num_tiles_x * num_tiles_y + ) * 64 * 1024; + + offset += ( + x % sparse_tile_size[0] + + (y % sparse_tile_size[1]) * sparse_tile_size[0] + + (z % sparse_tile_size[2]) * sparse_tile_size[0] * sparse_tile_size[1] + ) * util_format_get_blocksize(resource->format); + + return offset + lpr->mip_offsets[level] + lpr->img_stride[level] * layer; +} + static void * llvmpipe_transfer_map(struct pipe_context *pipe, @@ -952,18 +1109,39 @@ static void llvmpipe_transfer_unmap(struct pipe_context *pipe, struct pipe_transfer *transfer) { - assert(transfer->resource); + struct llvmpipe_transfer *lpt = (struct llvmpipe_transfer *)transfer; + struct pipe_resource *resource = transfer->resource; + struct llvmpipe_resource *lpr = llvmpipe_resource(resource); - llvmpipe_resource_unmap(transfer->resource, + assert(resource); + + if (llvmpipe_resource_is_texture(resource) && (resource->flags & PIPE_RESOURCE_FLAG_SPARSE) && + (transfer->usage & PIPE_MAP_WRITE)) { + uint32_t block_stride = util_format_get_blocksize(resource->format); + + const uint8_t *src = lpt->map; + uint8_t *dst = lpr->tex_data; + + for (uint32_t z = 0; z < lpt->block_box.depth; z++) { + for (uint32_t y = 0; y < lpt->block_box.height; y++) { + for (uint32_t x = 0; x < lpt->block_box.width; x++) { + memcpy(dst + llvmpipe_get_texel_offset(resource, transfer->level, + lpt->block_box.x + x, + lpt->block_box.y + y, + lpt->block_box.z + z), + src, block_stride); + src += block_stride; + } + } + } + } + + llvmpipe_resource_unmap(resource, transfer->level, transfer->box.z); - /* Effectively do the texture_update work here - if texture images - * needed post-processing to put them into hardware layout, this is - * where it would happen. For llvmpipe, nothing to do. - */ - assert (transfer->resource); - pipe_resource_reference(&transfer->resource, NULL); + pipe_resource_reference(&resource, NULL); + free(lpt->map); FREE(transfer); } @@ -1379,6 +1557,31 @@ llvmpipe_resource_bind_backing(struct pipe_screen *pscreen, if (!lpr->backable) return false; + if ((lpr->base.flags & PIPE_RESOURCE_FLAG_SPARSE) && offset < lpr->size_required) { +#if DETECT_OS_LINUX + struct llvmpipe_memory_allocation *mem = (struct llvmpipe_memory_allocation *)pmem; + if (mem) { + if (llvmpipe_resource_is_texture(&lpr->base)) { + mmap((char *)lpr->tex_data + offset, size, PROT_READ|PROT_WRITE, + MAP_SHARED|MAP_FIXED, mem->fd, mem->offset + fd_offset); + } else { + mmap((char *)lpr->data + offset, size, PROT_READ|PROT_WRITE, + MAP_SHARED|MAP_FIXED, mem->fd, mem->offset + fd_offset); + } + } else { + if (llvmpipe_resource_is_texture(&lpr->base)) { + mmap((char *)lpr->tex_data + offset, size, PROT_READ|PROT_WRITE, + MAP_SHARED|MAP_FIXED|MAP_ANONYMOUS, -1, 0); + } else { + mmap((char *)lpr->data + offset, size, PROT_READ|PROT_WRITE, + MAP_SHARED|MAP_FIXED|MAP_ANONYMOUS, -1, 0); + } + } +#endif + + return true; + } + addr = llvmpipe_map_memory(pscreen, pmem); if (llvmpipe_resource_is_texture(&lpr->base)) { diff --git a/src/gallium/drivers/llvmpipe/lp_texture.h b/src/gallium/drivers/llvmpipe/lp_texture.h index eda7bc81b7c..5f0cf7b1815 100644 --- a/src/gallium/drivers/llvmpipe/lp_texture.h +++ b/src/gallium/drivers/llvmpipe/lp_texture.h @@ -122,6 +122,8 @@ struct llvmpipe_resource struct llvmpipe_transfer { struct pipe_transfer base; + void *map; + struct pipe_box block_box; }; struct llvmpipe_memory_allocation @@ -300,4 +302,9 @@ llvmpipe_transfer_map_ms(struct pipe_context *pipe, const struct pipe_box *box, struct pipe_transfer **transfer); +uint32_t +llvmpipe_get_texel_offset(struct pipe_resource *resource, + uint32_t level, uint32_t x, + uint32_t y, uint32_t z); + #endif /* LP_TEXTURE_H */ diff --git a/src/gallium/drivers/llvmpipe/lp_texture_handle.c b/src/gallium/drivers/llvmpipe/lp_texture_handle.c index fac2a2be94e..6c4391a5f6f 100644 --- a/src/gallium/drivers/llvmpipe/lp_texture_handle.c +++ b/src/gallium/drivers/llvmpipe/lp_texture_handle.c @@ -122,7 +122,7 @@ llvmpipe_create_image_handle(struct pipe_context *pctx, const struct pipe_image_ if (view->u.tex.first_layer == view->u.tex.last_layer) { if (state.target == PIPE_TEXTURE_1D_ARRAY) state.target = PIPE_TEXTURE_1D; - else if (state.target == PIPE_TEXTURE_2D_ARRAY || state.target == PIPE_TEXTURE_3D) + else if (state.target == PIPE_TEXTURE_2D_ARRAY || (state.target == PIPE_TEXTURE_3D && !state.tiled)) state.target = PIPE_TEXTURE_2D; else if (state.target == PIPE_TEXTURE_CUBE_ARRAY) state.target = PIPE_TEXTURE_CUBE; diff --git a/src/gallium/drivers/zink/ci/zink-lvp-fails.txt b/src/gallium/drivers/zink/ci/zink-lvp-fails.txt index 8bfa354bde5..1fd8608acb2 100644 --- a/src/gallium/drivers/zink/ci/zink-lvp-fails.txt +++ b/src/gallium/drivers/zink/ci/zink-lvp-fails.txt @@ -140,3 +140,6 @@ spec@!opengl 1.0@gl-1.0-dlist-beginend,Crash spec@nv_texture_barrier@blending-in-shader,Crash spec@arb_viewport_array@display-list,Fail + +# passes locally +KHR-GL46.sparse_texture_tests.SparseTextureCommitment,Fail diff --git a/src/gallium/frontends/lavapipe/ci/lvp-asan-fails.txt b/src/gallium/frontends/lavapipe/ci/lvp-asan-fails.txt index 98772df1c69..0e5d833aaee 100644 --- a/src/gallium/frontends/lavapipe/ci/lvp-asan-fails.txt +++ b/src/gallium/frontends/lavapipe/ci/lvp-asan-fails.txt @@ -40,3 +40,6 @@ dEQP-VK.pipeline.shader_object_linked_binary.multisample.multisampled_render_to_ dEQP-VK.pipeline.shader_object_linked_spirv.multisample.multisampled_render_to_single_sampled.dynamic_rendering.multi_renderpass.r8g8b8a8_unorm_r16g16b16a16_sfloat_r16g16b16a16_sint_d32_sfloat_s8_uint.random_119,Fail dEQP-VK.pipeline.shader_object_linked_spirv.multisample.multisampled_render_to_single_sampled.multi_renderpass.r8g8b8a8_unorm_r16g16b16a16_sfloat_r16g16b16a16_sint_d32_sfloat_s8_uint.random_119,Fail dEQP-VK.pipeline.shader_object_linked_spirv.multisample.multisampled_render_to_single_sampled.multi_subpass.r8g8b8a8_unorm_r16g16b16a16_sfloat_r16g16b16a16_sint_d32_sfloat_s8_uint.random_119,Fail + +# The test has a loop that exceeds LP_MAX_TGSI_LOOP_ITERATIONS +dEQP-VK.sparse_resources.buffer.ssbo.sparse_residency.buffer_size_2_24 diff --git a/src/gallium/frontends/lavapipe/ci/lvp-asan-skips.txt b/src/gallium/frontends/lavapipe/ci/lvp-asan-skips.txt index 0cfdf78154e..cbf5dc725d3 100644 --- a/src/gallium/frontends/lavapipe/ci/lvp-asan-skips.txt +++ b/src/gallium/frontends/lavapipe/ci/lvp-asan-skips.txt @@ -9,3 +9,6 @@ dEQP-VK.subgroups.ballot_broadcast.compute.subgroupbroadcast.* dEQP-VK.ssbo.layout.3_level_unsized_array.scalar.* dEQP-VK.ssbo.layout.3_level_array.scalar.* dEQP-VK.ssbo.phys.layout.unsized_struct_array.per_block_buffer.*_instance_array_comp_access_store_cols + +# https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/5117 +dEQP-VK.image.host_image_copy.* diff --git a/src/gallium/frontends/lavapipe/ci/lvp-fails.txt b/src/gallium/frontends/lavapipe/ci/lvp-fails.txt index c38cdbe9f22..e37fdb5974f 100644 --- a/src/gallium/frontends/lavapipe/ci/lvp-fails.txt +++ b/src/gallium/frontends/lavapipe/ci/lvp-fails.txt @@ -5,6 +5,9 @@ dEQP-VK.rasterization.provoking_vertex.transform_feedback.per_pipeline.triangle_ dEQP-VK.api.version_check.unavailable_entry_points,Fail +# The test has a loop that exceeds LP_MAX_TGSI_LOOP_ITERATIONS +dEQP-VK.sparse_resources.buffer.ssbo.sparse_residency.buffer_size_2_24,Fail + # New failure since CTS 1.3.8.0 dEQP-VK.api.get_device_proc_addr.non_enabled,Fail diff --git a/src/gallium/frontends/lavapipe/ci/lvp-skips.txt b/src/gallium/frontends/lavapipe/ci/lvp-skips.txt index abf1734e13c..d2cfa2d4221 100644 --- a/src/gallium/frontends/lavapipe/ci/lvp-skips.txt +++ b/src/gallium/frontends/lavapipe/ci/lvp-skips.txt @@ -47,3 +47,6 @@ dEQP-VK.binding_model.descriptor_buffer.multiple.graphics_vert_buffers8_sets1 # New timeout since CTS 1.3.8.0 dEQP-VK.mesh_shader.ext.misc.many_task_work_groups_z + +# https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/5117 +dEQP-VK.image.host_image_copy.* diff --git a/src/gallium/frontends/lavapipe/lvp_device.c b/src/gallium/frontends/lavapipe/lvp_device.c index b17d655db4b..d627641bad8 100644 --- a/src/gallium/frontends/lavapipe/lvp_device.c +++ b/src/gallium/frontends/lavapipe/lvp_device.c @@ -47,6 +47,7 @@ #if DETECT_OS_LINUX #include +#include #endif #if defined(VK_USE_PLATFORM_WAYLAND_KHR) || \ @@ -343,6 +344,11 @@ lvp_get_features(const struct lvp_physical_device *pdevice, .shaderInt16 = (min_shader_param(pdevice->pscreen, PIPE_SHADER_CAP_INT16) == 1), .variableMultisampleRate = false, .inheritedQueries = false, + .sparseBinding = DETECT_OS_LINUX, + .sparseResidencyBuffer = DETECT_OS_LINUX, + .sparseResidencyImage2D = DETECT_OS_LINUX, + .sparseResidencyImage3D = DETECT_OS_LINUX, + .sparseResidencyAliased = DETECT_OS_LINUX, /* Vulkan 1.1 */ .storageBuffer16BitAccess = true, @@ -628,8 +634,8 @@ lvp_get_features(const struct lvp_physical_device *pdevice, .shaderSharedFloat64AtomicAdd = false, .shaderImageFloat32Atomics = true, .shaderImageFloat32AtomicAdd = true, - .sparseImageFloat32Atomics = false, - .sparseImageFloat32AtomicAdd = false, + .sparseImageFloat32Atomics = DETECT_OS_LINUX, + .sparseImageFloat32AtomicAdd = DETECT_OS_LINUX, /* VK_EXT_shader_atomic_float2 */ .shaderBufferFloat16Atomics = false, @@ -773,7 +779,7 @@ lvp_get_properties(const struct lvp_physical_device *device, struct vk_propertie .maxMemoryAllocationCount = UINT32_MAX, .maxSamplerAllocationCount = 32 * 1024, .bufferImageGranularity = 64, /* A cache line */ - .sparseAddressSpaceSize = 0, + .sparseAddressSpaceSize = 2UL*1024*1024*1024, .maxBoundDescriptorSets = MAX_SETS, .maxPerStageDescriptorSamplers = MAX_DESCRIPTORS, .maxPerStageDescriptorUniformBuffers = MAX_DESCRIPTORS, @@ -871,6 +877,9 @@ lvp_get_properties(const struct lvp_physical_device *device, struct vk_propertie .optimalBufferCopyOffsetAlignment = 128, .optimalBufferCopyRowPitchAlignment = 128, .nonCoherentAtomSize = 64, + .sparseResidencyStandard2DBlockShape = true, + .sparseResidencyStandard2DMultisampleBlockShape = true, + .sparseResidencyStandard3DBlockShape = true, /* Vulkan 1.1 */ /* The LUID is for Windows. */ @@ -1418,7 +1427,8 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetPhysicalDeviceQueueFamilyProperties2( p->queueFamilyProperties = (VkQueueFamilyProperties) { .queueFlags = VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT | - VK_QUEUE_TRANSFER_BIT, + VK_QUEUE_TRANSFER_BIT | + (DETECT_OS_LINUX ? VK_QUEUE_SPARSE_BINDING_BIT : 0), .queueCount = 1, .timestampValidBits = 64, .minImageTransferGranularity = (VkExtent3D) { 1, 1, 1 }, @@ -1535,6 +1545,24 @@ lvp_queue_submit(struct vk_queue *vk_queue, simple_mtx_lock(&queue->lock); + for (uint32_t i = 0; i < submit->buffer_bind_count; i++) { + VkSparseBufferMemoryBindInfo *bind = &submit->buffer_binds[i]; + + lvp_buffer_bind_sparse(queue->device, queue, bind); + } + + for (uint32_t i = 0; i < submit->image_opaque_bind_count; i++) { + VkSparseImageOpaqueMemoryBindInfo *bind = &submit->image_opaque_binds[i]; + + lvp_image_bind_opaque_sparse(queue->device, queue, bind); + } + + for (uint32_t i = 0; i < submit->image_bind_count; i++) { + VkSparseImageMemoryBindInfo *bind = &submit->image_binds[i]; + + lvp_image_bind_sparse(queue->device, queue, bind); + } + for (uint32_t i = 0; i < submit->command_buffer_count; i++) { struct lvp_cmd_buffer *cmd_buffer = container_of(submit->command_buffers[i], struct lvp_cmd_buffer, vk); @@ -1993,6 +2021,12 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetDeviceBufferMemoryRequirements( { pMemoryRequirements->memoryRequirements.memoryTypeBits = 1; pMemoryRequirements->memoryRequirements.alignment = 64; + + if (pInfo->pCreateInfo->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) { + uint64_t alignment; + os_get_page_size(&alignment); + pMemoryRequirements->memoryRequirements.alignment = alignment; + } pMemoryRequirements->memoryRequirements.size = 0; VkBuffer _buffer; @@ -2003,15 +2037,6 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetDeviceBufferMemoryRequirements( lvp_DestroyBuffer(_device, _buffer, NULL); } -VKAPI_ATTR void VKAPI_CALL lvp_GetDeviceImageSparseMemoryRequirements( - VkDevice device, - const VkDeviceImageMemoryRequirements* pInfo, - uint32_t* pSparseMemoryRequirementCount, - VkSparseImageMemoryRequirements2* pSparseMemoryRequirements) -{ - stub(); -} - VKAPI_ATTR void VKAPI_CALL lvp_GetDeviceImageMemoryRequirements( VkDevice _device, const VkDeviceImageMemoryRequirements* pInfo, @@ -2037,6 +2062,12 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetBufferMemoryRequirements( { LVP_FROM_HANDLE(lvp_buffer, buffer, _buffer); + pMemoryRequirements->alignment = 64; + if (buffer->vk.create_flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) { + uint64_t alignment; + os_get_page_size(&alignment); + pMemoryRequirements->alignment = alignment; + } /* The Vulkan spec (git aaed022) says: * * memoryTypeBits is a bitfield and contains one bit set for every @@ -2049,7 +2080,6 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetBufferMemoryRequirements( pMemoryRequirements->memoryTypeBits = 1; pMemoryRequirements->size = buffer->total_size; - pMemoryRequirements->alignment = 64; } VKAPI_ATTR void VKAPI_CALL lvp_GetBufferMemoryRequirements2( @@ -2109,24 +2139,6 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetImageMemoryRequirements2( } } -VKAPI_ATTR void VKAPI_CALL lvp_GetImageSparseMemoryRequirements( - VkDevice device, - VkImage image, - uint32_t* pSparseMemoryRequirementCount, - VkSparseImageMemoryRequirements* pSparseMemoryRequirements) -{ - stub(); -} - -VKAPI_ATTR void VKAPI_CALL lvp_GetImageSparseMemoryRequirements2( - VkDevice device, - const VkImageSparseMemoryRequirementsInfo2* pInfo, - uint32_t* pSparseMemoryRequirementCount, - VkSparseImageMemoryRequirements2* pSparseMemoryRequirements) -{ - stub(); -} - VKAPI_ATTR void VKAPI_CALL lvp_GetDeviceMemoryCommitment( VkDevice device, VkDeviceMemory memory, @@ -2291,15 +2303,6 @@ lvp_GetMemoryFdPropertiesKHR(VkDevice _device, #endif -VKAPI_ATTR VkResult VKAPI_CALL lvp_QueueBindSparse( - VkQueue queue, - uint32_t bindInfoCount, - const VkBindSparseInfo* pBindInfo, - VkFence fence) -{ - stub_return(VK_ERROR_INCOMPATIBLE_DRIVER); -} - VKAPI_ATTR VkResult VKAPI_CALL lvp_CreateEvent( VkDevice _device, const VkEventCreateInfo* pCreateInfo, diff --git a/src/gallium/frontends/lavapipe/lvp_formats.c b/src/gallium/frontends/lavapipe/lvp_formats.c index 86a6d74e555..d8f2111744a 100644 --- a/src/gallium/frontends/lavapipe/lvp_formats.c +++ b/src/gallium/frontends/lavapipe/lvp_formats.c @@ -508,18 +508,20 @@ VKAPI_ATTR VkResult VKAPI_CALL lvp_GetPhysicalDeviceImageFormatProperties2( return VK_SUCCESS; } -VKAPI_ATTR void VKAPI_CALL lvp_GetPhysicalDeviceSparseImageFormatProperties( - VkPhysicalDevice physicalDevice, - VkFormat format, - VkImageType type, - VkSampleCountFlagBits samples, - VkImageUsageFlags usage, - VkImageTiling tiling, - uint32_t* pNumProperties, - VkSparseImageFormatProperties* pProperties) +static void +fill_sparse_image_format_properties(struct lvp_physical_device *pdev, VkImageType type, + VkFormat format, VkSampleCountFlagBits samples, + VkSparseImageFormatProperties *prop) { - /* Sparse images are not yet supported. */ - *pNumProperties = 0; + enum pipe_format pformat = vk_format_to_pipe_format(format); + + prop->aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + prop->flags = 0; + prop->imageGranularity = (VkExtent3D){ + .width = util_format_get_tilesize(pformat, type + 1, samples, 0) * util_format_get_blockwidth(pformat), + .height = util_format_get_tilesize(pformat, type + 1, samples, 1) * util_format_get_blockheight(pformat), + .depth = util_format_get_tilesize(pformat, type + 1, samples, 2) * util_format_get_blockdepth(pformat), + }; } VKAPI_ATTR void VKAPI_CALL lvp_GetPhysicalDeviceSparseImageFormatProperties2( @@ -528,10 +530,98 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetPhysicalDeviceSparseImageFormatProperties2( uint32_t *pPropertyCount, VkSparseImageFormatProperties2 *pProperties) { - /* Sparse images are not yet supported. */ - *pPropertyCount = 0; + LVP_FROM_HANDLE(lvp_physical_device, physical_device, physicalDevice); + VkResult result; + + if (pFormatInfo->samples > VK_SAMPLE_COUNT_1_BIT) { + *pPropertyCount = 0; + return; + } + const VkPhysicalDeviceImageFormatInfo2 fmt_info = { + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2, + .format = pFormatInfo->format, + .type = pFormatInfo->type, + .tiling = pFormatInfo->tiling, + .usage = pFormatInfo->usage, + .flags = VK_IMAGE_CREATE_SPARSE_BINDING_BIT | VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT}; + + VkImageFormatProperties fmt_props; + result = lvp_get_image_format_properties(physical_device, &fmt_info, + &fmt_props); + if (result != VK_SUCCESS) { + *pPropertyCount = 0; + return; + } + + VK_OUTARRAY_MAKE_TYPED(VkSparseImageFormatProperties2, out, pProperties, pPropertyCount); + + vk_outarray_append_typed(VkSparseImageFormatProperties2, &out, prop) + { + fill_sparse_image_format_properties(physical_device, pFormatInfo->type, pFormatInfo->format, + pFormatInfo->samples, &prop->properties); + }; } +VKAPI_ATTR void VKAPI_CALL lvp_GetDeviceImageSparseMemoryRequirements( + VkDevice _device, + const VkDeviceImageMemoryRequirements* pInfo, + uint32_t* pSparseMemoryRequirementCount, + VkSparseImageMemoryRequirements2* pSparseMemoryRequirements) +{ + LVP_FROM_HANDLE(lvp_device, device, _device); + + if (!(pInfo->pCreateInfo->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT)) { + *pSparseMemoryRequirementCount = 0; + return; + } + + VK_OUTARRAY_MAKE_TYPED(VkSparseImageMemoryRequirements2, out, pSparseMemoryRequirements, + pSparseMemoryRequirementCount); + + vk_outarray_append_typed(VkSparseImageMemoryRequirements2, &out, req) + { + fill_sparse_image_format_properties(device->physical_device, pInfo->pCreateInfo->imageType, + pInfo->pCreateInfo->format, pInfo->pCreateInfo->samples, + &req->memoryRequirements.formatProperties); + + req->memoryRequirements.imageMipTailFirstLod = pInfo->pCreateInfo->mipLevels; + req->memoryRequirements.imageMipTailSize = 0; + req->memoryRequirements.imageMipTailOffset = 0; + req->memoryRequirements.imageMipTailStride = 0; + }; +} + +VKAPI_ATTR void VKAPI_CALL lvp_GetImageSparseMemoryRequirements2( + VkDevice _device, + const VkImageSparseMemoryRequirementsInfo2* pInfo, + uint32_t* pSparseMemoryRequirementCount, + VkSparseImageMemoryRequirements2* pSparseMemoryRequirements) +{ + LVP_FROM_HANDLE(lvp_device, device, _device); + LVP_FROM_HANDLE(lvp_image, image, pInfo->image); + + if (!(image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT)) { + *pSparseMemoryRequirementCount = 0; + return; + } + + VK_OUTARRAY_MAKE_TYPED(VkSparseImageMemoryRequirements2, out, pSparseMemoryRequirements, + pSparseMemoryRequirementCount); + + vk_outarray_append_typed(VkSparseImageMemoryRequirements2, &out, req) + { + fill_sparse_image_format_properties(device->physical_device, image->vk.image_type, + image->vk.format, image->vk.samples, + &req->memoryRequirements.formatProperties); + + req->memoryRequirements.imageMipTailFirstLod = image->vk.mip_levels; + req->memoryRequirements.imageMipTailSize = 0; + req->memoryRequirements.imageMipTailOffset = 0; + req->memoryRequirements.imageMipTailStride = 0; + }; +} + + VKAPI_ATTR void VKAPI_CALL lvp_GetPhysicalDeviceExternalBufferProperties( VkPhysicalDevice physicalDevice, const VkPhysicalDeviceExternalBufferInfo *pExternalBufferInfo, diff --git a/src/gallium/frontends/lavapipe/lvp_image.c b/src/gallium/frontends/lavapipe/lvp_image.c index c023b9efa1f..bd994f7cb2b 100644 --- a/src/gallium/frontends/lavapipe/lvp_image.c +++ b/src/gallium/frontends/lavapipe/lvp_image.c @@ -66,6 +66,9 @@ lvp_image_create(VkDevice _device, return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); image->alignment = 64; + if (image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) + image->alignment = 64 * 1024; + image->plane_count = vk_format_get_plane_count(pCreateInfo->format); image->disjoint = image->plane_count > 1 && (pCreateInfo->flags & VK_IMAGE_CREATE_DISJOINT_BIT); @@ -126,6 +129,9 @@ lvp_image_create(VkDevice _device, VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT)) template.bind |= PIPE_BIND_SHADER_IMAGE; + if (pCreateInfo->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) + template.flags |= PIPE_RESOURCE_FLAG_SPARSE; + template.width0 = pCreateInfo->extent.width / width_scale; template.height0 = pCreateInfo->extent.height / height_scale; template.depth0 = pCreateInfo->extent.depth; @@ -162,6 +168,8 @@ lvp_image_create(VkDevice _device, if (!image->planes[p].bo) return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + image->planes[p].size = align64(image->planes[p].size, image->alignment); + image->size += image->planes[p].size; } *pImage = lvp_image_to_handle(image); @@ -536,6 +544,8 @@ VKAPI_ATTR VkResult VKAPI_CALL lvp_CreateBuffer( if (buffer->vk.usage & VK_BUFFER_USAGE_2_STORAGE_TEXEL_BUFFER_BIT_KHR) template.bind |= PIPE_BIND_SHADER_IMAGE; template.flags = PIPE_RESOURCE_FLAG_DONT_OVER_ALLOCATE; + if (pCreateInfo->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) + template.flags |= PIPE_RESOURCE_FLAG_SPARSE; buffer->bo = device->pscreen->resource_create_unbacked(device->pscreen, &template, &buffer->total_size); @@ -543,6 +553,12 @@ VKAPI_ATTR VkResult VKAPI_CALL lvp_CreateBuffer( vk_free2(&device->vk.alloc, pAllocator, buffer); return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY); } + + if (pCreateInfo->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) { + buffer->map = device->queue.ctx->buffer_map(device->queue.ctx, buffer->bo, 0, + PIPE_MAP_READ | PIPE_MAP_WRITE | PIPE_MAP_PERSISTENT, + &(struct pipe_box){ 0 }, &buffer->transfer); + } } *pBuffer = lvp_buffer_to_handle(buffer); @@ -566,6 +582,9 @@ VKAPI_ATTR void VKAPI_CALL lvp_DestroyBuffer( if (he) _mesa_hash_table_remove(&device->bda, he); simple_mtx_unlock(&device->bda_lock); + + if (buffer->bo->flags & PIPE_RESOURCE_FLAG_SPARSE) + device->queue.ctx->buffer_unmap(device->queue.ctx, buffer->transfer); } pipe_resource_reference(&buffer->bo, NULL); vk_buffer_destroy(&device->vk, pAllocator, &buffer->vk); @@ -836,3 +855,134 @@ lvp_TransitionImageLayoutEXT(VkDevice device, uint32_t transitionCount, const Vk /* no-op */ return VK_SUCCESS; } + +VkResult +lvp_buffer_bind_sparse(struct lvp_device *device, + struct lvp_queue *queue, + VkSparseBufferMemoryBindInfo *bind) +{ + LVP_FROM_HANDLE(lvp_buffer, buffer, bind->buffer); + + for (uint32_t i = 0; i < bind->bindCount; i++) { + LVP_FROM_HANDLE(lvp_device_memory, mem, bind->pBinds[i].memory); + device->pscreen->resource_bind_backing(device->pscreen, + buffer->bo, + mem ? mem->pmem : NULL, + bind->pBinds[i].memoryOffset, + bind->pBinds[i].size, + bind->pBinds[i].resourceOffset); + } + + return VK_SUCCESS; +} + +VkResult +lvp_image_bind_opaque_sparse(struct lvp_device *device, + struct lvp_queue *queue, + VkSparseImageOpaqueMemoryBindInfo *bind_info) +{ + LVP_FROM_HANDLE(lvp_image, image, bind_info->image); + + for (uint32_t i = 0; i < bind_info->bindCount; i++) { + const VkSparseMemoryBind *bind = &bind_info->pBinds[i]; + LVP_FROM_HANDLE(lvp_device_memory, mem, bind->memory); + + uint32_t plane_index; + uint32_t offset; + if (bind->resourceOffset < image->planes[0].size) { + plane_index = 0; + offset = bind->resourceOffset; + } else if (bind->resourceOffset < image->planes[0].size + image->planes[1].size) { + plane_index = 1; + offset = bind->resourceOffset - image->planes[0].size; + } else { + plane_index = 2; + offset = bind->resourceOffset - image->planes[0].size - image->planes[1].size; + } + + device->pscreen->resource_bind_backing(device->pscreen, + image->planes[plane_index].bo, + mem ? mem->pmem : NULL, + bind->memoryOffset, + bind->size, + offset); + } + + return VK_SUCCESS; +} + +VkResult +lvp_image_bind_sparse(struct lvp_device *device, + struct lvp_queue *queue, + VkSparseImageMemoryBindInfo *bind_info) +{ + LVP_FROM_HANDLE(lvp_image, image, bind_info->image); + + enum pipe_format format = vk_format_to_pipe_format(image->vk.format); + + for (uint32_t i = 0; i < bind_info->bindCount; i++) { + const VkSparseImageMemoryBind *bind = &bind_info->pBinds[i]; + LVP_FROM_HANDLE(lvp_device_memory, mem, bind->memory); + + uint8_t plane = lvp_image_aspects_to_plane(image, bind->subresource.aspectMask); + + uint32_t depth = 1; + uint32_t z = 0; + uint32_t dimensions = 2; + switch (image->planes[plane].bo->target) { + case PIPE_TEXTURE_CUBE: + case PIPE_TEXTURE_CUBE_ARRAY: + case PIPE_TEXTURE_2D_ARRAY: + case PIPE_TEXTURE_1D_ARRAY: + /* these use layer */ + z = bind->subresource.arrayLayer; + break; + case PIPE_TEXTURE_3D: + /* this uses depth */ + z = bind->offset.z; + depth = bind->extent.depth; + dimensions = 3; + break; + default: + break; + } + + uint32_t sparse_tile_size[3] = { + util_format_get_tilesize(format, dimensions, image->vk.samples, 0), + util_format_get_tilesize(format, dimensions, image->vk.samples, 1), + util_format_get_tilesize(format, dimensions, image->vk.samples, 2), + }; + + uint32_t sparse_block_base[3] = { + bind->offset.x / (sparse_tile_size[0] * util_format_get_blockwidth(format)), + bind->offset.y / (sparse_tile_size[1] * util_format_get_blockheight(format)), + z / (sparse_tile_size[2] * util_format_get_blockdepth(format)), + }; + + uint32_t sparse_block_counts[3] = { + DIV_ROUND_UP(bind->extent.width, sparse_tile_size[0] * util_format_get_blockwidth(format)), + DIV_ROUND_UP(bind->extent.height, sparse_tile_size[1] * util_format_get_blockheight(format)), + DIV_ROUND_UP(depth, sparse_tile_size[2] * util_format_get_blockdepth(format)), + }; + + uint32_t sparse_block_count = sparse_block_counts[0] * sparse_block_counts[1] * sparse_block_counts[2]; + + for (uint32_t block = 0; block < sparse_block_count; block++) { + uint32_t start_x = (sparse_block_base[0] + block % sparse_block_counts[0]) * sparse_tile_size[0]; + uint32_t start_y = (sparse_block_base[1] + (block / sparse_block_counts[0]) % sparse_block_counts[1]) * + sparse_tile_size[1]; + uint32_t start_z = (sparse_block_base[2] + (block / sparse_block_counts[0] / sparse_block_counts[1]) % sparse_block_counts[2]) * + sparse_tile_size[2]; + + uint64_t offset = llvmpipe_get_texel_offset(image->planes[plane].bo, bind->subresource.mipLevel, start_x, start_y, start_z); + device->pscreen->resource_bind_backing(device->pscreen, + image->planes[plane].bo, + mem ? mem->pmem : NULL, + bind->memoryOffset + block * 64 * 1024, + 64 * 1024, + offset); + } + } + + return VK_SUCCESS; +} diff --git a/src/gallium/frontends/lavapipe/lvp_private.h b/src/gallium/frontends/lavapipe/lvp_private.h index 4dadd308c9a..22786b3bb54 100644 --- a/src/gallium/frontends/lavapipe/lvp_private.h +++ b/src/gallium/frontends/lavapipe/lvp_private.h @@ -582,6 +582,7 @@ struct lvp_buffer { uint64_t total_size; uint64_t offset; void *map; + struct pipe_transfer *transfer; }; struct lvp_buffer_view { @@ -687,6 +688,16 @@ VK_DEFINE_NONDISP_HANDLE_CASTS(lvp_indirect_command_layout_nv, base, VkIndirectC void lvp_add_enqueue_cmd_entrypoints(struct vk_device_dispatch_table *disp); +VkResult lvp_buffer_bind_sparse(struct lvp_device *device, + struct lvp_queue *queue, + VkSparseBufferMemoryBindInfo *bind); +VkResult lvp_image_bind_opaque_sparse(struct lvp_device *device, + struct lvp_queue *queue, + VkSparseImageOpaqueMemoryBindInfo *bind); +VkResult lvp_image_bind_sparse(struct lvp_device *device, + struct lvp_queue *queue, + VkSparseImageMemoryBindInfo *bind); + VkResult lvp_execute_cmds(struct lvp_device *device, struct lvp_queue *queue, struct lvp_cmd_buffer *cmd_buffer);