diff --git a/src/nouveau/vulkan/nvk_buffer.c b/src/nouveau/vulkan/nvk_buffer.c index 8e62950d108..1a53d81eec8 100644 --- a/src/nouveau/vulkan/nvk_buffer.c +++ b/src/nouveau/vulkan/nvk_buffer.c @@ -8,6 +8,7 @@ #include "nvk_device.h" #include "nvk_device_memory.h" #include "nvk_physical_device.h" +#include "nvkmd/nvkmd.h" static uint32_t nvk_get_buffer_alignment(const struct nvk_physical_device *pdev, @@ -81,6 +82,7 @@ nvk_CreateBuffer(VkDevice device, { VK_FROM_HANDLE(nvk_device, dev, device); struct nvk_buffer *buffer; + VkResult result; if (pCreateInfo->size > NVK_MAX_BUFFER_SIZE) return vk_error(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY); @@ -98,26 +100,31 @@ nvk_CreateBuffer(VkDevice device, buffer->vk.usage, buffer->vk.create_flags); assert(alignment >= 4096); - buffer->vma_size_B = align64(buffer->vk.size, alignment); + const uint64_t va_size_B = align64(buffer->vk.size, alignment); - const bool sparse_residency = - buffer->vk.create_flags & VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT; - const bool bda_capture_replay = - buffer->vk.create_flags & VK_BUFFER_CREATE_DEVICE_ADDRESS_CAPTURE_REPLAY_BIT; + enum nvkmd_va_flags va_flags = 0; + if (buffer->vk.create_flags & VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT) + va_flags |= NVKMD_VA_SPARSE; - uint64_t bda_replay_addr = 0; - if (bda_capture_replay) - bda_replay_addr = nvk_get_bda_replay_addr(pCreateInfo); + uint64_t fixed_addr = 0; + if (buffer->vk.create_flags & VK_BUFFER_CREATE_DEVICE_ADDRESS_CAPTURE_REPLAY_BIT) { + va_flags |= NVKMD_VA_REPLAY; - buffer->addr = nouveau_ws_alloc_vma(dev->ws_dev, bda_replay_addr, - buffer->vma_size_B, - alignment, bda_capture_replay, - sparse_residency); - if (buffer->addr == 0) { - vk_buffer_destroy(&dev->vk, pAllocator, &buffer->vk); - return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, - "Sparse VMA allocation failed"); + fixed_addr = nvk_get_bda_replay_addr(pCreateInfo); + if (fixed_addr != 0) + va_flags |= NVKMD_VA_ALLOC_FIXED; } + + result = nvkmd_dev_alloc_va(dev->nvkmd, &dev->vk.base, + va_flags, 0 /* pte_kind */, + va_size_B, alignment, fixed_addr, + &buffer->va); + if (result != VK_SUCCESS) { + vk_buffer_destroy(&dev->vk, pAllocator, &buffer->vk); + return result; + } + + buffer->addr = buffer->va->addr; } *pBuffer = nvk_buffer_to_handle(buffer); @@ -136,16 +143,8 @@ nvk_DestroyBuffer(VkDevice device, if (!buffer) return; - if (buffer->vma_size_B > 0) { - const bool sparse_residency = - buffer->vk.create_flags & VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT; - const bool bda_capture_replay = - buffer->vk.create_flags & VK_BUFFER_CREATE_DEVICE_ADDRESS_CAPTURE_REPLAY_BIT; - - nouveau_ws_bo_unbind_vma(dev->ws_dev, buffer->addr, buffer->vma_size_B); - nouveau_ws_free_vma(dev->ws_dev, buffer->addr, buffer->vma_size_B, - bda_capture_replay, sparse_residency); - } + if (buffer->va != NULL) + nvkmd_va_free(buffer->va); vk_buffer_destroy(&dev->vk, pAllocator, &buffer->vk); } @@ -236,20 +235,19 @@ nvk_bind_buffer_memory(struct nvk_device *dev, { VK_FROM_HANDLE(nvk_device_memory, mem, info->memory); VK_FROM_HANDLE(nvk_buffer, buffer, info->buffer); + VkResult result = VK_SUCCESS; - buffer->is_local = !(mem->bo->flags & NOUVEAU_WS_BO_GART); - if (buffer->vma_size_B) { - nouveau_ws_bo_bind_vma(dev->ws_dev, - mem->bo, - buffer->addr, - buffer->vma_size_B, - info->memoryOffset, - 0 /* pte_kind */); + buffer->is_local = !(mem->mem->flags & NVKMD_MEM_GART); + if (buffer->va != NULL) { + result = nvkmd_va_bind_mem(buffer->va, &buffer->vk.base, + 0 /* va_offset */, + mem->mem, info->memoryOffset, + buffer->va->size_B); } else { - buffer->addr = mem->bo->offset + info->memoryOffset; + buffer->addr = mem->mem->va->addr + info->memoryOffset; } - return VK_SUCCESS; + return result; } VKAPI_ATTR VkResult VKAPI_CALL diff --git a/src/nouveau/vulkan/nvk_buffer.h b/src/nouveau/vulkan/nvk_buffer.h index d13cbc047e2..64a428944d3 100644 --- a/src/nouveau/vulkan/nvk_buffer.h +++ b/src/nouveau/vulkan/nvk_buffer.h @@ -8,22 +8,23 @@ #include "nvk_private.h" #include "nvk_device_memory.h" -#include "nouveau_bo.h" #include "vk_buffer.h" struct nvk_device_memory; struct nvk_physical_device; +struct nvkmd_va; struct nvk_buffer { struct vk_buffer vk; uint64_t addr; - /** Size of the reserved VMA range for sparse buffers, zero otherwise. */ - uint64_t vma_size_B; + /** Reserved VA for sparse buffers, NULL otherwise. */ + struct nvkmd_va *va; bool is_local; }; -VK_DEFINE_NONDISP_HANDLE_CASTS(nvk_buffer, vk.base, VkBuffer, VK_OBJECT_TYPE_BUFFER) +VK_DEFINE_NONDISP_HANDLE_CASTS(nvk_buffer, vk.base, VkBuffer, + VK_OBJECT_TYPE_BUFFER) static inline uint64_t nvk_buffer_address(const struct nvk_buffer *buffer, uint64_t offset) diff --git a/src/nouveau/vulkan/nvk_device_memory.c b/src/nouveau/vulkan/nvk_device_memory.c index 6509a9b60e7..fe989e8f802 100644 --- a/src/nouveau/vulkan/nvk_device_memory.c +++ b/src/nouveau/vulkan/nvk_device_memory.c @@ -4,22 +4,17 @@ */ #include "nvk_device_memory.h" -#include "nouveau_bo.h" - #include "nvk_device.h" #include "nvk_entrypoints.h" #include "nvk_image.h" #include "nvk_physical_device.h" +#include "nvkmd/nvkmd.h" -#include "nv_push.h" #include "util/u_atomic.h" #include #include -#include "nvtypes.h" -#include "nv_push_cl90b5.h" - /* Supports opaque fd only */ const VkExternalMemoryProperties nvk_opaque_fd_mem_props = { .externalMemoryFeatures = @@ -44,27 +39,27 @@ const VkExternalMemoryProperties nvk_dma_buf_mem_props = { VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, }; -static enum nouveau_ws_bo_flags +static enum nvkmd_mem_flags nvk_memory_type_flags(const VkMemoryType *type, VkExternalMemoryHandleTypeFlagBits handle_types) { - enum nouveau_ws_bo_flags flags = 0; + enum nvkmd_mem_flags flags = 0; if (type->propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) - flags = NOUVEAU_WS_BO_LOCAL; + flags = NVKMD_MEM_LOCAL; else - flags = NOUVEAU_WS_BO_GART; + flags = NVKMD_MEM_GART; if (type->propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) - flags |= NOUVEAU_WS_BO_MAP; + flags |= NVKMD_MEM_CAN_MAP; /* For dma-bufs, we have to allow them to live in GART because they might * get forced there by the kernel if they're shared with another GPU. */ if (handle_types & VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT) - flags |= NOUVEAU_WS_BO_GART; + flags |= NVKMD_MEM_GART; if (handle_types == 0) - flags |= NOUVEAU_WS_BO_NO_SHARE; + flags |= NVKMD_MEM_NO_SHARE; return flags; } @@ -77,14 +72,15 @@ nvk_GetMemoryFdPropertiesKHR(VkDevice device, { VK_FROM_HANDLE(nvk_device, dev, device); struct nvk_physical_device *pdev = nvk_device_physical(dev); - struct nouveau_ws_bo *bo; + struct nvkmd_mem *mem; + VkResult result; switch (handleType) { case VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT: case VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT: - bo = nouveau_ws_bo_from_dma_buf(dev->ws_dev, fd); - if (bo == NULL) - return vk_error(dev, VK_ERROR_INVALID_EXTERNAL_HANDLE); + result = nvkmd_dev_import_dma_buf(dev->nvkmd, &dev->vk.base, fd, &mem); + if (result != VK_SUCCESS) + return result; break; default: return vk_error(dev, VK_ERROR_INVALID_EXTERNAL_HANDLE); @@ -98,16 +94,16 @@ nvk_GetMemoryFdPropertiesKHR(VkDevice device, type_bits = BITFIELD_MASK(pdev->mem_type_count); } else { for (unsigned t = 0; t < ARRAY_SIZE(pdev->mem_types); t++) { - const enum nouveau_ws_bo_flags flags = + const enum nvkmd_mem_flags flags = nvk_memory_type_flags(&pdev->mem_types[t], handleType); - if (!(flags & ~bo->flags)) + if (!(flags & ~mem->flags)) type_bits |= (1 << t); } } pMemoryFdProperties->memoryTypeBits = type_bits; - nouveau_ws_bo_destroy(bo); + nvkmd_mem_unref(mem); return VK_SUCCESS; } @@ -138,11 +134,10 @@ nvk_AllocateMemory(VkDevice device, if (fd_info != NULL) handle_types |= fd_info->handleType; - const enum nouveau_ws_bo_flags flags = - nvk_memory_type_flags(type, handle_types); + const enum nvkmd_mem_flags flags = nvk_memory_type_flags(type, handle_types); uint32_t alignment = (1ULL << 12); - if (flags & NOUVEAU_WS_BO_LOCAL) + if (flags & NVKMD_MEM_LOCAL) alignment = (1ULL << 16); uint8_t pte_kind = 0, tile_mode = 0; @@ -168,59 +163,58 @@ nvk_AllocateMemory(VkDevice device, if (!mem) return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY); - mem->map = NULL; if (fd_info && fd_info->handleType) { assert(fd_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT || fd_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT); - mem->bo = nouveau_ws_bo_from_dma_buf(dev->ws_dev, fd_info->fd); - if (mem->bo == NULL) { - result = vk_error(dev, VK_ERROR_INVALID_EXTERNAL_HANDLE); + result = nvkmd_dev_import_dma_buf(dev->nvkmd, &dev->vk.base, + fd_info->fd, &mem->mem); + if (result != VK_SUCCESS) goto fail_alloc; - } /* We can't really assert anything for dma-bufs because they could come * in from some other device. */ if (fd_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT) - assert(!(flags & ~mem->bo->flags)); + assert(!(flags & ~mem->mem->flags)); } else if (pte_kind != 0 || tile_mode != 0) { - mem->bo = nouveau_ws_bo_new_tiled(dev->ws_dev, aligned_size, alignment, - pte_kind, tile_mode, flags); - if (!mem->bo) { - result = vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, "%m"); + result = nvkmd_dev_alloc_tiled_mem(dev->nvkmd, &dev->vk.base, + aligned_size, alignment, + pte_kind, tile_mode, flags, + &mem->mem); + if (result != VK_SUCCESS) goto fail_alloc; - } } else { - mem->bo = nouveau_ws_bo_new(dev->ws_dev, aligned_size, alignment, flags); - if (!mem->bo) { - result = vk_error(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY); + result = nvkmd_dev_alloc_mem(dev->nvkmd, &dev->vk.base, + aligned_size, alignment, flags, + &mem->mem); + if (result != VK_SUCCESS) goto fail_alloc; - } } if (pdev->debug_flags & NVK_DEBUG_ZERO_MEMORY) { if (type->propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) { - void *map = nouveau_ws_bo_map(mem->bo, NOUVEAU_WS_BO_RDWR, NULL); - if (map == NULL) { - result = vk_errorf(dev, VK_ERROR_OUT_OF_HOST_MEMORY, - "Memory map failed"); - goto fail_bo; - } - memset(map, 0, mem->bo->size); - nouveau_ws_bo_unmap(mem->bo, map); + void *map; + result = nvkmd_mem_map(mem->mem, &dev->vk.base, + NVKMD_MEM_MAP_RDWR, NULL, &map); + if (result != VK_SUCCESS) + goto fail_mem; + + memset(map, 0, mem->mem->size_B); + nvkmd_mem_unmap(mem->mem); } else { result = nvk_upload_queue_fill(dev, &dev->upload, - mem->bo->offset, 0, mem->bo->size); + mem->mem->va->addr, + 0, mem->mem->size_B); if (result != VK_SUCCESS) - goto fail_bo; + goto fail_mem; /* Since we don't know when the memory will be freed, sync now */ result = nvk_upload_queue_sync(dev, &dev->upload); if (result != VK_SUCCESS) - goto fail_bo; + goto fail_mem; } } @@ -238,14 +232,14 @@ nvk_AllocateMemory(VkDevice device, } struct nvk_memory_heap *heap = &pdev->mem_heaps[type->heapIndex]; - p_atomic_add(&heap->used, mem->bo->size); + p_atomic_add(&heap->used, mem->mem->size_B); *pMem = nvk_device_memory_to_handle(mem); return VK_SUCCESS; -fail_bo: - nouveau_ws_bo_destroy(mem->bo); +fail_mem: + nvkmd_mem_unref(mem->mem); fail_alloc: vk_device_memory_destroy(&dev->vk, pAllocator, &mem->vk); return result; @@ -263,14 +257,11 @@ nvk_FreeMemory(VkDevice device, if (!mem) return; - if (mem->map) - nouveau_ws_bo_unmap(mem->bo, mem->map); - const VkMemoryType *type = &pdev->mem_types[mem->vk.memory_type_index]; struct nvk_memory_heap *heap = &pdev->mem_heaps[type->heapIndex]; - p_atomic_add(&heap->used, -((int64_t)mem->bo->size)); + p_atomic_add(&heap->used, -((int64_t)mem->mem->size_B)); - nouveau_ws_bo_destroy(mem->bo); + nvkmd_mem_unref(mem->mem); vk_device_memory_destroy(&dev->vk, pAllocator, &mem->vk); } @@ -282,6 +273,7 @@ nvk_MapMemory2KHR(VkDevice device, { VK_FROM_HANDLE(nvk_device, dev, device); VK_FROM_HANDLE(nvk_device_memory, mem, pMemoryMapInfo->memory); + VkResult result; if (mem == NULL) { *ppData = NULL; @@ -293,10 +285,13 @@ nvk_MapMemory2KHR(VkDevice device, vk_device_memory_range(&mem->vk, pMemoryMapInfo->offset, pMemoryMapInfo->size); + enum nvkmd_mem_map_flags map_flags = NVKMD_MEM_MAP_RDWR; + void *fixed_addr = NULL; if (pMemoryMapInfo->flags & VK_MEMORY_MAP_PLACED_BIT_EXT) { const VkMemoryMapPlacedInfoEXT *placed_info = vk_find_struct_const(pMemoryMapInfo->pNext, MEMORY_MAP_PLACED_INFO_EXT); + map_flags |= NVKMD_MEM_MAP_FIXED; fixed_addr = placed_info->pPlacedAddress; } @@ -308,7 +303,7 @@ nvk_MapMemory2KHR(VkDevice device, * equal to the size of the memory minus offset */ assert(size > 0); - assert(offset + size <= mem->bo->size); + assert(offset + size <= mem->mem->size_B); if (size != (size_t)size) { return vk_errorf(dev, VK_ERROR_MEMORY_MAP_FAILED, @@ -320,18 +315,17 @@ nvk_MapMemory2KHR(VkDevice device, * * "memory must not be currently host mapped" */ - if (mem->map != NULL) { + if (mem->mem->map != NULL) { return vk_errorf(dev, VK_ERROR_MEMORY_MAP_FAILED, "Memory object already mapped."); } - mem->map = nouveau_ws_bo_map(mem->bo, NOUVEAU_WS_BO_RDWR, fixed_addr); - if (mem->map == NULL) { - return vk_errorf(dev, VK_ERROR_MEMORY_MAP_FAILED, - "Memory object couldn't be mapped."); - } + void *mem_map; + result = nvkmd_mem_map(mem->mem, &mem->vk.base, map_flags, fixed_addr, &mem_map); + if (result != VK_SUCCESS) + return result; - *ppData = mem->map + offset; + *ppData = mem_map + offset; return VK_SUCCESS; } @@ -340,25 +334,17 @@ VKAPI_ATTR VkResult VKAPI_CALL nvk_UnmapMemory2KHR(VkDevice device, const VkMemoryUnmapInfoKHR *pMemoryUnmapInfo) { - VK_FROM_HANDLE(nvk_device, dev, device); VK_FROM_HANDLE(nvk_device_memory, mem, pMemoryUnmapInfo->memory); if (mem == NULL) return VK_SUCCESS; if (pMemoryUnmapInfo->flags & VK_MEMORY_UNMAP_RESERVE_BIT_EXT) { - int err = nouveau_ws_bo_overmap(mem->bo, mem->map); - if (err) { - return vk_errorf(dev, VK_ERROR_MEMORY_MAP_FAILED, - "Failed to map over original mapping"); - } + return nvkmd_mem_overmap(mem->mem, &mem->vk.base); } else { - nouveau_ws_bo_unmap(mem->bo, mem->map); + nvkmd_mem_unmap(mem->mem); + return VK_SUCCESS; } - - mem->map = NULL; - - return VK_SUCCESS; } VKAPI_ATTR VkResult VKAPI_CALL @@ -384,7 +370,7 @@ nvk_GetDeviceMemoryCommitment(VkDevice device, { VK_FROM_HANDLE(nvk_device_memory, mem, _mem); - *pCommittedMemoryInBytes = mem->bo->size; + *pCommittedMemoryInBytes = mem->mem->size_B; } VKAPI_ATTR VkResult VKAPI_CALL @@ -393,15 +379,12 @@ nvk_GetMemoryFdKHR(VkDevice device, int *pFD) { VK_FROM_HANDLE(nvk_device, dev, device); - VK_FROM_HANDLE(nvk_device_memory, memory, pGetFdInfo->memory); + VK_FROM_HANDLE(nvk_device_memory, mem, pGetFdInfo->memory); switch (pGetFdInfo->handleType) { case VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT: case VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT: - if (nouveau_ws_bo_dma_buf(memory->bo, pFD)) - return vk_errorf(dev, VK_ERROR_TOO_MANY_OBJECTS, - "Failed to export dma-buf: %m"); - return VK_SUCCESS; + return nvkmd_mem_export_dma_buf(mem->mem, &mem->vk.base, pFD); default: assert(!"unsupported handle type"); return vk_error(dev, VK_ERROR_FEATURE_NOT_PRESENT); @@ -415,5 +398,5 @@ nvk_GetDeviceMemoryOpaqueCaptureAddress( { VK_FROM_HANDLE(nvk_device_memory, mem, pInfo->memory); - return mem->bo->offset; + return mem->mem->va->addr; } diff --git a/src/nouveau/vulkan/nvk_device_memory.h b/src/nouveau/vulkan/nvk_device_memory.h index a2c556c747c..4d2f751f111 100644 --- a/src/nouveau/vulkan/nvk_device_memory.h +++ b/src/nouveau/vulkan/nvk_device_memory.h @@ -11,18 +11,16 @@ #include "util/list.h" -struct nvk_device; -struct nvk_image_plane; +struct nvkmd_mem; struct nvk_device_memory { struct vk_device_memory vk; - struct nouveau_ws_bo *bo; - - void *map; + struct nvkmd_mem *mem; }; -VK_DEFINE_NONDISP_HANDLE_CASTS(nvk_device_memory, vk.base, VkDeviceMemory, VK_OBJECT_TYPE_DEVICE_MEMORY) +VK_DEFINE_NONDISP_HANDLE_CASTS(nvk_device_memory, vk.base, VkDeviceMemory, + VK_OBJECT_TYPE_DEVICE_MEMORY) extern const VkExternalMemoryProperties nvk_opaque_fd_mem_props; extern const VkExternalMemoryProperties nvk_dma_buf_mem_props; diff --git a/src/nouveau/vulkan/nvk_image.c b/src/nouveau/vulkan/nvk_image.c index 4bddecff7d6..12390ee73f6 100644 --- a/src/nouveau/vulkan/nvk_image.c +++ b/src/nouveau/vulkan/nvk_image.c @@ -9,6 +9,7 @@ #include "nvk_entrypoints.h" #include "nvk_format.h" #include "nvk_physical_device.h" +#include "nvkmd/nvkmd.h" #include "vk_enum_to_str.h" #include "vk_format.h" @@ -822,10 +823,12 @@ nvk_image_plane_size_align_B(const struct nvk_image *image, } static VkResult -nvk_image_plane_alloc_vma(struct nvk_device *dev, - const struct nvk_image *image, - struct nvk_image_plane *plane) +nvk_image_plane_alloc_va(struct nvk_device *dev, + const struct nvk_image *image, + struct nvk_image_plane *plane) { + VkResult result; + const bool sparse_bound = image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT; const bool sparse_resident = @@ -833,15 +836,21 @@ nvk_image_plane_alloc_vma(struct nvk_device *dev, assert(sparse_bound || !sparse_resident); if (sparse_bound || plane->nil.pte_kind) { - uint64_t vma_align_B; - nvk_image_plane_size_align_B(image, plane, &plane->vma_size_B, - &vma_align_B); - plane->addr = nouveau_ws_alloc_vma(dev->ws_dev, 0, plane->vma_size_B, - vma_align_B, false, sparse_resident); - if (plane->addr == 0) { - return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, - "Sparse VMA allocation failed"); - } + enum nvkmd_va_flags va_flags = 0; + if (sparse_resident) + va_flags |= NVKMD_VA_SPARSE; + + uint64_t va_size_B, va_align_B; + nvk_image_plane_size_align_B(image, plane, &va_size_B, &va_align_B); + + result = nvkmd_dev_alloc_va(dev->nvkmd, &dev->vk.base, + va_flags, plane->nil.pte_kind, + va_size_B, va_align_B, + 0 /* fixed_addr */, &plane->va); + if (result != VK_SUCCESS) + return result; + + plane->addr = plane->va->addr; } return VK_SUCCESS; @@ -853,14 +862,8 @@ nvk_image_plane_finish(struct nvk_device *dev, VkImageCreateFlags create_flags, const VkAllocationCallbacks *pAllocator) { - if (plane->vma_size_B) { - const bool sparse_resident = - create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT; - - nouveau_ws_bo_unbind_vma(dev->ws_dev, plane->addr, plane->vma_size_B); - nouveau_ws_free_vma(dev->ws_dev, plane->addr, plane->vma_size_B, - false, sparse_resident); - } + if (plane->va != NULL) + nvkmd_va_free(plane->va); } static void @@ -878,7 +881,7 @@ nvk_image_finish(struct nvk_device *dev, struct nvk_image *image, } if (image->linear_tiled_shadow.nil.size_B > 0) { - assert(image->linear_tiled_shadow.vma_size_B == 0); + assert(image->linear_tiled_shadow.va == NULL); nouveau_ws_bo_destroy(image->linear_tiled_shadow_bo); } @@ -923,7 +926,7 @@ nvk_CreateImage(VkDevice _device, } for (uint8_t plane = 0; plane < image->plane_count; plane++) { - result = nvk_image_plane_alloc_vma(dev, image, &image->planes[plane]); + result = nvk_image_plane_alloc_va(dev, image, &image->planes[plane]); if (result != VK_SUCCESS) { nvk_image_finish(dev, image, pAllocator); vk_free2(&dev->vk.alloc, pAllocator, image); @@ -932,7 +935,7 @@ nvk_CreateImage(VkDevice _device, } if (image->stencil_copy_temp.nil.size_B > 0) { - result = nvk_image_plane_alloc_vma(dev, image, &image->stencil_copy_temp); + result = nvk_image_plane_alloc_va(dev, image, &image->stencil_copy_temp); if (result != VK_SUCCESS) { nvk_image_finish(dev, image, pAllocator); vk_free2(&dev->vk.alloc, pAllocator, image); @@ -1250,7 +1253,7 @@ nvk_GetDeviceImageSubresourceLayoutKHR( nvk_image_finish(dev, &image, NULL); } -static void +static VkResult nvk_image_plane_bind(struct nvk_device *dev, struct nvk_image *image, struct nvk_image_plane *plane, @@ -1261,19 +1264,20 @@ nvk_image_plane_bind(struct nvk_device *dev, nvk_image_plane_size_align_B(image, plane, &plane_size_B, &plane_align_B); *offset_B = align64(*offset_B, plane_align_B); - if (plane->vma_size_B) { - nouveau_ws_bo_bind_vma(dev->ws_dev, - mem->bo, - plane->addr, - plane->vma_size_B, - *offset_B, - plane->nil.pte_kind); + if (plane->va != NULL) { + VkResult result = nvkmd_va_bind_mem(plane->va, &image->vk.base, 0, + mem->mem, *offset_B, + plane->va->size_B); + if (result != VK_SUCCESS) + return result; } else { assert(plane->nil.pte_kind == 0); - plane->addr = mem->bo->offset + *offset_B; + plane->addr = mem->mem->va->addr + *offset_B; } *offset_B += plane_size_B; + + return VK_SUCCESS; } static VkResult @@ -1282,6 +1286,7 @@ nvk_bind_image_memory(struct nvk_device *dev, { VK_FROM_HANDLE(nvk_device_memory, mem, info->memory); VK_FROM_HANDLE(nvk_image, image, info->image); + VkResult result; /* Ignore this struct on Android, we cannot access swapchain structures there. */ #ifdef NVK_USE_WSI_PLATFORM @@ -1312,18 +1317,24 @@ nvk_bind_image_memory(struct nvk_device *dev, vk_find_struct_const(info->pNext, BIND_IMAGE_PLANE_MEMORY_INFO); const uint8_t plane = nvk_image_memory_aspects_to_plane(image, plane_info->planeAspect); - nvk_image_plane_bind(dev, image, &image->planes[plane], - mem, &offset_B); + result = nvk_image_plane_bind(dev, image, &image->planes[plane], + mem, &offset_B); + if (result != VK_SUCCESS) + return result; } else { for (unsigned plane = 0; plane < image->plane_count; plane++) { - nvk_image_plane_bind(dev, image, &image->planes[plane], - mem, &offset_B); + result = nvk_image_plane_bind(dev, image, &image->planes[plane], + mem, &offset_B); + if (result != VK_SUCCESS) + return result; } } if (image->stencil_copy_temp.nil.size_B > 0) { - nvk_image_plane_bind(dev, image, &image->stencil_copy_temp, - mem, &offset_B); + result = nvk_image_plane_bind(dev, image, &image->stencil_copy_temp, + mem, &offset_B); + if (result != VK_SUCCESS) + return result; } return VK_SUCCESS; diff --git a/src/nouveau/vulkan/nvk_image.h b/src/nouveau/vulkan/nvk_image.h index 89c8004dc86..a81406398de 100644 --- a/src/nouveau/vulkan/nvk_image.h +++ b/src/nouveau/vulkan/nvk_image.h @@ -37,6 +37,7 @@ struct nvk_device_memory; struct nvk_physical_device; +struct nvkmd_va; VkFormatFeatureFlags2 nvk_get_image_format_features(struct nvk_physical_device *pdevice, @@ -56,8 +57,8 @@ struct nvk_image_plane { struct nil_image nil; uint64_t addr; - /** Size of the reserved VMA range for sparse images, zero otherwise. */ - uint64_t vma_size_B; + /** Reserved VA for sparse images, NULL otherwise. */ + struct nvkmd_va *va; }; struct nvk_image { diff --git a/src/nouveau/vulkan/nvk_queue_drm_nouveau.c b/src/nouveau/vulkan/nvk_queue_drm_nouveau.c index 392d813ba69..692c3b5f0d4 100644 --- a/src/nouveau/vulkan/nvk_queue_drm_nouveau.c +++ b/src/nouveau/vulkan/nvk_queue_drm_nouveau.c @@ -11,6 +11,8 @@ #include "nvk_image.h" #include "nvk_device_memory.h" #include "nvk_physical_device.h" +#include "nvkmd/nvkmd.h" +#include "nvkmd/nouveau/nvkmd_nouveau.h" #include "nouveau_context.h" @@ -125,6 +127,12 @@ push_bind(struct push_builder *pb, const struct drm_nouveau_vm_bind_op *bind) pb->bind_ops[pb->vmbind.op_count++] = *bind; } +static uint32_t +mem_handle(struct nvk_device_memory *mem) +{ + return nvkmd_nouveau_mem(mem->mem)->bo->handle; +} + static void push_add_buffer_bind(struct push_builder *pb, VkSparseBufferMemoryBindInfo *bind_info) @@ -134,13 +142,13 @@ push_add_buffer_bind(struct push_builder *pb, const VkSparseMemoryBind *bind = &bind_info->pBinds[i]; VK_FROM_HANDLE(nvk_device_memory, mem, bind->memory); - assert(bind->resourceOffset + bind->size <= buffer->vma_size_B); + assert(bind->resourceOffset + bind->size <= buffer->va->size_B); assert(!mem || bind->memoryOffset + bind->size <= mem->vk.size); push_bind(pb, &(struct drm_nouveau_vm_bind_op) { .op = mem ? DRM_NOUVEAU_VM_BIND_OP_MAP : DRM_NOUVEAU_VM_BIND_OP_UNMAP, - .handle = mem ? mem->bo->handle : 0, + .handle = mem ? mem_handle(mem) : 0, .addr = buffer->addr + bind->resourceOffset, .bo_offset = bind->memoryOffset, .range = bind->size, @@ -234,7 +242,7 @@ push_add_image_plane_bind(struct push_builder *pb, push_bind(pb, &(struct drm_nouveau_vm_bind_op) { .op = mem ? DRM_NOUVEAU_VM_BIND_OP_MAP : DRM_NOUVEAU_VM_BIND_OP_UNMAP, - .handle = mem ? mem->bo->handle : 0, + .handle = mem ? mem_handle(mem) : 0, .addr = plane->addr + image_bind_offset_B + image_row_start_tl * tile_size_B, .bo_offset = mem_bind_offset_B + @@ -326,14 +334,14 @@ push_add_image_plane_opaque_bind(struct push_builder *pb, VK_FROM_HANDLE(nvk_device_memory, mem, bind->memory); - assert(plane->vma_size_B == plane->nil.size_B); - assert(plane_offset_B + bind_size_B <= plane->vma_size_B); + assert(plane->va->size_B == plane->nil.size_B); + assert(plane_offset_B + bind_size_B <= plane->va->size_B); assert(!mem || mem_offset_B + bind_size_B <= mem->vk.size); push_bind(pb, &(struct drm_nouveau_vm_bind_op) { .op = mem ? DRM_NOUVEAU_VM_BIND_OP_MAP : DRM_NOUVEAU_VM_BIND_OP_UNMAP, - .handle = mem ? mem->bo->handle : 0, + .handle = mem ? mem_handle(mem) : 0, .addr = plane->addr + plane_offset_B, .bo_offset = mem_offset_B, .range = bind_size_B, @@ -398,7 +406,7 @@ push_add_image_plane_mip_tail_bind(struct push_builder *pb, push_bind(pb, &(struct drm_nouveau_vm_bind_op) { .op = mem ? DRM_NOUVEAU_VM_BIND_OP_MAP : DRM_NOUVEAU_VM_BIND_OP_UNMAP, - .handle = mem ? mem->bo->handle : 0, + .handle = mem ? mem_handle(mem) : 0, .addr = plane->addr + a_image_offset_B, .bo_offset = mem_offset_B + a_bind_offset_B, .range = a_range_B,