OpenHarmony/third_party_mesa3d
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

zink: redesign the allocation try loop to test all heaps

Browse Source 
Before the first time an allocation failed the heap was changed,
now we only change the heap type when checking all heaps results
in allocation failure.

Signed-off-by: Gert Wollny <gert.wollny@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/27155>
This commit is contained in:
Gert Wollny
2024-01-29 19:51:17 +01:00
committed by Marge Bot
parent 252e877a87
commit eb394f5316
2 changed files with 173 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
src/gallium/drivers/zink/zink_resource.c
View File

@@ -1098,25 +1098,26 @@ allocate_bo(struct zink_screen *screen, const struct pipe_resource *templ,
assert(zink_mem_type_idx_from_bits(screen, heap, reqs->memoryTypeBits) != UINT32_MAX);
}
retry:
/* iterate over all available memory types to reduce chance of oom */
for (unsigned i = 0; !obj->bo && i < screen->heap_count[heap]; i++) {
if (!(reqs->memoryTypeBits & BITFIELD_BIT(screen->heap_map[heap][i])))
continue;
while (1) {
/* iterate over all available memory types to reduce chance of oom */
for (unsigned i = 0; !obj->bo && i < screen->heap_count[heap]; i++) {
if (!(reqs->memoryTypeBits & BITFIELD_BIT(screen->heap_map[heap][i])))
continue;
mai.memoryTypeIndex = screen->heap_map[heap][i];
obj->bo = zink_bo(zink_bo_create(screen, reqs->size, alignment, heap, mai.pNext ? ZINK_ALLOC_NO_SUBALLOC : 0, mai.memoryTypeIndex, mai.pNext));
if (!obj->bo) {
if (heap == ZINK_HEAP_DEVICE_LOCAL_VISIBLE) {
/* demote BAR allocations to a different heap on failure to avoid oom */
if (templ->flags & PIPE_RESOURCE_FLAG_MAP_COHERENT || templ->usage == PIPE_USAGE_DYNAMIC)
heap = ZINK_HEAP_HOST_VISIBLE_COHERENT;
else
heap = ZINK_HEAP_DEVICE_LOCAL;
goto retry;
}
mai.memoryTypeIndex = screen->heap_map[heap][i];
obj->bo = zink_bo(zink_bo_create(screen, reqs->size, alignment, heap, mai.pNext ? ZINK_ALLOC_NO_SUBALLOC : 0, mai.memoryTypeIndex, mai.pNext));
}
}
if (obj->bo || heap != ZINK_HEAP_DEVICE_LOCAL_VISIBLE) {
break;
/* demote BAR allocations to a different heap on failure to avoid oom */
if (templ->flags & PIPE_RESOURCE_FLAG_MAP_COHERENT || templ->usage == PIPE_USAGE_DYNAMIC)
heap = ZINK_HEAP_HOST_VISIBLE_COHERENT;
else
heap = ZINK_HEAP_DEVICE_LOCAL;
}
};
return obj->bo ? 0: -2;
}

387
src/gallium/drivers/zink/zink_synchronization.cpp
View File

@@ -162,6 +162,10 @@ zink_resource_access_is_write(VkAccessFlags flags)
bool
zink_resource_image_needs_barrier(struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
{
if (!pipeline)
pipeline = pipeline_dst_stage(new_layout);
if (!flags)
flags = access_dst_flags(new_layout);
return res->layout != new_layout || (res->obj->access_stage & pipeline) != pipeline ||
(res->obj->access & flags) != flags ||
zink_resource_access_is_write(res->obj->access) ||
@@ -258,36 +262,29 @@ unordered_res_exec(const struct zink_context *ctx, const struct zink_resource *r
return res->obj->unordered_write || !zink_batch_usage_matches(res->obj->bo->writes.u, ctx->batch.state);
}
static inline bool
check_unordered_exec(struct zink_context *ctx, struct zink_resource *res, bool is_write)
{
if (res) {
if (!res->obj->is_buffer) {
/* TODO: figure out how to link up unordered layout -> ordered layout and delete this conditionals */
if (zink_resource_usage_is_unflushed(res) && !res->obj->unordered_read && !res->obj->unordered_write)
return false;
}
return unordered_res_exec(ctx, res, is_write);
}
return true;
}
VkCommandBuffer
zink_get_cmdbuf(struct zink_context *ctx, struct zink_resource *src, struct zink_resource *dst)
{
bool unordered_exec = (zink_debug & ZINK_DEBUG_NOREORDER) == 0;
unordered_exec &= check_unordered_exec(ctx, src, false) &&
check_unordered_exec(ctx, dst, true);
/* TODO: figure out how to link up unordered layout -> ordered layout and delete these two conditionals */
if (src && !src->obj->is_buffer) {
if (zink_resource_usage_is_unflushed(src) && !src->obj->unordered_read && !src->obj->unordered_write)
unordered_exec = false;
}
if (dst && !dst->obj->is_buffer) {
if (zink_resource_usage_is_unflushed(dst) && !dst->obj->unordered_read && !dst->obj->unordered_write)
unordered_exec = false;
}
if (src && unordered_exec)
unordered_exec &= unordered_res_exec(ctx, src, false);
if (dst && unordered_exec)
unordered_exec &= unordered_res_exec(ctx, dst, true);
if (src)
src->obj->unordered_read = unordered_exec;
if (dst)
dst->obj->unordered_write = unordered_exec;
if (!unordered_exec || ctx->unordered_blitting)
zink_batch_no_rp(ctx);
if (unordered_exec) {
ctx->batch.state->has_barriers = true;
ctx->batch.has_work = true;
@@ -323,201 +320,7 @@ resource_check_defer_image_barrier(struct zink_context *ctx, struct zink_resourc
_mesa_set_add(ctx->need_barriers[is_compute], res);
}
enum barrier_type {
barrier_default,
barrier_KHR_synchronzation2
};
template <barrier_type BARRIER_API>
struct emit_memory_barrier {
static void for_image(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout,
VkAccessFlags flags, VkPipelineStageFlags pipeline, bool completed, VkCommandBuffer cmdbuf,
bool *queue_import) {
VkImageMemoryBarrier imb;
zink_resource_image_barrier_init(&imb, res, new_layout, flags, pipeline);
if (!res->obj->access_stage || completed)
imb.srcAccessMask = 0;
if (res->obj->needs_zs_evaluate)
imb.pNext = &res->obj->zs_evaluate;
res->obj->needs_zs_evaluate = false;
if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) {
imb.srcQueueFamilyIndex = res->queue;
imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue;
res->queue = VK_QUEUE_FAMILY_IGNORED;
*queue_import = true;
}
VKCTX(CmdPipelineBarrier)(
cmdbuf,
res->obj->access_stage ? res->obj->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
pipeline,
0,
0, NULL,
0, NULL,
1, &imb
);
}
static void for_buffer(struct zink_context *ctx, struct zink_resource *res,
VkPipelineStageFlags pipeline,
VkAccessFlags flags,
bool unordered,
bool usage_matches,
VkPipelineStageFlags stages,
VkCommandBuffer cmdbuf) {
VkMemoryBarrier bmb;
bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
bmb.pNext = NULL;
if (unordered) {
stages = usage_matches ? res->obj->unordered_access_stage : stages;
bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access;
} else {
bmb.srcAccessMask = res->obj->access;
}
VKCTX(CmdPipelineBarrier)(
cmdbuf,
stages,
pipeline,
0,
1, &bmb,
0, NULL,
0, NULL);
}
};
template <>
struct emit_memory_barrier<barrier_KHR_synchronzation2> {
static void for_image(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout,
VkAccessFlags flags, VkPipelineStageFlags pipeline, bool completed, VkCommandBuffer cmdbuf,
bool *queue_import) {
VkImageMemoryBarrier2 imb;
zink_resource_image_barrier2_init(&imb, res, new_layout, flags, pipeline);
if (!res->obj->access_stage || completed)
imb.srcAccessMask = 0;
if (res->obj->needs_zs_evaluate)
imb.pNext = &res->obj->zs_evaluate;
res->obj->needs_zs_evaluate = false;
if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) {
imb.srcQueueFamilyIndex = res->queue;
imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue;
res->queue = VK_QUEUE_FAMILY_IGNORED;
*queue_import = true;
}
VkDependencyInfo dep = {
VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
NULL,
0,
0,
NULL,
0,
NULL,
1,
&imb
};
VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep);
}
static void for_buffer(struct zink_context *ctx, struct zink_resource *res,
VkPipelineStageFlags pipeline,
VkAccessFlags flags,
bool unordered,
bool usage_matches,
VkPipelineStageFlags stages,
VkCommandBuffer cmdbuf) {
VkMemoryBarrier2 bmb;
bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2;
bmb.pNext = NULL;
if (unordered) {
bmb.srcStageMask = usage_matches ? res->obj->unordered_access_stage : stages;
bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access;
} else {
bmb.srcStageMask = stages;
bmb.srcAccessMask = res->obj->access;
}
bmb.dstStageMask = pipeline;
bmb.dstAccessMask = flags;
VkDependencyInfo dep = {
VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
NULL,
0,
1,
&bmb,
0,
NULL,
0,
NULL
};
VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep);
}
};
template <bool UNSYNCHRONIZED>
struct update_unordered_access_and_get_cmdbuf
{
/* use base template to make the cases for true and false more explicite below */
};
template <>
struct update_unordered_access_and_get_cmdbuf<true> {
static VkCommandBuffer apply(struct zink_context *ctx, struct zink_resource *res, bool usage_matches, bool is_write) {
assert(!usage_matches);
res->obj->unordered_write = true;
res->obj->unordered_read = true;
ctx->batch.state->has_unsync = true;
return ctx->batch.state->unsynchronized_cmdbuf;
}
};
template <>
struct update_unordered_access_and_get_cmdbuf<false> {
static VkCommandBuffer apply(struct zink_context *ctx, struct zink_resource *res, bool usage_matches, bool is_write) {
VkCommandBuffer cmdbuf;
if (!usage_matches) {
res->obj->unordered_write = true;
if (is_write || zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, ZINK_RESOURCE_ACCESS_RW))
res->obj->unordered_read = true;
}
if (zink_resource_usage_matches(res, ctx->batch.state) && !ctx->unordered_blitting &&
/* if current batch usage exists with ordered non-transfer access, never promote
* this avoids layout dsync
*/
(!res->obj->unordered_read || !res->obj->unordered_write)) {
cmdbuf = ctx->batch.state->cmdbuf;
res->obj->unordered_write = false;
res->obj->unordered_read = false;
/* it's impossible to detect this from the caller
* there should be no valid case where this barrier can occur inside a renderpass
*/
zink_batch_no_rp(ctx);
} else {
cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL);
/* force subsequent barriers to be ordered to avoid layout desync */
if (cmdbuf != ctx->batch.state->reordered_cmdbuf) {
res->obj->unordered_write = false;
res->obj->unordered_read = false;
}
}
return cmdbuf;
}
};
template <bool UNSYNCHRONIZED>
struct check_defer_image_barrier {
static void apply(UNUSED struct zink_context *ctx, UNUSED struct zink_resource *res, UNUSED VkImageLayout new_layout,
UNUSED VkPipelineStageFlags pipeline) {
}
};
template <>
struct check_defer_image_barrier<false> {
static void apply(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout,
VkPipelineStageFlags pipeline) {
resource_check_defer_image_barrier(ctx, res, new_layout, pipeline);
}
};
template <barrier_type BARRIER_API, bool UNSYNCHRONIZED>
template <bool HAS_SYNC2, bool UNSYNCHRONIZED>
void
zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
{
@@ -533,12 +336,92 @@ zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res,
enum zink_resource_access rw = is_write ? ZINK_RESOURCE_ACCESS_RW : ZINK_RESOURCE_ACCESS_WRITE;
bool completed = zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, rw);
bool usage_matches = !completed && zink_resource_usage_matches(res, ctx->batch.state);
VkCommandBuffer cmdbuf = update_unordered_access_and_get_cmdbuf<UNSYNCHRONIZED>::apply(ctx, res, usage_matches, is_write);
VkCommandBuffer cmdbuf;
if (!usage_matches) {
res->obj->unordered_write = true;
if (is_write || zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, ZINK_RESOURCE_ACCESS_RW))
res->obj->unordered_read = true;
} else {
assert(!UNSYNCHRONIZED);
}
if (UNSYNCHRONIZED) {
cmdbuf = ctx->batch.state->unsynchronized_cmdbuf;
res->obj->unordered_write = true;
res->obj->unordered_read = true;
ctx->batch.state->has_unsync = true;
} else if (zink_resource_usage_matches(res, ctx->batch.state) && !ctx->unordered_blitting &&
/* if current batch usage exists with ordered non-transfer access, never promote
* this avoids layout dsync
*/
(!res->obj->unordered_read || !res->obj->unordered_write)) {
cmdbuf = ctx->batch.state->cmdbuf;
res->obj->unordered_write = false;
res->obj->unordered_read = false;
/* it's impossible to detect this from the caller
* there should be no valid case where this barrier can occur inside a renderpass
*/
zink_batch_no_rp(ctx);
} else {
cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL);
/* force subsequent barriers to be ordered to avoid layout desync */
if (cmdbuf != ctx->batch.state->reordered_cmdbuf) {
res->obj->unordered_write = false;
res->obj->unordered_read = false;
}
}
assert(new_layout);
bool marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "image_barrier(%s->%s)", vk_ImageLayout_to_str(res->layout), vk_ImageLayout_to_str(new_layout));
bool queue_import = false;
emit_memory_barrier<BARRIER_API>::for_image(ctx, res, new_layout, flags, pipeline, completed, cmdbuf, &queue_import);
if (HAS_SYNC2) {
VkImageMemoryBarrier2 imb;
zink_resource_image_barrier2_init(&imb, res, new_layout, flags, pipeline);
if (!res->obj->access_stage || completed)
imb.srcAccessMask = 0;
if (res->obj->needs_zs_evaluate)
imb.pNext = &res->obj->zs_evaluate;
res->obj->needs_zs_evaluate = false;
if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) {
imb.srcQueueFamilyIndex = res->queue;
imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue;
res->queue = VK_QUEUE_FAMILY_IGNORED;
queue_import = true;
}
VkDependencyInfo dep = {
VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
NULL,
0,
0,
NULL,
0,
NULL,
1,
&imb
};
VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep);
} else {
VkImageMemoryBarrier imb;
zink_resource_image_barrier_init(&imb, res, new_layout, flags, pipeline);
if (!res->obj->access_stage || completed)
imb.srcAccessMask = 0;
if (res->obj->needs_zs_evaluate)
imb.pNext = &res->obj->zs_evaluate;
res->obj->needs_zs_evaluate = false;
if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) {
imb.srcQueueFamilyIndex = res->queue;
imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue;
res->queue = VK_QUEUE_FAMILY_IGNORED;
queue_import = true;
}
VKCTX(CmdPipelineBarrier)(
cmdbuf,
res->obj->access_stage ? res->obj->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
pipeline,
0,
0, NULL,
0, NULL,
1, &imb
);
}
zink_cmd_debug_marker_end(ctx, cmdbuf, marker);
if (!UNSYNCHRONIZED)
@@ -550,10 +433,6 @@ zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res,
res->obj->access = flags;
res->obj->access_stage = pipeline;
res->layout = new_layout;
if (new_layout != VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL)
zink_resource_copies_reset(res);
if (res->obj->exportable)
simple_mtx_lock(&ctx->batch.state->exportable_lock);
if (res->obj->dt) {
@@ -569,6 +448,8 @@ zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res,
pipe_resource_reference(&pres, &res->base.b);
}
}
if (new_layout != VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL)
zink_resource_copies_reset(res);
if (res->obj->exportable && queue_import) {
for (struct zink_resource *r = res; r; r = zink_resource(r->base.b.next)) {
VkSemaphore sem = zink_screen_export_dmabuf_semaphore(zink_screen(ctx->base.screen), r);
@@ -694,9 +575,7 @@ buffer_needs_barrier(struct zink_resource *res, VkAccessFlags flags, VkPipelineS
((unordered ? res->obj->unordered_access : res->obj->access) & flags) != flags;
}
template <barrier_type BARRIER_API>
template <bool HAS_SYNC2>
void
zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline)
{
@@ -759,7 +638,51 @@ zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_resource *res
}
VkPipelineStageFlags stages = res->obj->access_stage ? res->obj->access_stage : pipeline_access_stage(res->obj->access);;
emit_memory_barrier<BARRIER_API>::for_buffer(ctx, res, pipeline, flags, unordered,usage_matches, stages, cmdbuf);
if (HAS_SYNC2) {
VkMemoryBarrier2 bmb;
bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2;
bmb.pNext = NULL;
if (unordered) {
bmb.srcStageMask = usage_matches ? res->obj->unordered_access_stage : stages;
bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access;
} else {
bmb.srcStageMask = stages;
bmb.srcAccessMask = res->obj->access;
}
bmb.dstStageMask = pipeline;
bmb.dstAccessMask = flags;
VkDependencyInfo dep = {
VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
NULL,
0,
1,
&bmb,
0,
NULL,
0,
NULL
};
VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep);
} else {
VkMemoryBarrier bmb;
bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
bmb.pNext = NULL;
if (unordered) {
stages = usage_matches ? res->obj->unordered_access_stage : stages;
bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access;
} else {
bmb.srcAccessMask = res->obj->access;
}
VKCTX(CmdPipelineBarrier)(
cmdbuf,
stages,
pipeline,
0,
1, &bmb,
0, NULL,
0, NULL
);
}
zink_cmd_debug_marker_end(ctx, cmdbuf, marker);
}
@@ -790,12 +713,12 @@ void
zink_synchronization_init(struct zink_screen *screen)
{
if (screen->info.have_vulkan13 || screen->info.have_KHR_synchronization2) {
screen->buffer_barrier = zink_resource_buffer_barrier<barrier_KHR_synchronzation2>;
screen->image_barrier = zink_resource_image_barrier<barrier_KHR_synchronzation2, false>;
screen->image_barrier_unsync = zink_resource_image_barrier<barrier_KHR_synchronzation2, true>;
screen->buffer_barrier = zink_resource_buffer_barrier<true>;
screen->image_barrier = zink_resource_image_barrier<true, false>;
screen->image_barrier_unsync = zink_resource_image_barrier<true, true>;
} else {
screen->buffer_barrier = zink_resource_buffer_barrier<barrier_default>;
screen->image_barrier = zink_resource_image_barrier<barrier_default, false>;
screen->image_barrier_unsync = zink_resource_image_barrier<barrier_default, true>;
screen->buffer_barrier = zink_resource_buffer_barrier<false>;
screen->image_barrier = zink_resource_image_barrier<false, false>;
screen->image_barrier_unsync = zink_resource_image_barrier<false, true>;
}
}