OpenHarmony/third_party_mesa3d
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lavapipe: Implement VK_KHR_acceleration_structure

Browse Source 
Acked-By: Mike Blumenkrantz <michael.blumenkrantz@gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/25616>
This commit is contained in:
Konstantin Seurer
2023-07-14 08:13:54 +02:00
committed by Marge Bot
parent ff09e95080
commit 897ccbd180
9 changed files with 937 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/gallium/auxiliary/gallivm/lp_bld_jit_types.h
View File

@@ -237,6 +237,7 @@ struct lp_descriptor {
struct lp_jit_image image;
};
struct lp_jit_buffer buffer;
uint64_t accel_struct;
};
/* Store sample/image functions in the same location since some d3d12 games

611
src/gallium/frontends/lavapipe/lvp_acceleration_structure.c Normal file
View File

@@ -0,0 +1,611 @@
/*
* Copyright © 2023 Valve Corporation
* SPDX-License-Identifier: MIT
*/
#include "lvp_acceleration_structure.h"
#include "lvp_entrypoints.h"
#include "util/format/format_utils.h"
#include "util/half_float.h"
static_assert(sizeof(struct lvp_bvh_triangle_node) % 8 == 0, "lvp_bvh_triangle_node is not padded");
static_assert(sizeof(struct lvp_bvh_aabb_node) % 8 == 0, "lvp_bvh_aabb_node is not padded");
static_assert(sizeof(struct lvp_bvh_instance_node) % 8 == 0, "lvp_bvh_instance_node is not padded");
static_assert(sizeof(struct lvp_bvh_box_node) % 8 == 0, "lvp_bvh_box_node is not padded");
VKAPI_ATTR void VKAPI_CALL
lvp_GetAccelerationStructureBuildSizesKHR(
VkDevice _device, VkAccelerationStructureBuildTypeKHR buildType,
const VkAccelerationStructureBuildGeometryInfoKHR *pBuildInfo,
const uint32_t *pMaxPrimitiveCounts, VkAccelerationStructureBuildSizesInfoKHR *pSizeInfo)
{
pSizeInfo->buildScratchSize = 64;
pSizeInfo->updateScratchSize = 64;
uint32_t leaf_count = 0;
for (uint32_t i = 0; i < pBuildInfo->geometryCount; i++)
leaf_count += pMaxPrimitiveCounts[i];
uint32_t internal_count = MAX2(leaf_count, 2) - 1;
VkGeometryTypeKHR geometry_type = VK_GEOMETRY_TYPE_TRIANGLES_KHR;
if (pBuildInfo->geometryCount) {
if (pBuildInfo->pGeometries)
geometry_type = pBuildInfo->pGeometries[0].geometryType;
else
geometry_type = pBuildInfo->ppGeometries[0]->geometryType;
}
uint32_t leaf_size;
switch (geometry_type) {
case VK_GEOMETRY_TYPE_TRIANGLES_KHR:
leaf_size = sizeof(struct lvp_bvh_triangle_node);
break;
case VK_GEOMETRY_TYPE_AABBS_KHR:
leaf_size = sizeof(struct lvp_bvh_aabb_node);
break;
case VK_GEOMETRY_TYPE_INSTANCES_KHR:
leaf_size = sizeof(struct lvp_bvh_instance_node);
break;
default:
unreachable("Unknown VkGeometryTypeKHR");
}
uint32_t bvh_size = sizeof(struct lvp_bvh_header);
bvh_size += leaf_count * leaf_size;
bvh_size += internal_count * sizeof(struct lvp_bvh_box_node);
pSizeInfo->accelerationStructureSize = bvh_size;
}
VKAPI_ATTR VkResult VKAPI_CALL
lvp_WriteAccelerationStructuresPropertiesKHR(
VkDevice _device, uint32_t accelerationStructureCount,
const VkAccelerationStructureKHR *pAccelerationStructures, VkQueryType queryType,
size_t dataSize, void *pData, size_t stride)
{
unreachable("Unimplemented");
return VK_ERROR_FEATURE_NOT_PRESENT;
}
VKAPI_ATTR VkResult VKAPI_CALL
lvp_BuildAccelerationStructuresKHR(
VkDevice _device, VkDeferredOperationKHR deferredOperation, uint32_t infoCount,
const VkAccelerationStructureBuildGeometryInfoKHR *pInfos,
const VkAccelerationStructureBuildRangeInfoKHR *const *ppBuildRangeInfos)
{
unreachable("Unimplemented");
return VK_ERROR_FEATURE_NOT_PRESENT;
}
VKAPI_ATTR void VKAPI_CALL
lvp_GetDeviceAccelerationStructureCompatibilityKHR(
VkDevice _device, const VkAccelerationStructureVersionInfoKHR *pVersionInfo,
VkAccelerationStructureCompatibilityKHR *pCompatibility)
{
uint8_t uuid[VK_UUID_SIZE];
lvp_device_get_cache_uuid(uuid);
bool compat = memcmp(pVersionInfo->pVersionData, uuid, VK_UUID_SIZE) == 0;
*pCompatibility = compat ? VK_ACCELERATION_STRUCTURE_COMPATIBILITY_COMPATIBLE_KHR
: VK_ACCELERATION_STRUCTURE_COMPATIBILITY_INCOMPATIBLE_KHR;
}
VKAPI_ATTR VkResult VKAPI_CALL
lvp_CopyAccelerationStructureKHR(VkDevice _device, VkDeferredOperationKHR deferredOperation,
const VkCopyAccelerationStructureInfoKHR *pInfo)
{
unreachable("Unimplemented");
return VK_ERROR_FEATURE_NOT_PRESENT;
}
VKAPI_ATTR VkResult VKAPI_CALL
lvp_CopyMemoryToAccelerationStructureKHR(VkDevice _device, VkDeferredOperationKHR deferredOperation,
const VkCopyMemoryToAccelerationStructureInfoKHR *pInfo)
{
unreachable("Unimplemented");
return VK_ERROR_FEATURE_NOT_PRESENT;
}
VKAPI_ATTR VkResult VKAPI_CALL
lvp_CopyAccelerationStructureToMemoryKHR(VkDevice _device, VkDeferredOperationKHR deferredOperation,
const VkCopyAccelerationStructureToMemoryInfoKHR *pInfo)
{
unreachable("Unimplemented");
return VK_ERROR_FEATURE_NOT_PRESENT;
}
static uint32_t
lvp_pack_geometry_id_and_flags(uint32_t geometry_id, uint32_t flags)
{
uint32_t geometry_id_and_flags = geometry_id;
if (flags & VK_GEOMETRY_OPAQUE_BIT_KHR)
geometry_id_and_flags |= LVP_GEOMETRY_OPAQUE;
return geometry_id_and_flags;
}
static uint32_t
lvp_pack_sbt_offset_and_flags(uint32_t sbt_offset, VkGeometryInstanceFlagsKHR flags)
{
uint32_t ret = sbt_offset;
if (flags & VK_GEOMETRY_INSTANCE_FORCE_OPAQUE_BIT_KHR)
ret |= LVP_INSTANCE_FORCE_OPAQUE;
if (!(flags & VK_GEOMETRY_INSTANCE_FORCE_NO_OPAQUE_BIT_KHR))
ret |= LVP_INSTANCE_NO_FORCE_NOT_OPAQUE;
if (flags & VK_GEOMETRY_INSTANCE_TRIANGLE_FACING_CULL_DISABLE_BIT_KHR)
ret |= LVP_INSTANCE_TRIANGLE_FACING_CULL_DISABLE;
if (flags & VK_GEOMETRY_INSTANCE_TRIANGLE_FLIP_FACING_BIT_KHR)
ret |= LVP_INSTANCE_TRIANGLE_FLIP_FACING;
return ret;
}
struct lvp_build_internal_ctx {
uint8_t *dst;
uint32_t dst_offset;
void *leaf_nodes;
uint32_t leaf_nodes_offset;
uint32_t leaf_node_type;
uint32_t leaf_node_size;
};
static uint32_t
lvp_build_internal_node(struct lvp_build_internal_ctx *ctx, uint32_t first_leaf, uint32_t last_leaf)
{
uint32_t dst_offset = ctx->dst_offset;
ctx->dst_offset += sizeof(struct lvp_bvh_box_node);
uint32_t node_id = dst_offset | lvp_bvh_node_internal;
struct lvp_bvh_box_node *node = (void *)(ctx->dst + dst_offset);
uint32_t split = (first_leaf + last_leaf) / 2;
if (first_leaf < split)
node->children[0] = lvp_build_internal_node(ctx, first_leaf, split);
else
node->children[0] =
(ctx->leaf_nodes_offset + (first_leaf * ctx->leaf_node_size)) | ctx->leaf_node_type;
if (first_leaf < last_leaf) {
if (split + 1 < last_leaf)
node->children[1] = lvp_build_internal_node(ctx, split + 1, last_leaf);
else
node->children[1] =
(ctx->leaf_nodes_offset + (last_leaf * ctx->leaf_node_size)) | ctx->leaf_node_type;
} else {
node->children[1] = LVP_BVH_INVALID_NODE;
}
for (uint32_t i = 0; i < 2; i++) {
struct lvp_aabb *aabb = &node->bounds[i];
if (node->children[i] == LVP_BVH_INVALID_NODE) {
aabb->min.x = INFINITY;
aabb->min.y = INFINITY;
aabb->min.z = INFINITY;
aabb->max.x = -INFINITY;
aabb->max.y = -INFINITY;
aabb->max.z = -INFINITY;
continue;
}
uint32_t child_offset = node->children[i] & (~3u);
uint32_t child_type = node->children[i] & 3u;
void *child_node = (void *)(ctx->dst + child_offset);
switch (child_type) {
case lvp_bvh_node_triangle: {
struct lvp_bvh_triangle_node *triangle = child_node;
aabb->min.x = MIN3(triangle->coords[0][0], triangle->coords[1][0], triangle->coords[2][0]);
aabb->min.y = MIN3(triangle->coords[0][1], triangle->coords[1][1], triangle->coords[2][1]);
aabb->min.z = MIN3(triangle->coords[0][2], triangle->coords[1][2], triangle->coords[2][2]);
aabb->max.x = MAX3(triangle->coords[0][0], triangle->coords[1][0], triangle->coords[2][0]);
aabb->max.y = MAX3(triangle->coords[0][1], triangle->coords[1][1], triangle->coords[2][1]);
aabb->max.z = MAX3(triangle->coords[0][2], triangle->coords[1][2], triangle->coords[2][2]);
break;
}
case lvp_bvh_node_internal: {
struct lvp_bvh_box_node *box = child_node;
aabb->min.x = MIN2(box->bounds[0].min.x, box->bounds[1].min.x);
aabb->min.y = MIN2(box->bounds[0].min.y, box->bounds[1].min.y);
aabb->min.z = MIN2(box->bounds[0].min.z, box->bounds[1].min.z);
aabb->max.x = MAX2(box->bounds[0].max.x, box->bounds[1].max.x);
aabb->max.y = MAX2(box->bounds[0].max.y, box->bounds[1].max.y);
aabb->max.z = MAX2(box->bounds[0].max.z, box->bounds[1].max.z);
break;
}
case lvp_bvh_node_instance: {
struct lvp_bvh_instance_node *instance = child_node;
struct lvp_bvh_header *instance_header = (void *)(uintptr_t)instance->bvh_ptr;
float bounds[2][3];
float header_bounds[2][3];
memcpy(header_bounds, &instance_header->bounds, sizeof(struct lvp_aabb));
for (unsigned j = 0; j < 3; ++j) {
bounds[0][j] = instance->otw_matrix.values[j][3];
bounds[1][j] = instance->otw_matrix.values[j][3];
for (unsigned k = 0; k < 3; ++k) {
bounds[0][j] += MIN2(instance->otw_matrix.values[j][k] * header_bounds[0][k],
instance->otw_matrix.values[j][k] * header_bounds[1][k]);
bounds[1][j] += MAX2(instance->otw_matrix.values[j][k] * header_bounds[0][k],
instance->otw_matrix.values[j][k] * header_bounds[1][k]);
}
}
memcpy(aabb, bounds, sizeof(struct lvp_aabb));
break;
}
case lvp_bvh_node_aabb: {
struct lvp_bvh_aabb_node *aabb_node = child_node;
memcpy(aabb, &aabb_node->bounds, sizeof(struct lvp_aabb));
break;
}
default:
unreachable("Invalid node type");
}
}
return node_id;
}
void
lvp_build_acceleration_structure(VkAccelerationStructureBuildGeometryInfoKHR *info,
const VkAccelerationStructureBuildRangeInfoKHR *ranges)
{
VK_FROM_HANDLE(vk_acceleration_structure, accel_struct, info->dstAccelerationStructure);
void *dst = (void *)(uintptr_t)vk_acceleration_structure_get_va(accel_struct);
memset(dst, 0, accel_struct->size);
struct lvp_bvh_header *header = dst;
header->instance_count = 0;
struct lvp_bvh_box_node *root = (void *)((uint8_t *)dst + sizeof(struct lvp_bvh_header));
uint32_t leaf_count = 0;
for (unsigned i = 0; i < info->geometryCount; i++)
leaf_count += ranges[i].primitiveCount;
if (!leaf_count) {
for (uint32_t i = 0; i < 2; i++) {
root->bounds[i].min.x = INFINITY;
root->bounds[i].min.y = INFINITY;
root->bounds[i].min.z = INFINITY;
root->bounds[i].max.x = -INFINITY;
root->bounds[i].max.y = -INFINITY;
root->bounds[i].max.z = -INFINITY;
}
return;
}
uint32_t internal_count = MAX2(leaf_count, 2) - 1;
uint32_t primitive_index = 0;
header->leaf_nodes_offset =
sizeof(struct lvp_bvh_header) + sizeof(struct lvp_bvh_box_node) * internal_count;
void *leaf_nodes = (void *)((uint8_t *)dst + header->leaf_nodes_offset);
for (unsigned i = 0; i < info->geometryCount; i++) {
const VkAccelerationStructureGeometryKHR *geom =
info->pGeometries ? &info->pGeometries[i] : info->ppGeometries[i];
const VkAccelerationStructureBuildRangeInfoKHR *range = &ranges[i];
uint32_t geometry_id_and_flags = lvp_pack_geometry_id_and_flags(i, geom->flags);
switch (geom->geometryType) {
case VK_GEOMETRY_TYPE_TRIANGLES_KHR: {
assert(info->type == VK_ACCELERATION_STRUCTURE_TYPE_BOTTOM_LEVEL_KHR);
const uint8_t *vertex_data_base = geom->geometry.triangles.vertexData.hostAddress;
vertex_data_base += range->firstVertex * geom->geometry.triangles.vertexStride;
const uint8_t *index_data = geom->geometry.triangles.indexData.hostAddress;
if (geom->geometry.triangles.indexType == VK_INDEX_TYPE_NONE_KHR)
vertex_data_base += range->primitiveOffset;
else
index_data += range->primitiveOffset;
VkTransformMatrixKHR transform_matrix = {
.matrix =
{
{1.0, 0.0, 0.0, 0.0},
{0.0, 1.0, 0.0, 0.0},
{0.0, 0.0, 1.0, 0.0},
},
};
const uint8_t *transform = geom->geometry.triangles.transformData.hostAddress;
if (transform) {
transform += range->transformOffset;
transform_matrix = *(VkTransformMatrixKHR *)transform;
}
VkDeviceSize stride = geom->geometry.triangles.vertexStride;
VkFormat vertex_format = geom->geometry.triangles.vertexFormat;
VkIndexType index_type = geom->geometry.triangles.indexType;
for (uint32_t j = 0; j < range->primitiveCount; j++) {
struct lvp_bvh_triangle_node *node = leaf_nodes;
node += primitive_index;
node->primitive_id = j;
node->geometry_id_and_flags = geometry_id_and_flags;
for (uint32_t v = 0; v < 3; v++) {
uint32_t index = range->firstVertex;
switch (index_type) {
case VK_INDEX_TYPE_NONE_KHR:
index += j * 3 + v;
break;
case VK_INDEX_TYPE_UINT8_EXT:
index += *(const uint8_t *)index_data;
index_data += 1;
break;
case VK_INDEX_TYPE_UINT16:
index += *(const uint16_t *)index_data;
index_data += 2;
break;
case VK_INDEX_TYPE_UINT32:
index += *(const uint32_t *)index_data;
index_data += 4;
break;
case VK_INDEX_TYPE_MAX_ENUM:
unreachable("Unhandled VK_INDEX_TYPE_MAX_ENUM");
break;
}
const uint8_t *vertex_data = vertex_data_base + index * stride;
float coords[4];
switch (vertex_format) {
case VK_FORMAT_R32G32_SFLOAT:
coords[0] = *(const float *)(vertex_data + 0);
coords[1] = *(const float *)(vertex_data + 4);
coords[2] = 0.0f;
coords[3] = 1.0f;
break;
case VK_FORMAT_R32G32B32_SFLOAT:
coords[0] = *(const float *)(vertex_data + 0);
coords[1] = *(const float *)(vertex_data + 4);
coords[2] = *(const float *)(vertex_data + 8);
coords[3] = 1.0f;
break;
case VK_FORMAT_R32G32B32A32_SFLOAT:
coords[0] = *(const float *)(vertex_data + 0);
coords[1] = *(const float *)(vertex_data + 4);
coords[2] = *(const float *)(vertex_data + 8);
coords[3] = *(const float *)(vertex_data + 12);
break;
case VK_FORMAT_R16G16_SFLOAT:
coords[0] = _mesa_half_to_float(*(const uint16_t *)(vertex_data + 0));
coords[1] = _mesa_half_to_float(*(const uint16_t *)(vertex_data + 2));
coords[2] = 0.0f;
coords[3] = 1.0f;
break;
case VK_FORMAT_R16G16B16_SFLOAT:
coords[0] = _mesa_half_to_float(*(const uint16_t *)(vertex_data + 0));
coords[1] = _mesa_half_to_float(*(const uint16_t *)(vertex_data + 2));
coords[2] = _mesa_half_to_float(*(const uint16_t *)(vertex_data + 4));
coords[3] = 1.0f;
break;
case VK_FORMAT_R16G16B16A16_SFLOAT:
coords[0] = _mesa_half_to_float(*(const uint16_t *)(vertex_data + 0));
coords[1] = _mesa_half_to_float(*(const uint16_t *)(vertex_data + 2));
coords[2] = _mesa_half_to_float(*(const uint16_t *)(vertex_data + 4));
coords[3] = _mesa_half_to_float(*(const uint16_t *)(vertex_data + 6));
break;
case VK_FORMAT_R16G16_SNORM:
coords[0] = _mesa_snorm_to_float(*(const int16_t *)(vertex_data + 0), 16);
coords[1] = _mesa_snorm_to_float(*(const int16_t *)(vertex_data + 2), 16);
coords[2] = 0.0f;
coords[3] = 1.0f;
break;
case VK_FORMAT_R16G16_UNORM:
coords[0] = _mesa_unorm_to_float(*(const uint16_t *)(vertex_data + 0), 16);
coords[1] = _mesa_unorm_to_float(*(const uint16_t *)(vertex_data + 2), 16);
coords[2] = 0.0f;
coords[3] = 1.0f;
break;
case VK_FORMAT_R16G16B16A16_SNORM:
coords[0] = _mesa_snorm_to_float(*(const int16_t *)(vertex_data + 0), 16);
coords[1] = _mesa_snorm_to_float(*(const int16_t *)(vertex_data + 2), 16);
coords[2] = _mesa_snorm_to_float(*(const int16_t *)(vertex_data + 4), 16);
coords[3] = _mesa_snorm_to_float(*(const int16_t *)(vertex_data + 6), 16);
break;
case VK_FORMAT_R16G16B16A16_UNORM:
coords[0] = _mesa_unorm_to_float(*(const uint16_t *)(vertex_data + 0), 16);
coords[1] = _mesa_unorm_to_float(*(const uint16_t *)(vertex_data + 2), 16);
coords[2] = _mesa_unorm_to_float(*(const uint16_t *)(vertex_data + 4), 16);
coords[3] = _mesa_unorm_to_float(*(const uint16_t *)(vertex_data + 6), 16);
break;
case VK_FORMAT_R8G8_SNORM:
coords[0] = _mesa_snorm_to_float(*(const int8_t *)(vertex_data + 0), 8);
coords[1] = _mesa_snorm_to_float(*(const int8_t *)(vertex_data + 1), 8);
coords[2] = 0.0f;
coords[3] = 1.0f;
break;
case VK_FORMAT_R8G8_UNORM:
coords[0] = _mesa_unorm_to_float(*(const uint8_t *)(vertex_data + 0), 8);
coords[1] = _mesa_unorm_to_float(*(const uint8_t *)(vertex_data + 1), 8);
coords[2] = 0.0f;
coords[3] = 1.0f;
break;
case VK_FORMAT_R8G8B8A8_SNORM:
coords[0] = _mesa_snorm_to_float(*(const int8_t *)(vertex_data + 0), 8);
coords[1] = _mesa_snorm_to_float(*(const int8_t *)(vertex_data + 1), 8);
coords[2] = _mesa_snorm_to_float(*(const int8_t *)(vertex_data + 2), 8);
coords[3] = _mesa_snorm_to_float(*(const int8_t *)(vertex_data + 3), 8);
break;
case VK_FORMAT_R8G8B8A8_UNORM:
coords[0] = _mesa_unorm_to_float(*(const uint8_t *)(vertex_data + 0), 8);
coords[1] = _mesa_unorm_to_float(*(const uint8_t *)(vertex_data + 1), 8);
coords[2] = _mesa_unorm_to_float(*(const uint8_t *)(vertex_data + 2), 8);
coords[3] = _mesa_unorm_to_float(*(const uint8_t *)(vertex_data + 3), 8);
break;
case VK_FORMAT_A2B10G10R10_UNORM_PACK32: {
uint32_t val = *(const uint32_t *)vertex_data;
coords[0] = _mesa_unorm_to_float((val >> 0) & 0x3FF, 10);
coords[1] = _mesa_unorm_to_float((val >> 10) & 0x3FF, 10);
coords[2] = _mesa_unorm_to_float((val >> 20) & 0x3FF, 10);
coords[3] = _mesa_unorm_to_float((val >> 30) & 0x3, 2);
} break;
default:
unreachable("Unhandled vertex format in BVH build");
}
for (unsigned comp = 0; comp < 3; comp++) {
float r = 0;
for (unsigned col = 0; col < 4; col++)
r += transform_matrix.matrix[comp][col] * coords[col];
node->coords[v][comp] = r;
}
}
primitive_index++;
}
break;
}
case VK_GEOMETRY_TYPE_AABBS_KHR: {
assert(info->type == VK_ACCELERATION_STRUCTURE_TYPE_BOTTOM_LEVEL_KHR);
const uint8_t *data = geom->geometry.aabbs.data.hostAddress;
data += range->primitiveOffset;
VkDeviceSize stride = geom->geometry.aabbs.stride;
for (uint32_t j = 0; j < range->primitiveCount; j++) {
struct lvp_bvh_aabb_node *node = leaf_nodes;
node += primitive_index;
node->primitive_id = j;
node->geometry_id_and_flags = geometry_id_and_flags;
const VkAabbPositionsKHR *aabb = (const VkAabbPositionsKHR *)(data + j * stride);
node->bounds.min.x = aabb->minX;
node->bounds.min.y = aabb->minY;
node->bounds.min.z = aabb->minZ;
node->bounds.max.x = aabb->maxX;
node->bounds.max.y = aabb->maxY;
node->bounds.max.z = aabb->maxZ;
primitive_index++;
}
break;
}
case VK_GEOMETRY_TYPE_INSTANCES_KHR: {
assert(info->type == VK_ACCELERATION_STRUCTURE_TYPE_TOP_LEVEL_KHR);
const uint8_t *data = geom->geometry.instances.data.hostAddress;
data += range->primitiveOffset;
for (uint32_t j = 0; j < range->primitiveCount; j++) {
struct lvp_bvh_instance_node *node = leaf_nodes;
node += primitive_index;
const VkAccelerationStructureInstanceKHR *instance =
geom->geometry.instances.arrayOfPointers
? (((const VkAccelerationStructureInstanceKHR *const *)data)[j])
: &((const VkAccelerationStructureInstanceKHR *)data)[j];
if (!instance->accelerationStructureReference)
continue;
node->bvh_ptr = instance->accelerationStructureReference;
float transform[16], inv_transform[16];
memcpy(transform, &instance->transform.matrix, sizeof(instance->transform.matrix));
transform[12] = transform[13] = transform[14] = 0.0f;
transform[15] = 1.0f;
util_invert_mat4x4(inv_transform, transform);
memcpy(node->wto_matrix.values, inv_transform, sizeof(node->wto_matrix.values));
node->custom_instance_and_mask = instance->instanceCustomIndex | (instance->mask << 24);
node->sbt_offset_and_flags = lvp_pack_sbt_offset_and_flags(
instance->instanceShaderBindingTableRecordOffset, instance->flags);
node->instance_id = j;
memcpy(node->otw_matrix.values, instance->transform.matrix,
sizeof(node->otw_matrix.values));
primitive_index++;
header->instance_count++;
}
break;
}
default:
unreachable("Unknown geometryType");
}
}
leaf_count = primitive_index;
struct lvp_build_internal_ctx internal_ctx = {
.dst = dst,
.dst_offset = sizeof(struct lvp_bvh_header),
.leaf_nodes = leaf_nodes,
.leaf_nodes_offset = header->leaf_nodes_offset,
};
VkGeometryTypeKHR geometry_type = VK_GEOMETRY_TYPE_TRIANGLES_KHR;
if (info->geometryCount) {
if (info->pGeometries)
geometry_type = info->pGeometries[0].geometryType;
else
geometry_type = info->ppGeometries[0]->geometryType;
}
switch (geometry_type) {
case VK_GEOMETRY_TYPE_TRIANGLES_KHR:
internal_ctx.leaf_node_type = lvp_bvh_node_triangle;
internal_ctx.leaf_node_size = sizeof(struct lvp_bvh_triangle_node);
break;
case VK_GEOMETRY_TYPE_AABBS_KHR:
internal_ctx.leaf_node_type = lvp_bvh_node_aabb;
internal_ctx.leaf_node_size = sizeof(struct lvp_bvh_aabb_node);
break;
case VK_GEOMETRY_TYPE_INSTANCES_KHR:
internal_ctx.leaf_node_type = lvp_bvh_node_instance;
internal_ctx.leaf_node_size = sizeof(struct lvp_bvh_instance_node);
break;
default:
unreachable("Unknown VkGeometryTypeKHR");
}
if (leaf_count) {
lvp_build_internal_node(&internal_ctx, 0, leaf_count - 1);
} else {
root->children[0] = LVP_BVH_INVALID_NODE;
root->children[1] = LVP_BVH_INVALID_NODE;
}
header->bounds.min.x = MIN2(root->bounds[0].min.x, root->bounds[1].min.x);
header->bounds.min.y = MIN2(root->bounds[0].min.y, root->bounds[1].min.y);
header->bounds.min.z = MIN2(root->bounds[0].min.z, root->bounds[1].min.z);
header->bounds.max.x = MAX2(root->bounds[0].max.x, root->bounds[1].max.x);
header->bounds.max.y = MAX2(root->bounds[0].max.y, root->bounds[1].max.y);
header->bounds.max.z = MAX2(root->bounds[0].max.z, root->bounds[1].max.z);
header->serialization_size = sizeof(struct lvp_accel_struct_serialization_header) +
sizeof(uint64_t) * header->instance_count + accel_struct->size;
}

108
src/gallium/frontends/lavapipe/lvp_acceleration_structure.h Normal file
View File

@@ -0,0 +1,108 @@
/*
* Copyright © 2021 Google
* Copyright © 2023 Valve Corporation
* SPDX-License-Identifier: MIT
*/
#ifndef LVP_ACCELERATION_STRUCTURE_H
#define LVP_ACCELERATION_STRUCTURE_H
#include "lvp_private.h"
#define LVP_GEOMETRY_OPAQUE (1u << 31)
#define LVP_INSTANCE_FORCE_OPAQUE (1u << 31)
#define LVP_INSTANCE_NO_FORCE_NOT_OPAQUE (1u << 30)
#define LVP_INSTANCE_TRIANGLE_FACING_CULL_DISABLE (1u << 29)
#define LVP_INSTANCE_TRIANGLE_FLIP_FACING (1u << 28)
#define lvp_bvh_node_triangle 0
#define lvp_bvh_node_internal 1
#define lvp_bvh_node_instance 2
#define lvp_bvh_node_aabb 3
typedef struct {
float values[3][4];
} lvp_mat3x4;
typedef struct {
float x;
float y;
float z;
} lvp_vec3;
typedef struct lvp_aabb {
lvp_vec3 min;
lvp_vec3 max;
} lvp_aabb;
struct lvp_bvh_triangle_node {
float coords[3][3];
uint32_t padding;
uint32_t primitive_id;
/* flags in upper 4 bits */
uint32_t geometry_id_and_flags;
};
struct lvp_bvh_aabb_node {
lvp_aabb bounds;
uint32_t primitive_id;
/* flags in upper 4 bits */
uint32_t geometry_id_and_flags;
};
struct lvp_bvh_instance_node {
uint64_t bvh_ptr;
/* lower 24 bits are the custom instance index, upper 8 bits are the visibility mask */
uint32_t custom_instance_and_mask;
/* lower 24 bits are the sbt offset, upper 8 bits are VkGeometryInstanceFlagsKHR */
uint32_t sbt_offset_and_flags;
lvp_mat3x4 wto_matrix;
uint32_t padding;
uint32_t instance_id;
/* Object to world matrix transposed from the initial transform. */
lvp_mat3x4 otw_matrix;
};
struct lvp_bvh_box_node {
lvp_aabb bounds[2];
uint32_t children[2];
};
struct lvp_bvh_header {
lvp_aabb bounds;
uint32_t serialization_size;
uint32_t instance_count;
uint32_t leaf_nodes_offset;
uint32_t padding;
};
struct lvp_accel_struct_serialization_header {
uint8_t driver_uuid[VK_UUID_SIZE];
uint8_t accel_struct_compat[VK_UUID_SIZE];
uint64_t serialization_size;
uint64_t compacted_size;
uint64_t instance_count;
uint64_t instances[];
};
/* The root node is the first node after the header. */
#define LVP_BVH_ROOT_NODE_OFFSET (sizeof(struct lvp_bvh_header))
#define LVP_BVH_ROOT_NODE (LVP_BVH_ROOT_NODE_OFFSET | lvp_bvh_node_internal)
#define LVP_BVH_INVALID_NODE 0xFFFFFFFF
void
lvp_build_acceleration_structure(VkAccelerationStructureBuildGeometryInfoKHR *info,
const VkAccelerationStructureBuildRangeInfoKHR *ranges);
#endif

20
src/gallium/frontends/lavapipe/lvp_cmd_buffer.c
View File

@@ -138,6 +138,9 @@ VKAPI_ATTR void VKAPI_CALL lvp_CmdPushDescriptorSetWithTemplate2KHR(
case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
info_size += sizeof(VkBufferView) * entry->descriptorCount;
break;
case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR:
info_size += sizeof(VkAccelerationStructureKHR) * entry->descriptorCount;
break;
case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
@@ -301,6 +304,23 @@ VKAPI_ATTR void VKAPI_CALL lvp_CmdPushDescriptorSet2KHR(
}
break;
case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR: {
const VkWriteDescriptorSetAccelerationStructureKHR *accel_structs =
vk_find_struct_const(write->pNext, WRITE_DESCRIPTOR_SET_ACCELERATION_STRUCTURE_KHR);
uint32_t accel_structs_size = sizeof(VkAccelerationStructureKHR) * accel_structs->accelerationStructureCount;
VkWriteDescriptorSetAccelerationStructureKHR *write_accel_structs =
rzalloc_size(ctx, sizeof(VkWriteDescriptorSetAccelerationStructureKHR) + accel_structs_size);
write_accel_structs->sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET_ACCELERATION_STRUCTURE_KHR;
write_accel_structs->accelerationStructureCount = accel_structs->accelerationStructureCount;
write_accel_structs->pAccelerationStructures = (void *)&write_accel_structs[1];
memcpy((void *)write_accel_structs->pAccelerationStructures, accel_structs->pAccelerationStructures, accel_structs_size);
dstwrite->pNext = write_accel_structs;
break;
}
default:
break;
}

27
src/gallium/frontends/lavapipe/lvp_descriptor_set.c
View File

@@ -22,6 +22,7 @@
*/
#include "lvp_private.h"
#include "vk_acceleration_structure.h"
#include "vk_descriptors.h"
#include "vk_util.h"
#include "util/u_math.h"
@@ -617,7 +618,18 @@ VKAPI_ATTR void VKAPI_CALL lvp_UpdateDescriptorSets(
}
break;
case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR:
for (uint32_t j = 0; j < write->descriptorCount; j++) {
const VkWriteDescriptorSetAccelerationStructureKHR *accel_structs =
vk_find_struct_const(write->pNext, WRITE_DESCRIPTOR_SET_ACCELERATION_STRUCTURE_KHR);
VK_FROM_HANDLE(vk_acceleration_structure, accel_struct, accel_structs->pAccelerationStructures[j]);
desc[j].accel_struct = accel_struct ? vk_acceleration_structure_get_va(accel_struct) : 0;
}
break;
default:
unreachable("Unsupported descriptor type");
break;
}
}
@@ -802,6 +814,8 @@ lvp_descriptor_update_template_entry_size(VkDescriptorType type)
case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
return sizeof(VkBufferView);
case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR:
return sizeof(VkAccelerationStructureKHR);
case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
@@ -984,7 +998,15 @@ lvp_descriptor_set_update_with_template(VkDevice _device, VkDescriptorSet descri
}
break;
}
case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR: {
VK_FROM_HANDLE(vk_acceleration_structure, accel_struct, *(VkAccelerationStructureKHR *)pSrc);
desc[idx].accel_struct = accel_struct ? vk_acceleration_structure_get_va(accel_struct) : 0;
break;
}
default:
unreachable("Unsupported descriptor type");
break;
}
@@ -1159,7 +1181,12 @@ VKAPI_ATTR void VKAPI_CALL lvp_GetDescriptorEXT(
}
break;
}
case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR: {
desc->accel_struct = pCreateInfo->data.accelerationStructure;
break;
}
default:
unreachable("Unsupported descriptor type");
break;
}
}

130
src/gallium/frontends/lavapipe/lvp_execute.c
View File

@@ -24,6 +24,7 @@
/* use a gallium context to execute a command buffer */
#include "lvp_private.h"
#include "lvp_acceleration_structure.h"
#include "pipe/p_context.h"
#include "pipe/p_state.h"
@@ -3829,7 +3830,7 @@ static void handle_draw_mesh_tasks_indirect_count(struct vk_cmd_queue_entry *cmd
}
static VkBuffer
get_buffer(struct rendering_state *state, uint8_t *ptr, size_t *offset)
get_buffer(struct rendering_state *state, const uint8_t *ptr, size_t *offset)
{
simple_mtx_lock(&state->device->bda_lock);
hash_table_foreach(&state->device->bda, he) {
@@ -4309,6 +4310,109 @@ handle_dispatch_graph(struct vk_cmd_queue_entry *cmd, struct rendering_state *st
}
#endif
static struct pipe_resource *
get_buffer_pipe(struct rendering_state *state, const void *ptr)
{
size_t offset;
VK_FROM_HANDLE(lvp_buffer, buffer, get_buffer(state, ptr, &offset));
return buffer->bo;
}
static void
handle_copy_acceleration_structure(struct vk_cmd_queue_entry *cmd, struct rendering_state *state)
{
struct vk_cmd_copy_acceleration_structure_khr *copy = &cmd->u.copy_acceleration_structure_khr;
VK_FROM_HANDLE(vk_acceleration_structure, src, copy->info->src);
VK_FROM_HANDLE(vk_acceleration_structure, dst, copy->info->dst);
struct pipe_box box = { 0 };
u_box_1d(src->offset, MIN2(src->size, dst->size), &box);
state->pctx->resource_copy_region(state->pctx, lvp_buffer_from_handle(dst->buffer)->bo, 0,
dst->offset, 0, 0,
lvp_buffer_from_handle(src->buffer)->bo, 0, &box);
}
static void
handle_copy_memory_to_acceleration_structure(struct vk_cmd_queue_entry *cmd, struct rendering_state *state)
{
struct vk_cmd_copy_memory_to_acceleration_structure_khr *copy = &cmd->u.copy_memory_to_acceleration_structure_khr;
VK_FROM_HANDLE(vk_acceleration_structure, accel_struct, copy->info->dst);
struct lvp_bvh_header *dst = (void *)(uintptr_t)vk_acceleration_structure_get_va(accel_struct);
const struct lvp_accel_struct_serialization_header *src = copy->info->src.hostAddress;
memcpy(dst, &src->instances[src->instance_count], src->compacted_size);
for (uint32_t i = 0; i < src->instance_count; i++) {
uint8_t *leaf_nodes = (uint8_t *)dst;
leaf_nodes += dst->leaf_nodes_offset;
struct lvp_bvh_instance_node *node = (struct lvp_bvh_instance_node *)leaf_nodes;
node[i].bvh_ptr = src->instances[i];
}
}
static void
handle_copy_acceleration_structure_to_memory(struct vk_cmd_queue_entry *cmd, struct rendering_state *state)
{
struct vk_cmd_copy_acceleration_structure_to_memory_khr *copy = &cmd->u.copy_acceleration_structure_to_memory_khr;
VK_FROM_HANDLE(vk_acceleration_structure, accel_struct, copy->info->src);
struct lvp_bvh_header *src = (void *)(uintptr_t)vk_acceleration_structure_get_va(accel_struct);
struct lvp_accel_struct_serialization_header *dst = copy->info->dst.hostAddress;
lvp_device_get_cache_uuid(dst->driver_uuid);
lvp_device_get_cache_uuid(dst->accel_struct_compat);
dst->serialization_size = src->serialization_size;
dst->compacted_size = accel_struct->size;
dst->instance_count = src->instance_count;
for (uint32_t i = 0; i < src->instance_count; i++) {
uint8_t *leaf_nodes = (uint8_t *)src;
leaf_nodes += src->leaf_nodes_offset;
struct lvp_bvh_instance_node *node = (struct lvp_bvh_instance_node *)leaf_nodes;
dst->instances[i] = node[i].bvh_ptr;
}
memcpy(&dst->instances[dst->instance_count], src, accel_struct->size);
}
static void
handle_build_acceleration_structures(struct vk_cmd_queue_entry *cmd, struct rendering_state *state)
{
struct vk_cmd_build_acceleration_structures_khr *build = &cmd->u.build_acceleration_structures_khr;
for (uint32_t i = 0; i < build->info_count; i++)
lvp_build_acceleration_structure(&build->infos[i], build->pp_build_range_infos[i]);
}
static void
handle_write_acceleration_structures_properties(struct vk_cmd_queue_entry *cmd, struct rendering_state *state)
{
struct vk_cmd_write_acceleration_structures_properties_khr *write = &cmd->u.write_acceleration_structures_properties_khr;
VK_FROM_HANDLE(lvp_query_pool, pool, write->query_pool);
uint64_t *dst = pool->data;
dst += write->first_query;
for (uint32_t i = 0; i < write->acceleration_structure_count; i++) {
VK_FROM_HANDLE(vk_acceleration_structure, accel_struct, write->acceleration_structures[i]);
if (write->query_type == VK_QUERY_TYPE_ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR) {
dst[i] = accel_struct->size;
continue;
}
assert (write->query_type == VK_QUERY_TYPE_ACCELERATION_STRUCTURE_SERIALIZATION_SIZE_KHR);
struct lvp_bvh_header *header = (void *)(uintptr_t)vk_acceleration_structure_get_va(accel_struct);
dst[i] = header->serialization_size;
}
}
void lvp_add_enqueue_cmd_entrypoints(struct vk_device_dispatch_table *disp)
{
struct vk_device_dispatch_table cmd_enqueue_dispatch;
@@ -4449,6 +4553,13 @@ void lvp_add_enqueue_cmd_entrypoints(struct vk_device_dispatch_table *disp)
ENQUEUE_CMD(CmdSetRenderingAttachmentLocationsKHR)
ENQUEUE_CMD(CmdSetRenderingInputAttachmentIndicesKHR)
ENQUEUE_CMD(CmdCopyAccelerationStructureKHR)
ENQUEUE_CMD(CmdCopyMemoryToAccelerationStructureKHR)
ENQUEUE_CMD(CmdCopyAccelerationStructureToMemoryKHR)
ENQUEUE_CMD(CmdBuildAccelerationStructuresKHR)
ENQUEUE_CMD(CmdBuildAccelerationStructuresIndirectKHR)
ENQUEUE_CMD(CmdWriteAccelerationStructuresPropertiesKHR)
#undef ENQUEUE_CMD
}
@@ -4815,6 +4926,23 @@ static void lvp_execute_cmd_buffer(struct list_head *cmds,
case VK_CMD_SET_RENDERING_INPUT_ATTACHMENT_INDICES_KHR:
handle_rendering_input_attachment_indices(cmd, state);
break;
case VK_CMD_COPY_ACCELERATION_STRUCTURE_KHR:
handle_copy_acceleration_structure(cmd, state);
break;
case VK_CMD_COPY_MEMORY_TO_ACCELERATION_STRUCTURE_KHR:
handle_copy_memory_to_acceleration_structure(cmd, state);
break;
case VK_CMD_COPY_ACCELERATION_STRUCTURE_TO_MEMORY_KHR:
handle_copy_acceleration_structure_to_memory(cmd, state);
break;
case VK_CMD_BUILD_ACCELERATION_STRUCTURES_KHR:
handle_build_acceleration_structures(cmd, state);
break;
case VK_CMD_BUILD_ACCELERATION_STRUCTURES_INDIRECT_KHR:
break;
case VK_CMD_WRITE_ACCELERATION_STRUCTURES_PROPERTIES_KHR:
handle_write_acceleration_structures_properties(cmd, state);
break;
default:
fprintf(stderr, "Unsupported command %s\n", vk_cmd_queue_type_names[cmd->type]);
unreachable("Unsupported command");

5
src/gallium/frontends/lavapipe/lvp_private.h
View File

@@ -56,6 +56,7 @@ typedef uint32_t xcb_window_t;
#include <vulkan/vk_icd.h>
#include "lvp_entrypoints.h"
#include "vk_acceleration_structure.h"
#include "vk_buffer.h"
#include "vk_buffer_view.h"
#include "vk_device.h"
@@ -581,12 +582,16 @@ struct lvp_buffer_view {
struct lp_texture_handle *image_handle;
};
#define LVP_QUERY_ACCELERATION_STRUCTURE_COMPACTED_SIZE (PIPE_QUERY_TYPES)
#define LVP_QUERY_ACCELERATION_STRUCTURE_SERIALIZATION_SIZE (PIPE_QUERY_TYPES + 1)
struct lvp_query_pool {
struct vk_object_base base;
VkQueryType type;
uint32_t count;
VkQueryPipelineStatisticFlags pipeline_stats;
enum pipe_query_type base_type;
void *data; /* Used by queries that are not implemented by pipe_query */
struct pipe_query *queries[0];
};

39
src/gallium/frontends/lavapipe/lvp_query.c
View File

@@ -32,6 +32,7 @@ VKAPI_ATTR VkResult VKAPI_CALL lvp_CreateQueryPool(
{
LVP_FROM_HANDLE(lvp_device, device, _device);
uint32_t query_size = sizeof(struct pipe_query *);
enum pipe_query_type pipeq;
switch (pCreateInfo->queryType) {
case VK_QUERY_TYPE_OCCLUSION:
@@ -50,13 +51,21 @@ VKAPI_ATTR VkResult VKAPI_CALL lvp_CreateQueryPool(
case VK_QUERY_TYPE_MESH_PRIMITIVES_GENERATED_EXT:
pipeq = PIPE_QUERY_PRIMITIVES_GENERATED;
break;
case VK_QUERY_TYPE_ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR:
query_size = sizeof(uint64_t);
pipeq = LVP_QUERY_ACCELERATION_STRUCTURE_COMPACTED_SIZE;
break;
case VK_QUERY_TYPE_ACCELERATION_STRUCTURE_SERIALIZATION_SIZE_KHR:
query_size = sizeof(uint64_t);
pipeq = LVP_QUERY_ACCELERATION_STRUCTURE_SERIALIZATION_SIZE;
break;
default:
return VK_ERROR_FEATURE_NOT_PRESENT;
}
struct lvp_query_pool *pool;
size_t pool_size = sizeof(*pool)
+ pCreateInfo->queryCount * sizeof(struct pipe_query *);
+ pCreateInfo->queryCount * query_size;
pool = vk_zalloc2(&device->vk.alloc, pAllocator,
pool_size, 8,
@@ -70,6 +79,7 @@ VKAPI_ATTR VkResult VKAPI_CALL lvp_CreateQueryPool(
pool->count = pCreateInfo->queryCount;
pool->base_type = pipeq;
pool->pipeline_stats = pCreateInfo->pipelineStatistics;
pool->data = &pool->queries;
*pQueryPool = lvp_query_pool_to_handle(pool);
return VK_SUCCESS;
@@ -86,9 +96,12 @@ VKAPI_ATTR void VKAPI_CALL lvp_DestroyQueryPool(
if (!pool)
return;
for (unsigned i = 0; i < pool->count; i++)
if (pool->queries[i])
device->queue.ctx->destroy_query(device->queue.ctx, pool->queries[i]);
if (pool->base_type != LVP_QUERY_ACCELERATION_STRUCTURE_COMPACTED_SIZE &&
pool->base_type != LVP_QUERY_ACCELERATION_STRUCTURE_SERIALIZATION_SIZE) {
for (unsigned i = 0; i < pool->count; i++)
if (pool->queries[i])
device->queue.ctx->destroy_query(device->queue.ctx, pool->queries[i]);
}
vk_object_base_finish(&pool->base);
vk_free2(&device->vk.alloc, pAllocator, pool);
}
@@ -114,6 +127,20 @@ VKAPI_ATTR VkResult VKAPI_CALL lvp_GetQueryPoolResults(
union pipe_query_result result;
bool ready = false;
if (pool->base_type == LVP_QUERY_ACCELERATION_STRUCTURE_COMPACTED_SIZE ||
pool->base_type == LVP_QUERY_ACCELERATION_STRUCTURE_SERIALIZATION_SIZE) {
if (flags & VK_QUERY_RESULT_64_BIT) {
uint64_t *dst = (uint64_t *)dest;
uint64_t *src = (uint64_t *)pool->data;
*dst = src[i];
} else {
uint32_t *dst = (uint32_t *)dest;
uint64_t *src = (uint64_t *)pool->data;
*dst = src[i];
}
continue;
}
if (pool->queries[i]) {
ready = device->queue.ctx->get_query_result(device->queue.ctx,
pool->queries[i],
@@ -194,6 +221,10 @@ VKAPI_ATTR void VKAPI_CALL lvp_ResetQueryPool(
LVP_FROM_HANDLE(lvp_device, device, _device);
LVP_FROM_HANDLE(lvp_query_pool, pool, queryPool);
if (pool->base_type == LVP_QUERY_ACCELERATION_STRUCTURE_COMPACTED_SIZE ||
pool->base_type == LVP_QUERY_ACCELERATION_STRUCTURE_SERIALIZATION_SIZE)
return;
for (uint32_t i = 0; i < queryCount; i++) {
uint32_t idx = i + firstQuery;

1
src/gallium/frontends/lavapipe/meson.build
View File

@@ -12,6 +12,7 @@ lvp_entrypoints = custom_target(
)
liblvp_files = files(
'lvp_acceleration_structure.c',
'lvp_device.c',
'lvp_cmd_buffer.c',
'lvp_descriptor_set.c',