gfxstream: guest: enable virtio-gpu kumquat

This uses libvirtgpu_kumquat_ffi.so to send commands to the
Kumquat Media Server.  The implementation of Kumquat is found
at crrev.com/c/5645904.

This is a more complex, but more fully featured end-to-end testing
framework.  This biggest benefit it is can run complex apps (vkcube,
gfxbench + ANGLE/Zink) on Linux, without a VM or a full Android tree.
The rutabaga FFI path -- which relied on a nested Vulkan loader -- didn't
work for complex apps and funkiness was observed with the nested
Vulkan loader.

Plus, app 1 + app 2 could connect Kumquat at the same time.

This may also benefit snapshot tests, particular since end to end
flow relies on external blob.  For example, a snapshot save command
could actually call stream_renderer_teardown(..) on the host-side,
while the guest retains it's state (ASG mappings + vulkano mappings).

A snapshot restore would actually call stream_renderer_init(..)
+ stream_renderer_restore(..), which an actual upstream VMM would
do.

Another additional benefit is multi-context testing.  Since
libvirtgpu_kumquat_ffi.so is portable, other libraries (minigbm)
can also call it.  We can mimic the cross-domain context allocating
via libminigbm.so, while context libgfxstream_vulkan.so imports the
file descriptor.

This change only transitions the meson build since the main goal
is once again is Linux on Linux testing w/o an Android tree.

Reviewed-by: Aaron Ruby <aruby@blackberry.com>
Acked-by: Yonggang Luo <luoyonggang@gmail.com>
Acked-by: Adam Jackson <ajax@redhat.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/27246>
This commit is contained in:
Gurchetan Singh
2024-06-11 10:31:17 -07:00
committed by Marge Bot
parent d02b246fbc
commit a5788932c9
8 changed files with 578 additions and 1 deletions

View File

@@ -0,0 +1,83 @@
/*
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "VirtGpu.h"
#include "virtgpu_kumquat/virtgpu_kumquat_ffi.h"
class VirtGpuKumquatResource : public std::enable_shared_from_this<VirtGpuKumquatResource>,
public VirtGpuResource {
public:
VirtGpuKumquatResource(struct virtgpu_kumquat* virtGpu, uint32_t blobHandle,
uint32_t resourceHandle, uint64_t size);
~VirtGpuKumquatResource();
uint32_t getResourceHandle() const override;
uint32_t getBlobHandle() const override;
int wait() override;
VirtGpuResourceMappingPtr createMapping(void) override;
int exportBlob(struct VirtGpuExternalHandle& handle) override;
int transferFromHost(uint32_t x, uint32_t y, uint32_t w, uint32_t h) override;
int transferToHost(uint32_t x, uint32_t y, uint32_t w, uint32_t h) override;
private:
// Not owned. Really should use a ScopedFD for this, but doesn't matter since we have a
// singleton deviceimplemenentation anyways.
struct virtgpu_kumquat* mVirtGpu = nullptr;
;
uint32_t mBlobHandle;
uint32_t mResourceHandle;
uint64_t mSize;
};
class VirtGpuKumquatResourceMapping : public VirtGpuResourceMapping {
public:
VirtGpuKumquatResourceMapping(VirtGpuResourcePtr blob, uint8_t* ptr, uint64_t size);
~VirtGpuKumquatResourceMapping(void);
uint8_t* asRawPtr(void) override;
private:
VirtGpuResourcePtr mBlob;
uint8_t* mPtr;
uint64_t mSize;
};
class VirtGpuKumquatDevice : public VirtGpuDevice {
public:
VirtGpuKumquatDevice(enum VirtGpuCapset capset, int fd = -1);
virtual ~VirtGpuKumquatDevice();
virtual int64_t getDeviceHandle(void);
virtual struct VirtGpuCaps getCaps(void);
VirtGpuResourcePtr createBlob(const struct VirtGpuCreateBlob& blobCreate) override;
VirtGpuResourcePtr createResource(uint32_t width, uint32_t height, uint32_t virglFormat,
uint32_t target, uint32_t bind, uint32_t bpp) override;
virtual VirtGpuResourcePtr importBlob(const struct VirtGpuExternalHandle& handle);
virtual int execBuffer(struct VirtGpuExecBuffer& execbuffer, const VirtGpuResource* blob);
private:
struct virtgpu_kumquat* mVirtGpu = nullptr;
;
struct VirtGpuCaps mCaps;
};

View File

@@ -0,0 +1,135 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <cutils/log.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <unistd.h>
#include <cerrno>
#include <cstring>
#include "VirtGpuKumquat.h"
#include "virtgpu_kumquat/virtgpu_kumquat_ffi.h"
VirtGpuKumquatResource::VirtGpuKumquatResource(struct virtgpu_kumquat* virtGpu, uint32_t blobHandle,
uint32_t resourceHandle, uint64_t size)
: mVirtGpu(virtGpu), mBlobHandle(blobHandle), mResourceHandle(resourceHandle), mSize(size) {}
VirtGpuKumquatResource::~VirtGpuKumquatResource() {
struct drm_kumquat_resource_unref unref {
.bo_handle = mBlobHandle, .pad = 0,
};
int ret = virtgpu_kumquat_resource_unref(mVirtGpu, &unref);
if (ret) {
ALOGE("Closed failed with : [%s, blobHandle %u, resourceHandle: %u]", strerror(errno),
mBlobHandle, mResourceHandle);
}
}
uint32_t VirtGpuKumquatResource::getBlobHandle() const { return mBlobHandle; }
uint32_t VirtGpuKumquatResource::getResourceHandle() const { return mResourceHandle; }
VirtGpuResourceMappingPtr VirtGpuKumquatResource::createMapping() {
int ret;
struct drm_kumquat_map map {
.bo_handle = mBlobHandle, .ptr = NULL, .size = mSize,
};
ret = virtgpu_kumquat_resource_map(mVirtGpu, &map);
if (ret < 0) {
ALOGE("Mapping failed with %s", strerror(errno));
return nullptr;
}
return std::make_shared<VirtGpuKumquatResourceMapping>(shared_from_this(), (uint8_t*)map.ptr,
mSize);
}
int VirtGpuKumquatResource::exportBlob(struct VirtGpuExternalHandle& handle) {
int ret;
struct drm_kumquat_resource_export exp = {0};
exp.bo_handle = mBlobHandle;
ret = virtgpu_kumquat_resource_export(mVirtGpu, &exp);
if (ret) {
ALOGE("Failed to export blob with %s", strerror(errno));
return ret;
}
handle.osHandle = static_cast<int64_t>(exp.os_handle);
handle.type = static_cast<VirtGpuHandleType>(exp.handle_type);
return 0;
}
int VirtGpuKumquatResource::wait() {
int ret;
struct drm_kumquat_wait wait = {
.handle = mBlobHandle,
.flags = 0,
};
ret = virtgpu_kumquat_wait(mVirtGpu, &wait);
if (ret < 0) {
ALOGE("Wait failed with %s", strerror(errno));
return ret;
}
return 0;
}
int VirtGpuKumquatResource::transferToHost(uint32_t x, uint32_t y, uint32_t w, uint32_t h) {
int ret;
struct drm_kumquat_transfer_to_host xfer = {0};
xfer.box.x = x;
xfer.box.y = y;
xfer.box.w = w;
xfer.box.h = h;
xfer.box.d = 1;
xfer.bo_handle = mBlobHandle;
ret = virtgpu_kumquat_transfer_to_host(mVirtGpu, &xfer);
if (ret < 0) {
ALOGE("Transfer to host failed with %s", strerror(errno));
return ret;
}
return 0;
}
int VirtGpuKumquatResource::transferFromHost(uint32_t x, uint32_t y, uint32_t w, uint32_t h) {
int ret;
struct drm_kumquat_transfer_from_host xfer = {0};
xfer.box.x = x;
xfer.box.y = y;
xfer.box.w = w;
xfer.box.h = h;
xfer.box.d = 1;
xfer.bo_handle = mBlobHandle;
ret = virtgpu_kumquat_transfer_from_host(mVirtGpu, &xfer);
if (ret < 0) {
ALOGE("Transfer from host failed with %s", strerror(errno));
return ret;
}
return 0;
}

View File

@@ -0,0 +1,25 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "VirtGpuKumquat.h"
VirtGpuKumquatResourceMapping::VirtGpuKumquatResourceMapping(VirtGpuResourcePtr blob, uint8_t* ptr,
uint64_t size)
: mBlob(blob), mPtr(ptr), mSize(size) {}
VirtGpuKumquatResourceMapping::~VirtGpuKumquatResourceMapping(void) { return; }
uint8_t* VirtGpuKumquatResourceMapping::asRawPtr(void) { return mPtr; }

View File

@@ -0,0 +1,247 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <cutils/log.h>
#include <pthread.h>
#include <stdlib.h>
#include <unistd.h>
#include <cerrno>
#include <cstring>
#include <fstream>
#include <string>
#include "VirtGpuKumquat.h"
#include "virtgpu_gfxstream_protocol.h"
#include "virtgpu_kumquat/virtgpu_kumquat_ffi.h"
#define PARAM(x) \
(struct VirtGpuParam) { x, #x, 0 }
static inline uint32_t align_up(uint32_t n, uint32_t a) { return ((n + a - 1) / a) * a; }
VirtGpuKumquatDevice::VirtGpuKumquatDevice(enum VirtGpuCapset capset, int fd)
: VirtGpuDevice(capset) {
struct VirtGpuParam params[] = {
PARAM(VIRTGPU_KUMQUAT_PARAM_3D_FEATURES),
PARAM(VIRTGPU_KUMQUAT_PARAM_CAPSET_QUERY_FIX),
PARAM(VIRTGPU_KUMQUAT_PARAM_RESOURCE_BLOB),
PARAM(VIRTGPU_KUMQUAT_PARAM_HOST_VISIBLE),
PARAM(VIRTGPU_KUMQUAT_PARAM_CROSS_DEVICE),
PARAM(VIRTGPU_KUMQUAT_PARAM_CONTEXT_INIT),
PARAM(VIRTGPU_KUMQUAT_PARAM_SUPPORTED_CAPSET_IDs),
PARAM(VIRTGPU_KUMQUAT_PARAM_EXPLICIT_DEBUG_NAME),
PARAM(VIRTGPU_KUMQUAT_PARAM_CREATE_GUEST_HANDLE),
};
int ret;
struct drm_kumquat_get_caps get_caps = {0};
struct drm_kumquat_context_init init = {0};
struct drm_kumquat_context_set_param ctx_set_params[3] = {{0}};
const char* processName = nullptr;
memset(&mCaps, 0, sizeof(struct VirtGpuCaps));
#ifdef __ANDROID__
processName = getprogname();
#endif
ret = virtgpu_kumquat_init(&mVirtGpu);
if (ret) {
ALOGV("Failed to init virtgpu kumquat");
return;
}
for (uint32_t i = 0; i < kParamMax; i++) {
struct drm_kumquat_getparam get_param = {0};
get_param.param = params[i].param;
get_param.value = (uint64_t)(uintptr_t)&params[i].value;
ret = virtgpu_kumquat_get_param(mVirtGpu, &get_param);
if (ret) {
ALOGV("virtgpu backend not enabling %s", params[i].name);
continue;
}
mCaps.params[i] = params[i].value;
}
get_caps.cap_set_id = static_cast<uint32_t>(capset);
switch (capset) {
case kCapsetGfxStreamVulkan:
get_caps.size = sizeof(struct vulkanCapset);
get_caps.addr = (unsigned long long)&mCaps.vulkanCapset;
break;
case kCapsetGfxStreamMagma:
get_caps.size = sizeof(struct magmaCapset);
get_caps.addr = (unsigned long long)&mCaps.magmaCapset;
break;
case kCapsetGfxStreamGles:
get_caps.size = sizeof(struct vulkanCapset);
get_caps.addr = (unsigned long long)&mCaps.glesCapset;
break;
case kCapsetGfxStreamComposer:
get_caps.size = sizeof(struct vulkanCapset);
get_caps.addr = (unsigned long long)&mCaps.composerCapset;
break;
default:
get_caps.size = 0;
}
ret = virtgpu_kumquat_get_caps(mVirtGpu, &get_caps);
if (ret) {
// Don't fail get capabilities just yet, AEMU doesn't use this API
// yet (b/272121235);
ALOGE("DRM_IOCTL_VIRTGPU_KUMQUAT_GET_CAPS failed with %s", strerror(errno));
}
// We always need an ASG blob in some cases, so always define blobAlignment
if (!mCaps.vulkanCapset.blobAlignment) {
mCaps.vulkanCapset.blobAlignment = 4096;
}
ctx_set_params[0].param = VIRTGPU_KUMQUAT_CONTEXT_PARAM_NUM_RINGS;
ctx_set_params[0].value = 2;
init.num_params = 1;
if (capset != kCapsetNone) {
ctx_set_params[init.num_params].param = VIRTGPU_KUMQUAT_CONTEXT_PARAM_CAPSET_ID;
ctx_set_params[init.num_params].value = static_cast<uint32_t>(capset);
init.num_params++;
}
if (mCaps.params[kParamExplicitDebugName] && processName) {
ctx_set_params[init.num_params].param = VIRTGPU_KUMQUAT_CONTEXT_PARAM_DEBUG_NAME;
ctx_set_params[init.num_params].value = reinterpret_cast<uint64_t>(processName);
init.num_params++;
}
init.ctx_set_params = (unsigned long long)&ctx_set_params[0];
ret = virtgpu_kumquat_context_init(mVirtGpu, &init);
if (ret) {
ALOGE(
"DRM_IOCTL_VIRTGPU_KUMQUAT_CONTEXT_INIT failed with %s, continuing without context...",
strerror(errno));
}
}
VirtGpuKumquatDevice::~VirtGpuKumquatDevice() { virtgpu_kumquat_finish(&mVirtGpu); }
struct VirtGpuCaps VirtGpuKumquatDevice::getCaps(void) { return mCaps; }
int64_t VirtGpuKumquatDevice::getDeviceHandle(void) { return -1; }
VirtGpuResourcePtr VirtGpuKumquatDevice::createResource(uint32_t width, uint32_t height,
uint32_t virglFormat, uint32_t target,
uint32_t bind, uint32_t bpp) {
struct drm_kumquat_resource_create_3d create = {
.target = target,
.format = virglFormat,
.bind = bind,
.width = width,
.height = height,
.depth = 1U,
.array_size = 1U,
.last_level = 0,
.nr_samples = 0,
.size = width * height * bpp,
.stride = width * bpp,
};
int ret = virtgpu_kumquat_resource_create_3d(mVirtGpu, &create);
if (ret) {
ALOGE("DRM_IOCTL_VIRTGPU_KUMQUAT_RESOURCE_CREATE failed with %s", strerror(errno));
return nullptr;
}
return std::make_shared<VirtGpuKumquatResource>(mVirtGpu, create.bo_handle, create.res_handle,
static_cast<uint64_t>(create.size));
}
VirtGpuResourcePtr VirtGpuKumquatDevice::createBlob(const struct VirtGpuCreateBlob& blobCreate) {
int ret;
struct drm_kumquat_resource_create_blob create = {0};
create.size = blobCreate.size;
create.blob_mem = blobCreate.blobMem;
create.blob_flags = blobCreate.flags;
create.blob_id = blobCreate.blobId;
create.cmd = (uint64_t)(uintptr_t)blobCreate.blobCmd;
create.cmd_size = blobCreate.blobCmdSize;
ret = virtgpu_kumquat_resource_create_blob(mVirtGpu, &create);
if (ret < 0) {
ALOGE("DRM_VIRTGPU_KUMQUAT_RESOURCE_CREATE_BLOB failed with %s", strerror(errno));
return nullptr;
}
return std::make_shared<VirtGpuKumquatResource>(mVirtGpu, create.bo_handle, create.res_handle,
blobCreate.size);
}
VirtGpuResourcePtr VirtGpuKumquatDevice::importBlob(const struct VirtGpuExternalHandle& handle) {
int ret;
struct drm_kumquat_resource_import resource_import = {0};
resource_import.os_handle = static_cast<uint64_t>(handle.osHandle);
resource_import.handle_type = static_cast<uint32_t>(handle.type);
ret = virtgpu_kumquat_resource_import(mVirtGpu, &resource_import);
if (ret < 0) {
ALOGE("DRM_VIRTGPU_KUMQUAT_RESOURCE_IMPORT failed with %s", strerror(errno));
return nullptr;
}
return std::make_shared<VirtGpuKumquatResource>(
mVirtGpu, resource_import.bo_handle, resource_import.res_handle, resource_import.size);
return nullptr;
}
int VirtGpuKumquatDevice::execBuffer(struct VirtGpuExecBuffer& execbuffer,
const VirtGpuResource* blob) {
int ret;
struct drm_kumquat_execbuffer exec = {0};
uint32_t blobHandle;
exec.flags = execbuffer.flags;
exec.size = execbuffer.command_size;
exec.ring_idx = execbuffer.ring_idx;
exec.command = (uint64_t)(uintptr_t)(execbuffer.command);
exec.fence_fd = -1;
if (blob) {
blobHandle = blob->getBlobHandle();
exec.bo_handles = (uint64_t)(uintptr_t)(&blobHandle);
exec.num_bo_handles = 1;
}
ret = virtgpu_kumquat_execbuffer(mVirtGpu, &exec);
if (ret) {
ALOGE("DRM_IOCTL_VIRTGPU_KUMQUAT_EXECBUFFER failed: %s", strerror(errno));
return ret;
}
if (execbuffer.flags & kFenceOut) {
execbuffer.handle.osHandle = exec.fence_fd;
execbuffer.handle.type = kFenceHandleSyncFd;
}
return 0;
}
VirtGpuDevice* createPlatformVirtGpuDevice(enum VirtGpuCapset capset, int fd) {
return new VirtGpuKumquatDevice(capset, fd);
}

View File

@@ -0,0 +1,37 @@
/*
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "VirtGpuKumquatSync.h"
#include <unistd.h>
namespace gfxstream {
VirtGpuKumquatSyncHelper::VirtGpuKumquatSyncHelper() {}
int VirtGpuKumquatSyncHelper::wait(int syncFd, int timeoutMilliseconds) {
(void)syncFd;
(void)timeoutMilliseconds;
return -1;
}
int VirtGpuKumquatSyncHelper::dup(int syncFd) { return ::dup(syncFd); }
int VirtGpuKumquatSyncHelper::close(int syncFd) { return ::close(syncFd); }
SyncHelper* createPlatformSyncHelper() { return new VirtGpuKumquatSyncHelper(); }
} // namespace gfxstream

View File

@@ -0,0 +1,32 @@
// Copyright 2023 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either expresso or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "Sync.h"
namespace gfxstream {
class VirtGpuKumquatSyncHelper : public SyncHelper {
public:
VirtGpuKumquatSyncHelper();
int wait(int syncFd, int timeoutMilliseconds) override;
int dup(int syncFd) override;
int close(int syncFd) override;
};
} // namespace gfxstream

View File

@@ -0,0 +1,18 @@
# Copyright 2022 Android Open Source Project
# SPDX-License-Identifier: MIT
files_lib_platform = files(
'../VirtGpu.cpp',
'VirtGpuKumquatDevice.cpp',
'VirtGpuKumquatBlobMapping.cpp',
'VirtGpuKumquatBlob.cpp',
'VirtGpuKumquatSync.cpp',
)
lib_platform = static_library(
'platform',
files_lib_platform,
cpp_args: gfxstream_guest_args,
include_directories: [inc_platform, inc_android_compat],
dependencies: virtgpu_kumquat_dep,
)

View File

@@ -3,7 +3,7 @@
inc_platform = include_directories('include')
if with_guest_test
subdir('rutabaga')
subdir('kumquat')
else
subdir('linux')
subdir('stub')