anv/image: Add ANV_IMAGE_MEMORY_BINDING_PRIVATE

Used for storing the aux surface and/or aux state data for some images
that have a DRM format modifier.

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/1466>

src/intel/vulkan/anv_formats.c

@@ -1102,6 +1102,23 @@ anv_get_image_format_properties(
       }
    }
 
+   if (info->flags & VK_IMAGE_CREATE_ALIAS_BIT) {
+      /* Reject aliasing of images with non-linear DRM format modifiers because:
+       *
+       * 1. For modifiers with compression, we store aux tracking state in
+       *    ANV_IMAGE_MEMORY_BINDING_PRIVATE, which is not aliasable because it's
+       *    not client-bound.
+       *
+       * 2. For tiled modifiers without compression, we may attempt to compress
+       *    them behind the scenes, in which case both the aux tracking state
+       *    and the CCS data are bound to ANV_IMAGE_MEMORY_BINDING_PRIVATE.
+       */
+      if (info->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT &&
+          isl_mod_info->modifier != DRM_FORMAT_MOD_LINEAR) {
+         goto unsupported;
+      }
+   }
+
    if (info->usage & VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT) {
       /* Nothing to check. */
    }

src/intel/vulkan/anv_image.c

@@ -76,6 +76,8 @@ image_binding_grow(struct anv_image *image,
       if (!image->disjoint)
          binding = ANV_IMAGE_MEMORY_BINDING_MAIN;
       break;
+   case ANV_IMAGE_MEMORY_BINDING_PRIVATE:
+      break;
    case ANV_IMAGE_MEMORY_BINDING_END:
       unreachable("ANV_IMAGE_MEMORY_BINDING_END");
    }
@@ -396,12 +398,17 @@ add_aux_state_tracking_buffer(struct anv_device *device,
       }
    }
 
+   enum anv_image_memory_binding binding =
+      ANV_IMAGE_MEMORY_BINDING_PLANE_0 + plane;
+
+   if (image->drm_format_mod != DRM_FORMAT_MOD_INVALID)
+       binding = ANV_IMAGE_MEMORY_BINDING_PRIVATE;
+
    /* We believe that 256B alignment may be sufficient, but we choose 4K due to
     * lack of testing.  And MI_LOAD/STORE operations require dword-alignment.
     */
    image->planes[plane].fast_clear_memory_range =
-      image_binding_grow(image, ANV_IMAGE_MEMORY_BINDING_PLANE_0 + plane,
+      image_binding_grow(image, binding, state_size, 4096);
-                         state_size, 4096);
 }
 
 /**
@@ -509,6 +516,11 @@ add_aux_surface_if_supported(struct anv_device *device,
       if ((image->create_flags & VK_IMAGE_CREATE_ALIAS_BIT)) {
          /* The image may alias a plane of a multiplanar image. Above we ban
           * CCS on multiplanar images.
+          *
+          * We must also reject aliasing of any image that uses
+          * ANV_IMAGE_MEMORY_BINDING_PRIVATE. Since we're already rejecting all
+          * aliasing here, there's no need to further analyze if the image needs
+          * a private binding.
           */
          return VK_SUCCESS;
       }
@@ -575,9 +587,16 @@ add_aux_surface_if_supported(struct anv_device *device,
          image->planes[plane].aux_usage = ISL_AUX_USAGE_CCS_D;
       }
 
-      if (!device->physical->has_implicit_ccs)
+      if (!device->physical->has_implicit_ccs) {
-         add_surface(image, ANV_IMAGE_MEMORY_BINDING_PLANE_0 + plane,
+         enum anv_image_memory_binding binding =
-                     &image->planes[plane].aux_surface);
+            ANV_IMAGE_MEMORY_BINDING_PLANE_0 + plane;
+
+         if (image->drm_format_mod != DRM_FORMAT_MOD_INVALID &&
+             !isl_drm_modifier_has_aux(image->drm_format_mod))
+            binding = ANV_IMAGE_MEMORY_BINDING_PRIVATE;
+
+         add_surface(image, binding, &image->planes[plane].aux_surface);
+      }
 
       add_aux_state_tracking_buffer(device, image, plane);
    } else if ((aspect & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) && image->samples > 1) {
@@ -746,6 +765,12 @@ check_memory_bindings(const struct anv_device *device,
          ? ANV_IMAGE_MEMORY_BINDING_PLANE_0 + p
          : ANV_IMAGE_MEMORY_BINDING_MAIN;
 
+      /* Aliasing is incompatible with the private binding because it does not
+       * live in a VkDeviceMemory.
+       */
+      assert(!(image->create_flags & VK_IMAGE_CREATE_ALIAS_BIT) ||
+             image->bindings[ANV_IMAGE_MEMORY_BINDING_PRIVATE].memory_range.size == 0);
+
       /* Check primary surface */
       check_memory_range(accum_ranges,
                          .test_surface = &plane->primary_surface,
@@ -760,6 +785,12 @@ check_memory_bindings(const struct anv_device *device,
 
       /* Check aux_surface */
       if (anv_surface_is_valid(&plane->aux_surface)) {
+         enum anv_image_memory_binding binding = primary_binding;
+
+         if (image->drm_format_mod != DRM_FORMAT_MOD_INVALID &&
+             !isl_drm_modifier_has_aux(image->drm_format_mod))
+            binding = ANV_IMAGE_MEMORY_BINDING_PRIVATE;
+
          /* Display hardware requires that the aux surface start at
           * a higher address than the primary surface. The 3D hardware
           * doesn't care, but we enforce the display requirement in case
@@ -767,7 +798,7 @@ check_memory_bindings(const struct anv_device *device,
           */
          check_memory_range(accum_ranges,
                             .test_surface = &plane->aux_surface,
-                            .expect_binding = primary_binding);
+                            .expect_binding = binding);
       }
 
       /* Check fast clear state */
@@ -776,6 +807,11 @@ check_memory_bindings(const struct anv_device *device,
               image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV));
 
       if (plane->fast_clear_memory_range.size > 0) {
+         enum anv_image_memory_binding binding = primary_binding;
+
+         if (image->drm_format_mod != DRM_FORMAT_MOD_INVALID)
+            binding = ANV_IMAGE_MEMORY_BINDING_PRIVATE;
+
          /* We believe that 256B alignment may be sufficient, but we choose 4K
           * due to lack of testing.  And MI_LOAD/STORE operations require
           * dword-alignment.
@@ -783,7 +819,7 @@ check_memory_bindings(const struct anv_device *device,
          assert(plane->fast_clear_memory_range.alignment == 4096);
          check_memory_range(accum_ranges,
                             .test_range = &plane->fast_clear_memory_range,
-                            .expect_binding = primary_binding);
+                            .expect_binding = binding);
       }
    }
 #endif
@@ -969,6 +1005,30 @@ anv_image_create_usage(const VkImageCreateInfo *pCreateInfo,
    return usage;
 }
 
+static VkResult MUST_CHECK
+alloc_private_binding(struct anv_device *device,
+                      struct anv_image *image,
+                      const VkImageCreateInfo *create_info)
+{
+   struct anv_image_binding *binding =
+      &image->bindings[ANV_IMAGE_MEMORY_BINDING_PRIVATE];
+
+   if (binding->memory_range.size == 0)
+      return VK_SUCCESS;
+
+   const VkImageSwapchainCreateInfoKHR *swapchain_info =
+      vk_find_struct_const(create_info->pNext, IMAGE_SWAPCHAIN_CREATE_INFO_KHR);
+
+   if (swapchain_info && swapchain_info->swapchain != VK_NULL_HANDLE) {
+      /* The image will be bound to swapchain memory. */
+      return VK_SUCCESS;
+   }
+
+   return anv_device_alloc_bo(device, "image-binding-private",
+                              binding->memory_range.size, 0, 0,
+                              &binding->address.bo);
+}
+
 VkResult
 anv_image_create(VkDevice _device,
                  const struct anv_image_create_info *create_info,
@@ -1075,6 +1135,10 @@ anv_image_create(VkDevice _device,
    if (r != VK_SUCCESS)
       goto fail;
 
+   r = alloc_private_binding(device, image, pCreateInfo);
+   if (r != VK_SUCCESS)
+      goto fail;
+
    r = check_drm_format_mod(device, image);
    if (r != VK_SUCCESS)
       goto fail;
@@ -1224,6 +1288,10 @@ anv_DestroyImage(VkDevice _device, VkImage _image,
       anv_device_release_bo(device, image->bindings[ANV_IMAGE_MEMORY_BINDING_MAIN].address.bo);
    }
 
+   struct anv_bo *private_bo = image->bindings[ANV_IMAGE_MEMORY_BINDING_PRIVATE].address.bo;
+   if (private_bo)
+      anv_device_release_bo(device, private_bo);
+
    vk_object_base_finish(&image->base);
    vk_free2(&device->vk.alloc, pAllocator, image);
 }
@@ -1436,15 +1504,19 @@ VkResult anv_BindImageMemory2(
          case VK_STRUCTURE_TYPE_BIND_IMAGE_PLANE_MEMORY_INFO: {
             const VkBindImagePlaneMemoryInfo *plane_info =
                (const VkBindImagePlaneMemoryInfo *) s;
+
             uint32_t plane = anv_image_aspect_to_plane(image->aspects,
                                                        plane_info->planeAspect);
 
-            /* Unlike VkImagePlaneMemoryRequirementsInfo, which requires that
+            /* Workaround for possible spec bug.
+             *
+             * Unlike VkImagePlaneMemoryRequirementsInfo, which requires that
              * the image be disjoint (that is, multi-planar format and
              * VK_IMAGE_CREATE_DISJOINT_BIT), VkBindImagePlaneMemoryInfo allows
              * the image to be non-disjoint and requires only that the image
-             * have the DISJOINT flag. (This may be a spec bug). In this case,
+             * have the DISJOINT flag. In this case, regardless of the value of
-             * we continue as if VkImagePlaneMemoryRequirementsInfo was omitted.
+             * VkImagePlaneMemoryRequirementsInfo::planeAspect, the behavior is
+             * the same as if VkImagePlaneMemoryRequirementsInfo were omitted.
              */
             if (!image->disjoint) {
                assert(plane == 0);
@@ -1470,14 +1542,16 @@ VkResult anv_BindImageMemory2(
             assert(image->aspects == swapchain_image->aspects);
             assert(mem == NULL);
 
-            /* The Vulkan 1.2 spec ensures that the image is not disjoint. See
+            for (int j = 0; j < ARRAY_SIZE(image->bindings); ++j)
-             * the table of implied image creation parameters for swapchains
+               image->bindings[j].address = swapchain_image->bindings[j].address;
-             * <vkspec.html#swapchain-wsi-image-create-info>.
-             */
-            assert(!swapchain_image->disjoint);
 
-            image->bindings[ANV_IMAGE_MEMORY_BINDING_MAIN].address =
+            /* We must bump the private binding's bo's refcount because, unlike the other
-               swapchain_image->bindings[ANV_IMAGE_MEMORY_BINDING_MAIN].address;
+             * bindings, its lifetime is not application-managed.
+             */
+            struct anv_bo *private_bo =
+               image->bindings[ANV_IMAGE_MEMORY_BINDING_PRIVATE].address.bo;
+            if (private_bo)
+               anv_bo_ref(private_bo);
 
             did_bind = true;
             break;

src/intel/vulkan/anv_private.h

@@ -3643,6 +3643,12 @@ struct anv_image_memory_range {
       ANV_IMAGE_MEMORY_BINDING_PLANE_1,
       ANV_IMAGE_MEMORY_BINDING_PLANE_2,
 
+      /**
+       * Driver-private bo. In special cases we may store the aux surface and/or
+       * aux state in this binding.
+       */
+      ANV_IMAGE_MEMORY_BINDING_PRIVATE,
+
       /** Sentinel */
       ANV_IMAGE_MEMORY_BINDING_END,
    } binding;
@@ -3725,6 +3731,8 @@ struct anv_image {
    /**
     * The memory bindings created by vkCreateImage and vkBindImageMemory.
     *
+    * For details on the image's memory layout, see check_memory_bindings().
+    *
     * vkCreateImage constructs the `memory_range` for each
     * anv_image_memory_binding.  After vkCreateImage, each binding is valid if
     * and only if `memory_range::size > 0`.
@@ -3751,29 +3759,6 @@ struct anv_image {
     * reside in the same VkImage.  To satisfy both the hardware and Vulkan, we
     * allocate the depth and stencil buffers as separate surfaces in the same
     * bo.
-    *
-    * Memory layout :
-    *
-    * -----------------------
-    * |     surface0        |   /|\
-    * -----------------------    |
-    * |   shadow surface0   |    |
-    * -----------------------    | Plane 0
-    * |    aux surface0     |    |
-    * -----------------------    |
-    * | fast clear colors0  |   \|/
-    * -----------------------
-    * |     surface1        |   /|\
-    * -----------------------    |
-    * |   shadow surface1   |    |
-    * -----------------------    | Plane 1
-    * |    aux surface1     |    |
-    * -----------------------    |
-    * | fast clear colors1  |   \|/
-    * -----------------------
-    * |        ...          |
-    * |                     |
-    * -----------------------
     */
    struct anv_image_plane {
       struct anv_surface primary_surface;