iris: Move suballocated resources to a dedicated allocation on export

We don't want to export suballocated resources to external consumers,
for a variety of reasons.  First of all, it would be exporting random
other pieces of memory which we may not want those external consumers
to have access to.  Secondly, external clients wouldn't be aware of
what buffers are packed together and busy-tracking implications there.
Nor should they be.  And those are just the obvious reasons.

When we allocate a resource with the PIPE_BIND_SHARED flag, indicating
that it's going to be used externally, we avoid suballocation.

However, there are times when the client may suddenly decide to export
a texture or buffer, without any prior warning.  Since we had no idea
this buffer would be exported, we suballocated it.  Unfortunately, this
means we need to transition it to a dedicated allocation on the fly, by
allocating a new buffer and copying the contents over.

Making things worse, this often happens in DRI hooks that don't have an
associated context (which we need to say, run BLORP commands).  We have
to create an temporary context for this purpose, perform our blit, then
destroy it.  The radeonsi driver uses a permanent auxiliary context
stored in the screen for this purpose, but we can't do that because it
causes circular reference counting.  radeonsi doesn't do the reference
counting that we do, but also doesn't use u_transfer_helper, so they
get lucky in avoiding stale resource->screen pointers.  Other drivers
don't create an auxiliary context, so they avoid this problem for now.

For auxiliary data, rather than copying it over bit-for-bit, we simply
copy over the underlying data using iris_copy_region (GPU memcpy), and
take whatever the resulting aux state is from that operation.  Assuming
the copy operation compresses, the result will be compressed.

v2: Stop using a screen->aux_context and just invent one on the fly to
    avoid circular reference counting issues.

Acked-by: Paulo Zanoni <paulo.r.zanoni@intel.com> [v1]
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/12623>

src/gallium/drivers/iris/iris_context.c

@@ -212,7 +212,7 @@ iris_flush_dirty_dmabufs(struct iris_context *ice)
 /**
  * Destroy a context, freeing any associated memory.
  */
-static void
+void
 iris_destroy_context(struct pipe_context *ctx)
 {
    struct iris_context *ice = (struct iris_context *)ctx;

src/gallium/drivers/iris/iris_context.h

@@ -848,6 +848,7 @@ struct iris_context {
 
 struct pipe_context *
 iris_create_context(struct pipe_screen *screen, void *priv, unsigned flags);
+void iris_destroy_context(struct pipe_context *ctx);
 
 void iris_lost_context_state(struct iris_batch *batch);
 

src/gallium/drivers/iris/iris_resource.c

@@ -1394,6 +1394,139 @@ iris_flush_resource(struct pipe_context *ctx, struct pipe_resource *resource)
    }
 }
 
+/**
+ * Reallocate a (non-external) resource into new storage, copying the data
+ * and modifying the original resource to point at the new storage.
+ *
+ * This is useful for e.g. moving a suballocated internal resource to a
+ * dedicated allocation that can be exported by itself.
+ */
+static void
+iris_reallocate_resource_inplace(struct iris_context *ice,
+                                 struct iris_resource *old_res,
+                                 unsigned new_bind_flag)
+{
+   struct pipe_screen *pscreen = ice->ctx.screen;
+
+   if (iris_bo_is_external(old_res->bo))
+      return;
+
+   assert(old_res->mod_info == NULL);
+   assert(old_res->bo == old_res->aux.bo || old_res->aux.bo == NULL);
+   assert(old_res->bo == old_res->aux.clear_color_bo ||
+          old_res->aux.clear_color_bo == NULL);
+   assert(old_res->external_format == PIPE_FORMAT_NONE);
+
+   struct pipe_resource templ = old_res->base.b;
+   templ.bind |= new_bind_flag;
+
+   struct iris_resource *new_res =
+      (void *) pscreen->resource_create(pscreen, &templ);
+
+   assert(iris_bo_is_real(new_res->bo));
+
+   struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
+
+   if (old_res->base.b.target == PIPE_BUFFER) {
+      struct pipe_box box = (struct pipe_box) {
+         .width = old_res->base.b.width0,
+         .height = 1,
+      };
+
+      iris_copy_region(&ice->blorp, batch, &new_res->base.b, 0, 0, 0, 0,
+                       &old_res->base.b, 0, &box);
+   } else {
+      for (unsigned l = 0; l <= templ.last_level; l++) {
+         struct pipe_box box = (struct pipe_box) {
+            .width = u_minify(templ.width0, l),
+            .height = u_minify(templ.height0, l),
+            .depth = util_num_layers(&templ, l),
+         };
+
+         iris_copy_region(&ice->blorp, batch, &new_res->base.b, 0, 0, 0, l,
+                          &old_res->base.b, l, &box);
+      }
+   }
+
+   iris_flush_resource(&ice->ctx, &new_res->base.b);
+
+   struct iris_bo *old_bo = old_res->bo;
+   struct iris_bo *old_aux_bo = old_res->aux.bo;
+   struct iris_bo *old_clear_color_bo = old_res->aux.clear_color_bo;
+
+   /* Replace the structure fields with the new ones */
+   old_res->base.b.bind = templ.bind;
+   old_res->bo = new_res->bo;
+   old_res->aux.surf = new_res->aux.surf;
+   old_res->aux.bo = new_res->aux.bo;
+   old_res->aux.offset = new_res->aux.offset;
+   old_res->aux.extra_aux.surf = new_res->aux.extra_aux.surf;
+   old_res->aux.extra_aux.offset = new_res->aux.extra_aux.offset;
+   old_res->aux.clear_color_bo = new_res->aux.clear_color_bo;
+   old_res->aux.clear_color_offset = new_res->aux.clear_color_offset;
+   old_res->aux.usage = new_res->aux.usage;
+   old_res->aux.possible_usages = new_res->aux.possible_usages;
+   old_res->aux.sampler_usages = new_res->aux.sampler_usages;
+
+   if (new_res->aux.state) {
+      assert(old_res->aux.state);
+      for (unsigned l = 0; l <= templ.last_level; l++) {
+         unsigned layers = util_num_layers(&templ, l);
+         for (unsigned z = 0; z < layers; z++) {
+            enum isl_aux_state aux =
+               iris_resource_get_aux_state(new_res, l, z);
+            iris_resource_set_aux_state(ice, old_res, l, z, 1, aux);
+         }
+      }
+   }
+
+   /* old_res now points at the new BOs, make new_res point at the old ones
+    * so they'll be freed when we unreference the resource below.
+    */
+   new_res->bo = old_bo;
+   new_res->aux.bo = old_aux_bo;
+   new_res->aux.clear_color_bo = old_clear_color_bo;
+
+   pipe_resource_reference((struct pipe_resource **)&new_res, NULL);
+}
+
+static void
+iris_resource_disable_suballoc_on_first_query(struct pipe_screen *pscreen,
+                                              struct pipe_context *ctx,
+                                              struct iris_resource *res)
+{
+   if (iris_bo_is_real(res->bo))
+      return;
+
+   assert(!(res->base.b.bind & PIPE_BIND_SHARED));
+
+   bool destroy_context;
+   if (ctx) {
+      ctx = threaded_context_unwrap_sync(ctx);
+      destroy_context = false;
+   } else {
+      /* We need to execute a blit on some GPU context, but the DRI layer
+       * often doesn't give us one.  So we have to invent a temporary one.
+       *
+       * We can't store a permanent context in the screen, as it would cause
+       * circular refcounting where screens reference contexts that reference
+       * resources, while resources reference screens...causing nothing to be
+       * freed.  So we just create and destroy a temporary one here.
+       */
+      ctx = iris_create_context(pscreen, NULL, 0);
+      destroy_context = true;
+   }
+
+   struct iris_context *ice = (struct iris_context *)ctx;
+
+   iris_reallocate_resource_inplace(ice, res, PIPE_BIND_SHARED);
+   assert(res->base.b.bind & PIPE_BIND_SHARED);
+
+   if (destroy_context)
+      iris_destroy_context(ctx);
+}
+
+
 static void
 iris_resource_disable_aux_on_first_query(struct pipe_resource *resource,
                                          unsigned usage)
@@ -1415,7 +1548,7 @@ iris_resource_disable_aux_on_first_query(struct pipe_resource *resource,
 
 static bool
 iris_resource_get_param(struct pipe_screen *pscreen,
-                        struct pipe_context *context,
+                        struct pipe_context *ctx,
                         struct pipe_resource *resource,
                         unsigned plane,
                         unsigned layer,
@@ -1429,11 +1562,15 @@ iris_resource_get_param(struct pipe_screen *pscreen,
    bool mod_with_aux =
       res->mod_info && res->mod_info->aux_usage != ISL_AUX_USAGE_NONE;
    bool wants_aux = mod_with_aux && plane > 0;
-   struct iris_bo *bo = wants_aux ? res->aux.bo : res->bo;
    bool result;
    unsigned handle;
 
    iris_resource_disable_aux_on_first_query(resource, handle_usage);
+   iris_resource_disable_suballoc_on_first_query(pscreen, ctx, res);
+
+   struct iris_bo *bo = wants_aux ? res->aux.bo : res->bo;
+
+   assert(iris_bo_is_real(bo));
 
    switch (param) {
    case PIPE_RESOURCE_PARAM_NPLANES:
@@ -1496,7 +1633,7 @@ iris_resource_get_param(struct pipe_screen *pscreen,
 
 static bool
 iris_resource_get_handle(struct pipe_screen *pscreen,
-                         struct pipe_context *unused_ctx,
+                         struct pipe_context *ctx,
                          struct pipe_resource *resource,
                          struct winsys_handle *whandle,
                          unsigned usage)
@@ -1506,9 +1643,10 @@ iris_resource_get_handle(struct pipe_screen *pscreen,
    bool mod_with_aux =
       res->mod_info && res->mod_info->aux_usage != ISL_AUX_USAGE_NONE;
 
-   /* if ctx is ever used, do ctx = threaded_context_unwrap_sync(ctx) */
-
    iris_resource_disable_aux_on_first_query(resource, usage);
+   iris_resource_disable_suballoc_on_first_query(pscreen, ctx, res);
+
+   assert(iris_bo_is_real(res->bo));
 
    struct iris_bo *bo;
    if (res->mod_info &&