panfrost: Replace resource shadowing flush

The entire point of resource shadowing is to avoid unnecessary flushing.
Flushing readers after shadowing is counterproductive. A refresher on
how resource shadowing is supposed to work:

First, we determine if it's beneficial to shadow resources. If so, we
create a new backing buffer object. We flush the current writer of the
resource, if there is one, so the current contents become known to the
CPU. If we are not discarding the original resource, we then copy the
existing contents of the buffer to the new shadow buffer on the CPU.
Finally, we swap the resource's backing buffer for our shadow. Any batch
that reads the resource will continue to read the old copy of the
resource, and any future draw calls will see the new copy with the
change implemented.

Where did we go wrong?

In 988d5aae74 ("panfrost: Flush resources when shadowing"), we started
flushing all readers. We didn't actually need to flush, we just needed
to avoid dangling references on the batches reading the old copy of the
resource. But that's easily enough avoided: just remove the references.
The batches still hold a reference to the underlying BO, which will be
freed at the right time regardless.

Originally motivated by glmark2 -bbuffer:update-method=subdata, which
has some pathological access paterns.

Firefox is a lot faster anecdotally (now scrolling at 60fps in firefox).

But what actually motivated this is an apitrace from Duckstation's GLES
renderer. With this patch, the in-game portion is improved 3fps to 21fps.

Closes: #4028
Fixes: 988d5aae74 ("panfrost: Flush resources when shadowing")
Signed-off-by: Alyssa Rosenzweig <alyssa@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/19361>

src/gallium/drivers/panfrost/pan_job.c

@@ -122,7 +122,7 @@ panfrost_batch_cleanup(struct panfrost_context *ctx, struct panfrost_batch *batc
                 panfrost_bo_unreference(bo);
         }
 
-        set_foreach_remove(batch->resources, entry) {
+        set_foreach(batch->resources, entry) {
                 struct panfrost_resource *rsrc = (void *) entry->key;
 
                 if (_mesa_hash_table_search(ctx->writers, rsrc)) {
@@ -364,6 +364,44 @@ panfrost_batch_write_rsrc(struct panfrost_batch *batch,
         panfrost_batch_update_access(batch, rsrc, true);
 }
 
+void
+panfrost_resource_swap_bo(struct panfrost_context *ctx,
+                          struct panfrost_resource *rsrc,
+                          struct panfrost_bo *newbo)
+{
+        /* Any batch writing this resource is writing to the old BO, not the
+         * new BO. After swapping the resource's backing BO, there will be no
+         * writers of the updated resource. Existing writers still hold a
+         * reference to the old BO for reference counting.
+         */
+        struct hash_entry *writer = _mesa_hash_table_search(ctx->writers, rsrc);
+        if (writer) {
+                _mesa_hash_table_remove(ctx->writers, writer);
+                rsrc->track.nr_writers--;
+        }
+
+        /* Likewise, any batch reading this resource is reading the old BO, and
+         * after swapping will not be reading this resource.
+         */
+        unsigned i;
+        foreach_batch(ctx, i) {
+                struct panfrost_batch *batch = &ctx->batches.slots[i];
+                struct set_entry *ent = _mesa_set_search(batch->resources, rsrc);
+
+                if (!ent)
+                        continue;
+
+                _mesa_set_remove(batch->resources, ent);
+                rsrc->track.nr_users--;
+        }
+
+        /* Swap the pointers, dropping a reference to the old BO which is no
+         * long referenced from the resource
+         */
+        panfrost_bo_unreference(rsrc->image.data.bo);
+        rsrc->image.data.bo = newbo;
+}
+
 struct panfrost_bo *
 panfrost_batch_create_bo(struct panfrost_batch *batch, size_t size,
                          uint32_t create_flags, enum pipe_shader_type stage,

src/gallium/drivers/panfrost/pan_job.h

@@ -219,6 +219,11 @@ panfrost_batch_write_rsrc(struct panfrost_batch *batch,
                           struct panfrost_resource *rsrc,
                           enum pipe_shader_type stage);
 
+void
+panfrost_resource_swap_bo(struct panfrost_context *ctx,
+                          struct panfrost_resource *rsrc,
+                          struct panfrost_bo *newbo);
+
 struct panfrost_bo *
 panfrost_batch_create_bo(struct panfrost_batch *batch, size_t size,
                          uint32_t create_flags, enum pipe_shader_type stage,

src/gallium/drivers/panfrost/pan_resource.c

@@ -1051,6 +1051,14 @@ panfrost_ptr_map(struct pipe_context *pctx,
                 copy_resource = !panfrost_box_covers_resource(resource, box);
         }
 
+        /* Shadowing with separate stencil may require additional accounting.
+         * Bail in these exotic cases.
+         */
+        if (rsrc->separate_stencil) {
+                create_new_bo = false;
+                copy_resource = false;
+        }
+
         if (create_new_bo) {
                 /* Make sure we re-emit any descriptors using this resource */
                 panfrost_dirty_state_all(ctx);
@@ -1078,14 +1086,7 @@ panfrost_ptr_map(struct pipe_context *pctx,
                                 if (copy_resource)
                                         memcpy(newbo->ptr.cpu, rsrc->image.data.bo->ptr.cpu, bo->size);
 
-                                panfrost_bo_unreference(bo);
-                                rsrc->image.data.bo = newbo;
-
-                                /* Swapping out the BO will invalidate batches
-                                 * accessing this resource, flush them but do
-                                 * not wait for them.
-                                 */
-                                panfrost_flush_batches_accessing_rsrc(ctx, rsrc, "Resource shadowing");
+                                panfrost_resource_swap_bo(ctx, rsrc, newbo);
 
 	                        if (!copy_resource &&
                                     drm_is_afbc(rsrc->image.layout.modifier))