freedreno/a6xx: Introduce batch subpasses

Just the scaffolding for now, nothing actually creates multiple sub-
passes yet.  For now, only planning to use this for a6xx, as other
gens are doing clears on 3d.

Signed-off-by: Rob Clark <robdclark@chromium.org>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/22895>

5 files changed, 145 insertions, 51 deletions

diff --git a/src/gallium/drivers/freedreno/a6xx/fd6_draw.cc b/src/gallium/drivers/freedreno/a6xx/fd6_draw.cc
index b218180e591..bc425463f53 100644
--- a/src/gallium/drivers/freedreno/a6xx/fd6_draw.cc
+++ b/src/gallium/drivers/freedreno/a6xx/fd6_draw.cc
@@ -461,14 +461,16 @@ fd6_clear(struct fd_context *ctx, enum fd_buffer_mask buffers,
    if (pfb->samples > 1)
       return false;
 
+   struct fd_batch_subpass *subpass = ctx->batch->subpass;
+
    u_foreach_bit (i, color_buffers)
-      ctx->batch->clear_color[i] = *color;
+      subpass->clear_color[i] = *color;
    if (buffers & FD_BUFFER_DEPTH)
-      ctx->batch->clear_depth = depth;
+      subpass->clear_depth = depth;
    if (buffers & FD_BUFFER_STENCIL)
-      ctx->batch->clear_stencil = stencil;
+      subpass->clear_stencil = stencil;
 
-   ctx->batch->fast_cleared |= buffers;
+   subpass->fast_cleared |= buffers;
 
    return true;
 }
diff --git a/src/gallium/drivers/freedreno/a6xx/fd6_gmem.cc b/src/gallium/drivers/freedreno/a6xx/fd6_gmem.cc
index 79dd66c7453..26ddbf43f9b 100644
--- a/src/gallium/drivers/freedreno/a6xx/fd6_gmem.cc
+++ b/src/gallium/drivers/freedreno/a6xx/fd6_gmem.cc
@@ -785,7 +785,9 @@ emit_binning_pass(struct fd_batch *batch) assert_dt
 
    /* emit IB to binning drawcmds: */
    trace_start_binning_ib(&batch->trace, ring);
-   fd6_emit_ib(ring, batch->draw);
+   foreach_subpass (subpass, batch) {
+      fd6_emit_ib(ring, subpass->draw);
+   }
    trace_end_binning_ib(&batch->trace, ring);
 
    fd_reset_wfi(batch);
@@ -1124,18 +1126,19 @@ emit_restore_blit(struct fd_batch *batch, struct fd_ringbuffer *ring,
 }
 
 static void
-emit_clears(struct fd_batch *batch, struct fd_ringbuffer *ring)
+emit_subpass_clears(struct fd_batch *batch, struct fd_batch_subpass *subpass)
 {
    struct pipe_framebuffer_state *pfb = &batch->framebuffer;
    const struct fd_gmem_stateobj *gmem = batch->gmem_state;
+   struct fd_ringbuffer *ring = subpass->subpass_clears;
    enum a3xx_msaa_samples samples = fd_msaa_samples(pfb->samples);
 
-   uint32_t buffers = batch->fast_cleared;
+   uint32_t buffers = subpass->fast_cleared;
 
    if (buffers & PIPE_CLEAR_COLOR) {
 
       for (int i = 0; i < pfb->nr_cbufs; i++) {
-         union pipe_color_union *color = &batch->clear_color[i];
+         union pipe_color_union *color = &subpass->clear_color[i];
          union util_color uc = {0};
 
          if (!pfb->cbufs[i])
@@ -1216,11 +1219,11 @@ emit_clears(struct fd_batch *batch, struct fd_ringbuffer *ring)
 
       if (has_separate_stencil) {
          pfmt = util_format_get_depth_only(pfb->zsbuf->format);
-         clear_value = util_pack_z(pfmt, batch->clear_depth);
+         clear_value = util_pack_z(pfmt, subpass->clear_depth);
       } else {
          pfmt = pfb->zsbuf->format;
          clear_value =
-            util_pack_z_stencil(pfmt, batch->clear_depth, batch->clear_stencil);
+            util_pack_z_stencil(pfmt, subpass->clear_depth, subpass->clear_stencil);
       }
 
       if (buffers & PIPE_CLEAR_DEPTH)
@@ -1274,7 +1277,7 @@ emit_clears(struct fd_batch *batch, struct fd_ringbuffer *ring)
       OUT_RING(ring, 0);
 
       OUT_PKT4(ring, REG_A6XX_RB_BLIT_CLEAR_COLOR_DW0, 1);
-      OUT_RING(ring, batch->clear_stencil & 0xff);
+      OUT_RING(ring, subpass->clear_stencil & 0xff);
 
       fd6_emit_blit(batch, ring);
    }
@@ -1326,12 +1329,15 @@ prepare_tile_setup(struct fd_batch *batch)
       emit_restore_blits(batch, batch->tile_loads);
    }
 
-   if (batch->fast_cleared) {
-      batch->tile_clears =
+   foreach_subpass (subpass, batch) {
+      if (!subpass->fast_cleared)
+         continue;
+
+      subpass->subpass_clears =
          fd_submit_new_ringbuffer(batch->submit, 0x1000, FD_RINGBUFFER_STREAMING);
 
-      set_blit_scissor(batch, batch->tile_clears);
-      emit_clears(batch, batch->tile_clears);
+      set_blit_scissor(batch, subpass->subpass_clears);
+      emit_subpass_clears(batch, subpass);
    }
 }
 
@@ -1352,12 +1358,6 @@ fd6_emit_tile_renderprep(struct fd_batch *batch, const struct fd_tile *tile)
       emit_conditional_ib(batch, tile, batch->tile_loads);
       trace_end_tile_loads(&batch->trace, batch->gmem);
    }
-
-   if (batch->tile_clears) {
-      trace_start_clears(&batch->trace, batch->gmem, batch->fast_cleared);
-      emit_conditional_ib(batch, tile, batch->tile_clears);
-      trace_end_clears(&batch->trace, batch->gmem);
-   }
 }
 
 static bool
@@ -1518,7 +1518,15 @@ prepare_tile_fini(struct fd_batch *batch)
 static void
 fd6_emit_tile(struct fd_batch *batch, const struct fd_tile *tile)
 {
-   fd6_emit_ib(batch->gmem, batch->draw);
+   foreach_subpass (subpass, batch) {
+      if (subpass->subpass_clears) {
+         trace_start_clears(&batch->trace, batch->gmem, subpass->fast_cleared);
+         emit_conditional_ib(batch, tile, subpass->subpass_clears);
+         trace_end_clears(&batch->trace, batch->gmem);
+      }
+
+      fd6_emit_ib(batch->gmem, subpass->draw);
+   }
 
    if (batch->tile_epilogue)
       fd6_emit_ib(batch->gmem, batch->tile_epilogue);
@@ -1580,12 +1588,14 @@ fd6_emit_tile_fini(struct fd_batch *batch)
 
 template <chip CHIP>
 static void
-emit_sysmem_clears(struct fd_batch *batch, struct fd_ringbuffer *ring) assert_dt
+emit_sysmem_clears(struct fd_batch *batch, struct fd_batch_subpass *subpass)
+   assert_dt
 {
    struct fd_context *ctx = batch->ctx;
+   struct fd_ringbuffer *ring = batch->gmem;
    struct pipe_framebuffer_state *pfb = &batch->framebuffer;
 
-   uint32_t buffers = batch->fast_cleared;
+   uint32_t buffers = subpass->fast_cleared;
 
    if (!buffers)
       return;
@@ -1597,7 +1607,7 @@ emit_sysmem_clears(struct fd_batch *batch, struct fd_ringbuffer *ring) assert_dt
 
    if (buffers & PIPE_CLEAR_COLOR) {
       for (int i = 0; i < pfb->nr_cbufs; i++) {
-         union pipe_color_union color = batch->clear_color[i];
+         union pipe_color_union color = subpass->clear_color[i];
 
          if (!pfb->cbufs[i])
             continue;
@@ -1618,14 +1628,14 @@ emit_sysmem_clears(struct fd_batch *batch, struct fd_ringbuffer *ring) assert_dt
             : NULL;
 
       if ((buffers & PIPE_CLEAR_DEPTH) || (!separate_stencil && (buffers & PIPE_CLEAR_STENCIL))) {
-         value.f[0] = batch->clear_depth;
-         value.ui[1] = batch->clear_stencil;
+         value.f[0] = subpass->clear_depth;
+         value.ui[1] = subpass->clear_stencil;
          fd6_clear_surface<CHIP>(ctx, ring, pfb->zsbuf, &box2d,
                                  &value, fd6_unknown_8c01(pfb->zsbuf->format, buffers));
       }
 
       if (separate_stencil && (buffers & PIPE_CLEAR_STENCIL)) {
-         value.ui[0] = batch->clear_stencil;
+         value.ui[0] = subpass->clear_stencil;
 
          struct pipe_surface stencil_surf = *pfb->zsbuf;
          stencil_surf.format = PIPE_FORMAT_S8_UINT;
@@ -1712,18 +1722,26 @@ fd6_emit_sysmem(struct fd_batch *batch)
    struct fd_ringbuffer *ring = batch->gmem;
    struct fd_screen *screen = batch->ctx->screen;
 
-   emit_sysmem_clears<CHIP>(batch, ring);
+   foreach_subpass (subpass, batch) {
+      if (subpass->fast_cleared) {
+         unsigned flushes = 0;
+         if (subpass->fast_cleared & FD_BUFFER_COLOR)
+            flushes |= FD6_INVALIDATE_CCU_COLOR;
+         if (subpass->fast_cleared & (FD_BUFFER_DEPTH | FD_BUFFER_STENCIL))
+            flushes |= FD6_INVALIDATE_CCU_DEPTH;
 
-   fd6_event_write(batch, ring, PC_CCU_INVALIDATE_COLOR, false);
-   fd6_cache_inv(batch, ring);
+         fd6_emit_flushes(batch->ctx, ring, flushes);
+         emit_sysmem_clears<CHIP>(batch, subpass);
+      }
 
-   fd_wfi(batch, ring);
-   fd6_emit_ccu_cntl(ring, screen, false);
+      fd_wfi(batch, ring);
+      fd6_emit_ccu_cntl(ring, screen, false);
 
-   struct pipe_framebuffer_state *pfb = &batch->framebuffer;
-   update_render_cntl<CHIP>(batch, pfb, false);
+      struct pipe_framebuffer_state *pfb = &batch->framebuffer;
+      update_render_cntl<CHIP>(batch, pfb, false);
 
-   fd6_emit_ib(ring, batch->draw);
+      fd6_emit_ib(ring, subpass->draw);
+   }
 }
 
 static void
diff --git a/src/gallium/drivers/freedreno/freedreno_batch.c b/src/gallium/drivers/freedreno/freedreno_batch.c
index 7e4ac307a40..46f26387267 100644
--- a/src/gallium/drivers/freedreno/freedreno_batch.c
+++ b/src/gallium/drivers/freedreno/freedreno_batch.c
@@ -57,6 +57,35 @@ alloc_ring(struct fd_batch *batch, unsigned sz, enum fd_ringbuffer_flags flags)
    return fd_submit_new_ringbuffer(batch->submit, sz, flags);
 }
 
+static struct fd_batch_subpass *
+subpass_create(struct fd_batch *batch)
+{
+   struct fd_batch_subpass *subpass = CALLOC_STRUCT(fd_batch_subpass);
+
+   subpass->draw = alloc_ring(batch, 0x100000, 0);
+
+   /* Replace batch->draw with reference to current subpass, for
+    * backwards compat with code that is not subpass aware.
+    */
+   if (batch->draw)
+      fd_ringbuffer_del(batch->draw);
+   batch->draw = fd_ringbuffer_ref(subpass->draw);
+
+   list_addtail(&subpass->node, &batch->subpasses);
+
+   return subpass;
+}
+
+static void
+subpass_destroy(struct fd_batch_subpass *subpass)
+{
+   fd_ringbuffer_del(subpass->draw);
+   if (subpass->subpass_clears)
+      fd_ringbuffer_del(subpass->subpass_clears);
+   list_del(&subpass->node);
+   free(subpass);
+}
+
 struct fd_batch *
 fd_batch_create(struct fd_context *ctx, bool nondraw)
 {
@@ -74,13 +103,13 @@ fd_batch_create(struct fd_context *ctx, bool nondraw)
    batch->resources =
       _mesa_set_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal);
 
+   list_inithead(&batch->subpasses);
+
    batch->submit = fd_submit_new(ctx->pipe);
    if (batch->nondraw) {
       batch->gmem = alloc_ring(batch, 0x1000, FD_RINGBUFFER_PRIMARY);
-      batch->draw = alloc_ring(batch, 0x100000, 0);
    } else {
       batch->gmem = alloc_ring(batch, 0x100000, FD_RINGBUFFER_PRIMARY);
-      batch->draw = alloc_ring(batch, 0x100000, 0);
 
       /* a6xx+ re-uses draw rb for both draw and binning pass: */
       if (ctx->screen->gen < 6) {
@@ -88,6 +117,8 @@ fd_batch_create(struct fd_context *ctx, bool nondraw)
       }
    }
 
+   batch->subpass = subpass_create(batch);
+
    batch->in_fence_fd = -1;
    batch->fence = NULL;
 
@@ -130,6 +161,10 @@ cleanup_submit(struct fd_batch *batch)
    if (!batch->submit)
       return;
 
+   foreach_subpass_safe (subpass, batch) {
+      subpass_destroy(subpass);
+   }
+
    fd_ringbuffer_del(batch->draw);
    fd_ringbuffer_del(batch->gmem);
 
@@ -158,11 +193,6 @@ cleanup_submit(struct fd_batch *batch)
       batch->tile_loads = NULL;
    }
 
-   if (batch->tile_clears) {
-      fd_ringbuffer_del(batch->tile_clears);
-      batch->tile_clears = NULL;
-   }
-
    if (batch->tile_store) {
       fd_ringbuffer_del(batch->tile_store);
       batch->tile_store = NULL;
diff --git a/src/gallium/drivers/freedreno/freedreno_batch.h b/src/gallium/drivers/freedreno/freedreno_batch.h
index a799570f0c2..7d99a0e5807 100644
--- a/src/gallium/drivers/freedreno/freedreno_batch.h
+++ b/src/gallium/drivers/freedreno/freedreno_batch.h
@@ -45,7 +45,38 @@ struct fd_resource;
 struct fd_batch_key;
 struct fd_batch_result;
 
-/* A batch tracks everything about a cmdstream batch/submit, including the
+/**
+ * A subpass is a fragment of a batch potentially starting with a clear.
+ * If the app does a mid-batch clear, that clear and subsequent draws
+ * can be split out into another sub-pass.  At gmem time, the appropriate
+ * sysmem or gmem clears can be interleaved with the CP_INDIRECT_BUFFER
+ * to the subpass's draw cmdstream.
+ */
+struct fd_batch_subpass {
+   struct list_head node;
+
+   /** draw pass cmdstream: */
+   struct fd_ringbuffer *draw;
+
+   /** for the gmem code to stash per tile per subpass clears */
+   struct fd_ringbuffer *subpass_clears;
+
+   BITMASK_ENUM(fd_buffer_mask) fast_cleared;
+
+   union pipe_color_union clear_color[MAX_RENDER_TARGETS];
+   double clear_depth;
+   unsigned clear_stencil;
+
+   /**
+    * The number of draws emitted to this subpass.  If it is greater than
+    * zero, a clear triggers creating a new subpass (because clears must
+    * always come at the start of a subpass).
+    */
+   unsigned num_draws;
+};
+
+/**
+ * A batch tracks everything about a cmdstream batch/submit, including the
  * ringbuffers used for binning, draw, and gmem cmds, list of associated
  * fd_resource-s, etc.
  */
@@ -76,7 +107,7 @@ struct fd_batch {
     * where the contents are undefined, ie. what we don't need to restore
     * to gmem.
     */
-   BITMASK_ENUM(fd_buffer_mask) invalidated, cleared, fast_cleared, restore, resolve;
+   BITMASK_ENUM(fd_buffer_mask) invalidated, cleared, restore, resolve;
 
    /* is this a non-draw batch (ie compute/blit which has no pfb state)? */
    bool nondraw : 1;
@@ -190,7 +221,24 @@ struct fd_batch {
 
    struct fd_submit *submit;
 
-   /** draw pass cmdstream: */
+   /**
+    * List of fd_batch_subpass.
+    */
+   struct list_head subpasses;
+
+#define foreach_subpass(subpass, batch) \
+   list_for_each_entry (struct fd_batch_subpass, subpass, &batch->subpasses, node)
+#define foreach_subpass_safe(subpass, batch) \
+   list_for_each_entry_safe (struct fd_batch_subpass, subpass, &batch->subpasses, node)
+
+   /**
+    * The current subpass.
+    */
+   struct fd_batch_subpass *subpass;
+
+   /**
+    * just a reference to the current subpass's draw cmds for backwards compat.
+    */
    struct fd_ringbuffer *draw;
    /** binning pass cmdstream: */
    struct fd_ringbuffer *binning;
@@ -207,13 +255,8 @@ struct fd_batch {
    struct fd_ringbuffer *epilogue;
 
    struct fd_ringbuffer *tile_loads;
-   struct fd_ringbuffer *tile_clears;
    struct fd_ringbuffer *tile_store;
 
-   union pipe_color_union clear_color[MAX_RENDER_TARGETS];
-   double clear_depth;
-   unsigned clear_stencil;
-
    /**
     * hw query related state:
     */
diff --git a/src/gallium/drivers/freedreno/freedreno_draw.c b/src/gallium/drivers/freedreno/freedreno_draw.c
index db03244017f..ed39669ff1f 100644
--- a/src/gallium/drivers/freedreno/freedreno_draw.c
+++ b/src/gallium/drivers/freedreno/freedreno_draw.c
@@ -369,6 +369,7 @@ fd_draw_vbo(struct pipe_context *pctx, const struct pipe_draw_info *info,
    }
 
    batch->num_draws++;
+   batch->subpass->num_draws++;
 
    fd_print_dirty_state(ctx->dirty);