freedreno/a6xx: Move LRZ clears to gmem

If we have multiple LRZ clears, emit them all at once.  This also avoids
redundant LRZ clears if app does multiple clears in sequence.

Signed-off-by: Rob Clark <robdclark@chromium.org>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/22895>

5 files changed, 96 insertions, 37 deletions

diff --git a/src/gallium/drivers/freedreno/a6xx/fd6_blitter.cc b/src/gallium/drivers/freedreno/a6xx/fd6_blitter.cc
index c5e64b80f22..38e676aa7ad 100644
--- a/src/gallium/drivers/freedreno/a6xx/fd6_blitter.cc
+++ b/src/gallium/drivers/freedreno/a6xx/fd6_blitter.cc
@@ -802,10 +802,10 @@ emit_clear_color(struct fd_ringbuffer *ring, enum pipe_format pfmt,
 
 template <chip CHIP>
 void
-fd6_clear_lrz(struct fd_batch *batch, struct fd_resource *zsbuf, double depth)
+fd6_clear_lrz(struct fd_batch *batch, struct fd_resource *zsbuf,
+              struct fd_bo *lrz, double depth)
 {
    struct fd_ringbuffer *ring = fd_batch_get_prologue(batch);
-   struct fd_context *ctx = batch->ctx;
 
    if (DEBUG_BLIT) {
       fprintf(stderr, "lrz clear:\ndst resource: ");
@@ -813,13 +813,6 @@ fd6_clear_lrz(struct fd_batch *batch, struct fd_resource *zsbuf, double depth)
       fprintf(stderr, "\n");
    }
 
-   fd6_emit_ccu_cntl(ring, ctx->screen, false);
-
-   OUT_PKT7(ring, CP_SET_MARKER, 1);
-   OUT_RING(ring, A6XX_CP_SET_MARKER_0_MODE(RM6_BLIT2DSCALE));
-
-   fd6_event_write(batch, ring, CACHE_FLUSH_TS, true);
-
    OUT_PKT4(ring, REG_A6XX_GRAS_2D_DST_TL, 2);
    OUT_RING(ring, A6XX_GRAS_2D_DST_TL_X(0) | A6XX_GRAS_2D_DST_TL_Y(0));
    OUT_RING(ring, A6XX_GRAS_2D_DST_BR_X(zsbuf->lrz_width - 1) |
@@ -837,7 +830,7 @@ fd6_clear_lrz(struct fd_batch *batch, struct fd_resource *zsbuf, double depth)
                  .color_swap = WZYX,
            ),
            A6XX_RB_2D_DST(
-                 .bo = zsbuf->lrz,
+                 .bo = lrz,
            ),
            A6XX_RB_2D_DST_PITCH(zsbuf->lrz_pitch * 2),
    );
@@ -846,33 +839,12 @@ fd6_clear_lrz(struct fd_batch *batch, struct fd_resource *zsbuf, double depth)
     * Blit command:
     */
 
-   if (ctx->screen->info->a6xx.magic.RB_DBG_ECO_CNTL_blit != ctx->screen->info->a6xx.magic.RB_DBG_ECO_CNTL) {
-      /* This a non-context register, so we have to WFI before changing. */
-      OUT_WFI5(ring);
-      OUT_PKT4(ring, REG_A6XX_RB_DBG_ECO_CNTL, 1);
-      OUT_RING(ring, ctx->screen->info->a6xx.magic.RB_DBG_ECO_CNTL_blit);
-   }
-
    OUT_PKT7(ring, CP_BLIT, 1);
    OUT_RING(ring, CP_BLIT_0_OP(BLIT_OP_SCALE));
-
-   if (ctx->screen->info->a6xx.magic.RB_DBG_ECO_CNTL_blit != ctx->screen->info->a6xx.magic.RB_DBG_ECO_CNTL) {
-      OUT_WFI5(ring);
-      OUT_PKT4(ring, REG_A6XX_RB_DBG_ECO_CNTL, 1);
-      OUT_RING(ring, ctx->screen->info->a6xx.magic.RB_DBG_ECO_CNTL);
-   }
-
-   fd6_event_write(batch, ring, PC_CCU_FLUSH_COLOR_TS, true);
-   fd6_event_write(batch, ring, PC_CCU_FLUSH_DEPTH_TS, true);
-   fd6_event_write(batch, ring, CACHE_FLUSH_TS, true);
-   fd_wfi(batch, ring);
-
-   zsbuf->lrz_valid = true;
-   zsbuf->lrz_direction = FD_LRZ_UNKNOWN;
 }
 
-template void fd6_clear_lrz<A6XX>(struct fd_batch *batch, struct fd_resource *zsbuf, double depth);
-template void fd6_clear_lrz<A7XX>(struct fd_batch *batch, struct fd_resource *zsbuf, double depth);
+template void fd6_clear_lrz<A6XX>(struct fd_batch *batch, struct fd_resource *zsbuf, struct fd_bo *lrz, double depth);
+template void fd6_clear_lrz<A7XX>(struct fd_batch *batch, struct fd_resource *zsbuf, struct fd_bo *lrz, double depth);
 
 /**
  * Handle conversion of clear color
diff --git a/src/gallium/drivers/freedreno/a6xx/fd6_blitter.h b/src/gallium/drivers/freedreno/a6xx/fd6_blitter.h
index 5c213768399..b803e59cc54 100644
--- a/src/gallium/drivers/freedreno/a6xx/fd6_blitter.h
+++ b/src/gallium/drivers/freedreno/a6xx/fd6_blitter.h
@@ -43,7 +43,8 @@ unsigned fd6_tile_mode(const struct pipe_resource *tmpl);
  */
 
 template <chip CHIP>
-void fd6_clear_lrz(struct fd_batch *batch, struct fd_resource *zsbuf, double depth) assert_dt;
+void fd6_clear_lrz(struct fd_batch *batch, struct fd_resource *zsbuf,
+                   struct fd_bo *lrz, double depth) assert_dt;
 template <chip CHIP>
 void fd6_clear_surface(struct fd_context *ctx, struct fd_ringbuffer *ring,
                        struct pipe_surface *psurf, const struct pipe_box *box2d,
diff --git a/src/gallium/drivers/freedreno/a6xx/fd6_draw.cc b/src/gallium/drivers/freedreno/a6xx/fd6_draw.cc
index c991dab7de1..9936a57b4a9 100644
--- a/src/gallium/drivers/freedreno/a6xx/fd6_draw.cc
+++ b/src/gallium/drivers/freedreno/a6xx/fd6_draw.cc
@@ -447,7 +447,6 @@ do_lrz_clear(struct fd_context *ctx, enum fd_buffer_mask buffers)
          zsbuf->lrz && !is_z32(pfb->zsbuf->format);
 }
 
-template <chip CHIP>
 static bool
 fd6_clear(struct fd_context *ctx, enum fd_buffer_mask buffers,
           const union pipe_color_union *color, double depth,
@@ -486,7 +485,13 @@ fd6_clear(struct fd_context *ctx, enum fd_buffer_mask buffers,
 
    if (do_lrz_clear(ctx, buffers)) {
       struct fd_resource *zsbuf = fd_resource(pfb->zsbuf->texture);
-      fd6_clear_lrz<CHIP>(ctx->batch, zsbuf, depth);
+
+      zsbuf->lrz_valid = true;
+      zsbuf->lrz_direction = FD_LRZ_UNKNOWN;
+      subpass->clear_depth = depth;
+      subpass->fast_cleared |= FD_BUFFER_LRZ;
+
+      STATIC_ASSERT((FD_BUFFER_LRZ & FD_BUFFER_ALL) == 0);
    }
 
    /* we need to do multisample clear on 3d pipe, so fallback to u_blitter: */
@@ -511,7 +516,7 @@ fd6_draw_init(struct pipe_context *pctx)
    disable_thread_safety_analysis
 {
    struct fd_context *ctx = fd_context(pctx);
-   ctx->clear = fd6_clear<CHIP>;
+   ctx->clear = fd6_clear;
    ctx->draw_vbos = fd6_draw_vbos<CHIP>;
 }
 
diff --git a/src/gallium/drivers/freedreno/a6xx/fd6_gmem.cc b/src/gallium/drivers/freedreno/a6xx/fd6_gmem.cc
index 5ac3c9c39de..780552d0ae1 100644
--- a/src/gallium/drivers/freedreno/a6xx/fd6_gmem.cc
+++ b/src/gallium/drivers/freedreno/a6xx/fd6_gmem.cc
@@ -257,6 +257,78 @@ emit_lrz(struct fd_batch *batch, struct fd_batch_subpass *subpass)
            A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE());
 }
 
+/* Emit any needed lrz clears to the prologue cmds
+ */
+template <chip CHIP>
+static void
+emit_lrz_clears(struct fd_batch *batch)
+{
+   struct pipe_framebuffer_state *pfb = &batch->framebuffer;
+   struct fd_context *ctx = batch->ctx;
+   unsigned count = 0;
+
+   if (!pfb->zsbuf)
+      return;
+
+   struct fd_resource *zsbuf = fd_resource(pfb->zsbuf->texture);
+
+   foreach_subpass (subpass, batch) {
+      if (!(subpass->fast_cleared & FD_BUFFER_LRZ))
+         continue;
+
+      subpass->fast_cleared &= ~FD_BUFFER_LRZ;
+
+      /* prep before first clear: */
+      if (count == 0) {
+         struct fd_ringbuffer *ring = fd_batch_get_prologue(batch);
+
+         fd6_emit_ccu_cntl(ring, ctx->screen, false);
+
+         OUT_PKT7(ring, CP_SET_MARKER, 1);
+         OUT_RING(ring, A6XX_CP_SET_MARKER_0_MODE(RM6_BLIT2DSCALE));
+
+         fd6_event_write(batch, ring, CACHE_FLUSH_TS, true);
+
+         if (ctx->screen->info->a6xx.magic.RB_DBG_ECO_CNTL_blit !=
+             ctx->screen->info->a6xx.magic.RB_DBG_ECO_CNTL) {
+            /* This a non-context register, so we have to WFI before changing. */
+            OUT_WFI5(ring);
+            OUT_PKT4(ring, REG_A6XX_RB_DBG_ECO_CNTL, 1);
+            OUT_RING(ring, ctx->screen->info->a6xx.magic.RB_DBG_ECO_CNTL_blit);
+         }
+      }
+
+      fd6_clear_lrz<CHIP>(batch, zsbuf, subpass->lrz, subpass->clear_depth);
+
+      count++;
+   }
+
+   /* cleanup after last clear: */
+   if (count > 0) {
+      struct fd_ringbuffer *ring = fd_batch_get_prologue(batch);
+
+      if (ctx->screen->info->a6xx.magic.RB_DBG_ECO_CNTL_blit !=
+          ctx->screen->info->a6xx.magic.RB_DBG_ECO_CNTL) {
+         OUT_WFI5(ring);
+         OUT_PKT4(ring, REG_A6XX_RB_DBG_ECO_CNTL, 1);
+         OUT_RING(ring, ctx->screen->info->a6xx.magic.RB_DBG_ECO_CNTL);
+      }
+
+      /* Clearing writes via CCU color in the PS stage, and LRZ is read via
+       * UCHE in the earlier GRAS stage.
+       *
+       * Note tu also asks for WFI but maybe that is only needed if
+       * has_ccu_flush_bug (and it is added by fd6_emit_flushes() already
+       * in that case)
+       */
+      fd6_emit_flushes(batch->ctx, ring,
+                       FD6_FLUSH_CCU_COLOR |
+                       FD6_INVALIDATE_CACHE);
+
+      fd_wfi(batch, ring);
+   }
+}
+
 static bool
 use_hw_binning(struct fd_batch *batch)
 {
@@ -877,6 +949,8 @@ fd6_emit_tile_init(struct fd_batch *batch) assert_dt
    const struct fd_gmem_stateobj *gmem = batch->gmem_state;
    struct fd_screen *screen = batch->ctx->screen;
 
+   emit_lrz_clears<CHIP>(batch);
+
    fd6_emit_restore<CHIP>(batch, ring);
 
    fd6_emit_lrz_flush(ring);
@@ -1672,6 +1746,8 @@ fd6_emit_sysmem_prep(struct fd_batch *batch) assert_dt
 {
    struct fd_ringbuffer *ring = batch->gmem;
 
+   emit_lrz_clears<CHIP>(batch);
+
    fd6_emit_restore<CHIP>(batch, ring);
    fd6_emit_lrz_flush(ring);
 
diff --git a/src/gallium/drivers/freedreno/freedreno_context.h b/src/gallium/drivers/freedreno/freedreno_context.h
index 346b78c5e91..1c8f06c289c 100644
--- a/src/gallium/drivers/freedreno/freedreno_context.h
+++ b/src/gallium/drivers/freedreno/freedreno_context.h
@@ -258,6 +258,11 @@ enum fd_buffer_mask {
    FD_BUFFER_DEPTH = PIPE_CLEAR_DEPTH,
    FD_BUFFER_STENCIL = PIPE_CLEAR_STENCIL,
    FD_BUFFER_ALL = FD_BUFFER_COLOR | FD_BUFFER_DEPTH | FD_BUFFER_STENCIL,
+
+   /* A special internal buffer bit to signify that the LRZ buffer needs
+    * clearing
+    */
+   FD_BUFFER_LRZ = BIT(15),
 };
 
 #define MAX_HW_SAMPLE_PROVIDERS 7