tu: Support bufferDeviceAddressCaptureReplay on kgsl

We use the method used by the blob, which sets the USE_CPU_MAP flag,
originally intended for SVM, to allocate from a separate address range
and to control the address by passing a preferred address to mmap().

With this we can capture and replay gfxreconstruct traces on kgsl for
apps that use BDA, and we can replay them on msm with a small hack to
increase the address space size:

echo 274877906944 > /sys/module/msm/parameters/address_space_size

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/29251>

3 files changed, 48 insertions, 3 deletions

diff --git a/src/freedreno/vulkan/tu_knl.cc b/src/freedreno/vulkan/tu_knl.cc
index 4c674416ad8..a9ff89b8f02 100644
--- a/src/freedreno/vulkan/tu_knl.cc
+++ b/src/freedreno/vulkan/tu_knl.cc
@@ -72,7 +72,7 @@ tu_bo_map(struct tu_device *dev, struct tu_bo *bo, void *placed_addr)
 VkResult
 tu_bo_unmap(struct tu_device *dev, struct tu_bo *bo, bool reserve)
 {
-   if (!bo->map)
+   if (!bo->map || bo->never_unmap)
       return VK_SUCCESS;
 
    TU_RMV(bo_unmap, dev, bo);
diff --git a/src/freedreno/vulkan/tu_knl.h b/src/freedreno/vulkan/tu_knl.h
index 2f23377cb9d..4306c0cbed8 100644
--- a/src/freedreno/vulkan/tu_knl.h
+++ b/src/freedreno/vulkan/tu_knl.h
@@ -63,6 +63,7 @@ struct tu_bo {
 #endif
 
    bool implicit_sync : 1;
+   bool never_unmap : 1;
 };
 
 struct tu_knl {
diff --git a/src/freedreno/vulkan/tu_knl_kgsl.cc b/src/freedreno/vulkan/tu_knl_kgsl.cc
index e8b6e236c61..6c7dd102605 100644
--- a/src/freedreno/vulkan/tu_knl_kgsl.cc
+++ b/src/freedreno/vulkan/tu_knl_kgsl.cc
@@ -77,6 +77,8 @@ kgsl_submitqueue_close(struct tu_device *dev, uint32_t queue_id)
    safe_ioctl(dev->physical_device->local_fd, IOCTL_KGSL_DRAWCTXT_DESTROY, &req);
 }
 
+static void kgsl_bo_finish(struct tu_device *dev, struct tu_bo *bo);
+
 static VkResult
 bo_init_new_dmaheap(struct tu_device *dev, struct tu_bo **out_bo, uint64_t size,
                 enum tu_bo_alloc_flags flags)
@@ -170,9 +172,16 @@ kgsl_bo_init(struct tu_device *dev,
              enum tu_bo_alloc_flags flags,
              const char *name)
 {
-   assert(client_iova == 0);
-
    if (flags & TU_BO_ALLOC_SHAREABLE) {
+      /* The Vulkan spec doesn't forbid allocating exportable memory with a
+       * fixed address, only imported memory, but on kgsl we can't sensibly
+       * implement it so just always reject it.
+       */
+      if (client_iova) {
+         return vk_errorf(dev, VK_ERROR_INVALID_OPAQUE_CAPTURE_ADDRESS,
+                          "cannot allocate an exportable BO with a fixed address");
+      }
+
       switch(dev->physical_device->kgsl_dma_type) {
       case TU_KGSL_DMA_TYPE_DMAHEAP:
          return bo_init_new_dmaheap(dev, out_bo, size, flags);
@@ -200,6 +209,9 @@ kgsl_bo_init(struct tu_device *dev,
    if (flags & TU_BO_ALLOC_GPU_READ_ONLY)
       req.flags |= KGSL_MEMFLAGS_GPUREADONLY;
 
+   if (flags & TU_BO_ALLOC_REPLAYABLE)
+      req.flags |= KGSL_MEMFLAGS_USE_CPU_MAP;
+
    int ret;
 
    ret = safe_ioctl(dev->physical_device->local_fd,
@@ -221,6 +233,35 @@ kgsl_bo_init(struct tu_device *dev,
       .shared_fd = -1,
    };
 
+   if (flags & TU_BO_ALLOC_REPLAYABLE) {
+      uint64_t offset = req.id << 12;
+      void *map = mmap((void *)client_iova, bo->size, PROT_READ | PROT_WRITE,
+                       MAP_SHARED, dev->physical_device->local_fd, offset);
+      if (map == MAP_FAILED) {
+         kgsl_bo_finish(dev, bo);
+
+         return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY,
+                          "mmap failed (%s)", strerror(errno));
+      }
+
+      if (client_iova && (uint64_t)map != client_iova) {
+         kgsl_bo_finish(dev, bo);
+
+         return vk_errorf(dev, VK_ERROR_INVALID_OPAQUE_CAPTURE_ADDRESS,
+                          "mmap could not map the given address");
+      }
+
+      bo->map = map;
+      bo->iova = (uint64_t)map;
+
+      /* Because we're using SVM, the CPU mapping and GPU mapping are the same
+       * and the CPU mapping must stay fixed for the lifetime of the BO.
+       */
+      bo->never_unmap = true;
+
+   }
+
+
    *out_bo = bo;
 
    TU_RMV(bo_allocate, dev, bo);
@@ -1600,6 +1641,9 @@ tu_knl_kgsl_load(struct tu_instance *instance, int fd)
    device->heap.used = 0u;
    device->heap.flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT;
 
+   device->has_set_iova = kgsl_is_memory_type_supported(
+      fd, KGSL_MEMFLAGS_USE_CPU_MAP);
+
    /* Even if kernel is new enough, the GPU itself may not support it. */
    device->has_cached_coherent_memory = kgsl_is_memory_type_supported(
       fd, KGSL_MEMFLAGS_IOCOHERENT |