path: root/backend/src/driver/cl_gen_mem.cpp

/*
 * Copyright © 2012 Intel Corporation
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library. If not, see <http://www.gnu.org/licenses/>.
 *
 */

#ifdef __cplusplus
extern "C" {  // for the C header files
#endif /* __cplusplus */
#include <unistd.h>
#include <string.h>
#include "cl_gen_driver.h"
#include "cl_mem.h"
#include "cl_event.h"
#ifdef __cplusplus
}
#endif /* __cplusplus */
#include "cl_gen_driver.hpp"
#include "sys/assert.hpp"
#include "sys/alloc.hpp"

#ifdef HAS_USERPTR
static drm_intel_bo* genBufferAllocUserptr(dri_bufmgr* bufmgr, const char* name,
    void *data, size_t size, unsigned long flags)
{
  drm_intel_bo *bo;
  bo = drm_intel_bo_alloc_userptr(bufmgr, name, data, I915_TILING_NONE, 0, size, flags);
  /* Fallback to unsynchronized userptr allocation if kernel has no MMU notifier enabled. */
  if (bo == NULL)
    bo = drm_intel_bo_alloc_userptr(bufmgr, name, data, I915_TILING_NONE, 0, size,
                                    flags | I915_USERPTR_UNSYNCHRONIZED);
  return bo;
}
#endif

GenGPUMem::~GenGPUMem(void)
{
  GBE_ASSERT(this->mappedAddr == NULL); // Should not still mapped.
  if (bo) {
    drm_intel_bo_unreference(bo);
    bo = NULL;
  }

  pthread_mutex_destroy(&this->mutex);
}

void GenGPUMem::genUnMapBo(cl_mem mem)
{
  /* From here, we want to lock. */
  GenGPULockerHelper mutexAlloc(&this->mutex);

  this->mapRef--;
  if (this->mapRef > 0) {
    /* We do not unmap if someone still mapped.
       e.g. We have already mapped two different regions of one buffer,
       when we unmap one of them, we should not make the address of
       the other map invalid. */
    return;
  }

  if (this->mappedAddr == NULL) {
    GBE_ASSERT(this->mappedGtt == 0);
    GBE_ASSERT(this->writeMap == false);
    return;
  }

  if (this->mappedGtt == 1) {
    drm_intel_gem_bo_unmap_gtt(this->bo);
    this->mappedGtt = 0;
  } else if (!this->alignedHostPtr)
    drm_intel_bo_unmap(this->bo);

  this->mappedAddr = NULL;
  this->writeMap = false;
}

void* GenGPUMem::genMapBo(cl_mem mem, bool write)
{
  /* From here, we want to lock. */
  GenGPULockerHelper mutexAlloc(&this->mutex);

  if (this->mappedAddr && this->writeMap != write)
    return NULL; // Not the same operation.

  if (this->mappedAddr != NULL) {
    this->mapRef++;
    return this->mappedAddr;
  }

  GBE_ASSERT(this->mapRef == 0);
  if (IS_IMAGE(mem) && this->tiling != GPGPU_NO_TILE) {
    /* If we are image from User PTR, the tiling must be GPGPU_NO_TILE. */
    drm_intel_gem_bo_map_gtt(this->bo);
    GBE_ASSERT(this->bo->virt);
    this->mappedGtt = 1;
    this->mappedAddr = this->bo->virt;
  } else {
    if (this->alignedHostPtr) {
      //not created from userptr, the offset should not be always zero.
      drm_intel_bo_wait_rendering(this->bo);
      this->mappedAddr = mem->host_ptr;
    } else {
      this->writeMap = write;
      drm_intel_bo_map(this->bo, write);
      GBE_ASSERT(this->bo->virt);
      this->mappedAddr = this->bo->virt;
    }
  }

  this->mapRef++;
  return this->mappedAddr;
}

bool GenGPUMem::genAllocMemBo(cl_mem mem)
{
  /* From here, we want to lock. */
  GenGPULockerHelper mutexAlloc(&this->mutex);

  if (this->bo != NULL) {
    return true;
  }

  if (!IS_IMAGE(mem)) {
    if (mem->flags & CL_MEM_USE_HOST_PTR) {
#ifdef HAS_USERPTR
      if (queue->device->host_unified_memory) { // Need to alloc host accessible mem.
        int page_size = getpagesize();
        int cacheline_size = queue->device->global_mem_cache_line_size;

        if (ALIGN((unsigned long)(mem->host_ptr), cacheline_size) == (unsigned long)(mem->host_ptr)) {
          this->alignedHostPtr = (void*)(((unsigned long)(mem->host_ptr)) & (~(page_size - 1)));
          this->realSize =
            ALIGN(((unsigned long)mem->host_ptr - (unsigned long)alignedHostPtr + mem->size), page_size);
          this->bo = genBufferAllocUserptr(gpuCtx->bufmgr, "CL userptr memory object",
                                           this->alignedHostPtr, this->realSize, 0);
        }
      }

      /* We can not create host accessible mem for GPU, we need to fallback to a fake
         CL_MEM_USE_HOST_PTR policy. We just keep user ptr, every time, when MAP,
         we copy the data to GPU and when UNMAP, copy back the data to host. */
      if (this->bo == NULL)
        this->alignedHostPtr = NULL;
#endif
    }

    if (this->bo == NULL) {
      this->bo = drm_intel_bo_alloc(this->gpuCtx->bufmgr, "CL memory object", this->realSize, 64);
      if (UNLIKELY(this->bo == NULL)) {
        return false;
      }
    }
  }

  /* Copy the data if required */
  if ((mem->flags & CL_MEM_COPY_HOST_PTR) ||
      (mem->flags & CL_MEM_USE_HOST_PTR && this->alignedHostPtr == NULL)) {
    if (IS_IMAGE(mem)) {
      drm_intel_bo_subdata(this->bo, 0, mem->size, mem->host_ptr);
    } else {
      cl_mem_buffer buffer = cl_mem_to_buffer(mem);
      drm_intel_bo_subdata(this->bo, buffer->sub_offset, mem->size, mem->host_ptr);
    }
  }

  return true;
}

static uint32_t cl_image_get_intel_format(const cl_image_format *fmt)
{
  const uint32_t type = fmt->image_channel_data_type;
  const uint32_t order = fmt->image_channel_order;
  switch (order) {
    case CL_R:
#if 0
    case CL_Rx:
    case CL_A:
    case CL_INTENSITY:
    case CL_LUMINANCE:
      if ((order == CL_INTENSITY || order == CL_LUMINANCE)
          && (type != CL_UNORM_INT8 && type != CL_UNORM_INT16
              && type != CL_SNORM_INT8 && type != CL_SNORM_INT16
              && type != CL_HALF_FLOAT && type != CL_FLOAT))
        return INTEL_UNSUPPORTED_FORMAT;
#endif

      /* XXX it seems we have some acuracy compatible issue with snomr_int8/16,
       * have to disable those formats currently. */

      switch (type) {
        case CL_HALF_FLOAT:
          return I965_SURFACEFORMAT_R16_FLOAT;
        case CL_FLOAT:
          return I965_SURFACEFORMAT_R32_FLOAT;
//        case CL_SNORM_INT16:    return I965_SURFACEFORMAT_R16_SNORM;
//        case CL_SNORM_INT8:     return I965_SURFACEFORMAT_R8_SNORM;
        case CL_UNORM_INT8:
          return I965_SURFACEFORMAT_R8_UNORM;
        case CL_UNORM_INT16:
          return I965_SURFACEFORMAT_R16_UNORM;
        case CL_SIGNED_INT8:
          return I965_SURFACEFORMAT_R8_SINT;
        case CL_SIGNED_INT16:
          return I965_SURFACEFORMAT_R16_SINT;
        case CL_SIGNED_INT32:
          return I965_SURFACEFORMAT_R32_SINT;
        case CL_UNSIGNED_INT8:
          return I965_SURFACEFORMAT_R8_UINT;
        case CL_UNSIGNED_INT16:
          return I965_SURFACEFORMAT_R16_UINT;
        case CL_UNSIGNED_INT32:
          return I965_SURFACEFORMAT_R32_UINT;
        default:
          return INTEL_UNSUPPORTED_FORMAT;
      };
    case CL_RG:
      switch (type) {
        case CL_UNORM_INT8:
          return I965_SURFACEFORMAT_R8G8_UNORM;
        case CL_UNSIGNED_INT8:
          return I965_SURFACEFORMAT_R8G8_UINT;
        default:
          return INTEL_UNSUPPORTED_FORMAT;
      };
#if 0
    case CL_RG:
    case CL_RA:
      switch (type) {
        case CL_HALF_FLOAT:
          return I965_SURFACEFORMAT_R16G16_FLOAT;
        case CL_FLOAT:
          return I965_SURFACEFORMAT_R32G32_FLOAT;
        case CL_SNORM_INT16:
          return I965_SURFACEFORMAT_R16G16_SNORM;
        case CL_SNORM_INT8:
          return I965_SURFACEFORMAT_R8G8_SNORM;
        case CL_UNORM_INT8:
          return I965_SURFACEFORMAT_R8G8_UNORM;
        case CL_UNORM_INT16:
          return I965_SURFACEFORMAT_R16G16_UNORM;
        case CL_SIGNED_INT8:
          return I965_SURFACEFORMAT_R8G8_SINT;
        case CL_SIGNED_INT16:
          return I965_SURFACEFORMAT_R16G16_SINT;
        case CL_SIGNED_INT32:
          return I965_SURFACEFORMAT_R32G32_SINT;
        case CL_UNSIGNED_INT8:
          return I965_SURFACEFORMAT_R8G8_UINT;
        case CL_UNSIGNED_INT16:
          return I965_SURFACEFORMAT_R16G16_UINT;
        case CL_UNSIGNED_INT32:
          return I965_SURFACEFORMAT_R32G32_UINT;
        default:
          return INTEL_UNSUPPORTED_FORMAT;
      };
    case CL_RGB:
    case CL_RGBx:
      switch (type) {
        case CL_UNORM_INT_101010:
          return I965_SURFACEFORMAT_R10G10B10A2_UNORM;
        case CL_UNORM_SHORT_565:
        case CL_UNORM_SHORT_555:
        default:
          return INTEL_UNSUPPORTED_FORMAT;
      };
#endif
    case CL_RGBA:
      switch (type) {
        case CL_HALF_FLOAT:
          return I965_SURFACEFORMAT_R16G16B16A16_FLOAT;
        case CL_FLOAT:
          return I965_SURFACEFORMAT_R32G32B32A32_FLOAT;
//        case CL_SNORM_INT16:    return I965_SURFACEFORMAT_R16G16B16A16_SNORM;
//        case CL_SNORM_INT8:     return I965_SURFACEFORMAT_R8G8B8A8_SNORM;
        case CL_UNORM_INT8:
          return I965_SURFACEFORMAT_R8G8B8A8_UNORM;
        case CL_UNORM_INT16:
          return I965_SURFACEFORMAT_R16G16B16A16_UNORM;
        case CL_SIGNED_INT8:
          return I965_SURFACEFORMAT_R8G8B8A8_SINT;
        case CL_SIGNED_INT16:
          return I965_SURFACEFORMAT_R16G16B16A16_SINT;
        case CL_SIGNED_INT32:
          return I965_SURFACEFORMAT_R32G32B32A32_SINT;
        case CL_UNSIGNED_INT8:
          return I965_SURFACEFORMAT_R8G8B8A8_UINT;
        case CL_UNSIGNED_INT16:
          return I965_SURFACEFORMAT_R16G16B16A16_UINT;
        case CL_UNSIGNED_INT32:
          return I965_SURFACEFORMAT_R32G32B32A32_UINT;
        default:
          return INTEL_UNSUPPORTED_FORMAT;
      };
    case CL_ARGB:
      return INTEL_UNSUPPORTED_FORMAT;
    case CL_BGRA:
      switch (type) {
        case CL_UNORM_INT8:
          return I965_SURFACEFORMAT_B8G8R8A8_UNORM;
        default:
          return INTEL_UNSUPPORTED_FORMAT;
      };
    default:
      return INTEL_UNSUPPORTED_FORMAT;
  };
}

extern "C"
cl_int GenSupportImageFmt(const cl_device_id device, cl_mem_object_type image_type,
                          cl_image_format* image_formats)
{
  if (image_formats == NULL)
    return CL_INVALID_VALUE;

  if (cl_image_get_intel_format(image_formats) == INTEL_UNSUPPORTED_FORMAT)
    return CL_IMAGE_FORMAT_NOT_SUPPORTED;

  return CL_SUCCESS;
}

extern "C"
cl_int GenCreateBuffer(cl_mem mem, const cl_device_id device)
{
  if (mem->type != CL_MEM_OBJECT_BUFFER) {
    return CL_INVALID_VALUE;
  }

  GBE_ASSERT(getGenMemPrivate(mem, device) == NULL);

  cl_mem_buffer buffer = cl_mem_to_buffer(mem);
  if (!buffer->parent) {
    GenGPUContext* gpuCtx = reinterpret_cast<GenGPUContext*>(getGenContextPrivate(mem->ctx, device));
    GenGPUMem* genMem = GBE_NEW(GenGPUMem, gpuCtx);

    /* HSW: Byte scattered Read/Write has limitation that
       the buffer size must be a multiple of 4 bytes. */
    genMem->realSize = ALIGN(mem->size, 4);
    setGenMemPrivate(mem, device, genMem);
  } else {
    /* A sub buffer, just ref the parent's GenGPUMem. */
    GenGPUMem* genMem = reinterpret_cast<GenGPUMem*>(getGenMemPrivate((cl_mem)(buffer->parent), device));
    GBE_ASSERT(genMem);
    setGenMemPrivate(mem, device, genMem);
  }
  return CL_SUCCESS;
}

extern "C"
cl_int GenReleaseMem(cl_mem mem, const cl_device_id device)
{
  if (mem->type != CL_MEM_OBJECT_BUFFER) {
    return CL_INVALID_VALUE;
  }
  cl_mem_buffer buffer = cl_mem_to_buffer(mem);

  GenGPUMem* genMem = reinterpret_cast<GenGPUMem*>(getGenMemPrivate(mem, device));
  if (genMem == NULL)
    return CL_INVALID_VALUE;

  if (!buffer->parent)
    GBE_DELETE(genMem);

  setGenMemPrivate(mem, device, NULL);
  return CL_SUCCESS;
}

static void* genDoMapBuffer(GenGPUMem* genMem, cl_mem mem, cl_map_flags flags, size_t offset, size_t size)
{
  cl_mem_buffer buffer = cl_mem_to_buffer(mem);
  char* retAddr = NULL;
  bool mapWrite = flags & (CL_MAP_WRITE | CL_MAP_WRITE_INVALIDATE_REGION);

  void* mappedAddr = genMem->genMapBo(mem, mapWrite);
  if (mappedAddr == NULL) {
    return NULL;
  }

  if (mem->flags & CL_MEM_USE_HOST_PTR) {
    retAddr = (char*)mem->host_ptr + buffer->sub_offset + offset;
  } else {
    retAddr = (char*)mappedAddr + buffer->sub_offset + offset;
  }

  /* We might need to call back the data in GPU side to host. */
  if ((mem->flags & CL_MEM_USE_HOST_PTR) && genMem->alignedHostPtr == NULL) {
    char* ptr = (char*)mappedAddr + buffer->sub_offset + offset;
    memcpy((char*)mem->host_ptr + buffer->sub_offset + offset, ptr, size);
  }

  return (void*)retAddr;
}

GenGPUWorkItemMapBuf::GenGPUWorkItemMapBuf(cl_mem mem, GenGPUMem* genMem, void* addr, cl_map_flags flags,
    size_t offset, size_t size, cl_event event, const cl_event* dependEvents, cl_uint num_events) :
  GenGPUWorkItem(event, dependEvents, num_events)
{
  if (addr)
    mappedAddr = addr;
  else
    mappedAddr = NULL;

  GBE_ASSERT(mem);
  this->mem = mem;
  GBE_ASSERT(genMem);
  this->genMem = genMem;
  this->offset = offset;
  this->size = size;
  this->flags = flags;
}

bool GenGPUWorkItemMapBuf::complete(void)
{
  /* Nothing to do here. */
  return true;
}

bool GenGPUWorkItemMapBuf::submit(void)
{
  void* retAddr = genDoMapBuffer(genMem, mem, flags, offset, size);
  if (retAddr == NULL) {
    return false;
  }

  /* Need to copy to the pre-allocated mem if not using host mem. */
  if (this->mappedAddr) {
    memcpy(this->mappedAddr, retAddr, this->size);
  }
  return true;
}

extern "C"
cl_int GenEnqueueMapBuffer(cl_command_queue queue, cl_mem mem, void** ret_addr, cl_bool block,
                           cl_map_flags flags, size_t offset, size_t size, cl_command_queue_work_item item)
{
  assert(item);

  if (mem->type != CL_MEM_OBJECT_BUFFER) {
    return CL_INVALID_VALUE;
  }

  GenGPUMem* genMem = reinterpret_cast<GenGPUMem*>(getGenMemPrivate(mem, queue->device));
  GenGPUCommandQueue* gpuQueue = (GenGPUCommandQueue*)getGenCommandQueuePrivate(queue);
  GBE_ASSERT(gpuQueue);
  void* retAddr = NULL;

  if (genMem == NULL)
    return CL_INVALID_VALUE;

  if (genMem->genAllocMemBo(mem) == false) {
    return CL_MEM_OBJECT_ALLOCATION_FAILURE;
  }

  if (block || item->depend_events == NULL) { // We do not need to do it async, just map it.
    retAddr = genDoMapBuffer(genMem, mem, flags, offset, size);
    if (retAddr == NULL) {
      return CL_MAP_FAILURE;
    }

    if (ret_addr)
      *ret_addr = retAddr;

    item->status = CL_COMPLETE;
    return CL_SUCCESS;
  }

  return CL_MAP_FAILURE;
}

static bool genDoUnMapBuffer(GenGPUMem* genMem, cl_mem mem, void *mapped_ptr,
                             size_t mapped_size, size_t mapped_offset)
{
  /* The fake CL_MEM_USE_HOST_PTR case, we need to copy back the data to GPU. */
  if ((mem->flags & CL_MEM_USE_HOST_PTR) && genMem->alignedHostPtr == NULL) {
    cl_mem_buffer buffer = cl_mem_to_buffer(mem);
    GBE_ASSERT((char*)mapped_ptr >= (char*)mem->host_ptr + buffer->sub_offset);
    GBE_ASSERT((char*)mapped_ptr + mapped_size <=
               (char*)mem->host_ptr + buffer->sub_offset + mem->size);
    /* Sync the data. */
    memcpy((char*)mem->host_ptr + buffer->sub_offset + mapped_offset, mapped_ptr, mapped_size);
  }

  genMem->genUnMapBo(mem);
  return true;
}

GenGPUWorkItemUnMapBuf::GenGPUWorkItemUnMapBuf(cl_mem mem, GenGPUMem* genMem, void* mappedAddr, size_t mappedSize,
    size_t mappedOffset, bool needToFree, cl_event event, const cl_event* dependEvents, cl_uint num_events) :
  GenGPUWorkItem(event, dependEvents, num_events)
{
  this->mappedAddr = mappedAddr;
  this->mappedSize = mappedSize;
  this->mappedOffset = mappedOffset;
  this->needToFree = needToFree;
  this->mem = mem;
  this->genMem = genMem;
}

bool GenGPUWorkItemUnMapBuf::complete(void)
{
  /* Nothing to do here. */
  return true;
}

bool GenGPUWorkItemUnMapBuf::submit(void)
{
  if (needToFree) {
    // Sync back
    cl_mem_buffer buffer = cl_mem_to_buffer(this->mem);
    memcpy((char*)genMem->mappedAddr + buffer->sub_offset + mappedOffset, mappedAddr, mappedSize);
  }

  genDoUnMapBuffer(genMem, mem, mappedAddr, mappedSize, mappedOffset);

  if (needToFree)
    GBE_FREE(mappedAddr);

  return true;
}

extern "C"
cl_int GenEnqueueUnmapMem(cl_command_queue queue, cl_mem mem, void *mapped_ptr,
                          cl_uint num_events, const cl_event *event_list, cl_event event_ret)
{
  if (mapped_ptr == NULL)
    return CL_INVALID_VALUE;

  GenGPUMem* genMem = reinterpret_cast<GenGPUMem*>(getGenMemPrivate(mem, queue->device));
  if (genMem == NULL)
    return CL_INVALID_VALUE;

  size_t mapped_size = 0;
  size_t mapped_offset = 0;
  size_t origin[3], region[3];
  int i = 0;
  // FIXME: Have no choice but to lock mem again.
  pthread_mutex_lock(&mem->lock);
  for (; i < mem->mapped_ptr_sz; i++) {
    if (mem->mapped_ptr[i].ptr == mapped_ptr) {
      mem->mapped_ptr[i].ptr = NULL;
      mapped_size = mem->mapped_ptr[i].size;
      mapped_offset = mem->mapped_ptr[i].offset;
      for(int j = 0; j < 3; j++) {
        region[j] = mem->mapped_ptr[i].region[j];
        origin[j] = mem->mapped_ptr[i].origin[j];
      }
      break;
    }
  }
  if (i == mem->mapped_ptr_sz) {
    pthread_mutex_unlock(&mem->lock);
    return CL_INVALID_VALUE;
  }
  pthread_mutex_unlock(&mem->lock);

  GenGPUCommandQueue* gpuQueue = (GenGPUCommandQueue*)getGenCommandQueuePrivate(queue);
  GBE_ASSERT(gpuQueue);

  bool needFree = false;
  if ((!mem->flags & CL_MEM_USE_HOST_PTR) &&
      ((mapped_ptr < genMem->mappedAddr) || (mapped_ptr > (char*)genMem->mappedAddr + genMem->realSize))) {
    needFree = true;
  }

  if (event_list == NULL) {
    /* We do it in sync mode. */
    if (needFree) {
      // Sync back
      cl_mem_buffer buffer = cl_mem_to_buffer(mem);
      memcpy((char*)genMem->mappedAddr + buffer->sub_offset + mapped_offset, mapped_ptr, mapped_size);
    }

    genDoUnMapBuffer(genMem, mem, mapped_ptr, mapped_size, mapped_offset);

    if (needFree)
      GBE_FREE(mapped_ptr);

    if (event_ret) {
      event_ret->set_status(event_ret, CL_COMPLETE);
    }
    return CL_SUCCESS;
  } else {
    GBE_ASSERT(num_events > 0);
    GenGPUWorkItemUnMapBuf* unmapItem = GBE_NEW(GenGPUWorkItemUnMapBuf, mem, genMem, mapped_ptr, mapped_size,
                                        mapped_offset, needFree, event_ret, event_list, num_events);
    if (unmapItem == NULL) {
      return CL_OUT_OF_HOST_MEMORY;
    }

    /* Final,enqueue it in the queue worker thread. */
    if (gpuQueue->enqueueWorkItem(unmapItem) == false) {
      GBE_DELETE(unmapItem);
      return CL_OUT_OF_RESOURCES;
    }
    return CL_SUCCESS;
  }

  GBE_ASSERT(0); // Never get here.
  return CL_INVALID_VALUE;
}