path: root/libclapi/cl_mem.c

/*
 * Copyright © 2012 Intel Corporation
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library. If not, see <http://www.gnu.org/licenses/>.
 *
 */

#include <string.h>
#include <assert.h>
#include <unistd.h>
#include "cl_context.h"
#include "cl_internals.h"
#include "cl_alloc.h"
#include "cl_mutex.h"
#include "cl_driver.h"
#include "cl_device_id.h"
#include "cl_context.h"
#include "cl_platform_id.h"
#include "cl_mem.h"
#include "cl_command_queue.h"
#include "cl_image.h"

#define FIELD_SIZE(CASE,TYPE)               \
  case JOIN(CL_,CASE):                      \
    if(param_value_size_ret)                \
      *param_value_size_ret = sizeof(TYPE); \
    if(!param_value)                        \
      return CL_SUCCESS;                    \
    if(param_value_size < sizeof(TYPE))     \
      return CL_INVALID_VALUE;              \
    break;

static cl_int cl_get_mem_object_info(cl_mem mem, cl_mem_info param_name, size_t param_value_size,
                                     void *param_value, size_t *param_value_size_ret)
{
  switch(param_name) {
      FIELD_SIZE(MEM_TYPE, cl_mem_object_type);
      FIELD_SIZE(MEM_FLAGS, cl_mem_flags);
      FIELD_SIZE(MEM_SIZE, size_t);
      FIELD_SIZE(MEM_HOST_PTR, void *);
      FIELD_SIZE(MEM_MAP_COUNT, cl_uint);
      FIELD_SIZE(MEM_REFERENCE_COUNT, cl_uint);
      FIELD_SIZE(MEM_CONTEXT, cl_context);
      FIELD_SIZE(MEM_ASSOCIATED_MEMOBJECT, cl_mem);
      FIELD_SIZE(MEM_OFFSET, size_t);
    default:
      return CL_INVALID_VALUE;
  }

  switch(param_name) {
    case CL_MEM_TYPE:
      *((cl_mem_object_type *)param_value) = mem->type;
      break;
    case CL_MEM_FLAGS:
      *((cl_mem_flags *)param_value) = mem->flags;
      break;
    case CL_MEM_SIZE:
      *((size_t *)param_value) = mem->size;
      break;
    case CL_MEM_HOST_PTR:
      if(mem->type != CL_MEM_OBJECT_BUFFER) {
        *((size_t *)param_value) = (size_t)mem->host_ptr;
      } else {
        cl_mem_buffer buf = cl_mem_to_buffer(mem);
        *((size_t *)param_value) = (size_t)mem->host_ptr + buf->sub_offset;
      }
      break;
    case CL_MEM_MAP_COUNT:
      *((cl_uint *)param_value) = mem->map_ref;
      break;
    case CL_MEM_REFERENCE_COUNT:
      *((cl_uint *)param_value) = cl_ref_get_val(&mem->ref_n);
      break;
    case CL_MEM_CONTEXT:
      *((cl_context *)param_value) = mem->ctx;
      break;
    case CL_MEM_ASSOCIATED_MEMOBJECT:
      if(mem->type != CL_MEM_OBJECT_BUFFER) {
        *((cl_mem *)param_value) = NULL;
      } else {
        cl_mem_buffer buf = cl_mem_to_buffer(mem);
        *((cl_mem *)param_value) = (cl_mem)(buf->parent);
      }
      break;
    case CL_MEM_OFFSET:
      if(mem->type != CL_MEM_OBJECT_BUFFER) {
        *((size_t *)param_value) = 0;
      } else {
        cl_mem_buffer buf = cl_mem_to_buffer(mem);
        *((size_t *)param_value) = buf->sub_offset;
      }
      break;
  }

  return CL_SUCCESS;
}

#undef FIELD_SIZE

LOCAL cl_int cl_retain_mem(cl_mem mem)
{
  assert(mem);
  int ret = cl_ref_inc_if_positive(&mem->ref_n);
  if (ret > 0)
    return ret;

  return 0;
}

LOCAL cl_mem cl_mem_new(cl_mem_object_type type, cl_context ctx, cl_mem_flags flags, size_t sz)
{
  cl_mem mem = NULL;

  /* Allocate and inialize the structure itself */
  if (type == CL_MEM_OBJECT_IMAGE2D || type == CL_MEM_OBJECT_IMAGE3D
      || type == CL_MEM_OBJECT_IMAGE2D_ARRAY || type == CL_MEM_OBJECT_IMAGE1D
      || type == CL_MEM_OBJECT_IMAGE1D_ARRAY || type == CL_MEM_OBJECT_IMAGE1D_BUFFER) {
    struct _cl_mem_image *image = NULL;
    image = CL_CALLOC(1, sizeof(struct _cl_mem_image));
    mem = &image->base;
  } else {
    struct _cl_mem_buffer *buffer = NULL;
    buffer = CL_CALLOC(1, sizeof(struct _cl_mem_buffer));
    mem = &buffer->base;
  }

  if (mem == NULL)
    return NULL;

  /* Create the private pointer array if device > 1 */
  if (ctx->device_num > 1) {
    mem->pdata = CL_CALLOC(ctx->device_num, sizeof(void*));
    if (mem->pdata == NULL) {
      CL_FREE(mem);
      return NULL;
    }
  }
  mem->type = type;
  cl_ref_set_val(&mem->ref_n, 1);
  mem->magic = CL_MAGIC_MEM_HEADER;
  mem->flags = flags;
  mem->size = sz;
  CL_MUTEX_INIT(&mem->lock);

  cl_retain_context(ctx);
  mem->ctx = ctx;

  /* Append the buffer in the context buffer list */
  CL_MUTEX_LOCK(&ctx->lock);
  mem->next = ctx->buffers;
  if (ctx->buffers != NULL)
    ctx->buffers->prev = mem;
  ctx->buffers = mem;
  CL_MUTEX_UNLOCK(&ctx->lock);

  return mem;
}

static void cl_mem_delete(cl_mem mem)
{
  assert(mem);
  assert(mem->ctx);

  /* Remove it from the list */
  CL_MUTEX_LOCK(&mem->ctx->lock);
  if (mem->prev)
    mem->prev->next = mem->next;
  if (mem->next)
    mem->next->prev = mem->prev;
  if (mem->ctx->buffers == mem)
    mem->ctx->buffers = mem->next;
  CL_MUTEX_UNLOCK(&mem->ctx->lock);
  cl_release_context(mem->ctx);

  if (mem->flags & CL_MEM_ALLOC_HOST_PTR) {
    assert(mem->host_ptr);
    CL_FREE(mem->host_ptr);
  }

  if (mem->mapped_ptr)
    CL_FREE(mem->mapped_ptr);

  if (mem->dstr_cb) {
    cl_mem_dstr_cb cb = NULL;
    while (mem->dstr_cb) {
      cb = mem->dstr_cb;
      mem->dstr_cb = cb->next;
      CL_FREE(cb);
    }
  }

  CL_MUTEX_DESTROY(&mem->lock);

  if (mem->ctx->device_num > 1) {
    CL_FREE(mem->pdata);
  }
  CL_FREE(mem);
}

LOCAL void cl_release_mem(cl_mem mem)
{
  cl_uint i;

  if (UNLIKELY(mem == NULL))
    return;
  if (cl_ref_dec(&mem->ref_n) > 1)
    return;

  /* Call the user callback. No need to lock, we are the last user. */
  if (mem->dstr_cb) {
    cl_mem_dstr_cb cb = mem->dstr_cb;
    while (mem->dstr_cb) {
      cb = mem->dstr_cb;
      cb->pfn_notify(mem, cb->user_data);
      mem->dstr_cb = cb->next;
    }
  }

  for (i = 0; i < mem->ctx->device_num; i++) {
    mem->ctx->devices[i]->driver->release_mem(mem, mem->ctx->devices[i]);
  }

  /* iff we are a image, delete the buffer we use. */
  if (IS_IMAGE(mem)) {
    if (cl_mem_to_image(mem)->buffer_from) {
      assert(mem->type == CL_MEM_OBJECT_IMAGE1D_BUFFER || mem->type == CL_MEM_OBJECT_IMAGE2D);
      cl_release_mem(cl_mem_to_image(mem)->buffer_from);
    }
  }

  /* Someone still mapped? Assert */
  if (mem->map_ref != 0) {
    FATAL("Free the mem %p, while there are still %d user mapping it\n", mem, mem->map_ref);
  }

  /* Iff we are sub. */
  if (cl_mem_to_buffer(mem)->parent) {
    cl_mem_buffer buffer = cl_mem_to_buffer(mem);
    /* Remove it from the parent's list */
    CL_MUTEX_LOCK(&buffer->parent->base.lock);
    if (buffer->sub_prev)
      buffer->sub_prev->sub_next = buffer->sub_next;
    if (buffer->sub_next)
      buffer->sub_next->sub_prev = buffer->sub_prev;
    if (buffer->parent->subs == buffer)
      buffer->parent->subs = buffer->sub_next;
    CL_MUTEX_UNLOCK(&buffer->parent->base.lock);
    cl_release_mem((cl_mem )(buffer->parent));
  }

  cl_mem_delete(mem);
}

static cl_mem cl_mem_create_buffer(cl_context ctx, cl_mem_flags flags, size_t sz,
                                   void *data, cl_int *errcode_ret)
{
  /* Possible mem type combination:
       CL_MEM_ALLOC_HOST_PTR
       CL_MEM_ALLOC_HOST_PTR | CL_MEM_COPY_HOST_PTR
       CL_MEM_USE_HOST_PTR
       CL_MEM_COPY_HOST_PTR   */

  cl_int err = CL_SUCCESS;
  cl_mem mem = NULL;
  cl_ulong max_mem_size;
  cl_uint i = 0;

  if (UNLIKELY(sz == 0)) {
    err = CL_INVALID_BUFFER_SIZE;
    goto error;
  }

  if (UNLIKELY(((flags & CL_MEM_READ_WRITE)
                && (flags & (CL_MEM_READ_ONLY | CL_MEM_WRITE_ONLY)))
               || ((flags & CL_MEM_READ_ONLY) && (flags & (CL_MEM_WRITE_ONLY)))
               || ((flags & CL_MEM_ALLOC_HOST_PTR) && (flags & CL_MEM_USE_HOST_PTR))
               || ((flags & CL_MEM_COPY_HOST_PTR) && (flags & CL_MEM_USE_HOST_PTR))
               || ((flags & CL_MEM_HOST_READ_ONLY) && (flags & CL_MEM_HOST_NO_ACCESS))
               || ((flags & CL_MEM_HOST_READ_ONLY) && (flags & CL_MEM_HOST_WRITE_ONLY))
               || ((flags & CL_MEM_HOST_WRITE_ONLY) && (flags & CL_MEM_HOST_NO_ACCESS))
               || ((flags & (~(CL_MEM_READ_WRITE | CL_MEM_WRITE_ONLY | CL_MEM_READ_ONLY
                               | CL_MEM_ALLOC_HOST_PTR | CL_MEM_COPY_HOST_PTR
                               | CL_MEM_USE_HOST_PTR | CL_MEM_HOST_WRITE_ONLY
                               | CL_MEM_HOST_READ_ONLY | CL_MEM_HOST_NO_ACCESS))) != 0))) {
    err = CL_INVALID_VALUE;
    goto error;
  }

  /* This flag is valid only if host_ptr is not NULL */
  if (UNLIKELY((((flags & CL_MEM_COPY_HOST_PTR) || (flags & CL_MEM_USE_HOST_PTR)) && data == NULL))
      || (!(flags & (CL_MEM_COPY_HOST_PTR |CL_MEM_USE_HOST_PTR)) && (data != NULL))) {
    err = CL_INVALID_HOST_PTR;
    goto error;
  }

  /* return CL_INVALID_BUFFER_SIZE if size is greater than
     CL_DEVICE_MAX_MEM_ALLOC_SIZE value for all devices in context. */
  for (i = 0; i < ctx->device_num; i++) {
    cl_device_id device = ctx->devices[i];

    if ((err = cl_get_device_info(device, CL_DEVICE_MAX_MEM_ALLOC_SIZE, sizeof(max_mem_size),
                                  &max_mem_size, NULL)) != CL_SUCCESS) {
      goto error;
    }

    if (UNLIKELY(sz > max_mem_size)) {
      err = CL_INVALID_BUFFER_SIZE;
      goto error;
    }
  }

  mem = cl_mem_new(CL_MEM_OBJECT_BUFFER, ctx, flags, sz);
  if (mem == NULL) {
    err = CL_OUT_OF_HOST_MEMORY;
    goto error;
  }

  if (flags & CL_MEM_ALLOC_HOST_PTR) {
    /*FIXME: For peformance and HW limitation, we need the address align to page size. */
    int page_size = getpagesize();
    mem->host_ptr = CL_MEMALIGN(sz, page_size);
    if (mem->host_ptr == NULL) {
      err = CL_OUT_OF_HOST_MEMORY;
      goto error;
    }

    if (flags & CL_MEM_COPY_HOST_PTR)
      memcpy(mem->host_ptr, data, sz);
  } else if (flags & (CL_MEM_USE_HOST_PTR|CL_MEM_COPY_HOST_PTR)) {
    mem->host_ptr = data;
  }

  for (i = 0; i < ctx->device_num; i++) {
    err = ctx->devices[i]->driver->create_buffer(mem, ctx->devices[i]);
    if (err != CL_SUCCESS)
      goto error;
  }

exit:
  if (errcode_ret)
    *errcode_ret = err;
  return mem;

error:
  cl_release_mem(mem);
  mem = NULL;
  goto exit;
}

static cl_mem cl_mem_create_sub_buffer(cl_mem buffer, cl_mem_flags flags, cl_buffer_create_type create_type,
                                       const void *create_info, cl_int *errcode_ret)
{
  cl_int err = CL_SUCCESS;
  cl_mem mem = NULL;
  struct _cl_mem_buffer *sub_buf = NULL;
  cl_uint i;

  if (buffer->type != CL_MEM_OBJECT_BUFFER) {
    err = CL_INVALID_MEM_OBJECT;
    goto error;
  }

  if (flags && (((buffer->flags & CL_MEM_WRITE_ONLY) && (flags & (CL_MEM_READ_WRITE|CL_MEM_READ_ONLY)))
                || ((buffer->flags & CL_MEM_READ_ONLY) && (flags & (CL_MEM_READ_WRITE|CL_MEM_WRITE_ONLY)))
                || (flags & (CL_MEM_USE_HOST_PTR | CL_MEM_ALLOC_HOST_PTR | CL_MEM_COPY_HOST_PTR))
                || ((flags & CL_MEM_HOST_READ_ONLY) && (flags & CL_MEM_HOST_NO_ACCESS))
                || ((flags & CL_MEM_HOST_READ_ONLY) && (flags & CL_MEM_HOST_WRITE_ONLY))
                || ((flags & CL_MEM_HOST_WRITE_ONLY) && (flags & CL_MEM_HOST_NO_ACCESS)))) {
    err = CL_INVALID_VALUE;
    goto error;
  }

  if((flags & (CL_MEM_WRITE_ONLY | CL_MEM_READ_ONLY | CL_MEM_READ_WRITE)) == 0) {
    flags |= buffer->flags & (CL_MEM_WRITE_ONLY | CL_MEM_READ_ONLY | CL_MEM_READ_WRITE);
  }
  flags |= buffer->flags & (CL_MEM_USE_HOST_PTR | CL_MEM_ALLOC_HOST_PTR | CL_MEM_COPY_HOST_PTR);
  if((flags & (CL_MEM_HOST_WRITE_ONLY | CL_MEM_HOST_READ_ONLY | CL_MEM_HOST_NO_ACCESS)) == 0) {
    flags |= buffer->flags & (CL_MEM_HOST_WRITE_ONLY | CL_MEM_HOST_READ_ONLY | CL_MEM_HOST_NO_ACCESS);
  }

  if (create_type != CL_BUFFER_CREATE_TYPE_REGION) {
    err = CL_INVALID_VALUE;
    goto error;
  }

  if (!create_info) {
    err = CL_INVALID_VALUE;
    goto error;
  }

  cl_buffer_region *info = (cl_buffer_region *)create_info;

  if (!info->size) {
    err = CL_INVALID_BUFFER_SIZE;
    goto error;
  }

  if (info->origin > buffer->size || info->origin + info->size > buffer->size) {
    err = CL_INVALID_VALUE;
    goto error;
  }

  for (i = 0; i < buffer->ctx->device_num; i++) {
    if (info->origin & (buffer->ctx->devices[i]->mem_base_addr_align / 8 - 1)) {
      err = CL_MISALIGNED_SUB_BUFFER_OFFSET;
      goto error;
    }
  }

  /* Now create the sub buffer and link it to the context. */
  mem = cl_mem_new(CL_MEM_OBJECT_BUFFER, buffer->ctx, flags, info->size);
  if (mem == NULL) {
    err = CL_OUT_OF_HOST_MEMORY;
    goto error;
  }

  if (mem == NULL || err != CL_SUCCESS)
    goto error;

  sub_buf = cl_mem_to_buffer(mem);

  /* Hold a ref of parent. */
  cl_retain_mem(buffer);
  sub_buf->parent = (struct _cl_mem_buffer*)buffer;

  /* Append the buffer in the parent buffer list */
  CL_MUTEX_LOCK(&buffer->lock);
  sub_buf->sub_next = cl_mem_to_buffer(buffer)->subs;
  if (cl_mem_to_buffer(buffer)->subs != NULL)
    cl_mem_to_buffer(buffer)->subs->sub_prev = sub_buf;
  cl_mem_to_buffer(buffer)->subs = sub_buf;
  CL_MUTEX_UNLOCK(&buffer->lock);

  mem->size = info->size;
  sub_buf->sub_offset = info->origin;
  if (buffer->flags & (CL_MEM_USE_HOST_PTR|CL_MEM_COPY_HOST_PTR)) {
    mem->host_ptr = buffer->host_ptr;
  }

  for (i = 0; i < mem->ctx->device_num; i++) {
    err = mem->ctx->devices[i]->driver->create_buffer(mem, mem->ctx->devices[i]);
    if (err != CL_SUCCESS)
      goto error;
  }

exit:
  if (errcode_ret)
    *errcode_ret = err;
  return mem;
error:
  cl_release_mem(mem);
  mem = NULL;
  goto exit;
}

static cl_int cl_mem_find_mapped(cl_mem mem, size_t offset, cl_map_flags map_flags, size_t size)
{
  int i;

  CL_MUTEX_LOCK(&mem->lock);
  if (!mem->mapped_ptr_sz) {
    CL_MUTEX_UNLOCK(&mem->lock);
    return CL_SUCCESS;
  }

  for (i = 0; i < mem->mapped_ptr_sz; i++) {
    if (offset + size <= mem->mapped_ptr[i].offset ||
        offset >= mem->mapped_ptr[i].offset + mem->mapped_ptr[i].size)
      continue; // No overlap, continue.

    /* overlap, check the flags. Write is mutual exclusive. */
    if (map_flags != CL_MAP_READ || mem->mapped_ptr[i].flags != CL_MAP_READ) {
      CL_MUTEX_UNLOCK(&mem->lock);
      return CL_INVALID_OPERATION;
    }
  }

  CL_MUTEX_UNLOCK(&mem->lock);
  return CL_SUCCESS;
}

static cl_int cl_mem_delete_mapped(cl_mem mem, void *mapped_ptr, cl_mapped_ptr_info info)
{
  int i, j;
  cl_bool ptr_invalid = CL_FALSE;

  CL_MUTEX_LOCK(&mem->lock);
  assert(mem->mapped_ptr_sz >= mem->map_ref);

  for (i = 0; i < mem->mapped_ptr_sz; i++) {
    if (mem->mapped_ptr[i].ptr == mapped_ptr) {
      /* We may find several slots have same mapped_ptr, but we will be sure that
         the overlapped map should be read only map, and so just find the first
         one and unmap it. */
      *info = mem->mapped_ptr[i]; // copy it.
      break;
    }
  }

  if (i == mem->mapped_ptr_sz) {
    ptr_invalid = CL_TRUE;
  } else {
    /* So some shrink thing.*/
    mem->mapped_ptr[i].ptr = NULL;
    mem->map_ref--;
    if (mem->mapped_ptr_sz/2 > mem->map_ref) {
      j = 0;
      cl_mapped_ptr_info new_ptr = CL_MALLOC(sizeof(_cl_mapped_ptr_info) * (mem->mapped_ptr_sz/2));
      if (!new_ptr) {
        /* Just do nothing. */
        CL_MUTEX_UNLOCK(&mem->lock);
        goto RETURN;
      }
      memset(new_ptr, 0, (mem->mapped_ptr_sz/2) * sizeof(_cl_mapped_ptr_info));

      for (i = 0; i < mem->mapped_ptr_sz; i++) {
        if (mem->mapped_ptr[i].ptr) {
          new_ptr[j] = mem->mapped_ptr[i];
          j++;
          assert(j < mem->mapped_ptr_sz/2);
        }
      }
      mem->mapped_ptr_sz = mem->mapped_ptr_sz/2;
      CL_FREE(mem->mapped_ptr);
      mem->mapped_ptr = new_ptr;
    }
  }
  CL_MUTEX_UNLOCK(&mem->lock);

RETURN:
  if (ptr_invalid)
    return CL_INVALID_VALUE;

  return CL_SUCCESS;
}

static cl_int cl_mem_record_mapped(cl_mem mem, void *ptr, size_t offset, cl_map_flags map_flags,
                                   size_t size, const size_t *origin, const size_t *region)
{
  cl_int slot = -1;
  int i;

  CL_MUTEX_LOCK(&mem->lock);

  if (!mem->mapped_ptr_sz) {
    mem->mapped_ptr_sz = 16;
    mem->mapped_ptr = CL_MALLOC(sizeof(_cl_mapped_ptr_info) * mem->mapped_ptr_sz);
    if (!mem->mapped_ptr) {
      CL_MUTEX_UNLOCK(&mem->lock);
      return CL_OUT_OF_HOST_MEMORY;
    }
    memset(mem->mapped_ptr, 0, mem->mapped_ptr_sz * sizeof(_cl_mapped_ptr_info));
    slot = 0;
  } else {
    /* Someone may already add a slot when we do the map, may conflict.  check it again*/
    for (i = 0; i < mem->mapped_ptr_sz; i++) {
      if (offset + size <= mem->mapped_ptr[i].offset ||
          offset >= mem->mapped_ptr[i].offset + mem->mapped_ptr[i].size)
        continue; // No overlap, continue.

      /* overlap, check the flags. Write is mutual exclusive. */
      if (map_flags != CL_MAP_READ || mem->mapped_ptr[i].flags != CL_MAP_READ) {
        CL_MUTEX_UNLOCK(&mem->lock);
        return CL_INVALID_OPERATION;
      }
    }

    /* Insert a new one. */
    for (i = 0; i < mem->mapped_ptr_sz; i++) {
      if (mem->mapped_ptr[i].ptr == NULL) {
        slot = i;
        break;
      }
    }

    if (i == mem->mapped_ptr_sz) { /* Expand the list double. */
      cl_mapped_ptr_info new_ptr = CL_MALLOC(sizeof(_cl_mapped_ptr_info) * mem->mapped_ptr_sz * 2);
      if (!new_ptr) {
        CL_MUTEX_UNLOCK(&mem->lock);
        return CL_OUT_OF_HOST_MEMORY;
      }
      memset(new_ptr, 0, 2 * mem->mapped_ptr_sz * sizeof(_cl_mapped_ptr_info));
      memcpy(new_ptr, mem->mapped_ptr, mem->mapped_ptr_sz * sizeof(_cl_mapped_ptr_info));
      slot = mem->mapped_ptr_sz;
      mem->mapped_ptr_sz *= 2;
      CL_FREE(mem->mapped_ptr);
      mem->mapped_ptr = new_ptr;
    }
  }

  assert(slot != -1);
  mem->mapped_ptr[slot].ptr = ptr;
  mem->mapped_ptr[slot].offset = offset;
  mem->mapped_ptr[slot].size = size;
  mem->mapped_ptr[slot].flags = map_flags;
  if(origin) {
    assert(region);
    mem->mapped_ptr[slot].origin[0] = origin[0];
    mem->mapped_ptr[slot].origin[1] = origin[1];
    mem->mapped_ptr[slot].origin[2] = origin[2];
    mem->mapped_ptr[slot].region[0] = region[0];
    mem->mapped_ptr[slot].region[1] = region[1];
    mem->mapped_ptr[slot].region[2] = region[2];
  } else {
    mem->mapped_ptr[slot].origin[0]
      = mem->mapped_ptr[slot].origin[1]
        = mem->mapped_ptr[slot].origin[2]
          = mem->mapped_ptr[slot].region[0]
            = mem->mapped_ptr[slot].region[1]
              = mem->mapped_ptr[slot].region[2] = 0;
  }
  mem->map_ref++;

  CL_MUTEX_UNLOCK(&mem->lock);
  return CL_SUCCESS;
}

static void* cl_enqueue_map_buffer(cl_command_queue queue, cl_mem buffer, cl_bool blocking_map,
                                   cl_map_flags map_flags, size_t offset, size_t size, cl_uint num_events,
                                   const cl_event *event_list, cl_event *event_ret, cl_int *errcode_ret)
{
  cl_event event = NULL;
  cl_command_queue_work_item it = NULL;
  cl_int err = CL_SUCCESS;
  void *mem_ptr = NULL;

  err = cl_mem_find_mapped(buffer, offset, map_flags, size);
  if (err != CL_SUCCESS)
    goto error;

  if (blocking_map) {
    /* According to spec, when in block mode, we need to ensure all the
     commands in queue are flushed. */
    err = cl_enqueue_wait_for_flush(queue);
    if (err != CL_SUCCESS)
      goto error;
  }

  it = cl_enqueue_create_work_item(queue, num_events, event_list);
  if (it == NULL) {
    err = CL_OUT_OF_HOST_MEMORY;
    goto error;
  }

  if (event_ret) {
    event = cl_create_event(buffer->ctx, queue, CL_FALSE, num_events, event_list, &err);
    if (event == NULL)
      goto error;
  }

  CL_MUTEX_LOCK(&buffer->lock);
  if (buffer->enqueued_device && buffer->enqueued_device != queue->device) {
    CL_MUTEX_UNLOCK(&buffer->lock);
    err = CL_INVALID_OPERATION;
    goto error;
  } else if (buffer->enqueued_device == NULL) {
    cl_retain_device_id(queue->device);
    buffer->enqueued_device = queue->device;
  }
  CL_MUTEX_UNLOCK(&buffer->lock);

  err = queue->device->driver->enqueue_map_buffer(queue, buffer, &mem_ptr,
        blocking_map, map_flags, offset, size, it);
  if (err != CL_SUCCESS)
    goto error;

  if (event)
    cl_enqueue_set_work_item_event(it, event);

  /* We need to store the map info for unmap and debug. */
  err = cl_mem_record_mapped(buffer, mem_ptr, offset, map_flags, size, NULL, NULL);
  if (err != CL_SUCCESS) {
    // Unmap and return error.
    queue->device->driver->enqueue_unmap_mem(queue, buffer, mem_ptr, 0, NULL, NULL);
    goto error;
  }

  if (it->status > CL_COMPLETE) { // Still something todo
    err = cl_enqueue_insert_work_item(queue, it);
    assert(err == CL_SUCCESS); // queue must be avaible.
  } else {
    cl_enqueue_destroy_work_item(queue, it);
    it = NULL;
  }

  if (errcode_ret)
    *errcode_ret = err;
  if (event_ret)
    *event_ret = event;
  return mem_ptr;

error:
  if (it)
    cl_enqueue_destroy_work_item(queue, it);
  if (errcode_ret)
    *errcode_ret = err;
  if (event)
    cl_release_event(event);

  return NULL;
}

static cl_int cl_enqueue_unmap_mem(cl_command_queue queue, cl_mem memobj, void *mapped_ptr,
                                   cl_uint num_events, const cl_event *event_list, cl_event *event_ret)
{
  cl_event event = NULL;
  cl_int err = CL_SUCCESS;
  cl_int index;
  _cl_mapped_ptr_info ptr_info;

  if (event_ret) {
    event = cl_create_event(memobj->ctx, queue, CL_FALSE, num_events, event_list, &err);
    if (event == NULL)
      goto error;
  }

  CL_MUTEX_LOCK(&memobj->lock);
  if (memobj->enqueued_device && memobj->enqueued_device != queue->device) {
    CL_MUTEX_UNLOCK(&memobj->lock);
    err = CL_INVALID_OPERATION;
    goto error;
  } else if (memobj->enqueued_device == NULL) {
    cl_retain_device_id(queue->device);
    memobj->enqueued_device = queue->device;
  }
  CL_MUTEX_UNLOCK(&memobj->lock);

  /* Check the pointer valid. */
  INVALID_VALUE_IF(!mapped_ptr);

  /* can not find a mapped address? */
  err = cl_mem_delete_mapped(memobj, mapped_ptr, &ptr_info);
  if (err != CL_SUCCESS)
    goto error;

  err = queue->device->driver->enqueue_unmap_mem(queue, memobj, mapped_ptr,
        num_events, event_list, event);
  if (err != CL_SUCCESS)
    goto error;

  if (event_ret)
    *event_ret = event;

  return err;

error:
  if (event)
    cl_release_event(event);
  return err;
}

/**************************************************************************************
 *************************           CL APIs           ********************************
 **************************************************************************************/
cl_mem
clCreateBuffer(cl_context    context,
               cl_mem_flags  flags,
               size_t        size,
               void *        host_ptr,
               cl_int *      errcode_ret)
{
  cl_mem mem = NULL;
  cl_int err = CL_SUCCESS;
  CHECK_CONTEXT (context);

  mem = cl_mem_create_buffer(context, flags, size, host_ptr, &err);
error:
  if (errcode_ret)
    *errcode_ret = err;
  return mem;
}

cl_mem
clCreateSubBuffer(cl_mem                buffer,
                  cl_mem_flags          flags,
                  cl_buffer_create_type buffer_create_type,
                  const void *          buffer_create_info,
                  cl_int *              errcode_ret)
{
  cl_mem mem = NULL;
  cl_int err = CL_SUCCESS;

  CHECK_MEM(buffer);

  mem = cl_mem_create_sub_buffer(buffer, flags, buffer_create_type,
                                 buffer_create_info, &err);
error:
  if (errcode_ret)
    *errcode_ret = err;
  return mem;
}

cl_int
clRetainMemObject(cl_mem memobj)
{
  cl_int err = CL_SUCCESS;
  CHECK_MEM(memobj);
  if (cl_retain_mem(memobj) == 0) {
    err = CL_INVALID_MEM_OBJECT;
  }

error:
  return err;
}

cl_int
clReleaseMemObject(cl_mem memobj)
{
  cl_int err = CL_SUCCESS;
  CHECK_MEM(memobj);
  cl_release_mem(memobj);
error:
  return err;
}

cl_int
clSetMemObjectDestructorCallback(cl_mem  memobj,
                                 void (CL_CALLBACK *pfn_notify) (cl_mem, void*),
                                 void * user_data)
{
  cl_int err = CL_SUCCESS;
  CHECK_MEM(memobj);
  INVALID_VALUE_IF (pfn_notify == 0);

  cl_mem_dstr_cb cb = malloc(sizeof(_cl_mem_dstr_cb));
  if (!cb) {
    err = CL_OUT_OF_HOST_MEMORY;
    goto error;
  }

  CL_MUTEX_LOCK(&memobj->lock);
  memset(cb, 0, sizeof(_cl_mem_dstr_cb));
  cb->pfn_notify = pfn_notify;
  cb->user_data = user_data;
  cb->next = memobj->dstr_cb;
  memobj->dstr_cb = cb;
  CL_MUTEX_UNLOCK(&memobj->lock);

error:
  return err;
}

cl_int
clGetMemObjectInfo(cl_mem      memobj,
                   cl_mem_info param_name,
                   size_t      param_value_size,
                   void *      param_value,
                   size_t *    param_value_size_ret)
{
  cl_int err = CL_SUCCESS;
  CHECK_MEM(memobj);

  err = cl_get_mem_object_info(memobj, param_name, param_value_size,
                               param_value, param_value_size_ret);
error:
  return err;
}

void *
clEnqueueMapBuffer(cl_command_queue  command_queue,
                   cl_mem            buffer,
                   cl_bool           blocking_map,
                   cl_map_flags      map_flags,
                   size_t            offset,
                   size_t            size,
                   cl_uint           num_events_in_wait_list,
                   const cl_event *  event_wait_list,
                   cl_event *        event,
                   cl_int *          errcode_ret)
{
  cl_int err = CL_SUCCESS;
  void *ptr = NULL;

  CHECK_QUEUE(command_queue);
  CHECK_MEM(buffer);

  err = cl_event_check_waitlist(num_events_in_wait_list, event_wait_list,
                                NULL, command_queue->ctx);
  if (err != CL_SUCCESS)
    goto error;

  if (command_queue->ctx != buffer->ctx) {
    err = CL_INVALID_CONTEXT;
    goto error;
  }

  if (!size || offset + size > buffer->size) {
    err = CL_INVALID_VALUE;
    goto error;
  }

  if ((map_flags & CL_MAP_READ &&
       buffer->flags & (CL_MEM_HOST_WRITE_ONLY | CL_MEM_HOST_NO_ACCESS)) ||
      (map_flags & (CL_MAP_WRITE | CL_MAP_WRITE_INVALIDATE_REGION) &&
       buffer->flags & (CL_MEM_HOST_READ_ONLY | CL_MEM_HOST_NO_ACCESS))) {
    err = CL_INVALID_OPERATION;
    goto error;
  }

  ptr = cl_enqueue_map_buffer(command_queue, buffer, blocking_map, map_flags, offset, size,
                              num_events_in_wait_list, event_wait_list, event, &err);
  if (ptr == NULL || err != CL_SUCCESS) {
    goto error;
  }

  if (errcode_ret)
    *errcode_ret = err;

  return ptr;

error:
  if (errcode_ret)
    *errcode_ret = err;
  return NULL;
}

cl_int
clEnqueueUnmapMemObject(cl_command_queue  command_queue,
                        cl_mem            memobj,
                        void *            mapped_ptr,
                        cl_uint           num_events_in_wait_list,
                        const cl_event *  event_wait_list,
                        cl_event *        event)
{
  cl_int err = CL_SUCCESS;

  CHECK_QUEUE(command_queue);
  CHECK_MEM(memobj);

  err = cl_event_check_waitlist(num_events_in_wait_list, event_wait_list,
                                NULL, command_queue->ctx);
  if (err != CL_SUCCESS)
    goto error;

  if (command_queue->ctx != memobj->ctx) {
    err = CL_INVALID_CONTEXT;
    goto error;
  }

  err = cl_enqueue_unmap_mem(command_queue, memobj, mapped_ptr, num_events_in_wait_list,
                             event_wait_list, event);

error:
  return err;
}