path: root/libclapi/cl_mem.c

/*
 * Copyright © 2012 Intel Corporation
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library. If not, see <http://www.gnu.org/licenses/>.
 *
 */

#include <string.h>
#include <assert.h>
#include "cl_context.h"
#include "cl_internals.h"
#include "cl_alloc.h"
#include "cl_driver.h"
#include "cl_device_id.h"
#include "cl_context.h"
#include "cl_platform_id.h"
#include "cl_mem.h"
#include "cl_image.h"

#define FIELD_SIZE(CASE,TYPE)               \
  case JOIN(CL_,CASE):                      \
    if(param_value_size_ret)                \
      *param_value_size_ret = sizeof(TYPE); \
    if(!param_value)                        \
      return CL_SUCCESS;                    \
    if(param_value_size < sizeof(TYPE))     \
      return CL_INVALID_VALUE;              \
    break;

static cl_int cl_get_mem_object_info(cl_mem mem, cl_mem_info param_name, size_t param_value_size,
                                     void *param_value, size_t *param_value_size_ret)
{
  switch(param_name) {
      FIELD_SIZE(MEM_TYPE, cl_mem_object_type);
      FIELD_SIZE(MEM_FLAGS, cl_mem_flags);
      FIELD_SIZE(MEM_SIZE, size_t);
      FIELD_SIZE(MEM_HOST_PTR, void *);
      FIELD_SIZE(MEM_MAP_COUNT, cl_uint);
      FIELD_SIZE(MEM_REFERENCE_COUNT, cl_uint);
      FIELD_SIZE(MEM_CONTEXT, cl_context);
      FIELD_SIZE(MEM_ASSOCIATED_MEMOBJECT, cl_mem);
      FIELD_SIZE(MEM_OFFSET, size_t);
    default:
      return CL_INVALID_VALUE;
  }

  switch(param_name) {
    case CL_MEM_TYPE:
      *((cl_mem_object_type *)param_value) = mem->type;
      break;
    case CL_MEM_FLAGS:
      *((cl_mem_flags *)param_value) = mem->flags;
      break;
    case CL_MEM_SIZE:
      *((size_t *)param_value) = mem->size;
      break;
    case CL_MEM_HOST_PTR:
      if(mem->type != CL_MEM_OBJECT_BUFFER) {
        *((size_t *)param_value) = (size_t)mem->host_ptr;
      } else {
        cl_mem_buffer buf = cl_mem_to_buffer(mem);
        *((size_t *)param_value) = (size_t)mem->host_ptr + buf->sub_offset;
      }
      break;
    case CL_MEM_MAP_COUNT:
      *((cl_uint *)param_value) = mem->map_ref;
      break;
    case CL_MEM_REFERENCE_COUNT:
      *((cl_uint *)param_value) = mem->ref_n;
      break;
    case CL_MEM_CONTEXT:
      *((cl_context *)param_value) = mem->ctx;
      break;
    case CL_MEM_ASSOCIATED_MEMOBJECT:
      if(mem->type != CL_MEM_OBJECT_BUFFER) {
        *((cl_mem *)param_value) = NULL;
      } else {
        cl_mem_buffer buf = cl_mem_to_buffer(mem);
        *((cl_mem *)param_value) = (cl_mem)(buf->parent);
      }
      break;
    case CL_MEM_OFFSET:
      if(mem->type != CL_MEM_OBJECT_BUFFER) {
        *((size_t *)param_value) = 0;
      } else {
        cl_mem_buffer buf = cl_mem_to_buffer(mem);
        *((size_t *)param_value) = buf->sub_offset;
      }
      break;
  }

  return CL_SUCCESS;
}

#undef FIELD_SIZE

LOCAL void cl_retain_mem(cl_mem mem)
{
  assert(mem);
  atomic_inc(&mem->ref_n);
}

LOCAL cl_mem cl_mem_new(cl_mem_object_type type, cl_context ctx, cl_mem_flags flags, size_t sz)
{
  cl_mem mem = NULL;

  /* Allocate and inialize the structure itself */
  if (type == CL_MEM_OBJECT_IMAGE2D || type == CL_MEM_OBJECT_IMAGE3D
      || type == CL_MEM_OBJECT_IMAGE2D_ARRAY || type == CL_MEM_OBJECT_IMAGE1D
      || type == CL_MEM_OBJECT_IMAGE1D_ARRAY || type == CL_MEM_OBJECT_IMAGE1D_BUFFER) {
    struct _cl_mem_image *image = NULL;
    image = CALLOC(struct _cl_mem_image);
    mem = &image->base;
  } else {
    struct _cl_mem_buffer *buffer = NULL;
    buffer = CALLOC(struct _cl_mem_buffer);
    mem = &buffer->base;
  }

  if (mem == NULL)
    return NULL;

  /* Create the private pointer array if device > 1 */
  if (ctx->device_num > 1) {
    mem->pdata = CALLOC_ARRAY(void*, ctx->device_num);
    if (mem->pdata == NULL) {
      cl_free(mem);
      return NULL;
    }
  }
  mem->type = type;
  mem->ref_n = 1;
  mem->magic = CL_MAGIC_MEM_HEADER;
  mem->flags = flags;
  mem->offset = 0;
  mem->size = sz;
  cl_retain_context(ctx);
  mem->ctx = ctx;

  /* Append the buffer in the context buffer list */
  pthread_mutex_lock(&ctx->lock);
  mem->next = ctx->buffers;
  if (ctx->buffers != NULL)
    ctx->buffers->prev = mem;
  ctx->buffers = mem;
  pthread_mutex_unlock(&ctx->lock);

  return mem;
}

static void cl_mem_delete(cl_mem mem)
{
  assert(mem);
  /* Remove it from the list */
  if (mem->ctx) {
    pthread_mutex_lock(&mem->ctx->lock);
    if (mem->prev)
      mem->prev->next = mem->next;
    if (mem->next)
      mem->next->prev = mem->prev;
    if (mem->ctx->buffers == mem)
      mem->ctx->buffers = mem->next;
    pthread_mutex_unlock(&mem->ctx->lock);
    cl_release_context(mem->ctx);
  } else {
    assert((mem->prev == 0) && (mem->next == 0));
  }

  if (mem->mapped_ptr)
    free(mem->mapped_ptr); // Allocated by users

  if (mem->dstr_cb)
    cl_free(mem->dstr_cb);

  if (mem->ctx->device_num > 1) {
    cl_free(mem->pdata);
  }
  cl_free(mem);
}

LOCAL void cl_release_mem(cl_mem mem)
{
  cl_uint i;

  if (UNLIKELY(mem == NULL))
    return;
  if (atomic_dec(&mem->ref_n) > 1)
    return;

  for (i = 0; i < mem->ctx->device_num; i++) {
    mem->ctx->devices[i]->driver->release_mem(mem, mem->ctx->devices[i]);
  }

  /* iff we are a image, delete the buffer we use. */
  if (IS_IMAGE(mem)) {
    if (cl_mem_to_image(mem)->buffer_from) {
      assert(mem->type == CL_MEM_OBJECT_IMAGE1D_BUFFER || mem->type == CL_MEM_OBJECT_IMAGE2D);
      cl_release_mem(cl_mem_to_image(mem)->buffer_from);
    }
  }

  /* Someone still mapped? Assert */
  if (mem->map_ref != 0) {
    FATAL("Free the mem %p, while there are still %d user mapping it\n", mem, mem->map_ref);
  }

  /* Call the user callback. */
  if (mem->dstr_cb) {
    cl_mem_dstr_cb *cb = mem->dstr_cb;
    while (mem->dstr_cb) {
      cb = mem->dstr_cb;
      cb->pfn_notify(mem, cb->user_data);
      mem->dstr_cb = cb->next;
    }
  }

  /* Iff we are sub. */
  if (cl_mem_to_buffer(mem)->parent) {
    cl_mem_buffer buffer = cl_mem_to_buffer(mem);
    /* Remove it from the parent's list */
    pthread_mutex_lock(&buffer->parent->sub_lock);
    if (buffer->sub_prev)
      buffer->sub_prev->sub_next = buffer->sub_next;
    if (buffer->sub_next)
      buffer->sub_next->sub_prev = buffer->sub_prev;
    if (buffer->parent->subs == buffer)
      buffer->parent->subs = buffer->sub_next;
    pthread_mutex_unlock(&buffer->parent->sub_lock);
    cl_release_mem((cl_mem )(buffer->parent));
  }

  cl_mem_delete(mem);
}

static cl_mem cl_mem_create_buffer(cl_context ctx, cl_mem_flags flags, size_t sz,
                                   void *data, cl_int *errcode_ret)
{
  /* Possible mem type combination:
       CL_MEM_ALLOC_HOST_PTR
       CL_MEM_ALLOC_HOST_PTR | CL_MEM_COPY_HOST_PTR
       CL_MEM_USE_HOST_PTR
       CL_MEM_COPY_HOST_PTR   */

  cl_int err = CL_SUCCESS;
  cl_mem mem = NULL;
  cl_ulong max_mem_size;
  cl_uint i = 0;

  if (UNLIKELY(sz == 0)) {
    err = CL_INVALID_BUFFER_SIZE;
    goto error;
  }

  if (UNLIKELY(((flags & CL_MEM_READ_WRITE)
                && (flags & (CL_MEM_READ_ONLY | CL_MEM_WRITE_ONLY)))
               || ((flags & CL_MEM_READ_ONLY) && (flags & (CL_MEM_WRITE_ONLY)))
               || ((flags & CL_MEM_ALLOC_HOST_PTR) && (flags & CL_MEM_USE_HOST_PTR))
               || ((flags & CL_MEM_COPY_HOST_PTR) && (flags & CL_MEM_USE_HOST_PTR))
               || ((flags & CL_MEM_HOST_READ_ONLY) && (flags & CL_MEM_HOST_NO_ACCESS))
               || ((flags & CL_MEM_HOST_READ_ONLY) && (flags & CL_MEM_HOST_WRITE_ONLY))
               || ((flags & CL_MEM_HOST_WRITE_ONLY) && (flags & CL_MEM_HOST_NO_ACCESS))
               || ((flags & (~(CL_MEM_READ_WRITE | CL_MEM_WRITE_ONLY | CL_MEM_READ_ONLY
                               | CL_MEM_ALLOC_HOST_PTR | CL_MEM_COPY_HOST_PTR
                               | CL_MEM_USE_HOST_PTR | CL_MEM_HOST_WRITE_ONLY
                               | CL_MEM_HOST_READ_ONLY | CL_MEM_HOST_NO_ACCESS))) != 0))) {
    err = CL_INVALID_VALUE;
    goto error;
  }

  /* This flag is valid only if host_ptr is not NULL */
  if (UNLIKELY((((flags & CL_MEM_COPY_HOST_PTR) ||
                 (flags & CL_MEM_USE_HOST_PTR)) &&
                data == NULL))
      || (!(flags & (CL_MEM_COPY_HOST_PTR
                     |CL_MEM_USE_HOST_PTR))
          && (data != NULL))) {
    err = CL_INVALID_HOST_PTR;
    goto error;
  }

  /* return CL_INVALID_BUFFER_SIZE if size is greater than
     CL_DEVICE_MAX_MEM_ALLOC_SIZE value for all devices in context. */
  for (i = 0; i < ctx->device_num; i++) {
    cl_device_id device = ctx->devices[i];

    if ((err = cl_get_device_info(device, CL_DEVICE_MAX_MEM_ALLOC_SIZE, sizeof(max_mem_size),
                                  &max_mem_size, NULL)) != CL_SUCCESS) {
      goto error;
    }

    if (UNLIKELY(sz > max_mem_size)) {
      err = CL_INVALID_BUFFER_SIZE;
      goto error;
    }
  }

  mem = cl_mem_new(CL_MEM_OBJECT_BUFFER, ctx, flags, sz);
  if (mem == NULL) {
    err = CL_OUT_OF_HOST_MEMORY;
    goto error;
  }

  if (flags & CL_MEM_USE_HOST_PTR)
    mem->host_ptr = data;

  for (i = 0; i < ctx->device_num; i++) {
    err = ctx->devices[i]->driver->create_buffer(mem,
          ctx->devices[i], ctx, flags, sz, data);
    if (err != CL_SUCCESS)
      goto error;
  }

exit:
  if (errcode_ret)
    *errcode_ret = err;
  return mem;

error:
  cl_release_mem(mem);
  mem = NULL;
  goto exit;
}

static cl_mem cl_mem_create_sub_buffer(cl_mem buffer, cl_mem_flags flags, cl_buffer_create_type create_type,
                                       const void *create_info, cl_int *errcode_ret)
{
  cl_int err = CL_SUCCESS;
  cl_mem mem = NULL;
  struct _cl_mem_buffer *sub_buf = NULL;
  cl_uint i;

  if (buffer->type != CL_MEM_OBJECT_BUFFER) {
    err = CL_INVALID_MEM_OBJECT;
    goto error;
  }

  if (flags && (((buffer->flags & CL_MEM_WRITE_ONLY) && (flags & (CL_MEM_READ_WRITE|CL_MEM_READ_ONLY)))
                || ((buffer->flags & CL_MEM_READ_ONLY) && (flags & (CL_MEM_READ_WRITE|CL_MEM_WRITE_ONLY)))
                || (flags & (CL_MEM_USE_HOST_PTR | CL_MEM_ALLOC_HOST_PTR | CL_MEM_COPY_HOST_PTR))
                || ((flags & CL_MEM_HOST_READ_ONLY) && (flags & CL_MEM_HOST_NO_ACCESS))
                || ((flags & CL_MEM_HOST_READ_ONLY) && (flags & CL_MEM_HOST_WRITE_ONLY))
                || ((flags & CL_MEM_HOST_WRITE_ONLY) && (flags & CL_MEM_HOST_NO_ACCESS)))) {
    err = CL_INVALID_VALUE;
    goto error;
  }

  if((flags & (CL_MEM_WRITE_ONLY | CL_MEM_READ_ONLY | CL_MEM_READ_WRITE)) == 0) {
    flags |= buffer->flags & (CL_MEM_WRITE_ONLY | CL_MEM_READ_ONLY | CL_MEM_READ_WRITE);
  }
  flags |= buffer->flags & (CL_MEM_USE_HOST_PTR | CL_MEM_ALLOC_HOST_PTR | CL_MEM_COPY_HOST_PTR);
  if((flags & (CL_MEM_HOST_WRITE_ONLY | CL_MEM_HOST_READ_ONLY | CL_MEM_HOST_NO_ACCESS)) == 0) {
    flags |= buffer->flags & (CL_MEM_HOST_WRITE_ONLY | CL_MEM_HOST_READ_ONLY | CL_MEM_HOST_NO_ACCESS);
  }

  if (create_type != CL_BUFFER_CREATE_TYPE_REGION) {
    err = CL_INVALID_VALUE;
    goto error;
  }

  if (!create_info) {
    err = CL_INVALID_VALUE;
    goto error;
  }

  cl_buffer_region *info = (cl_buffer_region *)create_info;

  if (!info->size) {
    err = CL_INVALID_BUFFER_SIZE;
    goto error;
  }

  if (info->origin > buffer->size || info->origin + info->size > buffer->size) {
    err = CL_INVALID_VALUE;
    goto error;
  }

  for (i = 0; i < buffer->ctx->device_num; i++) {
    if (info->origin & (buffer->ctx->devices[i]->mem_base_addr_align / 8 - 1)) {
      err = CL_MISALIGNED_SUB_BUFFER_OFFSET;
      goto error;
    }
  }

  /* Now create the sub buffer and link it to the context. */
  mem = cl_mem_new(CL_MEM_OBJECT_BUFFER, buffer->ctx, flags, info->size);
  if (mem == NULL) {
    err = CL_OUT_OF_HOST_MEMORY;
    goto error;
  }

  if (mem == NULL || err != CL_SUCCESS)
    goto error;

  sub_buf = cl_mem_to_buffer(mem);
  mem->offset = buffer->offset;

  /* Hold a ref of parent. */
  cl_retain_mem(buffer);
  sub_buf->parent = (struct _cl_mem_buffer*)buffer;

  /* Append the buffer in the parent buffer list */
  pthread_mutex_lock(&((struct _cl_mem_buffer*)buffer)->sub_lock);
  sub_buf->sub_next = ((struct _cl_mem_buffer*)buffer)->subs;
  if (((struct _cl_mem_buffer*)buffer)->subs != NULL)
    ((struct _cl_mem_buffer*)buffer)->subs->sub_prev = sub_buf;
  ((struct _cl_mem_buffer*)buffer)->subs = sub_buf;
  pthread_mutex_unlock(&((struct _cl_mem_buffer*)buffer)->sub_lock);

  mem->size = info->size;
  sub_buf->sub_offset = info->origin;
  if (buffer->flags & CL_MEM_USE_HOST_PTR || buffer->flags & CL_MEM_COPY_HOST_PTR) {
    mem->host_ptr = buffer->host_ptr;
  }

  for (i = 0; i < mem->ctx->device_num; i++) {
    err = mem->ctx->devices[i]->driver->create_subbuffer(mem,
          mem->ctx->devices[i], buffer, flags, create_type, create_info);
    if (err != CL_SUCCESS)
      goto error;
  }

exit:
  if (errcode_ret)
    *errcode_ret = err;
  return mem;
error:
  cl_release_mem(mem);
  mem = NULL;
  goto exit;
}

/**************************************************************************************
 *************************           CL APIs           ********************************
 **************************************************************************************/
cl_mem
clCreateBuffer(cl_context    context,
               cl_mem_flags  flags,
               size_t        size,
               void *        host_ptr,
               cl_int *      errcode_ret)
{
  cl_mem mem = NULL;
  cl_int err = CL_SUCCESS;
  CHECK_CONTEXT (context);

  mem = cl_mem_create_buffer(context, flags, size, host_ptr, &err);
error:
  if (errcode_ret)
    *errcode_ret = err;
  return mem;
}

cl_mem
clCreateSubBuffer(cl_mem                buffer,
                  cl_mem_flags          flags,
                  cl_buffer_create_type buffer_create_type,
                  const void *          buffer_create_info,
                  cl_int *              errcode_ret)
{
  cl_mem mem = NULL;
  cl_int err = CL_SUCCESS;

  CHECK_MEM(buffer);

  mem = cl_mem_create_sub_buffer(buffer, flags, buffer_create_type,
                                 buffer_create_info, &err);
error:
  if (errcode_ret)
    *errcode_ret = err;
  return mem;
}

cl_int
clRetainMemObject(cl_mem memobj)
{
  cl_int err = CL_SUCCESS;
  CHECK_MEM(memobj);
  cl_retain_mem(memobj);
error:
  return err;
}

cl_int
clReleaseMemObject(cl_mem memobj)
{
  cl_int err = CL_SUCCESS;
  CHECK_MEM(memobj);
  cl_release_mem(memobj);
error:
  return err;
}

cl_int
clSetMemObjectDestructorCallback(cl_mem  memobj,
                                 void (CL_CALLBACK *pfn_notify) (cl_mem, void*),
                                 void * user_data)
{
  cl_int err = CL_SUCCESS;
  CHECK_MEM(memobj);
  INVALID_VALUE_IF (pfn_notify == 0);

  cl_mem_dstr_cb *cb = (cl_mem_dstr_cb*)malloc(sizeof(cl_mem_dstr_cb));
  if (!cb) {
    err = CL_OUT_OF_HOST_MEMORY;
    goto error;
  }

  memset(cb, 0, sizeof(cl_mem_dstr_cb));
  cb->pfn_notify = pfn_notify;
  cb->user_data = user_data;
  cb->next = memobj->dstr_cb;
  memobj->dstr_cb = cb;

error:
  return err;
}

cl_int
clGetMemObjectInfo(cl_mem      memobj,
                   cl_mem_info param_name,
                   size_t      param_value_size,
                   void *      param_value,
                   size_t *    param_value_size_ret)
{
  cl_int err = CL_SUCCESS;
  CHECK_MEM(memobj);

  err = cl_get_mem_object_info(memobj, param_name, param_value_size,
                               param_value, param_value_size_ret);
error:
  return err;
}