path: root/backend/src/driver/cl_gen_kernel.cpp

/*
 * Copyright © 2012 Intel Corporation
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library. If not, see <http://www.gnu.org/licenses/>.
 *
 */

#ifdef __cplusplus
extern "C" {  // for the C header files
#endif /* __cplusplus */
#include <string.h>
#include "cl_program.h"
#include "cl_kernel.h"
#include "cl_image.h"
#include "cl_gen_driver.h"
#include "cl_gen_devices.h"
#ifdef __cplusplus
}
#endif /* __cplusplus */
#include "sys/assert.hpp"
#include "sys/alloc.hpp"
#include "cl_gen_driver.hpp"
#include "backend/program.hpp"

using namespace gbe;

static int checkBuiltinKernelDimension(cl_kernel kernel, cl_device_id device)
{
  const char * n = kernel->name;
  const char * builtin_kernels_2d = "__cl_copy_image_2d_to_2d;__cl_copy_image_2d_to_buffer;"
                                    "__cl_copy_buffer_to_image_2d;__cl_fill_image_2d;__cl_fill_image_2d_array;";
  const char * builtin_kernels_3d = "__cl_copy_image_3d_to_2d;__cl_copy_image_2d_to_3d;__cl_copy_image_3d_to_3d;"
                                    "__cl_copy_image_3d_to_buffer;__cl_copy_buffer_to_image_3d;__cl_fill_image_3d";
  if (!strstr(device->built_in_kernels, n)) {
    return 0;
  } else if (strstr(builtin_kernels_2d, n)) {
    return 2;
  } else if (strstr(builtin_kernels_3d, n)) {
    return 3;
  } else
    return 1;
}

static size_t genGetKernelMaxWorkGroupSize(cl_kernel kernel, Kernel* ker, const cl_device_id device)
{
  size_t work_group_size, thread_cnt;
  int simd_width = ker->getSIMDWidth();
  GenGPUDevice* gpuDev = reinterpret_cast<GenGPUDevice*>(getGenDevicePrivate(device));
  GBE_ASSERT(gpuDev);

  if (!ker->getUseSLM()) {
    if (!IS_BAYTRAIL_T(gpuDev->device_id) || simd_width == 16)
      work_group_size = simd_width * 64;
    else
      work_group_size = device->max_compute_unit * gpuDev->max_thread_per_unit * simd_width;

  } else {
    thread_cnt = device->max_compute_unit *
                 gpuDev->max_thread_per_unit / gpuDev->sub_slice_count;
    if(thread_cnt > 64)
      thread_cnt = 64;
    work_group_size = thread_cnt * simd_width;
  }

  if(work_group_size > device->max_work_group_size)
    work_group_size = device->max_work_group_size;
  return work_group_size;
}

extern "C"
cl_int GenGetKernelWorkgroupInfo(cl_kernel kernel, const cl_device_id device, cl_kernel_workgroup_info wgInfo)
{
  Kernel* ker = reinterpret_cast<Kernel*>(getGenKernelPrivate(kernel, device));
  if (ker == NULL)
    return CL_INVALID_VALUE;

  ker->getCompileWorkGroupSize(wgInfo->compile_wg_sz);
  wgInfo->local_mem_sz = ker->getSLMSize();
  wgInfo->private_mem_sz = ker->getStackSize();
  wgInfo->work_group_sz = genGetKernelMaxWorkGroupSize(kernel, ker, device);
  wgInfo->work_group_sz_multiple = ker->getSIMDWidth();

  int dimension = checkBuiltinKernelDimension(kernel, device);
  memset(wgInfo->global_work_sz, 0, sizeof(wgInfo->global_work_sz));
  if (dimension == 1) {
    memcpy(wgInfo->global_work_sz, device->max_1d_global_work_sizes, sizeof(device->max_1d_global_work_sizes));
  } else if (dimension == 2) {
    memcpy(wgInfo->global_work_sz, device->max_2d_global_work_sizes, sizeof(device->max_2d_global_work_sizes));
  } else if (dimension == 3) {
    memcpy(wgInfo->global_work_sz, device->max_3d_global_work_sizes, sizeof(device->max_3d_global_work_sizes));
  }

  return CL_SUCCESS;
}

extern "C"
cl_int GenCreateKernel(cl_kernel kernel, const cl_device_id device)
{
  GBE_ASSERT(getGenKernelPrivate(kernel, device) == NULL);

  Program* p = reinterpret_cast<Program*>(getGenProgramPrivate(kernel->program, device));
  if (p == NULL)
    return CL_INVALID_PROGRAM;

  Kernel* ker = p->getKernel(kernel->name);
  if (ker == NULL)
    return CL_INVALID_KERNEL_NAME;

  setGenKernelPrivate(kernel, device, ker);
  return CL_SUCCESS;
}

extern "C"
cl_int GenReleaseKernel(cl_kernel kernel, const cl_device_id device)
{
  Kernel* ker = reinterpret_cast<Kernel*>(getGenKernelPrivate(kernel, device));
  if (ker == NULL)
    return CL_INVALID_VALUE;

  setGenKernelPrivate(kernel, device, NULL);
  return CL_SUCCESS;
}

extern "C"
cl_int GenGetKernelAttr(cl_kernel kernel, const cl_device_id device, cl_uint size, char* attr, cl_uint* ret_sz)
{
  Kernel* ker = reinterpret_cast<Kernel*>(getGenKernelPrivate(kernel, device));
  if (ker == NULL)
    return CL_INVALID_VALUE;

  cl_uint sz = 0;
  const char* attr_str = ker->getFunctionAttributes();
  if (attr_str && attr_str[0] != 0) {
    sz = strlen(attr_str) + 1;
  }

  if (ret_sz)
    *ret_sz = sz;

  if (!attr)
    return CL_SUCCESS;

  if (sz && sz <= size) {
    strcpy(attr, attr_str);
    return CL_SUCCESS;
  }

  return CL_INVALID_VALUE;
}

extern "C"
cl_int GenGetKernelArgNum(cl_kernel kernel, const cl_device_id device, cl_uint* ret_num)
{
  Kernel* ker = reinterpret_cast<Kernel*>(getGenKernelPrivate(kernel, device));
  if (ker == NULL)
    return CL_INVALID_VALUE;

  if (ret_num)
    *ret_num = ker->getArgNum();

  return CL_SUCCESS;
}

extern "C"
cl_int GenGetKernelArgName(cl_kernel kernel, const cl_device_id device, cl_uint index,
                           char *name, cl_uint name_sz, cl_uint* ret_sz)
{
  Kernel* ker = reinterpret_cast<Kernel*>(getGenKernelPrivate(kernel, device));
  if (ker == NULL)
    return CL_INVALID_VALUE;

  if (index >= ker->getArgNum())
    return CL_INVALID_VALUE;

  KernelArgument::ArgInfo* info = ker->getArgInfo(index);
  if (info == NULL)
    return CL_INVALID_VALUE;

  cl_uint sz = strlen(info->argName.c_str()) + 1;

  if (ret_sz)
    *ret_sz = sz;

  if (!name)
    return CL_SUCCESS;

  if (sz <= name_sz) {
    strcpy(name, info->argName.c_str());
    return CL_SUCCESS;
  }

  return CL_INVALID_VALUE;
}

extern "C"
cl_int GenGetKernelArgTypeName(cl_kernel kernel, const cl_device_id device, cl_uint index,
                               char *name, cl_uint name_sz, cl_uint* ret_sz)
{
  Kernel* ker = reinterpret_cast<Kernel*>(getGenKernelPrivate(kernel, device));
  if (ker == NULL)
    return CL_INVALID_VALUE;

  if (index >= ker->getArgNum())
    return CL_INVALID_VALUE;

  KernelArgument::ArgInfo* info = ker->getArgInfo(index);
  if (info == NULL)
    return CL_INVALID_VALUE;

  cl_uint sz = strlen(info->typeName.c_str()) + 1;

  if (ret_sz)
    *ret_sz = sz;

  if (!name)
    return CL_SUCCESS;

  if (sz <= name_sz) {
    strcpy(name, info->typeName.c_str());
    return CL_SUCCESS;
  }

  return CL_INVALID_VALUE;
}

extern "C"
cl_int GenGetKernelArgInfo(cl_kernel kernel, const cl_device_id device, cl_uint index, size_t* size,
                           cl_kernel_arg_type *type, cl_kernel_arg_address_qualifier *qualifier,
                           cl_kernel_arg_access_qualifier *access, cl_kernel_arg_type_qualifier *type_qualifier)
{
  Kernel* ker = reinterpret_cast<Kernel*>(getGenKernelPrivate(kernel, device));
  if (ker == NULL)
    return CL_INVALID_VALUE;

  if (index >= ker->getArgNum())
    return CL_INVALID_VALUE;

  gbe_arg_type gbeType = ker->getArgType(index);
  if (gbeType) {
    if (gbeType == GBE_ARG_GLOBAL_PTR) {
      *type = CL_KERNEL_ARG_GLOBAL_PTR;
    } else if (gbeType == GBE_ARG_CONSTANT_PTR) {
      *type = CL_KERNEL_ARG_CONST_PTR;
    } else if (gbeType == GBE_ARG_LOCAL_PTR) {
      *type = CL_KERNEL_ARG_LOCAL_PTR;
    } else if (gbeType == GBE_ARG_VALUE) {
      *type = CL_KERNEL_ARG_VALUE;
    } else if (gbeType == GBE_ARG_IMAGE) {
      *type = CL_KERNEL_ARG_IMAGE;
    } else if (gbeType == GBE_ARG_SAMPLER) {
      *type = CL_KERNEL_ARG_SAMPLER;
    } else
      GBE_ASSERT(0);
  }

  if (size) {
    *size = ker->getArgSize(index);
  }

  if (qualifier == NULL && access == NULL && type_qualifier == NULL)
    return CL_SUCCESS;

  KernelArgument::ArgInfo* info = ker->getArgInfo(index);
  if (info == NULL)
    return CL_INVALID_VALUE;

  if (qualifier) {
    if (info->addrSpace == 0) {
      *qualifier = CL_KERNEL_ARG_ADDRESS_PRIVATE;
    } else if (info->addrSpace == 1 || info->addrSpace == 4) {
      *qualifier = CL_KERNEL_ARG_ADDRESS_GLOBAL;
    } else if (info->addrSpace == 2) {
      *qualifier = CL_KERNEL_ARG_ADDRESS_CONSTANT;
    } else if (info->addrSpace == 3) {
      *qualifier = CL_KERNEL_ARG_ADDRESS_LOCAL;
    } else {
      /* If no address qualifier is specified, the default address qualifier
         which is CL_KERNEL_ARG_ADDRESS_PRIVATE is returned. */
      *qualifier = CL_KERNEL_ARG_ADDRESS_PRIVATE;
    }
  }

  if (access) {
    if (!strcmp(info->accessQual.c_str(), "write_only")) {
      *access = CL_KERNEL_ARG_ACCESS_WRITE_ONLY;
    } else if (!strcmp(info->accessQual.c_str(), "read_only")) {
      *access = CL_KERNEL_ARG_ACCESS_READ_ONLY;
    } else if (!strcmp(info->accessQual.c_str(), "read_write")) {
      *access = CL_KERNEL_ARG_ACCESS_READ_WRITE;
    } else {
      *access = CL_KERNEL_ARG_ACCESS_NONE;
    }
  }

  if (type_qualifier) {
    cl_kernel_arg_type_qualifier type_qual = CL_KERNEL_ARG_TYPE_NONE;
    if (strstr(info->typeQual.c_str(), "const") &&
        (gbeType == GBE_ARG_GLOBAL_PTR ||
         gbeType == GBE_ARG_CONSTANT_PTR ||
         gbeType == GBE_ARG_LOCAL_PTR))
      type_qual = type_qual | CL_KERNEL_ARG_TYPE_CONST;
    if (strstr(info->typeQual.c_str(), "volatile"))
      type_qual = type_qual | CL_KERNEL_ARG_TYPE_VOLATILE;
    if (strstr(info->typeQual.c_str(), "restrict"))
      type_qual = type_qual | CL_KERNEL_ARG_TYPE_RESTRICT;
    *type_qualifier = type_qual;
  }

  return CL_SUCCESS;
}

GenGPUWorkItemNDRange::GenGPUWorkItemNDRange(dri_bufmgr *bufmgr, drm_intel_context *ctx, int device_id,
    cl_event event, const cl_event* dependEvents, cl_uint num_events)
  : GenGPUWorkItem(event, dependEvents, num_events)
{
  if (IS_GEN9(device_id)) {

  } else if (IS_GEN8(device_id)) {

  } else if (IS_GEN75(device_id)) {

  } else if (IS_GEN7(device_id)) {
    this->gpuState = GBE_NEW(Gen7GPUState, bufmgr, ctx, device_id);
  } else
    GBE_ASSERT(0); // not support any more.
}

bool GenGPUWorkItemNDRange::submit(void)
{
  bool ret;
  ret = this->gpuState->flush();
  return ret;
}

bool GenGPUWorkItemNDRange::complete(void)
{
  /* Wait it to complete. */
  this->gpuState->sync();
  return true;
}

static cl_int genNDRangeRun(cl_command_queue_work_item it)
{
  GBE_ASSERT(0);
  return CL_SUCCESS;
}

static cl_int genNDRangeSubmit(cl_command_queue_work_item it)
{
  bool ret;
  GenGPUState* gpuState = reinterpret_cast<GenGPUState*>(it->pdata);
  ret = gpuState->flush();
  if (ret)
    return CL_RUNNING; // set to running, we do not have run function.

  return -1;
}

static cl_int genNDRangeComplete(cl_command_queue_work_item it)
{
  GenGPUState* gpuState = reinterpret_cast<GenGPUState*>(it->pdata);
  /* Wait it to complete. */
  gpuState->sync();
  return CL_COMPLETE;
}

/* Will return the total amount of slm used */
static int32_t genFillCurbe(cl_kernel kernel, Kernel* ker, char* curbe, const uint32_t work_dim,
                            const size_t *global_wk_off, const size_t *global_wk_sz,
                            const size_t *local_wk_sz, size_t thread_n)
{
  int32_t offset;
#define UPLOAD(ENUM, VALUE) \
       if ((offset = ker->getCurbeOffset(ENUM, 0)) >= 0) \
          *((uint32_t *) (curbe + offset)) = VALUE;

  UPLOAD(GBE_CURBE_LOCAL_SIZE_X, local_wk_sz[0]);
  UPLOAD(GBE_CURBE_LOCAL_SIZE_Y, local_wk_sz[1]);
  UPLOAD(GBE_CURBE_LOCAL_SIZE_Z, local_wk_sz[2]);
  UPLOAD(GBE_CURBE_GLOBAL_SIZE_X, global_wk_sz[0]);
  UPLOAD(GBE_CURBE_GLOBAL_SIZE_Y, global_wk_sz[1]);
  UPLOAD(GBE_CURBE_GLOBAL_SIZE_Z, global_wk_sz[2]);
  UPLOAD(GBE_CURBE_GLOBAL_OFFSET_X, global_wk_off[0]);
  UPLOAD(GBE_CURBE_GLOBAL_OFFSET_Y, global_wk_off[1]);
  UPLOAD(GBE_CURBE_GLOBAL_OFFSET_Z, global_wk_off[2]);
  UPLOAD(GBE_CURBE_GROUP_NUM_X, global_wk_sz[0]/local_wk_sz[0]);
  UPLOAD(GBE_CURBE_GROUP_NUM_Y, global_wk_sz[1]/local_wk_sz[1]);
  UPLOAD(GBE_CURBE_GROUP_NUM_Z, global_wk_sz[2]/local_wk_sz[2]);
  UPLOAD(GBE_CURBE_THREAD_NUM, thread_n);
  UPLOAD(GBE_CURBE_WORK_DIM, work_dim);
#undef UPLOAD

  /* Handle the various offsets to SLM */
  const int32_t arg_n = ker->getArgNum();
  int32_t arg;
  int32_t slm_offset = ker->getSLMSize();

  for (arg = 0; arg < arg_n; ++arg) {
    const enum gbe_arg_type type = ker->getArgType(arg);
    if (type != GBE_ARG_LOCAL_PTR)
      continue;
    uint32_t align = ker->getArgAlign(arg);
    assert(align != 0);
    slm_offset = ALIGN(slm_offset, align);
    offset = ker->getCurbeOffset(GBE_CURBE_KERNEL_ARGUMENT, arg);
    if (offset < 0)
      continue;
    uint32_t *slmptr = (uint32_t *)(curbe + offset);
    *slmptr = slm_offset;
    slm_offset += kernel->args[arg]->local_sz;
  }

  return slm_offset;
}

static cl_int genGPUBindSurfaces(GenGPUState& gpuState, cl_command_queue queue, cl_kernel kernel, Kernel* ker)
{
  /* Bind all user buffers (given by clSetKernelArg) */
  uint32_t i;
  enum gbe_arg_type arg_type; /* kind of argument */
  for (i = 0; i < ker->getArgNum(); ++i) {
    int32_t offset; // location of the address in the curbe
    arg_type = ker->getArgType(i);
    cl_mem mem = kernel->args[i]->mem;
    if (arg_type != GBE_ARG_GLOBAL_PTR || !mem)
      continue;

    offset = ker->getCurbeOffset(GBE_CURBE_KERNEL_ARGUMENT, i);
    if (offset < 0)
      continue;

    cl_mem_buffer buffer = cl_mem_to_buffer(mem);
    uint8_t bti = ker->getArgBTI(i);
    GenGPUMem* genMem = reinterpret_cast<GenGPUMem*>(getGenMemPrivate(mem, queue->device));
    GBE_ASSERT(genMem != NULL);
    GBE_ASSERT(genMem->bo != NULL);

    if (genMem->alignedHostPtr == NULL) {
      gpuState.bindBuf(genMem->bo, offset, buffer->sub_offset, genMem->realSize, bti);
    } else {
      gpuState.bindBuf(genMem->bo, offset, (char*)mem->host_ptr - (char*)genMem->alignedHostPtr
                       + buffer->sub_offset, genMem->realSize, bti);
    }
  }

  return CL_SUCCESS;
}

static cl_int genGPUBindImage(char* curbe, GenGPUState& gpuState, cl_kernel kernel, Kernel* ker)
{
  uint32_t i;
  size_t image_sz = ker->getImageSize();
  ImageInfo *images = (ImageInfo *)alloca(image_sz * sizeof(ImageInfo));
  if (image_sz > 0)
    ker->getImageData(images);

  for (i = 0; i < image_sz; i++) {
    int id = images[i].arg_idx;
    cl_mem_image image;
    GBE_ASSERT(ker->getArgType(id) == GBE_ARG_IMAGE);

    image = cl_mem_to_image(kernel->args[id]->mem);

    if (images[i].wSlot >= 0)
      *(uint32_t*)(curbe + images[i].wSlot) = image->w;
    if (images[i].hSlot >= 0)
      *(uint32_t*)(curbe + images[i].hSlot) = image->h;
    if (images[i].depthSlot >= 0)
      *(uint32_t*)(curbe + images[i].depthSlot) = image->depth;
    if (images[i].channelOrderSlot >= 0)
      *(uint32_t*)(curbe + images[i].channelOrderSlot) = image->fmt.image_channel_order;
    if (images[i].dataTypeSlot >= 0)
      *(uint32_t*)(curbe + images[i].dataTypeSlot) = image->fmt.image_channel_data_type;
    /*
        gpuState.bindImage(images[i].idx, image->base.bo,
                            image->offset + k->args[id].mem->offset, image->intel_fmt,
                            image->image_type, image->bpp, image->w, image->h, image->depth,
                            image->row_pitch, image->slice_pitch, (cl_gpgpu_tiling)image->tiling);
    // TODO, this workaround is for GEN7/GEN75 only, we may need to do it in the driver layer
    // on demand.
    if (image->image_type == CL_MEM_OBJECT_IMAGE1D_ARRAY)
      cl_gpgpu_bind_image(gpgpu, k->images[i].idx + BTI_WORKAROUND_IMAGE_OFFSET, image->base.bo, image->offset + k->args[id].mem->offset,
    					  image->intel_fmt, image->image_type, image->bpp,
    					  image->w, image->h, image->depth,
    					  image->row_pitch, image->slice_pitch, (cl_gpgpu_tiling)image->tiling);

    */
  }

  return CL_SUCCESS;
}

static cl_int genGPUBindSamplers(GenGPUState& gpuState, cl_kernel kernel, Kernel* ker)
{
  size_t sz = ker->getSamplerSize();
  uint32_t *samplers = (uint32_t *)alloca(GEN_MAX_SAMPLERS * sizeof(uint32_t));
  ker->getSamplerData(samplers);
  gpuState.bindSamplers(samplers, sz);
  return CL_SUCCESS;
}

static void genGPUSetStack(GenGPUState& gpuState, Kernel* ker,
                           cl_device_id device ,GenGPUDevice* gpuDev)
{
  const int32_t per_lane_stack_sz = ker->getStackSize();
  const int32_t offset = ker->getCurbeOffset(GBE_CURBE_EXTRA_ARGUMENT, GBE_STACK_BUFFER);
  int32_t stack_sz = per_lane_stack_sz;

  /* No stack required for this kernel */
  if (per_lane_stack_sz == 0)
    return;

  /* The stack size is given for *each* SIMD lane. So, we accordingly compute
   * the size we need for the complete machine
   */
  assert(offset >= 0);
  stack_sz *= ker->getSIMDWidth();
  stack_sz *= device->max_compute_unit * gpuDev->max_thread_per_unit;

  /* Because HSW calc stack offset per thread is relative with half slice, when
   thread schedule in half slice is not balance, would out of bound. Because
   the max half slice is 4 in GT4, multiply stack size with 4 for safe.
   */
  if(gpuDev->gen_ver == 75)
    stack_sz *= 4;

  gpuState.setStack(offset, stack_sz, BTI_PRIVATE);
}

static int genUploadConstantBuffer(cl_command_queue queue, char* curbe, GenGPUState& gpuState,
                                   cl_kernel kernel, Kernel* ker, Program* prog)
{
  /* calculate constant buffer size
   * we need raw_size & aligned_size
   */
  uint32_t raw_size = 0, aligned_size =0;
  int32_t arg;
  size_t offset = 0;
  const int32_t arg_n = ker->getArgNum();
  size_t global_const_size = prog->getGlobalConstantSize();
  raw_size = global_const_size;

  // Surface state need 4 byte alignment, and Constant argument's buffer size
  // have align to 4 byte when alloc, so align global constant size to 4 can
  // ensure the finally aligned_size align to 4.
  aligned_size =  ALIGN(raw_size, 4);

  /* Reserve 8 bytes to get rid of 0 address */
  if(global_const_size == 0)
    aligned_size = 8;

  for (arg = 0; arg < arg_n; ++arg) {
    const enum gbe_arg_type type = ker->getArgType(arg);
    if (type == GBE_ARG_CONSTANT_PTR && kernel->args[arg]->mem) {
      uint32_t alignment = ker->getArgAlign(arg);
      assert(alignment != 0);
      cl_mem mem = kernel->args[arg]->mem;
      raw_size += mem->size;
      aligned_size = ALIGN(aligned_size, alignment);
      aligned_size += mem->size;
    }
  }
  if(raw_size == 0)
    return 0;

  if (gpuState.allocConstantBuffer(aligned_size, BTI_CONSTANT) == false)
    return -1;
  drm_intel_bo* bo = gpuState.constant_b.bo;
  GBE_ASSERT(bo != NULL);

  drm_intel_bo_map(bo, 1);
  char *cst_addr = (char*)bo->virt;
  if (cst_addr == NULL)
    return -1;

  /* upload the global constant data */
  if (global_const_size > 0) {
    prog->getGlobalConstantData((char*)(cst_addr+offset));
    offset += global_const_size;
  }

  /* reserve 8 bytes to get rid of 0 address */
  if(global_const_size == 0) {
    offset = 8;
  }

  /* upload constant buffer argument */
  int32_t curbe_offset = 0;
  for (arg = 0; arg < arg_n; ++arg) {
    const enum gbe_arg_type type = ker->getArgType(arg);
    if (type == GBE_ARG_CONSTANT_PTR && kernel->args[arg]->mem) {
      cl_mem mem = kernel->args[arg]->mem;
      uint32_t alignment = ker->getArgAlign(arg);
      offset = ALIGN(offset, alignment);
      curbe_offset = ker->getCurbeOffset(GBE_CURBE_KERNEL_ARGUMENT, arg);
      if (curbe_offset < 0)
        continue;
      *(uint32_t *) (curbe + curbe_offset) = offset;

      GenGPUMem* genMem = reinterpret_cast<GenGPUMem*>(getGenMemPrivate(mem, queue->device));
      GBE_ASSERT(genMem != NULL);
      GBE_ASSERT(genMem->bo != NULL);
      drm_intel_bo_map(genMem->bo, 1);

      void *addr = genMem->bo->virt;
      memcpy(cst_addr + offset, addr, mem->size);
      drm_intel_bo_unmap(genMem->bo);
      offset += mem->size;
    }
  }
  drm_intel_bo_unmap(bo);
  return 0;
}

static cl_int genAllocateArgBufs(cl_kernel kernel, Kernel* ker, cl_command_queue queue)
{
  /* Bind all user buffers (given by clSetKernelArg) */
  uint32_t i;
  enum gbe_arg_type arg_type; /* kind of argument */
  for (i = 0; i < ker->getArgNum(); ++i) {
    arg_type = ker->getArgType(i);
    cl_mem mem = kernel->args[i]->mem;
    if ((arg_type != GBE_ARG_GLOBAL_PTR && arg_type != GBE_ARG_CONSTANT_PTR) || !mem)
      continue;

    GenGPUMem* genMem = reinterpret_cast<GenGPUMem*>(getGenMemPrivate(mem, queue->device));
    GBE_ASSERT(genMem != NULL);

    if (genMem->genAllocMemBo(mem) == false) {
      return CL_MEM_OBJECT_ALLOCATION_FAILURE;
    }
  }

  return CL_SUCCESS;
}

/* "Varing" payload is the part of the curbe that changes accross threads in the
 *  same work group. Right now, it consists in local IDs and block IPs
 */
static bool genSetVaryingPayload(Kernel* ker, char *data, const size_t *local_wk_sz,
                                 size_t simd_sz, size_t cst_sz, size_t thread_n)
{
  uint32_t *ids[3] = {NULL,NULL,NULL};
  uint16_t *block_ips = NULL;
  uint32_t *thread_ids = NULL;
  size_t i, j, k, curr = 0;
  int32_t id_offset[3], ip_offset, tid_offset;
  int32_t dw_ip_offset = -1;

  id_offset[0] = ker->getCurbeOffset(GBE_CURBE_LOCAL_ID_X, 0);
  id_offset[1] = ker->getCurbeOffset(GBE_CURBE_LOCAL_ID_Y, 0);
  id_offset[2] = ker->getCurbeOffset(GBE_CURBE_LOCAL_ID_Z, 0);
  ip_offset = ker->getCurbeOffset(GBE_CURBE_BLOCK_IP, 0);
  tid_offset = ker->getCurbeOffset(GBE_CURBE_THREAD_ID, 0);
  if (ip_offset < 0)
    dw_ip_offset = ker->getCurbeOffset(GBE_CURBE_DW_BLOCK_IP, 0);
  GBE_ASSERT(ip_offset < 0 || dw_ip_offset < 0);
  GBE_ASSERT(ip_offset >= 0 || dw_ip_offset >= 0);

  if (id_offset[0] >= 0) {
    ids[0] = (uint32_t*)alloca(sizeof(uint32_t)*thread_n*simd_sz);
    if (ids[0] == NULL)
      return false;
  }
  if (id_offset[1] >= 0) {
    ids[1] = (uint32_t*)alloca(sizeof(uint32_t)*thread_n*simd_sz);
    if (ids[1] == NULL)
      return false;
  }
  if (id_offset[2] >= 0) {
    ids[2] = (uint32_t*)alloca(sizeof(uint32_t)*thread_n*simd_sz);
    if (ids[2] == NULL)
      return false;
  }

  block_ips = (uint16_t*)alloca(sizeof(uint16_t)*thread_n*simd_sz);
  if (block_ips == NULL)
    return false;

  if (tid_offset >= 0) {
    thread_ids = (uint32_t*)alloca(sizeof(uint32_t)*thread_n);
    if (thread_ids == NULL)
      return false;
  }

  /* 0xffff means that the lane is inactivated */
  memset(block_ips, 0xff, sizeof(int16_t)*thread_n*simd_sz);

  /* Compute the IDs and the block IPs */
  for (k = 0; k < local_wk_sz[2]; ++k)
    for (j = 0; j < local_wk_sz[1]; ++j)
      for (i = 0; i < local_wk_sz[0]; ++i, ++curr) {
        if (id_offset[0] >= 0)
          ids[0][curr] = i;
        if (id_offset[1] >= 0)
          ids[1][curr] = j;
        if (id_offset[2] >= 0)
          ids[2][curr] = k;
        block_ips[curr] = 0;
        if (thread_ids)
          thread_ids[curr/simd_sz] = (k*local_wk_sz[2] + j*local_wk_sz[1] + i)/simd_sz;
      }

  /* Copy them to the curbe buffer */
  curr = 0;
  for (i = 0; i < thread_n; ++i, data += cst_sz) {
    uint32_t *ids0 = (uint32_t *) (data + id_offset[0]);
    uint32_t *ids1 = (uint32_t *) (data + id_offset[1]);
    uint32_t *ids2 = (uint32_t *) (data + id_offset[2]);
    uint16_t *ips  = (uint16_t *) (data + ip_offset);
    uint32_t *dw_ips  = (uint32_t *) (data + dw_ip_offset);

    if (thread_ids)
      *(uint32_t *)(data + tid_offset) = thread_ids[i];

    for (j = 0; j < simd_sz; ++j, ++curr) {
      if (id_offset[0] >= 0)
        ids0[j] = ids[0][curr];
      if (id_offset[1] >= 0)
        ids1[j] = ids[1][curr];
      if (id_offset[2] >= 0)
        ids2[j] = ids[2][curr];
      if (ip_offset >= 0)
        ips[j] = block_ips[curr];
      if (dw_ip_offset >= 0)
        dw_ips[j] = block_ips[curr];
    }
  }

  return true;
}

extern "C"
cl_int GenEnqueueNDRangeKernel(cl_command_queue queue, cl_kernel kernel, const uint32_t work_dim,
                               const size_t *global_wk_off, const size_t *global_wk_sz,
                               const size_t *local_wk_sz, cl_command_queue_work_item item)
{
  Kernel* ker = reinterpret_cast<Kernel*>(getGenKernelPrivate(kernel, queue->device));
  if (ker == NULL) {
    return CL_INVALID_VALUE;
  }

  GenGPUCommandQueue* gpuQueue = (GenGPUCommandQueue*)getGenCommandQueuePrivate(queue);
  if (gpuQueue == NULL) {
    return CL_INVALID_VALUE;
  }

  GenGPUDevice* gpuDev = reinterpret_cast<GenGPUDevice*>(getGenDevicePrivate(queue->device));
  GBE_ASSERT(gpuDev);
  Program* prog = reinterpret_cast<Program*>(getGenProgramPrivate(kernel->program, queue->device));
  GBE_ASSERT(prog);

  const uint32_t simd_sz = ker->getSIMDWidth();
  size_t cst_sz = ker->getCurbeSize();
  int32_t scratch_sz = ker->getScratchSize();
  size_t thread_n = 0u;
  cl_int err = CL_SUCCESS;
  //size_t global_size = global_wk_sz[0] * global_wk_sz[1] * global_wk_sz[2];
  bool use_slm = ker->getUseSLM();
  size_t local_sz = 0u;

  /* Calculate the workitems for each group. */
  local_sz = local_wk_sz[0];
  for (uint32_t i = 1; i < work_dim; ++i)
    local_sz *= local_wk_sz[i];

  /* Check the work group size again, we may have different number for
     different hardwares, even within one generation. */
  if (local_sz > genGetKernelMaxWorkGroupSize(kernel, ker, queue->device)) {
    printf("In GEN driver,workgroup items: %lu exceed the max num supported by HW\n", local_sz);
    return CL_INVALID_WORK_ITEM_SIZE;
  }

  thread_n = (local_sz + simd_sz - 1) / simd_sz;

  if ((uint32_t)scratch_sz > gpuDev->scratch_mem_size) {
    printf("Out of scratch memory %d.", scratch_sz);
    return CL_OUT_OF_RESOURCES;
  }

  char* curbe = NULL;
  if (cst_sz) {
    curbe = (char*)alloca(cst_sz);
    if (curbe == NULL)
      return CL_OUT_OF_HOST_MEMORY;

    memset(curbe, 0, cst_sz);
  }

  if (curbe) {
    int32_t slm_sz = genFillCurbe(kernel, ker, curbe, work_dim,
                                  global_wk_off, global_wk_sz, local_wk_sz, thread_n);
    if ((uint32_t)slm_sz > queue->device->local_mem_size) {
      printf("Out of shared local memory %d.\n", slm_sz);
      return CL_OUT_OF_RESOURCES;
    }
  }

  err = genAllocateArgBufs(kernel, ker, queue);
  if (err != CL_SUCCESS) {
    return err;
  }

  GenGPUState* gpuState = NULL;
  if (IS_GEN9(gpuDev->device_id)) {

  } else if (IS_GEN8(gpuDev->device_id)) {

  } else if (IS_GEN75(gpuDev->device_id)) {

  } else if (IS_GEN7(gpuDev->device_id)) {
    gpuState = GBE_NEW(Gen7GPUState, gpuQueue->bufmgr, gpuQueue->ctx, gpuDev->device_id);
  } else
    GBE_ASSERT(0); // not support any more.

  if (gpuState == NULL)
    return CL_OUT_OF_HOST_MEMORY;

  if (gpuState->stateInit(
        queue->device->max_compute_unit * gpuDev->max_thread_per_unit, cst_sz / 32) != true) {
    return CL_OUT_OF_RESOURCES;
  }

  err = genGPUBindSurfaces(*gpuState, queue, kernel, ker);
  if (err != CL_SUCCESS) {
    GBE_DELETE(gpuState);
    return err;
  }

  if (ker->getImageSize()) {
    err = genGPUBindImage(curbe, *gpuState, kernel, ker);
    if (err != CL_SUCCESS) {
      GBE_DELETE(gpuState);
      return err;
    }
  }

  if (ker->getSamplerSize()) {
    err = genGPUBindSamplers(*gpuState, kernel, ker);
    if (err != CL_SUCCESS) {
      GBE_DELETE(gpuState);
      return err;
    }
  }

  if (gpuState->setScratch(scratch_sz) != 0) {
    GBE_DELETE(gpuState);
    return CL_OUT_OF_RESOURCES;
  }

  /* Bind a stack if needed */
  genGPUSetStack(*gpuState, ker, queue->device, gpuDev);

  if (genUploadConstantBuffer(queue, curbe, *gpuState, kernel, ker, prog) != 0) {
    GBE_DELETE(gpuState);
    return CL_OUT_OF_RESOURCES;
  }

  gpuState->setKernel(ker->getCode(), ker->getCodeSize());
  gpuState->buildIdrt(cst_sz, use_slm, ker->getSLMSize(), thread_n);
  dri_bo_unmap(gpuState->aux_buf.bo);

  /* Curbe step 2. Give the localID and upload it to video memory */
  if (curbe) {
    GBE_ASSERT(cst_sz > 0);
    char *final_curbe = (char*)alloca(thread_n * cst_sz);
    if (final_curbe == NULL) {
      GBE_DELETE(gpuState);
      return CL_OUT_OF_HOST_MEMORY;
    }

    for (size_t i = 0; i < thread_n; ++i) {
      memcpy(final_curbe + cst_sz * i, curbe, cst_sz);
    }

    if (genSetVaryingPayload(ker, final_curbe, local_wk_sz, simd_sz, cst_sz, thread_n) != true) {
      GBE_DELETE(gpuState);
      return CL_OUT_OF_HOST_MEMORY;
    }

    if (gpuState->uploadCurbes(final_curbe, thread_n*cst_sz, thread_n, cst_sz) != true) {
      GBE_DELETE(gpuState);
      return CL_OUT_OF_RESOURCES;
    }
  }

  /* Start a new batch buffer */
  size_t batch_sz = 256 + 256; // Should be enough.
  gpuState->newBatchbuf(batch_sz);
  gpuState->batchStart(use_slm);
  /* Issue the GPGPU_WALKER command */
  gpuState->walker(simd_sz, thread_n, global_wk_off, global_wk_sz, local_wk_sz);
  /* Close the batch buffer and submit it */
  gpuState->batchEnd(0);

  item->pdata = gpuState;
  item->submit = genNDRangeSubmit;
  item->run = genNDRangeRun;
  item->complete = genNDRangeComplete;
  return CL_SUCCESS;
}