path: root/backend/src/driver/cl_gen_gpu_state.cpp

/*
 * Copyright © 2012 Intel Corporation
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library. If not, see <http://www.gnu.org/licenses/>.
 *
 */
#include "cl_gen_gpu_state.h"
#include "cl_gen_gpu_structs.h"
#include "../ocl_common_defines.h"
#include "../backend/program.h" // for BTI_RESERVED_NUM

typedef struct surface_heap {
  uint32_t binding_table[256];
  char surface[256*sizeof(gen_surface_state_t)];
} surface_heap_t;

static const uint32_t gpgpu_l3_config_reg1[] = {
  0x00080040, 0x02040040, 0x00800040, 0x01000038,
  0x02000030, 0x01000038, 0x00000038, 0x00000040,
  0x0A140091, 0x09100091, 0x08900091, 0x08900091,
  0x010000a1
};

static const uint32_t gpgpu_l3_config_reg2[] = {
  0x00000000, 0x00000000, 0x00080410, 0x00080410,
  0x00040410, 0x00040420, 0x00080420, 0x00080020,
  0x00204080, 0x00244890, 0x00284490, 0x002444A0,
  0x00040810
};

static int translate_wrap_mode(uint32_t cl_address_mode, int using_nearest)
{
  switch( cl_address_mode ) {
    case CLK_ADDRESS_NONE:
    case CLK_ADDRESS_REPEAT:
      return GEN_TEXCOORDMODE_WRAP;
    case CLK_ADDRESS_CLAMP:
      return GEN_TEXCOORDMODE_CLAMP_BORDER;
    case CLK_ADDRESS_CLAMP_TO_EDGE:
      return GEN_TEXCOORDMODE_CLAMP;
    case CLK_ADDRESS_MIRRORED_REPEAT:
      return GEN_TEXCOORDMODE_MIRROR;
    default:
      return GEN_TEXCOORDMODE_WRAP;
  }
}

static int is_surface_array(cl_mem_object_type type)
{
  if (type == CL_MEM_OBJECT_IMAGE1D_ARRAY ||
      type == CL_MEM_OBJECT_IMAGE2D_ARRAY)
    return 1;

  return 0;
}

static uint32_t _get_surface_type(cl_mem_object_type type)
{
  switch (type) {
    case CL_MEM_OBJECT_IMAGE1D:
    case CL_MEM_OBJECT_IMAGE1D_ARRAY:
      return I965_SURFACE_1D;

    case CL_MEM_OBJECT_IMAGE1D_BUFFER:
    case CL_MEM_OBJECT_IMAGE2D:
    case CL_MEM_OBJECT_IMAGE2D_ARRAY:
      return I965_SURFACE_2D;

    case CL_MEM_OBJECT_IMAGE3D:
      return I965_SURFACE_3D;

    default:
      GBE_ASSERT(0);
  }
  return 0;
}

/* Get fixed surface type. If it is a 1D array image with a large index,
   we need to fixup it to 2D type due to a Gen7/Gen75's sampler issue
   on a integer type surface with clamp address mode and nearest filter mode.
*/
static uint32_t get_surface_type(int device_id, int index, cl_mem_object_type type)
{
  uint32_t surface_type;
  if (((IS_IVYBRIDGE(device_id) || IS_HASWELL(device_id) ||
        IS_BROADWELL(device_id) || IS_CHERRYVIEW(device_id) ||
        IS_SKYLAKE(device_id) || IS_BROXTON(device_id))) &&
      index >= BTI_WORKAROUND_IMAGE_OFFSET + BTI_RESERVED_NUM &&
      type == CL_MEM_OBJECT_IMAGE1D_ARRAY)
    surface_type = I965_SURFACE_2D;
  else
    surface_type = _get_surface_type(type);
  return surface_type;
}

GenGPUState::GenGPUState(dri_bufmgr *bufmgr, drm_intel_context *ctx, int device_id)
{
  memset(this, 0, sizeof(GenGPUState));
  this->bufmgr = bufmgr;
  this->ctx = ctx;
  this->device_id = device_id;
}

GenGPUState::~GenGPUState(void)
{
  if(this->time_stamp_b.bo) {
    drm_intel_bo_unreference(this->time_stamp_b.bo);
    this->time_stamp_b.bo = NULL;
  }
  if(this->printf_b.bo) {
    drm_intel_bo_unreference(this->printf_b.bo);
    this->printf_b.bo = NULL;
  }
  if (this->aux_buf.bo) {
    drm_intel_bo_unreference(this->aux_buf.bo);
    this->aux_buf.bo = NULL;
  }
  if (this->stack_b.bo) {
    drm_intel_bo_unreference(this->stack_b.bo);
    this->stack_b.bo = NULL;
  }
  if (this->scratch_b.bo) {
    drm_intel_bo_unreference(this->scratch_b.bo);
    this->scratch_b.bo = NULL;
  }
  if (this->profiling_b.bo) {
    drm_intel_bo_unreference(this->profiling_b.bo);
    this->profiling_b.bo = NULL;
  }
  if(this->constant_b.bo) {
    drm_intel_bo_unreference(this->constant_b.bo);
    this->constant_b.bo = NULL;
  }

  if (this->batchbuf) {
    GBE_DELETE(this->batchbuf);
    this->batchbuf = NULL;
  }
}

void GenGPUState::sync(void)
{
  if (batchbuf)
    drm_intel_bo_wait_rendering((drm_intel_bo *)batchbuf->buffer);
}

void GenGPUState::bindBuf(drm_intel_bo *buf, uint32_t offset,
                          uint32_t internal_offset, size_t size, uint8_t bti)
{
  GBE_ASSERT(this->binded_n < max_buf_n);
  this->binded_buf[this->binded_n] = buf;
  this->target_buf_offset[this->binded_n] = internal_offset;
  this->binded_offset[this->binded_n] = offset;
  this->binded_n++;
  this->setupBTI(buf, internal_offset, size, bti, I965_SURFACEFORMAT_RAW);
}

void GenGPUState::setStack(uint32_t offset, uint32_t size, uint8_t bti)
{
  this->stack_b.bo = drm_intel_bo_alloc(bufmgr, "STACK", size, 64);
  this->bindBuf(this->stack_b.bo, offset, 0, size, bti);
}

bool GenGPUState::stateInit(uint32_t max_threads, uint32_t size_cs_entry, int profiling)
{
  drm_intel_bo *bo = NULL;

  /* Binded buffers */
  this->binded_n = 0;
  this->img_bitmap = 0;
  this->img_index_base = 3;
  this->sampler_bitmap = ~((1 << max_sampler_n) - 1);

  /* URB */
  this->curb.num_cs_entries = 64;
  this->curb.size_cs_entry = size_cs_entry;
  this->max_threads = max_threads;

  /* Set the profile buffer*/
  if (profiling) {
    bo = dri_bo_alloc(this->bufmgr, "timestamp query", 4096, 4096);
    this->time_stamp_b.bo = bo;
    if (!bo)
      fprintf(stderr, "Could not allocate buffer for profiling.\n");
  }

  /* Set the auxiliary buffer*/
  uint32_t size_aux = 0;
  /* begin with surface heap to make sure it's page aligned,
     because state base address use 20bit for the address */
  this->aux_offset.surface_heap_offset = size_aux;
  size_aux += sizeof(surface_heap);

  //curbe must be 32 bytes aligned
  size_aux = ALIGN(size_aux, 64);
  this->aux_offset.curbe_offset = size_aux;
  size_aux += this->curb.num_cs_entries * this->curb.size_cs_entry * 32;

  //idrt must be 32 bytes aligned
  size_aux = ALIGN(size_aux, 32);
  this->aux_offset.idrt_offset = size_aux;
  size_aux += GEN_MAX_IF_DESC * sizeof(struct gen6_interface_descriptor);

  //must be 32 bytes aligned
  //sampler state and vme state share the same buffer,
  size_aux = ALIGN(size_aux, 32);
  this->aux_offset.sampler_state_offset = size_aux;
  size_aux += GEN_MAX_SAMPLERS * sizeof(gen6_sampler_state_t) >=
              GEN_MAX_VME_STATES * sizeof(gen7_vme_state_t) ?
              GEN_MAX_SAMPLERS * sizeof(gen6_sampler_state_t) :
              GEN_MAX_VME_STATES * sizeof(gen7_vme_state_t);

  //sampler border color state must be 32 bytes aligned
  size_aux = ALIGN(size_aux, 32);
  this->aux_offset.sampler_border_color_state_offset = size_aux;
  size_aux += GEN_MAX_SAMPLERS * sizeof(gen7_sampler_border_color_t);

  /* make sure aux buffer is page aligned */
  size_aux = ALIGN(size_aux, 4096);

  bo = dri_bo_alloc(this->bufmgr, "AUX_BUFFER", size_aux, 4096);
  if (!bo || dri_bo_map(bo, 1) != 0) {
    fprintf(stderr, "%s:%d: %s.\n", __FILE__, __LINE__, strerror(errno));
    if (bo)
      dri_bo_unreference(bo);

    if (profiling && this->time_stamp_b.bo)
      dri_bo_unreference(this->time_stamp_b.bo);
    this->time_stamp_b.bo = NULL;
    return false;
  }

  memset(bo->virt, 0, size_aux);
  this->aux_buf.bo = bo;
  return true;
}

bool GenGPUState::allocConstantBuffer(uint32_t size, uint8_t bti)
{
  this->constant_b.bo = drm_intel_bo_alloc(this->bufmgr, "CONSTANT_BUFFER", size, 64);
  if (this->constant_b.bo == NULL)
    return false;

  this->setupBTI(this->constant_b.bo, 0, size, bti, I965_SURFACEFORMAT_R32G32B32A32_UINT);
  return true;
}

void GenGPUState::batchStart(uint32_t use_slm)
{
  this->batchbuf->requireSpace(256);
  this->pipeControl();
  this->setL3(use_slm);
  this->selectPipeline();
  this->setBaseAddress();
  this->loadVfeState();
  this->loadCurbeBuffer();
  this->loadIdrt();

  /* Insert PIPE_CONTROL for time stamp of start*/
  if (this->time_stamp_b.bo)
    this->writeTimestamp(0);
}

void GenGPUState::writeTimestamp(int idx)
{
  BEGIN_BATCH(this->batchbuf, 5);
  OUT_BATCH(this->batchbuf, CMD_PIPE_CONTROL | (5-2));
  OUT_BATCH(this->batchbuf, GEN7_PIPE_CONTROL_WRITE_TIMESTAMP);
  OUT_RELOC(this->batchbuf, this->time_stamp_b.bo,
            I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
            GEN7_PIPE_CONTROL_GLOBAL_GTT_WRITE | idx * sizeof(uint64_t));
  OUT_BATCH(this->batchbuf, 0);
  OUT_BATCH(this->batchbuf, 0);
  ADVANCE_BATCH();
}

void GenGPUState::walker(uint32_t simd_sz, uint32_t thread_n, const size_t global_wk_off[3],
                         const size_t global_wk_sz[3], const size_t local_wk_sz[3])
{
  const uint32_t global_wk_dim[3] = {
    (uint32_t)(global_wk_sz[0] / local_wk_sz[0]),
    (uint32_t)(global_wk_sz[1] / local_wk_sz[1]),
    (uint32_t)(global_wk_sz[2] / local_wk_sz[2])
  };
  uint32_t right_mask = ~0x0;
  size_t group_sz = local_wk_sz[0] * local_wk_sz[1] * local_wk_sz[2];

  assert(simd_sz == 8 || simd_sz == 16);

  uint32_t shift = (group_sz & (simd_sz - 1));
  shift = (shift == 0) ? simd_sz : shift;
  right_mask = (1 << shift) - 1;

  BEGIN_BATCH(this->batchbuf, 11);
  OUT_BATCH(this->batchbuf, CMD_GPGPU_WALKER | 9);
  OUT_BATCH(this->batchbuf, 0);                        /* kernel index == 0 */
  assert(thread_n <= 64);
  if (simd_sz == 16)
    OUT_BATCH(this->batchbuf, (1 << 30) | (thread_n-1)); /* SIMD16 | thread max */
  else
    OUT_BATCH(this->batchbuf, (0 << 30) | (thread_n-1)); /* SIMD8  | thread max */
  OUT_BATCH(this->batchbuf, 0);
  OUT_BATCH(this->batchbuf, global_wk_dim[0]);
  OUT_BATCH(this->batchbuf, 0);
  OUT_BATCH(this->batchbuf, global_wk_dim[1]);
  OUT_BATCH(this->batchbuf, 0);
  OUT_BATCH(this->batchbuf, global_wk_dim[2]);
  OUT_BATCH(this->batchbuf, right_mask);
  OUT_BATCH(this->batchbuf, ~0x0);   /* we always set height as 1, so set bottom mask as all 1*/
  ADVANCE_BATCH(this->batchbuf);

  BEGIN_BATCH(this->batchbuf, 2);
  OUT_BATCH(this->batchbuf, CMD_MEDIA_STATE_FLUSH | 0);
  OUT_BATCH(this->batchbuf, 0);                        /* kernel index == 0 */
  ADVANCE_BATCH(this->batchbuf);

  if (IS_IVYBRIDGE(device_id))
    this->pipeControl();
}

void GenGPUState::bindSamplers(uint32_t *samplers, size_t sampler_sz)
{
  uint32_t index;
  GBE_ASSERT(sampler_sz <= GEN_MAX_SAMPLERS);
  for(index = 0; index < sampler_sz; index++)
    this->insertSampler(index, samplers[index]);
}

bool GenGPUState::uploadCurbes(const void* data, uint32_t size, uint32_t thread_n, uint32_t curbe_sz)
{
  unsigned char *curbe = NULL;
  uint32_t i, j;

  /* Upload the data first */
  if (dri_bo_map(this->aux_buf.bo, 1) != 0) {
    fprintf(stderr, "%s:%d: %s.\n", __FILE__, __LINE__, strerror(errno));
    return false;
  }
  GBE_ASSERT(this->aux_buf.bo->virt);
  curbe = ((unsigned char *)this->aux_buf.bo->virt + this->aux_offset.curbe_offset);
  memcpy(curbe, data, size);

  /* Now put all the relocations for our flat address space */
  for (i = 0; i < thread_n; ++i)
    for (j = 0; j < this->binded_n; ++j) {
      *(uint32_t*)(curbe + this->binded_offset[j]+i*curbe_sz) =
        this->binded_buf[j]->offset + this->target_buf_offset[j];
      drm_intel_bo_emit_reloc(this->aux_buf.bo,
                              this->aux_offset.curbe_offset + this->binded_offset[j]+i*curbe_sz,
                              this->binded_buf[j],
                              this->target_buf_offset[j],
                              I915_GEM_DOMAIN_RENDER,
                              I915_GEM_DOMAIN_RENDER);
    }

  dri_bo_unmap(this->aux_buf.bo);
  return true;
}

int GenGPUState::getCurbeSize(void)
{
  int curbe_size = this->curb.size_cs_entry * this->curb.num_cs_entries;
  int max_curbe_size;

  if (IS_BAYTRAIL_T(device_id) || IS_IVB_GT1(device_id)) {
    max_curbe_size = 992;
  } else {
    max_curbe_size = 2016;
  }

  if (curbe_size > max_curbe_size) {
    fprintf(stderr, "warning, curbe size exceed limitation.\n");
    return max_curbe_size;
  } else
    return curbe_size;
}

/*****************************************************************************************
 **************************************  GEN7  *******************************************
 *****************************************************************************************/
void Gen7GPUState::selectPipeline(void)
{
  BEGIN_BATCH(this->batchbuf, 1);
  OUT_BATCH(this->batchbuf, CMD_PIPELINE_SELECT | PIPELINE_SELECT_GPGPU);
  ADVANCE_BATCH(this->batch);
}

uint32_t Gen7GPUState::getCacheCtrl(void)
{
  return cc_llc_l3;
}

void Gen7GPUState::setBaseAddress(void)
{
  const uint32_t def_cc = this->getCacheCtrl(); /* default Cache Control value */
  BEGIN_BATCH(this->batchbuf, 10);
  OUT_BATCH(this->batchbuf, CMD_STATE_BASE_ADDRESS | 8);
  /* 0, Gen State Mem Obj CC, Stateless Mem Obj CC, Stateless Access Write Back */
  /* General State Base Addr */
  OUT_BATCH(this->batchbuf, 0 | (def_cc << 8) | (def_cc << 4) | (0 << 3)| BASE_ADDRESS_MODIFY);
  /* 0, State Mem Obj CC */
  /* We use a state base address for the surface heap since IVB clamp the
   * binding table pointer at 11 bits. So, we cannot use pointers directly while
   * using the surface heap
   */
  GBE_ASSERT(this->aux_offset.surface_heap_offset % 4096 == 0);
  OUT_RELOC(this->batchbuf, this->aux_buf.bo,
            I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
            this->aux_offset.surface_heap_offset +
            (0 | (def_cc << 8) | (def_cc << 4) | (0 << 3)| BASE_ADDRESS_MODIFY));

  OUT_BATCH(this->batchbuf, 0 | (def_cc << 8) | BASE_ADDRESS_MODIFY); /* Dynamic State Base Addr */

  OUT_BATCH(this->batchbuf, 0 | (def_cc << 8) | BASE_ADDRESS_MODIFY); /* Indirect Obj Base Addr */
  OUT_BATCH(this->batchbuf, 0 | (def_cc << 8) | BASE_ADDRESS_MODIFY); /* Instruction Base Addr	*/
  OUT_BATCH(this->batchbuf, 0 | BASE_ADDRESS_MODIFY);
  /* According to mesa i965 driver code, we must set the dynamic state access upper bound
   * to a valid bound value, otherwise, the border color pointer may be rejected and you
   * may get incorrect border color. This is a known hardware bug. */
  OUT_BATCH(this->batchbuf, 0xfffff000 | BASE_ADDRESS_MODIFY);
  OUT_BATCH(this->batchbuf, 0 | BASE_ADDRESS_MODIFY);
  OUT_BATCH(this->batchbuf, 0 | BASE_ADDRESS_MODIFY);
  ADVANCE_BATCH(this->batchbuf);
}

void Gen7GPUState::setupBTI(drm_intel_bo *buf, uint32_t internal_offset,
                            size_t size, unsigned char index, uint32_t format)
{
  GBE_ASSERT(size <= (2ul<<30));

  size_t s = size - 1;
  surface_heap *heap = (surface_heap *)((char*)this->aux_buf.bo->virt
                                        + this->aux_offset.surface_heap_offset);
  gen7_surface_state_t *ss0 = (gen7_surface_state_t *) &heap->surface[index * sizeof(gen7_surface_state_t)];
  memset(ss0, 0, sizeof(gen7_surface_state_t));
  ss0->ss0.surface_type = I965_SURFACE_BUFFER;
  ss0->ss0.surface_format = format;
  ss0->ss2.width = s & 0x7f;   /* bits 6:0 of sz */
  // Per bspec, I965_SURFACE_BUFFER and RAW format, size must be a multiple of 4 byte.
  if(format == I965_SURFACEFORMAT_RAW)
    assert((ss0->ss2.width & 0x03) == 3);
  ss0->ss2.height = (s >> 7) & 0x3fff; /* bits 20:7 of sz */
  ss0->ss3.depth = (s >> 21) & 0x3ff; /* bits 30:21 of sz */
  ss0->ss5.cache_control = this->getCacheCtrl();
  heap->binding_table[index] = offsetof(surface_heap, surface) + index * sizeof(gen7_surface_state_t);

  ss0->ss1.base_addr = buf->offset + internal_offset;
  dri_bo_emit_reloc(this->aux_buf.bo, I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER,
                    internal_offset, this->aux_offset.surface_heap_offset +
                    heap->binding_table[index] + offsetof(gen7_surface_state_t, ss1), buf);
}

void Gen7GPUState::setL3(uint32_t use_slm)
{
  BEGIN_BATCH(this->batchbuf, 9);
  OUT_BATCH(this->batchbuf, CMD_LOAD_REGISTER_IMM | 1); /* length - 2 */
  OUT_BATCH(this->batchbuf, GEN7_L3_SQC_REG1_ADDRESS_OFFSET);
  OUT_BATCH(this->batchbuf, 0x00A00000);

  OUT_BATCH(this->batchbuf, CMD_LOAD_REGISTER_IMM | 1); /* length - 2 */
  OUT_BATCH(this->batchbuf, GEN7_L3_CNTL_REG2_ADDRESS_OFFSET);

  if (use_slm)
    OUT_BATCH(this->batchbuf, gpgpu_l3_config_reg1[12]);
  else
    OUT_BATCH(this->batchbuf, gpgpu_l3_config_reg1[4]);

  OUT_BATCH(this->batchbuf, CMD_LOAD_REGISTER_IMM | 1); /* length - 2 */
  OUT_BATCH(this->batchbuf, GEN7_L3_CNTL_REG3_ADDRESS_OFFSET);
  if (use_slm)
    OUT_BATCH(this->batchbuf, gpgpu_l3_config_reg2[12]);
  else
    OUT_BATCH(this->batchbuf, gpgpu_l3_config_reg2[4]);
  ADVANCE_BATCH(this->batchbuf);

  this->pipeControl();
}

void Gen7GPUState::pipeControl(void)
{
  gen6_pipe_control_t* pc = (gen6_pipe_control_t*)this->batchbuf->allocSpace(sizeof(gen6_pipe_control_t));
  memset(pc, 0, sizeof(*pc));
  pc->dw0.length = (sizeof(gen6_pipe_control_t)/sizeof(uint32_t)) - 2;
  pc->dw0.instruction_subopcode = GEN7_PIPE_CONTROL_SUBOPCODE_3D_CONTROL;
  pc->dw0.instruction_opcode = GEN7_PIPE_CONTROL_OPCODE_3D_CONTROL;
  pc->dw0.instruction_pipeline = GEN7_PIPE_CONTROL_3D;
  pc->dw0.instruction_type = GEN7_PIPE_CONTROL_INSTRUCTION_GFX;
  pc->dw1.render_target_cache_flush_enable = 1;
  pc->dw1.texture_cache_invalidation_enable = 1;
  pc->dw1.cs_stall = 1;
  pc->dw1.dc_flush_enable = 1;
  //pc->dw1.instruction_cache_invalidate_enable = 1;
  ADVANCE_BATCH(this->batchbuf);
}

void Gen7GPUState::loadIdrt(void)
{
  BEGIN_BATCH(this->batchbuf, 4);
  OUT_BATCH(this->batchbuf, CMD(2,0,2) | (4 - 2)); /* length-2 */
  OUT_BATCH(this->batchbuf, 0);                    /* mbz */
  OUT_BATCH(this->batchbuf, 1 << 5);
  OUT_RELOC(this->batchbuf, this->aux_buf.bo,
            I915_GEM_DOMAIN_INSTRUCTION, 0, this->aux_offset.idrt_offset);
  ADVANCE_BATCH(this->batchbuf);
}

void Gen7GPUState::insertSampler(uint32_t index, uint32_t clk_sampler)
{
  int using_nearest = 0;
  uint32_t wrap_mode;
  gen7_sampler_state_t *sampler;

  sampler = (gen7_sampler_state_t *)((char*)this->aux_buf.bo->virt
                                     + this->aux_offset.sampler_state_offset) + index;
  memset(sampler, 0, sizeof(*sampler));
  GBE_ASSERT((this->aux_buf.bo->offset
              + this->aux_offset.sampler_border_color_state_offset) % 32 == 0);
  sampler->ss2.default_color_pointer =
    (this->aux_buf.bo->offset + this->aux_offset.sampler_border_color_state_offset) >> 5;
  if ((clk_sampler & __CLK_NORMALIZED_MASK) == CLK_NORMALIZED_COORDS_FALSE)
    sampler->ss3.non_normalized_coord = 1;
  else
    sampler->ss3.non_normalized_coord = 0;

  switch (clk_sampler & __CLK_FILTER_MASK) {
    case CLK_FILTER_NEAREST:
      sampler->ss0.min_filter = GEN_MAPFILTER_NEAREST;
      sampler->ss0.mip_filter = GEN_MIPFILTER_NONE;
      sampler->ss0.mag_filter = GEN_MAPFILTER_NEAREST;
      using_nearest = 1;
      break;
    case CLK_FILTER_LINEAR:
      sampler->ss0.min_filter = GEN_MAPFILTER_LINEAR;
      sampler->ss0.mip_filter = GEN_MIPFILTER_NONE;
      sampler->ss0.mag_filter = GEN_MAPFILTER_LINEAR;
      break;
  }

  wrap_mode = translate_wrap_mode(clk_sampler & __CLK_ADDRESS_MASK, using_nearest);
  sampler->ss3.s_wrap_mode = wrap_mode;
  /* XXX mesa i965 driver code point out that if the surface is a 1D surface, we may need
   * to set t_wrap_mode to GEN_TEXCOORDMODE_WRAP. */
  sampler->ss3.t_wrap_mode = wrap_mode;
  sampler->ss3.r_wrap_mode = wrap_mode;

  sampler->ss0.lod_preclamp = 1; /* OpenGL mode */
  sampler->ss0.default_color_mode = 0; /* OpenGL/DX10 mode */

  sampler->ss0.base_level = 0;

  sampler->ss1.max_lod = 0;
  sampler->ss1.min_lod = 0;

  if (sampler->ss0.min_filter != GEN_MAPFILTER_NEAREST)
    sampler->ss3.address_round |= GEN_ADDRESS_ROUNDING_ENABLE_U_MIN |
                                  GEN_ADDRESS_ROUNDING_ENABLE_V_MIN |
                                  GEN_ADDRESS_ROUNDING_ENABLE_R_MIN;
  if (sampler->ss0.mag_filter != GEN_MAPFILTER_NEAREST)
    sampler->ss3.address_round |= GEN_ADDRESS_ROUNDING_ENABLE_U_MAG |
                                  GEN_ADDRESS_ROUNDING_ENABLE_V_MAG |
                                  GEN_ADDRESS_ROUNDING_ENABLE_R_MAG;

  dri_bo_emit_reloc(this->aux_buf.bo,
                    I915_GEM_DOMAIN_SAMPLER, 0,
                    this->aux_offset.sampler_border_color_state_offset,
                    this->aux_offset.sampler_state_offset +
                    index * sizeof(gen7_sampler_state_t) +
                    offsetof(gen7_sampler_state_t, ss2),
                    this->aux_buf.bo);
}

void Gen7GPUState::loadVfeState(void)
{
  int32_t scratch_index;
  BEGIN_BATCH(this->batchbuf, 8);
  OUT_BATCH(this->batchbuf, CMD_MEDIA_STATE_POINTERS | (8-2));

  if(this->per_thread_scratch > 0) {
    scratch_index = this->getScratchIndex(this->per_thread_scratch);
    OUT_RELOC(this->batchbuf, this->scratch_b.bo,
              I915_GEM_DOMAIN_RENDER,
              I915_GEM_DOMAIN_RENDER,
              scratch_index);
  } else {
    OUT_BATCH(this->batchbuf, 0);
  }
  /* max_thread | urb entries | (reset_gateway|bypass_gate_way | gpgpu_mode) */
  OUT_BATCH(this->batchbuf, 0 | ((this->max_threads - 1) << 16) | (0 << 8) | 0xc4);
  OUT_BATCH(this->batchbuf, 0);
  /* curbe_size */
  OUT_BATCH(this->batchbuf, this->getCurbeSize());
  OUT_BATCH(this->batchbuf, 0);
  OUT_BATCH(this->batchbuf, 0);
  OUT_BATCH(this->batchbuf, 0);
  ADVANCE_BATCH(this->batchbuf);
}

uint32_t Gen7GPUState::getScratchIndex(uint32_t size)
{
  return size / 1024 - 1;
}

void Gen7GPUState::loadCurbeBuffer(void)
{
  BEGIN_BATCH(this->batchbuf, 4);
  OUT_BATCH(this->batchbuf, CMD(2,0,1) | (4 - 2));  /* length-2 */
  OUT_BATCH(this->batchbuf, 0);                     /* mbz */
  OUT_BATCH(this->batchbuf, this->getCurbeSize() * 32);
  OUT_RELOC(this->batchbuf, this->aux_buf.bo, I915_GEM_DOMAIN_INSTRUCTION, 0, this->aux_offset.curbe_offset);
  ADVANCE_BATCH(this->batchbuf);
}

void Gen7GPUState::bindImage(uint32_t index, dri_bo* obj_bo, uint32_t obj_bo_offset,
                             uint32_t format, cl_mem_object_type type, uint32_t bpp,
                             int32_t w, int32_t h, int32_t depth, int32_t pitch,
                             int32_t slice_pitch, int32_t tiling)
{
  surface_heap_t *heap = (surface_heap_t *)((char *)this->aux_buf.bo->virt +
                         this->aux_offset.surface_heap_offset);
  gen7_surface_state_t *ss = (gen7_surface_state_t *)&heap->surface[index * sizeof(gen7_surface_state_t)];

  memset(ss, 0, sizeof(*ss));
  ss->ss0.vertical_line_stride = 0; // always choose VALIGN_2
  ss->ss0.surface_type = get_surface_type(this->device_id, index, type);
  if (is_surface_array(type)) {
    ss->ss0.surface_array = 1;
    ss->ss0.surface_array_spacing = 1;
  }
  ss->ss0.surface_format = format;
  ss->ss1.base_addr = obj_bo->offset + obj_bo_offset;
  ss->ss2.width = w - 1;

  ss->ss2.height = h - 1;
  ss->ss3.depth = depth - 1;
  ss->ss4.not_str_buf.rt_view_extent = depth - 1;
  ss->ss4.not_str_buf.min_array_element = 0;
  ss->ss3.pitch = pitch - 1;
  ss->ss5.cache_control = this->getCacheCtrl();
  if (tiling == GPGPU_TILE_X) {
    ss->ss0.tiled_surface = 1;
    ss->ss0.tile_walk = I965_TILEWALK_XMAJOR;
  } else if (tiling == GPGPU_TILE_Y) {
    ss->ss0.tiled_surface = 1;
    ss->ss0.tile_walk = I965_TILEWALK_YMAJOR;
  }
  ss->ss0.render_cache_rw_mode = 1; /* XXX do we need to set it? */

  heap->binding_table[index] = offsetof(surface_heap_t, surface) + index * sizeof(gen7_surface_state_t);
  dri_bo_emit_reloc(this->aux_buf.bo,
                    I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER,
                    obj_bo_offset,
                    this->aux_offset.surface_heap_offset + heap->binding_table[index] +
                    offsetof(gen7_surface_state_t, ss1),
                    obj_bo);

  GBE_ASSERT(index < GEN_MAX_SURFACES);
}

void Gen7GPUState::buildIdrt(drm_intel_bo* ker_bo, uint32_t curbe_sz,
                             uint32_t use_slm, uint32_t slm_sz, uint32_t thread_n)
{
  gen6_interface_descriptor_t *desc;

  desc = (gen6_interface_descriptor_t*) ((char *)this->aux_buf.bo->virt + this->aux_offset.idrt_offset);

  memset(desc, 0, sizeof(*desc));
  desc->desc0.kernel_start_pointer = ker_bo->offset >> 6; /* reloc */
  desc->desc1.single_program_flow = 0;
  desc->desc1.floating_point_mode = 0; /* use IEEE-754 rule */
  desc->desc5.rounding_mode = 0; /* round to nearest even */

  GBE_ASSERT((this->aux_buf.bo->offset + this->aux_offset.sampler_state_offset) % 32 == 0);
  desc->desc2.sampler_state_pointer = (this->aux_buf.bo->offset + this->aux_offset.sampler_state_offset) >> 5;
  desc->desc3.binding_table_entry_count = 0; /* no prefetch */
  desc->desc3.binding_table_pointer = 0;
  desc->desc4.curbe_read_len = curbe_sz / 32;
  desc->desc4.curbe_read_offset = 0;

  /* Barriers / SLM are automatically handled on Gen7+ */
  desc->desc5.group_threads_num = use_slm ? thread_n : 0;
  desc->desc5.barrier_enable = use_slm;
  if (slm_sz <= 4*KB)
    slm_sz = 4*KB;
  else if (slm_sz <= 8*KB)
    slm_sz = 8*KB;
  else if (slm_sz <= 16*KB)
    slm_sz = 16*KB;
  else if (slm_sz <= 32*KB)
    slm_sz = 32*KB;
  else
    slm_sz = 64*KB;
  slm_sz = slm_sz >> 12;
  desc->desc5.slm_sz = slm_sz;

  dri_bo_emit_reloc(this->aux_buf.bo,
                    I915_GEM_DOMAIN_INSTRUCTION, 0,
                    0,
                    this->aux_offset.idrt_offset + offsetof(gen6_interface_descriptor_t, desc0),
                    ker_bo);

  dri_bo_emit_reloc(this->aux_buf.bo,
                    I915_GEM_DOMAIN_SAMPLER, 0,
                    this->aux_offset.sampler_state_offset,
                    this->aux_offset.idrt_offset + offsetof(gen6_interface_descriptor_t, desc2),
                    this->aux_buf.bo);
}