haxNV_shader_atomic_int64

4 files changed, 45 insertions, 16 deletions

diff --git a/src/intel/compiler/brw_eu_emit.c b/src/intel/compiler/brw_eu_emit.c
index 3463d356b0a..658deee096c 100644
--- a/src/intel/compiler/brw_eu_emit.c
+++ b/src/intel/compiler/brw_eu_emit.c
@@ -2794,7 +2794,7 @@ brw_untyped_atomic(struct brw_codegen *p,
                           HSW_SFID_DATAPORT_DATA_CACHE_1 :
                           GEN7_SFID_DATAPORT_DATA_CACHE);
    const unsigned response_length = brw_surface_payload_size(
-      p, response_expected, devinfo->gen >= 8 || devinfo->is_haswell, true);
+      p, response_expected ? type_size / 4 : 0, devinfo->gen >= 8 || devinfo->is_haswell, true);
    const unsigned desc =
       brw_message_desc(devinfo, msg_length, response_length, header_present) |
       brw_dp_untyped_atomic_desc(p, atomic_op, response_expected, type_size);
diff --git a/src/intel/compiler/brw_fs.cpp b/src/intel/compiler/brw_fs.cpp
index 69ad4316885..107db2706fb 100644
--- a/src/intel/compiler/brw_fs.cpp
+++ b/src/intel/compiler/brw_fs.cpp
@@ -4774,7 +4774,8 @@ lower_surface_logical_send(const fs_builder &bld, fs_inst *inst, opcode op,
 
    /* Allocate space for the payload. */
    fs_reg *const components = new fs_reg[sz];
-   const fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, sz);
+   const fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, header_sz + addr_sz +
+                                   (src_sz * (type_sz(src.type) / 4)));
    unsigned n = 0;
 
    /* Construct the payload. */
@@ -4817,7 +4818,8 @@ lower_surface_logical_send(const fs_builder &bld, fs_inst *inst, opcode op,
 
    /* Update the original instruction. */
    inst->opcode = op;
-   inst->mlen = header_sz + (addr_sz + src_sz) * inst->exec_size / 8;
+   inst->mlen = header_sz + (addr_sz + (src_sz * (type_sz(src.type) / 4))) *
+                inst->exec_size / 8;
    inst->header_size = header_sz;
 
    inst->src[0] = payload;
diff --git a/src/intel/compiler/brw_fs_nir.cpp b/src/intel/compiler/brw_fs_nir.cpp
index 96db0f5f0d3..4f182fbcba1 100644
--- a/src/intel/compiler/brw_fs_nir.cpp
+++ b/src/intel/compiler/brw_fs_nir.cpp
@@ -4912,7 +4912,7 @@ fs_visitor::nir_emit_ssbo_atomic(const fs_builder &bld,
                             nir->info.num_ssbos - 1);
    }
 
-   fs_reg offset = get_nir_src(instr->src[1]);
+   fs_reg offset_reg = get_nir_src(instr->src[1]);
    fs_reg data1;
    if (op != BRW_AOP_INC && op != BRW_AOP_DEC && op != BRW_AOP_PREDEC)
       data1 = get_nir_src(instr->src[2]);
@@ -4922,13 +4922,19 @@ fs_visitor::nir_emit_ssbo_atomic(const fs_builder &bld,
 
    /* Emit the actual atomic operation */
 
-   fs_reg atomic_result = emit_untyped_atomic(bld, surface, offset,
+   fs_reg atomic_result = emit_untyped_atomic(bld, surface, offset_reg,
                                               data1, data2,
                                               1 /* dims */, 1 /* rsize */,
                                               op,
                                               BRW_PREDICATE_NONE);
-   dest.type = atomic_result.type;
-   bld.MOV(dest, atomic_result);
+   if (type_sz(dest.type) == 8) {
+         /* Shuffle the 32-bit load result into valid 64-bit data */
+         shuffle_from_32bit_read(bld, offset(dest, bld, 0), atomic_result,
+                                 0, 1);
+   } else {
+      dest.type = atomic_result.type;
+      bld.MOV(dest, atomic_result);
+   }
 }
 
 void
@@ -4988,7 +4994,7 @@ fs_visitor::nir_emit_shared_atomic(const fs_builder &bld,
       dest = get_nir_dest(instr->dest);
 
    fs_reg surface = brw_imm_ud(GEN7_BTI_SLM);
-   fs_reg offset;
+   fs_reg offset_reg;
    fs_reg data1;
    if (op != BRW_AOP_INC && op != BRW_AOP_DEC && op != BRW_AOP_PREDEC)
       data1 = get_nir_src(instr->src[1]);
@@ -4999,23 +5005,29 @@ fs_visitor::nir_emit_shared_atomic(const fs_builder &bld,
    /* Get the offset */
    nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]);
    if (const_offset) {
-      offset = brw_imm_ud(instr->const_index[0] + const_offset->u32[0]);
+      offset_reg = brw_imm_ud(instr->const_index[0] + const_offset->u32[0]);
    } else {
-      offset = vgrf(glsl_type::uint_type);
-      bld.ADD(offset,
+      offset_reg = vgrf(glsl_type::uint_type);
+      bld.ADD(offset_reg,
 	      retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_UD),
 	      brw_imm_ud(instr->const_index[0]));
    }
 
    /* Emit the actual atomic operation operation */
 
-   fs_reg atomic_result = emit_untyped_atomic(bld, surface, offset,
+   fs_reg atomic_result = emit_untyped_atomic(bld, surface, offset_reg,
                                               data1, data2,
                                               1 /* dims */, 1 /* rsize */,
                                               op,
                                               BRW_PREDICATE_NONE);
-   dest.type = atomic_result.type;
-   bld.MOV(dest, atomic_result);
+   if (type_sz(dest.type) == 8) {
+         /* Shuffle the 32-bit load result into valid 64-bit data */
+         shuffle_from_32bit_read(bld, offset(dest, bld, 0), atomic_result,
+                                 0, 1);
+   } else {
+      dest.type = atomic_result.type;
+      bld.MOV(dest, atomic_result);
+   }
 }
 
 void
diff --git a/src/intel/compiler/brw_fs_surface_builder.cpp b/src/intel/compiler/brw_fs_surface_builder.cpp
index 0b8418ca736..7badf8633ee 100644
--- a/src/intel/compiler/brw_fs_surface_builder.cpp
+++ b/src/intel/compiler/brw_fs_surface_builder.cpp
@@ -47,6 +47,12 @@ namespace brw {
             const fs_reg srcs[] = {
                addr, src, usurface, brw_imm_ud(dims), brw_imm_ud(arg)
             };
+
+            if (opcode == SHADER_OPCODE_UNTYPED_ATOMIC64_LOGICAL) {
+               assert(rsize == 0 || rsize == 1);
+               rsize *= 2;
+            }
+
             const fs_reg dst = bld.vgrf(BRW_REGISTER_TYPE_UD, rsize);
             fs_inst *inst = bld.emit(opcode, dst, srcs, ARRAY_SIZE(srcs));
 
@@ -103,10 +109,19 @@ namespace brw {
           */
          const unsigned n = (src0.file != BAD_FILE) + (src1.file != BAD_FILE);
          const fs_reg srcs[] = { src0, src1 };
-         const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD, n);
+         const enum brw_reg_type type = (src0.type == BRW_REGISTER_TYPE_Q ||
+                                         src0.type == BRW_REGISTER_TYPE_UQ) ? BRW_REGISTER_TYPE_UQ : BRW_REGISTER_TYPE_UD;
+         const fs_reg tmp = bld.vgrf(type, n);
          bld.LOAD_PAYLOAD(tmp, srcs, n, 0);
 
-         return emit_send(bld, SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL,
+         if (op == BRW_AOP_CMPWR && (src0.type == BRW_REGISTER_TYPE_Q ||
+                                     src0.type == BRW_REGISTER_TYPE_UQ)) {
+            op = BRW_AOP_CMPWR8;
+         }
+
+         return emit_send(bld,
+                          (src0.type == BRW_REGISTER_TYPE_Q || src0.type == BRW_REGISTER_TYPE_UQ)
+                          ? SHADER_OPCODE_UNTYPED_ATOMIC64_LOGICAL : SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL,
                           addr, tmp, surface, dims, op, rsize, pred);
       }