r600/sfn: correct allocating and emitting of atomics

Signed-off-by: Gert Wollny <gert.wollny@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/6025>

3 files changed, 23 insertions, 27 deletions

diff --git a/src/gallium/drivers/r600/sfn/sfn_emitssboinstruction.cpp b/src/gallium/drivers/r600/sfn/sfn_emitssboinstruction.cpp
index f7eb09936a4..458554eec8e 100644
--- a/src/gallium/drivers/r600/sfn/sfn_emitssboinstruction.cpp
+++ b/src/gallium/drivers/r600/sfn/sfn_emitssboinstruction.cpp
@@ -103,7 +103,7 @@ bool EmitSSBOInstruction::emit_atomic(const nir_intrinsic_instr* instr)
 
    GPRVector dest = make_dest(instr);
 
-   int base = nir_intrinsic_base(instr);
+   int base = remap_atomic_base(nir_intrinsic_base(instr));
 
    PValue uav_id = from_nir(instr->src[0], 0);
 
@@ -111,7 +111,7 @@ bool EmitSSBOInstruction::emit_atomic(const nir_intrinsic_instr* instr)
 
    GDSInstr *ir = nullptr;
    if (instr->intrinsic == nir_intrinsic_atomic_counter_comp_swap)  {
-      PValue value2 = from_nir_with_fetch_constant(instr->src[1], 1);
+      PValue value2 = from_nir_with_fetch_constant(instr->src[2], 0);
       ir = new GDSInstr(op, dest, value, value2, uav_id, base);
    } else {
       ir = new GDSInstr(op, dest, value, uav_id, base);
@@ -132,7 +132,7 @@ bool EmitSSBOInstruction::emit_unary_atomic(const nir_intrinsic_instr* instr)
 
    PValue uav_id = from_nir(instr->src[0], 0);
 
-   auto ir = new GDSInstr(op, dest, uav_id, nir_intrinsic_base(instr));
+   auto ir = new GDSInstr(op, dest, uav_id, remap_atomic_base(nir_intrinsic_base(instr)));
 
    emit_instruction(ir);
    return true;
@@ -221,7 +221,7 @@ bool EmitSSBOInstruction::emit_atomic_add(const nir_intrinsic_instr* instr)
    PValue uav_id = from_nir(instr->src[0], 0);
 
    auto ir = new GDSInstr(DS_OP_ADD_RET, dest, value, uav_id,
-                          nir_intrinsic_base(instr));
+                          remap_atomic_base(nir_intrinsic_base(instr)));
 
    emit_instruction(ir);
    return true;
@@ -240,7 +240,7 @@ bool EmitSSBOInstruction::emit_atomic_inc(const nir_intrinsic_instr* instr)
    PValue uav_id = from_nir(instr->src[0], 0);
    GPRVector dest = make_dest(instr);
    auto ir = new GDSInstr(DS_OP_ADD_RET, dest, m_atomic_update, uav_id,
-                          nir_intrinsic_base(instr));
+                          remap_atomic_base(nir_intrinsic_base(instr)));
    emit_instruction(ir);
    return true;
 }
@@ -252,9 +252,11 @@ bool EmitSSBOInstruction::emit_atomic_pre_dec(const nir_intrinsic_instr *instr)
    PValue uav_id = from_nir(instr->src[0], 0);
 
    auto ir = new GDSInstr(DS_OP_SUB_RET, dest, m_atomic_update, uav_id,
-                          nir_intrinsic_base(instr));
+                          remap_atomic_base(nir_intrinsic_base(instr)));
    emit_instruction(ir);
 
+   emit_instruction(new AluInstruction(op2_sub_int,  dest.x(), dest.x(), literal(1), last_write));
+
    return true;
 }
 
@@ -263,7 +265,7 @@ bool EmitSSBOInstruction::emit_load_ssbo(const nir_intrinsic_instr* instr)
    GPRVector dest = make_dest(instr);
 
    /** src0 not used, should be some offset */
-   auto addr = from_nir_with_fetch_constant(instr->src[1], 0);
+   auto addr = from_nir(instr->src[1], 0);
    PValue addr_temp = create_register_from_nir_src(instr->src[1], 1);
 
    /** Should be lowered in nir */
diff --git a/src/gallium/drivers/r600/sfn/sfn_ir_to_assembly.cpp b/src/gallium/drivers/r600/sfn/sfn_ir_to_assembly.cpp
index 6af1dd3fde6..6807eb9deee 100644
--- a/src/gallium/drivers/r600/sfn/sfn_ir_to_assembly.cpp
+++ b/src/gallium/drivers/r600/sfn/sfn_ir_to_assembly.cpp
@@ -893,26 +893,12 @@ bool AssemblyFromShaderLegacyImpl::emit_gds(const GDSInstr& instr)
          struct r600_bytecode_alu alu;
 
          memset(&alu, 0, sizeof(alu));
-         alu.op = opcode_map.at(op2_lshr_int);
-         alu.dst.sel = addr->sel();
-         alu.dst.chan = addr->chan();
-         alu.src[0].sel = addr->sel();
-         alu.src[0].chan = addr->chan();
-         alu.src[1].sel = ALU_SRC_LITERAL;
-         alu.src[1].value = 2;
-         alu.last = 1;
-         alu.dst.write = 1;
-         int r = r600_bytecode_add_alu(m_bc, &alu);
-         if (r)
-            return false;
-
-         memset(&alu, 0, sizeof(alu));
          alu.op = opcode_map.at(op1_mova_int);
          alu.dst.chan = 0;
          alu.src[0].sel = addr->sel();
          alu.src[0].chan = addr->chan();
          alu.last = 1;
-         r = r600_bytecode_add_alu(m_bc, &alu);
+         int r = r600_bytecode_add_alu(m_bc, &alu);
          if (r)
             return false;
 
@@ -934,7 +920,7 @@ bool AssemblyFromShaderLegacyImpl::emit_gds(const GDSInstr& instr)
       }
    } else {
       const LiteralValue& addr_reg = static_cast<const LiteralValue&>(*addr);
-      uav_idx = addr_reg.value() >> 2;
+      uav_idx = addr_reg.value();
    }
 
    memset(&gds, 0, sizeof(struct r600_bytecode_gds));
diff --git a/src/gallium/drivers/r600/sfn/sfn_shader_base.cpp b/src/gallium/drivers/r600/sfn/sfn_shader_base.cpp
index 5e48e6da0fa..1a8ae1c7b39 100644
--- a/src/gallium/drivers/r600/sfn/sfn_shader_base.cpp
+++ b/src/gallium/drivers/r600/sfn/sfn_shader_base.cpp
@@ -252,10 +252,15 @@ bool ShaderFromNirProcessor::process_uniforms(nir_variable *uniform)
       ++sh_info().nhwatomic_ranges;
       atom.buffer_id = uniform->data.binding;
       atom.hw_idx = m_atomic_base + m_next_hwatomic_loc;
-      atom.start = m_next_hwatomic_loc;
+
+      atom.start = uniform->data.offset >> 2;
       atom.end = atom.start + natomics - 1;
-      m_next_hwatomic_loc = atom.end + 1;
-      //atom.array_id = uniform->type->is_array() ? 1 : 0;
+
+      if (m_atomic_base_map.find(uniform->data.binding) ==
+          m_atomic_base_map.end())
+         m_atomic_base_map[uniform->data.binding] = m_next_hwatomic_loc;
+
+      m_next_hwatomic_loc += natomics;
 
       m_sel.info.file_count[TGSI_FILE_HW_ATOMIC] += atom.end  - atom.start + 1;
 
@@ -263,8 +268,11 @@ bool ShaderFromNirProcessor::process_uniforms(nir_variable *uniform)
               << m_sel.info.file_count[TGSI_FILE_HW_ATOMIC] << "\n";
    }
 
-   if (uniform->type->is_image() || uniform->data.mode == nir_var_mem_ssbo) {
+   auto type = uniform->type->is_array() ? uniform->type->without_array(): uniform->type;
+   if (type->is_image() || uniform->data.mode == nir_var_mem_ssbo) {
       sh_info().uses_images = 1;
+      if (uniform->type->is_array())
+         sh_info().indirect_files |= TGSI_FILE_IMAGE;
    }
 
    if (uniform->type->is_image()) {