libclc: Revert generic vload/vstore to pure CLC and move asm to R600

The assembly optimizations were making unsafe assumptions about which
address spaces had which identifiers.

Signed-off-by: Aaron Watry <awatry@gmail.com>

10 files changed, 219 insertions, 118 deletions

diff --git a/generic/lib/SOURCES b/generic/lib/SOURCES
index c2da3d7..ccbb59a 100644
--- a/generic/lib/SOURCES
+++ b/generic/lib/SOURCES
@@ -26,10 +26,6 @@ shared/clamp.cl
 shared/max.cl
 shared/min.cl
 shared/vload.cl
-shared/vload_if.ll
-shared/vload_impl.ll
 shared/vstore.cl
-shared/vstore_if.ll
-shared/vstore_impl.ll
 workitem/get_global_id.cl
 workitem/get_global_size.cl
diff --git a/generic/lib/shared/vload.cl b/generic/lib/shared/vload.cl
index dd28347..eba539e 100644
--- a/generic/lib/shared/vload.cl
+++ b/generic/lib/shared/vload.cl
@@ -27,12 +27,13 @@
     VLOAD_VECTORIZE(SCALAR_GENTYPE, __constant) \
     VLOAD_VECTORIZE(SCALAR_GENTYPE, __global) \
 
-//int/uint are special... see below
 #define VLOAD_TYPES() \
     VLOAD_ADDR_SPACES(char) \
     VLOAD_ADDR_SPACES(uchar) \
     VLOAD_ADDR_SPACES(short) \
     VLOAD_ADDR_SPACES(ushort) \
+    VLOAD_ADDR_SPACES(int) \
+    VLOAD_ADDR_SPACES(uint) \
     VLOAD_ADDR_SPACES(long) \
     VLOAD_ADDR_SPACES(ulong) \
     VLOAD_ADDR_SPACES(float) \
@@ -42,58 +43,4 @@ VLOAD_TYPES()
 #ifdef cl_khr_fp64
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
     VLOAD_ADDR_SPACES(double)
-#endif
-
-VLOAD_VECTORIZE(int, __private)
-VLOAD_VECTORIZE(int, __local)
-VLOAD_VECTORIZE(int, __constant)
-VLOAD_VECTORIZE(uint, __private)
-VLOAD_VECTORIZE(uint, __local)
-VLOAD_VECTORIZE(uint, __constant)
-
-_CLC_OVERLOAD _CLC_DEF int3 vload3(size_t offset, const global int *x) {
-  return (int3)(vload2(offset, x), x[offset+2]);
-}
-_CLC_OVERLOAD _CLC_DEF uint3 vload3(size_t offset, const global uint *x) {
-  return (uint3)(vload2(offset, x), x[offset+2]);
-}
-        
-/*Note: R600 doesn't support load <3 x ?>... so
- * those functions aren't actually overridden here
- */
-_CLC_DECL int2 __clc_vload2_int__global(size_t offset, const __global int *);
-_CLC_DECL int4 __clc_vload4_int__global(size_t offset, const __global int *);
-_CLC_DECL int8 __clc_vload8_int__global(size_t offset, const __global int *);
-_CLC_DECL int16 __clc_vload16_int__global(size_t offset, const __global int *);
-
-_CLC_OVERLOAD _CLC_DEF int2 vload2(size_t offset, const global int *x) {
-  return __clc_vload2_int__global(offset, x);
-}
-_CLC_OVERLOAD _CLC_DEF int4 vload4(size_t offset, const global int *x) {
-  return __clc_vload4_int__global(offset, x);
-}
-_CLC_OVERLOAD _CLC_DEF int8 vload8(size_t offset, const global int *x) {
-  return __clc_vload8_int__global(offset, x);
-}
-_CLC_OVERLOAD _CLC_DEF int16 vload16(size_t offset, const global int *x) {
-  return __clc_vload16_int__global(offset, x);
-}
-
-_CLC_DECL uint2 __clc_vload2_uint__global(size_t offset, const __global uint *);
-_CLC_DECL uint4 __clc_vload4_uint__global(size_t offset, const __global uint *);
-_CLC_DECL uint8 __clc_vload8_uint__global(size_t offset, const __global uint *);
-_CLC_DECL uint16 __clc_vload16_uint__global(size_t offset, const __global uint *);
-
-_CLC_OVERLOAD _CLC_DEF uint2 vload2(size_t offset, const global uint *x) {
-  return __clc_vload2_uint__global(offset, x);
-}
-_CLC_OVERLOAD _CLC_DEF uint4 vload4(size_t offset, const global uint *x) {
-  return __clc_vload4_uint__global(offset, x);
-}
-_CLC_OVERLOAD _CLC_DEF uint8 vload8(size_t offset, const global uint *x) {
-  return __clc_vload8_uint__global(offset, x);
-}
-_CLC_OVERLOAD _CLC_DEF uint16 vload16(size_t offset, const global uint *x) {
-  return __clc_vload16_uint__global(offset, x);
-}
-
+#endif
\ No newline at end of file
diff --git a/generic/lib/shared/vstore.cl b/generic/lib/shared/vstore.cl
index c8b8cd5..daab158 100644
--- a/generic/lib/shared/vstore.cl
+++ b/generic/lib/shared/vstore.cl
@@ -34,12 +34,13 @@
     VSTORE_VECTORIZE(SCALAR_GENTYPE, __local) \
     VSTORE_VECTORIZE(SCALAR_GENTYPE, __global) \
 
-//int/uint are special... see below
 #define VSTORE_TYPES() \
     VSTORE_ADDR_SPACES(char) \
     VSTORE_ADDR_SPACES(uchar) \
     VSTORE_ADDR_SPACES(short) \
     VSTORE_ADDR_SPACES(ushort) \
+    VSTORE_ADDR_SPACES(int) \
+    VSTORE_ADDR_SPACES(uint) \
     VSTORE_ADDR_SPACES(long) \
     VSTORE_ADDR_SPACES(ulong) \
     VSTORE_ADDR_SPACES(float) \
@@ -49,60 +50,4 @@ VSTORE_TYPES()
 #ifdef cl_khr_fp64
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
     VSTORE_ADDR_SPACES(double)
-#endif
-
-VSTORE_VECTORIZE(int, __private)
-VSTORE_VECTORIZE(int, __local)
-VSTORE_VECTORIZE(uint, __private)
-VSTORE_VECTORIZE(uint, __local)
-
-_CLC_OVERLOAD _CLC_DEF void vstore3(int3 vec, size_t offset, global int *mem) {
-    mem[offset] = vec.s0;
-    mem[offset+1] = vec.s1;
-    mem[offset+2] = vec.s2;
-}
-_CLC_OVERLOAD _CLC_DEF void vstore3(uint3 vec, size_t offset, global uint *mem) {
-    mem[offset] = vec.s0;
-    mem[offset+1] = vec.s1;
-    mem[offset+2] = vec.s2;
-}
-
-/*Note: R600 doesn't support store <3 x ?>... so
- * those functions aren't actually overridden here... lowest-common-denominator
- */
-_CLC_DECL void __clc_vstore2_int__global(int2 vec, size_t offset, __global int *);
-_CLC_DECL void __clc_vstore4_int__global(int4 vec, size_t offset, __global int *);
-_CLC_DECL void __clc_vstore8_int__global(int8 vec, size_t offset, __global int *);
-_CLC_DECL void __clc_vstore16_int__global(int16 vec, size_t offset, __global int *);
-
-_CLC_OVERLOAD _CLC_DEF void vstore2(int2 vec, size_t offset, global int *x) {
-    __clc_vstore2_int__global(vec, offset, x);
-}
-_CLC_OVERLOAD _CLC_DEF void vstore4(int4 vec, size_t offset, global int *x) {
-    __clc_vstore4_int__global(vec, offset, x);
-}
-_CLC_OVERLOAD _CLC_DEF void vstore8(int8 vec, size_t offset, global int *x) {
-    __clc_vstore8_int__global(vec, offset, x);
-}
-_CLC_OVERLOAD _CLC_DEF void vstore16(int16 vec, size_t offset, global int *x) {
-    __clc_vstore16_int__global(vec, offset, x);
-}
-
-_CLC_DECL void __clc_vstore2_uint__global(uint2 vec, size_t offset, __global uint *);
-_CLC_DECL void __clc_vstore4_uint__global(uint4 vec, size_t offset, __global uint *);
-_CLC_DECL void __clc_vstore8_uint__global(uint8 vec, size_t offset, __global uint *);
-_CLC_DECL void __clc_vstore16_uint__global(uint16 vec, size_t offset, __global uint *);
-
-_CLC_OVERLOAD _CLC_DEF void vstore2(uint2 vec, size_t offset, global uint *x) {
-    __clc_vstore2_uint__global(vec, offset, x);
-}
-_CLC_OVERLOAD _CLC_DEF void vstore4(uint4 vec, size_t offset, global uint *x) {
-    __clc_vstore4_uint__global(vec, offset, x);
-}
-_CLC_OVERLOAD _CLC_DEF void vstore8(uint8 vec, size_t offset, global uint *x) {
-    __clc_vstore8_uint__global(vec, offset, x);
-}
-_CLC_OVERLOAD _CLC_DEF void vstore16(uint16 vec, size_t offset, global uint *x) {
-    __clc_vstore16_uint__global(vec, offset, x);
-}
-
+#endif
\ No newline at end of file
diff --git a/r600/lib/SOURCES b/r600/lib/SOURCES
index 16ef3ac..889a697 100644
--- a/r600/lib/SOURCES
+++ b/r600/lib/SOURCES
@@ -4,3 +4,9 @@ workitem/get_local_id.ll
 workitem/get_global_size.ll
 synchronization/barrier.cl
 synchronization/barrier_impl.ll
+shared/vload.cl
+shared/vload_if.ll
+shared/vload_impl.ll
+shared/vstore.cl
+shared/vstore_if.ll
+shared/vstore_impl.ll
diff --git a/r600/lib/shared/vload.cl b/r600/lib/shared/vload.cl
new file mode 100644
index 0000000..dd28347
--- /dev/null
+++ b/r600/lib/shared/vload.cl
@@ -0,0 +1,99 @@
+#include <clc/clc.h>
+
+#define VLOAD_VECTORIZE(PRIM_TYPE, ADDR_SPACE) \
+  _CLC_OVERLOAD _CLC_DEF PRIM_TYPE##2 vload2(size_t offset, const ADDR_SPACE PRIM_TYPE *x) { \
+    return (PRIM_TYPE##2)(x[offset] , x[offset+1]); \
+  } \
+\
+  _CLC_OVERLOAD _CLC_DEF PRIM_TYPE##3 vload3(size_t offset, const ADDR_SPACE PRIM_TYPE *x) { \
+    return (PRIM_TYPE##3)(x[offset] , x[offset+1], x[offset+2]); \
+  } \
+\
+  _CLC_OVERLOAD _CLC_DEF PRIM_TYPE##4 vload4(size_t offset, const ADDR_SPACE PRIM_TYPE *x) { \
+    return (PRIM_TYPE##4)(x[offset], x[offset+1], x[offset+2], x[offset+3]); \
+  } \
+\
+  _CLC_OVERLOAD _CLC_DEF PRIM_TYPE##8 vload8(size_t offset, const ADDR_SPACE PRIM_TYPE *x) { \
+    return (PRIM_TYPE##8)(vload4(offset, x), vload4(offset+4, x)); \
+  } \
+\
+  _CLC_OVERLOAD _CLC_DEF PRIM_TYPE##16 vload16(size_t offset, const ADDR_SPACE PRIM_TYPE *x) { \
+    return (PRIM_TYPE##16)(vload8(offset, x), vload8(offset+8, x)); \
+  } \
+
+#define VLOAD_ADDR_SPACES(SCALAR_GENTYPE) \
+    VLOAD_VECTORIZE(SCALAR_GENTYPE, __private) \
+    VLOAD_VECTORIZE(SCALAR_GENTYPE, __local) \
+    VLOAD_VECTORIZE(SCALAR_GENTYPE, __constant) \
+    VLOAD_VECTORIZE(SCALAR_GENTYPE, __global) \
+
+//int/uint are special... see below
+#define VLOAD_TYPES() \
+    VLOAD_ADDR_SPACES(char) \
+    VLOAD_ADDR_SPACES(uchar) \
+    VLOAD_ADDR_SPACES(short) \
+    VLOAD_ADDR_SPACES(ushort) \
+    VLOAD_ADDR_SPACES(long) \
+    VLOAD_ADDR_SPACES(ulong) \
+    VLOAD_ADDR_SPACES(float) \
+
+VLOAD_TYPES()
+
+#ifdef cl_khr_fp64
+#pragma OPENCL EXTENSION cl_khr_fp64 : enable
+    VLOAD_ADDR_SPACES(double)
+#endif
+
+VLOAD_VECTORIZE(int, __private)
+VLOAD_VECTORIZE(int, __local)
+VLOAD_VECTORIZE(int, __constant)
+VLOAD_VECTORIZE(uint, __private)
+VLOAD_VECTORIZE(uint, __local)
+VLOAD_VECTORIZE(uint, __constant)
+
+_CLC_OVERLOAD _CLC_DEF int3 vload3(size_t offset, const global int *x) {
+  return (int3)(vload2(offset, x), x[offset+2]);
+}
+_CLC_OVERLOAD _CLC_DEF uint3 vload3(size_t offset, const global uint *x) {
+  return (uint3)(vload2(offset, x), x[offset+2]);
+}
+        
+/*Note: R600 doesn't support load <3 x ?>... so
+ * those functions aren't actually overridden here
+ */
+_CLC_DECL int2 __clc_vload2_int__global(size_t offset, const __global int *);
+_CLC_DECL int4 __clc_vload4_int__global(size_t offset, const __global int *);
+_CLC_DECL int8 __clc_vload8_int__global(size_t offset, const __global int *);
+_CLC_DECL int16 __clc_vload16_int__global(size_t offset, const __global int *);
+
+_CLC_OVERLOAD _CLC_DEF int2 vload2(size_t offset, const global int *x) {
+  return __clc_vload2_int__global(offset, x);
+}
+_CLC_OVERLOAD _CLC_DEF int4 vload4(size_t offset, const global int *x) {
+  return __clc_vload4_int__global(offset, x);
+}
+_CLC_OVERLOAD _CLC_DEF int8 vload8(size_t offset, const global int *x) {
+  return __clc_vload8_int__global(offset, x);
+}
+_CLC_OVERLOAD _CLC_DEF int16 vload16(size_t offset, const global int *x) {
+  return __clc_vload16_int__global(offset, x);
+}
+
+_CLC_DECL uint2 __clc_vload2_uint__global(size_t offset, const __global uint *);
+_CLC_DECL uint4 __clc_vload4_uint__global(size_t offset, const __global uint *);
+_CLC_DECL uint8 __clc_vload8_uint__global(size_t offset, const __global uint *);
+_CLC_DECL uint16 __clc_vload16_uint__global(size_t offset, const __global uint *);
+
+_CLC_OVERLOAD _CLC_DEF uint2 vload2(size_t offset, const global uint *x) {
+  return __clc_vload2_uint__global(offset, x);
+}
+_CLC_OVERLOAD _CLC_DEF uint4 vload4(size_t offset, const global uint *x) {
+  return __clc_vload4_uint__global(offset, x);
+}
+_CLC_OVERLOAD _CLC_DEF uint8 vload8(size_t offset, const global uint *x) {
+  return __clc_vload8_uint__global(offset, x);
+}
+_CLC_OVERLOAD _CLC_DEF uint16 vload16(size_t offset, const global uint *x) {
+  return __clc_vload16_uint__global(offset, x);
+}
+
diff --git a/generic/lib/shared/vload_if.ll b/r600/lib/shared/vload_if.ll
index bda592b..bda592b 100644
--- a/generic/lib/shared/vload_if.ll
+++ b/r600/lib/shared/vload_if.ll
diff --git a/generic/lib/shared/vload_impl.ll b/r600/lib/shared/vload_impl.ll
index 1333aac..1333aac 100644
--- a/generic/lib/shared/vload_impl.ll
+++ b/r600/lib/shared/vload_impl.ll
diff --git a/r600/lib/shared/vstore.cl b/r600/lib/shared/vstore.cl
new file mode 100644
index 0000000..c8b8cd5
--- /dev/null
+++ b/r600/lib/shared/vstore.cl
@@ -0,0 +1,108 @@
+#include <clc/clc.h>
+
+#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
+
+#define VSTORE_VECTORIZE(PRIM_TYPE, ADDR_SPACE) \
+  _CLC_OVERLOAD _CLC_DEF void vstore2(PRIM_TYPE##2 vec, size_t offset, ADDR_SPACE PRIM_TYPE *mem) { \
+    mem[offset] = vec.s0; \
+    mem[offset+1] = vec.s1; \
+  } \
+\
+  _CLC_OVERLOAD _CLC_DEF void vstore3(PRIM_TYPE##3 vec, size_t offset, ADDR_SPACE PRIM_TYPE *mem) { \
+    mem[offset] = vec.s0; \
+    mem[offset+1] = vec.s1; \
+    mem[offset+2] = vec.s2; \
+  } \
+\
+  _CLC_OVERLOAD _CLC_DEF void vstore4(PRIM_TYPE##4 vec, size_t offset, ADDR_SPACE PRIM_TYPE *mem) { \
+    vstore2(vec.lo, offset, mem); \
+    vstore2(vec.hi, offset+2, mem); \
+  } \
+\
+  _CLC_OVERLOAD _CLC_DEF void vstore8(PRIM_TYPE##8 vec, size_t offset, ADDR_SPACE PRIM_TYPE *mem) { \
+    vstore4(vec.lo, offset, mem); \
+    vstore4(vec.hi, offset+4, mem); \
+  } \
+\
+  _CLC_OVERLOAD _CLC_DEF void vstore16(PRIM_TYPE##16 vec, size_t offset, ADDR_SPACE PRIM_TYPE *mem) { \
+    vstore8(vec.lo, offset, mem); \
+    vstore8(vec.hi, offset+8, mem); \
+  } \
+
+#define VSTORE_ADDR_SPACES(SCALAR_GENTYPE) \
+    VSTORE_VECTORIZE(SCALAR_GENTYPE, __private) \
+    VSTORE_VECTORIZE(SCALAR_GENTYPE, __local) \
+    VSTORE_VECTORIZE(SCALAR_GENTYPE, __global) \
+
+//int/uint are special... see below
+#define VSTORE_TYPES() \
+    VSTORE_ADDR_SPACES(char) \
+    VSTORE_ADDR_SPACES(uchar) \
+    VSTORE_ADDR_SPACES(short) \
+    VSTORE_ADDR_SPACES(ushort) \
+    VSTORE_ADDR_SPACES(long) \
+    VSTORE_ADDR_SPACES(ulong) \
+    VSTORE_ADDR_SPACES(float) \
+
+VSTORE_TYPES()
+
+#ifdef cl_khr_fp64
+#pragma OPENCL EXTENSION cl_khr_fp64 : enable
+    VSTORE_ADDR_SPACES(double)
+#endif
+
+VSTORE_VECTORIZE(int, __private)
+VSTORE_VECTORIZE(int, __local)
+VSTORE_VECTORIZE(uint, __private)
+VSTORE_VECTORIZE(uint, __local)
+
+_CLC_OVERLOAD _CLC_DEF void vstore3(int3 vec, size_t offset, global int *mem) {
+    mem[offset] = vec.s0;
+    mem[offset+1] = vec.s1;
+    mem[offset+2] = vec.s2;
+}
+_CLC_OVERLOAD _CLC_DEF void vstore3(uint3 vec, size_t offset, global uint *mem) {
+    mem[offset] = vec.s0;
+    mem[offset+1] = vec.s1;
+    mem[offset+2] = vec.s2;
+}
+
+/*Note: R600 doesn't support store <3 x ?>... so
+ * those functions aren't actually overridden here... lowest-common-denominator
+ */
+_CLC_DECL void __clc_vstore2_int__global(int2 vec, size_t offset, __global int *);
+_CLC_DECL void __clc_vstore4_int__global(int4 vec, size_t offset, __global int *);
+_CLC_DECL void __clc_vstore8_int__global(int8 vec, size_t offset, __global int *);
+_CLC_DECL void __clc_vstore16_int__global(int16 vec, size_t offset, __global int *);
+
+_CLC_OVERLOAD _CLC_DEF void vstore2(int2 vec, size_t offset, global int *x) {
+    __clc_vstore2_int__global(vec, offset, x);
+}
+_CLC_OVERLOAD _CLC_DEF void vstore4(int4 vec, size_t offset, global int *x) {
+    __clc_vstore4_int__global(vec, offset, x);
+}
+_CLC_OVERLOAD _CLC_DEF void vstore8(int8 vec, size_t offset, global int *x) {
+    __clc_vstore8_int__global(vec, offset, x);
+}
+_CLC_OVERLOAD _CLC_DEF void vstore16(int16 vec, size_t offset, global int *x) {
+    __clc_vstore16_int__global(vec, offset, x);
+}
+
+_CLC_DECL void __clc_vstore2_uint__global(uint2 vec, size_t offset, __global uint *);
+_CLC_DECL void __clc_vstore4_uint__global(uint4 vec, size_t offset, __global uint *);
+_CLC_DECL void __clc_vstore8_uint__global(uint8 vec, size_t offset, __global uint *);
+_CLC_DECL void __clc_vstore16_uint__global(uint16 vec, size_t offset, __global uint *);
+
+_CLC_OVERLOAD _CLC_DEF void vstore2(uint2 vec, size_t offset, global uint *x) {
+    __clc_vstore2_uint__global(vec, offset, x);
+}
+_CLC_OVERLOAD _CLC_DEF void vstore4(uint4 vec, size_t offset, global uint *x) {
+    __clc_vstore4_uint__global(vec, offset, x);
+}
+_CLC_OVERLOAD _CLC_DEF void vstore8(uint8 vec, size_t offset, global uint *x) {
+    __clc_vstore8_uint__global(vec, offset, x);
+}
+_CLC_OVERLOAD _CLC_DEF void vstore16(uint16 vec, size_t offset, global uint *x) {
+    __clc_vstore16_uint__global(vec, offset, x);
+}
+
diff --git a/generic/lib/shared/vstore_if.ll b/r600/lib/shared/vstore_if.ll
index 382a8a8..382a8a8 100644
--- a/generic/lib/shared/vstore_if.ll
+++ b/r600/lib/shared/vstore_if.ll
diff --git a/generic/lib/shared/vstore_impl.ll b/r600/lib/shared/vstore_impl.ll
index 8790a8f..8790a8f 100644
--- a/generic/lib/shared/vstore_impl.ll
+++ b/r600/lib/shared/vstore_impl.ll