audioresample: added ARM NEON support

This adds ARM NEON accelerated code paths for 16-bit integer
and 32-bit floating point samples.

It is a modified combination of patches #3 and #5 from Jyri Sarha
( http://lists.xiph.org/pipermail/speex-dev/2011-September/008240.html &
http://lists.xiph.org/pipermail/speex-dev/2011-September/008238.html )

Signed-off-by: Carlos Rafael Giani <dv@pseudoterminal.org>

5 files changed, 244 insertions, 4 deletions

diff --git a/gst/audioresample/Makefile.am b/gst/audioresample/Makefile.am
index 0c7e871c7..78b4e24f9 100644
--- a/gst/audioresample/Makefile.am
+++ b/gst/audioresample/Makefile.am
@@ -39,6 +39,7 @@ noinst_HEADERS = \
 	gstaudioresample.h \
 	resample.c \
 	resample_sse.h \
+	resample_neon.h \
 	speex_resampler.h \
 	speex_resampler_wrapper.h
 
diff --git a/gst/audioresample/resample.c b/gst/audioresample/resample.c
index b6b1de660..98d006ca6 100644
--- a/gst/audioresample/resample.c
+++ b/gst/audioresample/resample.c
@@ -88,6 +88,12 @@
 #endif
 #endif
 
+#ifdef _USE_NEON
+#ifndef HAVE_ARM_NEON
+#undef _USE_NEON
+#endif
+#endif
+
 static inline void *
 speex_alloc (int size)
 {
@@ -134,6 +140,10 @@ speex_free (void *ptr)
 #include "resample_sse.h"
 #endif
 
+#ifdef _USE_NEON
+#include "resample_neon.h"
+#endif
+
 /* Numer of elements to allocate on the stack */
 #ifdef VAR_ARRAYS
 #define FIXED_STACK_ALLOC 8192
@@ -162,6 +172,16 @@ speex_free (void *ptr)
 #define SSE2_FALLBACK(macro)
 #endif
 
+#ifdef _USE_NEON
+#define NEON_FALLBACK(macro) \
+  if (st->use_neon) goto neon_##macro##_neon; {
+#define NEON_IMPLEMENTATION(macro) \
+  goto neon_##macro##_end; } neon_##macro##_neon: {
+#define NEON_END(macro) neon_##macro##_end:; }
+#else
+#define NEON_FALLBACK(macro)
+#endif
+
 
 typedef int (*resampler_basic_func) (SpeexResamplerState *, spx_uint32_t,
     const spx_word16_t *, spx_uint32_t *, spx_word16_t *, spx_uint32_t *);
@@ -201,6 +221,7 @@ struct SpeexResamplerState_
 
   int use_sse:1;
   int use_sse2:1;
+  int use_neon:1;
 };
 
 static double kaiser12_table[68] = {
@@ -457,6 +478,7 @@ resampler_basic_direct_single (SpeexResamplerState * st,
     const spx_word16_t *iptr = &in[last_sample];
 
     SSE_FALLBACK (INNER_PRODUCT_SINGLE)
+    NEON_FALLBACK (INNER_PRODUCT_SINGLE)
         sum = 0;
     for (j = 0; j < N; j++)
       sum += MULT16_16 (sinc[j], iptr[j]);
@@ -473,7 +495,11 @@ resampler_basic_direct_single (SpeexResamplerState * st,
       }
       sum = accum[0] + accum[1] + accum[2] + accum[3];
 */
-#ifdef OVERRIDE_INNER_PRODUCT_SINGLE
+#if defined(OVERRIDE_INNER_PRODUCT_SINGLE) && defined(_USE_NEON)
+    NEON_IMPLEMENTATION (INNER_PRODUCT_SINGLE)
+    sum = inner_product_single (sinc, iptr, N);
+    NEON_END(INNER_PRODUCT_SINGLE)
+#elif defined(OVERRIDE_INNER_PRODUCT_SINGLE) && defined(_USE_SSE)
     SSE_IMPLEMENTATION (INNER_PRODUCT_SINGLE)
         sum = inner_product_single (sinc, iptr, N);
     SSE_END (INNER_PRODUCT_SINGLE)
@@ -528,7 +554,7 @@ resampler_basic_direct_double (SpeexResamplerState * st,
       accum[3] += sinc[j + 3] * iptr[j + 3];
     }
     sum = accum[0] + accum[1] + accum[2] + accum[3];
-#ifdef OVERRIDE_INNER_PRODUCT_DOUBLE
+#if defined(OVERRIDE_INNER_PRODUCT_DOUBLE) && defined(_USE_SSE2)
     SSE2_IMPLEMENTATION (INNER_PRODUCT_DOUBLE)
         sum = inner_product_double (sinc, iptr, N);
     SSE2_END (INNER_PRODUCT_DOUBLE)
@@ -607,7 +633,7 @@ resampler_basic_interpolate_single (SpeexResamplerState * st,
             1)) + MULT16_32_Q15 (interp[1], SHR32 (accum[1],
             1)) + MULT16_32_Q15 (interp[2], SHR32 (accum[2],
             1)) + MULT16_32_Q15 (interp[3], SHR32 (accum[3], 1));
-#ifdef OVERRIDE_INTERPOLATE_PRODUCT_SINGLE
+#if defined(OVERRIDE_INTERPOLATE_PRODUCT_SINGLE) && defined(_USE_SSE)
     SSE_IMPLEMENTATION (INTERPOLATE_PRODUCT_SINGLE)
         cubic_coef (frac, interp);
     sum =
@@ -697,7 +723,7 @@ resampler_basic_interpolate_double (SpeexResamplerState * st,
         MULT16_32_Q15 (interp[0], accum[0]) + MULT16_32_Q15 (interp[1],
         accum[1]) + MULT16_32_Q15 (interp[2],
         accum[2]) + MULT16_32_Q15 (interp[3], accum[3]);
-#ifdef OVERRIDE_INTERPOLATE_PRODUCT_DOUBLE
+#if defined(OVERRIDE_INTERPOLATE_PRODUCT_DOUBLE) && defined(_USE_SSE2)
     SSE2_IMPLEMENTATION (INTERPOLATE_PRODUCT_DOUBLE)
         cubic_coef (frac, interp);
     sum =
@@ -933,10 +959,18 @@ check_insn_set (SpeexResamplerState * st, const char *name)
 {
   if (!name)
     return;
+#ifdef _USE_SSE
   if (!strcmp (name, "sse"))
     st->use_sse = 1;
+#endif
+#ifdef _USE_SSE2
   if (!strcmp (name, "sse2"))
     st->use_sse = st->use_sse2 = 1;
+#endif
+#ifdef _USE_NEON
+  if (!strcmp (name, "neon"))
+    st->use_neon = 1;
+#endif
 }
 #endif
 
@@ -997,6 +1031,7 @@ speex_resampler_init_frac (spx_uint32_t nb_channels, spx_uint32_t ratio_num,
 #endif
 
   st->use_sse = st->use_sse2 = 0;
+  st->use_neon = 0;
 #if defined HAVE_ORC && !defined DISABLE_ORC
   orc_init ();
   {
diff --git a/gst/audioresample/resample_neon.h b/gst/audioresample/resample_neon.h
new file mode 100644
index 000000000..478488f93
--- /dev/null
+++ b/gst/audioresample/resample_neon.h
@@ -0,0 +1,202 @@
+/* Copyright (C) 2007-2008 Jean-Marc Valin
+ * Copyright (C) 2008 Thorvald Natvig
+ * Copyright (C) 2011 Texas Instruments
+ *               author Jyri Sarha
+ */
+/**
+   @file resample_neon.h
+   @brief Resampler functions (NEON version)
+*/
+/*
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+   - Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+   - Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+   - Neither the name of the Xiph.org Foundation nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR
+   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#include <arm_neon.h>
+
+#ifdef FIXED_POINT
+#ifdef __thumb2__ 
+static inline int32_t saturate_32bit_to_16bit(int32_t a) {
+    int32_t ret;
+    asm ("ssat %[ret], #16, %[a]"
+         : [ret] "=&r" (ret)
+         : [a] "r" (a)
+         : );
+    return ret;
+}
+#else
+static inline int32_t saturate_32bit_to_16bit(int32_t a) {
+    int32_t ret;
+    asm ("vmov.s32 d0[0], %[a]\n"
+         "vqmovn.s32 d0, q0\n"
+         "vmov.s16 %[ret], d0[0]\n"
+         : [ret] "=&r" (ret)
+         : [a] "r" (a)
+         : "q0");
+    return ret;
+}
+#endif
+#undef WORD2INT
+#define WORD2INT(x) (saturate_32bit_to_16bit(x))
+
+#define OVERRIDE_INNER_PRODUCT_SINGLE
+/* Only works when len % 4 == 0 */
+static inline int32_t inner_product_single(const int16_t *a, const int16_t *b, unsigned int len)
+{
+    int32_t ret;
+    uint32_t remainder = len % 16;
+    len = len - remainder;
+
+    asm volatile ("	 cmp %[len], #0\n"
+		  "	 bne 1f\n"
+		  "	 vld1.16 {d16}, [%[b]]!\n"
+		  "	 vld1.16 {d20}, [%[a]]!\n"
+		  "	 subs %[remainder], %[remainder], #4\n"
+		  "	 vmull.s16 q0, d16, d20\n"
+		  "      beq 5f\n" 
+		  "	 b 4f\n"
+		  "1:"
+		  "	 vld1.16 {d16, d17, d18, d19}, [%[b]]!\n"
+		  "	 vld1.16 {d20, d21, d22, d23}, [%[a]]!\n"
+		  "	 subs %[len], %[len], #16\n"
+		  "	 vmull.s16 q0, d16, d20\n"
+		  "	 vmlal.s16 q0, d17, d21\n"
+		  "	 vmlal.s16 q0, d18, d22\n"
+		  "	 vmlal.s16 q0, d19, d23\n"
+		  "	 beq 3f\n"
+		  "2:"
+		  "	 vld1.16 {d16, d17, d18, d19}, [%[b]]!\n"
+		  "	 vld1.16 {d20, d21, d22, d23}, [%[a]]!\n"
+		  "	 subs %[len], %[len], #16\n"
+		  "	 vmlal.s16 q0, d16, d20\n"
+		  "	 vmlal.s16 q0, d17, d21\n"
+		  "	 vmlal.s16 q0, d18, d22\n"
+		  "	 vmlal.s16 q0, d19, d23\n"
+		  "	 bne 2b\n"
+		  "3:"
+		  "	 cmp %[remainder], #0\n"
+		  "	 beq 5f\n"
+		  "4:"
+		  "	 vld1.16 {d16}, [%[b]]!\n"
+		  "	 vld1.16 {d20}, [%[a]]!\n"
+		  "	 subs %[remainder], %[remainder], #4\n"
+		  "	 vmlal.s16 q0, d16, d20\n"
+		  "	 bne 4b\n"
+		  "5:"
+		  "	 vaddl.s32 q0, d0, d1\n"
+		  "	 vadd.s64 d0, d0, d1\n"
+		  "	 vqmovn.s64 d0, q0\n"
+		  "	 vqrshrn.s32 d0, q0, #15\n"
+		  "	 vmov.s16 %[ret], d0[0]\n"
+		  : [ret] "=&r" (ret), [a] "+r" (a), [b] "+r" (b),
+		    [len] "+r" (len), [remainder] "+r" (remainder)
+		  :
+		  : "cc", "q0",
+		    "d16", "d17", "d18", "d19",
+		    "d20", "d21", "d22", "d23");
+
+    return ret;
+}
+#elif defined(FLOATING_POINT)
+
+static inline int32_t saturate_float_to_16bit(float a) {
+    int32_t ret;
+    asm ("vmov.f32 d0[0], %[a]\n"
+         "vcvt.s32.f32 d0, d0, #15\n"
+         "vqrshrn.s32 d0, q0, #15\n"
+         "vmov.s16 %[ret], d0[0]\n"
+         : [ret] "=&r" (ret)
+         : [a] "r" (a)
+         : "q0");
+    return ret;
+}
+#undef WORD2INT
+#define WORD2INT(x) (saturate_float_to_16bit(x))
+
+#define OVERRIDE_INNER_PRODUCT_SINGLE
+/* Only works when len % 4 == 0 */
+static inline float inner_product_single(const float *a, const float *b, unsigned int len)
+{
+    float ret;
+    uint32_t remainder = len % 16;
+    len = len - remainder;
+
+    asm volatile ("	 cmp %[len], #0\n"
+		  "	 bne 1f\n"
+		  "	 vld1.32 {q4}, [%[b]]!\n"
+		  "	 vld1.32 {q8}, [%[a]]!\n"
+		  "	 subs %[remainder], %[remainder], #4\n"
+		  "	 vmul.f32 q0, q4, q8\n"
+		  "      bne 4f\n" 
+		  "	 b 5f\n"
+		  "1:"
+		  "	 vld1.32 {q4, q5}, [%[b]]!\n"
+		  "	 vld1.32 {q8, q9}, [%[a]]!\n"
+		  "	 vld1.32 {q6, q7}, [%[b]]!\n"
+		  "	 vld1.32 {q10, q11}, [%[a]]!\n"
+		  "	 subs %[len], %[len], #16\n"
+		  "	 vmul.f32 q0, q4, q8\n"
+		  "	 vmul.f32 q1, q5, q9\n"
+		  "	 vmul.f32 q2, q6, q10\n"
+		  "	 vmul.f32 q3, q7, q11\n"
+		  "	 beq 3f\n"
+		  "2:"
+		  "	 vld1.32 {q4, q5}, [%[b]]!\n"
+		  "	 vld1.32 {q8, q9}, [%[a]]!\n"
+		  "	 vld1.32 {q6, q7}, [%[b]]!\n"
+		  "	 vld1.32 {q10, q11}, [%[a]]!\n"
+		  "	 subs %[len], %[len], #16\n"
+		  "	 vmla.f32 q0, q4, q8\n"
+		  "	 vmla.f32 q1, q5, q9\n"
+		  "	 vmla.f32 q2, q6, q10\n"
+		  "	 vmla.f32 q3, q7, q11\n"
+		  "	 bne 2b\n"
+		  "3:"
+		  "	 vadd.f32 q4, q0, q1\n"
+		  "	 vadd.f32 q5, q2, q3\n"
+		  "	 cmp %[remainder], #0\n"
+		  "	 vadd.f32 q0, q4, q5\n"
+		  "	 beq 5f\n"
+		  "4:"
+		  "	 vld1.32 {q6}, [%[b]]!\n"
+		  "	 vld1.32 {q10}, [%[a]]!\n"
+		  "	 subs %[remainder], %[remainder], #4\n"
+		  "	 vmla.f32 q0, q6, q10\n"
+		  "	 bne 4b\n"
+		  "5:"
+		  "	 vadd.f32 d0, d0, d1\n"
+		  "	 vpadd.f32 d0, d0, d0\n"
+		  "	 vmov.f32 %[ret], d0[0]\n"
+		  : [ret] "=&r" (ret), [a] "+r" (a), [b] "+r" (b),
+		    [len] "+l" (len), [remainder] "+l" (remainder)
+		  :
+		  : "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8",
+                    "q9", "q10", "q11");
+    return ret;
+}
+#endif
+
diff --git a/gst/audioresample/speex_resampler_float.c b/gst/audioresample/speex_resampler_float.c
index ef3df1551..33d02d100 100644
--- a/gst/audioresample/speex_resampler_float.c
+++ b/gst/audioresample/speex_resampler_float.c
@@ -19,6 +19,7 @@
 
 #define _USE_SSE
 #define _USE_SSE2
+#define _USE_NEON
 #define FLOATING_POINT
 #define OUTSIDE_SPEEX
 #define RANDOM_PREFIX resample_float
diff --git a/gst/audioresample/speex_resampler_int.c b/gst/audioresample/speex_resampler_int.c
index 6b8586047..02b3aca27 100644
--- a/gst/audioresample/speex_resampler_int.c
+++ b/gst/audioresample/speex_resampler_int.c
@@ -19,6 +19,7 @@
 
 #define FIXED_POINT 1
 #define OUTSIDE_SPEEX 1
+#define _USE_NEON
 #define RANDOM_PREFIX resample_int
 
 #include "resample.c"