tdf#143450: Fix special fp+integer struct return case for gcc_*_x86-64

For one, the loop in x86_64::fill_struct was backwards.

And for another, privateSnippedExecutor does not need special handling of FLOAT
and DOUBLE return values (they can simply be moved to %xmm0, as covered by the
general case), but rather for those small structs where floating-point member(s)
in a first eightbyte (targeting %xmm0) are followed by integer member(s) in a
second eightbyte (targeting %rax).  Extended testtools to cover two such cases.

Change-Id: I8e775a1d1ce2312610f265bcc8e40b09bdac56df
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/119576
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>

10 files changed, 108 insertions, 52 deletions

diff --git a/bridges/source/cpp_uno/gcc3_linux_x86-64/abi.cxx b/bridges/source/cpp_uno/gcc3_linux_x86-64/abi.cxx
index 0f75b0616a37..355a0a7500d7 100644
--- a/bridges/source/cpp_uno/gcc3_linux_x86-64/abi.cxx
+++ b/bridges/source/cpp_uno/gcc3_linux_x86-64/abi.cxx
@@ -259,6 +259,20 @@ bool x86_64::return_in_hidden_param( typelib_TypeDescriptionReference *pTypeRef
     return classify_argument(pTypeRef, classes, 0) == 0;
 }
 
+x86_64::ReturnKind x86_64::getReturnKind(typelib_TypeDescriptionReference * type) noexcept {
+    x86_64_reg_class classes[MAX_CLASSES];
+    auto const n = classify_argument(type, classes, 0);
+    if (n == 0) {
+        return ReturnKind::Memory;
+    }
+    if (n == 2 && (classes[0] == X86_64_SSE_CLASS || classes[0] == X86_64_SSESF_CLASS)
+        && (classes[1] == X86_64_INTEGER_CLASS || classes[1] == X86_64_INTEGERSI_CLASS))
+    {
+        return ReturnKind::RegistersSpecial;
+    }
+    return ReturnKind::RegistersGeneral;
+}
+
 void x86_64::fill_struct( typelib_TypeDescriptionReference *pTypeRef, const sal_uInt64 *pGPR, const double *pSSE, void *pStruct ) noexcept
 {
     enum x86_64_reg_class classes[MAX_CLASSES];
@@ -267,8 +281,8 @@ void x86_64::fill_struct( typelib_TypeDescriptionReference *pTypeRef, const sal_
     n = classify_argument( pTypeRef, classes, 0 );
 
     sal_uInt64 *pStructAlign = static_cast<sal_uInt64 *>( pStruct );
-    for ( n--; n >= 0; n-- )
-        switch ( classes[n] )
+    for ( int i = 0; i != n; ++i )
+        switch ( classes[i] )
         {
             case X86_64_INTEGER_CLASS:
             case X86_64_INTEGERSI_CLASS:
diff --git a/bridges/source/cpp_uno/gcc3_linux_x86-64/abi.hxx b/bridges/source/cpp_uno/gcc3_linux_x86-64/abi.hxx
index 962801f9da17..db1ed75aa243 100644
--- a/bridges/source/cpp_uno/gcc3_linux_x86-64/abi.hxx
+++ b/bridges/source/cpp_uno/gcc3_linux_x86-64/abi.hxx
@@ -51,6 +51,14 @@ bool examine_argument( typelib_TypeDescriptionReference *pTypeRef, int &nUsedGPR
 */
 bool return_in_hidden_param( typelib_TypeDescriptionReference *pTypeRef ) noexcept;
 
+enum class ReturnKind {
+    Memory,
+    RegistersGeneral,
+    RegistersSpecial
+};
+
+ReturnKind getReturnKind(typelib_TypeDescriptionReference * type) noexcept;
+
 void fill_struct( typelib_TypeDescriptionReference *pTypeRef, const sal_uInt64* pGPR, const double* pSSE, void *pStruct ) noexcept;
 
 } // namespace x86_64
diff --git a/bridges/source/cpp_uno/gcc3_linux_x86-64/call.hxx b/bridges/source/cpp_uno/gcc3_linux_x86-64/call.hxx
index c381aa8a9668..08b19f06bbbb 100644
--- a/bridges/source/cpp_uno/gcc3_linux_x86-64/call.hxx
+++ b/bridges/source/cpp_uno/gcc3_linux_x86-64/call.hxx
@@ -23,7 +23,7 @@
 
 #include <sal/types.h>
 
-extern "C" typelib_TypeClass cpp_vtable_call(
+extern "C" int cpp_vtable_call(
     sal_Int32 nFunctionIndex, sal_Int32 nVtableOffset,
     void ** gpreg, void ** fpreg, void ** ovrflw,
     sal_uInt64 * pRegisterReturn /* space for register return */ );
diff --git a/bridges/source/cpp_uno/gcc3_linux_x86-64/call.s b/bridges/source/cpp_uno/gcc3_linux_x86-64/call.s
index 447ac0cecfdd..00909188a7b6 100644
--- a/bridges/source/cpp_uno/gcc3_linux_x86-64/call.s
+++ b/bridges/source/cpp_uno/gcc3_linux_x86-64/call.s
@@ -55,18 +55,17 @@ privateSnippetExecutor:
 	
 	call	cpp_vtable_call
 
-	cmp	$10, %rax					# typelib_TypeClass_FLOAT
-	je	.Lfloat
-	cmp	$11, %rax					# typelib_TypeClass_DOUBLE
-	je	.Lfloat
+	cmp	$1, %rax
+	je	.Lspecial
 
-	movq	-144(%rbp), %rax		# Return value (int case)
-	movq	-136(%rbp), %rdx		# Return value (int case)
-	movq	-144(%rbp), %xmm0		# Return value (int case)
-	movq	-136(%rbp), %xmm1		# Return value (int case)
+	movq	-144(%rbp), %rax		# Potential return value (general case)
+	movq	-136(%rbp), %rdx		# Potential return value (general case)
+	movq	-144(%rbp), %xmm0		# Potential return value (general case)
+	movq	-136(%rbp), %xmm1		# Potential return value (general case)
 	jmp	.Lfinish
-.Lfloat:
-	movlpd	-144(%rbp), %xmm0		# Return value (float/double case)
+.Lspecial:
+	movq	-144(%rbp), %xmm0		# Return value (special fp and integer case)
+	movq	-136(%rbp), %rax		# Return value (special fp and integer case)
 
 .Lfinish:
 	leave
diff --git a/bridges/source/cpp_uno/gcc3_linux_x86-64/cpp2uno.cxx b/bridges/source/cpp_uno/gcc3_linux_x86-64/cpp2uno.cxx
index 28f4d4ed9d8d..4bf5061a472f 100644
--- a/bridges/source/cpp_uno/gcc3_linux_x86-64/cpp2uno.cxx
+++ b/bridges/source/cpp_uno/gcc3_linux_x86-64/cpp2uno.cxx
@@ -54,7 +54,9 @@ using namespace ::com::sun::star::uno;
 // [ret *] is present when we are returning a structure bigger than 16 bytes
 // Simple types are returned in rax, rdx (int), or xmm0, xmm1 (fp).
 // Similarly structures <= 16 bytes are in rax, rdx, xmm0, xmm1 as necessary.
-static typelib_TypeClass cpp2uno_call(
+//
+// The return value is the same as for cpp_vtable_call.
+static int cpp2uno_call(
     bridges::cpp_uno::shared::CppInterfaceProxy * pThis,
     const typelib_TypeDescription * pMemberTypeDescr,
     typelib_TypeDescriptionReference * pReturnTypeRef, // 0 indicates void return
@@ -69,13 +71,16 @@ static typelib_TypeClass cpp2uno_call(
     typelib_TypeDescription * pReturnTypeDescr = nullptr;
     if (pReturnTypeRef)
         TYPELIB_DANGER_GET( &pReturnTypeDescr, pReturnTypeRef );
+    x86_64::ReturnKind returnKind
+        = (pReturnTypeRef == nullptr || pReturnTypeRef->eTypeClass == typelib_TypeClass_VOID)
+        ? x86_64::ReturnKind::RegistersGeneral : x86_64::getReturnKind(pReturnTypeRef);
 
     void * pUnoReturn = nullptr;
     void * pCppReturn = nullptr; // complex return ptr: if != 0 && != pUnoReturn, reconversion need
 
     if ( pReturnTypeDescr )
     {
-        if ( x86_64::return_in_hidden_param( pReturnTypeRef ) )
+        if ( returnKind == x86_64::ReturnKind::Memory )
         {
             pCppReturn = *gpreg++;
             nr_gpr++;
@@ -202,7 +207,7 @@ static typelib_TypeClass cpp2uno_call(
 
         CPPU_CURRENT_NAMESPACE::raiseException( &aUnoExc, pThis->getBridge()->getUno2Cpp() ); // has to destruct the any
         // is here for dummy
-        return typelib_TypeClass_VOID;
+        return 0;
     }
     else // else no exception occurred...
     {
@@ -239,17 +244,19 @@ static typelib_TypeClass cpp2uno_call(
         }
         if ( pReturnTypeDescr )
         {
-            typelib_TypeClass eRet = pReturnTypeDescr->eTypeClass;
             TYPELIB_DANGER_RELEASE( pReturnTypeDescr );
-            return eRet;
         }
-        else
-            return typelib_TypeClass_VOID;
+        return returnKind == x86_64::ReturnKind::RegistersSpecial ? 1 : 0;
     }
 }
 
-
-typelib_TypeClass cpp_vtable_call(
+// Returns 0 for the general case where potential return values from privateSnippetExecutor can be
+// copied from pRegisterReturn to both %rax and %rdx (in that order) and to %xmm0 and %xmm1 (in that
+// order)---each specific return type will only require a subset of that copy operations, but the
+// other copies to those non--callee-saved registers will be redundant and harmless.  And returns 1
+// for the special case where return values from privateSnippetExecutor must be copied from
+// pRegisterReturn to %xmm0 and %rax (in that order).
+int cpp_vtable_call(
     sal_Int32 nFunctionIndex, sal_Int32 nVtableOffset,
     void ** gpreg, void ** fpreg, void ** ovrflw,
     sal_uInt64 * pRegisterReturn /* space for register return */ )
@@ -294,7 +301,7 @@ typelib_TypeClass cpp_vtable_call(
 
     TypeDescription aMemberDescr( pTypeDescr->ppAllMembers[nMemberPos] );
 
-    typelib_TypeClass eRet;
+    int eRet;
     switch ( aMemberDescr.get()->eTypeClass )
     {
         case typelib_TypeClass_INTERFACE_ATTRIBUTE:
@@ -331,11 +338,11 @@ typelib_TypeClass cpp_vtable_call(
             {
                 case 1: // acquire()
                     pCppI->acquireProxy(); // non virtual call!
-                    eRet = typelib_TypeClass_VOID;
+                    eRet = 0;
                     break;
                 case 2: // release()
                     pCppI->releaseProxy(); // non virtual call!
-                    eRet = typelib_TypeClass_VOID;
+                    eRet = 0;
                     break;
                 case 0: // queryInterface() opt
                 {
@@ -359,7 +366,7 @@ typelib_TypeClass cpp_vtable_call(
                             TYPELIB_DANGER_RELEASE( pTD );
 
                             reinterpret_cast<void **>( pRegisterReturn )[0] = gpreg[0];
-                            eRet = typelib_TypeClass_ANY;
+                            eRet = 0;
                             break;
                         }
                         TYPELIB_DANGER_RELEASE( pTD );
diff --git a/bridges/source/cpp_uno/gcc3_macosx_x86-64/abi.cxx b/bridges/source/cpp_uno/gcc3_macosx_x86-64/abi.cxx
index 0f75b0616a37..355a0a7500d7 100644
--- a/bridges/source/cpp_uno/gcc3_macosx_x86-64/abi.cxx
+++ b/bridges/source/cpp_uno/gcc3_macosx_x86-64/abi.cxx
@@ -259,6 +259,20 @@ bool x86_64::return_in_hidden_param( typelib_TypeDescriptionReference *pTypeRef
     return classify_argument(pTypeRef, classes, 0) == 0;
 }
 
+x86_64::ReturnKind x86_64::getReturnKind(typelib_TypeDescriptionReference * type) noexcept {
+    x86_64_reg_class classes[MAX_CLASSES];
+    auto const n = classify_argument(type, classes, 0);
+    if (n == 0) {
+        return ReturnKind::Memory;
+    }
+    if (n == 2 && (classes[0] == X86_64_SSE_CLASS || classes[0] == X86_64_SSESF_CLASS)
+        && (classes[1] == X86_64_INTEGER_CLASS || classes[1] == X86_64_INTEGERSI_CLASS))
+    {
+        return ReturnKind::RegistersSpecial;
+    }
+    return ReturnKind::RegistersGeneral;
+}
+
 void x86_64::fill_struct( typelib_TypeDescriptionReference *pTypeRef, const sal_uInt64 *pGPR, const double *pSSE, void *pStruct ) noexcept
 {
     enum x86_64_reg_class classes[MAX_CLASSES];
@@ -267,8 +281,8 @@ void x86_64::fill_struct( typelib_TypeDescriptionReference *pTypeRef, const sal_
     n = classify_argument( pTypeRef, classes, 0 );
 
     sal_uInt64 *pStructAlign = static_cast<sal_uInt64 *>( pStruct );
-    for ( n--; n >= 0; n-- )
-        switch ( classes[n] )
+    for ( int i = 0; i != n; ++i )
+        switch ( classes[i] )
         {
             case X86_64_INTEGER_CLASS:
             case X86_64_INTEGERSI_CLASS:
diff --git a/bridges/source/cpp_uno/gcc3_macosx_x86-64/abi.hxx b/bridges/source/cpp_uno/gcc3_macosx_x86-64/abi.hxx
index 962801f9da17..db1ed75aa243 100644
--- a/bridges/source/cpp_uno/gcc3_macosx_x86-64/abi.hxx
+++ b/bridges/source/cpp_uno/gcc3_macosx_x86-64/abi.hxx
@@ -51,6 +51,14 @@ bool examine_argument( typelib_TypeDescriptionReference *pTypeRef, int &nUsedGPR
 */
 bool return_in_hidden_param( typelib_TypeDescriptionReference *pTypeRef ) noexcept;
 
+enum class ReturnKind {
+    Memory,
+    RegistersGeneral,
+    RegistersSpecial
+};
+
+ReturnKind getReturnKind(typelib_TypeDescriptionReference * type) noexcept;
+
 void fill_struct( typelib_TypeDescriptionReference *pTypeRef, const sal_uInt64* pGPR, const double* pSSE, void *pStruct ) noexcept;
 
 } // namespace x86_64
diff --git a/bridges/source/cpp_uno/gcc3_macosx_x86-64/call.cxx b/bridges/source/cpp_uno/gcc3_macosx_x86-64/call.cxx
index 2359d88c23ab..a0654104012d 100644
--- a/bridges/source/cpp_uno/gcc3_macosx_x86-64/call.cxx
+++ b/bridges/source/cpp_uno/gcc3_macosx_x86-64/call.cxx
@@ -55,19 +55,18 @@ void privateSnippetExecutor()
 
          "    call  _cpp_vtable_call\n"
 
-         "    cmp   $10, %rax               # typelib_TypeClass_FLOAT\n"
-         "    je    .Lfloat\n"
-         "    cmp   $11, %rax               # typelib_TypeClass_DOUBLE\n"
-         "    je    .Lfloat\n"
+         "    cmp   $1, %rax\n"
+         "    je    .Lspecial\n"
 
-         "    movq  -144(%rbp), %rax        # Return value (int case)\n"
-         "    movq  -136(%rbp), %rdx        # Return value (int case)\n"
-         "    movq  -144(%rbp), %xmm0       # Return value (int case)\n"
-         "    movq  -136(%rbp), %xmm1       # Return value (int case)\n"
+         "    movq  -144(%rbp), %rax        # Potential return value (general case)\n"
+         "    movq  -136(%rbp), %rdx        # Potential return value (general case)\n"
+         "    movq  -144(%rbp), %xmm0       # Potential return value (general case)\n"
+         "    movq  -136(%rbp), %xmm1       # Potential return value (general case)\n"
          "    jmp   .Lfinish\n"
 
-         ".Lfloat:\n"
-         "    movlpd    -144(%rbp), %xmm0   # Return value (float/double case)\n"
+         ".Lspecial:\n"
+         "    movq  -144(%rbp), %xmm0       # Return value (special fp and integer case)\n"
+         "    movq  -136(%rbp), %rax        # Return value (special fp and integer case)\n"
 
          ".Lfinish:\n"
          "    addq  $160, %rsp\n"
diff --git a/bridges/source/cpp_uno/gcc3_macosx_x86-64/call.hxx b/bridges/source/cpp_uno/gcc3_macosx_x86-64/call.hxx
index 7e7a68675648..efb5b9518989 100644
--- a/bridges/source/cpp_uno/gcc3_macosx_x86-64/call.hxx
+++ b/bridges/source/cpp_uno/gcc3_macosx_x86-64/call.hxx
@@ -22,9 +22,8 @@
 #include <sal/config.h>
 
 #include <sal/types.h>
-#include <typelib/typeclass.h>
 
-extern "C" typelib_TypeClass cpp_vtable_call(
+extern "C" int cpp_vtable_call(
     sal_Int32 nFunctionIndex, sal_Int32 nVtableOffset,
     void ** gpreg, void ** fpreg, void ** ovrflw,
     sal_uInt64 * pRegisterReturn /* space for register return */ );
diff --git a/bridges/source/cpp_uno/gcc3_macosx_x86-64/cpp2uno.cxx b/bridges/source/cpp_uno/gcc3_macosx_x86-64/cpp2uno.cxx
index ddbd76cb1889..4a150ef81a9a 100644
--- a/bridges/source/cpp_uno/gcc3_macosx_x86-64/cpp2uno.cxx
+++ b/bridges/source/cpp_uno/gcc3_macosx_x86-64/cpp2uno.cxx
@@ -54,7 +54,9 @@ using namespace ::com::sun::star::uno;
 // [ret *] is present when we are returning a structure bigger than 16 bytes
 // Simple types are returned in rax, rdx (int), or xmm0, xmm1 (fp).
 // Similarly structures <= 16 bytes are in rax, rdx, xmm0, xmm1 as necessary.
-static typelib_TypeClass cpp2uno_call(
+//
+// The return value is the same as for cpp_vtable_call.
+static int cpp2uno_call(
     bridges::cpp_uno::shared::CppInterfaceProxy * pThis,
     const typelib_TypeDescription * pMemberTypeDescr,
     typelib_TypeDescriptionReference * pReturnTypeRef, // 0 indicates void return
@@ -69,13 +71,16 @@ static typelib_TypeClass cpp2uno_call(
     typelib_TypeDescription * pReturnTypeDescr = nullptr;
     if (pReturnTypeRef)
         TYPELIB_DANGER_GET( &pReturnTypeDescr, pReturnTypeRef );
+    x86_64::ReturnKind returnKind
+        = (pReturnTypeRef == nullptr || pReturnTypeRef->eTypeClass == typelib_TypeClass_VOID)
+        ? x86_64::ReturnKind::RegistersGeneral : x86_64::getReturnKind(pReturnTypeRef);
 
     void * pUnoReturn = nullptr;
     void * pCppReturn = nullptr; // complex return ptr: if != 0 && != pUnoReturn, reconversion need
 
     if ( pReturnTypeDescr )
     {
-        if ( x86_64::return_in_hidden_param( pReturnTypeRef ) )
+        if ( returnKind == x86_64::ReturnKind::Memory )
         {
             pCppReturn = *gpreg++;
             nr_gpr++;
@@ -204,7 +209,7 @@ static typelib_TypeClass cpp2uno_call(
 
         CPPU_CURRENT_NAMESPACE::raiseException( &aUnoExc, pThis->getBridge()->getUno2Cpp() ); // has to destruct the any
         // is here for dummy
-        return typelib_TypeClass_VOID;
+        return 0;
     }
     else // else no exception occurred...
     {
@@ -241,16 +246,19 @@ static typelib_TypeClass cpp2uno_call(
         }
         if ( pReturnTypeDescr )
         {
-            typelib_TypeClass eRet = pReturnTypeDescr->eTypeClass;
             TYPELIB_DANGER_RELEASE( pReturnTypeDescr );
-            return eRet;
         }
-        else
-            return typelib_TypeClass_VOID;
+        return returnKind == x86_64::ReturnKind::RegistersSpecial ? 1 : 0;
     }
 }
 
-typelib_TypeClass cpp_vtable_call(
+// Returns 0 for the general case where potential return values from privateSnippetExecutor can be
+// copied from pRegisterReturn to both %rax and %rdx (in that order) and to %xmm0 and %xmm1 (in that
+// order)---each specific return type will only require a subset of that copy operations, but the
+// other copies to those non--callee-saved registers will be redundant and harmless.  And returns 1
+// for the special case where return values from privateSnippetExecutor must be copied from
+// pRegisterReturn to %xmm0 and %rax (in that order).
+int cpp_vtable_call(
     sal_Int32 nFunctionIndex, sal_Int32 nVtableOffset,
     void ** gpreg, void ** fpreg, void ** ovrflw,
     sal_uInt64 * pRegisterReturn /* space for register return */ )
@@ -295,7 +303,7 @@ typelib_TypeClass cpp_vtable_call(
 
     TypeDescription aMemberDescr( pTypeDescr->ppAllMembers[nMemberPos] );
 
-    typelib_TypeClass eRet;
+    int eRet;
     switch ( aMemberDescr.get()->eTypeClass )
     {
         case typelib_TypeClass_INTERFACE_ATTRIBUTE:
@@ -332,11 +340,11 @@ typelib_TypeClass cpp_vtable_call(
             {
                 case 1: // acquire()
                     pCppI->acquireProxy(); // non virtual call!
-                    eRet = typelib_TypeClass_VOID;
+                    eRet = 0;
                     break;
                 case 2: // release()
                     pCppI->releaseProxy(); // non virtual call!
-                    eRet = typelib_TypeClass_VOID;
+                    eRet = 0;
                     break;
                 case 0: // queryInterface() opt
                 {
@@ -360,7 +368,7 @@ typelib_TypeClass cpp_vtable_call(
                             TYPELIB_DANGER_RELEASE( pTD );
 
                             reinterpret_cast<void **>( pRegisterReturn )[0] = gpreg[0];
-                            eRet = typelib_TypeClass_ANY;
+                            eRet = 0;
                             break;
                         }
                         TYPELIB_DANGER_RELEASE( pTD );