jl164 #i109096# osl_createProcesess failed if the executable had a long path on windows

2 files changed, 46 insertions, 9 deletions

diff --git a/sal/osl/w32/module.cxx b/sal/osl/w32/module.cxx
index f303fd9b5..cac91f7c7 100644..100755
--- a/sal/osl/w32/module.cxx
+++ b/sal/osl/w32/module.cxx
@@ -36,6 +36,7 @@
 #include <osl/thread.h>
 #include <osl/file.h>
 #include <rtl/logfile.h>
+#include <vector>
 
 /*
     under WIN32, we use the void* oslModule 
@@ -72,21 +73,22 @@ oslModule SAL_CALL osl_loadModule(rtl_uString *strModuleName, sal_Int32 nRtldMod
 
     //In case of long path names (\\?\c:\...) try to shorten the filename.
     //LoadLibrary cannot handle file names which exceed 260 letters.
-    if (hInstance == NULL && GetLastError() == ERROR_FILENAME_EXCED_RANGE)
+    //In case the path is to long, the function will fail. However, the error
+    //code can be different. For example, it returned  ERROR_FILENAME_EXCED_RANGE
+    //on Windows XP and ERROR_INSUFFICIENT_BUFFER on Windows 7 (64bit)
+    if (hInstance == NULL && Module->length > 260)
     {
-        wchar_t * buff = new wchar_t[Module->length + 1];
+        std::vector<sal_Unicode, rtl::Allocator<sal_Unicode> > vec(Module->length + 1);
         DWORD len = GetShortPathNameW(reinterpret_cast<LPCWSTR>(Module->buffer), 
-            buff, Module->length + 1);
+                                      vec.begin(), Module->length + 1);
         if (len )
         {
-            hInstance = LoadLibraryW(buff);
+            hInstance = LoadLibraryW(vec.begin());
 
             if (hInstance == NULL)
-                hInstance = LoadLibraryExW(buff, NULL,
+                hInstance = LoadLibraryExW(vec.begin(), NULL,
                                   LOAD_WITH_ALTERED_SEARCH_PATH);
-
         }
-        delete[] buff;
     }
 
 
diff --git a/sal/osl/w32/procimpl.cxx b/sal/osl/w32/procimpl.cxx
index a2e645426..5a4188db9 100644..100755
--- a/sal/osl/w32/procimpl.cxx
+++ b/sal/osl/w32/procimpl.cxx
@@ -299,7 +299,40 @@ namespace /* private */
             
         return quoted.makeStringAndClear();
     }
-    
+
+    //The parameter path must be a system path. If it is longer than 260 characters
+    //then it is shortened using the GetShortPathName function. This function only
+    //works if the path exists. Because "path" can be the path to an executable, it
+    //may not have the file extension ".exe". However, if the file on disk has the
+    //".exe" extension, then the function will fail. In this case a second attempt
+    //is started by adding the parameter "extension" to "path".
+    rtl::OUString getShortPath(rtl::OUString const & path, rtl::OUString const & extension)
+    {
+        rtl::OUString ret(path);
+        if (path.getLength() > 260)
+        {
+            std::vector<sal_Unicode, rtl::Allocator<sal_Unicode> > vec(path.getLength() + 1);
+            //GetShortPathNameW only works if the file can be found!
+            const DWORD len = GetShortPathNameW(
+                path.getStr(), vec.begin(), path.getLength() + 1);
+
+            if (!len && GetLastError() == ERROR_FILE_NOT_FOUND
+                && extension.getLength())
+            {
+                const rtl::OUString extPath(path + extension);
+                std::vector<sal_Unicode, rtl::Allocator<sal_Unicode> > vec2(
+                    extPath.getLength() + 1);
+                const DWORD len2 = GetShortPathNameW(
+                    extPath.getStr(), vec2.begin(), extPath.getLength() + 1);
+                ret = rtl::OUString(vec2.begin(), len2);
+            }
+            else
+            {
+                ret = rtl::OUString(vec.begin(), len);
+            }
+        }
+        return ret;
+    }
     //##########################################################  
     // Returns the system path of the executable which can either
     // be provided via the strImageName parameter or as first 
@@ -325,9 +358,11 @@ namespace /* private */
         rtl::OUString exe_path;
         if (osl_File_E_None != osl::FileBase::getSystemPathFromFileURL(exe_url, exe_path))
             return rtl::OUString();
+
+        exe_path = getShortPath(exe_path, rtl::OUString(RTL_CONSTASCII_USTRINGPARAM(".exe")));
         
         if (exe_path.indexOf(' ') != -1)       
-            exe_path = quote_string(exe_path);            
+            exe_path = quote_string(exe_path);
         
         return exe_path;
     }