Avoid overflowing floating-point -> integer conversion

...seen at <https://ci.libreoffice.org/job/lo_ubsan/2113/> with recently-
introduced sw/qa/core/data/ooxml/fail/ofz37458-1.docx during
CppunitTest_sw_filters_test,

> /oox/source/vml/vmlformatting.cxx:202:38: runtime error: 6.40969e+23 is outside the range of representable values of type 'long'
>     #0 0x2b40e7a68a1e in oox::vml::ConversionHelper::decodeMeasureToEmu(oox::GraphicHelper const&, rtl::OUString const&, int, bool, bool) /oox/source/vml/vmlformatting.cxx:202:38
>     #1 0x2b40e7a68cf1 in oox::vml::ConversionHelper::decodeMeasureToHmm(oox::GraphicHelper const&, rtl::OUString const&, int, bool, bool) /oox/source/vml/vmlformatting.cxx:208:47
>     #2 0x2b40e7bc3bb7 in oox::vml::TextBoxContext::TextBoxContext(oox::core::ContextHandler2Helper const&, oox::vml::TextBox&, oox::AttributeList const&, oox::GraphicHelper const&) /oox/source/vml/vmltextboxcontext.cxx:190:39
>     #3 0x2b40e7b3190f in oox::vml::ShapeContext::onCreateContext(int, oox::AttributeList const&) /oox/source/vml/vmlshapecontext.cxx:526:24
>     #4 0x2b40e7b34ee9 in oox::vml::RectangleShapeContext::onCreateContext(int, oox::AttributeList const&) /oox/source/vml/vmlshapecontext.cxx:639:26
>     #5 0x2b40e7b34f42 in non-virtual thunk to oox::vml::RectangleShapeContext::onCreateContext(int, oox::AttributeList const&) /oox/source/vml/vmlshapecontext.cxx
>     #6 0x2b40e5fdae26 in oox::core::ContextHandler2Helper::implCreateChildContext(int, com::sun::star::uno::Reference<com::sun::star::xml::sax::XFastAttributeList> const&) /oox/source/core/contexthandler2.cxx:100:34
>     #7 0x2b40e5fddffb in oox::core::ContextHandler2::createFastChildContext(int, com::sun::star::uno::Reference<com::sun::star::xml::sax::XFastAttributeList> const&) /oox/source/core/contexthandler2.cxx:204:12
>     #8 0x2b40e5fdfb32 in non-virtual thunk to oox::core::ContextHandler2::createFastChildContext(int, com::sun::star::uno::Reference<com::sun::star::xml::sax::XFastAttributeList> const&) /oox/source/core/contexthandler2.cxx
>     #9 0x2b40fe645540 in writerfilter::ooxml::OOXMLFastContextHandlerWrapper::lcl_createFastChildContext(int, com::sun::star::uno::Reference<com::sun::star::xml::sax::XFastAttributeList> const&) /writerfilter/source/ooxml/OOXMLFastContextHandler.cxx:2006:38
>     #10 0x2b40fe60a6a4 in writerfilter::ooxml::OOXMLFastContextHandler::createFastChildContext(int, com::sun::star::uno::Reference<com::sun::star::xml::sax::XFastAttributeList> const&) /writerfilter/source/ooxml/OOXMLFastContextHandler.cxx:254:21
>     #11 0x2b40fe60a942 in non-virtual thunk to writerfilter::ooxml::OOXMLFastContextHandler::createFastChildContext(int, com::sun::star::uno::Reference<com::sun::star::xml::sax::XFastAttributeList> const&) /writerfilter/source/ooxml/OOXMLFastContextHandler.cxx
>     #12 0x2b40ca28e9de in (anonymous namespace)::Entity::startElement((anonymous namespace)::Event const*) /sax/source/fastparser/fastparser.cxx:465:44
>     #13 0x2b40ca2899ed in sax_fastparser::FastSaxParserImpl::callbackStartElement(unsigned char const*, unsigned char const*, unsigned char const*, int, unsigned char const**, int, unsigned char const**) /sax/source/fastparser/fastparser.cxx:1306:21

Apparently lacking a general concept to report failure for too-large input
values here, just silently do a saturating conversation, as seems to be the
general approach for this kind of filter code.

Change-Id: I4511a5dd63bdbc973d60a521bd3aef445dea3fb1
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/120941
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>

1 files changed, 10 insertions, 1 deletions

diff --git a/oox/source/vml/vmlformatting.cxx b/oox/source/vml/vmlformatting.cxx
index 0fca9b06a017..b7dc37b5e940 100644
--- a/oox/source/vml/vmlformatting.cxx
+++ b/oox/source/vml/vmlformatting.cxx
@@ -20,6 +20,7 @@
 #include <sal/config.h>
 
 #include <cstdlib>
+#include <limits>
 
 #include <oox/vml/vmlformatting.hxx>
 
@@ -29,6 +30,7 @@
 #include <com/sun/star/drawing/EnhancedCustomShapeTextPathMode.hpp>
 #include <com/sun/star/table/ShadowFormat.hpp>
 #include <com/sun/star/text/XTextRange.hpp>
+#include <o3tl/float_int_conversion.hxx>
 #include <o3tl/unit_conversion.hxx>
 #include <rtl/strbuf.hxx>
 #include <sal/log.hxx>
@@ -199,7 +201,14 @@ sal_Int64 ConversionHelper::decodeMeasureToEmu( const GraphicHelper& rGraphicHel
         OSL_FAIL( "ConversionHelper::decodeMeasureToEmu - unknown measure unit" );
         fValue = nRefValue;
     }
-    return static_cast< sal_Int64 >( fValue + 0.5 );
+    fValue += 0.5;
+    if (!o3tl::convertsToAtMost(fValue, std::numeric_limits<sal_Int64>::max())) {
+        return std::numeric_limits<sal_Int64>::max();
+    }
+    if (!o3tl::convertsToAtLeast(fValue, std::numeric_limits<sal_Int64>::min())) {
+        return std::numeric_limits<sal_Int64>::min();
+    }
+    return static_cast< sal_Int64 >( fValue );
 }
 
 sal_Int32 ConversionHelper::decodeMeasureToHmm( const GraphicHelper& rGraphicHelper,