cid#1607160 Overflowed return value

and

cid#1606794 Overflowed return value

Change-Id: I222f5fa18ed26ee92d970c744682bf21147265ea
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/176313
Tested-by: Jenkins
Reviewed-by: Caolán McNamara <caolan.mcnamara@collabora.com>

2 files changed, 4 insertions, 4 deletions

diff --git a/sc/source/core/data/formulacell.cxx b/sc/source/core/data/formulacell.cxx
index 80abda16f2ca..832c0db8843c 100644
--- a/sc/source/core/data/formulacell.cxx
+++ b/sc/source/core/data/formulacell.cxx
@@ -4420,10 +4420,10 @@ struct ScDependantsCalculator
             // This can end up negative! Was that the original intent, or
             // is it accidental? Was it not like that originally but the
             // surrounding conditions changed?
-            nRowLen = nLastRow - nRow + 1;
+            const bool bFail = o3tl::checked_sub(nLastRow + 1, nRow, nRowLen);
             // Anyway, let's assume it doesn't make sense to return a
             // negative or zero value here.
-            if (nRowLen <= 0)
+            if (bFail || nRowLen <= 0)
                 nRowLen = 1;
         }
         else if (nLastRow == 0)
diff --git a/sc/source/core/data/grouptokenconverter.cxx b/sc/source/core/data/grouptokenconverter.cxx
index 4d427fc32bab..18b2807515ee 100644
--- a/sc/source/core/data/grouptokenconverter.cxx
+++ b/sc/source/core/data/grouptokenconverter.cxx
@@ -70,13 +70,13 @@ SCROW ScGroupTokenConverter::trimLength(SCTAB nTab, SCCOL nCol1, SCCOL nCol2, SC
         // This can end up negative! Was that the original intent, or
         // is it accidental? Was it not like that originally but the
         // surrounding conditions changed?
-        nRowLen = nLastRow - nRow + 1;
+        const bool bFail = o3tl::checked_sub(nLastRow + 1, nRow, nRowLen);
         // Anyway, let's assume it doesn't make sense to return a
         // negative value here. But should we then return 0 or 1? In
         // the "Column is empty" case below, we return 1, why!? And,
         // at the callsites there are tests for a zero value returned
         // from this function (but not for a negative one).
-        if (nRowLen < 0)
+        if (bFail || nRowLen < 0)
             nRowLen = 0;
     }
     else if (nLastRow == 0)