ofz#5221 depth protect Prepare

Change-Id: I49804a8c65b3af204bfd79d6ff5b3280ab876ec4
Reviewed-on: https://gerrit.libreoffice.org/47693
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Caolán McNamara <caolanm@redhat.com>
Tested-by: Caolán McNamara <caolanm@redhat.com>

7 files changed, 53 insertions, 50 deletions

diff --git a/starmath/inc/node.hxx b/starmath/inc/node.hxx
index acef42a50cfd..01423959652b 100644
--- a/starmath/inc/node.hxx
+++ b/starmath/inc/node.hxx
@@ -151,7 +151,7 @@ public:
     /** Prepare preliminary settings about font and text
      *  (e.g. maFace, meRectHorAlign, mnFlags, mnAttributes, etc.)
      */
-    virtual void Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell);
+    virtual void Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell, int nDepth);
     void PrepareAttributes();
 
     void         SetRectHorAlign(RectHorAlign eHorAlign, bool bApplyToSubTree = true );
@@ -410,7 +410,7 @@ public:
     /** Set the index within GetText() where the selection end */
     void                SetSelectionEnd(sal_Int32 index) {mnSelectionEnd = index;}
 
-    virtual void Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell) override;
+    virtual void Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell, int nDepth) override;
     virtual void Arrange(OutputDevice &rDev, const SmFormat &rFormat) override;
     virtual void CreateTextFromNode(OUString &rText) override;
 
@@ -441,7 +441,7 @@ protected:
 public:
     explicit SmSpecialNode(const SmToken &rNodeToken);
 
-    virtual void Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell) override;
+    virtual void Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell, int nDepth) override;
     virtual void Arrange(OutputDevice &rDev, const SmFormat &rFormat) override;
 
     void Accept(SmVisitor* pVisitor) override;
@@ -490,7 +490,7 @@ public:
     virtual void AdaptToX(OutputDevice &rDev, sal_uLong nWidth) override;
     virtual void AdaptToY(OutputDevice &rDev, sal_uLong nHeight) override;
 
-    virtual void Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell) override;
+    virtual void Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell, int nDepth) override;
     virtual void Arrange(OutputDevice &rDev, const SmFormat &rFormat) override;
     void CreateTextFromNode(OUString &rText) override;
     void Accept(SmVisitor* pVisitor) override;
@@ -551,7 +551,7 @@ public:
     }
     SmPlaceNode() : SmMathSymbolNode(SmNodeType::Place, SmToken(TPLACE, MS_PLACE, "<?>")) {};
 
-    virtual void Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell) override;
+    virtual void Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell, int nDepth) override;
     virtual void Arrange(OutputDevice &rDev, const SmFormat &rFormat) override;
     void Accept(SmVisitor* pVisitor) override;
 };
@@ -571,7 +571,7 @@ public:
         SetText(OUString(MS_ERROR));
     }
 
-    virtual void Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell) override;
+    virtual void Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell, int nDepth) override;
     virtual void Arrange(OutputDevice &rDev, const SmFormat &rFormat) override;
     void Accept(SmVisitor* pVisitor) override;
 };
@@ -629,7 +629,7 @@ public:
     void  SetUseExtraSpaces(bool bVal) { mbUseExtraSpaces = bVal; }
     bool  IsUseExtraSpaces() const { return mbUseExtraSpaces; };
 
-    virtual void Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell) override;
+    virtual void Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell, int nDepth) override;
     virtual void Arrange(OutputDevice &rDev, const SmFormat &rFormat) override;
     void Accept(SmVisitor* pVisitor) override;
 };
@@ -1064,7 +1064,7 @@ public:
     const Fraction & GetSizeParameter() const {return maFontSize;}
     FontSizeType GetSizeType() const {return meSizeType;}
 
-    virtual void Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell) override;
+    virtual void Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell, int nDepth) override;
     virtual void Arrange(OutputDevice &rDev, const SmFormat &rFormat) override;
     void CreateTextFromNode(OUString &rText) override;
     void Accept(SmVisitor* pVisitor) override;
@@ -1121,7 +1121,7 @@ public:
     sal_uInt16   GetBlankNum() const { return mnNum; }
     void         SetBlankNum(sal_uInt16 nNumber) { mnNum = nNumber; }
 
-    virtual void Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell) override;
+    virtual void Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell, int nDepth) override;
     virtual void Arrange(OutputDevice &rDev, const SmFormat &rFormat) override;
     void Accept(SmVisitor* pVisitor) override;
     virtual void CreateTextFromNode(OUString &rText) override;
diff --git a/starmath/qa/cppunit/test_cursor.cxx b/starmath/qa/cppunit/test_cursor.cxx
index 02c7ff433371..702777a614f8 100644
--- a/starmath/qa/cppunit/test_cursor.cxx
+++ b/starmath/qa/cppunit/test_cursor.cxx
@@ -69,7 +69,7 @@ void Test::testCopyPaste()
 {
     OUString const sInput("a * b + c");
     std::unique_ptr<SmNode> xTree(SmParser().Parse(sInput));
-    xTree->Prepare(xDocShRef->GetFormat(), *xDocShRef);
+    xTree->Prepare(xDocShRef->GetFormat(), *xDocShRef, 0);
 
     SmCursor aCursor(xTree.get(), xDocShRef.get());
     ScopedVclPtrInstance<VirtualDevice> pOutputDevice;
@@ -92,7 +92,7 @@ void Test::testCopySelectPaste()
 {
     OUString const sInput("a * b + c");
     std::unique_ptr<SmNode> xTree(SmParser().Parse(sInput));
-    xTree->Prepare(xDocShRef->GetFormat(), *xDocShRef);
+    xTree->Prepare(xDocShRef->GetFormat(), *xDocShRef, 0);
 
     SmCursor aCursor(xTree.get(), xDocShRef.get());
     ScopedVclPtrInstance<VirtualDevice> pOutputDevice;
@@ -119,7 +119,7 @@ void Test::testCutPaste()
 {
     OUString const sInput("a * b + c");
     std::unique_ptr<SmNode> xTree(SmParser().Parse(sInput));
-    xTree->Prepare(xDocShRef->GetFormat(), *xDocShRef);
+    xTree->Prepare(xDocShRef->GetFormat(), *xDocShRef, 0);
 
     SmCursor aCursor(xTree.get(), xDocShRef.get());
     ScopedVclPtrInstance<VirtualDevice> pOutputDevice;
@@ -142,7 +142,7 @@ void Test::testCutSelectPaste()
 {
     OUString const sInput("a * b + c");
     std::unique_ptr<SmNode> xTree(SmParser().Parse(sInput));
-    xTree->Prepare(xDocShRef->GetFormat(), *xDocShRef);
+    xTree->Prepare(xDocShRef->GetFormat(), *xDocShRef, 0);
 
     SmCursor aCursor(xTree.get(), xDocShRef.get());
     ScopedVclPtrInstance<VirtualDevice> pOutputDevice;
diff --git a/starmath/qa/cppunit/test_nodetotextvisitors.cxx b/starmath/qa/cppunit/test_nodetotextvisitors.cxx
index dee561d371aa..6791e9d6270c 100644
--- a/starmath/qa/cppunit/test_nodetotextvisitors.cxx
+++ b/starmath/qa/cppunit/test_nodetotextvisitors.cxx
@@ -436,12 +436,12 @@ void Test::parseandparseagain(const char *formula, const char *test_name)
     // parse 1
     OUString input = OUString::createFromAscii(formula);
     pNode1 = SmParser().ParseExpression(input);
-    pNode1->Prepare(xDocShRef->GetFormat(), *xDocShRef);
+    pNode1->Prepare(xDocShRef->GetFormat(), *xDocShRef, 0);
     SmNodeToTextVisitor(pNode1, output1);
 
     // parse 2
     pNode2 = SmParser().ParseExpression(output1);
-    pNode2->Prepare(xDocShRef->GetFormat(), *xDocShRef);
+    pNode2->Prepare(xDocShRef->GetFormat(), *xDocShRef, 0);
     SmNodeToTextVisitor(pNode2, output2);
 
     // compare
@@ -466,7 +466,7 @@ void Test::ParseAndCheck(const char *formula, const char * expected, const char
     // parse
     OUString sInput = OUString::createFromAscii(formula);
     pNode = SmParser().ParseExpression(sInput);
-    pNode->Prepare(xDocShRef->GetFormat(), *xDocShRef);
+    pNode->Prepare(xDocShRef->GetFormat(), *xDocShRef, 0);
     SmNodeToTextVisitor(pNode, sOutput);
 
     // compare
@@ -491,13 +491,13 @@ void Test::ParseAndCompare(const char *formula1, const char *formula2, const cha
     // parse formula1
     OUString sInput1 = OUString(formula1, strlen(formula1), RTL_TEXTENCODING_UTF8);
     pNode1 = SmParser().ParseExpression(sInput1);
-    pNode1->Prepare(xDocShRef->GetFormat(), *xDocShRef);
+    pNode1->Prepare(xDocShRef->GetFormat(), *xDocShRef, 0);
     SmNodeToTextVisitor(pNode1, sOutput1);
 
     // parse formula2
     OUString sInput2 = OUString(formula2, strlen(formula2), RTL_TEXTENCODING_UTF8);
     pNode2 = SmParser().ParseExpression(sInput2);
-    pNode2->Prepare(xDocShRef->GetFormat(), *xDocShRef);
+    pNode2->Prepare(xDocShRef->GetFormat(), *xDocShRef, 0);
     SmNodeToTextVisitor(pNode2, sOutput2);
 
     CPPUNIT_ASSERT_EQUAL_MESSAGE(test_name, sOutput1, sOutput2);
@@ -519,7 +519,7 @@ void Test::testBinomInBinHor()
     // set up a binom (table) node
     sInput += "binom a b + c";
     pTree = SmParser().Parse(sInput);
-    pTree->Prepare(xDocShRef->GetFormat(), *xDocShRef);
+    pTree->Prepare(xDocShRef->GetFormat(), *xDocShRef, 0);
 
     SmCursor aCursor(pTree, xDocShRef.get());
     ScopedVclPtrInstance< VirtualDevice > pOutputDevice;
@@ -547,7 +547,7 @@ void Test::testBinVerInUnary()
     // set up a unary operator with operand
     sInput += "- 1";
     pTree = SmParser().Parse(sInput);
-    pTree->Prepare(xDocShRef->GetFormat(), *xDocShRef);
+    pTree->Prepare(xDocShRef->GetFormat(), *xDocShRef, 0);
 
     SmCursor aCursor(pTree, xDocShRef.get());
     ScopedVclPtrInstance< VirtualDevice > pOutputDevice;
@@ -574,7 +574,7 @@ void Test::testBinHorInSubSup()
 {
     // set up a blank formula
     SmNode* pTree = SmParser().Parse(OUString());
-    pTree->Prepare(xDocShRef->GetFormat(), *xDocShRef);
+    pTree->Prepare(xDocShRef->GetFormat(), *xDocShRef, 0);
 
     SmCursor aCursor(pTree, xDocShRef.get());
     ScopedVclPtrInstance< VirtualDevice > pOutputDevice;
@@ -602,7 +602,7 @@ void Test::testUnaryInMixedNumberAsNumerator()
     // set up a unary operator
     OUString sInput = "- 1";
     SmNode* pTree = SmParser().Parse(sInput);
-    pTree->Prepare(xDocShRef->GetFormat(), *xDocShRef);
+    pTree->Prepare(xDocShRef->GetFormat(), *xDocShRef, 0);
 
     SmCursor aCursor(pTree, xDocShRef.get());
     ScopedVclPtrInstance< VirtualDevice > pOutputDevice;
@@ -671,7 +671,7 @@ void Test::testParser()
     OUString sInput(u"{ \U0001D44E }"); // non-BMP Unicode
     OUString sExpected(u"\U0001D44E");
     std::unique_ptr<SmNode> pNode(SmParser().ParseExpression(sInput));
-    pNode->Prepare(xDocShRef->GetFormat(), *xDocShRef);
+    pNode->Prepare(xDocShRef->GetFormat(), *xDocShRef, 0);
     SmNodeToTextVisitor(pNode.get(), sOutput);
     CPPUNIT_ASSERT_EQUAL(sExpected, sOutput);
 }
diff --git a/starmath/source/ElementsDockingWindow.cxx b/starmath/source/ElementsDockingWindow.cxx
index 9ddf2d21f5c6..9ab977f489c0 100644
--- a/starmath/source/ElementsDockingWindow.cxx
+++ b/starmath/source/ElementsDockingWindow.cxx
@@ -525,7 +525,7 @@ void SmElementsControl::addElement(const OUString& aElementVisual, const OUStrin
 {
     std::unique_ptr<SmNode> pNode(SmParser().ParseExpression(aElementVisual));
 
-    pNode->Prepare(maFormat, *mpDocShell);
+    pNode->Prepare(maFormat, *mpDocShell, 0);
     pNode->SetSize(Fraction(10,8));
     pNode->Arrange(*this, maFormat);
 
diff --git a/starmath/source/cursor.cxx b/starmath/source/cursor.cxx
index 84c364bc17d6..033231a28b3a 100644
--- a/starmath/source/cursor.cxx
+++ b/starmath/source/cursor.cxx
@@ -528,7 +528,7 @@ void SmCursor::InsertSubSup(SmSubSup eSubSup) {
     } else {
         //Create a new place node
         pSubject = new SmPlaceNode();
-        pSubject->Prepare(mpDocShell->GetFormat(), *mpDocShell);
+        pSubject->Prepare(mpDocShell->GetFormat(), *mpDocShell, 0);
         it = pLineList->insert(it, pSubject);
         ++it;
         bPatchLine = true;  //We've modified the line it should be patched later.
@@ -637,7 +637,7 @@ void SmCursor::InsertBrackets(SmBracketType eBracketType) {
     SmBracebodyNode *pBody = new SmBracebodyNode(SmToken());
     pBody->SetSubNodes(pBodyNode, nullptr);
     pBrace->SetSubNodes(pLeft, pBody, pRight);
-    pBrace->Prepare(mpDocShell->GetFormat(), *mpDocShell);
+    pBrace->Prepare(mpDocShell->GetFormat(), *mpDocShell, 0);
 
     //Insert into line
     pLineList->insert(it, pBrace);
@@ -872,7 +872,7 @@ void SmCursor::InsertText(const OUString& aString)
     SmTextNode* pText = new SmTextNode(token, FNT_VARIABLE);
     pText->SetText(aString);
     pText->AdjustFontDesc();
-    pText->Prepare(mpDocShell->GetFormat(), *mpDocShell);
+    pText->Prepare(mpDocShell->GetFormat(), *mpDocShell, 0);
 
     SmNodeList* pList = new SmNodeList;
     pList->push_front(pText);
@@ -973,7 +973,7 @@ void SmCursor::InsertElement(SmFormulaElement element){
     assert(pNewNode);
 
     //Prepare the new node
-    pNewNode->Prepare(mpDocShell->GetFormat(), *mpDocShell);
+    pNewNode->Prepare(mpDocShell->GetFormat(), *mpDocShell, 0);
 
     //Insert new node
     SmNodeList* pList = new SmNodeList;
@@ -1000,7 +1000,7 @@ void SmCursor::InsertSpecial(const OUString& _aString)
     SmSpecialNode* pSpecial = new SmSpecialNode(token);
 
     //Prepare the special node
-    pSpecial->Prepare(mpDocShell->GetFormat(), *mpDocShell);
+    pSpecial->Prepare(mpDocShell->GetFormat(), *mpDocShell, 0);
 
     //Insert the node
     SmNodeList* pList = new SmNodeList;
@@ -1015,7 +1015,7 @@ void SmCursor::InsertCommandText(const OUString& aCommandText) {
     SmNode* pSubExpr = SmParser().ParseExpression(aCommandText);
 
     //Prepare the subtree
-    pSubExpr->Prepare(mpDocShell->GetFormat(), *mpDocShell);
+    pSubExpr->Prepare(mpDocShell->GetFormat(), *mpDocShell, 0);
 
     //Convert subtree to list
     SmNodeList* pLineList = NodeToList(pSubExpr);
@@ -1239,7 +1239,7 @@ void SmCursor::FinishEdit(SmNodeList* pLineList,
         SmBracebodyNode *pBody = new SmBracebodyNode(SmToken());
         pBody->SetSubNodes(pLine, nullptr);
         pBrace->SetSubNodes(pLeft, pBody, pRight);
-        pBrace->Prepare(mpDocShell->GetFormat(), *mpDocShell);
+        pBrace->Prepare(mpDocShell->GetFormat(), *mpDocShell, 0);
         pLine = pBrace;
         //TODO: Consider the following alternative behavior:
         //Consider the line: A + {B + C}^D lsub E
diff --git a/starmath/source/document.cxx b/starmath/source/document.cxx
index 9466d6635674..399ff0dece69 100644
--- a/starmath/source/document.cxx
+++ b/starmath/source/document.cxx
@@ -251,7 +251,7 @@ void SmDocShell::ArrangeFormula()
                "Sm : falscher MapMode");
 
     const SmFormat &rFormat = GetFormat();
-    mpTree->Prepare(rFormat, *this);
+    mpTree->Prepare(rFormat, *this, 0);
 
     // format/draw formulas always from left to right,
     // and numbers should not be converted
diff --git a/starmath/source/node.cxx b/starmath/source/node.cxx
index e48de29b8950..d316e9228023 100644
--- a/starmath/source/node.cxx
+++ b/starmath/source/node.cxx
@@ -217,8 +217,11 @@ void SmNode::PrepareAttributes()
 }
 
 
-void SmNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell)
+void SmNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell, int nDepth)
 {
+    if (nDepth > 2048)
+        throw std::range_error("parser depth limit");
+
     mbIsPhantom  = false;
     mnFlags      = FontChangeMask::None;
     mnAttributes = FontAttribute::None;
@@ -235,7 +238,7 @@ void SmNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell)
     GetFont().SetWeight(WEIGHT_NORMAL);
     GetFont().SetItalic(ITALIC_NONE);
 
-    ForEachNonNull(this, [&rFormat, &rDocShell](SmNode *pNode){pNode->Prepare(rFormat, rDocShell);});
+    ForEachNonNull(this, [&rFormat, &rDocShell, nDepth](SmNode *pNode){pNode->Prepare(rFormat, rDocShell, nDepth + 1);});
 }
 
 void SmNode::Move(const Point& rPosition)
@@ -545,9 +548,9 @@ long SmTableNode::GetFormulaBaseline() const
 /**************************************************************************/
 
 
-void SmLineNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell)
+void SmLineNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell, int nDepth)
 {
-    SmNode::Prepare(rFormat, rDocShell);
+    SmNode::Prepare(rFormat, rDocShell, nDepth);
 
     // Here we use the 'FNT_VARIABLE' font since it's ascent and descent in general fit better
     // to the rest of the formula compared to the 'FNT_MATH' font.
@@ -1823,10 +1826,10 @@ void SmFontNode::CreateTextFromNode(OUString &rText)
     rText += "} ";
 }
 
-void SmFontNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell)
+void SmFontNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell, int nDepth)
 {
     //! prepare subnodes first
-    SmNode::Prepare(rFormat, rDocShell);
+    SmNode::Prepare(rFormat, rDocShell, nDepth);
 
     int  nFnt = -1;
     switch (GetToken().eType)
@@ -2043,9 +2046,9 @@ SmTextNode::SmTextNode( const SmToken &rNodeToken, sal_uInt16 nFontDescP )
 {
 }
 
-void SmTextNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell)
+void SmTextNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell, int nDepth)
 {
-    SmNode::Prepare(rFormat, rDocShell);
+    SmNode::Prepare(rFormat, rDocShell, nDepth);
 
     // default setting for horizontal alignment of nodes with TTEXT
     // content is as alignl (cannot be done in Arrange since it would
@@ -2404,9 +2407,9 @@ void SmMathSymbolNode::AdaptToY(OutputDevice &rDev, sal_uLong nHeight)
 }
 
 
-void SmMathSymbolNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell)
+void SmMathSymbolNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell, int nDepth)
 {
-    SmNode::Prepare(rFormat, rDocShell);
+    SmNode::Prepare(rFormat, rDocShell, nDepth);
 
     GetFont() = rFormat.GetFont(GetFontDesc());
     // use same font size as is used for variables
@@ -2589,9 +2592,9 @@ SmSpecialNode::SmSpecialNode(const SmToken &rNodeToken)
 }
 
 
-void SmSpecialNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell)
+void SmSpecialNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell, int nDepth)
 {
-    SmNode::Prepare(rFormat, rDocShell);
+    SmNode::Prepare(rFormat, rDocShell, nDepth);
 
     const SmSym   *pSym;
     SmModule  *pp = SM_MOD();
@@ -2681,9 +2684,9 @@ void SmGlyphSpecialNode::Arrange(OutputDevice &rDev, const SmFormat &rFormat)
 /**************************************************************************/
 
 
-void SmPlaceNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell)
+void SmPlaceNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell, int nDepth)
 {
-    SmNode::Prepare(rFormat, rDocShell);
+    SmNode::Prepare(rFormat, rDocShell, nDepth);
 
     GetFont().SetColor(COL_GRAY);
     Flags() |= FontChangeMask::Color | FontChangeMask::Face | FontChangeMask::Italic;
@@ -2704,9 +2707,9 @@ void SmPlaceNode::Arrange(OutputDevice &rDev, const SmFormat &rFormat)
 /**************************************************************************/
 
 
-void SmErrorNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell)
+void SmErrorNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell, int nDepth)
 {
-    SmNode::Prepare(rFormat, rDocShell);
+    SmNode::Prepare(rFormat, rDocShell, nDepth);
 
     GetFont().SetColor(COL_RED);
     Flags() |= FontChangeMask::Phantom | FontChangeMask::Bold | FontChangeMask::Italic
@@ -2738,9 +2741,9 @@ void SmBlankNode::IncreaseBy(const SmToken &rToken, sal_uInt32 nMultiplyBy)
     }
 }
 
-void SmBlankNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell)
+void SmBlankNode::Prepare(const SmFormat &rFormat, const SmDocShell &rDocShell, int nDepth)
 {
-    SmNode::Prepare(rFormat, rDocShell);
+    SmNode::Prepare(rFormat, rDocShell, nDepth);
 
     // Here it need/should not be the StarMath font, so that for the character
     // used in Arrange a normal (non-clipped) rectangle is generated