Bug 101032 - assignments to nsSMState in nsCodingStateMachine result...

... in unspecified behavior.
When compiling with UBSan (-fsanitize=undefined), execution complains:
> runtime error: load of value 5, which is not a valid value for type 'nsSMState'
Since the machine states depend on every different charset's state
machine, it is not possible to simply extend the enum with more generic
values. Instead let's just make the state as an unsigned int value and
define the 3 generic states as constants.

9 files changed, 18 insertions, 17 deletions

diff --git a/src/nsBig5Prober.cpp b/src/nsBig5Prober.cpp
index 7a85abb..3e47d6e 100644
--- a/src/nsBig5Prober.cpp
+++ b/src/nsBig5Prober.cpp
@@ -46,7 +46,7 @@ void  nsBig5Prober::Reset(void)
 
 nsProbingState nsBig5Prober::HandleData(const char* aBuf, PRUint32 aLen)
 {
-  nsSMState codingState;
+  PRUint32 codingState;
 
   for (PRUint32 i = 0; i < aLen; i++)
   {
diff --git a/src/nsCodingStateMachine.h b/src/nsCodingStateMachine.h
index 819f9ab..8861118 100644
--- a/src/nsCodingStateMachine.h
+++ b/src/nsCodingStateMachine.h
@@ -39,11 +39,12 @@
 
 #include "nsPkgInt.h"
 
-typedef enum {
-   eStart = 0,
-   eError = 1,
-   eItsMe = 2 
-} nsSMState;
+/* Apart from these 3 generic states, machine states are specific to
+ * each charset prober.
+ */
+#define eStart 0
+#define eError 1
+#define eItsMe 2
 
 #define GETCLASS(c) GETFROMPCK(((unsigned char)(c)), mModel->classTable)
 
@@ -60,7 +61,7 @@ typedef struct
 class nsCodingStateMachine {
 public:
   nsCodingStateMachine(const SMModel* sm) : mModel(sm) { mCurrentState = eStart; }
-  nsSMState NextState(char c){
+  PRUint32 NextState(char c){
     //for each byte we get its class , if it is first byte, we also get byte length
     PRUint32 byteCls = GETCLASS(c);
     if (mCurrentState == eStart)
@@ -69,8 +70,8 @@ public:
       mCurrentCharLen = mModel->charLenTable[byteCls];
     }
     //from byte's class and stateTable, we get its next state
-    mCurrentState=(nsSMState)GETFROMPCK(mCurrentState*(mModel->classFactor)+byteCls,
-                                       mModel->stateTable);
+    mCurrentState = GETFROMPCK(mCurrentState * mModel->classFactor + byteCls,
+                               mModel->stateTable);
     mCurrentBytePos++;
     return mCurrentState;
   }
@@ -79,7 +80,7 @@ public:
   const char * GetCodingStateMachine() {return mModel->name;}
 
 protected:
-  nsSMState mCurrentState;
+  PRUint32 mCurrentState;
   PRUint32 mCurrentCharLen;
   PRUint32 mCurrentBytePos;
 
diff --git a/src/nsEUCJPProber.cpp b/src/nsEUCJPProber.cpp
index 54861b3..f84d154 100644
--- a/src/nsEUCJPProber.cpp
+++ b/src/nsEUCJPProber.cpp
@@ -52,7 +52,7 @@ void  nsEUCJPProber::Reset(void)
 
 nsProbingState nsEUCJPProber::HandleData(const char* aBuf, PRUint32 aLen)
 {
-  nsSMState codingState;
+  PRUint32 codingState;
 
   for (PRUint32 i = 0; i < aLen; i++)
   {
diff --git a/src/nsEUCKRProber.cpp b/src/nsEUCKRProber.cpp
index 3632f1f..6aae8ae 100644
--- a/src/nsEUCKRProber.cpp
+++ b/src/nsEUCKRProber.cpp
@@ -47,7 +47,7 @@ void  nsEUCKRProber::Reset(void)
 
 nsProbingState nsEUCKRProber::HandleData(const char* aBuf, PRUint32 aLen)
 {
-  nsSMState codingState;
+  PRUint32 codingState;
 
   for (PRUint32 i = 0; i < aLen; i++)
   {
diff --git a/src/nsEUCTWProber.cpp b/src/nsEUCTWProber.cpp
index a06e074..7e61ea1 100644
--- a/src/nsEUCTWProber.cpp
+++ b/src/nsEUCTWProber.cpp
@@ -47,7 +47,7 @@ void  nsEUCTWProber::Reset(void)
 
 nsProbingState nsEUCTWProber::HandleData(const char* aBuf, PRUint32 aLen)
 {
-  nsSMState codingState;
+  PRUint32 codingState;
 
   for (PRUint32 i = 0; i < aLen; i++)
   {
diff --git a/src/nsEscCharsetProber.cpp b/src/nsEscCharsetProber.cpp
index 464c753..d093ee4 100644
--- a/src/nsEscCharsetProber.cpp
+++ b/src/nsEscCharsetProber.cpp
@@ -75,7 +75,7 @@ void nsEscCharSetProber::Reset(void)
 
 nsProbingState nsEscCharSetProber::HandleData(const char* aBuf, PRUint32 aLen)
 {
-  nsSMState codingState;
+  PRUint32 codingState;
   PRInt32 j;
   PRUint32 i;
 
diff --git a/src/nsGB2312Prober.cpp b/src/nsGB2312Prober.cpp
index b6d469c..eac0762 100644
--- a/src/nsGB2312Prober.cpp
+++ b/src/nsGB2312Prober.cpp
@@ -52,7 +52,7 @@ void  nsGB18030Prober::Reset(void)
 
 nsProbingState nsGB18030Prober::HandleData(const char* aBuf, PRUint32 aLen)
 {
-  nsSMState codingState;
+  PRUint32 codingState;
 
   for (PRUint32 i = 0; i < aLen; i++)
   {
diff --git a/src/nsSJISProber.cpp b/src/nsSJISProber.cpp
index c7842f6..1c354a7 100644
--- a/src/nsSJISProber.cpp
+++ b/src/nsSJISProber.cpp
@@ -52,7 +52,7 @@ void  nsSJISProber::Reset(void)
 
 nsProbingState nsSJISProber::HandleData(const char* aBuf, PRUint32 aLen)
 {
-  nsSMState codingState;
+  PRUint32 codingState;
 
   for (PRUint32 i = 0; i < aLen; i++)
   {
diff --git a/src/nsUTF8Prober.cpp b/src/nsUTF8Prober.cpp
index ab8d9f7..93677de 100644
--- a/src/nsUTF8Prober.cpp
+++ b/src/nsUTF8Prober.cpp
@@ -46,7 +46,7 @@ void  nsUTF8Prober::Reset(void)
 
 nsProbingState nsUTF8Prober::HandleData(const char* aBuf, PRUint32 aLen)
 {
-  nsSMState codingState;
+  PRUint32 codingState;
 
   for (PRUint32 i = 0; i < aLen; i++)
   {