Convert code to use Xmumallocarray() & reallocarray()

Provides automatic integer overflow checking in allocation size calculations

Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>

3 files changed, 22 insertions, 13 deletions

diff --git a/src/Distinct.c b/src/Distinct.c
index 8c8bb5f..2260769 100644
--- a/src/Distinct.c
+++ b/src/Distinct.c
@@ -35,6 +35,7 @@ in this Software without prior written authorization from The Open Group.
 #include <stdlib.h>
 #include <X11/Xutil.h>
 #include <X11/Xmu/StdCmap.h>
+#include "Xmuint.h"
 
 /*
  * Distinguishable colors routine.  Determines if two colors are
@@ -77,7 +78,7 @@ XmuDistinguishablePixels(Display *dpy, Colormap cmap,
 	for (j = i + 1; j < count; j++)
 	    if (pixels[i] == pixels[j])
 		return False;
-    defs = malloc (count * sizeof (XColor));
+    defs = Xmumallocarray (count, sizeof (XColor));
     if (!defs)
 	return False;
     for (i = 0; i < count; i++)
diff --git a/src/LookupCmap.c b/src/LookupCmap.c
index 31b339a..a78d224 100644
--- a/src/LookupCmap.c
+++ b/src/LookupCmap.c
@@ -37,6 +37,7 @@ in this Software without prior written authorization from The Open Group.
 #include <X11/Xutil.h>
 #include <X11/Xmu/StdCmap.h>
 #include <stdlib.h>
+#include "Xmuint.h"
 
 /*
  * Prototypes
@@ -242,7 +243,7 @@ lookup(Display *dpy, int screen, VisualID visualid, Atom property,
 	if (cnew) {
 	    XStandardColormap	*m, *maps;
 
-	    s = malloc((unsigned) ((count+1) * sizeof(XStandardColormap)));
+	    s = Xmumallocarray((count+1), sizeof(XStandardColormap));
 
 	    for (i = 0, m = s, maps = stdcmaps; i < count; i++, m++, maps++) {
 		m->colormap   = maps->colormap;
diff --git a/src/Xct.c b/src/Xct.c
index 4ca64c5..bfa606e 100644
--- a/src/Xct.c
+++ b/src/Xct.c
@@ -30,6 +30,7 @@ in this Software without prior written authorization from The Open Group.
 #include <X11/Xfuncs.h>
 #include "Xct.h"
 #include <stdio.h>
+#include "Xmuint.h"
 
 #define UsedGraphic	0x0001
 #define UsedDirection	0x0002
@@ -298,6 +299,7 @@ HandleExtended(register XctData data, int c)
 	;
     if (i == priv->enc_count) {
 	XctString cp;
+	char **new_encodings;
 
 	for (cp = enc; cp != ptr; cp++) {
 	    if ((!IsGL(*cp) && !IsGR(*cp)) || (*cp == 0x2a) || (*cp == 0x3f))
@@ -307,11 +309,14 @@ HandleExtended(register XctData data, int c)
 	(void) memmove((char *)ptr, (char *)enc, len);
 	ptr[len] = 0x00;
 	priv->enc_count++;
-	if (priv->encodings)
-	    priv->encodings = realloc(priv->encodings,
-                                      priv->enc_count * sizeof(char *));
-	else
-	    priv->encodings = malloc(sizeof(char *));
+	new_encodings = reallocarray(priv->encodings,
+				     priv->enc_count, sizeof(char *));
+	if (new_encodings == NULL) {
+	    priv->enc_count--;
+	    free(ptr);
+	    return 0;
+	}
+	priv->encodings = new_encodings;
 	priv->encodings[i] = (char *)ptr;
     }
     data->encoding = priv->encodings[i];
@@ -498,14 +503,16 @@ XctNextItem(register XctData data)
 		    ((data->item[1] == 0x31) || (data->item[1] == 0x32))) {
 		    data->horz_depth++;
 		    if (priv->dirsize < data->horz_depth) {
+			XctHDirection *new_dirstack;
 			priv->dirsize += 10;
-			if (priv->dirstack)
-			    priv->dirstack = realloc(priv->dirstack,
-						     priv->dirsize *
-						     sizeof(XctHDirection));
-			else
-			    priv->dirstack = malloc(priv->dirsize *
+			new_dirstack = reallocarray(priv->dirstack,
+						    priv->dirsize,
 						    sizeof(XctHDirection));
+			if (new_dirstack == NULL) {
+			    priv->dirsize -= 10;
+			    return XctError;
+			}
+			priv->dirstack = new_dirstack;
 		    }
 		    priv->dirstack[data->horz_depth - 1] = data->horizontal;
 		    if (data->item[1] == 0x31)