Add XNFcallocarray() to allow xnfcalloc() to check for overflow

The xnfcalloc() macro took two arguments but simply multiplied them
together without checking for overflow and defeating any overflow
checking that calloc() might have done.  Let's not do that.

The original XNFcalloc() function is left for now to preserve driver
ABI, but is marked as deprecated so it can be removed in a future round
of ABI break/cleanup.

Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>
Reviewed-by: Matt Turner <mattst88@gmail.com>

4 files changed, 24 insertions, 6 deletions

diff --git a/config/config.c b/config/config.c
index b5d634b87..de45cc350 100644
--- a/config/config.c
+++ b/config/config.c
@@ -130,7 +130,8 @@ device_is_duplicate(const char *config_info)
 struct OdevAttributes *
 config_odev_allocate_attributes(void)
 {
-    struct OdevAttributes *attribs = XNFcalloc(sizeof (struct OdevAttributes));
+    struct OdevAttributes *attribs =
+        xnfcalloc(1, sizeof (struct OdevAttributes));
     attribs->fd = -1;
     return attribs;
 }
diff --git a/hw/xfree86/utils/cvt/cvt.c b/hw/xfree86/utils/cvt/cvt.c
index d5df17fd9..9413c20fa 100644
--- a/hw/xfree86/utils/cvt/cvt.c
+++ b/hw/xfree86/utils/cvt/cvt.c
@@ -54,11 +54,11 @@ XNFalloc(unsigned long n)
 
 /* xnfcalloc implementation used by the server code we built in */
 void *
-XNFcalloc(unsigned long n)
+XNFcallocarray(size_t nmemb, size_t size)
 {
     void *r;
 
-    r = calloc(1, n);
+    r = calloc(nmemb, size);
     if (!r) {
         perror("calloc failed");
         exit(1);
diff --git a/include/os.h b/include/os.h
index 3e68c4933..9d8b859d8 100644
--- a/include/os.h
+++ b/include/os.h
@@ -69,7 +69,7 @@ typedef struct _NewClientRec *NewClientPtr;
 
 #ifndef xnfalloc
 #define xnfalloc(size) XNFalloc((unsigned long)(size))
-#define xnfcalloc(_num, _size) XNFcalloc((unsigned long)(_num)*(unsigned long)(_size))
+#define xnfcalloc(_num, _size) XNFcallocarray((_num), (_size))
 #define xnfrealloc(ptr, size) XNFrealloc((void *)(ptr), (unsigned long)(size))
 
 #define xstrdup(s) Xstrdup(s)
@@ -222,7 +222,14 @@ XNFalloc(unsigned long /*amount */ );
  * enough memory.
  */
 extern _X_EXPORT void *
-XNFcalloc(unsigned long /*amount */ );
+XNFcalloc(unsigned long /*amount */ ) _X_DEPRECATED;
+
+/*
+ * This function calloc(3)s buffer, terminating the server if there is not
+ * enough memory or the arguments overflow when multiplied
+ */
+extern _X_EXPORT void *
+XNFcallocarray(size_t nmemb, size_t size);
 
 /*
  * This function realloc(3)s passed buffer, terminating the server if there is
diff --git a/os/utils.c b/os/utils.c
index 75769f17c..24a87516e 100644
--- a/os/utils.c
+++ b/os/utils.c
@@ -1128,10 +1128,20 @@ XNFalloc(unsigned long amount)
     return ptr;
 }
 
+/* The original XNFcalloc was used with the xnfcalloc macro which multiplied
+ * the arguments at the call site without allowing calloc to check for overflow.
+ * XNFcallocarray was added to fix that without breaking ABI.
+ */
 void *
 XNFcalloc(unsigned long amount)
 {
-    void *ret = calloc(1, amount);
+    return XNFcallocarray(1, amount);
+}
+
+void *
+XNFcallocarray(size_t nmemb, size_t size)
+{
+    void *ret = calloc(nmemb, size);
 
     if (!ret)
         FatalError("XNFcalloc: Out of memory");