orccodemem: Implement forwarding mmap() failure when allocating the region

Part-of: <https://gitlab.freedesktop.org/gstreamer/orc/-/merge_requests/176>

1 files changed, 24 insertions, 11 deletions

diff --git a/orc/orccodemem.c b/orc/orccodemem.c
index ca1f7e0..6431250 100644
--- a/orc/orccodemem.c
+++ b/orc/orccodemem.c
@@ -59,7 +59,7 @@ struct _OrcCodeChunk {
 };
 
 
-static void orc_code_region_allocate_codemem (OrcCodeRegion *region);
+static int orc_code_region_allocate_codemem (OrcCodeRegion *region);
 
 static OrcCodeRegion **orc_code_regions;
 static int orc_code_n_regions;
@@ -301,27 +301,27 @@ orc_code_region_allocate_codemem_anon_map (OrcCodeRegion *region)
   return TRUE;
 }
 
-static void
+int
 orc_code_region_allocate_codemem (OrcCodeRegion *region)
 {
   const char *tmpdir;
 
   tmpdir = getenv ("XDG_RUNTIME_DIR");
   if (tmpdir && orc_code_region_allocate_codemem_dual_map (region,
-        tmpdir, FALSE)) return;
+        tmpdir, FALSE)) return TRUE;
 
   tmpdir = getenv ("HOME");
   if (tmpdir && orc_code_region_allocate_codemem_dual_map (region,
-        tmpdir, FALSE)) return;
+        tmpdir, FALSE)) return TRUE;
 
   tmpdir = getenv ("TMPDIR");
   if (tmpdir && orc_code_region_allocate_codemem_dual_map (region,
-        tmpdir, FALSE)) return;
+        tmpdir, FALSE)) return TRUE;
 
   if (orc_code_region_allocate_codemem_dual_map (region,
-        "/tmp", FALSE)) return;
+        "/tmp", FALSE)) return TRUE;
 
-  if (orc_code_region_allocate_codemem_anon_map (region)) return;
+  if (orc_code_region_allocate_codemem_anon_map (region)) return TRUE;
 
 #ifdef __APPLE__
   ORC_ERROR("Failed to create write and exec mmap regions.  This "
@@ -332,30 +332,43 @@ orc_code_region_allocate_codemem (OrcCodeRegion *region)
       "is probably because SELinux execmem check is enabled (good) "
       "and $TMPDIR and $HOME are mounted noexec (bad).");
 #endif
+  return FALSE;
 }
 
 #endif
 
 #ifdef HAVE_CODEMEM_VIRTUALALLOC
-void
+int
 orc_code_region_allocate_codemem (OrcCodeRegion *region)
 {
   /* On UWP, we can't allocate memory as executable from the start. We can only
    * set that later after compiling and copying the code over. This is a good
    * idea in general to avoid security issues, so we do it on win32 too. */
-  region->write_ptr = _virtualalloc (NULL, SIZE, MEM_COMMIT, PAGE_READWRITE);
+  void *write_ptr;
+  write_ptr = _virtualalloc (NULL, SIZE, MEM_COMMIT, PAGE_READWRITE);
+  if (!write_ptr)
+    return FALSE;
+
+  region->write_ptr = write_ptr;
   region->exec_ptr = region->write_ptr;
   region->size = SIZE;
+  return TRUE;
 }
 #endif
 
 #ifdef HAVE_CODEMEM_MALLOC
-void
+int
 orc_code_region_allocate_codemem (OrcCodeRegion *region)
 {
-  region->write_ptr = malloc(SIZE);
+  void *write_ptr;
+  write_ptr = malloc(SIZE);
+  if (!write_ptr)
+    return FALSE;
+
+  region->write_ptr = write_ptr;
   region->exec_ptr = region->write_ptr;
   region->size = SIZE;
+  return TRUE;
 }
 #endif