os: Prevent core dump from being truncated.

The problem fixed by this patch can be reproduced on Linux with the
following steps.
- Access NULL pointer intentionally in ProcessOtherEvent on key press.
- Instead of saving core dump to a file, write it into a pipe.
  echo "|/usr/sbin/my-core-dumper" > /proc/sys/kernel/core_pattern
- Dump the core by pressing a key.

While the core is being dumped into the pipe, the smart schedule timer
will cause a pending SIGALRM. Linux kernel stops writing data to the
pipe when there are pending signals. This causes the core dump to be
truncated. On my system I'm expecting a 6 MB dump but the size will be
60 kB instead. The problem is solved if we block the SIGALRM caused by
expired smart schedule timer.

I haven't been able to reproduce this problem in the following cases.
- Save core dump to a file instead of a pipe.
- kill -SEGV `pidof Xorg`
- Press a key to dump core while gdb is attached to Xorg.
- Give option -dumbSched to Xorg.

Also note that the fix works only when NoTrapSignals has the default
value FALSE. The problem can still be reproduced if error signals
aren't trapped. In addition to pending SIGALRM, there is a similar
problem with pending SIGIO from the keyboard driver during core dump.

Signed-off-by: Rami Ylimaki <ext-rami.ylimaki@nokia.com>
Reviewed-by: Keith Packard <keithp@keithp.com>
Signed-off-by: Keith Packard <keithp@keithp.com>

2 files changed, 6 insertions, 6 deletions

diff --git a/miext/damage/damage.c b/miext/damage/damage.c
index b7ec92a5b..2851aed8d 100644
--- a/miext/damage/damage.c
+++ b/miext/damage/damage.c
@@ -1705,7 +1705,7 @@ damageRemoveDamage (DamagePtr *pPrev, DamagePtr pDamage)
     }
 #if DAMAGE_VALIDATE_ENABLE
     ErrorF ("Damage not on list\n");
-    abort ();
+    OsAbort ();
 #endif
 }
 
@@ -1718,7 +1718,7 @@ damageInsertDamage (DamagePtr *pPrev, DamagePtr pDamage)
     for (pOld = *pPrev; pOld; pOld = pOld->pNext)
 	if (pOld == pDamage) {
 	    ErrorF ("Damage already on list\n");
-	    abort ();
+	    OsAbort ();
 	}
 #endif
     pDamage->pNext = *pPrev;
@@ -1971,7 +1971,7 @@ DamageRegister (DrawablePtr pDrawable,
     if (pDrawable->pScreen != pDamage->pScreen)
     {
 	ErrorF ("DamageRegister called with mismatched screens\n");
-	abort ();
+	OsAbort ();
     }
 #endif
 
@@ -1986,7 +1986,7 @@ DamageRegister (DrawablePtr pDrawable,
 	for (pOld = *pPrev; pOld; pOld = pOld->pNextWin)
 	    if (pOld == pDamage) {
 		ErrorF ("Damage already on window list\n");
-		abort ();
+		OsAbort ();
 	    }
 #endif
 	pDamage->pNextWin = *pPrev;
@@ -2040,7 +2040,7 @@ DamageUnregister (DrawablePtr	    pDrawable,
 #if DAMAGE_VALIDATE_ENABLE
 	if (!found) {
 	    ErrorF ("Damage not on window list\n");
-	    abort ();
+	    OsAbort ();
 	}
 #endif
     }
diff --git a/miext/rootless/rootlessWindow.c b/miext/rootless/rootlessWindow.c
index 53209565d..e78e2c8f1 100644
--- a/miext/rootless/rootlessWindow.c
+++ b/miext/rootless/rootlessWindow.c
@@ -978,7 +978,7 @@ StartFrameResize(WindowPtr pWin, Bool gravity,
                 copy_rect.y2 = oldY2;
             }
             else
-                abort();
+                OsAbort();
 
             Bpp = winRec->win->drawable.bitsPerPixel / 8;
             copy_rect_width = copy_rect.x2 - copy_rect.x1;