XQuartz: RandR: Don't crash if X11 is launched while there are no attached displays

If CG reports no displays when launching, we could crash in RandR.  Instead, just
provide a fake 800x600 display until we are notified about displays being attached.

Signed-off-by: Jeremy Huddleston <jeremyhu@apple.com>

3 files changed, 117 insertions, 100 deletions

diff --git a/hw/xquartz/quartz.c b/hw/xquartz/quartz.c
index 0e71d3629..68a059c47 100644
--- a/hw/xquartz/quartz.c
+++ b/hw/xquartz/quartz.c
@@ -464,11 +464,15 @@ void QuartzSpaceChanged(uint32_t space_id) {
 void QuartzCopyDisplayIDs(ScreenPtr pScreen,
                           int displayCount, CGDirectDisplayID *displayIDs) {
     QuartzScreenPtr pQuartzScreen = QUARTZ_PRIV(pScreen);
-    int size = displayCount * sizeof(CGDirectDisplayID);
 
     free(pQuartzScreen->displayIDs);
-    pQuartzScreen->displayIDs = malloc(size);
-    memcpy(pQuartzScreen->displayIDs, displayIDs, size);
+    if(displayCount) {
+        size_t size = displayCount * sizeof(CGDirectDisplayID);
+        pQuartzScreen->displayIDs = malloc(size);
+        memcpy(pQuartzScreen->displayIDs, displayIDs, size);
+    } else {
+       pQuartzScreen->displayIDs = NULL;
+    }
     pQuartzScreen->displayCount = displayCount;
 }
 
diff --git a/hw/xquartz/quartzRandR.c b/hw/xquartz/quartzRandR.c
index d452b029a..98b382d65 100644
--- a/hw/xquartz/quartzRandR.c
+++ b/hw/xquartz/quartzRandR.c
@@ -3,7 +3,7 @@
  *
  * Copyright (c) 2001-2004 Greg Parker and Torrey T. Lyons,
  *               2010      Jan Hauffa.
- *               2010      Apple Inc.
+ *               2010-2011 Apple Inc.
  *                 All Rights Reserved.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -124,44 +124,48 @@ static Bool QuartzRandRSetCGMode (CGDirectDisplayID screenId,
 static Bool QuartzRandREnumerateModes (ScreenPtr pScreen,
                                        QuartzModeCallback callback,
                                        void *data) {
-    CFDictionaryRef curModeRef, modeRef;
-    long curBpp;
-    CFArrayRef modes;
-    QuartzModeInfo modeInfo;
-    int i;
-    BOOL retval = FALSE;
+    Bool retval = FALSE;
     QuartzScreenPtr pQuartzScreen = QUARTZ_PRIV(pScreen);
-    CGDirectDisplayID screenId = pQuartzScreen->displayIDs[0];
 
-    curModeRef = CGDisplayCurrentMode(screenId);
-    if (!curModeRef)
-        return FALSE;
-    curBpp = getDictLong(curModeRef, kCGDisplayBitsPerPixel);
+    /* Just an 800x600 fallback if we have no attached heads */
+    if(pQuartzScreen->displayIDs) {
+        CFDictionaryRef curModeRef, modeRef;
+        long curBpp;
+        CFArrayRef modes;
+        QuartzModeInfo modeInfo;
+        int i;
+        CGDirectDisplayID screenId = pQuartzScreen->displayIDs[0];
+
+        curModeRef = CGDisplayCurrentMode(screenId);
+        if (!curModeRef)
+            return FALSE;
+        curBpp = getDictLong(curModeRef, kCGDisplayBitsPerPixel);
 
-    modes = CGDisplayAvailableModes(screenId);
-    if (!modes)
-        return FALSE;
-    for (i = 0; i < CFArrayGetCount(modes); i++) {
-        int cb;
-        modeRef = (CFDictionaryRef) CFArrayGetValueAtIndex(modes, i);
-
-        /* Skip modes that are not usable on the current display or have a
-           different pixel encoding than the current mode. */
-        if (((unsigned long) getDictLong(modeRef, kCGDisplayIOFlags) &
-             kDisplayModeUsableFlags) != kDisplayModeUsableFlags)
-            continue;
-        if (getDictLong(modeRef, kCGDisplayBitsPerPixel) != curBpp)
-            continue;
-
-        QuartzRandRGetModeInfo(modeRef, &modeInfo);
-        modeInfo.ref = (void *)modeRef;
-        cb = callback(pScreen, &modeInfo, data);
-        if (cb == CALLBACK_CONTINUE)
-            retval = TRUE;
-        else if (cb == CALLBACK_SUCCESS)
-            return TRUE;
-        else if (cb == CALLBACK_ERROR)
+        modes = CGDisplayAvailableModes(screenId);
+        if (!modes)
             return FALSE;
+        for (i = 0; i < CFArrayGetCount(modes); i++) {
+            int cb;
+            modeRef = (CFDictionaryRef) CFArrayGetValueAtIndex(modes, i);
+
+            /* Skip modes that are not usable on the current display or have a
+               different pixel encoding than the current mode. */
+            if (((unsigned long) getDictLong(modeRef, kCGDisplayIOFlags) &
+                 kDisplayModeUsableFlags) != kDisplayModeUsableFlags)
+                continue;
+            if (getDictLong(modeRef, kCGDisplayBitsPerPixel) != curBpp)
+                continue;
+
+            QuartzRandRGetModeInfo(modeRef, &modeInfo);
+            modeInfo.ref = (void *)modeRef;
+            cb = callback(pScreen, &modeInfo, data);
+            if (cb == CALLBACK_CONTINUE)
+                retval = TRUE;
+            else if (cb == CALLBACK_SUCCESS)
+                return TRUE;
+            else if (cb == CALLBACK_ERROR)
+                return FALSE;
+        }
     }
 
     switch(callback(pScreen, &pQuartzScreen->rootlessMode, data)) {
@@ -225,61 +229,64 @@ static Bool QuartzRandRSetCGMode (CGDirectDisplayID screenId,
 static Bool QuartzRandREnumerateModes (ScreenPtr pScreen,
                                        QuartzModeCallback callback,
                                        void *data) {
-    CGDisplayModeRef curModeRef, modeRef;
-    CFStringRef curPixelEnc, pixelEnc;
-    CFComparisonResult pixelEncEqual;
-    CFArrayRef modes;
-    QuartzModeInfo modeInfo;
-    int i;
     Bool retval = FALSE;
-
     QuartzScreenPtr pQuartzScreen = QUARTZ_PRIV(pScreen);
-    CGDirectDisplayID screenId = pQuartzScreen->displayIDs[0];
 
-    curModeRef = CGDisplayCopyDisplayMode(screenId);
-    if (!curModeRef)
-        return FALSE;
-    curPixelEnc = CGDisplayModeCopyPixelEncoding(curModeRef);
-    CGDisplayModeRelease(curModeRef);
+    /* Just an 800x600 fallback if we have no attached heads */
+    if(pQuartzScreen->displayIDs) {
+        CGDisplayModeRef curModeRef, modeRef;
+        CFStringRef curPixelEnc, pixelEnc;
+        CFComparisonResult pixelEncEqual;
+        CFArrayRef modes;
+        QuartzModeInfo modeInfo;
+        int i;
+        CGDirectDisplayID screenId = pQuartzScreen->displayIDs[0];
+
+        curModeRef = CGDisplayCopyDisplayMode(screenId);
+        if (!curModeRef)
+            return FALSE;
+        curPixelEnc = CGDisplayModeCopyPixelEncoding(curModeRef);
+        CGDisplayModeRelease(curModeRef);
 
-    modes = CGDisplayCopyAllDisplayModes(screenId, NULL);
-    if (!modes) {
-        CFRelease(curPixelEnc);
-        return FALSE;
-    }
-    for (i = 0; i < CFArrayGetCount(modes); i++) {
-        int cb;
-        modeRef = (CGDisplayModeRef) CFArrayGetValueAtIndex(modes, i);
-
-        /* Skip modes that are not usable on the current display or have a
-           different pixel encoding than the current mode. */
-        if ((CGDisplayModeGetIOFlags(modeRef) & kDisplayModeUsableFlags) !=
-            kDisplayModeUsableFlags)
-            continue;
-        pixelEnc = CGDisplayModeCopyPixelEncoding(modeRef);
-        pixelEncEqual = CFStringCompare(pixelEnc, curPixelEnc, 0);
-        CFRelease(pixelEnc);
-        if (pixelEncEqual != kCFCompareEqualTo)
-            continue;
-
-        QuartzRandRGetModeInfo(modeRef, &modeInfo);
-        modeInfo.ref = modeRef;
-        cb = callback(pScreen, &modeInfo, data);
-        if (cb == CALLBACK_CONTINUE) {
-            retval = TRUE;
-        } else if (cb == CALLBACK_SUCCESS) {
-            CFRelease(modes);
-            CFRelease(curPixelEnc);
-            return TRUE;
-        } else if (cb == CALLBACK_ERROR) {
-            CFRelease(modes);
+        modes = CGDisplayCopyAllDisplayModes(screenId, NULL);
+        if (!modes) {
             CFRelease(curPixelEnc);
             return FALSE;
         }
-    }
+        for (i = 0; i < CFArrayGetCount(modes); i++) {
+            int cb;
+            modeRef = (CGDisplayModeRef) CFArrayGetValueAtIndex(modes, i);
+
+            /* Skip modes that are not usable on the current display or have a
+               different pixel encoding than the current mode. */
+            if ((CGDisplayModeGetIOFlags(modeRef) & kDisplayModeUsableFlags) !=
+                kDisplayModeUsableFlags)
+                continue;
+            pixelEnc = CGDisplayModeCopyPixelEncoding(modeRef);
+            pixelEncEqual = CFStringCompare(pixelEnc, curPixelEnc, 0);
+            CFRelease(pixelEnc);
+            if (pixelEncEqual != kCFCompareEqualTo)
+                continue;
+
+            QuartzRandRGetModeInfo(modeRef, &modeInfo);
+            modeInfo.ref = modeRef;
+            cb = callback(pScreen, &modeInfo, data);
+            if (cb == CALLBACK_CONTINUE) {
+                retval = TRUE;
+            } else if (cb == CALLBACK_SUCCESS) {
+                CFRelease(modes);
+                CFRelease(curPixelEnc);
+                return TRUE;
+            } else if (cb == CALLBACK_ERROR) {
+                CFRelease(modes);
+                CFRelease(curPixelEnc);
+                return FALSE;
+            }
+        }
 
-    CFRelease(modes);
-    CFRelease(curPixelEnc);
+        CFRelease(modes);
+        CFRelease(curPixelEnc);
+    }
 
     switch(callback(pScreen, &pQuartzScreen->rootlessMode, data)) {
         case CALLBACK_SUCCESS:
@@ -347,9 +354,13 @@ static int QuartzRandRRegisterModeCallback (ScreenPtr pScreen,
 
 static Bool QuartzRandRSetMode(ScreenPtr pScreen, QuartzModeInfoPtr pMode, BOOL doRegister) {
     QuartzScreenPtr pQuartzScreen = QUARTZ_PRIV(pScreen);
-    CGDirectDisplayID screenId = pQuartzScreen->displayIDs[0];
     Bool captureDisplay = (pMode->refresh != FAKE_REFRESH_FULLSCREEN && pMode->refresh != FAKE_REFRESH_ROOTLESS);
+    CGDirectDisplayID screenId;
 
+    if(pQuartzScreen->displayIDs == NULL)
+        return FALSE;
+
+    screenId = pQuartzScreen->displayIDs[0];
     if(XQuartzShieldingWindowLevel == 0 && captureDisplay) {
         if(!X11ApplicationCanEnterRandR())
             return FALSE;
@@ -379,7 +390,8 @@ static Bool QuartzRandRSetMode(ScreenPtr pScreen, QuartzModeInfoPtr pMode, BOOL
     if(pQuartzScreen->currentMode.ref)
         CFRelease(pQuartzScreen->currentMode.ref);
     pQuartzScreen->currentMode = *pMode;
-    CFRetain(pQuartzScreen->currentMode.ref);
+    if(pQuartzScreen->currentMode.ref)
+        CFRetain(pQuartzScreen->currentMode.ref);
     
     if(XQuartzShieldingWindowLevel != 0 && !captureDisplay) {
         CGReleaseAllDisplays();
@@ -406,13 +418,8 @@ static int QuartzRandRSetModeCallback (ScreenPtr pScreen,
 }
 
 static Bool QuartzRandRGetInfo (ScreenPtr pScreen, Rotation *rotations) {
-    QuartzScreenPtr pQuartzScreen = QUARTZ_PRIV(pScreen);
-
     *rotations = RR_Rotate_0;  /* TODO: support rotation */
 
-    if (pQuartzScreen->displayCount == 0)
-        return FALSE;
-
     return QuartzRandREnumerateModes(pScreen, QuartzRandRRegisterModeCallback, NULL);
 }
 
@@ -427,9 +434,6 @@ static Bool QuartzRandRSetConfig (ScreenPtr           pScreen,
     reqMode.height = pSize->height;
     reqMode.refresh = rate;
 
-    if (pQuartzScreen->displayCount == 0)
-        return FALSE;
-
     /* Do not switch modes if requested mode is equal to current mode. */
     if (QuartzRandRModesEqual(&reqMode, &pQuartzScreen->currentMode))
         return TRUE;
@@ -446,9 +450,16 @@ static Bool _QuartzRandRUpdateFakeModes (ScreenPtr pScreen) {
     QuartzScreenPtr pQuartzScreen = QUARTZ_PRIV(pScreen);
     QuartzModeInfo activeMode;
 
-    if (!QuartzRandRCopyCurrentModeInfo(pQuartzScreen->displayIDs[0], &activeMode)) {
-        ErrorF("Unable to determine current display mode.\n");
-        return FALSE;
+    if(pQuartzScreen->displayCount > 0) {
+        if(!QuartzRandRCopyCurrentModeInfo(pQuartzScreen->displayIDs[0], &activeMode)) {
+            ErrorF("Unable to determine current display mode.\n");
+            return FALSE;
+        }
+    } else {
+        memset(&activeMode, 0, sizeof(activeMode));
+        activeMode.width = 800;
+        activeMode.height = 600;
+        activeMode.refresh = 60;
     }
 
     if(pQuartzScreen->fullscreenMode.ref)
@@ -456,7 +467,7 @@ static Bool _QuartzRandRUpdateFakeModes (ScreenPtr pScreen) {
     if(pQuartzScreen->currentMode.ref)
         CFRelease(pQuartzScreen->currentMode.ref);
 
-    if (pQuartzScreen->displayCount > 1) {
+    if(pQuartzScreen->displayCount > 1) {
         activeMode.width = pScreen->width;
         activeMode.height = pScreen->height;
         if(XQuartzIsRootless)
@@ -479,7 +490,8 @@ static Bool _QuartzRandRUpdateFakeModes (ScreenPtr pScreen) {
     /* This extra retain is for currentMode's copy.
      * fullscreen and rootless share a retain.
      */
-    CFRetain(pQuartzScreen->currentMode.ref);
+    if(pQuartzScreen->currentMode.ref)
+        CFRetain(pQuartzScreen->currentMode.ref);
     
     DEBUG_LOG("rootlessMode: %d x %d\n", (int)pQuartzScreen->rootlessMode.width, (int)pQuartzScreen->rootlessMode.height);
     DEBUG_LOG("fullscreenMode: %d x %d\n", (int)pQuartzScreen->fullscreenMode.width, (int)pQuartzScreen->fullscreenMode.height);
diff --git a/hw/xquartz/xpr/xprScreen.c b/hw/xquartz/xpr/xprScreen.c
index d2968dc1e..98250e641 100644
--- a/hw/xquartz/xpr/xprScreen.c
+++ b/hw/xquartz/xpr/xprScreen.c
@@ -193,6 +193,7 @@ xprAddPseudoramiXScreens(int *x, int *y, int *width, int *height, ScreenPtr pScr
         *width = 800;
         *height = 600;
         PseudoramiXAddScreen(*x, *y, *width, *height);
+        QuartzCopyDisplayIDs(pScreen, 0, NULL);
         return;
     }