Fix Xinerama's consolidated visual handling.

Formerly the code claimed it could only handle up to 256 visuals, which
was true.  Also true, but not explicitly stated, was that it could only
handle visuals with VID < 256.  If you have enough screens, and subsystems
that add lots of visuals, you can easily run off the end.  (Made worse
because we allocate visual IDs from the same pool as XIDs.)  If your app
then chooses a visual > 256, then the Xinerama code would throw BadMatch
on CreateColormap and your app wouldn't start.

With this change, PanoramiXVisualTable is gone.  Other subsystems that
were using it as a translation table between each screen's visuals now
use a PanoramiXTranslateVisual() helper.

1 files changed, 134 insertions, 122 deletions

diff --git a/Xext/panoramiX.c b/Xext/panoramiX.c
index 75277beaa..d8ec588aa 100644
--- a/Xext/panoramiX.c
+++ b/Xext/panoramiX.c
@@ -81,9 +81,6 @@ static DepthPtr		PanoramiXDepths;
 static int		PanoramiXNumVisuals;
 static VisualPtr	PanoramiXVisuals;
 
-/* We support at most 256 visuals */
-_X_EXPORT XID		*PanoramiXVisualTable = NULL;
-
 _X_EXPORT unsigned long XRC_DRAWABLE;
 _X_EXPORT unsigned long XRT_WINDOW;
 _X_EXPORT unsigned long XRT_PIXMAP;
@@ -702,143 +699,133 @@ Bool PanoramiXCreateConnectionBlock(void)
     return TRUE;
 }
 
-extern
-void PanoramiXConsolidate(void)
+/*
+ * This isn't just memcmp(), bitsPerRGBValue is skipped.  markv made that
+ * change way back before xf86 4.0, but the comment for _why_ is a bit
+ * opaque, so I'm not going to question it for now.
+ *
+ * This is probably better done as a screen hook so DBE/EVI/GLX can add
+ * their own tests, and adding privates to VisualRec so they don't have to
+ * do their own back-mapping.
+ */
+static Bool
+VisualsEqual(VisualPtr a, VisualPtr b)
 {
-    int 	i, j, k;
-    VisualPtr   pVisual, pVisual2;
-    ScreenPtr   pScreen, pScreen2;
-    DepthPtr    pDepth, pDepth2;
-    PanoramiXRes *root, *defmap, *saver;
-    Bool        foundDepth, missingDepth;
-
-    if(!PanoramiXVisualTable)
-	PanoramiXVisualTable = xcalloc(256 * MAXSCREENS, sizeof(XID));
-
-    pScreen = screenInfo.screens[0];
-    pVisual = pScreen->visuals; 
-    pDepth  = pScreen->allowedDepths;
+    return ((a->class == b->class) &&
+	(a->ColormapEntries == b->ColormapEntries) &&
+	(a->nplanes == b->nplanes) &&
+	(a->redMask == b->redMask) &&
+	(a->greenMask == b->greenMask) &&
+	(a->blueMask == b->blueMask) &&
+	(a->offsetRed == b->offsetRed) &&
+	(a->offsetGreen == b->offsetGreen) &&
+	(a->offsetBlue == b->offsetBlue));
+}
 
-    PanoramiXNumDepths = 0;
-    PanoramiXDepths = xcalloc(pScreen->numDepths,sizeof(DepthRec));
-    PanoramiXNumVisuals = 0;
-    PanoramiXVisuals = xcalloc(pScreen->numVisuals,sizeof(VisualRec));
-
-    for (i = 0; i < pScreen->numDepths; i++, pDepth++) {
-        missingDepth = FALSE;
-        for (j = 1; j < PanoramiXNumScreens; j++) {
-             pScreen2 = screenInfo.screens[j];
-             pDepth2 = pScreen2->allowedDepths;
-
-             foundDepth = FALSE;
-             for (k = 0; k < pScreen2->numDepths; k++, pDepth2++) {
-                 if(pDepth2->depth == pDepth->depth) {
-                     foundDepth = TRUE;
-                     break;
-                 }
-             }
-
-             if(!foundDepth) {
-                missingDepth = TRUE;
-                break;
-             }
-        }
-          
-        if(!missingDepth) {
-            PanoramiXDepths[PanoramiXNumDepths].depth = pDepth->depth;
-            PanoramiXDepths[PanoramiXNumDepths].numVids = 0;
-            if(pDepth->numVids)
-                PanoramiXDepths[PanoramiXNumDepths].vids = 
-                      xalloc(sizeof(VisualID) * pDepth->numVids);      
-            else
-                PanoramiXDepths[PanoramiXNumDepths].vids = NULL;
-            PanoramiXNumDepths++;
-        }
+static void
+PanoramiXMaybeAddDepth(DepthPtr pDepth)
+{
+    ScreenPtr pScreen;
+    int j, k;
+    Bool found = FALSE;
+
+    for (j = 1; j < PanoramiXNumScreens; j++) {
+	pScreen = screenInfo.screens[j];
+	for (k = 0; k < pScreen->numDepths; k++) {
+	    if (pScreen->allowedDepths[k].depth == pDepth->depth) {
+		found = TRUE;
+		break;
+	    }
+	}
     }
 
-    for (i = 0; i < pScreen->numVisuals; i++, pVisual++) {
-	PanoramiXVisualTable[pVisual->vid * MAXSCREENS] = pVisual->vid;
+    if (!found)
+	return;
 
-	/* check if the visual exists on all screens */
-	for (j = 1; j < PanoramiXNumScreens; j++) {
-	    pScreen2 = screenInfo.screens[j];
+    j = PanoramiXNumDepths;
+    PanoramiXNumDepths++;
+    PanoramiXDepths = xrealloc(PanoramiXDepths,
+	    PanoramiXNumDepths * sizeof(DepthRec));
+    PanoramiXDepths[j].depth = pDepth->depth;
+    PanoramiXDepths[j].numVids = 0;
+    /* XXX suboptimal, should grow these dynamically */
+    if(pDepth->numVids)
+	PanoramiXDepths[j].vids = xalloc(sizeof(VisualID) * pDepth->numVids);
+    else
+	PanoramiXDepths[j].vids = NULL;
+}
 
+static void
+PanoramiXMaybeAddVisual(VisualPtr pVisual)
+{
+    ScreenPtr pScreen;
+    VisualPtr candidate = NULL;
+    int j, k;
+    Bool found = FALSE;
+
+    for (j = 1; j < PanoramiXNumScreens; j++) {
+	pScreen = screenInfo.screens[j];
+	found = FALSE;
+
+	candidate = pScreen->visuals;
+	for (k = 0; k < pScreen->numVisuals; k++) {
+	    candidate++;
+	    if (VisualsEqual(pVisual, candidate)
 #ifdef GLXPROXY
-	    pVisual2 = glxMatchVisual(pScreen, pVisual, pScreen2);
-	    if (pVisual2) {
-		PanoramiXVisualTable[(pVisual->vid * MAXSCREENS) + j] =
-		    pVisual2->vid;
-		continue;
-	    } else if (glxMatchVisual(pScreen, pVisual, pScreen)) {
-		PanoramiXVisualTable[(pVisual->vid * MAXSCREENS) + j] = 0;
-		break;
-	    }
+		&& glxMatchVisual(screenInfo.screens[0], pVisual, pScreen)
 #endif
-	    pVisual2 = pScreen2->visuals;
-
-	    for (k = 0; k < pScreen2->numVisuals; k++, pVisual2++) {
-		if ((pVisual->class == pVisual2->class) &&
-		    (pVisual->ColormapEntries == pVisual2->ColormapEntries) &&
-		    (pVisual->nplanes == pVisual2->nplanes) &&
-		    (pVisual->redMask == pVisual2->redMask) &&
-		    (pVisual->greenMask == pVisual2->greenMask) &&
-		    (pVisual->blueMask == pVisual2->blueMask) &&
-		    (pVisual->offsetRed == pVisual2->offsetRed) &&
-		    (pVisual->offsetGreen == pVisual2->offsetGreen) &&
-		    (pVisual->offsetBlue == pVisual2->offsetBlue))
-		{
-                /* We merely assign the first visual that matches.  OpenGL
-                   will need to get involved at some point if you want
-                   match GLX visuals */
-			PanoramiXVisualTable[(pVisual->vid * MAXSCREENS) + j] =
-					 pVisual2->vid;
-			break;
-		}
-	    }
-	}
-	
-	/* if it doesn't exist on all screens we can't use it */
-	for (j = 0; j < PanoramiXNumScreens; j++) {
-	    if (!PanoramiXVisualTable[(pVisual->vid * MAXSCREENS) + j]) {
-		PanoramiXVisualTable[pVisual->vid * MAXSCREENS] = 0;
+		    ) {
+		found = TRUE;
 		break;
 	    }
 	}
 
-	/* if it does, make sure it's in the list of supported depths and visuals */
-	if(PanoramiXVisualTable[pVisual->vid * MAXSCREENS]) {
-	    PanoramiXVisuals[PanoramiXNumVisuals].vid = pVisual->vid;
-	    PanoramiXVisuals[PanoramiXNumVisuals].class = pVisual->class;
-	    PanoramiXVisuals[PanoramiXNumVisuals].bitsPerRGBValue = pVisual->bitsPerRGBValue;
-	    PanoramiXVisuals[PanoramiXNumVisuals].ColormapEntries = pVisual->ColormapEntries;
-	    PanoramiXVisuals[PanoramiXNumVisuals].nplanes = pVisual->nplanes;
-	    PanoramiXVisuals[PanoramiXNumVisuals].redMask = pVisual->redMask;
-	    PanoramiXVisuals[PanoramiXNumVisuals].greenMask = pVisual->greenMask;
-	    PanoramiXVisuals[PanoramiXNumVisuals].blueMask = pVisual->blueMask;
-	    PanoramiXVisuals[PanoramiXNumVisuals].offsetRed = pVisual->offsetRed;
-	    PanoramiXVisuals[PanoramiXNumVisuals].offsetGreen = pVisual->offsetGreen;
-	    PanoramiXVisuals[PanoramiXNumVisuals].offsetBlue = pVisual->offsetBlue;
-	    PanoramiXNumVisuals++;	
-
-	    for (j = 0; j < PanoramiXNumDepths; j++) {
-	        if (PanoramiXDepths[j].depth == pVisual->nplanes) {
-		    PanoramiXDepths[j].vids[PanoramiXDepths[j].numVids] = pVisual->vid;
-		    PanoramiXDepths[j].numVids++;
-		    break;
-		}	
-	    }   
-	}
-    } 
+	if (!found)
+	    return;
+    }
+
+    /* found a matching visual on all screens, add it to the subset list */
+    j = PanoramiXNumVisuals;
+    PanoramiXNumVisuals++;
+    PanoramiXVisuals = xrealloc(PanoramiXVisuals,
+	    PanoramiXNumVisuals * sizeof(VisualRec));
+
+    memcpy(&PanoramiXVisuals[j], pVisual, sizeof(VisualRec));
+
+    for (k = 0; k < PanoramiXNumDepths; k++) {
+	if (PanoramiXDepths[k].depth == pVisual->nplanes) {
+	    PanoramiXDepths[k].vids[PanoramiXDepths[k].numVids] = pVisual->vid;
+	    PanoramiXDepths[k].numVids++;
+	    break;
+	}	
+    }   
+}
+
+extern void
+PanoramiXConsolidate(void)
+{
+    int 	i;
+    PanoramiXRes *root, *defmap, *saver;
+    ScreenPtr   pScreen = screenInfo.screens[0];
+    DepthPtr    pDepth = pScreen->allowedDepths;
+    VisualPtr   pVisual = pScreen->visuals;
+
+    PanoramiXNumDepths = 0;
+    PanoramiXNumVisuals = 0;
 
+    for (i = 0; i < pScreen->numDepths; i++)
+	PanoramiXMaybeAddDepth(pDepth++);
 
-    root = (PanoramiXRes *) xalloc(sizeof(PanoramiXRes));
+    for (i = 0; i < pScreen->numVisuals; i++)
+	PanoramiXMaybeAddVisual(pVisual++);
+
+    root = xalloc(sizeof(PanoramiXRes));
     root->type = XRT_WINDOW;
-    defmap = (PanoramiXRes *) xalloc(sizeof(PanoramiXRes));
+    defmap = xalloc(sizeof(PanoramiXRes));
     defmap->type = XRT_COLORMAP;
-    saver = (PanoramiXRes *) xalloc(sizeof(PanoramiXRes));
+    saver = xalloc(sizeof(PanoramiXRes));
     saver->type = XRT_WINDOW;
 
-
     for (i =  0; i < PanoramiXNumScreens; i++) {
 	root->info[i].id = WindowTable[i]->drawable.id;
 	root->u.win.class = InputOutput;
@@ -854,6 +841,31 @@ void PanoramiXConsolidate(void)
     AddResource(defmap->info[0].id, XRT_COLORMAP, defmap);
 }
 
+_X_EXPORT VisualID
+PanoramiXTranslateVisualID(int screen, VisualID orig)
+{
+    VisualPtr pVisual = NULL;
+    int i, j;
+
+    for (i = 0; i < PanoramiXNumVisuals; i++) {
+	if (orig == PanoramiXVisuals[i].vid) {
+	    pVisual = &PanoramiXVisuals[i];
+	    break;
+	}
+    }
+
+    if (!pVisual)
+	return 0;
+
+    /* found the original, now translate it relative to the backend screen */
+    for (i = 0; i < PanoramiXNumScreens; i++)
+	for (j = 0; j < screenInfo.screens[i]->numVisuals; j++)
+	    if (VisualsEqual(pVisual, &screenInfo.screens[i]->visuals[j]))
+		return screenInfo.screens[i]->visuals[j].vid;
+    
+    return 0;
+}
+
 
 /*
  *	PanoramiXResetProc()