Fix Xinerama's consolidated visual handling.

Formerly the code claimed it could only handle up to 256 visuals, which
was true.  Also true, but not explicitly stated, was that it could only
handle visuals with VID < 256.  If you have enough screens, and subsystems
that add lots of visuals, you can easily run off the end.  (Made worse
because we allocate visual IDs from the same pool as XIDs.)  If your app
then chooses a visual > 256, then the Xinerama code would throw BadMatch
on CreateColormap and your app wouldn't start.

With this change, PanoramiXVisualTable is gone.  Other subsystems that
were using it as a translation table between each screen's visuals now
use a PanoramiXTranslateVisual() helper.

6 files changed, 140 insertions, 141 deletions

diff --git a/Xext/panoramiX.c b/Xext/panoramiX.c
index 75277beaa..d8ec588aa 100644
--- a/Xext/panoramiX.c
+++ b/Xext/panoramiX.c
@@ -81,9 +81,6 @@ static DepthPtr		PanoramiXDepths;
 static int		PanoramiXNumVisuals;
 static VisualPtr	PanoramiXVisuals;
 
-/* We support at most 256 visuals */
-_X_EXPORT XID		*PanoramiXVisualTable = NULL;
-
 _X_EXPORT unsigned long XRC_DRAWABLE;
 _X_EXPORT unsigned long XRT_WINDOW;
 _X_EXPORT unsigned long XRT_PIXMAP;
@@ -702,143 +699,133 @@ Bool PanoramiXCreateConnectionBlock(void)
     return TRUE;
 }
 
-extern
-void PanoramiXConsolidate(void)
+/*
+ * This isn't just memcmp(), bitsPerRGBValue is skipped.  markv made that
+ * change way back before xf86 4.0, but the comment for _why_ is a bit
+ * opaque, so I'm not going to question it for now.
+ *
+ * This is probably better done as a screen hook so DBE/EVI/GLX can add
+ * their own tests, and adding privates to VisualRec so they don't have to
+ * do their own back-mapping.
+ */
+static Bool
+VisualsEqual(VisualPtr a, VisualPtr b)
 {
-    int 	i, j, k;
-    VisualPtr   pVisual, pVisual2;
-    ScreenPtr   pScreen, pScreen2;
-    DepthPtr    pDepth, pDepth2;
-    PanoramiXRes *root, *defmap, *saver;
-    Bool        foundDepth, missingDepth;
-
-    if(!PanoramiXVisualTable)
-	PanoramiXVisualTable = xcalloc(256 * MAXSCREENS, sizeof(XID));
-
-    pScreen = screenInfo.screens[0];
-    pVisual = pScreen->visuals; 
-    pDepth  = pScreen->allowedDepths;
+    return ((a->class == b->class) &&
+	(a->ColormapEntries == b->ColormapEntries) &&
+	(a->nplanes == b->nplanes) &&
+	(a->redMask == b->redMask) &&
+	(a->greenMask == b->greenMask) &&
+	(a->blueMask == b->blueMask) &&
+	(a->offsetRed == b->offsetRed) &&
+	(a->offsetGreen == b->offsetGreen) &&
+	(a->offsetBlue == b->offsetBlue));
+}
 
-    PanoramiXNumDepths = 0;
-    PanoramiXDepths = xcalloc(pScreen->numDepths,sizeof(DepthRec));
-    PanoramiXNumVisuals = 0;
-    PanoramiXVisuals = xcalloc(pScreen->numVisuals,sizeof(VisualRec));
-
-    for (i = 0; i < pScreen->numDepths; i++, pDepth++) {
-        missingDepth = FALSE;
-        for (j = 1; j < PanoramiXNumScreens; j++) {
-             pScreen2 = screenInfo.screens[j];
-             pDepth2 = pScreen2->allowedDepths;
-
-             foundDepth = FALSE;
-             for (k = 0; k < pScreen2->numDepths; k++, pDepth2++) {
-                 if(pDepth2->depth == pDepth->depth) {
-                     foundDepth = TRUE;
-                     break;
-                 }
-             }
-
-             if(!foundDepth) {
-                missingDepth = TRUE;
-                break;
-             }
-        }
-          
-        if(!missingDepth) {
-            PanoramiXDepths[PanoramiXNumDepths].depth = pDepth->depth;
-            PanoramiXDepths[PanoramiXNumDepths].numVids = 0;
-            if(pDepth->numVids)
-                PanoramiXDepths[PanoramiXNumDepths].vids = 
-                      xalloc(sizeof(VisualID) * pDepth->numVids);      
-            else
-                PanoramiXDepths[PanoramiXNumDepths].vids = NULL;
-            PanoramiXNumDepths++;
-        }
+static void
+PanoramiXMaybeAddDepth(DepthPtr pDepth)
+{
+    ScreenPtr pScreen;
+    int j, k;
+    Bool found = FALSE;
+
+    for (j = 1; j < PanoramiXNumScreens; j++) {
+	pScreen = screenInfo.screens[j];
+	for (k = 0; k < pScreen->numDepths; k++) {
+	    if (pScreen->allowedDepths[k].depth == pDepth->depth) {
+		found = TRUE;
+		break;
+	    }
+	}
     }
 
-    for (i = 0; i < pScreen->numVisuals; i++, pVisual++) {
-	PanoramiXVisualTable[pVisual->vid * MAXSCREENS] = pVisual->vid;
+    if (!found)
+	return;
 
-	/* check if the visual exists on all screens */
-	for (j = 1; j < PanoramiXNumScreens; j++) {
-	    pScreen2 = screenInfo.screens[j];
+    j = PanoramiXNumDepths;
+    PanoramiXNumDepths++;
+    PanoramiXDepths = xrealloc(PanoramiXDepths,
+	    PanoramiXNumDepths * sizeof(DepthRec));
+    PanoramiXDepths[j].depth = pDepth->depth;
+    PanoramiXDepths[j].numVids = 0;
+    /* XXX suboptimal, should grow these dynamically */
+    if(pDepth->numVids)
+	PanoramiXDepths[j].vids = xalloc(sizeof(VisualID) * pDepth->numVids);
+    else
+	PanoramiXDepths[j].vids = NULL;
+}
 
+static void
+PanoramiXMaybeAddVisual(VisualPtr pVisual)
+{
+    ScreenPtr pScreen;
+    VisualPtr candidate = NULL;
+    int j, k;
+    Bool found = FALSE;
+
+    for (j = 1; j < PanoramiXNumScreens; j++) {
+	pScreen = screenInfo.screens[j];
+	found = FALSE;
+
+	candidate = pScreen->visuals;
+	for (k = 0; k < pScreen->numVisuals; k++) {
+	    candidate++;
+	    if (VisualsEqual(pVisual, candidate)
 #ifdef GLXPROXY
-	    pVisual2 = glxMatchVisual(pScreen, pVisual, pScreen2);
-	    if (pVisual2) {
-		PanoramiXVisualTable[(pVisual->vid * MAXSCREENS) + j] =
-		    pVisual2->vid;
-		continue;
-	    } else if (glxMatchVisual(pScreen, pVisual, pScreen)) {
-		PanoramiXVisualTable[(pVisual->vid * MAXSCREENS) + j] = 0;
-		break;
-	    }
+		&& glxMatchVisual(screenInfo.screens[0], pVisual, pScreen)
 #endif
-	    pVisual2 = pScreen2->visuals;
-
-	    for (k = 0; k < pScreen2->numVisuals; k++, pVisual2++) {
-		if ((pVisual->class == pVisual2->class) &&
-		    (pVisual->ColormapEntries == pVisual2->ColormapEntries) &&
-		    (pVisual->nplanes == pVisual2->nplanes) &&
-		    (pVisual->redMask == pVisual2->redMask) &&
-		    (pVisual->greenMask == pVisual2->greenMask) &&
-		    (pVisual->blueMask == pVisual2->blueMask) &&
-		    (pVisual->offsetRed == pVisual2->offsetRed) &&
-		    (pVisual->offsetGreen == pVisual2->offsetGreen) &&
-		    (pVisual->offsetBlue == pVisual2->offsetBlue))
-		{
-                /* We merely assign the first visual that matches.  OpenGL
-                   will need to get involved at some point if you want
-                   match GLX visuals */
-			PanoramiXVisualTable[(pVisual->vid * MAXSCREENS) + j] =
-					 pVisual2->vid;
-			break;
-		}
-	    }
-	}
-	
-	/* if it doesn't exist on all screens we can't use it */
-	for (j = 0; j < PanoramiXNumScreens; j++) {
-	    if (!PanoramiXVisualTable[(pVisual->vid * MAXSCREENS) + j]) {
-		PanoramiXVisualTable[pVisual->vid * MAXSCREENS] = 0;
+		    ) {
+		found = TRUE;
 		break;
 	    }
 	}
 
-	/* if it does, make sure it's in the list of supported depths and visuals */
-	if(PanoramiXVisualTable[pVisual->vid * MAXSCREENS]) {
-	    PanoramiXVisuals[PanoramiXNumVisuals].vid = pVisual->vid;
-	    PanoramiXVisuals[PanoramiXNumVisuals].class = pVisual->class;
-	    PanoramiXVisuals[PanoramiXNumVisuals].bitsPerRGBValue = pVisual->bitsPerRGBValue;
-	    PanoramiXVisuals[PanoramiXNumVisuals].ColormapEntries = pVisual->ColormapEntries;
-	    PanoramiXVisuals[PanoramiXNumVisuals].nplanes = pVisual->nplanes;
-	    PanoramiXVisuals[PanoramiXNumVisuals].redMask = pVisual->redMask;
-	    PanoramiXVisuals[PanoramiXNumVisuals].greenMask = pVisual->greenMask;
-	    PanoramiXVisuals[PanoramiXNumVisuals].blueMask = pVisual->blueMask;
-	    PanoramiXVisuals[PanoramiXNumVisuals].offsetRed = pVisual->offsetRed;
-	    PanoramiXVisuals[PanoramiXNumVisuals].offsetGreen = pVisual->offsetGreen;
-	    PanoramiXVisuals[PanoramiXNumVisuals].offsetBlue = pVisual->offsetBlue;
-	    PanoramiXNumVisuals++;	
-
-	    for (j = 0; j < PanoramiXNumDepths; j++) {
-	        if (PanoramiXDepths[j].depth == pVisual->nplanes) {
-		    PanoramiXDepths[j].vids[PanoramiXDepths[j].numVids] = pVisual->vid;
-		    PanoramiXDepths[j].numVids++;
-		    break;
-		}	
-	    }   
-	}
-    } 
+	if (!found)
+	    return;
+    }
+
+    /* found a matching visual on all screens, add it to the subset list */
+    j = PanoramiXNumVisuals;
+    PanoramiXNumVisuals++;
+    PanoramiXVisuals = xrealloc(PanoramiXVisuals,
+	    PanoramiXNumVisuals * sizeof(VisualRec));
+
+    memcpy(&PanoramiXVisuals[j], pVisual, sizeof(VisualRec));
+
+    for (k = 0; k < PanoramiXNumDepths; k++) {
+	if (PanoramiXDepths[k].depth == pVisual->nplanes) {
+	    PanoramiXDepths[k].vids[PanoramiXDepths[k].numVids] = pVisual->vid;
+	    PanoramiXDepths[k].numVids++;
+	    break;
+	}	
+    }   
+}
+
+extern void
+PanoramiXConsolidate(void)
+{
+    int 	i;
+    PanoramiXRes *root, *defmap, *saver;
+    ScreenPtr   pScreen = screenInfo.screens[0];
+    DepthPtr    pDepth = pScreen->allowedDepths;
+    VisualPtr   pVisual = pScreen->visuals;
+
+    PanoramiXNumDepths = 0;
+    PanoramiXNumVisuals = 0;
 
+    for (i = 0; i < pScreen->numDepths; i++)
+	PanoramiXMaybeAddDepth(pDepth++);
 
-    root = (PanoramiXRes *) xalloc(sizeof(PanoramiXRes));
+    for (i = 0; i < pScreen->numVisuals; i++)
+	PanoramiXMaybeAddVisual(pVisual++);
+
+    root = xalloc(sizeof(PanoramiXRes));
     root->type = XRT_WINDOW;
-    defmap = (PanoramiXRes *) xalloc(sizeof(PanoramiXRes));
+    defmap = xalloc(sizeof(PanoramiXRes));
     defmap->type = XRT_COLORMAP;
-    saver = (PanoramiXRes *) xalloc(sizeof(PanoramiXRes));
+    saver = xalloc(sizeof(PanoramiXRes));
     saver->type = XRT_WINDOW;
 
-
     for (i =  0; i < PanoramiXNumScreens; i++) {
 	root->info[i].id = WindowTable[i]->drawable.id;
 	root->u.win.class = InputOutput;
@@ -854,6 +841,31 @@ void PanoramiXConsolidate(void)
     AddResource(defmap->info[0].id, XRT_COLORMAP, defmap);
 }
 
+_X_EXPORT VisualID
+PanoramiXTranslateVisualID(int screen, VisualID orig)
+{
+    VisualPtr pVisual = NULL;
+    int i, j;
+
+    for (i = 0; i < PanoramiXNumVisuals; i++) {
+	if (orig == PanoramiXVisuals[i].vid) {
+	    pVisual = &PanoramiXVisuals[i];
+	    break;
+	}
+    }
+
+    if (!pVisual)
+	return 0;
+
+    /* found the original, now translate it relative to the backend screen */
+    for (i = 0; i < PanoramiXNumScreens; i++)
+	for (j = 0; j < screenInfo.screens[i]->numVisuals; j++)
+	    if (VisualsEqual(pVisual, &screenInfo.screens[i]->visuals[j]))
+		return screenInfo.screens[i]->visuals[j].vid;
+    
+    return 0;
+}
+
 
 /*
  *	PanoramiXResetProc()
diff --git a/Xext/panoramiXprocs.c b/Xext/panoramiXprocs.c
index f9a579625..d19b3039a 100644
--- a/Xext/panoramiXprocs.c
+++ b/Xext/panoramiXprocs.c
@@ -150,7 +150,7 @@ int PanoramiXCreateWindow(ClientPtr client)
 	if (cmap)
 	    *((CARD32 *) &stuff[1] + cmap_offset) = cmap->info[j].id;
 	if ( orig_visual != CopyFromParent ) 
-	    stuff->visual = PanoramiXVisualTable[(orig_visual*MAXSCREENS) + j];
+	    stuff->visual = PanoramiXTranslateVisualID(j, orig_visual);
         result = (*SavedProcVector[X_CreateWindow])(client);
         if(result != Success) break;
     }
@@ -2077,9 +2077,6 @@ int PanoramiXCreateColormap(ClientPtr client)
 		client, stuff->window, XRT_WINDOW, DixReadAccess)))
 	return BadWindow;    
 
-    if(!stuff->visual || (stuff->visual > 255)) 
-	return BadValue;
-
     if(!(newCmap = (PanoramiXRes *) xalloc(sizeof(PanoramiXRes))))
         return BadAlloc;
 
@@ -2092,7 +2089,7 @@ int PanoramiXCreateColormap(ClientPtr client)
     FOR_NSCREENS_BACKWARD(j){
 	stuff->mid = newCmap->info[j].id;
 	stuff->window = win->info[j].id;
-	stuff->visual = PanoramiXVisualTable[(orig_visual * MAXSCREENS) + j];
+	stuff->visual = PanoramiXTranslateVisualID(j, orig_visual);
 	result = (* SavedProcVector[X_CreateColormap])(client);
 	if(result != Success) break;
     }
diff --git a/Xext/panoramiXsrv.h b/Xext/panoramiXsrv.h
index ae9024418..6d556e963 100644
--- a/Xext/panoramiXsrv.h
+++ b/Xext/panoramiXsrv.h
@@ -12,8 +12,8 @@ extern int PanoramiXNumScreens;
 extern PanoramiXData *panoramiXdataPtr;
 extern int PanoramiXPixWidth;
 extern int PanoramiXPixHeight;
-extern XID *PanoramiXVisualTable;
 
+extern VisualID PanoramiXTranslateVisualID(int screen, VisualID orig);
 extern void PanoramiXConsolidate(void);
 extern Bool PanoramiXCreateConnectionBlock(void);
 extern PanoramiXRes * PanoramiXFindIDByScrnum(RESTYPE, XID, int);
diff --git a/Xext/saver.c b/Xext/saver.c
index 9a6dd7757..feab972e2 100644
--- a/Xext/saver.c
+++ b/Xext/saver.c
@@ -1346,8 +1346,7 @@ ProcScreenSaverSetAttributes (ClientPtr client)
              *((CARD32 *) &stuff[1] + cmap_offset) = cmap->info[i].id;
 
           if (orig_visual != CopyFromParent) 
-            stuff->visualID = 
-                     PanoramiXVisualTable[(orig_visual*MAXSCREENS) + i];
+            stuff->visualID = PanoramiXTranslateVisualID(i, orig_visual);
 
           status = ScreenSaverSetAttributes(client);
        }
diff --git a/hw/dmx/glxProxy/glxcmds.c b/hw/dmx/glxProxy/glxcmds.c
index 6771cf1de..85e0f8701 100644
--- a/hw/dmx/glxProxy/glxcmds.c
+++ b/hw/dmx/glxProxy/glxcmds.c
@@ -66,7 +66,6 @@
 
 #ifdef PANORAMIX
 #include "panoramiXsrv.h"
-extern XID *PanoramiXVisualTable;
 #endif
 
 extern __GLXFBConfig **__glXFBConfigs;
@@ -2824,14 +2823,8 @@ int __glXGetFBConfigs(__GLXclientState *cl, GLbyte *pc)
 #ifdef PANORAMIX
 	   else if (!noPanoramiXExtension) {
 	      /* convert the associated visualId to the panoramix one */
-              for (v=0; v<255; v++) {
-		 if ( PanoramiXVisualTable[ v * MAXSCREENS + screen ] ==
-		      associatedVisualId ) {
-		    associatedVisualId = v;
-		    break;
-		 } 
-	      }
-	      pFBConfig->associatedVisualId = associatedVisualId;
+	      pFBConfig->associatedVisualId =
+		  PanoramiXTranslateVisualID(screen, v);
 	   }
 #endif
 	}
diff --git a/hw/xfree86/loader/extsym.c b/hw/xfree86/loader/extsym.c
index e09e9c084..1bdff9482 100644
--- a/hw/xfree86/loader/extsym.c
+++ b/hw/xfree86/loader/extsym.c
@@ -44,7 +44,6 @@ extern RESTYPE ShmSegType, ShmPixType;
 extern Bool noPanoramiXExtension;
 extern int PanoramiXNumScreens;
 extern PanoramiXData *panoramiXdataPtr;
-extern XID *PanoramiXVisualTable;
 extern unsigned long XRT_WINDOW;
 extern unsigned long XRT_PIXMAP;
 extern unsigned long XRT_GC;
@@ -69,7 +68,6 @@ _X_HIDDEN void *extLookupTab[] = {
     SYMFUNC(XineramaDeleteResource)
     SYMVAR(PanoramiXNumScreens)
     SYMVAR(panoramiXdataPtr)
-    SYMVAR(PanoramiXVisualTable)
     SYMVAR(XRT_WINDOW)
     SYMVAR(XRT_PIXMAP)
     SYMVAR(XRT_GC)