Add screen-specific privates.

Screen-specific privates areas are only allocated for objects related
to the target screen; objects allocated for other screens will not
have the private space reserved. This saves memory in these objects
while also allowing hot-plug screens to have additional private
allocation space beyond what the core screens are using.

Drivers are encouraged to switch to this mechanism as it will reduce
memory usage in multi-GPU environments, but it is only required for
drivers which will be loaded after the server starts, like
modesetting.

Objects providing screen-specific privates *must* be managed by the
screen-specific private API when allocating or initializing privates
so that the per-screen area can be initialized properly.

The objects which support screen-specific privates are:

	Windows
	Pixmaps
	GCs
	Pictures

Extending this list to include Colormaps would be possible, but
require slightly more work as the default colormap is created before
all colormap privates are allocated during server startup, and hence
gets a bunch of special treatment.

Of particular note, glyphs are *not* capable of supporting
screen-specific privates as they are global objects, not allocated on
a screen-specific basis, and so each driver must be able to see their
privates within the glyph.

Signed-off-by: Keith Packard <keithp@keithp.com>
Reviewed-by: Dave Airlie <airlied@redhat.com>

10 files changed, 400 insertions, 86 deletions

diff --git a/dix/dispatch.c b/dix/dispatch.c
index b88f9744d..7d2d3b727 100644
--- a/dix/dispatch.c
+++ b/dix/dispatch.c
@@ -3743,6 +3743,8 @@ AddScreen(Bool (*pfnInit) (ScreenPtr /*pScreen */ ,
     if (!pScreen)
         return -1;
 
+    dixInitScreenSpecificPrivates(pScreen);
+
     if (!dixAllocatePrivates(&pScreen->devPrivates, PRIVATE_SCREEN)) {
         free(pScreen);
         return -1;
@@ -3794,6 +3796,7 @@ AddScreen(Bool (*pfnInit) (ScreenPtr /*pScreen */ ,
     screenInfo.screens[i] = pScreen;
     screenInfo.numScreens++;
     if (!(*pfnInit) (pScreen, argc, argv)) {
+        dixFreeScreenSpecificPrivates(pScreen);
         dixFreePrivates(pScreen->devPrivates, PRIVATE_SCREEN);
         free(pScreen);
         screenInfo.numScreens--;
diff --git a/dix/gc.c b/dix/gc.c
index ac67643cf..60f54ec31 100644
--- a/dix/gc.c
+++ b/dix/gc.c
@@ -463,7 +463,7 @@ NewGCObject(ScreenPtr pScreen, int depth)
 {
     GCPtr pGC;
 
-    pGC = dixAllocateObjectWithPrivates(GC, PRIVATE_GC);
+    pGC = dixAllocateScreenObjectWithPrivates(pScreen, GC, PRIVATE_GC);
     if (!pGC) {
         return (GCPtr) NULL;
     }
diff --git a/dix/main.c b/dix/main.c
index e95ca1cc2..9524189d0 100644
--- a/dix/main.c
+++ b/dix/main.c
@@ -329,6 +329,7 @@ main(int argc, char *argv[], char *envp[])
             FreeScratchPixmapsForScreen(screenInfo.screens[i]);
             FreeGCperDepth(i);
             FreeDefaultStipple(i);
+            dixFreeScreenSpecificPrivates(screenInfo.screens[i]);
             (*screenInfo.screens[i]->CloseScreen) (screenInfo.screens[i]);
             dixFreePrivates(screenInfo.screens[i]->devPrivates, PRIVATE_SCREEN);
             free(screenInfo.screens[i]);
diff --git a/dix/pixmap.c b/dix/pixmap.c
index 545ff54ac..0c85c3fb1 100644
--- a/dix/pixmap.c
+++ b/dix/pixmap.c
@@ -88,7 +88,7 @@ CreateScratchPixmapsForScreen(ScreenPtr pScreen)
 {
     unsigned int pixmap_size;
 
-    pixmap_size = sizeof(PixmapRec) + dixPrivatesSize(PRIVATE_PIXMAP);
+    pixmap_size = sizeof(PixmapRec) + dixScreenSpecificPrivatesSize(pScreen, PRIVATE_PIXMAP);
     pScreen->totalPixmapSize =
         BitmapBytePad(pixmap_size * 8);
 
@@ -118,7 +118,7 @@ AllocatePixmap(ScreenPtr pScreen, int pixDataSize)
     if (!pPixmap)
         return NullPixmap;
 
-    dixInitPrivates(pPixmap, pPixmap + 1, PRIVATE_PIXMAP);
+    dixInitScreenPrivates(pScreen, pPixmap, pPixmap + 1, PRIVATE_PIXMAP);
     return pPixmap;
 }
 
diff --git a/dix/privates.c b/dix/privates.c
index 15fbf7590..e353108f8 100644
--- a/dix/privates.c
+++ b/dix/privates.c
@@ -63,12 +63,7 @@ from The Open Group.
 #include "scrnintstr.h"
 #include "extnsionst.h"
 
-static struct {
-    DevPrivateKey key;
-    unsigned offset;
-    int created;
-    int allocated;
-} keys[PRIVATE_LAST];
+static DevPrivateSetRec global_keys[PRIVATE_LAST];
 
 static const Bool xselinux_private[PRIVATE_LAST] = {
     [PRIVATE_SCREEN] = TRUE,
@@ -86,8 +81,57 @@ static const Bool xselinux_private[PRIVATE_LAST] = {
     [PRIVATE_GLYPHSET] = TRUE,
 };
 
+static const char *key_names[PRIVATE_LAST] = {
+    /* XSELinux uses the same private keys for numerous objects */
+    [PRIVATE_XSELINUX] = "XSELINUX",
+
+    /* Otherwise, you get a private in just the requested structure
+     */
+    /* These can have objects created before all of the keys are registered */
+    [PRIVATE_SCREEN] = "SCREEN",
+    [PRIVATE_EXTENSION] = "EXTENSION",
+    [PRIVATE_COLORMAP] = "COLORMAP",
+
+    /* These cannot have any objects before all relevant keys are registered */
+    [PRIVATE_DEVICE] = "DEVICE",
+    [PRIVATE_CLIENT] = "CLIENT",
+    [PRIVATE_PROPERTY] = "PROPERTY",
+    [PRIVATE_SELECTION] = "SELECTION",
+    [PRIVATE_WINDOW] = "WINDOW",
+    [PRIVATE_PIXMAP] = "PIXMAP",
+    [PRIVATE_GC] = "GC",
+    [PRIVATE_CURSOR] = "CURSOR",
+    [PRIVATE_CURSOR_BITS] = "CURSOR_BITS",
+
+    /* extension privates */
+    [PRIVATE_DBE_WINDOW] = "DBE_WINDOW",
+    [PRIVATE_DAMAGE] = "DAMAGE",
+    [PRIVATE_GLYPH] = "GLYPH",
+    [PRIVATE_GLYPHSET] = "GLYPHSET",
+    [PRIVATE_PICTURE] = "PICTURE",
+    [PRIVATE_SYNC_FENCE] = "SYNC_FENCE",
+};
+
+static const Bool screen_specific_private[PRIVATE_LAST] = {
+    [PRIVATE_SCREEN] = FALSE,
+    [PRIVATE_CLIENT] = FALSE,
+    [PRIVATE_WINDOW] = TRUE,
+    [PRIVATE_PIXMAP] = TRUE,
+    [PRIVATE_GC] = TRUE,
+    [PRIVATE_CURSOR] = FALSE,
+    [PRIVATE_COLORMAP] = FALSE,
+    [PRIVATE_DEVICE] = FALSE,
+    [PRIVATE_EXTENSION] = FALSE,
+    [PRIVATE_SELECTION] = FALSE,
+    [PRIVATE_PROPERTY] = FALSE,
+    [PRIVATE_PICTURE] = TRUE,
+    [PRIVATE_GLYPHSET] = FALSE,
+};
+
 typedef Bool (*FixupFunc) (PrivatePtr *privates, int offset, unsigned bytes);
 
+typedef enum { FixupMove, FixupRealloc } FixupType;
+
 static Bool
 dixReallocPrivates(PrivatePtr *privates, int old_offset, unsigned bytes)
 {
@@ -110,14 +154,72 @@ dixMovePrivates(PrivatePtr *privates, int new_offset, unsigned bytes)
 }
 
 static Bool
+fixupOneScreen(ScreenPtr pScreen, FixupFunc fixup, unsigned bytes)
+{
+    intptr_t        dist;
+    char            *old;
+    char            *new;
+    DevPrivateKey   *keyp, key;
+    DevPrivateType  type;
+    int             size;
+
+    old = (char *) pScreen->devPrivates;
+    size = global_keys[PRIVATE_SCREEN].offset;
+    if (!fixup (&pScreen->devPrivates, size, bytes))
+        return FALSE;
+
+    /* Screen privates can contain screen-specific private keys
+     * for other types. When they move, the linked list we use to
+     * track them gets scrambled. Fix that by computing the change
+     * in the location of each private adjusting our linked list
+     * pointers to match
+     */
+
+    new = (char *) pScreen->devPrivates;
+
+    /* Moving means everyone shifts up in the privates by 'bytes' amount,
+     * realloc means the base pointer moves
+     */
+    if (fixup == dixMovePrivates)
+        new += bytes;
+
+    dist = new - old;
+
+    if (dist) {
+        for (type = PRIVATE_XSELINUX; type < PRIVATE_LAST; type++)
+
+            /* Walk the privates list, being careful as the
+             * pointers are scrambled before we patch them.
+             */
+            for (keyp = &pScreen->screenSpecificPrivates[type].key;
+                 (key = *keyp) != NULL;
+                 keyp = &key->next)
+            {
+
+                /* Only mangle things if the private structure
+                 * is contained within the allocation. Privates
+                 * stored elsewhere will be left alone
+                 */
+                if (old <= (char *) key && (char *) key < old + size)
+                {
+                    /* Compute new location of key */
+                    key = (DevPrivateKey) ((char *) key + dist);
+
+                    /* Patch the list */
+                    *keyp = key;
+                }
+            }
+    }
+    return TRUE;
+}
+
+static Bool
 fixupScreens(FixupFunc fixup, unsigned bytes)
 {
     int s;
 
     for (s = 0; s < screenInfo.numScreens; s++)
-        if (!fixup
-            (&screenInfo.screens[s]->devPrivates, keys[PRIVATE_SCREEN].offset,
-             bytes))
+        if (!fixupOneScreen (screenInfo.screens[s], fixup, bytes))
             return FALSE;
     return TRUE;
 }
@@ -126,7 +228,7 @@ static Bool
 fixupServerClient(FixupFunc fixup, unsigned bytes)
 {
     if (serverClient)
-        return fixup(&serverClient->devPrivates, keys[PRIVATE_CLIENT].offset,
+        return fixup(&serverClient->devPrivates, global_keys[PRIVATE_CLIENT].offset,
                      bytes);
     return TRUE;
 }
@@ -140,7 +242,7 @@ fixupExtensions(FixupFunc fixup, unsigned bytes)
     for (major = EXTENSION_BASE; (extension = GetExtensionEntry(major));
          major++)
         if (!fixup
-            (&extension->devPrivates, keys[PRIVATE_EXTENSION].offset, bytes))
+            (&extension->devPrivates, global_keys[PRIVATE_EXTENSION].offset, bytes))
             return FALSE;
     return TRUE;
 }
@@ -157,17 +259,41 @@ fixupDefaultColormaps(FixupFunc fixup, unsigned bytes)
                                 screenInfo.screens[s]->defColormap, RT_COLORMAP,
                                 serverClient, DixCreateAccess);
         if (cmap &&
-            !fixup(&cmap->devPrivates, keys[PRIVATE_COLORMAP].offset, bytes))
+            !fixup(&cmap->devPrivates, screenInfo.screens[s]->screenSpecificPrivates[PRIVATE_COLORMAP].offset, bytes))
             return FALSE;
     }
     return TRUE;
 }
 
 static Bool (*const allocated_early[PRIVATE_LAST]) (FixupFunc, unsigned) = {
-[PRIVATE_SCREEN] = fixupScreens,
-        [PRIVATE_CLIENT] = fixupServerClient,
-        [PRIVATE_EXTENSION] = fixupExtensions,
-        [PRIVATE_COLORMAP] = fixupDefaultColormaps,};
+    [PRIVATE_SCREEN] = fixupScreens,
+    [PRIVATE_CLIENT] = fixupServerClient,
+    [PRIVATE_EXTENSION] = fixupExtensions,
+    [PRIVATE_COLORMAP] = fixupDefaultColormaps,
+};
+
+static void
+grow_private_set(DevPrivateSetPtr set, unsigned bytes)
+{
+    DevPrivateKey       k;
+
+    for (k = set->key; k; k = k->next)
+        k->offset += bytes;
+    set->offset += bytes;
+}
+
+static void
+grow_screen_specific_set(DevPrivateType type, unsigned bytes)
+{
+    int s;
+
+    /* Update offsets for all screen-specific keys */
+    for (s = 0; s < screenInfo.numScreens; s++) {
+        ScreenPtr       pScreen = screenInfo.screens[s];
+
+        grow_private_set(&pScreen->screenSpecificPrivates[type], bytes);
+    }
+}
 
 /*
  * Register a private key. This takes the type of object the key will
@@ -199,14 +325,13 @@ dixRegisterPrivateKey(DevPrivateKey key, DevPrivateType type, unsigned size)
 
     /* Update offsets for all affected keys */
     if (type == PRIVATE_XSELINUX) {
-        DevPrivateKey k;
 
         /* Resize if we can, or make sure nothing's allocated if we can't
          */
         for (t = PRIVATE_XSELINUX; t < PRIVATE_LAST; t++)
             if (xselinux_private[t]) {
                 if (!allocated_early[t])
-                    assert(!keys[t].created);
+                    assert(!global_keys[t].created);
                 else if (!allocated_early[t] (dixReallocPrivates, bytes))
                     return FALSE;
             }
@@ -216,12 +341,12 @@ dixRegisterPrivateKey(DevPrivateKey key, DevPrivateType type, unsigned size)
          */
         for (t = PRIVATE_XSELINUX; t < PRIVATE_LAST; t++) {
             if (xselinux_private[t]) {
-                for (k = keys[t].key; k; k = k->next)
-                    k->offset += bytes;
-                keys[t].offset += bytes;
+                grow_private_set(&global_keys[t], bytes);
+                grow_screen_specific_set(t, bytes);
                 if (allocated_early[t])
                     allocated_early[t] (dixMovePrivates, bytes);
             }
+
         }
 
         offset = 0;
@@ -229,11 +354,12 @@ dixRegisterPrivateKey(DevPrivateKey key, DevPrivateType type, unsigned size)
     else {
         /* Resize if we can, or make sure nothing's allocated if we can't */
         if (!allocated_early[type])
-            assert(!keys[type].created);
+            assert(!global_keys[type].created);
         else if (!allocated_early[type] (dixReallocPrivates, bytes))
             return FALSE;
-        offset = keys[type].offset;
-        keys[type].offset += bytes;
+        offset = global_keys[type].offset;
+        global_keys[type].offset += bytes;
+        grow_screen_specific_set(type, bytes);
     }
 
     /* Setup this key */
@@ -242,8 +368,8 @@ dixRegisterPrivateKey(DevPrivateKey key, DevPrivateType type, unsigned size)
     key->initialized = TRUE;
     key->type = type;
     key->allocated = FALSE;
-    key->next = keys[type].key;
-    keys[type].key = key;
+    key->next = global_keys[type].key;
+    global_keys[type].key = key;
 
     return TRUE;
 }
@@ -286,13 +412,15 @@ _dixGetScreenPrivateKey(const DevScreenPrivateKey key, ScreenPtr pScreen)
 void
 _dixInitPrivates(PrivatePtr *privates, void *addr, DevPrivateType type)
 {
-    keys[type].created++;
+    assert (!screen_specific_private[type]);
+
+    global_keys[type].created++;
     if (xselinux_private[type])
-        keys[PRIVATE_XSELINUX].created++;
-    if (keys[type].offset == 0)
+        global_keys[PRIVATE_XSELINUX].created++;
+    if (global_keys[type].offset == 0)
         addr = 0;
     *privates = addr;
-    memset(addr, '\0', keys[type].offset);
+    memset(addr, '\0', global_keys[type].offset);
 }
 
 /*
@@ -301,9 +429,9 @@ _dixInitPrivates(PrivatePtr *privates, void *addr, DevPrivateType type)
 void
 _dixFiniPrivates(PrivatePtr privates, DevPrivateType type)
 {
-    keys[type].created--;
+    global_keys[type].created--;
     if (xselinux_private[type])
-        keys[PRIVATE_XSELINUX].created--;
+        global_keys[PRIVATE_XSELINUX].created--;
 }
 
 /*
@@ -322,10 +450,11 @@ _dixAllocateObjectWithPrivates(unsigned baseSize, unsigned clear,
     PrivatePtr *devPrivates;
 
     assert(type > PRIVATE_SCREEN && type < PRIVATE_LAST);
+    assert(!screen_specific_private[type]);
 
     /* round up so that void * is aligned */
     baseSize = (baseSize + sizeof(void *) - 1) & ~(sizeof(void *) - 1);
-    totalSize = baseSize + keys[type].offset;
+    totalSize = baseSize + global_keys[type].offset;
     object = malloc(totalSize);
     if (!object)
         return NULL;
@@ -350,8 +479,9 @@ dixAllocatePrivates(PrivatePtr *privates, DevPrivateType type)
     PrivatePtr p;
 
     assert(type > PRIVATE_XSELINUX && type < PRIVATE_LAST);
+    assert(!screen_specific_private[type]);
 
-    size = keys[type].offset;
+    size = global_keys[type].offset;
     if (!size) {
         p = NULL;
     }
@@ -361,7 +491,7 @@ dixAllocatePrivates(PrivatePtr *privates, DevPrivateType type)
     }
 
     _dixInitPrivates(privates, p, type);
-    ++keys[type].allocated;
+    ++global_keys[type].allocated;
 
     return TRUE;
 }
@@ -387,7 +517,7 @@ void
 dixFreePrivates(PrivatePtr privates, DevPrivateType type)
 {
     _dixFiniPrivates(privates, type);
-    --keys[type].allocated;
+    --global_keys[type].allocated;
     free(privates);
 }
 
@@ -398,8 +528,9 @@ extern _X_EXPORT int
 dixPrivatesSize(DevPrivateType type)
 {
     assert(type >= PRIVATE_SCREEN && type < PRIVATE_LAST);
+    assert (!screen_specific_private[type]);
 
-    return keys[type].offset;
+    return global_keys[type].offset;
 }
 
 /* Table of devPrivates offsets */
@@ -434,36 +565,136 @@ dixLookupPrivateOffset(RESTYPE type)
     return -1;
 }
 
-static const char *key_names[PRIVATE_LAST] = {
-    /* XSELinux uses the same private keys for numerous objects */
-    [PRIVATE_XSELINUX] = "XSELINUX",
+/*
+ * Screen-specific privates
+ */
 
-    /* Otherwise, you get a private in just the requested structure
-     */
-    /* These can have objects created before all of the keys are registered */
-    [PRIVATE_SCREEN] = "SCREEN",
-    [PRIVATE_EXTENSION] = "EXTENSION",
-    [PRIVATE_COLORMAP] = "COLORMAP",
+extern _X_EXPORT Bool
+dixRegisterScreenSpecificPrivateKey(ScreenPtr pScreen, DevPrivateKey key,
+                                    DevPrivateType type, unsigned size)
+{
+    int offset;
+    unsigned bytes;
 
-    /* These cannot have any objects before all relevant keys are registered */
-    [PRIVATE_DEVICE] = "DEVICE",
-    [PRIVATE_CLIENT] = "CLIENT",
-    [PRIVATE_PROPERTY] = "PROPERTY",
-    [PRIVATE_SELECTION] = "SELECTION",
-    [PRIVATE_WINDOW] = "WINDOW",
-    [PRIVATE_PIXMAP] = "PIXMAP",
-    [PRIVATE_GC] = "GC",
-    [PRIVATE_CURSOR] = "CURSOR",
-    [PRIVATE_CURSOR_BITS] = "CURSOR_BITS",
+    if (!screen_specific_private[type])
+        FatalError("Attempt to allocate screen-specific private storage for type %s\n",
+                   key_names[type]);
 
-    /* extension privates */
-    [PRIVATE_DBE_WINDOW] = "DBE_WINDOW",
-    [PRIVATE_DAMAGE] = "DAMAGE",
-    [PRIVATE_GLYPH] = "GLYPH",
-    [PRIVATE_GLYPHSET] = "GLYPHSET",
-    [PRIVATE_PICTURE] = "PICTURE",
-    [PRIVATE_SYNC_FENCE] = "SYNC_FENCE",
-};
+    if (key->initialized) {
+        assert(size == key->size);
+        return TRUE;
+    }
+
+    /* Compute required space */
+    bytes = size;
+    if (size == 0)
+        bytes = sizeof(void *);
+
+    /* align to void * size */
+    bytes = (bytes + sizeof(void *) - 1) & ~(sizeof(void *) - 1);
+
+    assert (!allocated_early[type]);
+    assert (!pScreen->screenSpecificPrivates[type].created);
+    offset = pScreen->screenSpecificPrivates[type].offset;
+    pScreen->screenSpecificPrivates[type].offset += bytes;
+
+    /* Setup this key */
+    key->offset = offset;
+    key->size = size;
+    key->initialized = TRUE;
+    key->type = type;
+    key->allocated = FALSE;
+    key->next = pScreen->screenSpecificPrivates[type].key;
+    pScreen->screenSpecificPrivates[type].key = key;
+
+    return TRUE;
+}
+
+/* Clean up screen-specific privates before CloseScreen */
+void
+dixFreeScreenSpecificPrivates(ScreenPtr pScreen)
+{
+}
+
+/* Initialize screen-specific privates in AddScreen */
+void
+dixInitScreenSpecificPrivates(ScreenPtr pScreen)
+{
+    DevPrivateType      t;
+
+    for (t = PRIVATE_XSELINUX; t < PRIVATE_LAST; t++)
+        pScreen->screenSpecificPrivates[t].offset = global_keys[t].offset;
+}
+
+/* Initialize screen-specific privates in AddScreen */
+void
+_dixInitScreenPrivates(ScreenPtr pScreen, PrivatePtr *privates, void *addr, DevPrivateType type)
+{
+    int privates_size;
+    assert (screen_specific_private[type]);
+
+    if (pScreen) {
+        privates_size = pScreen->screenSpecificPrivates[type].offset;
+        pScreen->screenSpecificPrivates[type].created++;
+    }
+    else
+        privates_size = global_keys[type].offset;
+
+    global_keys[type].created++;
+    if (xselinux_private[type])
+        global_keys[PRIVATE_XSELINUX].created++;
+    if (privates_size == 0)
+        addr = 0;
+    *privates = addr;
+    memset(addr, '\0', privates_size);
+}
+
+void *
+_dixAllocateScreenObjectWithPrivates(ScreenPtr pScreen,
+                                     unsigned baseSize,
+                                     unsigned clear,
+                                     unsigned offset,
+                                     DevPrivateType type)
+{
+    unsigned totalSize;
+    void *object;
+    PrivatePtr privates;
+    PrivatePtr *devPrivates;
+    int privates_size;
+
+    assert(type > PRIVATE_SCREEN && type < PRIVATE_LAST);
+    assert (screen_specific_private[type]);
+
+    if (pScreen)
+        privates_size = pScreen->screenSpecificPrivates[type].offset;
+    else
+        privates_size = global_keys[type].offset;
+    /* round up so that void * is aligned */
+    baseSize = (baseSize + sizeof(void *) - 1) & ~(sizeof(void *) - 1);
+    totalSize = baseSize + privates_size;
+    object = malloc(totalSize);
+    if (!object)
+        return NULL;
+
+    memset(object, '\0', clear);
+    privates = (PrivatePtr) (((char *) object) + baseSize);
+    devPrivates = (PrivatePtr *) ((char *) object + offset);
+
+    _dixInitScreenPrivates(pScreen, devPrivates, privates, type);
+
+    return object;
+}
+
+int
+dixScreenSpecificPrivatesSize(ScreenPtr pScreen, DevPrivateType type)
+{
+    assert(type >= PRIVATE_SCREEN && type < PRIVATE_LAST);
+
+    if (screen_specific_private[type])
+        return pScreen->screenSpecificPrivates[type].offset;
+    else
+        return global_keys[type].offset;
+}
 
 void
 dixPrivateUsage(void)
@@ -474,14 +705,14 @@ dixPrivateUsage(void)
     DevPrivateType t;
 
     for (t = PRIVATE_XSELINUX + 1; t < PRIVATE_LAST; t++) {
-        if (keys[t].offset) {
+        if (global_keys[t].offset) {
             ErrorF
                 ("%s: %d objects of %d bytes = %d total bytes %d private allocs\n",
-                 key_names[t], keys[t].created, keys[t].offset,
-                 keys[t].created * keys[t].offset, keys[t].allocated);
-            bytes += keys[t].created * keys[t].offset;
-            objects += keys[t].created;
-            alloc += keys[t].allocated;
+                 key_names[t], global_keys[t].created, global_keys[t].offset,
+                 global_keys[t].created * global_keys[t].offset, global_keys[t].allocated);
+            bytes += global_keys[t].created * global_keys[t].offset;
+            objects += global_keys[t].created;
+            alloc += global_keys[t].allocated;
         }
     }
     ErrorF("TOTAL: %d objects, %d bytes, %d allocs\n", objects, bytes, alloc);
@@ -495,7 +726,7 @@ dixResetPrivates(void)
     for (t = PRIVATE_XSELINUX; t < PRIVATE_LAST; t++) {
         DevPrivateKey key, next;
 
-        for (key = keys[t].key; key; key = next) {
+        for (key = global_keys[t].key; key; key = next) {
             next = key->next;
             key->offset = 0;
             key->initialized = FALSE;
@@ -504,14 +735,14 @@ dixResetPrivates(void)
             if (key->allocated)
                 free(key);
         }
-        if (keys[t].created) {
+        if (global_keys[t].created) {
             ErrorF("%d %ss still allocated at reset\n",
-                   keys[t].created, key_names[t]);
+                   global_keys[t].created, key_names[t]);
             dixPrivateUsage();
         }
-        keys[t].key = NULL;
-        keys[t].offset = 0;
-        keys[t].created = 0;
-        keys[t].allocated = 0;
+        global_keys[t].key = NULL;
+        global_keys[t].offset = 0;
+        global_keys[t].created = 0;
+        global_keys[t].allocated = 0;
     }
 }
diff --git a/dix/window.c b/dix/window.c
index 5cc3a502d..b66080830 100644
--- a/dix/window.c
+++ b/dix/window.c
@@ -446,7 +446,7 @@ CreateRootWindow(ScreenPtr pScreen)
     BoxRec box;
     PixmapFormatRec *format;
 
-    pWin = dixAllocateObjectWithPrivates(WindowRec, PRIVATE_WINDOW);
+    pWin = dixAllocateScreenObjectWithPrivates(pScreen, WindowRec, PRIVATE_WINDOW);
     if (!pWin)
         return FALSE;
 
@@ -710,7 +710,7 @@ CreateWindow(Window wid, WindowPtr pParent, int x, int y, unsigned w,
         return NullWindow;
     }
 
-    pWin = dixAllocateObjectWithPrivates(WindowRec, PRIVATE_WINDOW);
+    pWin = dixAllocateScreenObjectWithPrivates(pScreen, WindowRec, PRIVATE_WINDOW);
     if (!pWin) {
         *error = BadAlloc;
         return NullWindow;
diff --git a/doc/Xserver-spec.xml b/doc/Xserver-spec.xml
index 31b6fb05d..cd1a9d07a 100644
--- a/doc/Xserver-spec.xml
+++ b/doc/Xserver-spec.xml
@@ -109,6 +109,12 @@
 	<revremark>Revised for Xorg 1.9 devPrivates changes
 	  and 1.8 CreateNewResourceType changes</revremark>
       </revision>
+      <revision>
+	<revnumber>3.6</revnumber>
+	<date>July 2012</date>
+	<authorinitials>kp</authorinitials>
+	<revremark>Revised for X server 1.13 screen-specific devPrivates changes</revremark>
+      </revision>
     </revhistory>
     <abstract>
       <para>The following document explains the structure of the X Window System display server and the interfaces among the larger pieces.  It is intended as a reference for programmers who are implementing an X Display Server on their workstation hardware.  It is included with the X Window System source tape, along with the document "Strategies for Porting the X v11 Sample Server."  The order in which you should read these documents is:
@@ -4714,7 +4720,8 @@ Two new extensibility concepts have been developed for release 4, Wrappers
 and devPrivates.  These replace the R3 GCInterest queues, which were not a
 general enough mechanism for many extensions and only provided hooks into a
 single data structure.  devPrivates have been revised substantially for
-X.Org X server release 1.5, and updated again for the 1.9 release.</para>
+X.Org X server release 1.5, updated again for the 1.9 release and extended
+again for the 1.13 relealse.</para>
 <section>
   <title>devPrivates</title>
 <para>
@@ -4758,6 +4765,23 @@ the specified type with distinct storage for the given
 that are otherwise equivalent to the following Private functions.</para>
 
 <para>
+  To request private space in objects created for a specific screen, use
+  <blockquote><programlisting>
+    Bool dixRegisterScreenSpecificPrivateKey(ScreenPtr pScreen, DevPrivateKey key, DevPrivateType type, unsigned size);
+  </programlisting></blockquote>
+  The <parameter>type</parameter> and <parameter>size</parameter> arguments are
+  the same as those to <function>dixRegisterPrivateKey</function> but this
+  function ensures only that the given <parameter>key</parameter> exists on objects of
+  the specified type that are allocated with reference to the specified
+  <parameter>pScreen</parameter>. Using the key on objects allocated for
+  other screens will result in incorrect results; there is no check made to
+  ensure that the caller's screen matches the private's screen. The key is
+  usable in any of the following functions. Screen-specific private storage is available
+  only for Windows, GCs, Pixmaps and Pictures. Attempts to allocate screen-specific
+  privates on other objects will result in a call to FatalError.
+</para>
+
+<para>
 To attach a piece of private data to an object, use:
 <blockquote><programlisting>
 	void dixSetPrivate(PrivateRec **privates, const DevPrivateKey key, pointer val)
diff --git a/include/privates.h b/include/privates.h
index c34b9512c..a0874f697 100644
--- a/include/privates.h
+++ b/include/privates.h
@@ -66,6 +66,13 @@ typedef struct _DevPrivateKeyRec {
     struct _DevPrivateKeyRec *next;
 } DevPrivateKeyRec, *DevPrivateKey;
 
+typedef struct _DevPrivateSetRec {
+    DevPrivateKey key;
+    unsigned offset;
+    int created;
+    int allocated;
+} DevPrivateSetRec, *DevPrivateSetPtr;
+
 typedef struct _DevScreenPrivateKeyRec {
     DevPrivateKeyRec screenKey;
 } DevScreenPrivateKeyRec, *DevScreenPrivateKey;
@@ -219,6 +226,51 @@ dixLookupScreenPrivateAddr(PrivatePtr *privates, const DevScreenPrivateKey key,
 }
 
 /*
+ * These functions relate to allocations related to a specific screen;
+ * space will only be available for objects allocated for use on that
+ * screen. As such, only objects which are related directly to a specific
+ * screen are candidates for allocation this way, this includes
+ * windows, pixmaps, gcs, pictures and colormaps. This key is
+ * used just like any other key using dixGetPrivate and friends.
+ *
+ * This is distinctly different from the ScreenPrivateKeys above which
+ * allocate space in global objects like cursor bits for a specific
+ * screen, allowing multiple screen-related chunks of storage in a
+ * single global object.
+ */
+
+#define HAVE_SCREEN_SPECIFIC_PRIVATE_KEYS       1
+
+extern _X_EXPORT Bool
+dixRegisterScreenSpecificPrivateKey(ScreenPtr pScreen, DevPrivateKey key,
+                                    DevPrivateType type, unsigned size);
+
+/* Clean up screen-specific privates before CloseScreen */
+extern void
+dixFreeScreenSpecificPrivates(ScreenPtr pScreen);
+
+/* Initialize screen-specific privates in AddScreen */
+extern void
+dixInitScreenSpecificPrivates(ScreenPtr pScreen);
+
+extern _X_EXPORT void *
+_dixAllocateScreenObjectWithPrivates(ScreenPtr pScreen,
+                                     unsigned size,
+                                     unsigned clear,
+                                     unsigned offset,
+                                     DevPrivateType type);
+
+#define dixAllocateScreenObjectWithPrivates(s, t, type) _dixAllocateScreenObjectWithPrivates(s, sizeof(t), sizeof(t), offsetof(t, devPrivates), type)
+
+extern _X_EXPORT int
+dixScreenSpecificPrivatesSize(ScreenPtr pScreen, DevPrivateType type);
+
+extern _X_EXPORT void
+_dixInitScreenPrivates(ScreenPtr pScreen, PrivatePtr *privates, void *addr, DevPrivateType type);
+
+#define dixInitScreenPrivates(s, o, v, type) _dixInitScreenPrivates(s, &(o)->devPrivates, (v), type);
+
+/*
  * Allocates private data separately from main object.
  *
  * For objects created during server initialization, this allows those
@@ -240,7 +292,7 @@ extern _X_EXPORT void
  * Initialize privates by zeroing them
  */
 extern _X_EXPORT void
- _dixInitPrivates(PrivatePtr *privates, void *addr, DevPrivateType type);
+_dixInitPrivates(PrivatePtr *privates, void *addr, DevPrivateType type);
 
 #define dixInitPrivates(o, v, type) _dixInitPrivates(&(o)->devPrivates, (v), type);
 
diff --git a/include/scrnintstr.h b/include/scrnintstr.h
index c592d1ffe..7af2bf53f 100644
--- a/include/scrnintstr.h
+++ b/include/scrnintstr.h
@@ -367,6 +367,8 @@ typedef struct _Screen {
     WindowPtr root;
     ScreenSaverStuffRec screensaver;
 
+    DevPrivateSetRec    screenSpecificPrivates[PRIVATE_LAST];
+
     /* Random screen procedures */
 
     CloseScreenProcPtr CloseScreen;
diff --git a/render/picture.c b/render/picture.c
index ebbfa29e9..2908b7629 100644
--- a/render/picture.c
+++ b/render/picture.c
@@ -763,7 +763,8 @@ CreatePicture(Picture pid,
     PicturePtr pPicture;
     PictureScreenPtr ps = GetPictureScreen(pDrawable->pScreen);
 
-    pPicture = dixAllocateObjectWithPrivates(PictureRec, PRIVATE_PICTURE);
+    pPicture = dixAllocateScreenObjectWithPrivates(pDrawable->pScreen,
+                                                   PictureRec, PRIVATE_PICTURE);
     if (!pPicture) {
         *error = BadAlloc;
         return 0;
@@ -853,7 +854,7 @@ createSourcePicture(void)
 {
     PicturePtr pPicture;
 
-    pPicture = dixAllocateObjectWithPrivates(PictureRec, PRIVATE_PICTURE);
+    pPicture = dixAllocateScreenObjectWithPrivates(NULL, PictureRec, PRIVATE_PICTURE);
     pPicture->pDrawable = 0;
     pPicture->pFormat = 0;
     pPicture->pNext = 0;