When TTM is available, use it instead of manual AGP allocations when possible.

This is a step towards being able to expose buffer objects through the screen
private to DRI clients, instead of having them have to use the fake buffer
object type.

This fails in two ways.  First, the kernel memory manager is not currently
suitable for doing the physical allocations we need, so we still use AGP for
those.  Additionally, the DRI lock can't be initialized early enough for us, so
these buffer object allocations fail.  This will be fixed by improving the
DRM interface.

6 files changed, 283 insertions, 257 deletions

diff --git a/man/intel.man b/man/intel.man
index 33dc319e..c682d550 100644
--- a/man/intel.man
+++ b/man/intel.man
@@ -128,9 +128,8 @@ Default: XVideo is enabled for configurations where it is supported.
 Enable support for the legacy i915_dri.so 3D driver.
 This will, among other things, make the 2D driver tell libGL to
 load the 3D driver i915_dri.so instead of the newer i915tex_dri.so.
-This option is only used for chipsets in the range i830-i945. 
-Default for i830-i945 series: Enabled for i915 drm versions < 1.7.0. Otherwise
-disabled. 
+This option is only used for chipsets in the range i830-i945.
+Default for i830-i945 series: Enabled.
 Default for i810: The option is not used.
 Default for i965: The option is always true.
 .TP
diff --git a/src/i830.h b/src/i830.h
index aa2b240c..6e1f88a2 100644
--- a/src/i830.h
+++ b/src/i830.h
@@ -158,6 +158,9 @@ struct _i830_memory {
     i830_memory *prev;
     /** @} */
 
+#ifdef XF86DRI_MM
+    drmBO bo;
+#endif
 };
 
 typedef struct {
@@ -328,8 +331,7 @@ typedef struct _I830Rec {
 
    int TexGranularity;
    int drmMinor;
-   int mmModeFlags;
-   int mmSize;
+   Bool allocate_classic_textures;
 
    unsigned int back_tiled;
    unsigned int third_tiled;
@@ -635,6 +637,7 @@ extern void I830SubsequentSolidFillRect(ScrnInfoPtr pScrn, int x, int y,
 
 Bool i830_allocator_init(ScrnInfoPtr pScrn, unsigned long offset,
 			 unsigned long size);
+void i830_allocator_fini(ScrnInfoPtr pScrn);
 void i830_describe_allocations(ScrnInfoPtr pScrn, int verbosity,
 			       const char *prefix);
 void i830_reset_allocations(ScrnInfoPtr pScrn);
@@ -744,6 +747,7 @@ extern const int I830CopyROP[16];
 /* Flags for memory allocation function */
 #define NEED_PHYSICAL_ADDR		0x00000001
 #define ALIGN_BOTH_ENDS			0x00000002
+#define NEED_NON_STOLEN			0x00000004
 
 /* Chipset registers for VIDEO BIOS memory RW access */
 #define _855_DRAM_RW_CONTROL 0x58
diff --git a/src/i830_accel.c b/src/i830_accel.c
index 5cbad44e..4d9ea790 100644
--- a/src/i830_accel.c
+++ b/src/i830_accel.c
@@ -254,6 +254,12 @@ I830RefreshRing(ScrnInfoPtr pScrn)
 {
    I830Ptr pI830 = I830PTR(pScrn);
 
+   /* If we're reaching RefreshRing as a result of grabbing the DRI lock
+    * before we've set up the ringbuffer, don't bother.
+    */
+   if (pI830->LpRing->mem == NULL)
+       return;
+
    pI830->LpRing->head = INREG(LP_RING + RING_HEAD) & I830_HEAD_MASK;
    pI830->LpRing->tail = INREG(LP_RING + RING_TAIL);
    pI830->LpRing->space = pI830->LpRing->head - (pI830->LpRing->tail + 8);
diff --git a/src/i830_dri.c b/src/i830_dri.c
index ca1190cf..0d511d22 100644
--- a/src/i830_dri.c
+++ b/src/i830_dri.c
@@ -546,16 +546,12 @@ I830DRIScreenInit(ScreenPtr pScreen)
    pDRIInfo->ddxDriverMajorVersion = I830_MAJOR_VERSION;
    pDRIInfo->ddxDriverMinorVersion = I830_MINOR_VERSION;
    pDRIInfo->ddxDriverPatchVersion = I830_PATCHLEVEL;
-#if 1 /* Remove this soon - see bug 5714 */
-   pDRIInfo->frameBufferPhysicalAddress = (char *) pI830->LinearAddr +
-					  pI830->front_buffer->offset;
-   pDRIInfo->frameBufferSize = ROUND_TO_PAGE(pScrn->displayWidth *
-					     pScrn->virtualY * pI830->cpp);
-#else
-   /* For rotation we map a 0 length framebuffer as we remap ourselves later */
-   pDRIInfo->frameBufferSize = 0;
-#endif
-   pDRIInfo->frameBufferStride = pScrn->displayWidth * pI830->cpp;
+   /* Supply a dummy mapping info required by DRI setup.  See bug #5714 for
+    * progress on removing this requirement.
+    */
+   pDRIInfo->frameBufferPhysicalAddress = (char *) pI830->LinearAddr;
+   pDRIInfo->frameBufferSize = GTT_PAGE_SIZE;
+   pDRIInfo->frameBufferStride = 1;
    pDRIInfo->ddxDrawableTableEntry = I830_MAX_DRAWABLES;
 
    if (SAREA_MAX_DRAWABLES < I830_MAX_DRAWABLES)
@@ -637,11 +633,11 @@ I830DRIScreenInit(ScreenPtr pScreen)
       return FALSE;
    }
 
-#if 0 /* disabled now, see frameBufferSize above being set to 0 */
-   /* for this driver, get rid of the front buffer mapping now */
+   /* Now, nuke dri.c's frontbuffer map setup. */
+#if 0
    if (xf86LoaderCheckSymbol("DRIGetScreenPrivate")) {
-      DRIScreenPrivPtr pDRIPriv 
-         = (DRIScreenPrivPtr) DRIGetScreenPrivate(pScreen);
+      DRIScreenPrivPtr pDRIPriv =
+	  (DRIScreenPrivPtr) DRIGetScreenPrivate(pScreen);
 
       if (pDRIPriv && pDRIPriv->drmFD && pDRIPriv->hFrameBuffer) {
          xf86DrvMsg(pScreen->myNum, X_ERROR,
@@ -652,11 +648,25 @@ I830DRIScreenInit(ScreenPtr pScreen)
                     "[intel] done removing original screen mapping\n");
       }
    }
-   else {
-      xf86DrvMsg(pScreen->myNum, X_ERROR,
-                 "[intel] DRIGetScreenPrivate not found!!!!\n");
-   }      
 #endif
+   {
+       int tmp;
+       unsigned int fb_handle;
+       void *ptmp;
+
+       /* With the compat method, it will continue to report
+	* the wrong map out of GetDeviceInfo, but we don't have any consumers
+	* of the frontbuffer handle from there.
+	*/
+       DRIGetDeviceInfo(pScreen, &fb_handle, &tmp, &tmp, &tmp, &tmp, &ptmp);
+       drmRmMap(pI830->drmSubFD, fb_handle);
+
+       xf86DrvMsg(pScrn->scrnIndex, X_WARNING,
+		  "Removed DRI frontbuffer mapping in compatibility mode.\n");
+       xf86DrvMsg(pScrn->scrnIndex, X_WARNING,
+		  "DRIGetDeviceInfo will report incorrect frontbuffer "
+		  "handle.\n");
+   }
 
    /* Check the i915 DRM versioning */
    {
@@ -724,27 +734,6 @@ I830DRIScreenInit(ScreenPtr pScreen)
 	    return FALSE;
 	 }
 	 pI830->drmMinor = version->version_minor;
-	 if (version->version_minor < 7) {
-	    if (pI830->mmModeFlags & I830_KERNEL_MM) {
-	       xf86DrvMsg(pScrn->scrnIndex, X_INFO,
-			  "Unable to use TTM-based memory manager with DRM version %d.%d\n",
-			  version->version_major, version->version_minor);
-	       pI830->mmModeFlags &= ~I830_KERNEL_MM;
-
-	       i830_free_memory(pScrn, pI830->memory_manager);
-	       pI830->memory_manager = NULL;
-
-	       if (!(pI830->mmModeFlags & I830_KERNEL_TEX)) {
-		  pI830->mmModeFlags |= I830_KERNEL_TEX;
-
-		  if (!i830_allocate_texture_memory(pScrn)) {
-		     I830DRICloseScreen(pScreen);
-		     drmFreeVersion(version);
-		     return FALSE;
-		  }
-	       }
-	    }
-	 }
 #ifdef DAMAGE
 	 if (pI830->allowPageFlip && pI830->drmMinor < 9) {
 	    xf86DrvMsg(pScrn->scrnIndex, X_WARNING,
@@ -762,7 +751,7 @@ I830DRIScreenInit(ScreenPtr pScreen)
     */
 
    if ((pDRIInfo->clientDriverName == I830ClientDriverName) && 
-       (pI830->mmModeFlags & I830_KERNEL_TEX)) {
+       (pI830->allocate_classic_textures)) {
       pDRIInfo->clientDriverName = I830LegacyClientDriverName;
    }
 
@@ -780,26 +769,21 @@ I830DRIMapScreenRegions(ScrnInfoPtr pScrn, drmI830Sarea *sarea)
 					     pScrn->virtualY * pI830->cpp);
 #endif
 
-   /* The I965G isn't ready for the front buffer mapping to be moved around,
-    * because of issues with rmmap, it seems.
-    */
-   if (!IS_I965G(pI830)) {
-      xf86DrvMsg(pScrn->scrnIndex, X_INFO,
-		 "[drm] Mapping front buffer\n");
-      if (drmAddMap(pI830->drmSubFD,
-		    (drm_handle_t)(sarea->front_offset + pI830->LinearAddr),
-		    sarea->front_size,
-		    DRM_AGP,
-		    0,
-		    (drmAddress) &sarea->front_handle) < 0) {
-	 xf86DrvMsg(pScrn->scrnIndex, X_ERROR,
-		    "[drm] drmAddMap(front_handle) failed. Disabling DRI\n");
-	 DRICloseScreen(pScreen);
-	 return FALSE;
-      }
-      xf86DrvMsg(pScrn->scrnIndex, X_INFO, "[drm] Front Buffer = 0x%08x\n",
-		 (int)sarea->front_handle);
+   xf86DrvMsg(pScrn->scrnIndex, X_INFO,
+	      "[drm] Mapping front buffer\n");
+   if (drmAddMap(pI830->drmSubFD,
+		 (drm_handle_t)(sarea->front_offset + pI830->LinearAddr),
+		 sarea->front_size,
+		 DRM_AGP,
+		 0,
+		 (drmAddress) &sarea->front_handle) < 0) {
+       xf86DrvMsg(pScrn->scrnIndex, X_ERROR,
+		  "[drm] drmAddMap(front_handle) failed. Disabling DRI\n");
+       DRICloseScreen(pScreen);
+       return FALSE;
    }
+   xf86DrvMsg(pScrn->scrnIndex, X_INFO, "[drm] Front Buffer = 0x%08x\n",
+	      (int)sarea->front_handle);
 
    if (drmAddMap(pI830->drmSubFD,
                  (drm_handle_t)(sarea->back_offset + pI830->LinearAddr),
@@ -841,7 +825,7 @@ I830DRIMapScreenRegions(ScrnInfoPtr pScrn, drmI830Sarea *sarea)
    xf86DrvMsg(pScrn->scrnIndex, X_INFO, "[drm] Depth Buffer = 0x%08x\n",
               (int)sarea->depth_handle);
 
-   if (pI830->mmModeFlags & I830_KERNEL_TEX) {
+   if (pI830->allocate_classic_textures) {
       if (drmAddMap(pI830->drmSubFD,
 		    (drm_handle_t)sarea->tex_offset + pI830->LinearAddr,
 		    sarea->tex_size, DRM_AGP, 0,
@@ -1710,7 +1694,7 @@ I830UpdateDRIBuffers(ScrnInfoPtr pScrn, drmI830Sarea *sarea)
 
    success = I830DRIMapScreenRegions(pScrn, sarea);
 
-   if (success && (pI830->mmModeFlags & I830_KERNEL_TEX))
+   if (success && pI830->allocate_classic_textures)
       I830InitTextureHeap(pScrn, sarea);
 
    return success;
diff --git a/src/i830_driver.c b/src/i830_driver.c
index 9bb12c6b..20c6c682 100644
--- a/src/i830_driver.c
+++ b/src/i830_driver.c
@@ -290,7 +290,6 @@ typedef enum {
    OPTION_TILING,
 #ifdef XF86DRI_MM
    OPTION_INTELTEXPOOL,
-   OPTION_INTELMMSIZE,
 #endif
    OPTION_TRIPLEBUFFER,
 } I830Opts;
@@ -313,7 +312,6 @@ static OptionInfoRec I830Options[] = {
    {OPTION_TILING,	"Tiling",	OPTV_BOOLEAN,	{0},	TRUE},
 #ifdef XF86DRI_MM
    {OPTION_INTELTEXPOOL,"Legacy3D",     OPTV_BOOLEAN,	{0},	FALSE},
-   {OPTION_INTELMMSIZE, "AperTexSize",  OPTV_INTEGER,	{0},	FALSE},
 #endif
    {OPTION_TRIPLEBUFFER, "TripleBuffer", OPTV_BOOLEAN,	{0},	FALSE},
    {-1,			NULL,		OPTV_NONE,	{0},	FALSE}
@@ -1319,50 +1317,22 @@ I830PreInit(ScrnInfoPtr pScrn, int flags)
 	 pI830->directRenderingDisabled = TRUE;
       }
 
-      pI830->mmModeFlags = 0;
-
       if (!pI830->directRenderingDisabled) {
-	 pI830->mmModeFlags = I830_KERNEL_TEX;
-#ifdef XF86DRI_MM
-	 Bool tmp = FALSE;
-
-	 if (!IS_I965G(pI830))
-	    pI830->mmModeFlags |= I830_KERNEL_MM;
-#endif
+	 pI830->allocate_classic_textures = TRUE;
 
 	 from = X_PROBED;
 
 #ifdef XF86DRI_MM
-	 if (xf86GetOptValBool(pI830->Options, 
-			       OPTION_INTELTEXPOOL, &tmp)) {
-	    from = X_CONFIG;
-	    if (tmp) {
-	       pI830->mmModeFlags |= I830_KERNEL_TEX;
-	       pI830->mmModeFlags &= ~I830_KERNEL_MM;
-	    } else {
-	       pI830->mmModeFlags &= ~I830_KERNEL_TEX;
-	       pI830->mmModeFlags |= I830_KERNEL_MM;
-	    }	       
-	 }
-#endif
-
-	 xf86DrvMsg(pScrn->scrnIndex, from,
-		    "Will %stry to allocate texture pool "
-		    "for old Mesa 3D driver.\n",
-		    (pI830->mmModeFlags & I830_KERNEL_TEX) ?
-		    "" : "not ");
+	 if (!IS_I965G(pI830)) {
+	    Bool tmp;
 
-#ifdef XF86DRI_MM
-	 pI830->mmSize = I830_MM_MAXSIZE;
-	 from = X_INFO;
-	 if (xf86GetOptValInteger(pI830->Options, OPTION_INTELMMSIZE,
-				  &(pI830->mmSize))) {
-	    from = X_CONFIG;
+	    if (xf86GetOptValBool(pI830->Options,
+				  OPTION_INTELTEXPOOL, &tmp)) {
+	       from = X_CONFIG;
+	       if (!tmp)
+		  pI830->allocate_classic_textures = FALSE;
+	    }
 	 }
-	 xf86DrvMsg(pScrn->scrnIndex, from, 
-		    "Will try to reserve %d kiB of AGP aperture space\n"
-		    "\tfor the DRM memory manager.\n",
-		    pI830->mmSize);
 #endif
       }
    } 
@@ -2128,12 +2098,6 @@ I830ScreenInit(int scrnIndex, ScreenPtr pScreen, int argc, char **argv)
    int i;
    Bool allocation_done = FALSE;
    MessageType from;
-#ifdef XF86DRI
-   Bool driDisabled;
-#ifdef XF86DRI_MM
-   unsigned long savedMMSize;
-#endif
-#endif
 
    pScrn = xf86Screens[pScreen->myNum];
    pI830 = I830PTR(pScrn);
@@ -2227,6 +2191,23 @@ I830ScreenInit(int scrnIndex, ScreenPtr pScreen, int argc, char **argv)
       pScrn->videoRam &= ~3;
    }
 
+#ifdef XF86DRI
+   /* Check for appropriate bpp and module support to initialize DRI. */
+   if (!I830CheckDRIAvailable(pScrn)) {
+      pI830->directRenderingDisabled = TRUE;
+   }
+
+   /* If DRI hasn't been explicitly disabled, try to initialize it.
+    * It will be used by the memory allocator.
+    */
+   if (!pI830->directRenderingDisabled)
+      pI830->directRenderingEnabled = I830DRIScreenInit(pScreen);
+   else
+      pI830->directRenderingEnabled = FALSE;
+#else
+   pI830->directRenderingEnabled = FALSE;
+#endif
+
    /* Set up our video memory allocator for the chosen videoRam */
    if (!i830_allocator_init(pScrn, 0, pScrn->videoRam * KB(1))) {
       xf86DrvMsg(pScrn->scrnIndex, X_ERROR,
@@ -2321,12 +2302,6 @@ I830ScreenInit(int scrnIndex, ScreenPtr pScreen, int argc, char **argv)
     * for it, and if there's also enough to allow tiling to be enabled.
     */
 
-   if (!I830CheckDRIAvailable(pScrn)) {
-      pI830->directRenderingDisabled = TRUE;
-#ifdef XF86DRI_MM
-      pI830->mmSize = 0;
-#endif
-   }
 
 #ifdef I830_XV
     /*
@@ -2336,7 +2311,7 @@ I830ScreenInit(int scrnIndex, ScreenPtr pScreen, int argc, char **argv)
     pI830->XvEnabled = !pI830->XvDisabled;
 #endif
 
-   if (!pI830->directRenderingDisabled) {
+   if (pI830->directRenderingEnabled) {
       int savedDisplayWidth = pScrn->displayWidth;
       Bool tiled = FALSE;
 
@@ -2366,25 +2341,18 @@ I830ScreenInit(int scrnIndex, ScreenPtr pScreen, int argc, char **argv)
 	 }
       }
 
-      /* Attempt several rounds of allocation to get 2d and 3d memory to fit:
+      /* Attempt two rounds of allocation to get 2d and 3d memory to fit:
        *
-       * 0: tiled, large memory manager
-       * 1: tiled, small memory manager
-       * 2: untiled, large
-       * 3: untiled, small
+       * 0: untiled
+       * 1: tiled
        */
 
-#ifdef XF86DRI_MM
-      savedMMSize = pI830->mmSize;
-#define MM_TURNS 4
-#else
 #define MM_TURNS 2
-#endif
       for (i = 0; i < MM_TURNS; i++) {
-	 if (!tiled && i < 2)
+	 if (!tiled && i == 0)
 	    continue;
 
-	 if (i >= MM_TURNS/2) {
+	 if (i >= 1) {
 	    /* For further allocations, disable tiling */
 	    pI830->tiling = FALSE;
 	    pScrn->displayWidth = savedDisplayWidth;
@@ -2395,26 +2363,9 @@ I830ScreenInit(int scrnIndex, ScreenPtr pScreen, int argc, char **argv)
 	    pI830->allowPageFlip = FALSE;
 	 }
 
-#ifdef XF86DRI_MM
-	 if (i & 1) {
-	    /* For this allocation, switch to a smaller DRI memory manager
-	     * size.
-	     */
-	    pI830->mmSize = I830_MM_MINPAGES * GTT_PAGE_SIZE / KB(1);
-	 } else {
-	    pI830->mmSize = savedMMSize;
-	 }
-
-	 xf86DrvMsg(pScrn->scrnIndex, X_INFO,
-		    "Attempting memory allocation with %s buffers and \n"
-		    "\t       %s DRI memory manager reservation:\n",
-		    (i & 2) ? "untiled" : "tiled",
-		    (i & 1) ? "small" : "large");
-#else
 	 xf86DrvMsg(pScrn->scrnIndex, X_INFO,
-		    "Attempting memory allocation with %s buffers:\n",
+		    "Attempting memory allocation with %s buffers.\n",
 		    (i & 1) ? "untiled" : "tiled");
-#endif
 
 	 if (i830_allocate_2d_memory(pScrn) &&
 	     i830_allocate_3d_memory(pScrn))
@@ -2436,10 +2387,7 @@ I830ScreenInit(int scrnIndex, ScreenPtr pScreen, int argc, char **argv)
       if (i == MM_TURNS) {
 	 xf86DrvMsg(pScrn->scrnIndex, X_WARNING,
 		    "Not enough video memory.  Disabling DRI.\n");
-#ifdef XF86DRI_MM
-	 pI830->mmSize = 0;
-#endif
-	 pI830->directRenderingDisabled = TRUE;
+	 pI830->directRenderingEnabled = FALSE;
       }
    }
 #endif
@@ -2461,7 +2409,7 @@ I830ScreenInit(int scrnIndex, ScreenPtr pScreen, int argc, char **argv)
       xf86DrvMsg(pScrn->scrnIndex, X_ERROR,
 		 "Cannot support DRI with frame buffer width > 2048.\n");
       pI830->tiling = FALSE;
-      pI830->directRenderingDisabled = TRUE;
+      pI830->directRenderingEnabled = FALSE;
    }
 
    pScrn->displayWidth = pScrn->displayWidth;
@@ -2563,11 +2511,6 @@ I830ScreenInit(int scrnIndex, ScreenPtr pScreen, int argc, char **argv)
 
 #ifdef XF86DRI
    /*
-    * pI830->directRenderingDisabled is set once in PreInit.  Reinitialise
-    * pI830->directRenderingEnabled based on it each generation.
-    */
-   pI830->directRenderingEnabled = !pI830->directRenderingDisabled;
-   /*
     * Setup DRI after visuals have been established, but before fbScreenInit
     * is called.   fbScreenInit will eventually call into the drivers
     * InitGLXVisuals call back.
@@ -2578,26 +2521,21 @@ I830ScreenInit(int scrnIndex, ScreenPtr pScreen, int argc, char **argv)
 	 xf86DrvMsg(pScrn->scrnIndex, X_PROBED, "DRI is disabled because it "
 		    "needs HW cursor, 2D accel and AGPGART.\n");
 	 pI830->directRenderingEnabled = FALSE;
-	 i830_free_3d_memory(pScrn);
       }
    }
 
-   driDisabled = !pI830->directRenderingEnabled;
-
    if (pI830->directRenderingEnabled)
-      pI830->directRenderingEnabled = I830DRIScreenInit(pScreen);
+      pI830->directRenderingEnabled = I830DRIDoMappings(pScreen);
 
-   if (!pI830->directRenderingEnabled) {
+   /* If we failed for any reason, free DRI memory. */
+   if (!pI830->directRenderingEnabled)
       i830_free_3d_memory(pScrn);
-   }
 
 #else
    pI830->directRenderingEnabled = FALSE;
 #endif
 
 #ifdef XF86DRI
-   if (pI830->directRenderingEnabled)
-      pI830->directRenderingEnabled = I830DRIDoMappings(pScreen);
 
    xf86DrvMsg(pScrn->scrnIndex, X_INFO, "Page Flipping %sabled\n",
 	      pI830->allowPageFlip ? "en" : "dis");
@@ -2717,9 +2655,8 @@ I830ScreenInit(int scrnIndex, ScreenPtr pScreen, int argc, char **argv)
 #endif
 
 #ifdef XF86DRI
-   if (pI830->directRenderingEnabled) {
+   if (pI830->directRenderingEnabled)
       pI830->directRenderingEnabled = I830DRIFinishScreenInit(pScreen);
-   }
 #endif
 
    /* Setup 3D engine, needed for rotation too */
@@ -2730,7 +2667,7 @@ I830ScreenInit(int scrnIndex, ScreenPtr pScreen, int argc, char **argv)
       pI830->directRenderingOpen = TRUE;
       xf86DrvMsg(pScrn->scrnIndex, X_INFO, "direct rendering: Enabled\n");
    } else {
-      if (driDisabled)
+      if (pI830->directRenderingDisabled)
 	 xf86DrvMsg(pScrn->scrnIndex, X_INFO, "direct rendering: Disabled\n");
       else
 	 xf86DrvMsg(pScrn->scrnIndex, X_INFO, "direct rendering: Failed\n");
@@ -2771,37 +2708,6 @@ I830ScreenInit(int scrnIndex, ScreenPtr pScreen, int argc, char **argv)
    pI830->closing = FALSE;
    pI830->suspended = FALSE;
 
-#ifdef XF86DRI_MM
-   if (pI830->directRenderingEnabled && (pI830->mmModeFlags & I830_KERNEL_MM))
-   {
-      if (pI830->memory_manager == NULL) {
-	 xf86DrvMsg(pScrn->scrnIndex, X_ERROR, 
-		    "Too little AGP aperture space for DRM memory manager.\n"
-		    "\tPlease increase AGP aperture size from BIOS configuration screen.\n"
-		    "\tDisabling DRI.\n");
-	 pI830->directRenderingOpen = FALSE;
-	 I830DRICloseScreen(pScreen);
-	 pI830->directRenderingEnabled = FALSE;
-      } else {
-	 unsigned long aperEnd = ROUND_DOWN_TO(pI830->memory_manager->offset +
-					       pI830->memory_manager->size,
-					       GTT_PAGE_SIZE) / GTT_PAGE_SIZE;
-	 unsigned long aperStart = ROUND_TO(pI830->memory_manager->offset,
-					    GTT_PAGE_SIZE) / GTT_PAGE_SIZE;
-
-	 if (drmMMInit(pI830->drmSubFD, aperStart, aperEnd - aperStart,
-		       DRM_BO_MEM_TT)) {
-	    xf86DrvMsg(pScrn->scrnIndex, X_ERROR, 
-		       "Could not initialize the DRM memory manager.\n");
-	    
-	    pI830->directRenderingOpen = FALSE;
-	    I830DRICloseScreen(pScreen);
-	    pI830->directRenderingEnabled = FALSE;
-	 }
-      }
-   }
-#endif /* XF86DRI_MM */
-
    return TRUE;
 }
 
@@ -2862,7 +2768,7 @@ I830LeaveVT(int scrnIndex, int flags)
    if (pI830->directRenderingOpen) {
       DRILock(screenInfo.screens[pScrn->scrnIndex], 0);
 #ifdef XF86DRI_MM
-      if (pI830->mmModeFlags & I830_KERNEL_MM) {
+      if (pI830->memory_manager != NULL) {
 	 drmMMLock(pI830->drmSubFD, DRM_BO_MEM_TT);
       }
 #endif /* XF86DRI_MM */
@@ -2967,7 +2873,7 @@ I830EnterVT(int scrnIndex, int flags)
 	    sarea->texList[i].age = sarea->texAge;
 
 #ifdef XF86DRI_MM
-	 if (pI830->mmModeFlags & I830_KERNEL_MM) {
+	 if (pI830->memory_manager != NULL) {
 	    drmMMUnlock(pI830->drmSubFD, DRM_BO_MEM_TT);
 	 }
 #endif /* XF86DRI_MM */
@@ -3014,26 +2920,6 @@ I830CloseScreen(int scrnIndex, ScreenPtr pScreen)
 #endif
 
    pI830->closing = TRUE;
-#ifdef XF86DRI
-   if (pI830->directRenderingOpen) {
-#ifdef DAMAGE
-      if (pI830->pDamage) {
-	 PixmapPtr pPix = pScreen->GetScreenPixmap(pScreen);
-
-	 DamageUnregister(&pPix->drawable, pI830->pDamage);
-	 DamageDestroy(pI830->pDamage);
-	 pI830->pDamage = NULL;
-      }
-#endif
-#ifdef XF86DRI_MM
-      if (pI830->mmModeFlags & I830_KERNEL_MM) {
-	 drmMMTakedown(pI830->drmSubFD, DRM_BO_MEM_TT);	 
-      }
-#endif /* XF86DRI_MM */
-      pI830->directRenderingOpen = FALSE;
-      I830DRICloseScreen(pScreen);
-   }
-#endif
 
    if (pScrn->vtSema == TRUE) {
       I830LeaveVT(scrnIndex, 0);
@@ -3068,7 +2954,22 @@ I830CloseScreen(int scrnIndex, ScreenPtr pScreen)
 #endif
    xf86_cursors_fini (pScreen);
 
-   i830_reset_allocations(pScrn);
+   i830_allocator_fini(pScrn);
+#ifdef XF86DRI
+   if (pI830->directRenderingOpen) {
+#ifdef DAMAGE
+      if (pI830->pDamage) {
+	 PixmapPtr pPix = pScreen->GetScreenPixmap(pScreen);
+
+	 DamageUnregister(&pPix->drawable, pI830->pDamage);
+	 DamageDestroy(pI830->pDamage);
+	 pI830->pDamage = NULL;
+      }
+#endif
+      pI830->directRenderingOpen = FALSE;
+      I830DRICloseScreen(pScreen);
+   }
+#endif
 
    if (I830IsPrimary(pScrn)) {
       xf86GARTCloseScreen(scrnIndex);
diff --git a/src/i830_memory.c b/src/i830_memory.c
index b38a5dfe..d4e9bff0 100644
--- a/src/i830_memory.c
+++ b/src/i830_memory.c
@@ -114,6 +114,9 @@ enum tile_format {
     TILING_YMAJOR
 };
 
+static i830_memory *
+i830_allocate_aperture(ScrnInfoPtr pScrn, const char *name,
+		       long size, unsigned long alignment, int flags);
 static void i830_set_fence(ScrnInfoPtr pScrn, int nr, unsigned int offset,
 			   unsigned int pitch, unsigned int size,
 			   enum tile_format tile_format);
@@ -182,10 +185,22 @@ i830_unbind_memory(ScrnInfoPtr pScrn, i830_memory *mem)
 void
 i830_free_memory(ScrnInfoPtr pScrn, i830_memory *mem)
 {
+#ifdef XF86DRI_MM
+    I830Ptr pI830 = I830PTR(pScrn);
+#endif
+
     if (mem == NULL)
 	return;
 
-    /* Disconnect from the list of allocations */
+#ifdef XF86DRI_MM
+    if (mem->bo.size != 0) {
+	drmBOUnReference(pI830->drmSubFD, &mem->bo);
+	xfree(mem->name);
+	xfree(mem);
+	return;
+    }
+#endif
+	    /* Disconnect from the list of allocations */
     if (mem->prev != NULL)
 	mem->prev->next = mem->next;
     if (mem->next != NULL)
@@ -237,7 +252,6 @@ i830_reset_allocations(ScrnInfoPtr pScrn)
     pI830->third_buffer = NULL;
     pI830->depth_buffer = NULL;
     pI830->textures = NULL;
-    pI830->memory_manager = NULL;
 #endif
     pI830->LpRing->mem = NULL;
 
@@ -260,21 +274,24 @@ i830_free_3d_memory(ScrnInfoPtr pScrn)
     pI830->depth_buffer = NULL;
     i830_free_memory(pScrn, pI830->textures);
     pI830->textures = NULL;
-    i830_free_memory(pScrn, pI830->memory_manager);
-    pI830->memory_manager = NULL;
 #endif
 }
 
 /**
  * Initialize's the driver's video memory allocator to allocate in the
  * given range.
+ *
+ * This sets up the kernel memory manager to manage as much of the memory
+ * as we think it can, while leaving enough to us to fulfill our non-TTM
+ * static allocations.  Some of these exist because of the need for physical
+ * addresses to reference.
  */
 Bool
-i830_allocator_init(ScrnInfoPtr pScrn, unsigned long offset,
-		    unsigned long size)
+i830_allocator_init(ScrnInfoPtr pScrn, unsigned long offset, unsigned long size)
 {
     I830Ptr pI830 = I830PTR(pScrn);
     i830_memory *start, *end;
+    int ret;
 
     start = xcalloc(1, sizeof(*start));
     if (start == NULL)
@@ -311,9 +328,82 @@ i830_allocator_init(ScrnInfoPtr pScrn, unsigned long offset,
 
     pI830->memory_list = start;
 
+#ifdef XF86DRI_MM
+    /* Now that we have our manager set up, initialize the kernel MM if
+     * possible, covering almost all of the aperture.
+     */
+    if (pI830->directRenderingEnabled && pI830->drmMinor >= 7) {
+	int mmsize;
+
+	/* Take over all of the graphics aperture minus enough to for
+	 * physical-address allocations of cursor/overlay registers.
+	 */
+	mmsize = size;
+	if (!OVERLAY_NOPHYSICAL(pI830) && !IS_I965G(pI830)) {
+	    mmsize -= ROUND_TO(OVERLAY_SIZE, GTT_PAGE_SIZE);
+	    if (pI830->CursorNeedsPhysical) {
+		mmsize -= 2 * (ROUND_TO(HWCURSOR_SIZE, GTT_PAGE_SIZE) +
+			     ROUND_TO(HWCURSOR_SIZE_ARGB, GTT_PAGE_SIZE));
+	    }
+	}
+	if (pI830->fb_compression)
+	    mmsize -= MB(6);
+	/* Can't do TTM on stolen memory */
+	mmsize -= pI830->stolen_size;
+
+	/* Create the aperture allocation */
+	pI830->memory_manager =
+	    i830_allocate_aperture(pScrn, "DRI memory manager",
+				   mmsize, GTT_PAGE_SIZE,
+				   ALIGN_BOTH_ENDS | NEED_NON_STOLEN);
+
+	if (pI830->memory_manager != NULL) {
+	    /* Tell the kernel to manage it */
+	    ret = drmMMInit(pI830->drmSubFD,
+			    pI830->memory_manager->offset / GTT_PAGE_SIZE,
+			    pI830->memory_manager->size / GTT_PAGE_SIZE,
+			    DRM_BO_MEM_TT);
+	    if (ret != 0) {
+		xf86DrvMsg(pScrn->scrnIndex, X_ERROR,
+			   "Failed to initialize kernel memory manager\n");
+		i830_free_memory(pScrn, pI830->memory_manager);
+		pI830->memory_manager = NULL;
+	    }
+	} else {
+	    xf86DrvMsg(pScrn->scrnIndex, X_ERROR,
+		       "Failed to allocate space for kernel memory manager\n");
+	    i830_free_memory(pScrn, pI830->memory_manager);
+	    pI830->memory_manager = NULL;
+	}
+    }
+#endif /* XF86DRI_MM */
+
     return TRUE;
 }
 
+void
+i830_allocator_fini(ScrnInfoPtr pScrn)
+{
+    I830Ptr pI830 = I830PTR(pScrn);
+
+    /* Free most of the allocations */
+    i830_reset_allocations(pScrn);
+
+#ifdef XF86DRI_MM
+    /* The memory manager is more special */
+    if (pI830->memory_manager) {
+	 drmMMTakedown(pI830->drmSubFD, DRM_BO_MEM_TT);
+	 i830_free_memory(pScrn, pI830->memory_manager);
+	 pI830->memory_manager = NULL;
+    }
+#endif /* XF86DRI_MM */
+
+    /* Free the start/end markers */
+    free(pI830->memory_list->next);
+    free(pI830->memory_list);
+    pI830->memory_list = NULL;
+}
+
 /**
  * Reads a GTT entry for the memory at the given offset and returns the
  * physical address.
@@ -447,6 +537,9 @@ i830_allocate_aperture(ScrnInfoPtr pScrn, const char *name,
 		mem->offset = ROUND_TO(pI830->stolen_size, alignment);
 	    }
 	}
+	if ((flags & NEED_NON_STOLEN) && mem->offset < pI830->stolen_size) {
+	    mem->offset = ROUND_TO(pI830->stolen_size, alignment);
+	}
 
 	mem->end = mem->offset + size;
 	if (flags & ALIGN_BOTH_ENDS)
@@ -515,25 +608,80 @@ i830_allocate_agp_memory(ScrnInfoPtr pScrn, i830_memory *mem, int flags)
     return TRUE;
 }
 
+#ifdef XF86DRI_MM
+static i830_memory *
+i830_allocate_memory_bo(ScrnInfoPtr pScrn, const char *name,
+			unsigned long size, unsigned long align, int flags)
+{
+    I830Ptr pI830 = I830PTR(pScrn);
+    i830_memory *mem;
+    unsigned long hint, mask;
+    int ret;
+
+    assert((flags & NEED_PHYSICAL_ADDR) == 0);
+
+    /* Only allocate page-sized increments. */
+    size = ALIGN(size, GTT_PAGE_SIZE);
+    align = ROUND_TO(align, GTT_PAGE_SIZE);
+
+    mem = xcalloc(1, sizeof(*mem));
+    if (mem == NULL)
+	return NULL;
+
+    mem->name = xstrdup(name);
+    if (name == NULL) {
+	xfree(mem);
+	return NULL;
+    }
+
+    mask = DRM_BO_FLAG_READ | DRM_BO_FLAG_WRITE | DRM_BO_FLAG_MAPPABLE |
+	DRM_BO_FLAG_NO_EVICT | DRM_BO_FLAG_NO_MOVE;
+    hint = DRM_BO_HINT_DONT_FENCE | DRM_BO_HINT_DONT_BLOCK |
+	DRM_BO_HINT_ALLOW_UNFENCED_MAP;
+
+    I830DRILock(pScrn);
+    ret = drmBOCreate(pI830->drmSubFD, 0, size, align / GTT_PAGE_SIZE, NULL,
+		      drm_bo_type_dc, mask, hint, &mem->bo);
+    I830DRIUnlock(pScrn);
+    if (ret) {
+	xfree(mem->name);
+	xfree(mem);
+	return NULL;
+    }
+
+    mem->offset = mem->bo.offset;
+    mem->end = mem->bo.start + size;
+    mem->size = size;
+
+    return mem;
+}
+#endif /* XF86DRI_MM */
 
 /* Allocates video memory at the given size and alignment.
  *
  * The memory will be bound automatically when the driver is in control of the
- * VT.
+ * VT.  When the kernel memory manager is available and compatible with flags
+ * (that is, flags doesn't say that the allocation must include a physical
+ * address), that will be used for the allocation.
  */
 static i830_memory *
 i830_allocate_memory(ScrnInfoPtr pScrn, const char *name,
 		     unsigned long size, unsigned long alignment, int flags)
 {
+    I830Ptr pI830 = I830PTR(pScrn);
     i830_memory *mem;
 
-    mem = i830_allocate_aperture(pScrn, name, size, alignment, flags);
-    if (mem == NULL)
-	return NULL;
+    if (pI830->memory_manager && !(flags & NEED_PHYSICAL_ADDR)) {
+	return i830_allocate_memory_bo(pScrn, name, size, alignment, flags);
+    } else {
+	mem = i830_allocate_aperture(pScrn, name, size, alignment, flags);
+	if (mem == NULL)
+	    return NULL;
 
-    if (!i830_allocate_agp_memory(pScrn, mem, flags)) {
-	i830_free_memory(pScrn, mem);
-	return NULL;
+	if (!i830_allocate_agp_memory(pScrn, mem, flags)) {
+	    i830_free_memory(pScrn, mem);
+	    return NULL;
+	}
     }
 
     return mem;
@@ -588,7 +736,7 @@ i830_allocate_memory_tiled(ScrnInfoPtr pScrn, const char *name,
 	aper_align = alignment;
 
     fence_divide = 1;
-    mem = i830_allocate_aperture(pScrn, name, aper_size, aper_align, flags);
+    mem = i830_allocate_memory(pScrn, name, aper_size, aper_align, flags);
     if (mem == NULL && !IS_I965G(pI830)) {
 	/* For the older hardware with stricter fencing limits, if we
 	 * couldn't allocate with the large alignment, try relaxing the
@@ -604,8 +752,8 @@ i830_allocate_memory_tiled(ScrnInfoPtr pScrn, const char *name,
 		break;
 	    }
 
-	    mem = i830_allocate_aperture(pScrn, name, aper_size,
-					 aper_align / fence_divide, flags);
+	    mem = i830_allocate_memory(pScrn, name, aper_size,
+				       aper_align / fence_divide, flags);
 	}
     }
 
@@ -623,12 +771,6 @@ i830_allocate_memory_tiled(ScrnInfoPtr pScrn, const char *name,
 	return NULL;
     }
 
-    /* Allocate any necessary AGP memory to back this allocation */
-    if (!i830_allocate_agp_memory(pScrn, mem, flags)) {
-	i830_free_memory(pScrn, mem);
-	return NULL;
-    }
-
     /* Set up the fence registers. */
     for (i = 0; i < fence_divide; i++) {
 	i830_set_fence(pScrn, pI830->next_fence++,
@@ -1341,17 +1483,7 @@ i830_allocate_texture_memory(ScrnInfoPtr pScrn)
     unsigned long size;
     int i;
 
-    if (pI830->mmModeFlags & I830_KERNEL_MM) {
-	pI830->memory_manager =
-	    i830_allocate_aperture(pScrn, "DRI memory manager",
-				   pI830->mmSize * KB(1), GTT_PAGE_SIZE,
-				   ALIGN_BOTH_ENDS);
-	/* XXX: try memory manager size backoff here? */
-	if (pI830->memory_manager == NULL)
-	    return FALSE;
-    }
-
-    if (pI830->mmModeFlags & I830_KERNEL_TEX) {
+    if (pI830->allocate_classic_textures) {
 	/* XXX: auto-sizing */
 	size = MB(32);
 	i = myLog2(size / I830_NR_TEX_REGIONS);
@@ -1368,7 +1500,7 @@ i830_allocate_texture_memory(ScrnInfoPtr pScrn)
 		       size / 1024);
 	    return FALSE;
 	}
-	pI830->textures = i830_allocate_memory(pScrn, "textures", size,
+	pI830->textures = i830_allocate_memory(pScrn, "classic textures", size,
 					       GTT_PAGE_SIZE, 0);
 	if (pI830->textures == NULL) {
 	    xf86DrvMsg(pScrn->scrnIndex, X_WARNING,