ttm: Backdoor aperture maps.

6 files changed, 106 insertions, 22 deletions

diff --git a/linux-core/drm_agpsupport.c b/linux-core/drm_agpsupport.c
index 395a069c..0aa32adc 100644
--- a/linux-core/drm_agpsupport.c
+++ b/linux-core/drm_agpsupport.c
@@ -647,14 +647,13 @@ void drm_agp_destroy_ttm(drm_ttm_backend_t *backend) {
 }
 	
 
-
-
 drm_ttm_backend_t *drm_agp_init_ttm_uncached(struct drm_device *dev) {
 
-	drm_ttm_backend_t *agp_be = drm_calloc(1, sizeof(*agp_be), DRM_MEM_MAPPINGS);
+        drm_ttm_backend_t *agp_be;
 	drm_agp_ttm_priv *agp_priv;
 
-	DRM_DEBUG("drm_agp_init_ttm\n");
+	agp_be = drm_calloc(1, sizeof(*agp_be), DRM_MEM_MAPPINGS);
+
 	if (!agp_be)
 		return NULL;
 	
@@ -669,6 +668,7 @@ drm_ttm_backend_t *drm_agp_init_ttm_uncached(struct drm_device *dev) {
 	agp_priv->mem_type = AGP_MEM_USER;
 	agp_priv->bridge = dev->agp->bridge;
 	agp_priv->populated = FALSE;
+	agp_be->aperture_base = dev->agp->agp_info.aper_base;
 	agp_be->private = (void *) agp_priv;
 	agp_be->needs_cache_adjust = drm_agp_needs_cache_adjust_true;
 	agp_be->populate = drm_agp_populate;
@@ -683,10 +683,11 @@ EXPORT_SYMBOL(drm_agp_init_ttm_uncached);
 
 drm_ttm_backend_t *drm_agp_init_ttm_cached(struct drm_device *dev) {
 
-	drm_ttm_backend_t *agp_be = drm_calloc(1, sizeof(*agp_be), DRM_MEM_MAPPINGS);
+        drm_ttm_backend_t *agp_be;
 	drm_agp_ttm_priv *agp_priv;
 
-	DRM_DEBUG("drm_agp_init_ttm\n");
+	agp_be = drm_calloc(1, sizeof(*agp_be), DRM_MEM_MAPPINGS);
+
 	if (!agp_be)
 		return NULL;
 	
@@ -701,6 +702,7 @@ drm_ttm_backend_t *drm_agp_init_ttm_cached(struct drm_device *dev) {
 	agp_priv->mem_type = AGP_MEM_UCACHED;
 	agp_priv->bridge = dev->agp->bridge;
 	agp_priv->populated = FALSE;
+	agp_be->aperture_base = dev->agp->agp_info.aper_base;
 	agp_be->private = (void *) agp_priv;
 	agp_be->needs_cache_adjust = drm_agp_needs_cache_adjust_false;
 	agp_be->populate = drm_agp_populate;
diff --git a/linux-core/drm_fops.c b/linux-core/drm_fops.c
index dce7d0e9..678f8bc5 100644
--- a/linux-core/drm_fops.c
+++ b/linux-core/drm_fops.c
@@ -362,6 +362,10 @@ int drm_release(struct inode *inode, struct file *filp)
 		if (dev->driver->reclaim_buffers_locked)
 			dev->driver->reclaim_buffers_locked(dev, filp);
 
+		if (dev->driver->ttm_driver) {
+			uint32_t fence =  dev->driver->ttm_driver->emit_fence(dev);
+			dev->driver->ttm_driver->wait_fence(dev, fence);
+		}
 		drm_lock_free(dev, &dev->lock.hw_lock->lock,
 			      _DRM_LOCKING_CONTEXT(dev->lock.hw_lock->lock));
 
@@ -369,7 +373,8 @@ int drm_release(struct inode *inode, struct file *filp)
 		   hardware at this point, possibly
 		   processed via a callback to the X
 		   server. */
-	} else if (dev->driver->reclaim_buffers_locked && priv->lock_count
+	} else if ((dev->driver->reclaim_buffers_locked || 
+		    dev->driver->ttm_driver) && priv->lock_count
 		   && dev->lock.hw_lock) {
 		/* The lock is required to reclaim buffers */
 		DECLARE_WAITQUEUE(entry, current);
@@ -399,7 +404,13 @@ int drm_release(struct inode *inode, struct file *filp)
 		__set_current_state(TASK_RUNNING);
 		remove_wait_queue(&dev->lock.lock_queue, &entry);
 		if (!retcode) {
-			dev->driver->reclaim_buffers_locked(dev, filp);
+			if (dev->driver->reclaim_buffers_locked)
+				dev->driver->reclaim_buffers_locked(dev, filp);
+			if (dev->driver->ttm_driver) {
+				uint32_t fence =  dev->driver->ttm_driver->emit_fence(dev);
+				dev->driver->ttm_driver->wait_fence(dev, fence);
+			}
+
 			drm_lock_free(dev, &dev->lock.hw_lock->lock,
 				      DRM_KERNEL_CONTEXT);
 		}
@@ -451,6 +462,7 @@ int drm_release(struct inode *inode, struct file *filp)
 	} else {
 		dev->file_last = priv->prev;
 	}
+
 	list_for_each_safe(list, next, &priv->ttms) {
 		drm_map_list_t *entry = list_entry(list, drm_map_list_t, head);
 		list_del(list);
@@ -462,6 +474,7 @@ int drm_release(struct inode *inode, struct file *filp)
 				 DRM_MEM_MAPS);
 		drm_free(entry, sizeof(*entry), DRM_MEM_MAPS);
 	}
+
 	list=NULL;
 	next=NULL;
 	list_for_each_safe(list, next, &priv->anon_ttm_regs) {
@@ -472,7 +485,8 @@ int drm_release(struct inode *inode, struct file *filp)
 			drm_remove_ht_val(&dev->ttmreghash, hash);
 		}
 		drm_user_destroy_region(entry);
- 	}
+ 	
+	}
 
 	up(&dev->struct_sem);
 
diff --git a/linux-core/drm_ttm.c b/linux-core/drm_ttm.c
index a390c620..432a848c 100644
--- a/linux-core/drm_ttm.c
+++ b/linux-core/drm_ttm.c
@@ -33,6 +33,7 @@
 #include <asm/tlbflush.h>
 #include <asm/pgtable.h>
 
+
 /*
  * DAVE: The below code needs to go to the linux mm subsystem. Most of it is already there.
  * Basically stolen from mprotect.c and rmap.c 
@@ -226,7 +227,8 @@ int drm_destroy_ttm(drm_ttm_t * ttm)
 	if (ttm->pages) {
 		for (i = 0; i < ttm->num_pages; ++i) {
 			cur_page = ttm->pages + i;
-			if (ttm->page_flags && ttm->page_flags[i] &&
+			if (ttm->page_flags && 
+			    (ttm->page_flags[i] & DRM_TTM_PAGE_UNCACHED) &&
 			    *cur_page && !PageHighMem(*cur_page)) {
 				change_page_attr(*cur_page, 1, PAGE_KERNEL);
 			}
@@ -322,7 +324,7 @@ drm_ttm_t *drm_init_ttm(struct drm_device * dev, unsigned long size)
  */
 
 static int drm_set_caching(drm_ttm_t * ttm, unsigned long page_offset,
-			   unsigned long num_pages, int noncached)
+			   unsigned long num_pages, uint32_t noncached)
 {
 	int i, cur;
 	struct page **cur_page;
@@ -352,8 +354,11 @@ static int drm_set_caching(drm_ttm_t * ttm, unsigned long page_offset,
 					}
 					return -EINVAL;
 				}
-			} else if (ttm->page_flags[cur] != noncached) {
-				ttm->page_flags[cur] = noncached;
+			} else if ((ttm->page_flags[cur] & 
+				    DRM_TTM_PAGE_UNCACHED) != noncached) {
+				DRM_MASK_VAL(ttm->page_flags[cur], 
+					     DRM_TTM_PAGE_UNCACHED, 
+					     noncached);
 				change_page_attr(*cur_page, 1, attr);
 			}
 		}
@@ -411,10 +416,11 @@ void drm_unbind_ttm_region(drm_ttm_backend_list_t * entry)
 		switch (entry->state) {
 		case ttm_bound:
 			be->unbind(entry->be);
+			/* Fall through */			  
 		case ttm_evicted:
 			if (ttm && be->needs_cache_adjust(be)) {
 				drm_set_caching(ttm, entry->page_offset,
-						entry->num_pages, FALSE);
+						entry->num_pages, 0);
 			}
 			break;
 		default:
@@ -440,7 +446,7 @@ void drm_destroy_ttm_region(drm_ttm_backend_list_t * entry)
 		be->clear(entry->be);
 		if (be->needs_cache_adjust(be)) {
 			drm_set_caching(ttm, entry->page_offset,
-					entry->num_pages, FALSE);
+					entry->num_pages, 0);
 		}
 		be->destroy(be);
 	}
@@ -537,6 +543,7 @@ int drm_create_ttm_region(drm_ttm_t * ttm, unsigned long page_offset,
 	}
 	entry->mm_node = NULL;
 	entry->mm = ttm->dev->driver->ttm_driver->ttm_mm(ttm->dev);
+	ttm->aperture_base = be->aperture_base;
 	*region = entry;
 	return 0;
 }
@@ -549,6 +556,8 @@ int drm_bind_ttm_region(drm_ttm_backend_list_t * region,
 			unsigned long aper_offset)
 {
 
+	int i;
+	uint32_t *cur_page_flag;
 	int ret;
 	drm_ttm_backend_t *be;
 	drm_ttm_t *ttm;
@@ -561,7 +570,7 @@ int drm_bind_ttm_region(drm_ttm_backend_list_t * region,
 
 	if (ttm && be->needs_cache_adjust(be)) {
 		drm_set_caching(ttm, region->page_offset, region->num_pages,
-				TRUE);
+				DRM_TTM_PAGE_UNCACHED);
 	} else {
 		flush_cache_all();
 	}
@@ -571,6 +580,14 @@ int drm_bind_ttm_region(drm_ttm_backend_list_t * region,
 		DRM_ERROR("Couldn't bind backend.\n");
 		return ret;
 	}
+
+	cur_page_flag = ttm->page_flags;
+	for (i = 0; i < region->num_pages; ++i) {
+		DRM_MASK_VAL(*cur_page_flag, DRM_TTM_MASK_PFN, 
+			     (i + aper_offset) << PAGE_SHIFT);
+		cur_page_flag++;
+	}
+
 	region->state = ttm_bound;
 	return 0;
 }
@@ -583,10 +600,26 @@ int drm_evict_ttm_region(drm_ttm_backend_list_t * entry)
 {
 
 	int ret;
+	drm_ttm_backend_t *be;
+	drm_ttm_t *ttm;
 
 	if (!entry || entry->state != ttm_bound)
 		return -EINVAL;
 
+	ttm = entry->owner;
+	be = entry->be;
+
+	/*
+	 * Unmap backdoor aperture map.
+	 */
+
+	if (ttm && be->needs_cache_adjust(be)) {
+		unmap_vma_pages(ttm, entry->page_offset, entry->num_pages);
+		/*
+		 * FIXME: Add partial tlb flush here.
+		 */
+	}
+
 	if (0 != (ret = entry->be->unbind(entry->be))) {
 		return ret;
 	}
@@ -606,6 +639,8 @@ int drm_rebind_ttm_region(drm_ttm_backend_list_t * entry,
 	int ret;
 	drm_ttm_backend_t *be;
 	drm_ttm_t *ttm;
+	int i;
+	uint32_t *cur_page_flag;
 
 	if (!entry || entry->state != ttm_evicted)
 		return -EINVAL;
@@ -620,6 +655,14 @@ int drm_rebind_ttm_region(drm_ttm_backend_list_t * entry,
 	if (0 != (ret = be->bind(be, aper_offset))) {
 		return ret;
 	}
+
+	cur_page_flag = ttm->page_flags;
+	for (i = 0; i < entry->num_pages; ++i) {
+		DRM_MASK_VAL(*cur_page_flag, DRM_TTM_MASK_PFN, 
+			     (i + aper_offset) << PAGE_SHIFT);
+		cur_page_flag++;
+	}
+
 	entry->state = ttm_bound;
 	return 0;
 }
@@ -851,7 +894,7 @@ static int drm_ttm_evict_lru_sl(drm_ttm_backend_list_t * entry,
 		*cur_fence = evict_fence;
 		*have_fence = TRUE;
 	} while (TRUE);
-
+	DRM_ERROR("Evicting\n");
 	evict_node = evict_priv->region->mm_node;
 	drm_evict_ttm_region(evict_priv->region);
 	list_del_init(list);
diff --git a/linux-core/drm_ttm.h b/linux-core/drm_ttm.h
index 8c91f0a8..ceb486d9 100644
--- a/linux-core/drm_ttm.h
+++ b/linux-core/drm_ttm.h
@@ -14,6 +14,7 @@
  */
 
 typedef struct drm_ttm_backend {
+	unsigned long aperture_base;
 	void *private;
 	int (*needs_cache_adjust) (struct drm_ttm_backend * backend);
 	int (*populate) (struct drm_ttm_backend * backend,
@@ -55,6 +56,7 @@ typedef struct drm_ttm_vma_list {
 } drm_ttm_vma_list_t;
 
 typedef struct drm_ttm {
+	unsigned long aperture_base;
 	struct page **pages;
 	uint32_t *page_flags;
 	unsigned long lhandle;
@@ -81,7 +83,7 @@ typedef struct drm_ttm_mm {
 
 typedef struct drm_ttm_driver {
 	int cached_pages;
-	 uint32_t(*emit_fence) (struct drm_device * dev);
+	uint32_t(*emit_fence) (struct drm_device * dev);
 	int (*wait_fence) (struct drm_device * dev, uint32_t fence);
 	int (*test_fence) (struct drm_device * dev, uint32_t fence);
 	void (*flush_caches) (struct drm_device * dev, int access);
@@ -144,4 +146,12 @@ void drm_user_destroy_region(drm_ttm_backend_list_t * entry);
 
 int drm_ttm_ioctl(DRM_IOCTL_ARGS);
 
+#define DRM_MASK_VAL(dest, mask, val)			\
+  (dest) = ((dest) & ~(mask)) | ((val) & (mask));
+ 
+#define DRM_TTM_MASK_FLAGS ((1 << PAGE_SHIFT) - 1)
+#define DRM_TTM_MASK_PFN (0xFFFFFFFFU - DRM_TTM_MASK_FLAGS)
+
+#define DRM_TTM_PAGE_UNCACHED 0x1
+
 #endif
diff --git a/linux-core/drm_vm.c b/linux-core/drm_vm.c
index 05988367..ab95b1ce 100644
--- a/linux-core/drm_vm.c
+++ b/linux-core/drm_vm.c
@@ -208,6 +208,8 @@ static __inline__ struct page *drm_do_vm_ttm_nopage(struct vm_area_struct *vma,
 	struct page *page;
 	drm_ttm_t *ttm; 
 	pgprot_t default_prot;
+	unsigned long aper_loc = 0;
+	uint32_t page_flags;
 
 	if (address > vma->vm_end)
 		return NOPAGE_SIGBUS;	/* Disallow mremap */
@@ -218,6 +220,16 @@ static __inline__ struct page *drm_do_vm_ttm_nopage(struct vm_area_struct *vma,
 	ttm = (drm_ttm_t *) map->offset;
 	page_offset = (address - vma->vm_start) >> PAGE_SHIFT;
 	page = ttm->pages[page_offset];
+
+	page_flags = ttm->page_flags[page_offset];
+	if (page_flags & DRM_TTM_PAGE_UNCACHED) {
+	        BUG_ON(!page);
+		aper_loc = ttm->aperture_base + 
+			(page_flags & DRM_TTM_MASK_PFN);
+		page = pfn_to_page(aper_loc >> PAGE_SHIFT);
+	}
+
+
 	if (!page) {
 		page = ttm->pages[page_offset] = 
 			alloc_page(GFP_USER);
@@ -233,11 +245,14 @@ static __inline__ struct page *drm_do_vm_ttm_nopage(struct vm_area_struct *vma,
 	 * duplicate code.
 	 */
 
+	
 	default_prot = drm_prot_map[vma->vm_flags & 0x0f];
-	vma->vm_page_prot = ttm->page_flags[page_offset] ? 
-		pgprot_noncached(default_prot):
-		default_prot;
 
+	if (page_flags & DRM_TTM_PAGE_UNCACHED) {
+ 	  pgprot_val(default_prot) |= _PAGE_PCD;
+	  pgprot_val(default_prot) &= ~_PAGE_PWT;
+	}
+	vma->vm_page_prot = default_prot;
 	return page;
 }
 
diff --git a/linux-core/i915_ttm.c b/linux-core/i915_ttm.c
index 593bd9c8..34dc3272 100644
--- a/linux-core/i915_ttm.c
+++ b/linux-core/i915_ttm.c
@@ -24,7 +24,7 @@ static drm_ttm_mm_t *i915_ttm_mm(drm_device_t * dev)
 void i915_init_ttm(drm_device_t * dev, drm_i915_private_t * dev_priv)
 {
 	drm_ttm_driver_t *ttm_driver = &dev_priv->ttm_driver;
-	drm_ttm_mm_init(dev, &dev_priv->ttm_mm, 32768, 10240);
+	drm_ttm_mm_init(dev, &dev_priv->ttm_mm, 32768, 20480);
 	ttm_driver->emit_fence = i915_emit_fence;
 	ttm_driver->wait_fence = i915_wait_fence;
 	ttm_driver->test_fence = i915_test_fence;