diff options
| -rw-r--r-- | drivers/gpu/drm/nouveau/nouveau_bo.c | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/radeon_ttm.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_util.c | 31 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_vm.c | 9 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_page_alloc.c | 57 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_tt.c | 91 | ||||
| -rw-r--r-- | drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c | 3 | ||||
| -rw-r--r-- | include/drm/ttm/ttm_bo_driver.h | 41 | ||||
| -rw-r--r-- | include/drm/ttm/ttm_page_alloc.h | 18 |
9 files changed, 135 insertions, 120 deletions
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 3f116ba38e1f..3271001cd0f2 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -28,6 +28,7 @@ */ #include "drmP.h" +#include "ttm/ttm_page_alloc.h" #include "nouveau_drm.h" #include "nouveau_drv.h" @@ -1050,6 +1051,8 @@ nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence) struct ttm_bo_driver nouveau_bo_driver = { .ttm_tt_create = &nouveau_ttm_tt_create, + .ttm_tt_populate = &ttm_pool_populate, + .ttm_tt_unpopulate = &ttm_pool_unpopulate, .invalidate_caches = nouveau_bo_invalidate_caches, .init_mem_type = nouveau_bo_init_mem_type, .evict_flags = nouveau_bo_evict_flags, diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c index d8d7a8acae0c..3a82fcee0c30 100644 --- a/drivers/gpu/drm/radeon/radeon_ttm.c +++ b/drivers/gpu/drm/radeon/radeon_ttm.c @@ -581,6 +581,8 @@ struct ttm_tt *radeon_ttm_tt_create(struct ttm_bo_device *bdev, static struct ttm_bo_driver radeon_bo_driver = { .ttm_tt_create = &radeon_ttm_tt_create, + .ttm_tt_populate = &ttm_pool_populate, + .ttm_tt_unpopulate = &ttm_pool_unpopulate, .invalidate_caches = &radeon_invalidate_caches, .init_mem_type = &radeon_init_mem_type, .evict_flags = &radeon_evict_flags, diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 082fcaea583f..60f204d67dbb 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -244,7 +244,7 @@ static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, unsigned long page, pgprot_t prot) { - struct page *d = ttm_tt_get_page(ttm, page); + struct page *d = ttm->pages[page]; void *dst; if (!d) @@ -281,7 +281,7 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, unsigned long page, pgprot_t prot) { - struct page *s = ttm_tt_get_page(ttm, page); + struct page *s = ttm->pages[page]; void *src; if (!s) @@ -342,6 +342,12 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, if (old_iomap == NULL && ttm == NULL) goto out2; + if (ttm->state == tt_unpopulated) { + ret = ttm->bdev->driver->ttm_tt_populate(ttm); + if (ret) + goto out1; + } + add = 0; dir = 1; @@ -502,10 +508,16 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, { struct ttm_mem_reg *mem = &bo->mem; pgprot_t prot; struct ttm_tt *ttm = bo->ttm; - struct page *d; - int i; + int ret; BUG_ON(!ttm); + + if (ttm->state == tt_unpopulated) { + ret = ttm->bdev->driver->ttm_tt_populate(ttm); + if (ret) + return ret; + } + if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) { /* * We're mapping a single page, and the desired @@ -513,18 +525,9 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, */ map->bo_kmap_type = ttm_bo_map_kmap; - map->page = ttm_tt_get_page(ttm, start_page); + map->page = ttm->pages[start_page]; map->virtual = kmap(map->page); } else { - /* - * Populate the part we're mapping; - */ - for (i = start_page; i < start_page + num_pages; ++i) { - d = ttm_tt_get_page(ttm, i); - if (!d) - return -ENOMEM; - } - /* * We need to use vmap to get the desired page protection * or to make the buffer object look contiguous. diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 221b924acebe..54412848de88 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -174,18 +174,23 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) vma->vm_page_prot = (bo->mem.placement & TTM_PL_FLAG_CACHED) ? vm_get_page_prot(vma->vm_flags) : ttm_io_prot(bo->mem.placement, vma->vm_page_prot); + + /* Allocate all page at once, most common usage */ + if (ttm->bdev->driver->ttm_tt_populate(ttm)) { + retval = VM_FAULT_OOM; + goto out_io_unlock; + } } /* * Speculatively prefault a number of pages. Only error on * first page. */ - for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) { if (bo->mem.bus.is_iomem) pfn = ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT) + page_offset; else { - page = ttm_tt_get_page(ttm, page_offset); + page = ttm->pages[page_offset]; if (unlikely(!page && i == 0)) { retval = VM_FAULT_OOM; goto out_io_unlock; diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c index 0f3e6d2395b3..8d6267e434ab 100644 --- a/drivers/gpu/drm/ttm/ttm_page_alloc.c +++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c @@ -855,6 +855,63 @@ void ttm_page_alloc_fini(void) _manager = NULL; } +int ttm_pool_populate(struct ttm_tt *ttm) +{ + struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; + unsigned i; + int ret; + + if (ttm->state != tt_unpopulated) + return 0; + + for (i = 0; i < ttm->num_pages; ++i) { + ret = ttm_get_pages(&ttm->pages[i], ttm->page_flags, + ttm->caching_state, 1, + &ttm->dma_address[i]); + if (ret != 0) { + ttm_pool_unpopulate(ttm); + return -ENOMEM; + } + + ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], + false, false); + if (unlikely(ret != 0)) { + ttm_pool_unpopulate(ttm); + return -ENOMEM; + } + } + + if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { + ret = ttm_tt_swapin(ttm); + if (unlikely(ret != 0)) { + ttm_pool_unpopulate(ttm); + return ret; + } + } + + ttm->state = tt_unbound; + return 0; +} +EXPORT_SYMBOL(ttm_pool_populate); + +void ttm_pool_unpopulate(struct ttm_tt *ttm) +{ + unsigned i; + + for (i = 0; i < ttm->num_pages; ++i) { + if (ttm->pages[i]) { + ttm_mem_global_free_page(ttm->glob->mem_glob, + ttm->pages[i]); + ttm_put_pages(&ttm->pages[i], 1, + ttm->page_flags, + ttm->caching_state, + ttm->dma_address); + } + } + ttm->state = tt_unpopulated; +} +EXPORT_SYMBOL(ttm_pool_unpopulate); + int ttm_page_alloc_debugfs(struct seq_file *m, void *data) { struct ttm_page_pool *p; diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index fbc90dce1de8..77f0e6f79f30 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -43,8 +43,6 @@ #include "ttm/ttm_placement.h" #include "ttm/ttm_page_alloc.h" -static int ttm_tt_swapin(struct ttm_tt *ttm); - /** * Allocates storage for pointers to the pages that back the ttm. */ @@ -63,69 +61,6 @@ static void ttm_tt_free_page_directory(struct ttm_tt *ttm) ttm->dma_address = NULL; } -static struct page *__ttm_tt_get_page(struct ttm_tt *ttm, int index) -{ - struct page *p; - struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; - int ret; - - if (NULL == (p = ttm->pages[index])) { - - ret = ttm_get_pages(&p, ttm->page_flags, ttm->caching_state, 1, - &ttm->dma_address[index]); - if (ret != 0) - return NULL; - - ret = ttm_mem_global_alloc_page(mem_glob, p, false, false); - if (unlikely(ret != 0)) - goto out_err; - - ttm->pages[index] = p; - } - return p; -out_err: - ttm_put_pages(&p, 1, ttm->page_flags, - ttm->caching_state, &ttm->dma_address[index]); - return NULL; -} - -struct page *ttm_tt_get_page(struct ttm_tt *ttm, int index) -{ - int ret; - - if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { - ret = ttm_tt_swapin(ttm); - if (unlikely(ret != 0)) - return NULL; - } - return __ttm_tt_get_page(ttm, index); -} - -int ttm_tt_populate(struct ttm_tt *ttm) -{ - struct page *page; - unsigned long i; - int ret; - - if (ttm->state != tt_unpopulated) - return 0; - - if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { - ret = ttm_tt_swapin(ttm); - if (unlikely(ret != 0)) - return ret; - } - - for (i = 0; i < ttm->num_pages; ++i) { - page = __ttm_tt_get_page(ttm, i); - if (!page) - return -ENOMEM; - } - ttm->state = tt_unbound; - return 0; -} -EXPORT_SYMBOL(ttm_tt_populate); - #ifdef CONFIG_X86 static inline int ttm_tt_set_page_caching(struct page *p, enum ttm_caching_state c_old, @@ -227,21 +162,6 @@ int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement) } EXPORT_SYMBOL(ttm_tt_set_placement_caching); -static void ttm_tt_free_alloced_pages(struct ttm_tt *ttm) -{ - unsigned i; - - for (i = 0; i < ttm->num_pages; ++i) { - if (ttm->pages[i]) { - ttm_mem_global_free_page(ttm->glob->mem_glob, - ttm->pages[i]); - ttm_put_pages(&ttm->pages[i], 1, ttm->page_flags, - ttm->caching_state, &ttm->dma_address[i]); - } - } - ttm->state = tt_unpopulated; -} - void ttm_tt_destroy(struct ttm_tt *ttm) { if (unlikely(ttm == NULL)) @@ -252,7 +172,7 @@ void ttm_tt_destroy(struct ttm_tt *ttm) } if (likely(ttm->pages != NULL)) { - ttm_tt_free_alloced_pages(ttm); + ttm->bdev->driver->ttm_tt_unpopulate(ttm); ttm_tt_free_page_directory(ttm); } @@ -307,7 +227,7 @@ int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) if (ttm->state == tt_bound) return 0; - ret = ttm_tt_populate(ttm); + ret = ttm->bdev->driver->ttm_tt_populate(ttm); if (ret) return ret; @@ -321,7 +241,7 @@ int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) } EXPORT_SYMBOL(ttm_tt_bind); -static int ttm_tt_swapin(struct ttm_tt *ttm) +int ttm_tt_swapin(struct ttm_tt *ttm) { struct address_space *swap_space; struct file *swap_storage; @@ -343,7 +263,7 @@ static int ttm_tt_swapin(struct ttm_tt *ttm) ret = PTR_ERR(from_page); goto out_err; } - to_page = __ttm_tt_get_page(ttm, i); + to_page = ttm->pages[i]; if (unlikely(to_page == NULL)) goto out_err; @@ -364,7 +284,6 @@ static int ttm_tt_swapin(struct ttm_tt *ttm) return 0; out_err: - ttm_tt_free_alloced_pages(ttm); return ret; } @@ -416,7 +335,7 @@ int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage) page_cache_release(to_page); } - ttm_tt_free_alloced_pages(ttm); + ttm->bdev->driver->ttm_tt_unpopulate(ttm); ttm->swap_storage = swap_storage; ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; if (persistent_swap_storage) diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c index cc7243592425..3986d7468232 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c @@ -28,6 +28,7 @@ #include "vmwgfx_drv.h" #include "ttm/ttm_bo_driver.h" #include "ttm/ttm_placement.h" +#include "ttm/ttm_page_alloc.h" static uint32_t vram_placement_flags = TTM_PL_FLAG_VRAM | TTM_PL_FLAG_CACHED; @@ -334,6 +335,8 @@ static int vmw_sync_obj_wait(void *sync_obj, void *sync_arg, struct ttm_bo_driver vmw_bo_driver = { .ttm_tt_create = &vmw_ttm_tt_create, + .ttm_tt_populate = &ttm_pool_populate, + .ttm_tt_unpopulate = &ttm_pool_unpopulate, .invalidate_caches = vmw_invalidate_caches, .init_mem_type = vmw_init_mem_type, .evict_flags = vmw_evict_flags, diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index 6b8c5cd690a0..ae06e421cf90 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -319,6 +319,26 @@ struct ttm_bo_driver { struct page *dummy_read_page); /** + * ttm_tt_populate + * + * @ttm: The struct ttm_tt to contain the backing pages. + * + * Allocate all backing pages + * Returns: + * -ENOMEM: Out of memory. + */ + int (*ttm_tt_populate)(struct ttm_tt *ttm); + + /** + * ttm_tt_unpopulate + * + * @ttm: The struct ttm_tt to contain the backing pages. + * + * Free all backing page + */ + void (*ttm_tt_unpopulate)(struct ttm_tt *ttm); + + /** * struct ttm_bo_driver member invalidate_caches * * @bdev: the buffer object device. @@ -585,15 +605,6 @@ extern int ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev, extern int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem); /** - * ttm_tt_populate: - * - * @ttm: The struct ttm_tt to contain the backing pages. - * - * Add backing pages to all of @ttm - */ -extern int ttm_tt_populate(struct ttm_tt *ttm); - -/** * ttm_ttm_destroy: * * @ttm: The struct ttm_tt. @@ -612,19 +623,13 @@ extern void ttm_tt_destroy(struct ttm_tt *ttm); extern void ttm_tt_unbind(struct ttm_tt *ttm); /** - * ttm_ttm_destroy: + * ttm_tt_swapin: * * @ttm: The struct ttm_tt. - * @index: Index of the desired page. * - * Return a pointer to the struct page backing @ttm at page - * index @index. If the page is unpopulated, one will be allocated to - * populate that index. - * - * Returns: - * NULL on OOM. + * Swap in a previously swap out ttm_tt. */ -extern struct page *ttm_tt_get_page(struct ttm_tt *ttm, int index); +extern int ttm_tt_swapin(struct ttm_tt *ttm); /** * ttm_tt_cache_flush: diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h index fe61c8d85684..18deeee23494 100644 --- a/include/drm/ttm/ttm_page_alloc.h +++ b/include/drm/ttm/ttm_page_alloc.h @@ -68,6 +68,24 @@ int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages); void ttm_page_alloc_fini(void); /** + * ttm_pool_populate: + * + * @ttm: The struct ttm_tt to contain the backing pages. + * + * Add backing pages to all of @ttm + */ +extern int ttm_pool_populate(struct ttm_tt *ttm); + +/** + * ttm_pool_unpopulate: + * + * @ttm: The struct ttm_tt which to free backing pages. + * + * Free all pages of @ttm + */ +extern void ttm_pool_unpopulate(struct ttm_tt *ttm); + +/** * Output the state of pools to debugfs file */ extern int ttm_page_alloc_debugfs(struct seq_file *m, void *data); |