diff options
| -rw-r--r-- | arch/x86/include/asm/cacheflush.h | 2 | ||||
| -rw-r--r-- | arch/x86/mm/pageattr.c | 53 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nouveau_debugfs.c | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/radeon/radeon_ttm.c | 13 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/Makefile | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_memory.c | 7 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_page_alloc.c | 845 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_tt.c | 44 | ||||
| -rw-r--r-- | include/drm/ttm/ttm_page_alloc.h | 74 |
9 files changed, 1007 insertions, 36 deletions
diff --git a/arch/x86/include/asm/cacheflush.h b/arch/x86/include/asm/cacheflush.h index 634c40a739a6..d92d63a6286b 100644 --- a/arch/x86/include/asm/cacheflush.h +++ b/arch/x86/include/asm/cacheflush.h @@ -139,9 +139,11 @@ int set_memory_np(unsigned long addr, int numpages); int set_memory_4k(unsigned long addr, int numpages); int set_memory_array_uc(unsigned long *addr, int addrinarray); +int set_memory_array_wc(unsigned long *addr, int addrinarray); int set_memory_array_wb(unsigned long *addr, int addrinarray); int set_pages_array_uc(struct page **pages, int addrinarray); +int set_pages_array_wc(struct page **pages, int addrinarray); int set_pages_array_wb(struct page **pages, int addrinarray); /* diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c index 28195c350b97..532e7933d606 100644 --- a/arch/x86/mm/pageattr.c +++ b/arch/x86/mm/pageattr.c @@ -997,7 +997,8 @@ out_err: } EXPORT_SYMBOL(set_memory_uc); -int set_memory_array_uc(unsigned long *addr, int addrinarray) +int _set_memory_array(unsigned long *addr, int addrinarray, + unsigned long new_type) { int i, j; int ret; @@ -1007,13 +1008,19 @@ int set_memory_array_uc(unsigned long *addr, int addrinarray) */ for (i = 0; i < addrinarray; i++) { ret = reserve_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE, - _PAGE_CACHE_UC_MINUS, NULL); + new_type, NULL); if (ret) goto out_free; } ret = change_page_attr_set(addr, addrinarray, __pgprot(_PAGE_CACHE_UC_MINUS), 1); + + if (!ret && new_type == _PAGE_CACHE_WC) + ret = change_page_attr_set_clr(addr, addrinarray, + __pgprot(_PAGE_CACHE_WC), + __pgprot(_PAGE_CACHE_MASK), + 0, CPA_ARRAY, NULL); if (ret) goto out_free; @@ -1025,8 +1032,19 @@ out_free: return ret; } + +int set_memory_array_uc(unsigned long *addr, int addrinarray) +{ + return _set_memory_array(addr, addrinarray, _PAGE_CACHE_UC_MINUS); +} EXPORT_SYMBOL(set_memory_array_uc); +int set_memory_array_wc(unsigned long *addr, int addrinarray) +{ + return _set_memory_array(addr, addrinarray, _PAGE_CACHE_WC); +} +EXPORT_SYMBOL(set_memory_array_wc); + int _set_memory_wc(unsigned long addr, int numpages) { int ret; @@ -1153,26 +1171,34 @@ int set_pages_uc(struct page *page, int numpages) } EXPORT_SYMBOL(set_pages_uc); -int set_pages_array_uc(struct page **pages, int addrinarray) +static int _set_pages_array(struct page **pages, int addrinarray, + unsigned long new_type) { unsigned long start; unsigned long end; int i; int free_idx; + int ret; for (i = 0; i < addrinarray; i++) { if (PageHighMem(pages[i])) continue; start = page_to_pfn(pages[i]) << PAGE_SHIFT; end = start + PAGE_SIZE; - if (reserve_memtype(start, end, _PAGE_CACHE_UC_MINUS, NULL)) + if (reserve_memtype(start, end, new_type, NULL)) goto err_out; } - if (cpa_set_pages_array(pages, addrinarray, - __pgprot(_PAGE_CACHE_UC_MINUS)) == 0) { - return 0; /* Success */ - } + ret = cpa_set_pages_array(pages, addrinarray, + __pgprot(_PAGE_CACHE_UC_MINUS)); + if (!ret && new_type == _PAGE_CACHE_WC) + ret = change_page_attr_set_clr(NULL, addrinarray, + __pgprot(_PAGE_CACHE_WC), + __pgprot(_PAGE_CACHE_MASK), + 0, CPA_PAGES_ARRAY, pages); + if (ret) + goto err_out; + return 0; /* Success */ err_out: free_idx = i; for (i = 0; i < free_idx; i++) { @@ -1184,8 +1210,19 @@ err_out: } return -EINVAL; } + +int set_pages_array_uc(struct page **pages, int addrinarray) +{ + return _set_pages_array(pages, addrinarray, _PAGE_CACHE_UC_MINUS); +} EXPORT_SYMBOL(set_pages_array_uc); +int set_pages_array_wc(struct page **pages, int addrinarray) +{ + return _set_pages_array(pages, addrinarray, _PAGE_CACHE_WC); +} +EXPORT_SYMBOL(set_pages_array_wc); + int set_pages_wb(struct page *page, int numpages) { unsigned long addr = (unsigned long)page_address(page); diff --git a/drivers/gpu/drm/nouveau/nouveau_debugfs.c b/drivers/gpu/drm/nouveau/nouveau_debugfs.c index a251886a0ce6..7933de4aff2e 100644 --- a/drivers/gpu/drm/nouveau/nouveau_debugfs.c +++ b/drivers/gpu/drm/nouveau/nouveau_debugfs.c @@ -33,6 +33,8 @@ #include "drmP.h" #include "nouveau_drv.h" +#include <ttm/ttm_page_alloc.h> + static int nouveau_debugfs_channel_info(struct seq_file *m, void *data) { @@ -159,6 +161,7 @@ static struct drm_info_list nouveau_debugfs_list[] = { { "chipset", nouveau_debugfs_chipset_info, 0, NULL }, { "memory", nouveau_debugfs_memory_info, 0, NULL }, { "vbios.rom", nouveau_debugfs_vbios_image, 0, NULL }, + { "ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL }, }; #define NOUVEAU_DEBUGFS_ENTRIES ARRAY_SIZE(nouveau_debugfs_list) diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c index d031b6863082..f06533676e7d 100644 --- a/drivers/gpu/drm/radeon/radeon_ttm.c +++ b/drivers/gpu/drm/radeon/radeon_ttm.c @@ -33,6 +33,7 @@ #include <ttm/ttm_bo_driver.h> #include <ttm/ttm_placement.h> #include <ttm/ttm_module.h> +#include <ttm/ttm_page_alloc.h> #include <drm/drmP.h> #include <drm/radeon_drm.h> #include <linux/seq_file.h> @@ -745,8 +746,8 @@ static int radeon_mm_dump_table(struct seq_file *m, void *data) static int radeon_ttm_debugfs_init(struct radeon_device *rdev) { #if defined(CONFIG_DEBUG_FS) - static struct drm_info_list radeon_mem_types_list[RADEON_DEBUGFS_MEM_TYPES]; - static char radeon_mem_types_names[RADEON_DEBUGFS_MEM_TYPES][32]; + static struct drm_info_list radeon_mem_types_list[RADEON_DEBUGFS_MEM_TYPES+1]; + static char radeon_mem_types_names[RADEON_DEBUGFS_MEM_TYPES+1][32]; unsigned i; for (i = 0; i < RADEON_DEBUGFS_MEM_TYPES; i++) { @@ -763,7 +764,13 @@ static int radeon_ttm_debugfs_init(struct radeon_device *rdev) radeon_mem_types_list[i].data = &rdev->mman.bdev.man[TTM_PL_TT].manager; } - return radeon_debugfs_add_files(rdev, radeon_mem_types_list, RADEON_DEBUGFS_MEM_TYPES); + /* Add ttm page pool to debugfs */ + sprintf(radeon_mem_types_names[i], "ttm_page_pool"); + radeon_mem_types_list[i].name = radeon_mem_types_names[i]; + radeon_mem_types_list[i].show = &ttm_page_alloc_debugfs; + radeon_mem_types_list[i].driver_features = 0; + radeon_mem_types_list[i].data = NULL; + return radeon_debugfs_add_files(rdev, radeon_mem_types_list, RADEON_DEBUGFS_MEM_TYPES+1); #endif return 0; diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile index 1e138f5bae09..4256e2006476 100644 --- a/drivers/gpu/drm/ttm/Makefile +++ b/drivers/gpu/drm/ttm/Makefile @@ -4,6 +4,6 @@ ccflags-y := -Iinclude/drm ttm-y := ttm_agp_backend.o ttm_memory.o ttm_tt.o ttm_bo.o \ ttm_bo_util.o ttm_bo_vm.o ttm_module.o ttm_global.o \ - ttm_object.o ttm_lock.o ttm_execbuf_util.o + ttm_object.o ttm_lock.o ttm_execbuf_util.o ttm_page_alloc.o obj-$(CONFIG_DRM_TTM) += ttm.o diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c index 801b702566e6..e70ddd82dc02 100644 --- a/drivers/gpu/drm/ttm/ttm_memory.c +++ b/drivers/gpu/drm/ttm/ttm_memory.c @@ -27,6 +27,7 @@ #include "ttm/ttm_memory.h" #include "ttm/ttm_module.h" +#include "ttm/ttm_page_alloc.h" #include <linux/spinlock.h> #include <linux/sched.h> #include <linux/wait.h> @@ -393,6 +394,7 @@ int ttm_mem_global_init(struct ttm_mem_global *glob) "Zone %7s: Available graphics memory: %llu kiB.\n", zone->name, (unsigned long long) zone->max_mem >> 10); } + ttm_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); return 0; out_no_zone: ttm_mem_global_release(glob); @@ -405,6 +407,9 @@ void ttm_mem_global_release(struct ttm_mem_global *glob) unsigned int i; struct ttm_mem_zone *zone; + /* let the page allocator first stop the shrink work. */ + ttm_page_alloc_fini(); + flush_workqueue(glob->swap_queue); destroy_workqueue(glob->swap_queue); glob->swap_queue = NULL; @@ -412,7 +417,7 @@ void ttm_mem_global_release(struct ttm_mem_global *glob) zone = glob->zones[i]; kobject_del(&zone->kobj); kobject_put(&zone->kobj); - } + } kobject_del(&glob->kobj); kobject_put(&glob->kobj); } diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c new file mode 100644 index 000000000000..0d9a42c2394f --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c @@ -0,0 +1,845 @@ +/* + * Copyright (c) Red Hat Inc. + + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sub license, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie <airlied@redhat.com> + * Jerome Glisse <jglisse@redhat.com> + * Pauli Nieminen <suokkos@gmail.com> + */ + +/* simple list based uncached page pool + * - Pool collects resently freed pages for reuse + * - Use page->lru to keep a free list + * - doesn't track currently in use pages + */ +#include <linux/list.h> +#include <linux/spinlock.h> +#include <linux/highmem.h> +#include <linux/mm_types.h> +#include <linux/module.h> +#include <linux/mm.h> +#include <linux/seq_file.h> /* for seq_printf */ +#include <linux/slab.h> + +#include <asm/atomic.h> +#include <asm/agp.h> + +#include "ttm/ttm_bo_driver.h" +#include "ttm/ttm_page_alloc.h" + + +#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) +#define SMALL_ALLOCATION 16 +#define FREE_ALL_PAGES (~0U) +/* times are in msecs */ +#define PAGE_FREE_INTERVAL 1000 + +/** + * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages. + * + * @lock: Protects the shared pool from concurrnet access. Must be used with + * irqsave/irqrestore variants because pool allocator maybe called from + * delayed work. + * @fill_lock: Prevent concurrent calls to fill. + * @list: Pool of free uc/wc pages for fast reuse. + * @gfp_flags: Flags to pass for alloc_page. + * @npages: Number of pages in pool. + */ +struct ttm_page_pool { + spinlock_t lock; + bool fill_lock; + struct list_head list; + int gfp_flags; + unsigned npages; + char *name; + unsigned long nfrees; + unsigned long nrefills; +}; + +/** + * Limits for the pool. They are handled without locks because only place where + * they may change is in sysfs store. They won't have immediate effect anyway + * so forcing serialiazation to access them is pointless. + */ + +struct ttm_pool_opts { + unsigned alloc_size; + unsigned max_size; + unsigned small; +}; + +#define NUM_POOLS 4 + +/** + * struct ttm_pool_manager - Holds memory pools for fst allocation + * + * Manager is read only object for pool code so it doesn't need locking. + * + * @free_interval: minimum number of jiffies between freeing pages from pool. + * @page_alloc_inited: reference counting for pool allocation. + * @work: Work that is used to shrink the pool. Work is only run when there is + * some pages to free. + * @small_allocation: Limit in number of pages what is small allocation. + * + * @pools: All pool objects in use. + **/ +struct ttm_pool_manager { + struct kobject kobj; + struct shrinker mm_shrink; + atomic_t page_alloc_inited; + struct ttm_pool_opts options; + + union { + struct ttm_page_pool pools[NUM_POOLS]; + struct { + struct ttm_page_pool wc_pool; + struct ttm_page_pool uc_pool; + struct ttm_page_pool wc_pool_dma32; + struct ttm_page_pool uc_pool_dma32; + } ; + }; +}; + +static struct attribute ttm_page_pool_max = { + .name = "pool_max_size", + .mode = S_IRUGO | S_IWUSR +}; +static struct attribute ttm_page_pool_small = { + .name = "pool_small_allocation", + .mode = S_IRUGO | S_IWUSR +}; +static struct attribute ttm_page_pool_alloc_size = { + .name = "pool_allocation_size", + .mode = S_IRUGO | S_IWUSR +}; + +static struct attribute *ttm_pool_attrs[] = { + &ttm_page_pool_max, + &ttm_page_pool_small, + &ttm_page_pool_alloc_size, + NULL +}; + +static void ttm_pool_kobj_release(struct kobject *kobj) +{ + struct ttm_pool_manager *m = + container_of(kobj, struct ttm_pool_manager, kobj); + (void)m; +} + +static ssize_t ttm_pool_store(struct kobject *kobj, + struct attribute *attr, const char *buffer, size_t size) +{ + struct ttm_pool_manager *m = + container_of(kobj, struct ttm_pool_manager, kobj); + int chars; + unsigned val; + chars = sscanf(buffer, "%u", &val); + if (chars == 0) + return size; + + /* Convert kb to number of pages */ + val = val / (PAGE_SIZE >> 10); + + if (attr == &ttm_page_pool_max) + m->options.max_size = val; + else if (attr == &ttm_page_pool_small) + m->options.small = val; + else if (attr == &ttm_page_pool_alloc_size) { + if (val > NUM_PAGES_TO_ALLOC*8) { + printk(KERN_ERR "[ttm] Setting allocation size to %lu " + "is not allowed. Recomended size is " + "%lu\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); + return size; + } else if (val > NUM_PAGES_TO_ALLOC) { + printk(KERN_WARNING "[ttm] Setting allocation size to " + "larger than %lu is not recomended.\n", + NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); + } + m->options.alloc_size = val; + } + + return size; +} + +static ssize_t ttm_pool_show(struct kobject *kobj, + struct attribute *attr, char *buffer) +{ + struct ttm_pool_manager *m = + container_of(kobj, struct ttm_pool_manager, kobj); + unsigned val = 0; + + if (attr == &ttm_page_pool_max) + val = m->options.max_size; + else if (attr == &ttm_page_pool_small) + val = m->options.small; + else if (attr == &ttm_page_pool_alloc_size) + val = m->options.alloc_size; + + val = val * (PAGE_SIZE >> 10); + + return snprintf(buffer, PAGE_SIZE, "%u\n", val); +} + +static const struct sysfs_ops ttm_pool_sysfs_ops = { + .show = &ttm_pool_show, + .store = &ttm_pool_store, +}; + +static struct kobj_type ttm_pool_kobj_type = { + .release = &ttm_pool_kobj_release, + .sysfs_ops = &ttm_pool_sysfs_ops, + .default_attrs = ttm_pool_attrs, +}; + +static struct ttm_pool_manager _manager = { + .page_alloc_inited = ATOMIC_INIT(0) +}; + +#ifndef CONFIG_X86 +static int set_pages_array_wb(struct page **pages, int addrinarray) +{ +#ifdef TTM_HAS_AGP + int i; + + for (i = 0; i < addrinarray; i++) + unmap_page_from_agp(pages[i]); +#endif + return 0; +} + +static int set_pages_array_wc(struct page **pages, int addrinarray) +{ +#ifdef TTM_HAS_AGP + int i; + + for (i = 0; i < addrinarray; i++) + map_page_into_agp(pages[i]); +#endif + return 0; +} + +static int set_pages_array_uc(struct page **pages, int addrinarray) +{ +#ifdef TTM_HAS_AGP + int i; + + for (i = 0; i < addrinarray; i++) + map_page_into_agp(pages[i]); +#endif + return 0; +} +#endif + +/** + * Select the right pool or requested caching state and ttm flags. */ +static struct ttm_page_pool *ttm_get_pool(int flags, + enum ttm_caching_state cstate) +{ + int pool_index; + + if (cstate == tt_cached) + return NULL; + + if (cstate == tt_wc) + pool_index = 0x0; + else + pool_index = 0x1; + + if (flags & TTM_PAGE_FLAG_DMA32) + pool_index |= 0x2; + + return &_manager.pools[pool_index]; +} + +/* set memory back to wb and free the pages. */ +static void ttm_pages_put(struct page *pages[], unsigned npages) +{ + unsigned i; + if (set_pages_array_wb(pages, npages)) + printk(KERN_ERR "[ttm] Failed to set %d pages to wb!\n", + npages); + for (i = 0; i < npages; ++i) + __free_page(pages[i]); +} + +static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, + unsigned freed_pages) +{ + pool->npages -= freed_pages; + pool->nfrees += freed_pages; +} + +/** + * Free pages from pool. + * + * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC + * number of pages in one go. + * + * @pool: to free the pages from + * @free_all: If set to true will free all pages in pool + **/ +static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free) +{ + unsigned long irq_flags; + struct page *p; + struct page **pages_to_free; + unsigned freed_pages = 0, + npages_to_free = nr_free; + + if (NUM_PAGES_TO_ALLOC < nr_free) + npages_to_free = NUM_PAGES_TO_ALLOC; + + pages_to_free = kmalloc(npages_to_free * sizeof(struct page *), + GFP_KERNEL); + if (!pages_to_free) { + printk(KERN_ERR "Failed to allocate memory for pool free operation.\n"); + return 0; + } + +restart: + spin_lock_irqsave(&pool->lock, irq_flags); + + list_for_each_entry_reverse(p, &pool->list, lru) { + if (freed_pages >= npages_to_free) + break; + + pages_to_free[freed_pages++] = p; + /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ + if (freed_pages >= NUM_PAGES_TO_ALLOC) { + /* remove range of pages from the pool */ + __list_del(p->lru.prev, &pool->list); + + ttm_pool_update_free_locked(pool, freed_pages); + /** + * Because changing page caching is costly + * we unlock the pool to prevent stalling. + */ + spin_unlock_irqrestore(&pool->lock, irq_flags); + + ttm_pages_put(pages_to_free, freed_pages); + if (likely(nr_free != FREE_ALL_PAGES)) + nr_free -= freed_pages; + + if (NUM_PAGES_TO_ALLOC >= nr_free) + npages_to_free = nr_free; + else + npages_to_free = NUM_PAGES_TO_ALLOC; + + freed_pages = 0; + + /* free all so restart the processing */ + if (nr_free) + goto restart; + + /* Not allowed to fall tough or break because + * following context is inside spinlock while we are + * outside here. + */ + goto out; + + } + } + + /* remove range of pages from the pool */ + if (freed_pages) { + __list_del(&p->lru, &pool->list); + + ttm_pool_update_free_locked(pool, freed_pages); + nr_free -= freed_pages; + } + + spin_unlock_irqrestore(&pool->lock, irq_flags); + + if (freed_pages) + ttm_pages_put(pages_to_free, freed_pages); +out: + kfree(pages_to_free); + return nr_free; +} + +/* Get good estimation how many pages are free in pools */ +static int ttm_pool_get_num_unused_pages(void) +{ + unsigned i; + int total = 0; + for (i = 0; i < NUM_POOLS; ++i) + total += _manager.pools[i].npages; + + return total; +} + +/** + * Calback for mm to request pool to reduce number of page held. + */ +static int ttm_pool_mm_shrink(int shrink_pages, gfp_t gfp_mask) +{ + static atomic_t start_pool = ATOMIC_INIT(0); + unsigned i; + unsigned pool_offset = atomic_add_return(1, &start_pool); + struct ttm_page_pool *pool; + + pool_offset = pool_offset % NUM_POOLS; + /* select start pool in round robin fashion */ + for (i = 0; i < NUM_POOLS; ++i) { + unsigned nr_free = shrink_pages; + if (shrink_pages == 0) + break; + pool = &_manager.pools[(i + pool_offset)%NUM_POOLS]; + shrink_pages = ttm_page_pool_free(pool, nr_free); + } + /* return estimated number of unused pages in pool */ + return ttm_pool_get_num_unused_pages(); +} + +static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) +{ + manager->mm_shrink.shrink = &ttm_pool_mm_shrink; + manager->mm_shrink.seeks = 1; + register_shrinker(&manager->mm_shrink); +} + +static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) +{ + unregister_shrinker(&manager->mm_shrink); +} + +static int ttm_set_pages_caching(struct page **pages, + enum ttm_caching_state cstate, unsigned cpages) +{ + int r = 0; + /* Set page caching */ + switch (cstate) { + case tt_uncached: + r = set_pages_array_uc(pages, cpages); + if (r) + printk(KERN_ERR "[ttm] Failed to set %d pages to uc!\n", + cpages); + break; + case tt_wc: + r = set_pages_array_wc(pages, cpages); + if (r) + printk(KERN_ERR "[ttm] Failed to set %d pages to wc!\n", + cpages); + break; + default: + break; + } + return r; +} + +/** + * Free pages the pages that failed to change the caching state. If there is + * any pages that have changed their caching state already put them to the + * pool. + */ +static void ttm_handle_caching_state_failure(struct list_head *pages, + int ttm_flags, enum ttm_caching_state cstate, + struct page **failed_pages, unsigned cpages) +{ + unsigned i; + /* Failed pages has to be reed */ + for (i = 0; i < cpages; ++i) { + list_del(&failed_pages[i]->lru); + __free_page(failed_pages[i]); + } +} + +/** + * Allocate new pages with correct caching. + * + * This function is reentrant if caller updates count depending on number of + * pages returned in pages array. + */ +static int ttm_alloc_new_pages(struct list_head *pages, int gfp_flags, + int ttm_flags, enum ttm_caching_state cstate, unsigned count) +{ + struct page **caching_array; + struct page *p; + int r = 0; + unsigned i, cpages; + unsigned max_cpages = min(count, + (unsigned)(PAGE_SIZE/sizeof(struct page *))); + + /* allocate array for page caching change */ + caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); + + if (!caching_array) { + printk(KERN_ERR "[ttm] unable to allocate table for new pages."); + return -ENOMEM; + } + + for (i = 0, cpages = 0; i < count; ++i) { + p = alloc_page(gfp_flags); + + if (!p) { + printk(KERN_ERR "[ttm] unable to get page %u\n", i); + + /* store already allocated pages in the pool after + * setting the caching state */ + if (cpages) { + r = ttm_set_pages_caching(caching_array, cstate, cpages); + if (r) + ttm_handle_caching_state_failure(pages, + ttm_flags, cstate, + caching_array, cpages); + } + r = -ENOMEM; + goto out; + } + +#ifdef CONFIG_HIGHMEM + /* gfp flags of highmem page should never be dma32 so we + * we should be fine in such case + */ + if (!PageHighMem(p)) +#endif + { + caching_array[cpages++] = p; + if (cpages == max_cpages) { + + r = ttm_set_pages_caching(caching_array, + cstate, cpages); + if (r) { + ttm_handle_caching_state_failure(pages, + ttm_flags, cstate, + caching_array, cpages); + goto out; + } + cpages = 0; + } + } + + list_add(&p->lru, pages); + } + + if (cpages) { + r = ttm_set_pages_caching(caching_array, cstate, cpages); + if (r) + ttm_handle_caching_state_failure(pages, + ttm_flags, cstate, + caching_array, cpages); + } +out: + kfree(caching_array); + + return r; +} + +/** + * Fill the given pool if there isn't enough pages and requested number of + * pages is small. + */ +static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, + int ttm_flags, enum ttm_caching_state cstate, unsigned count, + unsigned long *irq_flags) +{ + struct page *p; + int r; + unsigned cpages = 0; + /** + * Only allow one pool fill operation at a time. + * If pool doesn't have enough pages for the allocation new pages are + * allocated from outside of pool. + */ + if (pool->fill_lock) + return; + + pool->fill_lock = true; + + /* If allocation request is small and there is not enough + * pages in pool we fill the pool first */ + if (count < _manager.options.small + && count > pool->npages) { + struct list_head new_pages; + unsigned alloc_size = _manager.options.alloc_size; + + /** + * Can't change page caching if in irqsave context. We have to + * drop the pool->lock. + */ + spin_unlock_irqrestore(&pool->lock, *irq_flags); + + INIT_LIST_HEAD(&new_pages); + r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags, + cstate, alloc_size); + spin_lock_irqsave(&pool->lock, *irq_flags); + + if (!r) { + list_splice(&new_pages, &pool->list); + ++pool->nrefills; + pool->npages += alloc_size; + } else { + printk(KERN_ERR "[ttm] Failed to fill pool (%p).", pool); + /* If we have any pages left put them to the pool. */ + list_for_each_entry(p, &pool->list, lru) { + ++cpages; + } + list_splice(&new_pages, &pool->list); + pool->npages += cpages; + } + + } + pool->fill_lock = false; +} + +/** + * Cut count nubmer of pages from the pool and put them to return list + * + * @return count of pages still to allocate to fill the request. + */ +static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, + struct list_head *pages, int ttm_flags, + enum ttm_caching_state cstate, unsigned count) +{ + unsigned long irq_flags; + struct list_head *p; + unsigned i; + + spin_lock_irqsave(&pool->lock, irq_flags); + ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags); + + if (count >= pool->npages) { + /* take all pages from the pool */ + list_splice_init(&pool->list, pages); + count -= pool->npages; + pool->npages = 0; + goto out; + } + /* find the last pages to include for requested number of pages. Split + * pool to begin and halves to reduce search space. */ + if (count <= pool->npages/2) { + i = 0; + list_for_each(p, &pool->list) { + if (++i == count) + break; + } + } else { + i = pool->npages + 1; + list_for_each_prev(p, &pool->list) { + if (--i == count) + break; + } + } + /* Cut count number of pages from pool */ + list_cut_position(pages, &pool->list, p); + pool->npages -= count; + count = 0; +out: + spin_unlock_irqrestore(&pool->lock, irq_flags); + return count; +} + +/* + * On success pages list will hold count number of correctly + * cached pages. + */ +int ttm_get_pages(struct list_head *pages, int flags, + enum ttm_caching_state cstate, unsigned count) +{ + struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); + struct page *p = NULL; + int gfp_flags = 0; + int r; + + /* set zero flag for page allocation if required */ + if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) + gfp_flags |= __GFP_ZERO; + + /* No pool for cached pages */ + if (pool == NULL) { + if (flags & TTM_PAGE_FLAG_DMA32) + gfp_flags |= GFP_DMA32; + else + gfp_flags |= __GFP_HIGHMEM; + + for (r = 0; r < count; ++r) { + p = alloc_page(gfp_flags); + if (!p) { + + printk(KERN_ERR "[ttm] unable to allocate page."); + return -ENOMEM; + } + + list_add(&p->lru, pages); + } + return 0; + } + + + /* combine zero flag to pool flags */ + gfp_flags |= pool->gfp_flags; + + /* First we take pages from the pool */ + count = ttm_page_pool_get_pages(pool, pages, flags, cstate, count); + + /* clear the pages coming from the pool if requested */ + if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { + list_for_each_entry(p, pages, lru) { + clear_page(page_address(p)); + } + } + + /* If pool didn't have enough pages allocate new one. */ + if (count > 0) { + /* ttm_alloc_new_pages doesn't reference pool so we can run + * multiple requests in parallel. + **/ + r = ttm_alloc_new_pages(pages, gfp_flags, flags, cstate, count); + if (r) { + /* If there is any pages in the list put them back to + * the pool. */ + printk(KERN_ERR "[ttm] Failed to allocate extra pages " + "for large request."); + ttm_put_pages(pages, 0, flags, cstate); + return r; + } + } + + + return 0; +} + +/* Put all pages in pages list to correct pool to wait for reuse */ +void ttm_put_pages(struct list_head *pages, unsigned page_count, int flags, + enum ttm_caching_state cstate) +{ + unsigned long irq_flags; + struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); + struct page *p, *tmp; + + if (pool == NULL) { + /* No pool for this memory type so free the pages */ + + list_for_each_entry_safe(p, tmp, pages, lru) { + __free_page(p); + } + /* Make the pages list empty */ + INIT_LIST_HEAD(pages); + return; + } + if (page_count == 0) { + list_for_each_entry_safe(p, tmp, pages, lru) { + ++page_count; + } + } + + spin_lock_irqsave(&pool->lock, irq_flags); + list_splice_init(pages, &pool->list); + pool->npages += page_count; + /* Check that we don't go over the pool limit */ + page_count = 0; + if (pool->npages > _manager.options.max_size) { + page_count = pool->npages - _manager.options.max_size; + /* free at least NUM_PAGES_TO_ALLOC number of pages + * to reduce calls to set_memory_wb */ + if (page_count < NUM_PAGES_TO_ALLOC) + page_count = NUM_PAGES_TO_ALLOC; + } + spin_unlock_irqrestore(&pool->lock, irq_flags); + if (page_count) + ttm_page_pool_free(pool, page_count); +} + +static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags, + char *name) +{ + spin_lock_init(&pool->lock); + pool->fill_lock = false; + INIT_LIST_HEAD(&pool->list); + pool->npages = pool->nfrees = 0; + pool->gfp_flags = flags; + pool->name = name; +} + +int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) +{ + int ret; + if (atomic_add_return(1, &_manager.page_alloc_inited) > 1) + return 0; + + printk(KERN_INFO "[ttm] Initializing pool allocator.\n"); + + ttm_page_pool_init_locked(&_manager.wc_pool, GFP_HIGHUSER, "wc"); + + ttm_page_pool_init_locked(&_manager.uc_pool, GFP_HIGHUSER, "uc"); + + ttm_page_pool_init_locked(&_manager.wc_pool_dma32, GFP_USER | GFP_DMA32, + "wc dma"); + + ttm_page_pool_init_locked(&_manager.uc_pool_dma32, GFP_USER | GFP_DMA32, + "uc dma"); + + _manager.options.max_size = max_pages; + _manager.options.small = SMALL_ALLOCATION; + _manager.options.alloc_size = NUM_PAGES_TO_ALLOC; + + kobject_init(&_manager.kobj, &ttm_pool_kobj_type); + ret = kobject_add(&_manager.kobj, &glob->kobj, "pool"); + if (unlikely(ret != 0)) { + kobject_put(&_manager.kobj); + return ret; + } + + ttm_pool_mm_shrink_init(&_manager); + + return 0; +} + +void ttm_page_alloc_fini() +{ + int i; + + if (atomic_sub_return(1, &_manager.page_alloc_inited) > 0) + return; + + printk(KERN_INFO "[ttm] Finilizing pool allocator.\n"); + ttm_pool_mm_shrink_fini(&_manager); + + for (i = 0; i < NUM_POOLS; ++i) + ttm_page_pool_free(&_manager.pools[i], FREE_ALL_PAGES); + + kobject_put(&_manager.kobj); +} + +int ttm_page_alloc_debugfs(struct seq_file *m, void *data) +{ + struct ttm_page_pool *p; + unsigned i; + char *h[] = {"pool", "refills", "pages freed", "size"}; + if (atomic_read(&_manager.page_alloc_inited) == 0) { + seq_printf(m, "No pool allocator running.\n"); + return 0; + } + seq_printf(m, "%6s %12s %13s %8s\n", + h[0], h[1], h[2], h[3]); + for (i = 0; i < NUM_POOLS; ++i) { + p = &_manager.pools[i]; + + seq_printf(m, "%6s %12ld %13ld %8d\n", + p->name, p->nrefills, + p->nfrees, p->npages); + } + return 0; +} +EXPORT_SYMBOL(ttm_page_alloc_debugfs); diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index d5fd5b8faeb3..a7bab87a548b 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -39,6 +39,7 @@ #include "ttm/ttm_module.h" #include "ttm/ttm_bo_driver.h" #include "ttm/ttm_placement.h" +#include "ttm/ttm_page_alloc.h" static int ttm_tt_swapin(struct ttm_tt *ttm); @@ -56,21 +57,6 @@ static void ttm_tt_free_page_directory(struct ttm_tt *ttm) ttm->pages = NULL; } -static struct page *ttm_tt_alloc_page(unsigned page_flags) -{ - gfp_t gfp_flags = GFP_USER; - - if (page_flags & TTM_PAGE_FLAG_ZERO_ALLOC) - gfp_flags |= __GFP_ZERO; - - if (page_flags & TTM_PAGE_FLAG_DMA32) - gfp_flags |= __GFP_DMA32; - else - gfp_flags |= __GFP_HIGHMEM; - - return alloc_page(gfp_flags); -} - static void ttm_tt_free_user_pages(struct ttm_tt *ttm) { int write; @@ -111,15 +97,21 @@ static void ttm_tt_free_user_pages(struct ttm_tt *ttm) static struct page *__ttm_tt_get_page(struct ttm_tt *ttm, int index) { struct page *p; + struct list_head h; struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; int ret; while (NULL == (p = ttm->pages[index])) { - p = ttm_tt_alloc_page(ttm->page_flags); - if (!p) + INIT_LIST_HEAD(&h); + + ret = ttm_get_pages(&h, ttm->page_flags, ttm->caching_state, 1); + + if (ret != 0) return NULL; + p = list_first_entry(&h, struct page, lru); + ret = ttm_mem_global_alloc_page(mem_glob, p, false, false); if (unlikely(ret != 0)) goto out_err; @@ -228,10 +220,10 @@ static int ttm_tt_set_caching(struct ttm_tt *ttm, if (ttm->caching_state == c_state) return 0; - if (c_state != tt_cached) { - ret = ttm_tt_populate(ttm); - if (unlikely(ret != 0)) - return ret; + if (ttm->state == tt_unpopulated) { + /* Change caching but don't populate */ + ttm->caching_state = c_state; + return 0; } if (ttm->caching_state == tt_cached) @@ -282,13 +274,17 @@ EXPORT_SYMBOL(ttm_tt_set_placement_caching); static void ttm_tt_free_alloced_pages(struct ttm_tt *ttm) { int i; + unsigned count = 0; + struct list_head h; struct page *cur_page; struct ttm_backend *be = ttm->be; + INIT_LIST_HEAD(&h); + if (be) be->func->clear(be); - (void)ttm_tt_set_caching(ttm, tt_cached); for (i = 0; i < ttm->num_pages; ++i) { + cur_page = ttm->pages[i]; ttm->pages[i] = NULL; if (cur_page) { @@ -298,9 +294,11 @@ static void ttm_tt_free_alloced_pages(struct ttm_tt *ttm) "Leaking pages.\n"); ttm_mem_global_free_page(ttm->glob->mem_glob, cur_page); - __free_page(cur_page); + list_add(&cur_page->lru, &h); + count++; } } + ttm_put_pages(&h, count, ttm->page_flags, ttm->caching_state); ttm->state = tt_unpopulated; ttm->first_himem_page = ttm->num_pages; ttm->last_lomem_page = -1; diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h new file mode 100644 index 000000000000..8bb4de567b2c --- /dev/null +++ b/include/drm/ttm/ttm_page_alloc.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) Red Hat Inc. + + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sub license, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Authors: Dave Airlie <airlied@redhat.com> + * Jerome Glisse <jglisse@redhat.com> + */ +#ifndef TTM_PAGE_ALLOC +#define TTM_PAGE_ALLOC + +#include "ttm_bo_driver.h" +#include "ttm_memory.h" + +/** + * Get count number of pages from pool to pages list. + * + * @pages: heado of empty linked list where pages are filled. + * @flags: ttm flags for page allocation. + * @cstate: ttm caching state for the page. + * @count: number of pages to allocate. + */ +int ttm_get_pages(struct list_head *pages, + int flags, + enum ttm_caching_state cstate, + unsigned count); +/** + * Put linked list of pages to pool. + * + * @pages: list of pages to free. + * @page_count: number of pages in the list. Zero can be passed for unknown + * count. + * @flags: ttm flags for page allocation. + * @cstate: ttm caching state. + */ +void ttm_put_pages(struct list_head *pages, + unsigned page_count, + int flags, + enum ttm_caching_state cstate); +/** + * Initialize pool allocator. + * + * Pool allocator is internaly reference counted so it can be initialized + * multiple times but ttm_page_alloc_fini has to be called same number of + * times. + */ +int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages); +/** + * Free pool allocator. + */ +void ttm_page_alloc_fini(void); + +/** + * Output the state of pools to debugfs file + */ +extern int ttm_page_alloc_debugfs(struct seq_file *m, void *data); +#endif |