diff options
author | Vandana Salve <vsalve@nvidia.com> | 2014-01-31 20:54:02 +0530 |
---|---|---|
committer | Krishna Reddy <vdumpa@nvidia.com> | 2014-03-11 09:43:35 -0700 |
commit | 0e051fcb68126decee2fe053628e9e43c53626e8 (patch) | |
tree | 40471c6162b8e9fa7c5a0d2e328a49d759c0154f /drivers | |
parent | b5a5a3fe138da1994dc0919458994644e0a1f401 (diff) |
dma: coherent: Handle resizable coherent memory
Support to handle resizable/fixed size coherent memory
Add debugfs support
bug 1445123
Change-Id: I77c250ddfe95609a5e7edb9093b8f1c23652a0b4
Signed-off-by: Vandana Salve <vsalve@nvidia.com>
Reviewed-on: http://git-master/r/362440
Reviewed-by: Krishna Reddy <vdumpa@nvidia.com>
Diffstat (limited to 'drivers')
-rw-r--r-- | drivers/base/dma-coherent.c | 564 |
1 files changed, 506 insertions, 58 deletions
diff --git a/drivers/base/dma-coherent.c b/drivers/base/dma-coherent.c index 60ed4354a78d..d45369857aef 100644 --- a/drivers/base/dma-coherent.c +++ b/drivers/base/dma-coherent.c @@ -7,6 +7,42 @@ #include <linux/module.h> #include <linux/dma-mapping.h> #include <linux/dma-attrs.h> +#include <linux/dma-contiguous.h> +#include <linux/debugfs.h> + +struct heap_info { + const char *name; + /* number of devices pointed by devs */ + unsigned int num_devs; + /* indices for start and end device for resize support */ + unsigned int dev_start; + unsigned int dev_end; + /* devs to manage cma/coherent memory allocs, if resize allowed */ + struct device *devs; + /* device to allocate memory from cma */ + struct device *cma_dev; + /* flag that indicates whether heap can resize :shrink/grow */ + bool can_resize; + /* lock to synchronise heap resizing */ + struct mutex resize_lock; + /* CMA chunk size if resize supported */ + size_t cma_chunk_size; + /* heap base */ + phys_addr_t base; + /* heap size */ + size_t len; + phys_addr_t cma_base; + size_t cma_len; + struct dentry *dma_debug_root; + void (*update_resize_cfg)(phys_addr_t , size_t); +}; + +#define DMA_RESERVED_COUNT 8 +static struct dma_coherent_reserved { + const struct device *dev; +} dma_coherent_reserved[DMA_RESERVED_COUNT]; + +static unsigned dma_coherent_reserved_count; #ifdef CONFIG_ARM_DMA_IOMMU_ALIGNMENT #define DMA_BUF_ALIGNMENT CONFIG_ARM_DMA_IOMMU_ALIGNMENT @@ -23,6 +59,58 @@ struct dma_coherent_mem { unsigned long *bitmap; }; +static bool dma_is_coherent_dev(struct device *dev) +{ + int i; + struct dma_coherent_reserved *r = dma_coherent_reserved; + + for (i = 0; i < dma_coherent_reserved_count; i++, r++) { + if (dev == r->dev) + return true; + } + return false; +} +static void dma_debugfs_init(struct device *dev, struct heap_info *heap) +{ + if (!heap->dma_debug_root) { + heap->dma_debug_root = debugfs_create_dir(dev_name(dev), NULL); + if (IS_ERR_OR_NULL(heap->dma_debug_root)) { + dev_err(dev, "couldn't create debug files\n"); + return; + } + } + + debugfs_create_x32("base", S_IRUGO, + heap->dma_debug_root, (u32 *)&heap->base); + debugfs_create_x32("size", S_IRUGO, + heap->dma_debug_root, (u32 *)&heap->len); + if (heap->can_resize) { + debugfs_create_x32("cma_base", S_IRUGO, + heap->dma_debug_root, (u32 *)&heap->cma_base); + debugfs_create_x32("cma_size", S_IRUGO, + heap->dma_debug_root, (u32 *)&heap->cma_len); + debugfs_create_x32("cma_chunk_size", S_IRUGO, + heap->dma_debug_root, (u32 *)&heap->cma_chunk_size); + debugfs_create_x32("num_cma_chunks", S_IRUGO, + heap->dma_debug_root, (u32 *)&heap->num_devs); + } +} + +static struct device *dma_create_dma_devs(const char *name, int num_devs) +{ + int idx = 0; + struct device *devs; + + devs = kzalloc(num_devs * sizeof(*devs), GFP_KERNEL); + if (!devs) + return NULL; + + for (idx = 0; idx < num_devs; idx++) + dev_set_name(&devs[idx], "%s-heap-%d", name, idx); + + return devs; +} + int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr, dma_addr_t device_addr, size_t size, int flags) { @@ -78,58 +166,210 @@ skip_mapping: } EXPORT_SYMBOL(dma_declare_coherent_memory); -void dma_release_declared_memory(struct device *dev) +static int declare_coherent_heap(struct device *dev, phys_addr_t base, + size_t size) { - struct dma_coherent_mem *mem = dev->dma_mem; + int err; + + BUG_ON(dev->dma_mem); + dma_set_coherent_mask(dev, DMA_BIT_MASK(64)); + err = dma_declare_coherent_memory(dev, 0, + base, size, + DMA_MEMORY_NOMAP); + if (err & DMA_MEMORY_NOMAP) { + dev_dbg(dev, "dma coherent mem base (%pa) size (%zu)\n", + &base, size); + return 0; + } + dev_err(dev, "failed to declare dma coherent_mem (%pa)\n", + &base); + return -ENOMEM; +} - if (!mem) - return; - dev->dma_mem = NULL; +int dma_declare_coherent_resizable_cma_memory(struct device *dev, + struct dma_declare_info *dma_info) +{ + int err = 0; + struct heap_info *heap_info = NULL; + struct dma_coherent_reserved *r = + &dma_coherent_reserved[dma_coherent_reserved_count]; + + if (dma_coherent_reserved_count == ARRAY_SIZE(dma_coherent_reserved)) { + pr_err("Not enough slots for DMA Coherent reserved regions!\n"); + return -ENOSPC; + } - if (!(mem->flags & DMA_MEMORY_NOMAP)) - iounmap(mem->virt_base); + if (!dma_info || !dev) + return -EINVAL; + + heap_info = kzalloc(sizeof(*heap_info), GFP_KERNEL); + if (!heap_info) + return -ENOMEM; + + heap_info->name = dma_info->name; + dev_set_name(dev, "dma-%s", heap_info->name); + + if (!dma_info->resize) { + /* No Resize support */ + err = declare_coherent_heap(dev, dma_info->base, + dma_info->size); + if (err) + goto fail; + + heap_info->base = dma_info->base; + heap_info->len = dma_info->size; + heap_info->can_resize = false; + } else { + /* Resize Heap memory */ +#ifdef CONFIG_CMA + struct dma_contiguous_stats stats; + if (!dma_info->cma_dev) { + err = -EINVAL; + goto fail; + } + heap_info->cma_dev = dma_info->cma_dev; + dma_get_contiguous_stats(heap_info->cma_dev, &stats); + + heap_info->cma_chunk_size = dma_info->size; + heap_info->cma_base = stats.base; + heap_info->cma_len = stats.size; + heap_info->can_resize = true; + dev_set_name(heap_info->cma_dev, "cma-%s-heap", dma_info->name); + + if (heap_info->cma_len < heap_info->cma_chunk_size) { + dev_err(dev, "error cma_len < cma_chunk_size"); + err = -EINVAL; + goto fail; + } + if (heap_info->cma_len % heap_info->cma_chunk_size) { + dev_err(dev, + "size is not multiple of cma_chunk_size(%zu)\n" + "size truncated from %zu to %zu\n", + heap_info->cma_chunk_size, heap_info->cma_len, + round_down(heap_info->cma_len, + heap_info->cma_chunk_size)); + heap_info->cma_len = round_down( + heap_info->cma_len, + heap_info->cma_chunk_size); + } - kfree(mem->bitmap); - kfree(mem); + mutex_init(&heap_info->resize_lock); + heap_info->num_devs = div_u64(heap_info->cma_len, + heap_info->cma_chunk_size); + heap_info->devs = dma_create_dma_devs(dma_info->name, + heap_info->num_devs); + if (!heap_info->devs) { + dev_err(dev, "failed to alloc devices\n"); + err = -ENOMEM; + goto fail; + } + if (dma_info->notifier.ops) + heap_info->update_resize_cfg = + dma_info->notifier.ops->resize; +#else + err = -EINVAL; + goto fail; +#endif + } + + r->dev = dev; + dma_coherent_reserved_count++; + + dev_set_drvdata(dev, heap_info); + dma_debugfs_init(dev, heap_info); + return 0; +fail: + kfree(heap_info); + return err; } -EXPORT_SYMBOL(dma_release_declared_memory); +EXPORT_SYMBOL(dma_declare_coherent_resizable_cma_memory); -void *dma_mark_declared_memory_occupied(struct device *dev, - dma_addr_t device_addr, size_t size) +static phys_addr_t alloc_from_contiguous_heap( + struct heap_info *h, + phys_addr_t base, size_t len) { - struct dma_coherent_mem *mem = dev->dma_mem; - int pos, err; + size_t count; + struct page *page; + unsigned long order; + + order = get_order(len); + count = PAGE_ALIGN(len) >> PAGE_SHIFT; + page = dma_alloc_from_contiguous(h->cma_dev, count, order); + if (!page) { + dev_err(h->cma_dev, "failed to alloc dma contiguous mem\n"); + goto dma_alloc_err; + } + base = page_to_phys(page); + dev_dbg(h->cma_dev, "dma contiguous mem base (0x%pa) size (%zu)\n", + &base, len); + BUG_ON(base < h->cma_base || + base - h->cma_base + len > h->cma_len); + return base; + +dma_alloc_err: + return DMA_ERROR_CODE; +} - size += device_addr & ~PAGE_MASK; +static void release_from_contiguous_heap( + struct heap_info *h, + phys_addr_t base, size_t len) +{ + struct page *page = phys_to_page(base); + size_t count = PAGE_ALIGN(len) >> PAGE_SHIFT; - if (!mem) - return ERR_PTR(-EINVAL); + dma_release_from_contiguous(h->cma_dev, page, count); +} - pos = (device_addr - mem->device_base) >> PAGE_SHIFT; - err = bitmap_allocate_region(mem->bitmap, pos, get_order(size)); - if (err != 0) - return ERR_PTR(err); - return mem->virt_base + (pos << PAGE_SHIFT); +static int heap_resize_locked(struct heap_info *h) +{ + int idx; + phys_addr_t base; + bool at_bottom = false; + + base = alloc_from_contiguous_heap(h, 0, h->cma_chunk_size); + if (dma_mapping_error(h->cma_dev, base)) + return 1; + + idx = div_u64(base - h->cma_base, h->cma_chunk_size); + if (!h->len || base == h->base - h->cma_chunk_size) + /* new chunk can be added at bottom. */ + at_bottom = true; + else if (base != h->base + h->len) + /* new chunk can't be added at top */ + goto fail_non_contig; + + BUG_ON(h->dev_start - 1 != idx && h->dev_end + 1 != idx && h->len); + dev_dbg(&h->devs[idx], + "Resize VPR base from=0x%pa to=0x%pa, len from=%zu to=%zu\n", + &h->base, &base, h->len, h->len + h->cma_chunk_size); + + if (declare_coherent_heap(&h->devs[idx], base, h->cma_chunk_size)) + goto fail_declare; + dev_dbg(&h->devs[idx], + "Resize VPR base from=0x%pa to=0x%pa, len from=%zu to=%zu\n", + &h->base, &base, h->len, h->len + h->cma_chunk_size); + if (at_bottom) { + h->base = base; + h->dev_start = idx; + if (!h->len) + h->dev_end = h->dev_start; + } else { + h->dev_end = idx; + } + h->len += h->cma_chunk_size; + /* Handle VPR configuration updates*/ + if (h->update_resize_cfg) + h->update_resize_cfg(h->base, h->len); + return 0; + +fail_non_contig: + dev_dbg(&h->devs[idx], "Found Non-Contiguous block(0x%pa)\n", &base); +fail_declare: + release_from_contiguous_heap(h, base, h->cma_chunk_size); + return 1; } -EXPORT_SYMBOL(dma_mark_declared_memory_occupied); -/** - * dma_alloc_from_coherent_attr() - try to allocate memory from the per-device - * coherent area - * - * @dev: device from which we allocate memory - * @size: size of requested memory area - * @dma_handle: This will be filled with the correct dma handle - * @ret: This pointer will be filled with the virtual address - * to allocated area. - * @attrs: DMA Attribute - * This function should be only called from per-arch dma_alloc_coherent() - * to support allocation from per-device coherent memory pools. - * - * Returns 0 if dma_alloc_coherent_attr should continue with allocating from - * generic memory areas, or !0 if dma_alloc_coherent should return @ret. - */ -int dma_alloc_from_coherent_attr(struct device *dev, ssize_t size, +static int dma_alloc_from_coherent_dev(struct device *dev, ssize_t size, dma_addr_t *dma_handle, void **ret, struct dma_attrs *attrs) { @@ -187,25 +427,58 @@ err: */ return mem->flags & DMA_MEMORY_EXCLUSIVE; } -EXPORT_SYMBOL(dma_alloc_from_coherent_attr); -/** - * dma_release_from_coherent_attr() - try to free the memory allocated from - * per-device coherent memory pool - * @dev: device from which the memory was allocated - * @size: size of the memory area to free - * @vaddr: virtual address of allocated pages - * @attrs: DMA Attribute - * - * This checks whether the memory was allocated from the per-device - * coherent memory pool and if so, releases that memory. - * - * Returns 1 if we correctly released the memory, or 0 if - * dma_release_coherent_attr() should proceed with releasing memory from - * generic pools. - */ -int dma_release_from_coherent_attr(struct device *dev, size_t size, void *vaddr, - struct dma_attrs *attrs) +static int dma_alloc_from_coherent_heap_dev(struct device *dev, size_t len, + dma_addr_t *dma_handle, void **ret, + struct dma_attrs *attrs) +{ + int err = 0; + int idx = 0; + phys_addr_t pa; + struct heap_info *h = NULL; + struct device *d; + + if (!dma_is_coherent_dev(dev)) + return 0; + + h = dev_get_drvdata(dev); + if (!h) + return 0; + + if (!h->can_resize) + return 0; + + dma_set_attr(DMA_ATTR_ALLOC_EXACT_SIZE, attrs); + + mutex_lock(&h->resize_lock); +retry_alloc: + /* Try allocation from already existing CMA chunks */ + for (idx = h->dev_start; idx <= h->dev_end && h->len; idx++) { + d = &h->devs[idx]; + if (d->dma_mem) { + err = dma_alloc_from_coherent_dev(d, len, &pa, + ret, attrs); + if (err) { + dev_dbg(d, "Allocated addr %pa len %zu\n", + &pa, len); + *dma_handle = pa; + goto out; + } + } + } + + /* Check if a heap can be expanded */ + if (h->dev_end - h->dev_start + 1 < h->num_devs || !h->len) { + if (!heap_resize_locked(h)) + goto retry_alloc; + } +out: + mutex_unlock(&h->resize_lock); + return err; +} + +static int dma_release_from_coherent_dev(struct device *dev, size_t size, + void *vaddr, struct dma_attrs *attrs) { struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL; void *mem_addr; @@ -237,6 +510,181 @@ int dma_release_from_coherent_attr(struct device *dev, size_t size, void *vaddr, } return 0; } + +static int dma_release_from_coherent_heap_dev(struct device *dev, size_t len, + void *base, struct dma_attrs *attrs) +{ + int idx = 0; + int err = 0; + int resize_err = 0; + void *ret = NULL; + dma_addr_t dev_base; + struct heap_info *h = NULL; + + if (!dma_is_coherent_dev(dev)) + return 0; + + h = dev_get_drvdata(dev); + if (!h) + return 0; + + dma_set_attr(DMA_ATTR_ALLOC_EXACT_SIZE, attrs); + + if (!h->can_resize) + return 0; + + mutex_lock(&h->resize_lock); + + idx = div_u64((uintptr_t)base - h->cma_base, h->cma_chunk_size); + dev_dbg(&h->devs[idx], "dma free base (%pa) size (%zu) idx (%d)\n", + (void *)(uintptr_t)&base, len, idx); + err = dma_release_from_coherent_dev(&h->devs[idx], len, base, attrs); + + if (!err) + goto out_unlock; + +check_next_chunk: + /* Check if heap can be shrinked */ + if ((idx == h->dev_start || idx == h->dev_end) && h->len) { + /* check if entire chunk is free */ + resize_err = dma_alloc_from_coherent_dev(&h->devs[idx], + h->cma_chunk_size, + &dev_base, &ret, attrs); + if (!resize_err) + goto out_unlock; + else { + resize_err = dma_release_from_coherent_dev( + &h->devs[idx], + h->cma_chunk_size, + (void *)dev_base, attrs); + if (!resize_err) + goto out_unlock; + + dma_release_declared_memory( + &h->devs[idx]); + BUG_ON(h->devs[idx].dma_mem != NULL); + h->len -= h->cma_chunk_size; + + if ((idx == h->dev_start)) { + h->base += h->cma_chunk_size; + h->dev_start++; + dev_dbg(&h->devs[idx], + "Release Chunk at bottom\n"); + idx++; + } else { + h->dev_end--; + dev_dbg(&h->devs[idx], + "Release Chunk at top\n"); + idx--; + } + + /* Handle VPR configuration updates*/ + if (h->update_resize_cfg) + h->update_resize_cfg(h->base, h->len); + release_from_contiguous_heap(h, + dev_base, h->cma_chunk_size); + } + goto check_next_chunk; + } +out_unlock: + mutex_unlock(&h->resize_lock); + return err; +} + +void dma_release_declared_memory(struct device *dev) +{ + struct dma_coherent_mem *mem = dev->dma_mem; + + if (!mem) + return; + dev->dma_mem = NULL; + + if (!(mem->flags & DMA_MEMORY_NOMAP)) + iounmap(mem->virt_base); + + kfree(mem->bitmap); + kfree(mem); +} +EXPORT_SYMBOL(dma_release_declared_memory); + +void *dma_mark_declared_memory_occupied(struct device *dev, + dma_addr_t device_addr, size_t size) +{ + struct dma_coherent_mem *mem = dev->dma_mem; + int pos, err; + + size += device_addr & ~PAGE_MASK; + + if (!mem) + return ERR_PTR(-EINVAL); + + pos = (device_addr - mem->device_base) >> PAGE_SHIFT; + err = bitmap_allocate_region(mem->bitmap, pos, get_order(size)); + if (err != 0) + return ERR_PTR(err); + return mem->virt_base + (pos << PAGE_SHIFT); +} +EXPORT_SYMBOL(dma_mark_declared_memory_occupied); + +/** + * dma_alloc_from_coherent_attr() - try to allocate memory from the per-device + * coherent area + * + * @dev: device from which we allocate memory + * @size: size of requested memory area + * @dma_handle: This will be filled with the correct dma handle + * @ret: This pointer will be filled with the virtual address + * to allocated area. + * @attrs: DMA Attribute + * This function should be only called from per-arch dma_alloc_coherent() + * to support allocation from per-device coherent memory pools. + * + * Returns 0 if dma_alloc_coherent_attr should continue with allocating from + * generic memory areas, or !0 if dma_alloc_coherent should return @ret. + */ +int dma_alloc_from_coherent_attr(struct device *dev, ssize_t size, + dma_addr_t *dma_handle, void **ret, + struct dma_attrs *attrs) +{ + if (!dev) + return 0; + + if (dev->dma_mem) + return dma_alloc_from_coherent_dev(dev, size, dma_handle, ret, + attrs); + else + return dma_alloc_from_coherent_heap_dev(dev, size, dma_handle, + ret, attrs); +} +EXPORT_SYMBOL(dma_alloc_from_coherent_attr); + +/** + * dma_release_from_coherent_attr() - try to free the memory allocated from + * per-device coherent memory pool + * @dev: device from which the memory was allocated + * @size: size of the memory area to free + * @vaddr: virtual address of allocated pages + * @attrs: DMA Attribute + * + * This checks whether the memory was allocated from the per-device + * coherent memory pool and if so, releases that memory. + * + * Returns 1 if we correctly released the memory, or 0 if + * dma_release_coherent_attr() should proceed with releasing memory from + * generic pools. + */ +int dma_release_from_coherent_attr(struct device *dev, size_t size, void *vaddr, + struct dma_attrs *attrs) +{ + if (!dev) + return 0; + + if (dev->dma_mem) + return dma_release_from_coherent_dev(dev, size, vaddr, attrs); + else + return dma_release_from_coherent_heap_dev(dev, size, vaddr, + attrs); +} EXPORT_SYMBOL(dma_release_from_coherent_attr); /** |