diff options
-rw-r--r-- | drivers/video/tegra/host/dmabuf.c | 49 | ||||
-rw-r--r-- | drivers/video/tegra/host/dmabuf.h | 7 | ||||
-rw-r--r-- | drivers/video/tegra/host/nvhost_job.c | 8 | ||||
-rw-r--r-- | drivers/video/tegra/host/nvhost_job.h | 2 | ||||
-rw-r--r-- | drivers/video/tegra/host/nvhost_memmgr.c | 87 | ||||
-rw-r--r-- | drivers/video/tegra/host/nvhost_memmgr.h | 7 | ||||
-rw-r--r-- | drivers/video/tegra/host/nvmap.c | 188 | ||||
-rw-r--r-- | drivers/video/tegra/host/nvmap.h | 15 | ||||
-rw-r--r-- | drivers/video/tegra/host/vic03/vic03.c | 6 | ||||
-rw-r--r-- | include/trace/events/nvhost.h | 39 |
10 files changed, 215 insertions, 193 deletions
diff --git a/drivers/video/tegra/host/dmabuf.c b/drivers/video/tegra/host/dmabuf.c index 52d56638d413..62e5331c30b3 100644 --- a/drivers/video/tegra/host/dmabuf.c +++ b/drivers/video/tegra/host/dmabuf.c @@ -109,52 +109,3 @@ int nvhost_dmabuf_get_param(struct mem_mgr *memmgr, struct mem_handle *handle, /* TBD: find a way to associate size, kind, etc */ return -EINVAL; } - -int nvhost_dmabuf_pin_array_ids(struct platform_device *dev, - ulong *ids, - u32 id_type_mask, - u32 id_type, - u32 count, - struct nvhost_job_unpin *unpin_data, - dma_addr_t *phys_addr) { - int i; - int pin_count = 0; - int err; - - for (i = 0; i < count; i++) { - struct mem_handle *handle; - struct sg_table *sgt; - - if ((ids[i] & id_type_mask) != id_type) - continue; - - handle = nvhost_dmabuf_get(ids[i], dev); - - if (IS_ERR(handle)) { - err = PTR_ERR(handle); - goto fail; - } - - sgt = nvhost_dmabuf_pin(handle); - if (IS_ERR(sgt)) { - nvhost_dmabuf_put(handle); - err = PTR_ERR(sgt); - goto fail; - } - - phys_addr[i] = sg_dma_address(sgt->sgl); - - unpin_data[pin_count].h = handle; - unpin_data[pin_count].mem = sgt; - pin_count++; - } - return pin_count; -fail: - while (pin_count) { - pin_count--; - nvhost_dmabuf_unpin(unpin_data[pin_count].h, - unpin_data[pin_count].mem); - nvhost_dmabuf_put(unpin_data[pin_count].h); - } - return err; -} diff --git a/drivers/video/tegra/host/dmabuf.h b/drivers/video/tegra/host/dmabuf.h index ec723ab965ca..264618473572 100644 --- a/drivers/video/tegra/host/dmabuf.h +++ b/drivers/video/tegra/host/dmabuf.h @@ -43,11 +43,4 @@ void nvhost_dmabuf_kunmap(struct mem_handle *handle, unsigned int pagenum, struct mem_handle *nvhost_dmabuf_get(ulong id, struct platform_device *dev); int nvhost_dmabuf_get_param(struct mem_mgr *memmgr, struct mem_handle *handle, u32 param, u64 *result); -int nvhost_dmabuf_pin_array_ids(struct platform_device *dev, - ulong *ids, - u32 id_type_mask, - u32 id_type, - u32 count, - struct nvhost_job_unpin *unpin_data, - dma_addr_t *phys_addr); #endif diff --git a/drivers/video/tegra/host/nvhost_job.c b/drivers/video/tegra/host/nvhost_job.c index 1af2cf91a784..f820c9f31e58 100644 --- a/drivers/video/tegra/host/nvhost_job.c +++ b/drivers/video/tegra/host/nvhost_job.c @@ -48,7 +48,7 @@ static size_t job_size(u32 num_cmdbufs, u32 num_relocs, u32 num_waitchks, + num_waitchks * sizeof(struct nvhost_waitchk) + num_cmdbufs * sizeof(struct nvhost_job_gather) + num_unpins * sizeof(dma_addr_t) - + num_unpins * sizeof(u32 *) + + num_unpins * sizeof(struct nvhost_memmgr_pinid) + num_syncpts * sizeof(struct nvhost_job_syncpt); if(total > ULONG_MAX) @@ -85,7 +85,7 @@ static void init_fields(struct nvhost_job *job, job->addr_phys = num_unpins ? mem : NULL; mem += num_unpins * sizeof(dma_addr_t); job->pin_ids = num_unpins ? mem : NULL; - mem += num_unpins * sizeof(u32 *); + mem += num_unpins * sizeof(struct nvhost_memmgr_pinid); job->sp = num_syncpts ? mem : NULL; job->reloc_addr_phys = job->addr_phys; @@ -238,13 +238,13 @@ static int pin_job_mem(struct nvhost_job *job) for (i = 0; i < job->num_relocs; i++) { struct nvhost_reloc *reloc = &job->relocarray[i]; - job->pin_ids[count] = reloc->target; + job->pin_ids[count].id = reloc->target; count++; } for (i = 0; i < job->num_gathers; i++) { struct nvhost_job_gather *g = &job->gathers[i]; - job->pin_ids[count] = g->mem_id; + job->pin_ids[count].id = g->mem_id; count++; } diff --git a/drivers/video/tegra/host/nvhost_job.h b/drivers/video/tegra/host/nvhost_job.h index eb4eadc6c11a..931286e8bdbb 100644 --- a/drivers/video/tegra/host/nvhost_job.h +++ b/drivers/video/tegra/host/nvhost_job.h @@ -81,7 +81,7 @@ struct nvhost_job { struct nvhost_job_unpin *unpins; int num_unpins; - unsigned long *pin_ids; + struct nvhost_memmgr_pinid *pin_ids; dma_addr_t *addr_phys; dma_addr_t *gather_addr_phys; dma_addr_t *reloc_addr_phys; diff --git a/drivers/video/tegra/host/nvhost_memmgr.c b/drivers/video/tegra/host/nvhost_memmgr.c index 7ec1aa974bba..eb9b84056852 100644 --- a/drivers/video/tegra/host/nvhost_memmgr.c +++ b/drivers/video/tegra/host/nvhost_memmgr.c @@ -22,6 +22,7 @@ #include <linux/platform_device.h> #include <linux/err.h> #include <linux/bug.h> +#include <linux/platform_device.h> #include "nvhost_memmgr.h" #ifdef CONFIG_TEGRA_GRHOST_USE_NVMAP @@ -31,6 +32,7 @@ #ifdef CONFIG_TEGRA_GRHOST_USE_DMABUF #include "dmabuf.h" #endif +#include <linux/sort.h> #include "chip_support.h" struct mem_mgr *nvhost_memmgr_alloc_mgr(void) @@ -148,7 +150,7 @@ struct sg_table *nvhost_memmgr_pin(struct mem_mgr *mgr, switch (nvhost_memmgr_type((u32)((uintptr_t)handle))) { #ifdef CONFIG_TEGRA_GRHOST_USE_NVMAP case mem_mgr_type_nvmap: - return nvhost_nvmap_pin(mgr, handle); + return nvhost_nvmap_pin(mgr, handle, dev); break; #endif #ifdef CONFIG_TEGRA_GRHOST_USE_DMABUF @@ -169,7 +171,7 @@ void nvhost_memmgr_unpin(struct mem_mgr *mgr, switch (nvhost_memmgr_type((u32)((uintptr_t)handle))) { #ifdef CONFIG_TEGRA_GRHOST_USE_NVMAP case mem_mgr_type_nvmap: - nvhost_nvmap_unpin(mgr, handle, sgt); + nvhost_nvmap_unpin(mgr, handle, dev, sgt); break; #endif #ifdef CONFIG_TEGRA_GRHOST_USE_DMABUF @@ -284,53 +286,58 @@ void nvhost_memmgr_kunmap(struct mem_handle *handle, unsigned int pagenum, } } +static int id_cmp(const void *_id1, const void *_id2) +{ + u32 id1 = ((struct nvhost_memmgr_pinid *)_id1)->id; + u32 id2 = ((struct nvhost_memmgr_pinid *)_id2)->id; + + if (id1 < id2) + return -1; + if (id1 > id2) + return 1; + + return 0; +} + int nvhost_memmgr_pin_array_ids(struct mem_mgr *mgr, struct platform_device *dev, - ulong *ids, + struct nvhost_memmgr_pinid *ids, dma_addr_t *phys_addr, u32 count, struct nvhost_job_unpin *unpin_data) { - int pin_count = 0; + int i, pin_count = 0; + struct sg_table *sgt; + struct mem_handle *h; + u32 prev_id = 0; + dma_addr_t prev_addr = 0; -#ifdef CONFIG_TEGRA_GRHOST_USE_NVMAP - { - int nvmap_count = 0; - nvmap_count = nvhost_nvmap_pin_array_ids(mgr, - ids, MEMMGR_TYPE_MASK, - mem_mgr_type_nvmap, - count, unpin_data, - phys_addr); - if (nvmap_count < 0) - return nvmap_count; - pin_count += nvmap_count; - } -#endif -#ifdef CONFIG_TEGRA_GRHOST_USE_DMABUF - { - int dmabuf_count = 0; - dmabuf_count = nvhost_dmabuf_pin_array_ids(dev, - ids, MEMMGR_TYPE_MASK, - mem_mgr_type_dmabuf, - count, &unpin_data[pin_count], - phys_addr); - - if (dmabuf_count < 0) { - /* clean up previous handles */ - while (pin_count) { - pin_count--; - /* unpin, put */ - nvhost_memmgr_unpin(mgr, - unpin_data[pin_count].h, &dev->dev, - unpin_data[pin_count].mem); - nvhost_memmgr_put(mgr, - unpin_data[pin_count].h); - } - return dmabuf_count; + for (i = 0; i < count; i++) + ids[i].index = i; + + sort(ids, count, sizeof(*ids), id_cmp, NULL); + + for (i = 0; i < count; i++) { + if (ids[i].id == prev_id) { + phys_addr[ids[i].index] = prev_addr; + continue; } - pin_count += dmabuf_count; + + h = nvhost_memmgr_get(mgr, ids[i].id, dev); + if (IS_ERR(h)) + continue; + + sgt = nvhost_memmgr_pin(mgr, h, &dev->dev); + if (IS_ERR(sgt)) + return PTR_ERR(sgt); + + phys_addr[ids[i].index] = sg_dma_address(sgt->sgl); + unpin_data[pin_count].h = h; + unpin_data[pin_count++].mem = sgt; + + prev_id = ids[i].id; + prev_addr = phys_addr[ids[i].index]; } -#endif return pin_count; } diff --git a/drivers/video/tegra/host/nvhost_memmgr.h b/drivers/video/tegra/host/nvhost_memmgr.h index 789adfb9f1f6..817186b0c5a1 100644 --- a/drivers/video/tegra/host/nvhost_memmgr.h +++ b/drivers/video/tegra/host/nvhost_memmgr.h @@ -32,6 +32,11 @@ struct nvhost_job_unpin { struct sg_table *mem; }; +struct nvhost_memmgr_pinid { + u32 id; + u32 index; +}; + enum mem_mgr_flag { mem_mgr_flag_uncacheable = 0, mem_mgr_flag_write_combine = 1, @@ -77,7 +82,7 @@ u32 nvhost_memmgr_handle_to_id(struct mem_handle *handle); int nvhost_memmgr_pin_array_ids(struct mem_mgr *mgr, struct platform_device *dev, - ulong *ids, + struct nvhost_memmgr_pinid *ids, dma_addr_t *phys_addr, u32 count, struct nvhost_job_unpin *unpin_data); diff --git a/drivers/video/tegra/host/nvmap.c b/drivers/video/tegra/host/nvmap.c index f84be9bd5071..f4c2aefe55da 100644 --- a/drivers/video/tegra/host/nvmap.c +++ b/drivers/video/tegra/host/nvmap.c @@ -17,15 +17,30 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ - #include <linux/nvmap.h> #include <linux/slab.h> #include <linux/scatterlist.h> #include <linux/err.h> +#include <trace/events/nvhost.h> #include "nvmap.h" #include "nvhost_job.h" #include "chip_support.h" +#define FLAG_NVHOST_MAPPED 1 +#define FLAG_NVMAP_MAPPED 2 +#define FLAG_CARVEOUT 3 + +struct nvhost_nvmap_data { + struct nvmap_client *client; + struct nvmap_handle_ref *ref; + struct sg_table *sgt; + size_t len; + struct device *dev; + int pin_count; + int flags; + struct mutex lock; +}; + struct mem_mgr *nvhost_nvmap_alloc_mgr(void) { return (struct mem_mgr *)nvmap_create_client(nvmap_dev, "nvhost"); @@ -61,48 +76,113 @@ void nvhost_nvmap_put(struct mem_mgr *mgr, struct mem_handle *handle) (struct nvmap_handle_ref *)handle); } -static struct scatterlist *sg_kmalloc(unsigned int nents, gfp_t gfp_mask) +static void unmap_pages(struct device *dev, struct sg_table *sgt, size_t len) +{ + DEFINE_DMA_ATTRS(attrs); + dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs); + dma_unmap_sg_attrs(dev, sgt->sgl, sgt->nents, 0, &attrs); +} + +void delete_priv(void *_priv) { -#ifdef CONFIG_COMPAT - return (struct scatterlist *)(((uintptr_t)gfp_mask)|PAGE_OFFSET); -#else - return (struct scatterlist *)((uintptr_t)gfp_mask); -#endif + struct nvhost_nvmap_data *priv = _priv; + if (priv->sgt) { + if (priv->flags & BIT(FLAG_NVHOST_MAPPED)) + unmap_pages(priv->dev, priv->sgt, priv->len); + nvmap_free_sg_table(priv->client, priv->ref, priv->sgt); + } + kfree(priv); } struct sg_table *nvhost_nvmap_pin(struct mem_mgr *mgr, - struct mem_handle *handle) -{ - int err = 0; - dma_addr_t ret = 0; - struct sg_table *sgt = kmalloc(sizeof(*sgt) + sizeof(*sgt->sgl), - GFP_KERNEL); - if (!sgt) - return ERR_PTR(-ENOMEM); - - err = __sg_alloc_table(sgt, 1, 1, (gfp_t)((uintptr_t)(sgt+1)), sg_kmalloc); - if (err) { - kfree(sgt); - return ERR_PTR(err); + struct mem_handle *handle, + struct device *dev) +{ + struct nvmap_handle_ref *ref = (struct nvmap_handle_ref *)handle; + struct nvmap_client *nvmap = (struct nvmap_client *)mgr; + struct nvhost_nvmap_data *priv = nvmap_get_nvhost_private(ref); + struct sg_table *sgt; + int ret; + + if (!priv) { + u64 size = 0, addr = 0; + + ret = nvmap_get_handle_param(nvmap, ref, + NVMAP_HANDLE_PARAM_SIZE, &size); + if (ret) + return ERR_PTR(ret); + + ret = nvmap_get_handle_param(nvmap, ref, + NVMAP_HANDLE_PARAM_BASE, &addr); + if (ret) + return ERR_PTR(ret); + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return ERR_PTR(-ENOMEM); + + mutex_init(&priv->lock); + priv->client = nvmap; + priv->ref = ref; + priv->sgt = nvmap_sg_table(nvmap, ref); + priv->dev = dev; + priv->len = size; + if (IS_ERR(priv->sgt)) { + kfree(priv); + return priv->sgt; + } + nvmap_set_nvhost_private(ref, priv, delete_priv); + + if (!IS_ERR_VALUE(addr)) + priv->flags |= BIT(FLAG_NVMAP_MAPPED); } - err = nvmap_pin((struct nvmap_client *)mgr, - (struct nvmap_handle_ref *)handle, &ret); - if (err) { - kfree(sgt); - return ERR_PTR(err); + mutex_lock(&priv->lock); + sgt = priv->sgt; + + if (priv->flags & BIT(FLAG_NVMAP_MAPPED) && + sg_dma_address(sgt->sgl) == 0) { + dma_addr_t addr = 0; + int err = nvmap_pin(priv->client, ref, &addr); + if (err) + return ERR_PTR(err); + + sg_dma_address(sgt->sgl) = addr; + } else if (priv->pin_count == 0 && sg_dma_address(sgt->sgl) == 0) { + int ents; + DEFINE_DMA_ATTRS(attrs); + dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs); + + ents = dma_map_sg_attrs(dev, sgt->sgl, sgt->nents, 0, &attrs); + if (!ents) { + mutex_unlock(&priv->lock); + return ERR_PTR(-ENOMEM); + } + priv->flags |= BIT(FLAG_NVHOST_MAPPED); } - sg_dma_address(sgt->sgl) = ret; + trace_nvhost_nvmap_pin(dev_name(dev), + ref, priv->len, sg_dma_address(sgt->sgl)); + priv->pin_count++; + mutex_unlock(&priv->lock); + nvmap_flush_deferred_cache(nvmap, ref); return sgt; } -void nvhost_nvmap_unpin(struct mem_mgr *mgr, - struct mem_handle *handle, struct sg_table *sgt) +void nvhost_nvmap_unpin(struct mem_mgr *mgr, struct mem_handle *handle, + struct device *dev, struct sg_table *sgt) { - kfree(sgt); - return nvmap_unpin((struct nvmap_client *)mgr, - (struct nvmap_handle_ref *)handle); + struct nvmap_handle_ref *ref = (struct nvmap_handle_ref *)handle; + struct nvhost_nvmap_data *priv = nvmap_get_nvhost_private(ref); + if (!priv) + return; + + mutex_lock(&priv->lock); + priv->pin_count--; + WARN_ON(priv->pin_count < 0); + mutex_unlock(&priv->lock); + trace_nvhost_nvmap_unpin(dev_name(dev), + ref, priv->len, sg_dma_address(sgt->sgl)); } void *nvhost_nvmap_mmap(struct mem_handle *handle) @@ -126,52 +206,6 @@ void nvhost_nvmap_kunmap(struct mem_handle *handle, unsigned int pagenum, nvmap_kunmap((struct nvmap_handle_ref *)handle, pagenum, addr); } -int nvhost_nvmap_pin_array_ids(struct mem_mgr *mgr, - long unsigned *ids, - u32 id_type_mask, - u32 id_type, - u32 count, - struct nvhost_job_unpin *unpin_data, - dma_addr_t *phys_addr) -{ - int i; - int result = 0; - struct nvmap_handle **unique_handles; - struct nvmap_handle_ref **unique_handle_refs; - void *ptrs = kmalloc(sizeof(void *) * count * 2, - GFP_KERNEL); - - if (!ptrs) - return -ENOMEM; - - unique_handles = (struct nvmap_handle **) ptrs; - unique_handle_refs = (struct nvmap_handle_ref **) - &unique_handles[count]; - - result = nvmap_pin_array((struct nvmap_client *)mgr, - (long unsigned *)ids, id_type_mask, id_type, count, - unique_handles, - unique_handle_refs); - - if (result < 0) - goto fail; - - WARN_ON(result > count); - - for (i = 0; i < result; i++) - unpin_data[i].h = (struct mem_handle *)unique_handle_refs[i]; - - for (i = 0; i < count; i++) { - if ((ids[i] & id_type_mask) == id_type) - phys_addr[i] = (dma_addr_t)nvmap_get_addr_from_user_id( - ids[i]); - } - -fail: - kfree(ptrs); - return result; -} - struct mem_handle *nvhost_nvmap_get(struct mem_mgr *mgr, ulong id, struct platform_device *dev) { diff --git a/drivers/video/tegra/host/nvmap.h b/drivers/video/tegra/host/nvmap.h index bac79ac08465..a42a0a9f1327 100644 --- a/drivers/video/tegra/host/nvmap.h +++ b/drivers/video/tegra/host/nvmap.h @@ -35,9 +35,9 @@ struct mem_handle *nvhost_nvmap_alloc(struct mem_mgr *mgr, size_t size, size_t align, int flags, unsigned int heap_flags); void nvhost_nvmap_put(struct mem_mgr *mgr, struct mem_handle *handle); struct sg_table *nvhost_nvmap_pin(struct mem_mgr *mgr, - struct mem_handle *handle); -void nvhost_nvmap_unpin(struct mem_mgr *mgr, - struct mem_handle *handle, struct sg_table *sgt); + struct mem_handle *handle, struct device *dev); +void nvhost_nvmap_unpin(struct mem_mgr *mgr, struct mem_handle *handle, + struct device *dev, struct sg_table *sgt); void *nvhost_nvmap_mmap(struct mem_handle *handle); void nvhost_nvmap_munmap(struct mem_handle *handle, void *addr); void *nvhost_nvmap_kmap(struct mem_handle *handle, unsigned int pagenum); @@ -49,15 +49,6 @@ int nvhost_nvmap_get_param(struct mem_mgr *mgr, struct mem_handle *handle, u32 param, u64 *result); phys_addr_t nvhost_nvmap_get_addr_from_id(ulong id); -int nvhost_nvmap_pin_array_ids(struct mem_mgr *mgr, - long unsigned *ids, - u32 id_type_mask, - u32 id_type, - u32 count, - struct nvhost_job_unpin *unpin_data, - dma_addr_t *phys_addr); - void nvhost_nvmap_unpin_id(struct mem_mgr *mgr, ulong id); - #endif diff --git a/drivers/video/tegra/host/vic03/vic03.c b/drivers/video/tegra/host/vic03/vic03.c index cd45138819ab..aba7ad7589d9 100644 --- a/drivers/video/tegra/host/vic03/vic03.c +++ b/drivers/video/tegra/host/vic03/vic03.c @@ -518,7 +518,8 @@ static struct nvhost_hwctx *vic03_alloc_hwctx(struct nvhost_hwctx_handler *h, ctx->hwctx.save_thresh = 0; ctx->hwctx.save_slots = 0; - ctx->restore_sgt = nvhost_memmgr_pin(nvmap, ctx->restore); + ctx->restore_sgt = nvhost_memmgr_pin(nvmap, + ctx->restore, &ch->dev->dev); if (IS_ERR_VALUE(ctx->restore_phys)) goto fail_pin; ctx->restore_phys = sg_dma_address(ctx->restore_sgt->sgl); @@ -549,7 +550,8 @@ static void vic03_free_hwctx(struct kref *ref) nvhost_memmgr_munmap(ctx->restore, ctx->restore_virt); ctx->restore_virt = NULL; } - nvhost_memmgr_unpin(nvmap, ctx->restore, ctx->restore_sgt); + nvhost_memmgr_unpin(nvmap, ctx->restore, &nctx->channel->dev->dev, + ctx->restore_sgt); ctx->restore_phys = 0; nvhost_memmgr_put(nvmap, ctx->restore); ctx->restore = NULL; diff --git a/include/trace/events/nvhost.h b/include/trace/events/nvhost.h index 68fa3bf6c83f..b9b58216bcb5 100644 --- a/include/trace/events/nvhost.h +++ b/include/trace/events/nvhost.h @@ -900,6 +900,45 @@ TRACE_EVENT(nvhost_gk20a_mmu_fault, __entry->engine, __entry->client, __entry->fault_type) ); +DECLARE_EVENT_CLASS(nvhost_map, + TP_PROTO(const char *devname, void *handle, int size, + dma_addr_t iova), + + TP_ARGS(devname, handle, size, iova), + + TP_STRUCT__entry( + __field(const char *, devname) + __field(void *, handle) + __field(int, size) + __field(dma_addr_t, iova) + ), + + TP_fast_assign( + __entry->devname = devname; + __entry->handle = handle; + __entry->size = size; + __entry->iova = iova; + ), + + TP_printk("dev=%s, handle=%p, size=%d, iova=%08x", + __entry->devname, __entry->handle, __entry->size, + __entry->iova) +); + +DEFINE_EVENT(nvhost_map, nvhost_nvmap_pin, + TP_PROTO(const char *devname, void *handle, int size, + dma_addr_t iova), + + TP_ARGS(devname, handle, size, iova) +); + +DEFINE_EVENT(nvhost_map, nvhost_nvmap_unpin, + TP_PROTO(const char *devname, void *handle, int size, + dma_addr_t iova), + + TP_ARGS(devname, handle, size, iova) +); + #endif /* _TRACE_NVHOST_H */ /* This part must be outside protection */ |