nvmap: implement nvmap as a full memory manager driver

previously, the task of managing RM-managed memory handles was split
between nvos (OS page allocation), the RM (heap management for
carveout & IRAM heaps, and handle life-time management), nvreftrack
(abnormal process termination) and nvmap (user-space read/write/map
of memory handles). this resulted in an opaque system that was wasteful
of kernel virtual address space, didn't support CPU cache attributes for
kernel mappings and couldn't fully unwind leaked handles (e.g., if the
application leaked a pinned handle the memory might never be reclaimed).

nvmap is now a full re-implementation of the RM memory manager, unifying all
of the functionality from nvreftrack, nvos, nvmap and nvrm into one
driver used by both user and kernel-space clients.

add configs to control paranoid operation. when paranoid is enabled,
every handle reference passed into the kernel is verified to actually
have been created by nvmap; furthermore, handles which are not global
(the GET_ID ioctl has not been called for it) will fail validation
if they are referenced by any process other than the one which created
them, or a super-user process (opened via /dev/knvmap).

each file descriptor maintains its own table of nvmap_handle_ref
references, so the handle value returned to each process is unique;
furthermore, nvmap_handle_ref objects track how many times they have
been pinned, to ensure that processes which abnormally terminate with
pinned handles can be unwound correctly.

as a compile-time option, fully-unpinned handles which require IOVMM
mappings may be stored in a segmented (by size) MRU (most-recently
unpinned) eviction cache; if IOVMM space is over-committed across
multiple processes, a pin operation may reclaim any or all of the IOVMM
areas in the MRU cache. MRU is used as the eviction policy since
graphics operations frequently operate cyclically, and the least-recently
used entry may be needed almost immediately if the higher-level client
starts (e.g.) rendering the next frame.

introduce a concept of "secure" handles. secure handles may only
be mapped into IOVMM space, and when unpinned their mapping in IOVMM
space will be zapped immediately, to prevent malicious processes from
being able to access the handle.

expose carveout heap attributes for each carveout heap in sysfs,
under the nvmap device with sub-device name heap-<heap name>
 * total size
 * free size
 * total block count
 * free block count
 * largest block
 * largest free block
 * base address
 * name
 * heap usage bitmask

carveout heaps may be split at run-time, if sufficient memory is available
in the heap. the split heap can be (should be) assigned a different name
and usage bitmask than the original heap. this allows a large initial
carveout to be split into smaller carveouts, to reserve sections of carveout
memory for specific usages (e.g., camera and/or video clients).

add a split entry in the sysfs tree for each carveout heap, to support
run-time splitting of carveout heaps into reserved regions. format is:
 <size>,<usage>,<name>
 * size should be parsable with memparse (suffixes k/K and m/M are legal)
 * usage is the new heap's usage bitmask
 * name is the name of the new heap (must be unique)

carveout heaps are managed using a first-fit allocator with an explicit
free list, all blocks are kept in a dynamically-sized array (doubles
in size every time all blocks are exhausted); to reduce fragmentation
caused by allocations with different alignment requirements, the
allocator will compare left-justifying and right-justifying the
allocation within the first-fit block, and choose the justification
that results in the largest remaining free block (this is particularly
important for 1M-aligned split heaps).

other code which duplicated functionality subsumed by this changelist
(RM memory manager, NvOs carveout command line parser, etc.) is deleted;
implementations of the RM memory manager on top of nvmap are provided
to support backwards compatibility

bug 634812

Change-Id: Ic89d83fed31b4cadc68653d0e825c368b9c92f81
Reviewed-on: http://git-master/r/590
Reviewed-by: Gary King <gking@nvidia.com>
Tested-by: Gary King <gking@nvidia.com>

3 files changed, 2881 insertions, 223 deletions

diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig
index d9405d51b154..b23ecd1dd34f 100644
--- a/drivers/char/Kconfig
+++ b/drivers/char/Kconfig
@@ -129,6 +129,25 @@ config DEVNVMAP
 	  is used by NVIDIA Tegra graphics and multimedia drivers for managing
 	  graphics memory.
 
+config DEVNVMAP_PARANOID
+	bool "Validate all user-provided /dev/nvmap object references"
+	depends on DEVNVMAP
+	default n
+	help
+	  Say Y here to enable additional process-level validations and
+	  permissions for /dev/nvmap object references provided via ioctls.
+	  May result in a decrease in performance.
+
+config DEVNVMAP_RECLAIM_UNPINNED_VM
+	bool "Allow /dev/nvmap to reclaim unpinned I/O virtual memory"
+	depends on DEVNVMAP && TEGRA_IOVMM
+	default n
+	help
+	  Say Y here to enable /dev/nvmap to reclaim I/O virtual memory after
+	  it has been unpinned, and re-use it for other objects. This can
+	  allow a larger virtual I/O VM space than would normally be
+	  supported by the hardware, at a slight cost in performance.
+
 
 config SERIAL_NONSTANDARD
 	bool "Non-standard serial port support"
diff --git a/drivers/char/mem.c b/drivers/char/mem.c
index 9785f1091721..f91999c7b735 100644
--- a/drivers/char/mem.c
+++ b/drivers/char/mem.c
@@ -903,11 +903,6 @@ static const struct file_operations kmsg_fops = {
 	.write =	kmsg_write,
 };
 
-#ifdef CONFIG_DEVNVMAP
-extern const struct file_operations nvmap_fops;
-extern const struct file_operations knvmap_fops;
-#endif
-
 static int memory_open(struct inode * inode, struct file * filp)
 {
 	int ret = 0;
@@ -957,14 +952,6 @@ static int memory_open(struct inode * inode, struct file * filp)
 			filp->f_op = &oldmem_fops;
 			break;
 #endif
-#ifdef CONFIG_DEVNVMAP
-		case 13:
-			filp->f_op = &nvmap_fops;
-			break;
-		case 14:
-			filp->f_op = &knvmap_fops;
-			break;
-#endif
 		default:
 			unlock_kernel();
 			return -ENXIO;
@@ -1003,10 +990,6 @@ static const struct {
 #ifdef CONFIG_CRASH_DUMP
 	{12,"oldmem",    S_IRUSR | S_IWUSR | S_IRGRP, &oldmem_fops},
 #endif
-#ifdef CONFIG_DEVNVMAP
-	{13, "nvmap",  S_IRUGO | S_IWUGO, &nvmap_fops},
-	{14, "knvmap", S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP, &knvmap_fops },
-#endif
 };
 
 static struct class *mem_class;
diff --git a/drivers/char/nvmap.c b/drivers/char/nvmap.c
index f1a3f1ff11b9..8546061016b6 100644
--- a/drivers/char/nvmap.c
+++ b/drivers/char/nvmap.c
@@ -1,9 +1,9 @@
 /*
  * drivers/char/nvmap.c
  *
- * Memory mapping driver for Tegra anonymous memory handles
+ * Memory manager for Tegra GPU memory handles
  *
- * Copyright (c) 2009, NVIDIA Corporation.
+ * Copyright (c) 2009-2010, NVIDIA Corporation.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -20,6 +20,7 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
+#include <linux/vmalloc.h>
 #include <linux/module.h>
 #include <linux/bitmap.h>
 #include <linux/wait.h>
@@ -36,11 +37,17 @@
 #include <linux/sched.h>
 #include <linux/io.h>
 #include <linux/tegra_devices.h>
+#include <linux/rbtree.h>
+#include <linux/proc_fs.h>
+#include <linux/ctype.h>
 #include <asm/tlbflush.h>
+#include <mach/iovmm.h>
 #include "linux/nvmem_ioctl.h"
 #include "nvcommon.h"
 #include "nvrm_memmgr.h"
-#include "nvrm_memmgr_private.h"
+#include "nvbootargs.h"
+
+/*#define IOVMM_FIRST*/ /* enable to force most allocations from iovmm */
 
 static void nvmap_vma_open(struct vm_area_struct *vma);
 
@@ -57,11 +64,25 @@ static int nvmap_mmap(struct file *filp, struct vm_area_struct *vma);
 static long nvmap_ioctl(struct file *filp,
 	unsigned int cmd, unsigned long arg);
 
-static int nvmap_cache_maint(struct file *filp, void __user *arg);
+static int nvmap_ioctl_getid(struct file *filp, void __user *arg);
+
+static int nvmap_ioctl_get_param(struct file *filp, void __user* arg);
+
+static int nvmap_ioctl_alloc(struct file *filp, void __user *arg);
+
+static int nvmap_ioctl_free(struct file *filp, unsigned long arg);
+
+static int nvmap_ioctl_create(struct file *filp,
+	unsigned int cmd, void __user *arg);
+
+static int nvmap_ioctl_pinop(struct file *filp,
+	bool is_pin, void __user *arg);
+
+static int nvmap_ioctl_cache_maint(struct file *filp, void __user *arg);
 
 static int nvmap_map_into_caller_ptr(struct file *filp, void __user *arg);
 
-static int nvmap_rw_handle(struct file *filp, int is_read,
+static int nvmap_ioctl_rw_handle(struct file *filp, int is_read,
 	void __user* arg);
 
 extern void NvRmPrivMemIncrRef(NvRmMemHandle hmem);
@@ -81,9 +102,451 @@ static struct backing_dev_info nvmap_bdi = {
 
 static pte_t *nvmap_pte[NUM_NVMAP_PTES];
 static unsigned long nvmap_ptebits[NVMAP_PAGES/BITS_PER_LONG];
+
 static DEFINE_SPINLOCK(nvmap_ptelock);
 static DECLARE_WAIT_QUEUE_HEAD(nvmap_ptefull);
 
+/* used to lost the master tree of memory handles */
+static DEFINE_SPINLOCK(nvmap_handle_lock);
+
+/* only one task may be performing pin / unpin operations at once, to
+ * prevent deadlocks caused by interleaved IOVMM re-allocations */
+static DEFINE_MUTEX(nvmap_pin_lock);
+
+/* queue of tasks which are blocking on pin, for IOVMM room */
+static DECLARE_WAIT_QUEUE_HEAD(nvmap_pin_wait);
+static struct rb_root nvmap_handles = RB_ROOT;
+
+static struct tegra_iovmm_client *nvmap_vm_client = NULL;
+
+/* first-fit linear allocator carveout heap manager */
+struct nvmap_mem_block {
+	unsigned long	base;
+	size_t		size;
+	short		next; /* next absolute (address-order) block */
+	short		prev; /* previous absolute (address-order) block */
+	short		next_free;
+	short		prev_free;
+};
+
+struct nvmap_carveout {
+	unsigned short		num_blocks;
+	short			spare_index;
+	short			free_index;
+	short			block_index;
+	spinlock_t		lock;
+	const char		*name;
+	struct nvmap_mem_block	*blocks;
+};
+
+enum {
+	CARVEOUT_STAT_TOTAL_SIZE,
+	CARVEOUT_STAT_FREE_SIZE,
+	CARVEOUT_STAT_NUM_BLOCKS,
+	CARVEOUT_STAT_FREE_BLOCKS,
+	CARVEOUT_STAT_LARGEST_BLOCK,
+	CARVEOUT_STAT_LARGEST_FREE,
+	CARVEOUT_STAT_BASE,
+};
+
+static inline pgprot_t _nvmap_flag_to_pgprot(unsigned long flag, pgprot_t base)
+{
+	switch (flag) {
+	case NVMEM_HANDLE_UNCACHEABLE:
+		base = pgprot_noncached(base);
+		break;
+	case NVMEM_HANDLE_WRITE_COMBINE:
+		base = pgprot_writecombine(base);
+		break;
+	case NVMEM_HANDLE_INNER_CACHEABLE:
+		base = pgprot_inner_writeback(base);
+		break;
+	}
+	return base;
+}
+
+static unsigned long _nvmap_carveout_blockstat(struct nvmap_carveout *co,
+	int stat)
+{
+	unsigned long val = 0;
+	short idx;
+	spin_lock(&co->lock);
+
+	if (stat==CARVEOUT_STAT_BASE) {
+		if (co->block_index==-1)
+			val = ~0;
+		else
+			val = co->blocks[co->block_index].base;
+		spin_unlock(&co->lock);
+		return val;
+	}
+
+	if (stat==CARVEOUT_STAT_TOTAL_SIZE ||
+	    stat==CARVEOUT_STAT_NUM_BLOCKS ||
+	    stat==CARVEOUT_STAT_LARGEST_BLOCK)
+		idx = co->block_index;
+	else
+		idx = co->free_index;
+
+	while (idx!=-1) {
+		switch (stat) {
+		case CARVEOUT_STAT_TOTAL_SIZE:
+			val += co->blocks[idx].size;
+			idx = co->blocks[idx].next;
+			break;
+		case CARVEOUT_STAT_NUM_BLOCKS:
+			val++;
+			idx = co->blocks[idx].next;
+			break;
+		case CARVEOUT_STAT_LARGEST_BLOCK:
+			val = max_t(unsigned long, val, co->blocks[idx].size);
+			idx = co->blocks[idx].next;
+			break;
+		case CARVEOUT_STAT_FREE_SIZE:
+			val += co->blocks[idx].size;
+			idx = co->blocks[idx].next_free;
+			break;
+		case CARVEOUT_STAT_FREE_BLOCKS:
+			val ++;
+			idx = co->blocks[idx].next_free;
+			break;
+		case CARVEOUT_STAT_LARGEST_FREE:
+			val = max_t(unsigned long, val, co->blocks[idx].size);
+			idx = co->blocks[idx].next_free;
+			break;
+            }
+	}
+
+	spin_unlock(&co->lock);
+	return val;
+}
+
+#define co_is_free(_co, _idx) \
+	((_co)->free_index==(_idx) || ((_co)->blocks[(_idx)].prev_free!=-1))
+
+static int _nvmap_init_carveout(struct nvmap_carveout *co,
+	const char *name, unsigned long base_address, size_t len)
+{
+	const unsigned int min_blocks = 16;
+	struct nvmap_mem_block *blocks = NULL;
+	int i;
+
+	blocks = kzalloc(sizeof(*blocks)*min_blocks, GFP_KERNEL);
+
+	if (!blocks) goto fail;
+	co->name = kstrdup(name, GFP_KERNEL);
+	if (!co->name) goto fail;
+
+	for (i=1; i<min_blocks; i++) {
+		blocks[i].next = i+1;
+		blocks[i].prev = i-1;
+		blocks[i].next_free = -1;
+		blocks[i].prev_free = -1;
+	}
+	blocks[i-1].next = -1;
+	blocks[1].prev = -1;
+
+	blocks[0].next = blocks[0].prev = -1;
+	blocks[0].next_free = blocks[0].prev_free = -1;
+	blocks[0].base = base_address;
+	blocks[0].size = len;
+	co->blocks = blocks;
+	co->num_blocks = min_blocks;
+	spin_lock_init(&co->lock);
+	co->block_index = 0;
+	co->spare_index = 1;
+	co->free_index = 0;
+	return 0;
+
+fail:
+	if (blocks) kfree(blocks);
+	return -ENOMEM;
+}
+
+static int nvmap_grow_blocks(struct nvmap_carveout *co)
+{
+	struct nvmap_mem_block *blocks;
+	unsigned int i;
+
+	if (co->num_blocks >= 1<<(8*sizeof(co->free_index)-1)) return -ENOMEM;
+	blocks = kzalloc(sizeof(*blocks)*(co->num_blocks*2), GFP_KERNEL);
+	if (!blocks) return -ENOMEM;
+
+	memcpy(blocks, co->blocks, sizeof(*blocks)*(co->num_blocks));
+	kfree(co->blocks);
+	co->blocks = blocks;
+	for (i=co->num_blocks; i<co->num_blocks*2; i++) {
+		blocks[i].next = i+1;
+		blocks[i].prev = i-1;
+		blocks[i].next_free = -1;
+		blocks[i].prev_free = -1;
+	}
+	blocks[co->num_blocks].prev = -1;
+	blocks[i-1].next = -1;
+	blocks[i].next_free = -1;
+	blocks[i].prev_free = -1;
+	co->spare_index = co->num_blocks;
+	co->num_blocks *= 2;
+	return 0;
+}
+
+static int nvmap_get_spare(struct nvmap_carveout *co) {
+	int idx;
+
+	if (co->spare_index == -1)
+		if (nvmap_grow_blocks(co))
+			return -1;
+
+	BUG_ON(co->spare_index == -1);
+	idx = co->spare_index;
+	co->spare_index = co->blocks[idx].next;
+	co->blocks[idx].next = -1;
+	co->blocks[idx].prev = -1;
+	co->blocks[idx].next_free = -1;
+	co->blocks[idx].prev_free = -1;
+	return idx;
+}
+
+#define BLOCK(_co, _idx) ((_idx)==-1 ? NULL : &(_co)->blocks[(_idx)])
+
+static void nvmap_zap_free(struct nvmap_carveout *co, int idx)
+{
+	struct nvmap_mem_block *block;
+
+	block = BLOCK(co, idx);
+	if (block->prev_free != -1)
+		BLOCK(co, block->prev_free)->next_free = block->next_free;
+	else
+		co->free_index = block->next_free;
+
+	if (block->next_free != -1)
+		BLOCK(co, block->next_free)->prev_free = block->prev_free;
+
+	block->prev_free = -1;
+	block->next_free = -1;
+}
+
+static void nvmap_split_block(struct nvmap_carveout *co,
+	int idx, size_t start, size_t size)
+{
+	if (BLOCK(co, idx)->base < start) {
+		int spare_idx = nvmap_get_spare(co);
+		struct nvmap_mem_block *spare = BLOCK(co, spare_idx);
+		struct nvmap_mem_block *block = BLOCK(co, idx);
+		if (spare) {
+			spare->size = start - block->base;
+			spare->base = block->base;
+			block->size -= (start - block->base);
+			block->base = start;
+			spare->next = idx;
+			spare->prev = block->prev;
+			block->prev = spare_idx;
+			if (spare->prev != -1)
+				co->blocks[spare->prev].next = spare_idx;
+			else
+				co->block_index = spare_idx;
+			spare->prev_free = -1;
+			spare->next_free = co->free_index;
+			if (co->free_index != -1)
+				co->blocks[co->free_index].prev_free = spare_idx;
+			co->free_index = spare_idx;
+		} else {
+			if (block->prev != -1) {
+				spare = BLOCK(co, block->prev);
+				spare->size += start - block->base;
+				block->base = start;
+			}
+		}
+	}
+
+	if (BLOCK(co, idx)->size > size) {
+		int spare_idx = nvmap_get_spare(co);
+		struct nvmap_mem_block *spare = BLOCK(co, spare_idx);
+		struct nvmap_mem_block *block = BLOCK(co, idx);
+		if (spare) {
+			spare->base = block->base + size;
+			spare->size = block->size - size;
+			block->size = size;
+			spare->prev = idx;
+			spare->next = block->next;
+			block->next = spare_idx;
+			if (spare->next != -1)
+				co->blocks[spare->next].prev = spare_idx;
+			spare->prev_free = -1;
+			spare->next_free = co->free_index;
+			if (co->free_index != -1)
+				co->blocks[co->free_index].prev_free = spare_idx;
+			co->free_index = spare_idx;
+		}
+	}
+
+	nvmap_zap_free(co, idx);
+}
+
+#define next_spare next
+#define prev_spare prev
+
+#define nvmap_insert_block(_list, _co, _idx)				\
+	do {								\
+		struct nvmap_mem_block *b = BLOCK((_co), (_idx));	\
+		struct nvmap_mem_block *s = BLOCK((_co), (_co)->_list##_index);\
+		if (s) s->prev_##_list = (_idx);			\
+		b->prev_##_list = -1;					\
+		b->next_##_list = (_co)->_list##_index;			\
+		(_co)->_list##_index = (_idx);				\
+	} while (0);
+
+static void nvmap_carveout_free(struct nvmap_carveout *co, int idx)
+{
+	struct nvmap_mem_block *b;
+
+	spin_lock(&co->lock);
+
+	b = BLOCK(co, idx);
+
+	if (b->next!=-1 && co_is_free(co, b->next)) {
+		int zap = b->next;
+		struct nvmap_mem_block *n = BLOCK(co, zap);
+		b->size += n->size;
+
+		b->next = n->next;
+		if (n->next != -1) co->blocks[n->next].prev = idx;
+
+		nvmap_zap_free(co, zap);
+		nvmap_insert_block(spare, co, zap);
+	}
+
+	if (b->prev!=-1 && co_is_free(co, b->prev)) {
+		int zap = b->prev;
+		struct nvmap_mem_block *p = BLOCK(co, zap);
+
+		b->base = p->base;
+		b->size += p->size;
+
+		b->prev = p->prev;
+
+		if (p->prev != -1) co->blocks[p->prev].next = idx;
+		else co->block_index = idx;
+
+		nvmap_zap_free(co, zap);
+		nvmap_insert_block(spare, co, zap);
+	}
+
+	nvmap_insert_block(free, co, idx);
+	spin_unlock(&co->lock);
+}
+
+static int nvmap_carveout_alloc(struct nvmap_carveout *co,
+	size_t align, size_t size)
+{
+	short idx;
+
+	spin_lock(&co->lock);
+
+	idx = co->free_index;
+
+	while (idx != -1) {
+		struct nvmap_mem_block *b = BLOCK(co, idx);
+		/* try to be a bit more clever about generating block-
+		 * droppings by comparing the results of a left-justified vs
+		 * right-justified block split, and choosing the
+		 * justification style which yields the largest remaining
+		 * block */
+		size_t end = b->base + b->size;
+		size_t ljust = (b->base + align - 1) & ~(align-1);
+		size_t rjust = (end - size) & ~(align-1);
+		size_t l_max, r_max;
+
+		if (rjust < b->base) rjust = ljust;
+		l_max = max_t(size_t, ljust - b->base, end - (ljust + size));
+		r_max = max_t(size_t, rjust - b->base, end - (rjust + size));
+
+		if (b->base + b->size >= ljust + size) {
+			if (l_max >= r_max)
+				nvmap_split_block(co, idx, ljust, size);
+			else
+				nvmap_split_block(co, idx, rjust, size);
+			break;
+		}
+		idx = b->next_free;
+	}
+
+	spin_unlock(&co->lock);
+	return idx;
+}
+
+#undef next_spare
+#undef prev_spare
+
+struct nvmap_handle {
+	struct rb_node node;
+	atomic_t ref;
+	atomic_t pin;
+	unsigned long flags;
+	size_t size;
+	size_t orig_size;
+	struct task_struct *owner;
+	unsigned int poison;
+	union {
+		struct {
+			struct page **pages;
+			struct tegra_iovmm_area *area;
+			struct list_head mru_list;
+			bool contig;
+			bool dirty; /* IOVMM area allocated since last pin */
+		} pgalloc;
+		struct {
+			struct nvmap_carveout *co_heap;
+			int block_idx;
+			unsigned long base;
+			unsigned int key; /* preserved by bootloader */
+		} carveout;
+	};
+	bool global;
+	bool secure; /* only allocated in IOVM space, zapped on unpin */
+	bool heap_pgalloc;
+	bool alloc;
+	void *kern_map; /* used for RM memmgr backwards compat */
+};
+
+/* handle_ref objects are file-descriptor-local references to nvmap_handle
+ * objects. they track the number of references and pins performed by
+ * the specific caller (since nvmap_handle objects may be global), so that
+ * a client which terminates without properly unwinding all handles (or
+ * all nested pins) can be unwound by nvmap. */
+struct nvmap_handle_ref {
+	struct nvmap_handle *h;
+	struct rb_node node;
+	atomic_t refs;
+	atomic_t pin;
+};
+
+struct nvmap_file_priv {
+	struct rb_root handle_refs;
+	atomic_t iovm_commit;
+	size_t iovm_limit;
+	spinlock_t ref_lock;
+	bool su;
+};
+
+struct nvmap_carveout_node {
+	struct device		dev;
+	struct list_head	heap_list;
+	unsigned int		heap_bit;
+	struct nvmap_carveout	carveout;
+};
+
+/* the master structure for all nvmap-managed carveouts and all handle_ref
+ * objects allocated inside the kernel. heaps are sorted by their heap_bit
+ * (highest heap_bit first) so that carveout allocation will be first
+ * attempted by the heap with the highest heap_bit set in the allocation's
+ * heap mask */
+static struct {
+	struct nvmap_file_priv init_data;
+	struct rw_semaphore list_sem;
+	struct list_head heaps;
+} nvmap_context;
+
 static struct vm_operations_struct nvmap_vma_ops = {
 	.open	= nvmap_vma_open,
 	.close	= nvmap_vma_close,
@@ -107,20 +570,643 @@ const struct file_operations knvmap_fops = {
 };
 
 struct nvmap_vma_priv {
-	NvRmMemHandle	hmem;
-	size_t		offs;
-	atomic_t	ref;
+	struct nvmap_handle	*h;
+	size_t			offs;
+	atomic_t		ref;
 };
 
+static struct proc_dir_entry *nvmap_procfs_root;
+static struct proc_dir_entry *nvmap_procfs_proc;
+
+static void _nvmap_handle_free(struct nvmap_handle *h);
+
+#define NVMAP_CARVEOUT_ATTR_RO(_name)					\
+	struct device_attribute nvmap_heap_attr_##_name =		\
+		__ATTR(_name, S_IRUGO, _nvmap_sysfs_show_heap_##_name, NULL)
+
+#define NVMAP_CARVEOUT_ATTR_WO(_name, _mode)				\
+	struct device_attribute nvmap_heap_attr_##_name =		\
+		__ATTR(_name, _mode, NULL, _nvmap_sysfs_set_heap_##_name)
+
+
+static ssize_t _nvmap_sysfs_show_heap_usage(struct device *d,
+	struct device_attribute *attr, char *buf)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	return sprintf(buf, "%08x\n", c->heap_bit);
+}
+
+static ssize_t _nvmap_sysfs_show_heap_name(struct device *d,
+	struct device_attribute *attr, char *buf)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	return sprintf(buf, "%s\n", c->carveout.name);
+}
+
+static ssize_t _nvmap_sysfs_show_heap_base(struct device *d,
+	struct device_attribute *attr, char *buf)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	return sprintf(buf, "%08lx\n",
+		_nvmap_carveout_blockstat(&c->carveout, CARVEOUT_STAT_BASE));
+}
+
+static ssize_t _nvmap_sysfs_show_heap_free_size(struct device *d,
+	struct device_attribute *attr, char *buf)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	return sprintf(buf, "%lu\n",
+		_nvmap_carveout_blockstat(&c->carveout,
+			CARVEOUT_STAT_FREE_SIZE));
+}
+
+static ssize_t _nvmap_sysfs_show_heap_free_count(struct device *d,
+	struct device_attribute *attr, char *buf)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	return sprintf(buf, "%lu\n",
+		_nvmap_carveout_blockstat(&c->carveout,
+			CARVEOUT_STAT_FREE_BLOCKS));
+}
+
+static ssize_t _nvmap_sysfs_show_heap_free_max(struct device *d,
+	struct device_attribute *attr, char *buf)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	return sprintf(buf, "%lu\n",
+		_nvmap_carveout_blockstat(&c->carveout,
+			CARVEOUT_STAT_LARGEST_FREE));
+}
+
+static ssize_t _nvmap_sysfs_show_heap_total_count(struct device *d,
+	struct device_attribute *attr, char *buf)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	return sprintf(buf, "%lu\n",
+		_nvmap_carveout_blockstat(&c->carveout,
+			CARVEOUT_STAT_NUM_BLOCKS));
+}
+
+static ssize_t _nvmap_sysfs_show_heap_total_max(struct device *d,
+	struct device_attribute *attr, char *buf)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	return sprintf(buf, "%lu\n",
+		_nvmap_carveout_blockstat(&c->carveout,
+			CARVEOUT_STAT_LARGEST_BLOCK));
+}
+
+static ssize_t _nvmap_sysfs_show_heap_total_size(struct device *d,
+	struct device_attribute *attr, char *buf)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	return sprintf(buf, "%lu\n",
+		_nvmap_carveout_blockstat(&c->carveout,
+			CARVEOUT_STAT_TOTAL_SIZE));
+}
+
+static int nvmap_split_carveout_heap(struct nvmap_carveout *co, size_t size,
+	const char *name, unsigned int new_bitmask);
+
+static ssize_t _nvmap_sysfs_set_heap_split(struct device *d,
+	struct device_attribute *attr, const char * buf, size_t count)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	char *tmp, *local = kzalloc(count+1, GFP_KERNEL);
+	char *sizestr = NULL, *bitmaskstr = NULL, *name = NULL;
+	char **format[] = { &sizestr, &bitmaskstr, &name };
+	char ***f_iter = format;
+	unsigned int i;
+	unsigned long size, bitmask;
+	int err;
+
+	if (!local) {
+		pr_err("%s: unable to read string\n", __func__);
+		return -ENOMEM;
+	}
+
+	memcpy(local, buf, count);
+	tmp = local;
+	for (i=0, **f_iter = local; i<count &&
+	    (f_iter - format)<ARRAY_SIZE(format)-1; i++) {
+		if (local[i]==',') {
+			local[i] = '\0';
+			f_iter++;
+			**f_iter = &local[i+1];
+		}
+	}
+
+	if (!sizestr || !bitmaskstr || !name) {
+		pr_err("%s: format error\n", __func__);
+		kfree(tmp);
+		return -EINVAL;
+	}
+
+	for (local=name; !isspace(*local); local++);
+
+	if (local==name) {
+		pr_err("%s: invalid name %s\n", __func__, name);
+		kfree(tmp);
+		return -EINVAL;
+	}
+
+	*local=0;
+
+	size = memparse(sizestr, &sizestr);
+	if (!size) {
+		kfree(tmp);
+		return -EINVAL;
+	}
+
+	if (strict_strtoul(bitmaskstr, 0, &bitmask)==-EINVAL) {
+		kfree(tmp);
+		return -EINVAL;
+	}
+
+	err = nvmap_split_carveout_heap(&c->carveout, size, name, bitmask);
+
+	if (err) pr_err("%s: failed to create split heap %s\n", __func__, name);
+	kfree(tmp);
+	return err ? err : count;
+}
+
+static NVMAP_CARVEOUT_ATTR_RO(usage);
+static NVMAP_CARVEOUT_ATTR_RO(name);
+static NVMAP_CARVEOUT_ATTR_RO(base);
+static NVMAP_CARVEOUT_ATTR_RO(free_size);
+static NVMAP_CARVEOUT_ATTR_RO(free_count);
+static NVMAP_CARVEOUT_ATTR_RO(free_max);
+static NVMAP_CARVEOUT_ATTR_RO(total_size);
+static NVMAP_CARVEOUT_ATTR_RO(total_count);
+static NVMAP_CARVEOUT_ATTR_RO(total_max);
+static NVMAP_CARVEOUT_ATTR_WO(split, (S_IWUSR | S_IWGRP));
+
+static struct attribute *nvmap_heap_default_attrs[] = {
+	&nvmap_heap_attr_usage.attr,
+	&nvmap_heap_attr_name.attr,
+	&nvmap_heap_attr_split.attr,
+	&nvmap_heap_attr_base.attr,
+	&nvmap_heap_attr_total_size.attr,
+	&nvmap_heap_attr_free_size.attr,
+	&nvmap_heap_attr_total_count.attr,
+	&nvmap_heap_attr_free_count.attr,
+	&nvmap_heap_attr_total_max.attr,
+	&nvmap_heap_attr_free_max.attr,
+	NULL
+};
+
+static struct attribute_group nvmap_heap_defattr_group = {
+	.attrs = nvmap_heap_default_attrs
+};
+
+static struct device *__nvmap_heap_parent_dev(void);
+#define _nvmap_heap_parent_dev __nvmap_heap_parent_dev()
+
+/* unpinned I/O VMM areas may be reclaimed by nvmap to make room for
+ * new surfaces. unpinned surfaces are stored in segregated linked-lists
+ * sorted in most-recently-unpinned order (i.e., head insertion, head
+ * removal */
+#ifdef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM
+static DEFINE_SPINLOCK(nvmap_mru_vma_lock);
+static const size_t nvmap_mru_cutoff[] = {
+	262144, 393216, 786432, 1048576, 1572864
+};
+
+static struct list_head nvmap_mru_vma_lists[ARRAY_SIZE(nvmap_mru_cutoff)];
+
+static inline struct list_head *_nvmap_list(size_t size)
+{
+	unsigned int i;
+
+	for (i=0; i<ARRAY_SIZE(nvmap_mru_cutoff); i++)
+		if (size <= nvmap_mru_cutoff[i]) return &nvmap_mru_vma_lists[i];
+
+	return &nvmap_mru_vma_lists[ARRAY_SIZE(nvmap_mru_cutoff)-1];
+}
+#endif
+
+static inline struct nvmap_handle *_nvmap_handle_get(struct nvmap_handle *h)
+{
+	if (unlikely(h->poison)) {
+		pr_err("%s: %s getting poisoned handle\n", __func__,
+			current->group_leader->comm);
+		return NULL;
+	} else if (unlikely(atomic_inc_return(&h->ref)<=1)) {
+		pr_err("%s: %s getting a freed handle\n",
+			__func__, current->group_leader->comm);
+		return NULL;
+	}
+	return h;
+}
+
+static inline void _nvmap_handle_put(struct nvmap_handle *h)
+{
+	int cnt = atomic_dec_return(&h->ref);
+
+	if (unlikely(cnt<0)) {
+		pr_err("%s: %s put to negative references\n",
+			__func__, current->comm);
+		dump_stack();
+	} else if (!cnt) _nvmap_handle_free(h);
+}
+
+static struct nvmap_handle *_nvmap_claim_preserved(
+	struct task_struct *new_owner, unsigned long key)
+{
+	struct rb_node *n;
+	struct nvmap_handle *b = NULL;
+
+	if (!key) return NULL;
+
+	spin_lock(&nvmap_handle_lock);
+	n = rb_first(&nvmap_handles);
+
+	while (n) {
+		b = rb_entry(n, struct nvmap_handle, node);
+		if (b->alloc && !b->heap_pgalloc && b->carveout.key == key) {
+			b->carveout.key = 0;
+			b->owner = new_owner;
+			break;
+		}
+		b = NULL;
+		n = rb_next(n);
+	}
+
+	spin_unlock(&nvmap_handle_lock);
+	return b;
+}
+
+static struct nvmap_handle *_nvmap_validate_get(unsigned long handle, bool su)
+{
+	struct nvmap_handle *b = NULL;
+
+#ifdef CONFIG_DEVNVMAP_PARANOID
+	struct rb_node *n;
+
+	spin_lock(&nvmap_handle_lock);
+
+	n = nvmap_handles.rb_node;
+
+	while (n) {
+		b = rb_entry(n, struct nvmap_handle, node);
+		if ((unsigned long)b == handle) {
+			if (su || b->global || b->owner==current->group_leader)
+				b = _nvmap_handle_get(b);
+			else
+				b = NULL;
+			spin_unlock(&nvmap_handle_lock);
+			return b;
+		}
+		if (handle > (unsigned long)b) n = n->rb_right;
+		else n = n->rb_left;
+	}
+	spin_unlock(&nvmap_handle_lock);
+	return NULL;
+#else
+	if (!handle) return NULL;
+	b = _nvmap_handle_get((struct nvmap_handle *)handle);
+	return b;
+#endif
+}
+
+static inline void _nvmap_insert_mru_vma(struct nvmap_handle *h)
+{
+#ifdef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM
+	spin_lock(&nvmap_mru_vma_lock);
+	list_add(&h->pgalloc.mru_list, _nvmap_list(h->pgalloc.area->iovm_length));
+	spin_unlock(&nvmap_mru_vma_lock);
+#endif
+}
+
+static void _nvmap_remove_mru_vma(struct nvmap_handle *h)
+{
+#ifdef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM
+	spin_lock(&nvmap_mru_vma_lock);
+	if (!list_empty(&h->pgalloc.mru_list))
+		list_del(&h->pgalloc.mru_list);
+	spin_unlock(&nvmap_mru_vma_lock);
+	INIT_LIST_HEAD(&h->pgalloc.mru_list);
+#endif
+}
+
+static struct tegra_iovmm_area *_nvmap_get_vm(struct nvmap_handle *h)
+{
+#ifndef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM
+	BUG_ON(!h->pgalloc.area);
+	BUG_ON(h->size > h->pgalloc.area->iovm_length);
+	BUG_ON((h->size | h->pgalloc.area->iovm_length) & ~PAGE_MASK);
+	return h->pgalloc.area;
+#else
+	struct list_head *mru;
+	struct nvmap_handle *evict = NULL;
+	struct tegra_iovmm_area *vm = NULL;
+	unsigned int i, idx;
+
+	spin_lock(&nvmap_mru_vma_lock);
+
+	if (h->pgalloc.area) {
+		BUG_ON(list_empty(&h->pgalloc.mru_list));
+		list_del(&h->pgalloc.mru_list);
+		INIT_LIST_HEAD(&h->pgalloc.mru_list);
+		spin_unlock(&nvmap_mru_vma_lock);
+		return h->pgalloc.area;
+	}
+
+	vm = tegra_iovmm_create_vm(nvmap_vm_client, NULL, h->size,
+		_nvmap_flag_to_pgprot(h->flags, pgprot_kernel));
+
+	if (vm) {
+		INIT_LIST_HEAD(&h->pgalloc.mru_list);
+		spin_unlock(&nvmap_mru_vma_lock);
+		return vm;
+	}
+	/* attempt to re-use the most recently unpinned IOVMM area in the
+	 * same size bin as the current handle. If that fails, iteratively
+	 * evict handles (starting from the current bin) until an allocation
+	 * succeeds or no more areas can be evicted */
+
+	mru = _nvmap_list(h->size);
+	if (!list_empty(mru))
+		evict = list_first_entry(mru, struct nvmap_handle,
+			pgalloc.mru_list);
+	if (evict && evict->pgalloc.area->iovm_length >= h->size) {
+		list_del(&evict->pgalloc.mru_list);
+		vm = evict->pgalloc.area;
+		evict->pgalloc.area = NULL;
+		INIT_LIST_HEAD(&evict->pgalloc.mru_list);
+		spin_unlock(&nvmap_mru_vma_lock);
+		return vm;
+	}
+
+	idx = mru - nvmap_mru_vma_lists;
+
+	for (i=0; i<ARRAY_SIZE(nvmap_mru_vma_lists) && !vm; i++, idx++) {
+		if (idx >= ARRAY_SIZE(nvmap_mru_vma_lists))
+			idx -= ARRAY_SIZE(nvmap_mru_vma_lists);
+		mru = &nvmap_mru_vma_lists[idx];
+		while (!list_empty(mru) && !vm) {
+			evict = list_first_entry(mru, struct nvmap_handle,
+				pgalloc.mru_list);
+
+			BUG_ON(atomic_read(&evict->pin)!=0);
+			BUG_ON(!evict->pgalloc.area);
+			list_del(&evict->pgalloc.mru_list);
+			INIT_LIST_HEAD(&evict->pgalloc.mru_list);
+			tegra_iovmm_free_vm(evict->pgalloc.area);
+			evict->pgalloc.area = NULL;
+			vm = tegra_iovmm_create_vm(nvmap_vm_client,
+				NULL, h->size,
+				_nvmap_flag_to_pgprot(h->flags, pgprot_kernel));
+		}
+	}
+
+	spin_unlock(&nvmap_mru_vma_lock);
+	return vm;
+#endif
+}
+
+static int _nvmap_do_cache_maint(struct nvmap_handle *h,
+	unsigned long start, unsigned long end, unsigned long op, bool get);
+
+void _nvmap_handle_free(struct nvmap_handle *h)
+{
+	int e;
+	spin_lock(&nvmap_handle_lock);
+
+	/* if 2 contexts call _get and _put simultaneously, the reference
+	 * count may drop to 0 and then increase to 1 before the handle
+	 * can be freed. */
+	if (atomic_read(&h->ref)>0) {
+		spin_unlock(&nvmap_handle_lock);
+		return;
+	}
+	BUG_ON(atomic_read(&h->ref)<0);
+	BUG_ON(atomic_read(&h->pin)!=0);
+
+	rb_erase(&h->node, &nvmap_handles);
+
+	spin_unlock(&nvmap_handle_lock);
+
+	if (h->owner) put_task_struct(h->owner);
+
+	/* remove when NvRmMemMgr compatibility is eliminated */
+	if (h->kern_map) {
+		BUG_ON(!h->alloc);
+		if (h->heap_pgalloc)
+			vm_unmap_ram(h->kern_map, h->size>>PAGE_SHIFT);
+		else {
+			unsigned long addr = (unsigned long)h->kern_map;
+			addr &= ~PAGE_MASK;
+			iounmap((void *)addr);
+		}
+	}
+
+	/* ensure that no stale data remains in the cache for this handle */
+	e = _nvmap_do_cache_maint(h, 0, h->size, NVMEM_CACHE_OP_WB_INV, false);
+
+	if (h->alloc && !h->heap_pgalloc)
+		nvmap_carveout_free(h->carveout.co_heap, h->carveout.block_idx);
+	else if (h->alloc) {
+		unsigned int i;
+		BUG_ON(h->size & ~PAGE_MASK);
+		BUG_ON(!h->pgalloc.pages);
+		_nvmap_remove_mru_vma(h);
+		if (h->pgalloc.area) tegra_iovmm_free_vm(h->pgalloc.area);
+		for (i=0; i<h->size>>PAGE_SHIFT; i++) {
+			ClearPageReserved(h->pgalloc.pages[i]);
+			__free_page(h->pgalloc.pages[i]);
+		}
+		if ((h->size>>PAGE_SHIFT)*sizeof(struct page*)>=PAGE_SIZE)
+			vfree(h->pgalloc.pages);
+		else
+			kfree(h->pgalloc.pages);
+	}
+	h->poison = 0xa5a5a5a5;
+	kfree(h);
+}
+
+#define nvmap_gfp (GFP_KERNEL | __GFP_HIGHMEM | __GFP_NOWARN)
+
+static int _nvmap_alloc_do_coalloc(struct nvmap_handle *h,
+	struct nvmap_carveout *co, size_t align)
+{
+	int idx;
+
+	idx = nvmap_carveout_alloc(co, align, h->size);
+	if (idx != -1) {
+		h->alloc = true;
+		h->heap_pgalloc = false;
+		h->carveout.co_heap = co;
+		h->carveout.block_idx = idx;
+		spin_lock(&co->lock);
+		h->carveout.base = co->blocks[idx].base;
+		spin_unlock(&co->lock);
+	}
+
+	return (idx==-1) ? -ENOMEM : 0;
+}
+
+/* map the backing pages for a heap_pgalloc handle into its IOVMM area */
+static void _nvmap_handle_iovmm_map(struct nvmap_handle *h)
+{
+	tegra_iovmm_addr_t va;
+	unsigned long i;
+
+	BUG_ON(!h->heap_pgalloc || !h->pgalloc.area);
+	BUG_ON(h->size & ~PAGE_MASK);
+	WARN_ON(!h->pgalloc.dirty);
+
+	for (va = h->pgalloc.area->iovm_start, i=0;
+	    va < (h->pgalloc.area->iovm_start + h->size);
+	    i++, va+=PAGE_SIZE) {
+		BUG_ON(!pfn_valid(page_to_pfn(h->pgalloc.pages[i])));
+		tegra_iovmm_vm_insert_pfn(h->pgalloc.area, va,
+			page_to_pfn(h->pgalloc.pages[i]));
+	}
+	h->pgalloc.dirty = false;
+}
+
+static int _nvmap_alloc_do_pgalloc(struct nvmap_handle *h,
+	bool contiguous, bool secure)
+{
+	unsigned int i = 0, cnt = (h->size + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	struct page **pages;
+
+	if (cnt*sizeof(*pages)>=PAGE_SIZE)
+		pages = vmalloc(cnt*sizeof(*pages));
+	else
+		pages = kzalloc(sizeof(*pages)*cnt, GFP_KERNEL);
+
+	if (!pages) return -ENOMEM;
+
+	if (cnt==1 && !secure) contiguous = true;
+
+	/* secure surfaces should only be allocated in discontiguous (IOVM-
+	 * managed) space, so that the mapping can be zapped after it is
+	 * unpinned */
+	WARN_ON(secure && contiguous);
+
+	if (contiguous) {
+		size_t order = get_order(h->size);
+		struct page *compound_page;
+		compound_page = alloc_pages(nvmap_gfp, order);
+		if (!compound_page) goto fail;
+		split_page(compound_page, order);
+		for (i=0; i<cnt; i++)
+			pages[i] = nth_page(compound_page, i);
+		for (; i<(1<<order); i++)
+			__free_page(nth_page(compound_page, i));
+	} else {
+		for (i=0; i<cnt; i++) {
+			pages[i] = alloc_page(nvmap_gfp);
+			if (!pages[i]) {
+                            pr_err("failed to allocate %u pages after %u entries\n",
+                                   cnt, i);
+                            goto fail;
+                        }
+		}
+	}
+
+	h->pgalloc.area = NULL;
+#ifndef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM
+	if (!contiguous) {
+		h->pgalloc.area = tegra_iovmm_create_vm(nvmap_vm_client,
+			NULL, cnt << PAGE_SHIFT,
+			_nvmap_flag_to_pgprot(h->flags, pgprot_kernel));
+		if (!h->pgalloc.area) goto fail;
+		h->pgalloc.dirty = true;
+	}
+#endif
+
+	for (i=0; i<cnt; i++) {
+		void *km;
+		SetPageReserved(pages[i]);
+		km = kmap(pages[i]);
+		if (km) __cpuc_flush_dcache_page(km);
+		outer_flush_range(page_to_phys(pages[i]),
+			page_to_phys(pages[i])+PAGE_SIZE);
+		kunmap(pages[i]);
+	}
+
+	h->size = cnt<<PAGE_SHIFT;
+	h->pgalloc.pages = pages;
+	h->heap_pgalloc = true;
+	h->pgalloc.contig = contiguous;
+	INIT_LIST_HEAD(&h->pgalloc.mru_list);
+	h->alloc = true;
+	return 0;
+
+fail:
+	while (i--) __free_page(pages[i]);
+	if (pages && (cnt*sizeof(*pages)>=PAGE_SIZE)) vfree(pages);
+	else if (pages) kfree(pages);
+	return -ENOMEM;
+}
+
+static struct nvmap_handle *_nvmap_handle_create(
+	struct task_struct *owner, size_t size)
+{
+	struct nvmap_handle *h = kzalloc(sizeof(*h), GFP_KERNEL);
+	struct nvmap_handle *b;
+	struct rb_node **p;
+	struct rb_node *parent = NULL;
+
+	if (!h) return NULL;
+	atomic_set(&h->ref, 1);
+	atomic_set(&h->pin, 0);
+	h->owner = owner;
+	h->size = h->orig_size = size;
+	h->flags = NVMEM_HANDLE_WRITE_COMBINE;
+
+	spin_lock(&nvmap_handle_lock);
+	p = &nvmap_handles.rb_node;
+	while (*p) {
+		parent = *p;
+		b = rb_entry(parent, struct nvmap_handle, node);
+		if (h > b) p = &parent->rb_right;
+		else p = &parent->rb_left;
+	}
+	rb_link_node(&h->node, parent, p);
+	rb_insert_color(&h->node, &nvmap_handles);
+	spin_unlock(&nvmap_handle_lock);
+	if (owner) get_task_struct(owner);
+	return h;
+}
+
+/* nvmap pte manager */
+
+static void _nvmap_set_pte_at(unsigned long addr, unsigned long pfn,
+	pgprot_t prot)
+{
+	u32 off;
+	int idx;
+	pte_t *pte;
+
+	BUG_ON(!addr);
+	idx = NVMAP_PTE_INDEX(addr);
+	off = NVMAP_PTE_OFFSET(addr) & (PTRS_PER_PTE-1);
+
+	pte = nvmap_pte[idx] + off;
+	set_pte_ext(pte, pfn_pte(pfn, prot), 0);
+	flush_tlb_kernel_page(addr);
+}
+
 static int _nvmap_map_pte(unsigned long pfn, pgprot_t prot, void **vaddr)
 {
 	static unsigned int last_bit = 0;
 	unsigned long bit;
-	pte_t *pte;
 	unsigned long addr;
 	unsigned long flags;
-        u32 off;
-        int idx;
 
 	spin_lock_irqsave(&nvmap_ptelock, flags);
 
@@ -141,12 +1227,7 @@ static int _nvmap_map_pte(unsigned long pfn, pgprot_t prot, void **vaddr)
 
 	addr = NVMAP_BASE + bit*PAGE_SIZE;
 
-	idx = NVMAP_PTE_INDEX(addr);
-	off = NVMAP_PTE_OFFSET(addr) & (PTRS_PER_PTE-1);
-
-	pte = nvmap_pte[idx] + off;
-	set_pte_ext(pte, pfn_pte(pfn, prot), 0);
-	flush_tlb_kernel_page(addr);
+	_nvmap_set_pte_at(addr, pfn, prot);
 	*vaddr = (void *)addr;
 	return 0;
 }
@@ -195,33 +1276,20 @@ static void nvmap_vma_close(struct vm_area_struct *vma) {
 	struct nvmap_vma_priv *priv = vma->vm_private_data;
 
 	if (priv && !atomic_dec_return(&priv->ref)) {
-		NvRmMemHandle hmem = priv->hmem;
-		if (hmem) {
-			if (hmem->coherency==NvOsMemAttribute_WriteBack)
-				dmac_clean_all();
-			NvRmMemHandleFree(priv->hmem);
-		}
+		if (priv->h) _nvmap_handle_put(priv->h);
 		kfree(priv);
 	}
 	vma->vm_private_data = NULL;
 }
 
-extern struct page *NvOsPageGetPage(NvOsPageAllocHandle, size_t);
-#define nvmap_range(x,y) (x), (x)+(y)
-
-#define is_same_page(a, b)  \
-	((unsigned long)(a)>>PAGE_SHIFT == (unsigned long)(b)>>PAGE_SHIFT)
-
 static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
 	struct nvmap_vma_priv *priv;
-	struct page *page;
-	unsigned long pfn;
 	unsigned long offs;
 
 	offs = (unsigned long)(vmf->virtual_address - vma->vm_start);
 	priv = vma->vm_private_data;
-	if (!priv || !priv->hmem)
+	if (!priv || !priv->h || !priv->h->alloc)
 		return VM_FAULT_SIGBUS;
 
 	offs += priv->offs;
@@ -229,27 +1297,22 @@ static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 	 * offset from the original VMA */
 	offs += (vma->vm_pgoff << PAGE_SHIFT);
 
-	if (offs >= priv->hmem->size)
+	if (offs >= priv->h->size)
 		return VM_FAULT_SIGBUS;
 
-	switch (priv->hmem->heap) {
-	case NvRmHeap_ExternalCarveOut:
-	case NvRmHeap_IRam:
-		pfn = ((priv->hmem->PhysicalAddress+offs) >> PAGE_SHIFT);
+	if (!priv->h->heap_pgalloc) {
+		unsigned long pfn;
+		BUG_ON(priv->h->carveout.base & ~PAGE_MASK);
+		pfn = ((priv->h->carveout.base + offs) >> PAGE_SHIFT);
 		vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
 		return VM_FAULT_NOPAGE;
-
-	case NvRmHeap_GART:
-	case NvRmHeap_External:
-		if (!priv->hmem->hPageHandle)
-			return VM_FAULT_SIGBUS;
-		page = NvOsPageGetPage(priv->hmem->hPageHandle, offs);
+	} else {
+		struct page *page;
+		offs >>= PAGE_SHIFT;
+		page = priv->h->pgalloc.pages[offs];
 		if (page) get_page(page);
 		vmf->page = page;
 		return (page) ? 0 : VM_FAULT_SIGBUS;
-
-	default:
-		return VM_FAULT_SIGBUS;
 	}
 }
 
@@ -274,17 +1337,44 @@ static long nvmap_ioctl(struct file *filp,
 		return -EFAULT;
 
 	switch (cmd) {
+	case NVMEM_IOC_CREATE:
+	case NVMEM_IOC_CLAIM:
+	case NVMEM_IOC_FROM_ID:
+		err = nvmap_ioctl_create(filp, cmd, uarg);
+		break;
+
+	case NVMEM_IOC_GET_ID:
+		err = nvmap_ioctl_getid(filp, uarg);
+		break;
+
+	case NVMEM_IOC_PARAM:
+		err = nvmap_ioctl_get_param(filp, uarg);
+		break;
+
+	case NVMEM_IOC_UNPIN_MULT:
+	case NVMEM_IOC_PIN_MULT:
+		err = nvmap_ioctl_pinop(filp, cmd==NVMEM_IOC_PIN_MULT, uarg);
+		break;
+
+	case NVMEM_IOC_ALLOC:
+		err = nvmap_ioctl_alloc(filp, uarg);
+		break;
+
+	case NVMEM_IOC_FREE:
+		err = nvmap_ioctl_free(filp, arg);
+		break;
+
 	case NVMEM_IOC_MMAP:
 		err = nvmap_map_into_caller_ptr(filp, uarg);
 		break;
 
 	case NVMEM_IOC_WRITE:
 	case NVMEM_IOC_READ:
-		err = nvmap_rw_handle(filp, cmd==NVMEM_IOC_READ, uarg);
+		err = nvmap_ioctl_rw_handle(filp, cmd==NVMEM_IOC_READ, uarg);
 		break;
 
 	case NVMEM_IOC_CACHE:
-		err = nvmap_cache_maint(filp, uarg);
+		err = nvmap_ioctl_cache_maint(filp, uarg);
 		break;
 
 	default:
@@ -293,46 +1383,746 @@ static long nvmap_ioctl(struct file *filp,
 	return err;
 }
 
-static int nvmap_release(struct inode *inode, struct file *file)
+/* must be called with the ref_lock held - given a user-space handle ID
+ * ref, validate that the handle_ref object may be used by the caller */
+struct nvmap_handle_ref *_nvmap_ref_lookup_locked(
+	struct nvmap_file_priv *priv, unsigned long ref)
 {
+	struct rb_node *n = priv->handle_refs.rb_node;
+
+	while (n) {
+		struct nvmap_handle_ref *r;
+		r = rb_entry(n, struct nvmap_handle_ref, node);
+		if ((unsigned long)r->h == ref) return r;
+		else if (ref > (unsigned long)r->h) n = n->rb_right;
+		else n = n->rb_left;
+	}
+
+	return NULL;
+}
+
+/* must be called inside nvmap_pin_lock, to ensure that an entire stream
+ * of pins will complete without competition from a second stream. returns
+ * 0 if the pin was successful, -ENOMEM on failure */
+static int _nvmap_handle_pin_locked(struct nvmap_handle *h)
+{
+	struct tegra_iovmm_area *area;
+	BUG_ON(!h->alloc);
+
+	h = _nvmap_handle_get(h);
+	if (!h) return -ENOMEM;
+
+	if (atomic_inc_return(&h->pin)==1) {
+		if (h->heap_pgalloc && !h->pgalloc.contig) {
+			area = _nvmap_get_vm(h);
+			if (!area) {
+				/* no race here, inside the pin mutex */
+				atomic_dec(&h->pin);
+				_nvmap_handle_put(h);
+				return -ENOMEM;
+			}
+			if (area != h->pgalloc.area)
+				h->pgalloc.dirty = true;
+			h->pgalloc.area = area;
+		}
+	}
+	return 0;
+}
+
+/* doesn't need to be called inside nvmap_pin_lock, since this will only
+ * expand the available VM area */
+static int _nvmap_handle_unpin(struct nvmap_handle *h)
+{
+	int ret = 0;
+
+	if (atomic_read(&h->pin)==0) {
+		pr_err("%s: %s attempting to unpin an unpinned handle\n",
+			__func__, current->comm);
+		dump_stack();
+		return 0;
+	}
+
+	BUG_ON(!h->alloc || atomic_read(&h->pin)==0);
+	if (!atomic_dec_return(&h->pin)) {
+		if (h->heap_pgalloc && h->pgalloc.area) {
+			/* if a secure handle is clean (i.e., mapped into
+			 * IOVMM, it needs to be zapped on unpin. */
+			if (h->secure && !h->pgalloc.dirty) {
+				tegra_iovmm_zap_vm(h->pgalloc.area);
+				h->pgalloc.dirty = true;
+			}
+			_nvmap_insert_mru_vma(h);
+			ret=1;
+		}
+	}
+	_nvmap_handle_put(h);
+	return ret;
+}
+
+/* pin a list of handles, mapping IOVMM areas if needed. may sleep, if
+ * a handle's IOVMM area has been reclaimed and insufficient IOVMM space
+ * is available to complete the list pin. no intervening pin operations
+ * will interrupt this, and no validation is performed on the handles
+ * that are provided. */
+static int _nvmap_handle_pin_fast(unsigned int nr, struct nvmap_handle **h)
+{
+	unsigned int i;
+	int ret = 0;
+
+	mutex_lock(&nvmap_pin_lock);
+	for (i=0; i<nr && !ret; i++) {
+		ret = wait_event_interruptible(nvmap_pin_wait,
+			!_nvmap_handle_pin_locked(h[i]));
+	}
+	mutex_unlock(&nvmap_pin_lock);
+
+	if (ret) {
+		int do_wake = 0;
+		while (i--) do_wake |= _nvmap_handle_unpin(h[i]);
+		if (do_wake) wake_up(&nvmap_pin_wait);
+		return -EINTR;
+	} else {
+		for (i=0; i<nr; i++)
+			if (h[i]->heap_pgalloc && h[i]->pgalloc.dirty)
+				_nvmap_handle_iovmm_map(h[i]);
+	}
+
+	return 0;
+}
+
+static int _nvmap_do_global_unpin(unsigned long ref)
+{
+	struct nvmap_handle *h;
+	int w;
+
+	h = _nvmap_validate_get(ref, true);
+	if (unlikely(!h)) {
+		pr_err("%s: %s attempting to unpin non-existent handle\n",
+			__func__, current->group_leader->comm);
+		return 0;
+	}
+
+	pr_err("%s: %s unpinning %s's %uB %s handle without local context\n",
+		__func__, current->group_leader->comm,
+		(h->owner) ? h->owner->comm : "kernel", h->orig_size,
+		(h->heap_pgalloc && !h->pgalloc.contig) ? "iovmm" :
+		(h->heap_pgalloc) ? "sysmem" : "carveout");
+
+	w = _nvmap_handle_unpin(h);
+	_nvmap_handle_put(h);
+	return w;
+}
+
+static void _nvmap_do_unpin(struct nvmap_file_priv *priv,
+	unsigned int nr, unsigned long *refs)
+{
+	struct nvmap_handle_ref *r;
+	unsigned int i;
+	int do_wake = 0;
+
+	spin_lock(&priv->ref_lock);
+	for (i=0; i<nr; i++) {
+		if (!refs[i]) continue;
+		r = _nvmap_ref_lookup_locked(priv, refs[i]);
+		if (unlikely(!r)) {
+			if (priv->su)
+				do_wake |= _nvmap_do_global_unpin(refs[i]);
+			else
+				pr_err("%s: %s unpinning invalid handle\n",
+					__func__, current->comm);
+		} else if (unlikely(!atomic_add_unless(&r->pin, -1, 0)))
+			pr_err("%s: %s unpinning unpinned handle\n",
+				__func__, current->comm);
+		else
+			do_wake |= _nvmap_handle_unpin(r->h);
+	}
+	spin_unlock(&priv->ref_lock);
+	if (do_wake) wake_up(&nvmap_pin_wait);
+}
+
+/* pins a list of handle_ref objects; same conditions apply as to
+ * _nvmap_handle_pin, but also bumps the pin count of each handle_ref. */
+static int _nvmap_do_pin(struct nvmap_file_priv *priv,
+	unsigned int nr, unsigned long *refs)
+{
+	int ret = 0;
+	unsigned int i;
+	struct nvmap_handle **h = (struct nvmap_handle **)refs;
+	struct nvmap_handle_ref *r;
+
+	/* to optimize for the common case (client provided valid handle
+	 * references and the pin succeeds), increment the handle_ref pin
+	 * count during validation. in error cases, the tree will need to
+	 * be re-walked, since the handle_ref is discarded so that an
+	 * allocation isn't required. if a handle_ref is not found,
+	 * locally validate that the caller has permission to pin the handle;
+	 * handle_refs are not created in this case, so it is possible that
+	 * if the caller crashes after pinning a global handle, the handle
+	 * will be permanently leaked. */
+	spin_lock(&priv->ref_lock);
+	for (i=0; i<nr && !ret; i++) {
+		r = _nvmap_ref_lookup_locked(priv, refs[i]);
+		if (!r && (!(priv->su || h[i]->global ||
+		    current->group_leader == h[i]->owner)))
+			ret = -EPERM;
+		else if (r) atomic_inc(&r->pin);
+		else {
+			pr_err("%s: %s pinning %s's %uB handle without "
+				"local context\n", __func__,
+				current->group_leader->comm,
+				h[i]->owner->comm, h[i]->orig_size);
+                }
+	}
+
+	while (ret && i--) {
+		r = _nvmap_ref_lookup_locked(priv, refs[i]);
+		if (r) atomic_dec(&r->pin);
+	}
+	spin_unlock(&priv->ref_lock);
+
+	if (ret) return ret;
+
+	mutex_lock(&nvmap_pin_lock);
+	for (i=0; i<nr && !ret; i++) {
+		ret = wait_event_interruptible(nvmap_pin_wait,
+			!_nvmap_handle_pin_locked(h[i]));
+	}
+	mutex_unlock(&nvmap_pin_lock);
+
+	if (ret) {
+		int do_wake = 0;
+		spin_lock(&priv->ref_lock);
+		while (i--) {
+			r = _nvmap_ref_lookup_locked(priv, refs[i]);
+			do_wake |= _nvmap_handle_unpin(r->h);
+			if (r) atomic_dec(&r->pin);
+		}
+		spin_unlock(&priv->ref_lock);
+		if (do_wake) wake_up(&nvmap_pin_wait);
+		return -EINTR;
+	} else {
+		for (i=0; i<nr; i++) {
+			if (h[i]->heap_pgalloc && h[i]->pgalloc.dirty)
+				_nvmap_handle_iovmm_map(h[i]);
+		}
+	}
+
+	return 0;
+}
+
+static int nvmap_ioctl_pinop(struct file *filp,
+	bool is_pin, void __user *arg)
+{
+	struct nvmem_pin_handle op;
+	struct nvmap_handle *h;
+	unsigned long on_stack[16];
+	unsigned long *refs;
+	unsigned long __user *output;
+	unsigned int i;
+	int err;
+
+	err = copy_from_user(&op, arg, sizeof(op));
+	if (err) return err;
+
+	if (!op.count) return -EINVAL;
+
+	if (op.count > 1) {
+		size_t bytes = op.count * sizeof(unsigned long *);
+		if (!access_ok(VERIFY_READ, (void *)op.handles, bytes))
+			return -EPERM;
+		if (is_pin && !access_ok(VERIFY_WRITE, (void *)op.addr, bytes))
+			return -EPERM;
+
+		if (op.count <= ARRAY_SIZE(on_stack)) refs = on_stack;
+		else refs = kzalloc(bytes, GFP_KERNEL);
+
+		if (!refs) return -ENOMEM;
+		err = copy_from_user(refs, (void*)op.handles, bytes);
+		if (err) goto out;
+	} else {
+		refs = on_stack;
+		on_stack[0] = (unsigned long)op.handles;
+	}
+
+	if (is_pin)
+		err = _nvmap_do_pin(filp->private_data, op.count, refs);
+	else
+		_nvmap_do_unpin(filp->private_data, op.count, refs);
+
+	/* skip the output stage on unpin */
+	if (err || !is_pin) goto out;
+
+	/* it is guaranteed that if _nvmap_do_pin returns 0 that
+	 * all of the handle_ref objects are valid, so dereferencing directly
+	 * here is safe */
+	if (op.count > 1)
+		output = (unsigned long __user *)op.addr;
+	else {
+		struct nvmem_pin_handle __user *tmp = arg;
+		output = (unsigned long __user *)&(tmp->addr);
+	}
+
+	for (i=0; i<op.count; i++) {
+		unsigned long addr;
+		h = (struct nvmap_handle *)refs[i];
+		if (h->heap_pgalloc && h->pgalloc.contig)
+			addr = page_to_phys(h->pgalloc.pages[0]);
+		else if (h->heap_pgalloc)
+			addr = h->pgalloc.area->iovm_start;
+		else
+			addr = h->carveout.base;
+
+		__put_user(addr, &output[i]);
+	}
+
+out:
+	if (refs != on_stack) kfree(refs);
+	return err;
+}
+
+static int nvmap_release(struct inode *inode, struct file *filp)
+{
+	struct nvmap_file_priv *priv = filp->private_data;
+	struct rb_node *n;
+	struct nvmap_handle_ref *r;
+	int refs;
+	int do_wake = 0;
+	int pins;
+
+	if (!priv) return 0;
+
+	while ((n = rb_first(&priv->handle_refs))) {
+		r = rb_entry(n, struct nvmap_handle_ref, node);
+		rb_erase(&r->node, &priv->handle_refs);
+		pins = atomic_read(&r->pin);
+		atomic_set(&r->pin, 0);
+		while (pins--) do_wake |= _nvmap_handle_unpin(r->h);
+		refs = atomic_read(&r->refs);
+		if (r->h->alloc && r->h->heap_pgalloc && !r->h->pgalloc.contig)
+			atomic_sub(r->h->size, &priv->iovm_commit);
+		while (refs--) _nvmap_handle_put(r->h);
+		kfree(r);
+	}
+	if (do_wake) wake_up(&nvmap_pin_wait);
+	kfree(priv);
 	return 0;
 }
 
 static int nvmap_open(struct inode *inode, struct file *filp)
 {
 	/* eliminate read, write and llseek support on this node */
+	struct nvmap_file_priv *priv;
 	int ret;
 
+	/* nvmap doesn't track total number of pinned references, so its
+	 * IOVMM client is always locked. */
+	if (!nvmap_vm_client) {
+		mutex_lock(&nvmap_pin_lock);
+		if (!nvmap_vm_client) {
+			nvmap_vm_client = tegra_iovmm_alloc_client("gpu", NULL);
+			if (nvmap_vm_client)
+				tegra_iovmm_client_lock(nvmap_vm_client);
+		}
+		mutex_unlock(&nvmap_pin_lock);
+	}
+
 	ret = nonseekable_open(inode, filp);
 	if (unlikely(ret))
 		return ret;
 
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv) return -ENOMEM;
+	priv->handle_refs = RB_ROOT;
+	priv->su = (filp->f_op == &knvmap_fops);
+
+	atomic_set(&priv->iovm_commit, 0);
+
+	if (nvmap_vm_client)
+		priv->iovm_limit = tegra_iovmm_get_vm_size(nvmap_vm_client);
+#ifdef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM
+	/* to prevent fragmentation-caused deadlocks, derate the size of
+	 * the IOVM space to 75% */
+	priv->iovm_limit >>= 2;
+	priv->iovm_limit *= 3;
+#endif
+
+	spin_lock_init(&priv->ref_lock);
+
 	filp->f_mapping->backing_dev_info = &nvmap_bdi;
 
-	filp->private_data = NULL;
+	filp->private_data = priv;
+	return 0;
+}
+
+static int nvmap_ioctl_getid(struct file *filp, void __user *arg)
+{
+	struct nvmem_create_handle op;
+	struct nvmap_handle *h = NULL;
+	int err;
+
+	err = copy_from_user(&op, arg, sizeof(op));
+	if (err) return err;
+
+	if (!op.handle) return -EINVAL;
+
+	h = _nvmap_validate_get((unsigned long)op.handle,
+		filp->f_op==&knvmap_fops);
+
+	if (h) {
+		op.id = (__u32)h;
+		/* when the owner of a handle gets its ID, this is treated
+		 * as a granting of the handle for use by other processes.
+		 * however, the super-user is not capable of promoting a
+		 * handle to global status if it was created in another
+		 * process. */
+		if (current->group_leader == h->owner) h->global = true;
+
+		/* getid is not supposed to result in a ref count increase */
+		_nvmap_handle_put(h);
+
+		return copy_to_user(arg, &op, sizeof(op));
+	}
+	return -EPERM;
+}
+
+/* attempts to allocate from either contiguous system memory or IOVMM space */
+static int _nvmap_do_page_alloc(struct nvmap_file_priv *priv,
+	struct nvmap_handle *h, unsigned int heap_mask,
+	size_t align, bool secure)
+{
+	int ret = -ENOMEM;
+	size_t page_size = (h->size + PAGE_SIZE - 1) & ~(PAGE_SIZE-1);
+#ifdef IOVMM_FIRST
+	unsigned int fallback[] = { NVMEM_HEAP_IOVMM, NVMEM_HEAP_SYSMEM, 0 };
+#else
+	unsigned int fallback[] = { NVMEM_HEAP_SYSMEM, NVMEM_HEAP_IOVMM, 0 };
+#endif
+	unsigned int *m = fallback;
+
+	/* secure allocations must not be performed from sysmem */
+	if (secure) heap_mask &= ~NVMEM_HEAP_SYSMEM;
+
+	if (align > PAGE_SIZE) return -EINVAL;
+
+
+	while (*m && ret) {
+		if (heap_mask & NVMEM_HEAP_SYSMEM & *m)
+			ret = _nvmap_alloc_do_pgalloc(h, true, secure);
+
+		else if (heap_mask & NVMEM_HEAP_IOVMM & *m) {
+			/* increment the committed IOVM space prior to
+			 * allocation, to avoid race conditions with other
+			 * threads simultaneously allocating. this is
+			 * conservative, but guaranteed to work */
+
+			int oc;
+			oc = atomic_add_return(page_size, &priv->iovm_commit);
+
+			if (oc <= priv->iovm_limit)
+				ret = _nvmap_alloc_do_pgalloc(h, false, secure);
+			else
+				ret = -ENOMEM;
+			/* on failure, or when do_pgalloc promotes a non-
+			 * contiguous request into a contiguous request,
+			 * release the commited iovm space */
+			if (ret || h->pgalloc.contig)
+				atomic_sub(page_size, &priv->iovm_commit);
+		}
+		m++;
+	}
+	return ret;
+}
+
+/* attempts to allocate from the carveout heaps */
+static int _nvmap_do_carveout_alloc(struct nvmap_handle *h,
+	unsigned int heap_mask, size_t align)
+{
+	int ret = -ENOMEM;
+	struct nvmap_carveout_node *n;
+
+	down_read(&nvmap_context.list_sem);
+	list_for_each_entry(n, &nvmap_context.heaps, heap_list) {
+		if (heap_mask & n->heap_bit)
+			ret = _nvmap_alloc_do_coalloc(h, &n->carveout, align);
+		if (!ret) break;
+	}
+	up_read(&nvmap_context.list_sem);
+	return ret;
+}
+
+static int _nvmap_do_alloc(struct nvmap_file_priv *priv,
+	unsigned long href, unsigned int heap_mask, size_t align,
+	unsigned int flags, bool secure, bool carveout_first)
+{
+	int ret = -ENOMEM;
+	struct nvmap_handle_ref *r;
+	struct nvmap_handle *h;
+
+	if (!href) return -EINVAL;
+
+	spin_lock(&priv->ref_lock);
+	r = _nvmap_ref_lookup_locked(priv, href);
+	spin_unlock(&priv->ref_lock);
+
+	if (!r) return -EPERM;
+
+	h = r->h;
+	if (h->alloc) return 0;
+	h->flags = flags;
+
+	align = max_t(size_t, align, L1_CACHE_BYTES);
+
+	if (secure) heap_mask &= ~NVMEM_HEAP_CARVEOUT_MASK;
+
+	if (carveout_first || (heap_mask & NVMEM_HEAP_CARVEOUT_IRAM)) {
+		ret = _nvmap_do_carveout_alloc(h, heap_mask, align);
+		if (ret) ret = _nvmap_do_page_alloc(priv, h,
+				   heap_mask, align, secure);
+	} else {
+		ret = _nvmap_do_page_alloc(priv, h, heap_mask, align, secure);
+		if (ret) ret = _nvmap_do_carveout_alloc(h, heap_mask, align);
+	}
+
+	BUG_ON((!ret && !h->alloc) || (ret && h->alloc));
+	return ret;
+}
+
+static int nvmap_ioctl_alloc(struct file *filp, void __user *arg)
+{
+	struct nvmem_alloc_handle op;
+	struct nvmap_file_priv *priv = filp->private_data;
+	bool secure = false;
+#ifdef IOVMM_FIRST
+	bool carveout_first = false;
+#else
+	bool carveout_first = true;
+#endif
+	int err;
+
+	err = copy_from_user(&op, arg, sizeof(op));
+	if (err) return err;
+
+	if (op.align & (op.align-1)) return -EINVAL;
+
+	/* user-space handles are aligned to page boundaries, to prevent
+	 * data leakage. */
+	op.align = max_t(size_t, op.align, PAGE_SIZE);
+
+	if (op.flags & NVMEM_HANDLE_SECURE) secure = true;
+
+	/* TODO: implement a way to specify carveout-first vs
+	 * carveout-second */
+	return _nvmap_do_alloc(priv, op.handle, op.heap_mask,
+		op.align, (op.flags & 0x3), secure, carveout_first);
+}
+
+static int _nvmap_do_free(struct nvmap_file_priv *priv, unsigned long href)
+{
+	struct nvmap_handle_ref *r;
+	struct nvmap_handle *h;
+	int do_wake = 0;
 
+	if (!href) return 0;
+
+	spin_lock(&priv->ref_lock);
+	r = _nvmap_ref_lookup_locked(priv, href);
+
+	if (!r) {
+		spin_unlock(&priv->ref_lock);
+		pr_err("%s attempting to free unrealized handle\n",
+			current->group_leader->comm);
+		return -EPERM;
+	}
+
+	h = r->h;
+
+	if (!atomic_dec_return(&r->refs)) {
+		int pins = atomic_read(&r->pin);
+		rb_erase(&r->node, &priv->handle_refs);
+		spin_unlock(&priv->ref_lock);
+		if (pins) pr_err("%s: %s freeing %s's pinned %s %s %uB handle\n",
+			__func__, current->comm,
+			(r->h->owner) ? r->h->owner->comm : "kernel",
+			(r->h->global) ? "global" : "private",
+			(r->h->alloc && r->h->heap_pgalloc)?"page-alloc" :
+			(r->h->alloc) ? "carveout" : "unallocated",
+			r->h->orig_size);
+		while (pins--) do_wake |= _nvmap_handle_unpin(r->h);
+		kfree(r);
+		if (h->alloc && h->heap_pgalloc && !h->pgalloc.contig)
+			atomic_sub(h->size, &priv->iovm_commit);
+		if (do_wake) wake_up(&nvmap_pin_wait);
+	} else
+		spin_unlock(&priv->ref_lock);
+
+	BUG_ON(!atomic_read(&h->ref));
+	_nvmap_handle_put(h);
 	return 0;
 }
 
+static int nvmap_ioctl_free(struct file *filp, unsigned long arg)
+{
+	return _nvmap_do_free(filp->private_data, arg);
+}
+
+/* given a size, pre-existing handle ID, or a preserved handle key, create
+ * a handle and a reference to the handle in the per-context data */
+static int _nvmap_do_create(struct nvmap_file_priv *priv,
+	unsigned int cmd, unsigned long key, bool su,
+	struct nvmap_handle_ref **ref)
+{
+	struct nvmap_handle_ref *r = NULL;
+	struct nvmap_handle *h = NULL;
+	struct rb_node **p, *parent = NULL;
+
+	if (cmd == NVMEM_IOC_FROM_ID) {
+		/* only ugly corner case to handle with from ID:
+		 *
+		 * normally, if the handle that is being duplicated is IOVMM-
+		 * backed, the handle should fail to duplicate if duping it
+		 * would over-commit IOVMM space.  however, if the handle is
+		 * already duplicated in the client process (or the client
+		 * is duplicating a handle it created originally), IOVMM space
+		 * should not be doubly-reserved.
+		 */
+		h = _nvmap_validate_get(key, priv->su);
+
+		if (!h) {
+			pr_err("%s: %s duplicate handle failed\n", __func__,
+				current->group_leader->comm);
+			return -EPERM;
+		}
+
+		if (!h->alloc) {
+			pr_err("%s: attempting to clone unallocated "
+				"handle\n", __func__);
+			_nvmap_handle_put(h);
+			h = NULL;
+			return -EINVAL;
+		}
+
+		spin_lock(&priv->ref_lock);
+		r = _nvmap_ref_lookup_locked(priv, (unsigned long)h);
+		spin_unlock(&priv->ref_lock);
+		if (r) {
+			atomic_inc(&r->refs);
+			*ref = r;
+			return 0;
+		}
+
+		/* verify that adding this handle to the process' access list
+		 * won't exceed the IOVM limit */
+		if (h->heap_pgalloc && !h->pgalloc.contig) {
+			int oc = atomic_add_return(h->size, &priv->iovm_commit);
+			if (oc > priv->iovm_limit) {
+				atomic_sub(h->size, &priv->iovm_commit);
+				_nvmap_handle_put(h);
+				h = NULL;
+				pr_err("%s: %s duplicating handle would "
+					"over-commit iovmm space (%d / %dB\n",
+					__func__, current->group_leader->comm,
+					oc, priv->iovm_limit);
+				return -ENOMEM;
+			}
+		}
+	} else if (cmd == NVMEM_IOC_CREATE) {
+		h = _nvmap_handle_create(current->group_leader, key);
+		if (!h) return -ENOMEM;
+	} else {
+		h = _nvmap_claim_preserved(current->group_leader, key);
+		if (!h) return -EINVAL;
+	}
+
+	BUG_ON(!h);
+
+	/* if the client does something strange, like calling CreateFromId
+	 * when it was the original creator, avoid creating two handle refs
+	 * for the same handle */
+	spin_lock(&priv->ref_lock);
+	r = kzalloc(sizeof(*r), GFP_KERNEL);
+	if (!r) {
+		spin_unlock(&priv->ref_lock);
+		if (h) _nvmap_handle_put(h);
+		return -ENOMEM;
+	}
+
+	atomic_set(&r->refs, 1);
+	r->h = h;
+	atomic_set(&r->pin, 0);
+
+	p = &priv->handle_refs.rb_node;
+	while (*p) {
+		struct nvmap_handle_ref *l;
+		parent = *p;
+		l = rb_entry(parent, struct nvmap_handle_ref, node);
+		if (r->h > l->h) p = &parent->rb_right;
+		else p = &parent->rb_left;
+	}
+	rb_link_node(&r->node, parent, p);
+	rb_insert_color(&r->node, &priv->handle_refs);
+
+	spin_unlock(&priv->ref_lock);
+	*ref = r;
+	return 0;
+}
+
+static int nvmap_ioctl_create(struct file *filp,
+	unsigned int cmd, void __user *arg)
+{
+	struct nvmem_create_handle op;
+	struct nvmap_handle_ref *r = NULL;
+	struct nvmap_file_priv *priv = filp->private_data;
+	unsigned long key;
+	int err = 0;
+
+	err = copy_from_user(&op, arg, sizeof(op));
+	if (err) return err;
+
+	if (!priv) return -ENODEV;
+
+	/* user-space-created handles are expanded to be page-aligned,
+	 * so that mmap() will not accidentally leak a different allocation */
+	if (cmd==NVMEM_IOC_CREATE)
+		key = (op.size + PAGE_SIZE - 1) & ~(PAGE_SIZE-1);
+	else if (cmd==NVMEM_IOC_CLAIM)
+		key = op.key;
+	else if (cmd==NVMEM_IOC_FROM_ID)
+		key = op.id;
+
+	err = _nvmap_do_create(priv, cmd, key, (filp->f_op==&knvmap_fops), &r);
+
+	if (!err) {
+		op.handle = (uintptr_t)r->h;
+		/* since the size is spoofed to a page-multiple above,
+		 * clobber the orig_size field back to the requested value for
+		 * debugging. */
+		if (cmd == NVMEM_IOC_CREATE) r->h->orig_size = op.size;
+		err = copy_to_user(arg, &op, sizeof(op));
+		if (err) _nvmap_do_free(priv, op.handle);
+	}
+
+	return err;
+}
+
 static int nvmap_map_into_caller_ptr(struct file *filp, void __user *arg)
 {
 	struct nvmem_map_caller op;
-	struct nvmap_vma_priv *priv;
+	struct nvmap_vma_priv *vpriv;
 	struct vm_area_struct *vma;
-	NvRmMemHandle hmem;
+	struct nvmap_handle *h;
 	int err = 0;
 
 	err = copy_from_user(&op, arg, sizeof(op));
-	if (err)
-		return err;
+	if (err) return err;
 
-	hmem = (NvRmMemHandle)op.handle;
-	if (!hmem)
-		return -EINVAL;
+	if (!op.handle) return -EINVAL;
+
+	h = _nvmap_validate_get(op.handle, (filp->f_op==&knvmap_fops));
+	if (!h) return -EINVAL;
 
 	down_read(&current->mm->mmap_sem);
 
-	vma = find_vma(current->mm, (unsigned long)op.addr);
+	vma = find_vma(current->mm, op.addr);
 	if (!vma || !vma->vm_private_data) {
 		err = -ENOMEM;
 		goto out;
@@ -343,54 +2133,49 @@ static int nvmap_map_into_caller_ptr(struct file *filp, void __user *arg)
 		goto out;
 	}
 
-	if ((op.offset + op.length) >
-		((hmem->size+PAGE_SIZE-1)&PAGE_MASK)) {
+	if ((op.offset + op.length) > h->size) {
 		err = -EADDRNOTAVAIL;
 		goto out;
 	}
 
-	priv = vma->vm_private_data;
-	BUG_ON(!priv);
+	vpriv = vma->vm_private_data;
+	BUG_ON(!vpriv);
 
 	/* the VMA must exactly match the requested mapping operation, and the
 	 * VMA that is targetted must have been created originally by /dev/nvmap
 	 */
-	if (((void*)vma->vm_start != op.addr) ||
-		(vma->vm_ops != &nvmap_vma_ops) ||
-		(vma->vm_end-vma->vm_start != op.length)) {
+	if ((vma->vm_start != op.addr) || (vma->vm_ops != &nvmap_vma_ops) ||
+	    (vma->vm_end-vma->vm_start != op.length)) {
 		err = -EPERM;
 		goto out;
 	}
 
 	/* verify that each mmap() system call creates a unique VMA */
 
-	if (priv->hmem && hmem==priv->hmem)
+	if (vpriv->h && h==vpriv->h)
 		goto out;
-	else if (priv->hmem) {
+	else if (vpriv->h) {
 		err = -EADDRNOTAVAIL;
 		goto out;
 	}
 
-	if (hmem->alignment & ~PAGE_MASK) {
+	if (!h->heap_pgalloc && (h->carveout.base & ~PAGE_MASK)) {
 		err = -EFAULT;
 		goto out;
 	}
 
-	priv->hmem = hmem;
-	priv->offs = op.offset;
+	vpriv->h = h;
+	vpriv->offs = op.offset;
 
 	/* if the hmem is not writeback-cacheable, drop back to a page mapping
 	 * which will guarantee DMA coherency
 	 */
-	if (hmem->coherency == NvOsMemAttribute_WriteBack)
-		vma->vm_page_prot = pgprot_inner_writeback(vma->vm_page_prot);
-	else
-		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
-
-	NvRmPrivMemIncrRef(hmem);
+	vma->vm_page_prot = _nvmap_flag_to_pgprot(h->flags,
+		vma->vm_page_prot);
 
- out:
+out:
 	up_read(&current->mm->mmap_sem);
+	if (err) _nvmap_handle_put(h);
 	return err;
 }
 /* Initially, the nvmap mmap system call is used to allocate an inaccessible
@@ -412,7 +2197,7 @@ static int nvmap_mmap(struct file *filp, struct vm_area_struct *vma)
 		return -ENOMEM;
 
 	priv->offs = 0;
-	priv->hmem = NULL;
+	priv->h = NULL;
 	atomic_set(&priv->ref, 1);
 
 	vma->vm_flags |= VM_SHARED;
@@ -423,205 +2208,442 @@ static int nvmap_mmap(struct file *filp, struct vm_area_struct *vma)
 	return 0;
 }
 
-static int nvmap_cache_maint(struct file *filp, void __user *arg)
+/* perform cache maintenance on a handle; caller's handle must be pre-
+ * validated. */
+static int _nvmap_do_cache_maint(struct nvmap_handle *h,
+	unsigned long start, unsigned long end, unsigned long op, bool get)
+{
+	pgprot_t prot;
+	void *addr = NULL;
+	void (*inner_maint)(const void*, const void*);
+	void (*outer_maint)(unsigned long, unsigned long);
+	int err = 0;
+
+	if (get) h = _nvmap_handle_get(h);
+
+	if (!h) return -EINVAL;
+
+	/* don't waste time on cache maintenance if the handle isn't cached */
+	if (h->flags == NVMEM_HANDLE_UNCACHEABLE ||
+	    h->flags == NVMEM_HANDLE_WRITE_COMBINE)
+		goto out;
+
+	if (op == NVMEM_CACHE_OP_WB) {
+		inner_maint = smp_dma_clean_range;
+		if (h->flags == NVMEM_HANDLE_CACHEABLE)
+			outer_maint = outer_clean_range;
+		else
+			outer_maint = NULL;
+	} else if (op == NVMEM_CACHE_OP_WB_INV) {
+		inner_maint = smp_dma_flush_range;
+		if (h->flags == NVMEM_HANDLE_CACHEABLE)
+			outer_maint = outer_flush_range;
+		else
+			outer_maint = NULL;
+	} else {
+		inner_maint = smp_dma_inv_range;
+		if (h->flags == NVMEM_HANDLE_CACHEABLE)
+			outer_maint = outer_inv_range;
+		else
+			outer_maint = NULL;
+	}
+
+	prot = _nvmap_flag_to_pgprot(h->flags, pgprot_kernel);
+
+	/* for any write-back operation, it is safe to writeback the entire
+	 * cache rather than just the requested region.  for large regions, it
+	 * is faster to do this than to iterate over every line.
+	 * only implemented for L1-only cacheable handles currently */
+	if (h->flags == NVMEM_HANDLE_INNER_CACHEABLE &&
+	    end-start >= PAGE_SIZE*3 && op != NVMEM_CACHE_OP_INV) {
+		if (op==NVMEM_CACHE_OP_WB) dmac_clean_all();
+		else dmac_flush_all();
+		goto out;
+	}
+
+	while (start < end) {
+		struct page *page = NULL;
+		unsigned long phys;
+		void *src;
+		size_t count;
+
+		if (h->heap_pgalloc) {
+			page = h->pgalloc.pages[start>>PAGE_SHIFT];
+			BUG_ON(!page);
+			get_page(page);
+			phys = page_to_phys(page) + (start & ~PAGE_MASK);
+		} else {
+			phys = h->carveout.base + start;
+		}
+
+		if (!addr) {
+			err = nvmap_map_pte(__phys_to_pfn(phys), prot, &addr);
+			if (err) {
+				if (page) put_page(page);
+				break;
+			}
+		} else {
+			_nvmap_set_pte_at((unsigned long)addr,
+				__phys_to_pfn(phys), prot);
+		}
+
+		src = addr + (phys & ~PAGE_MASK);
+		count = min_t(size_t, end-start, PAGE_SIZE-(phys&~PAGE_MASK));
+
+		inner_maint(src, src+count);
+		if (outer_maint) outer_maint(phys, phys+count);
+		start += count;
+		if (page) put_page(page);
+	}
+
+out:
+	if (h->flags == NVMEM_HANDLE_INNER_CACHEABLE) outer_sync();
+	if (addr) nvmap_unmap_pte(addr);
+	if (get) _nvmap_handle_put(h);
+	return err;
+}
+
+static int nvmap_ioctl_cache_maint(struct file *filp, void __user *arg)
 {
 	struct nvmem_cache_op	op;
 	int			err = 0;
 	struct vm_area_struct	*vma;
-	struct nvmap_vma_priv	*priv;
-	size_t			offs;
-	size_t			end;
-	unsigned long		count;
-	pgprot_t		prot = pgprot_inner_writeback(pgprot_kernel);
+	struct nvmap_vma_priv	*vpriv;
+	unsigned long		start;
+	unsigned long		end;
 
 	err = copy_from_user(&op, arg, sizeof(op));
 	if (err) return err;
 
 	if (!op.handle || !op.addr || op.op<NVMEM_CACHE_OP_WB ||
-		op.op>NVMEM_CACHE_OP_WB_INV)
+	    op.op>NVMEM_CACHE_OP_WB_INV)
 		return -EINVAL;
 
 	vma = find_vma(current->active_mm, (unsigned long)op.addr);
 	if (!vma || vma->vm_ops!=&nvmap_vma_ops ||
-		(unsigned long)op.addr + op.len > vma->vm_end) {
-		err = -EADDRNOTAVAIL;
-		goto out;
-	}
+	    (unsigned long)op.addr + op.len > vma->vm_end)
+		return -EADDRNOTAVAIL;
 
-	priv = (struct nvmap_vma_priv *)vma->vm_private_data;
+	vpriv = (struct nvmap_vma_priv *)vma->vm_private_data;
 
-	if (priv->hmem != (NvRmMemHandle)op.handle) {
-		err = -EFAULT;
-		goto out;
-	}
+	if ((unsigned long)vpriv->h != op.handle)
+		return -EFAULT;
 
-	/* don't waste time on cache maintenance if the handle isn't cached */
-	if (priv->hmem->coherency != NvOsMemAttribute_WriteBack)
-		goto out;
+	start = (unsigned long)op.addr - vma->vm_start;
+	end  = start + op.len;
 
-	offs = (unsigned long)op.addr - vma->vm_start;
-	end  = offs + op.len;
+	return _nvmap_do_cache_maint(vpriv->h, start, end, op.op, true);
+}
 
-	/* for any write-back operation, it is safe to writeback the entire
-	 * cache rather than one line at a time.  for large regions, it
-	 * is faster to do this than to iterate over every line. */
-	if (end-offs >= PAGE_SIZE*3 && op.op != NVMEM_CACHE_OP_INV) {
-		if (op.op==NVMEM_CACHE_OP_WB) dmac_clean_all();
-		else dmac_flush_all();
-		goto out;
+/* copies a single element from the pre-get()'ed handle h, returns
+ * the number of bytes copied, and the address in the nvmap mapping range
+ * which was used (to eliminate re-allocation when copying multiple
+ * elements */
+static ssize_t _nvmap_do_one_rw_handle(struct nvmap_handle *h, int is_read,
+	int is_user, unsigned long start, unsigned long rw_addr,
+	unsigned long bytes, void **nvmap_addr)
+{
+	pgprot_t prot = _nvmap_flag_to_pgprot(h->flags, pgprot_kernel);
+	unsigned long end = start + bytes;
+	unsigned long orig_start = start;
+
+	if (is_user) {
+		if (is_read && !access_ok(VERIFY_WRITE, (void*)rw_addr, bytes))
+			return -EPERM;
+		if (!is_read && !access_ok(VERIFY_READ, (void*)rw_addr, bytes))
+			return -EPERM;
 	}
 
-	while (offs < end) {
+	while (start < end) {
 		struct page *page = NULL;
-		void *addr = NULL, *src;
+		unsigned long phys;
+		size_t count;
+		void *src;
 
-		if (priv->hmem->hPageHandle) {
-			struct page *page;
-			page = NvOsPageGetPage(priv->hmem->hPageHandle, offs);
+		if (h->heap_pgalloc) {
+			page = h->pgalloc.pages[start >> PAGE_SHIFT];
+			BUG_ON(!page);
 			get_page(page);
-			err = nvmap_map_pte(page_to_pfn(page), prot, &addr);
+			phys = page_to_phys(page) + (start & ~PAGE_MASK);
 		} else {
-			unsigned long phys = priv->hmem->PhysicalAddress + offs;
-			err = nvmap_map_pte(phys>>PAGE_SHIFT, prot, &addr);
+			phys = h->carveout.base + start;
 		}
 
-		if (err) {
-			if (page) put_page(page);
-			break;
+		if (!*nvmap_addr) {
+			int err = nvmap_map_pte(__phys_to_pfn(phys),
+				prot, nvmap_addr);
+			if (err) {
+				if (page) put_page(page);
+				count = start - orig_start;
+				return (count) ? count : err;
+			}
+		} else {
+			_nvmap_set_pte_at((unsigned long)*nvmap_addr,
+				__phys_to_pfn(phys), prot);
+
 		}
 
-		src = addr + (offs & ~PAGE_MASK);
-		count = min_t(size_t, end-offs, PAGE_SIZE-(offs & ~PAGE_MASK));
+		src = *nvmap_addr + (phys & ~PAGE_MASK);
+		count = min_t(size_t, end-start, PAGE_SIZE-(phys&~PAGE_MASK));
 
-		switch (op.op) {
-		case NVMEM_CACHE_OP_WB:
-			smp_dma_clean_range(src, src+count);
-			break;
-		case NVMEM_CACHE_OP_INV:
-			smp_dma_inv_range(src, src+count);
-			break;
-		case NVMEM_CACHE_OP_WB_INV:
-			smp_dma_flush_range(src, src+count);
-			break;
-		}
-		offs += count;
-		nvmap_unmap_pte(addr);
+		if (is_user && is_read)
+			copy_to_user((void*)rw_addr, src, count);
+		else if (is_user)
+			copy_from_user(src, (void*)rw_addr, count);
+		else if (is_read)
+			memcpy((void*)rw_addr, src, count);
+		else
+			memcpy(src, (void*)rw_addr, count);
+
+		rw_addr += count;
+		start += count;
 		if (page) put_page(page);
 	}
 
- out:
-	return err;
+	return (ssize_t)start - orig_start;
 }
 
-static int nvmap_do_rw_handle(NvRmMemHandle hmem, int is_read,
-	unsigned long l_hmem_offs, unsigned long l_user_addr,
-	unsigned long bytes, pgprot_t prot)
+static ssize_t _nvmap_do_rw_handle(struct nvmap_handle *h, int is_read,
+	int is_user, unsigned long h_offs, unsigned long sys_addr,
+	unsigned long h_stride, unsigned long sys_stride,
+	unsigned long elem_size, unsigned long count)
 {
-	void *addr = NULL, *dest;
-	struct page *page = NULL;
-	int ret = 0;
-
-        if (hmem->hPageHandle) {
-		page = NvOsPageGetPage(hmem->hPageHandle, l_hmem_offs);
-		get_page(page);
-		ret = nvmap_map_pte(page_to_pfn(page), prot, &addr);
-	} else {
-		unsigned long phys = hmem->PhysicalAddress + l_hmem_offs;
-		ret = nvmap_map_pte(phys>>PAGE_SHIFT, prot, &addr);
+	ssize_t bytes_copied = 0;
+	void *addr = NULL;
+
+	h = _nvmap_handle_get(h);
+	if (!h) return -EINVAL;
+
+	if (elem_size == h_stride &&
+	    elem_size == sys_stride) {
+		elem_size *= count;
+		h_stride = elem_size;
+		sys_stride = elem_size;
+		count = 1;
 	}
 
-	dest = addr + (l_hmem_offs & ~PAGE_MASK);
-
-	if (is_read && !access_ok(VERIFY_WRITE, (void *)l_user_addr, bytes))
-		ret = -EPERM;
-
-	if (!is_read && !access_ok(VERIFY_READ, (void *)l_user_addr, bytes))
-		ret = -EPERM;
-
-	if (!ret) {
-		if (is_read) copy_to_user((void *)l_user_addr, dest, bytes);
-		else copy_from_user(dest, (void*)l_user_addr, bytes);
+	while (count--) {
+		size_t ret = _nvmap_do_one_rw_handle(h, is_read,
+			is_user, h_offs, sys_addr, elem_size, &addr);
+		if (ret < 0) {
+			if (!bytes_copied) bytes_copied = ret;
+			break;
+		}
+		bytes_copied += ret;
+		if (ret < elem_size) break;
+		sys_addr += elem_size;
+		h_offs += elem_size;
 	}
 
 	if (addr) nvmap_unmap_pte(addr);
-	if (page) put_page(page);
-	return ret;
+	_nvmap_handle_put(h);
+	return bytes_copied;
 }
 
-static int nvmap_rw_handle(struct file *filp, int is_read,
-	void __user* arg)
+static int nvmap_ioctl_rw_handle(struct file *filp,
+	int is_read, void __user* arg)
 {
+	struct nvmem_rw_handle __user *uarg = arg;
 	struct nvmem_rw_handle op;
-	NvRmMemHandle hmem;
-	uintptr_t user_addr, hmem_offs;
+	struct nvmap_handle *h;
+	ssize_t copied;
 	int err = 0;
-	pgprot_t prot;
 
 	err = copy_from_user(&op, arg, sizeof(op));
-	if (err)
-		return err;
+	if (err) return err;
 
 	if (!op.handle || !op.addr || !op.count || !op.elem_size)
 		return -EINVAL;
 
-	hmem = (NvRmMemHandle)op.handle;
+	h = _nvmap_validate_get(op.handle, (filp->f_op == &knvmap_fops));
+	if (!h) return -EINVAL; /* -EPERM? */
 
-	if (op.elem_size == op.hmem_stride &&
-		op.elem_size == op.user_stride) {
-		op.elem_size *= op.count;
-		op.hmem_stride = op.elem_size;
-		op.user_stride = op.elem_size;
-		op.count = 1;
-	}
+	copied = _nvmap_do_rw_handle(h, is_read, 1, op.offset,
+		(unsigned long)op.addr, op.hmem_stride,
+		op.user_stride, op.elem_size, op.count);
 
-	user_addr = (uintptr_t)op.addr;
-	hmem_offs = (uintptr_t)op.offset;
+	if (copied < 0) { err = copied; copied = 0; }
+	else if (copied < (op.count*op.elem_size)) err = -EINTR;
 
-	if (hmem->coherency==NvOsMemAttribute_WriteBack)
-		prot = pgprot_inner_writeback(pgprot_kernel);
-	else if (hmem->coherency==NvOsMemAttribute_WriteCombined)
-		prot = pgprot_writecombine(pgprot_kernel);
-	else
-		prot = pgprot_noncached(pgprot_kernel);
+	__put_user(copied, &uarg->count);
 
-	while (op.count--) {
-		unsigned long remain;
-		unsigned long l_hmem_offs = hmem_offs;
-		unsigned long l_user_addr = user_addr;
+	_nvmap_handle_put(h);
 
-		remain = op.elem_size;
+	return err;
+}
 
-		while (remain && !err) {
-			unsigned long bytes;
+static unsigned int _nvmap_do_get_param(struct nvmap_handle *h,
+	unsigned int param)
+{
+	if (param==NVMEM_HANDLE_PARAM_SIZE)
+		return h->orig_size;
 
-			bytes = min(remain, PAGE_SIZE-(l_hmem_offs&~PAGE_MASK));
-			bytes = min(bytes, PAGE_SIZE-(l_user_addr&~PAGE_MASK));
+	else if (param==NVMEM_HANDLE_PARAM_ALIGNMENT) {
+		unsigned int i = 0;
 
-			err = nvmap_do_rw_handle(hmem, is_read, l_hmem_offs,
-				l_user_addr, bytes, prot);
+		if (!h->alloc) return 0;
 
-			if (!err) {
-				remain -= bytes;
-				l_hmem_offs += bytes;
-				l_user_addr += bytes;
-			}
+		if (h->heap_pgalloc) return PAGE_SIZE;
+		else {
+			while (!(i & h->carveout.base)) i<<=1;
 		}
+		return i;
+	} else if (param==NVMEM_HANDLE_PARAM_BASE) {
+
+		WARN_ON(!h->alloc || !atomic_read(&h->pin));
+
+		if (!h->alloc || !atomic_read(&h->pin)) return ~0ul;
+
+		if (!h->heap_pgalloc)
+			return h->carveout.base;
+
+		if (h->pgalloc.contig)
+			return page_to_phys(h->pgalloc.pages[0]);
 
-		user_addr += op.user_stride;
-		hmem_offs += op.hmem_stride;
+		if (h->pgalloc.area)
+			return h->pgalloc.area->iovm_start;
+
+		return ~0ul;
+	} else if (param==NVMEM_HANDLE_PARAM_HEAP) {
+
+		if (!h->alloc) return 0;
+
+		if (!h->heap_pgalloc) {
+			/* FIXME: hard-coded physical address */
+			if ((h->carveout.base & 0xf0000000ul)==0x40000000ul)
+				return NVMEM_HEAP_CARVEOUT_IRAM;
+			else
+				return NVMEM_HEAP_CARVEOUT_GENERIC;
+		}
+
+		if (!h->pgalloc.contig)
+			return NVMEM_HEAP_IOVMM;
+
+		return NVMEM_HEAP_SYSMEM;
 	}
-	dmb();
 
+	return 0;
+}
+
+static int nvmap_ioctl_get_param(struct file *filp, void __user* arg)
+{
+	struct nvmem_handle_param op;
+	struct nvmap_handle *h;
+	int err;
+
+	err = copy_from_user(&op, arg, sizeof(op));
+	if (err) return err;
+
+	if (op.param < NVMEM_HANDLE_PARAM_SIZE ||
+	    op.param > NVMEM_HANDLE_PARAM_HEAP)
+		return -EINVAL;
+
+	h = _nvmap_validate_get(op.handle, (filp->f_op==&knvmap_fops));
+	if (!h) return -EINVAL;
+
+	op.result = _nvmap_do_get_param(h, op.param);
+	err = copy_to_user(arg, &op, sizeof(op));
+
+	_nvmap_handle_put(h);
 	return err;
 }
 
-static int __init nvmap_pte_init(void)
+static struct miscdevice misc_nvmap_dev = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name = "nvmap",
+	.fops = &nvmap_fops
+};
+
+static struct miscdevice misc_knvmap_dev = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name = "knvmap",
+	.fops = &knvmap_fops
+};
+
+static struct device *__nvmap_heap_parent_dev(void)
+{
+	return misc_nvmap_dev.this_device;
+}
+
+/* creates the sysfs attribute files for a carveout heap; if called
+ * before fs initialization, silently returns.
+ */
+static void _nvmap_create_heap_attrs(struct nvmap_carveout_node *n)
+{
+	if (!_nvmap_heap_parent_dev) return;
+	dev_set_name(&n->dev, "heap-%s", n->carveout.name);
+	n->dev.parent = _nvmap_heap_parent_dev;
+	n->dev.driver = NULL;
+	n->dev.release = NULL;
+	if (device_register(&n->dev)) {
+		pr_err("%s: failed to create heap-%s device\n",
+			__func__, n->carveout.name);
+		return;
+        }
+	if (sysfs_create_group(&n->dev.kobj, &nvmap_heap_defattr_group))
+		pr_err("%s: failed to create attribute group for heap-%s "
+			"device\n", __func__, n->carveout.name);
+}
+
+static int __init nvmap_dev_init(void)
+{
+	struct nvmap_carveout_node *n;
+
+	if (misc_register(&misc_nvmap_dev))
+		pr_err("%s error registering %s\n", __func__,
+			misc_nvmap_dev.name);
+
+	if (misc_register(&misc_knvmap_dev))
+		pr_err("%s error registering %s\n", __func__,
+			misc_knvmap_dev.name);
+
+	/* create sysfs attribute entries for all the heaps which were
+	 * created prior to nvmap_dev_init */
+	down_read(&nvmap_context.list_sem);
+	list_for_each_entry(n, &nvmap_context.heaps, heap_list) {
+		_nvmap_create_heap_attrs(n);
+	}
+	up_read(&nvmap_context.list_sem);
+
+	nvmap_procfs_root = proc_mkdir("nvmap", NULL);
+	if (nvmap_procfs_root) {
+		nvmap_procfs_proc = proc_mkdir("proc", nvmap_procfs_root);
+	}
+	return 0;
+}
+fs_initcall(nvmap_dev_init);
+
+/* initialization of core data structures split out to earlier in the
+ * init sequence, to allow kernel drivers access to nvmap before devfs
+ * is initialized */
+#define NR_CARVEOUTS 2
+static unsigned int nvmap_carveout_cmds = 0;
+static unsigned long nvmap_carveout_cmd_base[NR_CARVEOUTS];
+static unsigned long nvmap_carveout_cmd_size[NR_CARVEOUTS];
+
+static int __init nvmap_core_init(void)
 {
 	u32 base = NVMAP_BASE;
 	pgd_t *pgd;
 	pmd_t *pmd;
 	pte_t *pte;
-	int i = 0;
+	unsigned int i;
+
+	init_rwsem(&nvmap_context.list_sem);
+	nvmap_context.init_data.handle_refs = RB_ROOT;
+	atomic_set(&nvmap_context.init_data.iovm_commit, 0);
+	/* no IOVMM allocations for kernel-created handles */
+	spin_lock_init(&nvmap_context.init_data.ref_lock);
+	nvmap_context.init_data.su = true;
+	nvmap_context.init_data.iovm_limit = 0;
+	INIT_LIST_HEAD(&nvmap_context.heaps);
+
+#ifdef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM
+	for (i=0; i<ARRAY_SIZE(nvmap_mru_cutoff); i++)
+		INIT_LIST_HEAD(&nvmap_mru_vma_lists[i]);
+#endif
+
+	i = 0;
 	do {
 		pgd = pgd_offset(&init_mm, base);
 		pmd = pmd_alloc(&init_mm, pgd, base);
@@ -638,7 +2660,641 @@ static int __init nvmap_pte_init(void)
 		base += (1<<PGDIR_SHIFT);
 	} while (base < NVMAP_END);
 
+	for (i=0; i<nvmap_carveout_cmds; i++) {
+		char tmp[16];
+		snprintf(tmp, sizeof(tmp), "generic-%u", i);
+		nvmap_add_carveout_heap(nvmap_carveout_cmd_base[i],
+			nvmap_carveout_cmd_size[i], tmp, 0x1);
+	}
+
 	return 0;
 }
-core_initcall(nvmap_pte_init);
+core_initcall(nvmap_core_init);
+
+static int __init nvmap_heap_arg(char *options)
+{
+	unsigned long start, size;
+	char *p = options;
+
+	start = -1;
+	size = memparse(p, &p);
+	if (*p == '@')
+		start = memparse(p + 1, &p);
+
+	if (nvmap_carveout_cmds < ARRAY_SIZE(nvmap_carveout_cmd_size)) {
+		nvmap_carveout_cmd_base[nvmap_carveout_cmds] = start;
+		nvmap_carveout_cmd_size[nvmap_carveout_cmds] = size;
+		nvmap_carveout_cmds++;
+	}
+	return 0;
+}
+__setup("nvmem=", nvmap_heap_arg);
+
+static int _nvmap_try_create_preserved(struct nvmap_carveout *co,
+	struct nvmap_handle *h, unsigned long base,
+	size_t size, unsigned int key)
+{
+	unsigned long end = base + size;
+	short idx;
+
+	h->carveout.base = ~0;
+	h->carveout.key = key;
+	h->carveout.co_heap = NULL;
+
+	spin_lock(&co->lock);
+	idx = co->free_index;
+	while (idx != -1) {
+		struct nvmap_mem_block *b = BLOCK(co, idx);
+		unsigned long blk_end = b->base + b->size;
+		if (b->base <= base && blk_end >= end) {
+			nvmap_split_block(co, idx, base, size);
+			h->carveout.block_idx = idx;
+			h->carveout.base = co->blocks[idx].base;
+			h->carveout.co_heap = co;
+			h->alloc = true;
+			break;
+		}
+		idx = b->next_free;
+	}
+	spin_unlock(&co->lock);
 
+	return (h->carveout.co_heap == NULL) ? -ENXIO : 0;
+}
+
+static void _nvmap_create_nvos_preserved(struct nvmap_carveout *co)
+{
+	unsigned int i, key;
+	NvBootArgsPreservedMemHandle mem;
+	static int was_created[NvBootArgKey_PreservedMemHandle_Num -
+		NvBootArgKey_PreservedMemHandle_0] = { 0 };
+
+	for (i=0, key=NvBootArgKey_PreservedMemHandle_0;
+	     i<ARRAY_SIZE(was_created); i++, key++) {
+		struct nvmap_handle *h;
+
+		if (was_created[i]) continue;
+
+		if (NvOsBootArgGet(key, &mem, sizeof(mem))!=NvSuccess) continue;
+		if (!mem.Address || !mem.Size) continue;
+
+		h = _nvmap_handle_create(NULL, mem.Size);
+		if (!h) continue;
+
+		if (!_nvmap_try_create_preserved(co, h, mem.Address,
+		    mem.Size, key))
+			was_created[i] = 1;
+		else
+			_nvmap_handle_put(h);
+	}
+}
+
+int nvmap_add_carveout_heap(unsigned long base, size_t size,
+	const char *name, unsigned int bitmask)
+{
+	struct nvmap_carveout_node *n;
+	struct nvmap_carveout_node *l;
+
+
+	n = kzalloc(sizeof(*n), GFP_KERNEL);
+	if (!n) return -ENOMEM;
+
+	BUG_ON(bitmask & ~NVMEM_HEAP_CARVEOUT_MASK);
+	n->heap_bit = bitmask;
+
+	if (_nvmap_init_carveout(&n->carveout, name, base, size)) {
+		kfree(n);
+		return -ENOMEM;
+	}
+
+	down_write(&nvmap_context.list_sem);
+
+	/* called inside the list_sem lock to ensure that the was_created
+	 * array is protected against simultaneous access */
+	_nvmap_create_nvos_preserved(&n->carveout);
+	_nvmap_create_heap_attrs(n);
+
+	list_for_each_entry(l, &nvmap_context.heaps, heap_list) {
+		if (n->heap_bit > l->heap_bit) {
+			list_add_tail(&n->heap_list, &l->heap_list);
+			up_write(&nvmap_context.list_sem);
+			return 0;
+		}
+	}
+	list_add_tail(&n->heap_list, &nvmap_context.heaps);
+	up_write(&nvmap_context.list_sem);
+	return 0;
+}
+
+int nvmap_create_preserved_handle(unsigned long base, size_t size,
+	unsigned int key)
+{
+	struct nvmap_carveout_node *i;
+	struct nvmap_handle *h;
+
+	h = _nvmap_handle_create(NULL, size);
+	if (!h) return -ENOMEM;
+
+	down_read(&nvmap_context.list_sem);
+	list_for_each_entry(i, &nvmap_context.heaps, heap_list) {
+		struct nvmap_carveout *co = &i->carveout;
+		if (!_nvmap_try_create_preserved(co, h, base, size, key))
+			break;
+	}
+	up_read(&nvmap_context.list_sem);
+
+	/* the base may not be correct if block splitting fails */
+	if (!h->carveout.co_heap || h->carveout.base != base) {
+		_nvmap_handle_put(h);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+/* attempts to create a new carveout heap with a new usage bitmask by
+ * taking an allocation from a previous carveout with a different bitmask */
+static int nvmap_split_carveout_heap(struct nvmap_carveout *co, size_t size,
+	const char *name, unsigned int new_bitmask)
+{
+	struct nvmap_carveout_node *i, *n;
+	int idx = -1;
+	unsigned int blkbase, blksize;
+
+
+	n = kzalloc(sizeof(*n), GFP_KERNEL);
+	if (!n) return -ENOMEM;
+	n->heap_bit = new_bitmask;
+
+	/* align split carveouts to 1M */
+	idx = nvmap_carveout_alloc(co, SZ_1M, size);
+	if (idx != -1) {
+		/* take the spin lock to avoid race conditions with
+		 * intervening allocations triggering grow_block operations */
+		spin_lock(&co->lock);
+		blkbase = co->blocks[idx].base;
+		blksize = co->blocks[idx].size;
+		spin_unlock(&co->lock);
+
+		if (_nvmap_init_carveout(&n->carveout,name, blkbase, blksize)) {
+			nvmap_carveout_free(&i->carveout, idx);
+			idx = -1;
+		} else {
+			spin_lock(&co->lock);
+			if (co->blocks[idx].prev) {
+				co->blocks[co->blocks[idx].prev].next =
+					co->blocks[idx].next;
+			}
+			if (co->blocks[idx].next) {
+				co->blocks[co->blocks[idx].next].prev =
+					co->blocks[idx].prev;
+			}
+			if (co->block_index==idx)
+				co->block_index = co->blocks[idx].next;
+			co->blocks[idx].next_free = -1;
+			co->blocks[idx].prev_free = -1;
+			co->blocks[idx].next = co->spare_index;
+			if (co->spare_index!=-1)
+				co->blocks[co->spare_index].prev = idx;
+			co->spare_index = idx;
+			spin_unlock(&co->lock);
+		}
+	}
+
+	if (idx==-1) {
+		kfree(n);
+		return -ENOMEM;
+	}
+
+	down_write(&nvmap_context.list_sem);
+	_nvmap_create_heap_attrs(n);
+	list_for_each_entry(i, &nvmap_context.heaps, heap_list) {
+		if (n->heap_bit > i->heap_bit) {
+			list_add_tail(&n->heap_list, &i->heap_list);
+			up_write(&nvmap_context.list_sem);
+			return 0;
+		}
+	}
+	list_add_tail(&n->heap_list, &nvmap_context.heaps);
+	up_write(&nvmap_context.list_sem);
+	return 0;
+}
+
+/* NvRmMemMgr APIs implemented on top of nvmap */
+
+#include <linux/freezer.h>
+
+NvU32 NvRmMemGetAddress(NvRmMemHandle hMem, NvU32 Offset)
+{
+	struct nvmap_handle *h = (struct nvmap_handle *)hMem;
+	unsigned long addr;
+
+	if (unlikely(!atomic_read(&h->pin) || !h->alloc)) return ~0ul;
+	if (unlikely(Offset >= h->orig_size)) return ~0UL;
+
+	if (h->heap_pgalloc && h->pgalloc.contig)
+		addr = page_to_phys(h->pgalloc.pages[0]);
+	else if (h->heap_pgalloc) {
+		BUG_ON(!h->pgalloc.area);
+		BUG_ON(!atomic_read(&h->pin));
+		addr = h->pgalloc.area->iovm_start;
+	} else
+		addr = h->carveout.base;
+
+	return (NvU32)addr+Offset;
+
+}
+
+void NvRmMemPinMult(NvRmMemHandle *hMems, NvU32 *addrs, NvU32 Count)
+{
+	struct nvmap_handle **h = (struct nvmap_handle **)hMems;
+	unsigned int i;
+	int ret;
+
+	do {
+		ret = _nvmap_handle_pin_fast(Count, h);
+		if (ret && !try_to_freeze()) {
+			pr_err("%s: failed to pin handles\n", __func__);
+			dump_stack();
+		}
+	} while (ret);
+
+	for (i=0; i<Count; i++) {
+		addrs[i] = NvRmMemGetAddress(hMems[i], 0);
+		BUG_ON(addrs[i]==~0ul);
+	}
+}
+
+void NvRmMemUnpinMult(NvRmMemHandle *hMems, NvU32 Count)
+{
+	int do_wake = 0;
+	unsigned int i;
+
+	for (i=0; i<Count; i++) {
+		struct nvmap_handle *h = (struct nvmap_handle *)hMems[i];
+		BUG_ON(atomic_read(&h->pin)==0);
+		do_wake |= _nvmap_handle_unpin(h);
+	}
+
+	if (do_wake) wake_up(&nvmap_pin_wait);
+}
+
+NvU32 NvRmMemPin(NvRmMemHandle hMem)
+{
+	NvU32 addr;
+	NvRmMemPinMult(&hMem, &addr, 1);
+	return addr;
+}
+
+void NvRmMemUnpin(NvRmMemHandle hMem)
+{
+	NvRmMemUnpinMult(&hMem, 1);
+}
+
+void NvRmMemHandleFree(NvRmMemHandle hMem)
+{
+	_nvmap_do_free(&nvmap_context.init_data, (unsigned long)hMem);
+}
+
+NvError NvRmMemMap(NvRmMemHandle hMem, NvU32 Offset, NvU32 Size,
+	NvU32 Flags, void **pVirtAddr)
+{
+	struct nvmap_handle *h = (struct nvmap_handle *)hMem;
+	pgprot_t prot = _nvmap_flag_to_pgprot(h->flags, pgprot_kernel);
+
+	BUG_ON(!h->alloc);
+
+	if (Offset+Size > h->size)
+		return NvError_BadParameter;
+
+	if (!h->kern_map && h->heap_pgalloc) {
+		BUG_ON(h->size & ~PAGE_MASK);
+		h->kern_map = vm_map_ram(h->pgalloc.pages,
+			h->size>>PAGE_SHIFT, -1, prot);
+	} else if (!h->kern_map) {
+		unsigned int size;
+		unsigned long addr;
+
+		addr = h->carveout.base;
+		size = h->size + (addr & ~PAGE_MASK);
+		addr &= PAGE_MASK;
+		size = (size + PAGE_SIZE - 1) & PAGE_MASK;
+
+		h->kern_map = ioremap_wc(addr, size);
+		if (h->kern_map) {
+			addr = h->carveout.base - addr;
+			h->kern_map += addr;
+		}
+	}
+
+	if (h->kern_map) {
+		*pVirtAddr = (h->kern_map + Offset);
+		return NvSuccess;
+	}
+
+	return NvError_InsufficientMemory;
+}
+
+void NvRmMemUnmap(NvRmMemHandle hMem, void *pVirtAddr, NvU32 Size)
+{
+	return;
+}
+
+NvU32 NvRmMemGetId(NvRmMemHandle hMem)
+{
+	struct nvmap_handle *h = (struct nvmap_handle *)hMem;
+	if (!h->owner) h->global = true;
+	return (NvU32)h;
+}
+
+NvError NvRmMemHandleFromId(NvU32 id, NvRmMemHandle *hMem)
+{
+	struct nvmap_handle_ref *r;
+
+	int err = _nvmap_do_create(&nvmap_context.init_data,
+		NVMEM_IOC_FROM_ID, id, true, &r);
+
+	if (err || !r) return NvError_NotInitialized;
+
+	*hMem = (NvRmMemHandle)r->h;
+	return NvSuccess;
+}
+
+NvError NvRmMemHandleClaimPreservedHandle(NvRmDeviceHandle hRm,
+	NvU32 Key, NvRmMemHandle *hMem)
+{
+	struct nvmap_handle_ref *r;
+
+	int err = _nvmap_do_create(&nvmap_context.init_data,
+		NVMEM_IOC_CLAIM, (unsigned long)Key, true, &r);
+
+	if (err || !r) return NvError_NotInitialized;
+
+	*hMem = (NvRmMemHandle)r->h;
+	return NvSuccess;
+}
+
+NvError NvRmMemHandleCreate(NvRmDeviceHandle hRm,
+	NvRmMemHandle *hMem, NvU32 Size)
+{
+	struct nvmap_handle_ref *r;
+	int err = _nvmap_do_create(&nvmap_context.init_data,
+		NVMEM_IOC_CREATE, (unsigned long)Size, true, &r);
+
+	if (err || !r) return NvError_InsufficientMemory;
+	*hMem = (NvRmMemHandle)r->h;
+	return NvSuccess;
+}
+
+NvError NvRmMemAlloc(NvRmMemHandle hMem, const NvRmHeap *Heaps,
+	NvU32 NumHeaps, NvU32 Alignment, NvOsMemAttribute Coherency)
+{
+	unsigned int heap_mask = 0;
+	unsigned int flags = pgprot_kernel;
+	int err;
+
+	BUG_ON(Alignment & (Alignment-1));
+
+	if (Coherency == NvOsMemAttribute_WriteBack)
+		flags = NVMEM_HANDLE_INNER_CACHEABLE;
+	else
+		flags = NVMEM_HANDLE_WRITE_COMBINE;
+
+	if (!NumHeaps || !Heaps)
+		heap_mask = (NVMEM_HEAP_SYSMEM | NVMEM_HEAP_CARVEOUT_GENERIC);
+	else {
+		unsigned int i;
+		for (i=0; i<NumHeaps; i++) {
+			switch (Heaps[i]) {
+			case NvRmHeap_GART:
+				heap_mask |= NVMEM_HEAP_IOVMM;
+				break;
+			case NvRmHeap_External:
+				heap_mask |= NVMEM_HEAP_SYSMEM;
+				break;
+			case NvRmHeap_ExternalCarveOut:
+				heap_mask |= NVMEM_HEAP_CARVEOUT_GENERIC;
+				break;
+			case NvRmHeap_IRam:
+				heap_mask |= NVMEM_HEAP_CARVEOUT_IRAM;
+				break;
+			default:
+				break;
+			}
+		}
+	}
+	if (!heap_mask) return NvError_InsufficientMemory;
+
+	err = _nvmap_do_alloc(&nvmap_context.init_data, (unsigned long)hMem,
+		heap_mask, (size_t)Alignment, flags, false, true);
+
+	if (err) return NvError_InsufficientMemory;
+	return NvSuccess;
+}
+
+void NvRmMemReadStrided(NvRmMemHandle hMem, NvU32 Offset, NvU32 SrcStride,
+	void *pDst, NvU32 DstStride, NvU32 ElementSize, NvU32 Count)
+{
+	ssize_t bytes = 0;
+
+	bytes = _nvmap_do_rw_handle((struct nvmap_handle *)hMem, true,
+		false, Offset, (unsigned long)pDst, SrcStride,
+		DstStride, ElementSize, Count);
+
+	BUG_ON(bytes != (ssize_t)(Count*ElementSize));
+}
+
+void NvRmMemWriteStrided(NvRmMemHandle hMem, NvU32 Offset, NvU32 DstStride,
+	const void *pSrc, NvU32 SrcStride, NvU32 ElementSize, NvU32 Count)
+{
+	ssize_t bytes = 0;
+
+	bytes = _nvmap_do_rw_handle((struct nvmap_handle *)hMem, false,
+		false, Offset, (unsigned long)pSrc, DstStride,
+		SrcStride, ElementSize, Count);
+
+	BUG_ON(bytes != (ssize_t)(Count*ElementSize));
+}
+
+NvU32 NvRmMemGetSize(NvRmMemHandle hMem)
+{
+	struct nvmap_handle *h = (struct nvmap_handle *)hMem;
+	return h->orig_size;
+}
+
+NvRmHeap NvRmMemGetHeapType(NvRmMemHandle hMem, NvU32 *BaseAddr)
+{
+	struct nvmap_handle *h = (struct nvmap_handle *)hMem;
+	NvRmHeap heap;
+
+	if (!h->alloc) {
+		*BaseAddr = ~0ul;
+		return (NvRmHeap)0;
+	}
+
+	if (h->heap_pgalloc && !h->pgalloc.contig)
+		heap = NvRmHeap_GART;
+	else if (h->heap_pgalloc)
+		heap = NvRmHeap_External;
+	else if ((h->carveout.base & 0xf0000000ul) == 0x40000000ul)
+		heap = NvRmHeap_IRam;
+	else
+		heap = NvRmHeap_ExternalCarveOut;
+
+	if (h->heap_pgalloc && h->pgalloc.contig)
+		*BaseAddr = (NvU32)page_to_phys(h->pgalloc.pages[0]);
+	else if (h->heap_pgalloc && atomic_read(&h->pin))
+		*BaseAddr = h->pgalloc.area->iovm_start;
+	else if (h->heap_pgalloc)
+		*BaseAddr = ~0ul;
+	else
+		*BaseAddr = (NvU32)h->carveout.base;
+
+	return heap;
+}
+
+void NvRmMemCacheMaint(NvRmMemHandle hMem, void *pMapping,
+	NvU32 Size, NvBool Writeback, NvBool Inv)
+{
+	struct nvmap_handle *h = (struct nvmap_handle *)hMem;
+	unsigned long start;
+	unsigned int op;
+
+	if (!h->kern_map || h->flags==NVMEM_HANDLE_UNCACHEABLE ||
+		h->flags==NVMEM_HANDLE_WRITE_COMBINE) return;
+
+	if (!Writeback && !Inv) return;
+
+	if (Writeback && Inv) op = NVMEM_CACHE_OP_WB_INV;
+	else if (Writeback) op = NVMEM_CACHE_OP_WB;
+	else op = NVMEM_CACHE_OP_INV;
+
+	start = (unsigned long)pMapping - (unsigned long)h->kern_map;
+
+	_nvmap_do_cache_maint(h, start, start+Size, op, true);
+	return;
+}
+
+NvU32 NvRmMemGetAlignment(NvRmMemHandle hMem)
+{
+	struct nvmap_handle *h = (struct nvmap_handle *)hMem;
+	return _nvmap_do_get_param(h, NVMEM_HANDLE_PARAM_ALIGNMENT);
+}
+
+NvError NvRmMemGetStat(NvRmMemStat Stat, NvS32 *Result)
+{
+	unsigned long total_co = 0;
+	unsigned long free_co = 0;
+	unsigned long max_free = 0;
+	struct nvmap_carveout_node *n;
+
+	down_read(&nvmap_context.list_sem);
+	list_for_each_entry(n, &nvmap_context.heaps, heap_list) {
+
+		if (!(n->heap_bit & NVMEM_HEAP_CARVEOUT_GENERIC)) continue;
+			total_co += _nvmap_carveout_blockstat(&n->carveout,
+					CARVEOUT_STAT_TOTAL_SIZE);
+			free_co += _nvmap_carveout_blockstat(&n->carveout,
+					CARVEOUT_STAT_FREE_SIZE);
+			max_free = max(max_free,
+				_nvmap_carveout_blockstat(&n->carveout,
+				    CARVEOUT_STAT_LARGEST_FREE));
+	}
+	up_read(&nvmap_context.list_sem);
+
+	if (Stat==NvRmMemStat_TotalCarveout) {
+		*Result = (NvU32)total_co;
+		return NvSuccess;
+	} else if (Stat==NvRmMemStat_UsedCarveout) {
+		*Result = (NvU32)total_co - (NvU32)free_co;
+		return NvSuccess;
+	} else if (Stat==NvRmMemStat_LargestFreeCarveoutBlock) {
+		*Result = (NvU32)max_free;
+		return NvSuccess;
+	}
+
+	return NvError_BadParameter;
+}
+
+NvU8 NvRmMemRd08(NvRmMemHandle hMem, NvU32 Offset)
+{
+	NvU8 val;
+	NvRmMemRead(hMem, Offset, &val, sizeof(val));
+	return val;
+}
+
+NvU16 NvRmMemRd16(NvRmMemHandle hMem, NvU32 Offset)
+{
+	NvU16 val;
+	NvRmMemRead(hMem, Offset, &val, sizeof(val));
+	return val;
+}
+
+NvU32 NvRmMemRd32(NvRmMemHandle hMem, NvU32 Offset)
+{
+	NvU32 val;
+	NvRmMemRead(hMem, Offset, &val, sizeof(val));
+	return val;
+}
+
+void NvRmMemWr08(NvRmMemHandle hMem, NvU32 Offset, NvU8 Data)
+{
+	NvRmMemWrite(hMem, Offset, &Data, sizeof(Data));
+}
+
+void NvRmMemWr16(NvRmMemHandle hMem, NvU32 Offset, NvU16 Data)
+{
+	NvRmMemWrite(hMem, Offset, &Data, sizeof(Data));
+}
+
+void NvRmMemWr32(NvRmMemHandle hMem, NvU32 Offset, NvU32 Data)
+{
+	NvRmMemWrite(hMem, Offset, &Data, sizeof(Data));
+}
+
+void NvRmMemRead(NvRmMemHandle hMem, NvU32 Offset, void *pDst, NvU32 Size)
+{
+	NvRmMemReadStrided(hMem, Offset, Size, pDst, Size, Size, 1);
+}
+
+void NvRmMemWrite(NvRmMemHandle hMem, NvU32 Offset,
+	const void *pSrc, NvU32 Size)
+{
+	NvRmMemWriteStrided(hMem, Offset, Size, pSrc, Size, Size, 1);
+}
+
+void NvRmMemMove(NvRmMemHandle dstHMem, NvU32 dstOffset,
+	NvRmMemHandle srcHMem, NvU32 srcOffset, NvU32 Size)
+{
+	while (Size--) {
+		NvU8 tmp = NvRmMemRd08(srcHMem, srcOffset);
+		NvRmMemWr08(dstHMem, dstOffset, tmp);
+		dstOffset++;
+		srcOffset++;
+	}
+}
+
+NvU32 NvRmMemGetCacheLineSize(void)
+{
+	return 32;
+}
+
+void *NvRmHostAlloc(size_t size)
+{
+	return NvOsAlloc(size);
+}
+
+void NvRmHostFree(void *ptr)
+{
+	NvOsFree(ptr);
+}
+
+NvError NvRmMemMapIntoCallerPtr(NvRmMemHandle hMem, void *pCallerPtr,
+	NvU32 Offset, NvU32 Size)
+{
+	return NvError_NotSupported;
+}
+
+NvError NvRmMemHandlePreserveHandle(NvRmMemHandle hMem, NvU32 *pKey)
+{
+	return NvError_NotSupported;
+}