ARM: errata: 727915: Background Clean & Invalidate by Way operation can cause data corruption.

PL310 implements the Clean & Invalidate by Way L2 cache maintenance operation
(offset 0x7FC). This operation runs in background so that PL310 can handle
normal accesses while it is in progress. Under very rare circumstances,
due to this erratum, write data can be lost when PL310 treats a cacheable
write transaction during a Clean & Invalidate by Way operation.

This fix is to replace the background Clean & Invalby Way operation
by a software loop on all sets/ways. This works for r2p0 and r3p0 as well.

Change-Id: I45e841d8049a18f2dd36ce13e8ef15322f14c5d5
Reviewed-on: http://git-master/r/29690
Reviewed-by: Varun Colbert <vcolbert@nvidia.com>
Tested-by: Varun Colbert <vcolbert@nvidia.com>

3 files changed, 57 insertions, 2 deletions

diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 79ad5f1efcaa..75226da44789 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -1148,6 +1148,17 @@ config ARM_ERRATA_752520
 	  and MMU are enabled, with the TLB descriptors marked as L1 cacheable,
 	  so that Page Table Walks are performed as cache linefills.
 
+config PL310_ERRATA_727915
+	bool "Background Clean & Invalidate by Way operation can cause data corruption"
+	depends on CACHE_L2X0
+	help
+	  PL310 implements the Clean & Invalidate by Way L2 cache maintenance
+	  operation (offset 0x7FC). This operation runs in background so that
+	  PL310 can handle normal accesses while it is in progress. Under very
+	  rare circumstances, due to this erratum, write data can be lost when
+	  PL310 treats a cacheable write transaction during a Clean &
+	  Invalidate by Way operation.
+
 endmenu
 
 source "arch/arm/common/Kconfig"
diff --git a/arch/arm/include/asm/hardware/cache-l2x0.h b/arch/arm/include/asm/hardware/cache-l2x0.h
index d62847df3df5..112b02d06ead 100644
--- a/arch/arm/include/asm/hardware/cache-l2x0.h
+++ b/arch/arm/include/asm/hardware/cache-l2x0.h
@@ -24,9 +24,14 @@
 #define   L2X0_CACHE_ID_PART_MASK	(0xf << 6)
 #define   L2X0_CACHE_ID_PART_L210	(1 << 6)
 #define   L2X0_CACHE_ID_PART_L310	(3 << 6)
+#define   L2X0_CACHE_ID_RTL_RELEASE_MASK	(0x3F << 0)
+#define   L2X0_CACHE_ID_RTL_RELEASE_R2P0	(4<< 0)
+#define   L2X0_CACHE_ID_RTL_RELEASE_R3P0	(5<< 0)
 #define L2X0_CACHE_TYPE			0x004
 #define L2X0_CTRL			0x100
 #define L2X0_AUX_CTRL			0x104
+#define   L2X0_AUX_CTRL_WAY_SIZE_SHIFT	17
+#define   L2X0_AUX_CTRL_WAY_SIZE_MASK	(0x7 << 17)
 #define L2X0_TAG_LATENCY_CTRL		0x108
 #define L2X0_DATA_LATENCY_CTRL		0x10C
 #define L2X0_EVENT_CNT_CTRL		0x200
@@ -46,6 +51,8 @@
 #define L2X0_CLEAN_WAY			0x7BC
 #define L2X0_CLEAN_INV_LINE_PA		0x7F0
 #define L2X0_CLEAN_INV_LINE_IDX		0x7F8
+#define   L2X0_CLEAN_INV_LINE_IDX_WAY_SHIFT	28
+#define   L2X0_CLEAN_INV_LINE_IDX_INDEX_SHIFT	5
 #define L2X0_CLEAN_INV_WAY		0x7FC
 #define L2X0_LOCKDOWN_WAY_D		0x900
 #define L2X0_LOCKDOWN_WAY_I		0x904
diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c
index 9027bd9a2a7c..02abaadabbc5 100644
--- a/arch/arm/mm/cache-l2x0.c
+++ b/arch/arm/mm/cache-l2x0.c
@@ -32,6 +32,9 @@ void callGenericSMC(u32 param0, u32 param1, u32 param2);
 
 static void __iomem *l2x0_base;
 static uint32_t l2x0_way_mask;	/* Bitmask of active ways */
+static uint32_t l2x0_size;
+static uint32_t l2x0_num_ways;
+static uint32_t l2x0_num_idxes;
 bool l2x0_disabled;
 
 static inline void cache_wait_always(void __iomem *reg, unsigned long mask)
@@ -151,11 +154,34 @@ static inline void l2x0_inv_all(void)
 static inline void l2x0_flush_all(void)
 {
 	unsigned long flags;
+#ifdef CONFIG_PL310_ERRATA_727915
+	int way;
+	int idx;
+	unsigned long way_idx;
+	unsigned long rtl_release;
+#endif
 
 	/* flush all ways */
 	_l2x0_lock(&l2x0_lock, flags);
-	writel(0xff, l2x0_base + L2X0_CLEAN_INV_WAY);
-	cache_wait_always(l2x0_base + L2X0_CLEAN_INV_WAY, 0xff);
+#ifdef CONFIG_PL310_ERRATA_727915
+	rtl_release = readl_relaxed(l2x0_base + L2X0_CACHE_ID);
+	rtl_release &= L2X0_CACHE_ID_RTL_RELEASE_MASK;
+	if ( (rtl_release == L2X0_CACHE_ID_RTL_RELEASE_R2P0) ||
+		(rtl_release == L2X0_CACHE_ID_RTL_RELEASE_R3P0) ) {
+		for (idx = 0; idx < l2x0_num_idxes; idx++) {
+			for (way = 0; way < l2x0_num_ways; way++) {
+				way_idx = (way << L2X0_CLEAN_INV_LINE_IDX_WAY_SHIFT) |
+					(idx << L2X0_CLEAN_INV_LINE_IDX_INDEX_SHIFT);
+				cache_wait(l2x0_base + L2X0_CLEAN_INV_LINE_IDX, 1);
+				writel_relaxed(way_idx, l2x0_base + L2X0_CLEAN_INV_LINE_IDX);
+			}
+		}
+	} else
+#endif
+	{
+		writel(0xff, l2x0_base + L2X0_CLEAN_INV_WAY);
+		cache_wait_always(l2x0_base + L2X0_CLEAN_INV_WAY, 0xff);
+	}
 	cache_sync();
 	_l2x0_unlock(&l2x0_lock, flags);
 }
@@ -315,6 +341,7 @@ static void l2x0_enable(__u32 aux_val, __u32 aux_mask)
 {
 	__u32 aux;
 	__u32 cache_id;
+	__u32 way_size = 0;
 	int ways;
 	const char *type;
 #if defined(CONFIG_SMP) && defined(CONFIG_TRUSTED_FOUNDATIONS)
@@ -355,6 +382,16 @@ static void l2x0_enable(__u32 aux_val, __u32 aux_mask)
 	l2x0_way_mask = (1 << ways) - 1;
 
 	/*
+	 * L2 cache Size =  Way size * Number of ways
+	 */
+	way_size = (aux & L2X0_AUX_CTRL_WAY_SIZE_MASK) >> 17;
+	way_size = 1 << (way_size + 3);
+	l2x0_size = ways * way_size * SZ_1K;
+
+	l2x0_num_ways = ways;
+	l2x0_num_idxes = l2x0_size / (ways * CACHE_LINE_SIZE);
+
+	/*
 	 * Check if l2x0 controller is already enabled.
 	 * If you are booting from non-secure mode
 	 * accessing the below registers will fault.