imx8m: Add DDR4 DVFS support

Add DDR4 DVFS support for i.MX8M. Currently, only tested on i.MX8MM.

Signed-off-by: Bai Ping <ping.bai@nxp.com>

4 files changed, 377 insertions, 9 deletions

diff --git a/plat/imx/common/imx8m/ddr4_dvfs.c b/plat/imx/common/imx8m/ddr4_dvfs.c
new file mode 100644
index 00000000..71237f0a
--- /dev/null
+++ b/plat/imx/common/imx8m/ddr4_dvfs.c
@@ -0,0 +1,355 @@
+/*
+ * Copyright 2018 NXP
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <debug.h>
+#include <ddrc.h>
+#include <dram.h>
+#include <mmio.h>
+
+unsigned fsp_init_reg[3][4] = {
+	{ DDRC_INIT3(0), DDRC_INIT4(0), DDRC_INIT6(0), DDRC_INIT7(0) },
+	{ DDRC_FREQ1_INIT3(0), DDRC_FREQ1_INIT4(0), DDRC_FREQ1_INIT6(0), DDRC_FREQ1_INIT7(0) },
+	{ DDRC_FREQ2_INIT3(0), DDRC_FREQ2_INIT4(0), DDRC_FREQ2_INIT6(0), DDRC_FREQ2_INIT7(0) },
+};
+
+static unsigned int mr_value[7];
+
+void ddr4_mr_write(unsigned int mr, unsigned int data, unsigned int mr_type, unsigned int rank)
+{
+	unsigned int tmp, mr_mirror, data_mirror;
+	/*
+	 * 1. Poll MRSTAT.mr_wr_busy until it is 0 to make sure that there is
+	 * no outstanding MR transAction. No
+	 */
+	do {
+		tmp = mmio_read_32(DDRC_MRSTAT(0));
+	} while (tmp & 0x1);
+
+	/*
+	 * 2. Write the MRCTRL0.mr_type, MRCTRL0.mr_addr, MRCTRL0.mr_rank
+	 * and (for MRWs) MRCTRL1.mr_data to define the MR transaction.
+	 */
+	tmp = mmio_read_32(DDRC_DIMMCTL(0));
+	if ((tmp & 0x2) && (rank == 0x2)) {
+		mr_mirror = (mr & 0x4) | ((mr & 0x1) << 1) | ((mr & 0x2) >> 1); /* BA0, BA1 swap */
+		data_mirror = (data & 0x1607) | ((data & 0x8) << 1) | ((data & 0x10) >> 1) |
+				((data & 0x20) << 1) | ((data & 0x40) >> 1) | ((data & 0x80) << 1) |
+				 ((data & 0x100) >> 1) | ((data & 0x800) << 2) | ((data & 0x2000) >> 2) ;
+	} else {
+		mr_mirror = mr;
+		data_mirror = data;
+	}
+
+	mmio_write_32(DDRC_MRCTRL0(0), mr_type | (mr_mirror << 12) | (rank << 4) );
+	mmio_write_32(DDRC_MRCTRL1(0), data_mirror );
+
+	/*
+	 * 3. In a separate APB transaction, write the MRCTRL0.mr_wr to 1. This bit is
+	 * self-clearing, and triggers the MR transaction. The uMCTL2 then asserts the
+	 * MRSTAT.mr_wr_busy while it performs the MR transaction to SDRAM, and no further
+	 * accesses can be initiated until it is deasserted.
+	 */
+	mmio_setbits_32(DDRC_MRCTRL0(0), (1 << 31));
+	do {
+		tmp = mmio_read_32(DDRC_MRSTAT(0));
+	} while (tmp);
+}
+
+void dram_cfg_all_mr(unsigned int num_rank, unsigned int pstate)
+{
+	/*
+	 * 15. Perform MRS commands as required to re-program timing registers
+	 * in the SDRAM for the new frequency (in particular, CL, CWL and WR
+	 * may need to be changed).
+	 */
+
+	for (int i = 1; i <= num_rank; i++) {
+		for (int j = 0; j < 7; j++)
+			ddr4_mr_write(j, mr_value[j], 0, i);
+	}
+}
+
+void sw_pstate(unsigned int pstate)
+{
+	volatile unsigned int tmp;
+	unsigned int i;
+
+	mmio_write_32(DDRC_SWCTL(0), 0x0000);
+	/* 12. Change the clock frequency to the desired value. */
+
+	/*
+	 * 13. Update any registers which may be required to change for the new frequency. This includes
+	 * quasidynamic and dynamic registers. This includes both uMCTL2 registers and PHY registers.
+	 */
+	mmio_write_32(DDRC_MSTR2(0), pstate); /* UMCTL2_REGS_FREQ1 */
+	mmio_setbits_32(DDRC_MSTR(0), (0x1 << 29));
+
+	/*
+	 * dvfs.18. Toggle RFSHCTL3.refresh_update_level to allow the new refresh-related
+	 * register values to propagate to the refresh logic.
+	 */
+	tmp = mmio_read_32(DDRC_RFSHCTL3(0));
+	if ((tmp & 0x2) == 0x2)
+		mmio_write_32(DDRC_RFSHCTL3(0), tmp & 0xFFFFFFFD);
+	else
+		mmio_write_32(DDRC_RFSHCTL3(0), tmp | 0x2);
+	/*
+	 *dvfs.19. If required, trigger the initialization in the PHY. If using the gen2
+	 * multiPHY, PLL initialization should be triggered at this point. See the PHY
+	 * databook for details about the frequency change procedure.
+	 */
+	/* before write Dynamic reg, sw_done should be 0 */
+	mmio_write_32(DDRC_DFIMISC(0), 0x00000000 | (pstate << 8));//pstate1
+	mmio_write_32(DDRC_DFIMISC(0), 0x00000020 | (pstate << 8));
+
+	/* wait DFISTAT.dfi_init_complete to 0 */
+	do {
+		tmp = 0x1 & mmio_read_32(DDRC_DFISTAT(0));
+	} while (tmp);
+
+	/* change the clock to the target frequency */
+	dram_clock_switch(pstate);
+
+	mmio_write_32(DDRC_DFIMISC(0), 0x00000000 | (pstate << 8));
+	/* wait DFISTAT.dfi_init_complete to 1 */
+	do {
+		tmp = 0x1 & mmio_read_32(DDRC_DFISTAT(0));
+	} while (!tmp);
+
+	/*
+	 * When changing frequencies the controller may violate the JEDEC
+	 * requirement that no more than 16 refreshes should be issued within
+	 * 2*tREFI. These extra refreshes are not expected to cause a problem
+	 * in the SDRAM. This issue can be avoided by waiting for at least 2*tREFI
+	 * before exiting self-refresh in step 19.
+	 */
+	for (i = 20; i > 0; i--)
+		printf("waiting for 2*tREF (2*7.8us)\n");
+
+	/* 14. Exit the self-refresh state by setting PWRCTL.selfref_sw = 0. */
+	mmio_clrbits_32(DDRC_PWRCTL(0), (1 << 5));
+	do {
+		tmp  = 0x3f & (mmio_read_32((DDRC_STAT(0))));
+	} while (tmp == 0x23);
+}
+
+void ddr4_dll_change(unsigned int num_rank, unsigned int pstate, unsigned int cur_pstate)
+{
+	volatile unsigned int tmp;
+	enum DLL_STATE { NO_CHANGE=0, ON2OFF=1, OFF2ON=2} dll_sw; /* 0-no change, 1-on2off, 2-off2on.; */
+
+	if (pstate != 0 && cur_pstate == 0)
+		dll_sw = ON2OFF;
+	else if (pstate == 0 && cur_pstate != 0)
+		dll_sw = OFF2ON;
+	else
+		dll_sw = NO_CHANGE;
+
+	/* the the following software programming sequence to switch from DLL-on to DLL-off, or reverse: */
+	mmio_write_32(DDRC_SWCTL(0), 0x0000);
+
+	mmio_write_32(DDRC_PCTRL_0(0), 0x00000000);
+
+	/* 1. Set the DBG1.dis_hif = 1. This prevents further reads/writes being received on the HIF. */
+	mmio_setbits_32(DDRC_DBG1(0), (0x1 << 1));
+
+	/* 2. Set ZQCTL0.dis_auto_zq=1, to disable automatic generation of ZQCS/MPC(ZQ calibration) */
+	mmio_setbits_32(DDRC_FREQ1_ZQCTL0(0), (1 << 31));
+	mmio_setbits_32(DDRC_FREQ2_ZQCTL0(0), (1 << 31));
+	mmio_setbits_32(DDRC_ZQCTL0(0), (1 << 31));
+
+	/* 3. Set RFSHCTL3.dis_auto_refresh=1, to disable automatic refreshes */
+	/* mmio_setbits_32(DDRC_RFSHCTL3(0), 0x1); */
+	/* 4. Ensure all commands have been flushed from the uMCTL2 by polling */
+	do {
+		tmp = 0x36000000 & mmio_read_32(DDRC_DBGCAM(0));
+	} while (tmp != 0x36000000);
+
+	/* mmio_write_32(DDRC_PCTRL_0(0), 0x00000000); */
+
+	/* 5. Perform an MRS command (using MRCTRL0 and MRCTRL1 registers) to disable RTT_NOM: */
+	/* a. DDR3: Write 0 to MR1[9], MR1[6] and MR1[2] */
+	/* b. DDR4: Write 0 to MR1[10:8] */
+	for (int i = 1; i <= num_rank; i++) {
+		if (mr_value[1] & 0x700)
+			ddr4_mr_write(1, mr_value[1] & 0xF8FF, 0, i);
+	}
+
+	/* 6. For DDR4 only: Perform an MRS command (using MRCTRL0 and MRCTRL1 registers) to write 0 to */
+	/* MR5[8:6] to disable RTT_PARK */
+	for (int i = i; i < num_rank; i++) {
+		if (mr_value[5] & 0x1C0)
+			ddr4_mr_write(5, mr_value[5] & 0xFE3F, 0, i);
+	}
+
+	if(dll_sw == ON2OFF) {
+	        /* 7. Perform an MRS command (using MRCTRL0 and MRCTRL1 registers) to write 0 to MR2[11:9], to */
+		/* disable RTT_WR (and therefore disable dynamic ODT). This applies for both DDR3 and DDR4. */
+		for (int i = 1; i <= num_rank; i++) {
+			if (mr_value[2] & 0xE00)
+				ddr4_mr_write(2, mr_value[2] & 0xF1FF, 0, i);
+		}
+
+		/* 8. Perform an MRS command (using MRCTRL0 and MRCTRL1 registers) to disable the DLL. The */
+		/* timing of this MRS is automatically handled by the uMCTL2. */
+		/* a. DDR3: Write 1 to MR1[0] */
+		/* b. DDR4: Write 0 to MR1[0] */
+		for (int i = 1; i <= num_rank; i++) {
+			ddr4_mr_write(1, mr_value[1] & 0xFFFE, 0, i);
+		}
+	}
+
+	/* 9. Put the SDRAM into self-refresh mode by setting PWRCTL.selfref_sw = 1, and polling */
+	/* STAT.operating_mode to ensure the DDRC has entered self-refresh. */
+	mmio_setbits_32(DDRC_PWRCTL(0), (1 << 5));
+	/*
+	 * 10. Wait until STAT.operating_mode[1:0]==11 indicating that the DWC_ddr_umctl2
+	 * core is in selfrefresh mode. Ensure transition to self-refresh was due to software
+	 * by checking that
+	 */
+	/* STAT.selfref_type[1:0]=2`b10. */
+	do {
+		tmp  = 0x3f & (mmio_read_32((DDRC_STAT(0))));
+	} while (tmp != 0x23);
+
+	/* FIXME Set RFSHCTL3.dis_auto_refresh=1, to disable automatic refreshes ??? */
+	mmio_setbits_32(DDRC_RFSHCTL3(0), 0x1);
+
+	/* 11. Set the MSTR.dll_off_mode = 1 or 0. */
+	if (dll_sw == ON2OFF)
+		mmio_setbits_32(DDRC_MSTR(0), (1 << 15));
+
+	if(dll_sw == OFF2ON)
+		mmio_clrbits_32(DDRC_MSTR(0), (1 << 15));
+
+	sw_pstate(pstate);
+
+	/* DRAM dll enable */
+	if (dll_sw == OFF2ON) {
+		for (int i = 1; i <= num_rank; i++) {
+			ddr4_mr_write(1, mr_value[1] | 0x1, 0, i);
+		}
+
+		for (int i = 1; i <= num_rank; i++) {
+			ddr4_mr_write(0, mr_value[0] | 0x100, 0, i);
+		}
+	}
+
+	dram_cfg_all_mr(num_rank, pstate);
+
+	/* 16. Re-enable automatic generation of ZQCS/MPC(ZQ calibration) commands */
+	mmio_clrbits_32(DDRC_FREQ1_ZQCTL0(0), (1 << 31));
+	mmio_clrbits_32(DDRC_FREQ2_ZQCTL0(0), (1 << 31));
+	mmio_clrbits_32(DDRC_ZQCTL0(0), (1 << 31));
+
+	/* 17. Re-enable automatic refreshes (RFSHCTL3.dis_auto_refresh = 0) if they have been previously disabled. */
+	mmio_clrbits_32(DDRC_RFSHCTL3(0), 0x1);
+	/* 18. Restore ZQCTL0.dis_srx_zqcl */
+	/* 19. Write DBG1.dis_hif = 0 to re-enable reads and writes. */
+	mmio_clrbits_32(DDRC_DBG1(0), (0x1 << 1));
+
+	mmio_write_32(DDRC_PCTRL_0(0), 0x00000001);
+	/* 27. Write 1 to SBRCTL.scrub_en. Enable SBR if desired, only required if SBR instantiated. */
+
+	/* set SWCTL.sw_done to enable quasi-dynamic register programming outside reset. */
+	mmio_write_32(DDRC_SWCTL(0), 0x0001);
+
+	/* wait SWSTAT.sw_done_ack to 1 */
+	do {
+		tmp  = 0x1 & mmio_read_32(DDRC_SWSTAT(0));
+	} while (!tmp);
+}
+
+void ddr4_dll_no_change(unsigned int num_rank, unsigned int pstate)
+{
+	volatile unsigned int tmp;
+	/* 1. set SWCTL.sw_done to disable quasi-dynamic register programming outside reset. */
+	mmio_write_32(DDRC_SWCTL(0), 0x0000);
+
+	/* 2. Write 0 to PCTRL_n.port_en. This blocks AXI port(s) from taking any transaction (blocks traffic on
+	AXI ports). */
+	mmio_write_32(DDRC_PCTRL_0(0), 0x00000000);
+
+	/* 3. Poll PSTAT.rd_port_busy_n=0 and PSTAT.wr_port_busy_n=0. Wait until all AXI ports are idle (the
+	 uMCTL2 core has to be idle). */
+	do {
+		tmp = mmio_read_32(DDRC_PSTAT(0));
+	} while (tmp & 0x10001);
+
+	/* 4. Write 0 to SBRCTL.scrub_en. Disable SBR, required only if SBR instantiated. */
+	/* 5. Poll SBRSTAT.scrub_busy=0. Indicates that there are no outstanding SBR read commands */
+	/* 6. Set DERATEEN.derate_enable = 0, if DERATEEN.derate_eanble = 1 and the read latency (RL) value */
+	//needs to change after the frequency change (LPDDR2/3/4 only).
+	/* 7. Set DBG1.dis_hif=1 so that no new commands will be accepted by the uMCTL2. */
+	mmio_setbits_32(DDRC_DBG1(0), (0x1 << 1));
+	/* 8. Poll DBGCAM.dbg_wr_q_empty and DBGCAM.dbg_rd_q_empty to ensure that write and read data buffers are empty. */
+	do {
+		tmp = 0x06000000 & mmio_read_32(DDRC_DBGCAM(0));
+	} while (tmp != 0x06000000);
+	/* 9. For DDR4, update MR6 with the new tDLLK value via the Mode Register Write signals */
+
+	/* 10. Set DFILPCFG0.dfi_lp_en_sr = 0, if DFILPCFG0.dfi_lp_en_sr = 1, and wait until DFISTAT.dfi_lp_ack */
+	/* 11. If DFI PHY Master interface is active in uMCTL2 (DFIPHYMSTR.phymstr_en == 1'b1) then disable it */
+	/* 12. Wait until STAT.operating_mode[1:0]!=11 indicating that the DWC_ddr_umctl2 controller is not in self-refresh mode. */
+	tmp  = 0x3 & (mmio_read_32((DDRC_STAT(0))));
+	if (tmp == 0x3) {
+		printf("C: Error DRAM should not in Self Refresh\n");
+	}
+	/* 13. Assert PWRCTL.selfref_sw for the DWC_ddr_umctl2 core to enter the self-refresh mode. */
+	mmio_setbits_32(DDRC_PWRCTL(0), (1 << 5));
+	/* 14. Wait until STAT.operating_mode[1:0]==11 indicating that the DWC_ddr_umctl2 core is in selfrefresh mode. */
+	do {
+		tmp  = 0x3f & (mmio_read_32((DDRC_STAT(0))));
+	} while (tmp != 0x23);
+
+	sw_pstate(pstate);
+	dram_cfg_all_mr(num_rank, pstate);
+
+	/* 23. Enable HIF commands by setting DBG1.dis_hif=0. */
+	mmio_clrbits_32(DDRC_DBG1(0), (0x1 << 1));
+	/* 24. Reset DERATEEN.derate_enable = 1 if DERATEEN.derate_enable has been set to 0 in step 6. */
+	/* 25. If DFI PHY Master interface was active in uMCTL2 (DFIPHYMSTR.phymstr_en == 1'b1) before the */
+	/* step 11 then enable it back by programming DFIPHYMSTR.phymstr_en = 1'b1. */
+	/* 26. Write 1 to PCTRL_n.port_en. AXI port(s) are no longer blocked from taking transactions (Re-enable traffic on AXI ports). */
+	/* Re-enable traffic on the AXI ports */
+	mmio_write_32(DDRC_PCTRL_0(0), 0x00000001);
+	/* 27. Write 1 to SBRCTL.scrub_en. Enable SBR if desired, only required if SBR instantiated. */
+
+	/* set SWCTL.sw_done to enable quasi-dynamic register programming outside reset. */
+	mmio_write_32(DDRC_SWCTL(0), 0x0001);
+
+	/* wait SWSTAT.sw_done_ack to 1 */
+	do {
+		tmp  = 0x1 & mmio_read_32(DDRC_SWSTAT(0));
+	} while (!tmp);
+}
+
+void get_mr_values(unsigned int pstate)
+{
+	uint32_t init_val;
+
+	for (int i = 0; i < 3; i++) {
+		init_val = mmio_read_32(fsp_init_reg[pstate][i]);
+		mr_value[2*i] = init_val >> 16;
+		mr_value[2*i + 1] = init_val & 0xFFFF;
+	}
+
+	mr_value[6] = mmio_read_32(fsp_init_reg[pstate][3]) & 0xFFFF;
+}
+
+void ddr4_swffc(struct dram_info *dram_info, unsigned int pstate)
+{
+	unsigned int cur_pstate = dram_info->current_fsp;
+
+	get_mr_values(pstate);
+
+	if ((pstate == 0 && cur_pstate !=0) || (pstate != 0 && cur_pstate == 0))
+		ddr4_dll_change(dram_info->num_rank, pstate, cur_pstate);
+	else
+		ddr4_dll_no_change(dram_info->num_rank, pstate);
+
+	dram_info->current_fsp = pstate;
+}
diff --git a/plat/imx/common/imx8m/dram.c b/plat/imx/common/imx8m/dram.c
index e00ca96d..df34c4b0 100644
--- a/plat/imx/common/imx8m/dram.c
+++ b/plat/imx/common/imx8m/dram.c
@@ -67,13 +67,17 @@ void dram_phy_init(void)
 	}
 }
 
+
 void dram_info_init(unsigned long dram_timing_base)
 {
-	uint32_t current_fsp, ddr_type;
+	uint32_t current_fsp, ddr_type, ddrc_mstr;
 
 	/* get the dram type */
-	ddr_type = mmio_read_32(DDRC_MSTR(0)) & DDR_TYPE_MASK;
+	ddrc_mstr = mmio_read_32(DDRC_MSTR(0));
+	ddr_type = ddrc_mstr & DDR_TYPE_MASK;
+
 	dram_info.dram_type = ddr_type;
+	dram_info.num_rank = (ddrc_mstr >> 24) & DDRC_ACTIVE_RANK_MASK;
 
 	/* init the boot_fsp & current_fsp */
 	current_fsp = mmio_read_32(DDRC_DFIMISC(0));
@@ -93,6 +97,10 @@ void dram_info_init(unsigned long dram_timing_base)
 		dcsw_op_all(DCCSW);
 		lpddr4_swffc(dev_fsp, 0x0);
 		dev_fsp = (~dev_fsp) & 0x1;
+	} else if (ddr_type == DDRC_DDR4 && current_fsp != 0x0) {
+		/* flush the L1/L2 cache */
+		dcsw_op_all(DCCSW);
+		ddr4_swffc(&dram_info, 0x0);
 	}
 }
 
@@ -122,10 +130,6 @@ int dram_dvfs_handler(uint32_t smc_fid,
 	unsigned int target_freq = x1;
 	uint32_t online_cores = x2;
 
-	/* TODO add ddr4 dvfs support later */
-	if (dram_info.dram_type != DDRC_LPDDR4)
-		return 0;
-
 	if (target_freq == 0xf) {
 		/* set the WFE done status */
 		spin_lock(&dfs_lock);
@@ -159,8 +163,12 @@ int dram_dvfs_handler(uint32_t smc_fid,
 		/* flush the L1/L2 cache */
 		dcsw_op_all(DCCSW);
 
-		lpddr4_swffc(dev_fsp, target_freq);
-		dev_fsp = (~dev_fsp) & 0x1;
+		if (dram_info.dram_type == DDRC_LPDDR4) {
+			lpddr4_swffc(dev_fsp, target_freq);
+			dev_fsp = (~dev_fsp) & 0x1;
+		} else if (dram_info.dram_type == DDRC_DDR4) {
+			ddr4_swffc(&dram_info, target_freq);
+		}
 
 		wait_ddrc_hwffc_done = false;
 		wfe_done = 0;
diff --git a/plat/imx/common/include/dram.h b/plat/imx/common/include/dram.h
index fa203da2..f3d72dd8 100644
--- a/plat/imx/common/include/dram.h
+++ b/plat/imx/common/include/dram.h
@@ -30,6 +30,7 @@
 #define DDRC_LPDDR4	BIT(5)
 #define DDRC_DDR4	BIT(4)
 #define DDR_TYPE_MASK	0x3f
+#define DDRC_ACTIVE_RANK_MASK	0x3
 
 #define DDRPHY_REG(x)	(0x3c000000 + 4*x)
 
@@ -59,6 +60,7 @@ struct dram_timing_info {
 
 struct dram_info {
 	int dram_type;
+	unsigned int num_rank;
 	int current_fsp;
 	int boot_fsp;
 	struct dram_timing_info *timing_info;
@@ -78,6 +80,8 @@ void ddr4_enter_retention(void);
 void ddr4_exit_retention(void);
 /* lpddr4 swffc for dvfs */
 void lpddr4_swffc(unsigned int dev_fsp, unsigned int tgt_freq);
+/* ddr4 swffw for dvfs */
+void ddr4_swffc(struct dram_info *dram_info, unsigned int target_fsp);
 
 /* dram retention */
 void dram_enter_retention(void);
diff --git a/plat/imx/imx8mm/platform.mk b/plat/imx/imx8mm/platform.mk
index ac9db97c..30052bcc 100644
--- a/plat/imx/imx8mm/platform.mk
+++ b/plat/imx/imx8mm/platform.mk
@@ -14,7 +14,8 @@ PLAT_DRAM_SOURCES	:=	plat/imx/common/imx8m/dram.c		\
 				plat/imx/common/imx8m/lpddr4_retention.c \
 				plat/imx/common/imx8m/ddr4_retention.c \
 				plat/imx/common/imx8m/lpddr4_helper.c	\
-				plat/imx/common/imx8m/lpddr4_dvfs.c
+				plat/imx/common/imx8m/lpddr4_dvfs.c	\
+				plat/imx/common/imx8m/ddr4_dvfs.c
 
 BL31_SOURCES		+=	plat/imx/common/imx8_helpers.S		\
 				plat/imx/common/mxcuart_console.S	\