src/soc/rockchip/rk3288/sdram.c

/*
 * This file is part of the coreboot project.
 *
 * Copyright 2014 Rockchip Inc.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; version 2 of the License.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
 */
#include <arch/io.h>
#include <string.h>
#include <types.h>
#include <console/console.h>
#include <delay.h>
#include "addressmap.h"
#include "clock.h"
#include "sdram.h"
#include "grf.h"
#include "soc.h"
#include "pmu.h"

struct rk3288_ddr_pctl_regs {
	u32 scfg;
	u32 sctl;
	u32 stat;
	u32 intrstat;
	u32 reserved0[12];
	u32 mcmd;
	u32 powctl;
	u32 powstat;
	u32 cmdtstat;
	u32 tstaten;
	u32 reserved1[3];
	u32 mrrcfg0;
	u32 mrrstat0;
	u32 mrrstat1;
	u32 reserved2[4];
	u32 mcfg1;
	u32 mcfg;
	u32 ppcfg;
	u32 mstat;
	u32 lpddr2zqcfg;
	u32 reserved3;
	u32 dtupdes;
	u32 dtuna;
	u32 dtune;
	u32 dtuprd0;
	u32 dtuprd1;
	u32 dtuprd2;
	u32 dtuprd3;
	u32 dtuawdt;
	u32 reserved4[3];
	u32 togcnt1u;
	u32 tinit;
	u32 trsth;
	u32 togcnt100n;
	u32 trefi;
	u32 tmrd;
	u32 trfc;
	u32 trp;
	u32 trtw;
	u32 tal;
	u32 tcl;
	u32 tcwl;
	u32 tras;
	u32 trc;
	u32 trcd;
	u32 trrd;
	u32 trtp;
	u32 twr;
	u32 twtr;
	u32 texsr;
	u32 txp;
	u32 txpdll;
	u32 tzqcs;
	u32 tzqcsi;
	u32 tdqs;
	u32 tcksre;
	u32 tcksrx;
	u32 tcke;
	u32 tmod;
	u32 trstl;
	u32 tzqcl;
	u32 tmrr;
	u32 tckesr;
	u32 tdpd;
	u32 reserved5[14];
	u32 ecccfg;
	u32 ecctst;
	u32 eccclr;
	u32 ecclog;
	u32 reserved6[28];
	u32 dtuwactl;
	u32 dturactl;
	u32 dtucfg;
	u32 dtuectl;
	u32 dtuwd0;
	u32 dtuwd1;
	u32 dtuwd2;
	u32 dtuwd3;
	u32 dtuwdm;
	u32 dturd0;
	u32 dturd1;
	u32 dturd2;
	u32 dturd3;
	u32 dtulfsrwd;
	u32 dtulfsrrd;
	u32 dtueaf;
	u32 dfitctrldelay;
	u32 dfiodtcfg;
	u32 dfiodtcfg1;
	u32 dfiodtrankmap;
	u32 dfitphywrdata;
	u32 dfitphywrlat;
	u32 reserved7[2];
	u32 dfitrddataen;
	u32 dfitphyrdlat;
	u32 reserved8[2];
	u32 dfitphyupdtype0;
	u32 dfitphyupdtype1;
	u32 dfitphyupdtype2;
	u32 dfitphyupdtype3;
	u32 dfitctrlupdmin;
	u32 dfitctrlupdmax;
	u32 dfitctrlupddly;
	u32 reserved9;
	u32 dfiupdcfg;
	u32 dfitrefmski;
	u32 dfitctrlupdi;
	u32 reserved10[4];
	u32 dfitrcfg0;
	u32 dfitrstat0;
	u32 dfitrwrlvlen;
	u32 dfitrrdlvlen;
	u32 dfitrrdlvlgateen;
	u32 dfiststat0;
	u32 dfistcfg0;
	u32 dfistcfg1;
	u32 reserved11;
	u32 dfitdramclken;
	u32 dfitdramclkdis;
	u32 dfistcfg2;
	u32 dfistparclr;
	u32 dfistparlog;
	u32 reserved12[3];
	u32 dfilpcfg0;
	u32 reserved13[3];
	u32 dfitrwrlvlresp0;
	u32 dfitrwrlvlresp1;
	u32 dfitrwrlvlresp2;
	u32 dfitrrdlvlresp0;
	u32 dfitrrdlvlresp1;
	u32 dfitrrdlvlresp2;
	u32 dfitrwrlvldelay0;
	u32 dfitrwrlvldelay1;
	u32 dfitrwrlvldelay2;
	u32 dfitrrdlvldelay0;
	u32 dfitrrdlvldelay1;
	u32 dfitrrdlvldelay2;
	u32 dfitrrdlvlgatedelay0;
	u32 dfitrrdlvlgatedelay1;
	u32 dfitrrdlvlgatedelay2;
	u32 dfitrcmd;
	u32 reserved14[46];
	u32 ipvr;
	u32 iptr;
};
check_member(rk3288_ddr_pctl_regs, iptr, 0x03fc);

struct rk3288_ddr_publ_datx {
	u32 dxgcr;
	u32 dxgsr[2];
	u32 dxdllcr;
	u32 dxdqtr;
	u32 dxdqstr;
	u32 reserved[10];
};

struct rk3288_ddr_publ_regs {
	u32 ridr;
	u32 pir;
	u32 pgcr;
	u32 pgsr;
	u32 dllgcr;
	u32 acdllcr;
	u32 ptr[3];
	u32 aciocr;
	u32 dxccr;
	u32 dsgcr;
	u32 dcr;
	u32 dtpr[3];
	u32 mr[4];
	u32 odtcr;
	u32 dtar;
	u32 dtdr[2];
	u32 reserved1[24];
	u32 dcuar;
	u32 dcudr;
	u32 dcurr;
	u32 dculr;
	u32 dcugcr;
	u32 dcutpr;
	u32 dcusr[2];
	u32 reserved2[8];
	u32 bist[17];
	u32 reserved3[15];
	u32 zq0cr[2];
	u32 zq0sr[2];
	u32 zq1cr[2];
	u32 zq1sr[2];
	u32 zq2cr[2];
	u32 zq2sr[2];
	u32 zq3cr[2];
	u32 zq3sr[2];
	struct rk3288_ddr_publ_datx datx8[4];
};
check_member(rk3288_ddr_publ_regs, datx8[3].dxdqstr, 0x0294);

struct rk3288_msch_regs {
	u32 coreid;
	u32 revisionid;
	u32 ddrconf;
	u32 ddrtiming;
	u32 ddrmode;
	u32 readlatency;
	u32 reserved1[8];
	u32 activate;
	u32 devtodev;
};
check_member(rk3288_msch_regs, devtodev, 0x003c);

static struct rk3288_ddr_pctl_regs * const rk3288_ddr_pctl[2] = {
	(void *)DDR_PCTL0_BASE, (void *)DDR_PCTL1_BASE};
static struct rk3288_ddr_publ_regs * const rk3288_ddr_publ[2] = {
	(void *)DDR_PUBL0_BASE, (void *)DDR_PUBL1_BASE};
static struct rk3288_msch_regs * const rk3288_msch[2] = {
	(void *)SERVICE_BUS_BASE, (void *)SERVICE_BUS_BASE + 0x80};

/* PCT_DFISTCFG0 */
#define DFI_INIT_START			(1 << 0)

/* PCT_DFISTCFG1 */
#define DFI_DRAM_CLK_SR_EN		(1 << 0)
#define DFI_DRAM_CLK_DPD_EN		(1 << 1)

/* PCT_DFISTCFG2 */
#define DFI_PARITY_INTR_EN		(1 << 0)
#define DFI_PARITY_EN			(1 << 1)

/* PCT_DFILPCFG0 */
#define TLP_RESP_TIME(n)		(n << 16)
#define LP_SR_EN			(1 << 8)
#define LP_PD_EN			(1 << 0)

/* PCT_DFITCTRLDELAY */
#define TCTRL_DELAY_TIME(n)		(n << 0)

/* PCT_DFITPHYWRDATA */
#define TPHY_WRDATA_TIME(n)		(n << 0)

/* PCT_DFITPHYRDLAT */
#define TPHY_RDLAT_TIME(n)		(n << 0)

/* PCT_DFITDRAMCLKDIS */
#define TDRAM_CLK_DIS_TIME(n)		(n << 0)

/* PCT_DFITDRAMCLKEN */
#define TDRAM_CLK_EN_TIME(n)		(n << 0)

/* PCTL_DFIODTCFG */
#define RANK0_ODT_WRITE_SEL		(1 << 3)
#define RANK1_ODT_WRITE_SEL		(1 << 11)

/* PCTL_DFIODTCFG1 */
#define ODT_LEN_BL8_W(n)		(n<<16)

/* PUBL_ACDLLCR */
#define ACDLLCR_DLLDIS			(1 << 31)
#define ACDLLCR_DLLSRST			(1 << 30)

/* PUBL_DXDLLCR */
#define DXDLLCR_DLLDIS			(1 << 31)
#define DXDLLCR_DLLSRST			(1 << 30)

/* PUBL_DLLGCR */
#define DLLGCR_SBIAS			(1 << 30)

/* PUBL_DXGCR */
#define DQSRTT				(1 << 9)
#define DQRTT				(1 << 10)

/* PIR */
#define PIR_INIT			(1 << 0)
#define PIR_DLLSRST			(1 << 1)
#define PIR_DLLLOCK			(1 << 2)
#define PIR_ZCAL			(1 << 3)
#define PIR_ITMSRST			(1 << 4)
#define PIR_DRAMRST			(1 << 5)
#define PIR_DRAMINIT			(1 << 6)
#define PIR_QSTRN			(1 << 7)
#define PIR_RVTRN			(1 << 8)
#define PIR_ICPC			(1 << 16)
#define PIR_DLLBYP			(1 << 17)
#define PIR_CTLDINIT			(1 << 18)
#define PIR_CLRSR			(1 << 28)
#define PIR_LOCKBYP			(1 << 29)
#define PIR_ZCALBYP			(1 << 30)
#define PIR_INITBYP			(1u << 31)

/* PGCR */
#define PGCR_DFTLMT(n)			((n) << 3)
#define PGCR_DFTCMP(n)			((n) << 2)
#define PGCR_DQSCFG(n)			((n) << 1)
#define PGCR_ITMDMD(n)			((n) << 0)

/* PGSR */
#define PGSR_IDONE			(1 << 0)
#define PGSR_DLDONE			(1 << 1)
#define PGSR_ZCDONE			(1 << 2)
#define PGSR_DIDONE			(1 << 3)
#define PGSR_DTDONE			(1 << 4)
#define PGSR_DTERR			(1 << 5)
#define PGSR_DTIERR			(1 << 6)
#define PGSR_DFTERR			(1 << 7)
#define PGSR_RVERR			(1 << 8)
#define PGSR_RVEIRR			(1 << 9)

/* PTR0 */
#define PRT_ITMSRST(n)			((n) << 18)
#define PRT_DLLLOCK(n)			((n) << 6)
#define PRT_DLLSRST(n)			((n) << 0)

/* PTR1 */
#define PRT_DINIT1(n)			((n) << 19)
#define PRT_DINIT0(n)			((n) << 0)

/* PTR2 */
#define PRT_DINIT3(n)			((n) << 17)
#define PRT_DINIT2(n)			((n) << 0)

/* DCR */
#define DDRMD_LPDDR			0
#define DDRMD_DDR			1
#define DDRMD_DDR2			2
#define DDRMD_DDR3			3
#define DDRMD_LPDDR2_LPDDR3		4
#define DDRMD_MSK			(7 << 0)
#define DDRMD_CFG(n)			((n) << 0)
#define PDQ_MSK				(7 << 4)
#define PDQ_CFG(n)			((n) << 4)

/* DXCCR */
#define DQSNRES_MSK			(0x0f << 8)
#define DQSNRES_CFG(n)			((n) << 8)
#define DQSRES_MSK			(0x0f << 4)
#define DQSRES_CFG(n)			((n) << 4)

/* DTPR */
#define TDQSCKMAX_VAL(n)		(((n) >> 27) & 7)
#define TDQSCK_VAL(n)			(((n) >> 24) & 7)

/* DSGCR */
#define DQSGX_MSK			(0x07 << 5)
#define DQSGX_CFG(n)			((n) << 5)
#define DQSGE_MSK			(0x07 << 8)
#define DQSGE_CFG(n)			((n) << 8)

/* SCTL */
#define INIT_STATE			(0)
#define CFG_STATE			(1)
#define GO_STATE			(2)
#define SLEEP_STATE			(3)
#define WAKEUP_STATE			(4)

/* STAT */
#define LP_TRIG_VAL(n)			(((n) >> 4) & 7)
#define PCTL_STAT_MSK			(7)
#define INIT_MEM			(0)
#define CONFIG				(1)
#define CONFIG_REQ			(2)
#define ACCESS				(3)
#define ACCESS_REQ			(4)
#define LOW_POWER			(5)
#define LOW_POWER_ENTRY_REQ		(6)
#define LOW_POWER_EXIT_REQ		(7)

/* ZQCR*/
#define PD_OUTPUT(n)			((n) << 0)
#define PU_OUTPUT(n)			((n) << 5)
#define PD_ONDIE(n)			((n) << 10)
#define PU_ONDIE(n)			((n) << 15)
#define ZDEN(n)				((n) << 28)

/* DDLGCR */
#define SBIAS_BYPASS			(1 << 23)

/* MCFG */
#define MDDR_LPDDR2_CLK_STOP_IDLE(n)	((n) << 24)
#define PD_IDLE(n)			((n) << 8)
#define MDDR_EN				(2 << 22)
#define LPDDR2_EN			(3 << 22)
#define DDR2_EN				(0 << 5)
#define DDR3_EN				(1 << 5)
#define LPDDR2_S2			(0 << 6)
#define LPDDR2_S4			(1 << 6)
#define MDDR_LPDDR2_BL_2		(0 << 20)
#define MDDR_LPDDR2_BL_4		(1 << 20)
#define MDDR_LPDDR2_BL_8		(2 << 20)
#define MDDR_LPDDR2_BL_16		(3 << 20)
#define DDR2_DDR3_BL_4			(0)
#define DDR2_DDR3_BL_8			(1)
#define TFAW_CFG(n)			(((n)-4) << 18)
#define PD_EXIT_SLOW			(0 << 17)
#define PD_EXIT_FAST			(1 << 17)
#define PD_TYPE(n)			((n) << 16)
#define BURSTLENGTH_CFG(n)		(((n) >> 1) << 20)

/* POWCTL */
#define POWER_UP_START			(1 << 0)

/* POWSTAT */
#define POWER_UP_DONE			(1 << 0)

/* MCMD */
#define DESELECT_CMD			(0)
#define PREA_CMD			(1)
#define REF_CMD				(2)
#define MRS_CMD				(3)
#define ZQCS_CMD			(4)
#define ZQCL_CMD			(5)
#define RSTL_CMD			(6)
#define MRR_CMD				(8)
#define DPDE_CMD			(9)

#define LPDDR2_MA(n)			(((n) & 0xff) << 4)

#define START_CMD			(1u << 31)

/* DEVTODEV */
#define BUSWRTORD(n)			((n) << 4)
#define BUSRDTOWR(n)			((n) << 2)
#define BUSRDTORD(n)			((n) << 0)

/* GRF_SOC_CON0 */
#define MSCH_MAINDDR3(ch, n)		(((n) << (3 + (ch))) \
					| ((1 << (3 + (ch))) << 16))

/* GRF_SOC_CON2 */
#define PUBL_LPDDR3_EN(ch, n) RK_CLRSETBITS(1 << (10 + (3 * (ch))), \
	(n) << (10 + (3 * (ch))))
#define PCTL_LPDDR3_ODT_EN(ch, n) RK_CLRSETBITS(1 << (9 + (3 * (ch))), \
	(n) << (9 + (3 * (ch))))
#define PCTL_BST_DISABLE(ch, n) RK_CLRSETBITS(1 << (8 + (3 * (ch))), \
	(n) << (8 + (3 * (ch))))

/* mr1 for ddr3 */
#define DDR3_DLL_ENABLE		(0)
#define DDR3_DLL_DISABLE	(1)

/*
 * sys_reg bitfield struct
 * [31] row_3_4_ch1
 * [30] row_3_4_ch0
 * [29:28] chinfo
 * [27] rank_ch1
 * [26:25] col_ch1
 * [24] bk_ch1
 * [23:22] cs0_row_ch1
 * [21:20] cs1_row_ch1
 * [19:18] bw_ch1
 * [17:16] dbw_ch1;
 * [15:13] ddrtype
 * [12] channelnum
 * [11] rank_ch0
 * [10:9] col_ch0
 * [8] bk_ch0
 * [7:6] cs0_row_ch0
 * [5:4] cs1_row_ch0
 * [3:2] bw_ch0
 * [1:0] dbw_ch0
*/
#define SYS_REG_DDRTYPE(n)		((n) << 13)
#define SYS_REG_NUM_CH(n)		(((n) - 1) << 12)
#define SYS_REG_ROW_3_4(n, ch)		((n) << (30 + (ch)))
#define SYS_REG_CHINFO(ch)		(1 << (28 + (ch)))
#define SYS_REG_RANK(n, ch)		(((n) - 1) << (11 + ((ch) * 16)))
#define SYS_REG_COL(n, ch)		(((n) - 9) << (9 + ((ch) * 16)))
#define SYS_REG_BK(n, ch)		(((n) == 3 ? 0 : 1) \
					<< (8 + ((ch) * 16)))
#define SYS_REG_CS0_ROW(n, ch)		(((n) - 13) << (6 + ((ch) * 16)))
#define SYS_REG_CS1_ROW(n, ch)		(((n) - 13) << (4 + ((ch) * 16)))
#define SYS_REG_BW(n, ch)		((2 >> (n)) << (2 + ((ch) * 16)))
#define SYS_REG_DBW(n, ch)		((2 >> (n)) << (0 + ((ch) * 16)))

static void copy_to_reg(u32 *dest, const u32 *src, u32 n)
{
	int i;
	for (i = 0; i < n / sizeof(u32); i++) {
		writel(*src, dest);
		src++;
		dest++;
	}
}

static void phy_pctrl_reset(struct rk3288_ddr_publ_regs *ddr_publ_regs,
			    u32 channel)
{
	int i;
	rkclk_ddr_reset(channel, 1, 1);
	udelay(1);
	clrbits_le32(&ddr_publ_regs->acdllcr, ACDLLCR_DLLSRST);
	for (i = 0; i < 4; i++)
		clrbits_le32(&ddr_publ_regs->datx8[i].dxdllcr, DXDLLCR_DLLSRST);

	udelay(10);
	setbits_le32(&ddr_publ_regs->acdllcr, ACDLLCR_DLLSRST);
	for (i = 0; i < 4; i++)
		setbits_le32(&ddr_publ_regs->datx8[i].dxdllcr, DXDLLCR_DLLSRST);

	udelay(10);
	rkclk_ddr_reset(channel, 1, 0);
	udelay(10);
	rkclk_ddr_reset(channel, 0, 0);
	udelay(1);
}

static void phy_dll_bypass_set(struct rk3288_ddr_publ_regs *ddr_publ_regs,
	u32 freq)
{
	int i;
	if (freq <= 250000000) {
		if (freq <= 150000000)
			clrbits_le32(&ddr_publ_regs->dllgcr, SBIAS_BYPASS);
		else
			setbits_le32(&ddr_publ_regs->dllgcr, SBIAS_BYPASS);
		setbits_le32(&ddr_publ_regs->acdllcr, ACDLLCR_DLLDIS);
		for (i = 0; i < 4; i++)
			setbits_le32(&ddr_publ_regs->datx8[i].dxdllcr,
				DXDLLCR_DLLDIS);

		setbits_le32(&ddr_publ_regs->pir, PIR_DLLBYP);
	} else {
		clrbits_le32(&ddr_publ_regs->dllgcr, SBIAS_BYPASS);
		clrbits_le32(&ddr_publ_regs->acdllcr, ACDLLCR_DLLDIS);
		for (i = 0; i < 4; i++)
			clrbits_le32(&ddr_publ_regs->datx8[i].dxdllcr,
				DXDLLCR_DLLDIS);

		clrbits_le32(&ddr_publ_regs->pir, PIR_DLLBYP);
	}
}

static void dfi_cfg(struct rk3288_ddr_pctl_regs *ddr_pctl_regs, u32 dramtype)
{
	writel(DFI_INIT_START, &ddr_pctl_regs->dfistcfg0);
	writel(DFI_DRAM_CLK_SR_EN | DFI_DRAM_CLK_DPD_EN,
			&ddr_pctl_regs->dfistcfg1);
	writel(DFI_PARITY_INTR_EN | DFI_PARITY_EN, &ddr_pctl_regs->dfistcfg2);
	writel(TLP_RESP_TIME(7) | LP_SR_EN | LP_PD_EN,
		&ddr_pctl_regs->dfilpcfg0);

	writel(TCTRL_DELAY_TIME(2), &ddr_pctl_regs->dfitctrldelay);
	writel(TPHY_WRDATA_TIME(1), &ddr_pctl_regs->dfitphywrdata);
	writel(TPHY_RDLAT_TIME(0xf), &ddr_pctl_regs->dfitphyrdlat);
	writel(TDRAM_CLK_DIS_TIME(2), &ddr_pctl_regs->dfitdramclkdis);
	writel(TDRAM_CLK_EN_TIME(2), &ddr_pctl_regs->dfitdramclken);
	writel(0x1, &ddr_pctl_regs->dfitphyupdtype0);

	/* cs0 and cs1 write odt enable */
	writel((RANK0_ODT_WRITE_SEL | RANK1_ODT_WRITE_SEL),
		&ddr_pctl_regs->dfiodtcfg);
	/* odt write length */
	writel(ODT_LEN_BL8_W(7), &ddr_pctl_regs->dfiodtcfg1);
	/* phyupd and ctrlupd disabled */
	writel(0, &ddr_pctl_regs->dfiupdcfg);
}

static void pctl_cfg(u32 channel,
		     const struct rk3288_sdram_params *sdram_params)
{
	unsigned int burstlen;
	struct rk3288_ddr_pctl_regs *ddr_pctl_regs = rk3288_ddr_pctl[channel];
	burstlen = (sdram_params->noc_timing >> 18) & 0x7;
	copy_to_reg(&ddr_pctl_regs->togcnt1u,
			&(sdram_params->pctl_timing.togcnt1u),
			sizeof(sdram_params->pctl_timing));
	switch (sdram_params->dramtype) {
	case LPDDR3:
		writel(sdram_params->pctl_timing.tcl - 1,
		       &ddr_pctl_regs->dfitrddataen);
		writel(sdram_params->pctl_timing.tcwl,
		       &ddr_pctl_regs->dfitphywrlat);
		writel(LPDDR2_S4 | MDDR_LPDDR2_CLK_STOP_IDLE(0) | LPDDR2_EN
			| BURSTLENGTH_CFG(burstlen) | TFAW_CFG(6) | PD_EXIT_FAST
			| PD_TYPE(1) | PD_IDLE(0), &ddr_pctl_regs->mcfg);
		writel(MSCH_MAINDDR3(channel, 0), &rk3288_grf->soc_con0);

		writel(PUBL_LPDDR3_EN(channel, 1)
		       | PCTL_BST_DISABLE(channel, 1)
		       | PCTL_LPDDR3_ODT_EN(channel, 1),
		       &rk3288_grf->soc_con2);

		break;
	case DDR3:
		if (sdram_params->phy_timing.mr[1] & DDR3_DLL_DISABLE)
			writel(sdram_params->pctl_timing.tcl - 3,
			       &ddr_pctl_regs->dfitrddataen);
		else
			writel(sdram_params->pctl_timing.tcl - 2,
			       &ddr_pctl_regs->dfitrddataen);
		writel(sdram_params->pctl_timing.tcwl - 1,
		       &ddr_pctl_regs->dfitphywrlat);
		writel(MDDR_LPDDR2_CLK_STOP_IDLE(0) | DDR3_EN
		       | DDR2_DDR3_BL_8 | TFAW_CFG(5) | PD_EXIT_SLOW
		       | PD_TYPE(1) | PD_IDLE(0), &ddr_pctl_regs->mcfg);
		writel(MSCH_MAINDDR3(channel, 1), &rk3288_grf->soc_con0);

		writel(PUBL_LPDDR3_EN(channel, 0)
		       | PCTL_BST_DISABLE(channel, 0)
		       | PCTL_LPDDR3_ODT_EN(channel, 0),
		       &rk3288_grf->soc_con2);

		break;
	}

	setbits_le32(&ddr_pctl_regs->scfg, 1);
}

static void phy_cfg(u32 channel, const struct rk3288_sdram_params *sdram_params)
{
	u32 i;
	struct rk3288_ddr_publ_regs *ddr_publ_regs = rk3288_ddr_publ[channel];
	struct rk3288_msch_regs *msch_regs = rk3288_msch[channel];

	/* DDR PHY Timing */
	copy_to_reg(&ddr_publ_regs->dtpr[0],
			&(sdram_params->phy_timing.dtpr0),
			sizeof(sdram_params->phy_timing));
	writel(sdram_params->noc_timing, &msch_regs->ddrtiming);
	writel(0x3f, &msch_regs->readlatency);
	writel(sdram_params->noc_activate, &msch_regs->activate);
	writel(BUSWRTORD(2) | BUSRDTOWR(2) | BUSRDTORD(1),
		&msch_regs->devtodev);
	writel(PRT_ITMSRST(8) | PRT_DLLLOCK(2750) | PRT_DLLSRST(27),
	       &ddr_publ_regs->ptr[0]);
	/* tDINIT1=400ns (533MHz), tDINIT0=500us (533MHz) */
	writel(PRT_DINIT1(213) | PRT_DINIT0(266525), &ddr_publ_regs->ptr[1]);
	/* tDINIT3=1us (533MHz), tDINIT2=200us (533MHz) */
	writel(PRT_DINIT3(534) | PRT_DINIT2(106610), &ddr_publ_regs->ptr[2]);

	switch (sdram_params->dramtype) {
	case LPDDR3:
		clrsetbits_le32(&ddr_publ_regs->pgcr, 0x1F, PGCR_DFTLMT(0)
		       | PGCR_DFTCMP(0) | PGCR_DQSCFG(1) | PGCR_ITMDMD(0));
		/* DDRMODE select LPDDR3 */
		clrsetbits_le32(&ddr_publ_regs->dcr, DDRMD_MSK,
			DDRMD_CFG(DDRMD_LPDDR2_LPDDR3));
		clrsetbits_le32(&ddr_publ_regs->dxccr, DQSNRES_MSK | DQSRES_MSK,
				DQSRES_CFG(4) | DQSNRES_CFG(0xc));
		i = TDQSCKMAX_VAL(readl(&ddr_publ_regs->dtpr[1]))
		    - TDQSCK_VAL(readl(&ddr_publ_regs->dtpr[1]));
		clrsetbits_le32(&ddr_publ_regs->dsgcr, DQSGE_MSK | DQSGX_MSK,
				DQSGE_CFG(i) | DQSGX_CFG(i));
		break;
	case DDR3:
		clrbits_le32(&ddr_publ_regs->pgcr, 0x1f);
		clrsetbits_le32(&ddr_publ_regs->dcr, DDRMD_MSK,
			DDRMD_CFG(DDRMD_DDR3));
		break;
	}
	if (sdram_params->odt) {
		/*dynamic RTT enable */
		for (i = 0; i < 4; i++)
			setbits_le32(&ddr_publ_regs->datx8[i].dxgcr,
				DQSRTT | DQRTT);
	} else {
		/*dynamic RTT disable */
		for (i = 0; i < 4; i++)
			clrbits_le32(&ddr_publ_regs->datx8[i].dxgcr,
				DQSRTT | DQRTT);

	}
}

static void phy_init(struct rk3288_ddr_publ_regs *ddr_publ_regs)
{
	setbits_le32(&ddr_publ_regs->pir, PIR_INIT | PIR_DLLSRST
		| PIR_DLLLOCK | PIR_ZCAL | PIR_ITMSRST | PIR_CLRSR);
	udelay(1);
	while ((readl(&ddr_publ_regs->pgsr) &
		(PGSR_IDONE | PGSR_DLDONE | PGSR_ZCDONE)) !=
		(PGSR_IDONE | PGSR_DLDONE | PGSR_ZCDONE))
		;
}

static void send_command(struct rk3288_ddr_pctl_regs *ddr_pctl_regs, u32 rank,
			 u32 cmd, u32 arg)
{
	writel((START_CMD | (rank << 20) | arg | cmd), &ddr_pctl_regs->mcmd);
	udelay(1);
	while (readl(&ddr_pctl_regs->mcmd) & START_CMD)
		;
}

static void memory_init(struct rk3288_ddr_publ_regs *ddr_publ_regs,
			u32 dramtype)
{
	setbits_le32(&ddr_publ_regs->pir,
		     (PIR_INIT | PIR_DRAMINIT | PIR_LOCKBYP
		      | PIR_ZCALBYP | PIR_CLRSR | PIR_ICPC
		      | (dramtype == DDR3 ? PIR_DRAMRST : 0)));
	udelay(1);
	while ((readl(&ddr_publ_regs->pgsr) & (PGSR_IDONE | PGSR_DLDONE))
		!= (PGSR_IDONE | PGSR_DLDONE))
		;
}

static void move_to_config_state(struct rk3288_ddr_publ_regs *ddr_publ_regs,
				 struct rk3288_ddr_pctl_regs *ddr_pctl_regs)
{
	unsigned int state;

	while (1) {
		state = readl(&ddr_pctl_regs->stat) & PCTL_STAT_MSK;

		switch (state) {
		case LOW_POWER:
			writel(WAKEUP_STATE, &ddr_pctl_regs->sctl);
			while ((readl(&ddr_pctl_regs->stat) & PCTL_STAT_MSK)
				!= ACCESS)
				;
			/* wait DLL lock */
			while ((readl(&ddr_publ_regs->pgsr) & PGSR_DLDONE)
				!= PGSR_DLDONE)
				;
			/* if at low power state,need wakeup first,
			 * and then enter the config
			 * so here no break.
			 */
		case ACCESS:
		case INIT_MEM:
			writel(CFG_STATE, &ddr_pctl_regs->sctl);
			while ((readl(&ddr_pctl_regs->stat) & PCTL_STAT_MSK)
				!= CONFIG)
				;
			break;
		case CONFIG:
			return;
		default:
			break;
		}
	}
}

static void set_bandwidth_ratio(u32 channel, u32 n)
{
	struct rk3288_ddr_pctl_regs *ddr_pctl_regs = rk3288_ddr_pctl[channel];
	struct rk3288_ddr_publ_regs *ddr_publ_regs = rk3288_ddr_publ[channel];
	struct rk3288_msch_regs *msch_regs = rk3288_msch[channel];

	if (n == 1) {
		setbits_le32(&ddr_pctl_regs->ppcfg, 1);
		writel(RK_SETBITS(1 << (8 + channel)),
		       &rk3288_grf->soc_con0);
		setbits_le32(&msch_regs->ddrtiming, 1 << 31);
		/* Data Byte disable*/
		clrbits_le32(&ddr_publ_regs->datx8[2].dxgcr, 1);
		clrbits_le32(&ddr_publ_regs->datx8[3].dxgcr, 1);
		/*disable DLL */
		setbits_le32(&ddr_publ_regs->datx8[2].dxdllcr,
			DXDLLCR_DLLDIS);
		setbits_le32(&ddr_publ_regs->datx8[3].dxdllcr,
			DXDLLCR_DLLDIS);
	} else {
		clrbits_le32(&ddr_pctl_regs->ppcfg, 1);
		writel(RK_CLRBITS(1 << (8 + channel)),
		       &rk3288_grf->soc_con0);
		clrbits_le32(&msch_regs->ddrtiming, 1 << 31);
		/* Data Byte enable*/
		setbits_le32(&ddr_publ_regs->datx8[2].dxgcr, 1);
		setbits_le32(&ddr_publ_regs->datx8[3].dxgcr, 1);

		/*enable DLL */
		clrbits_le32(&ddr_publ_regs->datx8[2].dxdllcr,
			DXDLLCR_DLLDIS);
		clrbits_le32(&ddr_publ_regs->datx8[3].dxdllcr,
			DXDLLCR_DLLDIS);
		/* reset DLL */
		clrbits_le32(&ddr_publ_regs->datx8[2].dxdllcr,
			DXDLLCR_DLLSRST);
		clrbits_le32(&ddr_publ_regs->datx8[3].dxdllcr,
			DXDLLCR_DLLSRST);
		udelay(10);
		setbits_le32(&ddr_publ_regs->datx8[2].dxdllcr,
			DXDLLCR_DLLSRST);
		setbits_le32(&ddr_publ_regs->datx8[3].dxdllcr,
			DXDLLCR_DLLSRST);
	}
	setbits_le32(&ddr_pctl_regs->dfistcfg0, 1 << 2);

}

static int data_training(u32 channel,
	const struct rk3288_sdram_params *sdram_params)
{
	unsigned int j;
	int ret = 0;
	u32 rank;
	int i;
	u32 step[2] = { PIR_QSTRN, PIR_RVTRN };
	struct rk3288_ddr_publ_regs *ddr_publ_regs = rk3288_ddr_publ[channel];
	struct rk3288_ddr_pctl_regs *ddr_pctl_regs = rk3288_ddr_pctl[channel];

	/* disable auto refresh */
	writel(0, &ddr_pctl_regs->trefi);

	if (sdram_params->dramtype != LPDDR3)
		setbits_le32(&ddr_publ_regs->pgcr, PGCR_DQSCFG(1));
	rank = sdram_params->ch[channel].rank | 1;
	for (j = 0; j < ARRAY_SIZE(step); j++) {
		/*
		 * trigger QSTRN and RVTRN
		 * clear DTDONE status
		 */
		setbits_le32(&ddr_publ_regs->pir, PIR_CLRSR);

		/* trigger DTT */
		setbits_le32(&ddr_publ_regs->pir,
			     PIR_INIT | step[j] | PIR_LOCKBYP | PIR_ZCALBYP |
			     PIR_CLRSR);
		udelay(1);
		/* wait echo byte DTDONE */
		while ((readl(&ddr_publ_regs->datx8[0].dxgsr[0]) & rank)
			!= rank)
			;
		while ((readl(&ddr_publ_regs->datx8[1].dxgsr[0]) & rank)
			!= rank)
			;
		if (!(readl(&ddr_pctl_regs->ppcfg) & 1)) {
			while ((readl(&ddr_publ_regs->datx8[2].dxgsr[0])
				& rank) != rank)
				;
			while ((readl(&ddr_publ_regs->datx8[3].dxgsr[0])
				& rank) != rank)
				;
		}
		if (readl(&ddr_publ_regs->pgsr) &
		    (PGSR_DTERR | PGSR_RVERR | PGSR_RVEIRR)) {
			ret = -1;
			break;
		}
	}
	/* send some auto refresh to complement the lost while DTT */
	for (i = 0; i < (rank > 1 ? 4 : 2); i++)
		send_command(ddr_pctl_regs, rank, REF_CMD, 0);

	if (sdram_params->dramtype != LPDDR3)
		clrbits_le32(&ddr_publ_regs->pgcr, PGCR_DQSCFG(1));

	/* resume auto refresh */
	writel(sdram_params->pctl_timing.trefi, &ddr_pctl_regs->trefi);

	return ret;
}

static void move_to_access_state(u32 chnum)
{
	struct rk3288_ddr_publ_regs *ddr_publ_regs = rk3288_ddr_publ[chnum];
	struct rk3288_ddr_pctl_regs *ddr_pctl_regs = rk3288_ddr_pctl[chnum];

	unsigned int state;

	while (1) {
		state = readl(&ddr_pctl_regs->stat) & PCTL_STAT_MSK;

		switch (state) {
		case LOW_POWER:
			if (LP_TRIG_VAL(readl(&ddr_pctl_regs->stat)) == 1)
				return;

			writel(WAKEUP_STATE, &ddr_pctl_regs->sctl);
			while ((readl(&ddr_pctl_regs->stat) & PCTL_STAT_MSK)
				!= ACCESS)
				;
			/* wait DLL lock */
			while ((readl(&ddr_publ_regs->pgsr) & PGSR_DLDONE)
				!= PGSR_DLDONE)
				;
			break;
		case INIT_MEM:
			writel(CFG_STATE, &ddr_pctl_regs->sctl);
			while ((readl(&ddr_pctl_regs->stat) & PCTL_STAT_MSK)
				!= CONFIG)
				;
		case CONFIG:
			writel(GO_STATE, &ddr_pctl_regs->sctl);
			while ((readl(&ddr_pctl_regs->stat) & PCTL_STAT_MSK)
				== CONFIG)
				;
			break;
		case ACCESS:
			return;
		default:
			break;
		}
	}
}

static void dram_cfg_rbc(u32 chnum,
	const struct rk3288_sdram_params *sdram_params)
{
	struct rk3288_ddr_publ_regs *ddr_publ_regs = rk3288_ddr_publ[chnum];
	struct rk3288_msch_regs *msch_regs = rk3288_msch[chnum];

	if (sdram_params->ch[chnum].bk == 3)
		clrsetbits_le32(&ddr_publ_regs->dcr, PDQ_MSK, PDQ_CFG(1));
	else
		clrbits_le32(&ddr_publ_regs->dcr, PDQ_MSK);

	writel(sdram_params->ddrconfig, &msch_regs->ddrconf);
}

static void dram_all_config(const struct rk3288_sdram_params *sdram_params)
{
	u32 sys_reg = 0;
	unsigned int channel;

	sys_reg |= SYS_REG_DDRTYPE(sdram_params->dramtype);
	sys_reg |= SYS_REG_NUM_CH(sdram_params->num_channels);
	for (channel = 0; channel < sdram_params->num_channels; channel++) {
		const struct rk3288_sdram_channel *info =
			&(sdram_params->ch[channel]);
		sys_reg |= SYS_REG_ROW_3_4(info->row_3_4, channel);
		sys_reg |= SYS_REG_CHINFO(channel);
		sys_reg |= SYS_REG_RANK(info->rank, channel);
		sys_reg |= SYS_REG_COL(info->col, channel);
		sys_reg |= SYS_REG_BK(info->bk, channel);
		sys_reg |= SYS_REG_CS0_ROW(info->cs0_row, channel);
		sys_reg |= SYS_REG_CS1_ROW(info->cs1_row, channel);
		sys_reg |= SYS_REG_BW(info->bw, channel);
		sys_reg |= SYS_REG_DBW(info->dbw, channel);

		dram_cfg_rbc(channel, sdram_params);
	}
	writel(sys_reg, &rk3288_pmu->sys_reg[2]);
	writel(RK_CLRSETBITS(0x1F, sdram_params->stride),
		&rk3288_sgrf->soc_con2);
}

void sdram_init(const struct rk3288_sdram_params *sdram_params)
{
	int channel;
	int zqcr;
	printk(BIOS_INFO, "Starting SDRAM initialization...\n");

	if (sdram_params->ddr_freq > 533000000)
		die("SDRAM frequency is to high!");

	rkclk_configure_ddr(sdram_params->ddr_freq);

	for (channel = 0; channel < sdram_params->num_channels; channel++) {
		struct rk3288_ddr_pctl_regs *ddr_pctl_regs =
		    rk3288_ddr_pctl[channel];
		struct rk3288_ddr_publ_regs *ddr_publ_regs =
		    rk3288_ddr_publ[channel];

		phy_pctrl_reset(ddr_publ_regs, channel);
		phy_dll_bypass_set(ddr_publ_regs, sdram_params->ddr_freq);

		dfi_cfg(ddr_pctl_regs, sdram_params->dramtype);

		pctl_cfg(channel, sdram_params);

		phy_cfg(channel, sdram_params);

		phy_init(ddr_publ_regs);

		writel(POWER_UP_START, &ddr_pctl_regs->powctl);
		while (!(readl(&ddr_pctl_regs->powstat) & POWER_UP_DONE))
			;
		send_command(ddr_pctl_regs, 3, DESELECT_CMD, 0);
		udelay(1);
		send_command(ddr_pctl_regs, 3, PREA_CMD, 0);

		memory_init(ddr_publ_regs, sdram_params->dramtype);
		move_to_config_state(ddr_publ_regs, ddr_pctl_regs);
		set_bandwidth_ratio(channel, sdram_params->ch[channel].bw);
		/*
		 * set cs
		 * CS0, n=1
		 * CS1, n=2
		 * CS0 & CS1, n = 3
		 */
		clrsetbits_le32(&ddr_publ_regs->pgcr, 0xF << 18,
				(sdram_params->ch[channel].rank | 1) << 18);
		/* DS=40ohm,ODT=155ohm */
		zqcr = ZDEN(1) | PU_ONDIE(0x2) | PD_ONDIE(0x2)
				| PU_OUTPUT(0x19) | PD_OUTPUT(0x19);
		writel(zqcr, &ddr_publ_regs->zq1cr[0]);
		writel(zqcr, &ddr_publ_regs->zq0cr[0]);

		if (sdram_params->dramtype == LPDDR3) {
			/* LPDDR2/LPDDR3 need to wait DAI complete, max 10us */
			udelay(10);
			if (channel == 0) {
				writel(0, &ddr_pctl_regs->mrrcfg0);
				send_command(ddr_pctl_regs, 1, MRR_CMD,
					LPDDR2_MA(0x8));
				/* S8 */
				if ((readl(&ddr_pctl_regs->mrrstat0) & 0x3)
					!= 3)
					die("SDRAM initialization failed!");
			}
		}

		if (-1 == data_training(channel, sdram_params)) {
			if (sdram_params->dramtype == LPDDR3) {
				rkclk_ddr_phy_ctl_reset(channel, 1);
				udelay(10);
				rkclk_ddr_phy_ctl_reset(channel, 0);
				udelay(10);
			}
			die("SDRAM initialization failed!");
		}

		if (sdram_params->dramtype == LPDDR3) {
			u32 i;
			writel(0, &ddr_pctl_regs->mrrcfg0);
			for (i = 0; i < 17; i++)
				send_command(ddr_pctl_regs, 1, MRR_CMD,
					LPDDR2_MA(i));
		}
		move_to_access_state(channel);
	}
	dram_all_config(sdram_params);
	printk(BIOS_INFO, "Finish SDRAM initialization...\n");
}