Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 1 | /* |
| 2 | * This file is part of the coreboot project. |
| 3 | * |
| 4 | * Copyright 2018 MediaTek Inc. |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or modify |
| 7 | * it under the terms of the GNU General Public License as published by |
| 8 | * the Free Software Foundation; version 2 of the License. |
| 9 | * |
| 10 | * This program is distributed in the hope that it will be useful, |
| 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 13 | * GNU General Public License for more details. |
| 14 | */ |
| 15 | |
| 16 | #include <assert.h> |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 17 | #include <console/console.h> |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 18 | #include <delay.h> |
Kyösti Mälkki | 13f6650 | 2019-03-03 08:01:05 +0200 | [diff] [blame] | 19 | #include <device/mmio.h> |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 20 | #include <soc/emi.h> |
| 21 | #include <soc/dramc_register.h> |
| 22 | #include <soc/dramc_pi_api.h> |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 23 | #include <timer.h> |
| 24 | |
| 25 | enum { |
| 26 | RX_VREF_BEGIN = 0, |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 27 | RX_VREF_END = 31, |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 28 | RX_VREF_STEP = 1, |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 29 | TX_VREF_BEGIN = 0, |
| 30 | TX_VREF_END = 50, |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 31 | TX_VREF_STEP = 2, |
| 32 | }; |
| 33 | |
| 34 | enum { |
| 35 | FIRST_DQ_DELAY = 0, |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 36 | FIRST_DQS_DELAY = -48, |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 37 | MAX_DQDLY_TAPS = 16, |
| 38 | MAX_RX_DQDLY_TAPS = 63, |
| 39 | }; |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 40 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 41 | #define WRITE_LEVELING_MOVD_DQS 1 |
| 42 | #define TEST2_1_CAL 0x55000000 |
| 43 | #define TEST2_2_CAL 0xaa000400 |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 44 | |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 45 | enum CAL_TYPE { |
| 46 | RX_WIN_RD_DQC = 0, |
| 47 | RX_WIN_TEST_ENG, |
| 48 | TX_WIN_DQ_ONLY, |
| 49 | TX_WIN_DQ_DQM, |
| 50 | }; |
| 51 | |
| 52 | enum RX_TYPE { |
| 53 | RX_DQ = 0, |
| 54 | RX_DQM, |
| 55 | RX_DQS, |
| 56 | }; |
| 57 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 58 | struct win_perbit_dly { |
| 59 | s16 first_pass; |
| 60 | s16 last_pass; |
| 61 | s16 best_first; |
| 62 | s16 best_last; |
| 63 | s16 best_dqdly; |
| 64 | s16 win_center; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 65 | }; |
| 66 | |
| 67 | struct vref_perbit_dly { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 68 | u8 best_vref; |
| 69 | u16 max_win_sum; |
| 70 | struct win_perbit_dly perbit_dly[DQ_DATA_WIDTH]; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 71 | }; |
| 72 | |
| 73 | struct tx_dly_tune { |
| 74 | u8 fine_tune; |
| 75 | u8 coarse_tune_large; |
| 76 | u8 coarse_tune_small; |
| 77 | u8 coarse_tune_large_oen; |
| 78 | u8 coarse_tune_small_oen; |
| 79 | }; |
| 80 | |
| 81 | struct per_byte_dly { |
| 82 | u16 max_center; |
| 83 | u16 min_center; |
| 84 | u16 final_dly; |
| 85 | }; |
| 86 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 87 | extern u8 MR01Value[FSP_MAX]; |
| 88 | extern u8 MR13Value; |
| 89 | |
mtk11195 | 16ad2d7 | 2019-01-23 11:41:19 +0800 | [diff] [blame] | 90 | static void dramc_auto_refresh_switch(u8 chn, bool option) |
Huayang Duan | cede791 | 2018-09-26 16:10:42 +0800 | [diff] [blame] | 91 | { |
| 92 | clrsetbits_le32(&ch[chn].ao.refctrl0, 1 << REFCTRL0_REFDIS_SHIFT, |
| 93 | (option ? 0 : 1) << REFCTRL0_REFDIS_SHIFT); |
| 94 | |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 95 | if (!option) { |
Huayang Duan | cede791 | 2018-09-26 16:10:42 +0800 | [diff] [blame] | 96 | /* |
| 97 | * Because HW will actually disable autorefresh after |
| 98 | * refresh_queue empty, we need to wait until queue empty. |
| 99 | */ |
| 100 | udelay(((read32(&ch[chn].nao.misc_statusa) & |
| 101 | MISC_STATUSA_REFRESH_QUEUE_CNT_MASK) >> |
| 102 | MISC_STATUSA_REFRESH_QUEUE_CNT_SHIFT) * 4); |
| 103 | } |
| 104 | } |
| 105 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 106 | void dramc_cke_fix_onoff(u8 chn, bool cke_on, bool cke_off) |
Huayang Duan | cede791 | 2018-09-26 16:10:42 +0800 | [diff] [blame] | 107 | { |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 108 | clrsetbits_le32(&ch[chn].ao.ckectrl, (0x1 << 6) | (0x1 << 7), |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 109 | ((cke_on ? 1 : 0) << 6) | ((cke_off ? 1 : 0) << 7)); |
Huayang Duan | cede791 | 2018-09-26 16:10:42 +0800 | [diff] [blame] | 110 | } |
| 111 | |
Huayang Duan | 7378015 | 2019-08-19 14:06:31 +0800 | [diff] [blame] | 112 | void dramc_mode_reg_write(u8 chn, u8 mr_idx, u8 value) |
Huayang Duan | cede791 | 2018-09-26 16:10:42 +0800 | [diff] [blame] | 113 | { |
| 114 | u32 ckectrl_bak = read32(&ch[chn].ao.ckectrl); |
| 115 | |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 116 | dramc_cke_fix_onoff(chn, true, false); |
Huayang Duan | cede791 | 2018-09-26 16:10:42 +0800 | [diff] [blame] | 117 | clrsetbits_le32(&ch[chn].ao.mrs, |
| 118 | MRS_MRSMA_MASK, mr_idx << MRS_MRSMA_SHIFT); |
| 119 | clrsetbits_le32(&ch[chn].ao.mrs, |
| 120 | MRS_MRSOP_MASK, value << MRS_MRSOP_SHIFT); |
| 121 | setbits_le32(&ch[chn].ao.spcmd, 1 << SPCMD_MRWEN_SHIFT); |
| 122 | |
| 123 | /* Wait MRW command fired */ |
| 124 | while ((read32(&ch[chn].nao.spcmdresp) & 1) == 0) |
| 125 | ; |
| 126 | |
| 127 | clrbits_le32(&ch[chn].ao.spcmd, 1 << SPCMD_MRWEN_SHIFT); |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 128 | write32(&ch[chn].ao.ckectrl, ckectrl_bak); |
| 129 | dramc_dbg("Write MR%d =0x%x\n", mr_idx, value); |
Huayang Duan | cede791 | 2018-09-26 16:10:42 +0800 | [diff] [blame] | 130 | } |
| 131 | |
| 132 | static void dramc_mode_reg_write_by_rank(u8 chn, u8 rank, |
| 133 | u8 mr_idx, u8 value) |
| 134 | { |
| 135 | u32 mrs_back = read32(&ch[chn].ao.mrs) & MRS_MRSRK_MASK; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 136 | dramc_dbg("Mode reg write rank%d MR%d = 0x%x\n", rank, mr_idx, value); |
Huayang Duan | cede791 | 2018-09-26 16:10:42 +0800 | [diff] [blame] | 137 | |
| 138 | clrsetbits_le32(&ch[chn].ao.mrs, |
| 139 | MRS_MRSRK_MASK, rank << MRS_MRSRK_SHIFT); |
| 140 | dramc_mode_reg_write(chn, mr_idx, value); |
| 141 | clrsetbits_le32(&ch[chn].ao.mrs, MRS_MRSRK_MASK, mrs_back); |
| 142 | } |
| 143 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 144 | static void move_dramc_delay(u32 *reg_0, u32 *reg_1, u8 shift, s8 shift_coarse_tune) |
| 145 | { |
| 146 | s32 sum; |
| 147 | u32 tmp_0p5t, tmp_2t; |
| 148 | |
| 149 | tmp_0p5t = ((read32(reg_0) >> shift) & DQ_DIV_MASK) & |
| 150 | ~(1 << DQ_DIV_SHIFT); |
| 151 | tmp_2t = (read32(reg_1) >> shift) & DQ_DIV_MASK; |
| 152 | |
| 153 | sum = (tmp_2t << DQ_DIV_SHIFT) + tmp_0p5t + shift_coarse_tune; |
| 154 | |
| 155 | if (sum < 0) { |
| 156 | tmp_0p5t = 0; |
| 157 | tmp_2t = 0; |
| 158 | } else { |
| 159 | tmp_2t = sum >> DQ_DIV_SHIFT; |
| 160 | tmp_0p5t = sum - (tmp_2t << DQ_DIV_SHIFT); |
| 161 | } |
| 162 | |
| 163 | clrsetbits_le32(reg_0, DQ_DIV_MASK << shift, tmp_0p5t << shift); |
| 164 | clrsetbits_le32(reg_1, DQ_DIV_MASK << shift, tmp_2t << shift); |
| 165 | } |
| 166 | |
| 167 | static void move_dramc_tx_dqs(u8 chn, u8 byte, s8 shift_coarse_tune) |
| 168 | { |
| 169 | move_dramc_delay(&ch[chn].ao.shu[0].selph_dqs1, |
| 170 | &ch[chn].ao.shu[0].selph_dqs0, byte * 4, shift_coarse_tune); |
| 171 | } |
| 172 | |
| 173 | static void move_dramc_tx_dqs_oen(u8 chn, u8 byte, |
| 174 | s8 shift_coarse_tune) |
| 175 | { |
| 176 | move_dramc_delay(&ch[chn].ao.shu[0].selph_dqs1, |
| 177 | &ch[chn].ao.shu[0].selph_dqs0, byte * 4 + OEN_SHIFT, shift_coarse_tune); |
| 178 | } |
| 179 | |
| 180 | static void move_dramc_tx_dq(u8 chn, u8 rank, u8 byte, s8 shift_coarse_tune) |
| 181 | { |
| 182 | /* DQM0 */ |
| 183 | move_dramc_delay(&ch[chn].ao.shu[0].rk[rank].selph_dq[3], |
| 184 | &ch[chn].ao.shu[0].rk[rank].selph_dq[1], byte * 4, shift_coarse_tune); |
| 185 | |
| 186 | /* DQ0 */ |
| 187 | move_dramc_delay(&ch[chn].ao.shu[0].rk[rank].selph_dq[2], |
| 188 | &ch[chn].ao.shu[0].rk[rank].selph_dq[0], byte * 4, shift_coarse_tune); |
| 189 | } |
| 190 | |
| 191 | static void move_dramc_tx_dq_oen(u8 chn, u8 rank, |
| 192 | u8 byte, s8 shift_coarse_tune) |
| 193 | { |
| 194 | /* DQM_OEN_0 */ |
| 195 | move_dramc_delay(&ch[chn].ao.shu[0].rk[rank].selph_dq[3], |
| 196 | &ch[chn].ao.shu[0].rk[rank].selph_dq[1], |
| 197 | byte * 4 + OEN_SHIFT, shift_coarse_tune); |
| 198 | |
| 199 | /* DQ_OEN_0 */ |
| 200 | move_dramc_delay(&ch[chn].ao.shu[0].rk[rank].selph_dq[2], |
| 201 | &ch[chn].ao.shu[0].rk[rank].selph_dq[0], |
| 202 | byte * 4 + OEN_SHIFT, shift_coarse_tune); |
| 203 | } |
| 204 | |
| 205 | static void write_leveling_move_dqs_instead_of_clk(u8 chn) |
| 206 | { |
| 207 | dramc_dbg("%s do ch:%d k\n", __func__, chn); |
| 208 | for (u8 byte = 0; byte < DQS_NUMBER; byte++) { |
| 209 | move_dramc_tx_dqs(chn, byte, -WRITE_LEVELING_MOVD_DQS); |
| 210 | move_dramc_tx_dqs_oen(chn, byte, -WRITE_LEVELING_MOVD_DQS); |
| 211 | |
| 212 | for (u8 rk = RANK_0; rk < RANK_MAX; rk++) { |
| 213 | move_dramc_tx_dq(chn, rk, byte, -WRITE_LEVELING_MOVD_DQS); |
| 214 | move_dramc_tx_dq_oen(chn, rk, byte, -WRITE_LEVELING_MOVD_DQS); |
| 215 | } |
| 216 | } |
| 217 | } |
| 218 | |
| 219 | static void dramc_write_leveling(u8 chn, u8 rank, u8 freq_group, |
Huayang Duan | 6bb72e4 | 2018-09-26 16:16:38 +0800 | [diff] [blame] | 220 | const u8 wr_level[CHANNEL_MAX][RANK_MAX][DQS_NUMBER]) |
| 221 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 222 | dramc_auto_refresh_switch(chn, false); |
| 223 | |
| 224 | if (rank == RANK_0 && (freq_group == LP4X_DDR3600 || |
| 225 | freq_group == LP4X_DDR1600 || |
| 226 | freq_group == LP4X_DDR2400)) |
| 227 | write_leveling_move_dqs_instead_of_clk(chn); |
| 228 | |
Huayang Duan | 6bb72e4 | 2018-09-26 16:16:38 +0800 | [diff] [blame] | 229 | clrsetbits_le32(&ch[chn].phy.shu[0].rk[rank].ca_cmd[9], |
| 230 | SHU1_CA_CMD9_RG_RK_ARFINE_TUNE_CLK_MASK, 0); |
| 231 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 232 | for (size_t byte = 0; byte < DQS_NUMBER; byte++) { |
| 233 | u32 wrlevel_dq_delay = wr_level[chn][rank][byte] + 0x10; |
| 234 | clrsetbits_le32(&ch[chn].phy.shu[0].rk[rank].b[byte].dq[7], |
| 235 | FINE_TUNE_PBYTE_MASK, |
| 236 | wr_level[chn][rank][byte] << FINE_TUNE_PBYTE_SHIFT); |
| 237 | if (wrlevel_dq_delay >= 0x40) { |
| 238 | wrlevel_dq_delay -= 0x40; |
| 239 | move_dramc_tx_dq(chn, rank, byte, 2); |
| 240 | move_dramc_tx_dq_oen(chn, rank, byte, 2); |
| 241 | } |
Huayang Duan | 6bb72e4 | 2018-09-26 16:16:38 +0800 | [diff] [blame] | 242 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 243 | clrsetbits_le32(&ch[chn].phy.shu[0].rk[rank].b[byte].dq[7], |
| 244 | FINE_TUNE_DQM_MASK | FINE_TUNE_DQ_MASK, |
Huayang Duan | 6bb72e4 | 2018-09-26 16:16:38 +0800 | [diff] [blame] | 245 | (wrlevel_dq_delay << FINE_TUNE_DQM_SHIFT) | |
| 246 | (wrlevel_dq_delay << FINE_TUNE_DQ_SHIFT)); |
| 247 | } |
| 248 | } |
| 249 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 250 | static void dramc_cmd_bus_training(u8 chn, u8 rank, u8 freq_group, |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 251 | const struct sdram_params *params, const bool fast_calib) |
Huayang Duan | cede791 | 2018-09-26 16:10:42 +0800 | [diff] [blame] | 252 | { |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 253 | u32 final_vref, clk_dly, cmd_dly, cs_dly; |
Huayang Duan | cede791 | 2018-09-26 16:10:42 +0800 | [diff] [blame] | 254 | |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 255 | clk_dly = params->cbt_clk_dly[chn][rank]; |
| 256 | cmd_dly = params->cbt_cmd_dly[chn][rank]; |
| 257 | cs_dly = params->cbt_cs_dly[chn][rank]; |
| 258 | final_vref = params->cbt_final_vref[chn][rank]; |
Huayang Duan | cede791 | 2018-09-26 16:10:42 +0800 | [diff] [blame] | 259 | |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 260 | if (fast_calib) { |
| 261 | /* Set CLK and CA delay */ |
| 262 | clrsetbits_le32(&ch[chn].phy.shu[0].rk[rank].ca_cmd[9], |
| 263 | (0x3f << 8) | (0x3f << 24), |
| 264 | (cmd_dly << 8) | (clk_dly << 24)); |
| 265 | udelay(1); |
| 266 | } |
| 267 | |
| 268 | /* Set CLK and CS delay */ |
| 269 | clrsetbits_le32(&ch[chn].phy.shu[0].rk[rank].ca_cmd[9], 0x3f, cs_dly << 0); |
Huayang Duan | cede791 | 2018-09-26 16:10:42 +0800 | [diff] [blame] | 270 | |
| 271 | /* CBT set vref */ |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 272 | dramc_mode_reg_write_by_rank(chn, rank, 12, final_vref); |
Huayang Duan | cede791 | 2018-09-26 16:10:42 +0800 | [diff] [blame] | 273 | } |
| 274 | |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 275 | static void dramc_read_dbi_onoff(bool on) |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 276 | { |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 277 | for (size_t chn = 0; chn < CHANNEL_MAX; chn++) |
| 278 | for (size_t b = 0; b < 2; b++) |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 279 | clrsetbits_le32(&ch[chn].phy.shu[0].b[b].dq[7], |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 280 | 0x1 << SHU1_BX_DQ7_R_DMDQMDBI_SHIFT, |
| 281 | (on ? 1 : 0) << SHU1_BX_DQ7_R_DMDQMDBI_SHIFT); |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 282 | } |
| 283 | |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 284 | static void dramc_write_dbi_onoff(bool on) |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 285 | { |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 286 | for (size_t chn = 0; chn < CHANNEL_MAX; chn++) |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 287 | clrsetbits_le32(&ch[chn].ao.shu[0].wodt, |
| 288 | 0x1 << SHU1_WODT_DBIWR_SHIFT, |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 289 | (on ? 1 : 0) << SHU1_WODT_DBIWR_SHIFT); |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 290 | } |
| 291 | |
Huayang Duan | 7b78a80 | 2018-09-26 21:09:54 +0800 | [diff] [blame] | 292 | static void dramc_phy_dcm_2_channel(u8 chn, bool en) |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 293 | { |
Huayang Duan | 7b78a80 | 2018-09-26 21:09:54 +0800 | [diff] [blame] | 294 | clrsetbits_le32(&ch[chn].phy.misc_cg_ctrl0, (0x3 << 19) | (0x3ff << 8), |
| 295 | ((en ? 0 : 0x1) << 19) | ((en ? 0 : 0x1ff) << 9) | (1 << 8)); |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 296 | |
| 297 | for (size_t i = 0; i < DRAM_DFS_SHUFFLE_MAX; i++) { |
| 298 | struct ddrphy_ao_shu *shu = &ch[chn].phy.shu[i]; |
Huayang Duan | 7b78a80 | 2018-09-26 21:09:54 +0800 | [diff] [blame] | 299 | for (size_t b = 0; b < 2; b++) |
| 300 | clrsetbits_le32(&shu->b[b].dq[8], 0x1fff << 19, |
| 301 | ((en ? 0 : 0x7ff) << 22) | (0x1 << 21) | |
| 302 | ((en ? 0 : 0x3) << 19)); |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 303 | clrbits_le32(&shu->ca_cmd[8], 0x1fff << 19); |
| 304 | } |
Huayang Duan | 7b78a80 | 2018-09-26 21:09:54 +0800 | [diff] [blame] | 305 | clrsetbits_le32(&ch[chn].phy.misc_cg_ctrl5, (0x7 << 16) | (0x7 << 20), |
| 306 | ((en ? 0x7 : 0) << 16) | ((en ? 0x7 : 0) << 20)); |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 307 | } |
| 308 | |
Huayang Duan | 7b78a80 | 2018-09-26 21:09:54 +0800 | [diff] [blame] | 309 | void dramc_enable_phy_dcm(bool en) |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 310 | { |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 311 | for (size_t chn = 0; chn < CHANNEL_MAX ; chn++) { |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 312 | clrbits_le32(&ch[chn].phy.b[0].dll_fine_tune[1], 0x1 << 20); |
| 313 | clrbits_le32(&ch[chn].phy.b[1].dll_fine_tune[1], 0x1 << 20); |
| 314 | clrbits_le32(&ch[chn].phy.ca_dll_fine_tune[1], 0x1 << 20); |
| 315 | |
| 316 | for (size_t i = 0; i < DRAM_DFS_SHUFFLE_MAX; i++) { |
| 317 | struct ddrphy_ao_shu *shu = &ch[chn].phy.shu[i]; |
| 318 | setbits_le32(&shu->b[0].dll[0], 0x1); |
| 319 | setbits_le32(&shu->b[1].dll[0], 0x1); |
| 320 | setbits_le32(&shu->ca_dll[0], 0x1); |
| 321 | } |
| 322 | |
| 323 | clrsetbits_le32(&ch[chn].ao.dramc_pd_ctrl, |
| 324 | (0x1 << 0) | (0x1 << 1) | (0x1 << 2) | |
| 325 | (0x1 << 5) | (0x1 << 26) | (0x1 << 30) | (0x1 << 31), |
| 326 | ((en ? 0x1 : 0) << 0) | ((en ? 0x1 : 0) << 1) | |
| 327 | ((en ? 0x1 : 0) << 2) | ((en ? 0 : 0x1) << 5) | |
| 328 | ((en ? 0 : 0x1) << 26) | ((en ? 0x1 : 0) << 30) | |
| 329 | ((en ? 0x1 : 0) << 31)); |
| 330 | |
| 331 | /* DCM on: CHANNEL_EMI free run; DCM off: mem_dcm */ |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 332 | write32(&ch[chn].phy.misc_cg_ctrl2, |
| 333 | 0x8060033e | (0x40 << (en ? 0x1 : 0))); |
| 334 | write32(&ch[chn].phy.misc_cg_ctrl2, |
| 335 | 0x8060033f | (0x40 << (en ? 0x1 : 0))); |
| 336 | write32(&ch[chn].phy.misc_cg_ctrl2, |
| 337 | 0x8060033e | (0x40 << (en ? 0x1 : 0))); |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 338 | |
| 339 | clrsetbits_le32(&ch[chn].phy.misc_ctrl3, 0x3 << 26, |
| 340 | (en ? 0 : 0x3) << 26); |
| 341 | for (size_t i = 0; i < DRAM_DFS_SHUFFLE_MAX; i++) { |
| 342 | u32 mask = 0x7 << 17; |
| 343 | u32 value = (en ? 0x7 : 0) << 17; |
| 344 | struct ddrphy_ao_shu *shu = &ch[chn].phy.shu[i]; |
| 345 | |
| 346 | clrsetbits_le32(&shu->b[0].dq[7], mask, value); |
| 347 | clrsetbits_le32(&shu->b[1].dq[7], mask, value); |
| 348 | clrsetbits_le32(&shu->ca_cmd[7], mask, value); |
| 349 | } |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 350 | |
Huayang Duan | 7b78a80 | 2018-09-26 21:09:54 +0800 | [diff] [blame] | 351 | dramc_phy_dcm_2_channel(chn, en); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 352 | } |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 353 | } |
| 354 | |
mtk11195 | 16ad2d7 | 2019-01-23 11:41:19 +0800 | [diff] [blame] | 355 | static void dramc_reset_delay_chain_before_calibration(void) |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 356 | { |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 357 | for (size_t chn = 0; chn < CHANNEL_MAX; chn++) |
| 358 | for (size_t rank = 0; rank < RANK_MAX; rank++) { |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 359 | struct dramc_ddrphy_regs_shu_rk *rk; |
| 360 | rk = &ch[chn].phy.shu[0].rk[rank]; |
| 361 | clrbits_le32(&rk->ca_cmd[0], 0xffffff << 0); |
| 362 | clrbits_le32(&rk->b[0].dq[0], 0xfffffff << 0); |
| 363 | clrbits_le32(&rk->b[1].dq[0], 0xfffffff << 0); |
| 364 | clrbits_le32(&rk->b[0].dq[1], 0xf << 0); |
| 365 | clrbits_le32(&rk->b[1].dq[1], 0xf << 0); |
| 366 | } |
| 367 | } |
| 368 | |
Huayang Duan | 7b78a80 | 2018-09-26 21:09:54 +0800 | [diff] [blame] | 369 | void dramc_hw_gating_onoff(u8 chn, bool on) |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 370 | { |
| 371 | clrsetbits_le32(&ch[chn].ao.shuctrl2, 0x3 << 14, |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 372 | (on ? 0x3 : 0) << 14); |
| 373 | clrsetbits_le32(&ch[chn].ao.stbcal2, 0x1 << 28, (on ? 0x1 : 0) << 28); |
| 374 | clrsetbits_le32(&ch[chn].ao.stbcal, 0x1 << 24, (on ? 0x1 : 0) << 24); |
| 375 | clrsetbits_le32(&ch[chn].ao.stbcal, 0x1 << 22, (on ? 0x1 : 0) << 22); |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 376 | } |
| 377 | |
| 378 | static void dramc_rx_input_delay_tracking_init_by_freq(u8 chn) |
| 379 | { |
| 380 | struct ddrphy_ao_shu *shu = &ch[chn].phy.shu[0]; |
| 381 | |
| 382 | clrsetbits_le32(&shu->b[0].dq[5], 0x7 << 20, 0x3 << 20); |
| 383 | clrsetbits_le32(&shu->b[1].dq[5], 0x7 << 20, 0x3 << 20); |
| 384 | clrbits_le32(&shu->b[0].dq[7], (0x1 << 12) | (0x1 << 13)); |
| 385 | clrbits_le32(&shu->b[1].dq[7], (0x1 << 12) | (0x1 << 13)); |
| 386 | } |
| 387 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 388 | void dramc_apply_config_before_calibration(u8 freq_group) |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 389 | { |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 390 | dramc_enable_phy_dcm(false); |
mtk11195 | 16ad2d7 | 2019-01-23 11:41:19 +0800 | [diff] [blame] | 391 | dramc_reset_delay_chain_before_calibration(); |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 392 | |
| 393 | setbits_le32(&ch[0].ao.shu[0].conf[3], 0x1ff << 16); |
| 394 | setbits_le32(&ch[0].ao.spcmdctrl, 0x1 << 24); |
| 395 | clrsetbits_le32(&ch[0].ao.shu[0].scintv, 0x1f << 1, 0x1b << 1); |
| 396 | |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 397 | for (size_t shu = 0; shu < DRAM_DFS_SHUFFLE_MAX; shu++) |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 398 | setbits_le32(&ch[0].ao.shu[shu].conf[3], 0x1ff << 0); |
| 399 | |
| 400 | clrbits_le32(&ch[0].ao.dramctrl, 0x1 << 18); |
| 401 | clrbits_le32(&ch[0].ao.spcmdctrl, 0x1 << 31); |
| 402 | clrbits_le32(&ch[0].ao.spcmdctrl, 0x1 << 30); |
| 403 | clrbits_le32(&ch[0].ao.dqsoscr, 0x1 << 26); |
| 404 | clrbits_le32(&ch[0].ao.dqsoscr, 0x1 << 25); |
| 405 | |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 406 | dramc_write_dbi_onoff(false); |
| 407 | dramc_read_dbi_onoff(false); |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 408 | |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 409 | for (size_t chn = 0; chn < CHANNEL_MAX; chn++) { |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 410 | setbits_le32(&ch[chn].ao.spcmdctrl, 0x1 << 29); |
| 411 | setbits_le32(&ch[chn].ao.dqsoscr, 0x1 << 24); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 412 | for (size_t shu = 0; shu < DRAM_DFS_SHUFFLE_MAX; shu++) |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 413 | setbits_le32(&ch[chn].ao.shu[shu].scintv, 0x1 << 30); |
| 414 | |
| 415 | clrbits_le32(&ch[chn].ao.dummy_rd, (0x1 << 7) | (0x7 << 20)); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 416 | dramc_hw_gating_onoff(chn, false); |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 417 | clrbits_le32(&ch[chn].ao.stbcal2, 0x1 << 28); |
| 418 | |
| 419 | setbits_le32(&ch[chn].phy.misc_ctrl1, |
| 420 | (0x1 << 7) | (0x1 << 11)); |
| 421 | clrbits_le32(&ch[chn].ao.refctrl0, 0x1 << 18); |
| 422 | clrbits_le32(&ch[chn].ao.mrs, 0x3 << 24); |
| 423 | setbits_le32(&ch[chn].ao.mpc_option, 0x1 << 17); |
| 424 | clrsetbits_le32(&ch[chn].phy.b[0].dq[6], 0x3 << 0, 0x1 << 0); |
| 425 | clrsetbits_le32(&ch[chn].phy.b[1].dq[6], 0x3 << 0, 0x1 << 0); |
| 426 | clrsetbits_le32(&ch[chn].phy.ca_cmd[6], 0x3 << 0, 0x1 << 0); |
| 427 | setbits_le32(&ch[chn].ao.dummy_rd, 0x1 << 25); |
| 428 | setbits_le32(&ch[chn].ao.drsctrl, 0x1 << 0); |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 429 | if (freq_group == LP4X_DDR3200 || freq_group == LP4X_DDR3600) |
| 430 | clrbits_le32(&ch[chn].ao.shu[1].drving[1], 0x1 << 31); |
| 431 | else |
| 432 | setbits_le32(&ch[chn].ao.shu[1].drving[1], 0x1 << 31); |
Huayang Duan | dac7f53 | 2018-09-26 15:53:23 +0800 | [diff] [blame] | 433 | |
| 434 | dramc_rx_input_delay_tracking_init_by_freq(chn); |
| 435 | } |
| 436 | |
| 437 | for (size_t r = 0; r < 2; r++) { |
| 438 | for (size_t b = 0; b < 2; b++) |
| 439 | clrbits_le32(&ch[0].phy.r[r].b[b].rxdvs[2], |
| 440 | (0x1 << 28) | (0x1 << 23) | (0x3 << 30)); |
| 441 | clrbits_le32(&ch[0].phy.r0_ca_rxdvs[2], 0x3 << 30); |
| 442 | } |
| 443 | } |
Huayang Duan | cede791 | 2018-09-26 16:10:42 +0800 | [diff] [blame] | 444 | |
mtk11195 | 16ad2d7 | 2019-01-23 11:41:19 +0800 | [diff] [blame] | 445 | static void dramc_set_mr13_vrcg_to_Normal(u8 chn) |
| 446 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 447 | MR13Value &= ~(0x1 << 3); |
mtk11195 | 16ad2d7 | 2019-01-23 11:41:19 +0800 | [diff] [blame] | 448 | for (u8 rank = 0; rank < RANK_MAX; rank++) |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 449 | dramc_mode_reg_write_by_rank(chn, rank, 13, MR13Value); |
mtk11195 | 16ad2d7 | 2019-01-23 11:41:19 +0800 | [diff] [blame] | 450 | |
| 451 | for (u8 shu = 0; shu < DRAM_DFS_SHUFFLE_MAX; shu++) |
| 452 | clrbits_le32(&ch[chn].ao.shu[shu].hwset_vrcg, 0x1 << 19); |
| 453 | } |
| 454 | |
| 455 | void dramc_apply_config_after_calibration(void) |
| 456 | { |
| 457 | for (size_t chn = 0; chn < CHANNEL_MAX; chn++) { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 458 | write32(&ch[chn].phy.misc_cg_ctrl4, 0x11400000); |
mtk11195 | 16ad2d7 | 2019-01-23 11:41:19 +0800 | [diff] [blame] | 459 | clrbits_le32(&ch[chn].ao.refctrl1, 0x1 << 7); |
| 460 | clrbits_le32(&ch[chn].ao.shuctrl, 0x1 << 2); |
| 461 | clrbits_le32(&ch[chn].phy.ca_cmd[6], 0x1 << 6); |
| 462 | dramc_set_mr13_vrcg_to_Normal(chn); |
| 463 | |
| 464 | clrbits_le32(&ch[chn].phy.b[0].dq[6], 0x3); |
| 465 | clrbits_le32(&ch[chn].phy.b[1].dq[6], 0x3); |
| 466 | clrbits_le32(&ch[chn].phy.ca_cmd[6], 0x3); |
| 467 | setbits_le32(&ch[chn].phy.b[0].dq[6], 0x1 << 5); |
| 468 | setbits_le32(&ch[chn].phy.b[1].dq[6], 0x1 << 5); |
| 469 | setbits_le32(&ch[chn].phy.ca_cmd[6], 0x1 << 5); |
| 470 | |
| 471 | clrbits_le32(&ch[chn].ao.impcal, 0x3 << 24); |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 472 | clrbits_le32(&ch[chn].phy.misc_imp_ctrl0, 0x4); |
| 473 | clrbits_le32(&ch[chn].phy.misc_cg_ctrl0, 0xf); |
mtk11195 | 16ad2d7 | 2019-01-23 11:41:19 +0800 | [diff] [blame] | 474 | |
| 475 | clrbits_le32(&ch[chn].phy.misc_ctrl0, 0x1 << 31); |
| 476 | clrbits_le32(&ch[chn].phy.misc_ctrl1, 0x1 << 25); |
| 477 | |
| 478 | setbits_le32(&ch[chn].ao.spcmdctrl, 1 << 29); |
| 479 | setbits_le32(&ch[chn].ao.dqsoscr, 1 << 24); |
| 480 | |
| 481 | for (u8 shu = 0; shu < DRAM_DFS_SHUFFLE_MAX; shu++) |
| 482 | clrbits_le32(&ch[chn].ao.shu[shu].scintv, 0x1 << 30); |
| 483 | |
| 484 | clrbits_le32(&ch[chn].ao.dummy_rd, (0x7 << 20) | (0x1 << 7)); |
| 485 | dramc_cke_fix_onoff(chn, false, false); |
| 486 | clrbits_le32(&ch[chn].ao.dramc_pd_ctrl, 0x1 << 26); |
| 487 | |
| 488 | clrbits_le32(&ch[chn].ao.eyescan, 0x7 << 8); |
| 489 | clrsetbits_le32(&ch[chn].ao.test2_4, 0x7 << 28, 0x4 << 28); |
| 490 | } |
| 491 | } |
| 492 | |
| 493 | static void dramc_rx_dqs_isi_pulse_cg_switch(u8 chn, bool flag) |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 494 | { |
| 495 | for (size_t b = 0; b < 2; b++) |
| 496 | clrsetbits_le32(&ch[chn].phy.b[b].dq[6], 1 << 5, |
| 497 | (flag ? 1 : 0) << 5); |
| 498 | } |
| 499 | |
| 500 | static void dramc_set_rank_engine2(u8 chn, u8 rank) |
| 501 | { |
| 502 | setbits_le32(&ch[chn].ao.dramctrl, 0x1 << 1); |
| 503 | clrbits_le32(&ch[chn].ao.test2_4, TEST2_4_TESTAGENTRKSEL_MASK); |
| 504 | clrsetbits_le32(&ch[chn].ao.test2_4, TEST2_4_TESTAGENTRK_MASK, |
| 505 | rank << TEST2_4_TESTAGENTRK_SHIFT); |
| 506 | } |
| 507 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 508 | static void dramc_engine2_setpat(u8 chn, bool test_pat) |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 509 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 510 | clrbits_le32(&ch[chn].ao.test2_4, |
| 511 | (0x1 << TEST2_4_TEST_REQ_LEN1_SHIFT) | |
| 512 | (0x1 << TEST2_4_TESTXTALKPAT_SHIFT) | |
| 513 | (0x1 << TEST2_4_TESTAUDMODE_SHIFT) | |
| 514 | (0x1 << TEST2_4_TESTAUDBITINV_SHIFT)); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 515 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 516 | if (!test_pat) { |
| 517 | setbits_le32(&ch[chn].ao.perfctl0, 1 << PERFCTL0_RWOFOEN_SHIFT); |
| 518 | |
| 519 | clrsetbits_le32(&ch[chn].ao.test2_4, |
| 520 | (0x1 << TEST2_4_TESTSSOPAT_SHIFT) | |
| 521 | (0x1 << TEST2_4_TESTSSOXTALKPAT_SHIFT), |
| 522 | (0x1 << TEST2_4_TESTXTALKPAT_SHIFT)); |
| 523 | } else { |
| 524 | clrsetbits_le32(&ch[chn].ao.test2_4, |
| 525 | TEST2_4_TESTAUDINIT_MASK | TEST2_4_TESTAUDINC_MASK, |
| 526 | (0x11 << 8) | (0xd << 0) | (0x1 << 14)); |
| 527 | } |
| 528 | clrsetbits_le32(&ch[chn].ao.test2_3, |
| 529 | (0x1 << TEST2_3_TESTAUDPAT_SHIFT) | TEST2_3_TESTCNT_MASK, |
| 530 | (test_pat ? 1 : 0) << TEST2_3_TESTAUDPAT_SHIFT); |
| 531 | } |
| 532 | |
| 533 | static void dramc_engine2_init(u8 chn, u8 rank, u32 t2_1, u32 t2_2, bool test_pat) |
| 534 | { |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 535 | dramc_set_rank_engine2(chn, rank); |
| 536 | |
| 537 | clrbits_le32(&ch[chn].ao.dummy_rd, |
| 538 | (0x1 << DUMMY_RD_DQSG_DMYRD_EN_SHIFT) | |
| 539 | (0x1 << DUMMY_RD_DQSG_DMYWR_EN_SHIFT) | |
| 540 | (0x1 << DUMMY_RD_DUMMY_RD_EN_SHIFT) | |
| 541 | (0x1 << DUMMY_RD_SREF_DMYRD_EN_SHIFT) | |
| 542 | (0x1 << DUMMY_RD_DMY_RD_DBG_SHIFT) | |
| 543 | (0x1 << DUMMY_RD_DMY_WR_DBG_SHIFT)); |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 544 | clrbits_le32(&ch[chn].nao.testchip_dma1, 0x1 << 12); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 545 | clrbits_le32(&ch[chn].ao.test2_3, |
| 546 | (0x1 << TEST2_3_TEST2W_SHIFT) | |
| 547 | (0x1 << TEST2_3_TEST2R_SHIFT) | |
| 548 | (0x1 << TEST2_3_TEST1_SHIFT)); |
| 549 | clrsetbits_le32(&ch[chn].ao.test2_0, |
| 550 | TEST2_0_PAT0_MASK | TEST2_0_PAT1_MASK, |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 551 | ((t2_1 >> 24) << TEST2_0_PAT0_SHIFT) | |
| 552 | ((t2_2 >> 24) << TEST2_0_PAT1_SHIFT)); |
| 553 | clrsetbits_le32(&ch[chn].ao.test2_1, 0xfffffff0, (t2_1 & 0x00ffffff) << 4); |
| 554 | clrsetbits_le32(&ch[chn].ao.test2_2, 0xfffffff0, (t2_2 & 0x00ffffff) << 4); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 555 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 556 | dramc_engine2_setpat(chn, test_pat); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 557 | } |
| 558 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 559 | static void dramc_engine2_check_complete(u8 chn, u8 status) |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 560 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 561 | u32 loop = 0; |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 562 | /* In some case test engine finished but the complete signal late come, |
| 563 | * system will wait very long time. Hence, we set a timeout here. |
| 564 | * After system receive complete signal or wait until time out |
| 565 | * it will return, the caller will check compare result to verify |
| 566 | * whether engine success. |
| 567 | */ |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 568 | while (wait_us(100, read32(&ch[chn].nao.testrpt) & status) != status) { |
| 569 | if (loop++ > 100) |
| 570 | dramc_dbg("MEASURE_A timeout\n"); |
| 571 | } |
| 572 | } |
| 573 | |
| 574 | static void dramc_engine2_compare(u8 chn, enum dram_te_op wr) |
| 575 | { |
| 576 | u8 rank_status = ((read32(&ch[chn].ao.test2_3) & 0xf) == 1) ? 3 : 1; |
| 577 | |
| 578 | if (wr == TE_OP_WRITE_READ_CHECK) { |
| 579 | dramc_engine2_check_complete(chn, rank_status); |
| 580 | |
| 581 | clrbits_le32(&ch[chn].ao.test2_3, (0x1 << TEST2_3_TEST2W_SHIFT) | |
| 582 | (0x1 << TEST2_3_TEST2R_SHIFT) | (0x1 << TEST2_3_TEST1_SHIFT)); |
| 583 | udelay(1); |
| 584 | setbits_le32(&ch[chn].ao.test2_3, 0x1 << TEST2_3_TEST2W_SHIFT); |
| 585 | } |
| 586 | |
| 587 | dramc_engine2_check_complete(chn, rank_status); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 588 | } |
| 589 | |
| 590 | static u32 dramc_engine2_run(u8 chn, enum dram_te_op wr) |
| 591 | { |
| 592 | u32 result; |
| 593 | |
| 594 | if (wr == TE_OP_READ_CHECK) { |
| 595 | clrbits_le32(&ch[chn].ao.test2_4, |
| 596 | 0x1 << TEST2_4_TESTAUDMODE_SHIFT); |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 597 | |
| 598 | clrsetbits_le32(&ch[chn].ao.test2_3, |
| 599 | (0x1 << TEST2_3_TEST2W_SHIFT) | (0x1 << TEST2_3_TEST2R_SHIFT) | |
| 600 | (0x1 << TEST2_3_TEST1_SHIFT), 0x1 << TEST2_3_TEST2R_SHIFT); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 601 | } else if (wr == TE_OP_WRITE_READ_CHECK) { |
| 602 | clrsetbits_le32(&ch[chn].ao.test2_3, |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 603 | (0x1 << TEST2_3_TEST2W_SHIFT) | (0x1 << TEST2_3_TEST2R_SHIFT) | |
| 604 | (0x1 << TEST2_3_TEST1_SHIFT), 0x1 << TEST2_3_TEST2W_SHIFT); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 605 | } |
| 606 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 607 | dramc_engine2_compare(chn, wr); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 608 | |
| 609 | udelay(1); |
| 610 | result = read32(&ch[chn].nao.cmp_err); |
| 611 | clrbits_le32(&ch[chn].ao.test2_3, |
| 612 | (0x1 << TEST2_3_TEST2W_SHIFT) | |
| 613 | (0x1 << TEST2_3_TEST2R_SHIFT) | |
| 614 | (0x1 << TEST2_3_TEST1_SHIFT)); |
| 615 | |
| 616 | return result; |
| 617 | } |
| 618 | |
Yu-Ping Wu | 093d8ea | 2019-10-02 13:06:42 +0800 | [diff] [blame] | 619 | static void dramc_engine2_end(u8 chn, u32 dummy_rd) |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 620 | { |
| 621 | clrbits_le32(&ch[chn].ao.test2_4, 0x1 << 17); |
Yu-Ping Wu | 093d8ea | 2019-10-02 13:06:42 +0800 | [diff] [blame] | 622 | write32(&ch[chn].ao.dummy_rd, dummy_rd); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 623 | } |
| 624 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 625 | static bool dramc_find_gating_window(u32 result_r, u32 result_f, u32 *debug_cnt, |
| 626 | u8 dly_coarse_large, u8 dly_coarse_0p5t, u8 *pass_begin, u8 *pass_count, |
| 627 | u8 *pass_count_1, u8 *dly_fine_xt, u8 *dqs_high, u8 *dqs_done) |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 628 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 629 | bool find_tune = false; |
| 630 | u16 debug_cnt_perbyte, current_pass = 0, pass_byte_cnt = 0; |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 631 | |
| 632 | for (u8 dqs = 0; dqs < DQS_NUMBER; dqs++) { |
| 633 | u8 dqs_result_r = (u8) ((result_r >> (8 * dqs)) & 0xff); |
| 634 | u8 dqs_result_f = (u8) ((result_f >> (8 * dqs)) & 0xff); |
| 635 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 636 | if (pass_byte_cnt & (1 << dqs)) |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 637 | continue; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 638 | current_pass = 0; |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 639 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 640 | debug_cnt_perbyte = (u16) debug_cnt[dqs]; |
| 641 | if (dqs_result_r == 0 && dqs_result_f == 0 && |
| 642 | debug_cnt_perbyte == GATING_GOLDEND_DQSCNT) |
| 643 | current_pass = 1; |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 644 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 645 | if (current_pass) { |
| 646 | if (pass_begin[dqs] == 0) { |
| 647 | pass_begin[dqs] = 1; |
| 648 | pass_count_1[dqs] = 0; |
| 649 | dramc_dbg("[Byte %d]First pass (%d, %d, %d)\n", |
| 650 | dqs, dly_coarse_large, dly_coarse_0p5t, *dly_fine_xt); |
| 651 | } |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 652 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 653 | if (pass_begin[dqs] == 1) |
| 654 | pass_count_1[dqs]++; |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 655 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 656 | if (pass_begin[dqs] == 1 && |
| 657 | pass_count_1[dqs] * DQS_GW_FINE_STEP > DQS_GW_FINE_END) { |
| 658 | dqs_high[dqs] = 0; |
| 659 | dqs_done[dqs] = 1; |
| 660 | } |
| 661 | |
| 662 | if (pass_count_1[0] * DQS_GW_FINE_STEP > DQS_GW_FINE_END && |
| 663 | pass_count_1[1] * DQS_GW_FINE_STEP > DQS_GW_FINE_END) { |
| 664 | dramc_dbg("All bytes gating window > 1 coarse_tune, Early break\n"); |
| 665 | *dly_fine_xt = DQS_GW_FINE_END; |
| 666 | find_tune = true; |
| 667 | } |
| 668 | } else { |
| 669 | if (pass_begin[dqs] != 1) |
| 670 | continue; |
| 671 | |
| 672 | dramc_dbg("[Byte %d] pass_begin[dqs]:%d, pass_count[dqs]:%d,pass_count_1:%d\n", |
| 673 | dqs, pass_begin[dqs], pass_count[dqs], pass_count_1[dqs]); |
| 674 | |
| 675 | pass_begin[dqs] = 0; |
| 676 | if (pass_count_1[dqs] > pass_count[dqs]) { |
| 677 | pass_count[dqs] = pass_count_1[dqs]; |
| 678 | if (pass_count_1[dqs] * DQS_GW_FINE_STEP > 32 && |
| 679 | pass_count_1[dqs] * DQS_GW_FINE_STEP < 96) |
| 680 | pass_byte_cnt |= (1 << dqs); |
| 681 | if (pass_byte_cnt == 3) { |
| 682 | *dly_fine_xt = DQS_GW_FINE_END; |
| 683 | find_tune = true; |
| 684 | } |
| 685 | } |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 686 | } |
| 687 | } |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 688 | |
| 689 | return find_tune; |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 690 | } |
| 691 | |
mtk11195 | 16ad2d7 | 2019-01-23 11:41:19 +0800 | [diff] [blame] | 692 | static void dramc_find_dly_tune(u8 chn, u8 dly_coarse_large, u8 dly_coarse_0p5t, |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 693 | u8 dly_fine_xt, u8 *dqs_high, u8 *dly_coarse_large_cnt, |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 694 | u8 *dly_coarse_0p5t_cnt, u8 *dly_fine_tune_cnt, u8 *dqs_trans, u8 *dqs_done) |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 695 | { |
| 696 | for (size_t dqs = 0; dqs < DQS_NUMBER; dqs++) { |
| 697 | u32 dqs_cnt = read32(&ch[chn].phy_nao.misc_phy_stben_b[dqs]); |
| 698 | dqs_cnt = (dqs_cnt >> 16) & 3; |
| 699 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 700 | if (dqs_done[dqs] == 1) |
| 701 | continue; |
| 702 | |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 703 | if (dqs_cnt == 3) |
| 704 | dqs_high[dqs]++; |
| 705 | |
| 706 | if (dqs_high[dqs] * DQS_GW_FINE_STEP <= 16) |
| 707 | continue; |
| 708 | |
| 709 | switch (dqs_cnt) { |
| 710 | case 3: |
| 711 | dly_coarse_large_cnt[dqs] = dly_coarse_large; |
| 712 | dly_coarse_0p5t_cnt[dqs] = dly_coarse_0p5t; |
| 713 | dly_fine_tune_cnt[dqs] = dly_fine_xt; |
| 714 | dqs_trans[dqs] = 1; |
| 715 | break; |
| 716 | case 2: |
| 717 | case 1: |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 718 | if (dqs_trans[dqs] == 1) |
| 719 | dramc_dbg("[Byte %ld] Lead/lag falling Transition" |
| 720 | " (%d, %d, %d)\n", |
| 721 | dqs, dly_coarse_large_cnt[dqs], |
| 722 | dly_coarse_0p5t_cnt[dqs], dly_fine_tune_cnt[dqs]); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 723 | dqs_trans[dqs]++; |
| 724 | break; |
| 725 | case 0: |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 726 | dramc_dbg("[Byte %ld] Lead/lag Transition tap number (%d)\n", |
| 727 | dqs, dqs_trans[dqs]); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 728 | dqs_high[dqs] = 0; |
| 729 | break; |
| 730 | } |
| 731 | } |
| 732 | } |
| 733 | |
| 734 | static void dram_phy_reset(u8 chn) |
| 735 | { |
| 736 | setbits_le32(&ch[chn].ao.ddrconf0, 1 << DDRCONF0_RDATRST_SHIFT); |
| 737 | setbits_le32(&ch[chn].phy.misc_ctrl1, 1 << MISC_CTRL1_R_DMPHYRST_SHIFT); |
| 738 | clrbits_le32(&ch[chn].phy.b[0].dq[9], (1 << 4) | (1 << 0)); |
| 739 | clrbits_le32(&ch[chn].phy.b[1].dq[9], (1 << 4) | (1 << 0)); |
| 740 | |
| 741 | udelay(1); |
| 742 | setbits_le32(&ch[chn].phy.b[1].dq[9], (1 << 4) | (1 << 0)); |
| 743 | setbits_le32(&ch[chn].phy.b[0].dq[9], (1 << 4) | (1 << 0)); |
| 744 | clrbits_le32(&ch[chn].phy.misc_ctrl1, 1 << MISC_CTRL1_R_DMPHYRST_SHIFT); |
| 745 | clrbits_le32(&ch[chn].ao.ddrconf0, 1 << DDRCONF0_RDATRST_SHIFT); |
| 746 | } |
| 747 | |
| 748 | static void dramc_set_gating_mode(u8 chn, bool mode) |
| 749 | { |
| 750 | u8 vref = 0, burst = 0; |
| 751 | |
| 752 | if (mode) { |
| 753 | vref = 2; |
| 754 | burst = 1; |
| 755 | } |
| 756 | |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 757 | for (size_t b = 0; b < 2; b++) { |
| 758 | clrsetbits_le32(&ch[chn].phy.b[b].dq[6], 0x3 << 14, vref << 14); |
| 759 | setbits_le32(&ch[chn].phy.b[b].dq[9], 0x1 << 5); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 760 | } |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 761 | |
| 762 | clrsetbits_le32(&ch[chn].ao.stbcal1, 0x1 << 5, burst << 5); |
| 763 | setbits_le32(&ch[chn].ao.stbcal, 0x1 << 30); |
| 764 | |
| 765 | clrbits_le32(&ch[chn].phy.b[0].dq[9], (0x1 << 4) | (0x1 << 0)); |
| 766 | clrbits_le32(&ch[chn].phy.b[1].dq[9], (0x1 << 4) | (0x1 << 0)); |
| 767 | udelay(1); |
| 768 | setbits_le32(&ch[chn].phy.b[1].dq[9], (0x1 << 4) | (0x1 << 0)); |
| 769 | setbits_le32(&ch[chn].phy.b[0].dq[9], (0x1 << 4) | (0x1 << 0)); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 770 | } |
| 771 | |
| 772 | static void dramc_rx_dqs_gating_cal_pre(u8 chn, u8 rank) |
| 773 | { |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 774 | clrbits_le32(&ch[chn].ao.refctrl0, 1 << REFCTRL0_PBREFEN_SHIFT); |
| 775 | |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 776 | dramc_hw_gating_onoff(chn, false); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 777 | |
| 778 | setbits_le32(&ch[chn].ao.stbcal1, 1 << STBCAL1_STBENCMPEN_SHIFT); |
| 779 | setbits_le32(&ch[chn].ao.stbcal1, 1 << STBCAL1_STBCNT_LATCH_EN_SHIFT); |
| 780 | clrbits_le32(&ch[chn].ao.ddrconf0, 1 << DDRCONF0_DM4TO1MODE_SHIFT); |
| 781 | setbits_le32(&ch[chn].ao.spcmd, 1 << SPCMD_DQSGCNTEN_SHIFT); |
| 782 | |
| 783 | udelay(4); |
| 784 | setbits_le32(&ch[chn].ao.spcmd, 1 << SPCMD_DQSGCNTRST_SHIFT); |
| 785 | udelay(1); |
| 786 | clrbits_le32(&ch[chn].ao.spcmd, 1 << SPCMD_DQSGCNTRST_SHIFT); |
| 787 | clrsetbits_le32(&ch[chn].phy.misc_ctrl1, |
| 788 | 1 << MISC_CTRL1_R_DMSTBENCMP_RK_OPT_SHIFT, |
| 789 | rank << MISC_CTRL1_R_DMSTBENCMP_RK_OPT_SHIFT); |
| 790 | |
| 791 | } |
| 792 | |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 793 | static void set_selph_gating_value(uint32_t *addr, u8 dly, u8 dly_p1) |
| 794 | { |
| 795 | clrsetbits_le32(addr, 0x77777777, |
| 796 | (dly << 0) | (dly << 8) | (dly << 16) | (dly << 24) | |
| 797 | (dly_p1 << 4) | (dly_p1 << 12) | (dly_p1 << 20) | (dly_p1 << 28)); |
| 798 | } |
| 799 | |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 800 | static void dramc_write_dqs_gating_result(u8 chn, u8 rank, |
| 801 | u8 *best_coarse_tune2t, u8 *best_coarse_tune0p5t, |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 802 | u8 *best_coarse_tune2t_p1, u8 *best_coarse_tune0p5t_p1) |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 803 | { |
| 804 | u8 best_coarse_rodt[DQS_NUMBER], best_coarse_0p5t_rodt[DQS_NUMBER]; |
| 805 | u8 best_coarse_rodt_p1[DQS_NUMBER]; |
| 806 | u8 best_coarse_0p5t_rodt_p1[DQS_NUMBER]; |
| 807 | |
mtk11195 | 16ad2d7 | 2019-01-23 11:41:19 +0800 | [diff] [blame] | 808 | dramc_rx_dqs_isi_pulse_cg_switch(chn, true); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 809 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 810 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[rank].selph_dqsg0, |
| 811 | 0x77777777, |
| 812 | (best_coarse_tune2t[0] << 0) | (best_coarse_tune2t[1] << 8) | |
| 813 | (best_coarse_tune2t_p1[0] << 4) | (best_coarse_tune2t_p1[1] << 12)); |
| 814 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[rank].selph_dqsg1, |
| 815 | 0x77777777, |
| 816 | (best_coarse_tune0p5t[0] << 0) | (best_coarse_tune0p5t[1] << 8) | |
| 817 | (best_coarse_tune0p5t_p1[0] << 4) | (best_coarse_tune0p5t_p1[1] << 12)); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 818 | |
| 819 | for (size_t dqs = 0; dqs < DQS_NUMBER; dqs++) { |
| 820 | u8 tmp_value = (best_coarse_tune2t[dqs] << 3) |
| 821 | + best_coarse_tune0p5t[dqs]; |
| 822 | |
| 823 | if (tmp_value >= 11) { |
| 824 | tmp_value -= 11; |
| 825 | best_coarse_rodt[dqs] = tmp_value >> 3; |
| 826 | best_coarse_0p5t_rodt[dqs] = |
| 827 | tmp_value - (best_coarse_rodt[dqs] << 3); |
| 828 | |
| 829 | tmp_value = (best_coarse_tune2t_p1[dqs] << 3) + |
| 830 | best_coarse_tune0p5t_p1[dqs] - 11; |
| 831 | best_coarse_rodt_p1[dqs] = tmp_value >> 3; |
| 832 | best_coarse_0p5t_rodt_p1[dqs] = |
| 833 | tmp_value - (best_coarse_rodt_p1[dqs] << 3); |
| 834 | |
| 835 | dramc_dbg("Best RODT dly(2T, 0.5T) = (%d, %d)\n", |
| 836 | best_coarse_rodt[dqs], |
| 837 | best_coarse_0p5t_rodt[dqs]); |
| 838 | } else { |
| 839 | best_coarse_rodt[dqs] = 0; |
| 840 | best_coarse_0p5t_rodt[dqs] = 0; |
| 841 | best_coarse_rodt_p1[dqs] = 4; |
| 842 | best_coarse_0p5t_rodt_p1[dqs] = 4; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 843 | dramc_dbg("RxdqsGatingCal error: best_coarse_tune2t:%zd" |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 844 | " is already 0. RODT cannot be -1 coarse\n", |
| 845 | dqs); |
| 846 | } |
| 847 | } |
| 848 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 849 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[rank].selph_odten0, |
| 850 | 0x77777777, |
| 851 | (best_coarse_rodt[0] << 0) | (best_coarse_rodt[1] << 8) | |
| 852 | (best_coarse_rodt_p1[0] << 4) | (best_coarse_rodt_p1[1] << 12)); |
| 853 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[rank].selph_odten1, |
| 854 | 0x77777777, |
| 855 | (best_coarse_0p5t_rodt[0] << 0) | (best_coarse_0p5t_rodt[1] << 8) | |
| 856 | (best_coarse_0p5t_rodt_p1[0] << 4) | (best_coarse_0p5t_rodt_p1[1] << 12)); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 857 | } |
| 858 | |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 859 | static void dramc_rx_dqs_gating_cal_partial(u8 chn, u8 rank, |
| 860 | u32 coarse_start, u32 coarse_end, u8 freqDiv, |
| 861 | u8 *pass_begin, u8 *pass_count, u8 *pass_count_1, u8 *dqs_done, |
| 862 | u8 *dqs_high, u8 *dqs_transition, u8 *dly_coarse_large_cnt, |
| 863 | u8 *dly_coarse_0p5t_cnt, u8 *dly_fine_tune_cnt) |
| 864 | { |
| 865 | u8 dqs; |
| 866 | u32 debug_cnt[DQS_NUMBER]; |
| 867 | |
| 868 | for (u32 coarse_tune = coarse_start; coarse_tune < coarse_end; |
| 869 | coarse_tune++) { |
| 870 | u32 dly_coarse_large_rodt = 0, dly_coarse_0p5t_rodt = 0; |
| 871 | u32 dly_coarse_large_rodt_p1 = 4, dly_coarse_0p5t_rodt_p1 = 4; |
| 872 | |
| 873 | u8 dly_coarse_large = coarse_tune / RX_DQS_CTL_LOOP; |
| 874 | u8 dly_coarse_0p5t = coarse_tune % RX_DQS_CTL_LOOP; |
| 875 | u32 dly_coarse_large_p1 = (coarse_tune + freqDiv) / RX_DQS_CTL_LOOP; |
| 876 | u32 dly_coarse_0p5t_p1 = (coarse_tune + freqDiv) % RX_DQS_CTL_LOOP; |
| 877 | u32 value = (dly_coarse_large << 3) + dly_coarse_0p5t; |
| 878 | |
| 879 | if (value >= 11) { |
| 880 | value -= 11; |
| 881 | dly_coarse_large_rodt = value >> 3; |
| 882 | dly_coarse_0p5t_rodt = |
| 883 | value - (dly_coarse_large_rodt << 3); |
| 884 | value = (dly_coarse_large << 3) + dly_coarse_0p5t - 11; |
| 885 | dly_coarse_large_rodt_p1 = value >> 3; |
| 886 | dly_coarse_0p5t_rodt_p1 = |
| 887 | value - (dly_coarse_large_rodt_p1 << 3); |
| 888 | } |
| 889 | |
| 890 | set_selph_gating_value(&ch[chn].ao.shu[0].rk[rank].selph_dqsg0, |
| 891 | dly_coarse_large, dly_coarse_large_p1); |
| 892 | set_selph_gating_value(&ch[chn].ao.shu[0].rk[rank].selph_dqsg1, |
| 893 | dly_coarse_0p5t, dly_coarse_0p5t_p1); |
| 894 | set_selph_gating_value(&ch[chn].ao.shu[0].rk[rank].selph_odten0, |
| 895 | dly_coarse_large_rodt, dly_coarse_large_rodt_p1); |
| 896 | set_selph_gating_value(&ch[chn].ao.shu[0].rk[rank].selph_odten1, |
| 897 | dly_coarse_0p5t_rodt, dly_coarse_0p5t_rodt_p1); |
| 898 | |
| 899 | for (u8 dly_fine_xt = 0; dly_fine_xt < DQS_GW_FINE_END; |
| 900 | dly_fine_xt += 4) { |
| 901 | dramc_set_gating_mode(chn, 0); |
| 902 | write32(&ch[chn].ao.shu[0].rk[rank].dqsien, |
| 903 | dly_fine_xt | (dly_fine_xt << 8)); |
| 904 | |
| 905 | dram_phy_reset(chn); |
| 906 | setbits_le32(&ch[chn].ao.spcmd, |
| 907 | 1 << SPCMD_DQSGCNTRST_SHIFT); |
| 908 | udelay(1); |
| 909 | clrbits_le32(&ch[chn].ao.spcmd, |
| 910 | 1 << SPCMD_DQSGCNTRST_SHIFT); |
| 911 | |
| 912 | dramc_engine2_run(chn, TE_OP_READ_CHECK); |
| 913 | |
| 914 | u32 result_r = read32(&ch[chn].phy.misc_stberr_rk0_r) & |
| 915 | MISC_STBERR_RK_R_STBERR_RK_R_MASK; |
| 916 | u32 result_f = read32(&ch[chn].phy.misc_stberr_rk0_f) & |
| 917 | MISC_STBERR_RK_F_STBERR_RK_F_MASK; |
| 918 | debug_cnt[0] = read32(&ch[chn].nao.dqsgnwcnt[0]); |
| 919 | debug_cnt[1] = (debug_cnt[0] >> 16) & 0xffff; |
| 920 | debug_cnt[0] &= 0xffff; |
| 921 | |
| 922 | dramc_set_gating_mode(chn, 1); |
| 923 | dramc_engine2_run(chn, TE_OP_READ_CHECK); |
| 924 | |
| 925 | dramc_find_dly_tune(chn, dly_coarse_large, |
| 926 | dly_coarse_0p5t, dly_fine_xt, dqs_high, |
| 927 | dly_coarse_large_cnt, dly_coarse_0p5t_cnt, |
| 928 | dly_fine_tune_cnt, dqs_transition, dqs_done); |
| 929 | |
| 930 | dramc_dbg("%d %d %d |", dly_coarse_large, |
| 931 | dly_coarse_0p5t, dly_fine_xt); |
| 932 | for (dqs = 0; dqs < DQS_NUMBER; dqs++) |
| 933 | dramc_dbg("%X ", debug_cnt[dqs]); |
| 934 | |
| 935 | dramc_dbg(" |"); |
| 936 | for (dqs = 0; dqs < DQS_NUMBER; dqs++) { |
| 937 | dramc_dbg("(%X %X)", |
| 938 | (result_f >> (DQS_BIT_NUMBER * dqs)) & 0xff, |
| 939 | (result_r >> (DQS_BIT_NUMBER * dqs)) & 0xff); |
| 940 | } |
| 941 | |
| 942 | dramc_dbg("\n"); |
| 943 | if (dramc_find_gating_window(result_r, result_f, debug_cnt, |
| 944 | dly_coarse_large, dly_coarse_0p5t, pass_begin, |
| 945 | pass_count, pass_count_1, &dly_fine_xt, |
| 946 | dqs_high, dqs_done)) |
| 947 | coarse_tune = coarse_end; |
| 948 | } |
| 949 | } |
| 950 | } |
| 951 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 952 | static void dramc_rx_dqs_gating_cal(u8 chn, u8 rank, u8 freq_group, |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 953 | const struct sdram_params *params, const bool fast_calib) |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 954 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 955 | u8 dqs, fsp, freqDiv = 4; |
| 956 | u8 pass_begin[DQS_NUMBER] = {0}, pass_count[DQS_NUMBER] = {0}, |
| 957 | pass_count_1[DQS_NUMBER] = {0}, dqs_done[DQS_NUMBER] = {0}; |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 958 | u8 min_coarse_tune2t[DQS_NUMBER], min_coarse_tune0p5t[DQS_NUMBER], |
| 959 | min_fine_tune[DQS_NUMBER]; |
| 960 | u8 best_fine_tune[DQS_NUMBER], best_coarse_tune0p5t[DQS_NUMBER], |
| 961 | best_coarse_tune2t[DQS_NUMBER]; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 962 | u8 best_coarse_tune0p5t_p1[DQS_NUMBER], best_coarse_tune2t_p1[DQS_NUMBER]; |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 963 | u8 dqs_high[DQS_NUMBER] = {0}, dqs_transition[DQS_NUMBER] = {0}; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 964 | u8 dly_coarse_large_cnt[DQS_NUMBER] = {0}, dly_coarse_0p5t_cnt[DQS_NUMBER] = {0}, |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 965 | dly_fine_tune_cnt[DQS_NUMBER] = {0}; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 966 | u32 coarse_start, coarse_end; |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 967 | |
| 968 | struct reg_value regs_bak[] = { |
| 969 | {&ch[chn].ao.stbcal, 0x0}, |
| 970 | {&ch[chn].ao.stbcal1, 0x0}, |
| 971 | {&ch[chn].ao.ddrconf0, 0x0}, |
| 972 | {&ch[chn].ao.spcmd, 0x0}, |
| 973 | {&ch[chn].ao.refctrl0, 0x0}, |
| 974 | {&ch[chn].phy.b[0].dq[6], 0x0}, |
| 975 | {&ch[chn].phy.b[1].dq[6], 0x0}, |
| 976 | }; |
| 977 | for (size_t i = 0; i < ARRAY_SIZE(regs_bak); i++) |
| 978 | regs_bak[i].value = read32(regs_bak[i].addr); |
| 979 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 980 | fsp = get_freq_fsq(freq_group); |
| 981 | dramc_rx_dqs_isi_pulse_cg_switch(chn, false); |
| 982 | |
| 983 | MR01Value[fsp] |= 0x80; |
| 984 | dramc_mode_reg_write_by_rank(chn, rank, 0x1, MR01Value[fsp]); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 985 | dramc_rx_dqs_gating_cal_pre(chn, rank); |
| 986 | |
| 987 | u32 dummy_rd_backup = read32(&ch[chn].ao.dummy_rd); |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 988 | dramc_engine2_init(chn, rank, TEST2_1_CAL, 0xaa000023, true); |
| 989 | |
| 990 | switch (freq_group) { |
| 991 | case LP4X_DDR1600: |
| 992 | coarse_start = 18; |
| 993 | break; |
| 994 | case LP4X_DDR2400: |
| 995 | coarse_start = 25; |
| 996 | break; |
| 997 | case LP4X_DDR3200: |
| 998 | coarse_start = 25; |
| 999 | break; |
| 1000 | case LP4X_DDR3600: |
| 1001 | coarse_start = 21; |
| 1002 | break; |
| 1003 | default: |
| 1004 | die("Invalid DDR frequency group %u\n", freq_group); |
| 1005 | return; |
| 1006 | } |
| 1007 | coarse_end = coarse_start + 12; |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 1008 | |
| 1009 | dramc_dbg("[Gating]\n"); |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1010 | |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1011 | if (!fast_calib) { |
| 1012 | dramc_rx_dqs_gating_cal_partial(chn, rank, |
| 1013 | coarse_start, coarse_end, |
| 1014 | freqDiv, pass_begin, pass_count, pass_count_1, dqs_done, |
| 1015 | dqs_high, dqs_transition, dly_coarse_large_cnt, |
| 1016 | dly_coarse_0p5t_cnt, dly_fine_tune_cnt); |
| 1017 | dramc_engine2_end(chn, dummy_rd_backup); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 1018 | } |
| 1019 | |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 1020 | for (dqs = 0; dqs < DQS_NUMBER; dqs++) { |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1021 | if (fast_calib) { |
| 1022 | dramc_dbg("[bypass Gating params] dqs: %d\n", dqs); |
| 1023 | pass_count[dqs] = params->gating_pass_count[chn][rank][dqs]; |
| 1024 | min_fine_tune[dqs] = params->gating_fine_tune[chn][rank][dqs]; |
| 1025 | min_coarse_tune0p5t[dqs] = params->gating05T[chn][rank][dqs]; |
| 1026 | min_coarse_tune2t[dqs] = params->gating2T[chn][rank][dqs]; |
| 1027 | } else { |
| 1028 | pass_count[dqs] = dqs_transition[dqs]; |
| 1029 | min_fine_tune[dqs] = dly_fine_tune_cnt[dqs]; |
| 1030 | min_coarse_tune0p5t[dqs] = dly_coarse_0p5t_cnt[dqs]; |
| 1031 | min_coarse_tune2t[dqs] = dly_coarse_large_cnt[dqs]; |
| 1032 | } |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 1033 | u8 tmp_offset = pass_count[dqs] * DQS_GW_FINE_STEP / 2; |
| 1034 | u8 tmp_value = min_fine_tune[dqs] + tmp_offset; |
| 1035 | best_fine_tune[dqs] = tmp_value % RX_DLY_DQSIENSTB_LOOP; |
| 1036 | |
| 1037 | tmp_offset = tmp_value / RX_DLY_DQSIENSTB_LOOP; |
| 1038 | tmp_value = min_coarse_tune0p5t[dqs] + tmp_offset; |
| 1039 | best_coarse_tune0p5t[dqs] = tmp_value % RX_DQS_CTL_LOOP; |
| 1040 | |
| 1041 | tmp_offset = tmp_value / RX_DQS_CTL_LOOP; |
| 1042 | best_coarse_tune2t[dqs] = min_coarse_tune2t[dqs] + tmp_offset; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1043 | |
| 1044 | tmp_value = best_coarse_tune0p5t[dqs] + freqDiv; |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 1045 | best_coarse_tune0p5t_p1[dqs] = tmp_value % RX_DQS_CTL_LOOP; |
| 1046 | |
| 1047 | tmp_offset = tmp_value / RX_DQS_CTL_LOOP; |
| 1048 | best_coarse_tune2t_p1[dqs] = |
| 1049 | best_coarse_tune2t[dqs] + tmp_offset; |
| 1050 | } |
| 1051 | |
| 1052 | for (dqs = 0; dqs < DQS_NUMBER; dqs++) |
| 1053 | dramc_show("Best DQS%d dly(2T, 0.5T, fine tune)" |
| 1054 | " = (%d, %d, %d)\n", dqs, best_coarse_tune2t[dqs], |
| 1055 | best_coarse_tune0p5t[dqs], best_fine_tune[dqs]); |
| 1056 | |
| 1057 | for (dqs = 0; dqs < DQS_NUMBER; dqs++) |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1058 | dramc_show("Best DQS%d P1 dly(2T, 0.5T, fine tune)" |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 1059 | " = (%d, %d, %d)\n", dqs, best_coarse_tune2t_p1[dqs], |
| 1060 | best_coarse_tune0p5t_p1[dqs], best_fine_tune[dqs]); |
| 1061 | |
| 1062 | for (size_t i = 0; i < ARRAY_SIZE(regs_bak); i++) |
| 1063 | write32(regs_bak[i].addr, regs_bak[i].value); |
| 1064 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1065 | MR01Value[fsp] &= 0x7f; |
| 1066 | dramc_mode_reg_write_by_rank(chn, rank, 0x1, MR01Value[fsp]); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 1067 | |
| 1068 | dramc_write_dqs_gating_result(chn, rank, best_coarse_tune2t, |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1069 | best_coarse_tune0p5t, best_coarse_tune2t_p1, best_coarse_tune0p5t_p1); |
| 1070 | |
| 1071 | write32(&ch[chn].ao.shu[0].rk[rank].dqsien, |
| 1072 | best_fine_tune[0] | (best_fine_tune[1] << 8)); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 1073 | |
| 1074 | dram_phy_reset(chn); |
| 1075 | } |
| 1076 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1077 | static void dramc_rx_rd_dqc_init(u8 chn, u8 rank) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1078 | { |
| 1079 | const u8 *lpddr_phy_mapping = phy_mapping[chn]; |
| 1080 | u16 temp_value = 0; |
| 1081 | |
| 1082 | for (size_t b = 0; b < 2; b++) |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1083 | clrbits_le32(&ch[chn].phy.shu[0].b[b].dq[7], 0x1 << 7); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1084 | |
| 1085 | clrsetbits_le32(&ch[chn].ao.mrs, |
| 1086 | MRS_MRSRK_MASK, rank << MRS_MRSRK_SHIFT); |
| 1087 | setbits_le32(&ch[chn].ao.mpc_option, |
| 1088 | 0x1 << MPC_OPTION_MPCRKEN_SHIFT); |
| 1089 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1090 | for (size_t bit = 0; bit < DQ_DATA_WIDTH; bit++) |
| 1091 | temp_value |= ((0x5555 >> bit) & 0x1) << lpddr_phy_mapping[bit]; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1092 | |
| 1093 | u16 mr15_golden_value = temp_value & 0xff; |
| 1094 | u16 mr20_golden_value = (temp_value >> 8) & 0xff; |
| 1095 | clrsetbits_le32(&ch[chn].ao.mr_golden, |
| 1096 | MR_GOLDEN_MR15_GOLDEN_MASK | MR_GOLDEN_MR20_GOLDEN_MASK, |
| 1097 | (mr15_golden_value << 8) | mr20_golden_value); |
| 1098 | } |
| 1099 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1100 | static u32 dramc_rx_rd_dqc_run(u8 chn) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1101 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1102 | u32 loop = 0; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1103 | setbits_le32(&ch[chn].ao.spcmdctrl, 1 << SPCMDCTRL_RDDQCDIS_SHIFT); |
| 1104 | setbits_le32(&ch[chn].ao.spcmd, 1 << SPCMD_RDDQCEN_SHIFT); |
| 1105 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1106 | while (!wait_us(10, read32(&ch[chn].nao.spcmdresp) & (0x1 << 7))) { |
| 1107 | if (loop++ > 10) |
| 1108 | dramc_dbg("[RDDQC] resp fail (time out)\n"); |
| 1109 | } |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1110 | |
| 1111 | u32 result = read32(&ch[chn].nao.rdqc_cmp); |
| 1112 | clrbits_le32(&ch[chn].ao.spcmd, 1 << SPCMD_RDDQCEN_SHIFT); |
| 1113 | clrbits_le32(&ch[chn].ao.spcmdctrl, 1 << SPCMDCTRL_RDDQCDIS_SHIFT); |
| 1114 | |
| 1115 | return result; |
| 1116 | } |
| 1117 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1118 | static void dramc_rx_rd_dqc_end(u8 chn) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1119 | { |
| 1120 | clrbits_le32(&ch[chn].ao.mrs, MRS_MRSRK_MASK); |
| 1121 | } |
| 1122 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1123 | static void dramc_rx_vref_pre_setting(u8 chn) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1124 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1125 | setbits_le32(&ch[chn].phy.b[0].dq[5], 0x1 << 16); |
| 1126 | setbits_le32(&ch[chn].phy.b[1].dq[5], 0x1 << 16); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1127 | } |
| 1128 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1129 | static void dramc_set_rx_vref(u8 chn, u8 vref) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1130 | { |
| 1131 | for (size_t b = 0; b < 2; b++) |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1132 | clrsetbits_le32(&ch[chn].phy.shu[0].b[b].dq[5], 0x3f, vref << 0); |
| 1133 | dramc_dbg("set rx vref :%d\n", vref); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1134 | } |
| 1135 | |
| 1136 | static void dramc_set_tx_vref(u8 chn, u8 rank, u8 value) |
| 1137 | { |
| 1138 | dramc_mode_reg_write_by_rank(chn, rank, 14, value); |
| 1139 | } |
| 1140 | |
| 1141 | static void dramc_set_vref(u8 chn, u8 rank, enum CAL_TYPE type, u8 vref) |
| 1142 | { |
| 1143 | if (type == RX_WIN_TEST_ENG) |
| 1144 | dramc_set_rx_vref(chn, vref); |
| 1145 | else |
| 1146 | dramc_set_tx_vref(chn, rank, vref); |
| 1147 | } |
| 1148 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1149 | static void dramc_transfer_dly_tune(u8 chn, u32 dly, u32 adjust_center, |
| 1150 | struct tx_dly_tune *dly_tune) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1151 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1152 | u8 tune = 3, fine_tune = 0; |
| 1153 | u16 tmp; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1154 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1155 | fine_tune = dly & (TX_DQ_COARSE_TUNE_TO_FINE_TUNE_TAP - 1); |
| 1156 | tmp = (dly / TX_DQ_COARSE_TUNE_TO_FINE_TUNE_TAP) << 1; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1157 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1158 | if (adjust_center) { |
| 1159 | if (fine_tune < 10) { |
| 1160 | fine_tune += TX_DQ_COARSE_TUNE_TO_FINE_TUNE_TAP >> 1; |
| 1161 | tmp--; |
| 1162 | } else if (fine_tune > TX_DQ_COARSE_TUNE_TO_FINE_TUNE_TAP - 10) { |
| 1163 | fine_tune -= TX_DQ_COARSE_TUNE_TO_FINE_TUNE_TAP >> 1; |
| 1164 | tmp++; |
| 1165 | } |
| 1166 | } |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1167 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1168 | dly_tune->fine_tune = fine_tune; |
| 1169 | dly_tune->coarse_tune_small = tmp - ((tmp >> tune) << tune); |
| 1170 | dly_tune->coarse_tune_large = tmp >> tune; |
| 1171 | |
| 1172 | tmp -= 3; |
| 1173 | dly_tune->coarse_tune_small_oen = tmp - ((tmp >> tune) << tune); |
| 1174 | dly_tune->coarse_tune_large_oen = tmp >> tune; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1175 | } |
| 1176 | |
mtk11195 | 16ad2d7 | 2019-01-23 11:41:19 +0800 | [diff] [blame] | 1177 | static void dramc_set_rx_dly_factor(u8 chn, u8 rank, enum RX_TYPE type, u32 val) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1178 | { |
| 1179 | u32 tmp, mask; |
| 1180 | |
| 1181 | switch (type) { |
| 1182 | case RX_DQ: |
| 1183 | tmp = (val << 24 | val << 16 | val << 8 | val); |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1184 | for (size_t dq = 2; dq < 6; dq++) { |
| 1185 | write32(&ch[chn].phy.shu[0].rk[rank].b[0].dq[dq], tmp); |
| 1186 | write32(&ch[chn].phy.shu[0].rk[rank].b[1].dq[dq], tmp); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1187 | } |
| 1188 | break; |
| 1189 | |
| 1190 | case RX_DQM: |
| 1191 | tmp = (val << 8 | val); |
| 1192 | mask = SHU1_B0_DQ6_RK_RX_ARDQM0_F_DLY_B0_MASK | |
| 1193 | SHU1_B0_DQ6_RK_RX_ARDQM0_R_DLY_B0_MASK; |
| 1194 | clrsetbits_le32(&ch[chn].phy.shu[0].rk[rank].b[0].dq[6], |
| 1195 | mask, tmp); |
| 1196 | clrsetbits_le32(&ch[chn].phy.shu[0].rk[rank].b[1].dq[6], |
| 1197 | mask, tmp); |
| 1198 | break; |
| 1199 | |
| 1200 | case RX_DQS: |
| 1201 | tmp = (val << 24 | val << 16); |
| 1202 | mask = SHU1_B0_DQ6_RK_RX_ARDQS0_F_DLY_B0_MASK | |
| 1203 | SHU1_B0_DQ6_RK_RX_ARDQS0_R_DLY_B0_MASK; |
| 1204 | clrsetbits_le32(&ch[chn].phy.shu[0].rk[rank].b[0].dq[6], |
| 1205 | mask, tmp); |
| 1206 | clrsetbits_le32(&ch[chn].phy.shu[0].rk[rank].b[1].dq[6], |
| 1207 | mask, tmp); |
| 1208 | break; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1209 | default: |
| 1210 | dramc_show("error calibration type:%d\n", type); |
| 1211 | break; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1212 | } |
| 1213 | } |
| 1214 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1215 | static void dramc_set_tx_dly_factor(u8 chn, u8 rk, |
| 1216 | enum CAL_TYPE type, u8 *dq_small_reg, u32 dly) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1217 | { |
| 1218 | struct tx_dly_tune dly_tune = {0}; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1219 | u32 dly_large = 0, dly_large_oen = 0, dly_small = 0, dly_small_oen = 0; |
| 1220 | u32 adjust_center = 0; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1221 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1222 | dramc_transfer_dly_tune(chn, dly, adjust_center, &dly_tune); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1223 | |
| 1224 | for (u8 i = 0; i < 4; i++) { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1225 | dly_large += dly_tune.coarse_tune_large << (i * 4); |
| 1226 | dly_large_oen += dly_tune.coarse_tune_large_oen << (i * 4); |
| 1227 | dly_small += dly_tune.coarse_tune_small << (i * 4); |
| 1228 | dly_small_oen += dly_tune.coarse_tune_small_oen << (i * 4); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1229 | } |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1230 | |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1231 | if (type == TX_WIN_DQ_DQM) |
| 1232 | dramc_dbg("%3d |%d %d %2d | [0]", |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1233 | dly, dly_tune.coarse_tune_large, |
| 1234 | dly_tune.coarse_tune_small, dly_tune.fine_tune); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1235 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1236 | if (*dq_small_reg != dly_tune.coarse_tune_small) { |
| 1237 | if (type == TX_WIN_DQ_DQM || type == TX_WIN_DQ_ONLY) { |
| 1238 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[rk].selph_dq[0], |
| 1239 | 0x77777777, dly_large | (dly_large_oen << 16)); |
| 1240 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[rk].selph_dq[2], |
| 1241 | 0x77777777, dly_small | (dly_small_oen << 16)); |
| 1242 | } |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1243 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1244 | if (type == TX_WIN_DQ_DQM) { |
| 1245 | /* Large coarse_tune setting */ |
| 1246 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[rk].selph_dq[1], |
| 1247 | 0x77777777, dly_large | (dly_large_oen << 16)); |
| 1248 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[rk].selph_dq[3], |
| 1249 | 0x77777777, dly_small | (dly_small_oen << 16)); |
| 1250 | } |
| 1251 | } |
| 1252 | *dq_small_reg = dly_tune.coarse_tune_small; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1253 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1254 | if (type == TX_WIN_DQ_DQM || type == TX_WIN_DQ_ONLY) { |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1255 | for (size_t b = 0; b < 2; b++) |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1256 | clrsetbits_le32(&ch[chn].phy.shu[0].rk[rk].b[b].dq[7], |
| 1257 | FINE_TUNE_DQ_MASK, dly_tune.fine_tune << 8); |
| 1258 | } |
| 1259 | if (type == TX_WIN_DQ_DQM) { |
| 1260 | for (size_t b = 0; b < 2; b++) |
| 1261 | clrsetbits_le32(&ch[chn].phy.shu[0].rk[rk].b[b].dq[7], |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1262 | FINE_TUNE_DQM_MASK, dly_tune.fine_tune << 16); |
| 1263 | } |
| 1264 | } |
| 1265 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1266 | static u32 dramc_get_smallest_dqs_dly(u8 chn, u8 rank, const struct sdram_params *params) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1267 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1268 | const u8 mck = 3; |
| 1269 | u32 min_dly = 0xffff, virtual_delay = 0; |
| 1270 | u32 tx_dly = read32(&ch[chn].ao.shu[0].selph_dqs0); |
| 1271 | u32 dly = read32(&ch[chn].ao.shu[0].selph_dqs1); |
| 1272 | u32 tmp; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1273 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1274 | for (size_t dqs = 0; dqs < DQS_NUMBER; dqs++) { |
| 1275 | tmp = ((tx_dly >> (dqs << 2) & 0x7) << mck) + |
| 1276 | (dly >> (dqs << 2) & 0x7); |
| 1277 | virtual_delay = (tmp << 5) + params->wr_level[chn][rank][dqs]; |
| 1278 | min_dly = MIN(min_dly, virtual_delay); |
| 1279 | } |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1280 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1281 | return min_dly; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1282 | } |
| 1283 | |
| 1284 | static void dramc_get_dly_range(u8 chn, u8 rank, enum CAL_TYPE type, |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1285 | u8 freq_group, u16 *pre_cal, s16 *begin, s16 *end, |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1286 | const struct sdram_params *params) |
| 1287 | { |
| 1288 | u16 pre_dq_dly; |
| 1289 | switch (type) { |
| 1290 | case RX_WIN_RD_DQC: |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1291 | case RX_WIN_TEST_ENG: |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1292 | switch (freq_group) { |
| 1293 | case LP4X_DDR1600: |
| 1294 | *begin = -48; |
| 1295 | break; |
| 1296 | case LP4X_DDR2400: |
| 1297 | *begin = -30; |
| 1298 | break; |
| 1299 | case LP4X_DDR3200: |
| 1300 | case LP4X_DDR3600: |
| 1301 | *begin = -26; |
| 1302 | break; |
| 1303 | default: |
| 1304 | die("Invalid DDR frequency group %u\n", freq_group); |
| 1305 | return; |
| 1306 | } |
| 1307 | |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1308 | *end = MAX_RX_DQDLY_TAPS; |
| 1309 | break; |
| 1310 | |
| 1311 | case TX_WIN_DQ_DQM: |
| 1312 | *begin = dramc_get_smallest_dqs_dly(chn, rank, params); |
| 1313 | *end = *begin + 256; |
| 1314 | break; |
| 1315 | |
| 1316 | case TX_WIN_DQ_ONLY: |
| 1317 | pre_dq_dly = MIN(pre_cal[0], pre_cal[1]); |
| 1318 | pre_dq_dly = (pre_dq_dly > 24) ? (pre_dq_dly - 24) : 0; |
| 1319 | *begin = pre_dq_dly; |
| 1320 | *end = *begin + 64; |
| 1321 | break; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1322 | default: |
| 1323 | dramc_show("error calibration type:%d\n", type); |
| 1324 | break; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1325 | } |
| 1326 | } |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1327 | |
| 1328 | static int dramc_check_dqdqs_win(struct win_perbit_dly *perbit_dly, |
| 1329 | s16 dly, s16 dly_end, bool fail_bit) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1330 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1331 | int pass_win = 0; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1332 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1333 | if (perbit_dly->first_pass == PASS_RANGE_NA) { |
| 1334 | if (!fail_bit) /* compare correct: pass */ |
| 1335 | perbit_dly->first_pass = dly; |
| 1336 | } else if (perbit_dly->last_pass == PASS_RANGE_NA) { |
| 1337 | if (fail_bit) /* compare error: fail */ |
| 1338 | perbit_dly->last_pass = dly - 1; |
| 1339 | else if (dly == dly_end) |
| 1340 | perbit_dly->last_pass = dly; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1341 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1342 | if (perbit_dly->last_pass != PASS_RANGE_NA) { |
| 1343 | pass_win = perbit_dly->last_pass - perbit_dly->first_pass; |
| 1344 | int best_pass_win = perbit_dly->best_last - perbit_dly->best_first; |
| 1345 | if (pass_win >= best_pass_win) { |
| 1346 | perbit_dly->best_last = perbit_dly->last_pass; |
| 1347 | perbit_dly->best_first = perbit_dly->first_pass; |
| 1348 | } |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1349 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1350 | /* Clear to find the next pass range if it has */ |
| 1351 | perbit_dly->first_pass = PASS_RANGE_NA; |
| 1352 | perbit_dly->last_pass = PASS_RANGE_NA; |
| 1353 | } |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1354 | } |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1355 | |
| 1356 | return pass_win; |
| 1357 | } |
| 1358 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1359 | static void dramc_set_vref_dly(struct vref_perbit_dly *vref_dly, struct win_perbit_dly delay[]) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1360 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1361 | struct win_perbit_dly *perbit_dly = vref_dly->perbit_dly; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1362 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1363 | for (u8 bit = 0; bit < DQ_DATA_WIDTH; bit++) { |
| 1364 | delay[bit].win_center = (delay[bit].best_first + delay[bit].best_last) >> 1; |
| 1365 | |
| 1366 | perbit_dly[bit].best_first = delay[bit].best_first; |
| 1367 | perbit_dly[bit].best_last = delay[bit].best_last; |
| 1368 | perbit_dly[bit].win_center = delay[bit].win_center; |
| 1369 | perbit_dly[bit].best_dqdly = delay[bit].best_dqdly; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1370 | } |
| 1371 | } |
| 1372 | |
| 1373 | static bool dramk_calc_best_vref(enum CAL_TYPE type, u8 vref, |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1374 | struct vref_perbit_dly *vref_dly, struct win_perbit_dly delay[], |
| 1375 | u32 *win_min_max) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1376 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1377 | u32 win_size, min_bit = 0xff, min_winsize = 0xffff, tmp_win_sum = 0; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1378 | |
| 1379 | switch (type) { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1380 | case RX_WIN_RD_DQC: |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1381 | case RX_WIN_TEST_ENG: |
| 1382 | for (size_t bit = 0; bit < DQ_DATA_WIDTH; bit++) { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1383 | win_size = delay[bit].best_last - delay[bit].best_first; |
| 1384 | |
| 1385 | if (win_size < min_winsize) { |
| 1386 | min_bit = bit; |
| 1387 | min_winsize = win_size; |
| 1388 | } |
| 1389 | tmp_win_sum += win_size; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1390 | } |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1391 | dramc_dbg("type:%d vref:%d Min Bit=%d, min_winsize=%d, win sum:%d\n", |
| 1392 | type, vref, min_bit, min_winsize, tmp_win_sum); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1393 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1394 | if (tmp_win_sum > vref_dly->max_win_sum) { |
| 1395 | *win_min_max = min_winsize; |
| 1396 | vref_dly->max_win_sum = tmp_win_sum; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1397 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1398 | /* best vref */ |
| 1399 | vref_dly->best_vref = vref; |
| 1400 | } |
| 1401 | dramc_dbg("type:%d vref:%d, win_sum_total:%d, tmp_win_sum:%d)\n", |
| 1402 | type, vref, vref_dly->max_win_sum, tmp_win_sum); |
| 1403 | dramc_set_vref_dly(vref_dly, delay); |
| 1404 | |
| 1405 | if (tmp_win_sum < vref_dly->max_win_sum * 95 / 100) { |
| 1406 | dramc_dbg("type:%d best vref found[%d], early break! (%d < %d)\n", |
| 1407 | type, vref_dly->best_vref, tmp_win_sum, |
| 1408 | vref_dly->max_win_sum * 95 / 100); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1409 | return true; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1410 | } |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1411 | |
| 1412 | break; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1413 | case TX_WIN_DQ_ONLY: |
| 1414 | case TX_WIN_DQ_DQM: |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1415 | for (size_t bit = 0; bit < DQ_DATA_WIDTH; bit++) { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1416 | win_size = delay[bit].best_last - delay[bit].best_first; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1417 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1418 | if (win_size < min_winsize) { |
| 1419 | min_bit = bit; |
| 1420 | min_winsize = win_size; |
| 1421 | } |
| 1422 | tmp_win_sum += win_size; |
| 1423 | } |
| 1424 | dramc_dbg("type:%d vref:%d Min Bit=%d, min_winsize=%d, win sum:%d\n", |
| 1425 | type, vref, min_bit, min_winsize, tmp_win_sum); |
| 1426 | |
| 1427 | if (min_winsize > *win_min_max || |
| 1428 | (min_winsize == *win_min_max && |
| 1429 | tmp_win_sum > vref_dly->max_win_sum)) { |
| 1430 | *win_min_max = min_winsize; |
| 1431 | vref_dly->max_win_sum = tmp_win_sum; |
| 1432 | |
| 1433 | /* best vref */ |
| 1434 | vref_dly->best_vref = vref; |
| 1435 | } |
| 1436 | dramc_dbg("type:%d vref:%d, win_sum_total:%d, tmp_win_sum:%d)\n", |
| 1437 | type, vref, vref_dly->max_win_sum, tmp_win_sum); |
| 1438 | dramc_set_vref_dly(vref_dly, delay); |
| 1439 | |
| 1440 | if (tmp_win_sum < vref_dly->max_win_sum * 95 / 100) { |
| 1441 | dramc_dbg("type:%d best vref found[%d], early break! (%d < %d)\n", |
| 1442 | type, vref_dly->best_vref, tmp_win_sum, |
| 1443 | vref_dly->max_win_sum * 95 / 100); |
| 1444 | return true; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1445 | } |
| 1446 | |
| 1447 | break; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1448 | |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1449 | default: |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1450 | dramc_show("error calibration type:%d\n", type); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1451 | break; |
| 1452 | } |
| 1453 | |
| 1454 | return false; |
| 1455 | } |
| 1456 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1457 | static void dramc_set_rx_dqdqs_dly(u8 chn, u8 rank, s32 dly) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1458 | { |
| 1459 | if (dly <= 0) { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1460 | /* Set DQS delay */ |
mtk11195 | 16ad2d7 | 2019-01-23 11:41:19 +0800 | [diff] [blame] | 1461 | dramc_set_rx_dly_factor(chn, rank, RX_DQS, -dly); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1462 | dram_phy_reset(chn); |
| 1463 | } else { |
| 1464 | /* Setup time calibration */ |
mtk11195 | 16ad2d7 | 2019-01-23 11:41:19 +0800 | [diff] [blame] | 1465 | dramc_set_rx_dly_factor(chn, rank, RX_DQM, dly); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1466 | dram_phy_reset(chn); |
mtk11195 | 16ad2d7 | 2019-01-23 11:41:19 +0800 | [diff] [blame] | 1467 | dramc_set_rx_dly_factor(chn, rank, RX_DQ, dly); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1468 | } |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1469 | } |
| 1470 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1471 | static void dramc_set_tx_best_dly_factor(u8 chn, u8 rank_start, u8 type, |
| 1472 | struct per_byte_dly *tx_perbyte_dly, u16 *dq_precal_dly, |
| 1473 | u8 use_delay_cell, u32 *byte_dly_cell) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1474 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1475 | u32 dq_large = 0, dq_large_oen = 0, dq_small = 0, dq_small_oen = 0, adjust_center = 1; |
| 1476 | u32 dqm_large = 0, dqm_large_oen = 0, dqm_small = 0, dqm_small_oen = 0; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1477 | u16 dq_oen[DQS_NUMBER] = {0}, dqm_oen[DQS_NUMBER] = {0}; |
| 1478 | struct tx_dly_tune dqdly_tune[DQS_NUMBER] = {0}; |
| 1479 | struct tx_dly_tune dqmdly_tune[DQS_NUMBER] = {0}; |
| 1480 | |
| 1481 | for (size_t i = 0; i < DQS_NUMBER; i++) { |
| 1482 | dramc_transfer_dly_tune(chn, tx_perbyte_dly[i].final_dly, |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1483 | adjust_center, &dqdly_tune[i]); |
| 1484 | dramc_transfer_dly_tune(chn, dq_precal_dly[i], |
| 1485 | adjust_center, &dqmdly_tune[i]); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1486 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1487 | dq_large += dqdly_tune[i].coarse_tune_large << (i * 4); |
| 1488 | dq_large_oen += dqdly_tune[i].coarse_tune_large_oen << (i * 4); |
| 1489 | dq_small += dqdly_tune[i].coarse_tune_small << (i * 4); |
| 1490 | dq_small_oen += dqdly_tune[i].coarse_tune_small_oen << (i * 4); |
| 1491 | |
| 1492 | dqm_large += dqmdly_tune[i].coarse_tune_large << (i * 4); |
| 1493 | dqm_large_oen += dqmdly_tune[i].coarse_tune_large_oen << (i * 4); |
| 1494 | dqm_small += dqmdly_tune[i].coarse_tune_small << (i * 4); |
| 1495 | dqm_small_oen += dqmdly_tune[i].coarse_tune_small_oen << (i * 4); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1496 | |
| 1497 | dq_oen[i] = (dqdly_tune[i].coarse_tune_large_oen << 3) + |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1498 | (dqdly_tune[i].coarse_tune_small_oen << 5) + dqdly_tune[i].fine_tune; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1499 | dqm_oen[i] = (dqmdly_tune[i].coarse_tune_large_oen << 3) + |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1500 | (dqmdly_tune[i].coarse_tune_small_oen << 5) + |
| 1501 | dqmdly_tune[i].fine_tune; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1502 | } |
| 1503 | |
| 1504 | for (size_t rank = rank_start; rank < RANK_MAX; rank++) { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1505 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[rank].selph_dq[0], |
| 1506 | 0x77777777, dq_large | (dq_large_oen << 16)); |
| 1507 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[rank].selph_dq[2], |
| 1508 | 0x77777777, dq_small | (dq_small_oen << 16)); |
| 1509 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[rank].selph_dq[1], |
| 1510 | 0x77777777, dqm_large | (dqm_large_oen << 16)); |
| 1511 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[rank].selph_dq[3], |
| 1512 | 0x77777777, dqm_small | (dqm_small_oen << 16)); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1513 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1514 | for (size_t byte = 0; byte < 2; byte++) |
| 1515 | clrsetbits_le32(&ch[chn].phy.shu[0].rk[rank].b[byte].dq[7], |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1516 | FINE_TUNE_DQ_MASK | FINE_TUNE_DQM_MASK, |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1517 | (dqdly_tune[byte].fine_tune << 8) | |
| 1518 | (dqmdly_tune[byte].fine_tune << 16)); |
| 1519 | if (use_delay_cell == 1) { |
| 1520 | for (size_t byte = 0; byte < DQS_NUMBER; byte++) |
| 1521 | write32(&ch[chn].phy.shu[0].rk[rank].b[byte].dq[0], |
| 1522 | byte_dly_cell[byte]); |
| 1523 | } |
| 1524 | |
| 1525 | if (type != TX_WIN_DQ_ONLY) |
| 1526 | continue; |
| 1527 | |
| 1528 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[rank].fine_tune, 0x3f3f3f3f, |
| 1529 | (dqdly_tune[0].fine_tune << 8) | (dqdly_tune[1].fine_tune << 0) | |
| 1530 | (dqmdly_tune[0].fine_tune << 24) | (dqmdly_tune[1].fine_tune << 16)); |
| 1531 | |
| 1532 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[rank].dqs2dq_cal1, 0x7ff | (0x7ff << 16), |
| 1533 | (dqdly_tune[0].fine_tune << 0) | (dqdly_tune[1].fine_tune << 16)); |
| 1534 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[rank].dqs2dq_cal2, 0x7ff | (0x7ff << 16), |
| 1535 | (dqdly_tune[0].fine_tune << 0) | (dqdly_tune[1].fine_tune << 16)); |
| 1536 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[rank].dqs2dq_cal5, 0x7ff | (0x7ff << 16), |
| 1537 | (dqmdly_tune[0].fine_tune << 0) | (dqmdly_tune[1].fine_tune << 16)); |
| 1538 | } |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1539 | } |
| 1540 | |
mtk11195 | 16ad2d7 | 2019-01-23 11:41:19 +0800 | [diff] [blame] | 1541 | static void dramc_set_rx_best_dly_factor(u8 chn, u8 rank, |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1542 | struct win_perbit_dly *dly, s32 *dqsdly_byte, s32 *dqmdly_byte) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1543 | { |
| 1544 | u32 value; |
| 1545 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1546 | /* set dqs delay, (dqm delay) */ |
| 1547 | for (u8 byte = 0; byte < DQS_NUMBER; byte++) { |
| 1548 | value = (dqsdly_byte[byte] << 24) | (dqsdly_byte[byte] << 16) | |
| 1549 | (dqmdly_byte[byte] << 8) | (dqmdly_byte[byte] << 0); |
| 1550 | clrsetbits_le32(&ch[chn].phy.shu[0].rk[rank].b[byte].dq[6], 0x7f7f3f3f, value); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1551 | } |
| 1552 | dram_phy_reset(chn); |
| 1553 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1554 | /* set dq delay */ |
| 1555 | for (u8 byte = 0; byte < DQS_NUMBER; byte++) { |
| 1556 | for (u8 bit = 0; bit < DQS_BIT_NUMBER; bit += 2) { |
| 1557 | u8 index = bit + byte * DQS_BIT_NUMBER; |
| 1558 | u8 dq_num = 2 + bit / 2; |
| 1559 | value = (dly[index + 1].best_dqdly << 24) | |
| 1560 | (dly[index + 1].best_dqdly << 16) | |
| 1561 | (dly[index].best_dqdly << 8) | (dly[index].best_dqdly << 0); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1562 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1563 | clrsetbits_le32(&ch[chn].phy.shu[0].rk[rank].b[byte].dq[dq_num], |
| 1564 | 0x3f3f3f3f, value); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1565 | } |
| 1566 | } |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1567 | } |
| 1568 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1569 | static void dramc_set_dqdqs_dly(u8 chn, u8 rank, enum CAL_TYPE type, u8 *small_value, s32 dly) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1570 | { |
Yu-Ping Wu | a39cd99 | 2019-08-29 10:54:14 +0800 | [diff] [blame] | 1571 | if (type == RX_WIN_RD_DQC || type == RX_WIN_TEST_ENG) |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1572 | dramc_set_rx_dqdqs_dly(chn, rank, dly); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1573 | else |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1574 | dramc_set_tx_dly_factor(chn, rank, type, small_value, dly); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1575 | } |
| 1576 | |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1577 | static void dramc_set_tx_dly_center(struct per_byte_dly *center_dly, |
| 1578 | const struct win_perbit_dly *vref_dly) |
| 1579 | { |
| 1580 | int index; |
| 1581 | struct per_byte_dly *dly; |
| 1582 | |
| 1583 | for (u8 byte = 0; byte < DQS_NUMBER; byte++) { |
| 1584 | dly = ¢er_dly[byte]; |
| 1585 | dly->min_center = 0xffff; |
| 1586 | dly->max_center = 0; |
| 1587 | |
| 1588 | for (u8 bit = 0; bit < DQS_BIT_NUMBER; bit++) { |
| 1589 | index = bit + 8 * byte; |
| 1590 | if (vref_dly[index].win_center < dly->min_center) |
| 1591 | dly->min_center = vref_dly[index].win_center; |
| 1592 | if (vref_dly[index].win_center > dly->max_center) |
| 1593 | dly->max_center = vref_dly[index].win_center; |
| 1594 | } |
| 1595 | dramc_dbg("center_dly[%d].min_center = %d, " |
| 1596 | "center_dly[%d].max_center = %d\n", |
| 1597 | byte, center_dly[byte].min_center, |
| 1598 | byte, center_dly[byte].max_center); |
| 1599 | } |
| 1600 | } |
| 1601 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1602 | static void dramc_set_tx_best_dly(u8 chn, u8 rank, bool bypass_tx, |
| 1603 | struct win_perbit_dly *vref_dly, enum CAL_TYPE type, u8 freq_group, |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1604 | u16 *tx_dq_precal_result, u16 dly_cell_unit, const struct sdram_params *params, |
| 1605 | const bool fast_calib) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1606 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1607 | int index, clock_rate; |
| 1608 | u8 use_delay_cell; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1609 | u32 byte_dly_cell[DQS_NUMBER] = {0}; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1610 | struct per_byte_dly center_dly[DQS_NUMBER]; |
| 1611 | u16 tune_diff, dq_delay_cell[DQ_DATA_WIDTH]; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1612 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1613 | switch (freq_group) { |
| 1614 | case LP4X_DDR1600: |
| 1615 | clock_rate = 800; |
| 1616 | break; |
| 1617 | case LP4X_DDR2400: |
| 1618 | clock_rate = 1200; |
| 1619 | break; |
| 1620 | case LP4X_DDR3200: |
| 1621 | clock_rate = 1600; |
| 1622 | break; |
| 1623 | case LP4X_DDR3600: |
| 1624 | clock_rate = 1866; |
| 1625 | break; |
| 1626 | default: |
| 1627 | die("Invalid DDR frequency group %u\n", freq_group); |
| 1628 | return; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1629 | } |
| 1630 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1631 | if (type == TX_WIN_DQ_ONLY && get_freq_fsq(freq_group) == FSP_1) |
| 1632 | use_delay_cell = 1; |
| 1633 | else |
| 1634 | use_delay_cell = 0; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1635 | |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1636 | if (fast_calib && bypass_tx) { |
| 1637 | dramc_dbg("bypass TX\n"); |
| 1638 | for (u8 byte = 0; byte < DQS_NUMBER; byte++) { |
| 1639 | center_dly[byte].min_center = params->tx_center_min[chn][rank][byte]; |
| 1640 | center_dly[byte].max_center = params->tx_center_max[chn][rank][byte]; |
| 1641 | for (u8 bit = 0; bit < DQS_BIT_NUMBER; bit++) { |
| 1642 | index = bit + 8 * byte; |
| 1643 | vref_dly[index].win_center = |
| 1644 | params->tx_win_center[chn][rank][index]; |
| 1645 | vref_dly[index].best_first = |
| 1646 | params->tx_first_pass[chn][rank][index]; |
| 1647 | vref_dly[index].best_last = |
| 1648 | params->tx_last_pass[chn][rank][index]; |
| 1649 | } |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1650 | } |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1651 | } else { |
| 1652 | dramc_set_tx_dly_center(center_dly, vref_dly); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1653 | } |
| 1654 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1655 | for (u8 byte = 0; byte < DQS_NUMBER; byte++) { |
| 1656 | if (use_delay_cell == 0) { |
| 1657 | center_dly[byte].final_dly = (center_dly[byte].min_center + |
| 1658 | center_dly[byte].max_center) >> 1; |
| 1659 | tx_dq_precal_result[byte] = center_dly[byte].final_dly; |
| 1660 | } else { |
| 1661 | center_dly[byte].final_dly = center_dly[byte].min_center; |
| 1662 | tx_dq_precal_result[byte] = (center_dly[byte].min_center |
| 1663 | + center_dly[byte].max_center) >> 1; |
| 1664 | |
| 1665 | for (u8 bit = 0; bit < DQS_BIT_NUMBER; bit++) { |
| 1666 | index = bit + 8 * byte; |
| 1667 | tune_diff = vref_dly[index].win_center - |
| 1668 | center_dly[byte].min_center; |
| 1669 | dq_delay_cell[index] = ((tune_diff * 100000000) / |
| 1670 | (clock_rate / 2 * 64)) / dly_cell_unit; |
| 1671 | byte_dly_cell[byte] |= (dq_delay_cell[index] << (bit * 4)); |
| 1672 | } |
| 1673 | } |
| 1674 | } |
| 1675 | |
| 1676 | dramc_set_tx_best_dly_factor(chn, rank, type, center_dly, tx_dq_precal_result, |
| 1677 | use_delay_cell, byte_dly_cell); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1678 | } |
| 1679 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1680 | static int dramc_set_rx_best_dly(u8 chn, u8 rank, struct win_perbit_dly *perbit_dly) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1681 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1682 | u8 bit_first, bit_last; |
| 1683 | u16 u2TmpDQMSum; |
| 1684 | s32 dqsdly_byte[DQS_NUMBER] = {0x0}, dqm_dly_byte[DQS_NUMBER] = {0x0}; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1685 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1686 | for (u8 byte = 0; byte < DQS_NUMBER; byte++) { |
| 1687 | u2TmpDQMSum = 0; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1688 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1689 | bit_first = DQS_BIT_NUMBER * byte; |
| 1690 | bit_last = DQS_BIT_NUMBER * byte + DQS_BIT_NUMBER - 1; |
| 1691 | dqsdly_byte[byte] = 64; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1692 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1693 | for (u8 bit = bit_first; bit <= bit_last; bit++) { |
| 1694 | if (perbit_dly[bit].win_center < dqsdly_byte[byte]) |
| 1695 | dqsdly_byte[byte] = perbit_dly[bit].win_center; |
| 1696 | } |
| 1697 | dqsdly_byte[byte] = (dqsdly_byte[byte] > 0) ? 0 : -dqsdly_byte[byte]; |
| 1698 | |
| 1699 | for (u8 bit = bit_first; bit <= bit_last; bit++) { |
| 1700 | perbit_dly[bit].best_dqdly = dqsdly_byte[byte] + |
| 1701 | perbit_dly[bit].win_center; |
| 1702 | u2TmpDQMSum += perbit_dly[bit].best_dqdly; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1703 | } |
| 1704 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1705 | dqm_dly_byte[byte] = u2TmpDQMSum / DQS_BIT_NUMBER; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1706 | } |
| 1707 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1708 | dramc_set_rx_best_dly_factor(chn, rank, perbit_dly, dqsdly_byte, dqm_dly_byte); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1709 | return 0; |
| 1710 | } |
| 1711 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1712 | static void dramc_get_vref_prop(u8 rank, enum CAL_TYPE type, u8 fsp, |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1713 | u8 *vref_scan_en, u8 *vref_begin, u8 *vref_end) |
| 1714 | { |
| 1715 | if (type == RX_WIN_TEST_ENG && rank == RANK_0) { |
| 1716 | *vref_scan_en = 1; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1717 | if (fsp == FSP_0) |
| 1718 | *vref_begin = 0x18; |
| 1719 | else |
| 1720 | *vref_begin = 0; |
| 1721 | *vref_end = RX_VREF_END; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1722 | } else if (type == TX_WIN_DQ_ONLY) { |
| 1723 | *vref_scan_en = 1; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1724 | if (fsp == FSP_0) { |
| 1725 | *vref_begin = 27 - 5; |
| 1726 | *vref_end = 27 + 5; |
| 1727 | } else { |
| 1728 | *vref_begin = TX_VREF_BEGIN; |
| 1729 | *vref_end = TX_VREF_END; |
| 1730 | } |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1731 | } else { |
| 1732 | *vref_scan_en = 0; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1733 | *vref_begin = 0; |
| 1734 | *vref_end = 1; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1735 | } |
| 1736 | } |
| 1737 | |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1738 | static u32 dram_k_perbit(u8 chn, enum CAL_TYPE type) |
| 1739 | { |
| 1740 | u32 err_value; |
| 1741 | |
| 1742 | if (type == RX_WIN_RD_DQC) { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1743 | err_value = dramc_rx_rd_dqc_run(chn); |
| 1744 | } else if (type == RX_WIN_TEST_ENG) { |
| 1745 | err_value = dramc_engine2_run(chn, TE_OP_WRITE_READ_CHECK); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1746 | } else { |
| 1747 | dramc_engine2_setpat(chn, true); |
| 1748 | err_value = dramc_engine2_run(chn, TE_OP_WRITE_READ_CHECK); |
| 1749 | dramc_engine2_setpat(chn, false); |
| 1750 | err_value |= dramc_engine2_run(chn, TE_OP_WRITE_READ_CHECK); |
| 1751 | } |
| 1752 | return err_value; |
| 1753 | } |
| 1754 | |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1755 | static void dramc_window_perbit_cal_partial(u8 chn, u8 rank, |
| 1756 | s16 dly_begin, s16 dly_end, s16 dly_step, |
| 1757 | enum CAL_TYPE type, u8 *small_value, u8 vref_scan_enable, |
| 1758 | struct win_perbit_dly *win_perbit) |
| 1759 | { |
| 1760 | u32 finish_bit = 0; |
| 1761 | |
| 1762 | for (s16 dly = dly_begin; dly < dly_end; dly += dly_step) { |
| 1763 | dramc_set_dqdqs_dly(chn, rank, type, small_value, dly); |
| 1764 | |
| 1765 | u32 err_value = dram_k_perbit(chn, type); |
| 1766 | if (!vref_scan_enable) |
| 1767 | dramc_dbg("%d ", dly); |
| 1768 | |
| 1769 | for (size_t bit = 0; bit < DQ_DATA_WIDTH; bit++) { |
| 1770 | bool bit_fail = (err_value & ((u32) 1 << bit)) != 0; |
| 1771 | |
| 1772 | /* pass window bigger than 7, |
| 1773 | * consider as real pass window. |
| 1774 | */ |
| 1775 | if (dramc_check_dqdqs_win(&(win_perbit[bit]), |
| 1776 | dly, dly_end, bit_fail) > 7) |
| 1777 | finish_bit |= (1 << bit); |
| 1778 | |
| 1779 | if (vref_scan_enable) |
| 1780 | continue; |
| 1781 | |
| 1782 | dramc_dbg("%s", bit_fail ? "x" : "o"); |
| 1783 | if (bit % DQS_BIT_NUMBER == 7) |
| 1784 | dramc_dbg(" "); |
| 1785 | } |
| 1786 | |
| 1787 | if (!vref_scan_enable) |
| 1788 | dramc_dbg(" [MSB]\n"); |
| 1789 | |
| 1790 | if (finish_bit == 0xffff && (err_value & 0xffff) == 0xffff) { |
| 1791 | dramc_dbg("all bits window found, " |
| 1792 | "early break! delay=%#x\n", dly); |
| 1793 | break; |
| 1794 | } |
| 1795 | } |
| 1796 | } |
| 1797 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1798 | static u8 dramc_window_perbit_cal(u8 chn, u8 rank, u8 freq_group, |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1799 | enum CAL_TYPE type, const struct sdram_params *params, |
| 1800 | const bool fast_calib) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1801 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1802 | u8 vref = 0, vref_begin = 0, vref_end = 1, vref_step = 1, vref_use = 0; |
| 1803 | u8 vref_scan_enable = 0, small_reg_value = 0xff; |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1804 | s16 dly_begin = 0, dly_end = 0, dly_step = 1; |
| 1805 | u32 dummy_rd_bak_engine2 = 0, finish_bit, win_min_max = 0; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1806 | static u16 dq_precal_result[DQS_NUMBER]; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1807 | struct vref_perbit_dly vref_dly; |
| 1808 | struct win_perbit_dly win_perbit[DQ_DATA_WIDTH]; |
| 1809 | u16 dly_cell_unit = params->delay_cell_unit; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1810 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1811 | u8 fsp = get_freq_fsq(freq_group); |
| 1812 | u8 vref_range = !fsp; |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1813 | bool bypass_tx = !fsp; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1814 | |
| 1815 | dramc_get_vref_prop(rank, type, fsp, |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1816 | &vref_scan_enable, &vref_begin, &vref_end); |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1817 | dramc_get_dly_range(chn, rank, type, freq_group, dq_precal_result, |
| 1818 | &dly_begin, &dly_end, params); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1819 | |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1820 | if (fast_calib) { |
| 1821 | if (type == RX_WIN_TEST_ENG && vref_scan_enable == 1) { |
| 1822 | vref_begin = params->rx_vref[chn]; |
| 1823 | vref_end = vref_begin + 1; |
| 1824 | dramc_dbg("bypass RX vref: %d\n", vref_begin); |
| 1825 | } else if (type == TX_WIN_DQ_ONLY) { |
| 1826 | vref_begin = params->tx_vref[chn][rank]; |
| 1827 | vref_end = vref_begin + 1; |
| 1828 | dramc_dbg("bypass TX vref: %d\n", vref_begin); |
| 1829 | } |
| 1830 | vref_dly.best_vref = vref_begin; |
| 1831 | } |
| 1832 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1833 | if ((type == RX_WIN_RD_DQC || type == RX_WIN_TEST_ENG) && fsp == FSP_0) |
| 1834 | dly_step = 2; |
| 1835 | |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1836 | dramc_dbg("[channel %d] [rank %d] type: %d, vref_enable: %d, vref range[%d : %d]\n", |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1837 | chn, rank, type, vref_scan_enable, vref_begin, vref_end); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1838 | |
Yu-Ping Wu | a39cd99 | 2019-08-29 10:54:14 +0800 | [diff] [blame] | 1839 | if (type == TX_WIN_DQ_ONLY || type == TX_WIN_DQ_DQM) { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1840 | for (size_t byte = 0; byte < 2; byte++) { |
| 1841 | write32(&ch[chn].phy.shu[0].rk[rank].b[byte].dq[0], 0); |
| 1842 | clrbits_le32(&ch[chn].phy.shu[0].rk[rank].b[byte].dq[1], |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1843 | 0xf); |
| 1844 | } |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1845 | setbits_le32(&ch[chn].phy.misc_ctrl1, 0x1 << 7); |
| 1846 | setbits_le32(&ch[chn].ao.dqsoscr, 0x1 << 7); |
| 1847 | if (fsp == FSP_1) |
| 1848 | vref_step = 2; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1849 | } |
| 1850 | |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1851 | if (fast_calib && bypass_tx && |
| 1852 | (type == TX_WIN_DQ_ONLY || type == TX_WIN_DQ_DQM)) { |
| 1853 | dramc_set_tx_best_dly(chn, rank, true, vref_dly.perbit_dly, |
| 1854 | type, freq_group, dq_precal_result, dly_cell_unit, |
| 1855 | params, fast_calib); |
| 1856 | |
| 1857 | if (vref_scan_enable) |
| 1858 | dramc_set_vref(chn, rank, type, vref_dly.best_vref); |
| 1859 | return 0; |
| 1860 | } |
| 1861 | |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1862 | if (type == RX_WIN_RD_DQC) { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1863 | dramc_rx_rd_dqc_init(chn, rank); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1864 | } else { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1865 | if (type == RX_WIN_TEST_ENG) |
| 1866 | dramc_rx_vref_pre_setting(chn); |
| 1867 | dummy_rd_bak_engine2 = read32(&ch[chn].ao.dummy_rd); |
| 1868 | dramc_engine2_init(chn, rank, TEST2_1_CAL, TEST2_2_CAL, false); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1869 | } |
| 1870 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1871 | vref_dly.max_win_sum = 0; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1872 | for (vref = vref_begin; vref < vref_end; vref += vref_step) { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1873 | small_reg_value = 0xff; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1874 | finish_bit = 0; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1875 | if (type == TX_WIN_DQ_ONLY) |
| 1876 | vref_use = vref | (vref_range << 6); |
| 1877 | else |
| 1878 | vref_use = vref; |
| 1879 | |
| 1880 | for (size_t bit = 0; bit < DQ_DATA_WIDTH; bit++) { |
| 1881 | win_perbit[bit].first_pass = PASS_RANGE_NA; |
| 1882 | win_perbit[bit].last_pass = PASS_RANGE_NA; |
| 1883 | win_perbit[bit].best_first = PASS_RANGE_NA; |
| 1884 | win_perbit[bit].best_last = PASS_RANGE_NA; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1885 | } |
| 1886 | |
| 1887 | if (vref_scan_enable) |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1888 | dramc_set_vref(chn, rank, type, vref_use); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1889 | |
Yu-Ping Wu | a39cd99 | 2019-08-29 10:54:14 +0800 | [diff] [blame] | 1890 | if (type == RX_WIN_RD_DQC || type == RX_WIN_TEST_ENG) { |
mtk11195 | 16ad2d7 | 2019-01-23 11:41:19 +0800 | [diff] [blame] | 1891 | dramc_set_rx_dly_factor(chn, rank, |
| 1892 | RX_DQM, FIRST_DQ_DELAY); |
| 1893 | dramc_set_rx_dly_factor(chn, rank, |
| 1894 | RX_DQ, FIRST_DQ_DELAY); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1895 | } |
| 1896 | |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1897 | if (fast_calib && |
| 1898 | (type == RX_WIN_RD_DQC || type == RX_WIN_TEST_ENG)) { |
| 1899 | dramc_dbg("bypass RX params\n"); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1900 | for (size_t bit = 0; bit < DQ_DATA_WIDTH; bit++) { |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1901 | win_perbit[bit].best_first = |
| 1902 | params->rx_firspass[chn][rank][bit]; |
| 1903 | win_perbit[bit].best_last = |
| 1904 | params->rx_lastpass[chn][rank][bit]; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1905 | } |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1906 | } else { |
| 1907 | dramc_window_perbit_cal_partial(chn, rank, |
| 1908 | dly_begin, dly_end, dly_step, |
| 1909 | type, &small_reg_value, |
| 1910 | vref_scan_enable, win_perbit); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1911 | } |
| 1912 | |
| 1913 | for (size_t bit = 0; bit < DQ_DATA_WIDTH; bit++) |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1914 | dramc_dbg("Dq[%zd] win width (%d ~ %d) %d\n", bit, |
| 1915 | win_perbit[bit].best_first, win_perbit[bit].best_last, |
| 1916 | win_perbit[bit].best_last - win_perbit[bit].best_first); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1917 | |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1918 | if (dramk_calc_best_vref(type, vref_use, &vref_dly, |
| 1919 | win_perbit, &win_min_max)) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1920 | break; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1921 | } |
| 1922 | |
Yu-Ping Wu | 093d8ea | 2019-10-02 13:06:42 +0800 | [diff] [blame] | 1923 | if (type == RX_WIN_RD_DQC) |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1924 | dramc_rx_rd_dqc_end(chn); |
Yu-Ping Wu | 093d8ea | 2019-10-02 13:06:42 +0800 | [diff] [blame] | 1925 | else |
| 1926 | dramc_engine2_end(chn, dummy_rd_bak_engine2); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1927 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1928 | if (vref_scan_enable && type == RX_WIN_TEST_ENG) |
| 1929 | dramc_set_vref(chn, rank, type, vref_dly.best_vref); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1930 | |
Yu-Ping Wu | a39cd99 | 2019-08-29 10:54:14 +0800 | [diff] [blame] | 1931 | if (type == RX_WIN_RD_DQC || type == RX_WIN_TEST_ENG) |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1932 | dramc_set_rx_best_dly(chn, rank, vref_dly.perbit_dly); |
| 1933 | else |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1934 | dramc_set_tx_best_dly(chn, rank, false, |
| 1935 | vref_dly.perbit_dly, type, freq_group, |
| 1936 | dq_precal_result, dly_cell_unit, params, fast_calib); |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1937 | |
| 1938 | if (vref_scan_enable && type == TX_WIN_DQ_ONLY) |
| 1939 | dramc_set_vref(chn, rank, type, vref_dly.best_vref); |
| 1940 | |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 1941 | return 0; |
| 1942 | } |
| 1943 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1944 | static void dramc_dle_factor_handler(u8 chn, u8 val, u8 freq_group) |
Huayang Duan | 6202d1b | 2018-09-26 17:50:44 +0800 | [diff] [blame] | 1945 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1946 | u8 start_ext2 = 0, start_ext3 = 0, last_ext2 = 0, last_ext3 = 0; |
| 1947 | |
Huayang Duan | 6202d1b | 2018-09-26 17:50:44 +0800 | [diff] [blame] | 1948 | val = MAX(val, 2); |
| 1949 | clrsetbits_le32(&ch[chn].ao.shu[0].conf[1], |
| 1950 | SHU_CONF1_DATLAT_MASK | SHU_CONF1_DATLAT_DSEL_MASK | |
| 1951 | SHU_CONF1_DATLAT_DSEL_PHY_MASK, |
| 1952 | (val << SHU_CONF1_DATLAT_SHIFT) | |
| 1953 | ((val - 2) << SHU_CONF1_DATLAT_DSEL_SHIFT) | |
| 1954 | ((val - 2) << SHU_CONF1_DATLAT_DSEL_PHY_SHIFT)); |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1955 | |
| 1956 | if (freq_group == LP4X_DDR3200 || freq_group == LP4X_DDR3600) |
| 1957 | start_ext2 = 1; |
| 1958 | |
| 1959 | if (val >= 24) |
| 1960 | last_ext2 = last_ext3 = 1; |
| 1961 | else if (val >= 18) |
| 1962 | last_ext2 = 1; |
| 1963 | |
| 1964 | clrsetbits_le32(&ch[chn].ao.shu[0].pipe, (0x1 << 31) | (0x1 << 30) | (0x1 << 29) | |
| 1965 | (0x1 << 28) | (0x1 << 27) | (0x1 << 26), |
| 1966 | (0x1 << 31) | (0x1 << 30) | (start_ext2 << 29) | |
| 1967 | (last_ext2 << 28) | (start_ext3 << 27) | (last_ext3 << 26)); |
Huayang Duan | 6202d1b | 2018-09-26 17:50:44 +0800 | [diff] [blame] | 1968 | dram_phy_reset(chn); |
| 1969 | } |
| 1970 | |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1971 | static u8 dramc_rx_datlat_cal(u8 chn, u8 rank, u8 freq_group, |
| 1972 | const struct sdram_params *params, const bool fast_calib) |
Huayang Duan | 6202d1b | 2018-09-26 17:50:44 +0800 | [diff] [blame] | 1973 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1974 | u32 datlat, begin = 0, first = 0, sum = 0, best_step; |
| 1975 | u32 datlat_start = 7; |
Huayang Duan | 6202d1b | 2018-09-26 17:50:44 +0800 | [diff] [blame] | 1976 | |
| 1977 | best_step = read32(&ch[chn].ao.shu[0].conf[1]) & SHU_CONF1_DATLAT_MASK; |
| 1978 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1979 | dramc_dbg("[DATLAT] start. CH%d RK%d DATLAT Default: 0x%x\n", |
| 1980 | chn, rank, best_step); |
Huayang Duan | 6202d1b | 2018-09-26 17:50:44 +0800 | [diff] [blame] | 1981 | |
| 1982 | u32 dummy_rd_backup = read32(&ch[chn].ao.dummy_rd); |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 1983 | dramc_engine2_init(chn, rank, TEST2_1_CAL, TEST2_2_CAL, false); |
Huayang Duan | 6202d1b | 2018-09-26 17:50:44 +0800 | [diff] [blame] | 1984 | |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1985 | if (fast_calib) { |
| 1986 | best_step = params->rx_datlat[chn][rank]; |
| 1987 | dramc_dbg("bypass DATLAT, best_step: %d\n", best_step); |
| 1988 | } else { |
| 1989 | for (datlat = datlat_start; datlat < DATLAT_TAP_NUMBER; datlat++) { |
| 1990 | dramc_dle_factor_handler(chn, datlat, freq_group); |
Huayang Duan | 6202d1b | 2018-09-26 17:50:44 +0800 | [diff] [blame] | 1991 | |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 1992 | u32 err = dramc_engine2_run(chn, TE_OP_WRITE_READ_CHECK); |
| 1993 | if (err == 0) { |
| 1994 | if (begin == 0) { |
| 1995 | first = datlat; |
| 1996 | begin = 1; |
| 1997 | } |
| 1998 | if (begin == 1) { |
| 1999 | sum++; |
| 2000 | if (sum > 4) |
| 2001 | break; |
| 2002 | } |
| 2003 | } else { |
| 2004 | if (begin == 1) |
| 2005 | begin = 0xff; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2006 | } |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 2007 | |
| 2008 | dramc_dbg("Datlat=%2d, err_value=0x%4x, sum=%d\n", datlat, err, sum); |
Huayang Duan | 6202d1b | 2018-09-26 17:50:44 +0800 | [diff] [blame] | 2009 | } |
| 2010 | |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 2011 | dramc_engine2_end(chn, dummy_rd_backup); |
| 2012 | |
| 2013 | assert(sum != 0); |
| 2014 | |
| 2015 | if (sum <= 3) |
| 2016 | best_step = first + (sum >> 1); |
| 2017 | else |
| 2018 | best_step = first + 2; |
| 2019 | dramc_dbg("First_step=%d, total pass=%d, best_step=%d\n", |
| 2020 | begin, sum, best_step); |
Huayang Duan | 6202d1b | 2018-09-26 17:50:44 +0800 | [diff] [blame] | 2021 | } |
| 2022 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2023 | dramc_dle_factor_handler(chn, best_step, freq_group); |
| 2024 | |
| 2025 | clrsetbits_le32(&ch[chn].ao.padctrl, 0x3 | (0x1 << 3), |
Huayang Duan | 6202d1b | 2018-09-26 17:50:44 +0800 | [diff] [blame] | 2026 | (0x1 << PADCTRL_DQIENQKEND_SHIFT) | |
| 2027 | (0x1 << PADCTRL_DQIENLATEBEGIN_SHIFT)); |
| 2028 | |
| 2029 | return (u8) best_step; |
| 2030 | } |
| 2031 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2032 | static void dramc_dual_rank_rx_datlat_cal(u8 chn, u8 freq_group, u8 datlat0, u8 datlat1) |
Huayang Duan | 6202d1b | 2018-09-26 17:50:44 +0800 | [diff] [blame] | 2033 | { |
| 2034 | u8 final_datlat = MAX(datlat0, datlat1); |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2035 | dramc_dle_factor_handler(chn, final_datlat, freq_group); |
Huayang Duan | 6202d1b | 2018-09-26 17:50:44 +0800 | [diff] [blame] | 2036 | } |
| 2037 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2038 | static void dramc_rx_dqs_gating_post_process(u8 chn, u8 freq_group) |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 2039 | { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2040 | s8 dqsinctl; |
| 2041 | u32 read_dqsinctl, rankinctl_root, reg_tx_dly_dqsgated_min = 3; |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 2042 | u8 txdly_cal_min = 0xff, txdly_cal_max = 0, tx_dly_dqs_gated = 0; |
| 2043 | u32 best_coarse_tune2t[RANK_MAX][DQS_NUMBER]; |
| 2044 | u32 best_coarse_tune2t_p1[RANK_MAX][DQS_NUMBER]; |
| 2045 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2046 | if (freq_group == LP4X_DDR3200 || freq_group == LP4X_DDR3600) |
| 2047 | reg_tx_dly_dqsgated_min = 2; |
| 2048 | else |
| 2049 | reg_tx_dly_dqsgated_min = 1; |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 2050 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2051 | /* get TXDLY_Cal_min and TXDLY_Cal_max value */ |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 2052 | for (size_t rank = 0; rank < RANK_MAX; rank++) { |
| 2053 | u32 dqsg0 = read32(&ch[chn].ao.shu[0].rk[rank].selph_dqsg0); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 2054 | for (size_t dqs = 0; dqs < DQS_NUMBER; dqs++) { |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2055 | best_coarse_tune2t[rank][dqs] = (dqsg0 >> (dqs * 8)) & 0x7; |
| 2056 | best_coarse_tune2t_p1[rank][dqs] = (dqsg0 >> (dqs * 8 + 4)) & 0x7; |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 2057 | dramc_dbg("Rank%zd best DQS%zd dly(2T,(P1)2T)=(%d, %d)\n", |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2058 | rank, dqs, best_coarse_tune2t[rank][dqs], |
| 2059 | best_coarse_tune2t_p1[rank][dqs]); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 2060 | |
| 2061 | tx_dly_dqs_gated = best_coarse_tune2t[rank][dqs]; |
| 2062 | txdly_cal_min = MIN(txdly_cal_min, tx_dly_dqs_gated); |
| 2063 | |
| 2064 | tx_dly_dqs_gated = best_coarse_tune2t_p1[rank][dqs]; |
| 2065 | txdly_cal_max = MAX(txdly_cal_max, tx_dly_dqs_gated); |
| 2066 | } |
| 2067 | } |
| 2068 | |
| 2069 | dqsinctl = reg_tx_dly_dqsgated_min - txdly_cal_min; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2070 | dramc_dbg("Dqsinctl:%d, dqsgated_min %d, txdly_cal_min %d, txdly_cal_max %d\n", |
| 2071 | dqsinctl, reg_tx_dly_dqsgated_min, txdly_cal_min, txdly_cal_max); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 2072 | |
| 2073 | if (dqsinctl != 0) { |
| 2074 | txdly_cal_min += dqsinctl; |
| 2075 | txdly_cal_max += dqsinctl; |
| 2076 | |
| 2077 | for (size_t rank = 0; rank < RANK_MAX; rank++) { |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 2078 | dramc_dbg("Rank: %zd\n", rank); |
| 2079 | for (size_t dqs = 0; dqs < DQS_NUMBER; dqs++) { |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 2080 | best_coarse_tune2t[rank][dqs] += dqsinctl; |
| 2081 | best_coarse_tune2t_p1[rank][dqs] += dqsinctl; |
| 2082 | |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 2083 | dramc_dbg("Best DQS%zd dly(2T) = (%d)\n", |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2084 | dqs, best_coarse_tune2t[rank][dqs]); |
Huayang Duan | 2b5067b | 2018-09-26 17:39:29 +0800 | [diff] [blame] | 2085 | dramc_dbg("Best DQS%zd P1 dly(2T) = (%d)\n", |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2086 | dqs, best_coarse_tune2t_p1[rank][dqs]); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 2087 | } |
| 2088 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2089 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[rank].selph_dqsg0, |
| 2090 | 0x77777777, |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 2091 | (best_coarse_tune2t[rank][0] << 0) | |
| 2092 | (best_coarse_tune2t[rank][1] << 8) | |
| 2093 | (best_coarse_tune2t_p1[rank][0] << 4) | |
| 2094 | (best_coarse_tune2t_p1[rank][1] << 12)); |
| 2095 | } |
| 2096 | } |
| 2097 | |
| 2098 | read_dqsinctl = (read32(&ch[chn].ao.shu[0].rk[0].dqsctl) & |
| 2099 | SHURK_DQSCTL_DQSINCTL_MASK) - dqsinctl; |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2100 | rankinctl_root = (read_dqsinctl >= 2) ? (read_dqsinctl - 2) : 0; |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 2101 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2102 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[0].dqsctl, 0xf, read_dqsinctl << 0); |
| 2103 | clrsetbits_le32(&ch[chn].ao.shu[0].rk[1].dqsctl, 0xf, read_dqsinctl << 0); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 2104 | clrsetbits_le32(&ch[chn].ao.shu[0].rankctl, |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2105 | (0xf << 28) | (0xf << 20) | (0xf << 24) | 0xf, |
| 2106 | (read_dqsinctl << 28) | (rankinctl_root << 20) | |
| 2107 | (rankinctl_root << 24) | rankinctl_root); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 2108 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2109 | u8 ROEN = read32(&ch[chn].ao.shu[0].odtctrl) & 0x1; |
| 2110 | clrsetbits_le32(&ch[chn].ao.shu[0].rodtenstb, (0xffff << 8) | (0x3f << 2) | (0x1), |
| 2111 | (0xff << 8) | (0x9 << 2) | ROEN); |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 2112 | } |
| 2113 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2114 | void dramc_calibrate_all_channels(const struct sdram_params *pams, u8 freq_group) |
Huayang Duan | cede791 | 2018-09-26 16:10:42 +0800 | [diff] [blame] | 2115 | { |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 2116 | bool fast_calib; |
| 2117 | switch (pams->source) { |
| 2118 | case DRAMC_PARAM_SOURCE_SDRAM_CONFIG: |
| 2119 | fast_calib = false; |
| 2120 | break; |
| 2121 | case DRAMC_PARAM_SOURCE_FLASH: |
| 2122 | fast_calib = true; |
| 2123 | break; |
| 2124 | default: |
| 2125 | die("Invalid DRAM param source %u\n", pams->source); |
| 2126 | return; |
| 2127 | } |
| 2128 | |
Huayang Duan | 6202d1b | 2018-09-26 17:50:44 +0800 | [diff] [blame] | 2129 | u8 rx_datlat[RANK_MAX] = {0}; |
Huayang Duan | cede791 | 2018-09-26 16:10:42 +0800 | [diff] [blame] | 2130 | for (u8 chn = 0; chn < CHANNEL_MAX; chn++) { |
| 2131 | for (u8 rk = RANK_0; rk < RANK_MAX; rk++) { |
| 2132 | dramc_show("Start K ch:%d, rank:%d\n", chn, rk); |
mtk11195 | 16ad2d7 | 2019-01-23 11:41:19 +0800 | [diff] [blame] | 2133 | dramc_auto_refresh_switch(chn, false); |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 2134 | dramc_cmd_bus_training(chn, rk, freq_group, pams, |
| 2135 | fast_calib); |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2136 | dramc_write_leveling(chn, rk, freq_group, pams->wr_level); |
mtk11195 | 16ad2d7 | 2019-01-23 11:41:19 +0800 | [diff] [blame] | 2137 | dramc_auto_refresh_switch(chn, true); |
Huayang Duan | 846be44 | 2019-08-30 18:01:19 +0800 | [diff] [blame] | 2138 | dramc_rx_dqs_gating_cal(chn, rk, freq_group, pams, |
| 2139 | fast_calib); |
| 2140 | dramc_window_perbit_cal(chn, rk, freq_group, |
| 2141 | RX_WIN_RD_DQC, pams, fast_calib); |
| 2142 | dramc_window_perbit_cal(chn, rk, freq_group, |
| 2143 | TX_WIN_DQ_DQM, pams, fast_calib); |
| 2144 | dramc_window_perbit_cal(chn, rk, freq_group, |
| 2145 | TX_WIN_DQ_ONLY, pams, fast_calib); |
| 2146 | rx_datlat[rk] = dramc_rx_datlat_cal(chn, rk, freq_group, |
| 2147 | pams, fast_calib); |
| 2148 | dramc_window_perbit_cal(chn, rk, freq_group, |
| 2149 | RX_WIN_TEST_ENG, pams, fast_calib); |
Huayang Duan | cede791 | 2018-09-26 16:10:42 +0800 | [diff] [blame] | 2150 | } |
Huayang Duan | fcdbce2 | 2018-09-26 16:33:18 +0800 | [diff] [blame] | 2151 | |
Huayang Duan | c157ee9 | 2019-07-14 15:46:08 +0800 | [diff] [blame] | 2152 | dramc_rx_dqs_gating_post_process(chn, freq_group); |
| 2153 | dramc_dual_rank_rx_datlat_cal(chn, freq_group, rx_datlat[0], rx_datlat[1]); |
Huayang Duan | cede791 | 2018-09-26 16:10:42 +0800 | [diff] [blame] | 2154 | } |
| 2155 | } |