Gitiles
Code Review Sign In
review.coreboot.org / coreboot / 9f1fbb9a3002e8d74d53d7973bd1c7e3d4879238 / . / src / northbridge / intel / sandybridge / raminit.c
blob: fa71bc10d32bef46164f860b9a0dba7edd64867b [file] [log] [blame]
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]1/*
2 * This file is part of the coreboot project.
3 *
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]4 * Copyright (C) 2014 Damien Zammit <damien@zamaudio.com>
5 * Copyright (C) 2014 Vladimir Serbinenko <phcoder@gmail.com>
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; version 2 of the License.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]15 */
16
17#include <console/console.h>
Kyösti Mälkki1d7541f2014-02-17 21:34:42 +0200[diff] [blame]18#include <console/usb.h>
Kyösti Mälkki5687fc92013-11-28 18:11:49 +0200[diff] [blame]19#include <bootmode.h>
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]20#include <string.h>
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]21#include <arch/io.h>
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]22#include <cbmem.h>
23#include <arch/cbfs.h>
24#include <cbfs.h>
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]25#include <halt.h>
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]26#include <ip_checksum.h>
27#include <pc80/mc146818rtc.h>
Duncan Laurie7b508dd2012-04-09 12:30:43 -0700[diff] [blame]28#include <device/pci_def.h>
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]29#include "raminit_native.h"
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]30#include "sandybridge.h"
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]31#include <delay.h>
32#include <lib.h>
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]33
34/* Management Engine is in the southbridge */
35#include "southbridge/intel/bd82x6x/me.h"
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]36/* For SPD. */
37#include "southbridge/intel/bd82x6x/smbus.h"
38#include "arch/cpu.h"
39#include "cpu/x86/msr.h"
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]40
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]41/* FIXME: no ECC support. */
42/* FIXME: no support for 3-channel chipsets. */
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]43
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]44#define BASEFREQ 133
45#define tDLLK 512
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]46
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]47#define IS_SANDY_CPU(x) ((x & 0xffff0) == 0x206a0)
48#define IS_SANDY_CPU_C(x) ((x & 0xf) == 4)
49#define IS_SANDY_CPU_D0(x) ((x & 0xf) == 5)
50#define IS_SANDY_CPU_D1(x) ((x & 0xf) == 6)
51#define IS_SANDY_CPU_D2(x) ((x & 0xf) == 7)
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]52
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]53#define IS_IVY_CPU(x) ((x & 0xffff0) == 0x306a0)
54#define IS_IVY_CPU_C(x) ((x & 0xf) == 4)
55#define IS_IVY_CPU_K(x) ((x & 0xf) == 5)
56#define IS_IVY_CPU_D(x) ((x & 0xf) == 6)
57#define IS_IVY_CPU_E(x) ((x & 0xf) >= 8)
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]58
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]59#define NUM_CHANNELS 2
60#define NUM_SLOTRANKS 4
61#define NUM_SLOTS 2
62#define NUM_LANES 8
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]63
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]64/* FIXME: Vendor BIOS uses 64 but our algorithms are less
65 performant and even 1 seems to be enough in practice. */
66#define NUM_PATTERNS 4
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]67
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]68typedef struct odtmap_st {
69 u16 rttwr;
70 u16 rttnom;
71} odtmap;
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]72
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]73typedef struct dimm_info_st {
74 dimm_attr dimm[NUM_CHANNELS][NUM_SLOTS];
75} dimm_info;
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]76
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]77struct ram_rank_timings {
78 /* Register 4024. One byte per slotrank. */
79 u8 val_4024;
80 /* Register 4028. One nibble per slotrank. */
81 u8 val_4028;
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]82
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]83 int val_320c;
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]84
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]85 struct ram_lane_timings {
86 /* lane register offset 0x10. */
87 u16 timA; /* bits 0 - 5, bits 16 - 18 */
88 u8 rising; /* bits 8 - 14 */
89 u8 falling; /* bits 20 - 26. */
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]90
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]91 /* lane register offset 0x20. */
92 int timC; /* bit 0 - 5, 19. */
93 u16 timB; /* bits 8 - 13, 15 - 17. */
94 } lanes[NUM_LANES];
95};
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]96
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]97struct ramctr_timing_st;
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]98
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]99typedef struct ramctr_timing_st {
100 int mobile;
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]101
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]102 u16 cas_supported;
103 /* tLatencies are in units of ns, scaled by x256 */
104 u32 tCK;
105 u32 tAA;
106 u32 tWR;
107 u32 tRCD;
108 u32 tRRD;
109 u32 tRP;
110 u32 tRAS;
111 u32 tRFC;
112 u32 tWTR;
113 u32 tRTP;
114 u32 tFAW;
115 /* Latencies in terms of clock cycles
116 * They are saved separately as they are needed for DRAM MRS commands*/
117 u8 CAS; /* CAS read latency */
118 u8 CWL; /* CAS write latency */
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]119
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]120 u32 tREFI;
121 u32 tMOD;
122 u32 tXSOffset;
123 u32 tWLO;
124 u32 tCKE;
125 u32 tXPDLL;
126 u32 tXP;
127 u32 tAONPD;
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]128
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]129 u16 reg_5064b0; /* bits 0-11. */
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]130
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]131 u8 rankmap[NUM_CHANNELS];
132 int ref_card_offset[NUM_CHANNELS];
133 u32 mad_dimm[NUM_CHANNELS];
134 int channel_size_mb[NUM_CHANNELS];
135 u32 cmd_stretch[NUM_CHANNELS];
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]136
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]137 int reg_c14_offset;
138 int reg_320c_range_threshold;
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]139
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]140 int edge_offset[3];
141 int timC_offset[3];
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]142
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]143 int extended_temperature_range;
144 int auto_self_refresh;
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]145
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]146 int rank_mirror[NUM_CHANNELS][NUM_SLOTRANKS];
147
148 struct ram_rank_timings timings[NUM_CHANNELS][NUM_SLOTRANKS];
149} ramctr_timing;
150
151#define SOUTHBRIDGE PCI_DEV(0, 0x1f, 0)
152#define NORTHBRIDGE PCI_DEV(0, 0x0, 0)
153#define FOR_ALL_LANES for (lane = 0; lane < NUM_LANES; lane++)
154#define FOR_ALL_CHANNELS for (channel = 0; channel < NUM_CHANNELS; channel++)
155#define FOR_ALL_POPULATED_RANKS for (slotrank = 0; slotrank < NUM_SLOTRANKS; slotrank++) if (ctrl->rankmap[channel] & (1 << slotrank))
156#define FOR_ALL_POPULATED_CHANNELS for (channel = 0; channel < NUM_CHANNELS; channel++) if (ctrl->rankmap[channel])
157#define MAX_EDGE_TIMING 71
158#define MAX_TIMC 127
159#define MAX_TIMB 511
160#define MAX_TIMA 127
161
162static void program_timings(ramctr_timing * ctrl, int channel);
163
164static const char *ecc_decoder[] = {
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]165 "inactive",
166 "active on IO",
167 "disabled on IO",
168 "active"
169};
170
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]171static void wait_txt_clear(void)
172{
173 struct cpuid_result cp;
174
175 cp = cpuid_ext(0x1, 0x0);
176 /* Check if TXT is supported? */
177 if (!(cp.ecx & 0x40))
178 return;
179 /* Some TXT public bit. */
180 if (!(read32((void *)0xfed30010) & 1))
181 return;
182 /* Wait for TXT clear. */
183 while (!(read8((void *)0xfed40000) & (1 << 7))) ;
184}
185
186static void sfence(void)
187{
188 asm volatile ("sfence");
189}
190
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]191/*
192 * Dump in the log memory controller configuration as read from the memory
193 * controller registers.
194 */
195static void report_memory_config(void)
196{
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]197 u32 addr_decoder_common, addr_decode_ch[NUM_CHANNELS];
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]198 int i;
199
200 addr_decoder_common = MCHBAR32(0x5000);
201 addr_decode_ch[0] = MCHBAR32(0x5004);
202 addr_decode_ch[1] = MCHBAR32(0x5008);
203
204 printk(BIOS_DEBUG, "memcfg DDR3 clock %d MHz\n",
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]205 (MCHBAR32(0x5e04) * 13333 * 2 + 50) / 100);
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]206 printk(BIOS_DEBUG, "memcfg channel assignment: A: %d, B % d, C % d\n",
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]207 addr_decoder_common & 3, (addr_decoder_common >> 2) & 3,
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]208 (addr_decoder_common >> 4) & 3);
209
210 for (i = 0; i < ARRAY_SIZE(addr_decode_ch); i++) {
211 u32 ch_conf = addr_decode_ch[i];
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]212 printk(BIOS_DEBUG, "memcfg channel[%d] config (%8.8x):\n", i,
213 ch_conf);
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]214 printk(BIOS_DEBUG, " ECC %s\n",
215 ecc_decoder[(ch_conf >> 24) & 3]);
216 printk(BIOS_DEBUG, " enhanced interleave mode %s\n",
217 ((ch_conf >> 22) & 1) ? "on" : "off");
218 printk(BIOS_DEBUG, " rank interleave %s\n",
219 ((ch_conf >> 21) & 1) ? "on" : "off");
220 printk(BIOS_DEBUG, " DIMMA %d MB width x%d %s rank%s\n",
221 ((ch_conf >> 0) & 0xff) * 256,
222 ((ch_conf >> 19) & 1) ? 16 : 8,
223 ((ch_conf >> 17) & 1) ? "dual" : "single",
224 ((ch_conf >> 16) & 1) ? "" : ", selected");
225 printk(BIOS_DEBUG, " DIMMB %d MB width x%d %s rank%s\n",
226 ((ch_conf >> 8) & 0xff) * 256,
227 ((ch_conf >> 20) & 1) ? 16 : 8,
228 ((ch_conf >> 18) & 1) ? "dual" : "single",
229 ((ch_conf >> 16) & 1) ? ", selected" : "");
230 }
231}
232
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]233void read_spd(spd_raw_data * spd, u8 addr)
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]234{
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]235 int j;
236 for (j = 0; j < 256; j++)
237 (*spd)[j] = do_smbus_read_byte(SMBUS_IO_BASE, addr, j);
238}
239
240static void dram_find_spds_ddr3(spd_raw_data * spd, dimm_info * dimm,
241 ramctr_timing * ctrl)
242{
243 int dimms = 0;
244 int channel, slot, spd_slot;
245
246 memset (ctrl->rankmap, 0, sizeof (ctrl->rankmap));
247
248 ctrl->extended_temperature_range = 1;
249 ctrl->auto_self_refresh = 1;
250
251 FOR_ALL_CHANNELS {
252 ctrl->channel_size_mb[channel] = 0;
253
254 for (slot = 0; slot < NUM_SLOTS; slot++) {
255 spd_slot = 2 * channel + slot;
256 spd_decode_ddr3(&dimm->dimm[channel][slot], spd[spd_slot]);
257 if (dimm->dimm[channel][slot].dram_type != SPD_MEMORY_TYPE_SDRAM_DDR3) {
258 // set dimm invalid
259 dimm->dimm[channel][slot].ranks = 0;
260 dimm->dimm[channel][slot].size_mb = 0;
261 continue;
262 }
263
264 dram_print_spd_ddr3(&dimm->dimm[channel][slot]);
265 dimms++;
266 ctrl->rank_mirror[channel][slot * 2] = 0;
267 ctrl->rank_mirror[channel][slot * 2 + 1] = dimm->dimm[channel][slot].flags.pins_mirrored;
268 ctrl->channel_size_mb[channel] += dimm->dimm[channel][slot].size_mb;
269
270 ctrl->auto_self_refresh &= dimm->dimm[channel][slot].flags.asr;
271 ctrl->extended_temperature_range &= dimm->dimm[channel][slot].flags.ext_temp_refresh;
272
273 ctrl->rankmap[channel] |= ((1 << dimm->dimm[channel][slot].ranks) - 1) << (2 * slot);
274 printk(BIOS_DEBUG, "rankmap[%d] = 0x%x\n", channel, ctrl->rankmap[channel]);
275 }
276 if ((ctrl->rankmap[channel] & 3) && (ctrl->rankmap[channel] & 0xc)
277 && dimm->dimm[channel][0].reference_card <= 5 && dimm->dimm[channel][1].reference_card <= 5) {
278 const int ref_card_offset_table[6][6] = {
279 { 0, 0, 0, 0, 2, 2, },
280 { 0, 0, 0, 0, 2, 2, },
281 { 0, 0, 0, 0, 2, 2, },
282 { 0, 0, 0, 0, 1, 1, },
283 { 2, 2, 2, 1, 0, 0, },
284 { 2, 2, 2, 1, 0, 0, },
285 };
286 ctrl->ref_card_offset[channel] = ref_card_offset_table[dimm->dimm[channel][0].reference_card]
287 [dimm->dimm[channel][1].reference_card];
288 } else
289 ctrl->ref_card_offset[channel] = 0;
290 }
291
292 if (!dimms)
293 die("No DIMMs were found");
294}
295
296static void dram_find_common_params(const dimm_info * dimms,
297 ramctr_timing * ctrl)
298{
299 size_t valid_dimms;
300 int channel, slot;
301 ctrl->cas_supported = 0xff;
302 valid_dimms = 0;
303 FOR_ALL_CHANNELS for (slot = 0; slot < 2; slot++) {
304 const dimm_attr *dimm = &dimms->dimm[channel][slot];
305 if (dimm->dram_type != SPD_MEMORY_TYPE_SDRAM_DDR3)
306 continue;
307 valid_dimms++;
308
309 /* Find all possible CAS combinations */
310 ctrl->cas_supported &= dimm->cas_supported;
311
312 /* Find the smallest common latencies supported by all DIMMs */
313 ctrl->tCK = MAX(ctrl->tCK, dimm->tCK);
314 ctrl->tAA = MAX(ctrl->tAA, dimm->tAA);
315 ctrl->tWR = MAX(ctrl->tWR, dimm->tWR);
316 ctrl->tRCD = MAX(ctrl->tRCD, dimm->tRCD);
317 ctrl->tRRD = MAX(ctrl->tRRD, dimm->tRRD);
318 ctrl->tRP = MAX(ctrl->tRP, dimm->tRP);
319 ctrl->tRAS = MAX(ctrl->tRAS, dimm->tRAS);
320 ctrl->tRFC = MAX(ctrl->tRFC, dimm->tRFC);
321 ctrl->tWTR = MAX(ctrl->tWTR, dimm->tWTR);
322 ctrl->tRTP = MAX(ctrl->tRTP, dimm->tRTP);
323 ctrl->tFAW = MAX(ctrl->tFAW, dimm->tFAW);
324 }
325
326 if (!ctrl->cas_supported)
327 die("Unsupported DIMM combination. "
328 "DIMMS do not support common CAS latency");
329 if (!valid_dimms)
330 die("No valid DIMMs found");
331}
332
333static u8 get_CWL(u8 CAS)
334{
335 /* Get CWL based on CAS using the following rule:
336 * _________________________________________
337 * CAS: | 4T | 5T | 6T | 7T | 8T | 9T | 10T | 11T |
338 * CWL: | 5T | 5T | 5T | 6T | 6T | 7T | 7T | 8T |
339 */
340 static const u8 cas_cwl_map[] = { 5, 5, 5, 6, 6, 7, 7, 8 };
341 if (CAS > 11)
342 return 8;
343 return cas_cwl_map[CAS - 4];
344}
345
346/* Frequency multiplier. */
347static u32 get_FRQ(u32 tCK)
348{
349 u32 FRQ;
350 FRQ = 256000 / (tCK * BASEFREQ);
351 if (FRQ > 8)
352 return 8;
353 if (FRQ < 3)
354 return 3;
355 return FRQ;
356}
357
358static u32 get_REFI(u32 tCK)
359{
360 /* Get REFI based on MCU frequency using the following rule:
361 * _________________________________________
362 * FRQ : | 3 | 4 | 5 | 6 | 7 | 8 |
363 * REFI: | 3120 | 4160 | 5200 | 6240 | 7280 | 8320 |
364 */
365 static const u32 frq_refi_map[] =
366 { 3120, 4160, 5200, 6240, 7280, 8320 };
367 return frq_refi_map[get_FRQ(tCK) - 3];
368}
369
370static u8 get_XSOffset(u32 tCK)
371{
372 /* Get XSOffset based on MCU frequency using the following rule:
373 * _________________________
374 * FRQ : | 3 | 4 | 5 | 6 | 7 | 8 |
375 * XSOffset : | 4 | 6 | 7 | 8 | 10 | 11 |
376 */
377 static const u8 frq_xs_map[] = { 4, 6, 7, 8, 10, 11 };
378 return frq_xs_map[get_FRQ(tCK) - 3];
379}
380
381static u8 get_MOD(u32 tCK)
382{
383 /* Get MOD based on MCU frequency using the following rule:
384 * _____________________________
385 * FRQ : | 3 | 4 | 5 | 6 | 7 | 8 |
386 * MOD : | 12 | 12 | 12 | 12 | 15 | 16 |
387 */
388 static const u8 frq_mod_map[] = { 12, 12, 12, 12, 15, 16 };
389 return frq_mod_map[get_FRQ(tCK) - 3];
390}
391
392static u8 get_WLO(u32 tCK)
393{
394 /* Get WLO based on MCU frequency using the following rule:
395 * _______________________
396 * FRQ : | 3 | 4 | 5 | 6 | 7 | 8 |
397 * WLO : | 4 | 5 | 6 | 6 | 8 | 8 |
398 */
399 static const u8 frq_wlo_map[] = { 4, 5, 6, 6, 8, 8 };
400 return frq_wlo_map[get_FRQ(tCK) - 3];
401}
402
403static u8 get_CKE(u32 tCK)
404{
405 /* Get CKE based on MCU frequency using the following rule:
406 * _______________________
407 * FRQ : | 3 | 4 | 5 | 6 | 7 | 8 |
408 * CKE : | 3 | 3 | 4 | 4 | 5 | 6 |
409 */
410 static const u8 frq_cke_map[] = { 3, 3, 4, 4, 5, 6 };
411 return frq_cke_map[get_FRQ(tCK) - 3];
412}
413
414static u8 get_XPDLL(u32 tCK)
415{
416 /* Get XPDLL based on MCU frequency using the following rule:
417 * _____________________________
418 * FRQ : | 3 | 4 | 5 | 6 | 7 | 8 |
419 * XPDLL : | 10 | 13 | 16 | 20 | 23 | 26 |
420 */
421 static const u8 frq_xpdll_map[] = { 10, 13, 16, 20, 23, 26 };
422 return frq_xpdll_map[get_FRQ(tCK) - 3];
423}
424
425static u8 get_XP(u32 tCK)
426{
427 /* Get XP based on MCU frequency using the following rule:
428 * _______________________
429 * FRQ : | 3 | 4 | 5 | 6 | 7 | 8 |
430 * XP : | 3 | 4 | 4 | 5 | 6 | 7 |
431 */
432 static const u8 frq_xp_map[] = { 3, 4, 4, 5, 6, 7 };
433 return frq_xp_map[get_FRQ(tCK) - 3];
434}
435
436static u8 get_AONPD(u32 tCK)
437{
438 /* Get AONPD based on MCU frequency using the following rule:
439 * ________________________
440 * FRQ : | 3 | 4 | 5 | 6 | 7 | 8 |
441 * AONPD : | 4 | 5 | 6 | 8 | 8 | 10 |
442 */
443 static const u8 frq_aonpd_map[] = { 4, 5, 6, 8, 8, 10 };
444 return frq_aonpd_map[get_FRQ(tCK) - 3];
445}
446
447static u32 get_COMP2(u32 tCK)
448{
449 /* Get COMP2 based on MCU frequency using the following rule:
450 * ___________________________________________________________
451 * FRQ : | 3 | 4 | 5 | 6 | 7 | 8 |
452 * COMP : | D6BEDCC | CE7C34C | CA57A4C | C6369CC | C42514C | C21410C |
453 */
454 static const u32 frq_comp2_map[] = { 0xD6BEDCC, 0xCE7C34C, 0xCA57A4C,
455 0xC6369CC, 0xC42514C, 0xC21410C
456 };
457 return frq_comp2_map[get_FRQ(tCK) - 3];
458}
459
460static void dram_timing(ramctr_timing * ctrl)
461{
462 u8 val;
463 u32 val32;
464
465 /* Maximum supported DDR3 frequency is 1066MHz (DDR3 2133) so make sure
466 * we cap it if we have faster DIMMs.
467 * Then, align it to the closest JEDEC standard frequency */
468 if (ctrl->tCK <= TCK_1066MHZ) {
469 ctrl->tCK = TCK_1066MHZ;
470 ctrl->edge_offset[0] = 16;
471 ctrl->edge_offset[1] = 7;
472 ctrl->edge_offset[2] = 7;
473 ctrl->timC_offset[0] = 18;
474 ctrl->timC_offset[1] = 7;
475 ctrl->timC_offset[2] = 7;
476 ctrl->reg_c14_offset = 16;
477 ctrl->reg_5064b0 = 0x218;
478 ctrl->reg_320c_range_threshold = 13;
479 } else if (ctrl->tCK <= TCK_933MHZ) {
480 ctrl->tCK = TCK_933MHZ;
481 ctrl->edge_offset[0] = 14;
482 ctrl->edge_offset[1] = 6;
483 ctrl->edge_offset[2] = 6;
484 ctrl->timC_offset[0] = 15;
485 ctrl->timC_offset[1] = 6;
486 ctrl->timC_offset[2] = 6;
487 ctrl->reg_c14_offset = 14;
488 ctrl->reg_5064b0 = 0x1d5;
489 ctrl->reg_320c_range_threshold = 15;
490 } else if (ctrl->tCK <= TCK_800MHZ) {
491 ctrl->tCK = TCK_800MHZ;
492 ctrl->edge_offset[0] = 13;
493 ctrl->edge_offset[1] = 5;
494 ctrl->edge_offset[2] = 5;
495 ctrl->timC_offset[0] = 14;
496 ctrl->timC_offset[1] = 5;
497 ctrl->timC_offset[2] = 5;
498 ctrl->reg_c14_offset = 12;
499 ctrl->reg_5064b0 = 0x193;
500 ctrl->reg_320c_range_threshold = 15;
501 } else if (ctrl->tCK <= TCK_666MHZ) {
502 ctrl->tCK = TCK_666MHZ;
503 ctrl->edge_offset[0] = 10;
504 ctrl->edge_offset[1] = 4;
505 ctrl->edge_offset[2] = 4;
506 ctrl->timC_offset[0] = 11;
507 ctrl->timC_offset[1] = 4;
508 ctrl->timC_offset[2] = 4;
509 ctrl->reg_c14_offset = 10;
510 ctrl->reg_5064b0 = 0x150;
511 ctrl->reg_320c_range_threshold = 16;
512 } else if (ctrl->tCK <= TCK_533MHZ) {
513 ctrl->tCK = TCK_533MHZ;
514 ctrl->edge_offset[0] = 8;
515 ctrl->edge_offset[1] = 3;
516 ctrl->edge_offset[2] = 3;
517 ctrl->timC_offset[0] = 9;
518 ctrl->timC_offset[1] = 3;
519 ctrl->timC_offset[2] = 3;
520 ctrl->reg_c14_offset = 8;
521 ctrl->reg_5064b0 = 0x10d;
522 ctrl->reg_320c_range_threshold = 17;
523 } else {
524 ctrl->tCK = TCK_400MHZ;
525 ctrl->edge_offset[0] = 6;
526 ctrl->edge_offset[1] = 2;
527 ctrl->edge_offset[2] = 2;
528 ctrl->timC_offset[0] = 6;
529 ctrl->timC_offset[1] = 2;
530 ctrl->timC_offset[2] = 2;
531 ctrl->reg_c14_offset = 8;
532 ctrl->reg_5064b0 = 0xcd;
533 ctrl->reg_320c_range_threshold = 17;
534 }
535
536 val32 = (1000 << 8) / ctrl->tCK;
537 printk(BIOS_DEBUG, "Selected DRAM frequency: %u MHz\n", val32);
538
539 /* Find CAS and CWL latencies */
540 val = (ctrl->tAA + ctrl->tCK - 1) / ctrl->tCK;
541 printk(BIOS_DEBUG, "Minimum CAS latency : %uT\n", val);
542 /* Find lowest supported CAS latency that satisfies the minimum value */
543 while (!((ctrl->cas_supported >> (val - 4)) & 1)
544 && (ctrl->cas_supported >> (val - 4))) {
545 val++;
546 }
547 /* Is CAS supported */
548 if (!(ctrl->cas_supported & (1 << (val - 4))))
549 printk(BIOS_DEBUG, "CAS not supported\n");
550 printk(BIOS_DEBUG, "Selected CAS latency : %uT\n", val);
551 ctrl->CAS = val;
552 ctrl->CWL = get_CWL(ctrl->CAS);
553 printk(BIOS_DEBUG, "Selected CWL latency : %uT\n", ctrl->CWL);
554
555 /* Find tRCD */
556 ctrl->tRCD = (ctrl->tRCD + ctrl->tCK - 1) / ctrl->tCK;
557 printk(BIOS_DEBUG, "Selected tRCD : %uT\n", ctrl->tRCD);
558
559 ctrl->tRP = (ctrl->tRP + ctrl->tCK - 1) / ctrl->tCK;
560 printk(BIOS_DEBUG, "Selected tRP : %uT\n", ctrl->tRP);
561
562 /* Find tRAS */
563 ctrl->tRAS = (ctrl->tRAS + ctrl->tCK - 1) / ctrl->tCK;
564 printk(BIOS_DEBUG, "Selected tRAS : %uT\n", ctrl->tRAS);
565
566 /* Find tWR */
567 ctrl->tWR = (ctrl->tWR + ctrl->tCK - 1) / ctrl->tCK;
568 printk(BIOS_DEBUG, "Selected tWR : %uT\n", ctrl->tWR);
569
570 /* Find tFAW */
571 ctrl->tFAW = (ctrl->tFAW + ctrl->tCK - 1) / ctrl->tCK;
572 printk(BIOS_DEBUG, "Selected tFAW : %uT\n", ctrl->tFAW);
573
574 /* Find tRRD */
575 ctrl->tRRD = (ctrl->tRRD + ctrl->tCK - 1) / ctrl->tCK;
576 printk(BIOS_DEBUG, "Selected tRRD : %uT\n", ctrl->tRRD);
577
578 /* Find tRTP */
579 ctrl->tRTP = (ctrl->tRTP + ctrl->tCK - 1) / ctrl->tCK;
580 printk(BIOS_DEBUG, "Selected tRTP : %uT\n", ctrl->tRTP);
581
582 /* Find tWTR */
583 ctrl->tWTR = (ctrl->tWTR + ctrl->tCK - 1) / ctrl->tCK;
584 printk(BIOS_DEBUG, "Selected tWTR : %uT\n", ctrl->tWTR);
585
586 /* Refresh-to-Active or Refresh-to-Refresh (tRFC) */
587 ctrl->tRFC = (ctrl->tRFC + ctrl->tCK - 1) / ctrl->tCK;
588 printk(BIOS_DEBUG, "Selected tRFC : %uT\n", ctrl->tRFC);
589
590 ctrl->tREFI = get_REFI(ctrl->tCK);
591 ctrl->tMOD = get_MOD(ctrl->tCK);
592 ctrl->tXSOffset = get_XSOffset(ctrl->tCK);
593 ctrl->tWLO = get_WLO(ctrl->tCK);
594 ctrl->tCKE = get_CKE(ctrl->tCK);
595 ctrl->tXPDLL = get_XPDLL(ctrl->tCK);
596 ctrl->tXP = get_XP(ctrl->tCK);
597 ctrl->tAONPD = get_AONPD(ctrl->tCK);
598}
599
600static void dram_freq(ramctr_timing * ctrl)
601{
602 if (ctrl->tCK > TCK_400MHZ) {
603 printk (BIOS_ERR, "DRAM frequency is under lowest supported frequency (400 MHz). Increasing to 400 MHz as last resort");
604 ctrl->tCK = TCK_400MHZ;
605 }
606 while (1) {
607 u8 val2;
608 u32 reg1 = 0;
609
610 /* Step 1 - Set target PCU frequency */
611
612 if (ctrl->tCK <= TCK_1066MHZ) {
613 ctrl->tCK = TCK_1066MHZ;
614 } else if (ctrl->tCK <= TCK_933MHZ) {
615 ctrl->tCK = TCK_933MHZ;
616 } else if (ctrl->tCK <= TCK_800MHZ) {
617 ctrl->tCK = TCK_800MHZ;
618 } else if (ctrl->tCK <= TCK_666MHZ) {
619 ctrl->tCK = TCK_666MHZ;
620 } else if (ctrl->tCK <= TCK_533MHZ) {
621 ctrl->tCK = TCK_533MHZ;
622 } else if (ctrl->tCK <= TCK_400MHZ) {
623 ctrl->tCK = TCK_400MHZ;
624 } else {
625 die ("No lock frequency found");
626 }
627
628 /* Frequency mulitplier. */
629 u32 FRQ = get_FRQ(ctrl->tCK);
630
631 /* Step 2 - Select frequency in the MCU */
632 reg1 = FRQ;
633 reg1 |= 0x80000000; // set running bit
634 MCHBAR32(0x5e00) = reg1;
635 while (reg1 & 0x80000000) {
636 printk(BIOS_DEBUG, " PLL busy...");
637 reg1 = MCHBAR32(0x5e00);
638 }
639 printk(BIOS_DEBUG, "done\n");
640
641 /* Step 3 - Verify lock frequency */
642 reg1 = MCHBAR32(0x5e04);
643 val2 = (u8) reg1;
644 if (val2 >= FRQ) {
645 printk(BIOS_DEBUG, "MCU frequency is set at : %d MHz\n",
646 (1000 << 8) / ctrl->tCK);
647 return;
648 }
649 printk(BIOS_DEBUG, "PLL didn't lock. Retrying at lower frequency\n");
650 ctrl->tCK++;
651 }
652}
653
654static void dram_xover(ramctr_timing * ctrl)
655{
656 u32 reg;
657 int channel;
658
659 FOR_ALL_CHANNELS {
660 // enable xover clk
661 printk(BIOS_DEBUG, "[%x] = %x\n", channel * 0x100 + 0xc14,
662 (ctrl->rankmap[channel] << 24));
663 MCHBAR32(channel * 0x100 + 0xc14) = (ctrl->rankmap[channel] << 24);
664
665 // enable xover ctl
666 reg = 0;
667 if (ctrl->rankmap[channel] & 0x5) {
668 reg |= 0x20000;
669 }
670 if (ctrl->rankmap[channel] & 0xa) {
671 reg |= 0x4000000;
672 }
673 // enable xover cmd
674 reg |= 0x4000;
675 printk(BIOS_DEBUG, "[%x] = %x\n", 0x100 * channel + 0x320c,
676 reg);
677 MCHBAR32(0x100 * channel + 0x320c) = reg;
678 }
679}
680
681static void dram_timing_regs(ramctr_timing * ctrl)
682{
683 u32 reg, addr, val32, cpu, stretch;
684 struct cpuid_result cpures;
685 int channel;
686
687 FOR_ALL_CHANNELS {
688 // DBP
689 reg = 0;
690 reg |= ctrl->tRCD;
691 reg |= (ctrl->tRP << 4);
692 reg |= (ctrl->CAS << 8);
693 reg |= (ctrl->CWL << 12);
694 reg |= (ctrl->tRAS << 16);
695 printk(BIOS_DEBUG, "[%x] = %x\n", 0x400 * channel + 0x4000,
696 reg);
697 MCHBAR32(0x400 * channel + 0x4000) = reg;
698
699 // RAP
700 reg = 0;
701 reg |= ctrl->tRRD;
702 reg |= (ctrl->tRTP << 4);
703 reg |= (ctrl->tCKE << 8);
704 reg |= (ctrl->tWTR << 12);
705 reg |= (ctrl->tFAW << 16);
706 reg |= (ctrl->tWR << 24);
707 reg |= (3 << 30);
708 printk(BIOS_DEBUG, "[%x] = %x\n", 0x400 * channel + 0x4004,
709 reg);
710 MCHBAR32(0x400 * channel + 0x4004) = reg;
711
712 // OTHP
713 addr = 0x400 * channel + 0x400c;
714 reg = 0;
715 reg |= ctrl->tXPDLL;
716 reg |= (ctrl->tXP << 5);
717 reg |= (ctrl->tAONPD << 8);
718 reg |= 0xa0000;
719 printk(BIOS_DEBUG, "[%x] = %x\n", addr, reg);
720 MCHBAR32(addr) = reg;
721
722 MCHBAR32(0x400 * channel + 0x4014) = 0;
723
724 MCHBAR32(addr) |= 0x00020000;
725
726 // ODT stretch
727 reg = 0;
728
729 cpures = cpuid(0);
730 cpu = cpures.eax;
731 if (IS_IVY_CPU(cpu)
732 || (IS_SANDY_CPU(cpu) && IS_SANDY_CPU_D2(cpu))) {
733 stretch = 2;
734 addr = 0x400 * channel + 0x400c;
735 printk(BIOS_DEBUG, "[%x] = %x\n",
736 0x400 * channel + 0x400c, reg);
737 reg = MCHBAR32(addr);
738
739 if (((ctrl->rankmap[channel] & 3) == 0)
740 || (ctrl->rankmap[channel] & 0xc) == 0) {
741
742 // Rank 0 - operate on rank 2
743 reg = (reg & ~0xc0000) | (stretch << 18);
744
745 // Rank 2 - operate on rank 0
746 reg = (reg & ~0x30000) | (stretch << 16);
747
748 printk(BIOS_DEBUG, "[%x] = %x\n", addr, reg);
749 MCHBAR32(addr) = reg;
750 }
751
752 } else if (IS_SANDY_CPU(cpu) && IS_SANDY_CPU_C(cpu)) {
753 stretch = 3;
754 addr = 0x400 * channel + 0x401c;
755 reg = MCHBAR32(addr);
756
757 if (((ctrl->rankmap[channel] & 3) == 0)
758 || (ctrl->rankmap[channel] & 0xc) == 0) {
759
760 // Rank 0 - operate on rank 2
761 reg = (reg & ~0x3000) | (stretch << 12);
762
763 // Rank 2 - operate on rank 0
764 reg = (reg & ~0xc00) | (stretch << 10);
765
766 printk(BIOS_DEBUG, "[%x] = %x\n", addr, reg);
767 MCHBAR32(addr) = reg;
768 }
769 } else {
770 stretch = 0;
771 }
772
773 // REFI
774 reg = 0;
775 val32 = ctrl->tREFI;
776 reg = (reg & ~0xffff) | val32;
777 val32 = ctrl->tRFC;
778 reg = (reg & ~0x1ff0000) | (val32 << 16);
779 val32 = (u32) (ctrl->tREFI * 9) / 1024;
780 reg = (reg & ~0xfe000000) | (val32 << 25);
781 printk(BIOS_DEBUG, "[%x] = %x\n", 0x400 * channel + 0x4298,
782 reg);
783 MCHBAR32(0x400 * channel + 0x4298) = reg;
784
785 MCHBAR32(0x400 * channel + 0x4294) |= 0xff;
786
787 // SRFTP
788 reg = 0;
789 val32 = tDLLK;
790 reg = (reg & ~0xfff) | val32;
791 val32 = ctrl->tXSOffset;
792 reg = (reg & ~0xf000) | (val32 << 12);
793 val32 = tDLLK - ctrl->tXSOffset;
794 reg = (reg & ~0x3ff0000) | (val32 << 16);
795 val32 = ctrl->tMOD - 8;
796 reg = (reg & ~0xf0000000) | (val32 << 28);
797 printk(BIOS_DEBUG, "[%x] = %x\n", 0x400 * channel + 0x42a4,
798 reg);
799 MCHBAR32(0x400 * channel + 0x42a4) = reg;
800 }
801}
802
803static void dram_dimm_mapping(dimm_info * info, ramctr_timing * ctrl)
804{
805 u32 reg, val32;
806 int channel;
807
808 FOR_ALL_CHANNELS {
809 dimm_attr *dimmA = 0;
810 dimm_attr *dimmB = 0;
811 reg = 0;
812 val32 = 0;
813 if (info->dimm[channel][0].size_mb >=
814 info->dimm[channel][1].size_mb) {
815 // dimm 0 is bigger, set it to dimmA
816 dimmA = &info->dimm[channel][0];
817 dimmB = &info->dimm[channel][1];
818 reg |= (0 << 16);
819 } else {
820 // dimm 1 is bigger, set it to dimmA
821 dimmA = &info->dimm[channel][1];
822 dimmB = &info->dimm[channel][0];
823 reg |= (1 << 16);
824 }
825 // dimmA
826 if (dimmA && (dimmA->ranks > 0)) {
827 val32 = dimmA->size_mb / 256;
828 reg = (reg & ~0xff) | val32;
829 val32 = dimmA->ranks - 1;
830 reg = (reg & ~0x20000) | (val32 << 17);
831 val32 = (dimmA->width / 8) - 1;
832 reg = (reg & ~0x80000) | (val32 << 19);
833 }
834 // dimmB
835 if (dimmB && (dimmB->ranks > 0)) {
836 val32 = dimmB->size_mb / 256;
837 reg = (reg & ~0xff00) | (val32 << 8);
838 val32 = dimmB->ranks - 1;
839 reg = (reg & ~0x40000) | (val32 << 18);
840 val32 = (dimmB->width / 8) - 1;
841 reg = (reg & ~0x100000) | (val32 << 20);
842 }
843 reg = (reg & ~0x200000) | (1 << 21); // rank interleave
844 reg = (reg & ~0x400000) | (1 << 22); // enhanced interleave
845
846 // Save MAD-DIMM register
847 if ((dimmA && (dimmA->ranks > 0))
848 || (dimmB && (dimmB->ranks > 0))) {
849 ctrl->mad_dimm[channel] = reg;
850 } else {
851 ctrl->mad_dimm[channel] = 0;
852 }
853 }
854}
855
856static void dram_dimm_set_mapping(ramctr_timing * ctrl)
857{
858 int channel;
859 FOR_ALL_CHANNELS {
860 MCHBAR32(0x5004 + channel * 4) = ctrl->mad_dimm[channel];
861 }
862}
863
864static void dram_zones(ramctr_timing * ctrl, int training)
865{
866 u32 reg, ch0size, ch1size;
867 u8 val;
868 reg = 0;
869 val = 0;
870 if (training) {
871 ch0size = ctrl->channel_size_mb[0] ? 256 : 0;
872 ch1size = ctrl->channel_size_mb[1] ? 256 : 0;
873 } else {
874 ch0size = ctrl->channel_size_mb[0];
875 ch1size = ctrl->channel_size_mb[1];
876 }
877
878 if (ch0size >= ch1size) {
879 reg = MCHBAR32(0x5014);
880 val = ch1size / 256;
881 reg = (reg & ~0xff000000) | val << 24;
882 reg = (reg & ~0xff0000) | (2 * val) << 16;
883 MCHBAR32(0x5014) = reg;
884 MCHBAR32(0x5000) = 0x24;
885 } else {
886 reg = MCHBAR32(0x5014);
887 val = ch0size / 256;
888 reg = (reg & ~0xff000000) | val << 24;
889 reg = (reg & ~0xff0000) | (2 * val) << 16;
890 MCHBAR32(0x5014) = reg;
891 MCHBAR32(0x5000) = 0x21;
892 }
893}
894
895static void dram_memorymap(ramctr_timing * ctrl, int me_uma_size)
896{
897 u32 reg, val, reclaim;
898 u32 tom, gfxstolen, gttsize;
899 size_t tsegsize, mmiosize, toludbase, touudbase, gfxstolenbase, gttbase,
900 tsegbase, mestolenbase;
901 size_t tsegbasedelta, remapbase, remaplimit;
902 uint16_t ggc;
903
904 mmiosize = 0x400;
905
906 ggc = pci_read_config16(NORTHBRIDGE, GGC);
907 if (!(ggc & 2)) {
908 gfxstolen = ((ggc >> 3) & 0x1f) * 32;
909 gttsize = ((ggc >> 8) & 0x3);
910 } else {
911 gfxstolen = 0;
912 gttsize = 0;
913 }
914
915 tsegsize = CONFIG_SMM_TSEG_SIZE >> 20;
916
917 tom = ctrl->channel_size_mb[0] + ctrl->channel_size_mb[1];
918
919 mestolenbase = tom - me_uma_size;
920
921 toludbase = MIN(4096 - mmiosize + gfxstolen + gttsize + tsegsize,
922 tom - me_uma_size);
923 gfxstolenbase = toludbase - gfxstolen;
924 gttbase = gfxstolenbase - gttsize;
925
926 tsegbase = gttbase - tsegsize;
927
928 // Round tsegbase down to nearest address aligned to tsegsize
929 tsegbasedelta = tsegbase & (tsegsize - 1);
930 tsegbase &= ~(tsegsize - 1);
931
932 gttbase -= tsegbasedelta;
933 gfxstolenbase -= tsegbasedelta;
934 toludbase -= tsegbasedelta;
935
936 // Test if it is possible to reclaim a hole in the ram addressing
937 if (tom - me_uma_size > toludbase) {
938 // Reclaim is possible
939 reclaim = 1;
940 remapbase = MAX(4096, tom - me_uma_size);
941 remaplimit =
942 remapbase + MIN(4096, tom - me_uma_size) - toludbase - 1;
943 touudbase = remaplimit + 1;
944 } else {
945 // Reclaim not possible
946 reclaim = 0;
947 touudbase = tom - me_uma_size;
948 }
949
950 // Update memory map in pci-e configuration space
951
952 // TOM (top of memory)
953 reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xa0);
954 val = tom & 0xfff;
955 reg = (reg & ~0xfff00000) | (val << 20);
956 printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xa0, reg);
957 pcie_write_config32(PCI_DEV(0, 0, 0), 0xa0, reg);
958
959 reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xa4);
960 val = tom & 0xfffff000;
961 reg = (reg & ~0x000fffff) | (val >> 12);
962 printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xa4, reg);
963 pcie_write_config32(PCI_DEV(0, 0, 0), 0xa4, reg);
964
965 // TOLUD (top of low used dram)
966 reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xbc);
967 val = toludbase & 0xfff;
968 reg = (reg & ~0xfff00000) | (val << 20);
969 printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xbc, reg);
970 pcie_write_config32(PCI_DEV(0, 0, 0), 0xbc, reg);
971
972 // TOUUD LSB (top of upper usable dram)
973 reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xa8);
974 val = touudbase & 0xfff;
975 reg = (reg & ~0xfff00000) | (val << 20);
976 printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xa8, reg);
977 pcie_write_config32(PCI_DEV(0, 0, 0), 0xa8, reg);
978
979 // TOUUD MSB
980 reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xac);
981 val = touudbase & 0xfffff000;
982 reg = (reg & ~0x000fffff) | (val >> 12);
983 printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xac, reg);
984 pcie_write_config32(PCI_DEV(0, 0, 0), 0xac, reg);
985
986 if (reclaim) {
987 // REMAP BASE
988 pcie_write_config32(PCI_DEV(0, 0, 0), 0x90, remapbase << 20);
989 pcie_write_config32(PCI_DEV(0, 0, 0), 0x94, remapbase >> 12);
990
991 // REMAP LIMIT
992 pcie_write_config32(PCI_DEV(0, 0, 0), 0x98, remaplimit << 20);
993 pcie_write_config32(PCI_DEV(0, 0, 0), 0x9c, remaplimit >> 12);
994 }
995 // TSEG
996 reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xb8);
997 val = tsegbase & 0xfff;
998 reg = (reg & ~0xfff00000) | (val << 20);
999 printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xb8, reg);
1000 pcie_write_config32(PCI_DEV(0, 0, 0), 0xb8, reg);
1001
1002 // GFX stolen memory
1003 reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xb0);
1004 val = gfxstolenbase & 0xfff;
1005 reg = (reg & ~0xfff00000) | (val << 20);
1006 printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xb0, reg);
1007 pcie_write_config32(PCI_DEV(0, 0, 0), 0xb0, reg);
1008
1009 // GTT stolen memory
1010 reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xb4);
1011 val = gttbase & 0xfff;
1012 reg = (reg & ~0xfff00000) | (val << 20);
1013 printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xb4, reg);
1014 pcie_write_config32(PCI_DEV(0, 0, 0), 0xb4, reg);
1015
1016 if (me_uma_size) {
1017 reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0x7c);
1018 val = (0x80000 - me_uma_size) & 0xfffff000;
1019 reg = (reg & ~0x000fffff) | (val >> 12);
1020 printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0x7c, reg);
1021 pcie_write_config32(PCI_DEV(0, 0, 0), 0x7c, reg);
1022
1023 // ME base
1024 reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0x70);
1025 val = mestolenbase & 0xfff;
1026 reg = (reg & ~0xfff00000) | (val << 20);
1027 printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0x70, reg);
1028 pcie_write_config32(PCI_DEV(0, 0, 0), 0x70, reg);
1029
1030 reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0x74);
1031 val = mestolenbase & 0xfffff000;
1032 reg = (reg & ~0x000fffff) | (val >> 12);
1033 printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0x74, reg);
1034 pcie_write_config32(PCI_DEV(0, 0, 0), 0x74, reg);
1035
1036 // ME mask
1037 reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0x78);
1038 val = (0x80000 - me_uma_size) & 0xfff;
1039 reg = (reg & ~0xfff00000) | (val << 20);
1040 reg = (reg & ~0x400) | (1 << 10); // set lockbit on ME mem
1041
1042 reg = (reg & ~0x800) | (1 << 11); // set ME memory enable
1043 printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0x78, reg);
1044 pcie_write_config32(PCI_DEV(0, 0, 0), 0x78, reg);
1045 }
1046}
1047
1048static void dram_ioregs(ramctr_timing * ctrl)
1049{
1050 u32 reg, comp2;
1051
1052 int channel;
1053
1054 // IO clock
1055 FOR_ALL_CHANNELS {
1056 MCHBAR32(0xc00 + 0x100 * channel) = ctrl->rankmap[channel];
1057 }
1058
1059 // IO command
1060 FOR_ALL_CHANNELS {
1061 MCHBAR32(0x3200 + 0x100 * channel) = ctrl->rankmap[channel];
1062 }
1063
1064 // IO control
1065 FOR_ALL_POPULATED_CHANNELS {
1066 program_timings(ctrl, channel);
1067 }
1068
1069 // Rcomp
1070 printk(BIOS_DEBUG, "RCOMP...");
1071 reg = 0;
1072 while (reg == 0) {
1073 reg = MCHBAR32(0x5084) & 0x10000;
1074 }
1075 printk(BIOS_DEBUG, "done\n");
1076
1077 // Set comp2
1078 comp2 = get_COMP2(ctrl->tCK);
1079 MCHBAR32(0x3714) = comp2;
1080 printk(BIOS_DEBUG, "COMP2 done\n");
1081
1082 // Set comp1
1083 FOR_ALL_POPULATED_CHANNELS {
1084 reg = MCHBAR32(0x1810 + channel * 0x100); //ch0
1085 reg = (reg & ~0xe00) | (1 << 9); //odt
1086 reg = (reg & ~0xe00000) | (1 << 21); //clk drive up
1087 reg = (reg & ~0x38000000) | (1 << 27); //ctl drive up
1088 MCHBAR32(0x1810 + channel * 0x100) = reg;
1089 }
1090 printk(BIOS_DEBUG, "COMP1 done\n");
1091
1092 printk(BIOS_DEBUG, "FORCE RCOMP and wait 20us...");
1093 MCHBAR32(0x5f08) |= 0x100;
1094 udelay(20);
1095 printk(BIOS_DEBUG, "done\n");
1096}
1097
1098static void wait_428c(int channel)
1099{
1100 while (1) {
1101 if (read32(DEFAULT_MCHBAR + 0x428c + (channel << 10)) & 0x50)
1102 return;
1103 }
1104}
1105
1106static void write_reset(ramctr_timing * ctrl)
1107{
1108 int channel, slotrank;
1109
1110 /* choose a populated channel. */
1111 channel = (ctrl->rankmap[0]) ? 0 : 1;
1112
1113 wait_428c(channel);
1114
1115 /* choose a populated rank. */
1116 slotrank = (ctrl->rankmap[channel] & 1) ? 0 : 2;
1117
1118 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
1119 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x80c01);
1120
1121 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
1122 (slotrank << 24) | 0x60000);
1123
1124 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
1125
1126 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0x400001);
1127 wait_428c(channel);
1128}
1129
1130static void dram_jedecreset(ramctr_timing * ctrl)
1131{
1132 u32 reg, addr;
1133 int channel;
1134
1135 while (!(MCHBAR32(0x5084) & 0x10000)) ;
1136 do {
1137 reg = MCHBAR32(0x428c);
1138 } while ((reg & 0x14) == 0);
1139
1140 // Set state of memory controller
1141 reg = 0x112;
1142 MCHBAR32(0x5030) = reg;
1143 MCHBAR32(0x4ea0) = 0;
1144 reg |= 2; //ddr reset
1145 MCHBAR32(0x5030) = reg;
1146
1147 // Assert dimm reset signal
1148 reg = MCHBAR32(0x5030);
1149 reg &= ~0x2;
1150 MCHBAR32(0x5030) = reg;
1151
1152 // Wait 200us
1153 udelay(200);
1154
1155 // Deassert dimm reset signal
1156 MCHBAR32(0x5030) |= 2;
1157
1158 // Wait 500us
1159 udelay(500);
1160
1161 // Enable DCLK
1162 MCHBAR32(0x5030) |= 4;
1163
1164 // XXX Wait 20ns
1165 udelay(1);
1166
1167 FOR_ALL_CHANNELS {
1168 // Set valid rank CKE
1169 reg = 0;
1170 reg = (reg & ~0xf) | ctrl->rankmap[channel];
1171 addr = 0x400 * channel + 0x42a0;
1172 MCHBAR32(addr) = reg;
1173
1174 // Wait 10ns for ranks to settle
1175 //udelay(0.01);
1176
1177 reg = (reg & ~0xf0) | (ctrl->rankmap[channel] << 4);
1178 MCHBAR32(addr) = reg;
1179
1180 // Write reset using a NOP
1181 write_reset(ctrl);
1182 }
1183}
1184
1185static odtmap get_ODT(ramctr_timing * ctrl, u8 rank)
1186{
1187 /* Get ODT based on rankmap: */
1188 int dimms_per_ch = 0;
1189 int channel;
1190
1191 FOR_ALL_CHANNELS {
1192 dimms_per_ch = max ((ctrl->rankmap[channel] & 1)
1193 + ((ctrl->rankmap[channel] >> 2) & 1),
1194 dimms_per_ch);
1195 }
1196
1197 if (dimms_per_ch == 1) {
1198 return (const odtmap){60, 60};
1199 } else if (dimms_per_ch == 2) {
1200 return (const odtmap){120, 30};
1201 } else {
1202 printk(BIOS_DEBUG,
1203 "Huh, no dimms? m0 = %d m1 = %d dpc = %d\n",
1204 ctrl->rankmap[0],
1205 ctrl->rankmap[1], dimms_per_ch);
1206 die("");
1207 }
1208}
1209
1210static void write_mrreg(ramctr_timing * ctrl, int channel, int slotrank,
1211 int reg, u32 val)
1212{
1213 wait_428c(channel);
1214
1215 printram("MRd: %x <= %x\n", reg, val);
1216
1217 if (ctrl->rank_mirror[channel][slotrank]) {
1218 /* DDR3 Rank1 Address mirror
1219 * swap the following pins:
1220 * A3<->A4, A5<->A6, A7<->A8, BA0<->BA1 */
1221 reg = ((reg >> 1) & 1) | ((reg << 1) & 2);
1222 val = (val & ~0x1f8) | ((val >> 1) & 0xa8)
1223 | ((val & 0xa8) << 1);
1224 }
1225
1226 printram("MRd: %x <= %x\n", reg, val);
1227
1228 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f000);
1229 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x41001);
1230 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
1231 (slotrank << 24) | (reg << 20) | val | 0x60000);
1232 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
1233
1234 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f000);
1235 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x41001);
1236 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
1237 (slotrank << 24) | (reg << 20) | val | 0x60000);
1238 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
1239
1240 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x0f000);
1241 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
1242 0x1001 | (ctrl->tMOD << 16));
1243 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
1244 (slotrank << 24) | (reg << 20) | val | 0x60000);
1245 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
1246 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0x80001);
1247}
1248
1249static u32 make_mr0(ramctr_timing * ctrl, u8 rank)
1250{
1251 u16 mr0reg, mch_cas, mch_wr;
1252 static const u8 mch_wr_t[12] = { 1, 2, 3, 4, 0, 5, 0, 6, 0, 7, 0, 0 };
1253 mr0reg = 0x100;
1254
1255 // Convert CAS to MCH register friendly
1256 if (ctrl->CAS < 12) {
1257 mch_cas = (u16) ((ctrl->CAS - 4) << 1);
1258 } else {
1259 mch_cas = (u16) (ctrl->CAS - 12);
1260 mch_cas = ((mch_cas << 1) | 0x1);
1261 }
1262
1263 // Convert tWR to MCH register friendly
1264 mch_wr = mch_wr_t[ctrl->tWR - 5];
1265
1266 mr0reg = (mr0reg & ~0x4) | (mch_cas & 0x1);
1267 mr0reg = (mr0reg & ~0x70) | ((mch_cas & 0xe) << 3);
1268 mr0reg = (mr0reg & ~0xe00) | (mch_wr << 9);
1269 // Fast (desktop) 0x1 or slow (mobile) 0x0
1270 mr0reg = (mr0reg & ~0x1000) | (!ctrl->mobile << 12);
1271 return mr0reg;
1272}
1273
1274static void dram_mr0(ramctr_timing * ctrl, u8 rank)
1275{
1276 int channel;
1277
1278 FOR_ALL_POPULATED_CHANNELS write_mrreg(ctrl, channel, rank, 0,
1279 make_mr0(ctrl, rank));
1280}
1281
1282static u32 encode_odt(u32 odt)
1283{
1284 switch (odt) {
1285 case 30:
1286 return (1 << 9) | (1 << 2); // RZQ/8, RZQ/4
1287 case 60:
1288 return (1 << 2); // RZQ/4
1289 case 120:
1290 return (1 << 6); // RZQ/2
1291 default:
1292 case 0:
1293 return 0;
1294 }
1295}
1296
1297static u32 make_mr1(ramctr_timing * ctrl, u8 rank)
1298{
1299 odtmap odt;
1300 u32 mr1reg;
1301
1302 odt = get_ODT(ctrl, rank);
1303 mr1reg = 0x2;
1304
1305 mr1reg |= encode_odt(odt.rttnom);
1306
1307 return mr1reg;
1308}
1309
1310static void dram_mr1(ramctr_timing * ctrl, u8 rank)
1311{
1312 u16 mr1reg;
1313 int channel;
1314
1315 mr1reg = make_mr1(ctrl, rank);
1316
1317 FOR_ALL_CHANNELS {
1318 write_mrreg(ctrl, channel, rank, 1, mr1reg);
1319 }
1320}
1321
1322static void dram_mr2(ramctr_timing * ctrl, u8 rank)
1323{
1324 u16 pasr, cwl, mr2reg;
1325 odtmap odt;
1326 int channel;
1327 int srt;
1328
1329 pasr = 0;
1330 cwl = ctrl->CWL - 5;
1331 odt = get_ODT(ctrl, rank);
1332
1333 srt = ctrl->extended_temperature_range && !ctrl->auto_self_refresh;
1334
1335 mr2reg = 0;
1336 mr2reg = (mr2reg & ~0x7) | pasr;
1337 mr2reg = (mr2reg & ~0x38) | (cwl << 3);
1338 mr2reg = (mr2reg & ~0x40) | (ctrl->auto_self_refresh << 6);
1339 mr2reg = (mr2reg & ~0x80) | (srt << 7);
1340 mr2reg |= (odt.rttwr / 60) << 9;
1341
1342 FOR_ALL_CHANNELS {
1343 write_mrreg(ctrl, channel, rank, 2, mr2reg);
1344 }
1345}
1346
1347static void dram_mr3(ramctr_timing * ctrl, u8 rank)
1348{
1349 int channel;
1350
1351 FOR_ALL_CHANNELS {
1352 write_mrreg(ctrl, channel, rank, 3, 0);
1353 }
1354}
1355
1356static void dram_mrscommands(ramctr_timing * ctrl)
1357{
1358 u8 rank;
1359 u32 reg, addr;
1360 int channel;
1361
1362 for (rank = 0; rank < 4; rank++) {
1363 // MR2
1364 printram("MR2 rank %d...", rank);
1365 dram_mr2(ctrl, rank);
1366 printram("done\n");
1367
1368 // MR3
1369 printram("MR3 rank %d...", rank);
1370 dram_mr3(ctrl, rank);
1371 printram("done\n");
1372
1373 // MR1
1374 printram("MR1 rank %d...", rank);
1375 dram_mr1(ctrl, rank);
1376 printram("done\n");
1377
1378 // MR0
1379 printram("MR0 rank %d...", rank);
1380 dram_mr0(ctrl, rank);
1381 printram("done\n");
1382 }
1383
1384 write32(DEFAULT_MCHBAR + 0x4e20, 0x7);
1385 write32(DEFAULT_MCHBAR + 0x4e30, 0xf1001);
1386 write32(DEFAULT_MCHBAR + 0x4e00, 0x60002);
1387 write32(DEFAULT_MCHBAR + 0x4e10, 0);
1388 write32(DEFAULT_MCHBAR + 0x4e24, 0x1f003);
1389 write32(DEFAULT_MCHBAR + 0x4e34, 0x1901001);
1390 write32(DEFAULT_MCHBAR + 0x4e04, 0x60400);
1391 write32(DEFAULT_MCHBAR + 0x4e14, 0x288);
1392 write32(DEFAULT_MCHBAR + 0x4e84, 0x40004);
1393
1394 // Drain
1395 FOR_ALL_CHANNELS {
1396 // Wait for ref drained
1397 wait_428c(channel);
1398 }
1399
1400 // Refresh enable
1401 MCHBAR32(0x5030) |= 8;
1402
1403 FOR_ALL_POPULATED_CHANNELS {
1404 addr = 0x400 * channel + 0x4020;
1405 reg = MCHBAR32(addr);
1406 reg &= ~0x200000;
1407 MCHBAR32(addr) = reg;
1408
1409 wait_428c(channel);
1410
1411 rank = (ctrl->rankmap[channel] & 1) ? 0 : 2;
1412
1413 // Drain
1414 wait_428c(channel);
1415
1416 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
1417 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x659001);
1418 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
1419 (rank << 24) | 0x60000);
1420 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
1421 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0x1);
1422
1423 // Drain
1424 wait_428c(channel);
1425 }
1426}
1427
1428const u32 lane_registers[] = {
1429 0x0000, 0x0200, 0x0400, 0x0600,
1430 0x1000, 0x1200, 0x1400, 0x1600,
1431 0x0800
1432};
1433
1434static void program_timings(ramctr_timing * ctrl, int channel)
1435{
1436 u32 reg32, reg_4024, reg_c14, reg_c18, reg_4028;
1437 int lane;
1438 int slotrank, slot;
1439 int full_shift = 0;
1440 u16 slot320c[NUM_SLOTS];
1441
1442 FOR_ALL_POPULATED_RANKS {
1443 if (full_shift < -ctrl->timings[channel][slotrank].val_320c)
1444 full_shift = -ctrl->timings[channel][slotrank].val_320c;
1445 }
1446
1447 for (slot = 0; slot < NUM_SLOTS; slot++)
1448 switch ((ctrl->rankmap[channel] >> (2 * slot)) & 3) {
1449 case 0:
1450 default:
1451 slot320c[slot] = 0x7f;
1452 break;
1453 case 1:
1454 slot320c[slot] =
1455 ctrl->timings[channel][2 * slot + 0].val_320c +
1456 full_shift;
1457 break;
1458 case 2:
1459 slot320c[slot] =
1460 ctrl->timings[channel][2 * slot + 1].val_320c +
1461 full_shift;
1462 break;
1463 case 3:
1464 slot320c[slot] =
1465 (ctrl->timings[channel][2 * slot].val_320c +
1466 ctrl->timings[channel][2 * slot +
1467 1].val_320c) / 2 +
1468 full_shift;
1469 break;
1470 }
1471
1472 reg32 = (1 << 17) | (1 << 14);
1473 reg32 |= ((slot320c[0] & 0x3f) << 6) | ((slot320c[0] & 0x40) << 9);
1474 reg32 |= (slot320c[1] & 0x7f) << 18;
1475 reg32 |= (full_shift & 0x3f) | ((full_shift & 0x40) << 6);
1476
1477 MCHBAR32(0x320c + 0x100 * channel) = reg32;
1478
1479 reg_c14 = ctrl->rankmap[channel] << 24;
1480 reg_c18 = 0;
1481
1482 FOR_ALL_POPULATED_RANKS {
1483 int shift =
1484 ctrl->timings[channel][slotrank].val_320c + full_shift;
1485 int offset_val_c14;
1486 if (shift < 0)
1487 shift = 0;
1488 offset_val_c14 = ctrl->reg_c14_offset + shift;
1489 reg_c14 |= (offset_val_c14 & 0x3f) << (6 * slotrank);
1490 reg_c18 |= ((offset_val_c14 >> 6) & 1) << slotrank;
1491 }
1492
1493 MCHBAR32(0xc14 + channel * 0x100) = reg_c14;
1494 MCHBAR32(0xc18 + channel * 0x100) = reg_c18;
1495
1496 reg_4028 = MCHBAR32(0x4028 + 0x400 * channel);
1497 reg_4028 &= 0xffff0000;
1498
1499 reg_4024 = 0;
1500
1501 FOR_ALL_POPULATED_RANKS {
1502 int post_timA_min_high = 7, post_timA_max_high = 0;
1503 int pre_timA_min_high = 7, pre_timA_max_high = 0;
1504 int shift_402x = 0;
1505 int shift =
1506 ctrl->timings[channel][slotrank].val_320c + full_shift;
1507
1508 if (shift < 0)
1509 shift = 0;
1510
1511 FOR_ALL_LANES {
1512 if (post_timA_min_high >
1513 ((ctrl->timings[channel][slotrank].lanes[lane].
1514 timA + shift) >> 6))
1515 post_timA_min_high =
1516 ((ctrl->timings[channel][slotrank].
1517 lanes[lane].timA + shift) >> 6);
1518 if (pre_timA_min_high >
1519 (ctrl->timings[channel][slotrank].lanes[lane].
1520 timA >> 6))
1521 pre_timA_min_high =
1522 (ctrl->timings[channel][slotrank].
1523 lanes[lane].timA >> 6);
1524 if (post_timA_max_high <
1525 ((ctrl->timings[channel][slotrank].lanes[lane].
1526 timA + shift) >> 6))
1527 post_timA_max_high =
1528 ((ctrl->timings[channel][slotrank].
1529 lanes[lane].timA + shift) >> 6);
1530 if (pre_timA_max_high <
1531 (ctrl->timings[channel][slotrank].lanes[lane].
1532 timA >> 6))
1533 pre_timA_max_high =
1534 (ctrl->timings[channel][slotrank].
1535 lanes[lane].timA >> 6);
1536 }
1537
1538 if (pre_timA_max_high - pre_timA_min_high <
1539 post_timA_max_high - post_timA_min_high)
1540 shift_402x = +1;
1541 else if (pre_timA_max_high - pre_timA_min_high >
1542 post_timA_max_high - post_timA_min_high)
1543 shift_402x = -1;
1544
1545 reg_4028 |=
1546 (ctrl->timings[channel][slotrank].val_4028 + shift_402x -
1547 post_timA_min_high) << (4 * slotrank);
1548 reg_4024 |=
1549 (ctrl->timings[channel][slotrank].val_4024 +
1550 shift_402x) << (8 * slotrank);
1551
1552 FOR_ALL_LANES {
1553 MCHBAR32(lane_registers[lane] + 0x10 + 0x100 * channel +
1554 4 * slotrank)
1555 =
1556 (((ctrl->timings[channel][slotrank].lanes[lane].
1557 timA + shift) & 0x3f)
1558 |
1559 ((ctrl->timings[channel][slotrank].lanes[lane].
1560 rising + shift) << 8)
1561 |
1562 (((ctrl->timings[channel][slotrank].lanes[lane].
1563 timA + shift -
1564 (post_timA_min_high << 6)) & 0x1c0) << 10)
1565 | (ctrl->timings[channel][slotrank].lanes[lane].
1566 falling << 20));
1567
1568 MCHBAR32(lane_registers[lane] + 0x20 + 0x100 * channel +
1569 4 * slotrank)
1570 =
1571 (((ctrl->timings[channel][slotrank].lanes[lane].
1572 timC + shift) & 0x3f)
1573 |
1574 (((ctrl->timings[channel][slotrank].lanes[lane].
1575 timB + shift) & 0x3f) << 8)
1576 |
1577 (((ctrl->timings[channel][slotrank].lanes[lane].
1578 timB + shift) & 0x1c0) << 9)
1579 |
1580 (((ctrl->timings[channel][slotrank].lanes[lane].
1581 timC + shift) & 0x40) << 13));
1582 }
1583 }
1584 MCHBAR32(0x4024 + 0x400 * channel) = reg_4024;
1585 MCHBAR32(0x4028 + 0x400 * channel) = reg_4028;
1586}
1587
1588static void test_timA(ramctr_timing * ctrl, int channel, int slotrank)
1589{
1590 wait_428c(channel);
1591
1592 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f000);
1593 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
1594 (0xc01 | (ctrl->tMOD << 16)));
1595 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
1596 (slotrank << 24) | 0x360004);
1597 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
1598
1599 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f105);
1600 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x4040c01);
1601 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel, (slotrank << 24));
1602 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
1603
1604 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f105);
1605 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
1606 0x100f | ((ctrl->CAS + 36) << 16));
1607 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
1608 (slotrank << 24) | 0x60000);
1609 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
1610
1611 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f000);
1612 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
1613 (0xc01 | (ctrl->tMOD << 16)));
1614 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
1615 (slotrank << 24) | 0x360000);
1616 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
1617
1618 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
1619
1620 wait_428c(channel);
1621}
1622
1623static int does_lane_work(ramctr_timing * ctrl, int channel, int slotrank,
1624 int lane)
1625{
1626 u32 timA = ctrl->timings[channel][slotrank].lanes[lane].timA;
1627 return ((read32
1628 (DEFAULT_MCHBAR + lane_registers[lane] + channel * 0x100 + 4 +
1629 ((timA / 32) & 1) * 4)
1630 >> (timA % 32)) & 1);
1631}
1632
1633struct run {
1634 int middle;
1635 int end;
1636 int start;
1637 int all;
1638 int length;
1639};
1640
1641static struct run get_longest_zero_run(int *seq, int sz)
1642{
1643 int i, ls;
1644 int bl = 0, bs = 0;
1645 struct run ret;
1646
1647 ls = 0;
1648 for (i = 0; i < 2 * sz; i++)
1649 if (seq[i % sz]) {
1650 if (i - ls > bl) {
1651 bl = i - ls;
1652 bs = ls;
1653 }
1654 ls = i + 1;
1655 }
1656 if (bl == 0) {
1657 ret.middle = sz / 2;
1658 ret.start = 0;
1659 ret.end = sz;
1660 ret.all = 1;
1661 return ret;
1662 }
1663
1664 ret.start = bs % sz;
1665 ret.end = (bs + bl - 1) % sz;
1666 ret.middle = (bs + (bl - 1) / 2) % sz;
1667 ret.length = bl;
1668 ret.all = 0;
1669
1670 return ret;
1671}
1672
1673static void discover_timA_coarse(ramctr_timing * ctrl, int channel,
1674 int slotrank, int *upperA)
1675{
1676 int timA;
1677 int statistics[NUM_LANES][128];
1678 int lane;
1679
1680 for (timA = 0; timA < 128; timA++) {
1681 FOR_ALL_LANES {
1682 ctrl->timings[channel][slotrank].lanes[lane].timA = timA;
1683 }
1684 program_timings(ctrl, channel);
1685
1686 test_timA(ctrl, channel, slotrank);
1687
1688 FOR_ALL_LANES {
1689 statistics[lane][timA] =
1690 !does_lane_work(ctrl, channel, slotrank, lane);
1691 printram("Astat: %d, %d, %d, %x, %x\n",
1692 channel, slotrank, lane, timA,
1693 statistics[lane][timA]);
1694 }
1695 }
1696 FOR_ALL_LANES {
1697 struct run rn = get_longest_zero_run(statistics[lane], 128);
1698 ctrl->timings[channel][slotrank].lanes[lane].timA = rn.middle;
1699 upperA[lane] = rn.end;
1700 if (upperA[lane] < rn.middle)
1701 upperA[lane] += 128;
1702 printram("Aval: %d, %d, %d, %x\n", channel, slotrank,
1703 lane, ctrl->timings[channel][slotrank].lanes[lane].timA);
1704 printram("Aend: %d, %d, %d, %x\n", channel, slotrank,
1705 lane, upperA[lane]);
1706 }
1707}
1708
1709static void discover_timA_fine(ramctr_timing * ctrl, int channel, int slotrank,
1710 int *upperA)
1711{
1712 int timA_delta;
1713 int statistics[NUM_LANES][51];
1714 int lane, i;
1715
1716 memset(statistics, 0, sizeof(statistics));
1717
1718 for (timA_delta = -25; timA_delta <= 25; timA_delta++) {
1719 FOR_ALL_LANES ctrl->timings[channel][slotrank].lanes[lane].
1720 timA = upperA[lane] + timA_delta + 0x40;
1721 program_timings(ctrl, channel);
1722
1723 for (i = 0; i < 100; i++) {
1724 test_timA(ctrl, channel, slotrank);
1725 FOR_ALL_LANES {
1726 statistics[lane][timA_delta + 25] +=
1727 does_lane_work(ctrl, channel, slotrank,
1728 lane);
1729 }
1730 }
1731 }
1732 FOR_ALL_LANES {
1733 int last_zero, first_all;
1734
1735 for (last_zero = -25; last_zero <= 25; last_zero++)
1736 if (statistics[lane][last_zero + 25])
1737 break;
1738 last_zero--;
1739 for (first_all = -25; first_all <= 25; first_all++)
1740 if (statistics[lane][first_all + 25] == 100)
1741 break;
1742
1743 printram("lane %d: %d, %d\n", lane, last_zero,
1744 first_all);
1745
1746 ctrl->timings[channel][slotrank].lanes[lane].timA =
1747 (last_zero + first_all) / 2 + upperA[lane];
1748 printram("Aval: %d, %d, %d, %x\n", channel, slotrank,
1749 lane, ctrl->timings[channel][slotrank].lanes[lane].timA);
1750 }
1751}
1752
1753static void discover_402x(ramctr_timing * ctrl, int channel, int slotrank,
1754 int *upperA)
1755{
1756 int works[NUM_LANES];
1757 int lane;
1758 while (1) {
1759 int all_works = 1, some_works = 0;
1760 program_timings(ctrl, channel);
1761 test_timA(ctrl, channel, slotrank);
1762 FOR_ALL_LANES {
1763 works[lane] =
1764 !does_lane_work(ctrl, channel, slotrank, lane);
1765 if (works[lane])
1766 some_works = 1;
1767 else
1768 all_works = 0;
1769 }
1770 if (all_works)
1771 return;
1772 if (!some_works) {
1773 if (ctrl->timings[channel][slotrank].val_4024 < 2)
1774 die("402x discovery failed");
1775 ctrl->timings[channel][slotrank].val_4024 -= 2;
1776 printram("4024 -= 2;\n");
1777 continue;
1778 }
1779 ctrl->timings[channel][slotrank].val_4028 += 2;
1780 printram("4028 += 2;\n");
1781 if (ctrl->timings[channel][slotrank].val_4028 >= 0x10)
1782 die("402x discovery failed");
1783 FOR_ALL_LANES if (works[lane]) {
1784 ctrl->timings[channel][slotrank].lanes[lane].timA +=
1785 128;
1786 upperA[lane] += 128;
1787 printram("increment %d, %d, %d\n", channel,
1788 slotrank, lane);
1789 }
1790 }
1791}
1792
1793struct timA_minmax {
1794 int timA_min_high, timA_max_high;
1795};
1796
1797static void pre_timA_change(ramctr_timing * ctrl, int channel, int slotrank,
1798 struct timA_minmax *mnmx)
1799{
1800 int lane;
1801 mnmx->timA_min_high = 7;
1802 mnmx->timA_max_high = 0;
1803
1804 FOR_ALL_LANES {
1805 if (mnmx->timA_min_high >
1806 (ctrl->timings[channel][slotrank].lanes[lane].timA >> 6))
1807 mnmx->timA_min_high =
1808 (ctrl->timings[channel][slotrank].lanes[lane].
1809 timA >> 6);
1810 if (mnmx->timA_max_high <
1811 (ctrl->timings[channel][slotrank].lanes[lane].timA >> 6))
1812 mnmx->timA_max_high =
1813 (ctrl->timings[channel][slotrank].lanes[lane].
1814 timA >> 6);
1815 }
1816}
1817
1818static void post_timA_change(ramctr_timing * ctrl, int channel, int slotrank,
1819 struct timA_minmax *mnmx)
1820{
1821 struct timA_minmax post;
1822 int shift_402x = 0;
1823
1824 /* Get changed maxima. */
1825 pre_timA_change(ctrl, channel, slotrank, &post);
1826
1827 if (mnmx->timA_max_high - mnmx->timA_min_high <
1828 post.timA_max_high - post.timA_min_high)
1829 shift_402x = +1;
1830 else if (mnmx->timA_max_high - mnmx->timA_min_high >
1831 post.timA_max_high - post.timA_min_high)
1832 shift_402x = -1;
1833 else
1834 shift_402x = 0;
1835
1836 ctrl->timings[channel][slotrank].val_4028 += shift_402x;
1837 ctrl->timings[channel][slotrank].val_4024 += shift_402x;
1838 printram("4024 += %d;\n", shift_402x);
1839 printram("4028 += %d;\n", shift_402x);
1840}
1841
1842static void read_training(ramctr_timing * ctrl)
1843{
1844 int channel, slotrank, lane;
1845
1846 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
1847 u32 r32;
1848 int all_high, some_high;
1849 int upperA[NUM_LANES];
1850 struct timA_minmax mnmx;
1851
1852 wait_428c(channel);
1853 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f002);
1854 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
1855 0xc01 | (ctrl->tRP << 16));
1856 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
1857 (slotrank << 24) | 0x60400);
1858 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
1859 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
1860
1861 write32(DEFAULT_MCHBAR + 0x3400, (slotrank << 2) | 0x8001);
1862
1863 ctrl->timings[channel][slotrank].val_4028 = 4;
1864 ctrl->timings[channel][slotrank].val_4024 = 55;
1865 program_timings(ctrl, channel);
1866
1867 discover_timA_coarse(ctrl, channel, slotrank, upperA);
1868
1869 all_high = 1;
1870 some_high = 0;
1871 FOR_ALL_LANES {
1872 if (ctrl->timings[channel][slotrank].lanes[lane].
1873 timA >= 0x40)
1874 some_high = 1;
1875 else
1876 all_high = 0;
1877 }
1878
1879 if (all_high) {
1880 ctrl->timings[channel][slotrank].val_4028--;
1881 printram("4028--;\n");
1882 FOR_ALL_LANES {
1883 ctrl->timings[channel][slotrank].lanes[lane].
1884 timA -= 0x40;
1885 upperA[lane] -= 0x40;
1886
1887 }
1888 } else if (some_high) {
1889 ctrl->timings[channel][slotrank].val_4024++;
1890 ctrl->timings[channel][slotrank].val_4028++;
1891 printram("4024++;\n");
1892 printram("4028++;\n");
1893 }
1894
1895 program_timings(ctrl, channel);
1896
1897 pre_timA_change(ctrl, channel, slotrank, &mnmx);
1898
1899 discover_402x(ctrl, channel, slotrank, upperA);
1900
1901 post_timA_change(ctrl, channel, slotrank, &mnmx);
1902 pre_timA_change(ctrl, channel, slotrank, &mnmx);
1903
1904 discover_timA_fine(ctrl, channel, slotrank, upperA);
1905
1906 post_timA_change(ctrl, channel, slotrank, &mnmx);
1907 pre_timA_change(ctrl, channel, slotrank, &mnmx);
1908
1909 FOR_ALL_LANES {
1910 ctrl->timings[channel][slotrank].lanes[lane].timA -= mnmx.timA_min_high * 0x40;
1911 }
1912 ctrl->timings[channel][slotrank].val_4028 -= mnmx.timA_min_high;
1913 printram("4028 -= %d;\n", mnmx.timA_min_high);
1914
1915 post_timA_change(ctrl, channel, slotrank, &mnmx);
1916
1917 printram("4/8: %d, %d, %x, %x\n", channel, slotrank,
1918 ctrl->timings[channel][slotrank].val_4024,
1919 ctrl->timings[channel][slotrank].val_4028);
1920
1921 FOR_ALL_LANES
1922 printram("%d, %d, %d, %x\n", channel, slotrank,
1923 lane,
1924 ctrl->timings[channel][slotrank].lanes[lane].timA);
1925
1926 write32(DEFAULT_MCHBAR + 0x3400, 0);
1927
1928 r32 = read32(DEFAULT_MCHBAR + 0x5030);
1929 write32(DEFAULT_MCHBAR + 0x5030, r32 | 0x20);
1930 udelay(1);
1931
1932 write32(DEFAULT_MCHBAR + 0x5030, r32 & ~0x20);
1933
1934 udelay(1);
1935 }
1936
1937 FOR_ALL_POPULATED_CHANNELS {
1938 program_timings(ctrl, channel);
1939 }
1940 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
1941 write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel
1942 + 4 * lane, 0);
1943 }
1944}
1945
1946static void test_timC(ramctr_timing * ctrl, int channel, int slotrank)
1947{
1948 int lane;
1949
1950 FOR_ALL_LANES {
1951 write32(DEFAULT_MCHBAR + 0x4340 + 0x400 * channel + 4 * lane, 0);
1952 read32(DEFAULT_MCHBAR + 0x4140 + 0x400 * channel + 4 * lane);
1953 }
1954
1955 wait_428c(channel);
1956
1957 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f006);
1958 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
1959 (max((ctrl->tFAW >> 2) + 1, ctrl->tRRD) << 10)
1960 | 4 | (ctrl->tRCD << 16));
1961
1962 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
1963 (slotrank << 24) | (6 << 16));
1964
1965 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x244);
1966
1967 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f207);
1968 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x8041001);
1969 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
1970 (slotrank << 24) | 8);
1971 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0x3e0);
1972
1973 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f201);
1974 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel, 0x80411f4);
1975 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel, (slotrank << 24));
1976 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0x242);
1977
1978 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f207);
1979 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
1980 0x8000c01 | ((ctrl->CWL + ctrl->tWTR + 5) << 16));
1981 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
1982 (slotrank << 24) | 8);
1983 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0x3e0);
1984
1985 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
1986
1987 wait_428c(channel);
1988
1989 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f002);
1990 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
1991 0xc01 | (ctrl->tRP << 16));
1992 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
1993 (slotrank << 24) | 0x60400);
1994 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x240);
1995
1996 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f006);
1997 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
1998 (max(ctrl->tRRD, (ctrl->tFAW >> 2) + 1) << 10)
1999 | 8 | (ctrl->CAS << 16));
2000
2001 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
2002 (slotrank << 24) | 0x60000);
2003
2004 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0x244);
2005
2006 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f105);
2007 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
2008 0x40011f4 | (max(ctrl->tRTP, 8) << 16));
2009 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel, (slotrank << 24));
2010 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0x242);
2011
2012 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f002);
2013 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
2014 0xc01 | (ctrl->tRP << 16));
2015 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
2016 (slotrank << 24) | 0x60400);
2017 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0x240);
2018 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
2019 wait_428c(channel);
2020}
2021
2022static void discover_timC(ramctr_timing * ctrl, int channel, int slotrank)
2023{
2024 int timC;
2025 int statistics[NUM_LANES][MAX_TIMC + 1];
2026 int lane;
2027
2028 wait_428c(channel);
2029
2030 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f002);
2031 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
2032 0xc01 | (ctrl->tRP << 16));
2033 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2034 (slotrank << 24) | 0x60400);
2035 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x240);
2036 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
2037
2038 for (timC = 0; timC <= MAX_TIMC; timC++) {
2039 FOR_ALL_LANES ctrl->timings[channel][slotrank].lanes[lane].
2040 timC = timC;
2041 program_timings(ctrl, channel);
2042
2043 test_timC(ctrl, channel, slotrank);
2044
2045 FOR_ALL_LANES {
2046 statistics[lane][timC] =
2047 read32(DEFAULT_MCHBAR + 0x4340 + 4 * lane +
2048 0x400 * channel);
2049 printram("Cstat: %d, %d, %d, %x, %x\n",
2050 channel, slotrank, lane, timC,
2051 statistics[lane][timC]);
2052 }
2053 }
2054 FOR_ALL_LANES {
2055 struct run rn =
2056 get_longest_zero_run(statistics[lane], MAX_TIMC + 1);
2057 ctrl->timings[channel][slotrank].lanes[lane].timC = rn.middle;
2058 if (rn.all)
2059 printk(BIOS_CRIT, "timC discovery failed");
2060 printram("Cval: %d, %d, %d, %x\n", channel, slotrank,
2061 lane, ctrl->timings[channel][slotrank].lanes[lane].timC);
2062 }
2063}
2064
2065static int get_precedening_channels(ramctr_timing * ctrl, int target_channel)
2066{
2067 int channel, ret = 0;
2068 FOR_ALL_POPULATED_CHANNELS if (channel < target_channel)
2069 ret++;
2070 return ret;
2071}
2072
2073static void fill_pattern0(ramctr_timing * ctrl, int channel, u32 a, u32 b)
2074{
2075 unsigned j;
2076 unsigned channel_offset =
2077 get_precedening_channels(ctrl, channel) * 0x40;
2078 printram("channel_offset=%x\n", channel_offset);
2079 for (j = 0; j < 16; j++)
2080 write32((void *)(0x04000000 + channel_offset + 4 * j), j & 2 ? b : a);
2081 sfence();
2082}
2083
2084static int num_of_channels(const ramctr_timing * ctrl)
2085{
2086 int ret = 0;
2087 int channel;
2088 FOR_ALL_POPULATED_CHANNELS ret++;
2089 return ret;
2090}
2091
2092static void fill_pattern1(ramctr_timing * ctrl, int channel)
2093{
2094 unsigned j;
2095 unsigned channel_offset =
2096 get_precedening_channels(ctrl, channel) * 0x40;
2097 unsigned channel_step = 0x40 * num_of_channels(ctrl);
2098 for (j = 0; j < 16; j++)
2099 write32((void *)(0x04000000 + channel_offset + j * 4), 0xffffffff);
2100 for (j = 0; j < 16; j++)
2101 write32((void *)(0x04000000 + channel_offset + channel_step + j * 4), 0);
2102 sfence();
2103}
2104
2105static void precharge(ramctr_timing * ctrl)
2106{
2107 int channel, slotrank, lane;
2108
2109 FOR_ALL_POPULATED_CHANNELS {
2110 FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2111 ctrl->timings[channel][slotrank].lanes[lane].falling =
2112 16;
2113 ctrl->timings[channel][slotrank].lanes[lane].rising =
2114 16;
2115 } program_timings(ctrl, channel);
2116
2117 FOR_ALL_POPULATED_RANKS {
2118 wait_428c(channel);
2119
2120 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
2121 0x1f000);
2122 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
2123 0xc01 | (ctrl->tMOD << 16));
2124 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2125 (slotrank << 24) | 0x360004);
2126 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
2127
2128 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
2129 0x1f105);
2130 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
2131 0x4041003);
2132 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
2133 (slotrank << 24) | 0);
2134 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
2135
2136 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
2137 0x1f105);
2138 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
2139 0x1001 | ((ctrl->CAS + 8) << 16));
2140 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
2141 (slotrank << 24) | 0x60000);
2142 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
2143
2144 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
2145 0x1f000);
2146 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
2147 0xc01 | (ctrl->tMOD << 16));
2148 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
2149 (slotrank << 24) | 0x360000);
2150 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
2151 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
2152 0xc0001);
2153
2154 wait_428c(channel);
2155 }
2156
2157 FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2158 ctrl->timings[channel][slotrank].lanes[lane].falling =
2159 48;
2160 ctrl->timings[channel][slotrank].lanes[lane].rising =
2161 48;
2162 }
2163
2164 program_timings(ctrl, channel);
2165
2166 FOR_ALL_POPULATED_RANKS {
2167 wait_428c(channel);
2168
2169 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
2170 0x1f000);
2171 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
2172 0xc01 | (ctrl->tMOD << 16));
2173 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2174 (slotrank << 24) | 0x360004);
2175 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
2176
2177 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
2178 0x1f105);
2179 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
2180 0x4041003);
2181 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
2182 (slotrank << 24) | 0);
2183 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
2184
2185 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
2186 0x1f105);
2187 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
2188 0x1001 | ((ctrl->CAS + 8) << 16));
2189 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
2190 (slotrank << 24) | 0x60000);
2191 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
2192
2193 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
2194 0x1f000);
2195 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
2196 0xc01 | (ctrl->tMOD << 16));
2197
2198 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
2199 (slotrank << 24) | 0x360000);
2200 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
2201
2202 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
2203 0xc0001);
2204 wait_428c(channel);
2205 }
2206 }
2207}
2208
2209static void test_timB(ramctr_timing * ctrl, int channel, int slotrank)
2210{
2211 write_mrreg(ctrl, channel, slotrank, 1,
2212 0x80 | make_mr1(ctrl, slotrank));
2213
2214 wait_428c(channel);
2215 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f207);
2216 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
2217 0x8000c01 | ((ctrl->CWL + ctrl->tWLO) << 16));
2218 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2219 8 | (slotrank << 24));
2220 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
2221
2222 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f107);
2223 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
2224 0x4000c01 | ((ctrl->CAS + 38) << 16));
2225 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
2226 (slotrank << 24) | 4);
2227 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
2228
2229 write32(DEFAULT_MCHBAR + 0x400 * channel + 0x4284, 0x40001);
2230 wait_428c(channel);
2231
2232 write_mrreg(ctrl, channel, slotrank, 1,
2233 0x1080 | make_mr1(ctrl, slotrank));
2234}
2235
2236static void discover_timB(ramctr_timing * ctrl, int channel, int slotrank)
2237{
2238 int timB;
2239 int statistics[NUM_LANES][128];
2240 int lane;
2241
2242 write32(DEFAULT_MCHBAR + 0x3400, 0x108052 | (slotrank << 2));
2243
2244 for (timB = 0; timB < 128; timB++) {
2245 FOR_ALL_LANES {
2246 ctrl->timings[channel][slotrank].lanes[lane].timB = timB;
2247 }
2248 program_timings(ctrl, channel);
2249
2250 test_timB(ctrl, channel, slotrank);
2251
2252 FOR_ALL_LANES {
2253 statistics[lane][timB] =
2254 !((read32
2255 (DEFAULT_MCHBAR + lane_registers[lane] +
2256 channel * 0x100 + 4 + ((timB / 32) & 1) * 4)
2257 >> (timB % 32)) & 1);
2258 printram("Bstat: %d, %d, %d, %x, %x\n",
2259 channel, slotrank, lane, timB,
2260 statistics[lane][timB]);
2261 }
2262 }
2263 FOR_ALL_LANES {
2264 struct run rn = get_longest_zero_run(statistics[lane], 128);
Patrick Rudolph9f1fbb92015-08-17 19:24:12 +0200[diff] [blame^]2265 if (rn.start < rn.middle) {
2266 ctrl->timings[channel][slotrank].lanes[lane].timB = rn.start;
2267 } else {
2268 /* In this case statistics[lane][7f] and statistics[lane][0] are
2269 * both zero.
2270 * Prefer a smaller value over rn.start to prevent failures in
2271 * the following write tests.
2272 */
2273 ctrl->timings[channel][slotrank].lanes[lane].timB = 0;
2274 }
2275
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]2276 if (rn.all)
2277 die("timB discovery failed");
2278 printram("Bval: %d, %d, %d, %x\n", channel, slotrank,
2279 lane, ctrl->timings[channel][slotrank].lanes[lane].timB);
2280 }
2281}
2282
2283static int get_timB_high_adjust(u64 val)
2284{
2285 int i;
2286
2287 /* good */
2288 if (val == 0xffffffffffffffffLL)
2289 return 0;
2290
2291 if (val >= 0xf000000000000000LL) {
2292 /* needs negative adjustment */
2293 for (i = 0; i < 8; i++)
2294 if (val << (8 * (7 - i) + 4))
2295 return -i;
2296 } else {
2297 /* needs positive adjustment */
2298 for (i = 0; i < 8; i++)
2299 if (val >> (8 * (7 - i) + 4))
2300 return i;
2301 }
2302 return 8;
2303}
2304
2305static void adjust_high_timB(ramctr_timing * ctrl)
2306{
2307 int channel, slotrank, lane, old;
2308 write32(DEFAULT_MCHBAR + 0x3400, 0x200);
2309 FOR_ALL_POPULATED_CHANNELS {
2310 fill_pattern1(ctrl, channel);
2311 write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 1);
2312 }
2313 FOR_ALL_POPULATED_CHANNELS FOR_ALL_POPULATED_RANKS {
2314
2315 write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x10001);
2316
2317 wait_428c(channel);
2318
2319 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f006);
2320 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
2321 0xc01 | (ctrl->tRCD << 16));
2322 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2323 (slotrank << 24) | 0x60000);
2324 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
2325
2326 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f207);
2327 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x8040c01);
2328 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
2329 (slotrank << 24) | 0x8);
2330 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0x3e0);
2331
2332 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f201);
2333 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel, 0x8041003);
2334 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
2335 (slotrank << 24));
2336 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0x3e2);
2337
2338 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f207);
2339 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
2340 0x8000c01 | ((ctrl->CWL + ctrl->tWTR + 5) << 16));
2341 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
2342 (slotrank << 24) | 0x8);
2343 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0x3e0);
2344
2345 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
2346
2347 wait_428c(channel);
2348
2349 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f002);
2350 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
2351 0xc01 | ((ctrl->tRP) << 16));
2352 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2353 (slotrank << 24) | 0x60400);
2354 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x240);
2355
2356 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f006);
2357 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
2358 0xc01 | ((ctrl->tRCD) << 16));
2359 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
2360 (slotrank << 24) | 0x60000);
2361 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
2362
2363 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x3f105);
2364 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
2365 0x4000c01 |
2366 ((ctrl->tRP +
2367 ctrl->timings[channel][slotrank].val_4024 +
2368 ctrl->timings[channel][slotrank].val_4028) << 16));
2369 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
2370 (slotrank << 24) | 0x60008);
2371 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
2372
2373 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0x80001);
2374 wait_428c(channel);
2375 FOR_ALL_LANES {
2376 u64 res =
2377 read32(DEFAULT_MCHBAR + lane_registers[lane] +
2378 0x100 * channel + 4);
2379 res |=
2380 ((u64) read32(DEFAULT_MCHBAR + lane_registers[lane] +
2381 0x100 * channel + 8)) << 32;
2382 old = ctrl->timings[channel][slotrank].lanes[lane].timB;
2383 ctrl->timings[channel][slotrank].lanes[lane].timB +=
2384 get_timB_high_adjust(res) * 64;
2385
2386 printk(BIOS_DEBUG, "High adjust %d:%016llx\n", lane, res);
2387 printram("Bval+: %d, %d, %d, %x -> %x\n", channel,
2388 slotrank, lane, old,
2389 ctrl->timings[channel][slotrank].lanes[lane].
2390 timB);
2391 }
2392 }
2393 write32(DEFAULT_MCHBAR + 0x3400, 0);
2394}
2395
2396static void write_op(ramctr_timing * ctrl, int channel)
2397{
2398 int slotrank;
2399
2400 wait_428c(channel);
2401
2402 /* choose an existing rank. */
2403 slotrank = !(ctrl->rankmap[channel] & 1) ? 2 : 0;
2404
2405 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
2406 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x41001);
2407
2408 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2409 (slotrank << 24) | 0x60000);
2410
2411 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
2412
2413 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
2414 wait_428c(channel);
2415}
2416
2417static void write_training(ramctr_timing * ctrl)
2418{
2419 int channel, slotrank, lane;
2420 u32 r32;
2421
2422 FOR_ALL_POPULATED_CHANNELS
2423 write32(DEFAULT_MCHBAR + 0x4008 + 0x400 * channel,
2424 read32(DEFAULT_MCHBAR + 0x4008 +
2425 0x400 * channel) | 0x8000000);
2426
2427 FOR_ALL_POPULATED_CHANNELS {
2428 write_op(ctrl, channel);
2429 write32(DEFAULT_MCHBAR + 0x4020 + 0x400 * channel,
2430 read32(DEFAULT_MCHBAR + 0x4020 +
2431 0x400 * channel) | 0x200000);
2432 }
2433 write32(DEFAULT_MCHBAR + 0x5030, read32(DEFAULT_MCHBAR + 0x5030) & ~8);
2434 FOR_ALL_POPULATED_CHANNELS {
2435 write_op(ctrl, channel);
2436 }
2437
2438 FOR_ALL_CHANNELS
2439 FOR_ALL_POPULATED_RANKS
2440 write_mrreg(ctrl, channel, slotrank, 1,
2441 make_mr1(ctrl, slotrank) | 0x1080);
2442
2443 write32(DEFAULT_MCHBAR + 0x3400, 0x108052);
2444
2445 r32 = read32(DEFAULT_MCHBAR + 0x5030);
2446 write32(DEFAULT_MCHBAR + 0x5030, r32 | 0x20);
2447 udelay(1);
2448
2449 write32(DEFAULT_MCHBAR + 0x5030, r32 & ~0x20);
2450
2451 udelay(1);
2452
2453 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS
2454 discover_timB(ctrl, channel, slotrank);
2455
2456 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS
2457 write_mrreg(ctrl, channel,
2458 slotrank, 1, make_mr1(ctrl, slotrank));
2459
2460 write32(DEFAULT_MCHBAR + 0x3400, 0);
2461
2462 FOR_ALL_POPULATED_CHANNELS
2463 wait_428c(channel);
2464
2465 write32(DEFAULT_MCHBAR + 0x5030, read32(DEFAULT_MCHBAR + 0x5030) | 8);
2466
2467 FOR_ALL_POPULATED_CHANNELS {
2468 write32(DEFAULT_MCHBAR + 0x4020 + 0x400 * channel,
2469 ~0x00200000 & read32(DEFAULT_MCHBAR + 0x4020 +
2470 0x400 * channel));
2471 read32(DEFAULT_MCHBAR + 0x428c + 0x400 * channel);
2472 wait_428c(channel);
2473
2474 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
2475 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x659001);
2476 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel, 0x60000);
2477 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
2478
2479 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
2480 wait_428c(channel);
2481 }
2482
2483 r32 = read32(DEFAULT_MCHBAR + 0x5030);
2484 write32(DEFAULT_MCHBAR + 0x5030, r32 | 0x20);
2485 udelay(1);
2486
2487 write32(DEFAULT_MCHBAR + 0x5030, r32 & ~0x20);
2488
2489 udelay(1);
2490
2491 printram("CPE\n");
2492 precharge(ctrl);
2493 printram("CPF\n");
2494
2495 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2496 read32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane);
2497 write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane,
2498 0);
2499 }
2500
2501 FOR_ALL_POPULATED_CHANNELS {
2502 fill_pattern0(ctrl, channel, 0xaaaaaaaa, 0x55555555);
2503 write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 0);
2504 }
2505
2506 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS
2507 discover_timC(ctrl, channel, slotrank);
2508
2509 FOR_ALL_POPULATED_CHANNELS
2510 program_timings(ctrl, channel);
2511
2512 adjust_high_timB(ctrl);
2513
2514 FOR_ALL_POPULATED_CHANNELS
2515 program_timings(ctrl, channel);
2516
2517 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2518 read32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane);
2519 write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane,
2520 0);
2521 }
2522}
2523
2524static int test_320c(ramctr_timing * ctrl, int channel, int slotrank)
2525{
2526 struct ram_rank_timings saved_rt = ctrl->timings[channel][slotrank];
2527 int timC_delta;
2528 int lanes_ok = 0;
2529 int ctr = 0;
2530 int lane;
2531
2532 for (timC_delta = -5; timC_delta <= 5; timC_delta++) {
2533 FOR_ALL_LANES {
2534 ctrl->timings[channel][slotrank].lanes[lane].timC =
2535 saved_rt.lanes[lane].timC + timC_delta;
2536 }
2537 program_timings(ctrl, channel);
2538 FOR_ALL_LANES {
2539 write32(DEFAULT_MCHBAR + 4 * lane + 0x4f40, 0);
2540 }
2541
2542 write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x1f);
2543
2544 wait_428c(channel);
2545
2546 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f006);
2547 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
2548 ((max(ctrl->tRRD, (ctrl->tFAW >> 2) + 1)) << 10)
2549 | 8 | (ctrl->tRCD << 16));
2550
2551 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2552 (slotrank << 24) | ctr | 0x60000);
2553
2554 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x244);
2555
2556 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f201);
2557 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
2558 0x8001020 | ((ctrl->CWL + ctrl->tWTR + 8) << 16));
2559 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
2560 (slotrank << 24));
2561 write32(DEFAULT_MCHBAR + 0x4244 + 0x400 * channel, 0x389abcd);
2562 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0x20e42);
2563
2564 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f105);
2565 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
2566 0x4001020 | (max(ctrl->tRTP, 8) << 16));
2567 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
2568 (slotrank << 24));
2569 write32(DEFAULT_MCHBAR + 0x4248 + 0x400 * channel, 0x389abcd);
2570 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0x20e42);
2571
2572 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f002);
2573 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel, 0xf1001);
2574 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
2575 (slotrank << 24) | 0x60400);
2576 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0x240);
2577
2578 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
2579 wait_428c(channel);
2580 FOR_ALL_LANES {
2581 u32 r32 =
2582 read32(DEFAULT_MCHBAR + 0x4340 + 4 * lane +
2583 0x400 * channel);
2584
2585 if (r32 == 0)
2586 lanes_ok |= 1 << lane;
2587 }
2588 ctr++;
2589 if (lanes_ok == ((1 << NUM_LANES) - 1))
2590 break;
2591 }
2592
2593 ctrl->timings[channel][slotrank] = saved_rt;
2594
2595 printram("3lanes: %x\n", lanes_ok);
2596 return lanes_ok != ((1 << NUM_LANES) - 1);
2597}
2598
2599#include "raminit_patterns.h"
2600
2601static void fill_pattern5(ramctr_timing * ctrl, int channel, int patno)
2602{
2603 unsigned i, j;
2604 unsigned channel_offset =
2605 get_precedening_channels(ctrl, channel) * 0x40;
2606 unsigned channel_step = 0x40 * num_of_channels(ctrl);
2607
2608 if (patno) {
2609 u8 base8 = 0x80 >> ((patno - 1) % 8);
2610 u32 base = base8 | (base8 << 8) | (base8 << 16) | (base8 << 24);
2611 for (i = 0; i < 32; i++) {
2612 for (j = 0; j < 16; j++) {
2613 u32 val = use_base[patno - 1][i] & (1 << (j / 2)) ? base : 0;
2614 if (invert[patno - 1][i] & (1 << (j / 2)))
2615 val = ~val;
2616 write32((void *)(0x04000000 + channel_offset + i * channel_step +
2617 j * 4), val);
2618 }
2619 }
2620
2621 } else {
2622 for (i = 0; i < sizeof(pattern) / sizeof(pattern[0]); i++) {
2623 for (j = 0; j < 16; j++)
2624 write32((void *)(0x04000000 + channel_offset + i * channel_step +
2625 j * 4), pattern[i][j]);
2626 }
2627 sfence();
2628 }
2629}
2630
2631static void reprogram_320c(ramctr_timing * ctrl)
2632{
2633 int channel, slotrank;
2634 u32 r32;
2635
2636 FOR_ALL_POPULATED_CHANNELS {
2637 wait_428c(channel);
2638
2639 /* choose an existing rank. */
2640 slotrank = !(ctrl->rankmap[channel] & 1) ? 2 : 0;
2641
2642 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
2643 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x41001);
2644
2645 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2646 (slotrank << 24) | 0x60000);
2647
2648 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
2649
2650 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
2651 wait_428c(channel);
2652 write32(DEFAULT_MCHBAR + 0x4020 + 0x400 * channel,
2653 read32(DEFAULT_MCHBAR + 0x4020 +
2654 0x400 * channel) | 0x200000);
2655 }
2656 write32(DEFAULT_MCHBAR + 0x5030, read32(DEFAULT_MCHBAR + 0x5030) & ~8);
2657 FOR_ALL_POPULATED_CHANNELS {
2658 wait_428c(channel);
2659
2660 /* choose an existing rank. */
2661 slotrank = !(ctrl->rankmap[channel] & 1) ? 2 : 0;
2662
2663 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
2664 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x41001);
2665
2666 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2667 (slotrank << 24) | 0x60000);
2668
2669 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
2670
2671 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
2672 wait_428c(channel);
2673 }
2674
2675 /* jedec reset */
2676 dram_jedecreset(ctrl);
2677 /* mrs commands. */
2678 dram_mrscommands(ctrl);
2679
2680 r32 = read32(DEFAULT_MCHBAR + 0x5030);
2681 write32(DEFAULT_MCHBAR + 0x5030, r32 | 0x20);
2682 udelay(1);
2683
2684 write32(DEFAULT_MCHBAR + 0x5030, r32 & ~0x20);
2685
2686 udelay(1);
2687}
2688
2689#define MIN_C320C_LEN 13
2690
2691static int try_cmd_stretch(ramctr_timing * ctrl, int cmd_stretch)
2692{
2693 struct ram_rank_timings saved_timings[NUM_CHANNELS][NUM_SLOTRANKS];
2694 int channel, slotrank;
2695 int c320c;
2696 int stat[NUM_SLOTRANKS][256];
2697
2698 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
2699 saved_timings[channel][slotrank] = ctrl->timings[channel][slotrank];
2700 }
2701
2702 FOR_ALL_POPULATED_CHANNELS {
2703 ctrl->cmd_stretch[channel] = cmd_stretch;
2704 }
2705
2706 FOR_ALL_POPULATED_CHANNELS
2707 MCHBAR32(0x4004 + 0x400 * channel) =
2708 ctrl->tRRD
2709 | (ctrl->tRTP << 4)
2710 | (ctrl->tCKE << 8)
2711 | (ctrl->tWTR << 12)
2712 | (ctrl->tFAW << 16)
2713 | (ctrl->tWR << 24)
2714 | (ctrl->cmd_stretch[channel] << 30);
2715
2716
2717 FOR_ALL_CHANNELS {
2718 int delta = 0;
2719 if (ctrl->cmd_stretch[channel] == 2)
2720 delta = 2;
2721 else if (ctrl->cmd_stretch[channel] == 0)
2722 delta = 4;
2723
2724 FOR_ALL_POPULATED_RANKS {
2725 ctrl->timings[channel][slotrank].val_4024 -= delta;
2726 }
2727 }
2728
2729 FOR_ALL_POPULATED_CHANNELS {
2730 for (c320c = -127; c320c <= 127; c320c++) {
2731 FOR_ALL_POPULATED_RANKS {
2732 ctrl->timings[channel][slotrank].val_320c = c320c;
2733 }
2734 program_timings(ctrl, channel);
2735 reprogram_320c(ctrl);
2736 FOR_ALL_POPULATED_RANKS {
2737 stat[slotrank][c320c + 127] =
2738 test_320c(ctrl, channel, slotrank);
2739 printram("3stat: %d, %d, %d: %d\n",
2740 channel, slotrank, c320c,
2741 stat[slotrank][c320c + 127]);
2742 }
2743 }
2744 FOR_ALL_POPULATED_RANKS {
2745 struct run rn =
2746 get_longest_zero_run(stat[slotrank], 255);
2747 ctrl->timings[channel][slotrank].val_320c =
2748 rn.middle - 127;
2749 printram("3val: %d, %d: %d\n", channel,
2750 slotrank,
2751 ctrl->timings[channel][slotrank].val_320c);
2752 if (rn.all || rn.length < MIN_C320C_LEN) {
2753 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
2754 ctrl->timings[channel][slotrank] = saved_timings[channel][slotrank];
2755 }
2756 return 0;
2757 }
2758 }
2759 }
2760 return 1;
2761}
2762
2763static void command_training(ramctr_timing * ctrl)
2764{
2765 int channel;
2766
2767 FOR_ALL_POPULATED_CHANNELS {
2768 fill_pattern5(ctrl, channel, 0);
2769 write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x1f);
2770 }
2771
2772 /* try command rate 1T and 2T */
2773 if (!try_cmd_stretch(ctrl, 0) && !try_cmd_stretch(ctrl, 2))
2774 die("c320c discovery failed");
2775
2776 FOR_ALL_POPULATED_CHANNELS {
2777 program_timings(ctrl, channel);
2778 }
2779
2780 reprogram_320c(ctrl);
2781}
2782
2783static void discover_edges_real(ramctr_timing * ctrl, int channel, int slotrank,
2784 int *edges)
2785{
2786 int edge;
2787 int statistics[NUM_LANES][MAX_EDGE_TIMING + 1];
2788 int lane;
2789
2790 for (edge = 0; edge <= MAX_EDGE_TIMING; edge++) {
2791 FOR_ALL_LANES {
2792 ctrl->timings[channel][slotrank].lanes[lane].rising =
2793 edge;
2794 ctrl->timings[channel][slotrank].lanes[lane].falling =
2795 edge;
2796 }
2797 printram("edge %02x\n", edge);
2798 program_timings(ctrl, channel);
2799
2800 FOR_ALL_LANES {
2801 write32(DEFAULT_MCHBAR + 0x4340 + 0x400 * channel +
2802 4 * lane, 0);
2803 read32(DEFAULT_MCHBAR + 0x400 * channel + 4 * lane +
2804 0x4140);
2805 }
2806
2807 wait_428c(channel);
2808
2809 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f000);
2810 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
2811 (0xc01 | (ctrl->tMOD << 16)));
2812 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2813 (slotrank << 24) | 0x360004);
2814 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
2815
2816 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f105);
2817 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x40411f4);
2818 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
2819 (slotrank << 24));
2820 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
2821
2822 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f105);
2823 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
2824 0x1001 | ((ctrl->CAS + 8) << 16));
2825 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
2826 (slotrank << 24) | 0x60000);
2827 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
2828
2829 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f000);
2830 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
2831 (0xc01 | (ctrl->tMOD << 16)));
2832 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
2833 (slotrank << 24) | 0x360000);
2834 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
2835
2836 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
2837
2838 wait_428c(channel);
2839
2840 FOR_ALL_LANES {
2841 statistics[lane][edge] =
2842 read32(DEFAULT_MCHBAR + 0x4340 + 0x400 * channel +
2843 lane * 4);
2844 }
2845 }
2846 FOR_ALL_LANES {
2847 struct run rn =
2848 get_longest_zero_run(statistics[lane], MAX_EDGE_TIMING + 1);
2849 edges[lane] = rn.middle;
2850 if (rn.all)
2851 die("edge discovery failed");
2852 printram("eval %d, %d, %d, %02x\n", channel, slotrank,
2853 lane, edges[lane]);
2854 }
2855}
2856
2857static void discover_edges(ramctr_timing * ctrl)
2858{
2859 int falling_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
2860 int rising_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
2861 int channel, slotrank, lane;
2862 u32 r32;
2863
2864 write32(DEFAULT_MCHBAR + 0x3400, 0);
2865
2866 r32 = read32(DEFAULT_MCHBAR + 0x5030);
2867 write32(DEFAULT_MCHBAR + 0x5030, r32 | 0x20);
2868 udelay(1);
2869
2870 write32(DEFAULT_MCHBAR + 0x5030, r32 & ~0x20);
2871
2872 udelay(1);
2873
2874 FOR_ALL_POPULATED_CHANNELS FOR_ALL_LANES {
2875 write32(DEFAULT_MCHBAR + 4 * lane +
2876 0x400 * channel + 0x4080, 0);
2877 }
2878
2879 FOR_ALL_POPULATED_CHANNELS {
2880 fill_pattern0(ctrl, channel, 0, 0);
2881 write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 0);
2882 FOR_ALL_LANES {
2883 read32(DEFAULT_MCHBAR + 0x400 * channel +
2884 lane * 4 + 0x4140);
2885 }
2886
2887 FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2888 ctrl->timings[channel][slotrank].lanes[lane].falling =
2889 16;
2890 ctrl->timings[channel][slotrank].lanes[lane].rising =
2891 16;
2892 }
2893
2894 program_timings(ctrl, channel);
2895
2896 FOR_ALL_POPULATED_RANKS {
2897 wait_428c(channel);
2898
2899 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
2900 0x1f000);
2901 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
2902 0xc01 | (ctrl->tMOD << 16));
2903 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2904 (slotrank << 24) | 0x360004);
2905 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
2906
2907 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
2908 0x1f105);
2909 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
2910 0x4041003);
2911 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
2912 (slotrank << 24) | 0);
2913 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
2914
2915 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
2916 0x1f105);
2917 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
2918 0x1001 | ((ctrl->CAS + 8) << 16));
2919 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
2920 (slotrank << 24) | 0x60000);
2921 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
2922
2923 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
2924 0x1f000);
2925 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
2926 0xc01 | (ctrl->tMOD << 16));
2927 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
2928 (slotrank << 24) | 0x360000);
2929 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
2930 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
2931 0xc0001);
2932
2933 wait_428c(channel);
2934 }
2935
2936 FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
2937 ctrl->timings[channel][slotrank].lanes[lane].falling =
2938 48;
2939 ctrl->timings[channel][slotrank].lanes[lane].rising =
2940 48;
2941 }
2942
2943 program_timings(ctrl, channel);
2944
2945 FOR_ALL_POPULATED_RANKS {
2946 wait_428c(channel);
2947
2948 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
2949 0x1f000);
2950 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
2951 0xc01 | (ctrl->tMOD << 16));
2952 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
2953 (slotrank << 24) | 0x360004);
2954 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
2955
2956 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
2957 0x1f105);
2958 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
2959 0x4041003);
2960 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
2961 (slotrank << 24) | 0);
2962 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
2963
2964 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
2965 0x1f105);
2966 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
2967 0x1001 | ((ctrl->CAS + 8) << 16));
2968 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
2969 (slotrank << 24) | 0x60000);
2970 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
2971
2972 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
2973 0x1f000);
2974 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
2975 0xc01 | (ctrl->tMOD << 16));
2976 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
2977 (slotrank << 24) | 0x360000);
2978 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
2979
2980 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
2981 0xc0001);
2982 wait_428c(channel);
2983 }
2984
2985 FOR_ALL_LANES {
2986 write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel +
2987 lane * 4,
2988 ~read32(DEFAULT_MCHBAR + 0x4040 +
2989 0x400 * channel + lane * 4) & 0xff);
2990 }
2991
2992 fill_pattern0(ctrl, channel, 0, 0xffffffff);
2993 write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 0);
2994 }
2995
2996 /* FIXME: under some conditions (older chipsets?) vendor BIOS sets both edges to the same value. */
2997 write32(DEFAULT_MCHBAR + 0x4eb0, 0x300);
2998
2999 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
3000 discover_edges_real(ctrl, channel, slotrank,
3001 falling_edges[channel][slotrank]);
3002 }
3003
3004 write32(DEFAULT_MCHBAR + 0x4eb0, 0x200);
3005
3006 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
3007 discover_edges_real(ctrl, channel, slotrank,
3008 rising_edges[channel][slotrank]);
3009 }
3010
3011 write32(DEFAULT_MCHBAR + 0x4eb0, 0);
3012
3013 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
3014 ctrl->timings[channel][slotrank].lanes[lane].falling =
3015 falling_edges[channel][slotrank][lane];
3016 ctrl->timings[channel][slotrank].lanes[lane].rising =
3017 rising_edges[channel][slotrank][lane];
3018 }
3019
3020 FOR_ALL_POPULATED_CHANNELS {
3021 program_timings(ctrl, channel);
3022 }
3023
3024 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
3025 write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane,
3026 0);
3027 }
3028}
3029
3030static void discover_edges_write_real(ramctr_timing * ctrl, int channel,
3031 int slotrank, int *edges)
3032{
3033 int edge;
3034 u32 raw_statistics[MAX_EDGE_TIMING + 1];
3035 int statistics[MAX_EDGE_TIMING + 1];
3036 const int reg3000b24[] = { 0, 0xc, 0x2c };
3037 int lane, i;
3038 int lower[NUM_LANES];
3039 int upper[NUM_LANES];
3040 int pat;
3041
3042 FOR_ALL_LANES {
3043 lower[lane] = 0;
3044 upper[lane] = MAX_EDGE_TIMING;
3045 }
3046
3047 for (i = 0; i < 3; i++) {
3048 write32(DEFAULT_MCHBAR + 0x3000 + 0x100 * channel,
3049 reg3000b24[i] << 24);
3050 for (pat = 0; pat < NUM_PATTERNS; pat++) {
3051 fill_pattern5(ctrl, channel, pat);
3052 write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x1f);
3053 printram("patterned\n");
3054 printram("[%x] = 0x%08x\n(%d, %d)\n",
3055 0x3000 + 0x100 * channel, reg3000b24[i] << 24, channel,
3056 slotrank);
3057 for (edge = 0; edge <= MAX_EDGE_TIMING; edge++) {
3058 FOR_ALL_LANES {
3059 ctrl->timings[channel][slotrank].lanes[lane].
3060 rising = edge;
3061 ctrl->timings[channel][slotrank].lanes[lane].
3062 falling = edge;
3063 }
3064 program_timings(ctrl, channel);
3065
3066 FOR_ALL_LANES {
3067 write32(DEFAULT_MCHBAR + 0x4340 +
3068 0x400 * channel + 4 * lane, 0);
3069 read32(DEFAULT_MCHBAR + 0x400 * channel +
3070 4 * lane + 0x4140);
3071 }
3072 wait_428c(channel);
3073
3074 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
3075 0x1f006);
3076 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
3077 0x4 | (ctrl->tRCD << 16)
3078 | (max(ctrl->tRRD, (ctrl->tFAW >> 2) + 1) <<
3079 10));
3080 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
3081 (slotrank << 24) | 0x60000);
3082 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel,
3083 0x240);
3084
3085 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
3086 0x1f201);
3087 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
3088 0x8005020 | ((ctrl->tWTR + ctrl->CWL + 8) <<
3089 16));
3090 write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
3091 (slotrank << 24));
3092 write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel,
3093 0x242);
3094
3095 write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
3096 0x1f105);
3097 write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
3098 0x4005020 | (max(ctrl->tRTP, 8) << 16));
3099 write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
3100 (slotrank << 24));
3101 write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel,
3102 0x242);
3103
3104 write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
3105 0x1f002);
3106 write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
3107 0xc01 | (ctrl->tRP << 16));
3108 write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
3109 (slotrank << 24) | 0x60400);
3110 write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
3111
3112 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
3113 0xc0001);
3114 wait_428c(channel);
3115 FOR_ALL_LANES {
3116 read32(DEFAULT_MCHBAR + 0x4340 +
3117 0x400 * channel + lane * 4);
3118 }
3119
3120 raw_statistics[edge] =
3121 MCHBAR32(0x436c + 0x400 * channel);
3122 }
3123 FOR_ALL_LANES {
3124 struct run rn;
3125 for (edge = 0; edge <= MAX_EDGE_TIMING; edge++)
3126 statistics[edge] =
3127 ! !(raw_statistics[edge] & (1 << lane));
3128 rn = get_longest_zero_run(statistics,
3129 MAX_EDGE_TIMING + 1);
3130 printram("edges: %d, %d, %d: 0x%x-0x%x-0x%x, 0x%x-0x%x\n",
3131 channel, slotrank, i, rn.start, rn.middle,
3132 rn.end, rn.start + ctrl->edge_offset[i],
3133 rn.end - ctrl->edge_offset[i]);
3134 lower[lane] =
3135 max(rn.start + ctrl->edge_offset[i], lower[lane]);
3136 upper[lane] =
3137 min(rn.end - ctrl->edge_offset[i], upper[lane]);
3138 edges[lane] = (lower[lane] + upper[lane]) / 2;
Patrick Rudolph9733e282015-08-16 17:06:30 +0200[diff] [blame]3139 if (rn.all || (lower[lane] > upper[lane]))
3140 die("edge write discovery failed");
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]3141
3142 }
3143 }
3144 }
3145
3146 write32(DEFAULT_MCHBAR + 0x3000, 0);
3147 printram("CPA\n");
3148}
3149
3150static void discover_edges_write(ramctr_timing * ctrl)
3151{
3152 int falling_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
3153 int rising_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
3154 int channel, slotrank, lane;
3155
3156 /* FIXME: under some conditions (older chipsets?) vendor BIOS sets both edges to the same value. */
3157 write32(DEFAULT_MCHBAR + 0x4eb0, 0x300);
3158
3159 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
3160 discover_edges_write_real(ctrl, channel, slotrank,
3161 falling_edges[channel][slotrank]);
3162 }
3163
3164 write32(DEFAULT_MCHBAR + 0x4eb0, 0x200);
3165
3166 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
3167 discover_edges_write_real(ctrl, channel, slotrank,
3168 rising_edges[channel][slotrank]);
3169 }
3170
3171 write32(DEFAULT_MCHBAR + 0x4eb0, 0);
3172
3173 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
3174 ctrl->timings[channel][slotrank].lanes[lane].falling =
3175 falling_edges[channel][slotrank][lane];
3176 ctrl->timings[channel][slotrank].lanes[lane].rising =
3177 rising_edges[channel][slotrank][lane];
3178 }
3179
3180 FOR_ALL_POPULATED_CHANNELS
3181 program_timings(ctrl, channel);
3182
3183 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
3184 write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane,
3185 0);
3186 }
3187}
3188
3189static void test_timC_write(ramctr_timing *ctrl, int channel, int slotrank)
3190{
3191 wait_428c(channel);
3192 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f006);
3193 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
3194 (max((ctrl->tFAW >> 2) + 1, ctrl->tRRD)
3195 << 10) | (ctrl->tRCD << 16) | 4);
3196 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
3197 (slotrank << 24) | 0x60000);
3198 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x244);
3199
3200 write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f201);
3201 write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
3202 0x80011e0 |
3203 ((ctrl->tWTR + ctrl->CWL + 8) << 16));
3204 write32(DEFAULT_MCHBAR + 0x4204 +
3205 0x400 * channel, (slotrank << 24));
3206 write32(DEFAULT_MCHBAR + 0x4214 +
3207 0x400 * channel, 0x242);
3208
3209 write32(DEFAULT_MCHBAR + 0x4228 +
3210 0x400 * channel, 0x1f105);
3211 write32(DEFAULT_MCHBAR + 0x4238 +
3212 0x400 * channel,
3213 0x40011e0 | (max(ctrl->tRTP, 8) << 16));
3214 write32(DEFAULT_MCHBAR + 0x4208 +
3215 0x400 * channel, (slotrank << 24));
3216 write32(DEFAULT_MCHBAR + 0x4218 +
3217 0x400 * channel, 0x242);
3218
3219 write32(DEFAULT_MCHBAR + 0x422c +
3220 0x400 * channel, 0x1f002);
3221 write32(DEFAULT_MCHBAR + 0x423c +
3222 0x400 * channel,
3223 0x1001 | (ctrl->tRP << 16));
3224 write32(DEFAULT_MCHBAR + 0x420c +
3225 0x400 * channel,
3226 (slotrank << 24) | 0x60400);
3227 write32(DEFAULT_MCHBAR + 0x421c +
3228 0x400 * channel, 0);
3229
3230 write32(DEFAULT_MCHBAR + 0x4284 +
3231 0x400 * channel, 0xc0001);
3232 wait_428c(channel);
3233}
3234
3235static void discover_timC_write(ramctr_timing * ctrl)
3236{
3237 const u8 rege3c_b24[3] = { 0, 0xf, 0x2f };
3238 int i, pat;
3239
3240 int lower[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
3241 int upper[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
3242 int channel, slotrank, lane;
3243
3244 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
3245 lower[channel][slotrank][lane] = 0;
3246 upper[channel][slotrank][lane] = MAX_TIMC;
3247 }
3248
3249 write32(DEFAULT_MCHBAR + 0x4ea8, 1);
3250
3251 for (i = 0; i < 3; i++)
3252 FOR_ALL_POPULATED_CHANNELS {
3253 write32(DEFAULT_MCHBAR + 0xe3c + (channel * 0x100),
3254 (rege3c_b24[i] << 24)
3255 | (read32(DEFAULT_MCHBAR + 0xe3c + (channel * 0x100))
3256 & ~0x3f000000));
3257 udelay(2);
3258 for (pat = 0; pat < NUM_PATTERNS; pat++) {
3259 FOR_ALL_POPULATED_RANKS {
3260 int timC;
3261 u32 raw_statistics[MAX_TIMC + 1];
3262 int statistics[MAX_TIMC + 1];
3263
3264 fill_pattern5(ctrl, channel, pat);
3265 write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x1f);
3266 for (timC = 0; timC < MAX_TIMC + 1; timC++) {
3267 FOR_ALL_LANES
3268 ctrl->timings[channel][slotrank].lanes[lane].timC = timC;
3269 program_timings(ctrl, channel);
3270
3271 test_timC_write (ctrl, channel, slotrank);
3272
3273 raw_statistics[timC] =
3274 MCHBAR32(0x436c + 0x400 * channel);
3275 }
3276 FOR_ALL_LANES {
3277 struct run rn;
3278 for (timC = 0; timC <= MAX_TIMC; timC++)
3279 statistics[timC] =
3280 !!(raw_statistics[timC] &
3281 (1 << lane));
3282 rn = get_longest_zero_run(statistics,
3283 MAX_TIMC + 1);
3284 if (rn.all)
3285 die("timC write discovery failed");
3286 printram("timC: %d, %d, %d: 0x%x-0x%x-0x%x, 0x%x-0x%x\n",
3287 channel, slotrank, i, rn.start,
3288 rn.middle, rn.end,
3289 rn.start + ctrl->timC_offset[i],
3290 rn.end - ctrl->timC_offset[i]);
3291 lower[channel][slotrank][lane] =
3292 max(rn.start + ctrl->timC_offset[i],
3293 lower[channel][slotrank][lane]);
3294 upper[channel][slotrank][lane] =
3295 min(rn.end - ctrl->timC_offset[i],
3296 upper[channel][slotrank][lane]);
3297
3298 }
3299 }
3300 }
3301 }
3302
3303 FOR_ALL_CHANNELS {
3304 write32(DEFAULT_MCHBAR + (channel * 0x100) + 0xe3c,
3305 0 | (read32(DEFAULT_MCHBAR + (channel * 0x100) + 0xe3c) &
3306 ~0x3f000000));
3307 udelay(2);
3308 }
3309
3310 write32(DEFAULT_MCHBAR + 0x4ea8, 0);
3311
3312 printram("CPB\n");
3313
3314 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
3315 printram("timC [%d, %d, %d] = 0x%x\n", channel,
3316 slotrank, lane,
3317 (lower[channel][slotrank][lane] +
3318 upper[channel][slotrank][lane]) / 2);
3319 ctrl->timings[channel][slotrank].lanes[lane].timC =
3320 (lower[channel][slotrank][lane] +
3321 upper[channel][slotrank][lane]) / 2;
3322 }
3323 FOR_ALL_POPULATED_CHANNELS {
3324 program_timings(ctrl, channel);
3325 }
3326}
3327
3328static void normalize_training(ramctr_timing * ctrl)
3329{
3330 int channel, slotrank, lane;
3331 int mat = 0;
3332
3333 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
3334 int delta;
3335 FOR_ALL_LANES mat =
3336 max(ctrl->timings[channel][slotrank].lanes[lane].timA, mat);
3337 delta = (mat >> 6) - ctrl->timings[channel][slotrank].val_4028;
3338 ctrl->timings[channel][slotrank].val_4024 += delta;
3339 ctrl->timings[channel][slotrank].val_4028 += delta;
3340 }
3341
3342 FOR_ALL_POPULATED_CHANNELS {
3343 program_timings(ctrl, channel);
3344 }
3345}
3346
3347static void write_controller_mr(ramctr_timing * ctrl)
3348{
3349 int channel, slotrank;
3350
3351 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
3352 write32(DEFAULT_MCHBAR + 0x0004 + (channel << 8) +
3353 lane_registers[slotrank], make_mr0(ctrl, slotrank));
3354 write32(DEFAULT_MCHBAR + 0x0008 + (channel << 8) +
3355 lane_registers[slotrank], make_mr1(ctrl, slotrank));
3356 }
3357}
3358
3359static void channel_test(ramctr_timing * ctrl)
3360{
3361 int channel, slotrank, lane;
3362
3363 FOR_ALL_POPULATED_CHANNELS
3364 if (read32(DEFAULT_MCHBAR + 0x42a0 + (channel << 10)) & 0xa000)
3365 die("Mini channel test failed (1)\n");
3366 FOR_ALL_POPULATED_CHANNELS {
3367 fill_pattern0(ctrl, channel, 0x12345678, 0x98765432);
3368
3369 write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 0);
3370 }
3371
3372 for (slotrank = 0; slotrank < 4; slotrank++)
3373 FOR_ALL_CHANNELS
3374 if (ctrl->rankmap[channel] & (1 << slotrank)) {
3375 FOR_ALL_LANES {
3376 write32(DEFAULT_MCHBAR + (0x4f40 + 4 * lane), 0);
3377 write32(DEFAULT_MCHBAR + (0x4d40 + 4 * lane), 0);
3378 }
3379 wait_428c(channel);
3380 write32(DEFAULT_MCHBAR + 0x4220 + (channel << 10), 0x0001f006);
3381 write32(DEFAULT_MCHBAR + 0x4230 + (channel << 10), 0x0028a004);
3382 write32(DEFAULT_MCHBAR + 0x4200 + (channel << 10),
3383 0x00060000 | (slotrank << 24));
3384 write32(DEFAULT_MCHBAR + 0x4210 + (channel << 10), 0x00000244);
3385 write32(DEFAULT_MCHBAR + 0x4224 + (channel << 10), 0x0001f201);
3386 write32(DEFAULT_MCHBAR + 0x4234 + (channel << 10), 0x08281064);
3387 write32(DEFAULT_MCHBAR + 0x4204 + (channel << 10),
3388 0x00000000 | (slotrank << 24));
3389 write32(DEFAULT_MCHBAR + 0x4214 + (channel << 10), 0x00000242);
3390 write32(DEFAULT_MCHBAR + 0x4228 + (channel << 10), 0x0001f105);
3391 write32(DEFAULT_MCHBAR + 0x4238 + (channel << 10), 0x04281064);
3392 write32(DEFAULT_MCHBAR + 0x4208 + (channel << 10),
3393 0x00000000 | (slotrank << 24));
3394 write32(DEFAULT_MCHBAR + 0x4218 + (channel << 10), 0x00000242);
3395 write32(DEFAULT_MCHBAR + 0x422c + (channel << 10), 0x0001f002);
3396 write32(DEFAULT_MCHBAR + 0x423c + (channel << 10), 0x00280c01);
3397 write32(DEFAULT_MCHBAR + 0x420c + (channel << 10),
3398 0x00060400 | (slotrank << 24));
3399 write32(DEFAULT_MCHBAR + 0x421c + (channel << 10), 0x00000240);
3400 write32(DEFAULT_MCHBAR + 0x4284 + (channel << 10), 0x000c0001);
3401 wait_428c(channel);
3402 FOR_ALL_LANES
3403 if (read32(DEFAULT_MCHBAR + 0x4340 + (channel << 10) + 4 * lane))
3404 die("Mini channel test failed (2)\n");
3405 }
3406}
3407
3408static void set_scrambling_seed(ramctr_timing * ctrl)
3409{
3410 int channel;
3411
3412 /* FIXME: we hardcode seeds. Do we need to use some PRNG for them?
3413 I don't think so. */
3414 static u32 seeds[NUM_CHANNELS][3] = {
3415 {0x00009a36, 0xbafcfdcf, 0x46d1ab68},
3416 {0x00028bfa, 0x53fe4b49, 0x19ed5483}
3417 };
3418 FOR_ALL_POPULATED_CHANNELS {
3419 MCHBAR32(0x4020 + 0x400 * channel) &= ~0x10000000;
3420 write32(DEFAULT_MCHBAR + 0x4034, seeds[channel][0]);
3421 write32(DEFAULT_MCHBAR + 0x403c, seeds[channel][1]);
3422 write32(DEFAULT_MCHBAR + 0x4038, seeds[channel][2]);
3423 }
3424}
3425
3426static void set_4f8c(void)
3427{
3428 struct cpuid_result cpures;
3429 u32 cpu;
3430
3431 cpures = cpuid(0);
3432 cpu = (cpures.eax);
3433 if (IS_SANDY_CPU(cpu) && (IS_SANDY_CPU_D0(cpu) || IS_SANDY_CPU_D1(cpu))) {
3434 MCHBAR32(0x4f8c) = 0x141D1519;
3435 } else {
3436 MCHBAR32(0x4f8c) = 0x551D1519;
3437 }
3438}
3439
3440static void prepare_training(ramctr_timing * ctrl)
3441{
3442 int channel;
3443
3444 FOR_ALL_POPULATED_CHANNELS {
3445 // Always drive command bus
3446 MCHBAR32(0x4004 + 0x400 * channel) |= 0x20000000;
3447 }
3448
3449 udelay(1);
3450
3451 FOR_ALL_POPULATED_CHANNELS {
3452 wait_428c(channel);
3453 }
3454}
3455
3456static void set_4008c(ramctr_timing * ctrl)
3457{
3458 int channel, slotrank;
3459 u32 reg;
3460 FOR_ALL_POPULATED_CHANNELS {
3461 u32 b20, b4_8_12;
3462 int min_320c = 10000;
3463 int max_320c = -10000;
3464
3465 FOR_ALL_POPULATED_RANKS {
3466 max_320c = max(ctrl->timings[channel][slotrank].val_320c, max_320c);
3467 min_320c = min(ctrl->timings[channel][slotrank].val_320c, min_320c);
3468 }
3469
3470 if (max_320c - min_320c > 51)
3471 b20 = 0;
3472 else
3473 b20 = ctrl->ref_card_offset[channel];
3474
3475 if (ctrl->reg_320c_range_threshold < max_320c - min_320c)
3476 b4_8_12 = 0x3330;
3477 else
3478 b4_8_12 = 0x2220;
3479
3480 reg = read32(DEFAULT_MCHBAR + 0x400c + (channel << 10));
3481 write32(DEFAULT_MCHBAR + 0x400c + (channel << 10),
3482 (reg & 0xFFF0FFFF)
3483 | (ctrl->ref_card_offset[channel] << 16)
3484 | (ctrl->ref_card_offset[channel] << 18));
3485 write32(DEFAULT_MCHBAR + 0x4008 + (channel << 10),
3486 0x0a000000
3487 | (b20 << 20)
3488 | ((ctrl->ref_card_offset[channel] + 2) << 16)
3489 | b4_8_12);
3490 }
3491}
3492
3493static void set_42a0(ramctr_timing * ctrl)
3494{
3495 int channel;
3496 FOR_ALL_POPULATED_CHANNELS {
3497 write32(DEFAULT_MCHBAR + (0x42a0 + 0x400 * channel),
3498 0x00001000 | ctrl->rankmap[channel]);
3499 MCHBAR32(0x4004 + 0x400 * channel) &= ~0x20000000; // OK
3500 }
3501}
3502
3503static int encode_5d10(int ns)
3504{
3505 return (ns + 499) / 500;
3506}
3507
3508/* FIXME: values in this function should be hardware revision-dependent. */
3509static void final_registers(ramctr_timing * ctrl)
3510{
3511 int channel;
3512 int t1_cycles = 0, t1_ns = 0, t2_ns;
3513 int t3_ns;
3514 u32 r32;
3515
3516 write32(DEFAULT_MCHBAR + 0x4cd4, 0x00000046);
3517
3518 write32(DEFAULT_MCHBAR + 0x400c, (read32(DEFAULT_MCHBAR + 0x400c) & 0xFFFFCFFF) | 0x1000); // OK
3519 write32(DEFAULT_MCHBAR + 0x440c, (read32(DEFAULT_MCHBAR + 0x440c) & 0xFFFFCFFF) | 0x1000); // OK
3520 write32(DEFAULT_MCHBAR + 0x4cb0, 0x00000740);
3521 write32(DEFAULT_MCHBAR + 0x4380, 0x00000aaa); // OK
3522 write32(DEFAULT_MCHBAR + 0x4780, 0x00000aaa); // OK
3523 write32(DEFAULT_MCHBAR + 0x4f88, 0x5f7003ff); // OK
3524 write32(DEFAULT_MCHBAR + 0x5064, 0x00073000 | ctrl->reg_5064b0); // OK
3525
3526 FOR_ALL_CHANNELS {
3527 switch (ctrl->rankmap[channel]) {
3528 /* Unpopulated channel. */
3529 case 0:
3530 write32(DEFAULT_MCHBAR + 0x4384 + channel * 0x400, 0);
3531 break;
3532 /* Only single-ranked dimms. */
3533 case 1:
3534 case 4:
3535 case 5:
3536 write32(DEFAULT_MCHBAR + 0x4384 + channel * 0x400, 0x373131);
3537 break;
3538 /* Dual-ranked dimms present. */
3539 default:
3540 write32(DEFAULT_MCHBAR + 0x4384 + channel * 0x400, 0x9b6ea1);
3541 break;
3542 }
3543 }
3544
3545 write32 (DEFAULT_MCHBAR + 0x5880, 0xca9171e5);
3546 write32 (DEFAULT_MCHBAR + 0x5888,
3547 (read32 (DEFAULT_MCHBAR + 0x5888) & ~0xffffff) | 0xe4d5d0);
3548 write32 (DEFAULT_MCHBAR + 0x58a8, read32 (DEFAULT_MCHBAR + 0x58a8) & ~0x1f);
3549 write32 (DEFAULT_MCHBAR + 0x4294,
3550 (read32 (DEFAULT_MCHBAR + 0x4294) & ~0x30000)
3551 | (1 << 16));
3552 write32 (DEFAULT_MCHBAR + 0x4694,
3553 (read32 (DEFAULT_MCHBAR + 0x4694) & ~0x30000)
3554 | (1 << 16));
3555
3556 MCHBAR32(0x5030) |= 1; // OK
3557 MCHBAR32(0x5030) |= 0x80; // OK
3558 MCHBAR32(0x5f18) = 0xfa; // OK
3559
3560 /* Find a populated channel. */
3561 FOR_ALL_POPULATED_CHANNELS
3562 break;
3563
3564 t1_cycles = ((read32(DEFAULT_MCHBAR + 0x4290 + channel * 0x400) >> 8) & 0xff);
3565 r32 = read32(DEFAULT_MCHBAR + 0x5064);
3566 if (r32 & 0x20000)
3567 t1_cycles += (r32 & 0xfff);
3568 t1_cycles += (read32(DEFAULT_MCHBAR + channel * 0x400 + 0x42a4) & 0xfff);
3569 t1_ns = t1_cycles * ctrl->tCK / 256 + 544;
3570 if (!(r32 & 0x20000))
3571 t1_ns += 500;
3572
3573 t2_ns = 10 * ((read32(DEFAULT_MCHBAR + 0x5f10) >> 8) & 0xfff);
3574 if ( read32(DEFAULT_MCHBAR + 0x5f00) & 8 )
3575 {
3576 t3_ns = 10 * ((read32(DEFAULT_MCHBAR + 0x5f20) >> 8) & 0xfff);
3577 t3_ns += 10 * (read32(DEFAULT_MCHBAR + 0x5f18) & 0xff);
3578 }
3579 else
3580 {
3581 t3_ns = 500;
3582 }
3583 printk(BIOS_DEBUG, "t123: %d, %d, %d\n",
3584 t1_ns, t2_ns, t3_ns);
3585 write32 (DEFAULT_MCHBAR + 0x5d10,
3586 ((encode_5d10(t1_ns) + encode_5d10(t2_ns)) << 16)
3587 | (encode_5d10(t1_ns) << 8)
3588 | ((encode_5d10(t3_ns) + encode_5d10(t2_ns) + encode_5d10(t1_ns)) << 24)
3589 | (read32(DEFAULT_MCHBAR + 0x5d10) & 0xC0C0C0C0)
3590 | 0xc);
3591}
3592
3593static void save_timings(ramctr_timing * ctrl)
3594{
3595 struct mrc_data_container *mrcdata;
3596 int output_len = ALIGN(sizeof (*ctrl), 16);
3597
3598 /* Save the MRC S3 restore data to cbmem */
3599 mrcdata = cbmem_add
3600 (CBMEM_ID_MRCDATA,
3601 output_len + sizeof(struct mrc_data_container));
3602
3603 printk(BIOS_DEBUG, "Relocate MRC DATA from %p to %p (%u bytes)\n",
3604 ctrl, mrcdata, output_len);
3605
3606 mrcdata->mrc_signature = MRC_DATA_SIGNATURE;
3607 mrcdata->mrc_data_size = output_len;
3608 mrcdata->reserved = 0;
3609 memcpy(mrcdata->mrc_data, ctrl, sizeof (*ctrl));
3610
3611 /* Zero the unused space in aligned buffer. */
3612 if (output_len > sizeof (*ctrl))
3613 memset(mrcdata->mrc_data+sizeof (*ctrl), 0,
3614 output_len - sizeof (*ctrl));
3615
3616 mrcdata->mrc_checksum = compute_ip_checksum(mrcdata->mrc_data,
3617 mrcdata->mrc_data_size);
3618}
3619
3620static void restore_timings(ramctr_timing * ctrl)
3621{
3622 int channel, slotrank, lane;
3623
3624 FOR_ALL_POPULATED_CHANNELS
3625 MCHBAR32(0x4004 + 0x400 * channel) =
3626 ctrl->tRRD
3627 | (ctrl->tRTP << 4)
3628 | (ctrl->tCKE << 8)
3629 | (ctrl->tWTR << 12)
3630 | (ctrl->tFAW << 16)
3631 | (ctrl->tWR << 24)
3632 | (ctrl->cmd_stretch[channel] << 30);
3633
3634 udelay(1);
3635
3636 FOR_ALL_POPULATED_CHANNELS {
3637 wait_428c(channel);
3638 }
3639
3640 FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
3641 write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel
3642 + 4 * lane, 0);
3643 }
3644
3645 FOR_ALL_POPULATED_CHANNELS
3646 write32(DEFAULT_MCHBAR + 0x4008 + 0x400 * channel,
3647 read32(DEFAULT_MCHBAR + 0x4008 +
3648 0x400 * channel) | 0x8000000);
3649
3650 FOR_ALL_POPULATED_CHANNELS {
3651 udelay (1);
3652 write32(DEFAULT_MCHBAR + 0x4020 + 0x400 * channel,
3653 read32(DEFAULT_MCHBAR + 0x4020 +
3654 0x400 * channel) | 0x200000);
3655 }
3656
3657 printram("CPE\n");
3658
3659 write32(DEFAULT_MCHBAR + 0x3400, 0);
3660 write32(DEFAULT_MCHBAR + 0x4eb0, 0);
3661
3662 printram("CP5b\n");
3663
3664 FOR_ALL_POPULATED_CHANNELS {
3665 program_timings(ctrl, channel);
3666 }
3667
3668 u32 reg, addr;
3669
3670 while (!(MCHBAR32(0x5084) & 0x10000)) ;
3671 do {
3672 reg = MCHBAR32(0x428c);
3673 } while ((reg & 0x14) == 0);
3674
3675 // Set state of memory controller
3676 MCHBAR32(0x5030) = 0x116;
3677 MCHBAR32(0x4ea0) = 0;
3678
3679 // Wait 500us
3680 udelay(500);
3681
3682 FOR_ALL_CHANNELS {
3683 // Set valid rank CKE
3684 reg = 0;
3685 reg = (reg & ~0xf) | ctrl->rankmap[channel];
3686 addr = 0x400 * channel + 0x42a0;
3687 MCHBAR32(addr) = reg;
3688
3689 // Wait 10ns for ranks to settle
3690 //udelay(0.01);
3691
3692 reg = (reg & ~0xf0) | (ctrl->rankmap[channel] << 4);
3693 MCHBAR32(addr) = reg;
3694
3695 // Write reset using a NOP
3696 write_reset(ctrl);
3697 }
3698
3699 /* mrs commands. */
3700 dram_mrscommands(ctrl);
3701
3702 printram("CP5c\n");
3703
3704 write32(DEFAULT_MCHBAR + 0x3000, 0);
3705
3706 FOR_ALL_CHANNELS {
3707 write32(DEFAULT_MCHBAR + (channel * 0x100) + 0xe3c,
3708 0 | (read32(DEFAULT_MCHBAR + (channel * 0x100) + 0xe3c) &
3709 ~0x3f000000));
3710 udelay(2);
3711 }
3712
3713 write32(DEFAULT_MCHBAR + 0x4ea8, 0);
3714}
3715
3716void init_dram_ddr3(spd_raw_data * spds, int mobile, int min_tck,
3717 int s3resume)
3718{
3719 int me_uma_size;
3720 int cbmem_was_inited;
3721
3722 MCHBAR32(0x5f00) |= 1;
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]3723
Vadim Bendebury7a3f36a2012-04-18 15:47:32 -0700[diff] [blame]3724 report_platform_info();
3725
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]3726 /* Wait for ME to be ready */
3727 intel_early_me_init();
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]3728 me_uma_size = intel_early_me_uma_size();
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]3729
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]3730 printk(BIOS_DEBUG, "Starting native Platform init\n");
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]3731
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]3732 u32 reg_5d10;
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]3733
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]3734 wait_txt_clear();
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]3735
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]3736 wrmsr(0x000002e6, (msr_t) { .lo = 0, .hi = 0 });
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]3737
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]3738 reg_5d10 = read32(DEFAULT_MCHBAR + 0x5d10); // !!! = 0x00000000
3739 if ((pcie_read_config16(SOUTHBRIDGE, 0xa2) & 0xa0) == 0x20 /* 0x0004 */
3740 && reg_5d10 && !s3resume) {
3741 write32(DEFAULT_MCHBAR + 0x5d10, 0);
3742 /* Need reset. */
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]3743 outb(0x6, 0xcf9);
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]3744
Patrick Georgi546953c2014-11-29 10:38:17 +0100[diff] [blame]3745 halt();
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]3746 }
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]3747
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]3748 ramctr_timing ctrl;
Vadim Bendebury48a4a7f2012-06-07 18:47:13 -0700[diff] [blame]3749
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]3750 memset(&ctrl, 0, sizeof (ctrl));
3751
3752 early_pch_init_native();
3753 early_thermal_init();
3754
3755 ctrl.mobile = mobile;
3756 ctrl.tCK = min_tck;
3757
3758 /* FIXME: for non-S3 we should be able to use timing caching with
3759 proper verification. Right now we use timings only for S3 case.
3760 */
3761 if (s3resume) {
3762 struct mrc_data_container *mrc_cache;
3763
3764 mrc_cache = find_current_mrc_cache();
3765 if (!mrc_cache || mrc_cache->mrc_data_size < sizeof (ctrl)) {
3766 /* Failed S3 resume, reset to come up cleanly */
3767 outb(0x6, 0xcf9);
3768 halt();
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]3769 }
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]3770 memcpy(&ctrl, mrc_cache->mrc_data, sizeof (ctrl));
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]3771 }
3772
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]3773 if (!s3resume) {
3774 dimm_info info;
Sven Schnelled4ee8082012-07-28 09:28:56 +0200[diff] [blame]3775
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]3776 /* Get DDR3 SPD data */
3777 dram_find_spds_ddr3(spds, &info, &ctrl);
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]3778
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]3779 /* Find fastest common supported parameters */
3780 dram_find_common_params(&info, &ctrl);
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]3781
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]3782 dram_dimm_mapping(&info, &ctrl);
3783 }
3784
3785 /* Set MCU frequency */
3786 dram_freq(&ctrl);
3787
3788 if (!s3resume) {
3789 /* Calculate timings */
3790 dram_timing(&ctrl);
3791 }
3792
3793 /* Set version register */
3794 MCHBAR32(0x5034) = 0xC04EB002;
3795
3796 /* Enable crossover */
3797 dram_xover(&ctrl);
3798
3799 /* Set timing and refresh registers */
3800 dram_timing_regs(&ctrl);
3801
3802 /* Power mode preset */
3803 MCHBAR32(0x4e80) = 0x5500;
3804
3805 /* Set scheduler parameters */
3806 MCHBAR32(0x4c20) = 0x10100005;
3807
3808 /* Set cpu specific register */
3809 set_4f8c();
3810
3811 /* Clear IO reset bit */
3812 MCHBAR32(0x5030) &= ~0x20;
3813
3814 /* Set MAD-DIMM registers */
3815 dram_dimm_set_mapping(&ctrl);
3816 printk(BIOS_DEBUG, "Done dimm mapping\n");
3817
3818 /* Zone config */
3819 dram_zones(&ctrl, 1);
3820
3821 /* Set memory map */
3822 dram_memorymap(&ctrl, me_uma_size);
3823 printk(BIOS_DEBUG, "Done memory map\n");
3824
3825 /* Set IO registers */
3826 dram_ioregs(&ctrl);
3827 printk(BIOS_DEBUG, "Done io registers\n");
3828
3829 udelay(1);
3830
3831 if (s3resume) {
3832 restore_timings(&ctrl);
3833 } else {
3834 /* Do jedec ddr3 reset sequence */
3835 dram_jedecreset(&ctrl);
3836 printk(BIOS_DEBUG, "Done jedec reset\n");
3837
3838 /* MRS commands */
3839 dram_mrscommands(&ctrl);
3840 printk(BIOS_DEBUG, "Done MRS commands\n");
3841 dram_mrscommands(&ctrl);
3842
3843 /* Prepare for memory training */
3844 prepare_training(&ctrl);
3845
3846 read_training(&ctrl);
3847 write_training(&ctrl);
3848
3849 printram("CP5a\n");
3850
3851 discover_edges(&ctrl);
3852
3853 printram("CP5b\n");
3854
3855 command_training(&ctrl);
3856
3857 printram("CP5c\n");
3858
3859 discover_edges_write(&ctrl);
3860
3861 discover_timC_write(&ctrl);
3862
3863 normalize_training(&ctrl);
3864 }
3865
3866 set_4008c(&ctrl);
3867
3868 write_controller_mr(&ctrl);
3869
3870 if (!s3resume) {
3871 channel_test(&ctrl);
3872 }
3873
3874 /* FIXME: should be hardware revision-dependent. */
3875 write32(DEFAULT_MCHBAR + 0x5024, 0x00a030ce);
3876
3877 set_scrambling_seed(&ctrl);
3878
3879 set_42a0(&ctrl);
3880
3881 final_registers(&ctrl);
3882
3883 /* Zone config */
3884 dram_zones(&ctrl, 0);
3885
3886 if (!s3resume)
3887 quick_ram_check();
3888
3889 intel_early_me_status();
3890 intel_early_me_init_done(ME_INIT_STATUS_SUCCESS);
3891 intel_early_me_status();
3892
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]3893 report_memory_config();
Alexandru Gagniucecf2eb42015-09-28 21:39:12 -0700[diff] [blame]3894
3895 cbmem_was_inited = !cbmem_recovery(s3resume);
3896 if (!s3resume)
3897 save_timings(&ctrl);
3898 if (s3resume && !cbmem_was_inited) {
3899 /* Failed S3 resume, reset to come up cleanly */
3900 outb(0x6, 0xcf9);
3901 halt();
3902 }
Stefan Reinauer00636b02012-04-04 00:08:51 +0200[diff] [blame]3903}