Gitiles
Code Review Sign In
review.coreboot.org / coreboot / 7116ac803736345cc7c7b73ac435efa50c4cd2b0 / . / src / northbridge / intel / nehalem / raminit.c
blob: 6673b5a846765019c47da7c40d4397ae9f689aa4 [file] [log] [blame]
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]1/*
2 * This file is part of the coreboot project.
3 *
4 * Copyright (C) 2013 Vladimir Serbinenko.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21/* Please don't remove this. It's needed it to do debugging
22 and reverse engineering to support in futur more nehalem variants. */
23#ifndef REAL
24#define REAL 1
25#endif
26
27#if REAL
Kyösti Mälkki931c1dc2014-06-30 09:40:19 +0300[diff] [blame]28#include <stdlib.h>
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]29#include <console/console.h>
30#include <string.h>
31#include <arch/hlt.h>
32#include <arch/io.h>
33#include <cpu/x86/msr.h>
34#include <cbmem.h>
35#include <arch/cbfs.h>
36#include <cbfs.h>
37#include <ip_checksum.h>
38#include <pc80/mc146818rtc.h>
39#include <device/pci_def.h>
40#include <arch/cpu.h>
41#include <spd.h>
42#include "raminit.h"
43#include <timestamp.h>
44#include <cpu/x86/mtrr.h>
45#include <cpu/intel/speedstep.h>
46#include <cpu/intel/turbo.h>
47#endif
48
49#if !REAL
50typedef unsigned char u8;
51typedef unsigned short u16;
52typedef unsigned int u32;
53typedef u32 device_t;
54#endif
55
56#include "nehalem.h"
57
58#include "southbridge/intel/ibexpeak/me.h"
59
60#if REAL
61#include <delay.h>
62#endif
63
64#define NORTHBRIDGE PCI_DEV(0, 0, 0)
65#define SOUTHBRIDGE PCI_DEV(0, 0x1f, 0)
66#define GMA PCI_DEV (0, 0x2, 0x0)
67#define HECIDEV PCI_DEV(0, 0x16, 0)
68#define HECIBAR 0x10
69
70#define FOR_ALL_RANKS \
71 for (channel = 0; channel < NUM_CHANNELS; channel++) \
72 for (slot = 0; slot < NUM_SLOTS; slot++) \
73 for (rank = 0; rank < NUM_RANKS; rank++)
74
75#define FOR_POPULATED_RANKS \
76 for (channel = 0; channel < NUM_CHANNELS; channel++) \
77 for (slot = 0; slot < NUM_SLOTS; slot++) \
78 for (rank = 0; rank < NUM_RANKS; rank++) \
79 if (info->populated_ranks[channel][slot][rank])
80
81#define FOR_POPULATED_RANKS_BACKWARDS \
82 for (channel = NUM_CHANNELS - 1; channel >= 0; channel--) \
83 for (slot = 0; slot < NUM_SLOTS; slot++) \
84 for (rank = 0; rank < NUM_RANKS; rank++) \
85 if (info->populated_ranks[channel][slot][rank])
86
87/* [REG_178][CHANNEL][2 * SLOT + RANK][LANE] */
88typedef struct {
89 u8 smallest;
90 u8 largest;
91} timing_bounds_t[2][2][2][9];
92
93struct ram_training {
94 /* [TM][CHANNEL][SLOT][RANK][LANE] */
95 u16 lane_timings[4][2][2][2][9];
96 u16 reg_178;
97 u16 reg_10b;
98
99 u8 reg178_center;
100 u8 reg178_smallest;
101 u8 reg178_largest;
102 timing_bounds_t timing_bounds[2];
103 u16 timing_offset[2][2][2][9];
104 u16 timing2_offset[2][2][2][9];
105 u16 timing2_bounds[2][2][2][9][2];
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]106 u8 reg274265[2][3]; /* [CHANNEL][REGISTER] */
107 u8 reg2ca9_bit0;
108 u32 reg_6dc;
109 u32 reg_6e8;
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]110};
111
112#if !REAL
113#include "raminit_fake.c"
114#else
115
116#include <lib.h> /* Prototypes */
117
118static inline void write_mchbar32(u32 addr, u32 val)
119{
120 MCHBAR32(addr) = val;
121}
122
123static inline void write_mchbar16(u32 addr, u16 val)
124{
125 MCHBAR16(addr) = val;
126}
127
128static inline void write_mchbar8(u32 addr, u8 val)
129{
130 MCHBAR8(addr) = val;
131}
132
133
134static inline u32 read_mchbar32(u32 addr)
135{
136 return MCHBAR32(addr);
137}
138
139static inline u16 read_mchbar16(u32 addr)
140{
141 return MCHBAR16(addr);
142}
143
144static inline u8 read_mchbar8(u32 addr)
145{
146 return MCHBAR8(addr);
147}
148
149static inline u8 read_mchbar8_bypass(u32 addr)
150{
151 return MCHBAR8(addr);
152}
153
154static void clflush(u32 addr)
155{
156 asm volatile ("clflush (%0)"::"r" (addr));
157}
158
159typedef struct _u128 {
160 u64 lo;
161 u64 hi;
162} u128;
163
164static void read128(u32 addr, u64 * out)
165{
166 u128 ret;
167 u128 stor;
168 asm volatile ("movdqu %%xmm0, %0\n"
169 "movdqa (%2), %%xmm0\n"
170 "movdqu %%xmm0, %1\n"
171 "movdqu %0, %%xmm0":"+m" (stor), "=m"(ret):"r"(addr));
172 out[0] = ret.lo;
173 out[1] = ret.hi;
174}
175
176#endif
177
178/* OK */
179static void write_1d0(u32 val, u16 addr, int bits, int flag)
180{
181 write_mchbar32(0x1d0, 0);
182 while (read_mchbar32(0x1d0) & 0x800000) ;
183 write_mchbar32(0x1d4,
184 (val & ((1 << bits) - 1)) | (2 << bits) | (flag <<
185 bits));
186 write_mchbar32(0x1d0, 0x40000000 | addr);
187 while (read_mchbar32(0x1d0) & 0x800000) ;
188}
189
190/* OK */
191static u16 read_1d0(u16 addr, int split)
192{
193 u32 val;
194 write_mchbar32(0x1d0, 0);
195 while (read_mchbar32(0x1d0) & 0x800000) ;
196 write_mchbar32(0x1d0,
197 0x80000000 | (((read_mchbar8(0x246) >> 2) & 3) +
198 0x361 - addr));
199 while (read_mchbar32(0x1d0) & 0x800000) ;
200 val = read_mchbar32(0x1d8);
201 write_1d0(0, 0x33d, 0, 0);
202 write_1d0(0, 0x33d, 0, 0);
203 val &= ((1 << split) - 1);
204 // printk (BIOS_ERR, "R1D0C [%x] => %x\n", addr, val);
205 return val;
206}
207
208static void sfence(void)
209{
210#if REAL
211 asm volatile ("sfence");
212#endif
213}
214
215static inline u16 get_lane_offset(int slot, int rank, int lane)
216{
217 return 0x124 * lane + ((lane & 4) ? 0x23e : 0) + 11 * rank + 22 * slot -
218 0x452 * (lane == 8);
219}
220
221static inline u16 get_timing_register_addr(int lane, int tm, int slot, int rank)
222{
223 const u16 offs[] = { 0x1d, 0xa8, 0xe6, 0x5c };
224 return get_lane_offset(slot, rank, lane) + offs[(tm + 3) % 4];
225}
226
227#if REAL
228static u32 gav_real(int line, u32 in)
229{
230 // printk (BIOS_DEBUG, "%d: GAV: %x\n", line, in);
231 return in;
232}
233
234#define gav(x) gav_real (__LINE__, (x))
235#endif
236struct raminfo {
237 u16 clock_speed_index; /* clock_speed (REAL, not DDR) / 133.(3) - 3 */
238 u16 fsb_frequency; /* in 1.(1)/2 MHz. */
239 u8 is_x16_module[2][2]; /* [CHANNEL][SLOT] */
240 u8 density[2][2]; /* [CHANNEL][SLOT] */
241 u8 populated_ranks[2][2][2]; /* [CHANNEL][SLOT][RANK] */
242 int rank_start[2][2][2];
243 u8 cas_latency;
244 u8 board_lane_delay[9];
245 u8 use_ecc;
246 u8 revision;
247 u8 max_supported_clock_speed_index;
248 u8 uma_enabled;
249 u8 spd[2][2][151]; /* [CHANNEL][SLOT][BYTE] */
250 u8 silicon_revision;
251 u8 populated_ranks_mask[2];
252 u8 max_slots_used_in_channel;
253 u8 mode4030[2];
254 u16 avg4044[2];
255 u16 max4048[2];
256 unsigned total_memory_mb;
257 unsigned interleaved_part_mb;
258 unsigned non_interleaved_part_mb;
259
260 u32 heci_bar;
261 u64 heci_uma_addr;
262 unsigned memory_reserved_for_heci_mb;
263
264 struct ram_training training;
265 u32 last_500_command[2];
266
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]267 u32 delay46_ps[2];
268 u32 delay54_ps[2];
269 u8 revision_flag_1;
270 u8 some_delay_1_cycle_floor;
271 u8 some_delay_2_halfcycles_ceil;
272 u8 some_delay_3_ps_rounded;
273
274 const struct ram_training *cached_training;
275};
276
277static void
278write_500(struct raminfo *info, int channel, u32 val, u16 addr, int bits,
279 int flag);
280
281/* OK */
282static u16
283read_500(struct raminfo *info, int channel, u16 addr, int split)
284{
285 u32 val;
286 info->last_500_command[channel] = 0x80000000;
287 write_mchbar32(0x500 + (channel << 10), 0);
288 while (read_mchbar32(0x500 + (channel << 10)) & 0x800000) ;
289 write_mchbar32(0x500 + (channel << 10),
290 0x80000000 |
291 (((read_mchbar8(0x246 + (channel << 10)) >> 2) &
292 3) + 0xb88 - addr));
293 while (read_mchbar32(0x500 + (channel << 10)) & 0x800000) ;
294 val = read_mchbar32(0x508 + (channel << 10));
295 return val & ((1 << split) - 1);
296}
297
298/* OK */
299static void
300write_500(struct raminfo *info, int channel, u32 val, u16 addr, int bits,
301 int flag)
302{
303 if (info->last_500_command[channel] == 0x80000000) {
304 info->last_500_command[channel] = 0x40000000;
305 write_500(info, channel, 0, 0xb61, 0, 0);
306 }
307 write_mchbar32(0x500 + (channel << 10), 0);
308 while (read_mchbar32(0x500 + (channel << 10)) & 0x800000) ;
309 write_mchbar32(0x504 + (channel << 10),
310 (val & ((1 << bits) - 1)) | (2 << bits) | (flag <<
311 bits));
312 write_mchbar32(0x500 + (channel << 10), 0x40000000 | addr);
313 while (read_mchbar32(0x500 + (channel << 10)) & 0x800000) ;
314}
315
316static int rw_test(int rank)
317{
318 const u32 mask = 0xf00fc33c;
319 int ok = 0xff;
320 int i;
321 for (i = 0; i < 64; i++)
322 write32((rank << 28) | (i << 2), 0);
323 sfence();
324 for (i = 0; i < 64; i++)
325 gav(read32((rank << 28) | (i << 2)));
326 sfence();
327 for (i = 0; i < 32; i++) {
328 u32 pat = (((mask >> i) & 1) ? 0xffffffff : 0);
329 write32((rank << 28) | (i << 3), pat);
330 write32((rank << 28) | (i << 3) | 4, pat);
331 }
332 sfence();
333 for (i = 0; i < 32; i++) {
334 u8 pat = (((mask >> i) & 1) ? 0xff : 0);
335 int j;
336 u32 val;
337 gav(val = read32((rank << 28) | (i << 3)));
338 for (j = 0; j < 4; j++)
339 if (((val >> (j * 8)) & 0xff) != pat)
340 ok &= ~(1 << j);
341 gav(val = read32((rank << 28) | (i << 3) | 4));
342 for (j = 0; j < 4; j++)
343 if (((val >> (j * 8)) & 0xff) != pat)
344 ok &= ~(16 << j);
345 }
346 sfence();
347 for (i = 0; i < 64; i++)
348 write32((rank << 28) | (i << 2), 0);
349 sfence();
350 for (i = 0; i < 64; i++)
351 gav(read32((rank << 28) | (i << 2)));
352
353 return ok;
354}
355
356static void
357program_timings(struct raminfo *info, u16 base, int channel, int slot, int rank)
358{
359 int lane;
360 for (lane = 0; lane < 8; lane++) {
361 write_500(info, channel,
362 base +
363 info->training.
364 lane_timings[2][channel][slot][rank][lane],
365 get_timing_register_addr(lane, 2, slot, rank), 9, 0);
366 write_500(info, channel,
367 base +
368 info->training.
369 lane_timings[3][channel][slot][rank][lane],
370 get_timing_register_addr(lane, 3, slot, rank), 9, 0);
371 }
372}
373
374static void write_26c(int channel, u16 si)
375{
376 write_mchbar32(0x26c + (channel << 10), 0x03243f35);
377 write_mchbar32(0x268 + (channel << 10), 0xcfc00000 | (si << 9));
378 write_mchbar16(0x2b9 + (channel << 10), si);
379}
380
381static u32 get_580(int channel, u8 addr)
382{
383 u32 ret;
384 gav(read_1d0(0x142, 3));
385 write_mchbar8(0x5ff, 0x0); /* OK */
386 write_mchbar8(0x5ff, 0x80); /* OK */
387 write_mchbar32(0x580 + (channel << 10), 0x8493c012 | addr);
388 write_mchbar8(0x580 + (channel << 10),
389 read_mchbar8(0x580 + (channel << 10)) | 1);
390 while (!((ret = read_mchbar32(0x580 + (channel << 10))) & 0x10000)) ;
391 write_mchbar8(0x580 + (channel << 10),
392 read_mchbar8(0x580 + (channel << 10)) & ~1);
393 return ret;
394}
395
396const int cached_config = 0;
397
398#define NUM_CHANNELS 2
399#define NUM_SLOTS 2
400#define NUM_RANKS 2
401#define RANK_SHIFT 28
402#define CHANNEL_SHIFT 10
403
404#include "raminit_tables.c"
405
406static void seq9(struct raminfo *info, int channel, int slot, int rank)
407{
408 int i, lane;
409
410 for (i = 0; i < 2; i++)
411 for (lane = 0; lane < 8; lane++)
412 write_500(info, channel,
413 info->training.lane_timings[i +
414 1][channel][slot]
415 [rank][lane], get_timing_register_addr(lane,
416 i + 1,
417 slot,
418 rank),
419 9, 0);
420
421 write_1d0(1, 0x103, 6, 1);
422 for (lane = 0; lane < 8; lane++)
423 write_500(info, channel,
424 info->training.
425 lane_timings[0][channel][slot][rank][lane],
426 get_timing_register_addr(lane, 0, slot, rank), 9, 0);
427
428 for (i = 0; i < 2; i++) {
429 for (lane = 0; lane < 8; lane++)
430 write_500(info, channel,
431 info->training.lane_timings[i +
432 1][channel][slot]
433 [rank][lane], get_timing_register_addr(lane,
434 i + 1,
435 slot,
436 rank),
437 9, 0);
438 gav(get_580(channel, ((i + 1) << 2) | (rank << 5)));
439 }
440
441 gav(read_1d0(0x142, 3)); // = 0x10408118
442 write_mchbar8(0x5ff, 0x0); /* OK */
443 write_mchbar8(0x5ff, 0x80); /* OK */
444 write_1d0(0x2, 0x142, 3, 1);
445 for (lane = 0; lane < 8; lane++) {
446 // printk (BIOS_ERR, "before: %x\n", info->training.lane_timings[2][channel][slot][rank][lane]);
447 info->training.lane_timings[2][channel][slot][rank][lane] =
448 read_500(info, channel,
449 get_timing_register_addr(lane, 2, slot, rank), 9);
450 //printk (BIOS_ERR, "after: %x\n", info->training.lane_timings[2][channel][slot][rank][lane]);
451 info->training.lane_timings[3][channel][slot][rank][lane] =
452 info->training.lane_timings[2][channel][slot][rank][lane] +
453 0x20;
454 }
455}
456
457static int count_ranks_in_channel(struct raminfo *info, int channel)
458{
459 int slot, rank;
460 int res = 0;
461 for (slot = 0; slot < NUM_SLOTS; slot++)
462 for (rank = 0; rank < NUM_SLOTS; rank++)
463 res += info->populated_ranks[channel][slot][rank];
464 return res;
465}
466
467static void
468config_rank(struct raminfo *info, int s3resume, int channel, int slot, int rank)
469{
470 int add;
471
472 write_1d0(0, 0x178, 7, 1);
473 seq9(info, channel, slot, rank);
474 program_timings(info, 0x80, channel, slot, rank);
475
476 if (channel == 0)
477 add = count_ranks_in_channel(info, 1);
478 else
479 add = 0;
480 if (!s3resume)
481 gav(rw_test(rank + add));
482 program_timings(info, 0x00, channel, slot, rank);
483 if (!s3resume)
484 gav(rw_test(rank + add));
485 if (!s3resume)
486 gav(rw_test(rank + add));
487 write_1d0(0, 0x142, 3, 1);
488 write_1d0(0, 0x103, 6, 1);
489
490 gav(get_580(channel, 0xc | (rank << 5)));
491 gav(read_1d0(0x142, 3));
492
493 write_mchbar8(0x5ff, 0x0); /* OK */
494 write_mchbar8(0x5ff, 0x80); /* OK */
495}
496
497static void set_4cf(struct raminfo *info, int channel, u8 val)
498{
499 gav(read_500(info, channel, 0x4cf, 4)); // = 0xc2300cf9
500 write_500(info, channel, val, 0x4cf, 4, 1);
501 gav(read_500(info, channel, 0x659, 4)); // = 0x80300839
502 write_500(info, channel, val, 0x659, 4, 1);
503 gav(read_500(info, channel, 0x697, 4)); // = 0x80300839
504 write_500(info, channel, val, 0x697, 4, 1);
505}
506
507static void set_334(int zero)
508{
509 int j, k, channel;
510 const u32 val3[] = { 0x2a2b2a2b, 0x26272627, 0x2e2f2e2f, 0x2a2b };
511 u32 vd8[2][16];
512
513 for (channel = 0; channel < NUM_CHANNELS; channel++) {
514 for (j = 0; j < 4; j++) {
515 u32 a = (j == 1) ? 0x29292929 : 0x31313131;
516 u32 lmask = (j == 3) ? 0xffff : 0xffffffff;
517 u16 c;
518 if ((j == 0 || j == 3) && zero)
519 c = 0;
520 else if (j == 3)
521 c = 0x5f;
522 else
523 c = 0x5f5f;
524
525 for (k = 0; k < 2; k++) {
526 write_mchbar32(0x138 + 8 * k,
527 (channel << 26) | (j << 24));
528 gav(vd8[1][(channel << 3) | (j << 1) | k] =
529 read_mchbar32(0x138 + 8 * k));
530 gav(vd8[0][(channel << 3) | (j << 1) | k] =
531 read_mchbar32(0x13c + 8 * k));
532 }
533
534 write_mchbar32(0x334 + (channel << 10) + (j * 0x44),
535 zero ? 0 : val3[j]);
536 write_mchbar32(0x32c + (channel << 10) + (j * 0x44),
537 zero ? 0 : (0x18191819 & lmask));
538 write_mchbar16(0x34a + (channel << 10) + (j * 0x44), c);
539 write_mchbar32(0x33c + (channel << 10) + (j * 0x44),
540 zero ? 0 : (a & lmask));
541 write_mchbar32(0x344 + (channel << 10) + (j * 0x44),
542 zero ? 0 : (a & lmask));
543 }
544 }
545
546 write_mchbar32(0x130, read_mchbar32(0x130) | 1); /* OK */
547 while (read_mchbar8(0x130) & 1) ; /* OK */
548}
549
550static void rmw_1d0(u16 addr, u32 and, u32 or, int split, int flag)
551{
552 u32 v;
553 v = read_1d0(addr, split);
554 write_1d0((v & and) | or, addr, split, flag);
555}
556
557static int find_highest_bit_set(u16 val)
558{
559 int i;
560 for (i = 15; i >= 0; i--)
561 if (val & (1 << i))
562 return i;
563 return -1;
564}
565
566static int find_lowest_bit_set32(u32 val)
567{
568 int i;
569 for (i = 0; i < 32; i++)
570 if (val & (1 << i))
571 return i;
572 return -1;
573}
574
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]575enum {
576 DEVICE_TYPE = 2,
577 MODULE_TYPE = 3,
578 DENSITY = 4,
579 RANKS_AND_DQ = 7,
580 MEMORY_BUS_WIDTH = 8,
581 TIMEBASE_DIVIDEND = 10,
582 TIMEBASE_DIVISOR = 11,
583 CYCLETIME = 12,
584
585 CAS_LATENCIES_LSB = 14,
586 CAS_LATENCIES_MSB = 15,
587 CAS_LATENCY_TIME = 16,
588 THERMAL_AND_REFRESH = 31,
589 REFERENCE_RAW_CARD_USED = 62,
590 RANK1_ADDRESS_MAPPING = 63
591};
592
593static void calculate_timings(struct raminfo *info)
594{
595 unsigned cycletime;
596 unsigned cas_latency_time;
597 unsigned supported_cas_latencies;
598 unsigned channel, slot;
599 unsigned clock_speed_index;
600 unsigned min_cas_latency;
601 unsigned cas_latency;
602 unsigned max_clock_index;
603
604 /* Find common CAS latency */
605 supported_cas_latencies = 0x3fe;
606 for (channel = 0; channel < NUM_CHANNELS; channel++)
607 for (slot = 0; slot < NUM_SLOTS; slot++)
608 if (info->populated_ranks[channel][slot][0])
609 supported_cas_latencies &=
610 2 *
611 (info->
612 spd[channel][slot][CAS_LATENCIES_LSB] |
613 (info->
614 spd[channel][slot][CAS_LATENCIES_MSB] <<
615 8));
616
617 max_clock_index = min(3, info->max_supported_clock_speed_index);
618
619 cycletime = min_cycletime[max_clock_index];
620 cas_latency_time = min_cas_latency_time[max_clock_index];
621
622 for (channel = 0; channel < NUM_CHANNELS; channel++)
623 for (slot = 0; slot < NUM_SLOTS; slot++)
624 if (info->populated_ranks[channel][slot][0]) {
625 unsigned timebase;
626 timebase =
627 1000 *
628 info->
629 spd[channel][slot][TIMEBASE_DIVIDEND] /
630 info->spd[channel][slot][TIMEBASE_DIVISOR];
631 cycletime =
632 max(cycletime,
633 timebase *
634 info->spd[channel][slot][CYCLETIME]);
635 cas_latency_time =
636 max(cas_latency_time,
637 timebase *
638 info->
639 spd[channel][slot][CAS_LATENCY_TIME]);
640 }
641 for (clock_speed_index = 0; clock_speed_index < 3; clock_speed_index++) {
642 if (cycletime == min_cycletime[clock_speed_index])
643 break;
644 if (cycletime > min_cycletime[clock_speed_index]) {
645 clock_speed_index--;
646 cycletime = min_cycletime[clock_speed_index];
647 break;
648 }
649 }
Edward O'Callaghan7116ac82014-07-08 01:53:24 +1000[diff] [blame^]650 min_cas_latency = CEIL_DIV(cas_latency_time, cycletime);
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]651 cas_latency = 0;
652 while (supported_cas_latencies) {
653 cas_latency = find_highest_bit_set(supported_cas_latencies) + 3;
654 if (cas_latency <= min_cas_latency)
655 break;
656 supported_cas_latencies &=
657 ~(1 << find_highest_bit_set(supported_cas_latencies));
658 }
659
660 if (cas_latency != min_cas_latency && clock_speed_index)
661 clock_speed_index--;
662
663 if (cas_latency * min_cycletime[clock_speed_index] > 20000)
664 die("Couldn't configure DRAM");
665 info->clock_speed_index = clock_speed_index;
666 info->cas_latency = cas_latency;
667}
668
669static void program_base_timings(struct raminfo *info)
670{
671 unsigned channel;
672 unsigned slot, rank, lane;
673 unsigned extended_silicon_revision;
674 int i;
675
676 extended_silicon_revision = info->silicon_revision;
677 if (info->silicon_revision == 0)
678 for (channel = 0; channel < NUM_CHANNELS; channel++)
679 for (slot = 0; slot < NUM_SLOTS; slot++)
680 if ((info->
681 spd[channel][slot][MODULE_TYPE] & 0xF) ==
682 3)
683 extended_silicon_revision = 4;
684
685 for (channel = 0; channel < NUM_CHANNELS; channel++) {
686 for (slot = 0; slot < NUM_SLOTS; slot++)
687 for (rank = 0; rank < NUM_SLOTS; rank++) {
688 int card_timing_2;
689 if (!info->populated_ranks[channel][slot][rank])
690 continue;
691
692 for (lane = 0; lane < 9; lane++) {
693 int tm_reg;
694 int card_timing;
695
696 card_timing = 0;
697 if ((info->
698 spd[channel][slot][MODULE_TYPE] &
699 0xF) == 3) {
700 int reference_card;
701 reference_card =
702 info->
703 spd[channel][slot]
704 [REFERENCE_RAW_CARD_USED] &
705 0x1f;
706 if (reference_card == 3)
707 card_timing =
708 u16_ffd1188[0][lane]
709 [info->
710 clock_speed_index];
711 if (reference_card == 5)
712 card_timing =
713 u16_ffd1188[1][lane]
714 [info->
715 clock_speed_index];
716 }
717
718 info->training.
719 lane_timings[0][channel][slot][rank]
720 [lane] =
721 u8_FFFD1218[info->
722 clock_speed_index];
723 info->training.
724 lane_timings[1][channel][slot][rank]
725 [lane] = 256;
726
727 for (tm_reg = 2; tm_reg < 4; tm_reg++)
728 info->training.
729 lane_timings[tm_reg]
730 [channel][slot][rank][lane]
731 =
732 u8_FFFD1240[channel]
733 [extended_silicon_revision]
734 [lane][2 * slot +
735 rank][info->
736 clock_speed_index]
737 + info->max4048[channel]
738 +
739 u8_FFFD0C78[channel]
740 [extended_silicon_revision]
741 [info->
742 mode4030[channel]][slot]
743 [rank][info->
744 clock_speed_index]
745 + card_timing;
746 for (tm_reg = 0; tm_reg < 4; tm_reg++)
747 write_500(info, channel,
748 info->training.
749 lane_timings[tm_reg]
750 [channel][slot][rank]
751 [lane],
752 get_timing_register_addr
753 (lane, tm_reg, slot,
754 rank), 9, 0);
755 }
756
757 card_timing_2 = 0;
758 if (!(extended_silicon_revision != 4
759 || (info->
760 populated_ranks_mask[channel] & 5) ==
761 5)) {
762 if ((info->
763 spd[channel][slot]
764 [REFERENCE_RAW_CARD_USED] & 0x1F)
765 == 3)
766 card_timing_2 =
767 u16_FFFE0EB8[0][info->
768 clock_speed_index];
769 if ((info->
770 spd[channel][slot]
771 [REFERENCE_RAW_CARD_USED] & 0x1F)
772 == 5)
773 card_timing_2 =
774 u16_FFFE0EB8[1][info->
775 clock_speed_index];
776 }
777
778 for (i = 0; i < 3; i++)
779 write_500(info, channel,
780 (card_timing_2 +
781 info->max4048[channel]
782 +
783 u8_FFFD0EF8[channel]
784 [extended_silicon_revision]
785 [info->
786 mode4030[channel]][info->
787 clock_speed_index]),
788 u16_fffd0c50[i][slot][rank],
789 8, 1);
790 write_500(info, channel,
791 (info->max4048[channel] +
792 u8_FFFD0C78[channel]
793 [extended_silicon_revision][info->
794 mode4030
795 [channel]]
796 [slot][rank][info->
797 clock_speed_index]),
798 u16_fffd0c70[slot][rank], 7, 1);
799 }
800 if (!info->populated_ranks_mask[channel])
801 continue;
802 for (i = 0; i < 3; i++)
803 write_500(info, channel,
804 (info->max4048[channel] +
805 info->avg4044[channel]
806 +
807 u8_FFFD17E0[channel]
808 [extended_silicon_revision][info->
809 mode4030
810 [channel]][info->
811 clock_speed_index]),
812 u16_fffd0c68[i], 8, 1);
813 }
814}
815
816static unsigned int fsbcycle_ps(struct raminfo *info)
817{
818 return 900000 / info->fsb_frequency;
819}
820
821/* The time of DDR transfer in ps. */
822static unsigned int halfcycle_ps(struct raminfo *info)
823{
824 return 3750 / (info->clock_speed_index + 3);
825}
826
827/* The time of clock cycle in ps. */
828static unsigned int cycle_ps(struct raminfo *info)
829{
830 return 2 * halfcycle_ps(info);
831}
832
833/* Frequency in 1.(1)=10/9 MHz units. */
834static unsigned frequency_11(struct raminfo *info)
835{
836 return (info->clock_speed_index + 3) * 120;
837}
838
839/* Frequency in 0.1 MHz units. */
840static unsigned frequency_01(struct raminfo *info)
841{
842 return 100 * frequency_11(info) / 9;
843}
844
845static unsigned ps_to_halfcycles(struct raminfo *info, unsigned int ps)
846{
847 return (frequency_11(info) * 2) * ps / 900000;
848}
849
850static unsigned ns_to_cycles(struct raminfo *info, unsigned int ns)
851{
852 return (frequency_11(info)) * ns / 900;
853}
854
855static void compute_derived_timings(struct raminfo *info)
856{
857 unsigned channel, slot, rank;
858 int extended_silicon_revision;
859 int some_delay_1_ps;
860 int some_delay_2_ps;
861 int some_delay_2_halfcycles_ceil;
862 int some_delay_2_halfcycles_floor;
863 int some_delay_3_ps;
864 int some_delay_3_halfcycles;
865 int some_delay_3_ps_rounded;
866 int some_delay_1_cycle_ceil;
867 int some_delay_1_cycle_floor;
868
869 some_delay_3_halfcycles = 0;
870 some_delay_3_ps_rounded = 0;
871 extended_silicon_revision = info->silicon_revision;
872 if (!info->silicon_revision)
873 for (channel = 0; channel < NUM_CHANNELS; channel++)
874 for (slot = 0; slot < NUM_SLOTS; slot++)
875 if ((info->
876 spd[channel][slot][MODULE_TYPE] & 0xF) ==
877 3)
878 extended_silicon_revision = 4;
879 if (info->board_lane_delay[7] < 5)
880 info->board_lane_delay[7] = 5;
881 info->revision_flag_1 = 2;
882 if (info->silicon_revision == 2 || info->silicon_revision == 3)
883 info->revision_flag_1 = 0;
884 if (info->revision < 16)
885 info->revision_flag_1 = 0;
886
887 if (info->revision < 8)
888 info->revision_flag_1 = 0;
889 if (info->revision >= 8 && (info->silicon_revision == 0
890 || info->silicon_revision == 1))
891 some_delay_2_ps = 735;
892 else
893 some_delay_2_ps = 750;
894
895 if (info->revision >= 0x10 && (info->silicon_revision == 0
896 || info->silicon_revision == 1))
897 some_delay_1_ps = 3929;
898 else
899 some_delay_1_ps = 3490;
900
901 some_delay_1_cycle_floor = some_delay_1_ps / cycle_ps(info);
902 some_delay_1_cycle_ceil = some_delay_1_ps / cycle_ps(info);
903 if (some_delay_1_ps % cycle_ps(info))
904 some_delay_1_cycle_ceil++;
905 else
906 some_delay_1_cycle_floor--;
907 info->some_delay_1_cycle_floor = some_delay_1_cycle_floor;
908 if (info->revision_flag_1)
909 some_delay_2_ps = halfcycle_ps(info) >> 6;
910 some_delay_2_ps +=
911 max(some_delay_1_ps - 30,
912 2 * halfcycle_ps(info) * (some_delay_1_cycle_ceil - 1) + 1000) +
913 375;
914 some_delay_3_ps =
915 halfcycle_ps(info) - some_delay_2_ps % halfcycle_ps(info);
916 if (info->revision_flag_1) {
917 if (some_delay_3_ps < 150)
918 some_delay_3_halfcycles = 0;
919 else
920 some_delay_3_halfcycles =
921 (some_delay_3_ps << 6) / halfcycle_ps(info);
922 some_delay_3_ps_rounded =
923 halfcycle_ps(info) * some_delay_3_halfcycles >> 6;
924 }
925 some_delay_2_halfcycles_ceil =
926 (some_delay_2_ps + halfcycle_ps(info) - 1) / halfcycle_ps(info) -
927 2 * (some_delay_1_cycle_ceil - 1);
928 if (info->revision_flag_1 && some_delay_3_ps < 150)
929 some_delay_2_halfcycles_ceil++;
930 some_delay_2_halfcycles_floor = some_delay_2_halfcycles_ceil;
931 if (info->revision < 0x10)
932 some_delay_2_halfcycles_floor =
933 some_delay_2_halfcycles_ceil - 1;
934 if (!info->revision_flag_1)
935 some_delay_2_halfcycles_floor++;
936 info->some_delay_2_halfcycles_ceil = some_delay_2_halfcycles_ceil;
937 info->some_delay_3_ps_rounded = some_delay_3_ps_rounded;
938 if ((info->populated_ranks[0][0][0] && info->populated_ranks[0][1][0])
939 || (info->populated_ranks[1][0][0]
940 && info->populated_ranks[1][1][0]))
941 info->max_slots_used_in_channel = 2;
942 else
943 info->max_slots_used_in_channel = 1;
944 for (channel = 0; channel < 2; channel++)
945 write_mchbar32(0x244 + (channel << 10),
946 ((info->revision < 8) ? 1 : 0x200)
947 | ((2 - info->max_slots_used_in_channel) << 17) |
948 (channel << 21) | (info->
949 some_delay_1_cycle_floor <<
950 18) | 0x9510);
951 if (info->max_slots_used_in_channel == 1) {
952 info->mode4030[0] = (count_ranks_in_channel(info, 0) == 2);
953 info->mode4030[1] = (count_ranks_in_channel(info, 1) == 2);
954 } else {
955 info->mode4030[0] = ((count_ranks_in_channel(info, 0) == 1) || (count_ranks_in_channel(info, 0) == 2)) ? 2 : 3; /* 2 if 1 or 2 ranks */
956 info->mode4030[1] = ((count_ranks_in_channel(info, 1) == 1)
957 || (count_ranks_in_channel(info, 1) ==
958 2)) ? 2 : 3;
959 }
960 for (channel = 0; channel < NUM_CHANNELS; channel++) {
961 int max_of_unk;
962 int min_of_unk_2;
963
964 int i, count;
965 int sum;
966
967 if (!info->populated_ranks_mask[channel])
968 continue;
969
970 max_of_unk = 0;
971 min_of_unk_2 = 32767;
972
973 sum = 0;
974 count = 0;
975 for (i = 0; i < 3; i++) {
976 int unk1;
977 if (info->revision < 8)
978 unk1 =
979 u8_FFFD1891[0][channel][info->
980 clock_speed_index]
981 [i];
982 else if (!
983 (info->revision >= 0x10
984 || info->revision_flag_1))
985 unk1 =
986 u8_FFFD1891[1][channel][info->
987 clock_speed_index]
988 [i];
989 else
990 unk1 = 0;
991 for (slot = 0; slot < NUM_SLOTS; slot++)
992 for (rank = 0; rank < NUM_RANKS; rank++) {
993 int a = 0;
994 int b = 0;
995
996 if (!info->
997 populated_ranks[channel][slot]
998 [rank])
999 continue;
1000 if (extended_silicon_revision == 4
1001 && (info->
1002 populated_ranks_mask[channel] &
1003 5) != 5) {
1004 if ((info->
1005 spd[channel][slot]
1006 [REFERENCE_RAW_CARD_USED] &
1007 0x1F) == 3) {
1008 a = u16_ffd1178[0]
1009 [info->
1010 clock_speed_index];
1011 b = u16_fe0eb8[0][info->
1012 clock_speed_index];
1013 } else
1014 if ((info->
1015 spd[channel][slot]
1016 [REFERENCE_RAW_CARD_USED]
1017 & 0x1F) == 5) {
1018 a = u16_ffd1178[1]
1019 [info->
1020 clock_speed_index];
1021 b = u16_fe0eb8[1][info->
1022 clock_speed_index];
1023 }
1024 }
1025 min_of_unk_2 = min(min_of_unk_2, a);
1026 min_of_unk_2 = min(min_of_unk_2, b);
1027 if (rank == 0) {
1028 sum += a;
1029 count++;
1030 }
1031 {
1032 int t;
1033 t = b +
1034 u8_FFFD0EF8[channel]
1035 [extended_silicon_revision]
1036 [info->
1037 mode4030[channel]][info->
1038 clock_speed_index];
1039 if (unk1 >= t)
1040 max_of_unk =
1041 max(max_of_unk,
1042 unk1 - t);
1043 }
1044 }
1045 {
1046 int t =
1047 u8_FFFD17E0[channel]
1048 [extended_silicon_revision][info->
1049 mode4030
1050 [channel]]
1051 [info->clock_speed_index] + min_of_unk_2;
1052 if (unk1 >= t)
1053 max_of_unk = max(max_of_unk, unk1 - t);
1054 }
1055 }
1056
1057 info->avg4044[channel] = sum / count;
1058 info->max4048[channel] = max_of_unk;
1059 }
1060}
1061
1062static void jedec_read(struct raminfo *info,
1063 int channel, int slot, int rank,
1064 int total_rank, u8 addr3, unsigned int value)
1065{
1066 /* Handle mirrored mapping. */
1067 if ((rank & 1) && (info->spd[channel][slot][RANK1_ADDRESS_MAPPING] & 1))
1068 addr3 =
1069 (addr3 & 0xCF) | ((addr3 & 0x10) << 1) | ((addr3 >> 1) &
1070 0x10);
1071 write_mchbar8(0x271, addr3 | (read_mchbar8(0x271) & 0xC1));
1072 write_mchbar8(0x671, addr3 | (read_mchbar8(0x671) & 0xC1));
1073
1074 /* Handle mirrored mapping. */
1075 if ((rank & 1) && (info->spd[channel][slot][RANK1_ADDRESS_MAPPING] & 1))
1076 value =
1077 (value & ~0x1f8) | ((value >> 1) & 0xa8) | ((value & 0xa8)
1078 << 1);
1079
1080 read32((value << 3) | (total_rank << 28));
1081
1082 write_mchbar8(0x271, (read_mchbar8(0x271) & 0xC3) | 2);
1083 write_mchbar8(0x671, (read_mchbar8(0x671) & 0xC3) | 2);
1084
1085 read32(total_rank << 28);
1086}
1087
1088enum {
1089 MR1_RZQ12 = 512,
1090 MR1_RZQ2 = 64,
1091 MR1_RZQ4 = 4,
1092 MR1_ODS34OHM = 2
1093};
1094
1095enum {
1096 MR0_BT_INTERLEAVED = 8,
1097 MR0_DLL_RESET_ON = 256
1098};
1099
1100enum {
1101 MR2_RTT_WR_DISABLED = 0,
1102 MR2_RZQ2 = 1 << 10
1103};
1104
1105static void jedec_init(struct raminfo *info)
1106{
1107 int write_recovery;
1108 int channel, slot, rank;
1109 int total_rank;
1110 int dll_on;
1111 int self_refresh_temperature;
1112 int auto_self_refresh;
1113
1114 auto_self_refresh = 1;
1115 self_refresh_temperature = 1;
1116 if (info->board_lane_delay[3] <= 10) {
1117 if (info->board_lane_delay[3] <= 8)
1118 write_recovery = info->board_lane_delay[3] - 4;
1119 else
1120 write_recovery = 5;
1121 } else {
1122 write_recovery = 6;
1123 }
1124 FOR_POPULATED_RANKS {
1125 auto_self_refresh &=
1126 (info->spd[channel][slot][THERMAL_AND_REFRESH] >> 2) & 1;
1127 self_refresh_temperature &=
1128 info->spd[channel][slot][THERMAL_AND_REFRESH] & 1;
1129 }
1130 if (auto_self_refresh == 1)
1131 self_refresh_temperature = 0;
1132
1133 dll_on = ((info->silicon_revision != 2 && info->silicon_revision != 3)
1134 || (info->populated_ranks[0][0][0]
1135 && info->populated_ranks[0][1][0])
1136 || (info->populated_ranks[1][0][0]
1137 && info->populated_ranks[1][1][0]));
1138
1139 total_rank = 0;
1140
1141 for (channel = NUM_CHANNELS - 1; channel >= 0; channel--) {
1142 int rtt, rtt_wr = MR2_RTT_WR_DISABLED;
1143 int rzq_reg58e;
1144
1145 if (info->silicon_revision == 2 || info->silicon_revision == 3) {
1146 rzq_reg58e = 64;
1147 rtt = MR1_RZQ2;
1148 if (info->clock_speed_index != 0) {
1149 rzq_reg58e = 4;
1150 if (info->populated_ranks_mask[channel] == 3)
1151 rtt = MR1_RZQ4;
1152 }
1153 } else {
1154 if ((info->populated_ranks_mask[channel] & 5) == 5) {
1155 rtt = MR1_RZQ12;
1156 rzq_reg58e = 64;
1157 rtt_wr = MR2_RZQ2;
1158 } else {
1159 rzq_reg58e = 4;
1160 rtt = MR1_RZQ4;
1161 }
1162 }
1163
1164 write_mchbar16(0x588 + (channel << 10), 0x0);
1165 write_mchbar16(0x58a + (channel << 10), 0x4);
1166 write_mchbar16(0x58c + (channel << 10), rtt | MR1_ODS34OHM);
1167 write_mchbar16(0x58e + (channel << 10), rzq_reg58e | 0x82);
1168 write_mchbar16(0x590 + (channel << 10), 0x1282);
1169
1170 for (slot = 0; slot < NUM_SLOTS; slot++)
1171 for (rank = 0; rank < NUM_RANKS; rank++)
1172 if (info->populated_ranks[channel][slot][rank]) {
1173 jedec_read(info, channel, slot, rank,
1174 total_rank, 0x28,
1175 rtt_wr | (info->
1176 clock_speed_index
1177 << 3)
1178 | (auto_self_refresh << 6) |
1179 (self_refresh_temperature <<
1180 7));
1181 jedec_read(info, channel, slot, rank,
1182 total_rank, 0x38, 0);
1183 jedec_read(info, channel, slot, rank,
1184 total_rank, 0x18,
1185 rtt | MR1_ODS34OHM);
1186 jedec_read(info, channel, slot, rank,
1187 total_rank, 6,
1188 (dll_on << 12) |
1189 (write_recovery << 9)
1190 | ((info->cas_latency - 4) <<
1191 4) | MR0_BT_INTERLEAVED |
1192 MR0_DLL_RESET_ON);
1193 total_rank++;
1194 }
1195 }
1196}
1197
1198static void program_modules_memory_map(struct raminfo *info, int pre_jedec)
1199{
1200 unsigned channel, slot, rank;
1201 unsigned int total_mb[2] = { 0, 0 }; /* total memory per channel in MB */
1202 unsigned int channel_0_non_interleaved;
1203
1204 FOR_ALL_RANKS {
1205 if (info->populated_ranks[channel][slot][rank]) {
1206 total_mb[channel] +=
1207 pre_jedec ? 256 : (256 << info->
1208 density[channel][slot] >> info->
1209 is_x16_module[channel][slot]);
1210 write_mchbar8(0x208 + rank + 2 * slot + (channel << 10),
1211 (pre_jedec ? (1 | ((1 + 1) << 1))
1212 : (info->
1213 is_x16_module[channel][slot] |
1214 ((info->density[channel][slot] +
1215 1) << 1))) | 0x80);
1216 }
1217 write_mchbar16(0x200 + (channel << 10) + 4 * slot + 2 * rank,
1218 total_mb[channel] >> 6);
1219 }
1220
1221 info->total_memory_mb = total_mb[0] + total_mb[1];
1222
1223 info->interleaved_part_mb =
1224 pre_jedec ? 0 : 2 * min(total_mb[0], total_mb[1]);
1225 info->non_interleaved_part_mb =
1226 total_mb[0] + total_mb[1] - info->interleaved_part_mb;
1227 channel_0_non_interleaved = total_mb[0] - info->interleaved_part_mb / 2;
1228 write_mchbar32(0x100,
1229 channel_0_non_interleaved | (info->
1230 non_interleaved_part_mb <<
1231 16));
1232 if (!pre_jedec)
1233 write_mchbar16(0x104, info->interleaved_part_mb);
1234}
1235
1236static void program_board_delay(struct raminfo *info)
1237{
1238 int cas_latency_shift;
1239 int some_delay_ns;
1240 int some_delay_3_half_cycles;
1241
1242 unsigned channel, i;
1243 int high_multiplier;
1244 int lane_3_delay;
1245 int cas_latency_derived;
1246
1247 high_multiplier = 0;
1248 some_delay_ns = 200;
1249 some_delay_3_half_cycles = 4;
1250 cas_latency_shift = info->silicon_revision == 0
1251 || info->silicon_revision == 1 ? 1 : 0;
1252 if (info->revision < 8) {
1253 some_delay_ns = 600;
1254 cas_latency_shift = 0;
1255 }
1256 {
1257 int speed_bit;
1258 speed_bit =
1259 ((info->clock_speed_index > 1
1260 || (info->silicon_revision != 2
1261 && info->silicon_revision != 3))) ^ (info->revision >=
1262 0x10);
1263 write_500(info, 0, speed_bit | ((!info->use_ecc) << 1), 0x60e,
1264 3, 1);
1265 write_500(info, 1, speed_bit | ((!info->use_ecc) << 1), 0x60e,
1266 3, 1);
1267 if (info->revision >= 0x10 && info->clock_speed_index <= 1
1268 && (info->silicon_revision == 2
1269 || info->silicon_revision == 3))
1270 rmw_1d0(0x116, 5, 2, 4, 1);
1271 }
1272 write_mchbar32(0x120,
1273 (1 << (info->max_slots_used_in_channel + 28)) |
1274 0x188e7f9f);
1275
1276 write_mchbar8(0x124,
1277 info->board_lane_delay[4] +
1278 ((frequency_01(info) + 999) / 1000));
1279 write_mchbar16(0x125, 0x1360);
1280 write_mchbar8(0x127, 0x40);
1281 if (info->fsb_frequency < frequency_11(info) / 2) {
1282 unsigned some_delay_2_half_cycles;
1283 high_multiplier = 1;
1284 some_delay_2_half_cycles = ps_to_halfcycles(info,
1285 ((3 *
1286 fsbcycle_ps(info))
1287 >> 1) +
1288 (halfcycle_ps(info)
1289 *
1290 reg178_min[info->
1291 clock_speed_index]
1292 >> 6)
1293 +
1294 4 *
1295 halfcycle_ps(info)
1296 + 2230);
1297 some_delay_3_half_cycles =
1298 min((some_delay_2_half_cycles +
1299 (frequency_11(info) * 2) * (28 -
1300 some_delay_2_half_cycles) /
1301 (frequency_11(info) * 2 -
1302 4 * (info->fsb_frequency))) >> 3, 7);
1303 }
1304 if (read_mchbar8(0x2ca9) & 1)
1305 some_delay_3_half_cycles = 3;
1306 for (channel = 0; channel < NUM_CHANNELS; channel++) {
1307 write_mchbar32(0x220 + (channel << 10),
1308 read_mchbar32(0x220 +
1309 (channel << 10)) | 0x18001117);
1310 write_mchbar32(0x224 + (channel << 10),
1311 (info->max_slots_used_in_channel - 1)
1312 |
1313 ((info->cas_latency - 5 -
1314 info->clock_speed_index) << 21)
1315 |
1316 ((info->max_slots_used_in_channel +
1317 info->cas_latency - cas_latency_shift -
1318 4) << 16)
1319 | ((info->cas_latency - cas_latency_shift - 4) <<
1320 26)
1321 |
1322 ((info->cas_latency - info->clock_speed_index +
1323 info->max_slots_used_in_channel - 6) << 8));
1324 write_mchbar32(0x228 + (channel << 10),
1325 info->max_slots_used_in_channel);
1326 write_mchbar8(0x239 + (channel << 10), 32);
1327 write_mchbar32(0x248 + (channel << 10),
1328 (high_multiplier << 24) |
1329 (some_delay_3_half_cycles << 25) | 0x840000);
1330 write_mchbar32(0x278 + (channel << 10), 0xc362042);
1331 write_mchbar32(0x27c + (channel << 10), 0x8b000062);
1332 write_mchbar32(0x24c + (channel << 10),
1333 ((! !info->
1334 clock_speed_index) << 17) | (((2 +
1335 info->
1336 clock_speed_index
1337 -
1338 (! !info->
1339 clock_speed_index)))
1340 << 12) | 0x10200);
1341
1342 write_mchbar8(0x267 + (channel << 10), 0x4);
1343 write_mchbar16(0x272 + (channel << 10), 0x155);
1344 write_mchbar32(0x2bc + (channel << 10),
1345 (read_mchbar32(0x2bc + (channel << 10)) &
1346 0xFF000000)
1347 | 0x707070);
1348
1349 write_500(info, channel,
1350 ((!info->populated_ranks[channel][1][1])
1351 | (!info->populated_ranks[channel][1][0] << 1)
1352 | (!info->populated_ranks[channel][0][1] << 2)
1353 | (!info->populated_ranks[channel][0][0] << 3)),
1354 0x4c9, 4, 1);
1355 }
1356
1357 write_mchbar8(0x2c4, ((1 + (info->clock_speed_index != 0)) << 6) | 0xC);
1358 {
1359 u8 freq_divisor = 2;
1360 if (info->fsb_frequency == frequency_11(info))
1361 freq_divisor = 3;
1362 else if (2 * info->fsb_frequency < 3 * (frequency_11(info) / 2))
1363 freq_divisor = 1;
1364 else
1365 freq_divisor = 2;
1366 write_mchbar32(0x2c0, (freq_divisor << 11) | 0x6009c400);
1367 }
1368
1369 if (info->board_lane_delay[3] <= 10) {
1370 if (info->board_lane_delay[3] <= 8)
1371 lane_3_delay = info->board_lane_delay[3];
1372 else
1373 lane_3_delay = 10;
1374 } else {
1375 lane_3_delay = 12;
1376 }
1377 cas_latency_derived = info->cas_latency - info->clock_speed_index + 2;
1378 if (info->clock_speed_index > 1)
1379 cas_latency_derived++;
1380 for (channel = 0; channel < NUM_CHANNELS; channel++) {
1381 write_mchbar32(0x240 + (channel << 10),
1382 ((info->clock_speed_index ==
1383 0) * 0x11000) | 0x1002100 | ((2 +
1384 info->
1385 clock_speed_index)
1386 << 4) | (info->
1387 cas_latency
1388 - 3));
1389 write_500(info, channel, (info->clock_speed_index << 1) | 1,
1390 0x609, 6, 1);
1391 write_500(info, channel,
1392 info->clock_speed_index + 2 * info->cas_latency - 7,
1393 0x601, 6, 1);
1394
1395 write_mchbar32(0x250 + (channel << 10),
1396 ((lane_3_delay + info->clock_speed_index +
1397 9) << 6)
1398 | (info->board_lane_delay[7] << 2) | (info->
1399 board_lane_delay
1400 [4] << 16)
1401 | (info->board_lane_delay[1] << 25) | (info->
1402 board_lane_delay
1403 [1] << 29)
1404 | 1);
1405 write_mchbar32(0x254 + (channel << 10),
1406 (info->
1407 board_lane_delay[1] >> 3) | ((info->
1408 board_lane_delay
1409 [8] +
1410 4 *
1411 info->
1412 use_ecc) << 6) |
1413 0x80 | (info->board_lane_delay[6] << 1) | (info->
1414 board_lane_delay
1415 [2] <<
1416 28) |
1417 (cas_latency_derived << 16) | 0x4700000);
1418 write_mchbar32(0x258 + (channel << 10),
1419 ((info->board_lane_delay[5] +
1420 info->clock_speed_index +
1421 9) << 12) | ((info->clock_speed_index -
1422 info->cas_latency + 12) << 8)
1423 | (info->board_lane_delay[2] << 17) | (info->
1424 board_lane_delay
1425 [4] << 24)
1426 | 0x47);
1427 write_mchbar32(0x25c + (channel << 10),
1428 (info->board_lane_delay[1] << 1) | (info->
1429 board_lane_delay
1430 [0] << 8) |
1431 0x1da50000);
1432 write_mchbar8(0x264 + (channel << 10), 0xff);
1433 write_mchbar8(0x5f8 + (channel << 10),
1434 (cas_latency_shift << 3) | info->use_ecc);
1435 }
1436
1437 program_modules_memory_map(info, 1);
1438
1439 write_mchbar16(0x610,
1440 (min(ns_to_cycles(info, some_delay_ns) / 2, 127) << 9)
1441 | (read_mchbar16(0x610) & 0x1C3) | 0x3C);
1442 write_mchbar16(0x612, read_mchbar16(0x612) | 0x100);
1443 write_mchbar16(0x214, read_mchbar16(0x214) | 0x3E00);
1444 for (i = 0; i < 8; i++) {
1445 pcie_write_config32(PCI_DEV (QUICKPATH_BUS, 0, 1), 0x80 + 4 * i,
1446 (info->total_memory_mb - 64) | !i | 2);
1447 pcie_write_config32(PCI_DEV (QUICKPATH_BUS, 0, 1), 0xc0 + 4 * i, 0);
1448 }
1449}
1450
1451#define BETTER_MEMORY_MAP 0
1452
1453static void program_total_memory_map(struct raminfo *info)
1454{
1455 unsigned int TOM, TOLUD, TOUUD;
1456 unsigned int quickpath_reserved;
1457 unsigned int REMAPbase;
1458 unsigned int uma_base_igd;
1459 unsigned int uma_base_gtt;
1460 int memory_remap;
1461 unsigned int memory_map[8];
1462 int i;
1463 unsigned int current_limit;
1464 unsigned int tseg_base;
1465 int uma_size_igd = 0, uma_size_gtt = 0;
1466
1467 memset(memory_map, 0, sizeof(memory_map));
1468
1469#if REAL
1470 if (info->uma_enabled) {
1471 u16 t = pcie_read_config16(NORTHBRIDGE, D0F0_GGC);
1472 gav(t);
1473 const int uma_sizes_gtt[16] =
1474 { 0, 1, 0, 2, 0, 0, 0, 0, 0, 2, 3, 4, 42, 42, 42, 42 };
1475 /* Igd memory */
1476 const int uma_sizes_igd[16] = {
1477 0, 0, 0, 0, 0, 32, 48, 64, 128, 256, 96, 160, 224, 352,
1478 256, 512
1479 };
1480
1481 uma_size_igd = uma_sizes_igd[(t >> 4) & 0xF];
1482 uma_size_gtt = uma_sizes_gtt[(t >> 8) & 0xF];
1483 }
1484#endif
1485
1486 TOM = info->total_memory_mb;
1487 if (TOM == 4096)
1488 TOM = 4032;
1489 TOUUD = ALIGN_DOWN(TOM - info->memory_reserved_for_heci_mb, 64);
1490 TOLUD = ALIGN_DOWN(min(3072 + ALIGN_UP(uma_size_igd + uma_size_gtt, 64)
1491 , TOUUD), 64);
1492 memory_remap = 0;
1493 if (TOUUD - TOLUD > 64) {
1494 memory_remap = 1;
1495 REMAPbase = max(4096, TOUUD);
1496 TOUUD = TOUUD - TOLUD + 4096;
1497 }
1498 if (TOUUD > 4096)
1499 memory_map[2] = TOUUD | 1;
1500 quickpath_reserved = 0;
1501
1502 {
1503 u32 t;
1504
1505 gav(t = pcie_read_config32(PCI_DEV(QUICKPATH_BUS, 0, 1), 0x68));
1506 if (t & 0x800)
1507 quickpath_reserved =
1508 (1 << find_lowest_bit_set32(t >> 20));
1509 }
1510 if (memory_remap)
1511 TOUUD -= quickpath_reserved;
1512
1513#if !REAL
1514 if (info->uma_enabled) {
1515 u16 t = pcie_read_config16(NORTHBRIDGE, D0F0_GGC);
1516 gav(t);
1517 const int uma_sizes_gtt[16] =
1518 { 0, 1, 0, 2, 0, 0, 0, 0, 0, 2, 3, 4, 42, 42, 42, 42 };
1519 /* Igd memory */
1520 const int uma_sizes_igd[16] = {
1521 0, 0, 0, 0, 0, 32, 48, 64, 128, 256, 96, 160, 224, 352,
1522 256, 512
1523 };
1524
1525 uma_size_igd = uma_sizes_igd[(t >> 4) & 0xF];
1526 uma_size_gtt = uma_sizes_gtt[(t >> 8) & 0xF];
1527 }
1528#endif
1529
1530 uma_base_igd = TOLUD - uma_size_igd;
1531 uma_base_gtt = uma_base_igd - uma_size_gtt;
1532 tseg_base = ALIGN_DOWN(uma_base_gtt, 64) - (CONFIG_SMM_TSEG_SIZE >> 20);
1533 if (!memory_remap)
1534 tseg_base -= quickpath_reserved;
1535 tseg_base = ALIGN_DOWN(tseg_base, 8);
1536
1537 pcie_write_config16(NORTHBRIDGE, D0F0_TOLUD, TOLUD << 4);
1538 pcie_write_config16(NORTHBRIDGE, D0F0_TOM, TOM >> 6);
1539 if (memory_remap) {
1540 pcie_write_config16(NORTHBRIDGE, D0F0_REMAPBASE, REMAPbase >> 6);
1541 pcie_write_config16(NORTHBRIDGE, D0F0_REMAPLIMIT, (TOUUD - 64) >> 6);
1542 }
1543 pcie_write_config16(NORTHBRIDGE, D0F0_TOUUD, TOUUD);
1544
1545 if (info->uma_enabled) {
1546 pcie_write_config32(NORTHBRIDGE, D0F0_IGD_BASE, uma_base_igd << 20);
1547 pcie_write_config32(NORTHBRIDGE, D0F0_GTT_BASE, uma_base_gtt << 20);
1548 }
1549 pcie_write_config32(NORTHBRIDGE, TSEG, tseg_base << 20);
1550
1551 current_limit = 0;
1552 memory_map[0] = ALIGN_DOWN(uma_base_gtt, 64) | 1;
1553 memory_map[1] = 4096;
1554 for (i = 0; i < ARRAY_SIZE(memory_map); i++) {
1555 current_limit = max(current_limit, memory_map[i] & ~1);
1556 pcie_write_config32(PCI_DEV(QUICKPATH_BUS, 0, 1), 4 * i + 0x80,
1557 (memory_map[i] & 1) | ALIGN_DOWN(current_limit -
1558 1, 64) | 2);
1559 pcie_write_config32(PCI_DEV(QUICKPATH_BUS, 0, 1), 4 * i + 0xc0, 0);
1560 }
1561}
1562
1563static void collect_system_info(struct raminfo *info)
1564{
1565 u32 capid0[3];
1566 int i;
1567 unsigned channel;
1568
1569 /* Wait for some bit, maybe TXT clear. */
1570 while (!(read8(0xfed40000) & (1 << 7))) ;
1571
1572 if (!info->heci_bar)
1573 gav(info->heci_bar =
1574 pcie_read_config32(HECIDEV, HECIBAR) & 0xFFFFFFF8);
1575 if (!info->memory_reserved_for_heci_mb) {
1576 /* Wait for ME to be ready */
1577 intel_early_me_init();
1578 info->memory_reserved_for_heci_mb = intel_early_me_uma_size();
1579 }
1580
1581 for (i = 0; i < 3; i++)
1582 gav(capid0[i] =
1583 pcie_read_config32(NORTHBRIDGE, D0F0_CAPID0 | (i << 2)));
1584 gav(info->revision = pcie_read_config8(NORTHBRIDGE, PCI_REVISION_ID));
1585 info->max_supported_clock_speed_index = (~capid0[1] & 7);
1586
1587 if ((capid0[1] >> 11) & 1)
1588 info->uma_enabled = 0;
1589 else
1590 gav(info->uma_enabled =
1591 pcie_read_config8(NORTHBRIDGE, D0F0_DEVEN) & 8);
1592 /* Unrecognised: [0000:fffd3d2d] 37f81.37f82 ! CPUID: eax: 00000001; ecx: 00000e00 => 00020655.00010800.029ae3ff.bfebfbff */
1593 info->silicon_revision = 0;
1594
1595 if (capid0[2] & 2) {
1596 info->silicon_revision = 0;
1597 info->max_supported_clock_speed_index = 2;
1598 for (channel = 0; channel < NUM_CHANNELS; channel++)
1599 if (info->populated_ranks[channel][0][0]
1600 && (info->spd[channel][0][MODULE_TYPE] & 0xf) ==
1601 3) {
1602 info->silicon_revision = 2;
1603 info->max_supported_clock_speed_index = 1;
1604 }
1605 } else {
1606 switch (((capid0[2] >> 18) & 1) + 2 * ((capid0[1] >> 3) & 1)) {
1607 case 1:
1608 case 2:
1609 info->silicon_revision = 3;
1610 break;
1611 case 3:
1612 info->silicon_revision = 0;
1613 break;
1614 case 0:
1615 info->silicon_revision = 2;
1616 break;
1617 }
1618 switch (pcie_read_config16(NORTHBRIDGE, PCI_DEVICE_ID)) {
1619 case 0x40:
1620 info->silicon_revision = 0;
1621 break;
1622 case 0x48:
1623 info->silicon_revision = 1;
1624 break;
1625 }
1626 }
1627}
1628
1629static void write_training_data(struct raminfo *info)
1630{
1631 int tm, channel, slot, rank, lane;
1632 if (info->revision < 8)
1633 return;
1634
1635 for (tm = 0; tm < 4; tm++)
1636 for (channel = 0; channel < NUM_CHANNELS; channel++)
1637 for (slot = 0; slot < NUM_SLOTS; slot++)
1638 for (rank = 0; rank < NUM_RANKS; rank++)
1639 for (lane = 0; lane < 9; lane++)
1640 write_500(info, channel,
1641 info->
1642 cached_training->
1643 lane_timings[tm]
1644 [channel][slot][rank]
1645 [lane],
1646 get_timing_register_addr
1647 (lane, tm, slot,
1648 rank), 9, 0);
1649 write_1d0(info->cached_training->reg_178, 0x178, 7, 1);
1650 write_1d0(info->cached_training->reg_10b, 0x10b, 6, 1);
1651}
1652
1653static void dump_timings(struct raminfo *info)
1654{
1655#if REAL
1656 int channel, slot, rank, lane, i;
1657 printk(BIOS_DEBUG, "Timings:\n");
1658 FOR_POPULATED_RANKS {
1659 printk(BIOS_DEBUG, "channel %d, slot %d, rank %d\n", channel,
1660 slot, rank);
1661 for (lane = 0; lane < 9; lane++) {
1662 printk(BIOS_DEBUG, "lane %d: ", lane);
1663 for (i = 0; i < 4; i++) {
1664 printk(BIOS_DEBUG, "%x (%x) ",
1665 read_500(info, channel,
1666 get_timing_register_addr
1667 (lane, i, slot, rank),
1668 9),
1669 info->training.
1670 lane_timings[i][channel][slot][rank]
1671 [lane]);
1672 }
1673 printk(BIOS_DEBUG, "\n");
1674 }
1675 }
1676 printk(BIOS_DEBUG, "[178] = %x (%x)\n", read_1d0(0x178, 7),
1677 info->training.reg_178);
1678 printk(BIOS_DEBUG, "[10b] = %x (%x)\n", read_1d0(0x10b, 6),
1679 info->training.reg_10b);
1680#endif
1681}
1682
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]1683/* Read timings and other registers that need to be restored verbatim and
1684 put them to CBMEM.
1685 */
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]1686static void save_timings(struct raminfo *info)
1687{
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]1688 struct ram_training train;
1689 struct mrc_data_container *mrcdata;
1690 int output_len = ALIGN(sizeof(train), 16);
1691 int channel, slot, rank, lane, i;
1692
1693 train = info->training;
1694 FOR_POPULATED_RANKS for (lane = 0; lane < 9; lane++)
1695 for (i = 0; i < 4; i++)
1696 train.lane_timings[i][channel][slot][rank][lane] =
1697 read_500(info, channel,
1698 get_timing_register_addr(lane, i, slot,
1699 rank), 9);
1700 train.reg_178 = read_1d0(0x178, 7);
1701 train.reg_10b = read_1d0(0x10b, 6);
1702
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]1703 for (channel = 0; channel < NUM_CHANNELS; channel++) {
1704 u32 reg32;
1705 reg32 = read_mchbar32 ((channel << 10) + 0x274);
1706 train.reg274265[channel][0] = reg32 >> 16;
1707 train.reg274265[channel][1] = reg32 & 0xffff;
1708 train.reg274265[channel][2] = read_mchbar16 ((channel << 10) + 0x265) >> 8;
1709 }
1710 train.reg2ca9_bit0 = read_mchbar8(0x2ca9) & 1;
1711 train.reg_6dc = read_mchbar32 (0x6dc);
1712 train.reg_6e8 = read_mchbar32 (0x6e8);
1713
1714 printk (BIOS_SPEW, "[6dc] = %x\n", train.reg_6dc);
1715 printk (BIOS_SPEW, "[6e8] = %x\n", train.reg_6e8);
1716
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]1717 /* Save the MRC S3 restore data to cbmem */
Kyösti Mälkki2d8520b2014-01-06 17:20:31 +0200[diff] [blame]1718 cbmem_recovery(0);
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]1719 mrcdata = cbmem_add
1720 (CBMEM_ID_MRCDATA, output_len + sizeof(struct mrc_data_container));
1721
Kyösti Mälkki743a2182014-06-15 15:59:44 +0300[diff] [blame]1722 if (mrcdata != NULL) {
1723 printk(BIOS_DEBUG, "Relocate MRC DATA from %p to %p (%u bytes)\n",
1724 &train, mrcdata, output_len);
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]1725
Kyösti Mälkki743a2182014-06-15 15:59:44 +0300[diff] [blame]1726 mrcdata->mrc_signature = MRC_DATA_SIGNATURE;
1727 mrcdata->mrc_data_size = output_len;
1728 mrcdata->reserved = 0;
1729 memcpy(mrcdata->mrc_data, &train, sizeof(train));
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]1730
Kyösti Mälkki743a2182014-06-15 15:59:44 +0300[diff] [blame]1731 /* Zero the unused space in aligned buffer. */
1732 if (output_len > sizeof(train))
1733 memset(mrcdata->mrc_data + sizeof(train), 0,
1734 output_len - sizeof(train));
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]1735
Kyösti Mälkki743a2182014-06-15 15:59:44 +0300[diff] [blame]1736 mrcdata->mrc_checksum = compute_ip_checksum(mrcdata->mrc_data,
1737 mrcdata->mrc_data_size);
1738 }
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]1739}
1740
1741#if REAL
1742static const struct ram_training *get_cached_training(void)
1743{
1744 struct mrc_data_container *cont;
1745 cont = find_current_mrc_cache();
1746 if (!cont)
1747 return 0;
1748 return (void *)cont->mrc_data;
1749}
1750#endif
1751
1752/* FIXME: add timeout. */
1753static void wait_heci_ready(void)
1754{
1755 while (!(read32(DEFAULT_HECIBAR | 0xc) & 8)) ; // = 0x8000000c
1756 write32((DEFAULT_HECIBAR | 0x4),
1757 (read32(DEFAULT_HECIBAR | 0x4) & ~0x10) | 0xc);
1758}
1759
1760/* FIXME: add timeout. */
1761static void wait_heci_cb_avail(int len)
1762{
1763 union {
1764 struct mei_csr csr;
1765 u32 raw;
1766 } csr;
1767
1768 while (!(read32(DEFAULT_HECIBAR | 0xc) & 8)) ;
1769
1770 do
1771 csr.raw = read32(DEFAULT_HECIBAR | 0x4);
1772 while (len >
1773 csr.csr.buffer_depth - (csr.csr.buffer_write_ptr -
1774 csr.csr.buffer_read_ptr));
1775}
1776
1777static void send_heci_packet(struct mei_header *head, u32 * payload)
1778{
1779 int len = (head->length + 3) / 4;
1780 int i;
1781
1782 wait_heci_cb_avail(len + 1);
1783
1784 /* FIXME: handle leftovers correctly. */
1785 write32(DEFAULT_HECIBAR | 0, *(u32 *) head);
1786 for (i = 0; i < len - 1; i++)
1787 write32(DEFAULT_HECIBAR | 0, payload[i]);
1788
1789 write32(DEFAULT_HECIBAR | 0, payload[i] & ((1 << (8 * len)) - 1));
1790 write32(DEFAULT_HECIBAR | 0x4, read32(DEFAULT_HECIBAR | 0x4) | 0x4);
1791}
1792
1793static void
1794send_heci_message(u8 * msg, int len, u8 hostaddress, u8 clientaddress)
1795{
1796 struct mei_header head;
1797 int maxlen;
1798
1799 wait_heci_ready();
1800 maxlen = (read32(DEFAULT_HECIBAR | 0x4) >> 24) * 4 - 4;
1801
1802 while (len) {
1803 int cur = len;
1804 if (cur > maxlen) {
1805 cur = maxlen;
1806 head.is_complete = 0;
1807 } else
1808 head.is_complete = 1;
1809 head.length = cur;
1810 head.reserved = 0;
1811 head.client_address = clientaddress;
1812 head.host_address = hostaddress;
1813 send_heci_packet(&head, (u32 *) msg);
1814 len -= cur;
1815 msg += cur;
1816 }
1817}
1818
1819/* FIXME: Add timeout. */
1820static int
1821recv_heci_packet(struct raminfo *info, struct mei_header *head, u32 * packet,
1822 u32 * packet_size)
1823{
1824 union {
1825 struct mei_csr csr;
1826 u32 raw;
1827 } csr;
1828 int i = 0;
1829
1830 write32(DEFAULT_HECIBAR | 0x4, read32(DEFAULT_HECIBAR | 0x4) | 2);
1831 do {
1832 csr.raw = read32(DEFAULT_HECIBAR | 0xc);
1833#if !REAL
1834 if (i++ > 346)
1835 return -1;
1836#endif
1837 }
1838 while (csr.csr.buffer_write_ptr == csr.csr.buffer_read_ptr);
1839 *(u32 *) head = read32(DEFAULT_HECIBAR | 0x8);
1840 if (!head->length) {
1841 write32(DEFAULT_HECIBAR | 0x4,
1842 read32(DEFAULT_HECIBAR | 0x4) | 2);
1843 *packet_size = 0;
1844 return 0;
1845 }
1846 if (head->length + 4 > 4 * csr.csr.buffer_depth
1847 || head->length > *packet_size) {
1848 *packet_size = 0;
1849 return -1;
1850 }
1851
1852 do
1853 csr.raw = read32(DEFAULT_HECIBAR | 0xc);
1854 while ((head->length + 3) >> 2 >
1855 csr.csr.buffer_write_ptr - csr.csr.buffer_read_ptr);
1856
1857 for (i = 0; i < (head->length + 3) >> 2; i++)
1858 packet[i++] = read32(DEFAULT_HECIBAR | 0x8);
1859 *packet_size = head->length;
1860 if (!csr.csr.ready)
1861 *packet_size = 0;
1862 write32(DEFAULT_HECIBAR | 0x4, read32(DEFAULT_HECIBAR | 0x4) | 4);
1863 return 0;
1864}
1865
1866/* FIXME: Add timeout. */
1867static int
1868recv_heci_message(struct raminfo *info, u32 * message, u32 * message_size)
1869{
1870 struct mei_header head;
1871 int current_position;
1872
1873 current_position = 0;
1874 while (1) {
1875 u32 current_size;
1876 current_size = *message_size - current_position;
1877 if (recv_heci_packet
1878 (info, &head, message + (current_position >> 2),
1879 &current_size) == -1)
1880 break;
1881 if (!current_size)
1882 break;
1883 current_position += current_size;
1884 if (head.is_complete) {
1885 *message_size = current_position;
1886 return 0;
1887 }
1888
1889 if (current_position >= *message_size)
1890 break;
1891 }
1892 *message_size = 0;
1893 return -1;
1894}
1895
1896static void send_heci_uma_message(struct raminfo *info)
1897{
1898 struct uma_reply {
1899 u8 group_id;
1900 u8 command;
1901 u8 reserved;
1902 u8 result;
1903 u8 field2;
1904 u8 unk3[0x48 - 4 - 1];
1905 } __attribute__ ((packed)) reply;
1906 struct uma_message {
1907 u8 group_id;
1908 u8 cmd;
1909 u8 reserved;
1910 u8 result;
1911 u32 c2;
1912 u64 heci_uma_addr;
1913 u32 memory_reserved_for_heci_mb;
1914 u16 c3;
1915 } __attribute__ ((packed)) msg = {
1916 0, MKHI_SET_UMA, 0, 0,
1917 0x82,
1918 info->heci_uma_addr, info->memory_reserved_for_heci_mb, 0};
1919 u32 reply_size;
1920
1921 send_heci_message((u8 *) & msg, sizeof(msg), 0, 7);
1922
1923 reply_size = sizeof(reply);
1924 if (recv_heci_message(info, (u32 *) & reply, &reply_size) == -1)
1925 return;
1926
1927 if (reply.command != (MKHI_SET_UMA | (1 << 7)))
1928 die("HECI init failed\n");
1929}
1930
1931static void setup_heci_uma(struct raminfo *info)
1932{
1933 u32 reg44;
1934
1935 reg44 = pcie_read_config32(HECIDEV, 0x44); // = 0x80010020
1936 info->memory_reserved_for_heci_mb = 0;
1937 info->heci_uma_addr = 0;
1938 if (!((reg44 & 0x10000) && !(pcie_read_config32(HECIDEV, 0x40) & 0x20)))
1939 return;
1940
1941 info->heci_bar = pcie_read_config32(HECIDEV, 0x10) & 0xFFFFFFF0;
1942 info->memory_reserved_for_heci_mb = reg44 & 0x3f;
1943 info->heci_uma_addr =
1944 ((u64)
1945 ((((u64) pcie_read_config16(NORTHBRIDGE, D0F0_TOM)) << 6) -
1946 info->memory_reserved_for_heci_mb)) << 20;
1947
1948 pcie_read_config32(NORTHBRIDGE, DMIBAR);
1949 if (info->memory_reserved_for_heci_mb) {
1950 write32(DEFAULT_DMIBAR | 0x14,
1951 read32(DEFAULT_DMIBAR | 0x14) & ~0x80);
1952 write32(DEFAULT_RCBA | 0x14,
1953 read32(DEFAULT_RCBA | 0x14) & ~0x80);
1954 write32(DEFAULT_DMIBAR | 0x20,
1955 read32(DEFAULT_DMIBAR | 0x20) & ~0x80);
1956 write32(DEFAULT_RCBA | 0x20,
1957 read32(DEFAULT_RCBA | 0x20) & ~0x80);
1958 write32(DEFAULT_DMIBAR | 0x2c,
1959 read32(DEFAULT_DMIBAR | 0x2c) & ~0x80);
1960 write32(DEFAULT_RCBA | 0x30,
1961 read32(DEFAULT_RCBA | 0x30) & ~0x80);
1962 write32(DEFAULT_DMIBAR | 0x38,
1963 read32(DEFAULT_DMIBAR | 0x38) & ~0x80);
1964 write32(DEFAULT_RCBA | 0x40,
1965 read32(DEFAULT_RCBA | 0x40) & ~0x80);
1966
1967 write32(DEFAULT_RCBA | 0x40, 0x87000080); // OK
1968 write32(DEFAULT_DMIBAR | 0x38, 0x87000080); // OK
1969 while (read16(DEFAULT_RCBA | 0x46) & 2
1970 && read16(DEFAULT_DMIBAR | 0x3e) & 2) ;
1971 }
1972
1973 write_mchbar32(0x24, 0x10000 + info->memory_reserved_for_heci_mb);
1974
1975 send_heci_uma_message(info);
1976
1977 pcie_write_config32(HECIDEV, 0x10, 0x0);
1978 pcie_write_config8(HECIDEV, 0x4, 0x0);
1979
1980}
1981
1982static int have_match_ranks(struct raminfo *info, int channel, int ranks)
1983{
1984 int ranks_in_channel;
1985 ranks_in_channel = info->populated_ranks[channel][0][0]
1986 + info->populated_ranks[channel][0][1]
1987 + info->populated_ranks[channel][1][0]
1988 + info->populated_ranks[channel][1][1];
1989
1990 /* empty channel */
1991 if (ranks_in_channel == 0)
1992 return 1;
1993
1994 if (ranks_in_channel != ranks)
1995 return 0;
1996 /* single slot */
1997 if (info->populated_ranks[channel][0][0] !=
1998 info->populated_ranks[channel][1][0])
1999 return 1;
2000 if (info->populated_ranks[channel][0][1] !=
2001 info->populated_ranks[channel][1][1])
2002 return 1;
2003 if (info->is_x16_module[channel][0] != info->is_x16_module[channel][1])
2004 return 0;
2005 if (info->density[channel][0] != info->density[channel][1])
2006 return 0;
2007 return 1;
2008}
2009
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]2010static void read_4090(struct raminfo *info)
2011{
2012 int i, channel, slot, rank, lane;
2013 for (i = 0; i < 2; i++)
2014 for (slot = 0; slot < NUM_SLOTS; slot++)
2015 for (rank = 0; rank < NUM_RANKS; rank++)
2016 for (lane = 0; lane < 9; lane++)
2017 info->training.
2018 lane_timings[0][i][slot][rank][lane]
2019 = 32;
2020
2021 for (i = 1; i < 4; i++)
2022 for (channel = 0; channel < NUM_CHANNELS; channel++)
2023 for (slot = 0; slot < NUM_SLOTS; slot++)
2024 for (rank = 0; rank < NUM_RANKS; rank++)
2025 for (lane = 0; lane < 9; lane++) {
2026 info->training.
2027 lane_timings[i][channel]
2028 [slot][rank][lane] =
2029 read_500(info, channel,
2030 get_timing_register_addr
2031 (lane, i, slot,
2032 rank), 9)
2033 + (i == 1) * 11; // !!!!
2034 }
2035
2036}
2037
2038static u32 get_etalon2(int flip, u32 addr)
2039{
2040 const u16 invmask[] = {
2041 0xaaaa, 0x6db6, 0x4924, 0xeeee, 0xcccc, 0x8888, 0x7bde, 0x739c,
2042 0x6318, 0x4210, 0xefbe, 0xcf3c, 0x8e38, 0x0c30, 0x0820
2043 };
2044 u32 ret;
2045 u32 comp4 = addr / 480;
2046 addr %= 480;
2047 u32 comp1 = addr & 0xf;
2048 u32 comp2 = (addr >> 4) & 1;
2049 u32 comp3 = addr >> 5;
2050
2051 if (comp4)
2052 ret = 0x1010101 << (comp4 - 1);
2053 else
2054 ret = 0;
2055 if (flip ^ (((invmask[comp3] >> comp1) ^ comp2) & 1))
2056 ret = ~ret;
2057
2058 return ret;
2059}
2060
2061static void disable_cache(void)
2062{
2063 msr_t msr = {.lo = 0, .hi = 0 };
2064
2065 wrmsr(MTRRphysBase_MSR(3), msr);
2066 wrmsr(MTRRphysMask_MSR(3), msr);
2067}
2068
2069static void enable_cache(unsigned int base, unsigned int size)
2070{
2071 msr_t msr;
2072 msr.lo = base | MTRR_TYPE_WRPROT;
2073 msr.hi = 0;
2074 wrmsr(MTRRphysBase_MSR(3), msr);
2075 msr.lo = ((~(ALIGN_DOWN(size + 4096, 4096) - 1) | MTRRdefTypeEn)
2076 & 0xffffffff);
2077 msr.hi = 0x0000000f;
2078 wrmsr(MTRRphysMask_MSR(3), msr);
2079}
2080
2081static void flush_cache(u32 start, u32 size)
2082{
2083 u32 end;
2084 u32 addr;
2085
2086 end = start + (ALIGN_DOWN(size + 4096, 4096));
2087 for (addr = start; addr < end; addr += 64)
2088 clflush(addr);
2089}
2090
2091static void clear_errors(void)
2092{
2093 pcie_write_config8(NORTHBRIDGE, 0xc0, 0x01);
2094}
2095
2096static void write_testing(struct raminfo *info, int totalrank, int flip)
2097{
2098 int nwrites = 0;
2099 /* in 8-byte units. */
2100 u32 offset;
2101 u32 base;
2102
2103 base = totalrank << 28;
2104 for (offset = 0; offset < 9 * 480; offset += 2) {
2105 write32(base + offset * 8, get_etalon2(flip, offset));
2106 write32(base + offset * 8 + 4, get_etalon2(flip, offset));
2107 write32(base + offset * 8 + 8, get_etalon2(flip, offset + 1));
2108 write32(base + offset * 8 + 12, get_etalon2(flip, offset + 1));
2109 nwrites += 4;
2110 if (nwrites >= 320) {
2111 clear_errors();
2112 nwrites = 0;
2113 }
2114 }
2115}
2116
2117static u8 check_testing(struct raminfo *info, u8 total_rank, int flip)
2118{
2119 u8 failmask = 0;
2120 int i;
2121 int comp1, comp2, comp3;
2122 u32 failxor[2] = { 0, 0 };
2123
2124 enable_cache((total_rank << 28), 1728 * 5 * 4);
2125
2126 for (comp3 = 0; comp3 < 9 && failmask != 0xff; comp3++) {
2127 for (comp1 = 0; comp1 < 4; comp1++)
2128 for (comp2 = 0; comp2 < 60; comp2++) {
2129 u32 re[4];
2130 u32 curroffset =
2131 comp3 * 8 * 60 + 2 * comp1 + 8 * comp2;
2132 read128((total_rank << 28) | (curroffset << 3),
2133 (u64 *) re);
2134 failxor[0] |=
2135 get_etalon2(flip, curroffset) ^ re[0];
2136 failxor[1] |=
2137 get_etalon2(flip, curroffset) ^ re[1];
2138 failxor[0] |=
2139 get_etalon2(flip, curroffset | 1) ^ re[2];
2140 failxor[1] |=
2141 get_etalon2(flip, curroffset | 1) ^ re[3];
2142 }
2143 for (i = 0; i < 8; i++)
2144 if ((0xff << (8 * (i % 4))) & failxor[i / 4])
2145 failmask |= 1 << i;
2146 }
2147 disable_cache();
2148 flush_cache((total_rank << 28), 1728 * 5 * 4);
2149 return failmask;
2150}
2151
2152const u32 seed1[0x18] = {
2153 0x3a9d5ab5, 0x576cb65b, 0x555773b6, 0x2ab772ee,
2154 0x555556ee, 0x3a9d5ab5, 0x576cb65b, 0x555773b6,
2155 0x2ab772ee, 0x555556ee, 0x5155a555, 0x5155a555,
2156 0x5155a555, 0x5155a555, 0x3a9d5ab5, 0x576cb65b,
2157 0x555773b6, 0x2ab772ee, 0x555556ee, 0x55d6b4a5,
2158 0x366d6b3a, 0x2ae5ddbb, 0x3b9ddbb7, 0x55d6b4a5,
2159};
2160
2161static u32 get_seed2(int a, int b)
2162{
2163 const u32 seed2[5] = {
2164 0x55555555, 0x33333333, 0x2e555a55, 0x55555555,
2165 0x5b6db6db,
2166 };
2167 u32 r;
2168 r = seed2[(a + (a >= 10)) / 5];
2169 return b ? ~r : r;
2170}
2171
2172static int make_shift(int comp2, int comp5, int x)
2173{
2174 const u8 seed3[32] = {
2175 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2176 0x00, 0x00, 0x38, 0x1c, 0x3c, 0x18, 0x38, 0x38,
2177 0x38, 0x38, 0x38, 0x38, 0x0f, 0x0f, 0x0f, 0x0f,
2178 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
2179 };
2180
2181 return (comp2 - ((seed3[comp5] >> (x & 7)) & 1)) & 0x1f;
2182}
2183
2184static u32 get_etalon(int flip, u32 addr)
2185{
2186 u32 mask_byte = 0;
2187 int comp1 = (addr >> 1) & 1;
2188 int comp2 = (addr >> 3) & 0x1f;
2189 int comp3 = (addr >> 8) & 0xf;
2190 int comp4 = (addr >> 12) & 0xf;
2191 int comp5 = (addr >> 16) & 0x1f;
2192 u32 mask_bit = ~(0x10001 << comp3);
2193 u32 part1;
2194 u32 part2;
2195 int byte;
2196
2197 part2 =
2198 ((seed1[comp5] >>
2199 make_shift(comp2, comp5,
2200 (comp3 >> 3) | (comp1 << 2) | 2)) & 1) ^ flip;
2201 part1 =
2202 ((seed1[comp5] >>
2203 make_shift(comp2, comp5,
2204 (comp3 >> 3) | (comp1 << 2) | 0)) & 1) ^ flip;
2205
2206 for (byte = 0; byte < 4; byte++)
2207 if ((get_seed2(comp5, comp4) >>
2208 make_shift(comp2, comp5, (byte | (comp1 << 2)))) & 1)
2209 mask_byte |= 0xff << (8 * byte);
2210
2211 return (mask_bit & mask_byte) | (part1 << comp3) | (part2 <<
2212 (comp3 + 16));
2213}
2214
2215static void
2216write_testing_type2(struct raminfo *info, u8 totalrank, u8 region, u8 block,
2217 char flip)
2218{
2219 int i;
2220 for (i = 0; i < 2048; i++)
2221 write32((totalrank << 28) | (region << 25) | (block << 16) |
2222 (i << 2), get_etalon(flip, (block << 16) | (i << 2)));
2223}
2224
2225static u8
2226check_testing_type2(struct raminfo *info, u8 totalrank, u8 region, u8 block,
2227 char flip)
2228{
2229 u8 failmask = 0;
2230 u32 failxor[2];
2231 int i;
2232 int comp1, comp2, comp3;
2233
2234 failxor[0] = 0;
2235 failxor[1] = 0;
2236
2237 enable_cache(totalrank << 28, 134217728);
2238 for (comp3 = 0; comp3 < 2 && failmask != 0xff; comp3++) {
2239 for (comp1 = 0; comp1 < 16; comp1++)
2240 for (comp2 = 0; comp2 < 64; comp2++) {
2241 u32 addr =
2242 (totalrank << 28) | (region << 25) | (block
2243 << 16)
2244 | (comp3 << 12) | (comp2 << 6) | (comp1 <<
2245 2);
2246 failxor[comp1 & 1] |=
2247 read32(addr) ^ get_etalon(flip, addr);
2248 }
2249 for (i = 0; i < 8; i++)
2250 if ((0xff << (8 * (i % 4))) & failxor[i / 4])
2251 failmask |= 1 << i;
2252 }
2253 disable_cache();
2254 flush_cache((totalrank << 28) | (region << 25) | (block << 16), 16384);
2255 return failmask;
2256}
2257
2258static int check_bounded(unsigned short *vals, u16 bound)
2259{
2260 int i;
2261
2262 for (i = 0; i < 8; i++)
2263 if (vals[i] < bound)
2264 return 0;
2265 return 1;
2266}
2267
2268enum state {
2269 BEFORE_USABLE = 0, AT_USABLE = 1, AT_MARGIN = 2, COMPLETE = 3
2270};
2271
2272static int validate_state(enum state *in)
2273{
2274 int i;
2275 for (i = 0; i < 8; i++)
2276 if (in[i] != COMPLETE)
2277 return 0;
2278 return 1;
2279}
2280
2281static void
2282do_fsm(enum state *state, u16 * counter,
2283 u8 fail_mask, int margin, int uplimit,
2284 u8 * res_low, u8 * res_high, u8 val)
2285{
2286 int lane;
2287
2288 for (lane = 0; lane < 8; lane++) {
2289 int is_fail = (fail_mask >> lane) & 1;
2290 switch (state[lane]) {
2291 case BEFORE_USABLE:
2292 if (!is_fail) {
2293 counter[lane] = 1;
2294 state[lane] = AT_USABLE;
2295 break;
2296 }
2297 counter[lane] = 0;
2298 state[lane] = BEFORE_USABLE;
2299 break;
2300 case AT_USABLE:
2301 if (!is_fail) {
2302 ++counter[lane];
2303 if (counter[lane] >= margin) {
2304 state[lane] = AT_MARGIN;
2305 res_low[lane] = val - margin + 1;
2306 break;
2307 }
2308 state[lane] = 1;
2309 break;
2310 }
2311 counter[lane] = 0;
2312 state[lane] = BEFORE_USABLE;
2313 break;
2314 case AT_MARGIN:
2315 if (is_fail) {
2316 state[lane] = COMPLETE;
2317 res_high[lane] = val - 1;
2318 } else {
2319 counter[lane]++;
2320 state[lane] = AT_MARGIN;
2321 if (val == uplimit) {
2322 state[lane] = COMPLETE;
2323 res_high[lane] = uplimit;
2324 }
2325 }
2326 break;
2327 case COMPLETE:
2328 break;
2329 }
2330 }
2331}
2332
2333static void
2334train_ram_at_178(struct raminfo *info, u8 channel, int slot, int rank,
2335 u8 total_rank, u8 reg_178, int first_run, int niter,
2336 timing_bounds_t * timings)
2337{
2338 int lane;
2339 enum state state[8];
2340 u16 count[8];
2341 u8 lower_usable[8];
2342 u8 upper_usable[8];
2343 unsigned short num_sucessfully_checked[8];
2344 u8 secondary_total_rank;
2345 u8 reg1b3;
2346
2347 if (info->populated_ranks_mask[1]) {
2348 if (channel == 1)
2349 secondary_total_rank =
2350 info->populated_ranks[1][0][0] +
2351 info->populated_ranks[1][0][1]
2352 + info->populated_ranks[1][1][0] +
2353 info->populated_ranks[1][1][1];
2354 else
2355 secondary_total_rank = 0;
2356 } else
2357 secondary_total_rank = total_rank;
2358
2359 {
2360 int i;
2361 for (i = 0; i < 8; i++)
2362 state[i] = BEFORE_USABLE;
2363 }
2364
2365 if (!first_run) {
2366 int is_all_ok = 1;
2367 for (lane = 0; lane < 8; lane++)
2368 if (timings[reg_178][channel][slot][rank][lane].
2369 smallest ==
2370 timings[reg_178][channel][slot][rank][lane].
2371 largest) {
2372 timings[reg_178][channel][slot][rank][lane].
2373 smallest = 0;
2374 timings[reg_178][channel][slot][rank][lane].
2375 largest = 0;
2376 is_all_ok = 0;
2377 }
2378 if (is_all_ok) {
2379 int i;
2380 for (i = 0; i < 8; i++)
2381 state[i] = COMPLETE;
2382 }
2383 }
2384
2385 for (reg1b3 = 0; reg1b3 < 0x30 && !validate_state(state); reg1b3++) {
2386 u8 failmask = 0;
2387 write_1d0(reg1b3 ^ 32, 0x1b3, 6, 1);
2388 write_1d0(reg1b3 ^ 32, 0x1a3, 6, 1);
2389 failmask = check_testing(info, total_rank, 0);
2390 write_mchbar32(0xfb0, read_mchbar32(0xfb0) | 0x00030000);
2391 do_fsm(state, count, failmask, 5, 47, lower_usable,
2392 upper_usable, reg1b3);
2393 }
2394
2395 if (reg1b3) {
2396 write_1d0(0, 0x1b3, 6, 1);
2397 write_1d0(0, 0x1a3, 6, 1);
2398 for (lane = 0; lane < 8; lane++) {
2399 if (state[lane] == COMPLETE) {
2400 timings[reg_178][channel][slot][rank][lane].
2401 smallest =
2402 lower_usable[lane] +
2403 (info->training.
2404 lane_timings[0][channel][slot][rank][lane]
2405 & 0x3F) - 32;
2406 timings[reg_178][channel][slot][rank][lane].
2407 largest =
2408 upper_usable[lane] +
2409 (info->training.
2410 lane_timings[0][channel][slot][rank][lane]
2411 & 0x3F) - 32;
2412 }
2413 }
2414 }
2415
2416 if (!first_run) {
2417 for (lane = 0; lane < 8; lane++)
2418 if (state[lane] == COMPLETE) {
2419 write_500(info, channel,
2420 timings[reg_178][channel][slot][rank]
2421 [lane].smallest,
2422 get_timing_register_addr(lane, 0,
2423 slot, rank),
2424 9, 1);
2425 write_500(info, channel,
2426 timings[reg_178][channel][slot][rank]
2427 [lane].smallest +
2428 info->training.
2429 lane_timings[1][channel][slot][rank]
2430 [lane]
2431 -
2432 info->training.
2433 lane_timings[0][channel][slot][rank]
2434 [lane], get_timing_register_addr(lane,
2435 1,
2436 slot,
2437 rank),
2438 9, 1);
2439 num_sucessfully_checked[lane] = 0;
2440 } else
2441 num_sucessfully_checked[lane] = -1;
2442
2443 do {
2444 u8 failmask = 0;
2445 int i;
2446 for (i = 0; i < niter; i++) {
2447 if (failmask == 0xFF)
2448 break;
2449 failmask |=
2450 check_testing_type2(info, total_rank, 2, i,
2451 0);
2452 failmask |=
2453 check_testing_type2(info, total_rank, 3, i,
2454 1);
2455 }
2456 write_mchbar32(0xfb0,
2457 read_mchbar32(0xfb0) | 0x00030000);
2458 for (lane = 0; lane < 8; lane++)
2459 if (num_sucessfully_checked[lane] != 0xffff) {
2460 if ((1 << lane) & failmask) {
2461 if (timings[reg_178][channel]
2462 [slot][rank][lane].
2463 largest <=
2464 timings[reg_178][channel]
2465 [slot][rank][lane].smallest)
2466 num_sucessfully_checked
2467 [lane] = -1;
2468 else {
2469 num_sucessfully_checked
2470 [lane] = 0;
2471 timings[reg_178]
2472 [channel][slot]
2473 [rank][lane].
2474 smallest++;
2475 write_500(info, channel,
2476 timings
2477 [reg_178]
2478 [channel]
2479 [slot][rank]
2480 [lane].
2481 smallest,
2482 get_timing_register_addr
2483 (lane, 0,
2484 slot, rank),
2485 9, 1);
2486 write_500(info, channel,
2487 timings
2488 [reg_178]
2489 [channel]
2490 [slot][rank]
2491 [lane].
2492 smallest +
2493 info->
2494 training.
2495 lane_timings
2496 [1][channel]
2497 [slot][rank]
2498 [lane]
2499 -
2500 info->
2501 training.
2502 lane_timings
2503 [0][channel]
2504 [slot][rank]
2505 [lane],
2506 get_timing_register_addr
2507 (lane, 1,
2508 slot, rank),
2509 9, 1);
2510 }
2511 } else
2512 num_sucessfully_checked[lane]++;
2513 }
2514 }
2515 while (!check_bounded(num_sucessfully_checked, 2));
2516
2517 for (lane = 0; lane < 8; lane++)
2518 if (state[lane] == COMPLETE) {
2519 write_500(info, channel,
2520 timings[reg_178][channel][slot][rank]
2521 [lane].largest,
2522 get_timing_register_addr(lane, 0,
2523 slot, rank),
2524 9, 1);
2525 write_500(info, channel,
2526 timings[reg_178][channel][slot][rank]
2527 [lane].largest +
2528 info->training.
2529 lane_timings[1][channel][slot][rank]
2530 [lane]
2531 -
2532 info->training.
2533 lane_timings[0][channel][slot][rank]
2534 [lane], get_timing_register_addr(lane,
2535 1,
2536 slot,
2537 rank),
2538 9, 1);
2539 num_sucessfully_checked[lane] = 0;
2540 } else
2541 num_sucessfully_checked[lane] = -1;
2542
2543 do {
2544 int failmask = 0;
2545 int i;
2546 for (i = 0; i < niter; i++) {
2547 if (failmask == 0xFF)
2548 break;
2549 failmask |=
2550 check_testing_type2(info, total_rank, 2, i,
2551 0);
2552 failmask |=
2553 check_testing_type2(info, total_rank, 3, i,
2554 1);
2555 }
2556
2557 write_mchbar32(0xfb0,
2558 read_mchbar32(0xfb0) | 0x00030000);
2559 for (lane = 0; lane < 8; lane++) {
2560 if (num_sucessfully_checked[lane] != 0xffff) {
2561 if ((1 << lane) & failmask) {
2562 if (timings[reg_178][channel]
2563 [slot][rank][lane].
2564 largest <=
2565 timings[reg_178][channel]
2566 [slot][rank][lane].
2567 smallest) {
2568 num_sucessfully_checked
2569 [lane] = -1;
2570 } else {
2571 num_sucessfully_checked
2572 [lane] = 0;
2573 timings[reg_178]
2574 [channel][slot]
2575 [rank][lane].
2576 largest--;
2577 write_500(info, channel,
2578 timings
2579 [reg_178]
2580 [channel]
2581 [slot][rank]
2582 [lane].
2583 largest,
2584 get_timing_register_addr
2585 (lane, 0,
2586 slot, rank),
2587 9, 1);
2588 write_500(info, channel,
2589 timings
2590 [reg_178]
2591 [channel]
2592 [slot][rank]
2593 [lane].
2594 largest +
2595 info->
2596 training.
2597 lane_timings
2598 [1][channel]
2599 [slot][rank]
2600 [lane]
2601 -
2602 info->
2603 training.
2604 lane_timings
2605 [0][channel]
2606 [slot][rank]
2607 [lane],
2608 get_timing_register_addr
2609 (lane, 1,
2610 slot, rank),
2611 9, 1);
2612 }
2613 } else
2614 num_sucessfully_checked[lane]++;
2615 }
2616 }
2617 }
2618 while (!check_bounded(num_sucessfully_checked, 3));
2619
2620 for (lane = 0; lane < 8; lane++) {
2621 write_500(info, channel,
2622 info->training.
2623 lane_timings[0][channel][slot][rank][lane],
2624 get_timing_register_addr(lane, 0, slot, rank),
2625 9, 1);
2626 write_500(info, channel,
2627 info->training.
2628 lane_timings[1][channel][slot][rank][lane],
2629 get_timing_register_addr(lane, 1, slot, rank),
2630 9, 1);
2631 if (timings[reg_178][channel][slot][rank][lane].
2632 largest <=
2633 timings[reg_178][channel][slot][rank][lane].
2634 smallest) {
2635 timings[reg_178][channel][slot][rank][lane].
2636 largest = 0;
2637 timings[reg_178][channel][slot][rank][lane].
2638 smallest = 0;
2639 }
2640 }
2641 }
2642}
2643
2644static void set_10b(struct raminfo *info, u8 val)
2645{
2646 int channel;
2647 int slot, rank;
2648 int lane;
2649
2650 if (read_1d0(0x10b, 6) == val)
2651 return;
2652
2653 write_1d0(val, 0x10b, 6, 1);
2654
2655 FOR_POPULATED_RANKS_BACKWARDS for (lane = 0; lane < 9; lane++) {
2656 u16 reg_500;
2657 reg_500 = read_500(info, channel,
2658 get_timing_register_addr(lane, 0, slot,
2659 rank), 9);
2660 if (val == 1) {
2661 if (lut16[info->clock_speed_index] <= reg_500)
2662 reg_500 -= lut16[info->clock_speed_index];
2663 else
2664 reg_500 = 0;
2665 } else {
2666 reg_500 += lut16[info->clock_speed_index];
2667 }
2668 write_500(info, channel, reg_500,
2669 get_timing_register_addr(lane, 0, slot, rank), 9, 1);
2670 }
2671}
2672
2673static void set_ecc(int onoff)
2674{
2675 int channel;
2676 for (channel = 0; channel < NUM_CHANNELS; channel++) {
2677 u8 t;
2678 t = read_mchbar8((channel << 10) + 0x5f8);
2679 if (onoff)
2680 t |= 1;
2681 else
2682 t &= ~1;
2683 write_mchbar8((channel << 10) + 0x5f8, t);
2684 }
2685}
2686
2687static void set_178(u8 val)
2688{
2689 if (val >= 31)
2690 val = val - 31;
2691 else
2692 val = 63 - val;
2693
2694 write_1d0(2 * val, 0x178, 7, 1);
2695}
2696
2697static void
2698write_500_timings_type(struct raminfo *info, int channel, int slot, int rank,
2699 int type)
2700{
2701 int lane;
2702
2703 for (lane = 0; lane < 8; lane++)
2704 write_500(info, channel,
2705 info->training.
2706 lane_timings[type][channel][slot][rank][lane],
2707 get_timing_register_addr(lane, type, slot, rank), 9,
2708 0);
2709}
2710
2711static void
2712try_timing_offsets(struct raminfo *info, int channel,
2713 int slot, int rank, int totalrank)
2714{
2715 u16 count[8];
2716 enum state state[8];
2717 u8 lower_usable[8], upper_usable[8];
2718 int lane;
2719 int i;
2720 int flip = 1;
2721 int timing_offset;
2722
2723 for (i = 0; i < 8; i++)
2724 state[i] = BEFORE_USABLE;
2725
2726 memset(count, 0, sizeof(count));
2727
2728 for (lane = 0; lane < 8; lane++)
2729 write_500(info, channel,
2730 info->training.
2731 lane_timings[2][channel][slot][rank][lane] + 32,
2732 get_timing_register_addr(lane, 3, slot, rank), 9, 1);
2733
2734 for (timing_offset = 0; !validate_state(state) && timing_offset < 64;
2735 timing_offset++) {
2736 u8 failmask;
2737 write_1d0(timing_offset ^ 32, 0x1bb, 6, 1);
2738 failmask = 0;
2739 for (i = 0; i < 2 && failmask != 0xff; i++) {
2740 flip = !flip;
2741 write_testing(info, totalrank, flip);
2742 failmask |= check_testing(info, totalrank, flip);
2743 }
2744 do_fsm(state, count, failmask, 10, 63, lower_usable,
2745 upper_usable, timing_offset);
2746 }
2747 write_1d0(0, 0x1bb, 6, 1);
2748 dump_timings(info);
2749 if (!validate_state(state))
2750 die("Couldn't discover DRAM timings (1)\n");
2751
2752 for (lane = 0; lane < 8; lane++) {
2753 u8 bias = 0;
2754
2755 if (info->silicon_revision) {
2756 int usable_length;
2757
2758 usable_length = upper_usable[lane] - lower_usable[lane];
2759 if (usable_length >= 20) {
2760 bias = usable_length / 2 - 10;
2761 if (bias >= 2)
2762 bias = 2;
2763 }
2764 }
2765 write_500(info, channel,
2766 info->training.
2767 lane_timings[2][channel][slot][rank][lane] +
2768 (upper_usable[lane] + lower_usable[lane]) / 2 - bias,
2769 get_timing_register_addr(lane, 3, slot, rank), 9, 1);
2770 info->training.timing2_bounds[channel][slot][rank][lane][0] =
2771 info->training.lane_timings[2][channel][slot][rank][lane] +
2772 lower_usable[lane];
2773 info->training.timing2_bounds[channel][slot][rank][lane][1] =
2774 info->training.lane_timings[2][channel][slot][rank][lane] +
2775 upper_usable[lane];
2776 info->training.timing2_offset[channel][slot][rank][lane] =
2777 info->training.lane_timings[2][channel][slot][rank][lane];
2778 }
2779}
2780
2781static u8
2782choose_training(struct raminfo *info, int channel, int slot, int rank,
2783 int lane, timing_bounds_t * timings, u8 center_178)
2784{
2785 u16 central_weight;
2786 u16 side_weight;
2787 unsigned int sum = 0, count = 0;
2788 u8 span;
2789 u8 lower_margin, upper_margin;
2790 u8 reg_178;
2791 u8 result;
2792
2793 span = 12;
2794 central_weight = 20;
2795 side_weight = 20;
2796 if (info->silicon_revision == 1 && channel == 1) {
2797 central_weight = 5;
2798 side_weight = 20;
2799 if ((info->
2800 populated_ranks_mask[1] ^ (info->
2801 populated_ranks_mask[1] >> 2)) &
2802 1)
2803 span = 18;
2804 }
2805 if ((info->populated_ranks_mask[0] & 5) == 5) {
2806 central_weight = 20;
2807 side_weight = 20;
2808 }
2809 if (info->clock_speed_index >= 2
2810 && (info->populated_ranks_mask[0] & 5) == 5 && slot == 1) {
2811 if (info->silicon_revision == 1) {
2812 switch (channel) {
2813 case 0:
2814 if (lane == 1) {
2815 central_weight = 10;
2816 side_weight = 20;
2817 }
2818 break;
2819 case 1:
2820 if (lane == 6) {
2821 side_weight = 5;
2822 central_weight = 20;
2823 }
2824 break;
2825 }
2826 }
2827 if (info->silicon_revision == 0 && channel == 0 && lane == 0) {
2828 side_weight = 5;
2829 central_weight = 20;
2830 }
2831 }
2832 for (reg_178 = center_178 - span; reg_178 <= center_178 + span;
2833 reg_178 += span) {
2834 u8 smallest;
2835 u8 largest;
2836 largest = timings[reg_178][channel][slot][rank][lane].largest;
2837 smallest = timings[reg_178][channel][slot][rank][lane].smallest;
2838 if (largest - smallest + 1 >= 5) {
2839 unsigned int weight;
2840 if (reg_178 == center_178)
2841 weight = central_weight;
2842 else
2843 weight = side_weight;
2844 sum += weight * (largest + smallest);
2845 count += weight;
2846 }
2847 }
2848 dump_timings(info);
2849 if (count == 0)
2850 die("Couldn't discover DRAM timings (2)\n");
2851 result = sum / (2 * count);
2852 lower_margin =
2853 result - timings[center_178][channel][slot][rank][lane].smallest;
2854 upper_margin =
2855 timings[center_178][channel][slot][rank][lane].largest - result;
2856 if (upper_margin < 10 && lower_margin > 10)
2857 result -= min(lower_margin - 10, 10 - upper_margin);
2858 if (upper_margin > 10 && lower_margin < 10)
2859 result += min(upper_margin - 10, 10 - lower_margin);
2860 return result;
2861}
2862
2863#define STANDARD_MIN_MARGIN 5
2864
2865static u8 choose_reg178(struct raminfo *info, timing_bounds_t * timings)
2866{
2867 u16 margin[64];
2868 int lane, rank, slot, channel;
2869 u8 reg178;
2870 int count = 0, sum = 0;
2871
2872 for (reg178 = reg178_min[info->clock_speed_index];
2873 reg178 < reg178_max[info->clock_speed_index];
2874 reg178 += reg178_step[info->clock_speed_index]) {
2875 margin[reg178] = -1;
2876 FOR_POPULATED_RANKS_BACKWARDS for (lane = 0; lane < 8; lane++) {
2877 int curmargin =
2878 timings[reg178][channel][slot][rank][lane].largest -
2879 timings[reg178][channel][slot][rank][lane].
2880 smallest + 1;
2881 if (curmargin < margin[reg178])
2882 margin[reg178] = curmargin;
2883 }
2884 if (margin[reg178] >= STANDARD_MIN_MARGIN) {
2885 u16 weight;
2886 weight = margin[reg178] - STANDARD_MIN_MARGIN;
2887 sum += weight * reg178;
2888 count += weight;
2889 }
2890 }
2891 dump_timings(info);
2892 if (count == 0)
2893 die("Couldn't discover DRAM timings (3)\n");
2894
2895 u8 threshold;
2896
2897 for (threshold = 30; threshold >= 5; threshold--) {
2898 int usable_length = 0;
2899 int smallest_fount = 0;
2900 for (reg178 = reg178_min[info->clock_speed_index];
2901 reg178 < reg178_max[info->clock_speed_index];
2902 reg178 += reg178_step[info->clock_speed_index])
2903 if (margin[reg178] >= threshold) {
2904 usable_length +=
2905 reg178_step[info->clock_speed_index];
2906 info->training.reg178_largest =
2907 reg178 -
2908 2 * reg178_step[info->clock_speed_index];
2909
2910 if (!smallest_fount) {
2911 smallest_fount = 1;
2912 info->training.reg178_smallest =
2913 reg178 +
2914 reg178_step[info->
2915 clock_speed_index];
2916 }
2917 }
2918 if (usable_length >= 0x21)
2919 break;
2920 }
2921
2922 return sum / count;
2923}
2924
2925static int check_cached_sanity(struct raminfo *info)
2926{
2927 int lane;
2928 int slot, rank;
2929 int channel;
2930
2931 if (!info->cached_training)
2932 return 0;
2933
2934 for (channel = 0; channel < NUM_CHANNELS; channel++)
2935 for (slot = 0; slot < NUM_SLOTS; slot++)
2936 for (rank = 0; rank < NUM_RANKS; rank++)
2937 for (lane = 0; lane < 8 + info->use_ecc; lane++) {
2938 u16 cached_value, estimation_value;
2939 cached_value =
2940 info->cached_training->
2941 lane_timings[1][channel][slot][rank]
2942 [lane];
2943 if (cached_value >= 0x18
2944 && cached_value <= 0x1E7) {
2945 estimation_value =
2946 info->training.
2947 lane_timings[1][channel]
2948 [slot][rank][lane];
2949 if (estimation_value <
2950 cached_value - 24)
2951 return 0;
2952 if (estimation_value >
2953 cached_value + 24)
2954 return 0;
2955 }
2956 }
2957 return 1;
2958}
2959
2960static int try_cached_training(struct raminfo *info)
2961{
2962 u8 saved_243[2];
2963 u8 tm;
2964
2965 int channel, slot, rank, lane;
2966 int flip = 1;
2967 int i, j;
2968
2969 if (!check_cached_sanity(info))
2970 return 0;
2971
2972 info->training.reg178_center = info->cached_training->reg178_center;
2973 info->training.reg178_smallest = info->cached_training->reg178_smallest;
2974 info->training.reg178_largest = info->cached_training->reg178_largest;
2975 memcpy(&info->training.timing_bounds,
2976 &info->cached_training->timing_bounds,
2977 sizeof(info->training.timing_bounds));
2978 memcpy(&info->training.timing_offset,
2979 &info->cached_training->timing_offset,
2980 sizeof(info->training.timing_offset));
2981
2982 write_1d0(2, 0x142, 3, 1);
2983 saved_243[0] = read_mchbar8(0x243);
2984 saved_243[1] = read_mchbar8(0x643);
2985 write_mchbar8(0x243, saved_243[0] | 2);
2986 write_mchbar8(0x643, saved_243[1] | 2);
2987 set_ecc(0);
2988 pcie_write_config16(NORTHBRIDGE, 0xc8, 3);
2989 if (read_1d0(0x10b, 6) & 1)
2990 set_10b(info, 0);
2991 for (tm = 0; tm < 2; tm++) {
2992 int totalrank;
2993
2994 set_178(tm ? info->cached_training->reg178_largest : info->
2995 cached_training->reg178_smallest);
2996
2997 totalrank = 0;
2998 /* Check timing ranges. With i == 0 we check smallest one and with
2999 i == 1 the largest bound. With j == 0 we check that on the bound
3000 it still works whereas with j == 1 we check that just outside of
3001 bound we fail.
3002 */
3003 FOR_POPULATED_RANKS_BACKWARDS {
3004 for (i = 0; i < 2; i++) {
3005 for (lane = 0; lane < 8; lane++) {
3006 write_500(info, channel,
3007 info->cached_training->
3008 timing2_bounds[channel][slot]
3009 [rank][lane][i],
3010 get_timing_register_addr(lane,
3011 3,
3012 slot,
3013 rank),
3014 9, 1);
3015
3016 if (!i)
3017 write_500(info, channel,
3018 info->
3019 cached_training->
3020 timing2_offset
3021 [channel][slot][rank]
3022 [lane],
3023 get_timing_register_addr
3024 (lane, 2, slot, rank),
3025 9, 1);
3026 write_500(info, channel,
3027 i ? info->cached_training->
3028 timing_bounds[tm][channel]
3029 [slot][rank][lane].
3030 largest : info->
3031 cached_training->
3032 timing_bounds[tm][channel]
3033 [slot][rank][lane].smallest,
3034 get_timing_register_addr(lane,
3035 0,
3036 slot,
3037 rank),
3038 9, 1);
3039 write_500(info, channel,
3040 info->cached_training->
3041 timing_offset[channel][slot]
3042 [rank][lane] +
3043 (i ? info->cached_training->
3044 timing_bounds[tm][channel]
3045 [slot][rank][lane].
3046 largest : info->
3047 cached_training->
3048 timing_bounds[tm][channel]
3049 [slot][rank][lane].
3050 smallest) - 64,
3051 get_timing_register_addr(lane,
3052 1,
3053 slot,
3054 rank),
3055 9, 1);
3056 }
3057 for (j = 0; j < 2; j++) {
3058 u8 failmask;
3059 u8 expected_failmask;
3060 char reg1b3;
3061
3062 reg1b3 = (j == 1) + 4;
3063 reg1b3 =
3064 j == i ? reg1b3 : (-reg1b3) & 0x3f;
3065 write_1d0(reg1b3, 0x1bb, 6, 1);
3066 write_1d0(reg1b3, 0x1b3, 6, 1);
3067 write_1d0(reg1b3, 0x1a3, 6, 1);
3068
3069 flip = !flip;
3070 write_testing(info, totalrank, flip);
3071 failmask =
3072 check_testing(info, totalrank,
3073 flip);
3074 expected_failmask =
3075 j == 0 ? 0x00 : 0xff;
3076 if (failmask != expected_failmask)
3077 goto fail;
3078 }
3079 }
3080 totalrank++;
3081 }
3082 }
3083
3084 set_178(info->cached_training->reg178_center);
3085 if (info->use_ecc)
3086 set_ecc(1);
3087 write_training_data(info);
3088 write_1d0(0, 322, 3, 1);
3089 info->training = *info->cached_training;
3090
3091 write_1d0(0, 0x1bb, 6, 1);
3092 write_1d0(0, 0x1b3, 6, 1);
3093 write_1d0(0, 0x1a3, 6, 1);
3094 write_mchbar8(0x243, saved_243[0]);
3095 write_mchbar8(0x643, saved_243[1]);
3096
3097 return 1;
3098
3099fail:
3100 FOR_POPULATED_RANKS {
3101 write_500_timings_type(info, channel, slot, rank, 1);
3102 write_500_timings_type(info, channel, slot, rank, 2);
3103 write_500_timings_type(info, channel, slot, rank, 3);
3104 }
3105
3106 write_1d0(0, 0x1bb, 6, 1);
3107 write_1d0(0, 0x1b3, 6, 1);
3108 write_1d0(0, 0x1a3, 6, 1);
3109 write_mchbar8(0x243, saved_243[0]);
3110 write_mchbar8(0x643, saved_243[1]);
3111
3112 return 0;
3113}
3114
3115static void do_ram_training(struct raminfo *info)
3116{
3117 u8 saved_243[2];
3118 int totalrank = 0;
3119 u8 reg_178;
3120 int niter;
3121
3122 timing_bounds_t timings[64];
3123 int lane, rank, slot, channel;
3124 u8 reg178_center;
3125
3126 write_1d0(2, 0x142, 3, 1);
3127 saved_243[0] = read_mchbar8(0x243);
3128 saved_243[1] = read_mchbar8(0x643);
3129 write_mchbar8(0x243, saved_243[0] | 2);
3130 write_mchbar8(0x643, saved_243[1] | 2);
3131 switch (info->clock_speed_index) {
3132 case 0:
3133 niter = 5;
3134 break;
3135 case 1:
3136 niter = 10;
3137 break;
3138 default:
3139 niter = 19;
3140 break;
3141 }
3142 set_ecc(0);
3143
3144 FOR_POPULATED_RANKS_BACKWARDS {
3145 int i;
3146
3147 write_500_timings_type(info, channel, slot, rank, 0);
3148
3149 write_testing(info, totalrank, 0);
3150 for (i = 0; i < niter; i++) {
3151 write_testing_type2(info, totalrank, 2, i, 0);
3152 write_testing_type2(info, totalrank, 3, i, 1);
3153 }
3154 pcie_write_config8(NORTHBRIDGE, 0xc0, 0x01);
3155 totalrank++;
3156 }
3157
3158 if (reg178_min[info->clock_speed_index] <
3159 reg178_max[info->clock_speed_index])
3160 memset(timings[reg178_min[info->clock_speed_index]], 0,
3161 sizeof(timings[0]) *
3162 (reg178_max[info->clock_speed_index] -
3163 reg178_min[info->clock_speed_index]));
3164 for (reg_178 = reg178_min[info->clock_speed_index];
3165 reg_178 < reg178_max[info->clock_speed_index];
3166 reg_178 += reg178_step[info->clock_speed_index]) {
3167 totalrank = 0;
3168 set_178(reg_178);
3169 for (channel = NUM_CHANNELS - 1; channel >= 0; channel--)
3170 for (slot = 0; slot < NUM_SLOTS; slot++)
3171 for (rank = 0; rank < NUM_RANKS; rank++) {
3172 memset(&timings[reg_178][channel][slot]
3173 [rank][0].smallest, 0, 16);
3174 if (info->
3175 populated_ranks[channel][slot]
3176 [rank]) {
3177 train_ram_at_178(info, channel,
3178 slot, rank,
3179 totalrank,
3180 reg_178, 1,
3181 niter,
3182 timings);
3183 totalrank++;
3184 }
3185 }
3186 }
3187
3188 reg178_center = choose_reg178(info, timings);
3189
3190 FOR_POPULATED_RANKS_BACKWARDS for (lane = 0; lane < 8; lane++) {
3191 info->training.timing_bounds[0][channel][slot][rank][lane].
3192 smallest =
3193 timings[info->training.
3194 reg178_smallest][channel][slot][rank][lane].
3195 smallest;
3196 info->training.timing_bounds[0][channel][slot][rank][lane].
3197 largest =
3198 timings[info->training.
3199 reg178_smallest][channel][slot][rank][lane].largest;
3200 info->training.timing_bounds[1][channel][slot][rank][lane].
3201 smallest =
3202 timings[info->training.
3203 reg178_largest][channel][slot][rank][lane].smallest;
3204 info->training.timing_bounds[1][channel][slot][rank][lane].
3205 largest =
3206 timings[info->training.
3207 reg178_largest][channel][slot][rank][lane].largest;
3208 info->training.timing_offset[channel][slot][rank][lane] =
3209 info->training.lane_timings[1][channel][slot][rank][lane]
3210 -
3211 info->training.lane_timings[0][channel][slot][rank][lane] +
3212 64;
3213 }
3214
3215 if (info->silicon_revision == 1
3216 && (info->
3217 populated_ranks_mask[1] ^ (info->
3218 populated_ranks_mask[1] >> 2)) & 1) {
3219 int ranks_after_channel1;
3220
3221 totalrank = 0;
3222 for (reg_178 = reg178_center - 18;
3223 reg_178 <= reg178_center + 18; reg_178 += 18) {
3224 totalrank = 0;
3225 set_178(reg_178);
3226 for (slot = 0; slot < NUM_SLOTS; slot++)
3227 for (rank = 0; rank < NUM_RANKS; rank++) {
3228 if (info->
3229 populated_ranks[1][slot][rank]) {
3230 train_ram_at_178(info, 1, slot,
3231 rank,
3232 totalrank,
3233 reg_178, 0,
3234 niter,
3235 timings);
3236 totalrank++;
3237 }
3238 }
3239 }
3240 ranks_after_channel1 = totalrank;
3241
3242 for (reg_178 = reg178_center - 12;
3243 reg_178 <= reg178_center + 12; reg_178 += 12) {
3244 totalrank = ranks_after_channel1;
3245 set_178(reg_178);
3246 for (slot = 0; slot < NUM_SLOTS; slot++)
3247 for (rank = 0; rank < NUM_RANKS; rank++)
3248 if (info->
3249 populated_ranks[0][slot][rank]) {
3250 train_ram_at_178(info, 0, slot,
3251 rank,
3252 totalrank,
3253 reg_178, 0,
3254 niter,
3255 timings);
3256 totalrank++;
3257 }
3258
3259 }
3260 } else {
3261 for (reg_178 = reg178_center - 12;
3262 reg_178 <= reg178_center + 12; reg_178 += 12) {
3263 totalrank = 0;
3264 set_178(reg_178);
3265 FOR_POPULATED_RANKS_BACKWARDS {
3266 train_ram_at_178(info, channel, slot, rank,
3267 totalrank, reg_178, 0, niter,
3268 timings);
3269 totalrank++;
3270 }
3271 }
3272 }
3273
3274 set_178(reg178_center);
3275 FOR_POPULATED_RANKS_BACKWARDS for (lane = 0; lane < 8; lane++) {
3276 u16 tm0;
3277
3278 tm0 =
3279 choose_training(info, channel, slot, rank, lane, timings,
3280 reg178_center);
3281 write_500(info, channel, tm0,
3282 get_timing_register_addr(lane, 0, slot, rank), 9, 1);
3283 write_500(info, channel,
3284 tm0 +
3285 info->training.
3286 lane_timings[1][channel][slot][rank][lane] -
3287 info->training.
3288 lane_timings[0][channel][slot][rank][lane],
3289 get_timing_register_addr(lane, 1, slot, rank), 9, 1);
3290 }
3291
3292 totalrank = 0;
3293 FOR_POPULATED_RANKS_BACKWARDS {
3294 try_timing_offsets(info, channel, slot, rank, totalrank);
3295 totalrank++;
3296 }
3297 write_mchbar8(0x243, saved_243[0]);
3298 write_mchbar8(0x643, saved_243[1]);
3299 write_1d0(0, 0x142, 3, 1);
3300 info->training.reg178_center = reg178_center;
3301}
3302
3303static void ram_training(struct raminfo *info)
3304{
3305 u16 saved_fc4;
3306
3307 saved_fc4 = read_mchbar16(0xfc4);
3308 write_mchbar16(0xfc4, 0xffff);
3309
3310 if (info->revision >= 8)
3311 read_4090(info);
3312
3313 if (!try_cached_training(info))
3314 do_ram_training(info);
3315 if ((info->silicon_revision == 2 || info->silicon_revision == 3)
3316 && info->clock_speed_index < 2)
3317 set_10b(info, 1);
3318 write_mchbar16(0xfc4, saved_fc4);
3319}
3320
3321static unsigned gcd(unsigned a, unsigned b)
3322{
3323 unsigned t;
3324 if (a > b) {
3325 t = a;
3326 a = b;
3327 b = t;
3328 }
3329 /* invariant a < b. */
3330 while (a) {
3331 t = b % a;
3332 b = a;
3333 a = t;
3334 }
3335 return b;
3336}
3337
3338static inline int div_roundup(int a, int b)
3339{
Edward O'Callaghan7116ac82014-07-08 01:53:24 +1000[diff] [blame^]3340 return CEIL_DIV(a, b);
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3341}
3342
3343static unsigned lcm(unsigned a, unsigned b)
3344{
3345 return (a * b) / gcd(a, b);
3346}
3347
3348struct stru1 {
3349 u8 freqs_reversed;
3350 u8 freq_diff_reduced;
3351 u8 freq_min_reduced;
3352 u8 divisor_f4_to_fmax;
3353 u8 divisor_f3_to_fmax;
3354 u8 freq4_to_max_remainder;
3355 u8 freq3_to_2_remainder;
3356 u8 freq3_to_2_remaindera;
3357 u8 freq4_to_2_remainder;
3358 int divisor_f3_to_f1, divisor_f4_to_f2;
3359 int common_time_unit_ps;
3360 int freq_max_reduced;
3361};
3362
3363static void
3364compute_frequence_ratios(struct raminfo *info, u16 freq1, u16 freq2,
3365 int num_cycles_2, int num_cycles_1, int round_it,
3366 int add_freqs, struct stru1 *result)
3367{
3368 int g;
3369 int common_time_unit_ps;
3370 int freq1_reduced, freq2_reduced;
3371 int freq_min_reduced;
3372 int freq_max_reduced;
3373 int freq3, freq4;
3374
3375 g = gcd(freq1, freq2);
3376 freq1_reduced = freq1 / g;
3377 freq2_reduced = freq2 / g;
3378 freq_min_reduced = min(freq1_reduced, freq2_reduced);
3379 freq_max_reduced = max(freq1_reduced, freq2_reduced);
3380
3381 common_time_unit_ps = div_roundup(900000, lcm(freq1, freq2));
3382 freq3 = div_roundup(num_cycles_2, common_time_unit_ps) - 1;
3383 freq4 = div_roundup(num_cycles_1, common_time_unit_ps) - 1;
3384 if (add_freqs) {
3385 freq3 += freq2_reduced;
3386 freq4 += freq1_reduced;
3387 }
3388
3389 if (round_it) {
3390 result->freq3_to_2_remainder = 0;
3391 result->freq3_to_2_remaindera = 0;
3392 result->freq4_to_max_remainder = 0;
3393 result->divisor_f4_to_f2 = 0;
3394 result->divisor_f3_to_f1 = 0;
3395 } else {
3396 if (freq2_reduced < freq1_reduced) {
3397 result->freq3_to_2_remainder =
3398 result->freq3_to_2_remaindera =
3399 freq3 % freq1_reduced - freq1_reduced + 1;
3400 result->freq4_to_max_remainder =
3401 -(freq4 % freq1_reduced);
3402 result->divisor_f3_to_f1 = freq3 / freq1_reduced;
3403 result->divisor_f4_to_f2 =
3404 (freq4 -
3405 (freq1_reduced - freq2_reduced)) / freq2_reduced;
3406 result->freq4_to_2_remainder =
3407 -(char)((freq1_reduced - freq2_reduced) +
3408 ((u8) freq4 -
3409 (freq1_reduced -
3410 freq2_reduced)) % (u8) freq2_reduced);
3411 } else {
3412 if (freq2_reduced > freq1_reduced) {
3413 result->freq4_to_max_remainder =
3414 (freq4 % freq2_reduced) - freq2_reduced + 1;
3415 result->freq4_to_2_remainder =
3416 freq4 % freq_max_reduced -
3417 freq_max_reduced + 1;
3418 } else {
3419 result->freq4_to_max_remainder =
3420 -(freq4 % freq2_reduced);
3421 result->freq4_to_2_remainder =
3422 -(char)(freq4 % freq_max_reduced);
3423 }
3424 result->divisor_f4_to_f2 = freq4 / freq2_reduced;
3425 result->divisor_f3_to_f1 =
3426 (freq3 -
3427 (freq2_reduced - freq1_reduced)) / freq1_reduced;
3428 result->freq3_to_2_remainder = -(freq3 % freq2_reduced);
3429 result->freq3_to_2_remaindera =
3430 -(char)((freq_max_reduced - freq_min_reduced) +
3431 (freq3 -
3432 (freq_max_reduced -
3433 freq_min_reduced)) % freq1_reduced);
3434 }
3435 }
3436 result->divisor_f3_to_fmax = freq3 / freq_max_reduced;
3437 result->divisor_f4_to_fmax = freq4 / freq_max_reduced;
3438 if (round_it) {
3439 if (freq2_reduced > freq1_reduced) {
3440 if (freq3 % freq_max_reduced)
3441 result->divisor_f3_to_fmax++;
3442 }
3443 if (freq2_reduced < freq1_reduced) {
3444 if (freq4 % freq_max_reduced)
3445 result->divisor_f4_to_fmax++;
3446 }
3447 }
3448 result->freqs_reversed = (freq2_reduced < freq1_reduced);
3449 result->freq_diff_reduced = freq_max_reduced - freq_min_reduced;
3450 result->freq_min_reduced = freq_min_reduced;
3451 result->common_time_unit_ps = common_time_unit_ps;
3452 result->freq_max_reduced = freq_max_reduced;
3453}
3454
3455static void
3456set_2d5x_reg(struct raminfo *info, u16 reg, u16 freq1, u16 freq2,
3457 int num_cycles_2, int num_cycles_1, int num_cycles_3,
3458 int num_cycles_4, int reverse)
3459{
3460 struct stru1 vv;
3461 char multiplier;
3462
3463 compute_frequence_ratios(info, freq1, freq2, num_cycles_2, num_cycles_1,
3464 0, 1, &vv);
3465
3466 multiplier =
3467 div_roundup(max
3468 (div_roundup(num_cycles_2, vv.common_time_unit_ps) +
3469 div_roundup(num_cycles_3, vv.common_time_unit_ps),
3470 div_roundup(num_cycles_1,
3471 vv.common_time_unit_ps) +
3472 div_roundup(num_cycles_4, vv.common_time_unit_ps))
3473 + vv.freq_min_reduced - 1, vv.freq_max_reduced) - 1;
3474
3475 u32 y =
3476 (u8) ((vv.freq_max_reduced - vv.freq_min_reduced) +
3477 vv.freq_max_reduced * multiplier)
3478 | (vv.
3479 freqs_reversed << 8) | ((u8) (vv.freq_min_reduced *
3480 multiplier) << 16) | ((u8) (vv.
3481 freq_min_reduced
3482 *
3483 multiplier)
3484 << 24);
3485 u32 x =
3486 vv.freq3_to_2_remaindera | (vv.freq4_to_2_remainder << 8) | (vv.
3487 divisor_f3_to_f1
3488 << 16)
3489 | (vv.divisor_f4_to_f2 << 20) | (vv.freq_min_reduced << 24);
3490 if (reverse) {
3491 write_mchbar32(reg, y);
3492 write_mchbar32(reg + 4, x);
3493 } else {
3494 write_mchbar32(reg + 4, y);
3495 write_mchbar32(reg, x);
3496 }
3497}
3498
3499static void
3500set_6d_reg(struct raminfo *info, u16 reg, u16 freq1, u16 freq2,
3501 int num_cycles_1, int num_cycles_2, int num_cycles_3,
3502 int num_cycles_4)
3503{
3504 struct stru1 ratios1;
3505 struct stru1 ratios2;
3506
3507 compute_frequence_ratios(info, freq1, freq2, num_cycles_1, num_cycles_2,
3508 0, 1, &ratios2);
3509 compute_frequence_ratios(info, freq1, freq2, num_cycles_3, num_cycles_4,
3510 0, 1, &ratios1);
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]3511 printk (BIOS_SPEW, "[%x] <= %x\n", reg,
3512 ratios1.freq4_to_max_remainder | (ratios2.
3513 freq4_to_max_remainder
3514 << 8)
3515 | (ratios1.divisor_f4_to_fmax << 16) | (ratios2.
3516 divisor_f4_to_fmax
3517 << 20));
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3518 write_mchbar32(reg,
3519 ratios1.freq4_to_max_remainder | (ratios2.
3520 freq4_to_max_remainder
3521 << 8)
3522 | (ratios1.divisor_f4_to_fmax << 16) | (ratios2.
3523 divisor_f4_to_fmax
3524 << 20));
3525}
3526
3527static void
3528set_2dx8_reg(struct raminfo *info, u16 reg, u8 mode, u16 freq1, u16 freq2,
3529 int num_cycles_2, int num_cycles_1, int round_it, int add_freqs)
3530{
3531 struct stru1 ratios;
3532
3533 compute_frequence_ratios(info, freq1, freq2, num_cycles_2, num_cycles_1,
3534 round_it, add_freqs, &ratios);
3535 switch (mode) {
3536 case 0:
3537 write_mchbar32(reg + 4,
3538 ratios.freq_diff_reduced | (ratios.
3539 freqs_reversed <<
3540 8));
3541 write_mchbar32(reg,
3542 ratios.freq3_to_2_remainder | (ratios.
3543 freq4_to_max_remainder
3544 << 8)
3545 | (ratios.divisor_f3_to_fmax << 16) | (ratios.
3546 divisor_f4_to_fmax
3547 << 20) |
3548 (ratios.freq_min_reduced << 24));
3549 break;
3550
3551 case 1:
3552 write_mchbar32(reg,
3553 ratios.freq3_to_2_remainder | (ratios.
3554 divisor_f3_to_fmax
3555 << 16));
3556 break;
3557
3558 case 2:
3559 write_mchbar32(reg,
3560 ratios.freq3_to_2_remainder | (ratios.
3561 freq4_to_max_remainder
3562 << 8) | (ratios.
3563 divisor_f3_to_fmax
3564 << 16) |
3565 (ratios.divisor_f4_to_fmax << 20));
3566 break;
3567
3568 case 4:
3569 write_mchbar32(reg, (ratios.divisor_f3_to_fmax << 4)
3570 | (ratios.divisor_f4_to_fmax << 8) | (ratios.
3571 freqs_reversed
3572 << 12) |
3573 (ratios.freq_min_reduced << 16) | (ratios.
3574 freq_diff_reduced
3575 << 24));
3576 break;
3577 }
3578}
3579
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]3580static void set_2dxx_series(struct raminfo *info, int s3resume)
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3581{
3582 set_2dx8_reg(info, 0x2d00, 0, 0x78, frequency_11(info) / 2, 1359, 1005,
3583 0, 1);
3584 set_2dx8_reg(info, 0x2d08, 0, 0x78, 0x78, 3273, 5033, 1, 1);
3585 set_2dx8_reg(info, 0x2d10, 0, 0x78, info->fsb_frequency, 1475, 1131, 0,
3586 1);
3587 set_2dx8_reg(info, 0x2d18, 0, 2 * info->fsb_frequency,
3588 frequency_11(info), 1231, 1524, 0, 1);
3589 set_2dx8_reg(info, 0x2d20, 0, 2 * info->fsb_frequency,
3590 frequency_11(info) / 2, 1278, 2008, 0, 1);
3591 set_2dx8_reg(info, 0x2d28, 0, info->fsb_frequency, frequency_11(info),
3592 1167, 1539, 0, 1);
3593 set_2dx8_reg(info, 0x2d30, 0, info->fsb_frequency,
3594 frequency_11(info) / 2, 1403, 1318, 0, 1);
3595 set_2dx8_reg(info, 0x2d38, 0, info->fsb_frequency, 0x78, 3460, 5363, 1,
3596 1);
3597 set_2dx8_reg(info, 0x2d40, 0, info->fsb_frequency, 0x3c, 2792, 5178, 1,
3598 1);
3599 set_2dx8_reg(info, 0x2d48, 0, 2 * info->fsb_frequency, 0x78, 2738, 4610,
3600 1, 1);
3601 set_2dx8_reg(info, 0x2d50, 0, info->fsb_frequency, 0x78, 2819, 5932, 1,
3602 1);
3603 set_2dx8_reg(info, 0x6d4, 1, info->fsb_frequency,
3604 frequency_11(info) / 2, 4000, 0, 0, 0);
3605 set_2dx8_reg(info, 0x6d8, 2, info->fsb_frequency,
3606 frequency_11(info) / 2, 4000, 4000, 0, 0);
3607
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]3608 if (s3resume) {
3609 printk (BIOS_SPEW, "[6dc] <= %x\n", info->cached_training->reg_6dc);
3610 write_mchbar32(0x6dc, info->cached_training->reg_6dc);
3611 } else
3612 set_6d_reg(info, 0x6dc, 2 * info->fsb_frequency, frequency_11(info), 0,
3613 info->delay46_ps[0], 0,
3614 info->delay54_ps[0]);
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3615 set_2dx8_reg(info, 0x6e0, 1, 2 * info->fsb_frequency,
3616 frequency_11(info), 2500, 0, 0, 0);
3617 set_2dx8_reg(info, 0x6e4, 1, 2 * info->fsb_frequency,
3618 frequency_11(info) / 2, 3500, 0, 0, 0);
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]3619 if (s3resume) {
3620 printk (BIOS_SPEW, "[6e8] <= %x\n", info->cached_training->reg_6e8);
3621 write_mchbar32(0x6e8, info->cached_training->reg_6e8);
3622 } else
3623 set_6d_reg(info, 0x6e8, 2 * info->fsb_frequency, frequency_11(info), 0,
3624 info->delay46_ps[1], 0,
3625 info->delay54_ps[1]);
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3626 set_2d5x_reg(info, 0x2d58, 0x78, 0x78, 864, 1195, 762, 786, 0);
3627 set_2d5x_reg(info, 0x2d60, 0x195, info->fsb_frequency, 1352, 725, 455,
3628 470, 0);
3629 set_2d5x_reg(info, 0x2d68, 0x195, 0x3c, 2707, 5632, 3277, 2207, 0);
3630 set_2d5x_reg(info, 0x2d70, 0x195, frequency_11(info) / 2, 1276, 758,
3631 454, 459, 0);
3632 set_2d5x_reg(info, 0x2d78, 0x195, 0x78, 1021, 799, 510, 513, 0);
3633 set_2d5x_reg(info, 0x2d80, info->fsb_frequency, 0xe1, 0, 2862, 2579,
3634 2588, 0);
3635 set_2d5x_reg(info, 0x2d88, info->fsb_frequency, 0xe1, 0, 2690, 2405,
3636 2405, 0);
3637 set_2d5x_reg(info, 0x2da0, 0x78, 0xe1, 0, 2560, 2264, 2251, 0);
3638 set_2d5x_reg(info, 0x2da8, 0x195, frequency_11(info), 1060, 775, 484,
3639 480, 0);
3640 set_2d5x_reg(info, 0x2db0, 0x195, 0x78, 4183, 6023, 2217, 2048, 0);
3641 write_mchbar32(0x2dbc, ((frequency_11(info) / 2) - 1) | 0xe00000);
3642 write_mchbar32(0x2db8, ((info->fsb_frequency - 1) << 16) | 0x77);
3643}
3644
3645static u16 get_max_timing(struct raminfo *info, int channel)
3646{
3647 int slot, rank, lane;
3648 u16 ret = 0;
3649
3650 if ((read_mchbar8(0x2ca8) >> 2) < 1)
3651 return 384;
3652
3653 if (info->revision < 8)
3654 return 256;
3655
3656 for (slot = 0; slot < NUM_SLOTS; slot++)
3657 for (rank = 0; rank < NUM_RANKS; rank++)
3658 if (info->populated_ranks[channel][slot][rank])
3659 for (lane = 0; lane < 8 + info->use_ecc; lane++)
3660 ret = max(ret, read_500(info, channel,
3661 get_timing_register_addr
3662 (lane, 0, slot,
3663 rank), 9));
3664 return ret;
3665}
3666
3667static void set_274265(struct raminfo *info)
3668{
3669 int delay_a_ps, delay_b_ps, delay_c_ps, delay_d_ps;
3670 int delay_e_ps, delay_e_cycles, delay_f_cycles;
3671 int delay_e_over_cycle_ps;
3672 int cycletime_ps;
3673 int channel;
3674
3675 delay_a_ps = 4 * halfcycle_ps(info) + 6 * fsbcycle_ps(info);
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]3676 info->training.reg2ca9_bit0 = 0;
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3677 for (channel = 0; channel < NUM_CHANNELS; channel++) {
3678 cycletime_ps =
3679 900000 / lcm(2 * info->fsb_frequency, frequency_11(info));
3680 delay_d_ps =
3681 (halfcycle_ps(info) * get_max_timing(info, channel) >> 6)
3682 - info->some_delay_3_ps_rounded + 200;
3683 if (!
3684 ((info->silicon_revision == 0
3685 || info->silicon_revision == 1)
3686 && (info->revision >= 8)))
3687 delay_d_ps += halfcycle_ps(info) * 2;
3688 delay_d_ps +=
3689 halfcycle_ps(info) * (!info->revision_flag_1 +
3690 info->some_delay_2_halfcycles_ceil +
3691 2 * info->some_delay_1_cycle_floor +
3692 info->clock_speed_index +
3693 2 * info->cas_latency - 7 + 11);
3694 delay_d_ps += info->revision >= 8 ? 2758 : 4428;
3695
3696 write_mchbar32(0x140,
3697 (read_mchbar32(0x140) & 0xfaffffff) | 0x2000000);
3698 write_mchbar32(0x138,
3699 (read_mchbar32(0x138) & 0xfaffffff) | 0x2000000);
3700 if ((read_mchbar8(0x144) & 0x1f) > 0x13)
3701 delay_d_ps += 650;
3702 delay_c_ps = delay_d_ps + 1800;
3703 if (delay_c_ps <= delay_a_ps)
3704 delay_e_ps = 0;
3705 else
3706 delay_e_ps =
3707 cycletime_ps * div_roundup(delay_c_ps - delay_a_ps,
3708 cycletime_ps);
3709
3710 delay_e_over_cycle_ps = delay_e_ps % (2 * halfcycle_ps(info));
3711 delay_e_cycles = delay_e_ps / (2 * halfcycle_ps(info));
3712 delay_f_cycles =
3713 div_roundup(2500 - delay_e_over_cycle_ps,
3714 2 * halfcycle_ps(info));
3715 if (delay_f_cycles > delay_e_cycles) {
3716 info->delay46_ps[channel] = delay_e_ps;
3717 delay_e_cycles = 0;
3718 } else {
3719 info->delay46_ps[channel] =
3720 delay_e_over_cycle_ps +
3721 2 * halfcycle_ps(info) * delay_f_cycles;
3722 delay_e_cycles -= delay_f_cycles;
3723 }
3724
3725 if (info->delay46_ps[channel] < 2500) {
3726 info->delay46_ps[channel] = 2500;
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]3727 info->training.reg2ca9_bit0 = 1;
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3728 }
3729 delay_b_ps = halfcycle_ps(info) + delay_c_ps;
3730 if (delay_b_ps <= delay_a_ps)
3731 delay_b_ps = 0;
3732 else
3733 delay_b_ps -= delay_a_ps;
3734 info->delay54_ps[channel] =
3735 cycletime_ps * div_roundup(delay_b_ps,
3736 cycletime_ps) -
3737 2 * halfcycle_ps(info) * delay_e_cycles;
3738 if (info->delay54_ps[channel] < 2500)
3739 info->delay54_ps[channel] = 2500;
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]3740 info->training.reg274265[channel][0] = delay_e_cycles;
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3741 if (delay_d_ps + 7 * halfcycle_ps(info) <=
3742 24 * halfcycle_ps(info))
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]3743 info->training.reg274265[channel][1] = 0;
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3744 else
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]3745 info->training.reg274265[channel][1] =
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3746 div_roundup(delay_d_ps + 7 * halfcycle_ps(info),
3747 4 * halfcycle_ps(info)) - 6;
3748 write_mchbar32((channel << 10) + 0x274,
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]3749 info->training.reg274265[channel][1]
3750 | (info->training.reg274265[channel][0] << 16));
3751 info->training.reg274265[channel][2] =
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3752 div_roundup(delay_c_ps + 3 * fsbcycle_ps(info),
3753 4 * halfcycle_ps(info)) + 1;
3754 write_mchbar16((channel << 10) + 0x265,
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]3755 info->training.reg274265[channel][2] << 8);
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3756 }
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]3757 if (info->training.reg2ca9_bit0)
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3758 write_mchbar8(0x2ca9, read_mchbar8(0x2ca9) | 1);
3759 else
3760 write_mchbar8(0x2ca9, read_mchbar8(0x2ca9) & ~1);
3761}
3762
3763static void restore_274265(struct raminfo *info)
3764{
3765 int channel;
3766
3767 for (channel = 0; channel < NUM_CHANNELS; channel++) {
3768 write_mchbar32((channel << 10) + 0x274,
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]3769 (info->cached_training->reg274265[channel][0] << 16)
3770 | info->cached_training->reg274265[channel][1]);
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3771 write_mchbar16((channel << 10) + 0x265,
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]3772 info->cached_training->reg274265[channel][2] << 8);
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3773 }
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]3774 if (info->cached_training->reg2ca9_bit0)
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3775 write_mchbar8(0x2ca9, read_mchbar8(0x2ca9) | 1);
3776 else
3777 write_mchbar8(0x2ca9, read_mchbar8(0x2ca9) & ~1);
3778}
3779
3780#if REAL
3781static void dmi_setup(void)
3782{
3783 gav(read8(DEFAULT_DMIBAR | 0x254));
3784 write8(DEFAULT_DMIBAR | 0x254, 0x1);
3785 write16(DEFAULT_DMIBAR | 0x1b8, 0x18f2);
3786 read_mchbar16(0x48);
3787 write_mchbar16(0x48, 0x2);
3788
3789 write32(DEFAULT_DMIBAR | 0xd68, read32(DEFAULT_DMIBAR | 0xd68) | 0x08000000);
3790
3791 outl((gav(inl(DEFAULT_GPIOBASE | 0x38)) & ~0x140000) | 0x400000,
3792 DEFAULT_GPIOBASE | 0x38);
3793 gav(inb(DEFAULT_GPIOBASE | 0xe)); // = 0xfdcaff6e
3794}
3795#endif
3796
Vladimir Serbinenko9817a372014-02-19 22:07:12 +0100[diff] [blame]3797void chipset_init(const int s3resume)
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3798{
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3799 u8 x2ca8;
3800
Vladimir Serbinenko9817a372014-02-19 22:07:12 +0100[diff] [blame]3801 x2ca8 = read_mchbar8(0x2ca8);
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3802 if ((x2ca8 & 1) || (x2ca8 == 8 && !s3resume)) {
3803 printk(BIOS_DEBUG, "soft reset detected, rebooting properly\n");
3804 write_mchbar8(0x2ca8, 0);
Vladimir Serbinenkoe1eef692014-02-19 22:08:51 +0100[diff] [blame]3805 outb(0x6, 0xcf9);
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3806#if REAL
3807 while (1) {
3808 asm volatile ("hlt");
3809 }
3810#else
3811 printf("CP5\n");
3812 exit(0);
3813#endif
3814 }
3815#if !REAL
3816 if (!s3resume) {
3817 pre_raminit_3(x2ca8);
3818 }
Vladimir Serbinenkof62669c2014-01-09 10:59:38 +0100[diff] [blame]3819 pre_raminit_4a(x2ca8);
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3820#endif
3821
3822 dmi_setup();
3823
3824 write_mchbar16(0x1170, 0xa880);
3825 write_mchbar8(0x11c1, 0x1);
3826 write_mchbar16(0x1170, 0xb880);
3827 read_mchbar8(0x1210);
3828 write_mchbar8(0x1210, 0x84);
3829 pcie_read_config8(NORTHBRIDGE, D0F0_GGC); // = 0x52
3830 pcie_write_config8(NORTHBRIDGE, D0F0_GGC, 0x2);
3831 pcie_read_config8(NORTHBRIDGE, D0F0_GGC); // = 0x2
3832 pcie_write_config8(NORTHBRIDGE, D0F0_GGC, 0x52);
3833 pcie_read_config16(NORTHBRIDGE, D0F0_GGC); // = 0xb52
3834
3835 pcie_write_config16(NORTHBRIDGE, D0F0_GGC, 0xb52);
3836
3837 u16 deven;
3838 deven = pcie_read_config16(NORTHBRIDGE, D0F0_DEVEN); // = 0x3
3839
3840 if (deven & 8) {
3841 write_mchbar8(0x2c30, 0x20);
3842 pcie_read_config8(NORTHBRIDGE, 0x8); // = 0x18
3843 write_mchbar16(0x2c30, read_mchbar16(0x2c30) | 0x200);
3844 write_mchbar16(0x2c32, 0x434);
3845 read_mchbar32(0x2c44);
3846 write_mchbar32(0x2c44, 0x1053687);
3847 pcie_read_config8(GMA, 0x62); // = 0x2
3848 pcie_write_config8(GMA, 0x62, 0x2);
3849 read8(DEFAULT_RCBA | 0x2318);
3850 write8(DEFAULT_RCBA | 0x2318, 0x47);
3851 read8(DEFAULT_RCBA | 0x2320);
3852 write8(DEFAULT_RCBA | 0x2320, 0xfc);
3853 }
3854
3855 read_mchbar32(0x30);
3856 write_mchbar32(0x30, 0x40);
3857
3858 pcie_read_config8(SOUTHBRIDGE, 0x8); // = 0x6
3859 pcie_read_config16(NORTHBRIDGE, D0F0_GGC); // = 0xb52
3860 pcie_write_config16(NORTHBRIDGE, D0F0_GGC, 0xb50);
3861 gav(read32(DEFAULT_RCBA | 0x3428));
3862 write32(DEFAULT_RCBA | 0x3428, 0x1d);
Vladimir Serbinenko9817a372014-02-19 22:07:12 +0100[diff] [blame]3863}
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3864
Vladimir Serbinenko9817a372014-02-19 22:07:12 +0100[diff] [blame]3865void raminit(const int s3resume, const u8 *spd_addrmap)
3866{
3867 unsigned channel, slot, lane, rank;
3868 int i;
3869 struct raminfo info;
3870 u8 x2ca8;
3871 u16 deven;
3872
3873 x2ca8 = read_mchbar8(0x2ca8);
3874 deven = pcie_read_config16(NORTHBRIDGE, D0F0_DEVEN);
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3875
3876 memset(&info, 0x5a, sizeof(info));
3877
3878 info.last_500_command[0] = 0;
3879 info.last_500_command[1] = 0;
3880
3881 info.fsb_frequency = 135 * 2;
3882 info.board_lane_delay[0] = 0x14;
3883 info.board_lane_delay[1] = 0x07;
3884 info.board_lane_delay[2] = 0x07;
3885 info.board_lane_delay[3] = 0x08;
3886 info.board_lane_delay[4] = 0x56;
3887 info.board_lane_delay[5] = 0x04;
3888 info.board_lane_delay[6] = 0x04;
3889 info.board_lane_delay[7] = 0x05;
3890 info.board_lane_delay[8] = 0x10;
3891
3892 info.training.reg_178 = 0;
3893 info.training.reg_10b = 0;
3894
3895 info.heci_bar = 0;
3896 info.memory_reserved_for_heci_mb = 0;
3897
3898 /* before SPD */
3899 timestamp_add_now(101);
3900
3901 if (!s3resume || REAL) {
3902 pcie_read_config8(SOUTHBRIDGE, GEN_PMCON_2); // = 0x80
3903
3904 collect_system_info(&info);
3905
3906#if REAL
3907 /* Enable SMBUS. */
3908 enable_smbus();
3909#endif
3910
3911 memset(&info.populated_ranks, 0, sizeof(info.populated_ranks));
3912
3913 info.use_ecc = 1;
3914 for (channel = 0; channel < NUM_CHANNELS; channel++)
Vladimir Serbinenko2ab8ec72014-02-20 14:34:56 +0100[diff] [blame]3915 for (slot = 0; slot < NUM_SLOTS; slot++) {
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3916 int v;
3917 int try;
3918 int addr;
3919 const u8 useful_addresses[] = {
3920 DEVICE_TYPE,
3921 MODULE_TYPE,
3922 DENSITY,
3923 RANKS_AND_DQ,
3924 MEMORY_BUS_WIDTH,
3925 TIMEBASE_DIVIDEND,
3926 TIMEBASE_DIVISOR,
3927 CYCLETIME,
3928 CAS_LATENCIES_LSB,
3929 CAS_LATENCIES_MSB,
3930 CAS_LATENCY_TIME,
3931 0x11, 0x12, 0x13, 0x14, 0x15,
3932 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b,
3933 0x1c, 0x1d,
3934 THERMAL_AND_REFRESH,
3935 0x20,
3936 REFERENCE_RAW_CARD_USED,
3937 RANK1_ADDRESS_MAPPING,
3938 0x75, 0x76, 0x77, 0x78,
3939 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e,
3940 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84,
3941 0x85, 0x86, 0x87, 0x88,
3942 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e,
3943 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94,
3944 0x95
3945 };
Vladimir Serbinenko902626c2014-02-16 17:22:26 +0100[diff] [blame]3946 if (!spd_addrmap[2 * channel + slot])
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3947 continue;
3948 for (try = 0; try < 5; try++) {
Vladimir Serbinenko902626c2014-02-16 17:22:26 +0100[diff] [blame]3949 v = smbus_read_byte(spd_addrmap[2 * channel + slot],
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3950 DEVICE_TYPE);
3951 if (v >= 0)
3952 break;
3953 }
3954 if (v < 0)
3955 continue;
3956 for (addr = 0;
3957 addr <
3958 sizeof(useful_addresses) /
3959 sizeof(useful_addresses[0]); addr++)
3960 gav(info.
3961 spd[channel][0][useful_addresses
3962 [addr]] =
Vladimir Serbinenko902626c2014-02-16 17:22:26 +0100[diff] [blame]3963 smbus_read_byte(spd_addrmap[2 * channel + slot],
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]3964 useful_addresses
3965 [addr]));
3966 if (info.spd[channel][0][DEVICE_TYPE] != 11)
3967 die("Only DDR3 is supported");
3968
3969 v = info.spd[channel][0][RANKS_AND_DQ];
3970 info.populated_ranks[channel][0][0] = 1;
3971 info.populated_ranks[channel][0][1] =
3972 ((v >> 3) & 7);
3973 if (((v >> 3) & 7) > 1)
3974 die("At most 2 ranks are supported");
3975 if ((v & 7) == 0 || (v & 7) > 2)
3976 die("Only x8 and x16 modules are supported");
3977 if ((info.
3978 spd[channel][slot][MODULE_TYPE] & 0xF) != 2
3979 && (info.
3980 spd[channel][slot][MODULE_TYPE] & 0xF)
3981 != 3)
3982 die("Registered memory is not supported");
3983 info.is_x16_module[channel][0] = (v & 7) - 1;
3984 info.density[channel][slot] =
3985 info.spd[channel][slot][DENSITY] & 0xF;
3986 if (!
3987 (info.
3988 spd[channel][slot][MEMORY_BUS_WIDTH] &
3989 0x18))
3990 info.use_ecc = 0;
3991 }
3992
3993 gav(0x55);
3994
3995 for (channel = 0; channel < NUM_CHANNELS; channel++) {
3996 int v = 0;
3997 for (slot = 0; slot < NUM_SLOTS; slot++)
3998 for (rank = 0; rank < NUM_RANKS; rank++)
3999 v |= info.
4000 populated_ranks[channel][slot][rank]
4001 << (2 * slot + rank);
4002 info.populated_ranks_mask[channel] = v;
4003 }
4004
4005 gav(0x55);
4006
4007 gav(pcie_read_config32(NORTHBRIDGE, D0F0_CAPID0 + 4));
4008 }
4009
4010 /* after SPD */
4011 timestamp_add_now(102);
4012
4013 write_mchbar8(0x2ca8, read_mchbar8(0x2ca8) & 0xfc);
4014#if !REAL
4015 rdmsr (MTRRphysMask_MSR (3));
4016#endif
4017
4018 collect_system_info(&info);
4019 calculate_timings(&info);
4020
4021#if !REAL
4022 pcie_write_config8(NORTHBRIDGE, 0xdf, 0x82);
4023#endif
4024
4025 if (!s3resume) {
4026 u8 reg8 = pcie_read_config8(SOUTHBRIDGE, GEN_PMCON_2);
4027 if (x2ca8 == 0 && (reg8 & 0x80)) {
4028 /* Don't enable S4-assertion stretch. Makes trouble on roda/rk9.
4029 reg8 = pci_read_config8(PCI_DEV(0, 0x1f, 0), 0xa4);
4030 pci_write_config8(PCI_DEV(0, 0x1f, 0), 0xa4, reg8 | 0x08);
4031 */
4032
4033 /* Clear bit7. */
4034
4035 pci_write_config8(SOUTHBRIDGE, GEN_PMCON_2,
4036 (reg8 & ~(1 << 7)));
4037
4038 printk(BIOS_INFO,
4039 "Interrupted RAM init, reset required.\n");
4040 outb(0x6, 0xcf9);
4041#if REAL
4042 while (1) {
4043 asm volatile ("hlt");
4044 }
4045#endif
4046 }
4047 }
4048#if !REAL
4049 gav(read_mchbar8(0x2ca8)); ///!!!!
4050#endif
4051
4052 if (!s3resume && x2ca8 == 0)
4053 pcie_write_config8(SOUTHBRIDGE, GEN_PMCON_2,
4054 pcie_read_config8(SOUTHBRIDGE, GEN_PMCON_2) | 0x80);
4055
4056 compute_derived_timings(&info);
4057
4058 if (x2ca8 == 0) {
4059 gav(read_mchbar8(0x164));
4060 write_mchbar8(0x164, 0x26);
4061 write_mchbar16(0x2c20, 0x10);
4062 }
4063
4064 write_mchbar32(0x18b4, read_mchbar32(0x18b4) | 0x210000); /* OK */
4065 write_mchbar32(0x1890, read_mchbar32(0x1890) | 0x2000000); /* OK */
4066 write_mchbar32(0x18b4, read_mchbar32(0x18b4) | 0x8000);
4067
4068 gav(pcie_read_config32(PCI_DEV(0xff, 2, 1), 0x50)); // !!!!
4069 pcie_write_config8(PCI_DEV(0xff, 2, 1), 0x54, 0x12);
4070
4071 gav(read_mchbar16(0x2c10)); // !!!!
4072 write_mchbar16(0x2c10, 0x412);
4073 gav(read_mchbar16(0x2c10)); // !!!!
4074 write_mchbar16(0x2c12, read_mchbar16(0x2c12) | 0x100); /* OK */
4075
4076 gav(read_mchbar8(0x2ca8)); // !!!!
4077 write_mchbar32(0x1804,
4078 (read_mchbar32(0x1804) & 0xfffffffc) | 0x8400080);
4079
4080 pcie_read_config32(PCI_DEV(0xff, 2, 1), 0x6c); // !!!!
4081 pcie_write_config32(PCI_DEV(0xff, 2, 1), 0x6c, 0x40a0a0);
4082 gav(read_mchbar32(0x1c04)); // !!!!
4083 gav(read_mchbar32(0x1804)); // !!!!
4084
4085 if (x2ca8 == 0) {
4086 write_mchbar8(0x2ca8, read_mchbar8(0x2ca8) | 1);
4087 }
4088
4089 write_mchbar32(0x18d8, 0x120000);
4090 write_mchbar32(0x18dc, 0x30a484a);
4091 pcie_write_config32(PCI_DEV(0xff, 2, 1), 0xe0, 0x0);
4092 pcie_write_config32(PCI_DEV(0xff, 2, 1), 0xf4, 0x9444a);
4093 write_mchbar32(0x18d8, 0x40000);
4094 write_mchbar32(0x18dc, 0xb000000);
4095 pcie_write_config32(PCI_DEV(0xff, 2, 1), 0xe0, 0x60000);
4096 pcie_write_config32(PCI_DEV(0xff, 2, 1), 0xf4, 0x0);
4097 write_mchbar32(0x18d8, 0x180000);
4098 write_mchbar32(0x18dc, 0xc0000142);
4099 pcie_write_config32(PCI_DEV(0xff, 2, 1), 0xe0, 0x20000);
4100 pcie_write_config32(PCI_DEV(0xff, 2, 1), 0xf4, 0x142);
4101 write_mchbar32(0x18d8, 0x1e0000);
4102
4103 gav(read_mchbar32(0x18dc)); // !!!!
4104 write_mchbar32(0x18dc, 0x3);
4105 gav(read_mchbar32(0x18dc)); // !!!!
4106
4107 if (x2ca8 == 0) {
4108 write_mchbar8(0x2ca8, read_mchbar8(0x2ca8) | 1); // guess
4109 }
4110
4111 write_mchbar32(0x188c, 0x20bc09);
4112 pcie_write_config32(PCI_DEV(0xff, 2, 1), 0xd0, 0x40b0c09);
4113 write_mchbar32(0x1a10, 0x4200010e);
4114 write_mchbar32(0x18b8, read_mchbar32(0x18b8) | 0x200);
4115 gav(read_mchbar32(0x1918)); // !!!!
4116 write_mchbar32(0x1918, 0x332);
4117
4118 gav(read_mchbar32(0x18b8)); // !!!!
4119 write_mchbar32(0x18b8, 0xe00);
4120 gav(read_mchbar32(0x182c)); // !!!!
4121 write_mchbar32(0x182c, 0x10202);
4122 gav(pcie_read_config32(PCI_DEV(0xff, 2, 1), 0x94)); // !!!!
4123 pcie_write_config32(PCI_DEV(0xff, 2, 1), 0x94, 0x10202);
4124 write_mchbar32(0x1a1c, read_mchbar32(0x1a1c) & 0x8fffffff);
4125 write_mchbar32(0x1a70, read_mchbar32(0x1a70) | 0x100000);
4126
4127 write_mchbar32(0x18b4, read_mchbar32(0x18b4) & 0xffff7fff);
4128 gav(read_mchbar32(0x1a68)); // !!!!
4129 write_mchbar32(0x1a68, 0x343800);
4130 gav(read_mchbar32(0x1e68)); // !!!!
4131 gav(read_mchbar32(0x1a68)); // !!!!
4132
4133 if (x2ca8 == 0) {
4134 write_mchbar8(0x2ca8, read_mchbar8(0x2ca8) | 1); // guess
4135 }
4136
4137 pcie_read_config32(PCI_DEV(0xff, 2, 0), 0x048); // !!!!
4138 pcie_write_config32(PCI_DEV(0xff, 2, 0), 0x048, 0x140000);
4139 pcie_read_config32(PCI_DEV(0xff, 2, 0), 0x058); // !!!!
4140 pcie_write_config32(PCI_DEV(0xff, 2, 0), 0x058, 0x64555);
4141 pcie_read_config32(PCI_DEV(0xff, 2, 0), 0x058); // !!!!
4142 pcie_read_config32(PCI_DEV (0xff, 0, 0), 0xd0); // !!!!
4143 pcie_write_config32(PCI_DEV (0xff, 0, 0), 0xd0, 0x180);
4144 gav(read_mchbar32(0x1af0)); // !!!!
4145 gav(read_mchbar32(0x1af0)); // !!!!
4146 write_mchbar32(0x1af0, 0x1f020003);
4147 gav(read_mchbar32(0x1af0)); // !!!!
4148
Edward O'Callaghan42b716f2014-06-26 21:38:52 +1000[diff] [blame]4149 if (x2ca8 == 0) {
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]4150 write_mchbar8(0x2ca8, read_mchbar8(0x2ca8) | 1); // guess
4151 }
4152
4153 gav(read_mchbar32(0x1890)); // !!!!
4154 write_mchbar32(0x1890, 0x80102);
4155 gav(read_mchbar32(0x18b4)); // !!!!
4156 write_mchbar32(0x18b4, 0x216000);
4157 write_mchbar32(0x18a4, 0x22222222);
4158 write_mchbar32(0x18a8, 0x22222222);
4159 write_mchbar32(0x18ac, 0x22222);
4160
4161 udelay(1000);
4162
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]4163 info.cached_training = get_cached_training();
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]4164
Vladimir Serbinenkof7a42de2014-01-09 11:10:04 +0100[diff] [blame]4165 if (x2ca8 == 0) {
4166 int j;
4167 if (s3resume && info.cached_training) {
4168 restore_274265(&info);
4169 printk(BIOS_DEBUG, "reg2ca9_bit0 = %x\n",
4170 info.cached_training->reg2ca9_bit0);
4171 for (i = 0; i < 2; i++)
4172 for (j = 0; j < 3; j++)
4173 printk(BIOS_DEBUG, "reg274265[%d][%d] = %x\n",
4174 i, j, info.cached_training->reg274265[i][j]);
4175 } else {
4176 set_274265(&info);
4177 printk(BIOS_DEBUG, "reg2ca9_bit0 = %x\n",
4178 info.training.reg2ca9_bit0);
4179 for (i = 0; i < 2; i++)
4180 for (j = 0; j < 3; j++)
4181 printk(BIOS_DEBUG, "reg274265[%d][%d] = %x\n",
4182 i, j, info.training.reg274265[i][j]);
4183 }
4184
4185 set_2dxx_series(&info, s3resume);
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]4186
4187 if (!(deven & 8)) {
4188 read_mchbar32(0x2cb0);
4189 write_mchbar32(0x2cb0, 0x40);
4190 }
4191
4192 udelay(1000);
4193
4194 if (deven & 8) {
4195 write_mchbar32(0xff8, 0x1800 | read_mchbar32(0xff8));
4196 read_mchbar32(0x2cb0);
4197 write_mchbar32(0x2cb0, 0x00);
4198 pcie_read_config8(PCI_DEV (0, 0x2, 0x0), 0x4c);
4199 pcie_read_config8(PCI_DEV (0, 0x2, 0x0), 0x4c);
4200 pcie_read_config8(PCI_DEV (0, 0x2, 0x0), 0x4e);
4201
4202 read_mchbar8(0x1150);
4203 read_mchbar8(0x1151);
4204 read_mchbar8(0x1022);
4205 read_mchbar8(0x16d0);
4206 write_mchbar32(0x1300, 0x60606060);
4207 write_mchbar32(0x1304, 0x60606060);
4208 write_mchbar32(0x1308, 0x78797a7b);
4209 write_mchbar32(0x130c, 0x7c7d7e7f);
4210 write_mchbar32(0x1310, 0x60606060);
4211 write_mchbar32(0x1314, 0x60606060);
4212 write_mchbar32(0x1318, 0x60606060);
4213 write_mchbar32(0x131c, 0x60606060);
4214 write_mchbar32(0x1320, 0x50515253);
4215 write_mchbar32(0x1324, 0x54555657);
4216 write_mchbar32(0x1328, 0x58595a5b);
4217 write_mchbar32(0x132c, 0x5c5d5e5f);
4218 write_mchbar32(0x1330, 0x40414243);
4219 write_mchbar32(0x1334, 0x44454647);
4220 write_mchbar32(0x1338, 0x48494a4b);
4221 write_mchbar32(0x133c, 0x4c4d4e4f);
4222 write_mchbar32(0x1340, 0x30313233);
4223 write_mchbar32(0x1344, 0x34353637);
4224 write_mchbar32(0x1348, 0x38393a3b);
4225 write_mchbar32(0x134c, 0x3c3d3e3f);
4226 write_mchbar32(0x1350, 0x20212223);
4227 write_mchbar32(0x1354, 0x24252627);
4228 write_mchbar32(0x1358, 0x28292a2b);
4229 write_mchbar32(0x135c, 0x2c2d2e2f);
4230 write_mchbar32(0x1360, 0x10111213);
4231 write_mchbar32(0x1364, 0x14151617);
4232 write_mchbar32(0x1368, 0x18191a1b);
4233 write_mchbar32(0x136c, 0x1c1d1e1f);
4234 write_mchbar32(0x1370, 0x10203);
4235 write_mchbar32(0x1374, 0x4050607);
4236 write_mchbar32(0x1378, 0x8090a0b);
4237 write_mchbar32(0x137c, 0xc0d0e0f);
4238 write_mchbar8(0x11cc, 0x4e);
4239 write_mchbar32(0x1110, 0x73970404);
4240 write_mchbar32(0x1114, 0x72960404);
4241 write_mchbar32(0x1118, 0x6f950404);
4242 write_mchbar32(0x111c, 0x6d940404);
4243 write_mchbar32(0x1120, 0x6a930404);
4244 write_mchbar32(0x1124, 0x68a41404);
4245 write_mchbar32(0x1128, 0x66a21404);
4246 write_mchbar32(0x112c, 0x63a01404);
4247 write_mchbar32(0x1130, 0x609e1404);
4248 write_mchbar32(0x1134, 0x5f9c1404);
4249 write_mchbar32(0x1138, 0x5c961404);
4250 write_mchbar32(0x113c, 0x58a02404);
4251 write_mchbar32(0x1140, 0x54942404);
4252 write_mchbar32(0x1190, 0x900080a);
4253 write_mchbar16(0x11c0, 0xc40b);
4254 write_mchbar16(0x11c2, 0x303);
4255 write_mchbar16(0x11c4, 0x301);
4256 read_mchbar32(0x1190);
4257 write_mchbar32(0x1190, 0x8900080a);
4258 write_mchbar32(0x11b8, 0x70c3000);
4259 write_mchbar8(0x11ec, 0xa);
4260 write_mchbar16(0x1100, 0x800);
4261 read_mchbar32(0x11bc);
4262 write_mchbar32(0x11bc, 0x1e84800);
4263 write_mchbar16(0x11ca, 0xfa);
4264 write_mchbar32(0x11e4, 0x4e20);
4265 write_mchbar8(0x11bc, 0xf);
4266 write_mchbar16(0x11da, 0x19);
4267 write_mchbar16(0x11ba, 0x470c);
4268 write_mchbar32(0x1680, 0xe6ffe4ff);
4269 write_mchbar32(0x1684, 0xdeffdaff);
4270 write_mchbar32(0x1688, 0xd4ffd0ff);
4271 write_mchbar32(0x168c, 0xccffc6ff);
4272 write_mchbar32(0x1690, 0xc0ffbeff);
4273 write_mchbar32(0x1694, 0xb8ffb0ff);
4274 write_mchbar32(0x1698, 0xa8ff0000);
4275 write_mchbar32(0x169c, 0xc00);
4276 write_mchbar32(0x1290, 0x5000000);
4277 }
4278
4279 write_mchbar32(0x124c, 0x15040d00);
4280 write_mchbar32(0x1250, 0x7f0000);
4281 write_mchbar32(0x1254, 0x1e220004);
4282 write_mchbar32(0x1258, 0x4000004);
4283 write_mchbar32(0x1278, 0x0);
4284 write_mchbar32(0x125c, 0x0);
4285 write_mchbar32(0x1260, 0x0);
4286 write_mchbar32(0x1264, 0x0);
4287 write_mchbar32(0x1268, 0x0);
4288 write_mchbar32(0x126c, 0x0);
4289 write_mchbar32(0x1270, 0x0);
4290 write_mchbar32(0x1274, 0x0);
4291 }
4292
4293 if ((deven & 8) && x2ca8 == 0) {
4294 write_mchbar16(0x1214, 0x320);
4295 write_mchbar32(0x1600, 0x40000000);
4296 read_mchbar32(0x11f4);
4297 write_mchbar32(0x11f4, 0x10000000);
4298 read_mchbar16(0x1230);
4299 write_mchbar16(0x1230, 0x8000);
4300 write_mchbar32(0x1400, 0x13040020);
4301 write_mchbar32(0x1404, 0xe090120);
4302 write_mchbar32(0x1408, 0x5120220);
4303 write_mchbar32(0x140c, 0x5120330);
4304 write_mchbar32(0x1410, 0xe090220);
4305 write_mchbar32(0x1414, 0x1010001);
4306 write_mchbar32(0x1418, 0x1110000);
4307 write_mchbar32(0x141c, 0x9020020);
4308 write_mchbar32(0x1420, 0xd090220);
4309 write_mchbar32(0x1424, 0x2090220);
4310 write_mchbar32(0x1428, 0x2090330);
4311 write_mchbar32(0x142c, 0xd090220);
4312 write_mchbar32(0x1430, 0x1010001);
4313 write_mchbar32(0x1434, 0x1110000);
4314 write_mchbar32(0x1438, 0x11040020);
4315 write_mchbar32(0x143c, 0x4030220);
4316 write_mchbar32(0x1440, 0x1060220);
4317 write_mchbar32(0x1444, 0x1060330);
4318 write_mchbar32(0x1448, 0x4030220);
4319 write_mchbar32(0x144c, 0x1010001);
4320 write_mchbar32(0x1450, 0x1110000);
4321 write_mchbar32(0x1454, 0x4010020);
4322 write_mchbar32(0x1458, 0xb090220);
4323 write_mchbar32(0x145c, 0x1090220);
4324 write_mchbar32(0x1460, 0x1090330);
4325 write_mchbar32(0x1464, 0xb090220);
4326 write_mchbar32(0x1468, 0x1010001);
4327 write_mchbar32(0x146c, 0x1110000);
4328 write_mchbar32(0x1470, 0xf040020);
4329 write_mchbar32(0x1474, 0xa090220);
4330 write_mchbar32(0x1478, 0x1120220);
4331 write_mchbar32(0x147c, 0x1120330);
4332 write_mchbar32(0x1480, 0xa090220);
4333 write_mchbar32(0x1484, 0x1010001);
4334 write_mchbar32(0x1488, 0x1110000);
4335 write_mchbar32(0x148c, 0x7020020);
4336 write_mchbar32(0x1490, 0x1010220);
4337 write_mchbar32(0x1494, 0x10210);
4338 write_mchbar32(0x1498, 0x10320);
4339 write_mchbar32(0x149c, 0x1010220);
4340 write_mchbar32(0x14a0, 0x1010001);
4341 write_mchbar32(0x14a4, 0x1110000);
4342 write_mchbar32(0x14a8, 0xd040020);
4343 write_mchbar32(0x14ac, 0x8090220);
4344 write_mchbar32(0x14b0, 0x1111310);
4345 write_mchbar32(0x14b4, 0x1111420);
4346 write_mchbar32(0x14b8, 0x8090220);
4347 write_mchbar32(0x14bc, 0x1010001);
4348 write_mchbar32(0x14c0, 0x1110000);
4349 write_mchbar32(0x14c4, 0x3010020);
4350 write_mchbar32(0x14c8, 0x7090220);
4351 write_mchbar32(0x14cc, 0x1081310);
4352 write_mchbar32(0x14d0, 0x1081420);
4353 write_mchbar32(0x14d4, 0x7090220);
4354 write_mchbar32(0x14d8, 0x1010001);
4355 write_mchbar32(0x14dc, 0x1110000);
4356 write_mchbar32(0x14e0, 0xb040020);
4357 write_mchbar32(0x14e4, 0x2030220);
4358 write_mchbar32(0x14e8, 0x1051310);
4359 write_mchbar32(0x14ec, 0x1051420);
4360 write_mchbar32(0x14f0, 0x2030220);
4361 write_mchbar32(0x14f4, 0x1010001);
4362 write_mchbar32(0x14f8, 0x1110000);
4363 write_mchbar32(0x14fc, 0x5020020);
4364 write_mchbar32(0x1500, 0x5090220);
4365 write_mchbar32(0x1504, 0x2071310);
4366 write_mchbar32(0x1508, 0x2071420);
4367 write_mchbar32(0x150c, 0x5090220);
4368 write_mchbar32(0x1510, 0x1010001);
4369 write_mchbar32(0x1514, 0x1110000);
4370 write_mchbar32(0x1518, 0x7040120);
4371 write_mchbar32(0x151c, 0x2090220);
4372 write_mchbar32(0x1520, 0x70b1210);
4373 write_mchbar32(0x1524, 0x70b1310);
4374 write_mchbar32(0x1528, 0x2090220);
4375 write_mchbar32(0x152c, 0x1010001);
4376 write_mchbar32(0x1530, 0x1110000);
4377 write_mchbar32(0x1534, 0x1010110);
4378 write_mchbar32(0x1538, 0x1081310);
4379 write_mchbar32(0x153c, 0x5041200);
4380 write_mchbar32(0x1540, 0x5041310);
4381 write_mchbar32(0x1544, 0x1081310);
4382 write_mchbar32(0x1548, 0x1010001);
4383 write_mchbar32(0x154c, 0x1110000);
4384 write_mchbar32(0x1550, 0x1040120);
4385 write_mchbar32(0x1554, 0x4051210);
4386 write_mchbar32(0x1558, 0xd051200);
4387 write_mchbar32(0x155c, 0xd051200);
4388 write_mchbar32(0x1560, 0x4051210);
4389 write_mchbar32(0x1564, 0x1010001);
4390 write_mchbar32(0x1568, 0x1110000);
4391 write_mchbar16(0x1222, 0x220a);
4392 write_mchbar16(0x123c, 0x1fc0);
4393 write_mchbar16(0x1220, 0x1388);
4394 }
4395
4396 read_mchbar32(0x2c80); // !!!!
4397 write_mchbar32(0x2c80, 0x1053688);
4398 read_mchbar32(0x1c04); // !!!!
4399 write_mchbar32(0x1804, 0x406080);
4400
4401 read_mchbar8(0x2ca8);
4402
4403 if (x2ca8 == 0) {
4404 write_mchbar8(0x2ca8, read_mchbar8(0x2ca8) & ~3);
4405 write_mchbar8(0x2ca8, read_mchbar8(0x2ca8) + 4);
4406 write_mchbar32(0x1af0, read_mchbar32(0x1af0) | 0x10);
4407#if REAL
4408 while (1) {
4409 asm volatile ("hlt");
4410 }
4411#else
4412 printf("CP5\n");
4413 exit(0);
4414#endif
4415 }
4416
4417 write_mchbar8(0x2ca8, read_mchbar8(0x2ca8));
4418 read_mchbar32(0x2c80); // !!!!
4419 write_mchbar32(0x2c80, 0x53688);
4420 pcie_write_config32(PCI_DEV (0xff, 0, 0), 0x60, 0x20220);
4421 read_mchbar16(0x2c20); // !!!!
4422 read_mchbar16(0x2c10); // !!!!
4423 read_mchbar16(0x2c00); // !!!!
4424 write_mchbar16(0x2c00, 0x8c0);
4425 udelay(1000);
4426 write_1d0(0, 0x33d, 0, 0);
4427 write_500(&info, 0, 0, 0xb61, 0, 0);
4428 write_500(&info, 1, 0, 0xb61, 0, 0);
4429 write_mchbar32(0x1a30, 0x0);
4430 write_mchbar32(0x1a34, 0x0);
4431 write_mchbar16(0x614,
4432 0xb5b | (info.populated_ranks[1][0][0] *
4433 0x404) | (info.populated_ranks[0][0][0] *
4434 0xa0));
4435 write_mchbar16(0x616, 0x26a);
4436 write_mchbar32(0x134, 0x856000);
4437 write_mchbar32(0x160, 0x5ffffff);
4438 read_mchbar32(0x114); // !!!!
4439 write_mchbar32(0x114, 0xc2024440);
4440 read_mchbar32(0x118); // !!!!
4441 write_mchbar32(0x118, 0x4);
4442 for (channel = 0; channel < NUM_CHANNELS; channel++)
4443 write_mchbar32(0x260 + (channel << 10),
4444 0x30809ff |
4445 ((info.
4446 populated_ranks_mask[channel] & 3) << 20));
4447 for (channel = 0; channel < NUM_CHANNELS; channel++) {
4448 write_mchbar16(0x31c + (channel << 10), 0x101);
4449 write_mchbar16(0x360 + (channel << 10), 0x909);
4450 write_mchbar16(0x3a4 + (channel << 10), 0x101);
4451 write_mchbar16(0x3e8 + (channel << 10), 0x101);
4452 write_mchbar32(0x320 + (channel << 10), 0x29002900);
4453 write_mchbar32(0x324 + (channel << 10), 0x0);
4454 write_mchbar32(0x368 + (channel << 10), 0x32003200);
4455 write_mchbar16(0x352 + (channel << 10), 0x505);
4456 write_mchbar16(0x354 + (channel << 10), 0x3c3c);
4457 write_mchbar16(0x356 + (channel << 10), 0x1040);
4458 write_mchbar16(0x39a + (channel << 10), 0x73e4);
4459 write_mchbar16(0x3de + (channel << 10), 0x77ed);
4460 write_mchbar16(0x422 + (channel << 10), 0x1040);
4461 }
4462
4463 write_1d0(0x4, 0x151, 4, 1);
4464 write_1d0(0, 0x142, 3, 1);
4465 rdmsr(0x1ac); // !!!!
4466 write_500(&info, 1, 1, 0x6b3, 4, 1);
4467 write_500(&info, 1, 1, 0x6cf, 4, 1);
4468
4469 rmw_1d0(0x21c, 0x38, 0, 6, 1);
4470
4471 write_1d0(((!info.populated_ranks[1][0][0]) << 1) | ((!info.
4472 populated_ranks[0]
4473 [0][0]) << 0),
4474 0x1d1, 3, 1);
4475 for (channel = 0; channel < NUM_CHANNELS; channel++) {
4476 write_mchbar16(0x38e + (channel << 10), 0x5f5f);
4477 write_mchbar16(0x3d2 + (channel << 10), 0x5f5f);
4478 }
4479
4480 set_334(0);
4481
4482 program_base_timings(&info);
4483
4484 write_mchbar8(0x5ff, read_mchbar8(0x5ff) | 0x80); /* OK */
4485
4486 write_1d0(0x2, 0x1d5, 2, 1);
4487 write_1d0(0x20, 0x166, 7, 1);
4488 write_1d0(0x0, 0xeb, 3, 1);
4489 write_1d0(0x0, 0xf3, 6, 1);
4490
4491 for (channel = 0; channel < NUM_CHANNELS; channel++)
4492 for (lane = 0; lane < 9; lane++) {
4493 u16 addr = 0x125 + get_lane_offset(0, 0, lane);
4494 u8 a;
4495 a = read_500(&info, channel, addr, 6); // = 0x20040080 //!!!!
4496 write_500(&info, channel, a, addr, 6, 1);
4497 }
4498
4499 udelay(1000);
4500
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]4501 if (s3resume) {
4502 if (info.cached_training == NULL) {
4503 u32 reg32;
4504 printk(BIOS_ERR,
4505 "Couldn't find training data. Rebooting\n");
4506 reg32 = inl(DEFAULT_PMBASE + 0x04);
4507 outl(reg32 & ~(7 << 10), DEFAULT_PMBASE + 0x04);
4508 outb(0xe, 0xcf9);
4509
4510#if REAL
4511 while (1) {
4512 asm volatile ("hlt");
4513 }
4514#else
4515 printf("CP5\n");
4516 exit(0);
4517#endif
4518 }
4519 int tm;
4520 info.training = *info.cached_training;
4521 for (tm = 0; tm < 4; tm++)
4522 for (channel = 0; channel < NUM_CHANNELS; channel++)
4523 for (slot = 0; slot < NUM_SLOTS; slot++)
4524 for (rank = 0; rank < NUM_RANKS; rank++)
4525 for (lane = 0; lane < 9; lane++)
4526 write_500(&info,
4527 channel,
4528 info.training.
4529 lane_timings
4530 [tm][channel]
4531 [slot][rank]
4532 [lane],
4533 get_timing_register_addr
4534 (lane, tm,
4535 slot, rank),
4536 9, 0);
4537 write_1d0(info.cached_training->reg_178, 0x178, 7, 1);
4538 write_1d0(info.cached_training->reg_10b, 0x10b, 6, 1);
4539 }
4540
4541 read_mchbar32(0x1f4); // !!!!
4542 write_mchbar32(0x1f4, 0x20000);
4543 write_mchbar32(0x1f0, 0x1d000200);
4544 read_mchbar8(0x1f0); // !!!!
4545 write_mchbar8(0x1f0, 0x1);
4546 read_mchbar8(0x1f0); // !!!!
4547
4548 program_board_delay(&info);
4549
4550 write_mchbar8(0x5ff, 0x0); /* OK */
4551 write_mchbar8(0x5ff, 0x80); /* OK */
4552 write_mchbar8(0x5f4, 0x1); /* OK */
4553
4554 write_mchbar32(0x130, read_mchbar32(0x130) & 0xfffffffd); // | 2 when ?
4555 while (read_mchbar32(0x130) & 1) ;
4556 gav(read_1d0(0x14b, 7)); // = 0x81023100
4557 write_1d0(0x30, 0x14b, 7, 1);
4558 read_1d0(0xd6, 6); // = 0xfa008080 // !!!!
4559 write_1d0(7, 0xd6, 6, 1);
4560 read_1d0(0x328, 6); // = 0xfa018080 // !!!!
4561 write_1d0(7, 0x328, 6, 1);
4562
4563 for (channel = 0; channel < NUM_CHANNELS; channel++)
4564 set_4cf(&info, channel,
4565 info.populated_ranks[channel][0][0] ? 8 : 0);
4566
4567 read_1d0(0x116, 4); // = 0x4040432 // !!!!
4568 write_1d0(2, 0x116, 4, 1);
4569 read_1d0(0xae, 6); // = 0xe8088080 // !!!!
4570 write_1d0(0, 0xae, 6, 1);
4571 read_1d0(0x300, 4); // = 0x48088080 // !!!!
4572 write_1d0(0, 0x300, 6, 1);
4573 read_mchbar16(0x356); // !!!!
4574 write_mchbar16(0x356, 0x1040);
4575 read_mchbar16(0x756); // !!!!
4576 write_mchbar16(0x756, 0x1040);
4577 write_mchbar32(0x140, read_mchbar32(0x140) & ~0x07000000);
4578 write_mchbar32(0x138, read_mchbar32(0x138) & ~0x07000000);
4579 write_mchbar32(0x130, 0x31111301);
4580 while (read_mchbar32(0x130) & 1) ;
4581
4582 {
4583 u32 t;
4584 u8 val_a1;
4585 val_a1 = read_1d0(0xa1, 6); // = 0x1cf4040 // !!!!
4586 t = read_1d0(0x2f3, 6); // = 0x10a4040 // !!!!
4587 rmw_1d0(0x320, 0x07,
4588 (t & 4) | ((t & 8) >> 2) | ((t & 0x10) >> 4), 6, 1);
4589 rmw_1d0(0x14b, 0x78,
4590 ((((val_a1 >> 2) & 4) | (val_a1 & 8)) >> 2) | (val_a1 &
4591 4), 7,
4592 1);
4593 rmw_1d0(0xce, 0x38,
4594 ((((val_a1 >> 2) & 4) | (val_a1 & 8)) >> 2) | (val_a1 &
4595 4), 6,
4596 1);
4597 }
4598
4599 for (channel = 0; channel < NUM_CHANNELS; channel++)
4600 set_4cf(&info, channel,
4601 info.populated_ranks[channel][0][0] ? 9 : 1);
4602
4603 rmw_1d0(0x116, 0xe, 1, 4, 1); // = 0x4040432 // !!!!
4604 read_mchbar32(0x144); // !!!!
4605 write_1d0(2, 0xae, 6, 1);
4606 write_1d0(2, 0x300, 6, 1);
4607 write_1d0(2, 0x121, 3, 1);
4608 read_1d0(0xd6, 6); // = 0xfa00c0c7 // !!!!
4609 write_1d0(4, 0xd6, 6, 1);
4610 read_1d0(0x328, 6); // = 0xfa00c0c7 // !!!!
4611 write_1d0(4, 0x328, 6, 1);
4612
4613 for (channel = 0; channel < NUM_CHANNELS; channel++)
4614 set_4cf(&info, channel,
4615 info.populated_ranks[channel][0][0] ? 9 : 0);
4616
4617 write_mchbar32(0x130,
4618 0x11111301 | (info.
4619 populated_ranks[1][0][0] << 30) | (info.
4620 populated_ranks
4621 [0][0]
4622 [0] <<
4623 29));
4624 while (read_mchbar8(0x130) & 1) ; // !!!!
4625 read_1d0(0xa1, 6); // = 0x1cf4054 // !!!!
4626 read_1d0(0x2f3, 6); // = 0x10a4054 // !!!!
4627 read_1d0(0x21c, 6); // = 0xafa00c0 // !!!!
4628 write_1d0(0, 0x21c, 6, 1);
4629 read_1d0(0x14b, 7); // = 0x810231b0 // !!!!
4630 write_1d0(0x35, 0x14b, 7, 1);
4631
4632 for (channel = 0; channel < NUM_CHANNELS; channel++)
4633 set_4cf(&info, channel,
4634 info.populated_ranks[channel][0][0] ? 0xb : 0x2);
4635
4636 set_334(1);
4637
4638 write_mchbar8(0x1e8, 0x4); /* OK */
4639
4640 for (channel = 0; channel < NUM_CHANNELS; channel++) {
4641 write_500(&info, channel,
4642 0x3 & ~(info.populated_ranks_mask[channel]), 0x6b7, 2,
4643 1);
4644 write_500(&info, channel, 0x3, 0x69b, 2, 1);
4645 }
4646 write_mchbar32(0x2d0, (read_mchbar32(0x2d0) & 0xff2c01ff) | 0x200000); /* OK */
4647 write_mchbar16(0x6c0, 0x14a0); /* OK */
4648 write_mchbar32(0x6d0, (read_mchbar32(0x6d0) & 0xff0080ff) | 0x8000); /* OK */
4649 write_mchbar16(0x232, 0x8);
4650 write_mchbar32(0x234, (read_mchbar32(0x234) & 0xfffbfffb) | 0x40004); /* 0x40004 or 0 depending on ? */
4651 write_mchbar32(0x34, (read_mchbar32(0x34) & 0xfffffffd) | 5); /* OK */
4652 write_mchbar32(0x128, 0x2150d05);
4653 write_mchbar8(0x12c, 0x1f); /* OK */
4654 write_mchbar8(0x12d, 0x56); /* OK */
4655 write_mchbar8(0x12e, 0x31);
4656 write_mchbar8(0x12f, 0x0); /* OK */
4657 write_mchbar8(0x271, 0x2); /* OK */
4658 write_mchbar8(0x671, 0x2); /* OK */
4659 write_mchbar8(0x1e8, 0x4); /* OK */
4660 for (channel = 0; channel < NUM_CHANNELS; channel++)
4661 write_mchbar32(0x294 + (channel << 10),
4662 (info.populated_ranks_mask[channel] & 3) << 16);
4663 write_mchbar32(0x134, (read_mchbar32(0x134) & 0xfc01ffff) | 0x10000); /* OK */
4664 write_mchbar32(0x134, (read_mchbar32(0x134) & 0xfc85ffff) | 0x850000); /* OK */
4665 for (channel = 0; channel < NUM_CHANNELS; channel++)
4666 write_mchbar32(0x260 + (channel << 10),
4667 (read_mchbar32(0x260 + (channel << 10)) &
4668 ~0xf00000) | 0x8000000 | ((info.
4669 populated_ranks_mask
4670 [channel] & 3) <<
4671 20));
4672
4673 if (!s3resume)
4674 jedec_init(&info);
4675
4676 int totalrank = 0;
4677 for (channel = 0; channel < NUM_CHANNELS; channel++)
4678 for (slot = 0; slot < NUM_SLOTS; slot++)
4679 for (rank = 0; rank < NUM_RANKS; rank++)
4680 if (info.populated_ranks[channel][slot][rank]) {
4681 jedec_read(&info, channel, slot, rank,
4682 totalrank, 0xa, 0x400);
4683 totalrank++;
4684 }
4685
4686 write_mchbar8(0x12c, 0x9f);
4687
4688 read_mchbar8(0x271); // 2 // !!!!
4689 write_mchbar8(0x271, 0xe);
4690 read_mchbar8(0x671); // !!!!
4691 write_mchbar8(0x671, 0xe);
4692
4693 if (!s3resume) {
4694 for (channel = 0; channel < NUM_CHANNELS; channel++) {
4695 write_mchbar32(0x294 + (channel << 10),
4696 (info.
4697 populated_ranks_mask[channel] & 3) <<
4698 16);
4699 write_mchbar16(0x298 + (channel << 10),
4700 (info.
4701 populated_ranks[channel][0][0]) | (info.
4702 populated_ranks
4703 [channel]
4704 [0]
4705 [1]
4706 <<
4707 5));
4708 write_mchbar32(0x29c + (channel << 10), 0x77a);
4709 }
4710 read_mchbar32(0x2c0); /// !!!
4711 write_mchbar32(0x2c0, 0x6009cc00);
4712
4713 {
4714 u8 a, b;
4715 a = read_mchbar8(0x243); // !!!!
4716 b = read_mchbar8(0x643); // !!!!
4717 write_mchbar8(0x243, a | 2);
4718 write_mchbar8(0x643, b | 2);
4719 }
4720
4721 write_1d0(7, 0x19b, 3, 1);
4722 write_1d0(7, 0x1c0, 3, 1);
4723 write_1d0(4, 0x1c6, 4, 1);
4724 write_1d0(4, 0x1cc, 4, 1);
4725 read_1d0(0x151, 4); // = 0x408c6d74 // !!!!
4726 write_1d0(4, 0x151, 4, 1);
4727 write_mchbar32(0x584, 0xfffff);
4728 write_mchbar32(0x984, 0xfffff);
4729
4730 for (channel = 0; channel < NUM_CHANNELS; channel++)
4731 for (slot = 0; slot < NUM_SLOTS; slot++)
4732 for (rank = 0; rank < NUM_RANKS; rank++)
4733 if (info.
4734 populated_ranks[channel][slot]
4735 [rank])
4736 config_rank(&info, s3resume,
4737 channel, slot,
4738 rank);
4739
4740 write_mchbar8(0x243, 0x1);
4741 write_mchbar8(0x643, 0x1);
4742 }
4743
4744 /* was == 1 but is common */
4745 pcie_write_config16(NORTHBRIDGE, 0xc8, 3);
4746 write_26c(0, 0x820);
4747 write_26c(1, 0x820);
4748 write_mchbar32(0x130, read_mchbar32(0x130) | 2);
4749 /* end */
4750
4751 if (s3resume) {
4752 for (channel = 0; channel < NUM_CHANNELS; channel++) {
4753 write_mchbar32(0x294 + (channel << 10),
4754 (info.
4755 populated_ranks_mask[channel] & 3) <<
4756 16);
4757 write_mchbar16(0x298 + (channel << 10),
4758 (info.
4759 populated_ranks[channel][0][0]) | (info.
4760 populated_ranks
4761 [channel]
4762 [0]
4763 [1]
4764 <<
4765 5));
4766 write_mchbar32(0x29c + (channel << 10), 0x77a);
4767 }
4768 read_mchbar32(0x2c0); /// !!!
4769 write_mchbar32(0x2c0, 0x6009cc00);
4770 }
4771
4772 write_mchbar32(0xfa4, read_mchbar32(0xfa4) & ~0x01000002);
4773 write_mchbar32(0xfb0, 0x2000e019);
4774
4775#if !REAL
4776 printf("CP16\n");
4777#endif
4778
4779 /* Before training. */
4780 timestamp_add_now(103);
4781
4782 if (!s3resume)
4783 ram_training(&info);
4784
4785 /* After training. */
4786 timestamp_add_now (104);
4787
4788 dump_timings(&info);
4789
4790#if 0
4791 ram_check(0x100000, 0x200000);
4792#endif
4793 program_modules_memory_map(&info, 0);
4794 program_total_memory_map(&info);
4795
4796 if (info.non_interleaved_part_mb != 0 && info.interleaved_part_mb != 0)
4797 write_mchbar8(0x111, 0x20 | (0 << 2) | (1 << 6) | (0 << 7));
4798 else if (have_match_ranks(&info, 0, 4) && have_match_ranks(&info, 1, 4))
4799 write_mchbar8(0x111, 0x20 | (3 << 2) | (0 << 6) | (1 << 7));
4800 else if (have_match_ranks(&info, 0, 2) && have_match_ranks(&info, 1, 2))
4801 write_mchbar8(0x111, 0x20 | (3 << 2) | (0 << 6) | (0 << 7));
4802 else
4803 write_mchbar8(0x111, 0x20 | (3 << 2) | (1 << 6) | (0 << 7));
4804
4805 write_mchbar32(0xfac, read_mchbar32(0xfac) & ~0x80000000); // OK
4806 write_mchbar32(0xfb4, 0x4800); // OK
4807 write_mchbar32(0xfb8, (info.revision < 8) ? 0x20 : 0x0); // OK
4808 write_mchbar32(0xe94, 0x7ffff); // OK
4809 write_mchbar32(0xfc0, 0x80002040); // OK
4810 write_mchbar32(0xfc4, 0x701246); // OK
4811 write_mchbar8(0xfc8, read_mchbar8(0xfc8) & ~0x70); // OK
4812 write_mchbar32(0xe5c, 0x1000000 | read_mchbar32(0xe5c)); // OK
4813 write_mchbar32(0x1a70, (read_mchbar32(0x1a70) | 0x00200000) & ~0x00100000); // OK
4814 write_mchbar32(0x50, 0x700b0); // OK
4815 write_mchbar32(0x3c, 0x10); // OK
4816 write_mchbar8(0x1aa8, (read_mchbar8(0x1aa8) & ~0x35) | 0xa); // OK
4817 write_mchbar8(0xff4, read_mchbar8(0xff4) | 0x2); // OK
4818 write_mchbar32(0xff8, (read_mchbar32(0xff8) & ~0xe008) | 0x1020); // OK
4819
4820#if REAL
4821 write_mchbar32(0xd00, IOMMU_BASE2 | 1);
4822 write_mchbar32(0xd40, IOMMU_BASE1 | 1);
4823 write_mchbar32(0xdc0, IOMMU_BASE4 | 1);
4824
4825 write32(IOMMU_BASE1 | 0xffc, 0x80000000);
4826 write32(IOMMU_BASE2 | 0xffc, 0xc0000000);
4827 write32(IOMMU_BASE4 | 0xffc, 0x80000000);
4828
4829#else
4830 {
4831 u32 eax;
4832 eax = read32(0xffc + (read_mchbar32(0xd00) & ~1)) | 0x08000000; // = 0xe911714b// OK
4833 write32(0xffc + (read_mchbar32(0xd00) & ~1), eax); // OK
4834 eax = read32(0xffc + (read_mchbar32(0xdc0) & ~1)) | 0x40000000; // = 0xe911714b// OK
4835 write32(0xffc + (read_mchbar32(0xdc0) & ~1), eax); // OK
4836 }
4837#endif
4838
4839 {
4840 u32 eax;
4841
4842 eax = info.fsb_frequency / 9;
4843 write_mchbar32(0xfcc, (read_mchbar32(0xfcc) & 0xfffc0000) | (eax * 0x280) | (eax * 0x5000) | eax | 0x40000); // OK
4844 write_mchbar32(0x20, 0x33001); //OK
4845 }
4846
4847 for (channel = 0; channel < NUM_CHANNELS; channel++) {
4848 write_mchbar32(0x220 + (channel << 10), read_mchbar32(0x220 + (channel << 10)) & ~0x7770); //OK
4849 if (info.max_slots_used_in_channel == 1)
4850 write_mchbar16(0x237 + (channel << 10), (read_mchbar16(0x237 + (channel << 10)) | 0x0201)); //OK
4851 else
4852 write_mchbar16(0x237 + (channel << 10), (read_mchbar16(0x237 + (channel << 10)) & ~0x0201)); //OK
4853
4854 write_mchbar8(0x241 + (channel << 10), read_mchbar8(0x241 + (channel << 10)) | 1); // OK
4855
4856 if (info.clock_speed_index <= 1
4857 && (info.silicon_revision == 2
4858 || info.silicon_revision == 3))
4859 write_mchbar32(0x248 + (channel << 10), (read_mchbar32(0x248 + (channel << 10)) | 0x00102000)); // OK
4860 else
4861 write_mchbar32(0x248 + (channel << 10), (read_mchbar32(0x248 + (channel << 10)) & ~0x00102000)); // OK
4862 }
4863
4864 write_mchbar32(0x115, read_mchbar32(0x115) | 0x1000000); // OK
4865
4866 {
4867 u8 al;
4868 al = 0xd;
4869 if (!(info.silicon_revision == 0 || info.silicon_revision == 1))
4870 al += 2;
4871 al |= ((1 << (info.max_slots_used_in_channel - 1)) - 1) << 4;
4872 write_mchbar32(0x210, (al << 16) | 0x20); // OK
4873 }
4874
4875 for (channel = 0; channel < NUM_CHANNELS; channel++) {
4876 write_mchbar32(0x288 + (channel << 10), 0x70605040); // OK
4877 write_mchbar32(0x28c + (channel << 10), 0xfffec080); // OK
4878 write_mchbar32(0x290 + (channel << 10), 0x282091c | ((info.max_slots_used_in_channel - 1) << 0x16)); // OK
4879 }
4880 u32 reg1c;
4881 pcie_read_config32(NORTHBRIDGE, 0x40); // = DEFAULT_EPBAR | 0x001 // OK
4882 reg1c = read32(DEFAULT_EPBAR | 0x01c); // = 0x8001 // OK
4883 pcie_read_config32(NORTHBRIDGE, 0x40); // = DEFAULT_EPBAR | 0x001 // OK
4884 write32(DEFAULT_EPBAR | 0x01c, reg1c); // OK
4885 read_mchbar8(0xe08); // = 0x0
4886 pcie_read_config32(NORTHBRIDGE, 0xe4); // = 0x316126
4887 write_mchbar8(0x1210, read_mchbar8(0x1210) | 2); // OK
4888 write_mchbar32(0x1200, 0x8800440); // OK
4889 write_mchbar32(0x1204, 0x53ff0453); // OK
4890 write_mchbar32(0x1208, 0x19002043); // OK
4891 write_mchbar16(0x1214, 0x320); // OK
4892
4893 if (info.revision == 0x10 || info.revision == 0x11) {
4894 write_mchbar16(0x1214, 0x220); // OK
4895 write_mchbar8(0x1210, read_mchbar8(0x1210) | 0x40); // OK
4896 }
4897
4898 write_mchbar8(0x1214, read_mchbar8(0x1214) | 0x4); // OK
4899 write_mchbar8(0x120c, 0x1); // OK
4900 write_mchbar8(0x1218, 0x3); // OK
4901 write_mchbar8(0x121a, 0x3); // OK
4902 write_mchbar8(0x121c, 0x3); // OK
4903 write_mchbar16(0xc14, 0x0); // OK
4904 write_mchbar16(0xc20, 0x0); // OK
4905 write_mchbar32(0x1c, 0x0); // OK
4906
4907 /* revision dependent here. */
4908
4909 write_mchbar16(0x1230, read_mchbar16(0x1230) | 0x1f07); // OK
4910
4911 if (info.uma_enabled)
4912 write_mchbar32(0x11f4, read_mchbar32(0x11f4) | 0x10000000); // OK
4913
4914 write_mchbar16(0x1230, read_mchbar16(0x1230) | 0x8000); // OK
4915 write_mchbar8(0x1214, read_mchbar8(0x1214) | 1); // OK
4916
4917 u8 bl, ebpb;
4918 u16 reg_1020;
4919
4920 reg_1020 = read_mchbar32(0x1020); // = 0x6c733c // OK
4921 write_mchbar8(0x1070, 0x1); // OK
4922
4923 write_mchbar32(0x1000, 0x100); // OK
4924 write_mchbar8(0x1007, 0x0); // OK
4925
4926 if (reg_1020 != 0) {
4927 write_mchbar16(0x1018, 0x0); // OK
4928 bl = reg_1020 >> 8;
4929 ebpb = reg_1020 & 0xff;
4930 } else {
4931 ebpb = 0;
4932 bl = 8;
4933 }
4934
4935 rdmsr(0x1a2);
4936
4937 write_mchbar32(0x1014, 0xffffffff); // OK
4938
4939 write_mchbar32(0x1010, ((((ebpb + 0x7d) << 7) / bl) & 0xff) * (! !reg_1020)); // OK
4940
4941 write_mchbar8(0x101c, 0xb8); // OK
4942
4943 write_mchbar8(0x123e, (read_mchbar8(0x123e) & 0xf) | 0x60); // OK
4944 if (reg_1020 != 0) {
4945 write_mchbar32(0x123c, (read_mchbar32(0x123c) & ~0x00900000) | 0x600000); // OK
4946 write_mchbar8(0x101c, 0xb8); // OK
4947 }
4948
4949 setup_heci_uma(&info);
4950
4951 if (info.uma_enabled) {
4952 u16 ax;
4953 write_mchbar32(0x11b0, read_mchbar32(0x11b0) | 0x4000); // OK
4954 write_mchbar32(0x11b4, read_mchbar32(0x11b4) | 0x4000); // OK
4955 write_mchbar16(0x1190, read_mchbar16(0x1190) | 0x4000); // OK
4956
4957 ax = read_mchbar16(0x1190) & 0xf00; // = 0x480a // OK
4958 write_mchbar16(0x1170, ax | (read_mchbar16(0x1170) & 0x107f) | 0x4080); // OK
4959 write_mchbar16(0x1170, read_mchbar16(0x1170) | 0x1000); // OK
4960#if REAL
4961 udelay(1000);
4962#endif
4963 u16 ecx;
4964 for (ecx = 0xffff; ecx && (read_mchbar16(0x1170) & 0x1000); ecx--) ; // OK
4965 write_mchbar16(0x1190, read_mchbar16(0x1190) & ~0x4000); // OK
4966 }
4967
4968 pcie_write_config8(SOUTHBRIDGE, GEN_PMCON_2,
4969 pcie_read_config8(SOUTHBRIDGE, GEN_PMCON_2) & ~0x80);
4970 udelay(10000);
Vladimir Serbinenkoc7db28c2014-02-19 22:09:33 +0100[diff] [blame]4971 write_mchbar16(0x2ca8, 0x8);
Vladimir Serbinenkoc6f6be02013-11-12 22:32:08 +0100[diff] [blame]4972
4973#if REAL
4974 udelay(1000);
4975 dump_timings(&info);
4976 if (!s3resume)
4977 save_timings(&info);
4978#endif
4979}