src/northbridge/intel/pineview/raminit.c

/*
 * This file is part of the coreboot project.
 *
 * Copyright (C) 2015  Damien Zammit <damien@zamaudio.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of
 * the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

#include <arch/io.h>
#include <cbmem.h>
#include <console/console.h>
#include <cpu/x86/cache.h>
#include <cpu/x86/mtrr.h>
#include <delay.h>
#include <halt.h>
#include <lib.h>
#include "pineview.h"
#include "raminit.h"
#include <pc80/mc146818rtc.h>
#include <spd.h>
#include <string.h>

/* Debugging macros. */
#if CONFIG_DEBUG_RAM_SETUP
#define PRINTK_DEBUG(x...)	printk(BIOS_DEBUG, x)
#else
#define PRINTK_DEBUG(x...)
#endif

#define MAX_TCLK_667	0x30
#define MAX_TCLK_800	0x25
#define MAX_TAC_667	0x45
#define MAX_TAC_800	0x40

#define NOP_CMD		(1 << 1)
#define PRE_CHARGE_CMD	(1 << 2)
#define MRS_CMD		((1 << 2) | (1 << 1))
#define EMRS_CMD	(1 << 3)
#define EMRS1_CMD	(EMRS_CMD | (1 << 4))
#define EMRS2_CMD	(EMRS_CMD | (1 << 5))
#define EMRS3_CMD	(EMRS_CMD | (1 << 5) | (1 << 4))
#define ZQCAL_CMD	((1 << 3) | (1 << 1))
#define CBR_CMD		((1 << 3) | (1 << 2))
#define NORMAL_OP_CMD	((1 << 3) | (1 << 2) | (1 << 1))

#define UBDIMM 1
#define SODIMM 2

#define TOTAL_CHANNELS 1
#define TOTAL_DIMMS 2

#define DIMM_IS_POPULATED(dimms, idx) (dimms[idx].card_type != 0)
#define IF_DIMM_POPULATED(dimms, idx) if (dimms[idx].card_type != 0)
#define ONLY_DIMMA_IS_POPULATED(dimms, ch) ( \
	(DIMM_IS_POPULATED(dimms, (ch == 0) ? 0 : 2) && \
	!DIMM_IS_POPULATED(dimms, (ch == 0) ? 1 : 3)))
#define ONLY_DIMMB_IS_POPULATED(dimms, ch) ( \
	(DIMM_IS_POPULATED(dimms, (ch == 0) ? 1 : 3) && \
	!DIMM_IS_POPULATED(dimms, (ch == 0) ? 0 : 2)))
#define BOTH_DIMMS_ARE_POPULATED(dimms, ch) ( \
	(DIMM_IS_POPULATED(dimms, (ch == 0) ? 0 : 2) && \
	(DIMM_IS_POPULATED(dimms, (ch == 0) ? 1 : 3))))
#define FOR_EACH_DIMM(idx) \
	for (idx = 0; idx < TOTAL_DIMMS; ++idx)
#define FOR_EACH_POPULATED_DIMM(dimms, idx) \
	FOR_EACH_DIMM(idx) IF_DIMM_POPULATED(dimms, idx)
#define CHANNEL_IS_POPULATED(dimms, idx) ((dimms[idx<<1].card_type != 0) || (dimms[(idx<<1) + 1].card_type != 0))
#define CHANNEL_IS_CARDF(dimms, idx) ((dimms[idx<<1].card_type == 0xf) || (dimms[(idx<<1) + 1].card_type == 0xf))
#define IF_CHANNEL_POPULATED(dimms, idx) if ((dimms[idx<<1].card_type != 0) || (dimms[(idx<<1) + 1].card_type != 0))
#define FOR_EACH_CHANNEL(idx) \
	for (idx = 0; idx < TOTAL_CHANNELS; ++idx)
#define FOR_EACH_POPULATED_CHANNEL(dimms, idx) \
	FOR_EACH_CHANNEL(idx) IF_CHANNEL_POPULATED(dimms, idx)

#define RANKS_PER_CHANNEL 4

#define FOR_EACH_RANK_IN_CHANNEL(r) \
	for (r = 0; r < RANKS_PER_CHANNEL; ++r)
#define FOR_EACH_POPULATED_RANK_IN_CHANNEL(dimms, ch, r) \
	FOR_EACH_RANK_IN_CHANNEL(r) if (rank_is_populated(dimms, ch, r))
#define FOR_EACH_RANK(ch, r) \
	FOR_EACH_CHANNEL(ch) FOR_EACH_RANK_IN_CHANNEL(r)
#define FOR_EACH_POPULATED_RANK(dimms, ch, r) \
	FOR_EACH_RANK(ch, r) if (rank_is_populated(dimms, ch, r))

static bool rank_is_populated(struct dimminfo dimms[], u8 ch, u8 r)
{
	return ((dimms[ch<<1].card_type && ((r) < dimms[ch<<1].ranks))
		|| (dimms[(ch<<1) + 1].card_type
			&& ((r) >= 2)
			&& ((r) < (dimms[(ch<<1) + 1].ranks + 2))));
}

static inline void barrier(void)
{
	 __asm__ __volatile__("": : :"memory");
}

static inline int spd_read_byte(unsigned device, unsigned address)
{
	return smbus_read_byte(device, address);
}

static int decode_spd(struct dimminfo *d)
{
	d->type = 0;
	if (d->spd_data[20] == 0x2) {
		d->type = UBDIMM;
	} else if (d->spd_data[20] == 0x4) {
		d->type = SODIMM;
	}
	d->sides = (d->spd_data[5] & 0x7) + 1;
	d->banks = (d->spd_data[17] >> 2) - 1;
	d->chip_capacity = d->banks;
	d->rows = d->spd_data[3];// - 12;
	d->cols = d->spd_data[4];// - 9;
	d->cas_latencies = 0x78;
	d->cas_latencies &= d->spd_data[18];
	if (d->cas_latencies == 0)
		d->cas_latencies = 7;
	d->tAAmin = d->spd_data[26];
	d->tCKmin = d->spd_data[25];
	d->width = (d->spd_data[13] >> 3) - 1;
	d->page_size = (d->width+1) * (1 << d->cols); // Bytes
	d->tRAS = d->spd_data[30];
	d->tRP = d->spd_data[27];
	d->tRCD = d->spd_data[29];
	d->tWR = d->spd_data[36];
	d->ranks = d->sides; // XXX
#if CONFIG_DEBUG_RAM_SETUP
	const char *ubso[] = { {"UB"}, {"SO"} };
#endif
	PRINTK_DEBUG("%s-DIMM %d\n", &ubso[d->type][0], i);
	PRINTK_DEBUG("  Sides     : %d\n", d->sides);
	PRINTK_DEBUG("  Banks     : %d\n", d->banks);
	PRINTK_DEBUG("  Ranks     : %d\n", d->ranks);
	PRINTK_DEBUG("  Rows      : %d\n", d->rows);
	PRINTK_DEBUG("  Cols      : %d\n", d->cols);
	PRINTK_DEBUG("  Page size : %d\n", d->page_size);
	PRINTK_DEBUG("  Width     : %d\n", (d->width + 1) * 8);

	return 0;
}

/* Ram Config:    DIMMB-DIMMA
 *		0 EMPTY-EMPTY
 *		1 EMPTY-x16SS
 *		2 EMPTY-x16DS
 *		3 x16SS-x16SS
 *		4 x16DS-x16DS
 *		5 EMPTY- x8DS
 *		6 x8DS - x8DS
 */
static void find_ramconfig(struct sysinfo *s, u32 chan)
{
	if (s->dimms[chan>>1].sides == 0) {
		// NC
		if (s->dimms[(chan>>1) + 1].sides == 0) {
			// NC/NC
			s->dimm_config[chan] = 0;
		} else if (s->dimms[(chan>>1) + 1].sides == 1) {
			// NC/SS
			if (s->dimms[(chan>>1) + 1].width == 0) {
				// NC/8SS
				s->dimm_config[chan] = 1;
			} else {
				// NC/16SS
				s->dimm_config[chan] = 1;
			}
		} else {
			// NC/DS
			if (s->dimms[(chan>>1) + 1].width == 0) {
				// NC/8DS
				s->dimm_config[chan] = 5;
			} else {
				// NC/16DS
				s->dimm_config[chan] = 2;
			}
		}
	} else if (s->dimms[chan>>1].sides == 1) {
		// SS
		if (s->dimms[(chan>>1) + 1].sides == 0) {
			// SS/NC
			if (s->dimms[chan>>1].width == 0) {
				// 8SS/NC
				s->dimm_config[chan] = 1;
			} else {
				// 16SS/NC
				s->dimm_config[chan] = 1;
			}
		} else if (s->dimms[(chan>>1) + 1].sides == 1) {
			// SS/SS
			if (s->dimms[chan>>1].width == 0) {
				if (s->dimms[(chan>>1) + 1].width == 0) {
					// 8SS/8SS
					s->dimm_config[chan] = 3;
				} else {
					// 8SS/16SS
					die("Mixed Not supported\n");
				}
			} else {
				if (s->dimms[(chan>>1) + 1].width == 0) {
					// 16SS/8SS
					die("Mixed Not supported\n");
				} else {
					// 16SS/16SS
					s->dimm_config[chan] = 3;
				}
			}
		} else {
			// SS/DS
			if (s->dimms[chan>>1].width == 0) {
				if (s->dimms[(chan>>1) + 1].width == 0) {
					// 8SS/8DS
					die("Mixed Not supported\n");
				} else {
					die("Mixed Not supported\n");
				}
			} else {
				if (s->dimms[(chan>>1) + 1].width == 0) {
					// 16SS/8DS
					die("Mixed Not supported\n");
				} else {
					die("Mixed Not supported\n");
				}
			}
		}
	} else {
		// DS
		if (s->dimms[(chan>>1) + 1].sides == 0) {
			// DS/NC
			if (s->dimms[chan>>1].width == 0) {
				// 8DS/NC
				s->dimm_config[chan] = 5;
			} else {
				s->dimm_config[chan] = 4;
			}
		} else if (s->dimms[(chan>>1) + 1].sides == 1) {
			// DS/SS
			if (s->dimms[chan>>1].width == 0) {
				if (s->dimms[(chan>>1) + 1].width == 0) {
					// 8DS/8SS
					die("Mixed Not supported\n");
				} else {
					// 8DS/16SS
					die("Mixed Not supported\n");
				}
			} else {
				if (s->dimms[(chan>>1) + 1].width == 0) {
					die("Mixed Not supported\n");
				} else {
					// 16DS/16DS
					s->dimm_config[chan] = 4;
				}
			}
		} else {
			// DS/DS
			if (s->dimms[chan>>1].width == 0 && s->dimms[(chan>>1)+1].width == 0) {
				// 8DS/8DS
				s->dimm_config[chan] = 6;
			}
		}
	}
}

static void sdram_read_spds(struct sysinfo *s)
{
	u8 i, j, chan;
	int status = 0;
	s->dt0mode = 0;
	FOR_EACH_DIMM(i) {
		for (j = 0; j < 64; j++) {
			status = spd_read_byte(s->spd_map[i], j);
			if (status < 0) {
				s->dimms[i].card_type = 0;
				break;
			}
			s->dimms[i].spd_data[j] = (u8) status;
			if (j == 62)
				s->dimms[i].card_type = ((u8) status) & 0x1f;
		}
		hexdump(s->dimms[i].spd_data, 64);
	}

	s->spd_type = 0;
	FOR_EACH_POPULATED_DIMM(s->dimms, i) {
		switch (s->dimms[i].spd_data[2]) {
			case 0x8:
				s->spd_type = DDR2;
				break;
			case 0xb:
			default:
				die("DIMM type mismatch\n");
				break;
		}
	}

	int err = 1;
	FOR_EACH_POPULATED_DIMM(s->dimms, i) {
		err = decode_spd(&s->dimms[i]);
		s->dt0mode |= (s->dimms[i].spd_data[49] & 0x2) >> 1;
	}
	if (err) {
		die("No memory dimms, halt\n");
	}

	FOR_EACH_POPULATED_CHANNEL(s->dimms, chan) {
		find_ramconfig(s, chan);
		PRINTK_DEBUG("  Config[CH%d] : %d\n",
			chan, s->dimm_config[chan]);
	}
}

#if CONFIG_DEBUG_RAM_SETUP
static u32 fsb_reg_to_mhz(u32 speed)
{
	return (speed * 133) + 667;
}

static u32 ddr_reg_to_mhz(u32 speed)
{
	u32 mhz;
	mhz = (speed == 0) ? 667 :
		(speed == 1) ? 800 :
		0;
	return mhz;
}
#endif

static u8 lsbpos(u8 val) //Forward
{
	u8 i;
	for (i = 0; (i < 8) && ((val & (1 << i)) == 0); i++);
	return i;
}

static u8 msbpos(u8 val) //Reverse
{
	u8 i;
	for (i = 7; (i >= 0) && ((val & (1 << i)) == 0); i--);
	return i;
}

static void sdram_detect_smallest_params(struct sysinfo *s)
{
	u16 mult[6] = {
		3000, // 667
		2500, // 800
	};

	u8 i;
	u32 tmp;
	u32 maxtras = 0;
	u32 maxtrp = 0;
	u32 maxtrcd = 0;
	u32 maxtwr = 0;
	u32 maxtrfc = 0;
	u32 maxtwtr = 0;
	u32 maxtrrd = 0;
	u32 maxtrtp = 0;

	FOR_EACH_POPULATED_DIMM(s->dimms, i) {
		maxtras = max(maxtras, s->dimms[i].spd_data[30] * 1000);
		maxtrp = max(maxtrp, (s->dimms[i].spd_data[27] * 1000) >> 2);
		maxtrcd = max(maxtrcd, (s->dimms[i].spd_data[29] * 1000) >> 2);
		maxtwr = max(maxtwr, (s->dimms[i].spd_data[36] * 1000) >> 2);
		maxtrfc = max(maxtrfc, s->dimms[i].spd_data[42] * 1000 +
				(s->dimms[i].spd_data[40] & 0xf));
		maxtwtr = max(maxtwtr, (s->dimms[i].spd_data[37] * 1000) >> 2);
		maxtrrd = max(maxtrrd, (s->dimms[i].spd_data[28] * 1000) >> 2);
		maxtrtp = max(maxtrtp, (s->dimms[i].spd_data[38] * 1000) >> 2);
	}
	for (i = 9; i < 24; i++) { // 16
		tmp = mult[s->selected_timings.mem_clock] * i;
		if (tmp >= maxtras) {
			s->selected_timings.tRAS = i;
			break;
		}
	}
	for (i = 3; i < 10; i++) {
		tmp = mult[s->selected_timings.mem_clock] * i;
		if (tmp >= maxtrp) {
			s->selected_timings.tRP = i;
			break;
		}
	}
	for (i = 3; i < 10; i++) {
		tmp = mult[s->selected_timings.mem_clock] * i;
		if (tmp >= maxtrcd) {
			s->selected_timings.tRCD = i;
			break;
		}
	}
	for (i = 3; i < 15; i++) {
		tmp = mult[s->selected_timings.mem_clock] * i;
		if (tmp >= maxtwr) {
			s->selected_timings.tWR = i;
			break;
		}
	}
	for (i = 15; i < 78; i++) {
		tmp = mult[s->selected_timings.mem_clock] * i;
		if (tmp >= maxtrfc) {
			s->selected_timings.tRFC = ((i + 16) & 0xfe) - 15;
			break;
		}
	}
	for (i = 4; i < 15; i++) {
		tmp = mult[s->selected_timings.mem_clock] * i;
		if (tmp >= maxtwtr) {
			s->selected_timings.tWTR = i;
			break;
		}
	}
	for (i = 2; i < 15; i++) {
		tmp = mult[s->selected_timings.mem_clock] * i;
		if (tmp >= maxtrrd) {
			s->selected_timings.tRRD = i;
			break;
		}
	}
	for (i = 4; i < 15; i++) {
		tmp = mult[s->selected_timings.mem_clock] * i;
		if (tmp >= maxtrtp) {
			s->selected_timings.tRTP = i;
			break;
		}
	}

	PRINTK_DEBUG("Selected timings:\n");
	PRINTK_DEBUG("\tFSB:  %dMHz\n", fsb_reg_to_mhz(s->selected_timings.fsb_clock));
	PRINTK_DEBUG("\tDDR:  %dMHz\n", ddr_reg_to_mhz(s->selected_timings.mem_clock));

	PRINTK_DEBUG("\tCAS:  %d\n", s->selected_timings.CAS);
	PRINTK_DEBUG("\ttRAS: %d\n", s->selected_timings.tRAS);
	PRINTK_DEBUG("\ttRP:  %d\n", s->selected_timings.tRP);
	PRINTK_DEBUG("\ttRCD: %d\n", s->selected_timings.tRCD);
	PRINTK_DEBUG("\ttWR:  %d\n", s->selected_timings.tWR);
	PRINTK_DEBUG("\ttRFC: %d\n", s->selected_timings.tRFC);
	PRINTK_DEBUG("\ttWTR: %d\n", s->selected_timings.tWTR);
	PRINTK_DEBUG("\ttRRD: %d\n", s->selected_timings.tRRD);
	PRINTK_DEBUG("\ttRTP: %d\n", s->selected_timings.tRTP);
}

static void sdram_detect_ram_speed(struct sysinfo *s)
{
	u8 cas, reg8;
	u32 reg32;
	u32 freq = 0;
	u32 fsb = 0;
	u8 i;
	u8 commoncas = 0;
	u8 highcas = 0;
	u8 lowcas = 0;

	// Core frequency
	fsb = (pci_read_config8(PCI_DEV(0,0,0), 0xe3) & 0x70) >> 4;
	if (fsb) {
		fsb = 5 - fsb;
	} else {
		fsb = FSB_CLOCK_800MHz;
	}

	// DDR frequency
	freq = (pci_read_config8(PCI_DEV(0,0,0), 0xe3) & 0x80) >> 7;
	freq |= (pci_read_config8(PCI_DEV(0,0,0), 0xe4) & 0x3) << 1;
	if (freq) {
		freq = 6 - freq;
	} else {
		freq = MEM_CLOCK_800MHz;
	}

	// Detect a common CAS latency
	commoncas = 0xff;
	FOR_EACH_POPULATED_DIMM(s->dimms, i) {
		commoncas &= s->dimms[i].spd_data[18];
	}
	if (commoncas == 0) {
		die("No common CAS among dimms\n");
	}

	// Start with fastest common CAS
	cas = 0;
	highcas = msbpos(commoncas);
	lowcas = max(lsbpos(commoncas), 5);

	while (cas == 0 && highcas >= lowcas) {
		FOR_EACH_POPULATED_DIMM(s->dimms, i) {
			switch (freq) {
			case MEM_CLOCK_800MHz:
				if ((s->dimms[i].spd_data[9] > 0x25) ||
				    (s->dimms[i].spd_data[10] > 0x40)) {
					// CAS too fast, lower it
					highcas--;
					break;
				} else {
					cas = highcas;
				}
				break;
			case MEM_CLOCK_667MHz:
			default:
				if ((s->dimms[i].spd_data[9] > 0x30) ||
				    (s->dimms[i].spd_data[10] > 0x45)) {
					// CAS too fast, lower it
					highcas--;
					break;
				} else {
					cas = highcas;
				}
				break;
			}
		}
	}
	if (highcas < lowcas) {
		// Timings not supported by MCH, lower the frequency
		if(freq == MEM_CLOCK_800MHz) {
			freq--;
			PRINTK_DEBUG("Run DDR clock speed reduced due to timings\n");
		} else {
			die("Timings not supported by MCH\n");
		}
		cas = 0;
		highcas = msbpos(commoncas);
		lowcas = lsbpos(commoncas);
		while (cas == 0 && highcas >= lowcas) {
			FOR_EACH_POPULATED_DIMM(s->dimms, i) {
				switch (freq) {
				case MEM_CLOCK_800MHz:
					if ((s->dimms[i].spd_data[9] > 0x25) ||
					    (s->dimms[i].spd_data[10] > 0x40)) {
						// CAS too fast, lower it
						highcas--;
						break;
					} else {
						cas = highcas;
					}
					break;
				case MEM_CLOCK_667MHz:
				default:
					if ((s->dimms[i].spd_data[9] > 0x30) ||
					    (s->dimms[i].spd_data[10] > 0x45)) {
						// CAS too fast, lower it
						highcas--;
						break;
					} else {
						cas = highcas;
					}
					break;
				}
			}
		}
		if (cas == 0) {
			die("Unsupported dimms\n");
		}
	}

	s->selected_timings.CAS = cas;
	s->selected_timings.mem_clock = freq;
	s->selected_timings.fsb_clock = fsb;

	PRINTK_DEBUG("Drive Memory at %dMHz with CAS = %d clocks\n", ddr_reg_to_mhz(s->selected_timings.mem_clock), s->selected_timings.CAS);

	// Set memory frequency
	MCHBAR32(0xf14) = MCHBAR32(0xf14) | 0x1;
	reg32 = (MCHBAR32(0xc00) & (~0x70)) | (1 << 10);
	if (s->selected_timings.mem_clock == MEM_CLOCK_800MHz) {
		reg8 = 3;
	} else {
		reg8 = 2;
	}
	reg32 |= reg8 << 4;
	MCHBAR32(0xc00) = reg32;
	s->selected_timings.mem_clock = ((MCHBAR32(0xc00) >> 4) & 0x7) - 2;
	if (s->selected_timings.mem_clock == MEM_CLOCK_800MHz) {
		PRINTK_DEBUG("MCH validated at 800MHz\n");
		s->nodll = 0;
		s->maxpi = 63;
		s->pioffset = 0;
	} else if (s->selected_timings.mem_clock == MEM_CLOCK_667MHz) {
		PRINTK_DEBUG("MCH validated at 667MHz\n");
		s->nodll = 1;
		s->maxpi = 15;
		s->pioffset = 1;
	} else {
		PRINTK_DEBUG("MCH set to unknown (%02x)\n",
			(uint8_t) s->selected_timings.mem_clock & 0xff);
	}
}

#define HPET_BASE 0xfed00000
#define HPET32(x) *((volatile u32 *)(HPET_BASE + x))
static void enable_hpet(void)
{
	u32 reg32;
	reg32 = RCBA32(0x3404);
	reg32 &= ~0x3;
	reg32 |= (1 << 7);
	RCBA32(0x3404) = reg32;
	HPET32(0x10) = HPET32(0x10) | 1;
}

static void sdram_clk_crossing(struct sysinfo *s)
{
	u8 clk_idx, fsb_idx;
	const u32 clkcross[2][2][4] = {
	{
		{0xFFFFFFFF, 0x05030305, 0x0000FFFF, 0x00000000},  //667  667
		{0x1F1F1F1F, 0x2A1F1FA5, 0x00000000, 0x05000002},  //667  800
	},
	{
		{0x1F1F1F1F, 0x0D07070B, 0x00000000, 0x00000000},  //800  667
		{0xFFFFFFFF, 0x05030305, 0x0000FFFF, 0x00000000},  //800  800
	}
	};
	clk_idx = s->selected_timings.mem_clock;
	fsb_idx = s->selected_timings.fsb_clock;

	MCHBAR32(0xc04) = clkcross[fsb_idx][clk_idx][0];
	MCHBAR32(0xc50) = clkcross[fsb_idx][clk_idx][1];
	MCHBAR32(0xc54) = clkcross[fsb_idx][clk_idx][2];
	MCHBAR32(0xc28) = 0;
	MCHBAR32(0xc2c) = clkcross[fsb_idx][clk_idx][3];
	MCHBAR32(0xc08) = MCHBAR32(0xc08) | (1 << 7);

	if ((fsb_idx == 0) && (clk_idx == 1)) {
		MCHBAR8(0x6d4) = 0;
		MCHBAR32(0x700) = 0;
		MCHBAR32(0x704) = 0;
	}

	const u32 clkcross2[2][2][8] = {
	{
		{  0,  0x08010204,  0,  0x08010204, 0, 0,  0,  0x04080102},  //  667  667
		{  0x04080000,  0x10010002,  0x10000000,  0x20010208,  0, 0x00000004,  0x02040000,  0x08100102},  //  667  800
	},
	{
		{  0x10000000,  0x20010208,  0x04080000,  0x10010002,  0, 0,  0x08000000,  0x10200204},  //  800  667
		{  0x00000000,  0x08010204,  0,  0x08010204,  0, 0,  0,  0x04080102},  //  800  800
	}
	};

	MCHBAR32(0x6d8) = clkcross2[fsb_idx][clk_idx][0];
	MCHBAR32(0x6e0) = clkcross2[fsb_idx][clk_idx][0];
	MCHBAR32(0x6e8) = clkcross2[fsb_idx][clk_idx][0];
	MCHBAR32(0x6d8+4) = clkcross2[fsb_idx][clk_idx][1];
	MCHBAR32(0x6e0+4) = clkcross2[fsb_idx][clk_idx][1];
	MCHBAR32(0x6e8+4) = clkcross2[fsb_idx][clk_idx][1];
	MCHBAR32(0x6f0) = clkcross2[fsb_idx][clk_idx][2];
	MCHBAR32(0x6f4) = clkcross2[fsb_idx][clk_idx][3];
	MCHBAR32(0x6f8) = clkcross2[fsb_idx][clk_idx][4];
	MCHBAR32(0x6fc) = clkcross2[fsb_idx][clk_idx][5];
	MCHBAR32(0x708) = clkcross2[fsb_idx][clk_idx][6];
	MCHBAR32(0x70c) = clkcross2[fsb_idx][clk_idx][7];
}

static void sdram_clkmode(struct sysinfo *s)
{
	u8 reg8;
	u16 reg16;

	MCHBAR16(0x1b6) = MCHBAR16(0x1b6) & ~(1 << 8);
	MCHBAR8(0x1b6) = MCHBAR8(0x1b6) & ~0x3f;

	if (s->selected_timings.mem_clock == MEM_CLOCK_667MHz) {
		reg8 = 0;
		reg16 = 1;
	} else {
		reg8 = 1;
		reg16 = (1 << 8) | (1 << 5);
	}
	MCHBAR16(0x1c0) = (MCHBAR16(0x1c0) & ~(0x033f)) | reg16;

	MCHBAR32(0x220) = 0x58001117;
	MCHBAR32(0x248) = (MCHBAR32(0x248) | (1 << 23));

	const u32 cas_to_reg[2][4] = {
		{0x00000000, 0x00030100, 0x0C240201, 0x00000000}, // 667
		{0x00000000, 0x00030100, 0x0C240201, 0x10450302}  // 800
	};

	MCHBAR32(0x224) = cas_to_reg[reg8][s->selected_timings.CAS - 3];
}

static void sdram_timings(struct sysinfo *s)
{
	u8 i, j, ch, r, ta1, ta2, ta3, ta4, trp, bank, page, flag;
	u8 reg8, wl;
	u16 reg16;
	u32 reg32, reg2;
	u8 pagetab[2][2] = {{0xe, 0x12}, {0x10, 0x14}};

	// Only consider DDR2
	wl = s->selected_timings.CAS - 1;
	ta1 = ta2 = 6;
	ta3 = s->selected_timings.CAS;
	ta4 = 8;
	s->selected_timings.tRFC = (s->selected_timings.tRFC + 1) & 0xfe;
	trp = 0;
	bank = 1;
	page = 0;

	MCHBAR8(0x240) = ((wl - 3) << 4) | (s->selected_timings.CAS - 3);

	FOR_EACH_POPULATED_RANK(s->dimms, ch, r) {
		i = ch << 1;
		if (s->dimms[i].banks == 1) {
			trp = 1;
			bank = 0;
		}
		if (s->dimms[i].page_size == 2048) {
			page = 1;
		}
	}
	PRINTK_DEBUG("trp=%d bank=%d page=%d\n",trp, bank, page);

	if (s->selected_timings.mem_clock == MEM_CLOCK_667MHz) {
		flag = 0;
	} else {
		flag = 1;
	}

	MCHBAR8(0x26f) = MCHBAR8(0x26f) | 0x3;
	MCHBAR16(0x250) = ((wl + 4 + s->selected_timings.tWR) << 6) |
				((2 + MAX(s->selected_timings.tRTP, 2)) << 2) | 1;
	reg32 = (bank << 21) | (s->selected_timings.tRRD << 17) |
		(s->selected_timings.tRP << 13) |
		((s->selected_timings.tRP + trp) << 9) |
		s->selected_timings.tRFC;
	if (bank == 0) {
		reg32 |= (pagetab[flag][page] << 22);
	}
	MCHBAR16(0x252) = (u16) reg32;
	MCHBAR16(0x254) = (u16) (reg32 >> 16);

	reg16 = (MCHBAR16(0x254) & 0xfc0) >> 6;
	MCHBAR16(0x62c) = (MCHBAR16(0x62c) & ~0x1f80) | (reg16 << 7);

	reg16 = (s->selected_timings.tRCD << 12) | (4 << 8) | (ta2 << 4) | ta4;
	MCHBAR16(0x256) = reg16;

	reg32 = (s->selected_timings.tRCD << 17) |
		((wl + 4 + s->selected_timings.tWTR) << 12) |
		(ta3 << 8) | (4 << 4) | ta1;
	MCHBAR32(0x258) = reg32;

	reg16 = ((s->selected_timings.tRP + trp) << 9) |
		s->selected_timings.tRFC;
	MCHBAR8(0x25b) = (u8) reg16;
	MCHBAR8(0x25c) = (u8) (reg16 >> 8);

	MCHBAR16(0x260) = (MCHBAR16(0x260) & ~0x3fe) | (100 << 1);
	MCHBAR8(0x25d) = (MCHBAR8(0x25d) & ~0x3f) | s->selected_timings.tRAS;
	MCHBAR16(0x244) = 0x2310;

	MCHBAR8(0x246) = (MCHBAR8(0x246) & ~0x1f) | 1;

	if (s->selected_timings.mem_clock == MEM_CLOCK_667MHz) {
		reg32 = 3000;
	} else {
		reg32 = 2500;
	}
	if (s->selected_timings.fsb_clock == FSB_CLOCK_667MHz) {
		reg2 = 6000;
	} else {
		reg2 = 5000;
	}
	reg16 = (u16)((((s->selected_timings.CAS + 7)*(reg32)) / reg2) << 8);
	MCHBAR16(0x248) = (MCHBAR16(0x248) & ~0x1f00) | reg16;

	flag = 0;
	if (wl > 2) {
		flag = 1;
	}
	reg16 = (u8) (wl - 1 - flag);
	reg16 |= reg16 << 4;
	reg16 |= flag << 8;
	MCHBAR16(0x24d) = (MCHBAR16(0x24d) & ~0x1ff) | reg16;

	MCHBAR16(0x25e) = 0x1585;
	MCHBAR8(0x265) = MCHBAR8(0x265) & ~0x1f;
	MCHBAR16(0x265) = (MCHBAR16(0x265) & ~0x3f00) |
		((s->selected_timings.CAS + 9) << 8);

	if (s->selected_timings.mem_clock == MEM_CLOCK_667MHz) {
		reg16 = 0x514;
		reg32 = 0xa28;
	} else {
		reg16 = 0x618;
		reg32 = 0xc30;
	}
	MCHBAR32(0x268) = (MCHBAR32(0x268) & ~0xfffff00) |
		(0x3f << 22) | (reg32 << 8);
	MCHBAR8(0x26c) = 0x00;
	MCHBAR16(0x2b8) = (MCHBAR16(0x2b8) & 0xc000) | reg16;
	MCHBAR8(0x274) = MCHBAR8(0x274) | 1;

	MCHBAR32(0x248) = (MCHBAR32(0x248) & ~0x7f000000) | (0xb << 25);
	i = s->selected_timings.mem_clock;
	j = s->selected_timings.fsb_clock;
	if (i > j) {
		MCHBAR32(0x248) = MCHBAR32(0x248) | (1 << 24);
	}

	MCHBAR8(0x24c) = MCHBAR8(0x24c) & ~0x3;
	MCHBAR16(0x24d) = (MCHBAR16(0x24d) & ~0x7c00) | ((wl + 10) << 10);
	MCHBAR32(0x260) = (MCHBAR32(0x260) & ~0x70e0000) | (3 << 24) | (3 << 17);
	reg16 = 0x15 << 6;
	reg16 |= 0x1f;
	reg16 |= (0x6 << 12);
	MCHBAR16(0x26d) = (MCHBAR16(0x26d) & ~0x7fff) | reg16;

	reg32 = (0x6 << 27) | (1 << 25);
	MCHBAR32(0x268) = (MCHBAR32(0x268) & ~0x30000000) | ((u32)(reg32 << 8));
	MCHBAR8(0x26c) = (MCHBAR8(0x26c) & ~0xfa) | ((u8)(reg32 >> 24));
	MCHBAR8(0x271) = MCHBAR8(0x271) & ~(1 << 7);
	MCHBAR8(0x274) = MCHBAR8(0x274) & ~0x6;
	reg32 = (u32) ((6 << 30) | (4 << 25) | (1 << 20) | (8 << 15) |
			(6 << 10) | (4 << 5) | 1);
	MCHBAR32(0x278) = reg32;

	MCHBAR16(0x27c) = (MCHBAR16(0x27c) & ~0x1ff) | (8 << 3) | (6 >> 2);
	MCHBAR16(0x125) = MCHBAR16(0x125) | 0x1c00 | (0x1f << 5);
	MCHBAR8(0x127) = (MCHBAR8(0x127) & ~0xff) | 0x40;
	MCHBAR8(0x128) = (MCHBAR8(0x128) & ~0x7) | 0x5;
	MCHBAR8(0x129) = MCHBAR8(0x129) | 0x1f;
	reg8 = 3 << 6;
	reg8 |= (s->dt0mode << 4);
	reg8 |= 0x0c;
	MCHBAR8(0x12f) = (MCHBAR8(0x12f) & ~0xdf) | reg8;
	MCHBAR8(0x18c) = MCHBAR8(0x18c) & ~0x2;
	MCHBAR8(0x228) = (MCHBAR8(0x228) & ~0x7) | 0x2;
	MCHBAR16(0x241) = (MCHBAR16(0x241) & ~0x3fc) | (4 << 2);
	reg32 = (2 << 29) | (1 << 28) | (1 << 23);
	MCHBAR32(0x120) = (MCHBAR32(0x120) & ~0xffb00000) | reg32;

	reg8 = (u8) ((MCHBAR16(0x252) & 0xe000) >> 13);
	reg8 |= (u8) ((MCHBAR16(0x254) & 1) << 3);
	MCHBAR8(0x12d) = (MCHBAR8(0x12d) & ~0xf0) | (reg8 << 4);
	reg8 = (u8) ((MCHBAR32(0x258) & 0xf0000) >> 17);
	MCHBAR8(0x12d) = (MCHBAR8(0x12d) & ~0xf) | reg8;
	MCHBAR8(0x12e) = MCHBAR8(0x12e) & ~0xfc;
	MCHBAR8(0x12e) = MCHBAR8(0x12e) & ~0x3;
	MCHBAR8(0x12f) = MCHBAR8(0x12f) & ~0x3;
	MCHBAR8(0x241) = MCHBAR8(0x241) | 1;
	MCHBAR16(0x1b6) = MCHBAR16(0x1b6) | (1 << 9);
	for (i = 0; i < 8; i++) {
		MCHBAR32(0x540 + i*4) = (MCHBAR32(0x540 + i*4) & ~0x3f3f3f3f) |
			0x0c0c0c0c;
	}
	MCHBAR32(0x248) = (MCHBAR32(0x248) & ~0xf0000) |
		((s->selected_timings.CAS + 1) << 16);
	for (i = 0; i < 8; i++) {
		MCHBAR8(0x560 + i*4) = MCHBAR8(0x560 + i*4) & ~0x3f;
		MCHBAR16(0x58c) = MCHBAR16(0x58c) & ((u16) (~(3 << (i*2))));
		MCHBAR16(0x588) = MCHBAR16(0x588) & ((u16) (~(3 << (i*2))));
		MCHBAR16(0x5fa) = MCHBAR16(0x5fa) & ((u16) (~(3 << (i*2))));
	}
	MCHBAR8(0x5f0) = MCHBAR8(0x5f0) & ~0x1;
	MCHBAR8(0x5f0) = MCHBAR8(0x5f0) | 0x2;
	MCHBAR8(0x5f0) = MCHBAR8(0x5f0) | 0x4;
	MCHBAR32(0x2c0) = MCHBAR32(0x2c0) | 0xc0400;
	MCHBAR32(0x594) = MCHBAR32(0x594) | (1 << 31);
}

static void sdram_p_clkset0(struct pllparam *pll, u8 f, u8 i)
{
	MCHBAR16(0x5a0) = (MCHBAR16(0x5a0) & ~0xc440) |
			(pll->clkdelay[f][i] << 14) |
			(pll->dben[f][i] << 10) |
			(pll->dbsel[f][i] << 6);
	MCHBAR8(0x581) = (MCHBAR8(0x581) & ~0x3f) | pll->pi[f][i];
}

static void sdram_p_clkset1(struct pllparam *pll, u8 f, u8 i)
{
	MCHBAR16(0x5a0) = (MCHBAR16(0x5a0) & ~0x30880) |
			(pll->clkdelay[f][i] << 16) |
			(pll->dben[f][i] << 11) |
			(pll->dbsel[f][i] << 7);
	MCHBAR8(0x582) = (MCHBAR8(0x582) & ~0x3f) | pll->pi[f][i];
}

static void sdram_p_cmd(struct pllparam *pll, u8 f, u8 i)
{
	u8 reg8;
	reg8 = pll->dbsel[f][i] << 5;
	reg8 |= pll->dben[f][i] << 6;
	MCHBAR8(0x594) = (MCHBAR8(0x594) & ~0x60) | reg8;

	reg8 = pll->clkdelay[f][i] << 4;
	MCHBAR8(0x598) = (MCHBAR8(0x598) & ~0x30) | reg8;

	reg8 = pll->pi[f][i];
	MCHBAR8(0x580) = (MCHBAR8(0x580) & ~0x3f) | reg8;
	MCHBAR8(0x583) = (MCHBAR8(0x583) & ~0x3f) | reg8;
}

static void sdram_p_ctrl(struct pllparam *pll, u8 f, u8 i)
{
	u8 reg8;
	u32 reg32;
	reg32 = ((u32) pll->dbsel[f][i]) << 20;
	reg32 |= ((u32) pll->dben[f][i]) << 21;
	reg32 |= ((u32) pll->dbsel[f][i]) << 22;
	reg32 |= ((u32) pll->dben[f][i]) << 23;
	reg32 |= ((u32) pll->clkdelay[f][i]) << 24;
	reg32 |= ((u32) pll->clkdelay[f][i]) << 27;
	MCHBAR32(0x59c) = (MCHBAR32(0x59c) & ~0x1bf0000) | reg32;

	reg8 = pll->pi[f][i];
	MCHBAR8(0x584) = (MCHBAR8(0x584) & ~0x3f) | reg8;
	MCHBAR8(0x585) = (MCHBAR8(0x585) & ~0x3f) | reg8;

	reg32 = ((u32) pll->dbsel[f][i]) << 12;
	reg32 |= ((u32) pll->dben[f][i]) << 13;
	reg32 |= ((u32) pll->dbsel[f][i]) << 8;
	reg32 |= ((u32) pll->dben[f][i]) << 9;
	reg32 |= ((u32) pll->clkdelay[f][i]) << 14;
	reg32 |= ((u32) pll->clkdelay[f][i]) << 10;
	MCHBAR32(0x598) = (MCHBAR32(0x598) & ~0xff00) | reg32;

	reg8 = pll->pi[f][i];
	MCHBAR8(0x586) = (MCHBAR8(0x586) & ~0x3f) | reg8;
	MCHBAR8(0x587) = (MCHBAR8(0x587) & ~0x3f) | reg8;
}

static void sdram_p_dqs(struct pllparam *pll, u8 f, u8 clk)
{
	u8 rank, dqs, reg8, j;
	u32 reg32;

	j = clk - 40;
	reg8 = 0;
	reg32 = 0;
	rank = j % 4;
	dqs = j / 4;

	reg32 |= ((u32) pll->dben[f][clk]) << (dqs + 9);
	reg32 |= ((u32) pll->dbsel[f][clk]) << dqs;
	MCHBAR32(0x5b4+rank*4) = (MCHBAR32(0x5b4+rank*4) &
		~( (1 << (dqs+9))|(1 << dqs) )) | reg32;

	reg32 = ((u32) pll->clkdelay[f][clk]) << ((dqs*2) + 16);
	MCHBAR32(0x5c8+rank*4) = (MCHBAR32(0x5c8+rank*4) &
		~( (1 << (dqs*2 + 17))|(1 << (dqs*2 + 16)) )) | reg32;

	reg8 = pll->pi[f][clk];
	MCHBAR8(0x520+j) = (MCHBAR8(0x520+j) & ~0x3f) | reg8;
}


static void sdram_p_dq(struct pllparam *pll, u8 f, u8 clk)
{
	u8 rank, dq, reg8, j;
	u32 reg32;

	j = clk - 8;
	reg8 = 0;
	reg32 = 0;
	rank = j % 4;
	dq = j / 4;

	reg32 |= ((u32) pll->dben[f][clk]) << (dq + 9);
	reg32 |= ((u32) pll->dbsel[f][clk]) << dq;
	MCHBAR32(0x5a4+rank*4) = (MCHBAR32(0x5a4+rank*4) &
		~( (1 << (dq+9))|(1 << dq) )) | reg32;

	reg32 = ((u32) pll->clkdelay[f][clk]) << (dq*2);
	MCHBAR32(0x5c8+rank*4) = (MCHBAR32(0x5c8+rank*4) &
		~( (1 << (dq*2 + 1))|(1 << (dq*2)) )) | reg32;

	reg8 = pll->pi[f][clk];
	MCHBAR8(0x500+j) = (MCHBAR8(0x500+j) & ~0x3f) | reg8;
}

static void sdram_calibratepll(struct sysinfo *s, u8 pidelay)
{
	struct pllparam pll = {
		.pi = {
		{	// 667
			3, 3, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
			7, 7, 7, 7, 4, 4, 4, 4, 4, 4, 4, 4,
			4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5,
			7, 7, 7, 7, 3, 3, 3, 3, 3, 3, 3, 3,
			0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
			0, 0, 0, 0, 1, 1, 1, 1, 3, 3, 3, 3
		},
		{	// 800
			53, 53, 10, 10, 5,  5,  5,  5,  27, 27, 27, 27,
			34, 34, 34, 34, 34, 34, 34, 34, 39, 39, 39, 39,
			47, 47, 47, 47, 44, 44, 44, 44, 47, 47, 47, 47,
			47, 47, 47, 47, 59, 59, 59, 59, 2,  2,  2,  2,
			2,  2,  2,  2,  7,  7,  7,  7,  15, 15, 15, 15,
			12, 12, 12, 12, 15, 15, 15, 15, 15, 15, 15, 15
		}},

		.dben = {
		{	// 667
			0,0,1,1,1,1,1,1,1,1,1,1,
			1,1,1,1,0,0,0,0,0,0,0,0,
			0,0,0,0,0,0,0,0,0,0,0,0,
			1,1,1,1,0,0,0,0,0,0,0,0,
			0,0,0,0,0,0,0,0,0,0,0,0,
			0,0,0,0,0,0,0,0,0,0,0,0
		},
		{	// 800
			1,1,1,1,1,1,1,1,0,0,0,0,
			0,0,0,0,0,0,0,0,0,0,0,0,
			0,0,0,0,0,0,0,0,0,0,0,0,
			0,0,0,0,1,1,1,1,1,1,1,1,
			1,1,1,1,1,1,1,1,0,0,0,0,
			1,1,1,1,0,0,0,0,0,0,0,0
		}},

		.dbsel = {
		{	// 667
			0,0,1,1,1,1,1,1,1,1,1,1,
			1,1,1,1,0,0,0,0,0,0,0,0,
			0,0,0,0,0,0,0,0,0,0,0,0,
			1,1,1,1,0,0,0,0,0,0,0,0,
			0,0,0,0,0,0,0,0,0,0,0,0,
			0,0,0,0,0,0,0,0,0,0,0,0
		},
		{	// 800
			0,0,1,1,1,1,1,1,0,0,0,0,
			0,0,0,0,0,0,0,0,0,0,0,0,
			0,0,0,0,0,0,0,0,0,0,0,0,
			0,0,0,0,0,0,0,0,1,1,1,1,
			1,1,1,1,1,1,1,1,0,0,0,0,
			1,1,1,1,0,0,0,0,0,0,0,0
		}},

		.clkdelay = {
		{	// 667
			0,0,1,1,0,0,0,0,1,1,1,1,
			1,1,1,1,1,1,1,1,1,1,1,1,
			1,1,1,1,1,1,1,1,1,1,1,1,
			1,1,1,1,0,0,0,0,0,0,0,0,
			0,0,0,0,0,0,0,0,0,0,0,0,
			0,0,0,0,0,0,0,0,0,0,0,0
		},
		{	// 800
			0,0,0,0,0,0,0,0,1,1,1,1,
			1,1,1,1,1,1,1,1,1,1,1,1,
			1,1,1,1,1,1,1,1,1,1,1,1,
			1,1,1,1,0,0,0,0,1,1,1,1,
			1,1,1,1,1,1,1,1,1,1,1,1,
			1,1,1,1,1,1,1,1,1,1,1,1
		}}
	};

	u8 i, f;
	if (s->selected_timings.mem_clock == MEM_CLOCK_667MHz) {
		f = 0;
	} else {
		f = 1;
	}
	for (i = 0; i < 72; i++) {
		pll.pi[f][i] += pidelay;
	}

	MCHBAR8(0x1a4) = MCHBAR8(0x1a4) & ~(1 << 7);
	MCHBAR16(0x190) = (MCHBAR16(0x190) & (u16) ~(0x3fff)) | 0x1fff;

	sdram_p_clkset0(&pll, f, 0);
	sdram_p_clkset1(&pll, f, 1);
	sdram_p_cmd(&pll, f, 2);
	sdram_p_ctrl(&pll, f, 4);
	for (i = 0; i < 32; i++) {
		sdram_p_dqs(&pll, f, i+40);
	}
	for (i = 0; i < 32; i++) {
		sdram_p_dq(&pll, f, i+8);
	}
}

static void sdram_calibratehwpll(struct sysinfo *s)
{
	u8 reg8;

	s->async = 0;
	reg8 = 0;
	MCHBAR16(0x180) = MCHBAR16(0x180) | (1 << 15);
	MCHBAR8(0x180) = MCHBAR8(0x180) & ~(1 << 7);
	MCHBAR8(0x180) = MCHBAR8(0x180) | (1 << 3);
	MCHBAR8(0x180) = MCHBAR8(0x180) | (1 << 2);

	MCHBAR8(0x180) = MCHBAR8(0x180) | (1 << 7);
	while ((MCHBAR8(0x180) & (1 << 2)) == 0);

	reg8 = (MCHBAR8(0x180) & (1 << 3)) >> 3;
	if (reg8 != 0) {
		s->async = 1;
	}
}

static void sdram_dlltiming(struct sysinfo *s)
{
	u8 reg8, i, pipost;
	u16 reg16;
	u32 reg32;

	if (s->selected_timings.mem_clock == MEM_CLOCK_667MHz) {
		reg32 = 0x8014227;
	} else {
		reg32 = 0x14221;
	}
	MCHBAR32(0x19c) = (MCHBAR32(0x19c) & ~0xfffffff) | reg32;
	MCHBAR32(0x19c) = MCHBAR32(0x19c) | (1 << 23);
	MCHBAR32(0x19c) = MCHBAR32(0x19c) | (1 << 15);
	MCHBAR32(0x19c) = MCHBAR32(0x19c) & ~(1 << 15);

	if (s->nodll) {
		MCHBAR16(0x198) = MCHBAR16(0x198) | (1 << 0);
		MCHBAR16(0x198) = MCHBAR16(0x198) | (1 << 2);
		MCHBAR16(0x198) = MCHBAR16(0x198) | (1 << 4);
		MCHBAR16(0x198) = MCHBAR16(0x198) | (1 << 8);
		MCHBAR16(0x198) = MCHBAR16(0x198) | (1 << 10);
		MCHBAR16(0x198) = MCHBAR16(0x198) | (1 << 12);
		MCHBAR16(0x198) = MCHBAR16(0x198) | (1 << 14);
	} else {
		MCHBAR16(0x198) = MCHBAR16(0x198) & ~(1 << 0);
		MCHBAR16(0x198) = MCHBAR16(0x198) & ~(1 << 2);
		MCHBAR16(0x198) = MCHBAR16(0x198) & ~(1 << 4);
		MCHBAR16(0x198) = MCHBAR16(0x198) & ~(1 << 8);
		MCHBAR16(0x198) = MCHBAR16(0x198) & ~(1 << 10);
		MCHBAR16(0x198) = MCHBAR16(0x198) & ~(1 << 12);
		MCHBAR16(0x198) = MCHBAR16(0x198) & ~(1 << 14);
	}

	if (s->nodll) {
		MCHBAR8(0x1c8) = (MCHBAR8(0x1c8) & ~0x3f) | 0x7;
	} else {
		MCHBAR8(0x1c8) = (MCHBAR8(0x1c8) & ~0x3f);
	}

	sdram_calibratepll(s, 0); // XXX check

	MCHBAR16(0x5f0) = MCHBAR16(0x5f0) | (1 << 11);
	MCHBAR16(0x5f0) = MCHBAR16(0x5f0) | (1 << 12);

	for (i = 0; i < 8; i++) {
		MCHBAR16(0x5f0) = MCHBAR16(0x5f0) | ((1 << 10) >> i);
	}
	MCHBAR8(0x2c14) = MCHBAR8(0x2c14) | 1;
	MCHBAR16(0x182) = 0x5005;
	MCHBAR16(0x18a) = (MCHBAR16(0x18a) & ~0x1f1f) | 0x51a;
	MCHBAR16(0x2c00) = (MCHBAR16(0x2c00) & ~0xbf3f) | 0x9010;

	if (s->nodll) {
		MCHBAR8(0x18e) = (MCHBAR8(0x18e) & ~0x7f) | 0x6b;
	} else {
		MCHBAR8(0x18e) = (MCHBAR8(0x18e) & ~0x7f) | 0x55;
		sdram_calibratehwpll(s);
	}
	pipost = 0x34;

	MCHBAR32(0x248) = MCHBAR32(0x248) & ~(1 << 22);
	MCHBAR8(0x5d9) = MCHBAR8(0x5d9) & ~0x2;
	MCHBAR8(0x189) = MCHBAR8(0x189) | 0xc0;
	MCHBAR8(0x189) = MCHBAR8(0x189) & ~(1 << 5);
	MCHBAR8(0x189) = (MCHBAR8(0x189) & ~0xc0) | (1 << 6);
	MCHBAR8(0x188) = (MCHBAR8(0x188) & ~0x3f) | 0x1a;
	MCHBAR8(0x188) = MCHBAR8(0x188) | 1;

	MCHBAR8(0x1a8) = MCHBAR8(0x1a8) | 1;
	MCHBAR32(0x1a0) = 0x551803;
	if (ONLY_DIMMA_IS_POPULATED(s->dimms, 0)) {
		reg8 = 0x3c;
	} else if (ONLY_DIMMB_IS_POPULATED(s->dimms, 0)) {
		reg8 = 0x27;
	} else if (BOTH_DIMMS_ARE_POPULATED(s->dimms, 0)) {
		reg8 = 0x24;
	} else {
		// None
		reg8 = 0x3f;
	}
	reg8 = 0x00; //switch all clocks on anyway

	MCHBAR32(0x5a0) = (MCHBAR32(0x5a0) & ~0x3f000000) | (reg8 << 24);
	MCHBAR8(0x594) = MCHBAR8(0x594) & ~1;
	reg16 = 0;
	if (!rank_is_populated(s->dimms, 0, 0)) {
		reg16 |= (1 << 8) | (1 << 4) | (1 << 0);
	}
	if (!rank_is_populated(s->dimms, 0, 1)) {
		reg16 |= (1 << 9) | (1 << 5) | (1 << 1);
	}
	if (!rank_is_populated(s->dimms, 0, 2)) {
		reg16 |= (1 << 10) | (1 << 6) | (1 << 2);
	}
	if (!rank_is_populated(s->dimms, 0, 3)) {
		reg16 |= (1 << 11) | (1 << 7) | (1 << 3);
	}
	MCHBAR16(0x59c) = MCHBAR16(0x59c) | reg16;
}

static void sdram_rcomp(struct sysinfo *s)
{
	u8 i, j, reg8, f, rcompp, rcompn, srup, srun;
	u16 reg16;
	u32 reg32, rcomp1, rcomp2;

	u8 rcompupdate[7] = { 0, 0, 0, 1, 1, 0, 0 };
	u8 rcompslew = 0xa;
	u8 rcompstr[7] = { 0x66, 0, 0xaa, 0x55, 0x55, 0x77, 0x77 };
	u16 rcompscomp[7] = { 0xa22a, 0, 0xe22e, 0xe22e, 0xe22e, 0xa22a, 0xa22a };
	u8 rcompdelay[7] = { 1, 0, 0, 0, 0, 1, 1 };
	u16 rcompctl[7] = { 0x31c, 0, 0x374, 0x3a2, 0x3d0, 0x3fe, 0x42c };
	u16 rcompf[7] = { 0x1114, 0, 0x0505, 0x0909, 0x0909, 0x0a0a, 0x0a0a };

	//                   NC-NC   x16SS   x16DS  x16SS2  x16DS2 x8DS, x8DS2
	u8 rcompstr2[7]    = {   0x00,   0x55,   0x55,   0xaa,   0xaa , 0x55,   0xaa};
	u16 rcompscomp2[7] = { 0x0000, 0xe22e, 0xe22e, 0xe22e, 0x8228 , 0xe22e, 0x8228 };
	u8 rcompdelay2[7]  = {      0,      0,      0,      0,      2 , 0,      2};

	u8 rcomplut[64][12] = {
		{ 9, 9,11,11, 2, 2, 5,5, 6, 6,5, 5},
		{ 9, 9,11, 11, 2, 2, 5,5, 6, 6,5, 5},
		{ 9, 9,11, 11, 2, 2, 5,5, 6, 6,5, 5},
		{ 9, 9,11, 11, 2, 2, 5,5, 6, 6,5, 5},
		{ 9, 9,11, 11, 2, 2, 5,5, 6, 6,5, 5},
		{ 9, 9,11, 11, 2, 2, 5,5, 6, 6,5, 5},
		{ 9, 9,11, 11, 2, 2, 5,5, 6, 6,5, 5},
		{ 9, 9,11, 11, 2, 2, 5,5, 6, 6,5, 5},
		{ 9, 9,11, 11, 2, 2, 5,5, 6, 6,5, 5},
		{ 9, 9,11, 11, 2, 2, 5,5, 6, 6,5, 5},
		{ 9, 9,11, 11, 2, 2, 5,5, 6, 6,5, 5},
		{ 9, 9,11, 11, 2, 2, 5,5, 6, 6,5, 5},
		{ 9, 9,11, 11, 2, 2, 5,5, 6, 6,5, 5},
		{ 9, 9,11, 11, 2, 2, 5,5, 6, 6,5, 5},
		{ 9, 9,11, 11, 2, 2, 5,5, 6, 6,5, 5},
		{ 10,9,12, 11, 2, 2, 5,5, 6, 6,5, 5},
		{ 10,9,12, 11, 2, 2, 6,5, 7, 6,6, 5},
		{ 10,10,12, 12, 2, 2, 6,5, 7, 6,6, 5},
		{ 10,10,12, 12, 2, 2, 6,6, 7, 7,6, 6},
		{ 10,10,12, 12, 3, 2, 6,6, 7, 7,6, 6},
		{ 10,10,12, 12, 3, 2, 6,6, 7, 7,6, 6},
		{ 10,10,12, 12, 3, 2, 6,6, 7, 7,6, 6},
		{ 10,10,12, 12, 3, 3, 6,6, 7, 7,6, 6},
		{ 10,10,12, 12, 3, 3, 6,6, 7, 7,6, 6},
		{ 10,10,12, 12, 3, 3, 6,6, 7, 7,6, 6},
		{ 10,10,12, 12, 3, 3, 6,6, 7, 7,6, 6},
		{ 10,10,12, 12, 3, 3, 6,6, 7, 7,6, 6},
		{ 11,10,12, 12, 3, 3, 6,6, 7, 7,6, 6},
		{ 11,10,14, 13, 3, 3, 6,6, 7, 7,6, 6},
		{ 12,10,14, 13, 3, 3, 6,6, 7, 7,6, 6},
		{ 12,12,14, 13, 3, 3, 7,6, 7, 7,7, 6},
		{ 13,12,16, 15, 3, 3, 7,6, 8, 7,7, 6},
		{ 13,14,16, 15, 4, 3, 7,7, 8, 8,7, 7},
		{ 14,14,16, 17, 4, 3, 7,7, 8, 8,7, 7},
		{ 14,16,18, 17, 4, 4, 8,7, 8, 8,8, 7},
		{ 15,16,18, 19, 4, 4, 8,7, 9, 8,8, 7},
		{ 15,18,18, 19, 4, 4, 8,8, 9, 9,8, 8},
		{ 16,18,20, 21, 4, 4, 8,8, 9, 9,8, 8},
		{ 16,19,20, 21, 5, 4, 9,8, 10, 9,9, 8},
		{ 16,19,20, 23, 5, 5, 9,9, 10, 10,9, 9},
		{ 17,19,22, 23, 5, 5, 9,9, 10, 10,9, 9},
		{ 17,20,22, 25, 5, 5, 9,9, 10, 10,9, 9},
		{ 17,20,22, 25, 5, 5, 9,9, 10, 10,9, 9},
		{ 18,20,22, 25, 5, 5, 9,9, 10, 10,9, 9},
		{ 18,21,24, 25, 5, 5, 9,9, 11, 10,9, 9},
		{ 19,21,24, 27, 5, 5, 9, 9, 11, 11,9, 9},
		{ 19,22,24, 27, 5, 5, 10,9, 11, 11,10, 9},
		{ 20,22,24, 27, 6, 5, 10,10, 11, 11,10, 10},
		{ 20,23,26, 27, 6, 6, 10,10, 12, 12,10, 10},
		{ 20,23,26, 29, 6, 6, 10,10, 12, 12,10, 10},
		{ 21,24,26, 29, 6, 6, 10,10, 12, 12,10, 10},
		{ 21,24,26, 29, 6, 6, 11,10, 12, 13,11, 10},
		{ 22,25,28, 29, 6, 6, 11,11, 13, 13,11, 11},
		{ 22,25,28, 31, 6, 6, 11,11, 13, 13,11, 11},
		{ 22,26,28, 31, 6, 6, 11,11, 13, 14,11, 11},
		{ 23,26,30, 31, 7, 6, 12,11, 14, 14,12, 11},
		{ 23,27,30, 33, 7, 7, 12,12, 14, 14,12, 12},
		{ 23,27,30, 33, 7, 7, 12,12, 14, 15,12, 12},
		{ 24,28,32, 33, 7, 7, 12,12, 15, 15,12, 12},
		{ 24,28,32, 33, 7, 7, 12,12, 15, 16,12, 12},
		{ 24,29,32, 35, 7, 7, 12,12, 15, 16,12, 12},
		{ 25,29,32, 35, 7, 7, 12,12, 15, 17,12, 12},
		{ 25,30,32, 35, 7, 7, 12,12, 15, 17,12, 12},
		{ 25,30,32, 35, 7, 7, 12,12, 15, 17,12, 12},
	};

	srup = 0;
	srun = 0;

	if (s->selected_timings.mem_clock == MEM_CLOCK_667MHz) {
		f = 0;
		rcomp1 = 0x00050431;
	} else {
		f = 1;
		rcomp1 = 0x00050542;
	}
	if (s->selected_timings.fsb_clock == FSB_CLOCK_667MHz) {
		rcomp2 = 0x14C42827;
	} else {
		rcomp2 = 0x19042827;
	}

	for (i = 0; i < 7; i++) {
		if (i == 1)
			continue;
		reg8 = rcompupdate[i];
		MCHBAR8(rcompctl[i]) = (MCHBAR8(rcompctl[i]) & ~0x1) | reg8;
		MCHBAR8(rcompctl[i]) = MCHBAR8(rcompctl[i]) & ~0x2;
		reg16 = (u16) rcompslew;
		MCHBAR16(rcompctl[i]) = (MCHBAR16(rcompctl[i]) & ~0xf000) |
				(reg16 << 12);
		MCHBAR8(rcompctl[i]+4) = rcompstr[i];
		MCHBAR16(rcompctl[i]+0xe) = rcompscomp[i];
		MCHBAR8(rcompctl[i]+0x14) = (MCHBAR8(rcompctl[i]+0x14) & ~0x3) |
				rcompdelay[i];
		if (i == 2) {
			reg16 = (u16) rcompslew;
			MCHBAR16(rcompctl[i]) = (MCHBAR16(rcompctl[i]) &
					~0xf000) | (reg16 << 12);
			MCHBAR8(rcompctl[i]+4) = rcompstr2[s->dimm_config[0]];
			MCHBAR16(rcompctl[i]+0xe) = rcompscomp2[s->dimm_config[0]];
			MCHBAR8(rcompctl[i]+0x14) = (MCHBAR8(rcompctl[i]+0x14) &
					~0x3) | rcompdelay2[s->dimm_config[0]];
		}

		MCHBAR16(rcompctl[i]+0x16) = MCHBAR16(rcompctl[i]+0x16) & ~0x7f7f;
		MCHBAR16(rcompctl[i]+0x18) = MCHBAR16(rcompctl[i]+0x18) & ~0x3f3f;
		MCHBAR16(rcompctl[i]+0x1a) = MCHBAR16(rcompctl[i]+0x1a) & ~0x3f3f;
		MCHBAR16(rcompctl[i]+0x1c) = MCHBAR16(rcompctl[i]+0x1c) & ~0x3f3f;
		MCHBAR16(rcompctl[i]+0x1e) = MCHBAR16(rcompctl[i]+0x1e) & ~0x3f3f;
	}

	MCHBAR8(0x45a) = (MCHBAR8(0x45a) & ~0x3f) | 0x36;
	MCHBAR8(0x462) = (MCHBAR8(0x462) & ~0x3f) | 0x36;

	for (i = 0; i < 7; i++) {
		if (i == 1)
			continue;
		MCHBAR8(rcompctl[i]) = MCHBAR8(rcompctl[i]) & ~0x60;
		MCHBAR16(rcompctl[i]+2) = MCHBAR16(rcompctl[i]+2) & ~0x706;
		MCHBAR16(rcompctl[i]+0xa) = MCHBAR16(rcompctl[i]+0xa) & ~0x7f7f;
		MCHBAR16(rcompctl[i]+0x12) = MCHBAR16(rcompctl[i]+0x12) & ~0x3f3f;
		MCHBAR16(rcompctl[i]+0x24) = MCHBAR16(rcompctl[i]+0x24) & ~0x1f1f;
		MCHBAR8(rcompctl[i]+0x26) = MCHBAR8(rcompctl[i]+0x26) & ~0x1f;
	}

	MCHBAR16(0x45a) = MCHBAR16(0x45a) & ~0xffc0;
	MCHBAR16(0x45c) = MCHBAR16(0x45c) & ~0xf;
	MCHBAR16(0x462) = MCHBAR16(0x462) & ~0xffc0;
	MCHBAR16(0x464) = MCHBAR16(0x464) & ~0xf;

	for (i = 0; i < 7; i++) {
		if (i == 1)
			continue;
		MCHBAR16(rcompctl[i]+0x10) = rcompf[i];
		MCHBAR16(rcompctl[i]+0x20) = 0x1219;
		MCHBAR16(rcompctl[i]+0x22) = 0x000C;
	}

	MCHBAR32(0x164) = (MCHBAR32(0x164) & ~0x1f1f1f) | 0x0c1219;
	MCHBAR16(0x4b0) = (MCHBAR16(0x4b0) & ~0x1f00) | 0x1200;
	MCHBAR8(0x4b0) = (MCHBAR8(0x4b0) & ~0x1f) | 0x12;
	MCHBAR32(0x138) = 0x007C9007;
	MCHBAR32(0x16c) = rcomp1;
	MCHBAR16(0x17a) = 0x1f7f;
	MCHBAR32(0x134) = rcomp2;
	MCHBAR16(0x170) = (MCHBAR16(0x170) & ~0xf) | 1;
	MCHBAR16(0x178) = 0x134;
	MCHBAR32(0x130) = 0x4C293600;
	MCHBAR8(0x133) = (MCHBAR8(0x133) & ~0x44) | (1 << 6) | (1 << 2);
	MCHBAR16(0x4b0) = MCHBAR16(0x4b0) & ~(1 << 13);
	MCHBAR8(0x4b0) = MCHBAR8(0x4b0) & ~(1 << 5);

	for (i = 0; i < 7; i++) {
		if (i == 1)
			continue;
		MCHBAR8(rcompctl[i]+2) = MCHBAR8(rcompctl[i]) & ~0x71;
	}

	if ((MCHBAR32(0x130) & (1 << 30)) == 0) {
		MCHBAR8(0x130) = MCHBAR8(0x130) | 0x1;
		while ((MCHBAR8(0x130) & 0x1) != 0);

		reg32 = MCHBAR32(0x13c);
		rcompp = (u8) ((reg32 & ~(1 << 31)) >> 24);
		rcompn = (u8) ((reg32 & ~(0xff800000)) >> 16);

		for (i = 0; i < 7; i++) {
			if (i == 1)
				continue;
			srup = (MCHBAR8(rcompctl[i]+1) & 0xc0) >> 6;
			srun = (MCHBAR8(rcompctl[i]+1) & 0x30) >> 4;
			reg16 = (u16)(rcompp - (1 << (srup + 1))) << 8;
			MCHBAR16(rcompctl[i]+0x16) = (MCHBAR16(rcompctl[i]+0x16)
							& ~0x7f00) | reg16;
			reg16 = (u16)(rcompn - (1 << (srun + 1)));
			MCHBAR8(rcompctl[i]+0x16) = (MCHBAR8(rcompctl[i]+0x16) &
							~0x7f) | (u8)reg16;
		}

		reg8 = rcompp - (1 << (srup + 1));
		for (i = 0, j = reg8; i < 4; i++, j += (1 << srup)) {
			MCHBAR8(rcompctl[0]+0x18+i) =
					(MCHBAR8(rcompctl[0]+0x18+i) & ~0x3f) |
					rcomplut[j][0];
		}

		for (i = 0, j = reg8; i < 4; i++, j += (1 << srup)) {
			if (s->dimm_config[0] < 3 || s->dimm_config[0] == 5) {
				MCHBAR8(rcompctl[2]+0x18+i) =
					(MCHBAR8(rcompctl[2]+0x18+i) & ~0x3f) |
					rcomplut[j][10];
			}
		}

		for (i = 0, j = reg8; i < 4; i++, j += (1 << srup)) {
			MCHBAR8(rcompctl[3]+0x18+i) =
					(MCHBAR8(rcompctl[3]+0x18+i) & ~0x3f) |
					rcomplut[j][6];
			MCHBAR8(rcompctl[4]+0x18+i) =
					(MCHBAR8(rcompctl[4]+0x18+i) & ~0x3f) |
					rcomplut[j][6];
		}

		for (i = 0, j = reg8; i < 4; i++, j += (1 << srup)) {
			MCHBAR8(rcompctl[5]+0x18+i) =
					(MCHBAR8(rcompctl[5]+0x18+i) & ~0x3f) |
					rcomplut[j][8];
			MCHBAR8(rcompctl[6]+0x18+i) =
					(MCHBAR8(rcompctl[6]+0x18+i) & ~0x3f) |
					rcomplut[j][8];
		}

		reg8 = rcompn - (1 << (srun + 1));
		for (i = 0, j = reg8; i < 4; i++, j += (1 << srun)) {
			MCHBAR8(rcompctl[0]+0x1c+i) =
					(MCHBAR8(rcompctl[0]+0x1c+i) & ~0x3f) |
					rcomplut[j][1];
		}

		for (i = 0, j = reg8; i < 4; i++, j += (1 << srun)) {
			if (s->dimm_config[0] < 3 || s->dimm_config[0] == 5) {
				MCHBAR8(rcompctl[2]+0x1c+i) =
					(MCHBAR8(rcompctl[2]+0x1c+i) & ~0x3f) |
					rcomplut[j][11];
			}
		}

		for (i = 0, j = reg8; i < 4; i++, j += (1 << srun)) {
			MCHBAR8(rcompctl[3]+0x1c+i) =
					(MCHBAR8(rcompctl[3]+0x1c+i) & ~0x3f) |
					rcomplut[j][7];
			MCHBAR8(rcompctl[4]+0x1c+i) =
					(MCHBAR8(rcompctl[4]+0x1c+i) & ~0x3f) |
					rcomplut[j][7];
		}

		for (i = 0, j = reg8; i < 4; i++, j += (1 << srun)) {
			MCHBAR8(rcompctl[5]+0x1c+i) =
					(MCHBAR8(rcompctl[5]+0x1c+i) & ~0x3f) |
					rcomplut[j][9];
			MCHBAR8(rcompctl[6]+0x1c+i) =
					(MCHBAR8(rcompctl[6]+0x1c+i) & ~0x3f) |
					rcomplut[j][9];
		}
	}
	MCHBAR8(0x130) = MCHBAR8(0x130) | 1;
}

static void sdram_odt(struct sysinfo *s)
{
	u8 rankindex = 0;

	u16 odt294[16] = {
			0x0000, 0x0000, 0x0000, 0x0000,
			0x0044, 0x1111, 0x0000, 0x1111,
			0x0000, 0x0000, 0x0000, 0x0000,
			0x0044, 0x1111, 0x0000, 0x1111
			};
	u16 odt298[16] = {
			0x0000, 0x0011, 0x0000, 0x0011,
			0x0000, 0x4444, 0x0000, 0x4444,
			0x0000, 0x0000, 0x0000, 0x0000,
			0x0000, 0x4444, 0x0000, 0x4444
			};

	switch (s->dimms[0].ranks) {
	case 0:
		if (s->dimms[1].ranks == 0) {
			rankindex = 0;
		} else if (s->dimms[1].ranks == 1) {
			rankindex = 4;
		} else if (s->dimms[1].ranks == 2) {
			rankindex = 12;
		}
		break;
	case 1:
		if (s->dimms[1].ranks == 0) {
			rankindex = 1;
		} else if (s->dimms[1].ranks == 1) {
			rankindex = 5;
		} else if (s->dimms[1].ranks == 2) {
			rankindex = 13;
		}
		break;
	case 2:
		if (s->dimms[1].ranks == 0) {
			rankindex = 3;
		} else if (s->dimms[1].ranks == 1) {
			rankindex = 7;
		} else if (s->dimms[1].ranks == 2) {
			rankindex = 15;
		}
		break;
	}

	MCHBAR16(0x298) = odt298[rankindex];
	MCHBAR16(0x294) = odt294[rankindex];
}

static void sdram_mmap(struct sysinfo *s)
{
	u32 w260[7] = {0, 0x400001, 0xc00001, 0x500000, 0xf00000, 0xc00001, 0xf00000};
	u32 w208[7] = {0, 0x10000, 0x1010000, 0x10001, 0x1010101, 0x1010000, 0x1010101};
	u32 w200[7] = {0, 0, 0, 0x20002, 0x40002, 0, 0x40002};
	u32 w204[7] = {0, 0x20002, 0x40002, 0x40004, 0x80006, 0x40002, 0x80006};

	u16 tolud[7] = {0x800, 0x800, 0x1000, 0x1000, 0x2000, 0x1000, 0x2000};
	u16 tom[7] = {0x2, 0x2, 0x4, 0x4, 0x8, 0x4, 0x8};
	u16 touud[7] = {0x80, 0x80, 0x100, 0x100, 0x200, 0x100, 0x200};
	u32 gbsm[7] = {0x8000000, 0x8000000, 0x10000000, 0x8000000, 0x20000000, 0x10000000, 0x20000000};
	u32 bgsm[7] = {0x8000000, 0x8000000, 0x10000000, 0x8000000, 0x20000000, 0x10000000, 0x20000000};
	u32 tsegmb[7] = {0x8000000, 0x8000000, 0x10000000, 0x8000000, 0x20000000, 0x10000000, 0x20000000};

	if ((s->dimm_config[0] < 3) && rank_is_populated(s->dimms, 0, 0)) {
		if (s->dimms[0].sides > 1) {
			// 2R/NC
			MCHBAR32(0x260) = (MCHBAR32(0x260) & ~0x1) | 0x300001;
			MCHBAR32(0x208) = 0x101;
			MCHBAR32(0x200) = 0x40002;
			MCHBAR32(0x204) = w204[s->dimm_config[0]];
		} else {
			// 1R/NC
			MCHBAR32(0x260) = (MCHBAR32(0x260) & ~0x1) | 0x100001;
			MCHBAR32(0x208) = 0x1;
			MCHBAR32(0x200) = 0x20002;
			MCHBAR32(0x204) = w204[s->dimm_config[0]];
		}
	} else if ((s->dimm_config[0] == 5) && rank_is_populated(s->dimms, 0, 0)) {

		MCHBAR32(0x260) = (MCHBAR32(0x260) & ~0x1) | 0x300001;
		MCHBAR32(0x208) = 0x101;
		MCHBAR32(0x200) = 0x40002;
		MCHBAR32(0x204) = 0x40004;
	} else {
		MCHBAR32(0x260) = (MCHBAR32(0x260) & ~0x1) | w260[s->dimm_config[0]];
		MCHBAR32(0x208) = w208[s->dimm_config[0]];
		MCHBAR32(0x200) = w200[s->dimm_config[0]];
		MCHBAR32(0x204) = w204[s->dimm_config[0]];
	}
	pci_write_config16(PCI_DEV(0, 0, 0), 0xb0, tolud[s->dimm_config[0]]);
	pci_write_config16(PCI_DEV(0, 0, 0), 0xa0, tom[s->dimm_config[0]]);
	pci_write_config16(PCI_DEV(0, 0, 0), 0xa2, touud[s->dimm_config[0]]);
	pci_write_config32(PCI_DEV(0, 0, 0), 0xa4, gbsm[s->dimm_config[0]]);
	pci_write_config32(PCI_DEV(0, 0, 0), 0xa8, bgsm[s->dimm_config[0]]);
	pci_write_config32(PCI_DEV(0, 0, 0), 0xac, tsegmb[s->dimm_config[0]]);
}

#if 1
static void hpet_udelay(u32 del)
{
	u32 start, finish, now;

	del *= 15; /* now in usec */

	start = HPET32(0xf0);
	finish = start + del;
	while (1) {
		now = HPET32(0xf0);
		if (finish > start) {
			if (now >= finish)
				break;
		} else {
			if ((now < start) && (now >= finish)) {
				break;
			}
		}
	}
}
#endif

static u8 sdram_checkrcompoverride(void)
{
	u32 xcomp;
	u8 aa, bb, a, b, c, d;

	xcomp = MCHBAR32(0x13c);
	a = (u8)((xcomp & 0x7f000000) >> 24);
	b = (u8)((xcomp & 0x7f0000) >> 16);
	c = (u8)((xcomp & 0x3f00) >> 8);
	d = (u8)(xcomp & 0x3f);

	if (a > b) {
		aa = a - b;
	} else {
		aa = b - a;
	}
	if (c > d) {
		bb = c - d;
	} else {
		bb = d - c;
	}
	if ((aa > 18) || (bb > 7) ||
			(a <= 5) || (b <= 5) || (c <= 5) || (d <= 5) ||
			(a >= 0x7a) || (b >= 0x7a) || (c >= 0x3a) || (d >= 0x3a)) {
		MCHBAR32(0x140) = 0x9718a729;
		return 1;
	}
	return 0;
}

static void sdram_rcompupdate(struct sysinfo *s)
{
	u8 i, ok;
	u32 reg32a, reg32b;

	ok = 0;
	MCHBAR8(0x170) = MCHBAR8(0x170) & ~(1 << 3);
	MCHBAR8(0x130) = MCHBAR8(0x130) & ~(1 << 7);
	for (i = 0; i < 3; i++) {
		MCHBAR8(0x130) = MCHBAR8(0x130) | 1;
		hpet_udelay(1000);
		while ((MCHBAR8(0x130) & 0x1) != 0);
		ok |= sdram_checkrcompoverride();
	}
	if (!ok) {
		reg32a = MCHBAR32(0x13c);
		reg32b = (reg32a >> 16) & 0x0000ffff;
		reg32a = ((reg32a << 16) & 0xffff0000) | reg32b;
		reg32a |= (1 << 31) | (1 << 15);
		MCHBAR32(0x140) = reg32a;
	}
	MCHBAR8(0x130) = MCHBAR8(0x130) | 1;
	hpet_udelay(1000);
	while ((MCHBAR8(0x130) & 0x1) != 0);
}

static void __attribute__((noinline))
sdram_jedec(struct sysinfo *s, u8 rank, u8 jmode, u16 jval)
{
	u32 reg32;

	reg32 = jval << 3;
	reg32 |= rank * 0x8000000;
	MCHBAR8(0x271) = (MCHBAR8(0x271) & ~0x3e) | jmode;
	read32((void *)reg32);
	barrier();
	hpet_udelay(1); // 1us
}

static void sdram_zqcl(struct sysinfo *s)
{
	if (s->boot_path == BOOT_PATH_RESUME) {
		MCHBAR32(0x260) = MCHBAR32(0x260) | (1 << 27);
		MCHBAR8(0x271) = (MCHBAR8(0x271) & ~0xe) | NORMAL_OP_CMD;
		MCHBAR8(0x271) = MCHBAR8(0x271) & ~0x30;
		MCHBAR32(0x268) = (MCHBAR32(0x268) & ~((1 << 30) | (1 << 31))) |
				(1 << 30) | (1 << 31);
	}
}

static void sdram_jedecinit(struct sysinfo *s)
{
	u8 r, i, ch;
	u16 reg16, mrs, rttnom;
	struct jedeclist {
		char debug[15];
		u8 cmd;
		u16 val;
	};

	struct jedeclist jedec[12] = {
			{ "   NOP        ", NOP_CMD, 0 },
			{ "   PRE CHARGE ", PRE_CHARGE_CMD, 0 },
			{ "   EMRS2      ", EMRS2_CMD, 0 },
			{ "   EMRS3      ", EMRS3_CMD, 0 },
			{ "   EMRS1      ", EMRS1_CMD, 0 },
			{ "   DLL RESET  ", MRS_CMD, (1 << 8) },
			{ "   PRE CHARGE ", PRE_CHARGE_CMD, 0 },
			{ "   AUTOREFRESH", CBR_CMD, 0 },
			{ "   AUTOREFRESH", CBR_CMD, 0 },
			{ "   INITIALISE ", MRS_CMD, 0 },
			{ "   EMRS1 OCD  ", EMRS1_CMD, (1 << 9) | (1 << 8) | (1 << 7) },
			{ "   EMRS1 EXIT ", EMRS1_CMD, 0 }
	};

	mrs = (s->selected_timings.CAS << 4) |
		((s->selected_timings.tWR - 1) << 9) | (1 << 3) | (1 << 1) | 1;
	rttnom = (1 << 2);
	if (rank_is_populated(s->dimms, 0, 0) && rank_is_populated(s->dimms, 0, 2)) {
		rttnom |= (1 << 6);
	}

	hpet_udelay(200); // 200us
	reg16 = 0;
	FOR_EACH_POPULATED_RANK(s->dimms, ch, r) {
		for (i = 0; i < 12; i++) {
			PRINTK_DEBUG("Rank:%d Jedec:%14s...", r, jedec[i].debug);
			reg16 = jedec[i].val;
			switch (jedec[i].cmd) {
			case EMRS1_CMD:
				reg16 |= rttnom;
				break;
			case MRS_CMD:
				reg16 |= mrs;
				break;
			default:
				break;
			}
			sdram_jedec(s, r, jedec[i].cmd, reg16);
			PRINTK_DEBUG("done\n");
		}
	}
}

static void sdram_misc(struct sysinfo *s)
{
	u32 reg32;

	reg32 = 0;
	reg32 |= (0x4 << 13);
	reg32 |= (0x6 << 8);
	MCHBAR32(0x274) = (MCHBAR32(0x274) & ~0x3ff00) | reg32;
	MCHBAR8(0x274) = MCHBAR8(0x274) & ~(1 << 7);
	MCHBAR8(0x26c) = MCHBAR8(0x26c) | 1;
	if (s->boot_path != BOOT_PATH_RESUME) {
		MCHBAR8(0x271) = (MCHBAR8(0x271) & ~0xe) | NORMAL_OP_CMD;
		MCHBAR8(0x271) = MCHBAR8(0x271) & ~0x30;
	} else {
		sdram_zqcl(s);
	}
}

static void sdram_checkreset(void)
{
	u8 pmcon2, pmcon3, reset;

	pmcon2 = pci_read_config8(PCI_DEV(0, 0x1f, 0), 0xa2);
	pmcon3 = pci_read_config8(PCI_DEV(0, 0x1f, 0), 0xa4);
	pmcon3 &= ~0x2;
	if (pmcon2 & 0x80) {
		pmcon2 &= ~0x80;
		reset = 1;
	} else {
		pmcon2 |= 0x80;
		reset = 0;
	}
	if (pmcon2 & 0x4) {
		pmcon2 |= 0x4;
		pmcon3 = (pmcon3 & ~0x30) | 0x30;
		pmcon3 |= (1 << 3);
	}
	pci_write_config8(PCI_DEV(0, 0x1f, 0), 0xa2, pmcon2);
	pci_write_config8(PCI_DEV(0, 0x1f, 0), 0xa4, pmcon3);
	if (reset) {
		printk(BIOS_DEBUG, "Power cycle reset...\n");
		outb(0xe, 0xcf9);
	}
}

static void sdram_dradrb(struct sysinfo *s)
{
	u8 i, reg8, ch, r;
	u32 reg32, ind, c0dra, c0drb, dra;
	u16 addr;
	i = 0;
	u8 dratab[2][2][2][4] =
	{{
		{
		 {0xff, 0xff, 0xff, 0xff},
		 {0xff, 0x00, 0x02, 0xff}
		},
		{
		 {0xff, 0x01, 0xff, 0xff},
		 {0xff, 0x03, 0xff, 0x06}
		}
	},
	{
		{
		 {0xff, 0xff, 0xff, 0xff},
		 {0xff, 0x04, 0x06, 0x08}
		},
		{
		 {0xff, 0xff, 0xff, 0xff},
		 {0x05, 0x07, 0x09, 0xff}
		}
	}};

	u8 dradrb[10][6]  =  {
		//Row   Col   Bank  Width         DRB
		{0x01,  0x01,  0x00,  0x08,  0,  0x04},
		{0x01,  0x00,  0x00,  0x10,  0,  0x02},
		{0x02,  0x01,  0x00,  0x08,  1,  0x08},
		{0x01,  0x01,  0x00,  0x10,  1,  0x04},
		{0x01,  0x01,  0x01,  0x08,  1,  0x08},
		{0x00,  0x01,  0x01,  0x10,  1,  0x04},
		{0x02,  0x01,  0x01,  0x08,  2,  0x10},
		{0x01,  0x01,  0x01,  0x10,  2,  0x08},
		{0x03,  0x01,  0x01,  0x08,  3,  0x20},
		{0x02,  0x01,  0x01,  0x10,  3,  0x10},
	};

	reg32 = 0;
	FOR_EACH_POPULATED_RANK(s->dimms, ch, r) {
		i = r / 2;
		PRINTK_DEBUG("RANK %d PRESENT\n", r);
		dra = dratab[s->dimms[i].banks]
			[s->dimms[i].width]
			[s->dimms[i].cols - 9]
			[s->dimms[i].rows - 12];

		if (s->dimms[i].banks == 1) {
			dra |= (1 << 7);
		}
		reg32 |= (dra << (r*8));
	}
	MCHBAR32(0x208) = reg32;
	c0dra = reg32;
	PRINTK_DEBUG("C0DRA = 0x%08x\n", c0dra);

	reg32 = 0;
	FOR_EACH_POPULATED_RANK(s->dimms, ch, r) {
		reg32 |= (1 << r);
	}
	reg8 = (u8)(reg32 << 4) & 0xf0;
	MCHBAR8(0x262) = (MCHBAR8(0x262) & ~0xf0) | reg8;
	if (ONLY_DIMMA_IS_POPULATED(s->dimms, 0) ||
			ONLY_DIMMB_IS_POPULATED(s->dimms, 0)) {
		MCHBAR8(0x260) = MCHBAR8(0x260) | 1;
	}

	addr = 0x200;
	c0drb = 0;
	FOR_EACH_RANK(ch, r) {
		if (rank_is_populated(s->dimms, ch, r)) {
			ind = (c0dra >> (8*r)) & 0x7f;
			c0drb = (u16)(c0drb + dradrb[ind][5]);
			s->channel_capacity[0] += dradrb[ind][5] << 6;
		}
		MCHBAR16(addr) = c0drb;
		addr += 2;
	}
	printk(BIOS_DEBUG, "Total memory = %dMB\n", s->channel_capacity[0]);
}

static u8 sampledqs(u32 dqshighaddr, u32 strobeaddr, u8 highlow, u8 count)
{
	volatile u32 strobedata;
	u8 dqsmatches = 1;
	while (count--) {
		MCHBAR8(0x5d8) = MCHBAR8(0x5d8) & ~0x2;
		hpet_udelay(1);
		MCHBAR8(0x5d8) = MCHBAR8(0x5d8) | 0x2;
		hpet_udelay(1);
		barrier();
		strobedata = read32((void*)strobeaddr);
		barrier();
		hpet_udelay(1);

		if (((MCHBAR8(dqshighaddr) & 0x40) >> 6) != highlow) {
			dqsmatches = 0;
		}
	}

	return dqsmatches;
}

static void rcvenclock(u8 *coarse, u8 *medium, u8 bytelane)
{
	if (*medium < 3) {
		(*medium)++;
		MCHBAR16(0x58c) = (MCHBAR16(0x58c) & (u16)(~(0x3 << (bytelane*2))))
					| (*medium << (bytelane*2));
	} else {
		*medium = 0;
		(*coarse)++;
		MCHBAR32(0x248) = (MCHBAR32(0x248) & ~0xf0000) | (*coarse << 16);
		MCHBAR16(0x58c) = (MCHBAR16(0x58c) & (u16)(~0x3 << (bytelane*2)))
					| (*medium << (bytelane*2));
	}
}

static void sdram_rcven(struct sysinfo *s)
{
	u8 curcoarse, savecoarse;
	u8 curmedium, savemedium;
	u8 pi, savepi;
	u8 bytelane;
	u8 bytelanecoarse[8] = { 0 };
	u8 minbytelanecoarse = 0xff;
	u8 bytelaneoffset;
	u8 maxbytelane = 8;
	u32 strobeaddr = (rank_is_populated(s->dimms, 0, 0)) ? 0 : 2*128*1024*1024;
	u32 dqshighaddr;

	MCHBAR8(0x5d8) = MCHBAR8(0x5d8) & ~0xc;
	MCHBAR8(0x5dc) = MCHBAR8(0x5dc) & ~0x80;

	PRINTK_DEBUG("rcven 0\n");
	for (bytelane = 0; bytelane < maxbytelane; bytelane++) {
		PRINTK_DEBUG("rcven bytelane %d\n", bytelane);
//trylaneagain:
		dqshighaddr = 0x561 + (bytelane << 2);

		curcoarse = s->selected_timings.CAS + 1;
		pi = 0;
		curmedium = 0;

		MCHBAR32(0x248) = (MCHBAR32(0x248) & ~0xf0000) | (curcoarse << 16);
		MCHBAR16(0x58c) = (MCHBAR16(0x58c) & (u16)(~(0x3 << (bytelane*2))))
						| (curmedium << (bytelane*2));
		MCHBAR8(0x560+bytelane*4) = MCHBAR8(0x560+bytelane*4) & ~0x3f;

		savecoarse = curcoarse;
		savemedium = curmedium;
		savepi = pi;

		PRINTK_DEBUG("rcven 0.1\n");

		//MCHBAR16(0x588) = (MCHBAR16(0x588) & (u16)~(0x3 << (bytelane*2))) | (1 << (bytelane*2)); // XXX comment out

		while (sampledqs(dqshighaddr, strobeaddr, 0, 3) == 0) {
			//printk(BIOS_DEBUG, "coarse=%d medium=%d\n", curcoarse, curmedium);
			rcvenclock(&curcoarse, &curmedium, bytelane);
			if (curcoarse > 0xf) {
				PRINTK_DEBUG("Error: coarse > 0xf\n");
				//goto trylaneagain;
				break;
			}
		}
		PRINTK_DEBUG("rcven 0.2\n");

		savecoarse = curcoarse;
		savemedium = curmedium;
		rcvenclock(&curcoarse, &curmedium, bytelane);

		while (sampledqs(dqshighaddr, strobeaddr, 1, 3) == 0) {
			savecoarse = curcoarse;
			savemedium = curmedium;
			rcvenclock(&curcoarse, &curmedium, bytelane);
			if (curcoarse > 0xf) {
				PRINTK_DEBUG("Error: coarse > 0xf\n");
				//goto trylaneagain;
				break;
			}
		}

		PRINTK_DEBUG("rcven 0.3\n");
		curcoarse = savecoarse;
		curmedium = savemedium;
		MCHBAR32(0x248) = (MCHBAR32(0x248) & ~0xf0000) | (curcoarse << 16);
		MCHBAR16(0x58c) = (MCHBAR16(0x58c) & (u16)(~(0x3 << bytelane*2)))
						| (curmedium << (bytelane*2));

		while (sampledqs(dqshighaddr, strobeaddr, 1, 3) == 0) {
			savepi = pi;
			pi++;
			if (pi > s->maxpi) {
				//if (s->nodll) {
					pi = savepi = s->maxpi;
					break;
				//}
			}
			MCHBAR8(0x560 + bytelane*4) = (MCHBAR8(0x560 + bytelane*4)
					& ~0x3f) | (pi << s->pioffset);
		}
		PRINTK_DEBUG("rcven 0.4\n");

		pi = savepi;
		MCHBAR8(0x560 + bytelane*4) = (MCHBAR8(0x560 + bytelane*4) & ~0x3f)
						| (pi << s->pioffset);
		rcvenclock(&curcoarse, &curmedium, bytelane);
		if (sampledqs(dqshighaddr, strobeaddr, 1, 1) == 0) {
			PRINTK_DEBUG("Error: DQS not high\n");
			//goto trylaneagain;
		}
		PRINTK_DEBUG("rcven 0.5\n");
		while (sampledqs(dqshighaddr, strobeaddr, 0, 3) == 0) {
			curcoarse--;
			MCHBAR32(0x248) = (MCHBAR32(0x248