blob: 3076d070ec31c2c5d36119bd50ed84d5c0f16f27 [file] [log] [blame]
Aaron Durbin305b1f02013-01-15 08:27:05 -06001/*
2 * This file is part of the coreboot project.
3 *
4 * Copyright (C) 2013 ChromeOS Authors
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as
8 * published by the Free Software Foundation; version 2 of
9 * the License.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
19 * MA 02110-1301 USA
20 */
21
22#include <console/console.h>
23#include <stdint.h>
24#include <rmodule.h>
25#include <arch/cpu.h>
26#include <cpu/cpu.h>
27#include <cpu/intel/microcode.h>
28#include <cpu/x86/cache.h>
29#include <cpu/x86/lapic.h>
30#include <cpu/x86/msr.h>
31#include <cpu/x86/mtrr.h>
32#include <cpu/x86/smm.h>
33#include <delay.h>
34#include <device/device.h>
35#include <device/path.h>
36#include <lib.h>
37#include <smp/atomic.h>
38#include <smp/spinlock.h>
39#include "haswell.h"
40
41/* This needs to match the layout in the .module_parametrs section. */
42struct sipi_params {
43 u16 gdtlimit;
44 u32 gdt;
45 u16 unused;
46 u32 idt_ptr;
47 u32 stack_top;
48 u32 stack_size;
49 u32 microcode_ptr;
50 u32 msr_table_ptr;
51 u32 msr_count;
52 u32 c_handler;
53 u32 c_handler_arg;
54 u8 apic_to_cpu_num[CONFIG_MAX_CPUS];
55} __attribute__((packed));
56
57/* This also needs to match the assembly code for saved MSR encoding. */
58struct saved_msr {
59 u32 index;
60 u32 lo;
61 u32 hi;
62} __attribute__((packed));
63
64
65/* The sipi vector rmodule is included in the ramstage using 'objdump -B'. */
66extern char _binary_sipi_vector_start[];
67/* These symbols are defined in c_start.S. */
68extern char gdt[];
69extern char gdt_limit[];
70extern char idtarg[];
71
72/* This table keeps track of each CPU's APIC id. */
73static u8 apic_id_table[CONFIG_MAX_CPUS];
74static device_t cpu_devs[CONFIG_MAX_CPUS];
75
76/* Number of APs checked that have checked in. */
77static atomic_t num_aps;
78/* Barrier to stop APs from performing SMM relcoation. */
79static int smm_relocation_barrier_begin __attribute__ ((aligned (64)));
80
81static inline void wait_for_barrier(volatile int *barrier)
82{
83 while (*barrier == 0) {
84 asm ("pause");
85 }
86}
87
88static inline void release_barrier(volatile int *barrier)
89{
90 *barrier = 1;
91}
92
93static void ap_wait_for_smm_relocation_begin(void)
94{
95 wait_for_barrier(&smm_relocation_barrier_begin);
96}
97
98
99/* Returns 1 if timeout waiting for APs. 0 if target aps found. */
100static int wait_for_aps(int target, int total_delay, int delay_step)
101{
102 int timeout = 0;
103 int delayed = 0;
104 while (atomic_read(&num_aps) != target) {
105 udelay(delay_step);
106 delayed += delay_step;
107 if (delayed >= total_delay) {
108 timeout = 1;
109 break;
110 }
111 }
112
113 return timeout;
114}
115
116void release_aps_for_smm_relocation(void)
117{
118 release_barrier(&smm_relocation_barrier_begin);
119}
120
121/* The mtrr code sets up ROM caching on the BSP, but not the others. However,
122 * the boot loader payload disables this. In order for Linux not to complain
123 * ensure the caching is disabled for tha APs before going to sleep. */
124static void cleanup_rom_caching(void)
125{
126#if CONFIG_CACHE_ROM
127 msr_t msr;
128 unsigned int last_var_mtrr;
129
130 msr = rdmsr(MTRRcap_MSR);
131 last_var_mtrr = (msr.lo & 0xff) - 1;
132
133 /* Check if the MTRR is valid. */
134 msr = rdmsr(MTRRphysMask_MSR(last_var_mtrr));
135 if ((msr.lo & MTRRphysMaskValid) == 0)
136 return;
137 msr = rdmsr(MTRRphysBase_MSR(last_var_mtrr));
138 /* Assum that if the MTRR is of write protected type, the MTRR is used
139 * to cache the ROM. */
140 if ((msr.lo & MTRR_NUM_TYPES) == MTRR_TYPE_WRPROT) {
141 msr.lo = msr.hi = 0;
142 disable_cache();
143 wrmsr(MTRRphysMask_MSR(last_var_mtrr), msr);
144 wrmsr(MTRRphysBase_MSR(last_var_mtrr), msr);
145 enable_cache();
146 }
147#endif
148}
149
150/* By the time APs call ap_init() caching has been setup, and microcode has
151 * been loaded. */
152static void ap_init(unsigned int cpu, void *microcode_ptr)
153{
154 struct cpu_info *info;
155
156 /* Signal that the AP has arrived. */
157 atomic_inc(&num_aps);
158
159 /* Ensure the local apic is enabled */
160 enable_lapic();
161
162 info = cpu_info();
163 info->index = cpu;
164 info->cpu = cpu_devs[cpu];
165
166 apic_id_table[info->index] = lapicid();
167 info->cpu->path.apic.apic_id = apic_id_table[info->index];
168
169 /* Call through the cpu driver's initialization. */
170 cpu_initialize(info->index);
171
172 ap_wait_for_smm_relocation_begin();
173
174 smm_initiate_relocation();
175
176 /* After SMM relocation a 2nd microcode load is required. */
177 intel_microcode_load_unlocked(microcode_ptr);
178
179 /* Cleanup ROM caching. */
180 cleanup_rom_caching();
181
182 /* FIXME(adurbin): park CPUs properly -- preferably somewhere in a
183 * reserved part of memory that the OS cannot get to. */
184 stop_this_cpu();
185}
186
187static void setup_default_sipi_vector_params(struct sipi_params *sp)
188{
189 int i;
190
191 sp->gdt = (u32)&gdt;
192 sp->gdtlimit = (u32)&gdt_limit;
193 sp->idt_ptr = (u32)&idtarg;
194 sp->stack_size = CONFIG_STACK_SIZE;
195 sp->stack_top = (u32)&_estack;
196 /* Adjust the stack top to take into account cpu_info. */
197 sp->stack_top -= sizeof(struct cpu_info);
198 /* Default to linear APIC id space. */
199 for (i = 0; i < CONFIG_MAX_CPUS; i++)
200 sp->apic_to_cpu_num[i] = i;
201}
202
203#define NUM_FIXED_MTRRS 11
204static unsigned int fixed_mtrrs[NUM_FIXED_MTRRS] = {
205 MTRRfix64K_00000_MSR, MTRRfix16K_80000_MSR, MTRRfix16K_A0000_MSR,
206 MTRRfix4K_C0000_MSR, MTRRfix4K_C8000_MSR, MTRRfix4K_D0000_MSR,
207 MTRRfix4K_D8000_MSR, MTRRfix4K_E0000_MSR, MTRRfix4K_E8000_MSR,
208 MTRRfix4K_F0000_MSR, MTRRfix4K_F8000_MSR,
209};
210
211static inline struct saved_msr *save_msr(int index, struct saved_msr *entry)
212{
213 msr_t msr;
214
215 msr = rdmsr(index);
216 entry->index = index;
217 entry->lo = msr.lo;
218 entry->hi = msr.hi;
219
220 /* Return the next entry. */
221 entry++;
222 return entry;
223}
224
225static int save_bsp_msrs(char *start, int size)
226{
227 int msr_count;
228 int num_var_mtrrs;
229 struct saved_msr *msr_entry;
230 int i;
231 msr_t msr;
232
233 /* Determine number of MTRRs need to be saved. */
234 msr = rdmsr(MTRRcap_MSR);
235 num_var_mtrrs = msr.lo & 0xff;
236
237 /* 2 * num_var_mtrrs for base and mask. +1 for IA32_MTRR_DEF_TYPE. */
238 msr_count = 2 * num_var_mtrrs + NUM_FIXED_MTRRS + 1;
239
240 if ((msr_count * sizeof(struct saved_msr)) > size) {
241 printk(BIOS_CRIT, "Cannot mirror all %d msrs.\n", msr_count);
242 return -1;
243 }
244
245 msr_entry = (void *)start;
246 for (i = 0; i < NUM_FIXED_MTRRS; i++) {
247 msr_entry = save_msr(fixed_mtrrs[i], msr_entry);
248 }
249
250 for (i = 0; i < num_var_mtrrs; i++) {
251 msr_entry = save_msr(MTRRphysBase_MSR(i), msr_entry);
252 msr_entry = save_msr(MTRRphysMask_MSR(i), msr_entry);
253 }
254
255 msr_entry = save_msr(MTRRdefType_MSR, msr_entry);
256
257 return msr_count;
258}
259
260/* The SIPI vector is loaded at the SMM_DEFAULT_BASE. The reason is at the
261 * memory range is already reserved so the OS cannot use it. That region is
262 * free to use for AP bringup before SMM is initialized. */
263static u32 sipi_vector_location = SMM_DEFAULT_BASE;
264static int sipi_vector_location_size = SMM_DEFAULT_SIZE;
265
266static int load_sipi_vector(const void *microcode_patch)
267{
268 struct rmodule sipi_mod;
269 int module_size;
270 int num_msrs;
271 struct sipi_params *sp;
272 char *mod_loc = (void *)sipi_vector_location;
273 const int loc_size = sipi_vector_location_size;
274
275 if (rmodule_parse(&_binary_sipi_vector_start, &sipi_mod)) {
276 printk(BIOS_CRIT, "Unable to parse sipi module.\n");
277 return -1;
278 }
279
280 if (rmodule_entry_offset(&sipi_mod) != 0) {
281 printk(BIOS_CRIT, "SIPI module entry offset is not 0!\n");
282 return -1;
283 }
284
285 if (rmodule_load_alignment(&sipi_mod) != 4096) {
286 printk(BIOS_CRIT, "SIPI module load alignment(%d) != 4096.\n",
287 rmodule_load_alignment(&sipi_mod));
288 return -1;
289 }
290
291 module_size = rmodule_memory_size(&sipi_mod);
292
293 /* Align to 4 bytes. */
294 module_size += 3;
295 module_size &= ~3;
296
297 if (module_size > loc_size) {
298 printk(BIOS_CRIT, "SIPI module size (%d) > region size (%d).\n",
299 module_size, loc_size);
300 return -1;
301 }
302
303 num_msrs = save_bsp_msrs(&mod_loc[module_size], loc_size - module_size);
304
305 if (num_msrs < 0) {
306 printk(BIOS_CRIT, "Error mirroring BSP's msrs.\n");
307 return -1;
308 }
309
310 if (rmodule_load(mod_loc, &sipi_mod)) {
311 printk(BIOS_CRIT, "Unable to load SIPI module.\n");
312 return -1;
313 }
314
315 sp = rmodule_parameters(&sipi_mod);
316
317 if (sp == NULL) {
318 printk(BIOS_CRIT, "SIPI module has no parameters.\n");
319 return -1;
320 }
321
322 setup_default_sipi_vector_params(sp);
323 /* Setup MSR table. */
324 sp->msr_table_ptr = (u32)&mod_loc[module_size];
325 sp->msr_count = num_msrs;
326 /* Provide pointer to microcode patch. */
327 sp->microcode_ptr = (u32)microcode_patch;
328 /* The microcode pointer is passed on through to the c handler so
329 * that it can be loaded again after SMM relocation. */
330 sp->c_handler_arg = (u32)microcode_patch;
331 sp->c_handler = (u32)&ap_init;
332
333 /* Make sure SIPI vector hits RAM so the APs that come up will see
334 * the startup code even if the caches are disabled. */
335 wbinvd();
336
337 return 0;
338}
339
340static int allocate_cpu_devices(struct bus *cpu_bus, int *total_hw_threads)
341{
342 int i;
343 int num_threads;
344 int num_cores;
345 int max_cpus;
346 struct cpu_info *info;
347 msr_t msr;
348
349 info = cpu_info();
350 cpu_devs[info->index] = info->cpu;
351 apic_id_table[info->index] = info->cpu->path.apic.apic_id;
352
353 msr = rdmsr(CORE_THREAD_COUNT_MSR);
354 num_threads = (msr.lo >> 0) & 0xffff;
355 num_cores = (msr.lo >> 16) & 0xffff;
356 printk(BIOS_DEBUG, "CPU has %u cores, %u threads enabled.\n",
357 num_cores, num_threads);
358
359 max_cpus = num_threads;
360 *total_hw_threads = num_threads;
361 if (num_threads > CONFIG_MAX_CPUS) {
362 printk(BIOS_CRIT, "CPU count(%d) exceeds CONFIG_MAX_CPUS(%d)\n",
363 num_threads, CONFIG_MAX_CPUS);
364 max_cpus = CONFIG_MAX_CPUS;
365 }
366
367 for (i = 1; i < max_cpus; i++) {
368 struct device_path cpu_path;
369 device_t new;
370
371 /* Build the cpu device path */
372 cpu_path.type = DEVICE_PATH_APIC;
373 cpu_path.apic.apic_id = info->cpu->path.apic.apic_id + i;
374
375 /* Allocate the new cpu device structure */
376 new = alloc_find_dev(cpu_bus, &cpu_path);
377 if (new == NULL) {
378 printk(BIOS_CRIT, "Could not allocte cpu device\n");
379 max_cpus--;
380 }
381 cpu_devs[i] = new;
382 }
383
384 return max_cpus;
385}
386
387int setup_ap_init(struct bus *cpu_bus, int *max_cpus,
388 const void *microcode_patch)
389{
390 int num_cpus;
391 int hw_threads;
392
393 /* Default to currently running CPU. */
394 num_cpus = allocate_cpu_devices(cpu_bus, &hw_threads);
395
396 /* Load the SIPI vector. */
397 if (load_sipi_vector(microcode_patch))
398 return -1;
399
400 *max_cpus = num_cpus;
401
402 if (num_cpus < hw_threads) {
403 printk(BIOS_CRIT,
404 "ERROR: More HW threads (%d) than support (%d).\n",
405 hw_threads, num_cpus);
406 return -1;
407 }
408
409 return 0;
410}
411
412/* Returns 1 for timeout. 0 on success. */
413static int apic_wait_timeout(int total_delay, int delay_step)
414{
415 int total = 0;
416 int timeout = 0;
417
418 while (lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY) {
419 udelay(delay_step);
420 total += delay_step;
421 if (total >= total_delay) {
422 timeout = 1;
423 break;
424 }
425 }
426
427 return timeout;
428}
429
430int start_aps(struct bus *cpu_bus, int ap_count)
431{
432 int sipi_vector;
433
434 if (ap_count == 0)
435 return 0;
436
437 /* The vector is sent as a 4k aligned address in one byte. */
438 sipi_vector = sipi_vector_location >> 12;
439
440 if (sipi_vector > 256) {
441 printk(BIOS_CRIT, "SIPI vector too large! 0x%08x\n",
442 sipi_vector);
443 return -1;
444 }
445
446 printk(BIOS_DEBUG, "Attempting to start %d APs\n", ap_count);
447
448 if ((lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY)) {
449 printk(BIOS_DEBUG, "Waiting for ICR not to be busy...");
450 if (apic_wait_timeout(1000 /* 1 ms */, 50)) {
451 printk(BIOS_DEBUG, "timed out. Aborting.\n");
452 return -1;
453 } else
454 printk(BIOS_DEBUG, "done.\n");
455 }
456
457 /* Send INIT IPI to all but self. */
458 lapic_write_around(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(0));
459 lapic_write_around(LAPIC_ICR, LAPIC_DEST_ALLBUT | LAPIC_INT_ASSERT |
460 LAPIC_DM_INIT);
Aaron Durbin8c203992013-01-17 11:13:46 -0600461 printk(BIOS_DEBUG, "Waiting for 10ms after sending INIT.\n");
462 mdelay(10);
Aaron Durbin305b1f02013-01-15 08:27:05 -0600463
464 /* Send 1st SIPI */
465 if ((lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY)) {
466 printk(BIOS_DEBUG, "Waiting for ICR not to be busy...");
467 if (apic_wait_timeout(1000 /* 1 ms */, 50)) {
468 printk(BIOS_DEBUG, "timed out. Aborting.\n");
469 return -1;
470 } else
471 printk(BIOS_DEBUG, "done.\n");
472 }
473
474 lapic_write_around(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(0));
475 lapic_write_around(LAPIC_ICR, LAPIC_DEST_ALLBUT | LAPIC_INT_ASSERT |
476 LAPIC_DM_STARTUP | sipi_vector);
477 printk(BIOS_DEBUG, "Waiting for 1st SIPI to complete...");
478 if (apic_wait_timeout(10000 /* 10 ms */, 50 /* us */)) {
479 printk(BIOS_DEBUG, "timed out.\n");
480 return -1;
481 } else {
482 printk(BIOS_DEBUG, "done.\n");
483 }
484 /* Wait for CPUs to check in up to 200 us. */
485 wait_for_aps(ap_count, 200 /* us */, 15 /* us */);
486
487 /* Send 2nd SIPI */
488 if ((lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY)) {
489 printk(BIOS_DEBUG, "Waiting for ICR not to be busy...");
490 if (apic_wait_timeout(1000 /* 1 ms */, 50)) {
491 printk(BIOS_DEBUG, "timed out. Aborting.\n");
492 return -1;
493 } else
494 printk(BIOS_DEBUG, "done.\n");
495 }
496
497 lapic_write_around(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(0));
498 lapic_write_around(LAPIC_ICR, LAPIC_DEST_ALLBUT | LAPIC_INT_ASSERT |
499 LAPIC_DM_STARTUP | sipi_vector);
500 printk(BIOS_DEBUG, "Waiting for 2nd SIPI to complete...");
501 if (apic_wait_timeout(10000 /* 10 ms */, 50 /* us */)) {
502 printk(BIOS_DEBUG, "timed out.\n");
503 return -1;
504 } else {
505 printk(BIOS_DEBUG, "done.\n");
506 }
507
508 /* Wait for CPUs to check in. */
509 if (wait_for_aps(ap_count, 10000 /* 10 ms */, 50 /* us */)) {
510 printk(BIOS_DEBUG, "Not all APs checked in: %d/%d.\n",
511 atomic_read(&num_aps), ap_count);
512 return -1;
513 }
514
515 return 0;
516}
517
518DECLARE_SPIN_LOCK(smm_relocation_lock);
519
520void smm_initiate_relocation(void)
521{
522 spin_lock(&smm_relocation_lock);
523
524 if ((lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY)) {
525 printk(BIOS_DEBUG, "Waiting for ICR not to be busy...");
526 if (apic_wait_timeout(1000 /* 1 ms */, 50)) {
527 printk(BIOS_DEBUG, "timed out. Aborting.\n");
528 spin_unlock(&smm_relocation_lock);
529 return;
530 } else
531 printk(BIOS_DEBUG, "done.\n");
532 }
533
534 lapic_write_around(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(lapicid()));
535 lapic_write_around(LAPIC_ICR, LAPIC_INT_ASSERT | LAPIC_DM_SMI);
536 if (apic_wait_timeout(1000 /* 1 ms */, 100 /* us */)) {
537 printk(BIOS_DEBUG, "SMI Relocation timed out.\n");
538 } else
539 printk(BIOS_DEBUG, "Relocation complete.\n");
540
541 spin_unlock(&smm_relocation_lock);
542}
543