Angel Pons | f23ae0b | 2020-04-02 23:48:12 +0200 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 2 | |
| 3 | #include <console/console.h> |
Jonathan Zhang | 6ec322e | 2020-01-16 11:11:09 -0800 | [diff] [blame] | 4 | #include <stddef.h> |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 5 | #include <stdint.h> |
Elyes HAOUAS | a1e22b8 | 2019-03-18 22:49:36 +0100 | [diff] [blame] | 6 | #include <string.h> |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 7 | #include <rmodule.h> |
| 8 | #include <arch/cpu.h> |
Kyösti Mälkki | 2fbb677 | 2018-05-15 19:50:20 +0300 | [diff] [blame] | 9 | #include <commonlib/helpers.h> |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 10 | #include <cpu/cpu.h> |
| 11 | #include <cpu/intel/microcode.h> |
| 12 | #include <cpu/x86/cache.h> |
Kyösti Mälkki | bae775a | 2014-12-18 10:36:33 +0200 | [diff] [blame] | 13 | #include <cpu/x86/gdt.h> |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 14 | #include <cpu/x86/lapic.h> |
| 15 | #include <cpu/x86/name.h> |
| 16 | #include <cpu/x86/msr.h> |
| 17 | #include <cpu/x86/mtrr.h> |
| 18 | #include <cpu/x86/smm.h> |
| 19 | #include <cpu/x86/mp.h> |
| 20 | #include <delay.h> |
| 21 | #include <device/device.h> |
| 22 | #include <device/path.h> |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 23 | #include <smp/atomic.h> |
| 24 | #include <smp/spinlock.h> |
Julius Werner | ec5e5e0 | 2014-08-20 15:29:56 -0700 | [diff] [blame] | 25 | #include <symbols.h> |
Elyes HAOUAS | add76f9 | 2019-03-21 09:55:49 +0100 | [diff] [blame] | 26 | #include <timer.h> |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 27 | #include <thread.h> |
| 28 | |
Eugene Myers | ae438be | 2020-01-21 17:01:47 -0500 | [diff] [blame] | 29 | #include <security/intel/stm/SmmStm.h> |
| 30 | |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 31 | #define MAX_APIC_IDS 256 |
Aaron Durbin | 770d7c7 | 2016-05-03 17:49:57 -0500 | [diff] [blame] | 32 | |
Aaron Durbin | 223fb43 | 2018-05-03 13:49:41 -0600 | [diff] [blame] | 33 | struct mp_callback { |
Subrata Banik | 3337497 | 2018-04-24 13:45:30 +0530 | [diff] [blame] | 34 | void (*func)(void *); |
| 35 | void *arg; |
Subrata Banik | 8a25cae | 2018-05-03 18:48:41 +0530 | [diff] [blame] | 36 | int logical_cpu_number; |
Aaron Durbin | 223fb43 | 2018-05-03 13:49:41 -0600 | [diff] [blame] | 37 | }; |
Aaron Durbin | 770d7c7 | 2016-05-03 17:49:57 -0500 | [diff] [blame] | 38 | |
Naresh G Solanki | 2463533 | 2018-05-31 23:13:18 +0530 | [diff] [blame] | 39 | static char processor_name[49]; |
| 40 | |
Aaron Durbin | 770d7c7 | 2016-05-03 17:49:57 -0500 | [diff] [blame] | 41 | /* |
| 42 | * A mp_flight_record details a sequence of calls for the APs to perform |
| 43 | * along with the BSP to coordinate sequencing. Each flight record either |
| 44 | * provides a barrier for each AP before calling the callback or the APs |
| 45 | * are allowed to perform the callback without waiting. Regardless, each |
| 46 | * record has the cpus_entered field incremented for each record. When |
| 47 | * the BSP observes that the cpus_entered matches the number of APs |
| 48 | * the bsp_call is called with bsp_arg and upon returning releases the |
| 49 | * barrier allowing the APs to make further progress. |
| 50 | * |
| 51 | * Note that ap_call() and bsp_call() can be NULL. In the NULL case the |
| 52 | * callback will just not be called. |
| 53 | */ |
| 54 | struct mp_flight_record { |
| 55 | atomic_t barrier; |
| 56 | atomic_t cpus_entered; |
Aaron Durbin | 223fb43 | 2018-05-03 13:49:41 -0600 | [diff] [blame] | 57 | void (*ap_call)(void); |
| 58 | void (*bsp_call)(void); |
Aaron Durbin | 381feb8 | 2018-05-02 22:38:58 -0600 | [diff] [blame] | 59 | } __aligned(CACHELINE_SIZE); |
Aaron Durbin | 770d7c7 | 2016-05-03 17:49:57 -0500 | [diff] [blame] | 60 | |
| 61 | #define _MP_FLIGHT_RECORD(barrier_, ap_func_, bsp_func_) \ |
| 62 | { \ |
| 63 | .barrier = ATOMIC_INIT(barrier_), \ |
| 64 | .cpus_entered = ATOMIC_INIT(0), \ |
| 65 | .ap_call = ap_func_, \ |
| 66 | .bsp_call = bsp_func_, \ |
| 67 | } |
| 68 | |
| 69 | #define MP_FR_BLOCK_APS(ap_func_, bsp_func_) \ |
| 70 | _MP_FLIGHT_RECORD(0, ap_func_, bsp_func_) |
| 71 | |
| 72 | #define MP_FR_NOBLOCK_APS(ap_func_, bsp_func_) \ |
| 73 | _MP_FLIGHT_RECORD(1, ap_func_, bsp_func_) |
| 74 | |
| 75 | /* The mp_params structure provides the arguments to the mp subsystem |
| 76 | * for bringing up APs. */ |
| 77 | struct mp_params { |
| 78 | int num_cpus; /* Total cpus include BSP */ |
| 79 | int parallel_microcode_load; |
| 80 | const void *microcode_pointer; |
Aaron Durbin | 770d7c7 | 2016-05-03 17:49:57 -0500 | [diff] [blame] | 81 | /* Flight plan for APs and BSP. */ |
| 82 | struct mp_flight_record *flight_plan; |
| 83 | int num_records; |
| 84 | }; |
| 85 | |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 86 | /* This needs to match the layout in the .module_parametrs section. */ |
| 87 | struct sipi_params { |
| 88 | uint16_t gdtlimit; |
| 89 | uint32_t gdt; |
| 90 | uint16_t unused; |
| 91 | uint32_t idt_ptr; |
| 92 | uint32_t stack_top; |
| 93 | uint32_t stack_size; |
| 94 | uint32_t microcode_lock; /* 0xffffffff means parallel loading. */ |
| 95 | uint32_t microcode_ptr; |
| 96 | uint32_t msr_table_ptr; |
| 97 | uint32_t msr_count; |
| 98 | uint32_t c_handler; |
| 99 | atomic_t ap_count; |
Stefan Reinauer | 6a00113 | 2017-07-13 02:20:27 +0200 | [diff] [blame] | 100 | } __packed; |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 101 | |
| 102 | /* This also needs to match the assembly code for saved MSR encoding. */ |
| 103 | struct saved_msr { |
| 104 | uint32_t index; |
| 105 | uint32_t lo; |
| 106 | uint32_t hi; |
Stefan Reinauer | 6a00113 | 2017-07-13 02:20:27 +0200 | [diff] [blame] | 107 | } __packed; |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 108 | |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 109 | /* The sipi vector rmodule is included in the ramstage using 'objdump -B'. */ |
| 110 | extern char _binary_sipi_vector_start[]; |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 111 | |
| 112 | /* The SIPI vector is loaded at the SMM_DEFAULT_BASE. The reason is at the |
| 113 | * memory range is already reserved so the OS cannot use it. That region is |
| 114 | * free to use for AP bringup before SMM is initialized. */ |
Patrick Rudolph | 6c46b6f | 2020-08-21 16:43:15 +0200 | [diff] [blame] | 115 | static const uintptr_t sipi_vector_location = SMM_DEFAULT_BASE; |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 116 | static const int sipi_vector_location_size = SMM_DEFAULT_SIZE; |
| 117 | |
| 118 | struct mp_flight_plan { |
| 119 | int num_records; |
| 120 | struct mp_flight_record *records; |
| 121 | }; |
| 122 | |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 123 | static int global_num_aps; |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 124 | static struct mp_flight_plan mp_info; |
| 125 | |
Subrata Banik | 7bc9036 | 2019-05-10 11:58:37 +0530 | [diff] [blame] | 126 | /* Keep track of device structure for each CPU. */ |
| 127 | static struct device *cpus_dev[CONFIG_MAX_CPUS]; |
Aaron Durbin | 5a1f9a8 | 2017-09-07 21:17:33 -0600 | [diff] [blame] | 128 | |
Aaron Durbin | 4c16f8f | 2018-05-02 22:35:33 -0600 | [diff] [blame] | 129 | static inline void barrier_wait(atomic_t *b) |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 130 | { |
Lee Leahy | a15d8af | 2017-03-15 14:49:35 -0700 | [diff] [blame] | 131 | while (atomic_read(b) == 0) |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 132 | asm ("pause"); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 133 | mfence(); |
| 134 | } |
| 135 | |
Aaron Durbin | 4c16f8f | 2018-05-02 22:35:33 -0600 | [diff] [blame] | 136 | static inline void release_barrier(atomic_t *b) |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 137 | { |
| 138 | mfence(); |
| 139 | atomic_set(b, 1); |
| 140 | } |
| 141 | |
| 142 | /* Returns 1 if timeout waiting for APs. 0 if target aps found. */ |
| 143 | static int wait_for_aps(atomic_t *val, int target, int total_delay, |
Lee Leahy | a07d0dd | 2017-03-15 14:25:22 -0700 | [diff] [blame] | 144 | int delay_step) |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 145 | { |
| 146 | int timeout = 0; |
| 147 | int delayed = 0; |
| 148 | while (atomic_read(val) != target) { |
| 149 | udelay(delay_step); |
| 150 | delayed += delay_step; |
| 151 | if (delayed >= total_delay) { |
| 152 | timeout = 1; |
| 153 | break; |
| 154 | } |
| 155 | } |
| 156 | |
| 157 | return timeout; |
| 158 | } |
| 159 | |
| 160 | static void ap_do_flight_plan(void) |
| 161 | { |
| 162 | int i; |
| 163 | |
| 164 | for (i = 0; i < mp_info.num_records; i++) { |
| 165 | struct mp_flight_record *rec = &mp_info.records[i]; |
| 166 | |
| 167 | atomic_inc(&rec->cpus_entered); |
| 168 | barrier_wait(&rec->barrier); |
| 169 | |
Lee Leahy | a15d8af | 2017-03-15 14:49:35 -0700 | [diff] [blame] | 170 | if (rec->ap_call != NULL) |
Aaron Durbin | 0e55632 | 2016-04-29 23:15:12 -0500 | [diff] [blame] | 171 | rec->ap_call(); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 172 | } |
| 173 | } |
| 174 | |
Subrata Banik | 3337497 | 2018-04-24 13:45:30 +0530 | [diff] [blame] | 175 | static void park_this_cpu(void *unused) |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 176 | { |
| 177 | stop_this_cpu(); |
| 178 | } |
| 179 | |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 180 | /* By the time APs call ap_init() caching has been setup, and microcode has |
| 181 | * been loaded. */ |
| 182 | static void asmlinkage ap_init(unsigned int cpu) |
| 183 | { |
| 184 | struct cpu_info *info; |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 185 | |
Elyes HAOUAS | d6e9686 | 2016-08-21 10:12:15 +0200 | [diff] [blame] | 186 | /* Ensure the local APIC is enabled */ |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 187 | enable_lapic(); |
| 188 | |
| 189 | info = cpu_info(); |
| 190 | info->index = cpu; |
Subrata Banik | 7bc9036 | 2019-05-10 11:58:37 +0530 | [diff] [blame] | 191 | info->cpu = cpus_dev[cpu]; |
Aaron Durbin | 5a1f9a8 | 2017-09-07 21:17:33 -0600 | [diff] [blame] | 192 | |
Subrata Banik | 7bc9036 | 2019-05-10 11:58:37 +0530 | [diff] [blame] | 193 | cpu_add_map_entry(info->index); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 194 | thread_init_cpu_info_non_bsp(info); |
| 195 | |
Aaron Durbin | 5a1f9a8 | 2017-09-07 21:17:33 -0600 | [diff] [blame] | 196 | /* Fix up APIC id with reality. */ |
| 197 | info->cpu->path.apic.apic_id = lapicid(); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 198 | |
Arthur Heymans | 9887264 | 2021-01-22 19:05:55 +0100 | [diff] [blame] | 199 | if (cpu_is_intel()) |
| 200 | printk(BIOS_INFO, "AP: slot %d apic_id %x, MCU rev: 0x%08x\n", cpu, |
| 201 | info->cpu->path.apic.apic_id, get_current_microcode_rev()); |
| 202 | else |
| 203 | printk(BIOS_INFO, "AP: slot %d apic_id %x\n", cpu, |
| 204 | info->cpu->path.apic.apic_id); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 205 | |
| 206 | /* Walk the flight plan */ |
| 207 | ap_do_flight_plan(); |
| 208 | |
| 209 | /* Park the AP. */ |
Subrata Banik | 3337497 | 2018-04-24 13:45:30 +0530 | [diff] [blame] | 210 | park_this_cpu(NULL); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 211 | } |
| 212 | |
| 213 | static void setup_default_sipi_vector_params(struct sipi_params *sp) |
| 214 | { |
Kyösti Mälkki | 2fbb677 | 2018-05-15 19:50:20 +0300 | [diff] [blame] | 215 | sp->gdt = (uintptr_t)&gdt; |
| 216 | sp->gdtlimit = (uintptr_t)&gdt_end - (uintptr_t)&gdt - 1; |
| 217 | sp->idt_ptr = (uintptr_t)&idtarg; |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 218 | sp->stack_size = CONFIG_STACK_SIZE; |
Kyösti Mälkki | 2fbb677 | 2018-05-15 19:50:20 +0300 | [diff] [blame] | 219 | sp->stack_top = ALIGN_DOWN((uintptr_t)&_estack, CONFIG_STACK_SIZE); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 220 | /* Adjust the stack top to take into account cpu_info. */ |
| 221 | sp->stack_top -= sizeof(struct cpu_info); |
| 222 | } |
| 223 | |
| 224 | #define NUM_FIXED_MTRRS 11 |
| 225 | static const unsigned int fixed_mtrrs[NUM_FIXED_MTRRS] = { |
Alexandru Gagniuc | 86091f9 | 2015-09-30 20:23:09 -0700 | [diff] [blame] | 226 | MTRR_FIX_64K_00000, MTRR_FIX_16K_80000, MTRR_FIX_16K_A0000, |
| 227 | MTRR_FIX_4K_C0000, MTRR_FIX_4K_C8000, MTRR_FIX_4K_D0000, |
| 228 | MTRR_FIX_4K_D8000, MTRR_FIX_4K_E0000, MTRR_FIX_4K_E8000, |
| 229 | MTRR_FIX_4K_F0000, MTRR_FIX_4K_F8000, |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 230 | }; |
| 231 | |
| 232 | static inline struct saved_msr *save_msr(int index, struct saved_msr *entry) |
| 233 | { |
| 234 | msr_t msr; |
| 235 | |
| 236 | msr = rdmsr(index); |
| 237 | entry->index = index; |
| 238 | entry->lo = msr.lo; |
| 239 | entry->hi = msr.hi; |
| 240 | |
| 241 | /* Return the next entry. */ |
| 242 | entry++; |
| 243 | return entry; |
| 244 | } |
| 245 | |
| 246 | static int save_bsp_msrs(char *start, int size) |
| 247 | { |
| 248 | int msr_count; |
| 249 | int num_var_mtrrs; |
| 250 | struct saved_msr *msr_entry; |
| 251 | int i; |
| 252 | msr_t msr; |
| 253 | |
| 254 | /* Determine number of MTRRs need to be saved. */ |
Alexandru Gagniuc | 86091f9 | 2015-09-30 20:23:09 -0700 | [diff] [blame] | 255 | msr = rdmsr(MTRR_CAP_MSR); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 256 | num_var_mtrrs = msr.lo & 0xff; |
| 257 | |
| 258 | /* 2 * num_var_mtrrs for base and mask. +1 for IA32_MTRR_DEF_TYPE. */ |
| 259 | msr_count = 2 * num_var_mtrrs + NUM_FIXED_MTRRS + 1; |
| 260 | |
| 261 | if ((msr_count * sizeof(struct saved_msr)) > size) { |
| 262 | printk(BIOS_CRIT, "Cannot mirror all %d msrs.\n", msr_count); |
| 263 | return -1; |
| 264 | } |
| 265 | |
Marshall Dawson | c0dbeda | 2017-10-19 09:45:16 -0600 | [diff] [blame] | 266 | fixed_mtrrs_expose_amd_rwdram(); |
| 267 | |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 268 | msr_entry = (void *)start; |
Lee Leahy | a15d8af | 2017-03-15 14:49:35 -0700 | [diff] [blame] | 269 | for (i = 0; i < NUM_FIXED_MTRRS; i++) |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 270 | msr_entry = save_msr(fixed_mtrrs[i], msr_entry); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 271 | |
| 272 | for (i = 0; i < num_var_mtrrs; i++) { |
Alexandru Gagniuc | 86091f9 | 2015-09-30 20:23:09 -0700 | [diff] [blame] | 273 | msr_entry = save_msr(MTRR_PHYS_BASE(i), msr_entry); |
| 274 | msr_entry = save_msr(MTRR_PHYS_MASK(i), msr_entry); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 275 | } |
| 276 | |
Alexandru Gagniuc | 86091f9 | 2015-09-30 20:23:09 -0700 | [diff] [blame] | 277 | msr_entry = save_msr(MTRR_DEF_TYPE_MSR, msr_entry); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 278 | |
Marshall Dawson | c0dbeda | 2017-10-19 09:45:16 -0600 | [diff] [blame] | 279 | fixed_mtrrs_hide_amd_rwdram(); |
| 280 | |
Richard Spiegel | 43bd594 | 2018-08-08 09:45:23 -0700 | [diff] [blame] | 281 | /* Tell static analysis we know value is left unused. */ |
| 282 | (void)msr_entry; |
| 283 | |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 284 | return msr_count; |
| 285 | } |
| 286 | |
| 287 | static atomic_t *load_sipi_vector(struct mp_params *mp_params) |
| 288 | { |
| 289 | struct rmodule sipi_mod; |
| 290 | int module_size; |
| 291 | int num_msrs; |
| 292 | struct sipi_params *sp; |
| 293 | char *mod_loc = (void *)sipi_vector_location; |
| 294 | const int loc_size = sipi_vector_location_size; |
| 295 | atomic_t *ap_count = NULL; |
| 296 | |
| 297 | if (rmodule_parse(&_binary_sipi_vector_start, &sipi_mod)) { |
| 298 | printk(BIOS_CRIT, "Unable to parse sipi module.\n"); |
| 299 | return ap_count; |
| 300 | } |
| 301 | |
| 302 | if (rmodule_entry_offset(&sipi_mod) != 0) { |
| 303 | printk(BIOS_CRIT, "SIPI module entry offset is not 0!\n"); |
| 304 | return ap_count; |
| 305 | } |
| 306 | |
| 307 | if (rmodule_load_alignment(&sipi_mod) != 4096) { |
| 308 | printk(BIOS_CRIT, "SIPI module load alignment(%d) != 4096.\n", |
| 309 | rmodule_load_alignment(&sipi_mod)); |
| 310 | return ap_count; |
| 311 | } |
| 312 | |
| 313 | module_size = rmodule_memory_size(&sipi_mod); |
| 314 | |
| 315 | /* Align to 4 bytes. */ |
Felix Held | f0cbb09 | 2019-06-20 14:45:16 +0200 | [diff] [blame] | 316 | module_size = ALIGN_UP(module_size, 4); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 317 | |
| 318 | if (module_size > loc_size) { |
| 319 | printk(BIOS_CRIT, "SIPI module size (%d) > region size (%d).\n", |
| 320 | module_size, loc_size); |
| 321 | return ap_count; |
| 322 | } |
| 323 | |
| 324 | num_msrs = save_bsp_msrs(&mod_loc[module_size], loc_size - module_size); |
| 325 | |
| 326 | if (num_msrs < 0) { |
| 327 | printk(BIOS_CRIT, "Error mirroring BSP's msrs.\n"); |
| 328 | return ap_count; |
| 329 | } |
| 330 | |
| 331 | if (rmodule_load(mod_loc, &sipi_mod)) { |
| 332 | printk(BIOS_CRIT, "Unable to load SIPI module.\n"); |
| 333 | return ap_count; |
| 334 | } |
| 335 | |
| 336 | sp = rmodule_parameters(&sipi_mod); |
| 337 | |
| 338 | if (sp == NULL) { |
| 339 | printk(BIOS_CRIT, "SIPI module has no parameters.\n"); |
| 340 | return ap_count; |
| 341 | } |
| 342 | |
| 343 | setup_default_sipi_vector_params(sp); |
| 344 | /* Setup MSR table. */ |
Patrick Rudolph | 6c46b6f | 2020-08-21 16:43:15 +0200 | [diff] [blame] | 345 | sp->msr_table_ptr = (uintptr_t)&mod_loc[module_size]; |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 346 | sp->msr_count = num_msrs; |
| 347 | /* Provide pointer to microcode patch. */ |
Patrick Rudolph | 6c46b6f | 2020-08-21 16:43:15 +0200 | [diff] [blame] | 348 | sp->microcode_ptr = (uintptr_t)mp_params->microcode_pointer; |
Elyes HAOUAS | 1a8dbfc | 2019-12-18 13:40:50 +0100 | [diff] [blame] | 349 | /* Pass on ability to load microcode in parallel. */ |
Lee Leahy | a15d8af | 2017-03-15 14:49:35 -0700 | [diff] [blame] | 350 | if (mp_params->parallel_microcode_load) |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 351 | sp->microcode_lock = ~0; |
Patrick Rudolph | 393992f | 2021-01-11 09:35:49 +0100 | [diff] [blame] | 352 | else |
| 353 | sp->microcode_lock = 0; |
Patrick Rudolph | 6c46b6f | 2020-08-21 16:43:15 +0200 | [diff] [blame] | 354 | sp->c_handler = (uintptr_t)&ap_init; |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 355 | ap_count = &sp->ap_count; |
| 356 | atomic_set(ap_count, 0); |
| 357 | |
| 358 | return ap_count; |
| 359 | } |
| 360 | |
| 361 | static int allocate_cpu_devices(struct bus *cpu_bus, struct mp_params *p) |
| 362 | { |
| 363 | int i; |
| 364 | int max_cpus; |
| 365 | struct cpu_info *info; |
| 366 | |
| 367 | max_cpus = p->num_cpus; |
| 368 | if (max_cpus > CONFIG_MAX_CPUS) { |
| 369 | printk(BIOS_CRIT, "CPU count(%d) exceeds CONFIG_MAX_CPUS(%d)\n", |
| 370 | max_cpus, CONFIG_MAX_CPUS); |
| 371 | max_cpus = CONFIG_MAX_CPUS; |
| 372 | } |
| 373 | |
| 374 | info = cpu_info(); |
| 375 | for (i = 1; i < max_cpus; i++) { |
| 376 | struct device_path cpu_path; |
Edward O'Callaghan | 2c9d2cf | 2014-10-27 23:29:29 +1100 | [diff] [blame] | 377 | struct device *new; |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 378 | |
Elyes HAOUAS | d82be92 | 2016-07-28 18:58:27 +0200 | [diff] [blame] | 379 | /* Build the CPU device path */ |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 380 | cpu_path.type = DEVICE_PATH_APIC; |
| 381 | |
Aaron Durbin | 5a1f9a8 | 2017-09-07 21:17:33 -0600 | [diff] [blame] | 382 | /* Assuming linear APIC space allocation. AP will set its own |
| 383 | APIC id in the ap_init() path above. */ |
| 384 | cpu_path.apic.apic_id = info->cpu->path.apic.apic_id + i; |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 385 | |
Elyes HAOUAS | d82be92 | 2016-07-28 18:58:27 +0200 | [diff] [blame] | 386 | /* Allocate the new CPU device structure */ |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 387 | new = alloc_find_dev(cpu_bus, &cpu_path); |
| 388 | if (new == NULL) { |
Elyes HAOUAS | d82be92 | 2016-07-28 18:58:27 +0200 | [diff] [blame] | 389 | printk(BIOS_CRIT, "Could not allocate CPU device\n"); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 390 | max_cpus--; |
Richard Spiegel | 569711a | 2018-08-07 15:59:34 -0700 | [diff] [blame] | 391 | continue; |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 392 | } |
Naresh G Solanki | 2463533 | 2018-05-31 23:13:18 +0530 | [diff] [blame] | 393 | new->name = processor_name; |
Subrata Banik | 7bc9036 | 2019-05-10 11:58:37 +0530 | [diff] [blame] | 394 | cpus_dev[i] = new; |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 395 | } |
| 396 | |
| 397 | return max_cpus; |
| 398 | } |
| 399 | |
| 400 | /* Returns 1 for timeout. 0 on success. */ |
| 401 | static int apic_wait_timeout(int total_delay, int delay_step) |
| 402 | { |
| 403 | int total = 0; |
| 404 | int timeout = 0; |
| 405 | |
| 406 | while (lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY) { |
| 407 | udelay(delay_step); |
| 408 | total += delay_step; |
| 409 | if (total >= total_delay) { |
| 410 | timeout = 1; |
| 411 | break; |
| 412 | } |
| 413 | } |
| 414 | |
| 415 | return timeout; |
| 416 | } |
| 417 | |
| 418 | static int start_aps(struct bus *cpu_bus, int ap_count, atomic_t *num_aps) |
| 419 | { |
| 420 | int sipi_vector; |
| 421 | /* Max location is 4KiB below 1MiB */ |
| 422 | const int max_vector_loc = ((1 << 20) - (1 << 12)) >> 12; |
| 423 | |
| 424 | if (ap_count == 0) |
| 425 | return 0; |
| 426 | |
| 427 | /* The vector is sent as a 4k aligned address in one byte. */ |
| 428 | sipi_vector = sipi_vector_location >> 12; |
| 429 | |
| 430 | if (sipi_vector > max_vector_loc) { |
| 431 | printk(BIOS_CRIT, "SIPI vector too large! 0x%08x\n", |
| 432 | sipi_vector); |
| 433 | return -1; |
| 434 | } |
| 435 | |
| 436 | printk(BIOS_DEBUG, "Attempting to start %d APs\n", ap_count); |
| 437 | |
Wonkyu Kim | 26ab9bf | 2021-03-22 19:59:18 -0700 | [diff] [blame] | 438 | if (is_x2apic_mode()) { |
| 439 | x2apic_send_ipi(LAPIC_DM_INIT | LAPIC_INT_LEVELTRIG | |
| 440 | LAPIC_INT_ASSERT | LAPIC_DEST_ALLBUT, 0); |
| 441 | mdelay(10); |
| 442 | x2apic_send_ipi(LAPIC_DM_STARTUP | LAPIC_INT_LEVELTRIG | |
| 443 | LAPIC_DEST_ALLBUT | sipi_vector, 0); |
| 444 | |
| 445 | /* Wait for CPUs to check in up to 200 us. */ |
| 446 | wait_for_aps(num_aps, ap_count, 200 /* us */, 15 /* us */); |
| 447 | |
| 448 | x2apic_send_ipi(LAPIC_DM_STARTUP | LAPIC_INT_LEVELTRIG | |
| 449 | LAPIC_DEST_ALLBUT | sipi_vector, 0); |
| 450 | |
| 451 | /* Wait for CPUs to check in. */ |
| 452 | if (wait_for_aps(num_aps, ap_count, 100000 /* 100 ms */, 50 /* us */)) { |
| 453 | printk(BIOS_ERR, "Not all APs checked in: %d/%d.\n", |
| 454 | atomic_read(num_aps), ap_count); |
| 455 | return -1; |
| 456 | } |
| 457 | return 0; |
| 458 | } |
| 459 | |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 460 | if ((lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY)) { |
| 461 | printk(BIOS_DEBUG, "Waiting for ICR not to be busy..."); |
| 462 | if (apic_wait_timeout(1000 /* 1 ms */, 50)) { |
Jonathan Zhang | cbbce66b | 2020-10-28 11:35:40 -0700 | [diff] [blame] | 463 | printk(BIOS_ERR, "timed out. Aborting.\n"); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 464 | return -1; |
Lee Leahy | a15d8af | 2017-03-15 14:49:35 -0700 | [diff] [blame] | 465 | } |
| 466 | printk(BIOS_DEBUG, "done.\n"); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 467 | } |
| 468 | |
| 469 | /* Send INIT IPI to all but self. */ |
| 470 | lapic_write_around(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(0)); |
| 471 | lapic_write_around(LAPIC_ICR, LAPIC_DEST_ALLBUT | LAPIC_INT_ASSERT | |
Lee Leahy | a07d0dd | 2017-03-15 14:25:22 -0700 | [diff] [blame] | 472 | LAPIC_DM_INIT); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 473 | printk(BIOS_DEBUG, "Waiting for 10ms after sending INIT.\n"); |
| 474 | mdelay(10); |
| 475 | |
| 476 | /* Send 1st SIPI */ |
| 477 | if ((lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY)) { |
| 478 | printk(BIOS_DEBUG, "Waiting for ICR not to be busy..."); |
| 479 | if (apic_wait_timeout(1000 /* 1 ms */, 50)) { |
Jonathan Zhang | cbbce66b | 2020-10-28 11:35:40 -0700 | [diff] [blame] | 480 | printk(BIOS_ERR, "timed out. Aborting.\n"); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 481 | return -1; |
Lee Leahy | a15d8af | 2017-03-15 14:49:35 -0700 | [diff] [blame] | 482 | } |
| 483 | printk(BIOS_DEBUG, "done.\n"); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 484 | } |
| 485 | |
| 486 | lapic_write_around(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(0)); |
| 487 | lapic_write_around(LAPIC_ICR, LAPIC_DEST_ALLBUT | LAPIC_INT_ASSERT | |
Lee Leahy | a07d0dd | 2017-03-15 14:25:22 -0700 | [diff] [blame] | 488 | LAPIC_DM_STARTUP | sipi_vector); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 489 | printk(BIOS_DEBUG, "Waiting for 1st SIPI to complete..."); |
| 490 | if (apic_wait_timeout(10000 /* 10 ms */, 50 /* us */)) { |
Jonathan Zhang | cbbce66b | 2020-10-28 11:35:40 -0700 | [diff] [blame] | 491 | printk(BIOS_ERR, "timed out.\n"); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 492 | return -1; |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 493 | } |
Lee Leahy | a15d8af | 2017-03-15 14:49:35 -0700 | [diff] [blame] | 494 | printk(BIOS_DEBUG, "done.\n"); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 495 | |
| 496 | /* Wait for CPUs to check in up to 200 us. */ |
| 497 | wait_for_aps(num_aps, ap_count, 200 /* us */, 15 /* us */); |
| 498 | |
Marshall Dawson | 98f43a1 | 2019-08-05 16:18:56 -0600 | [diff] [blame] | 499 | if (CONFIG(X86_AMD_INIT_SIPI)) |
| 500 | return 0; |
| 501 | |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 502 | /* Send 2nd SIPI */ |
| 503 | if ((lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY)) { |
| 504 | printk(BIOS_DEBUG, "Waiting for ICR not to be busy..."); |
| 505 | if (apic_wait_timeout(1000 /* 1 ms */, 50)) { |
Jonathan Zhang | cbbce66b | 2020-10-28 11:35:40 -0700 | [diff] [blame] | 506 | printk(BIOS_ERR, "timed out. Aborting.\n"); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 507 | return -1; |
Lee Leahy | a15d8af | 2017-03-15 14:49:35 -0700 | [diff] [blame] | 508 | } |
| 509 | printk(BIOS_DEBUG, "done.\n"); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 510 | } |
| 511 | |
| 512 | lapic_write_around(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(0)); |
| 513 | lapic_write_around(LAPIC_ICR, LAPIC_DEST_ALLBUT | LAPIC_INT_ASSERT | |
Lee Leahy | a07d0dd | 2017-03-15 14:25:22 -0700 | [diff] [blame] | 514 | LAPIC_DM_STARTUP | sipi_vector); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 515 | printk(BIOS_DEBUG, "Waiting for 2nd SIPI to complete..."); |
| 516 | if (apic_wait_timeout(10000 /* 10 ms */, 50 /* us */)) { |
Jonathan Zhang | cbbce66b | 2020-10-28 11:35:40 -0700 | [diff] [blame] | 517 | printk(BIOS_ERR, "timed out.\n"); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 518 | return -1; |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 519 | } |
Lee Leahy | a15d8af | 2017-03-15 14:49:35 -0700 | [diff] [blame] | 520 | printk(BIOS_DEBUG, "done.\n"); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 521 | |
| 522 | /* Wait for CPUs to check in. */ |
Jonathan Zhang | cbbce66b | 2020-10-28 11:35:40 -0700 | [diff] [blame] | 523 | if (wait_for_aps(num_aps, ap_count, 100000 /* 100 ms */, 50 /* us */)) { |
| 524 | printk(BIOS_ERR, "Not all APs checked in: %d/%d.\n", |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 525 | atomic_read(num_aps), ap_count); |
| 526 | return -1; |
| 527 | } |
| 528 | |
| 529 | return 0; |
| 530 | } |
| 531 | |
| 532 | static int bsp_do_flight_plan(struct mp_params *mp_params) |
| 533 | { |
| 534 | int i; |
| 535 | int ret = 0; |
Furquan Shaikh | fa9f107 | 2018-03-01 16:37:06 -0800 | [diff] [blame] | 536 | /* |
Jonathan Zhang | 6ec322e | 2020-01-16 11:11:09 -0800 | [diff] [blame] | 537 | * Set time out for flight plan to a huge minimum value (>=1 second). |
| 538 | * CPUs with many APs may take longer if there is contention for |
| 539 | * resources such as UART, so scale the time out up by increments of |
| 540 | * 100ms if needed. |
Furquan Shaikh | fa9f107 | 2018-03-01 16:37:06 -0800 | [diff] [blame] | 541 | */ |
Jonathan Zhang | 6ec322e | 2020-01-16 11:11:09 -0800 | [diff] [blame] | 542 | const int timeout_us = MAX(1000000, 100000 * mp_params->num_cpus); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 543 | const int step_us = 100; |
| 544 | int num_aps = mp_params->num_cpus - 1; |
Furquan Shaikh | 5d8faef | 2018-03-07 23:16:57 -0800 | [diff] [blame] | 545 | struct stopwatch sw; |
| 546 | |
| 547 | stopwatch_init(&sw); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 548 | |
| 549 | for (i = 0; i < mp_params->num_records; i++) { |
| 550 | struct mp_flight_record *rec = &mp_params->flight_plan[i]; |
| 551 | |
| 552 | /* Wait for APs if the record is not released. */ |
| 553 | if (atomic_read(&rec->barrier) == 0) { |
| 554 | /* Wait for the APs to check in. */ |
| 555 | if (wait_for_aps(&rec->cpus_entered, num_aps, |
Lee Leahy | a07d0dd | 2017-03-15 14:25:22 -0700 | [diff] [blame] | 556 | timeout_us, step_us)) { |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 557 | printk(BIOS_ERR, "MP record %d timeout.\n", i); |
| 558 | ret = -1; |
| 559 | } |
| 560 | } |
| 561 | |
Lee Leahy | a15d8af | 2017-03-15 14:49:35 -0700 | [diff] [blame] | 562 | if (rec->bsp_call != NULL) |
Aaron Durbin | 0e55632 | 2016-04-29 23:15:12 -0500 | [diff] [blame] | 563 | rec->bsp_call(); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 564 | |
| 565 | release_barrier(&rec->barrier); |
| 566 | } |
Furquan Shaikh | 5d8faef | 2018-03-07 23:16:57 -0800 | [diff] [blame] | 567 | |
| 568 | printk(BIOS_INFO, "%s done after %ld msecs.\n", __func__, |
| 569 | stopwatch_duration_msecs(&sw)); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 570 | return ret; |
| 571 | } |
| 572 | |
| 573 | static void init_bsp(struct bus *cpu_bus) |
| 574 | { |
| 575 | struct device_path cpu_path; |
| 576 | struct cpu_info *info; |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 577 | |
| 578 | /* Print processor name */ |
| 579 | fill_processor_name(processor_name); |
| 580 | printk(BIOS_INFO, "CPU: %s.\n", processor_name); |
| 581 | |
Elyes HAOUAS | d6e9686 | 2016-08-21 10:12:15 +0200 | [diff] [blame] | 582 | /* Ensure the local APIC is enabled */ |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 583 | enable_lapic(); |
| 584 | |
Elyes HAOUAS | d6e9686 | 2016-08-21 10:12:15 +0200 | [diff] [blame] | 585 | /* Set the device path of the boot CPU. */ |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 586 | cpu_path.type = DEVICE_PATH_APIC; |
| 587 | cpu_path.apic.apic_id = lapicid(); |
| 588 | |
Elyes HAOUAS | d6e9686 | 2016-08-21 10:12:15 +0200 | [diff] [blame] | 589 | /* Find the device structure for the boot CPU. */ |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 590 | info = cpu_info(); |
| 591 | info->cpu = alloc_find_dev(cpu_bus, &cpu_path); |
Naresh G Solanki | 2463533 | 2018-05-31 23:13:18 +0530 | [diff] [blame] | 592 | info->cpu->name = processor_name; |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 593 | |
| 594 | if (info->index != 0) |
| 595 | printk(BIOS_CRIT, "BSP index(%d) != 0!\n", info->index); |
| 596 | |
| 597 | /* Track BSP in cpu_map structures. */ |
Subrata Banik | 7bc9036 | 2019-05-10 11:58:37 +0530 | [diff] [blame] | 598 | cpu_add_map_entry(info->index); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 599 | } |
| 600 | |
Aaron Durbin | 770d7c7 | 2016-05-03 17:49:57 -0500 | [diff] [blame] | 601 | /* |
| 602 | * mp_init() will set up the SIPI vector and bring up the APs according to |
| 603 | * mp_params. Each flight record will be executed according to the plan. Note |
| 604 | * that the MP infrastructure uses SMM default area without saving it. It's |
| 605 | * up to the chipset or mainboard to either e820 reserve this area or save this |
| 606 | * region prior to calling mp_init() and restoring it after mp_init returns. |
| 607 | * |
| 608 | * At the time mp_init() is called the MTRR MSRs are mirrored into APs then |
| 609 | * caching is enabled before running the flight plan. |
| 610 | * |
| 611 | * The MP initialization has the following properties: |
| 612 | * 1. APs are brought up in parallel. |
Elyes HAOUAS | d82be92 | 2016-07-28 18:58:27 +0200 | [diff] [blame] | 613 | * 2. The ordering of coreboot CPU number and APIC ids is not deterministic. |
Aaron Durbin | 770d7c7 | 2016-05-03 17:49:57 -0500 | [diff] [blame] | 614 | * Therefore, one cannot rely on this property or the order of devices in |
| 615 | * the device tree unless the chipset or mainboard know the APIC ids |
| 616 | * a priori. |
| 617 | * |
| 618 | * mp_init() returns < 0 on error, 0 on success. |
| 619 | */ |
| 620 | static int mp_init(struct bus *cpu_bus, struct mp_params *p) |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 621 | { |
| 622 | int num_cpus; |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 623 | atomic_t *ap_count; |
| 624 | |
| 625 | init_bsp(cpu_bus); |
| 626 | |
| 627 | if (p == NULL || p->flight_plan == NULL || p->num_records < 1) { |
| 628 | printk(BIOS_CRIT, "Invalid MP parameters\n"); |
| 629 | return -1; |
| 630 | } |
| 631 | |
| 632 | /* Default to currently running CPU. */ |
| 633 | num_cpus = allocate_cpu_devices(cpu_bus, p); |
| 634 | |
| 635 | if (num_cpus < p->num_cpus) { |
| 636 | printk(BIOS_CRIT, |
| 637 | "ERROR: More cpus requested (%d) than supported (%d).\n", |
| 638 | p->num_cpus, num_cpus); |
| 639 | return -1; |
| 640 | } |
| 641 | |
| 642 | /* Copy needed parameters so that APs have a reference to the plan. */ |
| 643 | mp_info.num_records = p->num_records; |
| 644 | mp_info.records = p->flight_plan; |
| 645 | |
| 646 | /* Load the SIPI vector. */ |
| 647 | ap_count = load_sipi_vector(p); |
| 648 | if (ap_count == NULL) |
| 649 | return -1; |
| 650 | |
| 651 | /* Make sure SIPI data hits RAM so the APs that come up will see |
| 652 | * the startup code even if the caches are disabled. */ |
| 653 | wbinvd(); |
| 654 | |
| 655 | /* Start the APs providing number of APs and the cpus_entered field. */ |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 656 | global_num_aps = p->num_cpus - 1; |
| 657 | if (start_aps(cpu_bus, global_num_aps, ap_count) < 0) { |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 658 | mdelay(1000); |
| 659 | printk(BIOS_DEBUG, "%d/%d eventually checked in?\n", |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 660 | atomic_read(ap_count), global_num_aps); |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 661 | return -1; |
| 662 | } |
| 663 | |
| 664 | /* Walk the flight plan for the BSP. */ |
| 665 | return bsp_do_flight_plan(p); |
| 666 | } |
| 667 | |
Aaron Durbin | 770d7c7 | 2016-05-03 17:49:57 -0500 | [diff] [blame] | 668 | /* Calls cpu_initialize(info->index) which calls the coreboot CPU drivers. */ |
| 669 | static void mp_initialize_cpu(void) |
Aaron Durbin | e0785c0 | 2013-10-21 12:15:29 -0500 | [diff] [blame] | 670 | { |
| 671 | /* Call back into driver infrastructure for the AP initialization. */ |
| 672 | struct cpu_info *info = cpu_info(); |
| 673 | cpu_initialize(info->index); |
| 674 | } |
| 675 | |
Aaron Durbin | cd3f8ad | 2013-10-21 22:24:40 -0500 | [diff] [blame] | 676 | void smm_initiate_relocation_parallel(void) |
| 677 | { |
Wonkyu Kim | 26ab9bf | 2021-03-22 19:59:18 -0700 | [diff] [blame] | 678 | if (is_x2apic_mode()) { |
| 679 | x2apic_send_ipi(LAPIC_DM_SMI | LAPIC_INT_LEVELTRIG, lapicid()); |
| 680 | return; |
| 681 | } |
| 682 | |
Aaron Durbin | cd3f8ad | 2013-10-21 22:24:40 -0500 | [diff] [blame] | 683 | if ((lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY)) { |
| 684 | printk(BIOS_DEBUG, "Waiting for ICR not to be busy..."); |
| 685 | if (apic_wait_timeout(1000 /* 1 ms */, 50)) { |
| 686 | printk(BIOS_DEBUG, "timed out. Aborting.\n"); |
| 687 | return; |
Lee Leahy | a15d8af | 2017-03-15 14:49:35 -0700 | [diff] [blame] | 688 | } |
| 689 | printk(BIOS_DEBUG, "done.\n"); |
Aaron Durbin | cd3f8ad | 2013-10-21 22:24:40 -0500 | [diff] [blame] | 690 | } |
| 691 | |
| 692 | lapic_write_around(LAPIC_ICR2, SET_LAPIC_DEST_FIELD(lapicid())); |
| 693 | lapic_write_around(LAPIC_ICR, LAPIC_INT_ASSERT | LAPIC_DM_SMI); |
Lee Leahy | a15d8af | 2017-03-15 14:49:35 -0700 | [diff] [blame] | 694 | if (apic_wait_timeout(1000 /* 1 ms */, 100 /* us */)) |
Aaron Durbin | cd3f8ad | 2013-10-21 22:24:40 -0500 | [diff] [blame] | 695 | printk(BIOS_DEBUG, "SMI Relocation timed out.\n"); |
Lee Leahy | a15d8af | 2017-03-15 14:49:35 -0700 | [diff] [blame] | 696 | else |
Aaron Durbin | cd3f8ad | 2013-10-21 22:24:40 -0500 | [diff] [blame] | 697 | printk(BIOS_DEBUG, "Relocation complete.\n"); |
Aaron Durbin | cd3f8ad | 2013-10-21 22:24:40 -0500 | [diff] [blame] | 698 | } |
| 699 | |
| 700 | DECLARE_SPIN_LOCK(smm_relocation_lock); |
| 701 | |
| 702 | /* Send SMI to self with single user serialization. */ |
| 703 | void smm_initiate_relocation(void) |
| 704 | { |
| 705 | spin_lock(&smm_relocation_lock); |
| 706 | smm_initiate_relocation_parallel(); |
| 707 | spin_unlock(&smm_relocation_lock); |
| 708 | } |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 709 | |
| 710 | struct mp_state { |
| 711 | struct mp_ops ops; |
| 712 | int cpu_count; |
| 713 | uintptr_t perm_smbase; |
| 714 | size_t perm_smsize; |
Arthur Heymans | 478f3d8 | 2021-02-15 19:39:01 +0100 | [diff] [blame] | 715 | /* Size of the real CPU save state */ |
| 716 | size_t smm_real_save_state_size; |
| 717 | /* Size of allocated CPU save state, MAX(real save state size, stub size) */ |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 718 | size_t smm_save_state_size; |
Arthur Heymans | 1dfa46e | 2021-02-15 16:19:33 +0100 | [diff] [blame] | 719 | uintptr_t reloc_start32_offset; |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 720 | int do_smm; |
| 721 | } mp_state; |
| 722 | |
| 723 | static int is_smm_enabled(void) |
| 724 | { |
Julius Werner | cd49cce | 2019-03-05 16:53:33 -0800 | [diff] [blame] | 725 | return CONFIG(HAVE_SMI_HANDLER) && mp_state.do_smm; |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 726 | } |
| 727 | |
| 728 | static void smm_disable(void) |
| 729 | { |
| 730 | mp_state.do_smm = 0; |
| 731 | } |
| 732 | |
| 733 | static void smm_enable(void) |
| 734 | { |
Julius Werner | cd49cce | 2019-03-05 16:53:33 -0800 | [diff] [blame] | 735 | if (CONFIG(HAVE_SMI_HANDLER)) |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 736 | mp_state.do_smm = 1; |
| 737 | } |
| 738 | |
| 739 | static void asmlinkage smm_do_relocation(void *arg) |
| 740 | { |
| 741 | const struct smm_module_params *p; |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 742 | int cpu; |
Arthur Heymans | 50e849f | 2021-02-15 16:43:19 +0100 | [diff] [blame] | 743 | const uintptr_t curr_smbase = SMM_DEFAULT_BASE; |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 744 | uintptr_t perm_smbase; |
| 745 | |
| 746 | p = arg; |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 747 | cpu = p->cpu; |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 748 | |
| 749 | if (cpu >= CONFIG_MAX_CPUS) { |
| 750 | printk(BIOS_CRIT, |
| 751 | "Invalid CPU number assigned in SMM stub: %d\n", cpu); |
| 752 | return; |
| 753 | } |
| 754 | |
| 755 | /* |
| 756 | * The permanent handler runs with all cpus concurrently. Precalculate |
| 757 | * the location of the new SMBASE. If using SMM modules then this |
| 758 | * calculation needs to match that of the module loader. |
| 759 | */ |
Arthur Heymans | 88407bc | 2021-03-02 16:07:52 +0100 | [diff] [blame] | 760 | perm_smbase = smm_get_cpu_smbase(cpu); |
| 761 | if (!perm_smbase) { |
| 762 | printk(BIOS_ERR, "%s: bad SMBASE for CPU %d\n", __func__, cpu); |
| 763 | return; |
Rocky Phagura | afb7a81 | 2020-07-21 14:48:48 -0700 | [diff] [blame] | 764 | } |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 765 | |
| 766 | /* Setup code checks this callback for validity. */ |
Rocky Phagura | afb7a81 | 2020-07-21 14:48:48 -0700 | [diff] [blame] | 767 | printk(BIOS_INFO, "%s : curr_smbase 0x%x perm_smbase 0x%x, cpu = %d\n", |
| 768 | __func__, (int)curr_smbase, (int)perm_smbase, cpu); |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 769 | mp_state.ops.relocation_handler(cpu, curr_smbase, perm_smbase); |
Eugene Myers | ae438be | 2020-01-21 17:01:47 -0500 | [diff] [blame] | 770 | |
| 771 | if (CONFIG(STM)) { |
Eugene Myers | 53e9236 | 2020-02-10 15:44:38 -0500 | [diff] [blame] | 772 | uintptr_t mseg; |
Eugene Myers | ae438be | 2020-01-21 17:01:47 -0500 | [diff] [blame] | 773 | |
Eugene Myers | 53e9236 | 2020-02-10 15:44:38 -0500 | [diff] [blame] | 774 | mseg = mp_state.perm_smbase + |
| 775 | (mp_state.perm_smsize - CONFIG_MSEG_SIZE); |
Eugene Myers | ae438be | 2020-01-21 17:01:47 -0500 | [diff] [blame] | 776 | |
Eugene D Myers | f213f17 | 2020-04-15 19:11:52 -0400 | [diff] [blame] | 777 | stm_setup(mseg, p->cpu, |
Eugene Myers | 53e9236 | 2020-02-10 15:44:38 -0500 | [diff] [blame] | 778 | perm_smbase, |
| 779 | mp_state.perm_smbase, |
Arthur Heymans | 1dfa46e | 2021-02-15 16:19:33 +0100 | [diff] [blame] | 780 | mp_state.reloc_start32_offset); |
Eugene Myers | ae438be | 2020-01-21 17:01:47 -0500 | [diff] [blame] | 781 | } |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 782 | } |
| 783 | |
| 784 | static void adjust_smm_apic_id_map(struct smm_loader_params *smm_params) |
| 785 | { |
| 786 | int i; |
Arthur Heymans | ed4be45 | 2021-02-15 13:20:35 +0100 | [diff] [blame] | 787 | struct smm_stub_params *stub_params = smm_params->stub_params; |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 788 | |
| 789 | for (i = 0; i < CONFIG_MAX_CPUS; i++) |
Arthur Heymans | ed4be45 | 2021-02-15 13:20:35 +0100 | [diff] [blame] | 790 | stub_params->apic_id_to_cpu[i] = cpu_get_apic_id(i); |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 791 | } |
| 792 | |
Arthur Heymans | 478f3d8 | 2021-02-15 19:39:01 +0100 | [diff] [blame] | 793 | static int install_relocation_handler(int num_cpus, size_t real_save_state_size, |
Arthur Heymans | e6c3523 | 2021-02-16 13:19:18 +0100 | [diff] [blame] | 794 | size_t save_state_size, uintptr_t perm_smbase) |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 795 | { |
| 796 | struct smm_loader_params smm_params = { |
Marshall Dawson | 46fc6847 | 2018-10-25 13:01:55 -0600 | [diff] [blame] | 797 | .per_cpu_stack_size = CONFIG_SMM_STUB_STACK_SIZE, |
Arthur Heymans | e6c3523 | 2021-02-16 13:19:18 +0100 | [diff] [blame] | 798 | .num_concurrent_stacks = num_cpus, |
Arthur Heymans | 478f3d8 | 2021-02-15 19:39:01 +0100 | [diff] [blame] | 799 | .real_cpu_save_state_size = real_save_state_size, |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 800 | .per_cpu_save_state_size = save_state_size, |
| 801 | .num_concurrent_save_states = 1, |
| 802 | .handler = smm_do_relocation, |
| 803 | }; |
| 804 | |
| 805 | /* Allow callback to override parameters. */ |
| 806 | if (mp_state.ops.adjust_smm_params != NULL) |
| 807 | mp_state.ops.adjust_smm_params(&smm_params, 0); |
| 808 | |
Arthur Heymans | e6c3523 | 2021-02-16 13:19:18 +0100 | [diff] [blame] | 809 | if (smm_setup_relocation_handler((void *)perm_smbase, &smm_params)) { |
Rocky Phagura | afb7a81 | 2020-07-21 14:48:48 -0700 | [diff] [blame] | 810 | printk(BIOS_ERR, "%s: smm setup failed\n", __func__); |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 811 | return -1; |
Rocky Phagura | afb7a81 | 2020-07-21 14:48:48 -0700 | [diff] [blame] | 812 | } |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 813 | adjust_smm_apic_id_map(&smm_params); |
| 814 | |
Arthur Heymans | 1dfa46e | 2021-02-15 16:19:33 +0100 | [diff] [blame] | 815 | mp_state.reloc_start32_offset = smm_params.stub_params->start32_offset; |
| 816 | |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 817 | return 0; |
| 818 | } |
| 819 | |
| 820 | static int install_permanent_handler(int num_cpus, uintptr_t smbase, |
Arthur Heymans | 478f3d8 | 2021-02-15 19:39:01 +0100 | [diff] [blame] | 821 | size_t smsize, size_t real_save_state_size, |
| 822 | size_t save_state_size) |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 823 | { |
Rocky Phagura | afb7a81 | 2020-07-21 14:48:48 -0700 | [diff] [blame] | 824 | /* |
| 825 | * All the CPUs will relocate to permanaent handler now. Set parameters |
| 826 | * needed for all CPUs. The placement of each CPUs entry point is |
| 827 | * determined by the loader. This code simply provides the beginning of |
| 828 | * SMRAM region, the number of CPUs who will use the handler, the stack |
| 829 | * size and save state size for each CPU. |
| 830 | */ |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 831 | struct smm_loader_params smm_params = { |
Raul E Rangel | d3b8393 | 2018-06-12 10:43:09 -0600 | [diff] [blame] | 832 | .per_cpu_stack_size = CONFIG_SMM_MODULE_STACK_SIZE, |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 833 | .num_concurrent_stacks = num_cpus, |
Arthur Heymans | 478f3d8 | 2021-02-15 19:39:01 +0100 | [diff] [blame] | 834 | .real_cpu_save_state_size = real_save_state_size, |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 835 | .per_cpu_save_state_size = save_state_size, |
| 836 | .num_concurrent_save_states = num_cpus, |
| 837 | }; |
| 838 | |
| 839 | /* Allow callback to override parameters. */ |
| 840 | if (mp_state.ops.adjust_smm_params != NULL) |
| 841 | mp_state.ops.adjust_smm_params(&smm_params, 1); |
| 842 | |
Rocky Phagura | afb7a81 | 2020-07-21 14:48:48 -0700 | [diff] [blame] | 843 | printk(BIOS_DEBUG, "Installing permanent SMM handler to 0x%08lx\n", smbase); |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 844 | |
| 845 | if (smm_load_module((void *)smbase, smsize, &smm_params)) |
| 846 | return -1; |
| 847 | |
| 848 | adjust_smm_apic_id_map(&smm_params); |
| 849 | |
| 850 | return 0; |
| 851 | } |
| 852 | |
| 853 | /* Load SMM handlers as part of MP flight record. */ |
| 854 | static void load_smm_handlers(void) |
| 855 | { |
Arthur Heymans | 478f3d8 | 2021-02-15 19:39:01 +0100 | [diff] [blame] | 856 | size_t real_save_state_size = mp_state.smm_real_save_state_size; |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 857 | size_t smm_save_state_size = mp_state.smm_save_state_size; |
| 858 | |
| 859 | /* Do nothing if SMM is disabled.*/ |
| 860 | if (!is_smm_enabled()) |
| 861 | return; |
| 862 | |
| 863 | /* Install handlers. */ |
Arthur Heymans | e6c3523 | 2021-02-16 13:19:18 +0100 | [diff] [blame] | 864 | if (install_relocation_handler(mp_state.cpu_count, real_save_state_size, |
| 865 | smm_save_state_size, mp_state.perm_smbase) < 0) { |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 866 | printk(BIOS_ERR, "Unable to install SMM relocation handler.\n"); |
| 867 | smm_disable(); |
| 868 | } |
| 869 | |
| 870 | if (install_permanent_handler(mp_state.cpu_count, mp_state.perm_smbase, |
Arthur Heymans | 478f3d8 | 2021-02-15 19:39:01 +0100 | [diff] [blame] | 871 | mp_state.perm_smsize, real_save_state_size, |
| 872 | smm_save_state_size) < 0) { |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 873 | printk(BIOS_ERR, "Unable to install SMM permanent handler.\n"); |
| 874 | smm_disable(); |
| 875 | } |
| 876 | |
| 877 | /* Ensure the SMM handlers hit DRAM before performing first SMI. */ |
| 878 | wbinvd(); |
| 879 | |
| 880 | /* |
| 881 | * Indicate that the SMM handlers have been loaded and MP |
| 882 | * initialization is about to start. |
| 883 | */ |
| 884 | if (is_smm_enabled() && mp_state.ops.pre_mp_smm_init != NULL) |
| 885 | mp_state.ops.pre_mp_smm_init(); |
| 886 | } |
| 887 | |
| 888 | /* Trigger SMM as part of MP flight record. */ |
| 889 | static void trigger_smm_relocation(void) |
| 890 | { |
| 891 | /* Do nothing if SMM is disabled.*/ |
| 892 | if (!is_smm_enabled() || mp_state.ops.per_cpu_smm_trigger == NULL) |
| 893 | return; |
| 894 | /* Trigger SMM mode for the currently running processor. */ |
| 895 | mp_state.ops.per_cpu_smm_trigger(); |
| 896 | } |
| 897 | |
Aaron Durbin | 223fb43 | 2018-05-03 13:49:41 -0600 | [diff] [blame] | 898 | static struct mp_callback *ap_callbacks[CONFIG_MAX_CPUS]; |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 899 | |
Aaron Durbin | 223fb43 | 2018-05-03 13:49:41 -0600 | [diff] [blame] | 900 | static struct mp_callback *read_callback(struct mp_callback **slot) |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 901 | { |
Aaron Durbin | 223fb43 | 2018-05-03 13:49:41 -0600 | [diff] [blame] | 902 | struct mp_callback *ret; |
| 903 | |
| 904 | asm volatile ("mov %1, %0\n" |
| 905 | : "=r" (ret) |
| 906 | : "m" (*slot) |
| 907 | : "memory" |
| 908 | ); |
| 909 | return ret; |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 910 | } |
| 911 | |
Aaron Durbin | 223fb43 | 2018-05-03 13:49:41 -0600 | [diff] [blame] | 912 | static void store_callback(struct mp_callback **slot, struct mp_callback *val) |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 913 | { |
Aaron Durbin | 223fb43 | 2018-05-03 13:49:41 -0600 | [diff] [blame] | 914 | asm volatile ("mov %1, %0\n" |
| 915 | : "=m" (*slot) |
| 916 | : "r" (val) |
| 917 | : "memory" |
| 918 | ); |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 919 | } |
| 920 | |
Aaron Durbin | 223fb43 | 2018-05-03 13:49:41 -0600 | [diff] [blame] | 921 | static int run_ap_work(struct mp_callback *val, long expire_us) |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 922 | { |
| 923 | int i; |
| 924 | int cpus_accepted; |
| 925 | struct stopwatch sw; |
Jacob Garber | bc67476 | 2019-05-14 11:21:41 -0600 | [diff] [blame] | 926 | int cur_cpu; |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 927 | |
Julius Werner | cd49cce | 2019-03-05 16:53:33 -0800 | [diff] [blame] | 928 | if (!CONFIG(PARALLEL_MP_AP_WORK)) { |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 929 | printk(BIOS_ERR, "APs already parked. PARALLEL_MP_AP_WORK not selected.\n"); |
| 930 | return -1; |
| 931 | } |
| 932 | |
Jacob Garber | bc67476 | 2019-05-14 11:21:41 -0600 | [diff] [blame] | 933 | cur_cpu = cpu_index(); |
| 934 | |
| 935 | if (cur_cpu < 0) { |
| 936 | printk(BIOS_ERR, "Invalid CPU index.\n"); |
| 937 | return -1; |
| 938 | } |
| 939 | |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 940 | /* Signal to all the APs to run the func. */ |
| 941 | for (i = 0; i < ARRAY_SIZE(ap_callbacks); i++) { |
| 942 | if (cur_cpu == i) |
| 943 | continue; |
Aaron Durbin | 223fb43 | 2018-05-03 13:49:41 -0600 | [diff] [blame] | 944 | store_callback(&ap_callbacks[i], val); |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 945 | } |
| 946 | mfence(); |
| 947 | |
| 948 | /* Wait for all the APs to signal back that call has been accepted. */ |
Subrata Banik | 838f296 | 2018-04-11 18:45:57 +0530 | [diff] [blame] | 949 | if (expire_us > 0) |
| 950 | stopwatch_init_usecs_expire(&sw, expire_us); |
| 951 | |
Paul Menzel | 6bb8ff4 | 2017-06-19 13:02:31 +0200 | [diff] [blame] | 952 | do { |
Aaron Durbin | 046848c | 2017-06-15 08:47:04 -0500 | [diff] [blame] | 953 | cpus_accepted = 0; |
| 954 | |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 955 | for (i = 0; i < ARRAY_SIZE(ap_callbacks); i++) { |
| 956 | if (cur_cpu == i) |
| 957 | continue; |
| 958 | if (read_callback(&ap_callbacks[i]) == NULL) |
| 959 | cpus_accepted++; |
| 960 | } |
Aaron Durbin | 046848c | 2017-06-15 08:47:04 -0500 | [diff] [blame] | 961 | |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 962 | if (cpus_accepted == global_num_aps) |
| 963 | return 0; |
Subrata Banik | 838f296 | 2018-04-11 18:45:57 +0530 | [diff] [blame] | 964 | } while (expire_us <= 0 || !stopwatch_expired(&sw)); |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 965 | |
Subrata Banik | dbcb0ce | 2020-03-19 20:51:09 +0530 | [diff] [blame] | 966 | printk(BIOS_CRIT, "CRITICAL ERROR: AP call expired. %d/%d CPUs accepted.\n", |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 967 | cpus_accepted, global_num_aps); |
| 968 | return -1; |
| 969 | } |
| 970 | |
| 971 | static void ap_wait_for_instruction(void) |
| 972 | { |
Aaron Durbin | 223fb43 | 2018-05-03 13:49:41 -0600 | [diff] [blame] | 973 | struct mp_callback lcb; |
| 974 | struct mp_callback **per_cpu_slot; |
Subrata Banik | 8a25cae | 2018-05-03 18:48:41 +0530 | [diff] [blame] | 975 | int cur_cpu; |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 976 | |
Julius Werner | cd49cce | 2019-03-05 16:53:33 -0800 | [diff] [blame] | 977 | if (!CONFIG(PARALLEL_MP_AP_WORK)) |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 978 | return; |
| 979 | |
Subrata Banik | 8a25cae | 2018-05-03 18:48:41 +0530 | [diff] [blame] | 980 | cur_cpu = cpu_index(); |
Jacob Garber | bc67476 | 2019-05-14 11:21:41 -0600 | [diff] [blame] | 981 | |
| 982 | if (cur_cpu < 0) { |
| 983 | printk(BIOS_ERR, "Invalid CPU index.\n"); |
| 984 | return; |
| 985 | } |
| 986 | |
Subrata Banik | 8a25cae | 2018-05-03 18:48:41 +0530 | [diff] [blame] | 987 | per_cpu_slot = &ap_callbacks[cur_cpu]; |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 988 | |
Aaron Durbin | 223fb43 | 2018-05-03 13:49:41 -0600 | [diff] [blame] | 989 | while (1) { |
| 990 | struct mp_callback *cb = read_callback(per_cpu_slot); |
| 991 | |
| 992 | if (cb == NULL) { |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 993 | asm ("pause"); |
| 994 | continue; |
| 995 | } |
| 996 | |
Raul E Rangel | 9ea7762 | 2018-08-02 15:12:17 -0600 | [diff] [blame] | 997 | /* Copy to local variable before signaling consumption. */ |
Aaron Durbin | 223fb43 | 2018-05-03 13:49:41 -0600 | [diff] [blame] | 998 | memcpy(&lcb, cb, sizeof(lcb)); |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 999 | mfence(); |
Aaron Durbin | 223fb43 | 2018-05-03 13:49:41 -0600 | [diff] [blame] | 1000 | store_callback(per_cpu_slot, NULL); |
Subrata Banik | 8a25cae | 2018-05-03 18:48:41 +0530 | [diff] [blame] | 1001 | if (lcb.logical_cpu_number && (cur_cpu != |
| 1002 | lcb.logical_cpu_number)) |
| 1003 | continue; |
| 1004 | else |
| 1005 | lcb.func(lcb.arg); |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 1006 | } |
| 1007 | } |
| 1008 | |
Subrata Banik | 8a25cae | 2018-05-03 18:48:41 +0530 | [diff] [blame] | 1009 | int mp_run_on_aps(void (*func)(void *), void *arg, int logical_cpu_num, |
| 1010 | long expire_us) |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 1011 | { |
Subrata Banik | 8a25cae | 2018-05-03 18:48:41 +0530 | [diff] [blame] | 1012 | struct mp_callback lcb = { .func = func, .arg = arg, |
| 1013 | .logical_cpu_number = logical_cpu_num}; |
Aaron Durbin | 223fb43 | 2018-05-03 13:49:41 -0600 | [diff] [blame] | 1014 | return run_ap_work(&lcb, expire_us); |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 1015 | } |
| 1016 | |
Aamir Bohra | 7e0019e | 2021-03-05 09:41:20 +0530 | [diff] [blame] | 1017 | int mp_run_on_all_aps(void (*func)(void *), void *arg, long expire_us, bool run_parallel) |
| 1018 | { |
| 1019 | int ap_index, bsp_index; |
| 1020 | |
| 1021 | if (run_parallel) |
| 1022 | return mp_run_on_aps(func, arg, 0, expire_us); |
| 1023 | |
| 1024 | bsp_index = cpu_index(); |
| 1025 | |
| 1026 | const int total_threads = global_num_aps + 1; /* +1 for BSP */ |
| 1027 | |
| 1028 | for (ap_index = 0; ap_index < total_threads; ap_index++) { |
| 1029 | /* skip if BSP */ |
| 1030 | if (ap_index == bsp_index) |
| 1031 | continue; |
| 1032 | if (mp_run_on_aps(func, arg, ap_index, expire_us)) |
| 1033 | return CB_ERR; |
| 1034 | } |
| 1035 | |
| 1036 | return CB_SUCCESS; |
| 1037 | } |
| 1038 | |
Patrick Rudolph | 5ec97ce | 2019-07-26 14:47:32 +0200 | [diff] [blame] | 1039 | int mp_run_on_all_cpus(void (*func)(void *), void *arg) |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 1040 | { |
| 1041 | /* Run on BSP first. */ |
Subrata Banik | 3337497 | 2018-04-24 13:45:30 +0530 | [diff] [blame] | 1042 | func(arg); |
| 1043 | |
Patrick Rudolph | 5ec97ce | 2019-07-26 14:47:32 +0200 | [diff] [blame] | 1044 | /* For up to 1 second for AP to finish previous work. */ |
| 1045 | return mp_run_on_aps(func, arg, MP_RUN_ON_ALL_CPUS, 1000 * USECS_PER_MSEC); |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 1046 | } |
| 1047 | |
| 1048 | int mp_park_aps(void) |
| 1049 | { |
Furquan Shaikh | d6630d1 | 2018-03-29 00:10:02 -0700 | [diff] [blame] | 1050 | struct stopwatch sw; |
| 1051 | int ret; |
| 1052 | long duration_msecs; |
| 1053 | |
| 1054 | stopwatch_init(&sw); |
| 1055 | |
Subrata Banik | 8a25cae | 2018-05-03 18:48:41 +0530 | [diff] [blame] | 1056 | ret = mp_run_on_aps(park_this_cpu, NULL, MP_RUN_ON_ALL_CPUS, |
Patrick Rudolph | 5ec97ce | 2019-07-26 14:47:32 +0200 | [diff] [blame] | 1057 | 1000 * USECS_PER_MSEC); |
Furquan Shaikh | d6630d1 | 2018-03-29 00:10:02 -0700 | [diff] [blame] | 1058 | |
| 1059 | duration_msecs = stopwatch_duration_msecs(&sw); |
| 1060 | |
| 1061 | if (!ret) |
| 1062 | printk(BIOS_DEBUG, "%s done after %ld msecs.\n", __func__, |
| 1063 | duration_msecs); |
| 1064 | else |
| 1065 | printk(BIOS_ERR, "%s failed after %ld msecs.\n", __func__, |
| 1066 | duration_msecs); |
| 1067 | |
| 1068 | return ret; |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 1069 | } |
| 1070 | |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 1071 | static struct mp_flight_record mp_steps[] = { |
| 1072 | /* Once the APs are up load the SMM handlers. */ |
| 1073 | MP_FR_BLOCK_APS(NULL, load_smm_handlers), |
| 1074 | /* Perform SMM relocation. */ |
| 1075 | MP_FR_NOBLOCK_APS(trigger_smm_relocation, trigger_smm_relocation), |
Elyes HAOUAS | d82be92 | 2016-07-28 18:58:27 +0200 | [diff] [blame] | 1076 | /* Initialize each CPU through the driver framework. */ |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 1077 | MP_FR_BLOCK_APS(mp_initialize_cpu, mp_initialize_cpu), |
Aaron Durbin | b21e362 | 2016-12-07 00:32:19 -0600 | [diff] [blame] | 1078 | /* Wait for APs to finish then optionally start looking for work. */ |
| 1079 | MP_FR_BLOCK_APS(ap_wait_for_instruction, NULL), |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 1080 | }; |
| 1081 | |
Arthur Heymans | 478f3d8 | 2021-02-15 19:39:01 +0100 | [diff] [blame] | 1082 | static size_t smm_stub_size(void) |
| 1083 | { |
| 1084 | extern unsigned char _binary_smmstub_start[]; |
| 1085 | struct rmodule smm_stub; |
| 1086 | |
| 1087 | if (rmodule_parse(&_binary_smmstub_start, &smm_stub)) { |
| 1088 | printk(BIOS_ERR, "%s: unable to get SMM module size\n", __func__); |
| 1089 | return 0; |
| 1090 | } |
| 1091 | |
| 1092 | return rmodule_memory_size(&smm_stub); |
| 1093 | } |
| 1094 | |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 1095 | static void fill_mp_state(struct mp_state *state, const struct mp_ops *ops) |
| 1096 | { |
| 1097 | /* |
| 1098 | * Make copy of the ops so that defaults can be set in the non-const |
| 1099 | * structure if needed. |
| 1100 | */ |
| 1101 | memcpy(&state->ops, ops, sizeof(*ops)); |
| 1102 | |
| 1103 | if (ops->get_cpu_count != NULL) |
| 1104 | state->cpu_count = ops->get_cpu_count(); |
| 1105 | |
| 1106 | if (ops->get_smm_info != NULL) |
| 1107 | ops->get_smm_info(&state->perm_smbase, &state->perm_smsize, |
Arthur Heymans | 478f3d8 | 2021-02-15 19:39:01 +0100 | [diff] [blame] | 1108 | &state->smm_real_save_state_size); |
| 1109 | |
| 1110 | state->smm_save_state_size = MAX(state->smm_real_save_state_size, smm_stub_size()); |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 1111 | |
| 1112 | /* |
Eugene Myers | ae438be | 2020-01-21 17:01:47 -0500 | [diff] [blame] | 1113 | * Make sure there is enough room for the SMM descriptor |
| 1114 | */ |
Eugene Myers | faa1118 | 2020-02-06 10:37:01 -0500 | [diff] [blame] | 1115 | if (CONFIG(STM)) { |
Eugene Myers | ae438be | 2020-01-21 17:01:47 -0500 | [diff] [blame] | 1116 | state->smm_save_state_size += |
Eugene Myers | 970ed2a | 2020-02-10 15:02:27 -0500 | [diff] [blame] | 1117 | ALIGN_UP(sizeof(TXT_PROCESSOR_SMM_DESCRIPTOR), 0x100); |
Eugene Myers | faa1118 | 2020-02-06 10:37:01 -0500 | [diff] [blame] | 1118 | } |
Eugene Myers | ae438be | 2020-01-21 17:01:47 -0500 | [diff] [blame] | 1119 | |
| 1120 | /* |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 1121 | * Default to smm_initiate_relocation() if trigger callback isn't |
| 1122 | * provided. |
| 1123 | */ |
Julius Werner | cd49cce | 2019-03-05 16:53:33 -0800 | [diff] [blame] | 1124 | if (CONFIG(HAVE_SMI_HANDLER) && |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 1125 | ops->per_cpu_smm_trigger == NULL) |
| 1126 | mp_state.ops.per_cpu_smm_trigger = smm_initiate_relocation; |
| 1127 | } |
| 1128 | |
| 1129 | int mp_init_with_smm(struct bus *cpu_bus, const struct mp_ops *mp_ops) |
| 1130 | { |
| 1131 | int ret; |
| 1132 | void *default_smm_area; |
| 1133 | struct mp_params mp_params; |
| 1134 | |
| 1135 | if (mp_ops->pre_mp_init != NULL) |
| 1136 | mp_ops->pre_mp_init(); |
| 1137 | |
| 1138 | fill_mp_state(&mp_state, mp_ops); |
| 1139 | |
| 1140 | memset(&mp_params, 0, sizeof(mp_params)); |
| 1141 | |
| 1142 | if (mp_state.cpu_count <= 0) { |
| 1143 | printk(BIOS_ERR, "Invalid cpu_count: %d\n", mp_state.cpu_count); |
| 1144 | return -1; |
| 1145 | } |
| 1146 | |
| 1147 | /* Sanity check SMM state. */ |
| 1148 | if (mp_state.perm_smsize != 0 && mp_state.smm_save_state_size != 0 && |
| 1149 | mp_state.ops.relocation_handler != NULL) |
| 1150 | smm_enable(); |
| 1151 | |
| 1152 | if (is_smm_enabled()) |
| 1153 | printk(BIOS_INFO, "Will perform SMM setup.\n"); |
| 1154 | |
| 1155 | mp_params.num_cpus = mp_state.cpu_count; |
| 1156 | /* Gather microcode information. */ |
| 1157 | if (mp_state.ops.get_microcode_info != NULL) |
| 1158 | mp_state.ops.get_microcode_info(&mp_params.microcode_pointer, |
| 1159 | &mp_params.parallel_microcode_load); |
Aaron Durbin | 8250192 | 2016-04-29 22:55:49 -0500 | [diff] [blame] | 1160 | mp_params.flight_plan = &mp_steps[0]; |
| 1161 | mp_params.num_records = ARRAY_SIZE(mp_steps); |
| 1162 | |
| 1163 | /* Perform backup of default SMM area. */ |
| 1164 | default_smm_area = backup_default_smm_area(); |
| 1165 | |
| 1166 | ret = mp_init(cpu_bus, &mp_params); |
| 1167 | |
| 1168 | restore_default_smm_area(default_smm_area); |
| 1169 | |
| 1170 | /* Signal callback on success if it's provided. */ |
| 1171 | if (ret == 0 && mp_state.ops.post_mp_init != NULL) |
| 1172 | mp_state.ops.post_mp_init(); |
| 1173 | |
| 1174 | return ret; |
| 1175 | } |