Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 1 | #ifndef CPU_X86_LAPIC_H |
| 2 | #define CPU_X86_LAPIC_H |
| 3 | |
Patrick Rudolph | fc57d6c | 2019-11-12 16:30:14 +0100 | [diff] [blame] | 4 | #include <arch/mmio.h> |
Wonkyu Kim | 26ab9bf | 2021-03-22 19:59:18 -0700 | [diff] [blame] | 5 | #include <arch/cpu.h> |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 6 | #include <cpu/x86/lapic_def.h> |
| 7 | #include <cpu/x86/msr.h> |
Patrick Georgi | bd79c5e | 2014-11-28 22:35:36 +0100 | [diff] [blame] | 8 | #include <halt.h> |
Patrick Rudolph | fc57d6c | 2019-11-12 16:30:14 +0100 | [diff] [blame] | 9 | #include <stdint.h> |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 10 | |
Wonkyu Kim | 26ab9bf | 2021-03-22 19:59:18 -0700 | [diff] [blame] | 11 | static inline bool is_x2apic_mode(void) |
| 12 | { |
Kyösti Mälkki | 176c887 | 2021-05-29 20:33:22 +0300 | [diff] [blame^] | 13 | if (CONFIG(XAPIC_ONLY)) |
| 14 | return false; |
| 15 | |
| 16 | if (CONFIG(X2APIC_ONLY)) |
| 17 | return true; |
| 18 | |
Wonkyu Kim | 26ab9bf | 2021-03-22 19:59:18 -0700 | [diff] [blame] | 19 | msr_t msr; |
| 20 | msr = rdmsr(LAPIC_BASE_MSR); |
Wonkyu Kim | a04256f | 2021-04-27 01:52:57 -0700 | [diff] [blame] | 21 | return ((msr.lo & LAPIC_BASE_X2APIC_ENABLED) == LAPIC_BASE_X2APIC_ENABLED); |
Wonkyu Kim | 26ab9bf | 2021-03-22 19:59:18 -0700 | [diff] [blame] | 22 | } |
| 23 | |
| 24 | static inline void x2apic_send_ipi(uint32_t icrlow, uint32_t apicid) |
| 25 | { |
| 26 | msr_t icr; |
| 27 | icr.hi = apicid; |
| 28 | icr.lo = icrlow; |
| 29 | wrmsr(X2APIC_MSR_ICR_ADDRESS, icr); |
| 30 | } |
| 31 | |
Patrick Rudolph | fc57d6c | 2019-11-12 16:30:14 +0100 | [diff] [blame] | 32 | static __always_inline uint32_t lapic_read(unsigned int reg) |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 33 | { |
Wonkyu Kim | 26ab9bf | 2021-03-22 19:59:18 -0700 | [diff] [blame] | 34 | uint32_t value, index; |
| 35 | msr_t msr; |
| 36 | |
| 37 | if (is_x2apic_mode()) { |
| 38 | index = X2APIC_MSR_BASE_ADDRESS + (uint32_t)(reg >> 4); |
| 39 | msr = rdmsr(index); |
| 40 | value = msr.lo; |
| 41 | } else { |
| 42 | value = read32((volatile void *)(uintptr_t)(LAPIC_DEFAULT_BASE + reg)); |
| 43 | } |
| 44 | return value; |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 45 | } |
| 46 | |
Patrick Rudolph | fc57d6c | 2019-11-12 16:30:14 +0100 | [diff] [blame] | 47 | static __always_inline void lapic_write(unsigned int reg, uint32_t v) |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 48 | { |
Wonkyu Kim | 26ab9bf | 2021-03-22 19:59:18 -0700 | [diff] [blame] | 49 | msr_t msr; |
| 50 | uint32_t index; |
| 51 | if (is_x2apic_mode()) { |
| 52 | index = X2APIC_MSR_BASE_ADDRESS + (uint32_t)(reg >> 4); |
| 53 | msr.hi = 0x0; |
| 54 | msr.lo = v; |
| 55 | wrmsr(index, msr); |
| 56 | } else { |
| 57 | write32((volatile void *)(uintptr_t)(LAPIC_DEFAULT_BASE + reg), v); |
| 58 | } |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 59 | } |
| 60 | |
Aaron Durbin | 75a62e7 | 2018-09-13 02:10:45 -0600 | [diff] [blame] | 61 | static __always_inline void lapic_wait_icr_idle(void) |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 62 | { |
Lee Leahy | 91d1e76 | 2017-03-07 14:31:19 -0800 | [diff] [blame] | 63 | do { } while (lapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY); |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 64 | } |
| 65 | |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 66 | static inline void enable_lapic(void) |
| 67 | { |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 68 | msr_t msr; |
| 69 | msr = rdmsr(LAPIC_BASE_MSR); |
| 70 | msr.hi &= 0xffffff00; |
Kyösti Mälkki | ff284f6 | 2017-08-18 12:11:16 +0300 | [diff] [blame] | 71 | msr.lo &= ~LAPIC_BASE_MSR_ADDR_MASK; |
| 72 | msr.lo |= LAPIC_DEFAULT_BASE; |
| 73 | msr.lo |= LAPIC_BASE_MSR_ENABLE; |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 74 | wrmsr(LAPIC_BASE_MSR, msr); |
| 75 | } |
| 76 | |
| 77 | static inline void disable_lapic(void) |
| 78 | { |
| 79 | msr_t msr; |
| 80 | msr = rdmsr(LAPIC_BASE_MSR); |
Kyösti Mälkki | ff284f6 | 2017-08-18 12:11:16 +0300 | [diff] [blame] | 81 | msr.lo &= ~LAPIC_BASE_MSR_ENABLE; |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 82 | wrmsr(LAPIC_BASE_MSR, msr); |
| 83 | } |
| 84 | |
Wonkyu Kim | 26ab9bf | 2021-03-22 19:59:18 -0700 | [diff] [blame] | 85 | static __always_inline unsigned int initial_lapicid(void) |
| 86 | { |
| 87 | uint32_t lapicid; |
Wonkyu Kim | a04256f | 2021-04-27 01:52:57 -0700 | [diff] [blame] | 88 | if (is_x2apic_mode() && cpuid_get_max_func() >= 0xb) |
| 89 | lapicid = cpuid_ext(0xb, 0).edx; |
Wonkyu Kim | 26ab9bf | 2021-03-22 19:59:18 -0700 | [diff] [blame] | 90 | else |
| 91 | lapicid = cpuid_ebx(1) >> 24; |
| 92 | return lapicid; |
| 93 | } |
| 94 | |
Patrick Rudolph | fc57d6c | 2019-11-12 16:30:14 +0100 | [diff] [blame] | 95 | static __always_inline unsigned int lapicid(void) |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 96 | { |
Wonkyu Kim | 26ab9bf | 2021-03-22 19:59:18 -0700 | [diff] [blame] | 97 | uint32_t lapicid = lapic_read(LAPIC_ID); |
| 98 | |
| 99 | /* check x2apic mode and return accordingly */ |
| 100 | if (!is_x2apic_mode()) |
| 101 | lapicid >>= 24; |
| 102 | return lapicid; |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 103 | } |
| 104 | |
Julius Werner | cd49cce | 2019-03-05 16:53:33 -0800 | [diff] [blame] | 105 | #if !CONFIG(AP_IN_SIPI_WAIT) |
Sven Schnelle | 51676b1 | 2012-07-29 19:18:03 +0200 | [diff] [blame] | 106 | /* If we need to go back to sipi wait, we use the long non-inlined version of |
| 107 | * this function in lapic_cpu_init.c |
| 108 | */ |
Aaron Durbin | 75a62e7 | 2018-09-13 02:10:45 -0600 | [diff] [blame] | 109 | static __always_inline void stop_this_cpu(void) |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 110 | { |
Sven Schnelle | 51676b1 | 2012-07-29 19:18:03 +0200 | [diff] [blame] | 111 | /* Called by an AP when it is ready to halt and wait for a new task */ |
Patrick Georgi | bd79c5e | 2014-11-28 22:35:36 +0100 | [diff] [blame] | 112 | halt(); |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 113 | } |
Sven Schnelle | 51676b1 | 2012-07-29 19:18:03 +0200 | [diff] [blame] | 114 | #else |
| 115 | void stop_this_cpu(void); |
| 116 | #endif |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 117 | |
Patrick Rudolph | fc57d6c | 2019-11-12 16:30:14 +0100 | [diff] [blame] | 118 | static inline void lapic_write_atomic(unsigned long reg, uint32_t v) |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 119 | { |
Patrick Rudolph | fc57d6c | 2019-11-12 16:30:14 +0100 | [diff] [blame] | 120 | volatile uint32_t *ptr; |
| 121 | |
| 122 | ptr = (volatile uint32_t *)(LAPIC_DEFAULT_BASE + reg); |
| 123 | |
| 124 | asm volatile ("xchgl %0, %1\n" |
| 125 | : "+r" (v), "+m" (*(ptr)) |
| 126 | : : "memory", "cc"); |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 127 | } |
| 128 | |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 129 | # define lapic_read_around(x) lapic_read(x) |
Lee Leahy | ae3fd34 | 2017-03-07 12:55:23 -0800 | [diff] [blame] | 130 | # define lapic_write_around(x, y) lapic_write_atomic((x), (y)) |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 131 | |
Felix Held | 215ac3b | 2020-10-30 15:40:54 +0100 | [diff] [blame] | 132 | void lapic_virtual_wire_mode_init(void); |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 133 | |
Kyösti Mälkki | ff284f6 | 2017-08-18 12:11:16 +0300 | [diff] [blame] | 134 | /* See if I need to initialize the local APIC */ |
| 135 | static inline int need_lapic_init(void) |
| 136 | { |
Julius Werner | cd49cce | 2019-03-05 16:53:33 -0800 | [diff] [blame] | 137 | return CONFIG(SMP) || CONFIG(IOAPIC); |
Kyösti Mälkki | ff284f6 | 2017-08-18 12:11:16 +0300 | [diff] [blame] | 138 | } |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 139 | |
Kyösti Mälkki | ff284f6 | 2017-08-18 12:11:16 +0300 | [diff] [blame] | 140 | static inline void setup_lapic(void) |
| 141 | { |
| 142 | if (need_lapic_init()) |
Felix Held | 215ac3b | 2020-10-30 15:40:54 +0100 | [diff] [blame] | 143 | lapic_virtual_wire_mode_init(); |
Kyösti Mälkki | ff284f6 | 2017-08-18 12:11:16 +0300 | [diff] [blame] | 144 | else |
| 145 | disable_lapic(); |
| 146 | } |
| 147 | |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 148 | #endif /* CPU_X86_LAPIC_H */ |