Martin Roth | 239b5df | 2022-07-26 22:18:26 -0600 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
| 2 | |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 3 | #ifndef CPU_X86_LAPIC_H |
| 4 | #define CPU_X86_LAPIC_H |
| 5 | |
Patrick Rudolph | fc57d6c | 2019-11-12 16:30:14 +0100 | [diff] [blame] | 6 | #include <arch/mmio.h> |
Wonkyu Kim | 26ab9bf | 2021-03-22 19:59:18 -0700 | [diff] [blame] | 7 | #include <arch/cpu.h> |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 8 | #include <cpu/x86/lapic_def.h> |
| 9 | #include <cpu/x86/msr.h> |
Patrick Georgi | bd79c5e | 2014-11-28 22:35:36 +0100 | [diff] [blame] | 10 | #include <halt.h> |
Patrick Rudolph | fc57d6c | 2019-11-12 16:30:14 +0100 | [diff] [blame] | 11 | #include <stdint.h> |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 12 | |
Kyösti Mälkki | 41e6216 | 2021-05-31 12:38:41 +0300 | [diff] [blame] | 13 | static __always_inline uint32_t xapic_read(unsigned int reg) |
| 14 | { |
| 15 | return read32((volatile void *)(uintptr_t)(LAPIC_DEFAULT_BASE + reg)); |
| 16 | } |
| 17 | |
| 18 | static __always_inline void xapic_write(unsigned int reg, uint32_t v) |
| 19 | { |
| 20 | write32((volatile void *)(uintptr_t)(LAPIC_DEFAULT_BASE + reg), v); |
| 21 | } |
| 22 | |
Kyösti Mälkki | 0c1158b | 2022-01-03 14:54:27 +0200 | [diff] [blame] | 23 | static __always_inline void xapic_send_ipi(uint32_t icrlow, uint32_t icrhi) |
Arthur Heymans | 6f77ff7 | 2021-05-21 09:32:45 +0200 | [diff] [blame] | 24 | { |
Kyösti Mälkki | 0c1158b | 2022-01-03 14:54:27 +0200 | [diff] [blame] | 25 | xapic_write(LAPIC_ICR2, icrhi); |
Kyösti Mälkki | b31b033 | 2021-05-31 14:41:15 +0300 | [diff] [blame] | 26 | xapic_write(LAPIC_ICR, icrlow); |
Arthur Heymans | 6f77ff7 | 2021-05-21 09:32:45 +0200 | [diff] [blame] | 27 | } |
| 28 | |
Arthur Heymans | a4ceba4 | 2021-05-21 09:32:45 +0200 | [diff] [blame] | 29 | static __always_inline int xapic_busy(void) |
| 30 | { |
| 31 | return xapic_read(LAPIC_ICR) & LAPIC_ICR_BUSY; |
| 32 | } |
| 33 | |
Kyösti Mälkki | 41e6216 | 2021-05-31 12:38:41 +0300 | [diff] [blame] | 34 | static __always_inline uint32_t x2apic_read(unsigned int reg) |
| 35 | { |
| 36 | uint32_t value, index; |
| 37 | msr_t msr; |
| 38 | |
| 39 | index = X2APIC_MSR_BASE_ADDRESS + (uint32_t)(reg >> 4); |
| 40 | msr = rdmsr(index); |
| 41 | value = msr.lo; |
| 42 | return value; |
| 43 | } |
| 44 | |
| 45 | static __always_inline void x2apic_write(unsigned int reg, uint32_t v) |
| 46 | { |
| 47 | uint32_t index; |
| 48 | msr_t msr; |
| 49 | |
| 50 | index = X2APIC_MSR_BASE_ADDRESS + (uint32_t)(reg >> 4); |
| 51 | msr.hi = 0x0; |
| 52 | msr.lo = v; |
| 53 | wrmsr(index, msr); |
| 54 | } |
| 55 | |
Kyösti Mälkki | 0c1158b | 2022-01-03 14:54:27 +0200 | [diff] [blame] | 56 | static __always_inline void x2apic_send_ipi(uint32_t icrlow, uint32_t icrhi) |
Kyösti Mälkki | 41e6216 | 2021-05-31 12:38:41 +0300 | [diff] [blame] | 57 | { |
| 58 | msr_t icr; |
Kyösti Mälkki | 0c1158b | 2022-01-03 14:54:27 +0200 | [diff] [blame] | 59 | icr.hi = icrhi; |
Kyösti Mälkki | 41e6216 | 2021-05-31 12:38:41 +0300 | [diff] [blame] | 60 | icr.lo = icrlow; |
| 61 | wrmsr(X2APIC_MSR_ICR_ADDRESS, icr); |
| 62 | } |
| 63 | |
Kyösti Mälkki | c8d26c0 | 2021-06-06 17:07:25 +0300 | [diff] [blame] | 64 | static __always_inline bool is_x2apic_mode(void) |
Wonkyu Kim | 26ab9bf | 2021-03-22 19:59:18 -0700 | [diff] [blame] | 65 | { |
Kyösti Mälkki | 176c887 | 2021-05-29 20:33:22 +0300 | [diff] [blame] | 66 | if (CONFIG(XAPIC_ONLY)) |
| 67 | return false; |
| 68 | |
| 69 | if (CONFIG(X2APIC_ONLY)) |
| 70 | return true; |
| 71 | |
Wonkyu Kim | 26ab9bf | 2021-03-22 19:59:18 -0700 | [diff] [blame] | 72 | msr_t msr; |
| 73 | msr = rdmsr(LAPIC_BASE_MSR); |
Wonkyu Kim | a04256f | 2021-04-27 01:52:57 -0700 | [diff] [blame] | 74 | return ((msr.lo & LAPIC_BASE_X2APIC_ENABLED) == LAPIC_BASE_X2APIC_ENABLED); |
Wonkyu Kim | 26ab9bf | 2021-03-22 19:59:18 -0700 | [diff] [blame] | 75 | } |
| 76 | |
Patrick Rudolph | fc57d6c | 2019-11-12 16:30:14 +0100 | [diff] [blame] | 77 | static __always_inline uint32_t lapic_read(unsigned int reg) |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 78 | { |
Kyösti Mälkki | 41e6216 | 2021-05-31 12:38:41 +0300 | [diff] [blame] | 79 | if (is_x2apic_mode()) |
| 80 | return x2apic_read(reg); |
| 81 | else |
| 82 | return xapic_read(reg); |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 83 | } |
| 84 | |
Patrick Rudolph | fc57d6c | 2019-11-12 16:30:14 +0100 | [diff] [blame] | 85 | static __always_inline void lapic_write(unsigned int reg, uint32_t v) |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 86 | { |
Kyösti Mälkki | 41e6216 | 2021-05-31 12:38:41 +0300 | [diff] [blame] | 87 | if (is_x2apic_mode()) |
| 88 | x2apic_write(reg, v); |
| 89 | else |
| 90 | xapic_write(reg, v); |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 91 | } |
| 92 | |
Kyösti Mälkki | 0cfa911 | 2021-05-31 10:38:45 +0300 | [diff] [blame] | 93 | static __always_inline void lapic_update32(unsigned int reg, uint32_t mask, uint32_t or) |
| 94 | { |
| 95 | if (is_x2apic_mode()) { |
| 96 | uint32_t index; |
| 97 | msr_t msr; |
| 98 | index = X2APIC_MSR_BASE_ADDRESS + (uint32_t)(reg >> 4); |
| 99 | msr = rdmsr(index); |
| 100 | msr.lo &= mask; |
| 101 | msr.lo |= or; |
| 102 | wrmsr(index, msr); |
| 103 | } else { |
| 104 | uint32_t value; |
| 105 | value = xapic_read(reg); |
| 106 | value &= mask; |
| 107 | value |= or; |
Kyösti Mälkki | b31b033 | 2021-05-31 14:41:15 +0300 | [diff] [blame] | 108 | xapic_write(reg, value); |
Kyösti Mälkki | 0cfa911 | 2021-05-31 10:38:45 +0300 | [diff] [blame] | 109 | } |
| 110 | } |
| 111 | |
Arthur Heymans | 6f77ff7 | 2021-05-21 09:32:45 +0200 | [diff] [blame] | 112 | static __always_inline void lapic_send_ipi(uint32_t icrlow, uint32_t apicid) |
| 113 | { |
| 114 | if (is_x2apic_mode()) |
| 115 | x2apic_send_ipi(icrlow, apicid); |
| 116 | else |
Kyösti Mälkki | 0c1158b | 2022-01-03 14:54:27 +0200 | [diff] [blame] | 117 | xapic_send_ipi(icrlow, SET_LAPIC_DEST_FIELD(apicid)); |
Arthur Heymans | 6f77ff7 | 2021-05-21 09:32:45 +0200 | [diff] [blame] | 118 | } |
| 119 | |
Arthur Heymans | a4ceba4 | 2021-05-21 09:32:45 +0200 | [diff] [blame] | 120 | static __always_inline int lapic_busy(void) |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 121 | { |
Arthur Heymans | a4ceba4 | 2021-05-21 09:32:45 +0200 | [diff] [blame] | 122 | if (is_x2apic_mode()) |
| 123 | return 0; |
| 124 | else |
| 125 | return xapic_busy(); |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 126 | } |
| 127 | |
Wonkyu Kim | 26ab9bf | 2021-03-22 19:59:18 -0700 | [diff] [blame] | 128 | static __always_inline unsigned int initial_lapicid(void) |
| 129 | { |
| 130 | uint32_t lapicid; |
Felix Held | 49be1a9 | 2022-02-09 22:07:29 +0100 | [diff] [blame] | 131 | if (is_x2apic_mode() && cpuid_get_max_func() >= 0xb) |
Wonkyu Kim | a04256f | 2021-04-27 01:52:57 -0700 | [diff] [blame] | 132 | lapicid = cpuid_ext(0xb, 0).edx; |
Wonkyu Kim | 26ab9bf | 2021-03-22 19:59:18 -0700 | [diff] [blame] | 133 | else |
| 134 | lapicid = cpuid_ebx(1) >> 24; |
| 135 | return lapicid; |
| 136 | } |
| 137 | |
Patrick Rudolph | fc57d6c | 2019-11-12 16:30:14 +0100 | [diff] [blame] | 138 | static __always_inline unsigned int lapicid(void) |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 139 | { |
Wonkyu Kim | 26ab9bf | 2021-03-22 19:59:18 -0700 | [diff] [blame] | 140 | uint32_t lapicid = lapic_read(LAPIC_ID); |
| 141 | |
| 142 | /* check x2apic mode and return accordingly */ |
| 143 | if (!is_x2apic_mode()) |
| 144 | lapicid >>= 24; |
| 145 | return lapicid; |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 146 | } |
| 147 | |
Kyösti Mälkki | 710bdc4 | 2021-10-15 17:14:20 +0300 | [diff] [blame] | 148 | static __always_inline void lapic_send_ipi_self(uint32_t icrlow) |
| 149 | { |
Kyösti Mälkki | 7aea15a | 2021-10-15 17:14:20 +0300 | [diff] [blame] | 150 | int i = 1000; |
| 151 | |
Kyösti Mälkki | 710bdc4 | 2021-10-15 17:14:20 +0300 | [diff] [blame] | 152 | /* LAPIC_DEST_SELF does not support all delivery mode -fields. */ |
| 153 | lapic_send_ipi(icrlow, lapicid()); |
Kyösti Mälkki | 7aea15a | 2021-10-15 17:14:20 +0300 | [diff] [blame] | 154 | |
| 155 | /* In case of X2APIC force a short delay, to prevent deadlock in a case |
| 156 | * the immediately following code acquires some lock, like with printk(). |
| 157 | */ |
Arthur Heymans | f158c9c | 2022-08-31 09:43:45 +0200 | [diff] [blame^] | 158 | const bool x2apic = is_x2apic_mode(); |
| 159 | |
| 160 | while (x2apic && i--) |
Kyösti Mälkki | 7aea15a | 2021-10-15 17:14:20 +0300 | [diff] [blame] | 161 | cpu_relax(); |
Kyösti Mälkki | 710bdc4 | 2021-10-15 17:14:20 +0300 | [diff] [blame] | 162 | } |
| 163 | |
| 164 | static __always_inline void lapic_send_ipi_others(uint32_t icrlow) |
| 165 | { |
| 166 | lapic_send_ipi(LAPIC_DEST_ALLBUT | icrlow, 0); |
| 167 | } |
| 168 | |
Julius Werner | cd49cce | 2019-03-05 16:53:33 -0800 | [diff] [blame] | 169 | #if !CONFIG(AP_IN_SIPI_WAIT) |
Sven Schnelle | 51676b1 | 2012-07-29 19:18:03 +0200 | [diff] [blame] | 170 | /* If we need to go back to sipi wait, we use the long non-inlined version of |
Kyösti Mälkki | 36c9017 | 2021-05-29 22:50:22 +0300 | [diff] [blame] | 171 | * this function in lapic_cpu_stop.c |
Sven Schnelle | 51676b1 | 2012-07-29 19:18:03 +0200 | [diff] [blame] | 172 | */ |
Aaron Durbin | 75a62e7 | 2018-09-13 02:10:45 -0600 | [diff] [blame] | 173 | static __always_inline void stop_this_cpu(void) |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 174 | { |
Sven Schnelle | 51676b1 | 2012-07-29 19:18:03 +0200 | [diff] [blame] | 175 | /* Called by an AP when it is ready to halt and wait for a new task */ |
Patrick Georgi | bd79c5e | 2014-11-28 22:35:36 +0100 | [diff] [blame] | 176 | halt(); |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 177 | } |
Sven Schnelle | 51676b1 | 2012-07-29 19:18:03 +0200 | [diff] [blame] | 178 | #else |
| 179 | void stop_this_cpu(void); |
| 180 | #endif |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 181 | |
Kyösti Mälkki | 242f1d9 | 2021-06-06 16:58:19 +0300 | [diff] [blame] | 182 | void enable_lapic(void); |
Subrata Banik | 2125a17 | 2022-07-12 10:55:21 +0000 | [diff] [blame] | 183 | void enable_lapic_mode(bool try_set_x2apic); |
Kyösti Mälkki | 242f1d9 | 2021-06-06 16:58:19 +0300 | [diff] [blame] | 184 | void disable_lapic(void); |
Kyösti Mälkki | 9ec7227 | 2021-10-17 08:34:31 +0300 | [diff] [blame] | 185 | void setup_lapic_interrupts(void); |
Kyösti Mälkki | ff284f6 | 2017-08-18 12:11:16 +0300 | [diff] [blame] | 186 | |
Eric Biederman | c84c190 | 2004-10-14 20:13:01 +0000 | [diff] [blame] | 187 | #endif /* CPU_X86_LAPIC_H */ |