blob: 2fe1bfbd64fda39063923e234c0734b2db948751 [file] [log] [blame]
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -05001// Code for manipulating stack locations.
2//
Kevin O'Connor62de31b2015-09-22 12:35:00 -04003// Copyright (C) 2009-2015 Kevin O'Connor <kevin@koconnor.net>
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -05004//
5// This file may be distributed under the terms of the GNU LGPLv3 license.
6
Kevin O'Connor46b82622012-05-13 12:10:30 -04007#include "biosvar.h" // GET_GLOBAL
Kevin O'Connorad901592009-12-13 11:25:25 -05008#include "bregs.h" // CR0_PE
Kevin O'Connor55215cd2014-04-11 11:20:41 -04009#include "fw/paravirt.h" // PORT_SMI_CMD
Kevin O'Connor8b7861c2013-09-15 02:29:06 -040010#include "hw/rtc.h" // rtc_use
Kevin O'Connor0039a942013-06-08 21:50:15 -040011#include "list.h" // hlist_node
Kevin O'Connor9dea5902013-09-14 20:23:54 -040012#include "malloc.h" // free
Kevin O'Connor2d2fa312013-09-14 21:55:26 -040013#include "output.h" // dprintf
Kevin O'Connord29ce622014-04-07 12:15:34 -040014#include "romfile.h" // romfile_loadint
Kevin O'Connor3df600b2013-09-14 19:28:55 -040015#include "stacks.h" // struct mutex_s
Kevin O'Connor46a346e2015-09-11 16:07:59 -040016#include "string.h" // memset
Kevin O'Connor2d2fa312013-09-14 21:55:26 -040017#include "util.h" // useRTC
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -050018
Kevin O'Connor7acaa3c2013-10-02 20:20:03 -040019#define MAIN_STACK_MAX (1024*1024)
20
Kevin O'Connore77c7052012-05-28 22:06:42 -040021
22/****************************************************************
Kevin O'Connore77c7052012-05-28 22:06:42 -040023 * 16bit / 32bit calling
Kevin O'Connor9c447c32010-05-23 10:24:22 -040024 ****************************************************************/
25
Kevin O'Connor15721bf2014-09-30 12:55:54 -040026struct {
27 u8 method;
28 u8 cmosindex;
Kevin O'Connorbfb7b582014-10-11 13:27:16 -040029 u8 a20;
Kevin O'Connor15721bf2014-09-30 12:55:54 -040030 u16 ss, fs, gs;
Kevin O'Connor62de31b2015-09-22 12:35:00 -040031 u32 cr0;
Kevin O'Connor15721bf2014-09-30 12:55:54 -040032 struct descloc_s gdt;
Kevin O'Connor4c568d82015-09-13 10:03:12 -040033} Call16Data VARLOW;
Kevin O'Connor15721bf2014-09-30 12:55:54 -040034
Kevin O'Connor46a346e2015-09-11 16:07:59 -040035#define C16_BIG 1
36#define C16_SMM 2
Kevin O'Connor55215cd2014-04-11 11:20:41 -040037
38int HaveSmmCall32 VARFSEG;
39
Kevin O'Connor25eb0612015-09-11 15:38:43 -040040// Backup state in preparation for call32
Kevin O'Connor62de31b2015-09-22 12:35:00 -040041static int
Kevin O'Connor25eb0612015-09-11 15:38:43 -040042call32_prep(u8 method)
Kevin O'Connor55215cd2014-04-11 11:20:41 -040043{
Kevin O'Connor4c568d82015-09-13 10:03:12 -040044 if (!CONFIG_CALL32_SMM || method != C16_SMM) {
Kevin O'Connor62de31b2015-09-22 12:35:00 -040045 // Backup cr0
46 u32 cr0 = cr0_read();
47 if (cr0 & CR0_PE)
48 // Called in 16bit protected mode?!
49 return -1;
50 SET_LOW(Call16Data.cr0, cr0);
51
Kevin O'Connor25eb0612015-09-11 15:38:43 -040052 // Backup fs/gs and gdt
Kevin O'Connor4c568d82015-09-13 10:03:12 -040053 SET_LOW(Call16Data.fs, GET_SEG(FS));
54 SET_LOW(Call16Data.gs, GET_SEG(GS));
Kevin O'Connor25eb0612015-09-11 15:38:43 -040055 struct descloc_s gdt;
56 sgdt(&gdt);
Kevin O'Connor4c568d82015-09-13 10:03:12 -040057 SET_LOW(Call16Data.gdt.length, gdt.length);
58 SET_LOW(Call16Data.gdt.addr, gdt.addr);
Kevin O'Connor25eb0612015-09-11 15:38:43 -040059
60 // Enable a20 and backup its previous state
Kevin O'Connor4c568d82015-09-13 10:03:12 -040061 SET_LOW(Call16Data.a20, set_a20(1));
Kevin O'Connor25eb0612015-09-11 15:38:43 -040062 }
63
Kevin O'Connor62de31b2015-09-22 12:35:00 -040064 // Backup ss
65 SET_LOW(Call16Data.ss, GET_SEG(SS));
66
67 // Backup cmos index register and disable nmi
68 u8 cmosindex = inb(PORT_CMOS_INDEX);
Kevin O'Connordee3c152017-05-16 11:59:10 -040069 if (!(cmosindex & NMI_DISABLE_BIT)) {
70 outb(cmosindex | NMI_DISABLE_BIT, PORT_CMOS_INDEX);
71 inb(PORT_CMOS_DATA);
72 }
Kevin O'Connor62de31b2015-09-22 12:35:00 -040073 SET_LOW(Call16Data.cmosindex, cmosindex);
74
Kevin O'Connor4c568d82015-09-13 10:03:12 -040075 SET_LOW(Call16Data.method, method);
Kevin O'Connor62de31b2015-09-22 12:35:00 -040076 return 0;
Kevin O'Connor55215cd2014-04-11 11:20:41 -040077}
78
Kevin O'Connor25eb0612015-09-11 15:38:43 -040079// Restore state backed up during call32
80static u8
81call32_post(void)
Kevin O'Connor55215cd2014-04-11 11:20:41 -040082{
Kevin O'Connor4c568d82015-09-13 10:03:12 -040083 u8 method = GET_LOW(Call16Data.method);
84 SET_LOW(Call16Data.method, 0);
85 SET_LOW(Call16Data.ss, 0);
Kevin O'Connor55215cd2014-04-11 11:20:41 -040086
Kevin O'Connor4c568d82015-09-13 10:03:12 -040087 if (!CONFIG_CALL32_SMM || method != C16_SMM) {
Kevin O'Connor25eb0612015-09-11 15:38:43 -040088 // Restore a20
Kevin O'Connor8ebb33b2017-05-16 11:47:27 -040089 u8 a20 = GET_LOW(Call16Data.a20);
90 if (!a20)
91 set_a20(0);
Kevin O'Connor25eb0612015-09-11 15:38:43 -040092
93 // Restore gdt and fs/gs
94 struct descloc_s gdt;
Kevin O'Connor4c568d82015-09-13 10:03:12 -040095 gdt.length = GET_LOW(Call16Data.gdt.length);
96 gdt.addr = GET_LOW(Call16Data.gdt.addr);
Kevin O'Connor25eb0612015-09-11 15:38:43 -040097 lgdt(&gdt);
Kevin O'Connor4c568d82015-09-13 10:03:12 -040098 SET_SEG(FS, GET_LOW(Call16Data.fs));
99 SET_SEG(GS, GET_LOW(Call16Data.gs));
Kevin O'Connor62de31b2015-09-22 12:35:00 -0400100
101 // Restore cr0
102 u32 cr0_caching = GET_LOW(Call16Data.cr0) & (CR0_CD|CR0_NW);
103 if (cr0_caching)
104 cr0_mask(CR0_CD|CR0_NW, cr0_caching);
Kevin O'Connor25eb0612015-09-11 15:38:43 -0400105 }
106
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400107 // Restore cmos index register
Kevin O'Connordee3c152017-05-16 11:59:10 -0400108 u8 cmosindex = GET_LOW(Call16Data.cmosindex);
109 if (!(cmosindex & NMI_DISABLE_BIT)) {
110 outb(cmosindex, PORT_CMOS_INDEX);
111 inb(PORT_CMOS_DATA);
112 }
Kevin O'Connor25eb0612015-09-11 15:38:43 -0400113 return method;
114}
115
Kevin O'Connor4c599ef2015-09-22 14:38:14 -0400116// Force next call16() to restore to a pristine cpu environment state
117static void
118call16_override(int big)
119{
120 ASSERT32FLAT();
121 if (getesp() > BUILD_STACK_ADDR)
122 panic("call16_override with invalid stack\n");
123 memset(&Call16Data, 0, sizeof(Call16Data));
124 if (big) {
125 Call16Data.method = C16_BIG;
126 Call16Data.a20 = 1;
127 } else {
128 Call16Data.a20 = !CONFIG_DISABLE_A20;
129 }
130}
131
132// 16bit handler code called from call16() / call16_smm()
Kevin O'Connor25eb0612015-09-11 15:38:43 -0400133u32 VISIBLE16
134call16_helper(u32 eax, u32 edx, u32 (*func)(u32 eax, u32 edx))
135{
136 u8 method = call32_post();
137 u32 ret = func(eax, edx);
138 call32_prep(method);
139 return ret;
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400140}
141
Kevin O'Connored675ad2014-12-03 12:53:01 -0500142#define ASM32_SWITCH16 " .pushsection .text.32fseg." UNIQSEC "\n .code16\n"
143#define ASM32_BACK32 " .popsection\n .code32\n"
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500144#define ASM16_SWITCH32 " .code32\n"
145#define ASM16_BACK16 " .code16gcc\n"
146
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400147// Call a SeaBIOS C function in 32bit mode using smm trampoline
148static u32
149call32_smm(void *func, u32 eax)
150{
151 ASSERT16();
152 dprintf(9, "call32_smm %p %x\n", func, eax);
Kevin O'Connor4c568d82015-09-13 10:03:12 -0400153 call32_prep(C16_SMM);
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400154 u32 bkup_esp;
155 asm volatile(
156 // Backup esp / set esp to flat stack location
157 " movl %%esp, %0\n"
158 " movl %%ss, %%eax\n"
159 " shll $4, %%eax\n"
160 " addl %%eax, %%esp\n"
161
162 // Transition to 32bit mode, call func, return to 16bit
163 " movl $" __stringify(CALL32SMM_CMDID) ", %%eax\n"
164 " movl $" __stringify(CALL32SMM_ENTERID) ", %%ecx\n"
165 " movl $(" __stringify(BUILD_BIOS_ADDR) " + 1f), %%ebx\n"
166 " outb %%al, $" __stringify(PORT_SMI_CMD) "\n"
167 " rep; nop\n"
168 " hlt\n"
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400169
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500170 ASM16_SWITCH32
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400171 "1:movl %1, %%eax\n"
172 " calll *%2\n"
173 " movl %%eax, %1\n"
174
175 " movl $" __stringify(CALL32SMM_CMDID) ", %%eax\n"
176 " movl $" __stringify(CALL32SMM_RETURNID) ", %%ecx\n"
177 " movl $2f, %%ebx\n"
178 " outb %%al, $" __stringify(PORT_SMI_CMD) "\n"
179 " rep; nop\n"
180 " hlt\n"
181
182 // Restore esp
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500183 ASM16_BACK16
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400184 "2:movl %0, %%esp\n"
185 : "=&r" (bkup_esp), "+r" (eax)
186 : "r" (func)
187 : "eax", "ecx", "edx", "ebx", "cc", "memory");
Kevin O'Connor25eb0612015-09-11 15:38:43 -0400188 call32_post();
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400189
190 dprintf(9, "call32_smm done %p %x\n", func, eax);
191 return eax;
192}
193
Kevin O'Connored675ad2014-12-03 12:53:01 -0500194static u32
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400195call16_smm(u32 eax, u32 edx, void *func)
196{
197 ASSERT32FLAT();
198 if (!CONFIG_CALL32_SMM)
199 return eax;
200 func -= BUILD_BIOS_ADDR;
201 dprintf(9, "call16_smm %p %x %x\n", func, eax, edx);
Kevin O'Connor4c568d82015-09-13 10:03:12 -0400202 u32 stackoffset = Call16Data.ss << 4;
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400203 asm volatile(
204 // Restore esp
205 " subl %0, %%esp\n"
206
207 // Transition to 16bit mode, call func, return to 32bit
208 " movl $" __stringify(CALL32SMM_CMDID) ", %%eax\n"
209 " movl $" __stringify(CALL32SMM_RETURNID) ", %%ecx\n"
210 " movl $(1f - " __stringify(BUILD_BIOS_ADDR) "), %%ebx\n"
211 " outb %%al, $" __stringify(PORT_SMI_CMD) "\n"
212 " rep; nop\n"
213 " hlt\n"
214
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500215 ASM32_SWITCH16
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400216 "1:movl %1, %%eax\n"
217 " movl %3, %%ecx\n"
Kevin O'Connor25eb0612015-09-11 15:38:43 -0400218 " calll _cfunc16_call16_helper\n"
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400219 " movl %%eax, %1\n"
220
221 " movl $" __stringify(CALL32SMM_CMDID) ", %%eax\n"
222 " movl $" __stringify(CALL32SMM_ENTERID) ", %%ecx\n"
223 " movl $2f, %%ebx\n"
224 " outb %%al, $" __stringify(PORT_SMI_CMD) "\n"
225 " rep; nop\n"
226 " hlt\n"
227
228 // Set esp to flat stack location
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500229 ASM32_BACK32
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400230 "2:addl %0, %%esp\n"
231 : "+r" (stackoffset), "+r" (eax), "+d" (edx)
232 : "r" (func)
233 : "eax", "ecx", "ebx", "cc", "memory");
234 return eax;
235}
Kevin O'Connor7acaa3c2013-10-02 20:20:03 -0400236
Kevin O'Connor63766c42015-09-11 16:15:23 -0400237// Call a 32bit SeaBIOS function from a 16bit SeaBIOS function.
238u32 VISIBLE16
Kevin O'Connorb4cca862015-10-09 11:53:02 -0400239__call32(void *func, u32 eax, u32 errret)
Kevin O'Connora7fc5de2009-12-13 11:48:18 -0500240{
241 ASSERT16();
Kevin O'Connor63766c42015-09-11 16:15:23 -0400242 if (CONFIG_CALL32_SMM && GET_GLOBAL(HaveSmmCall32))
243 return call32_smm(func, eax);
Kevin O'Connor63766c42015-09-11 16:15:23 -0400244 // Jump direclty to 32bit mode - this clobbers the 16bit segment
245 // selector registers.
Kevin O'Connor62de31b2015-09-22 12:35:00 -0400246 int ret = call32_prep(C16_BIG);
247 if (ret)
248 return errret;
Kevin O'Connora7fc5de2009-12-13 11:48:18 -0500249 u32 bkup_ss, bkup_esp;
250 asm volatile(
251 // Backup ss/esp / set esp to flat stack location
252 " movl %%ss, %0\n"
253 " movl %%esp, %1\n"
254 " shll $4, %0\n"
255 " addl %0, %%esp\n"
Kevin O'Connoraf9629b2010-11-25 09:17:31 -0500256 " shrl $4, %0\n"
Kevin O'Connora7fc5de2009-12-13 11:48:18 -0500257
Kevin O'Connorfb214dc2009-12-20 13:11:17 -0500258 // Transition to 32bit mode, call func, return to 16bit
Kevin O'Connor4057f982010-11-25 08:52:50 -0500259 " movl $(" __stringify(BUILD_BIOS_ADDR) " + 1f), %%edx\n"
Kevin O'Connor423542e2015-09-11 16:19:02 -0400260 " jmp transition32_nmi_off\n"
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500261 ASM16_SWITCH32
Kevin O'Connoraf9629b2010-11-25 09:17:31 -0500262 "1:calll *%3\n"
Kevin O'Connor4057f982010-11-25 08:52:50 -0500263 " movl $2f, %%edx\n"
Kevin O'Connora7fc5de2009-12-13 11:48:18 -0500264 " jmp transition16big\n"
265
266 // Restore ds/ss/esp
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500267 ASM16_BACK16
Kevin O'Connora7fc5de2009-12-13 11:48:18 -0500268 "2:movl %0, %%ds\n"
269 " movl %0, %%ss\n"
270 " movl %1, %%esp\n"
Kevin O'Connoraf9629b2010-11-25 09:17:31 -0500271 : "=&r" (bkup_ss), "=&r" (bkup_esp), "+a" (eax)
Kevin O'Connorbca07362010-03-20 20:41:38 -0400272 : "r" (func)
Kevin O'Connoraf9629b2010-11-25 09:17:31 -0500273 : "ecx", "edx", "cc", "memory");
Kevin O'Connor25eb0612015-09-11 15:38:43 -0400274 call32_post();
Kevin O'Connoraf9629b2010-11-25 09:17:31 -0500275 return eax;
Kevin O'Connora7fc5de2009-12-13 11:48:18 -0500276}
277
Kevin O'Connor46a346e2015-09-11 16:07:59 -0400278// Call a 16bit SeaBIOS function, restoring the mode from last call32().
Kevin O'Connored675ad2014-12-03 12:53:01 -0500279static u32
Kevin O'Connor4c599ef2015-09-22 14:38:14 -0400280call16(u32 eax, u32 edx, void *func)
Kevin O'Connor11884802014-09-30 12:13:44 -0400281{
282 ASSERT32FLAT();
283 if (getesp() > MAIN_STACK_MAX)
Kevin O'Connor4c599ef2015-09-22 14:38:14 -0400284 panic("call16 with invalid stack\n");
Kevin O'Connor46a346e2015-09-11 16:07:59 -0400285 if (CONFIG_CALL32_SMM && Call16Data.method == C16_SMM)
286 return call16_smm(eax, edx, func);
287
288 extern void transition16big(void);
289 extern void transition16(void);
290 void *thunk = transition16;
291 if (Call16Data.method == C16_BIG || in_post())
292 thunk = transition16big;
Kevin O'Connor11884802014-09-30 12:13:44 -0400293 func -= BUILD_BIOS_ADDR;
Kevin O'Connor4c568d82015-09-13 10:03:12 -0400294 u32 stackseg = Call16Data.ss;
Kevin O'Connor11884802014-09-30 12:13:44 -0400295 asm volatile(
296 // Transition to 16bit mode
297 " movl $(1f - " __stringify(BUILD_BIOS_ADDR) "), %%edx\n"
Kevin O'Connor46a346e2015-09-11 16:07:59 -0400298 " jmp *%%ecx\n"
Kevin O'Connor11884802014-09-30 12:13:44 -0400299 // Setup ss/esp and call func
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500300 ASM32_SWITCH16
Kevin O'Connor46a346e2015-09-11 16:07:59 -0400301 "1:movl %2, %%ecx\n"
302 " shll $4, %2\n"
Kevin O'Connor11884802014-09-30 12:13:44 -0400303 " movw %%cx, %%ss\n"
Kevin O'Connor46a346e2015-09-11 16:07:59 -0400304 " subl %2, %%esp\n"
Kevin O'Connor11884802014-09-30 12:13:44 -0400305 " movw %%cx, %%ds\n"
Kevin O'Connor46a346e2015-09-11 16:07:59 -0400306 " movl %4, %%edx\n"
307 " movl %3, %%ecx\n"
Kevin O'Connor25eb0612015-09-11 15:38:43 -0400308 " calll _cfunc16_call16_helper\n"
Kevin O'Connor11884802014-09-30 12:13:44 -0400309 // Return to 32bit and restore esp
310 " movl $2f, %%edx\n"
Kevin O'Connor423542e2015-09-11 16:19:02 -0400311 " jmp transition32_nmi_off\n"
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500312 ASM32_BACK32
Kevin O'Connor46a346e2015-09-11 16:07:59 -0400313 "2:addl %2, %%esp\n"
314 : "+a" (eax), "+c"(thunk), "+r"(stackseg)
315 : "r" (func), "r" (edx)
316 : "edx", "cc", "memory");
Kevin O'Connor11884802014-09-30 12:13:44 -0400317 return eax;
318}
319
Kevin O'Connorbfac2b52013-09-29 10:48:24 -0400320
321/****************************************************************
Kevin O'Connor1389eee2014-09-29 19:08:57 -0400322 * Extra 16bit stack
323 ****************************************************************/
324
325// Space for a stack for 16bit code.
326u8 ExtraStack[BUILD_EXTRA_STACK_SIZE+1] VARLOW __aligned(8);
327u8 *StackPos VARLOW;
328
329// Test if currently on the extra stack
Kevin O'Connorfabc1b52014-09-29 19:18:25 -0400330int
Kevin O'Connor1389eee2014-09-29 19:08:57 -0400331on_extra_stack(void)
332{
333 return MODE16 && GET_SEG(SS) == SEG_LOW && getesp() > (u32)ExtraStack;
334}
335
336// Switch to the extra stack and call a function.
337u32
Kevin O'Connorb4cca862015-10-09 11:53:02 -0400338__stack_hop(u32 eax, u32 edx, void *func)
Kevin O'Connor1389eee2014-09-29 19:08:57 -0400339{
340 if (on_extra_stack())
341 return ((u32 (*)(u32, u32))func)(eax, edx);
342 ASSERT16();
343 u16 stack_seg = SEG_LOW;
344 u32 bkup_ss, bkup_esp;
345 asm volatile(
346 // Backup current %ss/%esp values.
347 "movw %%ss, %w3\n"
348 "movl %%esp, %4\n"
349 // Copy stack seg to %ds/%ss and set %esp
350 "movw %w6, %%ds\n"
351 "movw %w6, %%ss\n"
352 "movl %5, %%esp\n"
353 "pushl %3\n"
354 "pushl %4\n"
355 // Call func
356 "calll *%2\n"
357 "popl %4\n"
358 "popl %3\n"
359 // Restore segments and stack
360 "movw %w3, %%ds\n"
361 "movw %w3, %%ss\n"
362 "movl %4, %%esp"
363 : "+a" (eax), "+d" (edx), "+c" (func), "=&r" (bkup_ss), "=&r" (bkup_esp)
364 : "m" (StackPos), "r" (stack_seg)
365 : "cc", "memory");
366 return eax;
367}
368
369// Switch back to original caller's stack and call a function.
370u32
Kevin O'Connorb4cca862015-10-09 11:53:02 -0400371__stack_hop_back(u32 eax, u32 edx, void *func)
Kevin O'Connor1389eee2014-09-29 19:08:57 -0400372{
Kevin O'Connor80568252014-09-29 19:39:31 -0400373 if (!MODESEGMENT)
Kevin O'Connor4c599ef2015-09-22 14:38:14 -0400374 return call16(eax, edx, func);
Kevin O'Connor83c82762014-11-12 17:49:33 -0500375 if (!MODE16 || !on_extra_stack())
Kevin O'Connor1389eee2014-09-29 19:08:57 -0400376 return ((u32 (*)(u32, u32))func)(eax, edx);
377 ASSERT16();
378 u16 bkup_ss;
379 u32 bkup_stack_pos, temp;
380 asm volatile(
381 // Backup stack_pos and current %ss/%esp
382 "movl %6, %4\n"
383 "movw %%ss, %w3\n"
384 "movl %%esp, %6\n"
385 // Restore original callers' %ss/%esp
386 "movl -4(%4), %5\n"
387 "movl %5, %%ss\n"
388 "movw %%ds:-8(%4), %%sp\n"
389 "movl %5, %%ds\n"
390 // Call func
391 "calll *%2\n"
392 // Restore %ss/%esp and stack_pos
393 "movw %w3, %%ds\n"
394 "movw %w3, %%ss\n"
395 "movl %6, %%esp\n"
396 "movl %4, %6"
397 : "+a" (eax), "+d" (edx), "+c" (func), "=&r" (bkup_ss)
398 , "=&r" (bkup_stack_pos), "=&r" (temp), "+m" (StackPos)
399 :
400 : "cc", "memory");
401 return eax;
402}
403
404
405/****************************************************************
Kevin O'Connorbfac2b52013-09-29 10:48:24 -0400406 * External 16bit interface calling
407 ****************************************************************/
408
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400409// Far call 16bit code with a specified register state.
410void VISIBLE16
Kevin O'Connor2d977562013-09-29 20:21:40 -0400411_farcall16(struct bregs *callregs, u16 callregseg)
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400412{
Kevin O'Connorfabc1b52014-09-29 19:18:25 -0400413 if (need_hop_back()) {
Kevin O'Connorb4cca862015-10-09 11:53:02 -0400414 stack_hop_back(_farcall16, callregs, callregseg);
Kevin O'Connorbfac2b52013-09-29 10:48:24 -0400415 return;
416 }
Kevin O'Connor80568252014-09-29 19:39:31 -0400417 ASSERT16();
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400418 asm volatile(
419 "calll __farcall16\n"
Kevin O'Connor2d977562013-09-29 20:21:40 -0400420 : "+a" (callregs), "+m" (*callregs), "+d" (callregseg)
421 :
422 : "ebx", "ecx", "esi", "edi", "cc", "memory");
Kevin O'Connore77c7052012-05-28 22:06:42 -0400423}
424
Kevin O'Connor4c599ef2015-09-22 14:38:14 -0400425// Invoke external 16bit code.
Kevin O'Connor79c3ab32014-09-30 00:11:38 -0400426void
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400427farcall16(struct bregs *callregs)
428{
Kevin O'Connor4c599ef2015-09-22 14:38:14 -0400429 call16_override(0);
430 _farcall16(callregs, 0);
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400431}
432
Kevin O'Connor4c599ef2015-09-22 14:38:14 -0400433// Invoke external 16bit code in "big real" mode.
Kevin O'Connor79c3ab32014-09-30 00:11:38 -0400434void
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400435farcall16big(struct bregs *callregs)
436{
Kevin O'Connor4c599ef2015-09-22 14:38:14 -0400437 call16_override(1);
438 _farcall16(callregs, 0);
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400439}
440
441// Invoke a 16bit software interrupt.
Kevin O'Connor79c3ab32014-09-30 00:11:38 -0400442void
Kevin O'Connore77c7052012-05-28 22:06:42 -0400443__call16_int(struct bregs *callregs, u16 offset)
444{
Kevin O'Connore77c7052012-05-28 22:06:42 -0400445 callregs->code.offset = offset;
Kevin O'Connor79c3ab32014-09-30 00:11:38 -0400446 if (!MODESEGMENT) {
447 callregs->code.seg = SEG_BIOS;
Kevin O'Connor4c568d82015-09-13 10:03:12 -0400448 _farcall16((void*)callregs - Call16Data.ss * 16, Call16Data.ss);
Kevin O'Connor79c3ab32014-09-30 00:11:38 -0400449 return;
450 }
451 callregs->code.seg = GET_SEG(CS);
452 _farcall16(callregs, GET_SEG(SS));
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500453}
454
Kevin O'Connor43197a22014-06-06 13:49:33 -0400455// Reset the machine
456void
457reset(void)
458{
459 extern void reset_vector(void) __noreturn;
Kevin O'Connor9984b9d2014-09-30 00:17:44 -0400460 if (!MODE16)
Kevin O'Connor4c599ef2015-09-22 14:38:14 -0400461 call16(0, 0, reset_vector);
Kevin O'Connor43197a22014-06-06 13:49:33 -0400462 reset_vector();
463}
464
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500465
466/****************************************************************
467 * Threads
468 ****************************************************************/
469
Kevin O'Connore77c7052012-05-28 22:06:42 -0400470// Thread info - stored at bottom of each thread stack - don't change
471// without also updating the inline assembler below.
472struct thread_info {
Kevin O'Connore77c7052012-05-28 22:06:42 -0400473 void *stackpos;
Kevin O'Connor0039a942013-06-08 21:50:15 -0400474 struct hlist_node node;
Kevin O'Connore77c7052012-05-28 22:06:42 -0400475};
Kevin O'Connor89a2f962013-02-18 23:36:03 -0500476struct thread_info MainThread VARFSEG = {
Kevin O'Connor0039a942013-06-08 21:50:15 -0400477 NULL, { &MainThread.node, &MainThread.node.next }
Kevin O'Connore77c7052012-05-28 22:06:42 -0400478};
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500479#define THREADSTACKSIZE 4096
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500480
Kevin O'Connor2a8633f2013-06-08 22:11:07 -0400481// Check if any threads are running.
482static int
483have_threads(void)
484{
Kevin O'Connor0039a942013-06-08 21:50:15 -0400485 return (CONFIG_THREADS
486 && GET_FLATPTR(MainThread.node.next) != &MainThread.node);
Kevin O'Connor2a8633f2013-06-08 22:11:07 -0400487}
488
Kevin O'Connorad901592009-12-13 11:25:25 -0500489// Return the 'struct thread_info' for the currently running thread.
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500490struct thread_info *
Kevin O'Connor1ca05b02010-01-03 17:43:37 -0500491getCurThread(void)
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500492{
493 u32 esp = getesp();
Kevin O'Connor7acaa3c2013-10-02 20:20:03 -0400494 if (esp <= MAIN_STACK_MAX)
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500495 return &MainThread;
496 return (void*)ALIGN_DOWN(esp, THREADSTACKSIZE);
497}
498
Kevin O'Connor8b9942f2015-07-14 15:44:26 -0400499static u8 CanInterrupt, ThreadControl;
Kevin O'Connord29ce622014-04-07 12:15:34 -0400500
501// Initialize the support for internal threads.
502void
Kevin O'Connor8b9942f2015-07-14 15:44:26 -0400503thread_setup(void)
Kevin O'Connord29ce622014-04-07 12:15:34 -0400504{
Kevin O'Connor8b9942f2015-07-14 15:44:26 -0400505 CanInterrupt = 1;
Kevin O'Connor5869a6b2017-05-16 11:36:43 -0400506 call16_override(1);
Kevin O'Connord29ce622014-04-07 12:15:34 -0400507 if (! CONFIG_THREADS)
508 return;
509 ThreadControl = romfile_loadint("etc/threads", 1);
510}
511
512// Should hardware initialization threads run during optionrom execution.
513int
514threads_during_optionroms(void)
515{
Kevin O'Connorbc46ebe2015-08-13 11:43:27 -0400516 return CONFIG_THREADS && CONFIG_RTC_TIMER && ThreadControl == 2 && in_post();
Kevin O'Connord29ce622014-04-07 12:15:34 -0400517}
518
Kevin O'Connorad901592009-12-13 11:25:25 -0500519// Switch to next thread stack.
520static void
521switch_next(struct thread_info *cur)
522{
Kevin O'Connor0039a942013-06-08 21:50:15 -0400523 struct thread_info *next = container_of(
524 cur->node.next, struct thread_info, node);
Kevin O'Connorfb214dc2009-12-20 13:11:17 -0500525 if (cur == next)
526 // Nothing to do.
527 return;
Kevin O'Connorad901592009-12-13 11:25:25 -0500528 asm volatile(
529 " pushl $1f\n" // store return pc
530 " pushl %%ebp\n" // backup %ebp
Kevin O'Connor0039a942013-06-08 21:50:15 -0400531 " movl %%esp, (%%eax)\n" // cur->stackpos = %esp
532 " movl (%%ecx), %%esp\n" // %esp = next->stackpos
Kevin O'Connorad901592009-12-13 11:25:25 -0500533 " popl %%ebp\n" // restore %ebp
534 " retl\n" // restore pc
535 "1:\n"
536 : "+a"(cur), "+c"(next)
537 :
538 : "ebx", "edx", "esi", "edi", "cc", "memory");
539}
540
Kevin O'Connor48367e22013-12-22 22:44:08 -0500541// Last thing called from a thread (called on MainThread stack).
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500542static void
543__end_thread(struct thread_info *old)
544{
Kevin O'Connor0039a942013-06-08 21:50:15 -0400545 hlist_del(&old->node);
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500546 dprintf(DEBUG_thread, "\\%08x/ End thread\n", (u32)old);
Kevin O'Connor48367e22013-12-22 22:44:08 -0500547 free(old);
Kevin O'Connor2a8633f2013-06-08 22:11:07 -0400548 if (!have_threads())
Kevin O'Connore438b0c2010-05-01 12:20:33 -0400549 dprintf(1, "All threads complete.\n");
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500550}
551
Kevin O'Connorad901592009-12-13 11:25:25 -0500552// Create a new thread and start executing 'func' in it.
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500553void
554run_thread(void (*func)(void*), void *data)
555{
Kevin O'Connor52a300f2009-12-26 23:32:57 -0500556 ASSERT32FLAT();
Kevin O'Connord29ce622014-04-07 12:15:34 -0400557 if (! CONFIG_THREADS || ! ThreadControl)
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500558 goto fail;
559 struct thread_info *thread;
560 thread = memalign_tmphigh(THREADSTACKSIZE, THREADSTACKSIZE);
561 if (!thread)
562 goto fail;
563
Kevin O'Connor48367e22013-12-22 22:44:08 -0500564 dprintf(DEBUG_thread, "/%08x\\ Start thread\n", (u32)thread);
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500565 thread->stackpos = (void*)thread + THREADSTACKSIZE;
566 struct thread_info *cur = getCurThread();
Kevin O'Connor0039a942013-06-08 21:50:15 -0400567 hlist_add_after(&thread->node, &cur->node);
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500568 asm volatile(
569 // Start thread
570 " pushl $1f\n" // store return pc
571 " pushl %%ebp\n" // backup %ebp
Kevin O'Connor0039a942013-06-08 21:50:15 -0400572 " movl %%esp, (%%edx)\n" // cur->stackpos = %esp
573 " movl (%%ebx), %%esp\n" // %esp = thread->stackpos
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500574 " calll *%%ecx\n" // Call func
575
576 // End thread
Kevin O'Connor48367e22013-12-22 22:44:08 -0500577 " movl %%ebx, %%eax\n" // %eax = thread
578 " movl 4(%%ebx), %%ebx\n" // %ebx = thread->node.next
579 " movl (%5), %%esp\n" // %esp = MainThread.stackpos
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500580 " calll %4\n" // call __end_thread(thread)
Kevin O'Connor48367e22013-12-22 22:44:08 -0500581 " movl -4(%%ebx), %%esp\n" // %esp = next->stackpos
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500582 " popl %%ebp\n" // restore %ebp
583 " retl\n" // restore pc
584 "1:\n"
585 : "+a"(data), "+c"(func), "+b"(thread), "+d"(cur)
Kevin O'Connor48367e22013-12-22 22:44:08 -0500586 : "m"(*(u8*)__end_thread), "m"(MainThread)
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500587 : "esi", "edi", "cc", "memory");
588 return;
589
590fail:
591 func(data);
592}
593
Kevin O'Connor3cf301f2013-06-08 22:05:19 -0400594
595/****************************************************************
596 * Thread helpers
597 ****************************************************************/
598
599// Low-level irq enable.
600void VISIBLE16
601check_irqs(void)
602{
Kevin O'Connor2a91b342015-08-07 16:13:18 -0400603 if (!MODESEGMENT && !CanInterrupt) {
Kevin O'Connorec520682015-08-07 16:12:02 -0400604 // Can't enable interrupts (PIC and/or IVT not yet setup)
Kevin O'Connor2a91b342015-08-07 16:13:18 -0400605 cpu_relax();
Kevin O'Connorec520682015-08-07 16:12:02 -0400606 return;
Kevin O'Connor2a91b342015-08-07 16:13:18 -0400607 }
Kevin O'Connorfabc1b52014-09-29 19:18:25 -0400608 if (need_hop_back()) {
Kevin O'Connorb4cca862015-10-09 11:53:02 -0400609 stack_hop_back(check_irqs, 0, 0);
Kevin O'Connorbfac2b52013-09-29 10:48:24 -0400610 return;
611 }
Kevin O'Connorbd5f6c72015-08-10 16:14:48 -0400612 if (MODE16)
613 clock_poll_irq();
Kevin O'Connor3cf301f2013-06-08 22:05:19 -0400614 asm volatile("sti ; nop ; rep ; nop ; cli ; cld" : : :"memory");
615}
616
617// Briefly permit irqs to occur.
618void
619yield(void)
620{
Kevin O'Connor80568252014-09-29 19:39:31 -0400621 if (MODESEGMENT || !CONFIG_THREADS) {
Kevin O'Connorec520682015-08-07 16:12:02 -0400622 check_irqs();
Kevin O'Connor3cf301f2013-06-08 22:05:19 -0400623 return;
624 }
Kevin O'Connor3cf301f2013-06-08 22:05:19 -0400625 struct thread_info *cur = getCurThread();
Kevin O'Connorec520682015-08-07 16:12:02 -0400626 if (cur == &MainThread)
Kevin O'Connor3cf301f2013-06-08 22:05:19 -0400627 // Permit irqs to fire
Kevin O'Connor80568252014-09-29 19:39:31 -0400628 check_irqs();
Kevin O'Connor3cf301f2013-06-08 22:05:19 -0400629
630 // Switch to the next thread
631 switch_next(cur);
632}
633
634void VISIBLE16
635wait_irq(void)
636{
Kevin O'Connorfabc1b52014-09-29 19:18:25 -0400637 if (need_hop_back()) {
Kevin O'Connorb4cca862015-10-09 11:53:02 -0400638 stack_hop_back(wait_irq, 0, 0);
Kevin O'Connorbfac2b52013-09-29 10:48:24 -0400639 return;
640 }
Kevin O'Connor3cf301f2013-06-08 22:05:19 -0400641 asm volatile("sti ; hlt ; cli ; cld": : :"memory");
642}
643
644// Wait for next irq to occur.
645void
646yield_toirq(void)
647{
Kevin O'Connorbd5f6c72015-08-10 16:14:48 -0400648 if (!CONFIG_HARDWARE_IRQ
649 || (!MODESEGMENT && (have_threads() || !CanInterrupt))) {
Kevin O'Connorec520682015-08-07 16:12:02 -0400650 // Threads still active or irqs not available - do a yield instead.
Kevin O'Connor3cf301f2013-06-08 22:05:19 -0400651 yield();
652 return;
653 }
Kevin O'Connorec520682015-08-07 16:12:02 -0400654 wait_irq();
Kevin O'Connor3cf301f2013-06-08 22:05:19 -0400655}
656
Kevin O'Connorad901592009-12-13 11:25:25 -0500657// Wait for all threads (other than the main thread) to complete.
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500658void
Kevin O'Connor1ca05b02010-01-03 17:43:37 -0500659wait_threads(void)
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500660{
Kevin O'Connor52a300f2009-12-26 23:32:57 -0500661 ASSERT32FLAT();
Kevin O'Connor2a8633f2013-06-08 22:11:07 -0400662 while (have_threads())
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500663 yield();
664}
Kevin O'Connorad901592009-12-13 11:25:25 -0500665
Kevin O'Connore9086652010-02-27 13:49:47 -0500666void
667mutex_lock(struct mutex_s *mutex)
668{
669 ASSERT32FLAT();
670 if (! CONFIG_THREADS)
671 return;
672 while (mutex->isLocked)
673 yield();
674 mutex->isLocked = 1;
675}
676
677void
678mutex_unlock(struct mutex_s *mutex)
679{
680 ASSERT32FLAT();
681 if (! CONFIG_THREADS)
682 return;
683 mutex->isLocked = 0;
684}
685
Kevin O'Connorad901592009-12-13 11:25:25 -0500686
687/****************************************************************
688 * Thread preemption
689 ****************************************************************/
690
Kevin O'Connor89a2f962013-02-18 23:36:03 -0500691int CanPreempt VARFSEG;
Kevin O'Connorad901592009-12-13 11:25:25 -0500692static u32 PreemptCount;
693
694// Turn on RTC irqs and arrange for them to check the 32bit threads.
695void
Kevin O'Connor1ca05b02010-01-03 17:43:37 -0500696start_preempt(void)
Kevin O'Connorad901592009-12-13 11:25:25 -0500697{
Kevin O'Connord29ce622014-04-07 12:15:34 -0400698 if (! threads_during_optionroms())
Kevin O'Connorad901592009-12-13 11:25:25 -0500699 return;
700 CanPreempt = 1;
701 PreemptCount = 0;
Kevin O'Connor8b7861c2013-09-15 02:29:06 -0400702 rtc_use();
Kevin O'Connorad901592009-12-13 11:25:25 -0500703}
704
705// Turn off RTC irqs / stop checking for thread execution.
706void
Kevin O'Connor1ca05b02010-01-03 17:43:37 -0500707finish_preempt(void)
Kevin O'Connorad901592009-12-13 11:25:25 -0500708{
Kevin O'Connord29ce622014-04-07 12:15:34 -0400709 if (! threads_during_optionroms()) {
Kevin O'Connora7eb8fc2010-04-02 13:13:23 -0400710 yield();
Kevin O'Connorad901592009-12-13 11:25:25 -0500711 return;
Kevin O'Connora7eb8fc2010-04-02 13:13:23 -0400712 }
Kevin O'Connorad901592009-12-13 11:25:25 -0500713 CanPreempt = 0;
Kevin O'Connor8b7861c2013-09-15 02:29:06 -0400714 rtc_release();
Kevin O'Connore438b0c2010-05-01 12:20:33 -0400715 dprintf(9, "Done preempt - %d checks\n", PreemptCount);
Kevin O'Connora7eb8fc2010-04-02 13:13:23 -0400716 yield();
Kevin O'Connorad901592009-12-13 11:25:25 -0500717}
718
Kevin O'Connord7eb27e2010-03-20 18:17:19 -0400719// Check if preemption is on, and wait for it to complete if so.
720int
721wait_preempt(void)
722{
Kevin O'Connord29ce622014-04-07 12:15:34 -0400723 if (MODESEGMENT || !CONFIG_THREADS || !CanPreempt
Kevin O'Connor7acaa3c2013-10-02 20:20:03 -0400724 || getesp() < MAIN_STACK_MAX)
Kevin O'Connord7eb27e2010-03-20 18:17:19 -0400725 return 0;
726 while (CanPreempt)
727 yield();
728 return 1;
729}
730
Kevin O'Connorad901592009-12-13 11:25:25 -0500731// Try to execute 32bit threads.
Kevin O'Connord1b4f962010-09-15 21:38:16 -0400732void VISIBLE32INIT
Kevin O'Connor1ca05b02010-01-03 17:43:37 -0500733yield_preempt(void)
Kevin O'Connorad901592009-12-13 11:25:25 -0500734{
735 PreemptCount++;
736 switch_next(&MainThread);
737}
Kevin O'Connorad901592009-12-13 11:25:25 -0500738
739// 16bit code that checks if threads are pending and executes them if so.
740void
Kevin O'Connor1ca05b02010-01-03 17:43:37 -0500741check_preempt(void)
Kevin O'Connorad901592009-12-13 11:25:25 -0500742{
Kevin O'Connord29ce622014-04-07 12:15:34 -0400743 if (CONFIG_THREADS && GET_GLOBAL(CanPreempt) && have_threads())
Kevin O'Connorb4cca862015-10-09 11:53:02 -0400744 call32(yield_preempt, 0, 0);
Kevin O'Connorad901592009-12-13 11:25:25 -0500745}
Kevin O'Connor7f54bf72013-12-30 22:04:28 -0500746
747
748/****************************************************************
749 * call32 helper
750 ****************************************************************/
751
752struct call32_params_s {
753 void *func;
754 u32 eax, edx, ecx;
755};
756
757u32 VISIBLE32FLAT
758call32_params_helper(struct call32_params_s *params)
759{
760 return ((u32 (*)(u32, u32, u32))params->func)(
761 params->eax, params->edx, params->ecx);
762}
763
764u32
Kevin O'Connorb4cca862015-10-09 11:53:02 -0400765__call32_params(void *func, u32 eax, u32 edx, u32 ecx, u32 errret)
Kevin O'Connor7f54bf72013-12-30 22:04:28 -0500766{
767 ASSERT16();
768 struct call32_params_s params = {func, eax, edx, ecx};
Kevin O'Connorb4cca862015-10-09 11:53:02 -0400769 return call32(call32_params_helper, MAKE_FLATPTR(GET_SEG(SS), &params)
770 , errret);
Kevin O'Connor7f54bf72013-12-30 22:04:28 -0500771}