blob: 1dbdfe9bbc525942e84c9b3aa66d016c1ce0e6ee [file] [log] [blame]
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -05001// Code for manipulating stack locations.
2//
Kevin O'Connor23421292014-09-24 17:48:28 -04003// Copyright (C) 2009-2014 Kevin O'Connor <kevin@koconnor.net>
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -05004//
5// This file may be distributed under the terms of the GNU LGPLv3 license.
6
Kevin O'Connor46b82622012-05-13 12:10:30 -04007#include "biosvar.h" // GET_GLOBAL
Kevin O'Connorad901592009-12-13 11:25:25 -05008#include "bregs.h" // CR0_PE
Kevin O'Connor55215cd2014-04-11 11:20:41 -04009#include "fw/paravirt.h" // PORT_SMI_CMD
Kevin O'Connor8b7861c2013-09-15 02:29:06 -040010#include "hw/rtc.h" // rtc_use
Kevin O'Connor0039a942013-06-08 21:50:15 -040011#include "list.h" // hlist_node
Kevin O'Connor9dea5902013-09-14 20:23:54 -040012#include "malloc.h" // free
Kevin O'Connor2d2fa312013-09-14 21:55:26 -040013#include "output.h" // dprintf
Kevin O'Connord29ce622014-04-07 12:15:34 -040014#include "romfile.h" // romfile_loadint
Kevin O'Connor3df600b2013-09-14 19:28:55 -040015#include "stacks.h" // struct mutex_s
Kevin O'Connor2d2fa312013-09-14 21:55:26 -040016#include "util.h" // useRTC
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -050017
Kevin O'Connor7acaa3c2013-10-02 20:20:03 -040018#define MAIN_STACK_MAX (1024*1024)
19
Kevin O'Connore77c7052012-05-28 22:06:42 -040020
21/****************************************************************
Kevin O'Connore77c7052012-05-28 22:06:42 -040022 * 16bit / 32bit calling
Kevin O'Connor9c447c32010-05-23 10:24:22 -040023 ****************************************************************/
24
Kevin O'Connor15721bf2014-09-30 12:55:54 -040025struct {
26 u8 method;
27 u8 cmosindex;
Kevin O'Connorbfb7b582014-10-11 13:27:16 -040028 u8 a20;
Kevin O'Connor15721bf2014-09-30 12:55:54 -040029 u16 ss, fs, gs;
30 struct descloc_s gdt;
31} Call32Data VARLOW;
32
Kevin O'Connordcacfa02014-09-29 23:59:47 -040033#define C32_SLOPPY 1
Kevin O'Connor55215cd2014-04-11 11:20:41 -040034#define C32_SMM 2
35
36int HaveSmmCall32 VARFSEG;
37
38// Backup state in preparation for call32_smm()
39static void
40call32_smm_prep(void)
41{
42 // Backup cmos index register and disable nmi
43 u8 cmosindex = inb(PORT_CMOS_INDEX);
44 outb(cmosindex | NMI_DISABLE_BIT, PORT_CMOS_INDEX);
45 inb(PORT_CMOS_DATA);
46 SET_LOW(Call32Data.cmosindex, cmosindex);
47
48 // Backup ss
49 SET_LOW(Call32Data.ss, GET_SEG(SS));
50
51 SET_LOW(Call32Data.method, C32_SMM);
52}
53
54// Restore state backed up during call32_smm()
55static void
56call32_smm_post(void)
57{
58 SET_LOW(Call32Data.method, 0);
59 SET_LOW(Call32Data.ss, 0);
60
61 // Restore cmos index register
62 outb(GET_LOW(Call32Data.cmosindex), PORT_CMOS_INDEX);
63 inb(PORT_CMOS_DATA);
64}
65
Kevin O'Connored675ad2014-12-03 12:53:01 -050066#define ASM32_SWITCH16 " .pushsection .text.32fseg." UNIQSEC "\n .code16\n"
67#define ASM32_BACK32 " .popsection\n .code32\n"
Kevin O'Connorff6d5512014-12-03 12:39:28 -050068#define ASM16_SWITCH32 " .code32\n"
69#define ASM16_BACK16 " .code16gcc\n"
70
Kevin O'Connor55215cd2014-04-11 11:20:41 -040071// Call a SeaBIOS C function in 32bit mode using smm trampoline
72static u32
73call32_smm(void *func, u32 eax)
74{
75 ASSERT16();
76 dprintf(9, "call32_smm %p %x\n", func, eax);
77 call32_smm_prep();
78 u32 bkup_esp;
79 asm volatile(
80 // Backup esp / set esp to flat stack location
81 " movl %%esp, %0\n"
82 " movl %%ss, %%eax\n"
83 " shll $4, %%eax\n"
84 " addl %%eax, %%esp\n"
85
86 // Transition to 32bit mode, call func, return to 16bit
87 " movl $" __stringify(CALL32SMM_CMDID) ", %%eax\n"
88 " movl $" __stringify(CALL32SMM_ENTERID) ", %%ecx\n"
89 " movl $(" __stringify(BUILD_BIOS_ADDR) " + 1f), %%ebx\n"
90 " outb %%al, $" __stringify(PORT_SMI_CMD) "\n"
91 " rep; nop\n"
92 " hlt\n"
Kevin O'Connor55215cd2014-04-11 11:20:41 -040093
Kevin O'Connorff6d5512014-12-03 12:39:28 -050094 ASM16_SWITCH32
Kevin O'Connor55215cd2014-04-11 11:20:41 -040095 "1:movl %1, %%eax\n"
96 " calll *%2\n"
97 " movl %%eax, %1\n"
98
99 " movl $" __stringify(CALL32SMM_CMDID) ", %%eax\n"
100 " movl $" __stringify(CALL32SMM_RETURNID) ", %%ecx\n"
101 " movl $2f, %%ebx\n"
102 " outb %%al, $" __stringify(PORT_SMI_CMD) "\n"
103 " rep; nop\n"
104 " hlt\n"
105
106 // Restore esp
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500107 ASM16_BACK16
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400108 "2:movl %0, %%esp\n"
109 : "=&r" (bkup_esp), "+r" (eax)
110 : "r" (func)
111 : "eax", "ecx", "edx", "ebx", "cc", "memory");
112 call32_smm_post();
113
114 dprintf(9, "call32_smm done %p %x\n", func, eax);
115 return eax;
116}
117
118// 16bit handler code called from call16_smm()
119u32 VISIBLE16
120call16_smm_helper(u32 eax, u32 edx, u32 (*func)(u32 eax, u32 edx))
121{
122 if (!CONFIG_CALL32_SMM)
123 return eax;
124 call32_smm_post();
125 u32 ret = func(eax, edx);
126 call32_smm_prep();
127 return ret;
128}
129
Kevin O'Connored675ad2014-12-03 12:53:01 -0500130static u32
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400131call16_smm(u32 eax, u32 edx, void *func)
132{
133 ASSERT32FLAT();
134 if (!CONFIG_CALL32_SMM)
135 return eax;
136 func -= BUILD_BIOS_ADDR;
137 dprintf(9, "call16_smm %p %x %x\n", func, eax, edx);
138 u32 stackoffset = Call32Data.ss << 4;
139 asm volatile(
140 // Restore esp
141 " subl %0, %%esp\n"
142
143 // Transition to 16bit mode, call func, return to 32bit
144 " movl $" __stringify(CALL32SMM_CMDID) ", %%eax\n"
145 " movl $" __stringify(CALL32SMM_RETURNID) ", %%ecx\n"
146 " movl $(1f - " __stringify(BUILD_BIOS_ADDR) "), %%ebx\n"
147 " outb %%al, $" __stringify(PORT_SMI_CMD) "\n"
148 " rep; nop\n"
149 " hlt\n"
150
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500151 ASM32_SWITCH16
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400152 "1:movl %1, %%eax\n"
153 " movl %3, %%ecx\n"
154 " calll _cfunc16_call16_smm_helper\n"
155 " movl %%eax, %1\n"
156
157 " movl $" __stringify(CALL32SMM_CMDID) ", %%eax\n"
158 " movl $" __stringify(CALL32SMM_ENTERID) ", %%ecx\n"
159 " movl $2f, %%ebx\n"
160 " outb %%al, $" __stringify(PORT_SMI_CMD) "\n"
161 " rep; nop\n"
162 " hlt\n"
163
164 // Set esp to flat stack location
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500165 ASM32_BACK32
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400166 "2:addl %0, %%esp\n"
167 : "+r" (stackoffset), "+r" (eax), "+d" (edx)
168 : "r" (func)
169 : "eax", "ecx", "ebx", "cc", "memory");
170 return eax;
171}
Kevin O'Connor7acaa3c2013-10-02 20:20:03 -0400172
Kevin O'Connor15721bf2014-09-30 12:55:54 -0400173// Backup state in preparation for call32_sloppy()
174static void
175call32_sloppy_prep(void)
176{
177 // Backup cmos index register and disable nmi
178 u8 cmosindex = inb(PORT_CMOS_INDEX);
179 outb(cmosindex | NMI_DISABLE_BIT, PORT_CMOS_INDEX);
180 inb(PORT_CMOS_DATA);
181 SET_LOW(Call32Data.cmosindex, cmosindex);
182
Kevin O'Connorbfb7b582014-10-11 13:27:16 -0400183 // Enable a20 and backup it's previous state
184 SET_LOW(Call32Data.a20, set_a20(1));
185
Kevin O'Connor15721bf2014-09-30 12:55:54 -0400186 // Backup ss/fs/gs and gdt
187 SET_LOW(Call32Data.ss, GET_SEG(SS));
188 SET_LOW(Call32Data.fs, GET_SEG(FS));
189 SET_LOW(Call32Data.gs, GET_SEG(GS));
190 struct descloc_s gdt;
191 sgdt(&gdt);
192 SET_LOW(Call32Data.gdt.length, gdt.length);
193 SET_LOW(Call32Data.gdt.addr, gdt.addr);
194
195 SET_LOW(Call32Data.method, C32_SLOPPY);
196}
197
198// Restore state backed up during call32_sloppy()
199static void
200call32_sloppy_post(void)
201{
202 SET_LOW(Call32Data.method, 0);
203 SET_LOW(Call32Data.ss, 0);
204
205 // Restore gdt and fs/gs
206 struct descloc_s gdt;
207 gdt.length = GET_LOW(Call32Data.gdt.length);
208 gdt.addr = GET_LOW(Call32Data.gdt.addr);
209 lgdt(&gdt);
210 SET_SEG(FS, GET_LOW(Call32Data.fs));
211 SET_SEG(GS, GET_LOW(Call32Data.gs));
212
Kevin O'Connorbfb7b582014-10-11 13:27:16 -0400213 // Restore a20
214 set_a20(GET_LOW(Call32Data.a20));
215
Kevin O'Connor15721bf2014-09-30 12:55:54 -0400216 // Restore cmos index register
217 outb(GET_LOW(Call32Data.cmosindex), PORT_CMOS_INDEX);
218 inb(PORT_CMOS_DATA);
219}
220
Kevin O'Connor11884802014-09-30 12:13:44 -0400221// Call a C function in 32bit mode. This clobbers the 16bit segment
222// selector registers.
223static u32
224call32_sloppy(void *func, u32 eax)
Kevin O'Connora7fc5de2009-12-13 11:48:18 -0500225{
226 ASSERT16();
Kevin O'Connor15721bf2014-09-30 12:55:54 -0400227 call32_sloppy_prep();
Kevin O'Connora7fc5de2009-12-13 11:48:18 -0500228 u32 bkup_ss, bkup_esp;
229 asm volatile(
230 // Backup ss/esp / set esp to flat stack location
231 " movl %%ss, %0\n"
232 " movl %%esp, %1\n"
233 " shll $4, %0\n"
234 " addl %0, %%esp\n"
Kevin O'Connoraf9629b2010-11-25 09:17:31 -0500235 " shrl $4, %0\n"
Kevin O'Connora7fc5de2009-12-13 11:48:18 -0500236
Kevin O'Connorfb214dc2009-12-20 13:11:17 -0500237 // Transition to 32bit mode, call func, return to 16bit
Kevin O'Connor4057f982010-11-25 08:52:50 -0500238 " movl $(" __stringify(BUILD_BIOS_ADDR) " + 1f), %%edx\n"
Kevin O'Connora7fc5de2009-12-13 11:48:18 -0500239 " jmp transition32\n"
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500240 ASM16_SWITCH32
Kevin O'Connoraf9629b2010-11-25 09:17:31 -0500241 "1:calll *%3\n"
Kevin O'Connor4057f982010-11-25 08:52:50 -0500242 " movl $2f, %%edx\n"
Kevin O'Connora7fc5de2009-12-13 11:48:18 -0500243 " jmp transition16big\n"
244
245 // Restore ds/ss/esp
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500246 ASM16_BACK16
Kevin O'Connora7fc5de2009-12-13 11:48:18 -0500247 "2:movl %0, %%ds\n"
248 " movl %0, %%ss\n"
249 " movl %1, %%esp\n"
Kevin O'Connoraf9629b2010-11-25 09:17:31 -0500250 : "=&r" (bkup_ss), "=&r" (bkup_esp), "+a" (eax)
Kevin O'Connorbca07362010-03-20 20:41:38 -0400251 : "r" (func)
Kevin O'Connoraf9629b2010-11-25 09:17:31 -0500252 : "ecx", "edx", "cc", "memory");
Kevin O'Connor15721bf2014-09-30 12:55:54 -0400253 call32_sloppy_post();
Kevin O'Connoraf9629b2010-11-25 09:17:31 -0500254 return eax;
Kevin O'Connora7fc5de2009-12-13 11:48:18 -0500255}
256
Kevin O'Connor15721bf2014-09-30 12:55:54 -0400257// 16bit handler code called from call16_sloppy()
258u32 VISIBLE16
259call16_sloppy_helper(u32 eax, u32 edx, u32 (*func)(u32 eax, u32 edx))
260{
261 call32_sloppy_post();
262 u32 ret = func(eax, edx);
263 call32_sloppy_prep();
264 return ret;
265}
266
Kevin O'Connor11884802014-09-30 12:13:44 -0400267// Jump back to 16bit mode while in 32bit mode from call32_sloppy()
Kevin O'Connored675ad2014-12-03 12:53:01 -0500268static u32
Kevin O'Connor11884802014-09-30 12:13:44 -0400269call16_sloppy(u32 eax, u32 edx, void *func)
270{
271 ASSERT32FLAT();
272 if (getesp() > MAIN_STACK_MAX)
273 panic("call16_sloppy with invalid stack\n");
274 func -= BUILD_BIOS_ADDR;
Kevin O'Connor15721bf2014-09-30 12:55:54 -0400275 u32 stackseg = Call32Data.ss;
Kevin O'Connor11884802014-09-30 12:13:44 -0400276 asm volatile(
277 // Transition to 16bit mode
278 " movl $(1f - " __stringify(BUILD_BIOS_ADDR) "), %%edx\n"
279 " jmp transition16big\n"
280 // Setup ss/esp and call func
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500281 ASM32_SWITCH16
Kevin O'Connor11884802014-09-30 12:13:44 -0400282 "1:movl %3, %%ecx\n"
283 " shll $4, %3\n"
284 " movw %%cx, %%ss\n"
285 " subl %3, %%esp\n"
286 " movw %%cx, %%ds\n"
287 " movl %2, %%edx\n"
Kevin O'Connor15721bf2014-09-30 12:55:54 -0400288 " movl %1, %%ecx\n"
289 " calll _cfunc16_call16_sloppy_helper\n"
Kevin O'Connor11884802014-09-30 12:13:44 -0400290 // Return to 32bit and restore esp
291 " movl $2f, %%edx\n"
292 " jmp transition32\n"
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500293 ASM32_BACK32
Kevin O'Connor11884802014-09-30 12:13:44 -0400294 "2:addl %3, %%esp\n"
295 : "+a" (eax)
296 : "r" (func), "r" (edx), "r" (stackseg)
297 : "edx", "ecx", "cc", "memory");
Kevin O'Connor11884802014-09-30 12:13:44 -0400298 return eax;
299}
300
301// Call a 32bit SeaBIOS function from a 16bit SeaBIOS function.
302u32 VISIBLE16
303call32(void *func, u32 eax, u32 errret)
304{
305 ASSERT16();
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400306 if (CONFIG_CALL32_SMM && GET_GLOBAL(HaveSmmCall32))
307 return call32_smm(func, eax);
Kevin O'Connor11884802014-09-30 12:13:44 -0400308 u32 cr0 = getcr0();
309 if (cr0 & CR0_PE)
310 // Called in 16bit protected mode?!
311 return errret;
312 return call32_sloppy(func, eax);
313}
314
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400315// Call a 16bit SeaBIOS function from a 32bit SeaBIOS function.
Kevin O'Connored675ad2014-12-03 12:53:01 -0500316static u32
Kevin O'Connor9e75b082013-09-29 19:58:09 -0400317call16(u32 eax, u32 edx, void *func)
Kevin O'Connore77c7052012-05-28 22:06:42 -0400318{
319 ASSERT32FLAT();
Kevin O'Connor23421292014-09-24 17:48:28 -0400320 if (getesp() > BUILD_STACK_ADDR)
Kevin O'Connore77c7052012-05-28 22:06:42 -0400321 panic("call16 with invalid stack\n");
Kevin O'Connor23421292014-09-24 17:48:28 -0400322 func -= BUILD_BIOS_ADDR;
323 asm volatile(
324 // Transition to 16bit mode
325 " movl $(1f - " __stringify(BUILD_BIOS_ADDR) "), %%edx\n"
326 " jmp transition16\n"
327 // Call func
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500328 ASM32_SWITCH16
Kevin O'Connor23421292014-09-24 17:48:28 -0400329 "1:movl %2, %%edx\n"
330 " calll *%1\n"
331 // Return to 32bit
332 " movl $2f, %%edx\n"
333 " jmp transition32\n"
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500334 ASM32_BACK32
Kevin O'Connor23421292014-09-24 17:48:28 -0400335 "2:\n"
336 : "+a" (eax)
337 : "r" (func), "r" (edx)
338 : "edx", "ecx", "cc", "memory");
339 return eax;
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400340}
341
Kevin O'Connordcacfa02014-09-29 23:59:47 -0400342// Call a 16bit SeaBIOS function in "big real" mode.
Kevin O'Connored675ad2014-12-03 12:53:01 -0500343static u32
Kevin O'Connor9e75b082013-09-29 19:58:09 -0400344call16big(u32 eax, u32 edx, void *func)
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400345{
346 ASSERT32FLAT();
Kevin O'Connor23421292014-09-24 17:48:28 -0400347 if (getesp() > BUILD_STACK_ADDR)
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400348 panic("call16big with invalid stack\n");
Kevin O'Connor23421292014-09-24 17:48:28 -0400349 func -= BUILD_BIOS_ADDR;
350 asm volatile(
351 // Transition to 16bit mode
352 " movl $(1f - " __stringify(BUILD_BIOS_ADDR) "), %%edx\n"
353 " jmp transition16big\n"
354 // Call func
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500355 ASM32_SWITCH16
Kevin O'Connor23421292014-09-24 17:48:28 -0400356 "1:movl %2, %%edx\n"
357 " calll *%1\n"
358 // Return to 32bit
359 " movl $2f, %%edx\n"
360 " jmp transition32\n"
Kevin O'Connorff6d5512014-12-03 12:39:28 -0500361 ASM32_BACK32
Kevin O'Connor23421292014-09-24 17:48:28 -0400362 "2:\n"
363 : "+a" (eax)
364 : "r" (func), "r" (edx)
365 : "edx", "ecx", "cc", "memory");
366 return eax;
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400367}
368
Kevin O'Connordcacfa02014-09-29 23:59:47 -0400369// Call a 16bit SeaBIOS function, restoring the mode from last call32().
370static u32
371call16_back(u32 eax, u32 edx, void *func)
372{
373 ASSERT32FLAT();
Kevin O'Connor55215cd2014-04-11 11:20:41 -0400374 if (CONFIG_CALL32_SMM && Call32Data.method == C32_SMM)
375 return call16_smm(eax, edx, func);
Kevin O'Connor15721bf2014-09-30 12:55:54 -0400376 if (Call32Data.method == C32_SLOPPY)
Kevin O'Connordcacfa02014-09-29 23:59:47 -0400377 return call16_sloppy(eax, edx, func);
378 if (in_post())
379 return call16big(eax, edx, func);
380 return call16(eax, edx, func);
381}
382
Kevin O'Connorbfac2b52013-09-29 10:48:24 -0400383
384/****************************************************************
Kevin O'Connor1389eee2014-09-29 19:08:57 -0400385 * Extra 16bit stack
386 ****************************************************************/
387
388// Space for a stack for 16bit code.
389u8 ExtraStack[BUILD_EXTRA_STACK_SIZE+1] VARLOW __aligned(8);
390u8 *StackPos VARLOW;
391
392// Test if currently on the extra stack
Kevin O'Connorfabc1b52014-09-29 19:18:25 -0400393int
Kevin O'Connor1389eee2014-09-29 19:08:57 -0400394on_extra_stack(void)
395{
396 return MODE16 && GET_SEG(SS) == SEG_LOW && getesp() > (u32)ExtraStack;
397}
398
399// Switch to the extra stack and call a function.
400u32
401stack_hop(u32 eax, u32 edx, void *func)
402{
403 if (on_extra_stack())
404 return ((u32 (*)(u32, u32))func)(eax, edx);
405 ASSERT16();
406 u16 stack_seg = SEG_LOW;
407 u32 bkup_ss, bkup_esp;
408 asm volatile(
409 // Backup current %ss/%esp values.
410 "movw %%ss, %w3\n"
411 "movl %%esp, %4\n"
412 // Copy stack seg to %ds/%ss and set %esp
413 "movw %w6, %%ds\n"
414 "movw %w6, %%ss\n"
415 "movl %5, %%esp\n"
416 "pushl %3\n"
417 "pushl %4\n"
418 // Call func
419 "calll *%2\n"
420 "popl %4\n"
421 "popl %3\n"
422 // Restore segments and stack
423 "movw %w3, %%ds\n"
424 "movw %w3, %%ss\n"
425 "movl %4, %%esp"
426 : "+a" (eax), "+d" (edx), "+c" (func), "=&r" (bkup_ss), "=&r" (bkup_esp)
427 : "m" (StackPos), "r" (stack_seg)
428 : "cc", "memory");
429 return eax;
430}
431
432// Switch back to original caller's stack and call a function.
433u32
434stack_hop_back(u32 eax, u32 edx, void *func)
435{
Kevin O'Connor80568252014-09-29 19:39:31 -0400436 if (!MODESEGMENT)
Kevin O'Connordcacfa02014-09-29 23:59:47 -0400437 return call16_back(eax, edx, func);
Kevin O'Connor83c82762014-11-12 17:49:33 -0500438 if (!MODE16 || !on_extra_stack())
Kevin O'Connor1389eee2014-09-29 19:08:57 -0400439 return ((u32 (*)(u32, u32))func)(eax, edx);
440 ASSERT16();
441 u16 bkup_ss;
442 u32 bkup_stack_pos, temp;
443 asm volatile(
444 // Backup stack_pos and current %ss/%esp
445 "movl %6, %4\n"
446 "movw %%ss, %w3\n"
447 "movl %%esp, %6\n"
448 // Restore original callers' %ss/%esp
449 "movl -4(%4), %5\n"
450 "movl %5, %%ss\n"
451 "movw %%ds:-8(%4), %%sp\n"
452 "movl %5, %%ds\n"
453 // Call func
454 "calll *%2\n"
455 // Restore %ss/%esp and stack_pos
456 "movw %w3, %%ds\n"
457 "movw %w3, %%ss\n"
458 "movl %6, %%esp\n"
459 "movl %4, %6"
460 : "+a" (eax), "+d" (edx), "+c" (func), "=&r" (bkup_ss)
461 , "=&r" (bkup_stack_pos), "=&r" (temp), "+m" (StackPos)
462 :
463 : "cc", "memory");
464 return eax;
465}
466
467
468/****************************************************************
Kevin O'Connorbfac2b52013-09-29 10:48:24 -0400469 * External 16bit interface calling
470 ****************************************************************/
471
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400472// Far call 16bit code with a specified register state.
473void VISIBLE16
Kevin O'Connor2d977562013-09-29 20:21:40 -0400474_farcall16(struct bregs *callregs, u16 callregseg)
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400475{
Kevin O'Connorfabc1b52014-09-29 19:18:25 -0400476 if (need_hop_back()) {
Kevin O'Connor80568252014-09-29 19:39:31 -0400477 extern void _cfunc16__farcall16(void);
478 stack_hop_back((u32)callregs, callregseg, _cfunc16__farcall16);
Kevin O'Connorbfac2b52013-09-29 10:48:24 -0400479 return;
480 }
Kevin O'Connor80568252014-09-29 19:39:31 -0400481 ASSERT16();
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400482 asm volatile(
483 "calll __farcall16\n"
Kevin O'Connor2d977562013-09-29 20:21:40 -0400484 : "+a" (callregs), "+m" (*callregs), "+d" (callregseg)
485 :
486 : "ebx", "ecx", "esi", "edi", "cc", "memory");
Kevin O'Connore77c7052012-05-28 22:06:42 -0400487}
488
Kevin O'Connor79c3ab32014-09-30 00:11:38 -0400489void
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400490farcall16(struct bregs *callregs)
491{
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400492 extern void _cfunc16__farcall16(void);
Kevin O'Connor79c3ab32014-09-30 00:11:38 -0400493 call16((u32)callregs, 0, _cfunc16__farcall16);
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400494}
495
Kevin O'Connor79c3ab32014-09-30 00:11:38 -0400496void
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400497farcall16big(struct bregs *callregs)
498{
499 extern void _cfunc16__farcall16(void);
Kevin O'Connor79c3ab32014-09-30 00:11:38 -0400500 call16big((u32)callregs, 0, _cfunc16__farcall16);
Kevin O'Connor2f898d52012-05-28 10:56:20 -0400501}
502
503// Invoke a 16bit software interrupt.
Kevin O'Connor79c3ab32014-09-30 00:11:38 -0400504void
Kevin O'Connore77c7052012-05-28 22:06:42 -0400505__call16_int(struct bregs *callregs, u16 offset)
506{
Kevin O'Connore77c7052012-05-28 22:06:42 -0400507 callregs->code.offset = offset;
Kevin O'Connor79c3ab32014-09-30 00:11:38 -0400508 if (!MODESEGMENT) {
509 callregs->code.seg = SEG_BIOS;
Kevin O'Connor15721bf2014-09-30 12:55:54 -0400510 _farcall16((void*)callregs - Call32Data.ss * 16, Call32Data.ss);
Kevin O'Connor79c3ab32014-09-30 00:11:38 -0400511 return;
512 }
513 callregs->code.seg = GET_SEG(CS);
514 _farcall16(callregs, GET_SEG(SS));
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500515}
516
Kevin O'Connor43197a22014-06-06 13:49:33 -0400517// Reset the machine
518void
519reset(void)
520{
521 extern void reset_vector(void) __noreturn;
Kevin O'Connor9984b9d2014-09-30 00:17:44 -0400522 if (!MODE16)
523 call16_back(0, 0, reset_vector);
Kevin O'Connor43197a22014-06-06 13:49:33 -0400524 reset_vector();
525}
526
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500527
528/****************************************************************
529 * Threads
530 ****************************************************************/
531
Kevin O'Connore77c7052012-05-28 22:06:42 -0400532// Thread info - stored at bottom of each thread stack - don't change
533// without also updating the inline assembler below.
534struct thread_info {
Kevin O'Connore77c7052012-05-28 22:06:42 -0400535 void *stackpos;
Kevin O'Connor0039a942013-06-08 21:50:15 -0400536 struct hlist_node node;
Kevin O'Connore77c7052012-05-28 22:06:42 -0400537};
Kevin O'Connor89a2f962013-02-18 23:36:03 -0500538struct thread_info MainThread VARFSEG = {
Kevin O'Connor0039a942013-06-08 21:50:15 -0400539 NULL, { &MainThread.node, &MainThread.node.next }
Kevin O'Connore77c7052012-05-28 22:06:42 -0400540};
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500541#define THREADSTACKSIZE 4096
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500542
Kevin O'Connor2a8633f2013-06-08 22:11:07 -0400543// Check if any threads are running.
544static int
545have_threads(void)
546{
Kevin O'Connor0039a942013-06-08 21:50:15 -0400547 return (CONFIG_THREADS
548 && GET_FLATPTR(MainThread.node.next) != &MainThread.node);
Kevin O'Connor2a8633f2013-06-08 22:11:07 -0400549}
550
Kevin O'Connorad901592009-12-13 11:25:25 -0500551// Return the 'struct thread_info' for the currently running thread.
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500552struct thread_info *
Kevin O'Connor1ca05b02010-01-03 17:43:37 -0500553getCurThread(void)
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500554{
555 u32 esp = getesp();
Kevin O'Connor7acaa3c2013-10-02 20:20:03 -0400556 if (esp <= MAIN_STACK_MAX)
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500557 return &MainThread;
558 return (void*)ALIGN_DOWN(esp, THREADSTACKSIZE);
559}
560
Kevin O'Connord29ce622014-04-07 12:15:34 -0400561static int ThreadControl;
562
563// Initialize the support for internal threads.
564void
565thread_init(void)
566{
567 if (! CONFIG_THREADS)
568 return;
569 ThreadControl = romfile_loadint("etc/threads", 1);
570}
571
572// Should hardware initialization threads run during optionrom execution.
573int
574threads_during_optionroms(void)
575{
Kevin O'Connordd7527c2014-05-24 09:37:57 -0400576 return CONFIG_THREADS && ThreadControl == 2 && in_post();
Kevin O'Connord29ce622014-04-07 12:15:34 -0400577}
578
Kevin O'Connorad901592009-12-13 11:25:25 -0500579// Switch to next thread stack.
580static void
581switch_next(struct thread_info *cur)
582{
Kevin O'Connor0039a942013-06-08 21:50:15 -0400583 struct thread_info *next = container_of(
584 cur->node.next, struct thread_info, node);
Kevin O'Connorfb214dc2009-12-20 13:11:17 -0500585 if (cur == next)
586 // Nothing to do.
587 return;
Kevin O'Connorad901592009-12-13 11:25:25 -0500588 asm volatile(
589 " pushl $1f\n" // store return pc
590 " pushl %%ebp\n" // backup %ebp
Kevin O'Connor0039a942013-06-08 21:50:15 -0400591 " movl %%esp, (%%eax)\n" // cur->stackpos = %esp
592 " movl (%%ecx), %%esp\n" // %esp = next->stackpos
Kevin O'Connorad901592009-12-13 11:25:25 -0500593 " popl %%ebp\n" // restore %ebp
594 " retl\n" // restore pc
595 "1:\n"
596 : "+a"(cur), "+c"(next)
597 :
598 : "ebx", "edx", "esi", "edi", "cc", "memory");
599}
600
Kevin O'Connor48367e22013-12-22 22:44:08 -0500601// Last thing called from a thread (called on MainThread stack).
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500602static void
603__end_thread(struct thread_info *old)
604{
Kevin O'Connor0039a942013-06-08 21:50:15 -0400605 hlist_del(&old->node);
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500606 dprintf(DEBUG_thread, "\\%08x/ End thread\n", (u32)old);
Kevin O'Connor48367e22013-12-22 22:44:08 -0500607 free(old);
Kevin O'Connor2a8633f2013-06-08 22:11:07 -0400608 if (!have_threads())
Kevin O'Connore438b0c2010-05-01 12:20:33 -0400609 dprintf(1, "All threads complete.\n");
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500610}
611
Kevin O'Connorad901592009-12-13 11:25:25 -0500612// Create a new thread and start executing 'func' in it.
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500613void
614run_thread(void (*func)(void*), void *data)
615{
Kevin O'Connor52a300f2009-12-26 23:32:57 -0500616 ASSERT32FLAT();
Kevin O'Connord29ce622014-04-07 12:15:34 -0400617 if (! CONFIG_THREADS || ! ThreadControl)
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500618 goto fail;
619 struct thread_info *thread;
620 thread = memalign_tmphigh(THREADSTACKSIZE, THREADSTACKSIZE);
621 if (!thread)
622 goto fail;
623
Kevin O'Connor48367e22013-12-22 22:44:08 -0500624 dprintf(DEBUG_thread, "/%08x\\ Start thread\n", (u32)thread);
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500625 thread->stackpos = (void*)thread + THREADSTACKSIZE;
626 struct thread_info *cur = getCurThread();
Kevin O'Connor0039a942013-06-08 21:50:15 -0400627 hlist_add_after(&thread->node, &cur->node);
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500628 asm volatile(
629 // Start thread
630 " pushl $1f\n" // store return pc
631 " pushl %%ebp\n" // backup %ebp
Kevin O'Connor0039a942013-06-08 21:50:15 -0400632 " movl %%esp, (%%edx)\n" // cur->stackpos = %esp
633 " movl (%%ebx), %%esp\n" // %esp = thread->stackpos
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500634 " calll *%%ecx\n" // Call func
635
636 // End thread
Kevin O'Connor48367e22013-12-22 22:44:08 -0500637 " movl %%ebx, %%eax\n" // %eax = thread
638 " movl 4(%%ebx), %%ebx\n" // %ebx = thread->node.next
639 " movl (%5), %%esp\n" // %esp = MainThread.stackpos
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500640 " calll %4\n" // call __end_thread(thread)
Kevin O'Connor48367e22013-12-22 22:44:08 -0500641 " movl -4(%%ebx), %%esp\n" // %esp = next->stackpos
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500642 " popl %%ebp\n" // restore %ebp
643 " retl\n" // restore pc
644 "1:\n"
645 : "+a"(data), "+c"(func), "+b"(thread), "+d"(cur)
Kevin O'Connor48367e22013-12-22 22:44:08 -0500646 : "m"(*(u8*)__end_thread), "m"(MainThread)
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500647 : "esi", "edi", "cc", "memory");
648 return;
649
650fail:
651 func(data);
652}
653
Kevin O'Connor3cf301f2013-06-08 22:05:19 -0400654
655/****************************************************************
656 * Thread helpers
657 ****************************************************************/
658
659// Low-level irq enable.
660void VISIBLE16
661check_irqs(void)
662{
Kevin O'Connorfabc1b52014-09-29 19:18:25 -0400663 if (need_hop_back()) {
Kevin O'Connor80568252014-09-29 19:39:31 -0400664 extern void _cfunc16_check_irqs(void);
665 stack_hop_back(0, 0, _cfunc16_check_irqs);
Kevin O'Connorbfac2b52013-09-29 10:48:24 -0400666 return;
667 }
Kevin O'Connor3cf301f2013-06-08 22:05:19 -0400668 asm volatile("sti ; nop ; rep ; nop ; cli ; cld" : : :"memory");
669}
670
671// Briefly permit irqs to occur.
672void
673yield(void)
674{
Kevin O'Connor80568252014-09-29 19:39:31 -0400675 if (MODESEGMENT || !CONFIG_THREADS) {
Kevin O'Connorbfac2b52013-09-29 10:48:24 -0400676 check_irqs();
Kevin O'Connor3cf301f2013-06-08 22:05:19 -0400677 return;
678 }
Kevin O'Connor3cf301f2013-06-08 22:05:19 -0400679 struct thread_info *cur = getCurThread();
680 if (cur == &MainThread)
681 // Permit irqs to fire
Kevin O'Connor80568252014-09-29 19:39:31 -0400682 check_irqs();
Kevin O'Connor3cf301f2013-06-08 22:05:19 -0400683
684 // Switch to the next thread
685 switch_next(cur);
686}
687
688void VISIBLE16
689wait_irq(void)
690{
Kevin O'Connorfabc1b52014-09-29 19:18:25 -0400691 if (need_hop_back()) {
Kevin O'Connor80568252014-09-29 19:39:31 -0400692 extern void _cfunc16_wait_irq(void);
693 stack_hop_back(0, 0, _cfunc16_wait_irq);
Kevin O'Connorbfac2b52013-09-29 10:48:24 -0400694 return;
695 }
Kevin O'Connor3cf301f2013-06-08 22:05:19 -0400696 asm volatile("sti ; hlt ; cli ; cld": : :"memory");
697}
698
699// Wait for next irq to occur.
700void
701yield_toirq(void)
702{
Kevin O'Connor80568252014-09-29 19:39:31 -0400703 if (!MODESEGMENT && have_threads()) {
Kevin O'Connor3cf301f2013-06-08 22:05:19 -0400704 // Threads still active - do a yield instead.
705 yield();
706 return;
707 }
Kevin O'Connor80568252014-09-29 19:39:31 -0400708 wait_irq();
Kevin O'Connor3cf301f2013-06-08 22:05:19 -0400709}
710
Kevin O'Connorad901592009-12-13 11:25:25 -0500711// Wait for all threads (other than the main thread) to complete.
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500712void
Kevin O'Connor1ca05b02010-01-03 17:43:37 -0500713wait_threads(void)
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500714{
Kevin O'Connor52a300f2009-12-26 23:32:57 -0500715 ASSERT32FLAT();
Kevin O'Connor2a8633f2013-06-08 22:11:07 -0400716 while (have_threads())
Kevin O'Connor7cefbfa2009-12-10 21:35:49 -0500717 yield();
718}
Kevin O'Connorad901592009-12-13 11:25:25 -0500719
Kevin O'Connore9086652010-02-27 13:49:47 -0500720void
721mutex_lock(struct mutex_s *mutex)
722{
723 ASSERT32FLAT();
724 if (! CONFIG_THREADS)
725 return;
726 while (mutex->isLocked)
727 yield();
728 mutex->isLocked = 1;
729}
730
731void
732mutex_unlock(struct mutex_s *mutex)
733{
734 ASSERT32FLAT();
735 if (! CONFIG_THREADS)
736 return;
737 mutex->isLocked = 0;
738}
739
Kevin O'Connorad901592009-12-13 11:25:25 -0500740
741/****************************************************************
742 * Thread preemption
743 ****************************************************************/
744
Kevin O'Connor89a2f962013-02-18 23:36:03 -0500745int CanPreempt VARFSEG;
Kevin O'Connorad901592009-12-13 11:25:25 -0500746static u32 PreemptCount;
747
748// Turn on RTC irqs and arrange for them to check the 32bit threads.
749void
Kevin O'Connor1ca05b02010-01-03 17:43:37 -0500750start_preempt(void)
Kevin O'Connorad901592009-12-13 11:25:25 -0500751{
Kevin O'Connord29ce622014-04-07 12:15:34 -0400752 if (! threads_during_optionroms())
Kevin O'Connorad901592009-12-13 11:25:25 -0500753 return;
754 CanPreempt = 1;
755 PreemptCount = 0;
Kevin O'Connor8b7861c2013-09-15 02:29:06 -0400756 rtc_use();
Kevin O'Connorad901592009-12-13 11:25:25 -0500757}
758
759// Turn off RTC irqs / stop checking for thread execution.
760void
Kevin O'Connor1ca05b02010-01-03 17:43:37 -0500761finish_preempt(void)
Kevin O'Connorad901592009-12-13 11:25:25 -0500762{
Kevin O'Connord29ce622014-04-07 12:15:34 -0400763 if (! threads_during_optionroms()) {
Kevin O'Connora7eb8fc2010-04-02 13:13:23 -0400764 yield();
Kevin O'Connorad901592009-12-13 11:25:25 -0500765 return;
Kevin O'Connora7eb8fc2010-04-02 13:13:23 -0400766 }
Kevin O'Connorad901592009-12-13 11:25:25 -0500767 CanPreempt = 0;
Kevin O'Connor8b7861c2013-09-15 02:29:06 -0400768 rtc_release();
Kevin O'Connore438b0c2010-05-01 12:20:33 -0400769 dprintf(9, "Done preempt - %d checks\n", PreemptCount);
Kevin O'Connora7eb8fc2010-04-02 13:13:23 -0400770 yield();
Kevin O'Connorad901592009-12-13 11:25:25 -0500771}
772
Kevin O'Connord7eb27e2010-03-20 18:17:19 -0400773// Check if preemption is on, and wait for it to complete if so.
774int
775wait_preempt(void)
776{
Kevin O'Connord29ce622014-04-07 12:15:34 -0400777 if (MODESEGMENT || !CONFIG_THREADS || !CanPreempt
Kevin O'Connor7acaa3c2013-10-02 20:20:03 -0400778 || getesp() < MAIN_STACK_MAX)
Kevin O'Connord7eb27e2010-03-20 18:17:19 -0400779 return 0;
780 while (CanPreempt)
781 yield();
782 return 1;
783}
784
Kevin O'Connorad901592009-12-13 11:25:25 -0500785// Try to execute 32bit threads.
Kevin O'Connord1b4f962010-09-15 21:38:16 -0400786void VISIBLE32INIT
Kevin O'Connor1ca05b02010-01-03 17:43:37 -0500787yield_preempt(void)
Kevin O'Connorad901592009-12-13 11:25:25 -0500788{
789 PreemptCount++;
790 switch_next(&MainThread);
791}
Kevin O'Connorad901592009-12-13 11:25:25 -0500792
793// 16bit code that checks if threads are pending and executes them if so.
794void
Kevin O'Connor1ca05b02010-01-03 17:43:37 -0500795check_preempt(void)
Kevin O'Connorad901592009-12-13 11:25:25 -0500796{
Kevin O'Connorf3fe3aa2010-12-05 12:38:33 -0500797 extern void _cfunc32flat_yield_preempt(void);
Kevin O'Connord29ce622014-04-07 12:15:34 -0400798 if (CONFIG_THREADS && GET_GLOBAL(CanPreempt) && have_threads())
Kevin O'Connor2a8633f2013-06-08 22:11:07 -0400799 call32(_cfunc32flat_yield_preempt, 0, 0);
Kevin O'Connorad901592009-12-13 11:25:25 -0500800}
Kevin O'Connor7f54bf72013-12-30 22:04:28 -0500801
802
803/****************************************************************
804 * call32 helper
805 ****************************************************************/
806
807struct call32_params_s {
808 void *func;
809 u32 eax, edx, ecx;
810};
811
812u32 VISIBLE32FLAT
813call32_params_helper(struct call32_params_s *params)
814{
815 return ((u32 (*)(u32, u32, u32))params->func)(
816 params->eax, params->edx, params->ecx);
817}
818
819u32
820call32_params(void *func, u32 eax, u32 edx, u32 ecx, u32 errret)
821{
822 ASSERT16();
823 struct call32_params_s params = {func, eax, edx, ecx};
824 extern void _cfunc32flat_call32_params_helper(void);
825 return call32(_cfunc32flat_call32_params_helper
826 , (u32)MAKE_FLATPTR(GET_SEG(SS), &params), errret);
827}