mb/emulation/qemu: Copy page tables to DRAM in assembly
To work around various bugs running KVM enabled, copy page tables to
DRAM in assembly before jumping to x86_64 mode.
Tested on QEMU using KVM, no more stange bugs happen:
Tested on host
- CPU Intel(R) Core(TM) i7-7700HQ
- Linux 5.9
- qemu 4.2.1
Used to crash on emulating MMX instructions and failed to translate
some addresses using the virtual MMU when running in long mode.
Tested on host
- CPU AMD EPYC 7401P 24-Core Processor
- Linux 5.4
- qemu 4.2.1
Used to crash on jumping to long mode.
Change-Id: Ic0bdd2bef7197edd2e7488a8efdeba7eb4ab0dd4
Signed-off-by: Patrick Rudolph <patrick.rudolph@9elements.com>
Reviewed-on: https://review.coreboot.org/c/coreboot/+/49228
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: Angel Pons <th3fanbus@gmail.com>
Reviewed-by: Arthur Heymans <arthur@aheymans.xyz>
diff --git a/src/cpu/qemu-x86/Makefile.inc b/src/cpu/qemu-x86/Makefile.inc
index fb560d6..3f27e8b 100644
--- a/src/cpu/qemu-x86/Makefile.inc
+++ b/src/cpu/qemu-x86/Makefile.inc
@@ -2,6 +2,7 @@
bootblock-y += cache_as_ram_bootblock.S
bootblock-y += bootblock.c
+bootblock-$(CONFIG_ARCH_BOOTBLOCK_X86_64) += $(top)/src/arch/x86/walkcbfs.S
romstage-y += ../intel/car/romstage.c
diff --git a/src/cpu/qemu-x86/cache_as_ram_bootblock.S b/src/cpu/qemu-x86/cache_as_ram_bootblock.S
index e3a26b0..07f848a 100644
--- a/src/cpu/qemu-x86/cache_as_ram_bootblock.S
+++ b/src/cpu/qemu-x86/cache_as_ram_bootblock.S
@@ -2,7 +2,14 @@
#include <cpu/x86/post_code.h>
+#define CBFS_FILE_MAGIC 0
+#define CBFS_FILE_LEN (CBFS_FILE_MAGIC + 8)
+#define CBFS_FILE_TYPE (CBFS_FILE_LEN + 4)
+#define CBFS_FILE_CHECKSUM (CBFS_FILE_TYPE + 4)
+#define CBFS_FILE_OFFSET (CBFS_FILE_CHECKSUM + 4)
+
.section .init, "ax", @progbits
+.code32
.global bootblock_pre_c_entry
bootblock_pre_c_entry:
@@ -24,12 +31,55 @@
post_code(0x21)
+#if defined(__x86_64__)
+ /*
+ * Copy page tables to final location in DRAM. This prevents some strange
+ * bugs when running KVM enabled:
+ * Accessing MMX instructions in long mode causes an abort
+ * Some physical addresses aren't properly translated
+ * Emulation fault on every instruction fetched due to page tables in ROM
+ * Enabling or disabling paging causes a fault
+ *
+ * First, find page tables in CBFS:
+ */
+ lea pagetables_name, %esi
+ mov $1f, %esp
+ jmp walkcbfs_asm
+1:
+ cmpl $0, %eax
+ je .Lhlt
+
+ /* Test if page tables are memory-mapped and skip relocation */
+ cmpl $(CONFIG_ARCH_X86_64_PGTBL_LOC), %eax
+ je pages_done
+
+ movl CBFS_FILE_OFFSET(%eax), %ebx
+ bswap %ebx
+ addl %eax, %ebx
+ movl %ebx, %esi
+
+ movl CBFS_FILE_LEN(%eax), %ecx
+ bswap %ecx
+ shr $2, %ecx
+
+ movl $(CONFIG_ARCH_X86_64_PGTBL_LOC), %edi
+
+loop:
+ movl (%esi), %eax
+ movl %eax, (%edi)
+ addl $4, %esi
+ addl $4, %edi
+ decl %ecx
+ jnz loop
+pages_done:
+#endif
+
movl $_ecar_stack, %esp
/* Align the stack and keep aligned for call to bootblock_c_entry() */
and $0xfffffff0, %esp
- /* entry64.inc preserves ebx. */
+ /* entry64.inc preserves ebx. */
#include <cpu/x86/64bit/entry64.inc>
/* Restore the BIST result and timestamps. */
@@ -60,3 +110,6 @@
post_code(POST_DEAD_CODE)
hlt
jmp .Lhlt
+
+pagetables_name:
+ .string "pagetables"
diff --git a/src/mainboard/emulation/qemu-i440fx/Kconfig b/src/mainboard/emulation/qemu-i440fx/Kconfig
index 62c23ca..1b61871 100644
--- a/src/mainboard/emulation/qemu-i440fx/Kconfig
+++ b/src/mainboard/emulation/qemu-i440fx/Kconfig
@@ -27,6 +27,15 @@
select GBB_FLAG_DISABLE_EC_SOFTWARE_SYNC
select GBB_FLAG_DISABLE_FWMP
+if ARCH_BOOTBLOCK_X86_64
+# Need to install page tables in DRAM as the virtual MMU has problems translating paging
+# request when the page table resides in emulated ROM. This causes undefined behaviour
+# when handling data requests, as well as fetching and decoding instructions
+# Real hardware didn't show any problems until now.
+config ARCH_X86_64_PGTBL_LOC
+ default 0x8000
+endif
+
if VBOOT
config VBOOT_SLOTS_RW_A
diff --git a/src/mainboard/emulation/qemu-i440fx/mainboard.c b/src/mainboard/emulation/qemu-i440fx/mainboard.c
index 929743a..bc97554 100644
--- a/src/mainboard/emulation/qemu-i440fx/mainboard.c
+++ b/src/mainboard/emulation/qemu-i440fx/mainboard.c
@@ -28,8 +28,19 @@
pci_assign_irqs(pcidev_on_root(i, 0), qemu_i440fx_irqs + (i % 4));
}
+static void qemu_nb_read_resources(struct device *dev)
+{
+ pci_dev_read_resources(dev);
+
+ if (CONFIG(ARCH_RAMSTAGE_X86_64)) {
+ /* Reserve page tables in DRAM. FIXME: Remove once x86_64 page tables reside in CBMEM */
+ reserved_ram_resource(dev, 0, CONFIG_ARCH_X86_64_PGTBL_LOC / KiB,
+ (6 * 0x1000) / KiB);
+ }
+}
+
static struct device_operations nb_operations = {
- .read_resources = pci_dev_read_resources,
+ .read_resources = qemu_nb_read_resources,
.set_resources = pci_dev_set_resources,
.enable_resources = pci_dev_enable_resources,
.init = qemu_nb_init,
diff --git a/src/mainboard/emulation/qemu-q35/Kconfig b/src/mainboard/emulation/qemu-q35/Kconfig
index edd2b2c..d88d0da 100644
--- a/src/mainboard/emulation/qemu-q35/Kconfig
+++ b/src/mainboard/emulation/qemu-q35/Kconfig
@@ -32,6 +32,15 @@
default "src/mainboard/\$(CONFIG_MAINBOARD_DIR)/vboot-rwa-16M.fmd" if VBOOT_SLOTS_RW_A && !VBOOT_SLOTS_RW_AB
default "src/mainboard/\$(CONFIG_MAINBOARD_DIR)/vboot-rwab-16M.fmd" if VBOOT_SLOTS_RW_AB
+if ARCH_BOOTBLOCK_X86_64
+# Need to install page tables in DRAM as the virtual MMU has problems translating paging
+# request when the page table resides in emulated ROM. This causes undefined behaviour
+# when handling data requests, as well as fetching and decoding instructions
+# Real hardware didn't show any problems until now.
+config ARCH_X86_64_PGTBL_LOC
+ default 0x8000
+endif
+
if VBOOT
config VBOOT_SLOTS_RW_A
diff --git a/src/mainboard/emulation/qemu-q35/mainboard.c b/src/mainboard/emulation/qemu-q35/mainboard.c
index d329e5a..c88874f 100644
--- a/src/mainboard/emulation/qemu-q35/mainboard.c
+++ b/src/mainboard/emulation/qemu-q35/mainboard.c
@@ -44,6 +44,12 @@
/* reserve mmconfig */
fixed_mem_resource(dev, 2, CONFIG_MMCONF_BASE_ADDRESS >> 10, 0x10000000 >> 10,
IORESOURCE_RESERVE);
+
+ if (CONFIG(ARCH_RAMSTAGE_X86_64)) {
+ /* Reserve page tables in DRAM. FIXME: Remove once x86_64 page tables reside in CBMEM */
+ reserved_ram_resource(dev, 0, CONFIG_ARCH_X86_64_PGTBL_LOC / KiB,
+ (6 * 0x1000) / KiB);
+ }
}