src/cpu/x86/pae/pgtbl.c

/* SPDX-License-Identifier: GPL-2.0-only */

#include <cbfs.h>
#include <commonlib/helpers.h>
#include <console/console.h>
#include <cpu/cpu.h>
#include <cpu/x86/cr.h>
#include <cpu/x86/msr.h>
#include <cpu/x86/pae.h>
#include <string.h>
#include <symbols.h>
#include <types.h>

#define PDPTE_PRES (1ULL << 0)
#define PDPTE_ADDR_MASK (~((1ULL << 12) - 1))

#define PDE_PRES (1ULL << 0)
#define PDE_RW   (1ULL << 1)
#define PDE_US   (1ULL << 2)
#define PDE_PWT  (1ULL << 3)
#define PDE_PCD  (1ULL << 4)
#define PDE_A    (1ULL << 5)
#define PDE_D    (1ULL << 6) // only valid with PS=1
#define PDE_PS   (1ULL << 7)
#define PDE_G    (1ULL << 8)  // only valid with PS=1
#define PDE_PAT  (1ULL << 12) // only valid with PS=1
#define PDE_XD   (1ULL << 63)
#define PDE_ADDR_MASK (~((1ULL << 12) - 1))

#define PTE_PRES (1ULL << 0)
#define PTE_RW   (1ULL << 1)
#define PTE_US   (1ULL << 2)
#define PTE_PWT  (1ULL << 3)
#define PTE_PCD  (1ULL << 4)
#define PTE_A    (1ULL << 5)
#define PTE_D    (1ULL << 6)
#define PTE_PAT  (1ULL << 7)
#define PTE_G    (1ULL << 8)
#define PTE_XD   (1ULL << 63)

#define PDPTE_IDX_SHIFT 30
#define PDPTE_IDX_MASK  0x3

#define PDE_IDX_SHIFT 21
#define PDE_IDX_MASK  0x1ff

#define PTE_IDX_SHIFT 12
#define PTE_IDX_MASK  0x1ff

#define OVERLAP(a, b, s, e) ((b) > (s) && (a) < (e))

static const size_t s2MiB = 2 * MiB;
static const size_t s4KiB = 4 * KiB;

struct pde {
	uint32_t addr_lo;
	uint32_t addr_hi;
} __packed;
struct pg_table {
	struct pde pd[2048];
	struct pde pdp[512];
} __packed;

void paging_enable_pae_cr3(uintptr_t cr3)
{
	/* Load the page table address */
	write_cr3(cr3);
	paging_enable_pae();
}

void paging_enable_pae(void)
{
	CRx_TYPE cr0;
	CRx_TYPE cr4;

	/* Enable PAE */
	cr4 = read_cr4();
	cr4 |= CR4_PAE;
	write_cr4(cr4);

	/* Enable Paging */
	cr0 = read_cr0();
	cr0 |= CR0_PG;
	write_cr0(cr0);
}

void paging_disable_pae(void)
{
	CRx_TYPE cr0;
	CRx_TYPE cr4;

	/* Disable Paging */
	cr0 = read_cr0();
	cr0 &= ~(CRx_TYPE)CR0_PG;
	write_cr0(cr0);

	/* Disable PAE */
	cr4 = read_cr4();
	cr4 &= ~(CRx_TYPE)CR4_PAE;
	write_cr4(cr4);
}

/*
 * Use PAE to map a page and then memset it with the pattern specified.
 * In order to use PAE pagetables for virtual addressing are set up and reloaded
 * on a 2MiB boundary. After the function is done, virtual addressing mode is
 * disabled again. The PAT are set to all cacheable, but MTRRs still apply.
 *
 * Requires a scratch memory for pagetables and a virtual address for
 * non identity mapped memory.
 *
 * The scratch memory area containing pagetables must not overlap with the
 * memory range to be cleared.
 * The scratch memory area containing pagetables must not overlap with the
 * virtual address for non identity mapped memory.
 *
 * @param vmem_addr Where the virtual non identity mapped page resides, must
 *                  be 2 aligned MiB and at least 2 MiB in size.
 *                  Content at physical address is preserved.
 * @param pgtbl     Where pagetables reside, must be 4 KiB aligned and 20 KiB in
 *                  size.
 *                  Must not overlap memory range pointed to by dest.
 *                  Must not overlap memory range pointed to by vmem_addr.
 *                  Content at physical address isn't preserved.
 * @param length    The length of the memory segment to memset
 * @param dest      Physical memory address to memset
 * @param pat       The pattern to write to the physical memory
 * @return 0 on success, 1 on error
 */
int memset_pae(uint64_t dest, unsigned char pat, uint64_t length, void *pgtbl,
	       void *vmem_addr)
{
	struct pg_table *pgtbl_buf = (struct pg_table *)pgtbl;
	ssize_t offset;

	printk(BIOS_DEBUG, "%s: Using virtual address %p as scratchpad\n",
	       __func__, vmem_addr);
	printk(BIOS_DEBUG, "%s: Using address %p for page tables\n",
	       __func__, pgtbl_buf);

	/* Cover some basic error conditions */
	if (!IS_ALIGNED((uintptr_t)pgtbl_buf, s4KiB) ||
	    !IS_ALIGNED((uintptr_t)vmem_addr, s2MiB)) {
		printk(BIOS_ERR, "%s: Invalid alignment\n", __func__);
		return 1;
	}
	const uintptr_t pgtbl_s = (uintptr_t)pgtbl_buf;
	const uintptr_t pgtbl_e = pgtbl_s + sizeof(struct pg_table);

	if (OVERLAP(dest, dest + length, pgtbl_s, pgtbl_e)) {
		printk(BIOS_ERR, "%s: destination overlaps page tables\n",
		       __func__);
		return 1;
	}

	if (OVERLAP((uintptr_t)vmem_addr, (uintptr_t)vmem_addr + s2MiB,
		    pgtbl_s, pgtbl_e)) {
		printk(BIOS_ERR, "%s: vmem address overlaps page tables\n",
		       __func__);
		return 1;
	}

	paging_disable_pae();

	struct pde *pd = pgtbl_buf->pd, *pdp = pgtbl_buf->pdp;
	/* Point the page directory pointers at the page directories. */
	memset(pgtbl_buf->pdp, 0, sizeof(pgtbl_buf->pdp));

	pdp[0].addr_lo = ((uintptr_t)&pd[512*0]) | PDPTE_PRES;
	pdp[1].addr_lo = ((uintptr_t)&pd[512*1]) | PDPTE_PRES;
	pdp[2].addr_lo = ((uintptr_t)&pd[512*2]) | PDPTE_PRES;
	pdp[3].addr_lo = ((uintptr_t)&pd[512*3]) | PDPTE_PRES;

	offset = dest - ALIGN_DOWN(dest, s2MiB);
	dest = ALIGN_DOWN(dest, s2MiB);

	/* Identity map the whole 32-bit address space */
	for (size_t i = 0; i < 2048; i++) {
		pd[i].addr_lo = (i << PDE_IDX_SHIFT) | PDE_PS | PDE_PRES | PDE_RW;
		pd[i].addr_hi = 0;
	}

	/* Get pointer to PD that's not identity mapped */
	pd = &pgtbl_buf->pd[((uintptr_t)vmem_addr) >> PDE_IDX_SHIFT];

	paging_enable_pae_cr3((uintptr_t)pdp);

	do {
		const size_t len = MIN(length, s2MiB - offset);

		/*
		 * Map a page using PAE at virtual address vmem_addr.
		 * dest is already 2 MiB aligned.
		 */
		pd->addr_lo = dest | PDE_PS | PDE_PRES | PDE_RW;
		pd->addr_hi = dest >> 32;

		/* Update page tables */
		asm volatile ("invlpg (%0)" :: "b"(vmem_addr) : "memory");

		printk(BIOS_SPEW, "%s: Clearing %llx[%lx] - %zx\n", __func__,
		       dest + offset, (uintptr_t)vmem_addr + offset, len);

		memset(vmem_addr + offset, pat, len);

		dest += s2MiB;
		length -= len;
		offset = 0;
	} while (length > 0);

	paging_disable_pae();

	return 0;
}

void paging_set_nxe(int enable)
{
	msr_t msr = rdmsr(IA32_EFER);

	if (enable)
		msr.lo |= EFER_NXE;
	else
		msr.lo &= ~EFER_NXE;

	wrmsr(IA32_EFER, msr);
}

void paging_set_pat(uint64_t pat)
{
	msr_t msr;
	msr.lo = pat;
	msr.hi = pat >> 32;
	wrmsr(IA32_PAT, msr);
}

/* PAT encoding used in util/x86/x86_page_tables.go. It matches the linux
 * kernel settings:
 *  PTE encoding:
 *      PAT
 *      |PCD
 *      ||PWT  PAT
 *      |||    slot
 *      000    0    WB : _PAGE_CACHE_MODE_WB
 *      001    1    WC : _PAGE_CACHE_MODE_WC
 *      010    2    UC-: _PAGE_CACHE_MODE_UC_MINUS
 *      011    3    UC : _PAGE_CACHE_MODE_UC
 *      100    4    WB : Reserved
 *      101    5    WP : _PAGE_CACHE_MODE_WP
 *      110    6    UC-: Reserved
 *      111    7    WT : _PAGE_CACHE_MODE_WT
 */
void paging_set_default_pat(void)
{
	uint64_t pat =  PAT_ENCODE(WB, 0) | PAT_ENCODE(WC, 1) |
			PAT_ENCODE(UC_MINUS, 2) | PAT_ENCODE(UC, 3) |
			PAT_ENCODE(WB, 4) | PAT_ENCODE(WP, 5) |
			PAT_ENCODE(UC_MINUS, 6) | PAT_ENCODE(WT, 7);
	paging_set_pat(pat);
}

int paging_enable_for_car(const char *pdpt_name, const char *pt_name)
{
	if (!preram_symbols_available())
		return -1;

	if (!cbfs_load(pdpt_name, _pdpt, REGION_SIZE(pdpt))) {
		printk(BIOS_ERR, "Couldn't load pdpt\n");
		return -1;
	}

	if (!cbfs_load(pt_name, _pagetables, REGION_SIZE(pagetables))) {
		printk(BIOS_ERR, "Couldn't load page tables\n");
		return -1;
	}

	paging_enable_pae_cr3((uintptr_t)_pdpt);

	return 0;
}