[boot] Set up initial page tables

Set up initial page tables for both the offset-mapped area and the
loaded kernel code and data.

* Got rid of the `loaded_elf` struct - the loader now runs after the
  initial PML4 is created and maps the ELF sections itself.
* Copied in the `page_table` and `page_table_indices` from the kernel,
  still need to clean this up and extract it into shared code.
* Added `page_table_cache` to the kernel args to pass along free pages
  that can be used for initial page tables.

Tags: paging

modules.yaml

@@ -62,6 +62,7 @@ modules:
             - src/boot/hardware.cpp
             - src/boot/loader.cpp
             - src/boot/memory.cpp
+            - src/boot/paging.cpp
             - src/boot/support.cpp
 
     nulldrv:

src/boot/loader.cpp

@@ -6,6 +6,7 @@
 #include "elf.h"
 #include "error.h"
 #include "memory.h"
+#include "paging.h"
 
 namespace boot {
 namespace loader {
@@ -25,10 +26,19 @@ is_elfheader_valid(const elf::header *header)
 		header->header_version == elf::version;
 }
 
-loaded_elf
+static void
+map_pages(
+	paging::page_table *pml4,
+	kernel::args::header *args,
+	uintptr_t phys, uintptr_t virt,
+	size_t bytes)
+{
+}
+
+kernel::entrypoint
 load(
-	const void *data,
-	size_t size,
+	const void *data, size_t size,
+	kernel::args::header *args,
 	uefi::boot_services *bs)
 {
 	status_line status(L"Loading kernel ELF binary");
@@ -37,29 +47,7 @@ load(
 	if (size < sizeof(elf::header) || !is_elfheader_valid(header))
 		error::raise(uefi::status::load_error, L"Kernel ELF not valid");
 
-	uintptr_t kernel_start = 0;
-	uintptr_t kernel_end = 0;
-	for (int i = 0; i < header->ph_num; ++i) {
-		ptrdiff_t offset = header->ph_offset + i * header->ph_entsize;
-		const elf::program_header *pheader =
-			offset_ptr<elf::program_header>(data, offset);
-
-		if (pheader->type != elf::PT_LOAD)
-			continue;
-
-		if (kernel_start == 0 || pheader->vaddr < kernel_start)
-			kernel_start = pheader->vaddr;
-
-		if (pheader->vaddr + pheader->mem_size > kernel_end)
-			kernel_end = pheader->vaddr + pheader->mem_size;
-	}
-
-	void *pages = nullptr;
-	size_t num_pages = memory::bytes_to_pages(kernel_end - kernel_start);
-	try_or_raise(
-		bs->allocate_pages(uefi::allocate_type::any_pages,
-			memory::kernel_type, num_pages, &pages),
-		L"Failed allocating space for kernel code");
+	paging::page_table *pml4 = reinterpret_cast<paging::page_table*>(args->pml4);
 
 	for (int i = 0; i < header->ph_num; ++i) {
 		ptrdiff_t offset = header->ph_offset + i * header->ph_entsize;
@@ -69,20 +57,26 @@ load(
 		if (pheader->type != elf::PT_LOAD)
 			continue;
 
+		size_t num_pages = memory::bytes_to_pages(pheader->mem_size);
+		void *pages = nullptr;
+
+		try_or_raise(
+			bs->allocate_pages(uefi::allocate_type::any_pages,
+				memory::kernel_type, num_pages, &pages),
+			L"Failed allocating space for kernel code");
+
 		void *data_start = offset_ptr<void>(data, pheader->offset);
-		void *program_start = offset_ptr<void>(pages, pheader->vaddr - kernel_start);
-		bs->copy_mem(program_start, data_start, pheader->mem_size);
+		bs->copy_mem(pages, data_start, pheader->mem_size);
+
+		console::print(L"          Kernel section %d physical addr: 0x%lx\r\n", i, pages);
+		console::print(L"          Kernel section %d virtual addr:  0x%lx\r\n", i, pheader->vaddr);
+
+		// TODO: map these pages into kernel args' page tables
+		// remember to set appropriate RWX permissions
+		map_pages(pml4, args, reinterpret_cast<uintptr_t>(pages), pheader->vaddr, pheader->mem_size);
 	}
 
-	loaded_elf result;
-	result.data = pages;
-	result.vaddr = kernel_start;
-	result.entrypoint = header->entrypoint;
-	console::print(L"          Kernel loaded at: 0x%lx\r\n", result.data);
-	console::print(L"    Kernel virtual address: 0x%lx\r\n", result.vaddr);
-	console::print(L"         Kernel entrypoint: 0x%lx\r\n", result.entrypoint);
-
-	return result;
+	return reinterpret_cast<kernel::entrypoint>(header->entrypoint);
 }
 
 } // namespace loader

src/boot/loader.h

@@ -2,22 +2,22 @@
 /// Definitions for loading the kernel into memory
 #pragma once
 
+#include <uefi/boot_services.h>
+
+#include "kernel_args.h"
+
 namespace boot {
 namespace loader {
 
-/// Structure to hold information about loaded binary image.
-struct loaded_elf
-{
-	void *data;           ///< Start of the kernel in memory
-	uintptr_t vaddr;      ///< Virtual address to map to
-	uintptr_t entrypoint; ///< (Virtual) address of the kernel entrypoint
-};
-
 /// Parse and load an ELF file in memory into a loaded image.
 /// \arg data  The start of the ELF file in memory
 /// \arg size  The size of the ELF file in memory
-/// \returns   A `loaded_elf` structure defining the loaded image
-loaded_elf load(const void *data, size_t size, uefi::boot_services *bs);
+/// \arg args  The kernel args, used for modifying page tables
+/// \returns   A descriptor defining the loaded image
+kernel::entrypoint load(
+	const void *data, size_t size,
+	kernel::args::header *args,
+	uefi::boot_services *bs);
 
 } // namespace loader
 } // namespace boot

src/boot/main.cpp

@@ -13,6 +13,7 @@
 #include "hardware.h"
 #include "loader.h"
 #include "memory.h"
+#include "paging.h"
 
 #include "kernel_args.h"
 
@@ -136,7 +137,7 @@ load_module(
 /// The main procedure for the portion of the loader that runs while
 /// UEFI is still in control of the machine. (ie, while the loader still
 /// has access to boot services.
-loader::loaded_elf
+kernel::entrypoint
 bootloader_main_uefi(uefi::handle image, uefi::system_table *st, console &con, size_t *map_key)
 {
 	error::uefi_handler handler(con);
@@ -160,12 +161,14 @@ bootloader_main_uefi(uefi::handle image, uefi::system_table *st, console &con, s
 	kernel::args::module *kernel =
 		load_module(disk, args, L"kernel", L"jsix.elf", kernel::args::mod_type::kernel);
 
+	paging::allocate_tables(args, bs);
 
-	loader::loaded_elf kernel_elf =
-		loader::load(kernel->location, kernel->size, bs);
+	kernel::entrypoint kentry =
+		loader::load(kernel->location, kernel->size, args, bs);
 
 	*map_key = memory::build_kernel_mem_map(args, bs);
-	return kernel_elf;
+
+	return kentry;
 }
 
 } // namespace boot
@@ -180,7 +183,7 @@ efi_main(uefi::handle image_handle, uefi::system_table *st)
 	console con(st->boot_services, st->con_out);
 
 	size_t map_key;
-	loader::loaded_elf kernel =
+	kernel::entrypoint kentry =
 		bootloader_main_uefi(image_handle, st, con, &map_key);
 
 	try_or_raise(

src/boot/memory.cpp

@@ -13,7 +13,6 @@ using mem_type = kernel::args::mem_type;
 
 size_t fixup_pointer_index = 0;
 void **fixup_pointers[64];
-uint64_t *new_pml4 = 0;
 
 static const wchar_t *memory_type_names[] = {
 	L"reserved memory type",
@@ -78,19 +77,6 @@ init_pointer_fixup(uefi::boot_services *bs, uefi::runtime_services *rs)
 			rs,
 			&event),
 		L"Error creating memory virtualization event");
-
-	// Reserve a page for our replacement PML4, plus some pages for the kernel to use
-	// as page tables while it gets started.
-	void *addr = nullptr;
-	try_or_raise(
-		bs->allocate_pages(
-			uefi::allocate_type::any_pages,
-			table_type,
-			64,
-			&addr),
-		L"Error allocating page table pages.");
-
-	new_pml4 = reinterpret_cast<uint64_t*>(addr);
 }
 
 void

src/boot/paging.cpp

@@ -0,0 +1,127 @@
+#include "console.h"
+#include "error.h"
+#include "loader.h"
+#include "memory.h"
+#include "paging.h"
+#include "pointer_manipulation.h"
+
+namespace boot {
+namespace paging {
+
+using memory::page_size;
+
+void allocate_tables(kernel::args::header *args, uefi::boot_services *bs)
+{
+	status_line status(L"Allocating initial page tables");
+
+	static constexpr size_t offset_map_tables = 128 + 1;
+	static constexpr size_t tables_needed = offset_map_tables + 49;
+
+	void *addr = nullptr;
+	try_or_raise(
+		bs->allocate_pages(
+			uefi::allocate_type::any_pages,
+			memory::table_type,
+			tables_needed,
+			&addr),
+		L"Error allocating page table pages.");
+
+	bs->set_mem(addr, tables_needed*page_size, 0);
+
+	kernel::args::module &mod = args->modules[++args->num_modules];
+	mod.type = kernel::args::mod_type::page_tables;
+	mod.location = addr;
+	mod.size = tables_needed*page_size;
+
+	args->pml4 = addr;
+	args->num_free_tables = tables_needed - offset_map_tables;
+	args->page_table_cache = offset_ptr<void>(addr, offset_map_tables*page_size);
+
+	page_table *tables = reinterpret_cast<page_table*>(addr);
+
+	// Create the PML4 pointing to the following tables
+	for (int i = 0; i < offset_map_tables - 1; ++i) {
+		tables[0].set(384 + i, &tables[i+1], 0x0003);
+
+		uint64_t start = i * 0x8000000000;
+		for (int j = 0; j < 512; ++j)
+		{
+			void *p = reinterpret_cast<void*>(start + (j * 0x40000000ull));
+			tables[i+1].set(j, p, 0x0183);
+		}
+	}
+}
+
+void
+check_needs_page(page_table *table, int idx, kernel::args::header *args)
+{
+	if (table->entries[idx] & 0x1)
+		return;
+
+	uintptr_t new_table =
+		reinterpret_cast<uintptr_t>(args->page_table_cache);
+	table->entries[idx] = new_table | 0x0003;
+
+	args->page_table_cache = offset_ptr<void>(args->page_table_cache, page_size);
+	args->num_free_tables--;
+}
+
+void
+map_in(
+	page_table *pml4,
+	kernel::args::header *args,
+	uintptr_t phys, uintptr_t virt,
+	size_t size)
+{
+	page_table_indices idx{virt};
+	page_table *tables[4] = {pml4, nullptr, nullptr, nullptr};
+
+	size_t pages = memory::bytes_to_pages(size);
+
+	for (; idx[0] < 512; idx[0] += 1, idx[1] = 0, idx[2] = 0, idx[3] = 0) {
+		check_needs_page(tables[0], idx[0], args);
+		tables[1] = tables[0]->get(idx[0]);
+
+		for (; idx[1] < 512; idx[1] += 1, idx[2] = 0, idx[3] = 0) {
+			check_needs_page(tables[1], idx[1], args);
+			tables[2] = tables[1]->get(idx[1]);
+
+			for (; idx[2] < 512; idx[2] += 1, idx[3] = 0) {
+				check_needs_page(tables[2], idx[2], args);
+				tables[3] = tables[2]->get(idx[2]);
+
+				for (; idx[3] < 512; idx[3] += 1) {
+					tables[3]->entries[idx[3]] = phys | 0x003;
+					phys += page_size;
+					if (--pages == 0) return;
+				}
+			}
+		}
+	}
+}
+
+
+page_table_indices::page_table_indices(uint64_t v) :
+	index{
+		(v >> 39) & 0x1ff,
+		(v >> 30) & 0x1ff,
+		(v >> 21) & 0x1ff,
+		(v >> 12) & 0x1ff }
+{}
+
+uintptr_t
+page_table_indices::addr() const
+{
+	return
+		(index[0] << 39) |
+		(index[1] << 30) |
+		(index[2] << 21) |
+		(index[3] << 12);
+}
+
+bool operator==(const page_table_indices &l, const page_table_indices &r)
+{
+	return l[0] == r[0] && l[1] == r[1] && l[2] == r[2] && l[3] == r[3];
+}
+} // namespace paging
+} // namespace boot

src/boot/paging.h

@@ -0,0 +1,81 @@
+#pragma once
+/// \file paging.h
+/// Page table structure and related definitions
+#include <stdint.h>
+#include <uefi/boot_services.h>
+#include "kernel_args.h"
+
+namespace boot {
+namespace paging {
+
+/// Struct to allow easy accessing of a memory page being used as a page table.
+struct page_table
+{
+	enum class level : unsigned { pml4, pdp, pd, pt };
+	inline static level deeper(level l) {
+		return static_cast<level>(static_cast<unsigned>(l) + 1);
+	}
+
+	uint64_t entries[512];
+
+	inline page_table * get(int i, uint16_t *flags = nullptr) const {
+		uint64_t entry = entries[i];
+		if ((entry & 0x1) == 0) return nullptr;
+		if (flags) *flags = entry & 0xfffull;
+		return reinterpret_cast<page_table *>(entry & ~0xfffull);
+	}
+
+	inline void set(int i, void *p, uint16_t flags) {
+		entries[i] = reinterpret_cast<uint64_t>(p) | (flags & 0xfff);
+	}
+
+	inline bool is_present(int i) const { return (entries[i] & 0x1) == 0x1; }
+
+	inline bool is_large_page(level l, int i) const {
+		return
+			(l == level::pdp || l == level::pd) &&
+			(entries[i] & 0x80) == 0x80;
+	}
+};
+
+/// Helper struct for computing page table indices of a given address.
+struct page_table_indices
+{
+	page_table_indices(uint64_t v = 0);
+
+	uintptr_t addr() const;
+
+	inline operator uintptr_t() const { return addr(); }
+
+	/// Get the index for a given level of page table.
+	uint64_t & operator[](int i) { return index[i]; }
+	uint64_t operator[](int i) const { return index[i]; }
+	uint64_t & operator[](page_table::level i) { return index[static_cast<unsigned>(i)]; }
+	uint64_t operator[](page_table::level i) const { return index[static_cast<unsigned>(i)]; }
+	uint64_t index[4]; ///< Indices for each level of tables.
+};
+
+bool operator==(const page_table_indices &l, const page_table_indices &r);
+
+/// Allocate memory to be used for initial page tables. Initial offset-mapped
+/// page tables are pre-filled. All pages are saved as a module in kernel args
+/// and kernel args' `page_table_cache` and `num_free_tables` are updated with
+/// the leftover space.
+void allocate_tables(
+	kernel::args::header *args,
+	uefi::boot_services *bs);
+
+/// Map a physical address to a virtual address in the given page tables.
+/// \arg pml4  The root of the set of page tables to be updated
+/// \arg args  The kernel args header, used for the page table cache
+/// \arg phys  The phyiscal address to map in
+/// \arg virt  The virtual address to map in
+/// \arg size  The size in bytes of the mapping 
+void map_in(
+	page_table *pml4,
+	kernel::args::header *args,
+	uintptr_t phys, uintptr_t virt,
+	size_t bytes);
+
+} // namespace paging
+} // namespace boot

src/include/kernel_args.h

@@ -22,6 +22,7 @@ enum class mod_type : uint32_t {
 	initrd,
 
 	memory_map,
+	page_tables,
 	framebuffer,
 
 	max
@@ -72,7 +73,10 @@ struct header {
 
 	uint8_t _reserved0;
 
-	uint32_t _reserved1;
+	void *pml4;
+	void *page_table_cache;
+	uint32_t num_free_tables;
+
 	uint32_t num_modules;
 	module *modules;