[boot] Virtualize memory in the bootloader

Finish updating the page tables, call UEFI's `set_virtual_address_map`
and jump to the kernel!

src/boot/main.cpp

@@ -189,12 +189,15 @@ efi_main(uefi::handle image_handle, uefi::system_table *st)
 	kernel::args::header *args =
 		bootloader_main_uefi(image_handle, st, con, &kentry);
 
-	size_t map_key = memory::build_kernel_mem_map(args, st->boot_services);
+	memory::efi_mem_map map =
+		memory::build_kernel_mem_map(args, st->boot_services);
 
 	try_or_raise(
-		st->boot_services->exit_boot_services(image_handle, map_key),
+		st->boot_services->exit_boot_services(image_handle, map.key),
 		L"Failed to exit boot services");
 
+	memory::virtualize(args->pml4, map, st->runtime_services);
+
 	kentry(args);
 	debug_break();
 	return uefi::status::unsupported;

src/boot/memory.cpp

@@ -1,9 +1,12 @@
 #include <stddef.h>
 #include <uefi/types.h>
 
+#include "kernel_memory.h"
+
 #include "console.h"
 #include "error.h"
 #include "memory.h"
+#include "paging.h"
 
 namespace boot {
 namespace memory {
@@ -85,51 +88,6 @@ mark_pointer_fixup(void **p)
 	fixup_pointers[fixup_pointer_index++] = p;
 }
 
-/*
-void
-virtualize(paging::page_table *pml4, uefi::runtime_services *rs, efi_mem_map *map)
-{
-	// Get the pointer to the start of PML4
-	uint64_t* cr3 = 0;
-	__asm__ __volatile__ ( "mov %%cr3, %0" : "=r" (cr3) );
-
-	// PML4 is indexed with bits 39:47 of the virtual address
-	uint64_t offset = (KERNEL_VIRT_ADDRESS >> 39) & 0x1ff;
-
-	// Double map the lower half pages that are present into the higher half
-	for (unsigned i = 0; i < offset; ++i) {
-		if (cr3[i] & 0x1)
-			new_pml4[i] = new_pml4[offset+i] = cr3[i];
-		else
-			new_pml4[i] = new_pml4[offset+i] = 0;
-	}
-
-	// Write our new PML4 pointer back to CR3
-	__asm__ __volatile__ ( "mov %0, %%cr3" :: "r" (new_pml4) );
-
-	EFI_MEMORY_DESCRIPTOR *end = INCREMENT_DESC(map->entries, map->length);
-	EFI_MEMORY_DESCRIPTOR *d = map->entries;
-	while (d < end) {
-		switch (d->Type) {
-		case memtype_kernel:
-		case memtype_data:
-		case memtype_initrd:
-		case memtype_scratch:
-			d->Attribute |= EFI_MEMORY_RUNTIME;
-			d->VirtualStart = d->PhysicalStart + KERNEL_VIRT_ADDRESS;
-
-		default:
-			if (d->Attribute & EFI_MEMORY_RUNTIME) {
-				d->VirtualStart = d->PhysicalStart + KERNEL_VIRT_ADDRESS;
-			}
-		}
-		d = INCREMENT_DESC(d, map->size);
-	}
-
-	runsvc->SetVirtualAddressMap(map->length, map->size, map->version, map->entries);
-}
-*/
-
 bool
 can_merge(mem_entry &prev, mem_type type, uefi::memory_descriptor *next)
 {
@@ -165,7 +123,7 @@ get_uefi_mappings(efi_mem_map *map, bool allocate, uefi::boot_services *bs)
 	}
 }
 
-size_t
+efi_mem_map
 build_kernel_mem_map(kernel::args::header *args, uefi::boot_services *bs)
 {
 	status_line(L"Creating kernel memory map");
@@ -286,7 +244,25 @@ build_kernel_mem_map(kernel::args::header *args, uefi::boot_services *bs)
 	}
 	*/
 
-	return efi_map.key;
+	return efi_map;
+}
+
+void
+virtualize(void *pml4, efi_mem_map &map, uefi::runtime_services *rs)
+{
+	paging::add_current_mappings(reinterpret_cast<paging::page_table*>(pml4));
+
+	for (auto desc : map)
+		desc->virtual_start = desc->physical_start + ::memory::page_offset;
+
+	// Write our new PML4 pointer to CR3
+	asm volatile ( "mov %0, %%cr3" :: "r" (pml4) );
+	__sync_synchronize();
+
+	try_or_raise(
+		rs->set_virtual_address_map(
+			map.length, map.size, map.version, map.entries),
+		L"Error setting virtual address map");
 }
 
 

src/boot/memory.h

@@ -81,9 +81,17 @@ struct efi_mem_map
 };
 
 /// Add the kernel's memory map as a module to the kernel args.
-/// \returns  The uefi memory map key for the version used to build
-///           this map
-size_t build_kernel_mem_map(kernel::args::header *args, uefi::boot_services *bs);
+/// \returns  The uefi memory map used to build the kernel map
+efi_mem_map build_kernel_mem_map(kernel::args::header *args, uefi::boot_services *bs);
+
+/// Activate the given memory mappings. Sets the given page tables live as well
+/// as informs UEFI runtime services of the new mappings.
+/// \arg pml4  The root page table for the new mappings
+/// \arg map   The UEFI memory map, used to update runtime services
+void virtualize(
+	void *pml4,
+	efi_mem_map &map,
+	uefi::runtime_services *rs);
 
 } // namespace boot
 } // namespace memory

src/boot/paging.cpp

@@ -1,3 +1,5 @@
+#include "kernel_memory.h"
+
 #include "console.h"
 #include "error.h"
 #include "loader.h"
@@ -123,7 +125,46 @@ private:
 };
 
 
-void allocate_tables(kernel::args::header *args, uefi::boot_services *bs)
+static void
+add_offset_mappings(page_table *pml4, void *&page_cache, uint32_t &num_pages)
+{
+	uintptr_t phys = 0;
+	uintptr_t virt = ::memory::page_offset; // Start of offset-mapped area
+	size_t pages = 64 * 1024; // 64 TiB of 1 GiB pages
+	constexpr size_t GiB = 0x40000000ull;
+
+	page_entry_iterator<2> iterator{
+		virt, pml4,
+		page_cache,
+		num_pages};
+
+	while (true) {
+		*iterator = phys | huge_page_flags;
+		if (--pages == 0)
+			break;
+
+		iterator.increment();
+		phys += GiB;
+	}
+}
+
+void
+add_current_mappings(page_table *new_pml4)
+{
+	// Get the pointer to the current PML4
+	page_table *old_pml4 = 0;
+	asm volatile ( "mov %%cr3, %0" : "=r" (old_pml4) );
+
+	// Only copy mappings in the lower half
+	for (int i = 0; i < 256; ++i) {
+		uint64_t entry = old_pml4->entries[i];
+		if (entry & 1)
+			new_pml4->entries[i] = entry;
+	}
+}
+
+void
+allocate_tables(kernel::args::header *args, uefi::boot_services *bs)
 {
 	status_line status(L"Allocating initial page tables");
 
@@ -151,25 +192,7 @@ void allocate_tables(kernel::args::header *args, uefi::boot_services *bs)
 	args->page_table_cache = offset_ptr<void>(addr, page_size);
 
 	page_table *pml4 = reinterpret_cast<page_table*>(addr);
-
-	uintptr_t phys = 0;
-	uintptr_t virt = 0xffffc00000000000ull; // Start of offset-mapped area
-	size_t pages = 64 * 1024; // 64 TiB of 1 GiB pages
-	constexpr size_t GiB = 0x40000000ull;
-
-	page_entry_iterator<2> iterator{
-		virt, pml4,
-		args->page_table_cache,
-		args->num_free_tables};
-
-	while (true) {
-		*iterator = phys | huge_page_flags;
-		if (--pages == 0)
-			break;
-
-		iterator.increment();
-		phys += GiB;
-	}
+	add_offset_mappings(pml4, args->page_table_cache, args->num_free_tables);
 
 	console::print(L"    Set up initial mappings, %d spare tables.\r\n", args->num_free_tables);
 }

src/boot/paging.h

@@ -33,6 +33,11 @@ void allocate_tables(
 	kernel::args::header *args,
 	uefi::boot_services *bs);
 
+/// Copy existing page table entries to a new page table. Does not do a deep
+/// copy - the new PML4 is updated to point to the existing next-level page
+/// tables in the current PML4.
+void add_current_mappings(page_table *new_pml4);
+
 /// Map a physical address to a virtual address in the given page tables.
 /// \arg pml4  The root of the set of page tables to be updated
 /// \arg args  The kernel args header, used for the page table cache

src/include/kernel_memory.h

@@ -11,10 +11,10 @@ namespace memory {
 	static const size_t frame_size = 0x1000;
 
 	/// Start of kernel memory.
-	static const uintptr_t kernel_offset = 0xffffff0000000000;
+	static const uintptr_t kernel_offset = 0xffff800000000000;
 
 	/// Offset from physical where page tables are mapped.
-	static const uintptr_t page_offset = 0xffffff8000000000;
+	static const uintptr_t page_offset = 0xffffc00000000000;
 
 	/// Initial process thread's stack address
 	static const uintptr_t initial_stack = 0x0000800000000000;