Move kernel to higher half.

Return to having the bootloader re-map the kernel into the higher
half before jumping into the kernel entrypoint, so we don't have
to juggle pointers inside the kernel.

src/boot/loader.h

@@ -8,6 +8,10 @@
 #define KERNEL_PHYS_ADDRESS 0x100000
 #endif
 
+#ifndef KERNEL_VIRT_ADDRESS
+#define KERNEL_VIRT_ADDRESS 0xFFFF800000000000
+#endif
+
 #ifndef VIRTUAL_OFFSET
 #define VIRTUAL_OFFSET 0xf00000000
 #endif
@@ -32,6 +36,10 @@
 #define KERNEL_LOG_PAGES 4
 #endif
 
+#ifndef KERNEL_PT_MEMTYPE
+#define KERNEL_PT_MEMTYPE 0x80000004
+#endif
+
 #ifndef KERNEL_FILENAME
 #define KERNEL_FILENAME L"kernel.bin"
 #endif

src/boot/main.c

@@ -37,6 +37,7 @@ efi_main(EFI_HANDLE image_handle, EFI_SYSTEM_TABLE *system_table)
 {
 	EFI_STATUS status;
 	EFI_BOOT_SERVICES *bootsvc = system_table->BootServices;
+	EFI_RUNTIME_SERVICES *runsvc = system_table->RuntimeServices;
 
 	// When checking console initialization, use CHECK_EFI_STATUS_OR_RETURN
 	// because we can't be sure if the console was fully set up
@@ -44,6 +45,8 @@ efi_main(EFI_HANDLE image_handle, EFI_SYSTEM_TABLE *system_table)
 	CHECK_EFI_STATUS_OR_RETURN(status, "con_initialize");
 	// From here on out, we can use CHECK_EFI_STATUS_OR_FAIL instead
 
+	memory_init_pointer_fixup(bootsvc, runsvc);
+
 	// Find ACPI tables. Ignore ACPI 1.0 if a 2.0 table is found.
 	//
 	void *acpi_table = NULL;
@@ -100,10 +103,12 @@ efi_main(EFI_HANDLE image_handle, EFI_SYSTEM_TABLE *system_table)
 	con_printf(L"    Entrypoint 0x%x\r\n", version->entrypoint);
 
 	void (*kernel_main)() = version->entrypoint;
+	memory_mark_pointer_fixup((void **)&kernel_main);
 
 	// Set up the kernel data pages to pass to the kernel
 	//
 	struct popcorn_data *data_header = (struct popcorn_data *)load.data; 
+	memory_mark_pointer_fixup((void **)&data_header);
 
 	data_header->magic = DATA_HEADER_MAGIC;
 	data_header->version = DATA_HEADER_VERSION;
@@ -113,16 +118,24 @@ efi_main(EFI_HANDLE image_handle, EFI_SYSTEM_TABLE *system_table)
 
 	data_header->font = load.font;
 	data_header->font_length = load.font_length;
+	memory_mark_pointer_fixup((void **)&data_header->font);
 
 	data_header->data = load.data;
 	data_header->data_length = load.data_length;
+	memory_mark_pointer_fixup((void **)&data_header->data);
 
 	data_header->log = load.log;
 	data_header->log_length = load.log_length;
+	memory_mark_pointer_fixup((void **)&data_header->log);
 
 	data_header->memory_map = (EFI_MEMORY_DESCRIPTOR *)(data_header + 1);
-	data_header->runtime = system_table->RuntimeServices;
+	memory_mark_pointer_fixup((void **)&data_header->memory_map);
+
+	data_header->runtime = runsvc;
+	memory_mark_pointer_fixup((void **)&data_header->runtime);
+
 	data_header->acpi_table = acpi_table;
+	memory_mark_pointer_fixup((void **)&data_header->acpi_table);
 
 	data_header->_reserved0 = 0;
 	data_header->_reserved1 = 0;
@@ -139,6 +152,7 @@ efi_main(EFI_HANDLE image_handle, EFI_SYSTEM_TABLE *system_table)
 			&data_header->gmask,
 			&data_header->bmask);
 	CHECK_EFI_STATUS_OR_FAIL(status);
+	memory_mark_pointer_fixup((void **)&data_header->frame_buffer);
 
 	// Save the memory map and tell the firmware we're taking control.
 	//
@@ -154,6 +168,8 @@ efi_main(EFI_HANDLE image_handle, EFI_SYSTEM_TABLE *system_table)
 	status = bootsvc->ExitBootServices(image_handle, map.key);
 	CHECK_EFI_STATUS_OR_ASSERT(status, 0);
 
+	memory_virtualize(runsvc, &map);
+
 	// Hand control to the kernel
 	//
 	kernel_main(data_header);

src/boot/memory.c

@@ -8,6 +8,10 @@
 
 #define INCREMENT_DESC(p, b)  (EFI_MEMORY_DESCRIPTOR*)(((uint8_t*)(p))+(b))
 
+size_t fixup_pointer_index = 0;
+void **fixup_pointers[64];
+uint64_t *new_pml4 = 0;
+
 const CHAR16 *memory_type_names[] = {
 	L"EfiReservedMemoryType",
 	L"EfiLoaderCode",
@@ -37,6 +41,49 @@ memory_type_name(UINT32 value)
 	return memory_type_names[value];
 }
 
+void EFIAPI
+memory_update_marked_addresses(EFI_EVENT UNUSED *event, void *context)
+{
+	EFI_RUNTIME_SERVICES *runsvc = (EFI_RUNTIME_SERVICES*)context;
+	for (size_t i = 0; i < fixup_pointer_index; ++i) {
+		if (fixup_pointers[i])
+			runsvc->ConvertPointer(0, fixup_pointers[i]);
+	}
+}
+
+EFI_STATUS
+memory_init_pointer_fixup(EFI_BOOT_SERVICES *bootsvc, EFI_RUNTIME_SERVICES *runsvc)
+{
+	EFI_STATUS status;
+	EFI_EVENT event;
+
+	status = bootsvc->CreateEvent(
+			EVT_SIGNAL_VIRTUAL_ADDRESS_CHANGE,
+			TPL_CALLBACK,
+			(EFI_EVENT_NOTIFY)&memory_update_marked_addresses,
+			runsvc,
+			&event);
+
+	CHECK_EFI_STATUS_OR_RETURN(status, "Failed to initialize pointer update event.");
+
+	// Reserve a page for our replacement PML4
+	EFI_PHYSICAL_ADDRESS addr = 0;
+	status = bootsvc->AllocatePages(AllocateAnyPages, EfiLoaderData, 1, &addr);
+	CHECK_EFI_STATUS_OR_RETURN(status, "Failed to allocate PML4 page.");
+
+	new_pml4 = (uint64_t *)addr;
+}
+
+void
+memory_mark_pointer_fixup(void **p)
+{
+	if (fixup_pointer_index == 0) {
+		const size_t count = sizeof(fixup_pointers) / sizeof(void*);
+		for (size_t i = 0; i < count; ++i) fixup_pointers[i] = 0;
+	}
+	fixup_pointers[fixup_pointer_index++] = p;
+}
+
 void
 copy_desc(EFI_MEMORY_DESCRIPTOR *src, EFI_MEMORY_DESCRIPTOR *dst, size_t len)
 {
@@ -111,3 +158,50 @@ memory_dump_map(struct memory_map *map)
 
 	return EFI_SUCCESS;
 }
+
+void
+memory_virtualize(EFI_RUNTIME_SERVICES *runsvc, struct memory_map *map)
+{
+	memory_mark_pointer_fixup((void **)&runsvc);
+	memory_mark_pointer_fixup((void **)&map);
+
+	// Get the pointer to the start of PML4
+	uint64_t* cr3 = 0;
+	__asm__ __volatile__ ( "mov %%cr3, %0" : "=r" (cr3) );
+
+	// PML4 is indexed with bits 39:47 of the virtual address
+	uint64_t offset = (KERNEL_VIRT_ADDRESS >> 39) & 0x1ff;
+
+	// Double map the lower half pages that are present into the higher half
+	for (unsigned i = 0; i < offset; ++i) {
+		if (cr3[i] & 0x1)
+			new_pml4[i] = new_pml4[offset+i] = cr3[i];
+		else
+			new_pml4[i] = new_pml4[offset+i] = 0;
+	}
+
+	// Write our new PML4 pointer back to CR3
+	__asm__ __volatile__ ( "mov %0, %%cr3" :: "r" (new_pml4) );
+
+	EFI_MEMORY_DESCRIPTOR *end = INCREMENT_DESC(map->entries, map->length);
+	EFI_MEMORY_DESCRIPTOR *d = map->entries;
+	while (d < end) {
+		switch (d->Type) {
+		case KERNEL_MEMTYPE:
+		case KERNEL_FONT_MEMTYPE:
+		case KERNEL_DATA_MEMTYPE:
+		case KERNEL_LOG_MEMTYPE:
+			d->Attribute |= EFI_MEMORY_RUNTIME;
+			d->VirtualStart = d->PhysicalStart + KERNEL_VIRT_ADDRESS;
+			break;
+
+		default:
+			if (d->Attribute & EFI_MEMORY_RUNTIME) {
+				d->VirtualStart = d->PhysicalStart;
+			}
+		}
+		d = INCREMENT_DESC(d, map->size);
+	}
+
+	runsvc->SetVirtualAddressMap(map->length, map->size, map->version, map->entries);
+}

src/boot/memory.h

@@ -9,6 +9,11 @@ struct memory_map {
 	EFI_MEMORY_DESCRIPTOR *entries;
 };
 
+EFI_STATUS memory_init_pointer_fixup(EFI_BOOT_SERVICES *bootsvc, EFI_RUNTIME_SERVICES *runsvc);
+void memory_mark_pointer_fixup(void **p);
+
 EFI_STATUS memory_get_map_length(EFI_BOOT_SERVICES *bootsvc, size_t *size);
 EFI_STATUS memory_get_map(EFI_BOOT_SERVICES *bootsvc, struct memory_map *map);
 EFI_STATUS memory_dump_map(struct memory_map *map);
+
+void memory_virtualize(EFI_RUNTIME_SERVICES *runsvc, struct memory_map *map);