diff --git a/src/boot/main.cpp b/src/boot/main.cpp index 7652d93..550b60b 100644 --- a/src/boot/main.cpp +++ b/src/boot/main.cpp @@ -189,12 +189,15 @@ efi_main(uefi::handle image_handle, uefi::system_table *st) kernel::args::header *args = bootloader_main_uefi(image_handle, st, con, &kentry); - size_t map_key = memory::build_kernel_mem_map(args, st->boot_services); + memory::efi_mem_map map = + memory::build_kernel_mem_map(args, st->boot_services); try_or_raise( - st->boot_services->exit_boot_services(image_handle, map_key), + st->boot_services->exit_boot_services(image_handle, map.key), L"Failed to exit boot services"); + memory::virtualize(args->pml4, map, st->runtime_services); + kentry(args); debug_break(); return uefi::status::unsupported; diff --git a/src/boot/memory.cpp b/src/boot/memory.cpp index 70ad356..aa0ac45 100644 --- a/src/boot/memory.cpp +++ b/src/boot/memory.cpp @@ -1,9 +1,12 @@ #include #include +#include "kernel_memory.h" + #include "console.h" #include "error.h" #include "memory.h" +#include "paging.h" namespace boot { namespace memory { @@ -85,51 +88,6 @@ mark_pointer_fixup(void **p) fixup_pointers[fixup_pointer_index++] = p; } -/* -void -virtualize(paging::page_table *pml4, uefi::runtime_services *rs, efi_mem_map *map) -{ - // Get the pointer to the start of PML4 - uint64_t* cr3 = 0; - __asm__ __volatile__ ( "mov %%cr3, %0" : "=r" (cr3) ); - - // PML4 is indexed with bits 39:47 of the virtual address - uint64_t offset = (KERNEL_VIRT_ADDRESS >> 39) & 0x1ff; - - // Double map the lower half pages that are present into the higher half - for (unsigned i = 0; i < offset; ++i) { - if (cr3[i] & 0x1) - new_pml4[i] = new_pml4[offset+i] = cr3[i]; - else - new_pml4[i] = new_pml4[offset+i] = 0; - } - - // Write our new PML4 pointer back to CR3 - __asm__ __volatile__ ( "mov %0, %%cr3" :: "r" (new_pml4) ); - - EFI_MEMORY_DESCRIPTOR *end = INCREMENT_DESC(map->entries, map->length); - EFI_MEMORY_DESCRIPTOR *d = map->entries; - while (d < end) { - switch (d->Type) { - case memtype_kernel: - case memtype_data: - case memtype_initrd: - case memtype_scratch: - d->Attribute |= EFI_MEMORY_RUNTIME; - d->VirtualStart = d->PhysicalStart + KERNEL_VIRT_ADDRESS; - - default: - if (d->Attribute & EFI_MEMORY_RUNTIME) { - d->VirtualStart = d->PhysicalStart + KERNEL_VIRT_ADDRESS; - } - } - d = INCREMENT_DESC(d, map->size); - } - - runsvc->SetVirtualAddressMap(map->length, map->size, map->version, map->entries); -} -*/ - bool can_merge(mem_entry &prev, mem_type type, uefi::memory_descriptor *next) { @@ -165,7 +123,7 @@ get_uefi_mappings(efi_mem_map *map, bool allocate, uefi::boot_services *bs) } } -size_t +efi_mem_map build_kernel_mem_map(kernel::args::header *args, uefi::boot_services *bs) { status_line(L"Creating kernel memory map"); @@ -286,7 +244,25 @@ build_kernel_mem_map(kernel::args::header *args, uefi::boot_services *bs) } */ - return efi_map.key; + return efi_map; +} + +void +virtualize(void *pml4, efi_mem_map &map, uefi::runtime_services *rs) +{ + paging::add_current_mappings(reinterpret_cast(pml4)); + + for (auto desc : map) + desc->virtual_start = desc->physical_start + ::memory::page_offset; + + // Write our new PML4 pointer to CR3 + asm volatile ( "mov %0, %%cr3" :: "r" (pml4) ); + __sync_synchronize(); + + try_or_raise( + rs->set_virtual_address_map( + map.length, map.size, map.version, map.entries), + L"Error setting virtual address map"); } diff --git a/src/boot/memory.h b/src/boot/memory.h index 968c9e3..2a51688 100644 --- a/src/boot/memory.h +++ b/src/boot/memory.h @@ -81,9 +81,17 @@ struct efi_mem_map }; /// Add the kernel's memory map as a module to the kernel args. -/// \returns The uefi memory map key for the version used to build -/// this map -size_t build_kernel_mem_map(kernel::args::header *args, uefi::boot_services *bs); +/// \returns The uefi memory map used to build the kernel map +efi_mem_map build_kernel_mem_map(kernel::args::header *args, uefi::boot_services *bs); + +/// Activate the given memory mappings. Sets the given page tables live as well +/// as informs UEFI runtime services of the new mappings. +/// \arg pml4 The root page table for the new mappings +/// \arg map The UEFI memory map, used to update runtime services +void virtualize( + void *pml4, + efi_mem_map &map, + uefi::runtime_services *rs); } // namespace boot } // namespace memory diff --git a/src/boot/paging.cpp b/src/boot/paging.cpp index fa168e7..1e5332f 100644 --- a/src/boot/paging.cpp +++ b/src/boot/paging.cpp @@ -1,3 +1,5 @@ +#include "kernel_memory.h" + #include "console.h" #include "error.h" #include "loader.h" @@ -123,7 +125,46 @@ private: }; -void allocate_tables(kernel::args::header *args, uefi::boot_services *bs) +static void +add_offset_mappings(page_table *pml4, void *&page_cache, uint32_t &num_pages) +{ + uintptr_t phys = 0; + uintptr_t virt = ::memory::page_offset; // Start of offset-mapped area + size_t pages = 64 * 1024; // 64 TiB of 1 GiB pages + constexpr size_t GiB = 0x40000000ull; + + page_entry_iterator<2> iterator{ + virt, pml4, + page_cache, + num_pages}; + + while (true) { + *iterator = phys | huge_page_flags; + if (--pages == 0) + break; + + iterator.increment(); + phys += GiB; + } +} + +void +add_current_mappings(page_table *new_pml4) +{ + // Get the pointer to the current PML4 + page_table *old_pml4 = 0; + asm volatile ( "mov %%cr3, %0" : "=r" (old_pml4) ); + + // Only copy mappings in the lower half + for (int i = 0; i < 256; ++i) { + uint64_t entry = old_pml4->entries[i]; + if (entry & 1) + new_pml4->entries[i] = entry; + } +} + +void +allocate_tables(kernel::args::header *args, uefi::boot_services *bs) { status_line status(L"Allocating initial page tables"); @@ -151,25 +192,7 @@ void allocate_tables(kernel::args::header *args, uefi::boot_services *bs) args->page_table_cache = offset_ptr(addr, page_size); page_table *pml4 = reinterpret_cast(addr); - - uintptr_t phys = 0; - uintptr_t virt = 0xffffc00000000000ull; // Start of offset-mapped area - size_t pages = 64 * 1024; // 64 TiB of 1 GiB pages - constexpr size_t GiB = 0x40000000ull; - - page_entry_iterator<2> iterator{ - virt, pml4, - args->page_table_cache, - args->num_free_tables}; - - while (true) { - *iterator = phys | huge_page_flags; - if (--pages == 0) - break; - - iterator.increment(); - phys += GiB; - } + add_offset_mappings(pml4, args->page_table_cache, args->num_free_tables); console::print(L" Set up initial mappings, %d spare tables.\r\n", args->num_free_tables); } diff --git a/src/boot/paging.h b/src/boot/paging.h index f8c0e7c..c47af99 100644 --- a/src/boot/paging.h +++ b/src/boot/paging.h @@ -33,6 +33,11 @@ void allocate_tables( kernel::args::header *args, uefi::boot_services *bs); +/// Copy existing page table entries to a new page table. Does not do a deep +/// copy - the new PML4 is updated to point to the existing next-level page +/// tables in the current PML4. +void add_current_mappings(page_table *new_pml4); + /// Map a physical address to a virtual address in the given page tables. /// \arg pml4 The root of the set of page tables to be updated /// \arg args The kernel args header, used for the page table cache diff --git a/src/include/kernel_memory.h b/src/include/kernel_memory.h index 582d777..cef2be8 100644 --- a/src/include/kernel_memory.h +++ b/src/include/kernel_memory.h @@ -11,10 +11,10 @@ namespace memory { static const size_t frame_size = 0x1000; /// Start of kernel memory. - static const uintptr_t kernel_offset = 0xffffff0000000000; + static const uintptr_t kernel_offset = 0xffff800000000000; /// Offset from physical where page tables are mapped. - static const uintptr_t page_offset = 0xffffff8000000000; + static const uintptr_t page_offset = 0xffffc00000000000; /// Initial process thread's stack address static const uintptr_t initial_stack = 0x0000800000000000;