From 35b1d37df0bfe913e1bb006e547f1f8cb396a461 Mon Sep 17 00:00:00 2001 From: "Justin C. Miller" Date: Sun, 24 May 2020 16:27:48 -0700 Subject: [PATCH] [memory] Rework memory_initialize for new loader Created a new `memory_initialize()` function that uses the new-style kernel args structure from the new bootloader. Additionally: * Fixed a hard-coded interrupt EOI address that didn't work with new memory locations * Make the `page_manager::fault_handler()` automatically grant pages in the kernel heap Tags: boot page fault --- src/kernel/interrupts.cpp | 8 +- src/kernel/main.cpp | 8 +- src/kernel/memory_bootstrap.cpp | 272 ++++++++------------------------ src/kernel/page_manager.cpp | 12 +- src/kernel/page_manager.h | 13 +- src/kernel/page_table.h | 4 + 6 files changed, 95 insertions(+), 222 deletions(-) diff --git a/src/kernel/interrupts.cpp b/src/kernel/interrupts.cpp index c71225a..51d0bd5 100644 --- a/src/kernel/interrupts.cpp +++ b/src/kernel/interrupts.cpp @@ -1,5 +1,7 @@ #include +#include "kernel_memory.h" + #include "apic.h" #include "console.h" #include "cpu.h" @@ -15,6 +17,8 @@ static const uint16_t PIC1 = 0x20; static const uint16_t PIC2 = 0xa0; +constexpr uintptr_t apic_eoi_addr = 0xfee000b0 + ::memory::page_offset; + extern "C" { void _halt(); @@ -273,7 +277,7 @@ isr_handler(cpu_state *regs) print_stacktrace(2); _halt(); } - *reinterpret_cast(0xffffff80fee000b0) = 0; + *reinterpret_cast(apic_eoi_addr) = 0; } void @@ -290,5 +294,5 @@ irq_handler(cpu_state *regs) _halt(); } - *reinterpret_cast(0xffffff80fee000b0) = 0; + *reinterpret_cast(apic_eoi_addr) = 0; } diff --git a/src/kernel/main.cpp b/src/kernel/main.cpp index d147464..0c47ab1 100644 --- a/src/kernel/main.cpp +++ b/src/kernel/main.cpp @@ -84,13 +84,9 @@ kernel_main(args::header *header) cpu_id cpu; cpu.validate(); - /* - memory_initialize( - header->scratch_pages, - header->memory_map, - header->memory_map_length, - header->memory_map_desc_size); + memory_initialize(header); + /* kutil::allocator &heap = g_kernel_heap; if (header->frame_buffer && header->frame_buffer_length) { diff --git a/src/kernel/memory_bootstrap.cpp b/src/kernel/memory_bootstrap.cpp index ceca412..fc7b0dd 100644 --- a/src/kernel/memory_bootstrap.cpp +++ b/src/kernel/memory_bootstrap.cpp @@ -1,6 +1,8 @@ #include #include +#include "kernel_args.h" + #include "kutil/assert.h" #include "kutil/heap_allocator.h" #include "kutil/vm_space.h" @@ -16,232 +18,98 @@ using memory::kernel_max_heap; using memory::kernel_offset; using memory::page_offset; -kutil::vm_space g_kspace; +using namespace kernel; + +kutil::vm_space g_kernel_space; kutil::heap_allocator g_kernel_heap; -bool g_memory_initialized = false; void * operator new(size_t size) { return g_kernel_heap.allocate(size); } void * operator new [] (size_t size) { return g_kernel_heap.allocate(size); } void operator delete (void *p) noexcept { return g_kernel_heap.free(p); } void operator delete [] (void *p) noexcept { return g_kernel_heap.free(p); } -enum class efi_memory_type : uint32_t +void walk_page_table( + page_table *table, + page_table::level level, + uintptr_t ¤t_start, + size_t ¤t_bytes, + kutil::vm_space &kspace) { - reserved, - loader_code, - loader_data, - boot_services_code, - boot_services_data, - runtime_services_code, - runtime_services_data, - available, - unusable, - acpi_reclaim, - acpi_nvs, - mmio, - mmio_port, - pal_code, - persistent, + constexpr size_t huge_page_size = (1ull<<30); + constexpr size_t large_page_size = (1ull<<21); - efi_max, + for (unsigned i = 0; i < 512; ++i) { + page_table *next = table->get(i); + if (!next) { + kspace.commit(current_start, current_bytes); + current_start = 0; + current_bytes = 0; + continue; + } else if (table->is_page(level, i)) { + if (!current_bytes) + current_start = reinterpret_cast(next); + current_bytes += + (level == page_table::level::pt + ? frame_size + : level == page_table::level::pd + ? large_page_size + : huge_page_size); + } else { + page_table::level deeper = + static_cast( + static_cast(level) + 1); - jsix_kernel = 0x80000000, - jsix_data, - jsix_initrd, - jsix_scratch, - - jsix_max -}; - -struct efi_memory_descriptor -{ - efi_memory_type type; - uint32_t pad; - uint64_t physical_start; - uint64_t virtual_start; - uint64_t pages; - uint64_t flags; -}; - -struct memory_map -{ - memory_map(const void *efi_map, size_t map_length, size_t desc_length) : - efi_map(efi_map), map_length(map_length), desc_length(desc_length) {} - - class iterator - { - public: - iterator(const memory_map &map, efi_memory_descriptor const *item) : - map(map), item(item) {} - - inline efi_memory_descriptor const * operator*() const { return item; } - inline bool operator!=(const iterator &other) { return item != other.item; } - inline iterator & operator++() { - item = kutil::offset_pointer(item, map.desc_length); - return *this; - } - - private: - const memory_map ↦ - efi_memory_descriptor const *item; - }; - - iterator begin() const { - return iterator(*this, reinterpret_cast(efi_map)); - } - - iterator end() const { - const void *end = kutil::offset_pointer(efi_map, map_length); - return iterator(*this, reinterpret_cast(end)); - } - - const void *efi_map; - size_t map_length; - size_t desc_length; -}; - -class memory_bootstrap -{ -public: - memory_bootstrap(const void *memory_map, size_t map_length, size_t desc_length) : - map(memory_map, map_length, desc_length) {} - - void add_free_frames(frame_allocator &fa) { - for (auto *desc : map) { - if (desc->type == efi_memory_type::loader_code || - desc->type == efi_memory_type::loader_data || - desc->type == efi_memory_type::boot_services_code || - desc->type == efi_memory_type::boot_services_data || - desc->type == efi_memory_type::available) - { - fa.free(desc->physical_start, desc->pages); - } + walk_page_table( + next, deeper, current_start, current_bytes, kspace); } } - - void add_used_frames(kutil::vm_space &vm) { - for (auto *desc : map) { - if (desc->type == efi_memory_type::jsix_data || - desc->type == efi_memory_type::jsix_initrd || - desc->type == efi_memory_type::jsix_kernel) - { - uintptr_t virt_addr = desc->physical_start + kernel_offset; - vm.commit(virt_addr, desc->pages * frame_size); - } - } - } - - void page_in_kernel(page_manager &pm, page_table *pml4) { - for (auto *desc : map) { - if (desc->type == efi_memory_type::jsix_kernel || - desc->type == efi_memory_type::jsix_data || - desc->type == efi_memory_type::jsix_initrd) - { - uintptr_t virt_addr = desc->physical_start + kernel_offset; - pm.page_in(pml4, desc->physical_start, virt_addr, desc->pages); - } - - if (desc->type == efi_memory_type::acpi_reclaim) { - pm.page_in(pml4, desc->physical_start, desc->physical_start, desc->pages); - } - } - - // Put our new PML4 into CR3 to start using it - page_manager::set_pml4(pml4); - pm.m_kernel_pml4 = pml4; - } - -private: - const memory_map map; -}; +} void -memory_initialize(uint16_t scratch_pages, const void *memory_map, size_t map_length, size_t desc_length) +memory_initialize(args::header *kargs) { - g_memory_initialized = false; + args::mem_entry *entries = kargs->mem_map; + size_t entry_count = kargs->num_map_entries; - // make sure the options we want in CR4 are set - uint64_t cr4; - __asm__ __volatile__ ( "mov %%cr4, %0" : "=r" (cr4) ); - cr4 |= - 0x000080 | // Enable global pages - 0x000200 | // Enable FXSAVE/FXRSTOR - 0x010000 | // Enable FSGSBASE - 0x020000 | // Enable PCIDs - 0; - __asm__ __volatile__ ( "mov %0, %%cr4" :: "r" (cr4) ); + new (&g_kernel_heap) kutil::heap_allocator {heap_start, kernel_max_heap}; - // The bootloader reserved "scratch_pages" pages for page tables and - // scratch space, which we'll use to bootstrap. The first one is the - // already-installed PML4, so grab it from CR3. - uint64_t scratch_phys; - __asm__ __volatile__ ( "mov %%cr3, %0" : "=r" (scratch_phys) ); - scratch_phys &= ~0xfffull; - - // The tables are ident-mapped currently, so the cr3 physical address works. But let's - // get them into the offset-mapped area asap. - page_table *tables = reinterpret_cast(scratch_phys); - - page_table *id_pml4 = &tables[0]; - page_table *id_pdp = &tables[1]; - - // Flags: 0 0 0 0 1 1 0 0 0 0 0 1 1 = 0x0183 - // | IGN | | | | | | | | +- Present - // | | | | | | | | +--- Writeable - // | | | | | | | +----- Supervisor only - // | | | | | | +------- PWT (determining memory type for page) - // | | | | | +---------- PCD (determining memory type for page) - // | | | | +------------ Accessed flag (not accessed yet) - // | | | +-------------- Dirty (not dirtied yet) - // | | +---------------- Page size (1GiB page) - // | +------------------- Global - // +---------------------------- PAT (determining memory type for page) - for (int i=0; i<512; ++i) - id_pdp->entries[i] = (static_cast(i) << 30) | 0x0183; - - // Flags: 0 0 0 0 0 0 0 0 0 0 1 1 = 0x0003 - // IGNORED | | | | | | | +- Present - // | | | | | | +--- Writeable - // | | | | | +----- Supervisor only - // | | | | +------- PWT (determining memory type for pdpt) - // | | | +---------- PCD (determining memory type for pdpt) - // | | +------------ Accessed flag (not accessed yet) - // | +-------------- Ignored - // +---------------- Reserved 0 - id_pml4->entries[511] = reinterpret_cast(id_pdp) | 0x003; - - // Make sure the page table is finished updating before we write to memory - __sync_synchronize(); - io_wait(); - - uintptr_t scratch_virt = scratch_phys + page_offset; - memory_bootstrap bootstrap {memory_map, map_length, desc_length}; - - // Now tell the frame allocator what's free frame_allocator *fa = new (&g_frame_allocator) frame_allocator; - bootstrap.add_free_frames(*fa); - - new (&g_kernel_heap) kutil::heap_allocator(heap_start, kernel_max_heap); - new (&g_kspace) kutil::vm_space(kernel_offset, (page_offset-kernel_offset), g_kernel_heap); - bootstrap.add_used_frames(g_kspace); + for (unsigned i = 0; i < entry_count; ++i) { + // TODO: use entry attributes + args::mem_entry &e = entries[i]; + if (e.type == args::mem_type::free) + fa->free(e.start, e.pages); + } // Create the page manager page_manager *pm = new (&g_page_manager) page_manager(*fa); - // Give the frame_allocator back the rest of the scratch pages - fa->free(scratch_phys + (3 * frame_size), scratch_pages - 3); + new (&g_kernel_space) kutil::vm_space { + kernel_offset, + (page_offset-kernel_offset), + g_kernel_heap}; - // Finally, build an acutal set of kernel page tables where we'll only add - // what the kernel actually has mapped, but making everything writable - // (especially the page tables themselves) - page_table *pml4 = &tables[2]; - pml4 = kutil::offset_pointer(pml4, page_offset); + page_table *kpml4 = reinterpret_cast(kargs->pml4); - kutil::memset(pml4, 0, sizeof(page_table)); - pml4->entries[511] = reinterpret_cast(id_pdp) | 0x003; + uintptr_t current_start = 0; + size_t current_bytes = 0; - bootstrap.page_in_kernel(*pm, pml4); + // TODO: Should we exclude the top of this area? (eg, buffers, stacks, etc) + for (unsigned i = 256; i < 384; ++i) { + page_table *pdp = kpml4->get(i); - // Reclaim the old PML4 - fa->free(scratch_phys, 1); + if (!pdp) { + g_kernel_space.commit(current_start, current_bytes); + current_start = 0; + current_bytes = 0; + continue; + } + + walk_page_table( + pdp, page_table::level::pdp, + current_start, current_bytes, + g_kernel_space); + } + + fa->free(reinterpret_cast(kargs->page_table_cache), kargs->num_free_tables); } diff --git a/src/kernel/page_manager.cpp b/src/kernel/page_manager.cpp index 9be0a3e..051ec00 100644 --- a/src/kernel/page_manager.cpp +++ b/src/kernel/page_manager.cpp @@ -15,7 +15,7 @@ using memory::page_offset; using memory::page_mappable; page_manager g_page_manager(g_frame_allocator); -extern kutil::vm_space g_kspace; +extern kutil::vm_space g_kernel_space; // NB: in 4KiB page table entries, bit 7 isn't pagesize but PAT. Currently this // doesn't matter, becasue in the default PAT table, both 000 and 100 are WB. @@ -47,8 +47,7 @@ struct free_page_header page_manager::page_manager(frame_allocator &frames) : m_page_cache(nullptr), - m_frames(frames), - m_memory_initialized(false) + m_frames(frames) { } @@ -322,8 +321,11 @@ page_manager::fault_handler(uintptr_t addr) if (!addr) return false; - if (m_memory_initialized && - g_kspace.get(addr) != kutil::vm_state::committed) + bool is_heap = addr >= ::memory::heap_start && + addr < ::memory::heap_start + ::memory::kernel_max_heap; + + if (!is_heap && + g_kernel_space.get(addr) != kutil::vm_state::committed) return false; uintptr_t page = addr & ~0xfffull; diff --git a/src/kernel/page_manager.h b/src/kernel/page_manager.h index 6d45b79..d2f7759 100644 --- a/src/kernel/page_manager.h +++ b/src/kernel/page_manager.h @@ -175,8 +175,6 @@ private: frame_allocator &m_frames; - bool m_memory_initialized; - friend class memory_bootstrap; page_manager(const page_manager &) = delete; }; @@ -208,10 +206,11 @@ page_table_align(T p) return ((p - 1) & ~0x1fffffull) + 0x200000; } +namespace kernel { +namespace args { + struct header; +} +} /// Bootstrap the memory managers. -void memory_initialize( - uint16_t scratch_pages, - const void *memory_map, - size_t map_length, - size_t desc_length); +void memory_initialize(kernel::args::header *mem_map); diff --git a/src/kernel/page_table.h b/src/kernel/page_table.h index 9204a50..ae55406 100644 --- a/src/kernel/page_table.h +++ b/src/kernel/page_table.h @@ -36,6 +36,10 @@ struct page_table (entries[i] & 0x80) == 0x80; } + inline bool is_page(level l, int i) const { + return (l == level::pt) || is_large_page(l, i); + } + void dump( level lvl = level::pml4, bool recurse = true);