[boot] Restructure boot paging and program loading

Restructuring paging into an object that carries its page cache with it
and makes for simpler code. Program loading is also changed to not copy
the pages loaded from the file into new pages - we can impose a new
constraint that anything loaded by boot have a simple, page-aligned
layout so that we can just map the existing pages into the right
addresses. Also included are some linker script changes to help
accommodate this.

src/boot/loader.cpp

@@ -1,7 +1,6 @@
 #include <uefi/boot_services.h>
 #include <uefi/types.h>
 
-#include <bootproto/init.h>
 #include <elf/file.h>
 #include <elf/headers.h>
 #include <util/pointers.h>
@@ -22,9 +21,7 @@ namespace loader {
 using memory::alloc_type;
 
 util::buffer
-load_file(
+load_file(fs::file &disk, const wchar_t *path)
-    fs::file &disk,
-    const wchar_t *path)
 {
     status_line status(L"Loading file", path);
 
@@ -61,73 +58,108 @@ verify_kernel_header(elf::file &kernel, util::const_buffer data)
             header->version_gitsha);
 }
 
-bootproto::program *
+inline void
-load_program(
+elf_error(const elf::file &elf, util::const_buffer data)
-    fs::file &disk,
-    const wchar_t *name,
-    const descriptor &desc,
-    bool verify)
 {
-    status_line status(L"Loading program", name);
+    auto *header = elf.header();
+    console::print(L"        progam size: %d\r\n", data.count);
+    console::print(L"          word size: %d\r\n", header->word_size);
+    console::print(L"         endianness: %d\r\n", header->endianness);
+    console::print(L"  ELF ident version: %d\r\n", header->ident_version);
+    console::print(L"             OS ABI: %d\r\n", header->os_abi);
+    console::print(L"          file type: %d\r\n", header->file_type);
+    console::print(L"       machine type: %d\r\n", header->machine_type);
+    console::print(L"        ELF version: %d\r\n", header->version);
 
-    util::const_buffer data = load_file(disk, desc.path);
+    error::raise(uefi::status::load_error, L"ELF file not valid");
+}
 
-    elf::file program {data};
+inline uintptr_t
-    if (!program.valid()) {
+allocate_bss(elf::segment_header seg)
-        auto *header = program.header();
+{
-        console::print(L"        progam size: %d\r\n", data.count);
+    size_t page_count = memory::bytes_to_pages(seg.mem_size);
-        console::print(L"          word size: %d\r\n", header->word_size);
+    void *pages = g_alloc.allocate_pages(page_count, alloc_type::program, true);
-        console::print(L"         endianness: %d\r\n", header->endianness);
+    return reinterpret_cast<uintptr_t>(pages);
-        console::print(L"  ELF ident version: %d\r\n", header->ident_version);
+}
-        console::print(L"             OS ABI: %d\r\n", header->os_abi);
-        console::print(L"          file type: %d\r\n", header->file_type);
-        console::print(L"       machine type: %d\r\n", header->machine_type);
-        console::print(L"        ELF version: %d\r\n", header->version);
 
-        error::raise(uefi::status::load_error, L"ELF file not valid");
-    }
 
-    if (verify)
+void
-        verify_kernel_header(program, data);
+parse_program(const wchar_t *name, util::const_buffer data, bootproto::program &program)
+{
+    status_line status(L"Preparing program", name);
+
+    elf::file elf {data};
+    if (!elf.valid())
+        elf_error(elf, data); // does not return
 
     size_t num_sections = 0;
-    for (auto &seg : program.segments()) {
+    for (auto &seg : elf.segments()) {
         if (seg.type == elf::segment_type::load)
             ++num_sections;
     }
 
-    bootproto::program_section *sections = new bootproto::program_section [num_sections];
+    bootproto::program_section *sections =
+        new bootproto::program_section [num_sections];
 
     size_t next_section = 0;
-    for (auto &seg : program.segments()) {
+    for (auto &seg : elf.segments()) {
         if (seg.type != elf::segment_type::load)
             continue;
 
         bootproto::program_section &section = sections[next_section++];
-
+        section.phys_addr = elf.base() + seg.offset;
-        uintptr_t virt_addr = seg.vaddr;
+        section.virt_addr = seg.vaddr;
-        size_t mem_size = seg.mem_size;
+        section.size = seg.mem_size;
-
-        // Page-align the section, which may require increasing the size
-        size_t prelude = virt_addr & 0xfff;
-        mem_size += prelude;
-        virt_addr &= ~0xfffull;
-
-        size_t page_count = memory::bytes_to_pages(mem_size);
-        void *pages = g_alloc.allocate_pages(page_count, alloc_type::program, true);
-        const void *source = util::offset_pointer(data.pointer, seg.offset);
-        g_alloc.copy(util::offset_pointer(pages, prelude), source, seg.file_size);
-        section.phys_addr = reinterpret_cast<uintptr_t>(pages);
-        section.virt_addr = virt_addr;
-        section.size = mem_size;
         section.type = static_cast<bootproto::section_flags>(seg.flags);
+
+        if (seg.mem_size != seg.file_size)
+            section.phys_addr = allocate_bss(seg);
     }
 
-    bootproto::program *prog = new bootproto::program;
+    program.sections = { .pointer = sections, .count = num_sections };
-    prog->sections = { .pointer = sections, .count = num_sections };
+    program.phys_base = elf.base();
-    prog->phys_base = program.base();
+    program.entrypoint = elf.entrypoint();
-    prog->entrypoint = program.entrypoint();
+}
-    return prog;
+
+uintptr_t
+load_program(
+    util::const_buffer data,
+    const wchar_t *name,
+    paging::pager &pager,
+    bool verify)
+{
+    using util::bits::has;
+
+    status_line status(L"Loading program", name);
+
+    elf::file elf {data};
+    if (!elf.valid())
+        elf_error(elf, data); // does not return
+
+    if (verify)
+        verify_kernel_header(elf, data);
+
+    size_t num_sections = 0;
+    for (auto &seg : elf.segments()) {
+        if (seg.type == elf::segment_type::load)
+            ++num_sections;
+    }
+
+    for (auto &seg : elf.segments()) {
+        if (seg.type != elf::segment_type::load)
+            continue;
+
+        uintptr_t phys_addr = elf.base() + seg.offset;
+        if (seg.mem_size != seg.file_size)
+            phys_addr = allocate_bss(seg);
+
+        pager.map_pages(phys_addr, seg.vaddr,
+            memory::bytes_to_pages(seg.mem_size),
+            has(seg.flags, elf::segment_flags::write),
+            has(seg.flags, elf::segment_flags::exec));
+    }
+
+    return elf.entrypoint();
 }
 
 void

src/boot/loader.h

@@ -3,6 +3,7 @@
 #pragma once
 
 #include <bootproto/init.h>
+#include <bootproto/kernel.h>
 #include <util/counted.h>
 
 namespace bootproto {
@@ -11,32 +12,47 @@ namespace bootproto {
 
 namespace boot {
 
-class descriptor;
+namespace fs { class file; }
-
+namespace paging { class pager; }
-namespace fs {
-    class file;
-}
 
 namespace loader {
 
+// Bootloader ELF file requirements
+// ================================
+// The bootloader accepts a subset of valid ELF files to load, with
+// the following requiresments:
+//  1. All program segments are page-aligned.
+//  2. PT_LOAD segments cannot contain a mix of PROGBITS and NOBITS
+//     sections. i.e., section memory size must equal either zero or
+//     its file size.
+//  3. There are only one or zero PT_LOAD NOBITS program segments.
+
+
 /// Load a file from disk into memory.
 /// \arg disk  The opened UEFI filesystem to load from
 /// \arg path  The path of the file to load
-util::buffer
+util::buffer load_file(fs::file &disk, const wchar_t *path);
-load_file(
-    fs::file &disk,
-    const wchar_t *path);
 
-/// Parse and load an ELF file in memory into a loaded image.
+/// Parse a buffer holding ELF data into a bootproto::program
-/// \arg disk   The opened UEFI filesystem to load from
-/// \arg desc   The descriptor identifying the program
 /// \arg name   The human-readable name of the program to load
-/// \arg verify If this is the kernel and should have its header verified
+/// \arg data    A buffer containing an ELF executable
-bootproto::program *
+/// \arg program A program structure to fill
-load_program(
+void parse_program(
-    fs::file &disk,
     const wchar_t *name,
-    const descriptor &desc,
+    util::const_buffer data,
+    bootproto::program &program);
+
+/// Parse a buffer holding ELF data and map it to be runnable
+/// \arg data   The ELF data in memory
+/// \arg name   The human-readable name of the program to load
+/// \arg pager  The kernel space pager, to map programs into
+/// \arg verify If this is the kernel and should have its header verified
+/// \returns    The entrypoint to the loaded program
+uintptr_t
+load_program(
+    util::const_buffer data,
+    const wchar_t *name,
+    paging::pager &pager,
     bool verify = false);
 
 /// Load a file from disk into memory, creating an init args module

src/boot/main.cpp

@@ -60,14 +60,17 @@ uefi_preboot(uefi::handle image, uefi::system_table *st)
     args->acpi_table = hw::find_acpi_table(st);
     memory::mark_pointer_fixup(&args->runtime_services);
 
-    paging::allocate_tables(args);
-
     return args;
 }
 
 /// Load the kernel and other programs from disk
-void
+bootproto::entrypoint
-load_resources(bootproto::args *args, video::screen *screen, uefi::handle image, uefi::boot_services *bs)
+load_resources(
+        bootproto::args *args,
+        video::screen *screen,
+        uefi::handle image,
+        paging::pager &pager,
+        uefi::boot_services *bs)
 {
     status_line status {L"Loading programs"};
 
@@ -75,17 +78,16 @@ load_resources(bootproto::args *args, video::screen *screen, uefi::handle image,
     util::buffer bc_data = loader::load_file(disk, L"jsix\\boot.conf");
     bootconfig bc {bc_data, bs};
 
-    args->kernel = loader::load_program(disk, L"kernel", bc.kernel(), true);
+    util::buffer kernel = loader::load_file(disk, bc.kernel().path);
-    args->init = loader::load_program(disk, L"init server", bc.init());
+    uintptr_t kentry = loader::load_program(kernel, L"jsix kernel", pager, true);
-    args->flags = static_cast<bootproto::boot_flags>(bc.flags());
 
-    loader::load_module(disk, L"initrd", bc.initrd(),
+    args->flags = static_cast<bootproto::boot_flags>(bc.flags());
-            bootproto::module_type::initrd, 0);
 
     namespace bits = util::bits;
     using bootproto::desc_flags;
 
     bool has_panic = false;
+    util::buffer panic;
 
     if (screen) {
         video::make_module(screen);
@@ -94,7 +96,7 @@ load_resources(bootproto::args *args, video::screen *screen, uefi::handle image,
         // give it priority
         for (const descriptor &d : bc.panics()) {
             if (bits::has(d.flags, desc_flags::graphical)) {
-                args->panic = loader::load_program(disk, L"panic handler", d);
+                panic = loader::load_file(disk, d.path);
                 has_panic = true;
                 break;
             }
@@ -104,16 +106,28 @@ load_resources(bootproto::args *args, video::screen *screen, uefi::handle image,
     if (!has_panic) {
         for (const descriptor &d : bc.panics()) {
             if (!bits::has(d.flags, desc_flags::graphical)) {
-                args->panic = loader::load_program(disk, L"panic handler", d);
+                panic = loader::load_file(disk, d.path);
                 has_panic = true;
                 break;
             }
         }
     }
 
-    const wchar_t *symbol_file = bc.symbols();
+    if (has_panic) {
-    if (has_panic && symbol_file && *symbol_file)
+        args->panic_handler = loader::load_program(panic, L"panic handler", pager);
-        args->symbol_table = loader::load_file(disk, symbol_file).pointer;
+
+        const wchar_t *symbol_file = bc.symbols();
+        if (symbol_file && *symbol_file)
+            args->symbol_table = loader::load_file(disk, symbol_file);
+    }
+
+    util::buffer init = loader::load_file(disk, bc.init().path);
+    loader::parse_program(L"init server", init, args->init);
+
+    loader::load_module(disk, L"initrd", bc.initrd(),
+            bootproto::module_type::initrd, 0);
+
+    return reinterpret_cast<bootproto::entrypoint>(kentry);
 }
 
 memory::efi_mem_map
@@ -156,35 +170,31 @@ efi_main(uefi::handle image, uefi::system_table *st)
     con.announce();
 
     bootproto::args *args = uefi_preboot(image, st);
-    load_resources(args, screen, image, bs);
+
+    paging::pager pager {bs};
+
+    bootproto::entrypoint kentry =
+        load_resources(args, screen, image, pager, bs);
+
+    pager.update_kernel_args(args);
     memory::efi_mem_map map = uefi_exit(args, image, st->boot_services);
 
     args->allocations = allocs;
-    args->modules = reinterpret_cast<uintptr_t>(modules);
+    args->init_modules = reinterpret_cast<uintptr_t>(modules);
 
     status_bar status {screen}; // Switch to fb status display
 
-    // Map the kernel and panic handler to the appropriate addresses
+    memory::fix_frame_blocks(args, pager);
-    paging::map_program(args, *args->kernel);
-    paging::map_program(args, *args->panic);
 
-    memory::fix_frame_blocks(args);
-
-    bootproto::entrypoint kentry =
-        reinterpret_cast<bootproto::entrypoint>(args->kernel->entrypoint);
     //status.next();
 
     hw::setup_control_regs();
-    memory::virtualize(args->pml4, map, st->runtime_services);
+    memory::virtualize(pager, map, st->runtime_services);
     //status.next();
 
     change_pointer(args);
     change_pointer(args->pml4);
-
+    change_pointer(args->init.sections.pointer);
-    change_pointer(args->kernel);
-    change_pointer(args->kernel->sections.pointer);
-    change_pointer(args->init);
-    change_pointer(args->init->sections.pointer);
 
     //status.next();
 

src/boot/memory.cpp

@@ -57,16 +57,14 @@ mark_pointer_fixup(void **p)
 }
 
 void
-virtualize(void *pml4, efi_mem_map &map, uefi::runtime_services *rs)
+virtualize(paging::pager &pager, efi_mem_map &map, uefi::runtime_services *rs)
 {
-    paging::add_current_mappings(reinterpret_cast<paging::page_table*>(pml4));
+    pager.add_current_mappings();
 
     for (auto &desc : map)
         desc.virtual_start = desc.physical_start + bootproto::mem::linear_offset;
 
-    // Write our new PML4 pointer to CR3
+    pager.install();
-    asm volatile ( "mov %0, %%cr3" :: "r" (pml4) );
-    __sync_synchronize();
 
     try_or_raise(
         rs->set_virtual_address_map(

src/boot/memory.h

@@ -10,6 +10,11 @@ namespace uefi {
 }
 
 namespace boot {
+
+namespace paging {
+    class pager;
+}
+
 namespace memory {
 
 class efi_mem_map;
@@ -42,7 +47,7 @@ void mark_pointer_fixup(void **p);
 /// \arg pml4  The root page table for the new mappings
 /// \arg map   The UEFI memory map, used to update runtime services
 void virtualize(
-    void *pml4,
+    paging::pager &pager,
     efi_mem_map &map,
     uefi::runtime_services *rs);
 

src/boot/memory_map.cpp

@@ -278,7 +278,7 @@ build_frame_blocks(const util::counted<bootproto::mem_entry> &kmap)
 }
 
 void
-fix_frame_blocks(bootproto::args *args)
+fix_frame_blocks(bootproto::args *args, paging::pager &pager)
 {
     util::counted<frame_block> &blocks = args->frame_blocks;
 
@@ -290,8 +290,7 @@ fix_frame_blocks(bootproto::args *args)
     uintptr_t addr = reinterpret_cast<uintptr_t>(blocks.pointer);
 
     // Map the frame blocks to the appropriate address
-    paging::map_pages(args, addr,
+    pager.map_pages(addr, bootproto::mem::bitmap_offset, pages, true, false);
-        bootproto::mem::bitmap_offset, pages, true, false);
 
     uintptr_t offset = bootproto::mem::bitmap_offset - addr;
 

src/boot/memory_map.h

@@ -17,6 +17,11 @@ namespace bootproto {
 }
 
 namespace boot {
+
+namespace paging {
+    class pager;
+}
+
 namespace memory {
 
 /// Struct that represents UEFI's memory map. Contains a pointer to the map data
@@ -56,7 +61,7 @@ util::counted<bootproto::mem_entry> build_kernel_map(efi_mem_map &map);
 util::counted<bootproto::frame_block> build_frame_blocks(const util::counted<bootproto::mem_entry> &kmap);
 
 /// Map the frame allocation maps to the right spot and fix up pointers
-void fix_frame_blocks(bootproto::args *args);
+void fix_frame_blocks(bootproto::args *args, paging::pager &pager);
 
 } // namespace boot
 } // namespace memory

src/boot/paging.cpp

@@ -4,7 +4,6 @@
 #include <util/pointers.h>
 
 #include "allocator.h"
-#include "console.h"
 #include "error.h"
 #include "loader.h"
 #include "memory.h"
@@ -20,7 +19,7 @@ using memory::page_size;
 // Flags: 0 0 0 1  0 0 0 0  0 0 0 1 = 0x0101
 //         IGN  |  | | | |  | | | +- Present
 //              |  | | | |  | | +--- Writeable
-//              |  | | | |  | +----- Usermode access (supervisor only)
+//              |  | | | |  | +----- Usermode access (Supervisor only)
 //              |  | | | |  +------- PWT (determining memory type for page)
 //              |  | | | +---------- PCD (determining memory type for page)
 //              |  | | +------------ Accessed flag (not accessed yet)
@@ -33,7 +32,7 @@ constexpr uint64_t page_flags = 0x101;
 // Flags: 0  0 0 0 1  1 0 0 0  1 0 1 1 = 0x018b
 //        |   IGN  |  | | | |  | | | +- Present
 //        |        |  | | | |  | | +--- Writeable
-//        |        |  | | | |  | +----- Supervisor only
+//        |        |  | | | |  | +----- Usermode access (Supervisor only)
 //        |        |  | | | |  +------- PWT (determining memory type for page)
 //        |        |  | | | +---------- PCD (determining memory type for page)
 //        |        |  | | +------------ Accessed flag (not accessed yet)
@@ -57,17 +56,22 @@ constexpr uint64_t huge_page_flags = 0x18b;
 constexpr uint64_t table_flags = 0x003;
 
 
-inline void *
+/// Struct to allow easy accessing of a memory page being used as a page table.
-pop_pages(util::counted<void> &pages, size_t count)
+struct page_table
 {
-    if (count > pages.count)
+    uint64_t entries[512];
-        error::raise(uefi::status::out_of_resources, L"Page table cache empty", 0x7ab1e5);
 
-    void *next = pages.pointer;
+    inline page_table * get(int i, uint16_t *flags = nullptr) const {
-    pages.pointer = util::offset_pointer(pages.pointer, count*page_size);
+        uint64_t entry = entries[i];
-    pages.count -= count;
+        if ((entry & 1) == 0) return nullptr;
-    return next;
+        if (flags) *flags = entry & 0xfff;
-}
+        return reinterpret_cast<page_table *>(entry & ~0xfffull);
+    }
+
+    inline void set(int i, void *p, uint16_t flags) {
+        entries[i] = reinterpret_cast<uint64_t>(p) | (flags & 0xfff);
+    }
+};
 
 /// Iterator over page table entries.
 template <unsigned D = 4>
@@ -75,25 +79,23 @@ class page_entry_iterator
 {
 public:
     /// Constructor.
-    /// \arg virt    Virtual address this iterator is starting at
+    /// \arg virt      Virtual address this iterator is starting at
-    /// \arg pml4    Root of the page tables to iterate
+    /// \arg pgr       The pager, used for its page table cache
-    /// \arg pages   Cache of usable table pages
     page_entry_iterator(
             uintptr_t virt,
-            page_table *pml4,
+            pager &pgr) :
-            util::counted<void> &pages) :
+        m_pager(pgr)
-        m_pages(pages)
     {
-        m_table[0] = pml4;
+        m_table[0] = pgr.m_pml4;
         for (unsigned i = 0; i < D; ++i) {
             m_index[i] = static_cast<uint16_t>((virt >> (12 + 9*(3-i))) & 0x1ff);
             ensure_table(i);
         }
     }
 
-    uintptr_t vaddress() const {
+    uintptr_t vaddress(unsigned level = D) const {
         uintptr_t address = 0;
-        for (unsigned i = 0; i < D; ++i)
+        for (unsigned i = 0; i < level; ++i)
             address |= static_cast<uintptr_t>(m_index[i]) << (12 + 9*(3-i));
         if (address & (1ull<<47)) // canonicalize the address
             address |= (0xffffull<<48);
@@ -114,6 +116,7 @@ public:
     }
 
     uint64_t & operator*() { return entry(D-1); }
+    uint64_t operator[](int i) { return i < D ? m_index[i] : 0; }
 
 private:
     inline uint64_t & entry(unsigned level) { return m_table[level]->entries[m_index[level]]; }
@@ -129,7 +132,7 @@ private:
         uint64_t & parent_ent = entry(level - 1);
 
         if (!(parent_ent & 1)) {
-            page_table *table = reinterpret_cast<page_table*>(pop_pages(m_pages, 1));
+            page_table *table = reinterpret_cast<page_table*>(m_pager.pop_pages(1));
             parent_ent = (reinterpret_cast<uintptr_t>(table) & ~0xfffull) | table_flags;
             m_table[level] = table;
         } else {
@@ -137,44 +140,62 @@ private:
         }
     }
 
-    util::counted<void> &m_pages;
+    pager &m_pager;
     page_table *m_table[D];
     uint16_t m_index[D];
 };
 
-
+pager::pager(uefi::boot_services *bs) :
-static void
+    m_bs {bs}
-add_offset_mappings(page_table *pml4, util::counted<void> &pages)
 {
-    uintptr_t phys = 0;
+    status_line status(L"Allocating initial page tables");
-    uintptr_t virt = bootproto::mem::linear_offset; // Start of offset-mapped area
-    size_t page_count = 64 * 1024; // 64 TiB of 1 GiB pages
-    constexpr size_t GiB = 0x40000000ull;
 
-    page_entry_iterator<2> iterator{virt, pml4, pages};
+    // include 1 extra for kernel PML4
+    static constexpr size_t tables_needed = pd_tables + extra_tables + 1;
 
-    while (true) {
+    void *addr = g_alloc.allocate_pages(tables_needed, alloc_type::page_table, true);
-        *iterator = phys | huge_page_flags;
+    m_bs->set_mem(addr, tables_needed * page_size, 0);
-        if (--page_count == 0)
+    m_table_pages = { .pointer = addr, .count = tables_needed };
-            break;
 
-        iterator.increment();
+    create_kernel_tables();
-        phys += GiB;
-    }
-}
-
-static void
-add_kernel_pds(page_table *pml4, util::counted<void> &pages)
-{
-    constexpr unsigned start = arch::kernel_root_index; 
-    constexpr unsigned end = arch::table_entries; 
-
-    for (unsigned i = start; i < end; ++i)
-        pml4->set(i, pop_pages(pages, 1), table_flags);
 }
 
 void
-add_current_mappings(page_table *new_pml4)
+pager::map_pages(
+    uintptr_t phys, uintptr_t virt, size_t count,
+    bool write_flag, bool exe_flag)
+{
+    if (!count)
+        return;
+
+    page_entry_iterator<4> iterator {virt, *this};
+
+    uint64_t flags = page_flags;
+    if (!exe_flag)  flags |= (1ull << 63); // set NX bit
+    if (write_flag) flags |= 2;
+
+    while (true) {
+        uint64_t entry = phys | flags;
+        *iterator = entry;
+
+        if (--count == 0)
+            break;
+
+        iterator.increment();
+        phys += page_size;
+    }
+}
+
+void
+pager::update_kernel_args(bootproto::args *args)
+{
+    status_line status {L"Updating kernel args"};
+    args->pml4 = reinterpret_cast<void*>(m_pml4);
+    args->page_tables = m_table_pages;
+}
+
+void
+pager::add_current_mappings()
 {
     // Get the pointer to the current PML4
     page_table *old_pml4 = 0;
@@ -185,98 +206,49 @@ add_current_mappings(page_table *new_pml4)
     for (int i = 0; i < halfway; ++i) {
         uint64_t entry = old_pml4->entries[i];
         if (entry & 1)
-            new_pml4->entries[i] = entry;
+            m_pml4->entries[i] = entry;
     }
 }
 
-void
+page_table *
-allocate_tables(bootproto::args *args)
+pager::pop_pages(size_t count)
 {
-    status_line status(L"Allocating initial page tables");
+    if (count > m_table_pages.count)
+        error::raise(uefi::status::out_of_resources, L"Page table cache empty", 0x7ab1e5);
 
-    static constexpr size_t pd_tables = 256;   // number of pages for kernelspace PDs
+    page_table *next = reinterpret_cast<page_table*>(m_table_pages.pointer);
-    static constexpr size_t extra_tables = 64; // number of extra pages
+    m_table_pages.pointer = util::offset_pointer(m_table_pages.pointer, count*page_size);
-
+    m_table_pages.count -= count;
-    // number of pages for kernelspace PDs + PML4
+    return next;
-    static constexpr size_t kernel_tables = pd_tables + 1;
-
-    static constexpr size_t tables_needed = kernel_tables + extra_tables;
-
-    void *addr = g_alloc.allocate_pages(tables_needed, alloc_type::page_table, true);
-    page_table *pml4 = reinterpret_cast<page_table*>(addr);
-
-    args->pml4 = pml4;
-    args->page_tables = { .pointer = pml4 + 1, .count = tables_needed - 1 };
-
-    console::print(L"    First page (pml4) at: 0x%lx\r\n", pml4);
-
-    add_kernel_pds(pml4, args->page_tables);
-    add_offset_mappings(pml4, args->page_tables);
-
-    //console::print(L"    Set up initial mappings, %d spare tables.\r\n", args->table_count);
-}
-
-template <typename E>
-constexpr bool has_flag(E set, E flag) {
-    return
-        (static_cast<uint64_t>(set) & static_cast<uint64_t>(flag)) ==
-         static_cast<uint64_t>(flag);
 }
 
 void
-map_pages(
+pager::create_kernel_tables()
-    bootproto::args *args,
-    uintptr_t phys, uintptr_t virt,
-    size_t count, bool write_flag, bool exe_flag)
 {
-    if (!count)
+    m_pml4 = pop_pages(1);
-        return;
+    page_table *pds = pop_pages(pd_tables);
 
-    paging::page_table *pml4 =
+    // Add PDs for all of high memory into the PML4
-        reinterpret_cast<paging::page_table*>(args->pml4);
+    constexpr unsigned start = arch::kernel_root_index; 
+    constexpr unsigned end = arch::table_entries; 
+    for (unsigned i = start; i < end; ++i)
+        m_pml4->set(i, &pds[i - start], table_flags);
 
-    page_entry_iterator<4> iterator{virt, pml4, args->page_tables};
+    // Add the linear offset-mapped area
-
+    uintptr_t phys = 0;
-    uint64_t flags = page_flags;
+    uintptr_t virt_base = bootproto::mem::linear_offset;
-    if (!exe_flag)
+    size_t page_count = 64 * 1024; // 64 TiB of 1 GiB pages
-        flags |= (1ull << 63); // set NX bit
+    constexpr size_t GiB = 0x40000000ull;
-    if (write_flag)
-        flags |= 2;
 
+    page_entry_iterator<2> iterator {virt_base, *this};
     while (true) {
-        *iterator = phys | flags;
+        *iterator = phys | huge_page_flags;
-        if (--count == 0)
+        if (--page_count == 0)
             break;
-
         iterator.increment();
-        phys += page_size;
+        phys += GiB;
     }
 }
 
-void
-map_section(
-    bootproto::args *args,
-    const bootproto::program_section &section)
-{
-    using bootproto::section_flags;
-
-    map_pages(
-        args,
-        section.phys_addr,
-        section.virt_addr,
-        memory::bytes_to_pages(section.size),
-        has_flag(section.type, section_flags::write),
-        has_flag(section.type, section_flags::execute));
-}
-
-void
-map_program(
-    bootproto::args *args,
-    bootproto::program &program)
-{
-    for (auto &section : program.sections)
-        paging::map_section(args, section);
-}
-
 } // namespace paging
 } // namespace boot

src/boot/paging.h

@@ -3,60 +3,67 @@
 /// Page table structure and related definitions
 #include <stdint.h>
 #include <uefi/boot_services.h>
-#include <bootproto/kernel.h>
+
+namespace bootproto {
+    struct args;
+}
 
 namespace boot {
 namespace paging {
 
-/// Struct to allow easy accessing of a memory page being used as a page table.
+struct page_table;
-struct page_table
+
+class pager
 {
-    uint64_t entries[512];
+public:
+    static constexpr size_t pd_tables = 256;    // number of pages for kernelspace PDs
+    static constexpr size_t extra_tables = 64;  // number of extra pages
 
-    inline page_table * get(int i, uint16_t *flags = nullptr) const {
+    pager(uefi::boot_services *bs);
-        uint64_t entry = entries[i];
+
-        if ((entry & 1) == 0) return nullptr;
+    /// Map physical memory pages to virtual addresses in the given page tables.
-        if (flags) *flags = entry & 0xfff;
+    /// \arg phys       The physical address of the pages to map
-        return reinterpret_cast<page_table *>(entry & ~0xfffull);
+    /// \arg virt       The virtual address at which to map the pages
+    /// \arg count      The number of pages to map
+    /// \arg write_flag If true, mark the pages writeable
+    /// \arg exe_flag   If true, mark the pages executable
+    void map_pages(
+        uintptr_t phys,
+        uintptr_t virt,
+        size_t count,
+        bool write_flag,
+        bool exe_flag);
+
+    /// Update the kernel args structure before handing it off to the kernel
+    void update_kernel_args(bootproto::args *args);
+
+    /// Copy existing page table entries from the UEFI PML4 into our new PML4.
+    /// Does not do a deep copy - the new PML4 is updated to point to the
+    /// existing next-level page tables in the current PML4.
+    void add_current_mappings();
+
+    /// Write the pager's PML4 pointer to CR3
+    inline void install() const {
+        asm volatile ( "mov %0, %%cr3" :: "r" (m_pml4) );
+        __sync_synchronize();
     }
 
-    inline void set(int i, void *p, uint16_t flags) {
+private:
-        entries[i] = reinterpret_cast<uint64_t>(p) | (flags & 0xfff);
+    template <unsigned D>
-    }
+    friend class page_entry_iterator;
+
+    /// Get `count` table pages from the cache
+    page_table * pop_pages(size_t count);
+
+    /// Allocate the kernel PML4 and PD tables out of the cache, and add the
+    /// linear offset mappings to them
+    void create_kernel_tables();
+
+    uefi::boot_services *m_bs;
+    util::counted<void> m_table_pages;
+    page_table *m_pml4;
+    page_table *m_kernel_pds;
 };
 
-/// Allocate memory to be used for initial page tables. Initial offset-mapped
-/// page tables are pre-filled. All pages are saved as a module in kernel args
-/// and kernel args' `page_table_cache` and `num_free_tables` are updated with
-/// the leftover space.
-/// \arg args    The kernel args struct, used for the page table cache and pml4
-void allocate_tables(bootproto::args *args);
-
-/// Copy existing page table entries to a new page table. Does not do a deep
-/// copy - the new PML4 is updated to point to the existing next-level page
-/// tables in the current PML4.
-/// \arg new_pml4  The new PML4 to copy into
-void add_current_mappings(page_table *new_pml4);
-
-/// Map physical memory pages to virtual addresses in the given page tables.
-/// \arg args       The kernel args struct, used for the page table cache and pml4
-/// \arg phys       The physical address of the pages to map
-/// \arg virt       The virtual address at which to map the pages
-/// \arg count      The number of pages to map
-/// \arg write_flag If true, mark the pages writeable
-/// \arg exe_flag   If true, mark the pages executable
-void map_pages(
-    bootproto::args *args,
-    uintptr_t phys, uintptr_t virt,
-    size_t count, bool write_flag, bool exe_flag);
-
-/// Map the sections of a program in physical memory to their virtual memory
-/// addresses in the given page tables.
-/// \arg args    The kernel args struct, used for the page table cache and pml4
-/// \arg program The program to load
-void map_program(
-    bootproto::args *args,
-    bootproto::program &program);
-
 } // namespace paging
 } // namespace boot

src/kernel/assert.cpp

@@ -7,10 +7,10 @@ uint32_t *apic_icr = reinterpret_cast<uint32_t*>(0xffffc000fee00300);
 void const *symbol_table = nullptr;
 
 void
-install(uintptr_t entrypoint, const void *symbol_data)
+install(uintptr_t entrypoint, util::const_buffer symbol_data)
 {
     IDT::set_nmi_handler(entrypoint);
-    symbol_table = symbol_data;
+    symbol_table = symbol_data.pointer;
 }
 
 } // namespace panic

src/kernel/assert.h

@@ -1,6 +1,8 @@
 #pragma once
 
 #include <stdint.h>
+
+#include <util/counted.h>
 #include "cpu.h"
 
 namespace panic {
@@ -47,8 +49,8 @@ inline void panic(
 
 /// Install a panic handler.
 /// \arg entrypoint   Virtual address of the panic handler's entrypoint
-/// \arg symbol_data  Pointer to the symbol table data
+/// \arg symbol_data  Symbol table data
-void install(uintptr_t entrypoint, const void *symbol_data);
+void install(uintptr_t entrypoint, util::const_buffer symbol_data);
 
 } // namespace panic
 

src/kernel/kernel_main.cpp

@@ -32,9 +32,8 @@ void load_init_server(bootproto::program &program, uintptr_t modules_address);
 void
 kernel_main(bootproto::args *args)
 {
-    if (args->panic) {
+    if (args->panic_handler) {
-        const void *syms = util::offset_pointer(args->symbol_table, mem::linear_offset);
+        panic::install(args->panic_handler, args->symbol_table);
-        panic::install(args->panic->entrypoint, syms);
     }
 
     logger_init();
@@ -75,7 +74,7 @@ kernel_main(bootproto::args *args)
     smp::ready();
 
     // Load the init server
-    load_init_server(*args->init, args->modules);
+    load_init_server(args->init, args->init_modules);
 
     sched->start();
 }

src/kernel/panic.serial/panic.serial.ld

@@ -1,20 +1,36 @@
+PHDRS
+{
+    rodata PT_LOAD PHDRS FILEHDR FLAGS (4) /* read-only */;
+    text PT_LOAD ;
+    rwdata PT_LOAD ;
+    bss PT_LOAD ;
+}
+
+MEMORY
+{
+    panic (rwxa) : ORIGIN = 0xFFFF800080000000, LENGTH = 256M
+}
+
 ENTRY(_panic_entry)
+
 SECTIONS
 {
-	. = 0xFFFF800080000000;
+    .rodata ORIGIN(panic) + SIZEOF_HEADERS : {
-
-	.text ALIGN(4096) : {
-		*(.text*)
-	}
-
-	.data ALIGN(4096) : {
-		 *(.data*)
 		 *(.rodata*)
-	}
+    } :rodata
 
-	.bss ALIGN(4096) : {
+	.text ALIGN(4K) : {
+		*(.text*)
+	} :text
+
+	.data ALIGN(4K) : {
+		 *(.data*)
+		 *(.init_array*)
+	} :rwdata
+
+	.bss ALIGN(4K) : {
 		__bss_start = .;
 		*(.bss*)
 		__bss_end = .;
-	}
+	} :bss
 }

src/libraries/bootproto/bootproto/kernel.h

@@ -129,13 +129,13 @@ struct args
     util::counted<void> page_tables;
     util::counted<mem_entry> mem_map;
     util::counted<frame_block> frame_blocks;
-
-    program *kernel;
-    program *init;
-    program *panic;
     allocation_register *allocations;
-    uintptr_t modules;
+
-    void const *symbol_table;
+    uintptr_t panic_handler;
+    util::buffer symbol_table;
+
+    program init;
+    uintptr_t init_modules;
 
     void *runtime_services;
     void *acpi_table;

src/user/srv.init/init.ld

@@ -0,0 +1,43 @@
+PHDRS
+{
+    rodata PT_LOAD PHDRS FILEHDR FLAGS (4) /* read-only */;
+    text PT_LOAD ;
+    rwdata PT_LOAD ;
+    bss PT_LOAD ;
+}
+
+SECTIONS
+{
+    . = 0x20000000 + SIZEOF_HEADERS ;
+
+    .rodata : {
+		 *(.rodata*)
+    } :rodata
+
+	.text ALIGN(4K) : {
+		*(.text*)
+	} :text
+
+	.data ALIGN(4K) : {
+		 *(.data*)
+	} :rwdata
+
+    .init_array : {
+		 *(.init_array*)
+    } :rwdata
+
+    .got : {
+		 *(.got*)
+    } :rwdata
+
+	.bss ALIGN(4K) : {
+		__bss_start = .;
+		*(.bss*)
+		__bss_end = .;
+	} :bss
+
+    /DISCARD/ : {
+        *(.gcc_except_table*)
+        *(.eh_frame*)
+    }
+}

src/user/srv.init/init.module

@@ -4,6 +4,7 @@ init = module("srv.init",
     targets = [ "user" ],
     deps = [ "libc", "elf", "bootproto", "zstd" ],
     description = "Init server",
+    ld_script = "init.ld",
     sources = [
         "j6romfs.cpp",
         "loader.cpp",