[kernel] Get rid of page_manager

page_manager is dead - final uses replaced in vm_space (page_in and
clear). Removed the header and cpp, and other lingering references.

src/kernel/apic.cpp

@@ -5,7 +5,6 @@
 #include "io.h"
 #include "kernel_memory.h"
 #include "log.h"
-#include "page_manager.h"
 
 static constexpr uint16_t lapic_spurious   = 0x00f0;
 

src/kernel/buffer_cache.cpp

@@ -2,7 +2,6 @@
 #include "buffer_cache.h"
 #include "kernel_memory.h"
 #include "objects/vm_area.h"
-#include "page_manager.h"
 #include "vm_space.h"
 
 using memory::frame_size;

src/kernel/debug.cpp

@@ -4,7 +4,6 @@
 #include "gdt.h"
 #include "objects/process.h"
 #include "objects/thread.h"
-#include "page_manager.h"
 #include "symbol_table.h"
 
 size_t __counter_syscall_enter = 0;
@@ -18,6 +17,9 @@ print_regs(const cpu_state &regs)
 	uint64_t cr2 = 0;
 	__asm__ __volatile__ ("mov %%cr2, %0" : "=r"(cr2));
 
+	uintptr_t cr3 = 0;
+	__asm__ __volatile__ ( "mov %%cr3, %0" : "=r" (cr3) );
+
 	cons->printf("       process: %llx", bsp_cpu_data.p->koid());
 	cons->printf("   thread: %llx\n", bsp_cpu_data.t->koid());
 
@@ -44,7 +46,7 @@ print_regs(const cpu_state &regs)
 	print_regR("sp0", bsp_cpu_data.rsp0);
 
 	print_regL("rip", regs.rip);
-	print_regM("cr3", page_manager::get()->get_pml4());
+	print_regM("cr3", cr3);
 	print_regR("cr2", cr2);
 
 	cons->puts("\n");

src/kernel/device_manager.cpp

@@ -9,8 +9,8 @@
 #include "console.h"
 #include "device_manager.h"
 #include "interrupts.h"
+#include "kernel_memory.h"
 #include "log.h"
-#include "page_manager.h"
 
 
 static const char expected_signature[] = "RSD PTR ";
@@ -248,8 +248,6 @@ device_manager::load_mcfg(const acpi_mcfg *mcfg)
 	m_pci.set_size(count);
 	m_devices.set_capacity(16);
 
-	page_manager *pm = page_manager::get();
-
 	for (unsigned i = 0; i < count; ++i) {
 		const acpi_mcfg_entry &mcfge = mcfg->entries[i];
 

src/kernel/main.cpp

@@ -19,7 +19,6 @@
 #include "objects/channel.h"
 #include "objects/event.h"
 #include "objects/handle.h"
-#include "page_manager.h"
 #include "scheduler.h"
 #include "serial.h"
 #include "symbol_table.h"

src/kernel/memory_bootstrap.cpp

@@ -11,7 +11,6 @@
 #include "log.h"
 #include "objects/process.h"
 #include "objects/vm_area.h"
-#include "page_manager.h"
 #include "vm_space.h"
 
 using memory::frame_size;
@@ -33,9 +32,6 @@ using namespace kernel;
 static kutil::no_construct<kutil::heap_allocator> __g_kernel_heap_storage;
 kutil::heap_allocator &g_kernel_heap = __g_kernel_heap_storage.value;
 
-static kutil::no_construct<page_manager> __g_page_manager_storage;
-page_manager &g_page_manager = __g_page_manager_storage.value;
-
 static kutil::no_construct<frame_allocator> __g_frame_allocator_storage;
 frame_allocator &g_frame_allocator = __g_frame_allocator_storage.value;
 
@@ -106,9 +102,6 @@ memory_initialize_pre_ctors(args::header *kargs)
 			g_frame_allocator.free(e.start, e.pages);
 	}
 
-	// Create the page manager
-	new (&g_page_manager) page_manager {g_frame_allocator, kpml4};
-
 	process *kp = process::create_kernel_process(kpml4);
 	vm_space &vm = kp->space();
 	vm.allow(memory::heap_start, memory::kernel_max_heap, true);

src/kernel/objects/endpoint.cpp

@@ -1,7 +1,6 @@
 #include "objects/endpoint.h"
 #include "objects/process.h"
 #include "objects/thread.h"
-#include "page_manager.h"
 #include "scheduler.h"
 #include "vm_space.h"
 
@@ -87,7 +86,6 @@ endpoint::do_message_copy(const endpoint::thread_data &sender, endpoint::thread_
 	if (sender.len > *receiver.len_p)
 		return j6_err_insufficient;
 
-	page_manager *pm = page_manager::get();
 	vm_space &source = sender.th->parent().space();
 	vm_space &dest = receiver.th->parent().space();
 	vm_space::copy(source, dest, sender.data, receiver.data, sender.len);

src/kernel/objects/process.cpp

@@ -4,7 +4,6 @@
 #include "cpu.h"
 #include "objects/process.h"
 #include "objects/thread.h"
-#include "page_manager.h"
 
 // This object is initialized _before_ global constructors are called,
 // so we don't want it to have a global constructor at all, lest it

src/kernel/objects/vm_area.cpp

@@ -51,7 +51,7 @@ vm_area::remove_from(vm_space *space)
 	if (space && base) {
 		for (auto &m : m_mappings)
 			if (m.state == state::mapped)
-				space->page_out(*base + m.offset, m.count);
+				space->clear(*base + m.offset, m.count);
 		m_procs.erase(space);
 	}
 	return j6_status_ok;
@@ -272,7 +272,7 @@ vm_area::unmap(uintptr_t offset, size_t count)
 	for (auto &it : m_procs) {
 		uintptr_t addr = it.val + offset;
 		vm_space *space = it.key;
-		space->page_out(addr, count);
+		space->clear(addr, count);
 	}
 }
 

src/kernel/page_manager.cpp

@@ -1,189 +0,0 @@
-#include "kutil/assert.h"
-#include "console.h"
-#include "io.h"
-#include "log.h"
-#include "objects/process.h"
-#include "objects/vm_area.h"
-#include "page_manager.h"
-#include "vm_space.h"
-
-using memory::frame_size;
-using memory::heap_start;
-using memory::kernel_max_heap;
-using memory::kernel_offset;
-using memory::page_offset;
-using memory::page_mappable;
-using memory::pml4e_kernel;
-using memory::table_entries;
-
-// NB: in 4KiB page table entries, bit 7 isn't pagesize but PAT. Currently this
-// doesn't matter, becasue in the default PAT table, both 000 and 100 are WB.
-constexpr uint64_t sys_page_flags   = 0x183; // global, pagesize, write, present
-constexpr uint64_t sys_table_flags  = 0x003; // write, present
-constexpr uint64_t user_page_flags  = 0x087; // pagesize, user, write, present
-constexpr uint64_t user_table_flags = 0x007; // user, write, present
-
-static uintptr_t
-pt_to_phys(page_table *pt)
-{
-	return reinterpret_cast<uintptr_t>(pt) - page_offset;
-}
-
-
-static page_table *
-pt_from_phys(uintptr_t p)
-{
-	return reinterpret_cast<page_table *>((p + page_offset) & ~0xfffull);
-}
-
-
-
-
-page_manager::page_manager(frame_allocator &frames, page_table *pml4) :
-	m_kernel_pml4(pml4),
-	m_frames(frames)
-{
-}
-
-void
-page_manager::dump_pml4(page_table *pml4, bool recurse)
-{
-	if (pml4 == nullptr) pml4 = get_pml4();
-	pml4->dump(page_table::level::pml4, recurse);
-}
-
-void
-page_manager::check_needs_page(page_table *table, unsigned index, bool user)
-{
-	if ((table->entries[index] & 0x1) == 1) return;
-
-	page_table *new_table = page_table::get_table_page();
-	for (int i=0; i<table_entries; ++i) new_table->entries[i] = 0;
-	table->entries[index] = pt_to_phys(new_table) | (user ? user_table_flags : sys_table_flags);
-}
-
-void
-page_manager::page_in(page_table *pml4, uintptr_t phys_addr, uintptr_t virt_addr, size_t count, bool user, bool large)
-{
-	/*
-	log::debug(logs::paging, "page_in for table %016lx p:%016lx v:%016lx c:%4d u:%d l:%d",
-			pml4, phys_addr, virt_addr, count, user, large);
-	*/
-
-	page_table_indices idx{virt_addr};
-	page_table *tables[4] = {pml4, nullptr, nullptr, nullptr};
-
-	uint64_t flags = user ? user_table_flags : sys_table_flags;
-
-	for (; idx[0] < table_entries; idx[0] += 1) {
-		check_needs_page(tables[0], idx[0], user);
-		tables[1] = tables[0]->get(idx[0]);
-
-		for (; idx[1] < table_entries; idx[1] += 1, idx[2] = 0, idx[3] = 0) {
-			check_needs_page(tables[1], idx[1], user);
-			tables[2] = tables[1]->get(idx[1]);
-
-			for (; idx[2] < table_entries; idx[2] += 1, idx[3] = 0) {
-				if (large &&
-					idx[3] == 0 &&
-					count >= table_entries &&
-					tables[2]->get(idx[2]) == nullptr) {
-					// Do a 2MiB page instead
-					tables[2]->entries[idx[2]] = phys_addr | flags | 0x80;
-					phys_addr += frame_size * table_entries;
-					count -= table_entries;
-					if (count == 0) return;
-					continue;
-				}
-
-				check_needs_page(tables[2], idx[2], user);
-				tables[3] = tables[2]->get(idx[2]);
-
-				for (; idx[3] < table_entries; idx[3] += 1) {
-					tables[3]->entries[idx[3]] = phys_addr | flags;
-					phys_addr += frame_size;
-					if (--count == 0) return;
-				}
-			}
-		}
-	}
-
-	kassert(0, "Ran to end of page_in");
-}
-
-void
-page_manager::page_out(page_table *pml4, uintptr_t virt_addr, size_t count, bool free)
-{
-	page_table_indices idx{virt_addr};
-	page_table *tables[4] = {pml4, nullptr, nullptr, nullptr};
-
-	uintptr_t free_start = 0;
-	unsigned free_count = 0;
-
-	for (; idx[0] < table_entries; idx[0] += 1) {
-		page_table *table = tables[0]->get(idx[0]);
-		if (!table) {
-			constexpr size_t skip = 512 * 512 * 512;
-			if (count > skip) {
-				count -= skip;
-				continue;
-			}
-			goto page_out_end;
-		}
-
-		tables[1] = table;
-
-		for (; idx[1] < table_entries; idx[1] += 1) {
-			page_table *table = tables[1]->get(idx[1]);
-			if (!table) {
-				constexpr size_t skip = 512 * 512;
-				if (count > skip) {
-					count -= skip;
-					continue;
-				}
-				goto page_out_end;
-			}
-
-			tables[2] = table;
-
-			for (; idx[2] < table_entries; idx[2] += 1) {
-				page_table *table = tables[2]->get(idx[2]);
-				if (!table) {
-					constexpr size_t skip = 512;
-					if (count > skip) {
-						count -= skip;
-						continue;
-					}
-					goto page_out_end;
-				}
-
-				tables[3] = table;
-
-				for (; idx[3] < table_entries; idx[3] += 1) {
-					uintptr_t entry = tables[3]->entries[idx[3]];
-					bool present = entry & 1;
-
-					if (present) {
-						entry &= ~0xfffull;
-						if (!free_count || entry != free_start + free_count * frame_size) {
-							if (free_count && free) m_frames.free(free_start, free_count);
-							free_start = tables[3]->entries[idx[3]] & ~0xfffull;
-							free_count = 1;
-						} else {
-							free_count++;
-						}
-
-						tables[3]->entries[idx[3]] = 0;
-					}
-
-					if (--count == 0)
-						goto page_out_end;
-				}
-			}
-		}
-	}
-
-page_out_end:
-	if (free && free_count)
-		m_frames.free(free_start, free_count);
-}

src/kernel/page_manager.h

@@ -1,133 +0,0 @@
-#pragma once
-/// \file page_manager.h
-/// The page memory manager and related definitions.
-
-#include <stddef.h>
-#include <stdint.h>
-
-#include "kutil/enum_bitfields.h"
-#include "kutil/linked_list.h"
-#include "kutil/memory.h"
-#include "frame_allocator.h"
-#include "kernel_memory.h"
-#include "page_table.h"
-
-struct free_page_header;
-
-/// Manager for allocation and mapping of pages
-class page_manager
-{
-public:
-	/// Constructor.
-	/// \arg frames  The frame allocator to get physical frames from
-	/// \arg pml4    The initial kernel-space pml4
-	page_manager(frame_allocator &frames, page_table *pml4);
-
-	/// Helper to get the number of pages needed for a given number of bytes.
-	/// \arg bytes  The number of bytes desired
-	/// \returns    The number of pages needed to contain the desired bytes
-	static inline size_t page_count(size_t bytes)
-	{
-		return (bytes - 1) / memory::frame_size + 1;
-	}
-
-	/// Helper to read the PML4 table from CR3.
-	/// \returns  A pointer to the current PML4 table.
-	static inline page_table * get_pml4()
-	{
-		uintptr_t pml4 = 0;
-		__asm__ __volatile__ ( "mov %%cr3, %0" : "=r" (pml4) );
-		return reinterpret_cast<page_table *>((pml4 & ~0xfffull) + memory::page_offset);
-	}
-
-	/// Helper to set the PML4 table pointer in CR3.
-	/// \arg pml4  A pointer to the PML4 table to install.
-	static inline void set_pml4(page_table *pml4)
-	{
-		constexpr uint64_t phys_mask = ~memory::page_offset & ~0xfffull;
-		uintptr_t p = reinterpret_cast<uintptr_t>(pml4) & phys_mask;
-		__asm__ __volatile__ ( "mov %0, %%cr3" :: "r" (p) );
-	}
-
-	/// Dump the given or current PML4 to the console
-	/// \arg pml4     The page table to use, null for the current one
-	/// \arg recurse  Whether to print sub-tables
-	void dump_pml4(page_table *pml4 = nullptr, bool recurse = true);
-
-	/// Get the system page manager.
-	/// \returns  A pointer to the system page manager
-	static page_manager * get();
-
-	/// Get a pointer to the kernel's PML4
-	inline page_table * get_kernel_pml4() { return m_kernel_pml4; }
-
-private:
-	/// Helper function to allocate a new page table. If table entry `i` in
-	/// table `base` is empty, allocate a new page table and point `base[i]` at
-	/// it.
-	/// \arg base  Existing page table being indexed into
-	/// \arg i     Index into the existing table to check
-	/// \art user  True if this is a userspace mapping
-	void check_needs_page(page_table *base, unsigned i, bool user);
-
-	/// Low-level routine for mapping a number of pages into the given page table.
-	/// \arg pml4       The root page table to map into
-	/// \arg phys_addr  The starting physical address of the pages to be mapped
-	/// \arg virt_addr  The starting virtual address ot the memory to be mapped
-	/// \arg count      The number of pages to map
-	/// \arg user       True if this is a userspace mapping
-	/// \arg large      Whether to allow large pages
-	void page_in(
-			page_table *pml4,
-			uintptr_t phys_addr,
-			uintptr_t virt_addr,
-			size_t count,
-			bool user = false,
-			bool large = false);
-
-	/// Low-level routine for unmapping a number of pages from the given page table.
-	/// \arg pml4       The root page table for this mapping
-	/// \arg virt_addr  The starting virtual address ot the memory to be unmapped
-	/// \arg count      The number of pages to unmap
-	/// \arg free       Whether to return the pages to the frame allocator
-	void page_out(
-			page_table *pml4,
-			uintptr_t virt_addr,
-			size_t count,
-			bool free = false);
-
-	page_table *m_kernel_pml4; ///< The PML4 of just kernel pages
-
-	frame_allocator &m_frames;
-
-	friend class memory_bootstrap;
-	friend class vm_space;
-	page_manager(const page_manager &) = delete;
-};
-
-/// Global page manager.
-extern page_manager &g_page_manager;
-
-inline page_manager * page_manager::get() { return &g_page_manager; }
-
-
-/// Calculate a page-aligned address.
-/// \arg p    The address to align.
-/// \returns  The next page-aligned address _after_ `p`.
-template <typename T> inline T
-page_align(T p)
-{
-	return reinterpret_cast<T>(
-		((reinterpret_cast<uintptr_t>(p) - 1) & ~(memory::frame_size - 1))
-		+ memory::frame_size);
-}
-
-/// Calculate a page-table-aligned address. That is, an address that is
-/// page-aligned to the first page in a page table.
-/// \arg p    The address to align.
-/// \returns  The next page-table-aligned address _after_ `p`.
-template <typename T> inline T
-page_table_align(T p)
-{
-	return ((p - 1) & ~0x1fffffull) + 0x200000;
-}

src/kernel/page_table.h

@@ -7,7 +7,6 @@
 #include "kernel_memory.h"
 
 struct free_page_header;
-class page_manager;
 
 /// Struct to allow easy accessing of a memory page being used as a page table.
 struct page_table

src/kernel/scheduler.cpp

@@ -13,7 +13,6 @@
 #include "msr.h"
 #include "objects/channel.h"
 #include "objects/process.h"
-#include "page_manager.h"
 #include "scheduler.h"
 
 #include "elf/elf.h"
@@ -41,7 +40,6 @@ scheduler::scheduler(lapic *apic) :
 	kassert(!s_instance, "Multiple schedulers created!");
 	s_instance = this;
 
-	page_table *pml4 = page_manager::get_pml4();
 	process *kp = &process::kernel_process();
 
 	log::debug(logs::task, "Kernel process koid %llx", kp->koid());
@@ -50,7 +48,6 @@ scheduler::scheduler(lapic *apic) :
 		reinterpret_cast<uintptr_t>(&idle_stack_end));
 
 	log::debug(logs::task, "Idle thread koid %llx", idle->koid());
-	log::debug(logs::task, "Kernel PML4 %llx", pml4);
 
 	auto *tcb = idle->tcb();
 	m_runlists[max_priority].push_back(tcb);
@@ -85,7 +82,7 @@ load_process_image(const void *image_start, size_t bytes, TCB *tcb)
 
 		uintptr_t aligned = header->vaddr & ~(memory::frame_size - 1);
 		size_t size = (header->vaddr + header->mem_size) - aligned;
-		size_t pagesize = page_manager::page_count(size) * memory::frame_size;
+		size_t pagesize = memory::page_count(size) * memory::frame_size;
 
 		log::debug(logs::loader, "  Loadable segment %02u: vaddr %016lx  size %016lx",
 			i, header->vaddr, header->mem_size);

src/kernel/vm_space.cpp

@@ -3,7 +3,6 @@
 #include "objects/process.h"
 #include "objects/thread.h"
 #include "objects/vm_area.h"
-#include "page_manager.h"
 #include "vm_space.h"
 
 extern frame_allocator &g_frame_allocator;
@@ -93,17 +92,32 @@ vm_space::get(uintptr_t addr, uintptr_t *base)
 }
 
 void
-vm_space::page_in(uintptr_t addr, size_t count, uintptr_t phys)
+vm_space::page_in(uintptr_t virt, uintptr_t phys, size_t count)
 {
-	page_manager *pm = page_manager::get();
+	page_table::iterator it {virt, m_pml4};
-	pm->page_in(m_pml4, phys, addr, count, is_kernel());
+	for (size_t i = 0; i < count; ++i) {
+		uint64_t &e = it.entry(page_table::level::pt);
+		bool allowed = (e & page_table::flag::allowed);
+		e = (phys + i * memory::frame_size) |
+			(allowed ? page_table::flag::allowed : page_table::flag::none);
+		++it;
+	}
 }
 
 void
-vm_space::page_out(uintptr_t addr, size_t count)
+vm_space::clear(uintptr_t addr, size_t count)
 {
-	page_manager *pm = page_manager::get();
+	page_table::iterator it {addr, m_pml4};
-	pm->page_out(m_pml4, addr, count, false);
+	while (count--) {
+		uint64_t &e = it.entry(page_table::level::pt);
+		if (e & page_table::flag::present) {
+			g_frame_allocator.free(e & ~0xfffull, 1);
+		}
+		bool allowed = (e & page_table::flag::allowed);
+		e = 0;
+		if (allowed) e |= page_table::flag::allowed;
+		++it;
+	}
 }
 
 void

src/kernel/vm_space.h

@@ -47,16 +47,16 @@ public:
 	/// Get the kernel virtual memory space
 	static vm_space & kernel_space();
 
-	/// Add page mappings into this space's page tables
+	/// Copy a range of mappings from the given address space
-	/// \arg addr   The virtual address to map at
+	/// \arg virt  The starting virutal address
-	/// \arg count  The number of pages
+	/// \arg phys  The starting physical address
-	/// \arg phys   The physical address of the first page
+	/// \arg count The number of contiugous physical pages to map
-	void page_in(uintptr_t addr, size_t count, uintptr_t phys);
+	void page_in(uintptr_t virt, uintptr_t phys, size_t count);
 
-	/// Remove page mappings from this space's page tables
+	/// Clear mappings from the given region
-	/// \arg addr   The virtual address to unmap
+	/// \arg start  The starting virutal address to clear
-	/// \arg count  The number of pages
+	/// \arg count  The number of pages worth of mappings to clear
-	void page_out(uintptr_t addr, size_t count);
+	void clear(uintptr_t start, size_t count);
 
 	/// Mark whether allocation is allowed or not in a range of
 	/// virtual memory.