[boot][kernel] Replace frame allocator with bitmap-based one

The previous frame allocator involved a lot of splitting and merging
linked lists and lost all information about frames while they were
allocated. The new allocator is based on an array of descriptor
structures and a bitmap. Each memory map region of allocatable memory
becomes one or more descriptors, each mapping up to 1GiB of physical
memory. The descriptors implement two levels of a bitmap tree, and have
a pointer into the large contiguous bitmap to track individual pages.

src/kernel/frame_allocator.cpp

@@ -2,20 +2,11 @@
 #include "kutil/assert.h"
 #include "kutil/memory.h"
 #include "frame_allocator.h"
+#include "kernel_args.h"
+#include "kernel_memory.h"
+#include "log.h"
 
 using memory::frame_size;
-using memory::page_offset;
-using frame_block_node = kutil::list_node<frame_block>;
-
-int
-frame_block::compare(const frame_block &rhs) const
-{
-	if (address < rhs.address)
-		return -1;
-	else if (address > rhs.address)
-		return 1;
-	return 0;
-}
 
 
 frame_allocator &
@@ -25,54 +16,142 @@ frame_allocator::get()
 	return g_frame_allocator;
 }
 
-frame_allocator::frame_allocator() {}
+frame_allocator::frame_allocator(kernel::args::frame_block *frames, size_t count) :
+	m_blocks(frames),
+	m_count(count)
+{
+}
+
+inline unsigned
+bsf(uint64_t v)
+{
+	asm ("tzcntq %q0, %q1" : "=r"(v) : "r"(v) : "cc");
+	return v;
+}
 
 size_t
 frame_allocator::allocate(size_t count, uintptr_t *address)
 {
-	kassert(!m_free.empty(), "frame_allocator::pop_frames ran out of free frames!");
-	if (m_free.empty())
-		return 0;
+	for (long i = m_count - 1; i >= 0; ++i) {
+		frame_block &block = m_blocks[i];
 
-	auto *first = m_free.front();
+		if (!block.map1)
+			continue;
+
+		// Tree walk to find the first available page
+		unsigned o1 = bsf(block.map1);
+
+		uint64_t m2 = block.map2[o1];
+		unsigned o2 = bsf(m2);
+
+		uint64_t m3 = block.bitmap[(o1 << 6) + o2];
+		unsigned o3 = bsf(m3);
+
+		unsigned frame = (o1 << 12) + (o2 << 6) + o3;
+
+		// See how many contiguous pages are here
+		unsigned n = bsf(~m3 >> o3);
+		if (n > count)
+			n = count;
+
+		*address = block.base + frame * frame_size;
+
+		// Clear the bits to mark these pages allocated
+		m3 &= ~(((1 << n) - 1) << o3);
+		block.bitmap[(o1 << 6) + o2] = m3;
+		if (!m3) {
+			// if that was it for this group, clear the next level bit
+			m2 &= ~(1 << o2);
+			block.map2[o1] = m2;
+
+			if (!m2) {
+				// if that was cleared too, update the top level
+				block.map1 &= ~(1 << o1);
+			}
+		}
 
-	if (count >= first->count) {
-		*address = first->address;
-		m_free.remove(first);
-		return first->count;
-	} else {
-		first->count -= count;
-		*address = first->address + (first->count * frame_size);
 		return count;
 	}
-}
 
-inline uintptr_t end(frame_block *node) { return node->address + node->count * frame_size; }
+	kassert(false, "frame_allocator ran out of free frames!");
+	return 0;
+}
 
 void
 frame_allocator::free(uintptr_t address, size_t count)
 {
 	kassert(address % frame_size == 0, "Trying to free a non page-aligned frame!");
 
-	frame_block_node *node =
-		reinterpret_cast<frame_block_node*>(address + page_offset);
+	if (!count)
+		return;
 
-	kutil::memset(node, 0, sizeof(frame_block_node));
-	node->address = address;
-	node->count = count;
+	for (long i = 0; i < m_count; ++i) {
+		frame_block &block = m_blocks[i];
+		uintptr_t end = block.base + block.count * frame_size;
 
-	m_free.sorted_insert(node);
+		if (address < block.base || address >= end)
+			continue;
 
-	frame_block_node *next = node->next();
-	if (next && end(node) == next->address) {
-		node->count += next->count;
-		m_free.remove(next);
-	}
+		uint64_t frame = (address - block.base) >> 12;
+		unsigned o1 = (frame >> 12) & 0x3f;
+		unsigned o2 = (frame >> 6) & 0x3f;
+		unsigned o3 = frame & 0x3f;
 
-	frame_block_node *prev = node->prev();
-	if (prev && end(prev) == address) {
-		prev->count += node->count;
-		m_free.remove(node);
+		while (count--) {
+			block.map1 |= (1 << o1);
+			block.map2[o1] |= (1 << o2);
+			block.bitmap[o2] |= (1 << o3);
+			if (++o3 == 64) {
+				o3 = 0;
+				if (++o2 == 64) {
+					o2 = 0;
+					++o1;
+					kassert(o1 < 64, "Tried to free pages past the end of a block");
+				}
+			}
+		}
+	}
+}
+
+void
+frame_allocator::used(uintptr_t address, size_t count)
+{
+	kassert(address % frame_size == 0, "Trying to mark a non page-aligned frame!");
+
+	if (!count)
+		return;
+
+	for (long i = 0; i < m_count; ++i) {
+		frame_block &block = m_blocks[i];
+		uintptr_t end = block.base + block.count * frame_size;
+
+		if (address < block.base || address >= end)
+			continue;
+
+		uint64_t frame = (address - block.base) >> 12;
+		unsigned o1 = (frame >> 12) & 0x3f;
+		unsigned o2 = (frame >> 6) & 0x3f;
+		unsigned o3 = frame & 0x3f;
+
+		while (count--) {
+			block.bitmap[o2] &= ~(1 << o3);
+			if (!block.bitmap[o2]) {
+				block.map2[o1] &= ~(1 << o2);
+
+				if (!block.map2[o1]) {
+					block.map1 &= ~(1 << o1);
+				}
+			}
+
+			if (++o3 == 64) {
+				o3 = 0;
+				if (++o2 == 64) {
+					o2 = 0;
+					++o1;
+					kassert(o1 < 64, "Tried to mark pages past the end of a block");
+				}
+			}
+		}
 	}
 }
 

src/kernel/frame_allocator.h

@@ -4,17 +4,21 @@
 
 #include <stdint.h>
 
-#include "kutil/linked_list.h"
-
-struct frame_block;
-using frame_block_list = kutil::linked_list<frame_block>;
+namespace kernel {
+namespace args {
+	struct frame_block;
+}}
 
 /// Allocator for physical memory frames
 class frame_allocator
 {
 public:
-	/// Default constructor
-	frame_allocator();
+	using frame_block = kernel::args::frame_block;
+
+	/// Constructor
+	/// \arg blocks The bootloader-supplied frame bitmap block list
+	/// \arg count  Number of entries in the block list
+	frame_allocator(frame_block *frames, size_t count);
 
 	/// Get free frames from the free list. Only frames from the first free block
 	/// are returned, so the number may be less than requested, but they will
@@ -29,26 +33,18 @@ public:
 	/// \arg count    The number of frames to be freed
 	void free(uintptr_t address, size_t count);
 
+	/// Mark frames as used
+	/// \arg address  The physical address of the first frame to free
+	/// \arg count    The number of frames to be freed
+	void used(uintptr_t address, size_t count);
+
 	/// Get the global frame allocator
 	static frame_allocator & get();
 
 private:
-	frame_block_list m_free; ///< Free frames list
+	frame_block *m_blocks;
+	long m_count;
 
+	frame_allocator() = delete;
 	frame_allocator(const frame_allocator &) = delete;
 };
-
-
-/// A block of contiguous frames. Each `frame_block` represents contiguous
-/// physical frames with the same attributes.
-struct frame_block
-{
-	uintptr_t address;
-	uint32_t count;
-
-	/// Compare two blocks by address.
-	/// \arg rhs   The right-hand comparator
-	/// \returns   <0 if this is sorts earlier, >0 if this sorts later, 0 for equal
-	int compare(const frame_block &rhs) const;
-};
-

src/kernel/main.cpp

@@ -37,8 +37,8 @@ extern void __kernel_assert(const char *, unsigned, const char *);
 
 /// Bootstrap the memory managers.
 void setup_pat();
-void memory_initialize_pre_ctors(kernel::args::header *kargs);
-void memory_initialize_post_ctors(kernel::args::header *kargs);
+void memory_initialize_pre_ctors(kernel::args::header &kargs);
+void memory_initialize_post_ctors(kernel::args::header &kargs);
 
 using namespace kernel;
 
@@ -92,9 +92,9 @@ kernel_main(args::header *header)
 	gdt_init();
 	interrupts_init();
 
-	memory_initialize_pre_ctors(header);
+	memory_initialize_pre_ctors(*header);
 	run_constructors();
-	memory_initialize_post_ctors(header);
+	memory_initialize_post_ctors(*header);
 
 	for (size_t i = 0; i < header->num_modules; ++i) {
 		args::module &mod = header->modules[i];

src/kernel/memory_bootstrap.cpp

@@ -14,15 +14,8 @@
 #include "objects/vm_area.h"
 #include "vm_space.h"
 
-using memory::frame_size;
 using memory::heap_start;
 using memory::kernel_max_heap;
-using memory::kernel_offset;
-using memory::heap_start;
-using memory::page_offset;
-using memory::pml4e_kernel;
-using memory::pml4e_offset;
-using memory::table_entries;
 
 using namespace kernel;
 
@@ -57,49 +50,68 @@ void operator delete (void *p) noexcept    { return g_kernel_heap.free(p); }
 void operator delete [] (void *p) noexcept { return g_kernel_heap.free(p); }
 
 namespace kutil {
-void * kalloc(size_t size) { return g_kernel_heap.allocate(size); }
-void kfree(void *p) { return g_kernel_heap.free(p); }
+	void * kalloc(size_t size) { return g_kernel_heap.allocate(size); }
+	void kfree(void *p) { return g_kernel_heap.free(p); }
 }
 
-/*
-void walk_page_table(
-	page_table *table,
-	page_table::level level,
-	uintptr_t &current_start,
-	size_t &current_bytes,
-	vm_area &karea)
+void
+memory_initialize_pre_ctors(args::header &kargs)
 {
-	constexpr size_t huge_page_size = (1ull<<30);
-	constexpr size_t large_page_size = (1ull<<21);
+	using kernel::args::frame_block;
 
-	for (unsigned i = 0; i < table_entries; ++i) {
-		page_table *next = table->get(i);
-		if (!next) {
-			if (current_bytes)
-				karea.commit(current_start, current_bytes);
-			current_start = 0;
-			current_bytes = 0;
-			continue;
-		} else if (table->is_page(level, i)) {
-			if (!current_bytes)
-				current_start = reinterpret_cast<uintptr_t>(next);
-			current_bytes +=
-				(level == page_table::level::pt
-					? frame_size
-					: level == page_table::level::pd
-						? large_page_size
-						: huge_page_size);
-		} else {
-			page_table::level deeper =
-				static_cast<page_table::level>(
-					static_cast<unsigned>(level) + 1);
+	new (&g_kernel_heap) kutil::heap_allocator {heap_start, kernel_max_heap};
 
-			walk_page_table(
-				next, deeper, current_start, current_bytes, kspace);
+	frame_block *blocks = reinterpret_cast<frame_block*>(memory::bitmap_start);
+	new (&g_frame_allocator) frame_allocator {blocks, kargs.frame_block_count};
+
+	// Mark all the things the bootloader allocated for us as used
+	g_frame_allocator.used(
+		reinterpret_cast<uintptr_t>(kargs.frame_blocks),
+		kargs.frame_block_pages);
+
+	g_frame_allocator.used(
+		reinterpret_cast<uintptr_t>(kargs.pml4),
+		kargs.table_pages);
+
+	for (unsigned i = 0; i < kargs.num_modules; ++i) {
+		const kernel::args::module &mod = kargs.modules[i];
+		g_frame_allocator.used(
+			reinterpret_cast<uintptr_t>(mod.location),
+			memory::page_count(mod.size));
+	}
+
+	for (unsigned i = 0; i < kargs.num_programs; ++i) {
+		const kernel::args::program &prog = kargs.programs[i];
+		for (auto &sect : prog.sections) {
+			if (!sect.size) continue;
+			g_frame_allocator.used(
+				sect.phys_addr,
+				memory::page_count(sect.size));
 		}
 	}
+
+	page_table *kpml4 = reinterpret_cast<page_table*>(kargs.pml4);
+	process *kp = process::create_kernel_process(kpml4);
+	vm_space &vm = kp->space();
+
+	vm_area *heap = new (&g_kernel_heap_area)
+		vm_area_open(kernel_max_heap, vm, vm_flags::write);
+
+	vm.add(heap_start, heap);
+}
+
+void
+memory_initialize_post_ctors(args::header &kargs)
+{
+	vm_space &vm = vm_space::kernel_space();
+	vm.add(memory::stacks_start, &g_kernel_stacks);
+	vm.add(memory::buffers_start, &g_kernel_buffers);
+
+	g_frame_allocator.free(
+		reinterpret_cast<uintptr_t>(kargs.page_tables),
+		kargs.table_count);
+
 }
-*/
 
 static void
 log_mtrrs()
@@ -166,60 +178,3 @@ setup_pat()
 }
 
 
-void
-memory_initialize_pre_ctors(args::header *kargs)
-{
-	new (&g_kernel_heap) kutil::heap_allocator {heap_start, kernel_max_heap};
-	new (&g_frame_allocator) frame_allocator;
-
-	args::mem_entry *entries = kargs->mem_map;
-	const size_t count = kargs->map_count;
-	for (unsigned i = 0; i < count; ++i) {
-		// TODO: use entry attributes
-		// TODO: copy anything we need from "pending" memory and free it
-		args::mem_entry &e = entries[i];
-		if (e.type == args::mem_type::free)
-			g_frame_allocator.free(e.start, e.pages);
-	}
-
-	page_table *kpml4 = reinterpret_cast<page_table*>(kargs->pml4);
-	process *kp = process::create_kernel_process(kpml4);
-	vm_space &vm = kp->space();
-
-	vm_area *heap = new (&g_kernel_heap_area)
-		vm_area_open(memory::kernel_max_heap, vm, vm_flags::write);
-
-	vm.add(memory::heap_start, heap);
-}
-
-void
-memory_initialize_post_ctors(args::header *kargs)
-{
-	/*
-	uintptr_t current_start = 0;
-	size_t current_bytes = 0;
-
-	// TODO: Should we exclude the top of this area? (eg, buffers, stacks, etc)
-	page_table *kpml4 = reinterpret_cast<page_table*>(kargs->pml4);
-	for (unsigned i = pml4e_kernel; i < pml4e_offset; ++i) {
-		page_table *pdp = kpml4->get(i);
-		kassert(pdp, "Bootloader did not create all kernelspace PDs");
-
-		walk_page_table(
-			pdp, page_table::level::pdp,
-			current_start, current_bytes,
-			g_kernel_space);
-	}
-
-	if (current_bytes)
-		g_kernel_space.commit(current_start, current_bytes);
-	*/
-	vm_space &vm = vm_space::kernel_space();
-	vm.add(memory::stacks_start, &g_kernel_stacks);
-	vm.add(memory::buffers_start, &g_kernel_buffers);
-
-	g_frame_allocator.free(
-		reinterpret_cast<uintptr_t>(kargs->page_tables),
-		kargs->table_count);
-
-}