[kernel] Move kernel stacks out of the heap

We were previously allocating kernel stacks as large objects on the
heap. Now keep track of areas of the kernel stack area that are in use,
and allocate them from there. Also required actually implementing
vm_space::commit(). This still needs more work.

modules.yaml

@@ -46,6 +46,7 @@ modules:
             - src/kernel/scheduler.cpp
             - src/kernel/screen.cpp
             - src/kernel/serial.cpp
+            - src/kernel/stack_cache.cpp
             - src/kernel/syscall.cpp
             - src/kernel/syscall.s
             - src/kernel/syscalls/channel.cpp

src/include/kernel_memory.h

@@ -16,8 +16,8 @@ namespace memory {
 	/// Offset from physical where page tables are mapped.
 	constexpr uintptr_t page_offset = 0xffffc00000000000;
 
-	/// Initial process thread's stack size, in pages
-	constexpr unsigned initial_stack_pages = 1;
+	/// Number of pages for a kernel stack
+	constexpr unsigned kernel_stack_pages = 1;
 
 	/// Max size of the kernel heap
 	constexpr size_t kernel_max_heap = 0x8000000000; // 512GiB
@@ -25,6 +25,12 @@ namespace memory {
 	/// Start of the kernel heap
 	constexpr uintptr_t heap_start = page_offset - kernel_max_heap;
 
+	/// Start of the kernel stacks
+	constexpr uintptr_t stacks_start = heap_start - kernel_max_heap;
+
+	/// Max size of the kernel stacks area
+	constexpr size_t kernel_max_stacks = 0x8000000000; // 512GiB
+
 	/// First kernel space PML4 entry
 	constexpr unsigned pml4e_kernel = 256;
 

src/kernel/memory_bootstrap.cpp

@@ -24,7 +24,8 @@ using memory::table_entries;
 
 using namespace kernel;
 
-kutil::vm_space g_kernel_space;
+kutil::vm_space g_kernel_space {kernel_offset, (page_offset-kernel_offset)};
+
 
 // These objects are initialized _before_ global constructors are called,
 // so we don't want them to have global constructors at all, lest they
@@ -61,6 +62,7 @@ void walk_page_table(
 	for (unsigned i = 0; i < table_entries; ++i) {
 		page_table *next = table->get(i);
 		if (!next) {
+			if (current_bytes)
 				kspace.commit(current_start, current_bytes);
 			current_start = 0;
 			current_bytes = 0;
@@ -109,10 +111,6 @@ memory_initialize_pre_ctors(args::header *kargs)
 void
 memory_initialize_post_ctors(args::header *kargs)
 {
-	new (&g_kernel_space) kutil::vm_space {
-		kernel_offset,
-		(page_offset-kernel_offset)};
-
 	uintptr_t current_start = 0;
 	size_t current_bytes = 0;
 
@@ -122,6 +120,7 @@ memory_initialize_post_ctors(args::header *kargs)
 		page_table *pdp = kpml4->get(i);
 
 		if (!pdp) {
+			if (current_bytes)
 				g_kernel_space.commit(current_start, current_bytes);
 			current_start = 0;
 			current_bytes = 0;

src/kernel/objects/thread.cpp

@@ -3,6 +3,7 @@
 #include "objects/thread.h"
 #include "objects/process.h"
 #include "scheduler.h"
+#include "stack_cache.h"
 
 extern "C" void kernel_to_user_trampoline();
 static constexpr j6_signal_t thread_default_signals = 0;
@@ -15,9 +16,8 @@ thread::thread(process &parent, uint8_t pri, bool user) :
 	m_wait_data(0),
 	m_wait_obj(0)
 {
-	TCB *tcbp = tcb();
-	tcbp->pml4 = parent.pml4();
-	tcbp->priority = pri;
+	m_tcb.pml4 = parent.pml4();
+	m_tcb.priority = pri;
 	setup_kernel_stack();
 	set_state(state::ready);
 }
@@ -30,16 +30,15 @@ thread::thread(process &parent, uint8_t pri, uintptr_t rsp0) :
 	m_wait_data(0),
 	m_wait_obj(0)
 {
-	TCB *tcbp = tcb();
-	tcbp->pml4 = parent.pml4();
-	tcbp->priority = pri;
-	tcbp->rsp0 = rsp0;
+	m_tcb.pml4 = parent.pml4();
+	m_tcb.priority = pri;
+	m_tcb.rsp0 = rsp0;
 	set_state(state::ready);
 }
 
 thread::~thread()
 {
-	kutil::kfree(reinterpret_cast<void*>(m_tcb.kernel_stack));
+	stack_cache::get().return_stack(m_tcb.kernel_stack);
 }
 
 thread *
@@ -153,27 +152,25 @@ thread::add_thunk_user(uintptr_t rip)
 void
 thread::setup_kernel_stack()
 {
-	constexpr size_t initial_stack_size = 0x1000;
+	using memory::frame_size;
+	using memory::kernel_stack_pages;
+	static constexpr size_t stack_bytes = kernel_stack_pages * frame_size;
+
 	constexpr unsigned null_frame_entries = 2;
 	constexpr size_t null_frame_size = null_frame_entries * sizeof(uint64_t);
 
-	void *stack_bottom = kutil::kalloc(initial_stack_size);
-	kutil::memset(stack_bottom, 0, initial_stack_size);
+	uintptr_t stack_addr = stack_cache::get().get_stack();
+	uintptr_t stack_end = stack_addr + stack_bytes;
 
-	log::debug(logs::memory, "Created kernel stack at %016lx size 0x%lx",
-			stack_bottom, initial_stack_size);
-
-	void *stack_top =
-		kutil::offset_pointer(stack_bottom,
-				initial_stack_size - null_frame_size);
-
-	uint64_t *null_frame = reinterpret_cast<uint64_t*>(stack_top);
+	uint64_t *null_frame = reinterpret_cast<uint64_t*>(stack_end - null_frame_size);
 	for (unsigned i = 0; i < null_frame_entries; ++i)
 		null_frame[i] = 0;
 
-	m_tcb.kernel_stack_size = initial_stack_size;
-	m_tcb.kernel_stack = reinterpret_cast<uintptr_t>(stack_bottom);
-	m_tcb.rsp0 = reinterpret_cast<uintptr_t>(stack_top);
+	log::debug(logs::memory, "Created kernel stack at %016lx size 0x%lx",
+			stack_addr, stack_bytes);
+
+	m_tcb.kernel_stack = stack_addr;
+	m_tcb.rsp0 = reinterpret_cast<uintptr_t>(null_frame);
 	m_tcb.rsp = m_tcb.rsp0;
 }
 

src/kernel/objects/thread.h

@@ -23,7 +23,6 @@ struct TCB
 	// TODO: move state into TCB?
 
 	uintptr_t kernel_stack;
-	size_t kernel_stack_size;
 
 	uint32_t time_left;
 	uint64_t last_ran;

src/kernel/page_manager.cpp

@@ -325,6 +325,10 @@ page_manager::fault_handler(uintptr_t addr)
 	bool user = addr < kernel_offset;
 	map_pages(page, 1, user);
 
+	// Kernel stacks: zero them upon mapping them
+	if (addr >= memory::stacks_start && addr < memory::heap_start)
+		kutil::memset(reinterpret_cast<void*>(page), 0, memory::frame_size);
+
 	return true;
 }
 

src/kernel/stack_cache.cpp

@@ -0,0 +1,41 @@
+#include "kutil/vm_space.h"
+#include "kernel_memory.h"
+#include "page_manager.h"
+#include "stack_cache.h"
+
+extern kutil::vm_space g_kernel_space;
+
+using memory::frame_size;
+using memory::kernel_stack_pages;
+static constexpr size_t stack_bytes = kernel_stack_pages * frame_size;
+
+stack_cache stack_cache::s_instance(memory::stacks_start, memory::kernel_max_stacks);
+
+stack_cache::stack_cache(uintptr_t start, size_t size) :
+	m_next(start), m_end(start+size)
+{
+}
+
+uintptr_t
+stack_cache::get_stack()
+{
+	uintptr_t stack = 0;
+	if (m_cache.count() > 0) {
+		stack = m_cache.pop();
+	} else {
+		stack = m_next;
+		m_next += stack_bytes;
+	}
+
+	g_kernel_space.commit(stack, stack_bytes);
+	return stack;
+}
+
+void
+stack_cache::return_stack(uintptr_t addr)
+{
+	void *ptr = reinterpret_cast<void*>(addr);
+	page_manager::get()->unmap_pages(ptr, kernel_stack_pages);
+	g_kernel_space.unreserve(addr, stack_bytes);
+	m_cache.append(addr);
+}

src/kernel/stack_cache.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <kutil/vector.h>
+
+/// A cache of kernel stack address ranges
+class stack_cache
+{
+public:
+	/// Constructor.
+	/// \args start  Start of virtual memory area to contain stacks
+	/// \args size   Size of virtual memory area in bytes
+	stack_cache(uintptr_t start, size_t size);
+
+	/// Get an available stack address
+	uintptr_t get_stack();
+
+	/// Return a stack address to the available pool
+	void return_stack(uintptr_t addr);
+
+	static stack_cache & get() { return s_instance; }
+
+private:
+	kutil::vector<uintptr_t> m_cache;
+	uintptr_t m_next;
+	const uintptr_t m_end;
+	static stack_cache s_instance;
+};

src/libraries/kutil/vm_space.cpp

@@ -184,7 +184,23 @@ vm_space::unreserve(uintptr_t start, size_t size)
 uintptr_t
 vm_space::commit(uintptr_t start, size_t size)
 {
+	log::debug(logs::vmem, "Committing region %016llx-%016llx", start, start+size);
+
+	node_type *node = find_overlapping(m_ranges.root(), start, size);
+	if (!node) {
+		log::debug(logs::vmem, "  found no match");
 		return 0;
+	} else if (node->state == vm_state::unknown || node->state == vm_state::mapped) {
+		log::debug(logs::vmem, "  found wrong state %016llx-%016llx[%d]",
+			node->address, node->address + node->size, node->state);
+		return 0;
+	}
+
+	if (node->address <= start && node->size >= size && node->state == vm_state::committed)
+		return start;
+
+	node = split_out(node, start, size, vm_state::committed);
+	return node ? start : 0;
 }
 
 void