From 579eaaf4a01a3b49e31a5e92fca5fb4241d4ea62 Mon Sep 17 00:00:00 2001 From: "Justin C. Miller" Date: Sun, 2 Aug 2020 18:15:28 -0700 Subject: [PATCH] [kernel] Move kernel stacks out of the heap We were previously allocating kernel stacks as large objects on the heap. Now keep track of areas of the kernel stack area that are in use, and allocate them from there. Also required actually implementing vm_space::commit(). This still needs more work. --- modules.yaml | 1 + src/include/kernel_memory.h | 10 ++++++-- src/kernel/memory_bootstrap.cpp | 13 +++++----- src/kernel/objects/thread.cpp | 41 +++++++++++++++----------------- src/kernel/objects/thread.h | 1 - src/kernel/page_manager.cpp | 4 ++++ src/kernel/stack_cache.cpp | 41 ++++++++++++++++++++++++++++++++ src/kernel/stack_cache.h | 27 +++++++++++++++++++++ src/libraries/kutil/vm_space.cpp | 18 +++++++++++++- 9 files changed, 123 insertions(+), 33 deletions(-) create mode 100644 src/kernel/stack_cache.cpp create mode 100644 src/kernel/stack_cache.h diff --git a/modules.yaml b/modules.yaml index 25e739a..c29aaab 100644 --- a/modules.yaml +++ b/modules.yaml @@ -46,6 +46,7 @@ modules: - src/kernel/scheduler.cpp - src/kernel/screen.cpp - src/kernel/serial.cpp + - src/kernel/stack_cache.cpp - src/kernel/syscall.cpp - src/kernel/syscall.s - src/kernel/syscalls/channel.cpp diff --git a/src/include/kernel_memory.h b/src/include/kernel_memory.h index 4c0c868..32ffccf 100644 --- a/src/include/kernel_memory.h +++ b/src/include/kernel_memory.h @@ -16,8 +16,8 @@ namespace memory { /// Offset from physical where page tables are mapped. constexpr uintptr_t page_offset = 0xffffc00000000000; - /// Initial process thread's stack size, in pages - constexpr unsigned initial_stack_pages = 1; + /// Number of pages for a kernel stack + constexpr unsigned kernel_stack_pages = 1; /// Max size of the kernel heap constexpr size_t kernel_max_heap = 0x8000000000; // 512GiB @@ -25,6 +25,12 @@ namespace memory { /// Start of the kernel heap constexpr uintptr_t heap_start = page_offset - kernel_max_heap; + /// Start of the kernel stacks + constexpr uintptr_t stacks_start = heap_start - kernel_max_heap; + + /// Max size of the kernel stacks area + constexpr size_t kernel_max_stacks = 0x8000000000; // 512GiB + /// First kernel space PML4 entry constexpr unsigned pml4e_kernel = 256; diff --git a/src/kernel/memory_bootstrap.cpp b/src/kernel/memory_bootstrap.cpp index 83f5de3..404a59a 100644 --- a/src/kernel/memory_bootstrap.cpp +++ b/src/kernel/memory_bootstrap.cpp @@ -24,7 +24,8 @@ using memory::table_entries; using namespace kernel; -kutil::vm_space g_kernel_space; +kutil::vm_space g_kernel_space {kernel_offset, (page_offset-kernel_offset)}; + // These objects are initialized _before_ global constructors are called, // so we don't want them to have global constructors at all, lest they @@ -61,7 +62,8 @@ void walk_page_table( for (unsigned i = 0; i < table_entries; ++i) { page_table *next = table->get(i); if (!next) { - kspace.commit(current_start, current_bytes); + if (current_bytes) + kspace.commit(current_start, current_bytes); current_start = 0; current_bytes = 0; continue; @@ -109,10 +111,6 @@ memory_initialize_pre_ctors(args::header *kargs) void memory_initialize_post_ctors(args::header *kargs) { - new (&g_kernel_space) kutil::vm_space { - kernel_offset, - (page_offset-kernel_offset)}; - uintptr_t current_start = 0; size_t current_bytes = 0; @@ -122,7 +120,8 @@ memory_initialize_post_ctors(args::header *kargs) page_table *pdp = kpml4->get(i); if (!pdp) { - g_kernel_space.commit(current_start, current_bytes); + if (current_bytes) + g_kernel_space.commit(current_start, current_bytes); current_start = 0; current_bytes = 0; continue; diff --git a/src/kernel/objects/thread.cpp b/src/kernel/objects/thread.cpp index 0a5c1d4..003de68 100644 --- a/src/kernel/objects/thread.cpp +++ b/src/kernel/objects/thread.cpp @@ -3,6 +3,7 @@ #include "objects/thread.h" #include "objects/process.h" #include "scheduler.h" +#include "stack_cache.h" extern "C" void kernel_to_user_trampoline(); static constexpr j6_signal_t thread_default_signals = 0; @@ -15,9 +16,8 @@ thread::thread(process &parent, uint8_t pri, bool user) : m_wait_data(0), m_wait_obj(0) { - TCB *tcbp = tcb(); - tcbp->pml4 = parent.pml4(); - tcbp->priority = pri; + m_tcb.pml4 = parent.pml4(); + m_tcb.priority = pri; setup_kernel_stack(); set_state(state::ready); } @@ -30,16 +30,15 @@ thread::thread(process &parent, uint8_t pri, uintptr_t rsp0) : m_wait_data(0), m_wait_obj(0) { - TCB *tcbp = tcb(); - tcbp->pml4 = parent.pml4(); - tcbp->priority = pri; - tcbp->rsp0 = rsp0; + m_tcb.pml4 = parent.pml4(); + m_tcb.priority = pri; + m_tcb.rsp0 = rsp0; set_state(state::ready); } thread::~thread() { - kutil::kfree(reinterpret_cast(m_tcb.kernel_stack)); + stack_cache::get().return_stack(m_tcb.kernel_stack); } thread * @@ -153,27 +152,25 @@ thread::add_thunk_user(uintptr_t rip) void thread::setup_kernel_stack() { - constexpr size_t initial_stack_size = 0x1000; + using memory::frame_size; + using memory::kernel_stack_pages; + static constexpr size_t stack_bytes = kernel_stack_pages * frame_size; + constexpr unsigned null_frame_entries = 2; constexpr size_t null_frame_size = null_frame_entries * sizeof(uint64_t); - void *stack_bottom = kutil::kalloc(initial_stack_size); - kutil::memset(stack_bottom, 0, initial_stack_size); + uintptr_t stack_addr = stack_cache::get().get_stack(); + uintptr_t stack_end = stack_addr + stack_bytes; - log::debug(logs::memory, "Created kernel stack at %016lx size 0x%lx", - stack_bottom, initial_stack_size); - - void *stack_top = - kutil::offset_pointer(stack_bottom, - initial_stack_size - null_frame_size); - - uint64_t *null_frame = reinterpret_cast(stack_top); + uint64_t *null_frame = reinterpret_cast(stack_end - null_frame_size); for (unsigned i = 0; i < null_frame_entries; ++i) null_frame[i] = 0; - m_tcb.kernel_stack_size = initial_stack_size; - m_tcb.kernel_stack = reinterpret_cast(stack_bottom); - m_tcb.rsp0 = reinterpret_cast(stack_top); + log::debug(logs::memory, "Created kernel stack at %016lx size 0x%lx", + stack_addr, stack_bytes); + + m_tcb.kernel_stack = stack_addr; + m_tcb.rsp0 = reinterpret_cast(null_frame); m_tcb.rsp = m_tcb.rsp0; } diff --git a/src/kernel/objects/thread.h b/src/kernel/objects/thread.h index b2b909c..73ba3a3 100644 --- a/src/kernel/objects/thread.h +++ b/src/kernel/objects/thread.h @@ -23,7 +23,6 @@ struct TCB // TODO: move state into TCB? uintptr_t kernel_stack; - size_t kernel_stack_size; uint32_t time_left; uint64_t last_ran; diff --git a/src/kernel/page_manager.cpp b/src/kernel/page_manager.cpp index 2edbc8d..68163ba 100644 --- a/src/kernel/page_manager.cpp +++ b/src/kernel/page_manager.cpp @@ -325,6 +325,10 @@ page_manager::fault_handler(uintptr_t addr) bool user = addr < kernel_offset; map_pages(page, 1, user); + // Kernel stacks: zero them upon mapping them + if (addr >= memory::stacks_start && addr < memory::heap_start) + kutil::memset(reinterpret_cast(page), 0, memory::frame_size); + return true; } diff --git a/src/kernel/stack_cache.cpp b/src/kernel/stack_cache.cpp new file mode 100644 index 0000000..becbf4a --- /dev/null +++ b/src/kernel/stack_cache.cpp @@ -0,0 +1,41 @@ +#include "kutil/vm_space.h" +#include "kernel_memory.h" +#include "page_manager.h" +#include "stack_cache.h" + +extern kutil::vm_space g_kernel_space; + +using memory::frame_size; +using memory::kernel_stack_pages; +static constexpr size_t stack_bytes = kernel_stack_pages * frame_size; + +stack_cache stack_cache::s_instance(memory::stacks_start, memory::kernel_max_stacks); + +stack_cache::stack_cache(uintptr_t start, size_t size) : + m_next(start), m_end(start+size) +{ +} + +uintptr_t +stack_cache::get_stack() +{ + uintptr_t stack = 0; + if (m_cache.count() > 0) { + stack = m_cache.pop(); + } else { + stack = m_next; + m_next += stack_bytes; + } + + g_kernel_space.commit(stack, stack_bytes); + return stack; +} + +void +stack_cache::return_stack(uintptr_t addr) +{ + void *ptr = reinterpret_cast(addr); + page_manager::get()->unmap_pages(ptr, kernel_stack_pages); + g_kernel_space.unreserve(addr, stack_bytes); + m_cache.append(addr); +} diff --git a/src/kernel/stack_cache.h b/src/kernel/stack_cache.h new file mode 100644 index 0000000..de12f80 --- /dev/null +++ b/src/kernel/stack_cache.h @@ -0,0 +1,27 @@ +#pragma once + +#include + +/// A cache of kernel stack address ranges +class stack_cache +{ +public: + /// Constructor. + /// \args start Start of virtual memory area to contain stacks + /// \args size Size of virtual memory area in bytes + stack_cache(uintptr_t start, size_t size); + + /// Get an available stack address + uintptr_t get_stack(); + + /// Return a stack address to the available pool + void return_stack(uintptr_t addr); + + static stack_cache & get() { return s_instance; } + +private: + kutil::vector m_cache; + uintptr_t m_next; + const uintptr_t m_end; + static stack_cache s_instance; +}; diff --git a/src/libraries/kutil/vm_space.cpp b/src/libraries/kutil/vm_space.cpp index eecc82b..77405eb 100644 --- a/src/libraries/kutil/vm_space.cpp +++ b/src/libraries/kutil/vm_space.cpp @@ -184,7 +184,23 @@ vm_space::unreserve(uintptr_t start, size_t size) uintptr_t vm_space::commit(uintptr_t start, size_t size) { - return 0; + log::debug(logs::vmem, "Committing region %016llx-%016llx", start, start+size); + + node_type *node = find_overlapping(m_ranges.root(), start, size); + if (!node) { + log::debug(logs::vmem, " found no match"); + return 0; + } else if (node->state == vm_state::unknown || node->state == vm_state::mapped) { + log::debug(logs::vmem, " found wrong state %016llx-%016llx[%d]", + node->address, node->address + node->size, node->state); + return 0; + } + + if (node->address <= start && node->size >= size && node->state == vm_state::committed) + return start; + + node = split_out(node, start, size, vm_state::committed); + return node ? start : 0; } void