From f7f8bb3f454f604a02a5d8d6d764e0b5685de470 Mon Sep 17 00:00:00 2001 From: "Justin C. Miller" Date: Sun, 27 Sep 2020 15:34:24 -0700 Subject: [PATCH] [kernel] Replace buffer_cache with vm_area_buffers In order to reduce the amount of tracked state, now use the vm_area_buffers instead of a VMA with buffer_cache on top. --- modules.yaml | 1 - src/include/kernel_memory.h | 7 ++++ src/kernel/buffer_cache.cpp | 47 ------------------------ src/kernel/buffer_cache.h | 29 --------------- src/kernel/memory_bootstrap.cpp | 15 ++++++++ src/kernel/objects/channel.cpp | 12 +++--- src/kernel/objects/process.cpp | 7 +++- src/kernel/objects/thread.cpp | 8 ++-- src/kernel/objects/vm_area.cpp | 48 ++++++++++++++++++++++++ src/kernel/objects/vm_area.h | 65 ++++++++++++++++++++------------- src/kernel/scheduler.cpp | 23 ++++++++---- src/kernel/vm_mapper.h | 2 + src/kernel/vm_space.cpp | 14 ++++++- src/kernel/vm_space.h | 3 ++ 14 files changed, 161 insertions(+), 120 deletions(-) delete mode 100644 src/kernel/buffer_cache.cpp delete mode 100644 src/kernel/buffer_cache.h diff --git a/modules.yaml b/modules.yaml index d1fdc32..b9c5e37 100644 --- a/modules.yaml +++ b/modules.yaml @@ -48,7 +48,6 @@ modules: - src/kernel/scheduler.cpp - src/kernel/screen.cpp - src/kernel/serial.cpp - - src/kernel/buffer_cache.cpp - src/kernel/symbol_table.cpp - src/kernel/syscall.cpp - src/kernel/syscall.s diff --git a/src/include/kernel_memory.h b/src/include/kernel_memory.h index 5296220..f875521 100644 --- a/src/include/kernel_memory.h +++ b/src/include/kernel_memory.h @@ -64,4 +64,11 @@ namespace memory { inline size_t page_count(size_t bytes) { return ((bytes - 1) >> 12) + 1; } + + /// Get the given address, aligned to the next lowest page + inline uintptr_t page_align_down(uintptr_t a) { return a & ~(frame_size-1); } + + /// Get the given address, aligned to the next page + inline uintptr_t page_align_up(uintptr_t a) { return page_align_down(a-1) + frame_size; } + } // namespace memory diff --git a/src/kernel/buffer_cache.cpp b/src/kernel/buffer_cache.cpp deleted file mode 100644 index 1933575..0000000 --- a/src/kernel/buffer_cache.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#include "kutil/assert.h" -#include "buffer_cache.h" -#include "kernel_memory.h" -#include "objects/vm_area.h" -#include "vm_space.h" - -using memory::frame_size; -using memory::kernel_stack_pages; -using memory::kernel_buffer_pages; - -static constexpr size_t stack_bytes = kernel_stack_pages * frame_size; -static constexpr size_t buffer_bytes = kernel_buffer_pages * frame_size; - -buffer_cache g_kstack_cache {memory::stacks_start, stack_bytes, memory::kernel_max_stacks}; -buffer_cache g_kbuffer_cache {memory::buffers_start, buffer_bytes, memory::kernel_max_buffers}; - -buffer_cache::buffer_cache(uintptr_t start, size_t size, size_t length) : - m_next(start), m_end(start+length), m_size(size) -{ - kassert((size % frame_size) == 0, "buffer_cache given a non-page-multiple size!"); -} - -uintptr_t -buffer_cache::get_buffer() -{ - uintptr_t addr = 0; - if (m_cache.count() > 0) { - addr = m_cache.pop(); - } else { - addr = m_next; - m_next += m_size; - } - - vm_space &vm = vm_space::kernel_space(); - vm.allow(addr, m_size, true); - - return addr; -} - -void -buffer_cache::return_buffer(uintptr_t addr) -{ - vm_space &vm = vm_space::kernel_space(); - vm.allow(addr, m_size, false); - - m_cache.append(addr); -} diff --git a/src/kernel/buffer_cache.h b/src/kernel/buffer_cache.h deleted file mode 100644 index 01994d4..0000000 --- a/src/kernel/buffer_cache.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include - -/// A cache of kernel stack address ranges -class buffer_cache -{ -public: - /// Constructor. - /// \args start Start of virtual memory area to contain buffers - /// \args size Size of individual buffers in bytes - /// \args length Size of virtual memory area in bytes - buffer_cache(uintptr_t start, size_t size, size_t length); - - /// Get an available stack address - uintptr_t get_buffer(); - - /// Return a buffer address to the available pool - void return_buffer(uintptr_t addr); - -private: - kutil::vector m_cache; - uintptr_t m_next; - const uintptr_t m_end; - const size_t m_size; -}; - -extern buffer_cache g_kstack_cache; -extern buffer_cache g_kbuffer_cache; diff --git a/src/kernel/memory_bootstrap.cpp b/src/kernel/memory_bootstrap.cpp index 5b69074..a3c4129 100644 --- a/src/kernel/memory_bootstrap.cpp +++ b/src/kernel/memory_bootstrap.cpp @@ -38,6 +38,18 @@ frame_allocator &g_frame_allocator = __g_frame_allocator_storage.value; static kutil::no_construct __g_kernel_heap_area_storage; vm_area_open &g_kernel_heap_area = __g_kernel_heap_area_storage.value; +vm_area_buffers g_kernel_stacks { + memory::kernel_max_stacks, + vm_space::kernel_space(), + vm_flags::write, + memory::kernel_stack_pages}; + +vm_area_buffers g_kernel_buffers { + memory::kernel_max_buffers, + vm_space::kernel_space(), + vm_flags::write, + memory::kernel_buffer_pages}; + void * operator new(size_t size) { return g_kernel_heap.allocate(size); } void * operator new [] (size_t size) { return g_kernel_heap.allocate(size); } void operator delete (void *p) noexcept { return g_kernel_heap.free(p); } @@ -136,6 +148,9 @@ memory_initialize_post_ctors(args::header *kargs) if (current_bytes) g_kernel_space.commit(current_start, current_bytes); */ + vm_space &vm = vm_space::kernel_space(); + vm.add(memory::stacks_start, &g_kernel_stacks); + vm.add(memory::buffers_start, &g_kernel_buffers); g_frame_allocator.free( reinterpret_cast(kargs->page_table_cache), diff --git a/src/kernel/objects/channel.cpp b/src/kernel/objects/channel.cpp index 9428adb..de34cf9 100644 --- a/src/kernel/objects/channel.cpp +++ b/src/kernel/objects/channel.cpp @@ -1,16 +1,16 @@ #include "kutil/assert.h" -#include "buffer_cache.h" #include "kernel_memory.h" #include "objects/channel.h" +#include "objects/vm_area.h" -using memory::frame_size; -using memory::kernel_buffer_pages; -static constexpr size_t buffer_bytes = kernel_buffer_pages * frame_size; +extern vm_area_buffers g_kernel_buffers; + +constexpr size_t buffer_bytes = memory::kernel_buffer_pages * memory::frame_size; channel::channel() : m_len(0), - m_data(g_kbuffer_cache.get_buffer()), + m_data(g_kernel_buffers.get_buffer()), m_buffer(reinterpret_cast(m_data), buffer_bytes), kobject(kobject::type::channel, j6_signal_channel_can_send) { @@ -78,7 +78,7 @@ channel::dequeue(size_t *len, void *data) void channel::close() { - g_kbuffer_cache.return_buffer(m_data); + g_kernel_buffers.return_buffer(m_data); assert_signal(j6_signal_channel_closed); } diff --git a/src/kernel/objects/process.cpp b/src/kernel/objects/process.cpp index 41f50b2..243ff55 100644 --- a/src/kernel/objects/process.cpp +++ b/src/kernel/objects/process.cpp @@ -4,6 +4,7 @@ #include "cpu.h" #include "objects/process.h" #include "objects/thread.h" +#include "objects/vm_area.h" // This object is initialized _before_ global constructors are called, // so we don't want it to have a global constructor at all, lest it @@ -94,7 +95,11 @@ process::create_thread(uint8_t priority, bool user) if (user) { uintptr_t stack_top = stacks_top - (m_threads.count() * stack_size); - m_space.allow(stack_top - stack_size, stack_size, true); + + vm_area *vma = new vm_area_open(stack_size, m_space, + vm_flags::zero|vm_flags::write); + m_space.add(stack_top - stack_size, vma); + th->tcb()->rsp3 = stack_top; } diff --git a/src/kernel/objects/thread.cpp b/src/kernel/objects/thread.cpp index f51079e..a73af25 100644 --- a/src/kernel/objects/thread.cpp +++ b/src/kernel/objects/thread.cpp @@ -3,12 +3,14 @@ #include "log.h" #include "objects/thread.h" #include "objects/process.h" +#include "objects/vm_area.h" #include "scheduler.h" -#include "buffer_cache.h" extern "C" void kernel_to_user_trampoline(); static constexpr j6_signal_t thread_default_signals = 0; +extern vm_area_buffers g_kernel_stacks; + thread::thread(process &parent, uint8_t pri, uintptr_t rsp0) : kobject(kobject::type::thread, thread_default_signals), m_parent(parent), @@ -30,7 +32,7 @@ thread::thread(process &parent, uint8_t pri, uintptr_t rsp0) : thread::~thread() { - g_kstack_cache.return_buffer(m_tcb.kernel_stack); + g_kernel_stacks.return_buffer(m_tcb.kernel_stack); } thread * @@ -179,7 +181,7 @@ thread::setup_kernel_stack() constexpr unsigned null_frame_entries = 2; constexpr size_t null_frame_size = null_frame_entries * sizeof(uint64_t); - uintptr_t stack_addr = g_kstack_cache.get_buffer(); + uintptr_t stack_addr = g_kernel_stacks.get_buffer(); uintptr_t stack_end = stack_addr + stack_bytes; uint64_t *null_frame = reinterpret_cast(stack_end - null_frame_size); diff --git a/src/kernel/objects/vm_area.cpp b/src/kernel/objects/vm_area.cpp index b6cac83..d1a25f5 100644 --- a/src/kernel/objects/vm_area.cpp +++ b/src/kernel/objects/vm_area.cpp @@ -211,3 +211,51 @@ vm_area_open::uncommit(uintptr_t offset, size_t count) m_mapper.unmap(offset, count); } + +vm_area_buffers::vm_area_buffers(size_t size, vm_space &space, vm_flags flags, size_t buf_pages) : + m_mapper {*this, space}, + m_pages {buf_pages}, + m_next {memory::frame_size}, + vm_area {size, flags} +{ +} + +uintptr_t +vm_area_buffers::get_buffer() +{ + if (m_cache.count() > 0) { + return m_cache.pop(); + } + + uintptr_t addr = m_next; + m_next += (m_pages + 1) * memory::frame_size; + return m_mapper.space().lookup(*this, addr); +} + +void +vm_area_buffers::return_buffer(uintptr_t addr) +{ + m_cache.append(addr); +} + +bool +vm_area_buffers::allowed(uintptr_t offset) const +{ + if (offset >= m_next) return false; + + // Buffers are m_pages big plus 1 leading guard page + return memory::page_align_down(offset) % (m_pages+1); +} + +void +vm_area_buffers::commit(uintptr_t phys, uintptr_t offset, size_t count) +{ + m_mapper.map(offset, count, phys); +} + +void +vm_area_buffers::uncommit(uintptr_t offset, size_t count) +{ + m_mapper.unmap(offset, count); +} + diff --git a/src/kernel/objects/vm_area.h b/src/kernel/objects/vm_area.h index da1e1c9..96c659a 100644 --- a/src/kernel/objects/vm_area.h +++ b/src/kernel/objects/vm_area.h @@ -124,8 +124,9 @@ private: kutil::vector m_mappings; }; -/// Area split into standard-sized segments -class vm_area_buffers : + +/// Area that allows open allocation (eg, kernel heap) +class vm_area_open : public vm_area { public: @@ -133,7 +134,40 @@ public: /// \arg size Initial virtual size of the memory area /// \arg space The address space this area belongs to /// \arg flags Flags for this memory area - vm_area_buffers(size_t size, vm_space &space, vm_flags flags); + vm_area_open(size_t size, vm_space &space, vm_flags flags); + + virtual vm_mapper & mapper() override { return m_mapper; } + virtual const vm_mapper & mapper() const override { return m_mapper; } + + virtual void commit(uintptr_t phys, uintptr_t offset, size_t count) override; + virtual void uncommit(uintptr_t offset, size_t count) override; + +private: + vm_mapper_single m_mapper; +}; + + +/// Area split into standard-sized segments +class vm_area_buffers : + public vm_area +{ +public: + /// Constructor. + /// \arg size Initial virtual size of the memory area + /// \arg space The address space this area belongs to + /// \arg flags Flags for this memory area + /// \arg buf_pages Pages in an individual buffer + vm_area_buffers( + size_t size, + vm_space &space, + vm_flags flags, + size_t buf_pages); + + /// Get an available stack address + uintptr_t get_buffer(); + + /// Return a buffer address to the available pool + void return_buffer(uintptr_t addr); virtual vm_mapper & mapper() override { return m_mapper; } virtual const vm_mapper & mapper() const override { return m_mapper; } @@ -144,6 +178,9 @@ public: private: vm_mapper_single m_mapper; + kutil::vector m_cache; + size_t m_pages; + uintptr_t m_next; }; @@ -167,26 +204,4 @@ private: vm_mapper_multi m_mapper; }; - -/// Area that allows open allocation (eg, kernel heap) -class vm_area_open : - public vm_area -{ -public: - /// Constructor. - /// \arg size Initial virtual size of the memory area - /// \arg space The address space this area belongs to - /// \arg flags Flags for this memory area - vm_area_open(size_t size, vm_space &space, vm_flags flags); - - virtual vm_mapper & mapper() override { return m_mapper; } - virtual const vm_mapper & mapper() const override { return m_mapper; } - - virtual void commit(uintptr_t phys, uintptr_t offset, size_t count) override; - virtual void uncommit(uintptr_t offset, size_t count) override; - -private: - vm_mapper_single m_mapper; -}; - IS_BITFIELD(vm_flags); diff --git a/src/kernel/scheduler.cpp b/src/kernel/scheduler.cpp index 7fd3949..2caf91d 100644 --- a/src/kernel/scheduler.cpp +++ b/src/kernel/scheduler.cpp @@ -13,6 +13,7 @@ #include "msr.h" #include "objects/channel.h" #include "objects/process.h" +#include "objects/vm_area.h" #include "scheduler.h" #include "elf/elf.h" @@ -62,6 +63,9 @@ scheduler::scheduler(lapic *apic) : uintptr_t load_process_image(const void *image_start, size_t bytes, TCB *tcb) { + using memory::page_align_down; + using memory::page_align_up; + // We're now in the process space for this process, allocate memory for the // process code and load it process &proc = process::current(); @@ -73,6 +77,9 @@ load_process_image(const void *image_start, size_t bytes, TCB *tcb) elf::elf image(image_start, bytes); kassert(image.valid(), "Invalid ELF passed to load_process_image"); + uintptr_t vma_base = -1; + uintptr_t vma_end = 0; + const unsigned program_count = image.program_count(); for (unsigned i = 0; i < program_count; ++i) { const elf::program_header *header = image.program(i); @@ -80,20 +87,22 @@ load_process_image(const void *image_start, size_t bytes, TCB *tcb) if (header->type != elf::segment_type::load) continue; - uintptr_t aligned = header->vaddr & ~(memory::frame_size - 1); - size_t size = (header->vaddr + header->mem_size) - aligned; - size_t pagesize = memory::page_count(size) * memory::frame_size; + uintptr_t base = page_align_down(header->vaddr); + uintptr_t end = page_align_up(header->vaddr + header->mem_size); + if (base < vma_base) vma_base = base; + if (end > vma_end) vma_end = end; log::debug(logs::loader, " Loadable segment %02u: vaddr %016lx size %016lx", i, header->vaddr, header->mem_size); log::debug(logs::loader, " - aligned to: vaddr %016lx pages %d", - aligned, pagesize >> 12); - - space.allow(aligned, pagesize, true); - kutil::memset(reinterpret_cast(aligned), 0, pagesize); + base, memory::page_count(end-base)); } + vm_area *vma = new vm_area_open(vma_end - vma_base, space, + vm_flags::zero|vm_flags::write); + space.add(vma_base, vma); + const unsigned section_count = image.section_count(); for (unsigned i = 0; i < section_count; ++i) { const elf::section_header *header = image.section(i); diff --git a/src/kernel/vm_mapper.h b/src/kernel/vm_mapper.h index af4a908..2ac8f63 100644 --- a/src/kernel/vm_mapper.h +++ b/src/kernel/vm_mapper.h @@ -52,6 +52,8 @@ public: virtual void map(uintptr_t offset, size_t count, uintptr_t phys) override; virtual void unmap(uintptr_t offset, size_t count) override; + vm_space & space() { return m_space; } + private: vm_area &m_area; vm_space &m_space; diff --git a/src/kernel/vm_space.cpp b/src/kernel/vm_space.cpp index d17e1f0..0e6dd65 100644 --- a/src/kernel/vm_space.cpp +++ b/src/kernel/vm_space.cpp @@ -220,6 +220,15 @@ vm_space::clear(const vm_area &vma, uintptr_t offset, size_t count, bool free) fa.free(free_start, free_count); } +uintptr_t +vm_space::lookup(const vm_area &vma, uintptr_t offset) +{ + uintptr_t base = 0; + if (!find_vma(vma, base)) + return 0; + return base + offset; +} + void vm_space::allow(uintptr_t start, size_t length, bool allow) { @@ -276,7 +285,7 @@ vm_space::handle_fault(uintptr_t addr, fault_type fault) uintptr_t base = 0; vm_area *area = get(addr, &base); - if ((!area || !area->allowed(page)) && !it.allowed()) + if ((!area || !area->allowed(page-base)) && !it.allowed()) return false; uintptr_t phys = 0; @@ -294,6 +303,9 @@ vm_space::handle_fault(uintptr_t addr, fault_type fault) : page_table::flag::user); if (area) { + if (area->flags() && vm_flags::zero) + kutil::memset(memory::to_virtual(phys), 0, memory::frame_size); + uintptr_t offset = page - base; area->commit(phys, offset, 1); return true; diff --git a/src/kernel/vm_space.h b/src/kernel/vm_space.h index 8583aef..7124453 100644 --- a/src/kernel/vm_space.h +++ b/src/kernel/vm_space.h @@ -61,6 +61,9 @@ public: /// \arg free If true, free the pages back to the system void clear(const vm_area &vma, uintptr_t start, size_t count, bool free = false); + /// Look up the address of a given VMA's offset + uintptr_t lookup(const vm_area &vma, uintptr_t offset); + /// Mark whether allocation is allowed or not in a range of /// virtual memory. /// \arg start The starting virtual address of the area