Overhaul memory allocation model
This commit makes several fundamental changes to memory handling:
- the frame allocator is now only an allocator for free frames, and does
not track used frames.
- the frame allocator now stores its free list inside the free frames
themselves, as a hybrid stack/span model.
- This has the implication that all frames must currently fit within
the offset area.
- kutil has a new allocator interface, which is the only allowed way for
any code outside of src/kernel to allocate. Code under src/kernel
_may_ use new/delete, but should prefer the allocator interface.
- the heap manager has become heap_allocator, which is merely an
implementation of kutil::allocator which doles out sections of a given
address range.
- the heap manager now only writes block headers when necessary,
avoiding page faults until they're actually needed
- page_manager now has a page fault handler, which checks with the
address_manager to see if the address is known, and provides a frame
mapping if it is, allowing heap manager to work with its entire
address size from the start. (Currently 32GiB.)
This commit is contained in:
Justin C. Miller
@@ -14,7 +14,7 @@
|
||||
|
||||
static const char expected_signature[] = "RSD PTR ";
|
||||
|
||||
device_manager device_manager::s_instance(nullptr);
|
||||
device_manager device_manager::s_instance(nullptr, kutil::allocator::invalid);
|
||||
|
||||
struct acpi1_rsdp
|
||||
{
|
||||
@@ -59,8 +59,13 @@ void irq4_callback(void *)
|
||||
}
|
||||
|
||||
|
||||
device_manager::device_manager(const void *root_table) :
|
||||
m_lapic(nullptr)
|
||||
device_manager::device_manager(const void *root_table, kutil::allocator &alloc) :
|
||||
m_lapic(nullptr),
|
||||
m_ioapics(alloc),
|
||||
m_pci(alloc),
|
||||
m_devices(alloc),
|
||||
m_irqs(alloc),
|
||||
m_blockdevs(alloc)
|
||||
{
|
||||
kassert(root_table != 0, "ACPI root table pointer is null.");
|
||||
|
||||
@@ -93,7 +98,7 @@ device_manager::device_manager(const void *root_table) :
|
||||
ioapic *
|
||||
device_manager::get_ioapic(int i)
|
||||
{
|
||||
return (i < m_ioapics.count()) ? m_ioapics[i] : nullptr;
|
||||
return (i < m_ioapics.count()) ? &m_ioapics[i] : nullptr;
|
||||
}
|
||||
|
||||
static void
|
||||
@@ -148,19 +153,31 @@ device_manager::load_apic(const acpi_apic *apic)
|
||||
uint8_t const *p = apic->controller_data;
|
||||
uint8_t const *end = p + count;
|
||||
|
||||
// Pass one: set up IOAPIC objcts
|
||||
// Pass one: count IOAPIC objcts
|
||||
int num_ioapics = 0;
|
||||
while (p < end) {
|
||||
const uint8_t type = p[0];
|
||||
const uint8_t length = p[1];
|
||||
if (type == 1) num_ioapics++;
|
||||
p += length;
|
||||
}
|
||||
|
||||
m_ioapics.set_capacity(num_ioapics);
|
||||
|
||||
// Pass two: set up IOAPIC objcts
|
||||
p = apic->controller_data;
|
||||
while (p < end) {
|
||||
const uint8_t type = p[0];
|
||||
const uint8_t length = p[1];
|
||||
if (type == 1) {
|
||||
uint32_t *base = reinterpret_cast<uint32_t *>(kutil::read_from<uint32_t>(p+4));
|
||||
uint32_t base_gsr = kutil::read_from<uint32_t>(p+8);
|
||||
m_ioapics.append(new ioapic(base, base_gsr));
|
||||
m_ioapics.emplace(base, base_gsr);
|
||||
}
|
||||
p += length;
|
||||
}
|
||||
|
||||
// Pass two: configure APIC objects
|
||||
// Pass three: configure APIC objects
|
||||
p = apic->controller_data;
|
||||
while (p < end) {
|
||||
const uint8_t type = p[0];
|
||||
@@ -186,7 +203,7 @@ device_manager::load_apic(const acpi_apic *apic)
|
||||
source, gsi, (flags & 0x3), ((flags >> 2) & 0x3));
|
||||
|
||||
// TODO: in a multiple-IOAPIC system this might be elsewhere
|
||||
m_ioapics[0]->redirect(source, static_cast<isr>(gsi), flags, true);
|
||||
m_ioapics[0].redirect(source, static_cast<isr>(gsi), flags, true);
|
||||
}
|
||||
break;
|
||||
|
||||
@@ -212,10 +229,10 @@ device_manager::load_apic(const acpi_apic *apic)
|
||||
p += length;
|
||||
}
|
||||
|
||||
for (uint8_t i = 0; i < m_ioapics[0]->get_num_gsi(); ++i) {
|
||||
for (uint8_t i = 0; i < m_ioapics[0].get_num_gsi(); ++i) {
|
||||
switch (i) {
|
||||
case 2: break;
|
||||
default: m_ioapics[0]->mask(i, false);
|
||||
default: m_ioapics[0].mask(i, false);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,14 +2,13 @@
|
||||
/// \file device_manager.h
|
||||
/// The device manager definition
|
||||
#include "kutil/vector.h"
|
||||
#include "apic.h"
|
||||
#include "pci.h"
|
||||
|
||||
struct acpi_xsdt;
|
||||
struct acpi_apic;
|
||||
struct acpi_mcfg;
|
||||
class block_device;
|
||||
class lapic;
|
||||
class ioapic;
|
||||
|
||||
using irq_callback = void (*)(void *);
|
||||
|
||||
@@ -19,8 +18,9 @@ class device_manager
|
||||
{
|
||||
public:
|
||||
/// Constructor.
|
||||
/// \arg root_table Pointer to the ACPI RSDP
|
||||
device_manager(const void *root_table);
|
||||
/// \arg root_table Pointer to the ACPI RSDP
|
||||
/// \arg alloc Allocator for device arrays
|
||||
device_manager(const void *root_table, kutil::allocator &alloc);
|
||||
|
||||
/// Get the system global device manager.
|
||||
/// \returns A reference to the system device manager
|
||||
@@ -105,7 +105,7 @@ private:
|
||||
void bad_irq(uint8_t irq);
|
||||
|
||||
lapic *m_lapic;
|
||||
kutil::vector<ioapic *> m_ioapics;
|
||||
kutil::vector<ioapic> m_ioapics;
|
||||
|
||||
kutil::vector<pci_group> m_pci;
|
||||
kutil::vector<pci_device> m_devices;
|
||||
|
||||
93
src/kernel/frame_allocator.cpp
Normal file
93
src/kernel/frame_allocator.cpp
Normal file
@@ -0,0 +1,93 @@
|
||||
#include "kutil/assert.h"
|
||||
#include "kutil/memory.h"
|
||||
#include "frame_allocator.h"
|
||||
|
||||
using memory::frame_size;
|
||||
using memory::page_offset;
|
||||
using frame_block_node = kutil::list_node<frame_block>;
|
||||
|
||||
frame_allocator g_frame_allocator;
|
||||
|
||||
int
|
||||
frame_block::compare(const frame_block *rhs) const
|
||||
{
|
||||
if (address < rhs->address)
|
||||
return -1;
|
||||
else if (address > rhs->address)
|
||||
return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
frame_allocator::raw_alloc::raw_alloc(frame_allocator &fa) : m_fa(fa) {}
|
||||
|
||||
void *
|
||||
frame_allocator::raw_alloc::allocate(size_t size)
|
||||
{
|
||||
kassert(size <= frame_size, "Raw allocator only allocates a single page");
|
||||
|
||||
uintptr_t addr = 0;
|
||||
if (size <= frame_size)
|
||||
m_fa.allocate(1, &addr);
|
||||
return reinterpret_cast<void*>(addr + page_offset);
|
||||
}
|
||||
|
||||
void
|
||||
frame_allocator::raw_alloc::free(void *p)
|
||||
{
|
||||
m_fa.free(reinterpret_cast<uintptr_t>(p), 1);
|
||||
}
|
||||
|
||||
|
||||
frame_allocator::frame_allocator() :
|
||||
m_raw_alloc(*this)
|
||||
{
|
||||
}
|
||||
|
||||
size_t
|
||||
frame_allocator::allocate(size_t count, uintptr_t *address)
|
||||
{
|
||||
kassert(!m_free.empty(), "frame_allocator::pop_frames ran out of free frames!");
|
||||
if (m_free.empty())
|
||||
return 0;
|
||||
|
||||
auto *first = m_free.front();
|
||||
|
||||
if (count >= first->count) {
|
||||
*address = first->address;
|
||||
m_free.remove(first);
|
||||
return first->count;
|
||||
} else {
|
||||
first->count -= count;
|
||||
*address = first->address + (first->count * frame_size);
|
||||
return count;
|
||||
}
|
||||
}
|
||||
|
||||
inline uintptr_t end(frame_block *node) { return node->address + node->count * frame_size; }
|
||||
|
||||
void
|
||||
frame_allocator::free(uintptr_t address, size_t count)
|
||||
{
|
||||
frame_block_node *node =
|
||||
reinterpret_cast<frame_block_node*>(address + page_offset);
|
||||
|
||||
kutil::memset(node, 0, sizeof(frame_block_node));
|
||||
node->address = address;
|
||||
node->count = count;
|
||||
|
||||
m_free.sorted_insert(node);
|
||||
|
||||
frame_block_node *next = node->next();
|
||||
if (next && end(node) == next->address) {
|
||||
node->count += next->count;
|
||||
m_free.remove(next);
|
||||
}
|
||||
|
||||
frame_block_node *prev = node->prev();
|
||||
if (prev && end(prev) == address) {
|
||||
prev->count += node->count;
|
||||
m_free.remove(node);
|
||||
}
|
||||
}
|
||||
|
||||
70
src/kernel/frame_allocator.h
Normal file
70
src/kernel/frame_allocator.h
Normal file
@@ -0,0 +1,70 @@
|
||||
#pragma once
|
||||
/// \file frame_allocator.h
|
||||
/// Allocator for physical memory frames
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "kutil/allocator.h"
|
||||
#include "kutil/linked_list.h"
|
||||
|
||||
struct frame_block;
|
||||
using frame_block_list = kutil::linked_list<frame_block>;
|
||||
|
||||
/// Allocator for physical memory frames
|
||||
class frame_allocator
|
||||
{
|
||||
public:
|
||||
/// Default constructor
|
||||
frame_allocator();
|
||||
|
||||
/// Get free frames from the free list. Only frames from the first free block
|
||||
/// are returned, so the number may be less than requested, but they will
|
||||
/// be contiguous.
|
||||
/// \arg count The maximum number of frames to get
|
||||
/// \arg address [out] The physical address of the first frame
|
||||
/// \returns The number of frames retrieved
|
||||
size_t allocate(size_t count, uintptr_t *address);
|
||||
|
||||
/// Free previously allocated frames.
|
||||
/// \arg address The physical address of the first frame to free
|
||||
/// \arg count The number of frames to be freed
|
||||
void free(uintptr_t address, size_t count);
|
||||
|
||||
/// Get a memory allocator that allocates raw pages
|
||||
/// \returns The allocator ojbect
|
||||
kutil::allocator & raw_allocator() { return m_raw_alloc; }
|
||||
|
||||
private:
|
||||
class raw_alloc :
|
||||
public kutil::allocator
|
||||
{
|
||||
public:
|
||||
raw_alloc(frame_allocator &fa);
|
||||
virtual void * allocate(size_t size) override;
|
||||
virtual void free(void *p) override;
|
||||
private:
|
||||
frame_allocator &m_fa;
|
||||
};
|
||||
|
||||
raw_alloc m_raw_alloc;
|
||||
frame_block_list m_free; ///< Free frames list
|
||||
|
||||
frame_allocator(const frame_allocator &) = delete;
|
||||
};
|
||||
|
||||
|
||||
/// A block of contiguous frames. Each `frame_block` represents contiguous
|
||||
/// physical frames with the same attributes.
|
||||
struct frame_block
|
||||
{
|
||||
uintptr_t address;
|
||||
uint32_t count;
|
||||
|
||||
/// Compare two blocks by address.
|
||||
/// \arg rhs The right-hand comparator
|
||||
/// \returns <0 if this is sorts earlier, >0 if this sorts later, 0 for equal
|
||||
int compare(const frame_block *rhs) const;
|
||||
};
|
||||
|
||||
|
||||
extern frame_allocator g_frame_allocator;
|
||||
@@ -181,21 +181,26 @@ isr_handler(cpu_state *regs)
|
||||
break;
|
||||
|
||||
case isr::isrPageFault: {
|
||||
cons->set_color(11);
|
||||
cons->puts("\nPage Fault:\n");
|
||||
cons->set_color();
|
||||
uintptr_t cr2 = 0;
|
||||
__asm__ __volatile__ ("mov %%cr2, %0" : "=r"(cr2));
|
||||
|
||||
cons->puts(" flags:");
|
||||
if (regs->errorcode & 0x01) cons->puts(" present");
|
||||
if (regs->errorcode & 0x02) cons->puts(" write");
|
||||
if (regs->errorcode & 0x04) cons->puts(" user");
|
||||
if (regs->errorcode & 0x08) cons->puts(" reserved");
|
||||
if (regs->errorcode & 0x10) cons->puts(" ip");
|
||||
cons->puts("\n");
|
||||
print_regs(*regs);
|
||||
print_stacktrace(2);
|
||||
if (!page_manager::get()->fault_handler(cr2)) {
|
||||
cons->set_color(11);
|
||||
cons->puts("\nPage Fault:\n");
|
||||
cons->set_color();
|
||||
|
||||
cons->puts(" flags:");
|
||||
if (regs->errorcode & 0x01) cons->puts(" present");
|
||||
if (regs->errorcode & 0x02) cons->puts(" write");
|
||||
if (regs->errorcode & 0x04) cons->puts(" user");
|
||||
if (regs->errorcode & 0x08) cons->puts(" reserved");
|
||||
if (regs->errorcode & 0x10) cons->puts(" ip");
|
||||
cons->puts("\n");
|
||||
print_regs(*regs);
|
||||
print_stacktrace(2);
|
||||
_halt();
|
||||
}
|
||||
}
|
||||
_halt();
|
||||
break;
|
||||
|
||||
case isr::isrTimer:
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
#include "log.h"
|
||||
#include "page_manager.h"
|
||||
#include "scheduler.h"
|
||||
#include "screen.h"
|
||||
#include "serial.h"
|
||||
#include "syscall.h"
|
||||
|
||||
@@ -51,7 +50,7 @@ kernel_main(kernel_args *header)
|
||||
gdt_init();
|
||||
interrupts_init();
|
||||
|
||||
memory_initialize(
|
||||
kutil::allocator &heap = memory_initialize(
|
||||
header->scratch_pages,
|
||||
header->memory_map,
|
||||
header->memory_map_length,
|
||||
@@ -72,7 +71,7 @@ kernel_main(kernel_args *header)
|
||||
log::debug(logs::boot, "ACPI root table is at: %016lx", header->acpi_table);
|
||||
log::debug(logs::boot, "Runtime service is at: %016lx", header->runtime);
|
||||
|
||||
initrd::disk ird(header->initrd);
|
||||
initrd::disk ird(header->initrd, heap);
|
||||
log::info(logs::boot, "initrd loaded with %d files.", ird.files().count());
|
||||
for (auto &f : ird.files())
|
||||
log::info(logs::boot, " %s%s (%d bytes).", f.executable() ? "*" : "", f.name(), f.size());
|
||||
@@ -83,7 +82,7 @@ kernel_main(kernel_args *header)
|
||||
*/
|
||||
|
||||
device_manager *devices =
|
||||
new (&device_manager::get()) device_manager(header->acpi_table);
|
||||
new (&device_manager::get()) device_manager(header->acpi_table, heap);
|
||||
|
||||
interrupts_enable();
|
||||
|
||||
@@ -130,7 +129,7 @@ kernel_main(kernel_args *header)
|
||||
devices->get_lapic()->calibrate_timer();
|
||||
|
||||
syscall_enable();
|
||||
scheduler *sched = new (&scheduler::get()) scheduler(devices->get_lapic());
|
||||
scheduler *sched = new (&scheduler::get()) scheduler(devices->get_lapic(), heap);
|
||||
|
||||
sched->create_kernel_task(-1, logger_task);
|
||||
|
||||
|
||||
@@ -2,72 +2,26 @@
|
||||
#include <utility>
|
||||
#include "kutil/address_manager.h"
|
||||
#include "kutil/assert.h"
|
||||
#include "kutil/frame_allocator.h"
|
||||
#include "kutil/heap_manager.h"
|
||||
#include "kutil/heap_allocator.h"
|
||||
#include "frame_allocator.h"
|
||||
#include "io.h"
|
||||
#include "log.h"
|
||||
#include "page_manager.h"
|
||||
|
||||
using kutil::frame_block;
|
||||
using kutil::frame_block_flags;
|
||||
using kutil::frame_block_list;
|
||||
using memory::frame_size;
|
||||
using memory::kernel_max_heap;
|
||||
using memory::kernel_offset;
|
||||
using memory::page_offset;
|
||||
|
||||
static const unsigned ident_page_flags = 0xb;
|
||||
|
||||
kutil::frame_allocator g_frame_allocator;
|
||||
kutil::address_manager g_kernel_address_manager;
|
||||
kutil::heap_manager g_kernel_heap_manager;
|
||||
kutil::heap_allocator g_kernel_heap;
|
||||
|
||||
void * mm_grow_callback(size_t length)
|
||||
{
|
||||
kassert(length % frame_size == 0,
|
||||
"Heap manager requested a fractional page.");
|
||||
|
||||
size_t pages = length / frame_size;
|
||||
log::info(logs::memory, "Heap manager growing heap by %d pages.", pages);
|
||||
|
||||
uintptr_t addr = g_kernel_address_manager.allocate(length);
|
||||
g_page_manager.map_pages(addr, pages);
|
||||
|
||||
return reinterpret_cast<void *>(addr);
|
||||
}
|
||||
|
||||
|
||||
namespace {
|
||||
// Page-by-page initial allocator for the initial frame_block allocator
|
||||
struct page_consumer
|
||||
{
|
||||
page_consumer(uintptr_t start, unsigned count, unsigned used = 0) :
|
||||
current(start + used * frame_size),
|
||||
used(used),
|
||||
max(count) {}
|
||||
|
||||
void * get_page() {
|
||||
kassert(used++ < max, "page_consumer ran out of pages");
|
||||
void *retval = reinterpret_cast<void *>(current);
|
||||
current += frame_size;
|
||||
return retval;
|
||||
}
|
||||
|
||||
void * operator()(size_t size) {
|
||||
kassert(size == frame_size, "page_consumer used with non-page size!");
|
||||
return get_page();
|
||||
}
|
||||
|
||||
unsigned left() const { return max - used; }
|
||||
|
||||
uintptr_t current;
|
||||
unsigned used, max;
|
||||
};
|
||||
|
||||
using block_allocator =
|
||||
kutil::slab_allocator<kutil::frame_block, page_consumer &>;
|
||||
using region_allocator =
|
||||
kutil::slab_allocator<kutil::buddy_region, page_consumer &>;
|
||||
}
|
||||
void * operator new(size_t size) { return g_kernel_heap.allocate(size); }
|
||||
void * operator new [] (size_t size) { return g_kernel_heap.allocate(size); }
|
||||
void operator delete (void *p) noexcept { return g_kernel_heap.free(p); }
|
||||
void operator delete [] (void *p) noexcept { return g_kernel_heap.free(p); }
|
||||
|
||||
enum class efi_memory_type : uint32_t
|
||||
{
|
||||
@@ -107,92 +61,103 @@ struct efi_memory_descriptor
|
||||
uint64_t flags;
|
||||
};
|
||||
|
||||
static const efi_memory_descriptor *
|
||||
desc_incr(const efi_memory_descriptor *d, size_t desc_length)
|
||||
struct memory_map
|
||||
{
|
||||
return reinterpret_cast<const efi_memory_descriptor *>(
|
||||
reinterpret_cast<const uint8_t *>(d) + desc_length);
|
||||
}
|
||||
memory_map(const void *efi_map, size_t map_length, size_t desc_length) :
|
||||
efi_map(efi_map), map_length(map_length), desc_length(desc_length) {}
|
||||
|
||||
void
|
||||
gather_block_lists(
|
||||
block_allocator &allocator,
|
||||
frame_block_list &used,
|
||||
frame_block_list &free,
|
||||
const void *memory_map,
|
||||
size_t map_length,
|
||||
size_t desc_length)
|
||||
{
|
||||
efi_memory_descriptor const *desc = reinterpret_cast<efi_memory_descriptor const *>(memory_map);
|
||||
efi_memory_descriptor const *end = desc_incr(desc, map_length);
|
||||
class iterator
|
||||
{
|
||||
public:
|
||||
iterator(const memory_map &map, efi_memory_descriptor const *item) :
|
||||
map(map), item(item) {}
|
||||
|
||||
while (desc < end) {
|
||||
auto *block = allocator.pop();
|
||||
block->address = desc->physical_start;
|
||||
block->count = desc->pages;
|
||||
bool block_used;
|
||||
|
||||
switch (desc->type) {
|
||||
case efi_memory_type::loader_code:
|
||||
case efi_memory_type::loader_data:
|
||||
block_used = true;
|
||||
block->flags = frame_block_flags::pending_free;
|
||||
break;
|
||||
|
||||
case efi_memory_type::boot_services_code:
|
||||
case efi_memory_type::boot_services_data:
|
||||
case efi_memory_type::available:
|
||||
block_used = false;
|
||||
break;
|
||||
|
||||
case efi_memory_type::acpi_reclaim:
|
||||
block_used = true;
|
||||
block->flags =
|
||||
frame_block_flags::acpi_wait |
|
||||
frame_block_flags::map_ident;
|
||||
break;
|
||||
|
||||
case efi_memory_type::persistent:
|
||||
block_used = false;
|
||||
block->flags = frame_block_flags::nonvolatile;
|
||||
break;
|
||||
|
||||
case efi_memory_type::popcorn_kernel:
|
||||
block_used = true;
|
||||
block->flags =
|
||||
frame_block_flags::permanent |
|
||||
frame_block_flags::map_kernel;
|
||||
break;
|
||||
|
||||
case efi_memory_type::popcorn_data:
|
||||
case efi_memory_type::popcorn_initrd:
|
||||
block_used = true;
|
||||
block->flags =
|
||||
frame_block_flags::pending_free |
|
||||
frame_block_flags::map_kernel;
|
||||
break;
|
||||
|
||||
case efi_memory_type::popcorn_scratch:
|
||||
block_used = true;
|
||||
block->flags = frame_block_flags::map_offset;
|
||||
break;
|
||||
|
||||
default:
|
||||
block_used = true;
|
||||
block->flags = frame_block_flags::permanent;
|
||||
break;
|
||||
inline efi_memory_descriptor const * operator*() const { return item; }
|
||||
inline bool operator!=(const iterator &other) { return item != other.item; }
|
||||
inline iterator & operator++() {
|
||||
item = kutil::offset_pointer(item, map.desc_length);
|
||||
return *this;
|
||||
}
|
||||
|
||||
if (block_used)
|
||||
used.push_back(block);
|
||||
else
|
||||
free.push_back(block);
|
||||
private:
|
||||
const memory_map ↦
|
||||
efi_memory_descriptor const *item;
|
||||
};
|
||||
|
||||
desc = desc_incr(desc, desc_length);
|
||||
iterator begin() const {
|
||||
return iterator(*this, reinterpret_cast<efi_memory_descriptor const *>(efi_map));
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
iterator end() const {
|
||||
const void *end = kutil::offset_pointer(efi_map, map_length);
|
||||
return iterator(*this, reinterpret_cast<efi_memory_descriptor const *>(end));
|
||||
}
|
||||
|
||||
const void *efi_map;
|
||||
size_t map_length;
|
||||
size_t desc_length;
|
||||
};
|
||||
|
||||
class memory_bootstrap
|
||||
{
|
||||
public:
|
||||
memory_bootstrap(const void *memory_map, size_t map_length, size_t desc_length) :
|
||||
map(memory_map, map_length, desc_length) {}
|
||||
|
||||
void add_free_frames(frame_allocator &fa) {
|
||||
for (auto *desc : map) {
|
||||
if (desc->type == efi_memory_type::loader_code ||
|
||||
desc->type == efi_memory_type::loader_data ||
|
||||
desc->type == efi_memory_type::boot_services_code ||
|
||||
desc->type == efi_memory_type::boot_services_data ||
|
||||
desc->type == efi_memory_type::available)
|
||||
{
|
||||
fa.free(desc->physical_start, desc->pages);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void add_used_frames(kutil::address_manager &am) {
|
||||
for (auto *desc : map) {
|
||||
if (desc->type == efi_memory_type::popcorn_data ||
|
||||
desc->type == efi_memory_type::popcorn_initrd)
|
||||
{
|
||||
uintptr_t virt_addr = desc->physical_start + kernel_offset;
|
||||
am.mark(virt_addr, desc->pages * frame_size);
|
||||
}
|
||||
else if (desc->type == efi_memory_type::popcorn_kernel)
|
||||
{
|
||||
uintptr_t virt_addr = desc->physical_start + kernel_offset;
|
||||
am.mark_permanent(virt_addr, desc->pages * frame_size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void page_in_kernel(page_manager &pm, page_table *pml4) {
|
||||
for (auto *desc : map) {
|
||||
if (desc->type == efi_memory_type::popcorn_kernel ||
|
||||
desc->type == efi_memory_type::popcorn_data ||
|
||||
desc->type == efi_memory_type::popcorn_initrd)
|
||||
{
|
||||
uintptr_t virt_addr = desc->physical_start + kernel_offset;
|
||||
pm.page_in(pml4, desc->physical_start, virt_addr, desc->pages);
|
||||
}
|
||||
|
||||
if (desc->type == efi_memory_type::acpi_reclaim) {
|
||||
pm.page_in(pml4, desc->physical_start, desc->physical_start, desc->pages);
|
||||
}
|
||||
}
|
||||
|
||||
// Put our new PML4 into CR3 to start using it
|
||||
page_manager::set_pml4(pml4);
|
||||
pm.m_kernel_pml4 = pml4;
|
||||
}
|
||||
|
||||
private:
|
||||
const memory_map map;
|
||||
};
|
||||
|
||||
kutil::allocator &
|
||||
memory_initialize(uint16_t scratch_pages, const void *memory_map, size_t map_length, size_t desc_length)
|
||||
{
|
||||
// make sure the options we want in CR4 are set
|
||||
@@ -227,81 +192,51 @@ memory_initialize(uint16_t scratch_pages, const void *memory_map, size_t map_len
|
||||
__sync_synchronize();
|
||||
io_wait();
|
||||
|
||||
// We now have pages starting at "scratch_virt" to bootstrap ourselves. Start by
|
||||
// taking inventory of free pages.
|
||||
uintptr_t scratch_virt = scratch_phys + page_offset;
|
||||
uint64_t used_pages = 2; // starts with PML4 + offset PDP
|
||||
page_consumer allocator(scratch_virt, scratch_pages, used_pages);
|
||||
memory_bootstrap bootstrap {memory_map, map_length, desc_length};
|
||||
|
||||
block_allocator block_slab(frame_size, allocator);
|
||||
frame_block_list used;
|
||||
frame_block_list free;
|
||||
// Now tell the frame allocator what's free
|
||||
frame_allocator *fa = new (&g_frame_allocator) frame_allocator;
|
||||
bootstrap.add_free_frames(*fa);
|
||||
|
||||
gather_block_lists(block_slab, used, free, memory_map, map_length, desc_length);
|
||||
block_slab.allocate(); // Make sure we have extra
|
||||
// Build an initial address manager that we'll copy into the real
|
||||
// address manager later (so that we can use a raw allocator now)
|
||||
kutil::allocator &alloc = fa->raw_allocator();
|
||||
kutil::address_manager init_am(alloc);
|
||||
init_am.add_regions(kernel_offset, page_offset - kernel_offset);
|
||||
bootstrap.add_used_frames(init_am);
|
||||
|
||||
// Now go back through these lists and consolidate
|
||||
block_slab.append(frame_block::consolidate(free));
|
||||
// Add the heap into the address manager
|
||||
uintptr_t heap_start = page_offset - kernel_max_heap;
|
||||
init_am.mark(heap_start, kernel_max_heap);
|
||||
|
||||
region_allocator region_slab(frame_size, allocator);
|
||||
region_slab.allocate(); // Allocate some buddy regions for the address_manager
|
||||
kutil::allocator *heap_alloc =
|
||||
new (&g_kernel_heap) kutil::heap_allocator(heap_start, kernel_max_heap);
|
||||
|
||||
// Copy everything into the real address manager
|
||||
kutil::address_manager *am =
|
||||
new (&g_kernel_address_manager) kutil::address_manager(std::move(region_slab));
|
||||
new (&g_kernel_address_manager) kutil::address_manager(
|
||||
std::move(init_am), *heap_alloc);
|
||||
|
||||
am->add_regions(kernel_offset, page_offset - kernel_offset);
|
||||
// Create the page manager
|
||||
page_manager *pm = new (&g_page_manager) page_manager(*fa, *am);
|
||||
|
||||
// Finally, build an acutal set of kernel page tables that just contains
|
||||
// Give the frame_allocator back the rest of the scratch pages
|
||||
fa->free(scratch_phys + (3 * frame_size), scratch_pages - 3);
|
||||
|
||||
// Finally, build an acutal set of kernel page tables where we'll only add
|
||||
// what the kernel actually has mapped, but making everything writable
|
||||
// (especially the page tables themselves)
|
||||
page_table *pml4 = reinterpret_cast<page_table *>(allocator.get_page());
|
||||
page_table *pml4 = &tables[2];
|
||||
pml4 = kutil::offset_pointer(pml4, page_offset);
|
||||
|
||||
kutil::memset(pml4, 0, sizeof(page_table));
|
||||
pml4->entries[511] = reinterpret_cast<uintptr_t>(id_pdp) | 0x10b;
|
||||
|
||||
kutil::frame_allocator *fa =
|
||||
new (&g_frame_allocator) kutil::frame_allocator(std::move(block_slab));
|
||||
page_manager *pm = new (&g_page_manager) page_manager(*fa, *am);
|
||||
bootstrap.page_in_kernel(*pm, pml4);
|
||||
|
||||
// Give the rest to the page_manager's cache for use in page_in
|
||||
pm->free_table_pages(
|
||||
reinterpret_cast<void *>(allocator.current),
|
||||
allocator.left());
|
||||
// Reclaim the old PML4
|
||||
fa->free(scratch_phys, 1);
|
||||
|
||||
for (auto *block : used) {
|
||||
uintptr_t virt_addr = 0;
|
||||
|
||||
switch (block->flags & frame_block_flags::map_mask) {
|
||||
case frame_block_flags::map_ident:
|
||||
virt_addr = block->address;
|
||||
break;
|
||||
|
||||
case frame_block_flags::map_kernel:
|
||||
virt_addr = block->address + kernel_offset;
|
||||
if (block->flags && frame_block_flags::permanent)
|
||||
am->mark_permanent(virt_addr, block->count * frame_size);
|
||||
else
|
||||
am->mark(virt_addr, block->count * frame_size);
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
block->flags -= frame_block_flags::map_mask;
|
||||
if (virt_addr)
|
||||
pm->page_in(pml4, block->address, virt_addr, block->count);
|
||||
}
|
||||
|
||||
fa->init(std::move(free), std::move(used));
|
||||
|
||||
// Put our new PML4 into CR3 to start using it
|
||||
page_manager::set_pml4(pml4);
|
||||
pm->m_kernel_pml4 = pml4;
|
||||
|
||||
// Give the old pml4 back to the page_manager to recycle
|
||||
pm->free_table_pages(reinterpret_cast<void *>(scratch_virt), 1);
|
||||
|
||||
// Set the heap manager
|
||||
new (&g_kernel_heap_manager) kutil::heap_manager(mm_grow_callback);
|
||||
kutil::setup::set_heap(&g_kernel_heap_manager);
|
||||
return *heap_alloc;
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ using memory::kernel_offset;
|
||||
using memory::page_offset;
|
||||
using memory::page_mappable;
|
||||
|
||||
extern kutil::frame_allocator g_frame_allocator;
|
||||
extern kutil::address_manager g_kernel_address_manager;
|
||||
page_manager g_page_manager(
|
||||
g_frame_allocator,
|
||||
@@ -40,7 +39,7 @@ struct free_page_header
|
||||
|
||||
|
||||
page_manager::page_manager(
|
||||
kutil::frame_allocator &frames,
|
||||
frame_allocator &frames,
|
||||
kutil::address_manager &addrs) :
|
||||
m_page_cache(nullptr),
|
||||
m_frames(frames),
|
||||
@@ -341,13 +340,31 @@ page_manager::unmap_table(page_table *table, page_table::level lvl, bool free, p
|
||||
void
|
||||
page_manager::unmap_pages(void* address, size_t count, page_table *pml4)
|
||||
{
|
||||
if (!pml4) pml4 = get_pml4();
|
||||
page_out(pml4, reinterpret_cast<uintptr_t>(address), count, true);
|
||||
if (address >= kernel_offset) {
|
||||
m_addrs.free(address, count);
|
||||
if (!pml4)
|
||||
pml4 = get_pml4();
|
||||
|
||||
uintptr_t iaddr = reinterpret_cast<uintptr_t>(address);
|
||||
|
||||
page_out(pml4, iaddr, count, true);
|
||||
if (iaddr >= kernel_offset) {
|
||||
// TODO
|
||||
// m_addrs.free(address, count);
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
page_manager::fault_handler(uintptr_t addr)
|
||||
{
|
||||
if (!m_addrs.contains(addr))
|
||||
return false;
|
||||
|
||||
uintptr_t page = addr & ~0xfffull;
|
||||
bool user = addr < kernel_offset;
|
||||
map_pages(page, 1, user);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void
|
||||
page_manager::check_needs_page(page_table *table, unsigned index, bool user)
|
||||
{
|
||||
|
||||
@@ -7,9 +7,9 @@
|
||||
|
||||
#include "kutil/address_manager.h"
|
||||
#include "kutil/enum_bitfields.h"
|
||||
#include "kutil/frame_allocator.h"
|
||||
#include "kutil/linked_list.h"
|
||||
#include "kutil/slab_allocator.h"
|
||||
#include "frame_allocator.h"
|
||||
#include "kernel_memory.h"
|
||||
#include "page_table.h"
|
||||
|
||||
@@ -20,7 +20,7 @@ class page_manager
|
||||
{
|
||||
public:
|
||||
page_manager(
|
||||
kutil::frame_allocator &frames,
|
||||
frame_allocator &frames,
|
||||
kutil::address_manager &addrs);
|
||||
|
||||
/// Helper to get the number of pages needed for a given number of bytes.
|
||||
@@ -44,8 +44,9 @@ public:
|
||||
/// \arg pml4 A pointer to the PML4 table to install.
|
||||
static inline void set_pml4(page_table *pml4)
|
||||
{
|
||||
uintptr_t p = reinterpret_cast<uintptr_t>(pml4) - memory::page_offset;
|
||||
__asm__ __volatile__ ( "mov %0, %%cr3" :: "r" (p & ~0xfffull) );
|
||||
constexpr uint64_t phys_mask = ~memory::page_offset & ~0xfffull;
|
||||
uintptr_t p = reinterpret_cast<uintptr_t>(pml4) & phys_mask;
|
||||
__asm__ __volatile__ ( "mov %0, %%cr3" :: "r" (p) );
|
||||
}
|
||||
|
||||
/// Allocate but don't switch to a new PML4 table. This table
|
||||
@@ -113,6 +114,11 @@ public:
|
||||
/// Get a pointer to the kernel's PML4
|
||||
inline page_table * get_kernel_pml4() { return m_kernel_pml4; }
|
||||
|
||||
/// Attempt to handle a page fault.
|
||||
/// \arg addr Address that triggered the fault
|
||||
/// \returns True if the fault was handled
|
||||
bool fault_handler(uintptr_t addr);
|
||||
|
||||
private:
|
||||
/// Copy a physical page
|
||||
/// \arg orig Physical address of the page to copy
|
||||
@@ -169,10 +175,10 @@ private:
|
||||
page_table *m_kernel_pml4; ///< The PML4 of just kernel pages
|
||||
free_page_header *m_page_cache; ///< Cache of free pages to use for tables
|
||||
|
||||
kutil::frame_allocator &m_frames;
|
||||
frame_allocator &m_frames;
|
||||
kutil::address_manager &m_addrs;
|
||||
|
||||
friend void memory_initialize(uint16_t, const void *, size_t, size_t);
|
||||
friend class memory_bootstrap;
|
||||
page_manager(const page_manager &) = delete;
|
||||
};
|
||||
|
||||
@@ -205,4 +211,8 @@ page_table_align(T p)
|
||||
|
||||
|
||||
/// Bootstrap the memory managers.
|
||||
void memory_initialize(uint16_t scratch_pages, const void *memory_map, size_t map_length, size_t desc_length);
|
||||
kutil::allocator & memory_initialize(
|
||||
uint16_t scratch_pages,
|
||||
const void *memory_map,
|
||||
size_t map_length,
|
||||
size_t desc_length);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
#include "kutil/heap_allocator.h"
|
||||
#include "cpu.h"
|
||||
#include "debug.h"
|
||||
#include "log.h"
|
||||
@@ -5,7 +6,7 @@
|
||||
#include "scheduler.h"
|
||||
|
||||
extern "C" void task_fork_return_thunk();
|
||||
|
||||
extern kutil::heap_allocator g_kernel_heap; // TODO: this is a bad hack to get access to the heap
|
||||
|
||||
void
|
||||
process::exit(uint32_t code)
|
||||
@@ -67,7 +68,7 @@ process::setup_kernel_stack()
|
||||
constexpr unsigned null_frame_entries = 2;
|
||||
constexpr size_t null_frame_size = null_frame_entries * sizeof(uint64_t);
|
||||
|
||||
void *stack_bottom = kutil::malloc(initial_stack_size);
|
||||
void *stack_bottom = g_kernel_heap.allocate(initial_stack_size);
|
||||
kutil::memset(stack_bottom, 0, initial_stack_size);
|
||||
|
||||
log::debug(logs::memory, "Created kernel stack at %016lx size 0x%lx",
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
|
||||
using memory::initial_stack;
|
||||
|
||||
scheduler scheduler::s_instance(nullptr);
|
||||
scheduler scheduler::s_instance(nullptr, kutil::allocator::invalid);
|
||||
|
||||
const uint64_t rflags_noint = 0x002;
|
||||
const uint64_t rflags_int = 0x202;
|
||||
@@ -28,9 +28,10 @@ extern "C" {
|
||||
|
||||
extern uint64_t idle_stack_end;
|
||||
|
||||
scheduler::scheduler(lapic *apic) :
|
||||
scheduler::scheduler(lapic *apic, kutil::allocator &alloc) :
|
||||
m_apic(apic),
|
||||
m_next_pid(1)
|
||||
m_next_pid(1),
|
||||
m_process_allocator(alloc)
|
||||
{
|
||||
auto *idle = m_process_allocator.pop();
|
||||
uint8_t last_pri = num_priorities - 1;
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
/// The task scheduler and related definitions
|
||||
|
||||
#include <stdint.h>
|
||||
#include "kutil/allocator.h"
|
||||
#include "kutil/slab_allocator.h"
|
||||
#include "process.h"
|
||||
|
||||
@@ -30,7 +31,8 @@ public:
|
||||
|
||||
/// Constructor.
|
||||
/// \arg apic Pointer to the local APIC object
|
||||
scheduler(lapic *apic);
|
||||
/// \arg alloc Allocator to use for TCBs
|
||||
scheduler(lapic *apic, kutil::allocator &alloc);
|
||||
|
||||
/// Create a new process from a program image in memory.
|
||||
/// \arg name Name of the program image
|
||||
|
||||
Reference in New Issue
Block a user