justin/jsix
1
1
Fork 0
mirror of https://github.com/justinian/jsix.git synced 2025-12-10 08:24:32 -08:00
Code Issues Packages Projects Releases Wiki Activity

[kernel] Change heap alloc for better alignment

Browse Source 
Created a new util/node_map.h that implements a map that grows in-place.
Now this is used for tracking blocks' size orders, instead of a header
at the start of the memory block. This allows the whole buddy block to
be allocated, allowing for page-aligned (or greater) blocks to be
requested from the heap.
This commit is contained in:
Justin C. Miller
2022-10-02 17:27:21 -07:00
parent 11b61ab345
commit e90647d498
9 changed files with 518 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

233
src/kernel/heap_allocator.cpp
View File

@@ -9,69 +9,43 @@
#include "heap_allocator.h"
#include "memory.h"
struct heap_allocator::mem_header
uint32_t & get_map_key(heap_allocator::block_info &info) { return info.offset; }
struct heap_allocator::free_header
{
mem_header(mem_header *prev, mem_header *next, uint8_t order) :
m_prev(prev), m_next(next)
{
set_order(order);
}
inline void set_order(uint8_t order) {
m_prev = reinterpret_cast<mem_header *>(
reinterpret_cast<uintptr_t>(prev()) | (order & 0x3f));
}
inline void set_used(bool used) {
m_next = reinterpret_cast<mem_header *>(
reinterpret_cast<uintptr_t>(next()) | (used ? 1 : 0));
}
inline void set_next(mem_header *next) {
bool u = used();
m_next = next;
set_used(u);
}
inline void set_prev(mem_header *prev) {
uint8_t s = order();
m_prev = prev;
set_order(s);
void clear(unsigned new_order) {
prev = next = nullptr;
order = new_order;
}
void remove() {
if (next()) next()->set_prev(prev());
if (prev()) prev()->set_next(next());
set_prev(nullptr);
set_next(nullptr);
if (next) next->prev = prev;
if (prev) prev->next = next;
prev = next = nullptr;
}
inline mem_header * next() { return util::mask_pointer(m_next, 0x3f); }
inline mem_header * prev() { return util::mask_pointer(m_prev, 0x3f); }
inline mem_header * buddy() const {
return reinterpret_cast<mem_header *>(
reinterpret_cast<uintptr_t>(this) ^ (1 << order()));
inline free_header * buddy() const {
return reinterpret_cast<free_header *>(
reinterpret_cast<uintptr_t>(this) ^ (1 << order));
}
inline bool eldest() const { return this < buddy(); }
inline uint8_t order() const { return reinterpret_cast<uintptr_t>(m_prev) & 0x3f; }
inline bool used() const { return reinterpret_cast<uintptr_t>(m_next) & 0x1; }
private:
mem_header *m_prev;
mem_header *m_next;
free_header *prev;
free_header *next;
unsigned order;
};
heap_allocator::heap_allocator() : m_start {0}, m_end {0} {}
heap_allocator::heap_allocator(uintptr_t start, size_t size) :
heap_allocator::heap_allocator(uintptr_t start, size_t size, uintptr_t heapmap) :
m_start {start},
m_end {start+size},
m_blocks {0},
m_allocated_size {0}
m_end {start},
m_maxsize {size},
m_allocated_size {0},
m_map (reinterpret_cast<block_info*>(heapmap), 512)
{
memset(m_free, 0, sizeof(m_free));
}
@@ -79,12 +53,10 @@ heap_allocator::heap_allocator(uintptr_t start, size_t size) :
void *
heap_allocator::allocate(size_t length)
{
size_t total = length + sizeof(mem_header);
if (length == 0)
return nullptr;
unsigned order = util::log2(total);
unsigned order = util::log2(length);
if (order < min_order)
order = min_order;
@@ -94,10 +66,16 @@ heap_allocator::allocate(size_t length)
util::scoped_lock lock {m_lock};
mem_header *header = pop_free(order);
header->set_used(true);
m_allocated_size += (1 << order);
return header + 1;
free_header *block = pop_free(order);
if (!block && !split_off(order, block)) {
return new_block(order);
}
m_map[map_key(block)].free = false;
return block;
}
void
@@ -111,70 +89,107 @@ heap_allocator::free(void *p)
util::scoped_lock lock {m_lock};
mem_header *header = reinterpret_cast<mem_header *>(p);
header -= 1; // p points after the header
header->set_used(false);
m_allocated_size -= (1 << header->order());
free_header *block = reinterpret_cast<free_header *>(p);
block_info *info = m_map.find(map_key(block));
kassert(info, "Attempt to free pointer not known to the heap");
if (!info) return;
while (header->order() != max_order) {
auto order = header->order();
m_allocated_size -= (1 << info->order);
mem_header *buddy = header->buddy();
if (buddy->used() || buddy->order() != order)
break;
if (get_free(order) == buddy)
get_free(order) = buddy->next();
buddy->remove();
header = header->eldest() ? header : buddy;
header->set_order(order + 1);
}
uint8_t order = header->order();
header->set_next(get_free(order));
get_free(order) = header;
if (header->next())
header->next()->set_prev(header);
block->clear(info->order);
block = merge_block(block);
register_free_block(block, block->order);
}
void
heap_allocator::ensure_block(unsigned order)
{
if (get_free(order) != nullptr)
return;
if (order == max_order) {
size_t bytes = (1 << max_order);
uintptr_t next = m_start + m_blocks * bytes;
if (next + bytes <= m_end) {
mem_header *nextp = reinterpret_cast<mem_header *>(next);
new (nextp) mem_header(nullptr, nullptr, order);
get_free(order) = nextp;
++m_blocks;
}
} else {
mem_header *orig = pop_free(order + 1);
if (orig) {
mem_header *next = util::offset_pointer(orig, 1 << order);
new (next) mem_header(orig, nullptr, order);
orig->set_next(next);
orig->set_order(order);
get_free(order) = orig;
}
}
}
heap_allocator::mem_header *
heap_allocator::free_header *
heap_allocator::pop_free(unsigned order)
{
ensure_block(order);
mem_header *block = get_free(order);
free_header *block = get_free(order);
if (block) {
get_free(order) = block->next();
get_free(order) = block->next;
block->remove();
}
return block;
}
heap_allocator::free_header *
heap_allocator::merge_block(free_header *block)
{
// The lock needs to be held while calling merge_block
unsigned order = block->order;
while (order < max_order) {
block_info *info = m_map.find(map_key(block->buddy()));
if (!info || !info->free || info->order != order)
break;
free_header *buddy = block->buddy();
if (get_free(order) == buddy)
get_free(order) = buddy->next;
buddy->remove();
block = block->eldest() ? block : buddy;
m_map.erase(map_key(block->buddy()));
block->order = m_map[map_key(block)].order = ++order;
}
return block;
}
void *
heap_allocator::new_block(unsigned order)
{
// The lock needs to be held while calling new_block
// Add the largest blocks possible until m_end is
// aligned to be a block of the requested order
unsigned current = address_order(m_end);
while (current < order) {
register_free_block(reinterpret_cast<free_header*>(m_end), current);
m_end += 1 << current;
current = address_order(m_end);
}
void *block = reinterpret_cast<void*>(m_end);
m_end += 1 << order;
m_map[map_key(block)].order = order;
return block;
}
void
heap_allocator::register_free_block(free_header *block, unsigned order)
{
// The lock needs to be held while calling register_free_block
block_info &info = m_map[map_key(block)];
info.free = true;
info.order = order;
block->clear(order);
block->next = get_free(order);
get_free(order) = block;
}
bool
heap_allocator::split_off(unsigned order, free_header *&block)
{
// The lock needs to be held while calling split_off
const unsigned next = order + 1;
if (next > max_order) {
block = nullptr;
return false;
}
block = pop_free(next);
if (!block && !split_off(next, block))
return false;
block->order = order;
free_header *buddy = block->buddy();
register_free_block(block->buddy(), order);
m_map[map_key(block)].order = order;
return true;
}

75
src/kernel/heap_allocator.h
View File

@@ -5,7 +5,7 @@
#include <stddef.h>
#include <util/spinlock.h>
#include <util/node_map.h>
/// Allocator for a given heap range
class heap_allocator
@@ -15,9 +15,10 @@ public:
heap_allocator();
/// Constructor. The given memory area must already have been reserved.
/// \arg start Starting address of the heap
/// \arg size Size of the heap in bytes
heap_allocator(uintptr_t start, size_t size);
/// \arg start Starting address of the heap
/// \arg size Size of the heap in bytes
/// \arg heapmap Starting address of the heap tracking map
heap_allocator(uintptr_t start, size_t size, uintptr_t heapmap);
/// Allocate memory from the area managed.
/// \arg length The amount of memory to allocate, in bytes
@@ -30,34 +31,74 @@ public:
void free(void *p);
/// Minimum block size is (2^min_order). Must be at least 6.
static const unsigned min_order = 6;
static const unsigned min_order = 6; // 2^6 == 64 B
/// Maximum block size is (2^max_order). Must be less than 64.
static const unsigned max_order = 22;
/// Maximum block size is (2^max_order). Must be less than 32 + min_order.
static const unsigned max_order = 22; // 2^22 == 4 MiB
protected:
class mem_header;
struct free_header;
struct block_info
{
uint32_t offset;
uint8_t order;
bool free;
};
friend uint32_t & get_map_key(block_info &info);
/// Ensure there is a block of a given order, recursively splitting
/// \arg order Order (2^N) of the block we want
void ensure_block(unsigned order);
inline uint32_t map_key(void *p) const {
return static_cast<uint32_t>(
(reinterpret_cast<uintptr_t>(p) - m_start) >> min_order);
}
using block_map = util::inplace_map<uint32_t, block_info, -1u>;
/// Get the largest block size order that aligns with this address
inline unsigned address_order(uintptr_t addr) {
unsigned tz = __builtin_ctzll(addr);
return tz > max_order ? max_order : tz;
}
/// Helper accessor for the list of blocks of a given order
/// \arg order Order (2^N) of the block we want
/// \returns A mutable reference to the head of the list
mem_header *& get_free(unsigned order) { return m_free[order - min_order]; }
free_header *& get_free(unsigned order) { return m_free[order - min_order]; }
/// Helper to get a block of the given order, growing if necessary
/// Helper to remove and return the first block in the free
/// list for the given order.
free_header * pop_free(unsigned order);
/// Merge the given block with any currently free buddies to
/// create the largest block possible.
/// \arg block The current block
/// \returns The fully-merged block
free_header * merge_block(free_header *block);
/// Create a new block of the given order past the end of the existing
/// heap. The block will be marked as non-free.
/// \arg order The requested size order
/// \returns A pointer to the block's memory
void * new_block(unsigned order);
/// Register the given block as free with the given order.
/// \arg block The newly-created or freed block
/// \arg order The size order to set on the block
void register_free_block(free_header *block, unsigned order);
/// Helper to get a block of the given order by splitting existing
/// larger blocks. Returns false if there were no larger blocks.
/// \arg order Order (2^N) of the block we want
/// \returns A detached block of the given order
mem_header * pop_free(unsigned order);
/// \arg block [out] Receives a pointer to the requested block
/// \returns True if a split was done
bool split_off(unsigned order, free_header *&block);
uintptr_t m_start, m_end;
size_t m_blocks;
mem_header *m_free[max_order - min_order + 1];
size_t m_maxsize;
free_header *m_free[max_order - min_order + 1];
size_t m_allocated_size;
util::spinlock m_lock;
block_map m_map;
heap_allocator(const heap_allocator &) = delete;
};

11
src/kernel/memory_bootstrap.cpp
View File

@@ -37,6 +37,9 @@ frame_allocator &g_frame_allocator = __g_frame_allocator_storage.value;
static util::no_construct<obj::vm_area_untracked> __g_kernel_heap_area_storage;
obj::vm_area_untracked &g_kernel_heap_area = __g_kernel_heap_area_storage.value;
static util::no_construct<obj::vm_area_untracked> __g_kernel_heapmap_area_storage;
obj::vm_area_untracked &g_kernel_heapmap_area = __g_kernel_heapmap_area_storage.value;
static util::no_construct<obj::vm_area_guarded> __g_kernel_stacks_storage;
obj::vm_area_guarded &g_kernel_stacks = __g_kernel_stacks_storage.value;
@@ -73,7 +76,6 @@ memory_initialize_pre_ctors(bootproto::args &kargs)
page_table *kpml4 = static_cast<page_table*>(kargs.pml4);
new (&g_kernel_heap) heap_allocator {mem::heap_offset, mem::heap_size};
frame_block *blocks = reinterpret_cast<frame_block*>(mem::bitmap_offset);
new (&g_frame_allocator) frame_allocator {blocks, kargs.frame_blocks.count};
@@ -87,13 +89,20 @@ memory_initialize_pre_ctors(bootproto::args &kargs)
reg = reg->next;
}
obj::process *kp = obj::process::create_kernel_process(kpml4);
vm_space &vm = kp->space();
obj::vm_area *heap = new (&g_kernel_heap_area)
obj::vm_area_untracked(mem::heap_size, vm_flags::write);
obj::vm_area *heap_map = new (&g_kernel_heapmap_area)
obj::vm_area_untracked(mem::heapmap_size, vm_flags::write);
vm.add(mem::heap_offset, heap);
vm.add(mem::heapmap_offset, heap_map);
new (&g_kernel_heap) heap_allocator {mem::heap_offset, mem::heap_size, mem::heapmap_offset};
obj::vm_area *stacks = new (&g_kernel_stacks) obj::vm_area_guarded {
mem::stacks_offset,