Implement free() to finish buddy allocator

NOTES.md

@@ -7,10 +7,10 @@
   - The objects get created, but GSI lookup only uses the one at index 0
 - Slab allocator for kernel structures
 - mark kernel memory pages global
-- kernel allocator `free()`
 - lock `memory_manager` and `page_manager` structures
 - Serial out based on circular/bip biffer and interrupts, not spinning on
   `write_ready()`
+- Split out more code into kutil for testing
 
 
 - Device Tree

src/kernel/main.cpp

@@ -46,9 +46,9 @@ init_console(const popcorn_data *header)
 	cons->puts(GIT_VERSION " booting...\n");
 
 	log::init(cons);
-	log::enable(logs::apic, log::level::debug);
+	log::enable(logs::apic, log::level::info);
-	log::enable(logs::devices, log::level::debug);
+	log::enable(logs::devices, log::level::info);
-	log::enable(logs::memory, log::level::info);
+	log::enable(logs::memory, log::level::debug);
 }
 
 void do_error_3() { int x = 1 / 0; }
@@ -81,7 +81,6 @@ kernel_main(popcorn_data *header)
 	log::info(logs::boot, "CPU Family %x Model %x Stepping %x",
 			cpu.family(), cpu.model(), cpu.stepping());
 
-
 	// do_error_1();
 	// __asm__ __volatile__("int $15");
 

src/kernel/memory.cpp

@@ -1,3 +1,4 @@
+#include "kutil/enum_bitfields.h"
 #include "kutil/memory.h"
 #include "assert.h"
 #include "log.h"
@@ -6,18 +7,66 @@
 
 memory_manager g_kernel_memory_manager;
 
-struct memory_manager::alloc_header
+struct memory_manager::mem_header
 {
-	uint64_t size;
+	mem_header(mem_header *prev, mem_header *next, uint8_t size) :
-	uint64_t reserved;
+		m_prev(prev), m_next(next)
-	uint8_t data[0];
+	{
-} __attribute__ ((packed));
+		set_size(size);
+	}
+
+	inline void set_size(uint8_t size)
+	{
+		m_prev = reinterpret_cast<mem_header *>(
+			reinterpret_cast<addr_t>(prev()) | (size & 0x3f));
+	}
+
+	inline void set_used(bool used)
+	{
+		m_next = reinterpret_cast<mem_header *>(
+			reinterpret_cast<addr_t>(next()) | (used ? 1 : 0));
+	}
+
+	inline void set_next(mem_header *next)
+	{
+		bool u = used();
+		m_next = next;
+		set_used(u);
+	}
+
+	inline void set_prev(mem_header *prev)
+	{
+		uint8_t s = size();
+		m_prev = prev;
+		set_size(s);
+	}
+
+	void remove()
+	{
+		if (next()) next()->set_prev(prev());
+		if (prev()) prev()->set_next(next());
+		set_prev(nullptr);
+		set_next(nullptr);
+	}
+
+	inline mem_header * next() { return kutil::mask_pointer(m_next, 0x3f); }
+	inline mem_header * prev() { return kutil::mask_pointer(m_prev, 0x3f); }
+
+	inline mem_header * buddy() const {
+		return reinterpret_cast<mem_header *>(
+			reinterpret_cast<addr_t>(this) ^ (1 << size()));
+	}
+
+	inline bool eldest() const { return this < buddy(); }
+
+	inline uint8_t size() const { return reinterpret_cast<addr_t>(m_prev) & 0x3f; }
+	inline bool used() const { return reinterpret_cast<addr_t>(m_next) & 0x1; }
+
+private:
+	mem_header *m_prev;
+	mem_header *m_next;
+};
 
-struct memory_manager::free_header
-{
-	free_header *next;
-	uint64_t size;
-} __attribute__ ((packed));
 
 memory_manager::memory_manager() :
 	m_start(nullptr),
@@ -37,23 +86,43 @@ memory_manager::memory_manager(void *start) :
 void *
 memory_manager::allocate(size_t length)
 {
-	size_t total = length + sizeof(alloc_header);
+	size_t total = length + sizeof(mem_header);
 	unsigned size = min_size;
 	while (total > (1 << size)) size++;
 	kassert(size < max_size, "Tried to allocate a block bigger than max_size");
 	log::debug(logs::memory, "Allocating %d bytes, which is size %d", total, size);
 
-	alloc_header *header = reinterpret_cast<alloc_header *>(pop_free(size));
+	mem_header *header = pop_free(size);
-	header->size = size;
+	header->set_used(true);
 
-	log::debug(logs::memory, "  Returning %d bytes at %lx", length, &header->data);
+	log::debug(logs::memory, "  Returning %d bytes at %lx", length, header + 1);
-	return &header->data;
+	return header + 1;
 }
 
 void
 memory_manager::free(void *p)
 {
-	// In this simple version, we don't care about freed pointers
+	mem_header *header = reinterpret_cast<mem_header *>(p);
+	header -= 1; // p points after the header
+	header->set_used(false);
+
+	log::debug(logs::memory, "Freeing a block of size %2d at %lx", header->size(), header);
+
+	while (true) {
+		mem_header *buddy = header->buddy();
+		if (buddy->used() || buddy->size() != header->size()) break;
+		log::debug(logs::memory, "  buddy is same size at %lx", buddy);
+		buddy->remove();
+		header = header->eldest() ? header : buddy;
+		header->set_size(header->size() + 1);
+		log::debug(logs::memory, "  joined into size %2d at %lx", header->size(), header);
+	}
+
+	uint8_t size = header->size();
+	header->set_next(get_free(size));
+	get_free(size) = header;
+	if (header->next())
+		header->next()->set_prev(header);
 }
 
 void
@@ -61,57 +130,53 @@ memory_manager::grow_memory()
 {
 	size_t length = (1 << max_size);
 
-	free_header *block = reinterpret_cast<free_header *>(
+	void *next = kutil::offset_pointer(m_start, m_length);
-			reinterpret_cast<uint64_t>(m_start) + m_length);
 
 	g_page_manager.map_pages(
-			reinterpret_cast<page_manager::addr_t>(block),
+			reinterpret_cast<page_manager::addr_t>(next),
 			length / page_manager::page_size);
 
-	block->size = max_size;
+	mem_header *block = new (next) mem_header(nullptr, get_free(max_size), max_size);
-	block->next = get_free(max_size);
 	get_free(max_size) = block;
+	if (block->next())
+		block->next()->set_prev(block);
 	m_length += length;
 
 	log::debug(logs::memory, "Allocated new block at %lx: size %d next %lx",
-			block, block->size, block->next);
+			block, max_size, block->next());
 }
 
 void
 memory_manager::ensure_block(unsigned size)
 {
-	free_header *header = get_free(size);
+	if (get_free(size) != nullptr) return;
-	if (header != nullptr) return;
 	else if (size == max_size) {
 		grow_memory();
 		return;
 	}
 
-	free_header *bigger = pop_free(size + 1);
+	mem_header *orig = pop_free(size + 1);
 
-	uint64_t new_size = bigger->size - 1;
+	mem_header *next = kutil::offset_pointer(orig, 1 << size);
-	free_header *next = reinterpret_cast<free_header *>(
+	new (next) mem_header(orig, nullptr, size);
-			reinterpret_cast<uint64_t>(bigger) + (1 << new_size));
 
-	next->next = bigger->next;
+	orig->set_next(next);
-	next->size = new_size;
+	orig->set_size(size);
+	get_free(size) = orig;
 
-	bigger->next = next;
-	bigger->size = new_size;
 	log::debug(logs::memory, "ensure_block[%2d] split blocks:", size);
-	log::debug(logs::memory, "   %lx: size %d next %lx", bigger, bigger->size, bigger->next);
+	log::debug(logs::memory, "   %lx: size %d next %lx", orig, size, orig->next());
-	log::debug(logs::memory, "   %lx: size %d next %lx", next, next->size, next->next);
+	log::debug(logs::memory, "   %lx: size %d next %lx", next, size, next->next());
-
-	get_free(size) = bigger;
 }
 
-memory_manager::free_header *
+memory_manager::mem_header *
 memory_manager::pop_free(unsigned size)
 {
 	ensure_block(size);
-	free_header *block = get_free(size);
+	mem_header *block = get_free(size);
-	get_free(size) = block->next;
+	get_free(size) = block->next();
-	block->next = nullptr;
+
+	block->remove();
 	return block;
 }
 

src/kernel/memory.h

@@ -24,8 +24,7 @@ public:
 	void free(void *p);
 
 private:
-	struct alloc_header;
+	class mem_header;
-	struct free_header;
 
 	/// Expand the size of memory
 	void grow_memory();
@@ -35,15 +34,18 @@ private:
 	void ensure_block(unsigned size);
 
 	/// Helper accessor for the list of blocks of a given size
-	free_header *& get_free(unsigned size)  { return m_free[size - min_size]; }
+	mem_header *& get_free(unsigned size)  { return m_free[size - min_size]; }
 
 	/// Helper to get a block of the given size, growing if necessary
-	free_header * pop_free(unsigned size);
+	mem_header * pop_free(unsigned size);
 
-	static const unsigned min_size = 6;  ///< Minimum block size is (2^min_size)
+	/// Minimum block size is (2^min_size). Must be at least 6.
-	static const unsigned max_size = 16; ///< Maximum block size is (2^max_size)
+	static const unsigned min_size = 6;
 
-	free_header *m_free[max_size - min_size];
+	/// Maximum block size is (2^max_size). Must be less than 64.
+	static const unsigned max_size = 16;
+
+	mem_header *m_free[max_size - min_size];
 	void *m_start;
 	size_t m_length;
 
@@ -65,3 +67,5 @@ inline void * kalloc(size_t length) { return g_kernel_memory_manager.allocate(le
 /// Free kernel space memory.
 /// \arg p   The pointer to free
 inline void kfree(void *p) { g_kernel_memory_manager.free(p); }
+
+void * operator new (size_t, void *p);

src/modules/kutil/memory.h

@@ -19,4 +19,10 @@ inline T * offset_pointer(T *p, size_t offset)
 	return reinterpret_cast<T *>(reinterpret_cast<addr_t>(p) + offset);
 }
 
+template <typename T>
+inline T* mask_pointer(T *p, addr_t mask)
+{
+	return reinterpret_cast<T *>(reinterpret_cast<addr_t>(p) & ~mask);
+}
+
 } // namespace kutil