[kernel] Add VMA interface

Finished the VMA kobject and added the related syscalls. Processes can
now allocate memory! Other changes in this commit:

- stop using g_frame_allocator and add frame_allocator::get()
- make sure to release all handles in the process dtor
- fix kutil::map::iterator never comparing to end()

modules.yaml

@@ -43,7 +43,6 @@ modules:
             - src/kernel/objects/thread.cpp
             - src/kernel/objects/process.cpp
             - src/kernel/objects/vm_area.cpp
-            - src/kernel/page_manager.cpp
             - src/kernel/page_table.cpp
             - src/kernel/pci.cpp
             - src/kernel/scheduler.cpp
@@ -59,6 +58,7 @@ modules:
             - src/kernel/syscalls/process.cpp
             - src/kernel/syscalls/system.cpp
             - src/kernel/syscalls/thread.cpp
+            - src/kernel/syscalls/vm_area.cpp
             - src/kernel/task.s
             - src/kernel/vm_space.cpp
             - src/kernel/crtn.s

src/drivers/nulldrv/main.cpp

@@ -61,7 +61,17 @@ main(int argc, const char **argv)
 
 	_syscall_system_log("main thread starting");
 
-	j6_status_t result = _syscall_endpoint_create(&endp);
+	uintptr_t base = 0xcc0000000;
+	j6_handle_t vma = j6_handle_invalid;
+	j6_status_t result = _syscall_vma_create_map(&vma, 0x100000, base);
+	if (result != j6_status_ok)
+		return result;
+
+	uint64_t *vma_ptr = reinterpret_cast<uint64_t*>(base);
+	for (int i = 0; i < 4096; ++i)
+		vma_ptr[i] = uint64_t(i);
+
+	result = _syscall_endpoint_create(&endp);
 	if (result != j6_status_ok)
 		return result;
 

src/include/syscalls.inc

@@ -1,7 +1,7 @@
-SYSCALL(0x00, system_log,       const char *)
+SYSCALL(0x00, system_log,        const char *)
-SYSCALL(0x01, system_noop,      void)
+SYSCALL(0x01, system_noop,       void)
 
-SYSCALL(0x09, object_wait,      j6_handle_t, j6_signal_t, j6_signal_t *)
+SYSCALL(0x09, object_wait,       j6_handle_t, j6_signal_t, j6_signal_t *)
 SYSCALL(0x0a, object_signal,     j6_handle_t, j6_signal_t)
 
 SYSCALL(0x10, process_koid,      j6_koid_t *)
@@ -24,3 +24,8 @@ SYSCALL(0x2a, endpoint_send,     j6_handle_t, size_t, void *)
 SYSCALL(0x2b, endpoint_receive,  j6_handle_t, size_t *, void *)
 SYSCALL(0x2c, endpoint_sendrecv, j6_handle_t, size_t *, void *)
 
+SYSCALL(0x30, vma_create,        j6_handle_t *, size_t)
+SYSCALL(0x31, vma_create_map,    j6_handle_t *, size_t, uintptr_t)
+SYSCALL(0x32, vma_close,         j6_handle_t)
+SYSCALL(0x33, vma_map,           j6_handle_t, uintptr_t)
+SYSCALL(0x34, vma_unmap,         j6_handle_t)

src/kernel/frame_allocator.cpp

@@ -18,6 +18,13 @@ frame_block::compare(const frame_block &rhs) const
 }
 
 
+frame_allocator &
+frame_allocator::get()
+{
+	extern frame_allocator &g_frame_allocator;
+	return g_frame_allocator;
+}
+
 frame_allocator::frame_allocator() {}
 
 size_t

src/kernel/frame_allocator.h

@@ -29,6 +29,9 @@ public:
 	/// \arg count    The number of frames to be freed
 	void free(uintptr_t address, size_t count);
 
+	/// Get the global frame allocator
+	static frame_allocator & get();
+
 private:
 	frame_block_list m_free; ///< Free frames list
 

src/kernel/objects/process.cpp

@@ -32,6 +32,8 @@ process::process(page_table *kpml4) :
 
 process::~process()
 {
+	for (auto &it : m_handles)
+		if (it.val) it.val->handle_release();
 	s_processes.remove_swap(this);
 }
 

src/kernel/objects/vm_area.cpp

@@ -1,6 +1,7 @@
+#include "frame_allocator.h"
 #include "kernel_memory.h"
-#include "objects/process.h"
 #include "objects/vm_area.h"
+#include "vm_space.h"
 
 using memory::frame_size;
 
@@ -13,6 +14,12 @@ vm_area::vm_area(size_t size, vm_flags flags) :
 
 vm_area::~vm_area()
 {
+	for (auto &it : m_spaces)
+		remove_from(it.key);
+
+	frame_allocator &fa = frame_allocator::get();
+	for (auto &m : m_mappings)
+		fa.free(m.phys, m.count);
 }
 
 size_t
@@ -21,40 +28,38 @@ vm_area::resize(size_t size)
 	return m_size;
 }
 
-j6_status_t
+void
-vm_area::add_to(vm_space *space, uintptr_t *base)
+vm_area::add_to(vm_space *space, uintptr_t base)
 {
-	if (!base || !space)
+	kassert(space, "null vm_space passed to vm_area::add_to");
-		return j6_err_invalid_arg;
 
-	uintptr_t *prev = m_procs.find(space);
+	// Multiple copies in the same space not yet supported
-	if (prev) {
+	uintptr_t *prev = m_spaces.find(space);
-		*base = *prev;
+	if (prev) return;
-		return j6_status_exists;
+
+	if (m_spaces.count()) {
+		// Just get the first address space in the map
+		auto it = m_spaces.begin();
+		vm_space *source = it->key;
+		uintptr_t from = it->val;
+		size_t pages = memory::page_count(m_size);
+		space->copy_from(*source, from, base, pages);
 	}
 
-	if (!*base)
+	m_spaces.insert(space, base);
-		return j6_err_nyi;
+	space->add(base, this);
-
-	m_procs.insert(space, *base);
-	for (auto &m : m_mappings)
-		if (m.state == state::mapped)
-			space->page_in(*base + m.offset, m.count, m.phys);
-
-	return j6_status_ok;
 }
 
-j6_status_t
+void
 vm_area::remove_from(vm_space *space)
 {
-	uintptr_t *base = m_procs.find(space);
+	size_t pages = memory::page_count(m_size);
+	uintptr_t *base = m_spaces.find(space);
 	if (space && base) {
-		for (auto &m : m_mappings)
+		space->clear(*base, pages);
-			if (m.state == state::mapped)
+		m_spaces.erase(space);
-				space->clear(*base + m.offset, m.count);
-		m_procs.erase(space);
 	}
-	return j6_status_ok;
+	space->remove(this);
 }
 
 size_t
@@ -82,84 +87,34 @@ vm_area::overlaps(uintptr_t offset, size_t pages, size_t *count)
 bool
 vm_area::commit(uintptr_t phys, uintptr_t offset, size_t count)
 {
-	return add(offset, count, state::mapped, phys);
-}
-
-bool
-vm_area::uncommit(uintptr_t offset, size_t count)
-{
-	return remove(offset, count, state::reserved);
-}
-
-bool
-vm_area::reserve(uintptr_t offset, size_t count)
-{
-	return add(offset, count, state::reserved, 0);
-}
-
-bool
-vm_area::unreserve(uintptr_t offset, size_t count)
-{
-	return remove(offset, count, state::reserved);
-}
-
-vm_area::state
-vm_area::get(uintptr_t offset, uintptr_t *phys)
-{
-	size_t n = 0;
-	size_t o = overlaps(offset, 1, &n);
-	if (n) {
-		mapping &m = m_mappings[o];
-		if (phys) *phys = m.phys;
-		return m.state;
-	}
-
-	return state::none;
-}
-
-bool
-vm_area::add(uintptr_t offset, size_t count, state desired, uintptr_t phys)
-{
-	const bool do_map = desired == state::mapped;
-
 	size_t n = 0;
 	size_t o = overlaps(offset, count, &n);
-	if (!n) {
-		// In the clear, map it
-		size_t o = m_mappings.sorted_insert({
-			.offset = offset,
-			.count = count,
-			.phys = phys,
-			.state = desired});
-		n = 1;
 
-		if (do_map)
+	// Mapping overlaps not allowed
-			map(offset, count, phys);
+	if (n) return false;
-	} else if (desired == state::mapped) {
-		// Mapping overlaps not allowed
-		return false;
-	}
 
-	// Any overlaps with different states is not allowed
+	o = m_mappings.sorted_insert({
-	for (size_t i = o; i < o+n; ++i)
+		.offset = offset,
-		if (m_mappings[i].state != desired)
+		.count = count,
-			return false;
+		.phys = phys});
+	n = 1;
+
+	map(offset, count, phys);
 
 	// Try to expand to abutting similar areas
 	if (o > 0 &&
-		m_mappings[o-1].state == desired &&
 		m_mappings[o-1].end() == offset &&
-		(!do_map || m_mappings[o-1].phys_end() == phys)) {
+		m_mappings[o-1].phys_end() == phys) {
 		--o;
 		++n;
 	}
 
 	uintptr_t end = offset + count * frame_size;
-	uintptr_t pend = offset + count * frame_size;
+	uintptr_t pend = phys + count * frame_size;
+
 	if (o + n < m_mappings.count() &&
-		m_mappings[o+n].state == desired &&
 		end == m_mappings[o+n].offset &&
-		(!do_map || m_mappings[o-1].phys == pend)) {
+		m_mappings[o-1].phys == pend) {
 		++n;
 	}
 
@@ -174,7 +129,7 @@ vm_area::add(uintptr_t offset, size_t count, state desired, uintptr_t phys)
 		(end > last.end() ? end : last.end()) -
 		first.offset;
 
-	first.count = diff / frame_size;
+	first.count = memory::page_count(diff);
 
 	if (n > 1)
 		m_mappings.remove_at(o+1, n-1);
@@ -182,19 +137,13 @@ vm_area::add(uintptr_t offset, size_t count, state desired, uintptr_t phys)
 	return true;
 }
 
-
 bool
-vm_area::remove(uintptr_t offset, size_t count, state expected)
+vm_area::uncommit(uintptr_t offset, size_t count)
 {
 	size_t n = 0;
 	size_t o = overlaps(offset, count, &n);
 	if (!n) return true;
 
-	// Any overlaps with different states is not allowed
-	for (size_t i = o; i < o+n; ++i)
-		if (m_mappings[i].state != expected)
-			return false;
-
 	mapping *first = &m_mappings[o];
 	mapping *last = &m_mappings[o+n-1];
 
@@ -208,22 +157,19 @@ vm_area::remove(uintptr_t offset, size_t count, state expected)
 		size_t i = m_mappings.sorted_insert({
 			.offset = end,
 			.count = trailing / frame_size,
-			.state = first->state,
 			});
 		last = &m_mappings[i];
 		trailing = 0;
 
 		first->count -= last->count;
-		if (first->state == state::mapped)
+		last->phys = first->phys + first->count * frame_size;
-			last->phys = first->phys + first->count * frame_size;
 	}
 
 	if (leading) {
 		size_t remove_pages = first->count; 
 		first->count = leading / frame_size;
 		remove_pages -= first->count;
-		if (expected == state::mapped)
+		unmap(first->end(), remove_pages);
-			unmap(first->end(), remove_pages);
 	}
 
 	if (trailing) {
@@ -232,10 +178,8 @@ vm_area::remove(uintptr_t offset, size_t count, state expected)
 		last->offset = end;
 		last->count = trailing / frame_size;
 		remove_pages -= last->count;
-		if (expected == state::mapped) {
+		unmap(remove_off, remove_pages);
-			unmap(remove_off, remove_pages);
+		last->phys += remove_pages * frame_size;
-			last->phys += remove_pages * frame_size;
-		}
 	}
 
 	size_t delete_start = 0;
@@ -245,8 +189,7 @@ vm_area::remove(uintptr_t offset, size_t count, state expected)
 		if (offset <= m.offset && end >= m.end()) {
 			if (!delete_count) delete_start = i;
 			++delete_count;
-			if (expected == state::mapped)
+			unmap(m.offset, m.count);
-				unmap(m.offset, m.count);
 		}
 	}
 
@@ -259,20 +202,26 @@ vm_area::remove(uintptr_t offset, size_t count, state expected)
 void
 vm_area::map(uintptr_t offset, size_t count, uintptr_t phys)
 {
-	for (auto &it : m_procs) {
+	for (auto &it : m_spaces) {
 		uintptr_t addr = it.val + offset;
 		vm_space *space = it.key;
-		space->page_in(addr, count, phys);
+		space->page_in(addr, phys, count);
 	} 
 }
 
 void
 vm_area::unmap(uintptr_t offset, size_t count)
 {
-	for (auto &it : m_procs) {
+	for (auto &it : m_spaces) {
 		uintptr_t addr = it.val + offset;
 		vm_space *space = it.key;
 		space->clear(addr, count);
 	}
 }
 
+void
+vm_area::on_no_handles()
+{
+	kobject::on_no_handles();
+	delete this;
+}

src/kernel/objects/vm_area.h

@@ -19,9 +19,7 @@ enum class vm_flags : uint32_t
 	contiguous      = 0x00000002,
 
 	large_pages     = 0x00000100,
-	huge_pages      = 0x00000200,
+	huge_pages      = 0x00000200
-
-	offset_linear   = 0x80000000
 };
 
 IS_BITFIELD(vm_flags);
@@ -55,14 +53,12 @@ public:
 	/// the given base address is zero, a base address will be chosen
 	/// automatically.
 	/// \arg s    The target address space
-	/// \arg base [in] The desired base address [out] the actual base address
+	/// \arg base The base address this area will be mapped to
-	/// \returns  j6_status_ok on success
+	void add_to(vm_space *s, uintptr_t base);
-	j6_status_t add_to(vm_space *s, uintptr_t *base);
 
 	/// Remove this virtual area from a process' virtual address space.
 	/// \arg s    The target address space
-	/// \returns  j6_status_ok on success
+	void remove_from(vm_space *s);
-	j6_status_t remove_from(vm_space *s);
 
 	/// Commit contiguous physical pages to this area
 	/// \arg phys   The physical address of the first page
@@ -77,35 +73,17 @@ public:
 	/// \returns    True if successful
 	bool uncommit(uintptr_t offset, size_t count);
 
-	/// Reserve a range of this area to never commit
-	/// \arg offset The offset from the start of this area
-	/// \arg count  The number of pages
-	/// \returns    True if successful
-	bool reserve(uintptr_t offset, size_t count);
-
-	/// Unreserve a range of this area to allow commits
-	/// \arg offset The offset from the start of this area
-	/// \arg count  The number of pages
-	/// \returns    True if successful
-	bool unreserve(uintptr_t offset, size_t count);
-
-	enum class state : uint8_t { none, reserved, mapped };
-
-	/// Get the physical page representing an offset in this area
-	/// \arg offset  The offset into the area
-	/// \arg phys    [out] The physical page address
-	/// \returns     State of the given address
-	state get(uintptr_t offset, uintptr_t *phys);
-
 	/// Get the flags set for this area
 	vm_flags flags() const { return m_flags; }
 
+protected:
+	virtual void on_no_handles() override;
+
 private:
 	struct mapping {
 		uintptr_t offset;
 		size_t count;
 		uintptr_t phys;
-		state state;
 
 		int compare(const struct mapping &o) const {
 			return offset > o.offset ? 1 : offset < o.offset ? -1 : 0;
@@ -116,14 +94,12 @@ private:
 	};
 
 	size_t overlaps(uintptr_t offset, size_t pages, size_t *count);
-	bool add(uintptr_t offset, size_t count, state desired, uintptr_t phys);
-	bool remove(uintptr_t offset, size_t count, state expected);
 
 	void map(uintptr_t offset, size_t count, uintptr_t phys);
 	void unmap(uintptr_t offset, size_t count);
 
 	size_t m_size;
 	vm_flags m_flags;
-	kutil::map<vm_space*, uintptr_t> m_procs;
+	kutil::map<vm_space*, uintptr_t> m_spaces;
 	kutil::vector<mapping> m_mappings;
 };

src/kernel/page_table.cpp

@@ -8,8 +8,6 @@
 using memory::page_offset;
 using level = page_table::level;
 
-extern frame_allocator &g_frame_allocator;
-
 free_page_header * page_table::s_page_cache = nullptr;
 size_t page_table::s_cache_count = 0;
 constexpr size_t page_table::entry_sizes[4];
@@ -157,17 +155,16 @@ page_table::iterator::ensure_table(level l)
 	if (l == level::pml4 || l > level::pt) return;
 	if (check_table(l)) return;
 
-	uintptr_t phys = 0;
+	page_table *table = page_table::get_table_page();
-	size_t n = g_frame_allocator.allocate(1, &phys);
+	uintptr_t phys = reinterpret_cast<uintptr_t>(table) & ~page_offset;
-	kassert(n, "Failed to allocate a page table");
 
 	uint64_t &parent = entry(l - 1);
 	flag flags = table_flags | (parent & flag::allowed);
 	if (m_index[0] < memory::pml4e_kernel)
 		flags |= flag::user;
 
-	m_table[unsigned(l)] = reinterpret_cast<page_table*>(phys | page_offset);
+	m_table[unsigned(l)] = table;
-	parent  = (reinterpret_cast<uintptr_t>(phys) & ~0xfffull) | flags;
+	parent = (phys & ~0xfffull) | flags;
 }
 
 page_table *
@@ -223,9 +220,10 @@ page_table::fill_table_page_cache()
 {
 	constexpr size_t min_pages = 16;
 
+	frame_allocator &fa = frame_allocator::get();
 	while (s_cache_count < min_pages) {
 		uintptr_t phys = 0;
-		size_t n = g_frame_allocator.allocate(min_pages - s_cache_count, &phys);
+		size_t n = fa.allocate(min_pages - s_cache_count, &phys);
 
 		free_page_header *start =
 			memory::to_virtual<free_page_header>(phys);
@@ -250,11 +248,12 @@ page_table::free(page_table::level l)
 		? memory::pml4e_kernel
 		: memory::table_entries;
 
+	frame_allocator &fa = frame_allocator::get();
 	for (unsigned i = 0; i < last; ++i) {
 		if (!is_present(i)) continue;
 		if (is_page(l, i)) {
 			size_t count = memory::page_count(entry_sizes[unsigned(l)]);
-			g_frame_allocator.free(entries[i] & ~0xfffull, count);
+			fa.free(entries[i] & ~0xfffull, count);
 		} else {
 			get(i)->free(l + 1);
 		}

src/kernel/syscalls/vm_area.cpp

@@ -0,0 +1,61 @@
+#include "j6/errors.h"
+#include "j6/signals.h"
+#include "j6/types.h"
+
+#include "log.h"
+#include "objects/process.h"
+#include "objects/vm_area.h"
+#include "syscalls/helpers.h"
+#include "vm_space.h"
+
+namespace syscalls {
+
+j6_status_t
+vma_create(j6_handle_t *handle, size_t size)
+{
+	construct_handle<vm_area>(handle, size);
+	return j6_status_ok;
+}
+
+j6_status_t
+vma_create_map(j6_handle_t *handle, size_t size, uintptr_t base)
+{
+	vm_area *a = construct_handle<vm_area>(handle, size);
+	a->add_to(&process::current().space(), base);
+	return j6_status_ok;
+}
+
+j6_status_t
+vma_map(j6_handle_t handle, uintptr_t base)
+{
+	vm_area *a = get_handle<vm_area>(handle);
+	if (!a) return j6_err_invalid_arg;
+
+	a->add_to(&process::current().space(), base);
+	return j6_status_ok;
+}
+
+j6_status_t
+vma_unmap(j6_handle_t handle)
+{
+	vm_area *a = get_handle<vm_area>(handle);
+	if (!a) return j6_err_invalid_arg;
+
+	a->remove_from(&process::current().space());
+	return j6_status_ok;
+}
+
+j6_status_t
+vma_close(j6_handle_t handle)
+{
+	j6_status_t status = vma_unmap(handle);
+	if (status != j6_status_ok)
+		return status;
+
+	remove_handle<vm_area>(handle);
+	return j6_status_ok;
+}
+
+
+
+} // namespace syscalls

src/kernel/vm_space.cpp

@@ -5,8 +5,6 @@
 #include "objects/vm_area.h"
 #include "vm_space.h"
 
-extern frame_allocator &g_frame_allocator;
-
 int
 vm_space::area::compare(const vm_space::area &o) const
 {
@@ -91,6 +89,21 @@ vm_space::get(uintptr_t addr, uintptr_t *base)
 	return nullptr;
 }
 
+void
+vm_space::copy_from(const vm_space &source, uintptr_t from, uintptr_t to, size_t count)
+{
+	page_table::iterator sit {from, source.m_pml4};
+	page_table::iterator dit {to, m_pml4};
+
+	while (count--) {
+		uint64_t &e = dit.entry(page_table::level::pt);
+		if (e & page_table::flag::present) {
+			// TODO: handle clobbering mapping
+		}
+		e = sit.entry(page_table::level::pt);
+	}
+}
+
 void
 vm_space::page_in(uintptr_t virt, uintptr_t phys, size_t count)
 {
@@ -107,17 +120,37 @@ vm_space::page_in(uintptr_t virt, uintptr_t phys, size_t count)
 void
 vm_space::clear(uintptr_t addr, size_t count)
 {
+	using memory::frame_size;
+
+	uintptr_t free_start = 0;
+	size_t free_count = 0;
+
+	frame_allocator &fa = frame_allocator::get();
 	page_table::iterator it {addr, m_pml4};
+
 	while (count--) {
 		uint64_t &e = it.entry(page_table::level::pt);
-		if (e & page_table::flag::present) {
-			g_frame_allocator.free(e & ~0xfffull, 1);
-		}
 		bool allowed = (e & page_table::flag::allowed);
-		e = 0;
+		uintptr_t phys = e & ~0xfffull;
-		if (allowed) e |= page_table::flag::allowed;
+
+		if (e & page_table::flag::present) {
+			if (free_count && phys == free_start + (free_count * frame_size)) {
+				++free_count;
+			} else {
+				if (free_count)
+					fa.free(free_start, free_count);
+				free_start = phys;
+				free_count = 1;
+			}
+			fa.free(e & ~0xfffull, 1);
+		}
+
+		e = 0 | (allowed ? page_table::flag::allowed : page_table::flag::none);
 		++it;
 	}
+
+	if (free_count)
+		fa.free(free_start, free_count);
 }
 
 void
@@ -173,22 +206,32 @@ vm_space::handle_fault(uintptr_t addr, fault_type fault)
 	if (fault && fault_type::present)
 		return false;
 
-	if (!it.allowed())
+	uintptr_t base = 0;
+	vm_area *area = get(addr, &base);
+
+	if (!area && !it.allowed())
 		return false;
 
 	uintptr_t phys = 0;
-	size_t n = g_frame_allocator.allocate(1, &phys);
+	size_t n = frame_allocator::get().allocate(1, &phys);
 	kassert(n, "Failed to allocate a new page during page fault");
 
 	page_table::flag flags =
 		page_table::flag::present |
 		page_table::flag::write |
-		page_table::flag::allowed |
+		(area
+		 ? page_table::flag::none
+		 : page_table::flag::allowed) |
 		(is_kernel()
 		 ? page_table::flag::global
 		 : page_table::flag::user);
 
 	it.entry(page_table::level::pt) = phys | flags;
+	if (area) {
+		uintptr_t offset = page - base;
+		area->commit(phys, offset, 1);
+	}
+
 	return true;
 }
 

src/kernel/vm_space.h

@@ -48,6 +48,13 @@ public:
 	static vm_space & kernel_space();
 
 	/// Copy a range of mappings from the given address space
+	/// \arg source The address space that already contains the mappings
+	/// \arg from   The starting virtual address in the source address space
+	/// \arg to     The starting virtual address in this address space
+	/// \arg count  The number of page entries to copy
+	void copy_from(const vm_space &source, uintptr_t from, uintptr_t to, size_t count);
+
+	/// Map virtual addressses to the given physical pages
 	/// \arg virt  The starting virutal address
 	/// \arg phys  The starting physical address
 	/// \arg count The number of contiugous physical pages to map

src/libraries/kutil/include/kutil/map.h

@@ -73,7 +73,6 @@ public:
 	class iterator
 	{
 	public:
-		iterator(node *n) : m_node(n) {}
 		inline node & operator*() { return *m_node; }
 		inline node * operator->() { return m_node; }
 		inline const node & operator*() const { return *m_node; }
@@ -81,8 +80,12 @@ public:
 		inline iterator operator++(int) { node *old = m_node; incr(); return iterator(old); }
 		inline bool operator!=(const iterator &o) { return m_node != o.m_node; }
 	private:
-		void incr() { do { m_node++; } while ( m_node && m_node->hash() == 0 ); }
+		friend class base_map;
+		iterator(node *n) : m_node(n), m_end(n) {}
+		iterator(node *n, node *end) : m_node(n), m_end(end) {}
+		void incr() { while (m_node < m_end) { ++m_node; if (m_node->hash()) break; } }
 		node *m_node;
+		node *m_end;
 	};
 
 	/// Default constructor. Creates an empty map with the given capacity.
@@ -102,11 +105,13 @@ public:
 	}
 
 	iterator begin() {
-		return iterator(m_nodes);
+		iterator it {m_nodes - 1, m_nodes + m_capacity};
+		return ++it;
 	}
 
 	const iterator begin() const {
-		return iterator(m_nodes);
+		iterator it {m_nodes - 1, m_nodes + m_capacity};
+		return ++it;
 	}
 
 	const iterator end() const {
@@ -206,6 +211,9 @@ protected:
 	}
 
 	node * lookup(const K &k) {
+		if (!m_count)
+			return nullptr;
+
 		uint64_t h = hash(k);
 		size_t i = mod(h);
 		size_t dist = 0;
@@ -223,8 +231,10 @@ protected:
 		}
 	}
 
-	const node * lookup(const K &k) const
+	const node * lookup(const K &k) const {
-	{
+		if (!m_count)
+			return nullptr;
+
 		uint64_t h = hash(k);
 		size_t i = mod(h);
 		size_t dist = 0;