Recycle old initial pml4

Instead of building nested page tables for the offset region, just
offset map the entire thing into kernel memory with one PDP mapping
1GiB large pages. This is more efficient and avoids the "need a
page table to map in a page table" dependency loop.

NOTES.md

@@ -2,7 +2,9 @@
 
 ## TODO
 
-- Better page-allocation model
+- Paging manager
+  - Copy-on-write pages
+  - Better page-allocation model?
 - Allow for more than one IOAPIC in ACPI module
   - The objects get created, but GSI lookup only uses the one at index 0
 - mark kernel memory pages global
@@ -19,12 +21,18 @@
   - Parse initrd and pre-load certain ELF images, eg the process loader process?
   - Do initial memory bootstrap?
 - Calling global ctors
-
-
 - Device Tree
   - Actual serial driver
   - Disk driver
     - File system
 - Multiprocessing
-  - Syscalls
+  - Fast syscalls using syscall/sysret
+
+### Build
+
+- Clean up build generator and its templates
+  - More robust objects to represent modules & targets to pass to templates
+  - Read project setup from a simple JSON/TOML/etc
+  - Better compartmentalizing when doing template inheritance
+- Move to LLD as sysroot linker
 

generate_build.py

@@ -7,56 +7,102 @@ program = namedtuple('program', ['path', 'deps', 'output', 'targets'])
 source = namedtuple('source', ['name', 'input', 'output', 'action'])
 version = namedtuple('version', ['major', 'minor', 'patch', 'sha', 'dirty'])
 
-MODULES = {
-    "elf":       library('src/libraries/elf', ['kutil']),
-    "initrd":    library('src/libraries/initrd', ["kutil"]),
-    "kutil":     library('src/libraries/kutil', []),
+MODULES = {}
 
-    "makerd":    program('src/tools/makerd', ["initrd", "kutil"], "makerd", ["native"]),
+class Source:
+    Actions = {'.c': 'cc', '.cpp': 'cxx', '.s': 'nasm'}
 
-    "boot":      program('src/boot', [], "boot.elf", ["boot"]),
+    def __init__(self, path, root, modroot):
+        from os.path import relpath, splitext
+        self.input = path
+        self.name = relpath(path, root)
+        self.output = relpath(path, modroot) + ".o"
+        self.action = self.Actions.get(splitext(path)[1], None)
 
-    "nulldrv":   program('src/drivers/nulldrv', [], "nulldrv", ["host"]),
-    "kernel":    program('src/kernel', ["elf", "initrd", "kutil"], "popcorn.elf", ["host"]),
-}
+    def __str__(self):
+        return "{} {}:{}:{}".format(self.action, self.output, self.name, self.input)
+
+
+class Module:
+    def __init__(self, name, output, root, **kwargs):
+        from os.path import commonpath, dirname, isdir, join
+
+        self.name = name
+        self.output = output
+        self.kind = kwargs.get("kind", "exe")
+        self.target = kwargs.get("target", None)
+        self.deps = kwargs.get("deps", tuple())
+        self.includes = kwargs.get("includes", tuple())
+        self.defines = kwargs.get("defines", tuple())
+        self.depmods = []
+
+        sources = [join(root, f) for f in kwargs.get("source", tuple())]
+        modroot = commonpath(sources)
+        while not isdir(modroot):
+            modroot = dirname(modroot)
+
+        self.sources = [Source(f, root, modroot) for f in sources]
+
+    def __str__(self):
+        return "Module {} {}\n\t".format(self.kind, self.name)
+
+    def __find_depmods(self, modules):
+        self.depmods = set()
+        open_list = set(self.deps)
+        closed_list = set()
+
+        while open_list:
+            dep = modules[open_list.pop()]
+            open_list |= (set(dep.deps) - closed_list)
+            self.depmods.add(dep)
+
+        self.libdeps = [d for d in self.depmods if d.kind == "lib"]
+        self.exedeps = [d for d in self.depmods if d.kind != "lib"]
+
+    @classmethod
+    def load(cls, filename):
+        from os.path import abspath, dirname
+        from yaml import load
+
+        root = dirname(filename)
+        modules = {}
+        moddata = load(open(filename, "r"))
+
+        for name, data in moddata.items():
+            modules[name] = cls(name, root=root, **data)
+
+        for mod in modules.values():
+            mod.__find_depmods(modules)
+
+        targets = {}
+        for mod in modules.values():
+            if mod.target is None: continue
+            if mod.target not in targets:
+                targets[mod.target] = set()
+            targets[mod.target].add(mod)
+            targets[mod.target] |= mod.depmods
+
+        return modules.values(), targets
 
 
 def get_template(env, typename, name):
     from jinja2.exceptions import TemplateNotFound
     try:
-        return env.get_template("{}.{}.ninja.j2".format(typename, name))
+        return env.get_template("{}.{}.j2".format(typename, name))
     except TemplateNotFound:
-        return env.get_template("{}.default.ninja.j2".format(typename))
-
-
-def get_sources(path, srcroot):
-    import os
-    from os.path import abspath, join, relpath, splitext
-
-    actions = {'.c': 'cc', '.cpp': 'cxx', '.s': 'nasm'}
-
-    sources = []
-    for root, dirs, files in os.walk(path):
-        for f in files:
-            base, ext = splitext(f)
-            if not ext in actions: continue
-            name = join(root, f)
-            sources.append(
-                source(
-                    relpath(name, srcroot),
-                    abspath(name),
-                    relpath(abspath(name), path) + ".o",
-                    actions[ext]))
-
-    return sources
+        return env.get_template("{}.default.j2".format(typename))
 
 
 def get_git_version():
     from subprocess import run
-    cp = run(['git', 'describe', '--dirty', '--abbrev=7'],
+    cp = run(['git', 'describe', '--dirty'],
             check=True, capture_output=True)
     full_version = cp.stdout.decode('utf-8').strip()
 
+    cp = run(['git', 'rev-parse', 'HEAD'],
+            check=True, capture_output=True)
+    full_sha = cp.stdout.decode('utf-8').strip()
+
     dirty = False
     parts1 = full_version.split('-')
     if parts1[-1] == "dirty":
@@ -69,19 +115,21 @@ def get_git_version():
         parts2[0],
         parts2[1],
         parts2[2],
-        parts1[-1][1:],
+        full_sha[:7],
         dirty)
 
 
-def main(buildroot):
+def main(buildroot="build", modulefile="modules.yaml"):
     import os
-    from os.path import abspath, dirname, isdir, join
+    from os.path import abspath, dirname, isabs, isdir, join
 
     generator = abspath(__file__)
     srcroot = dirname(generator)
+    if not isabs(modulefile):
+        modulefile = join(srcroot, modulefile)
 
-    if buildroot is None:
-        buildroot = join(srcroot, "build")
+    if not isabs(buildroot):
+        buildroot = join(srcroot, buildroot)
 
     if not isdir(buildroot):
         os.mkdir(buildroot)
@@ -91,35 +139,21 @@ def main(buildroot):
         git_version.major, git_version.minor, git_version.patch, git_version.sha))
 
     from jinja2 import Environment, FileSystemLoader
-    env = Environment(loader=FileSystemLoader(join(srcroot, "scripts", "templates")))
+    template_dir = join(srcroot, "scripts", "templates")
+    env = Environment(loader=FileSystemLoader(template_dir))
 
-    targets = {}
-    templates = set()
     buildfiles = []
-    for name, mod in MODULES.items():
-        if isinstance(mod, program):
-            for target in mod.targets:
-                if not target in targets:
-                    targets[target] = set()
+    templates = set()
+    modules, targets = Module.load(modulefile)
 
-                open_list = [name]
-                while open_list:
-                    depname = open_list.pop()
-                    dep = MODULES[depname]
-                    open_list.extend(dep.deps)
-                    targets[target].add(depname)
-
-        sources = get_sources(join(srcroot, mod.path), join(srcroot, "src"))
-        buildfile = join(buildroot, name + ".ninja")
+    for mod in modules:
+        buildfile = join(buildroot, mod.name + ".ninja")
         buildfiles.append(buildfile)
         with open(buildfile, 'w') as out:
-            #print("Generating module", name)
-            template = get_template(env, type(mod).__name__, name)
+            template = get_template(env, mod.kind, mod.name)
             templates.add(template.filename)
             out.write(template.render(
-                name=name,
                 module=mod,
-                sources=sources,
                 buildfile=buildfile,
                 version=git_version))
 
@@ -131,7 +165,6 @@ def main(buildroot):
         buildfile = join(root, "target.ninja")
         buildfiles.append(buildfile)
         with open(buildfile, 'w') as out:
-            #print("Generating target", target)
             template = get_template(env, "target", target)
             templates.add(template.filename)
             out.write(template.render(
@@ -140,11 +173,14 @@ def main(buildroot):
                 buildfile=buildfile,
                 version=git_version))
 
-    buildfile = join(buildroot, "build.ninja")
+    # Top level buildfile cannot use an absolute path or ninja won't
+    # reload itself properly on changes.
+    # See: https://github.com/ninja-build/ninja/issues/1240
+    buildfile = "build.ninja"
     buildfiles.append(buildfile)
-    with open(buildfile, 'w') as out:
-        #print("Generating main build.ninja")
-        template = env.get_template('build.ninja.j2')
+
+    with open(join(buildroot, buildfile), 'w') as out:
+        template = env.get_template("build.ninja.j2")
         templates.add(template.filename)
 
         out.write(template.render(
@@ -155,11 +191,9 @@ def main(buildroot):
             buildfiles=buildfiles,
             templates=[abspath(f) for f in templates],
             generator=generator,
+            modulefile=modulefile,
             version=git_version))
 
 if __name__ == "__main__":
     import sys
-    buildroot = None
-    if len(sys.argv) > 1:
-        buildroot = sys.argv[1]
-    main(buildroot)
+    main(*sys.argv[1:])

modules.yaml

@@ -0,0 +1,116 @@
+kernel:
+    output: popcorn.elf
+    target: host
+    deps:
+        - elf
+        - initrd
+        - kutil
+    includes:
+        - src/kernel
+    source:
+        - src/kernel/crti.s
+        - src/kernel/apic.cpp
+        - src/kernel/assert.cpp
+        - src/kernel/boot.s
+        - src/kernel/console.cpp
+        - src/kernel/cpprt.cpp
+        - src/kernel/cpu.cpp
+        - src/kernel/debug.cpp
+        - src/kernel/debug.s
+        - src/kernel/device_manager.cpp
+        - src/kernel/font.cpp
+        - src/kernel/fs/gpt.cpp
+        - src/kernel/gdt.cpp
+        - src/kernel/gdt.s
+        - src/kernel/interrupts.cpp
+        - src/kernel/interrupts.s
+        - src/kernel/io.cpp
+        - src/kernel/loader.s
+        - src/kernel/log.cpp
+        - src/kernel/main.cpp
+        - src/kernel/memory_bootstrap.cpp
+        - src/kernel/msr.cpp
+        - src/kernel/page_manager.cpp
+        - src/kernel/pci.cpp
+        - src/kernel/process.cpp
+        - src/kernel/scheduler.cpp
+        - src/kernel/screen.cpp
+        - src/kernel/serial.cpp
+        - src/kernel/syscall.cpp
+        - src/kernel/syscall.s
+        - src/kernel/crtn.s
+
+boot:
+    kind: exe
+    target: boot
+    output: boot.elf
+    source:
+        - src/boot/crt0.s
+        - src/boot/console.cpp
+        - src/boot/guids.cpp
+        - src/boot/loader.cpp
+        - src/boot/main.cpp
+        - src/boot/memory.cpp
+        - src/boot/reloc.cpp
+        - src/boot/utility.cpp
+
+nulldrv:
+    kind: exe
+    target: host
+    output: nulldrv
+    source:
+        - src/drivers/nulldrv/main.s
+
+elf:
+    kind: lib
+    output: libelf.a
+    deps:
+        - kutil
+    includes:
+        - src/libraries/elf/include
+    source:
+        - src/libraries/elf/elf.cpp
+
+initrd:
+    kind: lib
+    output: libinitrd.a
+    deps:
+        - kutil
+    includes:
+        - src/libraries/initrd/include
+    source:
+        - src/libraries/initrd/initrd.cpp
+
+kutil:
+    kind: lib
+    output: libkutil.a
+    includes:
+        - src/libraries/kutil/include
+    source:
+        - src/libraries/kutil/assert.cpp
+        - src/libraries/kutil/frame_allocator.cpp
+        - src/libraries/kutil/heap_manager.cpp
+        - src/libraries/kutil/memory.cpp
+
+makerd:
+    kind: exe
+    target: native
+    output: makerd
+    deps:
+        - initrd
+    source:
+        - src/tools/makerd/entry.cpp
+        - src/tools/makerd/main.cpp
+
+tests:
+    kind: exe
+    target: native
+    output: tests
+    deps:
+        - kutil
+    source:
+        - src/tests/address_manager.cpp
+        - src/tests/frame_allocator.cpp
+        - src/tests/linked_list.cpp
+        - src/tests/heap_manager.cpp
+        - src/tests/main.cpp

qemu.sh

@@ -1,13 +1,29 @@
 #!/usr/bin/env bash
 
-build=${1:-"$(dirname $0)/build"}
-ninja -C $build
+build="$(dirname $0)/build"
+debug=""
+gfx="-nographic"
+
+for arg in $@; do
+	case "${arg}" in
+		--debug)
+			debug="-s"
+			;;
+		--gfx)
+			gfx=""
+			;;
+		*)
+			build="${arg}"
+			;;
+	esac
+done
 
 kvm=""
-if [[ -f /dev/kvm ]]; then
-	kvm="--enable-kvm"
+if [[ -c /dev/kvm ]]; then
+	kvm="-enable-kvm"
 fi
 
+ninja -C "${build}" && \
 exec qemu-system-x86_64 \
 	-drive "if=pflash,format=raw,file=${build}/flash.img" \
 	-drive "format=raw,file=${build}/popcorn.img" \
@@ -18,5 +34,4 @@ exec qemu-system-x86_64 \
 	-cpu Broadwell \
 	-M q35 \
 	-no-reboot \
-	-nographic \
-	$kvm
+	$gfx $kvm $debug

scripts/build_sysroot_clang.sh

@@ -216,5 +216,5 @@ update_links
 
 export CC="${SYSROOT}/bin/clang"
 export CXX="${SYSROOT}/bin/clang++"
-export LD="${SYSROOT}/bin/ld.lld"
+export LD="${SYSROOT}/bin/ld"
 build_libc

scripts/templates/build.ninja.j2

@@ -1,6 +1,7 @@
 ninja_required_version = 1.3
 builddir = {{ buildroot }}
 srcroot = {{ srcroot }}
+modulefile = {{ modulefile }}
 
 warnflags = $
     -Wformat=2 $
@@ -52,12 +53,34 @@ rule cc
     description = Compiling $name
     command = $cc -MMD -MF $out.d $ccflags $cflags -o $out -c $in
 
+rule dump_cc_defs
+    description = Dumping CC defines for $target
+    command = echo "" | $cc $ccflags $cflags -dM -E - > $out
+
+rule dump_cc_run
+    description = Dumping CC arguments for $target
+    command = $
+        echo "#!/bin/bash" > $out; $
+        echo '$cc $ccflags $cflags $$*' > $out; $
+        chmod a+x $out
+
 rule cxx
     deps = gcc
     depfile = $out.d
     description = Compiling $name
     command = $cxx -MMD -MF $out.d $cxxflags $ccflags -o $out -c $in
 
+rule dump_cxx_defs
+    description = Dumping C++ defines for $target
+    command = echo "" | $cxx -x c++ $cxxflags $ccflags -dM -E - > $out
+
+rule dump_cxx_run
+    description = Dumping C++ arguments for $target
+    command = $
+        echo "#!/bin/bash" > $out; $
+        echo '$cc $cxxflags $ccflags $$*' > $out; $
+        chmod a+x $out
+
 rule nasm
     deps = gcc
     depfile = $out.d
@@ -75,7 +98,7 @@ rule lib
 rule regen
     generator = true
     description = Regenrating build files
-    command = {{ generator }} $builddir
+    command = {{ generator }} $builddir $modulefile
 
 rule cp
     description = Copying $name
@@ -103,7 +126,16 @@ rule makefat
     description = Creating $name
     command = $
         cp $srcroot/assets/diskbase.img $out; $
-        mcopy -s -D o -i $out@@1024K $builddir/fatroot/* ::/
+        mcopy -s -D o -i $out@@1M $builddir/fatroot/* ::/
+
+rule strip
+    description = Stripping $name
+    command = $
+        cp $in $out; $
+        objcopy --only-keep-debug $out $out.debug; $
+        strip -g $out; $
+        objcopy --add-gnu-debuglink=$out.debug $out
+
 
 {% for target in targets %}
 subninja {{ target }}/target.ninja
@@ -117,12 +149,16 @@ build $
     {%- for template in templates %}
     {{ template }} $
     {%- endfor %}
+    $modulefile $
     {{ generator }}
 
 build $builddir/flash.img : cp $srcroot/assets/ovmf/x64/OVMF.fd
     name = flash.img
 
-build $builddir/fatroot/popcorn.elf : cp $builddir/host/popcorn.elf
+build $builddir/popcorn.elf | $builddir/popcorn.elf.debug : strip $builddir/host/popcorn.elf
+    name = kernel
+
+build $builddir/fatroot/popcorn.elf : cp $builddir/popcorn.elf
     name = kernel to FAT image
 
 build $builddir/fatroot/efi/boot/bootx64.efi : cp $builddir/boot/boot.efi

scripts/templates/exe.boot.j2

@@ -1,4 +1,4 @@
-{% extends "program.default.ninja.j2" %}
+{% extends "exe.default.j2" %}
 {% block variables %}
 {{ super() }}
 
@@ -12,8 +12,8 @@ ccflags = $ccflags $
     -DHAVE_USE_MS_ABI $
     -DEFI_DEBUG=0 $
     -DEFI_DEBUG_CLEAR_MEMORY=0 $
-    -fPIC $
-    -fshort-wchar
+	-DBOOTLOADER_DEBUG $
+    -fPIC
 
 ldflags = $ldflags $
     -T ${srcroot}/src/arch/x86_64/boot.ld $

scripts/templates/exe.default.j2

@@ -0,0 +1,12 @@
+{% extends "module.base.j2" %}
+{% block variables %}
+{{ super() }}
+
+libs = $
+    -L${builddir} $
+    {%- for dep in module.libdeps %}
+    -l{{ dep.name }} $
+    {%- endfor %}
+    $libs
+
+{% endblock %}

scripts/templates/exe.kernel.j2

@@ -1,4 +1,4 @@
-{% extends "program.default.ninja.j2" %}
+{% extends "exe.default.j2" %}
 {% block variables %}
 {{ super() }}
 

scripts/templates/exe.tests.j2

@@ -0,0 +1,7 @@
+{% extends "exe.default.j2" %}
+{% block variables %}
+{{ super() }}
+
+ccflags = $ccflags -ggdb
+
+{% endblock %}

scripts/templates/lib.default.j2

@@ -0,0 +1 @@
+{% extends "module.base.j2" %}

scripts/templates/library.default.ninja.j2

@@ -1 +0,0 @@
-{% extends "module.base.ninja.j2" %}

scripts/templates/module.base.j2

@@ -0,0 +1,41 @@
+moddir = ${builddir}/{{ module.name }}.dir
+
+{% block variables %}
+ccflags = $ccflags $
+    {%- for dep in module.depmods %}
+    {%- for inc in dep.includes %}
+    -I${srcroot}/{{ inc }} $
+    {%- endfor %}
+    {%- endfor %}
+    {%- for inc in module.includes %}
+    -I${srcroot}/{{ inc }} $
+    {%- endfor %}
+    {%- for define in module.defines %}
+    -D{{ define }} $
+    {%- endfor %}
+{% endblock %}
+
+{% for source in module.sources %}
+build ${moddir}/{{ source.output }} : {{ source.action }} {{ source.input }} || {{ buildfile }}
+    name = {{ source.name }}
+{% endfor %}
+
+build ${builddir}/{{ module.output }} : {{ module.kind }} $
+{%- for source in module.sources %}
+    ${moddir}/{{ source.output }} $
+{%- endfor -%}
+{%- for dep in module.libdeps %}
+    ${builddir}/{{ dep.output }} $
+{%- endfor %}
+    | $
+{% for dep in module.exedeps %}
+    ${builddir}/{{ dep.output }} $
+{%- endfor -%}
+    {{ buildfile }}
+    name = {{ module.name }}
+
+{% block extra %}
+{% endblock %}
+
+# vim: ft=ninja et ts=4 sts=4 sw=4
+

scripts/templates/module.base.ninja.j2

@@ -1,29 +0,0 @@
-moddir = ${builddir}/{{ name }}.dir
-
-{% block variables %}
-ccflags = $ccflags $
-    {%- for dep in module.deps %}
-    -I${srcroot}/src/libraries/{{ dep }}/include $
-    {%- endfor %}
-    -I${srcroot}/{{ module.path }} $
-    -I${srcroot}/{{ module.path }}/include
-{% endblock %}
-
-{% for source in sources %}
-build ${moddir}/{{ source.output }} : {{ source.action }} {{ source.input }} || {{ buildfile }}
-    name = {{ source.name }}
-{% endfor %}
-
-build {% block artifact %} ${builddir}/lib{{ name }}.a : lib {% endblock %} $
-{%- block extrasources %}{% endblock -%}
-{%- for source in sources %}
-    ${moddir}/{{ source.output }}{% if not loop.last %} ${% endif %}
-{%- endfor -%}
-{%- if module.deps %}| {% for dep in module.deps %} ${builddir}/lib{{ dep }}.a {% endfor %}{% endif %}
-    name = {{ name }}
-
-{% block extra %}
-{% endblock %}
-
-# vim: ft=ninja et ts=4 sts=4 sw=4
-

scripts/templates/program.default.ninja.j2

@@ -1,13 +0,0 @@
-{% extends "module.base.ninja.j2" %}
-{% block variables %}
-{{ super() }}
-
-libs = $
-    -L${builddir} $
-    {%- for dep in module.deps %}
-    -l{{dep}} $
-    {%- endfor %}
-    $libs
-
-{% endblock %}
-{% block artifact %}${builddir}/{{ module.output }} : exe{% endblock %}

scripts/templates/target.boot.j2

@@ -1,4 +1,4 @@
-{% extends "target.default.ninja.j2" %}
+{% extends "target.default.j2" %}
 
 {% block binaries %}
 ld = ld
@@ -17,11 +17,11 @@ ccflags = $ccflags $
     -fno-builtin $
     -mno-sse $
     -fno-omit-frame-pointer $
-    -mno-red-zone
+    -mno-red-zone $
+    -fshort-wchar
 
 cxxflags = $cxxflags $
     -nostdlibinc $
-    -isystem${srcroot}/sysroot/include/c++/v1 $
     -fno-exceptions $
     -fno-rtti
 

scripts/templates/target.default.j2

@@ -0,0 +1,24 @@
+builddir = $builddir/{{ target }}
+target = {{ target }}
+
+{% block variables %}
+{% endblock %}
+
+{% block binaries %}
+cc = clang
+cxx = clang++
+ld = clang++
+ar = ar
+nasm = nasm
+objcopy = objcopy
+{% endblock %}
+
+{% for module in modules %}
+subninja {{ module.name }}.ninja
+{% endfor %}
+
+build ${builddir}/cc.defs : dump_cc_defs | {{ buildfile }}
+build ${builddir}/cxx.defs : dump_cxx_defs | {{ buildfile }}
+build ${builddir}/cc.run : dump_cc_run | {{ buildfile }}
+build ${builddir}/cxx.run : dump_cxx_run | {{ buildfile }}
+

scripts/templates/target.default.ninja.j2

@@ -1,17 +0,0 @@
-builddir = $builddir/{{ target }}
-
-{% block variables %}
-{% endblock %}
-
-{% block binaries %}
-cc = clang
-cxx = clang++
-ld = clang++
-ar = ar
-nasm = nasm
-objcopy = objcopy
-{% endblock %}
-
-{% for module in modules %}
-subninja {{ module }}.ninja
-{% endfor %}

scripts/templates/target.host.j2

@@ -1,4 +1,5 @@
-{% extends "target.default.ninja.j2" %}
+{% extends "target.default.j2" %}
+
 {% block binaries %}
 cc = ${srcroot}/sysroot/bin/clang
 cxx = ${srcroot}/sysroot/bin/clang++

scripts/templates/target.native.j2

@@ -0,0 +1,8 @@
+{% extends "target.default.j2" %}
+
+{% block variables %}
+
+ccflags = $ccflags -g -ggdb
+
+{% endblock %}
+# vim: et ts=4 sts=4 sw=4

scripts/templates/target.native.ninja.j2

@@ -1 +0,0 @@
-{% extends "target.default.ninja.j2" %}

src/boot/console.cpp

@@ -1,4 +1,3 @@
-#include <stdarg.h>
 #include <stddef.h>
 #include <stdint.h>
 
@@ -9,16 +8,55 @@
 size_t ROWS = 0;
 size_t COLS = 0;
 
-static EFI_SIMPLE_TEXT_OUT_PROTOCOL *con_out = 0;
+static EFI_SIMPLE_TEXT_OUT_PROTOCOL *m_out = 0;
 
-const CHAR16 digits[] = {u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9', u'a', u'b', u'c', u'd', u'e', u'f'};
+static const wchar_t digits[] = {u'0', u'1', u'2', u'3', u'4', u'5',
+	u'6', u'7', u'8', u'9', u'a', u'b', u'c', u'd', u'e', u'f'};
+
+console::console(EFI_SYSTEM_TABLE *system_table) :
+	m_rows(0),
+	m_cols(0),
+	m_out(nullptr)
+{
+	s_console = this;
+	m_boot = system_table->BootServices;
+	m_out = system_table->ConOut;
+}
 
 EFI_STATUS
-con_pick_mode(EFI_BOOT_SERVICES *bootsvc)
+console::initialize(const wchar_t *version)
+{
+	EFI_STATUS status;
+
+	// Might not find a video device at all, so ignore not found errors
+	status = pick_mode();
+	if (status != EFI_NOT_FOUND)
+		CHECK_EFI_STATUS_OR_FAIL(status);
+
+	status = m_out->QueryMode(m_out, m_out->Mode->Mode, &m_cols, &m_rows);
+	CHECK_EFI_STATUS_OR_RETURN(status, "QueryMode");
+
+	status = m_out->ClearScreen(m_out);
+	CHECK_EFI_STATUS_OR_RETURN(status, "ClearScreen");
+
+	m_out->SetAttribute(m_out, EFI_LIGHTCYAN);
+	m_out->OutputString(m_out, (wchar_t *)L"Popcorn loader ");
+
+	m_out->SetAttribute(m_out, EFI_LIGHTMAGENTA);
+	m_out->OutputString(m_out, (wchar_t *)version);
+
+	m_out->SetAttribute(m_out, EFI_LIGHTGRAY);
+	m_out->OutputString(m_out, (wchar_t *)L" booting...\r\n\n");
+
+	return status;
+}
+
+EFI_STATUS
+console::pick_mode()
 {
 	EFI_STATUS status;
 	EFI_GRAPHICS_OUTPUT_PROTOCOL *gfx_out_proto;
-	status = bootsvc->LocateProtocol(&guid_gfx_out, NULL, (void **)&gfx_out_proto);
+	status = m_boot->LocateProtocol(&guid_gfx_out, NULL, (void **)&gfx_out_proto);
 	CHECK_EFI_STATUS_OR_RETURN(status, "LocateProtocol gfx");
 
 	const uint32_t modes = gfx_out_proto->Mode->MaxMode;
@@ -53,106 +91,72 @@ con_pick_mode(EFI_BOOT_SERVICES *bootsvc)
 	return EFI_SUCCESS;
 }
 
-EFI_STATUS
-con_initialize(EFI_SYSTEM_TABLE *system_table, const CHAR16 *version)
-{
-	EFI_STATUS status;
-
-	EFI_BOOT_SERVICES *bootsvc = system_table->BootServices;
-	con_out = system_table->ConOut;
-
-	// Might not find a video device at all, so ignore not found errors
-	status = con_pick_mode(bootsvc);
-	if (status != EFI_NOT_FOUND)
-		CHECK_EFI_STATUS_OR_RETURN(status, "con_pick_mode");
-
-	status = con_out->QueryMode(con_out, con_out->Mode->Mode, &COLS, &ROWS);
-	CHECK_EFI_STATUS_OR_RETURN(status, "QueryMode");
-
-	status = con_out->ClearScreen(con_out);
-	CHECK_EFI_STATUS_OR_RETURN(status, "ClearScreen");
-
-	con_out->SetAttribute(con_out, EFI_LIGHTCYAN);
-	con_out->OutputString(con_out, (CHAR16 *)L"Popcorn loader ");
-
-	con_out->SetAttribute(con_out, EFI_LIGHTMAGENTA);
-	con_out->OutputString(con_out, (CHAR16 *)version);
-
-	con_out->SetAttribute(con_out, EFI_LIGHTGRAY);
-	con_out->OutputString(con_out, (CHAR16 *)L" booting...\r\n\n");
-
-	return status;
-}
-
 size_t
-con_print_hex(uint32_t n)
+console::print_hex(uint32_t n) const
 {
-	CHAR16 buffer[9];
-	CHAR16 *p = buffer;
+	wchar_t buffer[9];
+	wchar_t *p = buffer;
 	for (int i = 7; i >= 0; --i) {
 		uint8_t nibble = (n & (0xf << (i*4))) >> (i*4);
 		*p++ = digits[nibble];
 	}
 	*p = 0;
-	con_out->OutputString(con_out, buffer);
+	m_out->OutputString(m_out, buffer);
 	return 8;
 }
 
 size_t
-con_print_long_hex(uint64_t n)
+console::print_long_hex(uint64_t n) const
 {
-	CHAR16 buffer[17];
-	CHAR16 *p = buffer;
+	wchar_t buffer[17];
+	wchar_t *p = buffer;
 	for (int i = 15; i >= 0; --i) {
 		uint8_t nibble = (n & (0xf << (i*4))) >> (i*4);
 		*p++ = digits[nibble];
 	}
 	*p = 0;
-	con_out->OutputString(con_out, buffer);
+	m_out->OutputString(m_out, buffer);
 	return 16;
 }
 
 size_t
-con_print_dec(uint32_t n)
+console::print_dec(uint32_t n) const
 {
-	CHAR16 buffer[11];
-	CHAR16 *p = buffer + 10;
+	wchar_t buffer[11];
+	wchar_t *p = buffer + 10;
 	*p-- = 0;
 	do {
 		*p-- = digits[n % 10];
 		n /= 10;
 	} while (n != 0);
 
-	con_out->OutputString(con_out, ++p);
+	m_out->OutputString(m_out, ++p);
 	return 10 - (p - buffer);
 }
 
 size_t
-con_print_long_dec(uint64_t n)
+console::print_long_dec(uint64_t n) const
 {
-	CHAR16 buffer[21];
-	CHAR16 *p = buffer + 20;
+	wchar_t buffer[21];
+	wchar_t *p = buffer + 20;
 	*p-- = 0;
 	do {
 		*p-- = digits[n % 10];
 		n /= 10;
 	} while (n != 0);
 
-	con_out->OutputString(con_out, ++p);
+	m_out->OutputString(m_out, ++p);
 	return 20 - (p - buffer);
 }
 
 size_t
-con_printf(const CHAR16 *fmt, ...)
+console::vprintf(const wchar_t *fmt, va_list args) const
 {
-	CHAR16 buffer[256];
-	const CHAR16 *r = fmt;
-	CHAR16 *w = buffer;
-	va_list args;
+	wchar_t buffer[256];
+	const wchar_t *r = fmt;
+	wchar_t *w = buffer;
 	size_t count = 0;
 
-	va_start(args, fmt);
-
 	while (r && *r) {
 		if (*r != L'%') {
 			count++;
@@ -161,43 +165,43 @@ con_printf(const CHAR16 *fmt, ...)
 		}
 
 		*w = 0;
-		con_out->OutputString(con_out, buffer);
+		m_out->OutputString(m_out, buffer);
 		w = buffer;
 
 		r++; // chomp the %
 
 		switch (*r++) {
 			case L'%':
-				con_out->OutputString(con_out, L"%");
+				m_out->OutputString(m_out, const_cast<wchar_t*>(L"%"));
 				count++;
 				break;
 
 			case L'x':
-				count += con_print_hex(va_arg(args, uint32_t));
+				count += print_hex(va_arg(args, uint32_t));
 				break;
 
 			case L'd':
 			case L'u':
-				count += con_print_dec(va_arg(args, uint32_t));
+				count += print_dec(va_arg(args, uint32_t));
 				break;
 
 			case L's':
 				{
-					CHAR16 *s = va_arg(args, CHAR16*);
+					wchar_t *s = va_arg(args, wchar_t*);
 					count += wstrlen(s);
-					con_out->OutputString(con_out, s);
+					m_out->OutputString(m_out, s);
 				}
 				break;
 
 			case L'l':
 				switch (*r++) {
 					case L'x':
-						count += con_print_long_hex(va_arg(args, uint64_t));
+						count += print_long_hex(va_arg(args, uint64_t));
 						break;
 
 					case L'd':
 					case L'u':
-						count += con_print_long_dec(va_arg(args, uint64_t));
+						count += print_long_dec(va_arg(args, uint64_t));
 						break;
 
 					default:
@@ -211,44 +215,66 @@ con_printf(const CHAR16 *fmt, ...)
 	}
 
 	*w = 0;
-	con_out->OutputString(con_out, buffer);
-
-	va_end(args);
+	m_out->OutputString(m_out, buffer);
 	return count;
 }
 
-void
-con_status_begin(const CHAR16 *message)
+size_t
+console::printf(const wchar_t *fmt, ...) const
 {
-	con_out->SetAttribute(con_out, EFI_LIGHTGRAY);
-	con_out->OutputString(con_out, (CHAR16 *)message);
+	va_list args;
+	va_start(args, fmt);
+
+	size_t result = vprintf(fmt, args);
+
+	va_end(args);
+	return result;
+}
+
+size_t
+console::print(const wchar_t *fmt, ...)
+{
+	va_list args;
+	va_start(args, fmt);
+
+	size_t result = get().vprintf(fmt, args);
+
+	va_end(args);
+	return result;
 }
 
 void
-con_status_ok()
+console::status_begin(const wchar_t *message) const
 {
-	con_out->SetAttribute(con_out, EFI_LIGHTGRAY);
-	con_out->OutputString(con_out, (CHAR16 *)L"[");
-	con_out->SetAttribute(con_out, EFI_GREEN);
-	con_out->OutputString(con_out, (CHAR16 *)L"  ok  ");
-	con_out->SetAttribute(con_out, EFI_LIGHTGRAY);
-	con_out->OutputString(con_out, (CHAR16 *)L"]\r\n");
+	m_out->SetAttribute(m_out, EFI_LIGHTGRAY);
+	m_out->OutputString(m_out, (wchar_t *)message);
 }
 
 void
-con_status_fail(const CHAR16 *error)
+console::status_ok() const
 {
-	con_out->SetAttribute(con_out, EFI_LIGHTGRAY);
-	con_out->OutputString(con_out, (CHAR16 *)L"[");
-	con_out->SetAttribute(con_out, EFI_LIGHTRED);
-	con_out->OutputString(con_out, (CHAR16 *)L"failed");
-	con_out->SetAttribute(con_out, EFI_LIGHTGRAY);
-	con_out->OutputString(con_out, (CHAR16 *)L"]\r\n");
+	m_out->SetAttribute(m_out, EFI_LIGHTGRAY);
+	m_out->OutputString(m_out, (wchar_t *)L"[");
+	m_out->SetAttribute(m_out, EFI_GREEN);
+	m_out->OutputString(m_out, (wchar_t *)L"  ok  ");
+	m_out->SetAttribute(m_out, EFI_LIGHTGRAY);
+	m_out->OutputString(m_out, (wchar_t *)L"]\r\n");
+}
 
-	con_out->SetAttribute(con_out, EFI_RED);
-	con_out->OutputString(con_out, (CHAR16 *)error);
-	con_out->SetAttribute(con_out, EFI_LIGHTGRAY);
-	con_out->OutputString(con_out, (CHAR16 *)L"\r\n");
+void
+console::status_fail(const wchar_t *error) const
+{
+	m_out->SetAttribute(m_out, EFI_LIGHTGRAY);
+	m_out->OutputString(m_out, (wchar_t *)L"[");
+	m_out->SetAttribute(m_out, EFI_LIGHTRED);
+	m_out->OutputString(m_out, (wchar_t *)L"failed");
+	m_out->SetAttribute(m_out, EFI_LIGHTGRAY);
+	m_out->OutputString(m_out, (wchar_t *)L"]\r\n");
+
+	m_out->SetAttribute(m_out, EFI_RED);
+	m_out->OutputString(m_out, (wchar_t *)error);
+	m_out->SetAttribute(m_out, EFI_LIGHTGRAY);
+	m_out->OutputString(m_out, (wchar_t *)L"\r\n");
 }
 
 EFI_STATUS

src/boot/console.h

@@ -1,12 +1,38 @@
 #pragma once
+#include <stdarg.h>
 #include <stddef.h>
 #include <efi/efi.h>
 
-EFI_STATUS con_initialize(EFI_SYSTEM_TABLE *system_table, const CHAR16 *version);
-void con_status_begin(const CHAR16 *message);
-void con_status_ok();
-void con_status_fail(const CHAR16 *error);
-size_t con_printf(const CHAR16 *fmt, ...);
+class console
+{
+public:
+	console(EFI_SYSTEM_TABLE *system_table);
+
+	EFI_STATUS initialize(const wchar_t *version);
+
+	void status_begin(const wchar_t *message) const;
+	void status_fail(const wchar_t *error) const;
+	void status_ok() const;
+
+	size_t print_hex(uint32_t n) const;
+	size_t print_dec(uint32_t n) const;
+	size_t print_long_hex(uint64_t n) const;
+	size_t print_long_dec(uint64_t n) const;
+	size_t printf(const wchar_t *fmt, ...) const;
+
+	static const console & get() { return *s_console; }
+	static size_t print(const wchar_t *fmt, ...);
+
+private:
+	EFI_STATUS pick_mode();
+	size_t vprintf(const wchar_t *fmt, va_list args) const;
+
+	size_t m_rows, m_cols;
+	EFI_BOOT_SERVICES *m_boot;
+	EFI_SIMPLE_TEXT_OUT_PROTOCOL *m_out;
+
+	static console *s_console;
+};
 
 EFI_STATUS
 con_get_framebuffer(

src/boot/loader.cpp

@@ -6,21 +6,28 @@
 
 #define PAGE_SIZE 0x1000
 
-static CHAR16 kernel_name[] = KERNEL_FILENAME;
-static CHAR16 initrd_name[] = INITRD_FILENAME;
+static wchar_t kernel_name[] = KERNEL_FILENAME;
+static wchar_t initrd_name[] = INITRD_FILENAME;
 
 EFI_STATUS
 loader_alloc_pages(
 	EFI_BOOT_SERVICES *bootsvc,
 	EFI_MEMORY_TYPE mem_type,
 	size_t *length,
-	void **pages)
+	void **pages,
+	bool align)
 {
 	EFI_STATUS status;
 
 	size_t page_count = ((*length - 1) / PAGE_SIZE) + 1;
 	EFI_PHYSICAL_ADDRESS addr = (EFI_PHYSICAL_ADDRESS)*pages;
 
+	if (align) {
+		// Align addr to the next multiple of N pages
+		size_t align_size = page_count * PAGE_SIZE;
+		addr = ((addr - 1) & ~(align_size - 1)) + align_size;
+	}
+
 	status = bootsvc->AllocatePages(AllocateAddress, mem_type, page_count, &addr);
 	if (status == EFI_NOT_FOUND || status == EFI_OUT_OF_RESOURCES) {
 		// couldn't get the address we wanted, try loading the kernel anywhere
@@ -46,7 +53,7 @@ loader_load_initrd(
 	EFI_STATUS status;
 
 	EFI_FILE_PROTOCOL *file = NULL;
-	status = root->Open(root, &file, (CHAR16 *)initrd_name, EFI_FILE_MODE_READ,
+	status = root->Open(root, &file, (wchar_t *)initrd_name, EFI_FILE_MODE_READ,
 						EFI_FILE_READ_ONLY | EFI_FILE_HIDDEN | EFI_FILE_SYSTEM);
 
 	if (status == EFI_NOT_FOUND)
@@ -64,9 +71,10 @@ loader_load_initrd(
 
 	status = loader_alloc_pages(
 			bootsvc,
-			INITRD_MEMTYPE,
+			memtype_initrd,
 			&data->initrd_length,
-			&data->initrd);
+			&data->initrd,
+			true);
 	CHECK_EFI_STATUS_OR_RETURN(status, L"Allocating pages");
 
 	status = file->Read(file, &data->initrd_length, data->initrd);
@@ -87,10 +95,10 @@ loader_load_elf(
 {
 	EFI_STATUS status;
 
-	con_debug(L"Opening kernel file %s\r\n", (CHAR16 *)kernel_name);
+	con_debug(L"Opening kernel file %s\r\n", (wchar_t *)kernel_name);
 
 	EFI_FILE_PROTOCOL *file = NULL;
-	status = root->Open(root, &file, (CHAR16 *)kernel_name, EFI_FILE_MODE_READ,
+	status = root->Open(root, &file, (wchar_t *)kernel_name, EFI_FILE_MODE_READ,
 						EFI_FILE_READ_ONLY | EFI_FILE_HIDDEN | EFI_FILE_SYSTEM);
 
 	if (status == EFI_NOT_FOUND)
@@ -141,7 +149,6 @@ loader_load_elf(
 
 	struct elf_program_header prog_header;
 	for (int i = 0; i < header.ph_num; ++i) {
-		con_debug(L"Reading ELF program header %d\r\n", i);
 
 		status = file->SetPosition(file, header.ph_offset + i * header.ph_entsize);
 		CHECK_EFI_STATUS_OR_RETURN(status, L"Setting ELF file position");
@@ -154,18 +161,17 @@ loader_load_elf(
 
 		length = prog_header.mem_size;
 		void *addr = (void *)(prog_header.vaddr - KERNEL_VIRT_ADDRESS);
-		status = loader_alloc_pages(bootsvc, KERNEL_MEMTYPE, &length, &addr);
+		status = loader_alloc_pages(bootsvc, memtype_kernel, &length, &addr, false);
 		CHECK_EFI_STATUS_OR_RETURN(status, L"Allocating kernel pages");
 
 		if (data->kernel == 0)
 			data->kernel = addr;
 		data->kernel_length = (uint64_t)addr + length - (uint64_t)data->kernel;
 	}
+	con_debug(L"Read %d ELF program headers\r\n", header.ph_num);
 
 	struct elf_section_header sec_header;
 	for (int i = 0; i < header.sh_num; ++i) {
-		con_debug(L"Reading ELF section header %d ", i);
-
 		status = file->SetPosition(file, header.sh_offset + i * header.sh_entsize);
 		CHECK_EFI_STATUS_OR_RETURN(status, L"Setting ELF file position");
 
@@ -174,14 +180,12 @@ loader_load_elf(
 		CHECK_EFI_STATUS_OR_RETURN(status, L"Reading ELF section header");
 
 		if ((sec_header.flags & ELF_SHF_ALLOC) == 0) {
-			con_debug(L"SHF_ALLOC section\r\n");
 			continue;
 		}
 
 		void *addr = (void *)(sec_header.addr - KERNEL_VIRT_ADDRESS);
 
 		if (sec_header.type == ELF_ST_PROGBITS) {
-			con_debug(L"PROGBITS section\r\n");
 			status = file->SetPosition(file, sec_header.offset);
 			CHECK_EFI_STATUS_OR_RETURN(status, L"Setting ELF file position");
 
@@ -189,12 +193,10 @@ loader_load_elf(
 			status = file->Read(file, &length, addr);
 			CHECK_EFI_STATUS_OR_RETURN(status, L"Reading file");
 		} else if (sec_header.type == ELF_ST_NOBITS) {
-			con_debug(L"NOBITS section\r\n");
 			bootsvc->SetMem(addr, sec_header.size, 0);
-		} else {
-			con_debug(L"other section\r\n");
 		}
 	}
+	con_debug(L"Read %d ELF section headers\r\n", header.ph_num);
 
 	status = file->Close(file);
 	CHECK_EFI_STATUS_OR_RETURN(status, L"Closing file handle");
@@ -243,9 +245,10 @@ loader_load_kernel(
 		data->data_length += PAGE_SIZE; // extra page for map growth
 		status = loader_alloc_pages(
 				bootsvc,
-				KERNEL_DATA_MEMTYPE,
+				memtype_data,
 				&data->data_length,
-				&data->data);
+				&data->data,
+				true);
 		CHECK_EFI_STATUS_OR_RETURN(status, L"loader_alloc_pages: kernel data");
 
 		return EFI_SUCCESS;

src/boot/loader.h

@@ -12,18 +12,6 @@
 #define KERNEL_VIRT_ADDRESS 0xFFFFFF0000000000
 #endif
 
-#ifndef KERNEL_MEMTYPE
-#define KERNEL_MEMTYPE 0x80000000
-#endif
-
-#ifndef INITRD_MEMTYPE
-#define INITRD_MEMTYPE 0x80000001
-#endif
-
-#ifndef KERNEL_DATA_MEMTYPE
-#define KERNEL_DATA_MEMTYPE 0x80000002
-#endif
-
 #ifndef KERNEL_FILENAME
 #define KERNEL_FILENAME L"kernel.elf"
 #endif

src/boot/main.cpp

@@ -10,6 +10,10 @@
 #include "memory.h"
 #include "utility.h"
 
+#ifndef SCRATCH_PAGES
+#define SCRATCH_PAGES 64
+#endif
+
 #ifndef GIT_VERSION_WIDE
 #define GIT_VERSION_WIDE L"no version"
 #endif
@@ -30,20 +34,24 @@ struct kernel_header {
 };
 #pragma pack(pop)
 
-EFI_STATUS
+using kernel_entry = void (*)(popcorn_data *);
+
+extern "C" EFI_STATUS
 efi_main(EFI_HANDLE image_handle, EFI_SYSTEM_TABLE *system_table)
 {
 	EFI_STATUS status;
 	EFI_BOOT_SERVICES *bootsvc = system_table->BootServices;
 	EFI_RUNTIME_SERVICES *runsvc = system_table->RuntimeServices;
 
+	console con(system_table);
+
 	// When checking console initialization, use CHECK_EFI_STATUS_OR_RETURN
 	// because we can't be sure if the console was fully set up
-	status = con_initialize(system_table, GIT_VERSION_WIDE);
-	CHECK_EFI_STATUS_OR_RETURN(status, "con_initialize");
+	status = con.initialize(GIT_VERSION_WIDE);
+	CHECK_EFI_STATUS_OR_RETURN(status, "console::initialize");
 	// From here on out, we can use CHECK_EFI_STATUS_OR_FAIL instead
 
-	memory_init_pointer_fixup(bootsvc, runsvc);
+	memory_init_pointer_fixup(bootsvc, runsvc, SCRATCH_PAGES);
 
 	// Find ACPI tables. Ignore ACPI 1.0 if a 2.0 table is found.
 	//
@@ -65,8 +73,8 @@ efi_main(EFI_HANDLE image_handle, EFI_SYSTEM_TABLE *system_table)
 	status = memory_get_map_length(bootsvc, &data_length);
 	CHECK_EFI_STATUS_OR_FAIL(status);
 
-	size_t header_size = sizeof(struct popcorn_data);
-	const size_t header_align = alignof(struct popcorn_data);
+	size_t header_size = sizeof(popcorn_data);
+	const size_t header_align = alignof(popcorn_data);
 	if (header_size % header_align)
 		header_size += header_align - (header_size % header_align);
 
@@ -75,29 +83,30 @@ efi_main(EFI_HANDLE image_handle, EFI_SYSTEM_TABLE *system_table)
 
 	// Load the kernel image from disk and check it
 	//
-	con_printf(L"Loading kernel into memory...\r\n");
+	console::print(L"Loading kernel into memory...\r\n");
 
 	struct loader_data load;
 	load.data_length = data_length;
 	status = loader_load_kernel(bootsvc, &load);
 	CHECK_EFI_STATUS_OR_FAIL(status);
 
-	con_printf(L"    %u image bytes at 0x%x\r\n", load.kernel_length, load.kernel);
-	con_printf(L"    %u initrd bytes at 0x%x\r\n", load.initrd_length, load.initrd);
-	con_printf(L"    %u data bytes at 0x%x\r\n", load.data_length, load.data);
+	console::print(L"    %u image bytes at 0x%x\r\n", load.kernel_length, load.kernel);
+	console::print(L"    %u initrd bytes at 0x%x\r\n", load.initrd_length, load.initrd);
+	console::print(L"    %u data bytes at 0x%x\r\n", load.data_length, load.data);
 
 	struct kernel_header *version = (struct kernel_header *)load.kernel;
 	if (version->magic != KERNEL_HEADER_MAGIC) {
-		con_printf(L"    bad magic %x\r\n", version->magic);
+		console::print(L"    bad magic %x\r\n", version->magic);
 		CHECK_EFI_STATUS_OR_FAIL(EFI_CRC_ERROR);
 	}
 
-	con_printf(L"    Kernel version %d.%d.%d %x%s\r\n",
+	console::print(L"    Kernel version %d.%d.%d %x%s\r\n",
 			version->major, version->minor, version->patch, version->gitsha & 0x0fffffff,
 			version->gitsha & 0xf0000000 ? "*" : "");
-	con_printf(L"    Entrypoint 0x%x\r\n", load.kernel_entry);
+	console::print(L"    Entrypoint 0x%x\r\n", load.kernel_entry);
 
-	void (*kernel_main)() = load.kernel_entry;
+	kernel_entry kernel_main =
+		reinterpret_cast<kernel_entry>(load.kernel_entry);
 	memory_mark_pointer_fixup((void **)&kernel_main);
 
 	// Set up the kernel data pages to pass to the kernel
@@ -109,6 +118,7 @@ efi_main(EFI_HANDLE image_handle, EFI_SYSTEM_TABLE *system_table)
 	data_header->version = DATA_HEADER_VERSION;
 	data_header->length = sizeof(struct popcorn_data);
 
+	data_header->scratch_pages = SCRATCH_PAGES;
 	data_header->flags = 0;
 
 	data_header->initrd = load.initrd;
@@ -148,8 +158,9 @@ efi_main(EFI_HANDLE image_handle, EFI_SYSTEM_TABLE *system_table)
 	// Save the memory map and tell the firmware we're taking control.
 	//
 	struct memory_map map;
-	map.entries = data_header->memory_map;
 	map.length = (load.data_length - header_size);
+	map.entries =
+		reinterpret_cast<EFI_MEMORY_DESCRIPTOR *>(data_header->memory_map);
 
 	status = memory_get_map(bootsvc, &map);
 	CHECK_EFI_STATUS_OR_FAIL(status);

src/boot/memory.cpp

@@ -4,13 +4,18 @@
 #include "memory.h"
 #include "utility.h"
 
+const EFI_MEMORY_TYPE memtype_kernel  = static_cast<EFI_MEMORY_TYPE>(0x80000000);
+const EFI_MEMORY_TYPE memtype_data    = static_cast<EFI_MEMORY_TYPE>(0x80000001);
+const EFI_MEMORY_TYPE memtype_initrd  = static_cast<EFI_MEMORY_TYPE>(0x80000002);
+const EFI_MEMORY_TYPE memtype_scratch = static_cast<EFI_MEMORY_TYPE>(0x80000003);
+
 #define INCREMENT_DESC(p, b)  (EFI_MEMORY_DESCRIPTOR*)(((uint8_t*)(p))+(b))
 
 size_t fixup_pointer_index = 0;
 void **fixup_pointers[64];
 uint64_t *new_pml4 = 0;
 
-const CHAR16 *memory_type_names[] = {
+const wchar_t *memory_type_names[] = {
 	L"EfiReservedMemoryType",
 	L"EfiLoaderCode",
 	L"EfiLoaderData",
@@ -28,13 +33,17 @@ const CHAR16 *memory_type_names[] = {
 	L"EfiPersistentMemory",
 };
 
-static const CHAR16 *
+static const wchar_t *
 memory_type_name(UINT32 value)
 {
-	if (value >= (sizeof(memory_type_names) / sizeof(CHAR16 *))) {
-		if (value == KERNEL_DATA_MEMTYPE) return L"Kernel Data";
-		else if (value == KERNEL_MEMTYPE) return L"Kernel Image";
-		else return L"Bad Type Value";
+	if (value >= (sizeof(memory_type_names) / sizeof(wchar_t *))) {
+		switch (value) {
+			case memtype_kernel:  return L"Kernel Data";
+			case memtype_data:    return L"Kernel Data";
+			case memtype_initrd:  return L"Initial Ramdisk";
+			case memtype_scratch: return L"Kernel Scratch Space";
+			default: return L"Bad Type Value";
+		}
 	}
 	return memory_type_names[value];
 }
@@ -50,7 +59,7 @@ memory_update_marked_addresses(EFI_EVENT UNUSED *event, void *context)
 }
 
 EFI_STATUS
-memory_init_pointer_fixup(EFI_BOOT_SERVICES *bootsvc, EFI_RUNTIME_SERVICES *runsvc)
+memory_init_pointer_fixup(EFI_BOOT_SERVICES *bootsvc, EFI_RUNTIME_SERVICES *runsvc, unsigned scratch_pages)
 {
 	EFI_STATUS status;
 	EFI_EVENT event;
@@ -67,7 +76,7 @@ memory_init_pointer_fixup(EFI_BOOT_SERVICES *bootsvc, EFI_RUNTIME_SERVICES *runs
 	// Reserve a page for our replacement PML4, plus some pages for the kernel to use
 	// as page tables while it gets started.
 	EFI_PHYSICAL_ADDRESS addr = 0;
-	status = bootsvc->AllocatePages(AllocateAnyPages, EfiLoaderData, 16, &addr);
+	status = bootsvc->AllocatePages(AllocateAnyPages, memtype_scratch, scratch_pages, &addr);
 	CHECK_EFI_STATUS_OR_RETURN(status, "Failed to allocate page table pages.");
 
 	new_pml4 = (uint64_t *)addr;
@@ -139,19 +148,19 @@ memory_dump_map(struct memory_map *map)
 
 	const size_t count = map->length / map->size;
 
-	con_printf(L"Memory map:\n");
-	con_printf(L"\t	Descriptor Count: %d (%d bytes)\n", count, map->length);
-	con_printf(L"\t  Descriptor Size: %d bytes\n", map->size);
-	con_printf(L"\t      Type offset: %d\n\n", offsetof(EFI_MEMORY_DESCRIPTOR, Type));
+	console::print(L"Memory map:\n");
+	console::print(L"\t	Descriptor Count: %d (%d bytes)\n", count, map->length);
+	console::print(L"\t  Descriptor Size: %d bytes\n", map->size);
+	console::print(L"\t      Type offset: %d\n\n", offsetof(EFI_MEMORY_DESCRIPTOR, Type));
 
 	EFI_MEMORY_DESCRIPTOR *end = INCREMENT_DESC(map->entries, map->length);
 	EFI_MEMORY_DESCRIPTOR *d = map->entries;
 	while (d < end) {
 		int runtime = (d->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME;
-		con_printf(L"%s%s ", memory_type_name(d->Type), runtime ? L"*" : L" ");
-		con_printf(L"%lx ", d->PhysicalStart);
-		con_printf(L"%lx ", d->VirtualStart);
-		con_printf(L"[%4d]\n", d->NumberOfPages);
+		console::print(L"%s%s ", memory_type_name(d->Type), runtime ? L"*" : L" ");
+		console::print(L"%lx ", d->PhysicalStart);
+		console::print(L"%lx ", d->VirtualStart);
+		console::print(L"[%4d]\n", d->NumberOfPages);
 
 		d = INCREMENT_DESC(d, map->size);
 	}
@@ -187,9 +196,10 @@ memory_virtualize(EFI_RUNTIME_SERVICES *runsvc, struct memory_map *map)
 	EFI_MEMORY_DESCRIPTOR *d = map->entries;
 	while (d < end) {
 		switch (d->Type) {
-		case KERNEL_MEMTYPE:
-		case INITRD_MEMTYPE:
-		case KERNEL_DATA_MEMTYPE:
+		case memtype_kernel:
+		case memtype_data:
+		case memtype_initrd:
+		case memtype_scratch:
 			d->Attribute |= EFI_MEMORY_RUNTIME;
 			d->VirtualStart = d->PhysicalStart + KERNEL_VIRT_ADDRESS;
 

src/boot/memory.h

@@ -1,6 +1,11 @@
 #pragma once
 #include <efi/efi.h>
 
+extern const EFI_MEMORY_TYPE memtype_kernel;
+extern const EFI_MEMORY_TYPE memtype_data;
+extern const EFI_MEMORY_TYPE memtype_initrd;
+extern const EFI_MEMORY_TYPE memtype_scratch;
+
 struct memory_map {
 	size_t length;
 	size_t size;
@@ -9,7 +14,10 @@ struct memory_map {
 	EFI_MEMORY_DESCRIPTOR *entries;
 };
 
-EFI_STATUS memory_init_pointer_fixup(EFI_BOOT_SERVICES *bootsvc, EFI_RUNTIME_SERVICES *runsvc);
+EFI_STATUS memory_init_pointer_fixup(
+		EFI_BOOT_SERVICES *bootsvc,
+		EFI_RUNTIME_SERVICES *runsvc,
+		unsigned scratch_pages);
 void memory_mark_pointer_fixup(void **p);
 
 EFI_STATUS memory_get_map_length(EFI_BOOT_SERVICES *bootsvc, size_t *size);

src/boot/reloc.cpp

@@ -38,6 +38,7 @@
 #include <efi/efi.h>
 #include <elf.h>
 
+extern "C"
 EFI_STATUS _relocate (long ldbase, Elf64_Dyn *dyn,
 		      EFI_HANDLE image EFI_UNUSED,
 		      EFI_SYSTEM_TABLE *systab EFI_UNUSED)

src/boot/utility.cpp

@@ -2,7 +2,7 @@
 
 struct error_code_desc {
 	EFI_STATUS code;
-	CHAR16 *name;
+	const wchar_t *name;
 };
 
 // Based off the gnu-efi table
@@ -47,7 +47,7 @@ struct error_code_desc error_table[] = {
 	{ 0, NULL }
 };
 
-const CHAR16 *
+const wchar_t *
 util_error_message(EFI_STATUS status)
 {
 	int32_t i = -1;
@@ -62,7 +62,7 @@ util_error_message(EFI_STATUS status)
 }
 
 size_t
-wstrlen(const CHAR16 *s)
+wstrlen(const wchar_t *s)
 {
 	size_t count = 0;
 	while (s && *s++) count++;

src/boot/utility.h

@@ -5,18 +5,18 @@
 
 #define UNUSED __attribute__((unused))
 
-size_t wstrlen(const CHAR16 *s);
-const CHAR16 *util_error_message(EFI_STATUS status);
+size_t wstrlen(const wchar_t *s);
+const wchar_t *util_error_message(EFI_STATUS status);
 
 #define CHECK_EFI_STATUS_OR_RETURN(s, msg, ...) \
 	if (EFI_ERROR((s))) { \
-		con_printf(L"ERROR: " msg L": %s\r\n", ##__VA_ARGS__, util_error_message(s)); \
+		console::print(L"ERROR: " msg L": %s\r\n", ##__VA_ARGS__, util_error_message(s)); \
 		return (s); \
 	}
 
 #define CHECK_EFI_STATUS_OR_FAIL(s) \
 	if (EFI_ERROR((s))) { \
-		con_status_fail(util_error_message(s)); \
+		console::get().status_fail(util_error_message(s)); \
 		while (1) __asm__("hlt"); \
 	}
 
@@ -34,7 +34,7 @@ const CHAR16 *util_error_message(EFI_STATUS status);
 	}
 
 #ifdef BOOTLOADER_DEBUG
-#define con_debug(...) con_printf(L"DEBUG: " __VA_ARGS__)
+#define con_debug(...) console::print(L"DEBUG: " __VA_ARGS__)
 #else
 #define con_debug(...)
 #endif

src/drivers/nulldrv/main.s

@@ -1,11 +1,35 @@
+section .bss
+mypid: resq 1
+mychild: resq 1
+
+section .text
 global _start
 _start:
 	xor rbp, rbp	; Sentinel rbp
+
+	mov rax, 5		; GETPID syscall
+	int 0xee
+	mov [mypid], rax
+
+	mov rax, 8		; FORK syscall
+	int 0xee
+	mov [mychild], rax
+
+	mov r12, [mypid]
+	mov r13, [mychild]
+	mov rax, 1		; DEBUG syscall
+	int 0xee
+
+	cmp r12, 1
+	je .dosend
+	jne .doreceive
+
+.preloop:
 	mov r11, 0		; counter
 	mov rbx, 20		; sleep timeout
 
 .loop:
-	mov rax, 1		; DEBUG syscall
+	mov rax, 1		; MESSAGE syscall
 	;mov rax, 0		; NOOP syscall
 	;syscall
 	int 0xee
@@ -23,3 +47,15 @@ _start:
 
 	mov r11, 0
 	jmp .loop
+
+.dosend:
+	mov rax, 6      ; SEND syscall
+	mov rdi, 2      ; target is pid 2
+	int 0xee
+	jmp .preloop
+
+.doreceive:
+	mov rax, 7      ; RECEIVE syscall
+	mov rdi, 1      ; source is pid 2
+	int 0xee
+	jmp .preloop

src/include/efi/efidef.h

@@ -20,9 +20,9 @@ Revision History
 
 --*/
 
-typedef UINT16          CHAR16;
-typedef UINT8           CHAR8;
-typedef UINT8           BOOLEAN;
+typedef wchar_t         CHAR16;
+typedef char            CHAR8;
+typedef uint8_t         BOOLEAN;
 #ifndef CONST
    #define CONST const
 #endif

src/include/kernel_data.h

@@ -13,7 +13,8 @@ struct popcorn_data {
 	uint16_t version;
 	uint16_t length;
 
-	uint32_t _reserved0;
+	uint16_t _reserved0;
+	uint16_t scratch_pages;
 	uint32_t flags;
 
 	void *initrd;

src/kernel/cpu.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <stdint.h>
+
 struct cpu_state
 {
 	uint64_t ds;

src/kernel/crti.s

@@ -0,0 +1,14 @@
+section .init
+global _init:function
+_init:
+	push rbp
+	mov rbp, rsp
+	; Control flow falls through to other .init sections
+
+section .fini
+global _fini:function
+_fini:
+	push rbp
+	mov rbp, rsp
+	; Control flow falls through to other .fini sections
+

src/kernel/crtn.s

@@ -0,0 +1,10 @@
+section .init
+	; Control flow falls through to here from other .init sections
+	pop rbp
+	ret
+
+section .fini
+	; Control flow falls through to here from other .fini sections
+	pop rbp
+	ret
+

src/kernel/debug.cpp

@@ -2,6 +2,7 @@
 #include "cpu.h"
 #include "debug.h"
 #include "gdt.h"
+#include "page_manager.h"
 
 #define print_reg(name, value) cons->printf("         %s: %016lx\n", name, (value));
 
@@ -39,6 +40,9 @@ print_regs(const cpu_state &regs)
 
 	cons->puts("\n");
 	print_reg("rip", regs.rip);
+
+	cons->puts("\n");
+	print_reg("cr3", page_manager::get()->get_pml4());
 }
 
 void

src/kernel/gdt.s

@@ -25,7 +25,7 @@ gdt_write:
 	push rax
 	o64 retf
 .next:
-	ltr dx
+	ltr dx ; third arg is the TSS
 	ret
 
 global gdt_load

src/kernel/interrupts.cpp

@@ -18,8 +18,8 @@ static const uint16_t PIC2 = 0xa0;
 extern "C" {
 	void _halt();
 
-	uintptr_t isr_handler(uintptr_t, cpu_state);
-	uintptr_t irq_handler(uintptr_t, cpu_state);
+	uintptr_t isr_handler(uintptr_t, cpu_state*);
+	uintptr_t irq_handler(uintptr_t, cpu_state*);
 	uintptr_t syscall_handler(uintptr_t, cpu_state);
 
 #define ISR(i, name)     extern void name ();
@@ -105,23 +105,23 @@ interrupts_init()
 }
 
 uintptr_t
-isr_handler(uintptr_t return_rsp, cpu_state regs)
+isr_handler(uintptr_t return_rsp, cpu_state *regs)
 {
 	console *cons = console::get();
 
-	switch (static_cast<isr>(regs.interrupt & 0xff)) {
+	switch (static_cast<isr>(regs->interrupt & 0xff)) {
 
 	case isr::isrGPFault: {
 			cons->set_color(9);
 			cons->puts("\nGeneral Protection Fault:\n");
 			cons->set_color();
 
-			cons->printf("   errorcode: %lx", regs.errorcode);
-			if (regs.errorcode & 0x01) cons->puts(" external");
+			cons->printf("   errorcode: %lx", regs->errorcode);
+			if (regs->errorcode & 0x01) cons->puts(" external");
 
-			int index = (regs.errorcode & 0xffff) >> 4;
+			int index = (regs->errorcode & 0xffff) >> 4;
 			if (index) {
-				switch ((regs.errorcode & 0x07) >> 1) {
+				switch ((regs->errorcode & 0x07) >> 1) {
 				case 0:
 					cons->printf(" GDT[%x]\n", index);
 					gdt_dump();
@@ -140,10 +140,10 @@ isr_handler(uintptr_t return_rsp, cpu_state regs)
 			} else {
 				cons->putc('\n');
 			}
-			print_regs(regs);
+			print_regs(*regs);
 			/*
 			print_stacktrace(2);
-			print_stack(regs);
+			print_stack(*regs);
 			*/
 
 		}
@@ -156,22 +156,22 @@ isr_handler(uintptr_t return_rsp, cpu_state regs)
 			cons->set_color();
 
 			cons->puts("       flags:");
-			if (regs.errorcode & 0x01) cons->puts(" present");
-			if (regs.errorcode & 0x02) cons->puts(" write");
-			if (regs.errorcode & 0x04) cons->puts(" user");
-			if (regs.errorcode & 0x08) cons->puts(" reserved");
-			if (regs.errorcode & 0x10) cons->puts(" ip");
+			if (regs->errorcode & 0x01) cons->puts(" present");
+			if (regs->errorcode & 0x02) cons->puts(" write");
+			if (regs->errorcode & 0x04) cons->puts(" user");
+			if (regs->errorcode & 0x08) cons->puts(" reserved");
+			if (regs->errorcode & 0x10) cons->puts(" ip");
 			cons->puts("\n");
 
 			uint64_t cr2 = 0;
 			__asm__ __volatile__ ("mov %%cr2, %0" : "=r"(cr2));
 			print_reg("cr2", cr2);
 
-			print_reg("rsp", regs.user_rsp);
-			print_reg("rip", regs.rip);
+			print_reg("rsp", regs->user_rsp);
+			print_reg("rip", regs->rip);
 
 			cons->puts("\n");
-			print_stacktrace(2);
+			//print_stacktrace(2);
 		}
 		_halt();
 		break;
@@ -192,14 +192,14 @@ isr_handler(uintptr_t return_rsp, cpu_state regs)
 
 	case isr::isrAssert: {
 			cons->set_color();
-			print_regs(regs);
+			print_regs(*regs);
 			print_stacktrace(2);
 		}
 		_halt();
 		break;
 
 	case isr::isrSyscall: {
-			return_rsp = syscall_dispatch(return_rsp, regs);
+			return_rsp = syscall_dispatch(return_rsp, *regs);
 		}
 		break;
 
@@ -215,7 +215,7 @@ isr_handler(uintptr_t return_rsp, cpu_state regs)
 	case isr::isrIgnore5:
 	case isr::isrIgnore6:
 	case isr::isrIgnore7:
-		//cons->printf("\nIGNORED: %02x\n", regs.interrupt);
+		//cons->printf("\nIGNORED: %02x\n", regs->interrupt);
 		outb(PIC1, 0x20);
 		break;
 
@@ -227,7 +227,7 @@ isr_handler(uintptr_t return_rsp, cpu_state regs)
 	case isr::isrIgnoreD:
 	case isr::isrIgnoreE:
 	case isr::isrIgnoreF:
-		//cons->printf("\nIGNORED: %02x\n", regs.interrupt);
+		//cons->printf("\nIGNORED: %02x\n", regs->interrupt);
 		outb(PIC1, 0x20);
 		outb(PIC2, 0x20);
 		break;
@@ -235,13 +235,13 @@ isr_handler(uintptr_t return_rsp, cpu_state regs)
 	default:
 		cons->set_color(9);
 		cons->printf("\nReceived %02x interrupt:\n",
-				(static_cast<isr>(regs.interrupt)));
+				(static_cast<isr>(regs->interrupt)));
 
 		cons->set_color();
 		cons->printf("         ISR: %02lx  ERR: %lx\n\n",
-				regs.interrupt, regs.errorcode);
+				regs->interrupt, regs->errorcode);
 
-		print_regs(regs);
+		print_regs(*regs);
 		//print_stacktrace(2);
 		_halt();
 	}
@@ -251,16 +251,16 @@ isr_handler(uintptr_t return_rsp, cpu_state regs)
 }
 
 uintptr_t
-irq_handler(uintptr_t return_rsp, cpu_state regs)
+irq_handler(uintptr_t return_rsp, cpu_state *regs)
 {
 	console *cons = console::get();
-	uint8_t irq = get_irq(regs.interrupt);
+	uint8_t irq = get_irq(regs->interrupt);
 	if (! device_manager::get().dispatch_irq(irq)) {
 		cons->set_color(11);
 		cons->printf("\nReceived unknown IRQ: %d (vec %d)\n",
-				irq, regs.interrupt);
+				irq, regs->interrupt);
 		cons->set_color();
-		print_regs(regs);
+		print_regs(*regs);
 		_halt();
 	}
 

src/kernel/interrupts.s

@@ -6,6 +6,7 @@ isr_handler_prelude:
 	push_all_and_segments
 
 	mov rdi, rsp
+	mov rsi, rsp
 	call isr_handler
 	mov rsp, rax
 
@@ -21,6 +22,7 @@ irq_handler_prelude:
 	push_all_and_segments
 
 	mov rdi, rsp
+	mov rsi, rsp
 	call irq_handler
 	mov rsp, rax
 
@@ -56,8 +58,8 @@ irq_handler_prelude:
 		jmp irq_handler_prelude
 %endmacro
 
-%define EISR(i, name)     EMIT_EISR name, i
-%define UISR(i, name)     EMIT_ISR name, i
+%define EISR(i, name)     EMIT_EISR name, i   ; ISR with error code
+%define UISR(i, name)     EMIT_ISR name, i    ; ISR callable from user space
 %define  ISR(i, name)     EMIT_ISR name, i
 %define  IRQ(i, q, name)  EMIT_IRQ name, i
 

src/kernel/kernel_memory.h

@@ -0,0 +1,22 @@
+#pragma once
+/// \file kernel_memory.h
+/// Constants related to the kernel's memory layout
+
+namespace memory {
+
+	/// Size of a single page frame.
+	static const size_t frame_size = 0x1000;
+
+	/// Start of kernel memory.
+	static const uintptr_t kernel_offset = 0xffffff0000000000;
+
+	/// Offset from physical where page tables are mapped.
+	static const uintptr_t page_offset = 0xffffff8000000000;
+
+	/// Initial process thread's stack address
+	static const uintptr_t initial_stack = 0x0000800000000000;
+
+	/// Initial process thread's stack size, in pages
+	static const unsigned initial_stack_pages = 1;
+
+} // namespace memory

src/kernel/log.cpp

@@ -17,6 +17,7 @@ static const char *areas[] = {
 	"driver",
 	"file  ",
 	"task  ",
+	"paging",
 
 	nullptr
 };

src/kernel/log.h

@@ -14,6 +14,7 @@ enum class logs
 	driver,
 	fs,
 	task,
+	paging,
 
 	max
 };

src/kernel/main.cpp

@@ -38,12 +38,15 @@ init_console()
 	cons->puts(GIT_VERSION " booting...\n");
 
 	log::init(cons);
-	log::enable(logs::apic, log::level::debug);
+	log::enable(logs::apic, log::level::info);
 	log::enable(logs::device, log::level::info);
+
 	log::enable(logs::driver, log::level::debug);
-	log::enable(logs::memory, log::level::info);
+	log::enable(logs::memory, log::level::debug);
 	log::enable(logs::fs, log::level::debug);
 	log::enable(logs::task, log::level::debug);
+	log::enable(logs::boot, log::level::debug);
+	log::enable(logs::paging, log::level::debug);
 }
 
 void
@@ -64,16 +67,17 @@ kernel_main(popcorn_data *header)
 	gdt_init();
 	interrupts_init();
 
-	page_manager *pager = new (&g_page_manager) page_manager;
-
 	memory_initialize(
+			header->scratch_pages,
 			header->memory_map,
 			header->memory_map_length,
 			header->memory_map_desc_size);
 
-	pager->map_offset_pointer(
-			&header->frame_buffer,
-			header->frame_buffer_length);
+	if (header->frame_buffer && header->frame_buffer_length) {
+		page_manager::get()->map_offset_pointer(
+				&header->frame_buffer,
+				header->frame_buffer_length);
+	}
 
 	init_console();
 
@@ -90,8 +94,8 @@ kernel_main(popcorn_data *header)
 		log::info(logs::boot, "  %s%s (%d bytes).", f.executable() ? "*" : "", f.name(), f.size());
 
 	/*
-	pager->dump_pml4(nullptr, 0);
-	pager->dump_blocks(true);
+	   page_manager::get()->dump_pml4(nullptr, 0);
+	   page_manager::get()->dump_blocks(true);
 	*/
 
 	device_manager *devices =
@@ -144,12 +148,10 @@ kernel_main(popcorn_data *header)
 	syscall_enable();
 	scheduler *sched = new (&scheduler::get()) scheduler(devices->get_lapic());
 
-	/*
 	for (auto &f : ird.files()) {
 		if (f.executable())
-			sched->create_process(f.name(), f.data(), f.size());
+			sched->load_process(f.name(), f.data(), f.size());
 	}
-	*/
 
 	sched->start();
 }

src/kernel/memory_bootstrap.cpp

@@ -1,32 +1,73 @@
+#include <algorithm>
 #include <utility>
+#include "kutil/address_manager.h"
 #include "kutil/assert.h"
-#include "kutil/linked_list.h"
-#include "kutil/slab_allocator.h"
+#include "kutil/frame_allocator.h"
+#include "kutil/heap_manager.h"
 #include "io.h"
+#include "log.h"
 #include "page_manager.h"
 
-const unsigned efi_page_size = 0x1000;
-const unsigned ident_page_flags = 0xb;
+using kutil::frame_block;
+using kutil::frame_block_flags;
+using kutil::frame_block_list;
+using memory::frame_size;
+using memory::kernel_offset;
+using memory::page_offset;
+
+static const unsigned ident_page_flags = 0xb;
+
+kutil::frame_allocator g_frame_allocator;
+kutil::address_manager g_kernel_address_manager;
+kutil::heap_manager g_kernel_heap_manager;
+
+void * mm_grow_callback(size_t length)
+{
+	kassert(length % frame_size == 0,
+			"Heap manager requested a fractional page.");
+
+	size_t pages = length / frame_size;
+	log::info(logs::memory, "Heap manager growing heap by %d pages.", pages);
+
+	uintptr_t addr = g_kernel_address_manager.allocate(length);
+	g_page_manager.map_pages(addr, pages);
+
+	return reinterpret_cast<void *>(addr);
+}
+
 
 namespace {
-	// Page-by-page initial allocator for the initial page_block allocator
+	// Page-by-page initial allocator for the initial frame_block allocator
 	struct page_consumer
 	{
-		page_consumer(uintptr_t start) : current(start) {}
+		page_consumer(uintptr_t start, unsigned count, unsigned used = 0) :
+			current(start + used * frame_size),
+			used(used),
+			max(count) {}
 
-		void * operator()(size_t size) {
-			kassert(size == page_manager::page_size, "page_consumer used with non-page size!");
+		void * get_page() {
+			kassert(used++ < max, "page_consumer ran out of pages");
 			void *retval = reinterpret_cast<void *>(current);
-			current += size;
+			current += frame_size;
 			return retval;
 		}
 
-		uintptr_t current;
-	};
-}
+		void * operator()(size_t size) {
+			kassert(size == frame_size, "page_consumer used with non-page size!");
+			return get_page();
+		}
 
-using block_list = kutil::linked_list<page_block>;
-using block_allocator = kutil::slab_allocator<page_block, page_consumer &>;
+		unsigned left() const { return max - used; }
+
+		uintptr_t current;
+		unsigned used, max;
+	};
+
+	using block_allocator =
+		kutil::slab_allocator<kutil::frame_block, page_consumer &>;
+	using region_allocator =
+		kutil::slab_allocator<kutil::buddy_region, page_consumer &>;
+}
 
 enum class efi_memory_type : uint32_t
 {
@@ -49,66 +90,13 @@ enum class efi_memory_type : uint32_t
 	efi_max,
 
 	popcorn_kernel = 0x80000000,
-	popcorn_font,
 	popcorn_data,
-	popcorn_log,
-	popcorn_pml4,
+	popcorn_initrd,
+	popcorn_scratch,
 
 	popcorn_max
 };
 
-const char *efi_memory_type_names[] = {
-	"             reserved",
-	"          loader_code",
-	"          loader_data",
-	"   boot_services_code",
-	"   boot_services_data",
-	"runtime_services_code",
-	"runtime_services_data",
-	"            available",
-	"             unusable",
-	"         acpi_reclaim",
-	"             acpi_nvs",
-	"                 mmio",
-	"            mmio_port",
-	"             pal_code",
-
-	"       popcorn_kernel",
-	"         popcorn_font",
-	"         popcorn_data",
-	"          popcorn_log",
-	"         popcorn_pml4",
-};
-
-static const char *
-get_efi_name(efi_memory_type t)
-{
-	static const unsigned offset =
-		(unsigned)efi_memory_type::popcorn_kernel - (unsigned)efi_memory_type::efi_max;
-
-	return t >= efi_memory_type::popcorn_kernel ?
-		efi_memory_type_names[(unsigned)t - offset] :
-		efi_memory_type_names[(unsigned)t];
-}
-
-enum class efi_memory_flag : uint64_t
-{
-	can_mark_uc  = 0x0000000000000001, // uc = un-cacheable
-	can_mark_wc  = 0x0000000000000002, // wc = write-combining
-	can_mark_wt  = 0x0000000000000004, // wt = write through
-	can_mark_wb  = 0x0000000000000008, // wb = write back
-	can_mark_uce = 0x0000000000000010, // uce = un-cacheable exported
-	can_mark_wp  = 0x0000000000001000, // wp = write protected
-	can_mark_rp  = 0x0000000000002000, // rp = read protected
-	can_mark_xp  = 0x0000000000004000, // xp = exceute protected
-	can_mark_ro  = 0x0000000000020000, // ro = read only
-
-	non_volatile  = 0x0000000000008000,
-	more_reliable = 0x0000000000010000,
-	runtime       = 0x8000000000000000
-};
-IS_BITFIELD(efi_memory_flag);
-
 struct efi_memory_descriptor
 {
 	efi_memory_type type;
@@ -116,7 +104,7 @@ struct efi_memory_descriptor
 	uint64_t physical_start;
 	uint64_t virtual_start;
 	uint64_t pages;
-	efi_memory_flag flags;
+	uint64_t flags;
 };
 
 static const efi_memory_descriptor *
@@ -126,68 +114,11 @@ desc_incr(const efi_memory_descriptor *d, size_t desc_length)
 			reinterpret_cast<const uint8_t *>(d) + desc_length);
 }
 
-page_block_list::item_type *
-remove_block_for(page_block_list &list, uintptr_t phys_start, size_t pages, page_block_list &cache)
-{
-	// This is basically just the removal portion of page_manager::unmap_pages,
-	// but with physical addresses, and only ever removing a single block.
-
-	for (auto *item : list) {
-		if (!item->contains_physical(phys_start))
-			continue;
-
-		uint64_t size = page_manager::page_size * pages;
-		uint64_t end = phys_start + size;
-		uint64_t leading = phys_start - item->physical_address;
-		uint64_t trailing = item->physical_end() - end;
-
-		if (leading) {
-			uint64_t pages = leading / page_manager::page_size;
-
-			page_block_list::item_type *lead_block = cache.pop_front();
-
-			lead_block->copy(item);
-			lead_block->count = pages;
-
-			item->count -= pages;
-			item->physical_address += leading;
-
-			if (item->virtual_address)
-				item->virtual_address += leading;
-
-			list.insert_before(item, lead_block);
-		}
-
-		if (trailing) {
-			uint64_t pages = trailing / page_manager::page_size;
-
-			page_block_list::item_type *trail_block = cache.pop_front();
-
-			trail_block->copy(item);
-			trail_block->count = pages;
-			trail_block->physical_address += size;
-
-			item->count -= pages;
-
-			if (item->virtual_address)
-				trail_block->virtual_address += size;
-
-			list.insert_before(item, trail_block);
-		}
-
-		list.remove(item);
-		return item;
-	}
-
-	kassert(false, "Couldn't find block to remove");
-	return nullptr;
-}
-
 void
 gather_block_lists(
 		block_allocator &allocator,
-		block_list &used,
-		block_list &free,
+		frame_block_list &used,
+		frame_block_list &free,
 		const void *memory_map,
 		size_t map_length,
 		size_t desc_length)
@@ -197,192 +128,74 @@ gather_block_lists(
 
 	while (desc < end) {
 		auto *block = allocator.pop();
-		block->physical_address = desc->physical_start;
-		block->virtual_address = desc->virtual_start;
+		block->address = desc->physical_start;
 		block->count = desc->pages;
+		bool block_used;
 
 		switch (desc->type) {
 		case efi_memory_type::loader_code:
 		case efi_memory_type::loader_data:
-			block->flags = page_block_flags::used | page_block_flags::pending_free;
+			block_used = true;
+			block->flags = frame_block_flags::pending_free;
 			break;
 
 		case efi_memory_type::boot_services_code:
 		case efi_memory_type::boot_services_data:
 		case efi_memory_type::available:
-			block->flags = page_block_flags::free;
+			block_used = false;
 			break;
 
 		case efi_memory_type::acpi_reclaim:
+			block_used = true;
 			block->flags =
-				page_block_flags::used |
-				page_block_flags::mapped |
-				page_block_flags::acpi_wait;
-
-			block->virtual_address = block->physical_address;
+				frame_block_flags::acpi_wait |
+				frame_block_flags::map_ident;
 			break;
 
 		case efi_memory_type::persistent:
-			block->flags = page_block_flags::nonvolatile;
+			block_used = false;
+			block->flags = frame_block_flags::nonvolatile;
+			break;
+
+		case efi_memory_type::popcorn_kernel:
+			block_used = true;
+			block->flags =
+				frame_block_flags::permanent |
+				frame_block_flags::map_kernel;
+			break;
+
+		case efi_memory_type::popcorn_data:
+		case efi_memory_type::popcorn_initrd:
+			block_used = true;
+			block->flags =
+				frame_block_flags::pending_free |
+				frame_block_flags::map_kernel;
+			break;
+
+		case efi_memory_type::popcorn_scratch:
+			block_used = true;
+			block->flags = frame_block_flags::map_offset;
 			break;
 
 		default:
-			block->flags = page_block_flags::used | page_block_flags::permanent;
+			block_used = true;
+			block->flags = frame_block_flags::permanent;
 			break;
 		}
 
-		if (block->has_flag(page_block_flags::used)) {
-			if (block->virtual_address || !block->physical_address)
-				block->flags |= page_block_flags::mapped;
-
+		if (block_used) 
 			used.push_back(block);
-		} else {
+		else
 			free.push_back(block);
-		}
 
 		desc = desc_incr(desc, desc_length);
 	}
 }
 
 void
-copy_new_table(page_table *base, unsigned index, page_table *new_table)
+memory_initialize(uint16_t scratch_pages, const void *memory_map, size_t map_length, size_t desc_length)
 {
-	uint64_t entry = base->entries[index];
-
-	// If this is a large page and not a a table, bail out.
-	if(entry & 0x80) return;
-
-	if (entry & 0x1) {
-		page_table *old_next = reinterpret_cast<page_table *>(
-				base->entries[index] & ~0xffful);
-		for (int i = 0; i < 512; ++i) new_table->entries[i] = old_next->entries[i];
-	} else {
-		for (int i = 0; i < 512; ++i) new_table->entries[i] = 0;
-	}
-
-	base->entries[index] = reinterpret_cast<uint64_t>(new_table) | ident_page_flags;
-}
-
-static uint64_t
-find_efi_free_aligned_pages(const void *memory_map, size_t map_length, size_t desc_length, unsigned pages)
-{
-	efi_memory_descriptor const *desc =
-		reinterpret_cast<efi_memory_descriptor const *>(memory_map);
-	efi_memory_descriptor const *end = desc_incr(desc, map_length);
-
-	const unsigned want_space = pages * page_manager::page_size;
-	uint64_t start_phys = 0;
-
-	for (; desc < end; desc = desc_incr(desc, desc_length)) {
-		if (desc->type != efi_memory_type::available)
-			continue;
-
-		// See if the first wanted pages fit in one page table. If we
-		// find free memory at zero, skip ahead because we're not ready
-		// to deal with 0 being a valid pointer yet.
-		start_phys = desc->physical_start;
-		if (start_phys == 0)
-			start_phys += efi_page_size;
-
-		const uint64_t desc_end =
-			desc->physical_start + desc->pages * efi_page_size;
-
-		uint64_t end = start_phys + want_space;
-		if (end < desc_end) {
-			page_table_indices start_idx{start_phys};
-			page_table_indices end_idx{end};
-			if (start_idx[0] == end_idx[0] &&
-				start_idx[1] == end_idx[1] &&
-				start_idx[2] == end_idx[2])
-				break;
-
-			// Try seeing if the page-table-aligned version fits
-			start_phys = page_table_align(start_phys);
-			end = start_phys + want_space;
-			if (end < desc_end)
-				break;
-		}
-	}
-
-	kassert(desc < end, "Couldn't find wanted pages of aligned scratch space.");
-	return start_phys;
-}
-
-static unsigned
-check_needs_page_ident(page_table *table, unsigned index, page_table **free_pages)
-{
-	if ((table->entries[index] & 0x1) == 1) return 0;
-
-	kassert(*free_pages, "check_needs_page_ident needed to allocate but had no free pages");
-
-	page_table *new_table = (*free_pages)++;
-	for (int i=0; i<512; ++i) new_table->entries[i] = 0;
-	table->entries[index] = reinterpret_cast<uint64_t>(new_table) | ident_page_flags;
-	return 1;
-}
-
-static unsigned
-page_in_ident(
-		page_table *pml4,
-		uint64_t phys_addr,
-		uint64_t virt_addr,
-		uint64_t count,
-		page_table *free_pages)
-{
-	page_table_indices idx{virt_addr};
-	page_table *tables[4] = {pml4, nullptr, nullptr, nullptr};
-
-	unsigned pages_consumed = 0;
-	for (; idx[0] < 512; idx[0] += 1) {
-		pages_consumed += check_needs_page_ident(tables[0], idx[0], &free_pages);
-		tables[1] = reinterpret_cast<page_table *>(
-				tables[0]->entries[idx[0]] & ~0xfffull);
-
-		for (; idx[1] < 512; idx[1] += 1, idx[2] = 0, idx[3] = 0) {
-			pages_consumed += check_needs_page_ident(tables[1], idx[1], &free_pages);
-			tables[2] = reinterpret_cast<page_table *>(
-					tables[1]->entries[idx[1]] & ~0xfffull);
-
-			for (; idx[2] < 512; idx[2] += 1, idx[3] = 0) {
-				if (idx[3] == 0 &&
-					count >= 512 &&
-					tables[2]->get(idx[2]) == nullptr) {
-					// Do a 2MiB page instead
-					tables[2]->entries[idx[2]] = phys_addr | 0x80 | ident_page_flags;
-
-					phys_addr += page_manager::page_size * 512;
-					count -= 512;
-					if (count == 0) return pages_consumed;
-					continue;
-				}
-
-				pages_consumed += check_needs_page_ident(tables[2], idx[2], &free_pages);
-				tables[3] = reinterpret_cast<page_table *>(
-						tables[2]->entries[idx[2]] & ~0xfffull);
-
-				for (; idx[3] < 512; idx[3] += 1) {
-					tables[3]->entries[idx[3]] = phys_addr | ident_page_flags;
-					phys_addr += page_manager::page_size;
-					if (--count == 0) return pages_consumed;
-				}
-			}
-		}
-	}
-
-	kassert(0, "Ran to end of page_in_ident");
-	return 0; // Cannot reach
-}
-
-void
-memory_initialize(const void *memory_map, size_t map_length, size_t desc_length)
-{
-	// The bootloader reserved 16 pages for page tables, which we'll use to bootstrap.
-	// The first one is the already-installed PML4, so grab it from CR3.
-	uint64_t cr3;
-	__asm__ __volatile__ ( "mov %%cr3, %0" : "=r" (cr3) );
-	page_table *tables = reinterpret_cast<page_table *>(cr3 & ~0xfffull);
-
-	// We'll need to make sure the options we want in CR4 are set
+	// make sure the options we want in CR4 are set
 	uint64_t cr4;
 	__asm__ __volatile__ ( "mov %%cr4, %0" : "=r" (cr4) );
 	cr4 |= 0x00080; // Enable global pages
@@ -390,114 +203,102 @@ memory_initialize(const void *memory_map, size_t map_length, size_t desc_length)
 	cr4 |= 0x20000; // Enable PCIDs
 	__asm__ __volatile__ ( "mov %0, %%cr4" :: "r" (cr4) );
 
-	// Now go through EFi's memory map and find a region of scratch space.
-	const unsigned want_pages = 32;
-	uint64_t free_region_start_phys =
-		find_efi_free_aligned_pages(memory_map, map_length, desc_length, want_pages);
+	// The bootloader reserved "scratch_pages" pages for page tables and
+	// scratch space, which we'll use to bootstrap.  The first one is the
+	// already-installed PML4, so grab it from CR3.
+	uint64_t scratch_phys;
+	__asm__ __volatile__ ( "mov %%cr3, %0" : "=r" (scratch_phys) );
+	scratch_phys &= ~0xfffull;
 
-	// Offset-map this region into the higher half.
-	uint64_t free_region_start_virt =
-		free_region_start_phys + page_manager::page_offset;
+	// The tables are ident-mapped currently, so the cr3 physical address works. But let's
+	// get them into the offset-mapped area asap.
+	page_table *tables = reinterpret_cast<page_table *>(scratch_phys);
 
-	uint64_t free_next = free_region_start_virt;
-
-	// We'll need to copy any existing tables (except the PML4 which the
-	// bootloader gave us) into our reserved pages so we can edit them.
-	page_table_indices fr_idx{free_region_start_virt};
-
-	copy_new_table(&tables[0], fr_idx[0], &tables[1]);
-	copy_new_table(&tables[1], fr_idx[1], &tables[2]);
-	copy_new_table(&tables[2], fr_idx[2], &tables[3]);
-	page_in_ident(&tables[0], free_region_start_phys, free_region_start_virt, want_pages, nullptr);
-
-	// We now have pages starting at "free_next" to bootstrap ourselves. Start by
-	// taking inventory of free pages.
-	page_consumer allocator(free_next);
-	block_allocator block_slab(page_manager::page_size, allocator);
-	block_list used;
-	block_list free;
-
-	gather_block_lists(block_slab, used, free, memory_map, map_length, desc_length);
-	block_slab.allocate(); // Make sure we have extra
-
-	free_next = allocator.current;
-
-	// Now go back through these lists and consolidate
-	block_slab.append(page_block::consolidate(free));
-	block_slab.append(page_block::consolidate(used));
-
-	// Pull out the block that represents the bootstrap pages we've used
-	uint64_t used_bytes = free_next - free_region_start_virt;
-	uint64_t used_pages = used_bytes / page_manager::page_size;
-	uint64_t remaining_pages = want_pages - used_pages;
-
-	auto *removed = remove_block_for(free, free_region_start_phys,
-			used_pages, block_slab);
-
-	kassert(removed, "remove_block_for didn't find the bootstrap region.");
-	kassert(removed->physical_address == free_region_start_phys,
-			"remove_block_for found the wrong region.");
-
-	// Add it to the used list
-	removed->virtual_address = free_region_start_virt;
-	removed->flags = page_block_flags::used | page_block_flags::mapped;
-	used.sorted_insert(removed);
-
-	// Pull out the block that represents the rest
-	uint64_t free_next_phys = free_region_start_phys + used_bytes;
-
-	removed = remove_block_for(free, free_next_phys,
-			remaining_pages, block_slab);
-
-	kassert(removed, "remove_block_for didn't find the page table region.");
-	kassert(removed->physical_address == free_next_phys,
-			"remove_block_for found the wrong region.");
-
-	uint64_t pt_start_phys = removed->physical_address;
-	uint64_t pt_start_virt = removed->physical_address + page_manager::page_offset;
-
-	// Record that we're about to remap it into the page table address space
-	removed->virtual_address = pt_start_virt;
-	removed->flags = page_block_flags::used | page_block_flags::mapped;
-	used.sorted_insert(removed);
-
-	page_manager *pm = &g_page_manager;
-
-	// Actually remap them into page table space
-	pm->page_out(&tables[0], free_next, remaining_pages);
-
-	page_table_indices pg_idx{pt_start_virt};
-	copy_new_table(&tables[0], pg_idx[0], &tables[4]);
-	copy_new_table(&tables[4], pg_idx[1], &tables[5]);
-	copy_new_table(&tables[5], pg_idx[2], &tables[6]);
-
-	page_in_ident(&tables[0], pt_start_phys, pt_start_virt, remaining_pages, tables + 4);
+	page_table *id_pml4 = &tables[0];
+	page_table *id_pdp = &tables[1];
+	for (int i=0; i<512; ++i)
+		id_pdp->entries[i] = (static_cast<uintptr_t>(i) << 30) | 0x18b;
+	id_pml4->entries[511] = reinterpret_cast<uintptr_t>(id_pdp) | 0x10b;
 
 	// Make sure the page table is finished updating before we write to memory
 	__sync_synchronize();
 	io_wait();
 
+	// We now have pages starting at "scratch_virt" to bootstrap ourselves. Start by
+	// taking inventory of free pages.
+	uintptr_t scratch_virt = scratch_phys + page_offset;
+	uint64_t used_pages = 2; // starts with PML4 + offset PDP
+	page_consumer allocator(scratch_virt, scratch_pages, used_pages);
+
+	block_allocator block_slab(frame_size, allocator);
+	frame_block_list used;
+	frame_block_list free;
+
+	gather_block_lists(block_slab, used, free, memory_map, map_length, desc_length);
+	block_slab.allocate(); // Make sure we have extra
+
+	// Now go back through these lists and consolidate
+	block_slab.append(frame_block::consolidate(free));
+
+	region_allocator region_slab(frame_size, allocator);
+	region_slab.allocate(); // Allocate some buddy regions for the address_manager
+
+	kutil::address_manager *am =
+		new (&g_kernel_address_manager) kutil::address_manager(std::move(region_slab));
+
+	am->add_regions(kernel_offset, page_offset - kernel_offset);
+
 	// Finally, build an acutal set of kernel page tables that just contains
 	// what the kernel actually has mapped, but making everything writable
 	// (especially the page tables themselves)
-	page_table *pml4 = reinterpret_cast<page_table *>(pt_start_virt);
-	for (int i=0; i<512; ++i) pml4->entries[i] = 0;
+	page_table *pml4 = reinterpret_cast<page_table *>(allocator.get_page());
+	kutil::memset(pml4, 0, sizeof(page_table));
+	pml4->entries[511] = reinterpret_cast<uintptr_t>(id_pdp) | 0x10b;
+
+	kutil::frame_allocator *fa =
+		new (&g_frame_allocator) kutil::frame_allocator(std::move(block_slab));
+	page_manager *pm = new (&g_page_manager) page_manager(*fa, *am);
 
 	// Give the rest to the page_manager's cache for use in page_in
-	pm->free_table_pages(pml4 + 1, remaining_pages - 1);
+	pm->free_table_pages(
+			reinterpret_cast<void *>(allocator.current),
+			allocator.left());
 
 	for (auto *block : used) {
-		if (!block->has_flag(page_block_flags::mapped)) continue;
-		pm->page_in(pml4, block->physical_address, block->virtual_address, block->count);
+		uintptr_t virt_addr = 0;
+
+		switch (block->flags & frame_block_flags::map_mask) {
+			case frame_block_flags::map_ident:
+				virt_addr = block->address;
+				break;
+
+			case frame_block_flags::map_kernel:
+				virt_addr = block->address + kernel_offset;
+				if (block->flags && frame_block_flags::permanent)
+					am->mark_permanent(virt_addr, block->count * frame_size);
+				else
+					am->mark(virt_addr, block->count * frame_size);
+				break;
+
+			default:
+				break;
+		}
+
+		block->flags -= frame_block_flags::map_mask;
+		if (virt_addr)
+			pm->page_in(pml4, block->address, virt_addr, block->count);
 	}
 
+	fa->init(std::move(free), std::move(used));
+
 	// Put our new PML4 into CR3 to start using it
 	page_manager::set_pml4(pml4);
+	pm->m_kernel_pml4 = pml4;
 
-	// We now have all used memory mapped ourselves. Let the page_manager take
-	// over from here.
-	g_page_manager.init(
-			std::move(free),
-			std::move(used),
-			std::move(block_slab));
+	// Give the old pml4 back to the page_manager to recycle
+	pm->free_table_pages(reinterpret_cast<void *>(scratch_virt), 1);
+
+	// Set the heap manager
+	new (&g_kernel_heap_manager) kutil::heap_manager(mm_grow_callback);
+	kutil::setup::set_heap(&g_kernel_heap_manager);
 }

src/kernel/page_manager.cpp

@@ -1,26 +1,32 @@
 #include <algorithm>
 
 #include "kutil/assert.h"
-#include "kutil/memory_manager.h"
 #include "console.h"
 #include "log.h"
 #include "page_manager.h"
 
-page_manager g_page_manager;
-kutil::memory_manager g_kernel_memory_manager;
+using memory::frame_size;
+using memory::kernel_offset;
+using memory::page_offset;
+
+extern kutil::frame_allocator g_frame_allocator;
+extern kutil::address_manager g_kernel_address_manager;
+page_manager g_page_manager(
+	g_frame_allocator,
+	g_kernel_address_manager);
 
 
 static uintptr_t
 pt_to_phys(page_table *pt)
 {
-	return reinterpret_cast<uintptr_t>(pt) - page_manager::page_offset;
+	return reinterpret_cast<uintptr_t>(pt) - page_offset;
 }
 
 
 static page_table *
 pt_from_phys(uintptr_t p)
 {
-	return reinterpret_cast<page_table *>((p + page_manager::page_offset) & ~0xfffull);
+	return reinterpret_cast<page_table *>((p + page_offset) & ~0xfffull);
 }
 
 
@@ -31,142 +37,13 @@ struct free_page_header
 };
 
 
-void mm_grow_callback(void *next, size_t length)
+page_manager::page_manager(
+		kutil::frame_allocator &frames,
+		kutil::address_manager &addrs) :
+	m_page_cache(nullptr),
+	m_frames(frames),
+	m_addrs(addrs)
 {
-	kassert(length % page_manager::page_size == 0,
-			"Heap manager requested a fractional page.");
-
-	size_t pages = length / page_manager::page_size;
-	log::info(logs::memory, "Heap manager growing heap by %d pages.", pages);
-	g_page_manager.map_pages(reinterpret_cast<uintptr_t>(next), pages);
-}
-
-
-int
-page_block::compare(const page_block *rhs) const
-{
-	if (virtual_address < rhs->virtual_address)
-		return -1;
-	else if (virtual_address > rhs->virtual_address)
-		return 1;
-
-	if (physical_address < rhs->physical_address)
-		return -1;
-	else if (physical_address > rhs->physical_address)
-		return 1;
-
-	return 0;
-}
-
-page_block_list
-page_block::consolidate(page_block_list &list)
-{
-	page_block_list freed;
-
-	for (auto *cur : list) {
-		auto *next = cur->next();
-
-		while (next &&
-			cur->flags == next->flags &&
-			cur->physical_end() == next->physical_address &&
-			(!cur->has_flag(page_block_flags::mapped) ||
-			cur->virtual_end() == next->virtual_address)) {
-
-			cur->count += next->count;
-			list.remove(next);
-			freed.push_back(next);
-		}
-	}
-
-	return freed;
-}
-
-void
-page_block::dump(const page_block_list &list, const char *name, bool show_unmapped)
-{
-	log::info(logs::memory, "Block list %s:", name);
-
-	int count = 0;
-	for (auto *cur : list) {
-		count += 1;
-		if (!(show_unmapped || cur->has_flag(page_block_flags::mapped)))
-			continue;
-
-		if (cur->virtual_address) {
-			page_table_indices start{cur->virtual_address};
-			log::info(logs::memory, "  %016lx %08x [%6d] %016lx (%d,%d,%d,%d)",
-					cur->physical_address,
-					cur->flags,
-					cur->count,
-					cur->virtual_address,
-					start[0], start[1], start[2], start[3]);
-		} else {
-			page_table_indices start{cur->virtual_address};
-			log::info(logs::memory, "  %016lx %08x [%6d]",
-					cur->physical_address,
-					cur->flags,
-					cur->count);
-		}
-	}
-
-	log::info(logs::memory, "  Total: %d", count);
-}
-
-void
-page_block::zero()
-{
-	physical_address = 0;
-	virtual_address = 0;
-	count = 0;
-	flags = page_block_flags::free;
-}
-
-void
-page_block::copy(page_block *other)
-{
-	physical_address = other->physical_address;
-	virtual_address = other->virtual_address;
-	count = other->count;
-	flags = other->flags;
-}
-
-
-page_manager::page_manager() :
-	m_block_slab(page_size),
-	m_page_cache(nullptr)
-{
-	kassert(this == &g_page_manager, "Attempt to create another page_manager.");
-}
-
-void
-page_manager::init(
-	page_block_list free,
-	page_block_list used,
-	page_block_list cache)
-{
-	m_free.append(free);
-	m_used.append(used);
-	m_block_slab.append(cache);
-
-	consolidate_blocks();
-
-	// Initialize the kernel memory manager
-	uintptr_t end = 0;
-	for (auto *block : m_used) {
-		if (block->virtual_address &&
-			block->virtual_address < page_offset) {
-			end = block->virtual_end();
-		} else {
-			break;
-		}
-	}
-
-	new (&g_kernel_memory_manager) kutil::memory_manager(
-			reinterpret_cast<void *>(end),
-			mm_grow_callback);
-	kutil::setup::set_heap(&g_kernel_memory_manager);
-
-	m_kernel_pml4 = get_pml4();
 }
 
 page_table *
@@ -174,67 +51,107 @@ page_manager::create_process_map()
 {
 	page_table *table = get_table_page();
 
-	kutil::memset(table, 0, page_size);
+	kutil::memset(table, 0, frame_size);
 	table->entries[510] = m_kernel_pml4->entries[510];
 	table->entries[511] = m_kernel_pml4->entries[511];
 
 	// Create the initial user stack
 	map_pages(
-		initial_stack - (initial_stack_pages * page_size),
-		initial_stack_pages,
+		memory::initial_stack - (memory::initial_stack_pages * frame_size),
+		memory::initial_stack_pages,
 		true, // This is the ring3 stack, user = true
 		table);
 
 	return table;
 }
 
-void
-page_manager::delete_process_map(page_table *table)
+uintptr_t
+page_manager::copy_page(uintptr_t orig)
 {
-	// TODO: recurse table entries and mark them free
-	unmap_pages(table, 1);
+	uintptr_t virt = m_addrs.allocate(2 * frame_size);
+	uintptr_t copy = 0;
+
+	size_t n = m_frames.allocate(1, &copy);
+	kassert(n, "copy_page could not allocate page");
+
+	page_in(get_pml4(), orig, virt, 1);
+	page_in(get_pml4(), copy, virt + frame_size, 1);
+
+	kutil::memcpy(
+			reinterpret_cast<void *>(virt + frame_size),
+			reinterpret_cast<void *>(virt),
+			frame_size);
+
+	page_out(get_pml4(), virt, 2);
+
+	m_addrs.free(virt);
+	return copy;
+}
+
+page_table *
+page_manager::copy_table(page_table *from, page_table::level lvl)
+{
+	page_table *to = get_table_page();
+	log::debug(logs::paging, "Page manager copying level %d table at %016lx to %016lx.", lvl, from, to);
+
+	if (lvl == page_table::level::pml4) {
+		to->entries[510] = m_kernel_pml4->entries[510];
+		to->entries[511] = m_kernel_pml4->entries[511];
+	}
+
+	const int max =
+		lvl == page_table::level::pml4 ?
+			510 :
+			512;
+
+	unsigned pages_copied = 0;
+	for (int i = 0; i < max; ++i) {
+		if (!from->is_present(i)) {
+			to->entries[i] = 0;
+			continue;
+		}
+
+		bool is_page =
+			lvl == page_table::level::pt ||
+			from->is_large_page(lvl, i);
+
+		if (is_page) {
+			uint16_t flags = from->entries[i] &  0xfffull;
+			uintptr_t orig = from->entries[i] & ~0xfffull;
+			to->entries[i] = copy_page(orig) | flags;
+			pages_copied++;
+		} else {
+			uint16_t flags = 0;
+			page_table *next_from = from->get(i, &flags);
+			page_table *next_to = copy_table(next_from, page_table::deeper(lvl));
+			to->set(i, next_to, flags);
+		}
+	}
+
+	if (pages_copied)
+		log::debug(logs::paging, "   copied %3u pages", pages_copied);
+
+	return to;
+}
+
+void
+page_manager::delete_process_map(page_table *pml4)
+{
+	unmap_table(pml4, page_table::level::pml4, true);
 }
 
 void
 page_manager::map_offset_pointer(void **pointer, size_t length)
 {
-	uintptr_t *p = reinterpret_cast<uintptr_t *>(pointer);
-	uintptr_t v = *p + page_offset;
-	uintptr_t c = ((length - 1) / page_size) + 1;
-
-	// TODO: cleanly search/split this as a block out of used/free if possible
-	auto *block = m_block_slab.pop();
-
-	// TODO: page-align
-	block->physical_address = *p;
-	block->virtual_address = v;
-	block->count = c;
-	block->flags =
-		page_block_flags::used |
-		page_block_flags::mapped |
-		page_block_flags::mmio;
-
-	m_used.sorted_insert(block);
-
-	page_table *pml4 = get_pml4();
-	page_in(pml4, *p, v, c);
-	*p = v;
+	log::info(logs::paging, "Mapping offset pointer region at %016lx size 0x%lx", *pointer, length);
+	*pointer = kutil::offset_pointer(*pointer, page_offset);
 }
 
 void
-page_manager::dump_blocks(bool used_only)
+page_manager::dump_pml4(page_table *pml4, bool recurse)
 {
-	page_block::dump(m_used, "used", true);
-	if (!used_only)
-		page_block::dump(m_free, "free", true);
-}
-
-void
-page_manager::dump_pml4(page_table *pml4, int max_index)
-{
-	if (pml4 == nullptr)
-		pml4 = get_pml4();
-	pml4->dump(4, max_index);
+	if (pml4 == nullptr) pml4 = get_pml4();
+	pml4->dump(page_table::level::pml4, recurse);
 }
 
 page_table *
@@ -242,31 +159,21 @@ page_manager::get_table_page()
 {
 	if (!m_page_cache) {
 		uintptr_t phys = 0;
-		size_t n = pop_pages(32, &phys);
+		size_t n = m_frames.allocate(32, &phys); // TODO: indicate frames must be offset-mappable
 		uintptr_t virt = phys + page_offset;
 
-		auto *block = m_block_slab.pop();
-
-		block->physical_address = phys;
-		block->virtual_address = virt;
-		block->count = n;
-
-		m_used.sorted_insert(block);
-
-		page_in(get_pml4(), phys, virt, n);
-
 		m_page_cache = reinterpret_cast<free_page_header *>(virt);
 
 		// The last one needs to be null, so do n-1
-		uintptr_t end = virt + (n-1) * page_size;
+		uintptr_t end = virt + (n-1) * frame_size;
 		while (virt < end) {
 			reinterpret_cast<free_page_header *>(virt)->next =
-				reinterpret_cast<free_page_header *>(virt + page_size);
-			virt += page_size;
+				reinterpret_cast<free_page_header *>(virt + frame_size);
+			virt += frame_size;
 		}
 		reinterpret_cast<free_page_header *>(virt)->next = nullptr;
 
-		log::debug(logs::memory, "Mappd %d new page table pages at %lx", n, phys);
+		log::info(logs::paging, "Mappd %d new page table pages at %lx", n, phys);
 	}
 
 	free_page_header *page = m_page_cache;
@@ -279,7 +186,7 @@ page_manager::free_table_pages(void *pages, size_t count)
 {
 	uintptr_t start = reinterpret_cast<uintptr_t>(pages);
 	for (size_t i = 0; i < count; ++i) {
-		uintptr_t addr = start + (i * page_size);
+		uintptr_t addr = start + (i * frame_size);
 		free_page_header *header = reinterpret_cast<free_page_header *>(addr);
 		header->count = 1;
 		header->next = m_page_cache;
@@ -287,13 +194,6 @@ page_manager::free_table_pages(void *pages, size_t count)
 	}
 }
 
-void
-page_manager::consolidate_blocks()
-{
-	m_block_slab.append(page_block::consolidate(m_free));
-	m_block_slab.append(page_block::consolidate(m_used));
-}
-
 void *
 page_manager::map_pages(uintptr_t address, size_t count, bool user, page_table *pml4)
 {
@@ -301,129 +201,69 @@ page_manager::map_pages(uintptr_t address, size_t count, bool user, page_table *
 	if (!pml4) pml4 = get_pml4();
 
 	while (count) {
-		kassert(!m_free.empty(), "page_manager::map_pages ran out of free pages!");
-
 		uintptr_t phys = 0;
-		size_t n = pop_pages(count, &phys);
+		size_t n = m_frames.allocate(count, &phys);
 
-		auto *block = m_block_slab.pop();
-
-		block->physical_address = phys;
-		block->virtual_address = address;
-		block->count = n;
-		block->flags =
-				page_block_flags::used |
-				page_block_flags::mapped;
-
-		m_used.sorted_insert(block);
-
-		log::debug(logs::memory, "Paging in %d pages at p:%016lx to v:%016lx into %016lx table",
+		log::info(logs::paging, "Paging in %d pages at p:%016lx to v:%016lx into %016lx table",
 				n, phys, address, pml4);
 
 		page_in(pml4, phys, address, n, user);
 
-		address += n * page_size;
+		address += n * frame_size;
 		count -= n;
 	}
 
 	return ret;
 }
 
-void *
-page_manager::map_offset_pages(size_t count)
+void
+page_manager::unmap_table(page_table *table, page_table::level lvl, bool free)
 {
-	page_table *pml4 = get_pml4();
+	const int max =
+		lvl == page_table::level::pml4 ?
+			510 :
+			512;
 
-	for (auto *free : m_free) {
-		if (free->count < count) continue;
+	uintptr_t free_start = 0;
+	uintptr_t free_count = 0;
 
-		auto *used = m_block_slab.pop();
+	size_t size =
+		lvl == page_table::level::pdp ? (1<<30) :
+		lvl == page_table::level::pd  ? (1<<21) :
+		lvl == page_table::level::pt  ? (1<<12) :
+		0;
 
-		used->count = count;
-		used->physical_address = free->physical_address;
-		used->virtual_address = used->physical_address + page_offset;
-		used->flags =
-			page_block_flags::used |
-			page_block_flags::mapped;
+	for (int i = 0; i < max; ++i) {
+		if (!table->is_present(i)) continue;
 
-		m_used.sorted_insert(used);
+		bool is_page =
+			lvl == page_table::level::pt ||
+			table->is_large_page(lvl, i);
 
-		free->physical_address += count * page_size;
-		free->count -= count;
-
-		if (free->count == 0) {
-			m_free.remove(free);
-			free->zero();
-			m_block_slab.push(free);
+		if (is_page) {
+			uintptr_t frame = table->entries[i] & ~0xfffull;
+			if (!free_count || free_start != frame + free_count * size) {
+				if (free_count && free)
+					m_frames.free(free_start, free_count * size / frame_size);
+				free_start = frame;
+				free_count = 1;
+			}
+		} else {
+			page_table *next = table->get(i);
+			unmap_table(next, page_table::deeper(lvl), free);
 		}
-
-		log::debug(logs::memory, "Got request for offset map %016lx [%d]", used->virtual_address, count);
-		page_in(pml4, used->physical_address, used->virtual_address, count);
-		return reinterpret_cast<void *>(used->virtual_address);
 	}
 
-	return nullptr;
+	if (free_count && free)
+		m_frames.free(free_start, free_count * size / frame_size);
+	free_table_pages(table, 1);
 }
 
 void
-page_manager::unmap_pages(void* address, size_t count)
+page_manager::unmap_pages(void* address, size_t count, page_table *pml4)
 {
-	uintptr_t addr = reinterpret_cast<uintptr_t>(address);
-	size_t block_count = 0;
-
-	for (auto *block : m_used) {
-		if (!block->contains(addr)) continue;
-
-		size_t size = page_size * count;
-		uintptr_t end = addr + size;
-
-		size_t leading = addr - block->virtual_address;
-		size_t trailing =
-			end > block->virtual_end() ?
-			0 : (block->virtual_end() - end);
-
-		if (leading) {
-			size_t pages = leading / page_size;
-
-			auto *lead_block = m_block_slab.pop();
-
-			lead_block->copy(block);
-			lead_block->count = pages;
-
-			block->count -= pages;
-			block->physical_address += leading;
-			block->virtual_address += leading;
-
-			m_used.insert_before(block, lead_block);
-		}
-
-		if (trailing) {
-			size_t pages = trailing / page_size;
-
-			auto *trail_block = m_block_slab.pop();
-
-			trail_block->copy(block);
-			trail_block->count = pages;
-			trail_block->physical_address += size;
-			trail_block->virtual_address += size;
-
-			block->count -= pages;
-
-			m_used.insert_after(block, trail_block);
-		}
-
-		addr += block->count * page_size;
-
-		block->virtual_address = 0;
-		block->flags = block->flags &
-			~(page_block_flags::used | page_block_flags::mapped);
-
-		m_used.remove(block);
-		m_free.sorted_insert(block);
-		++block_count;
-	}
-
-	kassert(block_count, "Couldn't find existing mapped pages to unmap");
+	if (!pml4) pml4 = get_pml4();
+	page_out(pml4, reinterpret_cast<uintptr_t>(address), count, true);
 }
 
 void
@@ -437,8 +277,11 @@ page_manager::check_needs_page(page_table *table, unsigned index, bool user)
 }
 
 void
-page_manager::page_in(page_table *pml4, uintptr_t phys_addr, uintptr_t virt_addr, size_t count, bool user)
+page_manager::page_in(page_table *pml4, uintptr_t phys_addr, uintptr_t virt_addr, size_t count, bool user, bool large)
 {
+	log::debug(logs::paging, "page_in for table %016lx p:%016lx v:%016lx c:%4d u:%d l:%d",
+			pml4, phys_addr, virt_addr, count, user, large);
+
 	page_table_indices idx{virt_addr};
 	page_table *tables[4] = {pml4, nullptr, nullptr, nullptr};
 
@@ -455,12 +298,13 @@ page_manager::page_in(page_table *pml4, uintptr_t phys_addr, uintptr_t virt_addr
 			tables[2] = tables[1]->get(idx[1]);
 
 			for (; idx[2] < 512; idx[2] += 1, idx[3] = 0) {
-				if (idx[3] == 0 &&
+				if (large &&
+					idx[3] == 0 &&
 					count >= 512 &&
 					tables[2]->get(idx[2]) == nullptr) {
 					// Do a 2MiB page instead
 					tables[2]->entries[idx[2]] = phys_addr | flags | 0x80;
-					phys_addr += page_size * 512;
+					phys_addr += frame_size * 512;
 					count -= 512;
 					if (count == 0) return;
 					continue;
@@ -471,7 +315,7 @@ page_manager::page_in(page_table *pml4, uintptr_t phys_addr, uintptr_t virt_addr
 
 				for (; idx[3] < 512; idx[3] += 1) {
 					tables[3]->entries[idx[3]] = phys_addr | flags;
-					phys_addr += page_size;
+					phys_addr += frame_size;
 					if (--count == 0) return;
 				}
 			}
@@ -482,26 +326,41 @@ page_manager::page_in(page_table *pml4, uintptr_t phys_addr, uintptr_t virt_addr
 }
 
 void
-page_manager::page_out(page_table *pml4, uintptr_t virt_addr, size_t count)
+page_manager::page_out(page_table *pml4, uintptr_t virt_addr, size_t count, bool free)
 {
 	page_table_indices idx{virt_addr};
 	page_table *tables[4] = {pml4, nullptr, nullptr, nullptr};
 
+	bool found = false;
+	uintptr_t free_start = 0;
+	unsigned free_count = 0;
+
 	for (; idx[0] < 512; idx[0] += 1) {
-		tables[1] = reinterpret_cast<page_table *>(
-				tables[0]->entries[idx[0]] & ~0xfffull);
+		tables[1] = tables[0]->get(idx[0]);
 
 		for (; idx[1] < 512; idx[1] += 1) {
-			tables[2] = reinterpret_cast<page_table *>(
-					tables[1]->entries[idx[1]] & ~0xfffull);
+			tables[2] = tables[1]->get(idx[1]);
 
 			for (; idx[2] < 512; idx[2] += 1) {
-				tables[3] = reinterpret_cast<page_table *>(
-						tables[2]->entries[idx[2]] & ~0xfffull);
+				tables[3] = tables[2]->get(idx[2]);
 
 				for (; idx[3] < 512; idx[3] += 1) {
+					uintptr_t entry = tables[3]->entries[idx[3]] & ~0xfffull;
+					if (!found || entry != free_start + free_count * frame_size) {
+						if (found && free) m_frames.free(free_start, free_count);
+						free_start = tables[3]->entries[idx[3]] & ~0xfffull;
+						free_count = 1;
+						found = true;
+					} else {
+						free_count++;
+					}
+
 					tables[3]->entries[idx[3]] = 0;
-					if (--count == 0) return;
+
+					if (--count == 0) {
+						if (free) m_frames.free(free_start, free_count);
+						return;
+					}
 				}
 			}
 		}
@@ -510,62 +369,60 @@ page_manager::page_out(page_table *pml4, uintptr_t virt_addr, size_t count)
 	kassert(0, "Ran to end of page_out");
 }
 
-size_t
-page_manager::pop_pages(size_t count, uintptr_t *address)
-{
-	kassert(!m_free.empty(), "page_manager::pop_pages ran out of free pages!");
-
-	auto *first = m_free.front();
-
-	unsigned n = std::min(count, static_cast<size_t>(first->count));
-	*address = first->physical_address;
-
-	first->physical_address += n * page_size;
-	first->count -= n;
-	if (first->count == 0)
-		m_block_slab.push(m_free.pop_front());
-
-	return n;
-}
-
-
 void
-page_table::dump(int level, int max_index, uint64_t offset)
+page_table::dump(page_table::level lvl, bool recurse)
 {
 	console *cons = console::get();
 
-	cons->printf("\nLevel %d page table @ %lx (off %lx):\n", level, this, offset);
+	cons->printf("\nLevel %d page table @ %lx:\n", lvl, this);
 	for (int i=0; i<512; ++i) {
 		uint64_t ent = entries[i];
-		if (ent == 0) continue;
 
-		if ((ent & 0x1) == 0) {
+		if ((ent & 0x1) == 0)
 			cons->printf("  %3d: %016lx   NOT PRESENT\n", i, ent);
-			continue;
-		}
 
-		if ((level == 2 || level == 3) && (ent & 0x80) == 0x80) {
+		else if ((lvl == level::pdp || lvl == level::pd) && (ent & 0x80) == 0x80)
 			cons->printf("  %3d: %016lx -> Large page at    %016lx\n", i, ent, ent & ~0xfffull);
-			continue;
-		} else if (level == 1) {
+
+		else if (lvl == level::pt)
 			cons->printf("  %3d: %016lx -> Page at          %016lx\n", i, ent, ent & ~0xfffull);
-		} else {
+
+		else
 			cons->printf("  %3d: %016lx -> Level %d table at %016lx\n",
-					i, ent, level - 1, (ent & ~0xfffull) + offset);
-			continue;
-		}
+					i, ent, deeper(lvl), (ent & ~0xfffull) + page_offset);
 	}
 
-	if (--level > 0) {
-		for (int i=0; i<=max_index; ++i) {
-			uint64_t ent = entries[i];
-			if ((ent & 0x1) == 0) continue;
-			if ((ent & 0x80)) continue;
+	if (lvl != level::pt && recurse) {
+		for (int i=0; i<=512; ++i) {
+			if (is_large_page(lvl, i))
+				continue;
 
-			page_table *next = reinterpret_cast<page_table *>((ent & ~0xffful) + offset); 
-			next->dump(level, 511, offset);
+			page_table *next = get(i);
+			if (next)
+				next->dump(deeper(lvl), true);
 		}
 	}
 }
 
+page_table_indices::page_table_indices(uint64_t v) :
+	index{
+		(v >> 39) & 0x1ff,
+		(v >> 30) & 0x1ff,
+		(v >> 21) & 0x1ff,
+		(v >> 12) & 0x1ff }
+{}
 
+uintptr_t
+page_table_indices::addr() const
+{
+	return
+		(index[0] << 39) |
+		(index[1] << 30) |
+		(index[2] << 21) |
+		(index[3] << 12);
+}
+
+bool operator==(const page_table_indices &l, const page_table_indices &r)
+{
+	return l[0] == r[0] && l[1] == r[1] && l[2] == r[2] && l[3] == r[3];
+}

src/kernel/page_manager.h

@@ -5,44 +5,30 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include "kutil/address_manager.h"
 #include "kutil/enum_bitfields.h"
+#include "kutil/frame_allocator.h"
 #include "kutil/linked_list.h"
 #include "kutil/slab_allocator.h"
+#include "kernel_memory.h"
+#include "page_table.h"
 
-struct page_block;
-struct page_table;
 struct free_page_header;
 
-using page_block_list = kutil::linked_list<page_block>;
-using page_block_slab = kutil::slab_allocator<page_block>;
-
-/// Manager for allocation of physical pages.
+/// Manager for allocation and mapping of pages
 class page_manager
 {
 public:
-	/// Size of a single page.
-	static const size_t page_size = 0x1000;
-
-	/// Start of the higher half.
-	static const uintptr_t high_offset = 0xffffff0000000000;
-
-	/// Offset from physical where page tables are mapped.
-	static const uintptr_t page_offset = 0xffffff8000000000;
-
-	/// Initial process thread's stack address
-	static const uintptr_t initial_stack = 0x0000800000000000;
-
-	/// Initial process thread's stack size, in pages
-	static const unsigned initial_stack_pages = 1;
-
-	page_manager();
+	page_manager(
+		kutil::frame_allocator &frames,
+		kutil::address_manager &addrs);
 
 	/// Helper to get the number of pages needed for a given number of bytes.
 	/// \arg bytes  The number of bytes desired
 	/// \returns    The number of pages needed to contain the desired bytes
 	static inline size_t page_count(size_t bytes)
 	{
-		return (bytes - 1) / page_size + 1;
+		return (bytes - 1) / memory::frame_size + 1;
 	}
 
 	/// Helper to read the PML4 table from CR3.
@@ -51,14 +37,14 @@ public:
 	{
 		uintptr_t pml4 = 0;
 		__asm__ __volatile__ ( "mov %%cr3, %0" : "=r" (pml4) );
-		return reinterpret_cast<page_table *>((pml4 & ~0xfffull) + page_offset);
+		return reinterpret_cast<page_table *>((pml4 & ~0xfffull) + memory::page_offset);
 	}
 
 	/// Helper to set the PML4 table pointer in CR3.
 	/// \arg pml4  A pointer to the PML4 table to install.
 	static inline void set_pml4(page_table *pml4)
 	{
-		uintptr_t p = reinterpret_cast<uintptr_t>(pml4) - page_offset;
+		uintptr_t p = reinterpret_cast<uintptr_t>(pml4) - memory::page_offset;
 		__asm__ __volatile__ ( "mov %0, %%cr3" :: "r" (p & ~0xfffull) );
 	}
 
@@ -67,8 +53,16 @@ public:
 	/// \returns      A pointer to the PML4 table
 	page_table * create_process_map();
 
-	/// Deallocate a process' PML4 table.
-	void delete_process_map(page_table *table);
+	/// Deallocate a process' PML4 table and entries.
+	/// \arg pml4  The process' PML4 table
+	void delete_process_map(page_table *pml4);
+
+	/// Copy a process' memory mappings (and memory pages).
+	/// \arg from  Page table to copy from
+	/// \arg lvl   Level of the given tables (default is PML4)
+	/// \returns   The new page table
+	page_table * copy_table(page_table *from,
+			page_table::level lvl = page_table::level::pml4);
 
 	/// Allocate and map pages into virtual memory.
 	/// \arg address  The virtual address at which to map the pages
@@ -78,17 +72,11 @@ public:
 	/// \returns      A pointer to the start of the mapped region
 	void * map_pages(uintptr_t address, size_t count, bool user = false, page_table *pml4 = nullptr);
 
-	/// Allocate and map contiguous pages into virtual memory, with
-	/// a constant offset from their physical address.
-	/// \arg count    The number of pages to map
-	/// \returns      A pointer to the start of the mapped region, or
-	/// nullptr if no region could be found to fit the request.
-	void * map_offset_pages(size_t count);
-
-	/// Unmap existing pages from memory.
+	/// Unmap and free existing pages from memory.
 	/// \arg address  The virtual address of the memory to unmap
 	/// \arg count    The number of pages to unmap
-	void unmap_pages(void *address, size_t count);
+	/// \arg pml4     The pml4 to unmap from - null for the current one
+	void unmap_pages(void *address, size_t count, page_table *pml4 = nullptr);
 
 	/// Offset-map a pointer. No physical pages will be mapped.
 	/// \arg pointer  Pointer to a pointer to the memory area to be mapped
@@ -100,7 +88,7 @@ public:
 	/// \returns Physical address of the memory pointed to by p
 	inline uintptr_t offset_phys(void *p) const
 	{
-		return reinterpret_cast<uintptr_t>(kutil::offset_pointer(p, -page_offset));
+		return reinterpret_cast<uintptr_t>(kutil::offset_pointer(p, -memory::page_offset));
 	}
 
 	/// Get the virtual address of an offset-mapped physical address
@@ -108,39 +96,23 @@ public:
 	/// \returns Virtual address of the memory at address a
 	inline void * offset_virt(uintptr_t a) const
 	{
-		return kutil::offset_pointer(reinterpret_cast<void *>(a), page_offset);
+		return kutil::offset_pointer(reinterpret_cast<void *>(a), memory::page_offset);
 	}
 
-	/// Log the current free/used block lists.
-	/// \arg used_only  If true, skip printing free list. Default false.
-	void dump_blocks(bool used_only = false);
-
 	/// Dump the given or current PML4 to the console
-	/// \arg pml4       The page table to use, null for the current one
-	/// \arg max_index  The max index of pml4 to print
-	void dump_pml4(page_table *pml4 = nullptr, int max_index = 511);
+	/// \arg pml4     The page table to use, null for the current one
+	/// \arg recurse  Whether to print sub-tables
+	void dump_pml4(page_table *pml4 = nullptr, bool recurse = true);
 
 	/// Get the system page manager.
 	/// \returns  A pointer to the system page manager
 	static page_manager * get();
 
 private:
-	/// Set up the memory manager from bootstraped memory
-	void init(
-			page_block_list free,
-			page_block_list used,
-			page_block_list cache);
-
-	/// Initialize the virtual memory manager based on this object's state
-	void init_memory_manager();
-
-	/// Create a `page_block` struct or pull one from the cache.
-	/// \returns  An empty `page_block` struct
-	page_block * get_block();
-
-	/// Return a list of `page_block` structs to the cache.
-	/// \arg block   A list of `page_block` structs
-	void free_blocks(page_block *block);
+	/// Copy a physical page
+	/// \arg orig  Physical address of the page to copy
+	/// \returns   Physical address of the new page
+	uintptr_t copy_page(uintptr_t orig);
 
 	/// Allocate a page for a page table, or pull one from the cache
 	/// \returns  An empty page mapped in page space
@@ -151,10 +123,6 @@ private:
 	/// \arg count  Number of pages in the range
 	void free_table_pages(void *pages, size_t count);
 
-	/// Consolidate the free and used block lists. Return freed blocks
-	/// to the cache.
-	void consolidate_blocks();
-
 	/// Helper function to allocate a new page table. If table entry `i` in
 	/// table `base` is empty, allocate a new page table and point `base[i]` at
 	/// it.
@@ -168,40 +136,37 @@ private:
 	/// \arg phys_addr  The starting physical address of the pages to be mapped
 	/// \arg virt_addr  The starting virtual address ot the memory to be mapped
 	/// \arg count      The number of pages to map
-	/// \art user  True if this is a userspace mapping
+	/// \arg user       True if this is a userspace mapping
+	/// \arg large      Whether to allow large pages
 	void page_in(
 			page_table *pml4,
 			uintptr_t phys_addr,
 			uintptr_t virt_addr,
 			size_t count,
-			bool user = false);
+			bool user = false,
+			bool large = false);
 
 	/// Low-level routine for unmapping a number of pages from the given page table.
 	/// \arg pml4       The root page table for this mapping
 	/// \arg virt_addr  The starting virtual address ot the memory to be unmapped
 	/// \arg count      The number of pages to unmap
+	/// \arg free       Whether to return the pages to the frame allocator
 	void page_out(
 			page_table *pml4,
 			uintptr_t virt_addr,
-			size_t count);
+			size_t count,
+			bool free = false);
 
-	/// Get free pages from the free list. Only pages from the first free block
-	/// are returned, so the number may be less than requested, but they will
-	/// be contiguous. Pages will not be mapped into virtual memory.
-	/// \arg count    The maximum number of pages to get
-	/// \arg address  [out] The address of the first page
-	/// \returns      The number of pages retrieved
-	size_t pop_pages(size_t count, uintptr_t *address);
+	/// Low-level routine for unmapping an entire table of memory at once
+	void unmap_table(page_table *table, page_table::level lvl, bool free);
 
 	page_table *m_kernel_pml4; ///< The PML4 of just kernel pages
-
-	page_block_list m_free; ///< Free pages list
-	page_block_list m_used; ///< In-use pages list
-	page_block_slab m_block_slab; ///< page_block slab allocator
-
 	free_page_header *m_page_cache; ///< Cache of free pages to use for tables
 
-	friend void memory_initialize(const void *, size_t, size_t);
+	kutil::frame_allocator &m_frames;
+	kutil::address_manager &m_addrs;
+
+	friend void memory_initialize(uint16_t, const void *, size_t, size_t);
 	page_manager(const page_manager &) = delete;
 };
 
@@ -210,105 +175,6 @@ extern page_manager g_page_manager;
 
 inline page_manager * page_manager::get() { return &g_page_manager; }
 
-/// Flags used by `page_block`.
-enum class page_block_flags : uint32_t
-{
-	free         = 0x00000000,  ///< Not a flag, value for free memory
-	used         = 0x00000001,  ///< Memory is in use
-	mapped       = 0x00000002,  ///< Memory is mapped to virtual address
-
-	mmio         = 0x00000010,  ///< Memory is a MMIO region
-	nonvolatile  = 0x00000020,  ///< Memory is non-volatile storage
-
-	pending_free = 0x10000000,  ///< Memory should be freed
-	acpi_wait    = 0x40000000,  ///< Memory should be freed after ACPI init
-	permanent    = 0x80000000,  ///< Memory is permanently unusable
-
-	max_flags
-};
-IS_BITFIELD(page_block_flags);
-
-
-/// A block of contiguous pages. Each `page_block` represents contiguous
-/// physical pages with the same attributes. A `page_block *` is also a
-/// linked list of such structures.
-struct page_block
-{
-	uintptr_t physical_address;
-	uintptr_t virtual_address;
-	uint32_t count;
-	page_block_flags flags;
-
-	inline bool has_flag(page_block_flags f) const { return bitfield_has(flags, f); }
-	inline uintptr_t physical_end() const { return physical_address + (count * page_manager::page_size); }
-	inline uintptr_t virtual_end() const { return virtual_address + (count * page_manager::page_size); }
-
-	inline bool contains(uintptr_t vaddr) const { return vaddr >= virtual_address && vaddr < virtual_end(); }
-	inline bool contains_physical(uintptr_t addr) const { return addr >= physical_address && addr < physical_end(); }
-
-	/// Helper to zero out a block and optionally set the next pointer.
-	void zero();
-
-	/// Helper to copy a bock from another block
-	/// \arg other  The block to copy from
-	void copy(page_block *other);
-
-	/// Compare two blocks by address.
-	/// \arg rhs   The right-hand comparator
-	/// \returns   <0 if this is sorts earlier, >0 if this sorts later, 0 for equal
-	int compare(const page_block *rhs) const;
-
-	/// Traverse the list, joining adjacent blocks where possible.
-	/// \arg list  The list to consolidate
-	/// \returns   A linked list of freed page_block structures.
-	static page_block_list consolidate(page_block_list &list);
-
-	/// Traverse the list, printing debug info on this list.
-	/// \arg list  The list to print
-	/// \arg name  [optional] String to print as the name of this list
-	/// \arg show_permanent [optional] If false, hide unmapped blocks
-	static void dump(const page_block_list &list, const char *name = nullptr, bool show_unmapped = false);
-};
-
-
-
-/// Struct to allow easy accessing of a memory page being used as a page table.
-struct page_table
-{
-	using pm = page_manager;
-
-	uint64_t entries[512];
-
-	inline page_table * get(int i) const {
-		uint64_t entry = entries[i];
-		if ((entry & 0x1) == 0) return nullptr;
-		return reinterpret_cast<page_table *>((entry & ~0xfffull) + pm::page_offset);
-	}
-
-	inline void set(int i, page_table *p, uint16_t flags) {
-		entries[i] = (reinterpret_cast<uint64_t>(p) - pm::page_offset) | (flags & 0xfff);
-	}
-
-	void dump(int level = 4, int max_index = 511, uint64_t offset = page_manager::page_offset);
-};
-
-
-/// Helper struct for computing page table indices of a given address.
-struct page_table_indices
-{
-	page_table_indices(uint64_t v = 0) :
-		index{
-			(v >> 39) & 0x1ff,
-			(v >> 30) & 0x1ff,
-			(v >> 21) & 0x1ff,
-			(v >> 12) & 0x1ff }
-	{}
-
-	/// Get the index for a given level of page table.
-	uint64_t & operator[](size_t i) { return index[i]; }
-	uint64_t index[4]; ///< Indices for each level of tables.
-};
-
 
 /// Calculate a page-aligned address.
 /// \arg p    The address to align.
@@ -317,8 +183,8 @@ template <typename T> inline T
 page_align(T p)
 {
 	return reinterpret_cast<T>(
-		((reinterpret_cast<uintptr_t>(p) - 1) & ~(page_manager::page_size - 1))
-		+ page_manager::page_size);
+		((reinterpret_cast<uintptr_t>(p) - 1) & ~(memory::frame_size - 1))
+		+ memory::frame_size);
 }
 
 /// Calculate a page-table-aligned address. That is, an address that is
@@ -332,11 +198,5 @@ page_table_align(T p)
 }
 
 
-/// Calculate the number of pages needed for the give number of bytes.
-/// \arg n   Number of bytes
-/// \returns Number of pages
-inline size_t page_count(size_t n) { return ((n - 1) / page_manager::page_size) + 1; }
-
-
 /// Bootstrap the memory managers.
-void memory_initialize(const void *memory_map, size_t map_length, size_t desc_length);
+void memory_initialize(uint16_t scratch_pages, const void *memory_map, size_t map_length, size_t desc_length);

src/kernel/page_table.h

@@ -0,0 +1,63 @@
+#pragma once
+/// \file page_table.h
+/// Helper structures for dealing with page tables.
+
+#include <stdint.h>
+#include "kernel_memory.h"
+
+class page_manager;
+
+/// Struct to allow easy accessing of a memory page being used as a page table.
+struct page_table
+{
+	enum class level : unsigned { pml4, pdp, pd, pt };
+	inline static level deeper(level l) {
+		return static_cast<level>(static_cast<unsigned>(l) + 1);
+	}
+
+	uint64_t entries[512];
+
+	inline page_table * get(int i, uint16_t *flags = nullptr) const {
+		uint64_t entry = entries[i];
+		if ((entry & 0x1) == 0) return nullptr;
+		if (flags) *flags = entry & 0xfffull;
+		return reinterpret_cast<page_table *>((entry & ~0xfffull) + memory::page_offset);
+	}
+
+	inline void set(int i, page_table *p, uint16_t flags) {
+		entries[i] = (reinterpret_cast<uint64_t>(p) - memory::page_offset) | (flags & 0xfff);
+	}
+
+	inline bool is_present(int i) const { return (entries[i] & 0x1) == 0x1; }
+
+	inline bool is_large_page(level l, int i) const {
+		return
+			(l == level::pdp || l == level::pd) &&
+			(entries[i] & 0x80) == 0x80;
+	}
+
+	void dump(
+		level lvl = level::pml4,
+		bool recurse = true);
+};
+
+
+/// Helper struct for computing page table indices of a given address.
+struct page_table_indices
+{
+	page_table_indices(uint64_t v = 0);
+
+	uintptr_t addr() const;
+
+	inline operator uintptr_t() const { return addr(); }
+
+	/// Get the index for a given level of page table.
+	uint64_t & operator[](int i) { return index[i]; }
+	uint64_t operator[](int i) const { return index[i]; }
+	uint64_t & operator[](page_table::level i) { return index[static_cast<unsigned>(i)]; }
+	uint64_t operator[](page_table::level i) const { return index[static_cast<unsigned>(i)]; }
+	uint64_t index[4]; ///< Indices for each level of tables.
+};
+
+bool operator==(const page_table_indices &l, const page_table_indices &r);
+

src/kernel/process.cpp

@@ -1,28 +1,117 @@
+#include "cpu.h"
+#include "log.h"
 #include "process.h"
+#include "scheduler.h"
 
 
-void
+pid_t
+process::fork(uintptr_t in_rsp)
+{
+	auto &sched = scheduler::get();
+	auto *child = sched.create_process();
+	kassert(child, "process::fork() got null child");
+
+	child->ppid = pid;
+	child->flags =
+		process_flags::running |
+		process_flags::ready;
+
+	sched.m_runlists[child->priority].push_back(child);
+
+	child->rsp = in_rsp;
+
+	child->pml4 = page_manager::get()->copy_table(pml4);
+	kassert(child->pml4, "process::fork() got null pml4");
+
+	child->setup_kernel_stack(kernel_stack_size, kernel_stack);
+	child->rsp = child->kernel_stack + (in_rsp - kernel_stack);
+
+	log::debug(logs::task, "Copied process %d to %d, new PML4 %016lx.",
+			pid, child->pid, child->pml4);
+	log::debug(logs::task, "  copied stack %016lx to %016lx, rsp %016lx to %016lx.",
+			kernel_stack, child->kernel_stack, in_rsp, child->rsp);
+
+	// Add in the faked fork return value
+	cpu_state *regs = reinterpret_cast<cpu_state *>(child->rsp);
+	regs->rax = 0;
+
+	return child->pid;
+}
+
+void *
+process::setup_kernel_stack(size_t size, uintptr_t orig)
+{
+	void *stack0 = kutil::malloc(size);
+
+	if (orig)
+		kutil::memcpy(stack0, reinterpret_cast<void*>(orig), size);
+	else
+		kutil::memset(stack0, 0, size);
+
+	kernel_stack_size = size;
+	kernel_stack = reinterpret_cast<uintptr_t>(stack0);
+
+	return stack0;
+}
+
+bool
 process::wait_on_signal(uint64_t sigmask)
 {
 	waiting = process_wait::signal;
 	waiting_info = sigmask;
 	flags -= process_flags::ready;
+	return true;
 }
 
-void
+bool
 process::wait_on_child(uint32_t pid)
 {
 	waiting = process_wait::child;
 	waiting_info = pid;
 	flags -= process_flags::ready;
+	return true;
 }
 
-void
+bool
 process::wait_on_time(uint64_t time)
 {
 	waiting = process_wait::time;
 	waiting_info = time;
 	flags -= process_flags::ready;
+	return true;
+}
+
+bool
+process::wait_on_send(uint32_t target_id)
+{
+	scheduler &s = scheduler::get();
+	process *target = s.get_process_by_id(target_id);
+	if (!target) return false;
+
+	if (!target->wake_on_receive(this)) {
+		waiting = process_wait::send;
+		waiting_info = target_id;
+		flags -= process_flags::ready;
+	}
+
+	return true;
+}
+
+bool
+process::wait_on_receive(uint32_t source_id)
+{
+	scheduler &s = scheduler::get();
+	process *source = s.get_process_by_id(source_id);
+	if (!source) return false;
+
+	if (!source->wake_on_send(this)) {
+		waiting = process_wait::receive;
+		waiting_info = source_id;
+		flags -= process_flags::ready;
+		return true;
+	}
+
+	return false;
 }
 
 bool
@@ -49,7 +138,6 @@ process::wake_on_child(process *child)
 	return true;
 }
 
-
 bool
 process::wake_on_time(uint64_t now)
 {
@@ -62,3 +150,28 @@ process::wake_on_time(uint64_t now)
 	return true;
 }
 
+bool
+process::wake_on_send(process *target)
+{
+	if (waiting != process_wait::send ||
+		waiting_info != target->pid)
+		return false;
+
+	waiting = process_wait::none;
+	flags += process_flags::ready;
+	return true;
+}
+
+
+bool
+process::wake_on_receive(process *source)
+{
+	if (waiting != process_wait::receive ||
+		waiting_info != source->pid)
+		return false;
+
+	waiting = process_wait::none;
+	flags += process_flags::ready;
+	return true;
+}
+

src/kernel/process.h

@@ -7,6 +7,8 @@
 #include "kutil/linked_list.h"
 #include "page_manager.h"
 
+typedef uint32_t pid_t;
+
 
 enum class process_flags : uint32_t
 {
@@ -25,14 +27,16 @@ enum class process_wait : uint8_t
 	none,
 	signal,
 	child,
-	time
+	time,
+	send,
+	receive
 };
 
 /// A process
 struct process
 {
-	uint32_t pid;
-	uint32_t ppid;
+	pid_t pid;
+	pid_t ppid;
 
 	process_flags flags;
 
@@ -50,17 +54,41 @@ struct process
 	uintptr_t rsp;
 	page_table *pml4;
 
+	uintptr_t kernel_stack;
+	size_t kernel_stack_size;
+
+	/// Copy this process.
+	/// \arg in_rsp The RSP of the calling process
+	/// \returns    Returns the child's pid to the parent, and
+	///             0 to the child.
+	pid_t fork(uint64_t in_rsp);
+
 	/// Unready this process until it gets a signal
 	/// \arg sigmask  A bitfield of signals to wake on
-	void wait_on_signal(uint64_t sigmask);
+	/// \returns  Whether the process should be rescheduled
+	bool wait_on_signal(uint64_t sigmask);
 
 	/// Unready this process until a child exits
 	/// \arg pid  PID of the child to wait for, or 0 for any
-	void wait_on_child(uint32_t pid);
+	/// \returns  Whether the process should be rescheduled
+	bool wait_on_child(uint32_t pid);
 
 	/// Unready this process until after the given time
 	/// \arg time  The time after which to wake
-	void wait_on_time(uint64_t time);
+	/// \returns  Whether the process should be rescheduled
+	bool wait_on_time(uint64_t time);
+
+	/// Try to send to the target process, becoming unready if it
+	/// is not waiting on receive.
+	/// \arg target_id  The process to send to
+	/// \returns  Whether the process should be rescheduled
+	bool wait_on_send(uint32_t target_id);
+
+	/// Try to receive from one or more processes, becoming unready
+	/// if none of them are waiting on a send to this process.
+	/// \arg source_id  The process to receive from
+	/// \returns  Whether the process should be rescheduled
+	bool wait_on_receive(uint32_t source_id);
 
 	/// If this process is waiting on the given signal, wake it
 	/// \argument signal  The signal sent to the process
@@ -76,6 +104,27 @@ struct process
 	/// \argument now  The current time
 	/// \returns  True if this wake was handled
 	bool wake_on_time(uint64_t now);
+
+	/// If this process is waiting to send to this target, wake it
+	/// \argument target  The target process
+	/// \returns  True if this wake was handled
+	bool wake_on_send(process *target);
+
+	/// If this process is waiting to receieve from this source, wake it
+	/// \argument source  The process that is sending
+	/// \returns  True if this wake was handled
+	bool wake_on_receive(process *source);
+
+private:
+	friend class scheduler;
+
+	/// Set up a new kernel stack for this process, optionally copying the
+	/// given stack. Sets the kernel stack on the process object, but also
+	/// returns it.
+	/// \arg size  Size of the stack to allocate
+	/// \arg orig  Address of a stack to copy, or 0 for no copying.
+	/// \returns   The address of the new stack as a pointer
+	void * setup_kernel_stack(size_t size, uintptr_t orig);
 };
 
 using process_list = kutil::linked_list<process>;

src/kernel/scheduler.cpp

@@ -4,6 +4,7 @@
 #include "gdt.h"
 #include "interrupts.h"
 #include "io.h"
+#include "kernel_memory.h"
 #include "log.h"
 #include "msr.h"
 #include "page_manager.h"
@@ -12,6 +13,8 @@
 #include "elf/elf.h"
 #include "kutil/assert.h"
 
+using memory::initial_stack;
+
 scheduler scheduler::s_instance(nullptr);
 
 const int stack_size = 0x1000;
@@ -67,7 +70,7 @@ load_process(const void *image_start, size_t bytes, process *proc, cpu_state sta
 		if (header->type != elf::segment_type::load)
 			continue;
 
-		uintptr_t aligned = header->vaddr & ~(page_manager::page_size - 1);
+		uintptr_t aligned = header->vaddr & ~(memory::frame_size - 1);
 		size_t size = (header->vaddr + header->mem_size) - aligned;
 		size_t pages = page_manager::page_count(size);
 
@@ -80,7 +83,7 @@ load_process(const void *image_start, size_t bytes, process *proc, cpu_state sta
 		void *mapped = pager->map_pages(aligned, pages, true);
 		kassert(mapped, "Tried to map userspace pages and failed!");
 
-		kutil::memset(mapped, 0, pages * page_manager::page_size);
+		kutil::memset(mapped, 0, pages * memory::frame_size);
 	}
 
 	const unsigned section_count = image.section_count();
@@ -105,9 +108,20 @@ load_process(const void *image_start, size_t bytes, process *proc, cpu_state sta
 	log::debug(logs::task, "  Loaded! New process rip: %016lx", state.rip);
 }
 
-void
-scheduler::create_process(const char *name, const void *data, size_t size)
+process_node *
+scheduler::create_process()
 {
+	auto *proc = m_process_allocator.pop();
+	proc->pid = m_next_pid++;
+	proc->priority = default_priority;
+	return proc;
+}
+
+void
+scheduler::load_process(const char *name, const void *data, size_t size)
+{
+	auto *proc = create_process();
+
 	uint16_t kcs = (1 << 3) | 0; // Kernel CS is GDT entry 1, ring 0
 	uint16_t cs = (5 << 3) | 3;  // User CS is GDT entry 5, ring 3
 
@@ -118,8 +132,7 @@ scheduler::create_process(const char *name, const void *data, size_t size)
 	page_table *pml4 = page_manager::get()->create_process_map();
 
 	// Create a one-page kernel stack space
-	void *stack0 = kutil::malloc(stack_size);
-	kutil::memset(stack0, 0, stack_size);
+	void *stack0 = proc->setup_kernel_stack(stack_size, 0);
 
 	// Stack grows down, point to the end
 	void *sp0 = kutil::offset_pointer(stack0, stack_size);
@@ -132,7 +145,7 @@ scheduler::create_process(const char *name, const void *data, size_t size)
 	state->cs = cs;
 	state->rflags = rflags_int;
 	state->rip = 0; // to be filled by the loader
-	state->user_rsp = page_manager::initial_stack;
+	state->user_rsp = initial_stack;
 
 	// Next state in the stack is the loader's kernel stack. The scheduler will
 	// iret to this which will kick off the loading:
@@ -147,12 +160,6 @@ scheduler::create_process(const char *name, const void *data, size_t size)
 	loader_state->rax = reinterpret_cast<uint64_t>(data);
 	loader_state->rbx = size;
 
-	uint16_t pid = m_next_pid++;
-	auto *proc = m_process_allocator.pop();
-
-	proc->pid = pid;
-	proc->ppid = 0; // TODO
-	proc->priority = default_priority;
 	proc->rsp = reinterpret_cast<uintptr_t>(loader_state);
 	proc->pml4 = pml4;
 	proc->quanta = process_quanta;
@@ -216,13 +223,11 @@ void scheduler::prune(uint64_t now)
 	while (proc) {
 		bool ready = proc->flags && process_flags::ready;
 		ready |= proc->wake_on_time(now);
-		if (!ready) {
-			proc = proc->next();
-			continue;
-		}
 
 		auto *remove = proc;
 		proc = proc->next();
+		if (!ready) continue;
+
 		m_blocked.remove(remove);
 		m_runlists[remove->priority].push_front(remove);
 	}
@@ -234,15 +239,17 @@ scheduler::schedule(uintptr_t rsp0)
 
 	// TODO: lol a real clock
 	static uint64_t now = 0;
-	prune(++now);
 
 	m_current->rsp = rsp0;
 	m_runlists[m_current->priority].remove(m_current);
 
-	if (m_current->flags && process_flags::ready)
+	if (m_current->flags && process_flags::ready) {
 		m_runlists[m_current->priority].push_back(m_current);
-	else
+	} else {
 		m_blocked.push_back(m_current);
+	}
+
+	prune(++now);
 
 	uint8_t pri = 0;
 	while (m_runlists[pri].empty()) {

src/kernel/scheduler.h

@@ -10,7 +10,7 @@ class lapic;
 struct page_table;
 struct cpu_state;
 
-extern "C" uintptr_t isr_handler(uintptr_t, cpu_state);
+extern "C" uintptr_t isr_handler(uintptr_t, cpu_state*);
 
 
 /// The task scheduler
@@ -34,7 +34,7 @@ public:
 	/// \arg name  Name of the program image
 	/// \arg data  Pointer to the image data
 	/// \arg size  Size of the program image, in bytes
-	void create_process(const char *name, const void *data, size_t size);
+	void load_process(const char *name, const void *data, size_t size);
 
 	/// Start the scheduler working. This may involve starting
 	/// timer interrupts or other preemption methods.
@@ -59,8 +59,14 @@ public:
 	static scheduler & get() { return s_instance; }
 
 private:
-	friend uintptr_t syscall_dispatch(uintptr_t, const cpu_state &);
-	friend uintptr_t isr_handler(uintptr_t, cpu_state);
+	friend uintptr_t syscall_dispatch(uintptr_t, cpu_state &);
+	friend uintptr_t isr_handler(uintptr_t, cpu_state*);
+	friend class process;
+
+	/// Create a new process object. This process will have its pid
+	/// set but nothing else.
+	/// \returns  The new process object
+	process_node * create_process();
 
 	/// Handle a timer tick
 	/// \arg rsp0  The stack pointer of the current interrupt handler

src/kernel/syscall.cpp

@@ -35,25 +35,28 @@ syscall_enable()
 }
 
 uintptr_t
-syscall_dispatch(uintptr_t return_rsp, const cpu_state &regs)
+syscall_dispatch(uintptr_t return_rsp, cpu_state &regs)
 {
 	console *cons = console::get();
 	syscall call = static_cast<syscall>(regs.rax);
 
+	auto &s = scheduler::get();
+	auto *p = s.current();
+
 	switch (call) {
 	case syscall::noop:
 		break;
 
 	case syscall::debug:
 		cons->set_color(11);
-		cons->printf("\nReceived DEBUG syscall\n");
+		cons->printf("\nProcess %u: Received DEBUG syscall\n", p->pid);
 		cons->set_color();
 		print_regs(regs);
 		break;
 
 	case syscall::message:
 		cons->set_color(11);
-		cons->printf("\nReceived MESSAGE syscall\n");
+		cons->printf("\nProcess %u: Received MESSAGE syscall\n", p->pid);
 		cons->set_color();
 		break;
 
@@ -64,22 +67,67 @@ syscall_dispatch(uintptr_t return_rsp, const cpu_state &regs)
 			auto &s = scheduler::get();
 			auto *p = s.current();
 			p->wait_on_signal(-1ull);
-			cons->printf("\nReceived PAUSE syscall\n");
-			return_rsp = s.tick(return_rsp);
+			cons->printf("\nProcess %u: Received PAUSE syscall\n", p->pid);
 			cons->set_color();
+			return_rsp = s.schedule(return_rsp);
 		}
 		break;
 
 	case syscall::sleep:
 		{
 			cons->set_color(11);
-
-			auto &s = scheduler::get();
-			auto *p = s.current();
-			p->wait_on_time(regs.rbx);
-			cons->printf("\nReceived SLEEP syscall\n");
-			return_rsp = s.tick(return_rsp);
+			cons->printf("\nProcess %u: Received SLEEP syscall\n", p->pid);
+			cons->printf("Sleeping until %lu\n", regs.rbx);
 			cons->set_color();
+
+			p->wait_on_time(regs.rbx);
+			return_rsp = s.schedule(return_rsp);
+		}
+		break;
+
+	case syscall::getpid:
+		cons->set_color(11);
+		cons->printf("\nProcess %u: Received GETPID syscall\n", p->pid);
+		cons->set_color();
+		regs.rax = p->pid;
+		break;
+
+	case syscall::send:
+		{
+			uint32_t target = regs.rdi;
+
+			cons->set_color(11);
+			cons->printf("\nProcess %u: Received SEND syscall, target %u\n", p->pid, target);
+			cons->set_color();
+
+			if (p->wait_on_send(target))
+				return_rsp = s.schedule(return_rsp);
+		}
+		break;
+
+	case syscall::receive:
+		{
+			uint32_t source = regs.rdi;
+
+			cons->set_color(11);
+			cons->printf("\nProcess %u: Received RECEIVE syscall, source %u\n", p->pid, source);
+			cons->set_color();
+
+			if (p->wait_on_receive(source))
+				return_rsp = s.schedule(return_rsp);
+		}
+		break;
+
+	case syscall::fork:
+		{
+			cons->set_color(11);
+			cons->printf("\nProcess %u: Received FORK syscall\n", p->pid);
+			cons->set_color();
+
+			pid_t pid = p->fork(return_rsp);
+			if (pid == scheduler::get().current()->pid)
+				pid = 0;
+			regs.rax = pid;
 		}
 		break;
 

src/kernel/syscall.h

@@ -6,15 +6,19 @@ struct cpu_state;
 
 enum class syscall : uint64_t
 {
-	noop,
-	debug,
-	message,
-	pause,
-	sleep,
+	noop           = 0x0000,
+	debug          = 0x0001,
+	message        = 0x0002,
+	pause          = 0x0003,
+	sleep          = 0x0004,
+	getpid         = 0x0005,
+	send           = 0x0006,
+	receive        = 0x0007,
+	fork           = 0x0008,
 
 	last_syscall
 };
 
 void syscall_enable();
-uintptr_t syscall_dispatch(uintptr_t, const cpu_state &);
+uintptr_t syscall_dispatch(uintptr_t, cpu_state &);
 

src/libraries/kutil/frame_allocator.cpp

@@ -0,0 +1,159 @@
+#include <algorithm>
+
+#include "kutil/frame_allocator.h"
+
+namespace kutil {
+
+int
+frame_block::compare(const frame_block *rhs) const
+{
+	if (address < rhs->address)
+		return -1;
+	else if (address > rhs->address)
+		return 1;
+
+	return 0;
+}
+
+frame_block_list
+frame_block::consolidate(frame_block_list &list)
+{
+	frame_block_list freed;
+
+	for (auto *cur : list) {
+		auto *next = cur->next();
+
+		while ( next &&
+				cur->flags == next->flags &&
+				cur->end() == next->address) {
+			cur->count += next->count;
+			list.remove(next);
+			freed.push_back(next);
+		}
+	}
+
+	return freed;
+}
+
+void
+frame_block::zero()
+{
+	address = 0;
+	count = 0;
+	flags = frame_block_flags::none;
+}
+
+void
+frame_block::copy(frame_block *other)
+{
+	address = other->address;
+	count = other->count;
+	flags = other->flags;
+}
+
+
+frame_allocator::frame_allocator(
+	frame_block_list cache)
+{
+	m_block_slab.append(cache);
+}
+
+void
+frame_allocator::init(
+	frame_block_list free,
+	frame_block_list used)
+{
+	m_free.append(free);
+	m_used.append(used);
+}
+
+void
+frame_allocator::consolidate_blocks()
+{
+	m_block_slab.append(frame_block::consolidate(m_free));
+	m_block_slab.append(frame_block::consolidate(m_used));
+}
+
+size_t
+frame_allocator::allocate(size_t count, uintptr_t *address)
+{
+	kassert(!m_free.empty(), "frame_allocator::pop_frames ran out of free frames!");
+
+	auto *first = m_free.front();
+
+	unsigned n = std::min(count, static_cast<size_t>(first->count));
+	*address = first->address;
+
+	if (count >= first->count) {
+		m_free.remove(first);
+		m_used.sorted_insert(first);
+	} else {
+		auto *used = m_block_slab.pop();
+		used->copy(first);
+		used->count = n;
+		m_used.sorted_insert(used);
+
+		first->address += n * frame_size;
+		first->count -= n;
+	}
+
+	m_block_slab.append(frame_block::consolidate(m_used));
+	return n;
+}
+
+void
+frame_allocator::free(uintptr_t address, size_t count)
+{
+	size_t block_count = 0;
+
+	for (auto *block : m_used) {
+		if (!block->contains(address)) continue;
+
+		size_t size = frame_size * count;
+		uintptr_t end = address + size;
+
+		size_t leading = address - block->address;
+		size_t trailing =
+			end > block->end() ?
+			0 : (block->end() - end);
+
+		if (leading) {
+			size_t frames = leading / frame_size;
+
+			auto *lead_block = m_block_slab.pop();
+
+			lead_block->copy(block);
+			lead_block->count = frames;
+
+			block->count -= frames;
+			block->address += leading;
+
+			m_used.insert_before(block, lead_block);
+		}
+
+		if (trailing) {
+			size_t frames = trailing / frame_size;
+
+			auto *trail_block = m_block_slab.pop();
+
+			trail_block->copy(block);
+			trail_block->count = frames;
+			trail_block->address += size;
+			trail_block->address += size;
+
+			block->count -= frames;
+
+			m_used.insert_after(block, trail_block);
+		}
+
+		address += block->count * frame_size;
+
+		m_used.remove(block);
+		m_free.sorted_insert(block);
+		++block_count;
+	}
+
+	kassert(block_count, "Couldn't find existing allocated frames to free");
+}
+
+} // namespace kutil

src/libraries/kutil/heap_manager.cpp

@@ -1,12 +1,12 @@
 #include <stdint.h>
 #include "kutil/assert.h"
 #include "kutil/memory.h"
-#include "kutil/memory_manager.h"
+#include "kutil/heap_manager.h"
 
 namespace kutil {
 
 
-struct memory_manager::mem_header
+struct heap_manager::mem_header
 {
 	mem_header(mem_header *prev, mem_header *next, uint8_t size) :
 		m_prev(prev), m_next(next)
@@ -67,17 +67,12 @@ private:
 };
 
 
-memory_manager::memory_manager() :
-	m_start(nullptr),
-	m_length(0),
+heap_manager::heap_manager() :
 	m_grow(nullptr)
 {
-	kutil::memset(m_free, 0, sizeof(m_free));
 }
 
-memory_manager::memory_manager(void *start, grow_callback grow_cb) :
-	m_start(start),
-	m_length(0),
+heap_manager::heap_manager(grow_callback grow_cb) :
 	m_grow(grow_cb)
 {
 	kutil::memset(m_free, 0, sizeof(m_free));
@@ -85,7 +80,7 @@ memory_manager::memory_manager(void *start, grow_callback grow_cb) :
 }
 
 void *
-memory_manager::allocate(size_t length)
+heap_manager::allocate(size_t length)
 {
 	size_t total = length + sizeof(mem_header);
 	unsigned size = min_size;
@@ -98,13 +93,13 @@ memory_manager::allocate(size_t length)
 }
 
 void
-memory_manager::free(void *p)
+heap_manager::free(void *p)
 {
 	mem_header *header = reinterpret_cast<mem_header *>(p);
 	header -= 1; // p points after the header
 	header->set_used(false);
 
-	while (true) {
+	while (header->size() != max_size) {
 		mem_header *buddy = header->buddy();
 		if (buddy->used() || buddy->size() != header->size()) break;
 		buddy->remove();
@@ -120,23 +115,21 @@ memory_manager::free(void *p)
 }
 
 void
-memory_manager::grow_memory()
+heap_manager::grow_memory()
 {
 	size_t length = (1 << max_size);
 
-	void *next = kutil::offset_pointer(m_start, m_length);
 	kassert(m_grow, "Tried to grow heap without a growth callback");
-	m_grow(next, length);
+	void *next = m_grow(length);
 
 	mem_header *block = new (next) mem_header(nullptr, get_free(max_size), max_size);
 	get_free(max_size) = block;
 	if (block->next())
 		block->next()->set_prev(block);
-	m_length += length;
 }
 
 void
-memory_manager::ensure_block(unsigned size)
+heap_manager::ensure_block(unsigned size)
 {
 	if (get_free(size) != nullptr) return;
 	else if (size == max_size) {
@@ -154,8 +147,8 @@ memory_manager::ensure_block(unsigned size)
 	get_free(size) = orig;
 }
 
-memory_manager::mem_header *
-memory_manager::pop_free(unsigned size)
+heap_manager::mem_header *
+heap_manager::pop_free(unsigned size)
 {
 	ensure_block(size);
 	mem_header *block = get_free(size);

src/libraries/kutil/include/kutil/address_manager.h

@@ -0,0 +1,15 @@
+#pragma once
+/// \file address_manager.h
+/// The virtual memory address space manager
+
+#include "kutil/buddy_allocator.h"
+
+namespace kutil {
+
+using address_manager =
+	buddy_allocator<
+		12,  // Min allocation:  4KiB
+		36   // Max allocation: 64GiB
+	>;
+
+} //namespace kutil

src/libraries/kutil/include/kutil/buddy_allocator.h

@@ -0,0 +1,276 @@
+#pragma once
+/// \file buddy_allocator.h
+/// Helper base class for buddy allocators with external node storage.
+
+#include <stdint.h>
+#include "kutil/assert.h"
+#include "kutil/linked_list.h"
+#include "kutil/slab_allocator.h"
+
+namespace kutil {
+
+struct buddy_region;
+
+
+template<
+	unsigned size_min,
+	unsigned size_max,
+	typename region_type = buddy_region>
+class buddy_allocator
+{
+public:
+	using region_node = list_node<region_type>;
+	using region_list = linked_list<region_type>;
+
+	static const size_t min_alloc = (1 << size_min);
+	static const size_t max_alloc = (1 << size_max);
+
+	/// Constructor.
+	buddy_allocator() {}
+
+	/// Constructor with an initial cache of region structs from bootstrapped
+	/// memory.
+	/// \arg cache  List of pre-allocated ununused region_type structures
+	buddy_allocator(region_list cache)
+	{
+		m_alloc.append(cache);
+	}
+
+	/// Add address space to be managed.
+	/// \arg start   Initial address in the managed range
+	/// \arg length  Size of the managed range, in bytes
+	void add_regions(uintptr_t start, size_t length)
+	{
+		uintptr_t p = start;
+		unsigned size = size_max;
+
+		while (size >= size_min) {
+			size_t chunk_size = 1ull << size;
+
+			while (p + chunk_size <= start + length) {
+				region_node *r = m_alloc.pop();
+				r->size = size_max;
+				r->address = p;
+
+				free_bucket(size).sorted_insert(r);
+				p += chunk_size;
+			}
+
+			size--;
+		}
+	}
+
+	/// Allocate address space from the managed area.
+	/// \arg length  The amount of memory to allocate, in bytes
+	/// \returns     The address of the start of the allocated area, or 0 on
+	///              failure
+	uintptr_t allocate(size_t length)
+	{
+		unsigned size = size_min;
+		while ((1ull << size) < length)
+			if (size++ == size_max)
+				return 0;
+
+		unsigned request = size;
+		while (free_bucket(request).empty())
+			if (request++ == size_max)
+				return 0;
+
+		region_node *r = nullptr;
+		while (request > size) {
+			r = free_bucket(request).pop_front();
+			region_node *n = split(r);
+			request--;
+
+			free_bucket(request).sorted_insert(n);
+			if (request != size)
+				free_bucket(request).sorted_insert(r);
+		}
+
+		if (r == nullptr) r = free_bucket(size).pop_front();
+		used_bucket(size).sorted_insert(r);
+
+		return r->address;
+	}
+
+	/// Mark a region as allocated.
+	/// \arg start   The start of the region
+	/// \arg length  The size of the region, in bytes
+	/// \returns     The address of the start of the allocated area, or 0 on
+	///              failure. This may be less than `start`.
+	uintptr_t mark(uintptr_t start, size_t length)
+	{
+		uintptr_t end = start + length;
+		region_node *found = nullptr;
+
+		for (unsigned i = size_max; i >= size_min && !found; --i) {
+			for (auto *r : free_bucket(i)) {
+				if (start >= r->address && end <= r->end()) {
+					free_bucket(i).remove(r);
+					found = r;
+					break;
+				}
+			}
+		}
+
+		kassert(found, "buddy_allocator::mark called for unknown region");
+		if (!found)
+			return 0;
+
+		found = maybe_split(found, start, end);
+		used_bucket(found->size).sorted_insert(found);
+		return found->address;
+	}
+
+	/// Mark a region as permanently allocated. The region is not returned,
+	/// as the block can never be freed. This may remove several smaller
+	/// regions in order to more closely fit the region described.
+	/// \arg start   The start of the region
+	/// \arg length  The size of the region, in bytes
+	/// \returns     The address of the start of the allocated area, or 0 on
+	///              failure. This may be less than `start`.
+	void mark_permanent(uintptr_t start, size_t length)
+	{
+		uintptr_t end = start + length;
+		for (unsigned i = size_max; i >= size_min; --i) {
+			for (auto *r : free_bucket(i)) {
+				if (start >= r->address && end <= r->end()) {
+					free_bucket(i).remove(r);
+					delete_region(r, start, end);
+					return;
+				}
+			}
+		}
+
+		kassert(false, "buddy_allocator::mark_permanent called for unknown region");
+	}
+
+	/// Free a previous allocation.
+	/// \arg p  An address previously retuned by allocate()
+	void free(uintptr_t p)
+	{
+		region_node *found = find(p, true);
+
+		kassert(found, "buddy_allocator::free called for unknown region");
+		if (!found)
+			return;
+
+		used_bucket(found->size).remove(found);
+		free_bucket(found->size).sorted_insert(found);
+
+		while (auto *bud = get_buddy(found)) {
+			region_node *eld = found->elder() ? found : bud;
+			region_node *other = found->elder() ? bud : found;
+
+			free_bucket(other->size).remove(other);
+			m_alloc.push(other);
+
+			free_bucket(eld->size).remove(eld);
+			eld->size++;
+			free_bucket(eld->size).sorted_insert(eld);
+
+			found = eld;
+		}
+	}
+
+protected:
+	/// Split a region of the given size into two smaller regions, returning
+	/// the new latter half
+	region_node * split(region_node *reg)
+	{
+		region_node *other = m_alloc.pop();
+
+		other->size = --reg->size;
+		other->address = reg->address + (1ull << reg->size);
+		return other;
+	}
+
+	/// Find a node with the given address
+	region_node * find(uintptr_t p, bool used = true)
+	{
+		for (unsigned size = size_max; size >= size_min; --size) {
+			auto &list = used ? used_bucket(size) : free_bucket(size);
+			for (auto *f : list) {
+				if (f->address < p) continue;
+				if (f->address == p) return f;
+				break;
+			}
+		}
+		return nullptr;
+	}
+
+	/// Helper to get the buddy for a node, if it's adjacent
+	region_node * get_buddy(region_node *r)
+	{
+		region_node *b = r->elder() ? r->next() : r->prev();
+		if (b && b->address == r->buddy())
+			return b;
+		return nullptr;
+	}
+
+	region_node * maybe_split(region_node *reg, uintptr_t start, uintptr_t end)
+	{
+		while (reg->size > size_min) {
+			// Split if the request fits in the second half
+			if (start >= reg->half()) {
+				region_node *other = split(reg);
+				free_bucket(reg->size).sorted_insert(reg);
+				reg = other;
+			}
+
+			// Split if the request fits in the first half
+			else if (end <= reg->half()) {
+				region_node *other = split(reg);
+				free_bucket(other->size).sorted_insert(other);
+			}
+
+			// If neither, we've split as much as possible
+			else break;
+		}
+
+		return reg;
+	}
+
+	void delete_region(region_node *reg, uintptr_t start, uintptr_t end)
+	{
+		reg = maybe_split(reg, start, end);
+
+		size_t leading = start - reg->address;
+		size_t trailing = reg->end() - end;
+		if (leading > (1<<size_min) || trailing > (1<<size_min)) {
+			region_node *tail = split(reg);
+			delete_region(reg, start, reg->end());
+			delete_region(tail, tail->address, end);
+		} else {
+			m_alloc.push(reg);
+		}
+	}
+
+	region_list & used_bucket(unsigned size) { return m_used[size - size_min]; }
+	region_list & free_bucket(unsigned size) { return m_free[size - size_min]; }
+
+	static const unsigned buckets = (size_max - size_min + 1);
+
+	region_list m_free[buckets];
+	region_list m_used[buckets];
+	slab_allocator<region_type> m_alloc;
+};
+
+
+struct buddy_region
+{
+	inline int compare(const buddy_region *o) const {
+		return address > o->address ? 1 : \
+			address < o->address ? -1 : 0;
+	}
+
+	inline uintptr_t end() const { return address + (1ull << size); }
+	inline uintptr_t half() const { return address + (1ull << (size - 1)); }
+	inline uintptr_t buddy() const { return address ^ (1ull << size); }
+	inline bool elder() const { return address < buddy(); }
+
+	uint16_t size;
+	uintptr_t address;
+};
+
+} // namespace kutil

src/libraries/kutil/include/kutil/frame_allocator.h

@@ -0,0 +1,122 @@
+#pragma once
+/// \file frame_allocator.h
+/// Allocator for physical memory frames
+
+#include <stdint.h>
+
+#include "kutil/enum_bitfields.h"
+#include "kutil/linked_list.h"
+#include "kutil/slab_allocator.h"
+
+namespace kutil {
+
+struct frame_block;
+using frame_block_list = kutil::linked_list<frame_block>;
+using frame_block_slab = kutil::slab_allocator<frame_block>;
+
+/// Allocator for physical memory frames
+class frame_allocator
+{
+public:
+	/// Size of a single page frame.
+	static const size_t frame_size = 0x1000;
+
+	/// Default constructor
+	frame_allocator() = default;
+
+	/// Constructor with a provided initial frame_block cache.
+	/// \arg cache  List of pre-allocated but unused frame_block structures
+	frame_allocator(frame_block_list cache);
+
+	/// Initialize the frame allocator from bootstraped memory.
+	/// \arg free   List of free blocks
+	/// \arg used   List of currently used blocks
+	void init(
+			frame_block_list free,
+			frame_block_list used);
+
+	/// Get free frames from the free list. Only frames from the first free block
+	/// are returned, so the number may be less than requested, but they will
+	/// be contiguous.
+	/// \arg count    The maximum number of frames to get
+	/// \arg address  [out] The physical address of the first frame
+	/// \returns      The number of frames retrieved
+	size_t allocate(size_t count, uintptr_t *address);
+
+	/// Free previously allocated frames.
+	/// \arg address  The physical address of the first frame to free
+	/// \arg count    The number of frames to be freed
+	void free(uintptr_t address, size_t count);
+
+	/// Consolidate the free and used block lists. Return freed blocks
+	/// to the cache.
+	void consolidate_blocks();
+
+private:
+	frame_block_list m_free; ///< Free frames list
+	frame_block_list m_used; ///< In-use frames list
+	frame_block_slab m_block_slab; ///< frame_block slab allocator
+
+	frame_allocator(const frame_allocator &) = delete;
+};
+
+
+/// Flags used by `frame_block`.
+enum class frame_block_flags : uint32_t
+{
+	none         = 0x0000,
+
+	mmio         = 0x0001,  ///< Memory is a MMIO region
+	nonvolatile  = 0x0002,  ///< Memory is non-volatile storage
+
+	pending_free = 0x0020,  ///< Memory should be freed
+	acpi_wait    = 0x0040,  ///< Memory should be freed after ACPI init
+	permanent    = 0x0080,  ///< Memory is permanently unusable
+
+	// The following are used only during the memory bootstraping
+	// process, and tell the page manager where to initially map
+	// the given block.
+	map_ident    = 0x0100,  ///< Identity map
+	map_kernel   = 0x0200,  ///< Map into normal kernel space
+	map_offset   = 0x0400,  ///< Map into offset kernel space
+	map_mask     = 0x0700,  ///< Mask of all map_* values
+};
+
+} // namespace kutil
+
+IS_BITFIELD(kutil::frame_block_flags);
+
+namespace kutil {
+
+/// A block of contiguous frames. Each `frame_block` represents contiguous
+/// physical frames with the same attributes.
+struct frame_block
+{
+	uintptr_t address;
+	uint32_t count;
+	frame_block_flags flags;
+
+	inline bool has_flag(frame_block_flags f) const { return bitfield_has(flags, f); }
+	inline uintptr_t end() const { return address + (count * frame_allocator::frame_size); }
+	inline bool contains(uintptr_t addr) const { return addr >= address && addr < end(); }
+
+	/// Helper to zero out a block and optionally set the next pointer.
+	void zero();
+
+	/// Helper to copy a bock from another block
+	/// \arg other  The block to copy from
+	void copy(frame_block *other);
+
+	/// Compare two blocks by address.
+	/// \arg rhs   The right-hand comparator
+	/// \returns   <0 if this is sorts earlier, >0 if this sorts later, 0 for equal
+	int compare(const frame_block *rhs) const;
+
+	/// Traverse the list, joining adjacent blocks where possible.
+	/// \arg list  The list to consolidate
+	/// \returns   A linked list of freed frame_block structures.
+	static frame_block_list consolidate(frame_block_list &list);
+};
+
+} // namespace kutil
+

src/libraries/kutil/include/kutil/heap_manager.h

@@ -1,5 +1,5 @@
 #pragma once
-/// \file memory_manager.h
+/// \file heap_manager.h
 /// A buddy allocator and related definitions.
 
 #include <stddef.h>
@@ -7,19 +7,20 @@
 namespace kutil {
 
 
-/// Manager for allocation of virtual memory.
-class memory_manager
+/// Manager for allocation of heap memory.
+class heap_manager
 {
 public:
-	using grow_callback = void (*)(void *start, size_t length);
+	/// Callback signature for growth function. Memory returned does not need
+	/// to be contiguous, but needs to be alined to the length requested.
+	using grow_callback = void * (*)(size_t length);
 
 	/// Default constructor. Creates an invalid manager.
-	memory_manager();
+	heap_manager();
 
 	/// Constructor.
-	/// \arg start    Pointer to the start of the heap to be managed
 	/// \arg grow_cb  Function pointer to grow the heap size
-	memory_manager(void *start, grow_callback grow_cb);
+	heap_manager(grow_callback grow_cb);
 
 	/// Allocate memory from the area managed.
 	/// \arg length  The amount of memory to allocate, in bytes
@@ -57,13 +58,11 @@ protected:
 	/// \returns    A detached block of the given size
 	mem_header * pop_free(unsigned size);
 
-	mem_header *m_free[max_size - min_size];
-	void *m_start;
-	size_t m_length;
+	mem_header *m_free[max_size - min_size + 1];
 
 	grow_callback m_grow;
 
-	memory_manager(const memory_manager &) = delete;
+	heap_manager(const heap_manager &) = delete;
 };
 
 } // namespace kutil

src/libraries/kutil/include/kutil/linked_list.h

@@ -189,10 +189,7 @@ public:
 	item_type * pop_front()
 	{
 		item_type *item = m_head;
-		if (m_head) {
-			m_head = item->m_next;
-			item->m_next = nullptr;
-		}
+		remove(item);
 		return item;
 	}
 
@@ -201,10 +198,7 @@ public:
 	item_type * pop_back()
 	{
 		item_type *item = m_tail;
-		if (m_tail) {
-			m_tail = item->m_prev;
-			item->m_prev = nullptr;
-		}
+		remove(item);
 		return item;
 	}
 

src/libraries/kutil/include/kutil/memory.h

@@ -68,13 +68,13 @@ inline T* mask_pointer(T *p, uintptr_t mask)
 uint8_t checksum(const void *p, size_t len, size_t off = 0);
 
 
-class memory_manager;
+class heap_manager;
 
 namespace setup {
 
 /// Set the heap that malloc() / free() will use.
 /// \arg mm  The heap manager for the heap to use.
-void set_heap(memory_manager *mm);
+void set_heap(heap_manager *mm);
 
 } // namespace kutil::setup
 } // namespace kutil

src/libraries/kutil/include/kutil/slab_allocator.h

@@ -1,6 +1,7 @@
 #pragma once
 /// \file slab_allocator.h
 /// A slab allocator and related definitions
+#include "kutil/assert.h"
 #include "kutil/linked_list.h"
 #include "kutil/memory.h"
 
@@ -29,6 +30,7 @@ public:
 	inline item_type * pop()
 	{
 		if (this->empty()) allocate();
+		kassert(!this->empty(), "Slab allocator is empty after allocate()");
 		item_type *item = this->pop_front();
 		kutil::memset(item, 0, sizeof(item_type));
 		return item;

src/libraries/kutil/memory.cpp

@@ -1,5 +1,5 @@
 #include "kutil/memory.h"
-#include "kutil/memory_manager.h"
+#include "kutil/heap_manager.h"
 
 namespace std {
 	enum class __attribute__ ((__type_visibility("default"))) align_val_t : size_t { };
@@ -17,10 +17,10 @@ namespace kutil {
 
 namespace setup {
 
-static memory_manager *heap_memory_manager;
+static heap_manager *heap_memory_manager;
 
 void
-set_heap(memory_manager *mm)
+set_heap(heap_manager *mm)
 {
 	setup::heap_memory_manager = mm;
 }

src/tests/address_manager.cpp

@@ -0,0 +1,68 @@
+#include <chrono>
+#include <random>
+#include <vector>
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdint.h>
+
+#include "kutil/address_manager.h"
+#include "catch.hpp"
+
+using namespace kutil;
+
+static const size_t max_block = 1ull << 36;
+static const size_t start = max_block;
+static const size_t GB = 1ull << 30;
+
+TEST_CASE( "Buddy addresses tests", "[address buddy]" )
+{
+	address_manager am;
+	am.add_regions(start, max_block * 2);
+
+	// Blocks should be:
+	// 36: 0-64G, 64-128G
+
+	uintptr_t a = am.allocate(0x400); // under min
+	uintptr_t b = am.allocate(0x400);
+	CHECK( b == a + address_manager::min_alloc);
+
+	am.free(a);
+	am.free(b);
+
+	// Should be back to
+	// 36: 0-64G, 64-128G
+
+	a = am.allocate(max_block);
+	CHECK(a == start);
+	am.free(a);
+
+	// Should be back to
+	// 36: 0-64G, 64-128G
+
+	// This should allocate the 66-68G block, not the
+	// 66-67G block.
+	a = am.mark( start + 66 * GB + 0x1000, GB );
+	CHECK( a == start + 66 * GB );
+
+	// Free blocks should be:
+	// 36: 0-64G
+	// 35: 96-128G
+	// 34: 80-96G
+	// 33: 72-80G
+	// 32: 68-72G
+	// 31: 64-66G
+
+	// This should cause a split of the one 31 block, NOT
+	// be a leftover of the above mark.
+	b = am.allocate(GB);
+	CHECK( b == start + 64 * GB );
+	am.free(b);
+	am.free(a);
+
+	// Should be back to
+	// 36: 0-64G, 64-128G
+	a = am.allocate(max_block);
+	b = am.allocate(max_block);
+	CHECK( b == start + max_block );
+	CHECK( a == start );
+}

src/tests/frame_allocator.cpp

@@ -0,0 +1,45 @@
+#include "kutil/frame_allocator.h"
+#include "catch.hpp"
+
+using namespace kutil;
+
+TEST_CASE( "Frame allocator tests", "[memory frame]" )
+{
+	frame_block_list free;
+	frame_block_list used;
+	frame_block_list cache;
+
+	auto *f = new frame_block_list::item_type;
+	f->address = 0x1000;
+	f->count = 1;
+	f->flags = kutil::frame_block_flags::none;
+	free.sorted_insert(f);
+
+	auto *g = new frame_block_list::item_type;
+	g->address = 0x2000;
+	g->count = 1;
+	g->flags = kutil::frame_block_flags::none;
+	free.sorted_insert(g);
+
+	frame_allocator fa(std::move(cache));
+	fa.init(std::move(free), std::move(used));
+
+	fa.consolidate_blocks();
+
+	uintptr_t a = 0;
+	size_t c = fa.allocate(2, &a);
+	CHECK( a == 0x1000 );
+	CHECK( c == 2 );
+
+	fa.free(a, 2);
+	a = 0;
+
+	fa.consolidate_blocks();
+
+	c = fa.allocate(2, &a);
+	CHECK( a == 0x1000 );
+	CHECK( c == 2 );
+
+	delete f;
+	delete g;
+}

src/tests/heap_manager.cpp

@@ -0,0 +1,191 @@
+#include <chrono>
+#include <random>
+#include <vector>
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdint.h>
+
+#include "kutil/memory.h"
+#include "kutil/heap_manager.h"
+#include "catch.hpp"
+
+using namespace kutil;
+
+static std::vector<void *> memory;
+
+static size_t total_alloc_size = 0;
+static size_t total_alloc_calls = 0;
+
+const size_t hs = 0x10; // header size
+const size_t max_block = 1 << 16;
+
+std::vector<size_t> sizes = {
+	16000, 8000, 4000, 4000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 150,
+	150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 48, 48, 48, 13 };
+
+void * grow_callback(size_t length)
+{
+	total_alloc_calls += 1;
+	total_alloc_size += length;
+
+	void *p = aligned_alloc(max_block, length * 2);
+	memory.push_back(p);
+	return p;
+}
+
+void free_memory()
+{
+	for (void *p : memory) ::free(p);
+	memory.clear();
+	total_alloc_size = 0;
+	total_alloc_calls = 0;
+}
+
+TEST_CASE( "Buddy blocks tests", "[memory buddy]" )
+{
+	using clock = std::chrono::system_clock;
+	unsigned seed = clock::now().time_since_epoch().count();
+	std::default_random_engine rng(seed);
+
+	heap_manager mm(grow_callback);
+
+	// The ctor should have allocated an initial block
+	CHECK( total_alloc_size == max_block );
+	CHECK( total_alloc_calls == 1 );
+
+	// Blocks should be:
+	// 16: 0-64K
+
+	std::vector<void *> allocs(6);
+	for (int i = 0; i < 6; ++i)
+		allocs[i] = mm.allocate(150); // size 8
+
+	// Should not have grown
+	CHECK( total_alloc_size == max_block );
+	CHECK( total_alloc_calls == 1 );
+	CHECK( memory[0] != nullptr );
+
+	// Blocks should be:
+	// 16: [0-64K]
+	// 15: [0-32K], 32K-64K
+	// 14: [0-16K], 16K-32K
+	// 13: [0-8K], 8K-16K
+	// 12: [0-4K], 4K-8K
+	// 11: [0-2K], 2K-4K
+	// 10: [0-1K, 1-2K]
+	//  9: [0, 512, 1024], 1536
+	//  8: [0, 256, 512, 768, 1024, 1280]
+
+	// We have free memory at 1526 and 2K, but we should get 4K
+	void *big = mm.allocate(4000); // size 12
+	REQUIRE( big == offset_pointer(memory[0], 4096 + hs) );
+	mm.free(big);
+
+	// free up 512
+	mm.free(allocs[3]);
+	mm.free(allocs[4]);
+
+	// Blocks should be:
+	// ...
+	//  9: [0, 512, 1024], 1536
+	//  8: [0, 256, 512], 768, 1024, [1280]
+
+	// A request for a 512-block should not cross the buddy divide
+	big = mm.allocate(500); // size 9
+	REQUIRE( big >= offset_pointer(memory[0], 1536 + hs) );
+	mm.free(big);
+
+	mm.free(allocs[0]);
+	mm.free(allocs[1]);
+	mm.free(allocs[2]);
+	mm.free(allocs[5]);
+	allocs.clear();
+
+	std::shuffle(sizes.begin(), sizes.end(), rng);
+
+	allocs.reserve(sizes.size());
+	for (size_t size : sizes)
+		allocs.push_back(mm.allocate(size));
+
+	std::shuffle(allocs.begin(), allocs.end(), rng);
+	for (void *p: allocs)
+		mm.free(p);
+	allocs.clear();
+
+	big = mm.allocate(64000);
+
+	// If everything was freed / joined correctly, that should not have allocated
+	CHECK( total_alloc_size == max_block );
+	CHECK( total_alloc_calls == 1 );
+
+	// And we should have gotten back the start of memory
+	CHECK( big == offset_pointer(memory[0], hs) );
+
+	free_memory();
+}
+
+bool check_in_memory(void *p)
+{
+	for (void *mem : memory)
+		if (p >= mem && p <= offset_pointer(mem, max_block))
+			return true;
+	return false;
+}
+
+TEST_CASE( "Non-contiguous blocks tests", "[memory buddy]" )
+{
+	using clock = std::chrono::system_clock;
+	unsigned seed = clock::now().time_since_epoch().count();
+	std::default_random_engine rng(seed);
+
+	heap_manager mm(grow_callback);
+	std::vector<void *> allocs;
+
+	const int blocks = 3;
+	for (int i = 0; i < blocks; ++i) {
+		void *p = mm.allocate(64000);
+		REQUIRE( memory[i] != nullptr );
+		REQUIRE( p == offset_pointer(memory[i], hs) );
+		allocs.push_back(p);
+	}
+
+	CHECK( total_alloc_size == max_block * blocks );
+	CHECK( total_alloc_calls == blocks );
+
+	for (void *p : allocs)
+		mm.free(p);
+	allocs.clear();
+
+	allocs.reserve(sizes.size() * blocks);
+
+	for (int i = 0; i < blocks; ++i) {
+		std::shuffle(sizes.begin(), sizes.end(), rng);
+
+		for (size_t size : sizes)
+			allocs.push_back(mm.allocate(size));
+	}
+
+	for (void *p : allocs)
+		CHECK( check_in_memory(p) );
+
+	std::shuffle(allocs.begin(), allocs.end(), rng);
+	for (void *p: allocs)
+		mm.free(p);
+	allocs.clear();
+
+	CHECK( total_alloc_size == max_block * blocks );
+	CHECK( total_alloc_calls == blocks );
+
+	for (int i = 0; i < blocks; ++i)
+		allocs.push_back(mm.allocate(64000));
+
+	// If everything was freed / joined correctly, that should not have allocated
+	CHECK( total_alloc_size == max_block * blocks );
+	CHECK( total_alloc_calls == blocks );
+
+	for (void *p : allocs)
+		CHECK( check_in_memory(p) );
+
+	free_memory();
+}
+

src/tests/main.cpp

@@ -7,3 +7,40 @@ void * operator new[] (size_t n)				{ return ::malloc(n); }
 void operator delete (void *p) noexcept			{ return ::free(p); }
 void operator delete[] (void *p) noexcept		{ return ::free(p); }
 
+#include "kutil/heap_manager.h"
+#include "kutil/memory.h"
+
+struct default_heap_listener : 
+	public Catch::TestEventListenerBase
+{
+	using TestEventListenerBase::TestEventListenerBase;
+
+	virtual void testCaseStarting(Catch::TestCaseInfo const& info) override
+	{
+		heap = new kutil::heap_manager(heap_grow_callback);
+		kutil::setup::set_heap(heap);
+	}
+
+	virtual void testCaseEnded(Catch::TestCaseStats const& stats) override
+	{
+		kutil::setup::set_heap(nullptr);
+		delete heap;
+
+		for (void *p : memory) ::free(p);
+		memory.clear();
+	}
+
+	static std::vector<void *> memory;
+	static kutil::heap_manager *heap;
+
+	static void * heap_grow_callback(size_t length) {
+		void *p = aligned_alloc(length, length);
+		memory.push_back(p);
+		return p;
+	}
+};
+
+std::vector<void *> default_heap_listener::memory;
+kutil::heap_manager *default_heap_listener::heap;
+
+CATCH_REGISTER_LISTENER( default_heap_listener );

src/tests/memory_manager.cpp

@@ -1,114 +0,0 @@
-#include <chrono>
-#include <random>
-#include <vector>
-#include <stddef.h>
-#include <stdlib.h>
-#include <stdint.h>
-
-#include "kutil/memory.h"
-#include "kutil/memory_manager.h"
-#include "catch.hpp"
-
-using namespace kutil;
-
-
-void *memory = nullptr;
-size_t total_alloc_size = 0;
-size_t total_alloc_calls = 0;
-
-const size_t hs = 0x10; // header size
-const size_t max_block = 1 << 16;
-
-
-void grow_callback(void *start, size_t length)
-{
-	total_alloc_size += length;
-	total_alloc_calls += 1;
-}
-
-
-TEST_CASE( "Buddy blocks tests", "[memory buddy]" )
-{
-	using clock = std::chrono::system_clock;
-	unsigned seed = clock::now().time_since_epoch().count();
-	std::default_random_engine rng(seed);
-
-	memory = aligned_alloc(max_block, 4 * max_block);
-
-	memory_manager mm(memory, grow_callback);
-
-	// The ctor should have allocated an initial block
-	CHECK( total_alloc_size == max_block );
-	CHECK( total_alloc_calls == 1 );
-
-	// Blocks should be:
-	// 16: 0-64K
-
-	std::vector<void *> allocs(6);
-	for (int i = 0; i < 6; ++i)
-		allocs[i] = mm.allocate(150); // size 8
-
-	// Should not have grown
-	CHECK( total_alloc_size == max_block );
-	CHECK( total_alloc_calls == 1 );
-
-	// Blocks should be:
-	// 16: [0-64K]
-	// 15: [0-32K], 32K-64K
-	// 14: [0-16K], 16K-32K
-	// 13: [0-8K], 8K-16K
-	// 12: [0-4K], 4K-8K
-	// 11: [0-2K], 2K-4K
-	// 10: [0-1K, 1-2K]
-	//  9: [0, 512, 1024], 1536
-	//  8: [0, 256, 512, 768, 1024, 1280]
-
-	// We have free memory at 1526 and 2K, but we should get 4K
-	void *big = mm.allocate(4000); // size 12
-	REQUIRE( big == offset_pointer(memory, 4096 + hs) );
-	mm.free(big);
-
-	// free up 512
-	mm.free(allocs[3]);
-	mm.free(allocs[4]);
-
-	// Blocks should be:
-	// ...
-	//  9: [0, 512, 1024], 1536
-	//  8: [0, 256, 512], 768, 1024, [1280]
-
-	// A request for a 512-block should not cross the buddy divide
-	big = mm.allocate(500); // size 9
-	REQUIRE( big >= offset_pointer(memory, 1536 + hs) );
-	mm.free(big);
-
-	mm.free(allocs[0]);
-	mm.free(allocs[1]);
-	mm.free(allocs[2]);
-	mm.free(allocs[5]);
-	allocs.clear();
-
-	std::vector<size_t> sizes = {
-		16000, 8000, 4000, 4000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 150,
-		150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 150, 48, 48, 48, 13 };
-
-	std::shuffle(sizes.begin(), sizes.end(), rng);
-
-	allocs.reserve(sizes.size());
-	for (size_t size : sizes)
-		allocs.push_back(mm.allocate(size));
-
-	std::shuffle(allocs.begin(), allocs.end(), rng);
-	for (void *p: allocs)
-		mm.free(p);
-	allocs.clear();
-
-	big = mm.allocate(64000);
-
-	// If everything was freed / joined correctly, that should not have allocated
-	CHECK( total_alloc_size == max_block );
-	CHECK( total_alloc_calls == 1 );
-
-	// And we should have gotten back the start of memory
-	CHECK( big == offset_pointer(memory, hs) );
-}