Recycle old initial pml4

Instead of building nested page tables for the offset region, just
offset map the entire thing into kernel memory with one PDP mapping
1GiB large pages. This is more efficient and avoids the "need a
page table to map in a page table" dependency loop.

.gitignore

@@ -1,7 +1,9 @@
 .lock*
-build
+/build*
 *.bak
 tags
 .gdbinit
 popcorn.log
-.waf-*
+*.o
+*.a
+sysroot

LICENSE.md

@@ -2,7 +2,8 @@
 
 Popcorn itself is released under the terms of the MIT license:
 
-> Copyright © 2018 Justin C. Miller, http://devjustinian.com <justin@devjustinian.com>
+> Copyright © 2018 Justin C. Miller, https://devjustinian.com
+> <justin@devjustinian.com>
 > 
 > Permission is hereby granted, free of charge, to any person obtaining a copy
 > of this software and associated documentation files (the “Software”), to deal
@@ -19,8 +20,8 @@ Popcorn itself is released under the terms of the MIT license:
 > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 > AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 > LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-> THE SOFTWARE.
+> SOFTWARE.
 
 # Included works
 
@@ -28,7 +29,7 @@ Popcorn includes and/or is derived from a number of other works, listed here.
 
 ## Catch2
 
-Popcorn uses [Catch2](http://github.com/catchorg/Catch2) for testing. Catch2 is
+Popcorn uses [Catch2](https://github.com/catchorg/Catch2) for testing. Catch2 is
 released under the terms of the Boost Software license:
 
 > Boost Software License - Version 1.0 - August 17th, 2003
@@ -55,44 +56,30 @@ released under the terms of the Boost Software license:
 > ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 > DEALINGS IN THE SOFTWARE.
 
-## Intel EFI Application Toolkit
+## cpptoml
 
-Popcorn's UEFI loader uses code from Intel's EFI Application toolkit. Relevant
+Popcorn uses the [cpptoml](https://github.com/skystrife/cpptoml) library for
-code includes license statements at the top of each file.
+parsing TOML configuration files. cpptoml is released under the terms of the
+MIT license:
 
-## Waf
+> Copyright (c) 2014 Chase Geigle
-
+>
-Popcorn's build system uses [Waf](https://waf.io/), which claims to be released
+> Permission is hereby granted, free of charge, to any person obtaining a copy of
-under the BSD license. I could not find its specific license file, so I am
+> this software and associated documentation files (the "Software"), to deal in
-reproducing a generic 3-clause BSD license (the most restrictive, so as not to
+> the Software without restriction, including without limitation the rights to
-assume any extra rights that may not actually be granted) for it here:
+> use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-
+> the Software, and to permit persons to whom the Software is furnished to do so,
-> Copyright © 2005-2018 Thomas Nagy
+> subject to the following conditions:
 > 
-> Redistribution and use in source and binary forms, with or without
+> The above copyright notice and this permission notice shall be included in all
-> modification, are permitted provided that the following conditions are met:
+> copies or substantial portions of the Software.
 > 
-> 1. Redistributions of source code must retain the above copyright notice, this
+> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
->    list of conditions and the following disclaimer.
+> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-> 
+> FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-> 2. Redistributions in binary form must reproduce the above copyright notice,
+> COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
->    this list of conditions and the following disclaimer in the documentation
+> IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
->    and/or other materials provided with the distribution.
+> CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-> 
-> 3. Neither the name of the copyright holder nor the names of its contributors
->    may be used to endorse or promote products derived from this software
->    without specific prior written permission.
-> 
-> THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-> ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-> WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-> DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-> FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-> DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-> SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-> CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-> OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-> OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ## GNU-EFI
 
@@ -130,3 +117,7 @@ to assume any extra rights that may not actually be granted) for it here:
 > OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 > OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+## Intel EFI Application Toolkit
+
+Popcorn's UEFI loader uses code from Intel's EFI Application toolkit. Relevant
+code includes license statements at the top of each file.

NOTES.md

@@ -2,23 +2,37 @@
 
 ## TODO
 
-- Better page-allocation model
+- Paging manager
+  - Copy-on-write pages
+  - Better page-allocation model?
 - Allow for more than one IOAPIC in ACPI module
   - The objects get created, but GSI lookup only uses the one at index 0
-- Slab allocator for kernel structures
 - mark kernel memory pages global
-- lock `memory_manager` and `page_manager` structures
 - Serial out based on circular/bip biffer and interrupts, not spinning on
   `write_ready()`
 - Split out more code into kutil for testing
-
+- AHCI / MSI interrupts on Vbox break?
-
+- FXSAVE to save XMM registers.
+  - optimization using #NM (0x7) to detect SSE usage
+- Clean up of process memory maps
+- Better stack tracer
+- Bootloader rewrite
+  - C++ and sharing library code for ELF, initrd, etc
+  - Parse initrd and pre-load certain ELF images, eg the process loader process?
+  - Do initial memory bootstrap?
+- Calling global ctors
 - Device Tree
   - Actual serial driver
   - Disk driver
     - File system
-- Memory map swapping
+- Multiprocessing
-  - Multiprocessing
+  - Fast syscalls using syscall/sysret
-    - Processes in Ring 3
+
-- Stack tracer
+### Build
-- build system upgrade (CMake / Waf / etc)
+
+- Clean up build generator and its templates
+  - More robust objects to represent modules & targets to pass to templates
+  - Read project setup from a simple JSON/TOML/etc
+  - Better compartmentalizing when doing template inheritance
+- Move to LLD as sysroot linker
+

README.md

@@ -27,14 +27,34 @@ The design goals of the project are:
 
 ## Building
 
-Popcorn uses the `waf` build tool, which is included in the repo. The other
+Popcorn uses the `ninja` build tool, and generates the build files for it with
-requirements are:
+the `generate_build.py` script.  The other requirements are:
 
-* python (to run waf)
+* python 3 for generating the build config
+  * The Jinja2 package is also required
 * clang
-* nasm
+* mtools
+* ninja
+* curl for downloading the toolchain
+
+### Setting up the cross toolchain
+
+If you have `clang` and `curl` installed, runing the `scripts/build_sysroot_clang.sh`
+script will download and build a nasm/binutils/LLVM toolchain configured for building
+Popcorn host binaries.
+
+### Building and running Popcorn
+
+Once the toolchain has been set up, running `generate_build.py` will set up the
+build configuration, and `ninja -C build` will actually run the build.  If you
+have `qemu-system-x86_64` installed, the `qemu.sh` script will to run Popcorn
+in QEMU `-nographic` mode.
+
+I personally run this either from a real debian amd64 testing/buster machine or
+a windows WSL debian testing/buster installation. The following should be
+enough to set up such a system to build the kernel:
+
+    sudo apt install qemu-system-x86 nasm clang-6.0 mtools
+	sudo update-alternatives /usr/bin/clang clang /usr/bin/clang-6.0 1000
+	sudo update-alternatives /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 1000
 
-After cloning, run `waf configure`. Then you can run `waf build` to build the
-project, and `waf test` to run the tests. A floppy disk image will be built in
-`build/popcorn.img`. If you have `qemu-system-x86_64` installed, then you can
-run `waf qemu` to run it in `-nographic` mode.

assets/initrd.toml

@@ -0,0 +1,23 @@
+# This is the manifest for the initial ramdisk, read by the `makerd` tool.
+# The contents should be a table array of files to add to the ramdistk:
+#
+#  [[files]]
+#  dest = "foo.bar"             # Name of the file in the ramdisk
+#  source = "build/foo/foo.bar" # Location of the file from the project root
+#  executable = true            # Optional, default false. Whether this is an
+#                               # initial application for the kernel to execute
+#                               # on startup
+
+[[files]]
+dest = "screenfont.psf"
+source = "../assets/fonts/tamsyn8x16r.psf"
+
+[[files]]
+dest = "nulldrv1"
+source = "host/nulldrv"
+executable = true
+
+[[files]]
+dest = "nulldrv2"
+source = "host/nulldrv"
+executable = true

generate_build.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+
+from collections import namedtuple
+
+library = namedtuple('library', ['path', 'deps'])
+program = namedtuple('program', ['path', 'deps', 'output', 'targets'])
+source = namedtuple('source', ['name', 'input', 'output', 'action'])
+version = namedtuple('version', ['major', 'minor', 'patch', 'sha', 'dirty'])
+
+MODULES = {}
+
+class Source:
+    Actions = {'.c': 'cc', '.cpp': 'cxx', '.s': 'nasm'}
+
+    def __init__(self, path, root, modroot):
+        from os.path import relpath, splitext
+        self.input = path
+        self.name = relpath(path, root)
+        self.output = relpath(path, modroot) + ".o"
+        self.action = self.Actions.get(splitext(path)[1], None)
+
+    def __str__(self):
+        return "{} {}:{}:{}".format(self.action, self.output, self.name, self.input)
+
+
+class Module:
+    def __init__(self, name, output, root, **kwargs):
+        from os.path import commonpath, dirname, isdir, join
+
+        self.name = name
+        self.output = output
+        self.kind = kwargs.get("kind", "exe")
+        self.target = kwargs.get("target", None)
+        self.deps = kwargs.get("deps", tuple())
+        self.includes = kwargs.get("includes", tuple())
+        self.defines = kwargs.get("defines", tuple())
+        self.depmods = []
+
+        sources = [join(root, f) for f in kwargs.get("source", tuple())]
+        modroot = commonpath(sources)
+        while not isdir(modroot):
+            modroot = dirname(modroot)
+
+        self.sources = [Source(f, root, modroot) for f in sources]
+
+    def __str__(self):
+        return "Module {} {}\n\t".format(self.kind, self.name)
+
+    def __find_depmods(self, modules):
+        self.depmods = set()
+        open_list = set(self.deps)
+        closed_list = set()
+
+        while open_list:
+            dep = modules[open_list.pop()]
+            open_list |= (set(dep.deps) - closed_list)
+            self.depmods.add(dep)
+
+        self.libdeps = [d for d in self.depmods if d.kind == "lib"]
+        self.exedeps = [d for d in self.depmods if d.kind != "lib"]
+
+    @classmethod
+    def load(cls, filename):
+        from os.path import abspath, dirname
+        from yaml import load
+
+        root = dirname(filename)
+        modules = {}
+        moddata = load(open(filename, "r"))
+
+        for name, data in moddata.items():
+            modules[name] = cls(name, root=root, **data)
+
+        for mod in modules.values():
+            mod.__find_depmods(modules)
+
+        targets = {}
+        for mod in modules.values():
+            if mod.target is None: continue
+            if mod.target not in targets:
+                targets[mod.target] = set()
+            targets[mod.target].add(mod)
+            targets[mod.target] |= mod.depmods
+
+        return modules.values(), targets
+
+
+def get_template(env, typename, name):
+    from jinja2.exceptions import TemplateNotFound
+    try:
+        return env.get_template("{}.{}.j2".format(typename, name))
+    except TemplateNotFound:
+        return env.get_template("{}.default.j2".format(typename))
+
+
+def get_git_version():
+    from subprocess import run
+    cp = run(['git', 'describe', '--dirty'],
+            check=True, capture_output=True)
+    full_version = cp.stdout.decode('utf-8').strip()
+
+    cp = run(['git', 'rev-parse', 'HEAD'],
+            check=True, capture_output=True)
+    full_sha = cp.stdout.decode('utf-8').strip()
+
+    dirty = False
+    parts1 = full_version.split('-')
+    if parts1[-1] == "dirty":
+        dirty = True
+        parts1 = parts1[:-1]
+
+    parts2 = parts1[0].split('.')
+
+    return version(
+        parts2[0],
+        parts2[1],
+        parts2[2],
+        full_sha[:7],
+        dirty)
+
+
+def main(buildroot="build", modulefile="modules.yaml"):
+    import os
+    from os.path import abspath, dirname, isabs, isdir, join
+
+    generator = abspath(__file__)
+    srcroot = dirname(generator)
+    if not isabs(modulefile):
+        modulefile = join(srcroot, modulefile)
+
+    if not isabs(buildroot):
+        buildroot = join(srcroot, buildroot)
+
+    if not isdir(buildroot):
+        os.mkdir(buildroot)
+
+    git_version = get_git_version()
+    print("Generating build files for Popcorn {}.{}.{}-{}...".format(
+        git_version.major, git_version.minor, git_version.patch, git_version.sha))
+
+    from jinja2 import Environment, FileSystemLoader
+    template_dir = join(srcroot, "scripts", "templates")
+    env = Environment(loader=FileSystemLoader(template_dir))
+
+    buildfiles = []
+    templates = set()
+    modules, targets = Module.load(modulefile)
+
+    for mod in modules:
+        buildfile = join(buildroot, mod.name + ".ninja")
+        buildfiles.append(buildfile)
+        with open(buildfile, 'w') as out:
+            template = get_template(env, mod.kind, mod.name)
+            templates.add(template.filename)
+            out.write(template.render(
+                module=mod,
+                buildfile=buildfile,
+                version=git_version))
+
+    for target, mods in targets.items():
+        root = join(buildroot, target)
+        if not isdir(root):
+            os.mkdir(root)
+
+        buildfile = join(root, "target.ninja")
+        buildfiles.append(buildfile)
+        with open(buildfile, 'w') as out:
+            template = get_template(env, "target", target)
+            templates.add(template.filename)
+            out.write(template.render(
+                target=target,
+                modules=mods,
+                buildfile=buildfile,
+                version=git_version))
+
+    # Top level buildfile cannot use an absolute path or ninja won't
+    # reload itself properly on changes.
+    # See: https://github.com/ninja-build/ninja/issues/1240
+    buildfile = "build.ninja"
+    buildfiles.append(buildfile)
+
+    with open(join(buildroot, buildfile), 'w') as out:
+        template = env.get_template("build.ninja.j2")
+        templates.add(template.filename)
+
+        out.write(template.render(
+            targets=targets,
+            buildroot=buildroot,
+            srcroot=srcroot,
+            buildfile=buildfile,
+            buildfiles=buildfiles,
+            templates=[abspath(f) for f in templates],
+            generator=generator,
+            modulefile=modulefile,
+            version=git_version))
+
+if __name__ == "__main__":
+    import sys
+    main(*sys.argv[1:])

modules.yaml

@@ -0,0 +1,116 @@
+kernel:
+    output: popcorn.elf
+    target: host
+    deps:
+        - elf
+        - initrd
+        - kutil
+    includes:
+        - src/kernel
+    source:
+        - src/kernel/crti.s
+        - src/kernel/apic.cpp
+        - src/kernel/assert.cpp
+        - src/kernel/boot.s
+        - src/kernel/console.cpp
+        - src/kernel/cpprt.cpp
+        - src/kernel/cpu.cpp
+        - src/kernel/debug.cpp
+        - src/kernel/debug.s
+        - src/kernel/device_manager.cpp
+        - src/kernel/font.cpp
+        - src/kernel/fs/gpt.cpp
+        - src/kernel/gdt.cpp
+        - src/kernel/gdt.s
+        - src/kernel/interrupts.cpp
+        - src/kernel/interrupts.s
+        - src/kernel/io.cpp
+        - src/kernel/loader.s
+        - src/kernel/log.cpp
+        - src/kernel/main.cpp
+        - src/kernel/memory_bootstrap.cpp
+        - src/kernel/msr.cpp
+        - src/kernel/page_manager.cpp
+        - src/kernel/pci.cpp
+        - src/kernel/process.cpp
+        - src/kernel/scheduler.cpp
+        - src/kernel/screen.cpp
+        - src/kernel/serial.cpp
+        - src/kernel/syscall.cpp
+        - src/kernel/syscall.s
+        - src/kernel/crtn.s
+
+boot:
+    kind: exe
+    target: boot
+    output: boot.elf
+    source:
+        - src/boot/crt0.s
+        - src/boot/console.cpp
+        - src/boot/guids.cpp
+        - src/boot/loader.cpp
+        - src/boot/main.cpp
+        - src/boot/memory.cpp
+        - src/boot/reloc.cpp
+        - src/boot/utility.cpp
+
+nulldrv:
+    kind: exe
+    target: host
+    output: nulldrv
+    source:
+        - src/drivers/nulldrv/main.s
+
+elf:
+    kind: lib
+    output: libelf.a
+    deps:
+        - kutil
+    includes:
+        - src/libraries/elf/include
+    source:
+        - src/libraries/elf/elf.cpp
+
+initrd:
+    kind: lib
+    output: libinitrd.a
+    deps:
+        - kutil
+    includes:
+        - src/libraries/initrd/include
+    source:
+        - src/libraries/initrd/initrd.cpp
+
+kutil:
+    kind: lib
+    output: libkutil.a
+    includes:
+        - src/libraries/kutil/include
+    source:
+        - src/libraries/kutil/assert.cpp
+        - src/libraries/kutil/frame_allocator.cpp
+        - src/libraries/kutil/heap_manager.cpp
+        - src/libraries/kutil/memory.cpp
+
+makerd:
+    kind: exe
+    target: native
+    output: makerd
+    deps:
+        - initrd
+    source:
+        - src/tools/makerd/entry.cpp
+        - src/tools/makerd/main.cpp
+
+tests:
+    kind: exe
+    target: native
+    output: tests
+    deps:
+        - kutil
+    source:
+        - src/tests/address_manager.cpp
+        - src/tests/frame_allocator.cpp
+        - src/tests/linked_list.cpp
+        - src/tests/heap_manager.cpp
+        - src/tests/main.cpp

qemu.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+build="$(dirname $0)/build"
+debug=""
+gfx="-nographic"
+
+for arg in $@; do
+	case "${arg}" in
+		--debug)
+			debug="-s"
+			;;
+		--gfx)
+			gfx=""
+			;;
+		*)
+			build="${arg}"
+			;;
+	esac
+done
+
+kvm=""
+if [[ -c /dev/kvm ]]; then
+	kvm="-enable-kvm"
+fi
+
+ninja -C "${build}" && \
+exec qemu-system-x86_64 \
+	-drive "if=pflash,format=raw,file=${build}/flash.img" \
+	-drive "format=raw,file=${build}/popcorn.img" \
+	-smp 1 \
+	-m 512 \
+	-d mmu,int,guest_errors \
+	-D popcorn.log \
+	-cpu Broadwell \
+	-M q35 \
+	-no-reboot \
+	$gfx $kvm $debug

scripts/build_sysroot_clang.sh

@@ -0,0 +1,220 @@
+#!/usr/bin/env bash
+
+TARGET="x86_64-elf"
+NASM_VERSION="2.13.03"
+BINUTILS_VERSION="2.31.1"
+
+TOOLS="clang" # lld libunwind libcxxabi libcxx"
+PROJECTS="compiler-rt libcxxabi libcxx libunwind"
+#RUNTIMES="compiler-rt libcxxabi libcxx libunwind"
+
+set -e
+
+SYSROOT=$(realpath "$(dirname $0)/../sysroot")
+WORK=$(realpath "$(dirname $0)/sysroot")
+mkdir -p "${SYSROOT}"
+mkdir -p "${WORK}"
+
+export CC=clang
+export CXX=clang++
+
+function build_nasm() {
+	if [[ ! -d "${WORK}/nasm-${NASM_VERSION}" ]]; then
+		echo "Downloading NASM..."
+		tarball="nasm-${NASM_VERSION}.tar.gz"
+		curl -sSOL "https://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/${tarball}"
+		tar xzf "${tarball}" -C "${WORK}" && rm "${tarball}"
+	fi
+
+	mkdir -p "${WORK}/build/nasm"
+	pushd "${WORK}/build/nasm"
+
+	if [[ ! -f "${WORK}/build/nasm/config.cache" ]]; then
+		echo "Configuring NASM..."
+		"${WORK}/nasm-${NASM_VERSION}/configure" \
+			--quiet \
+			--config-cache \
+			--disable-werror \
+			--prefix="${SYSROOT}" \
+			--srcdir="${WORK}/nasm-${NASM_VERSION}"
+	fi
+
+	echo "Building NASM..."
+	(make -j && make install) > "${WORK}/build/nasm_build.log"
+	popd
+}
+
+function build_binutils() {
+	if [[ ! -d "${WORK}/binutils-${BINUTILS_VERSION}" ]]; then
+		echo "Downloading binutils..."
+		tarball="binutils-${BINUTILS_VERSION}.tar.gz"
+		curl -sSOL "https://ftp.gnu.org/gnu/binutils/${tarball}"
+		tar xzf "${tarball}" -C "${WORK}" && rm "${tarball}"
+	fi
+
+	mkdir -p "${WORK}/build/binutils"
+	pushd "${WORK}/build/binutils"
+
+	if [[ ! -f "${WORK}/build/binutils/config.cache" ]]; then
+		echo "Configuring binutils..."
+		"${WORK}/binutils-${BINUTILS_VERSION}/configure" \
+			--quiet \
+			--config-cache \
+			--target="${TARGET}" \
+			--prefix="${SYSROOT}" \
+			--with-sysroot="${SYSROOT}" \
+			--with-lib-path="${SYSROOT}/lib" \
+			--disable-nls \
+			--disable-werror
+	fi
+
+	echo "Building binutils..."
+	(make -j && make install) > "${WORK}/build/binutils_build.log"
+	popd
+}
+
+function build_llvm() {
+	if [[ ! -d "${WORK}/llvm" ]]; then
+		echo "Downloading LLVM..."
+		git clone -q \
+			--branch release_70 \
+			--depth 1 \
+			"https://git.llvm.org/git/llvm.git" "${WORK}/llvm"
+	fi
+
+	for tool in ${TOOLS}; do
+		if [[ ! -d "${WORK}/llvm/tools/${tool}" ]]; then
+			echo "Downloading ${tool}..."
+			git clone -q \
+				--branch release_70 \
+				--depth 1 \
+				"https://git.llvm.org/git/${tool}.git" "${WORK}/llvm/tools/${tool}"
+		fi
+	done
+
+	if [[ ! -d "${WORK}/llvm/tools/clang/tools/extra" ]]; then
+		echo "Downloading clang-tools-extra..."
+		git clone -q \
+			--branch release_70 \
+			--depth 1 \
+			"https://git.llvm.org/git/clang-tools-extra.git" "${WORK}/llvm/tools/clang/tools/extra"
+	fi
+
+	for proj in ${PROJECTS}; do
+		if [[ ! -d "${WORK}/llvm/projects/${proj}" ]]; then
+			echo "Downloading ${proj}..."
+			git clone -q \
+				--branch release_70 \
+				--depth 1 \
+				"https://git.llvm.org/git/${proj}.git" "${WORK}/llvm/projects/${proj}"
+		fi
+	done
+
+	for proj in ${RUNTIMES}; do
+		if [[ ! -d "${WORK}/llvm/runtimes/${proj}" ]]; then
+			echo "Downloading ${proj}..."
+			git clone -q \
+				--branch release_70 \
+				--depth 1 \
+				"https://git.llvm.org/git/${proj}.git" "${WORK}/llvm/runtime/${proj}"
+		fi
+	done
+
+	mkdir -p "${WORK}/build/llvm"
+	pushd "${WORK}/build/llvm"
+
+	echo "Configuring LLVM..."
+
+	cmake -G Ninja \
+		-DCLANG_DEFAULT_RTLIB=compiler-rt \
+		-DCLANG_DEFAULT_STD_C=c11 \
+		-DCLANG_DEFAULT_STD_CXX=cxx14 \
+		-DCMAKE_BUILD_TYPE=Release \
+		-DCMAKE_C_COMPILER="clang" \
+		-DCMAKE_CXX_COMPILER="clang++" \
+		-DCMAKE_CXX_FLAGS="-Wno-unused-parameter -D_LIBCPP_HAS_NO_ALIGNED_ALLOCATION -D_LIBUNWIND_IS_BAREMETAL=1 -U_LIBUNWIND_SUPPORT_DWARF_UNWIND" \
+		-DCMAKE_INSTALL_PREFIX="${SYSROOT}" \
+		-DCMAKE_MAKE_PROGRAM=`which ninja` \
+		-DDEFAULT_SYSROOT="${SYSROOT}" \
+		-DLIBCXX_CXX_ABI=libcxxabi \
+		-DLIBCXX_CXX_ABI_INCLUDE_PATHS="${WORK}/llvm/projects/libcxxabi/include" \
+		-DLIBCXX_CXX_ABI_LIBRARY_PATH=lib \
+		-DLIBCXX_ENABLE_EXPERIMENTAL_LIBRARY=OFF \
+		-DLIBCXX_ENABLE_NEW_DELETE_DEFINITIONS=ON \
+		-DLIBCXX_ENABLE_SHARED=OFF \
+		-DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \
+		-DLIBCXX_ENABLE_THREADS=OFF \
+		-DLIBCXX_INCLUDE_BENCHMARKS=OFF \
+		-DLIBCXX_USE_COMPILER_RT=ON \
+		-DLIBCXXABI_ENABLE_NEW_DELETE_DEFINITIONS=OFF \
+		-DLIBCXXABI_ENABLE_SHARED=OFF \
+		-DLIBCXXABI_ENABLE_STATIC_UNWINDER=ON \
+		-DLIBCXXABI_ENABLE_THREADS=OFF \
+		-DLIBCXXABI_LIBCXX_PATH="${WORK}/llvm/projects/libcxx" \
+		-DLIBCXXABI_USE_COMPILER_RT=ON \
+		-DLIBCXXABI_USE_LLVM_UNWINDER=ON \
+		-DLIBUNWIND_ENABLE_SHARED=OFF \
+		-DLIBUNWIND_ENABLE_THREADS=OFF \
+		-DLIBUNWIND_USE_COMPILER_RT=ON \
+		-DLLVM_CONFIG_PATH="${SYSROOT}/bin/llvm-config" \
+		-DLLVM_DEFAULT_TARGET_TRIPLE="x86_64-unknown-elf" \
+		-DLLVM_ENABLE_LIBCXX=ON \
+		-DLLVM_ENABLE_PIC=OFF \
+		-DLLVM_ENABLE_THREADS=OFF \
+		-DLLVM_INSTALL_BINUTILS_SYMLINKS=ON \
+		-DLLVM_TARGETS_TO_BUILD="X86" \
+		${WORK}/llvm > cmake_configure.log
+
+	#  -DCMAKE_ASM_COMPILER=nasm \
+	#  -DCMAKE_LINKER="${SYSROOT}/bin/ld.lld" \
+	#  -DCOMPILER_RT_ENABLE_LLD=ON \
+	#  -DLIBCXX_ENABLE_LLD=ON \
+	#  -DLIBCXX_ENABLE_STATIC_UNWINDER=ON \
+	#  -DLIBCXXABI_ENABLE_LLD=ON \
+	#  -DLIBUNWIND_ENABLE_LLD=ON \
+	#  -DLLVM_ENABLE_LLD=ON \
+	#  -DLLVM_ENABLE_PROJECTS="libcxx;libcxxabi;libunwind;compiler-rt" \
+	#  -DCOMPILER_RT_BAREMETAL_BUILD=ON \
+	#  -DLIBCXXABI_BAREMETAL=ON \
+
+	echo "Building LLVM..."
+	ninja && ninja install
+	ninja cxx cxxabi compiler-rt
+	ninja install-compiler-rt install-cxx install-cxxabi
+	popd
+}
+
+function build_libc() {
+	if [[ ! -d "${WORK}/poplibc" ]]; then
+		echo "Downloading poplibc..."
+		git clone \
+			"https://github.com/justinian/poplibc.git" \
+			"${WORK}/poplibc"
+	else
+		echo "Updating poplibc..."
+		git -C "${WORK}/poplibc" pull
+	fi
+
+	pushd "${WORK}/poplibc"
+	echo "Building poplibc..."
+	make install PREFIX="${SYSROOT}"
+	popd
+}
+
+function update_links() {
+	for exe in `ls "${SYSROOT}/bin/${TARGET}-"*`; do
+		base=$(echo "$exe" | sed -e "s/${TARGET}-//")
+		ln -fs "${exe}" "${base}"
+	done
+}
+
+build_nasm
+build_binutils
+build_libc
+build_llvm
+update_links
+
+export CC="${SYSROOT}/bin/clang"
+export CXX="${SYSROOT}/bin/clang++"
+export LD="${SYSROOT}/bin/ld"
+build_libc

scripts/build_sysroot_gcc.sh

@@ -0,0 +1,186 @@
+#!/usr/bin/env bash
+
+TARGET="x86_64-elf"
+NASM_VERSION="2.14.02"
+GCC_VERSION="7.4.0"
+BINUTILS_VERSION="2.31.1"
+
+SYSROOT=$(realpath "$(dirname $0)/../sysroot")
+WORK=$(realpath "$(dirname $0)/sysroot")
+
+echo "Not currently supported"
+exit 1
+
+set -e
+mkdir -p "${SYSROOT}"
+mkdir -p "${WORK}"
+
+
+function build_nasm() {
+	if [[ ! -d "${WORK}/nasm-${NASM_VERSION}" ]]; then
+		echo "Downloading NASM..."
+		tarball="nasm-${NASM_VERSION}.tar.gz"
+		curl -sSOL "https://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/${tarball}"
+		tar xzf "${tarball}" -C "${WORK}" && rm "${tarball}"
+	fi
+
+	mkdir -p "${WORK}/build/nasm"
+	pushd "${WORK}/build/nasm"
+
+	if [[ ! -f "${WORK}/build/nasm/config.cache" ]]; then
+		echo "Configuring NASM..."
+		"${WORK}/nasm-${NASM_VERSION}/configure" \
+			--quiet \
+			--config-cache \
+			--disable-werror \
+			--prefix="${SYSROOT}" \
+			--srcdir="${WORK}/nasm-${NASM_VERSION}"
+	fi
+
+	echo "Building NASM..."
+	(make -j && make install) > "${WORK}/build/nasm_build.log"
+	popd
+}
+
+function build_binutils() {
+	if [[ ! -d "${WORK}/binutils-${BINUTILS_VERSION}" ]]; then
+		echo "Downloading binutils..."
+		tarball="binutils-${BINUTILS_VERSION}.tar.gz"
+		curl -sSOL "https://ftp.gnu.org/gnu/binutils/${tarball}"
+		tar xzf "${tarball}" -C "${WORK}" && rm "${tarball}"
+	fi
+
+	mkdir -p "${WORK}/build/binutils"
+	pushd "${WORK}/build/binutils"
+
+	if [[ ! -f "${WORK}/build/binutils/config.cache" ]]; then
+		echo "Configuring binutils..."
+		"${WORK}/binutils-${BINUTILS_VERSION}/configure" \
+			--quiet \
+			--config-cache \
+			--target="${TARGET}" \
+			--prefix="${SYSROOT}" \
+			--with-sysroot="${SYSROOT}" \
+			--with-lib-path="${SYSROOT}/lib" \
+			--disable-nls \
+			--disable-werror
+	fi
+
+	echo "Building binutils..."
+	(make -j && make install) > "${WORK}/build/binutils_build.log"
+	popd
+}
+
+function build_gcc() {
+	if [[ ! -d "${WORK}/gcc-${GCC_VERSION}" ]]; then
+		echo "Downloading GCC..."
+		tarball="gcc-${GCC_VERSION}.tar.gz"
+		curl -sSOL "https://ftp.gnu.org/gnu/gcc/gcc-${GCC_VERSION}/${tarball}"
+		tar xzf "${tarball}" -C "${WORK}" && rm "${tarball}"
+
+		# no-red-zone support version of libgcc
+		echo "MULTILIB_OPTIONS += mno-red-zone" >  "${WORK}/gcc-${GCC_VERSION}/gcc/config/i386/t-${TARGET}"
+		echo "MULTILIB_DIRNAMES += no-red-zone" >> "${WORK}/gcc-${GCC_VERSION}/gcc/config/i386/t-${TARGET}"
+
+cat <<EOF >> "${WORK}/gcc-${GCC_VERSION}/gcc/config.gcc"
+case \${target} in
+	${TARGET})
+		tmake_file="\${tmake_file} i386/t-${TARGET}"
+		;;
+esac
+EOF
+	fi
+
+	mkdir -p "${WORK}/build/gcc"
+	pushd "${WORK}/build/gcc"
+
+	if [[ ! -f "${WORK}/build/gcc/config.cache" ]]; then
+		echo "Configuring GCC..."
+		"${WORK}/gcc-${GCC_VERSION}/configure" \
+			--quiet \
+			--config-cache \
+			--target="${TARGET}" \
+			--prefix="${SYSROOT}" \
+			--with-sysroot="${SYSROOT}" \
+			--with-native-system-header-dir="${SYSROOT}/include" \
+			--with-newlib \
+			--without-headers \
+			--disable-nls \
+			--enable-languages=c,c++ \
+			--disable-shared \
+			--disable-multilib \
+			--disable-decimal-float \
+			--disable-threads \
+			--disable-libatomic \
+			--disable-libgomp \
+			--disable-libmpx \
+			--disable-libquadmath \
+			--disable-libssp \
+			--disable-libvtv \
+			--disable-libstdcxx
+	fi
+
+	echo "Building GCC..."
+	(make -j all-gcc && make -j all-target-libgcc && \
+		make install-gcc && make install-target-libgcc) > "${WORK}/build/gcc_build.log"
+	popd
+}
+
+function build_libstdcxx() {
+	mkdir -p "${WORK}/build/libstdcxx"
+	pushd "${WORK}/build/libstdcxx"
+
+	if [[ ! -f "${WORK}/build/libstdcxx/config.cache" ]]; then
+		echo "Configuring libstdc++..."
+		CFLAGS="-I${SYSROOT}/include" \
+		CXXFLAGS="-I${SYSROOT}/include" \
+		"${WORK}/gcc-${GCC_VERSION}/libstdc++-v3/configure" \
+			--config-cache \
+			--host="${TARGET}" \
+			--target="${TARGET}" \
+			--prefix="${SYSROOT}" \
+			--disable-nls \
+			--disable-multilib \
+			--with-newlib \
+			--disable-libstdcxx-threads \
+			--disable-libstdcxx-pch \
+			--with-gxx-include-dir="${SYSROOT}/include/c++"
+	fi
+
+	echo "Building libstdc++..."
+	(make -j && make install) > "${WORK}/build/libstdcxx_build.log"
+	popd
+}
+
+function build_libc() {
+	if [[ ! -d "${WORK}/poplibc" ]]; then
+		echo "Downloading poplibc..."
+		git clone \
+			"https://github.com/justinian/poplibc.git" \
+			"${WORK}/poplibc"
+	else
+		echo "Updating poplibc..."
+		git -C "${WORK}/poplibc" pull
+	fi
+
+	pushd "${WORK}/poplibc"
+	echo "Building poplibc..."
+	make install PREFIX="${SYSROOT}"
+	popd
+}
+
+function update_links() {
+	for exe in `ls "${SYSROOT}/bin/${TARGET}-"*`; do
+		base=$(echo "$exe" | sed -e "s/${TARGET}-//")
+		ln -fs "${exe}" "${base}"
+	done
+}
+
+build_nasm
+build_binutils
+build_gcc
+
+update_links
+export PATH="${SYSROOT}/bin:${PATH}"
+build_libc
+build_libstdcxx

scripts/templates/build.ninja.j2

@@ -0,0 +1,178 @@
+ninja_required_version = 1.3
+builddir = {{ buildroot }}
+srcroot = {{ srcroot }}
+modulefile = {{ modulefile }}
+
+warnflags = $
+    -Wformat=2 $
+    -Winit-self $
+    -Wfloat-equal $
+    -Winline $
+    -Wmissing-format-attribute $
+    -Wmissing-include-dirs $
+    -Wswitch $
+    -Wundef $
+    -Wdisabled-optimization $
+    -Wpointer-arith $
+    -Wno-attributes $
+    -Wno-sign-compare $
+    -Wno-multichar $
+    -Wno-div-by-zero $
+    -Wno-endif-labels $
+    -Wno-pragmas $
+    -Wno-format-extra-args $
+    -Wno-unused-result $
+    -Wno-deprecated-declarations $
+    -Wno-unused-function $
+    -Werror
+
+ccflags = $
+    -I${srcroot}/src/include $
+    -I${srcroot}/src/include/x86_64 $
+    -DVERSION_MAJOR={{ version.major }} $
+    -DVERSION_MINOR={{ version.minor }} $
+    -DVERSION_PATCH={{ version.patch }} $
+    -DVERSION_GITSHA=0x{% if version.dirty %}1{% else %}0{% endif %}{{ version.sha }} $
+    -DGIT_VERSION=\"{{ version.major }}.{{ version.minor }}.{{ version.patch }}-{{ version.sha }}\" $
+    -DGIT_VERSION_WIDE=L\"{{ version.major }}.{{ version.minor }}.{{ version.patch }}-{{ version.sha }}\" $
+    $warnflags
+
+asflags = $
+    -DVERSION_MAJOR={{ version.major }} $
+    -DVERSION_MINOR={{ version.minor }} $
+    -DVERSION_PATCH={{ version.patch }} $
+    -DVERSION_GITSHA=0x{% if version.dirty %}1{% else %}0{% endif %}{{ version.sha }}
+
+cflags = -std=c11
+cxxflags = -std=c++14
+libs =
+
+rule cc
+    deps = gcc
+    depfile = $out.d
+    description = Compiling $name
+    command = $cc -MMD -MF $out.d $ccflags $cflags -o $out -c $in
+
+rule dump_cc_defs
+    description = Dumping CC defines for $target
+    command = echo "" | $cc $ccflags $cflags -dM -E - > $out
+
+rule dump_cc_run
+    description = Dumping CC arguments for $target
+    command = $
+        echo "#!/bin/bash" > $out; $
+        echo '$cc $ccflags $cflags $$*' > $out; $
+        chmod a+x $out
+
+rule cxx
+    deps = gcc
+    depfile = $out.d
+    description = Compiling $name
+    command = $cxx -MMD -MF $out.d $cxxflags $ccflags -o $out -c $in
+
+rule dump_cxx_defs
+    description = Dumping C++ defines for $target
+    command = echo "" | $cxx -x c++ $cxxflags $ccflags -dM -E - > $out
+
+rule dump_cxx_run
+    description = Dumping C++ arguments for $target
+    command = $
+        echo "#!/bin/bash" > $out; $
+        echo '$cc $cxxflags $ccflags $$*' > $out; $
+        chmod a+x $out
+
+rule nasm
+    deps = gcc
+    depfile = $out.d
+    description = Assembling $name
+    command = $nasm -o $out -felf64 -MD $out.d $asflags $in
+
+rule exe
+    description = Linking $name
+    command = $ld $ldflags -o $out $in $libs
+
+rule lib
+    description = Archiving $name
+    command = $ar qcs $out $in
+
+rule regen
+    generator = true
+    description = Regenrating build files
+    command = {{ generator }} $builddir $modulefile
+
+rule cp
+    description = Copying $name
+    command = cp $in $out
+
+rule makerd
+    description = Making init ramdisk
+    command = $builddir/native/makerd $in $out
+
+rule makeefi
+    description = Converting $name
+    command = objcopy $
+        -j .text $
+        -j .sdata $
+        -j .data $
+        -j .dynamic $
+        -j .dynsym $
+        -j .rel $
+        -j .rela $
+        -j .reloc $
+        --target=efi-app-x86_64 $
+        $in $out
+
+rule makefat
+    description = Creating $name
+    command = $
+        cp $srcroot/assets/diskbase.img $out; $
+        mcopy -s -D o -i $out@@1M $builddir/fatroot/* ::/
+
+rule strip
+    description = Stripping $name
+    command = $
+        cp $in $out; $
+        objcopy --only-keep-debug $out $out.debug; $
+        strip -g $out; $
+        objcopy --add-gnu-debuglink=$out.debug $out
+
+
+{% for target in targets %}
+subninja {{ target }}/target.ninja
+{% endfor %}
+
+build $
+    {%- for buildfile in buildfiles %}
+    {{ buildfile }} $
+    {%- endfor %}
+    : regen | $
+    {%- for template in templates %}
+    {{ template }} $
+    {%- endfor %}
+    $modulefile $
+    {{ generator }}
+
+build $builddir/flash.img : cp $srcroot/assets/ovmf/x64/OVMF.fd
+    name = flash.img
+
+build $builddir/popcorn.elf | $builddir/popcorn.elf.debug : strip $builddir/host/popcorn.elf
+    name = kernel
+
+build $builddir/fatroot/popcorn.elf : cp $builddir/popcorn.elf
+    name = kernel to FAT image
+
+build $builddir/fatroot/efi/boot/bootx64.efi : cp $builddir/boot/boot.efi
+    name = bootloader to FAT image
+
+build $builddir/fatroot/initrd.img : makerd ${srcroot}/assets/initrd.toml | $
+    ${builddir}/native/makerd $
+    ${builddir}/host/nulldrv
+
+build $builddir/popcorn.img : makefat | $
+    $builddir/fatroot/initrd.img $
+    $builddir/fatroot/popcorn.elf $
+    $builddir/fatroot/efi/boot/bootx64.efi
+    name = popcorn.img
+
+# vim: et ts=4 sts=4 sw=4
+

scripts/templates/exe.boot.j2

@@ -0,0 +1,29 @@
+{% extends "exe.default.j2" %}
+{% block variables %}
+{{ super() }}
+
+ld = ld
+cc = clang
+cxx = clang++
+
+ccflags = $ccflags $
+    -DKERNEL_FILENAME=L\"popcorn.elf\" $
+    -DGNU_EFI_USE_MS_ABI $
+    -DHAVE_USE_MS_ABI $
+    -DEFI_DEBUG=0 $
+    -DEFI_DEBUG_CLEAR_MEMORY=0 $
+	-DBOOTLOADER_DEBUG $
+    -fPIC
+
+ldflags = $ldflags $
+    -T ${srcroot}/src/arch/x86_64/boot.ld $
+    -shared
+
+{% endblock %}
+
+{% block extra %}
+
+build $builddir/boot.efi : makeefi ${builddir}/{{ module.output }}
+    name = boot.efi
+
+{% endblock %}

scripts/templates/exe.default.j2

@@ -0,0 +1,12 @@
+{% extends "module.base.j2" %}
+{% block variables %}
+{{ super() }}
+
+libs = $
+    -L${builddir} $
+    {%- for dep in module.libdeps %}
+    -l{{ dep.name }} $
+    {%- endfor %}
+    $libs
+
+{% endblock %}

scripts/templates/exe.kernel.j2

@@ -0,0 +1,9 @@
+{% extends "exe.default.j2" %}
+{% block variables %}
+{{ super() }}
+
+asflags = $asflags -I${srcroot}/src/kernel/
+libs = $libs
+ldflags = $ldflags -T ${srcroot}/src/arch/x86_64/kernel.ld
+
+{% endblock %}

scripts/templates/exe.tests.j2

@@ -0,0 +1,7 @@
+{% extends "exe.default.j2" %}
+{% block variables %}
+{{ super() }}
+
+ccflags = $ccflags -ggdb
+
+{% endblock %}

scripts/templates/lib.default.j2

@@ -0,0 +1 @@
+{% extends "module.base.j2" %}

scripts/templates/module.base.j2

@@ -0,0 +1,41 @@
+moddir = ${builddir}/{{ module.name }}.dir
+
+{% block variables %}
+ccflags = $ccflags $
+    {%- for dep in module.depmods %}
+    {%- for inc in dep.includes %}
+    -I${srcroot}/{{ inc }} $
+    {%- endfor %}
+    {%- endfor %}
+    {%- for inc in module.includes %}
+    -I${srcroot}/{{ inc }} $
+    {%- endfor %}
+    {%- for define in module.defines %}
+    -D{{ define }} $
+    {%- endfor %}
+{% endblock %}
+
+{% for source in module.sources %}
+build ${moddir}/{{ source.output }} : {{ source.action }} {{ source.input }} || {{ buildfile }}
+    name = {{ source.name }}
+{% endfor %}
+
+build ${builddir}/{{ module.output }} : {{ module.kind }} $
+{%- for source in module.sources %}
+    ${moddir}/{{ source.output }} $
+{%- endfor -%}
+{%- for dep in module.libdeps %}
+    ${builddir}/{{ dep.output }} $
+{%- endfor %}
+    | $
+{% for dep in module.exedeps %}
+    ${builddir}/{{ dep.output }} $
+{%- endfor -%}
+    {{ buildfile }}
+    name = {{ module.name }}
+
+{% block extra %}
+{% endblock %}
+
+# vim: ft=ninja et ts=4 sts=4 sw=4
+

scripts/templates/target.boot.j2

@@ -0,0 +1,35 @@
+{% extends "target.default.j2" %}
+
+{% block binaries %}
+ld = ld
+cc = clang
+cxx = clang++
+nasm = nasm
+{% endblock %}
+
+{% block variables %}
+
+ccflags = $ccflags $
+    -ggdb $
+    -nostdlib $
+    -ffreestanding $
+    -nodefaultlibs $
+    -fno-builtin $
+    -mno-sse $
+    -fno-omit-frame-pointer $
+    -mno-red-zone $
+    -fshort-wchar
+
+cxxflags = $cxxflags $
+    -nostdlibinc $
+    -fno-exceptions $
+    -fno-rtti
+
+ldflags = $ldflags $
+    -g $
+    -nostdlib $
+    -znocombreloc $
+	-Bsymbolic $
+    -nostartfiles
+
+{% endblock %}

scripts/templates/target.default.j2

@@ -0,0 +1,24 @@
+builddir = $builddir/{{ target }}
+target = {{ target }}
+
+{% block variables %}
+{% endblock %}
+
+{% block binaries %}
+cc = clang
+cxx = clang++
+ld = clang++
+ar = ar
+nasm = nasm
+objcopy = objcopy
+{% endblock %}
+
+{% for module in modules %}
+subninja {{ module.name }}.ninja
+{% endfor %}
+
+build ${builddir}/cc.defs : dump_cc_defs | {{ buildfile }}
+build ${builddir}/cxx.defs : dump_cxx_defs | {{ buildfile }}
+build ${builddir}/cc.run : dump_cc_run | {{ buildfile }}
+build ${builddir}/cxx.run : dump_cxx_run | {{ buildfile }}
+

scripts/templates/target.host.j2

@@ -0,0 +1,43 @@
+{% extends "target.default.j2" %}
+
+{% block binaries %}
+cc = ${srcroot}/sysroot/bin/clang
+cxx = ${srcroot}/sysroot/bin/clang++
+ld = ${srcroot}/sysroot/bin/x86_64-elf-ld
+ar = ${srcroot}/sysroot/bin/x86_64-elf-ar
+nasm = ${srcroot}/sysroot/bin/nasm
+objcopy = ${srcroot}/sysroot/bin/x86_64-elf-objcopy
+{% endblock %}
+
+{% block variables %}
+
+ccflags = $ccflags $
+    -nostdlib $
+    -ffreestanding $
+    -nodefaultlibs $
+    -fno-builtin $
+    -mno-sse $
+    -fno-omit-frame-pointer $
+    -mno-red-zone $
+    -g $
+    -mcmodel=large $
+    -D__ELF__ $
+    -D__POPCORN__ $
+    -isystem${srcroot}/sysroot/include $
+    --sysroot="${srcroot}/sysroot"
+
+cxxflags = $cxxflags $
+    -fno-exceptions $
+    -fno-rtti $
+    -isystem${srcroot}/sysroot/include/c++/v1
+
+ldflags = $ldflags $
+    -g $
+    -nostdlib $
+    -znocombreloc $
+    -Bsymbolic $
+    -nostartfiles $
+    -Bstatic
+
+{% endblock %}
+# vim: et ts=4 sts=4 sw=4

scripts/templates/target.native.j2

@@ -0,0 +1,8 @@
+{% extends "target.default.j2" %}
+
+{% block variables %}
+
+ccflags = $ccflags -g -ggdb
+
+{% endblock %}
+# vim: et ts=4 sts=4 sw=4

scripts/vmem_translate.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+
+def translate(i4 = 0, i3 = 0, i2 = 0, i1 = 0, offset = 0):
+    addr = (i4 << 39) + (i3 << 30) + (i2 << 21) + (i1 << 12) + offset
+    if addr & (1 << 47):
+        addr |= 0xffff000000000000
+    return addr
+
+if __name__ == "__main__":
+    import sys
+    print("{:016x}".format(translate(*map(int, sys.argv[1:]))))
+

src/arch/x86_64/kernel.ld

@@ -1,7 +1,7 @@
 ENTRY(_start)
 SECTIONS
 {
-	OFFSET = 0xFFFF800000000000;
+	OFFSET = 0xFFFFFF0000000000;
 	. = OFFSET + 0x100000;
 
 	.header : {
@@ -27,11 +27,24 @@ SECTIONS
 		*(.note.*)
 	}
 
-	.bss ALIGN(0x1000) : {
+	.bss ALIGN(16) : {
 		__bss_start = .;
 		*(.bss)
 		__bss_end = .;
 	}
 
+	.eh_frame : {
+		__eh_frame_start = .;
+		KEEP(*(.eh_frame))
+		__eh_frame_end = .;
+	}
+
+	.eh_frame_hdr : {
+		KEEP(*(.eh_frame_hdr))
+	}
+
+	__eh_frame_hdr_start = SIZEOF(.eh_frame_hdr) > 0 ? ADDR(.eh_frame_hdr) : 0;
+	__eh_frame_hdr_end = SIZEOF(.eh_frame_hdr) > 0 ? . : 0;
+
 	kernel_end = ALIGN(4096);
 }

src/boot/console.cpp

@@ -1,4 +1,3 @@
-#include <stdarg.h>
 #include <stddef.h>
 #include <stdint.h>
 
@@ -9,16 +8,55 @@
 size_t ROWS = 0;
 size_t COLS = 0;
 
-static EFI_SIMPLE_TEXT_OUT_PROTOCOL *con_out = 0;
+static EFI_SIMPLE_TEXT_OUT_PROTOCOL *m_out = 0;
 
-const CHAR16 digits[] = {u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9', u'a', u'b', u'c', u'd', u'e', u'f'};
+static const wchar_t digits[] = {u'0', u'1', u'2', u'3', u'4', u'5',
+	u'6', u'7', u'8', u'9', u'a', u'b', u'c', u'd', u'e', u'f'};
+
+console::console(EFI_SYSTEM_TABLE *system_table) :
+	m_rows(0),
+	m_cols(0),
+	m_out(nullptr)
+{
+	s_console = this;
+	m_boot = system_table->BootServices;
+	m_out = system_table->ConOut;
+}
 
 EFI_STATUS
-con_pick_mode(EFI_BOOT_SERVICES *bootsvc)
+console::initialize(const wchar_t *version)
+{
+	EFI_STATUS status;
+
+	// Might not find a video device at all, so ignore not found errors
+	status = pick_mode();
+	if (status != EFI_NOT_FOUND)
+		CHECK_EFI_STATUS_OR_FAIL(status);
+
+	status = m_out->QueryMode(m_out, m_out->Mode->Mode, &m_cols, &m_rows);
+	CHECK_EFI_STATUS_OR_RETURN(status, "QueryMode");
+
+	status = m_out->ClearScreen(m_out);
+	CHECK_EFI_STATUS_OR_RETURN(status, "ClearScreen");
+
+	m_out->SetAttribute(m_out, EFI_LIGHTCYAN);
+	m_out->OutputString(m_out, (wchar_t *)L"Popcorn loader ");
+
+	m_out->SetAttribute(m_out, EFI_LIGHTMAGENTA);
+	m_out->OutputString(m_out, (wchar_t *)version);
+
+	m_out->SetAttribute(m_out, EFI_LIGHTGRAY);
+	m_out->OutputString(m_out, (wchar_t *)L" booting...\r\n\n");
+
+	return status;
+}
+
+EFI_STATUS
+console::pick_mode()
 {
 	EFI_STATUS status;
 	EFI_GRAPHICS_OUTPUT_PROTOCOL *gfx_out_proto;
-	status = bootsvc->LocateProtocol(&guid_gfx_out, NULL, (void **)&gfx_out_proto);
+	status = m_boot->LocateProtocol(&guid_gfx_out, NULL, (void **)&gfx_out_proto);
 	CHECK_EFI_STATUS_OR_RETURN(status, "LocateProtocol gfx");
 
 	const uint32_t modes = gfx_out_proto->Mode->MaxMode;
@@ -53,106 +91,72 @@ con_pick_mode(EFI_BOOT_SERVICES *bootsvc)
 	return EFI_SUCCESS;
 }
 
-EFI_STATUS
-con_initialize(EFI_SYSTEM_TABLE *system_table, const CHAR16 *version)
-{
-	EFI_STATUS status;
-
-	EFI_BOOT_SERVICES *bootsvc = system_table->BootServices;
-	con_out = system_table->ConOut;
-
-	// Might not find a video device at all, so ignore not found errors
-	status = con_pick_mode(bootsvc);
-	if (status != EFI_NOT_FOUND)
-		CHECK_EFI_STATUS_OR_RETURN(status, "con_pick_mode");
-
-	status = con_out->QueryMode(con_out, con_out->Mode->Mode, &COLS, &ROWS);
-	CHECK_EFI_STATUS_OR_RETURN(status, "QueryMode");
-
-	status = con_out->ClearScreen(con_out);
-	CHECK_EFI_STATUS_OR_RETURN(status, "ClearScreen");
-
-	con_out->SetAttribute(con_out, EFI_LIGHTCYAN);
-	con_out->OutputString(con_out, (CHAR16 *)L"Popcorn loader ");
-
-	con_out->SetAttribute(con_out, EFI_LIGHTMAGENTA);
-	con_out->OutputString(con_out, (CHAR16 *)version);
-
-	con_out->SetAttribute(con_out, EFI_LIGHTGRAY);
-	con_out->OutputString(con_out, (CHAR16 *)L" booting...\r\n\n");
-
-	return status;
-}
-
 size_t
-con_print_hex(uint32_t n)
+console::print_hex(uint32_t n) const
 {
-	CHAR16 buffer[9];
+	wchar_t buffer[9];
-	CHAR16 *p = buffer;
+	wchar_t *p = buffer;
 	for (int i = 7; i >= 0; --i) {
 		uint8_t nibble = (n & (0xf << (i*4))) >> (i*4);
 		*p++ = digits[nibble];
 	}
 	*p = 0;
-	con_out->OutputString(con_out, buffer);
+	m_out->OutputString(m_out, buffer);
 	return 8;
 }
 
 size_t
-con_print_long_hex(uint64_t n)
+console::print_long_hex(uint64_t n) const
 {
-	CHAR16 buffer[17];
+	wchar_t buffer[17];
-	CHAR16 *p = buffer;
+	wchar_t *p = buffer;
 	for (int i = 15; i >= 0; --i) {
 		uint8_t nibble = (n & (0xf << (i*4))) >> (i*4);
 		*p++ = digits[nibble];
 	}
 	*p = 0;
-	con_out->OutputString(con_out, buffer);
+	m_out->OutputString(m_out, buffer);
 	return 16;
 }
 
 size_t
-con_print_dec(uint32_t n)
+console::print_dec(uint32_t n) const
 {
-	CHAR16 buffer[11];
+	wchar_t buffer[11];
-	CHAR16 *p = buffer + 10;
+	wchar_t *p = buffer + 10;
 	*p-- = 0;
 	do {
 		*p-- = digits[n % 10];
 		n /= 10;
 	} while (n != 0);
 
-	con_out->OutputString(con_out, ++p);
+	m_out->OutputString(m_out, ++p);
 	return 10 - (p - buffer);
 }
 
 size_t
-con_print_long_dec(uint64_t n)
+console::print_long_dec(uint64_t n) const
 {
-	CHAR16 buffer[21];
+	wchar_t buffer[21];
-	CHAR16 *p = buffer + 20;
+	wchar_t *p = buffer + 20;
 	*p-- = 0;
 	do {
 		*p-- = digits[n % 10];
 		n /= 10;
 	} while (n != 0);
 
-	con_out->OutputString(con_out, ++p);
+	m_out->OutputString(m_out, ++p);
 	return 20 - (p - buffer);
 }
 
 size_t
-con_printf(const CHAR16 *fmt, ...)
+console::vprintf(const wchar_t *fmt, va_list args) const
 {
-	CHAR16 buffer[256];
+	wchar_t buffer[256];
-	const CHAR16 *r = fmt;
+	const wchar_t *r = fmt;
-	CHAR16 *w = buffer;
+	wchar_t *w = buffer;
-	va_list args;
 	size_t count = 0;
 
-	va_start(args, fmt);
-
 	while (r && *r) {
 		if (*r != L'%') {
 			count++;
@@ -161,43 +165,43 @@ con_printf(const CHAR16 *fmt, ...)
 		}
 
 		*w = 0;
-		con_out->OutputString(con_out, buffer);
+		m_out->OutputString(m_out, buffer);
 		w = buffer;
 
 		r++; // chomp the %
 
 		switch (*r++) {
 			case L'%':
-				con_out->OutputString(con_out, L"%");
+				m_out->OutputString(m_out, const_cast<wchar_t*>(L"%"));
 				count++;
 				break;
 
 			case L'x':
-				count += con_print_hex(va_arg(args, uint32_t));
+				count += print_hex(va_arg(args, uint32_t));
 				break;
 
 			case L'd':
 			case L'u':
-				count += con_print_dec(va_arg(args, uint32_t));
+				count += print_dec(va_arg(args, uint32_t));
 				break;
 
 			case L's':
 				{
-					CHAR16 *s = va_arg(args, CHAR16*);
+					wchar_t *s = va_arg(args, wchar_t*);
 					count += wstrlen(s);
-					con_out->OutputString(con_out, s);
+					m_out->OutputString(m_out, s);
 				}
 				break;
 
 			case L'l':
 				switch (*r++) {
 					case L'x':
-						count += con_print_long_hex(va_arg(args, uint64_t));
+						count += print_long_hex(va_arg(args, uint64_t));
 						break;
 
 					case L'd':
 					case L'u':
-						count += con_print_long_dec(va_arg(args, uint64_t));
+						count += print_long_dec(va_arg(args, uint64_t));
 						break;
 
 					default:
@@ -211,44 +215,66 @@ con_printf(const CHAR16 *fmt, ...)
 	}
 
 	*w = 0;
-	con_out->OutputString(con_out, buffer);
+	m_out->OutputString(m_out, buffer);
-
-	va_end(args);
 	return count;
 }
 
-void
+size_t
-con_status_begin(const CHAR16 *message)
+console::printf(const wchar_t *fmt, ...) const
 {
-	con_out->SetAttribute(con_out, EFI_LIGHTGRAY);
+	va_list args;
-	con_out->OutputString(con_out, (CHAR16 *)message);
+	va_start(args, fmt);
+
+	size_t result = vprintf(fmt, args);
+
+	va_end(args);
+	return result;
+}
+
+size_t
+console::print(const wchar_t *fmt, ...)
+{
+	va_list args;
+	va_start(args, fmt);
+
+	size_t result = get().vprintf(fmt, args);
+
+	va_end(args);
+	return result;
 }
 
 void
-con_status_ok()
+console::status_begin(const wchar_t *message) const
 {
-	con_out->SetAttribute(con_out, EFI_LIGHTGRAY);
+	m_out->SetAttribute(m_out, EFI_LIGHTGRAY);
-	con_out->OutputString(con_out, (CHAR16 *)L"[");
+	m_out->OutputString(m_out, (wchar_t *)message);
-	con_out->SetAttribute(con_out, EFI_GREEN);
-	con_out->OutputString(con_out, (CHAR16 *)L"  ok  ");
-	con_out->SetAttribute(con_out, EFI_LIGHTGRAY);
-	con_out->OutputString(con_out, (CHAR16 *)L"]\r\n");
 }
 
 void
-con_status_fail(const CHAR16 *error)
+console::status_ok() const
 {
-	con_out->SetAttribute(con_out, EFI_LIGHTGRAY);
+	m_out->SetAttribute(m_out, EFI_LIGHTGRAY);
-	con_out->OutputString(con_out, (CHAR16 *)L"[");
+	m_out->OutputString(m_out, (wchar_t *)L"[");
-	con_out->SetAttribute(con_out, EFI_LIGHTRED);
+	m_out->SetAttribute(m_out, EFI_GREEN);
-	con_out->OutputString(con_out, (CHAR16 *)L"failed");
+	m_out->OutputString(m_out, (wchar_t *)L"  ok  ");
-	con_out->SetAttribute(con_out, EFI_LIGHTGRAY);
+	m_out->SetAttribute(m_out, EFI_LIGHTGRAY);
-	con_out->OutputString(con_out, (CHAR16 *)L"]\r\n");
+	m_out->OutputString(m_out, (wchar_t *)L"]\r\n");
+}
 
-	con_out->SetAttribute(con_out, EFI_RED);
+void
-	con_out->OutputString(con_out, (CHAR16 *)error);
+console::status_fail(const wchar_t *error) const
-	con_out->SetAttribute(con_out, EFI_LIGHTGRAY);
+{
-	con_out->OutputString(con_out, (CHAR16 *)L"\r\n");
+	m_out->SetAttribute(m_out, EFI_LIGHTGRAY);
+	m_out->OutputString(m_out, (wchar_t *)L"[");
+	m_out->SetAttribute(m_out, EFI_LIGHTRED);
+	m_out->OutputString(m_out, (wchar_t *)L"failed");
+	m_out->SetAttribute(m_out, EFI_LIGHTGRAY);
+	m_out->OutputString(m_out, (wchar_t *)L"]\r\n");
+
+	m_out->SetAttribute(m_out, EFI_RED);
+	m_out->OutputString(m_out, (wchar_t *)error);
+	m_out->SetAttribute(m_out, EFI_LIGHTGRAY);
+	m_out->OutputString(m_out, (wchar_t *)L"\r\n");
 }
 
 EFI_STATUS

src/boot/console.h

@@ -1,12 +1,38 @@
 #pragma once
+#include <stdarg.h>
 #include <stddef.h>
 #include <efi/efi.h>
 
-EFI_STATUS con_initialize(EFI_SYSTEM_TABLE *system_table, const CHAR16 *version);
+class console
-void con_status_begin(const CHAR16 *message);
+{
-void con_status_ok();
+public:
-void con_status_fail(const CHAR16 *error);
+	console(EFI_SYSTEM_TABLE *system_table);
-size_t con_printf(const CHAR16 *fmt, ...);
+
+	EFI_STATUS initialize(const wchar_t *version);
+
+	void status_begin(const wchar_t *message) const;
+	void status_fail(const wchar_t *error) const;
+	void status_ok() const;
+
+	size_t print_hex(uint32_t n) const;
+	size_t print_dec(uint32_t n) const;
+	size_t print_long_hex(uint64_t n) const;
+	size_t print_long_dec(uint64_t n) const;
+	size_t printf(const wchar_t *fmt, ...) const;
+
+	static const console & get() { return *s_console; }
+	static size_t print(const wchar_t *fmt, ...);
+
+private:
+	EFI_STATUS pick_mode();
+	size_t vprintf(const wchar_t *fmt, va_list args) const;
+
+	size_t m_rows, m_cols;
+	EFI_BOOT_SERVICES *m_boot;
+	EFI_SIMPLE_TEXT_OUT_PROTOCOL *m_out;
+
+	static console *s_console;
+};
 
 EFI_STATUS
 con_get_framebuffer(

src/boot/loader.cpp

@@ -6,21 +6,28 @@
 
 #define PAGE_SIZE 0x1000
 
-static CHAR16 kernel_name[] = KERNEL_FILENAME;
+static wchar_t kernel_name[] = KERNEL_FILENAME;
-static CHAR16 font_name[] = KERNEL_FONT;
+static wchar_t initrd_name[] = INITRD_FILENAME;
 
 EFI_STATUS
 loader_alloc_pages(
 	EFI_BOOT_SERVICES *bootsvc,
 	EFI_MEMORY_TYPE mem_type,
 	size_t *length,
-	void **pages)
+	void **pages,
+	bool align)
 {
 	EFI_STATUS status;
 
 	size_t page_count = ((*length - 1) / PAGE_SIZE) + 1;
 	EFI_PHYSICAL_ADDRESS addr = (EFI_PHYSICAL_ADDRESS)*pages;
 
+	if (align) {
+		// Align addr to the next multiple of N pages
+		size_t align_size = page_count * PAGE_SIZE;
+		addr = ((addr - 1) & ~(align_size - 1)) + align_size;
+	}
+
 	status = bootsvc->AllocatePages(AllocateAddress, mem_type, page_count, &addr);
 	if (status == EFI_NOT_FOUND || status == EFI_OUT_OF_RESOURCES) {
 		// couldn't get the address we wanted, try loading the kernel anywhere
@@ -38,7 +45,7 @@ loader_alloc_pages(
 }
 
 EFI_STATUS
-loader_load_font(
+loader_load_initrd(
 	EFI_BOOT_SERVICES *bootsvc,
 	EFI_FILE_PROTOCOL *root,
 	struct loader_data *data)
@@ -46,13 +53,13 @@ loader_load_font(
 	EFI_STATUS status;
 
 	EFI_FILE_PROTOCOL *file = NULL;
-	status = root->Open(root, &file, (CHAR16 *)font_name, EFI_FILE_MODE_READ,
+	status = root->Open(root, &file, (wchar_t *)initrd_name, EFI_FILE_MODE_READ,
 						EFI_FILE_READ_ONLY | EFI_FILE_HIDDEN | EFI_FILE_SYSTEM);
 
 	if (status == EFI_NOT_FOUND)
 		return status;
 
-	CHECK_EFI_STATUS_OR_RETURN(status, L"Opening file %s", font_name);
+	CHECK_EFI_STATUS_OR_RETURN(status, L"Opening file %s", initrd_name);
 
 	char info[sizeof(EFI_FILE_INFO) + 100];
 	size_t info_length = sizeof(info);
@@ -60,16 +67,17 @@ loader_load_font(
 	status = file->GetInfo(file, &guid_file_info, &info_length, info);
 	CHECK_EFI_STATUS_OR_RETURN(status, L"Getting file info");
 
-	data->font_length = ((EFI_FILE_INFO *)info)->FileSize;
+	data->initrd_length = ((EFI_FILE_INFO *)info)->FileSize;
 
 	status = loader_alloc_pages(
 			bootsvc,
-			KERNEL_FONT_MEMTYPE,
+			memtype_initrd,
-			&data->font_length,
+			&data->initrd_length,
-			&data->font);
+			&data->initrd,
+			true);
 	CHECK_EFI_STATUS_OR_RETURN(status, L"Allocating pages");
 
-	status = file->Read(file, &data->font_length, data->font);
+	status = file->Read(file, &data->initrd_length, data->initrd);
 	CHECK_EFI_STATUS_OR_RETURN(status, L"Reading file");
 
 	status = file->Close(file);
@@ -87,8 +95,10 @@ loader_load_elf(
 {
 	EFI_STATUS status;
 
+	con_debug(L"Opening kernel file %s\r\n", (wchar_t *)kernel_name);
+
 	EFI_FILE_PROTOCOL *file = NULL;
-	status = root->Open(root, &file, (CHAR16 *)kernel_name, EFI_FILE_MODE_READ,
+	status = root->Open(root, &file, (wchar_t *)kernel_name, EFI_FILE_MODE_READ,
 						EFI_FILE_READ_ONLY | EFI_FILE_HIDDEN | EFI_FILE_SYSTEM);
 
 	if (status == EFI_NOT_FOUND)
@@ -107,6 +117,8 @@ loader_load_elf(
 	status = file->Read(file, &length, &header);
 	CHECK_EFI_STATUS_OR_RETURN(status, L"Reading ELF header");
 
+	con_debug(L"Read %u bytes of ELF header\r\n", length);
+
 	if (length < sizeof(struct elf_header))
 		CHECK_EFI_STATUS_OR_RETURN(EFI_LOAD_ERROR, L"Incomplete read of ELF header");
 
@@ -131,10 +143,13 @@ loader_load_elf(
 		header.machine != 0x3e)
 		CHECK_EFI_STATUS_OR_RETURN(EFI_LOAD_ERROR, L"ELF load error: wrong machine architecture");
 
+	con_debug(L"ELF is valid, entrypoint %lu\r\n", header.entrypoint);
+
 	data->kernel_entry = (void *)header.entrypoint;
 
 	struct elf_program_header prog_header;
 	for (int i = 0; i < header.ph_num; ++i) {
+
 		status = file->SetPosition(file, header.ph_offset + i * header.ph_entsize);
 		CHECK_EFI_STATUS_OR_RETURN(status, L"Setting ELF file position");
 
@@ -146,13 +161,14 @@ loader_load_elf(
 
 		length = prog_header.mem_size;
 		void *addr = (void *)(prog_header.vaddr - KERNEL_VIRT_ADDRESS);
-		status = loader_alloc_pages(bootsvc, KERNEL_MEMTYPE, &length, &addr);
+		status = loader_alloc_pages(bootsvc, memtype_kernel, &length, &addr, false);
 		CHECK_EFI_STATUS_OR_RETURN(status, L"Allocating kernel pages");
 
 		if (data->kernel == 0)
 			data->kernel = addr;
 		data->kernel_length = (uint64_t)addr + length - (uint64_t)data->kernel;
 	}
+	con_debug(L"Read %d ELF program headers\r\n", header.ph_num);
 
 	struct elf_section_header sec_header;
 	for (int i = 0; i < header.sh_num; ++i) {
@@ -163,7 +179,9 @@ loader_load_elf(
 		status = file->Read(file, &length, &sec_header);
 		CHECK_EFI_STATUS_OR_RETURN(status, L"Reading ELF section header");
 
-		if ((sec_header.flags & ELF_SHF_ALLOC) == 0) continue;
+		if ((sec_header.flags & ELF_SHF_ALLOC) == 0) {
+			continue;
+		}
 
 		void *addr = (void *)(sec_header.addr - KERNEL_VIRT_ADDRESS);
 
@@ -178,6 +196,7 @@ loader_load_elf(
 			bootsvc->SetMem(addr, sec_header.size, 0);
 		}
 	}
+	con_debug(L"Read %d ELF section headers\r\n", header.ph_num);
 
 	status = file->Close(file);
 	CHECK_EFI_STATUS_OR_RETURN(status, L"Closing file handle");
@@ -215,20 +234,21 @@ loader_load_kernel(
 		if (status == EFI_NOT_FOUND)
 			continue;
 
-		CHECK_EFI_STATUS_OR_RETURN(status, L"loader_load_file: %s", kernel_name);
+		CHECK_EFI_STATUS_OR_RETURN(status, L"loader_load_elf: %s", kernel_name);
 
-		data->font = (void *)((uint64_t)data->kernel + data->kernel_length);
+		data->initrd = (void *)((uint64_t)data->kernel + data->kernel_length);
-		status = loader_load_font(bootsvc, root, data);
+		status = loader_load_initrd(bootsvc, root, data);
 
-		CHECK_EFI_STATUS_OR_RETURN(status, L"loader_load_file: %s", font_name);
+		CHECK_EFI_STATUS_OR_RETURN(status, L"loader_load_file: %s", initrd_name);
 
-		data->data = (void *)((uint64_t)data->font + data->font_length);
+		data->data = (void *)((uint64_t)data->initrd + data->initrd_length);
 		data->data_length += PAGE_SIZE; // extra page for map growth
 		status = loader_alloc_pages(
 				bootsvc,
-				KERNEL_DATA_MEMTYPE,
+				memtype_data,
 				&data->data_length,
-				&data->data);
+				&data->data,
+				true);
 		CHECK_EFI_STATUS_OR_RETURN(status, L"loader_alloc_pages: kernel data");
 
 		return EFI_SUCCESS;

src/boot/loader.h

@@ -9,35 +9,15 @@
 #endif
 
 #ifndef KERNEL_VIRT_ADDRESS
-#define KERNEL_VIRT_ADDRESS 0xFFFF800000000000
+#define KERNEL_VIRT_ADDRESS 0xFFFFFF0000000000
-#endif
-
-#ifndef VIRTUAL_OFFSET
-#define VIRTUAL_OFFSET 0xf00000000
-#endif
-
-#ifndef KERNEL_MEMTYPE
-#define KERNEL_MEMTYPE 0x80000000
-#endif
-
-#ifndef KERNEL_FONT_MEMTYPE
-#define KERNEL_FONT_MEMTYPE 0x80000001
-#endif
-
-#ifndef KERNEL_DATA_MEMTYPE
-#define KERNEL_DATA_MEMTYPE 0x80000002
-#endif
-
-#ifndef KERNEL_PT_MEMTYPE
-#define KERNEL_PT_MEMTYPE 0x80000004
 #endif
 
 #ifndef KERNEL_FILENAME
 #define KERNEL_FILENAME L"kernel.elf"
 #endif
 
-#ifndef KERNEL_FONT
+#ifndef INITRD_FILENAME
-#define KERNEL_FONT L"screenfont.psf"
+#define INITRD_FILENAME L"initrd.img"
 #endif
 
 struct loader_data {
@@ -45,8 +25,8 @@ struct loader_data {
 	void *kernel_entry;
 	size_t kernel_length;
 
-	void *font;
+	void *initrd;
-	size_t font_length;
+	size_t initrd_length;
 
 	void *data;
 	size_t data_length;

src/boot/main.cpp

@@ -10,6 +10,10 @@
 #include "memory.h"
 #include "utility.h"
 
+#ifndef SCRATCH_PAGES
+#define SCRATCH_PAGES 64
+#endif
+
 #ifndef GIT_VERSION_WIDE
 #define GIT_VERSION_WIDE L"no version"
 #endif
@@ -30,20 +34,24 @@ struct kernel_header {
 };
 #pragma pack(pop)
 
-EFI_STATUS
+using kernel_entry = void (*)(popcorn_data *);
+
+extern "C" EFI_STATUS
 efi_main(EFI_HANDLE image_handle, EFI_SYSTEM_TABLE *system_table)
 {
 	EFI_STATUS status;
 	EFI_BOOT_SERVICES *bootsvc = system_table->BootServices;
 	EFI_RUNTIME_SERVICES *runsvc = system_table->RuntimeServices;
 
+	console con(system_table);
+
 	// When checking console initialization, use CHECK_EFI_STATUS_OR_RETURN
 	// because we can't be sure if the console was fully set up
-	status = con_initialize(system_table, GIT_VERSION_WIDE);
+	status = con.initialize(GIT_VERSION_WIDE);
-	CHECK_EFI_STATUS_OR_RETURN(status, "con_initialize");
+	CHECK_EFI_STATUS_OR_RETURN(status, "console::initialize");
 	// From here on out, we can use CHECK_EFI_STATUS_OR_FAIL instead
 
-	memory_init_pointer_fixup(bootsvc, runsvc);
+	memory_init_pointer_fixup(bootsvc, runsvc, SCRATCH_PAGES);
 
 	// Find ACPI tables. Ignore ACPI 1.0 if a 2.0 table is found.
 	//
@@ -65,8 +73,8 @@ efi_main(EFI_HANDLE image_handle, EFI_SYSTEM_TABLE *system_table)
 	status = memory_get_map_length(bootsvc, &data_length);
 	CHECK_EFI_STATUS_OR_FAIL(status);
 
-	size_t header_size = sizeof(struct popcorn_data);
+	size_t header_size = sizeof(popcorn_data);
-	const size_t header_align = alignof(struct popcorn_data);
+	const size_t header_align = alignof(popcorn_data);
 	if (header_size % header_align)
 		header_size += header_align - (header_size % header_align);
 
@@ -75,29 +83,30 @@ efi_main(EFI_HANDLE image_handle, EFI_SYSTEM_TABLE *system_table)
 
 	// Load the kernel image from disk and check it
 	//
-	con_printf(L"Loading kernel into memory...\r\n");
+	console::print(L"Loading kernel into memory...\r\n");
 
 	struct loader_data load;
 	load.data_length = data_length;
 	status = loader_load_kernel(bootsvc, &load);
 	CHECK_EFI_STATUS_OR_FAIL(status);
 
-	con_printf(L"    %u image bytes at 0x%x\r\n", load.kernel_length, load.kernel);
+	console::print(L"    %u image bytes at 0x%x\r\n", load.kernel_length, load.kernel);
-	con_printf(L"    %u font bytes at 0x%x\r\n", load.font_length, load.font);
+	console::print(L"    %u initrd bytes at 0x%x\r\n", load.initrd_length, load.initrd);
-	con_printf(L"    %u data bytes at 0x%x\r\n", load.data_length, load.data);
+	console::print(L"    %u data bytes at 0x%x\r\n", load.data_length, load.data);
 
 	struct kernel_header *version = (struct kernel_header *)load.kernel;
 	if (version->magic != KERNEL_HEADER_MAGIC) {
-		con_printf(L"    bad magic %x\r\n", version->magic);
+		console::print(L"    bad magic %x\r\n", version->magic);
 		CHECK_EFI_STATUS_OR_FAIL(EFI_CRC_ERROR);
 	}
 
-	con_printf(L"    Kernel version %d.%d.%d %x%s\r\n",
+	console::print(L"    Kernel version %d.%d.%d %x%s\r\n",
 			version->major, version->minor, version->patch, version->gitsha & 0x0fffffff,
 			version->gitsha & 0xf0000000 ? "*" : "");
-	con_printf(L"    Entrypoint 0x%x\r\n", load.kernel_entry);
+	console::print(L"    Entrypoint 0x%x\r\n", load.kernel_entry);
 
-	void (*kernel_main)() = load.kernel_entry;
+	kernel_entry kernel_main =
+		reinterpret_cast<kernel_entry>(load.kernel_entry);
 	memory_mark_pointer_fixup((void **)&kernel_main);
 
 	// Set up the kernel data pages to pass to the kernel
@@ -109,11 +118,12 @@ efi_main(EFI_HANDLE image_handle, EFI_SYSTEM_TABLE *system_table)
 	data_header->version = DATA_HEADER_VERSION;
 	data_header->length = sizeof(struct popcorn_data);
 
+	data_header->scratch_pages = SCRATCH_PAGES;
 	data_header->flags = 0;
 
-	data_header->font = load.font;
+	data_header->initrd = load.initrd;
-	data_header->font_length = load.font_length;
+	data_header->initrd_length = load.initrd_length;
-	memory_mark_pointer_fixup((void **)&data_header->font);
+	memory_mark_pointer_fixup((void **)&data_header->initrd);
 
 	data_header->data = load.data;
 	data_header->data_length = load.data_length;
@@ -148,8 +158,9 @@ efi_main(EFI_HANDLE image_handle, EFI_SYSTEM_TABLE *system_table)
 	// Save the memory map and tell the firmware we're taking control.
 	//
 	struct memory_map map;
-	map.entries = data_header->memory_map;
 	map.length = (load.data_length - header_size);
+	map.entries =
+		reinterpret_cast<EFI_MEMORY_DESCRIPTOR *>(data_header->memory_map);
 
 	status = memory_get_map(bootsvc, &map);
 	CHECK_EFI_STATUS_OR_FAIL(status);

src/boot/memory.cpp

@@ -4,13 +4,18 @@
 #include "memory.h"
 #include "utility.h"
 
+const EFI_MEMORY_TYPE memtype_kernel  = static_cast<EFI_MEMORY_TYPE>(0x80000000);
+const EFI_MEMORY_TYPE memtype_data    = static_cast<EFI_MEMORY_TYPE>(0x80000001);
+const EFI_MEMORY_TYPE memtype_initrd  = static_cast<EFI_MEMORY_TYPE>(0x80000002);
+const EFI_MEMORY_TYPE memtype_scratch = static_cast<EFI_MEMORY_TYPE>(0x80000003);
+
 #define INCREMENT_DESC(p, b)  (EFI_MEMORY_DESCRIPTOR*)(((uint8_t*)(p))+(b))
 
 size_t fixup_pointer_index = 0;
 void **fixup_pointers[64];
 uint64_t *new_pml4 = 0;
 
-const CHAR16 *memory_type_names[] = {
+const wchar_t *memory_type_names[] = {
 	L"EfiReservedMemoryType",
 	L"EfiLoaderCode",
 	L"EfiLoaderData",
@@ -28,13 +33,17 @@ const CHAR16 *memory_type_names[] = {
 	L"EfiPersistentMemory",
 };
 
-static const CHAR16 *
+static const wchar_t *
 memory_type_name(UINT32 value)
 {
-	if (value >= (sizeof(memory_type_names) / sizeof(CHAR16 *))) {
+	if (value >= (sizeof(memory_type_names) / sizeof(wchar_t *))) {
-		if (value == KERNEL_DATA_MEMTYPE) return L"Kernel Data";
+		switch (value) {
-		else if (value == KERNEL_MEMTYPE) return L"Kernel Image";
+			case memtype_kernel:  return L"Kernel Data";
-		else return L"Bad Type Value";
+			case memtype_data:    return L"Kernel Data";
+			case memtype_initrd:  return L"Initial Ramdisk";
+			case memtype_scratch: return L"Kernel Scratch Space";
+			default: return L"Bad Type Value";
+		}
 	}
 	return memory_type_names[value];
 }
@@ -50,7 +59,7 @@ memory_update_marked_addresses(EFI_EVENT UNUSED *event, void *context)
 }
 
 EFI_STATUS
-memory_init_pointer_fixup(EFI_BOOT_SERVICES *bootsvc, EFI_RUNTIME_SERVICES *runsvc)
+memory_init_pointer_fixup(EFI_BOOT_SERVICES *bootsvc, EFI_RUNTIME_SERVICES *runsvc, unsigned scratch_pages)
 {
 	EFI_STATUS status;
 	EFI_EVENT event;
@@ -67,7 +76,7 @@ memory_init_pointer_fixup(EFI_BOOT_SERVICES *bootsvc, EFI_RUNTIME_SERVICES *runs
 	// Reserve a page for our replacement PML4, plus some pages for the kernel to use
 	// as page tables while it gets started.
 	EFI_PHYSICAL_ADDRESS addr = 0;
-	status = bootsvc->AllocatePages(AllocateAnyPages, EfiLoaderData, 16, &addr);
+	status = bootsvc->AllocatePages(AllocateAnyPages, memtype_scratch, scratch_pages, &addr);
 	CHECK_EFI_STATUS_OR_RETURN(status, "Failed to allocate page table pages.");
 
 	new_pml4 = (uint64_t *)addr;
@@ -139,19 +148,19 @@ memory_dump_map(struct memory_map *map)
 
 	const size_t count = map->length / map->size;
 
-	con_printf(L"Memory map:\n");
+	console::print(L"Memory map:\n");
-	con_printf(L"\t	Descriptor Count: %d (%d bytes)\n", count, map->length);
+	console::print(L"\t	Descriptor Count: %d (%d bytes)\n", count, map->length);
-	con_printf(L"\t  Descriptor Size: %d bytes\n", map->size);
+	console::print(L"\t  Descriptor Size: %d bytes\n", map->size);
-	con_printf(L"\t      Type offset: %d\n\n", offsetof(EFI_MEMORY_DESCRIPTOR, Type));
+	console::print(L"\t      Type offset: %d\n\n", offsetof(EFI_MEMORY_DESCRIPTOR, Type));
 
 	EFI_MEMORY_DESCRIPTOR *end = INCREMENT_DESC(map->entries, map->length);
 	EFI_MEMORY_DESCRIPTOR *d = map->entries;
 	while (d < end) {
 		int runtime = (d->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME;
-		con_printf(L"%s%s ", memory_type_name(d->Type), runtime ? L"*" : L" ");
+		console::print(L"%s%s ", memory_type_name(d->Type), runtime ? L"*" : L" ");
-		con_printf(L"%lx ", d->PhysicalStart);
+		console::print(L"%lx ", d->PhysicalStart);
-		con_printf(L"%lx ", d->VirtualStart);
+		console::print(L"%lx ", d->VirtualStart);
-		con_printf(L"[%4d]\n", d->NumberOfPages);
+		console::print(L"[%4d]\n", d->NumberOfPages);
 
 		d = INCREMENT_DESC(d, map->size);
 	}
@@ -187,10 +196,12 @@ memory_virtualize(EFI_RUNTIME_SERVICES *runsvc, struct memory_map *map)
 	EFI_MEMORY_DESCRIPTOR *d = map->entries;
 	while (d < end) {
 		switch (d->Type) {
-		case KERNEL_MEMTYPE:
+		case memtype_kernel:
-		case KERNEL_FONT_MEMTYPE:
+		case memtype_data:
-		case KERNEL_DATA_MEMTYPE:
+		case memtype_initrd:
+		case memtype_scratch:
 			d->Attribute |= EFI_MEMORY_RUNTIME;
+			d->VirtualStart = d->PhysicalStart + KERNEL_VIRT_ADDRESS;
 
 		default:
 			if (d->Attribute & EFI_MEMORY_RUNTIME) {

src/boot/memory.h

@@ -1,6 +1,11 @@
 #pragma once
 #include <efi/efi.h>
 
+extern const EFI_MEMORY_TYPE memtype_kernel;
+extern const EFI_MEMORY_TYPE memtype_data;
+extern const EFI_MEMORY_TYPE memtype_initrd;
+extern const EFI_MEMORY_TYPE memtype_scratch;
+
 struct memory_map {
 	size_t length;
 	size_t size;
@@ -9,7 +14,10 @@ struct memory_map {
 	EFI_MEMORY_DESCRIPTOR *entries;
 };
 
-EFI_STATUS memory_init_pointer_fixup(EFI_BOOT_SERVICES *bootsvc, EFI_RUNTIME_SERVICES *runsvc);
+EFI_STATUS memory_init_pointer_fixup(
+		EFI_BOOT_SERVICES *bootsvc,
+		EFI_RUNTIME_SERVICES *runsvc,
+		unsigned scratch_pages);
 void memory_mark_pointer_fixup(void **p);
 
 EFI_STATUS memory_get_map_length(EFI_BOOT_SERVICES *bootsvc, size_t *size);

src/boot/reloc.cpp

@@ -38,6 +38,7 @@
 #include <efi/efi.h>
 #include <elf.h>
 
+extern "C"
 EFI_STATUS _relocate (long ldbase, Elf64_Dyn *dyn,
 		      EFI_HANDLE image EFI_UNUSED,
 		      EFI_SYSTEM_TABLE *systab EFI_UNUSED)

src/boot/utility.cpp

@@ -2,7 +2,7 @@
 
 struct error_code_desc {
 	EFI_STATUS code;
-	CHAR16 *name;
+	const wchar_t *name;
 };
 
 // Based off the gnu-efi table
@@ -47,7 +47,7 @@ struct error_code_desc error_table[] = {
 	{ 0, NULL }
 };
 
-const CHAR16 *
+const wchar_t *
 util_error_message(EFI_STATUS status)
 {
 	int32_t i = -1;
@@ -62,7 +62,7 @@ util_error_message(EFI_STATUS status)
 }
 
 size_t
-wstrlen(const CHAR16 *s)
+wstrlen(const wchar_t *s)
 {
 	size_t count = 0;
 	while (s && *s++) count++;

src/boot/utility.h

@@ -5,18 +5,18 @@
 
 #define UNUSED __attribute__((unused))
 
-size_t wstrlen(const CHAR16 *s);
+size_t wstrlen(const wchar_t *s);
-const CHAR16 *util_error_message(EFI_STATUS status);
+const wchar_t *util_error_message(EFI_STATUS status);
 
 #define CHECK_EFI_STATUS_OR_RETURN(s, msg, ...) \
 	if (EFI_ERROR((s))) { \
-		con_printf(L"ERROR: " msg L": %s\r\n", ##__VA_ARGS__, util_error_message(s)); \
+		console::print(L"ERROR: " msg L": %s\r\n", ##__VA_ARGS__, util_error_message(s)); \
 		return (s); \
 	}
 
 #define CHECK_EFI_STATUS_OR_FAIL(s) \
 	if (EFI_ERROR((s))) { \
-		con_status_fail(util_error_message(s)); \
+		console::get().status_fail(util_error_message(s)); \
 		while (1) __asm__("hlt"); \
 	}
 
@@ -32,3 +32,9 @@ const CHAR16 *util_error_message(EFI_STATUS status);
 			: "r"((uint64_t)s), "r"((uint64_t)d), "r"((uint64_t)__LINE__) \
 			: "rax", "rdx", "r8", "r9", "r10"); \
 	}
+
+#ifdef BOOTLOADER_DEBUG
+#define con_debug(...) console::print(L"DEBUG: " __VA_ARGS__)
+#else
+#define con_debug(...)
+#endif

src/boot/wscript

@@ -1,61 +0,0 @@
-
-def configure(ctx):
-    from os.path import join
-
-    lds_path = join(ctx.env.ARCH_D, "boot.ld")
-
-    ctx.env.append_value('DEFINES_EFI', [
-        'KERNEL_FILENAME=L"{}"'.format(ctx.env.KERNEL_FILENAME),
-        'GNU_EFI_USE_MS_ABI',
-        'HAVE_USE_MS_ABI',
-        'EFI_DEBUG=0',
-        'EFI_DEBUG_CLEAR_MEMORY=0',
-    ])
-    ctx.env.append_value('CFLAGS_EFI', ['-fPIC', '-fshort-wchar'])
-    ctx.env.append_value('LINKFLAGS_EFI', [
-        '-shared',
-        '-T', lds_path,
-    ])
-    ctx.env.append_value('SECTIONS_EFI', [
-    ])
-
-def build(bld):
-    sources = bld.path.ant_glob("**/*.c")
-    sources += bld.path.ant_glob("**/*.s")
-
-    bld.program(
-        source      = sources,
-        target      = "boot.elf",
-        use         = 'EFI',
-    )
-
-    from waflib.Task import Task
-    class make_efi(Task):
-        color = 'YELLOW'
-        def keyword(self):
-            return "Creating"
-        def __str__(self):
-            node = self.outputs[0]
-            return node.path_from(node.ctx.launch_node())
-        def run(self):
-            from subprocess import check_call as call
-
-            args = self.env.objcopy
-            sections = [".text", ".sdata", ".data", ".dynamic",
-                        ".dynsym", ".rel", ".rela", ".reloc"]
-            for s in sections: args.extend(['-j', s])
-            args.append('--target=efi-app-' + self.env.POPCORN_ARCH)
-            args.append(self.inputs[0].abspath())
-            args.append(self.outputs[0].abspath())
-            call(args)
-
-    src = bld.path
-    out = bld.path.get_bld()
-
-    efi = make_efi(env=bld.env)
-    efi.set_inputs([out.make_node("boot.elf")])
-    efi.set_outputs([out.make_node("boot.efi")])
-    bld.add_to_group(efi)
-
-
-# vim: ft=python et sw=4

src/drivers/ahci/driver.cpp

@@ -3,13 +3,15 @@
 #include "log.h"
 #include "pci.h"
 
+namespace ahci {
 
-ahci_driver::ahci_driver()
+
+driver::driver()
 {
 }
 
 void
-ahci_driver::register_device(pci_device *device)
+driver::register_device(pci_device *device)
 {
 	log::info(logs::driver, "AHCI registering device %d:%d:%d:",
 			device->bus(), device->device(), device->function());
@@ -17,12 +19,4 @@ ahci_driver::register_device(pci_device *device)
 	ahci::hba &hba = m_devices.emplace(device);
 }
 
-ahci::port *
+} // namespace
-ahci_driver::find_disk()
-{
-	for (auto &hba : m_devices) {
-		ahci::port *d = hba.find_disk();
-		if (d) return d;
-	}
-	return nullptr;
-}

src/drivers/ahci/driver.h

@@ -6,13 +6,15 @@
 
 class pci_device;
 
+namespace ahci {
+
 
 /// Basic AHCI driver
-class ahci_driver
+class driver
 {
 public:
 	/// Constructor.
-	ahci_driver();
+	driver();
 
 	/// Register a device with the driver
 	/// \arg device  The PCI device to handle
@@ -22,10 +24,8 @@ public:
 	/// \arg device  The PCI device to remove
 	void unregister_device(pci_device *device);
 
-	/// Debug: find the first disk
-	ahci::port * find_disk();
-
 private:
 	kutil::vector<ahci::hba> m_devices;
 };
 
+} // namespace

src/drivers/ahci/hba.cpp

@@ -1,7 +1,9 @@
 #include <stdint.h>
 #include "ahci/ata.h"
 #include "ahci/hba.h"
+#include "console.h"
 #include "device_manager.h"
+#include "fs/gpt.h"
 #include "log.h"
 #include "page_manager.h"
 #include "pci.h"
@@ -57,10 +59,14 @@ void irq_cb(void *data)
 hba::hba(pci_device *device)
 {
 	page_manager *pm = page_manager::get();
+	device_manager &dm = device_manager::get();
 
 	uint32_t bar5 = device->get_bar(5);
-	m_data = reinterpret_cast<hba_data *>(bar5 & ~0xfffull);
+	log::debug(logs::driver, "HBA raw BAR5 is %08lx", bar5);
-	pm->map_offset_pointer(reinterpret_cast<void **>(&m_data), 0x2000);
+
+	void *data = reinterpret_cast<void *>(bar5 & ~0xfffull);
+	pm->map_offset_pointer(&data, 0x2000);
+	m_data = reinterpret_cast<hba_data volatile *>(data);
 
 	if (! bitfield_has(m_data->cap, hba_cap::ahci_only))
 		m_data->host_control |= 0x80000000; // Enable AHCI mode
@@ -70,17 +76,17 @@ hba::hba(pci_device *device)
 	unsigned ports = (icap & 0xf) + 1;
 	unsigned slots = ((icap >> 8) & 0x1f) + 1;
 
-	log::debug(logs::driver, "  %d ports", ports);
+	log::debug(logs::driver, "  %d ports: %08x", ports, m_data->port_impl);
 	log::debug(logs::driver, "  %d command slots", slots);
 
-	port_data *pd = reinterpret_cast<port_data *>(
+	auto *pd = reinterpret_cast<port_data volatile *>(
 			kutil::offset_pointer(m_data, 0x100));
 
 	bool needs_interrupt = false;
 	m_ports.ensure_capacity(ports);
 	for (unsigned i = 0; i < ports; ++i) {
 		bool impl = ((m_data->port_impl & (1 << i)) != 0);
-		port &p = m_ports.emplace(i, kutil::offset_pointer(pd, 0x80 * i), impl);
+		port &p = m_ports.emplace(this, i, kutil::offset_pointer(pd, 0x80 * i), impl);
 		if (p.get_state() == port::state::active)
 			needs_interrupt = true;
 	}
@@ -89,29 +95,48 @@ hba::hba(pci_device *device)
 		device_manager::get().allocate_msi("AHCI Device", *device, irq_cb, this);
 		m_data->host_control |= 0x02; // enable interrupts
 	}
-}
 
-port *
+	for (auto &p : m_ports) {
-hba::find_disk()
+		if (!p.active()) continue;
-{
+
-	for (auto &port : m_ports) {
+		if (p.get_type() == sata_signature::sata_drive) {
-		if (port.get_state() == port::state::active &&
+			p.sata_reconnect();
-			port.get_type() == sata_signature::sata_drive)
+			/*
-			return &port;
+			if (fs::partition::load(&p) == 0)
+				dm.register_block_device(&p);
+			*/
+		}
 	}
-
-	return nullptr;
 }
 
 void
 hba::handle_interrupt()
 {
+	uint32_t status = m_data->int_status;
 	for (auto &port : m_ports) {
-		if (m_data->int_status & (1 << port.index())) {
+		if (status & (1 << port.index())) {
 			port.handle_interrupt();
 		}
 	}
-	m_data->int_status = 0;
+	// Write 1 to the handled interrupts
+	m_data->int_status = status;
+}
+
+void
+hba::dump()
+{
+	console *cons = console::get();
+	static const char *regs[] = {
+		" CAP", " GHC", "  IS", "  PI", "  VS", " C3C",
+		" C3P", " EML", " EMC", "CAP2", "BOHC"
+	};
+
+	cons->printf("HBA Registers:\n");
+	auto *data = reinterpret_cast<uint32_t volatile *>(m_data);
+	for (int i = 0; i < 11; ++i) {
+		cons->printf("  %s: %08x\n", regs[i], data[i]);
+	}
+	cons->putc('\n');
 }
 
 } // namespace ahci

src/drivers/ahci/hba.h

@@ -25,12 +25,12 @@ public:
 	/// Interrupt handler.
 	void handle_interrupt();
 
-	/// Debug: find the first disk
+	/// Dump the HBA registers to the console
-	port * find_disk();
+	void dump();
 
 private:
 	pci_device *m_device;
-	hba_data *m_data;
+	hba_data volatile *m_data;
 	kutil::vector<port> m_ports;
 };
 

src/drivers/ahci/port.cpp

@@ -2,13 +2,21 @@
 #include "kutil/assert.h"
 #include "kutil/enum_bitfields.h"
 #include "ahci/ata.h"
+#include "ahci/hba.h"
 #include "ahci/fis.h"
 #include "ahci/port.h"
+#include "console.h"
 #include "io.h"
 #include "log.h"
 #include "page_manager.h"
 
+namespace ahci {
+	enum class cmd_list_flags : uint16_t;
+}
+
 IS_BITFIELD(ahci::port_cmd);
+IS_BITFIELD(volatile ahci::port_cmd);
+IS_BITFIELD(ahci::cmd_list_flags);
 
 namespace ahci {
 
@@ -110,10 +118,11 @@ struct port_data
 } __attribute__ ((packed));
 
 
-port::port(uint8_t index, port_data *data, bool impl) :
+port::port(hba *device, uint8_t index, port_data volatile *data, bool impl) :
 	m_index(index),
 	m_type(sata_signature::none),
 	m_state(state::unimpl),
+	m_hba(device),
 	m_data(data),
 	m_fis(nullptr),
 	m_cmd_list(nullptr),
@@ -154,7 +163,13 @@ port::update()
 
 		rebase();
 		m_pending.set_size(32);
-		m_data->interrupt_enable = 1;
+		for (auto &pend : m_pending) {
+			pend.type = command_type::none;
+		}
+
+		// Clear any old pending interrupts and enable interrupts
+		m_data->interrupt_status = m_data->interrupt_status;
+		m_data->interrupt_enable = 0xffffffff;
 	} else {
 		m_state = state::inactive;
 	}
@@ -190,19 +205,27 @@ port::stop_commands()
 }
 
 int
-port::make_command(size_t length)
+port::make_command(size_t length, fis_register_h2d **fis)
 {
 	int slot = -1;
-	uint32_t used_slots = (m_data->serial_active | m_data->cmd_issue);
+	uint32_t used_slots =
+		m_data->serial_active |
+		m_data->cmd_issue |
+		m_data->interrupt_status;
+
 	for (int i = 0; i < 32; ++i) {
-		if ((used_slots & (1 << i)) == 0) {
+		if (used_slots & (1 << i)) continue;
+
+		if (m_pending[i].type == command_type::none) {
 			slot = i;
 			break;
+		} else {
+			log::debug(logs::driver, "Type is %d", m_pending[i].type);
 		}
 	}
 
 	if (slot < 0) {
-		log::info(logs::driver, "AHCI could not get a free command slot.");
+		log::error(logs::driver, "AHCI could not get a free command slot.");
 		return -1;
 	}
 
@@ -214,6 +237,14 @@ port::make_command(size_t length)
 	kutil::memset(&cmdt, 0, sizeof(cmd_table) +
 			max_prd_count * sizeof(prdt_entry));
 
+	ent.flags = cmd_list_fis_size(sizeof(fis_register_h2d));
+
+	fis_register_h2d *cfis = reinterpret_cast<fis_register_h2d *>(&cmdt.cmd_fis);
+	kutil::memset(cfis, 0, sizeof(fis_register_h2d));
+	cfis->type = fis_type::register_h2d;
+	cfis->pm_port = 0x80; // set command register flag
+	*fis = cfis;
+
 	size_t remaining = length;
 	for (int i = 0; i < max_prd_count; ++i) {
 		size_t prd_len = std::min(remaining, 0x200000ul);
@@ -222,7 +253,7 @@ port::make_command(size_t length)
 		void *mem = pm->map_offset_pages(page_count(prd_len));
 		kutil::memset(mem, 0xaf, prd_len);
 
-		addr_t phys = pm->offset_phys(mem);
+		uintptr_t phys = pm->offset_phys(mem);
 		cmdt.entries[i].data_base_low = phys & 0xffffffff;
 		cmdt.entries[i].data_base_high = phys >> 32;
 		cmdt.entries[i].byte_count = prd_len - 1;
@@ -240,21 +271,20 @@ port::make_command(size_t length)
 	return slot;
 }
 
-bool
+int
-port::read(uint64_t sector, size_t length, void *dest)
+port::read_async(uint64_t offset, size_t length, void *dest)
 {
-	int slot = make_command(length);
+	fis_register_h2d *fis;
+	int slot = make_command(length, &fis);
 	if (slot < 0)
-		return false;
+		return 0;
 
 	cmd_table &cmdt = m_cmd_table[slot];
 
-	fis_register_h2d *fis = reinterpret_cast<fis_register_h2d *>(&cmdt.cmd_fis);
-	fis->type = fis_type::register_h2d;
-	fis->pm_port = 0x80; // set command register flag
 	fis->command = ata_cmd::read_dma_ext;
 	fis->device = 0x40; // ATA8-ACS p.175
 
+	uint64_t sector = offset >> 9;
 	fis->lba0 = (sector      ) & 0xff;
 	fis->lba1 = (sector >>  8) & 0xff;
 	fis->lba2 = (sector >> 16) & 0xff;
@@ -270,8 +300,64 @@ port::read(uint64_t sector, size_t length, void *dest)
 			count, sector, sector*512);
 
 	m_pending[slot].type = command_type::read;
+	m_pending[slot].offset = offset % 512;
+	m_pending[slot].count = 0;
 	m_pending[slot].data = dest;
-	return issue_command(slot);
+	if(issue_command(slot))
+		return slot;
+	else
+		return -1;
+}
+
+size_t
+port::read(uint64_t offset, size_t length, void *dest)
+{
+	int slot = read_async(offset, length, dest);
+
+	int timeout = 0;
+	while (m_pending[slot].type == command_type::read) {
+		if (timeout++ > 5) {
+			return 0;
+		}
+		asm("hlt");
+	}
+	kassert(m_pending[slot].type == command_type::finished,
+			"Read got unexpected command type");
+
+	size_t count = m_pending[slot].count;
+	m_pending[slot].type = command_type::none;
+	m_pending[slot].count = 0;
+
+	return count;
+}
+
+int
+port::identify_async()
+{
+	fis_register_h2d *fis;
+	int slot = make_command(512, &fis);
+	if (slot < 0)
+		return 0;
+
+	fis->command = ata_cmd::identify;
+
+	m_pending[slot].type = command_type::identify;
+	m_pending[slot].offset = 0;
+	m_pending[slot].count = 0;
+	m_pending[slot].data = 0;
+
+	if(issue_command(slot))
+		return slot;
+	else
+		return -1;
+}
+
+void
+port::sata_reconnect()
+{
+	m_data->serial_control |= 1;
+	io_wait(1000); // About 1ms
+	m_data->serial_control &= ~1;
 }
 
 bool
@@ -297,14 +383,23 @@ port::issue_command(int slot)
 void
 port::handle_interrupt()
 {
-	log::debug(logs::driver, "AHCI port %d got an interrupt");
+	log::debug(logs::driver, "AHCI port %d got an interrupt", m_index);
 
 	// TODO: handle other states in interrupt_status
 
-	if (m_data->interrupt_status & 0x40000000) {
+	uint32_t is = m_data->interrupt_status;
+
+	if (is & 0x00000040) {
+		// Port connect status change: For now clear the "exchange"
+		// bit in SERR, this should probably kick off diagnostics.
+		m_data->serial_error = 0x04000000;
+		identify_async();
+	}
+
+	if (is & 0x40000000) {
 		log::error(logs::driver, "AHCI task file error");
-		// TODO: clean up!
+		dump();
-		return;
+		kassert(0, "Task file error");
 	}
 
 	log::debug(logs::driver, "AHCI interrupt status: %08lx  %08lx",
@@ -312,19 +407,25 @@ port::handle_interrupt()
 
 	uint32_t ci = m_data->cmd_issue;
 	for (int i = 0; i < 32; ++i) {
+		// Skip commands still listed as "issued"
 		if (ci & (1 << i)) continue;
 
+		// Any commands not still listed as "issued" that are still pending for
+		// the driver are now finished, so handle them.
 		pending &p = m_pending[i];
 		switch (p.type) {
 		case command_type::read:
 			finish_read(i);
 			break;
+		case command_type::identify:
+			finish_identify(i);
+			break;
 		default:
 			break;
 		}
-		p.type = command_type::none;
-		p.data = nullptr;
 	}
+
+	// Clear the whole status register to mark it as handled
 	m_data->interrupt_status = m_data->interrupt_status;
 }
 
@@ -335,21 +436,92 @@ port::finish_read(int slot)
 	cmd_table &cmdt = m_cmd_table[slot];
 	cmd_list_entry &ent = m_cmd_list[slot];
 
+	size_t count = 0;
 	void *p = m_pending[slot].data;
+	uint8_t offset = m_pending[slot].offset;
 	for (int i = 0; i < ent.prd_table_length; ++i) {
 		size_t prd_len = (cmdt.entries[i].byte_count & 0x7fffffff) + 1;
 
-		addr_t phys = 
+		uintptr_t phys =
-			static_cast<addr_t>(cmdt.entries[i].data_base_low) |
+			static_cast<uintptr_t>(cmdt.entries[i].data_base_low) |
-			static_cast<addr_t>(cmdt.entries[i].data_base_high) << 32;
+			static_cast<uintptr_t>(cmdt.entries[i].data_base_high) << 32;
+
+		void *mem = kutil::offset_pointer(pm->offset_virt(phys), offset);
+
+		log::debug(logs::driver, "Reading PRD %2d: %016lx->%016lx [%lxb]", i, mem, p, prd_len);
 
-		void *mem = pm->offset_virt(phys);
 		kutil::memcpy(p, mem, prd_len);
-		p = kutil::offset_pointer(p, prd_len);
+		p = kutil::offset_pointer(p, prd_len - offset);
+		count += (prd_len - offset);
+		offset = 0;
 
 		pm->unmap_pages(mem, page_count(prd_len));
 	}
 
+	m_pending[slot].count = count;
+	m_pending[slot].type = command_type::finished;
+	m_pending[slot].data = nullptr;
+}
+
+static void
+ident_strcpy(uint16_t *from, int words, char *dest)
+{
+	for (int i = 0; i < words; ++i) {
+		*dest++ = *from >> 8;
+		*dest++ = *from & 0xff;
+		from++;
+	}
+	*dest = 0;
+}
+
+void
+port::finish_identify(int slot)
+{
+	page_manager *pm = page_manager::get();
+	cmd_table &cmdt = m_cmd_table[slot];
+	cmd_list_entry &ent = m_cmd_list[slot];
+
+	kassert(ent.prd_table_length == 1, "AHCI identify used multiple PRDs");
+
+	size_t prd_len = (cmdt.entries[0].byte_count & 0x7fffffff) + 1;
+
+	uintptr_t phys =
+		static_cast<uintptr_t>(cmdt.entries[0].data_base_low) |
+		static_cast<uintptr_t>(cmdt.entries[0].data_base_high) << 32;
+
+	log::debug(logs::driver, "Reading ident PRD:");
+
+	uint16_t *mem = reinterpret_cast<uint16_t *>(pm->offset_virt(phys));
+	char string[41];
+
+	ident_strcpy(&mem[10], 10, &string[0]);
+	log::debug(logs::driver, "    Device serial: %s", string);
+
+	ident_strcpy(&mem[23], 4, &string[0]);
+	log::debug(logs::driver, "   Device version: %s", string);
+
+	ident_strcpy(&mem[27], 20, &string[0]);
+	log::debug(logs::driver, "     Device model: %s", string);
+
+	uint32_t sectors = mem[60] | (mem[61] << 16);
+	log::debug(logs::driver, "      Max sectors: %xh", sectors);
+
+	uint16_t lb_size = mem[106];
+	log::debug(logs::driver, " lsects per psect: %d %s %s", 1 << (lb_size & 0xf),
+			lb_size & 0x20 ? "multiple logical per physical" : "",
+			lb_size & 0x10 ? "physical > 512b" : "");
+
+	uint32_t b_per_ls = 2 * (mem[117] | (mem[118] << 16));
+	log::debug(logs::driver, "      b per lsect: %d", b_per_ls);
+
+	/*
+	for (int i=0; i<256; i += 4)
+		log::debug(logs::driver, "  %3d: %04x  %3d: %04x  %3d: %04x  %3d: %04x",
+				i, mem[i], i+1, mem[i+1], i+2, mem[i+2], i+3, mem[i+3]);
+	*/
+
+	pm->unmap_pages(mem, page_count(prd_len));
+	m_pending[slot].type = command_type::none;
 }
 
 void
@@ -363,9 +535,9 @@ port::free_command(int slot)
 	for (int i = 0; i < ent.prd_table_length; ++i) {
 		size_t prd_len = (cmdt.entries[i].byte_count & 0x7fffffff) + 1;
 
-		addr_t phys =
+		uintptr_t phys =
-			static_cast<addr_t>(cmdt.entries[i].data_base_low) |
+			static_cast<uintptr_t>(cmdt.entries[i].data_base_low) |
-			static_cast<addr_t>(cmdt.entries[i].data_base_high) << 32;
+			static_cast<uintptr_t>(cmdt.entries[i].data_base_high) << 32;
 		void *mem = pm->offset_virt(phys);
 		pm->unmap_pages(mem, page_count(prd_len));
 	}
@@ -385,7 +557,7 @@ port::rebase()
 	size_t pages = 1 + page_count(prd_size * 32);
 
 	void *mem = pm->map_offset_pages(pages);
-	addr_t phys = pm->offset_phys(mem);
+	uintptr_t phys = pm->offset_phys(mem);
 
 	log::debug(logs::driver, "Rebasing address for AHCI port %d to %lx [%d]", m_index, mem, pages);
 
@@ -430,4 +602,22 @@ port::rebase()
 	start_commands();
 }
 
+void
+port::dump()
+{
+	console *cons = console::get();
+	static const char *regs[] = {
+		" CLB", "+CLB", "  FB", " +FB", "  IS", "  IE",
+		" CMD", nullptr, " TFD", " SIG", "SSTS", "SCTL", "SERR",
+		"SACT", "  CI", "SNTF", " FBS", "DEVS"
+	};
+
+	cons->printf("Port Registers:\n");
+	auto *data = reinterpret_cast<volatile uint32_t *>(m_data);
+	for (int i = 0; i < 18; ++i) {
+		if (regs[i]) cons->printf("  %s: %08x\n", regs[i], data[i]);
+	}
+	cons->putc('\n');
+}
+
 } // namespace ahci

src/drivers/ahci/port.h

@@ -4,25 +4,30 @@
 #include <stddef.h>
 #include <stdint.h>
 #include "kutil/vector.h"
+#include "block_device.h"
 
 namespace ahci {
 
 struct cmd_list_entry;
 struct cmd_table;
+struct fis_register_h2d;
+class hba;
 enum class sata_signature : uint32_t;
 enum class port_cmd : uint32_t;
 struct port_data;
 
 
 /// A port on an AHCI HBA
-class port
+class port :
+	public block_device
 {
 public:
 	/// Constructor.
-	/// \arg index Index of the port on its HBA
+	/// \arg device The HBA device this port belongs to
-	/// \arg data  Pointer to the device's registers for this port
+	/// \arg index  Index of the port on its HBA
-	/// \arg impl  Whether this port is marked as implemented in the HBA
+	/// \arg data   Pointer to the device's registers for this port
-	port(uint8_t index, port_data *data, bool impl);
+	/// \arg impl   Whether this port is marked as implemented in the HBA
+	port(hba *device, uint8_t index, port_data volatile *data, bool impl);
 
 	/// Destructor
 	~port();
@@ -37,6 +42,10 @@ public:
 	/// \returns  An enum representing the state
 	inline state get_state() const { return m_state; }
 
+	/// Check if this device is active
+	/// \returns  True if the device state is active
+	inline bool active() const { return m_state == state::active; }
+
 	/// Get the type signature of this device
 	/// \returns  An enum representing the type of device
 	inline sata_signature get_type() const { return m_type; }
@@ -53,16 +62,34 @@ public:
 	/// Stop command processing from this port
 	void stop_commands();
 
-	/// Read data from the drive.
+	/// Start a read operation from the drive.
-	/// \arg sector  Starting sector to read
+	/// \arg offset  Offset to start from
 	/// \arg length  Number of bytes to read
 	/// \arg dest    A buffer where the data will be placed
-	/// \returns     True if the command succeeded
+	/// \returns     A handle to the read operation, or -1 on error
-	bool read(uint64_t sector, size_t length, void *dest);
+	int read_async(uint64_t offset, size_t length, void *dest);
+
+	/// Read from the drive, blocking until finished.
+	/// \arg offset  Offset to start from
+	/// \arg length  Number of bytes to read
+	/// \arg dest    A buffer where the data will be placed
+	/// \returns     The number of bytes read
+	virtual size_t read(uint64_t offset, size_t length, void *dest);
+
+	/// Start an identify operation for the drive.
+	/// \returns     A handle to the read operation, or -1 on error
+	int identify_async();
+
+	/// Tell the HBA to reconnect to the SATA device. A successful
+	/// reconnect will kick off an identify command.
+	void sata_reconnect();
 
 	/// Handle an incoming interrupt
 	void handle_interrupt();
 
+	/// Dump the port registers to the console
+	void dump();
+
 private:
 	/// Rebase the port command structures to a new location in system
 	/// memory, to be allocated from the page manager.
@@ -70,8 +97,9 @@ private:
 
 	/// Initialize a command structure
 	/// \arg length  The number of bytes of data needed in the PRDs
+	/// \arg fis     [out] The FIS for this command
 	/// \returns     The index of the command slot, or -1 if none available
-	int make_command(size_t length);
+	int make_command(size_t length, fis_register_h2d **fis);
 
 	/// Send a constructed command to the hardware
 	/// \arg slot   The index of the command slot used
@@ -88,20 +116,29 @@ private:
 	/// \arg slot  The command slot that the read command used
 	void finish_read(int slot);
 
+	/// Finish an identify command started by `identify_async()`.
+	/// This will free the structures allocated, so `free_command()` is
+	/// not necessary.
+	/// \arg slot  The command slot that the read command used
+	void finish_identify(int slot);
+
 	sata_signature m_type;
 	uint8_t m_index;
 	state m_state;
 
-	port_data *m_data;
+	hba *m_hba;
+	port_data volatile *m_data;
 	void *m_fis;
 	cmd_list_entry *m_cmd_list;
 	cmd_table *m_cmd_table;
 
-	enum class command_type : uint8_t { none, read, write };
+	enum class command_type : uint8_t { none, read, write, identify, finished };
 
 	struct pending
 	{
 		command_type type;
+		uint8_t offset;
+		size_t count;
 		void *data;
 	};
 

src/drivers/nulldrv/main.cpp

@@ -0,0 +1 @@
+int main(int argc, const char **argv) { return 0; }

src/drivers/nulldrv/main.s

@@ -0,0 +1,61 @@
+section .bss
+mypid: resq 1
+mychild: resq 1
+
+section .text
+global _start
+_start:
+	xor rbp, rbp	; Sentinel rbp
+
+	mov rax, 5		; GETPID syscall
+	int 0xee
+	mov [mypid], rax
+
+	mov rax, 8		; FORK syscall
+	int 0xee
+	mov [mychild], rax
+
+	mov r12, [mypid]
+	mov r13, [mychild]
+	mov rax, 1		; DEBUG syscall
+	int 0xee
+
+	cmp r12, 1
+	je .dosend
+	jne .doreceive
+
+.preloop:
+	mov r11, 0		; counter
+	mov rbx, 20		; sleep timeout
+
+.loop:
+	mov rax, 1		; MESSAGE syscall
+	;mov rax, 0		; NOOP syscall
+	;syscall
+	int 0xee
+
+	inc r11
+	cmp r11, 2
+
+	jle .loop
+
+	mov rax, 4		; SLEEP syscall
+	; syscall
+	int 0xee
+
+	add rbx, 20
+
+	mov r11, 0
+	jmp .loop
+
+.dosend:
+	mov rax, 6      ; SEND syscall
+	mov rdi, 2      ; target is pid 2
+	int 0xee
+	jmp .preloop
+
+.doreceive:
+	mov rax, 7      ; RECEIVE syscall
+	mov rdi, 1      ; source is pid 2
+	int 0xee
+	jmp .preloop

src/include/efi/efidef.h

@@ -20,9 +20,9 @@ Revision History
 
 --*/
 
-typedef UINT16          CHAR16;
+typedef wchar_t         CHAR16;
-typedef UINT8           CHAR8;
+typedef char            CHAR8;
-typedef UINT8           BOOLEAN;
+typedef uint8_t         BOOLEAN;
 #ifndef CONST
    #define CONST const
 #endif

src/include/efi/efisetjmp.h

@@ -4,7 +4,16 @@
 #include <efi/eficompiler.h>
 #include <efi/efisetjmp_arch.h>
 
+#ifndef __has_builtin
+#define __has_builtin(x) 0
+#endif
+
+#if ! __has_builtin(setjmp)
 extern UINTN setjmp(jmp_buf *env) __attribute__((returns_twice));
+#endif
+
+#if ! __has_builtin(longjmp)
 extern VOID longjmp(jmp_buf *env, UINTN value) __attribute__((noreturn));
+#endif
 
 #endif /* GNU_EFI_SETJMP_H */

src/include/elf.h

@@ -0,0 +1,48 @@
+#pragma once
+
+/* elf.h - basic defines for external code written assuming <elf.h> works. Only
+ * Elf64 values are included.
+ */
+
+typedef uint16_t Elf64_Half;
+typedef uint32_t Elf64_Word;
+typedef int32_t Elf64_Sword;
+typedef uint64_t Elf64_Xword;
+typedef int64_t Elf64_Sxword;
+typedef uint64_t Elf64_Addr;
+typedef uint64_t Elf64_Off;
+typedef uint16_t Elf64_Section;
+typedef Elf64_Half Elf64_Versym;
+
+typedef struct {
+	Elf64_Addr r_offset;
+	Elf64_Xword r_info;
+} Elf64_Rel;
+
+typedef struct {
+	Elf64_Addr r_offset;
+	Elf64_Word r_info;
+	Elf64_Sword r_addend;
+} Elf64_Rela;
+
+typedef struct {
+	Elf64_Sxword d_tag;
+	union {
+		Elf64_Xword d_val;
+		Elf64_Addr d_ptr;
+	} d_un;
+} Elf64_Dyn;
+
+#define ELF64_R_TYPE(x) ((x) & 0xffffffff)
+
+typedef enum {
+	DT_NULL = 0,
+	DT_RELA = 7,
+	DT_RELASZ = 8,
+	DT_RELAENT = 9
+} ElfDynTag;
+
+typedef enum {
+	R_X86_64_NONE = 0,
+	R_X86_64_RELATIVE = 8
+} Elf_x86_64_RelType;

src/include/kernel_data.h

@@ -13,11 +13,12 @@ struct popcorn_data {
 	uint16_t version;
 	uint16_t length;
 
-	uint32_t _reserved0;
+	uint16_t _reserved0;
+	uint16_t scratch_pages;
 	uint32_t flags;
 
-	void *font;
+	void *initrd;
-	size_t font_length;
+	size_t initrd_length;
 
 	void *data;
 	size_t data_length;

src/kernel/allocator.cpp

@@ -1,9 +0,0 @@
-#include "kutil/memory_manager.h"
-
-kutil::memory_manager g_kernel_memory_manager;
-
-// kutil malloc/free implementation
-namespace kutil {
-	void * malloc(size_t n) { return g_kernel_memory_manager.allocate(n); }
-	void free(void *p) { g_kernel_memory_manager.free(p); }
-}

src/kernel/apic.cpp

@@ -1,6 +1,7 @@
 #include "kutil/assert.h"
 #include "apic.h"
 #include "interrupts.h"
+#include "io.h"
 #include "log.h"
 #include "page_manager.h"
 
@@ -50,30 +51,96 @@ lapic::lapic(uint32_t *base, isr spurious) :
 }
 
 void
-lapic::enable_timer(isr vector, uint8_t divisor, uint32_t count, bool repeat)
+lapic::calibrate_timer()
 {
+	interrupts_disable();
+
+	log::info(logs::apic, "Calibrating APIC timer...");
+
+	// Set up PIT sleep
+	uint8_t command = 0x30; // channel 0, loybyte/highbyte, mode 0
+	outb(0x43, command);
+
+	const uint32_t initial = -1u;
+	enable_timer_internal(isr::isrSpurious, 1, initial, false);
+
+	const int iterations = 5;
+	for (int i=0; i<iterations; ++i) {
+		const uint16_t pit_33ms = 39375;
+		uint16_t pit_count = pit_33ms;
+		outb(0x40, pit_count & 0xff);
+		io_wait();
+		outb(0x40, (pit_count >> 8) & 0xff);
+
+
+		while (pit_count <= pit_33ms) {
+			outb(0x43, 0); // latch counter values
+			pit_count =
+				static_cast<uint16_t>(inb(0x40)) |
+				static_cast<uint16_t>(inb(0x40)) << 8;
+		}
+	}
+
+	uint32_t remain = stop_timer();
+	uint32_t ticks_total = initial - remain;
+	m_ticks_per_us = ticks_total / (iterations * 33000);
+	log::info(logs::apic, "APIC timer ticks %d times per nanosecond.", m_ticks_per_us);
+
+	interrupts_enable();
+}
+
+uint32_t
+lapic::enable_timer_internal(isr vector, uint8_t divisor, uint32_t count, bool repeat)
+{
+	uint32_t divbits = 0;
+
 	switch (divisor) {
-	case   1: divisor = 11; break;
+	case   1: divbits = 0xb; break;
-	case   2: divisor = 0; break;
+	case   2: divbits = 0x0; break;
-	case   4: divisor = 1; break;
+	case   4: divbits = 0x1; break;
-	case   8: divisor = 2; break;
+	case   8: divbits = 0x2; break;
-	case  16: divisor = 3; break;
+	case  16: divbits = 0x3; break;
-	case  32: divisor = 8; break;
+	case  32: divbits = 0x8; break;
-	case  64: divisor = 9; break;
+	case  64: divbits = 0x9; break;
-	case 128: divisor = 10; break;
+	case 128: divbits = 0xa; break;
 	default:
 		kassert(0, "Invalid divisor passed to lapic::enable_timer");
 	}
 
-	apic_write(m_base, 0x3e0, divisor);
-	apic_write(m_base, 0x380, count);
-
 	uint32_t lvte = static_cast<uint8_t>(vector);
 	if (repeat)
 		lvte |= 0x20000;
 
-	log::debug(logs::apic, "Enabling APIC timer with isr %d.", vector);
+	log::debug(logs::apic, "Enabling APIC timer count %ld, divisor %d, isr %02x",
+			count, divisor, vector);
+
 	apic_write(m_base, 0x320, lvte);
+	apic_write(m_base, 0x3e0, divbits);
+
+	reset_timer(count);
+	return count;
+}
+
+uint32_t
+lapic::enable_timer(isr vector, uint64_t interval, bool repeat)
+{
+	uint64_t ticks = interval * m_ticks_per_us;
+
+	int divisor = 1;
+	while (ticks > -1u) {
+		ticks /= 2;
+		divisor *= 2;
+	}
+
+	return enable_timer_internal(vector, divisor, static_cast<uint32_t>(ticks), repeat);
+}
+
+uint32_t
+lapic::reset_timer(uint32_t count)
+{
+	uint32_t remaining = apic_read(m_base, 0x390);
+	apic_write(m_base, 0x380, count);
+	return remaining;
 }
 
 void

src/kernel/apic.h

@@ -32,10 +32,19 @@ public:
 
 	/// Enable interrupts for the LAPIC timer.
 	/// \arg vector   Interrupt vector the timer should use
-	/// \arg divisor  The frequency divisor of the bus Hz (power of 2, <= 128)
+	/// \arg interval The timer interval, in microseconds
-	/// \arg count    The count of ticks before an interrupt
 	/// \arg repeat   If false, this timer is one-off, otherwise repeating
-	void enable_timer(isr vector, uint8_t divisor, uint32_t count, bool repeat = true);
+	/// \returns      The count of ticks the timer is set for
+	uint32_t enable_timer(isr vector, uint64_t interval, bool repeat = true);
+
+	/// Reset the timer countdown.
+	/// \arg count  The count of ticks before an interrupt, or 0 to stop the timer
+	/// \returns    The count of ticks that were remaining before reset
+	uint32_t reset_timer(uint32_t count);
+
+	/// Stop the timer.
+	/// \returns  The count of ticks remaining before an interrupt was to happen
+	inline uint32_t stop_timer() { return reset_timer(0); }
 
 	/// Enable interrupts for the LAPIC LINT0 pin.
 	/// \arg num      Local interrupt number (0 or 1)
@@ -46,6 +55,14 @@ public:
 
 	void enable();  ///< Enable servicing of interrupts
 	void disable(); ///< Disable (temporarily) servicing of interrupts
+
+	/// Calibrate the timer speed against the PIT
+	void calibrate_timer();
+
+private:
+	uint32_t enable_timer_internal(isr vector, uint8_t divisor, uint32_t count, bool repeat);
+
+	uint32_t m_ticks_per_us;
 };
 
 

src/kernel/assert.cpp

@@ -7,7 +7,7 @@ __kernel_assert(const char *file, unsigned line, const char *message)
 	console *cons = console::get();
 	if (cons) {
 		cons->set_color(9 , 0);
-		cons->puts("\n\n  ERROR: ");
+		cons->puts("\n\n       ERROR: ");
 		cons->puts(file);
 		cons->puts(":");
 		cons->put_dec(line);
@@ -15,15 +15,12 @@ __kernel_assert(const char *file, unsigned line, const char *message)
 		cons->puts(message);
 	}
 
-	__asm__ __volatile__( 
+	__asm__ ( "int $0e7h" );
-		"movq %0, %%r8;"
+	while (1) __asm__ ("hlt");
-		"movq %1, %%r9;"
+}
-		"movq %2, %%r10;"
+
-		"movq $0, %%rdx;"
+extern "C" [[noreturn]] void
-		"divq %%rdx;"
+__assert_fail(const char *message, const char *file, unsigned int line, const char *function)
-		: // no outputs
+{
-		: "r"((uint64_t)line), "r"(file), "r"(message)
+	__kernel_assert(file, line, message);
-		: "rax", "rdx", "r8", "r9", "r10");
-
-	while (1);
 }

src/kernel/block_device.h

@@ -0,0 +1,11 @@
+#pragma once
+/// \file block_device.h
+/// Interface definition for block devices
+#include <stddef.h>
+
+/// Interface for block devices
+class block_device
+{
+public:
+	virtual size_t read(size_t offset, size_t length, void *buffer) = 0;
+};

src/kernel/boot.s

@@ -28,8 +28,8 @@ _start:
 	extern kernel_main
 	call kernel_main
 
-	cli
+	; Kernel init is over, wait for the scheduler to
-
+	; take over
 .hang:
 	hlt
 	jmp .hang

src/kernel/console.cpp

@@ -1,8 +1,8 @@
 #include "kutil/coord.h"
+#include "kutil/guid.h"
 #include "kutil/memory.h"
 #include "console.h"
 #include "font.h"
-#include "memory.h"
 #include "screen.h"
 #include "serial.h"
 
@@ -334,6 +334,23 @@ void console::vprintf(const char *fmt, va_list args)
 					done = true;
 					break;
 
+				case 'G': {
+						// Special: GUID type
+						kutil::guid g = va_arg(args, kutil::guid);
+						put_hex<uint32_t>(g.a, 8, '0');
+						putc('-');
+						put_hex<uint16_t>((g.a >> 32) & 0xffff, 4, '0');
+						putc('-');
+						put_hex<uint16_t>((g.a >> 48) & 0xffff, 4, '0');
+						putc('-');
+						put_hex<uint16_t>((kutil::byteswap(g.b) >> 16) & 0xffff, 4, '0');
+						putc('-');
+						put_hex<uint16_t>(kutil::byteswap(g.b) & 0xffff, 4, '0');
+						put_hex<uint32_t>(kutil::byteswap(g.b >> 32), 8, '0');
+					}
+					done = true;
+					break;
+
 				case 'l':
 					switch (*r++) {
 						case 'x': put_hex<uint64_t>(va_arg(args, uint64_t), right ? width : -width, pad); done = true; break;

src/kernel/cpu.cpp

@@ -52,6 +52,7 @@ cpu_id::cpu_id()
 
 }
 
+
 cpu_id::regs
 cpu_id::get(uint32_t leaf, uint32_t sub) const
 {

src/kernel/cpu.h

@@ -1,5 +1,15 @@
 #pragma once
 
+#include <stdint.h>
+
+struct cpu_state
+{
+	uint64_t ds;
+	uint64_t r15, r14, r13, r12, r11, r10, r9,  r8;
+	uint64_t rdi, rsi, rbp, rbx, rdx, rcx, rax;
+	uint64_t interrupt, errorcode;
+	uint64_t rip, cs, rflags, user_rsp, ss;
+};
 
 class cpu_id
 {

src/kernel/crti.s

@@ -0,0 +1,14 @@
+section .init
+global _init:function
+_init:
+	push rbp
+	mov rbp, rsp
+	; Control flow falls through to other .init sections
+
+section .fini
+global _fini:function
+_fini:
+	push rbp
+	mov rbp, rsp
+	; Control flow falls through to other .fini sections
+

src/kernel/crtn.s

@@ -0,0 +1,10 @@
+section .init
+	; Control flow falls through to here from other .init sections
+	pop rbp
+	ret
+
+section .fini
+	; Control flow falls through to here from other .fini sections
+	pop rbp
+	ret
+

src/kernel/debug.cpp

@@ -0,0 +1,72 @@
+#include "console.h"
+#include "cpu.h"
+#include "debug.h"
+#include "gdt.h"
+#include "page_manager.h"
+
+#define print_reg(name, value) cons->printf("         %s: %016lx\n", name, (value));
+
+void
+print_regs(const cpu_state &regs)
+{
+	console *cons = console::get();
+
+	print_reg("rax", regs.rax);
+	print_reg("rbx", regs.rbx);
+	print_reg("rcx", regs.rcx);
+	print_reg("rdx", regs.rdx);
+	print_reg("rdi", regs.rdi);
+	print_reg("rsi", regs.rsi);
+
+	cons->puts("\n");
+	print_reg(" r8", regs.r8);
+	print_reg(" r9", regs.r9);
+	print_reg("r10", regs.r10);
+	print_reg("r11", regs.r11);
+	print_reg("r12", regs.r12);
+	print_reg("r13", regs.r13);
+	print_reg("r14", regs.r14);
+	print_reg("r15", regs.r15);
+
+	cons->puts("\n");
+	print_reg("rbp", regs.rbp);
+	print_reg("rsp", regs.user_rsp);
+	print_reg("sp0", tss_get_stack(0));
+
+	cons->puts("\n");
+	print_reg(" ds", regs.ds);
+	print_reg(" cs", regs.cs);
+	print_reg(" ss", regs.ss);
+
+	cons->puts("\n");
+	print_reg("rip", regs.rip);
+
+	cons->puts("\n");
+	print_reg("cr3", page_manager::get()->get_pml4());
+}
+
+void
+print_stacktrace(int skip)
+{
+	console *cons = console::get();
+	int frame = 0;
+	uint64_t bp = get_frame(skip);
+	while (bp) {
+		cons->printf("    frame %2d: %lx\n", frame, bp);
+		bp = get_frame(++frame + skip);
+	}
+}
+
+void
+print_stack(const cpu_state &regs)
+{
+	console *cons = console::get();
+
+	cons->puts("\nStack:\n");
+	uint64_t sp = regs.user_rsp;
+	while (sp <= regs.rbp) {
+		cons->printf("%016x: %016x\n", sp, *reinterpret_cast<uint64_t *>(sp));
+		sp += sizeof(uint64_t);
+	}
+}
+

src/kernel/debug.h

@@ -0,0 +1,19 @@
+#pragma once
+/// \file debug.h
+/// Debugging utilities
+
+#include <stdint.h>
+
+extern "C" {
+	uintptr_t get_rsp();
+	uintptr_t get_rip();
+	uintptr_t get_frame(int frame);
+
+}
+
+void print_regs(const cpu_state &regs);
+void print_stack(const cpu_state &regs);
+void print_stacktrace(int skip = 0);
+
+#define print_reg(name, value) cons->printf("         %s: %016lx\n", name, (value));
+

src/kernel/debug.s

@@ -1,10 +1,12 @@
+global get_rsp
+get_rsp:
+	mov rax, rsp
+	ret
 
-section .text
+global get_rip
-global do_the_set_registers
+get_rip:
-do_the_set_registers:
+	pop rax ; do the same thing as 'ret', except with 'jmp'
-	mov rax, 0xdeadbeef0badc0de
+	jmp rax ; with the return address still in rax
-	mov r8, rcx
-	mov r9, rdi
 
 global _halt
 _halt:

src/kernel/device_manager.cpp

@@ -4,20 +4,17 @@
 #include "kutil/assert.h"
 #include "kutil/memory.h"
 #include "acpi_tables.h"
-#include "ahci/driver.h"
 #include "apic.h"
 #include "console.h"
 #include "device_manager.h"
 #include "interrupts.h"
 #include "log.h"
-#include "memory.h"
 #include "page_manager.h"
 
 
 static const char expected_signature[] = "RSD PTR ";
 
 device_manager device_manager::s_instance(nullptr);
-ahci_driver ahcid;
 
 struct acpi1_rsdp
 {
@@ -42,29 +39,16 @@ struct acpi2_rsdp
 	uint8_t reserved[3];
 } __attribute__ ((packed));
 
-uint8_t
-acpi_checksum(const void *p, size_t len, size_t off = 0)
-{
-	uint8_t sum = 0;
-	const uint8_t *c = reinterpret_cast<const uint8_t *>(p);
-	for (int i = off; i < len; ++i) sum += c[i];
-	return sum;
-}
-
 bool
 acpi_table_header::validate(uint32_t expected_type) const
 {
-	if (acpi_checksum(this, length) != 0) return false;
+	if (kutil::checksum(this, length) != 0) return false;
 	return !expected_type || (expected_type == type);
 }
 
 
 void irq2_callback(void *)
 {
-	console *cons = console::get();
-	cons->set_color(11);
-	cons->puts(".");
-	cons->set_color();
 }
 
 void irq4_callback(void *)
@@ -87,7 +71,7 @@ device_manager::device_manager(const void *root_table) :
 		kassert(acpi1->signature[i] == expected_signature[i],
 				"ACPI RSDP table signature mismatch");
 
-	uint8_t sum = acpi_checksum(acpi1, sizeof(acpi1_rsdp), 0);
+	uint8_t sum = kutil::checksum(acpi1, sizeof(acpi1_rsdp), 0);
 	kassert(sum == 0, "ACPI 1.0 RSDP checksum mismatch.");
 
 	kassert(acpi1->revision > 1, "ACPI 1.0 not supported.");
@@ -95,7 +79,7 @@ device_manager::device_manager(const void *root_table) :
 	const acpi2_rsdp *acpi2 =
 		reinterpret_cast<const acpi2_rsdp *>(acpi1);
 
-	sum = acpi_checksum(acpi2, sizeof(acpi2_rsdp), sizeof(acpi1_rsdp));
+	sum = kutil::checksum(acpi2, sizeof(acpi2_rsdp), sizeof(acpi1_rsdp));
 	kassert(sum == 0, "ACPI 2.0 RSDP checksum mismatch.");
 
 	load_xsdt(reinterpret_cast<const acpi_xsdt *>(acpi2->xsdt_address));
@@ -222,8 +206,6 @@ device_manager::load_apic(const acpi_apic *apic)
 		p += length;
 	}
 
-	// m_lapic->enable_timer(isr::isrTimer, 128, 3000000);
-
 	for (uint8_t i = 0; i < m_ioapics[0]->get_num_gsi(); ++i) {
 		switch (i) {
 			case 2: break;
@@ -231,7 +213,6 @@ device_manager::load_apic(const acpi_apic *apic)
 		}
 	}
 
-	m_ioapics[0]->dump_redirs();
 	m_lapic->enable();
 }
 
@@ -292,6 +273,7 @@ device_manager::init_drivers()
 {
 	// Eventually this should be e.g. a lookup into a loadable driver list
 	// for now, just look for AHCI devices
+	/*
 	for (auto &device : m_devices) {
 		if (device.devclass() != 1 || device.subclass() != 6)
 			continue;
@@ -303,6 +285,7 @@ device_manager::init_drivers()
 
 		ahcid.register_device(&device);
 	}
+	*/
 }
 
 bool
@@ -320,3 +303,9 @@ device_manager::allocate_msi(const char *name, pci_device &device, irq_callback
 			static_cast<uint16_t>(vector));
 	return true;
 }
+
+void
+device_manager::register_block_device(block_device *blockdev)
+{
+	m_blockdevs.append(blockdev);
+}

src/kernel/device_manager.h

@@ -7,6 +7,7 @@
 struct acpi_xsdt;
 struct acpi_apic;
 struct acpi_mcfg;
+class block_device;
 class lapic;
 class ioapic;
 
@@ -65,6 +66,23 @@ public:
 		return false;
 	}
 
+	/// Register the existance of a block device.
+	/// \arg blockdev  Pointer to the block device
+	void register_block_device(block_device *blockdev);
+
+	/// Get the number of block devices in the system
+	/// \returns  A count of devices
+	inline unsigned get_num_block_devices() const { return m_blockdevs.count(); }
+
+	/// Get a block device
+	/// \arg i    Index of the device to get
+	/// \returns  A pointer to the requested device, or nullptr
+	inline block_device * get_block_device(unsigned i)
+	{
+		return i < m_blockdevs.count() ?
+			m_blockdevs[i] : nullptr;
+	}
+
 private:
 	/// Parse the ACPI XSDT and load relevant sub-tables.
 	/// \arg xsdt  Pointer to the XSDT from the firmware
@@ -100,6 +118,8 @@ private:
 	};
 	kutil::vector<irq_allocation> m_irqs;
 
+	kutil::vector<block_device *> m_blockdevs;
+
 	static device_manager s_instance;
 
 	device_manager() = delete;

src/kernel/fs/gpt.cpp

@@ -0,0 +1,114 @@
+#include "kutil/assert.h"
+#include "kutil/guid.h"
+#include "kutil/memory.h"
+#include "device_manager.h"
+#include "fs/gpt.h"
+#include "log.h"
+
+namespace fs {
+
+const kutil::guid efi_system_part = kutil::make_guid(0xC12A7328, 0xF81F, 0x11D2, 0xBA4B, 0x00A0C93EC93B);
+const kutil::guid efi_unused_part = kutil::make_guid(0, 0, 0, 0, 0);
+
+const uint64_t gpt_signature = 0x5452415020494645; // "EFI PART"
+const size_t block_size = 512;
+
+struct gpt_header
+{
+	uint64_t signature;
+	uint32_t revision;
+	uint32_t headersize;
+	uint32_t crc32;
+	uint32_t reserved;
+	uint64_t my_lba;
+	uint64_t alt_lba;
+	uint64_t first_usable_lba;
+	uint64_t last_usable_lba;
+	kutil::guid disk_guid;
+	uint64_t table_lba;
+	uint32_t entry_count;
+	uint32_t entry_length;
+	uint32_t table_crc32;
+} __attribute__ ((packed));
+
+struct gpt_entry
+{
+	kutil::guid type;
+	kutil::guid part_guid;
+	uint64_t start_lba;
+	uint64_t end_lba;
+	uint64_t attributes;
+	uint16_t name_wide[36];
+} __attribute__ ((packed));
+
+
+partition::partition(block_device *parent, size_t start, size_t length) :
+	m_parent(parent),
+	m_start(start),
+	m_length(length)
+{
+}
+
+size_t
+partition::read(size_t offset, size_t length, void *buffer)
+{
+	if (offset + length > m_length)
+		offset = m_length - offset;
+	return m_parent->read(m_start + offset, length, buffer);
+}
+
+
+unsigned
+partition::load(block_device *device)
+{
+	// Read LBA 1
+	uint8_t block[block_size];
+	size_t count = device->read(block_size, block_size, &block);
+	kassert(count == block_size, "Short read for GPT header.");
+
+	gpt_header *header = reinterpret_cast<gpt_header *>(&block);
+	if (header->signature != gpt_signature)
+		return 0;
+
+	size_t arraysize = header->entry_length * header->entry_count;
+	log::debug(logs::fs, "Found GPT header: %d paritions, size 0x%lx",
+			header->entry_count, arraysize);
+
+	uint8_t *array = new uint8_t[arraysize];
+	count = device->read(block_size * header->table_lba, arraysize, array);
+	kassert(count == arraysize, "Short read for GPT entry array.");
+
+	auto &dm = device_manager::get();
+
+	unsigned found = 0;
+	gpt_entry *entry0 = reinterpret_cast<gpt_entry *>(array);
+	for (uint32_t i = 0; i < header->entry_count; ++i) {
+		gpt_entry *entry = kutil::offset_pointer(entry0, i * header->entry_length);
+		if (entry->type == efi_unused_part) continue;
+
+		// TODO: real UTF16->UTF8
+		char name[sizeof(gpt_entry::name_wide) / 2];
+		for (int i = 0; i < sizeof(name); ++i)
+			name[i] = entry->name_wide[i];
+
+		log::debug(logs::fs, "Found partition %02x at %lx-%lx", i, entry->start_lba, entry->end_lba);
+		if (entry->type == efi_system_part)
+			log::debug(logs::fs, "   type EFI SYSTEM PARTITION");
+		else
+			log::debug(logs::fs, "   type %G", entry->type);
+		log::debug(logs::fs, "   name %s", name);
+		log::debug(logs::fs, "   attr %016lx", entry->attributes);
+
+		found += 1;
+		partition *part = new partition(
+				device,
+				entry->start_lba * block_size,
+				(entry->end_lba - entry->start_lba) * block_size);
+		dm.register_block_device(part);
+	}
+
+	return found;
+}
+
+
+} // namespace fs

src/kernel/fs/gpt.h

@@ -0,0 +1,37 @@
+#pragma once
+/// \file gpt.h
+/// Definitions for dealing with GUID Partition Tables
+#include "block_device.h"
+
+namespace fs {
+
+
+class partition :
+	public block_device
+{
+public:
+	/// Constructor.
+	/// \arg parent  The block device this partition is a part of
+	/// \arg start   The starting offset in bytes from the start of the parent
+	/// \arg lenght  The length in bytes of this partition
+	partition(block_device *parent, size_t start, size_t length);
+
+	/// Read bytes from the partition.
+	/// \arg offset  The offset in bytes at which to start reading
+	/// \arg length  The number of bytes to read
+	/// \arg buffer  [out] Data is read into this buffer
+	/// \returns     The number of bytes read
+	virtual size_t read(size_t offset, size_t length, void *buffer);
+
+	/// Find partitions on a block device and add them to the device manager
+	/// \arg device  The device to search for partitions
+	/// \returns     The number of partitions found
+	static unsigned load(block_device *device);
+
+private:
+	block_device *m_parent;
+	size_t m_start;
+	size_t m_length;
+};
+
+} // namespace fs

src/kernel/gdt.cpp

@@ -0,0 +1,273 @@
+#include <stdint.h>
+
+#include "kutil/assert.h"
+#include "kutil/enum_bitfields.h"
+#include "kutil/memory.h"
+#include "console.h"
+#include "log.h"
+
+
+enum class gdt_type : uint8_t
+{
+	accessed	= 0x01,
+	read_write	= 0x02,
+	conforming	= 0x04,
+	execute		= 0x08,
+	system		= 0x10,
+	ring1		= 0x20,
+	ring2		= 0x40,
+	ring3		= 0x60,
+	present		= 0x80
+};
+IS_BITFIELD(gdt_type);
+
+struct gdt_descriptor
+{
+	uint16_t limit_low;
+	uint16_t base_low;
+	uint8_t base_mid;
+	gdt_type type;
+	uint8_t size;
+	uint8_t base_high;
+} __attribute__ ((packed));
+
+struct tss_descriptor
+{
+	uint16_t limit_low;
+	uint16_t base_00;
+	uint8_t base_16;
+	gdt_type type;
+	uint8_t size;
+	uint8_t base_24;
+	uint32_t base_32;
+	uint32_t reserved;
+} __attribute__ ((packed));
+
+struct tss_entry
+{
+	uint32_t reserved0;
+
+	uint64_t rsp[3]; // stack pointers for CPL 0-2
+	uint64_t ist[8]; // ist[0] is reserved
+
+	uint64_t reserved1;
+	uint16_t reserved2;
+	uint16_t iomap_offset;
+} __attribute__ ((packed));
+
+struct idt_descriptor
+{
+	uint16_t base_low;
+	uint16_t selector;
+	uint8_t ist;
+	uint8_t flags;
+	uint16_t base_mid;
+	uint32_t base_high;
+	uint32_t reserved;		// must be zero
+} __attribute__ ((packed));
+
+struct table_ptr
+{
+	uint16_t limit;
+	uint64_t base;
+} __attribute__ ((packed));
+
+
+gdt_descriptor g_gdt_table[10];
+idt_descriptor g_idt_table[256];
+table_ptr g_gdtr;
+table_ptr g_idtr;
+tss_entry g_tss;
+
+
+extern "C" {
+	void idt_write();
+	void idt_load();
+
+	void gdt_write(uint16_t cs, uint16_t ds, uint16_t tr);
+	void gdt_load();
+}
+
+void
+gdt_set_entry(uint8_t i, uint32_t base, uint64_t limit, bool is64, gdt_type type)
+{
+	g_gdt_table[i].limit_low = limit & 0xffff;
+	g_gdt_table[i].size = (limit >> 16) & 0xf;
+	g_gdt_table[i].size |= (is64 ? 0xa0 : 0xc0);
+
+	g_gdt_table[i].base_low = base & 0xffff;
+	g_gdt_table[i].base_mid = (base >> 16) & 0xff;
+	g_gdt_table[i].base_high = (base >> 24) & 0xff;
+
+	g_gdt_table[i].type = type | gdt_type::system | gdt_type::present;
+}
+
+void
+tss_set_entry(uint8_t i, uint64_t base, uint64_t limit)
+{
+	tss_descriptor tssd;
+	tssd.limit_low = limit & 0xffff;
+	tssd.size = (limit >> 16) & 0xf;
+
+	tssd.base_00 = base & 0xffff;
+	tssd.base_16 = (base >> 16) & 0xff;
+	tssd.base_24 = (base >> 24) & 0xff;
+	tssd.base_32 = (base >> 32) & 0xffffffff;
+	tssd.reserved = 0;
+
+	tssd.type =
+		gdt_type::accessed |
+		gdt_type::execute |
+		gdt_type::ring3 |
+		gdt_type::present;
+	kutil::memcpy(&g_gdt_table[i], &tssd, sizeof(tss_descriptor));
+}
+
+void
+idt_set_entry(uint8_t i, uint64_t addr, uint16_t selector, uint8_t flags)
+{
+	g_idt_table[i].base_low = addr & 0xffff;
+	g_idt_table[i].base_mid = (addr >> 16) & 0xffff;
+	g_idt_table[i].base_high = (addr >> 32) & 0xffffffff;
+	g_idt_table[i].selector = selector;
+	g_idt_table[i].flags = flags;
+	g_idt_table[i].ist = 0;
+	g_idt_table[i].reserved = 0;
+}
+
+void
+tss_set_stack(int ring, uintptr_t rsp)
+{
+	kassert(ring < 3, "Bad ring passed to tss_set_stack.");
+	g_tss.rsp[ring] = rsp;
+}
+
+uintptr_t
+tss_get_stack(int ring)
+{
+	kassert(ring < 3, "Bad ring passed to tss_get_stack.");
+	return g_tss.rsp[ring];
+}
+
+void
+gdt_init()
+{
+	kutil::memset(&g_gdt_table, 0, sizeof(g_gdt_table));
+	kutil::memset(&g_idt_table, 0, sizeof(g_idt_table));
+
+	g_gdtr.limit = sizeof(g_gdt_table) - 1;
+	g_gdtr.base = reinterpret_cast<uint64_t>(&g_gdt_table);
+
+	// Kernel CS/SS - always 64bit
+	gdt_set_entry(1, 0, 0xfffff,  true, gdt_type::read_write | gdt_type::execute);
+	gdt_set_entry(2, 0, 0xfffff,  true, gdt_type::read_write);
+
+	// User CS32/SS/CS64 - layout expected by SYSRET
+	gdt_set_entry(3, 0, 0xfffff,  false, gdt_type::ring3 | gdt_type::read_write | gdt_type::execute);
+	gdt_set_entry(4, 0, 0xfffff,  true, gdt_type::ring3 | gdt_type::read_write);
+	gdt_set_entry(5, 0, 0xfffff,  true, gdt_type::ring3 | gdt_type::read_write | gdt_type::execute);
+
+	kutil::memset(&g_tss, 0, sizeof(tss_entry));
+	g_tss.iomap_offset = sizeof(tss_entry);
+
+	uintptr_t tss_base = reinterpret_cast<uintptr_t>(&g_tss);
+
+	// Note that this takes TWO GDT entries
+	tss_set_entry(6, tss_base, sizeof(tss_entry));
+
+	gdt_write(1 << 3, 2 << 3, 6 << 3);
+
+	g_idtr.limit = sizeof(g_idt_table) - 1;
+	g_idtr.base = reinterpret_cast<uint64_t>(&g_idt_table);
+
+	idt_write();
+}
+
+void
+gdt_dump()
+{
+	const table_ptr &table = g_gdtr;
+
+	console *cons = console::get();
+	cons->printf("         GDT: loc:%lx size:%d\n", table.base, table.limit+1);
+
+	int count = (table.limit + 1) / sizeof(gdt_descriptor);
+	const gdt_descriptor *gdt =
+		reinterpret_cast<const gdt_descriptor *>(table.base);
+
+	for (int i = 0; i < count; ++i) {
+		uint32_t base =
+			(gdt[i].base_high << 24) |
+			(gdt[i].base_mid << 16) |
+			gdt[i].base_low;
+
+		uint32_t limit =
+			static_cast<uint32_t>(gdt[i].size & 0x0f) << 16 |
+			gdt[i].limit_low;
+
+		cons->printf("          %02d:", i);
+		if (! bitfield_has(gdt[i].type, gdt_type::present)) {
+			cons->puts(" Not Present\n");
+			continue;
+		}
+
+		cons->printf(" Base %08x  limit %05x ", base, limit);
+
+		switch (gdt[i].type & gdt_type::ring3) {
+			case gdt_type::ring3: cons->puts("ring3"); break;
+			case gdt_type::ring2: cons->puts("ring2"); break;
+			case gdt_type::ring1: cons->puts("ring1"); break;
+			default: cons->puts("ring0"); break;
+		}
+
+		cons->printf(" %s %s %s %s %s %s %s\n",
+			bitfield_has(gdt[i].type, gdt_type::accessed) ? "A" : " ",
+			bitfield_has(gdt[i].type, gdt_type::read_write) ? "RW" : "  ",
+			bitfield_has(gdt[i].type, gdt_type::conforming) ? "C" : " ",
+			bitfield_has(gdt[i].type, gdt_type::execute) ? "EX" : "  ",
+			bitfield_has(gdt[i].type, gdt_type::system) ? "S" : " ",
+			(gdt[i].size & 0x80) ? "KB" : " B",
+			(gdt[i].size & 0x60) == 0x20 ? "64" :
+				(gdt[i].size & 0x60) == 0x40 ? "32" : "16");
+	}
+}
+
+void
+idt_dump()
+{
+	const table_ptr &table = g_idtr;
+
+	log::info(logs::boot, "Loaded IDT at: %lx size: %d bytes", table.base, table.limit+1);
+
+	int count = (table.limit + 1) / sizeof(idt_descriptor);
+	const idt_descriptor *idt =
+		reinterpret_cast<const idt_descriptor *>(table.base);
+
+	for (int i = 0; i < count; ++i) {
+		uint64_t base =
+			(static_cast<uint64_t>(idt[i].base_high) << 32) |
+			(static_cast<uint64_t>(idt[i].base_mid)  << 16) |
+			idt[i].base_low;
+
+		char const *type;
+		switch (idt[i].flags & 0xf) {
+			case 0x5: type = " 32tsk "; break;
+			case 0x6: type = " 16int "; break;
+			case 0x7: type = " 16trp "; break;
+			case 0xe: type = " 32int "; break;
+			case 0xf: type = " 32trp "; break;
+			default:  type = " ????? "; break;
+		}
+
+		if (idt[i].flags & 0x80) {
+			log::debug(logs::boot,
+					"   Entry %3d: Base:%lx Sel(rpl %d, ti %d, %3d) IST:%d %s DPL:%d", i, base,
+					(idt[i].selector & 0x3),
+					((idt[i].selector & 0x4) >> 2),
+					(idt[i].selector >> 3),
+					idt[i].ist,
+					type,
+					((idt[i].flags >> 5) & 0x3));
+		}
+	}
+}

src/kernel/gdt.h

@@ -0,0 +1,31 @@
+#pragma once
+/// \file gdt.h
+/// Definitions relating to system descriptor tables: GDT, IDT, TSS
+#include <stdint.h>
+
+/// Set up the GDT and TSS, and switch segment registers to point
+/// to them.
+void gdt_init();
+
+/// Set an entry in the IDT
+/// \arg i         Index in the IDT (vector of the interrupt this handles)
+/// \arg addr      Address of the handler
+/// \arg selector  GDT selector to set when invoking this handler
+/// \arg flags     Descriptor flags to set
+void idt_set_entry(uint8_t i, uint64_t addr, uint16_t selector, uint8_t flags);
+
+/// Set the stack pointer for a given ring in the TSS
+/// \arg ring  Ring to set for (0-2)
+/// \arg rsp   Stack pointer to set
+void tss_set_stack(int ring, uintptr_t rsp);
+
+/// Get the stack pointer for a given ring in the TSS
+/// \arg ring  Ring to get (0-2)
+/// \returns   Stack pointers for that ring
+uintptr_t tss_get_stack(int ring);
+
+/// Dump information about the current GDT to the screen
+void gdt_dump();
+
+/// Dump information about the current IDT to the screen
+void idt_dump();

src/kernel/gdt.s

@@ -0,0 +1,35 @@
+extern g_idtr
+extern g_gdtr
+
+global idt_write
+idt_write:
+	lidt [rel g_idtr]
+	ret
+
+global idt_load
+idt_load:
+	sidt [rel g_idtr]
+	ret
+
+global gdt_write
+gdt_write:
+	lgdt [rel g_gdtr]
+	mov ax, si ; second arg is data segment
+	mov ds, ax
+	mov es, ax
+	mov fs, ax
+	mov gs, ax
+	mov ss, ax
+	push qword rdi ; first arg is code segment
+	lea rax, [rel .next]
+	push rax
+	o64 retf
+.next:
+	ltr dx ; third arg is the TSS
+	ret
+
+global gdt_load
+gdt_load:
+	sgdt [rel g_gdtr]
+	ret
+

src/kernel/interrupt_isrs.inc

@@ -134,9 +134,115 @@ IRQ (0x7d, 0x5d, irq5D)
 IRQ (0x7e, 0x5e, irq5E)
 IRQ (0x7f, 0x5f, irq5F)
 
-ISR (0xec, isrTimer)
+IRQ (0x80, 0x60, irq60)
-ISR (0xed, isrLINT0)
+IRQ (0x81, 0x61, irq61)
-ISR (0xee, isrLINT1)
+IRQ (0x82, 0x62, irq62)
+IRQ (0x83, 0x63, irq63)
+IRQ (0x84, 0x64, irq64)
+IRQ (0x85, 0x65, irq65)
+IRQ (0x86, 0x66, irq66)
+IRQ (0x87, 0x67, irq67)
+IRQ (0x88, 0x68, irq68)
+IRQ (0x89, 0x69, irq69)
+IRQ (0x8a, 0x6a, irq6A)
+IRQ (0x8b, 0x6b, irq6B)
+IRQ (0x8c, 0x6c, irq6C)
+IRQ (0x8d, 0x6d, irq6D)
+IRQ (0x8e, 0x6e, irq6E)
+IRQ (0x8f, 0x6f, irq6F)
+
+IRQ (0x90, 0x70, irq70)
+IRQ (0x91, 0x71, irq71)
+IRQ (0x92, 0x72, irq72)
+IRQ (0x93, 0x73, irq73)
+IRQ (0x94, 0x74, irq74)
+IRQ (0x95, 0x75, irq75)
+IRQ (0x96, 0x76, irq76)
+IRQ (0x97, 0x77, irq77)
+IRQ (0x98, 0x78, irq78)
+IRQ (0x99, 0x79, irq79)
+IRQ (0x9a, 0x7a, irq7A)
+IRQ (0x9b, 0x7b, irq7B)
+IRQ (0x9c, 0x7c, irq7C)
+IRQ (0x9d, 0x7d, irq7D)
+IRQ (0x9e, 0x7e, irq7E)
+IRQ (0x9f, 0x7f, irq7F)
+
+IRQ (0xa0, 0x80, irq80)
+IRQ (0xa1, 0x81, irq81)
+IRQ (0xa2, 0x82, irq82)
+IRQ (0xa3, 0x83, irq83)
+IRQ (0xa4, 0x84, irq84)
+IRQ (0xa5, 0x85, irq85)
+IRQ (0xa6, 0x86, irq86)
+IRQ (0xa7, 0x87, irq87)
+IRQ (0xa8, 0x88, irq88)
+IRQ (0xa9, 0x89, irq89)
+IRQ (0xaa, 0x8a, irq8A)
+IRQ (0xab, 0x8b, irq8B)
+IRQ (0xac, 0x8c, irq8C)
+IRQ (0xad, 0x8d, irq8D)
+IRQ (0xae, 0x8e, irq8E)
+IRQ (0xaf, 0x8f, irq8F)
+
+IRQ (0xb0, 0x90, irq90)
+IRQ (0xb1, 0x91, irq91)
+IRQ (0xb2, 0x92, irq92)
+IRQ (0xb3, 0x93, irq93)
+IRQ (0xb4, 0x94, irq94)
+IRQ (0xb5, 0x95, irq95)
+IRQ (0xb6, 0x96, irq96)
+IRQ (0xb7, 0x97, irq97)
+IRQ (0xb8, 0x98, irq98)
+IRQ (0xb9, 0x99, irq99)
+IRQ (0xba, 0x9a, irq9A)
+IRQ (0xbb, 0x9b, irq9B)
+IRQ (0xbc, 0x9c, irq9C)
+IRQ (0xbd, 0x9d, irq9D)
+IRQ (0xbe, 0x9e, irq9E)
+IRQ (0xbf, 0x9f, irq9F)
+
+IRQ (0xc0, 0xa0, irqA0)
+IRQ (0xc1, 0xa1, irqA1)
+IRQ (0xc2, 0xa2, irqA2)
+IRQ (0xc3, 0xa3, irqA3)
+IRQ (0xc4, 0xa4, irqA4)
+IRQ (0xc5, 0xa5, irqA5)
+IRQ (0xc6, 0xa6, irqA6)
+IRQ (0xc7, 0xa7, irqA7)
+IRQ (0xc8, 0xa8, irqA8)
+IRQ (0xc9, 0xa9, irqA9)
+IRQ (0xca, 0xaa, irqAA)
+IRQ (0xcb, 0xab, irqAB)
+IRQ (0xcc, 0xac, irqAC)
+IRQ (0xcd, 0xad, irqAD)
+IRQ (0xce, 0xae, irqAE)
+IRQ (0xcf, 0xaf, irqAF)
+
+IRQ (0xd0, 0xb0, irqB0)
+IRQ (0xd1, 0xb1, irqB1)
+IRQ (0xd2, 0xb2, irqB2)
+IRQ (0xd3, 0xb3, irqB3)
+IRQ (0xd4, 0xb4, irqB4)
+IRQ (0xd5, 0xb5, irqB5)
+IRQ (0xd6, 0xb6, irqB6)
+IRQ (0xd7, 0xb7, irqB7)
+IRQ (0xd8, 0xb8, irqB8)
+IRQ (0xd9, 0xb9, irqB9)
+IRQ (0xda, 0xba, irqBA)
+IRQ (0xdb, 0xbb, irqBB)
+IRQ (0xdc, 0xbc, irqBC)
+IRQ (0xdd, 0xbd, irqBD)
+IRQ (0xde, 0xbe, irqBE)
+IRQ (0xdf, 0xbf, irqBF)
+
+
+ISR (0xe0, isrTimer)
+ISR (0xe1, isrLINT0)
+ISR (0xe2, isrLINT1)
+ISR (0xe4, isrAssert)
+
+UISR(0xee, isrSyscall)
 ISR (0xef, isrSpurious)
 
 ISR (0xf0, isrIgnore0)

src/kernel/interrupts.cpp

@@ -1,85 +1,38 @@
-#include <stddef.h>
 #include <stdint.h>
 
-#include "kutil/enum_bitfields.h"
+#include "apic.h"
-#include "kutil/memory.h"
 #include "console.h"
+#include "cpu.h"
+#include "debug.h"
 #include "device_manager.h"
+#include "gdt.h"
 #include "interrupts.h"
 #include "io.h"
 #include "log.h"
+#include "scheduler.h"
+#include "syscall.h"
 
-enum class gdt_flags : uint8_t
+static const uint16_t PIC1 = 0x20;
-{
+static const uint16_t PIC2 = 0xa0;
-	ac = 0x01,
-	rw = 0x02,
-	dc = 0x04,
-	ex = 0x08,
-	r1 = 0x20,
-	r2 = 0x40,
-	r3 = 0x60,
-	pr = 0x80,
-
-	res_1 = 0x10
-};
-IS_BITFIELD(gdt_flags);
-
-struct gdt_descriptor
-{
-	uint16_t limit_low;
-	uint16_t base_low;
-	uint8_t base_mid;
-	uint8_t flags;
-	uint8_t granularity;
-	uint8_t base_high;
-} __attribute__ ((packed));
-
-struct idt_descriptor
-{
-	uint16_t base_low;
-	uint16_t selector;
-	uint8_t ist;
-	uint8_t flags;
-	uint16_t base_mid;
-	uint32_t base_high;
-	uint32_t reserved;		// must be zero
-} __attribute__ ((packed));
-
-struct table_ptr
-{
-	uint16_t limit;
-	uint64_t base;
-} __attribute__ ((packed));
-
-gdt_descriptor g_gdt_table[10];
-idt_descriptor g_idt_table[256];
-table_ptr g_gdtr;
-table_ptr g_idtr;
-
-struct registers;
 
 extern "C" {
-	void idt_write();
+	void _halt();
-	void idt_load();
 
-	void gdt_write();
+	uintptr_t isr_handler(uintptr_t, cpu_state*);
-	void gdt_load();
+	uintptr_t irq_handler(uintptr_t, cpu_state*);
-
+	uintptr_t syscall_handler(uintptr_t, cpu_state);
-	void isr_handler(registers);
-	void irq_handler(registers);
 
 #define ISR(i, name)     extern void name ();
 #define EISR(i, name)    extern void name ();
+#define UISR(i, name)    extern void name ();
 #define IRQ(i, q, name)  extern void name ();
 #include "interrupt_isrs.inc"
 #undef IRQ
+#undef UISR
 #undef EISR
 #undef ISR
 }
 
-void idt_dump(const table_ptr &table);
-void gdt_dump(const table_ptr &table);
-
 isr
 operator+(const isr &lhs, int rhs)
 {
@@ -93,9 +46,11 @@ get_irq(unsigned vector)
 	switch (vector) {
 #define ISR(i, name)
 #define EISR(i, name)
+#define UISR(i, name)
 #define IRQ(i, q, name) case i : return q;
 #include "interrupt_isrs.inc"
 #undef IRQ
+#undef UISR
 #undef EISR
 #undef ISR
 
@@ -103,43 +58,12 @@ get_irq(unsigned vector)
 	}
 }
 
-void
-set_gdt_entry(uint8_t i, uint32_t base, uint32_t limit, bool is64, gdt_flags flags)
-{
-	g_gdt_table[i].limit_low = limit & 0xffff;
-
-	g_gdt_table[i].base_low = base & 0xffff;
-	g_gdt_table[i].base_mid = (base >> 16) & 0xff;
-	g_gdt_table[i].base_high = (base >> 24) & 0xff;
-
-	g_gdt_table[i].granularity =
-			((limit >> 16) & 0xf) | (is64 ? 0xa0 : 0xc0);
-
-	g_gdt_table[i].flags = static_cast<uint8_t>(flags | gdt_flags::res_1 | gdt_flags::pr);
-}
-
-void
-set_idt_entry(uint8_t i, uint64_t addr, uint16_t selector, uint8_t flags)
-{
-	g_idt_table[i].base_low = addr & 0xffff;
-	g_idt_table[i].base_mid = (addr >> 16) & 0xffff;
-	g_idt_table[i].base_high = (addr >> 32) & 0xffffffff;
-	g_idt_table[i].selector = selector;
-	g_idt_table[i].flags = flags;
-	g_idt_table[i].ist = 0;
-	g_idt_table[i].reserved = 0;
-}
-
 static void
 disable_legacy_pic()
 {
-
-	static const uint16_t PIC1 = 0x20;
-	static const uint16_t PIC2 = 0xa0;
-
 	// Mask all interrupts
-	outb(0xa1, 0xff);
+	outb(PIC2+1, 0xfc);
-	outb(0x21, 0xff);
+	outb(PIC1+1, 0xff);
 
 	// Start initialization sequence
 	outb(PIC1, 0x11); io_wait();
@@ -147,7 +71,7 @@ disable_legacy_pic()
 
 	// Remap into ignore ISRs
 	outb(PIC1+1, static_cast<uint8_t>(isr::isrIgnore0)); io_wait();
-	outb(PIC2+1, static_cast<uint8_t>(isr::isrIgnore0)); io_wait();
+	outb(PIC2+1, static_cast<uint8_t>(isr::isrIgnore8)); io_wait();
 
 	// Tell PICs about each other
 	outb(PIC1+1, 0x04); io_wait();
@@ -164,72 +88,98 @@ enable_serial_interrupts()
 void
 interrupts_init()
 {
-	kutil::memset(&g_gdt_table, 0, sizeof(g_gdt_table));
+#define ISR(i, name)     idt_set_entry(i, reinterpret_cast<uint64_t>(& name), 0x08, 0x8e);
-	kutil::memset(&g_idt_table, 0, sizeof(g_idt_table));
+#define EISR(i, name)    idt_set_entry(i, reinterpret_cast<uint64_t>(& name), 0x08, 0x8e);
-
+#define UISR(i, name)    idt_set_entry(i, reinterpret_cast<uint64_t>(& name), 0x08, 0xee);
-	g_gdtr.limit = sizeof(g_gdt_table) - 1;
+#define IRQ(i, q, name)  idt_set_entry(i, reinterpret_cast<uint64_t>(& name), 0x08, 0x8e);
-	g_gdtr.base = reinterpret_cast<uint64_t>(&g_gdt_table);
-
-	set_gdt_entry(1, 0, 0xfffff, false, gdt_flags::rw);
-	set_gdt_entry(2, 0, 0xfffff, false, gdt_flags::rw | gdt_flags::ex | gdt_flags::dc);
-	set_gdt_entry(3, 0, 0xfffff, false, gdt_flags::rw);
-	set_gdt_entry(4, 0, 0xfffff, false, gdt_flags::rw | gdt_flags::ex);
-
-	set_gdt_entry(6, 0, 0xfffff, false, gdt_flags::rw);
-	set_gdt_entry(7, 0, 0xfffff,  true, gdt_flags::rw | gdt_flags::ex);
-
-	gdt_write();
-
-	g_idtr.limit = sizeof(g_idt_table) - 1;
-	g_idtr.base = reinterpret_cast<uint64_t>(&g_idt_table);
-
-#define ISR(i, name)     set_idt_entry(i, reinterpret_cast<uint64_t>(& name), 0x38, 0x8e);
-#define EISR(i, name)    set_idt_entry(i, reinterpret_cast<uint64_t>(& name), 0x38, 0x8e);
-#define IRQ(i, q, name)  set_idt_entry(i, reinterpret_cast<uint64_t>(& name), 0x38, 0x8e);
 #include "interrupt_isrs.inc"
 #undef IRQ
+#undef UISR
 #undef EISR
 #undef ISR
 
-	idt_write();
 	disable_legacy_pic();
 	enable_serial_interrupts();
 
 	log::info(logs::boot, "Interrupts enabled.");
 }
 
-struct registers
+uintptr_t
-{
+isr_handler(uintptr_t return_rsp, cpu_state *regs)
-	uint64_t ds;
-	uint64_t rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax;
-	uint64_t interrupt, errorcode;
-	uint64_t rip, cs, eflags, user_esp, ss;
-};
-
-#define print_reg(name, value) cons->printf("         %s: %016lx\n", name, (value));
-
-extern "C" uint64_t get_frame(int frame);
-
-void
-print_stacktrace(int skip = 0)
-{
-	console *cons = console::get();
-	int frame = 0;
-	uint64_t bp = get_frame(skip);
-	while (bp) {
-		cons->printf("    frame %2d: %lx\n", frame, bp);
-		bp = get_frame(++frame + skip);
-	}
-}
-
-void
-isr_handler(registers regs)
 {
 	console *cons = console::get();
 
-	switch (static_cast<isr>(regs.interrupt & 0xff)) {
+	switch (static_cast<isr>(regs->interrupt & 0xff)) {
-	case isr::isrTimer:
+
-		cons->puts("\nTICK\n");
+	case isr::isrGPFault: {
+			cons->set_color(9);
+			cons->puts("\nGeneral Protection Fault:\n");
+			cons->set_color();
+
+			cons->printf("   errorcode: %lx", regs->errorcode);
+			if (regs->errorcode & 0x01) cons->puts(" external");
+
+			int index = (regs->errorcode & 0xffff) >> 4;
+			if (index) {
+				switch ((regs->errorcode & 0x07) >> 1) {
+				case 0:
+					cons->printf(" GDT[%x]\n", index);
+					gdt_dump();
+					break;
+
+				case 1:
+				case 3:
+					cons->printf(" IDT[%x]\n", index);
+					idt_dump();
+					break;
+
+				default:
+					cons->printf(" LDT[%x]??\n", index);
+					break;
+				}
+			} else {
+				cons->putc('\n');
+			}
+			print_regs(*regs);
+			/*
+			print_stacktrace(2);
+			print_stack(*regs);
+			*/
+
+		}
+		_halt();
+		break;
+
+	case isr::isrPageFault: {
+			cons->set_color(11);
+			cons->puts("\nPage Fault:\n");
+			cons->set_color();
+
+			cons->puts("       flags:");
+			if (regs->errorcode & 0x01) cons->puts(" present");
+			if (regs->errorcode & 0x02) cons->puts(" write");
+			if (regs->errorcode & 0x04) cons->puts(" user");
+			if (regs->errorcode & 0x08) cons->puts(" reserved");
+			if (regs->errorcode & 0x10) cons->puts(" ip");
+			cons->puts("\n");
+
+			uint64_t cr2 = 0;
+			__asm__ __volatile__ ("mov %%cr2, %0" : "=r"(cr2));
+			print_reg("cr2", cr2);
+
+			print_reg("rsp", regs->user_rsp);
+			print_reg("rip", regs->rip);
+
+			cons->puts("\n");
+			//print_stacktrace(2);
+		}
+		_halt();
+		break;
+
+	case isr::isrTimer: {
+			scheduler &s = scheduler::get();
+			return_rsp = s.tick(return_rsp);
+		}
 		break;
 
 	case isr::isrLINT0:
@@ -240,6 +190,23 @@ isr_handler(registers regs)
 		cons->puts("\nLINT1\n");
 		break;
 
+	case isr::isrAssert: {
+			cons->set_color();
+			print_regs(*regs);
+			print_stacktrace(2);
+		}
+		_halt();
+		break;
+
+	case isr::isrSyscall: {
+			return_rsp = syscall_dispatch(return_rsp, *regs);
+		}
+		break;
+
+	case isr::isrSpurious:
+		// No EOI for the spurious interrupt
+		return return_rsp;
+
 	case isr::isrIgnore0:
 	case isr::isrIgnore1:
 	case isr::isrIgnore2:
@@ -248,177 +215,61 @@ isr_handler(registers regs)
 	case isr::isrIgnore5:
 	case isr::isrIgnore6:
 	case isr::isrIgnore7:
-
+		//cons->printf("\nIGNORED: %02x\n", regs->interrupt);
-		/*
+		outb(PIC1, 0x20);
-		cons->printf("\nIGNORED PIC INTERRUPT %d\n",
-				(regs.interrupt % 0xff) - 0xf0);
-		*/
 		break;
 
-	case isr::isrPageFault: {
+	case isr::isrIgnore8:
-			cons->set_color(11);
+	case isr::isrIgnore9:
-			cons->puts("\nPage Fault:\n");
+	case isr::isrIgnoreA:
-			cons->set_color();
+	case isr::isrIgnoreB:
-
+	case isr::isrIgnoreC:
-			cons->puts("       flags:");
+	case isr::isrIgnoreD:
-			if (regs.errorcode & 0x01) cons->puts(" present");
+	case isr::isrIgnoreE:
-			if (regs.errorcode & 0x02) cons->puts(" write");
+	case isr::isrIgnoreF:
-			if (regs.errorcode & 0x04) cons->puts(" user");
+		//cons->printf("\nIGNORED: %02x\n", regs->interrupt);
-			if (regs.errorcode & 0x08) cons->puts(" reserved");
+		outb(PIC1, 0x20);
-			if (regs.errorcode & 0x10) cons->puts(" ip");
+		outb(PIC2, 0x20);
-			cons->puts("\n");
-
-			uint64_t cr2 = 0;
-			__asm__ __volatile__ ("mov %%cr2, %0" : "=r"(cr2));
-			print_reg("cr2", cr2);
-
-			print_reg("rip", regs.rip);
-
-			cons->puts("\n");
-			print_stacktrace(2);
-		}
-		while(1) asm("hlt");
 		break;
 
 	default:
 		cons->set_color(9);
-		cons->puts("\nReceived ISR interrupt:\n");
+		cons->printf("\nReceived %02x interrupt:\n",
+				(static_cast<isr>(regs->interrupt)));
+
 		cons->set_color();
+		cons->printf("         ISR: %02lx  ERR: %lx\n\n",
+				regs->interrupt, regs->errorcode);
 
-		cons->printf("         ISR: %02lx\n", regs.interrupt);
+		print_regs(*regs);
-		cons->printf("         ERR: %lx\n", regs.errorcode);
+		//print_stacktrace(2);
-		cons->puts("\n");
+		_halt();
-
-		print_reg(" ds", regs.ds);
-		print_reg("rdi", regs.rdi);
-		print_reg("rsi", regs.rsi);
-		print_reg("rbp", regs.rbp);
-		print_reg("rsp", regs.rsp);
-		print_reg("rbx", regs.rbx);
-		print_reg("rdx", regs.rdx);
-		print_reg("rcx", regs.rcx);
-		print_reg("rax", regs.rax);
-		cons->puts("\n");
-
-		print_reg("rip", regs.rip);
-		print_reg(" cs", regs.cs);
-		print_reg(" ef", regs.eflags);
-		print_reg("esp", regs.user_esp);
-		print_reg(" ss", regs.ss);
-
-		cons->puts("\n");
-		print_stacktrace(2);
-		while(1) asm("hlt");
 	}
-
 	*reinterpret_cast<uint32_t *>(0xffffff80fee000b0) = 0;
+
+	return return_rsp;
 }
 
-void
+uintptr_t
-irq_handler(registers regs)
+irq_handler(uintptr_t return_rsp, cpu_state *regs)
 {
 	console *cons = console::get();
-	uint8_t irq = get_irq(regs.interrupt);
+	uint8_t irq = get_irq(regs->interrupt);
 	if (! device_manager::get().dispatch_irq(irq)) {
 		cons->set_color(11);
 		cons->printf("\nReceived unknown IRQ: %d (vec %d)\n",
-				irq, regs.interrupt);
+				irq, regs->interrupt);
 		cons->set_color();
-
+		print_regs(*regs);
-		print_reg(" ds", regs.ds);
+		_halt();
-		print_reg("rdi", regs.rdi);
-		print_reg("rsi", regs.rsi);
-		print_reg("rbp", regs.rbp);
-		print_reg("rsp", regs.rsp);
-		print_reg("rbx", regs.rbx);
-		print_reg("rdx", regs.rdx);
-		print_reg("rcx", regs.rcx);
-		print_reg("rax", regs.rax);
-		cons->puts("\n");
-
-		print_reg("rip", regs.rip);
-		print_reg(" cs", regs.cs);
-		print_reg(" ef", regs.eflags);
-		print_reg("esp", regs.user_esp);
-		print_reg(" ss", regs.ss);
-		while(1) asm("hlt");
 	}
 
 	*reinterpret_cast<uint32_t *>(0xffffff80fee000b0) = 0;
+	return return_rsp;
 }
 
-
+uintptr_t
-void
+syscall_handler(uintptr_t return_rsp, cpu_state regs)
-gdt_dump(const table_ptr &table)
 {
-	log::info(logs::boot, "Loaded GDT at: %lx size: %d bytes", table.base, table.limit+1);
+	return syscall_dispatch(return_rsp, regs);
-
-	int count = (table.limit + 1) / sizeof(gdt_descriptor);
-	const gdt_descriptor *gdt =
-		reinterpret_cast<const gdt_descriptor *>(table.base);
-
-	for (int i = 0; i < count; ++i) {
-		uint32_t base =
-			(gdt[i].base_high << 24) |
-			(gdt[i].base_mid << 16) |
-			gdt[i].base_low;
-
-		uint32_t limit =
-			static_cast<uint32_t>(gdt[i].granularity & 0x0f) << 16 |
-			gdt[i].limit_low;
-
-		if (gdt[i].flags & 0x80) {
-			log::debug(logs::boot,
-					"   Entry %3d: Base %x  limit %x  privs %d  flags %s%s%s%s%s%s",
-					i, base, limit, ((gdt[i].flags >> 5) & 0x03),
-
-					(gdt[i].flags & 0x80) ? "P " : "  ",
-					(gdt[i].flags & 0x08) ? "ex " : "   ",
-					(gdt[i].flags & 0x04) ? "dc " : "   ",
-					(gdt[i].flags & 0x02) ? "rw " : "   ",
-
-					(gdt[i].granularity & 0x80) ? "kb " : "b  ",
-					(gdt[i].granularity & 0x60) == 0x60 ? "64" :
-						(gdt[i].granularity & 0x60) == 0x40 ? "32" : "16"
-					);
-		}
-	}
-}
-
-void
-idt_dump(const table_ptr &table)
-{
-	log::info(logs::boot, "Loaded IDT at: %lx size: %d bytes", table.base, table.limit+1);
-
-	int count = (table.limit + 1) / sizeof(idt_descriptor);
-	const idt_descriptor *idt =
-		reinterpret_cast<const idt_descriptor *>(table.base);
-
-	for (int i = 0; i < count; ++i) {
-		uint64_t base =
-			(static_cast<uint64_t>(idt[i].base_high) << 32) |
-			(static_cast<uint64_t>(idt[i].base_mid)  << 16) |
-			idt[i].base_low;
-
-		char const *type;
-		switch (idt[i].flags & 0xf) {
-			case 0x5: type = " 32tsk "; break;
-			case 0x6: type = " 16int "; break;
-			case 0x7: type = " 16trp "; break;
-			case 0xe: type = " 32int "; break;
-			case 0xf: type = " 32trp "; break;
-			default:  type = " ????? "; break;
-		}
-
-		if (idt[i].flags & 0x80) {
-			log::debug(logs::boot,
-					"   Entry %3d: Base:%lx Sel(rpl %d, ti %d, %3d) IST:%d %s DPL:%d", i, base,
-					(idt[i].selector & 0x3),
-					((idt[i].selector & 0x4) >> 2),
-					(idt[i].selector >> 3),
-					idt[i].ist,
-					type,
-					((idt[i].flags >> 5) & 0x3));
-		}
-	}
 }

src/kernel/interrupts.h

@@ -1,6 +1,7 @@
 #pragma once
 /// \file interrupts.h
 /// Free functions and definitions related to interrupt service vectors
+#include <stdint.h>
 
 
 /// Enum of all defined ISR/IRQ vectors
@@ -8,20 +9,28 @@ enum class isr : uint8_t
 {
 #define ISR(i, name)     name = i,
 #define EISR(i, name)    name = i,
+#define UISR(i, name)    name = i,
 #define IRQ(i, q, name)  name = i,
 #include "interrupt_isrs.inc"
 #undef IRQ
+#undef UISR
 #undef EISR
 #undef ISR
 
 	_zero = 0
 };
 
+/// Helper operator to add an offset to an isr vector
 isr operator+(const isr &lhs, int rhs);
 
 extern "C" {
+	/// Set the CPU interrupt enable flag (sti)
 	void interrupts_enable();
+
+	/// Set the CPU interrupt disable flag (cli)
 	void interrupts_disable();
 }
 
+/// Fill the IDT with our ISRs, and disable the legacy
+/// PIC interrupts.
 void interrupts_init();

src/kernel/interrupts.s

@@ -1,72 +1,14 @@
-extern g_idtr
+%include "push_all.inc"
-extern g_gdtr
-
-global idt_write
-idt_write:
-	lidt [rel g_idtr]
-	ret
-
-global idt_load
-idt_load:
-	sidt [rel g_idtr]
-	ret
-
-global gdt_write
-gdt_write:
-	lgdt [rel g_gdtr]
-	ret
-
-global gdt_load
-gdt_load:
-	sgdt [rel g_gdtr]
-	ret
-
-%macro push_all_and_segments 0
-	push rax
-	push rcx
-	push rdx
-	push rbx
-	push rsp
-	push rbp
-	push rsi
-	push rdi
-
-	mov ax, ds
-	push rax
-%endmacro
-
-%macro pop_all_and_segments 0
-	pop rax
-	mov ds, ax
-	mov es, ax
-	mov fs, ax
-	mov gs, ax
-
-	pop rdi
-	pop rsi
-	pop rbp
-	pop rsp
-	pop rbx
-	pop rdx
-	pop rcx
-	pop rax
-%endmacro
-
-%macro load_kernel_segments 0
-	mov ax, 0x10	; load the kernel data segment
-	mov ds, ax
-	mov es, ax
-	mov fs, ax
-	mov gs, ax
-%endmacro
 
 extern isr_handler
 global isr_handler_prelude
 isr_handler_prelude:
 	push_all_and_segments
-	load_kernel_segments
 
+	mov rdi, rsp
+	mov rsi, rsp
 	call isr_handler
+	mov rsp, rax
 
 	pop_all_and_segments
 
@@ -78,9 +20,11 @@ extern irq_handler
 global irq_handler_prelude
 irq_handler_prelude:
 	push_all_and_segments
-	load_kernel_segments
 
+	mov rdi, rsp
+	mov rsi, rsp
 	call irq_handler
+	mov rsp, rax
 
 	pop_all_and_segments
 
@@ -114,7 +58,8 @@ irq_handler_prelude:
 		jmp irq_handler_prelude
 %endmacro
 
-%define EISR(i, name)     EMIT_EISR name, i
+%define EISR(i, name)     EMIT_EISR name, i   ; ISR with error code
+%define UISR(i, name)     EMIT_ISR name, i    ; ISR callable from user space
 %define  ISR(i, name)     EMIT_ISR name, i
 %define  IRQ(i, q, name)  EMIT_IRQ name, i
 

src/kernel/io.cpp

@@ -14,24 +14,9 @@ outb(uint16_t port, uint8_t val)
 	__asm__ __volatile__ ( "outb %0, %1" :: "a"(val), "Nd"(port) );
 }
 
-uint64_t
-rdmsr(uint64_t addr)
-{
-	uint32_t low, high;
-	__asm__ __volatile__ ("rdmsr" : "=a"(low), "=d"(high) : "c"(addr));
-	return (static_cast<uint64_t>(high) << 32) | low;
-}
-
 void
-wrmsr(uint64_t addr, uint64_t value)
+io_wait(unsigned times)
 {
-	uint32_t low = value & 0xffffffff;
+	for (unsigned i = 0; i < times; ++i)
-	uint32_t high = value >> 32;
+		outb(0x80, 0);
-	__asm__ __volatile__ ("wrmsr" :: "c"(addr), "a"(low), "d"(high));
-}
-
-void
-io_wait()
-{
-	outb(0x80, 0);
 }

src/kernel/io.h

@@ -14,18 +14,9 @@ uint8_t inb(uint16_t port);
 /// \arg val   The byte to write
 void outb(uint16_t port, uint8_t val);
 
-/// Read the value of a MSR
-/// \arg addr   The MSR address
-/// \returns    The current value of the MSR
-uint64_t rdmsr(uint64_t addr);
-
-/// Write to a MSR
-/// \arg addr   The MSR address
-/// \arg value  The value to write
-void wrmsr(uint64_t addr, uint64_t value);
-
 /// Pause briefly by doing IO to port 0x80
-void io_wait();
+/// \arg times  Number of times to delay by writing
+void io_wait(unsigned times = 1);
 
 }
 

src/kernel/kernel_memory.h

@@ -0,0 +1,22 @@
+#pragma once
+/// \file kernel_memory.h
+/// Constants related to the kernel's memory layout
+
+namespace memory {
+
+	/// Size of a single page frame.
+	static const size_t frame_size = 0x1000;
+
+	/// Start of kernel memory.
+	static const uintptr_t kernel_offset = 0xffffff0000000000;
+
+	/// Offset from physical where page tables are mapped.
+	static const uintptr_t page_offset = 0xffffff8000000000;
+
+	/// Initial process thread's stack address
+	static const uintptr_t initial_stack = 0x0000800000000000;
+
+	/// Initial process thread's stack size, in pages
+	static const unsigned initial_stack_pages = 1;
+
+} // namespace memory

src/kernel/loader.s

@@ -0,0 +1,25 @@
+%include "push_all.inc"
+
+extern load_process
+
+global ramdisk_process_loader
+ramdisk_process_loader:
+
+	; create_process already pushed a cpu_state onto the stack for us, this
+	; acts both as the cpu_state parameter to load_process, and the saved
+	; state for the following iretq
+	;
+	; Additional parameters:
+	;  rax - the address of the program image
+	;  rbx - the size of the program image
+	;  rcx - the address of this process' process structure
+	mov rdi, rax
+	mov rsi, rbx
+	mov rdx, rcx
+	call load_process
+
+	pop_all_and_segments
+	add rsp, 16		; because the ISRs add err/num
+	iretq
+
+

src/kernel/log.cpp

@@ -10,11 +10,14 @@ static const uint8_t level_colors[] = {0x07, 0x0f, 0x0b, 0x09};
 
 static const char *levels[] = {"debug", " info", " warn", "error", "fatal"};
 static const char *areas[] = {
-	"boot",
+	"boot  ",
-	"mem ",
+	"memory",
-	"apic",
+	"apic  ",
-	"dev ",
+	"device",
-	"driv",
+	"driver",
+	"file  ",
+	"task  ",
+	"paging",
 
 	nullptr
 };

src/kernel/log.h

@@ -12,6 +12,9 @@ enum class logs
 	apic,
 	device,
 	driver,
+	fs,
+	task,
+	paging,
 
 	max
 };

src/kernel/main.cpp

@@ -1,54 +1,37 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include "initrd/initrd.h"
 #include "kutil/assert.h"
-#include "kutil/memory.h"
+#include "apic.h"
-
+#include "block_device.h"
-#include "ahci/driver.h"
-#include "ahci/port.h"
-
 #include "console.h"
 #include "cpu.h"
 #include "device_manager.h"
-#include "font.h"
+#include "gdt.h"
 #include "interrupts.h"
 #include "io.h"
 #include "kernel_data.h"
 #include "log.h"
-#include "memory.h"
 #include "page_manager.h"
+#include "scheduler.h"
 #include "screen.h"
 #include "serial.h"
+#include "syscall.h"
 
 extern "C" {
-	void do_the_set_registers(popcorn_data *header);
 	void kernel_main(popcorn_data *header);
-
 	void *__bss_start, *__bss_end;
 }
 
-extern ahci_driver ahcid;
-
 extern void __kernel_assert(const char *, unsigned, const char *);
 
 void
-init_console(const popcorn_data *header)
+init_console()
 {
 	serial_port *com1 = new (&g_com1) serial_port(COM1);
 	console *cons = new (&g_console) console(com1);
 
-	if (header->frame_buffer) {
-		screen *s = new screen(
-				header->frame_buffer,
-				header->hres,
-				header->vres,
-				header->rmask,
-				header->gmask,
-				header->bmask);
-		font *f = new font(header->font);
-		cons->init_screen(s, f);
-	}
-
 	cons->set_color(0x21, 0x00);
 	cons->puts("Popcorn OS ");
 	cons->set_color(0x08, 0x00);
@@ -56,69 +39,119 @@ init_console(const popcorn_data *header)
 
 	log::init(cons);
 	log::enable(logs::apic, log::level::info);
-	log::enable(logs::device, log::level::debug);
+	log::enable(logs::device, log::level::info);
+
 	log::enable(logs::driver, log::level::debug);
 	log::enable(logs::memory, log::level::debug);
+	log::enable(logs::fs, log::level::debug);
+	log::enable(logs::task, log::level::debug);
+	log::enable(logs::boot, log::level::debug);
+	log::enable(logs::paging, log::level::debug);
 }
 
-void do_error_3() { volatile int x = 1; volatile int y = 0; volatile int z = x / y; }
-void do_error_2() { do_error_3(); }
-void do_error_1() { do_error_2(); }
-
 void
 kernel_main(popcorn_data *header)
 {
+#ifdef DEBUG
+	// Run `waf configure --debug` to enable compiling with DEBUG turned on.
+	// Then attach to QEMU's gdb server and `set waiting = false` to start
+	// the kernel. This compensates for GDB's poor handling of QEMU going
+	// through the x86 PC startup and switching to 64 bit mode when you
+	// attach to qemu with the -S option.
+	bool waiting = true;
+	while (waiting);
+#endif
+
 	kutil::assert_set_callback(__kernel_assert);
 
-	page_manager *pager = new (&g_page_manager) page_manager;
+	gdt_init();
+	interrupts_init();
 
 	memory_initialize(
+			header->scratch_pages,
 			header->memory_map,
 			header->memory_map_length,
 			header->memory_map_desc_size);
 
-	pager->map_offset_pointer(
+	if (header->frame_buffer && header->frame_buffer_length) {
-			&header->frame_buffer,
+		page_manager::get()->map_offset_pointer(
-			header->frame_buffer_length);
+				&header->frame_buffer,
+				header->frame_buffer_length);
+	}
 
-	init_console(header);
+	init_console();
-	// pager->dump_blocks();
 
-	interrupts_init();
+	log::debug(logs::boot, " Popcorn header is at: %016lx", header);
+	log::debug(logs::boot, "    Framebuffer is at: %016lx", header->frame_buffer);
+	log::debug(logs::boot, "    Kernel data is at: %016lx", header->data);
+	log::debug(logs::boot, "     Memory map is at: %016lx", header->memory_map);
+	log::debug(logs::boot, "ACPI root table is at: %016lx", header->acpi_table);
+	log::debug(logs::boot, "Runtime service is at: %016lx", header->runtime);
+
+	initrd::disk ird(header->initrd);
+	log::info(logs::boot, "initrd loaded with %d files.", ird.files().count());
+	for (auto &f : ird.files())
+		log::info(logs::boot, "  %s%s (%d bytes).", f.executable() ? "*" : "", f.name(), f.size());
+
+	/*
+	   page_manager::get()->dump_pml4(nullptr, 0);
+	   page_manager::get()->dump_blocks(true);
+	*/
 
 	device_manager *devices =
 		new (&device_manager::get()) device_manager(header->acpi_table);
 
 	interrupts_enable();
 
+	/*
 	cpu_id cpu;
 	log::info(logs::boot, "CPU Vendor: %s", cpu.vendor_id());
 	log::info(logs::boot, "CPU Family %x Model %x Stepping %x",
 			cpu.family(), cpu.model(), cpu.stepping());
 
+	auto r = cpu.get(0x15);
+	log::info(logs::boot, "CPU Crystal: %dHz", r.ecx);
+
+	uintptr_t cr4 = 0;
+	__asm__ __volatile__ ( "mov %%cr4, %0" : "=r" (cr4) );
+	log::info(logs::boot, "cr4: %016x", cr4);
+	*/
+
 	devices->init_drivers();
 
-	ahci::port *disk = ahcid.find_disk();
+	/*
+	block_device *disk = devices->get_block_device(0);
 	if (disk) {
-		uint8_t buf[512];
+		for (int i=0; i<1; ++i) {
-		kutil::memset(buf, 0, 512);
+			uint8_t buf[512];
+			kutil::memset(buf, 0, 512);
 
-		disk->read(1, sizeof(buf), buf);
+			kassert(disk->read(0x200, sizeof(buf), buf),
-		while (buf[0] == 0) io_wait();
+					"Disk read returned 0");
 
-		console *cons = console::get();
+			console *cons = console::get();
-		uint8_t *p = &buf[0];
+			uint8_t *p = &buf[0];
-		for (int i = 0; i < 8; ++i) {
+			for (int i = 0; i < 8; ++i) {
-			for (int j = 0; j < 16; ++j) {
+				for (int j = 0; j < 16; ++j) {
-				cons->printf(" %02x", *p++);
+					cons->printf(" %02x", *p++);
+				}
+				cons->putc('\n');
 			}
-			cons->putc('\n');
 		}
+	} else {
+		log::warn(logs::boot, "No block devices present.");
+	}
+	*/
+
+	devices->get_lapic()->calibrate_timer();
+
+	syscall_enable();
+	scheduler *sched = new (&scheduler::get()) scheduler(devices->get_lapic());
+
+	for (auto &f : ird.files()) {
+		if (f.executable())
+			sched->load_process(f.name(), f.data(), f.size());
 	}
 
-	// do_error_1();
+	sched->start();
-	// __asm__ __volatile__("int $15");
-
-	g_console.puts("boogity!");
-	do_the_set_registers(header);
 }

src/kernel/memory_bootstrap.cpp

@@ -1,9 +1,73 @@
+#include <algorithm>
+#include <utility>
+#include "kutil/address_manager.h"
 #include "kutil/assert.h"
-#include "kutil/memory.h"
+#include "kutil/frame_allocator.h"
-#include "memory.h"
+#include "kutil/heap_manager.h"
+#include "io.h"
+#include "log.h"
 #include "page_manager.h"
 
-const unsigned efi_page_size = 0x1000;
+using kutil::frame_block;
+using kutil::frame_block_flags;
+using kutil::frame_block_list;
+using memory::frame_size;
+using memory::kernel_offset;
+using memory::page_offset;
+
+static const unsigned ident_page_flags = 0xb;
+
+kutil::frame_allocator g_frame_allocator;
+kutil::address_manager g_kernel_address_manager;
+kutil::heap_manager g_kernel_heap_manager;
+
+void * mm_grow_callback(size_t length)
+{
+	kassert(length % frame_size == 0,
+			"Heap manager requested a fractional page.");
+
+	size_t pages = length / frame_size;
+	log::info(logs::memory, "Heap manager growing heap by %d pages.", pages);
+
+	uintptr_t addr = g_kernel_address_manager.allocate(length);
+	g_page_manager.map_pages(addr, pages);
+
+	return reinterpret_cast<void *>(addr);
+}
+
+
+namespace {
+	// Page-by-page initial allocator for the initial frame_block allocator
+	struct page_consumer
+	{
+		page_consumer(uintptr_t start, unsigned count, unsigned used = 0) :
+			current(start + used * frame_size),
+			used(used),
+			max(count) {}
+
+		void * get_page() {
+			kassert(used++ < max, "page_consumer ran out of pages");
+			void *retval = reinterpret_cast<void *>(current);
+			current += frame_size;
+			return retval;
+		}
+
+		void * operator()(size_t size) {
+			kassert(size == frame_size, "page_consumer used with non-page size!");
+			return get_page();
+		}
+
+		unsigned left() const { return max - used; }
+
+		uintptr_t current;
+		unsigned used, max;
+	};
+
+	using block_allocator =
+		kutil::slab_allocator<kutil::frame_block, page_consumer &>;
+	using region_allocator =
+		kutil::slab_allocator<kutil::buddy_region, page_consumer &>;
+}
 
 enum class efi_memory_type : uint32_t
 {
@@ -26,66 +90,13 @@ enum class efi_memory_type : uint32_t
 	efi_max,
 
 	popcorn_kernel = 0x80000000,
-	popcorn_font,
 	popcorn_data,
-	popcorn_log,
+	popcorn_initrd,
-	popcorn_pml4,
+	popcorn_scratch,
 
 	popcorn_max
 };
 
-const char *efi_memory_type_names[] = {
-	"             reserved",
-	"          loader_code",
-	"          loader_data",
-	"   boot_services_code",
-	"   boot_services_data",
-	"runtime_services_code",
-	"runtime_services_data",
-	"            available",
-	"             unusable",
-	"         acpi_reclaim",
-	"             acpi_nvs",
-	"                 mmio",
-	"            mmio_port",
-	"             pal_code",
-
-	"       popcorn_kernel",
-	"         popcorn_font",
-	"         popcorn_data",
-	"          popcorn_log",
-	"         popcorn_pml4",
-};
-
-static const char *
-get_efi_name(efi_memory_type t)
-{
-	static const unsigned offset =
-		(unsigned)efi_memory_type::popcorn_kernel - (unsigned)efi_memory_type::efi_max;
-
-	return t >= efi_memory_type::popcorn_kernel ?
-		efi_memory_type_names[(unsigned)t - offset] :
-		efi_memory_type_names[(unsigned)t];
-}
-
-enum class efi_memory_flag : uint64_t
-{
-	can_mark_uc  = 0x0000000000000001, // uc = un-cacheable
-	can_mark_wc  = 0x0000000000000002, // wc = write-combining
-	can_mark_wt  = 0x0000000000000004, // wt = write through
-	can_mark_wb  = 0x0000000000000008, // wb = write back
-	can_mark_uce = 0x0000000000000010, // uce = un-cacheable exported
-	can_mark_wp  = 0x0000000000001000, // wp = write protected
-	can_mark_rp  = 0x0000000000002000, // rp = read protected
-	can_mark_xp  = 0x0000000000004000, // xp = exceute protected
-	can_mark_ro  = 0x0000000000020000, // ro = read only
-
-	non_volatile  = 0x0000000000008000,
-	more_reliable = 0x0000000000010000,
-	runtime       = 0x8000000000000000
-};
-IS_BITFIELD(efi_memory_flag);
-
 struct efi_memory_descriptor
 {
 	efi_memory_type type;
@@ -93,7 +104,7 @@ struct efi_memory_descriptor
 	uint64_t physical_start;
 	uint64_t virtual_start;
 	uint64_t pages;
-	efi_memory_flag flags;
+	uint64_t flags;
 };
 
 static const efi_memory_descriptor *
@@ -103,430 +114,191 @@ desc_incr(const efi_memory_descriptor *d, size_t desc_length)
 			reinterpret_cast<const uint8_t *>(d) + desc_length);
 }
 
-static unsigned
+void
-count_table_pages_needed(page_block *used)
-{
-	page_table_indices last_idx{~0ull};
-	unsigned counts[] = {1, 0, 0, 0};
-
-	for (page_block *cur = used; cur; cur = cur->next) {
-		if (!cur->has_flag(page_block_flags::mapped))
-			continue;
-
-		page_table_indices start{cur->virtual_address};
-		page_table_indices end{cur->virtual_address + (cur->count * page_manager::page_size)};
-
-		counts[1] +=
-			((start[0] == last_idx[0]) ? 0 : 1) +
-			(end[0] - start[0]);
-
-		counts[2] +=
-			((start[0] == last_idx[0] &&
-			 start[1] == last_idx[1]) ? 0 : 1) +
-			(end[1] - start[1]);
-
-		counts[3] +=
-			((start[0] == last_idx[0] &&
-			 start[1] == last_idx[1] &&
-			 start[2] == last_idx[2]) ? 0 : 1) +
-			(end[2] - start[2]);
-
-		last_idx = end;
-	}
-
-	return counts[0] + counts[1] + counts[2] + counts[3];
-
-}
-
-page_block *
-remove_block_for(page_block **list, uint64_t phys_start, uint64_t pages, page_block **cache)
-{
-	// This is basically just the removal portion of page_manager::unmap_pages,
-	// but with physical addresses, and only ever removing a single block.
-
-	page_block *prev = nullptr;
-	page_block *cur = *list;
-	while (cur && !cur->contains_physical(phys_start)) {
-		prev = cur;
-		cur = cur->next;
-	}
-
-	kassert(cur, "Couldn't find block to remove");
-
-	uint64_t size = page_manager::page_size * pages;
-	uint64_t end = phys_start + size;
-	uint64_t leading = phys_start - cur->physical_address;
-	uint64_t trailing = cur->physical_end() - end;
-
-	if (leading) {
-		uint64_t pages = leading / page_manager::page_size;
-
-		page_block *lead_block = *cache;
-		*cache = (*cache)->next;
-
-		lead_block->copy(cur);
-		lead_block->next = cur;
-		lead_block->count = pages;
-
-		cur->count -= pages;
-		cur->physical_address += leading;
-
-		if (cur->virtual_address)
-			cur->virtual_address += leading;
-
-		if (prev) {
-			prev->next = lead_block;
-		} else {
-			prev = lead_block;
-			*list = prev;
-		}
-	}
-
-	if (trailing) {
-		uint64_t pages = trailing / page_manager::page_size;
-
-		page_block *trail_block = *cache;
-		*cache = (*cache)->next;
-
-		trail_block->copy(cur);
-		trail_block->next = cur->next;
-		trail_block->count = pages;
-		trail_block->physical_address += size;
-
-		if (cur->virtual_address)
-			trail_block->virtual_address += size;
-
-		cur->count -= pages;
-		cur->next = trail_block;
-	}
-
-	prev->next = cur->next;
-	cur->next = nullptr;
-	return cur;
-}
-
-uint64_t
 gather_block_lists(
-		uint64_t scratch_virt,
+		block_allocator &allocator,
+		frame_block_list &used,
+		frame_block_list &free,
 		const void *memory_map,
 		size_t map_length,
-		size_t desc_length,
+		size_t desc_length)
-		page_block **free_head,
-		page_block **used_head)
 {
-	int i = 0;
-	page_block *free = nullptr;
-	page_block *used = nullptr;
-
-	page_block *block_list = reinterpret_cast<page_block *>(scratch_virt);
 	efi_memory_descriptor const *desc = reinterpret_cast<efi_memory_descriptor const *>(memory_map);
 	efi_memory_descriptor const *end = desc_incr(desc, map_length);
 
 	while (desc < end) {
-		page_block *block = &block_list[i++];
+		auto *block = allocator.pop();
-		block->physical_address = desc->physical_start;
+		block->address = desc->physical_start;
-		block->virtual_address = desc->virtual_start;
 		block->count = desc->pages;
-		block->next = nullptr;
+		bool block_used;
 
 		switch (desc->type) {
 		case efi_memory_type::loader_code:
 		case efi_memory_type::loader_data:
-			block->flags = page_block_flags::used | page_block_flags::pending_free;
+			block_used = true;
+			block->flags = frame_block_flags::pending_free;
 			break;
 
 		case efi_memory_type::boot_services_code:
 		case efi_memory_type::boot_services_data:
 		case efi_memory_type::available:
-			block->flags = page_block_flags::free;
+			block_used = false;
 			break;
 
 		case efi_memory_type::acpi_reclaim:
+			block_used = true;
 			block->flags =
-				page_block_flags::used |
+				frame_block_flags::acpi_wait |
-				page_block_flags::mapped |
+				frame_block_flags::map_ident;
-				page_block_flags::acpi_wait;
-
-			block->virtual_address = block->physical_address;
 			break;
 
 		case efi_memory_type::persistent:
-			block->flags = page_block_flags::nonvolatile;
+			block_used = false;
+			block->flags = frame_block_flags::nonvolatile;
+			break;
+
+		case efi_memory_type::popcorn_kernel:
+			block_used = true;
+			block->flags =
+				frame_block_flags::permanent |
+				frame_block_flags::map_kernel;
+			break;
+
+		case efi_memory_type::popcorn_data:
+		case efi_memory_type::popcorn_initrd:
+			block_used = true;
+			block->flags =
+				frame_block_flags::pending_free |
+				frame_block_flags::map_kernel;
+			break;
+
+		case efi_memory_type::popcorn_scratch:
+			block_used = true;
+			block->flags = frame_block_flags::map_offset;
 			break;
 
 		default:
-			block->flags = page_block_flags::used | page_block_flags::permanent;
+			block_used = true;
+			block->flags = frame_block_flags::permanent;
 			break;
 		}
 
-		if (block->has_flag(page_block_flags::used)) {
+		if (block_used) 
-			if (block->virtual_address || !block->physical_address)
+			used.push_back(block);
-				block->flags |= page_block_flags::mapped;
+		else
-
+			free.push_back(block);
-			used = page_block::insert(used, block);
-		} else {
-			free = page_block::insert(free, block);
-		}
 
 		desc = desc_incr(desc, desc_length);
 	}
-
-	*free_head = free;
-	*used_head = used;
-	return reinterpret_cast<uint64_t>(&block_list[i]);
-}
-
-page_block *
-fill_page_with_blocks(uint64_t start)
-{
-	uint64_t end = page_align(start);
-	uint64_t count = (end - start) / sizeof(page_block);
-	if (count == 0) return nullptr;
-
-	page_block *blocks = reinterpret_cast<page_block *>(start);
-	for (unsigned i = 0; i < count; ++i)
-		blocks[i].zero(&blocks[i+1]);
-	blocks[count - 1].next = nullptr;
-	return blocks;
 }
 
 void
-copy_new_table(page_table *base, unsigned index, page_table *new_table)
+memory_initialize(uint16_t scratch_pages, const void *memory_map, size_t map_length, size_t desc_length)
 {
-	uint64_t entry = base->entries[index];
+	// make sure the options we want in CR4 are set
+	uint64_t cr4;
+	__asm__ __volatile__ ( "mov %%cr4, %0" : "=r" (cr4) );
+	cr4 |= 0x00080; // Enable global pages
+	cr4 |= 0x00200; // Enable FXSAVE/FXRSTOR
+	cr4 |= 0x20000; // Enable PCIDs
+	__asm__ __volatile__ ( "mov %0, %%cr4" :: "r" (cr4) );
 
-	// If this is a large page and not a a table, bail out.
+	// The bootloader reserved "scratch_pages" pages for page tables and
-	if(entry & 0x80) return;
+	// scratch space, which we'll use to bootstrap.  The first one is the
+	// already-installed PML4, so grab it from CR3.
+	uint64_t scratch_phys;
+	__asm__ __volatile__ ( "mov %%cr3, %0" : "=r" (scratch_phys) );
+	scratch_phys &= ~0xfffull;
 
-	if (entry & 0x1) {
+	// The tables are ident-mapped currently, so the cr3 physical address works. But let's
-		page_table *old_next = reinterpret_cast<page_table *>(
+	// get them into the offset-mapped area asap.
-				base->entries[index] & ~0xffful);
+	page_table *tables = reinterpret_cast<page_table *>(scratch_phys);
-		for (int i = 0; i < 512; ++i) new_table->entries[i] = old_next->entries[i];
-	} else {
-		for (int i = 0; i < 512; ++i) new_table->entries[i] = 0;
-	}
 
-	base->entries[index] = reinterpret_cast<uint64_t>(new_table) | 0xb;
+	page_table *id_pml4 = &tables[0];
-}
+	page_table *id_pdp = &tables[1];
+	for (int i=0; i<512; ++i)
+		id_pdp->entries[i] = (static_cast<uintptr_t>(i) << 30) | 0x18b;
+	id_pml4->entries[511] = reinterpret_cast<uintptr_t>(id_pdp) | 0x10b;
 
-static uint64_t
+	// Make sure the page table is finished updating before we write to memory
-find_efi_free_aligned_pages(const void *memory_map, size_t map_length, size_t desc_length, unsigned pages)
+	__sync_synchronize();
-{
+	io_wait();
-	efi_memory_descriptor const *desc =
-		reinterpret_cast<efi_memory_descriptor const *>(memory_map);
-	efi_memory_descriptor const *end = desc_incr(desc, map_length);
 
-	const unsigned want_space = pages * page_manager::page_size;
+	// We now have pages starting at "scratch_virt" to bootstrap ourselves. Start by
-	uint64_t start_phys = 0;
-
-	for (; desc < end; desc = desc_incr(desc, desc_length)) {
-		if (desc->type != efi_memory_type::available)
-			continue;
-
-		// See if the first wanted pages fit in one page table. If we
-		// find free memory at zero, skip ahead because we're not ready
-		// to deal with 0 being a valid pointer yet.
-		start_phys = desc->physical_start;
-		if (start_phys == 0)
-			start_phys += efi_page_size;
-
-		const uint64_t desc_end =
-			desc->physical_start + desc->pages * efi_page_size;
-
-		uint64_t end = start_phys + want_space;
-		if (end < desc_end) {
-			page_table_indices start_idx{start_phys};
-			page_table_indices end_idx{end};
-			if (start_idx[0] == end_idx[0] &&
-				start_idx[1] == end_idx[1] &&
-				start_idx[2] == end_idx[2])
-				break;
-
-			// Try seeing if the page-table-aligned version fits
-			start_phys = page_table_align(start_phys);
-			end = start_phys + want_space;
-			if (end < desc_end)
-				break;
-		}
-	}
-
-	kassert(desc < end, "Couldn't find wanted pages of aligned scratch space.");
-	return start_phys;
-}
-
-static unsigned
-check_needs_page_ident(page_table *table, unsigned index, page_table **free_pages)
-{
-	if ((table->entries[index] & 0x1) == 1) return 0;
-
-	kassert(*free_pages, "check_needs_page_ident needed to allocate but had no free pages");
-
-	page_table *new_table = (*free_pages)++;
-	for (int i=0; i<512; ++i) new_table->entries[i] = 0;
-	table->entries[index] = reinterpret_cast<uint64_t>(new_table) | 0xb;
-	return 1;
-}
-
-static unsigned
-page_in_ident(
-		page_table *pml4,
-		uint64_t phys_addr,
-		uint64_t virt_addr,
-		uint64_t count,
-		page_table *free_pages)
-{
-	page_table_indices idx{virt_addr};
-	page_table *tables[4] = {pml4, nullptr, nullptr, nullptr};
-
-	unsigned pages_consumed = 0;
-	for (; idx[0] < 512; idx[0] += 1) {
-		pages_consumed += check_needs_page_ident(tables[0], idx[0], &free_pages);
-		tables[1] = reinterpret_cast<page_table *>(
-				tables[0]->entries[idx[0]] & ~0xfffull);
-
-		for (; idx[1] < 512; idx[1] += 1, idx[2] = 0, idx[3] = 0) {
-			pages_consumed += check_needs_page_ident(tables[1], idx[1], &free_pages);
-			tables[2] = reinterpret_cast<page_table *>(
-					tables[1]->entries[idx[1]] & ~0xfffull);
-
-			for (; idx[2] < 512; idx[2] += 1, idx[3] = 0) {
-				pages_consumed += check_needs_page_ident(tables[2], idx[2], &free_pages);
-				tables[3] = reinterpret_cast<page_table *>(
-						tables[2]->entries[idx[2]] & ~0xfffull);
-
-				for (; idx[3] < 512; idx[3] += 1) {
-					tables[3]->entries[idx[3]] = phys_addr | 0xb;
-					phys_addr += page_manager::page_size;
-					if (--count == 0) return pages_consumed;
-				}
-			}
-		}
-	}
-
-	kassert(0, "Ran to end of page_in_ident");
-	return 0; // Cannot reach
-}
-
-void
-memory_initialize(const void *memory_map, size_t map_length, size_t desc_length)
-{
-	// The bootloader reserved 16 pages for page tables, which we'll use to bootstrap.
-	// The first one is the already-installed PML4, so grab it from CR3.
-	uint64_t cr3;
-	__asm__ __volatile__ ( "mov %%cr3, %0" : "=r" (cr3) );
-	page_table *tables = reinterpret_cast<page_table *>(cr3 & ~0xfffull);
-
-	// Now go through EFi's memory map and find a region of scratch space.
-	const unsigned want_pages = 32;
-	uint64_t free_region_start_phys =
-		find_efi_free_aligned_pages(memory_map, map_length, desc_length, want_pages);
-
-	// Offset-map this region into the higher half.
-	uint64_t free_region_start_virt =
-		free_region_start_phys + page_manager::high_offset;
-
-	uint64_t free_next = free_region_start_virt;
-
-	// We'll need to copy any existing tables (except the PML4 which the
-	// bootloader gave us) into our 4 reserved pages so we can edit them.
-	page_table_indices fr_idx{free_region_start_virt};
-
-	copy_new_table(&tables[0], fr_idx[0], &tables[1]);
-	copy_new_table(&tables[1], fr_idx[1], &tables[2]);
-	copy_new_table(&tables[2], fr_idx[2], &tables[3]);
-	page_in_ident(&tables[0], free_region_start_phys, free_region_start_virt, want_pages, nullptr);
-
-	// We now have pages starting at "free_next" to bootstrap ourselves. Start by
 	// taking inventory of free pages.
-	page_block *free_head = nullptr;
+	uintptr_t scratch_virt = scratch_phys + page_offset;
-	page_block *used_head = nullptr;
+	uint64_t used_pages = 2; // starts with PML4 + offset PDP
-	free_next = gather_block_lists(
+	page_consumer allocator(scratch_virt, scratch_pages, used_pages);
-			free_next, memory_map, map_length, desc_length,
-			&free_head, &used_head);
 
-	// Unused page_block structs go here - finish out the current page with them
+	block_allocator block_slab(frame_size, allocator);
-	page_block *cache_head = fill_page_with_blocks(free_next);
+	frame_block_list used;
-	free_next = page_align(free_next);
+	frame_block_list free;
+
+	gather_block_lists(block_slab, used, free, memory_map, map_length, desc_length);
+	block_slab.allocate(); // Make sure we have extra
 
 	// Now go back through these lists and consolidate
-	page_block *freed = page_block::consolidate(free_head);
+	block_slab.append(frame_block::consolidate(free));
-	cache_head = page_block::append(cache_head, freed);
 
-	freed = page_block::consolidate(used_head);
+	region_allocator region_slab(frame_size, allocator);
-	cache_head = page_block::append(cache_head, freed);
+	region_slab.allocate(); // Allocate some buddy regions for the address_manager
 
+	kutil::address_manager *am =
+		new (&g_kernel_address_manager) kutil::address_manager(std::move(region_slab));
 
-	// Pull out the block that represents the bootstrap pages we've used
+	am->add_regions(kernel_offset, page_offset - kernel_offset);
-	uint64_t used = free_next - free_region_start_virt;
-	uint64_t used_pages = used / page_manager::page_size;
-	uint64_t remaining_pages = want_pages - used_pages;
-
-	page_block *removed = remove_block_for(
-			&free_head,
-			free_region_start_phys,
-			used_pages,
-			&cache_head);
-
-	kassert(removed, "remove_block_for didn't find the bootstrap region.");
-	kassert(removed->physical_address == free_region_start_phys,
-			"remove_block_for found the wrong region.");
-
-	// Add it to the used list
-	removed->virtual_address = free_region_start_virt;
-	removed->flags = page_block_flags::used | page_block_flags::mapped;
-	used_head = page_block::insert(used_head, removed);
-
-	// Pull out the block that represents the rest
-	uint64_t free_next_phys = free_region_start_phys + used;
-
-	removed = remove_block_for(
-			&free_head,
-			free_next_phys,
-			remaining_pages,
-			&cache_head);
-
-	kassert(removed, "remove_block_for didn't find the page table region.");
-	kassert(removed->physical_address == free_next_phys,
-			"remove_block_for found the wrong region.");
-
-	uint64_t pt_start_phys = removed->physical_address;
-	uint64_t pt_start_virt = removed->physical_address + page_manager::page_offset;
-
-	// Record that we're about to remap it into the page table address space
-	removed->virtual_address = pt_start_virt;
-	removed->flags = page_block_flags::used | page_block_flags::mapped;
-	used_head = page_block::insert(used_head, removed);
-
-	page_manager *pm = &g_page_manager;
-
-	// Actually remap them into page table space
-	pm->page_out(&tables[0], free_next, remaining_pages);
-
-	page_table_indices pg_idx{pt_start_virt};
-	copy_new_table(&tables[0], pg_idx[0], &tables[4]);
-	copy_new_table(&tables[4], pg_idx[1], &tables[5]);
-	copy_new_table(&tables[5], pg_idx[2], &tables[6]);
-
-	page_in_ident(&tables[0], pt_start_phys, pt_start_virt, remaining_pages, tables + 4);
 
 	// Finally, build an acutal set of kernel page tables that just contains
 	// what the kernel actually has mapped, but making everything writable
 	// (especially the page tables themselves)
-	page_table *pml4 = reinterpret_cast<page_table *>(pt_start_virt);
+	page_table *pml4 = reinterpret_cast<page_table *>(allocator.get_page());
-	for (int i=0; i<512; ++i) pml4->entries[i] = 0;
+	kutil::memset(pml4, 0, sizeof(page_table));
+	pml4->entries[511] = reinterpret_cast<uintptr_t>(id_pdp) | 0x10b;
+
+	kutil::frame_allocator *fa =
+		new (&g_frame_allocator) kutil::frame_allocator(std::move(block_slab));
+	page_manager *pm = new (&g_page_manager) page_manager(*fa, *am);
 
 	// Give the rest to the page_manager's cache for use in page_in
-	pm->free_table_pages(pml4 + 1, remaining_pages - 1);
+	pm->free_table_pages(
+			reinterpret_cast<void *>(allocator.current),
+			allocator.left());
 
-	for (page_block *cur = used_head; cur; cur = cur->next) {
+	for (auto *block : used) {
-		if (!cur->has_flag(page_block_flags::mapped)) continue;
+		uintptr_t virt_addr = 0;
-		pm->page_in(pml4, cur->physical_address, cur->virtual_address, cur->count);
+
+		switch (block->flags & frame_block_flags::map_mask) {
+			case frame_block_flags::map_ident:
+				virt_addr = block->address;
+				break;
+
+			case frame_block_flags::map_kernel:
+				virt_addr = block->address + kernel_offset;
+				if (block->flags && frame_block_flags::permanent)
+					am->mark_permanent(virt_addr, block->count * frame_size);
+				else
+					am->mark(virt_addr, block->count * frame_size);
+				break;
+
+			default:
+				break;
+		}
+
+		block->flags -= frame_block_flags::map_mask;
+		if (virt_addr)
+			pm->page_in(pml4, block->address, virt_addr, block->count);
 	}
 
+	fa->init(std::move(free), std::move(used));
+
 	// Put our new PML4 into CR3 to start using it
 	page_manager::set_pml4(pml4);
+	pm->m_kernel_pml4 = pml4;
 
-	// We now have all used memory mapped ourselves. Let the page_manager take
+	// Give the old pml4 back to the page_manager to recycle
-	// over from here.
+	pm->free_table_pages(reinterpret_cast<void *>(scratch_virt), 1);
-	g_page_manager.init(free_head, used_head, cache_head);
+
+	// Set the heap manager
+	new (&g_kernel_heap_manager) kutil::heap_manager(mm_grow_callback);
+	kutil::setup::set_heap(&g_kernel_heap_manager);
 }

src/kernel/msr.cpp

@@ -0,0 +1,18 @@
+#include "msr.h"
+
+uint64_t
+rdmsr(msr addr)
+{
+	uint32_t low, high;
+	__asm__ __volatile__ ("rdmsr" : "=a"(low), "=d"(high) : "c"(addr));
+	return (static_cast<uint64_t>(high) << 32) | low;
+}
+
+void
+wrmsr(msr addr, uint64_t value)
+{
+	uint32_t low = value & 0xffffffff;
+	uint32_t high = value >> 32;
+	__asm__ __volatile__ ("wrmsr" :: "c"(addr), "a"(low), "d"(high));
+}
+

src/kernel/msr.h

@@ -0,0 +1,27 @@
+#pragma once
+/// \file msr.h
+/// Routines and definitions for dealing with Model-Specific Registers
+
+#include <stdint.h>
+
+enum class msr : uint32_t
+{
+	ia32_efer			= 0xc0000080,
+	ia32_star			= 0xc0000081,
+	ia32_lstar			= 0xc0000082,
+	ia32_fmask			= 0xc0000084,
+
+	ia32_gs_base		= 0xc0000101,
+	ia32_kernel_gs_base = 0xc0000102
+};
+
+/// Read the value of a MSR
+/// \arg addr   The MSR address
+/// \returns    The current value of the MSR
+uint64_t rdmsr(msr addr);
+
+/// Write to a MSR
+/// \arg addr   The MSR address
+/// \arg value  The value to write
+void wrmsr(msr addr, uint64_t value);
+

src/kernel/page_manager.cpp

@@ -1,24 +1,32 @@
 #include <algorithm>
 
 #include "kutil/assert.h"
-#include "kutil/memory_manager.h"
+#include "console.h"
 #include "log.h"
 #include "page_manager.h"
 
-page_manager g_page_manager;
+using memory::frame_size;
+using memory::kernel_offset;
+using memory::page_offset;
+
+extern kutil::frame_allocator g_frame_allocator;
+extern kutil::address_manager g_kernel_address_manager;
+page_manager g_page_manager(
+	g_frame_allocator,
+	g_kernel_address_manager);
 
 
-static addr_t
+static uintptr_t
 pt_to_phys(page_table *pt)
 {
-	return reinterpret_cast<addr_t>(pt) - page_manager::page_offset;
+	return reinterpret_cast<uintptr_t>(pt) - page_offset;
 }
 
 
 static page_table *
-pt_from_phys(addr_t p)
+pt_from_phys(uintptr_t p)
 {
-	return reinterpret_cast<page_table *>((p + page_manager::page_offset) & ~0xfffull);
+	return reinterpret_cast<page_table *>((p + page_offset) & ~0xfffull);
 }
 
 
@@ -29,287 +37,143 @@ struct free_page_header
 };
 
 
-void mm_grow_callback(void *next, size_t length)
+page_manager::page_manager(
+		kutil::frame_allocator &frames,
+		kutil::address_manager &addrs) :
+	m_page_cache(nullptr),
+	m_frames(frames),
+	m_addrs(addrs)
 {
-	kassert(length % page_manager::page_size == 0,
-			"Heap manager requested a fractional page.");
-
-	size_t pages = length / page_manager::page_size;
-	log::info(logs::memory, "Heap manager growing heap by %d pages.", pages);
-	g_page_manager.map_pages(reinterpret_cast<addr_t>(next), pages);
 }
 
-
+page_table *
-size_t
+page_manager::create_process_map()
-page_block::length(page_block *list)
 {
-	size_t i = 0;
+	page_table *table = get_table_page();
-	for (page_block *b = list; b; b = b->next) ++i;
+
-	return i;
+	kutil::memset(table, 0, frame_size);
+	table->entries[510] = m_kernel_pml4->entries[510];
+	table->entries[511] = m_kernel_pml4->entries[511];
+
+	// Create the initial user stack
+	map_pages(
+		memory::initial_stack - (memory::initial_stack_pages * frame_size),
+		memory::initial_stack_pages,
+		true, // This is the ring3 stack, user = true
+		table);
+
+	return table;
 }
 
-page_block *
+uintptr_t
-page_block::append(page_block *list, page_block *extra)
+page_manager::copy_page(uintptr_t orig)
 {
-	if (list == nullptr) return extra;
+	uintptr_t virt = m_addrs.allocate(2 * frame_size);
-	else if (extra == nullptr) return list;
+	uintptr_t copy = 0;
 
-	page_block *cur = list;
+	size_t n = m_frames.allocate(1, &copy);
-	while (cur->next)
+	kassert(n, "copy_page could not allocate page");
-		cur = cur->next;
+
-	cur->next = extra;
+	page_in(get_pml4(), orig, virt, 1);
-	return list;
+	page_in(get_pml4(), copy, virt + frame_size, 1);
+
+	kutil::memcpy(
+			reinterpret_cast<void *>(virt + frame_size),
+			reinterpret_cast<void *>(virt),
+			frame_size);
+
+	page_out(get_pml4(), virt, 2);
+
+	m_addrs.free(virt);
+	return copy;
 }
 
-page_block *
+page_table *
-page_block::insert(page_block *list, page_block *block)
+page_manager::copy_table(page_table *from, page_table::level lvl)
 {
-	if (list == nullptr) return block;
+	page_table *to = get_table_page();
-	else if (block == nullptr) return list;
+	log::debug(logs::paging, "Page manager copying level %d table at %016lx to %016lx.", lvl, from, to);
 
-	page_block *cur = list;
+	if (lvl == page_table::level::pml4) {
-	page_block *prev = nullptr;
+		to->entries[510] = m_kernel_pml4->entries[510];
-	while (cur && page_block::compare(block, cur) > 0) {
+		to->entries[511] = m_kernel_pml4->entries[511];
-		prev = cur;
-		cur = cur->next;
 	}
 
-	block->next = cur;
+	const int max =
-	if (prev) {
+		lvl == page_table::level::pml4 ?
-		prev->next = block;
+			510 :
-		return list;
+			512;
-	}
-	return block;
-}
 
-int
+	unsigned pages_copied = 0;
-page_block::compare(const page_block *lhs, const page_block *rhs)
+	for (int i = 0; i < max; ++i) {
-{
+		if (!from->is_present(i)) {
-	if (lhs->virtual_address < rhs->virtual_address)
+			to->entries[i] = 0;
-		return -1;
-	else if (lhs->virtual_address > rhs->virtual_address)
-		return 1;
-
-	if (lhs->physical_address < rhs->physical_address)
-		return -1;
-	else if (lhs->physical_address > rhs->physical_address)
-		return 1;
-
-	return 0;
-}
-
-page_block *
-page_block::consolidate(page_block *list)
-{
-	page_block *freed = nullptr;
-	page_block *cur = list;
-
-	while (cur) {
-		page_block *next = cur->next;
-
-		if (next &&
-			cur->flags == next->flags &&
-			cur->physical_end() == next->physical_address &&
-			(!cur->has_flag(page_block_flags::mapped) ||
-			cur->virtual_end() == next->virtual_address)) {
-
-			cur->count += next->count;
-			cur->next = next->next;
-
-			next->zero(freed);
-			freed = next;
 			continue;
 		}
 
-		cur = cur->next;
+		bool is_page =
-	}
+			lvl == page_table::level::pt ||
+			from->is_large_page(lvl, i);
 
-	return freed;
+		if (is_page) {
-}
+			uint16_t flags = from->entries[i] &  0xfffull;
-
+			uintptr_t orig = from->entries[i] & ~0xfffull;
-void
+			to->entries[i] = copy_page(orig) | flags;
-page_block::dump(page_block *list, const char *name, bool show_unmapped)
+			pages_copied++;
-{
-	log::info(logs::memory, "Block list %s:", name);
-
-	int count = 0;
-	for (page_block *cur = list; cur; cur = cur->next) {
-		count += 1;
-		if (!(show_unmapped || cur->has_flag(page_block_flags::mapped)))
-			continue;
-
-		if (cur->virtual_address) {
-			page_table_indices start{cur->virtual_address};
-			log::info(logs::memory, "  %lx %x [%6d] %lx (%d,%d,%d,%d)",
-					cur->physical_address,
-					cur->flags,
-					cur->count,
-					cur->virtual_address,
-					start[0], start[1], start[2], start[3]);
 		} else {
-			page_table_indices start{cur->virtual_address};
+			uint16_t flags = 0;
-			log::info(logs::memory, "  %lx %x [%6d]",
+			page_table *next_from = from->get(i, &flags);
-					cur->physical_address,
+			page_table *next_to = copy_table(next_from, page_table::deeper(lvl));
-					cur->flags,
+			to->set(i, next_to, flags);
-					cur->count);
 		}
 	}
 
-	log::info(logs::memory, "  Total: %d", count);
+	if (pages_copied)
+		log::debug(logs::paging, "   copied %3u pages", pages_copied);
+
+	return to;
 }
 
 void
-page_block::zero(page_block *set_next)
+page_manager::delete_process_map(page_table *pml4)
 {
-	physical_address = 0;
+	unmap_table(pml4, page_table::level::pml4, true);
-	virtual_address = 0;
-	count = 0;
-	flags = page_block_flags::free;
-	next = set_next;
-}
-
-void
-page_block::copy(page_block *other)
-{
-	physical_address = other->physical_address;
-	virtual_address = other->virtual_address;
-	count = other->count;
-	flags = other->flags;
-	next = other->next;
-}
-
-
-page_manager::page_manager() :
-	m_free(nullptr),
-	m_used(nullptr),
-	m_block_cache(nullptr),
-	m_page_cache(nullptr)
-{
-	kassert(this == &g_page_manager, "Attempt to create another page_manager.");
-}
-
-void
-page_manager::init(
-	page_block *free,
-	page_block *used,
-	page_block *block_cache)
-{
-	m_free = free;
-	m_used = used;
-	m_block_cache = block_cache;
-
-	// For now we're ignoring that we've got the scratch pages
-	// allocated, full of page_block structs. Eventually hand
-	// control of that to a slab allocator.
-
-	consolidate_blocks();
-
-	// Initialize the kernel memory manager
-	addr_t end = 0;
-	for (page_block *b = m_used; b; b = b->next) {
-		if (b->virtual_address < page_offset) {
-			end = b->virtual_end();
-		} else {
-			break;
-		}
-	}
-
-	extern kutil::memory_manager g_kernel_memory_manager;
-	new (&g_kernel_memory_manager) kutil::memory_manager(
-			reinterpret_cast<void *>(end),
-			mm_grow_callback);
 }
 
 void
 page_manager::map_offset_pointer(void **pointer, size_t length)
 {
-	addr_t *p = reinterpret_cast<addr_t *>(pointer);
+	log::info(logs::paging, "Mapping offset pointer region at %016lx size 0x%lx", *pointer, length);
-	addr_t v = *p + page_offset;
+	*pointer = kutil::offset_pointer(*pointer, page_offset);
-	addr_t c = ((length - 1) / page_size) + 1;
-
-	// TODO: cleanly search/split this as a block out of used/free if possible
-	page_block *block = get_block();
-
-	// TODO: page-align
-	block->physical_address = *p;
-	block->virtual_address = v;
-	block->count = c;
-	block->flags =
-		page_block_flags::used |
-		page_block_flags::mapped |
-		page_block_flags::mmio;
-
-	m_used = page_block::insert(m_used, block);
-
-	page_table *pml4 = get_pml4();
-	page_in(pml4, *p, v, c);
-	*p = v;
 }
 
 void
-page_manager::dump_blocks()
+page_manager::dump_pml4(page_table *pml4, bool recurse)
 {
-	page_block::dump(m_used, "used", true);
+	if (pml4 == nullptr) pml4 = get_pml4();
-	page_block::dump(m_free, "free", true);
+	pml4->dump(page_table::level::pml4, recurse);
-}
-
-page_block *
-page_manager::get_block()
-{
-	page_block *block = m_block_cache;
-	if (block) {
-		m_block_cache = block->next;
-		block->next = 0;
-		return block;
-	} else {
-		kassert(0, "NYI: page_manager::get_block() needed to allocate.");
-		return nullptr;
-	}
-}
-
-void
-page_manager::free_blocks(page_block *block)
-{
-	if (!block) return;
-
-	page_block *cur = block;
-	while (cur) {
-		page_block *next = cur->next;
-		cur->zero(cur->next ? cur->next : m_block_cache);
-		cur = next;
-	}
-
-	m_block_cache = block;
 }
 
 page_table *
 page_manager::get_table_page()
 {
 	if (!m_page_cache) {
-		addr_t phys = 0;
+		uintptr_t phys = 0;
-		size_t n = pop_pages(32, &phys);
+		size_t n = m_frames.allocate(32, &phys); // TODO: indicate frames must be offset-mappable
-		addr_t virt = phys + page_offset;
+		uintptr_t virt = phys + page_offset;
-
-		page_block *block = get_block();
-		block->physical_address = phys;
-		block->virtual_address = virt;
-		block->count = n;
-		page_block::insert(m_used, block);
-
-		page_in(get_pml4(), phys, virt, n);
 
 		m_page_cache = reinterpret_cast<free_page_header *>(virt);
 
 		// The last one needs to be null, so do n-1
-		addr_t end = virt + (n-1) * page_size;
+		uintptr_t end = virt + (n-1) * frame_size;
 		while (virt < end) {
 			reinterpret_cast<free_page_header *>(virt)->next =
-				reinterpret_cast<free_page_header *>(virt + page_size);
+				reinterpret_cast<free_page_header *>(virt + frame_size);
-			virt += page_size;
+			virt += frame_size;
 		}
 		reinterpret_cast<free_page_header *>(virt)->next = nullptr;
 
-		log::info(logs::memory, "Mappd %d new page table pages at %lx", n, phys);
+		log::info(logs::paging, "Mappd %d new page table pages at %lx", n, phys);
 	}
 
 	free_page_header *page = m_page_cache;
@@ -320,9 +184,9 @@ page_manager::get_table_page()
 void
 page_manager::free_table_pages(void *pages, size_t count)
 {
-	addr_t start = reinterpret_cast<addr_t>(pages);
+	uintptr_t start = reinterpret_cast<uintptr_t>(pages);
 	for (size_t i = 0; i < count; ++i) {
-		addr_t addr = start + (i * page_size);
+		uintptr_t addr = start + (i * frame_size);
 		free_page_header *header = reinterpret_cast<free_page_header *>(addr);
 		header->count = 1;
 		header->next = m_page_cache;
@@ -330,186 +194,128 @@ page_manager::free_table_pages(void *pages, size_t count)
 	}
 }
 
-void
-page_manager::consolidate_blocks()
-{
-	m_block_cache = page_block::append(m_block_cache, page_block::consolidate(m_free));
-	m_block_cache = page_block::append(m_block_cache, page_block::consolidate(m_used));
-}
-
 void *
-page_manager::map_pages(addr_t address, size_t count)
+page_manager::map_pages(uintptr_t address, size_t count, bool user, page_table *pml4)
 {
 	void *ret = reinterpret_cast<void *>(address);
-	page_table *pml4 = get_pml4();
+	if (!pml4) pml4 = get_pml4();
 
 	while (count) {
-		kassert(m_free, "page_manager::map_pages ran out of free pages!");
+		uintptr_t phys = 0;
+		size_t n = m_frames.allocate(count, &phys);
 
-		addr_t phys = 0;
+		log::info(logs::paging, "Paging in %d pages at p:%016lx to v:%016lx into %016lx table",
-		size_t n = pop_pages(count, &phys);
+				n, phys, address, pml4);
 
-		page_block *block = get_block();
+		page_in(pml4, phys, address, n, user);
-		block->physical_address = phys;
-		block->virtual_address = address;
-		block->count = n;
-		block->flags =
-				page_block_flags::used |
-				page_block_flags::mapped;
-		page_block::insert(m_used, block);
 
-		page_in(pml4, phys, address, n);
+		address += n * frame_size;
-
-		address += n * page_size;
 		count -= n;
 	}
 
 	return ret;
 }
 
-void *
+void
-page_manager::map_offset_pages(size_t count)
+page_manager::unmap_table(page_table *table, page_table::level lvl, bool free)
 {
-	page_table *pml4 = get_pml4();
+	const int max =
-	page_block *free = m_free;
+		lvl == page_table::level::pml4 ?
-	page_block *prev = nullptr;
+			510 :
+			512;
 
-	log::debug(logs::memory, "Got request to offset map %d pages", count);
+	uintptr_t free_start = 0;
+	uintptr_t free_count = 0;
 
-	while (free) {
+	size_t size =
-		if (free->count < count) {
+		lvl == page_table::level::pdp ? (1<<30) :
-			prev = free;
+		lvl == page_table::level::pd  ? (1<<21) :
-			free = free->next;
+		lvl == page_table::level::pt  ? (1<<12) :
-			continue;
+		0;
+
+	for (int i = 0; i < max; ++i) {
+		if (!table->is_present(i)) continue;
+
+		bool is_page =
+			lvl == page_table::level::pt ||
+			table->is_large_page(lvl, i);
+
+		if (is_page) {
+			uintptr_t frame = table->entries[i] & ~0xfffull;
+			if (!free_count || free_start != frame + free_count * size) {
+				if (free_count && free)
+					m_frames.free(free_start, free_count * size / frame_size);
+				free_start = frame;
+				free_count = 1;
+			}
+		} else {
+			page_table *next = table->get(i);
+			unmap_table(next, page_table::deeper(lvl), free);
 		}
-
-		page_block *used = get_block();
-		used->count = count;
-		used->physical_address = free->physical_address;
-		used->virtual_address = used->physical_address + page_offset;
-		used->flags =
-			page_block_flags::used |
-			page_block_flags::mapped;
-		page_block::insert(m_used, used);
-
-		free->physical_address += count * page_size;
-		free->count -= count;
-
-		if (free->count == 0) {
-			if (prev)
-				prev->next = free->next;
-			else
-				m_free = free->next;
-
-			free->zero(m_block_cache);
-			m_block_cache = free;
-		}
-
-		page_in(pml4, used->physical_address, used->virtual_address, count);
-		return reinterpret_cast<void *>(used->virtual_address);
 	}
 
-	return nullptr;
+	if (free_count && free)
+		m_frames.free(free_start, free_count * size / frame_size);
+	free_table_pages(table, 1);
 }
 
 void
-page_manager::unmap_pages(void* address, size_t count)
+page_manager::unmap_pages(void* address, size_t count, page_table *pml4)
 {
-	addr_t addr = reinterpret_cast<addr_t>(address);
+	if (!pml4) pml4 = get_pml4();
-
+	page_out(pml4, reinterpret_cast<uintptr_t>(address), count, true);
-	page_block **prev = &m_used;
-	page_block *cur = m_used;
-	while (cur && !cur->contains(addr)) {
-		prev = &cur->next;
-		cur = cur->next;
-	}
-
-	kassert(cur, "Couldn't find existing mapped pages to unmap");
-
-	size_t size = page_size * count;
-	addr_t end = addr + size;
-
-	while (cur && cur->contains(addr)) {
-		size_t leading = addr - cur->virtual_address;
-		size_t trailing =
-			end > cur->virtual_end() ?
-			0 : (cur->virtual_end() - end);
-
-		if (leading) {
-			size_t pages = leading / page_size;
-
-			page_block *lead_block = get_block();
-			lead_block->copy(cur);
-			lead_block->next = cur;
-			lead_block->count = pages;
-
-			cur->count -= pages;
-			cur->physical_address += leading;
-			cur->virtual_address += leading;
-
-			*prev = lead_block;
-			prev = &lead_block->next;
-		}
-
-		if (trailing) {
-			size_t pages = trailing / page_size;
-
-			page_block *trail_block = get_block();
-			trail_block->copy(cur);
-			trail_block->next = cur->next;
-			trail_block->count = pages;
-			trail_block->physical_address += size;
-			trail_block->virtual_address += size;
-
-			cur->count -= pages;
-
-			cur->next = trail_block;
-		}
-
-		addr += cur->count * page_size;
-		page_block *next = cur->next;
-
-		*prev = cur->next;
-		cur->next = nullptr;
-		cur->virtual_address = 0;
-		cur->flags = cur->flags & ~(page_block_flags::used | page_block_flags::mapped);
-		m_free = page_block::insert(m_free, cur);
-
-		cur = next;
-	}
 }
 
 void
-page_manager::check_needs_page(page_table *table, unsigned index)
+page_manager::check_needs_page(page_table *table, unsigned index, bool user)
 {
 	if ((table->entries[index] & 0x1) == 1) return;
 
 	page_table *new_table = get_table_page();
 	for (int i=0; i<512; ++i) new_table->entries[i] = 0;
-	table->entries[index] = pt_to_phys(new_table) | 0xb;
+	table->entries[index] = pt_to_phys(new_table) | (user ? 0xf : 0xb);
 }
 
 void
-page_manager::page_in(page_table *pml4, addr_t phys_addr, addr_t virt_addr, size_t count)
+page_manager::page_in(page_table *pml4, uintptr_t phys_addr, uintptr_t virt_addr, size_t count, bool user, bool large)
 {
+	log::debug(logs::paging, "page_in for table %016lx p:%016lx v:%016lx c:%4d u:%d l:%d",
+			pml4, phys_addr, virt_addr, count, user, large);
+
 	page_table_indices idx{virt_addr};
 	page_table *tables[4] = {pml4, nullptr, nullptr, nullptr};
 
+	uint64_t flags = user ?
+		0x00f:	// writethru, user, write, present
+		0x10b;	// global, writethru, write, present
+
 	for (; idx[0] < 512; idx[0] += 1) {
-		check_needs_page(tables[0], idx[0]);
+		check_needs_page(tables[0], idx[0], user);
 		tables[1] = tables[0]->get(idx[0]);
 
 		for (; idx[1] < 512; idx[1] += 1, idx[2] = 0, idx[3] = 0) {
-			check_needs_page(tables[1], idx[1]);
+			check_needs_page(tables[1], idx[1], user);
 			tables[2] = tables[1]->get(idx[1]);
 
 			for (; idx[2] < 512; idx[2] += 1, idx[3] = 0) {
-				check_needs_page(tables[2], idx[2]);
+				if (large &&
+					idx[3] == 0 &&
+					count >= 512 &&
+					tables[2]->get(idx[2]) == nullptr) {
+					// Do a 2MiB page instead
+					tables[2]->entries[idx[2]] = phys_addr | flags | 0x80;
+					phys_addr += frame_size * 512;
+					count -= 512;
+					if (count == 0) return;
+					continue;
+				}
+
+				check_needs_page(tables[2], idx[2], user);
 				tables[3] = tables[2]->get(idx[2]);
 
 				for (; idx[3] < 512; idx[3] += 1) {
-					tables[3]->entries[idx[3]] = phys_addr | 0xb;
+					tables[3]->entries[idx[3]] = phys_addr | flags;
-					phys_addr += page_manager::page_size;
+					phys_addr += frame_size;
 					if (--count == 0) return;
 				}
 			}
@@ -520,26 +326,41 @@ page_manager::page_in(page_table *pml4, addr_t phys_addr, addr_t virt_addr, size
 }
 
 void
-page_manager::page_out(page_table *pml4, addr_t virt_addr, size_t count)
+page_manager::page_out(page_table *pml4, uintptr_t virt_addr, size_t count, bool free)
 {
 	page_table_indices idx{virt_addr};
 	page_table *tables[4] = {pml4, nullptr, nullptr, nullptr};
 
+	bool found = false;
+	uintptr_t free_start = 0;
+	unsigned free_count = 0;
+
 	for (; idx[0] < 512; idx[0] += 1) {
-		tables[1] = reinterpret_cast<page_table *>(
+		tables[1] = tables[0]->get(idx[0]);
-				tables[0]->entries[idx[0]] & ~0xfffull);
 
 		for (; idx[1] < 512; idx[1] += 1) {
-			tables[2] = reinterpret_cast<page_table *>(
+			tables[2] = tables[1]->get(idx[1]);
-					tables[1]->entries[idx[1]] & ~0xfffull);
 
 			for (; idx[2] < 512; idx[2] += 1) {
-				tables[3] = reinterpret_cast<page_table *>(
+				tables[3] = tables[2]->get(idx[2]);
-						tables[2]->entries[idx[2]] & ~0xfffull);
 
 				for (; idx[3] < 512; idx[3] += 1) {
+					uintptr_t entry = tables[3]->entries[idx[3]] & ~0xfffull;
+					if (!found || entry != free_start + free_count * frame_size) {
+						if (found && free) m_frames.free(free_start, free_count);
+						free_start = tables[3]->entries[idx[3]] & ~0xfffull;
+						free_count = 1;
+						found = true;
+					} else {
+						free_count++;
+					}
+
 					tables[3]->entries[idx[3]] = 0;
-					if (--count == 0) return;
+
+					if (--count == 0) {
+						if (free) m_frames.free(free_start, free_count);
+						return;
+					}
 				}
 			}
 		}
@@ -548,65 +369,60 @@ page_manager::page_out(page_table *pml4, addr_t virt_addr, size_t count)
 	kassert(0, "Ran to end of page_out");
 }
 
-size_t
-page_manager::pop_pages(size_t count, addr_t *address)
-{
-	kassert(m_free, "page_manager::pop_pages ran out of free pages!");
-
-	unsigned n = std::min(count, static_cast<size_t>(m_free->count));
-	*address = m_free->physical_address;
-
-	m_free->physical_address += n * page_size;
-	m_free->count -= n;
-	if (m_free->count == 0) {
-		page_block *block = m_free;
-		m_free = m_free->next;
-
-		block->zero(m_block_cache);
-		m_block_cache = block;
-	}
-
-	return n;
-}
-
-
 void
-page_table::dump(int level, uint64_t offset)
+page_table::dump(page_table::level lvl, bool recurse)
 {
-	log::info(logs::memory, "Level %d page table @ %lx (off %lx):", level, this, offset);
+	console *cons = console::get();
+
+	cons->printf("\nLevel %d page table @ %lx:\n", lvl, this);
 	for (int i=0; i<512; ++i) {
 		uint64_t ent = entries[i];
-		if (ent == 0) continue;
 
-		if ((ent & 0x1) == 0) {
+		if ((ent & 0x1) == 0)
-			log::info(logs::memory, "  %3d: %lx   NOT PRESENT", i, ent);
+			cons->printf("  %3d: %016lx   NOT PRESENT\n", i, ent);
-			continue;
-		}
 
-		if ((level == 2 || level == 3) && (ent & 0x80) == 0x80) {
+		else if ((lvl == level::pdp || lvl == level::pd) && (ent & 0x80) == 0x80)
-			log::info(logs::memory, "  %3d: %lx   -> Large page at     %lx",
+			cons->printf("  %3d: %016lx -> Large page at    %016lx\n", i, ent, ent & ~0xfffull);
-					i, ent, ent & ~0xfffull);
+
-			continue;
+		else if (lvl == level::pt)
-		} else if (level == 1) {
+			cons->printf("  %3d: %016lx -> Page at          %016lx\n", i, ent, ent & ~0xfffull);
-			log::info(logs::memory, "  %3d: %lx   -> Page at           %lx",
+
-					i, ent, ent & ~0xfffull);
+		else
-		} else {
+			cons->printf("  %3d: %016lx -> Level %d table at %016lx\n",
-			log::info(logs::memory, "  %3d: %lx   -> Level %d table at %lx",
+					i, ent, deeper(lvl), (ent & ~0xfffull) + page_offset);
-					i, ent, level - 1, (ent & ~0xfffull) + offset);
-			continue;
-		}
 	}
 
-	if (--level > 0) {
+	if (lvl != level::pt && recurse) {
-		for (int i=0; i<512; ++i) {
+		for (int i=0; i<=512; ++i) {
-			uint64_t ent = entries[i];
+			if (is_large_page(lvl, i))
-			if ((ent & 0x1) == 0) continue;
+				continue;
-			if ((ent & 0x80)) continue;
 
-			page_table *next = reinterpret_cast<page_table *>((ent & ~0xffful) + offset); 
+			page_table *next = get(i);
-			next->dump(level, offset);
+			if (next)
+				next->dump(deeper(lvl), true);
 		}
 	}
 }
 
+page_table_indices::page_table_indices(uint64_t v) :
+	index{
+		(v >> 39) & 0x1ff,
+		(v >> 30) & 0x1ff,
+		(v >> 21) & 0x1ff,
+		(v >> 12) & 0x1ff }
+{}
 
+uintptr_t
+page_table_indices::addr() const
+{
+	return
+		(index[0] << 39) |
+		(index[1] << 30) |
+		(index[2] << 21) |
+		(index[3] << 12);
+}
+
+bool operator==(const page_table_indices &l, const page_table_indices &r)
+{
+	return l[0] == r[0] && l[1] == r[1] && l[2] == r[2] && l[3] == r[3];
+}

src/kernel/page_manager.h

@@ -5,46 +5,78 @@
 #include <stddef.h>
 #include <stdint.h>
 
-#include "kutil/memory.h"
+#include "kutil/address_manager.h"
 #include "kutil/enum_bitfields.h"
+#include "kutil/frame_allocator.h"
+#include "kutil/linked_list.h"
+#include "kutil/slab_allocator.h"
+#include "kernel_memory.h"
+#include "page_table.h"
 
-struct page_block;
-struct page_table;
 struct free_page_header;
 
-
+/// Manager for allocation and mapping of pages
-/// Manager for allocation of physical pages.
 class page_manager
 {
 public:
-	/// Size of a single page.
+	page_manager(
-	static const size_t page_size = 0x1000;
+		kutil::frame_allocator &frames,
+		kutil::address_manager &addrs);
 
-	/// Start of the higher half.
+	/// Helper to get the number of pages needed for a given number of bytes.
-	static const addr_t high_offset = 0xffff800000000000;
+	/// \arg bytes  The number of bytes desired
+	/// \returns    The number of pages needed to contain the desired bytes
+	static inline size_t page_count(size_t bytes)
+	{
+		return (bytes - 1) / memory::frame_size + 1;
+	}
 
-	/// Offset from physical where page tables are mapped.
+	/// Helper to read the PML4 table from CR3.
-	static const addr_t page_offset = 0xffffff8000000000;
+	/// \returns  A pointer to the current PML4 table.
+	static inline page_table * get_pml4()
+	{
+		uintptr_t pml4 = 0;
+		__asm__ __volatile__ ( "mov %%cr3, %0" : "=r" (pml4) );
+		return reinterpret_cast<page_table *>((pml4 & ~0xfffull) + memory::page_offset);
+	}
 
-	page_manager();
+	/// Helper to set the PML4 table pointer in CR3.
+	/// \arg pml4  A pointer to the PML4 table to install.
+	static inline void set_pml4(page_table *pml4)
+	{
+		uintptr_t p = reinterpret_cast<uintptr_t>(pml4) - memory::page_offset;
+		__asm__ __volatile__ ( "mov %0, %%cr3" :: "r" (p & ~0xfffull) );
+	}
+
+	/// Allocate but don't switch to a new PML4 table. This table
+	/// should only have global kernel pages mapped.
+	/// \returns      A pointer to the PML4 table
+	page_table * create_process_map();
+
+	/// Deallocate a process' PML4 table and entries.
+	/// \arg pml4  The process' PML4 table
+	void delete_process_map(page_table *pml4);
+
+	/// Copy a process' memory mappings (and memory pages).
+	/// \arg from  Page table to copy from
+	/// \arg lvl   Level of the given tables (default is PML4)
+	/// \returns   The new page table
+	page_table * copy_table(page_table *from,
+			page_table::level lvl = page_table::level::pml4);
 
 	/// Allocate and map pages into virtual memory.
 	/// \arg address  The virtual address at which to map the pages
 	/// \arg count    The number of pages to map
+	/// \arg user     True is this memory is user-accessible
+	/// \arg pml4     The pml4 to map into - null for the current one
 	/// \returns      A pointer to the start of the mapped region
-	void * map_pages(addr_t address, size_t count);
+	void * map_pages(uintptr_t address, size_t count, bool user = false, page_table *pml4 = nullptr);
 
-	/// Allocate and map contiguous pages into virtual memory, with
+	/// Unmap and free existing pages from memory.
-	/// a constant offset from their physical address.
-	/// \arg count    The number of pages to map
-	/// \returns      A pointer to the start of the mapped region, or
-	/// nullptr if no region could be found to fit the request.
-	void * map_offset_pages(size_t count);
-
-	/// Unmap existing pages from memory.
 	/// \arg address  The virtual address of the memory to unmap
 	/// \arg count    The number of pages to unmap
-	void unmap_pages(void *address, size_t count);
+	/// \arg pml4     The pml4 to unmap from - null for the current one
+	void unmap_pages(void *address, size_t count, page_table *pml4 = nullptr);
 
 	/// Offset-map a pointer. No physical pages will be mapped.
 	/// \arg pointer  Pointer to a pointer to the memory area to be mapped
@@ -54,43 +86,33 @@ public:
 	/// Get the physical address of an offset-mapped pointer
 	/// \arg p   Virtual address of memory that has been offset-mapped
 	/// \returns Physical address of the memory pointed to by p
-	inline addr_t offset_phys(void *p) const
+	inline uintptr_t offset_phys(void *p) const
 	{
-		return reinterpret_cast<addr_t>(kutil::offset_pointer(p, -page_offset));
+		return reinterpret_cast<uintptr_t>(kutil::offset_pointer(p, -memory::page_offset));
 	}
 
 	/// Get the virtual address of an offset-mapped physical address
 	/// \arg a   Physical address of memory that has been offset-mapped
 	/// \returns Virtual address of the memory at address a
-	inline void * offset_virt(addr_t a) const
+	inline void * offset_virt(uintptr_t a) const
 	{
-		return kutil::offset_pointer(reinterpret_cast<void *>(a), page_offset);
+		return kutil::offset_pointer(reinterpret_cast<void *>(a), memory::page_offset);
 	}
 
-	/// Log the current free/used block lists.
+	/// Dump the given or current PML4 to the console
-	void dump_blocks();
+	/// \arg pml4     The page table to use, null for the current one
+	/// \arg recurse  Whether to print sub-tables
+	void dump_pml4(page_table *pml4 = nullptr, bool recurse = true);
 
 	/// Get the system page manager.
 	/// \returns  A pointer to the system page manager
 	static page_manager * get();
 
 private:
-	/// Set up the memory manager from bootstraped memory
+	/// Copy a physical page
-	void init(
+	/// \arg orig  Physical address of the page to copy
-			page_block *free,
+	/// \returns   Physical address of the new page
-			page_block *used,
+	uintptr_t copy_page(uintptr_t orig);
-			page_block *block_cache);
-
-	/// Initialize the virtual memory manager based on this object's state
-	void init_memory_manager();
-
-	/// Create a `page_block` struct or pull one from the cache.
-	/// \returns  An empty `page_block` struct
-	page_block * get_block();
-
-	/// Return a list of `page_block` structs to the cache.
-	/// \arg block   A list of `page_block` structs
-	void free_blocks(page_block *block);
 
 	/// Allocate a page for a page table, or pull one from the cache
 	/// \returns  An empty page mapped in page space
@@ -101,69 +123,50 @@ private:
 	/// \arg count  Number of pages in the range
 	void free_table_pages(void *pages, size_t count);
 
-	/// Consolidate the free and used block lists. Return freed blocks
-	/// to the cache.
-	void consolidate_blocks();
-
-	/// Helper to read the PML4 table from CR3.
-	/// \returns  A pointer to the current PML4 table.
-	static inline page_table * get_pml4()
-	{
-		addr_t pml4 = 0;
-		__asm__ __volatile__ ( "mov %%cr3, %0" : "=r" (pml4) );
-		return reinterpret_cast<page_table *>((pml4 & ~0xfffull) + page_offset);
-	}
-
-	/// Helper to set the PML4 table pointer in CR3.
-	/// \arg pml4  A pointer to the PML4 table to install.
-	static inline void set_pml4(page_table *pml4)
-	{
-		addr_t p = reinterpret_cast<addr_t>(pml4) - page_offset;
-		__asm__ __volatile__ ( "mov %0, %%cr3" :: "r" (p & ~0xfffull) );
-	}
-
 	/// Helper function to allocate a new page table. If table entry `i` in
 	/// table `base` is empty, allocate a new page table and point `base[i]` at
 	/// it.
 	/// \arg base  Existing page table being indexed into
 	/// \arg i     Index into the existing table to check
-	void check_needs_page(page_table *base, unsigned i);
+	/// \art user  True if this is a userspace mapping
+	void check_needs_page(page_table *base, unsigned i, bool user);
 
 	/// Low-level routine for mapping a number of pages into the given page table.
 	/// \arg pml4       The root page table to map into
 	/// \arg phys_addr  The starting physical address of the pages to be mapped
 	/// \arg virt_addr  The starting virtual address ot the memory to be mapped
 	/// \arg count      The number of pages to map
+	/// \arg user       True if this is a userspace mapping
+	/// \arg large      Whether to allow large pages
 	void page_in(
 			page_table *pml4,
-			addr_t phys_addr,
+			uintptr_t phys_addr,
-			addr_t virt_addr,
+			uintptr_t virt_addr,
-			size_t count);
+			size_t count,
+			bool user = false,
+			bool large = false);
 
 	/// Low-level routine for unmapping a number of pages from the given page table.
 	/// \arg pml4       The root page table for this mapping
 	/// \arg virt_addr  The starting virtual address ot the memory to be unmapped
 	/// \arg count      The number of pages to unmap
+	/// \arg free       Whether to return the pages to the frame allocator
 	void page_out(
 			page_table *pml4,
-			addr_t virt_addr,
+			uintptr_t virt_addr,
-			size_t count);
+			size_t count,
+			bool free = false);
 
-	/// Get free pages from the free list. Only pages from the first free block
+	/// Low-level routine for unmapping an entire table of memory at once
-	/// are returned, so the number may be less than requested, but they will
+	void unmap_table(page_table *table, page_table::level lvl, bool free);
-	/// be contiguous. Pages will not be mapped into virtual memory.
-	/// \arg count    The maximum number of pages to get
-	/// \arg address  [out] The address of the first page
-	/// \returns      The number of pages retrieved
-	size_t pop_pages(size_t count, addr_t *address);
 
-	page_block *m_free; ///< Free pages list
+	page_table *m_kernel_pml4; ///< The PML4 of just kernel pages
-	page_block *m_used; ///< In-use pages list
-
-	page_block *m_block_cache; ///< Cache of unused page_block structs
 	free_page_header *m_page_cache; ///< Cache of free pages to use for tables
 
-	friend void memory_initialize(const void *, size_t, size_t);
+	kutil::frame_allocator &m_frames;
+	kutil::address_manager &m_addrs;
+
+	friend void memory_initialize(uint16_t, const void *, size_t, size_t);
 	page_manager(const page_manager &) = delete;
 };
 
@@ -172,132 +175,6 @@ extern page_manager g_page_manager;
 
 inline page_manager * page_manager::get() { return &g_page_manager; }
 
-/// Flags used by `page_block`.
-enum class page_block_flags : uint32_t
-{
-	free         = 0x00000000,  ///< Not a flag, value for free memory
-	used         = 0x00000001,  ///< Memory is in use
-	mapped       = 0x00000002,  ///< Memory is mapped to virtual address
-
-	mmio         = 0x00000010,  ///< Memory is a MMIO region
-	nonvolatile  = 0x00000020,  ///< Memory is non-volatile storage
-
-	pending_free = 0x10000000,  ///< Memory should be freed
-	acpi_wait    = 0x40000000,  ///< Memory should be freed after ACPI init
-	permanent    = 0x80000000,  ///< Memory is permanently unusable
-
-	max_flags
-};
-IS_BITFIELD(page_block_flags);
-
-
-/// A block of contiguous pages. Each `page_block` represents contiguous
-/// physical pages with the same attributes. A `page_block *` is also a
-/// linked list of such structures.
-struct page_block
-{
-	addr_t physical_address;
-	addr_t virtual_address;
-	uint32_t count;
-	page_block_flags flags;
-	page_block *next;
-
-	inline bool has_flag(page_block_flags f) const { return bitfield_has(flags, f); }
-	inline addr_t physical_end() const { return physical_address + (count * page_manager::page_size); }
-	inline addr_t virtual_end() const { return virtual_address + (count * page_manager::page_size); }
-
-	inline bool contains(addr_t vaddr) const { return vaddr >= virtual_address && vaddr < virtual_end(); }
-	inline bool contains_physical(addr_t addr) const { return addr >= physical_address && addr < physical_end(); }
-
-	/// Helper to zero out a block and optionally set the next pointer.
-	/// \arg next  [optional] The value for the `next` pointer
-	void zero(page_block *set_next = nullptr);
-
-	/// Helper to copy a bock from another block
-	/// \arg other  The block to copy from
-	void copy(page_block *other);
-
-	/// \name Page block linked list functions
-	/// Functions to act on a `page_block *` as a linked list
-	/// @{
-
-	/// Count the items in the given linked list.
-	/// \arg list  The list to count
-	/// \returns   The number of entries in the list.
-	static size_t length(page_block *list);
-
-	/// Append a block or list to the given list.
-	/// \arg list   The list to append to
-	/// \arg extra  The list or block to be appended 
-	/// \returns    The new list head
-	static page_block * append(page_block *list, page_block *extra);
-
-	/// Sorted-insert of a block into the list by address.
-	/// \arg list   The list to insert into
-	/// \arg block  The single block to insert
-	/// \returns    The new list head
-	static page_block * insert(page_block *list, page_block *block);
-
-	/// Compare two blocks by address.
-	/// \arg lhs   The left-hand comparator
-	/// \arg rhs   The right-hand comparator
-	/// \returns   <0 if lhs is sorts earlier, >0 if lhs sorts later, 0 for equal
-	static int compare(const page_block *lhs, const page_block *rhs);
-
-	/// Traverse the list, joining adjacent blocks where possible.
-	/// \arg list  The list to consolidate
-	/// \returns   A linked list of freed page_block structures.
-	static page_block * consolidate(page_block *list);
-
-	/// Traverse the list, printing debug info on this list.
-	/// \arg list  The list to print
-	/// \arg name  [optional] String to print as the name of this list
-	/// \arg show_permanent [optional] If false, hide unmapped blocks
-	static void dump(page_block *list, const char *name = nullptr, bool show_unmapped = false);
-
-	/// @}
-
-};
-
-
-
-/// Struct to allow easy accessing of a memory page being used as a page table.
-struct page_table
-{
-	using pm = page_manager;
-
-	uint64_t entries[512];
-
-	inline page_table * get(int i) const {
-		uint64_t entry = entries[i];
-		if ((entry & 0x1) == 0) return nullptr;
-		return reinterpret_cast<page_table *>((entry & ~0xfffull) + pm::page_offset);
-	}
-
-	inline void set(int i, page_table *p, uint16_t flags) {
-		entries[i] = (reinterpret_cast<uint64_t>(p) - pm::page_offset) | (flags & 0xfff);
-	}
-
-	void dump(int level = 4, uint64_t offset = page_manager::page_offset);
-};
-
-
-/// Helper struct for computing page table indices of a given address.
-struct page_table_indices
-{
-	page_table_indices(uint64_t v = 0) :
-		index{
-			(v >> 39) & 0x1ff,
-			(v >> 30) & 0x1ff,
-			(v >> 21) & 0x1ff,
-			(v >> 12) & 0x1ff }
-	{}
-
-	/// Get the index for a given level of page table.
-	uint64_t & operator[](size_t i) { return index[i]; }
-	uint64_t index[4]; ///< Indices for each level of tables.
-};
-
 
 /// Calculate a page-aligned address.
 /// \arg p    The address to align.
@@ -306,8 +183,8 @@ template <typename T> inline T
 page_align(T p)
 {
 	return reinterpret_cast<T>(
-		((reinterpret_cast<addr_t>(p) - 1) & ~(page_manager::page_size - 1))
+		((reinterpret_cast<uintptr_t>(p) - 1) & ~(memory::frame_size - 1))
-		+ page_manager::page_size);
+		+ memory::frame_size);
 }
 
 /// Calculate a page-table-aligned address. That is, an address that is
@@ -321,11 +198,5 @@ page_table_align(T p)
 }
 
 
-/// Calculate the number of pages needed for the give number of bytes.
-/// \arg n   Number of bytes
-/// \returns Number of pages
-inline size_t page_count(size_t n) { return ((n - 1) / page_manager::page_size) + 1; }
-
-
 /// Bootstrap the memory managers.
-void memory_initialize(const void *memory_map, size_t map_length, size_t desc_length);
+void memory_initialize(uint16_t scratch_pages, const void *memory_map, size_t map_length, size_t desc_length);

src/kernel/page_table.h

@@ -0,0 +1,63 @@
+#pragma once
+/// \file page_table.h
+/// Helper structures for dealing with page tables.
+
+#include <stdint.h>
+#include "kernel_memory.h"
+
+class page_manager;
+
+/// Struct to allow easy accessing of a memory page being used as a page table.
+struct page_table
+{
+	enum class level : unsigned { pml4, pdp, pd, pt };
+	inline static level deeper(level l) {
+		return static_cast<level>(static_cast<unsigned>(l) + 1);
+	}
+
+	uint64_t entries[512];
+
+	inline page_table * get(int i, uint16_t *flags = nullptr) const {
+		uint64_t entry = entries[i];
+		if ((entry & 0x1) == 0) return nullptr;
+		if (flags) *flags = entry & 0xfffull;
+		return reinterpret_cast<page_table *>((entry & ~0xfffull) + memory::page_offset);
+	}
+
+	inline void set(int i, page_table *p, uint16_t flags) {
+		entries[i] = (reinterpret_cast<uint64_t>(p) - memory::page_offset) | (flags & 0xfff);
+	}
+
+	inline bool is_present(int i) const { return (entries[i] & 0x1) == 0x1; }
+
+	inline bool is_large_page(level l, int i) const {
+		return
+			(l == level::pdp || l == level::pd) &&
+			(entries[i] & 0x80) == 0x80;
+	}
+
+	void dump(
+		level lvl = level::pml4,
+		bool recurse = true);
+};
+
+
+/// Helper struct for computing page table indices of a given address.
+struct page_table_indices
+{
+	page_table_indices(uint64_t v = 0);
+
+	uintptr_t addr() const;
+
+	inline operator uintptr_t() const { return addr(); }
+
+	/// Get the index for a given level of page table.
+	uint64_t & operator[](int i) { return index[i]; }
+	uint64_t operator[](int i) const { return index[i]; }
+	uint64_t & operator[](page_table::level i) { return index[static_cast<unsigned>(i)]; }
+	uint64_t operator[](page_table::level i) const { return index[static_cast<unsigned>(i)]; }
+	uint64_t index[4]; ///< Indices for each level of tables.
+};
+
+bool operator==(const page_table_indices &l, const page_table_indices &r);
+

src/kernel/pci.cpp

@@ -1,11 +1,32 @@
 #include "kutil/assert.h"
+#include "console.h"
 #include "log.h"
 #include "interrupts.h"
 #include "pci.h"
 
-struct pci_cap_msi :
+struct pci_cap_msi
-	public pci_cap
 {
+	pci_cap::type id;
+	uint8_t next;
+	uint16_t control;
+} __attribute__ ((packed));
+
+struct pci_cap_msi32
+{
+	pci_cap::type id;
+	uint8_t next;
+	uint16_t control;
+	uint32_t address;
+	uint16_t data;
+	uint16_t reserved;
+	uint32_t mask;
+	uint32_t pending;
+} __attribute__ ((packed));
+
+struct pci_cap_msi64
+{
+	pci_cap::type id;
+	uint8_t next;
 	uint16_t control;
 	uint64_t address;
 	uint16_t data;
@@ -14,17 +35,33 @@ struct pci_cap_msi :
 	uint32_t pending;
 } __attribute__ ((packed));
 
-struct pci_cap_msix :
-	public pci_cap
-{
-	uint16_t control;
-	uint64_t address;
-	uint16_t data;
-	uint16_t reserved;
-	uint32_t mask;
-	uint32_t pending;
-} __attribute__ ((packed));
 
+void dump_msi(pci_cap_msi *cap)
+{
+	auto cons = console::get();
+	cons->printf("MSI Cap:\n");
+	cons->printf("     id: %02x\n", cap->id);
+	cons->printf("   next: %02x\n", cap->next);
+	cons->printf("control: %04x\n", cap->control);
+	if (cap->control & 0x0080) {
+		pci_cap_msi64 *cap64 = reinterpret_cast<pci_cap_msi64 *>(cap);
+		cons->printf("address: %016x\n", cap64->address);
+		cons->printf("   data: %04x\n", cap64->data);
+		if (cap->control & 0x100) {
+			cons->printf("   mask: %08x\n", cap64->mask);
+			cons->printf("pending: %08x\n", cap64->pending);
+		}
+	} else {
+		pci_cap_msi32 *cap32 = reinterpret_cast<pci_cap_msi32 *>(cap);
+		cons->printf("address: %08x\n", cap32->address);
+		cons->printf("   data: %04x\n", cap32->data);
+		if (cap->control & 0x100) {
+			cons->printf("   mask: %08x\n", cap32->mask);
+			cons->printf("pending: %08x\n", cap32->pending);
+		}
+	}
+	cons->putc('\n');
+};
 
 pci_device::pci_device() :
 	m_base(nullptr),
@@ -60,19 +97,25 @@ pci_device::pci_device(pci_group &group, uint8_t bus, uint8_t device, uint8_t fu
 	uint16_t *command = reinterpret_cast<uint16_t *>(&m_base[1]);
 	*command |= 0x400; // Mask old INTx style interrupts
 
-	log::info(logs::device, "Found PCIe device at %02d:%02d:%d of type %d.%d id %04x:%04x",
+	uint16_t *status = command + 1;
-			bus, device, func, m_class, m_subclass, m_vendor, m_device);
 
-	// Walk the extended capabilities list
+	log::info(logs::device, "Found PCIe device at %02d:%02d:%d of type %x.%x.%x id %04x:%04x",
-	uint8_t next = m_base[13] & 0xff;
+			bus, device, func, m_class, m_subclass, m_progif, m_vendor, m_device);
-	while (next) {
-		pci_cap *cap = reinterpret_cast<pci_cap *>(kutil::offset_pointer(m_base, next));
-		next = cap->next;
 
-		if (cap->id == pci_cap::type::msi) {
+	if (*status & 0x0010) {
-			m_msi = cap;
+		// Walk the extended capabilities list
-			pci_cap_msi *mcap = reinterpret_cast<pci_cap_msi *>(cap);
+		uint8_t next = m_base[13] & 0xff;
-			mcap->control |= ~0x1; // Mask interrupts
+		while (next) {
+			pci_cap *cap = reinterpret_cast<pci_cap *>(kutil::offset_pointer(m_base, next));
+			next = cap->next;
+			log::debug(logs::device, "  - found PCI cap type %02x", cap->id);
+
+			if (cap->id == pci_cap::type::msi) {
+				m_msi = cap;
+				pci_cap_msi *mcap = reinterpret_cast<pci_cap_msi *>(cap);
+				mcap->control &= ~0x70; // at most 1 vector allocated
+				mcap->control |= 0x01; // Enable interrupts, at most 1 vector allocated
+			}
 		}
 	}
 }
@@ -106,14 +149,24 @@ pci_device::set_bar(unsigned i, uint32_t val)
 }
 
 void
-pci_device::write_msi_regs(addr_t address, uint16_t data)
+pci_device::write_msi_regs(uintptr_t address, uint16_t data)
 {
 	kassert(m_msi, "Tried to write MSI for a device without that cap");
 	if (m_msi->id == pci_cap::type::msi) {
 		pci_cap_msi *mcap = reinterpret_cast<pci_cap_msi *>(m_msi);
-		mcap->address = address;
+		if (mcap->control & 0x0080) {
-		mcap->data = data;
+			pci_cap_msi64 *mcap64 = reinterpret_cast<pci_cap_msi64 *>(m_msi);
-		mcap->control |= 1;
+			mcap64->address = address;
+			mcap64->data = data;
+		} else {
+			pci_cap_msi32 *mcap32 = reinterpret_cast<pci_cap_msi32 *>(m_msi);
+			mcap32->address = address;
+			mcap32->data = data;
+		}
+		uint16_t control = mcap->control;
+		control &= 0xff8f; // We're allocating one vector, clear 6::4
+		control |= 0x0001; // Enable MSI
+		mcap->control = control;
 	} else {
 		kassert(0, "MIS-X is NYI");
 	}

src/kernel/pci.h

@@ -1,6 +1,7 @@
 #pragma once
 /// \file pci.h
 /// PCI devices and groups
+
 #include <stdint.h>
 #include "kutil/memory.h"
 
@@ -75,7 +76,7 @@ public:
 	/// Write to the MSI registers
 	/// \arg addr  The address to write to the MSI address registers
 	/// \arg data  The value to write to the MSI data register
-	void write_msi_regs(addr_t addr, uint16_t data);
+	void write_msi_regs(uintptr_t addr, uint16_t data);
 
 	/// Get a bus address, given the bus/device/function numbers.
 	/// \arg bus    Number of the bus

src/kernel/process.cpp

@@ -0,0 +1,177 @@
+#include "cpu.h"
+#include "log.h"
+#include "process.h"
+#include "scheduler.h"
+
+
+pid_t
+process::fork(uintptr_t in_rsp)
+{
+	auto &sched = scheduler::get();
+	auto *child = sched.create_process();
+	kassert(child, "process::fork() got null child");
+
+	child->ppid = pid;
+	child->flags =
+		process_flags::running |
+		process_flags::ready;
+
+	sched.m_runlists[child->priority].push_back(child);
+
+	child->rsp = in_rsp;
+
+	child->pml4 = page_manager::get()->copy_table(pml4);
+	kassert(child->pml4, "process::fork() got null pml4");
+
+	child->setup_kernel_stack(kernel_stack_size, kernel_stack);
+	child->rsp = child->kernel_stack + (in_rsp - kernel_stack);
+
+	log::debug(logs::task, "Copied process %d to %d, new PML4 %016lx.",
+			pid, child->pid, child->pml4);
+	log::debug(logs::task, "  copied stack %016lx to %016lx, rsp %016lx to %016lx.",
+			kernel_stack, child->kernel_stack, in_rsp, child->rsp);
+
+	// Add in the faked fork return value
+	cpu_state *regs = reinterpret_cast<cpu_state *>(child->rsp);
+	regs->rax = 0;
+
+	return child->pid;
+}
+
+void *
+process::setup_kernel_stack(size_t size, uintptr_t orig)
+{
+	void *stack0 = kutil::malloc(size);
+
+	if (orig)
+		kutil::memcpy(stack0, reinterpret_cast<void*>(orig), size);
+	else
+		kutil::memset(stack0, 0, size);
+
+	kernel_stack_size = size;
+	kernel_stack = reinterpret_cast<uintptr_t>(stack0);
+
+	return stack0;
+}
+
+bool
+process::wait_on_signal(uint64_t sigmask)
+{
+	waiting = process_wait::signal;
+	waiting_info = sigmask;
+	flags -= process_flags::ready;
+	return true;
+}
+
+bool
+process::wait_on_child(uint32_t pid)
+{
+	waiting = process_wait::child;
+	waiting_info = pid;
+	flags -= process_flags::ready;
+	return true;
+}
+
+bool
+process::wait_on_time(uint64_t time)
+{
+	waiting = process_wait::time;
+	waiting_info = time;
+	flags -= process_flags::ready;
+	return true;
+}
+
+bool
+process::wait_on_send(uint32_t target_id)
+{
+	scheduler &s = scheduler::get();
+	process *target = s.get_process_by_id(target_id);
+	if (!target) return false;
+
+	if (!target->wake_on_receive(this)) {
+		waiting = process_wait::send;
+		waiting_info = target_id;
+		flags -= process_flags::ready;
+	}
+
+	return true;
+}
+
+bool
+process::wait_on_receive(uint32_t source_id)
+{
+	scheduler &s = scheduler::get();
+	process *source = s.get_process_by_id(source_id);
+	if (!source) return false;
+
+	if (!source->wake_on_send(this)) {
+		waiting = process_wait::receive;
+		waiting_info = source_id;
+		flags -= process_flags::ready;
+		return true;
+	}
+
+	return false;
+}
+
+bool
+process::wake_on_signal(int signal)
+{
+	if (waiting != process_wait::signal ||
+		(waiting_info & (1 << signal)) == 0)
+		return false;
+
+	waiting = process_wait::none;
+	flags += process_flags::ready;
+	return true;
+}
+
+bool
+process::wake_on_child(process *child)
+{
+	if (waiting != process_wait::child ||
+		(waiting_info && waiting_info != child->pid))
+		return false;
+
+	waiting = process_wait::none;
+	flags += process_flags::ready;
+	return true;
+}
+
+bool
+process::wake_on_time(uint64_t now)
+{
+	if (waiting != process_wait::time ||
+		waiting_info > now)
+		return false;
+
+	waiting = process_wait::none;
+	flags += process_flags::ready;
+	return true;
+}
+
+bool
+process::wake_on_send(process *target)
+{
+	if (waiting != process_wait::send ||
+		waiting_info != target->pid)
+		return false;
+
+	waiting = process_wait::none;
+	flags += process_flags::ready;
+	return true;
+}
+
+
+bool
+process::wake_on_receive(process *source)
+{
+	if (waiting != process_wait::receive ||
+		waiting_info != source->pid)
+		return false;
+
+	waiting = process_wait::none;
+	flags += process_flags::ready;
+	return true;
+}
+

src/kernel/process.h

@@ -0,0 +1,131 @@
+#pragma once
+/// \file process.h
+/// The processes and related definitions
+
+#include <stdint.h>
+#include "kutil/enum_bitfields.h"
+#include "kutil/linked_list.h"
+#include "page_manager.h"
+
+typedef uint32_t pid_t;
+
+
+enum class process_flags : uint32_t
+{
+	running   = 0x00000001,
+	ready     = 0x00000002,
+	loading   = 0x00000004,
+
+	const_pri = 0x80000000,
+
+	none      = 0x00000000
+};
+IS_BITFIELD(process_flags);
+
+enum class process_wait : uint8_t
+{
+	none,
+	signal,
+	child,
+	time,
+	send,
+	receive
+};
+
+/// A process
+struct process
+{
+	pid_t pid;
+	pid_t ppid;
+
+	process_flags flags;
+
+	uint16_t quanta;
+
+	uint8_t priority;
+
+	process_wait waiting;
+	uint64_t waiting_info;
+
+	uint32_t return_code;
+
+	uint32_t reserved1;
+
+	uintptr_t rsp;
+	page_table *pml4;
+
+	uintptr_t kernel_stack;
+	size_t kernel_stack_size;
+
+	/// Copy this process.
+	/// \arg in_rsp The RSP of the calling process
+	/// \returns    Returns the child's pid to the parent, and
+	///             0 to the child.
+	pid_t fork(uint64_t in_rsp);
+
+	/// Unready this process until it gets a signal
+	/// \arg sigmask  A bitfield of signals to wake on
+	/// \returns  Whether the process should be rescheduled
+	bool wait_on_signal(uint64_t sigmask);
+
+	/// Unready this process until a child exits
+	/// \arg pid  PID of the child to wait for, or 0 for any
+	/// \returns  Whether the process should be rescheduled
+	bool wait_on_child(uint32_t pid);
+
+	/// Unready this process until after the given time
+	/// \arg time  The time after which to wake
+	/// \returns  Whether the process should be rescheduled
+	bool wait_on_time(uint64_t time);
+
+	/// Try to send to the target process, becoming unready if it
+	/// is not waiting on receive.
+	/// \arg target_id  The process to send to
+	/// \returns  Whether the process should be rescheduled
+	bool wait_on_send(uint32_t target_id);
+
+	/// Try to receive from one or more processes, becoming unready
+	/// if none of them are waiting on a send to this process.
+	/// \arg source_id  The process to receive from
+	/// \returns  Whether the process should be rescheduled
+	bool wait_on_receive(uint32_t source_id);
+
+	/// If this process is waiting on the given signal, wake it
+	/// \argument signal  The signal sent to the process
+	/// \returns  True if this wake was handled
+	bool wake_on_signal(int signal);
+
+	/// If this process is waiting on the given child, wake it
+	/// \argument child  The process that exited
+	/// \returns  True if this wake was handled
+	bool wake_on_child(process *child);
+
+	/// If this process is waiting on a time, check it
+	/// \argument now  The current time
+	/// \returns  True if this wake was handled
+	bool wake_on_time(uint64_t now);
+
+	/// If this process is waiting to send to this target, wake it
+	/// \argument target  The target process
+	/// \returns  True if this wake was handled
+	bool wake_on_send(process *target);
+
+	/// If this process is waiting to receieve from this source, wake it
+	/// \argument source  The process that is sending
+	/// \returns  True if this wake was handled
+	bool wake_on_receive(process *source);
+
+private:
+	friend class scheduler;
+
+	/// Set up a new kernel stack for this process, optionally copying the
+	/// given stack. Sets the kernel stack on the process object, but also
+	/// returns it.
+	/// \arg size  Size of the stack to allocate
+	/// \arg orig  Address of a stack to copy, or 0 for no copying.
+	/// \returns   The address of the new stack as a pointer
+	void * setup_kernel_stack(size_t size, uintptr_t orig);
+};
+
+using process_list = kutil::linked_list<process>;
+using process_node = process_list::item_type;

src/kernel/push_all.inc

@@ -0,0 +1,48 @@
+%macro push_all_and_segments 0
+	push rax
+	push rcx
+	push rdx
+	push rbx
+	push rbp
+	push rsi
+	push rdi
+
+	push r8
+	push r9
+	push r10
+	push r11
+	push r12
+	push r13
+	push r14
+	push r15
+
+	mov ax, ds
+	push rax
+%endmacro
+
+%macro pop_all_and_segments 0
+	pop rax
+	mov ds, ax
+	mov es, ax
+	mov fs, ax
+	mov gs, ax
+
+	pop r15
+	pop r14
+	pop r13
+	pop r12
+	pop r11
+	pop r10
+	pop r9
+	pop r8
+
+	pop rdi
+	pop rsi
+	pop rbp
+	pop rbx
+	pop rdx
+	pop rcx
+	pop rax
+%endmacro
+
+; vim: ft=asm

src/kernel/scheduler.cpp

@@ -0,0 +1,308 @@
+#include "apic.h"
+#include "console.h"
+#include "cpu.h"
+#include "gdt.h"
+#include "interrupts.h"
+#include "io.h"
+#include "kernel_memory.h"
+#include "log.h"
+#include "msr.h"
+#include "page_manager.h"
+#include "scheduler.h"
+
+#include "elf/elf.h"
+#include "kutil/assert.h"
+
+using memory::initial_stack;
+
+scheduler scheduler::s_instance(nullptr);
+
+const int stack_size = 0x1000;
+const uint64_t rflags_noint = 0x002;
+const uint64_t rflags_int = 0x202;
+
+extern "C" {
+	void ramdisk_process_loader();
+	void load_process(const void *image_start, size_t bytes, process *proc, cpu_state state);
+};
+
+scheduler::scheduler(lapic *apic) :
+	m_apic(apic),
+	m_next_pid(1)
+{
+	auto *idle = m_process_allocator.pop();
+
+	uint8_t last_pri = num_priorities - 1;
+
+	// The kernel idle task, also the thread we're in now
+	idle->pid = 0;
+	idle->ppid = 0;
+	idle->priority = last_pri;
+	idle->rsp = 0; // This will get set when we switch away
+	idle->pml4 = page_manager::get_pml4();
+	idle->quanta = process_quanta;
+	idle->flags =
+		process_flags::running |
+		process_flags::ready |
+		process_flags::const_pri;
+
+	m_runlists[last_pri].push_back(idle);
+	m_current = idle;
+}
+
+void
+load_process(const void *image_start, size_t bytes, process *proc, cpu_state state)
+{
+	// We're now in the process space for this process, allocate memory for the
+	// process code and load it
+	page_manager *pager = page_manager::get();
+
+	log::debug(logs::task, "Loading task! ELF: %016lx [%d]", image_start, bytes);
+
+	// TODO: Handle bad images gracefully
+	elf::elf image(image_start, bytes);
+	kassert(image.valid(), "Invalid ELF passed to load_process");
+
+	const unsigned program_count = image.program_count();
+	for (unsigned i = 0; i < program_count; ++i) {
+		const elf::program_header *header = image.program(i);
+
+		if (header->type != elf::segment_type::load)
+			continue;
+
+		uintptr_t aligned = header->vaddr & ~(memory::frame_size - 1);
+		size_t size = (header->vaddr + header->mem_size) - aligned;
+		size_t pages = page_manager::page_count(size);
+
+		log::debug(logs::task, "  Loadable segment %02u: vaddr %016lx  size %016lx",
+			i, header->vaddr, header->mem_size);
+
+		log::debug(logs::task, "         - aligned to: vaddr %016lx  pages %d",
+			aligned, pages);
+
+		void *mapped = pager->map_pages(aligned, pages, true);
+		kassert(mapped, "Tried to map userspace pages and failed!");
+
+		kutil::memset(mapped, 0, pages * memory::frame_size);
+	}
+
+	const unsigned section_count = image.section_count();
+	for (unsigned i = 0; i < section_count; ++i) {
+		const elf::section_header *header = image.section(i);
+
+		if (header->type != elf::section_type::progbits ||
+			!bitfield_has(header->flags, elf::section_flags::alloc))
+			continue;
+
+		log::debug(logs::task, "  Loadable section %02u: vaddr %016lx  size %016lx",
+			i, header->addr, header->size);
+
+		void *dest = reinterpret_cast<void *>(header->addr);
+		const void *src = kutil::offset_pointer(image_start, header->offset);
+		kutil::memcpy(dest, src, header->size);
+	}
+
+	state.rip = image.entrypoint();
+	proc->flags &= ~process_flags::loading;
+
+	log::debug(logs::task, "  Loaded! New process rip: %016lx", state.rip);
+}
+
+process_node *
+scheduler::create_process()
+{
+	auto *proc = m_process_allocator.pop();
+	proc->pid = m_next_pid++;
+	proc->priority = default_priority;
+	return proc;
+}
+
+void
+scheduler::load_process(const char *name, const void *data, size_t size)
+{
+	auto *proc = create_process();
+
+	uint16_t kcs = (1 << 3) | 0; // Kernel CS is GDT entry 1, ring 0
+	uint16_t cs = (5 << 3) | 3;  // User CS is GDT entry 5, ring 3
+
+	uint16_t kss = (2 << 3) | 0; // Kernel SS is GDT entry 2, ring 0
+	uint16_t ss = (4 << 3) | 3;  // User SS is GDT entry 4, ring 3
+
+	// Set up the page tables - this also allocates an initial user stack
+	page_table *pml4 = page_manager::get()->create_process_map();
+
+	// Create a one-page kernel stack space
+	void *stack0 = proc->setup_kernel_stack(stack_size, 0);
+
+	// Stack grows down, point to the end
+	void *sp0 = kutil::offset_pointer(stack0, stack_size);
+
+	cpu_state *state = reinterpret_cast<cpu_state *>(sp0) - 1;
+
+	// Highest state in the stack is the process' kernel stack for the loader
+	// to iret to:
+	state->ds = state->ss = ss;
+	state->cs = cs;
+	state->rflags = rflags_int;
+	state->rip = 0; // to be filled by the loader
+	state->user_rsp = initial_stack;
+
+	// Next state in the stack is the loader's kernel stack. The scheduler will
+	// iret to this which will kick off the loading:
+	cpu_state *loader_state = reinterpret_cast<cpu_state *>(sp0) - 2;
+
+	loader_state->ds = loader_state->ss = kss;
+	loader_state->cs = kcs;
+	loader_state->rflags = rflags_noint;
+	loader_state->rip = reinterpret_cast<uint64_t>(ramdisk_process_loader);
+	loader_state->user_rsp = reinterpret_cast<uint64_t>(state);
+
+	loader_state->rax = reinterpret_cast<uint64_t>(data);
+	loader_state->rbx = size;
+
+	proc->rsp = reinterpret_cast<uintptr_t>(loader_state);
+	proc->pml4 = pml4;
+	proc->quanta = process_quanta;
+	proc->flags =
+		process_flags::running |
+		process_flags::ready |
+		process_flags::loading;
+
+	m_runlists[default_priority].push_back(proc);
+
+	loader_state->rcx = reinterpret_cast<uint64_t>(proc);
+
+	log::debug(logs::task, "Creating process %s: pid %d  pri %d", name, proc->pid, proc->priority);
+	log::debug(logs::task, "     RSP0 %016lx", state);
+	log::debug(logs::task, "     PML4 %016lx", pml4);
+}
+
+void
+scheduler::start()
+{
+	log::info(logs::task, "Starting scheduler.");
+	m_tick_count = m_apic->enable_timer(isr::isrTimer, quantum_micros, false);
+}
+
+void scheduler::prune(uint64_t now)
+{
+	// Find processes that aren't ready or aren't running and
+	// move them to the appropriate lists.
+	for (auto &pri_list : m_runlists) {
+		auto *proc = pri_list.front();
+		while (proc) {
+			bool running = proc->flags && process_flags::running;
+			bool ready = proc->flags && process_flags::ready;
+			if (running && ready) {
+				proc = proc->next();
+				continue;
+			}
+
+			auto *remove = proc;
+			proc = proc->next();
+			pri_list.remove(remove);
+
+			if (!(remove->flags && process_flags::running)) {
+				auto *parent = get_process_by_id(remove->ppid);
+				if (parent && parent->wake_on_child(remove)) {
+					m_blocked.remove(parent);
+					m_runlists[parent->priority].push_front(parent);
+					m_process_allocator.push(remove);
+				} else {
+					m_exited.push_back(remove);
+				}
+			} else {
+				m_blocked.push_back(remove);
+			}
+		}
+	}
+
+	// Find blocked processes that are ready (possibly after waking wating
+	// ones) and move them to the appropriate runlist.
+	auto *proc = m_blocked.front();
+	while (proc) {
+		bool ready = proc->flags && process_flags::ready;
+		ready |= proc->wake_on_time(now);
+
+		auto *remove = proc;
+		proc = proc->next();
+		if (!ready) continue;
+
+		m_blocked.remove(remove);
+		m_runlists[remove->priority].push_front(remove);
+	}
+}
+
+uintptr_t
+scheduler::schedule(uintptr_t rsp0)
+{
+
+	// TODO: lol a real clock
+	static uint64_t now = 0;
+
+	m_current->rsp = rsp0;
+	m_runlists[m_current->priority].remove(m_current);
+
+	if (m_current->flags && process_flags::ready) {
+		m_runlists[m_current->priority].push_back(m_current);
+	} else {
+		m_blocked.push_back(m_current);
+	}
+
+	prune(++now);
+
+	uint8_t pri = 0;
+	while (m_runlists[pri].empty()) {
+		++pri;
+		kassert(pri < num_priorities, "All runlists are empty");
+	}
+
+	m_current = m_runlists[pri].pop_front();
+	rsp0 = m_current->rsp;
+
+	// Set rsp0 to after the end of the about-to-be-popped cpu state
+	tss_set_stack(0, rsp0 + sizeof(cpu_state));
+	wrmsr(msr::ia32_kernel_gs_base, rsp0);
+
+	// Swap page tables
+	page_table *pml4 = m_current->pml4;
+	page_manager::set_pml4(pml4);
+
+	bool loading = m_current->flags && process_flags::loading;
+	log::debug(logs::task, "Scheduler switched to process %d, priority %d%s.",
+			m_current->pid, m_current->priority, loading ? " (loading)" : "");
+
+	return rsp0;
+}
+
+uintptr_t
+scheduler::tick(uintptr_t rsp0)
+{
+	if (--m_current->quanta == 0) {
+		m_current->quanta = process_quanta;
+		rsp0 = schedule(rsp0);
+	}
+	m_apic->reset_timer(m_tick_count);
+	return rsp0;
+}
+
+process_node *
+scheduler::get_process_by_id(uint32_t pid)
+{
+	// TODO: this needs to be a hash map
+	for (auto *proc : m_blocked) {
+		if (proc->pid == pid) return proc;
+	}
+
+	for (int i = 0; i < num_priorities; ++i) {
+		for (auto *proc : m_runlists[i]) {
+			if (proc->pid == pid) return proc;
+		}
+	}
+
+	for (auto *proc : m_exited) {
+		if (proc->pid == pid) return proc;
+	}
+
+	return nullptr;
+}

src/kernel/scheduler.h

@@ -0,0 +1,92 @@
+#pragma once
+/// \file scheduler.h
+/// The task scheduler and related definitions
+
+#include <stdint.h>
+#include "kutil/slab_allocator.h"
+#include "process.h"
+
+class lapic;
+struct page_table;
+struct cpu_state;
+
+extern "C" uintptr_t isr_handler(uintptr_t, cpu_state*);
+
+
+/// The task scheduler
+class scheduler
+{
+public:
+	static const uint8_t num_priorities = 8;
+	static const uint8_t default_priority = num_priorities / 2;
+
+	/// How long the timer quantum is
+	static const uint64_t quantum_micros = 100000;
+
+	/// How many quantums a process gets before being rescheduled
+	static const uint16_t process_quanta = 10;
+
+	/// Constructor.
+	/// \arg apic  Pointer to the local APIC object
+	scheduler(lapic *apic);
+
+	/// Create a new process from a program image in memory.
+	/// \arg name  Name of the program image
+	/// \arg data  Pointer to the image data
+	/// \arg size  Size of the program image, in bytes
+	void load_process(const char *name, const void *data, size_t size);
+
+	/// Start the scheduler working. This may involve starting
+	/// timer interrupts or other preemption methods.
+	void start();
+
+	/// Run the scheduler, possibly switching to a new task
+	/// \arg rsp0  The stack pointer of the current interrupt handler
+	/// \returns   The stack pointer to switch to
+	uintptr_t schedule(uintptr_t rsp0);
+
+	/// Get the current process.
+	/// \returns  A pointer to the current process' process struct
+	inline process * current() { return m_current; }
+
+	/// Look up a process by its PID
+	/// \arg pid  The requested PID
+	/// \returns  The process matching that PID, or nullptr
+	process_node * get_process_by_id(uint32_t pid);
+
+	/// Get a reference to the system scheduler
+	/// \returns  A reference to the global system scheduler
+	static scheduler & get() { return s_instance; }
+
+private:
+	friend uintptr_t syscall_dispatch(uintptr_t, cpu_state &);
+	friend uintptr_t isr_handler(uintptr_t, cpu_state*);
+	friend class process;
+
+	/// Create a new process object. This process will have its pid
+	/// set but nothing else.
+	/// \returns  The new process object
+	process_node * create_process();
+
+	/// Handle a timer tick
+	/// \arg rsp0  The stack pointer of the current interrupt handler
+	/// \returns   The stack pointer to switch to
+	uintptr_t tick(uintptr_t rsp0);
+
+	void prune(uint64_t now);
+
+	lapic *m_apic;
+
+	uint32_t m_next_pid;
+	uint32_t m_tick_count;
+
+	using process_slab = kutil::slab_allocator<process>;
+	process_slab m_process_allocator;
+
+	process_node *m_current;
+	process_list m_runlists[num_priorities];
+	process_list m_blocked;
+	process_list m_exited;
+
+	static scheduler s_instance;
+};

src/kernel/syscall.cpp

@@ -0,0 +1,144 @@
+#include "console.h"
+#include "cpu.h"
+#include "debug.h"
+#include "msr.h"
+#include "process.h"
+#include "scheduler.h"
+#include "syscall.h"
+
+extern "C" {
+	void _halt();
+	void syscall_handler_prelude();
+}
+
+void
+syscall_enable()
+{
+	// IA32_EFER - set bit 0, syscall enable
+	uint64_t efer = rdmsr(msr::ia32_efer);
+	wrmsr(msr::ia32_efer, efer|1);
+
+	// IA32_STAR - high 32 bits contain k+u CS
+	// Kernel CS: GDT[1] ring 0 bits[47:32]
+	//   User CS: GDT[3] ring 3 bits[63:48]
+	uint64_t star =
+		(((1ull << 3) | 0) << 32) |
+		(((3ull << 3) | 3) << 48);
+	wrmsr(msr::ia32_star, star);
+
+	// IA32_LSTAR - RIP for syscall
+	wrmsr(msr::ia32_lstar,
+		reinterpret_cast<uintptr_t>(&syscall_handler_prelude));
+
+	// IA32_FMASK - FLAGS mask inside syscall
+	wrmsr(msr::ia32_fmask, 0x200);
+}
+
+uintptr_t
+syscall_dispatch(uintptr_t return_rsp, cpu_state &regs)
+{
+	console *cons = console::get();
+	syscall call = static_cast<syscall>(regs.rax);
+
+	auto &s = scheduler::get();
+	auto *p = s.current();
+
+	switch (call) {
+	case syscall::noop:
+		break;
+
+	case syscall::debug:
+		cons->set_color(11);
+		cons->printf("\nProcess %u: Received DEBUG syscall\n", p->pid);
+		cons->set_color();
+		print_regs(regs);
+		break;
+
+	case syscall::message:
+		cons->set_color(11);
+		cons->printf("\nProcess %u: Received MESSAGE syscall\n", p->pid);
+		cons->set_color();
+		break;
+
+	case syscall::pause:
+		{
+			cons->set_color(11);
+
+			auto &s = scheduler::get();
+			auto *p = s.current();
+			p->wait_on_signal(-1ull);
+			cons->printf("\nProcess %u: Received PAUSE syscall\n", p->pid);
+			cons->set_color();
+			return_rsp = s.schedule(return_rsp);
+		}
+		break;
+
+	case syscall::sleep:
+		{
+			cons->set_color(11);
+			cons->printf("\nProcess %u: Received SLEEP syscall\n", p->pid);
+			cons->printf("Sleeping until %lu\n", regs.rbx);
+			cons->set_color();
+
+			p->wait_on_time(regs.rbx);
+			return_rsp = s.schedule(return_rsp);
+		}
+		break;
+
+	case syscall::getpid:
+		cons->set_color(11);
+		cons->printf("\nProcess %u: Received GETPID syscall\n", p->pid);
+		cons->set_color();
+		regs.rax = p->pid;
+		break;
+
+	case syscall::send:
+		{
+			uint32_t target = regs.rdi;
+
+			cons->set_color(11);
+			cons->printf("\nProcess %u: Received SEND syscall, target %u\n", p->pid, target);
+			cons->set_color();
+
+			if (p->wait_on_send(target))
+				return_rsp = s.schedule(return_rsp);
+		}
+		break;
+
+	case syscall::receive:
+		{
+			uint32_t source = regs.rdi;
+
+			cons->set_color(11);
+			cons->printf("\nProcess %u: Received RECEIVE syscall, source %u\n", p->pid, source);
+			cons->set_color();
+
+			if (p->wait_on_receive(source))
+				return_rsp = s.schedule(return_rsp);
+		}
+		break;
+
+	case syscall::fork:
+		{
+			cons->set_color(11);
+			cons->printf("\nProcess %u: Received FORK syscall\n", p->pid);
+			cons->set_color();
+
+			pid_t pid = p->fork(return_rsp);
+			if (pid == scheduler::get().current()->pid)
+				pid = 0;
+			regs.rax = pid;
+		}
+		break;
+
+	default:
+		cons->set_color(9);
+		cons->printf("\nReceived unknown syscall: %02x\n", call);
+		cons->set_color();
+		_halt();
+		break;
+	}
+
+	return return_rsp;
+}
+

src/kernel/syscall.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <stdint.h>
+
+struct cpu_state;
+
+enum class syscall : uint64_t
+{
+	noop           = 0x0000,
+	debug          = 0x0001,
+	message        = 0x0002,
+	pause          = 0x0003,
+	sleep          = 0x0004,
+	getpid         = 0x0005,
+	send           = 0x0006,
+	receive        = 0x0007,
+	fork           = 0x0008,
+
+	last_syscall
+};
+
+void syscall_enable();
+uintptr_t syscall_dispatch(uintptr_t, cpu_state &);
+

src/kernel/syscall.s

@@ -0,0 +1,24 @@
+%include "push_all.inc"
+
+extern syscall_handler
+global syscall_handler_prelude
+syscall_handler_prelude:
+	push 0		; ss, doesn't matter here
+	push rsp
+	pushf
+	push 0		; cs, doesn't matter here
+	push rcx	; user rip
+	push 0		; bogus interrupt
+	push 0		; bogus errorcode
+	push_all_and_segments
+
+	mov rdi, rsp
+	call syscall_handler
+	mov rsp, rax
+
+	pop_all_and_segments
+	add rsp, 16	; ignore bogus interrupt / error
+	pop rcx		; user rip
+	add rsp, 32 ; ignore cs, flags, rsp, ss
+
+	o64 sysret

src/kernel/wscript

@@ -1,43 +0,0 @@
-
-def configure(ctx):
-    pass
-
-def build(bld):
-    from os.path import join
-
-    sources = bld.path.ant_glob("**/*.cpp")
-    sources += bld.path.ant_glob("**/*.s")
-
-    lds = join(bld.env.ARCH_D, 'kernel.ld')
-
-    bld.program(
-        source = sources,
-        name = 'kernel',
-        includes = '.',
-        target = bld.env.KERNEL_FILENAME,
-        use = 'kutil',
-        linkflags = "-T {}".format(lds),
-    )
-
-    from waflib.Task import Task
-    class objdump(Task):
-        color = 'PINK'
-        def keyword(self):
-            return "Dumping"
-        def __str__(self):
-            node = self.outputs[0]
-            return node.path_from(node.ctx.launch_node())
-        def run(self):
-            from subprocess import check_output
-            args = self.env.objdump + ["--source", "-D", self.inputs[0].abspath()]
-            with file(self.outputs[0].abspath(), 'w') as output:
-                output.write(check_output(args))
-
-    out = bld.path.get_bld()
-
-    dump = objdump(env=bld.env)
-    dump.set_inputs([out.make_node(bld.env.KERNEL_FILENAME)])
-    dump.set_outputs([out.make_node("kernel.dump")])
-    bld.add_to_group(dump)
-
-# vim: ft=python et