[boot] Create bootconfig to tell boot what to load

While bonnibel already had the concept of a manifest, which controls
what goes into the built disk image, the bootloader still had filenames
hard-coded. Now bonnibel creates a 'jsix_boot.dat' file that tells the
bootloader what it should load.

Changes include:

- Modules have two new fields: location and description. location is
  their intended directory on the EFI boot volume. description is
  self-explanatory, and is used in log messages.
- New class, boot::bootconfig, implements reading of jsix_boot.dat
- New header, bootproto/bootconfig.h, specifies flags used in the
  manifest and jsix_boot.dat
- New python module, bonnibel/manifest.py, encapsulates reading of the
  manifest and writing jsix_boot.dat
- Syntax of the manifest changed slightly, including adding flags
- Boot and Kernel target ccflags unified a bit (this was partly due to
  trying to get enum_bitfields to work in boot)
- util::counted gained operator+= and new free function util::read<T>

assets/manifests/default.yaml

@@ -1,8 +1,9 @@
 ---
+init: srv.init
 programs:
   - name: panic.serial
     target: kernel
-
+    flags: panic
   - name: drv.uefi_fb
+    flags: graphical
   - name: testapp
-  - name: srv.init

configs/boot-debug.yaml

@@ -5,6 +5,10 @@ variables:
   ld: clang++
 
   ccflags: [
+    "-nostdlib",
+    "-nodefaultlibs",
+    "-fno-builtin",
+
     "-I${source_root}/external",
     "--target=x86_64-unknown-windows",
     "-ffreestanding",
@@ -12,8 +16,7 @@ variables:
     "-fshort-wchar",
     "-fno-omit-frame-pointer",
     "-ggdb",
-    "-g3",
+    "-g3" ]
-    '-DKERNEL_FILENAME=L"jsix.elf"' ]
 
   cxxflags: [ "-fno-exceptions", "-fno-rtti" ]
 

configs/kernel-debug.yaml

@@ -8,6 +8,7 @@ variables:
     "--target=x86_64-unknown-elf",
     "-I${source_root}/external",
 
+    "-nostdinc",
     "-nostdlib",
     "-ffreestanding",
     "-nodefaultlibs",
@@ -18,7 +19,8 @@ variables:
     "-mno-red-zone",
     "-mcmodel=large",
 
-    "-g",
+    "-g3",
+    "-ggdb",
 
     "-D__ELF__",
     "-D__JSIX__",

scripts/bonnibel/manifest.py

@@ -0,0 +1,94 @@
+from . import BonnibelError
+
+class Manifest:
+    from collections import namedtuple
+    Entry = namedtuple("Entry", ("module", "target", "output", "location", "description", "flags"))
+
+    flags = {
+        "graphical": 0x01,
+        "panic":     0x02,
+        "symbols":   0x04,
+    }
+
+    def __init__(self, path, modules):
+        from . import load_config
+
+        config = load_config(path)
+
+        self.kernel = self.__build_entry(modules,
+                config.get("kernel", dict()),
+                name="kernel", target="kernel")
+
+        self.init = self.__build_entry(modules,
+                config.get("init", None))
+
+        self.programs = [self.__build_entry(modules, i)
+                for i in config.get("programs", tuple())]
+
+        self.data = []
+        for d in config.get("data", tuple()):
+            self.add_data(**d)
+
+    def __build_entry(self, modules, config, target="user", name=None):
+        flags = tuple()
+
+        if isinstance(config, str):
+            name = config
+        elif isinstance(config, dict):
+            name = config.get("name", name)
+            target = config.get("target", target)
+            flags = config.get("flags", tuple())
+            if isinstance(flags, str):
+                flags = flags.split()
+
+        mod = modules.get(name)
+        if not mod:
+            raise BonnibelError(f"Manifest specifies unknown module '{name}'")
+
+        for f in flags:
+            if not f in Manifest.flags:
+                raise BonnibelError(f"Manifest specifies unknown flag '{f}'")
+
+        return Manifest.Entry(name, target, mod.output, mod.location, mod.description, flags)
+
+    def add_data(self, output, location, desc, flags=tuple()):
+        e = Manifest.Entry(None, None, output, location, desc, flags)
+        self.data.append(e)
+        return e
+
+    def write_boot_config(self, path):
+        from os.path import join
+        import struct
+
+        with open(path, 'wb') as outfile:
+            magic = "jsixboot".encode("utf-8") # magic string
+            version = 0
+            reserved = 0
+
+            outfile.write(struct.pack("<8sBBHHH",
+                magic, version, reserved,
+                len(self.programs), len(self.data),
+                reserved))
+
+            def write_str(s):
+                outfile.write(struct.pack("<H", (len(s)+1)*2))
+                outfile.write(s.encode("utf-16le"))
+                outfile.write(b"\0\0")
+
+            def write_ent(ent):
+                flags = 0
+                for f in ent.flags:
+                    flags |= Manifest.flags[f]
+
+                outfile.write(struct.pack("<H", flags))
+                write_str(join(ent.location, ent.output).replace('/','\\'))
+                write_str(ent.description)
+
+            write_ent(self.kernel)
+            write_ent(self.init)
+
+            for p in self.programs:
+                write_ent(p)
+
+            for d in self.data:
+                write_ent(d)

scripts/bonnibel/module.py

@@ -15,6 +15,8 @@ class Module:
         "sources": (tuple, ()),
         "variables": (dict, ()),
         "default": (bool, False),
+        "location": (str, "jsix"),
+        "description": (str, None),
     }
 
     def __init__(self, name, modfile, root, **kwargs):

scripts/bonnibel/project.py

@@ -51,56 +51,57 @@ class Project:
             fatroot.mkdir(exist_ok=True)
 
             fatroot_content = []
-            def add_fatroot(name, target):
-                if not name in modules:
-                    raise BonnibelError(f"Manifest item '{name}' is not a known module")
 
-                mod = modules[name]
+            def add_fatroot(source, entry):
-                intermediary = f"${{build_root}}/{mod.output}"
+                output = join(entry.location, entry.output)
-                fatroot_output = f"${{build_root}}/fatroot/{mod.output}"
+                fatroot_output = f"${{build_root}}/fatroot/{output}"
-
-                build.build(
-                    rule = "strip",
-                    outputs = [intermediary],
-                    inputs = [f"${{build_root}}/{target}/{mod.output}"],
-                    implicit = [f"${{build_root}}/{target}/{mod.output}.dump"],
-                    variables = {
-                        "name": f"Stripping {name}",
-                        "debug": f"${{build_root}}/.debug/{mod.output}.debug",
-                    })
 
                 build.build(
                     rule = "cp",
                     outputs = [fatroot_output],
-                    inputs = [intermediary],
+                    inputs = [source],
                     variables = {
-                        "name": f"Installing {name}",
+                        "name": f"Installing {output}",
                     })
 
                 fatroot_content.append(fatroot_output)
                 build.newline()
 
-            manifest = load_config(manifest_file)
+            def add_fatroot_exe(entry):
-            programs = manifest.get("programs", tuple())
+                input_path = f"${{build_root}}/{entry.target}/{entry.output}"
+                intermediary = f"${{build_root}}/{entry.output}"
 
-            kernel = manifest.get("kernel", dict())
+                build.build(
-            add_fatroot(
+                    rule = "strip",
-                kernel.get("name", "kernel"),
+                    outputs = [intermediary],
-                kernel.get("target", "kernel"))
+                    inputs = [input_path],
+                    implicit = [f"{input_path}.dump"],
+                    variables = {
+                        "name": f"Stripping {entry.module}",
+                        "debug": f"${{build_root}}/.debug/{entry.output}.debug",
+                    })
 
-            for program in programs:
+                add_fatroot(intermediary, entry)
-                name = program["name"]
+                return mod.location
-                target = program.get("target", "user")
-                add_fatroot(name, target)
 
-            symbol_table = "${build_root}/fatroot/symbol_table.dat"
+            from .manifest import Manifest
+            manifest = Manifest(manifest_file, modules)
+
+            add_fatroot_exe(manifest.kernel)
+            add_fatroot_exe(manifest.init)
+            for program in manifest.programs:
+                add_fatroot_exe(program)
+
+            syms = manifest.add_data("symbol_table.dat",
+                    manifest.kernel.location, "Symbol table", ("symbols",))
+
+            sym_out = f"${{build_root}}/symbol_table.dat"
             build.build(
                 rule = "makest",
-                outputs = [symbol_table],
+                outputs = [sym_out],
-                inputs = ["${build_root}/fatroot/jsix.elf"],
+                inputs = [f"${{build_root}}/{modules['kernel'].output}"],
                 )
-            fatroot_content.append(symbol_table)
+            add_fatroot(sym_out, syms)
-            build.newline()
 
             bootloader = "${build_root}/fatroot/efi/boot/bootx64.efi"
             build.build(
@@ -110,14 +111,16 @@ class Project:
                 variables = {
                     "name": "Installing bootloader",
                 })
-            fatroot_content.append(bootloader)
             build.newline()
 
+            boot_config = join(fatroot, "jsix_boot.dat")
+            manifest.write_boot_config(boot_config)
+
             build.build(
                 rule = "makefat",
                 outputs = ["${build_root}/jsix.img"],
                 inputs = ["${source_root}/assets/diskbase.img"],
-                implicit = fatroot_content,
+                implicit = fatroot_content + [bootloader],
                 variables = {"name": "jsix.img"},
                 )
             build.newline()
@@ -159,7 +162,8 @@ class Project:
                 implicit = regen_implicits,
                 implicit_outputs = 
                     [f"{mod.name}.ninja" for mod in modules.values()] +
-                    [f"{target.name}/target.ninja" for target in targets],
+                    [f"{target.name}/target.ninja" for target in targets] +
+                    [boot_config],
                 )
 
             build.newline()

src/boot/boot.module

@@ -7,6 +7,7 @@ module("boot",
     deps = [ "cpu", "elf", "util", "bootproto" ],
     sources = [
         "allocator.cpp",
+        "bootconfig.cpp",
         "console.cpp",
         "error.cpp",
         "fs.cpp",

src/boot/bootconfig.cpp

@@ -0,0 +1,62 @@
+#include <uefi/boot_services.h>
+#include <uefi/types.h>
+
+#include "bootconfig.h"
+#include "error.h"
+#include "fs.h"
+#include "status.h"
+
+namespace boot {
+
+constexpr uint64_t jsixboot = 0x746f6f627869736a; // "jsixboot"
+
+static const wchar_t *
+read_string(util::buffer &data)
+{
+    uint16_t size = *util::read<uint16_t>(data);
+    const wchar_t *string = reinterpret_cast<const wchar_t*>(data.pointer);
+    data += size;
+    return string;
+}
+
+static void
+read_descriptor(descriptor &e, util::buffer &data)
+{
+    e.flags = static_cast<desc_flags>(*util::read<uint16_t>(data));
+    e.path = read_string(data);
+    e.desc = read_string(data);
+}
+
+bootconfig::bootconfig(util::buffer data, uefi::boot_services *bs)
+{
+    status_line status {L"Loading boot config"};
+
+    if (*util::read<uint64_t>(data) != jsixboot)
+        error::raise(uefi::status::load_error, L"Bad header in jsix_boot.dat");
+
+    const uint8_t version = *util::read<uint8_t>(data);
+    if (version != 0)
+        error::raise(uefi::status::incompatible_version, L"Bad version in jsix_boot.dat");
+
+    data += 1; // reserved byte
+    uint16_t num_programs = *util::read<uint16_t>(data);
+    uint16_t num_data = *util::read<uint16_t>(data);
+    data += 2; // reserved short
+
+    read_descriptor(m_kernel, data);
+    read_descriptor(m_init, data);
+
+    m_programs.count = num_programs;
+    m_programs.pointer = new descriptor [num_programs];
+
+    for (unsigned i = 0; i < num_programs; ++i)
+        read_descriptor(m_programs[i], data);
+
+    m_data.count = num_programs;
+    m_data.pointer = new descriptor [num_data];
+
+    for (unsigned i = 0; i < num_data; ++i)
+        read_descriptor(m_data[i], data);
+}
+
+} // namespace boot

src/boot/bootconfig.h

@@ -0,0 +1,43 @@
+/// \file bootconfig.h
+/// Definitions for reading the jsix bootconfig file
+#pragma once
+
+#include <bootproto/bootconfig.h>
+#include <util/counted.h>
+
+namespace uefi {
+    struct boot_services;
+}
+
+namespace boot {
+
+using desc_flags = bootproto::desc_flags;
+
+struct descriptor {
+    desc_flags flags;
+    wchar_t const *path;
+    wchar_t const *desc;
+};
+
+/// A bootconfig is a manifest of potential files.
+class bootconfig
+{
+public:
+    using descriptors = util::counted<descriptor>;
+
+    /// Constructor. Loads bootconfig from the given buffer.
+    bootconfig(util::buffer data, uefi::boot_services *bs);
+
+    inline const descriptor & kernel() { return m_kernel; }
+    inline const descriptor & init() { return m_init; }
+    descriptors programs() { return m_programs; }
+    descriptors data() { return m_data; }
+
+private:
+    descriptor m_kernel;
+    descriptor m_init;
+    descriptors m_programs;
+    descriptors m_data;
+};
+
+} // namespace boot

src/boot/loader.cpp

@@ -7,6 +7,7 @@
 #include <util/pointers.h>
 
 #include "allocator.h"
+#include "bootconfig.h"
 #include "console.h"
 #include "error.h"
 #include "fs.h"
@@ -23,7 +24,7 @@ using memory::alloc_type;
 util::buffer
 load_file(
     fs::file &disk,
-    const program_desc &desc)
+    const descriptor &desc)
 {
     status_line status(L"Loading file", desc.path);
 
@@ -36,7 +37,7 @@ load_file(
 
 
 static void
-create_module(util::buffer data, const program_desc &desc, bool loaded)
+create_module(util::buffer data, const descriptor &desc, bool loaded)
 {
     size_t path_len = wstrlen(desc.path);
     bootproto::module_program *mod = g_alloc.allocate_module<bootproto::module_program>(path_len);
@@ -50,18 +51,20 @@ create_module(util::buffer data, const program_desc &desc, bool loaded)
 
     // TODO: support non-ascii path characters and do real utf-16 to utf-8
     // conversion
-    for (int i = 0; i < path_len; ++i)
+    for (int i = 0; i < path_len; ++i) {
-        mod->filename[i] = desc.path[i];
+        char c = desc.path[i];
+        mod->filename[i] = c == '\\' ? '/' : c;
+    }
     mod->filename[path_len] = 0;
 }
 
 bootproto::program *
 load_program(
     fs::file &disk,
-    const program_desc &desc,
+    const descriptor &desc,
     bool add_module)
 {
-    status_line status(L"Loading program", desc.name);
+    status_line status(L"Loading program", desc.desc);
 
     util::buffer data = load_file(disk, desc);
 
@@ -113,9 +116,9 @@ load_program(
 void
 load_module(
     fs::file &disk,
-    const program_desc &desc)
+    const descriptor &desc)
 {
-    status_line status(L"Loading module", desc.name);
+    status_line status(L"Loading module", desc.desc);
 
     util::buffer data = load_file(disk, desc);
     create_module(data, desc, false);

src/boot/loader.h

@@ -11,34 +11,30 @@ namespace bootproto {
 
 namespace boot {
 
+class descriptor;
+
 namespace fs {
     class file;
 }
 
 namespace loader {
 
-struct program_desc
-{
-    const wchar_t *name;
-    const wchar_t *path;
-};
-
 /// Load a file from disk into memory.
 /// \arg disk  The opened UEFI filesystem to load from
 /// \arg desc  The program descriptor identifying the file
 util::buffer
 load_file(
     fs::file &disk,
-    const program_desc &desc);
+    const descriptor &desc);
 
 /// Parse and load an ELF file in memory into a loaded image.
 /// \arg disk       The opened UEFI filesystem to load from
-/// \arg desc       The program descriptor identifying the program
+/// \arg desc       The descriptor identifying the program
 /// \arg add_module Also create a module for this loaded program
 bootproto::program *
 load_program(
     fs::file &disk,
-    const program_desc &desc,
+    const descriptor &desc,
     bool add_module = false);
 
 /// Load a file from disk into memory, creating an init args module
@@ -47,7 +43,7 @@ load_program(
 void
 load_module(
     fs::file &disk,
-    const program_desc &desc);
+    const descriptor &desc);
 
 /// Verify that a loaded ELF has the j6 kernel header
 /// \arg program  The program to check for a header

src/boot/main.cpp

@@ -7,12 +7,14 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include <bootproto/bootconfig.h>
 #include <bootproto/kernel.h>
 #include <bootproto/memory.h>
 #include <util/counted.h>
 #include <util/pointers.h>
 
 #include "allocator.h"
+#include "bootconfig.h"
 #include "console.h"
 #include "error.h"
 #include "fs.h"
@@ -27,17 +29,6 @@
 
 namespace boot {
 
-const loader::program_desc kern_desc = {L"kernel", L"jsix.elf"};
-const loader::program_desc init_desc = {L"init server", L"srv.init.elf"};
-const loader::program_desc fb_driver = {L"UEFI framebuffer driver", L"drv.uefi_fb.elf"};
-const loader::program_desc uart_driver = {L"Serial port driver", L"drv.uart.elf"};
-
-const loader::program_desc panic_handler = {L"Serial panic handler", L"panic.serial.elf"};
-
-const loader::program_desc extra_programs[] = {
-    {L"test application", L"testapp.elf"},
-};
-
 /// Change a pointer to point to the higher-half linear-offset version
 /// of the address it points to.
 template <typename T>
@@ -81,23 +72,51 @@ load_resources(bootproto::args *args, video::screen *screen, uefi::handle image,
     status_line status {L"Loading programs"};
 
     fs::file disk = fs::get_boot_volume(image, bs);
+    fs::file bc_data = disk.open(L"jsix_boot.dat");
+    bootconfig bc {bc_data.load(), bs};
+
+    args->kernel = loader::load_program(disk, bc.kernel(), true);
+    args->init = loader::load_program(disk, bc.init());
+
+    namespace bits = util::bits;
+    using bootproto::desc_flags;
 
     if (screen) {
         video::make_module(screen);
-        loader::load_module(disk, fb_driver);
+
-    } else {
+        // Go through the screen-specific descriptors first to
-        loader::load_module(disk, uart_driver);
+        // give them priority
+        for (const descriptor &d : bc.programs()) {
+            if (!bits::has(d.flags, desc_flags::graphical))
+                continue;
+
+            if (bits::has(d.flags, desc_flags::panic))
+                args->panic = loader::load_program(disk, d);
+            else
+                loader::load_module(disk, d);
+        }
     }
 
-    util::buffer symbol_table = loader::load_file(disk, {L"symbol table", L"symbol_table.dat"});
+    // Load the non-graphical descriptors
+    for (const descriptor &d : bc.programs()) {
+        if (bits::has(d.flags, desc_flags::graphical))
+            continue;
+
+        if (bits::has(d.flags, desc_flags::panic) && !args->panic)
+            args->panic = loader::load_program(disk, d);
+        else
+            loader::load_module(disk, d);
+    }
+
+    // For now the only data we load is the symbol table
+    for (const descriptor &d : bc.data()) {
+        if (!bits::has(d.flags, desc_flags::symbols))
+            continue;
+
+        util::buffer symbol_table = loader::load_file(disk, d);
         args->symbol_table = reinterpret_cast<uintptr_t>(symbol_table.pointer);
-
+        break;
-    args->kernel = loader::load_program(disk, kern_desc, true);
+    }
-    args->init = loader::load_program(disk, init_desc);
-    args->panic = loader::load_program(disk, panic_handler);
-
-    for (auto &desc : extra_programs)
-        loader::load_module(disk, desc);
 
     loader::verify_kernel_header(*args->kernel);
 }

src/kernel/kernel.module

@@ -5,6 +5,7 @@ kernel = module("kernel",
     default = True,
     output = "jsix.elf",
     targets = [ "kernel" ],
+    description = "jsix kernel",
     deps = [ "util", "cpu", "bootproto" ],
     includes = [ "." ],
     sources = [

src/kernel/panic.serial/serial.module

@@ -2,9 +2,9 @@
 
 panic = module("panic.serial",
     kind = "exe",
-    output = "panic.serial.elf",
     targets = [ "kernel" ],
     deps = [ "util", "elf", "kernel" ],
+    description = "Serial panic handler",
     sources = [
         "display.cpp",
         "entry.s",

src/libraries/bootproto/include/bootproto/bootconfig.h

@@ -0,0 +1,17 @@
+#pragma once
+/// \file bootproto/bootconfig.h
+/// Data structures for reading jsix_boot.dat
+
+#include <stdint.h>
+#include <util/enum_bitfields.h>
+
+namespace bootproto {
+
+enum class desc_flags : uint16_t {
+    graphical = 0x01,
+    panic     = 0x02,
+    symbols   = 0x04,
+};
+is_bitfield(desc_flags);
+
+} // namespace bootproto

src/libraries/libc/libc.module

@@ -2,7 +2,6 @@
 
 libc = module("libc",
     kind = "lib",
-    output = "libc.a",
     includes = [ "include" ],
     deps = [ "j6" ],
     sources = [

src/libraries/util/include/util/counted.h

@@ -35,6 +35,14 @@ struct counted
 
     /// Return an iterator to the end of the array
     inline const_iterator end() const { return offset_pointer<const T>(pointer, sizeof(T)*count); }
+
+    /// Advance the pointer by N items
+    inline counted<T> & operator+=(unsigned i) {
+        if (i > count) i = count;
+        pointer += i;
+        count -= i;
+        return *this;
+    }
 };
 
 /// Specialize for `void` which cannot be indexed or iterated
@@ -43,8 +51,24 @@ struct counted<void>
 {
     void *pointer;
     size_t count;
+
+    /// Advance the pointer by N bytes
+    inline counted<void> & operator+=(unsigned i) {
+        if (i > count) i = count;
+        pointer = offset_pointer(pointer, i);
+        count -= i;
+        return *this;
+    }
 };
 
 using buffer = counted<void>;
 
+template <typename T>
+const T * read(buffer &b) {
+    const T *p = reinterpret_cast<const T*>(b.pointer);
+    b.pointer = offset_pointer(b.pointer, sizeof(T));
+    b.count -= sizeof(T);
+    return p;
+}
+
 } // namespace util

src/user/drv.uefi_fb/uefi_fb.module

@@ -3,6 +3,7 @@
 module("drv.uefi_fb",
     targets = [ "user" ],
     deps = [ "libc" ],
+    description = "UEFI framebuffer driver",
     sources = [
         "font.cpp",
         "main.cpp",

src/user/srv.init/init.module

@@ -3,6 +3,7 @@
 init = module("srv.init",
     targets = [ "user" ],
     deps = [ "libc", "elf", "bootproto" ],
+    description = "Init server",
     sources = [
         "loader.cpp",
         "main.cpp",

src/user/testapp/testapp.module

@@ -3,6 +3,7 @@
 module("testapp",
     targets = [ "user" ],
     deps = [ "libc" ],
+    description = "Testbed app",
     sources = [
         "io.cpp",
         "main.cpp",