[boot] More initrd format changes

CDB seemed to be too simple for the needs of init, and squashfs is too
laden with design choices to work around Linux's APIs. This commit adds
creation of an initrd image of a new format I've called `j6romfs`.

Note that this commit currently does not work! The initrd-reading code
still needs to be added.

configs/rules.ninja

@@ -60,16 +60,16 @@ rule makest
     description = Making symbol table
     command = nm -n -S --demangle $in | ${source_root}/scripts/build_symbol_table.py $out
 
+rule makeinitrd
+    description = Creating $name
+    command = ${source_root}/scripts/mkj6romfs.py -c $format $in $out
+
 rule makefat
     description = Creating $name
     command = $
         cp $in $out; $
         mcopy -s -D o -i $out@@1M ${build_root}/fatroot/* ::/
 
-rule mkinitrd
-    description = Creating initrd
-    command = ${source_root}/scripts/mkinitrd.py $out $in
-
 rule strip
     description = Stripping $name
     command = $

scripts/bonnibel/manifest.py

@@ -4,11 +4,6 @@ class Manifest:
     from collections import namedtuple
     Entry = namedtuple("Entry", ("module", "target", "output", "flags"))
 
-    formats = {
-        "none":      0x00,
-        "zstd":      0x01,
-    }
-
     flags = {
         "graphical": 0x01,
         "symbols":   0x80,
@@ -45,11 +40,10 @@ class Manifest:
         self.flags = config.get("flags", tuple())
 
         initrd = config.get("initrd", dict())
-        self.initrd = initrd.get("name", "initrd.dat")
+        self.initrd = {
-        self.initrd_format = initrd.get("format", "none")
+            "name": initrd.get("name", "initrd.dat"),
-
+            "format": initrd.get("format", "zstd"),
-        if not self.initrd_format in Manifest.formats:
+        }
-            raise BonnibelError(f"Manifest specifies unknown initrd format '{self.initrd_format}'")
 
         self.data = []
         for d in config.get("data", tuple()):
@@ -89,15 +83,12 @@ class Manifest:
         with open(path, 'wb') as outfile:
             magic = "jsixboot".encode("utf-8") # magic string
             version = 1
-            reserved = 0
 
-            initrd_format = Manifest.formats.get(self.initrd_format, 0)
             bootflags = sum([Manifest.boot_flags.get(s, 0) for s in self.flags])
 
-            outfile.write(struct.pack("<8s BBH HH",
+            outfile.write(struct.pack("<8s BBH",
                 magic,
-                version, reserved, len(self.panics),
+                version, len(self.panics), bootflags))
-                initrd_format, bootflags))
 
             def write_str(s):
                 data = s.encode("utf-16le")
@@ -118,7 +109,7 @@ class Manifest:
 
             write_ent(self.kernel)
             write_ent(self.init)
-            write_path(self.initrd)
+            write_path(self.initrd["name"])
 
             for p in self.panics:
                 write_ent(p)

scripts/bonnibel/project.py

@@ -64,20 +64,25 @@ class Project:
                 outputs = ["all-headers"],
                 inputs = ["${build_root}/.all_headers"])
 
+            from .manifest import Manifest
+            manifest = Manifest(manifest_file, modules)
+
             debugroot = output / ".debug"
             debugroot.mkdir(exist_ok=True)
 
             fatroot = output / "fatroot"
             fatroot.mkdir(exist_ok=True)
 
+            (fatroot / manifest.location).mkdir(exist_ok=True)
+
+            initrdroot = output / "initrd_root"
+            initrdroot.mkdir(exist_ok=True)
+
             fatroot_content = []
             initrd_content = []
 
-            from .manifest import Manifest
+            def add_fatroot(source, name):
-            manifest = Manifest(manifest_file, modules)
+                output = join(manifest.location, name)
-
-            def add_fatroot(source, entry):
-                output = join(manifest.location, entry.output)
                 fatroot_output = f"${{build_root}}/fatroot/{output}"
 
                 build.build(
@@ -85,7 +90,7 @@ class Project:
                     outputs = [fatroot_output],
                     inputs = [source],
                     variables = {
-                        "name": f"Installing {output}",
+                        "description": f"Installing {output}",
                     })
 
                 fatroot_content.append(fatroot_output)
@@ -105,9 +110,24 @@ class Project:
                         "debug": f"${{build_root}}/.debug/{entry.output}.debug",
                     })
 
-                add_fatroot(intermediary, entry)
+                add_fatroot(intermediary, entry.output)
 
-            def add_initrd_exe(entry):
+            def add_initrd_content(root, name):
+                output = join(root, name)
+                initrd_output = f"${{build_root}}/initrd_root/{output}"
+
+                build.build(
+                    rule = "cp",
+                    outputs = [initrd_output],
+                    inputs = [f"${{build_root}}/{name}"],
+                    variables = {
+                        "description": f"Installing {name}",
+                    })
+
+                initrd_content.append(initrd_output)
+                build.newline()
+
+            def add_initrd_stripped(root, entry):
                 input_path = f"${{build_root}}/{entry.target}/{entry.output}"
                 intermediary = f"${{build_root}}/{entry.output}"
 
@@ -121,13 +141,18 @@ class Project:
                         "debug": f"${{build_root}}/.debug/{entry.output}.debug",
                     })
 
-                initrd_content.append(intermediary)
+                add_initrd_content(root, entry.output)
 
             add_fatroot_exe(manifest.kernel)
             add_fatroot_exe(manifest.init)
-            for program in manifest.panics: add_fatroot_exe(program)
+            for program in manifest.panics:
-            for program in manifest.services: add_initrd_exe(program)
+                add_fatroot_exe(program)
-            for program in manifest.drivers: add_initrd_exe(program)
+
+            for program in manifest.services:
+                add_initrd_stripped("jsix/services", program)
+
+            for program in manifest.drivers:
+                add_initrd_stripped("jsix/drivers", program)
 
             syms = manifest.add_data("symbol_table.dat",
                     "Symbol table", ("symbols",))
@@ -136,9 +161,9 @@ class Project:
             build.build(
                 rule = "makest",
                 outputs = [sym_out],
-                inputs = [f"${{build_root}}/{modules['kernel'].output}"],
+                inputs = [f"${{build_root}}/kernel/{modules['kernel'].output}"],
                 )
-            initrd_content.append(sym_out)
+            add_initrd_content("jsix/data", "symbol_table.dat")
 
             bootloader = "${build_root}/fatroot/efi/boot/bootx64.efi"
             build.build(
@@ -146,23 +171,23 @@ class Project:
                 outputs = [bootloader],
                 inputs = ["${build_root}/boot/boot.efi"],
                 variables = {
-                    "name": "Installing bootloader",
+                    "description": "Installing bootloader",
                 })
             build.newline()
 
-            boot_config = str(fatroot / "jsix_boot.dat")
+            boot_config = join(fatroot, "jsix", "boot.conf")
-            init_config = str(output / "init.manifest")
             manifest.write_boot_config(boot_config)
-            manifest.write_init_config(init_config, modules)
-            initrd_content.append(init_config)
 
-            initrd = join(fatroot, manifest.location, "initrd.dat")
+            initrd = str(fatroot / manifest.location / manifest.initrd["name"])
             build.build(
-                rule = "mkinitrd",
+                rule = "makeinitrd",
                 outputs = [initrd],
-                inputs = initrd_content,
+                inputs = [str(initrdroot)],
+                implicit = initrd_content + ["${source_root}/scripts/mkj6romfs.py"],
+                variables = {"format": manifest.initrd["format"]},
             )
             build.newline()
+
             fatroot_content.append(initrd)
 
             build.build(

scripts/fnv.py

@@ -0,0 +1,17 @@
+"""Python implementation of FNV hashes."""
+
+_FNV1_prime32 = 16777619
+_FNV1_prime64 = 1099511628211
+_FNV1_offset32 = 2166136261
+_FNV1_offset64 = 14695981039346656037
+
+def _fnv1a(offset, prime, mask):
+    def hashfunc(data):
+        h = offset
+        for b in data:
+            h = ((h ^ b) * prime) & mask
+        return h
+    return hashfunc
+
+hash64 = _fnv1a(_FNV1_offset64, _FNV1_prime64, (2**64)-1)
+hash32 = _fnv1a(_FNV1_offset32, _FNV1_prime32, (2**32)-1)

scripts/mkinitrd.py

@@ -1,29 +0,0 @@
-#!/usr/bin/env python3
-
-compress_level = 19
-
-def write_image(image, files):
-    from pathlib import Path
-    from cdblib import Writer, djb_hash
-    from pyzstd import compress
-
-    with open(image, 'wb') as db:
-        with Writer(db) as writer:
-            for f in files:
-                key = Path(f).name.encode('utf-8')
-                with open(f, 'rb') as input_file:
-                    writer.put(key, compress(input_file.read(), compress_level))
-
-if __name__ == "__main__":
-    from argparse import ArgumentParser
-
-    p = ArgumentParser(description="Generate a jsix initrd image")
-
-    p.add_argument("image", metavar="INITRD", 
-            help="initrd image file to generate")
-
-    p.add_argument("files", metavar="FILE", nargs='+',
-            help="files to add to the image")
-
-    args = p.parse_args()
-    write_image(args.image, args.files)

scripts/mkj6romfs.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+#
+# Generate a j6romfs image from a given directory.
+
+from struct import calcsize, pack
+from fnv import hash64
+
+inode_type_dir = 1
+inode_type_file = 2
+
+uncompressed_limit = 0xffffffff
+compressed_limit = 0xffffff
+
+hasher = hash64
+
+class fs_exception(Exception): pass
+
+def compress_zstd(data):
+    from pyzstd import compress
+    return compress(data, 19)
+
+def compress_none(data):
+    return data
+
+compressors = {
+    'none': (0, compress_none),
+    'zstd': (1, compress_zstd),
+}
+
+def align(stream, size):
+    offset = stream.tell()
+    extra = offset % size
+    if extra != 0:
+        stream.seek(size-extra, 1)
+        offset = stream.tell()
+    return offset
+
+def add_file(path, inode_data, output, compress):
+    offset = output.tell()
+    comp_size = 0
+    uncomp_size = 0
+
+    with open(path, 'rb') as file:
+        uncompressed = file.read()
+        uncomp_size = len(uncompressed)
+        compressed = compress(uncompressed)
+        comp_size = len(compressed)
+
+        # Don't use more room for compression than
+        # the original file
+        if comp_size >= uncomp_size:
+            compressed = uncompressed
+            comp_size = uncomp_size
+
+        output.write(compressed)
+
+        if uncomp_size > uncompressed_limit:
+            raise fs_exception(f"File {path} too large: {uncomp_size} bytes.")
+
+        if comp_size > compressed_limit:
+            raise fs_exception(f"File {path} too large when compressed: {comp_size} bytes.")
+
+    inode_data.append((inode_type_file, offset, comp_size, uncomp_size))
+
+
+dirent_format = "<IHBBQ"
+dirent_size = calcsize(dirent_format)
+
+def make_dirent(inode, inode_type, name, strings, strings_offset):
+    strings_offset += len(strings)
+    name_utf = name.encode('utf-8')
+    name_hash = hasher(name_utf)
+    strings.extend(name_utf)
+    strings.append(0)
+
+    return pack(dirent_format, inode, strings_offset, inode_type, len(name_utf) + 1, name_hash)
+
+
+def add_dir(path, files, dirs, inode_data, dir_inodes, output, compress):
+    strings = bytearray()
+    uncompressed = bytearray()
+    offset = align(output, 0x10)
+    comp_size = 0
+    uncomp_size = 0
+
+    strings_offset = (len(dirs) + len(files)) * dirent_size
+
+    for name, realpath in dirs:
+        inode = dir_inodes[realpath]
+        uncompressed.extend(make_dirent(inode, inode_type_dir, name, strings, strings_offset))
+
+    for name, inode in files:
+        uncompressed.extend(make_dirent(inode, inode_type_file, name, strings, strings_offset))
+
+    uncompressed.extend(strings)
+
+    uncomp_size = len(uncompressed)
+    compressed = compress(uncompressed)
+    comp_size = len(compressed)
+
+    # Don't use more room for compression than
+    # the original file
+    if comp_size >= uncomp_size:
+        compressed = uncompressed
+        comp_size = uncomp_size
+
+    output.write(uncompressed)
+    inode_data.append((inode_type_dir, offset, comp_size, uncomp_size))
+
+
+def make_image(root, image, compressor):
+    import os
+    from os.path import dirname, join
+
+    compressor_id, compress = compressor
+
+    directories = []
+    inode_data = []
+
+    with open(image, 'wb') as output:
+        def write_header(inode_offset, inode_count, root_inode):
+            output.seek(0, 0)
+            output.write(pack("<8s Q II B7x",
+                b"j6romfs1", inode_offset, inode_count, root_inode, compressor_id))
+
+        write_header(0, 0, 0)
+
+        for (dirpath, dirs, files) in os.walk(root, topdown=False):
+            #print(f"{dirpath}:\n\t{dirs}\n\t{files}")
+
+            dir_inodes = []
+            for file in files:
+                dir_inodes.append((file, len(inode_data)))
+                add_file(join(dirpath, file), inode_data, output, compress)
+
+            parent = dirpath
+            if dirpath != root:
+                parent = dirname(dirpath)
+
+            dir_directories = [('.', dirpath), ('..', parent)]
+            dir_directories += [(d, join(dirpath, d)) for d in dirs]
+
+            directories.append((dirpath, dir_inodes, dir_directories))
+
+        dir_inodes = {directories[i][0]: len(inode_data) + i for i in range(len(directories))}
+        for d in directories:
+            add_dir(*d, inode_data, dir_inodes, output, compress)
+
+        inode_offset = align(output, 0x10) # align to a 16 byte value
+
+        for inode_type, offset, comp_size, uncomp_size in inode_data:
+            comp_size_type = (comp_size & 0xffffff) | (inode_type << 24)
+            output.write(pack("<IIQ", uncomp_size, comp_size_type, offset))
+
+        write_header(inode_offset, len(inode_data), len(inode_data) - 1)
+
+
+if __name__ == "__main__":
+    import sys
+    from argparse import ArgumentParser
+
+    p = ArgumentParser(description="Generate a j6romfs image from a directory")
+
+    p.add_argument("--compressor", "-c", metavar="NAME", default="zstd",
+            help="Which compressor to use (currently: 'none' or 'zstd')")
+
+    p.add_argument("--verbose", "-v", action="store_true",
+            help="Output more information on progress")
+
+    p.add_argument("input", metavar="DIR",
+            help="The source directory to use as the root of the image")
+
+    p.add_argument("output", metavar="FILE",
+            help="The output file for the image")
+
+    args = p.parse_args()
+
+    import logging
+    log_level = logging.ERROR
+    if args.verbose:
+        log_level = logging.INFO
+
+    logging.basicConfig(
+        format="%(levelname)s: %(message)s",
+        level=log_level,
+    )
+
+    compressor = compressors.get(args.compressor)
+    if compressor is None:
+        logging.error(f"No such compressor: {args.compressor}")
+        sys.exit(1)
+
+    try:
+        make_image(args.input, args.output, compressor)
+    except fs_exception as fse:
+        print(fse, file=sys.stderr)
+        sys.exit(1)

src/boot/bootconfig.cpp

@@ -37,9 +37,7 @@ bootconfig::bootconfig(util::buffer data, uefi::boot_services *bs)
     if (version != 1)
         error::raise(uefi::status::incompatible_version, L"Bad version in jsix_boot.dat");
 
-    data += 1; // reserved byte
+    uint8_t num_panics = *util::read<uint8_t>(data);
-    uint16_t num_panics = *util::read<uint16_t>(data);
-    m_initrd_format = *util::read<uint16_t>(data);
     m_flags = *util::read<uint16_t>(data);
 
     read_descriptor(m_kernel, data);

src/boot/bootconfig.h

@@ -31,12 +31,10 @@ public:
     inline const descriptor & kernel() const { return m_kernel; }
     inline const descriptor & init() const { return m_init; }
     inline const wchar_t * initrd() const { return m_initrd; }
-    inline uint16_t initrd_format() const { return m_initrd_format; }
     inline const descriptors & panics() { return m_panics; }
 
 private:
     uint16_t m_flags;
-    uint16_t m_initrd_format;
     descriptor m_kernel;
     descriptor m_init;
     descriptors m_panics;

src/boot/main.cpp

@@ -72,7 +72,7 @@ load_resources(bootproto::args *args, video::screen *screen, uefi::handle image,
     status_line status {L"Loading programs"};
 
     fs::file disk = fs::get_boot_volume(image, bs);
-    fs::file bc_data = disk.open(L"jsix_boot.dat");
+    fs::file bc_data = disk.open(L"jsix\\boot.conf");
     bootconfig bc {bc_data.load(), bs};
 
     args->kernel = loader::load_program(disk, L"kernel", bc.kernel());
@@ -80,7 +80,7 @@ load_resources(bootproto::args *args, video::screen *screen, uefi::handle image,
     args->flags = static_cast<bootproto::boot_flags>(bc.flags());
 
     loader::load_module(disk, L"initrd", bc.initrd(),
-            bootproto::module_type::initrd, bc.initrd_format());
+            bootproto::module_type::initrd, 0);
 
     namespace bits = util::bits;
     using bootproto::desc_flags;