From 6f7dd7fc05f8fcea2c9a63875dacb163b82b3bb4 Mon Sep 17 00:00:00 2001 From: "Justin C. Miller" Date: Sun, 29 Jan 2023 19:10:12 -0800 Subject: [PATCH] [boot] More initrd format changes CDB seemed to be too simple for the needs of init, and squashfs is too laden with design choices to work around Linux's APIs. This commit adds creation of an initrd image of a new format I've called `j6romfs`. Note that this commit currently does not work! The initrd-reading code still needs to be added. --- configs/rules.ninja | 8 +- scripts/bonnibel/manifest.py | 23 ++-- scripts/bonnibel/project.py | 71 +++++++++---- scripts/fnv.py | 17 +++ scripts/mkinitrd.py | 29 ------ scripts/mkj6romfs.py | 197 +++++++++++++++++++++++++++++++++++ src/boot/bootconfig.cpp | 4 +- src/boot/bootconfig.h | 2 - src/boot/main.cpp | 4 +- 9 files changed, 276 insertions(+), 79 deletions(-) create mode 100644 scripts/fnv.py delete mode 100755 scripts/mkinitrd.py create mode 100755 scripts/mkj6romfs.py diff --git a/configs/rules.ninja b/configs/rules.ninja index 0b0d661..ad32296 100644 --- a/configs/rules.ninja +++ b/configs/rules.ninja @@ -60,16 +60,16 @@ rule makest description = Making symbol table command = nm -n -S --demangle $in | ${source_root}/scripts/build_symbol_table.py $out +rule makeinitrd + description = Creating $name + command = ${source_root}/scripts/mkj6romfs.py -c $format $in $out + rule makefat description = Creating $name command = $ cp $in $out; $ mcopy -s -D o -i $out@@1M ${build_root}/fatroot/* ::/ -rule mkinitrd - description = Creating initrd - command = ${source_root}/scripts/mkinitrd.py $out $in - rule strip description = Stripping $name command = $ diff --git a/scripts/bonnibel/manifest.py b/scripts/bonnibel/manifest.py index 5cb4704..3cc44ad 100644 --- a/scripts/bonnibel/manifest.py +++ b/scripts/bonnibel/manifest.py @@ -4,11 +4,6 @@ class Manifest: from collections import namedtuple Entry = namedtuple("Entry", ("module", "target", "output", "flags")) - formats = { - "none": 0x00, - "zstd": 0x01, - } - flags = { "graphical": 0x01, "symbols": 0x80, @@ -45,11 +40,10 @@ class Manifest: self.flags = config.get("flags", tuple()) initrd = config.get("initrd", dict()) - self.initrd = initrd.get("name", "initrd.dat") - self.initrd_format = initrd.get("format", "none") - - if not self.initrd_format in Manifest.formats: - raise BonnibelError(f"Manifest specifies unknown initrd format '{self.initrd_format}'") + self.initrd = { + "name": initrd.get("name", "initrd.dat"), + "format": initrd.get("format", "zstd"), + } self.data = [] for d in config.get("data", tuple()): @@ -89,15 +83,12 @@ class Manifest: with open(path, 'wb') as outfile: magic = "jsixboot".encode("utf-8") # magic string version = 1 - reserved = 0 - initrd_format = Manifest.formats.get(self.initrd_format, 0) bootflags = sum([Manifest.boot_flags.get(s, 0) for s in self.flags]) - outfile.write(struct.pack("<8s BBH HH", + outfile.write(struct.pack("<8s BBH", magic, - version, reserved, len(self.panics), - initrd_format, bootflags)) + version, len(self.panics), bootflags)) def write_str(s): data = s.encode("utf-16le") @@ -118,7 +109,7 @@ class Manifest: write_ent(self.kernel) write_ent(self.init) - write_path(self.initrd) + write_path(self.initrd["name"]) for p in self.panics: write_ent(p) diff --git a/scripts/bonnibel/project.py b/scripts/bonnibel/project.py index 2b6ec39..685e0f8 100644 --- a/scripts/bonnibel/project.py +++ b/scripts/bonnibel/project.py @@ -64,20 +64,25 @@ class Project: outputs = ["all-headers"], inputs = ["${build_root}/.all_headers"]) + from .manifest import Manifest + manifest = Manifest(manifest_file, modules) + debugroot = output / ".debug" debugroot.mkdir(exist_ok=True) fatroot = output / "fatroot" fatroot.mkdir(exist_ok=True) + (fatroot / manifest.location).mkdir(exist_ok=True) + + initrdroot = output / "initrd_root" + initrdroot.mkdir(exist_ok=True) + fatroot_content = [] initrd_content = [] - from .manifest import Manifest - manifest = Manifest(manifest_file, modules) - - def add_fatroot(source, entry): - output = join(manifest.location, entry.output) + def add_fatroot(source, name): + output = join(manifest.location, name) fatroot_output = f"${{build_root}}/fatroot/{output}" build.build( @@ -85,7 +90,7 @@ class Project: outputs = [fatroot_output], inputs = [source], variables = { - "name": f"Installing {output}", + "description": f"Installing {output}", }) fatroot_content.append(fatroot_output) @@ -105,9 +110,24 @@ class Project: "debug": f"${{build_root}}/.debug/{entry.output}.debug", }) - add_fatroot(intermediary, entry) + add_fatroot(intermediary, entry.output) - def add_initrd_exe(entry): + def add_initrd_content(root, name): + output = join(root, name) + initrd_output = f"${{build_root}}/initrd_root/{output}" + + build.build( + rule = "cp", + outputs = [initrd_output], + inputs = [f"${{build_root}}/{name}"], + variables = { + "description": f"Installing {name}", + }) + + initrd_content.append(initrd_output) + build.newline() + + def add_initrd_stripped(root, entry): input_path = f"${{build_root}}/{entry.target}/{entry.output}" intermediary = f"${{build_root}}/{entry.output}" @@ -121,13 +141,18 @@ class Project: "debug": f"${{build_root}}/.debug/{entry.output}.debug", }) - initrd_content.append(intermediary) + add_initrd_content(root, entry.output) add_fatroot_exe(manifest.kernel) add_fatroot_exe(manifest.init) - for program in manifest.panics: add_fatroot_exe(program) - for program in manifest.services: add_initrd_exe(program) - for program in manifest.drivers: add_initrd_exe(program) + for program in manifest.panics: + add_fatroot_exe(program) + + for program in manifest.services: + add_initrd_stripped("jsix/services", program) + + for program in manifest.drivers: + add_initrd_stripped("jsix/drivers", program) syms = manifest.add_data("symbol_table.dat", "Symbol table", ("symbols",)) @@ -136,9 +161,9 @@ class Project: build.build( rule = "makest", outputs = [sym_out], - inputs = [f"${{build_root}}/{modules['kernel'].output}"], + inputs = [f"${{build_root}}/kernel/{modules['kernel'].output}"], ) - initrd_content.append(sym_out) + add_initrd_content("jsix/data", "symbol_table.dat") bootloader = "${build_root}/fatroot/efi/boot/bootx64.efi" build.build( @@ -146,23 +171,23 @@ class Project: outputs = [bootloader], inputs = ["${build_root}/boot/boot.efi"], variables = { - "name": "Installing bootloader", + "description": "Installing bootloader", }) build.newline() - boot_config = str(fatroot / "jsix_boot.dat") - init_config = str(output / "init.manifest") + boot_config = join(fatroot, "jsix", "boot.conf") manifest.write_boot_config(boot_config) - manifest.write_init_config(init_config, modules) - initrd_content.append(init_config) - initrd = join(fatroot, manifest.location, "initrd.dat") + initrd = str(fatroot / manifest.location / manifest.initrd["name"]) build.build( - rule = "mkinitrd", + rule = "makeinitrd", outputs = [initrd], - inputs = initrd_content, - ) + inputs = [str(initrdroot)], + implicit = initrd_content + ["${source_root}/scripts/mkj6romfs.py"], + variables = {"format": manifest.initrd["format"]}, + ) build.newline() + fatroot_content.append(initrd) build.build( diff --git a/scripts/fnv.py b/scripts/fnv.py new file mode 100644 index 0000000..3b39d4c --- /dev/null +++ b/scripts/fnv.py @@ -0,0 +1,17 @@ +"""Python implementation of FNV hashes.""" + +_FNV1_prime32 = 16777619 +_FNV1_prime64 = 1099511628211 +_FNV1_offset32 = 2166136261 +_FNV1_offset64 = 14695981039346656037 + +def _fnv1a(offset, prime, mask): + def hashfunc(data): + h = offset + for b in data: + h = ((h ^ b) * prime) & mask + return h + return hashfunc + +hash64 = _fnv1a(_FNV1_offset64, _FNV1_prime64, (2**64)-1) +hash32 = _fnv1a(_FNV1_offset32, _FNV1_prime32, (2**32)-1) diff --git a/scripts/mkinitrd.py b/scripts/mkinitrd.py deleted file mode 100755 index 07672c8..0000000 --- a/scripts/mkinitrd.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 - -compress_level = 19 - -def write_image(image, files): - from pathlib import Path - from cdblib import Writer, djb_hash - from pyzstd import compress - - with open(image, 'wb') as db: - with Writer(db) as writer: - for f in files: - key = Path(f).name.encode('utf-8') - with open(f, 'rb') as input_file: - writer.put(key, compress(input_file.read(), compress_level)) - -if __name__ == "__main__": - from argparse import ArgumentParser - - p = ArgumentParser(description="Generate a jsix initrd image") - - p.add_argument("image", metavar="INITRD", - help="initrd image file to generate") - - p.add_argument("files", metavar="FILE", nargs='+', - help="files to add to the image") - - args = p.parse_args() - write_image(args.image, args.files) diff --git a/scripts/mkj6romfs.py b/scripts/mkj6romfs.py new file mode 100755 index 0000000..94b6f88 --- /dev/null +++ b/scripts/mkj6romfs.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +# +# Generate a j6romfs image from a given directory. + +from struct import calcsize, pack +from fnv import hash64 + +inode_type_dir = 1 +inode_type_file = 2 + +uncompressed_limit = 0xffffffff +compressed_limit = 0xffffff + +hasher = hash64 + +class fs_exception(Exception): pass + +def compress_zstd(data): + from pyzstd import compress + return compress(data, 19) + +def compress_none(data): + return data + +compressors = { + 'none': (0, compress_none), + 'zstd': (1, compress_zstd), +} + +def align(stream, size): + offset = stream.tell() + extra = offset % size + if extra != 0: + stream.seek(size-extra, 1) + offset = stream.tell() + return offset + +def add_file(path, inode_data, output, compress): + offset = output.tell() + comp_size = 0 + uncomp_size = 0 + + with open(path, 'rb') as file: + uncompressed = file.read() + uncomp_size = len(uncompressed) + compressed = compress(uncompressed) + comp_size = len(compressed) + + # Don't use more room for compression than + # the original file + if comp_size >= uncomp_size: + compressed = uncompressed + comp_size = uncomp_size + + output.write(compressed) + + if uncomp_size > uncompressed_limit: + raise fs_exception(f"File {path} too large: {uncomp_size} bytes.") + + if comp_size > compressed_limit: + raise fs_exception(f"File {path} too large when compressed: {comp_size} bytes.") + + inode_data.append((inode_type_file, offset, comp_size, uncomp_size)) + + +dirent_format = "= uncomp_size: + compressed = uncompressed + comp_size = uncomp_size + + output.write(uncompressed) + inode_data.append((inode_type_dir, offset, comp_size, uncomp_size)) + + +def make_image(root, image, compressor): + import os + from os.path import dirname, join + + compressor_id, compress = compressor + + directories = [] + inode_data = [] + + with open(image, 'wb') as output: + def write_header(inode_offset, inode_count, root_inode): + output.seek(0, 0) + output.write(pack("<8s Q II B7x", + b"j6romfs1", inode_offset, inode_count, root_inode, compressor_id)) + + write_header(0, 0, 0) + + for (dirpath, dirs, files) in os.walk(root, topdown=False): + #print(f"{dirpath}:\n\t{dirs}\n\t{files}") + + dir_inodes = [] + for file in files: + dir_inodes.append((file, len(inode_data))) + add_file(join(dirpath, file), inode_data, output, compress) + + parent = dirpath + if dirpath != root: + parent = dirname(dirpath) + + dir_directories = [('.', dirpath), ('..', parent)] + dir_directories += [(d, join(dirpath, d)) for d in dirs] + + directories.append((dirpath, dir_inodes, dir_directories)) + + dir_inodes = {directories[i][0]: len(inode_data) + i for i in range(len(directories))} + for d in directories: + add_dir(*d, inode_data, dir_inodes, output, compress) + + inode_offset = align(output, 0x10) # align to a 16 byte value + + for inode_type, offset, comp_size, uncomp_size in inode_data: + comp_size_type = (comp_size & 0xffffff) | (inode_type << 24) + output.write(pack("(data); - m_initrd_format = *util::read(data); + uint8_t num_panics = *util::read(data); m_flags = *util::read(data); read_descriptor(m_kernel, data); diff --git a/src/boot/bootconfig.h b/src/boot/bootconfig.h index 66e6921..cb1d52f 100644 --- a/src/boot/bootconfig.h +++ b/src/boot/bootconfig.h @@ -31,12 +31,10 @@ public: inline const descriptor & kernel() const { return m_kernel; } inline const descriptor & init() const { return m_init; } inline const wchar_t * initrd() const { return m_initrd; } - inline uint16_t initrd_format() const { return m_initrd_format; } inline const descriptors & panics() { return m_panics; } private: uint16_t m_flags; - uint16_t m_initrd_format; descriptor m_kernel; descriptor m_init; descriptors m_panics; diff --git a/src/boot/main.cpp b/src/boot/main.cpp index fb418c2..8dd03d9 100644 --- a/src/boot/main.cpp +++ b/src/boot/main.cpp @@ -72,7 +72,7 @@ load_resources(bootproto::args *args, video::screen *screen, uefi::handle image, status_line status {L"Loading programs"}; fs::file disk = fs::get_boot_volume(image, bs); - fs::file bc_data = disk.open(L"jsix_boot.dat"); + fs::file bc_data = disk.open(L"jsix\\boot.conf"); bootconfig bc {bc_data.load(), bs}; args->kernel = loader::load_program(disk, L"kernel", bc.kernel()); @@ -80,7 +80,7 @@ load_resources(bootproto::args *args, video::screen *screen, uefi::handle image, args->flags = static_cast(bc.flags()); loader::load_module(disk, L"initrd", bc.initrd(), - bootproto::module_type::initrd, bc.initrd_format()); + bootproto::module_type::initrd, 0); namespace bits = util::bits; using bootproto::desc_flags;