diff --git a/configs/rules.ninja b/configs/rules.ninja index 0b0d661..ad32296 100644 --- a/configs/rules.ninja +++ b/configs/rules.ninja @@ -60,16 +60,16 @@ rule makest description = Making symbol table command = nm -n -S --demangle $in | ${source_root}/scripts/build_symbol_table.py $out +rule makeinitrd + description = Creating $name + command = ${source_root}/scripts/mkj6romfs.py -c $format $in $out + rule makefat description = Creating $name command = $ cp $in $out; $ mcopy -s -D o -i $out@@1M ${build_root}/fatroot/* ::/ -rule mkinitrd - description = Creating initrd - command = ${source_root}/scripts/mkinitrd.py $out $in - rule strip description = Stripping $name command = $ diff --git a/scripts/bonnibel/manifest.py b/scripts/bonnibel/manifest.py index 5cb4704..3cc44ad 100644 --- a/scripts/bonnibel/manifest.py +++ b/scripts/bonnibel/manifest.py @@ -4,11 +4,6 @@ class Manifest: from collections import namedtuple Entry = namedtuple("Entry", ("module", "target", "output", "flags")) - formats = { - "none": 0x00, - "zstd": 0x01, - } - flags = { "graphical": 0x01, "symbols": 0x80, @@ -45,11 +40,10 @@ class Manifest: self.flags = config.get("flags", tuple()) initrd = config.get("initrd", dict()) - self.initrd = initrd.get("name", "initrd.dat") - self.initrd_format = initrd.get("format", "none") - - if not self.initrd_format in Manifest.formats: - raise BonnibelError(f"Manifest specifies unknown initrd format '{self.initrd_format}'") + self.initrd = { + "name": initrd.get("name", "initrd.dat"), + "format": initrd.get("format", "zstd"), + } self.data = [] for d in config.get("data", tuple()): @@ -89,15 +83,12 @@ class Manifest: with open(path, 'wb') as outfile: magic = "jsixboot".encode("utf-8") # magic string version = 1 - reserved = 0 - initrd_format = Manifest.formats.get(self.initrd_format, 0) bootflags = sum([Manifest.boot_flags.get(s, 0) for s in self.flags]) - outfile.write(struct.pack("<8s BBH HH", + outfile.write(struct.pack("<8s BBH", magic, - version, reserved, len(self.panics), - initrd_format, bootflags)) + version, len(self.panics), bootflags)) def write_str(s): data = s.encode("utf-16le") @@ -118,7 +109,7 @@ class Manifest: write_ent(self.kernel) write_ent(self.init) - write_path(self.initrd) + write_path(self.initrd["name"]) for p in self.panics: write_ent(p) diff --git a/scripts/bonnibel/project.py b/scripts/bonnibel/project.py index 2b6ec39..685e0f8 100644 --- a/scripts/bonnibel/project.py +++ b/scripts/bonnibel/project.py @@ -64,20 +64,25 @@ class Project: outputs = ["all-headers"], inputs = ["${build_root}/.all_headers"]) + from .manifest import Manifest + manifest = Manifest(manifest_file, modules) + debugroot = output / ".debug" debugroot.mkdir(exist_ok=True) fatroot = output / "fatroot" fatroot.mkdir(exist_ok=True) + (fatroot / manifest.location).mkdir(exist_ok=True) + + initrdroot = output / "initrd_root" + initrdroot.mkdir(exist_ok=True) + fatroot_content = [] initrd_content = [] - from .manifest import Manifest - manifest = Manifest(manifest_file, modules) - - def add_fatroot(source, entry): - output = join(manifest.location, entry.output) + def add_fatroot(source, name): + output = join(manifest.location, name) fatroot_output = f"${{build_root}}/fatroot/{output}" build.build( @@ -85,7 +90,7 @@ class Project: outputs = [fatroot_output], inputs = [source], variables = { - "name": f"Installing {output}", + "description": f"Installing {output}", }) fatroot_content.append(fatroot_output) @@ -105,9 +110,24 @@ class Project: "debug": f"${{build_root}}/.debug/{entry.output}.debug", }) - add_fatroot(intermediary, entry) + add_fatroot(intermediary, entry.output) - def add_initrd_exe(entry): + def add_initrd_content(root, name): + output = join(root, name) + initrd_output = f"${{build_root}}/initrd_root/{output}" + + build.build( + rule = "cp", + outputs = [initrd_output], + inputs = [f"${{build_root}}/{name}"], + variables = { + "description": f"Installing {name}", + }) + + initrd_content.append(initrd_output) + build.newline() + + def add_initrd_stripped(root, entry): input_path = f"${{build_root}}/{entry.target}/{entry.output}" intermediary = f"${{build_root}}/{entry.output}" @@ -121,13 +141,18 @@ class Project: "debug": f"${{build_root}}/.debug/{entry.output}.debug", }) - initrd_content.append(intermediary) + add_initrd_content(root, entry.output) add_fatroot_exe(manifest.kernel) add_fatroot_exe(manifest.init) - for program in manifest.panics: add_fatroot_exe(program) - for program in manifest.services: add_initrd_exe(program) - for program in manifest.drivers: add_initrd_exe(program) + for program in manifest.panics: + add_fatroot_exe(program) + + for program in manifest.services: + add_initrd_stripped("jsix/services", program) + + for program in manifest.drivers: + add_initrd_stripped("jsix/drivers", program) syms = manifest.add_data("symbol_table.dat", "Symbol table", ("symbols",)) @@ -136,9 +161,9 @@ class Project: build.build( rule = "makest", outputs = [sym_out], - inputs = [f"${{build_root}}/{modules['kernel'].output}"], + inputs = [f"${{build_root}}/kernel/{modules['kernel'].output}"], ) - initrd_content.append(sym_out) + add_initrd_content("jsix/data", "symbol_table.dat") bootloader = "${build_root}/fatroot/efi/boot/bootx64.efi" build.build( @@ -146,23 +171,23 @@ class Project: outputs = [bootloader], inputs = ["${build_root}/boot/boot.efi"], variables = { - "name": "Installing bootloader", + "description": "Installing bootloader", }) build.newline() - boot_config = str(fatroot / "jsix_boot.dat") - init_config = str(output / "init.manifest") + boot_config = join(fatroot, "jsix", "boot.conf") manifest.write_boot_config(boot_config) - manifest.write_init_config(init_config, modules) - initrd_content.append(init_config) - initrd = join(fatroot, manifest.location, "initrd.dat") + initrd = str(fatroot / manifest.location / manifest.initrd["name"]) build.build( - rule = "mkinitrd", + rule = "makeinitrd", outputs = [initrd], - inputs = initrd_content, - ) + inputs = [str(initrdroot)], + implicit = initrd_content + ["${source_root}/scripts/mkj6romfs.py"], + variables = {"format": manifest.initrd["format"]}, + ) build.newline() + fatroot_content.append(initrd) build.build( diff --git a/scripts/fnv.py b/scripts/fnv.py new file mode 100644 index 0000000..3b39d4c --- /dev/null +++ b/scripts/fnv.py @@ -0,0 +1,17 @@ +"""Python implementation of FNV hashes.""" + +_FNV1_prime32 = 16777619 +_FNV1_prime64 = 1099511628211 +_FNV1_offset32 = 2166136261 +_FNV1_offset64 = 14695981039346656037 + +def _fnv1a(offset, prime, mask): + def hashfunc(data): + h = offset + for b in data: + h = ((h ^ b) * prime) & mask + return h + return hashfunc + +hash64 = _fnv1a(_FNV1_offset64, _FNV1_prime64, (2**64)-1) +hash32 = _fnv1a(_FNV1_offset32, _FNV1_prime32, (2**32)-1) diff --git a/scripts/mkinitrd.py b/scripts/mkinitrd.py deleted file mode 100755 index 07672c8..0000000 --- a/scripts/mkinitrd.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 - -compress_level = 19 - -def write_image(image, files): - from pathlib import Path - from cdblib import Writer, djb_hash - from pyzstd import compress - - with open(image, 'wb') as db: - with Writer(db) as writer: - for f in files: - key = Path(f).name.encode('utf-8') - with open(f, 'rb') as input_file: - writer.put(key, compress(input_file.read(), compress_level)) - -if __name__ == "__main__": - from argparse import ArgumentParser - - p = ArgumentParser(description="Generate a jsix initrd image") - - p.add_argument("image", metavar="INITRD", - help="initrd image file to generate") - - p.add_argument("files", metavar="FILE", nargs='+', - help="files to add to the image") - - args = p.parse_args() - write_image(args.image, args.files) diff --git a/scripts/mkj6romfs.py b/scripts/mkj6romfs.py new file mode 100755 index 0000000..94b6f88 --- /dev/null +++ b/scripts/mkj6romfs.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +# +# Generate a j6romfs image from a given directory. + +from struct import calcsize, pack +from fnv import hash64 + +inode_type_dir = 1 +inode_type_file = 2 + +uncompressed_limit = 0xffffffff +compressed_limit = 0xffffff + +hasher = hash64 + +class fs_exception(Exception): pass + +def compress_zstd(data): + from pyzstd import compress + return compress(data, 19) + +def compress_none(data): + return data + +compressors = { + 'none': (0, compress_none), + 'zstd': (1, compress_zstd), +} + +def align(stream, size): + offset = stream.tell() + extra = offset % size + if extra != 0: + stream.seek(size-extra, 1) + offset = stream.tell() + return offset + +def add_file(path, inode_data, output, compress): + offset = output.tell() + comp_size = 0 + uncomp_size = 0 + + with open(path, 'rb') as file: + uncompressed = file.read() + uncomp_size = len(uncompressed) + compressed = compress(uncompressed) + comp_size = len(compressed) + + # Don't use more room for compression than + # the original file + if comp_size >= uncomp_size: + compressed = uncompressed + comp_size = uncomp_size + + output.write(compressed) + + if uncomp_size > uncompressed_limit: + raise fs_exception(f"File {path} too large: {uncomp_size} bytes.") + + if comp_size > compressed_limit: + raise fs_exception(f"File {path} too large when compressed: {comp_size} bytes.") + + inode_data.append((inode_type_file, offset, comp_size, uncomp_size)) + + +dirent_format = "= uncomp_size: + compressed = uncompressed + comp_size = uncomp_size + + output.write(uncompressed) + inode_data.append((inode_type_dir, offset, comp_size, uncomp_size)) + + +def make_image(root, image, compressor): + import os + from os.path import dirname, join + + compressor_id, compress = compressor + + directories = [] + inode_data = [] + + with open(image, 'wb') as output: + def write_header(inode_offset, inode_count, root_inode): + output.seek(0, 0) + output.write(pack("<8s Q II B7x", + b"j6romfs1", inode_offset, inode_count, root_inode, compressor_id)) + + write_header(0, 0, 0) + + for (dirpath, dirs, files) in os.walk(root, topdown=False): + #print(f"{dirpath}:\n\t{dirs}\n\t{files}") + + dir_inodes = [] + for file in files: + dir_inodes.append((file, len(inode_data))) + add_file(join(dirpath, file), inode_data, output, compress) + + parent = dirpath + if dirpath != root: + parent = dirname(dirpath) + + dir_directories = [('.', dirpath), ('..', parent)] + dir_directories += [(d, join(dirpath, d)) for d in dirs] + + directories.append((dirpath, dir_inodes, dir_directories)) + + dir_inodes = {directories[i][0]: len(inode_data) + i for i in range(len(directories))} + for d in directories: + add_dir(*d, inode_data, dir_inodes, output, compress) + + inode_offset = align(output, 0x10) # align to a 16 byte value + + for inode_type, offset, comp_size, uncomp_size in inode_data: + comp_size_type = (comp_size & 0xffffff) | (inode_type << 24) + output.write(pack("(data); - m_initrd_format = *util::read(data); + uint8_t num_panics = *util::read(data); m_flags = *util::read(data); read_descriptor(m_kernel, data); diff --git a/src/boot/bootconfig.h b/src/boot/bootconfig.h index 66e6921..cb1d52f 100644 --- a/src/boot/bootconfig.h +++ b/src/boot/bootconfig.h @@ -31,12 +31,10 @@ public: inline const descriptor & kernel() const { return m_kernel; } inline const descriptor & init() const { return m_init; } inline const wchar_t * initrd() const { return m_initrd; } - inline uint16_t initrd_format() const { return m_initrd_format; } inline const descriptors & panics() { return m_panics; } private: uint16_t m_flags; - uint16_t m_initrd_format; descriptor m_kernel; descriptor m_init; descriptors m_panics; diff --git a/src/boot/main.cpp b/src/boot/main.cpp index fb418c2..8dd03d9 100644 --- a/src/boot/main.cpp +++ b/src/boot/main.cpp @@ -72,7 +72,7 @@ load_resources(bootproto::args *args, video::screen *screen, uefi::handle image, status_line status {L"Loading programs"}; fs::file disk = fs::get_boot_volume(image, bs); - fs::file bc_data = disk.open(L"jsix_boot.dat"); + fs::file bc_data = disk.open(L"jsix\\boot.conf"); bootconfig bc {bc_data.load(), bs}; args->kernel = loader::load_program(disk, L"kernel", bc.kernel()); @@ -80,7 +80,7 @@ load_resources(bootproto::args *args, video::screen *screen, uefi::handle image, args->flags = static_cast(bc.flags()); loader::load_module(disk, L"initrd", bc.initrd(), - bootproto::module_type::initrd, bc.initrd_format()); + bootproto::module_type::initrd, 0); namespace bits = util::bits; using bootproto::desc_flags;