justin/jsix_import
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

[boot] More initrd format changes

Browse Source 
CDB seemed to be too simple for the needs of init, and squashfs is too
laden with design choices to work around Linux's APIs. This commit adds
creation of an initrd image of a new format I've called `j6romfs`.

Note that this commit currently does not work! The initrd-reading code
still needs to be added.
This commit is contained in:
Justin C. Miller
2023-01-29 19:10:12 -08:00
parent 1f15d2ef49
commit 6f7dd7fc05
9 changed files with 276 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
scripts/bonnibel/manifest.py
View File

@@ -4,11 +4,6 @@ class Manifest:
from collections import namedtuple
Entry = namedtuple("Entry", ("module", "target", "output", "flags"))
formats = {
"none": 0x00,
"zstd": 0x01,
}
flags = {
"graphical": 0x01,
"symbols": 0x80,
@@ -45,11 +40,10 @@ class Manifest:
self.flags = config.get("flags", tuple())
initrd = config.get("initrd", dict())
self.initrd = initrd.get("name", "initrd.dat")
self.initrd_format = initrd.get("format", "none")
if not self.initrd_format in Manifest.formats:
raise BonnibelError(f"Manifest specifies unknown initrd format '{self.initrd_format}'")
self.initrd = {
"name": initrd.get("name", "initrd.dat"),
"format": initrd.get("format", "zstd"),
}
self.data = []
for d in config.get("data", tuple()):
@@ -89,15 +83,12 @@ class Manifest:
with open(path, 'wb') as outfile:
magic = "jsixboot".encode("utf-8") # magic string
version = 1
reserved = 0
initrd_format = Manifest.formats.get(self.initrd_format, 0)
bootflags = sum([Manifest.boot_flags.get(s, 0) for s in self.flags])
outfile.write(struct.pack("<8s BBH HH",
outfile.write(struct.pack("<8s BBH",
magic,
version, reserved, len(self.panics),
initrd_format, bootflags))
version, len(self.panics), bootflags))
def write_str(s):
data = s.encode("utf-16le")
@@ -118,7 +109,7 @@ class Manifest:
write_ent(self.kernel)
write_ent(self.init)
write_path(self.initrd)
write_path(self.initrd["name"])
for p in self.panics:
write_ent(p)

71
scripts/bonnibel/project.py
View File

@@ -64,20 +64,25 @@ class Project:
outputs = ["all-headers"],
inputs = ["${build_root}/.all_headers"])
from .manifest import Manifest
manifest = Manifest(manifest_file, modules)
debugroot = output / ".debug"
debugroot.mkdir(exist_ok=True)
fatroot = output / "fatroot"
fatroot.mkdir(exist_ok=True)
(fatroot / manifest.location).mkdir(exist_ok=True)
initrdroot = output / "initrd_root"
initrdroot.mkdir(exist_ok=True)
fatroot_content = []
initrd_content = []
from .manifest import Manifest
manifest = Manifest(manifest_file, modules)
def add_fatroot(source, entry):
output = join(manifest.location, entry.output)
def add_fatroot(source, name):
output = join(manifest.location, name)
fatroot_output = f"${{build_root}}/fatroot/{output}"
build.build(
@@ -85,7 +90,7 @@ class Project:
outputs = [fatroot_output],
inputs = [source],
variables = {
"name": f"Installing {output}",
"description": f"Installing {output}",
})
fatroot_content.append(fatroot_output)
@@ -105,9 +110,24 @@ class Project:
"debug": f"${{build_root}}/.debug/{entry.output}.debug",
})
add_fatroot(intermediary, entry)
add_fatroot(intermediary, entry.output)
def add_initrd_exe(entry):
def add_initrd_content(root, name):
output = join(root, name)
initrd_output = f"${{build_root}}/initrd_root/{output}"
build.build(
rule = "cp",
outputs = [initrd_output],
inputs = [f"${{build_root}}/{name}"],
variables = {
"description": f"Installing {name}",
})
initrd_content.append(initrd_output)
build.newline()
def add_initrd_stripped(root, entry):
input_path = f"${{build_root}}/{entry.target}/{entry.output}"
intermediary = f"${{build_root}}/{entry.output}"
@@ -121,13 +141,18 @@ class Project:
"debug": f"${{build_root}}/.debug/{entry.output}.debug",
})
initrd_content.append(intermediary)
add_initrd_content(root, entry.output)
add_fatroot_exe(manifest.kernel)
add_fatroot_exe(manifest.init)
for program in manifest.panics: add_fatroot_exe(program)
for program in manifest.services: add_initrd_exe(program)
for program in manifest.drivers: add_initrd_exe(program)
for program in manifest.panics:
add_fatroot_exe(program)
for program in manifest.services:
add_initrd_stripped("jsix/services", program)
for program in manifest.drivers:
add_initrd_stripped("jsix/drivers", program)
syms = manifest.add_data("symbol_table.dat",
"Symbol table", ("symbols",))
@@ -136,9 +161,9 @@ class Project:
build.build(
rule = "makest",
outputs = [sym_out],
inputs = [f"${{build_root}}/{modules['kernel'].output}"],
inputs = [f"${{build_root}}/kernel/{modules['kernel'].output}"],
)
initrd_content.append(sym_out)
add_initrd_content("jsix/data", "symbol_table.dat")
bootloader = "${build_root}/fatroot/efi/boot/bootx64.efi"
build.build(
@@ -146,23 +171,23 @@ class Project:
outputs = [bootloader],
inputs = ["${build_root}/boot/boot.efi"],
variables = {
"name": "Installing bootloader",
"description": "Installing bootloader",
})
build.newline()
boot_config = str(fatroot / "jsix_boot.dat")
init_config = str(output / "init.manifest")
boot_config = join(fatroot, "jsix", "boot.conf")
manifest.write_boot_config(boot_config)
manifest.write_init_config(init_config, modules)
initrd_content.append(init_config)
initrd = join(fatroot, manifest.location, "initrd.dat")
initrd = str(fatroot / manifest.location / manifest.initrd["name"])
build.build(
rule = "mkinitrd",
rule = "makeinitrd",
outputs = [initrd],
inputs = initrd_content,
)
inputs = [str(initrdroot)],
implicit = initrd_content + ["${source_root}/scripts/mkj6romfs.py"],
variables = {"format": manifest.initrd["format"]},
)
build.newline()
fatroot_content.append(initrd)
build.build(

17
scripts/fnv.py Normal file
View File

@@ -0,0 +1,17 @@
"""Python implementation of FNV hashes."""
_FNV1_prime32 = 16777619
_FNV1_prime64 = 1099511628211
_FNV1_offset32 = 2166136261
_FNV1_offset64 = 14695981039346656037
def _fnv1a(offset, prime, mask):
def hashfunc(data):
h = offset
for b in data:
h = ((h ^ b) * prime) & mask
return h
return hashfunc
hash64 = _fnv1a(_FNV1_offset64, _FNV1_prime64, (2**64)-1)
hash32 = _fnv1a(_FNV1_offset32, _FNV1_prime32, (2**32)-1)

29
scripts/mkinitrd.py
View File

@@ -1,29 +0,0 @@
#!/usr/bin/env python3
compress_level = 19
def write_image(image, files):
from pathlib import Path
from cdblib import Writer, djb_hash
from pyzstd import compress
with open(image, 'wb') as db:
with Writer(db) as writer:
for f in files:
key = Path(f).name.encode('utf-8')
with open(f, 'rb') as input_file:
writer.put(key, compress(input_file.read(), compress_level))
if __name__ == "__main__":
from argparse import ArgumentParser
p = ArgumentParser(description="Generate a jsix initrd image")
p.add_argument("image", metavar="INITRD",
help="initrd image file to generate")
p.add_argument("files", metavar="FILE", nargs='+',
help="files to add to the image")
args = p.parse_args()
write_image(args.image, args.files)

197
scripts/mkj6romfs.py Executable file
View File

@@ -0,0 +1,197 @@
#!/usr/bin/env python3
#
# Generate a j6romfs image from a given directory.
from struct import calcsize, pack
from fnv import hash64
inode_type_dir = 1
inode_type_file = 2
uncompressed_limit = 0xffffffff
compressed_limit = 0xffffff
hasher = hash64
class fs_exception(Exception): pass
def compress_zstd(data):
from pyzstd import compress
return compress(data, 19)
def compress_none(data):
return data
compressors = {
'none': (0, compress_none),
'zstd': (1, compress_zstd),
}
def align(stream, size):
offset = stream.tell()
extra = offset % size
if extra != 0:
stream.seek(size-extra, 1)
offset = stream.tell()
return offset
def add_file(path, inode_data, output, compress):
offset = output.tell()
comp_size = 0
uncomp_size = 0
with open(path, 'rb') as file:
uncompressed = file.read()
uncomp_size = len(uncompressed)
compressed = compress(uncompressed)
comp_size = len(compressed)
# Don't use more room for compression than
# the original file
if comp_size >= uncomp_size:
compressed = uncompressed
comp_size = uncomp_size
output.write(compressed)
if uncomp_size > uncompressed_limit:
raise fs_exception(f"File {path} too large: {uncomp_size} bytes.")
if comp_size > compressed_limit:
raise fs_exception(f"File {path} too large when compressed: {comp_size} bytes.")
inode_data.append((inode_type_file, offset, comp_size, uncomp_size))
dirent_format = "<IHBBQ"
dirent_size = calcsize(dirent_format)
def make_dirent(inode, inode_type, name, strings, strings_offset):
strings_offset += len(strings)
name_utf = name.encode('utf-8')
name_hash = hasher(name_utf)
strings.extend(name_utf)
strings.append(0)
return pack(dirent_format, inode, strings_offset, inode_type, len(name_utf) + 1, name_hash)
def add_dir(path, files, dirs, inode_data, dir_inodes, output, compress):
strings = bytearray()
uncompressed = bytearray()
offset = align(output, 0x10)
comp_size = 0
uncomp_size = 0
strings_offset = (len(dirs) + len(files)) * dirent_size
for name, realpath in dirs:
inode = dir_inodes[realpath]
uncompressed.extend(make_dirent(inode, inode_type_dir, name, strings, strings_offset))
for name, inode in files:
uncompressed.extend(make_dirent(inode, inode_type_file, name, strings, strings_offset))
uncompressed.extend(strings)
uncomp_size = len(uncompressed)
compressed = compress(uncompressed)
comp_size = len(compressed)
# Don't use more room for compression than
# the original file
if comp_size >= uncomp_size:
compressed = uncompressed
comp_size = uncomp_size
output.write(uncompressed)
inode_data.append((inode_type_dir, offset, comp_size, uncomp_size))
def make_image(root, image, compressor):
import os
from os.path import dirname, join
compressor_id, compress = compressor
directories = []
inode_data = []
with open(image, 'wb') as output:
def write_header(inode_offset, inode_count, root_inode):
output.seek(0, 0)
output.write(pack("<8s Q II B7x",
b"j6romfs1", inode_offset, inode_count, root_inode, compressor_id))
write_header(0, 0, 0)
for (dirpath, dirs, files) in os.walk(root, topdown=False):
#print(f"{dirpath}:\n\t{dirs}\n\t{files}")
dir_inodes = []
for file in files:
dir_inodes.append((file, len(inode_data)))
add_file(join(dirpath, file), inode_data, output, compress)
parent = dirpath
if dirpath != root:
parent = dirname(dirpath)
dir_directories = [('.', dirpath), ('..', parent)]
dir_directories += [(d, join(dirpath, d)) for d in dirs]
directories.append((dirpath, dir_inodes, dir_directories))
dir_inodes = {directories[i][0]: len(inode_data) + i for i in range(len(directories))}
for d in directories:
add_dir(*d, inode_data, dir_inodes, output, compress)
inode_offset = align(output, 0x10) # align to a 16 byte value
for inode_type, offset, comp_size, uncomp_size in inode_data:
comp_size_type = (comp_size & 0xffffff) | (inode_type << 24)
output.write(pack("<IIQ", uncomp_size, comp_size_type, offset))
write_header(inode_offset, len(inode_data), len(inode_data) - 1)
if __name__ == "__main__":
import sys
from argparse import ArgumentParser
p = ArgumentParser(description="Generate a j6romfs image from a directory")
p.add_argument("--compressor", "-c", metavar="NAME", default="zstd",
help="Which compressor to use (currently: 'none' or 'zstd')")
p.add_argument("--verbose", "-v", action="store_true",
help="Output more information on progress")
p.add_argument("input", metavar="DIR",
help="The source directory to use as the root of the image")
p.add_argument("output", metavar="FILE",
help="The output file for the image")
args = p.parse_args()
import logging
log_level = logging.ERROR
if args.verbose:
log_level = logging.INFO
logging.basicConfig(
format="%(levelname)s: %(message)s",
level=log_level,
)
compressor = compressors.get(args.compressor)
if compressor is None:
logging.error(f"No such compressor: {args.compressor}")
sys.exit(1)
try:
make_image(args.input, args.output, compressor)
except fs_exception as fse:
print(fse, file=sys.stderr)
sys.exit(1)