From c9d713fc7f7fab0b2a55d8913f13280f970c1171 Mon Sep 17 00:00:00 2001 From: "Justin C. Miller" Date: Sun, 5 Sep 2021 13:07:09 -0700 Subject: [PATCH] [build] Move to yaml-based build config and manifest Overall, I believe TOML to be a superior configuration format than YAML in many situations, but it gets ugly quickly when nesting data structures. The build configs were fine in TOML, but the manifest (and my future plans for it) got unwieldy. I also did not want different formats for each kind of configuration on top of also having a custom DSL for interface definitions, so I've switched all the TOML to YAML. Also of note is that this change actually adds structure to the manifest file, which was little more than a CSV previously. --- assets/manifests/default | 5 --- assets/manifests/default.yaml | 8 ++++ configs/{base.toml => base.yaml} | 37 ++++++++-------- configs/{boot-debug.toml => boot-debug.yaml} | 28 +++++-------- .../{kernel-debug.toml => kernel-debug.yaml} | 42 +++++++++---------- configs/{user-debug.toml => user-debug.yaml} | 32 +++++++------- configure | 2 +- requirements.txt | 2 +- scripts/bonnibel/__init__.py | 5 +++ scripts/bonnibel/project.py | 25 +++++++---- scripts/bonnibel/target.py | 6 +-- 11 files changed, 101 insertions(+), 91 deletions(-) delete mode 100644 assets/manifests/default create mode 100644 assets/manifests/default.yaml rename configs/{base.toml => base.yaml} (78%) rename configs/{boot-debug.toml => boot-debug.yaml} (64%) rename configs/{kernel-debug.toml => kernel-debug.yaml} (64%) rename configs/{user-debug.toml => user-debug.yaml} (61%) diff --git a/assets/manifests/default b/assets/manifests/default deleted file mode 100644 index eb76027..0000000 --- a/assets/manifests/default +++ /dev/null @@ -1,5 +0,0 @@ -kernel,kernel -kernel,panic.serial -user,drv.uefi_fb -user,testapp -user,srv.init diff --git a/assets/manifests/default.yaml b/assets/manifests/default.yaml new file mode 100644 index 0000000..2399b73 --- /dev/null +++ b/assets/manifests/default.yaml @@ -0,0 +1,8 @@ +--- +programs: + - name: panic.serial + target: kernel + + - name: drv.uefi_fb + - name: testapp + - name: srv.init diff --git a/configs/base.toml b/configs/base.yaml similarity index 78% rename from configs/base.toml rename to configs/base.yaml index df3e4f3..4d917bc 100644 --- a/configs/base.toml +++ b/configs/base.yaml @@ -1,13 +1,14 @@ -[variables] -cc = "clang" -cxx = "clang++" -ld = "ld.lld" -ar = "ar" -nasm = "nasm" -objcopy = "objcopy" -cog = "cog" +--- +variables: + cc: clang + cxx: clang++ + ld: ld.lld + ar: ar + nasm: nasm + objcopy: objcopy + cog: cog -ccflags = [ + ccflags: [ "-I${source_root}/src/include", "-I${source_root}/src/include/x86_64", "-fcolor-diagnostics", @@ -17,27 +18,25 @@ ccflags = [ "-DVERSION_GITSHA=0x${version_sha}", '-DGIT_VERSION=\"${version_major}.${version_minor}.${version_patch}+${version_sha}\"', '-DGIT_VERSION_WIDE=L\"${version_major}.${version_minor}.${version_patch}+${version_sha}\"', + "-Wformat=2", "-Winit-self", "-Wfloat-equal", "-Winline", "-Wmissing-format-attribute", "-Wmissing-include-dirs", "-Wswitch", "-Wundef", "-Wdisabled-optimization", "-Wpointer-arith", "-Wno-attributes", "-Wno-sign-compare", "-Wno-multichar", "-Wno-div-by-zero", "-Wno-endif-labels", "-Wno-pragmas", "-Wno-format-extra-args", "-Wno-unused-result", "-Wno-deprecated-declarations", "-Wno-unused-function", - "-Wno-address-of-packed-member", "-Wno-invalid-offsetof", "-Werror", -] + "-Wno-address-of-packed-member", "-Wno-invalid-offsetof", "-Werror" ] -asflags = [ + asflags: [ "-DVERSION_MAJOR=${version_major}", "-DVERSION_MINOR=${version_minor}", "-DVERSION_PATCH=${version_patch}", "-DVERSION_GITSHA=0x${version_sha}", - "-I${source_root}/src/include", -] + "-I${source_root}/src/include" ] -cflags = ["-std=c11"] -cxxflags = ["-std=c++17"] + cflags: [ "-std=c11" ] + cxxflags: [ "-std=c++17" ] -cogflags = [ + cogflags: [ "-I", "${source_root}/scripts", - "-D", "definitions_path=${source_root}/definitions" -] + "-D", "definitions_path=${source_root}/definitions" ] diff --git a/configs/boot-debug.toml b/configs/boot-debug.yaml similarity index 64% rename from configs/boot-debug.toml rename to configs/boot-debug.yaml index 3da0a15..507abe0 100644 --- a/configs/boot-debug.toml +++ b/configs/boot-debug.yaml @@ -1,10 +1,10 @@ -extends = "base" +--- +extends: base -[variables] +variables: + ld: clang++ -ld = "clang++" - -ccflags = [ + ccflags: [ "-I${source_root}/external", "--target=x86_64-unknown-windows", "-ffreestanding", @@ -13,23 +13,15 @@ ccflags = [ "-fno-omit-frame-pointer", "-ggdb", "-g3", - '-DKERNEL_FILENAME=L"jsix.elf"', -] + '-DKERNEL_FILENAME=L"jsix.elf"' ] + cxxflags: [ "-fno-exceptions", "-fno-rtti" ] -cflags = [ -] - -cxxflags = [ - "-fno-exceptions", - "-fno-rtti", -] - -ldflags = [ + ldflags: [ "--target=x86_64-unknown-windows", "-nostdlib", "-Wl,-entry:efi_main", "-Wl,-subsystem:efi_application", "-fuse-ld=lld-link", - "-g" -] + "-g" ] + diff --git a/configs/kernel-debug.toml b/configs/kernel-debug.yaml similarity index 64% rename from configs/kernel-debug.toml rename to configs/kernel-debug.yaml index 9358dde..3781aec 100644 --- a/configs/kernel-debug.toml +++ b/configs/kernel-debug.yaml @@ -1,47 +1,47 @@ -extends = "base" +--- +extends: base -[variables] -asflags = [ - "-I${source_root}/src/kernel/", -] +variables: + asflags: [ "-I${source_root}/src/kernel/" ] -ccflags = [ + ccflags: [ "--target=x86_64-unknown-elf", "-I${source_root}/external", + "-nostdlib", "-ffreestanding", "-nodefaultlibs", "-fno-builtin", + "-mno-sse", "-fno-omit-frame-pointer", "-mno-red-zone", - "-g", "-mcmodel=large", + + "-g", + "-D__ELF__", "-D__JSIX__", - "-isystem${source_root}/sysroot/include", - "-isystem${source_root}/src/libraries/libc/include", - "--sysroot='${source_root}/sysroot'", "-U__linux", "-U__linux__", -] -cflags = [ - '-nostdinc', -] + "-isystem${source_root}/sysroot/include", + "-isystem${source_root}/src/libraries/libc/include", + "--sysroot='${source_root}/sysroot'" ] -cxxflags = [ + + cflags: [ '-nostdinc' ] + + cxxflags: [ "-fno-exceptions", "-fno-rtti", - '-nostdinc', - "-isystem${source_root}/sysroot/include/c++/v1" -] + "-nostdinc", + "-isystem${source_root}/sysroot/include/c++/v1" ] -ldflags = [ + ldflags: [ "-g", "-nostdlib", "-Bstatic", "-z", "norelro", - "-z", "separate-code", -] + "-z", "separate-code" ] diff --git a/configs/user-debug.toml b/configs/user-debug.yaml similarity index 61% rename from configs/user-debug.toml rename to configs/user-debug.yaml index a8d882d..ae2d9ab 100644 --- a/configs/user-debug.toml +++ b/configs/user-debug.yaml @@ -1,34 +1,34 @@ -extends = "base" +--- +extends: base -[variables] -asflags = [ - "-I${source_root}/src/kernel/", -] +variables: + asflags: [ "-I${source_root}/src/kernel/" ] -ccflags = [ + ccflags: [ "-mno-sse", "-fno-omit-frame-pointer", + "-g", + "-D__ELF__", "-D__JSIX__", - "-isystem${source_root}/sysroot/include", - "-isystem${source_root}/src/libraries/libc/include", - "--sysroot='${source_root}/sysroot'", "-U__linux", "-U__linux__", -] -cxxflags = [ + "-isystem${source_root}/sysroot/include", + "-isystem${source_root}/src/libraries/libc/include", + "--sysroot='${source_root}/sysroot'" ] + + + cxxflags: [ "-fno-exceptions", "-fno-rtti", - "-isystem${source_root}/sysroot/include/c++/v1" -] + "-isystem${source_root}/sysroot/include/c++/v1" ] -ldflags = [ + ldflags: [ "-g", "-Bstatic", "--sysroot='${source_root}/sysroot'", "-L", "${source_root}/sysroot/lib", - "-z", "separate-code", -] + "-z", "separate-code" ] diff --git a/configure b/configure index 2576255..4ba5ff9 100755 --- a/configure +++ b/configure @@ -47,7 +47,7 @@ if __name__ == "__main__": from bonnibel import BonnibelError p = ArgumentParser(description="Generate jsix build files") - p.add_argument("--manifest", "-m", metavar="FILE", default="assets/manifests/default", + p.add_argument("--manifest", "-m", metavar="FILE", default="assets/manifests/default.yaml", help="File to use as the system manifest") p.add_argument("--config", "-c", metavar="NAME", default="debug", help="Configuration to build (eg, 'debug' or 'release')") diff --git a/requirements.txt b/requirements.txt index 10f8055..6cb549a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ cogapp >= 3 ninja >= 1.10.2 peru >= 1.2.1 -toml >= 0.10.2 +pyyaml >= 5.4 diff --git a/scripts/bonnibel/__init__.py b/scripts/bonnibel/__init__.py index 337ec82..1ba3953 100644 --- a/scripts/bonnibel/__init__.py +++ b/scripts/bonnibel/__init__.py @@ -1,3 +1,8 @@ class BonnibelError(Exception): def __init__(self, message): self.message = message + +def load_config(filename): + from yaml import safe_load + with open(filename, 'r') as infile: + return safe_load(infile.read()) diff --git a/scripts/bonnibel/project.py b/scripts/bonnibel/project.py index da72207..a995ae0 100644 --- a/scripts/bonnibel/project.py +++ b/scripts/bonnibel/project.py @@ -10,11 +10,12 @@ class Project: def __str__(self): return f"{self.name} {self.version.major}.{self.version.minor}.{self.version.patch}-{self.version.sha}" - def generate(self, root, output, modules, config, manifest): + def generate(self, root, output, modules, config, manifest_file): import sys import bonnibel from os.path import join from ninja.ninja_syntax import Writer + from . import load_config from .target import Target targets = set() @@ -47,11 +48,7 @@ class Project: fatroot.mkdir(exist_ok=True) fatroot_content = [] - for line in open(manifest, 'r'): - target, name = line.split(",", 1) - target = target.strip() - name = name.strip() - + def add_fatroot(name, target): if not name in modules: raise BonnibelError(f"Manifest item '{name}' is not a known module") @@ -66,9 +63,23 @@ class Project: "name": f"Installing {name}", "debug": f"${{build_root}}/{mod.output}.debug", }) + fatroot_content.append(fatroot_output) build.newline() + manifest = load_config(manifest_file) + programs = manifest.get("programs", tuple()) + + kernel = manifest.get("kernel", dict()) + add_fatroot( + kernel.get("name", "kernel"), + kernel.get("target", "kernel")) + + for program in programs: + name = program["name"] + target = program.get("target", "user") + add_fatroot(name, target) + symbol_table = "${build_root}/fatroot/symbol_table.dat" build.build( rule = "makest", @@ -123,7 +134,7 @@ class Project: build.newline() regen_implicits = \ - [f"{self.root}/configure", str(manifest)] + \ + [f"{self.root}/configure", str(manifest_file)] + \ [str(mod.modfile) for mod in modules.values()] for target in targets: diff --git a/scripts/bonnibel/target.py b/scripts/bonnibel/target.py index 8d3e8b4..30a076b 100644 --- a/scripts/bonnibel/target.py +++ b/scripts/bonnibel/target.py @@ -3,7 +3,7 @@ class Target(dict): @classmethod def load(cls, root, name, config=None): - import toml + from . import load_config if (name, config) in cls.__targets: return cls.__targets[(name, config)] @@ -17,9 +17,9 @@ class Target(dict): basename += f"-{config}" while basename is not None: - filename = str(configs / (basename + ".toml")) + filename = str(configs / (basename + ".yaml")) depfiles.append(filename) - desc = toml.load(filename) + desc = load_config(filename) basename = desc.get("extends") dicts.append(desc.get("variables", dict()))