[build] Move to yaml-based build config and manifest

Overall, I believe TOML to be a superior configuration format than YAML in many situations, but it gets ugly quickly when nesting data structures. The build configs were fine in TOML, but the manifest (and my future plans for it) got unwieldy. I also did not want different formats for each kind of configuration on top of also having a custom DSL for interface definitions, so I've switched all the TOML to YAML. Also of note is that this change actually adds structure to the manifest file, which was little more than a CSV previously.
2021-09-05 13:07:09 -07:00
parent 186724e751
commit c9d713fc7f
11 changed files with 101 additions and 91 deletions
--- a/assets/manifests/default
+++ b/assets/manifests/default
@@ -1,5 +0,0 @@
-kernel,kernel
-kernel,panic.serial
-user,drv.uefi_fb
-user,testapp
-user,srv.init
--- a/assets/manifests/default.yaml
+++ b/assets/manifests/default.yaml
@@ -0,0 +1,8 @@
+---
+programs:
+  - name: panic.serial
+    target: kernel
+
+  - name: drv.uefi_fb
+  - name: testapp
+  - name: srv.init
--- a/configs/base.yaml
+++ b/configs/base.yaml
@@ -1,13 +1,14 @@
-[variables]
-cc = "clang"
-cxx = "clang++"
-ld = "ld.lld"
-ar = "ar"
-nasm = "nasm"
-objcopy = "objcopy"
-cog = "cog"
+---
+variables:
+  cc: clang
+  cxx: clang++
+  ld: ld.lld
+  ar: ar
+  nasm: nasm
+  objcopy: objcopy
+  cog: cog

-ccflags = [
+  ccflags: [
    "-I${source_root}/src/include",
    "-I${source_root}/src/include/x86_64",
    "-fcolor-diagnostics",
@@ -17,27 +18,25 @@ ccflags = [
    "-DVERSION_GITSHA=0x${version_sha}",
    '-DGIT_VERSION=\"${version_major}.${version_minor}.${version_patch}+${version_sha}\"',
    '-DGIT_VERSION_WIDE=L\"${version_major}.${version_minor}.${version_patch}+${version_sha}\"',
+
    "-Wformat=2", "-Winit-self", "-Wfloat-equal", "-Winline", "-Wmissing-format-attribute",
    "-Wmissing-include-dirs", "-Wswitch", "-Wundef", "-Wdisabled-optimization",
    "-Wpointer-arith", "-Wno-attributes", "-Wno-sign-compare", "-Wno-multichar",
    "-Wno-div-by-zero", "-Wno-endif-labels", "-Wno-pragmas", "-Wno-format-extra-args",
    "-Wno-unused-result", "-Wno-deprecated-declarations", "-Wno-unused-function",
-    "-Wno-address-of-packed-member", "-Wno-invalid-offsetof", "-Werror",
-]
+    "-Wno-address-of-packed-member", "-Wno-invalid-offsetof", "-Werror" ]

-asflags = [
+  asflags: [
    "-DVERSION_MAJOR=${version_major}",
    "-DVERSION_MINOR=${version_minor}",
    "-DVERSION_PATCH=${version_patch}",
    "-DVERSION_GITSHA=0x${version_sha}",
-    "-I${source_root}/src/include",
-]
+    "-I${source_root}/src/include" ]

-cflags = ["-std=c11"]
-cxxflags = ["-std=c++17"]
+  cflags: [ "-std=c11" ]
+  cxxflags: [ "-std=c++17" ]

-cogflags = [
+  cogflags: [
    "-I", "${source_root}/scripts",
-    "-D", "definitions_path=${source_root}/definitions"
-]
+    "-D", "definitions_path=${source_root}/definitions" ]

--- a/configs/boot-debug.yaml
+++ b/configs/boot-debug.yaml
@@ -1,10 +1,10 @@
-extends = "base"
+---
+extends: base

-[variables]
+variables:
+  ld: clang++

-ld = "clang++"
-
-ccflags = [
+  ccflags: [
    "-I${source_root}/external",
    "--target=x86_64-unknown-windows",
    "-ffreestanding",
@@ -13,23 +13,15 @@ ccflags = [
    "-fno-omit-frame-pointer",
    "-ggdb",
    "-g3",
-    '-DKERNEL_FILENAME=L"jsix.elf"',
-]
+    '-DKERNEL_FILENAME=L"jsix.elf"' ]

+  cxxflags: [ "-fno-exceptions", "-fno-rtti" ]

-cflags = [
-]
-
-cxxflags = [
-    "-fno-exceptions",
-    "-fno-rtti",
-]
-
-ldflags = [
+  ldflags: [
    "--target=x86_64-unknown-windows",
    "-nostdlib",
    "-Wl,-entry:efi_main",
    "-Wl,-subsystem:efi_application",
    "-fuse-ld=lld-link",
-    "-g"
-]
+    "-g" ]
+
--- a/configs/kernel-debug.yaml
+++ b/configs/kernel-debug.yaml
@@ -1,47 +1,47 @@
-extends = "base"
+---
+extends: base

-[variables]
-asflags = [ 
-    "-I${source_root}/src/kernel/",
-]
+variables:
+  asflags: [ "-I${source_root}/src/kernel/" ]

-ccflags = [
+  ccflags: [
    "--target=x86_64-unknown-elf",
    "-I${source_root}/external",
+
    "-nostdlib",
    "-ffreestanding",
    "-nodefaultlibs",
    "-fno-builtin",
+
    "-mno-sse",
    "-fno-omit-frame-pointer",
    "-mno-red-zone",
-    "-g",
    "-mcmodel=large",
+
+    "-g",
+
    "-D__ELF__",
    "-D__JSIX__",
-    "-isystem${source_root}/sysroot/include",
-    "-isystem${source_root}/src/libraries/libc/include",
-    "--sysroot='${source_root}/sysroot'",
    "-U__linux",
    "-U__linux__",
-]

-cflags = [
-    '-nostdinc',
-]
+    "-isystem${source_root}/sysroot/include",
+    "-isystem${source_root}/src/libraries/libc/include",
+    "--sysroot='${source_root}/sysroot'" ]

-cxxflags = [
+
+  cflags: [ '-nostdinc' ]
+
+  cxxflags: [
    "-fno-exceptions",
    "-fno-rtti",
-    '-nostdinc',
-    "-isystem${source_root}/sysroot/include/c++/v1"
-]
+    "-nostdinc",
+    "-isystem${source_root}/sysroot/include/c++/v1" ]

-ldflags = [
+  ldflags: [
    "-g",
    "-nostdlib",
    "-Bstatic",
    "-z", "norelro",
-    "-z", "separate-code",
-]
+    "-z", "separate-code" ]

--- a/configs/user-debug.yaml
+++ b/configs/user-debug.yaml
@@ -1,34 +1,34 @@
-extends = "base"
+---
+extends: base

-[variables]
-asflags = [
-    "-I${source_root}/src/kernel/",
-]
+variables:
+  asflags: [ "-I${source_root}/src/kernel/" ]

-ccflags = [
+  ccflags: [
    "-mno-sse",
    "-fno-omit-frame-pointer",
+
    "-g",
+
    "-D__ELF__",
    "-D__JSIX__",
-    "-isystem${source_root}/sysroot/include",
-    "-isystem${source_root}/src/libraries/libc/include",
-    "--sysroot='${source_root}/sysroot'",
    "-U__linux",
    "-U__linux__",
-]

-cxxflags = [
+    "-isystem${source_root}/sysroot/include",
+    "-isystem${source_root}/src/libraries/libc/include",
+    "--sysroot='${source_root}/sysroot'" ]
+
+
+  cxxflags: [
    "-fno-exceptions",
    "-fno-rtti",
-    "-isystem${source_root}/sysroot/include/c++/v1"
-]
+    "-isystem${source_root}/sysroot/include/c++/v1" ]

-ldflags = [
+  ldflags: [
    "-g",
    "-Bstatic",
    "--sysroot='${source_root}/sysroot'",
    "-L", "${source_root}/sysroot/lib",
-    "-z", "separate-code",
-]
+    "-z", "separate-code" ]

--- a/2
+++ b/2
@@ -47,7 +47,7 @@ if __name__ == "__main__":
    from bonnibel import BonnibelError

    p = ArgumentParser(description="Generate jsix build files")
-    p.add_argument("--manifest", "-m", metavar="FILE", default="assets/manifests/default",
+    p.add_argument("--manifest", "-m", metavar="FILE", default="assets/manifests/default.yaml",
            help="File to use as the system manifest")
    p.add_argument("--config", "-c", metavar="NAME", default="debug",
            help="Configuration to build (eg, 'debug' or 'release')")
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
 cogapp >= 3
 ninja >= 1.10.2
 peru >= 1.2.1
-toml >= 0.10.2
+pyyaml >= 5.4
--- a/scripts/bonnibel/init.py
+++ b/scripts/bonnibel/init.py
@@ -1,3 +1,8 @@
 class BonnibelError(Exception):
    def __init__(self, message):
        self.message = message
+
+def load_config(filename):
+    from yaml import safe_load
+    with open(filename, 'r') as infile:
+        return safe_load(infile.read())
--- a/scripts/bonnibel/project.py
+++ b/scripts/bonnibel/project.py
@@ -10,11 +10,12 @@ class Project:
    def __str__(self):
        return f"{self.name} {self.version.major}.{self.version.minor}.{self.version.patch}-{self.version.sha}"

-    def generate(self, root, output, modules, config, manifest):
+    def generate(self, root, output, modules, config, manifest_file):
        import sys
        import bonnibel
        from os.path import join
        from ninja.ninja_syntax import Writer
+        from . import load_config
        from .target import Target

        targets = set()
@@ -47,11 +48,7 @@ class Project:
            fatroot.mkdir(exist_ok=True)

            fatroot_content = []
-            for line in open(manifest, 'r'):
-                target, name = line.split(",", 1)
-                target = target.strip()
-                name = name.strip()
-
+            def add_fatroot(name, target):
                if not name in modules:
                    raise BonnibelError(f"Manifest item '{name}' is not a known module")

@@ -66,9 +63,23 @@ class Project:
                        "name": f"Installing {name}",
                        "debug": f"${{build_root}}/{mod.output}.debug",
                    })
+
                fatroot_content.append(fatroot_output)
                build.newline()

+            manifest = load_config(manifest_file)
+            programs = manifest.get("programs", tuple())
+
+            kernel = manifest.get("kernel", dict())
+            add_fatroot(
+                kernel.get("name", "kernel"),
+                kernel.get("target", "kernel"))
+
+            for program in programs:
+                name = program["name"]
+                target = program.get("target", "user")
+                add_fatroot(name, target)
+
            symbol_table = "${build_root}/fatroot/symbol_table.dat"
            build.build(
                rule = "makest",
@@ -123,7 +134,7 @@ class Project:
            build.newline()

            regen_implicits = \
-                [f"{self.root}/configure", str(manifest)] + \
+                [f"{self.root}/configure", str(manifest_file)] + \
                [str(mod.modfile) for mod in modules.values()]

            for target in targets:
--- a/scripts/bonnibel/target.py
+++ b/scripts/bonnibel/target.py
@@ -3,7 +3,7 @@ class Target(dict):

    @classmethod
    def load(cls, root, name, config=None):
-        import toml
+        from . import load_config

        if (name, config) in cls.__targets:
            return cls.__targets[(name, config)]
@@ -17,9 +17,9 @@ class Target(dict):
            basename += f"-{config}"

        while basename is not None:
-            filename = str(configs / (basename + ".toml"))
+            filename = str(configs / (basename + ".yaml"))
            depfiles.append(filename)
-            desc = toml.load(filename)
+            desc = load_config(filename)
            basename = desc.get("extends")
            dicts.append(desc.get("variables", dict()))