[build] Refactor build options definitions

Split out build definition YAML files to allow different options based
on config, target, kind of module, and target/kind combination.

assets/build/base.yaml

@@ -1,40 +0,0 @@
----
-variables:
-  cc: "${source_root}/sysroot/bin/clang"
-  cxx: "${source_root}/sysroot/bin/clang++"
-  ld: "${source_root}/sysroot/bin/ld.lld"
-  ar: ar
-  nasm: nasm
-  objcopy: objcopy
-
-  ccflags: [
-    "-I${source_root}/src/include",
-    "-fcolor-diagnostics",
-    "-U__STDCPP_THREADS__",
-    "-D_LIBCPP_HAS_NO_THREADS",
-    "-D__jsix_config=${build_config}",
-    "-D__jsix_config_${build_config}",
-    "-DVERSION_MAJOR=${version_major}",
-    "-DVERSION_MINOR=${version_minor}",
-    "-DVERSION_PATCH=${version_patch}",
-    "-DVERSION_GITSHA=0x${version_sha}",
-    '-DGIT_VERSION=\"${version_major}.${version_minor}.${version_patch}+${version_sha}\"',
-    '-DGIT_VERSION_WIDE=L\"${version_major}.${version_minor}.${version_patch}+${version_sha}\"',
-
-    "-Wformat=2", "-Winit-self", "-Winline", "-Wmissing-format-attribute",
-    "-Wmissing-include-dirs", "-Wswitch", "-Wundef", "-Wdisabled-optimization",
-    "-Wpointer-arith", "-Wno-attributes", "-Wno-sign-compare", "-Wno-multichar",
-    "-Wno-div-by-zero", "-Wno-endif-labels", "-Wno-pragmas", "-Wno-format-extra-args",
-    "-Wno-unused-result", "-Wno-deprecated-declarations", "-Wno-unused-function",
-    "-Wno-address-of-packed-member", "-Wno-invalid-offsetof", "-Wno-format-nonliteral",
-    "-Werror" ]
-
-  asflags: [
-    "-DVERSION_MAJOR=${version_major}",
-    "-DVERSION_MINOR=${version_minor}",
-    "-DVERSION_PATCH=${version_patch}",
-    "-DVERSION_GITSHA=0x${version_sha}",
-    "-I${source_root}/src/include" ]
-
-  cflags: [ "-std=c11" ]
-  cxxflags: [ "-std=c++17" ]

assets/build/boot-debug.yaml

@@ -1,30 +0,0 @@
----
-extends: base
-
-variables:
-  ld: clang++
-
-  ccflags: [
-    "-nostdlib",
-    "-nodefaultlibs",
-    "-fno-builtin",
-
-    "-I${source_root}/external",
-    "--target=x86_64-unknown-windows",
-    "-ffreestanding",
-    "-mno-red-zone",
-    "-fshort-wchar",
-    "-fno-omit-frame-pointer",
-    "-ggdb",
-    "-g3" ]
-
-  cxxflags: [ "-fno-exceptions", "-fno-rtti" ]
-
-  ldflags: [
-    "--target=x86_64-unknown-windows",
-    "-nostdlib",
-    "-Wl,-entry:efi_main",
-    "-Wl,-subsystem:efi_application",
-    "-fuse-ld=lld-link",
-    "-g" ]
-

assets/build/global.yaml

@@ -0,0 +1,39 @@
+---
+cc: "${source_root}/sysroot/bin/clang"
+cxx: "${source_root}/sysroot/bin/clang++"
+ld: "${source_root}/sysroot/bin/ld.lld"
+ar: ar
+nasm: nasm
+objcopy: objcopy
+
+ccflags: [
+  "-I${source_root}/src/include",
+  "-fcolor-diagnostics",
+  "-U__STDCPP_THREADS__",
+  "-D_LIBCPP_HAS_NO_THREADS",
+  "-D__jsix_config=${build_config}",
+  "-D__jsix_config_${build_config}",
+  "-DVERSION_MAJOR=${version_major}",
+  "-DVERSION_MINOR=${version_minor}",
+  "-DVERSION_PATCH=${version_patch}",
+  "-DVERSION_GITSHA=0x${version_sha}",
+  '-DGIT_VERSION=\"${version_major}.${version_minor}.${version_patch}+${version_sha}\"',
+  '-DGIT_VERSION_WIDE=L\"${version_major}.${version_minor}.${version_patch}+${version_sha}\"',
+
+  "-Wformat=2", "-Winit-self", "-Winline", "-Wmissing-format-attribute",
+  "-Wmissing-include-dirs", "-Wswitch", "-Wundef", "-Wdisabled-optimization",
+  "-Wpointer-arith", "-Wno-attributes", "-Wno-sign-compare", "-Wno-multichar",
+  "-Wno-div-by-zero", "-Wno-endif-labels", "-Wno-pragmas", "-Wno-format-extra-args",
+  "-Wno-unused-result", "-Wno-deprecated-declarations", "-Wno-unused-function",
+  "-Wno-address-of-packed-member", "-Wno-invalid-offsetof", "-Wno-format-nonliteral",
+  "-Werror" ]
+
+asflags: [
+  "-DVERSION_MAJOR=${version_major}",
+  "-DVERSION_MINOR=${version_minor}",
+  "-DVERSION_PATCH=${version_patch}",
+  "-DVERSION_GITSHA=0x${version_sha}",
+  "-I${source_root}/src/include" ]
+
+cflags: [ "-std=c11" ]
+cxxflags: [ "-std=c++17" ]

assets/build/kernel-debug.yaml

@@ -1,57 +0,0 @@
----
-extends: base
-
-variables:
-  asflags: [ "-I${source_root}/src/kernel/" ]
-
-  ccflags: [
-    "--target=x86_64-jsix-elf",
-    "-fno-stack-protector",
-
-    "-I${source_root}/external",
-
-    "-nostdinc",
-    "-nostdlib",
-    "-ffreestanding",
-    "-nodefaultlibs",
-    "-fno-builtin",
-    "-fno-plt",
-
-    "-mno-sse",
-    "-fno-omit-frame-pointer",
-    "-mno-red-zone",
-    "-mcmodel=kernel",
-    "-fvisibility=hidden",
-    "-fvisibility-inlines-hidden",
-
-    "-g3",
-    "-ggdb",
-
-    "-D__ELF__",
-    "-D__jsix__",
-    "-D__j6kernel",
-    "-U__linux",
-    "-U__linux__",
-    "-DPRINTF_ALIAS_STANDARD_FUNCTION_NAMES=1",
-    "-DPRINTF_INCLUDE_CONFIG_H=1",
-
-    "--sysroot='${source_root}/sysroot'" ]
-
-
-  cflags: [ '-nostdinc' ]
-
-  cxxflags: [
-    "-fno-exceptions",
-    "-fno-rtti",
-    "-nostdinc",
-    ]
-
-  ldflags: [
-    "-g",
-    "-m", "elf_x86_64",
-    "-nostdlib",
-    "-Bstatic",
-    "--no-eh-frame-hdr",
-    "-z", "norelro",
-    "-z", "separate-code" ]
-

assets/build/rules.ninja

@@ -48,6 +48,10 @@ rule lib
   command = $ar qcs $out $in
   description = Archiving [$target]:$name
 
+rule driver
+  command = $ld $ldflags -shared -o $out $in $libs
+  description = Linking $name
+
 rule cp
   command = cp $in $out
   description = Copying [$target]:$name

assets/build/target.boot.yaml

@@ -0,0 +1,27 @@
+---
+ld: clang++
+
+ccflags: [
+  "-nostdlib",
+  "-nodefaultlibs",
+  "-fno-builtin",
+
+  "-I${source_root}/external",
+  "--target=x86_64-unknown-windows",
+  "-ffreestanding",
+  "-mno-red-zone",
+  "-fshort-wchar",
+  "-fno-omit-frame-pointer",
+  "-ggdb",
+  "-g3" ]
+
+cxxflags: [ "-fno-exceptions", "-fno-rtti" ]
+
+ldflags: [
+  "--target=x86_64-unknown-windows",
+  "-nostdlib",
+  "-Wl,-entry:efi_main",
+  "-Wl,-subsystem:efi_application",
+  "-fuse-ld=lld-link",
+  "-g" ]
+

assets/build/target.kernel.yaml

@@ -0,0 +1,54 @@
+---
+asflags: [ "-I${source_root}/src/kernel/" ]
+
+ccflags: [
+  "--target=x86_64-jsix-elf",
+  "-fno-stack-protector",
+
+  "-I${source_root}/external",
+
+  "-nostdinc",
+  "-nostdlib",
+  "-ffreestanding",
+  "-nodefaultlibs",
+  "-fno-builtin",
+  "-fno-plt",
+
+  "-mno-sse",
+  "-fno-omit-frame-pointer",
+  "-mno-red-zone",
+  "-mcmodel=kernel",
+  "-fvisibility=hidden",
+  "-fvisibility-inlines-hidden",
+
+  "-g3",
+  "-ggdb",
+
+  "-D__ELF__",
+  "-D__jsix__",
+  "-D__j6kernel",
+  "-U__linux",
+  "-U__linux__",
+  "-DPRINTF_ALIAS_STANDARD_FUNCTION_NAMES=1",
+  "-DPRINTF_INCLUDE_CONFIG_H=1",
+
+  "--sysroot='${source_root}/sysroot'" ]
+
+
+cflags: [ '-nostdinc' ]
+
+cxxflags: [
+  "-fno-exceptions",
+  "-fno-rtti",
+  "-nostdinc",
+  ]
+
+ldflags: [
+  "-g",
+  "-m", "elf_x86_64",
+  "-nostdlib",
+  "-Bstatic",
+  "--no-eh-frame-hdr",
+  "-z", "norelro",
+  "-z", "separate-code" ]
+

assets/build/target.user.yaml

@@ -0,0 +1,41 @@
+---
+asflags: [ "-I${source_root}/src/kernel/" ]
+
+ccflags: [
+  "--target=x86_64-jsix-elf",
+  "-fno-omit-frame-pointer",
+  "-fno-stack-protector",
+  "-fpic",
+  "-fpie",
+
+  "-g3",
+  "-ggdb",
+
+  "-fvisibility=hidden",
+  "-fvisibility-inlines-hidden",
+
+  "-D__ELF__",
+  "-D__jsix__",
+  "-U__linux",
+  "-U__linux__",
+
+  "--sysroot='${source_root}/sysroot'" ]
+
+
+cxxflags: [
+  "-fno-exceptions",
+  "-fno-rtti",
+  ]
+
+ldflags: [
+  "-g",
+  "-m", "elf_x86_64",
+  "--sysroot='${source_root}/sysroot'",
+  "--no-eh-frame-hdr",
+  "-L", "${source_root}/sysroot/lib",
+  "-z", "separate-code",
+  "-lc++", "-lc++abi", "-lunwind",
+  "--no-dependent-libraries",
+  "--dynamic-linker", "/tools/ld.so",
+]
+

assets/build/user-debug.yaml

@@ -1,43 +0,0 @@
----
-extends: base
-
-variables:
-  asflags: [ "-I${source_root}/src/kernel/" ]
-
-  ccflags: [
-    "--target=x86_64-jsix-elf",
-    "-fno-omit-frame-pointer",
-    "-fno-stack-protector",
-
-    "-g3",
-    "-ggdb",
-
-    "-fno-plt",
-    "-fvisibility=hidden",
-    "-fvisibility-inlines-hidden",
-
-    "-D__ELF__",
-    "-D__jsix__",
-    "-U__linux",
-    "-U__linux__",
-
-    "--sysroot='${source_root}/sysroot'" ]
-
-
-  cxxflags: [
-    "-fno-exceptions",
-    "-fno-rtti",
-    ]
-
-  ldflags: [
-    "-g",
-    "-m", "elf_x86_64",
-    "-Bstatic",
-    "--sysroot='${source_root}/sysroot'",
-    "--no-eh-frame-hdr",
-    "-L", "${source_root}/sysroot/lib",
-    "-z", "separate-code",
-    "-lc++", "-lc++abi", "-lunwind",
-    "--no-dependent-libraries",
-  ]
-

scripts/bonnibel/config.py

@@ -0,0 +1,68 @@
+_config_cache = {}
+
+def _load(filename, depfiles):
+    from . import load_config
+    if not filename in _config_cache:
+        if filename.exists():
+            depfiles.add(filename)
+            data = load_config(filename)
+            _config_cache[filename] = data
+    return _config_cache.get(filename, dict())
+
+
+def _build_config(chain, depfiles):
+    config = {}
+    for d in [_load(c, depfiles) for c in chain]:
+        for k, v in d.items():
+            if isinstance(v, (list, tuple)):
+                config[k] = config.get(k, list()) + list(v)
+            elif isinstance(v, dict):
+                config[k] = config.get(k, dict())
+                config[k].update(v)
+            else:
+                config[k] = v
+
+    return config
+
+
+def _make_ninja_config(outfile, config, files):
+    from os import makedirs
+    from ninja.ninja_syntax import Writer
+
+    makedirs(outfile.parent, exist_ok=True)
+
+    with open(outfile, "w") as buildfile:
+        build = Writer(buildfile)
+        build.comment("This file is automatically generated by bonnibel")
+        build.comment("Source config files:")
+        for f in files:
+            build.comment(f"  - {f}")
+
+        build.newline()
+        for k, v in config.items():
+            build.variable(k, v)
+
+ 
+def generate_configs(root, output, config, targets, kinds):
+
+    assets = root / "assets" / "build"
+    base = ["global.yaml", f"config.{config}.yaml"]
+
+    depfiles = set()
+
+    for target in targets:
+        chain = base + [f"target.{target}.yaml"]
+
+        files = [assets / f for f in chain]
+        config = _build_config(files, depfiles)
+        outfile = output / target / f"config.ninja"
+        _make_ninja_config(outfile, config, files)
+
+        for kind in kinds:
+            custom = [f"kind.{kind}.yaml", f"target.{target}.{kind}.yaml"]
+            files = [assets / f for f in chain + custom]
+            config = _build_config(files, depfiles)
+            outfile = output / target / f"config.{kind}.ninja"
+            _make_ninja_config(outfile, config, files)
+
+    return depfiles

scripts/bonnibel/manifest.py

@@ -37,6 +37,9 @@ class Manifest:
         self.drivers = [self.__build_entry(modules, i)
                 for i in config.get("drivers", tuple())]
 
+        self.tools = [self.__build_entry(modules, i)
+                for i in config.get("tools", tuple())]
+
         self.flags = config.get("flags", tuple())
 
         self.symbols = ""

scripts/bonnibel/module.py

@@ -88,6 +88,8 @@ class Module:
 
         if self.kind == "lib":
             return f"lib{self.name}.a"
+        elif self.kind == "driver":
+            return f"{self.name}.drv"
         else:
             return f"{self.name}.elf"
 
@@ -157,6 +159,11 @@ class Module:
             build.newline()
 
             build.variable("module_dir", target_rel(self.name + ".dir"))
+            build.variable("module_kind", self.kind)
+            build.newline()
+
+            build.include(f"${{target_dir}}/config.{self.kind}.ninja")
+            build.newline()
 
             modopts = BuildOptions(
                 local = [self.root, "${module_dir}"],

scripts/bonnibel/project.py

@@ -12,14 +12,17 @@ class Project:
 
     def generate(self, root, output, modules, config, manifest_file):
         import sys
-        import bonnibel
         from os.path import join
         from ninja.ninja_syntax import Writer
-        from .target import Target
 
         targets = set()
+        kinds = set()
         for mod in modules.values():
-            targets.update({Target.load(root, t, config) for t in mod.targets})
+            targets.update(mod.targets)
+            kinds.add(mod.kind)
+
+        from .config import generate_configs
+        config_deps = generate_configs(root, output, config, targets, kinds)
 
         with open(output / "build.ninja", "w") as buildfile:
             build = Writer(buildfile)
@@ -47,7 +50,7 @@ class Project:
             build.newline()
 
             for target in targets:
-                build.subninja(output / target.name / "target.ninja")
+                build.subninja(output / target / "target.ninja")
             build.newline()
 
             build.build(
@@ -151,6 +154,9 @@ class Project:
             for program in manifest.drivers:
                 add_initrd_stripped("jsix/drivers", program)
 
+            for program in manifest.tools:
+                add_initrd_stripped("jsix/tools", program)
+
             syms = manifest.add_data("symbol_table.dat",
                     "Symbol table", ("symbols",))
 
@@ -229,8 +235,7 @@ class Project:
                 [f"{self.root}/configure", str(manifest_file)] + \
                 [str(mod.modfile) for mod in modules.values()]
 
-            for target in targets:
+            regen_implicits += list(map(str, config_deps))
-                regen_implicits += target.depfiles
 
             build.build(
                 rule = "compdb",
@@ -246,7 +251,7 @@ class Project:
                 implicit = regen_implicits,
                 implicit_outputs = 
                     [f"module.{mod.name}.ninja" for mod in modules.values()] +
-                    [f"{target.name}/target.ninja" for target in targets] +
+                    [f"{target}/target.ninja" for target in targets] +
                     [boot_config],
                 )
 
@@ -254,9 +259,9 @@ class Project:
             build.default(["${build_root}/jsix.img"])
 
         for target in targets:
-            mods = [m.name for m in modules.values() if target.name in m.targets]
+            mods = [m.name for m in modules.values() if target in m.targets]
 
-            targetdir = output / target.name
+            targetdir = output / target
             targetdir.mkdir(exist_ok=True)
 
             buildfilename = str(targetdir / "target.ninja")
@@ -265,17 +270,16 @@ class Project:
                 build.comment("This file is automatically generated by bonnibel")
                 build.newline()
 
-                build.variable("target", target.name)
+                build.variable("target", target)
-                build.variable("target_dir", output / target.name)
+                build.variable("target_dir", output / target)
                 build.newline()
 
-                for name, value in target.items():
+                build.include(f"{target}/config.ninja")
-                    build.variable(name, value)
 
                 build.newline()
                 for kind in ('defs', 'run'):
                     for lang in ('c', 'cpp'):
-                        deffile = str(output / target.name / f"{lang}.{kind}")
+                        deffile = str(output / target / f"{lang}.{kind}")
 
                         build.build(
                             rule = f"dump_{lang}_{kind}",

scripts/bonnibel/target.py

@@ -1,50 +0,0 @@
-class Target(dict):
-    __targets = {}
-
-    @classmethod
-    def load(cls, root, name, config=None):
-        from . import load_config
-
-        if (name, config) in cls.__targets:
-            return cls.__targets[(name, config)]
-
-        configs = root / "assets/build"
-
-        dicts = []
-        depfiles = []
-        basename = name
-        if config:
-            basename += f"-{config}"
-
-        while basename is not None:
-            filename = str(configs / (basename + ".yaml"))
-            depfiles.append(filename)
-            desc = load_config(filename)
-            basename = desc.get("extends")
-            dicts.append(desc.get("variables", dict()))
-
-        t = Target(name, config, depfiles)
-        for d in reversed(dicts):
-            for k, v in d.items():
-                if isinstance(v, (list, tuple)):
-                    t[k] = t.get(k, list()) + list(v)
-                elif isinstance(v, dict):
-                    t[k] = t.get(k, dict())
-                    t[k].update(v)
-                else:
-                    t[k] = v
-
-        cls.__targets[(name, config)] = t
-        return t
-
-    def __init__(self, name, config, depfiles):
-        self.__name = name
-        self.__config = config
-        self.__depfiles = tuple(depfiles)
-
-    def __hash__(self):
-        return hash((self.__name, self.__config))
-
-    name = property(lambda self: self.__name)
-    config = property(lambda self: self.__config)
-    depfiles = property(lambda self: self.__depfiles)