WIP linux multiarch support

Changing bonnibel to respect the --arch flag to configure. This requires
some reworking of modules, mostly in the addition of the ModuleList
class instead of just a dict of modules.

assets/build/linux/config.debug.yaml

@@ -0,0 +1,9 @@
+---
+ccflags: [
+  "-g3",
+  "-ggdb",
+]
+
+ldflags: [
+  "-g",
+]

assets/build/linux/config.release.yaml

@@ -0,0 +1,3 @@
+ccflags: [
+  "-O3",
+]

assets/build/linux/global.yaml

@@ -0,0 +1,39 @@
+---
+cc: "clang-16"
+cxx: "clang++-16"
+ld: "ld.lld-16"
+ar: ar
+nasm: nasm
+objcopy: objcopy
+
+ccflags: [
+  "-I${source_root}/src/include",
+  "-I${source_root}/sysroot/include/c++/v1",
+  "-fcolor-diagnostics",
+  "-U__STDCPP_THREADS__",
+  "-D__jsix_config=${build_config}",
+  "-D__jsix_config_${build_config}",
+  "-DVERSION_MAJOR=${version_major}",
+  "-DVERSION_MINOR=${version_minor}",
+  "-DVERSION_PATCH=${version_patch}",
+  "-DVERSION_GITSHA=0x${version_sha}",
+  '-DGIT_VERSION=\"${version_major}.${version_minor}.${version_patch}+${version_sha}\"',
+  '-DGIT_VERSION_WIDE=L\"${version_major}.${version_minor}.${version_patch}+${version_sha}\"',
+
+  "-Wformat=2", "-Winit-self", "-Winline", "-Wmissing-format-attribute",
+  "-Wmissing-include-dirs", "-Wswitch", "-Wundef", "-Wdisabled-optimization",
+  "-Wpointer-arith", "-Wno-attributes", "-Wno-sign-compare", "-Wno-multichar",
+  "-Wno-div-by-zero", "-Wno-endif-labels", "-Wno-pragmas", "-Wno-format-extra-args",
+  "-Wno-unused-result", "-Wno-deprecated-declarations", "-Wno-unused-function",
+  "-Wno-address-of-packed-member", "-Wno-invalid-offsetof", "-Wno-format-nonliteral",
+  "-Werror" ]
+
+asflags: [
+  "-DVERSION_MAJOR=${version_major}",
+  "-DVERSION_MINOR=${version_minor}",
+  "-DVERSION_PATCH=${version_patch}",
+  "-DVERSION_GITSHA=0x${version_sha}",
+  "-I${source_root}/src/include" ]
+
+cflags: [ "-std=c11" ]
+cxxflags: [ "-std=c++17" ]

assets/build/linux/rules.ninja

@@ -0,0 +1,91 @@
+rule compile.c
+  command = $cc -MMD -MF $out.d $cflags $ccflags -o $out -c $in
+  description = Compiling [$target]:$name
+  depfile = $out.d
+  deps = gcc
+
+rule dump_c_defs
+  command = echo | $cc $ccflags $cflags -dM -E - > $out
+  description = Dumping C defines for $target
+
+rule dump_c_run
+  command = echo '#!/bin/bash' > $out; echo '$cc $ccflags $cflags $$*' >> $
+      $out; chmod a+x $out
+  description = Dumping C arguments for $target
+
+rule compile.cpp
+  command = $cxx -MMD -MF $out.d $cxxflags $ccflags -o $out -c $in
+  description = Compiling [$target]:$name
+  depfile = $out.d
+  deps = gcc
+
+rule dump_cpp_defs
+  command = echo | $cxx -x c++ $ccflags $cxxflags -dM -E - > $out
+  description = Dumping C++ defines for $target
+
+rule dump_cpp_run
+  command = echo '#!/bin/bash' > $out; echo '$cxx $ccflags $cxxflags $$*' $
+      >> $out; chmod a+x $out
+  description = Dumping C++ arguments for $target
+
+rule compile.s
+  command = $nasm -o $out -felf64 -MD $out.d $asflags $in
+  description = Assembling [$target]:$name
+  depfile = $out.d
+  deps = gcc
+
+rule parse.cog
+  command = cog -o $out -d -D target=$target $cogflags $in
+  description = Parsing [$target]:$name
+
+rule exe
+  command = $ld $ldflags -o $out $in $libs
+  description = Linking exe [$target]:$name
+
+rule driver
+  command = $ld $ldflags -o $out $in $libs
+  description = Linking driver [$target]:$name
+
+rule lib
+  command = $ld -shared -soname $soname $ldflags -o $out $in $libs
+  description = Linking [$target]:$name
+
+rule lib_static
+  command = $ar qcs $out $in
+  description = Archiving [$target]:$name
+
+rule cp
+  command = cp $in $out
+  description = Copying [$target]:$name
+
+rule dump
+  command = objdump -DSC -M intel $in > $out
+  description = Dumping decompiled $name
+
+rule makest
+    description = Making symbol table
+    command = nm -n -S --demangle $in | ${source_root}/scripts/build_symbol_table.py $out
+
+rule makeinitrd
+    description = Creating $name
+    command = ${source_root}/scripts/mkj6romfs.py -c $format $in $out
+
+rule makefat
+    description = Creating $name
+    command = $
+        cp $in $out; $
+        mcopy -s -D o -i $out@@1M ${build_root}/fatroot/* ::/
+
+rule strip
+    description = Stripping $name
+    command = $
+        cp $in $out; $
+        objcopy --only-keep-debug $out $debug; $
+        strip --discard-all -g $out; $
+        objcopy --add-gnu-debuglink=$debug $out
+
+rule touch
+    command = touch $out
+
+rule compdb
+    command = ninja -t compdb > $out

assets/build/linux/target.init.yaml

@@ -0,0 +1,39 @@
+---
+ccflags: [
+  "--target=x86_64-jsix-elf",
+  "-fno-omit-frame-pointer",
+  "-fno-stack-protector",
+
+  "-fvisibility=hidden",
+  "-fvisibility-inlines-hidden",
+
+  "-D__ELF__",
+  "-D__jsix__",
+  "-U__linux",
+  "-U__linux__",
+
+  "-DMSPACES",
+
+  "--sysroot='${source_root}/sysroot'"
+]
+
+
+cxxflags: [
+  "-fno-exceptions",
+  "-fno-rtti",
+]
+
+ldflags: [
+  "-Bstatic",
+  "-m", "elf_x86_64",
+  "--sysroot='${source_root}/sysroot'",
+  "--no-eh-frame-hdr",
+  "-L", "${source_root}/sysroot/lib",
+  "-z", "separate-code",
+  "-lc++", "-lc++abi", "-lunwind",
+  "--no-dependent-libraries",
+]
+
+libs: [
+  "${target_dir}/crt0.o",
+]

assets/build/linux/target.user.exe.yaml

@@ -0,0 +1,16 @@
+---
+
+ccflags: [
+  "-fpie"
+]
+
+ldflags: [
+  "-pie",
+  "-rpath", "${target_dir}",
+  "--dynamic-linker", "/lib64/ld-linux-x86-64.so.2",
+  "--push-state", "--as-needed", "-Bstatic", "-lc++", "-lc++abi", "-lunwind", "--pop-state",
+]
+
+libs: [
+  "${target_dir}/crt0.o",
+]

assets/build/linux/target.user.shared.yaml

@@ -0,0 +1,7 @@
+---
+ccflags: [
+]
+
+ldflags: [
+  "-shared",
+]

assets/build/linux/target.user.yaml

@@ -0,0 +1,33 @@
+---
+asflags: []
+
+ccflags: [
+  "--target=x86_64-jsix-elf",
+  "-fno-omit-frame-pointer",
+  "-fno-stack-protector",
+
+  "-fvisibility=hidden",
+  "-fvisibility-inlines-hidden",
+
+  "-D__ELF__",
+  "-D__jsix__",
+  "-U__linux",
+  "-U__linux__",
+
+  "--sysroot='${source_root}/sysroot'",
+  "-fpic",
+]
+
+cxxflags: [
+  "-fno-exceptions",
+  "-fno-rtti",
+  ]
+
+ldflags: [
+  "-m", "elf_x86_64",
+  "--sysroot='${source_root}/sysroot'",
+  "--no-eh-frame-hdr",
+  "-L", "${source_root}/sysroot/lib",
+  "-z", "separate-code",
+  "--no-dependent-libraries",
+]

configure

@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
 
-def generate(output, config, manifest):
+def generate(output, config, arch, manifest):
     from os import makedirs
     from glob import iglob
     from pathlib import Path
-    from bonnibel.module import Module
+    from bonnibel.module import Module, ModuleList
     from bonnibel.project import Project
 
     root = Path(__file__).parent.resolve()
@@ -18,16 +18,17 @@ def generate(output, config, manifest):
         str(root / "external/*.module"),
     ]
 
-    modules = {}
+    modules = ModuleList(arch)
     for source in sources:
         for modfile in iglob(source, recursive=True):
-            path = Path(modfile).parent
+            modfile = Path(modfile)
+            path = modfile.parent
 
             def module_init(name, **kwargs):
                 if not "root" in kwargs:
                     kwargs["root"] = path
                 m = Module(name, modfile, **kwargs)
-                modules[m.name] = m
+                modules.add(m)
                 return m
 
             glo = {
@@ -36,18 +37,16 @@ def generate(output, config, manifest):
                 "build_root": output,
                 "module_root": path,
                 "config": config,
+                "arch": arch,
             }
             code = compile(open(modfile, 'r').read(), modfile, "exec")
 
             loc = {}
             exec(code, glo, loc)
 
-    Module.update(modules)
-
     makedirs(output.resolve(), exist_ok=True)
-    project.generate(root, output, modules, config, manifest)
+    project.generate(root, output, modules, config, arch, manifest)
-    for mod in modules.values():
+    modules.generate(output)
-        mod.generate(output)
 
 if __name__ == "__main__":
     import sys
@@ -57,18 +56,25 @@ if __name__ == "__main__":
     from argparse import ArgumentParser
     from bonnibel import BonnibelError
 
+    default_arch = "amd64"
+
     p = ArgumentParser(description="Generate jsix build files")
     p.add_argument("--manifest", "-m", metavar="FILE", default="assets/manifests/default.yaml",
             help="File to use as the system manifest")
-    p.add_argument("--config", "-c", metavar="NAME", default="debug",
+    p.add_argument("--conf", "-c", metavar="NAME", default="debug",
             help="Configuration to build (eg, 'debug' or 'release')")
-    p.add_argument("output", metavar="DIR", default="build", nargs='?',
+    p.add_argument("--arch", "-a", metavar="NAME", default=default_arch,
+            help="Architecture to build (eg, 'amd64' or 'linux')")
+    p.add_argument("--verbose", "-v", action='count', default=0,
+            help="More verbose log output")
+    p.add_argument("output", metavar="DIR", default=None, nargs='?',
             help="Where to create the build root")
 
     args = p.parse_args()
 
+    output = args.output or f"build.{args.arch}"
     try:
-        generate(args.output, args.config, args.manifest)
+        generate(output, args.conf, args.arch, args.manifest)
 
     except BonnibelError as be:
         import sys

qemu.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 root=$(dirname $0)
-build="${root}/build"
+build="${root}/build.amd64"
 assets="${root}/assets"
 
 no_build=""

scripts/bonnibel/config.py

@@ -43,10 +43,10 @@ def _make_ninja_config(outfile, config, files):
             build.variable(k, v)
 
  
-def generate_configs(root, output, config, targets, kinds):
+def generate_configs(root, output, buildconfig, arch, targets, kinds):
 
-    assets = root / "assets" / "build"
+    assets = root / "assets" / "build" / arch
-    base = ["global.yaml", f"config.{config}.yaml"]
+    base = ["global.yaml", f"config.{buildconfig}.yaml"]
 
     depfiles = set()
 

scripts/bonnibel/manifest.py

@@ -37,9 +37,15 @@ class Manifest:
         self.drivers = [self.__build_entry(modules, i)
                 for i in config.get("drivers", tuple())]
 
-        libs = set(config.get("libs", tuple()))
+        def get_libdeps(names):
-        libs.update(self.__libdeps([modules[e.module] for e in self.services]))
+            libmods = modules.get_mods(names)
-        libs.update(self.__libdeps([modules[e.module] for e in self.drivers]))
+            deps = modules.all_deps(libmods, stop_at_static=True)
+            deps = [m.name for m in deps if m.kind == "lib"]
+            return deps
+
+        libs = set(get_libdeps(config.get("libs", tuple())))
+        libs.update(get_libdeps([e.module for e in self.services]))
+        libs.update(get_libdeps([e.module for e in self.drivers]))
 
         self.libs = [self.__build_entry(modules, i)
                 for i in libs]
@@ -80,13 +86,6 @@ class Manifest:
 
         return Manifest.Entry(name, target, mod.get_output(), flags)
 
-    def __libdeps(self, modules):
-        deps = set([m.name for m in modules if m.kind == "lib"])
-        for m in modules:
-            if m.static: continue
-            deps.update(self.__libdeps(m.depmods))
-        return deps
-
     def add_data(self, output, desc, flags=tuple()):
         e = Manifest.Entry(None, None, output, flags)
         self.data.append(e)

scripts/bonnibel/module.py

@@ -39,7 +39,7 @@ class Module:
         "kind": (str, "exe"),
         "outfile": (str, None),
         "basename": (str, None),
-        "targets": (set, ()),
+        "target": (str, None),
         "deps":  (set, ()),
         "public_headers":  (set, ()),
         "copy_headers": (bool, False),
@@ -53,6 +53,8 @@ class Module:
         "no_libc": (bool, False),
         "ld_script": (str, None),
         "static": (bool, False),
+        "arch_source": (dict, ()),
+        "skip_arches": (tuple, ()),
     }
 
     def __init__(self, name, modfile, root, **kwargs):
@@ -81,15 +83,14 @@ class Module:
 
         # Turn strings into real Source objects
         self.sources = [make_source(root, f) for f in self.sources]
+        for arch in self.arch_source:
+            self.arch_source[arch] = [make_source(root, f) for f in self.arch_source[arch]]
 
         header_source = lambda f: make_source(root, Path("include") / f)
         if self.copy_headers:
             header_source = lambda f: make_copy_source(root, f, "include")
         self.public_headers = [header_source(f) for f in self.public_headers]
 
-        # Filled by Module.update
-        self.depmods = set()
-
     def __repr__(self):
         return f"<Module {self.kind} {self.name}>"
 
@@ -115,227 +116,6 @@ class Module:
             ext = dict(exe=".elf", driver=".drv", lib=(static and ".a" or ".so"))
             return self.basename + ext.get(self.kind, "")
 
-    @classmethod
-    def update(cls, mods):
-        from . import BonnibelError
-
-        def resolve(source, modlist):
-            resolved = set()
-            for dep in modlist:
-                if not dep in mods:
-                    raise BonnibelError(f"module '{source.name}' references unknown module '{dep}'")
-                mod = mods[dep]
-                resolved.add(mod)
-            return resolved
-
-        for mod in mods.values():
-            mod.depmods = resolve(mod, mod.deps)
-
-        target_mods = [mod for mod in mods.values() if mod.targets]
-        for mod in target_mods:
-            closed = set()
-            children = set(mod.depmods)
-            while children:
-                child = children.pop()
-                closed.add(child)
-                child.targets |= mod.targets
-                children |= {m for m in child.depmods if not m in closed}
-
-    def generate(self, output):
-        from pathlib import Path
-        from collections import defaultdict
-        from ninja.ninja_syntax import Writer
-
-        def walk_deps(deps, static, results):
-            for mod in deps:
-                if static or mod.name not in results:
-                    results[mod.name] = (mod, static)
-                walk_deps(mod.depmods, static or mod.static, results)
-
-        all_deps = {}
-        walk_deps(self.depmods, self.static, all_deps)
-        all_deps = all_deps.values()
-
-        def gather_phony(build, deps, child_rel):
-            phony = ".headers.phony"
-            child_phony = [child_rel(phony, module=c.name)
-                    for c, _ in all_deps]
-
-            build.build(
-                rule = "touch",
-                outputs = [mod_rel(phony)],
-                implicit = child_phony,
-                order_only = list(map(mod_rel, deps)),
-            )
-
-        filename = str(output / f"module.{self.name}.ninja")
-        with open(filename, "w") as buildfile:
-            build = Writer(buildfile)
-
-            build.comment("This file is automatically generated by bonnibel")
-            build.newline()
-
-            build.variable("module_dir", target_rel(self.name + ".dir"))
-            build.variable("module_kind", self.kind)
-            build.newline()
-
-            build.include(f"${{target_dir}}/config.{self.kind}.ninja")
-            build.newline()
-
-            modopts = BuildOptions(
-                local = [self.root, "${module_dir}"],
-                ld_script = self.ld_script and self.root / self.ld_script,
-                )
-            if self.public_headers:
-                modopts.includes += [
-                    self.root / "include",
-                    f"${{target_dir}}/{self.name}.dir/include",
-                ]
-
-            for key, value in self.variables.items():
-                build.variable(key, value)
-            build.newline()
-
-            for include in self.includes:
-                p = Path(include)
-                if p.is_absolute():
-                    if not p in modopts.includes:
-                        modopts.includes.append(str(p.resolve()))
-                elif include != ".":
-                    incpath = self.root / p
-                    destpath = mod_rel(p)
-                    for header in incpath.rglob("*.h"):
-                        dest_header = f"{destpath}/" + str(header.relative_to(incpath))
-                    modopts.includes.append(str(incpath))
-                    modopts.includes.append(destpath)
-
-            for dep, static in all_deps:
-                if dep.public_headers:
-                    if dep.include_phase == "normal":
-                        modopts.includes += [dep.root / "include", f"${{target_dir}}/{dep.name}.dir/include"]
-                    elif dep.include_phase == "late":
-                        modopts.late += [dep.root / "include", f"${{target_dir}}/{dep.name}.dir/include"]
-                    else:
-                        from . import BonnibelError
-                        raise BonnibelError(f"Module {dep.name} has invalid include_phase={dep.include_phase}")
-
-                if dep.kind == "headers":
-                    continue
-                elif dep.kind == "lib":
-                    modopts.libs.append((target_rel(dep.get_output(static)), static))
-                else:
-                    modopts.order_only.append(target_rel(dep.get_output(static)))
-
-            cc_includes = []
-            if modopts.local:
-                cc_includes += [f"-iquote{i}" for i in modopts.local]
-
-            if modopts.includes:
-                cc_includes += [f"-I{i}" for i in modopts.includes]
-
-            if modopts.late:
-                cc_includes += [f"-idirafter{i}" for i in modopts.late]
-
-            if cc_includes:
-                build.variable("ccflags", ["${ccflags}"] + cc_includes)
-
-            as_includes = [f"-I{d}" for d in modopts.local + modopts.includes + modopts.late]
-            if as_includes:
-                build.variable("asflags", ["${asflags}"] + as_includes)
-
-            if modopts.libs:
-                build.variable("libs", ["-L${target_dir}", "${libs}"] + modopts.linker_args)
-
-            if modopts.ld_script:
-                build.variable("ldflags", ["${ldflags}"] + ["-T", modopts.ld_script])
-
-            header_deps = []
-            inputs = []
-            headers = set(self.public_headers)
-            while headers:
-                source = headers.pop()
-                headers.update(source.next)
-
-                if source.action:
-                    build.newline()
-                    build.build(rule=source.action, **source.args)
-
-                if source.gather:
-                    header_deps += list(source.outputs)
-
-                if source.input:
-                    inputs.extend(map(mod_rel, source.outputs))
-
-            build.newline()
-
-            inputs = []
-            sources = set(self.sources)
-            while sources:
-                source = sources.pop()
-                sources.update(source.next)
-
-                if source.action:
-                    build.newline()
-                    build.build(rule=source.action, **source.args)
-
-                if source.gather:
-                    header_deps += list(source.outputs)
-
-                if source.input:
-                    inputs.extend(map(mod_rel, source.outputs))
-
-            gather_phony(build, header_deps, target_rel)
-
-            if self.kind == "headers":
-                # Header-only, don't output a build rule
-                return
-
-            output = target_rel(self.get_output())
-            build.newline()
-            build.build(
-                rule = self.kind,
-                outputs = output,
-                inputs = inputs,
-                implicit = modopts.implicit,
-                order_only = modopts.order_only,
-                variables = {"name": self.name,
-                             "soname": self.get_output()},
-            )
-
-            dump = output + ".dump"
-            build.newline()
-            build.build(
-                rule = "dump",
-                outputs = dump,
-                inputs = output,
-                variables = {"name": self.name},
-            )
-
-            s_output = target_rel(self.get_output(static=True))
-            if s_output != output:
-                build.newline()
-                build.build(
-                    rule = self.kind + "_static",
-                    outputs = s_output,
-                    inputs = inputs,
-                    order_only = modopts.order_only,
-                    variables = {"name": self.name},
-                )
-
-                dump = s_output + ".dump"
-                build.newline()
-                build.build(
-                    rule = "dump",
-                    outputs = dump,
-                    inputs = s_output,
-                    variables = {"name": self.name},
-                )
-
-            if self.default:
-                build.newline()
-                build.default(output)
-                build.default(dump)
-
     def add_input(self, path, **kwargs):
         from .source import make_source
         s = make_source(self.root, path, **kwargs)
@@ -350,3 +130,285 @@ class Module:
         for source in self.public_headers:
             if source.path in paths:
                 source.add_deps(deps)
+
+
+class ModuleList:
+    def __init__(self, arch):
+        self.__arch = arch
+        self.__mods = {}
+        self.__used = {}
+        self.__targets = frozenset()
+        self.__kinds = frozenset()
+
+    def __getitem__(self, name):
+        return self.__mods.get(name)
+
+    def __contains__(self, name):
+        """Return if the module name is known."""
+        return name in self.__mods
+
+    def __iter__(self):
+        """Iterate over _non-skipped_ modules."""
+        return self.used.__iter__()
+
+    def __skip(self, mod):
+        if self.__arch in mod.skip_arches: return True
+        return False
+
+    @property
+    def used(self):
+        if self.__used is None:
+            self.__used = {n:m for n,m in self.__mods.items() if not self.__skip(m)}
+        return self.__used
+
+    @property
+    def targets(self):
+        if self.__targets is None:
+            self.__targets = frozenset([m.target for m in self.used.values() if m.target])
+        return self.__targets
+
+    @property
+    def kinds(self):
+        if self.__kinds is None:
+            self.__kinds = frozenset([m.kind for m in self.used.values()])
+        return self.__kinds
+
+    def get(self, name):
+        return self.__mods.get(name)
+
+    def add(self, mod):
+        if mod.name in self.__mods:
+            raise BonnibelError(f"re-adding module '{mod.name}' to this ModuleList")
+        self.__mods[mod.name] = mod
+        self.__used = None
+        self.__targets = None
+        self.__kinds = None
+
+    def get_mods(self, names, filt=None):
+        return {self[n] for n in names
+                if n in self.used
+                and ((not filt) or filt(self[n]))}
+
+    def all_deps(self, mods, stop_at_static=False):
+        search = set(mods)
+        closed = list()
+
+        while search:
+            mod = search.pop()
+            if mod in closed: continue
+            closed.append(mod)
+
+            if stop_at_static and mod.static:
+                continue
+
+            for dep in mod.deps:
+                if not dep in self.__mods:
+                    raise BonnibelError(f"module '{mod.name}' references unknown module '{dep}'")
+                if dep in self.used:
+                    search.add(self.used[dep])
+
+        return closed
+
+    def target_mods(self, target):
+        return self.all_deps([m for m in self.used.values() if m.target == target])
+
+    def generate(self, output):
+        from pathlib import Path
+        from collections import defaultdict
+        from ninja.ninja_syntax import Writer
+
+        def walk_deps(deps, static, results):
+            for modname in deps:
+                mod = self.used.get(modname)
+                if not mod: continue # skipped
+                if static or modname not in results:
+                    results[modname] = (mod, static)
+                walk_deps(mod.deps, static or mod.static, results)
+
+        for mod in self.used.values():
+            all_deps = {}
+            walk_deps(mod.deps, mod.static, all_deps)
+            all_deps = all_deps.values()
+
+            def gather_phony(build, deps, child_rel):
+                phony = ".headers.phony"
+                child_phony = [child_rel(phony, module=c.name)
+                        for c, _ in all_deps]
+
+                build.build(
+                    rule = "touch",
+                    outputs = [mod_rel(phony)],
+                    implicit = child_phony,
+                    order_only = list(map(mod_rel, deps)),
+                )
+
+            filename = str(output / f"module.{mod.name}.ninja")
+            with open(filename, "w") as buildfile:
+                build = Writer(buildfile)
+
+                build.comment("This file is automatically generated by bonnibel")
+                build.newline()
+
+                build.variable("module_dir", target_rel(mod.name + ".dir"))
+                build.variable("module_kind", mod.kind)
+                build.newline()
+
+                build.include(f"${{target_dir}}/config.{mod.kind}.ninja")
+                build.newline()
+
+                modopts = BuildOptions(
+                    local = [mod.root, "${module_dir}"],
+                    ld_script = mod.ld_script and mod.root / mod.ld_script,
+                    )
+                if mod.public_headers:
+                    modopts.includes += [
+                        mod.root / "include",
+                        f"${{target_dir}}/{mod.name}.dir/include",
+                    ]
+
+                for key, value in mod.variables.items():
+                    build.variable(key, value)
+                build.newline()
+
+                for include in mod.includes:
+                    p = Path(include)
+                    if p.is_absolute():
+                        if not p in modopts.includes:
+                            modopts.includes.append(str(p.resolve()))
+                    elif include != ".":
+                        incpath = mod.root / p
+                        destpath = mod_rel(p)
+                        for header in incpath.rglob("*.h"):
+                            dest_header = f"{destpath}/" + str(header.relative_to(incpath))
+                        modopts.includes.append(str(incpath))
+                        modopts.includes.append(destpath)
+
+                for dep, static in all_deps:
+                    if dep.public_headers:
+                        if dep.include_phase == "normal":
+                            modopts.includes += [dep.root / "include", f"${{target_dir}}/{dep.name}.dir/include"]
+                        elif dep.include_phase == "late":
+                            modopts.late += [dep.root / "include", f"${{target_dir}}/{dep.name}.dir/include"]
+                        else:
+                            from . import BonnibelError
+                            raise BonnibelError(f"Module {dep.name} has invalid include_phase={dep.include_phase}")
+
+                    if dep.kind == "headers":
+                        continue
+                    elif dep.kind == "lib":
+                        modopts.libs.append((target_rel(dep.get_output(static)), static))
+                    else:
+                        modopts.order_only.append(target_rel(dep.get_output(static)))
+
+                cc_includes = []
+                if modopts.local:
+                    cc_includes += [f"-iquote{i}" for i in modopts.local]
+
+                if modopts.includes:
+                    cc_includes += [f"-I{i}" for i in modopts.includes]
+
+                if modopts.late:
+                    cc_includes += [f"-idirafter{i}" for i in modopts.late]
+
+                if cc_includes:
+                    build.variable("ccflags", ["${ccflags}"] + cc_includes)
+
+                as_includes = [f"-I{d}" for d in modopts.local + modopts.includes + modopts.late]
+                if as_includes:
+                    build.variable("asflags", ["${asflags}"] + as_includes)
+
+                if modopts.libs:
+                    build.variable("libs", ["-L${target_dir}", "${libs}"] + modopts.linker_args)
+
+                if modopts.ld_script:
+                    build.variable("ldflags", ["${ldflags}"] + ["-T", modopts.ld_script])
+
+                header_deps = []
+                inputs = []
+                headers = set(mod.public_headers)
+                while headers:
+                    source = headers.pop()
+                    headers.update(source.next)
+
+                    if source.action:
+                        build.newline()
+                        build.build(rule=source.action, **source.args)
+
+                    if source.gather:
+                        header_deps += list(source.outputs)
+
+                    if source.input:
+                        inputs.extend(map(mod_rel, source.outputs))
+
+                build.newline()
+
+                inputs = []
+                sources = set(mod.sources)
+                sources.update(set(mod.arch_source.get(self.__arch, tuple())))
+                while sources:
+                    source = sources.pop()
+                    sources.update(source.next)
+
+                    if source.action:
+                        build.newline()
+                        build.build(rule=source.action, **source.args)
+
+                    if source.gather:
+                        header_deps += list(source.outputs)
+
+                    if source.input:
+                        inputs.extend(map(mod_rel, source.outputs))
+
+                gather_phony(build, header_deps, target_rel)
+
+                if mod.kind == "headers":
+                    # Header-only, don't output a build rule
+                    continue
+
+                mod_output = target_rel(mod.get_output())
+                build.newline()
+                build.build(
+                    rule = mod.kind,
+                    outputs = mod_output,
+                    inputs = inputs,
+                    implicit = modopts.implicit,
+                    order_only = modopts.order_only,
+                    variables = {"name": mod.name,
+                                 "soname": mod.get_output()},
+                )
+
+                dump = mod_output + ".dump"
+                build.newline()
+                build.build(
+                    rule = "dump",
+                    outputs = dump,
+                    inputs = mod_output,
+                    variables = {"name": mod.name},
+                )
+
+                s_output = target_rel(mod.get_output(static=True))
+                if s_output != mod_output:
+                    build.newline()
+                    build.build(
+                        rule = mod.kind + "_static",
+                        outputs = s_output,
+                        inputs = inputs,
+                        order_only = modopts.order_only,
+                        variables = {"name": mod.name},
+                    )
+
+                    dump = s_output + ".dump"
+                    build.newline()
+                    build.build(
+                        rule = "dump",
+                        outputs = dump,
+                        inputs = s_output,
+                        variables = {"name": mod.name},
+                    )
+
+                if mod.default:
+                    build.newline()
+                    build.default(mod_output)
+                    build.default(dump)
+
+

scripts/bonnibel/project.py

@@ -10,22 +10,17 @@ class Project:
     def __str__(self):
         return f"{self.name} {self.version.major}.{self.version.minor}.{self.version.patch}-{self.version.sha}"
 
-    def generate(self, root, output, modules, config, manifest_file):
+    def generate(self, root, output, modules, config, arch, manifest_file):
         import sys
         from os.path import join
         from ninja.ninja_syntax import Writer
 
-        targets = set()
-        kinds = set()
-        for mod in modules.values():
-            targets.update(mod.targets)
-            kinds.add(mod.kind)
-
         from .config import generate_configs
-        config_deps = generate_configs(root, output, config, targets, kinds)
+        config_deps = generate_configs(root, output, config, arch, modules.targets, modules.kinds)
 
         with open(output / "build.ninja", "w") as buildfile:
             build = Writer(buildfile)
+            default_builds = []
 
             build.comment("This file is automatically generated by bonnibel")
             build.variable("ninja_required_version", "1.3")
@@ -34,7 +29,7 @@ class Project:
             build.variable("build_config", config)
             build.newline()
 
-            build.include(root / "assets/build/rules.ninja")
+            build.include(root / "assets" / "build" / arch / "rules.ninja")
             build.newline()
 
             build.variable("version_major", self.version.major)
@@ -49,7 +44,7 @@ class Project:
                 ])
             build.newline()
 
-            for target in targets:
+            for target in modules.targets:
                 build.subninja(output / target / "target.ninja")
             build.newline()
 
@@ -57,7 +52,7 @@ class Project:
                 rule = "touch",
                 outputs = "${build_root}/.all_headers",
                 implicit = [f"${{build_root}}/include/{m.name}/.headers.phony"
-                    for m in modules.values() if m.public_headers],
+                    for m in modules.used.values() if m.public_headers],
                 )
             build.build(
                 rule = "phony",
@@ -78,56 +73,24 @@ class Project:
             initrdroot = output / "initrd_root"
             initrdroot.mkdir(exist_ok=True)
 
-            fatroot_content = []
+            image_content = {'initrd_root': [], 'fatroot': []}
-            initrd_content = []
 
-            def add_fatroot(source, name):
+            def add_image_content(image, path, name):
-                output = join(manifest.location, name)
+                output = join(path, name)
-                fatroot_output = f"${{build_root}}/fatroot/{output}"
+                image_output = f"${{build_root}}/{image}/{output}"
 
                 build.build(
                     rule = "cp",
-                    outputs = [fatroot_output],
+                    outputs = [image_output],
-                    inputs = [source],
-                    variables = {
-                        "description": f"Installing {output}",
-                    })
-
-                fatroot_content.append(fatroot_output)
-                build.newline()
-
-            def add_fatroot_exe(entry):
-                input_path = f"${{build_root}}/{entry.target}/{entry.output}"
-                intermediary = f"${{build_root}}/{entry.output}"
-
-                build.build(
-                    rule = "strip",
-                    outputs = [intermediary],
-                    inputs = [input_path],
-                    implicit = [f"{input_path}.dump"],
-                    variables = {
-                        "name": f"Stripping {entry.module}",
-                        "debug": f"${{build_root}}/.debug/{entry.output}.debug",
-                    })
-
-                add_fatroot(intermediary, entry.output)
-
-            def add_initrd_content(root, name):
-                output = join(root, name)
-                initrd_output = f"${{build_root}}/initrd_root/{output}"
-
-                build.build(
-                    rule = "cp",
-                    outputs = [initrd_output],
                     inputs = [f"${{build_root}}/{name}"],
                     variables = {
                         "description": f"Installing {name}",
                     })
 
-                initrd_content.append(initrd_output)
+                image_content[image].append(image_output)
                 build.newline()
 
-            def add_initrd_stripped(root, entry):
+            def add_image_content_exe(image, path, entry):
                 input_path = f"${{build_root}}/{entry.target}/{entry.output}"
                 intermediary = f"${{build_root}}/{entry.output}"
 
@@ -141,87 +104,108 @@ class Project:
                         "debug": f"${{build_root}}/.debug/{entry.output}.debug",
                     })
 
-                add_initrd_content(root, entry.output)
+                build.build(
+                    rule = "phony",
+                    outputs = [entry.output],
+                    inputs = [intermediary])
+
+                add_image_content(image, path, entry.output)
+
+            if 'kernel' in modules.used:
+                add_image_content_exe("fatroot", manifest.location, manifest.kernel)
+
+                syms = manifest.add_data("symbol_table.dat",
+                        "Symbol table", ("symbols",))
+
+                syms_file = "jsix.symbols"
+                syms_path = join(manifest.location, syms_file );
+                syms_out = join(fatroot, syms_path)
+                build.build(
+                    rule = "makest",
+                    outputs = [syms_out],
+                    inputs = [f"${{build_root}}/kernel/{modules['kernel'].get_output(static=True)}"],
+                    )
+                image_content['fatroot'].append(syms_out)
+                manifest.symbols = syms_file
+
+                build.newline()
+                default_builds.append("${build_root}/jsix.img")
+
+            def try_add_manifest_module(entry, section, image, path="jsix"):
+                if entry.module in modules.used:
+                    add_image_content_exe(image, path, entry)
+                elif entry.module not in modules:
+                    raise BonnibelError(f'unknown {section} module in manifest: {entry.module}')
+
+            try_add_manifest_module(manifest.init, "init", "fatroot")
 
-            add_fatroot_exe(manifest.kernel)
-            add_fatroot_exe(manifest.init)
             for program in manifest.panics:
-                add_fatroot_exe(program)
+                try_add_manifest_module(program, "panic", "fatroot")
 
             for program in manifest.services:
-                add_initrd_stripped("jsix/services", program)
+                try_add_manifest_module(program, "services", "initrd_root", "jsix/services")
 
             for program in manifest.drivers:
-                add_initrd_stripped("jsix/drivers", program)
+                try_add_manifest_module(program, "drivers", "initrd_root", "jsix/drivers")
 
             for program in manifest.libs:
-                add_initrd_stripped("jsix/lib", program)
+                try_add_manifest_module(program, "libs", "initrd_root", "jsix/lib")
 
-            syms = manifest.add_data("symbol_table.dat",
+            extra_regen_outputs = []
-                    "Symbol table", ("symbols",))
 
-            syms_file = "jsix.symbols"
+            if 'boot' in modules.used:
-            syms_path = join(manifest.location, syms_file );
+                bootloader = "${build_root}/fatroot/efi/boot/bootx64.efi"
-            syms_out = join(fatroot, syms_path)
-            build.build(
-                rule = "makest",
-                outputs = [syms_out],
-                inputs = [f"${{build_root}}/kernel/{modules['kernel'].get_output(static=True)}"],
-                )
-            fatroot_content.append(syms_out)
-            manifest.symbols = syms_file
-
-            bootloader = "${build_root}/fatroot/efi/boot/bootx64.efi"
-            build.build(
-                rule = "cp",
-                outputs = [bootloader],
-                inputs = ["${build_root}/boot/boot.efi"],
-                variables = {
-                    "description": "Installing bootloader",
-                })
-            build.newline()
-
-            boot_config = join(fatroot, "jsix", "boot.conf")
-            manifest.write_boot_config(boot_config)
-
-            initrd = str(fatroot / manifest.location / manifest.initrd["name"])
-            build.build(
-                rule = "makeinitrd",
-                outputs = [initrd],
-                inputs = [str(initrdroot)],
-                implicit = initrd_content + ["${source_root}/scripts/mkj6romfs.py"],
-                variables = {"format": manifest.initrd["format"]},
-            )
-            build.newline()
-
-            fatroot_content.append(initrd)
-
-            build.build(
-                rule = "makefat",
-                outputs = ["${build_root}/jsix.img"],
-                inputs = ["${source_root}/assets/diskbase.img"],
-                implicit = fatroot_content + [bootloader],
-                variables = {"name": "jsix.img"},
-                )
-            build.newline()
-
-            default_assets = {
-                "UEFI Variables": ("ovmf/x64/ovmf_vars.fd", "ovmf_vars.fd"),
-                "GDB Debug Helpers": ("debugging/jsix.elf-gdb.py", "jsix.elf-gdb.py"),
-            }
-
-            for name, assets in default_assets.items():
-                p = root / "assets" / assets[0]
-                out = f"${{build_root}}/{assets[1]}"
                 build.build(
                     rule = "cp",
-                    outputs = [out],
+                    outputs = [bootloader],
-                    inputs = [str(p)],
+                    inputs = ["${build_root}/boot/boot.efi"],
-                    variables = {"name": name},
+                    variables = {
-                    )
+                        "description": "Installing bootloader",
-                build.default([out])
+                    })
                 build.newline()
 
+                boot_config = join(fatroot, "jsix", "boot.conf")
+                manifest.write_boot_config(boot_config)
+                extra_regen_outputs.append(boot_config)
+
+                initrd = str(fatroot / manifest.location / manifest.initrd["name"])
+                build.build(
+                    rule = "makeinitrd",
+                    outputs = [initrd],
+                    inputs = [str(initrdroot)],
+                    implicit = image_content['initrd_root'] + ["${source_root}/scripts/mkj6romfs.py"],
+                    variables = {"format": manifest.initrd["format"]},
+                )
+                build.newline()
+
+                image_content['fatroot'].append(initrd)
+
+                build.build(
+                    rule = "makefat",
+                    outputs = ["${build_root}/jsix.img"],
+                    inputs = ["${source_root}/assets/diskbase.img"],
+                    implicit = image_content['fatroot'] + [bootloader],
+                    variables = {"name": "jsix.img"},
+                    )
+                build.newline()
+
+                default_assets = {
+                    "UEFI Variables": ("ovmf/x64/ovmf_vars.fd", "ovmf_vars.fd"),
+                    "GDB Debug Helpers": ("debugging/jsix.elf-gdb.py", "jsix.elf-gdb.py"),
+                }
+
+                for name, assets in default_assets.items():
+                    p = root / "assets" / assets[0]
+                    out = f"${{build_root}}/{assets[1]}"
+                    build.build(
+                        rule = "cp",
+                        outputs = [out],
+                        inputs = [str(p)],
+                        variables = {"name": name},
+                        )
+                    build.newline()
+                    default_builds.append(out)
+
             compdb = "${source_root}/compile_commands.json"
 
             build.rule("regen",
@@ -233,7 +217,7 @@ class Project:
 
             regen_implicits = \
                 [f"{self.root}/configure", str(manifest_file)] + \
-                [str(mod.modfile) for mod in modules.values()]
+                [str(mod.modfile) for mod in modules.used.values()]
 
             regen_implicits += list(map(str, config_deps))
 
@@ -242,24 +226,24 @@ class Project:
                 outputs = [compdb],
                 implicit = regen_implicits,
                 )
-            build.default([compdb])
             build.newline()
+            default_builds.append(compdb)
 
             build.build(
                 rule = "regen",
                 outputs = ['build.ninja'],
                 implicit = regen_implicits,
                 implicit_outputs = 
-                    [f"module.{mod.name}.ninja" for mod in modules.values()] +
+                    [f"module.{mod.name}.ninja" for mod in modules.used.values()] +
-                    [f"{target}/target.ninja" for target in targets] +
+                    [f"{target}/target.ninja" for target in modules.targets] +
-                    [boot_config],
+                    extra_regen_outputs,
                 )
 
             build.newline()
-            build.default(["${build_root}/jsix.img"])
+            build.default(default_builds)
 
-        for target in targets:
+        for target in modules.targets:
-            mods = [m.name for m in modules.values() if target in m.targets]
+            mods = modules.target_mods(target)
 
             targetdir = output / target
             targetdir.mkdir(exist_ok=True)
@@ -290,4 +274,4 @@ class Project:
                         build.newline()
 
                 for mod in mods:
-                    build.subninja(f"module.{mod}.ninja")
+                    build.subninja(f"module.{mod.name}.ninja")

src/boot/boot.module

@@ -3,9 +3,10 @@
 boot = module("boot",
     kind = "exe",
     outfile = "boot.efi",
-    targets = [ "boot" ],
+    target = "boot",
     deps = [ "cpu", "elf", "util", "bootproto" ],
     static = True,
+    skip_arches = [ "linux" ],
     sources = [
         "allocator.cpp",
         "bootconfig.cpp",

src/kernel/kernel.module

@@ -3,11 +3,12 @@
 kernel = module("kernel",
     default = True,
     basename = "jsix",
-    targets = [ "kernel" ],
+    target = "kernel",
     description = "jsix kernel",
     deps = [ "util", "cpu", "bootproto", "j6", "acpi" ],
     static = True,
     ld_script = "kernel.ld",
+    skip_arches = [ "linux" ],
     sources = [
         "apic.cpp",
         "kassert.cpp",

src/kernel/panic.serial/serial.module

@@ -1,12 +1,13 @@
 # vim: ft=python
 
 panic = module("panic.serial",
-    targets = [ "kernel" ],
+    target = "kernel",
     deps = [ "util", "elf", "kernel" ],
     static = True,
     includes = [ ".." ],
     description = "Serial panic handler",
     ld_script = "panic.serial.ld",
+    skip_arches = [ "linux" ],
     sources = [
         "display.cpp",
         "entry.s",

src/libraries/j6/j6.module

@@ -14,10 +14,17 @@ j6 = module("j6",
         "protocols/service_locator.cpp",
         "protocols/vfs.cpp",
         "ring_buffer.cpp",
-        "syscalls.s.cog",
         "sysconf.cpp.cog",
         "syslog.cpp",
     ],
+    arch_source = {
+        "amd64": [
+            "syscalls.s.cog",
+        ],
+        "linux": [
+            "linux/syscalls.s.cog",
+        ],
+    },
     public_headers = [
         "j6/cap_flags.h.cog",
         "j6/channel.hh",

src/libraries/j6/linux/syscalls.s.cog

@@ -0,0 +1,49 @@
+; vim: ft=asm
+
+%macro define_syscall 2
+    global j6_%1: function (j6_%1.end - j6_%1)
+    j6_%1:
+        push rbp
+        mov rbp, rsp
+
+        ; if the syscall has more than 6 arguments, the rest
+        ; will be pushed on the stack. in that case, we'd need
+        ; to pass this stack pointer to the kernel, so stash
+        ; off rbx (callee-saved) and pass the pointer to the
+        ; arguments there.
+        push rbx
+        mov rbx, rbp
+        add rbx, 16 ; account for stack frame
+
+        ; args should already be in rdi, etc, but rcx will
+        ; get stomped, so stash it in r10, which isn't a
+        ; callee-saved register, but also isn't used in the
+        ; function call ABI.
+        mov r10, rcx
+
+        mov rax, %2
+        syscall
+        ; result is now already in rax, so just return
+
+        pop rbx
+        pop rbp
+        ret
+    .end:
+%endmacro
+
+; [[[cog code generation
+;   from definitions.context import Context
+;
+;   ctx = Context(definitions_path)
+;   ctx.parse("syscalls.def")
+;   syscalls = ctx.interfaces['syscalls']
+;
+;   if target != "kernel":
+;       for id, scope, method in syscalls.methods:
+;           if scope:
+;               name = f"{scope.name}_{method.name}"
+;           else:
+;               name = method.name
+;           cog.outl(f"define_syscall {name:20}, {id}")
+; ]]]
+; [[[end]]]

src/user/6s/6s.module

@@ -1,7 +1,7 @@
 # vim: ft=python
 
 module("6s",
-    targets = [ "user" ],
+    target = "user",
     deps = [ "libc", "edit" ],
     description = "j6 shell",
     sources = [

src/user/drv.uart/uart.module

@@ -1,7 +1,7 @@
 # vim: ft=python
 
 module("drv.uart",
-    targets = [ "user" ],
+    target = "user",
     deps = [ "libc", "util" ],
     description = "UART driver",
     drivers = [ "pc.uart" ],

src/user/drv.uefi_fb/uefi_fb.module

@@ -1,7 +1,7 @@
 # vim: ft=python
 
 module("drv.uefi_fb",
-    targets = [ "user" ],
+    target = "user",
     deps = [ "libc", "bootproto" ],
     description = "UEFI framebuffer driver",
     drivers = [ "uefi.fb" ],

src/user/ld.so/ld.so.module

@@ -4,7 +4,7 @@ ldso = module("ld.so",
     kind = "lib",
     static = True,
     basename = "ld",
-    targets = [ "user" ],
+    target = "user",
     deps = [ "libc", "util", "elf" ],
     description = "Dynamic Linker",
     sources = [

src/user/srv.init/init.module

@@ -1,7 +1,7 @@
 # vim: ft=python
 
 init = module("srv.init",
-    targets = [ "init" ],
+    target = "init",
     deps = [ "libc", "elf", "bootproto", "zstd", "acpi", "pci" ],
     static = True,
     description = "Init server",

src/user/srv.logger/logger.module

@@ -1,7 +1,7 @@
 # vim: ft=python
 
 module("srv.logger",
-    targets = [ "user" ],
+    target = "user",
     deps = [ "libc" ],
     description = "Logging server",
     sources = [

src/user/test_runner/test_runner.module

@@ -1,7 +1,7 @@
 # vim: ft=python
 
 module("test_runner",
-    targets = [ "user" ],
+    target = "user",
     deps = [ "libc", "util" ],
     description = "Unit test runner",
     sources = [

src/user/testapp/testapp.module

@@ -1,7 +1,7 @@
 # vim: ft=python
 
 module("testapp",
-    targets = [ "user" ],
+    target = "user",
     deps = [ "libc" ],
     description = "Testbed app",
     sources = [