[build] Move headers out of target dirs

The great header shift: It didn't make sense to regenerate headers for
the same module for every target (boot/kernel/user) it appeared in. And
now that core headers are out of src/include, this was going to cause
problems for the new libc changes I've been working on. So I went back
to re-design how module headers work.

Pre-requisites:
- A module's public headers should all be available in one location, not
  tied to target.
- No accidental includes. Another module should not be able to include
  anything (creating an implicit dependency) from a module without
  declaring an explicit dependency.
- Exception to the previous: libc's headers should be available to all,
  at least for the freestanding headers.

New system:
- A new "public_headers" property of module declares all public headers
  that should be available to dependant modules
- All public headers (after possible processing) are installed relative
  to build/include/<module> with the same path as their source
- This also means no "include" dir in modules is necessary. If a header
  should be included as <j6/types.h> then its source should be
  src/libraries/j6/j6/types.h - this caused the most churn as all public
  header sources moved one directory up.
- The "includes" property of a module is local only to that module now,
  it does not create any implicit public interface

Other changes:
- The bonnibel concept of sources changed: instead of sources having
  actions, they themselves are an instance of a (sub)class of Source,
  which provides all the necessary information itself.
- Along with the above, rule names were standardized into <type>.<ext>,
  eg "compile.cpp" or "parse.cog"
- cog and cogflags variables moved from per-target scope to global scope
  in the build files.
- libc gained a more dynamic .module file

scripts/bonnibel/__init__.py

@@ -1,3 +1,5 @@
+from os.path import join
+
 class BonnibelError(Exception):
     def __init__(self, message):
         self.message = message
@@ -6,3 +8,18 @@ def load_config(filename):
     from yaml import safe_load
     with open(filename, 'r') as infile:
         return safe_load(infile.read())
+
+def mod_rel(path):
+    return join("${module_dir}", path)
+
+def target_rel(path, module=None):
+    if module:
+        return join("${target_dir}", module + ".dir", path)
+    else:
+        return join("${target_dir}", path)
+
+def include_rel(path, module=None):
+    if module:
+        return join("${build_root}", "include", module, path)
+    else:
+        return join("${build_root}", "include", path)

scripts/bonnibel/action.py

@@ -1,18 +0,0 @@
-class Action:
-    _action_map = {
-        '.c': Action_c,
-        '.C': Action_cxx,
-        '.cc': Action_cxx,
-        '.cpp': Action_cxx,
-        '.cxx': Action_cxx,
-        '.c++': Action_cxx,
-        '.s': Action_asm,
-        '.S': Action_asm,
-        '.cog': Action_cog,
-    }
-
-    @classmethod
-    def find(cls, ext):
-        return cls._action_map.get(ext)
-
-class Action_c:

scripts/bonnibel/module.py

@@ -1,9 +1,18 @@
+from . import include_rel, mod_rel, target_rel
+
 def resolve(path):
     if path.startswith('/') or path.startswith('$'):
         return path
     from pathlib import Path
     return str(Path(path).resolve())
 
+class BuildOptions:
+    def __init__(self, includes=tuple(), libs=tuple(), order_only=tuple()):
+        self.includes = list(includes)
+        self.libs = list(libs)
+        self.order_only = list(order_only)
+
+
 class Module:
     __fields = {
         # name: (type, default)
@@ -11,16 +20,18 @@ class Module:
         "output": (str, None),
         "targets": (set, ()),
         "deps":  (set, ()),
+        "public_headers":  (set, ()),
         "includes": (tuple, ()),
         "sources": (tuple, ()),
         "variables": (dict, ()),
         "default": (bool, False),
         "location": (str, "jsix"),
         "description": (str, None),
+        "no_libc": (bool, False),
     }
 
     def __init__(self, name, modfile, root, **kwargs):
-        from .source import Source
+        from .source import make_source
 
         # Required fields
         self.root = root
@@ -40,7 +51,8 @@ class Module:
                 raise AttributeError(f"No attribute named {name} on Module")
 
         # Turn strings into real Source objects
-        self.sources = [Source(root, f) for f in self.sources]
+        self.sources = [make_source(root, f) for f in self.sources]
+        self.public_headers = [make_source(root, f) for f in self.public_headers]
 
         # Filled by Module.update
         self.depmods = set()
@@ -66,13 +78,17 @@ class Module:
     def update(cls, mods):
         from . import BonnibelError
 
-        for mod in mods.values():
-            for dep in mod.deps:
+        def resolve(modlist):
+            resolved = set()
+            for dep in modlist:
                 if not dep in mods:
                     raise BonnibelError(f"module '{mod.name}' references unknown module '{dep}'")
+                mod = mods[dep]
+                resolved.add(mod)
+            return resolved
 
-                depmod = mods[dep]
-                mod.depmods.add(depmod)
+        for mod in mods.values():
+            mod.depmods = resolve(mod.deps)
 
         target_mods = [mod for mod in mods.values() if mod.targets]
         for mod in target_mods:
@@ -85,112 +101,150 @@ class Module:
                 children |= {m for m in child.depmods if not m in closed}
 
     def generate(self, output):
-        filename = str(output / f"module.{self.name}.ninja")
+        from pathlib import Path
+        from collections import defaultdict
+        from ninja.ninja_syntax import Writer
 
+        def walk_deps(deps):
+            open_set = set(deps)
+            closed_set = set()
+            while open_set:
+                dep = open_set.pop()
+                closed_set.add(dep)
+                open_set |= {m for m in dep.depmods if not m in closed_set}
+            return closed_set
+
+        all_deps = walk_deps(self.depmods)
+
+        def gather_phony(build, deps, child_rel, add_headers=False):
+            phony = ".headers.phony"
+            child_phony = [child_rel(phony, module=c.name)
+                    for c in all_deps]
+
+            header_targets = []
+            if add_headers:
+                if not self.no_libc:
+                    header_targets.append(f"${{build_root}}/include/libc/{phony}")
+                if self.public_headers:
+                    header_targets.append(f"${{build_root}}/include/{self.name}/{phony}")
+
+            build.build(
+                rule = "touch",
+                outputs = [mod_rel(phony)],
+                implicit = child_phony + header_targets,
+                order_only = list(map(mod_rel, deps)),
+            )
+
+        filename = str(output / f"headers.{self.name}.ninja")
         with open(filename, "w") as buildfile:
-            from pathlib import Path
-            from ninja.ninja_syntax import Writer
-
             build = Writer(buildfile)
 
             build.comment("This file is automatically generated by bonnibel")
             build.newline()
 
-            build.variable("module_dir", f"${{target_dir}}/{self.name}.dir")
+            build.variable("module_dir", f"${{build_root}}/include/{self.name}")
+
+            header_deps = []
+
+            inputs = []
+            headers = set(self.public_headers)
+            while headers:
+                source = headers.pop()
+                headers.update(source.next)
+
+                if source.action:
+                    build.newline()
+                    build.build(rule=source.action, **source.args)
+
+                if source.gather:
+                    header_deps += list(source.outputs)
+
+                if source.input:
+                    inputs.extend(map(mod_rel, source.outputs))
+
+            build.newline()
+            gather_phony(build, header_deps, include_rel)
+
+        filename = str(output / f"module.{self.name}.ninja")
+        with open(filename, "w") as buildfile:
+            build = Writer(buildfile)
+
+            build.comment("This file is automatically generated by bonnibel")
+            build.newline()
+
+            build.variable("module_dir", target_rel(self.name + ".dir"))
+
+            modopts = BuildOptions(
+                includes = [self.root, "${module_dir}"],
+                )
+            if self.public_headers:
+                modopts.includes += [f"${{build_root}}/include/{self.name}"]
 
             for key, value in self.variables.items():
                 build.variable(key, value)
             build.newline()
 
-            includes = [self.root, "${module_dir}"]
             for include in self.includes:
                 p = Path(include)
                 if p.is_absolute():
-                    if not p in includes:
-                        includes.append(str(p.resolve()))
+                    if not p in modopts.includes:
+                        modopts.includes.append(str(p.resolve()))
                 elif include != ".":
-                    includes.append(str(self.root / p))
-                    includes.append(f"${{module_dir}}/{p}")
+                    incpath = self.root / p
+                    destpath = mod_rel(p)
+                    for header in incpath.rglob("*.h"):
+                        dest_header = f"{destpath}/" + str(header.relative_to(incpath))
+                    modopts.includes.append(str(incpath))
+                    modopts.includes.append(destpath)
 
-            libs = []
-            child_deps = []
-            order_only = []
-            closed = set()
-            children = set(self.depmods)
-            while children:
-                child = children.pop()
-                closed.add(child)
+            all_deps = walk_deps(self.depmods)
+            for dep in all_deps:
+                if dep.public_headers:
+                    modopts.includes += [f"${{build_root}}/include/{dep.name}"]
 
-                includes += [f"${{target_dir}}/{child.name}.dir/{i}" for i in child.includes]
-                includes += [f"{child.root}/{i}" for i in child.includes]
-
-                child_deps.append(f"${{target_dir}}/{child.name}.dir/.parse_dep.phony")
-
-                if child.kind == "lib":
-                    libs.append(f"${{target_dir}}/{child.output}")
+                if dep.kind == "lib":
+                    modopts.libs.append(target_rel(dep.output))
                 else:
-                    order_only.append(f"${{target_dir}}/{child.output}")
-                children |= {m for m in child.depmods if not m in closed}
+                    modopts.order_only.append(target_rel(dep.output))
 
-            if includes:
-                build.variable("ccflags", ["${ccflags}"] + [f"-I{i}" for i in includes])
-                build.variable("asflags", ["${asflags}"] + [f"-I{i}" for i in includes])
+            if modopts.includes:
+                build.variable("ccflags", ["${ccflags}"] + [f"-I{i}" for i in modopts.includes])
+                build.variable("asflags", ["${asflags}"] + [f"-I{i}" for i in modopts.includes])
 
-            if libs:
-                build.variable("libs", ["${libs}"] + libs)
+            if modopts.libs:
+                build.variable("libs", ["${libs}"] + modopts.libs)
+
+            header_deps = []
 
             inputs = []
-            parse_deps = []
-            parse_depfile = "${module_dir}/.parse_dep.phony"
+            sources = set(self.sources)
+            while sources:
+                source = sources.pop()
+                sources.update(source.next)
 
-            for start in self.sources:
-                source = start
+                if source.action:
+                    build.newline()
+                    build.build(rule=source.action, **source.args)
 
-                while source and source.action:
-                    output = source.output
+                if source.gather:
+                    header_deps += list(source.outputs)
 
-                    if source.action.parse_deps:
-                        oodeps = [parse_depfile]
-                    else:
-                        oodeps = []
-
-                    if source.action.rule:
-                        build.newline()
-                        build.build(
-                            rule = source.action.rule,
-                            outputs = output.input,
-                            inputs = source.input,
-                            implicit = list(map(resolve, source.deps)) +
-                                       list(source.action.deps),
-                            order_only = oodeps,
-                            variables = {"name": source.name},
-                        )
-
-                    elif source.action.implicit:
-                        parse_deps.append(source.input)
-
-                    else:
-                        inputs.append(source.input)
-
-                    source = output
+                if source.input:
+                    inputs.extend(map(mod_rel, source.outputs))
 
             build.newline()
-            build.build(
-                rule = "touch",
-                outputs = [parse_depfile],
-                implicit = child_deps,
-                order_only = parse_deps,
-            )
 
-            output = f"${{target_dir}}/{self.output}"
-            dump = f"${{target_dir}}/{self.output}.dump"
+            gather_phony(build, header_deps, target_rel, add_headers=True)
+
+            output = target_rel(self.output)
+            dump = output + ".dump"
             build.newline()
             build.build(
                 rule = self.kind,
                 outputs = output,
                 inputs = inputs,
-                implicit = libs,
-                order_only = order_only,
+                implicit = modopts.libs,
+                order_only = modopts.order_only,
             )
 
             build.newline()
@@ -210,9 +264,13 @@ class Module:
         from .source import Source
         s = Source(self.root, path, **kwargs)
         self.sources.append(s)
-        return str(s.output)
+        return s.outputs
 
     def add_depends(self, paths, deps):
         for source in self.sources:
-            if source.name in paths:
+            if source.path in paths:
+                source.add_deps(deps)
+
+        for source in self.public_headers:
+            if source.path in paths:
                 source.add_deps(deps)

scripts/bonnibel/project.py

@@ -15,7 +15,6 @@ class Project:
         import bonnibel
         from os.path import join
         from ninja.ninja_syntax import Writer
-        from . import load_config
         from .target import Target
 
         targets = set()
@@ -40,10 +39,31 @@ class Project:
             build.variable("version_sha", self.version.sha)
             build.newline()
 
+            build.variable("cogflags", [
+                "-I", "${source_root}/scripts",
+                "-D", "definitions_path=${source_root}/definitions",
+                ])
+            build.newline()
+
             for target in targets:
                 build.subninja(output / target.name / "target.ninja")
             build.newline()
 
+            for mod in modules.values():
+                build.subninja(output / f"headers.{mod.name}.ninja")
+            build.newline()
+
+            build.build(
+                rule = "touch",
+                outputs = "${build_root}/.all_headers",
+                implicit = [f"${{build_root}}/include/{m.name}/.headers.phony"
+                    for m in modules.values() if m.public_headers],
+                )
+            build.build(
+                rule = "phony",
+                outputs = ["all-headers"],
+                inputs = ["${build_root}/.all_headers"])
+
             debugroot = output / ".debug"
             debugroot.mkdir(exist_ok=True)
 
@@ -82,7 +102,6 @@ class Project:
                     })
 
                 add_fatroot(intermediary, entry)
-                return mod.location
 
             from .manifest import Manifest
             manifest = Manifest(manifest_file, modules)

scripts/bonnibel/source.py

@@ -1,87 +1,119 @@
-class Action:
-    name = property(lambda self: self.__name)
-    implicit = property(lambda self: False)
-    rule = property(lambda self: None)
-    deps = property(lambda self: tuple())
-    parse_deps = property(lambda self: False)
+from os.path import join, splitext
+from . import mod_rel
 
-    def __init__(self, name):
-        self.__name = name
-
-    def output_of(self, path):
-        return None
-
-
-class Compile(Action):
-    rule = property(lambda self: f'compile_{self.name}')
-    deps = property(lambda self: ("${module_dir}/.parse_dep.phony",))
-    parse_deps = property(lambda self: True)
-
-    def __init__(self, name, suffix = ".o"):
-        super().__init__(name)
-        self.__suffix = suffix
-
-    def output_of(self, path):
-        return str(path) + self.__suffix
-
-
-class Parse(Action):
-    rule = property(lambda self: f'parse_{self.name}')
-
-    def output_of(self, path):
-        suffix = "." + self.name
-        if path.suffix == suffix:
-            return path.with_suffix('')
+def _resolve(path):
+    if path.startswith('/') or path.startswith('$'):
         return path
+    from pathlib import Path
+    return str(Path(path).resolve())
 
-
-class Link(Action): pass
-
-
-class Header(Action):
-    implicit = property(lambda self: True)
+def _dynamic_action(name):
+    def prop(self):
+        root, suffix = splitext(self.path)
+        return f"{name}{suffix}"
+    return prop
 
 
 class Source:
-    Actions = {
-        '.c': Compile('c'),
-        '.cpp': Compile('cxx'),
-        '.s': Compile('asm'),
-        '.cog': Parse('cog'),
-        '.o': Link('o'),
-        '.h': Header('h'),
-        '.inc': Header('inc'),
-    }
+    next = tuple()
+    action = None
+    args = dict()
+    gather = False
+    outputs = tuple()
+    input = False
 
-    def __init__(self, root, path, output=None, deps=tuple()):
-        from pathlib import Path
-        self.__root = Path(root)
-        self.__path = Path(path)
-        self.__output = output
-        self.__deps = tuple(deps)
-
-    def __str__(self):
-        return self.input
+    def __init__(self, path, root = "${module_dir}", deps=tuple()): 
+        self.path = path
+        self.root = root
+        self.deps = deps
 
     def add_deps(self, deps):
-        self.__deps += tuple(deps)
+        self.deps += tuple(deps)
 
     @property
-    def action(self):
-        suffix = self.__path.suffix
-        return self.Actions.get(suffix)
+    def fullpath(self):
+        return join(self.root, self.path)
+
+class ParseSource(Source):
+    action = property(_dynamic_action("parse"))
 
     @property
     def output(self):
-        if not self.action:
-            return None
+        root, _ = splitext(self.path)
+        return root
 
-        path = self.__output
-        if path is None:
-            path = self.action.output_of(self.__path)
+    @property
+    def outputs(self):
+        return (self.output,)
 
-        return path and Source("${module_dir}", path)
+    @property
+    def gather(self):
+        _, suffix = splitext(self.output)
+        return suffix in (".h", ".inc")
 
-    deps = property(lambda self: self.__deps)
-    name = property(lambda self: str(self.__path))
-    input = property(lambda self: str(self.__root / self.__path))
+    @property
+    def next(self):
+        _, suffix = splitext(self.output)
+        nextType = {
+            ".s": CompileSource,
+            ".cpp": CompileSource,
+            }.get(suffix)
+
+        if nextType:
+            return (nextType(self.output),)
+        return tuple()
+
+    @property
+    def args(self):
+        return dict(
+            outputs = list(map(mod_rel, self.outputs)),
+            inputs = [self.fullpath],
+            implicit = list(map(_resolve, self.deps)),
+            variables = dict(name=self.path),
+            )
+
+class HeaderSource(Source):
+    action = "cp"
+    gather = True
+
+    @property
+    def outputs(self):
+        return (self.path,)
+
+    @property
+    def args(self):
+        return dict(
+            outputs = [mod_rel(self.path)],
+            inputs = [join(self.root, self.path)],
+            implicit = list(map(_resolve, self.deps)),
+            variables = dict(name=self.path),
+            )
+
+class CompileSource(Source):
+    action = property(_dynamic_action("compile"))
+    input = True
+
+    @property
+    def outputs(self):
+        return (self.path + ".o",)
+
+    @property
+    def args(self):
+        return dict(
+            outputs = list(map(mod_rel, self.outputs)),
+            inputs = [join(self.root, self.path)],
+            implicit = list(map(_resolve, self.deps)) + [mod_rel(".headers.phony")],
+            variables = dict(name=self.path),
+            )
+
+def make_source(root, path):
+    _, suffix = splitext(path)
+
+    if suffix in (".s", ".c", ".cpp"):
+        return CompileSource(path, root)
+    elif suffix in (".cog",):
+        return ParseSource(path, root)
+    elif suffix in (".h", ".inc"):
+        return HeaderSource(path, root)
+    else:
+        raise RuntimeError(f"{path} has no Source type")