[all] Reference headers in src instead of copying

This is the second of two big changes to clean up includes throughout
the project. Since I've started using clangd with Neovim and using
VSCode's intellisense, my former strategy of copying all header files
into place in `build/include` means that the real files don't show up in
`compile_commands.json` and so display many include errors when viewing
those header files in those tools.

That setup was mostly predicated on a desire to keep directory depths
small, but really I don't think paths like `src/libraries/j6/j6` are
much better than `src/libraries/j6/include/j6`, and the latter doesn't
have the aforementioned issues, and is clearer to the casual observer as
well.

Some additional changes:

- Added a new module flag `copy_headers` for behavior similar to the old
  style, but placing headers in `$module_dir/include` instead of the
  global `build/include`. This was needed for external projects that
  don't follow the same source/headers folder structure - in this case,
  `zstd`.
- There is no longer an associated `headers.*.ninja` for each
  `module.*.ninja` file, as only parsed headers need to be listed; this
  functionality has been moved back into the module's ninja file.

assets/debugging/jsix.elf-gdb.py

@@ -300,7 +300,7 @@ class DumpLogCommand(gdb.Command):
 
         self.areas = []
         area_re = re.compile(r"LOG\(\s*(\w+).*")
-        with open("src/libraries/j6/j6/tables/log_areas.inc", 'r') as areas_inc:
+        with open("src/libraries/j6/include/j6/tables/log_areas.inc", 'r') as areas_inc:
             for line in areas_inc:
                 m = area_re.match(line)
                 if m:

external/zstd.module

@@ -3,6 +3,7 @@
 zstd = module("zstd",
     root = "${source_root}/external/zstd",
     kind = "lib",
+    copy_headers = True,
     deps = [ "libc" ],
     output = "libzstd.a",
     sources = [

scripts/bonnibel/module.py

@@ -31,6 +31,7 @@ class Module:
         "targets": (set, ()),
         "deps":  (set, ()),
         "public_headers":  (set, ()),
+        "copy_headers": (bool, False),
         "includes": (tuple, ()),
         "include_phase": (str, "normal"),
         "sources": (tuple, ()),
@@ -43,10 +44,11 @@ class Module:
     }
 
     def __init__(self, name, modfile, root, **kwargs):
-        from .source import make_source
+        from pathlib import Path
+        from .source import make_source, make_copy_source
 
         # Required fields
-        self.root = root
+        self.root = Path(root)
         self.name = name
         self.modfile = modfile
 
@@ -67,7 +69,11 @@ class Module:
 
         # Turn strings into real Source objects
         self.sources = [make_source(root, f) for f in self.sources]
-        self.public_headers = [make_source(root, f) for f in self.public_headers]
+
+        header_source = lambda f: make_source(root, Path("include") / f)
+        if self.copy_headers:
+            header_source = lambda f: make_copy_source(root, f, "include")
+        self.public_headers = [header_source(f) for f in self.public_headers]
 
         # Filled by Module.update
         self.depmods = set()
@@ -131,53 +137,18 @@ class Module:
 
         all_deps = walk_deps(self.depmods)
 
-        def gather_phony(build, deps, child_rel, add_headers=False):
+        def gather_phony(build, deps, child_rel):
             phony = ".headers.phony"
             child_phony = [child_rel(phony, module=c.name)
                     for c in all_deps]
 
-            header_targets = []
-            if add_headers:
-                if self.public_headers:
-                    header_targets.append(f"${{build_root}}/include/{self.name}/{phony}")
-
             build.build(
                 rule = "touch",
                 outputs = [mod_rel(phony)],
-                implicit = child_phony + header_targets,
+                implicit = child_phony,
                 order_only = list(map(mod_rel, deps)),
             )
 
-        filename = str(output / f"headers.{self.name}.ninja")
-        with open(filename, "w") as buildfile:
-            build = Writer(buildfile)
-
-            build.comment("This file is automatically generated by bonnibel")
-            build.newline()
-
-            build.variable("module_dir", f"${{build_root}}/include/{self.name}")
-
-            header_deps = []
-
-            inputs = []
-            headers = set(self.public_headers)
-            while headers:
-                source = headers.pop()
-                headers.update(source.next)
-
-                if source.action:
-                    build.newline()
-                    build.build(rule=source.action, **source.args)
-
-                if source.gather:
-                    header_deps += list(source.outputs)
-
-                if source.input:
-                    inputs.extend(map(mod_rel, source.outputs))
-
-            build.newline()
-            gather_phony(build, header_deps, include_rel)
-
         filename = str(output / f"module.{self.name}.ninja")
         with open(filename, "w") as buildfile:
             build = Writer(buildfile)
@@ -192,7 +163,10 @@ class Module:
                 ld_script = self.ld_script and self.root / self.ld_script,
                 )
             if self.public_headers:
-                modopts.includes += [f"${{build_root}}/include/{self.name}"]
+                modopts.includes += [
+                    self.root / "include",
+                    f"${{target_dir}}/{self.name}.dir/include",
+                ]
 
             for key, value in self.variables.items():
                 build.variable(key, value)
@@ -214,9 +188,9 @@ class Module:
             for dep in all_deps:
                 if dep.public_headers:
                     if dep.include_phase == "normal":
-                        modopts.includes += [f"${{build_root}}/include/{dep.name}"]
+                        modopts.includes += [dep.root / "include", f"${{target_dir}}/{dep.name}.dir/include"]
                     elif dep.include_phase == "late":
-                        modopts.late += [f"${{build_root}}/include/{dep.name}"]
+                        modopts.late += [dep.root / "include", f"${{target_dir}}/{dep.name}.dir/include"]
                     else:
                         from . import BonnibelError
                         raise BonnibelError(f"Module {dep.name} has invalid include_phase={dep.include_phase}")
@@ -250,6 +224,23 @@ class Module:
                 build.variable("ldflags", ["${ldflags}"] + ["-T", modopts.ld_script])
 
             header_deps = []
+            inputs = []
+            headers = set(self.public_headers)
+            while headers:
+                source = headers.pop()
+                headers.update(source.next)
+
+                if source.action:
+                    build.newline()
+                    build.build(rule=source.action, **source.args)
+
+                if source.gather:
+                    header_deps += list(source.outputs)
+
+                if source.input:
+                    inputs.extend(map(mod_rel, source.outputs))
+
+            build.newline()
 
             inputs = []
             sources = set(self.sources)
@@ -267,9 +258,7 @@ class Module:
                 if source.input:
                     inputs.extend(map(mod_rel, source.outputs))
 
-            build.newline()
+            gather_phony(build, header_deps, target_rel)
-
-            gather_phony(build, header_deps, target_rel, add_headers=True)
 
             output = target_rel(self.output)
             build.newline()

scripts/bonnibel/project.py

@@ -50,10 +50,6 @@ class Project:
                 build.subninja(output / target.name / "target.ninja")
             build.newline()
 
-            for mod in modules.values():
-                build.subninja(output / f"headers.{mod.name}.ninja")
-            build.newline()
-
             build.build(
                 rule = "touch",
                 outputs = "${build_root}/.all_headers",

scripts/bonnibel/source.py

@@ -72,22 +72,21 @@ class ParseSource(Source):
             variables = dict(name=self.path),
             )
 
-class HeaderSource(Source):
+class CopySource(ParseSource):
     action = "cp"
-    gather = True
+
+    def __init__(self, path, root = "${module_dir}", deps=tuple(), prefix = ""): 
+        self.path = path
+        self.root = root
+        self.deps = deps
+        self.prefix = prefix
 
     @property
-    def outputs(self):
+    def output(self):
-        return (self.path,)
+        from pathlib import Path
+        return Path(self.prefix) / self.path
 
-    @property
+class HeaderSource(Source): pass
-    def args(self):
-        return dict(
-            outputs = [mod_rel(self.path)],
-            inputs = [join(self.root, self.path)],
-            implicit = list(map(_resolve, self.deps)),
-            variables = dict(name=self.path),
-            )
 
 class CompileSource(Source):
     action = property(_dynamic_action("compile"))
@@ -117,3 +116,6 @@ def make_source(root, path):
         return HeaderSource(path, root)
     else:
         raise RuntimeError(f"{path} has no Source type")
+
+def make_copy_source(root, path, prefix = ""):
+    return CopySource(path, root, prefix=prefix)

src/libraries/bootproto/bootproto.module

@@ -14,4 +14,4 @@ bp = module("bootproto",
 from os.path import join
 
 layout = join(source_root, "definitions/memory_layout.yaml")
-bp.add_depends("bootproto/memory.h.cog", deps=[layout])
+bp.add_depends(["bootproto/memory.h.cog"], deps=[layout])

src/libraries/j6/include/j6/memutils.h

@@ -15,4 +15,4 @@ void *memset(void *s, int c, size_t n);
 
 #ifdef __cplusplus
 } // extern "C"
-#endif
+#endif

src/libraries/libc/include/string.h

@@ -13,6 +13,7 @@
 #include <__j6libc/null.h>
 #include <__j6libc/restrict.h>
 #include <__j6libc/size_t.h>
+#include <j6/memutils.h> // memcpy, memmove, memset are defined in libj6
 
 #ifdef __cplusplus
 extern "C" {
@@ -20,8 +21,6 @@ extern "C" {
 
 // Copying functions
 //
-void *memcpy(void * restrict s1, const void * restrict s2, size_t n);
-void *memmove(void * restrict s1, const void * restrict s2, size_t n);
 char *strncpy(char * restrict s1, const char * restrict s2, size_t n);
 
 // Concatenation functions
@@ -49,7 +48,6 @@ char *strtok(char * restrict s1, const char * restrict s2);
 
 // Miscellaneous functions
 //
-void *memset(void *s, int c, size_t n);
 char *strerror(int errnum);
 size_t strlen(const char *s);
 

src/libraries/libc/libc.module

@@ -5,14 +5,16 @@
 # many files that it's unweildy. So if a file is added or removed in
 # libc, remember to run configure again.
 
-def glob(ext):
+def glob(ext, root=''):
     from glob import glob
+    from pathlib import Path
+
+    base = Path(module_root) / root
 
     def resolve(path):
-        from pathlib import Path
+        return str(Path(path).relative_to(base))
-        return str(Path(path).relative_to(module_root))
 
-    return list(map(resolve, glob(f"{module_root}/**/*.{ext}", recursive=True)))
+    return list(map(resolve, glob(f"{base}/**/*.{ext}", recursive=True)))
 
 sources = []
 for ext in ("c", "cpp", "s", "inc"):
@@ -20,7 +22,7 @@ for ext in ("c", "cpp", "s", "inc"):
 
 headers = []
 for ext in ("h",):
-    headers += glob(ext) + glob(ext + ".cog")
+    headers += glob(ext, "include") + glob(ext + ".cog", "include")
 
 libc = module("libc",
     kind = "lib",

src/libraries/util/include/util/basic_types.h

@@ -90,4 +90,4 @@ template<typename T> T&& forward(typename types::remove_reference<T>::type&  par
 
 template<typename T> void swap(T &t1, T &t2) { T tmp = move(t1); t1 = move(t2); t2 = move(tmp); }
 
-} // namespace util
+} // namespace util