[build] Add build knowledge of dynamic libraries

Bonnibel will now build dynamic libraries when they're dependencies for
non-statically linked modules. It will also copy those shared libraries
into the initrd image for programs being copied into the image.

assets/build/rules.ninja

@@ -1,5 +1,3 @@
-# This file is automatically generated by bonnibel
-
 rule compile.c
   command = $cc -MMD -MF $out.d $cflags $ccflags -o $out -c $in
   description = Compiling [$target]:$name
@@ -42,15 +40,19 @@ rule parse.cog
 
 rule exe
   command = $ld $ldflags -o $out $in $libs
-  description = Linking $name
-
-rule lib
-  command = $ar qcs $out $in
-  description = Archiving [$target]:$name
+  description = Linking exe [$target]:$name
 
 rule driver
-  command = $ld $ldflags -shared -o $out $in $libs
-  description = Linking $name
+  command = $ld $ldflags -o $out $in $libs
+  description = Linking driver [$target]:$name
+
+rule lib
+  command = $ld -shared $ldflags -o $out $in $libs
+  description = Linking [$target]:$name
+
+rule lib_static
+  command = $ar qcs $out $in
+  description = Archiving [$target]:$name
 
 rule cp
   command = cp $in $out

assets/build/target.user.exe.yaml

@@ -5,6 +5,6 @@ ccflags: [
 
 ldflags: [
   "-pie",
-  "--dynamic-linker", "/jsix/tools/ld.so",
-  "-lc++", "-lc++abi", "-lunwind",
+  "--dynamic-linker", "/jsix/lib/ld.so",
+  "--push-state", "--as-needed", "-Bstatic", "-lc++", "-lc++abi", "-lunwind", "--pop-state",
 ]

assets/build/target.user.yaml

@@ -15,6 +15,7 @@ ccflags: [
   "-U__linux__",
 
   "--sysroot='${source_root}/sysroot'",
+  "-fpic",
 ]
 
 cxxflags: [
@@ -29,5 +30,4 @@ ldflags: [
   "-L", "${source_root}/sysroot/lib",
   "-z", "separate-code",
   "--no-dependent-libraries",
-  "-Bstatic",
 ]

assets/manifests/default.yaml

@@ -12,5 +12,5 @@ services:
 drivers:
   - drv.uart
   - drv.uefi_fb
-tools:
+libs:
   - ld.so

external/zstd.module

@@ -5,7 +5,6 @@ zstd = module("zstd",
     kind = "lib",
     copy_headers = True,
     deps = [ "libc" ],
-    output = "libzstd.a",
     sources = [
         "decompress/zstd_decompress.c",
         "decompress/zstd_ddict.c",

scripts/bonnibel/manifest.py

@@ -37,8 +37,12 @@ class Manifest:
         self.drivers = [self.__build_entry(modules, i)
                 for i in config.get("drivers", tuple())]
 
-        self.tools = [self.__build_entry(modules, i)
-                for i in config.get("tools", tuple())]
+        libs = set(config.get("libs", tuple()))
+        libs.update(self.__libdeps([modules[e.module] for e in self.services]))
+        libs.update(self.__libdeps([modules[e.module] for e in self.drivers]))
+
+        self.libs = [self.__build_entry(modules, i)
+                for i in libs]
 
         self.flags = config.get("flags", tuple())
 
@@ -74,7 +78,14 @@ class Manifest:
             if not f in Manifest.flags:
                 raise BonnibelError(f"Manifest specifies unknown flag '{f}'")
 
-        return Manifest.Entry(name, target, mod.output, flags)
+        return Manifest.Entry(name, target, mod.get_output(), flags)
+
+    def __libdeps(self, modules):
+        deps = set([m.name for m in modules if m.kind == "lib"])
+        for m in modules:
+            if m.static: continue
+            deps.update(self.__libdeps(m.depmods))
+        return deps
 
     def add_data(self, output, desc, flags=tuple()):
         e = Manifest.Entry(None, None, output, flags)

scripts/bonnibel/module.py

@@ -1,4 +1,5 @@
 from . import include_rel, mod_rel, target_rel
+from . import BonnibelError
 
 def resolve(path):
     if path.startswith('/') or path.startswith('$'):
@@ -17,17 +18,27 @@ class BuildOptions:
 
     @property
     def implicit(self):
+        libfiles = list(map(lambda x: x[0], self.libs))
         if self.ld_script is not None:
-            return self.libs + [self.ld_script]
+            return libfiles + [self.ld_script]
         else:
-            return self.libs
+            return libfiles
+
+    @property
+    def linker_args(self):
+        from pathlib import Path
+        def arg(path, static):
+            if static: return path
+            return "-l:" + Path(path).name
+        return [arg(*l) for l in self.libs]
 
 
 class Module:
     __fields = {
         # name: (type, default)
         "kind": (str, "exe"),
-        "output": (str, None),
+        "outfile": (str, None),
+        "basename": (str, None),
         "targets": (set, ()),
         "deps":  (set, ()),
         "public_headers":  (set, ()),
@@ -41,6 +52,7 @@ class Module:
         "description": (str, None),
         "no_libc": (bool, False),
         "ld_script": (str, None),
+        "static": (bool, False),
     }
 
     def __init__(self, name, modfile, root, **kwargs):
@@ -62,7 +74,7 @@ class Module:
 
         for name in kwargs:
             if not name in self.__fields:
-                raise AttributeError(f"No attribute named {name} on Module")
+                raise AttributeError(f"No attribute named {name} on Module ({modfile})")
 
         if not self.no_libc:
             self.deps.add("libc_free")
@@ -78,24 +90,30 @@ class Module:
         # Filled by Module.update
         self.depmods = set()
 
-    def __str__(self):
-        return "Module {} {}\n\t{}".format(self.kind, self.name, "\n\t".join(map(str, self.sources)))
+    def __repr__(self):
+        return f"<Module {self.kind} {self.name}>"
 
     @property
-    def output(self):
-        if self.__output is not None:
-            return self.__output
-
+    def basename(self):
+        if self.__basename is not None:
+            return self.__basename
         if self.kind == "lib":
-            return f"lib{self.name}.a"
-        elif self.kind == "driver":
-            return f"{self.name}.drv"
+            return f"lib{self.name}"
         else:
-            return f"{self.name}.elf"
+            return self.name
 
-    @output.setter
-    def output(self, value):
-        self.__output = value
+    @basename.setter
+    def basename(self, value):
+        self.__basename = value
+
+    def get_output(self, static=False):
+        if self.outfile is not None:
+            return self.outfile
+        elif self.kind == "headers":
+            return None
+        else:
+            ext = dict(exe=".elf", driver=".drv", lib=(static and ".a" or ".so"))
+            return self.basename + ext.get(self.kind, "")
 
     @classmethod
     def update(cls, mods):
@@ -128,21 +146,20 @@ class Module:
         from collections import defaultdict
         from ninja.ninja_syntax import Writer
 
-        def walk_deps(deps):
-            open_set = set(deps)
-            closed_set = set()
-            while open_set:
-                dep = open_set.pop()
-                closed_set.add(dep)
-                open_set |= {m for m in dep.depmods if not m in closed_set}
-            return closed_set
+        def walk_deps(deps, static, results):
+            for mod in deps:
+                if static or mod.name not in results:
+                    results[mod.name] = (mod, static)
+                walk_deps(mod.depmods, static or mod.static, results)
 
-        all_deps = walk_deps(self.depmods)
+        all_deps = {}
+        walk_deps(self.depmods, self.static, all_deps)
+        all_deps = all_deps.values()
 
         def gather_phony(build, deps, child_rel):
             phony = ".headers.phony"
             child_phony = [child_rel(phony, module=c.name)
-                    for c in all_deps]
+                    for c, _ in all_deps]
 
             build.build(
                 rule = "touch",
@@ -192,7 +209,7 @@ class Module:
                     modopts.includes.append(str(incpath))
                     modopts.includes.append(destpath)
 
-            for dep in all_deps:
+            for dep, static in all_deps:
                 if dep.public_headers:
                     if dep.include_phase == "normal":
                         modopts.includes += [dep.root / "include", f"${{target_dir}}/{dep.name}.dir/include"]
@@ -202,10 +219,12 @@ class Module:
                         from . import BonnibelError
                         raise BonnibelError(f"Module {dep.name} has invalid include_phase={dep.include_phase}")
 
-                if dep.kind == "lib":
-                    modopts.libs.append(target_rel(dep.output))
+                if dep.kind == "headers":
+                    continue
+                elif dep.kind == "lib":
+                    modopts.libs.append((target_rel(dep.get_output(static)), static))
                 else:
-                    modopts.order_only.append(target_rel(dep.output))
+                    modopts.order_only.append(target_rel(dep.get_output(static)))
 
             cc_includes = []
             if modopts.local:
@@ -225,7 +244,7 @@ class Module:
                 build.variable("asflags", ["${asflags}"] + as_includes)
 
             if modopts.libs:
-                build.variable("libs", ["${libs}"] + modopts.libs)
+                build.variable("libs", ["-L${target_dir}", "${libs}"] + modopts.linker_args)
 
             if modopts.ld_script:
                 build.variable("ldflags", ["${ldflags}"] + ["-T", modopts.ld_script])
@@ -267,7 +286,11 @@ class Module:
 
             gather_phony(build, header_deps, target_rel)
 
-            output = target_rel(self.output)
+            if self.kind == "headers":
+                # Header-only, don't output a build rule
+                return
+
+            output = target_rel(self.get_output())
             build.newline()
             build.build(
                 rule = self.kind,
@@ -275,6 +298,7 @@ class Module:
                 inputs = inputs,
                 implicit = modopts.implicit,
                 order_only = modopts.order_only,
+                variables = {"name": self.name},
             )
 
             dump = output + ".dump"
@@ -286,6 +310,26 @@ class Module:
                 variables = {"name": self.name},
             )
 
+            s_output = target_rel(self.get_output(static=True))
+            if s_output != output:
+                build.newline()
+                build.build(
+                    rule = self.kind + "_static",
+                    outputs = s_output,
+                    inputs = inputs,
+                    order_only = modopts.order_only,
+                    variables = {"name": self.name},
+                )
+
+                dump = s_output + ".dump"
+                build.newline()
+                build.build(
+                    rule = "dump",
+                    outputs = dump,
+                    inputs = s_output,
+                    variables = {"name": self.name},
+                )
+
             if self.default:
                 build.newline()
                 build.default(output)

scripts/bonnibel/project.py

@@ -154,8 +154,8 @@ class Project:
             for program in manifest.drivers:
                 add_initrd_stripped("jsix/drivers", program)
 
-            for program in manifest.tools:
-                add_initrd_stripped("jsix/tools", program)
+            for program in manifest.libs:
+                add_initrd_stripped("jsix/lib", program)
 
             syms = manifest.add_data("symbol_table.dat",
                     "Symbol table", ("symbols",))
@@ -166,7 +166,7 @@ class Project:
             build.build(
                 rule = "makest",
                 outputs = [syms_out],
-                inputs = [f"${{build_root}}/kernel/{modules['kernel'].output}"],
+                inputs = [f"${{build_root}}/kernel/{modules['kernel'].get_output(static=True)}"],
                 )
             fatroot_content.append(syms_out)
             manifest.symbols = syms_file

src/boot/boot.module

@@ -2,9 +2,10 @@
 
 boot = module("boot",
     kind = "exe",
-    output = "boot.efi",
+    outfile = "boot.efi",
     targets = [ "boot" ],
     deps = [ "cpu", "elf", "util", "bootproto" ],
+    static = True,
     sources = [
         "allocator.cpp",
         "bootconfig.cpp",

src/kernel/kernel.module

@@ -1,12 +1,12 @@
 # vim: ft=python
 
 kernel = module("kernel",
-    kind = "exe",
     default = True,
-    output = "jsix.elf",
+    basename = "jsix",
     targets = [ "kernel" ],
     description = "jsix kernel",
     deps = [ "util", "cpu", "bootproto", "j6" ],
+    static = True,
     ld_script = "kernel.ld",
     sources = [
         "apic.cpp",

src/kernel/panic.serial/serial.module

@@ -1,9 +1,9 @@
 # vim: ft=python
 
 panic = module("panic.serial",
-    kind = "exe",
     targets = [ "kernel" ],
     deps = [ "util", "elf", "kernel" ],
+    static = True,
     includes = [ ".." ],
     description = "Serial panic handler",
     ld_script = "panic.serial.ld",

src/libraries/bootproto/bootproto.module

@@ -1,7 +1,7 @@
 # vim: ft=python
 
 bp = module("bootproto",
-    kind = "lib",
+    kind = "headers",
     public_headers = [
         "bootproto/acpi.h",
         "bootproto/bootconfig.h",

src/libraries/libc_free/libc_free.module

@@ -1,7 +1,8 @@
 # vim: ft=python
 
 libc_free = module("libc_free",
-    kind = "lib",
+    kind = "headers",
+    basename = "libc_free",
     no_libc = True,
     include_phase = "late",
     public_headers = [

src/user/ld.so/ld.so.module

@@ -1,8 +1,9 @@
 # vim: ft=python
 
 ldso = module("ld.so",
-    kind = "shared",
-    output = "ld.so",
+    kind = "lib",
+    static = True,
+    basename = "ld",
     targets = [ "user" ],
     deps = [ "libc", "util", "elf" ],
     description = "Dynamic Linker",

src/user/srv.init/init.module

@@ -3,6 +3,7 @@
 init = module("srv.init",
     targets = [ "init" ],
     deps = [ "libc", "elf", "bootproto", "zstd" ],
+    static = True,
     description = "Init server",
     ld_script = "init.ld",
     sources = [