Refactoring build system for more control of inputs

generate_build.py

@@ -7,48 +7,89 @@ program = namedtuple('program', ['path', 'deps', 'output', 'targets'])
 source = namedtuple('source', ['name', 'input', 'output', 'action'])
 version = namedtuple('version', ['major', 'minor', 'patch', 'sha', 'dirty'])
 
-MODULES = {
+MODULES = {}
-    "elf":       library('src/libraries/elf', ['kutil']),
-    "initrd":    library('src/libraries/initrd', ["kutil"]),
-    "kutil":     library('src/libraries/kutil', []),
 
-    "makerd":    program('src/tools/makerd', ["initrd", "kutil"], "makerd", ["native"]),
+class Source:
+    Actions = {'.c': 'cc', '.cpp': 'cxx', '.s': 'nasm'}
 
-    "boot":      program('src/boot', [], "boot.elf", ["boot"]),
+    def __init__(self, path, root, modroot):
+        from os.path import relpath, splitext
+        self.input = path
+        self.name = relpath(path, root)
+        self.output = relpath(path, modroot) + ".o"
+        self.action = self.Actions.get(splitext(path)[1], None)
 
-    "nulldrv":   program('src/drivers/nulldrv', [], "nulldrv", ["host"]),
+    def __str__(self):
-    "kernel":    program('src/kernel', ["elf", "initrd", "kutil"], "popcorn.elf", ["host"]),
+        return "{} {}:{}:{}".format(self.action, self.output, self.name, self.input)
-}
+
+
+class Module:
+    def __init__(self, name, output, root, **kwargs):
+        from os.path import commonpath, dirname, isdir, join
+
+        self.name = name
+        self.output = output
+        self.kind = kwargs.get("kind", "exe")
+        self.target = kwargs.get("target", None)
+        self.deps = kwargs.get("deps", tuple())
+        self.includes = kwargs.get("includes", tuple())
+        self.depmods = []
+
+        sources = [join(root, f) for f in kwargs.get("source", tuple())]
+        modroot = commonpath(sources)
+        while not isdir(modroot):
+            modroot = dirname(modroot)
+
+        self.sources = [Source(f, root, modroot) for f in sources]
+
+    def __str__(self):
+        return "Module {} {}\n\t".format(self.kind, self.name)
+
+    def __find_depmods(self, modules):
+        self.depmods = set()
+        open_list = set(self.deps)
+        closed_list = set()
+
+        while open_list:
+            dep = modules[open_list.pop()]
+            open_list |= (set(dep.deps) - closed_list)
+            self.depmods.add(dep)
+
+        self.libdeps = [d for d in self.depmods if d.kind == "lib"]
+        self.exedeps = [d for d in self.depmods if d.kind != "lib"]
+
+    @classmethod
+    def load(cls, filename):
+        from os.path import abspath, dirname
+        from yaml import load
+
+        root = dirname(filename)
+        modules = {}
+        moddata = load(open(filename, "r"))
+
+        for name, data in moddata.items():
+            modules[name] = cls(name, root=root, **data)
+
+        for mod in modules.values():
+            mod.__find_depmods(modules)
+
+        targets = {}
+        for mod in modules.values():
+            if mod.target is None: continue
+            if mod.target not in targets:
+                targets[mod.target] = set()
+            targets[mod.target].add(mod)
+            targets[mod.target] |= mod.depmods
+
+        return modules.values(), targets
 
 
 def get_template(env, typename, name):
     from jinja2.exceptions import TemplateNotFound
     try:
-        return env.get_template("{}.{}.ninja.j2".format(typename, name))
+        return env.get_template("{}.{}.j2".format(typename, name))
     except TemplateNotFound:
-        return env.get_template("{}.default.ninja.j2".format(typename))
+        return env.get_template("{}.default.j2".format(typename))
-
-
-def get_sources(path, srcroot):
-    import os
-    from os.path import abspath, join, relpath, splitext
-
-    actions = {'.c': 'cc', '.cpp': 'cxx', '.s': 'nasm'}
-
-    sources = []
-    for root, dirs, files in os.walk(path):
-        for f in files:
-            base, ext = splitext(f)
-            if not ext in actions: continue
-            name = join(root, f)
-            sources.append(
-                source(
-                    relpath(name, srcroot),
-                    abspath(name),
-                    relpath(abspath(name), path) + ".o",
-                    actions[ext]))
-
-    return sources
 
 
 def get_git_version():
@@ -77,15 +118,17 @@ def get_git_version():
         dirty)
 
 
-def main(buildroot):
+def main(buildroot="build", modulefile="modules.yaml"):
     import os
-    from os.path import abspath, dirname, isdir, join
+    from os.path import abspath, dirname, isabs, isdir, join
 
     generator = abspath(__file__)
     srcroot = dirname(generator)
+    if not isabs(modulefile):
+        modulefile = join(srcroot, modulefile)
 
-    if buildroot is None:
+    if not isabs(buildroot):
-        buildroot = join(srcroot, "build")
+        buildroot = join(srcroot, buildroot)
 
     if not isdir(buildroot):
         os.mkdir(buildroot)
@@ -95,35 +138,21 @@ def main(buildroot):
         git_version.major, git_version.minor, git_version.patch, git_version.sha))
 
     from jinja2 import Environment, FileSystemLoader
-    env = Environment(loader=FileSystemLoader(join(srcroot, "scripts", "templates")))
+    template_dir = join(srcroot, "scripts", "templates")
+    env = Environment(loader=FileSystemLoader(template_dir))
 
-    targets = {}
-    templates = set()
     buildfiles = []
-    for name, mod in MODULES.items():
+    templates = set()
-        if isinstance(mod, program):
+    modules, targets = Module.load(modulefile)
-            for target in mod.targets:
-                if not target in targets:
-                    targets[target] = set()
 
-                open_list = [name]
+    for mod in modules:
-                while open_list:
+        buildfile = join(buildroot, mod.name + ".ninja")
-                    depname = open_list.pop()
-                    dep = MODULES[depname]
-                    open_list.extend(dep.deps)
-                    targets[target].add(depname)
-
-        sources = get_sources(join(srcroot, mod.path), join(srcroot, "src"))
-        buildfile = join(buildroot, name + ".ninja")
         buildfiles.append(buildfile)
         with open(buildfile, 'w') as out:
-            #print("Generating module", name)
+            template = get_template(env, mod.kind, mod.name)
-            template = get_template(env, type(mod).__name__, name)
             templates.add(template.filename)
             out.write(template.render(
-                name=name,
                 module=mod,
-                sources=sources,
                 buildfile=buildfile,
                 version=git_version))
 
@@ -135,7 +164,6 @@ def main(buildroot):
         buildfile = join(root, "target.ninja")
         buildfiles.append(buildfile)
         with open(buildfile, 'w') as out:
-            #print("Generating target", target)
             template = get_template(env, "target", target)
             templates.add(template.filename)
             out.write(template.render(
@@ -151,8 +179,7 @@ def main(buildroot):
     buildfiles.append(buildfile)
 
     with open(join(buildroot, buildfile), 'w') as out:
-        #print("Generating main build.ninja")
+        template = env.get_template("build.ninja.j2")
-        template = env.get_template('build.ninja.j2')
         templates.add(template.filename)
 
         out.write(template.render(
@@ -163,11 +190,9 @@ def main(buildroot):
             buildfiles=buildfiles,
             templates=[abspath(f) for f in templates],
             generator=generator,
+            modulefile=modulefile,
             version=git_version))
 
 if __name__ == "__main__":
     import sys
-    buildroot = None
+    main(*sys.argv[1:])
-    if len(sys.argv) > 1:
-        buildroot = sys.argv[1]
-    main(buildroot)

modules.yaml

@@ -0,0 +1,113 @@
+kernel:
+    output: popcorn.elf
+    target: host
+    deps:
+        - elf
+        - initrd
+        - kutil
+    includes:
+        - src/kernel
+    source:
+        - src/kernel/crti.s
+        - src/kernel/apic.cpp
+        - src/kernel/assert.cpp
+        - src/kernel/boot.s
+        - src/kernel/console.cpp
+        - src/kernel/cpprt.cpp
+        - src/kernel/cpu.cpp
+        - src/kernel/debug.cpp
+        - src/kernel/debug.s
+        - src/kernel/device_manager.cpp
+        - src/kernel/font.cpp
+        - src/kernel/fs/gpt.cpp
+        - src/kernel/gdt.cpp
+        - src/kernel/gdt.s
+        - src/kernel/interrupts.cpp
+        - src/kernel/interrupts.s
+        - src/kernel/io.cpp
+        - src/kernel/loader.s
+        - src/kernel/log.cpp
+        - src/kernel/main.cpp
+        - src/kernel/memory_bootstrap.cpp
+        - src/kernel/msr.cpp
+        - src/kernel/page_manager.cpp
+        - src/kernel/pci.cpp
+        - src/kernel/process.cpp
+        - src/kernel/scheduler.cpp
+        - src/kernel/screen.cpp
+        - src/kernel/serial.cpp
+        - src/kernel/syscall.cpp
+        - src/kernel/syscall.s
+        - src/kernel/crtn.s
+
+elf:
+    kind: lib
+    output: libelf.a
+    deps:
+        - kutil
+    includes:
+        - src/libraries/elf/include
+    source:
+        - src/libraries/elf/elf.cpp
+
+initrd:
+    kind: lib
+    output: libinitrd.a
+    deps:
+        - kutil
+    includes:
+        - src/libraries/initrd/include
+    source:
+        - src/libraries/initrd/initrd.cpp
+
+kutil:
+    kind: lib
+    output: libkutil.a
+    includes:
+        - src/libraries/kutil/include
+    source:
+        - src/libraries/kutil/assert.cpp
+        - src/libraries/kutil/memory.cpp
+        - src/libraries/kutil/memory_manager.cpp
+
+makerd:
+    kind: exe
+    target: native
+    output: makerd
+    deps:
+        - initrd
+    source:
+        - src/tools/makerd/entry.cpp
+        - src/tools/makerd/main.cpp
+
+tests:
+    kind: exe
+    target: native
+    output: tests
+    deps:
+        - kutil
+    source:
+        - src/tests/linked_list.cpp
+        - src/tests/main.cpp
+        - src/tests/memory_manager.cpp
+
+boot:
+    kind: exe
+    target: boot
+    output: boot.elf
+    source:
+        - src/boot/crt0.s
+        - src/boot/console.cpp
+        - src/boot/guids.cpp
+        - src/boot/loader.cpp
+        - src/boot/main.cpp
+        - src/boot/memory.cpp
+        - src/boot/reloc.cpp
+        - src/boot/utility.cpp
+
+nulldrv:
+    kind: exe
+    target: host
+    output: nulldrv
+    source:
+        - src/drivers/nulldrv/main.s

scripts/templates/build.ninja.j2

@@ -1,6 +1,7 @@
 ninja_required_version = 1.3
 builddir = {{ buildroot }}
 srcroot = {{ srcroot }}
+modulefile = {{ modulefile }}
 
 warnflags = $
     -Wformat=2 $
@@ -75,7 +76,7 @@ rule lib
 rule regen
     generator = true
     description = Regenrating build files
-    command = {{ generator }} $builddir
+    command = {{ generator }} $builddir $modulefile
 
 rule cp
     description = Copying $name
@@ -117,6 +118,7 @@ build $
     {%- for template in templates %}
     {{ template }} $
     {%- endfor %}
+    $modulefile $
     {{ generator }}
 
 build $builddir/flash.img : cp $srcroot/assets/ovmf/x64/OVMF.fd

scripts/templates/exe.boot.j2

@@ -1,4 +1,4 @@
-{% extends "program.default.ninja.j2" %}
+{% extends "exe.default.j2" %}
 {% block variables %}
 {{ super() }}
 

scripts/templates/exe.default.j2

@@ -0,0 +1,12 @@
+{% extends "module.base.j2" %}
+{% block variables %}
+{{ super() }}
+
+libs = $
+    -L${builddir} $
+    {%- for dep in module.libdeps %}
+    -l{{ dep.name }} $
+    {%- endfor %}
+    $libs
+
+{% endblock %}

scripts/templates/exe.kernel.j2

@@ -1,4 +1,4 @@
-{% extends "program.default.ninja.j2" %}
+{% extends "exe.default.j2" %}
 {% block variables %}
 {{ super() }}
 

scripts/templates/lib.default.j2

@@ -0,0 +1 @@
+{% extends "module.base.j2" %}

scripts/templates/library.default.ninja.j2

@@ -1 +0,0 @@
-{% extends "module.base.ninja.j2" %}

scripts/templates/module.base.j2

@@ -0,0 +1,38 @@
+moddir = ${builddir}/{{ module.name }}.dir
+
+{% block variables %}
+ccflags = $ccflags $
+    {%- for dep in module.depmods %}
+    {%- for inc in dep.includes %}
+    -I${srcroot}/{{ inc }} $
+    {%- endfor %}
+    {%- endfor %}
+    {%- for inc in module.includes %}
+    -I${srcroot}/{{ inc }} $
+    {%- endfor %}
+{% endblock %}
+
+{% for source in module.sources %}
+build ${moddir}/{{ source.output }} : {{ source.action }} {{ source.input }} || {{ buildfile }}
+    name = {{ source.name }}
+{% endfor %}
+
+build ${builddir}/{{ module.output }} : {{ module.kind }} $
+{%- for source in module.sources %}
+    ${moddir}/{{ source.output }} $
+{%- endfor -%}
+{%- for dep in module.libdeps %}
+    ${builddir}/{{ dep.output }} $
+{%- endfor %}
+    | $
+{% for dep in module.exedeps %}
+    ${builddir}/{{ dep.output }} $
+{%- endfor -%}
+    {{ buildfile }}
+    name = {{ module.name }}
+
+{% block extra %}
+{% endblock %}
+
+# vim: ft=ninja et ts=4 sts=4 sw=4
+

scripts/templates/module.base.ninja.j2

@@ -1,29 +0,0 @@
-moddir = ${builddir}/{{ name }}.dir
-
-{% block variables %}
-ccflags = $ccflags $
-    {%- for dep in module.deps %}
-    -I${srcroot}/src/libraries/{{ dep }}/include $
-    {%- endfor %}
-    -I${srcroot}/{{ module.path }} $
-    -I${srcroot}/{{ module.path }}/include
-{% endblock %}
-
-{% for source in sources %}
-build ${moddir}/{{ source.output }} : {{ source.action }} {{ source.input }} || {{ buildfile }}
-    name = {{ source.name }}
-{% endfor %}
-
-build {% block artifact %} ${builddir}/lib{{ name }}.a : lib {% endblock %} $
-{%- block extrasources %}{% endblock -%}
-{%- for source in sources %}
-    ${moddir}/{{ source.output }}{% if not loop.last %} ${% endif %}
-{%- endfor -%}
-{%- if module.deps %}| {% for dep in module.deps %} ${builddir}/lib{{ dep }}.a {% endfor %}{% endif %}
-    name = {{ name }}
-
-{% block extra %}
-{% endblock %}
-
-# vim: ft=ninja et ts=4 sts=4 sw=4
-

scripts/templates/program.default.ninja.j2

@@ -1,13 +0,0 @@
-{% extends "module.base.ninja.j2" %}
-{% block variables %}
-{{ super() }}
-
-libs = $
-    -L${builddir} $
-    {%- for dep in module.deps %}
-    -l{{dep}} $
-    {%- endfor %}
-    $libs
-
-{% endblock %}
-{% block artifact %}${builddir}/{{ module.output }} : exe{% endblock %}

scripts/templates/target.boot.j2

@@ -1,4 +1,4 @@
-{% extends "target.default.ninja.j2" %}
+{% extends "target.default.j2" %}
 
 {% block binaries %}
 ld = ld

scripts/templates/target.default.j2

@@ -13,5 +13,5 @@ objcopy = objcopy
 {% endblock %}
 
 {% for module in modules %}
-subninja {{ module }}.ninja
+subninja {{ module.name }}.ninja
 {% endfor %}

scripts/templates/target.host.j2

@@ -1,4 +1,5 @@
-{% extends "target.default.ninja.j2" %}
+{% extends "target.default.j2" %}
+
 {% block binaries %}
 cc = ${srcroot}/sysroot/bin/clang
 cxx = ${srcroot}/sysroot/bin/clang++

scripts/templates/target.native.ninja.j2

@@ -1 +0,0 @@
-{% extends "target.default.ninja.j2" %}