justin/jsix_import
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

[build] Move headers out of target dirs

Browse Source 
The great header shift: It didn't make sense to regenerate headers for
the same module for every target (boot/kernel/user) it appeared in. And
now that core headers are out of src/include, this was going to cause
problems for the new libc changes I've been working on. So I went back
to re-design how module headers work.

Pre-requisites:
- A module's public headers should all be available in one location, not
  tied to target.
- No accidental includes. Another module should not be able to include
  anything (creating an implicit dependency) from a module without
  declaring an explicit dependency.
- Exception to the previous: libc's headers should be available to all,
  at least for the freestanding headers.

New system:
- A new "public_headers" property of module declares all public headers
  that should be available to dependant modules
- All public headers (after possible processing) are installed relative
  to build/include/<module> with the same path as their source
- This also means no "include" dir in modules is necessary. If a header
  should be included as <j6/types.h> then its source should be
  src/libraries/j6/j6/types.h - this caused the most churn as all public
  header sources moved one directory up.
- The "includes" property of a module is local only to that module now,
  it does not create any implicit public interface

Other changes:
- The bonnibel concept of sources changed: instead of sources having
  actions, they themselves are an instance of a (sub)class of Source,
  which provides all the necessary information itself.
- Along with the above, rule names were standardized into <type>.<ext>,
  eg "compile.cpp" or "parse.cog"
- cog and cogflags variables moved from per-target scope to global scope
  in the build files.
- libc gained a more dynamic .module file
This commit is contained in:
Justin C. Miller
2022-02-06 10:18:51 -08:00
parent db23e4966e
commit 4545256b49
103 changed files with 362 additions and 5191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

218
scripts/bonnibel/module.py
View File

@@ -1,9 +1,18 @@
from . import include_rel, mod_rel, target_rel
def resolve(path):
if path.startswith('/') or path.startswith('$'):
return path
from pathlib import Path
return str(Path(path).resolve())
class BuildOptions:
def __init__(self, includes=tuple(), libs=tuple(), order_only=tuple()):
self.includes = list(includes)
self.libs = list(libs)
self.order_only = list(order_only)
class Module:
__fields = {
# name: (type, default)
@@ -11,16 +20,18 @@ class Module:
"output": (str, None),
"targets": (set, ()),
"deps": (set, ()),
"public_headers": (set, ()),
"includes": (tuple, ()),
"sources": (tuple, ()),
"variables": (dict, ()),
"default": (bool, False),
"location": (str, "jsix"),
"description": (str, None),
"no_libc": (bool, False),
}
def __init__(self, name, modfile, root, **kwargs):
from .source import Source
from .source import make_source
# Required fields
self.root = root
@@ -40,7 +51,8 @@ class Module:
raise AttributeError(f"No attribute named {name} on Module")
# Turn strings into real Source objects
self.sources = [Source(root, f) for f in self.sources]
self.sources = [make_source(root, f) for f in self.sources]
self.public_headers = [make_source(root, f) for f in self.public_headers]
# Filled by Module.update
self.depmods = set()
@@ -66,13 +78,17 @@ class Module:
def update(cls, mods):
from . import BonnibelError
for mod in mods.values():
for dep in mod.deps:
def resolve(modlist):
resolved = set()
for dep in modlist:
if not dep in mods:
raise BonnibelError(f"module '{mod.name}' references unknown module '{dep}'")
mod = mods[dep]
resolved.add(mod)
return resolved
depmod = mods[dep]
mod.depmods.add(depmod)
for mod in mods.values():
mod.depmods = resolve(mod.deps)
target_mods = [mod for mod in mods.values() if mod.targets]
for mod in target_mods:
@@ -85,112 +101,150 @@ class Module:
children |= {m for m in child.depmods if not m in closed}
def generate(self, output):
filename = str(output / f"module.{self.name}.ninja")
from pathlib import Path
from collections import defaultdict
from ninja.ninja_syntax import Writer
def walk_deps(deps):
open_set = set(deps)
closed_set = set()
while open_set:
dep = open_set.pop()
closed_set.add(dep)
open_set |= {m for m in dep.depmods if not m in closed_set}
return closed_set
all_deps = walk_deps(self.depmods)
def gather_phony(build, deps, child_rel, add_headers=False):
phony = ".headers.phony"
child_phony = [child_rel(phony, module=c.name)
for c in all_deps]
header_targets = []
if add_headers:
if not self.no_libc:
header_targets.append(f"${{build_root}}/include/libc/{phony}")
if self.public_headers:
header_targets.append(f"${{build_root}}/include/{self.name}/{phony}")
build.build(
rule = "touch",
outputs = [mod_rel(phony)],
implicit = child_phony + header_targets,
order_only = list(map(mod_rel, deps)),
)
filename = str(output / f"headers.{self.name}.ninja")
with open(filename, "w") as buildfile:
from pathlib import Path
from ninja.ninja_syntax import Writer
build = Writer(buildfile)
build.comment("This file is automatically generated by bonnibel")
build.newline()
build.variable("module_dir", f"${{target_dir}}/{self.name}.dir")
build.variable("module_dir", f"${{build_root}}/include/{self.name}")
header_deps = []
inputs = []
headers = set(self.public_headers)
while headers:
source = headers.pop()
headers.update(source.next)
if source.action:
build.newline()
build.build(rule=source.action, **source.args)
if source.gather:
header_deps += list(source.outputs)
if source.input:
inputs.extend(map(mod_rel, source.outputs))
build.newline()
gather_phony(build, header_deps, include_rel)
filename = str(output / f"module.{self.name}.ninja")
with open(filename, "w") as buildfile:
build = Writer(buildfile)
build.comment("This file is automatically generated by bonnibel")
build.newline()
build.variable("module_dir", target_rel(self.name + ".dir"))
modopts = BuildOptions(
includes = [self.root, "${module_dir}"],
)
if self.public_headers:
modopts.includes += [f"${{build_root}}/include/{self.name}"]
for key, value in self.variables.items():
build.variable(key, value)
build.newline()
includes = [self.root, "${module_dir}"]
for include in self.includes:
p = Path(include)
if p.is_absolute():
if not p in includes:
includes.append(str(p.resolve()))
if not p in modopts.includes:
modopts.includes.append(str(p.resolve()))
elif include != ".":
includes.append(str(self.root / p))
includes.append(f"${{module_dir}}/{p}")
incpath = self.root / p
destpath = mod_rel(p)
for header in incpath.rglob("*.h"):
dest_header = f"{destpath}/" + str(header.relative_to(incpath))
modopts.includes.append(str(incpath))
modopts.includes.append(destpath)
libs = []
child_deps = []
order_only = []
closed = set()
children = set(self.depmods)
while children:
child = children.pop()
closed.add(child)
all_deps = walk_deps(self.depmods)
for dep in all_deps:
if dep.public_headers:
modopts.includes += [f"${{build_root}}/include/{dep.name}"]
includes += [f"${{target_dir}}/{child.name}.dir/{i}" for i in child.includes]
includes += [f"{child.root}/{i}" for i in child.includes]
child_deps.append(f"${{target_dir}}/{child.name}.dir/.parse_dep.phony")
if child.kind == "lib":
libs.append(f"${{target_dir}}/{child.output}")
if dep.kind == "lib":
modopts.libs.append(target_rel(dep.output))
else:
order_only.append(f"${{target_dir}}/{child.output}")
children |= {m for m in child.depmods if not m in closed}
modopts.order_only.append(target_rel(dep.output))
if includes:
build.variable("ccflags", ["${ccflags}"] + [f"-I{i}" for i in includes])
build.variable("asflags", ["${asflags}"] + [f"-I{i}" for i in includes])
if modopts.includes:
build.variable("ccflags", ["${ccflags}"] + [f"-I{i}" for i in modopts.includes])
build.variable("asflags", ["${asflags}"] + [f"-I{i}" for i in modopts.includes])
if libs:
build.variable("libs", ["${libs}"] + libs)
if modopts.libs:
build.variable("libs", ["${libs}"] + modopts.libs)
header_deps = []
inputs = []
parse_deps = []
parse_depfile = "${module_dir}/.parse_dep.phony"
sources = set(self.sources)
while sources:
source = sources.pop()
sources.update(source.next)
for start in self.sources:
source = start
if source.action:
build.newline()
build.build(rule=source.action, **source.args)
while source and source.action:
output = source.output
if source.gather:
header_deps += list(source.outputs)
if source.action.parse_deps:
oodeps = [parse_depfile]
else:
oodeps = []
if source.action.rule:
build.newline()
build.build(
rule = source.action.rule,
outputs = output.input,
inputs = source.input,
implicit = list(map(resolve, source.deps)) +
list(source.action.deps),
order_only = oodeps,
variables = {"name": source.name},
)
elif source.action.implicit:
parse_deps.append(source.input)
else:
inputs.append(source.input)
source = output
if source.input:
inputs.extend(map(mod_rel, source.outputs))
build.newline()
build.build(
rule = "touch",
outputs = [parse_depfile],
implicit = child_deps,
order_only = parse_deps,
)
output = f"${{target_dir}}/{self.output}"
dump = f"${{target_dir}}/{self.output}.dump"
gather_phony(build, header_deps, target_rel, add_headers=True)
output = target_rel(self.output)
dump = output + ".dump"
build.newline()
build.build(
rule = self.kind,
outputs = output,
inputs = inputs,
implicit = libs,
order_only = order_only,
implicit = modopts.libs,
order_only = modopts.order_only,
)
build.newline()
@@ -210,9 +264,13 @@ class Module:
from .source import Source
s = Source(self.root, path, **kwargs)
self.sources.append(s)
return str(s.output)
return s.outputs
def add_depends(self, paths, deps):
for source in self.sources:
if source.name in paths:
if source.path in paths:
source.add_deps(deps)
for source in self.public_headers:
if source.path in paths:
source.add_deps(deps)