[project] Generate syscalls from new interface DSL

This change adds a new interface DSL for specifying objects (with
methods) and interfaces (that expose objects, and optionally have their
own methods).

Significant changes:

- Add the new scripts/definitions Python module to parse the DSL
- Add the new definitions directory containing DSL definition files
- Use cog to generate syscall-related code in kernel and libj6
- Unify ordering of pointer + length pairs in interfaces

scripts/definitions/context.py

@@ -0,0 +1,65 @@
+class NotFound(Exception): pass
+
+class Context:
+    def __init__(self, path, verbose=False):
+        if isinstance(path, str):
+            self.__paths = [path]
+        else:
+            self.__paths = path
+
+        self.__closed = set()
+        self.__verbose = verbose
+
+        self.__deps = {}
+
+        self.objects = dict()
+        self.interfaces = dict()
+
+    verbose = property(lambda self: self.__verbose)
+
+    def find(self, filename):
+        from os.path import exists, isabs, join
+
+        if exists(filename) or isabs(filename):
+            return filename
+
+        for path in self.__paths:
+            full = join(path, filename)
+            if exists(full):
+                return full
+
+        raise NotFound(filename)
+
+    def parse(self, filename):
+        pending = set()
+        pending.add(filename)
+
+        from .parser import Lark_StandAlone as Parser
+        from .transformer import DefTransformer
+
+        objects = {}
+        interfaces = {}
+
+        while pending:
+            name = pending.pop()
+            self.__closed.add(name)
+
+            path = self.find(name)
+
+            parser = Parser(transformer=DefTransformer(name))
+            imps, objs, ints = parser.parse(open(path, "r").read())
+            objects.update(objs)
+            interfaces.update(ints)
+
+            self.__deps[name] = imps
+
+            pending.update(imps.difference(self.__closed))
+
+        from .types import ObjectRef
+        ObjectRef.connect(objects)
+
+        self.objects.update(objects)
+        self.interfaces.update(interfaces)
+
+    def deps(self):
+        return {self.find(k): tuple(map(self.find, v)) for k, v in self.__deps.items()}

scripts/definitions/errors.py

@@ -0,0 +1,2 @@
+class InvalidType(Exception): pass
+class UnknownTypeError(Exception): pass

scripts/definitions/generate.py

@@ -0,0 +1,62 @@
+def generate_template(template, outfile, **kwargs):
+    from hashlib import md5
+    from os import makedirs
+    from os.path import dirname, exists
+
+    content = template.render(**kwargs)
+    h = md5(content.encode('utf-8')).hexdigest()
+
+    if exists(outfile):
+        existing = open(outfile, 'r').read().encode('utf-8')
+        if md5(existing).hexdigest() == h:
+            return False
+
+    makedirs(dirname(outfile), exist_ok=True)
+    open(outfile, 'w').write(content)
+
+    return True
+
+def generate(ctx, outdir, template_type):
+    from os.path import basename, join, split, splitext
+    from jinja2 import Environment, PackageLoader
+
+    for name, interface in ctx.interfaces.items():
+        base = "_".join(sorted(interface.options))
+
+        path = join("templates", base, template_type)
+        env = Environment(
+            loader = PackageLoader('definitions', package_path=path),
+            trim_blocks = True, lstrip_blocks = True)
+
+        env.filters
+
+        for template_name in env.list_templates():
+            template = env.get_template(template_name)
+
+            basepath, filename = split(template_name)
+            filename, ext = splitext(filename)
+
+            if filename == "_object_":
+                for obj in ctx.objects.values():
+                    outfile = join(outdir, basepath, obj.name + ext)
+                    wrote = generate_template(
+                                template, outfile,
+                                filename=obj.name + ext,
+                                basepath=basepath,
+                                object=obj,
+                                interface=interface,
+                                objects=ctx.objects)
+
+                    if wrote and ctx.verbose:
+                        print(f"Writing {outfile}")
+
+            else:
+                outfile = join(outdir, template_name)
+                wrote = generate_template(
+                            template, outfile,
+                            filename=basename(template_name),
+                            interface=interface,
+                            objects=ctx.objects)
+
+                if wrote and ctx.verbose:
+                    print(f"Writing {outfile}")

scripts/definitions/templates/syscall/client/private/syscall_trampolines.s

@@ -0,0 +1,23 @@
+%macro SYSCALL 2
+	global __syscall_%1
+	__syscall_%1:
+		push rbp
+		mov rbp, rsp
+
+		; args should already be in rdi, etc, but rcx will
+		; get stomped, so stash it in r10, which isn't a
+		; callee-saved register, but also isn't used in the
+		; function call ABI.
+		mov r10, rcx
+
+		mov rax, %2
+		syscall
+		; result is now already in rax, so just return
+
+		pop rbp
+		ret
+%endmacro
+
+{% for id, scope, method in interface.methods %}
+SYSCALL {% if scope %}{{ scope.name }}_{% endif %}{{ method.name }} {{ id }}
+{% endfor %}

scripts/definitions/templates/syscall/client/public/_object_.hh

@@ -0,0 +1,41 @@
+#pragma once
+/// \file {{filename}}
+{% if object.super %}
+
+#include <j6/{{ object.super.name }}.hh>
+{% endif %}
+
+namespace j6 {
+
+{% if object.desc %}
+{{ object.desc|indent('/// ', first=True) }}
+{% endif %}
+class {{ object.name }}
+{% if object.super %}    : public {{ object.super.name }}
+{% endif %}
+{
+public:
+{% macro argument(type, name, first, options=False) -%}
+    {%- for ctype, suffix in type.c_names(options) -%}
+        {%- if not first or not loop.first %}, {% endif %}{{ ctype }} {{ name }}{{ suffix }}
+    {%- endfor -%}
+{%- endmacro -%}
+
+{% for method in object.methods %}
+{% if method.desc %}    /// {{ method.desc|indent('    /// ') }}{% endif %} 
+{% for param in method.params %}
+{% if param.desc %}    /// \arg {{ "%-10s"|format(param.name) }} {{ param.desc }}
+{% endif %}
+{% endfor %}
+    {%+ if method.static %}static {% endif %}j6_status_t {{ method.name }} (
+{%- for param in method.params %}{{ argument(param.type, param.name, loop.first, options=param.options) }}{% endfor -%});
+
+{% endfor %}
+    ~{{ object.name }}();
+
+private:
+    {{ object.name }}(j6_handle_t handle) : m_handle {handle} {}
+    j6_handle_t m_handle;
+}
+
+} // namespace j6

scripts/definitions/templates/syscall/client/public/syscalls.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <j6/types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+{% macro argument(type, name, first, options=False) -%}
+    {%- for ctype, suffix in type.c_names(options) -%}
+        {%- if not first or not loop.first %}, {% endif %}{{ ctype }} {{ name }}{{ suffix }}
+    {%- endfor -%}
+{%- endmacro %}
+ 
+{% for id, scope, method in interface.methods %}
+j6_status_t __syscall_{% if scope %}{{ scope.name }}_{% endif %}{{ method.name }} (
+    {%- if not method.static -%}{{ argument(scope.reftype, "self", True) }}{% endif -%}
+    {%- set first = method.static -%}
+    {%- for param in method.params %}{{ argument(param.type, param.name, first, options=param.options) }}{% set first = False %}{% endfor -%}
+);
+{% endfor %}
+
+#ifdef __cplusplus
+}
+#endif
+

scripts/definitions/transformer.py

@@ -0,0 +1,142 @@
+from .parser import Transformer, v_args
+
+def get_opts(args):
+    from .types import Description, Options, Type, UID
+
+    kinds = {
+        Description: "desc",
+        Options: "opts",
+        UID: "uid",
+        Type: "typename",
+    }
+
+    result = dict()
+    outargs = []
+    for a in args:
+        for kind, name in kinds.items():
+            if isinstance(a, kind):
+                result[name] = a
+                break
+        else:
+            outargs.append(a)
+
+    return result, outargs
+
+class DefTransformer(Transformer):
+    def __init__(self, filename):
+        self.filename = filename
+
+    def start(self, args):
+        from .types import Import, Interface, Object
+
+        imports = set()
+        objects = dict()
+        interfaces = dict()
+
+        for o in args:
+            if isinstance(o, Object):
+                objects[o.name] = o
+
+            elif isinstance(o, Interface):
+                interfaces[o.name] = o
+
+            elif isinstance(o, Import):
+                imports.add(o)
+
+        return imports, objects, interfaces
+
+    @v_args(inline=True)
+    def import_statement(self, path):
+        from .types import Import
+        return Import(path)
+
+    def object(self, args):
+        from .types import Object
+        specials, args = get_opts(args)
+        name, args = args[0], args[1:]
+        return Object(name, children=args, **specials)
+
+    def interface(self, args):
+        from .types import Interface
+        specials, args = get_opts(args)
+        name, args = args[0], args[1:]
+        return Interface(name, children=args, **specials)
+
+    def method(self, args):
+        from .types import Method
+        specials, args = get_opts(args)
+        name, args = args[0], args[1:]
+        return Method(name, children=args, **specials)
+
+    def function(self, args):
+        from .types import Function
+        specials, args = get_opts(args)
+        name, args = args[0], args[1:]
+        return Function(name, children=args, **specials)
+
+    def param(self, args):
+        from .types import Param
+        specials, args = get_opts(args)
+        name = args[0]
+        return Param(name, **specials)
+
+    @v_args(inline=True)
+    def expose(self, s):
+        from .types import Expose
+        return Expose(s)
+
+    @v_args(inline=True)
+    def uid(self, s):
+        return s
+
+    @v_args(inline=True)
+    def name(self, s):
+        return s
+
+    @v_args(inline=True)
+    def type(self, s):
+        return s
+
+    @v_args(inline=True)
+    def super(self, s):
+        from .types import ObjectRef
+        return ObjectRef(s, self.filename)
+
+    def options(self, args):
+        from .types import Options
+        return Options([str(s) for s in args])
+
+    def description(self, s):
+        from .types import Description
+        return Description("\n".join(s))
+
+    @v_args(inline=True)
+    def object_name(self, n):
+        from .types import ObjectRef
+        return ObjectRef(n, self.filename)
+
+    def PRIMITIVE(self, s):
+        from .types import get_primitive
+        return get_primitive(s)
+
+    def UID(self, s):
+        from .types import UID
+        return UID(int(s, base=16))
+
+    def INT_TYPE(self, s):
+        return s
+
+    def NUMBER(self, s):
+        if s.startswith("0x"):
+            return int(s,16)
+        return int(s)
+
+    def COMMENT(self, s):
+        return s[2:].strip()
+
+    def IDENTIFIER(self, s):
+        return str(s)
+
+    def PATH(self, s):
+        return str(s[1:-1])
+

scripts/definitions/types/__init__.py

@@ -0,0 +1,23 @@
+def _indent(x):
+    from textwrap import indent
+    return indent(str(x), '    ')
+
+class Description(str): pass
+class Import(str): pass
+
+class Options(set):
+    def __str__(self):
+        if not self: return ""
+        return "[{}]".format(" ".join(self))
+
+class UID(int):
+    def __str__(self):
+        return f"{self:016x}"
+
+from .object import Object
+from .interface import Interface, Expose
+from .function import Function, Method, Param
+
+from .type import Type
+from .primitive import get_primitive
+from .objref import ObjectRef

scripts/definitions/types/function.py

@@ -0,0 +1,49 @@
+from . import _indent
+from . import Options
+
+def _hasopt(opt):
+    def test(self):
+        return opt in self.options
+    return test
+
+class Function:
+    typename = "function"
+
+    def __init__(self, name, opts=Options(), desc="", children=tuple()):
+        self.name = name
+        self.options = opts
+        self.desc = desc
+        self.params = [c for c in children if isinstance(c, Param)]
+        self.id = -1
+
+    def __str__(self):
+        parts = ["{} {}".format(self.typename, self.name)]
+        if self.desc:
+            parts.append(_indent(self.desc))
+        if self.options:
+            parts.append(f"    Options: {self.options}")
+        parts.extend(map(_indent, self.params))
+        return "\n".join(parts)
+
+    static = property(lambda x: True)
+    constructor = property(lambda x: False)
+
+
+class Method(Function):
+    typename = "method"
+
+    static = property(_hasopt("static"))
+    constructor = property(_hasopt("constructor"))
+
+
+class Param:
+    def __init__(self, name, typename, opts=Options(), desc=""):
+        self.name = name
+        self.type = typename
+        self.options = opts
+        self.desc = desc
+
+    def __str__(self):
+        return "param {} {} {}  {}".format(
+                self.name, repr(self.type), self.options, self.desc or "")
+

scripts/definitions/types/interface.py

@@ -0,0 +1,43 @@
+from . import _indent
+from . import Options
+
+class Expose(object):
+    def __init__(self, type):
+        self.type = type
+
+    def __repr__(self):
+        return f'expose {repr(self.type)}'
+
+class Interface:
+    def __init__(self, name, uid, opts=Options(), desc="", children=tuple()):
+        from .function import Function
+
+        self.name = name
+        self.uid = uid
+        self.options = opts
+        self.desc = desc
+
+        self.functions = [c for c in children if isinstance(c, Function)]
+        self.exposes = [e.type for e in children if isinstance(e, Expose)]
+
+    def __str__(self):
+        parts = [f"interface {self.name}: {self.uid}"]
+        if self.desc:
+            parts.append(_indent(self.desc))
+        if self.options:
+            parts.append(f"    Options: {self.options}")
+        parts.extend(map(_indent, self.exposes))
+        parts.extend(map(_indent, self.functions))
+        return "\n".join(parts)
+
+    def __methods(self):
+        mm = [(i, None, self.functions[i]) for i in range(len(self.functions))]
+
+        base = len(mm)
+        for o in [e.object for e in self.exposes]:
+            mm.extend([(base + i, o, o.methods[i]) for i in range(len(o.methods))])
+            base += len(o.methods)
+
+        return mm
+
+    methods = property(__methods)

scripts/definitions/types/object.py

@@ -0,0 +1,25 @@
+from . import _indent
+from . import Options
+
+class Object:
+    def __init__(self, name, uid, typename=None, opts=Options(), desc="", children=tuple()):
+        self.name = name
+        self.uid = uid
+        self.options = opts
+        self.desc = desc
+        self.super = typename
+        self.methods = children
+
+        from . import ObjectRef
+        self.__ref = ObjectRef(name)
+
+    def __str__(self):
+        parts = [f"object {self.name}: {self.uid}"]
+        if self.desc:
+            parts.append(_indent(self.desc))
+        if self.options:
+            parts.append(f"    Options: {self.options}")
+        parts.extend(map(_indent, self.methods))
+        return "\n".join(parts)
+
+    reftype = property(lambda self: self.__ref)

scripts/definitions/types/objref.py

@@ -0,0 +1,44 @@
+from .type import Type
+
+class ObjectRef(Type):
+    all_refs = {}
+
+    def __init__(self, name, filename=None):
+        super().__init__(name)
+        self.__c_type = "j6_handle_t"
+        self.__object = None
+        ObjectRef.all_refs[self] = filename
+
+    def __repr__(self):
+        return f'ObjectRef({self.name})'
+
+    object = property(lambda self: self.__object)
+
+    def c_names(self, options):
+        one = self.__c_type
+        out = bool({"out", "inout"}.intersection(options))
+
+        if "list" in options:
+            two = "size_t"
+            if out:
+                one = f"const {one} *"
+                two += " *"
+            else:
+                one = f"{one} *"
+            return ((one, ""), (two, "_count"))
+
+        else:
+            if out:
+                one += " *"
+            return ((one, ""),)
+
+    def cxx_names(self, options):
+        return self.c_names(options)
+
+    @classmethod
+    def connect(cls, objects):
+        for ref, filename in cls.all_refs.items():
+            ref.__object = objects.get(ref.name)
+            if ref.__object is None:
+                from ..errors import UnknownTypeError
+                raise UnknownTypeError(ref.name, filename)

scripts/definitions/types/primitive.py

@@ -0,0 +1,70 @@
+from .type import Type
+
+class Primitive(Type):
+    def __init__(self, name, c_type):
+        super().__init__(name)
+        self.c_type = c_type
+
+    def __repr__(self):
+        return f'Primitive({self.name})'
+
+    def c_names(self, options=None):
+        one = self.c_type
+        if options and "out" in options or "inout" in options:
+            one += " *"
+
+        return ((one, ""),)
+
+    def cxx_names(self, options):
+        return self.c_names(options)
+
+class PrimitiveRef(Primitive):
+    def __init__(self, name, c_type, counted=False):
+        super().__init__(name, c_type)
+        self.__counted = counted
+
+    def c_names(self, options=None):
+        one = f"{self.c_type} *"
+        two = "size_t"
+
+        if options and "out" in options or "inout" in options:
+            two += " *"
+        else:
+            one = "const " + one
+
+        if self.__counted:
+            return ((one, ""), (two, "_len"))
+        else:
+            return ((one, ""),)
+
+    def cxx_names(self, options):
+        return self.c_names(options)
+
+_primitives = {
+    "string": PrimitiveRef("string", "char"),
+    "buffer": PrimitiveRef("buffer", "void", counted=True),
+
+    "int": Primitive("int", "int"),
+    "uint": Primitive("uint", "unsigned"),
+    "size": Primitive("size", "size_t"),
+    "address": Primitive("address", "uintptr_t"),
+
+    "int8": Primitive("int8", "int8_t"),
+    "uint8": Primitive("uint8", "uint8_t"),
+
+    "int16": Primitive("int16", "int16_t"),
+    "uint16": Primitive("uint16", "uint16_t"),
+
+    "int32": Primitive("int32", "int32_t"),
+    "uint32": Primitive("uint32", "uint32_t"),
+
+    "int64": Primitive("int64", "int64_t"),
+    "uint64": Primitive("uint64", "uint64_t"),
+}
+
+def get_primitive(name):
+    p = _primitives.get(name)
+    if not p:
+        from ..errors import InvalidType
+        raise InvalidType(name)
+    return p

scripts/definitions/types/type.py

@@ -0,0 +1,12 @@
+class Type:
+    def __init__(self, name):
+        self.__name = name
+
+    name = property(lambda self: self.__name)
+
+    def c_names(self, options):
+        raise NotImplemented("Call to base Type.c_names")
+
+    def cxx_names(self, options):
+        raise NotImplemented("Call to base Type.c_names")
+