From 186724e751d952eb06dfbf5037ca7a7da13516c0 Mon Sep 17 00:00:00 2001 From: "Justin C. Miller" Date: Mon, 30 Aug 2021 01:05:32 -0700 Subject: [PATCH] [project] Generate syscalls from new interface DSL This change adds a new interface DSL for specifying objects (with methods) and interfaces (that expose objects, and optionally have their own methods). Significant changes: - Add the new scripts/definitions Python module to parse the DSL - Add the new definitions directory containing DSL definition files - Use cog to generate syscall-related code in kernel and libj6 - Unify ordering of pointer + length pairs in interfaces --- assets/grammars/definitions.g | 42 + definitions/objects/channel.def | 3 + definitions/objects/endpoint.def | 30 + definitions/objects/kobject.def | 25 + definitions/objects/mailbox.def | 33 + definitions/objects/process.def | 27 + definitions/objects/system.def | 29 + definitions/objects/thread.def | 18 + definitions/objects/vma.def | 29 + definitions/syscalls.def | 27 + scripts/definitions/__init__.py | 0 scripts/definitions/context.py | 65 + scripts/definitions/errors.py | 2 + scripts/definitions/generate.py | 62 + scripts/definitions/parser.py | 2809 +++++++++++++++++ .../client/private/syscall_trampolines.s | 23 + .../syscall/client/public/_object_.hh | 41 + .../syscall/client/public/syscalls.h | 25 + scripts/definitions/transformer.py | 142 + scripts/definitions/types/__init__.py | 23 + scripts/definitions/types/function.py | 49 + scripts/definitions/types/interface.py | 43 + scripts/definitions/types/object.py | 25 + scripts/definitions/types/objref.py | 44 + scripts/definitions/types/primitive.py | 70 + scripts/definitions/types/type.py | 12 + scripts/idgen | 11 + src/kernel/kernel.module | 15 +- src/kernel/objects/endpoint.cpp | 10 +- src/kernel/objects/endpoint.h | 11 +- src/kernel/syscall.cpp | 52 - src/kernel/syscall.cpp.cog | 56 + src/kernel/syscall.h | 23 - src/kernel/syscall.h.cog | 60 + src/kernel/syscall.s | 5 +- src/kernel/syscalls.inc.cog | 11 + src/kernel/syscalls/endpoint.cpp | 24 +- src/kernel/syscalls/object.cpp | 20 +- src/kernel/syscalls/process.cpp | 6 +- src/kernel/syscalls/system.cpp | 12 +- src/kernel/syscalls/thread.cpp | 17 +- src/kernel/vm_space.cpp | 2 +- src/kernel/vm_space.h | 2 +- src/libraries/j6/include/j6/syscalls.h | 15 - src/libraries/j6/include/j6/syscalls.h.cog | 39 + src/libraries/j6/j6.module | 9 +- src/libraries/j6/syscalls.s | 28 - src/libraries/j6/syscalls.s.cog | 37 + src/user/drv.uefi_fb/main.cpp | 8 +- src/user/srv.init/main.cpp | 4 +- src/user/srv.init/modules.cpp | 2 +- src/user/testapp/main.cpp | 54 +- 52 files changed, 4025 insertions(+), 206 deletions(-) create mode 100644 assets/grammars/definitions.g create mode 100644 definitions/objects/channel.def create mode 100644 definitions/objects/endpoint.def create mode 100644 definitions/objects/kobject.def create mode 100644 definitions/objects/mailbox.def create mode 100644 definitions/objects/process.def create mode 100644 definitions/objects/system.def create mode 100644 definitions/objects/thread.def create mode 100644 definitions/objects/vma.def create mode 100644 definitions/syscalls.def create mode 100644 scripts/definitions/__init__.py create mode 100644 scripts/definitions/context.py create mode 100644 scripts/definitions/errors.py create mode 100644 scripts/definitions/generate.py create mode 100644 scripts/definitions/parser.py create mode 100644 scripts/definitions/templates/syscall/client/private/syscall_trampolines.s create mode 100644 scripts/definitions/templates/syscall/client/public/_object_.hh create mode 100644 scripts/definitions/templates/syscall/client/public/syscalls.h create mode 100644 scripts/definitions/transformer.py create mode 100644 scripts/definitions/types/__init__.py create mode 100644 scripts/definitions/types/function.py create mode 100644 scripts/definitions/types/interface.py create mode 100644 scripts/definitions/types/object.py create mode 100644 scripts/definitions/types/objref.py create mode 100644 scripts/definitions/types/primitive.py create mode 100644 scripts/definitions/types/type.py create mode 100755 scripts/idgen delete mode 100644 src/kernel/syscall.cpp create mode 100644 src/kernel/syscall.cpp.cog delete mode 100644 src/kernel/syscall.h create mode 100644 src/kernel/syscall.h.cog create mode 100644 src/kernel/syscalls.inc.cog delete mode 100644 src/libraries/j6/include/j6/syscalls.h create mode 100644 src/libraries/j6/include/j6/syscalls.h.cog delete mode 100644 src/libraries/j6/syscalls.s create mode 100644 src/libraries/j6/syscalls.s.cog diff --git a/assets/grammars/definitions.g b/assets/grammars/definitions.g new file mode 100644 index 0000000..1523515 --- /dev/null +++ b/assets/grammars/definitions.g @@ -0,0 +1,42 @@ +start: import_statement* (object|interface)+ + +import_statement: "import" PATH + +object: description? "object" name options? super? "{" uid method* "}" + +interface: description? "interface" name options? "{" uid interface_param* "}" + +?interface_param: expose | function + +expose: "expose" type + +uid: "uid" UID + +super: ":" name + +function: description? "function" name options? ("{" param* "}")? + +method: description? "method" name options? ("{" param* "}")? + +param: "param" name type options? description? + +?type: PRIMITIVE | object_name + +object_name: "object" name + +id: NUMBER +name: IDENTIFIER +options: "[" IDENTIFIER+ "]" +description: COMMENT+ + +PRIMITIVE: INT_TYPE | "size" | "string" | "buffer" | "address" +INT_TYPE: /u?int(8|16|32|64)?/ +NUMBER: /(0x)?[0-9a-fA-F]+/ +UID: /[0-9a-fA-F]{16}/ +COMMENT: /#.*/ +PATH: /"[^"]*"/ + +%import common.LETTER +%import common.CNAME -> IDENTIFIER +%import common.WS +%ignore WS diff --git a/definitions/objects/channel.def b/definitions/objects/channel.def new file mode 100644 index 0000000..e019462 --- /dev/null +++ b/definitions/objects/channel.def @@ -0,0 +1,3 @@ +object channel : kobject { + uid 3ea38b96aa0e54c8 +} diff --git a/definitions/objects/endpoint.def b/definitions/objects/endpoint.def new file mode 100644 index 0000000..3dfc397 --- /dev/null +++ b/definitions/objects/endpoint.def @@ -0,0 +1,30 @@ +# Channels are objects that enable synchronous IPC of +# arbitrary-sized messages. + +object endpoint : kobject { + uid c5882f24a4c03b7e + + method create [constructor] + + # Send a message on a channel. Blocks until the message + # is received. + method send { + param tag uint64 + param data buffer + } + + # Receieve a message on a channel. Blocks until a message + # is available. + method receive { + param tag uint64 [out] + param data buffer [out] + } + + # Send a message on a channel and then await a new message. + # Equivalent to calling send and then recieve, as a single + # operation. + method sendrecv { + param tag uint64 [inout] + param data buffer [inout] + } +} diff --git a/definitions/objects/kobject.def b/definitions/objects/kobject.def new file mode 100644 index 0000000..1d6ebb6 --- /dev/null +++ b/definitions/objects/kobject.def @@ -0,0 +1,25 @@ +# The base type of all kernel-exposed objects +object kobject [virtual] { + uid 667f61fb2cd57bb4 + + # Get the internal kernel object id of an object + method koid { + param koid uint64 [out] + } + + # Block the current thread waiting for an object to assert + # one of a set of signals + method wait { + param mask uint64 # Bitmap of which signals to wait for + param signals uint64 [out] # Returns the state of the signals + } + + # Block the current thread waiting for an one of multiple + # objects to assert one of a set of signals + method wait_many [static] { + param handles object kobject [list] # The objects to wait on + param mask uint64 # Bitmap of which signals to wait for + param handle object kobject [out] # Returns the object that signalled + param signals uint64 [out] # Returns the state of the signals + } +} diff --git a/definitions/objects/mailbox.def b/definitions/objects/mailbox.def new file mode 100644 index 0000000..b036faa --- /dev/null +++ b/definitions/objects/mailbox.def @@ -0,0 +1,33 @@ +# Mailboxes are objects that enable asynchronous IPC via event notification. +# This is a second line of documentation + +object mailbox { + uid 99934ad04ece1e07 + + # Create an unbound mailbox + method create [constructor] + + method close [destructor] + + method bind { + param index uint + param source object kobject + param event uint + } + + method unbind { + param index uint + } + + method notify { + param index uint + } + + method wait { + param bitmap uint64 [out] + } + + method ack { + param bitmap uint64 + } +} diff --git a/definitions/objects/process.def b/definitions/objects/process.def new file mode 100644 index 0000000..8da379b --- /dev/null +++ b/definitions/objects/process.def @@ -0,0 +1,27 @@ +import "objects/kobject.def" + +# Processes are a collection of handles and a virtual memory +# space inside which threads are run. + +object process : kobject { + uid 0c69ee0b7502ba31 + + # Create a new empty process + method create [constructor] + + # Stop all threads and exit the given process + method kill [destructor] + + # Stop all threads and exit the current process + method exit [static] { + param result int32 # The result to retrun to the parent process + } + + # Start the given process running. Note that the entrypoint + # address must be specified in the address space of the new + # process. + method start { + param entrypoint address # The address of the main thread entrypoint + param handles object kobject [list] # A list of parent handles to send to the child process + } +} diff --git a/definitions/objects/system.def b/definitions/objects/system.def new file mode 100644 index 0000000..f206c12 --- /dev/null +++ b/definitions/objects/system.def @@ -0,0 +1,29 @@ +import "objects/endpoint.def" +import "objects/vma.def" + +# The system object represents a handle to kernel functionality +# needed by drivers and other priviledged services +object system : kobject { + uid fa72506a2cf71a30 + + # Get a log line from the kernel log + method get_log { + param buffer buffer [out] # Buffer for the log message data structure + } + + # Ask the kernel to send this process messages whenever + # the given IRQ fires + method bind_irq { + param dest object endpoint # Endpoint that will receive messages + param irq uint # IRQ number to bind + } + + # Create a VMA and map an area of physical memory into it, + # also mapping that VMA into the current process + method map_phys { + param area object vma [out] # Receives a handle to the VMA created + param phys address # The physical address of the area + param size size # Size of the area, in pages + param flags uint32 # Flags to apply to the created VMA + } +} diff --git a/definitions/objects/thread.def b/definitions/objects/thread.def new file mode 100644 index 0000000..1d5fab8 --- /dev/null +++ b/definitions/objects/thread.def @@ -0,0 +1,18 @@ +object thread : kobject { + uid 11f23e593d5761bd + + method create [constructor] { + param entrypoint address + } + + method kill [destructor] + + method exit [static] { + param status int32 + } + + method pause [static] + method sleep [static] { + param until uint64 + } +} diff --git a/definitions/objects/vma.def b/definitions/objects/vma.def new file mode 100644 index 0000000..68e24aa --- /dev/null +++ b/definitions/objects/vma.def @@ -0,0 +1,29 @@ +import "objects/process.def" + +object vma : kobject { + uid d6a12b63b3ed3937 + + method create [constructor] { + param size size + param flags uint32 + } + + method create_map [constructor] { + param size size + param address address + param flags uint32 + } + + method map { + param process object process + param address address + } + + method unmap { + param process object process + } + + method resize { + param size size [inout] + } +} diff --git a/definitions/syscalls.def b/definitions/syscalls.def new file mode 100644 index 0000000..23eefe2 --- /dev/null +++ b/definitions/syscalls.def @@ -0,0 +1,27 @@ +import "objects/system.def" +import "objects/kobject.def" +import "objects/process.def" +import "objects/thread.def" +import "objects/channel.def" +import "objects/endpoint.def" +import "objects/vma.def" + +interface syscalls [syscall] { + uid 01d9b6a948961097 + + expose object system + expose object kobject + expose object process + expose object thread + expose object channel + expose object endpoint + expose object vma + + # Simple no-op syscall for testing + function noop + + # Write a message to the kernel log + function log { + param message string + } +} diff --git a/scripts/definitions/__init__.py b/scripts/definitions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/definitions/context.py b/scripts/definitions/context.py new file mode 100644 index 0000000..341183c --- /dev/null +++ b/scripts/definitions/context.py @@ -0,0 +1,65 @@ +class NotFound(Exception): pass + +class Context: + def __init__(self, path, verbose=False): + if isinstance(path, str): + self.__paths = [path] + else: + self.__paths = path + + self.__closed = set() + self.__verbose = verbose + + self.__deps = {} + + self.objects = dict() + self.interfaces = dict() + + verbose = property(lambda self: self.__verbose) + + def find(self, filename): + from os.path import exists, isabs, join + + if exists(filename) or isabs(filename): + return filename + + for path in self.__paths: + full = join(path, filename) + if exists(full): + return full + + raise NotFound(filename) + + def parse(self, filename): + pending = set() + pending.add(filename) + + from .parser import Lark_StandAlone as Parser + from .transformer import DefTransformer + + objects = {} + interfaces = {} + + while pending: + name = pending.pop() + self.__closed.add(name) + + path = self.find(name) + + parser = Parser(transformer=DefTransformer(name)) + imps, objs, ints = parser.parse(open(path, "r").read()) + objects.update(objs) + interfaces.update(ints) + + self.__deps[name] = imps + + pending.update(imps.difference(self.__closed)) + + from .types import ObjectRef + ObjectRef.connect(objects) + + self.objects.update(objects) + self.interfaces.update(interfaces) + + def deps(self): + return {self.find(k): tuple(map(self.find, v)) for k, v in self.__deps.items()} diff --git a/scripts/definitions/errors.py b/scripts/definitions/errors.py new file mode 100644 index 0000000..c4034ba --- /dev/null +++ b/scripts/definitions/errors.py @@ -0,0 +1,2 @@ +class InvalidType(Exception): pass +class UnknownTypeError(Exception): pass diff --git a/scripts/definitions/generate.py b/scripts/definitions/generate.py new file mode 100644 index 0000000..03dfabe --- /dev/null +++ b/scripts/definitions/generate.py @@ -0,0 +1,62 @@ +def generate_template(template, outfile, **kwargs): + from hashlib import md5 + from os import makedirs + from os.path import dirname, exists + + content = template.render(**kwargs) + h = md5(content.encode('utf-8')).hexdigest() + + if exists(outfile): + existing = open(outfile, 'r').read().encode('utf-8') + if md5(existing).hexdigest() == h: + return False + + makedirs(dirname(outfile), exist_ok=True) + open(outfile, 'w').write(content) + + return True + +def generate(ctx, outdir, template_type): + from os.path import basename, join, split, splitext + from jinja2 import Environment, PackageLoader + + for name, interface in ctx.interfaces.items(): + base = "_".join(sorted(interface.options)) + + path = join("templates", base, template_type) + env = Environment( + loader = PackageLoader('definitions', package_path=path), + trim_blocks = True, lstrip_blocks = True) + + env.filters + + for template_name in env.list_templates(): + template = env.get_template(template_name) + + basepath, filename = split(template_name) + filename, ext = splitext(filename) + + if filename == "_object_": + for obj in ctx.objects.values(): + outfile = join(outdir, basepath, obj.name + ext) + wrote = generate_template( + template, outfile, + filename=obj.name + ext, + basepath=basepath, + object=obj, + interface=interface, + objects=ctx.objects) + + if wrote and ctx.verbose: + print(f"Writing {outfile}") + + else: + outfile = join(outdir, template_name) + wrote = generate_template( + template, outfile, + filename=basename(template_name), + interface=interface, + objects=ctx.objects) + + if wrote and ctx.verbose: + print(f"Writing {outfile}") diff --git a/scripts/definitions/parser.py b/scripts/definitions/parser.py new file mode 100644 index 0000000..46a19ac --- /dev/null +++ b/scripts/definitions/parser.py @@ -0,0 +1,2809 @@ +# The file was automatically generated by Lark v0.11.3 +__version__ = "0.11.3" + +# +# +# Lark Stand-alone Generator Tool +# ---------------------------------- +# Generates a stand-alone LALR(1) parser with a standard lexer +# +# Git: https://github.com/erezsh/lark +# Author: Erez Shinan (erezshin@gmail.com) +# +# +# >>> LICENSE +# +# This tool and its generated code use a separate license from Lark, +# and are subject to the terms of the Mozilla Public License, v. 2.0. +# If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# +# If you wish to purchase a commercial license for this tool and its +# generated code, you may contact me via email or otherwise. +# +# If MPL2 is incompatible with your free or open-source project, +# contact me and we'll work it out. +# +# + +from io import open + + + +class LarkError(Exception): + pass + + +class ConfigurationError(LarkError, ValueError): + pass + + +def assert_config(value, options, msg='Got %r, expected one of %s'): + if value not in options: + raise ConfigurationError(msg % (value, options)) + + +class GrammarError(LarkError): + pass + + +class ParseError(LarkError): + pass + + +class LexError(LarkError): + pass + + +class UnexpectedInput(LarkError): + #-- + pos_in_stream = None + _terminals_by_name = None + + def get_context(self, text, span=40): + #-- + assert self.pos_in_stream is not None, self + pos = self.pos_in_stream + start = max(pos - span, 0) + end = pos + span + if not isinstance(text, bytes): + before = text[start:pos].rsplit('\n', 1)[-1] + after = text[pos:end].split('\n', 1)[0] + return before + after + '\n' + ' ' * len(before.expandtabs()) + '^\n' + else: + before = text[start:pos].rsplit(b'\n', 1)[-1] + after = text[pos:end].split(b'\n', 1)[0] + return (before + after + b'\n' + b' ' * len(before.expandtabs()) + b'^\n').decode("ascii", "backslashreplace") + + def match_examples(self, parse_fn, examples, token_type_match_fallback=False, use_accepts=False): + #-- + assert self.state is not None, "Not supported for this exception" + + if isinstance(examples, dict): + examples = examples.items() + + candidate = (None, False) + for i, (label, example) in enumerate(examples): + assert not isinstance(example, STRING_TYPE) + + for j, malformed in enumerate(example): + try: + parse_fn(malformed) + except UnexpectedInput as ut: + if ut.state == self.state: + if use_accepts and hasattr(self, 'accepts') and ut.accepts != self.accepts: + logger.debug("Different accepts with same state[%d]: %s != %s at example [%s][%s]" % + (self.state, self.accepts, ut.accepts, i, j)) + continue + try: + if ut.token == self.token: ## + + logger.debug("Exact Match at example [%s][%s]" % (i, j)) + return label + + if token_type_match_fallback: + ## + + if (ut.token.type == self.token.type) and not candidate[-1]: + logger.debug("Token Type Fallback at example [%s][%s]" % (i, j)) + candidate = label, True + + except AttributeError: + pass + if candidate[0] is None: + logger.debug("Same State match at example [%s][%s]" % (i, j)) + candidate = label, False + + return candidate[0] + + def _format_expected(self, expected): + if self._terminals_by_name: + d = self._terminals_by_name + expected = [d[t_name].user_repr() if t_name in d else t_name for t_name in expected] + return "Expected one of: \n\t* %s\n" % '\n\t* '.join(expected) + + +class UnexpectedEOF(ParseError, UnexpectedInput): + def __init__(self, expected, state=None, terminals_by_name=None): + self.expected = expected + self.state = state + from .lexer import Token + self.token = Token("", "") ## + + self.pos_in_stream = -1 + self.line = -1 + self.column = -1 + self._terminals_by_name = terminals_by_name + + super(UnexpectedEOF, self).__init__() + + def __str__(self): + message = "Unexpected end-of-input. " + message += self._format_expected(self.expected) + return message + + +class UnexpectedCharacters(LexError, UnexpectedInput): + def __init__(self, seq, lex_pos, line, column, allowed=None, considered_tokens=None, state=None, token_history=None, + terminals_by_name=None, considered_rules=None): + ## + + self.line = line + self.column = column + self.pos_in_stream = lex_pos + self.state = state + self._terminals_by_name = terminals_by_name + + self.allowed = allowed + self.considered_tokens = considered_tokens + self.considered_rules = considered_rules + self.token_history = token_history + + if isinstance(seq, bytes): + self.char = seq[lex_pos:lex_pos + 1].decode("ascii", "backslashreplace") + else: + self.char = seq[lex_pos] + self._context = self.get_context(seq) + + super(UnexpectedCharacters, self).__init__() + + def __str__(self): + message = "No terminal matches '%s' in the current parser context, at line %d col %d" % (self.char, self.line, self.column) + message += '\n\n' + self._context + if self.allowed: + message += self._format_expected(self.allowed) + if self.token_history: + message += '\nPrevious tokens: %s\n' % ', '.join(repr(t) for t in self.token_history) + return message + + +class UnexpectedToken(ParseError, UnexpectedInput): + #-- + + def __init__(self, token, expected, considered_rules=None, state=None, interactive_parser=None, terminals_by_name=None, token_history=None): + ## + + self.line = getattr(token, 'line', '?') + self.column = getattr(token, 'column', '?') + self.pos_in_stream = getattr(token, 'pos_in_stream', None) + self.state = state + + self.token = token + self.expected = expected ## + + self._accepts = NO_VALUE + self.considered_rules = considered_rules + self.interactive_parser = interactive_parser + self._terminals_by_name = terminals_by_name + self.token_history = token_history + + super(UnexpectedToken, self).__init__() + + @property + def accepts(self): + if self._accepts is NO_VALUE: + self._accepts = self.interactive_parser and self.interactive_parser.accepts() + return self._accepts + + def __str__(self): + message = ("Unexpected token %r at line %s, column %s.\n%s" + % (self.token, self.line, self.column, self._format_expected(self.accepts or self.expected))) + if self.token_history: + message += "Previous tokens: %r\n" % self.token_history + + return message + + @property + def puppet(self): + warn("UnexpectedToken.puppet attribute has been renamed to interactive_parser", DeprecationWarning) + return self.interactive_parser + + + +class VisitError(LarkError): + #-- + + def __init__(self, rule, obj, orig_exc): + self.obj = obj + self.orig_exc = orig_exc + + message = 'Error trying to process rule "%s":\n\n%s' % (rule, orig_exc) + super(VisitError, self).__init__(message) + + +import sys, re +import logging +from io import open +logger = logging.getLogger("lark") +logger.addHandler(logging.StreamHandler()) +## + +## + +logger.setLevel(logging.CRITICAL) + +if sys.version_info[0]>2: + from abc import ABC, abstractmethod +else: + from abc import ABCMeta, abstractmethod + class ABC(object): ## + + __slots__ = () + __metclass__ = ABCMeta + + +Py36 = (sys.version_info[:2] >= (3, 6)) + +NO_VALUE = object() + + +def classify(seq, key=None, value=None): + d = {} + for item in seq: + k = key(item) if (key is not None) else item + v = value(item) if (value is not None) else item + if k in d: + d[k].append(v) + else: + d[k] = [v] + return d + + +def _deserialize(data, namespace, memo): + if isinstance(data, dict): + if '__type__' in data: ## + + class_ = namespace[data['__type__']] + return class_.deserialize(data, memo) + elif '@' in data: + return memo[data['@']] + return {key:_deserialize(value, namespace, memo) for key, value in data.items()} + elif isinstance(data, list): + return [_deserialize(value, namespace, memo) for value in data] + return data + + +class Serialize(object): + #-- + + def memo_serialize(self, types_to_memoize): + memo = SerializeMemoizer(types_to_memoize) + return self.serialize(memo), memo.serialize() + + def serialize(self, memo=None): + if memo and memo.in_types(self): + return {'@': memo.memoized.get(self)} + + fields = getattr(self, '__serialize_fields__') + res = {f: _serialize(getattr(self, f), memo) for f in fields} + res['__type__'] = type(self).__name__ + postprocess = getattr(self, '_serialize', None) + if postprocess: + postprocess(res, memo) + return res + + @classmethod + def deserialize(cls, data, memo): + namespace = getattr(cls, '__serialize_namespace__', {}) + namespace = {c.__name__:c for c in namespace} + + fields = getattr(cls, '__serialize_fields__') + + if '@' in data: + return memo[data['@']] + + inst = cls.__new__(cls) + for f in fields: + try: + setattr(inst, f, _deserialize(data[f], namespace, memo)) + except KeyError as e: + raise KeyError("Cannot find key for class", cls, e) + postprocess = getattr(inst, '_deserialize', None) + if postprocess: + postprocess() + return inst + + +class SerializeMemoizer(Serialize): + #-- + + __serialize_fields__ = 'memoized', + + def __init__(self, types_to_memoize): + self.types_to_memoize = tuple(types_to_memoize) + self.memoized = Enumerator() + + def in_types(self, value): + return isinstance(value, self.types_to_memoize) + + def serialize(self): + return _serialize(self.memoized.reversed(), None) + + @classmethod + def deserialize(cls, data, namespace, memo): + return _deserialize(data, namespace, memo) + + +try: + STRING_TYPE = basestring +except NameError: ## + + STRING_TYPE = str + + +import types +from functools import wraps, partial +from contextlib import contextmanager + +Str = type(u'') +try: + classtype = types.ClassType ## + +except AttributeError: + classtype = type ## + + + +def smart_decorator(f, create_decorator): + if isinstance(f, types.FunctionType): + return wraps(f)(create_decorator(f, True)) + + elif isinstance(f, (classtype, type, types.BuiltinFunctionType)): + return wraps(f)(create_decorator(f, False)) + + elif isinstance(f, types.MethodType): + return wraps(f)(create_decorator(f.__func__, True)) + + elif isinstance(f, partial): + ## + + return wraps(f.func)(create_decorator(lambda *args, **kw: f(*args[1:], **kw), True)) + + else: + return create_decorator(f.__func__.__call__, True) + + +try: + import regex +except ImportError: + regex = None + +import sre_parse +import sre_constants +categ_pattern = re.compile(r'\\p{[A-Za-z_]+}') + +def get_regexp_width(expr): + if regex: + ## + + ## + + ## + + regexp_final = re.sub(categ_pattern, 'A', expr) + else: + if re.search(categ_pattern, expr): + raise ImportError('`regex` module must be installed in order to use Unicode categories.', expr) + regexp_final = expr + try: + return [int(x) for x in sre_parse.parse(regexp_final).getwidth()] + except sre_constants.error: + raise ValueError(expr) + + +from collections import OrderedDict + + +class Meta: + def __init__(self): + self.empty = True + + +class Tree(object): + #-- + def __init__(self, data, children, meta=None): + self.data = data + self.children = children + self._meta = meta + + @property + def meta(self): + if self._meta is None: + self._meta = Meta() + return self._meta + + def __repr__(self): + return 'Tree(%r, %r)' % (self.data, self.children) + + def _pretty_label(self): + return self.data + + def _pretty(self, level, indent_str): + if len(self.children) == 1 and not isinstance(self.children[0], Tree): + return [indent_str*level, self._pretty_label(), '\t', '%s' % (self.children[0],), '\n'] + + l = [indent_str*level, self._pretty_label(), '\n'] + for n in self.children: + if isinstance(n, Tree): + l += n._pretty(level+1, indent_str) + else: + l += [indent_str*(level+1), '%s' % (n,), '\n'] + + return l + + def pretty(self, indent_str=' '): + #-- + return ''.join(self._pretty(0, indent_str)) + + def __eq__(self, other): + try: + return self.data == other.data and self.children == other.children + except AttributeError: + return False + + def __ne__(self, other): + return not (self == other) + + def __hash__(self): + return hash((self.data, tuple(self.children))) + + def iter_subtrees(self): + #-- + queue = [self] + subtrees = OrderedDict() + for subtree in queue: + subtrees[id(subtree)] = subtree + queue += [c for c in reversed(subtree.children) + if isinstance(c, Tree) and id(c) not in subtrees] + + del queue + return reversed(list(subtrees.values())) + + def find_pred(self, pred): + #-- + return filter(pred, self.iter_subtrees()) + + def find_data(self, data): + #-- + return self.find_pred(lambda t: t.data == data) + + +from inspect import getmembers, getmro + + +class Discard(Exception): + #-- + pass + +## + + + +class _Decoratable: + #-- + + @classmethod + def _apply_decorator(cls, decorator, **kwargs): + mro = getmro(cls) + assert mro[0] is cls + libmembers = {name for _cls in mro[1:] for name, _ in getmembers(_cls)} + for name, value in getmembers(cls): + + ## + + if name.startswith('_') or (name in libmembers and name not in cls.__dict__): + continue + if not callable(value): + continue + + ## + + if hasattr(cls.__dict__[name], 'vargs_applied') or hasattr(value, 'vargs_applied'): + continue + + static = isinstance(cls.__dict__[name], (staticmethod, classmethod)) + setattr(cls, name, decorator(value, static=static, **kwargs)) + return cls + + def __class_getitem__(cls, _): + return cls + + +class Transformer(_Decoratable): + #-- + __visit_tokens__ = True ## + + + def __init__(self, visit_tokens=True): + self.__visit_tokens__ = visit_tokens + + def _call_userfunc(self, tree, new_children=None): + ## + + children = new_children if new_children is not None else tree.children + try: + f = getattr(self, tree.data) + except AttributeError: + return self.__default__(tree.data, children, tree.meta) + else: + try: + wrapper = getattr(f, 'visit_wrapper', None) + if wrapper is not None: + return f.visit_wrapper(f, tree.data, children, tree.meta) + else: + return f(children) + except (GrammarError, Discard): + raise + except Exception as e: + raise VisitError(tree.data, tree, e) + + def _call_userfunc_token(self, token): + try: + f = getattr(self, token.type) + except AttributeError: + return self.__default_token__(token) + else: + try: + return f(token) + except (GrammarError, Discard): + raise + except Exception as e: + raise VisitError(token.type, token, e) + + def _transform_children(self, children): + for c in children: + try: + if isinstance(c, Tree): + yield self._transform_tree(c) + elif self.__visit_tokens__ and isinstance(c, Token): + yield self._call_userfunc_token(c) + else: + yield c + except Discard: + pass + + def _transform_tree(self, tree): + children = list(self._transform_children(tree.children)) + return self._call_userfunc(tree, children) + + def transform(self, tree): + #-- + return self._transform_tree(tree) + + def __mul__(self, other): + #-- + return TransformerChain(self, other) + + def __default__(self, data, children, meta): + #-- + return Tree(data, children, meta) + + def __default_token__(self, token): + #-- + return token + + +class InlineTransformer(Transformer): ## + + def _call_userfunc(self, tree, new_children=None): + ## + + children = new_children if new_children is not None else tree.children + try: + f = getattr(self, tree.data) + except AttributeError: + return self.__default__(tree.data, children, tree.meta) + else: + return f(*children) + + +class TransformerChain(object): + def __init__(self, *transformers): + self.transformers = transformers + + def transform(self, tree): + for t in self.transformers: + tree = t.transform(tree) + return tree + + def __mul__(self, other): + return TransformerChain(*self.transformers + (other,)) + + +class Transformer_InPlace(Transformer): + #-- + def _transform_tree(self, tree): ## + + return self._call_userfunc(tree) + + def transform(self, tree): + for subtree in tree.iter_subtrees(): + subtree.children = list(self._transform_children(subtree.children)) + + return self._transform_tree(tree) + + +class Transformer_NonRecursive(Transformer): + #-- + + def transform(self, tree): + ## + + rev_postfix = [] + q = [tree] + while q: + t = q.pop() + rev_postfix.append(t) + if isinstance(t, Tree): + q += t.children + + ## + + stack = [] + for x in reversed(rev_postfix): + if isinstance(x, Tree): + size = len(x.children) + if size: + args = stack[-size:] + del stack[-size:] + else: + args = [] + stack.append(self._call_userfunc(x, args)) + elif self.__visit_tokens__ and isinstance(x, Token): + stack.append(self._call_userfunc_token(x)) + else: + stack.append(x) + + t ,= stack ## + + return t + + +class Transformer_InPlaceRecursive(Transformer): + #-- + def _transform_tree(self, tree): + tree.children = list(self._transform_children(tree.children)) + return self._call_userfunc(tree) + + +## + + +class VisitorBase: + def _call_userfunc(self, tree): + return getattr(self, tree.data, self.__default__)(tree) + + def __default__(self, tree): + #-- + return tree + + def __class_getitem__(cls, _): + return cls + + +class Visitor(VisitorBase): + #-- + + def visit(self, tree): + #-- + for subtree in tree.iter_subtrees(): + self._call_userfunc(subtree) + return tree + + def visit_topdown(self,tree): + #-- + for subtree in tree.iter_subtrees_topdown(): + self._call_userfunc(subtree) + return tree + + +class Visitor_Recursive(VisitorBase): + #-- + + def visit(self, tree): + #-- + for child in tree.children: + if isinstance(child, Tree): + self.visit(child) + + self._call_userfunc(tree) + return tree + + def visit_topdown(self,tree): + #-- + self._call_userfunc(tree) + + for child in tree.children: + if isinstance(child, Tree): + self.visit_topdown(child) + + return tree + + +def visit_children_decor(func): + #-- + @wraps(func) + def inner(cls, tree): + values = cls.visit_children(tree) + return func(cls, values) + return inner + + +class Interpreter(_Decoratable): + #-- + + def visit(self, tree): + f = getattr(self, tree.data) + wrapper = getattr(f, 'visit_wrapper', None) + if wrapper is not None: + return f.visit_wrapper(f, tree.data, tree.children, tree.meta) + else: + return f(tree) + + def visit_children(self, tree): + return [self.visit(child) if isinstance(child, Tree) else child + for child in tree.children] + + def __getattr__(self, name): + return self.__default__ + + def __default__(self, tree): + return self.visit_children(tree) + + +## + + +def _apply_decorator(obj, decorator, **kwargs): + try: + _apply = obj._apply_decorator + except AttributeError: + return decorator(obj, **kwargs) + else: + return _apply(decorator, **kwargs) + + +def _inline_args__func(func): + @wraps(func) + def create_decorator(_f, with_self): + if with_self: + def f(self, children): + return _f(self, *children) + else: + def f(self, children): + return _f(*children) + return f + + return smart_decorator(func, create_decorator) + + +def inline_args(obj): ## + + return _apply_decorator(obj, _inline_args__func) + + +def _visitor_args_func_dec(func, visit_wrapper=None, static=False): + def create_decorator(_f, with_self): + if with_self: + def f(self, *args, **kwargs): + return _f(self, *args, **kwargs) + else: + def f(self, *args, **kwargs): + return _f(*args, **kwargs) + return f + + if static: + f = wraps(func)(create_decorator(func, False)) + else: + f = smart_decorator(func, create_decorator) + f.vargs_applied = True + f.visit_wrapper = visit_wrapper + return f + + +def _vargs_inline(f, _data, children, _meta): + return f(*children) +def _vargs_meta_inline(f, _data, children, meta): + return f(meta, *children) +def _vargs_meta(f, _data, children, meta): + return f(children, meta) ## + +def _vargs_tree(f, data, children, meta): + return f(Tree(data, children, meta)) + + +def v_args(inline=False, meta=False, tree=False, wrapper=None): + #-- + if tree and (meta or inline): + raise ValueError("Visitor functions cannot combine 'tree' with 'meta' or 'inline'.") + + func = None + if meta: + if inline: + func = _vargs_meta_inline + else: + func = _vargs_meta + elif inline: + func = _vargs_inline + elif tree: + func = _vargs_tree + + if wrapper is not None: + if func is not None: + raise ValueError("Cannot use 'wrapper' along with 'tree', 'meta' or 'inline'.") + func = wrapper + + def _visitor_args_dec(obj): + return _apply_decorator(obj, _visitor_args_func_dec, visit_wrapper=func) + return _visitor_args_dec + + + + +class Symbol(Serialize): + __slots__ = ('name',) + + is_term = NotImplemented + + def __init__(self, name): + self.name = name + + def __eq__(self, other): + assert isinstance(other, Symbol), other + return self.is_term == other.is_term and self.name == other.name + + def __ne__(self, other): + return not (self == other) + + def __hash__(self): + return hash(self.name) + + def __repr__(self): + return '%s(%r)' % (type(self).__name__, self.name) + + fullrepr = property(__repr__) + + +class Terminal(Symbol): + __serialize_fields__ = 'name', 'filter_out' + + is_term = True + + def __init__(self, name, filter_out=False): + self.name = name + self.filter_out = filter_out + + @property + def fullrepr(self): + return '%s(%r, %r)' % (type(self).__name__, self.name, self.filter_out) + + +class NonTerminal(Symbol): + __serialize_fields__ = 'name', + + is_term = False + + +class RuleOptions(Serialize): + __serialize_fields__ = 'keep_all_tokens', 'expand1', 'priority', 'template_source', 'empty_indices' + + def __init__(self, keep_all_tokens=False, expand1=False, priority=None, template_source=None, empty_indices=()): + self.keep_all_tokens = keep_all_tokens + self.expand1 = expand1 + self.priority = priority + self.template_source = template_source + self.empty_indices = empty_indices + + def __repr__(self): + return 'RuleOptions(%r, %r, %r, %r)' % ( + self.keep_all_tokens, + self.expand1, + self.priority, + self.template_source + ) + + +class Rule(Serialize): + #-- + __slots__ = ('origin', 'expansion', 'alias', 'options', 'order', '_hash') + + __serialize_fields__ = 'origin', 'expansion', 'order', 'alias', 'options' + __serialize_namespace__ = Terminal, NonTerminal, RuleOptions + + def __init__(self, origin, expansion, order=0, alias=None, options=None): + self.origin = origin + self.expansion = expansion + self.alias = alias + self.order = order + self.options = options or RuleOptions() + self._hash = hash((self.origin, tuple(self.expansion))) + + def _deserialize(self): + self._hash = hash((self.origin, tuple(self.expansion))) + + def __str__(self): + return '<%s : %s>' % (self.origin.name, ' '.join(x.name for x in self.expansion)) + + def __repr__(self): + return 'Rule(%r, %r, %r, %r)' % (self.origin, self.expansion, self.alias, self.options) + + def __hash__(self): + return self._hash + + def __eq__(self, other): + if not isinstance(other, Rule): + return False + return self.origin == other.origin and self.expansion == other.expansion + + + +from copy import copy + + +class Pattern(Serialize): + raw = None + type = None + + def __init__(self, value, flags=(), raw=None): + self.value = value + self.flags = frozenset(flags) + self.raw = raw + + def __repr__(self): + return repr(self.to_regexp()) + + ## + + def __hash__(self): + return hash((type(self), self.value, self.flags)) + + def __eq__(self, other): + return type(self) == type(other) and self.value == other.value and self.flags == other.flags + + def to_regexp(self): + raise NotImplementedError() + + def min_width(self): + raise NotImplementedError() + + def max_width(self): + raise NotImplementedError() + + if Py36: + ## + + def _get_flags(self, value): + for f in self.flags: + value = ('(?%s:%s)' % (f, value)) + return value + + else: + def _get_flags(self, value): + for f in self.flags: + value = ('(?%s)' % f) + value + return value + + + +class PatternStr(Pattern): + __serialize_fields__ = 'value', 'flags' + + type = "str" + + def to_regexp(self): + return self._get_flags(re.escape(self.value)) + + @property + def min_width(self): + return len(self.value) + max_width = min_width + + +class PatternRE(Pattern): + __serialize_fields__ = 'value', 'flags', '_width' + + type = "re" + + def to_regexp(self): + return self._get_flags(self.value) + + _width = None + def _get_width(self): + if self._width is None: + self._width = get_regexp_width(self.to_regexp()) + return self._width + + @property + def min_width(self): + return self._get_width()[0] + + @property + def max_width(self): + return self._get_width()[1] + + +class TerminalDef(Serialize): + __serialize_fields__ = 'name', 'pattern', 'priority' + __serialize_namespace__ = PatternStr, PatternRE + + def __init__(self, name, pattern, priority=1): + assert isinstance(pattern, Pattern), pattern + self.name = name + self.pattern = pattern + self.priority = priority + + def __repr__(self): + return '%s(%r, %r)' % (type(self).__name__, self.name, self.pattern) + + def user_repr(self): + if self.name.startswith('__'): ## + + return self.pattern.raw or self.name + else: + return self.name + + +class Token(Str): + #-- + __slots__ = ('type', 'pos_in_stream', 'value', 'line', 'column', 'end_line', 'end_column', 'end_pos') + + def __new__(cls, type_, value, pos_in_stream=None, line=None, column=None, end_line=None, end_column=None, end_pos=None): + try: + self = super(Token, cls).__new__(cls, value) + except UnicodeDecodeError: + value = value.decode('latin1') + self = super(Token, cls).__new__(cls, value) + + self.type = type_ + self.pos_in_stream = pos_in_stream + self.value = value + self.line = line + self.column = column + self.end_line = end_line + self.end_column = end_column + self.end_pos = end_pos + return self + + def update(self, type_=None, value=None): + return Token.new_borrow_pos( + type_ if type_ is not None else self.type, + value if value is not None else self.value, + self + ) + + @classmethod + def new_borrow_pos(cls, type_, value, borrow_t): + return cls(type_, value, borrow_t.pos_in_stream, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column, borrow_t.end_pos) + + def __reduce__(self): + return (self.__class__, (self.type, self.value, self.pos_in_stream, self.line, self.column)) + + def __repr__(self): + return 'Token(%r, %r)' % (self.type, self.value) + + def __deepcopy__(self, memo): + return Token(self.type, self.value, self.pos_in_stream, self.line, self.column) + + def __eq__(self, other): + if isinstance(other, Token) and self.type != other.type: + return False + + return Str.__eq__(self, other) + + __hash__ = Str.__hash__ + + +class LineCounter: + __slots__ = 'char_pos', 'line', 'column', 'line_start_pos', 'newline_char' + + def __init__(self, newline_char): + self.newline_char = newline_char + self.char_pos = 0 + self.line = 1 + self.column = 1 + self.line_start_pos = 0 + + def __eq__(self, other): + if not isinstance(other, LineCounter): + return NotImplemented + + return self.char_pos == other.char_pos and self.newline_char == other.newline_char + + def feed(self, token, test_newline=True): + #-- + if test_newline: + newlines = token.count(self.newline_char) + if newlines: + self.line += newlines + self.line_start_pos = self.char_pos + token.rindex(self.newline_char) + 1 + + self.char_pos += len(token) + self.column = self.char_pos - self.line_start_pos + 1 + + +class UnlessCallback: + def __init__(self, mres): + self.mres = mres + + def __call__(self, t): + for mre, type_from_index in self.mres: + m = mre.match(t.value) + if m: + t.type = type_from_index[m.lastindex] + break + return t + + +class CallChain: + def __init__(self, callback1, callback2, cond): + self.callback1 = callback1 + self.callback2 = callback2 + self.cond = cond + + def __call__(self, t): + t2 = self.callback1(t) + return self.callback2(t) if self.cond(t2) else t2 + + +def _create_unless(terminals, g_regex_flags, re_, use_bytes): + tokens_by_type = classify(terminals, lambda t: type(t.pattern)) + assert len(tokens_by_type) <= 2, tokens_by_type.keys() + embedded_strs = set() + callback = {} + for retok in tokens_by_type.get(PatternRE, []): + unless = [] + for strtok in tokens_by_type.get(PatternStr, []): + if strtok.priority > retok.priority: + continue + s = strtok.pattern.value + m = re_.match(retok.pattern.to_regexp(), s, g_regex_flags) + if m and m.group(0) == s: + unless.append(strtok) + if strtok.pattern.flags <= retok.pattern.flags: + embedded_strs.add(strtok) + if unless: + callback[retok.name] = UnlessCallback(build_mres(unless, g_regex_flags, re_, match_whole=True, use_bytes=use_bytes)) + + terminals = [t for t in terminals if t not in embedded_strs] + return terminals, callback + + +def _build_mres(terminals, max_size, g_regex_flags, match_whole, re_, use_bytes): + ## + + ## + + ## + + postfix = '$' if match_whole else '' + mres = [] + while terminals: + pattern = u'|'.join(u'(?P<%s>%s)' % (t.name, t.pattern.to_regexp() + postfix) for t in terminals[:max_size]) + if use_bytes: + pattern = pattern.encode('latin-1') + try: + mre = re_.compile(pattern, g_regex_flags) + except AssertionError: ## + + return _build_mres(terminals, max_size//2, g_regex_flags, match_whole, re_, use_bytes) + + mres.append((mre, {i: n for n, i in mre.groupindex.items()})) + terminals = terminals[max_size:] + return mres + + +def build_mres(terminals, g_regex_flags, re_, use_bytes, match_whole=False): + return _build_mres(terminals, len(terminals), g_regex_flags, match_whole, re_, use_bytes) + + +def _regexp_has_newline(r): + #-- + return '\n' in r or '\\n' in r or '\\s' in r or '[^' in r or ('(?s' in r and '.' in r) + + +class Lexer(object): + #-- + lex = NotImplemented + + def make_lexer_state(self, text): + line_ctr = LineCounter(b'\n' if isinstance(text, bytes) else '\n') + return LexerState(text, line_ctr) + + +class TraditionalLexer(Lexer): + + def __init__(self, conf): + terminals = list(conf.terminals) + assert all(isinstance(t, TerminalDef) for t in terminals), terminals + + self.re = conf.re_module + + if not conf.skip_validation: + ## + + for t in terminals: + try: + self.re.compile(t.pattern.to_regexp(), conf.g_regex_flags) + except self.re.error: + raise LexError("Cannot compile token %s: %s" % (t.name, t.pattern)) + + if t.pattern.min_width == 0: + raise LexError("Lexer does not allow zero-width terminals. (%s: %s)" % (t.name, t.pattern)) + + if not (set(conf.ignore) <= {t.name for t in terminals}): + raise LexError("Ignore terminals are not defined: %s" % (set(conf.ignore) - {t.name for t in terminals})) + + ## + + self.newline_types = frozenset(t.name for t in terminals if _regexp_has_newline(t.pattern.to_regexp())) + self.ignore_types = frozenset(conf.ignore) + + terminals.sort(key=lambda x: (-x.priority, -x.pattern.max_width, -len(x.pattern.value), x.name)) + self.terminals = terminals + self.user_callbacks = conf.callbacks + self.g_regex_flags = conf.g_regex_flags + self.use_bytes = conf.use_bytes + self.terminals_by_name = conf.terminals_by_name + + self._mres = None + + def _build(self): + terminals, self.callback = _create_unless(self.terminals, self.g_regex_flags, self.re, self.use_bytes) + assert all(self.callback.values()) + + for type_, f in self.user_callbacks.items(): + if type_ in self.callback: + ## + + self.callback[type_] = CallChain(self.callback[type_], f, lambda t: t.type == type_) + else: + self.callback[type_] = f + + self._mres = build_mres(terminals, self.g_regex_flags, self.re, self.use_bytes) + + @property + def mres(self): + if self._mres is None: + self._build() + return self._mres + + def match(self, text, pos): + for mre, type_from_index in self.mres: + m = mre.match(text, pos) + if m: + return m.group(0), type_from_index[m.lastindex] + + def lex(self, state, parser_state): + with suppress(EOFError): + while True: + yield self.next_token(state, parser_state) + + def next_token(self, lex_state, parser_state=None): + line_ctr = lex_state.line_ctr + while line_ctr.char_pos < len(lex_state.text): + res = self.match(lex_state.text, line_ctr.char_pos) + if not res: + allowed = {v for m, tfi in self.mres for v in tfi.values()} - self.ignore_types + if not allowed: + allowed = {""} + raise UnexpectedCharacters(lex_state.text, line_ctr.char_pos, line_ctr.line, line_ctr.column, + allowed=allowed, token_history=lex_state.last_token and [lex_state.last_token], + state=parser_state, terminals_by_name=self.terminals_by_name) + + value, type_ = res + + if type_ not in self.ignore_types: + t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column) + line_ctr.feed(value, type_ in self.newline_types) + t.end_line = line_ctr.line + t.end_column = line_ctr.column + t.end_pos = line_ctr.char_pos + if t.type in self.callback: + t = self.callback[t.type](t) + if not isinstance(t, Token): + raise LexError("Callbacks must return a token (returned %r)" % t) + lex_state.last_token = t + return t + else: + if type_ in self.callback: + t2 = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column) + self.callback[type_](t2) + line_ctr.feed(value, type_ in self.newline_types) + + ## + + raise EOFError(self) + + +class LexerState(object): + __slots__ = 'text', 'line_ctr', 'last_token' + + def __init__(self, text, line_ctr, last_token=None): + self.text = text + self.line_ctr = line_ctr + self.last_token = last_token + + def __eq__(self, other): + if not isinstance(other, LexerState): + return NotImplemented + + return self.text is other.text and self.line_ctr == other.line_ctr and self.last_token == other.last_token + + def __copy__(self): + return type(self)(self.text, copy(self.line_ctr), self.last_token) + + +class ContextualLexer(Lexer): + + def __init__(self, conf, states, always_accept=()): + terminals = list(conf.terminals) + terminals_by_name = conf.terminals_by_name + + trad_conf = copy(conf) + trad_conf.terminals = terminals + + lexer_by_tokens = {} + self.lexers = {} + for state, accepts in states.items(): + key = frozenset(accepts) + try: + lexer = lexer_by_tokens[key] + except KeyError: + accepts = set(accepts) | set(conf.ignore) | set(always_accept) + lexer_conf = copy(trad_conf) + lexer_conf.terminals = [terminals_by_name[n] for n in accepts if n in terminals_by_name] + lexer = TraditionalLexer(lexer_conf) + lexer_by_tokens[key] = lexer + + self.lexers[state] = lexer + + assert trad_conf.terminals is terminals + self.root_lexer = TraditionalLexer(trad_conf) + + def make_lexer_state(self, text): + return self.root_lexer.make_lexer_state(text) + + def lex(self, lexer_state, parser_state): + try: + while True: + lexer = self.lexers[parser_state.position] + yield lexer.next_token(lexer_state, parser_state) + except EOFError: + pass + except UnexpectedCharacters as e: + ## + + ## + + try: + last_token = lexer_state.last_token ## + + token = self.root_lexer.next_token(lexer_state, parser_state) + raise UnexpectedToken(token, e.allowed, state=parser_state, token_history=[last_token], terminals_by_name=self.root_lexer.terminals_by_name) + except UnexpectedCharacters: + raise e ## + + +class LexerThread(object): + #-- + + def __init__(self, lexer, text): + self.lexer = lexer + self.state = lexer.make_lexer_state(text) + + def lex(self, parser_state): + return self.lexer.lex(self.state, parser_state) + + def __copy__(self): + copied = object.__new__(LexerThread) + copied.lexer = self.lexer + copied.state = copy(self.state) + return copied + + + +class LexerConf(Serialize): + __serialize_fields__ = 'terminals', 'ignore', 'g_regex_flags', 'use_bytes', 'lexer_type' + __serialize_namespace__ = TerminalDef, + + def __init__(self, terminals, re_module, ignore=(), postlex=None, callbacks=None, g_regex_flags=0, skip_validation=False, use_bytes=False): + self.terminals = terminals + self.terminals_by_name = {t.name: t for t in self.terminals} + assert len(self.terminals) == len(self.terminals_by_name) + self.ignore = ignore + self.postlex = postlex + self.callbacks = callbacks or {} + self.g_regex_flags = g_regex_flags + self.re_module = re_module + self.skip_validation = skip_validation + self.use_bytes = use_bytes + self.lexer_type = None + + @property + def tokens(self): + warn("LexerConf.tokens is deprecated. Use LexerConf.terminals instead", DeprecationWarning) + return self.terminals + + def _deserialize(self): + self.terminals_by_name = {t.name: t for t in self.terminals} + + + +class ParserConf(Serialize): + __serialize_fields__ = 'rules', 'start', 'parser_type' + + def __init__(self, rules, callbacks, start): + assert isinstance(start, list) + self.rules = rules + self.callbacks = callbacks + self.start = start + + self.parser_type = None + + +from functools import partial, wraps +from itertools import repeat, product + + +class ExpandSingleChild: + def __init__(self, node_builder): + self.node_builder = node_builder + + def __call__(self, children): + if len(children) == 1: + return children[0] + else: + return self.node_builder(children) + + +class PropagatePositions: + def __init__(self, node_builder): + self.node_builder = node_builder + + def __call__(self, children): + res = self.node_builder(children) + + ## + + if isinstance(res, Tree): + res_meta = res.meta + for c in children: + if isinstance(c, Tree): + child_meta = c.meta + if not child_meta.empty: + res_meta.line = child_meta.line + res_meta.column = child_meta.column + res_meta.start_pos = child_meta.start_pos + res_meta.empty = False + break + elif isinstance(c, Token): + res_meta.line = c.line + res_meta.column = c.column + res_meta.start_pos = c.pos_in_stream + res_meta.empty = False + break + + for c in reversed(children): + if isinstance(c, Tree): + child_meta = c.meta + if not child_meta.empty: + res_meta.end_line = child_meta.end_line + res_meta.end_column = child_meta.end_column + res_meta.end_pos = child_meta.end_pos + res_meta.empty = False + break + elif isinstance(c, Token): + res_meta.end_line = c.end_line + res_meta.end_column = c.end_column + res_meta.end_pos = c.end_pos + res_meta.empty = False + break + + return res + + +class ChildFilter: + def __init__(self, to_include, append_none, node_builder): + self.node_builder = node_builder + self.to_include = to_include + self.append_none = append_none + + def __call__(self, children): + filtered = [] + + for i, to_expand, add_none in self.to_include: + if add_none: + filtered += [None] * add_none + if to_expand: + filtered += children[i].children + else: + filtered.append(children[i]) + + if self.append_none: + filtered += [None] * self.append_none + + return self.node_builder(filtered) + + +class ChildFilterLALR(ChildFilter): + #-- + + def __call__(self, children): + filtered = [] + for i, to_expand, add_none in self.to_include: + if add_none: + filtered += [None] * add_none + if to_expand: + if filtered: + filtered += children[i].children + else: ## + + filtered = children[i].children + else: + filtered.append(children[i]) + + if self.append_none: + filtered += [None] * self.append_none + + return self.node_builder(filtered) + + +class ChildFilterLALR_NoPlaceholders(ChildFilter): + #-- + def __init__(self, to_include, node_builder): + self.node_builder = node_builder + self.to_include = to_include + + def __call__(self, children): + filtered = [] + for i, to_expand in self.to_include: + if to_expand: + if filtered: + filtered += children[i].children + else: ## + + filtered = children[i].children + else: + filtered.append(children[i]) + return self.node_builder(filtered) + + +def _should_expand(sym): + return not sym.is_term and sym.name.startswith('_') + + +def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous, _empty_indices): + ## + + if _empty_indices: + assert _empty_indices.count(False) == len(expansion) + s = ''.join(str(int(b)) for b in _empty_indices) + empty_indices = [len(ones) for ones in s.split('0')] + assert len(empty_indices) == len(expansion)+1, (empty_indices, len(expansion)) + else: + empty_indices = [0] * (len(expansion)+1) + + to_include = [] + nones_to_add = 0 + for i, sym in enumerate(expansion): + nones_to_add += empty_indices[i] + if keep_all_tokens or not (sym.is_term and sym.filter_out): + to_include.append((i, _should_expand(sym), nones_to_add)) + nones_to_add = 0 + + nones_to_add += empty_indices[len(expansion)] + + if _empty_indices or len(to_include) < len(expansion) or any(to_expand for i, to_expand,_ in to_include): + if _empty_indices or ambiguous: + return partial(ChildFilter if ambiguous else ChildFilterLALR, to_include, nones_to_add) + else: + ## + + return partial(ChildFilterLALR_NoPlaceholders, [(i, x) for i,x,_ in to_include]) + + +class AmbiguousExpander: + #-- + def __init__(self, to_expand, tree_class, node_builder): + self.node_builder = node_builder + self.tree_class = tree_class + self.to_expand = to_expand + + def __call__(self, children): + def _is_ambig_tree(t): + return hasattr(t, 'data') and t.data == '_ambig' + + ## + + ## + + ## + + ## + + ambiguous = [] + for i, child in enumerate(children): + if _is_ambig_tree(child): + if i in self.to_expand: + ambiguous.append(i) + + to_expand = [j for j, grandchild in enumerate(child.children) if _is_ambig_tree(grandchild)] + child.expand_kids_by_index(*to_expand) + + if not ambiguous: + return self.node_builder(children) + + expand = [iter(child.children) if i in ambiguous else repeat(child) for i, child in enumerate(children)] + return self.tree_class('_ambig', [self.node_builder(list(f[0])) for f in product(zip(*expand))]) + + +def maybe_create_ambiguous_expander(tree_class, expansion, keep_all_tokens): + to_expand = [i for i, sym in enumerate(expansion) + if keep_all_tokens or ((not (sym.is_term and sym.filter_out)) and _should_expand(sym))] + if to_expand: + return partial(AmbiguousExpander, to_expand, tree_class) + + +class AmbiguousIntermediateExpander: + #-- + + def __init__(self, tree_class, node_builder): + self.node_builder = node_builder + self.tree_class = tree_class + + def __call__(self, children): + def _is_iambig_tree(child): + return hasattr(child, 'data') and child.data == '_iambig' + + def _collapse_iambig(children): + #-- + + ## + + ## + + if children and _is_iambig_tree(children[0]): + iambig_node = children[0] + result = [] + for grandchild in iambig_node.children: + collapsed = _collapse_iambig(grandchild.children) + if collapsed: + for child in collapsed: + child.children += children[1:] + result += collapsed + else: + new_tree = self.tree_class('_inter', grandchild.children + children[1:]) + result.append(new_tree) + return result + + collapsed = _collapse_iambig(children) + if collapsed: + processed_nodes = [self.node_builder(c.children) for c in collapsed] + return self.tree_class('_ambig', processed_nodes) + + return self.node_builder(children) + + +def ptb_inline_args(func): + @wraps(func) + def f(children): + return func(*children) + return f + + +def inplace_transformer(func): + @wraps(func) + def f(children): + ## + + tree = Tree(func.__name__, children) + return func(tree) + return f + + +def apply_visit_wrapper(func, name, wrapper): + if wrapper is _vargs_meta or wrapper is _vargs_meta_inline: + raise NotImplementedError("Meta args not supported for internal transformer") + + @wraps(func) + def f(children): + return wrapper(func, name, children, None) + return f + + +class ParseTreeBuilder: + def __init__(self, rules, tree_class, propagate_positions=False, ambiguous=False, maybe_placeholders=False): + self.tree_class = tree_class + self.propagate_positions = propagate_positions + self.ambiguous = ambiguous + self.maybe_placeholders = maybe_placeholders + + self.rule_builders = list(self._init_builders(rules)) + + def _init_builders(self, rules): + for rule in rules: + options = rule.options + keep_all_tokens = options.keep_all_tokens + expand_single_child = options.expand1 + + wrapper_chain = list(filter(None, [ + (expand_single_child and not rule.alias) and ExpandSingleChild, + maybe_create_child_filter(rule.expansion, keep_all_tokens, self.ambiguous, options.empty_indices if self.maybe_placeholders else None), + self.propagate_positions and PropagatePositions, + self.ambiguous and maybe_create_ambiguous_expander(self.tree_class, rule.expansion, keep_all_tokens), + self.ambiguous and partial(AmbiguousIntermediateExpander, self.tree_class) + ])) + + yield rule, wrapper_chain + + def create_callback(self, transformer=None): + callbacks = {} + + for rule, wrapper_chain in self.rule_builders: + + user_callback_name = rule.alias or rule.options.template_source or rule.origin.name + try: + f = getattr(transformer, user_callback_name) + ## + + wrapper = getattr(f, 'visit_wrapper', None) + if wrapper is not None: + f = apply_visit_wrapper(f, user_callback_name, wrapper) + else: + if isinstance(transformer, InlineTransformer): + f = ptb_inline_args(f) + elif isinstance(transformer, Transformer_InPlace): + f = inplace_transformer(f) + except AttributeError: + f = partial(self.tree_class, user_callback_name) + + for w in wrapper_chain: + f = w(f) + + if rule in callbacks: + raise GrammarError("Rule '%s' already exists" % (rule,)) + + callbacks[rule] = f + + return callbacks + + + +class LALR_Parser(Serialize): + def __init__(self, parser_conf, debug=False): + analysis = LALR_Analyzer(parser_conf, debug=debug) + analysis.compute_lalr() + callbacks = parser_conf.callbacks + + self._parse_table = analysis.parse_table + self.parser_conf = parser_conf + self.parser = _Parser(analysis.parse_table, callbacks, debug) + + @classmethod + def deserialize(cls, data, memo, callbacks, debug=False): + inst = cls.__new__(cls) + inst._parse_table = IntParseTable.deserialize(data, memo) + inst.parser = _Parser(inst._parse_table, callbacks, debug) + return inst + + def serialize(self, memo): + return self._parse_table.serialize(memo) + + def parse_interactive(self, lexer, start): + return self.parser.parse(lexer, start, start_interactive=True) + + def parse(self, lexer, start, on_error=None): + try: + return self.parser.parse(lexer, start) + except UnexpectedInput as e: + if on_error is None: + raise + + while True: + if isinstance(e, UnexpectedCharacters): + s = e.interactive_parser.lexer_state.state + p = s.line_ctr.char_pos + + if not on_error(e): + raise e + + if isinstance(e, UnexpectedCharacters): + ## + + if p == s.line_ctr.char_pos: + s.line_ctr.feed(s.text[p:p+1]) + + try: + return e.interactive_parser.resume_parse() + except UnexpectedToken as e2: + if (isinstance(e, UnexpectedToken) + and e.token.type == e2.token.type == '$END' + and e.interactive_parser == e2.interactive_parser): + ## + + raise e2 + e = e2 + except UnexpectedCharacters as e2: + e = e2 + + +class ParseConf(object): + __slots__ = 'parse_table', 'callbacks', 'start', 'start_state', 'end_state', 'states' + + def __init__(self, parse_table, callbacks, start): + self.parse_table = parse_table + + self.start_state = self.parse_table.start_states[start] + self.end_state = self.parse_table.end_states[start] + self.states = self.parse_table.states + + self.callbacks = callbacks + self.start = start + + +class ParserState(object): + __slots__ = 'parse_conf', 'lexer', 'state_stack', 'value_stack' + + def __init__(self, parse_conf, lexer, state_stack=None, value_stack=None): + self.parse_conf = parse_conf + self.lexer = lexer + self.state_stack = state_stack or [self.parse_conf.start_state] + self.value_stack = value_stack or [] + + @property + def position(self): + return self.state_stack[-1] + + ## + + def __eq__(self, other): + if not isinstance(other, ParserState): + return NotImplemented + return len(self.state_stack) == len(other.state_stack) and self.position == other.position + + def __copy__(self): + return type(self)( + self.parse_conf, + self.lexer, ## + + copy(self.state_stack), + deepcopy(self.value_stack), + ) + + def copy(self): + return copy(self) + + def feed_token(self, token, is_end=False): + state_stack = self.state_stack + value_stack = self.value_stack + states = self.parse_conf.states + end_state = self.parse_conf.end_state + callbacks = self.parse_conf.callbacks + + while True: + state = state_stack[-1] + try: + action, arg = states[state][token.type] + except KeyError: + expected = {s for s in states[state].keys() if s.isupper()} + raise UnexpectedToken(token, expected, state=self, interactive_parser=None) + + assert arg != end_state + + if action is Shift: + ## + + assert not is_end + state_stack.append(arg) + value_stack.append(token if token.type not in callbacks else callbacks[token.type](token)) + return + else: + ## + + rule = arg + size = len(rule.expansion) + if size: + s = value_stack[-size:] + del state_stack[-size:] + del value_stack[-size:] + else: + s = [] + + value = callbacks[rule](s) + + _action, new_state = states[state_stack[-1]][rule.origin.name] + assert _action is Shift + state_stack.append(new_state) + value_stack.append(value) + + if is_end and state_stack[-1] == end_state: + return value_stack[-1] + +class _Parser(object): + def __init__(self, parse_table, callbacks, debug=False): + self.parse_table = parse_table + self.callbacks = callbacks + self.debug = debug + + def parse(self, lexer, start, value_stack=None, state_stack=None, start_interactive=False): + parse_conf = ParseConf(self.parse_table, self.callbacks, start) + parser_state = ParserState(parse_conf, lexer, state_stack, value_stack) + if start_interactive: + return InteractiveParser(self, parser_state, parser_state.lexer) + return self.parse_from_state(parser_state) + + + def parse_from_state(self, state): + ## + + try: + token = None + for token in state.lexer.lex(state): + state.feed_token(token) + + token = Token.new_borrow_pos('$END', '', token) if token else Token('$END', '', 0, 1, 1) + return state.feed_token(token, True) + except UnexpectedInput as e: + try: + e.interactive_parser = InteractiveParser(self, state, state.lexer) + except NameError: + pass + raise e + except Exception as e: + if self.debug: + print("") + print("STATE STACK DUMP") + print("----------------") + for i, s in enumerate(state.state_stack): + print('%d)' % i , s) + print("") + + raise + + +class Action: + def __init__(self, name): + self.name = name + def __str__(self): + return self.name + def __repr__(self): + return str(self) + +Shift = Action('Shift') +Reduce = Action('Reduce') + + +class ParseTable: + def __init__(self, states, start_states, end_states): + self.states = states + self.start_states = start_states + self.end_states = end_states + + def serialize(self, memo): + tokens = Enumerator() + rules = Enumerator() + + states = { + state: {tokens.get(token): ((1, arg.serialize(memo)) if action is Reduce else (0, arg)) + for token, (action, arg) in actions.items()} + for state, actions in self.states.items() + } + + return { + 'tokens': tokens.reversed(), + 'states': states, + 'start_states': self.start_states, + 'end_states': self.end_states, + } + + @classmethod + def deserialize(cls, data, memo): + tokens = data['tokens'] + states = { + state: {tokens[token]: ((Reduce, Rule.deserialize(arg, memo)) if action==1 else (Shift, arg)) + for token, (action, arg) in actions.items()} + for state, actions in data['states'].items() + } + return cls(states, data['start_states'], data['end_states']) + + +class IntParseTable(ParseTable): + + @classmethod + def from_ParseTable(cls, parse_table): + enum = list(parse_table.states) + state_to_idx = {s:i for i,s in enumerate(enum)} + int_states = {} + + for s, la in parse_table.states.items(): + la = {k:(v[0], state_to_idx[v[1]]) if v[0] is Shift else v + for k,v in la.items()} + int_states[ state_to_idx[s] ] = la + + + start_states = {start:state_to_idx[s] for start, s in parse_table.start_states.items()} + end_states = {start:state_to_idx[s] for start, s in parse_table.end_states.items()} + return cls(int_states, start_states, end_states) + + + +def _wrap_lexer(lexer_class): + future_interface = getattr(lexer_class, '__future_interface__', False) + if future_interface: + return lexer_class + else: + class CustomLexerWrapper(Lexer): + def __init__(self, lexer_conf): + self.lexer = lexer_class(lexer_conf) + def lex(self, lexer_state, parser_state): + return self.lexer.lex(lexer_state.text) + return CustomLexerWrapper + + +class MakeParsingFrontend: + def __init__(self, parser_type, lexer_type): + self.parser_type = parser_type + self.lexer_type = lexer_type + + def __call__(self, lexer_conf, parser_conf, options): + assert isinstance(lexer_conf, LexerConf) + assert isinstance(parser_conf, ParserConf) + parser_conf.parser_type = self.parser_type + lexer_conf.lexer_type = self.lexer_type + return ParsingFrontend(lexer_conf, parser_conf, options) + + @classmethod + def deserialize(cls, data, memo, lexer_conf, callbacks, options): + parser_conf = ParserConf.deserialize(data['parser_conf'], memo) + parser = LALR_Parser.deserialize(data['parser'], memo, callbacks, options.debug) + parser_conf.callbacks = callbacks + return ParsingFrontend(lexer_conf, parser_conf, options, parser=parser) + + + + +class ParsingFrontend(Serialize): + __serialize_fields__ = 'lexer_conf', 'parser_conf', 'parser', 'options' + + def __init__(self, lexer_conf, parser_conf, options, parser=None): + self.parser_conf = parser_conf + self.lexer_conf = lexer_conf + self.options = options + + ## + + if parser: ## + + self.parser = parser + else: + create_parser = { + 'lalr': create_lalr_parser, + 'earley': create_earley_parser, + 'cyk': CYK_FrontEnd, + }[parser_conf.parser_type] + self.parser = create_parser(lexer_conf, parser_conf, options) + + ## + + lexer_type = lexer_conf.lexer_type + self.skip_lexer = False + if lexer_type in ('dynamic', 'dynamic_complete'): + assert lexer_conf.postlex is None + self.skip_lexer = True + return + + try: + create_lexer = { + 'standard': create_traditional_lexer, + 'contextual': create_contextual_lexer, + }[lexer_type] + except KeyError: + assert issubclass(lexer_type, Lexer), lexer_type + self.lexer = _wrap_lexer(lexer_type)(lexer_conf) + else: + self.lexer = create_lexer(lexer_conf, self.parser, lexer_conf.postlex) + + if lexer_conf.postlex: + self.lexer = PostLexConnector(self.lexer, lexer_conf.postlex) + + def _verify_start(self, start=None): + if start is None: + start = self.parser_conf.start + if len(start) > 1: + raise ConfigurationError("Lark initialized with more than 1 possible start rule. Must specify which start rule to parse", start) + start ,= start + elif start not in self.parser_conf.start: + raise ConfigurationError("Unknown start rule %s. Must be one of %r" % (start, self.parser_conf.start)) + return start + + def parse(self, text, start=None, on_error=None): + start = self._verify_start(start) + stream = text if self.skip_lexer else LexerThread(self.lexer, text) + kw = {} if on_error is None else {'on_error': on_error} + return self.parser.parse(stream, start, **kw) + + def parse_interactive(self, text=None, start=None): + start = self._verify_start(start) + if self.parser_conf.parser_type != 'lalr': + raise ConfigurationError("parse_interactive() currently only works with parser='lalr' ") + stream = text if self.skip_lexer else LexerThread(self.lexer, text) + return self.parser.parse_interactive(stream, start) + + +def get_frontend(parser, lexer): + assert_config(parser, ('lalr', 'earley', 'cyk')) + if not isinstance(lexer, type): ## + + expected = { + 'lalr': ('standard', 'contextual'), + 'earley': ('standard', 'dynamic', 'dynamic_complete'), + 'cyk': ('standard', ), + }[parser] + assert_config(lexer, expected, 'Parser %r does not support lexer %%r, expected one of %%s' % parser) + + return MakeParsingFrontend(parser, lexer) + + +def _get_lexer_callbacks(transformer, terminals): + result = {} + for terminal in terminals: + callback = getattr(transformer, terminal.name, None) + if callback is not None: + result[terminal.name] = callback + return result + +class PostLexConnector: + def __init__(self, lexer, postlexer): + self.lexer = lexer + self.postlexer = postlexer + + def make_lexer_state(self, text): + return self.lexer.make_lexer_state(text) + + def lex(self, lexer_state, parser_state): + i = self.lexer.lex(lexer_state, parser_state) + return self.postlexer.process(i) + + + +def create_traditional_lexer(lexer_conf, parser, postlex): + return TraditionalLexer(lexer_conf) + +def create_contextual_lexer(lexer_conf, parser, postlex): + states = {idx:list(t.keys()) for idx, t in parser._parse_table.states.items()} + always_accept = postlex.always_accept if postlex else () + return ContextualLexer(lexer_conf, states, always_accept=always_accept) + +def create_lalr_parser(lexer_conf, parser_conf, options=None): + debug = options.debug if options else False + return LALR_Parser(parser_conf, debug=debug) + + +create_earley_parser = NotImplemented +CYK_FrontEnd = NotImplemented + + + +class LarkOptions(Serialize): + #-- + OPTIONS_DOC = """ + **=== General Options ===** + + start + The start symbol. Either a string, or a list of strings for multiple possible starts (Default: "start") + debug + Display debug information and extra warnings. Use only when debugging (default: False) + When used with Earley, it generates a forest graph as "sppf.png", if 'dot' is installed. + transformer + Applies the transformer to every parse tree (equivalent to applying it after the parse, but faster) + propagate_positions + Propagates (line, column, end_line, end_column) attributes into all tree branches. + maybe_placeholders + When True, the ``[]`` operator returns ``None`` when not matched. + + When ``False``, ``[]`` behaves like the ``?`` operator, and returns no value at all. + (default= ``False``. Recommended to set to ``True``) + cache + Cache the results of the Lark grammar analysis, for x2 to x3 faster loading. LALR only for now. + + - When ``False``, does nothing (default) + - When ``True``, caches to a temporary file in the local directory + - When given a string, caches to the path pointed by the string + regex + When True, uses the ``regex`` module instead of the stdlib ``re``. + g_regex_flags + Flags that are applied to all terminals (both regex and strings) + keep_all_tokens + Prevent the tree builder from automagically removing "punctuation" tokens (default: False) + tree_class + Lark will produce trees comprised of instances of this class instead of the default ``lark.Tree``. + + **=== Algorithm Options ===** + + parser + Decides which parser engine to use. Accepts "earley" or "lalr". (Default: "earley"). + (there is also a "cyk" option for legacy) + lexer + Decides whether or not to use a lexer stage + + - "auto" (default): Choose for me based on the parser + - "standard": Use a standard lexer + - "contextual": Stronger lexer (only works with parser="lalr") + - "dynamic": Flexible and powerful (only with parser="earley") + - "dynamic_complete": Same as dynamic, but tries *every* variation of tokenizing possible. + ambiguity + Decides how to handle ambiguity in the parse. Only relevant if parser="earley" + + - "resolve": The parser will automatically choose the simplest derivation + (it chooses consistently: greedy for tokens, non-greedy for rules) + - "explicit": The parser will return all derivations wrapped in "_ambig" tree nodes (i.e. a forest). + - "forest": The parser will return the root of the shared packed parse forest. + + **=== Misc. / Domain Specific Options ===** + + postlex + Lexer post-processing (Default: None) Only works with the standard and contextual lexers. + priority + How priorities should be evaluated - auto, none, normal, invert (Default: auto) + lexer_callbacks + Dictionary of callbacks for the lexer. May alter tokens during lexing. Use with caution. + use_bytes + Accept an input of type ``bytes`` instead of ``str`` (Python 3 only). + edit_terminals + A callback for editing the terminals before parse. + import_paths + A List of either paths or loader functions to specify from where grammars are imported + source_path + Override the source of from where the grammar was loaded. Useful for relative imports and unconventional grammar loading + **=== End Options ===** + """ + if __doc__: + __doc__ += OPTIONS_DOC + + + ## + + ## + + ## + + ## + + ## + + ## + + ## + + ## + + _defaults = { + 'debug': False, + 'keep_all_tokens': False, + 'tree_class': None, + 'cache': False, + 'postlex': None, + 'parser': 'earley', + 'lexer': 'auto', + 'transformer': None, + 'start': 'start', + 'priority': 'auto', + 'ambiguity': 'auto', + 'regex': False, + 'propagate_positions': False, + 'lexer_callbacks': {}, + 'maybe_placeholders': False, + 'edit_terminals': None, + 'g_regex_flags': 0, + 'use_bytes': False, + 'import_paths': [], + 'source_path': None, + } + + def __init__(self, options_dict): + o = dict(options_dict) + + options = {} + for name, default in self._defaults.items(): + if name in o: + value = o.pop(name) + if isinstance(default, bool) and name not in ('cache', 'use_bytes'): + value = bool(value) + else: + value = default + + options[name] = value + + if isinstance(options['start'], STRING_TYPE): + options['start'] = [options['start']] + + self.__dict__['options'] = options + + + assert_config(self.parser, ('earley', 'lalr', 'cyk', None)) + + if self.parser == 'earley' and self.transformer: + raise ConfigurationError('Cannot specify an embedded transformer when using the Earley algorithm.' + 'Please use your transformer on the resulting parse tree, or use a different algorithm (i.e. LALR)') + + if o: + raise ConfigurationError("Unknown options: %s" % o.keys()) + + def __getattr__(self, name): + try: + return self.__dict__['options'][name] + except KeyError as e: + raise AttributeError(e) + + def __setattr__(self, name, value): + assert_config(name, self.options.keys(), "%r isn't a valid option. Expected one of: %s") + self.options[name] = value + + def serialize(self, memo): + return self.options + + @classmethod + def deserialize(cls, data, memo): + return cls(data) + + +## + +## + +_LOAD_ALLOWED_OPTIONS = {'postlex', 'transformer', 'lexer_callbacks', 'use_bytes', 'debug', 'g_regex_flags', 'regex', 'propagate_positions', 'tree_class'} + +_VALID_PRIORITY_OPTIONS = ('auto', 'normal', 'invert', None) +_VALID_AMBIGUITY_OPTIONS = ('auto', 'resolve', 'explicit', 'forest') + + +class PostLex(ABC): + @abstractmethod + def process(self, stream): + return stream + + always_accept = () + + +class Lark(Serialize): + #-- + def __init__(self, grammar, **options): + self.options = LarkOptions(options) + + ## + + use_regex = self.options.regex + if use_regex: + if regex: + re_module = regex + else: + raise ImportError('`regex` module must be installed if calling `Lark(regex=True)`.') + else: + re_module = re + + ## + + if self.options.source_path is None: + try: + self.source_path = grammar.name + except AttributeError: + self.source_path = '' + else: + self.source_path = self.options.source_path + + ## + + try: + read = grammar.read + except AttributeError: + pass + else: + grammar = read() + + cache_fn = None + cache_md5 = None + if isinstance(grammar, STRING_TYPE): + self.source_grammar = grammar + if self.options.use_bytes: + if not isascii(grammar): + raise ConfigurationError("Grammar must be ascii only, when use_bytes=True") + if sys.version_info[0] == 2 and self.options.use_bytes != 'force': + raise ConfigurationError("`use_bytes=True` may have issues on python2." + "Use `use_bytes='force'` to use it at your own risk.") + + if self.options.cache: + if self.options.parser != 'lalr': + raise ConfigurationError("cache only works with parser='lalr' for now") + + unhashable = ('transformer', 'postlex', 'lexer_callbacks', 'edit_terminals') + options_str = ''.join(k+str(v) for k, v in options.items() if k not in unhashable) + from . import __version__ + s = grammar + options_str + __version__ + str(sys.version_info[:2]) + cache_md5 = hashlib.md5(s.encode('utf8')).hexdigest() + + if isinstance(self.options.cache, STRING_TYPE): + cache_fn = self.options.cache + else: + if self.options.cache is not True: + raise ConfigurationError("cache argument must be bool or str") + ## + + cache_fn = tempfile.gettempdir() + '/.lark_cache_%s_%s_%s.tmp' % ((cache_md5,) + sys.version_info[:2]) + + if FS.exists(cache_fn): + logger.debug('Loading grammar from cache: %s', cache_fn) + ## + + for name in (set(options) - _LOAD_ALLOWED_OPTIONS): + del options[name] + with FS.open(cache_fn, 'rb') as f: + old_options = self.options + try: + file_md5 = f.readline().rstrip(b'\n') + cached_used_files = pickle.load(f) + if file_md5 == cache_md5.encode('utf8') and verify_used_files(cached_used_files): + cached_parser_data = pickle.load(f) + self._load(cached_parser_data, **options) + return + except Exception: ## + + logger.exception("Failed to load Lark from cache: %r. We will try to carry on." % cache_fn) + + ## + + ## + + self.options = old_options + + + ## + + self.grammar, used_files = load_grammar(grammar, self.source_path, self.options.import_paths, self.options.keep_all_tokens) + else: + assert isinstance(grammar, Grammar) + self.grammar = grammar + + + if self.options.lexer == 'auto': + if self.options.parser == 'lalr': + self.options.lexer = 'contextual' + elif self.options.parser == 'earley': + if self.options.postlex is not None: + logger.info("postlex can't be used with the dynamic lexer, so we use standard instead. " + "Consider using lalr with contextual instead of earley") + self.options.lexer = 'standard' + else: + self.options.lexer = 'dynamic' + elif self.options.parser == 'cyk': + self.options.lexer = 'standard' + else: + assert False, self.options.parser + lexer = self.options.lexer + if isinstance(lexer, type): + assert issubclass(lexer, Lexer) ## + + else: + assert_config(lexer, ('standard', 'contextual', 'dynamic', 'dynamic_complete')) + if self.options.postlex is not None and 'dynamic' in lexer: + raise ConfigurationError("Can't use postlex with a dynamic lexer. Use standard or contextual instead") + + if self.options.ambiguity == 'auto': + if self.options.parser == 'earley': + self.options.ambiguity = 'resolve' + else: + assert_config(self.options.parser, ('earley', 'cyk'), "%r doesn't support disambiguation. Use one of these parsers instead: %s") + + if self.options.priority == 'auto': + self.options.priority = 'normal' + + if self.options.priority not in _VALID_PRIORITY_OPTIONS: + raise ConfigurationError("invalid priority option: %r. Must be one of %r" % (self.options.priority, _VALID_PRIORITY_OPTIONS)) + assert self.options.ambiguity not in ('resolve__antiscore_sum', ), 'resolve__antiscore_sum has been replaced with the option priority="invert"' + if self.options.ambiguity not in _VALID_AMBIGUITY_OPTIONS: + raise ConfigurationError("invalid ambiguity option: %r. Must be one of %r" % (self.options.ambiguity, _VALID_AMBIGUITY_OPTIONS)) + + if self.options.postlex is not None: + terminals_to_keep = set(self.options.postlex.always_accept) + else: + terminals_to_keep = set() + + ## + + self.terminals, self.rules, self.ignore_tokens = self.grammar.compile(self.options.start, terminals_to_keep) + + if self.options.edit_terminals: + for t in self.terminals: + self.options.edit_terminals(t) + + self._terminals_dict = {t.name: t for t in self.terminals} + + ## + + ## + + if self.options.priority == 'invert': + for rule in self.rules: + if rule.options.priority is not None: + rule.options.priority = -rule.options.priority + ## + + ## + + ## + + elif self.options.priority is None: + for rule in self.rules: + if rule.options.priority is not None: + rule.options.priority = None + + ## + + self.lexer_conf = LexerConf( + self.terminals, re_module, self.ignore_tokens, self.options.postlex, + self.options.lexer_callbacks, self.options.g_regex_flags, use_bytes=self.options.use_bytes + ) + + if self.options.parser: + self.parser = self._build_parser() + elif lexer: + self.lexer = self._build_lexer() + + if cache_fn: + logger.debug('Saving grammar to cache: %s', cache_fn) + with FS.open(cache_fn, 'wb') as f: + f.write(cache_md5.encode('utf8') + b'\n') + pickle.dump(used_files, f) + self.save(f) + + if __doc__: + __doc__ += "\n\n" + LarkOptions.OPTIONS_DOC + + __serialize_fields__ = 'parser', 'rules', 'options' + + def _build_lexer(self, dont_ignore=False): + lexer_conf = self.lexer_conf + if dont_ignore: + from copy import copy + lexer_conf = copy(lexer_conf) + lexer_conf.ignore = () + return TraditionalLexer(lexer_conf) + + def _prepare_callbacks(self): + self._callbacks = {} + ## + + if self.options.ambiguity != 'forest': + self._parse_tree_builder = ParseTreeBuilder( + self.rules, + self.options.tree_class or Tree, + self.options.propagate_positions, + self.options.parser != 'lalr' and self.options.ambiguity == 'explicit', + self.options.maybe_placeholders + ) + self._callbacks = self._parse_tree_builder.create_callback(self.options.transformer) + self._callbacks.update(_get_lexer_callbacks(self.options.transformer, self.terminals)) + + def _build_parser(self): + self._prepare_callbacks() + parser_class = get_frontend(self.options.parser, self.options.lexer) + parser_conf = ParserConf(self.rules, self._callbacks, self.options.start) + return parser_class(self.lexer_conf, parser_conf, options=self.options) + + def save(self, f): + #-- + data, m = self.memo_serialize([TerminalDef, Rule]) + pickle.dump({'data': data, 'memo': m}, f, protocol=pickle.HIGHEST_PROTOCOL) + + @classmethod + def load(cls, f): + #-- + inst = cls.__new__(cls) + return inst._load(f) + + def _deserialize_lexer_conf(self, data, memo, options): + lexer_conf = LexerConf.deserialize(data['lexer_conf'], memo) + lexer_conf.callbacks = options.lexer_callbacks or {} + lexer_conf.re_module = regex if options.regex else re + lexer_conf.use_bytes = options.use_bytes + lexer_conf.g_regex_flags = options.g_regex_flags + lexer_conf.skip_validation = True + lexer_conf.postlex = options.postlex + return lexer_conf + + def _load(self, f, **kwargs): + if isinstance(f, dict): + d = f + else: + d = pickle.load(f) + memo = d['memo'] + data = d['data'] + + assert memo + memo = SerializeMemoizer.deserialize(memo, {'Rule': Rule, 'TerminalDef': TerminalDef}, {}) + options = dict(data['options']) + if (set(kwargs) - _LOAD_ALLOWED_OPTIONS) & set(LarkOptions._defaults): + raise ConfigurationError("Some options are not allowed when loading a Parser: {}" + .format(set(kwargs) - _LOAD_ALLOWED_OPTIONS)) + options.update(kwargs) + self.options = LarkOptions.deserialize(options, memo) + self.rules = [Rule.deserialize(r, memo) for r in data['rules']] + self.source_path = '' + parser_class = get_frontend(self.options.parser, self.options.lexer) + self.lexer_conf = self._deserialize_lexer_conf(data['parser'], memo, self.options) + self.terminals = self.lexer_conf.terminals + self._prepare_callbacks() + self._terminals_dict = {t.name: t for t in self.terminals} + self.parser = parser_class.deserialize( + data['parser'], + memo, + self.lexer_conf, + self._callbacks, + self.options, ## + + ) + return self + + @classmethod + def _load_from_dict(cls, data, memo, **kwargs): + inst = cls.__new__(cls) + return inst._load({'data': data, 'memo': memo}, **kwargs) + + @classmethod + def open(cls, grammar_filename, rel_to=None, **options): + #-- + if rel_to: + basepath = os.path.dirname(rel_to) + grammar_filename = os.path.join(basepath, grammar_filename) + with open(grammar_filename, encoding='utf8') as f: + return cls(f, **options) + + @classmethod + def open_from_package(cls, package, grammar_path, search_paths=("",), **options): + #-- + package = FromPackageLoader(package, search_paths) + full_path, text = package(None, grammar_path) + options.setdefault('source_path', full_path) + options.setdefault('import_paths', []) + options['import_paths'].append(package) + return cls(text, **options) + + def __repr__(self): + return 'Lark(open(%r), parser=%r, lexer=%r, ...)' % (self.source_path, self.options.parser, self.options.lexer) + + + def lex(self, text, dont_ignore=False): + #-- + if not hasattr(self, 'lexer') or dont_ignore: + lexer = self._build_lexer(dont_ignore) + else: + lexer = self.lexer + lexer_thread = LexerThread(lexer, text) + stream = lexer_thread.lex(None) + if self.options.postlex: + return self.options.postlex.process(stream) + return stream + + def get_terminal(self, name): + #-- + return self._terminals_dict[name] + + def parse_interactive(self, text=None, start=None): + return self.parser.parse_interactive(text, start=start) + + def parse(self, text, start=None, on_error=None): + #-- + return self.parser.parse(text, start=start, on_error=on_error) + + @property + def source(self): + warn("Lark.source attribute has been renamed to Lark.source_path", DeprecationWarning) + return self.source_path + + @source.setter + def source(self, value): + self.source_path = value + + @property + def grammar_source(self): + warn("Lark.grammar_source attribute has been renamed to Lark.source_grammar", DeprecationWarning) + return self.source_grammar + + @grammar_source.setter + def grammar_source(self, value): + self.source_grammar = value + + + +class DedentError(LarkError): + pass + +class Indenter(PostLex): + def __init__(self): + self.paren_level = None + self.indent_level = None + assert self.tab_len > 0 + + def handle_NL(self, token): + if self.paren_level > 0: + return + + yield token + + indent_str = token.rsplit('\n', 1)[1] ## + + indent = indent_str.count(' ') + indent_str.count('\t') * self.tab_len + + if indent > self.indent_level[-1]: + self.indent_level.append(indent) + yield Token.new_borrow_pos(self.INDENT_type, indent_str, token) + else: + while indent < self.indent_level[-1]: + self.indent_level.pop() + yield Token.new_borrow_pos(self.DEDENT_type, indent_str, token) + + if indent != self.indent_level[-1]: + raise DedentError('Unexpected dedent to column %s. Expected dedent to %s' % (indent, self.indent_level[-1])) + + def _process(self, stream): + for token in stream: + if token.type == self.NL_type: + for t in self.handle_NL(token): + yield t + else: + yield token + + if token.type in self.OPEN_PAREN_types: + self.paren_level += 1 + elif token.type in self.CLOSE_PAREN_types: + self.paren_level -= 1 + assert self.paren_level >= 0 + + while len(self.indent_level) > 1: + self.indent_level.pop() + yield Token(self.DEDENT_type, '') + + assert self.indent_level == [0], self.indent_level + + def process(self, stream): + self.paren_level = 0 + self.indent_level = [0] + return self._process(stream) + + ## + + @property + def always_accept(self): + return (self.NL_type,) + + +import pickle, zlib, base64 +DATA = ( +{'parser': {'lexer_conf': {'terminals': [{'@': 0}, {'@': 1}, {'@': 2}, {'@': 3}, {'@': 4}, {'@': 5}, {'@': 6}, {'@': 7}, {'@': 8}, {'@': 9}, {'@': 10}, {'@': 11}, {'@': 12}, {'@': 13}, {'@': 14}, {'@': 15}, {'@': 16}, {'@': 17}, {'@': 18}], 'ignore': ['WS'], 'g_regex_flags': 0, 'use_bytes': False, 'lexer_type': 'contextual', '__type__': 'LexerConf'}, 'parser_conf': {'rules': [{'@': 19}, {'@': 20}, {'@': 21}, {'@': 22}, {'@': 23}, {'@': 24}, {'@': 25}, {'@': 26}, {'@': 27}, {'@': 28}, {'@': 29}, {'@': 30}, {'@': 31}, {'@': 32}, {'@': 33}, {'@': 34}, {'@': 35}, {'@': 36}, {'@': 37}, {'@': 38}, {'@': 39}, {'@': 40}, {'@': 41}, {'@': 42}, {'@': 43}, {'@': 44}, {'@': 45}, {'@': 46}, {'@': 47}, {'@': 48}, {'@': 49}, {'@': 50}, {'@': 51}, {'@': 52}, {'@': 53}, {'@': 54}, {'@': 55}, {'@': 56}, {'@': 57}, {'@': 58}, {'@': 59}, {'@': 60}, {'@': 61}, {'@': 62}, {'@': 63}, {'@': 64}, {'@': 65}, {'@': 66}, {'@': 67}, {'@': 68}, {'@': 69}, {'@': 70}, {'@': 71}, {'@': 72}, {'@': 73}, {'@': 74}, {'@': 75}, {'@': 76}, {'@': 77}, {'@': 78}, {'@': 79}, {'@': 80}, {'@': 81}, {'@': 82}, {'@': 83}, {'@': 84}, {'@': 85}, {'@': 86}, {'@': 87}, {'@': 88}, {'@': 89}, {'@': 90}, {'@': 91}, {'@': 92}, {'@': 93}, {'@': 94}, {'@': 95}, {'@': 96}, {'@': 97}, {'@': 98}, {'@': 99}, {'@': 100}], 'start': ['start'], 'parser_type': 'lalr', '__type__': 'ParserConf'}, 'parser': {'tokens': {0: '__start_plus_1', 1: 'OBJECT', 2: 'description', 3: 'IMPORT', 4: 'INTERFACE', 5: 'start', 6: 'import_statement', 7: 'object', 8: '__start_star_0', 9: 'interface', 10: '__description_plus_6', 11: 'COMMENT', 12: 'name', 13: 'IDENTIFIER', 14: 'RBRACE', 15: 'PARAM', 16: 'METHOD', 17: 'method', 18: '__object_star_2', 19: '$END', 20: 'COLON', 21: 'LBRACE', 22: 'LSQB', 23: 'super', 24: 'options', 25: 'FUNCTION', 26: 'EXPOSE', 27: 'param', 28: 'object_name', 29: 'type', 30: 'PRIMITIVE', 31: '__function_star_4', 32: 'interface_param', 33: 'function', 34: 'expose', 35: 'PATH', 36: 'uid', 37: '__ANON_0', 38: 'RSQB', 39: '__interface_star_3', 40: 'UID', 41: '__options_plus_5'}, 'states': {0: {0: (0, 90), 1: (0, 129), 2: (0, 59), 3: (0, 39), 4: (0, 52), 5: (0, 25), 6: (0, 37), 7: (0, 8), 8: (0, 21), 9: (0, 66), 10: (0, 86), 11: (0, 111)}, 1: {12: (0, 112), 13: (0, 31)}, 2: {10: (0, 86), 11: (0, 111), 2: (0, 30), 14: (1, {'@': 76}), 15: (1, {'@': 76})}, 3: {12: (0, 95), 13: (0, 31)}, 4: {16: (1, {'@': 63}), 14: (1, {'@': 63}), 11: (1, {'@': 63})}, 5: {16: (0, 157), 17: (0, 126), 14: (0, 6), 2: (0, 10), 10: (0, 86), 18: (0, 22), 11: (0, 111)}, 6: {19: (1, {'@': 29}), 11: (1, {'@': 29}), 1: (1, {'@': 29}), 4: (1, {'@': 29})}, 7: {16: (1, {'@': 67}), 14: (1, {'@': 67}), 11: (1, {'@': 67})}, 8: {19: (1, {'@': 87}), 11: (1, {'@': 87}), 1: (1, {'@': 87}), 4: (1, {'@': 87})}, 9: {18: (0, 114), 16: (0, 157), 14: (0, 54), 17: (0, 126), 2: (0, 10), 10: (0, 86), 11: (0, 111)}, 10: {16: (0, 135)}, 11: {20: (0, 26), 21: (0, 134), 22: (0, 161), 23: (0, 15), 24: (0, 34)}, 12: {16: (0, 157), 17: (0, 126), 2: (0, 10), 10: (0, 86), 18: (0, 27), 11: (0, 111), 14: (0, 80)}, 13: {19: (1, {'@': 44}), 11: (1, {'@': 44}), 1: (1, {'@': 44}), 4: (1, {'@': 44})}, 14: {25: (1, {'@': 48}), 26: (1, {'@': 48}), 14: (1, {'@': 48}), 11: (1, {'@': 48})}, 15: {21: (0, 70)}, 16: {19: (1, {'@': 37}), 11: (1, {'@': 37}), 1: (1, {'@': 37}), 4: (1, {'@': 37})}, 17: {19: (1, {'@': 34}), 11: (1, {'@': 34}), 1: (1, {'@': 34}), 4: (1, {'@': 34})}, 18: {15: (0, 61), 14: (0, 105), 27: (0, 116)}, 19: {28: (0, 63), 29: (0, 14), 1: (0, 118), 30: (0, 127)}, 20: {15: (0, 61), 14: (0, 76), 27: (0, 116)}, 21: {1: (0, 129), 0: (0, 141), 2: (0, 59), 3: (0, 39), 4: (0, 52), 6: (0, 53), 7: (0, 8), 9: (0, 66), 10: (0, 86), 11: (0, 111)}, 22: {16: (0, 157), 2: (0, 10), 10: (0, 86), 11: (0, 111), 17: (0, 136), 14: (0, 115)}, 23: {14: (1, {'@': 77}), 15: (1, {'@': 77})}, 24: {25: (1, {'@': 46}), 26: (1, {'@': 46}), 14: (1, {'@': 46}), 11: (1, {'@': 46})}, 25: {}, 26: {13: (0, 31), 12: (0, 49)}, 27: {16: (0, 157), 2: (0, 10), 10: (0, 86), 11: (0, 111), 14: (0, 82), 17: (0, 136)}, 28: {15: (0, 61), 14: (0, 36), 27: (0, 116)}, 29: {15: (0, 61), 14: (0, 140), 27: (0, 116)}, 30: {14: (1, {'@': 75}), 15: (1, {'@': 75})}, 31: {25: (1, {'@': 82}), 22: (1, {'@': 82}), 26: (1, {'@': 82}), 14: (1, {'@': 82}), 11: (1, {'@': 82}), 21: (1, {'@': 82}), 30: (1, {'@': 82}), 1: (1, {'@': 82}), 20: (1, {'@': 82}), 15: (1, {'@': 82}), 16: (1, {'@': 82})}, 32: {21: (0, 151)}, 33: {31: (0, 29), 14: (0, 7), 27: (0, 72), 15: (0, 61)}, 34: {20: (0, 26), 21: (0, 60), 23: (0, 48)}, 35: {25: (1, {'@': 47}), 26: (1, {'@': 47}), 14: (1, {'@': 47}), 11: (1, {'@': 47})}, 36: {25: (1, {'@': 51}), 26: (1, {'@': 51}), 14: (1, {'@': 51}), 11: (1, {'@': 51})}, 37: {3: (1, {'@': 85}), 1: (1, {'@': 85}), 4: (1, {'@': 85}), 11: (1, {'@': 85})}, 38: {25: (0, 1), 2: (0, 43), 10: (0, 86), 32: (0, 84), 33: (0, 35), 34: (0, 24), 11: (0, 111), 26: (0, 19), 14: (0, 132)}, 39: {35: (0, 130)}, 40: {36: (0, 159), 37: (0, 158)}, 41: {19: (1, {'@': 39}), 11: (1, {'@': 39}), 1: (1, {'@': 39}), 4: (1, {'@': 39})}, 42: {25: (1, {'@': 81}), 26: (1, {'@': 81}), 14: (1, {'@': 81}), 11: (1, {'@': 81}), 22: (1, {'@': 81}), 15: (1, {'@': 81})}, 43: {25: (0, 3)}, 44: {19: (1, {'@': 42}), 11: (1, {'@': 42}), 1: (1, {'@': 42}), 4: (1, {'@': 42})}, 45: {25: (1, {'@': 58}), 26: (1, {'@': 58}), 14: (1, {'@': 58}), 11: (1, {'@': 58})}, 46: {21: (0, 138), 25: (1, {'@': 59}), 26: (1, {'@': 59}), 14: (1, {'@': 59}), 11: (1, {'@': 59})}, 47: {19: (1, {'@': 26}), 11: (1, {'@': 26}), 1: (1, {'@': 26}), 4: (1, {'@': 26})}, 48: {21: (0, 55)}, 49: {21: (1, {'@': 50})}, 50: {21: (0, 71), 16: (1, {'@': 71}), 14: (1, {'@': 71}), 11: (1, {'@': 71})}, 51: {16: (1, {'@': 64}), 14: (1, {'@': 64}), 11: (1, {'@': 64})}, 52: {12: (0, 100), 13: (0, 31)}, 53: {3: (1, {'@': 86}), 1: (1, {'@': 86}), 4: (1, {'@': 86}), 11: (1, {'@': 86})}, 54: {19: (1, {'@': 35}), 11: (1, {'@': 35}), 1: (1, {'@': 35}), 4: (1, {'@': 35})}, 55: {36: (0, 57), 37: (0, 158)}, 56: {16: (0, 157), 2: (0, 10), 14: (0, 104), 10: (0, 86), 11: (0, 111), 17: (0, 136)}, 57: {16: (0, 157), 14: (0, 83), 17: (0, 126), 2: (0, 10), 10: (0, 86), 18: (0, 85), 11: (0, 111)}, 58: {15: (0, 61), 14: (0, 92), 27: (0, 116)}, 59: {1: (0, 98), 4: (0, 67)}, 60: {36: (0, 12), 37: (0, 158)}, 61: {12: (0, 122), 13: (0, 31)}, 62: {25: (1, {'@': 93}), 26: (1, {'@': 93}), 14: (1, {'@': 93}), 11: (1, {'@': 93})}, 63: {25: (1, {'@': 80}), 26: (1, {'@': 80}), 14: (1, {'@': 80}), 11: (1, {'@': 80}), 15: (1, {'@': 80}), 22: (1, {'@': 80})}, 64: {19: (1, {'@': 40}), 11: (1, {'@': 40}), 1: (1, {'@': 40}), 4: (1, {'@': 40})}, 65: {25: (0, 1), 2: (0, 43), 10: (0, 86), 14: (0, 44), 32: (0, 84), 33: (0, 35), 34: (0, 24), 11: (0, 111), 26: (0, 19)}, 66: {19: (1, {'@': 88}), 11: (1, {'@': 88}), 1: (1, {'@': 88}), 4: (1, {'@': 88})}, 67: {12: (0, 162), 13: (0, 31)}, 68: {19: (1, {'@': 41}), 11: (1, {'@': 41}), 1: (1, {'@': 41}), 4: (1, {'@': 41})}, 69: {16: (1, {'@': 73}), 14: (1, {'@': 73}), 11: (1, {'@': 73})}, 70: {37: (0, 158), 36: (0, 77)}, 71: {31: (0, 18), 27: (0, 72), 15: (0, 61), 14: (0, 110)}, 72: {14: (1, {'@': 95}), 15: (1, {'@': 95})}, 73: {37: (0, 158), 36: (0, 154)}, 74: {25: (0, 1), 2: (0, 43), 10: (0, 86), 32: (0, 84), 33: (0, 35), 34: (0, 24), 11: (0, 111), 26: (0, 19), 14: (0, 64)}, 75: {19: (1, {'@': 27}), 11: (1, {'@': 27}), 1: (1, {'@': 27}), 4: (1, {'@': 27})}, 76: {25: (1, {'@': 57}), 26: (1, {'@': 57}), 14: (1, {'@': 57}), 11: (1, {'@': 57})}, 77: {16: (0, 157), 17: (0, 126), 2: (0, 10), 10: (0, 86), 18: (0, 145), 11: (0, 111), 14: (0, 75)}, 78: {19: (1, {'@': 90}), 11: (1, {'@': 90}), 1: (1, {'@': 90}), 4: (1, {'@': 90})}, 79: {16: (0, 157), 2: (0, 10), 14: (0, 119), 10: (0, 86), 11: (0, 111), 17: (0, 136)}, 80: {19: (1, {'@': 25}), 11: (1, {'@': 25}), 1: (1, {'@': 25}), 4: (1, {'@': 25})}, 81: {19: (1, {'@': 33}), 11: (1, {'@': 33}), 1: (1, {'@': 33}), 4: (1, {'@': 33})}, 82: {19: (1, {'@': 24}), 11: (1, {'@': 24}), 1: (1, {'@': 24}), 4: (1, {'@': 24})}, 83: {19: (1, {'@': 23}), 11: (1, {'@': 23}), 1: (1, {'@': 23}), 4: (1, {'@': 23})}, 84: {25: (1, {'@': 94}), 26: (1, {'@': 94}), 14: (1, {'@': 94}), 11: (1, {'@': 94})}, 85: {14: (0, 139), 16: (0, 157), 2: (0, 10), 10: (0, 86), 11: (0, 111), 17: (0, 136)}, 86: {11: (0, 133), 1: (1, {'@': 84}), 4: (1, {'@': 84}), 14: (1, {'@': 84}), 15: (1, {'@': 84}), 16: (1, {'@': 84}), 25: (1, {'@': 84})}, 87: {20: (0, 26), 23: (0, 150), 22: (0, 161), 24: (0, 164), 21: (0, 155)}, 88: {13: (1, {'@': 98}), 38: (1, {'@': 98})}, 89: {31: (0, 97), 27: (0, 72), 14: (0, 99), 15: (0, 61)}, 90: {1: (0, 129), 2: (0, 59), 9: (0, 78), 10: (0, 86), 4: (0, 52), 7: (0, 143), 11: (0, 111), 19: (1, {'@': 20})}, 91: {21: (0, 128), 25: (1, {'@': 53}), 26: (1, {'@': 53}), 14: (1, {'@': 53}), 11: (1, {'@': 53})}, 92: {16: (1, {'@': 72}), 14: (1, {'@': 72}), 11: (1, {'@': 72})}, 93: {19: (1, {'@': 30}), 11: (1, {'@': 30}), 1: (1, {'@': 30}), 4: (1, {'@': 30})}, 94: {19: (1, {'@': 43}), 11: (1, {'@': 43}), 1: (1, {'@': 43}), 4: (1, {'@': 43})}, 95: {24: (0, 91), 22: (0, 161), 21: (0, 108), 25: (1, {'@': 56}), 26: (1, {'@': 56}), 14: (1, {'@': 56}), 11: (1, {'@': 56})}, 96: {25: (1, {'@': 60}), 26: (1, {'@': 60}), 14: (1, {'@': 60}), 11: (1, {'@': 60})}, 97: {15: (0, 61), 14: (0, 96), 27: (0, 116)}, 98: {12: (0, 11), 13: (0, 31)}, 99: {25: (1, {'@': 61}), 26: (1, {'@': 61}), 14: (1, {'@': 61}), 11: (1, {'@': 61})}, 100: {24: (0, 153), 21: (0, 147), 22: (0, 161)}, 101: {37: (0, 158), 36: (0, 123)}, 102: {16: (0, 157), 17: (0, 126), 2: (0, 10), 10: (0, 86), 18: (0, 79), 11: (0, 111), 14: (0, 81)}, 103: {19: (1, {'@': 45}), 11: (1, {'@': 45}), 1: (1, {'@': 45}), 4: (1, {'@': 45})}, 104: {19: (1, {'@': 36}), 11: (1, {'@': 36}), 1: (1, {'@': 36}), 4: (1, {'@': 36})}, 105: {16: (1, {'@': 69}), 14: (1, {'@': 69}), 11: (1, {'@': 69})}, 106: {25: (1, {'@': 54}), 26: (1, {'@': 54}), 14: (1, {'@': 54}), 11: (1, {'@': 54})}, 107: {24: (0, 142), 21: (0, 33), 22: (0, 161), 16: (1, {'@': 68}), 14: (1, {'@': 68}), 11: (1, {'@': 68})}, 108: {31: (0, 144), 14: (0, 125), 27: (0, 72), 15: (0, 61)}, 109: {25: (0, 1), 39: (0, 38), 2: (0, 43), 10: (0, 86), 33: (0, 35), 34: (0, 24), 11: (0, 111), 26: (0, 19), 32: (0, 62), 14: (0, 41)}, 110: {16: (1, {'@': 70}), 14: (1, {'@': 70}), 11: (1, {'@': 70})}, 111: {11: (1, {'@': 99}), 1: (1, {'@': 99}), 4: (1, {'@': 99}), 14: (1, {'@': 99}), 15: (1, {'@': 99}), 16: (1, {'@': 99}), 25: (1, {'@': 99})}, 112: {24: (0, 46), 21: (0, 89), 22: (0, 161), 25: (1, {'@': 62}), 26: (1, {'@': 62}), 14: (1, {'@': 62}), 11: (1, {'@': 62})}, 113: {19: (1, {'@': 31}), 11: (1, {'@': 31}), 1: (1, {'@': 31}), 4: (1, {'@': 31})}, 114: {16: (0, 157), 2: (0, 10), 10: (0, 86), 14: (0, 17), 11: (0, 111), 17: (0, 136)}, 115: {19: (1, {'@': 28}), 11: (1, {'@': 28}), 1: (1, {'@': 28}), 4: (1, {'@': 28})}, 116: {14: (1, {'@': 96}), 15: (1, {'@': 96})}, 117: {31: (0, 148), 27: (0, 72), 15: (0, 61), 14: (0, 51)}, 118: {13: (0, 31), 12: (0, 42)}, 119: {19: (1, {'@': 32}), 11: (1, {'@': 32}), 1: (1, {'@': 32}), 4: (1, {'@': 32})}, 120: {24: (0, 2), 11: (0, 111), 22: (0, 161), 10: (0, 86), 2: (0, 23), 14: (1, {'@': 78}), 15: (1, {'@': 78})}, 121: {16: (0, 157), 2: (0, 10), 10: (0, 86), 14: (0, 93), 11: (0, 111), 17: (0, 136)}, 122: {29: (0, 120), 30: (0, 127), 1: (0, 118), 28: (0, 63)}, 123: {18: (0, 121), 16: (0, 157), 17: (0, 126), 2: (0, 10), 10: (0, 86), 14: (0, 113), 11: (0, 111)}, 124: {24: (0, 50), 21: (0, 131), 22: (0, 161), 16: (1, {'@': 74}), 14: (1, {'@': 74}), 11: (1, {'@': 74})}, 125: {25: (1, {'@': 55}), 26: (1, {'@': 55}), 14: (1, {'@': 55}), 11: (1, {'@': 55})}, 126: {16: (1, {'@': 91}), 14: (1, {'@': 91}), 11: (1, {'@': 91})}, 127: {25: (1, {'@': 79}), 26: (1, {'@': 79}), 14: (1, {'@': 79}), 11: (1, {'@': 79}), 15: (1, {'@': 79}), 22: (1, {'@': 79})}, 128: {31: (0, 28), 27: (0, 72), 15: (0, 61), 14: (0, 146)}, 129: {12: (0, 87), 13: (0, 31)}, 130: {3: (1, {'@': 21}), 1: (1, {'@': 21}), 4: (1, {'@': 21}), 11: (1, {'@': 21})}, 131: {31: (0, 58), 27: (0, 72), 15: (0, 61), 14: (0, 69)}, 132: {19: (1, {'@': 38}), 11: (1, {'@': 38}), 1: (1, {'@': 38}), 4: (1, {'@': 38})}, 133: {11: (1, {'@': 100}), 1: (1, {'@': 100}), 4: (1, {'@': 100}), 14: (1, {'@': 100}), 15: (1, {'@': 100}), 16: (1, {'@': 100}), 25: (1, {'@': 100})}, 134: {37: (0, 158), 36: (0, 5)}, 135: {12: (0, 107), 13: (0, 31)}, 136: {16: (1, {'@': 92}), 14: (1, {'@': 92}), 11: (1, {'@': 92})}, 137: {14: (0, 13), 25: (0, 1), 2: (0, 43), 10: (0, 86), 32: (0, 84), 33: (0, 35), 34: (0, 24), 11: (0, 111), 26: (0, 19)}, 138: {31: (0, 20), 27: (0, 72), 15: (0, 61), 14: (0, 45)}, 139: {19: (1, {'@': 22}), 11: (1, {'@': 22}), 1: (1, {'@': 22}), 4: (1, {'@': 22})}, 140: {16: (1, {'@': 66}), 14: (1, {'@': 66}), 11: (1, {'@': 66})}, 141: {1: (0, 129), 2: (0, 59), 9: (0, 78), 10: (0, 86), 4: (0, 52), 7: (0, 143), 11: (0, 111), 19: (1, {'@': 19})}, 142: {21: (0, 117), 16: (1, {'@': 65}), 14: (1, {'@': 65}), 11: (1, {'@': 65})}, 143: {19: (1, {'@': 89}), 11: (1, {'@': 89}), 1: (1, {'@': 89}), 4: (1, {'@': 89})}, 144: {15: (0, 61), 14: (0, 106), 27: (0, 116)}, 145: {14: (0, 47), 16: (0, 157), 2: (0, 10), 10: (0, 86), 11: (0, 111), 17: (0, 136)}, 146: {25: (1, {'@': 52}), 26: (1, {'@': 52}), 14: (1, {'@': 52}), 11: (1, {'@': 52})}, 147: {36: (0, 166), 37: (0, 158)}, 148: {15: (0, 61), 27: (0, 116), 14: (0, 4)}, 149: {20: (1, {'@': 83}), 21: (1, {'@': 83}), 25: (1, {'@': 83}), 26: (1, {'@': 83}), 14: (1, {'@': 83}), 11: (1, {'@': 83}), 16: (1, {'@': 83}), 15: (1, {'@': 83})}, 150: {21: (0, 152)}, 151: {37: (0, 158), 36: (0, 109)}, 152: {37: (0, 158), 36: (0, 9)}, 153: {21: (0, 40)}, 154: {25: (0, 1), 14: (0, 68), 39: (0, 74), 2: (0, 43), 10: (0, 86), 33: (0, 35), 34: (0, 24), 11: (0, 111), 26: (0, 19), 32: (0, 62)}, 155: {37: (0, 158), 36: (0, 167)}, 156: {38: (0, 149), 13: (0, 88)}, 157: {12: (0, 124), 13: (0, 31)}, 158: {40: (0, 165)}, 159: {25: (0, 1), 39: (0, 65), 2: (0, 43), 10: (0, 86), 14: (0, 94), 33: (0, 35), 34: (0, 24), 11: (0, 111), 26: (0, 19), 32: (0, 62)}, 160: {21: (0, 101)}, 161: {13: (0, 163), 41: (0, 156)}, 162: {24: (0, 32), 21: (0, 73), 22: (0, 161)}, 163: {13: (1, {'@': 97}), 38: (1, {'@': 97})}, 164: {21: (0, 168), 20: (0, 26), 23: (0, 160)}, 165: {25: (1, {'@': 49}), 26: (1, {'@': 49}), 14: (1, {'@': 49}), 11: (1, {'@': 49}), 16: (1, {'@': 49})}, 166: {25: (0, 1), 39: (0, 137), 2: (0, 43), 10: (0, 86), 33: (0, 35), 34: (0, 24), 11: (0, 111), 26: (0, 19), 14: (0, 103), 32: (0, 62)}, 167: {16: (0, 157), 17: (0, 126), 2: (0, 10), 10: (0, 86), 18: (0, 56), 14: (0, 16), 11: (0, 111)}, 168: {37: (0, 158), 36: (0, 102)}}, 'start_states': {'start': 0}, 'end_states': {'start': 25}}, 'options': {'debug': False, 'keep_all_tokens': False, 'tree_class': None, 'cache': False, 'postlex': None, 'parser': 'lalr', 'lexer': 'contextual', 'transformer': None, 'start': ['start'], 'priority': 'normal', 'ambiguity': 'auto', 'regex': False, 'propagate_positions': False, 'lexer_callbacks': {}, 'maybe_placeholders': False, 'edit_terminals': None, 'g_regex_flags': 0, 'use_bytes': False, 'import_paths': [], 'source_path': None}, '__type__': 'ParsingFrontend'}, 'rules': [{'@': 19}, {'@': 20}, {'@': 21}, {'@': 22}, {'@': 23}, {'@': 24}, {'@': 25}, {'@': 26}, {'@': 27}, {'@': 28}, {'@': 29}, {'@': 30}, {'@': 31}, {'@': 32}, {'@': 33}, {'@': 34}, {'@': 35}, {'@': 36}, {'@': 37}, {'@': 38}, {'@': 39}, {'@': 40}, {'@': 41}, {'@': 42}, {'@': 43}, {'@': 44}, {'@': 45}, {'@': 46}, {'@': 47}, {'@': 48}, {'@': 49}, {'@': 50}, {'@': 51}, {'@': 52}, {'@': 53}, {'@': 54}, {'@': 55}, {'@': 56}, {'@': 57}, {'@': 58}, {'@': 59}, {'@': 60}, {'@': 61}, {'@': 62}, {'@': 63}, {'@': 64}, {'@': 65}, {'@': 66}, {'@': 67}, {'@': 68}, {'@': 69}, {'@': 70}, {'@': 71}, {'@': 72}, {'@': 73}, {'@': 74}, {'@': 75}, {'@': 76}, {'@': 77}, {'@': 78}, {'@': 79}, {'@': 80}, {'@': 81}, {'@': 82}, {'@': 83}, {'@': 84}, {'@': 85}, {'@': 86}, {'@': 87}, {'@': 88}, {'@': 89}, {'@': 90}, {'@': 91}, {'@': 92}, {'@': 93}, {'@': 94}, {'@': 95}, {'@': 96}, {'@': 97}, {'@': 98}, {'@': 99}, {'@': 100}], 'options': {'debug': False, 'keep_all_tokens': False, 'tree_class': None, 'cache': False, 'postlex': None, 'parser': 'lalr', 'lexer': 'contextual', 'transformer': None, 'start': ['start'], 'priority': 'normal', 'ambiguity': 'auto', 'regex': False, 'propagate_positions': False, 'lexer_callbacks': {}, 'maybe_placeholders': False, 'edit_terminals': None, 'g_regex_flags': 0, 'use_bytes': False, 'import_paths': [], 'source_path': None}, '__type__': 'Lark'} +) +MEMO = ( +{0: {'name': 'IDENTIFIER', 'pattern': {'value': '(?:_|(?:[A-Z]|[a-z]))(?:(?:(?:_|(?:[A-Z]|[a-z]))|[0-9]))*', 'flags': [], '_width': [1, 4294967295], '__type__': 'PatternRE'}, 'priority': 1, '__type__': 'TerminalDef'}, 1: {'name': 'WS', 'pattern': {'value': '(?:[ \t\x0c\r\n])+', 'flags': [], '_width': [1, 4294967295], '__type__': 'PatternRE'}, 'priority': 1, '__type__': 'TerminalDef'}, 2: {'name': 'PRIMITIVE', 'pattern': {'value': '(?:(?:(?:(?:(?:u?int(8|16|32|64)?|size)|string)|funcptr)|buffer)|address)', 'flags': [], '_width': [3, 7], '__type__': 'PatternRE'}, 'priority': 1, '__type__': 'TerminalDef'}, 3: {'name': 'UID', 'pattern': {'value': '[0-9a-fA-F]{16}', 'flags': [], '_width': [16, 16], '__type__': 'PatternRE'}, 'priority': 1, '__type__': 'TerminalDef'}, 4: {'name': 'COMMENT', 'pattern': {'value': '#.*', 'flags': [], '_width': [1, 4294967295], '__type__': 'PatternRE'}, 'priority': 1, '__type__': 'TerminalDef'}, 5: {'name': 'PATH', 'pattern': {'value': '"[^"]*"', 'flags': [], '_width': [2, 4294967295], '__type__': 'PatternRE'}, 'priority': 1, '__type__': 'TerminalDef'}, 6: {'name': 'IMPORT', 'pattern': {'value': 'import', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 7: {'name': 'OBJECT', 'pattern': {'value': 'object', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 8: {'name': 'LBRACE', 'pattern': {'value': '{', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 9: {'name': 'RBRACE', 'pattern': {'value': '}', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 10: {'name': 'INTERFACE', 'pattern': {'value': 'interface', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 11: {'name': 'EXPOSE', 'pattern': {'value': 'expose', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 12: {'name': '__ANON_0', 'pattern': {'value': 'uid', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 13: {'name': 'COLON', 'pattern': {'value': ':', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 14: {'name': 'FUNCTION', 'pattern': {'value': 'function', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 15: {'name': 'METHOD', 'pattern': {'value': 'method', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 16: {'name': 'PARAM', 'pattern': {'value': 'param', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 17: {'name': 'LSQB', 'pattern': {'value': '[', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 18: {'name': 'RSQB', 'pattern': {'value': ']', 'flags': [], '__type__': 'PatternStr'}, 'priority': 1, '__type__': 'TerminalDef'}, 19: {'origin': {'name': 'start', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__start_star_0', '__type__': 'NonTerminal'}, {'name': '__start_plus_1', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 20: {'origin': {'name': 'start', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__start_plus_1', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 21: {'origin': {'name': 'import_statement', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'IMPORT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'PATH', 'filter_out': False, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 22: {'origin': {'name': 'object', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'OBJECT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'super', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': '__object_star_2', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 23: {'origin': {'name': 'object', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'OBJECT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'super', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 24: {'origin': {'name': 'object', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'OBJECT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': '__object_star_2', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 2, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 25: {'origin': {'name': 'object', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'OBJECT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 3, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 26: {'origin': {'name': 'object', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'OBJECT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'super', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': '__object_star_2', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 4, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 27: {'origin': {'name': 'object', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'OBJECT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'super', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 5, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 28: {'origin': {'name': 'object', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'OBJECT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': '__object_star_2', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 6, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 29: {'origin': {'name': 'object', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'OBJECT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 7, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 30: {'origin': {'name': 'object', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'OBJECT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'super', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': '__object_star_2', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 8, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 31: {'origin': {'name': 'object', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'OBJECT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'super', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 9, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 32: {'origin': {'name': 'object', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'OBJECT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': '__object_star_2', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 10, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 33: {'origin': {'name': 'object', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'OBJECT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 11, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 34: {'origin': {'name': 'object', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'OBJECT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'super', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': '__object_star_2', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 12, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 35: {'origin': {'name': 'object', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'OBJECT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'super', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 13, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 36: {'origin': {'name': 'object', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'OBJECT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': '__object_star_2', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 14, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 37: {'origin': {'name': 'object', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'OBJECT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 15, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 38: {'origin': {'name': 'interface', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'INTERFACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': '__interface_star_3', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 39: {'origin': {'name': 'interface', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'INTERFACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 40: {'origin': {'name': 'interface', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'INTERFACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': '__interface_star_3', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 2, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 41: {'origin': {'name': 'interface', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'INTERFACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 3, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 42: {'origin': {'name': 'interface', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'INTERFACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': '__interface_star_3', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 4, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 43: {'origin': {'name': 'interface', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'INTERFACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 5, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 44: {'origin': {'name': 'interface', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'INTERFACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': '__interface_star_3', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 6, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 45: {'origin': {'name': 'interface', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'INTERFACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'uid', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 7, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 46: {'origin': {'name': 'interface_param', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'expose', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': True, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 47: {'origin': {'name': 'interface_param', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'function', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': True, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 48: {'origin': {'name': 'expose', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'EXPOSE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'type', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 49: {'origin': {'name': 'uid', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__ANON_0', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'UID', 'filter_out': False, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 50: {'origin': {'name': 'super', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'COLON', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 51: {'origin': {'name': 'function', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'FUNCTION', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': '__function_star_4', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 52: {'origin': {'name': 'function', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'FUNCTION', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 53: {'origin': {'name': 'function', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'FUNCTION', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}], 'order': 2, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 54: {'origin': {'name': 'function', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'FUNCTION', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': '__function_star_4', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 3, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 55: {'origin': {'name': 'function', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'FUNCTION', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 4, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 56: {'origin': {'name': 'function', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'FUNCTION', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}], 'order': 5, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 57: {'origin': {'name': 'function', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'FUNCTION', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': '__function_star_4', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 6, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 58: {'origin': {'name': 'function', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'FUNCTION', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 7, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 59: {'origin': {'name': 'function', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'FUNCTION', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}], 'order': 8, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 60: {'origin': {'name': 'function', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'FUNCTION', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': '__function_star_4', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 9, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 61: {'origin': {'name': 'function', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'FUNCTION', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 10, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 62: {'origin': {'name': 'function', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'FUNCTION', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}], 'order': 11, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 63: {'origin': {'name': 'method', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'METHOD', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': '__function_star_4', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 64: {'origin': {'name': 'method', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'METHOD', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 65: {'origin': {'name': 'method', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'METHOD', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}], 'order': 2, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 66: {'origin': {'name': 'method', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'METHOD', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': '__function_star_4', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 3, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 67: {'origin': {'name': 'method', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'METHOD', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 4, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 68: {'origin': {'name': 'method', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'description', '__type__': 'NonTerminal'}, {'name': 'METHOD', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}], 'order': 5, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 69: {'origin': {'name': 'method', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'METHOD', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': '__function_star_4', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 6, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 70: {'origin': {'name': 'method', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'METHOD', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 7, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 71: {'origin': {'name': 'method', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'METHOD', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}], 'order': 8, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 72: {'origin': {'name': 'method', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'METHOD', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': '__function_star_4', '__type__': 'NonTerminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 9, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 73: {'origin': {'name': 'method', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'METHOD', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'LBRACE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'RBRACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 10, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 74: {'origin': {'name': 'method', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'METHOD', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}], 'order': 11, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 75: {'origin': {'name': 'param', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'PARAM', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'type', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}, {'name': 'description', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 76: {'origin': {'name': 'param', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'PARAM', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'type', '__type__': 'NonTerminal'}, {'name': 'options', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 77: {'origin': {'name': 'param', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'PARAM', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'type', '__type__': 'NonTerminal'}, {'name': 'description', '__type__': 'NonTerminal'}], 'order': 2, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 78: {'origin': {'name': 'param', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'PARAM', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}, {'name': 'type', '__type__': 'NonTerminal'}], 'order': 3, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 79: {'origin': {'name': 'type', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'PRIMITIVE', 'filter_out': False, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': True, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 80: {'origin': {'name': 'type', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'object_name', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': True, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 81: {'origin': {'name': 'object_name', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'OBJECT', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'name', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 82: {'origin': {'name': 'name', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'IDENTIFIER', 'filter_out': False, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 83: {'origin': {'name': 'options', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'LSQB', 'filter_out': True, '__type__': 'Terminal'}, {'name': '__options_plus_5', '__type__': 'NonTerminal'}, {'name': 'RSQB', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 84: {'origin': {'name': 'description', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__description_plus_6', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 85: {'origin': {'name': '__start_star_0', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'import_statement', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 86: {'origin': {'name': '__start_star_0', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__start_star_0', '__type__': 'NonTerminal'}, {'name': 'import_statement', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 87: {'origin': {'name': '__start_plus_1', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'object', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 88: {'origin': {'name': '__start_plus_1', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'interface', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 89: {'origin': {'name': '__start_plus_1', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__start_plus_1', '__type__': 'NonTerminal'}, {'name': 'object', '__type__': 'NonTerminal'}], 'order': 2, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 90: {'origin': {'name': '__start_plus_1', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__start_plus_1', '__type__': 'NonTerminal'}, {'name': 'interface', '__type__': 'NonTerminal'}], 'order': 3, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 91: {'origin': {'name': '__object_star_2', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'method', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 92: {'origin': {'name': '__object_star_2', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__object_star_2', '__type__': 'NonTerminal'}, {'name': 'method', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 93: {'origin': {'name': '__interface_star_3', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'interface_param', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 94: {'origin': {'name': '__interface_star_3', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__interface_star_3', '__type__': 'NonTerminal'}, {'name': 'interface_param', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 95: {'origin': {'name': '__function_star_4', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'param', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 96: {'origin': {'name': '__function_star_4', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__function_star_4', '__type__': 'NonTerminal'}, {'name': 'param', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 97: {'origin': {'name': '__options_plus_5', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'IDENTIFIER', 'filter_out': False, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 98: {'origin': {'name': '__options_plus_5', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__options_plus_5', '__type__': 'NonTerminal'}, {'name': 'IDENTIFIER', 'filter_out': False, '__type__': 'Terminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 99: {'origin': {'name': '__description_plus_6', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'COMMENT', 'filter_out': False, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 100: {'origin': {'name': '__description_plus_6', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__description_plus_6', '__type__': 'NonTerminal'}, {'name': 'COMMENT', 'filter_out': False, '__type__': 'Terminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}} +) +Shift = 0 +Reduce = 1 +def Lark_StandAlone(**kwargs): + return Lark._load_from_dict(DATA, MEMO, **kwargs) diff --git a/scripts/definitions/templates/syscall/client/private/syscall_trampolines.s b/scripts/definitions/templates/syscall/client/private/syscall_trampolines.s new file mode 100644 index 0000000..ab56323 --- /dev/null +++ b/scripts/definitions/templates/syscall/client/private/syscall_trampolines.s @@ -0,0 +1,23 @@ +%macro SYSCALL 2 + global __syscall_%1 + __syscall_%1: + push rbp + mov rbp, rsp + + ; args should already be in rdi, etc, but rcx will + ; get stomped, so stash it in r10, which isn't a + ; callee-saved register, but also isn't used in the + ; function call ABI. + mov r10, rcx + + mov rax, %2 + syscall + ; result is now already in rax, so just return + + pop rbp + ret +%endmacro + +{% for id, scope, method in interface.methods %} +SYSCALL {% if scope %}{{ scope.name }}_{% endif %}{{ method.name }} {{ id }} +{% endfor %} diff --git a/scripts/definitions/templates/syscall/client/public/_object_.hh b/scripts/definitions/templates/syscall/client/public/_object_.hh new file mode 100644 index 0000000..bb3f5b8 --- /dev/null +++ b/scripts/definitions/templates/syscall/client/public/_object_.hh @@ -0,0 +1,41 @@ +#pragma once +/// \file {{filename}} +{% if object.super %} + +#include +{% endif %} + +namespace j6 { + +{% if object.desc %} +{{ object.desc|indent('/// ', first=True) }} +{% endif %} +class {{ object.name }} +{% if object.super %} : public {{ object.super.name }} +{% endif %} +{ +public: +{% macro argument(type, name, first, options=False) -%} + {%- for ctype, suffix in type.c_names(options) -%} + {%- if not first or not loop.first %}, {% endif %}{{ ctype }} {{ name }}{{ suffix }} + {%- endfor -%} +{%- endmacro -%} + +{% for method in object.methods %} +{% if method.desc %} /// {{ method.desc|indent(' /// ') }}{% endif %} +{% for param in method.params %} +{% if param.desc %} /// \arg {{ "%-10s"|format(param.name) }} {{ param.desc }} +{% endif %} +{% endfor %} + {%+ if method.static %}static {% endif %}j6_status_t {{ method.name }} ( +{%- for param in method.params %}{{ argument(param.type, param.name, loop.first, options=param.options) }}{% endfor -%}); + +{% endfor %} + ~{{ object.name }}(); + +private: + {{ object.name }}(j6_handle_t handle) : m_handle {handle} {} + j6_handle_t m_handle; +} + +} // namespace j6 diff --git a/scripts/definitions/templates/syscall/client/public/syscalls.h b/scripts/definitions/templates/syscall/client/public/syscalls.h new file mode 100644 index 0000000..7efb3e0 --- /dev/null +++ b/scripts/definitions/templates/syscall/client/public/syscalls.h @@ -0,0 +1,25 @@ +#pragma once + +#include + +#ifdef __cplusplus +extern "C" { +#endif +{% macro argument(type, name, first, options=False) -%} + {%- for ctype, suffix in type.c_names(options) -%} + {%- if not first or not loop.first %}, {% endif %}{{ ctype }} {{ name }}{{ suffix }} + {%- endfor -%} +{%- endmacro %} + +{% for id, scope, method in interface.methods %} +j6_status_t __syscall_{% if scope %}{{ scope.name }}_{% endif %}{{ method.name }} ( + {%- if not method.static -%}{{ argument(scope.reftype, "self", True) }}{% endif -%} + {%- set first = method.static -%} + {%- for param in method.params %}{{ argument(param.type, param.name, first, options=param.options) }}{% set first = False %}{% endfor -%} +); +{% endfor %} + +#ifdef __cplusplus +} +#endif + diff --git a/scripts/definitions/transformer.py b/scripts/definitions/transformer.py new file mode 100644 index 0000000..d181682 --- /dev/null +++ b/scripts/definitions/transformer.py @@ -0,0 +1,142 @@ +from .parser import Transformer, v_args + +def get_opts(args): + from .types import Description, Options, Type, UID + + kinds = { + Description: "desc", + Options: "opts", + UID: "uid", + Type: "typename", + } + + result = dict() + outargs = [] + for a in args: + for kind, name in kinds.items(): + if isinstance(a, kind): + result[name] = a + break + else: + outargs.append(a) + + return result, outargs + +class DefTransformer(Transformer): + def __init__(self, filename): + self.filename = filename + + def start(self, args): + from .types import Import, Interface, Object + + imports = set() + objects = dict() + interfaces = dict() + + for o in args: + if isinstance(o, Object): + objects[o.name] = o + + elif isinstance(o, Interface): + interfaces[o.name] = o + + elif isinstance(o, Import): + imports.add(o) + + return imports, objects, interfaces + + @v_args(inline=True) + def import_statement(self, path): + from .types import Import + return Import(path) + + def object(self, args): + from .types import Object + specials, args = get_opts(args) + name, args = args[0], args[1:] + return Object(name, children=args, **specials) + + def interface(self, args): + from .types import Interface + specials, args = get_opts(args) + name, args = args[0], args[1:] + return Interface(name, children=args, **specials) + + def method(self, args): + from .types import Method + specials, args = get_opts(args) + name, args = args[0], args[1:] + return Method(name, children=args, **specials) + + def function(self, args): + from .types import Function + specials, args = get_opts(args) + name, args = args[0], args[1:] + return Function(name, children=args, **specials) + + def param(self, args): + from .types import Param + specials, args = get_opts(args) + name = args[0] + return Param(name, **specials) + + @v_args(inline=True) + def expose(self, s): + from .types import Expose + return Expose(s) + + @v_args(inline=True) + def uid(self, s): + return s + + @v_args(inline=True) + def name(self, s): + return s + + @v_args(inline=True) + def type(self, s): + return s + + @v_args(inline=True) + def super(self, s): + from .types import ObjectRef + return ObjectRef(s, self.filename) + + def options(self, args): + from .types import Options + return Options([str(s) for s in args]) + + def description(self, s): + from .types import Description + return Description("\n".join(s)) + + @v_args(inline=True) + def object_name(self, n): + from .types import ObjectRef + return ObjectRef(n, self.filename) + + def PRIMITIVE(self, s): + from .types import get_primitive + return get_primitive(s) + + def UID(self, s): + from .types import UID + return UID(int(s, base=16)) + + def INT_TYPE(self, s): + return s + + def NUMBER(self, s): + if s.startswith("0x"): + return int(s,16) + return int(s) + + def COMMENT(self, s): + return s[2:].strip() + + def IDENTIFIER(self, s): + return str(s) + + def PATH(self, s): + return str(s[1:-1]) + diff --git a/scripts/definitions/types/__init__.py b/scripts/definitions/types/__init__.py new file mode 100644 index 0000000..7aa2b04 --- /dev/null +++ b/scripts/definitions/types/__init__.py @@ -0,0 +1,23 @@ +def _indent(x): + from textwrap import indent + return indent(str(x), ' ') + +class Description(str): pass +class Import(str): pass + +class Options(set): + def __str__(self): + if not self: return "" + return "[{}]".format(" ".join(self)) + +class UID(int): + def __str__(self): + return f"{self:016x}" + +from .object import Object +from .interface import Interface, Expose +from .function import Function, Method, Param + +from .type import Type +from .primitive import get_primitive +from .objref import ObjectRef diff --git a/scripts/definitions/types/function.py b/scripts/definitions/types/function.py new file mode 100644 index 0000000..3119cc8 --- /dev/null +++ b/scripts/definitions/types/function.py @@ -0,0 +1,49 @@ +from . import _indent +from . import Options + +def _hasopt(opt): + def test(self): + return opt in self.options + return test + +class Function: + typename = "function" + + def __init__(self, name, opts=Options(), desc="", children=tuple()): + self.name = name + self.options = opts + self.desc = desc + self.params = [c for c in children if isinstance(c, Param)] + self.id = -1 + + def __str__(self): + parts = ["{} {}".format(self.typename, self.name)] + if self.desc: + parts.append(_indent(self.desc)) + if self.options: + parts.append(f" Options: {self.options}") + parts.extend(map(_indent, self.params)) + return "\n".join(parts) + + static = property(lambda x: True) + constructor = property(lambda x: False) + + +class Method(Function): + typename = "method" + + static = property(_hasopt("static")) + constructor = property(_hasopt("constructor")) + + +class Param: + def __init__(self, name, typename, opts=Options(), desc=""): + self.name = name + self.type = typename + self.options = opts + self.desc = desc + + def __str__(self): + return "param {} {} {} {}".format( + self.name, repr(self.type), self.options, self.desc or "") + diff --git a/scripts/definitions/types/interface.py b/scripts/definitions/types/interface.py new file mode 100644 index 0000000..db95975 --- /dev/null +++ b/scripts/definitions/types/interface.py @@ -0,0 +1,43 @@ +from . import _indent +from . import Options + +class Expose(object): + def __init__(self, type): + self.type = type + + def __repr__(self): + return f'expose {repr(self.type)}' + +class Interface: + def __init__(self, name, uid, opts=Options(), desc="", children=tuple()): + from .function import Function + + self.name = name + self.uid = uid + self.options = opts + self.desc = desc + + self.functions = [c for c in children if isinstance(c, Function)] + self.exposes = [e.type for e in children if isinstance(e, Expose)] + + def __str__(self): + parts = [f"interface {self.name}: {self.uid}"] + if self.desc: + parts.append(_indent(self.desc)) + if self.options: + parts.append(f" Options: {self.options}") + parts.extend(map(_indent, self.exposes)) + parts.extend(map(_indent, self.functions)) + return "\n".join(parts) + + def __methods(self): + mm = [(i, None, self.functions[i]) for i in range(len(self.functions))] + + base = len(mm) + for o in [e.object for e in self.exposes]: + mm.extend([(base + i, o, o.methods[i]) for i in range(len(o.methods))]) + base += len(o.methods) + + return mm + + methods = property(__methods) diff --git a/scripts/definitions/types/object.py b/scripts/definitions/types/object.py new file mode 100644 index 0000000..7f1dd45 --- /dev/null +++ b/scripts/definitions/types/object.py @@ -0,0 +1,25 @@ +from . import _indent +from . import Options + +class Object: + def __init__(self, name, uid, typename=None, opts=Options(), desc="", children=tuple()): + self.name = name + self.uid = uid + self.options = opts + self.desc = desc + self.super = typename + self.methods = children + + from . import ObjectRef + self.__ref = ObjectRef(name) + + def __str__(self): + parts = [f"object {self.name}: {self.uid}"] + if self.desc: + parts.append(_indent(self.desc)) + if self.options: + parts.append(f" Options: {self.options}") + parts.extend(map(_indent, self.methods)) + return "\n".join(parts) + + reftype = property(lambda self: self.__ref) diff --git a/scripts/definitions/types/objref.py b/scripts/definitions/types/objref.py new file mode 100644 index 0000000..c12573e --- /dev/null +++ b/scripts/definitions/types/objref.py @@ -0,0 +1,44 @@ +from .type import Type + +class ObjectRef(Type): + all_refs = {} + + def __init__(self, name, filename=None): + super().__init__(name) + self.__c_type = "j6_handle_t" + self.__object = None + ObjectRef.all_refs[self] = filename + + def __repr__(self): + return f'ObjectRef({self.name})' + + object = property(lambda self: self.__object) + + def c_names(self, options): + one = self.__c_type + out = bool({"out", "inout"}.intersection(options)) + + if "list" in options: + two = "size_t" + if out: + one = f"const {one} *" + two += " *" + else: + one = f"{one} *" + return ((one, ""), (two, "_count")) + + else: + if out: + one += " *" + return ((one, ""),) + + def cxx_names(self, options): + return self.c_names(options) + + @classmethod + def connect(cls, objects): + for ref, filename in cls.all_refs.items(): + ref.__object = objects.get(ref.name) + if ref.__object is None: + from ..errors import UnknownTypeError + raise UnknownTypeError(ref.name, filename) diff --git a/scripts/definitions/types/primitive.py b/scripts/definitions/types/primitive.py new file mode 100644 index 0000000..03350bf --- /dev/null +++ b/scripts/definitions/types/primitive.py @@ -0,0 +1,70 @@ +from .type import Type + +class Primitive(Type): + def __init__(self, name, c_type): + super().__init__(name) + self.c_type = c_type + + def __repr__(self): + return f'Primitive({self.name})' + + def c_names(self, options=None): + one = self.c_type + if options and "out" in options or "inout" in options: + one += " *" + + return ((one, ""),) + + def cxx_names(self, options): + return self.c_names(options) + +class PrimitiveRef(Primitive): + def __init__(self, name, c_type, counted=False): + super().__init__(name, c_type) + self.__counted = counted + + def c_names(self, options=None): + one = f"{self.c_type} *" + two = "size_t" + + if options and "out" in options or "inout" in options: + two += " *" + else: + one = "const " + one + + if self.__counted: + return ((one, ""), (two, "_len")) + else: + return ((one, ""),) + + def cxx_names(self, options): + return self.c_names(options) + +_primitives = { + "string": PrimitiveRef("string", "char"), + "buffer": PrimitiveRef("buffer", "void", counted=True), + + "int": Primitive("int", "int"), + "uint": Primitive("uint", "unsigned"), + "size": Primitive("size", "size_t"), + "address": Primitive("address", "uintptr_t"), + + "int8": Primitive("int8", "int8_t"), + "uint8": Primitive("uint8", "uint8_t"), + + "int16": Primitive("int16", "int16_t"), + "uint16": Primitive("uint16", "uint16_t"), + + "int32": Primitive("int32", "int32_t"), + "uint32": Primitive("uint32", "uint32_t"), + + "int64": Primitive("int64", "int64_t"), + "uint64": Primitive("uint64", "uint64_t"), +} + +def get_primitive(name): + p = _primitives.get(name) + if not p: + from ..errors import InvalidType + raise InvalidType(name) + return p diff --git a/scripts/definitions/types/type.py b/scripts/definitions/types/type.py new file mode 100644 index 0000000..565f871 --- /dev/null +++ b/scripts/definitions/types/type.py @@ -0,0 +1,12 @@ +class Type: + def __init__(self, name): + self.__name = name + + name = property(lambda self: self.__name) + + def c_names(self, options): + raise NotImplemented("Call to base Type.c_names") + + def cxx_names(self, options): + raise NotImplemented("Call to base Type.c_names") + diff --git a/scripts/idgen b/scripts/idgen new file mode 100755 index 0000000..564dd9d --- /dev/null +++ b/scripts/idgen @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 + +def hashid(s): + from hashlib import shake_128 as sh + return sh(s.encode('utf-8')).hexdigest(8) + +import sys + +for arg in sys.argv[1:]: + id = hashid(arg) + print(f"{arg}: {id}") diff --git a/src/kernel/kernel.module b/src/kernel/kernel.module index a0001e6..e1b424f 100644 --- a/src/kernel/kernel.module +++ b/src/kernel/kernel.module @@ -14,7 +14,6 @@ kernel = module("kernel", "clock.cpp", "console.cpp", "cpprt.cpp", - "cpu.cpp", "debug.cpp", "debug.s", "device_manager.cpp", @@ -27,7 +26,6 @@ kernel = module("kernel", "interrupts.s", "io.cpp", "log.cpp", - "main.cpp", "memory_bootstrap.cpp", "msr.cpp", "objects/channel.cpp", @@ -42,8 +40,6 @@ kernel = module("kernel", "pci.cpp", "scheduler.cpp", "serial.cpp", - "syscall.cpp", - "syscall.s", "syscalls/channel.cpp", "syscalls/endpoint.cpp", "syscalls/object.cpp", @@ -56,4 +52,15 @@ kernel = module("kernel", "vm_space.cpp", ]) +from glob import glob +definitions = glob('definitions/**/*.def', recursive=True) +sysinc = kernel.add_input("syscalls.inc.cog", deps=definitions) +kernel.add_input("syscall.s", deps=[sysinc]) + +sysh = kernel.add_input("syscall.h.cog", deps=definitions) +sysc = kernel.add_input("syscall.cpp.cog", deps=definitions + [sysh]) + +kernel.add_input("main.cpp", deps=[sysh]) +kernel.add_input("cpu.cpp", deps=[sysh]) + kernel.variables['ldflags'] = ["${ldflags}", "-T", "${source_root}/src/kernel/kernel.ld"] diff --git a/src/kernel/objects/endpoint.cpp b/src/kernel/objects/endpoint.cpp index 6b6a9ef..21a4e47 100644 --- a/src/kernel/objects/endpoint.cpp +++ b/src/kernel/objects/endpoint.cpp @@ -28,10 +28,10 @@ endpoint::close() } j6_status_t -endpoint::send(j6_tag_t tag, size_t len, void *data) +endpoint::send(j6_tag_t tag, const void *data, size_t data_len) { thread_data sender = { &thread::current(), data }; - sender.len = len; + sender.len = data_len; sender.tag = tag; if (!check_signal(j6_signal_endpoint_can_send)) { @@ -55,11 +55,11 @@ endpoint::send(j6_tag_t tag, size_t len, void *data) } j6_status_t -endpoint::receive(j6_tag_t *tag, size_t *len, void *data) +endpoint::receive(j6_tag_t *tag, void *data, size_t *data_len) { thread_data receiver = { &thread::current(), data }; receiver.tag_p = tag; - receiver.len_p = len; + receiver.len_p = data_len; if (!check_signal(j6_signal_endpoint_can_recv)) { assert_signal(j6_signal_endpoint_can_send); @@ -120,7 +120,7 @@ endpoint::do_message_copy(const endpoint::thread_data &sender, endpoint::thread_ if (sender.len) { vm_space &source = sender.th->parent().space(); vm_space &dest = receiver.th->parent().space(); - vm_space::copy(source, dest, sender.data, receiver.data, sender.len); + vm_space::copy(source, dest, sender.data, receiver.buffer, sender.len); } *receiver.len_p = sender.len; diff --git a/src/kernel/objects/endpoint.h b/src/kernel/objects/endpoint.h index 993387a..16d8183 100644 --- a/src/kernel/objects/endpoint.h +++ b/src/kernel/objects/endpoint.h @@ -27,10 +27,10 @@ public: /// Send a message to a thread waiting to receive on this endpoint. If no threads /// are currently trying to receive, block the current thread. /// \arg tag The application-specified message tag - /// \arg len The size in bytes of the message /// \arg data The message data + /// \arg len The size in bytes of the message /// \returns j6_status_ok on success - j6_status_t send(j6_tag_t tag, size_t len, void *data); + j6_status_t send(j6_tag_t tag, const void *data, size_t data_len); /// Receive a message from a thread waiting to send on this endpoint. If no threads /// are currently trying to send, block the current thread. @@ -38,7 +38,7 @@ public: /// \arg len [in] The size in bytes of the buffer [out] Number of bytes in the message /// \arg data Buffer for copying message data into /// \returns j6_status_ok on success - j6_status_t receive(j6_tag_t *tag, size_t *len, void *data); + j6_status_t receive(j6_tag_t *tag, void *data, size_t *data_len); /// Give the listener on the endpoint a message that a bound IRQ has been signalled /// \arg irq The IRQ that caused this signal @@ -48,7 +48,10 @@ private: struct thread_data { thread *th; - void *data; + union { + const void *data; + void *buffer; + }; union { j6_tag_t *tag_p; j6_tag_t tag; diff --git a/src/kernel/syscall.cpp b/src/kernel/syscall.cpp deleted file mode 100644 index ce89e4f..0000000 --- a/src/kernel/syscall.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#include - -#include "kutil/memory.h" - -#include "console.h" -#include "debug.h" -#include "log.h" -#include "syscall.h" - -extern "C" { - void syscall_invalid(uint64_t call); -} - -uintptr_t syscall_registry[256] __attribute__((section(".syscall_registry"))); -const char * syscall_names[256] __attribute__((section(".syscall_registry"))); -static constexpr size_t num_syscalls = sizeof(syscall_registry) / sizeof(syscall_registry[0]); - -void -syscall_invalid(uint64_t call) -{ - console *cons = console::get(); - cons->set_color(9); - cons->printf("\nReceived unknown syscall: %02x\n", call); - - cons->printf(" Known syscalls:\n"); - cons->printf(" invalid %016lx\n", syscall_invalid); - - for (unsigned i = 0; i < num_syscalls; ++i) { - const char *name = syscall_names[i]; - uintptr_t handler = syscall_registry[i]; - if (name) - cons->printf(" %02x %10s %016lx\n", i, name, handler); - } - - cons->set_color(); - _halt(); -} - -void -syscall_initialize() -{ - kutil::memset(&syscall_registry, 0, sizeof(syscall_registry)); - kutil::memset(&syscall_names, 0, sizeof(syscall_names)); - -#define SYSCALL(id, name, result, ...) \ - syscall_registry[id] = reinterpret_cast(syscalls::name); \ - syscall_names[id] = #name; \ - log::debug(logs::syscall, "Enabling syscall 0x%02x as " #name , id); -#include "j6/tables/syscalls.inc" -#undef SYSCALL -} - diff --git a/src/kernel/syscall.cpp.cog b/src/kernel/syscall.cpp.cog new file mode 100644 index 0000000..74e0807 --- /dev/null +++ b/src/kernel/syscall.cpp.cog @@ -0,0 +1,56 @@ +// vim: ft=cpp +#include + +#include "kutil/memory.h" + +#include "console.h" +#include "debug.h" +#include "log.h" +#include "syscall.h" + +extern "C" { + void syscall_invalid(uint64_t call); +} + +/*[[[cog code generation +from definitions.context import Context + +ctx = Context(definitions_path) +ctx.parse("syscalls.def") +syscalls = ctx.interfaces['syscalls'] + +cog.outl(f"constexpr size_t num_syscalls = {len(syscalls.methods)};") +]]]*/ +/// [[[end]]] +uintptr_t syscall_registry[num_syscalls] __attribute__((section(".syscall_registry"))); + +void +syscall_invalid(uint64_t call) +{ + console *cons = console::get(); + cons->set_color(9); + cons->printf("\nReceived unknown syscall: %02x\n", call); + + cons->set_color(); + _halt(); +} + +void +syscall_initialize() +{ + kutil::memset(&syscall_registry, 0, sizeof(syscall_registry)); + + /*[[[cog code generation + for id, scope, method in syscalls.methods: + if scope: + name = f"{scope.name}_{method.name}" + else: + name = method.name + + cog.outl(f"syscall_registry[{id}] = reinterpret_cast(syscalls::{name});") + cog.outl(f"""log::debug(logs::syscall, "Enabling syscall {id:x} as {name}");""") + cog.outl("") + ]]]*/ + //[[[end]]] +} + diff --git a/src/kernel/syscall.h b/src/kernel/syscall.h deleted file mode 100644 index e16849b..0000000 --- a/src/kernel/syscall.h +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#include -#include "j6/types.h" - -struct cpu_state; - -enum class syscall : uint64_t -{ -#define SYSCALL(id, name, ...) name = id, -#include "j6/tables/syscalls.inc" -#undef SYSCALL -}; - -void syscall_initialize(); -extern "C" void syscall_enable(); - -namespace syscalls -{ -#define SYSCALL(id, name, ...) j6_status_t name (__VA_ARGS__); -#include "j6/tables/syscalls.inc" -#undef SYSCALL -} diff --git a/src/kernel/syscall.h.cog b/src/kernel/syscall.h.cog new file mode 100644 index 0000000..2169025 --- /dev/null +++ b/src/kernel/syscall.h.cog @@ -0,0 +1,60 @@ +#pragma once +// vim: ft=cpp + +#include +#include "j6/types.h" + +struct cpu_state; + +/*[[[cog code generation +from definitions.context import Context + +ctx = Context(definitions_path) +ctx.parse("syscalls.def") +syscalls = ctx.interfaces["syscalls"] + +]]]*/ +/// [[[end]]] + +enum class syscall : uint64_t +{ + /*[[[cog code generation + + for id, scope, method in syscalls.methods: + if scope: + name = f"{scope.name}_{method.name}" + else: + name = method.name + cog.outl(f"{name:20} = {id},") + + ]]]*/ + //[[[end]]] +}; + +void syscall_initialize(); +extern "C" void syscall_enable(); + +namespace syscalls +{ +/*[[[cog code generation + +for id, scope, method in syscalls.methods: + if scope: + name = f"{scope.name}_{method.name}" + else: + name = method.name + + args = [] + if method.constructor: + args.append("j6_handle_t *handle") + elif not method.static: + args.append("j6_handle_t handle") + + for param in method.params: + for type, suffix in param.type.c_names(param.options): + args.append(f"{type} {param.name}{suffix}") + + cog.outl(f"""j6_status_t {name} ({", ".join(args)});""") +]]]*/ +//[[[end]]] +} diff --git a/src/kernel/syscall.s b/src/kernel/syscall.s index 15834a6..3fa6451 100644 --- a/src/kernel/syscall.s +++ b/src/kernel/syscall.s @@ -1,4 +1,5 @@ %include "tasking.inc" +%include "syscalls.inc" ; SYSCALL/SYSRET control MSRs MSR_STAR equ 0xc0000081 @@ -51,7 +52,9 @@ syscall_handler_prelude: inc qword [rel __counter_syscall_enter] - and rax, 0xff ; Only 256 possible syscall values + cmp rax, NUM_SYSCALLS + jge .bad_syscall + lea r11, [rel syscall_registry] mov r11, [r11 + rax * 8] cmp r11, 0 diff --git a/src/kernel/syscalls.inc.cog b/src/kernel/syscalls.inc.cog new file mode 100644 index 0000000..634bc94 --- /dev/null +++ b/src/kernel/syscalls.inc.cog @@ -0,0 +1,11 @@ +; vim: ft=asm +; [[[cog code generation +; from definitions.context import Context +; +; ctx = Context(definitions_path) +; ctx.parse("syscalls.def") +; syscalls = ctx.interfaces['syscalls'] +; cog.outl(f"NUM_SYSCALLS equ {len(syscalls.methods)}") +; ]]] +; [[[end]]] + diff --git a/src/kernel/syscalls/endpoint.cpp b/src/kernel/syscalls/endpoint.cpp index d0c95ee..1821829 100644 --- a/src/kernel/syscalls/endpoint.cpp +++ b/src/kernel/syscalls/endpoint.cpp @@ -15,7 +15,7 @@ endpoint_create(j6_handle_t *handle) } j6_status_t -endpoint_send(j6_handle_t handle, j6_tag_t tag, size_t len, void *data) +endpoint_send(j6_handle_t handle, uint64_t tag, const void * data, size_t data_len) { if (tag & j6_tag_system_flag) return j6_err_invalid_arg; @@ -23,28 +23,28 @@ endpoint_send(j6_handle_t handle, j6_tag_t tag, size_t len, void *data) endpoint *e = get_handle(handle); if (!e) return j6_err_invalid_arg; - return e->send(tag, len, data); + return e->send(tag, data, data_len); } j6_status_t -endpoint_receive(j6_handle_t handle, j6_tag_t *tag, size_t *len, void *data) +endpoint_receive(j6_handle_t handle, uint64_t * tag, void * data, size_t * data_len) { - if (!tag || !len || (*len && !data)) + if (!tag || !data_len || !data) return j6_err_invalid_arg; endpoint *e = get_handle(handle); if (!e) return j6_err_invalid_arg; j6_tag_t out_tag = j6_tag_invalid; - size_t out_len = *len; - j6_status_t s = e->receive(&out_tag, &out_len, data); + size_t out_len = *data_len; + j6_status_t s = e->receive(&out_tag, data, &out_len); *tag = out_tag; - *len = out_len; + *data_len = out_len; return s; } j6_status_t -endpoint_sendrecv(j6_handle_t handle, j6_tag_t *tag, size_t *len, void *data) +endpoint_sendrecv(j6_handle_t handle, uint64_t * tag, void * data, size_t * data_len) { if (!tag || (*tag & j6_tag_system_flag)) return j6_err_invalid_arg; @@ -52,15 +52,15 @@ endpoint_sendrecv(j6_handle_t handle, j6_tag_t *tag, size_t *len, void *data) endpoint *e = get_handle(handle); if (!e) return j6_err_invalid_arg; - j6_status_t status = e->send(*tag, *len, data); + j6_status_t status = e->send(*tag, data, *data_len); if (status != j6_status_ok) return status; j6_tag_t out_tag = j6_tag_invalid; - size_t out_len = *len; - j6_status_t s = e->receive(&out_tag, &out_len, data); + size_t out_len = *data_len; + j6_status_t s = e->receive(&out_tag, data, &out_len); *tag = out_tag; - *len = out_len; + *data_len = out_len; return s; } diff --git a/src/kernel/syscalls/object.cpp b/src/kernel/syscalls/object.cpp index 1882740..f079f69 100644 --- a/src/kernel/syscalls/object.cpp +++ b/src/kernel/syscalls/object.cpp @@ -9,7 +9,7 @@ namespace syscalls { j6_status_t -object_koid(j6_handle_t handle, j6_koid_t *koid) +kobject_koid(j6_handle_t handle, j6_koid_t *koid) { if (koid == nullptr) return j6_err_invalid_arg; @@ -23,7 +23,7 @@ object_koid(j6_handle_t handle, j6_koid_t *koid) } j6_status_t -object_wait(j6_handle_t handle, j6_signal_t mask, j6_signal_t *sigs) +kobject_wait(j6_handle_t handle, j6_signal_t mask, j6_signal_t *sigs) { kobject *obj = get_handle(handle); if (!obj) @@ -47,11 +47,11 @@ object_wait(j6_handle_t handle, j6_signal_t mask, j6_signal_t *sigs) } j6_status_t -object_wait_many(j6_handle_t *handles, uint32_t count, j6_signal_t mask, j6_handle_t *handle, j6_signal_t *sigs) +kobject_wait_many(j6_handle_t * handles, size_t handles_count, uint64_t mask, j6_handle_t * handle, uint64_t * signals) { - kutil::vector objects {count}; + kutil::vector objects {uint32_t(handles_count)}; - for (unsigned i = 0; i < count; ++i) { + for (unsigned i = 0; i < handles_count; ++i) { j6_handle_t h = handles[i]; if (h == j6_handle_invalid) continue; @@ -62,7 +62,7 @@ object_wait_many(j6_handle_t *handles, uint32_t count, j6_signal_t mask, j6_hand j6_signal_t current = obj->signals(); if ((current & mask) != 0) { - *sigs = current; + *signals = current; *handle = h; return j6_status_ok; } @@ -81,9 +81,9 @@ object_wait_many(j6_handle_t *handles, uint32_t count, j6_signal_t mask, j6_hand return result; *handle = j6_handle_invalid; - *sigs = th.get_wait_data(); + *signals = th.get_wait_data(); j6_koid_t koid = th.get_wait_object(); - for (unsigned i = 0; i < count; ++i) { + for (unsigned i = 0; i < handles_count; ++i) { if (koid == objects[i]->koid()) *handle = handles[i]; else @@ -96,7 +96,7 @@ object_wait_many(j6_handle_t *handles, uint32_t count, j6_signal_t mask, j6_hand } j6_status_t -object_signal(j6_handle_t handle, j6_signal_t signals) +kobject_signal(j6_handle_t handle, j6_signal_t signals) { if ((signals & j6_signal_user_mask) != signals) return j6_err_invalid_arg; @@ -110,7 +110,7 @@ object_signal(j6_handle_t handle, j6_signal_t signals) } j6_status_t -object_close(j6_handle_t handle) +kobject_close(j6_handle_t handle) { kobject *obj = get_handle(handle); if (!obj) diff --git a/src/kernel/syscalls/process.cpp b/src/kernel/syscalls/process.cpp index 914d66d..4b498cb 100644 --- a/src/kernel/syscalls/process.cpp +++ b/src/kernel/syscalls/process.cpp @@ -16,14 +16,14 @@ process_create(j6_handle_t *handle) } j6_status_t -process_start(j6_handle_t handle, uintptr_t entrypoint, j6_handle_t *handles, size_t handle_count) +process_start(j6_handle_t handle, uintptr_t entrypoint, j6_handle_t * handles, size_t handles_count) { process &p = process::current(); process *c = get_handle(handle); - if (handle_count && !handles) + if (handles_count && !handles) return j6_err_invalid_arg; - for (size_t i = 0; i < handle_count; ++i) { + for (size_t i = 0; i < handles_count; ++i) { kobject *o = p.lookup_handle(handles[i]); if (o) c->add_handle(o); } diff --git a/src/kernel/syscalls/system.cpp b/src/kernel/syscalls/system.cpp index 3be6f5a..90928e5 100644 --- a/src/kernel/syscalls/system.cpp +++ b/src/kernel/syscalls/system.cpp @@ -15,7 +15,7 @@ extern log::logger &g_logger; namespace syscalls { j6_status_t -system_log(const char *message) +log(const char *message) { if (message == nullptr) return j6_err_invalid_arg; @@ -26,7 +26,7 @@ system_log(const char *message) } j6_status_t -system_noop() +noop() { thread &th = thread::current(); log::debug(logs::syscall, "Thread %llx called noop syscall.", th.koid()); @@ -61,18 +61,18 @@ system_bind_irq(j6_handle_t sys, j6_handle_t endp, unsigned irq) } j6_status_t -system_map_phys(j6_handle_t sys, j6_handle_t *vma_handle, uintptr_t phys_addr, size_t size, uint32_t flags) +system_map_phys(j6_handle_t handle, j6_handle_t * area, uintptr_t phys, size_t size, uint32_t flags) { // TODO: check capabilities on sys handle - if (!vma_handle) return j6_err_invalid_arg; + if (!area) return j6_err_invalid_arg; // TODO: check to see if frames are already used? How would that collide with // the bootloader's allocated pages already being marked used? if (!(flags & vm_flags::mmio)) - frame_allocator::get().used(phys_addr, memory::page_count(size)); + frame_allocator::get().used(phys, memory::page_count(size)); vm_flags vmf = (static_cast(flags) & vm_flags::driver_mask); - construct_handle(vma_handle, phys_addr, size, vmf); + construct_handle(area, phys, size, vmf); return j6_status_ok; } diff --git a/src/kernel/syscalls/thread.cpp b/src/kernel/syscalls/thread.cpp index f35f67c..52a2633 100644 --- a/src/kernel/syscalls/thread.cpp +++ b/src/kernel/syscalls/thread.cpp @@ -4,17 +4,18 @@ #include "log.h" #include "objects/process.h" #include "objects/thread.h" +#include "syscalls/helpers.h" namespace syscalls { j6_status_t -thread_create(void *rip, j6_handle_t *handle) +thread_create(j6_handle_t *handle, uintptr_t entrypoint) { thread &parent = thread::current(); process &p = parent.parent(); thread *child = p.create_thread(); - child->add_thunk_user(reinterpret_cast(rip)); + child->add_thunk_user(entrypoint); *handle = child->self_handle(); child->clear_state(thread::state::loading); child->set_state(thread::state::ready); @@ -36,6 +37,18 @@ thread_exit(int32_t status) return j6_err_unexpected; } +j6_status_t +thread_kill(j6_handle_t handle) +{ + thread *th = get_handle(handle); + if (!th) + return j6_err_invalid_arg; + + log::debug(logs::task, "Killing thread %llx", th->koid()); + th->exit(-1); + return j6_status_ok; +} + j6_status_t thread_pause() { diff --git a/src/kernel/vm_space.cpp b/src/kernel/vm_space.cpp index 7e9a070..ae4d369 100644 --- a/src/kernel/vm_space.cpp +++ b/src/kernel/vm_space.cpp @@ -286,7 +286,7 @@ vm_space::handle_fault(uintptr_t addr, fault_type fault) } size_t -vm_space::copy(vm_space &source, vm_space &dest, void *from, void *to, size_t length) +vm_space::copy(vm_space &source, vm_space &dest, const void *from, void *to, size_t length) { uintptr_t ifrom = reinterpret_cast(from); uintptr_t ito = reinterpret_cast(to); diff --git a/src/kernel/vm_space.h b/src/kernel/vm_space.h index 92c2aaf..abdfd49 100644 --- a/src/kernel/vm_space.h +++ b/src/kernel/vm_space.h @@ -103,7 +103,7 @@ public: /// \arg to Pointer to the destination in the dest address space /// \arg length Amount of data to copy, in bytes /// \returnd The number of bytes copied - static size_t copy(vm_space &source, vm_space &dest, void *from, void *to, size_t length); + static size_t copy(vm_space &source, vm_space &dest, const void *from, void *to, size_t length); private: friend class vm_area; diff --git a/src/libraries/j6/include/j6/syscalls.h b/src/libraries/j6/include/j6/syscalls.h deleted file mode 100644 index 51b6a64..0000000 --- a/src/libraries/j6/include/j6/syscalls.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#define SYSCALL(n, name, ...) j6_status_t j6_ ## name (__VA_ARGS__); -#include "j6/tables/syscalls.inc" -#undef SYSCALL - -#ifdef __cplusplus -} -#endif diff --git a/src/libraries/j6/include/j6/syscalls.h.cog b/src/libraries/j6/include/j6/syscalls.h.cog new file mode 100644 index 0000000..78042a2 --- /dev/null +++ b/src/libraries/j6/include/j6/syscalls.h.cog @@ -0,0 +1,39 @@ +#pragma once +// vim: ft=cpp + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/*[[[cog code generation +from definitions.context import Context + +ctx = Context(definitions_path) +ctx.parse("syscalls.def") +syscalls = ctx.interfaces["syscalls"] + +for id, scope, method in syscalls.methods: + if scope: + name = f"{scope.name}_{method.name}" + else: + name = method.name + + args = [] + if method.constructor: + args.append("j6_handle_t *handle") + elif not method.static: + args.append("j6_handle_t handle") + + for param in method.params: + for type, suffix in param.type.c_names(param.options): + args.append(f"{type} {param.name}{suffix}") + + cog.outl(f"""j6_status_t j6_{name} ({", ".join(args)});""") +]]]*/ +/// [[[end]]] + +#ifdef __cplusplus +} +#endif diff --git a/src/libraries/j6/j6.module b/src/libraries/j6/j6.module index 001ab1f..e4cc7a5 100644 --- a/src/libraries/j6/j6.module +++ b/src/libraries/j6/j6.module @@ -1,9 +1,14 @@ # vim: ft=python -module("j6", +j6 = module("j6", kind = "lib", includes = [ "include" ], sources = [ "init.cpp", - "syscalls.s", ]) + +from glob import glob +definitions = glob('definitions/**/*.def', recursive=True) +j6.add_input("include/j6/syscalls.h.cog", deps=definitions) +j6.add_input("syscalls.s.cog", deps=definitions) + diff --git a/src/libraries/j6/syscalls.s b/src/libraries/j6/syscalls.s deleted file mode 100644 index dd1ecae..0000000 --- a/src/libraries/j6/syscalls.s +++ /dev/null @@ -1,28 +0,0 @@ -%macro SYSCALL 2 - global j6_%1 - j6_%1: - push rbp - mov rbp, rsp - - ; args should already be in rdi, etc, but rcx will - ; get stomped, so stash it in r10, which isn't a - ; callee-saved register, but also isn't used in the - ; function call ABI. - mov r10, rcx - - mov rax, %2 - syscall - ; result is now already in rax, so just return - - pop rbp - ret -%endmacro - -%define SYSCALL(n, name) SYSCALL name, n -%define SYSCALL(n, name, a) SYSCALL name, n -%define SYSCALL(n, name, a, b) SYSCALL name, n -%define SYSCALL(n, name, a, b, c) SYSCALL name, n -%define SYSCALL(n, name, a, b, c, d) SYSCALL name, n -%define SYSCALL(n, name, a, b, c, d, e) SYSCALL name, n - -%include "j6/tables/syscalls.inc" diff --git a/src/libraries/j6/syscalls.s.cog b/src/libraries/j6/syscalls.s.cog new file mode 100644 index 0000000..f1caf3f --- /dev/null +++ b/src/libraries/j6/syscalls.s.cog @@ -0,0 +1,37 @@ +; vim: ft=asm + +%macro define_syscall 2 + global j6_%1 + j6_%1: + push rbp + mov rbp, rsp + + ; args should already be in rdi, etc, but rcx will + ; get stomped, so stash it in r10, which isn't a + ; callee-saved register, but also isn't used in the + ; function call ABI. + mov r10, rcx + + mov rax, %2 + syscall + ; result is now already in rax, so just return + + pop rbp + ret +%endmacro + +; [[[cog code generation +; from definitions.context import Context +; +; ctx = Context(definitions_path) +; ctx.parse("syscalls.def") +; syscalls = ctx.interfaces['syscalls'] +; +; for id, scope, method in syscalls.methods: +; if scope: +; name = f"{scope.name}_{method.name}" +; else: +; name = method.name +; cog.outl(f"define_syscall {name:20}, {id}") +; ]]] +; [[[end]]] diff --git a/src/user/drv.uefi_fb/main.cpp b/src/user/drv.uefi_fb/main.cpp index 387247b..33f4e7b 100644 --- a/src/user/drv.uefi_fb/main.cpp +++ b/src/user/drv.uefi_fb/main.cpp @@ -33,7 +33,7 @@ struct entry int main(int argc, const char **argv) { - j6_system_log("fb driver starting"); + j6_log("fb driver starting"); size_t initc = 0; j6_init_value *initv = nullptr; @@ -48,7 +48,7 @@ main(int argc, const char **argv) } if (!fb || fb->addr == 0) { - j6_system_log("fb driver didn't find a framebuffer, exiting"); + j6_log("fb driver didn't find a framebuffer, exiting"); return 1; } @@ -109,7 +109,7 @@ main(int argc, const char **argv) buffer_size = size; continue; } else if (s != j6_status_ok) { - j6_system_log("fb driver got error from get_log, quitting"); + j6_log("fb driver got error from get_log, quitting"); return s; } @@ -134,7 +134,7 @@ main(int argc, const char **argv) } } - j6_system_log("fb driver done, exiting"); + j6_log("fb driver done, exiting"); return 0; } diff --git a/src/user/srv.init/main.cpp b/src/user/srv.init/main.cpp index 57f5961..3deed43 100644 --- a/src/user/srv.init/main.cpp +++ b/src/user/srv.init/main.cpp @@ -20,7 +20,7 @@ j6_handle_t handle_system = 2; // boot protocol is that init gets the system as int main(int argc, const char **argv) { - j6_system_log("srv.init starting"); + j6_log("srv.init starting"); modules mods = modules::load_modules(_arg_modules_phys, handle_system, handle_self); @@ -28,7 +28,7 @@ main(int argc, const char **argv) for (auto &mod : mods.of_type(module_type::program)) { auto &prog = static_cast(mod); sprintf(message, " program module '%s' at %lx", prog.filename, prog.base_address); - j6_system_log(message); + j6_log(message); } return 0; diff --git a/src/user/srv.init/modules.cpp b/src/user/srv.init/modules.cpp index eee0382..349e67b 100644 --- a/src/user/srv.init/modules.cpp +++ b/src/user/srv.init/modules.cpp @@ -70,7 +70,7 @@ modules::load_modules(uintptr_t address, j6_handle_t system, j6_handle_t self) char message[100]; sprintf(message, "srv.init found %d modules from page at 0x%lx", page->count, address); - j6_system_log(message); + j6_log(message); if (!first) first = page->modules; diff --git a/src/user/testapp/main.cpp b/src/user/testapp/main.cpp index d77d64b..3c9c19e 100644 --- a/src/user/testapp/main.cpp +++ b/src/user/testapp/main.cpp @@ -20,32 +20,32 @@ extern "C" { void thread_proc() { - j6_system_log("sub thread starting"); + j6_log("sub thread starting"); char buffer[512]; size_t len = sizeof(buffer); j6_tag_t tag = 0; - j6_status_t result = j6_endpoint_receive(endp, &tag, &len, (void*)buffer); + j6_status_t result = j6_endpoint_receive(endp, &tag, (void*)buffer, &len); if (result != j6_status_ok) j6_thread_exit(result); - j6_system_log("sub thread received message"); + j6_log("sub thread received message"); for (int i = 0; i < len; ++i) if (buffer[i] >= 'A' && buffer[i] <= 'Z') buffer[i] += 0x20; tag++; - result = j6_endpoint_send(endp, tag, len, (void*)buffer); + result = j6_endpoint_send(endp, tag, (void*)buffer, len); if (result != j6_status_ok) j6_thread_exit(result); - j6_system_log("sub thread sent message"); + j6_log("sub thread sent message"); for (int i = 1; i < 5; ++i) j6_thread_sleep(i*10); - j6_system_log("sub thread exiting"); + j6_log("sub thread exiting"); j6_thread_exit(0); } @@ -55,10 +55,10 @@ main(int argc, const char **argv) j6_handle_t children[2] = {j6_handle_invalid, j6_handle_invalid}; j6_signal_t out = 0; - j6_system_log("main thread starting"); + j6_log("main thread starting"); for (int i = 0; i < argc; ++i) - j6_system_log(argv[i]); + j6_log(argv[i]); void *base = malloc(0x1000); if (!base) @@ -68,57 +68,57 @@ main(int argc, const char **argv) for (int i = 0; i < 3; ++i) vma_ptr[i*100] = uint64_t(i); - j6_system_log("main thread wrote to memory area"); + j6_log("main thread wrote to memory area"); j6_status_t result = j6_endpoint_create(&endp); if (result != j6_status_ok) return result; - j6_system_log("main thread created endpoint"); + j6_log("main thread created endpoint"); - result = j6_thread_create(reinterpret_cast(&thread_proc), &children[1]); + result = j6_thread_create(&children[1], reinterpret_cast(&thread_proc)); if (result != j6_status_ok) return result; - j6_system_log("main thread created sub thread"); + j6_log("main thread created sub thread"); char message[] = "MAIN THREAD SUCCESSFULLY CALLED SENDRECV IF THIS IS LOWERCASE"; size_t size = sizeof(message); j6_tag_t tag = 16; - result = j6_endpoint_sendrecv(endp, &tag, &size, (void*)message); + result = j6_endpoint_sendrecv(endp, &tag, (void*)message, &size); if (result != j6_status_ok) return result; if (tag != 17) - j6_system_log("GOT WRONG TAG FROM SENDRECV"); + j6_log("GOT WRONG TAG FROM SENDRECV"); result = j6_system_bind_irq(__handle_sys, endp, 3); if (result != j6_status_ok) return result; - j6_system_log(message); + j6_log(message); - j6_system_log("main thread creating a new process"); + j6_log("main thread creating a new process"); result = j6_process_create(&children[0]); if (result != j6_status_ok) return result; - j6_system_log("main thread waiting on children"); + j6_log("main thread waiting on children"); j6_handle_t outhandle; - result = j6_object_wait_many(children, 2, -1ull, &outhandle, &out); + result = j6_kobject_wait_many(children, 2, -1ull, &outhandle, &out); if (result != j6_status_ok) return result; if (outhandle == children[1]) { - j6_system_log(" ok from child thread"); + j6_log(" ok from child thread"); } else if (outhandle == children[0]) { - j6_system_log(" ok from child process"); + j6_log(" ok from child process"); } else { - j6_system_log(" ... got unknown handle"); + j6_log(" ... got unknown handle"); } - j6_system_log("main testing irqs"); + j6_log("main testing irqs"); serial_port com2(COM2); @@ -137,16 +137,10 @@ main(int argc, const char **argv) return result; if (j6_tag_is_irq(tag)) - j6_system_log("main thread got irq!"); + j6_log("main thread got irq!"); } - j6_system_log("main thread closing endpoint"); - - result = j6_object_close(endp); - if (result != j6_status_ok) - return result; - - j6_system_log("main thread done, exiting"); + j6_log("main thread done, exiting"); return 0; }