uplox 3.0.0__py3-none-any.whl

uplox/__init__.py

@@ -0,0 +1,4 @@
+"""uplox — compiler front-end generator."""
+
+__version__ = "3.0.0"
+UPLOX_SCHEMA_VERSION = "1"

uplox/ast/__init__.py

@@ -0,0 +1,150 @@
+"""AST node schema and the default tree-builder action set.
+
+The parser runtime in :mod:`uplox.parse.runtime` produces :class:`uplox.parse.runtime.ParseNode`
+trees by default — a one-to-one mirror of the grammar's parse tree. That is fine
+for prototyping but generally too verbose for downstream tools.
+
+This module gives:
+
+* A neutral :class:`AstNode` schema that grammar authors can target through
+  semantic actions, with one node kind per concept rather than per production.
+* The :func:`build_ast` helper: walks a ParseNode tree and applies user-supplied
+  per-rule reducers to flatten chains, drop literal punctuation, and lift
+  meaningful children. This is the "default tree-builder" deliverable from the
+  Phase-4 plan.
+* JSON ser/deser in :func:`ast_to_json` / :func:`ast_from_json` so AST nodes
+  fit cleanly into the bundle's ``ast`` section once a backend wants to
+  pre-compute them.
+
+What this module deliberately does *not* do:
+
+* Type checking, name resolution, scope handling — those are hooks (see
+  :mod:`uplox.hooks`), not AST concerns.
+* Pretty-printing — would couple AST shape to host-language style; left to
+  callers.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Callable, Iterable, Optional, Union
+
+from ..lex.scanner import Token
+from ..parse.runtime import ParseNode
+
+AstChild = Union["AstNode", Token, str, int, float, bool, None]
+
+
+@dataclass
+class AstNode:
+    """Neutral AST node. Distinct from :class:`ParseNode` so the AST shape can
+    diverge from the grammar's parse tree.
+
+    ``kind`` is a stable string the user picks (e.g. ``"BinaryOp"``). ``attrs``
+    holds named per-node fields (operator, name, value). ``children`` are
+    ordered children in source order.
+    """
+
+    kind: str
+    attrs: dict[str, Any] = field(default_factory=dict)
+    children: list[AstChild] = field(default_factory=list)
+    position: Optional[tuple[int, int]] = None  # (line, column) of the first token
+
+    def __repr__(self) -> str:  # pragma: no cover - cosmetic
+        return f"AstNode({self.kind!r}, attrs={self.attrs!r}, children={self.children!r})"
+
+
+# A reducer takes a ParseNode (or token leaf) and returns an AstChild.
+# The dispatch is by ParseNode.kind (i.e. the grammar's LHS for that node).
+Reducer = Callable[[ParseNode, list[AstChild]], AstChild]
+
+
+def build_ast(
+    root: ParseNode | Token,
+    reducers: dict[str, Reducer],
+    *,
+    drop_tokens: Iterable[str] = (),
+) -> AstChild:
+    """Walk ``root`` bottom-up; transform each ParseNode through its reducer.
+
+    For each ParseNode encountered:
+    * Recursively transform every child (Token children become themselves
+      unless their token name is in ``drop_tokens``, in which case they are
+      filtered out — handy for punctuation like LPAREN/SEMI).
+    * Look up ``reducers[node.kind]``. If present, call it with the original
+      ParseNode and the (already-transformed) child list. The return value
+      is what ends up in the parent's child list.
+    * If no reducer is registered, fall back to wrapping the node in an
+      AstNode with the same ``kind``.
+
+    Token leaves at the root level pass through unchanged (the caller can
+    map them as they please).
+    """
+    drop_set = set(drop_tokens)
+
+    def visit(node: ParseNode | Token) -> AstChild:
+        if isinstance(node, Token):
+            return node
+        new_children: list[AstChild] = []
+        for c in node.children:
+            if isinstance(c, Token) and c.name in drop_set:
+                continue
+            new_children.append(visit(c) if isinstance(c, (ParseNode, Token)) else c)
+        reducer = reducers.get(node.kind)
+        if reducer is None:
+            return AstNode(kind=node.kind, children=new_children)
+        return reducer(node, new_children)
+
+    return visit(root)
+
+
+# ---- JSON serialisation ------------------------------------------------------
+
+
+def ast_to_json(node: AstChild) -> Any:
+    """Serialise an AST tree to JSON-compatible primitives.
+
+    AstNodes become dicts with ``kind``, ``attrs``, ``children`` (and ``position``
+    when set). Tokens become ``{"_token": name, "text": ..., "line": ..., "column": ...}``.
+    Plain Python primitives pass through untouched. The shape is deliberately
+    explicit so a backend in any language can reconstruct it without inferring
+    types.
+    """
+    if isinstance(node, AstNode):
+        out: dict[str, Any] = {
+            "_kind": node.kind,
+            "attrs": dict(node.attrs),
+            "children": [ast_to_json(c) for c in node.children],
+        }
+        if node.position is not None:
+            out["position"] = list(node.position)
+        return out
+    if isinstance(node, Token):
+        return {
+            "_token": node.name,
+            "text": node.text,
+            "line": node.line,
+            "column": node.column,
+        }
+    return node
+
+
+def ast_from_json(value: Any) -> AstChild:
+    if isinstance(value, dict):
+        if "_token" in value:
+            return Token(
+                name=value["_token"],
+                text=value["text"],
+                line=value["line"],
+                column=value["column"],
+                offset=value.get("offset", -1),
+            )
+        if "_kind" in value:
+            pos = value.get("position")
+            return AstNode(
+                kind=value["_kind"],
+                attrs=dict(value.get("attrs", {})),
+                children=[ast_from_json(c) for c in value.get("children", [])],
+                position=tuple(pos) if pos else None,
+            )
+    return value

uplox/cli/__init__.py

@@ -0,0 +1 @@
+"""``uplox`` command-line interface."""

uplox/cli/main.py

@@ -0,0 +1,312 @@
+"""``uplox`` command entry point.
+
+Subcommands:
+
+* ``uplox version``                — print uplox version and schema version.
+* ``uplox build <grammar.uplox> -o <out.json>``
+                                  — build the JSON bundle. In Phase 2 only the
+                                    lex section is populated; the parse / ast /
+                                    hooks sections are emitted empty so backends
+                                    can already start consuming bundles.
+* ``uplox check <grammar.uplox>``   — parse + lower without emitting JSON; reports
+                                    syntax errors and lex-construction failures.
+* ``uplox emit  <bundle.json> --target=c|cpp|py|lua --out=<dir>``
+                                  — drive a backend. Stubbed until Phase 7-8.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from .. import UPLOX_SCHEMA_VERSION, __version__
+from ..lex.build import lex_from_ir
+from ..lex.scanner import Scanner
+from ..parse.grammar import GrammarError, compile_grammar
+from ..parse.lr1 import build_lr1
+from ..gen.c import emit_c
+from ..gen.cpp import emit_cpp
+from ..gen.lua import emit_lua
+from ..gen.py import emit_py
+from ..parse.glr import GLRParseError, glr_from_lr, glr_parse
+from ..parse.glr.runtime import AmbiguityNode, GLRNode
+from ..parse.runtime import HookRegistry, ParseError, parse as run_parser
+from ..spec.reader import ReaderError, read_file
+from ..lex.build import balanced_tokens
+from ..tables import (
+    balanced_from_json,
+    dfa_from_json,
+    dfa_to_json,
+    dump_bundle,
+    empty_bundle,
+    table_from_json,
+    table_to_json,
+)
+
+
+def _cmd_version(_args: argparse.Namespace) -> int:
+    print(f"uplox {__version__} (schema {UPLOX_SCHEMA_VERSION})")
+    return 0
+
+
+def _cmd_build(args: argparse.Namespace) -> int:
+    try:
+        ir = read_file(args.source)
+    except ReaderError as e:
+        print(str(e), file=sys.stderr)
+        return 1
+    try:
+        dfa, tokens, skip = lex_from_ir(ir)
+    except ValueError as e:
+        print(f"{args.source}: {e}", file=sys.stderr)
+        return 1
+
+    bundle = empty_bundle(ir.name)
+    bundle["lex"] = dfa_to_json(
+        dfa, tokens=tokens, skip=skip, balanced=balanced_tokens(ir)
+    )
+
+    if not args.lex_only:
+        try:
+            grammar = compile_grammar(ir)
+            table = build_lr1(grammar)
+        except GrammarError as e:
+            print(f"{args.source}: {e}", file=sys.stderr)
+            return 1
+        if table.conflicts:
+            print(
+                f"{args.source}: refusing to build with {len(table.conflicts)} parser conflict(s):",
+                file=sys.stderr,
+            )
+            for c in table.conflicts:
+                print(c.describe(table.grammar), file=sys.stderr)
+                print("", file=sys.stderr)
+            return 1
+        bundle["parse"] = table_to_json(table)
+
+    text = dump_bundle(bundle)
+    if args.output == "-":
+        sys.stdout.write(text)
+    else:
+        with open(args.output, "w", encoding="utf-8") as fh:
+            fh.write(text)
+    return 0
+
+
+def _cmd_parse(args: argparse.Namespace) -> int:
+    """Smoke-parse: load a bundle, run scanner+parser on stdin or a path, dump tree."""
+    import json as _json
+
+    with open(args.bundle, "r", encoding="utf-8") as fh:
+        bundle = _json.load(fh)
+    if not bundle.get("parse"):
+        print(f"{args.bundle}: bundle has no parse section (was it built --lex-only?)", file=sys.stderr)
+        return 1
+
+    dfa, _tokens, skip = dfa_from_json(bundle["lex"])
+    scanner = Scanner(
+        dfa=dfa,
+        skip_tokens=frozenset(skip),
+        balanced=balanced_from_json(bundle["lex"]),
+    )
+    table = table_from_json(bundle["parse"])
+
+    if args.input == "-":
+        text = sys.stdin.read()
+    else:
+        with open(args.input, "r", encoding="utf-8") as fh:
+            text = fh.read()
+
+    try:
+        if args.glr:
+            tree = glr_parse(glr_from_lr(table), scanner.scan(text))
+        else:
+            # The LR runtime is a smoke tool here; we don't try to resolve
+            # hooks the way a real host driver would. Unknown names no-op so
+            # any grammar builds and parses end-to-end.
+            tree = run_parser(
+                table,
+                scanner.scan(text),
+                hooks=HookRegistry(ignore_missing=True),
+            )
+    except (ParseError, GLRParseError) as e:
+        print(f"{args.input}: {e}", file=sys.stderr)
+        return 1
+
+    sys.stdout.write(_render_tree(tree) + "\n")
+    return 0
+
+
+def _render_tree(tree, indent: int = 0) -> str:
+    from ..lex.scanner import Token
+    from ..parse.runtime import ParseNode
+    pad = "  " * indent
+    if isinstance(tree, Token):
+        return f"{pad}{tree.name} {tree.text!r}"
+    if isinstance(tree, (ParseNode, GLRNode)):
+        lines = [f"{pad}{tree.kind}"]
+        for c in tree.children:
+            lines.append(_render_tree(c, indent + 1))
+        return "\n".join(lines)
+    if isinstance(tree, AmbiguityNode):
+        lines = [f"{pad}AMBIGUITY[{tree.kind}] ({len(tree.alternatives)} alternatives)"]
+        for i, alt in enumerate(tree.alternatives):
+            lines.append(f"{pad}  alt {i + 1}:")
+            lines.append(_render_tree(alt, indent + 2))
+        return "\n".join(lines)
+    return f"{pad}{tree!r}"
+
+
+def _cmd_check(args: argparse.Namespace) -> int:
+    try:
+        ir = read_file(args.source)
+        lex_from_ir(ir)
+    except (ReaderError, ValueError) as e:
+        print(str(e), file=sys.stderr)
+        return 1
+
+    parser_summary = ""
+    parser_conflicts = 0
+    try:
+        grammar = compile_grammar(ir)
+        table = build_lr1(grammar)
+        parser_conflicts = len(table.conflicts)
+        if table.conflicts:
+            print(f"{args.source}: {parser_conflicts} parser conflict(s):", file=sys.stderr)
+            for c in table.conflicts:
+                print(c.describe(table.grammar), file=sys.stderr)
+                print("", file=sys.stderr)
+        parser_summary = (
+            f", {len(grammar.productions)} productions, "
+            f"{len(table.states)} states, {parser_conflicts} conflicts"
+        )
+    except GrammarError as e:
+        print(f"{args.source}: {e}", file=sys.stderr)
+        return 1
+
+    print(
+        f"{args.source}: {ir.name} — {len(ir.tokens)} tokens, "
+        f"{len(ir.hooks)} hooks{parser_summary}"
+    )
+    return 1 if parser_conflicts else 0
+
+
+def _cmd_emit(args: argparse.Namespace) -> int:
+    import json as _json
+    import os as _os
+
+    with open(args.bundle, "r", encoding="utf-8") as fh:
+        bundle = _json.load(fh)
+
+    grammar = (args.prefix or bundle.get("meta", {}).get("grammar") or "grammar").lower()
+    _os.makedirs(args.out, exist_ok=True)
+
+    try:
+        if args.target == "c":
+            header, impl = emit_c(bundle, prefix=args.prefix)
+            header_path = _os.path.join(args.out, f"uplox_{grammar}.h")
+            impl_path = _os.path.join(args.out, f"uplox_{grammar}.c")
+        elif args.target == "cpp":
+            header, impl = emit_cpp(bundle, prefix=args.prefix)
+            header_path = _os.path.join(args.out, f"uplox_{grammar}.hpp")
+            impl_path = _os.path.join(args.out, f"uplox_{grammar}.cpp")
+        elif args.target == "lua":
+            module_text = emit_lua(bundle, prefix=args.prefix)
+            module_path = _os.path.join(args.out, f"uplox_{grammar}.lua")
+            with open(module_path, "w", encoding="utf-8") as fh:
+                fh.write(module_text)
+            print(f"wrote {module_path}")
+            return 0
+        elif args.target == "py":
+            module_text = emit_py(bundle, prefix=args.prefix)
+            module_path = _os.path.join(args.out, f"uplox_{grammar}.py")
+            with open(module_path, "w", encoding="utf-8") as fh:
+                fh.write(module_text)
+            print(f"wrote {module_path}")
+            return 0
+        else:
+            print(
+                f"uplox emit --target={args.target}: unknown target",
+                file=sys.stderr,
+            )
+            return 2
+    except ValueError as e:
+        print(f"{args.bundle}: {e}", file=sys.stderr)
+        return 1
+
+    with open(header_path, "w", encoding="utf-8") as fh:
+        fh.write(header)
+    with open(impl_path, "w", encoding="utf-8") as fh:
+        fh.write(impl)
+    print(f"wrote {header_path}\nwrote {impl_path}")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="uplox",
+        description="Compiler front-end generator (grammar -> JSON tables + drivers)",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p_version = sub.add_parser("version", help="print uplox and schema versions")
+    p_version.set_defaults(func=_cmd_version)
+
+    p_build = sub.add_parser("build", help="compile a .uplox grammar to a JSON bundle")
+    p_build.add_argument("source", help="path to .uplox source file")
+    p_build.add_argument(
+        "-o", "--output",
+        default="-",
+        help="output bundle path; '-' (default) writes to stdout",
+    )
+    p_build.add_argument(
+        "--lex-only",
+        action="store_true",
+        help="emit only the lex section (skip the LR table)",
+    )
+    p_build.set_defaults(func=_cmd_build)
+
+    p_check = sub.add_parser("check", help="parse and validate a .uplox grammar without emitting")
+    p_check.add_argument("source", help="path to .uplox source file")
+    p_check.set_defaults(func=_cmd_check)
+
+    p_parse = sub.add_parser(
+        "parse",
+        help="parse input through a built bundle and pretty-print the parse tree",
+    )
+    p_parse.add_argument("bundle", help="path to JSON bundle (output of `uplox build`)")
+    p_parse.add_argument(
+        "input",
+        help="path to input file; '-' reads from stdin",
+        nargs="?",
+        default="-",
+    )
+    p_parse.add_argument(
+        "--glr",
+        action="store_true",
+        help="parse with the GLR runtime (handles ambiguous grammars; produces a parse forest)",
+    )
+    p_parse.set_defaults(func=_cmd_parse)
+
+    p_emit = sub.add_parser("emit", help="emit a C driver from a bundle (--target=c is supported in Phase 7)")
+    p_emit.add_argument("bundle", help="path to JSON bundle")
+    p_emit.add_argument("--target", required=True, choices=["c", "cpp", "py", "lua"])
+    p_emit.add_argument("--out", required=True, help="output directory")
+    p_emit.add_argument(
+        "--prefix",
+        default=None,
+        help="override grammar name used as the symbol prefix (default: meta.grammar)",
+    )
+    p_emit.set_defaults(func=_cmd_emit)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

uplox/gen/__init__.py

@@ -0,0 +1,5 @@
+"""Backends. Each subpackage emits a driver skeleton in one target language.
+
+Backends consume only the JSON bundle described in :mod:`uplox.tables.schema`.
+They never read the grammar source directly — the JSON is the contract.
+"""

uplox/gen/c/__init__.py

@@ -0,0 +1,12 @@
+"""C backend. Emits a self-contained ``.c`` / ``.h`` pair from a JSON bundle.
+
+Re-entrant by construction: all parser state lives on a ``uplox_<grammar>_ctx``
+struct, all symbols are prefixed with the grammar name, no file-scope
+mutable state. Two grammars built with uplox can link into the same binary
+without symbol collisions — the design's acceptance test (per
+``docs/c_backend.md``).
+"""
+
+from .emit import emit_c
+
+__all__ = ["emit_c"]