annoscript 0.1.0__py3-none-any.whl

annoscript/formatter.py

@@ -0,0 +1,171 @@
+"""Readable, non-executable notation for any version-1 AnnoScript IR program.
+
+format_ir(program, show_source=False) preserves every statement, temporary name,
+literal and control-flow block. It neither optimizes nor runs the program and
+does not reconstruct the original helper functions. format_value/format_string
+in the output denote the corresponding IR expressions, not Python functions.
+"""
+
+from __future__ import annotations
+
+import keyword
+
+from annoscript import ir
+
+
+_COMPARE = {
+    "eq": "==", "ne": "!=", "lt": "<", "le": "<=", "gt": ">", "ge": ">=",
+    "in": "in", "not_in": "not in", "is": "is", "is_not": "is not",
+}
+
+
+def _identifier(name: str) -> bool:
+    return name.isidentifier() and not keyword.iskeyword(name)
+
+
+def _name(name: str) -> str:
+    return name if _identifier(name) else f"var[{name!r}]"
+
+
+def _arguments(args, kwargs) -> str:
+    parts = [_expression(arg) for arg in args]
+    for name, value in kwargs:
+        if _identifier(name):
+            parts.append(f"{name}={_expression(value)}")
+        else:
+            parts.append(f"**{{{name!r}: {_expression(value)}}}")
+    return ", ".join(parts)
+
+
+def _base(node: ir.Expr) -> str:
+    text = _expression(node)
+    if isinstance(node, (ir.Var, ir.State, ir.GetItem, ir.Field, ir.Intrinsic)):
+        return text
+    return f"({text})"
+
+
+def _expression(node: ir.Expr) -> str:
+    match node:
+        case ir.Literal(value):
+            return repr(value)
+        case ir.Var(name):
+            return _name(name)
+        case ir.State():
+            return "anno.state"
+        case ir.GetItem(value, index):
+            return f"{_base(value)}[{_expression(index)}]"
+        case ir.Field(value, name):
+            return f"{_base(value)}.{name}"
+        case ir.ListExpr(items):
+            return "[" + ", ".join(map(_expression, items)) + "]"
+        case ir.TupleExpr(items):
+            return "(" + ", ".join(map(_expression, items)) + ("," if len(items) == 1 else "") + ")"
+        case ir.DictExpr(entries):
+            return "{" + ", ".join(f"{_expression(k)}: {_expression(v)}" for k, v in entries) + "}"
+        case ir.Unary(op, value):
+            prefix = {"not": "not ", "negative": "-", "positive": "+"}[op]
+            return f"({prefix}{_expression(value)})"
+        case ir.Binary("add", left, right):
+            return f"({_expression(left)} + {_expression(right)})"
+        case ir.Compare(left, ops, comparators):
+            parts = [_expression(left)]
+            for op, right in zip(ops, comparators):
+                parts.extend((_COMPARE[op], _expression(right)))
+            return "(" + " ".join(parts) + ")"
+        case ir.BoolOp(op, values):
+            return "(" + f" {op} ".join(map(_expression, values)) + ")"
+        case ir.Intrinsic(name, args, kwargs):
+            return f"{name}({_arguments(args, kwargs)})"
+        case ir.FormatValue(value, conversion, spec):
+            parts = [_expression(value)]
+            if conversion != "none":
+                parts.append(f"conversion={conversion!r}")
+            if spec is not None:
+                parts.append(f"spec={_expression(spec)}")
+            return "format_value(" + ", ".join(parts) + ")"
+        case ir.FormatString(parts):
+            return "format_string(" + ", ".join(map(_expression, parts)) + ")"
+    raise ValueError(f"Unsupported IR expression: {type(node).__name__}")
+
+
+def _target(node: ir.Target) -> str:
+    match node:
+        case ir.NameTarget(name):
+            return _name(name)
+        case ir.TupleTarget(items):
+            return "(" + ", ".join(map(_target, items)) + ("," if len(items) == 1 else "") + ")"
+    raise ValueError(f"Unsupported IR target: {type(node).__name__}")
+
+
+def _location(span: ir.SourceSpan) -> str:
+    return f"{span.module}:{span.line}:{span.column} ({span.function})"
+
+
+def _comment(text: str) -> str:
+    # Keep embedded newlines/control characters in metadata on one comment line.
+    return "# " + repr(text)[1:-1]
+
+
+class _Formatter:
+    def __init__(self, show_source):
+        self.show_source = show_source
+        self.lines = []
+
+    def line(self, depth, text):
+        self.lines.append("    " * depth + text)
+
+    def block(self, body, depth):
+        if not body:
+            self.line(depth, "pass")
+        for node in body:
+            if self.show_source:
+                text = f"[{node.id}] {_location(node.source.span)}"
+                if node.source.calls:
+                    text += "; inlined from " + " -> ".join(map(_location, node.source.calls))
+                self.line(depth, _comment(text))
+            match node:
+                case ir.Assign(target, value):
+                    self.line(depth, f"{_name(target)} = {_expression(value)}")
+                case ir.Unpack(target, value):
+                    self.line(depth, f"{_target(target)} = {_expression(value)}")
+                case ir.SetItem(value, index, item):
+                    self.line(depth, f"{_base(value)}[{_expression(index)}] = {_expression(item)}")
+                case ir.If(condition, then_body, else_body):
+                    self.line(depth, f"if {_expression(condition)}:")
+                    self.block(then_body, depth + 1)
+                    if else_body:
+                        self.line(depth, "else:")
+                        self.block(else_body, depth + 1)
+                case ir.For(target, iterable, body):
+                    self.line(depth, f"for {_target(target)} in {_expression(iterable)}:")
+                    self.block(body, depth + 1)
+                case ir.Infer(target, operation, args, kwargs):
+                    self.line(depth, f"{_name(target)} = anno.{operation}({_arguments(args, kwargs)})")
+                case ir.Constrain(condition, message):
+                    text = f"constrain({_expression(condition)}"
+                    if message is not None:
+                        text += f", message={_expression(message)}"
+                    self.line(depth, text + ")")
+                case ir.Return(value):
+                    self.line(depth, f"return {_expression(value)}")
+                case _:
+                    raise ValueError(f"Unsupported IR statement: {type(node).__name__}")
+
+
+def format_ir(program: ir.Program, *, show_source: bool = False) -> str:
+    """Render a Program as deterministic display text, ending with a newline.
+
+    Load external data through Program.from_json/from_dict first. The output is
+    an inspection format, not executable Python or a serialization format.
+    """
+    if not isinstance(program, ir.Program):
+        raise TypeError("format_ir expects an ir.Program")
+    formatter = _Formatter(show_source)
+    formatter.line(0, f"# AnnoScript IR v{program.version} (display notation)")
+    if show_source:
+        formatter.line(0, _comment(_location(program.source)))
+    if program.config.system_prompt is not None:
+        formatter.line(0, f"system_prompt {program.config.system_prompt!r}")
+    formatter.line(0, f"program {program.entry!r}:")
+    formatter.block(program.body, 1)
+    return "\n".join(formatter.lines) + "\n"

annoscript/grammar_tests.py

@@ -0,0 +1,176 @@
+"""Required CPU-only XGrammar acceptance: python3.12 -m annoscript.grammar_tests.
+
+Separate from unittest discovery: XGrammar is a test dependency, never a client
+or execution dependency. No model, tokenizer files or GPU are used or downloaded.
+Missing/wrong dependencies fail this entry point instead of skipping acceptance.
+"""
+
+from importlib.metadata import version
+import json
+import sys
+import unittest
+from types import SimpleNamespace
+
+from annoscript.backends._grammar import ITEMS_GRAMMAR, BATCH_GRAMMAR, batch_grammar
+
+
+class CompactGrammarTests(unittest.TestCase):
+    grammar = ITEMS_GRAMMAR
+    @classmethod
+    def setUpClass(cls):
+        if sys.version_info[:2] != (3, 12):
+            raise RuntimeError("grammar acceptance requires Python 3.12")
+        if version("xgrammar") != "0.2.0":
+            raise RuntimeError("install requirements-test-grammar.txt (xgrammar==0.2.0)")
+        import xgrammar as xgr
+        cls.xgr = xgr
+        grammar = xgr.Grammar.from_ebnf(cls.grammar)
+        # Empty vocabulary: compile/match strings without any model/tokenizer.
+        cls.compiled = xgr.GrammarCompiler(xgr.TokenizerInfo([]), cache_enabled=False).compile_grammar(grammar)
+
+    def accepts(self, text):
+        matcher = self.xgr.GrammarMatcher(self.compiled, terminate_without_stop_token=True)
+        return matcher.accept_string(text) and matcher.is_terminated()
+
+    def test_json_strings_unicode_and_escapes(self):
+        for values in ([], [""], ["中文", "😀", " A B "], ['"\\/\b\f\n\r\t'],
+                       ["\x00\x1f"], ["e\u0301", "\u2028\u2029"], ["条目"]):
+            for ascii_only in (True, False):
+                payload = json.dumps({"items": values}, ensure_ascii=ascii_only, separators=(",", ":"))
+                with self.subTest(payload=payload):
+                    self.assertTrue(self.accepts(payload))
+                    self.assertEqual(json.loads(payload)["items"], values)
+        self.assertTrue(self.accepts(r'{"items":["\/\u4E2D"]}'))
+
+    def test_structure_has_no_whitespace_outside_strings(self):
+        for payload in (' {"items":[]}', '{"items":[]}\n', '\n{"items":[]}',
+                        '{ "items":[]}', '{"items" :[]}', '{"items": []}',
+                        '{"items":[ ]}', '{"items":["a", "b"]}', '{"items":["a"] }'):
+            with self.subTest(payload=payload):
+                self.assertFalse(self.accepts(payload))
+
+    def test_extra_fields_types_and_invalid_strings_are_rejected(self):
+        for payload in ('{}', '[]', 'null', '{"items":null}', '{"items":"x"}',
+                        '{"items":[1]}', '{"items":[true]}', '{"items":[null]}',
+                        '{"items":[{}]}', '{"items":[[]]}', '{"items":[],"extra":0}',
+                        '{"items":[],"items":[]}', '{"other":[]}', '{"items":["a",]}',
+                        '{"items":["a\nb"]}', r'{"items":["\x20"]}',
+                        r'{"items":["\u12Q4"]}', '{"items":["\x00"]}'):
+            with self.subTest(payload=payload):
+                self.assertFalse(self.accepts(payload))
+
+    def test_incomplete_output_is_not_a_complete_match(self):
+        payload = '{"items":["中文","条目"]}'
+        for end in range(len(payload)):
+            with self.subTest(end=end):
+                self.assertFalse(self.accepts(payload[:end]))
+        self.assertTrue(self.accepts(payload))
+
+    def test_business_bounds_are_left_to_runtime(self):
+        # Grammar remains reusable for all programs, not just application field bounds.
+        self.assertTrue(self.accepts(json.dumps({"items": ["x" * 301] * 17}, separators=(",", ":"))))
+
+
+class BatchGrammarTests(unittest.TestCase):
+    grammar = BATCH_GRAMMAR
+    setUpClass = classmethod(CompactGrammarTests.setUpClass.__func__)
+    accepts = CompactGrammarTests.accepts
+
+    def test_mixed_answers_unicode_and_escapes(self):
+        for ascii_only in (False, True):
+            payload = json.dumps({"answers": {
+                "q1": ' 中文😀 "引号" \\路径\n\x00',
+                "q2": ["", "条目"],
+                "q3": [],
+            }}, ensure_ascii=ascii_only, separators=(",", ":"))
+            self.assertTrue(self.accepts(payload))
+            for end in range(len(payload)):
+                self.assertFalse(self.accepts(payload[:end]))
+
+    def test_bad_types_extra_fields_and_outer_whitespace(self):
+        good = '{"answers":{"q1":"x"}}'
+        for payload in (" " + good, good + "\n", good.replace(':"x"', ': "x"'),
+                        good.replace('"x"', 'null'), good.replace('"x"', 'true'),
+                        good.replace('"x"', '[null]'), good.replace('"x"', '{"text":"x"}'),
+                        good[:-1] + ',"extra":0}', '{}', '[]'):
+            with self.subTest(payload=payload):
+                self.assertFalse(self.accepts(payload))
+
+
+class BoundBatchGrammarTests(unittest.TestCase):
+    calls = [SimpleNamespace(id='extract"\n😀', operation="extract", arguments={}),
+             SimpleNamespace(id="list", operation="generate_list", arguments={}),
+             SimpleNamespace(id="text", operation="generate", arguments={})]
+    grammar = batch_grammar(calls)
+    setUpClass = classmethod(CompactGrammarTests.setUpClass.__func__)
+    accepts = CompactGrammarTests.accepts
+
+    def answer(self):
+        return {"answers": {
+            "q1": ["广州"], "q2": ["😀", "", '"\\\n'], "q3": "标题"}}
+
+    def encode(self, value):
+        return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
+
+    def test_exact_mapping_unicode_escaping_and_completion(self):
+        payload = self.encode(self.answer())
+        self.assertTrue(self.accepts(payload))
+        for end in range(len(payload)):
+            self.assertFalse(self.accepts(payload[:end]))
+
+    def test_wrong_type_for_extract_id_is_impossible(self):
+        for value in ("广州", {"type": "extract", "items": ["广州"]}):
+            answer = self.answer()
+            answer["answers"]["q1"] = value
+            self.assertFalse(self.accepts(self.encode(answer)))
+
+    def test_unknown_missing_and_duplicate_ids_are_impossible(self):
+        answer = self.answer()
+        del answer["answers"]["q2"]
+        self.assertFalse(self.accepts(self.encode(answer)))
+        answer = self.answer()
+        answer["answers"]["unknown"] = answer["answers"]["q2"]
+        self.assertFalse(self.accepts(self.encode(answer)))
+        good = self.encode(self.answer())
+        self.assertFalse(self.accepts(good[:-2] + ',"q3":"x"}}'))
+        self.assertFalse(self.accepts('{"answers":{}}'))
+
+
+class NamedBatchGrammarTests(unittest.TestCase):
+    calls = [SimpleNamespace(id="a", operation="extract", arguments={"field_name": ' 地点"\\\n😀 '}),
+             SimpleNamespace(id="b", operation="generate_list", arguments={"field_name": "tags"}),
+             SimpleNamespace(id="c", operation="generate", arguments={})]
+    grammar = batch_grammar(calls)
+    setUpClass = classmethod(CompactGrammarTests.setUpClass.__func__)
+    accepts = CompactGrammarTests.accepts
+
+    def test_named_and_automatic_keys_and_escaping(self):
+        value = {"answers": {' 地点"\\\n😀 ': ["广州"], "tags": ["条目"], "q3": "标题"}}
+        # Grammar key literals use JSON's ASCII escapes; decoded names are exact.
+        encoded = json.dumps(value, ensure_ascii=True, separators=(",", ":"))
+        self.assertTrue(self.accepts(encoded))
+        self.assertFalse(self.accepts(encoded.replace('tags', 'q2')))
+        value["answers"]["q3"] = []
+        self.assertFalse(self.accepts(json.dumps(value, ensure_ascii=True, separators=(",", ":"))))
+
+
+class NullableBatchGrammarTests(unittest.TestCase):
+    grammar = batch_grammar([
+        SimpleNamespace(id="a", operation="generate", arguments={"nullable": True}),
+        SimpleNamespace(id="b", operation="generate", arguments={}),
+        SimpleNamespace(id="c", operation="generate_list", arguments={}),
+    ])
+    setUpClass = classmethod(CompactGrammarTests.setUpClass.__func__)
+    accepts = CompactGrammarTests.accepts
+
+    def test_only_nullable_text_accepts_null(self):
+        for value in (None, "null", "中文", ""):
+            good = json.dumps({"answers": {"q1": value, "q2": "标题", "q3": []}}, ensure_ascii=False, separators=(",", ":"))
+            self.assertTrue(self.accepts(good))
+        for value in ({"q1": [], "q2": "标题", "q3": []},
+                      {"q1": None, "q2": None, "q3": []}, {"q1": None, "q2": "标题", "q3": None}):
+            self.assertFalse(self.accepts(json.dumps({"answers": value}, separators=(",", ":"))))
+
+
+if __name__ == "__main__":
+    unittest.main()

annoscript/inspect.py

@@ -0,0 +1,77 @@
+"""Inspect AnnoScript IR or an Execution Plan without importing or running its source.
+
+Usage from the repository root (or with AnnoScript on PYTHONPATH):
+
+    python3 -m annoscript.inspect path/to/program.json
+    python3 -m annoscript.inspect path/to/program.json --show-source
+    cat path/to/program.json | python3 -m annoscript.inspect -
+    python3 -m annoscript.inspect request.execution_plan.json --mode strict
+    python3 -m annoscript.inspect request.execution_plan.json --format html -o plan.html
+
+For Python callers, use Program.from_json + annoscript.formatter.format_ir, or
+ExecutionPlan.from_json + annoscript.plan_formatter.format_plan / format_plan_html.
+Output goes to stdout unless --output is provided. Inspection does not lower
+or optimize IR. HTML is self-contained and can be opened offline in a browser.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+import sys
+
+from annoscript.formatter import format_ir
+from annoscript.ir import IRDecodeError, Program
+from annoscript.plan import ExecutionPlan
+from annoscript.plan_formatter import format_plan, format_plan_html, format_plan_mermaid
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="python3 -m annoscript.inspect",
+        description="Inspect AnnoScript IR or visualize Execution Plan Infer dependencies without execution.",
+    )
+    parser.add_argument("path", metavar="PATH", help="AnnoScript IR / Execution Plan JSON, or - for stdin")
+    parser.add_argument("--show-source", action="store_true", help="show statement IDs, source locations and inline call chains")
+    parser.add_argument("--format", choices=("text", "html", "mermaid"), default="text", help="output format (HTML/Mermaid require an Execution Plan)")
+    parser.add_argument("--mode", choices=("eager", "strict", "speculative"), default="eager", help="Execution Plan failure mode; HTML includes all three modes")
+    parser.add_argument("-o", "--output", metavar="PATH", help="write UTF-8 output to this file instead of stdout")
+    args = parser.parse_args(argv)
+    try:
+        text = sys.stdin.read() if args.path == "-" else Path(args.path).read_text(encoding="utf-8")
+        # Only sniff the format here. Each strict decoder parses the original
+        # text again, so duplicate JSON keys and non-finite values stay errors.
+        header = json.loads(text)
+        if isinstance(header, dict) and header.get("format") == "lev-execution-plan":
+            plan = ExecutionPlan.from_json(text)
+            formatter = {"text": format_plan, "html": format_plan_html, "mermaid": format_plan_mermaid}[args.format]
+            output = formatter(plan, mode=args.mode, show_source=args.show_source)
+        else:
+            program = Program.from_json(text)
+            if args.format != "text" or args.mode != "eager":
+                raise IRDecodeError("--format html/mermaid and --mode strict/speculative require an Execution Plan; lower the IR explicitly first")
+            output = format_ir(program, show_source=args.show_source)
+        if args.output and args.output != "-":
+            if args.path != "-" and Path(args.output).resolve() == Path(args.path).resolve():
+                raise ValueError("output path must differ from the input JSON file")
+            Path(args.output).write_text(output, encoding="utf-8")
+            return 0
+    except (OSError, UnicodeError, ValueError, RecursionError) as exc:
+        print(f"annoscript.inspect: {exc}", file=sys.stderr)
+        return 2
+    try:
+        sys.stdout.write(output)
+        sys.stdout.flush()
+    except BrokenPipeError:
+        # A pager may stop reading early. Closing also prevents a second failed
+        # flush during interpreter shutdown.
+        try:
+            sys.stdout.close()
+        except BrokenPipeError:
+            pass
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())