reeval-python 0.10.0__py3-none-any.whl

reeval_python/__init__.py

@@ -0,0 +1,23 @@
+"""Python eval authoring API for reeval (*.eval.py)."""
+
+from reeval_python.context import AttemptContext, SyncAttemptContext
+from reeval_python.declare import collect_module_tree, evaluate, group, reset_collector
+from reeval_python.errors import ReevalAssertionError
+from reeval_python.expect import expect, soft
+from reeval_python.types import BackgroundHandle, ExecResult, Turn, Verdict
+
+__all__ = [
+    "AttemptContext",
+    "BackgroundHandle",
+    "ExecResult",
+    "ReevalAssertionError",
+    "SyncAttemptContext",
+    "Turn",
+    "Verdict",
+    "collect_module_tree",
+    "evaluate",
+    "expect",
+    "group",
+    "reset_collector",
+    "soft",
+]

reeval_python/__main__.py

@@ -0,0 +1,4 @@
+from reeval_python.runner import main
+
+if __name__ == "__main__":
+    main()

reeval_python/assertion.py

@@ -0,0 +1,168 @@
+"""Turn plain Python assert failures into attempt-protocol expect calls."""
+
+from __future__ import annotations
+
+import ast
+import linecache
+from dataclasses import dataclass
+from types import FrameType
+from typing import Any
+
+
+@dataclass(frozen=True, slots=True)
+class AssertExpect:
+    subject: dict[str, Any]
+    matcher: str
+    args: list[Any]
+    label: str | None
+    not_: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class _EvalFrame:
+    filename: str
+    lineno: int
+    frame: FrameType
+
+
+def _safe_eval(node: ast.expr, frame: FrameType) -> Any:
+    code = compile(ast.Expression(node), "<assert>", "eval")
+    return eval(code, frame.f_globals, frame.f_locals)  # noqa: S307
+
+
+def _show(value: Any) -> str:
+    try:
+        return repr(value)
+    except Exception:  # noqa: BLE001
+        return "<unrepr>"
+
+
+def _frame_for_assert(exc: BaseException) -> _EvalFrame | None:
+    tb = exc.__traceback__
+    found: _EvalFrame | None = None
+    while tb is not None:
+        if tb.tb_frame.f_code.co_filename.endswith(".eval.py"):
+            found = _EvalFrame(tb.tb_frame.f_code.co_filename, tb.tb_lineno, tb.tb_frame)
+        tb = tb.tb_next
+    return found
+
+
+def _parse_assert_line(source: str, lineno: int, filename: str) -> ast.Assert | None:
+    try:
+        tree = ast.parse(source, mode="exec")
+    except SyntaxError:
+        tree = None
+    if tree is not None:
+        for node in ast.walk(tree):
+            if isinstance(node, ast.Assert) and node.lineno == lineno:
+                return node
+    line = linecache.getline(filename, lineno).strip()
+    if not line.startswith("assert"):
+        return None
+    try:
+        parsed = ast.parse(line)
+    except SyntaxError:
+        return None
+    stmt = parsed.body[0] if parsed.body else None
+    return stmt if isinstance(stmt, ast.Assert) else None
+
+
+def _label_from_assert(node: ast.Assert, exc: AssertionError) -> str | None:
+    if node.msg is not None:
+        try:
+            if isinstance(node.msg, ast.Constant) and isinstance(node.msg.value, str):
+                return node.msg.value
+        except Exception:  # noqa: BLE001
+            pass
+    if exc.args and isinstance(exc.args[0], str):
+        return exc.args[0]
+    return None
+
+
+def _wire_for_test(test: ast.expr, frame: FrameType, label: str | None) -> AssertExpect:
+    if isinstance(test, ast.Compare) and len(test.ops) == 1 and len(test.comparators) == 1:
+        op = test.ops[0]
+        left = _safe_eval(test.left, frame)
+        right = _safe_eval(test.comparators[0], frame)
+        if isinstance(op, ast.Eq):
+            enriched = label or f"assert {_show(left)} == {_show(right)}"
+            return AssertExpect(
+                subject={"value": left},
+                matcher="toEqual",
+                args=[right],
+                label=enriched,
+            )
+        if isinstance(op, ast.Is):
+            enriched = label or f"assert {_show(left)} is {_show(right)}"
+            return AssertExpect(
+                subject={"value": left},
+                matcher="toBe",
+                args=[right],
+                label=enriched,
+            )
+        if isinstance(op, ast.IsNot):
+            enriched = label or f"assert {_show(left)} is not {_show(right)}"
+            return AssertExpect(
+                subject={"value": left},
+                matcher="toBe",
+                args=[right],
+                label=enriched,
+                not_=True,
+            )
+
+    if isinstance(test, ast.Constant) and test.value is False:
+        return AssertExpect(
+            subject={"value": False},
+            matcher="toBeTruthy",
+            args=[],
+            label=label or "assert False",
+        )
+
+    actual = _safe_eval(test, frame)
+    enriched = label
+    if enriched is None:
+        enriched = f"assert {_show(actual)}"
+    elif not actual:
+        enriched = f"{enriched} ({_show(actual)} is falsy)"
+    return AssertExpect(
+        subject={"value": actual},
+        matcher="toBeTruthy",
+        args=[],
+        label=enriched,
+    )
+
+
+def assert_to_expect(exc: AssertionError) -> AssertExpect:
+    frame = _frame_for_assert(exc)
+    if frame is None:
+        msg = str(exc) if str(exc) else "assertion failed"
+        return AssertExpect(
+            subject={"value": False},
+            matcher="toBeTruthy",
+            args=[],
+            label=msg,
+        )
+
+    source = linecache.getlines(frame.filename)
+    if not source:
+        msg = str(exc) if str(exc) else "assertion failed"
+        return AssertExpect(
+            subject={"value": False},
+            matcher="toBeTruthy",
+            args=[],
+            label=msg,
+        )
+
+    block = "".join(source)
+    node = _parse_assert_line(block, frame.lineno, frame.filename)
+    if node is None:
+        msg = str(exc) if str(exc) else "assertion failed"
+        return AssertExpect(
+            subject={"value": False},
+            matcher="toBeTruthy",
+            args=[],
+            label=msg,
+        )
+
+    label = _label_from_assert(node, exc)
+    return _wire_for_test(node.test, frame.frame, label)

reeval_python/context.py

@@ -0,0 +1,163 @@
+"""Attempt context: native Python API over the attempt protocol."""
+
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable, Coroutine
+from typing import Any, cast
+
+from reeval_python.assertion import assert_to_expect
+from reeval_python.errors import ReevalAssertionError
+from reeval_python.expect import bind_expect
+from reeval_python.types import BackgroundHandle, ExecResult, Turn, Verdict
+from reeval_python.wire import options_to_wire
+
+SendFn = Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]
+
+
+class AttemptContext:
+    def __init__(self, attempt_id: str, send: SendFn) -> None:
+        self._attempt = attempt_id
+        self._send = send
+        self._seq = 0
+        self.turns: list[Turn] = []
+
+    async def _call(self, op: str, **fields: Any) -> dict[str, Any]:
+        self._seq += 1
+        call: dict[str, Any] = {"op": op, **fields}
+        reply = await self._send({"call": call})
+        if not reply.get("ok", False):
+            err = reply.get("error") or {"name": "Error", "message": "call failed"}
+            raise RuntimeError(f"{err.get('name')}: {err.get('message')}")
+        return reply
+
+    async def send(self, prompt: str) -> Turn:
+        reply = await self._call("send", prompt=prompt)
+        turn = Turn(index=len(self.turns), raw=cast(dict[str, Any], reply["turn"]))
+        self.turns.append(turn)
+        return turn
+
+    async def exec(
+        self,
+        command: str,
+        *,
+        background: bool = False,
+    ) -> ExecResult | BackgroundHandle:
+        fields: dict[str, Any] = {"command": command}
+        if background:
+            fields["background"] = True
+        reply = await self._call("exec", **fields)
+        result = cast(dict[str, Any], reply["result"])
+        if background:
+            return BackgroundHandle.from_wire(result)
+        return ExecResult.from_wire(command, result)
+
+    async def read_file(self, path: str) -> str | None:
+        reply = await self._call("readFile", path=path)
+        contents = reply.get("contents")
+        return None if contents is None else str(contents)
+
+    async def file(self, path: str) -> str | None:
+        return await self.read_file(path)
+
+    async def write_file(self, path: str, content: str) -> None:
+        await self._call("writeFile", path=path, content=content)
+
+    async def kill(self, handle: BackgroundHandle) -> None:
+        await self._call("kill", handle=handle.id)
+
+    async def judge(self, question: str, **options: Any) -> Verdict:
+        wire_opts = options_to_wire(options) if options else None
+        reply = await self._call("judge", question=question, options=wire_opts)
+        return Verdict.from_wire(cast(dict[str, Any], reply["verdict"]))
+
+    async def run_expect(
+        self,
+        *,
+        subject: dict[str, Any],
+        matcher: str,
+        args: list[Any],
+        label: str | None,
+        not_: bool,
+        soft: bool,
+    ) -> None:
+        call: dict[str, Any] = {
+            "op": "expect",
+            "subject": subject,
+            "matcher": matcher,
+            "args": args,
+            "not": not_,
+            "soft": soft,
+        }
+        if label is not None:
+            call["label"] = label
+        reply = await self._send({"call": call})
+        if not reply.get("ok", False):
+            err = reply.get("error") or {"name": "Error", "message": "call failed"}
+            raise RuntimeError(f"{err.get('name')}: {err.get('message')}")
+        assertion_raw = reply.get("assertion")
+        assertion = cast(dict[str, Any], assertion_raw) if isinstance(assertion_raw, dict) else {}
+        if assertion.get("status") == "fail":
+            raise ReevalAssertionError(str(assertion.get("message") or "expect failed"))
+
+    async def record_plain_assert(self, exc: AssertionError) -> None:
+        wire = assert_to_expect(exc)
+        await self.run_expect(
+            subject=wire.subject,
+            matcher=wire.matcher,
+            args=wire.args,
+            label=wire.label,
+            not_=wire.not_,
+            soft=False,
+        )
+
+    def bind_expect(self) -> None:
+        async def submit(**kwargs: Any) -> None:
+            await self.run_expect(**kwargs)
+
+        submit._sync_mode = False  # type: ignore[attr-defined]
+        bind_expect(submit)
+
+
+class SyncAttemptContext:
+    """Sync façade: each method blocks on the runner's asyncio loop."""
+
+    def __init__(self, async_ctx: AttemptContext, loop: Any) -> None:
+        self._ctx = async_ctx
+        self._loop = loop
+
+    def _run(self, coro: Coroutine[Any, Any, Any]) -> Any:
+        import asyncio
+
+        fut = asyncio.run_coroutine_threadsafe(coro, self._loop)
+        return fut.result()
+
+    def send(self, prompt: str) -> Turn:
+        return self._run(self._ctx.send(prompt))
+
+    def exec(self, command: str, *, background: bool = False) -> ExecResult | BackgroundHandle:
+        return cast(
+            ExecResult | BackgroundHandle,
+            self._run(self._ctx.exec(command, background=background)),  # type: ignore[arg-type]
+        )
+
+    def read_file(self, path: str) -> str | None:
+        return self._run(self._ctx.read_file(path))
+
+    def file(self, path: str) -> str | None:
+        return self._run(self._ctx.file(path))
+
+    def write_file(self, path: str, content: str) -> None:
+        self._run(self._ctx.write_file(path, content))
+
+    def kill(self, handle: BackgroundHandle) -> None:
+        self._run(self._ctx.kill(handle))
+
+    def judge(self, question: str, **options: Any) -> Verdict:
+        return self._run(self._ctx.judge(question, **options))
+
+    def bind_expect(self) -> None:
+        def submit(**kwargs: Any) -> None:
+            self._run(self._ctx.run_expect(**kwargs))
+
+        submit._sync_mode = True  # type: ignore[attr-defined]
+        bind_expect(submit)

reeval_python/declare.py

@@ -0,0 +1,263 @@
+"""Build a declaration tree matching reeval's TypeScript collector ordering."""
+
+from __future__ import annotations
+
+import importlib.util
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal, cast
+
+from reeval_python.wire import options_to_wire
+
+Mode = Literal["run", "skip", "only", "todo"]
+
+
+@dataclass
+class DeclaredFamily:
+    name: str
+    case: int
+    cases: int
+
+    def as_json(self) -> dict[str, Any]:
+        return {"name": self.name, "case": self.case, "cases": self.cases}
+
+
+@dataclass
+class DeclaredEval:
+    name: str
+    mode: Mode
+    options: dict[str, Any]
+    body: str
+    family: DeclaredFamily | None = None
+
+    def as_json(self) -> dict[str, Any]:
+        out: dict[str, Any] = {
+            "type": "eval",
+            "name": self.name,
+            "mode": self.mode,
+            "options": self.options,
+            "body": self.body,
+        }
+        if self.family is not None:
+            out["family"] = self.family.as_json()
+        return out
+
+
+@dataclass
+class DeclaredGroup:
+    name: str
+    mode: Mode
+    options: dict[str, Any]
+    children: list[DeclNode] = field(default_factory=list)  # pyright: ignore[reportUnknownVariableType]
+
+    def as_json(self) -> dict[str, Any]:
+        return {
+            "type": "group",
+            "name": self.name,
+            "mode": self.mode,
+            "options": self.options,
+            "children": [c.as_json() for c in self.children],
+        }
+
+
+@dataclass
+class DeclaredTodo:
+    name: str
+
+    def as_json(self) -> dict[str, Any]:
+        return {"type": "todo", "name": self.name}
+
+
+DeclNode = DeclaredEval | DeclaredGroup | DeclaredTodo
+
+
+@dataclass
+class _GroupFrame:
+    name: str
+    options: dict[str, Any]
+    node: DeclaredGroup
+
+
+def _empty_suites() -> list[_Suite]:
+    return []
+
+
+@dataclass
+class _Suite:
+    factory: Callable[[], None] | None = None
+    suites: list[_Suite] = field(default_factory=_empty_suites)
+
+
+_tree_children: list[DeclNode] | None = None
+_group_stack: list[_GroupFrame] = []
+_suites: list[_Suite] = []
+_bodies: dict[str, Callable[..., Any]] = {}
+
+
+def reset_collector() -> None:
+    global _tree_children, _group_stack, _suites, _bodies
+    _tree_children = []
+    _group_stack = []
+    _suites = [_Suite()]
+    _bodies = {}
+
+
+def _next_body(fn: Callable[..., Any]) -> str:
+    body = f"b{len(_bodies) + 1}"
+    _bodies[body] = fn
+    return body
+
+
+def _declare(node: DeclNode) -> None:
+    parent = _group_stack[-1].node if _group_stack else None
+    if parent is not None:
+        parent.children.append(node)
+    else:
+        assert _tree_children is not None
+        _tree_children.append(node)
+
+
+def _register_group(
+    name: str, mode: Mode, options: dict[str, Any], factory: Callable[[], None]
+) -> None:
+    suite = _Suite(factory=factory)
+    _suites[-1].suites.append(suite)
+
+
+def _wire_options(options: dict[str, Any]) -> dict[str, Any]:
+    return options_to_wire(options) or {}
+
+
+def group(
+    name: str,
+    arg: Callable[[], None] | dict[str, Any] | None = None,
+    arg2: Callable[[], None] | None = None,
+    /,
+    *,
+    mode: Mode = "run",
+) -> None:
+    if callable(arg) and arg2 is None:
+        fn, opts = arg, {}
+    elif isinstance(arg, dict) and callable(arg2):
+        opts, fn = dict[str, Any](cast(dict[str, Any], arg)), arg2
+    else:
+        raise TypeError("group(name, options, fn) or group(name, fn)")
+
+    node = DeclaredGroup(name=name, mode=mode, options=_wire_options(opts), children=[])
+    _declare(node)
+    frame = _GroupFrame(name=name, options=opts, node=node)
+
+    def factory() -> None:
+        outer = _group_stack.copy()
+        _group_stack.clear()
+        _group_stack.extend([*outer, frame])
+        try:
+            fn()
+        finally:
+            _group_stack.clear()
+            _group_stack.extend(outer)
+
+    _register_group(name, mode, opts, factory)
+
+
+def evaluate(
+    name: str,
+    arg: Callable[..., Any] | dict[str, Any] | None = None,
+    arg2: Callable[..., Any] | None = None,
+    /,
+    *,
+    mode: Mode = "run",
+) -> Callable[[Callable[..., Any]], Callable[..., Any]] | None:
+    if callable(arg) and arg2 is None:
+        _register_eval(name, mode, {}, arg, None)
+        return arg
+    if isinstance(arg, dict) and callable(arg2):
+        _register_eval(name, mode, dict[str, Any](cast(dict[str, Any], arg)), arg2, None)
+        return arg2
+
+    opts = dict[str, Any](cast(dict[str, Any], arg)) if isinstance(arg, dict) else {}
+
+    def decorator(fn: Callable[..., Any]) -> Callable[..., Any]:
+        _register_eval(name, mode, opts, fn, None)
+        return fn
+
+    return decorator
+
+
+def evaluate_each(cases: Sequence[Any], name_template: str, options: dict[str, Any] | None = None):
+    opts = dict[str, Any](options or {})
+
+    def decorator(fn: Callable[..., Any]) -> Callable[..., Any]:
+        for index, case in enumerate(cases):
+            family = DeclaredFamily(name=name_template, case=index, cases=len(cases))
+            case_value: Any = case
+            if isinstance(case_value, dict) and "n" in case_value:
+                display = name_template.replace("$n", str(cast(Any, case_value["n"])))
+            else:
+                display = name_template.replace("$n", str(index + 1)).replace("%#", str(index))
+            _register_eval(display, "run", opts, fn, family)
+        return fn
+
+    return decorator
+
+
+def _register_eval(
+    name: str,
+    mode: Mode,
+    options: dict[str, Any],
+    fn: Callable[..., Any],
+    family: DeclaredFamily | None,
+) -> None:
+    body = _next_body(fn)
+    _declare(
+        DeclaredEval(
+            name=name,
+            mode=mode,
+            options=_wire_options(options),
+            body=body,
+            family=family,
+        )
+    )
+
+
+def _collect_suite_sync(suite: _Suite) -> None:
+    if suite.factory is not None:
+        suite.factory()
+    for nested in suite.suites:
+        _collect_suite_sync(nested)
+
+
+def _tree_after_loader(loader: Callable[[], None]) -> dict[str, Any]:
+    reset_collector()
+    loader()
+    _collect_suite_sync(_suites[0])
+    assert _tree_children is not None
+    return {"children": [c.as_json() for c in _tree_children]}
+
+
+async def collect_module_tree(loader: Callable[[], None]) -> dict[str, Any]:
+    return _tree_after_loader(loader)
+
+
+def collect_module_tree_sync(loader: Callable[[], None]) -> dict[str, Any]:
+    return _tree_after_loader(loader)
+
+
+def get_body(body_id: str) -> Callable[..., Any]:
+    if body_id not in _bodies:
+        raise KeyError(f"unknown eval body {body_id}")
+    return _bodies[body_id]
+
+
+def collect_eval_file(path: Path) -> dict[str, Any]:
+    """Import one *.eval.py file and return its declaration tree JSON."""
+
+    def loader() -> None:
+        spec = importlib.util.spec_from_file_location("reeval_eval", path)
+        if spec is None or spec.loader is None:
+            raise ImportError(f"can't load {path}")
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+
+    return collect_module_tree_sync(loader)

reeval_python/errors.py

@@ -0,0 +1,7 @@
+"""Errors raised by the public Python API."""
+
+from __future__ import annotations
+
+
+class ReevalAssertionError(AssertionError):
+    """An expect matcher failed (mirrors reeval's assertion record message)."""