reeval-python 0.10.0__py3-none-any.whl

reeval_python/expect.py

@@ -0,0 +1,95 @@
+"""Fluent expect() API; maps to wire matchers internally."""
+
+from __future__ import annotations
+
+from contextvars import ContextVar
+from typing import Any
+
+from reeval_python.wire import MATCHER_TO_WIRE, subject_for_matcher, wire_args
+
+SubmitFn = Any
+
+_submit: ContextVar[SubmitFn | None] = ContextVar("_submit", default=None)
+
+
+def bind_expect(submit: SubmitFn) -> None:
+    _submit.set(submit)
+
+
+def expect(actual: Any, label: str | None = None) -> _Expectation:
+    submit = _submit.get()
+    if submit is None:
+        raise RuntimeError("expect() must run inside an eval body")
+    return _Expectation(actual, submit, label=label)
+
+
+class _Expectation:
+    def __init__(
+        self,
+        actual: Any,
+        submit: SubmitFn,
+        *,
+        label: str | None = None,
+        not_: bool = False,
+        soft: bool = False,
+    ) -> None:
+        self._actual = actual
+        self._submit = submit
+        self._label = label
+        self._not = not_
+        self._soft = soft
+        self._sync = bool(getattr(submit, "_sync_mode", False))
+
+    @property
+    def not_(self) -> _Expectation:
+        return _Expectation(
+            self._actual,
+            self._submit,
+            label=self._label,
+            not_=not self._not,
+            soft=self._soft,
+        )
+
+    def __getattr__(self, name: str) -> Any:
+        if not name.startswith("to_"):
+            raise AttributeError(name)
+        wire = MATCHER_TO_WIRE.get(name)
+        if wire is None:
+            raise AttributeError(f"unknown matcher {name}")
+
+        async def _async_matcher(*args: Any) -> None:
+            subject = subject_for_matcher(self._actual, wire)
+            await self._submit(
+                subject=subject,
+                matcher=wire,
+                args=wire_args(args),
+                label=self._label,
+                not_=self._not,
+                soft=self._soft,
+            )
+
+        def _sync_matcher(*args: Any) -> None:
+            subject = subject_for_matcher(self._actual, wire)
+            self._submit(
+                subject=subject,
+                matcher=wire,
+                args=wire_args(args),
+                label=self._label,
+                not_=self._not,
+                soft=self._soft,
+            )
+
+        return _sync_matcher if self._sync else _async_matcher
+
+
+class soft:  # noqa: N801
+    """expect.soft(subject, label?) — soft assertion."""
+
+    def __init__(self, actual: Any, label: str | None = None) -> None:
+        submit = _submit.get()
+        if submit is None:
+            raise RuntimeError("expect.soft() must run inside an eval body")
+        self._inner = _Expectation(actual, submit, label=label, soft=True)
+
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self._inner, name)

reeval_python/runner.py

@@ -0,0 +1,190 @@
+"""Out-of-process runner: collect declaration trees and run eval bodies over attempt protocol v2."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import json
+import os
+import sys
+import traceback
+from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
+from typing import Any, cast
+
+from reeval_python.context import AttemptContext, SyncAttemptContext
+from reeval_python.declare import collect_eval_file, get_body
+from reeval_python.errors import ReevalAssertionError
+
+RUNNER_PROTOCOL = 2
+
+
+def sdk_version() -> str:
+    try:
+        return version("reeval-python")
+    except PackageNotFoundError:
+        return "0.0.0"
+
+
+class _Connection:
+    def __init__(self, writer: asyncio.StreamWriter) -> None:
+        self._writer = writer
+        self._pending: dict[tuple[str, str], asyncio.Future[dict[str, Any]]] = {}
+
+    def write(self, msg: dict[str, Any]) -> None:
+        self._writer.write((json.dumps(msg) + "\n").encode())
+
+    async def drain(self) -> None:
+        await self._writer.drain()
+
+    def complete_reply(self, msg: dict[str, Any]) -> None:
+        attempt = msg.get("attempt")
+        req_id = msg.get("id")
+        if attempt is None or req_id is None:
+            return
+        key = (str(attempt), str(req_id))
+        fut = self._pending.pop(key, None)
+        if fut is not None and not fut.done():
+            fut.set_result(msg)
+
+    async def request(self, attempt_id: str, req_id: str, call: dict[str, Any]) -> dict[str, Any]:
+        key = (attempt_id, req_id)
+        fut = asyncio.get_running_loop().create_future()
+        self._pending[key] = fut
+        self.write({"id": req_id, "call": call, "attempt": attempt_id})
+        await self.drain()
+        return await fut
+
+
+async def _run_body(conn: _Connection, attempt_id: str, body_id: str) -> None:
+    fn = get_body(body_id)
+    seq = 0
+
+    async def send_request(message: dict[str, Any]) -> dict[str, Any]:
+        nonlocal seq
+        seq += 1
+        req_id = str(seq)
+        return await conn.request(attempt_id, req_id, message["call"])
+
+    loop = asyncio.get_running_loop()
+    async_ctx = AttemptContext(attempt_id, send_request)
+    error: dict[str, Any] | None = None
+
+    async def run_async_body() -> None:
+        async_ctx.bind_expect()
+        await fn(async_ctx)
+
+    async def run_sync_body() -> None:
+        sync_ctx = SyncAttemptContext(async_ctx, loop)
+        sync_ctx.bind_expect()
+        await asyncio.to_thread(fn, sync_ctx)
+
+    try:
+        if inspect.iscoroutinefunction(fn):
+            await run_async_body()
+        else:
+            await run_sync_body()
+    except ReevalAssertionError as exc:
+        error = {
+            "name": "AssertionError",
+            "message": str(exc),
+            "stack": traceback.format_exc(),
+        }
+    except AssertionError as exc:
+        try:
+            await async_ctx.record_plain_assert(exc)
+        except ReevalAssertionError as recorded:
+            error = {
+                "name": "AssertionError",
+                "message": str(recorded),
+                "stack": traceback.format_exc(),
+            }
+        except AssertionError as nested:
+            error = {
+                "name": "AssertionError",
+                "message": str(nested),
+                "stack": traceback.format_exc(),
+            }
+    except Exception as exc:  # noqa: BLE001
+        error = {"name": type(exc).__name__, "message": str(exc), "stack": traceback.format_exc()}
+    conn.write({"type": "done", "attempt": attempt_id, **({"error": error} if error else {})})
+    await conn.drain()
+
+
+async def serve() -> None:
+    host_port = os.environ.get("REEVAL_RUNNER", "")
+    token = os.environ.get("REEVAL_RUNNER_TOKEN", "")
+    root = Path(os.environ.get("REEVAL_PROJECT_ROOT", os.getcwd()))
+    if ":" not in host_port or not token:
+        print("This is reeval's Python runner; run evals with reeval run.", file=sys.stderr)
+        sys.exit(2)
+
+    host, port_str = host_port.rsplit(":", 1)
+    reader, writer = await asyncio.open_connection(host, int(port_str))
+    conn = _Connection(writer)
+
+    hello = {
+        "t": "hello",
+        "protocol": RUNNER_PROTOCOL,
+        "token": token,
+        "sdk": "python",
+        "version": sdk_version(),
+        "runtime": sys.version.split()[0],
+    }
+    writer.write((json.dumps(hello) + "\n").encode())
+    await writer.drain()
+
+    tasks: set[asyncio.Task[None]] = set()
+    buf = b""
+    while True:
+        chunk = await reader.read(65536)
+        if chunk == b"":
+            break
+        buf += chunk
+        while True:
+            newline = buf.find(b"\n")
+            if newline < 0:
+                break
+            line = buf[:newline]
+            buf = buf[newline + 1 :]
+            if not line.strip():
+                continue
+            msg = cast(dict[str, Any], json.loads(line.decode()))
+            if msg.get("t") == "collect":
+                rel = str(msg["file"])
+                try:
+                    tree = collect_eval_file(root / rel)
+                    conn.write({"t": "declared", "file": rel, "tree": tree})
+                except Exception as exc:  # noqa: BLE001
+                    conn.write(
+                        {
+                            "t": "load-error",
+                            "error": {
+                                "name": type(exc).__name__,
+                                "message": str(exc),
+                                "stack": traceback.format_exc(),
+                            },
+                        }
+                    )
+                await conn.drain()
+            elif msg.get("type") == "start":
+                attempt_id = str(msg["attempt"]["id"])
+                body_id = str(msg["body"])
+                task = asyncio.create_task(_run_body(conn, attempt_id, body_id))
+                tasks.add(task)
+                task.add_done_callback(tasks.discard)
+            elif msg.get("type") == "abort":
+                for task in list(tasks):
+                    task.cancel()
+            elif "id" in msg and "ok" in msg:
+                conn.complete_reply(msg)
+
+    writer.close()
+
+
+def main() -> None:
+    asyncio.run(serve())
+
+
+if __name__ == "__main__":
+    main()

reeval_python/types.py

@@ -0,0 +1,97 @@
+"""Typed results for attempt calls (public, not wire-shaped)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Literal, cast
+
+
+@dataclass(frozen=True, slots=True)
+class ExecResult:
+    command: str
+    exit_code: int
+    stdout: str
+    stderr: str
+
+    @property
+    def ok(self) -> bool:
+        return self.exit_code == 0
+
+    @classmethod
+    def from_wire(cls, command: str, data: dict[str, Any]) -> ExecResult:
+        return cls(
+            command=command,
+            exit_code=int(data.get("exitCode", data.get("exit_code", 1))),
+            stdout=str(data.get("stdout") or ""),
+            stderr=str(data.get("stderr") or ""),
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class BackgroundHandle:
+    id: str
+
+    @classmethod
+    def from_wire(cls, data: dict[str, Any]) -> BackgroundHandle:
+        return cls(id=str(data.get("handle") or data.get("id") or ""))
+
+
+@dataclass(frozen=True, slots=True)
+class Turn:
+    index: int
+    raw: dict[str, Any]
+
+    @property
+    def prompt(self) -> str:
+        return str(self.raw.get("prompt") or "")
+
+    @property
+    def text(self) -> str:
+        return str(self.raw.get("text") or "")
+
+    @property
+    def status(self) -> str:
+        return str(self.raw.get("status") or "")
+
+    @property
+    def diff(self) -> dict[str, Any] | None:
+        diff = self.raw.get("diff")
+        if isinstance(diff, dict):
+            return cast(dict[str, Any], diff)
+        return None
+
+
+VerdictKind = Literal["pass", "fail", "error"]
+
+
+@dataclass(frozen=True, slots=True)
+class Verdict:
+    verdict: VerdictKind
+    score: float
+    threshold: float
+    question: str
+    model: str
+    raw: dict[str, Any]
+
+    @property
+    def passed(self) -> bool:
+        return self.verdict == "pass"
+
+    @property
+    def reason(self) -> str | None:
+        for key in ("reason", "explanation", "rationale"):
+            value = self.raw.get(key)
+            if isinstance(value, str) and value:
+                return value
+        return None
+
+    @classmethod
+    def from_wire(cls, data: dict[str, Any]) -> Verdict:
+        return cls(
+            verdict=data.get("verdict", "fail"),  # type: ignore[arg-type]
+            score=float(data.get("score", 0)),
+            threshold=float(data.get("threshold", 0)),
+            question=str(data.get("question") or ""),
+            model=str(data.get("model") or ""),
+            raw=dict(data),
+        )

reeval_python/wire.py

@@ -0,0 +1,81 @@
+"""Map public Python values to attempt-protocol wire shapes (internal)."""
+
+from __future__ import annotations
+
+import re
+from typing import Any, cast
+
+from reeval_python.types import ExecResult, Turn, Verdict
+
+# Python snake_case matcher method -> wire matcher name (Vitest + reeval table).
+MATCHER_TO_WIRE: dict[str, str] = {
+    "to_succeed": "toSucceed",
+    "to_pass": "toPass",
+    "to_equal": "toEqual",
+    "to_contain": "toContain",
+    "to_be_truthy": "toBeTruthy",
+    "to_be": "toBe",
+    "to_be_null": "toBeNull",
+    "to_have_file": "toHaveFile",
+    "to_have_edited": "toHaveEdited",
+    "to_have_run": "toHaveRun",
+    "to_have_searched": "toHaveSearched",
+    "to_have_fetched": "toHaveFetched",
+    "to_have_called_skill": "toHaveCalledSkill",
+    "to_have_called_mcp": "toHaveCalledMcp",
+    "to_have_called_tool": "toHaveCalledTool",
+    "to_have_changed": "toHaveChanged",
+}
+
+# Judge / evaluate options: snake_case kwargs -> camelCase keys on the wire.
+OPTION_KEY_TO_WIRE: dict[str, str] = {
+    "max_retries": "maxRetries",
+    "max_tokens": "maxTokens",
+    "read_only": "readOnly",
+}
+
+
+def options_to_wire(options: dict[str, Any] | None) -> dict[str, Any] | None:
+    if not options:
+        return None
+    out: dict[str, Any] = {}
+    for key, value in options.items():
+        wire_key = OPTION_KEY_TO_WIRE.get(key, key)
+        out[wire_key] = value
+    return out
+
+
+def _pattern_arg(value: re.Pattern[str]) -> Any:
+    return {"regexp": value.pattern, "flags": value.flags and int(value.flags) or None}
+
+
+def wire_args(args: tuple[Any, ...]) -> list[Any]:
+    out: list[Any] = []
+    for a in args:
+        if isinstance(a, re.Pattern):
+            out.append(_pattern_arg(cast(re.Pattern[str], a)))
+        else:
+            out.append(a)
+    return out
+
+
+def subject_for_matcher(actual: Any, matcher: str) -> dict[str, Any]:
+    if isinstance(actual, ExecResult) and matcher == "toSucceed":
+        return {"exec": actual.command}
+    if isinstance(actual, ExecResult):
+        return {
+            "value": {
+                "exitCode": actual.exit_code,
+                "stdout": actual.stdout,
+                "stderr": actual.stderr,
+            }
+        }
+    if isinstance(actual, Verdict):
+        return {"value": actual.raw}
+    if isinstance(actual, Turn):
+        return {"turn": actual.index}
+    if isinstance(actual, dict):
+        wire_keys = ("exec", "turn", "attempt", "readFile", "value")
+        if any(k in actual for k in wire_keys):
+            return cast(dict[str, Any], actual)
+    return {"value": actual}

reeval_python-0.10.0.dist-info/METADATA

@@ -0,0 +1,85 @@
+Metadata-Version: 2.5
+Name: reeval-python
+Version: 0.10.0
+Summary: Python authoring SDK for reeval (*.eval.py eval files)
+Project-URL: Homepage, https://reeval.ai
+Project-URL: Repository, https://github.com/invertase/reeval
+Project-URL: Documentation, https://github.com/invertase/reeval/tree/main/packages/reeval-python#readme
+Project-URL: Issues, https://github.com/invertase/reeval/issues
+Author: Invertase
+License-Expression: Apache-2.0
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+
+# reeval-python
+
+Python authoring SDK for [reeval](https://github.com/invertase/reeval). Write `*.eval.py` files,
+declare evals with the same options as TypeScript, and run them through reeval's language runner
+(loopback TCP, attempt protocol v2).
+
+Import as `reeval_python` (`from reeval_python import evaluate`). The PyPI distribution is
+**reeval-python** (the name `reeval` is already taken on PyPI by another project).
+
+With reeval installed, default discovery includes `evals/**/*.eval.py`. When your project has
+`pyproject.toml` or `uv.lock` and [uv](https://docs.astral.sh/uv/) is on `PATH`, reeval starts the
+runner with `uv run --project <root> python -m reeval_python`. Otherwise it uses `<root>/.venv/bin/python`
+when present, then `python3`. Set `REEVAL_PYTHON` to override the interpreter.
+
+## Install
+
+```bash
+uv add reeval-python
+# or: pip install reeval-python
+```
+
+Monorepo development:
+
+```bash
+cd packages/reeval-python
+uv sync --dev
+uv run pytest
+uv run ruff check .
+uv run pyright
+```
+
+Declare trees are checked against `packages/reeval/test/conformance/declare/` goldens. From the
+reeval package, `test/python-sdk.test.ts` runs the same suite in CI.
+
+## Writing evals
+
+```python
+from reeval_python import evaluate, expect
+
+
+@evaluate("lands")
+def lands(c):
+    c.send("Create greet.py that prints hi")
+    assert c.exec("test -f greet.py").ok
+    expect(c.judge("Does greet.py print hi?")).to_pass()
+```
+
+Plain `assert` is first-class: failures become hard checks in the run record (same shape as
+TypeScript). Use typed results (`ExecResult.ok`, `Verdict.passed`, `Turn.text`, …) and `c.file(path)`
+for quick checks. Keep `expect` for reeval matchers (`to_have_file`, `to_succeed`, soft checks,
+labels) — Playwright-style.
+
+Use snake_case on the context (`read_file`, `write_file`, `file`) and on matchers (`to_succeed`,
+`to_pass`, `to_equal`, `to_contain`, …). Matcher failures raise `ReevalAssertionError`. Async bodies use
+`async def`, `await c.send(...)`, and `await expect(await c.exec(...)).to_succeed()`; sync bodies
+use plain `def` and block on the runner loop.
+
+## Examples
+
+- Feature smoke: `packages/reeval/test/features/python-sdk/hello/` (uv project with a project dependency)
+- Plan case: `packages/reeval/test/plan/python/`
+- Contributor guide: `.agents/skills/add-a-language/SKILL.md` in the reeval repo

reeval_python-0.10.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+reeval_python/__init__.py,sha256=Mzv45KSuKaWmcKO7yZ-jzk3DcnYDrvzavbxZzJsf6AM,655
+reeval_python/__main__.py,sha256=0AezY7FdEV8RqwkKhR0twWdvhaiJkt02ps7U1TEujzs,77
+reeval_python/assertion.py,sha256=AnRa_8KAWBw-RHnZFG2NuO2qimLvGS3ZuxsBq79Oqv8,5077
+reeval_python/context.py,sha256=i0__DQIZ0KBCqRgbhjjZ-VZGd_9O5NBWIPYRQT2grQA,5818
+reeval_python/declare.py,sha256=7ceRmccd2GrvX_yVQfqfzU5o3W4v3_SjEwQvkEgE8OY,7090
+reeval_python/errors.py,sha256=q6xm8zwEzS4LewZf1h5oKYfnXHc2SvuKghew0ryia4A,208
+reeval_python/expect.py,sha256=giZTP3h2xavXow2GOe4N9QaxLXG1wV6IeblA0Otaym0,2777
+reeval_python/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reeval_python/runner.py,sha256=-9-QSh5cPr-xCbYLEoa2D4CtVWT7z1CCam4mnMeEx4Q,6230
+reeval_python/types.py,sha256=qcZA1u0c7uxu4hI5tATXylH0ru0_V65QuWja7B3UeXI,2453
+reeval_python/wire.py,sha256=ECX1t8ETFigEOFHGhXmSn7YnYYU-27tNCJMyh-eUJG4,2564
+reeval_python-0.10.0.dist-info/METADATA,sha256=vVKz67mwpgt_x3KKjFFrSZY4KF7m6zUFfXBd0ldArTA,3308
+reeval_python-0.10.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+reeval_python-0.10.0.dist-info/licenses/LICENSE,sha256=ksfhGdGZnBIw2o77d0o71LHzgK-bRGUx1qS6nzYMjK4,11348
+reeval_python-0.10.0.dist-info/RECORD,,

reeval_python-0.10.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any