ancha 0.1.0__py3-none-any.whl

ancha/__init__.py

@@ -0,0 +1,19 @@
+"""Ancha for Python: wrap your agent's own tools so a test made from a production trace answers them as production did
+(`ancha.tool`, `ancha.wrap_tools`, `ancha.entry`; see ancha/tools.py), and pytest for Ancha: a fresh simulated world for each test, your agent run against it, and checks on what people learned
+and what the tools hold.
+
+    import ancha, pytest
+
+    @pytest.mark.ancha(test="ancha/tests/relay-export-bug.yaml")
+    @ancha.attempts(10, min_pass_rate=0.9)
+    def test_relay(world):
+        world.start("npm start")
+        world.wait_until_quiet()
+        assert world.people["ben"].knows("laura.beta_on")
+        assert world.slack.messages(to="laura", in_thread_of="$laura_report", from_="$slack_bot")
+        assert not world.people["laura"].knows("ben.root_cause")
+"""
+from .tools import ToolError, call_tool, entry, tool, wrap_tools
+from .world import AnchaError, CheckReport, People, Person, Slack, World, attempts
+
+__all__ = ["AnchaError", "CheckReport", "People", "Person", "Slack", "ToolError", "World", "attempts", "call_tool", "entry", "tool", "wrap_tools"]

ancha/plugin.py

@@ -0,0 +1,85 @@
+"""The pytest plugin: the `world` fixture, configured per test with `@pytest.mark.ancha(...)`."""
+from __future__ import annotations
+
+import os
+from typing import Any, Iterator
+
+import pytest
+
+from .recording import Recorder, current, error_code
+from .world import World
+
+OPTIONS = ("test", "scene", "world", "on", "seed", "ttl")
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    parser.getgroup("ancha").addoption("--on", choices=("local", "cloud"), default=None,
+        help="where Ancha worlds run; overrides the ancha marker (pytest and the agent stay local)")
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    config.addinivalue_line("markers", "ancha(test=None, scene=None, world=None, on=None, seed=None, ttl=None, cwd=None): "
+                                       "what the `world` fixture starts: a test or scene file, or a saved world; where (local or cloud)")
+    config._ancha_recording_failures = []
+    config._ancha_recorded = 0
+
+
+@pytest.hookimpl(hookwrapper=True, tryfirst=True)
+def pytest_runtest_protocol(item: pytest.Item, nextitem: pytest.Item | None):
+    marker = item.get_closest_marker("ancha")
+    recorder = Recorder(item.nodeid, str((marker.kwargs.get("cwd") if marker else None) or os.getcwd()))
+    item._ancha_recorder = recorder
+    token = current.set(recorder)
+    try:
+        yield
+    finally:
+        try:
+            recorder.finish()
+            recorder.write()
+            item.config._ancha_recorded += 1
+            if recorder.infrastructure_failed:
+                item.config._ancha_recording_failures.append((recorder.payload["runId"], "INFRASTRUCTURE_ERROR"))
+        except Exception as error:
+            # Recording cannot turn a failing test into an internal pytest error or expose its exception text.
+            item.config._ancha_recording_failures.append((recorder.payload["runId"], error_code(error)))
+        finally:
+            current.reset(token)
+
+
+@pytest.hookimpl(hookwrapper=True, tryfirst=True)
+def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo):
+    outcome = yield
+    recorder = getattr(item, "_ancha_recorder", None)
+    if recorder is not None:
+        recorder.report(outcome.get_result(), call.excinfo.value if call.excinfo is not None else None)
+
+
+def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
+    if session.config._ancha_recording_failures and session.exitstatus == pytest.ExitCode.OK:
+        session.exitstatus = pytest.ExitCode.TESTS_FAILED
+
+
+def pytest_terminal_summary(terminalreporter: Any) -> None:
+    config = terminalreporter.config
+    if config._ancha_recorded:
+        terminalreporter.write_line(f"Ancha recorded {config._ancha_recorded} pytest run(s) locally in .ancha/runs.")
+    for run_id, code in config._ancha_recording_failures:
+        terminalreporter.write_line(f"Ancha run {run_id}: infrastructure or recording failed ({code}); pytest session failed.", red=True)
+
+
+@pytest.fixture
+def world(request: pytest.FixtureRequest) -> Iterator[World]:
+    """A fresh world for this test (`ancha world up`), removed afterwards. Without a marker: the one scene ancha.yaml names."""
+    marker = request.node.get_closest_marker("ancha")
+    options = dict(marker.kwargs) if marker else {}
+    unknown = set(options) - {*OPTIONS, "cwd"}
+    if unknown:
+        raise pytest.UsageError(f"@pytest.mark.ancha takes {', '.join(OPTIONS)} and cwd, not {', '.join(sorted(unknown))}")
+    if request.config.getoption("--on") is not None:
+        options["on"] = request.config.getoption("--on")
+    args = [arg for key in OPTIONS if options.get(key) is not None for arg in (f"--{key}", str(options[key]))]
+    live = World(args, cwd=str(options.get("cwd") or os.getcwd()))
+    try:
+        yield live
+    finally:
+        live.down()

ancha/recording.py

@@ -0,0 +1,173 @@
+"""Private per-test observations. The CLI validates and stores the finished run."""
+from __future__ import annotations
+
+import contextvars
+import datetime as dt
+import hashlib
+import json
+import os
+import re
+import tempfile
+import uuid
+from typing import Any
+
+
+current: contextvars.ContextVar[Recorder | None] = contextvars.ContextVar("ancha_pytest_recorder", default=None)
+_SECRET_NAME = re.compile(r"secret|token|password|passwd|api.?key|private.?key|credential|cookie|authorization", re.I)
+_CODE = re.compile(r"^[A-Za-z_][A-Za-z0-9_.:-]{0,127}$")
+
+
+def now() -> str:
+    return dt.datetime.now(dt.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
+def error_code(error: BaseException | None) -> str:
+    code = getattr(error, "code", None) or (type(error).__name__ if error is not None else "PYTEST_ERROR")
+    return code if isinstance(code, str) and _CODE.fullmatch(code) else "PYTEST_ERROR"
+
+
+def node_id(value: str) -> str:
+    # Parameter IDs can include credentials, payloads or user data. Keep test identity without their contents.
+    if "[" not in value:
+        return value
+    plain, parameters = value.split("[", 1)
+    return f"{plain}[case-{hashlib.sha256(parameters.encode()).hexdigest()[:16]}]"
+
+
+class Recorder:
+    def __init__(self, node: str, cwd: str) -> None:
+        self.cwd = cwd
+        self.secrets: set[str] = set()
+        self.add_secrets(os.environ)
+        self.payload: dict[str, Any] = {
+            "version": 1, "runId": f"run_pytest_{uuid.uuid4().hex}", "nodeId": node_id(node),
+            "startedAt": now(), "finishedAt": now(), "outcome": "passed", "minPassRate": 1.0,
+            "phases": [], "attempts": [], "requestedTrials": 1,
+        }
+        self.worlds: list[Any] = []
+        self.repeated = False
+        self.infrastructure_failed = False
+        self.begin_attempt()
+
+    @property
+    def attempt(self) -> dict[str, Any]:
+        return self.payload["attempts"][-1]
+
+    def add_secrets(self, values: Any) -> None:
+        self.secrets.update(value for key, value in values.items()
+                            if _SECRET_NAME.search(key) and isinstance(value, str) and value)
+
+    def begin_attempt(self) -> None:
+        self.payload["attempts"].append({"number": len(self.payload["attempts"]) + 1,
+            "outcome": "passed", "startedAt": now(), "finishedAt": now(), "worlds": [], "errors": []})
+
+    def add_world(self, world: Any) -> dict[str, Any]:
+        value: dict[str, Any] = {"envId": world.id, "workspace": world.workspace, "checks": [], "errors": []}
+        self.worlds.append(world)
+        self.attempt["worlds"].append(value)
+        self.add_secrets(world.env)
+        return value
+
+    def infrastructure_error(self, phase: str, error: BaseException, world: dict | None = None) -> None:
+        issue = {"phase": phase, "code": error_code(error)}
+        if world is not None:
+            world["errors"].append(issue)
+        else:
+            self.attempt["errors"].append(issue)
+        self.attempt["outcome"] = "invalid"
+        self.infrastructure_failed = True
+
+    def close_worlds(self) -> BaseException | None:
+        first: BaseException | None = None
+        # World.down owns error recording and is idempotent, including after a failed removal.
+        for world in self.worlds:
+            try:
+                world.down()
+            except BaseException as error:
+                first = first or error
+        return first
+
+    def finish_attempt(self, outcome: str) -> BaseException | None:
+        if self.attempt["outcome"] != "invalid":
+            self.attempt["outcome"] = outcome
+        error = self.close_worlds()
+        self.attempt["finishedAt"] = now()
+        return error
+
+    def report(self, report: Any, error: BaseException | None) -> None:
+        phase: dict[str, Any] = {"when": report.when, "outcome": report.outcome,
+                                "durationMs": max(0, report.duration * 1000)}
+        if report.failed:
+            phase["errorCode"] = error_code(error)
+        self.payload["phases"].append(phase)
+        if report.skipped:
+            outcome = "xfailed" if getattr(report, "wasxfail", None) is not None else "skipped"
+            if self.payload["outcome"] not in ("failed", "error"):
+                self.payload["outcome"] = outcome
+            if not self.repeated:
+                self.attempt["outcome"] = outcome
+        elif report.failed:
+            # A failed assertion/pytest.fail is a test verdict. Other exceptions and fixture errors are invalid.
+            import pytest
+            assertion = isinstance(error, (AssertionError, pytest.fail.Exception))
+            strict_xpass = error is None and report.when == "call" and str(report.longrepr).startswith("[XPASS(strict)]")
+            if report.when != "call" or (not assertion and not strict_xpass):
+                stage = getattr(error, "_ancha_phase", report.when)
+                self.infrastructure_error(stage, error or RuntimeError())
+                self.payload["outcome"] = "error"
+            elif self.payload["outcome"] != "error":
+                self.payload["outcome"] = "failed"
+                if not self.repeated and not strict_xpass:
+                    self.attempt["outcome"] = "failed"
+        elif report.when == "call" and getattr(report, "wasxfail", None) is not None:
+            self.payload["outcome"] = "xpassed"
+            if not self.repeated:
+                self.attempt["outcome"] = "xpassed"
+
+    def _redact(self, value: Any) -> Any:
+        if isinstance(value, str):
+            for secret in sorted(self.secrets, key=len, reverse=True):
+                if len(secret) >= 4:
+                    value = value.replace(secret, "[redacted]")
+                elif value == secret:
+                    value = "[redacted]"
+            return value
+        if isinstance(value, list):
+            return [self._redact(item) for item in value]
+        if isinstance(value, dict):
+            return {self._redact(key): self._redact(item) for key, item in value.items()}
+        return value
+
+    def safe_payload(self) -> dict:
+        # Keep protocol enums/timestamps/IDs intact even if a credential happens to equal one of their values.
+        def code(value: str) -> str:
+            return value if self._redact(value) == value else "REDACTED_ERROR"
+
+        def errors(values: list[dict]) -> list[dict]:
+            return [{**value, "code": code(value["code"])} for value in values]
+
+        def world(value: dict) -> dict:
+            return {**{key: self._redact(item) for key, item in value.items() if key != "errors"}, "errors": errors(value["errors"])}
+
+        return {**self.payload, "nodeId": self._redact(self.payload["nodeId"]),
+                "phases": [{**phase, **({"errorCode": code(phase["errorCode"])} if "errorCode" in phase else {})}
+                           for phase in self.payload["phases"]],
+                "attempts": [{**attempt, "worlds": [world(value) for value in attempt["worlds"]], "errors": errors(attempt["errors"])}
+                             for attempt in self.payload["attempts"]]}
+
+    def finish(self) -> None:
+        self.close_worlds()
+        if self.infrastructure_failed:
+            self.payload["outcome"] = "error"
+        self.attempt["finishedAt"] = now()
+        self.payload["finishedAt"] = now()
+
+    def write(self) -> dict:
+        from .world import run_cli
+        fd, path = tempfile.mkstemp(prefix="ancha-pytest-", suffix=".json")
+        try:
+            with os.fdopen(fd, "w", encoding="utf-8") as file:
+                json.dump(self.safe_payload(), file, ensure_ascii=False)
+            return run_cli(["world", "record", "--file", path], self.cwd)
+        finally:
+            os.unlink(path)

ancha/tools.py

@@ -0,0 +1,232 @@
+"""Test an agent's own tools against production's recorded calls (TRACE_PLAN_V2 §3).
+
+    import ancha
+
+    @function_tool          # the framework's decorator, if any, outside
+    @ancha.tool             # innermost, on the function itself
+    def lookup_order(order_id: str) -> Order: ...
+
+    tools = ancha.wrap_tools([lookup_order, refund_order])     # or wrap tools the framework already built
+
+    ancha.entry(lambda input: run_agent(input["user"], input["messages"]))    # the command ancha.yaml runs
+
+In production nothing changes: without ANCHA_TOOLS_URL a wrapped tool is the original function. In a test, `ancha run`
+sets ANCHA_TOOLS_URL and ANCHA_TOOLS_TOKEN, and each call goes to Ancha, which answers as production did: the recorded
+result (restored to the function's pydantic return type), or the recorded error raised as `ancha.ToolError`.
+"""
+from __future__ import annotations
+
+import asyncio
+import dataclasses
+import functools
+import inspect
+import json
+import os
+import sys
+import typing
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Any, Callable
+
+__all__ = ["ToolError", "call_tool", "entry", "tool", "wrap_tools"]
+
+# Parameters frameworks pass that the model never sees (the OpenAI Agents SDK's context, LangChain's config).
+_CONTEXT = {"self", "cls", "ctx", "context", "config", "run_manager", "callbacks", "wrapper"}
+_wrapped: set[str] = set()
+
+
+class ToolError(Exception):
+    """Ancha answered with the error production's tool raised, or with no recorded answer."""
+
+    def __init__(self, message: str, source: str | None = None):
+        super().__init__(message)
+        self.source = source
+
+
+def _service() -> tuple[str, str] | None:
+    url = os.environ.get("ANCHA_TOOLS_URL")
+    return (url.rstrip("/"), os.environ.get("ANCHA_TOOLS_TOKEN", "")) if url else None
+
+
+def _post(url: str, token: str, body: Any) -> tuple[int, Any]:
+    request = urllib.request.Request(url, data=json.dumps(body, default=_jsonable).encode(), method="POST",
+                                     headers={"content-type": "application/json", "authorization": f"Bearer {token}"})
+    try:
+        with urllib.request.urlopen(request, timeout=60) as res:
+            return res.status, json.loads(res.read() or b"{}")
+    except urllib.error.HTTPError as error:
+        try:
+            return error.code, json.loads(error.read() or b"{}")
+        except ValueError:
+            return error.code, {}
+
+
+def _jsonable(value: Any) -> Any:
+    if hasattr(value, "model_dump"):
+        return value.model_dump(mode="json")
+    if dataclasses.is_dataclass(value) and not isinstance(value, type):
+        return dataclasses.asdict(value)
+    if isinstance(value, (set, tuple)):
+        return list(value)
+    return str(value)
+
+
+def _register(names: list[str]) -> None:
+    """Tells Ancha which tools this process wraps, so a test stops before the agent calls an unwrapped one for real."""
+    _wrapped.update(names)
+    service = _service()
+    if service:
+        try:
+            _post(f"{service[0]}/register", service[1], {"tools": names})
+        except OSError:
+            pass  # entry() checks again, and stops the test if a tool is missing
+
+
+_PASSTHROUGH = object()
+
+
+def call_tool(name: str, args: Any) -> Any:
+    """One call to a tool as Ancha answers it in a test."""
+    answer = _ask(name, args)
+    if answer is _PASSTHROUGH:
+        raise ToolError(f"production never made this call to {name}; the tool itself runs against the twin in a test", "twin")
+    return answer
+
+
+def _ask(name: str, args: Any) -> Any:
+    """Ancha's answer: production's result, its error raised, or _PASSTHROUGH (run the tool itself against its twin)."""
+    service = _service()
+    if not service:
+        raise RuntimeError("call_tool runs inside an Ancha test: ANCHA_TOOLS_URL is not set")
+    status, body = _post(f"{service[0]}/tools/{urllib.parse.quote(name, safe='')}", service[1], {"args": args if args is not None else {}})
+    if status == 401:
+        raise RuntimeError("Ancha refused the tool call: ANCHA_TOOLS_TOKEN does not match this test")
+    if body.get("stop"):  # Ancha stopped the test: end the agent before it calls an unwrapped tool for real
+        print(f"ancha: {body.get('error')}", file=sys.stderr, flush=True)
+        os._exit(2)
+    if status >= 400:
+        raise RuntimeError(f"Ancha's tool service answered {status}: {body.get('error', '')}")
+    if "error" in body:
+        raise ToolError(body["error"], body.get("source"))
+    return _PASSTHROUGH if body.get("passthrough") else body.get("result")
+
+
+def _arguments(fn: Callable[..., Any], args: tuple, kwargs: dict) -> dict:
+    """The arguments as the model sent them: the call bound to the signature, without what the framework adds."""
+    try:
+        bound = inspect.signature(fn).bind_partial(*args, **kwargs)
+    except (TypeError, ValueError):
+        return {"args": list(args), **kwargs}
+    out = {}
+    for name, value in bound.arguments.items():
+        annotation = str(inspect.signature(fn).parameters[name].annotation)
+        if name in _CONTEXT or "RunContext" in annotation or "ToolContext" in annotation:
+            continue
+        param = inspect.signature(fn).parameters[name]
+        if param.kind is inspect.Parameter.VAR_KEYWORD:
+            out.update(value)
+        else:
+            out[name] = value
+    return json.loads(json.dumps(out, default=_jsonable))
+
+
+def _restore(fn: Callable[..., Any], result: Any) -> Any:
+    """The recorded result as the function's return type: a pydantic model, or a list of them, from the JSON."""
+    try:
+        annotation = typing.get_type_hints(fn).get("return")
+    except Exception:  # forward references the module can't resolve: keep the JSON
+        return result
+    if annotation is not None and hasattr(annotation, "model_validate") and isinstance(result, dict):
+        return annotation.model_validate(result)
+    origin, inner = typing.get_origin(annotation), typing.get_args(annotation)
+    if origin in (list, typing.List) and inner and hasattr(inner[0], "model_validate") and isinstance(result, list):
+        return [inner[0].model_validate(item) for item in result]
+    return result
+
+
+def tool(fn: Callable[..., Any] | None = None, *, name: str | None = None):
+    """Wraps one function as a tool named `name` (the function's name by default, the name the model sees)."""
+
+    def decorate(f: Callable[..., Any]) -> Callable[..., Any]:
+        tool_name = name or f.__name__
+        _register([tool_name])
+        if inspect.iscoroutinefunction(f):
+            @functools.wraps(f)
+            async def async_tool(*args: Any, **kwargs: Any) -> Any:
+                if not _service():
+                    return await f(*args, **kwargs)
+                answer = await asyncio.to_thread(_ask, tool_name, _arguments(f, args, kwargs))
+                return await f(*args, **kwargs) if answer is _PASSTHROUGH else _restore(f, answer)
+            return async_tool
+
+        @functools.wraps(f)
+        def sync_tool(*args: Any, **kwargs: Any) -> Any:
+            if not _service():
+                return f(*args, **kwargs)
+            answer = _ask(tool_name, _arguments(f, args, kwargs))
+            return f(*args, **kwargs) if answer is _PASSTHROUGH else _restore(f, answer)
+        return sync_tool
+
+    return decorate(fn) if fn is not None else decorate
+
+
+def wrap_tools(tools: Any, names: dict[str, str] | None = None) -> Any:
+    """Every tool in a list or dict, wrapped: functions, LangChain tools (their func and coroutine), and OpenAI Agents
+    SDK function tools (their on_invoke_tool). `names` maps a tool to another name when the trace knows it by one."""
+    names = names or {}
+    items = list(tools.items()) if isinstance(tools, dict) else [(None, t) for t in tools]
+    out = []
+    for key, value in items:
+        label = getattr(value, "name", None) or key or getattr(value, "__name__", None)
+        tool_name = names.get(label, label)
+        if hasattr(value, "on_invoke_tool"):          # OpenAI Agents SDK FunctionTool
+            original = value.on_invoke_tool
+
+            async def invoke(ctx: Any, raw: str, _original=original, _name=tool_name) -> Any:
+                if not _service():
+                    return await _original(ctx, raw)
+                answer = await asyncio.to_thread(_ask, _name, json.loads(raw or "{}"))
+                return await _original(ctx, raw) if answer is _PASSTHROUGH else answer
+            value.on_invoke_tool = invoke
+            _register([tool_name])
+            out.append(value)
+        elif hasattr(value, "func") or hasattr(value, "coroutine"):   # LangChain StructuredTool / Tool
+            if getattr(value, "func", None) is not None:
+                value.func = tool(value.func, name=tool_name)
+            if getattr(value, "coroutine", None) is not None:
+                value.coroutine = tool(value.coroutine, name=tool_name)
+            out.append(value)
+        elif callable(value):
+            out.append(tool(value, name=tool_name))
+        else:
+            raise TypeError(f"wrap_tools: {label!r} is neither a function nor a tool object")
+    return dict(zip((k for k, _ in items), out)) if isinstance(tools, dict) else out
+
+
+def entry(run: Callable[[dict], Any]) -> Any:
+    """The command a test runs: tells Ancha every wrapped tool (a test stops here if the trace holds one the agent does
+    not wrap), calls `run` with the invocation's input (ANCHA_INPUT_JSON: the conversation so far and the user, plus
+    `task`), and hands the answer back to Ancha. Outside a test it runs `run` with ANCHA_TASK, or with nothing."""
+    service = _service()
+    if service:
+        status, body = _post(f"{service[0]}/register", service[1], {"tools": sorted(_wrapped), "complete": True})
+        if status == 409:
+            print(f"ancha: {body.get('error')}", file=sys.stderr)
+            sys.exit(2)
+    task_input = {"task": os.environ.get("ANCHA_TASK", ""), **json.loads(os.environ.get("ANCHA_INPUT_JSON") or "{}")}
+    output = run(task_input)
+    if inspect.isawaitable(output):
+        output = asyncio.run(output) if not _running_loop() else output
+    done = os.environ.get("ANCHA_DONE_URL")
+    if done:
+        _post(done, os.environ.get("ANCHA_TRIAL_TOKEN", ""), {"output": output})
+    return output
+
+
+def _running_loop() -> bool:
+    try:
+        asyncio.get_running_loop()
+        return True
+    except RuntimeError:
+        return False

ancha/world.py

@@ -0,0 +1,309 @@
+"""A live Ancha world for a test, driven through the `ancha` CLI's lifecycle commands (build plan §9.4b).
+
+`ancha world up --json` starts the world and prints the agent's variables; the test runs its agent with them; `ancha check`
+grades the world; `ancha world down` removes it. `ANCHA_CLI` names the command (default `ancha`, e.g.
+`npx -y @ancha-labs/cli`).
+"""
+from __future__ import annotations
+
+import functools
+import json
+import os
+import shlex
+import subprocess
+import tempfile
+from typing import Any, Callable, Iterable
+
+from .recording import current, error_code
+
+
+class AnchaError(Exception):
+    """An error the CLI reported, with its stable code and the hint that names the next step."""
+
+    def __init__(self, code: str, message: str, hint: str | None = None) -> None:
+        super().__init__(f"{code}: {message}" + (f" (hint: {hint})" if hint else ""))
+        self.code, self.message, self.hint = code, message, hint
+
+
+def _cli() -> list[str]:
+    return shlex.split(os.environ.get("ANCHA_CLI", "ancha"))
+
+
+def run_cli(args: Iterable[str], cwd: str, ok: tuple[int, ...] = (0,)) -> Any:
+    """Runs one `ancha … --json` command and returns its JSON result; an error result raises AnchaError."""
+    args = list(args)
+    proc = subprocess.run([*_cli(), *args, "--json"], cwd=cwd, capture_output=True, text=True)
+    value: Any = None
+    for line in reversed([line for line in proc.stdout.splitlines() if line.strip()]):
+        try:
+            value = json.loads(line)
+            break
+        except json.JSONDecodeError:
+            continue
+    if proc.returncode not in ok:
+        if isinstance(value, dict) and isinstance(value.get("error"), dict):
+            error = value["error"]
+            raise AnchaError(error.get("code", "INTERNAL"), error.get("message", ""), error.get("hint"))
+        raise AnchaError("INTERNAL", f"ancha {' '.join(args[:2])} exited {proc.returncode}: {proc.stderr.strip()[-500:]}")
+    return value
+
+
+class CheckReport:
+    """What `ancha check` said: the verdict, every check with expected and actual, and the failures. Truthy when it passed."""
+
+    def __init__(self, value: dict) -> None:
+        self.verdict: str = value.get("verdict", "fail")
+        self.checks: list[dict] = value.get("checks", [])
+        self.failures: list[dict] = value.get("failures", [])
+
+    @property
+    def passed(self) -> bool:
+        return self.verdict == "pass"
+
+    def __bool__(self) -> bool:
+        return self.passed
+
+    def __repr__(self) -> str:
+        if self.passed:
+            return f"<CheckReport pass: {len(self.checks)} checks>"
+        return "<CheckReport fail: " + "; ".join(f"{f['check']}: expected {f.get('expected')}; got {f.get('actual')}" for f in self.failures) + ">"
+
+
+class Person:
+    """A person in the world, by ref: what they know and what they decided, read from the engine's records."""
+
+    def __init__(self, world: World, ref: str) -> None:
+        self._world, self.ref = world, ref
+
+    def knows(self, *facts: str) -> bool:
+        """The person knows every one of these facts (`<owner>.<fact id>`), e.g. people["ben"].knows("laura.beta_on")."""
+        return self._world.holds({"knows": [self.ref, *facts]})
+
+    told = knows
+
+    def decided(self, decision: str, option: str | None = None) -> bool:
+        """The person made this decision, with this option when given."""
+        return self._world.holds({"decided": [self.ref, decision, *([option] if option is not None else [])]})
+
+
+class People:
+    def __init__(self, world: World) -> None:
+        self._world = world
+
+    def __getitem__(self, ref: str) -> Person:
+        return Person(self._world, ref)
+
+
+class Slack:
+    """Messages written during the attempt (the seed never counts), by the selectors of a `message` check."""
+
+    def __init__(self, world: World) -> None:
+        self._world = world
+
+    def messages(self, *, to: str | None = None, in_: str | None = None, in_thread_of: str | None = None, from_: str | None = None,
+                 text_contains: str | list[str] | None = None, fact: str | None = None, count_gte: int | None = None) -> bool:
+        """At least one (or count_gte) matching message, e.g. slack.messages(to="laura", in_thread_of="$laura_report", from_="$slack_bot")."""
+        selector = {key: value for key, value in {"to": to, "in": in_, "in_thread_of": in_thread_of, "from": from_, "text_contains": text_contains,
+                                                  "fact": fact, "count_gte": count_gte}.items() if value is not None}
+        if not selector:
+            raise ValueError("give at least one of to, in_, in_thread_of, from_, text_contains, fact")
+        return self._world.holds({"message": selector})
+
+
+class World:
+    """A fresh world: `env` holds the agent's variables (the tools' URLs and fake credentials, ANCHA_TASK); never the control URL."""
+
+    def __init__(self, up_args: Iterable[str] = (), cwd: str | None = None) -> None:
+        self._args = list(up_args)
+        self.cwd = cwd or os.getcwd()
+        try:
+            up = run_cli(["world", "up", *self._args], self.cwd)
+        except Exception as error:
+            error._ancha_phase = "up"
+            raise
+        self.id: str = up["envId"]
+        self.env: dict[str, str] = dict(up["env"])
+        self.services: dict = up["services"]
+        self.workspace: str = up["workspace"]
+        self.agent: subprocess.Popen | None = None
+        self._agents: list[subprocess.Popen] = []
+        self._recorded_stop_error: BaseException | None = None
+        self._log: str | None = None
+        self._down = False
+        self._recorder = current.get()
+        self._record = self._recorder.add_world(self) if self._recorder is not None else None
+        self.people = People(self)
+        self.slack = Slack(self)
+
+    @property
+    def task(self) -> str | None:
+        return self.env.get("ANCHA_TASK")
+
+    def start(self, agent: str | list[str], *, cwd: str | None = None, env: dict[str, str] | None = None) -> subprocess.Popen:
+        """Starts the agent with this world's variables (and PATH, HOME, locale from here); its output goes to `log()`."""
+        command = shlex.split(agent) if isinstance(agent, str) else list(agent)
+        if self._down:
+            raise AnchaError("ENV_NOT_FOUND", "this world was already removed")
+        if self._recorder is not None:
+            self._recorder.add_secrets(env or {})
+        base = {key: os.environ[key] for key in ("PATH", "HOME", "LANG", "LC_ALL", "TMPDIR", "SYSTEMROOT") if key in os.environ}
+        handle, self._log = tempfile.mkstemp(prefix="ancha-agent-", suffix=".log")
+        with os.fdopen(handle, "w") as log:
+            try:
+                self.agent = subprocess.Popen(command, cwd=cwd or self.cwd, env={**base, **self.env, **(env or {})}, stdout=log, stderr=subprocess.STDOUT)
+            except Exception as error:
+                error._ancha_phase = "agent"
+                raise
+        self._agents.append(self.agent)
+        return self.agent
+
+    def wait(self, timeout: float = 120) -> int | None:
+        """Waits for the agent to exit and returns its exit code; stops it and raises TimeoutError after `timeout` seconds."""
+        if self.agent is None:
+            return None
+        try:
+            return self.agent.wait(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            self.stop()
+            raise TimeoutError(f"the agent was still running after {timeout} s; its output: {self.log()[-2000:]}") from None
+
+    wait_until_quiet = wait
+
+    def log(self) -> str:
+        if self._log is None:
+            return ""
+        with open(self._log, encoding="utf-8", errors="replace") as log:
+            return log.read()
+
+    def stop(self) -> None:
+        first: BaseException | None = None
+        for agent in self._agents:
+            try:
+                if agent.poll() is None:
+                    agent.terminate()
+                    try:
+                        agent.wait(timeout=5)
+                    except subprocess.TimeoutExpired:
+                        agent.kill()
+                        agent.wait(timeout=5)
+            except Exception as error:
+                first = first or error
+        if first is not None:
+            first._ancha_phase = "agent"
+            if self._recorder is not None:
+                self._recorder.infrastructure_error("agent", first, self._record)
+                self._recorded_stop_error = first
+            raise first
+
+    def check(self, *checks: dict) -> CheckReport:
+        """The world's own checks (its scene or test), or these check objects, e.g. check({"id": "x", "decided": ["mike", "severity"]})."""
+        args = [arg for index, item in enumerate(checks) for arg in ("--check", json.dumps({"id": f"check-{index + 1}", **item}))]
+        value = run_cli(["check", "--env", self.id, *args], self.cwd, ok=(0, 1))
+        if self._record is not None:
+            self._record["checks"].append(value)
+        report = CheckReport(value)
+        # A check the world couldn't evaluate is a mistake in the test, not a verdict on the agent.
+        broken = [result for result in report.checks if result.get("error")]
+        if broken:
+            raise AnchaError("USAGE", "; ".join(f"check {result['id']} could not be evaluated: {result['error']}" for result in broken))
+        return report
+
+    def holds(self, check: dict) -> bool:
+        return self.check(check).passed
+
+    def state(self) -> dict:
+        return run_cli(["world", "state", "--env", self.id], self.cwd)
+
+    def trace(self) -> list:
+        return run_cli(["world", "trace", "--env", self.id], self.cwd)
+
+    def fresh(self) -> World:
+        """Another world from the same scene or test."""
+        return World(self._args, self.cwd)
+
+    def down(self) -> None:
+        if self._down:
+            return
+        self._down = True
+        first: BaseException | None = None
+        try:
+            self.stop()
+        except BaseException as error:
+            first = error
+            if self._recorder is not None and error is not self._recorded_stop_error:
+                self._recorder.infrastructure_error("agent", error, self._record)
+        if self._record is not None:
+            try:
+                capture = run_cli(["world", "capture", "--env", self.id], self.cwd)
+                # Native state/trace are evidence; free-form service logs are not part of the pytest record.
+                if isinstance(capture, dict) and isinstance(capture.get("observation"), dict):
+                    capture["observation"]["logs"] = []
+                self._record["capture"] = capture
+            except BaseException as error:
+                first = first or error
+                self._recorder.infrastructure_error("capture", error, self._record)
+        try:
+            run_cli(["world", "down", "--env", self.id], self.cwd)
+        except BaseException as error:
+            first = first or error
+            if self._recorder is not None:
+                self._recorder.infrastructure_error("cleanup", error, self._record)
+        if first is not None:
+            if not isinstance(first, Exception):
+                raise first
+            # The record keeps stage/code evidence. Do not attach raw output or exception text to teardown errors.
+            raise AnchaError(error_code(first), "could not capture or remove this test's world") from None
+
+
+def attempts(n: int, min_pass_rate: float = 1.0) -> Callable:
+    """Runs the test `n` times, each in a fresh world, and passes when at least `min_pass_rate` of them pass."""
+    if n < 1 or not 0 <= min_pass_rate <= 1:
+        raise ValueError("attempts(n, min_pass_rate): n >= 1 and 0 <= min_pass_rate <= 1")
+
+    def decorate(test: Callable) -> Callable:
+        @functools.wraps(test)
+        def wrapper(*args: Any, **kwargs: Any) -> None:
+            import pytest
+            first: World | None = kwargs.get("world")
+            if first is None:
+                raise TypeError("ancha.attempts needs the world fixture: def test_x(world)")
+            passed, failures = 0, []
+            recorder = current.get()
+            if recorder is not None:
+                recorder.repeated = True
+                recorder.payload["minPassRate"] = min_pass_rate
+                recorder.payload["requestedTrials"] = n
+            for attempt in range(n):
+                if attempt > 0 and recorder is not None:
+                    recorder.begin_attempt()
+                world = first if attempt == 0 else first.fresh()
+                outcome = "passed"
+                raised: BaseException | None = None
+                try:
+                    test(*args, **{**kwargs, "world": world})
+                    passed += 1
+                except (AssertionError, pytest.fail.Exception) as error:
+                    outcome = "failed"
+                    failures.append(f"attempt {attempt + 1}: {error}")
+                except BaseException as error:
+                    outcome = "xfailed" if isinstance(error, pytest.xfail.Exception) else "skipped" if isinstance(error, pytest.skip.Exception) else "invalid"
+                    raised = error
+                finally:
+                    cleanup = recorder.finish_attempt(outcome) if recorder is not None else None
+                    if recorder is None:
+                        try:
+                            world.down()
+                        except BaseException as error:
+                            cleanup = error
+                if raised is not None:
+                    raise raised
+                if cleanup is not None:
+                    raise cleanup
+            from fractions import Fraction
+            threshold = Fraction(str(min_pass_rate))
+            if passed * threshold.denominator < n * threshold.numerator:
+                raise AssertionError(f"{passed} of {n} attempts passed; at least {min_pass_rate:.0%} must. " + " | ".join(failures[:3]))
+
+        return wrapper
+
+    return decorate

ancha-0.1.0.dist-info/METADATA

@@ -0,0 +1,78 @@
+Metadata-Version: 2.4
+Name: ancha
+Version: 0.1.0
+Summary: Ancha for Python: test your agent's own tools against production's recorded calls, and pytest worlds for agent tests
+License-Expression: MIT
+Classifier: Framework :: Pytest
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Provides-Extra: pytest
+Requires-Dist: pytest>=7; extra == "pytest"
+
+# ancha
+
+Ancha for Python. Two parts, one `import ancha`:
+
+- **Tool wrappers** (no dependencies): wrap your agent's own tools once with `@ancha.tool` or `ancha.wrap_tools`, and
+  run the agent through `ancha.entry`. In production nothing changes. In an [Ancha](https://ancha.io) test made from a
+  production trace, each call answers as production did. See
+  [Tool wrappers](https://docs.ancha.io/reference/tool-wrappers).
+- **A pytest plugin** (`pip install "ancha[pytest]"`): a fresh simulated world for each test. See
+  [Test with pytest](https://docs.ancha.io/guides/pytest).
+
+```sh
+pip install ancha
+```
+
+```python
+import ancha
+
+@ancha.tool
+def lookup_order(order_id: str) -> dict: ...
+
+ancha.entry(lambda input: run_agent(input["task"]))
+```
+
+## The pytest plugin
+
+pytest for Ancha: a fresh simulated world for each test, your agent run against it, and checks on what people learned and what the tools hold. It drives the `ancha` CLI. This development version needs the matching CLI with `world capture` and `world record`; published CLI 0.1.4 does not support run recording. `ANCHA_CLI` names another CLI command.
+
+```python
+import ancha, pytest
+
+@pytest.mark.ancha(test="ancha/tests/relay-export-bug.yaml")
+@ancha.attempts(10, min_pass_rate=0.9)
+def test_relay(world):
+    world.start("npm start")                 # the agent gets world.env: the tools' URLs and fake credentials, ANCHA_TASK
+    world.wait_until_quiet()
+    assert world.people["ben"].knows("laura.beta_on")
+    assert world.slack.messages(to="laura", in_thread_of="$laura_report", from_="$slack_bot")
+    assert not world.people["laura"].knows("ben.root_cause")
+```
+
+- `@pytest.mark.ancha(test=…, scene=…, world=…, on="local"|"cloud", seed=…, ttl=…)` says what the `world` fixture starts (`ancha world up`). Without it, the one scene `ancha.yaml` names.
+- `world.env`, `world.task`; `world.start(command)`, `world.wait_until_quiet(timeout)`, `world.log()`, `world.stop()`.
+- `world.check()` applies the test's own checks; `world.check({...})` applies check objects (the kinds in `ancha docs scenes`). A report is truthy when every check passed; `report.failures` has expected and actual.
+- `world.people[ref].knows(fact)`, `.decided(decision, option)`; `world.slack.messages(to=, in_=, in_thread_of=, from_=, text_contains=, fact=)`.
+- `world.state()`, `world.trace()`; `world.fresh()` for another world; the fixture removes its world (`ancha world down`).
+- `@ancha.attempts(n, min_pass_rate)` runs the test in `n` fresh worlds and passes when enough of them do.
+
+An evaluated check that fails returns a false `CheckReport`; `report.failures` explains the expected and actual values. A check that cannot be evaluated raises `ancha.AnchaError` instead. For example, a `knows` check naming an unknown person raises an error with `code == "USAGE"` and a message naming the invalid reference. Correct the test definition; this error is not a verdict on the agent. The people and Slack helpers use the same check path and can raise the same error. CLI failures also raise `AnchaError`, retaining the CLI's `code`, `message` and optional `hint`.
+
+Worlds start locally by default, including after `ancha login`. Run `pytest --on cloud` to put worlds in your hosted workspace; pytest and the agent still run on your machine. `--on local` forces local worlds. The pytest option overrides the marker's `on="local"` or `on="cloud"`; without either, the CLI's normal world selection applies. An explicit server URL can also select a world server. Never give the agent the test files or the checks.
+
+## Recorded runs
+
+While the plugin is enabled, each executed pytest item creates one local run under `.ancha/runs/`, with `source: pytest` and `target: local`. This includes tests without a world, parameterized cases and skipped tests. Collection alone creates no records. Disable the plugin with `-p no:ancha` for suites that should not record. Run files stay local even when worlds run in the cloud; uploading pytest records is not supported.
+
+```sh
+ancha --local runs list --json
+ancha --local runs show <run-id> --json
+ancha --local explain <run-id> --json
+```
+
+Pytest assertions decide the test outcome. A deliberately false `world.check()` remains evidence and does not fail a passing negative assertion. Records preserve setup, call and teardown outcomes separately. Skipped and expected-failure cases are ungraded; setup, capture and cleanup failures are invalid, not failed agent answers. `@ancha.attempts` records each invocation and its fresh world under the same run, including failed attempts when the overall pass-rate threshold is met. The requested repetition count is recorded before execution. Reports keep every requested attempt in the pass-rate denominator, including invalid, skipped and missing attempts; all-k requires all requested attempts to finish validly and pass. Thresholds use exact integer counts. An old baseline without verified requested counts must be regenerated before comparison.
+
+The plugin stops its agent, captures a coherent world observation, then removes the world. This observation is not a sealed managed-trial capture. Capture failures remain visible and cleanup is still attempted. Recording failures are reported and make an otherwise successful pytest session fail.
+
+Parameter values in test IDs are replaced by a hash. Records omit environment values, raw exception messages, captured pytest output and agent logs. Known credentials are redacted from evidence; keep `.ancha/` private and out of version control. Abrupt termination such as SIGKILL cannot finish a record or guarantee cleanup; world TTLs still apply.

ancha-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+ancha/__init__.py,sha256=NZ3HpuooOdG1eDMYe5fZWtAdo5wB7RsgjF5k3FFUH2E,1086
+ancha/plugin.py,sha256=4NqKTFDfaD7oF0pZFtuydPfTndGewFoVu2N80qdGxw8,3787
+ancha/recording.py,sha256=S9fBn5pnkV17euWLovniSoaqINfIZja8lRFvoYjTpO8,7745
+ancha/tools.py,sha256=c62jEwTBxKumuRCV0QkwgKrujPJF6ruoU1n8-JgC2nw,10445
+ancha/world.py,sha256=vW8lGMCB21Yl5yt4NGQy_omS-2WsTbfPqpm63fRnn-E,13982
+ancha-0.1.0.dist-info/METADATA,sha256=a1eQTpQ4CujCTQElbYvuRa9P30RetXFpKXo9fU0puWE,6005
+ancha-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+ancha-0.1.0.dist-info/entry_points.txt,sha256=Ce3WFSBftLgXwuUUi3ME1dzUTuXKauOuq-Zyl8w49XM,32
+ancha-0.1.0.dist-info/top_level.txt,sha256=cYaGV4PP4Q52VJVUOUxGbXTxNhFB_P533AVWGRUlIe4,6
+ancha-0.1.0.dist-info/RECORD,,

ancha-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

ancha-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[pytest11]
+ancha = ancha.plugin

ancha-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+ancha