readyagentsdev 0.8.2__py3-none-any.whl

readyagents/testing/eval.py

@@ -0,0 +1,219 @@
+"""Tiny eval harness: score fixture workflows (pass and fail cases)."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from readyagents.errors import ConfigError
+from readyagents.testing.helpers import run_workflow_file_test, run_workflow_spec
+from readyagents.tools import ToolRegistry
+from readyagents.workflow.schema import WorkflowSpec
+from readyagents.workflow.state import RunState
+
+
+@dataclass
+class EvalCase:
+    name: str
+    workflow: Mapping[str, Any] | WorkflowSpec | Path | str
+    inputs: dict[str, Any] = field(default_factory=dict)
+    decisions: dict[str, str] = field(default_factory=dict)
+    expect_status: str = "succeeded"
+    expect_outputs: dict[str, Any] | None = None
+    expect_contains: dict[str, str] | None = None
+
+
+@dataclass
+class EvalResult:
+    name: str
+    passed: bool
+    reason: str = ""
+    state: RunState | None = None
+
+
+@dataclass
+class EvalReport:
+    results: list[EvalResult]
+
+    @property
+    def passed(self) -> int:
+        return sum(1 for row in self.results if row.passed)
+
+    @property
+    def failed(self) -> int:
+        return sum(1 for row in self.results if not row.passed)
+
+    @property
+    def ok(self) -> bool:
+        return self.failed == 0
+
+    def assert_passing(self) -> None:
+        if self.ok:
+            return
+        lines = [f"{row.name}: {row.reason}" for row in self.results if not row.passed]
+        raise AssertionError("eval failures:\n" + "\n".join(lines))
+
+
+def load_eval_suite(path: Path | str) -> list[EvalCase]:
+    """Load a YAML/JSON mapping with a ``cases:`` list of :class:`EvalCase` fields."""
+    file = Path(path)
+    if not file.is_file():
+        raise ConfigError(f"Eval suite file not found: {file}")
+    text = file.read_text(encoding="utf-8")
+    try:
+        if file.suffix.lower() in {".json"}:
+            data = json.loads(text)
+        else:
+            data = yaml.safe_load(text)
+    except (json.JSONDecodeError, yaml.YAMLError) as exc:
+        raise ConfigError(f"Could not parse eval suite {file}: {exc}") from exc
+    if not isinstance(data, Mapping):
+        raise ConfigError(f"Eval suite {file} must be a mapping with a 'cases' list")
+    raw_cases = data.get("cases")
+    if not isinstance(raw_cases, list):
+        raise ConfigError(f"Eval suite {file} must be a mapping with a 'cases' list")
+    if not raw_cases:
+        raise ConfigError(f"Eval suite {file} has no cases")
+    return [
+        _case_from_mapping(row, index=i, suite=file) for i, row in enumerate(raw_cases, start=1)
+    ]
+
+
+def _case_from_mapping(raw: object, *, index: int, suite: Path) -> EvalCase:
+    if not isinstance(raw, Mapping):
+        raise ConfigError(f"Eval suite {suite} case #{index} must be a mapping")
+    name = raw.get("name")
+    if not isinstance(name, str) or not name.strip():
+        raise ConfigError(f"Eval suite {suite} case #{index} needs a name")
+    name = name.strip()
+    workflow_field = raw.get("workflow")
+    if isinstance(workflow_field, str):
+        rel = workflow_field.strip()
+        if not rel:
+            raise ConfigError(f"Eval case {name!r} has an empty workflow path")
+        # File workflows are resolved next to the suite, not the process cwd.
+        workflow: Mapping[str, Any] | WorkflowSpec | Path | str = Path(suite).parent / rel
+    elif isinstance(workflow_field, Mapping):
+        workflow = dict(workflow_field)
+    else:
+        raise ConfigError(f"Eval case {name!r} needs a workflow path or inline mapping")
+    expect_status = raw.get("expect_status", "succeeded")
+    if not isinstance(expect_status, str) or not expect_status.strip():
+        raise ConfigError(f"Eval case {name!r} field 'expect_status' must be a string")
+    return EvalCase(
+        name=name,
+        workflow=workflow,
+        inputs=_mapping_field(raw, "inputs", name, default={}),
+        decisions=_str_mapping_field(raw, "decisions", name, default={}),
+        expect_status=expect_status.strip(),
+        expect_outputs=_optional_mapping_field(raw, "expect_outputs", name),
+        expect_contains=_optional_str_mapping_field(raw, "expect_contains", name),
+    )
+
+
+def _mapping_field(
+    raw: Mapping[str, Any],
+    key: str,
+    name: str,
+    *,
+    default: dict[str, Any],
+) -> dict[str, Any]:
+    if key not in raw or raw[key] is None:
+        return default
+    value = raw[key]
+    if not isinstance(value, Mapping):
+        raise ConfigError(f"Eval case {name!r} field '{key}' must be a mapping")
+    return dict(value)
+
+
+def _str_mapping_field(
+    raw: Mapping[str, Any],
+    key: str,
+    name: str,
+    *,
+    default: dict[str, str],
+) -> dict[str, str]:
+    data = _mapping_field(raw, key, name, default=default)
+    return {str(k): str(v) for k, v in data.items()}
+
+
+def _optional_mapping_field(
+    raw: Mapping[str, Any],
+    key: str,
+    name: str,
+) -> dict[str, Any] | None:
+    if key not in raw or raw[key] is None:
+        return None
+    value = raw[key]
+    if not isinstance(value, Mapping):
+        raise ConfigError(f"Eval case {name!r} field '{key}' must be a mapping")
+    return dict(value)
+
+
+def _optional_str_mapping_field(
+    raw: Mapping[str, Any],
+    key: str,
+    name: str,
+) -> dict[str, str] | None:
+    data = _optional_mapping_field(raw, key, name)
+    if data is None:
+        return None
+    return {str(k): str(v) for k, v in data.items()}
+
+
+def _score(state: RunState, case: EvalCase) -> tuple[bool, str]:
+    if state.status != case.expect_status:
+        return False, f"status {state.status!r} != {case.expect_status!r}"
+    outputs = state.output_keys or state.node_outputs
+    if case.expect_outputs:
+        for key, expected in case.expect_outputs.items():
+            actual = outputs.get(key)
+            if actual != expected:
+                return False, f"output {key!r}={actual!r} != {expected!r}"
+    if case.expect_contains:
+        for key, needle in case.expect_contains.items():
+            hay = outputs.get(key)
+            if needle not in str(hay):
+                return False, f"output {key!r}={hay!r} does not contain {needle!r}"
+    return True, "ok"
+
+
+def run_eval(
+    cases: Sequence[EvalCase],
+    *,
+    llm: Any = None,
+    tools: ToolRegistry | None = None,
+    settings: Any | None = None,
+) -> EvalReport:
+    results: list[EvalResult] = []
+    for case in cases:
+        try:
+            if isinstance(case.workflow, (Path, str)):
+                state = run_workflow_file_test(
+                    case.workflow,
+                    inputs=case.inputs,
+                    llm=llm,
+                    settings=settings,
+                    persist=False,
+                    decisions=case.decisions,
+                    extra_tools=tools,
+                )
+            else:
+                state = run_workflow_spec(
+                    case.workflow,
+                    inputs=case.inputs,
+                    llm=llm,
+                    tools=tools,
+                    decisions=case.decisions,
+                )
+        except Exception as exc:  # noqa: BLE001
+            results.append(EvalResult(name=case.name, passed=False, reason=str(exc)))
+            continue
+        ok, reason = _score(state, case)
+        results.append(EvalResult(name=case.name, passed=ok, reason=reason, state=state))
+    return EvalReport(results)

readyagents/testing/helpers.py

@@ -0,0 +1,128 @@
+"""Helpers that wrap the same engine path the CLI uses."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+from readyagents.errors import LLMError
+from readyagents.llm.base import CompletionResult, Message, ToolCall
+from readyagents.tools import ToolRegistry
+from readyagents.workflow.engine import run_workflow
+from readyagents.workflow.nodes import ExecutionContext
+from readyagents.workflow.runner import run_workflow_file
+from readyagents.workflow.schema import WorkflowSpec
+from readyagents.workflow.state import RunState
+
+
+class ScriptedLLM:
+    """In-process LLM stand-in. Queue completions or errors per model id."""
+
+    name = "scripted"
+
+    def __init__(self) -> None:
+        self.calls: list[dict[str, Any]] = []
+        self._by_model: dict[str, list[CompletionResult | BaseException]] = {}
+        self._queue: list[CompletionResult | BaseException] = []
+
+    def enqueue(
+        self,
+        text: str = "ok",
+        *,
+        model: str | None = None,
+        usage: Mapping[str, Any] | None = None,
+        error: BaseException | None = None,
+        tool_calls: Sequence[ToolCall] | None = None,
+    ) -> ScriptedLLM:
+        item: CompletionResult | BaseException
+        if error is not None:
+            item = error
+        else:
+            item = CompletionResult(
+                text=text,
+                model=model or "scripted",
+                usage=dict(usage or {}),
+                tool_calls=list(tool_calls or []),
+            )
+        if model:
+            self._by_model.setdefault(model, []).append(item)
+        else:
+            self._queue.append(item)
+        return self
+
+    def complete(
+        self,
+        messages: list[Message],
+        *,
+        model: str,
+        tools: Any = None,
+        **kwargs: Any,
+    ) -> CompletionResult:
+        self.calls.append({"model": model, "messages": messages, "tools": tools})
+        item: CompletionResult | BaseException | None = None
+        bucket = self._by_model.get(model)
+        if bucket:
+            item = bucket.pop(0)
+        elif self._queue:
+            item = self._queue.pop(0)
+        if item is None:
+            return CompletionResult(text="ok", model=model, usage={})
+        if isinstance(item, BaseException):
+            raise item
+        if not item.model:
+            item.model = model
+        return item
+
+
+def run_workflow_spec(
+    spec: Mapping[str, Any] | WorkflowSpec,
+    *,
+    inputs: Mapping[str, Any] | None = None,
+    llm: Any | None = None,
+    tools: ToolRegistry | None = None,
+    decisions: Mapping[str, str] | None = None,
+    **ctx_kwargs: Any,
+) -> RunState:
+    """Validate a workflow mapping and run it through ``run_workflow``."""
+    workflow = spec if isinstance(spec, WorkflowSpec) else WorkflowSpec.model_validate(dict(spec))
+    merged = dict(workflow.input_defaults())
+    if inputs:
+        merged.update(inputs)
+    ctx = ExecutionContext(
+        workflow,
+        tools or ToolRegistry(),
+        llm=llm,
+        default_model=ctx_kwargs.pop("default_model", workflow.default_model or "mock:test"),
+        decisions=decisions,
+        **ctx_kwargs,
+    )
+    return run_workflow(workflow, merged, ctx)
+
+
+def run_workflow_file_test(
+    path: Path | str,
+    *,
+    inputs: Mapping[str, Any] | None = None,
+    llm: Any | None = None,
+    settings: Any | None = None,
+    persist: bool = False,
+    decisions: Mapping[str, str] | None = None,
+    extra_tools: ToolRegistry | None = None,
+    **kwargs: Any,
+) -> RunState:
+    """``run_workflow_file`` with test-friendly defaults (no persist)."""
+    return run_workflow_file(
+        path,
+        inputs=inputs,
+        llm=llm,
+        settings=settings,
+        persist=persist,
+        decisions=decisions,
+        extra_tools=extra_tools,
+        **kwargs,
+    )
+
+
+def fail(message: str) -> LLMError:
+    return LLMError(message)

readyagents/testing/recorded.py

@@ -0,0 +1,68 @@
+"""Offline / recorded LLM: replay a cassette file, no network."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from readyagents.errors import LLMError
+from readyagents.llm.base import CompletionResult, Message
+from readyagents.llm.tool_calls import tool_calls_from_json, tool_calls_to_json
+
+
+class RecordedLLM:
+    """Replay stored completions. Optional ``inner`` records new calls to disk."""
+
+    name = "recorded"
+
+    def __init__(self, cassette: Path | str, *, inner: Any | None = None) -> None:
+        self.cassette = Path(cassette)
+        self.inner = inner
+        self.calls: list[list[Message]] = []
+        self.models: list[str] = []
+        self._tape: list[dict[str, Any]] = []
+        if self.cassette.is_file():
+            loaded = json.loads(self.cassette.read_text(encoding="utf-8"))
+            if isinstance(loaded, list):
+                self._tape = [row for row in loaded if isinstance(row, dict)]
+        self._index = 0
+
+    def complete(
+        self,
+        messages: list[Message],
+        *,
+        model: str,
+        tools: Any = None,
+        **kwargs: Any,
+    ) -> CompletionResult:
+        self.calls.append(messages)
+        self.models.append(model)
+        if self._index < len(self._tape):
+            row = self._tape[self._index]
+            self._index += 1
+            usage = row.get("usage") if isinstance(row.get("usage"), dict) else {}
+            return CompletionResult(
+                text=str(row.get("text") or ""),
+                model=str(row.get("model") or model),
+                usage=dict(usage),
+                tool_calls=tool_calls_from_json(row.get("tool_calls")),
+            )
+        if self.inner is None:
+            raise LLMError(
+                f"No recorded completion at index {self._index} in {self.cassette} "
+                "(offline replay — no network)"
+            )
+        result = self.inner.complete(messages, model=model, tools=tools, **kwargs)
+        self._tape.append(
+            {
+                "text": result.text,
+                "model": result.model or model,
+                "usage": dict(result.usage or {}),
+                "tool_calls": tool_calls_to_json(result.tool_calls),
+            }
+        )
+        self._index += 1
+        self.cassette.parent.mkdir(parents=True, exist_ok=True)
+        self.cassette.write_text(json.dumps(self._tape, indent=2, ensure_ascii=False) + "\n")
+        return result

readyagents/tools/__init__.py

@@ -0,0 +1,67 @@
+"""Common Tool protocol wrapping builtin and MCP tools."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+
+from readyagents.errors import ToolError
+
+
+class Tool(Protocol):
+    name: str
+    description: str
+    schema: dict[str, Any]
+
+    def run(self, **kwargs: Any) -> Any: ...
+
+
+@dataclass
+class FunctionTool:
+    name: str
+    description: str
+    handler: Callable[..., Any]
+    schema: dict[str, Any] = field(default_factory=dict)
+
+    def run(self, **kwargs: Any) -> Any:
+        try:
+            return self.handler(**kwargs)
+        except TypeError as exc:
+            raise ToolError(f"Tool '{self.name}' got invalid arguments: {exc}") from exc
+
+
+class ToolRegistry:
+    def __init__(self) -> None:
+        self._tools: dict[str, Tool] = {}
+
+    def register(self, tool: Tool) -> None:
+        self._tools[tool.name] = tool
+
+    def get(self, name: str) -> Tool:
+        if name not in self._tools:
+            known = ", ".join(sorted(self._tools)) or "(none)"
+            raise ToolError(f"Unknown tool '{name}'. Available: {known}")
+        return self._tools[name]
+
+    def names(self) -> list[str]:
+        return sorted(self._tools)
+
+    def as_dict(self) -> Mapping[str, Tool]:
+        return dict(self._tools)
+
+    def merge(self, other: Mapping[str, Tool] | ToolRegistry) -> None:
+        items = other.as_dict() if isinstance(other, ToolRegistry) else other
+        for tool in items.values():
+            if tool.name in self._tools:
+                continue
+            self.register(tool)
+
+
+def default_registry(*, allow_http: bool, workspace: Any) -> ToolRegistry:
+    from readyagents.mcp.builtin import builtin_tools
+
+    registry = ToolRegistry()
+    for tool in builtin_tools(allow_http=allow_http, workspace=workspace):
+        registry.register(tool)
+    return registry

readyagents/workflow/__init__.py

@@ -0,0 +1,3 @@
+from readyagents.workflow.schema import WorkflowSpec
+
+__all__ = ["WorkflowSpec"]

readyagents/workflow/cancellation.py

@@ -0,0 +1,88 @@
+"""Cooperative cancellation for in-flight workflow runs."""
+
+from __future__ import annotations
+
+import threading
+import time
+from collections.abc import Callable
+
+from readyagents.errors import CancellationRequested
+
+_SLEEP_CHUNK = 0.05
+
+
+class CancellationToken:
+    """Thread-safe cooperative cancellation flag (threading.Event)."""
+
+    def __init__(self) -> None:
+        self._event = threading.Event()
+        self._lock = threading.Lock()
+        self._actor: str | None = None
+        self._reason: str | None = None
+        self._listeners: list[Callable[[], None]] = []
+
+    def request(self, *, actor: str | None = None, reason: str | None = None) -> None:
+        listeners: list[Callable[[], None]]
+        with self._lock:
+            first = not self._event.is_set()
+            if first:
+                self._actor = actor
+                self._reason = reason
+            listeners = list(self._listeners)
+            self._event.set()
+        if first:
+            for listener in listeners:
+                listener()
+
+    def is_requested(self) -> bool:
+        return self._event.is_set()
+
+    def raise_if_requested(self, *, run_id: str | None = None) -> None:
+        if not self.is_requested():
+            return
+        raise CancellationRequested(run_id=run_id, reason=self.reason)
+
+    def wait(self, timeout: float) -> bool:
+        """Block up to ``timeout`` seconds. True if cancellation was requested."""
+        return self._event.wait(max(float(timeout), 0.0))
+
+    @property
+    def actor(self) -> str | None:
+        with self._lock:
+            return self._actor
+
+    @property
+    def reason(self) -> str | None:
+        with self._lock:
+            return self._reason
+
+    def add_listener(self, callback: Callable[[], None]) -> None:
+        """Invoke ``callback`` on the first ``request()`` (or immediately if already set)."""
+        with self._lock:
+            self._listeners.append(callback)
+            pending = self._event.is_set()
+        if pending:
+            callback()
+
+    def clear_listeners(self) -> None:
+        with self._lock:
+            self._listeners.clear()
+
+
+def cancellable_sleep(seconds: float, token: CancellationToken | None) -> None:
+    """Sleep ``seconds``, or until ``token`` is requested (then raise CancellationRequested)."""
+    delay = max(float(seconds), 0.0)
+    if token is None:
+        time.sleep(delay)
+        return
+    if delay <= 0:
+        token.raise_if_requested()
+        return
+    remaining = delay
+    while remaining > 0:
+        chunk = remaining if remaining < _SLEEP_CHUNK else _SLEEP_CHUNK
+        if token.wait(chunk):
+            token.raise_if_requested()
+            return
+        remaining -= chunk
+    token.raise_if_requested()