agentforge-py 0.2.1__py3-none-any.whl

agentforge/testing/__init__.py

@@ -0,0 +1,69 @@
+"""Public testing API (feat-016).
+
+`agentforge.testing` is the v0.1 public test-helper surface. It
+ships scripted-response mocks, fake tools, an `agent_factory`
+helper, pytest fixtures, conformance harnesses re-exported from
+`agentforge_core.testing`, and recording / replay helpers.
+
+Typical usage:
+
+    from agentforge.testing import (
+        MockLLMClient,
+        FakeTool,
+        agent_factory,
+        run_memory_conformance,
+    )
+
+    async def test_population_lookup() -> None:
+        llm = MockLLMClient.from_script([
+            {"text": "I need to search.",
+             "tool_calls": [{"name": "search", "args": {"q": "x"}}]},
+            {"text": "47.5M", "stop_reason": "end_turn"},
+        ])
+        web = FakeTool.fake("search", lambda **kw: "47.5M")
+        agent = agent_factory(model=llm, tools=[web])
+        result = await agent.run("How many people live in Spain?")
+        assert "47.5M" in result.output
+        assert llm.call_count == 2
+
+The private `agentforge._testing` namespace is preserved as a
+compatibility shim for the framework's own pre-feat-016 internal
+tests; new code should import from `agentforge.testing`.
+"""
+
+from __future__ import annotations
+
+from agentforge._testing.fake_llm import FakeLLMClient, echo_response
+from agentforge._testing.fake_tool import FakeTool
+from agentforge.testing.conformance import (
+    run_hybrid_search_conformance,
+    run_input_validator_conformance,
+    run_memory_conformance,
+    run_output_validator_conformance,
+    run_strategy_conformance,
+    run_task_conformance,
+    run_tool_gate_conformance,
+    run_vector_conformance,
+)
+from agentforge.testing.factory import agent_factory
+from agentforge.testing.llm import MockLLMClient, ScriptedResponse
+from agentforge.testing.recording import load_recording, record_llm
+
+__all__ = [
+    "FakeLLMClient",
+    "FakeTool",
+    "MockLLMClient",
+    "ScriptedResponse",
+    "agent_factory",
+    "echo_response",
+    "load_recording",
+    "record_llm",
+    "run_hybrid_search_conformance",
+    "run_input_validator_conformance",
+    "run_memory_conformance",
+    "run_output_validator_conformance",
+    "run_strategy_conformance",
+    "run_task_conformance",
+    "run_tool_gate_conformance",
+    "run_vector_conformance",
+]

agentforge/testing/conformance.py

@@ -0,0 +1,40 @@
+"""Conformance harnesses (feat-016).
+
+Re-exports the ABC-conformance functions from
+`agentforge_core.testing.conformance` so users have one canonical
+import path. The actual harnesses live in `agentforge-core` because
+they exercise contracts defined there.
+
+Typical usage:
+
+    from agentforge.testing import run_memory_conformance
+    from my_pkg import MyMemoryStore
+
+    async def test_my_driver() -> None:
+        async with MyMemoryStore.from_url("...") as store:
+            await run_memory_conformance(store)
+"""
+
+from __future__ import annotations
+
+from agentforge_core.testing.conformance import (
+    run_hybrid_search_conformance,
+    run_input_validator_conformance,
+    run_memory_conformance,
+    run_output_validator_conformance,
+    run_strategy_conformance,
+    run_task_conformance,
+    run_tool_gate_conformance,
+    run_vector_conformance,
+)
+
+__all__ = [
+    "run_hybrid_search_conformance",
+    "run_input_validator_conformance",
+    "run_memory_conformance",
+    "run_output_validator_conformance",
+    "run_strategy_conformance",
+    "run_task_conformance",
+    "run_tool_gate_conformance",
+    "run_vector_conformance",
+]

agentforge/testing/factory.py

@@ -0,0 +1,89 @@
+"""`agent_factory` — safe Agent constructor for unit tests (feat-016).
+
+Defaults: `MockLLMClient.deterministic("ok")`, no tools, in-memory
+store, low budget (0.10 USD), low iteration cap (3). Override any
+kwarg explicitly. The intent is fast, deterministic, isolated tests
+— no network, no secrets, < 100 ms per case.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from agentforge_core.contracts.llm import LLMClient
+from agentforge_core.contracts.strategy import ReasoningStrategy
+from agentforge_core.contracts.tool import Tool
+from agentforge_core.values.state import AgentState, Step
+
+from agentforge.agent import Agent
+from agentforge.memory import InMemoryStore
+from agentforge.testing.llm import MockLLMClient
+
+
+class _SingleStepStrategy(ReasoningStrategy):
+    """Trivial strategy: makes a single LLM call, returns its content.
+
+    The factory wires this in when no `strategy` override is passed
+    so tests don't need to install feat-002's strategies just to
+    exercise `Agent.run`. Real production agents pass an explicit
+    strategy.
+    """
+
+    async def run(self, state: AgentState) -> AgentState:
+        from agentforge.strategies._base import get_runtime  # noqa: PLC0415
+
+        runtime = get_runtime(state)
+        response = await runtime.llm.call(
+            system="",
+            messages=[],
+            tools=None,
+        )
+        state.steps.append(
+            Step(
+                iteration=0,
+                kind="observe",
+                content=response.content,
+                cost_usd=response.cost_usd,
+            )
+        )
+        return state
+
+
+def agent_factory(
+    *,
+    model: str | LLMClient | None = None,
+    tools: list[Tool] | None = None,
+    strategy: str | ReasoningStrategy | None = None,
+    **overrides: Any,
+) -> Agent:
+    """Construct an `Agent` with safe test defaults.
+
+    Equivalent to::
+
+        Agent(
+            model=model or MockLLMClient.deterministic("ok"),
+            tools=tools or [],
+            strategy=strategy or _SingleStepStrategy(),
+            memory=InMemoryStore(),
+            budget_usd=0.10,
+            max_iterations=3,
+            install_log_filter=False,
+            **overrides,
+        )
+
+    The `install_log_filter=False` default keeps test runs from
+    mutating root-logger handlers across the suite.
+    """
+    return Agent(
+        model=model if model is not None else MockLLMClient.deterministic("ok"),
+        tools=tools if tools is not None else [],
+        strategy=strategy if strategy is not None else _SingleStepStrategy(),
+        memory=overrides.pop("memory", None) or InMemoryStore(),
+        budget_usd=overrides.pop("budget_usd", 0.10),
+        max_iterations=overrides.pop("max_iterations", 3),
+        install_log_filter=overrides.pop("install_log_filter", False),
+        **overrides,
+    )
+
+
+__all__ = ["agent_factory"]

agentforge/testing/fixtures.py

@@ -0,0 +1,42 @@
+"""Pytest fixtures (feat-016).
+
+Importing this module registers the fixtures into the active
+session via `pytest.fixture` — typical usage is to re-export the
+two fixtures from a project's `conftest.py`:
+
+    # tests/conftest.py
+    from agentforge.testing.fixtures import mock_llm, temp_memory_store
+
+The fixtures are framework-agnostic in spirit but the decorator is
+pytest-specific. Other test runners can construct the helpers
+themselves via `MockLLMClient.deterministic(...)` and
+`InMemoryStore()` directly.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+
+import pytest
+
+from agentforge.memory import InMemoryStore
+from agentforge.testing.llm import MockLLMClient
+
+
+@pytest.fixture
+def mock_llm() -> MockLLMClient:
+    """A fresh `MockLLMClient.deterministic("ok")` per test."""
+    return MockLLMClient.deterministic("ok")
+
+
+@pytest.fixture
+async def temp_memory_store() -> AsyncIterator[InMemoryStore]:
+    """An `InMemoryStore` that's cleared after the test."""
+    store = InMemoryStore()
+    try:
+        yield store
+    finally:
+        await store.close()
+
+
+__all__ = ["mock_llm", "temp_memory_store"]

agentforge/testing/llm.py

@@ -0,0 +1,235 @@
+"""`MockLLMClient` — the v0.1 public scripted-response `LLMClient`.
+
+Richer than the private `_testing.FakeLLMClient`:
+
+- `MockLLMClient.from_script([...])` accepts a list of dict
+  "scripted responses" with text / tool_calls / stop_reason / usage
+  — far less boilerplate than constructing `LLMResponse` by hand.
+- `MockLLMClient.deterministic("answer")` always returns the same
+  short response, perfect for the `agent_factory` default.
+- `MockLLMClient.from_recording(path)` (added in chunk 3) replays
+  a recorded transcript.
+- Tracks `call_count` and `tool_calls_observed` (a list of every
+  `(tool_name, arguments)` pair the script emitted) so tests can
+  assert on what the agent asked the LLM to call.
+
+Implementation defers as much as possible to the existing
+`FakeLLMClient` so we share its captured-call ergonomics; the
+public surface is what changed.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+from agentforge_core.contracts.llm import LLMClient
+from agentforge_core.production.exceptions import ModuleError
+from agentforge_core.values.messages import (
+    LLMResponse,
+    Message,
+    StopReason,
+    TokenUsage,
+    ToolCall,
+    ToolSpec,
+)
+
+ScriptedResponse = dict[str, Any]
+"""Lightweight scripted response.
+
+Keys (all optional except `text`):
+- ``text``: str — the assistant's content.
+- ``tool_calls``: list[{name, args, id?}] — emitted tool calls.
+- ``stop_reason``: "end_turn" | "tool_use" | ... — defaults to
+  "tool_use" when tool_calls are present, "end_turn" otherwise.
+- ``usage``: {input_tokens, output_tokens, ...} — defaults to a
+  small non-zero usage.
+- ``cost_usd``, ``model``, ``provider`` — defaults to 0.0 / "mock"
+  / "mock".
+"""
+
+
+class MockLLMClient(LLMClient):
+    """Scripted-response `LLMClient`. Public test helper."""
+
+    def __init__(
+        self,
+        *,
+        responses: list[LLMResponse],
+        provider: str = "mock",
+        model: str = "mock",
+    ) -> None:
+        self._responses = list(responses)
+        self._cursor = 0
+        self._observed_tool_calls: list[tuple[str, dict[str, Any]]] = []
+        self._provider = provider
+        self._model = model
+
+    # ------------------------------------------------------------------
+    # Factories
+    # ------------------------------------------------------------------
+
+    @classmethod
+    def from_script(
+        cls,
+        responses: list[ScriptedResponse],
+        *,
+        provider: str = "mock",
+        model: str = "mock",
+    ) -> MockLLMClient:
+        """Build a mock from a list of lightweight dict responses."""
+        return cls(
+            responses=[_scripted_to_llm_response(r, provider, model) for r in responses],
+            provider=provider,
+            model=model,
+        )
+
+    @classmethod
+    def from_recording(
+        cls,
+        path: str,
+        *,
+        provider: str = "mock",
+        model: str = "mock",
+    ) -> MockLLMClient:
+        """Replay a recording produced by `record_llm`.
+
+        Responses are returned in on-disk order — same as the
+        original real-provider sequence. Matching by request hash
+        is exposed for callers that need it; the default replay
+        is sequential because tests usually exercise the same flow
+        as the recording.
+        """
+        from agentforge.testing.recording import load_recording  # noqa: PLC0415
+
+        _, entries = load_recording(path)
+        responses: list[LLMResponse] = []
+        for entry in entries:
+            resp = dict(entry["response"])
+            # Pydantic strict mode requires tuples for tuple-typed
+            # fields; JSON serialisation lowered them to lists.
+            if isinstance(resp.get("tool_calls"), list):
+                resp["tool_calls"] = tuple(resp["tool_calls"])
+            responses.append(LLMResponse.model_validate(resp))
+        return cls(responses=responses, provider=provider, model=model)
+
+    @classmethod
+    def deterministic(
+        cls,
+        response: str,
+        *,
+        provider: str = "mock",
+        model: str = "mock",
+    ) -> MockLLMClient:
+        """Always returns the same single-response transcript.
+
+        Useful for `agent_factory()` defaults: a one-shot reply
+        with no tool calls, no cost, that lets `Agent.run` complete
+        cleanly in unit tests.
+        """
+        return cls.from_script(
+            [{"text": response, "stop_reason": "end_turn"}],
+            provider=provider,
+            model=model,
+        )
+
+    # ------------------------------------------------------------------
+    # Observation surface
+    # ------------------------------------------------------------------
+
+    @property
+    def call_count(self) -> int:
+        """Number of `.call()` invocations so far."""
+        return self._cursor
+
+    @property
+    def tool_calls_observed(self) -> list[tuple[str, dict[str, Any]]]:
+        """Every (tool_name, arguments) pair the script has emitted.
+
+        Tests use this to assert on the agent's tool-call sequence
+        without having to inspect each `LLMResponse` manually.
+        """
+        return list(self._observed_tool_calls)
+
+    # ------------------------------------------------------------------
+    # LLMClient contract
+    # ------------------------------------------------------------------
+
+    async def call(
+        self,
+        system: str,
+        messages: list[Message],
+        tools: list[ToolSpec] | None = None,
+    ) -> LLMResponse:
+        del system, messages, tools
+        if self._cursor >= len(self._responses):
+            msg = (
+                f"MockLLMClient exhausted after {self._cursor} call(s); "
+                f"add more scripted responses or check the strategy loop."
+            )
+            raise ModuleError(msg)
+        response = self._responses[self._cursor]
+        self._cursor += 1
+        for tc in response.tool_calls:
+            self._observed_tool_calls.append((tc.name, dict(tc.arguments)))
+        return response
+
+    async def close(self) -> None:
+        return
+
+
+def _scripted_to_llm_response(
+    spec: ScriptedResponse,
+    provider: str,
+    model: str,
+) -> LLMResponse:
+    """Normalise a `ScriptedResponse` dict into an `LLMResponse`."""
+    text = spec.get("text", "")
+    raw_tool_calls = spec.get("tool_calls", []) or []
+    tool_calls = tuple(
+        ToolCall(
+            id=tc.get("id") or _synth_tool_id(tc),
+            name=tc["name"],
+            arguments=dict(tc.get("args", tc.get("arguments", {}))),
+        )
+        for tc in raw_tool_calls
+    )
+    stop_reason: StopReason = spec.get(
+        "stop_reason",
+        "tool_use" if tool_calls else "end_turn",
+    )
+    usage_raw: dict[str, Any] = spec.get("usage") or {}
+    usage = TokenUsage(
+        input_tokens=int(usage_raw.get("input_tokens", 1)),
+        output_tokens=int(usage_raw.get("output_tokens", 1)),
+        cache_read_tokens=int(usage_raw.get("cache_read_tokens", 0)),
+        cache_write_tokens=int(usage_raw.get("cache_write_tokens", 0)),
+        thinking_tokens=int(usage_raw.get("thinking_tokens", 0)),
+    )
+    return LLMResponse(
+        content=text,
+        tool_calls=tool_calls,
+        stop_reason=stop_reason,
+        usage=usage,
+        cost_usd=float(spec.get("cost_usd", 0.0)),
+        model=spec.get("model", model),
+        provider=spec.get("provider", provider),
+    )
+
+
+def _synth_tool_id(tc: dict[str, Any]) -> str:
+    """Synthesize a stable tool-call id from name + arguments.
+
+    Real providers always set their own id; the mock fills one in
+    when the script omits it so downstream code that keys on
+    ToolCall.id keeps working.
+    """
+    serialised = json.dumps(
+        {"name": tc["name"], "args": tc.get("args", tc.get("arguments", {}))},
+        sort_keys=True,
+    )
+    return "mock-" + hashlib.sha256(serialised.encode("utf-8")).hexdigest()[:12]
+
+
+__all__ = ["MockLLMClient", "ScriptedResponse"]

agentforge/testing/recording.py

@@ -0,0 +1,177 @@
+"""LLM recording + replay (feat-016 chunk 3).
+
+`record_llm(real_client, path)` returns a wrapper that proxies
+every `.call(...)` to the real provider and appends one JSON line
+to `path` recording `{request_hash, request, response}`. Tests
+replay via `MockLLMClient.from_recording(path)` which matches by
+request hash so reordered calls still work.
+
+JSONL format (versioned):
+
+    {"format_version": 1, "redactions": ["api_key", "authorization"]}
+    {"request_hash": "...", "request": {...}, "response": {...}}
+    {"request_hash": "...", "request": {...}, "response": {...}}
+
+Header always at line 0. Recordings made under format_version 1
+remain loadable when the version bumps.
+
+Redaction applies to system / messages / metadata fields whose
+key (case-insensitive) matches one of the redactions list. Values
+are replaced with `"<redacted>"` before write. By default,
+`api_key` and `authorization` are redacted; pass `redactions=[...]`
+to extend or replace.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+from agentforge_core.contracts.llm import LLMClient
+from agentforge_core.production.exceptions import ModuleError
+from agentforge_core.values.messages import LLMResponse, Message, ToolSpec
+
+_FORMAT_VERSION = 1
+_DEFAULT_REDACTIONS: tuple[str, ...] = ("api_key", "authorization", "bearer")
+
+
+class _RecordingLLMClient(LLMClient):
+    """Wraps a real `LLMClient`; records each call to a JSONL path."""
+
+    def __init__(
+        self,
+        *,
+        real: LLMClient,
+        path: Path,
+        redactions: tuple[str, ...],
+    ) -> None:
+        self._real = real
+        self._path = path
+        self._redactions = redactions
+        self._initialised = False
+
+    async def call(
+        self,
+        system: str,
+        messages: list[Message],
+        tools: list[ToolSpec] | None = None,
+    ) -> LLMResponse:
+        response = await self._real.call(system, messages, tools)
+        self._ensure_header()
+        request = _redact(
+            {
+                "system": system,
+                "messages": [m.model_dump(mode="json") for m in messages],
+                "tools": [t.model_dump(mode="json", by_alias=True) for t in (tools or [])],
+            },
+            self._redactions,
+        )
+        record = {
+            "request_hash": _hash_request(request),
+            "request": request,
+            "response": response.model_dump(mode="json"),
+        }
+        with self._path.open("a", encoding="utf-8") as f:
+            f.write(json.dumps(record) + "\n")
+        return response
+
+    async def close(self) -> None:
+        await self._real.close()
+
+    def _ensure_header(self) -> None:
+        if self._initialised:
+            return
+        # Header is written exactly once. Re-recording over an
+        # existing file replaces the file outright; appending without
+        # a header would corrupt format detection.
+        if not self._path.exists() or self._path.stat().st_size == 0:
+            self._path.parent.mkdir(parents=True, exist_ok=True)
+            with self._path.open("w", encoding="utf-8") as f:
+                f.write(
+                    json.dumps(
+                        {
+                            "format_version": _FORMAT_VERSION,
+                            "redactions": list(self._redactions),
+                        }
+                    )
+                    + "\n"
+                )
+        self._initialised = True
+
+
+def record_llm(
+    real: LLMClient,
+    path: str | Path,
+    *,
+    redactions: tuple[str, ...] | None = None,
+) -> LLMClient:
+    """Return an `LLMClient` that proxies `real` and records to `path`.
+
+    Replay via `MockLLMClient.from_recording(path)`.
+
+    Redactions default to `("api_key", "authorization", "bearer")`.
+    Pass an empty tuple to disable redaction entirely (only for
+    truly local recordings).
+    """
+    return _RecordingLLMClient(
+        real=real,
+        path=Path(path),
+        redactions=redactions if redactions is not None else _DEFAULT_REDACTIONS,
+    )
+
+
+def load_recording(path: str | Path) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+    """Parse a recording file into `(header, entries)`.
+
+    `header` carries `format_version` + `redactions`. `entries` is
+    the list of `{request_hash, request, response}` records in
+    on-disk order.
+    """
+    p = Path(path)
+    if not p.exists():
+        msg = f"recording {p!r} does not exist."
+        raise ModuleError(msg)
+    lines = [line for line in p.read_text(encoding="utf-8").splitlines() if line.strip()]
+    if not lines:
+        msg = f"recording {p!r} is empty."
+        raise ModuleError(msg)
+    header = json.loads(lines[0])
+    if header.get("format_version") != _FORMAT_VERSION:
+        msg = (
+            f"recording {p!r}: format_version={header.get('format_version')!r} "
+            f"not supported (this build accepts version {_FORMAT_VERSION})."
+        )
+        raise ModuleError(msg)
+    entries = [json.loads(line) for line in lines[1:]]
+    return header, entries
+
+
+def _hash_request(request: dict[str, Any]) -> str:
+    """Stable hash of a redacted request for replay lookup.
+
+    Hashing happens AFTER redaction so the cassette is portable
+    across environments with different api_key values.
+    """
+    blob = json.dumps(request, sort_keys=True).encode("utf-8")
+    return hashlib.sha256(blob).hexdigest()[:16]
+
+
+def _redact(payload: Any, keys: tuple[str, ...]) -> Any:
+    """Recursively replace any dict-value whose key matches one of
+    `keys` (case-insensitive) with `"<redacted>"`."""
+    if isinstance(payload, dict):
+        result: dict[str, Any] = {}
+        for k, v in payload.items():
+            if isinstance(k, str) and k.lower() in keys:
+                result[k] = "<redacted>"
+            else:
+                result[k] = _redact(v, keys)
+        return result
+    if isinstance(payload, list):
+        return [_redact(item, keys) for item in payload]
+    return payload
+
+
+__all__ = ["load_recording", "record_llm"]

agentforge/tools/__init__.py

@@ -0,0 +1,41 @@
+"""Default tools shipped with `agentforge` (feat-004).
+
+Public surface — pre-built `Tool` instances ready to pass into
+`Agent(tools=[...])`. Implementations live under
+`agentforge._tools/`; this module re-exports them.
+
+```python
+from agentforge import Agent
+from agentforge.tools import calculator, file_read
+
+agent = Agent(model="...", tools=[calculator, file_read])
+```
+
+For tools that take configuration (e.g., `FileReadTool` with a
+custom sandbox and size cap), import the class directly:
+
+```python
+from agentforge.tools import FileReadTool
+
+custom = FileReadTool(work_dir="/srv/data", max_bytes=10_485_760)
+agent = Agent(model="...", tools=[custom])
+```
+"""
+
+from __future__ import annotations
+
+from agentforge._tools.calculator import calculator
+from agentforge._tools.file_read import FileReadTool, file_read
+from agentforge._tools.shell import ShellTool, shell
+from agentforge._tools.web_search import SearchResult, WebSearchTool, web_search
+
+__all__ = [
+    "FileReadTool",
+    "SearchResult",
+    "ShellTool",
+    "WebSearchTool",
+    "calculator",
+    "file_read",
+    "shell",
+    "web_search",
+]