benchpress-agent 0.1.0__py3-none-any.whl

benchpress/__init__.py

@@ -0,0 +1,36 @@
+"""Benchpress — the reliability layer for AI agents with write access.
+
+Policy-first, authority-gated, evidence-verified execution for multi-app agents.
+Nothing in this package is keyed to a task id, a seeded name, or a seeded domain:
+every task fact is discovered at runtime from the prompt and from provider reads.
+
+    import benchpress
+
+    agent = benchpress.wrap("deepseek-v4-pro", gateway.execute_tool, providers=["hubspot", "stripe"])
+    result = await agent.run("Move Acme's renewal notices to ap@acme.example")
+"""
+
+from __future__ import annotations
+
+from benchpress.api import DEFAULT_SYSTEM_PROMPT, Benchpress, wrap
+from benchpress.controller import TrialResult, run_trial
+from benchpress.gate import Gate, GateRefusal
+from benchpress.model import ModelConfig
+from benchpress.phases.common import Ablations
+from benchpress.tools import ToolExecutor
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "DEFAULT_SYSTEM_PROMPT",
+    "Ablations",
+    "Benchpress",
+    "Gate",
+    "GateRefusal",
+    "ModelConfig",
+    "ToolExecutor",
+    "TrialResult",
+    "__version__",
+    "run_trial",
+    "wrap",
+]

benchpress/adapter.py

@@ -0,0 +1,138 @@
+"""The ArgaBench candidate contract: `invoke(model_id, system_prompt, user_prompt, tool_schema,
+execute_tool, max_tool_calls, timeout_seconds, *, api_effort, thinking)`.
+
+`src/benchpress` never imports the harness, so this returns an `InvocationRecord` with the
+same fields as the harness's `ModelInvocationResult`; the harness-side shim converts it.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping, Sequence
+from dataclasses import asdict, dataclass, field, replace
+from pathlib import Path
+from typing import Any, Literal, cast
+
+from benchpress.controller import run_trial
+from benchpress.model import ModelConfig
+from benchpress.phases.common import Ablations
+from benchpress.tools import ToolExecutor
+
+KNOWN_ROLES: frozenset[str] = frozenset(
+    {
+        "code_host",
+        "email",
+        "calendar",
+        "file_storage",
+        "hubspot_crm",
+        "jira_tracker",
+        "linear_tracker",
+        "professional_network",
+        "knowledge_base",
+        "salesforce_crm",
+        "team_chat",
+        "payments",
+    }
+)
+
+
+@dataclass(frozen=True)
+class InvocationRecord:
+    requested_model: str
+    response_model: str | None
+    provider: Literal["anthropic", "openai", "google"]
+    final_text: str
+    status: str
+    stop_reason: str
+    system_prompt: str
+    user_prompt: str
+    events: tuple[dict[str, Any], ...] = ()
+    usage: dict[str, Any] = field(default_factory=lambda: dict[str, Any]())
+    config: dict[str, Any] = field(default_factory=lambda: dict[str, Any]())
+    latency_ms: int = 0
+    tool_calls: int = 0
+
+    def as_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+def providers_from_tool_schema(tool_schema: object) -> list[str]:
+    """Provisioned provider names from the `provider_api` tool's enum (roles excluded)."""
+    schemas: Sequence[Mapping[str, Any]]
+    if isinstance(tool_schema, Mapping):
+        schemas = [cast(Mapping[str, Any], tool_schema)]
+    else:
+        schemas = cast(Sequence[Mapping[str, Any]], tool_schema)
+    for schema in schemas:
+        if schema.get("name") != "provider_api":
+            continue
+        parameters = cast(Mapping[str, Any], schema.get("input_schema", schema.get("parameters", {})))
+        properties = cast(Mapping[str, Any], parameters.get("properties", {}))
+        provider = cast(Mapping[str, Any], properties.get("provider", {}))
+        enum = cast(Sequence[object], provider.get("enum", []))
+        return [str(item) for item in enum if str(item) not in KNOWN_ROLES]
+    return []
+
+
+async def invoke(
+    model_id: str,
+    system_prompt: str,
+    user_prompt: str,
+    tool_schema: object,
+    execute_tool: ToolExecutor,
+    max_tool_calls: int,
+    timeout_seconds: float,
+    *,
+    api_effort: str = "default",
+    thinking: str = "model_default",
+    ablations: Ablations | None = None,
+    trace_dir: Path | None = None,
+    trial_id: str | None = None,
+) -> InvocationRecord:
+    inner_model = model_id.split("/", 1)[1] if model_id.startswith("benchpress/") else model_id
+    config = ModelConfig.from_env(model=inner_model)
+    if api_effort and api_effort != "default":
+        config = replace(config, effort=api_effort)
+    resolved_ablations = ablations or Ablations.parse(os.environ.get("BENCHPRESS_ABLATIONS"))
+    result = await run_trial(
+        system_prompt=system_prompt,
+        user_prompt=user_prompt,
+        providers=providers_from_tool_schema(tool_schema),
+        execute_tool=execute_tool,
+        config=config,
+        ablations=resolved_ablations,
+        trace_dir=trace_dir,
+        trial_id=trial_id,
+    )
+    if result.error == "timeout":
+        status, stop_reason = "timed_out", "timeout"
+    elif result.error:
+        status, stop_reason = "incomplete", result.error[:80]
+    else:
+        status, stop_reason = "completed", "end_turn"
+    provider: Literal["anthropic", "openai", "google"] = "anthropic" if inner_model.startswith("claude") else "openai"
+    return InvocationRecord(
+        requested_model=model_id,
+        response_model=model_id,
+        final_text=result.final_text,
+        provider=provider,
+        status=status,
+        stop_reason=stop_reason,
+        system_prompt=system_prompt,
+        user_prompt=user_prompt,
+        events=result.tool_events,
+        usage=result.usage.harness_usage(),
+        config={
+            "model": inner_model,
+            "provider": config.provider,
+            "effort": config.effort,
+            "thinking": {"type": thinking},
+            "scaffold": "benchpress",
+            "ablations": list(result.ablations),
+            "max_tool_calls": max_tool_calls,
+            "timeout_seconds": timeout_seconds,
+            "benchpress_status": result.status,
+        },
+        latency_ms=result.latency_ms,
+        tool_calls=result.provider_calls + result.docs_calls,
+    )

benchpress/api.py

@@ -0,0 +1,133 @@
+"""The public entry point: `wrap(model, executor)` puts the Benchpress loop around any tool layer.
+
+    agent = benchpress.wrap("deepseek-v4-pro", gateway.execute_tool, providers=["hubspot", "stripe"])
+    result = await agent.run("Move Acme's renewal notices to ap@acme.example")
+
+The executor is anything with the harness `execute_tool(tool_name, tool_input)` shape, either a
+coroutine function or an object exposing `execute_tool`. Every write it receives has already passed
+the gate; every run ends with an evidence-only status and, when `trace_dir` is set, a receipt.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Protocol, runtime_checkable
+
+from benchpress.controller import TrialResult, run_trial
+from benchpress.model import ModelClient, ModelConfig, ModelTransport
+from benchpress.phases.common import Ablations
+from benchpress.playbooks import Playbook
+from benchpress.prompts import system_text
+from benchpress.tools import ToolExecutor
+
+DEFAULT_SYSTEM_PROMPT = (
+    "You are an operations agent working across the provisioned business systems. Complete the request "
+    "through the provider_api tool using ordinary data-plane routes only. Minimize mutations, respect every "
+    "explicit prohibition, verify the final state through reads, and return exactly the output requested."
+)
+
+
+@runtime_checkable
+class HasExecuteTool(Protocol):
+    def execute_tool(self, tool_name: str, tool_input: dict[str, Any]) -> Awaitable[object]: ...
+
+
+type ExecutorLike = ToolExecutor | HasExecuteTool
+type ModelLike = str | ModelConfig | None
+
+
+@dataclass(frozen=True)
+class Benchpress:
+    """A configured loop. Stateless between runs: each `run` gets a fresh context, gate and model client."""
+
+    executor: ToolExecutor
+    providers: tuple[str, ...]
+    config: ModelConfig
+    system_prompt: str = DEFAULT_SYSTEM_PROMPT
+    ablations: Ablations = field(default_factory=Ablations)
+    playbooks: Mapping[str, Playbook] | None = None
+    transport: ModelTransport | None = None
+
+    async def run(
+        self,
+        request: str,
+        *,
+        trace_dir: str | Path | None = None,
+        trial_id: str | None = None,
+    ) -> TrialResult:
+        if not request.strip():
+            raise ValueError("request must be a non-empty string")
+        client = None
+        if self.transport is not None:
+            client = ModelClient(self.config, system_text(self.system_prompt), transport=self.transport)
+        return await run_trial(
+            system_prompt=self.system_prompt,
+            user_prompt=request,
+            providers=self.providers,
+            execute_tool=self.executor,
+            config=self.config,
+            ablations=self.ablations,
+            trace_dir=Path(trace_dir) if trace_dir is not None else None,
+            trial_id=trial_id,
+            model_client=client,
+            playbooks=self.playbooks,
+        )
+
+    def run_sync(
+        self,
+        request: str,
+        *,
+        trace_dir: str | Path | None = None,
+        trial_id: str | None = None,
+    ) -> TrialResult:
+        """`run` for synchronous callers. Raises if called from inside a running event loop."""
+        return asyncio.run(self.run(request, trace_dir=trace_dir, trial_id=trial_id))
+
+
+def _config(model: ModelLike) -> ModelConfig:
+    if isinstance(model, ModelConfig):
+        return model
+    return ModelConfig.from_env(model=model)
+
+
+def _executor(executor: ExecutorLike) -> ToolExecutor:
+    if isinstance(executor, HasExecuteTool):
+        return executor.execute_tool
+    if callable(executor):
+        return executor
+    raise TypeError("executor must be an async callable (tool_name, tool_input) or expose execute_tool")
+
+
+def wrap(
+    model: ModelLike,
+    executor: ExecutorLike,
+    *,
+    providers: Sequence[str],
+    system_prompt: str = DEFAULT_SYSTEM_PROMPT,
+    ablations: Ablations | None = None,
+    playbooks: Mapping[str, Playbook] | None = None,
+    transport: ModelTransport | None = None,
+) -> Benchpress:
+    """Wrap a tool layer in the Benchpress loop.
+
+    `model` is a model id (credentials from the environment: `DEEPSEEK_API_KEY`/`BENCHPRESS_API_KEY`
+    for OpenAI-compatible endpoints, `ANTHROPIC_API_KEY` for `claude-*`), a full `ModelConfig`, or
+    None for `BENCHPRESS_MODEL`. `providers` names the systems the executor can reach; built-in
+    playbooks cover slack, gmail, hubspot and stripe, and `playbooks` supplies your own. `transport`
+    swaps the model wire protocol (bring your own LLM client, or a scripted one in tests).
+    """
+    names = tuple(name.strip() for name in providers if name.strip())
+    if not names:
+        raise ValueError("providers must name at least one system the executor can reach")
+    return Benchpress(
+        executor=_executor(executor),
+        providers=names,
+        config=_config(model),
+        system_prompt=system_prompt,
+        ablations=ablations or Ablations(),
+        playbooks=playbooks,
+        transport=transport,
+    )

benchpress/cli.py

@@ -0,0 +1,112 @@
+"""`benchpress` command line: run a request against real apps or local twins, print receipts."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import importlib
+import json
+import os
+import sys
+from collections.abc import Sequence
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, Protocol, cast
+
+from benchpress.api import DEFAULT_SYSTEM_PROMPT
+from benchpress.controller import run_trial
+from benchpress.model import ModelConfig
+from benchpress.phases.common import Ablations
+from benchpress.receipt_html import write_receipt_html
+from benchpress.report import receipt_summary
+
+
+class GatewayLike(Protocol):
+    async def execute_tool(self, tool_name: str, tool_input: dict[str, Any]) -> object: ...
+
+    async def aclose(self) -> None: ...
+
+
+class GatewayFactory(Protocol):
+    def __call__(self, providers: Sequence[str]) -> GatewayLike: ...
+
+
+def _load_env() -> None:
+    try:
+        from dotenv import load_dotenv
+    except ImportError:  # pragma: no cover - dotenv is a dev dependency
+        return
+    load_dotenv(Path.cwd() / ".env")
+
+
+async def _run(args: argparse.Namespace) -> int:
+    prompt = Path(args.prompt_file).read_text() if args.prompt_file else str(args.prompt or "")
+    if not prompt.strip():
+        print("a --prompt or --prompt-file is required", file=sys.stderr)
+        return 2
+    providers = [name.strip() for name in str(args.providers).split(",") if name.strip()]
+    try:
+        realapp = importlib.import_module("benchpress.realapp")
+    except ImportError as exc:
+        print(f"real-app gateway unavailable: {exc}", file=sys.stderr)
+        return 2
+    factory = cast(GatewayFactory, getattr(realapp, "gateway_from_env"))  # noqa: B009 - module resolved at runtime
+    gateway = factory(providers)
+    trace_dir = Path(args.trace_dir or f"runs/local/{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}")
+    config = ModelConfig.from_env(model=args.model)
+    result = await run_trial(
+        system_prompt=DEFAULT_SYSTEM_PROMPT,
+        user_prompt=prompt,
+        providers=providers,
+        execute_tool=gateway.execute_tool,
+        config=config,
+        ablations=Ablations.parse(args.ablations),
+        trace_dir=trace_dir,
+    )
+    await gateway.aclose()
+    receipt = json.loads((trace_dir / "receipt.json").read_text())
+    print(receipt_summary(receipt))
+    print(f"\nfinal:\n{result.final_text}")
+    print(f"\nreceipt: {trace_dir / 'receipt.json'}")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    _load_env()
+    parser = argparse.ArgumentParser(
+        prog="benchpress", description="the reliability layer for agents with write access"
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+    run = sub.add_parser("run", help="run one request end to end")
+    run.add_argument("--prompt")
+    run.add_argument("--prompt-file")
+    run.add_argument("--providers", default="slack,gmail,hubspot,stripe")
+    run.add_argument("--trace-dir")
+    run.add_argument("--model", default=os.environ.get("BENCHPRESS_MODEL"))
+    run.add_argument("--ablations", default=os.environ.get("BENCHPRESS_ABLATIONS"))
+    receipt = sub.add_parser("receipt", help="print a receipt summary")
+    receipt.add_argument("path")
+    receipt.add_argument(
+        "--html",
+        nargs="?",
+        const="",
+        metavar="OUT",
+        help="also render the self-contained receipt page; defaults to receipt.html next to the JSON",
+    )
+    args = parser.parse_args(argv)
+    if args.command == "run":
+        return asyncio.run(_run(args))
+    if args.command == "receipt":
+        path = Path(args.path)
+        if path.is_dir():
+            path = path / "receipt.json"
+        print(receipt_summary(json.loads(path.read_text())))
+        if args.html is not None:
+            out = Path(args.html) if args.html else path.with_suffix(".html")
+            print(f"\nreceipt page: {write_receipt_html(path, out)}")
+        return 0
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())