algo-cli-runtime 0.14.0__py3-none-any.whl

algo_cli/intelligence/agent_benchmark.py

@@ -0,0 +1,236 @@
+"""B47. Gym-like Agent Evaluation Environment (TextWorld Pattern).
+
+Benchmark Algo CLI agent performance with reproducible episodes, score
+tracking, and step limits.  Uses a Gym-like API: reset() → (obs, infos),
+step(action) → (obs, score, done, infos).
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+
+@dataclass
+class EpisodeResult:
+    task: str
+    steps: int
+    max_steps: int
+    score: float
+    done: bool
+    success: bool
+    tool_calls: list[dict] = field(default_factory=list)
+    errors: list[str] = field(default_factory=list)
+    trace: list[dict] = field(default_factory=list)
+    duration_ms: float = 0.0
+
+
+@dataclass
+class BenchmarkScenario:
+    """A single benchmark scenario definition."""
+    name: str
+    task: str
+    seed: int = 42
+    max_steps: int = 50
+    setup_fn: Callable[[], dict] | None = None
+    verify_fn: Callable[[str, dict], tuple[bool, float]] | None = None
+    # verify_fn returns (success, score)
+
+
+class AgentEnv:
+    """Gym-like environment for a single benchmark episode."""
+
+    def __init__(self, scenario: BenchmarkScenario):
+        self.scenario = scenario
+        self.task = scenario.task
+        self.seed = scenario.seed
+        self.max_steps = scenario.max_steps
+        self._step = 0
+        self._done = False
+        self._score = 0.0
+        self._tool_calls: list[dict] = []
+        self._errors: list[str] = []
+        self._trace: list[dict] = []
+        self._state: dict[str, Any] = {}
+
+    def reset(self) -> tuple[str, dict]:
+        """Reset environment to initial state."""
+        self._step = 0
+        self._done = False
+        self._score = 0.0
+        self._tool_calls = []
+        self._errors = []
+        self._trace = []
+        if self.scenario.setup_fn:
+            self._state = self.scenario.setup_fn()
+        else:
+            self._state = {}
+        return self.task, {"step": 0, "state": self._state}
+
+    def step(self, action: dict) -> tuple[str, float, bool, dict]:
+        """Execute one agent action.
+
+        action is a dict with:
+        - "type": "tool_call", "response", "done"
+        - "tool": tool name (for tool_call)
+        - "params": tool params
+        - "content": response text
+        """
+        self._step += 1
+        action_type = action.get("type", "response")
+        reward = 0.0
+
+        if action_type == "tool_call":
+            self._tool_calls.append({
+                "tool": action.get("tool", ""),
+                "params": action.get("params", {}),
+                "step": self._step,
+            })
+            self._trace.append({"step": self._step, "type": "tool_call", "tool": action.get("tool", "")})
+            # Small penalty for each tool call (encourages efficiency)
+            reward = -0.1
+
+        elif action_type == "response":
+            content = action.get("content", "")
+            self._trace.append({"step": self._step, "type": "response", "content": content[:200]})
+            # Check if the response is correct
+            if self.scenario.verify_fn:
+                success, score = self.scenario.verify_fn(content, self._state)
+                reward = score
+                if success:
+                    self._done = True
+            else:
+                reward = 0.0
+
+        elif action_type == "done":
+            self._done = True
+
+        elif action_type == "error":
+            self._errors.append(action.get("content", "unknown error"))
+            reward = -0.5
+
+        self._score += reward
+
+        if self._step >= self.max_steps:
+            self._done = True
+
+        infos = {
+            "step": self._step,
+            "score": self._score,
+            "tool_calls": list(self._tool_calls),
+            "errors": list(self._errors),
+            "trace": list(self._trace),
+            "success": self._done and self._score > 0,
+        }
+        return "", reward, self._done, infos
+
+
+class AgentBenchmark:
+    """Runs benchmark episodes and aggregates results."""
+
+    def __init__(self, max_steps: int = 50):
+        self.max_steps = max_steps
+        self.scenarios: dict[str, BenchmarkScenario] = {}
+        self.results: list[EpisodeResult] = []
+
+    def add_scenario(self, scenario: BenchmarkScenario) -> None:
+        self.scenarios[scenario.name] = scenario
+
+    def run_episode(self, scenario_name: str, agent_fn: Callable[[str, dict], dict]) -> EpisodeResult:
+        """Run a single episode with the given agent function."""
+        scenario = self.scenarios.get(scenario_name)
+        if not scenario:
+            return EpisodeResult(
+                task="", steps=0, max_steps=0, score=0,
+                done=True, success=False, errors=["scenario not found"],
+            )
+        env = AgentEnv(scenario)
+        obs, infos = env.reset()
+        start = time.monotonic()
+        while not env._done:
+            action = agent_fn(obs, infos)
+            obs, reward, done, infos = env.step(action)
+            if done:
+                break
+        duration = (time.monotonic() - start) * 1000
+        result = EpisodeResult(
+            task=scenario.task,
+            steps=env._step,
+            max_steps=scenario.max_steps,
+            score=env._score,
+            done=env._done,
+            success=infos.get("success", False),
+            tool_calls=env._tool_calls,
+            errors=env._errors,
+            trace=env._trace,
+            duration_ms=duration,
+        )
+        self.results.append(result)
+        return result
+
+    def report(self) -> dict[str, Any]:
+        """Aggregate results across all episodes."""
+        if not self.results:
+            return {"total": 0}
+        scores = [r.score for r in self.results]
+        steps = [r.steps for r in self.results]
+        successes = sum(1 for r in self.results if r.success)
+        sorted_scores = sorted(scores)
+        n = len(sorted_scores)
+        p50 = sorted_scores[n // 2]
+        p90 = sorted_scores[int(n * 0.9)] if n > 1 else sorted_scores[0]
+        return {
+            "total": len(self.results),
+            "successes": successes,
+            "success_rate": successes / len(self.results),
+            "avg_score": sum(scores) / len(scores),
+            "p50_score": p50,
+            "p90_score": p90,
+            "avg_steps": sum(steps) / len(steps),
+            "avg_duration_ms": sum(r.duration_ms for r in self.results) / len(self.results),
+        }
+
+
+# ── built-in scenarios ────────────────────────────────────────────────
+
+
+def make_find_file_scenario() -> BenchmarkScenario:
+    """Scenario: find a file and extract a value."""
+    state = {"target_file": "EST-2026-002_200A.html", "target_value": "8291.60"}
+
+    def setup() -> dict:
+        return state
+
+    def verify(content: str, st: dict) -> tuple[bool, float]:
+        if st["target_value"] in content:
+            return True, 1.0
+        return False, 0.0
+
+    return BenchmarkScenario(
+        name="find_file",
+        task=f"Find the file {state['target_file']} and extract the total price.",
+        setup_fn=setup,
+        verify_fn=verify,
+        max_steps=10,
+    )
+
+
+def make_pdf_extract_scenario() -> BenchmarkScenario:
+    """Scenario: extract line items from a PDF."""
+
+    def setup() -> dict:
+        return {"expected_items": ["panel", "breakers", "conductors", "meter base", "grounding"]}
+
+    def verify(content: str, st: dict) -> tuple[bool, float]:
+        found = sum(1 for item in st["expected_items"] if item.lower() in content.lower())
+        score = found / len(st["expected_items"])
+        return (score >= 0.8, score)
+
+    return BenchmarkScenario(
+        name="pdf_extract",
+        task="Read the attached PDF and list all line items.",
+        setup_fn=setup,
+        verify_fn=verify,
+        max_steps=15,
+    )

algo_cli/intelligence/agent_runtime.py

@@ -0,0 +1,171 @@
+"""B36. Layered Agent Runtime + Workbench Boundary (AutoGen Pattern).
+
+Separates agent message passing (runtime), agent logic (agent API), and
+tool execution (workbench) into distinct layers with a trust boundary.
+"""
+
+from __future__ import annotations
+
+import time
+import uuid
+from dataclasses import dataclass, field
+from typing import Any, Callable, Protocol
+
+
+# ── message layer ─────────────────────────────────────────────────────
+
+
+@dataclass
+class Message:
+    id: str
+    role: str  # "user", "assistant", "tool", "system"
+    content: str
+    agent_name: str = ""
+    tool_call_id: str = ""
+    timestamp: float = field(default_factory=time.time)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class TraceEvent:
+    type: str  # "message", "tool_call", "tool_result", "cancel", "handoff"
+    data: dict[str, Any] = field(default_factory=dict)
+    timestamp: float = field(default_factory=time.time)
+
+
+# ── runtime layer ─────────────────────────────────────────────────────
+
+
+class AgentRuntime:
+    """Message bus + trace recorder + cancellation support."""
+
+    def __init__(self) -> None:
+        self.messages: list[Message] = []
+        self.traces: list[TraceEvent] = []
+        self._cancelled = False
+        self._trace_id = str(uuid.uuid4())
+
+    @property
+    def trace_id(self) -> str:
+        return self._trace_id
+
+    def post(self, msg: Message) -> None:
+        self.messages.append(msg)
+        self.traces.append(TraceEvent(type="message", data={"role": msg.role, "agent": msg.agent_name}))
+
+    def cancel(self) -> None:
+        self._cancelled = True
+        self.traces.append(TraceEvent(type="cancel"))
+
+    @property
+    def cancelled(self) -> bool:
+        return self._cancelled
+
+    def get_history(self, agent_name: str | None = None) -> list[Message]:
+        if agent_name:
+            return [m for m in self.messages if m.agent_name == agent_name]
+        return list(self.messages)
+
+    def handoff(self, from_agent: str, to_agent: str, context: str = "") -> None:
+        self.traces.append(TraceEvent(
+            type="handoff",
+            data={"from": from_agent, "to": to_agent, "context": context},
+        ))
+
+
+# ── agent API layer ───────────────────────────────────────────────────
+
+
+class Agent(Protocol):
+    """Protocol for agent implementations."""
+
+    name: str
+    role: str
+
+    def respond(self, runtime: AgentRuntime, task: str) -> str: ...
+
+
+class SimpleAgent:
+    """A basic agent that uses a callable to generate responses."""
+
+    def __init__(self, name: str, role: str, fn: Callable[[str, list[Message]], str]):
+        self.name = name
+        self.role = role
+        self._fn = fn
+
+    def respond(self, runtime: AgentRuntime, task: str) -> str:
+        history = runtime.get_history()
+        result = self._fn(task, history)
+        runtime.post(Message(
+            id=str(uuid.uuid4()),
+            role="assistant",
+            content=result,
+            agent_name=self.name,
+        ))
+        return result
+
+
+# ── workbench layer (tool execution boundary) ────────────────────────
+
+
+@dataclass
+class ToolAction:
+    tool: str
+    params: dict[str, Any]
+    requires_approval: bool = False
+
+
+@dataclass
+class ToolResult:
+    tool: str
+    success: bool
+    output: str
+    error: str | None = None
+
+
+class Workbench:
+    """Untrusted tool execution boundary.
+
+    Tools cannot directly mutate files or send externally — they return
+    action proposals that the host gates.
+    """
+
+    def __init__(self, approval_fn: Callable[[ToolAction], bool] | None = None):
+        self._tools: dict[str, Callable[[dict], str]] = {}
+        self._approval_fn = approval_fn or (lambda a: not a.requires_approval)
+        self._allowlisted: set[str] = set()
+
+    def register_tool(self, name: str, fn: Callable[[dict], str], allowlisted: bool = False) -> None:
+        self._tools[name] = fn
+        if allowlisted:
+            self._allowlisted.add(name)
+
+    def execute(self, action: ToolAction) -> ToolResult:
+        if action.tool not in self._tools:
+            return ToolResult(tool=action.tool, success=False, output="", error="unknown tool")
+        if action.requires_approval and not self._approval_fn(action):
+            return ToolResult(tool=action.tool, success=False, output="", error="approval denied")
+        try:
+            output = self._tools[action.tool](action.params)
+            return ToolResult(tool=action.tool, success=True, output=output)
+        except Exception as e:
+            return ToolResult(tool=action.tool, success=False, output="", error=str(e))
+
+    @property
+    def allowlisted_tools(self) -> set[str]:
+        return set(self._allowlisted)
+
+
+# ── agent-as-tool wrapper ─────────────────────────────────────────────
+
+
+class AgentAsTool:
+    """Wraps an agent so it can be invoked as a tool by another agent."""
+
+    def __init__(self, agent: Agent, runtime: AgentRuntime):
+        self._agent = agent
+        self._runtime = runtime
+
+    def __call__(self, params: dict) -> str:
+        task = params.get("task", "")
+        return self._agent.respond(self._runtime, task)

algo_cli/intelligence/agents_as_tools.py

@@ -0,0 +1,70 @@
+"""B73. Agents-as-Tools + Handoffs.
+
+Delegate to other agents via tool calls or handoffs.
+Source: OpenAI Agents SDK pattern.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum, auto
+from typing import Callable
+
+
+class DelegationMode(Enum):
+    AS_TOOL = auto()    # parent calls agent, gets result, continues
+    HANDOFF = auto()    # parent transfers control entirely
+
+
+@dataclass
+class AgentTool:
+    name: str
+    description: str
+    agent_id: str
+    run_fn: Callable[[str], str]
+    mode: DelegationMode = DelegationMode.AS_TOOL
+
+
+@dataclass
+class HandoffResult:
+    transferred_to: str
+    reason: str = ""
+    context_summary: str = ""
+
+
+class AgentsAsTools:
+    """Register agents as callable tools or handoff targets."""
+
+    def __init__(self) -> None:
+        self._tools: dict[str, AgentTool] = {}
+        self._handoff_targets: dict[str, AgentTool] = {}
+
+    def register_as_tool(self, tool: AgentTool) -> None:
+        tool.mode = DelegationMode.AS_TOOL
+        self._tools[tool.name] = tool
+
+    def register_as_handoff(self, tool: AgentTool) -> None:
+        tool.mode = DelegationMode.HANDOFF
+        self._handoff_targets[tool.agent_id] = tool
+
+    def call_tool(self, tool_name: str, input: str) -> str:
+        """Call an agent as a tool — parent continues after."""
+        tool = self._tools.get(tool_name)
+        if not tool:
+            raise KeyError(f"Tool '{tool_name}' not found")
+        return tool.run_fn(input)
+
+    def handoff(self, agent_id: str, context: str, reason: str = "") -> HandoffResult:
+        """Transfer control to another agent entirely."""
+        tool = self._handoff_targets.get(agent_id)
+        if not tool:
+            raise KeyError(f"Handoff target '{agent_id}' not found")
+        result = HandoffResult(transferred_to=agent_id, reason=reason, context_summary=context[:500])
+        tool.run_fn(context)
+        return result
+
+    def list_tools(self) -> list[dict[str, str]]:
+        return [{"name": t.name, "description": t.description, "agent": t.agent_id}
+                for t in self._tools.values()]
+
+    def list_handoff_targets(self) -> list[str]:
+        return list(self._handoff_targets.keys())

algo_cli/intelligence/artifact_binding.py

@@ -0,0 +1,80 @@
+"""H9 — Ground-Truth Artifact Binding.
+
+Every claim binds to a raw artifact with a content hash.
+Mined from GLOSSOPETRAE 78 raw JSONs + T3MP3ST bench/.
+"""
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from typing import Any
+
+
+def _compute_hash(content: bytes | str) -> str:
+    if isinstance(content, str):
+        content = content.encode("utf-8")
+    return hashlib.sha256(content).hexdigest()
+
+
+@dataclass
+class ArtifactBinding:
+    """A binding between a claim and a raw artifact."""
+
+    claim_id: str
+    artifact_path: str
+    content_hash: str
+    artifact_type: str = "json"
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "claim_id": self.claim_id,
+            "artifact_path": self.artifact_path,
+            "content_hash": self.content_hash,
+            "artifact_type": self.artifact_type,
+            "metadata": dict(self.metadata),
+        }
+
+
+class ArtifactBinder:
+    """Bind claims to raw artifacts with content hashes."""
+
+    def __init__(self) -> None:
+        self._bindings: dict[str, ArtifactBinding] = {}
+
+    def bind(
+        self,
+        claim_id: str,
+        artifact_path: str,
+        content: bytes | str,
+        artifact_type: str = "json",
+        metadata: dict[str, Any] | None = None,
+    ) -> ArtifactBinding:
+        content_hash = _compute_hash(content)
+        binding = ArtifactBinding(
+            claim_id=claim_id,
+            artifact_path=artifact_path,
+            content_hash=content_hash,
+            artifact_type=artifact_type,
+            metadata=metadata or {},
+        )
+        self._bindings[claim_id] = binding
+        return binding
+
+    def verify(self, claim_id: str, content: bytes | str) -> bool:
+        binding = self._bindings.get(claim_id)
+        if binding is None:
+            return False
+        return _compute_hash(content) == binding.content_hash
+
+    def get(self, claim_id: str) -> ArtifactBinding | None:
+        return self._bindings.get(claim_id)
+
+    def all(self) -> list[ArtifactBinding]:
+        return list(self._bindings.values())
+
+    def count(self) -> int:
+        return len(self._bindings)
+
+    def remove(self, claim_id: str) -> bool:
+        return self._bindings.pop(claim_id, None) is not None