smythe 0.2.0__py3-none-any.whl

smythe/__init__.py

@@ -0,0 +1,83 @@
+"""Smythe: task-based personalized agent swarms with dynamic execution topology."""
+
+from smythe.budget import Sentinel, SentinelAlert
+from smythe.checkpoint import CheckpointStore, FileCheckpointStore
+from smythe.constrained_planner import ConstrainedArchitect, SubGraphTemplate
+from smythe.graph import FailurePolicy
+from smythe.loader import load_graph
+from smythe.mcp import MCPConfigError, MCPServerSpec, MCPSkillProvider, MCPToolRuntime
+from smythe.memory import PlannerMemory
+from smythe.planner import ArchitectError, DeterministicArchitect, LLMArchitect, SimpleArchitect
+from smythe.provider import (
+    AnthropicProvider,
+    CompletionResult,
+    GeminiProvider,
+    OfflineProvider,
+    OpenAIProvider,
+    Provider,
+)
+from smythe.router import WhiteRabbit
+from smythe.skills import (
+    CapabilityHydrationMode,
+    CapabilityMapper,
+    DefaultCapabilityMapper,
+    SkillProvider,
+    SkillRef,
+)
+from smythe.swarm import Swarm, SwarmResult
+from smythe.synthesizer import Synthesizer, SynthesisStrategy
+from smythe.task import Task
+from smythe.tools import (
+    ChatMessage,
+    ToolCall,
+    ToolLoopLimitError,
+    ToolResult,
+    ToolRuntime,
+    ToolSession,
+    ToolSpec,
+)
+
+__all__ = [
+    "AnthropicProvider",
+    "ArchitectError",
+    "CapabilityHydrationMode",
+    "CapabilityMapper",
+    "ChatMessage",
+    "CheckpointStore",
+    "CompletionResult",
+    "ConstrainedArchitect",
+    "DefaultCapabilityMapper",
+    "DeterministicArchitect",
+    "FailurePolicy",
+    "FileCheckpointStore",
+    "GeminiProvider",
+    "LLMArchitect",
+    "MCPConfigError",
+    "MCPServerSpec",
+    "MCPSkillProvider",
+    "MCPToolRuntime",
+    "OfflineProvider",
+    "OpenAIProvider",
+    "PlannerMemory",
+    "Provider",
+    "Sentinel",
+    "SentinelAlert",
+    "SimpleArchitect",
+    "SkillProvider",
+    "SkillRef",
+    "SubGraphTemplate",
+    "Swarm",
+    "SwarmResult",
+    "Synthesizer",
+    "SynthesisStrategy",
+    "Task",
+    "ToolCall",
+    "ToolLoopLimitError",
+    "ToolResult",
+    "ToolRuntime",
+    "ToolSession",
+    "ToolSpec",
+    "WhiteRabbit",
+    "load_graph",
+]
+__version__ = "0.2.0"

smythe/agent.py

@@ -0,0 +1,47 @@
+"""Agent model — persistent identity with capabilities and memory."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+from uuid import uuid4
+
+if TYPE_CHECKING:
+    from smythe.mcp import MCPServerSpec
+
+
+@dataclass
+class AgentProfile:
+    """Static description of what an agent can do and how it behaves.
+
+    Attributes:
+        name: Human-readable agent name.
+        persona: System-prompt-level personality / role description.
+        capabilities: Tags describing areas of expertise (e.g. "research", "critique").
+        mcp_servers: MCP servers this agent may use as tool sources
+            (executed via MCPToolRuntime; see smythe.mcp).
+    """
+
+    name: str
+    persona: str = ""
+    capabilities: list[str] = field(default_factory=list)
+    mcp_servers: list[MCPServerSpec] = field(default_factory=list)
+
+
+@dataclass
+class Agent:
+    """A persistent agent instance tracked across executions.
+
+    Attributes:
+        id: Unique identifier (auto-generated).
+        profile: Static capability/persona description.
+        history: Append-only log of past execution summaries for learning.
+    """
+
+    profile: AgentProfile
+    id: str = field(default_factory=lambda: uuid4().hex[:12])
+    history: list[dict[str, Any]] = field(default_factory=list)
+
+    @property
+    def name(self) -> str:
+        return self.profile.name

smythe/async_executor.py

@@ -0,0 +1,148 @@
+"""AsyncExecutor — concurrent DAG execution via asyncio."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Callable
+
+from smythe.budget import Sentinel
+from smythe.executor_base import DEFAULT_MAX_TOOL_ITERATIONS, ExecutorBase
+from smythe.graph import ExecutionGraph, FailurePolicy, Node, NodeStatus
+from smythe.provider import Provider
+from smythe.registry import Registry
+from smythe.tools import ToolRuntime
+from smythe.tracer import Tracer
+
+
+class AsyncExecutor(ExecutorBase):
+    """Executes an assigned graph with maximum concurrency.
+
+    Independent nodes (those whose dependencies are all completed)
+    are launched in parallel via asyncio.gather().  The executor
+    runs in waves until every node has completed or failed.
+
+    Budget safety: before each wave, estimated cost is *reserved* for
+    every ready node so concurrent nodes cannot collectively overshoot
+    the budget.  Reservations are reconciled with actual cost on
+    completion, or released on failure.
+
+    Concurrency safety: ``max_concurrency`` caps in-flight provider
+    calls across a wave, so a wide broadcast doesn't fire every call
+    at once and trip provider rate limits.  None means unlimited.
+    """
+
+    DEFAULT_ESTIMATED_TOKENS = 2000
+
+    def __init__(
+        self,
+        provider: Provider,
+        registry: Registry,
+        tracer: Tracer,
+        budget: Sentinel | None = None,
+        estimated_tokens_per_node: int = DEFAULT_ESTIMATED_TOKENS,
+        max_concurrency: int | None = None,
+        on_node_update: Callable[[Node], None] | None = None,
+        tool_runtime: ToolRuntime | None = None,
+        max_tool_iterations: int = DEFAULT_MAX_TOOL_ITERATIONS,
+    ) -> None:
+        super().__init__(
+            provider=provider, registry=registry, tracer=tracer, budget=budget,
+            on_node_update=on_node_update, tool_runtime=tool_runtime,
+            max_tool_iterations=max_tool_iterations,
+        )
+        self._estimated_tokens_per_node = estimated_tokens_per_node
+        if max_concurrency is not None and max_concurrency < 1:
+            raise ValueError(f"max_concurrency must be >= 1, got {max_concurrency}")
+        self._max_concurrency = max_concurrency
+
+    async def run(self, graph: ExecutionGraph) -> ExecutionGraph:
+        """Execute every node, fanning out independent nodes concurrently."""
+        semaphore = (
+            asyncio.Semaphore(self._max_concurrency)
+            if self._max_concurrency is not None
+            else None
+        )
+
+        async def bounded(node: Node) -> None:
+            if semaphore is None:
+                await self._execute_node(node, graph)
+            else:
+                async with semaphore:
+                    await self._execute_node(node, graph)
+
+        first_error: Exception | None = None
+        while True:
+            pending = [n for n in graph.nodes if n.status == NodeStatus.PENDING]
+            if not pending:
+                break
+            ready = [n for n in pending if graph.is_ready(n)]
+            if not ready:
+                if first_error is not None:
+                    raise first_error
+                failed = [n for n in graph.nodes if n.status == NodeStatus.FAILED]
+                if failed:
+                    raise RuntimeError(
+                        f"Execution halted: {len(pending)} node(s) blocked by "
+                        f"{len(failed)} failed upstream node(s)"
+                    )
+                raise RuntimeError("Deadlock: pending nodes exist but none are ready")
+
+            if self._budget:
+                estimated_cost = (
+                    self._estimated_tokens_per_node * self._budget.cost_per_token
+                )
+                reserved_ids: list[str] = []
+                try:
+                    for n in ready:
+                        self._budget.reserve(n.id, estimated_cost)
+                        reserved_ids.append(n.id)
+                except Exception:
+                    for rid in reserved_ids:
+                        self._budget.release(rid)
+                    raise
+
+            try:
+                await asyncio.gather(*(bounded(n) for n in ready))
+            except Exception as exc:
+                if first_error is None:
+                    first_error = exc
+
+        if first_error is not None:
+            raise first_error
+        return graph
+
+    async def _execute_node(self, node: Node, graph: ExecutionGraph) -> None:
+        """Run a single node through the provider, respecting its failure policy."""
+        last_exc: Exception | None = None
+        attempts = 1 + max(node.max_retries, 0) if node.failure_policy == FailurePolicy.RETRY else 1
+
+        for attempt in range(attempts):
+            node.status = NodeStatus.RUNNING
+            self._tracer.on_node_start(node)
+
+            try:
+                # Cost recording happens inside acall_node (per provider call).
+                result = await self.acall_node(node, graph)
+                node.result = result.text
+                node.status = NodeStatus.COMPLETED
+                self._tracer.on_node_end(node)
+                self.notify_update(node)
+                return
+            except Exception as exc:
+                last_exc = exc
+                self._tracer.on_node_error(node, exc)
+                self._tracer.on_node_end(node)
+
+        if self._budget:
+            self._budget.release(node.id)
+
+        node.result = str(last_exc)
+
+        if node.failure_policy == FailurePolicy.SKIP:
+            node.status = NodeStatus.SKIPPED
+            self.notify_update(node)
+            return
+
+        node.status = NodeStatus.FAILED
+        self.notify_update(node)
+        raise last_exc  # type: ignore[misc]

smythe/budget.py

@@ -0,0 +1,115 @@
+"""Sentinel — deterministic cost guardrails for execution.
+
+The Sentinels patrol the boundaries.  They enforce the budget.
+"""
+
+from __future__ import annotations
+
+from smythe.provider import CompletionResult
+
+
+class SentinelAlert(Exception):
+    """Raised when cumulative execution cost would exceed the budget limit."""
+
+    def __init__(self, spent: float, limit: float, node_id: str) -> None:
+        self.spent = spent
+        self.limit = limit
+        self.node_id = node_id
+        super().__init__(
+            f"Budget exhausted before node {node_id!r}: "
+            f"${spent:.4f} spent of ${limit:.4f} limit"
+        )
+
+
+class Sentinel:
+    """Accumulates token costs per node and enforces a USD spending cap.
+
+    Supports a reservation protocol for parallel execution: ``reserve()``
+    pre-commits estimated cost so concurrent nodes cannot collectively
+    exceed the budget.  ``record()`` reconciles the reservation with the
+    actual cost once the node completes.  ``release()`` frees a reservation
+    if the node fails before ``record()`` is called.
+
+    Attributes:
+        max_budget_usd: Hard cap in USD.  None means unlimited.
+        cost_per_token: Blended $/token rate (default ~$3/1M tokens).
+    """
+
+    def __init__(
+        self,
+        max_budget_usd: float | None = None,
+        cost_per_token: float = 0.000003,
+    ) -> None:
+        self.max_budget_usd = max_budget_usd
+        self.cost_per_token = cost_per_token
+        self._node_costs: dict[str, float] = {}
+        self._reservations: dict[str, float] = {}
+        self._spent: float = 0.0
+
+    @property
+    def total_cost_usd(self) -> float:
+        return self._spent
+
+    def check(self, node_id: str) -> None:
+        """Raise SentinelAlert if the budget is already exhausted.
+
+        Used by the serial executor where reservation is unnecessary.
+        """
+        if self.max_budget_usd is not None and self._spent >= self.max_budget_usd:
+            raise SentinelAlert(self._spent, self.max_budget_usd, node_id)
+
+    def reserve(self, node_id: str, estimated_cost: float) -> None:
+        """Pre-commit estimated cost before a node starts executing.
+
+        Raises SentinelAlert if the reservation would exceed the
+        budget.  The reservation is held until ``record()`` replaces it
+        with actual cost, or ``release()`` cancels it on failure.
+        """
+        if self.max_budget_usd is not None:
+            if self._spent + estimated_cost > self.max_budget_usd:
+                raise SentinelAlert(self._spent, self.max_budget_usd, node_id)
+        self._reservations[node_id] = estimated_cost
+        self._spent += estimated_cost
+
+    def release(self, node_id: str) -> None:
+        """Cancel a reservation (e.g. on node failure before record)."""
+        reserved = self._reservations.pop(node_id, 0.0)
+        self._spent -= reserved
+
+    def record(self, node_id: str, result: CompletionResult) -> float:
+        """Record the actual cost, replacing any outstanding reservation.
+
+        Overwrites the node's cost — use add_cost() for multi-call
+        nodes (tool loops), where costs must accumulate.
+        """
+        reserved = self._reservations.pop(node_id, 0.0)
+        self._spent -= reserved
+        cost = result.total_tokens * self.cost_per_token
+        self._node_costs[node_id] = cost
+        self._spent += cost
+        return cost
+
+    def add_cost(self, node_id: str, result: CompletionResult) -> float:
+        """Accumulate cost for a node across multiple provider calls.
+
+        The first call for a node also releases any outstanding
+        reservation (the estimate is superseded by actuals).  Returns
+        the node's cumulative cost.
+        """
+        reserved = self._reservations.pop(node_id, 0.0)
+        self._spent -= reserved
+        cost = result.total_tokens * self.cost_per_token
+        self._node_costs[node_id] = self._node_costs.get(node_id, 0.0) + cost
+        self._spent += cost
+        return self._node_costs[node_id]
+
+    def breakdown(self) -> dict[str, float]:
+        """Per-node cost map in USD."""
+        return dict(self._node_costs)
+
+    def restore(self, node_costs: dict[str, float]) -> None:
+        """Seed per-node costs from a checkpoint so a resumed execution
+        keeps counting against the same budget."""
+        for node_id, cost in node_costs.items():
+            self._node_costs[node_id] = cost
+        self._spent = sum(self._node_costs.values()) + sum(self._reservations.values())

smythe/checkpoint.py

@@ -0,0 +1,267 @@
+"""Checkpointing — durable, resumable execution state.
+
+After each node reaches a terminal status, the Swarm persists the full
+execution state (graph, node results, agents, budget consumed) through a
+CheckpointStore.  A crashed or interrupted execution can then be picked
+up with ``swarm.resume(execution_id)``, re-running only the nodes that
+never completed.
+
+The state is a plain JSON document (version 1) so users can inspect or
+repair checkpoints by hand.  See docs/checkpoint-format.md for the full
+schema.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import threading
+import time
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any
+
+from smythe.agent import Agent, AgentProfile
+from smythe.graph import ExecutionGraph, FailurePolicy, Node, NodeStatus, Topology
+from smythe.registry import Registry
+from smythe.task import Task
+
+CHECKPOINT_VERSION = 1
+
+_EXECUTION_ID_RE = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
+
+
+def _jsonable(value: Any) -> Any:
+    """Return *value* if JSON-serializable, otherwise its str() form."""
+    try:
+        json.dumps(value)
+        return value
+    except (TypeError, ValueError):
+        return str(value)
+
+
+def node_to_dict(node: Node) -> dict[str, Any]:
+    return {
+        "id": node.id,
+        "label": node.label,
+        "agent_id": node.agent_id,
+        "depends_on": list(node.depends_on),
+        "result": _jsonable(node.result),
+        "status": node.status.value,
+        "metadata": {k: _jsonable(v) for k, v in node.metadata.items()},
+        "failure_policy": node.failure_policy.value,
+        "max_retries": node.max_retries,
+        "required_capabilities": list(node.required_capabilities),
+        "timeout_s": node.timeout_s,
+        "max_tool_iterations": node.max_tool_iterations,
+    }
+
+
+def node_from_dict(data: dict[str, Any]) -> Node:
+    return Node(
+        id=data["id"],
+        label=data["label"],
+        agent_id=data.get("agent_id"),
+        depends_on=list(data.get("depends_on", [])),
+        result=data.get("result"),
+        status=NodeStatus(data.get("status", "pending")),
+        metadata=dict(data.get("metadata", {})),
+        failure_policy=FailurePolicy(data.get("failure_policy", "halt")),
+        max_retries=data.get("max_retries", 1),
+        required_capabilities=list(data.get("required_capabilities", [])),
+        timeout_s=data.get("timeout_s"),
+        max_tool_iterations=data.get("max_tool_iterations"),
+    )
+
+
+def graph_to_dict(graph: ExecutionGraph) -> dict[str, Any]:
+    return {
+        "topology": [t.value for t in graph.topology],
+        "estimated_cost_usd": graph.estimated_cost_usd,
+        "nodes": [node_to_dict(n) for n in graph.nodes],
+    }
+
+
+def graph_from_dict(data: dict[str, Any]) -> ExecutionGraph:
+    """Exact restore of a serialized graph, including statuses and results."""
+    return ExecutionGraph(
+        topology=[Topology(t) for t in data.get("topology", ["serial"])],
+        nodes=[node_from_dict(n) for n in data.get("nodes", [])],
+        estimated_cost_usd=data.get("estimated_cost_usd"),
+    )
+
+
+def agents_to_list(registry: Registry) -> list[dict[str, Any]]:
+    out = []
+    for agent in registry.list_agents():
+        entry: dict[str, Any] = {
+            "id": agent.id,
+            "name": agent.profile.name,
+            "persona": agent.profile.persona,
+            "capabilities": list(agent.profile.capabilities),
+        }
+        if agent.profile.mcp_servers:
+            # env_passthrough stores variable NAMES only; secret values
+            # never touch the checkpoint (see smythe.mcp).
+            entry["mcp_servers"] = [s.to_dict() for s in agent.profile.mcp_servers]
+        out.append(entry)
+    return out
+
+
+def agents_from_list(data: list[dict[str, Any]]) -> list[Agent]:
+    agents = []
+    for entry in data:
+        mcp_servers = []
+        if entry.get("mcp_servers"):
+            from smythe.mcp import MCPServerSpec
+
+            mcp_servers = [MCPServerSpec.from_dict(s) for s in entry["mcp_servers"]]
+        agents.append(Agent(
+            id=entry["id"],
+            profile=AgentProfile(
+                name=entry.get("name", entry["id"]),
+                persona=entry.get("persona", ""),
+                capabilities=list(entry.get("capabilities", [])),
+                mcp_servers=mcp_servers,
+            ),
+        ))
+    return agents
+
+
+def task_to_dict(task: Task | None) -> dict[str, Any] | None:
+    if task is None:
+        return None
+    return {
+        "goal": task.goal,
+        "constraints": list(task.constraints),
+        "context": {k: _jsonable(v) for k, v in task.context.items()},
+    }
+
+
+def task_from_dict(data: dict[str, Any] | None) -> Task | None:
+    if data is None:
+        return None
+    return Task(
+        goal=data["goal"],
+        constraints=list(data.get("constraints", [])),
+        context=dict(data.get("context", {})),
+    )
+
+
+def reset_incomplete_nodes(graph: ExecutionGraph) -> list[str]:
+    """Reset RUNNING and FAILED nodes to PENDING so resume re-runs them.
+
+    COMPLETED and SKIPPED nodes keep their status and results.
+    Returns the IDs of the nodes that were reset.
+    """
+    reset: list[str] = []
+    for node in graph.nodes:
+        if node.status in (NodeStatus.RUNNING, NodeStatus.FAILED):
+            node.status = NodeStatus.PENDING
+            node.result = None
+            reset.append(node.id)
+    return reset
+
+
+class CheckpointStore(ABC):
+    """Persistence interface for execution checkpoints.
+
+    Implementations must make ``save()`` atomic per execution_id: a
+    reader must never observe a partially written state.
+    """
+
+    @abstractmethod
+    def save(self, execution_id: str, state: dict[str, Any]) -> None:
+        """Persist the full state for an execution, replacing any prior state."""
+
+    @abstractmethod
+    def load(self, execution_id: str) -> dict[str, Any] | None:
+        """Return the last saved state, or None if the id is unknown."""
+
+    @abstractmethod
+    def delete(self, execution_id: str) -> None:
+        """Remove a checkpoint.  Deleting an unknown id is a no-op."""
+
+    @abstractmethod
+    def list_ids(self) -> list[str]:
+        """Return all known execution ids, sorted."""
+
+
+class FileCheckpointStore(CheckpointStore):
+    """Filesystem-backed store: one JSON file per execution.
+
+    Files live in ``~/.smythe/checkpoints/`` by default.  Writes go to a
+    temp file and are moved into place with os.replace, so a crash
+    mid-write never corrupts the previous checkpoint.
+    """
+
+    def __init__(self, directory: str | Path | None = None) -> None:
+        self._dir = (
+            Path(directory)
+            if directory is not None
+            else Path.home() / ".smythe" / "checkpoints"
+        )
+        self._dir.mkdir(parents=True, exist_ok=True)
+        self._lock = threading.Lock()
+
+    def _path(self, execution_id: str) -> Path:
+        if not _EXECUTION_ID_RE.match(execution_id):
+            raise ValueError(
+                f"Invalid execution_id {execution_id!r}: must match "
+                f"{_EXECUTION_ID_RE.pattern}"
+            )
+        return self._dir / f"{execution_id}.json"
+
+    def save(self, execution_id: str, state: dict[str, Any]) -> None:
+        path = self._path(execution_id)
+        tmp = path.with_suffix(".json.tmp")
+        with self._lock:
+            tmp.write_text(json.dumps(state, indent=2), encoding="utf-8")
+            os.replace(tmp, path)
+
+    def load(self, execution_id: str) -> dict[str, Any] | None:
+        path = self._path(execution_id)
+        try:
+            return json.loads(path.read_text(encoding="utf-8"))
+        except FileNotFoundError:
+            return None
+
+    def delete(self, execution_id: str) -> None:
+        self._path(execution_id).unlink(missing_ok=True)
+
+    def list_ids(self) -> list[str]:
+        return sorted(p.stem for p in self._dir.glob("*.json"))
+
+
+def build_state(
+    *,
+    execution_id: str,
+    status: str,
+    model: str,
+    graph: ExecutionGraph,
+    registry: Registry,
+    task: Task | None,
+    max_budget_usd: float | None,
+    node_costs: dict[str, float],
+    output: str | None = None,
+    created_at: float | None = None,
+) -> dict[str, Any]:
+    """Assemble a version-1 checkpoint state document."""
+    now = time.time()
+    return {
+        "version": CHECKPOINT_VERSION,
+        "execution_id": execution_id,
+        "status": status,
+        "created_at": created_at if created_at is not None else now,
+        "updated_at": now,
+        "model": model,
+        "task": task_to_dict(task),
+        "graph": graph_to_dict(graph),
+        "agents": agents_to_list(registry),
+        "budget": {
+            "max_budget_usd": max_budget_usd,
+            "node_costs": dict(node_costs),
+        },
+        "output": output,
+    }