readyagentsdev 0.8.2__py3-none-any.whl

readyagents/workflow/runner.py

@@ -0,0 +1,375 @@
+"""Load a workflow file, execute it, persist the run record."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import ValidationError
+
+from readyagents.audit import audit_dir_for, make_auditor
+from readyagents.config import Settings, get_settings
+from readyagents.errors import ConfigError, WorkflowError
+from readyagents.llm.base import LLMProvider
+from readyagents.llm.cache import LLMCache
+from readyagents.llm.resilience import CircuitBreaker, usd_to_micros
+from readyagents.logging import configure_logging, get_logger
+from readyagents.notify import post_json
+from readyagents.packs.loader import (
+    collect_pack_authorizers,
+    collect_pack_nodes,
+    collect_pack_secrets,
+    collect_pack_tools,
+    discover_packs,
+)
+from readyagents.policy import redactor_from_settings, resolve_authorizer
+from readyagents.tools import ToolRegistry, default_registry
+from readyagents.workflow.cancellation import CancellationToken
+from readyagents.workflow.engine import run_workflow
+from readyagents.workflow.nodes import ExecutionContext
+from readyagents.workflow.schema import WorkflowSpec, validate_required_inputs
+from readyagents.workflow.state import RunState, load_decision_file, load_run, persist_run
+
+log = get_logger("runner")
+
+_APPROVE = {"approve", "approved", "yes", "true", "accept", "ok"}
+
+
+def load_workflow(path: Path | str) -> WorkflowSpec:
+    file = Path(path)
+    if not file.is_file():
+        raise ConfigError(f"Workflow file not found: {file}")
+    text = file.read_text(encoding="utf-8")
+    try:
+        if file.suffix.lower() in {".json"}:
+            data = json.loads(text)
+        else:
+            data = yaml.safe_load(text)
+    except (json.JSONDecodeError, yaml.YAMLError) as exc:
+        raise WorkflowError(f"Could not parse {file}: {exc}") from exc
+    if not isinstance(data, dict):
+        raise WorkflowError(f"Workflow {file} must be a mapping")
+    try:
+        return WorkflowSpec.model_validate(data)
+    except ValidationError as exc:
+        raise WorkflowError(_format_validation(file, exc)) from exc
+
+
+def _format_validation(path: Path, exc: ValidationError) -> str:
+    lines = [f"Invalid workflow {path}:"]
+    for err in exc.errors():
+        loc = ".".join(str(p) for p in err.get("loc", ()))
+        lines.append(f"  - {loc}: {err.get('msg')}")
+    return "\n".join(lines)
+
+
+def confine_under(raw: str | Path, root: Path, *, what: str) -> Path:
+    """Resolve `raw` and refuse anything outside `root` (symlink-aware)."""
+    root = Path(root).resolve()
+    text = str(raw).strip()
+    if not text or "\x00" in text:
+        raise ConfigError(f"{what} must be a path under {root}")
+    candidate = Path(text)
+    if not candidate.is_absolute():
+        candidate = root / candidate
+    resolved = candidate.resolve()
+    if not resolved.is_relative_to(root):
+        raise ConfigError(
+            f"{what} is outside the workspace: {raw} "
+            f"(resolved to {resolved}, must stay under {root})"
+        )
+    return resolved
+
+
+def merge_inputs(workflow: WorkflowSpec, overrides: Mapping[str, Any] | None) -> dict[str, Any]:
+    merged = dict(workflow.input_defaults())
+    if overrides:
+        merged.update(overrides)
+    validate_required_inputs(workflow, merged)
+    return merged
+
+
+def run_workflow_file(
+    path: Path | str,
+    *,
+    inputs: Mapping[str, Any] | None = None,
+    dry_run: bool = False,
+    settings: Settings | None = None,
+    llm: LLMProvider | None = None,
+    persist: bool = True,
+    extra_tools: ToolRegistry | None = None,
+    extra_packs: Sequence[Any] | None = None,
+    decisions: Mapping[str, str] | None = None,
+    resume_state: RunState | None = None,
+    actor: str | None = None,
+    authorizer: Any | None = None,
+    secrets: Any | None = None,
+    decision_file: Path | str | None = None,
+    on_pause: Any | None = None,
+    no_cache: bool = False,
+    run_id: str | None = None,
+    initial_state: RunState | None = None,
+    cancellation: CancellationToken | None = None,
+) -> RunState:
+    settings = settings or get_settings()
+    workflow = load_workflow(path)
+    if initial_state is not None and resume_state is not None:
+        raise WorkflowError("initial_state and resume_state are mutually exclusive")
+    merged_decisions: dict[str, str] = {}
+    if decision_file:
+        merged_decisions.update(load_decision_file(decision_file))
+    if decisions:
+        merged_decisions.update(
+            {str(k): str(v).strip().lower() for k, v in dict(decisions).items()}
+        )
+    if resume_state is not None:
+        merged = dict(resume_state.inputs)
+        if inputs:
+            merged.update(inputs)
+        validate_required_inputs(workflow, merged)
+    elif initial_state is not None:
+        overrides = dict(initial_state.inputs)
+        if inputs:
+            overrides.update(inputs)
+        merged = merge_inputs(workflow, overrides)
+    else:
+        merged = merge_inputs(workflow, inputs)
+
+    source_path = Path(path).resolve()
+    workflow_dir = source_path.parent
+    if settings.workspace is not None:
+        root = settings.workspace_path()
+    else:
+        root = workflow_dir
+    declared = (workflow.workspace or "").strip()
+    workspace = confine_under(declared, root, what="workspace") if declared else root
+    allow_http = bool(workflow.allow_http or settings.allow_http)
+
+    tools = default_registry(allow_http=allow_http, workspace=workspace)
+    packs = list(discover_packs())
+    if extra_packs:
+        packs.extend(list(extra_packs))
+    tools.merge(collect_pack_tools(packs))
+    if extra_tools:
+        tools.merge(extra_tools)
+
+    pack_secrets = list(collect_pack_secrets(packs))
+    if secrets is not None:
+        from readyagents.secrets import as_backends
+
+        pack_secrets = as_backends(secrets) + pack_secrets
+    pack_authorizers = list(collect_pack_authorizers(packs))
+    if authorizer is not None:
+        pack_authorizers = [authorizer, *pack_authorizers]
+    resolved_authorizer = resolve_authorizer(pack_authorizers)
+    resolved_actor = actor if actor is not None else settings.actor
+    action = "resume" if resume_state is not None else "run"
+    resource = resume_state.run_id if resume_state is not None else workflow.name
+    resolved_authorizer.check(resolved_actor, action, resource)
+    for node_id, decision in merged_decisions.items():
+        gate = "approve" if str(decision).strip().lower() in _APPROVE else "reject"
+        resolved_authorizer.check(resolved_actor, gate, node_id)
+
+    redact_on = bool(settings.redact if workflow.redact is None else workflow.redact)
+    redactor = redactor_from_settings(
+        enabled=redact_on,
+        patterns=settings.redact_pattern_list(),
+        literals=settings.redact_literal_list(),
+    )
+    if redactor is not None:
+        configure_logging(settings.log_level, fmt=settings.log_format, redactor=redactor)
+
+    mcp = None
+    if workflow.mcp_servers and not dry_run:
+        from readyagents.mcp.client import MCPClient
+
+        mcp = MCPClient(workflow.mcp_servers, workspace)
+        tools.merge(mcp.tools())
+
+    runs_dir = settings.runs_dir()
+    auditor = None
+    if persist:
+        auditor = make_auditor(audit_dir_for(settings.home_path()), redactor=redactor)
+
+    def _save(state: RunState) -> None:
+        persist_run(state, runs_dir, redactor=redactor)
+
+    budget = workflow.budget
+    if budget and budget.max_tokens is not None:
+        budget_tokens = budget.max_tokens
+    else:
+        budget_tokens = settings.max_tokens
+    budget_cost = (
+        usd_to_micros(budget.max_cost_usd)
+        if budget and budget.max_cost_usd is not None
+        else usd_to_micros(settings.max_cost_usd)
+    )
+    circuit_spec = workflow.circuit
+    breaker = CircuitBreaker(
+        failure_threshold=(
+            circuit_spec.failure_threshold if circuit_spec else settings.circuit_failure_threshold
+        ),
+        cooldown_seconds=(
+            circuit_spec.cooldown_seconds if circuit_spec else settings.circuit_cooldown_seconds
+        ),
+    )
+    cache_enabled = bool(settings.llm_cache if workflow.cache_llm is None else workflow.cache_llm)
+    if no_cache:
+        cache_enabled = False
+    llm_cache = LLMCache(settings.cache_dir()) if cache_enabled else None
+    fallback = list(workflow.fallback_models or []) + settings.fallback_model_list()
+    pause_url = workflow.on_pause_url or settings.pause_notify_url
+
+    def _pause(exc: Any, state: RunState) -> None:
+        if on_pause is not None:
+            on_pause(exc, state)
+        if pause_url:
+            payload = {
+                "event": "approval_required",
+                "run_id": state.run_id,
+                "node_id": getattr(exc, "node_id", state.pending_node),
+                "prompt": getattr(exc, "prompt", ""),
+                "resume": (
+                    f"readyagents resume {state.run_id} "
+                    f"--approve {getattr(exc, 'node_id', state.pending_node)}"
+                ),
+            }
+            try:
+                post_json(pause_url, payload)
+            except Exception as notify_exc:  # noqa: BLE001
+                log.warning("pause webhook failed: %s", notify_exc)
+
+    ctx = ExecutionContext(
+        workflow,
+        tools,
+        dry_run=dry_run,
+        llm=llm,
+        default_model=workflow.default_model or settings.default_model,
+        extra_handlers=collect_pack_nodes(packs),
+        decisions=merged_decisions,
+        on_persist=_save if persist else None,
+        workflow_dir=source_path.parent,
+        circuit_breaker=breaker,
+        llm_cache=llm_cache,
+        budget_tokens=budget_tokens,
+        budget_cost_micros=budget_cost,
+        secrets=pack_secrets or None,
+        authorizer=resolved_authorizer,
+        actor=resolved_actor,
+        redactor=redactor,
+        auditor=auditor,
+        on_pause=_pause if (on_pause is not None or pause_url) else None,
+        fallback_models=fallback,
+        cache_llm=cache_enabled,
+        usage_state=resume_state,
+        cancellation=cancellation,
+    )
+    metadata = {
+        "source": str(source_path),
+        "allow_http": allow_http,
+        "dry_run": dry_run,
+        "workspace": str(workspace),
+        "actor": resolved_actor,
+    }
+    try:
+        state = run_workflow(
+            workflow,
+            merged,
+            ctx,
+            metadata=metadata,
+            state=resume_state if resume_state is not None else initial_state,
+            run_id=run_id,
+        )
+    finally:
+        if mcp is not None:
+            mcp.close()
+    return state
+
+
+def resume_run(
+    run_id: str,
+    *,
+    settings: Settings | None = None,
+    path: Path | str | None = None,
+    inputs: Mapping[str, Any] | None = None,
+    dry_run: bool = False,
+    persist: bool = True,
+    extra_tools: ToolRegistry | None = None,
+    extra_packs: Sequence[Any] | None = None,
+    decisions: Mapping[str, str] | None = None,
+    llm: LLMProvider | None = None,
+    actor: str | None = None,
+    authorizer: Any | None = None,
+    secrets: Any | None = None,
+    decision_file: Path | str | None = None,
+    on_pause: Any | None = None,
+    no_cache: bool = False,
+    cancellation: CancellationToken | None = None,
+) -> RunState:
+    settings = settings or get_settings()
+    state = load_run(settings.runs_dir(), run_id)
+    source = path or state.metadata.get("source")
+    if not source:
+        raise ConfigError(f"Run {state.run_id} has no stored workflow path. Pass --workflow PATH.")
+    return run_workflow_file(
+        source,
+        inputs=inputs,
+        dry_run=dry_run,
+        settings=settings,
+        llm=llm,
+        persist=persist,
+        extra_tools=extra_tools,
+        extra_packs=extra_packs,
+        decisions=decisions,
+        resume_state=state,
+        actor=actor,
+        authorizer=authorizer,
+        secrets=secrets,
+        decision_file=decision_file,
+        on_pause=on_pause,
+        no_cache=no_cache,
+        cancellation=cancellation,
+    )
+
+
+def replay_run(
+    run_id: str,
+    *,
+    settings: Settings | None = None,
+    persist: bool = True,
+    dry_run: bool = False,
+    extra_tools: ToolRegistry | None = None,
+    extra_packs: Sequence[Any] | None = None,
+    decisions: Mapping[str, str] | None = None,
+    llm: LLMProvider | None = None,
+    actor: str | None = None,
+    authorizer: Any | None = None,
+    secrets: Any | None = None,
+    decision_file: Path | str | None = None,
+    no_cache: bool = False,
+) -> RunState:
+    """Start a new run with the stored workflow path and inputs."""
+    settings = settings or get_settings()
+    previous = load_run(settings.runs_dir(), run_id)
+    source = previous.metadata.get("source")
+    if not source:
+        raise ConfigError(f"Run {previous.run_id} has no stored workflow path. Cannot replay.")
+    return run_workflow_file(
+        source,
+        inputs=previous.inputs,
+        dry_run=dry_run,
+        settings=settings,
+        llm=llm,
+        persist=persist,
+        extra_tools=extra_tools,
+        extra_packs=extra_packs,
+        decisions=decisions,
+        actor=actor,
+        authorizer=authorizer,
+        secrets=secrets,
+        decision_file=decision_file,
+        no_cache=no_cache,
+    )

readyagents/workflow/schema.py

@@ -0,0 +1,287 @@
+"""Pydantic models for YAML/JSON workflow definitions."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+from readyagents.errors import WorkflowError
+
+
+class NodeType(StrEnum):
+    agent = "agent"
+    tool = "tool"
+    condition = "condition"
+    transform = "transform"
+    approval = "approval"
+    parallel = "parallel"
+    include = "include"
+    foreach = "foreach"
+
+
+class RetrySpec(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    max_attempts: int = Field(default=1, ge=1, le=20)
+    backoff_seconds: float = Field(default=1.0, ge=0)
+    backoff_multiplier: float = Field(default=2.0, ge=1.0)
+
+
+class BudgetSpec(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    max_tokens: int | None = Field(default=None, ge=0)
+    max_cost_usd: float | None = Field(default=None, ge=0)
+
+
+class CircuitSpec(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    failure_threshold: int = Field(default=3, ge=1)
+    cooldown_seconds: float = Field(default=60.0, ge=0)
+
+
+class MCPServerSpec(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    command: str
+    args: list[str] = Field(default_factory=list)
+    env: dict[str, str] = Field(default_factory=dict)
+    cwd: str | None = None
+
+
+class NodeSpec(BaseModel):
+    """One node in the workflow graph."""
+
+    model_config = ConfigDict(extra="allow", populate_by_name=True)
+
+    id: str = Field(min_length=1)
+    type: str
+    timeout_seconds: float | None = Field(default=None, gt=0)
+    retry: RetrySpec | None = None
+    next: str | None = None
+    output_key: str | None = None
+    description: str | None = None
+
+    # agent
+    prompt: str | None = None
+    system: str | None = None
+    model: str | None = None
+
+    # tool
+    tool: str | None = None
+    arguments: dict[str, Any] = Field(default_factory=dict)
+
+    # condition
+    when: str | None = None
+    then: str | None = None
+    else_: str | None = Field(default=None, alias="else")
+
+    # transform
+    template: str | None = None
+    source: str | None = None
+    json_path: str | None = None
+    parse_json: bool = False
+
+    # parallel
+    branches: list[NodeSpec] = Field(default_factory=list)
+
+    # include (sub-workflow)
+    path: str | None = None
+    call_inputs: dict[str, Any] = Field(default_factory=dict, alias="inputs")
+
+    # agent extras
+    fallback_models: list[str] = Field(default_factory=list)
+    output_schema: dict[str, Any] | None = None
+    cache: bool | None = None
+    tools: list[str] = Field(default_factory=list)
+    max_tool_rounds: int | None = Field(default=None, ge=1, le=20)
+
+    # foreach
+    items: str | None = None
+    max_items: int | None = Field(default=None, ge=1, le=100)
+    body: NodeSpec | None = None
+
+    @field_validator("id")
+    @classmethod
+    def _id_token(cls, value: str) -> str:
+        if not value.replace("_", "").replace("-", "").isalnum():
+            raise ValueError(f"Invalid node id '{value}' (use letters, numbers, _ or -)")
+        return value
+
+    @field_validator("type")
+    @classmethod
+    def _type_token(cls, value: str) -> str:
+        cleaned = value.strip().lower()
+        if not cleaned.replace("_", "").replace("-", "").isalnum():
+            raise ValueError(f"Invalid node type '{value}'")
+        return cleaned
+
+    @field_validator("tools")
+    @classmethod
+    def _tools_tokens(cls, value: list[str]) -> list[str]:
+        cleaned: list[str] = []
+        seen: set[str] = set()
+        for raw in value:
+            name = str(raw).strip()
+            if not name:
+                raise ValueError("tool names must be non-empty")
+            token = name.replace("_", "").replace("-", "").replace(".", "")
+            if not token.isalnum():
+                raise ValueError(f"Invalid tool name '{name}' (use letters, numbers, _, -, .)")
+            if name in seen:
+                raise ValueError(f"Duplicate tool name '{name}'")
+            seen.add(name)
+            cleaned.append(name)
+        return cleaned
+
+    @model_validator(mode="after")
+    def _type_fields(self) -> NodeSpec:
+        t = self.type
+        if t != NodeType.agent.value and self.tools:
+            raise ValueError(f"Node '{self.id}': 'tools' is only valid on agent nodes")
+        if t == NodeType.agent.value and not self.prompt:
+            raise ValueError(f"Node '{self.id}': agent nodes require 'prompt'")
+        if t == NodeType.tool.value and not self.tool:
+            raise ValueError(f"Node '{self.id}': tool nodes require 'tool'")
+        if t == NodeType.condition.value:
+            if not self.when:
+                raise ValueError(f"Node '{self.id}': condition nodes require 'when'")
+            if not self.then and not self.else_:
+                raise ValueError(f"Node '{self.id}': condition nodes require 'then' and/or 'else'")
+        if t == NodeType.transform.value:
+            if self.template is None and not self.json_path and not self.parse_json:
+                raise ValueError(
+                    f"Node '{self.id}': transform nodes require "
+                    "'template', 'json_path', or parse_json"
+                )
+        if t == NodeType.approval.value:
+            if not self.prompt:
+                raise ValueError(f"Node '{self.id}': approval nodes require 'prompt'")
+            if not self.then and not self.else_ and not self.next:
+                raise ValueError(
+                    f"Node '{self.id}': approval nodes require 'then', 'else', or 'next'"
+                )
+        if t == NodeType.parallel.value and not self.branches:
+            raise ValueError(f"Node '{self.id}': parallel nodes require 'branches'")
+        if t == NodeType.include.value and not self.path:
+            raise ValueError(f"Node '{self.id}': include nodes require 'path'")
+        if t == NodeType.foreach.value:
+            if not self.items:
+                raise ValueError(f"Node '{self.id}': foreach nodes require 'items'")
+            if self.body is None:
+                raise ValueError(f"Node '{self.id}': foreach nodes require 'body'")
+            if self.body.type == NodeType.foreach.value:
+                raise ValueError(f"Node '{self.id}': nested foreach is not supported")
+        return self
+
+
+class EdgeSpec(BaseModel):
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+    from_: str = Field(alias="from")
+    to: str
+    when: str | None = None
+
+
+class WorkflowSpec(BaseModel):
+    """A validated workflow document."""
+
+    model_config = ConfigDict(extra="ignore", populate_by_name=True)
+
+    name: str
+    version: str = "1"
+    description: str | None = None
+    inputs: dict[str, Any] = Field(default_factory=dict)
+    required_inputs: list[str] = Field(default_factory=list)
+    start: str | None = None
+    nodes: list[NodeSpec]
+    edges: list[EdgeSpec] = Field(default_factory=list)
+    mcp_servers: dict[str, MCPServerSpec] = Field(default_factory=dict)
+    allow_http: bool = False
+    workspace: str | None = None
+    default_model: str | None = None
+    budget: BudgetSpec | None = None
+    fallback_models: list[str] = Field(default_factory=list)
+    circuit: CircuitSpec | None = None
+    on_pause_url: str | None = None
+    cache_llm: bool | None = None
+    redact: bool | None = None
+
+    @model_validator(mode="after")
+    def _graph(self) -> WorkflowSpec:
+        if not self.nodes:
+            raise ValueError("Workflow must declare at least one node")
+        ids = [n.id for n in self.nodes]
+        if len(ids) != len(set(ids)):
+            raise ValueError("Duplicate node ids")
+        known = set(ids)
+        start = self.start or self.nodes[0].id
+        if start not in known:
+            raise ValueError(f"start node '{start}' does not exist")
+        self.start = start
+        for node in self.nodes:
+            for ref in (node.next, node.then, node.else_):
+                if ref is not None and ref not in known:
+                    raise ValueError(f"Node '{node.id}' references unknown node '{ref}'")
+            if node.branches:
+                branch_ids = [b.id for b in node.branches]
+                if len(branch_ids) != len(set(branch_ids)):
+                    raise ValueError(f"Node '{node.id}': duplicate parallel branch ids")
+        for edge in self.edges:
+            if edge.from_ not in known:
+                raise ValueError(f"Edge from unknown node '{edge.from_}'")
+            if edge.to not in known:
+                raise ValueError(f"Edge to unknown node '{edge.to}'")
+        self._assert_acyclic()
+        return self
+
+    def _assert_acyclic(self) -> None:
+        graph: dict[str, list[str]] = {node.id: [] for node in self.nodes}
+        for node in self.nodes:
+            for ref in (node.next, node.then, node.else_):
+                if ref is not None:
+                    graph[node.id].append(ref)
+        for edge in self.edges:
+            graph[edge.from_].append(edge.to)
+
+        visiting: set[str] = set()
+        done: set[str] = set()
+
+        def dfs(nid: str) -> None:
+            visiting.add(nid)
+            for nxt in graph[nid]:
+                if nxt in visiting:
+                    raise ValueError(f"Cycle detected at node '{nxt}'")
+                if nxt not in done and nxt in graph:
+                    dfs(nxt)
+            visiting.remove(nid)
+            done.add(nid)
+
+        for nid in graph:
+            if nid not in done:
+                dfs(nid)
+
+    def node_map(self) -> dict[str, NodeSpec]:
+        return {n.id: n for n in self.nodes}
+
+    def input_defaults(self) -> dict[str, Any]:
+        defaults: dict[str, Any] = {}
+        for key, raw in self.inputs.items():
+            if isinstance(raw, dict) and "default" in raw:
+                defaults[key] = raw["default"]
+            else:
+                defaults[key] = raw
+        return defaults
+
+
+def validate_required_inputs(workflow: WorkflowSpec, provided: dict[str, Any]) -> None:
+    missing = [name for name in workflow.required_inputs if name not in provided]
+    if missing:
+        example = " ".join(f"--input {name}=..." for name in missing)
+        raise WorkflowError(f"Missing required inputs: {', '.join(missing)}. Pass {example}.")
+
+
+NodeSpec.model_rebuild()