verifyloop 0.1.0__py3-none-any.whl

verifyloop/__init__.py

@@ -0,0 +1,41 @@
+"""VerifyLoop — Plan → Execute → Verify → Recover agent framework.
+
+The verify step uses a trained verification model, not just a prompt.
+"""
+
+from verifyloop.models import (
+    AgentRun,
+    ExecuteStep,
+    PlanStep,
+    RecoverStep,
+    Step,
+    Substep,
+    VerifyStep,
+)
+from verifyloop.pipeline import AgentPipeline, PipelineConfig
+from verifyloop.executor import Executor
+from verifyloop.planner import PlanGenerator
+from verifyloop.verifier import Verifier, VerifierConfig
+from verifyloop.recoverer import Recoverer
+from verifyloop.memory import InMemoryStore, FileStore
+
+__all__ = [
+    "AgentPipeline",
+    "PipelineConfig",
+    "Executor",
+    "PlanGenerator",
+    "Verifier",
+    "VerifierConfig",
+    "Recoverer",
+    "InMemoryStore",
+    "FileStore",
+    "Step",
+    "PlanStep",
+    "ExecuteStep",
+    "VerifyStep",
+    "RecoverStep",
+    "Substep",
+    "AgentRun",
+]
+
+__version__ = "0.1.0"

verifyloop/cli.py

@@ -0,0 +1,186 @@
+"""CLI interface for VerifyLoop."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+import click
+from rich.console import Console
+from rich.live import Live
+from rich.panel import Panel
+from rich.progress import Progress, SpinnerColumn, TextColumn
+from rich.table import Table
+
+from verifyloop.models import AgentRun, PipelineConfig, RunStatus
+from verifyloop.pipeline import AgentPipeline
+
+console = Console()
+
+
+def format_step_table(run: AgentRun) -> Table:
+    table = Table(title="VerifyLoop Execution", show_lines=True)
+    table.add_column("#", style="dim", width=4)
+    table.add_column("Phase", style="bold")
+    table.add_column("Content", max_width=60)
+    table.add_column("Confidence", justify="right")
+
+    phase_colors = {
+        "plan": "cyan",
+        "execute": "green",
+        "verify": "yellow",
+        "recover": "red",
+    }
+
+    for i, step in enumerate(run.steps, 1):
+        color = phase_colors.get(step.step_type.value, "white")
+        table.add_row(
+            str(i),
+            f"[{color}]{step.step_type.value}[/{color}]",
+            step.content[:120] + ("..." if len(step.content) > 120 else ""),
+            f"{step.confidence:.0%}",
+        )
+    return table
+
+
+async def run_pipeline(
+    task: str,
+    context: str = "",
+    model: str = "gpt-4o",
+    verify_model: str = "reason-critic-7b",
+    max_iterations: int = 5,
+    working_dir: str = ".",
+    dry_run: bool = False,
+    interactive: bool = False,
+    sandbox: bool = False,
+) -> AgentRun:
+    config = PipelineConfig(
+        model=model,
+        verify_model=verify_model,
+        max_iterations=max_iterations,
+        working_dir=working_dir,
+        dry_run=dry_run,
+        interactive=interactive,
+        sandbox=sandbox,
+    )
+    pipeline = AgentPipeline(config)
+
+    events_log: list[dict[str, Any]] = []
+
+    async def on_event(event: str, data: dict[str, Any]) -> None:
+        events_log.append({"event": event, **data})
+        if event == "phase_start":
+            phase = data.get("phase", "")
+            color_map = {"plan": "cyan", "execute": "green", "verify": "yellow", "recover": "red"}
+            color = color_map.get(phase, "white")
+            console.print(f"\n[bold {color}]═══ {phase.upper()} ═══[/]")
+        elif event == "step_complete":
+            status = "✓" if data.get("success") else "✗"
+            console.print(f"  {status} {data.get('tool', '')} (iteration {data.get('iteration', '')})")
+        elif event == "phase_complete":
+            phase = data.get("phase", "")
+            if phase == "verify":
+                passed = data.get("passed", False)
+                confidence = data.get("confidence", 0)
+                icon = "✓" if passed else "✗"
+                console.print(f"  {icon} Verification: confidence={confidence:.0%}")
+        elif event == "recovery_attempt":
+            console.print(
+                f"  ⟳ Recovery #{data.get('attempt', '')}: {data.get('description', '')}"
+            )
+
+    pipeline.on_event(on_event)
+
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        console=console,
+    ) as progress:
+        task_display = progress.add_task("Running VerifyLoop...", total=None)
+        result = await pipeline.run(task, context, max_iterations)
+        progress.update(task_display, completed=1)
+
+    console.print()
+    console.print(format_step_table(result))
+
+    status_style = "bold green" if result.status == RunStatus.COMPLETED else "bold red"
+    console.print(f"\n[{status_style}]Status: {result.status.value}[/{status_style}]")
+    console.print(f"Duration: {result.duration_seconds:.2f}s")
+    console.print(f"Iterations: {result.iteration}")
+    console.print(
+        f"Tokens: {result.token_usage.prompt_tokens} prompt + "
+        f"{result.token_usage.completion_tokens} completion = "
+        f"{result.token_usage.total_tokens} total"
+    )
+
+    return result
+
+
+@click.group()
+def cli() -> None:
+    """VerifyLoop — Plan → Execute → Verify → Recover agent framework."""
+    pass
+
+
+@cli.command()
+@click.argument("task", required=False)
+@click.option("--task-file", type=click.Path(exists=True), help="Path to JSON task file")
+@click.option("--context", default="", help="Additional context for the task")
+@click.option("--model", default="gpt-4o", help="LLM model for planning")
+@click.option("--verify-model", default="reason-critic-7b", help="Verification model")
+@click.option("--max-iterations", default=5, type=int, help="Maximum plan-execute-verify loops")
+@click.option("--working-dir", default=".", help="Working directory")
+@click.option("--dry-run", is_flag=True, help="Plan only, don't execute")
+@click.option("--interactive", is_flag=True, help="Confirm each step before execution")
+@click.option("--sandbox", is_flag=True, help="Run bash commands in Docker sandbox")
+@click.option("--output", type=click.Path(), help="Save results to JSON file")
+def run(
+    task: str | None,
+    task_file: str | None,
+    context: str,
+    model: str,
+    verify_model: str,
+    max_iterations: int,
+    working_dir: str,
+    dry_run: bool,
+    interactive: bool,
+    sandbox: bool,
+    output: str | None,
+) -> None:
+    """Run a task through the Plan → Execute → Verify → Recover pipeline."""
+    if task_file:
+        path = Path(task_file)
+        data = json.loads(path.read_text())
+        task = data.get("task", "")
+        context = data.get("context", context)
+        model = data.get("model", model)
+        verify_model = data.get("verify_model", verify_model)
+        max_iterations = data.get("max_iterations", max_iterations)
+    elif not task:
+        console.print("[bold red]Error:[/] Provide a task string or --task-file")
+        sys.exit(1)
+
+    result = asyncio.run(run_pipeline(
+        task=task,
+        context=context,
+        model=model,
+        verify_model=verify_model,
+        max_iterations=max_iterations,
+        working_dir=working_dir,
+        dry_run=dry_run,
+        interactive=interactive,
+        sandbox=sandbox,
+    ))
+
+    if output:
+        output_path = Path(output)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        output_path.write_text(result.model_dump_json(indent=2))
+        console.print(f"\n[dim]Results saved to {output}[/]")
+
+
+if __name__ == "__main__":
+    cli()

verifyloop/executor.py

@@ -0,0 +1,330 @@
+"""Execute phase: run tools with structured results."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+from verifyloop.models import ExecuteStep
+
+
+class Executor:
+    def __init__(
+        self,
+        working_dir: str = ".",
+        sandbox: bool = False,
+        sandbox_image: str = "python:3.11-slim",
+        timeout: int = 120,
+    ) -> None:
+        self.working_dir = Path(working_dir).resolve()
+        self.sandbox = sandbox
+        self.sandbox_image = sandbox_image
+        self.timeout = timeout
+        self._file_history: dict[str, list[str]] = {}
+
+    async def execute(self, tool: str, arguments: dict[str, Any]) -> ExecuteStep:
+        handlers: dict[str, Any] = {
+            "bash": self.bash,
+            "edit": self.edit,
+            "read": self.read,
+            "write": self.write,
+            "web_search": self.web_search,
+            "web_fetch": self.web_fetch,
+        }
+        handler = handlers.get(tool)
+        if handler is None:
+            return ExecuteStep(
+                tool=tool,
+                arguments=arguments,
+                result=f"Unknown tool: {tool}",
+                success=False,
+                error=f"Unknown tool: {tool}",
+            )
+        try:
+            return await handler(**arguments)
+        except Exception as exc:
+            return ExecuteStep(
+                tool=tool,
+                arguments=arguments,
+                result="",
+                success=False,
+                error=str(exc),
+            )
+
+    async def bash(
+        self,
+        command: str,
+        working_dir: str | None = None,
+        timeout: int | None = None,
+    ) -> ExecuteStep:
+        start = time.monotonic()
+        cwd = str(Path(working_dir).resolve()) if working_dir else str(self.working_dir)
+        exec_timeout = timeout or self.timeout
+
+        try:
+            if self.sandbox:
+                result = await self._bash_docker(command, cwd, exec_timeout)
+            else:
+                proc = await asyncio.create_subprocess_shell(
+                    command,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE,
+                    cwd=cwd,
+                )
+                stdout, stderr = await asyncio.wait_for(
+                    proc.communicate(), timeout=exec_timeout
+                )
+                result = ExecuteStep(
+                    tool="bash",
+                    arguments={"command": command, "working_dir": cwd},
+                    result=stdout.decode(errors="replace"),
+                    success=proc.returncode == 0,
+                    duration_seconds=time.monotonic() - start,
+                    exit_code=proc.returncode,
+                    error=stderr.decode(errors="replace") if proc.returncode else None,
+                )
+
+            result.duration_seconds = time.monotonic() - start
+            return result
+
+        except asyncio.TimeoutError:
+            return ExecuteStep(
+                tool="bash",
+                arguments={"command": command},
+                result="",
+                success=False,
+                duration_seconds=time.monotonic() - start,
+                error=f"Command timed out after {exec_timeout}s",
+            )
+        except Exception as exc:
+            return ExecuteStep(
+                tool="bash",
+                arguments={"command": command},
+                result="",
+                success=False,
+                duration_seconds=time.monotonic() - start,
+                error=str(exc),
+            )
+
+    async def _bash_docker(
+        self, command: str, working_dir: str, timeout: int
+    ) -> ExecuteStep:
+        start = time.monotonic()
+        docker_cmd = [
+            "docker", "run", "--rm",
+            "-v", f"{working_dir}:/workspace",
+            "-w", "/workspace",
+            self.sandbox_image,
+            "sh", "-c", command,
+        ]
+        proc = await asyncio.create_subprocess_exec(
+            *docker_cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
+        return ExecuteStep(
+            tool="bash",
+            arguments={"command": command, "working_dir": working_dir, "sandbox": True},
+            result=stdout.decode(errors="replace"),
+            success=proc.returncode == 0,
+            duration_seconds=time.monotonic() - start,
+            exit_code=proc.returncode,
+            error=stderr.decode(errors="replace") if proc.returncode else None,
+        )
+
+    async def edit(
+        self,
+        file_path: str,
+        old_content: str,
+        new_content: str,
+    ) -> ExecuteStep:
+        start = time.monotonic()
+        target = Path(file_path)
+        if not target.is_absolute():
+            target = self.working_dir / target
+
+        try:
+            content = target.read_text()
+            if old_content not in content:
+                return ExecuteStep(
+                    tool="edit",
+                    arguments={"file_path": str(target), "old_content": old_content, "new_content": new_content},
+                    result=f"old_content not found in {target}",
+                    success=False,
+                    duration_seconds=time.monotonic() - start,
+                    error=f"old_content not found in {target}",
+                )
+
+            self._file_history.setdefault(str(target), []).append(content)
+            updated = content.replace(old_content, new_content, 1)
+            target.write_text(updated)
+
+            return ExecuteStep(
+                tool="edit",
+                arguments={"file_path": str(target), "old_content": old_content, "new_content": new_content},
+                result=f"Edited {target}: replaced 1 occurrence",
+                success=True,
+                duration_seconds=time.monotonic() - start,
+            )
+        except FileNotFoundError:
+            return ExecuteStep(
+                tool="edit",
+                arguments={"file_path": str(target)},
+                result="",
+                success=False,
+                duration_seconds=time.monotonic() - start,
+                error=f"File not found: {target}",
+            )
+        except Exception as exc:
+            return ExecuteStep(
+                tool="edit",
+                arguments={"file_path": str(target)},
+                result="",
+                success=False,
+                duration_seconds=time.monotonic() - start,
+                error=str(exc),
+            )
+
+    async def read(self, file_path: str) -> ExecuteStep:
+        start = time.monotonic()
+        target = Path(file_path)
+        if not target.is_absolute():
+            target = self.working_dir / target
+
+        try:
+            content = target.read_text()
+            self._file_history.setdefault(str(target), []).append(content)
+            return ExecuteStep(
+                tool="read",
+                arguments={"file_path": str(target)},
+                result=content,
+                success=True,
+                duration_seconds=time.monotonic() - start,
+            )
+        except FileNotFoundError:
+            return ExecuteStep(
+                tool="read",
+                arguments={"file_path": str(target)},
+                result="",
+                success=False,
+                duration_seconds=time.monotonic() - start,
+                error=f"File not found: {target}",
+            )
+        except Exception as exc:
+            return ExecuteStep(
+                tool="read",
+                arguments={"file_path": str(target)},
+                result="",
+                success=False,
+                duration_seconds=time.monotonic() - start,
+                error=str(exc),
+            )
+
+    async def write(
+        self,
+        file_path: str,
+        content: str,
+    ) -> ExecuteStep:
+        start = time.monotonic()
+        target = Path(file_path)
+        if not target.is_absolute():
+            target = self.working_dir / target
+
+        try:
+            target.parent.mkdir(parents=True, exist_ok=True)
+
+            if target.exists():
+                self._file_history.setdefault(str(target), []).append(target.read_text())
+
+            target.write_text(content)
+            return ExecuteStep(
+                tool="write",
+                arguments={"file_path": str(target), "content_length": len(content)},
+                result=f"Wrote {len(content)} chars to {target}",
+                success=True,
+                duration_seconds=time.monotonic() - start,
+            )
+        except Exception as exc:
+            return ExecuteStep(
+                tool="write",
+                arguments={"file_path": str(target)},
+                result="",
+                success=False,
+                duration_seconds=time.monotonic() - start,
+                error=str(exc),
+            )
+
+    async def web_search(self, query: str) -> ExecuteStep:
+        start = time.monotonic()
+        try:
+            async with httpx.AsyncClient(timeout=30) as client:
+                resp = await client.get(
+                    "https://api.duckduckgo.com/",
+                    params={"q": query, "format": "json", "no_html": 1},
+                )
+                data = resp.json()
+                results = []
+                for key in ("AbstractText", "Answer", "Definition"):
+                    if data.get(key):
+                        results.append(data[key])
+                for topic in data.get("RelatedTopics", [])[:5]:
+                    if isinstance(topic, dict) and topic.get("Text"):
+                        results.append(topic["Text"])
+                result_text = "\n".join(results) if results else "No results found"
+                return ExecuteStep(
+                    tool="web_search",
+                    arguments={"query": query},
+                    result=result_text,
+                    success=True,
+                    duration_seconds=time.monotonic() - start,
+                )
+        except Exception as exc:
+            return ExecuteStep(
+                tool="web_search",
+                arguments={"query": query},
+                result="",
+                success=False,
+                duration_seconds=time.monotonic() - start,
+                error=str(exc),
+            )
+
+    async def web_fetch(self, url: str) -> ExecuteStep:
+        start = time.monotonic()
+        try:
+            async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client:
+                resp = await client.get(url)
+                resp.raise_for_status()
+                return ExecuteStep(
+                    tool="web_fetch",
+                    arguments={"url": url},
+                    result=resp.text[:50000],
+                    success=True,
+                    duration_seconds=time.monotonic() - start,
+                )
+        except Exception as exc:
+            return ExecuteStep(
+                tool="web_fetch",
+                arguments={"url": url},
+                result="",
+                success=False,
+                duration_seconds=time.monotonic() - start,
+                error=str(exc),
+            )
+
+    def get_file_history(self, file_path: str) -> list[str]:
+        return self._file_history.get(file_path, [])
+
+    def rollback_file(self, file_path: str) -> bool:
+        history = self._file_history.get(file_path, [])
+        if not history:
+            return False
+        Path(file_path).write_text(history.pop())
+        return True