verifyloop 0.1.0__py3-none-any.whl

verifyloop/memory.py

@@ -0,0 +1,197 @@
+"""Memory system: short-term (in-process) and long-term (persistent file) stores."""
+
+from __future__ import annotations
+
+import json
+import time
+from abc import ABC, abstractmethod
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import aiofiles
+
+
+class MemoryStore(ABC):
+    @abstractmethod
+    async def store(self, key: str, value: Any, namespace: str = "default") -> None:
+        ...
+
+    @abstractmethod
+    async def retrieve(self, key: str, namespace: str = "default") -> Any | None:
+        ...
+
+    @abstractmethod
+    async def search(self, query: str, namespace: str = "default", limit: int = 10) -> list[dict[str, Any]]:
+        ...
+
+    @abstractmethod
+    async def delete(self, key: str, namespace: str = "default") -> bool:
+        ...
+
+    @abstractmethod
+    async def list_keys(self, namespace: str = "default") -> list[str]:
+        ...
+
+
+class InMemoryStore(MemoryStore):
+    def __init__(self) -> None:
+        self._store: dict[str, dict[str, dict[str, Any]]] = {}
+
+    def _ns(self, namespace: str) -> dict[str, dict[str, Any]]:
+        if namespace not in self._store:
+            self._store[namespace] = {}
+        return self._store[namespace]
+
+    async def store(self, key: str, value: Any, namespace: str = "default") -> None:
+        ns = self._ns(namespace)
+        ns[key] = {
+            "value": value,
+            "stored_at": datetime.now(timezone.utc).isoformat(),
+            "access_count": ns.get(key, {}).get("access_count", 0),
+        }
+
+    async def retrieve(self, key: str, namespace: str = "default") -> Any | None:
+        ns = self._ns(namespace)
+        entry = ns.get(key)
+        if entry is None:
+            return None
+        entry["access_count"] = entry.get("access_count", 0) + 1
+        return entry["value"]
+
+    async def search(self, query: str, namespace: str = "default", limit: int = 10) -> list[dict[str, Any]]:
+        ns = self._ns(namespace)
+        query_lower = query.lower()
+        results = []
+        for key, entry in ns.items():
+            value_str = str(entry.get("value", "")).lower()
+            if query_lower in value_str or query_lower in key.lower():
+                results.append({"key": key, **entry})
+        results.sort(key=lambda r: r.get("access_count", 0), reverse=True)
+        return results[:limit]
+
+    async def delete(self, key: str, namespace: str = "default") -> bool:
+        ns = self._ns(namespace)
+        if key in ns:
+            del ns[key]
+            return True
+        return False
+
+    async def list_keys(self, namespace: str = "default") -> list[str]:
+        return list(self._ns(namespace).keys())
+
+
+class FileStore(MemoryStore):
+    def __init__(self, base_dir: str = ".verifyloop_memory") -> None:
+        self.base_dir = Path(base_dir)
+        self._cache: dict[str, dict[str, dict[str, Any]]] = {}
+
+    def _ns_path(self, namespace: str) -> Path:
+        return self.base_dir / f"{namespace}.json"
+
+    async def _load_ns(self, namespace: str) -> dict[str, dict[str, Any]]:
+        if namespace in self._cache:
+            return self._cache[namespace]
+        path = self._ns_path(namespace)
+        if path.exists():
+            async with aiofiles.open(path, "r") as f:
+                data = json.loads(await f.read())
+            self._cache[namespace] = data
+            return data
+        self._cache[namespace] = {}
+        return {}
+
+    async def _save_ns(self, namespace: str, data: dict[str, dict[str, Any]]) -> None:
+        self._cache[namespace] = data
+        path = self._ns_path(namespace)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        async with aiofiles.open(path, "w") as f:
+            await f.write(json.dumps(data, indent=2, default=str))
+
+    async def store(self, key: str, value: Any, namespace: str = "default") -> None:
+        data = await self._load_ns(namespace)
+        data[key] = {
+            "value": value,
+            "stored_at": datetime.now(timezone.utc).isoformat(),
+            "access_count": data.get(key, {}).get("access_count", 0),
+        }
+        await self._save_ns(namespace, data)
+
+    async def retrieve(self, key: str, namespace: str = "default") -> Any | None:
+        data = await self._load_ns(namespace)
+        entry = data.get(key)
+        if entry is None:
+            return None
+        entry["access_count"] = entry.get("access_count", 0) + 1
+        await self._save_ns(namespace, data)
+        return entry["value"]
+
+    async def search(self, query: str, namespace: str = "default", limit: int = 10) -> list[dict[str, Any]]:
+        data = await self._load_ns(namespace)
+        query_lower = query.lower()
+        results = []
+        for key, entry in data.items():
+            value_str = str(entry.get("value", "")).lower()
+            if query_lower in value_str or query_lower in key.lower():
+                results.append({"key": key, **entry})
+        results.sort(key=lambda r: r.get("access_count", 0), reverse=True)
+        return results[:limit]
+
+    async def delete(self, key: str, namespace: str = "default") -> bool:
+        data = await self._load_ns(namespace)
+        if key in data:
+            del data[key]
+            await self._save_ns(namespace, data)
+            return True
+        return False
+
+    async def list_keys(self, namespace: str = "default") -> list[str]:
+        data = await self._load_ns(namespace)
+        return list(data.keys())
+
+
+class ConversationContext:
+    def __init__(self, memory: MemoryStore | None = None) -> None:
+        self.memory = memory or InMemoryStore()
+        self._messages: list[dict[str, str]] = []
+        self._file_context: dict[str, str] = {}
+
+    def add_message(self, role: str, content: str) -> None:
+        self._messages.append({"role": role, "content": content})
+
+    def get_messages(self) -> list[dict[str, str]]:
+        return list(self._messages)
+
+    def add_file_context(self, file_path: str, content: str) -> None:
+        self._file_context[file_path] = content
+        if self.memory:
+            import asyncio as _asyncio
+            try:
+                loop = _asyncio.get_event_loop()
+                if loop.is_running():
+                    _asyncio.ensure_future(
+                        self.memory.store(f"file:{file_path}", content, namespace="files")
+                    )
+                else:
+                    loop.run_until_complete(
+                        self.memory.store(f"file:{file_path}", content, namespace="files")
+                    )
+            except RuntimeError:
+                pass
+
+    def get_file_context(self, file_path: str) -> str | None:
+        return self._file_context.get(file_path)
+
+    def get_all_file_paths(self) -> list[str]:
+        return list(self._file_context.keys())
+
+    def build_context_string(self, max_files: int = 5) -> str:
+        parts = []
+        if self._messages:
+            last_msg = self._messages[-1] if self._messages else {}
+            parts.append(f"Last message: {last_msg.get('content', '')[:500]}")
+        if self._file_context:
+            for path, content in list(self._file_context.items())[:max_files]:
+                preview = content[:300] + "..." if len(content) > 300 else content
+                parts.append(f"File {path}:\n{preview}")
+        return "\n\n".join(parts)

verifyloop/models.py

@@ -0,0 +1,146 @@
+"""Core data models for the VerifyLoop framework."""
+
+from __future__ import annotations
+
+import time
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+class StepType(str, Enum):
+    PLAN = "plan"
+    EXECUTE = "execute"
+    VERIFY = "verify"
+    RECOVER = "recover"
+
+
+class Step(BaseModel):
+    step_type: StepType
+    content: str
+    tool_calls: list[dict[str, Any]] = Field(default_factory=list)
+    confidence: float = 0.0
+    timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+
+
+class Substep(BaseModel):
+    description: str
+    tool: str
+    arguments: dict[str, Any] = Field(default_factory=dict)
+    order: int = 0
+
+
+class PlanStep(BaseModel):
+    description: str
+    substeps: list[str] = Field(default_factory=list)
+    estimated_tools: list[str] = Field(default_factory=list)
+    substep_details: list[Substep] = Field(default_factory=list)
+    complexity: Literal["low", "medium", "high"] = "medium"
+    context_tokens: int = 0
+    estimated_duration_seconds: float = 0.0
+
+
+class ExecuteStep(BaseModel):
+    tool: str
+    arguments: dict[str, Any] = Field(default_factory=dict)
+    result: str = ""
+    success: bool = False
+    duration_seconds: float = 0.0
+    exit_code: int | None = None
+    error: str | None = None
+    artifacts: dict[str, str] = Field(default_factory=dict)
+
+    @property
+    def failed(self) -> bool:
+        return not self.success
+
+
+class VerifyCheckResult(BaseModel):
+    check: str
+    passed: bool
+    detail: str = ""
+
+
+class VerifyStep(BaseModel):
+    checks: list[str] = Field(default_factory=list)
+    check_results: list[VerifyCheckResult] = Field(default_factory=list)
+    passed: bool = False
+    confidence: float = 0.0
+    failures: list[str] = Field(default_factory=list)
+    fix_suggestions: list[str] = Field(default_factory=list)
+    verification_model: str = "reason-critic-7b"
+    used_trained_model: bool = False
+
+
+class RecoverStep(BaseModel):
+    original_error: str
+    recovery_attempt: str = ""
+    recovery_type: Literal["edit", "create", "retry", "simplify", "analyze"] = "edit"
+    success: bool = False
+    attempt_number: int = 1
+    max_attempts: int = 3
+    patched_arguments: dict[str, Any] = Field(default_factory=dict)
+
+    @property
+    def exhausted(self) -> bool:
+        return self.attempt_number >= self.max_attempts and not self.success
+
+
+class RunStatus(str, Enum):
+    PENDING = "pending"
+    PLANNING = "planning"
+    EXECUTING = "executing"
+    VERIFYING = "verifying"
+    RECOVERING = "recovering"
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+
+class TokenUsage(BaseModel):
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    total_tokens: int = 0
+
+    def merge(self, other: TokenUsage) -> TokenUsage:
+        return TokenUsage(
+            prompt_tokens=self.prompt_tokens + other.prompt_tokens,
+            completion_tokens=self.completion_tokens + other.completion_tokens,
+            total_tokens=self.total_tokens + other.total_tokens,
+        )
+
+
+class AgentRun(BaseModel):
+    task: str
+    steps: list[Step] = Field(default_factory=list)
+    status: RunStatus = RunStatus.PENDING
+    token_usage: TokenUsage = Field(default_factory=TokenUsage)
+    duration_seconds: float = 0.0
+    iteration: int = 0
+    max_iterations: int = 5
+    created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    completed_at: datetime | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    def add_step(self, step: Step) -> None:
+        self.steps.append(step)
+
+    def elapsed(self) -> float:
+        if self.completed_at:
+            return (self.completed_at - self.created_at).total_seconds()
+        return (datetime.now(timezone.utc) - self.created_at).total_seconds()
+
+
+class PipelineConfig(BaseModel):
+    model: str = "gpt-4o"
+    verify_model: str = "reason-critic-7b"
+    max_iterations: int = 5
+    confidence_threshold: float = 0.8
+    max_recovery_attempts: int = 3
+    working_dir: str = "."
+    dry_run: bool = False
+    interactive: bool = False
+    sandbox: bool = False
+    sandbox_image: str = "python:3.11-slim"
+    callbacks: dict[str, Any] = Field(default_factory=dict)

verifyloop/pipeline.py

@@ -0,0 +1,246 @@
+"""Full Pipeline: Plan → Execute → Verify → Recover loop."""
+
+from __future__ import annotations
+
+import time
+from datetime import datetime, timezone
+from typing import Any, Callable, Coroutine
+
+from verifyloop.executor import Executor
+from verifyloop.memory import ConversationContext, InMemoryStore, MemoryStore
+from verifyloop.models import (
+    AgentRun,
+    ExecuteStep,
+    PipelineConfig,
+    PlanStep,
+    RecoverStep,
+    RunStatus,
+    Step,
+    StepType,
+    TokenUsage,
+    VerifyStep,
+)
+from verifyloop.planner import PlanGenerator
+from verifyloop.recoverer import Recoverer
+from verifyloop.verifier import Verifier, VerifierConfig
+
+
+CallbackFn = Callable[[str, dict[str, Any]], Coroutine[Any, Any, None]] | None
+
+
+class AgentPipeline:
+    def __init__(self, config: PipelineConfig | None = None) -> None:
+        self.config = config or PipelineConfig()
+        self._planner = PlanGenerator(
+            model=self.config.model,
+        )
+        self._executor = Executor(
+            working_dir=self.config.working_dir,
+            sandbox=self.config.sandbox,
+            sandbox_image=self.config.sandbox_image,
+        )
+        self._verifier = Verifier(
+            VerifierConfig(
+                verify_model=self.config.verify_model,
+                confidence_threshold=self.config.confidence_threshold,
+            )
+        )
+        self._recoverer = Recoverer(
+            model=self.config.model,
+            max_recovery_attempts=self.config.max_recovery_attempts,
+        )
+        self._memory: MemoryStore = InMemoryStore()
+        self._context = ConversationContext(self._memory)
+        self._callbacks: list[CallbackFn] = []
+
+    @property
+    def token_usage(self) -> TokenUsage:
+        return (
+            self._planner.token_usage
+            .merge(self._verifier.token_usage)
+            .merge(self._recoverer.token_usage)
+        )
+
+    def on_event(self, callback: CallbackFn) -> None:
+        self._callbacks.append(callback)
+
+    async def _emit(self, event: str, data: dict[str, Any]) -> None:
+        for cb in self._callbacks:
+            if cb is not None:
+                try:
+                    await cb(event, data)
+                except Exception:
+                    pass
+
+    async def run(
+        self,
+        task: str,
+        context: str = "",
+        max_iterations: int | None = None,
+    ) -> AgentRun:
+        max_iters = max_iterations or self.config.max_iterations
+        run = AgentRun(
+            task=task,
+            max_iterations=max_iters,
+            status=RunStatus.PENDING,
+        )
+        start_time = time.monotonic()
+
+        try:
+            await self._emit("run_start", {"task": task})
+
+            for iteration in range(1, max_iters + 1):
+                run.iteration = iteration
+                await self._emit("iteration_start", {"iteration": iteration})
+
+                # Phase 1: Plan
+                run.status = RunStatus.PLANNING
+                await self._emit("phase_start", {"phase": "plan", "iteration": iteration})
+
+                plan = await self._planner.generate_plan(task, context or self._context.build_context_string())
+                run.add_step(Step(
+                    step_type=StepType.PLAN,
+                    content=plan.description,
+                    confidence=0.7,
+                ))
+                await self._emit("phase_complete", {
+                    "phase": "plan",
+                    "description": plan.description,
+                    "substeps": plan.substeps,
+                })
+
+                if self.config.dry_run:
+                    run.status = RunStatus.COMPLETED
+                    run.duration_seconds = time.monotonic() - start_time
+                    return run
+
+                # Phase 2: Execute
+                run.status = RunStatus.EXECUTING
+                await self._emit("phase_start", {"phase": "execute", "iteration": iteration})
+
+                execute_steps: list[ExecuteStep] = []
+                for substep in plan.substep_details:
+                    if self.config.interactive:
+                        proceed = await self._confirm_substep(substep)
+                        if not proceed:
+                            continue
+
+                    step_result = await self._executor.execute(substep.tool, substep.arguments)
+                    execute_steps.append(step_result)
+                    run.add_step(Step(
+                        step_type=StepType.EXECUTE,
+                        content=f"{substep.tool}: {substep.description}",
+                        tool_calls=[{"tool": substep.tool, "args": substep.arguments}],
+                        confidence=1.0 if step_result.success else 0.0,
+                    ))
+                    await self._emit("step_complete", {
+                        "tool": substep.tool,
+                        "success": step_result.success,
+                        "iteration": iteration,
+                    })
+
+                    if substep.tool == "read" and step_result.success:
+                        self._context.add_file_context(
+                            substep.arguments.get("file_path", ""), step_result.result
+                        )
+
+                # Phase 3: Verify
+                run.status = RunStatus.VERIFYING
+                await self._emit("phase_start", {"phase": "verify", "iteration": iteration})
+
+                verification = await self._verifier.verify_code_edits(plan, execute_steps)
+                run.add_step(Step(
+                    step_type=StepType.VERIFY,
+                    content=f"Passed: {verification.passed}, Confidence: {verification.confidence:.2f}",
+                    confidence=verification.confidence,
+                ))
+                await self._emit("phase_complete", {
+                    "phase": "verify",
+                    "passed": verification.passed,
+                    "confidence": verification.confidence,
+                    "failures": verification.failures,
+                })
+
+                if verification.passed and verification.confidence >= self.config.confidence_threshold:
+                    run.status = RunStatus.COMPLETED
+                    run.duration_seconds = time.monotonic() - start_time
+                    run.completed_at = datetime.now(timezone.utc)
+                    run.token_usage = self.token_usage
+                    await self._emit("run_complete", {"status": "completed", "iterations": iteration})
+                    return run
+
+                # Phase 4: Recover (if verification failed)
+                run.status = RunStatus.RECOVERING
+                await self._emit("phase_start", {"phase": "recover", "iteration": iteration})
+
+                failure_messages = verification.failures or ["Verification failed"]
+                all_errors = "; ".join(failure_messages)
+
+                for recovery_attempt in range(1, self.config.max_recovery_attempts + 1):
+                    recovery = await self._recoverer.recover(
+                        error=all_errors,
+                        context=self._context.build_context_string(),
+                        attempt=recovery_attempt,
+                        failed_step=execute_steps[-1] if execute_steps else None,
+                    )
+                    run.add_step(Step(
+                        step_type=StepType.RECOVER,
+                        content=f"Recovery attempt {recovery_attempt}: {recovery.recovery_attempt}",
+                        confidence=0.5,
+                    ))
+                    await self._emit("recovery_attempt", {
+                        "attempt": recovery_attempt,
+                        "type": recovery.recovery_type,
+                        "description": recovery.recovery_attempt,
+                    })
+
+                    if recovery.patched_arguments:
+                        tool = recovery.patched_arguments.get("tool", "bash")
+                        args = recovery.patched_arguments.get("arguments", {})
+                        recovery_exec = await self._executor.execute(tool, args)
+                        execute_steps.append(recovery_exec)
+
+                        # Re-verify after recovery
+                        recheck = await self._verifier.verify_code_edits(plan, execute_steps)
+                        if recheck.passed and recheck.confidence >= self.config.confidence_threshold:
+                            run.status = RunStatus.COMPLETED
+                            run.duration_seconds = time.monotonic() - start_time
+                            run.completed_at = datetime.now(timezone.utc)
+                            run.token_usage = self.token_usage
+                            await self._emit("run_complete", {"status": "completed_after_recovery"})
+                            return run
+
+                    if recovery.exhausted:
+                        break
+
+                # If we get here, recovery didn't fix it — loop back for next iteration
+                context = self._context.build_context_string() + f"\nPrevious failures: {all_errors}"
+
+            run.status = RunStatus.FAILED
+            run.duration_seconds = time.monotonic() - start_time
+            run.completed_at = datetime.now(timezone.utc)
+            run.token_usage = self.token_usage
+            await self._emit("run_complete", {"status": "failed", "iterations": max_iters})
+            return run
+
+        except Exception as exc:
+            run.status = RunStatus.FAILED
+            run.duration_seconds = time.monotonic() - start_time
+            run.completed_at = datetime.now(timezone.utc)
+            run.metadata["error"] = str(exc)
+            run.token_usage = self.token_usage
+            await self._emit("run_error", {"error": str(exc)})
+            return run
+
+    async def _confirm_substep(self, substep: Any) -> bool:
+        try:
+            from rich.console import Console
+            from rich.prompt import Confirm
+
+            console = Console()
+            console.print(f"\n[bold blue]Step:[/] {substep.description}")
+            console.print(f"  [dim]Tool: {substep.tool}[/dim]")
+            console.print(f"  [dim]Args: {substep.arguments}[/dim]")
+            return Confirm.ask("Execute this step?", default=True)
+        except ImportError:
+            return True