loopengt 0.1.0__py3-none-any.whl

loopengt/cli/commands/trace.py

@@ -0,0 +1,114 @@
+"""``loopengt trace`` — inspect the execution trace of a run."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import typer
+
+from loopengt.cli.formatters import (
+    console,
+    print_error,
+    print_info,
+    print_run_trace_tree,
+)
+
+
+def trace_cmd(
+    run_id: str = typer.Argument(..., help="Run ID to inspect."),
+    runs_dir: Path = typer.Option(
+        Path(".loopengt/runs"),
+        "--runs-dir",
+        help="Directory containing run traces.",
+    ),
+    output_format: str = typer.Option(
+        "tree",
+        "--format",
+        "-f",
+        help="Output format: tree, json, markdown.",
+    ),
+) -> None:
+    """Show step-by-step execution trace for a completed run."""
+    trace_path = runs_dir / f"{run_id}.json"
+
+    if not trace_path.exists():
+        # Try to find a partial match
+        candidates = list(runs_dir.glob(f"{run_id}*"))
+        if candidates:
+            trace_path = candidates[0]
+        else:
+            print_error(f"Trace not found: {trace_path}")
+            print_info(f"Available runs in {runs_dir}:")
+            if runs_dir.exists():
+                for f in sorted(runs_dir.glob("*.json")):
+                    print_info(f"  {f.stem}")
+            raise typer.Exit(code=1)
+
+    try:
+        trace_data = json.loads(trace_path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError) as exc:
+        print_error(f"Failed to read trace: {exc}")
+        raise typer.Exit(code=1) from exc
+
+    if output_format == "json":
+        console.print_json(json.dumps(trace_data, indent=2, default=str))
+    elif output_format == "markdown":
+        _print_markdown_trace(trace_data)
+    else:
+        _print_tree_trace(trace_data)
+
+
+def _print_tree_trace(trace_data: dict) -> None:
+    """Render trace as a Rich tree."""
+    # Reshape data for the tree formatter
+    steps = []
+    for step_name, step_state in trace_data.get("step_states", {}).items():
+        steps.append(
+            {
+                "step_name": step_name,
+                "status": step_state.get("status", "unknown"),
+                "duration_seconds": (
+                    step_state.get("result", {}) or {}
+                ).get("duration_seconds", 0),
+                "error": (step_state.get("result", {}) or {}).get("error"),
+            }
+        )
+
+    print_run_trace_tree(
+        {
+            "run_id": trace_data.get("run_id", "unknown"),
+            "status": trace_data.get("status", "unknown"),
+            "steps": steps,
+        }
+    )
+
+
+def _print_markdown_trace(trace_data: dict) -> None:
+    """Render trace as markdown text."""
+    lines = [
+        f"# Run Trace: {trace_data.get('run_id', 'unknown')}",
+        f"",
+        f"**Status**: {trace_data.get('status', 'unknown')}",
+        f"**Loop**: {trace_data.get('loop_name', 'unknown')}",
+        f"**Turns**: {trace_data.get('turn', 0)}",
+        f"",
+        f"## Steps",
+        f"",
+    ]
+
+    for step_name, step_state in trace_data.get("step_states", {}).items():
+        status = step_state.get("status", "unknown")
+        attempts = step_state.get("attempts", 0)
+        lines.append(f"### {step_name}")
+        lines.append(f"- Status: {status}")
+        lines.append(f"- Attempts: {attempts}")
+        result = step_state.get("result")
+        if result:
+            if result.get("error"):
+                lines.append(f"- Error: {result['error']}")
+            if result.get("duration_seconds"):
+                lines.append(f"- Duration: {result['duration_seconds']:.2f}s")
+        lines.append("")
+
+    console.print("\n".join(lines))

loopengt/cli/formatters.py

@@ -0,0 +1,125 @@
+"""Rich formatters for CLI output."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+from rich.text import Text
+from rich.tree import Tree
+
+console = Console()
+error_console = Console(stderr=True)
+
+
+def print_success(message: str) -> None:
+    """Print a success message with a green checkmark."""
+    console.print(f"[bold green]✓[/bold green] {message}")
+
+
+def print_error(message: str) -> None:
+    """Print an error message with a red cross."""
+    error_console.print(f"[bold red]✗[/bold red] {message}")
+
+
+def print_warning(message: str) -> None:
+    """Print a warning message with a yellow exclamation."""
+    console.print(f"[bold yellow]⚠[/bold yellow] {message}")
+
+
+def print_info(message: str) -> None:
+    """Print an info message with a blue arrow."""
+    console.print(f"[bold blue]→[/bold blue] {message}")
+
+
+def print_header(title: str, subtitle: str = "") -> None:
+    """Print a styled header panel."""
+    content = f"[bold]{title}[/bold]"
+    if subtitle:
+        content += f"\n[dim]{subtitle}[/dim]"
+    console.print(Panel(content, border_style="blue", padding=(1, 2)))
+
+
+def print_loop_spec_table(spec: dict[str, Any]) -> None:
+    """Print a loop spec summary as a Rich table."""
+    table = Table(title="Loop Specification", show_header=True, border_style="blue")
+    table.add_column("Field", style="cyan", width=20)
+    table.add_column("Value", style="white")
+
+    for key in ("name", "version", "goal", "pattern"):
+        if key in spec:
+            table.add_row(key, str(spec[key]))
+
+    if "agents" in spec:
+        agents = ", ".join(
+            a["name"] if isinstance(a, dict) else str(a)
+            for a in spec["agents"]
+        )
+        table.add_row("agents", agents)
+
+    if "steps" in spec:
+        steps = ", ".join(
+            s["name"] if isinstance(s, dict) else str(s)
+            for s in spec["steps"]
+        )
+        table.add_row("steps", steps)
+
+    console.print(table)
+
+
+def print_run_trace_tree(trace: dict[str, Any]) -> None:
+    """Print an execution trace as a Rich tree."""
+    run_id = trace.get("run_id", "unknown")
+    status = trace.get("status", "unknown")
+
+    status_color = {
+        "completed": "green",
+        "failed": "red",
+        "running": "yellow",
+        "pending": "dim",
+        "cancelled": "red",
+    }.get(status, "white")
+
+    tree = Tree(
+        Text.from_markup(
+            f"[bold]Run:[/bold] {run_id}  "
+            f"[{status_color}]{status}[/{status_color}]"
+        )
+    )
+
+    for step in trace.get("steps", []):
+        step_name = step.get("step_name", "unknown")
+        step_status = step.get("status", "unknown")
+        duration = step.get("duration_seconds", 0)
+
+        s_color = {
+            "completed": "green",
+            "failed": "red",
+            "running": "yellow",
+            "skipped": "dim",
+        }.get(step_status, "white")
+
+        step_node = tree.add(
+            Text.from_markup(
+                f"[{s_color}]●[/{s_color}] {step_name}  "
+                f"[dim]{duration:.2f}s[/dim]"
+            )
+        )
+
+        if step.get("error"):
+            step_node.add(Text(f"Error: {step['error']}", style="red"))
+
+    console.print(tree)
+
+
+def print_doctor_result(
+    name: str, passed: bool, detail: str = ""
+) -> None:
+    """Print a single diagnostic check result."""
+    icon = "[bold green]✓[/bold green]" if passed else "[bold red]✗[/bold red]"
+    msg = f"{icon} {name}"
+    if detail:
+        msg += f"  [dim]{detail}[/dim]"
+    console.print(msg)

loopengt/cli/main.py

@@ -0,0 +1,66 @@
+"""loopengt CLI — the main Typer application entry point."""
+
+from __future__ import annotations
+
+import typer
+
+from loopengt import __version__
+
+app = typer.Typer(
+    name="loopengt",
+    help="Loop Engineering Agent — design, orchestrate, and evaluate agent loops.",
+    no_args_is_help=True,
+    rich_markup_mode="rich",
+    add_completion=True,
+)
+
+
+def version_callback(value: bool) -> None:
+    """Print version and exit."""
+    if value:
+        typer.echo(f"loopengt {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def main(
+    version: bool = typer.Option(
+        False,
+        "--version",
+        "-v",
+        help="Show version and exit.",
+        callback=version_callback,
+        is_eager=True,
+    ),
+) -> None:
+    """Loop Engineering Agent CLI."""
+
+
+# ── Register sub-commands ──────────────────────────────────────────────
+
+def _register_commands() -> None:
+    """Import and register all CLI command groups."""
+    from loopengt.cli.commands.design import design_cmd
+    from loopengt.cli.commands.doctor import doctor_cmd
+    from loopengt.cli.commands.eval import eval_cmd
+    from loopengt.cli.commands.init import init_cmd
+    from loopengt.cli.commands.mcp_serve import mcp_cmd
+    from loopengt.cli.commands.run import run_cmd
+    from loopengt.cli.commands.template import template_app
+    from loopengt.cli.commands.trace import trace_cmd
+
+    app.command("init", help="Scaffold a new .loopengt/ project directory.")(init_cmd)
+    app.command("design", help="Design a loop from a natural-language goal.")(design_cmd)
+    app.command("run", help="Execute a loop from a YAML spec.")(run_cmd)
+    app.command("trace", help="Inspect the execution trace of a run.")(trace_cmd)
+    app.command("eval", help="Run evaluations against a completed run.")(eval_cmd)
+    app.command("doctor", help="Diagnose configuration and dependencies.")(doctor_cmd)
+    app.command("mcp", help="Start the MCP server.")(mcp_cmd)
+    app.add_typer(template_app, name="template", help="Manage loop templates.")
+
+
+_register_commands()
+
+
+if __name__ == "__main__":
+    app()

loopengt/core/__init__.py

@@ -0,0 +1 @@
+"""Core engine for loop design, execution, and evaluation."""

loopengt/core/evals/__init__.py

@@ -0,0 +1 @@
+"""Evaluation subsystem."""

loopengt/core/evals/judges.py

@@ -0,0 +1,216 @@
+"""LLM-as-judge evaluators."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+
+class JudgeResult:
+    """Result from an LLM judge evaluation."""
+
+    __slots__ = ("judge_name", "score", "passed", "reasoning", "raw_output")
+
+    def __init__(
+        self,
+        judge_name: str,
+        score: float,
+        passed: bool,
+        reasoning: str = "",
+        raw_output: str = "",
+    ) -> None:
+        self.judge_name = judge_name
+        self.score = score
+        self.passed = passed
+        self.reasoning = reasoning
+        self.raw_output = raw_output
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "judge_name": self.judge_name,
+            "score": self.score,
+            "passed": self.passed,
+            "reasoning": self.reasoning,
+        }
+
+
+class LLMJudge:
+    """Base class for LLM-as-judge evaluators.
+
+    Subclass and implement ``_build_prompt`` and ``_parse_response``
+    to create custom judges.
+
+    Usage::
+
+        judge = GoalAchievementJudge()
+        result = await judge.evaluate(trace_data, goal="build an API")
+    """
+
+    def __init__(self, name: str = "llm_judge") -> None:
+        self._name = name
+        self._log = logger.bind(judge=name)
+
+    async def evaluate(
+        self, trace: dict[str, Any], **kwargs: Any
+    ) -> JudgeResult:
+        """Evaluate a loop run trace.
+
+        Returns a ``JudgeResult`` with score and reasoning.
+        In stub mode (no LLM configured), returns a default result.
+        """
+        prompt = self._build_prompt(trace, **kwargs)
+
+        try:
+            response = await self._call_llm(prompt)
+            return self._parse_response(response)
+        except Exception as exc:  # noqa: BLE001
+            self._log.warning("judge.error", error=str(exc))
+            return JudgeResult(
+                judge_name=self._name,
+                score=0.0,
+                passed=False,
+                reasoning=f"Judge failed: {exc}",
+            )
+
+    def _build_prompt(self, trace: dict[str, Any], **kwargs: Any) -> str:
+        """Build the evaluation prompt.  Override in subclasses."""
+        return (
+            f"Evaluate this loop execution trace:\n"
+            f"{trace}\n\n"
+            f"Score from 0.0 to 1.0 and explain your reasoning."
+        )
+
+    def _parse_response(self, response: str) -> JudgeResult:
+        """Parse the LLM response.  Override in subclasses."""
+        return JudgeResult(
+            judge_name=self._name,
+            score=0.5,
+            passed=True,
+            reasoning=response,
+            raw_output=response,
+        )
+
+    async def _call_llm(self, prompt: str) -> str:
+        """Call the LLM to evaluate the trace."""
+        try:
+            import os
+            from openai import AsyncOpenAI
+        except ImportError:
+            self._log.warning("openai_not_installed", fallback="stub")
+            return '{"score": 0.5, "reasoning": "[stub] Install loopengt[llm] for real evaluation"}'
+
+        provider = os.environ.get("LOOPENGT_LLM_PROVIDER", "openai").lower()
+        if provider == "huggingface":
+            api_key = os.environ.get("HF_TOKEN")
+            base_url = "https://api-inference.huggingface.co/v1/"
+        else:
+            api_key = os.environ.get("OPENAI_API_KEY")
+            base_url = None
+
+        if not api_key:
+            self._log.warning("no_api_key", fallback="stub")
+            return '{"score": 0.5, "reasoning": "[stub] Missing API key"}'
+
+        client = AsyncOpenAI(api_key=api_key, base_url=base_url)
+        model = os.environ.get("LOOPENGT_EVAL_MODEL", "gpt-4o")
+
+        try:
+            response = await client.chat.completions.create(
+                model=model,
+                messages=[
+                    {"role": "system", "content": "You are an objective AI evaluator. Return JSON format."},
+                    {"role": "user", "content": prompt}
+                ]
+            )
+            return response.choices[0].message.content or ""
+        except Exception as e:
+            self._log.error("llm_eval_failed", error=str(e))
+            return f'{{"score": 0.0, "reasoning": "LLM call failed: {e}"}}'
+
+
+class GoalAchievementJudge(LLMJudge):
+    """Judge that evaluates whether the loop achieved its stated goal."""
+
+    def __init__(self) -> None:
+        super().__init__(name="goal_achievement")
+
+    def _build_prompt(self, trace: dict[str, Any], **kwargs: Any) -> str:
+        goal = kwargs.get("goal", trace.get("context", {}).get("goal", "unknown"))
+        status = trace.get("status", "unknown")
+        history = trace.get("history", [])
+
+        return (
+            f"## Goal\n{goal}\n\n"
+            f"## Run Status\n{status}\n\n"
+            f"## Execution History\n{history}\n\n"
+            f"## Instructions\n"
+            f"Score this run from 0.0 to 1.0 on goal achievement.\n"
+            f"Respond with JSON: {{\"score\": float, \"reasoning\": \"...\"}}"
+        )
+
+    def _parse_response(self, response: str) -> JudgeResult:
+        import json
+        score = 0.0
+        reasoning = "Failed to parse JSON"
+        passed = False
+        try:
+            content = response.strip()
+            if content.startswith("```json"):
+                content = content[7:-3].strip()
+            data = json.loads(content)
+            score = float(data.get("score", 0.0))
+            reasoning = data.get("reasoning", "")
+            passed = score >= 0.8
+        except (json.JSONDecodeError, ValueError):
+            reasoning = response
+        
+        return JudgeResult(
+            judge_name=self._name,
+            score=score,
+            passed=passed,
+            reasoning=reasoning,
+            raw_output=response,
+        )
+
+
+class QualityJudge(LLMJudge):
+    """Judge that evaluates the overall quality of loop outputs."""
+
+    def __init__(self) -> None:
+        super().__init__(name="quality")
+
+    def _build_prompt(self, trace: dict[str, Any], **kwargs: Any) -> str:
+        return (
+            f"Evaluate the quality of this loop's outputs:\n"
+            f"{trace.get('history', [])}\n\n"
+            f"Consider: completeness, correctness, clarity, efficiency.\n"
+            f"Score from 0.0 to 1.0 with reasoning.\n"
+            f"Respond with JSON: {{\"score\": float, \"reasoning\": \"...\"}}"
+        )
+
+    def _parse_response(self, response: str) -> JudgeResult:
+        import json
+        score = 0.0
+        reasoning = "Failed to parse JSON"
+        passed = False
+        try:
+            content = response.strip()
+            if content.startswith("```json"):
+                content = content[7:-3].strip()
+            data = json.loads(content)
+            score = float(data.get("score", 0.0))
+            reasoning = data.get("reasoning", "")
+            passed = score >= 0.8
+        except (json.JSONDecodeError, ValueError):
+            reasoning = response
+        
+        return JudgeResult(
+            judge_name=self._name,
+            score=score,
+            passed=passed,
+            reasoning=reasoning,
+            raw_output=response,
+        )

loopengt/core/evals/metrics.py

@@ -0,0 +1,119 @@
+"""Built-in evaluation metrics for loop runs."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+
+class MetricResult:
+    """Result of a single metric evaluation."""
+
+    __slots__ = ("name", "value", "passed", "detail")
+
+    def __init__(
+        self, name: str, value: Any, passed: bool, detail: str = ""
+    ) -> None:
+        self.name = name
+        self.value = value
+        self.passed = passed
+        self.detail = detail
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "value": self.value,
+            "passed": self.passed,
+            "detail": self.detail,
+        }
+
+
+def total_turns(trace: dict[str, Any]) -> MetricResult:
+    """Count total turns executed."""
+    turns = trace.get("turn", 0)
+    return MetricResult("total_turns", turns, True)
+
+
+def total_steps(trace: dict[str, Any]) -> MetricResult:
+    """Count total steps defined."""
+    count = len(trace.get("step_states", {}))
+    return MetricResult("total_steps", count, True)
+
+
+def completed_steps(trace: dict[str, Any]) -> MetricResult:
+    """Count successfully completed steps."""
+    count = sum(
+        1
+        for s in trace.get("step_states", {}).values()
+        if s.get("status") == "completed"
+    )
+    return MetricResult("completed_steps", count, True)
+
+
+def failed_steps(trace: dict[str, Any]) -> MetricResult:
+    """Count failed steps."""
+    count = sum(
+        1
+        for s in trace.get("step_states", {}).values()
+        if s.get("status") == "failed"
+    )
+    return MetricResult("failed_steps", count, count == 0)
+
+
+def success_rate(trace: dict[str, Any]) -> MetricResult:
+    """Calculate step success rate."""
+    total = len(trace.get("step_states", {}))
+    if total == 0:
+        return MetricResult("success_rate", 0.0, False, "No steps")
+
+    completed = sum(
+        1
+        for s in trace.get("step_states", {}).values()
+        if s.get("status") == "completed"
+    )
+    rate = completed / total
+    return MetricResult("success_rate", round(rate, 4), rate >= 0.5)
+
+
+def total_duration(trace: dict[str, Any]) -> MetricResult:
+    """Sum of all step durations."""
+    duration = sum(
+        (s.get("result") or {}).get("duration_seconds", 0)
+        for s in trace.get("step_states", {}).values()
+    )
+    return MetricResult("total_duration_seconds", round(duration, 2), True)
+
+
+def run_status(trace: dict[str, Any]) -> MetricResult:
+    """Check overall run status."""
+    status = trace.get("status", "unknown")
+    return MetricResult("run_status", status, status == "completed")
+
+
+# Registry of all built-in metrics
+BUILTIN_METRICS = {
+    "total_turns": total_turns,
+    "total_steps": total_steps,
+    "completed_steps": completed_steps,
+    "failed_steps": failed_steps,
+    "success_rate": success_rate,
+    "total_duration": total_duration,
+    "run_status": run_status,
+}
+
+
+def evaluate_all(
+    trace: dict[str, Any],
+    metric_names: list[str] | None = None,
+) -> list[MetricResult]:
+    """Run all (or selected) metrics against a trace."""
+    selected = metric_names or list(BUILTIN_METRICS.keys())
+    results = []
+    for name in selected:
+        fn = BUILTIN_METRICS.get(name)
+        if fn:
+            results.append(fn(trace))
+    return results