fluxloop-cli 0.1.0__py3-none-any.whl

fluxloop_cli/commands/parse.py

@@ -0,0 +1,293 @@
+"""Parse command for generating human-readable experiment artifacts."""
+
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, Iterable, List, Literal, Optional
+
+import typer
+from rich.console import Console
+
+console = Console()
+app = typer.Typer()
+
+
+@dataclass
+class Observation:
+    """A single observation entry parsed from observations.jsonl."""
+
+    trace_id: str
+    type: str
+    name: Optional[str]
+    start_time: Optional[str]
+    end_time: Optional[str]
+    level: Optional[str]
+    input: Optional[dict]
+    output: Optional[dict]
+    raw: dict
+
+    @property
+    def duration_ms(self) -> Optional[float]:
+        """Return duration in milliseconds if timestamps are available."""
+
+        if not self.start_time or not self.end_time:
+            return None
+
+        try:
+            start = datetime.fromisoformat(self.start_time.replace("Z", "+00:00"))
+            end = datetime.fromisoformat(self.end_time.replace("Z", "+00:00"))
+            return (end - start).total_seconds() * 1000
+        except ValueError:
+            return None
+
+
+@dataclass
+class TraceSummary:
+    """Reduced structure for entries inside trace_summary.jsonl."""
+
+    trace_id: str
+    iteration: int
+    persona: Optional[str]
+    input_text: str
+    output_text: Optional[str]
+    duration_ms: float
+    success: bool
+    raw: dict
+
+
+def _load_observations(path: Path) -> Dict[str, List[Observation]]:
+    """Load observations grouped by trace_id."""
+
+    grouped: Dict[str, List[Observation]] = defaultdict(list)
+    if not path.exists():
+        raise FileNotFoundError(f"observations.jsonl not found at {path}")
+
+    with path.open("r", encoding="utf-8") as handle:
+        for line_no, line in enumerate(handle, start=1):
+            line = line.strip()
+            if not line:
+                continue
+
+            try:
+                payload = json.loads(line)
+            except json.JSONDecodeError as exc:
+                raise ValueError(
+                    f"Invalid JSON in observations.jsonl at line {line_no}: {exc}"
+                ) from exc
+
+            trace_id = payload.get("trace_id")
+            if not trace_id:
+                # Observations without trace are not relevant for per-trace visualization
+                continue
+
+            grouped[trace_id].append(
+                Observation(
+                    trace_id=trace_id,
+                    type=payload.get("type", "unknown"),
+                    name=payload.get("name"),
+                    start_time=payload.get("start_time"),
+                    end_time=payload.get("end_time"),
+                    level=payload.get("level"),
+                    input=payload.get("input"),
+                    output=payload.get("output"),
+                    raw=payload,
+                )
+            )
+
+    return grouped
+
+
+def _load_trace_summaries(path: Path) -> Iterable[TraceSummary]:
+    """Yield trace summaries from trace_summary.jsonl."""
+
+    if not path.exists():
+        raise FileNotFoundError(f"trace_summary.jsonl not found at {path}")
+
+    with path.open("r", encoding="utf-8") as handle:
+        for line_no, line in enumerate(handle, start=1):
+            line = line.strip()
+            if not line:
+                continue
+
+            try:
+                payload = json.loads(line)
+            except json.JSONDecodeError as exc:
+                raise ValueError(
+                    f"Invalid JSON in trace_summary.jsonl at line {line_no}: {exc}"
+                ) from exc
+
+            trace_id = payload.get("trace_id")
+            if not trace_id:
+                continue
+
+            yield TraceSummary(
+                trace_id=trace_id,
+                iteration=payload.get("iteration", 0),
+                persona=payload.get("persona"),
+                input_text=payload.get("input", ""),
+                output_text=payload.get("output"),
+                duration_ms=payload.get("duration_ms", 0.0),
+                success=payload.get("success", False),
+                raw=payload,
+            )
+
+
+def _format_json_block(data: Optional[dict], *, indent: int = 2) -> str:
+    """Render a JSON dictionary as a fenced code block."""
+
+    if data is None:
+        return "(no data)"
+
+    try:
+        return "```json\n" + json.dumps(data, indent=indent, ensure_ascii=False) + "\n```"
+    except (TypeError, ValueError):
+        # Fallback to raw repr when data contains non-serializable content
+        return "```\n" + repr(data) + "\n```"
+
+
+def _format_markdown(
+    trace: TraceSummary,
+    observations: List[Observation],
+) -> str:
+    """Create markdown visualization for a single trace."""
+
+    observations_sorted = sorted(
+        observations,
+        key=lambda obs: (obs.start_time or "", obs.end_time or ""),
+    )
+
+    header = (
+        "---\n"
+        f"trace_id: \"{trace.trace_id}\"\n"
+        f"iteration: {trace.iteration}\n"
+        f"persona: {json.dumps(trace.persona) if trace.persona else 'null'}\n"
+        f"duration_ms: {trace.duration_ms:.2f}\n"
+        f"success: {'true' if trace.success else 'false'}\n"
+        "---\n\n"
+    )
+
+    summary_section = (
+        "# Trace Analysis\n\n"
+        "## Summary\n"
+        f"- Trace ID: `{trace.trace_id}`\n"
+        f"- Iteration: `{trace.iteration}`\n"
+        f"- Persona: `{trace.persona or 'N/A'}`\n"
+        f"- Duration: `{trace.duration_ms:.2f} ms`\n"
+        f"- Success: `{trace.success}`\n"
+        "\n"
+        "### Input\n"
+        f"{_format_json_block({'input': trace.input_text})}\n\n"
+        "### Output\n"
+        f"{_format_json_block({'output': trace.output_text})}\n\n"
+    )
+
+    timeline_lines = ["## Timeline\n"]
+
+    for index, obs in enumerate(observations_sorted, start=1):
+        duration = obs.duration_ms
+        duration_str = f"{duration:.2f} ms" if duration is not None else "N/A"
+        start = obs.start_time or "N/A"
+        end = obs.end_time or "N/A"
+        timeline_lines.append(
+            "---\n"
+            f"### Step {index}: [{obs.type}] {obs.name or 'unknown'}\n"
+            f"- Start: `{start}`\n"
+            f"- End: `{end}`\n"
+            f"- Duration: `{duration_str}`\n"
+            f"- Level: `{obs.level or 'N/A'}`\n"
+            "\n"
+            "**Input**\n"
+            f"{_format_json_block(obs.input)}\n\n"
+            "**Output**\n"
+            f"{_format_json_block(obs.output)}\n\n"
+        )
+
+    if not observations_sorted:
+        timeline_lines.append("(no observations recorded)\n")
+
+    return header + summary_section + "".join(timeline_lines)
+
+
+def _slugify(name: str) -> str:
+    """Create a filesystem-safe slug from a trace identifier."""
+
+    return "".join(c if c.isalnum() or c in {"-", "_"} else "-" for c in name)
+
+
+def _ensure_experiment_dir(path: Path) -> Path:
+    if not path.is_dir():
+        raise typer.BadParameter(f"Experiment directory not found: {path}")
+    return path
+
+
+@app.command()
+def experiment(
+    experiment_dir: Path = typer.Argument(..., help="Path to the experiment output directory"),
+    output: Path = typer.Option(
+        Path("per_trace_analysis"),
+        "--output",
+        "-o",
+        help="Directory name (relative to experiment_dir) to store parsed files",
+    ),
+    fmt: Literal["md"] = typer.Option(
+        "md",
+        "--format",
+        "-f",
+        help="Output format (currently only 'md' supported)",
+    ),
+    overwrite: bool = typer.Option(
+        False,
+        "--overwrite",
+        help="Overwrite the output directory if it already exists",
+    ),
+):
+    """Parse experiment artifacts into readable per-trace files."""
+
+    if fmt != "md":
+        raise typer.BadParameter("Only 'md' format is currently supported")
+
+    experiment_dir = _ensure_experiment_dir(experiment_dir)
+    output_dir = experiment_dir / output
+
+    if output_dir.exists():
+        if not overwrite:
+            raise typer.BadParameter(
+                f"Output directory already exists: {output_dir}. Use --overwrite to replace."
+            )
+    else:
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+    console.print(
+        f"📂 Loading experiment from: [cyan]{experiment_dir.resolve()}[/cyan]"
+    )
+
+    observations_path = experiment_dir / "observations.jsonl"
+    trace_summary_path = experiment_dir / "trace_summary.jsonl"
+
+    observations = _load_observations(observations_path)
+    summaries = list(_load_trace_summaries(trace_summary_path))
+
+    if not summaries:
+        console.print("[yellow]No trace summaries found. Nothing to parse.[/yellow]")
+        raise typer.Exit(0)
+
+    console.print(
+        f"📝 Found {len(summaries)} trace summaries. Generating markdown..."
+    )
+
+    for summary in summaries:
+        trace_observations = observations.get(summary.trace_id, [])
+        content = _format_markdown(summary, trace_observations)
+        file_name = f"{summary.iteration:02d}_{_slugify(summary.trace_id)}.{fmt}"
+        target_path = output_dir / file_name
+        target_path.write_text(content, encoding="utf-8")
+
+    console.print(
+        f"✅ Generated {len(summaries)} files in: [green]{output_dir.resolve()}[/green]"
+    )
+
+

fluxloop_cli/commands/run.py

@@ -0,0 +1,310 @@
+"""
+Run command for executing experiments and simulations.
+"""
+
+import asyncio
+import sys
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.console import Console
+from rich.live import Live
+from rich.panel import Panel
+from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TimeRemainingColumn
+from rich.table import Table
+
+from ..runner import ExperimentRunner
+from ..config_loader import load_experiment_config
+from ..constants import DEFAULT_CONFIG_PATH, DEFAULT_ROOT_DIR_NAME
+from ..project_paths import (
+    resolve_config_path,
+    resolve_project_relative,
+)
+
+app = typer.Typer()
+console = Console()
+
+
+@app.command()
+def experiment(
+    config_file: Path = typer.Option(
+        DEFAULT_CONFIG_PATH,
+        "--config",
+        "-c",
+        help="Path to experiment configuration file",
+    ),
+    project: Optional[str] = typer.Option(
+        None,
+        "--project",
+        help="Project name under the FluxLoop root",
+    ),
+    root: Path = typer.Option(
+        Path(DEFAULT_ROOT_DIR_NAME),
+        "--root",
+        help="FluxLoop root directory",
+    ),
+    iterations: Optional[int] = typer.Option(
+        None,
+        "--iterations",
+        "-i",
+        help="Override number of iterations",
+    ),
+    personas: Optional[str] = typer.Option(
+        None,
+        "--personas",
+        "-p",
+        help="Comma-separated list of personas to use",
+    ),
+    output_dir: Optional[Path] = typer.Option(
+        None,
+        "--output",
+        "-o",
+        help="Override output directory",
+    ),
+    no_collector: bool = typer.Option(
+        False,
+        "--no-collector",
+        help="Run without sending data to collector",
+    ),
+    dry_run: bool = typer.Option(
+        False,
+        "--dry-run",
+        help="Show what would be run without executing",
+    ),
+):
+    """
+    Run an experiment based on configuration file.
+    
+    This command:
+    - Loads experiment configuration
+    - Generates prompt variations
+    - Runs agent with each variation
+    - Collects traces and metrics
+    - Generates summary report
+    """
+    # Check if config file exists
+    resolved_config = resolve_config_path(config_file, project, root)
+    if not resolved_config.exists():
+        console.print(f"[red]Error:[/red] Configuration file not found: {config_file}")
+        console.print("\nRun [cyan]fluxloop init project[/cyan] to create a configuration file.")
+        raise typer.Exit(1)
+    
+    # Load configuration
+    console.print(f"📋 Loading configuration from: [cyan]{resolved_config}[/cyan]")
+    
+    try:
+        config = load_experiment_config(resolved_config)
+    except Exception as e:
+        console.print(f"[red]Error loading configuration:[/red] {e}")
+        raise typer.Exit(1)
+    
+    # Override configuration if needed
+    if iterations is not None:
+        config.iterations = iterations
+    
+    if personas:
+        persona_list = [p.strip() for p in personas.split(",")]
+        config.personas = [p for p in config.personas if p.name in persona_list]
+    
+    if output_dir:
+        resolved_output = resolve_project_relative(output_dir, project, root)
+        config.output_directory = str(resolved_output)
+    
+    # Load inputs to ensure accurate counts before showing the summary
+    try:
+        runner = ExperimentRunner(config, no_collector=no_collector)
+        loaded_inputs = asyncio.run(runner._load_inputs())  # type: ignore[attr-defined]
+    except Exception as e:
+        console.print(f"[red]Error preparing inputs:[/red] {e}")
+        raise typer.Exit(1)
+
+    config.set_resolved_input_count(len(loaded_inputs))
+    total_runs = config.estimate_total_runs()
+    
+    summary = Table(title="Experiment Summary", show_header=False)
+    summary.add_column("Property", style="cyan")
+    summary.add_column("Value", style="white")
+    
+    summary.add_row("Name", config.name)
+    summary.add_row("Iterations", str(config.iterations))
+    summary.add_row("Personas", str(len(config.personas)))
+    summary.add_row(
+        "Input Source",
+        "external file" if config.has_external_inputs() else "base_inputs",
+    )
+    summary.add_row("Total Runs", str(total_runs))
+    summary.add_row("Output", config.output_directory)
+    
+    console.print(summary)
+    
+    if dry_run:
+        console.print("\n[yellow]Dry run mode - no execution will occur[/yellow]")
+        return
+    
+    # Confirm execution
+    if total_runs > 100:
+        console.print(
+            f"\n[yellow]Warning:[/yellow] This will execute {total_runs} runs. "
+            "This may take a while and incur API costs."
+        )
+    else:
+        console.print(
+            f"\nThis will execute {total_runs} runs."
+        )
+
+    if not typer.confirm("Continue?"):
+        raise typer.Abort()
+    
+    # Create runner
+    # Run experiment with progress tracking
+    console.print("\n[bold green]▶️ Starting experiment...[/bold green]\n")
+
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
+        TimeRemainingColumn(),
+        TextColumn("({task.completed} of {task.total})"),
+        console=console,
+    ) as progress:
+        # Create main task
+        main_task = progress.add_task(
+            f"Running {config.name}",
+            total=total_runs,
+        )
+
+        def _progress_callback():
+            progress.advance(main_task)
+
+        # Run experiment
+        try:
+            results = asyncio.run(
+                runner.run_experiment(
+                    progress_callback=_progress_callback
+                )
+            )
+        except KeyboardInterrupt:
+            console.print("\n[yellow]Experiment interrupted by user[/yellow]")
+            raise typer.Exit(1)
+        except Exception as e:
+            console.print(f"\n[red]Error during experiment:[/red] {e}")
+            # Debug mode - show full traceback
+            if "--debug" in sys.argv:
+                console.print_exception()
+            raise typer.Exit(1)
+    
+    config.set_resolved_input_count(results.get("input_count", config.get_input_count()))
+
+    input_total_runs = config.estimate_total_runs()
+
+    if input_total_runs != total_runs:
+        console.print(
+            f"\n[yellow]Notice:[/yellow] Effective total runs adjusted to {input_total_runs} "
+            "after loading inputs."
+        )
+
+    _display_results(results)
+
+
+@app.command()
+def single(
+    agent_path: str = typer.Argument(
+        ...,
+        help="Module path to agent (e.g., my_agent.main)",
+    ),
+    input_text: str = typer.Argument(
+        ...,
+        help="Input text for the agent",
+    ),
+    function_name: str = typer.Option(
+        "run",
+        "--function",
+        "-f",
+        help="Function name to call",
+    ),
+    trace_name: Optional[str] = typer.Option(
+        None,
+        "--trace-name",
+        help="Name for the trace",
+    ),
+    no_collector: bool = typer.Option(
+        False,
+        "--no-collector",
+        help="Run without sending data to collector",
+    ),
+):
+    """
+    Run a single agent execution.
+    
+    Quick way to test an agent without a full experiment configuration.
+    """
+    console.print(f"🤖 Running agent: [cyan]{agent_path}.{function_name}[/cyan]")
+    console.print(f"📝 Input: {input_text[:100]}{'...' if len(input_text) > 100 else ''}")
+    
+    # Create minimal runner
+    from ..runner import SingleRunner
+    
+    runner = SingleRunner(
+        module_path=agent_path,
+        function_name=function_name,
+        trace_name=trace_name or f"single_{agent_path}",
+        no_collector=no_collector,
+    )
+    
+    # Run agent
+    console.print("\n[bold]Executing...[/bold]\n")
+    
+    try:
+        result = asyncio.run(runner.run(input_text))
+        
+        console.print(Panel(
+            str(result),
+            title="[bold green]Result[/bold green]",
+            border_style="green",
+        ))
+        
+    except Exception as e:
+        console.print(f"[red]Error:[/red] {e}")
+        # Debug mode - show full traceback
+        if "--debug" in sys.argv:
+            console.print_exception()
+        raise typer.Exit(1)
+
+
+def _display_results(results: dict):
+    """Display experiment results."""
+    console.print("\n" + "="*50)
+    console.print("[bold green]Experiment Complete![/bold green]")
+    console.print("="*50 + "\n")
+    
+    # Create results table
+    table = Table(title="Results Summary")
+    table.add_column("Metric", style="cyan")
+    table.add_column("Value", style="white")
+    
+    # Add metrics
+    table.add_row("Total Runs", str(results.get("total_runs", 0)))
+    table.add_row("Successful", str(results.get("successful", 0)))
+    table.add_row("Failed", str(results.get("failed", 0)))
+    
+    success_rate = results.get("success_rate", 0) * 100
+    table.add_row("Success Rate", f"{success_rate:.1f}%")
+    
+    avg_duration = results.get("avg_duration_ms", 0)
+    table.add_row("Avg Duration", f"{avg_duration:.0f}ms")
+    
+    console.print(table)
+    
+    # Show output location
+    if results.get("output_dir"):
+        console.print(f"\n📁 Results saved to: [cyan]{results['output_dir']}[/cyan]")
+    
+    # Show trace URLs if available
+    if results.get("trace_urls"):
+        console.print("\n🔗 View traces:")
+        for url in results["trace_urls"][:5]:  # Show first 5
+            console.print(f"   {url}")
+        if len(results["trace_urls"]) > 5:
+            console.print(f"   ... and {len(results['trace_urls']) - 5} more")