fluxloop-cli 0.2.16__tar.gz → 0.2.18__tar.gz

{fluxloop_cli-0.2.16 → fluxloop_cli-0.2.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fluxloop-cli
-Version: 0.2.16
+Version: 0.2.18
 Summary: FluxLoop CLI for running agent simulations
 Author-email: FluxLoop Team <team@fluxloop.dev>
 License: Apache-2.0
@@ -66,11 +66,30 @@ The legacy `setting.yaml` is still supported, but new projects created with
 - `fluxloop generate inputs` – produce input variations for the active project
 - `fluxloop run experiment` – execute an experiment using `configs/simulation.yaml`
 - `fluxloop parse experiment` – convert experiment outputs into readable artifacts
+- `fluxloop evaluate experiment` – score experiment outputs and generate evaluation reports
 - `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
 - `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
+- `fluxloop doctor` – summarize Python, FluxLoop CLI/MCP, and MCP index state for the active environment
 
 Run `fluxloop --help` or `fluxloop <command> --help` for more detail.
 
+## Quick Setup Script
+
+To prepare a fresh checkout (create `.venv`, install dependencies, and run diagnostics):
+
+```
+bash scripts/setup_fluxloop_env.sh --target-source-root path/to/your/source
+```
+
+Options:
+
+- `--python PATH` – choose a specific interpreter (default `python3`)
+- `--target-source-root PATH` – pre-populate VSCode `fluxloop.targetSourceRoot`
+- `--skip-doctor` – skip the final `fluxloop doctor` check
+
+After running the script, open the folder in VSCode and use `FluxLoop: Show Environment Info`
+or `FluxLoop: Run Doctor` to confirm the environment.
+
 ## Runner Integration Patterns
 
 Configure how FluxLoop calls your code in `configs/simulation.yaml`:

{fluxloop_cli-0.2.16 → fluxloop_cli-0.2.18}/README.md

@@ -26,11 +26,30 @@ The legacy `setting.yaml` is still supported, but new projects created with
 - `fluxloop generate inputs` – produce input variations for the active project
 - `fluxloop run experiment` – execute an experiment using `configs/simulation.yaml`
 - `fluxloop parse experiment` – convert experiment outputs into readable artifacts
+- `fluxloop evaluate experiment` – score experiment outputs and generate evaluation reports
 - `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
 - `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
+- `fluxloop doctor` – summarize Python, FluxLoop CLI/MCP, and MCP index state for the active environment
 
 Run `fluxloop --help` or `fluxloop <command> --help` for more detail.
 
+## Quick Setup Script
+
+To prepare a fresh checkout (create `.venv`, install dependencies, and run diagnostics):
+
+```
+bash scripts/setup_fluxloop_env.sh --target-source-root path/to/your/source
+```
+
+Options:
+
+- `--python PATH` – choose a specific interpreter (default `python3`)
+- `--target-source-root PATH` – pre-populate VSCode `fluxloop.targetSourceRoot`
+- `--skip-doctor` – skip the final `fluxloop doctor` check
+
+After running the script, open the folder in VSCode and use `FluxLoop: Show Environment Info`
+or `FluxLoop: Run Doctor` to confirm the environment.
+
 ## Runner Integration Patterns
 
 Configure how FluxLoop calls your code in `configs/simulation.yaml`:

{fluxloop_cli-0.2.16 → fluxloop_cli-0.2.18}/fluxloop_cli/__init__.py

@@ -2,7 +2,7 @@
 FluxLoop CLI - Command-line interface for running agent simulations.
 """
 
-__version__ = "0.2.16"
+__version__ = "0.2.18"
 
 from .main import app
 

fluxloop_cli-0.2.18/fluxloop_cli/commands/__init__.py

@@ -0,0 +1,15 @@
+"""CLI commands."""
+
+from . import config, doctor, evaluate, generate, init, parse, record, run, status
+
+__all__ = [
+    "config",
+    "doctor",
+    "evaluate",
+    "generate",
+    "init",
+    "parse",
+    "record",
+    "run",
+    "status",
+]

fluxloop_cli-0.2.18/fluxloop_cli/commands/doctor.py

@@ -0,0 +1,252 @@
+"""
+Doctor command for diagnosing FluxLoop environment issues.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import platform
+import subprocess
+import sys
+from dataclasses import dataclass, asdict
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+import typer
+from rich.console import Console
+from rich.padding import Padding
+from rich.panel import Panel
+from rich.table import Table
+
+from ..project_paths import resolve_config_directory, resolve_config_path
+from ..constants import DEFAULT_CONFIG_PATH, DEFAULT_ROOT_DIR_NAME
+
+app = typer.Typer(
+    help="Diagnose FluxLoop CLI, MCP and environment setup.",
+    invoke_without_command=True,
+    no_args_is_help=False,
+)
+console = Console()
+
+
+@dataclass
+class CommandResult:
+    success: bool
+    path: Optional[str]
+    output: Optional[str]
+    error: Optional[str]
+
+
+def _which(executable: str) -> Optional[str]:
+    from shutil import which
+
+    return which(executable)
+
+
+def _run_command(command: str, *args: str) -> CommandResult:
+    candidate = _which(command)
+    if not candidate:
+        return CommandResult(False, None, None, f"{command} not found on PATH")
+
+    try:
+        completed = subprocess.run(
+            [candidate, *args],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        return CommandResult(True, candidate, completed.stdout.strip(), completed.stderr.strip() or None)
+    except subprocess.CalledProcessError as exc:
+        return CommandResult(
+            False,
+            candidate,
+            exc.stdout.strip() or None,
+            exc.stderr.strip() or exc.stdout.strip() or str(exc),
+        )
+
+
+def _detect_virtual_environment() -> Dict[str, Any]:
+    env_vars = {
+        "VIRTUAL_ENV": os.environ.get("VIRTUAL_ENV"),
+        "CONDA_PREFIX": os.environ.get("CONDA_PREFIX"),
+        "UV_PROJECT_ENV": os.environ.get("UV_PROJECT_ENV"),
+    }
+    virtualized = sys.prefix != getattr(sys, "base_prefix", sys.prefix)
+    return {
+        "python_executable": sys.executable,
+        "python_version": platform.python_version(),
+        "platform": platform.platform(),
+        "virtual_env": virtualized,
+        "virtual_env_path": (
+            env_vars["VIRTUAL_ENV"]
+            or env_vars["CONDA_PREFIX"]
+            or (Path(sys.prefix).as_posix() if virtualized else None)
+        ),
+        "environment_variables": {k: v for k, v in env_vars.items() if v},
+    }
+
+
+def _check_index_directory(index_dir: Path) -> Dict[str, Any]:
+    exists = index_dir.exists()
+    chunks_file = index_dir / "chunks.jsonl"
+    chunks_exists = chunks_file.exists()
+    chunks_size = chunks_file.stat().st_size if chunks_exists else 0
+    return {
+        "exists": exists,
+        "path": index_dir.as_posix(),
+        "chunks_exists": chunks_exists,
+        "chunks_size": chunks_size,
+    }
+
+
+def _resolve_project_info(root: Path, project: Optional[str]) -> Dict[str, Any]:
+    config_path = resolve_config_path(DEFAULT_CONFIG_PATH, project, root)
+    config_dir = resolve_config_directory(project, root)
+    return {
+        "root": str(root.resolve()),
+        "config_directory": str(config_dir.resolve()),
+        "project_yaml": str(config_path.resolve()),
+        "project_yaml_exists": config_path.exists(),
+        "config_directory_exists": config_dir.exists(),
+    }
+
+
+def _run_diagnostics(
+    project: Optional[str],
+    root: Path,
+    index_dir: Optional[Path],
+    output_json: bool,
+) -> None:
+    console.print(Panel.fit("FluxLoop Environment Doctor", border_style="blue"))
+
+    virtual_info = _detect_virtual_environment()
+    fluxloop_result = _run_command("fluxloop", "--version")
+    mcp_result = _run_command("fluxloop-mcp", "--help")
+    python_result = _run_command(sys.executable, "--version")
+
+    resolved_root = root
+    project_info = _resolve_project_info(resolved_root, project)
+    resolved_index_dir = (
+        index_dir
+        if index_dir is not None
+        else Path.home() / ".fluxloop" / "mcp" / "index" / "dev"
+    )
+    index_info = _check_index_directory(resolved_index_dir)
+
+    diagnostics = {
+        "python": {
+            "executable": sys.executable,
+            "version": virtual_info["python_version"],
+            "platform": virtual_info["platform"],
+            "command_output": python_result.output,
+        },
+        "virtual_environment": virtual_info,
+        "fluxloop_cli": asdict(fluxloop_result),
+        "fluxloop_mcp": asdict(mcp_result),
+        "project": project_info,
+        "mcp_index": index_info,
+    }
+
+    if output_json:
+        typer.echo(json.dumps(diagnostics, indent=2, default=str))
+        raise typer.Exit()
+
+    items_panel = Table(show_header=False, box=None)
+    items_panel.add_column("Component", style="cyan", no_wrap=True)
+    items_panel.add_column("Status")
+    items_panel.add_column("Details", style="dim")
+
+    items_panel.add_row(
+        "Python",
+        "[green]✓[/green]" if python_result.success else "[red]✗[/red]",
+        f"{virtual_info['python_version']} ({sys.executable})",
+    )
+
+    items_panel.add_row(
+        "Virtual Env",
+        "[green]✓[/green]" if virtual_info["virtual_env"] else "[yellow]–[/yellow]",
+        virtual_info["virtual_env_path"] or "Global interpreter",
+    )
+
+    items_panel.add_row(
+        "FluxLoop CLI",
+        "[green]✓[/green]" if fluxloop_result.success else "[red]✗[/red]",
+        fluxloop_result.path or fluxloop_result.error or "Not found",
+    )
+
+    items_panel.add_row(
+        "FluxLoop MCP",
+        "[green]✓[/green]" if mcp_result.success else "[red]✗[/red]",
+        mcp_result.path or mcp_result.error or "Not found",
+    )
+
+    index_status = "[green]✓[/green]" if index_info["exists"] else "[yellow]–[/yellow]"
+    index_details = index_info["path"]
+    if index_info["exists"]:
+        chunk_detail = (
+            f"chunks.jsonl ({index_info['chunks_size']} bytes)"
+            if index_info["chunks_exists"]
+            else "missing chunks.jsonl"
+        )
+        index_details = f"{index_details} • {chunk_detail}"
+
+    items_panel.add_row("MCP Index", index_status, index_details)
+
+    config_status = (
+        "[green]✓[/green]" if project_info["project_yaml_exists"] else "[yellow]–[/yellow]"
+    )
+    config_details = (
+        f"{project_info['project_yaml']} (project.yaml)"
+        if project_info["project_yaml_exists"]
+        else "Run: fluxloop init project"
+    )
+    items_panel.add_row("Project Config", config_status, config_details)
+
+    console.print(Padding(items_panel, (1, 0, 1, 0)))
+
+    if fluxloop_result.error or mcp_result.error:
+        console.print("[bold red]Errors[/bold red]")
+        if fluxloop_result.error:
+            console.print(f"• FluxLoop CLI: {fluxloop_result.error}")
+        if mcp_result.error:
+            console.print(f"• fluxloop-mcp: {mcp_result.error}")
+        console.print()
+
+    if diagnostics["fluxloop_cli"]["output"]:
+        console.print(Panel(diagnostics["fluxloop_cli"]["output"], title="fluxloop --version", border_style="green"))
+    if diagnostics["fluxloop_mcp"]["output"]:
+        console.print(Panel(diagnostics["fluxloop_mcp"]["output"], title="fluxloop-mcp --help", border_style="green"))
+
+    console.print(Padding(Panel.fit("Doctor completed", border_style="green"), (1, 0, 0, 0)))
+
+
+@app.callback()
+def doctor_callback(
+    ctx: typer.Context,
+    project: Optional[str] = typer.Option(
+        None, "--project", "-p", help="Project name under the FluxLoop root directory."
+    ),
+    root: Path = typer.Option(
+        Path(DEFAULT_ROOT_DIR_NAME),
+        "--root",
+        help="FluxLoop root directory (defaults to ./fluxloop).",
+    ),
+    index_dir: Optional[Path] = typer.Option(
+        None,
+        "--index-dir",
+        help="Override FluxLoop MCP index directory.",
+    ),
+    output_json: bool = typer.Option(
+        False,
+        "--json",
+        help="Output diagnostic information as JSON.",
+    ),
+) -> None:
+    """
+    Diagnose the FluxLoop CLI, MCP installation, virtual environment, and project configuration.
+    """
+    if ctx.invoked_subcommand:
+        return
+    _run_diagnostics(project, root, index_dir, output_json)
+

fluxloop_cli-0.2.18/fluxloop_cli/commands/evaluate.py

@@ -0,0 +1,183 @@
+"""Evaluate command for scoring experiment outputs."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.console import Console
+
+from ..evaluation import EvaluationOptions, load_evaluation_config, run_evaluation
+
+console = Console()
+app = typer.Typer(help="Evaluate experiment outputs and generate reports.")
+
+
+@app.command()
+def experiment(
+    experiment_dir: Path = typer.Argument(
+        ...,
+        help="Path to the experiment output directory",
+        exists=True,
+        dir_okay=True,
+        file_okay=False,
+        resolve_path=True,
+    ),
+    config: Path = typer.Option(
+        Path("configs/evaluation.yaml"),
+        "--config",
+        "-c",
+        help="Path to evaluation configuration file",
+    ),
+    output: Path = typer.Option(
+        Path("evaluation"),
+        "--output",
+        "-o",
+        help="Output directory name (relative to the experiment directory)",
+    ),
+    overwrite: bool = typer.Option(
+        False,
+        "--overwrite",
+        help="Overwrite output directory if it already exists",
+    ),
+    llm_api_key: Optional[str] = typer.Option(
+        None,
+        "--llm-api-key",
+        help="LLM API key for judge evaluators (optional)",
+        envvar="FLUXLOOP_LLM_API_KEY",
+    ),
+    sample_rate: Optional[float] = typer.Option(
+        None,
+        "--sample-rate",
+        help="Override LLM evaluation sample rate (0.0-1.0)",
+    ),
+    max_llm_calls: Optional[int] = typer.Option(
+        None,
+        "--max-llm",
+        help="Maximum number of LLM evaluations to run",
+    ),
+    report: Optional[str] = typer.Option(
+        None,
+        "--report",
+        help="Report output format to generate (md, html, both)",
+        metavar="FORMAT",
+    ),
+    report_template: Optional[Path] = typer.Option(
+        None,
+        "--report-template",
+        help="Path to custom HTML report template",
+        exists=False,
+        file_okay=True,
+        dir_okay=False,
+    ),
+    baseline: Optional[Path] = typer.Option(
+        None,
+        "--baseline",
+        help="Path to baseline summary.json file for comparisons",
+        exists=False,
+        file_okay=True,
+        dir_okay=False,
+    ),
+    verbose: bool = typer.Option(
+        False,
+        "--verbose",
+        help="Enable verbose logging",
+    ),
+):
+    """
+    Evaluate experiment outputs and generate aggregate reports.
+    """
+
+    resolved_experiment_dir = experiment_dir.resolve()
+    if not resolved_experiment_dir.is_dir():
+        raise typer.BadParameter(f"Experiment directory not found: {resolved_experiment_dir}")
+
+    if not config.is_absolute():
+        config_path = (Path.cwd() / config).resolve()
+    else:
+        config_path = config
+
+    if sample_rate is not None and not 0.0 <= sample_rate <= 1.0:
+        raise typer.BadParameter("--sample-rate must be between 0.0 and 1.0")
+
+    if max_llm_calls is not None and max_llm_calls < 0:
+        raise typer.BadParameter("--max-llm must be a non-negative integer")
+
+    report_format: Optional[str] = None
+    if report is not None:
+        candidate = report.lower()
+        if candidate not in {"md", "html", "both"}:
+            raise typer.BadParameter("--report must be one of: md, html, both")
+        report_format = candidate
+
+    report_template_path: Optional[Path] = None
+    if report_template is not None:
+        resolved_template = report_template
+        if not resolved_template.is_absolute():
+            resolved_template = (Path.cwd() / resolved_template).resolve()
+        if not resolved_template.exists():
+            raise typer.BadParameter(f"Report template not found: {resolved_template}")
+        report_template_path = resolved_template
+
+    baseline_path: Optional[Path] = None
+    if baseline is not None:
+        resolved_baseline = baseline
+        if not resolved_baseline.is_absolute():
+            resolved_baseline = (Path.cwd() / resolved_baseline).resolve()
+        baseline_path = resolved_baseline
+
+    try:
+        evaluation_config = load_evaluation_config(config_path)
+    except FileNotFoundError as exc:
+        raise typer.BadParameter(str(exc)) from exc
+    except Exception as exc:  # noqa: BLE001
+        raise typer.BadParameter(f"Failed to load evaluation config: {exc}") from exc
+
+    output_dir = output
+    if not output_dir.is_absolute():
+        output_dir = resolved_experiment_dir / output_dir
+
+    options = EvaluationOptions(
+        output_dir=output_dir,
+        overwrite=overwrite,
+        llm_api_key=llm_api_key,
+        sample_rate=sample_rate,
+        max_llm_calls=max_llm_calls,
+        verbose=verbose,
+        report_format=report_format,  # type: ignore[arg-type]
+        report_template=report_template_path,
+        baseline_path=baseline_path,
+    )
+
+    effective_report_format = report_format or evaluation_config.report.output
+    template_display = (
+        str(report_template_path)
+        if report_template_path is not None
+        else evaluation_config.report.template_path or "default"
+    )
+    baseline_display = (
+        str(baseline_path)
+        if baseline_path is not None
+        else evaluation_config.additional_analysis.comparison.baseline_path
+    )
+
+    message_lines = [
+        f"📊 Evaluating experiment at [cyan]{resolved_experiment_dir}[/cyan]",
+        f"⚙️  Config: [magenta]{config_path}[/magenta]",
+        f"📁 Output: [green]{output_dir}[/green]",
+        f"📝 Report: [yellow]{effective_report_format.upper()}[/yellow] (template: [cyan]{template_display}[/cyan])",
+    ]
+    if baseline_display:
+        message_lines.append(f"📈 Baseline: [cyan]{baseline_display}[/cyan]")
+
+    console.print("\n".join(message_lines))
+
+    summary = run_evaluation(resolved_experiment_dir, evaluation_config, options)
+
+    if verbose:
+        console.print("\n[bold]Summary[/bold]")
+        for key, value in summary.items():
+            console.print(f"• {key}: {value}")
+
+

fluxloop_cli-0.2.18/fluxloop_cli/evaluation/__init__.py

@@ -0,0 +1,37 @@
+"""
+Evaluation framework for FluxLoop experiments.
+"""
+
+from .config import (
+    AdditionalAnalysisConfig,
+    AdvancedConfig,
+    AggregateConfig,
+    EvaluationConfig,
+    EvaluationGoalConfig,
+    EvaluatorConfig,
+    LimitsConfig,
+    ReportConfig,
+    ReportOutput,
+    RuleDefinition,
+    SuccessCriteriaConfig,
+    load_evaluation_config,
+)
+from .engine import EvaluationOptions, run_evaluation
+
+__all__ = [
+    "AdditionalAnalysisConfig",
+    "AdvancedConfig",
+    "AggregateConfig",
+    "EvaluationConfig",
+    "EvaluationGoalConfig",
+    "EvaluatorConfig",
+    "LimitsConfig",
+    "ReportConfig",
+    "ReportOutput",
+    "RuleDefinition",
+    "SuccessCriteriaConfig",
+    "EvaluationOptions",
+    "load_evaluation_config",
+    "run_evaluation",
+]
+