fluxloop-cli 0.2.30__tar.gz → 0.2.32__tar.gz

{fluxloop_cli-0.2.30 → fluxloop_cli-0.2.32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fluxloop-cli
-Version: 0.2.30
+Version: 0.2.32
 Summary: FluxLoop CLI for running agent simulations
 Author-email: FluxLoop Team <team@fluxloop.dev>
 License: Apache-2.0
@@ -27,6 +27,8 @@ Requires-Dist: rich>=13.0
 Requires-Dist: python-dotenv>=1.0.0
 Requires-Dist: fluxloop>=0.1.0
 Requires-Dist: ruamel.yaml>=0.17.0
+Requires-Dist: Jinja2>=3.0
+Requires-Dist: weasyprint>=61.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
@@ -67,7 +69,7 @@ The legacy `setting.yaml` is still supported, but new projects created with
 - `fluxloop generate inputs` – produce input variations for the active project
 - `fluxloop run experiment` – execute an experiment using `configs/simulation.yaml`
 - `fluxloop parse experiment` – convert experiment outputs into readable artifacts and emit structured per-trace JSON at `per_trace_analysis/per_trace.jsonl`
-- `fluxloop evaluate experiment` – score experiment outputs using rule-based and LLM evaluators (requires `fluxloop parse` output or `--per-trace`), generate reports with success criteria, analysis, and customizable templates
+- `fluxloop evaluate experiment` – run the LLM-driven evaluation pipeline (LLM-PT → rule aggregation → LLM-OV → HTML render). Requires the parsed per-trace file (or `--per-trace`) and writes an interactive report to `evaluation_report/report.html` by default.
 - `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
 - `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
 - `fluxloop doctor` – summarize Python, FluxLoop CLI/MCP, and MCP index state for the active environment
@@ -105,7 +107,7 @@ Evaluation now follows a two-step process so that multi-turn context is preserve
 
 1. `fluxloop run experiment` – produce `trace_summary.jsonl` (and optionally `observations.jsonl`).
 2. `fluxloop parse experiment <experiment_dir>` – generate markdown summaries and a structured artifact at `per_trace_analysis/per_trace.jsonl`.
-3. `fluxloop evaluate experiment <experiment_dir>` – consume the structured file to calculate scores and reports.
+3. `fluxloop evaluate experiment <experiment_dir>` – consume that structured file, run LLM-based per-trace + overall analysis, and emit an interactive dashboard at `<experiment_dir>/evaluation_report/report.html` (override with `--output`).
 
 `fluxloop evaluate` exits early with guidance when the per-trace artifact is missing. If you relocate the file, supply an explicit path with `--per-trace /path/to/per_trace.jsonl`.
 

{fluxloop_cli-0.2.30 → fluxloop_cli-0.2.32}/README.md

@@ -26,7 +26,7 @@ The legacy `setting.yaml` is still supported, but new projects created with
 - `fluxloop generate inputs` – produce input variations for the active project
 - `fluxloop run experiment` – execute an experiment using `configs/simulation.yaml`
 - `fluxloop parse experiment` – convert experiment outputs into readable artifacts and emit structured per-trace JSON at `per_trace_analysis/per_trace.jsonl`
-- `fluxloop evaluate experiment` – score experiment outputs using rule-based and LLM evaluators (requires `fluxloop parse` output or `--per-trace`), generate reports with success criteria, analysis, and customizable templates
+- `fluxloop evaluate experiment` – run the LLM-driven evaluation pipeline (LLM-PT → rule aggregation → LLM-OV → HTML render). Requires the parsed per-trace file (or `--per-trace`) and writes an interactive report to `evaluation_report/report.html` by default.
 - `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
 - `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
 - `fluxloop doctor` – summarize Python, FluxLoop CLI/MCP, and MCP index state for the active environment
@@ -64,7 +64,7 @@ Evaluation now follows a two-step process so that multi-turn context is preserve
 
 1. `fluxloop run experiment` – produce `trace_summary.jsonl` (and optionally `observations.jsonl`).
 2. `fluxloop parse experiment <experiment_dir>` – generate markdown summaries and a structured artifact at `per_trace_analysis/per_trace.jsonl`.
-3. `fluxloop evaluate experiment <experiment_dir>` – consume the structured file to calculate scores and reports.
+3. `fluxloop evaluate experiment <experiment_dir>` – consume that structured file, run LLM-based per-trace + overall analysis, and emit an interactive dashboard at `<experiment_dir>/evaluation_report/report.html` (override with `--output`).
 
 `fluxloop evaluate` exits early with guidance when the per-trace artifact is missing. If you relocate the file, supply an explicit path with `--per-trace /path/to/per_trace.jsonl`.
 

{fluxloop_cli-0.2.30 → fluxloop_cli-0.2.32}/fluxloop_cli/__init__.py

@@ -2,7 +2,7 @@
 FluxLoop CLI - Command-line interface for running agent simulations.
 """
 
-__version__ = "0.2.30"
+__version__ = "0.2.32"
 
 from .main import app
 

{fluxloop_cli-0.2.30 → fluxloop_cli-0.2.32}/fluxloop_cli/arg_binder.py

@@ -7,6 +7,8 @@ import json
 from pathlib import Path
 from typing import Any, Callable, Dict, Optional, Sequence
 
+from fluxloop.schemas import ExperimentConfig, ReplayArgsConfig, PersonaConfig
+
 
 class _AttrDict(dict):
     """Dictionary that also supports attribute access for keys."""
@@ -26,8 +28,6 @@ class _AttrDict(dict):
         except KeyError as exc:  # pragma: no cover
             raise AttributeError(item) from exc
 
-from fluxloop.schemas import ExperimentConfig, ReplayArgsConfig, PersonaConfig
-
 
 class _AwaitableNone:
     """Simple awaitable that resolves to ``None``."""
@@ -278,7 +278,6 @@ class ArgBinder:
 
             def _record(args: Any, kwargs: Any) -> None:
                 messages.append((args, kwargs))
-                pretty = args[0] if len(args) == 1 and not kwargs else {"args": args, "kwargs": kwargs}
 
             def send(*args: Any, **kwargs: Any) -> _AwaitableNone:
                 _record(args, kwargs)

{fluxloop_cli-0.2.30 → fluxloop_cli-0.2.32}/fluxloop_cli/commands/config.py

@@ -14,7 +14,6 @@ from rich.syntax import Syntax
 from rich.table import Table
 
 from ..config_loader import load_experiment_config
-from ..templates import create_env_file, create_gitignore, create_sample_agent
 from ..constants import DEFAULT_CONFIG_PATH, DEFAULT_ROOT_DIR_NAME
 from ..config_schema import CONFIG_SECTION_FILENAMES
 from ..project_paths import (

fluxloop_cli-0.2.32/fluxloop_cli/commands/evaluate.py

@@ -0,0 +1,194 @@
+"""Evaluate command for generating interactive reports."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import shutil
+from dataclasses import asdict
+from pathlib import Path
+from typing import Optional
+
+import typer
+import yaml
+from rich.console import Console
+
+from ..environment import load_env_chain
+from ..evaluation import load_evaluation_config
+from ..evaluation.artifacts import load_per_trace_records
+from ..evaluation.report.pipeline import ReportPipeline
+
+console = Console()
+app = typer.Typer(help="Evaluate experiment outputs and generate interactive reports.")
+
+
+def _load_yaml_file(path: Optional[Path]) -> dict:
+    if not path or not path.exists():
+        return {}
+    with path.open("r", encoding="utf-8") as handle:
+        data = yaml.safe_load(handle) or {}
+    if isinstance(data, dict):
+        return data
+    return {}
+
+
+def _resolve_project_root(config_path: Path) -> Path:
+    config_dir = config_path.parent
+    return config_dir.parent if config_dir.name == "configs" else config_dir
+
+
+def _find_config_file(config_path: Path, filename: str) -> Optional[Path]:
+    config_dir = config_path.parent
+    project_root = _resolve_project_root(config_path)
+    candidates = [
+        config_dir / filename,
+        project_root / "configs" / filename,
+        project_root / filename,
+    ]
+    for candidate in candidates:
+        if candidate.exists():
+            return candidate
+    return None
+
+
+def _prepare_output_directory(path: Path, overwrite: bool) -> None:
+    if path.exists():
+        if not overwrite:
+            raise typer.BadParameter(
+                f"Output directory already exists: {path}. Use --overwrite to replace it."
+            )
+        shutil.rmtree(path)
+    path.mkdir(parents=True, exist_ok=True)
+
+
+@app.command()
+def experiment(
+    experiment_dir: Path = typer.Argument(
+        ...,
+        help="Path to the experiment output directory",
+        exists=True,
+        dir_okay=True,
+        file_okay=False,
+        resolve_path=True,
+    ),
+    config: Path = typer.Option(
+        Path("configs/evaluation.yaml"),
+        "--config",
+        "-c",
+        help="Path to evaluation configuration file",
+    ),
+    output: Path = typer.Option(
+        Path("evaluation_report"),
+        "--output",
+        "-o",
+        help="Output directory name (relative to the experiment directory)",
+    ),
+    overwrite: bool = typer.Option(
+        False,
+        "--overwrite",
+        help="Overwrite output directory if it already exists",
+    ),
+    llm_api_key: Optional[str] = typer.Option(
+        None,
+        "--llm-api-key",
+        help="LLM API key for report generation (optional)",
+        envvar="FLUXLOOP_LLM_API_KEY",
+    ),
+    per_trace: Optional[Path] = typer.Option(
+        None,
+        "--per-trace",
+        help="Path to structured per-trace JSONL generated by `fluxloop parse`",
+    ),
+    verbose: bool = typer.Option(
+        False,
+        "--verbose",
+        help="Enable verbose logging",
+    ),
+):
+    """
+    Evaluate experiment outputs and generate an interactive HTML report.
+    """
+
+    logging.basicConfig(
+        level=logging.DEBUG if verbose else logging.INFO,
+        format="%(message)s",
+    )
+
+    resolved_experiment_dir = experiment_dir.resolve()
+    if not resolved_experiment_dir.is_dir():
+        raise typer.BadParameter(f"Experiment directory not found: {resolved_experiment_dir}")
+
+    config_path = config.resolve() if config.is_absolute() else (Path.cwd() / config).resolve()
+
+    if per_trace is not None:
+        per_trace_path = per_trace.resolve() if per_trace.is_absolute() else (Path.cwd() / per_trace).resolve()
+    else:
+        per_trace_path = resolved_experiment_dir / "per_trace_analysis" / "per_trace.jsonl"
+
+    per_trace_records = load_per_trace_records(resolved_experiment_dir, per_trace_path)
+    trace_summaries = [record.trace for record in per_trace_records]
+    if not trace_summaries:
+        raise typer.BadParameter("No traces found in per-trace artifacts.")
+
+    try:
+        evaluation_config = load_evaluation_config(config_path)
+    except FileNotFoundError as exc:
+        raise typer.BadParameter(str(exc)) from exc
+    except Exception as exc:  # noqa: BLE001
+        raise typer.BadParameter(f"Failed to load evaluation config: {exc}") from exc
+
+    def _log_env_error(path: Path, exc: Exception) -> None:
+        console.log(
+            f"[yellow]Warning:[/yellow] Failed to load environment from {path}: {exc}"
+        )
+
+    load_env_chain(
+        evaluation_config.get_source_dir(),
+        refresh_config=True,
+        on_error=_log_env_error,
+    )
+
+    if llm_api_key is None:
+        llm_api_key = os.getenv("FLUXLOOP_LLM_API_KEY") or os.getenv("OPENAI_API_KEY")
+
+    output_dir = output if output.is_absolute() else (resolved_experiment_dir / output)
+    _prepare_output_directory(output_dir, overwrite)
+
+    input_config_path = _find_config_file(config_path, "input.yaml")
+    project_config_path = _find_config_file(config_path, "project.yaml")
+
+    input_config = _load_yaml_file(input_config_path)
+    project_config = _load_yaml_file(project_config_path)
+
+    config_bundle = {
+        "name": project_config.get("name") or resolved_experiment_dir.name,
+        "evaluation": asdict(evaluation_config),
+        "input": input_config,
+    }
+
+    pipeline = ReportPipeline(
+        config=config_bundle,
+        output_dir=output_dir,
+        api_key=llm_api_key,
+    )
+
+    message_lines = [
+        f"📊 Evaluating experiment at [cyan]{resolved_experiment_dir}[/cyan]",
+        f"⚙️  Config: [magenta]{config_path}[/magenta]",
+        f"🧵 Per-trace data: [blue]{per_trace_path}[/blue]",
+        f"📁 Output: [green]{output_dir}[/green]",
+    ]
+    console.print("\n".join(message_lines))
+
+    artifacts = asyncio.run(pipeline.run(trace_summaries))
+    console.print(f"\n✅ Report ready: [bold cyan]{artifacts.html_path}[/bold cyan]")
+    if artifacts.pdf_path:
+        console.print(f"🖨️ PDF ready: [bold green]{artifacts.pdf_path}[/bold green]")
+    else:
+        console.print(
+            "⚠️ PDF export skipped (WeasyPrint is unavailable or PDF rendering failed)."
+        )
+
+
+

{fluxloop_cli-0.2.30 → fluxloop_cli-0.2.32}/fluxloop_cli/commands/init.py

@@ -188,7 +188,7 @@ def project(
     console.print("4. Generate inputs: [green]fluxloop generate inputs[/green]")
     console.print("5. Run the experiment: [green]fluxloop run experiment[/green]")
     console.print("6. Parse outputs: [green]fluxloop parse experiment[/green]")
-    console.print("7. Evaluate results (optional): [green]fluxloop evaluate experiment[/green]")
+    console.print("7. Generate the interactive report (optional): [green]fluxloop evaluate experiment[/green]")
     console.print("8. Diagnose environment anytime: [green]fluxloop doctor[/green]")
 
 

{fluxloop_cli-0.2.30 → fluxloop_cli-0.2.32}/fluxloop_cli/commands/record.py

@@ -16,7 +16,6 @@ from ..project_paths import (
     resolve_project_dir,
     resolve_config_section_path,
 )
-from ..config_schema import CONFIG_SECTION_FILENAMES
 
 
 app = typer.Typer()

{fluxloop_cli-0.2.30 → fluxloop_cli-0.2.32}/fluxloop_cli/commands/run.py

@@ -9,7 +9,6 @@ from typing import Callable, Optional
 
 import typer
 from rich.console import Console
-from rich.live import Live
 from rich.panel import Panel
 from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TimeRemainingColumn
 from rich.table import Table

{fluxloop_cli-0.2.30 → fluxloop_cli-0.2.32}/fluxloop_cli/commands/status.py

@@ -15,7 +15,6 @@ from ..config_schema import CONFIG_SECTION_FILENAMES, CONFIG_REQUIRED_KEYS
 from ..project_paths import (
     resolve_config_path,
     resolve_config_directory,
-    resolve_root_dir,
     resolve_project_relative,
 )
 
@@ -79,7 +78,7 @@ def check(
         from fluxloop.client import FluxLoopClient
         
         config = get_config()
-        client = FluxLoopClient()
+        _client = FluxLoopClient()  # noqa: F841 - instantiated for potential future use
         
         # Try to connect (this would need a health endpoint)
         status_table.add_row(

{fluxloop_cli-0.2.30 → fluxloop_cli-0.2.32}/fluxloop_cli/config_loader.py

@@ -12,7 +12,6 @@ from pydantic import ValidationError
 from .project_paths import resolve_config_path
 from .config_schema import (
     CONFIG_SECTION_FILENAMES,
-    CONFIG_SECTION_ORDER,
     CONFIG_REQUIRED_KEYS,
     iter_section_paths,
     is_legacy_config,
@@ -52,7 +51,6 @@ def load_experiment_config(
 
         for section_path in iter_section_paths(project_root):
             if not section_path.exists():
-                key = section_path.name
                 logical_key = _section_key_from_filename(section_path.name)
                 if logical_key in CONFIG_REQUIRED_KEYS:
                     missing_required.append(section_path.name)

{fluxloop_cli-0.2.30 → fluxloop_cli-0.2.32}/fluxloop_cli/evaluation/artifacts.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 import json
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Dict, Iterable, List, Optional
+from typing import Any, Dict, List, Optional
 
 
 @dataclass

{fluxloop_cli-0.2.30 → fluxloop_cli-0.2.32}/fluxloop_cli/evaluation/config.py

@@ -229,6 +229,8 @@ class EvaluationConfig:
     additional_analysis: AdditionalAnalysisConfig = field(default_factory=AdditionalAnalysisConfig)
     report: ReportConfig = field(default_factory=ReportConfig)
     advanced: AdvancedConfig = field(default_factory=AdvancedConfig)
+    metrics: Dict[str, Any] = field(default_factory=dict)
+    efficiency: Dict[str, Any] = field(default_factory=dict)
 
     def set_source_dir(self, source_dir: Path) -> None:
         """Remember the directory where this config file was loaded from."""
@@ -383,12 +385,45 @@ def _parse_limits(raw: Any) -> LimitsConfig:
     )
 
 
+def _parse_metrics(raw: Any) -> Dict[str, Any]:
+    if raw is None:
+        return {}
+    if not isinstance(raw, dict):
+        raise ValueError("metrics must be a mapping")
+    normalized: Dict[str, Any] = {}
+    for key, value in raw.items():
+        if not isinstance(value, dict):
+            raise ValueError(f"metrics.{key} must be a mapping")
+        normalized[key] = dict(value)
+    return normalized
+
+
+def _parse_efficiency(raw: Any) -> Dict[str, Any]:
+    if raw is None:
+        return {}
+    if not isinstance(raw, dict):
+        raise ValueError("efficiency must be a mapping")
+    normalized: Dict[str, Any] = {}
+    for key, value in raw.items():
+        if not isinstance(value, dict):
+            raise ValueError(f"efficiency.{key} must be a mapping")
+        normalized[key] = dict(value)
+    return normalized
+
+
 def _parse_evaluation_goal(raw: Any) -> EvaluationGoalConfig:
     if raw is None:
         return EvaluationGoalConfig()
+
+    if isinstance(raw, str):
+        # Backwards compatibility for older templates that used a bare string.
+        return EvaluationGoalConfig(text=raw.strip())
+
     if not isinstance(raw, dict):
-        raise ValueError("evaluation_goal must be a mapping")
-    text = str(raw.get("text", "")) if raw.get("text") is not None else ""
+        raise ValueError("evaluation_goal must be a mapping or string")
+
+    text_value = raw.get("text")
+    text = str(text_value) if text_value is not None else ""
     return EvaluationGoalConfig(text=text)
 
 
@@ -711,6 +746,8 @@ def load_evaluation_config(path: Path) -> EvaluationConfig:
     additional_analysis = _parse_additional_analysis(data.get("additional_analysis"))
     report = _parse_report(data.get("report"))
     advanced = _parse_advanced(data.get("advanced"))
+    metrics = _parse_metrics(data.get("metrics"))
+    efficiency = _parse_efficiency(data.get("efficiency"))
 
     config_dir = path.parent.resolve()
 
@@ -723,6 +760,8 @@ def load_evaluation_config(path: Path) -> EvaluationConfig:
         additional_analysis=additional_analysis,
         report=report,
         advanced=advanced,
+        metrics=metrics,
+        efficiency=efficiency,
     )
 
     config.set_source_dir(config_dir)