fluxloop-cli 0.2.29__tar.gz → 0.2.31__tar.gz

{fluxloop_cli-0.2.29 → fluxloop_cli-0.2.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fluxloop-cli
-Version: 0.2.29
+Version: 0.2.31
 Summary: FluxLoop CLI for running agent simulations
 Author-email: FluxLoop Team <team@fluxloop.dev>
 License: Apache-2.0
@@ -27,6 +27,7 @@ Requires-Dist: rich>=13.0
 Requires-Dist: python-dotenv>=1.0.0
 Requires-Dist: fluxloop>=0.1.0
 Requires-Dist: ruamel.yaml>=0.17.0
+Requires-Dist: Jinja2>=3.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
@@ -67,10 +68,11 @@ The legacy `setting.yaml` is still supported, but new projects created with
 - `fluxloop generate inputs` – produce input variations for the active project
 - `fluxloop run experiment` – execute an experiment using `configs/simulation.yaml`
 - `fluxloop parse experiment` – convert experiment outputs into readable artifacts and emit structured per-trace JSON at `per_trace_analysis/per_trace.jsonl`
-- `fluxloop evaluate experiment` – score experiment outputs using rule-based and LLM evaluators (requires `fluxloop parse` output or `--per-trace`), generate reports with success criteria, analysis, and customizable templates
+- `fluxloop evaluate experiment` – run the LLM-driven evaluation pipeline (LLM-PT → rule aggregation → LLM-OV → HTML render). Requires the parsed per-trace file (or `--per-trace`) and writes an interactive report to `evaluation_report/report.html` by default.
 - `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
 - `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
 - `fluxloop doctor` – summarize Python, FluxLoop CLI/MCP, and MCP index state for the active environment
+- `--yes/-y` (for `fluxloop run experiment`) – skip the interactive confirmation prompt, ideal for CI and the Pytest bridge
 
 ### Multi-turn supervisor options
 
@@ -88,13 +90,23 @@ These flags override the values in `configs/simulation.yaml` (`multi_turn` block
 
 Run `fluxloop --help` or `fluxloop <command> --help` for more detail.
 
+## Pytest Bridge (0.2.29+)
+
+- `fluxloop init pytest-template [project_root]` creates `tests/test_fluxloop_smoke.py`, already wired to the new `fluxloop_runner` fixture.
+- Fixtures live in `fluxloop_cli.testing.pytest_plugin` and return a `FluxLoopTestResult`, so you can assert on `total_runs`, `success_rate`, or call `require_success()`.
+- Full guide + CI example: see `docs/guides/pytest_bridge.md` (includes GitHub Actions workflow at `examples/ci/fluxloop_pytest.yml`).
+- Typical workflow:
+  1. `pip install -e packages/cli[dev]`
+  2. `fluxloop init pytest-template .`
+  3. `pytest -k fluxloop_smoke --maxfail=1`
+
 ## Evaluation Workflow
 
 Evaluation now follows a two-step process so that multi-turn context is preserved:
 
 1. `fluxloop run experiment` – produce `trace_summary.jsonl` (and optionally `observations.jsonl`).
 2. `fluxloop parse experiment <experiment_dir>` – generate markdown summaries and a structured artifact at `per_trace_analysis/per_trace.jsonl`.
-3. `fluxloop evaluate experiment <experiment_dir>` – consume the structured file to calculate scores and reports.
+3. `fluxloop evaluate experiment <experiment_dir>` – consume that structured file, run LLM-based per-trace + overall analysis, and emit an interactive dashboard at `<experiment_dir>/evaluation_report/report.html` (override with `--output`).
 
 `fluxloop evaluate` exits early with guidance when the per-trace artifact is missing. If you relocate the file, supply an explicit path with `--per-trace /path/to/per_trace.jsonl`.
 

{fluxloop_cli-0.2.29 → fluxloop_cli-0.2.31}/README.md

@@ -26,10 +26,11 @@ The legacy `setting.yaml` is still supported, but new projects created with
 - `fluxloop generate inputs` – produce input variations for the active project
 - `fluxloop run experiment` – execute an experiment using `configs/simulation.yaml`
 - `fluxloop parse experiment` – convert experiment outputs into readable artifacts and emit structured per-trace JSON at `per_trace_analysis/per_trace.jsonl`
-- `fluxloop evaluate experiment` – score experiment outputs using rule-based and LLM evaluators (requires `fluxloop parse` output or `--per-trace`), generate reports with success criteria, analysis, and customizable templates
+- `fluxloop evaluate experiment` – run the LLM-driven evaluation pipeline (LLM-PT → rule aggregation → LLM-OV → HTML render). Requires the parsed per-trace file (or `--per-trace`) and writes an interactive report to `evaluation_report/report.html` by default.
 - `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
 - `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
 - `fluxloop doctor` – summarize Python, FluxLoop CLI/MCP, and MCP index state for the active environment
+- `--yes/-y` (for `fluxloop run experiment`) – skip the interactive confirmation prompt, ideal for CI and the Pytest bridge
 
 ### Multi-turn supervisor options
 
@@ -47,13 +48,23 @@ These flags override the values in `configs/simulation.yaml` (`multi_turn` block
 
 Run `fluxloop --help` or `fluxloop <command> --help` for more detail.
 
+## Pytest Bridge (0.2.29+)
+
+- `fluxloop init pytest-template [project_root]` creates `tests/test_fluxloop_smoke.py`, already wired to the new `fluxloop_runner` fixture.
+- Fixtures live in `fluxloop_cli.testing.pytest_plugin` and return a `FluxLoopTestResult`, so you can assert on `total_runs`, `success_rate`, or call `require_success()`.
+- Full guide + CI example: see `docs/guides/pytest_bridge.md` (includes GitHub Actions workflow at `examples/ci/fluxloop_pytest.yml`).
+- Typical workflow:
+  1. `pip install -e packages/cli[dev]`
+  2. `fluxloop init pytest-template .`
+  3. `pytest -k fluxloop_smoke --maxfail=1`
+
 ## Evaluation Workflow
 
 Evaluation now follows a two-step process so that multi-turn context is preserved:
 
 1. `fluxloop run experiment` – produce `trace_summary.jsonl` (and optionally `observations.jsonl`).
 2. `fluxloop parse experiment <experiment_dir>` – generate markdown summaries and a structured artifact at `per_trace_analysis/per_trace.jsonl`.
-3. `fluxloop evaluate experiment <experiment_dir>` – consume the structured file to calculate scores and reports.
+3. `fluxloop evaluate experiment <experiment_dir>` – consume that structured file, run LLM-based per-trace + overall analysis, and emit an interactive dashboard at `<experiment_dir>/evaluation_report/report.html` (override with `--output`).
 
 `fluxloop evaluate` exits early with guidance when the per-trace artifact is missing. If you relocate the file, supply an explicit path with `--per-trace /path/to/per_trace.jsonl`.
 

{fluxloop_cli-0.2.29 → fluxloop_cli-0.2.31}/fluxloop_cli/__init__.py

@@ -2,7 +2,7 @@
 FluxLoop CLI - Command-line interface for running agent simulations.
 """
 
-__version__ = "0.2.29"
+__version__ = "0.2.31"
 
 from .main import app
 

{fluxloop_cli-0.2.29 → fluxloop_cli-0.2.31}/fluxloop_cli/arg_binder.py

@@ -7,6 +7,8 @@ import json
 from pathlib import Path
 from typing import Any, Callable, Dict, Optional, Sequence
 
+from fluxloop.schemas import ExperimentConfig, ReplayArgsConfig, PersonaConfig
+
 
 class _AttrDict(dict):
     """Dictionary that also supports attribute access for keys."""
@@ -26,8 +28,6 @@ class _AttrDict(dict):
         except KeyError as exc:  # pragma: no cover
             raise AttributeError(item) from exc
 
-from fluxloop.schemas import ExperimentConfig, ReplayArgsConfig, PersonaConfig
-
 
 class _AwaitableNone:
     """Simple awaitable that resolves to ``None``."""
@@ -278,7 +278,6 @@ class ArgBinder:
 
             def _record(args: Any, kwargs: Any) -> None:
                 messages.append((args, kwargs))
-                pretty = args[0] if len(args) == 1 and not kwargs else {"args": args, "kwargs": kwargs}
 
             def send(*args: Any, **kwargs: Any) -> _AwaitableNone:
                 _record(args, kwargs)

{fluxloop_cli-0.2.29 → fluxloop_cli-0.2.31}/fluxloop_cli/commands/config.py

@@ -14,7 +14,6 @@ from rich.syntax import Syntax
 from rich.table import Table
 
 from ..config_loader import load_experiment_config
-from ..templates import create_env_file, create_gitignore, create_sample_agent
 from ..constants import DEFAULT_CONFIG_PATH, DEFAULT_ROOT_DIR_NAME
 from ..config_schema import CONFIG_SECTION_FILENAMES
 from ..project_paths import (

fluxloop_cli-0.2.31/fluxloop_cli/commands/evaluate.py

@@ -0,0 +1,188 @@
+"""Evaluate command for generating interactive reports."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import shutil
+from dataclasses import asdict
+from pathlib import Path
+from typing import Optional
+
+import typer
+import yaml
+from rich.console import Console
+
+from ..environment import load_env_chain
+from ..evaluation import load_evaluation_config
+from ..evaluation.artifacts import load_per_trace_records
+from ..evaluation.report.pipeline import ReportPipeline
+
+console = Console()
+app = typer.Typer(help="Evaluate experiment outputs and generate interactive reports.")
+
+
+def _load_yaml_file(path: Optional[Path]) -> dict:
+    if not path or not path.exists():
+        return {}
+    with path.open("r", encoding="utf-8") as handle:
+        data = yaml.safe_load(handle) or {}
+    if isinstance(data, dict):
+        return data
+    return {}
+
+
+def _resolve_project_root(config_path: Path) -> Path:
+    config_dir = config_path.parent
+    return config_dir.parent if config_dir.name == "configs" else config_dir
+
+
+def _find_config_file(config_path: Path, filename: str) -> Optional[Path]:
+    config_dir = config_path.parent
+    project_root = _resolve_project_root(config_path)
+    candidates = [
+        config_dir / filename,
+        project_root / "configs" / filename,
+        project_root / filename,
+    ]
+    for candidate in candidates:
+        if candidate.exists():
+            return candidate
+    return None
+
+
+def _prepare_output_directory(path: Path, overwrite: bool) -> None:
+    if path.exists():
+        if not overwrite:
+            raise typer.BadParameter(
+                f"Output directory already exists: {path}. Use --overwrite to replace it."
+            )
+        shutil.rmtree(path)
+    path.mkdir(parents=True, exist_ok=True)
+
+
+@app.command()
+def experiment(
+    experiment_dir: Path = typer.Argument(
+        ...,
+        help="Path to the experiment output directory",
+        exists=True,
+        dir_okay=True,
+        file_okay=False,
+        resolve_path=True,
+    ),
+    config: Path = typer.Option(
+        Path("configs/evaluation.yaml"),
+        "--config",
+        "-c",
+        help="Path to evaluation configuration file",
+    ),
+    output: Path = typer.Option(
+        Path("evaluation_report"),
+        "--output",
+        "-o",
+        help="Output directory name (relative to the experiment directory)",
+    ),
+    overwrite: bool = typer.Option(
+        False,
+        "--overwrite",
+        help="Overwrite output directory if it already exists",
+    ),
+    llm_api_key: Optional[str] = typer.Option(
+        None,
+        "--llm-api-key",
+        help="LLM API key for report generation (optional)",
+        envvar="FLUXLOOP_LLM_API_KEY",
+    ),
+    per_trace: Optional[Path] = typer.Option(
+        None,
+        "--per-trace",
+        help="Path to structured per-trace JSONL generated by `fluxloop parse`",
+    ),
+    verbose: bool = typer.Option(
+        False,
+        "--verbose",
+        help="Enable verbose logging",
+    ),
+):
+    """
+    Evaluate experiment outputs and generate an interactive HTML report.
+    """
+
+    logging.basicConfig(
+        level=logging.DEBUG if verbose else logging.INFO,
+        format="%(message)s",
+    )
+
+    resolved_experiment_dir = experiment_dir.resolve()
+    if not resolved_experiment_dir.is_dir():
+        raise typer.BadParameter(f"Experiment directory not found: {resolved_experiment_dir}")
+
+    config_path = config.resolve() if config.is_absolute() else (Path.cwd() / config).resolve()
+
+    if per_trace is not None:
+        per_trace_path = per_trace.resolve() if per_trace.is_absolute() else (Path.cwd() / per_trace).resolve()
+    else:
+        per_trace_path = resolved_experiment_dir / "per_trace_analysis" / "per_trace.jsonl"
+
+    per_trace_records = load_per_trace_records(resolved_experiment_dir, per_trace_path)
+    trace_summaries = [record.trace for record in per_trace_records]
+    if not trace_summaries:
+        raise typer.BadParameter("No traces found in per-trace artifacts.")
+
+    try:
+        evaluation_config = load_evaluation_config(config_path)
+    except FileNotFoundError as exc:
+        raise typer.BadParameter(str(exc)) from exc
+    except Exception as exc:  # noqa: BLE001
+        raise typer.BadParameter(f"Failed to load evaluation config: {exc}") from exc
+
+    def _log_env_error(path: Path, exc: Exception) -> None:
+        console.log(
+            f"[yellow]Warning:[/yellow] Failed to load environment from {path}: {exc}"
+        )
+
+    load_env_chain(
+        evaluation_config.get_source_dir(),
+        refresh_config=True,
+        on_error=_log_env_error,
+    )
+
+    if llm_api_key is None:
+        llm_api_key = os.getenv("FLUXLOOP_LLM_API_KEY") or os.getenv("OPENAI_API_KEY")
+
+    output_dir = output if output.is_absolute() else (resolved_experiment_dir / output)
+    _prepare_output_directory(output_dir, overwrite)
+
+    input_config_path = _find_config_file(config_path, "input.yaml")
+    project_config_path = _find_config_file(config_path, "project.yaml")
+
+    input_config = _load_yaml_file(input_config_path)
+    project_config = _load_yaml_file(project_config_path)
+
+    config_bundle = {
+        "name": project_config.get("name") or resolved_experiment_dir.name,
+        "evaluation": asdict(evaluation_config),
+        "input": input_config,
+    }
+
+    pipeline = ReportPipeline(
+        config=config_bundle,
+        output_dir=output_dir,
+        api_key=llm_api_key,
+    )
+
+    message_lines = [
+        f"📊 Evaluating experiment at [cyan]{resolved_experiment_dir}[/cyan]",
+        f"⚙️  Config: [magenta]{config_path}[/magenta]",
+        f"🧵 Per-trace data: [blue]{per_trace_path}[/blue]",
+        f"📁 Output: [green]{output_dir}[/green]",
+    ]
+    console.print("\n".join(message_lines))
+
+    report_path = asyncio.run(pipeline.run(trace_summaries))
+    console.print(f"\n✅ Report ready: [bold cyan]{report_path}[/bold cyan]")
+
+
+

{fluxloop_cli-0.2.29 → fluxloop_cli-0.2.31}/fluxloop_cli/commands/init.py

@@ -18,6 +18,7 @@ from ..templates import (
     create_sample_agent,
     create_gitignore,
     create_env_file,
+    create_pytest_bridge_template,
 )
 from ..project_paths import resolve_root_dir, resolve_project_dir
 from ..constants import (
@@ -187,10 +188,77 @@ def project(
     console.print("4. Generate inputs: [green]fluxloop generate inputs[/green]")
     console.print("5. Run the experiment: [green]fluxloop run experiment[/green]")
     console.print("6. Parse outputs: [green]fluxloop parse experiment[/green]")
-    console.print("7. Evaluate results (optional): [green]fluxloop evaluate experiment[/green]")
+    console.print("7. Generate the interactive report (optional): [green]fluxloop evaluate experiment[/green]")
     console.print("8. Diagnose environment anytime: [green]fluxloop doctor[/green]")
 
 
+@app.command("pytest-template")
+def pytest_template(
+    project_root: Path = typer.Argument(
+        Path.cwd(),
+        help="Project root containing configs/ or setting.yaml",
+    ),
+    tests_dir: str = typer.Option(
+        "tests",
+        "--tests-dir",
+        help="Directory (relative to project root) where tests live",
+    ),
+    filename: str = typer.Option(
+        "test_fluxloop_smoke.py",
+        "--filename",
+        help="Test file name to create inside the tests directory",
+    ),
+    force: bool = typer.Option(
+        False,
+        "--force",
+        help="Overwrite existing template without confirmation",
+    ),
+) -> None:
+    """
+    Scaffold a pytest smoke test that uses the FluxLoop runner fixtures.
+    """
+
+    root_path = project_root.expanduser().resolve()
+    if not root_path.exists():
+        console.print(f"[red]Error:[/red] Project root {root_path} does not exist.")
+        raise typer.Exit(1)
+
+    tests_path = (root_path / tests_dir).resolve()
+    tests_path.mkdir(parents=True, exist_ok=True)
+
+    target_file = tests_path / filename
+
+    if target_file.exists() and not force:
+        if not Confirm.ask(
+            f"{target_file} already exists. Overwrite?",
+            default=False,
+        ):
+            raise typer.Abort()
+
+    configs_sim = root_path / CONFIG_DIRECTORY_NAME / CONFIG_SECTION_FILENAMES["simulation"]
+    legacy_config = root_path / "setting.yaml"
+
+    if configs_sim.exists():
+        relative_config = configs_sim.relative_to(root_path).as_posix()
+    elif legacy_config.exists():
+        relative_config = legacy_config.relative_to(root_path).as_posix()
+    else:
+        # Fall back to configs/simulation.yaml even if it does not exist yet
+        relative_config = (CONFIG_DIRECTORY_NAME + "/" + CONFIG_SECTION_FILENAMES["simulation"])
+        console.print(
+            "[yellow]Warning:[/yellow] Could not find configs/simulation.yaml or setting.yaml. "
+            "Template will reference the default simulation path."
+        )
+
+    template_content = create_pytest_bridge_template(relative_config)
+    target_file.write_text(template_content, encoding="utf-8")
+
+    console.print(
+        f"[green]✓[/green] Pytest template created at [cyan]{target_file}[/cyan]. "
+        "Run [bold]pytest -k fluxloop_smoke[/bold] to execute the sample test."
+    )
+
+
 @app.command()
 def agent(
     name: str = typer.Argument(

{fluxloop_cli-0.2.29 → fluxloop_cli-0.2.31}/fluxloop_cli/commands/record.py

@@ -16,7 +16,6 @@ from ..project_paths import (
     resolve_project_dir,
     resolve_config_section_path,
 )
-from ..config_schema import CONFIG_SECTION_FILENAMES
 
 
 app = typer.Typer()

{fluxloop_cli-0.2.29 → fluxloop_cli-0.2.31}/fluxloop_cli/commands/run.py

@@ -9,7 +9,6 @@ from typing import Callable, Optional
 
 import typer
 from rich.console import Console
-from rich.live import Live
 from rich.panel import Panel
 from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TimeRemainingColumn
 from rich.table import Table
@@ -118,6 +117,12 @@ def experiment(
         "--display/--no-display",
         help="Show rich console output (disable for plain log streaming)",
     ),
+    yes: bool = typer.Option(
+        False,
+        "--yes",
+        "-y",
+        help="Skip confirmation prompt and run immediately",
+    ),
 ):
     """
     Run an experiment based on configuration file.
@@ -248,7 +253,11 @@ def experiment(
             f"\nThis will execute {total_runs} runs."
         )
 
-    if not typer.confirm("Continue?"):
+    if yes:
+        proceed = True
+    else:
+        proceed = typer.confirm("Continue?")
+    if not proceed:
         raise typer.Abort()
     
     # Create runner

{fluxloop_cli-0.2.29 → fluxloop_cli-0.2.31}/fluxloop_cli/commands/status.py

@@ -15,7 +15,6 @@ from ..config_schema import CONFIG_SECTION_FILENAMES, CONFIG_REQUIRED_KEYS
 from ..project_paths import (
     resolve_config_path,
     resolve_config_directory,
-    resolve_root_dir,
     resolve_project_relative,
 )
 
@@ -79,7 +78,7 @@ def check(
         from fluxloop.client import FluxLoopClient
         
         config = get_config()
-        client = FluxLoopClient()
+        _client = FluxLoopClient()  # noqa: F841 - instantiated for potential future use
         
         # Try to connect (this would need a health endpoint)
         status_table.add_row(

{fluxloop_cli-0.2.29 → fluxloop_cli-0.2.31}/fluxloop_cli/config_loader.py

@@ -12,7 +12,6 @@ from pydantic import ValidationError
 from .project_paths import resolve_config_path
 from .config_schema import (
     CONFIG_SECTION_FILENAMES,
-    CONFIG_SECTION_ORDER,
     CONFIG_REQUIRED_KEYS,
     iter_section_paths,
     is_legacy_config,
@@ -52,7 +51,6 @@ def load_experiment_config(
 
         for section_path in iter_section_paths(project_root):
             if not section_path.exists():
-                key = section_path.name
                 logical_key = _section_key_from_filename(section_path.name)
                 if logical_key in CONFIG_REQUIRED_KEYS:
                     missing_required.append(section_path.name)

{fluxloop_cli-0.2.29 → fluxloop_cli-0.2.31}/fluxloop_cli/evaluation/artifacts.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 import json
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Dict, Iterable, List, Optional
+from typing import Any, Dict, List, Optional
 
 
 @dataclass