fluxloop-cli 0.2.28__py3-none-any.whl → 0.2.30__py3-none-any.whl

fluxloop_cli/__init__.py

@@ -2,7 +2,7 @@
 FluxLoop CLI - Command-line interface for running agent simulations.
 """
 
-__version__ = "0.2.28"
+__version__ = "0.2.30"
 
 from .main import app
 

fluxloop_cli/commands/init.py

@@ -18,6 +18,7 @@ from ..templates import (
     create_sample_agent,
     create_gitignore,
     create_env_file,
+    create_pytest_bridge_template,
 )
 from ..project_paths import resolve_root_dir, resolve_project_dir
 from ..constants import (
@@ -191,6 +192,73 @@ def project(
     console.print("8. Diagnose environment anytime: [green]fluxloop doctor[/green]")
 
 
+@app.command("pytest-template")
+def pytest_template(
+    project_root: Path = typer.Argument(
+        Path.cwd(),
+        help="Project root containing configs/ or setting.yaml",
+    ),
+    tests_dir: str = typer.Option(
+        "tests",
+        "--tests-dir",
+        help="Directory (relative to project root) where tests live",
+    ),
+    filename: str = typer.Option(
+        "test_fluxloop_smoke.py",
+        "--filename",
+        help="Test file name to create inside the tests directory",
+    ),
+    force: bool = typer.Option(
+        False,
+        "--force",
+        help="Overwrite existing template without confirmation",
+    ),
+) -> None:
+    """
+    Scaffold a pytest smoke test that uses the FluxLoop runner fixtures.
+    """
+
+    root_path = project_root.expanduser().resolve()
+    if not root_path.exists():
+        console.print(f"[red]Error:[/red] Project root {root_path} does not exist.")
+        raise typer.Exit(1)
+
+    tests_path = (root_path / tests_dir).resolve()
+    tests_path.mkdir(parents=True, exist_ok=True)
+
+    target_file = tests_path / filename
+
+    if target_file.exists() and not force:
+        if not Confirm.ask(
+            f"{target_file} already exists. Overwrite?",
+            default=False,
+        ):
+            raise typer.Abort()
+
+    configs_sim = root_path / CONFIG_DIRECTORY_NAME / CONFIG_SECTION_FILENAMES["simulation"]
+    legacy_config = root_path / "setting.yaml"
+
+    if configs_sim.exists():
+        relative_config = configs_sim.relative_to(root_path).as_posix()
+    elif legacy_config.exists():
+        relative_config = legacy_config.relative_to(root_path).as_posix()
+    else:
+        # Fall back to configs/simulation.yaml even if it does not exist yet
+        relative_config = (CONFIG_DIRECTORY_NAME + "/" + CONFIG_SECTION_FILENAMES["simulation"])
+        console.print(
+            "[yellow]Warning:[/yellow] Could not find configs/simulation.yaml or setting.yaml. "
+            "Template will reference the default simulation path."
+        )
+
+    template_content = create_pytest_bridge_template(relative_config)
+    target_file.write_text(template_content, encoding="utf-8")
+
+    console.print(
+        f"[green]✓[/green] Pytest template created at [cyan]{target_file}[/cyan]. "
+        "Run [bold]pytest -k fluxloop_smoke[/bold] to execute the sample test."
+    )
+
+
 @app.command()
 def agent(
     name: str = typer.Argument(

fluxloop_cli/commands/run.py

@@ -3,10 +3,9 @@ Run command for executing experiments and simulations.
 """
 
 import asyncio
-import json
 import sys
 from pathlib import Path
-from typing import Optional
+from typing import Callable, Optional
 
 import typer
 from rich.console import Console
@@ -28,16 +27,6 @@ app = typer.Typer()
 console = Console()
 
 
-def _emit_progress_event(event: str, **payload: object) -> None:
-    """Emit a structured progress event to stderr for machine consumption."""
-    data = {"event": event, **payload}
-    try:
-        print(json.dumps(data, default=str), file=sys.stderr, flush=True)
-    except Exception:
-        # Fall back to repr if serialization fails
-        print({"event": event, **{k: repr(v) for k, v in payload.items()}}, file=sys.stderr, flush=True)
-
-
 @app.command()
 def experiment(
     config_file: Path = typer.Option(
@@ -124,6 +113,17 @@ def experiment(
         "--dry-run",
         help="Show what would be run without executing",
     ),
+    display: bool = typer.Option(
+        True,
+        "--display/--no-display",
+        help="Show rich console output (disable for plain log streaming)",
+    ),
+    yes: bool = typer.Option(
+        False,
+        "--yes",
+        "-y",
+        help="Skip confirmation prompt and run immediately",
+    ),
 ):
     """
     Run an experiment based on configuration file.
@@ -239,19 +239,8 @@ def experiment(
     
     console.print(summary)
 
-    _emit_progress_event(
-        "experiment_summary",
-        name=config.name,
-        iterations=config.iterations,
-        personas=len(config.personas),
-        total_runs=total_runs,
-        multi_turn_enabled=bool(config.multi_turn and config.multi_turn.enabled),
-        output_dir=config.output_directory,
-    )
-    
     if dry_run:
         console.print("\n[yellow]Dry run mode - no execution will occur[/yellow]")
-        _emit_progress_event("experiment_status", status="dry_run", name=config.name)
         return
     
     # Confirm execution
@@ -265,128 +254,138 @@ def experiment(
             f"\nThis will execute {total_runs} runs."
         )
 
-    _emit_progress_event(
-        "experiment_ready",
-        name=config.name,
-        total_runs=total_runs,
-    )
-
-    if not typer.confirm("Continue?"):
-        _emit_progress_event("experiment_status", status="aborted", reason="user_declined", name=config.name)
+    if yes:
+        proceed = True
+    else:
+        proceed = typer.confirm("Continue?")
+    if not proceed:
         raise typer.Abort()
     
     # Create runner
     # Run experiment with progress tracking
     console.print("\n[bold green]▶️ Starting experiment...[/bold green]\n")
 
-    _emit_progress_event("experiment_started", name=config.name, total_runs=total_runs)
-
-    with Progress(
-        SpinnerColumn(),
-        TextColumn("[progress.description]{task.description}"),
-        BarColumn(),
-        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
-        TimeRemainingColumn(),
-        TextColumn("({task.completed} of {task.total})"),
-        console=console,
-    ) as progress:
-        # Create main task
-        main_task = progress.add_task(
-            f"Running {config.name}",
-            total=total_runs,
-        )
-        multi_turn_total = (
-            config.multi_turn.max_turns
-            if config.multi_turn and config.multi_turn.max_turns
-            else None
-        )
-        turn_task = progress.add_task(
-            "[cyan]Turn 0/0",
-            total=multi_turn_total or 1,
-            visible=False,
-        )
+    def _execute_with_callbacks(
+        progress_callback: Optional[Callable[[], None]],
+        turn_progress_callback: Optional[Callable[[int, int, Optional[str]], None]],
+    ):
+        try:
+            return asyncio.run(
+                runner.run_experiment(
+                    progress_callback=progress_callback,
+                    turn_progress_callback=turn_progress_callback,
+                )
+            )
+        except KeyboardInterrupt:
+            console.print("\n[yellow]Experiment interrupted by user[/yellow]")
+            raise typer.Exit(1)
+        except Exception as e:
+            console.print(f"\n[red]Error during experiment:[/red] {e}")
+            if "--debug" in sys.argv:
+                console.print_exception()
+            raise typer.Exit(1)
 
-        def _progress_callback():
-            progress.advance(main_task)
-            task = progress.tasks[main_task]
-            _emit_progress_event(
-                "run_progress",
-                completed=int(task.completed),
-                total=int(task.total or 0),
-                description=task.description,
-                experiment=config.name,
+    if display and console.is_terminal:
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            BarColumn(),
+            TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
+            TimeRemainingColumn(),
+            TextColumn("({task.completed} of {task.total})"),
+            console=console,
+        ) as progress:
+            main_task = progress.add_task(
+                f"Running {config.name}",
+                total=total_runs,
+            )
+            multi_turn_total = (
+                config.multi_turn.max_turns
+                if config.multi_turn and config.multi_turn.max_turns
+                else None
+            )
+            turn_task = progress.add_task(
+                "[cyan]Turn 0/0",
+                total=multi_turn_total or 1,
+                visible=False,
             )
 
-        def _turn_progress_callback(
-            current_turn: int,
-            total_turns: int,
-            message: Optional[str] = None,
-        ) -> None:
-            total = total_turns or 1
-            clamped_turn = max(0, current_turn)
-            if message is None:
-                completed = min(clamped_turn, total)
-                description = f"[cyan]Turn {min(clamped_turn, total)}/{total}: complete"
+            def _progress_callback() -> None:
+                progress.advance(main_task)
+
+            def _turn_progress_callback(
+                current_turn: int,
+                total_turns: int,
+                message: Optional[str] = None,
+            ) -> None:
+                total = total_turns or 1
+                clamped_turn = max(0, current_turn)
+                if message is None:
+                    completed = min(clamped_turn, total)
+                    description = f"[cyan]Turn {min(clamped_turn, total)}/{total}: complete"
+                    progress.update(
+                        turn_task,
+                        total=total,
+                        completed=completed,
+                        description=description,
+                        visible=False,
+                    )
+                    return
+
+                if not progress.tasks[turn_task].visible:
+                    progress.update(turn_task, visible=True)
+
+                preview = message.replace("\n", " ")
+                if len(preview) > 40:
+                    preview = preview[:37] + "..."
+
+                completed = max(0, min(clamped_turn - 1, total))
+                description = f"[cyan]Turn {min(clamped_turn, total)}/{total}: {preview}"
                 progress.update(
                     turn_task,
                     total=total,
                     completed=completed,
                     description=description,
-                    visible=False,
-                )
-                _emit_progress_event(
-                    "turn_progress",
-                    state="complete",
-                    current=min(clamped_turn, total),
-                    total=total,
-                experiment=config.name,
                 )
-                return
-
-            if not progress.tasks[turn_task].visible:
-                progress.update(turn_task, visible=True)
 
-            preview = message.replace("\n", " ")
-            if len(preview) > 40:
-                preview = preview[:37] + "..."
-
-            completed = max(0, min(clamped_turn - 1, total))
-            description = f"[cyan]Turn {min(clamped_turn, total)}/{total}: {preview}"
+            results = _execute_with_callbacks(_progress_callback, _turn_progress_callback)
+    else:
+        completed_runs = 0
 
-            progress.update(
-                turn_task,
-                total=total,
-                completed=completed,
-                description=description,
-            )
-            _emit_progress_event(
-                "turn_progress",
-                state="in_progress",
-                current=clamped_turn,
-                total=total,
-                preview=preview,
-                experiment=config.name,
+        def _progress_callback() -> None:
+            nonlocal completed_runs
+            completed_runs += 1
+            print(
+                f"[fluxloop] run progress {completed_runs}/{total_runs}",
+                flush=True,
             )
 
-        # Run experiment
-        try:
-            results = asyncio.run(
-                runner.run_experiment(
-                    progress_callback=_progress_callback,
-                    turn_progress_callback=_turn_progress_callback,
+        def _turn_progress_callback(
+            current_turn: int,
+            total_turns: int,
+            message: Optional[str] = None,
+        ) -> None:
+            total = total_turns or 1
+            clamped_turn = max(0, current_turn)
+            if message:
+                preview = message.replace("\n", " ")
+                if len(preview) > 80:
+                    preview = preview[:77] + "..."
+                print(
+                    f"[fluxloop] turn progress {min(clamped_turn, total)}/{total}: {preview}",
+                    flush=True,
                 )
-            )
-        except KeyboardInterrupt:
-            console.print("\n[yellow]Experiment interrupted by user[/yellow]")
-            _emit_progress_event("experiment_status", status="interrupted", name=config.name)
-            raise typer.Exit(1)
-        except Exception as e:
-            console.print(f"\n[red]Error during experiment:[/red] {e}")
-            # Debug mode - show full traceback
-            if "--debug" in sys.argv:
-                console.print_exception()
-            _emit_progress_event("experiment_status", status="error", error=str(e), name=config.name)
-            raise typer.Exit(1)
+            else:
+                print(
+                    f"[fluxloop] turn progress {min(clamped_turn, total)}/{total}: complete",
+                    flush=True,
+                )
+
+        print(
+            f"[fluxloop] running {config.name} ({total_runs} total runs)",
+            flush=True,
+        )
+        results = _execute_with_callbacks(_progress_callback, _turn_progress_callback)
     
     config.set_resolved_input_count(results.get("input_count", config.get_input_count()))
 
@@ -399,15 +398,6 @@ def experiment(
         )
 
     _display_results(results)
-    _emit_progress_event(
-        "experiment_status",
-        status="completed",
-        name=config.name,
-        total_runs=results.get("total_runs", total_runs),
-        successful=results.get("successful"),
-        failed=results.get("failed"),
-        success_rate=results.get("success_rate"),
-    )
 
 
 @app.command()

fluxloop_cli/templates.py

@@ -429,6 +429,31 @@ def create_evaluation_config() -> str:
     ).strip() + "\n"
 
 
+def create_pytest_bridge_template(config_relative_path: str) -> str:
+    """Return a ready-to-run pytest smoke test referencing the bridge fixtures."""
+
+    return dedent(
+        f"""
+        \"\"\"FluxLoop Pytest smoke test generated via `fluxloop init pytest-template`.\"\"\"
+
+        from pathlib import Path
+
+
+        PROJECT_ROOT = Path(__file__).resolve().parents[1]
+        SIMULATION_CONFIG = PROJECT_ROOT / "{config_relative_path}"
+
+
+        def test_fluxloop_smoke(fluxloop_runner):
+            result = fluxloop_runner(
+                project_root=PROJECT_ROOT,
+                simulation_config=SIMULATION_CONFIG,
+                env={{"PYTHONPATH": str(PROJECT_ROOT)}},
+            )
+            result.require_success(label="fluxloop smoke")
+        """
+    ).strip() + "\n"
+
+
 def create_sample_agent() -> str:
     """Create a sample agent implementation."""
 

fluxloop_cli/testing/__init__.py

@@ -0,0 +1,24 @@
+"""
+Testing utilities interface definitions for FluxLoop Pytest bridge.
+"""
+
+from .types import (
+    DEFAULT_SCENARIOS,
+    FluxLoopRunnerMode,
+    FluxLoopRunnerRequest,
+    FluxLoopRunnerOverrides,
+    FluxLoopTestError,
+    FluxLoopTestResult,
+    FluxLoopTestScenario,
+)
+
+__all__ = [
+    "DEFAULT_SCENARIOS",
+    "FluxLoopRunnerMode",
+    "FluxLoopRunnerOverrides",
+    "FluxLoopRunnerRequest",
+    "FluxLoopTestError",
+    "FluxLoopTestResult",
+    "FluxLoopTestScenario",
+]
+

fluxloop_cli/testing/pytest_plugin.py

@@ -0,0 +1,421 @@
+"""
+Pytest fixtures for executing FluxLoop experiments (Week 2/4 deliverables).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import json
+import os
+import subprocess
+import sys
+import uuid
+from pathlib import Path
+from typing import Any, Callable, Mapping, MutableMapping, Optional, Sequence, Union
+
+import pytest
+import yaml
+
+from fluxloop_cli.config_loader import load_experiment_config, merge_config_overrides
+from fluxloop_cli.runner import ExperimentRunner
+
+from .types import (
+    FluxLoopRunnerRequest,
+    FluxLoopRunnerOverrides,
+    FluxLoopTestResult,
+)
+
+PathLike = Union[str, Path]
+RunnerCallable = Callable[..., FluxLoopTestResult]
+WORKSPACE_DIR_NAME = ".fluxloop_pytest"
+
+
+def _build_request(
+    *,
+    project_root: PathLike,
+    simulation_config: Optional[PathLike],
+    overrides: Optional[Mapping[str, Any]],
+    mode: str,
+    timeout: Optional[int],
+    env: Optional[Mapping[str, str]],
+) -> FluxLoopRunnerRequest:
+    request = FluxLoopRunnerRequest(
+        project_root=Path(project_root),
+        simulation_config=Path(simulation_config) if simulation_config else None,
+        mode=mode,
+    )
+    if timeout is not None:
+        request.timeout_seconds = timeout
+    if overrides:
+        request.overrides.merge(overrides)
+    if env:
+        request.extra_env.update({k: str(v) for k, v in env.items()})
+    request.resolve_paths()
+    return request
+
+
+def _determine_config_path(request: FluxLoopRunnerRequest) -> Path:
+    if request.simulation_config and request.simulation_config.exists():
+        return request.simulation_config
+
+    candidates: Sequence[Path] = (
+        request.project_root / "configs",
+        request.project_root / "setting.yaml",
+    )
+    for candidate in candidates:
+        if candidate.exists():
+            return candidate
+    raise FileNotFoundError(
+        f"Could not locate configs/ or setting.yaml under {request.project_root}"
+    )
+
+
+@contextlib.contextmanager
+def _patched_environ(values: Mapping[str, str]):
+    previous: MutableMapping[str, Optional[str]] = {}
+    try:
+        for key, value in values.items():
+            previous[key] = os.environ.get(key)
+            os.environ[key] = value
+        yield
+    finally:
+        for key, old_value in previous.items():
+            if old_value is None:
+                os.environ.pop(key, None)
+            else:
+                os.environ[key] = old_value
+
+
+def _build_result_payload(runner: ExperimentRunner) -> dict[str, Any]:
+    results = runner.results
+    payload = {
+        "results": {
+            "total_runs": results.get("total_runs", 0),
+            "failed": results.get("failed", 0),
+            "success_rate": results.get("success_rate", 0.0),
+            "avg_duration_ms": results.get("avg_duration_ms", 0.0),
+        },
+        "errors": results.get("errors", []),
+        "output_dir": str(runner.output_dir),
+    }
+    return payload
+
+
+def _build_result_object(runner: ExperimentRunner) -> FluxLoopTestResult:
+    payload = _build_result_payload(runner)
+    trace_summary_path = runner.output_dir / "trace_summary.jsonl"
+    if not trace_summary_path.exists():
+        trace_summary_path = None
+    per_trace_path = runner.output_dir / "per_trace_analysis" / "per_trace.jsonl"
+    if not per_trace_path.exists():
+        per_trace_path = None
+    return FluxLoopTestResult.from_summary(
+        payload,
+        trace_summary_path=trace_summary_path,
+        per_trace_path=per_trace_path,
+    )
+
+
+def _run_with_timeout(
+    runner: ExperimentRunner,
+    *,
+    timeout: Optional[int],
+) -> None:
+    async def _execute():
+        coroutine = runner.run_experiment()
+        if timeout is not None:
+            return await asyncio.wait_for(coroutine, timeout=timeout)
+        return await coroutine
+
+    try:
+        asyncio.run(_execute())
+    except asyncio.TimeoutError as exc:
+        raise AssertionError(
+            f"FluxLoop experiment exceeded timeout ({timeout}s)"
+        ) from exc
+
+
+def _load_config_with_overrides(
+    request: FluxLoopRunnerRequest,
+    *,
+    require_inputs_file: bool,
+):
+    config_path = _determine_config_path(request)
+    config = load_experiment_config(
+        config_path,
+        require_inputs_file=require_inputs_file,
+    )
+    overrides = request.overrides.normalized()
+    if overrides:
+        source_dir = config.get_source_dir()
+        config = merge_config_overrides(config, overrides)
+        if source_dir:
+            config.set_source_dir(source_dir)
+    return config
+
+
+def _run_via_sdk(
+    request: FluxLoopRunnerRequest,
+    *,
+    require_inputs_file: bool,
+) -> FluxLoopTestResult:
+    config = _load_config_with_overrides(
+        request,
+        require_inputs_file=require_inputs_file,
+    )
+
+    runner = ExperimentRunner(config)
+
+    with _patched_environ(request.extra_env):
+        _run_with_timeout(runner, timeout=request.timeout_seconds)
+
+    return _build_result_object(runner)
+
+
+def _workspace_dir(project_root: Path) -> Path:
+    workspace = project_root / WORKSPACE_DIR_NAME
+    workspace.mkdir(exist_ok=True)
+    return workspace
+
+
+def _abs_path(value: Optional[str], base: Path) -> Optional[str]:
+    if not value:
+        return value
+    path = Path(value)
+    if path.is_absolute():
+        return str(path)
+    return str((base / path).resolve())
+
+
+def _config_to_python_dict(config) -> dict[str, Any]:
+    if hasattr(config, "model_dump"):
+        return config.model_dump(mode="json")  # type: ignore[call-arg]
+    return config.to_dict()
+
+
+def _materialize_cli_config(
+    config,
+    request: FluxLoopRunnerRequest,
+    workspace: Path,
+) -> tuple[Path, Path]:
+    source_dir = config.get_source_dir() or request.project_root
+    config_dict = _config_to_python_dict(config)
+    outputs_base = (workspace / "outputs").resolve()
+    outputs_base.mkdir(exist_ok=True)
+    config_dict["output_directory"] = str(outputs_base)
+
+    if config_dict.get("inputs_file"):
+        config_dict["inputs_file"] = _abs_path(config_dict["inputs_file"], source_dir)
+
+    runner_dict = config_dict.get("runner") or {}
+    if runner_dict.get("working_directory"):
+        runner_dict["working_directory"] = _abs_path(
+            runner_dict["working_directory"], source_dir
+        )
+    python_paths = runner_dict.get("python_path") or []
+    runner_dict["python_path"] = [
+        _abs_path(entry, source_dir) for entry in python_paths if entry
+    ]
+    config_dict["runner"] = runner_dict
+
+    replay_args = config_dict.get("replay_args") or {}
+    if replay_args.get("recording_file"):
+        replay_args["recording_file"] = _abs_path(replay_args["recording_file"], source_dir)
+    config_dict["replay_args"] = replay_args
+
+    config_path = request.project_root / f".cli_config_{uuid.uuid4().hex}.yaml"
+    with open(config_path, "w", encoding="utf-8") as handle:
+        yaml.safe_dump(config_dict, handle, sort_keys=False)
+    return config_path, outputs_base
+
+
+def _update_pythonpath(env: MutableMapping[str, str], project_root: Path) -> None:
+    existing = env.get("PYTHONPATH")
+    entries = [str(project_root)]
+    if existing:
+        entries.append(existing)
+    env["PYTHONPATH"] = os.pathsep.join(entries)
+
+
+def _append_log(path: Path, content: str) -> None:
+    with open(path, "a", encoding="utf-8") as handle:
+        handle.write(content)
+        if not content.endswith("\n"):
+            handle.write("\n")
+
+
+def _invoke_cli_command(
+    args: list[str],
+    *,
+    env: Mapping[str, str],
+    cwd: Path,
+    timeout: Optional[int],
+    stdout_log: Path,
+    stderr_log: Path,
+) -> subprocess.CompletedProcess[str]:
+    proc = subprocess.run(
+        [sys.executable, "-m", "fluxloop_cli.main", *args],
+        text=True,
+        capture_output=True,
+        cwd=str(cwd),
+        env=dict(env),
+        timeout=timeout,
+    )
+    _append_log(stdout_log, f"$ fluxloop {' '.join(args)}")
+    _append_log(stdout_log, proc.stdout)
+    _append_log(stderr_log, proc.stderr)
+    return proc
+
+
+def _latest_experiment_dir(outputs_base: Path) -> Path:
+    candidates = [p for p in outputs_base.iterdir() if p.is_dir()]
+    if not candidates:
+        raise AssertionError(f"No experiment directories found under {outputs_base}")
+    return max(candidates, key=lambda p: p.stat().st_mtime)
+
+
+def _run_via_cli(
+    request: FluxLoopRunnerRequest,
+    *,
+    require_inputs_file: bool,
+) -> FluxLoopTestResult:
+    config = _load_config_with_overrides(
+        request,
+        require_inputs_file=require_inputs_file,
+    )
+    workspace = _workspace_dir(request.project_root)
+    logs_dir = workspace / "logs"
+    logs_dir.mkdir(exist_ok=True)
+
+    run_id = uuid.uuid4().hex
+    stdout_log = logs_dir / f"{run_id}.stdout.log"
+    stderr_log = logs_dir / f"{run_id}.stderr.log"
+
+    config_path, outputs_base = _materialize_cli_config(config, request, workspace)
+
+    env = os.environ.copy()
+    env.update({k: v for k, v in request.extra_env.items()})
+    _update_pythonpath(env, request.project_root)
+
+    try:
+        run_args = [
+            "run",
+            "experiment",
+            "--config",
+            str(config_path),
+            "--yes",
+            "--no-display",
+            "--no-collector",
+        ]
+        run_proc = _invoke_cli_command(
+            run_args,
+            env=env,
+            cwd=request.project_root,
+            timeout=request.timeout_seconds,
+            stdout_log=stdout_log,
+            stderr_log=stderr_log,
+        )
+        if run_proc.returncode != 0:
+            raise AssertionError(
+                f"fluxloop run experiment failed (see {stderr_log})"
+            )
+
+        experiment_dir = _latest_experiment_dir(outputs_base)
+
+        parse_args = [
+            "parse",
+            "experiment",
+            str(experiment_dir),
+            "--overwrite",
+        ]
+        parse_proc = _invoke_cli_command(
+            parse_args,
+            env=env,
+            cwd=request.project_root,
+            timeout=request.timeout_seconds,
+            stdout_log=stdout_log,
+            stderr_log=stderr_log,
+        )
+        if parse_proc.returncode != 0:
+            raise AssertionError(
+                f"fluxloop parse experiment failed (see {stderr_log})"
+            )
+
+        summary_path = experiment_dir / "summary.json"
+        if not summary_path.exists():
+            raise AssertionError(f"summary.json missing under {experiment_dir}")
+        summary_payload = json.loads(summary_path.read_text(encoding="utf-8"))
+        summary_payload["output_dir"] = str(experiment_dir)
+
+        trace_summary_path = experiment_dir / "trace_summary.jsonl"
+        if not trace_summary_path.exists():
+            trace_summary_path = None
+
+        per_trace_path = experiment_dir / "per_trace_analysis" / "per_trace.jsonl"
+        if not per_trace_path.exists():
+            per_trace_path = None
+
+        cli_command = f"fluxloop {' '.join(run_args)}"
+
+        return FluxLoopTestResult.from_summary(
+            summary_payload,
+            trace_summary_path=trace_summary_path,
+            per_trace_path=per_trace_path,
+            stdout_path=stdout_log,
+            stderr_path=stderr_log,
+            cli_command=cli_command,
+        )
+    finally:
+        with contextlib.suppress(FileNotFoundError):
+            config_path.unlink()
+
+
+def _build_runner_fixture(mode: str) -> RunnerCallable:
+    def _runner(
+        *,
+        project_root: PathLike,
+        simulation_config: Optional[PathLike] = None,
+        overrides: Optional[Mapping[str, Any]] = None,
+        timeout: Optional[int] = None,
+        env: Optional[Mapping[str, str]] = None,
+        require_inputs_file: bool = True,
+    ) -> FluxLoopTestResult:
+        request = _build_request(
+            project_root=project_root,
+            simulation_config=simulation_config,
+            overrides=overrides,
+            mode=mode,
+            timeout=timeout,
+            env=env,
+        )
+        if mode == "cli":
+            return _run_via_cli(
+                request,
+                require_inputs_file=require_inputs_file,
+            )
+        return _run_via_sdk(
+            request,
+            require_inputs_file=require_inputs_file,
+        )
+
+    return _runner
+
+
+@pytest.fixture
+def fluxloop_runner() -> RunnerCallable:
+    """
+    Execute experiments via the SDK path (default fixture).
+    """
+
+    return _build_runner_fixture("sdk")
+
+
+@pytest.fixture
+def fluxloop_cli() -> RunnerCallable:
+    """
+    Execute experiments via the CLI path (subprocess parity).
+    """
+
+    return _build_runner_fixture("cli")
+

fluxloop_cli/testing/types.py

@@ -0,0 +1,188 @@
+"""
+Core data structures for the Pytest bridge (Week 1 deliverables).
+
+These definitions capture the request/result contract so fixtures can be
+implemented later without changing user-facing APIs.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, Mapping, MutableMapping, Optional, Sequence
+
+
+FluxLoopRunnerMode = str  # Literal["sdk", "cli"] 예정, but keep flexible for now
+
+
+@dataclass(slots=True)
+class FluxLoopRunnerOverrides:
+    """User supplied key-path → value overrides for ExperimentConfig."""
+
+    values: MutableMapping[str, Any] = field(default_factory=dict)
+
+    def normalized(self) -> Dict[str, Any]:
+        """Return a shallow copy for downstream mutation safety."""
+        return dict(self.values)
+
+    def merge(self, *overrides: Mapping[str, Any]) -> "FluxLoopRunnerOverrides":
+        merged = self.normalized()
+        for item in overrides:
+            merged.update(item)
+        self.values = merged
+        return self
+
+
+@dataclass(slots=True)
+class FluxLoopRunnerRequest:
+    """
+    Normalized request payload consumed by the future pytest fixtures.
+
+    `project_root` is required so we can locate configs/ and .env files.
+    """
+
+    project_root: Path
+    simulation_config: Optional[Path] = None
+    mode: FluxLoopRunnerMode = "sdk"
+    overrides: FluxLoopRunnerOverrides = field(default_factory=FluxLoopRunnerOverrides)
+    timeout_seconds: Optional[int] = 600
+    extra_env: MutableMapping[str, str] = field(default_factory=dict)
+
+    def resolve_paths(self) -> None:
+        """
+        Ensure all paths are absolute so fixtures can run from any cwd.
+        """
+        self.project_root = self.project_root.expanduser().resolve()
+        if self.simulation_config is not None:
+            config_path = self.simulation_config
+            if not config_path.is_absolute():
+                config_path = self.project_root / config_path
+            self.simulation_config = config_path.expanduser().resolve()
+
+
+@dataclass(slots=True)
+class FluxLoopTestError:
+    """Represents a single failure entry from the experiment summary."""
+
+    iteration: Optional[int]
+    persona: Optional[str]
+    input_text: Optional[str]
+    message: str
+
+    @classmethod
+    def from_dict(cls, payload: Mapping[str, Any]) -> "FluxLoopTestError":
+        return cls(
+            iteration=payload.get("iteration"),
+            persona=payload.get("persona"),
+            input_text=payload.get("input"),
+            message=str(payload.get("error", "")),
+        )
+
+
+@dataclass(slots=True)
+class FluxLoopTestResult:
+    """Aggregate summary emitted back to pytest test functions."""
+
+    total_runs: int
+    failed_runs: int
+    success_rate: float
+    avg_duration_ms: float
+    output_dir: Path
+    summary_path: Path
+    trace_summary_path: Optional[Path]
+    per_trace_path: Optional[Path]
+    stdout_path: Optional[Path] = None
+    stderr_path: Optional[Path] = None
+    cli_command: Optional[str] = None
+    errors: Sequence[FluxLoopTestError] = field(default_factory=tuple)
+
+    @property
+    def succeeded(self) -> bool:
+        return self.failed_runs == 0
+
+    def require_success(self, *, label: Optional[str] = None) -> None:
+        if self.succeeded:
+            return
+        reason = label or "FluxLoop experiment"
+        details = "; ".join(err.message for err in self.errors[:3]) or "unknown failure"
+        raise AssertionError(f"{reason} failed ({self.failed_runs} runs): {details}")
+
+    @classmethod
+    def from_summary(
+        cls,
+        summary: Mapping[str, Any],
+        *,
+        trace_summary_path: Optional[Path] = None,
+        per_trace_path: Optional[Path] = None,
+        stdout_path: Optional[Path] = None,
+        stderr_path: Optional[Path] = None,
+        cli_command: Optional[str] = None,
+    ) -> "FluxLoopTestResult":
+        results = summary.get("results", {})
+        errors = [FluxLoopTestError.from_dict(raw) for raw in summary.get("errors", [])]
+        output_dir = Path(summary.get("output_dir", summary.get("name", "experiments"))).expanduser()
+        summary_path = output_dir / "summary.json"
+        return cls(
+            total_runs=int(results.get("total_runs", 0)),
+            failed_runs=int(results.get("failed", 0)),
+            success_rate=float(results.get("success_rate", 0.0)),
+            avg_duration_ms=float(results.get("avg_duration_ms", 0.0)),
+            output_dir=output_dir,
+            summary_path=summary_path,
+            trace_summary_path=trace_summary_path,
+            per_trace_path=per_trace_path,
+            stdout_path=stdout_path,
+            stderr_path=stderr_path,
+            cli_command=cli_command,
+            errors=tuple(errors),
+        )
+
+
+@dataclass(slots=True)
+class FluxLoopTestScenario:
+    """Design-time catalog of QA scenarios the fixtures must support."""
+
+    name: str
+    description: str
+    tags: Sequence[str] = field(default_factory=tuple)
+
+
+DEFAULT_SCENARIOS: Sequence[FluxLoopTestScenario] = (
+    FluxLoopTestScenario(
+        name="single-turn-sync",
+        description="Basic python function target without multi-turn supervisor.",
+        tags=("sdk", "happy-path"),
+    ),
+    FluxLoopTestScenario(
+        name="multi-turn-supervisor",
+        description="ConversationSupervisor enabled with scripted user turns.",
+        tags=("sdk", "multi-turn"),
+    ),
+    FluxLoopTestScenario(
+        name="subprocess-jsonl",
+        description="Runner configured via subprocess JSONL stream to validate CLI parity.",
+        tags=("cli", "subprocess"),
+    ),
+    FluxLoopTestScenario(
+        name="mcp-adapter",
+        description="MCP server target requiring environment injection and guard handling.",
+        tags=("sdk", "mcp"),
+    ),
+    FluxLoopTestScenario(
+        name="failure-capture",
+        description="Intentional failure to verify error surfaces and pytest assertion formatting.",
+        tags=("sdk", "negative"),
+    ),
+)
+
+
+__all__ = [
+    "FluxLoopRunnerMode",
+    "FluxLoopRunnerOverrides",
+    "FluxLoopRunnerRequest",
+    "FluxLoopTestError",
+    "FluxLoopTestResult",
+    "FluxLoopTestScenario",
+    "DEFAULT_SCENARIOS",
+]
+

{fluxloop_cli-0.2.28.dist-info → fluxloop_cli-0.2.30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fluxloop-cli
-Version: 0.2.28
+Version: 0.2.30
 Summary: FluxLoop CLI for running agent simulations
 Author-email: FluxLoop Team <team@fluxloop.dev>
 License: Apache-2.0
@@ -71,6 +71,7 @@ The legacy `setting.yaml` is still supported, but new projects created with
 - `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
 - `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
 - `fluxloop doctor` – summarize Python, FluxLoop CLI/MCP, and MCP index state for the active environment
+- `--yes/-y` (for `fluxloop run experiment`) – skip the interactive confirmation prompt, ideal for CI and the Pytest bridge
 
 ### Multi-turn supervisor options
 
@@ -88,6 +89,16 @@ These flags override the values in `configs/simulation.yaml` (`multi_turn` block
 
 Run `fluxloop --help` or `fluxloop <command> --help` for more detail.
 
+## Pytest Bridge (0.2.29+)
+
+- `fluxloop init pytest-template [project_root]` creates `tests/test_fluxloop_smoke.py`, already wired to the new `fluxloop_runner` fixture.
+- Fixtures live in `fluxloop_cli.testing.pytest_plugin` and return a `FluxLoopTestResult`, so you can assert on `total_runs`, `success_rate`, or call `require_success()`.
+- Full guide + CI example: see `docs/guides/pytest_bridge.md` (includes GitHub Actions workflow at `examples/ci/fluxloop_pytest.yml`).
+- Typical workflow:
+  1. `pip install -e packages/cli[dev]`
+  2. `fluxloop init pytest-template .`
+  3. `pytest -k fluxloop_smoke --maxfail=1`
+
 ## Evaluation Workflow
 
 Evaluation now follows a two-step process so that multi-turn context is preserved:

{fluxloop_cli-0.2.28.dist-info → fluxloop_cli-0.2.30.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fluxloop_cli/__init__.py,sha256=Yy0ysK8xslb79WaB0apu48ZBwviey964qWc5uDat8Mc,143
+fluxloop_cli/__init__.py,sha256=9mSy8Uew5CRCHwCbTK63mr3OmKigybmLBSHEHuVz_1U,143
 fluxloop_cli/arg_binder.py,sha256=6-i81kK6kmR8inNvIp0GeGY51M5Jere-hAdignMl2bE,12765
 fluxloop_cli/config_loader.py,sha256=PYy0CfGVbU8jpPbx4sJzOu7i3BbrkQMNaRiSOp_uX9g,10307
 fluxloop_cli/config_schema.py,sha256=JZJRcMFun5hp3vKLAyek7W3NvISyzRzZt0BZAeSU38I,2415
@@ -11,17 +11,17 @@ fluxloop_cli/main.py,sha256=apLSNIgXv-V269ttKvd-t5za2UUjePswmR5KZGhOLPE,2772
 fluxloop_cli/project_paths.py,sha256=FoHp-g3aY1nytxGys85Oy3wJ6gmiKU6FVOwkgTtlHNA,4128
 fluxloop_cli/runner.py,sha256=ee-ue9wmN7X3VKuIk1Sg3z_vYiKwypWkKxFsTccH6rA,50199
 fluxloop_cli/target_loader.py,sha256=ACCu2izqGKoOrEiNnAajH0FgLZcw3j1pWn5rAhEuWFU,5528
-fluxloop_cli/templates.py,sha256=SvJrz0xqZ7VxkXnQ4FgffiClLuno3tMAG5qNJF4SwKc,19277
+fluxloop_cli/templates.py,sha256=880cum_f8J40gwQiR7HbmQuqWgUXBTmji5EsaCjQDm0,20070
 fluxloop_cli/validators.py,sha256=_bLXmxUSzVrDtLjqyTba0bDqamRIaOUHhV4xZ7K36Xw,1155
 fluxloop_cli/commands/__init__.py,sha256=i8xTg7uE9B6ExA0EP_j2h9DZWhD-eC94a-7khXIzRwo,243
 fluxloop_cli/commands/config.py,sha256=9moP68pcNBnJYGMcptgdU1UW-6WlI-Nan6eykPdwl8I,12121
 fluxloop_cli/commands/doctor.py,sha256=svoz9wbLv5Byj9G1Y9GI5lYpVdoG2e7IzfG7tZxuMK0,8184
 fluxloop_cli/commands/evaluate.py,sha256=itxb3TyfqV44DaCczC3oC94cJ2rmz2cIpY-794eXhvQ,7229
 fluxloop_cli/commands/generate.py,sha256=6tBanI8gj5Zzwz2cv1YeW60bYiO7iWivD0AgOcJ-GUU,7172
-fluxloop_cli/commands/init.py,sha256=eugHrsOTaXQmX5jxRNGSO3C3SpRVI5BCSqP9GqmGHPQ,8156
+fluxloop_cli/commands/init.py,sha256=1KUdOs9HqwgBZB94YyKjn95TrK3-t4cmBAUAMA6uwhc,10530
 fluxloop_cli/commands/parse.py,sha256=5dgoHt6x2h7SBf6AOlbafoIWz3psEBrJdZyErIvgSgw,15417
 fluxloop_cli/commands/record.py,sha256=IfoXdFAentyerBrIdQeweYr_kM7DjGks2aoKjjqCZ9Q,5139
-fluxloop_cli/commands/run.py,sha256=OMT9XrDBDZACyQ_8tlUiqSIRq6QROH25Uc1PHZoEHyU,16849
+fluxloop_cli/commands/run.py,sha256=Mw5f6OdunjaOtaZAZoT9Ct1AL8FiBksqh2S3cz8_cCk,16431
 fluxloop_cli/commands/status.py,sha256=ERZrWoSP3V7dz5A_TEE5b8E0nGwsPggP4nXw4tLOzxE,7841
 fluxloop_cli/evaluation/__init__.py,sha256=voMYkA4S5Gb3GyvpTU8JnKu0novOyZBEag1cYEqYi1Q,759
 fluxloop_cli/evaluation/artifacts.py,sha256=zDVB-SSQZ1JYWLPHwrfz5AuCv46pUFZfEfFY_5doAjQ,2176
@@ -41,8 +41,11 @@ fluxloop_cli/evaluation/prompts/information_completeness.py,sha256=BjZ-wEIANWWBQ
 fluxloop_cli/evaluation/prompts/intent_recognition.py,sha256=QlroUq1y0wwWsT2qcS1NXdo7OwNqUu4Tw2xx7Frr50Q,1572
 fluxloop_cli/evaluation/prompts/response_clarity.py,sha256=1OYyx39UfS-YR__VdqhLejHNxH1NBVFm7Bha3MuMN9Y,1418
 fluxloop_cli/evaluation/prompts/response_consistency.py,sha256=tyO3Wsf2J_CvErCuNc2bw-UQWqRW_BupaBj3CS66_Yk,1550
-fluxloop_cli-0.2.28.dist-info/METADATA,sha256=Vs5b3XsEL5irrC2_8DzahTy1bfD-O3JznpzTqB5hxsY,6329
-fluxloop_cli-0.2.28.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-fluxloop_cli-0.2.28.dist-info/entry_points.txt,sha256=NxOEMku4yLMY5kp_Qcd3JcevfXP6A98FsSf9xHcwkyE,51
-fluxloop_cli-0.2.28.dist-info/top_level.txt,sha256=ahLkaxzwhmVU4z-YhkmQVzAbW3-wez9cKnwPiDK7uKM,13
-fluxloop_cli-0.2.28.dist-info/RECORD,,
+fluxloop_cli/testing/__init__.py,sha256=3FZ0JzA6gbeDAOlvplEiNDplbOXmNT-TYQ-4C7eHUlo,482
+fluxloop_cli/testing/pytest_plugin.py,sha256=j9fAlh_uL_yRcXPPcx1AcvnxpKfUdfPnEf5ozJOQ7aA,12533
+fluxloop_cli/testing/types.py,sha256=CcpoSnV-eOBc_Ngvg6ZRpeb-Bnz4eT2zBdGz0-icVB4,6190
+fluxloop_cli-0.2.30.dist-info/METADATA,sha256=gp3keZepVZ1jT9xQTBG3KpyIsFfdKTCTdfmNi6mCK80,7074
+fluxloop_cli-0.2.30.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+fluxloop_cli-0.2.30.dist-info/entry_points.txt,sha256=NxOEMku4yLMY5kp_Qcd3JcevfXP6A98FsSf9xHcwkyE,51
+fluxloop_cli-0.2.30.dist-info/top_level.txt,sha256=ahLkaxzwhmVU4z-YhkmQVzAbW3-wez9cKnwPiDK7uKM,13
+fluxloop_cli-0.2.30.dist-info/RECORD,,