fluxloop-cli 0.1.0__tar.gz → 0.2.1__tar.gz

{fluxloop_cli-0.1.0 → fluxloop_cli-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fluxloop-cli
-Version: 0.1.0
+Version: 0.2.1
 Summary: FluxLoop CLI for running agent simulations
 Author-email: FluxLoop Team <team@fluxloop.dev>
 License: Apache-2.0
@@ -40,47 +40,51 @@ Requires-Dist: anthropic>=0.7.0; extra == "anthropic"
 
 # FluxLoop CLI
 
-Command-line interface for running agent simulations and managing FluxLoop workflows.
+Command-line interface for running agent simulations.
 
 ## Installation
 
-```bash
+```
 pip install fluxloop-cli
 ```
 
-## Quick Start
-
-```bash
-# Initialize a new FluxLoop project
-fluxloop init
+## Configuration Overview (v0.2.0)
 
-# Run agent simulations
-fluxloop run
+FluxLoop CLI now stores experiment settings in four files under `configs/`:
 
-# Generate test inputs
-fluxloop generate
+- `configs/project.yaml` – project metadata, collector defaults
+- `configs/input.yaml` – personas, base inputs, input generation options
+- `configs/simulation.yaml` – runtime parameters (iterations, runner, replay args)
+- `configs/evaluation.yaml` – evaluator definitions (rule-based, LLM judge, etc.)
 
-# Check status
-fluxloop status
-```
+The legacy `setting.yaml` is still supported, but new projects created with
+`fluxloop init project` will generate the structured layout above.
 
-## Features
+## Key Commands
 
-- 🚀 **Easy Setup**: Initialize projects with a single command
-- 🔄 **Simulation Runner**: Execute agent tests with various input scenarios
-- 📝 **Input Generation**: LLM-powered test input generation
-- 📊 **Rich Output**: Beautiful terminal UI with detailed progress tracking
+- `fluxloop init project` – scaffold a new project (configs, `.env`, examples)
+- `fluxloop generate inputs` – produce input variations for the active project
+- `fluxloop run experiment` – execute an experiment using `configs/simulation.yaml`
+- `fluxloop parse experiment` – convert experiment outputs into readable artifacts
+- `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
+- `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
 
-## Requirements
+Run `fluxloop --help` or `fluxloop <command> --help` for more detail.
 
-- Python 3.8 or higher
-- FluxLoop SDK
+## Developing
 
-## Documentation
+Install dependencies and run tests:
 
-For detailed documentation, visit [https://docs.fluxloop.dev](https://docs.fluxloop.dev)
+```
+python -m venv .venv
+source .venv/bin/activate
+pip install -e .[dev]
+pytest
+```
 
-## License
+To package the CLI:
 
-Apache License 2.0 - see LICENSE file for details
+```
+./build.sh
+```
 

fluxloop_cli-0.2.1/README.md

@@ -0,0 +1,50 @@
+# FluxLoop CLI
+
+Command-line interface for running agent simulations.
+
+## Installation
+
+```
+pip install fluxloop-cli
+```
+
+## Configuration Overview (v0.2.0)
+
+FluxLoop CLI now stores experiment settings in four files under `configs/`:
+
+- `configs/project.yaml` – project metadata, collector defaults
+- `configs/input.yaml` – personas, base inputs, input generation options
+- `configs/simulation.yaml` – runtime parameters (iterations, runner, replay args)
+- `configs/evaluation.yaml` – evaluator definitions (rule-based, LLM judge, etc.)
+
+The legacy `setting.yaml` is still supported, but new projects created with
+`fluxloop init project` will generate the structured layout above.
+
+## Key Commands
+
+- `fluxloop init project` – scaffold a new project (configs, `.env`, examples)
+- `fluxloop generate inputs` – produce input variations for the active project
+- `fluxloop run experiment` – execute an experiment using `configs/simulation.yaml`
+- `fluxloop parse experiment` – convert experiment outputs into readable artifacts
+- `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
+- `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
+
+Run `fluxloop --help` or `fluxloop <command> --help` for more detail.
+
+## Developing
+
+Install dependencies and run tests:
+
+```
+python -m venv .venv
+source .venv/bin/activate
+pip install -e .[dev]
+pytest
+```
+
+To package the CLI:
+
+```
+./build.sh
+```
+

{fluxloop_cli-0.1.0 → fluxloop_cli-0.2.1}/fluxloop_cli/__init__.py

@@ -2,7 +2,7 @@
 FluxLoop CLI - Command-line interface for running agent simulations.
 """
 
-__version__ = "0.1.0"
+__version__ = "0.2.1"
 
 from .main import app
 

{fluxloop_cli-0.1.0 → fluxloop_cli-0.2.1}/fluxloop_cli/commands/__init__.py

@@ -1,5 +1,5 @@
 """CLI commands."""
 
-from . import config, generate, init, parse, run, status
+from . import config, generate, init, parse, run, status, record
 
-__all__ = ["config", "generate", "init", "parse", "run", "status"]
+__all__ = ["config", "generate", "init", "parse", "run", "status", "record"]

{fluxloop_cli-0.1.0 → fluxloop_cli-0.2.1}/fluxloop_cli/commands/config.py

@@ -15,6 +15,7 @@ from rich.table import Table
 from ..config_loader import load_experiment_config
 from ..templates import create_env_file, create_gitignore, create_sample_agent
 from ..constants import DEFAULT_CONFIG_PATH, DEFAULT_ROOT_DIR_NAME
+from ..config_schema import CONFIG_SECTION_FILENAMES
 from ..project_paths import (
     resolve_config_path,
     resolve_env_path,
@@ -228,7 +229,12 @@ def set_llm(
     api_key: str = typer.Argument(..., help="API key or token for the provider"),
     model: Optional[str] = typer.Option(None, "--model", "-m", help="Default model to use"),
     overwrite_env: bool = typer.Option(False, "--overwrite-env", help="Overwrite existing key in .env"),
-    config_file: Path = typer.Option(DEFAULT_CONFIG_PATH, "--file", "-f", help="Configuration file to update"),
+    config_file: Path = typer.Option(
+        Path(CONFIG_SECTION_FILENAMES["input"]),
+        "--file",
+        "-f",
+        help="Configuration file to update",
+    ),
     env_file: Path = typer.Option(Path(".env"), "--env-file", help="Path to environment file"),
     project: Optional[str] = typer.Option(None, "--project", help="Project name under the FluxLoop root"),
     root: Path = typer.Option(Path(DEFAULT_ROOT_DIR_NAME), "--root", help="FluxLoop root directory"),
@@ -330,7 +336,11 @@ def validate(
         raise typer.Exit(1)
     
     try:
-        config = load_experiment_config(config_file)
+        config = load_experiment_config(
+            resolved_path,
+            project=project,
+            root=root,
+        )
         
         # Show validation results
         console.print("[green]✓[/green] Configuration is valid!\n")

{fluxloop_cli-0.1.0 → fluxloop_cli-0.2.1}/fluxloop_cli/commands/generate.py

@@ -1,10 +1,9 @@
 """Generate command for producing input datasets."""
 
-import json
 import os
 import sys
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Dict, List, Optional
 
 import typer
 from rich.console import Console
@@ -93,11 +92,6 @@ def inputs(
         "--llm-api-key",
         help="API key for LLM provider (falls back to FLUXLOOP_LLM_API_KEY)",
     ),
-    from_recording: Optional[Path] = typer.Option(
-        None,
-        "--from-recording",
-        help="Use a recorded call (JSONL) as template for generated inputs",
-    ),
 ):
     """Generate input variations for review before running experiments."""
     resolved_config = resolve_config_path(config_file, project, root)
@@ -136,6 +130,8 @@ def inputs(
     try:
         config = load_experiment_config(
             resolved_config,
+            project=project,
+            root=root,
             require_inputs_file=require_inputs_file,
         )
     except Exception as exc:
@@ -201,20 +197,8 @@ def inputs(
         llm_api_key_override=llm_api_key,
     )
 
-    recording_template = None
-    if from_recording:
-        try:
-            recording_template = _load_recording_template(from_recording, config)
-            console.print(
-                "📝 Generating inputs from recording"
-                f"\n  → Base content: [cyan]{recording_template['base_content'][:60]}[/cyan]"
-            )
-        except Exception as exc:
-            console.print(f"[red]Failed to load recording template:[/red] {exc}")
-            raise typer.Exit(1)
-
     try:
-        result = generate_inputs(config, settings, recording_template=recording_template)
+        result = generate_inputs(config, settings)
     except Exception as exc:
         console.print(f"[red]Generation failed:[/red] {exc}")
         if "--debug" in sys.argv:
@@ -238,67 +222,3 @@ def inputs(
         f"\n🧠 Mode: [magenta]{result.metadata.get('generation_mode', 'deterministic')}[/magenta]"
         f"\n🎯 Strategies: [cyan]{', '.join(strategy for strategy in strategies_used)}[/cyan]"
     )
-
-
-def _load_recording_template(path: Path, config: "ExperimentConfig") -> Dict[str, object]:
-    target_path = path
-    if not target_path.is_absolute():
-        source_dir = config.get_source_dir()
-        if source_dir:
-            target_path = (source_dir / target_path).resolve()
-        else:
-            target_path = target_path.resolve()
-
-    if not target_path.exists():
-        raise FileNotFoundError(f"Recording file not found: {target_path}")
-
-    with target_path.open("r", encoding="utf-8") as handle:
-        first_line = handle.readline()
-
-    if not first_line:
-        raise ValueError(f"Recording file is empty: {target_path}")
-
-    try:
-        record = json.loads(first_line)
-    except json.JSONDecodeError as exc:
-        raise ValueError(f"Invalid JSON in recording file {target_path}: {exc}")
-
-    kwargs = record.get("kwargs", {})
-    base_content = _extract_content(kwargs)
-    if not base_content:
-        sample = json.dumps(kwargs, indent=2)[:500]
-        raise ValueError(
-            "Unable to locate textual content in recording. "
-            "Expected keys such as data.content, input, input_text, message, query, text, or content." \
-            f"\nRecording snapshot:\n{sample}"
-        )
-
-    return {
-        "base_content": base_content,
-        "full_kwargs": kwargs,
-        "target": record.get("target"),
-    }
-
-
-def _extract_content(kwargs: Dict[str, Any]) -> Optional[str]:
-    content_paths = [
-        "data.content",
-        "input",
-        "input_text",
-        "message",
-        "query",
-        "text",
-        "content",
-    ]
-
-    for path in content_paths:
-        try:
-            current: Any = kwargs
-            for part in path.split('.'):
-                current = current[part]
-            if isinstance(current, str) and current:
-                return current
-        except (KeyError, TypeError):
-            continue
-
-    return None

{fluxloop_cli-0.1.0 → fluxloop_cli-0.2.1}/fluxloop_cli/commands/init.py

@@ -11,13 +11,21 @@ from rich.prompt import Confirm
 from rich.tree import Tree
 
 from ..templates import (
-    create_experiment_config,
+    create_project_config,
+    create_input_config,
+    create_simulation_config,
+    create_evaluation_config,
     create_sample_agent,
     create_gitignore,
     create_env_file,
 )
 from ..project_paths import resolve_root_dir, resolve_project_dir
-from ..constants import DEFAULT_ROOT_DIR_NAME
+from ..constants import (
+    DEFAULT_ROOT_DIR_NAME,
+    CONFIG_DIRECTORY_NAME,
+    CONFIG_SECTION_FILENAMES,
+    CONFIG_SECTION_ORDER,
+)
 
 app = typer.Typer()
 console = Console()
@@ -51,7 +59,7 @@ def project(
     Initialize a new FluxLoop project.
     
     This command creates:
-    - setting.yaml: Default experiment configuration
+    - configs/: Separated configuration files (project/input/simulation/evaluation)
     - .env: Environment variables template
     - examples/: Sample agent code (optional)
     """
@@ -89,14 +97,20 @@ def project(
             raise typer.Abort()
     
     # Check for existing files
-    config_file = project_path / "setting.yaml"
+    config_dir = project_path / CONFIG_DIRECTORY_NAME
+    section_paths = {
+        key: config_dir / CONFIG_SECTION_FILENAMES[key]
+        for key in CONFIG_SECTION_FILENAMES
+    }
     env_file = project_path / ".env"
     gitignore_file = project_path / ".gitignore"
     
     if not force:
         existing_files = []
-        if config_file.exists():
-            existing_files.append("setting.yaml")
+        for key in CONFIG_SECTION_ORDER:
+            path = section_paths[key]
+            if path.exists():
+                existing_files.append(path.relative_to(project_path).as_posix())
         if env_file.exists():
             existing_files.append(".env")
         
@@ -107,10 +121,21 @@ def project(
             if not Confirm.ask("Overwrite existing files?", default=False):
                 raise typer.Abort()
     
-    # Create configuration file
-    console.print("📝 Creating experiment configuration...")
-    config_content = create_experiment_config(project_name)
-    config_file.write_text(config_content)
+    # Create configuration files
+    console.print("📝 Creating configuration files...")
+    config_dir.mkdir(exist_ok=True)
+
+    section_writers = {
+        "project": lambda: create_project_config(project_name),
+        "input": create_input_config,
+        "simulation": lambda: create_simulation_config(project_name),
+        "evaluation": create_evaluation_config,
+    }
+
+    for key in CONFIG_SECTION_ORDER:
+        content = section_writers[key]()  # type: ignore[operator]
+        section_path = section_paths[key]
+        section_path.write_text(content)
     
     # Ensure root .env exists, create project override template
     root_env_file = root_dir / ".env"
@@ -145,7 +170,9 @@ def project(
     console.print("\n[bold green]✓ Project initialized successfully![/bold green]\n")
     
     tree = Tree(f"[bold]{project_name}/[/bold]")
-    tree.add("📄 setting.yaml")
+    configs_node = tree.add(f"📁 {CONFIG_DIRECTORY_NAME}/")
+    for key in CONFIG_SECTION_ORDER:
+        configs_node.add(f"📄 {CONFIG_SECTION_FILENAMES[key]}")
     tree.add("🔐 .env")
     tree.add("📄 .gitignore")
     tree.add("📁 recordings/")
@@ -158,7 +185,7 @@ def project(
     
     # Show next steps
     console.print("\n[bold]Next steps:[/bold]")
-    console.print("1. Edit [cyan]setting.yaml[/cyan] to configure your experiment")
+    console.print("1. Review configs in [cyan]configs/[/cyan] (project/input/simulation/evaluation)")
     console.print("2. Set up environment variables in [cyan].env[/cyan]")
     if with_example:
         console.print("3. Try running: [green]fluxloop run experiment[/green]")

fluxloop_cli-0.2.1/fluxloop_cli/commands/record.py

@@ -0,0 +1,150 @@
+"""Record command for managing argument recording mode."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Dict, Optional
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from ..constants import DEFAULT_ROOT_DIR_NAME
+from ..project_paths import (
+    resolve_env_path,
+    resolve_project_dir,
+    resolve_config_section_path,
+)
+from ..config_schema import CONFIG_SECTION_FILENAMES
+
+
+app = typer.Typer()
+console = Console()
+
+
+def _load_env(env_path: Path) -> Dict[str, str]:
+    values: Dict[str, str] = {}
+    if env_path.exists():
+        for line in env_path.read_text().splitlines():
+            line = line.strip()
+            if not line or line.startswith("#") or "=" not in line:
+                continue
+            key, value = line.split("=", 1)
+            values[key.strip()] = value.strip()
+    return values
+
+
+def _write_env(env_path: Path, values: Dict[str, str]) -> None:
+    env_path.write_text("\n".join(f"{k}={v}" for k, v in values.items()) + "\n")
+
+
+def _update_simulation(recording_enabled: bool, project: Optional[str], root: Path) -> None:
+    try:
+        simulation_path = resolve_config_section_path("simulation", project, root)
+    except KeyError:
+        return
+
+    data: Dict[str, object] = {}
+    if simulation_path.exists():
+        import yaml
+
+        data = yaml.safe_load(simulation_path.read_text()) or {}
+
+    replay = data.setdefault("replay_args", {})
+    replay["enabled"] = recording_enabled
+
+    import yaml
+
+    simulation_path.parent.mkdir(parents=True, exist_ok=True)
+    simulation_path.write_text(
+        yaml.dump(data, sort_keys=False, default_flow_style=False),
+        encoding="utf-8",
+    )
+
+
+@app.command()
+def enable(
+    project: Optional[str] = typer.Option(None, "--project", help="Project name"),
+    root: Path = typer.Option(Path(DEFAULT_ROOT_DIR_NAME), "--root", help="FluxLoop root directory"),
+    recording_file: Path = typer.Option(
+        Path("recordings/args_recording.jsonl"),
+        "--file",
+        help="Recording file path relative to project",
+    ),
+):
+    """Enable recording mode by updating .env and simulation config."""
+
+    env_path = resolve_env_path(Path(".env"), project, root)
+    env_values = _load_env(env_path)
+    env_values["FLUXLOOP_RECORD_ARGS"] = "true"
+    env_values["FLUXLOOP_RECORDING_FILE"] = str(recording_file)
+
+    env_path.parent.mkdir(parents=True, exist_ok=True)
+    _write_env(env_path, env_values)
+
+    _update_simulation(True, project, root)
+
+    console.print(
+        f"[green]✓[/green] Recording enabled. Edit {env_path} or configs/simulation.yaml to adjust settings."
+    )
+
+
+@app.command()
+def disable(
+    project: Optional[str] = typer.Option(None, "--project", help="Project name"),
+    root: Path = typer.Option(Path(DEFAULT_ROOT_DIR_NAME), "--root", help="FluxLoop root directory"),
+):
+    """Disable recording mode and reset settings."""
+
+    env_path = resolve_env_path(Path(".env"), project, root)
+    env_values = _load_env(env_path)
+    env_values["FLUXLOOP_RECORD_ARGS"] = "false"
+    env_path.parent.mkdir(parents=True, exist_ok=True)
+    _write_env(env_path, env_values)
+
+    _update_simulation(False, project, root)
+
+    console.print("[green]✓[/green] Recording disabled.")
+
+
+@app.command()
+def status(
+    project: Optional[str] = typer.Option(None, "--project", help="Project name"),
+    root: Path = typer.Option(Path(DEFAULT_ROOT_DIR_NAME), "--root", help="FluxLoop root directory"),
+):
+    """Display current recording status."""
+
+    env_path = resolve_env_path(Path(".env"), project, root)
+    env_values = _load_env(env_path)
+    record_args = env_values.get("FLUXLOOP_RECORD_ARGS", "false").lower() == "true"
+    recording_file = env_values.get("FLUXLOOP_RECORDING_FILE", "(default)")
+
+    simulation_path = resolve_config_section_path("simulation", project, root)
+    simulation_exists = simulation_path.exists()
+
+    table = Table(title="Recording Status", show_header=False)
+    table.add_column("Property", style="cyan")
+    table.add_column("Value")
+
+    table.add_row("Mode", "ENABLED" if record_args else "disabled")
+    table.add_row("Recording File", recording_file)
+    table.add_row(".env Path", str(env_path))
+    table.add_row("Simulation Config", str(simulation_path) if simulation_exists else "(missing)")
+
+    console.print(table)
+
+    project_dir = (
+        resolve_project_dir(project, root)
+        if project
+        else Path.cwd()
+    )
+    recordings_dir = project_dir / Path(recording_file).parent
+
+    if recordings_dir.exists():
+        console.print(
+            f"\n[dim]Recording directory:[/dim] {recordings_dir}\n"
+        )
+    else:
+        console.print(
+            f"\n[yellow]Recording directory does not exist yet:[/yellow] {recordings_dir}\n"
+        )

{fluxloop_cli-0.1.0 → fluxloop_cli-0.2.1}/fluxloop_cli/commands/run.py

@@ -94,7 +94,11 @@ def experiment(
     console.print(f"📋 Loading configuration from: [cyan]{resolved_config}[/cyan]")
     
     try:
-        config = load_experiment_config(resolved_config)
+        config = load_experiment_config(
+            resolved_config,
+            project=project,
+            root=root,
+        )
     except Exception as e:
         console.print(f"[red]Error loading configuration:[/red] {e}")
         raise typer.Exit(1)

{fluxloop_cli-0.1.0 → fluxloop_cli-0.2.1}/fluxloop_cli/commands/status.py

@@ -11,7 +11,13 @@ from rich.table import Table
 from rich.panel import Panel
 
 from ..constants import DEFAULT_CONFIG_PATH, DEFAULT_ROOT_DIR_NAME
-from ..project_paths import resolve_config_path, resolve_root_dir, resolve_project_relative
+from ..config_schema import CONFIG_SECTION_FILENAMES, CONFIG_REQUIRED_KEYS
+from ..project_paths import (
+    resolve_config_path,
+    resolve_config_directory,
+    resolve_root_dir,
+    resolve_project_relative,
+)
 
 app = typer.Typer()
 console = Console()
@@ -90,17 +96,40 @@ def check(
     
     # Check for configuration file
     resolved_config = resolve_config_path(DEFAULT_CONFIG_PATH, project, root)
-    if resolved_config.exists():
+    config_dir = resolve_config_directory(project, root)
+
+    missing_sections = [
+        CONFIG_SECTION_FILENAMES[key]
+        for key in CONFIG_SECTION_FILENAMES
+        if not (config_dir / CONFIG_SECTION_FILENAMES[key]).exists()
+    ]
+
+    if resolved_config.exists() and not missing_sections:
         status_table.add_row(
             "Config",
             "[green]✓ Found[/green]",
-            str(resolved_config)
+            str(config_dir)
         )
     else:
+        detail_parts = []
+        if not config_dir.exists():
+            detail_parts.append(f"Missing directory: {config_dir}")
+        if missing_sections:
+            required_missing = [
+                name for name in missing_sections
+                if _section_key_from_filename(name) in CONFIG_REQUIRED_KEYS
+            ]
+            if required_missing:
+                detail_parts.append(
+                    "Missing sections: " + ", ".join(required_missing)
+                )
+        if not detail_parts:
+            detail_parts.append("Run: fluxloop init project")
+
         status_table.add_row(
             "Config",
-            "[yellow]- Not found[/yellow]",
-            "Run: fluxloop init project"
+            "[yellow]- Incomplete[/yellow]",
+            "\n".join(detail_parts)
         )
     
     # Check environment
@@ -133,6 +162,13 @@ def check(
             console.print(f"  • {rec}")
 
 
+def _section_key_from_filename(filename: str) -> Optional[str]:
+    for key, value in CONFIG_SECTION_FILENAMES.items():
+        if value == filename:
+            return key
+    return None
+
+
 @app.command()
 def experiments(
     output_dir: Path = typer.Option(