fluxloop-cli 0.1.0__py3-none-any.whl

fluxloop_cli/commands/status.py

@@ -0,0 +1,227 @@
+"""
+Status command for checking system and experiment status.
+"""
+
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.console import Console
+from rich.table import Table
+from rich.panel import Panel
+
+from ..constants import DEFAULT_CONFIG_PATH, DEFAULT_ROOT_DIR_NAME
+from ..project_paths import resolve_config_path, resolve_root_dir, resolve_project_relative
+
+app = typer.Typer()
+console = Console()
+
+
+@app.command()
+def check(
+    verbose: bool = typer.Option(
+        False,
+        "--verbose",
+        "-v",
+        help="Show detailed information",
+    ),
+    project: Optional[str] = typer.Option(
+        None,
+        "--project",
+        help="Project name under the FluxLoop root",
+    ),
+    root: Path = typer.Option(
+        Path(DEFAULT_ROOT_DIR_NAME),
+        "--root",
+        help="FluxLoop root directory",
+    ),
+):
+    """
+    Check FluxLoop system status.
+    
+    Verifies:
+    - SDK installation
+    - Collector connectivity
+    - Configuration validity
+    """
+    console.print("[bold]FluxLoop Status Check[/bold]\n")
+    
+    status_table = Table(show_header=False)
+    status_table.add_column("Component", style="cyan")
+    status_table.add_column("Status")
+    status_table.add_column("Details", style="dim")
+    
+    # Check SDK
+    try:
+        import fluxloop
+        sdk_version = fluxloop.__version__
+        status_table.add_row(
+            "SDK",
+            "[green]✓ Installed[/green]",
+            f"Version {sdk_version}"
+        )
+    except ImportError:
+        status_table.add_row(
+            "SDK",
+            "[red]✗ Not installed[/red]",
+            "Run: pip install fluxloop-sdk"
+        )
+    
+    # Check collector connectivity
+    try:
+        from fluxloop import get_config
+        from fluxloop.client import FluxLoopClient
+        
+        config = get_config()
+        client = FluxLoopClient()
+        
+        # Try to connect (this would need a health endpoint)
+        status_table.add_row(
+            "Collector",
+            "[yellow]? Unknown[/yellow]",
+            f"URL: {config.collector_url}"
+        )
+    except Exception as e:
+        status_table.add_row(
+            "Collector",
+            "[red]✗ Error[/red]",
+            str(e) if verbose else "Connection failed"
+        )
+    
+    # Check for configuration file
+    resolved_config = resolve_config_path(DEFAULT_CONFIG_PATH, project, root)
+    if resolved_config.exists():
+        status_table.add_row(
+            "Config",
+            "[green]✓ Found[/green]",
+            str(resolved_config)
+        )
+    else:
+        status_table.add_row(
+            "Config",
+            "[yellow]- Not found[/yellow]",
+            "Run: fluxloop init project"
+        )
+    
+    # Check environment
+    import os
+    if os.getenv("FLUXLOOP_API_KEY"):
+        status_table.add_row(
+            "API Key",
+            "[green]✓ Set[/green]",
+            "****" + os.getenv("FLUXLOOP_API_KEY")[-4:] if verbose else "Configured"
+        )
+    else:
+        status_table.add_row(
+            "API Key",
+            "[yellow]- Not set[/yellow]",
+            "Set FLUXLOOP_API_KEY in .env"
+        )
+    
+    console.print(status_table)
+    
+    # Show recommendations
+    recommendations = []
+    if not resolved_config.exists():
+        recommendations.append("Initialize a project: [cyan]fluxloop init project[/cyan]")
+    if not os.getenv("FLUXLOOP_API_KEY"):
+        recommendations.append("Set up API key in .env file")
+    
+    if recommendations:
+        console.print("\n[bold]Recommendations:[/bold]")
+        for rec in recommendations:
+            console.print(f"  • {rec}")
+
+
+@app.command()
+def experiments(
+    output_dir: Path = typer.Option(
+        Path("./experiments"),
+        "--output",
+        "-o",
+        help="Directory containing experiment results",
+    ),
+    project: Optional[str] = typer.Option(
+        None,
+        "--project",
+        help="Project name under the FluxLoop root",
+    ),
+    root: Path = typer.Option(
+        Path(DEFAULT_ROOT_DIR_NAME),
+        "--root",
+        help="FluxLoop root directory",
+    ),
+    limit: int = typer.Option(
+        10,
+        "--limit",
+        "-l",
+        help="Number of experiments to show",
+    ),
+):
+    """
+    List recent experiments and their results.
+    """
+    resolved_output = resolve_project_relative(output_dir, project, root)
+
+    if not resolved_output.exists():
+        console.print(f"[yellow]No experiments found in:[/yellow] {resolved_output}")
+        console.print("\nRun an experiment first: [cyan]fluxloop run experiment[/cyan]")
+        return
+    
+    # Find experiment directories
+    exp_dirs = sorted(
+        [d for d in resolved_output.iterdir() if d.is_dir()],
+        key=lambda x: x.stat().st_mtime,
+        reverse=True
+    )[:limit]
+    
+    if not exp_dirs:
+        console.print("[yellow]No experiments found[/yellow]")
+        return
+    
+    console.print(f"[bold]Recent Experiments[/bold] (showing {len(exp_dirs)} of {len(list(resolved_output.iterdir()))})\n")
+    
+    for exp_dir in exp_dirs:
+        # Try to load summary
+        summary_file = exp_dir / "summary.json"
+        if summary_file.exists():
+            import json
+            summary = json.loads(summary_file.read_text())
+            
+            # Create mini table for each experiment
+            exp_panel = Panel(
+                f"[cyan]Name:[/cyan] {summary.get('name', 'Unknown')}\n"
+                f"[cyan]Date:[/cyan] {summary.get('date', 'Unknown')}\n"
+                f"[cyan]Runs:[/cyan] {summary.get('total_runs', 0)}\n"
+                f"[cyan]Success Rate:[/cyan] {summary.get('success_rate', 0)*100:.1f}%\n"
+                f"[cyan]Avg Duration:[/cyan] {summary.get('avg_duration_ms', 0):.0f}ms",
+                title=f"[bold]{exp_dir.name}[/bold]",
+                border_style="blue",
+            )
+            console.print(exp_panel)
+        else:
+            console.print(f"📁 {exp_dir.name} [dim](no summary available)[/dim]")
+        
+        console.print()  # Add spacing
+
+
+@app.command()
+def traces(
+    experiment_id: Optional[str] = typer.Argument(
+        None,
+        help="Experiment ID to show traces for",
+    ),
+    limit: int = typer.Option(
+        10,
+        "--limit",
+        "-l",
+        help="Number of traces to show",
+    ),
+):
+    """
+    List recent traces from experiments.
+    """
+    # This would connect to the collector service
+    console.print("[yellow]Trace viewing requires collector service[/yellow]")
+    console.print("\nThis feature will be available when the collector is running.")
+    console.print("For now, check the experiment output directories for trace data.")

fluxloop_cli/config_loader.py

@@ -0,0 +1,159 @@
+"""
+Configuration loader for experiments.
+"""
+
+import sys
+from pathlib import Path
+from typing import Any, Dict, List
+
+import yaml
+from pydantic import ValidationError
+
+from .project_paths import resolve_config_path
+
+# Add shared schemas to path
+sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / "shared"))
+
+from fluxloop.schemas import ExperimentConfig
+
+
+def load_experiment_config(
+    config_file: Path,
+    *,
+    require_inputs_file: bool = True,
+) -> ExperimentConfig:
+    """
+    Load and validate experiment configuration from YAML file.
+    """
+    resolved_path = resolve_config_path(config_file, project=None, root=None)
+    if not resolved_path.exists():
+        raise FileNotFoundError(f"Configuration file not found: {config_file}")
+
+    # Load YAML
+    with open(resolved_path) as f:
+        data = yaml.safe_load(f)
+
+    if not data:
+        raise ValueError("Configuration file is empty")
+
+    # Validate and create config object
+    try:
+        config = ExperimentConfig(**data)
+        config.set_source_dir(resolved_path.parent)
+        resolved_input_count = _resolve_input_count(
+            config,
+            require_inputs_file=require_inputs_file,
+        )
+        config.set_resolved_input_count(resolved_input_count)
+    except ValidationError as e:
+        # Format validation errors nicely
+        errors = []
+        for error in e.errors():
+            loc = ".".join(str(x) for x in error["loc"])
+            msg = error["msg"]
+            errors.append(f"  - {loc}: {msg}")
+
+        raise ValueError(
+            f"Invalid configuration:\n" + "\n".join(errors)
+        )
+
+    return config
+
+
+def _resolve_input_count(
+    config: ExperimentConfig,
+    *,
+    require_inputs_file: bool = True,
+) -> int:
+    """Determine the effective number of inputs for this configuration."""
+    if config.inputs_file:
+        inputs_path = (config.get_source_dir() / Path(config.inputs_file)
+                       if config.get_source_dir() and not Path(config.inputs_file).is_absolute()
+                       else Path(config.inputs_file)).resolve()
+
+        if not inputs_path.exists():
+            if require_inputs_file:
+                raise FileNotFoundError(
+                    f"Inputs file not found when loading config: {inputs_path}"
+                )
+            return len(config.base_inputs)
+
+        with open(inputs_path, "r", encoding="utf-8") as f:
+            payload = yaml.safe_load(f)
+
+        if not payload:
+            if require_inputs_file:
+                raise ValueError(f"Inputs file is empty: {inputs_path}")
+            return len(config.base_inputs)
+
+        if isinstance(payload, dict):
+            entries = payload.get("inputs")
+            if entries is None:
+                if require_inputs_file:
+                    raise ValueError("Inputs file must contain an 'inputs' list when using mapping format")
+                return len(config.base_inputs)
+        elif isinstance(payload, list):
+            entries = payload
+        else:
+            raise ValueError("Inputs file must be either a list or a mapping with an 'inputs' key")
+
+        if not isinstance(entries, list):
+            if require_inputs_file:
+                raise ValueError("Inputs entries must be provided as a list")
+            return len(config.base_inputs)
+
+        return len(entries)
+
+    # No external file – rely on base_inputs multiplied by variation count
+    base_count = len(config.base_inputs)
+    variation_multiplier = max(1, config.variation_count)
+    return base_count * variation_multiplier if base_count else variation_multiplier
+
+
+def save_experiment_config(config: ExperimentConfig, config_file: Path) -> None:
+    """
+    Save experiment configuration to YAML file.
+    
+    Args:
+        config: ExperimentConfig object to save
+        config_file: Path to save configuration to
+    """
+    # Convert to dict and save
+    data = config.to_dict()
+    
+    with open(config_file, "w") as f:
+        yaml.dump(data, f, default_flow_style=False, sort_keys=False)
+
+
+def merge_config_overrides(
+    config: ExperimentConfig,
+    overrides: Dict[str, Any]
+) -> ExperimentConfig:
+    """
+    Merge override values into configuration.
+    
+    Args:
+        config: Base configuration
+        overrides: Dictionary of overrides (dot notation supported)
+        
+    Returns:
+        New configuration with overrides applied
+    """
+    # Convert config to dict
+    data = config.to_dict()
+    
+    # Apply overrides
+    for key, value in overrides.items():
+        # Support dot notation (e.g., "runner.timeout")
+        keys = key.split(".")
+        current = data
+        
+        for k in keys[:-1]:
+            if k not in current:
+                current[k] = {}
+            current = current[k]
+        
+        current[keys[-1]] = value
+    
+    # Create new config
+    return ExperimentConfig(**data)

fluxloop_cli/constants.py

@@ -0,0 +1,12 @@
+"""Shared constants for the FluxLoop CLI."""
+
+from pathlib import Path
+
+
+DEFAULT_ROOT_DIR_NAME = "fluxloop"
+DEFAULT_CONFIG_FILENAME = "setting.yaml"
+LEGACY_CONFIG_FILENAMES = ("fluxloop.yaml",)
+
+
+DEFAULT_CONFIG_PATH = Path(DEFAULT_CONFIG_FILENAME)
+

fluxloop_cli/input_generator.py

@@ -0,0 +1,158 @@
+"""Utilities for generating input datasets."""
+
+from __future__ import annotations
+
+import datetime as dt
+import json
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Sequence
+
+import yaml
+
+from fluxloop.schemas import (
+    ExperimentConfig,
+    InputGenerationMode,
+    PersonaConfig,
+    VariationStrategy,
+)
+
+from .llm_generator import DEFAULT_STRATEGIES, LLMGenerationError, generate_llm_inputs
+
+if TYPE_CHECKING:
+    from .llm_generator import LLMClient
+
+
+@dataclass
+class GenerationSettings:
+    """Options controlling input generation."""
+
+    limit: Optional[int] = None
+    dry_run: bool = False
+    mode: Optional[InputGenerationMode] = None
+    strategies: Optional[Sequence[VariationStrategy]] = None
+    use_cache: bool = True
+    llm_api_key_override: Optional[str] = None
+    llm_client: Optional["LLMClient"] = None
+
+
+@dataclass
+class GeneratedInput:
+    """Represents a single generated input entry."""
+
+    input: str
+    metadata: Dict[str, object] = field(default_factory=dict)
+
+
+@dataclass
+class GenerationResult:
+    """Container for generation output."""
+
+    entries: List[GeneratedInput]
+    metadata: Dict[str, object]
+
+    def to_yaml(self) -> str:
+        payload = {
+            "generated_at": dt.datetime.utcnow().isoformat() + "Z",
+            "metadata": self.metadata,
+            "inputs": [
+                {
+                    "input": entry.input,
+                    "metadata": entry.metadata,
+                }
+                for entry in self.entries
+            ],
+        }
+        return yaml.safe_dump(payload, sort_keys=False, allow_unicode=True)
+
+    def to_json(self) -> str:
+        return json.dumps(
+            {
+                "generated_at": dt.datetime.utcnow().isoformat() + "Z",
+                "metadata": self.metadata,
+                "inputs": [
+                    {
+                        "input": entry.input,
+                        "metadata": entry.metadata,
+                    }
+                    for entry in self.entries
+                ],
+            },
+            indent=2,
+        )
+
+
+class GenerationError(Exception):
+    """Raised when input generation cannot proceed."""
+
+
+def generate_inputs(
+    config: ExperimentConfig,
+    settings: GenerationSettings,
+    recording_template: Optional[Dict[str, Any]] = None,
+) -> GenerationResult:
+    """Generate deterministic input entries based on configuration."""
+    base_inputs = config.base_inputs
+    config_for_generation = config
+    if recording_template:
+        base_inputs = [
+            {
+                "input": recording_template["base_content"],
+                "metadata": {
+                    "source": "recording",
+                    "target": recording_template.get("target"),
+                },
+            }
+        ]
+        config_for_generation = config.model_copy(update={"base_inputs": base_inputs})
+
+    if not base_inputs:
+        raise GenerationError("base_inputs must be defined to generate inputs")
+
+    mode = settings.mode or config_for_generation.input_generation.mode
+
+    if mode == InputGenerationMode.LLM:
+        strategies: Sequence[VariationStrategy]
+        if settings.strategies and len(settings.strategies) > 0:
+            strategies = list(settings.strategies)
+        elif config_for_generation.variation_strategies:
+            strategies = config_for_generation.variation_strategies
+        else:
+            strategies = DEFAULT_STRATEGIES
+
+        try:
+            raw_entries = generate_llm_inputs(
+                config=config_for_generation,
+                strategies=strategies,
+                settings=settings,
+            )
+        except LLMGenerationError as exc:
+            raise GenerationError(str(exc)) from exc
+
+        entries = [
+            GeneratedInput(input=item["input"], metadata=item.get("metadata", {}))
+            for item in raw_entries
+        ]
+
+        metadata = {
+            "config_name": config_for_generation.name,
+            "total_base_inputs": len(base_inputs),
+            "total_personas": len(config_for_generation.personas),
+            "strategies": [strategy.value for strategy in strategies],
+            "limit": settings.limit,
+            "generation_mode": InputGenerationMode.LLM.value,
+            "llm_provider": config_for_generation.input_generation.llm.provider,
+            "llm_model": config_for_generation.input_generation.llm.model,
+        }
+
+        if recording_template:
+            for entry in entries:
+                entry.metadata["args_template"] = "use_recorded"
+                entry.metadata["template_kwargs"] = recording_template.get("full_kwargs")
+            metadata["recording_target"] = recording_template.get("target")
+            metadata["recording_base_input"] = recording_template.get("base_content")
+
+        return GenerationResult(entries=entries, metadata=metadata)
+
+    raise GenerationError(
+        "Only LLM-based generation is supported. Set input_generation.mode to 'llm'"
+    )