fluxloop-cli 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

fluxloop_cli/config_schema.py

@@ -0,0 +1,83 @@
+"""Configuration section definitions and merge strategy for FluxLoop CLI."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Iterable, Tuple
+
+from .constants import (
+    CONFIG_DIRECTORY_NAME,
+    CONFIG_SECTION_FILENAMES,
+    CONFIG_SECTION_ORDER,
+    DEFAULT_CONFIG_FILENAME,
+    LEGACY_CONFIG_FILENAMES,
+)
+
+
+@dataclass(frozen=True)
+class ConfigSection:
+    """Represents an individual configuration section file."""
+
+    key: str
+    filename: str
+    description: str
+    required: bool = False
+
+    def resolve_path(self, root: Path) -> Path:
+        """Return the canonical path to this section within a project root."""
+
+        return root / CONFIG_DIRECTORY_NAME / self.filename
+
+
+CONFIG_SECTIONS: Tuple[ConfigSection, ...] = (
+    ConfigSection(
+        key="project",
+        filename=CONFIG_SECTION_FILENAMES["project"],
+        description="Project metadata, collector configuration, and shared defaults.",
+        required=True,
+    ),
+    ConfigSection(
+        key="input",
+        filename=CONFIG_SECTION_FILENAMES["input"],
+        description="Personas, base inputs, and input generation settings.",
+    ),
+    ConfigSection(
+        key="simulation",
+        filename=CONFIG_SECTION_FILENAMES["simulation"],
+        description="Experiment execution parameters (runner, iterations, environments).",
+        required=True,
+    ),
+    ConfigSection(
+        key="evaluation",
+        filename=CONFIG_SECTION_FILENAMES["evaluation"],
+        description="Post-run evaluation pipelines and scoring strategies.",
+    ),
+)
+
+CONFIG_SECTION_BY_KEY: Dict[str, ConfigSection] = {
+    section.key: section for section in CONFIG_SECTIONS
+}
+
+CONFIG_REQUIRED_KEYS: Tuple[str, ...] = tuple(
+    section.key for section in CONFIG_SECTIONS if section.required
+)
+
+
+def iter_section_paths(root: Path) -> Iterable[Path]:
+    """Yield section paths in merge order for the given project root."""
+
+    for key in CONFIG_SECTION_ORDER:
+        section = CONFIG_SECTION_BY_KEY[key]
+        yield section.resolve_path(root)
+
+
+def is_legacy_config(filename: str) -> bool:
+    """Return True if filename refers to a legacy single-file configuration."""
+
+    return filename in (DEFAULT_CONFIG_FILENAME, *LEGACY_CONFIG_FILENAMES)
+
+
+MERGE_PRIORITY: Tuple[str, ...] = CONFIG_SECTION_ORDER
+"""Order in which configuration sections should be merged."""
+

fluxloop_cli/constants.py

@@ -10,3 +10,18 @@ LEGACY_CONFIG_FILENAMES = ("fluxloop.yaml",)
 
 DEFAULT_CONFIG_PATH = Path(DEFAULT_CONFIG_FILENAME)
 
+CONFIG_DIRECTORY_NAME = "configs"
+PROJECT_CONFIG_FILENAME = "project.yaml"
+INPUT_CONFIG_FILENAME = "input.yaml"
+SIMULATION_CONFIG_FILENAME = "simulation.yaml"
+EVALUATION_CONFIG_FILENAME = "evaluation.yaml"
+
+CONFIG_SECTION_FILENAMES = {
+    "project": PROJECT_CONFIG_FILENAME,
+    "input": INPUT_CONFIG_FILENAME,
+    "simulation": SIMULATION_CONFIG_FILENAME,
+    "evaluation": EVALUATION_CONFIG_FILENAME,
+}
+
+CONFIG_SECTION_ORDER = tuple(CONFIG_SECTION_FILENAMES.keys())
+

fluxloop_cli/project_paths.py

@@ -3,7 +3,13 @@
 from pathlib import Path
 from typing import Optional
 
-from .constants import DEFAULT_CONFIG_FILENAME, DEFAULT_CONFIG_PATH, DEFAULT_ROOT_DIR_NAME
+from .constants import (
+    DEFAULT_CONFIG_FILENAME,
+    DEFAULT_CONFIG_PATH,
+    DEFAULT_ROOT_DIR_NAME,
+    CONFIG_DIRECTORY_NAME,
+    CONFIG_SECTION_FILENAMES,
+)
 
 
 def _normalize_path(path: Path) -> Path:
@@ -43,16 +49,37 @@ def resolve_project_relative(path: Path, project: Optional[str], root: Optional[
 def resolve_config_path(config_file: Path, project: Optional[str], root: Optional[Path]) -> Path:
     """Resolve the path to a configuration file, honoring project/root settings."""
 
-    if project:
-        if config_file == DEFAULT_CONFIG_PATH:
-            target = Path(DEFAULT_CONFIG_FILENAME)
-        else:
-            target = config_file
-        return resolve_project_relative(target, project, root)
-
     if config_file.is_absolute():
         return _normalize_path(config_file)
 
+    base_dir = (
+        resolve_project_dir(project, root)
+        if project
+        else _normalize_path(Path.cwd())
+    )
+
+    config_dir = base_dir / CONFIG_DIRECTORY_NAME
+
+    # Default: prefer simulation config inside configs/
+    if config_file == DEFAULT_CONFIG_PATH:
+        simulation_candidate = config_dir / CONFIG_SECTION_FILENAMES["simulation"]
+        if simulation_candidate.exists() or config_dir.exists():
+            return _normalize_path(simulation_candidate)
+
+    # Explicit section filenames resolve relative to configs/
+    if config_file.name in CONFIG_SECTION_FILENAMES.values():
+        return _normalize_path(config_dir / config_file.name)
+
+    # Explicit configs/ directory
+    if config_file == Path(CONFIG_DIRECTORY_NAME):
+        return _normalize_path(config_dir)
+
+    if config_file.parent == Path(CONFIG_DIRECTORY_NAME):
+        return _normalize_path(config_dir / config_file.name)
+
+    if project:
+        return resolve_project_relative(config_file, project, root)
+
     return _normalize_path(Path.cwd() / config_file)
 
 
@@ -78,3 +105,28 @@ def resolve_env_path(env_file: Path, project: Optional[str], root: Optional[Path
 
     return _normalize_path(Path.cwd() / env_file)
 
+
+def resolve_config_directory(project: Optional[str], root: Optional[Path]) -> Path:
+    """Return the canonical configs/ directory for the given context."""
+
+    base_dir = (
+        resolve_project_dir(project, root)
+        if project
+        else _normalize_path(Path.cwd())
+    )
+    return _normalize_path(base_dir / CONFIG_DIRECTORY_NAME)
+
+
+def resolve_config_section_path(
+    section_key: str,
+    project: Optional[str],
+    root: Optional[Path],
+) -> Path:
+    """Return the path to a specific configuration section file."""
+
+    if section_key not in CONFIG_SECTION_FILENAMES:
+        raise KeyError(f"Unknown configuration section: {section_key}")
+
+    config_dir = resolve_config_directory(project, root)
+    return _normalize_path(config_dir / CONFIG_SECTION_FILENAMES[section_key])
+

fluxloop_cli/templates.py

@@ -2,127 +2,297 @@
 Templates for generating configuration and code files.
 """
 
+from textwrap import dedent
+
+
+def create_project_config(project_name: str) -> str:
+    """Create default project-level configuration content."""
+
+    return dedent(
+        f"""
+        # FluxLoop Project Configuration
+        name: {project_name}
+        description: AI agent simulation project
+        version: 1.0.0
+
+        collector_url: null
+        collector_api_key: null
+
+        tags:
+          - simulation
+          - testing
+
+        metadata:
+          team: development
+          environment: local
+        """
+    ).strip() + "\n"
+
+
+def create_input_config() -> str:
+    """Create default input configuration content."""
+
+    return dedent(
+        """
+        # FluxLoop Input Configuration
+        personas:
+          - name: novice_user
+            description: A user new to the system
+            characteristics:
+              - Asks basic questions
+              - May use incorrect terminology
+              - Needs detailed explanations
+            language: en
+            expertise_level: novice
+            goals:
+              - Understand system capabilities
+              - Complete basic tasks
+
+          - name: expert_user
+            description: An experienced power user
+            characteristics:
+              - Uses technical terminology
+              - Asks complex questions
+              - Expects efficient responses
+            language: en
+            expertise_level: expert
+            goals:
+              - Optimize workflows
+              - Access advanced features
+
+        base_inputs:
+          - input: "How do I get started?"
+            expected_intent: help
+          - input: "What can you do?"
+            expected_intent: capabilities
+          - input: "Show me an example"
+            expected_intent: demo
+
+        variation_strategies:
+          - type: rephrase
+          - type: verbose
+          - type: error_prone
+
+        variation_count: 2
+        variation_temperature: 0.7
+
+        inputs_file: inputs/generated.yaml
+
+        input_generation:
+          mode: llm
+          llm:
+            enabled: true
+            provider: openai
+            model: gpt-4o-mini
+            api_key: null
+        """
+    ).strip() + "\n"
+
+
+def create_simulation_config(project_name: str) -> str:
+    """Create default simulation configuration content."""
+
+    return dedent(
+        f"""
+        # FluxLoop Simulation Configuration
+        name: {project_name}_experiment
+        description: AI agent simulation experiment
+
+        iterations: 10
+        parallel_runs: 1
+        run_delay_seconds: 0
+        seed: 42
+
+        runner:
+          module_path: "examples.simple_agent"
+          function_name: "run"
+          target: "examples.simple_agent:run"
+          working_directory: .
+          timeout_seconds: 120
+          max_retries: 3
+
+        replay_args:
+          enabled: false
+          recording_file: recordings/args_recording.jsonl
+          override_param_path: data.content
+
+        output_directory: experiments
+        save_traces: true
+        save_aggregated_metrics: true
+        """
+    ).strip() + "\n"
+
+
+def create_evaluation_config() -> str:
+    """Create default evaluation configuration content."""
+
+    return dedent(
+        """
+        # FluxLoop Evaluation Configuration
+        evaluators:
+          - name: success_checker
+            type: rule_based
+            enabled: true
+            rules:
+              - check: output_not_empty
+                weight: 1.0
+
+          - name: response_quality
+            type: llm_judge
+            enabled: false
+            model: gpt-4o-mini
+            prompt_template: |
+              Rate the quality of this response on a scale of 1-10:
+              Input: {input}
+              Output: {output}
+
+              Consider: relevance, completeness, clarity
+              Score:
+        """
+    ).strip() + "\n"
 
-def create_experiment_config(project_name: str) -> str:
-    """Create a default experiment configuration."""
-    return f"""# FluxLoop Experiment Configuration
-name: {project_name}_experiment
-description: AI agent simulation experiment
-version: 1.0.0
-
-# Simulation settings
-iterations: 10
-parallel_runs: 1
-seed: 42
-run_delay_seconds: 0
-
-# User personas for simulation
-personas:
-  - name: novice_user
-    description: A user new to the system
-    characteristics:
-      - Asks basic questions
-      - May use incorrect terminology
-      - Needs detailed explanations
-    language: en
-    expertise_level: novice
-    goals:
-      - Understand how the system works
-      - Complete simple tasks
-
-  - name: expert_user
-    description: An experienced power user
-    characteristics:
-      - Uses technical terminology
-      - Asks complex questions
-      - Expects efficient responses
-    language: en
-    expertise_level: expert
-    goals:
-      - Optimize workflows
-      - Access advanced features
-
-# Base inputs for reference (generate inputs from these)
-base_inputs:
-  - input: "How do I get started?"
-    expected_intent: help
-  - input: "What can you do?"
-    expected_intent: capabilities
-  - input: "Show me an example"
-    expected_intent: demo
-
-# Agent runner configuration
-runner:
-  target: "examples.simple_agent:run"  # module:function or module:Class.method
-  working_directory: .  # IMPORTANT: Set this to your project's root directory
-  timeout_seconds: 30
-  max_retries: 3
-
-# Recorded argument replay (optional)
-replay_args:
-  enabled: false
-  recording_file: recordings/args_recording.jsonl
-  callable_providers:
-    send_message_callback: "builtin:collector.send"
-    send_error_callback: "builtin:collector.error"
-  override_param_path: data.content
-
-# Input generation configuration
-input_generation:
-  mode: llm
-  llm:
-    enabled: true
-    provider: openai
-    model: gpt-5
-  strategies:
-    - type: rephrase
-    - type: verbose
-    - type: error_prone
-  variation_count: 2
-
-# Evaluation methods
-evaluators:
-  - name: success_checker
-    type: rule_based
-    enabled: true
-    rules:
-      - check: output_not_empty
-        weight: 1.0
-
-  - name: response_quality
-    type: llm_judge
-    enabled: false
-    model: gpt-5
-    prompt_template: |
-      Rate the quality of this response on a scale of 1-10:
-      Input: {{input}}
-      Output: {{output}}
-      
-      Consider: relevance, completeness, clarity
-      Score:
-
-# Output configuration
-output_directory: experiments
-save_traces: true
-save_aggregated_metrics: true
-
-# Inputs must be generated before running experiments
-inputs_file: inputs/generated.yaml
-
-# Collector settings (optional)
-# collector_url: http://localhost:8000
-# collector_api_key: your-api-key
-
-# Tags and metadata
-tags:
-  - simulation
-  - testing
-metadata:
-  team: development
-  environment: local
-"""
 
+def create_sample_agent() -> str:
+    """Create a sample agent implementation."""
+
+    return dedent(
+        '''
+        """Sample agent implementation for FluxLoop testing."""
+
+        import random
+        import time
+        from typing import Any, Dict
+
+        import fluxloop
+
+
+        @fluxloop.agent(name="SimpleAgent")
+        def run(input_text: str) -> str:
+            """Main agent entry point."""
+            processed = process_input(input_text)
+            response = generate_response(processed)
+            time.sleep(random.uniform(0.1, 0.5))
+            return response
+
+
+        @fluxloop.prompt(model="simple-model")
+        def generate_response(processed_input: Dict[str, Any]) -> str:
+            intent = processed_input.get("intent", "unknown")
+            responses = {
+                "greeting": "Hello! How can I help you today?",
+                "help": "I can assist you with various tasks. What would you like to know?",
+                "capabilities": "I can answer questions, provide information, and help with tasks.",
+                "demo": "Here's an example: You can ask me about any topic and I'll try to help.",
+                "unknown": "I'm not sure I understand. Could you please rephrase?",
+            }
+            return responses.get(intent, responses["unknown"])
+
+
+        @fluxloop.tool(description="Process and analyze input text")
+        def process_input(text: str) -> Dict[str, Any]:
+            text_lower = text.lower()
+
+            intent = "unknown"
+            if any(word in text_lower for word in ["hello", "hi", "hey"]):
+                intent = "greeting"
+            elif any(word in text_lower for word in ["help", "start", "begin"]):
+                intent = "help"
+            elif any(word in text_lower for word in ["can you", "what can", "capabilities"]):
+                intent = "capabilities"
+            elif "example" in text_lower or "demo" in text_lower:
+                intent = "demo"
+
+            return {
+                "original": text,
+                "intent": intent,
+                "word_count": len(text.split()),
+                "has_question": "?" in text,
+            }
+
+
+        if __name__ == "__main__":
+            with fluxloop.instrument("test_run"):
+                result = run("Hello, what can you help me with?")
+                print(f"Result: {result}")
+        '''
+    ).strip() + "\n"
+
+
+def create_gitignore() -> str:
+    """Create a .gitignore file."""
+
+    return dedent(
+        """
+        # Python
+        __pycache__/
+        *.py[cod]
+        *$py.class
+        *.so
+        .Python
+        venv/
+        env/
+        ENV/
+        .venv/
+
+        # FluxLoop
+        traces/
+        *.trace
+        *.log
+
+        # Environment
+        .env
+        .env.local
+        *.env
+
+        # IDE
+        .vscode/
+        .idea/
+        *.swp
+        *.swo
+
+        # OS
+        .DS_Store
+        Thumbs.db
+
+        # Testing
+        .pytest_cache/
+        .coverage
+        htmlcov/
+        *.coverage
+        """
+    ).strip() + "\n"
+
+
+def create_env_file() -> str:
+    """Create a .env template file."""
 
+    return dedent(
+        """
+        # FluxLoop Configuration
+        FLUXLOOP_COLLECTOR_URL=http://localhost:8000
+        FLUXLOOP_API_KEY=your-api-key-here
+        FLUXLOOP_ENABLED=true
+        FLUXLOOP_DEBUG=false
+        FLUXLOOP_SAMPLE_RATE=1.0
+        # Argument Recording (global toggle)
+        FLUXLOOP_RECORD_ARGS=false
+        FLUXLOOP_RECORDING_FILE=recordings/args_recording.jsonl
+
+        # Service Configuration
+        FLUXLOOP_SERVICE_NAME=my-agent
+        FLUXLOOP_ENVIRONMENT=development
+
+        # LLM API Keys (if needed)
+        # OPENAI_API_KEY=sk-...
+        # ANTHROPIC_API_KEY=sk-ant-...
+
+        # Other Configuration
+        # Add your custom environment variables here
+        """
+    ).strip() + "\n"
+"""
+Templates for generating configuration and code files.
+"""
 def create_sample_agent() -> str:
     """Create a sample agent implementation."""
     return '''"""

{fluxloop_cli-0.1.0.dist-info → fluxloop_cli-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fluxloop-cli
-Version: 0.1.0
+Version: 0.2.0
 Summary: FluxLoop CLI for running agent simulations
 Author-email: FluxLoop Team <team@fluxloop.dev>
 License: Apache-2.0
@@ -40,47 +40,51 @@ Requires-Dist: anthropic>=0.7.0; extra == "anthropic"
 
 # FluxLoop CLI
 
-Command-line interface for running agent simulations and managing FluxLoop workflows.
+Command-line interface for running agent simulations.
 
 ## Installation
 
-```bash
+```
 pip install fluxloop-cli
 ```
 
-## Quick Start
-
-```bash
-# Initialize a new FluxLoop project
-fluxloop init
+## Configuration Overview (v0.2.0)
 
-# Run agent simulations
-fluxloop run
+FluxLoop CLI now stores experiment settings in four files under `configs/`:
 
-# Generate test inputs
-fluxloop generate
+- `configs/project.yaml` – project metadata, collector defaults
+- `configs/input.yaml` – personas, base inputs, input generation options
+- `configs/simulation.yaml` – runtime parameters (iterations, runner, replay args)
+- `configs/evaluation.yaml` – evaluator definitions (rule-based, LLM judge, etc.)
 
-# Check status
-fluxloop status
-```
+The legacy `setting.yaml` is still supported, but new projects created with
+`fluxloop init project` will generate the structured layout above.
 
-## Features
+## Key Commands
 
-- 🚀 **Easy Setup**: Initialize projects with a single command
-- 🔄 **Simulation Runner**: Execute agent tests with various input scenarios
-- 📝 **Input Generation**: LLM-powered test input generation
-- 📊 **Rich Output**: Beautiful terminal UI with detailed progress tracking
+- `fluxloop init project` – scaffold a new project (configs, `.env`, examples)
+- `fluxloop generate inputs` – produce input variations for the active project
+- `fluxloop run experiment` – execute an experiment using `configs/simulation.yaml`
+- `fluxloop parse experiment` – convert experiment outputs into readable artifacts
+- `fluxloop config set-llm` – update LLM provider/model in `configs/input.yaml`
+- `fluxloop record enable|disable|status` – toggle recording mode across `.env` and simulation config
 
-## Requirements
+Run `fluxloop --help` or `fluxloop <command> --help` for more detail.
 
-- Python 3.8 or higher
-- FluxLoop SDK
+## Developing
 
-## Documentation
+Install dependencies and run tests:
 
-For detailed documentation, visit [https://docs.fluxloop.dev](https://docs.fluxloop.dev)
+```
+python -m venv .venv
+source .venv/bin/activate
+pip install -e .[dev]
+pytest
+```
 
-## License
+To package the CLI:
 
-Apache License 2.0 - see LICENSE file for details
+```
+./build.sh
+```