agent-learning-kit 0.1.0__py3-none-any.whl

fi/cli/config/loader.py

@@ -0,0 +1,155 @@
+"""Configuration file loading and discovery."""
+
+from pathlib import Path
+from typing import Optional, Union
+
+import yaml
+from pydantic import ValidationError
+
+from fi.cli.config.schema import FIEvaluationConfig
+
+
+# Default config file names to look for
+CONFIG_FILE_NAMES = [
+    "fi-evaluation.yaml",
+    "fi-evaluation.yml",
+    ".fi-evaluation.yaml",
+    ".fi-evaluation.yml",
+]
+
+
+def find_config_file(start_path: Optional[Path] = None) -> Optional[Path]:
+    """
+    Find a configuration file by searching current directory and parents.
+
+    Args:
+        start_path: Starting directory for search (default: current working directory)
+
+    Returns:
+        Path to configuration file if found, None otherwise
+    """
+    if start_path is None:
+        start_path = Path.cwd()
+
+    current = start_path.resolve()
+
+    # Search up the directory tree
+    while current != current.parent:
+        for config_name in CONFIG_FILE_NAMES:
+            config_path = current / config_name
+            if config_path.exists():
+                return config_path
+        current = current.parent
+
+    # Check root as well
+    for config_name in CONFIG_FILE_NAMES:
+        config_path = current / config_name
+        if config_path.exists():
+            return config_path
+
+    return None
+
+
+def load_config(
+    config_path: Optional[Union[str, Path]] = None
+) -> FIEvaluationConfig:
+    """
+    Load and validate configuration from a YAML file.
+
+    Args:
+        config_path: Path to configuration file. If not provided,
+                    will search for config file automatically.
+
+    Returns:
+        Validated FIEvaluationConfig object
+
+    Raises:
+        FileNotFoundError: If config file not found
+        ValueError: If config file is invalid YAML
+        ValidationError: If config doesn't match schema
+    """
+    if config_path is None:
+        config_path = find_config_file()
+        if config_path is None:
+            raise FileNotFoundError(
+                "No configuration file found. "
+                "Create a fi-evaluation.yaml file or specify --config path."
+            )
+    else:
+        config_path = Path(config_path)
+
+    if not config_path.exists():
+        raise FileNotFoundError(f"Configuration file not found: {config_path}")
+
+    # Load YAML
+    try:
+        with open(config_path, "r") as f:
+            raw_config = yaml.safe_load(f)
+    except yaml.YAMLError as e:
+        raise ValueError(f"Invalid YAML in configuration file: {e}")
+
+    if raw_config is None:
+        raise ValueError("Configuration file is empty")
+
+    # Validate against schema
+    try:
+        config = FIEvaluationConfig(**raw_config)
+    except ValidationError as e:
+        raise ValueError(f"Configuration validation failed: {e}")
+
+    return config
+
+
+def load_test_data(data_path: Union[str, Path]) -> list:
+    """
+    Load test data from a JSON, JSONL, or CSV file.
+
+    Args:
+        data_path: Path to test data file
+
+    Returns:
+        List of test case dictionaries
+
+    Raises:
+        FileNotFoundError: If data file not found
+        ValueError: If data format is unsupported or invalid
+    """
+    import json
+    import csv
+
+    data_path = Path(data_path)
+
+    if not data_path.exists():
+        raise FileNotFoundError(f"Test data file not found: {data_path}")
+
+    suffix = data_path.suffix.lower()
+
+    if suffix == ".json":
+        with open(data_path, "r") as f:
+            data = json.load(f)
+            if isinstance(data, dict):
+                return [data]
+            return data
+
+    elif suffix == ".jsonl":
+        data = []
+        with open(data_path, "r") as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    data.append(json.loads(line))
+        return data
+
+    elif suffix == ".csv":
+        data = []
+        with open(data_path, "r", newline="") as f:
+            reader = csv.DictReader(f)
+            for row in reader:
+                data.append(dict(row))
+        return data
+
+    else:
+        raise ValueError(
+            f"Unsupported data format: {suffix}. "
+            "Supported formats: .json, .jsonl, .csv"
+        )

fi/cli/config/schema.py

@@ -0,0 +1,174 @@
+"""Configuration schema definitions for fi-evaluation.yaml."""
+
+from typing import List, Optional, Dict, Any
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+class APIConfig(BaseModel):
+    """API configuration settings."""
+    base_url: str = Field(
+        default="https://api.futureagi.com",
+        description="Base URL for the Future AGI API"
+    )
+
+
+class DefaultsConfig(BaseModel):
+    """Default settings for evaluations."""
+    model: str = Field(
+        default="gpt-4o",
+        description="Default model for LLM-as-judge evaluations"
+    )
+    timeout: int = Field(
+        default=200,
+        description="Default timeout in seconds"
+    )
+    parallel_workers: int = Field(
+        default=8,
+        description="Number of parallel workers for evaluation"
+    )
+
+
+class EvaluationConfig(BaseModel):
+    """Configuration for a single evaluation."""
+    name: str = Field(..., description="Name of this evaluation")
+    template: Optional[str] = Field(
+        default=None,
+        description="Single evaluation template to use"
+    )
+    templates: Optional[List[str]] = Field(
+        default=None,
+        description="List of evaluation templates to use"
+    )
+    data: str = Field(..., description="Path to test data file")
+    config: Optional[Dict[str, Any]] = Field(
+        default=None,
+        description="Additional configuration for the evaluation"
+    )
+
+    @field_validator("templates", "template")
+    @classmethod
+    def validate_template_presence(cls, v, info):
+        """Ensure at least one template specification method is used."""
+        return v
+
+
+class AssertionConfig(BaseModel):
+    """Configuration for evaluation assertions."""
+    template: Optional[str] = Field(
+        default=None,
+        description="Template to assert on (mutually exclusive with 'global')"
+    )
+    conditions: List[str] = Field(
+        default_factory=list,
+        description="List of assertion conditions (e.g., 'pass_rate >= 0.85')"
+    )
+    on_fail: str = Field(
+        default="error",
+        description="Action on failure: error, warn, or skip"
+    )
+    is_global: bool = Field(
+        default=False,
+        alias="global",
+        description="If true, assertion applies globally across all templates"
+    )
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    @field_validator("on_fail")
+    @classmethod
+    def validate_on_fail(cls, v):
+        """Validate on_fail value."""
+        valid_values = ["warn", "error", "skip"]
+        if v not in valid_values:
+            raise ValueError(f"on_fail must be one of: {valid_values}")
+        return v
+
+
+class ThresholdOverrides(BaseModel):
+    """Per-template threshold overrides."""
+    model_config = ConfigDict(extra="allow")
+
+
+class ThresholdsConfig(BaseModel):
+    """Threshold shortcuts for assertions."""
+    default_pass_rate: Optional[float] = Field(
+        default=None,
+        description="Default pass rate threshold for all templates"
+    )
+    fail_fast: bool = Field(
+        default=False,
+        description="Stop on first assertion failure"
+    )
+    overrides: Optional[Dict[str, float]] = Field(
+        default=None,
+        description="Per-template threshold overrides"
+    )
+
+
+class OutputConfig(BaseModel):
+    """Output configuration settings."""
+    format: str = Field(
+        default="json",
+        description="Output format: json, table, csv, html"
+    )
+    path: str = Field(
+        default="./results/",
+        description="Path to save results"
+    )
+    include_metadata: bool = Field(
+        default=True,
+        description="Include metadata in output"
+    )
+
+    @field_validator("format")
+    @classmethod
+    def validate_format(cls, v):
+        """Validate output format."""
+        valid_formats = ["json", "table", "csv", "html"]
+        if v not in valid_formats:
+            raise ValueError(f"format must be one of: {valid_formats}")
+        return v
+
+
+class FIEvaluationConfig(BaseModel):
+    """Root configuration schema for fi-evaluation.yaml."""
+    version: str = Field(
+        default="1.0",
+        description="Configuration file version"
+    )
+    api: Optional[APIConfig] = Field(
+        default=None,
+        description="API configuration"
+    )
+    defaults: Optional[DefaultsConfig] = Field(
+        default=None,
+        description="Default evaluation settings"
+    )
+    evaluations: List[EvaluationConfig] = Field(
+        ...,
+        description="List of evaluation configurations"
+    )
+    output: Optional[OutputConfig] = Field(
+        default=None,
+        description="Output configuration"
+    )
+    assertions: Optional[List[AssertionConfig]] = Field(
+        default=None,
+        description="Assertions to run on evaluation results"
+    )
+    thresholds: Optional[ThresholdsConfig] = Field(
+        default=None,
+        description="Threshold shortcuts for assertions"
+    )
+
+    def get_defaults(self) -> DefaultsConfig:
+        """Get defaults config, creating one if not present."""
+        return self.defaults or DefaultsConfig()
+
+    def get_output_config(self) -> OutputConfig:
+        """Get output config, creating one if not present."""
+        return self.output or OutputConfig()
+
+    def get_api_config(self) -> APIConfig:
+        """Get API config, creating one if not present."""
+        return self.api or APIConfig()

fi/cli/main.py

@@ -0,0 +1,78 @@
+"""
+AI Evaluation CLI
+
+Command-line interface for running LLM evaluations with 60+ templates.
+"""
+
+import typer
+from rich.console import Console
+
+from fi.cli.commands.init import init_project
+from fi.cli.commands.run import run
+from fi.cli.commands.list_cmd import list_resources
+from fi.cli.commands.validate import validate
+from fi.cli.commands.config import config_app
+from fi.cli.commands.view import view
+from fi.cli.commands.export import export
+
+# Create main app
+app = typer.Typer(
+    name="fi",
+    help="AI Evaluation CLI - Evaluate LLM outputs with 60+ templates",
+    no_args_is_help=True,
+    rich_markup_mode="rich",
+)
+
+console = Console()
+
+
+# Register commands
+app.command("init", help="Initialize a new evaluation project")(init_project)
+app.command("run", help="Run evaluations from config or CLI")(run)
+app.command("list", help="List available templates and resources")(list_resources)
+app.command("validate", help="Validate configuration file")(validate)
+app.command("view", help="View evaluation results from previous runs")(view)
+app.command("export", help="Export evaluation results to file")(export)
+app.add_typer(config_app, name="config", help="Manage CLI configuration")
+
+
+@app.callback(invoke_without_command=True)
+def main_callback(
+    ctx: typer.Context,
+    version: bool = typer.Option(
+        False,
+        "--version", "-v",
+        help="Show version and exit",
+    ),
+) -> None:
+    """
+    AI Evaluation CLI by Future AGI.
+
+    Evaluate LLM outputs with 60+ pre-built evaluation templates.
+
+    Quick Start:
+        fi init my-project      # Initialize project
+        fi run                  # Run evaluations
+        fi list templates       # List available templates
+    """
+    if version:
+        from importlib.metadata import version as get_version
+        try:
+            v = get_version("ai-evaluation")
+        except Exception:
+            v = "1.0.0"
+        console.print(f"ai-evaluation version {v}")
+        raise typer.Exit()
+
+    # If no command provided and not asking for version, show help
+    if ctx.invoked_subcommand is None and not version:
+        console.print(ctx.get_help())
+
+
+def main() -> None:
+    """Main entry point for the CLI."""
+    app()
+
+
+if __name__ == "__main__":
+    main()

fi/cli/output/__init__.py

@@ -0,0 +1,6 @@
+"""Output formatters and reporters for CLI results."""
+
+from fi.cli.output.formatters import format_results
+from fi.cli.output.reporters import ResultReporter
+
+__all__ = ["format_results", "ResultReporter"]

fi/cli/output/formatters.py

@@ -0,0 +1,106 @@
+"""Format evaluation results for CLI output."""
+
+import csv
+import io
+import json
+from typing import Optional
+
+from rich.console import Console
+from rich.table import Table
+
+
+def format_results(
+    batch_result,
+    fmt: str,
+    console: Console,
+    output_path: Optional[str] = None,
+) -> Optional[str]:
+    """Format BatchRunResult for display or file output.
+
+    Args:
+        batch_result: BatchRunResult with eval_results list.
+        fmt: Output format — "table", "json", "csv", or "html".
+        console: Rich Console for table rendering.
+        output_path: Optional file path to write output to.
+
+    Returns:
+        Formatted string for non-table formats, None for table format.
+    """
+    results = [r for r in batch_result.eval_results if r is not None]
+
+    if fmt == "table":
+        _print_table(results, console)
+        return None
+    elif fmt == "json":
+        text = _to_json(results)
+    elif fmt == "csv":
+        text = _to_csv(results)
+    elif fmt == "html":
+        text = _to_html(results)
+    else:
+        text = _to_json(results)
+
+    if output_path:
+        with open(output_path, "w") as f:
+            f.write(text)
+
+    return text
+
+
+def _print_table(results, console: Console) -> None:
+    table = Table(title="Evaluation Results", show_lines=True)
+    table.add_column("Metric", style="cyan", no_wrap=True)
+    table.add_column("Output", style="green")
+    table.add_column("Reason", style="dim")
+    table.add_column("Runtime (ms)", justify="right")
+
+    for r in results:
+        table.add_row(
+            r.name,
+            str(r.output) if r.output is not None else "-",
+            (r.reason or "-")[:80],
+            str(r.runtime),
+        )
+
+    console.print(table)
+
+
+def _to_json(results) -> str:
+    rows = []
+    for r in results:
+        rows.append({
+            "name": r.name,
+            "output": r.output,
+            "reason": r.reason,
+            "runtime": r.runtime,
+        })
+    return json.dumps(rows, indent=2, default=str)
+
+
+def _to_csv(results) -> str:
+    buf = io.StringIO()
+    writer = csv.writer(buf)
+    writer.writerow(["name", "output", "reason", "runtime"])
+    for r in results:
+        writer.writerow([r.name, r.output, r.reason, r.runtime])
+    return buf.getvalue()
+
+
+def _to_html(results) -> str:
+    rows_html = ""
+    for r in results:
+        rows_html += (
+            f"<tr><td>{r.name}</td><td>{r.output}</td>"
+            f"<td>{r.reason or ''}</td><td>{r.runtime}</td></tr>\n"
+        )
+    return f"""<!DOCTYPE html>
+<html><head><title>Evaluation Results</title>
+<style>
+  body {{ font-family: sans-serif; margin: 2rem; }}
+  table {{ border-collapse: collapse; width: 100%; }}
+  th, td {{ border: 1px solid #ddd; padding: 8px; text-align: left; }}
+  th {{ background: #f5f5f5; }}
+</style></head><body>
+<h1>Evaluation Results</h1>
+<table><tr><th>Metric</th><th>Output</th><th>Reason</th><th>Runtime (ms)</th></tr>
+{rows_html}</table></body></html>"""

fi/cli/output/reporters.py

@@ -0,0 +1,46 @@
+"""Summary reporters for CLI evaluation results."""
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+
+
+class ResultReporter:
+    """Print a summary panel after evaluation results are displayed."""
+
+    def __init__(self, console: Console):
+        self.console = console
+
+    def report_summary(self, batch_result) -> None:
+        """Print a summary of the batch evaluation results."""
+        results = [r for r in batch_result.eval_results if r is not None]
+        total = len(results)
+
+        if total == 0:
+            self.console.print("[yellow]No results to summarise.[/yellow]")
+            return
+
+        # Count numeric scores (output that can be interpreted as a number)
+        scores = []
+        for r in results:
+            try:
+                scores.append(float(r.output))
+            except (TypeError, ValueError):
+                pass
+
+        table = Table(show_header=False, box=None, padding=(0, 2))
+        table.add_column("label", style="bold")
+        table.add_column("value")
+
+        table.add_row("Total metrics", str(total))
+
+        if scores:
+            avg = sum(scores) / len(scores)
+            table.add_row("Avg score", f"{avg:.3f}")
+            table.add_row("Min score", f"{min(scores):.3f}")
+            table.add_row("Max score", f"{max(scores):.3f}")
+
+        total_runtime = sum(r.runtime for r in results)
+        table.add_row("Total runtime", f"{total_runtime} ms")
+
+        self.console.print(Panel(table, title="Summary", border_style="blue"))

fi/cli/storage/__init__.py

@@ -0,0 +1,5 @@
+"""Storage module for run history."""
+
+from fi.cli.storage.run_history import RunHistory, RunRecord
+
+__all__ = ["RunHistory", "RunRecord"]