agent-learning-kit 0.1.0__py3-none-any.whl

fi/cli/commands/config.py

@@ -0,0 +1,165 @@
+"""Config command for managing CLI configuration."""
+
+import os
+from pathlib import Path
+from typing import Optional
+
+import typer
+
+from fi.cli.config.defaults import BASIC_TEMPLATE
+from fi.cli.utils.console import console, print_success, print_warning
+
+
+config_app = typer.Typer(
+    name="config",
+    help="Manage CLI configuration",
+    no_args_is_help=True,
+)
+
+
+@config_app.command("show")
+def show_config() -> None:
+    """Display current configuration settings."""
+    config_items = {
+        "FI_API_KEY": _mask_key(os.environ.get("FI_API_KEY")),
+        "FI_SECRET_KEY": _mask_key(os.environ.get("FI_SECRET_KEY")),
+        "FI_BASE_URL": os.environ.get("FI_BASE_URL", "https://api.futureagi.com"),
+        "FI_DEFAULT_MODEL": os.environ.get("FI_DEFAULT_MODEL", "gpt-4o"),
+    }
+
+    console.print("[bold]Current Configuration:[/bold]\n")
+    for key, value in config_items.items():
+        if value:
+            console.print(f"  [cyan]{key}[/cyan]: {value}")
+        else:
+            console.print(f"  [cyan]{key}[/cyan]: [dim]not set[/dim]")
+
+
+@config_app.command("get")
+def get_config(
+    key: str = typer.Argument(..., help="Configuration key to get"),
+) -> None:
+    """Get a specific configuration value."""
+    value = os.environ.get(key)
+    if value:
+        # Mask sensitive keys
+        if "KEY" in key.upper() or "SECRET" in key.upper():
+            console.print(_mask_key(value))
+        else:
+            console.print(value)
+    else:
+        print_warning(f"Configuration key not set: {key}")
+        raise typer.Exit(1)
+
+
+@config_app.command("set")
+def set_config(
+    key: str = typer.Argument(..., help="Configuration key to set"),
+    value: str = typer.Argument(..., help="Value to set"),
+    save: bool = typer.Option(
+        False,
+        "--save", "-s",
+        help="Save to .env file in current directory",
+    ),
+) -> None:
+    """
+    Set a configuration value.
+
+    Note: Without --save, the value is only set for the current session.
+    Use --save to persist the value to a .env file.
+    """
+    os.environ[key] = value
+
+    if save:
+        _save_to_env_file(key, value)
+        print_success(f"Saved {key} to .env file")
+    else:
+        print_success(f"Set {key} for current session")
+        console.print("[dim]Use --save to persist to .env file[/dim]")
+
+
+@config_app.command("init")
+def init_config(
+    path: Path = typer.Argument(
+        Path("."),
+        help="Directory to create config file in",
+    ),
+    force: bool = typer.Option(
+        False,
+        "--force", "-f",
+        help="Overwrite existing config file",
+    ),
+) -> None:
+    """Create a default configuration file."""
+    config_path = path / "fi-evaluation.yaml"
+
+    if config_path.exists() and not force:
+        print_warning(
+            f"Configuration file already exists: {config_path}\n"
+            "Use --force to overwrite."
+        )
+        raise typer.Exit(1)
+
+    with open(config_path, "w") as f:
+        f.write(BASIC_TEMPLATE)
+
+    print_success(f"Created configuration file: {config_path}")
+
+
+@config_app.command("env")
+def show_env_template() -> None:
+    """Show environment variable template."""
+    template = """# Future AGI Evaluation Configuration
+# Copy this to a .env file and fill in your values
+
+# Required: API credentials
+FI_API_KEY=your_api_key_here
+FI_SECRET_KEY=your_secret_key_here
+
+# Optional: Custom settings
+# FI_BASE_URL=https://api.futureagi.com
+# FI_DEFAULT_MODEL=gpt-4o
+
+# Optional: Langfuse integration
+# LANGFUSE_SECRET_KEY=your_langfuse_secret
+# LANGFUSE_PUBLIC_KEY=your_langfuse_public
+# LANGFUSE_HOST=https://cloud.langfuse.com
+"""
+    console.print(template)
+
+
+def _mask_key(value: Optional[str]) -> Optional[str]:
+    """Mask a sensitive key value."""
+    if not value:
+        return None
+    if len(value) <= 8:
+        return "*" * len(value)
+    return value[:4] + "*" * (len(value) - 8) + value[-4:]
+
+
+def _save_to_env_file(key: str, value: str) -> None:
+    """Save a key-value pair to .env file."""
+    env_path = Path(".env")
+
+    # Read existing content
+    existing_lines = []
+    if env_path.exists():
+        with open(env_path, "r") as f:
+            existing_lines = f.readlines()
+
+    # Update or add the key
+    key_found = False
+    new_lines = []
+    for line in existing_lines:
+        if line.startswith(f"{key}="):
+            new_lines.append(f"{key}={value}\n")
+            key_found = True
+        else:
+            new_lines.append(line)
+
+    if not key_found:
+        new_lines.append(f"{key}={value}\n")
+
+    # Write back
+    with open(env_path, "w") as f:
+        f.writelines(new_lines)

fi/cli/commands/export.py

@@ -0,0 +1,208 @@
+"""Export command for exporting evaluation results."""
+
+import json
+from pathlib import Path
+from typing import Optional
+
+import typer
+
+from fi.cli.storage import RunHistory
+from fi.cli.utils.console import print_error, print_success, print_warning
+
+
+def export(
+    run_id: Optional[str] = typer.Argument(
+        None,
+        help="Run ID to export (use 'fi view --list' to see available runs)",
+    ),
+    output: Path = typer.Option(
+        ...,
+        "--output", "-o",
+        help="Output file path",
+    ),
+    format: str = typer.Option(
+        "json",
+        "--format", "-f",
+        help="Export format: json, csv, html, junit",
+    ),
+    last: bool = typer.Option(
+        False,
+        "--last", "-l",
+        help="Export the most recent run",
+    ),
+) -> None:
+    """
+    Export evaluation results to a file.
+
+    Supported formats:
+        - json: JSON format with full details
+        - csv: CSV spreadsheet format
+        - html: HTML report for viewing in browser
+        - junit: JUnit XML format for CI/CD integration
+
+    Examples:
+        fi export --last -o results.json
+        fi export --last -o report.html -f html
+        fi export --last -o results.xml -f junit
+        fi export 20260123-143022-abc -o export.csv -f csv
+    """
+    history = RunHistory()
+
+    # Get the run to export
+    if last:
+        record = history.get_latest_run()
+        if record is None:
+            print_warning("No runs found. Run 'fi run' first to create evaluations.")
+            raise typer.Exit(1)
+    elif run_id:
+        record = history.get_run(run_id)
+        if record is None:
+            print_error(f"Run not found: {run_id}\nUse 'fi view --list' to see available runs.")
+            raise typer.Exit(1)
+    else:
+        print_error("Please specify a run ID or use --last to export the most recent run.")
+        raise typer.Exit(1)
+
+    # Load full results
+    results = history.load_results(record.run_id)
+    if results is None:
+        print_error(f"Results file not found for run: {record.run_id}")
+        raise typer.Exit(1)
+
+    # Export based on format
+    format = format.lower()
+    if format == "json":
+        _export_json(record, results, output)
+    elif format == "csv":
+        _export_csv(record, results, output)
+    elif format == "html":
+        _export_html(record, results, output)
+    elif format == "junit":
+        _export_junit(record, results, output)
+    else:
+        print_error(f"Unsupported format: {format}\nSupported: json, csv, html, junit")
+        raise typer.Exit(1)
+
+    print_success(f"Exported run {record.run_id} to {output}")
+
+
+def _export_json(record, results: dict, output: Path) -> None:
+    """Export results as JSON."""
+    export_data = {
+        "run_id": record.run_id,
+        "timestamp": record.timestamp,
+        "config_file": record.config_file,
+        "templates": record.templates,
+        "summary": {
+            "total": record.total_evaluations,
+            "successful": record.successful,
+            "failed": record.failed,
+            "pass_rate": record.pass_rate,
+            "avg_score": record.avg_score,
+        },
+        "eval_results": results.get("eval_results", []),
+    }
+
+    with open(output, "w") as f:
+        json.dump(export_data, f, indent=2)
+
+
+def _export_csv(record, results: dict, output: Path) -> None:
+    """Export results as CSV."""
+    import csv
+
+    with open(output, "w", newline="") as f:
+        writer = csv.writer(f)
+
+        # Header
+        writer.writerow([
+            "run_id",
+            "timestamp",
+            "template",
+            "output",
+            "reason",
+            "runtime",
+            "output_type",
+            "eval_id",
+        ])
+
+        # Data rows
+        for result in results.get("eval_results", []):
+            writer.writerow([
+                record.run_id,
+                record.timestamp,
+                result.get("name", ""),
+                result.get("output", ""),
+                result.get("reason", ""),
+                result.get("runtime", ""),
+                result.get("output_type", ""),
+                result.get("eval_id", ""),
+            ])
+
+
+def _export_html(record, results: dict, output: Path) -> None:
+    """Export results as HTML."""
+    from fi.cli.commands.view import _generate_html_report
+
+    html_content = _generate_html_report(record, results)
+
+    with open(output, "w") as f:
+        f.write(html_content)
+
+
+def _export_junit(record, results: dict, output: Path) -> None:
+    """Export results as JUnit XML for CI/CD integration."""
+    import xml.etree.ElementTree as ET
+    from xml.dom import minidom
+
+    # Create testsuites root
+    testsuites = ET.Element("testsuites")
+    testsuites.set("name", "AI Evaluation Results")
+    testsuites.set("tests", str(record.total_evaluations))
+    testsuites.set("failures", str(record.failed))
+    testsuites.set("time", "0")
+
+    # Create testsuite for this run
+    testsuite = ET.SubElement(testsuites, "testsuite")
+    testsuite.set("name", f"Run {record.run_id}")
+    testsuite.set("tests", str(record.total_evaluations))
+    testsuite.set("failures", str(record.failed))
+    testsuite.set("timestamp", record.timestamp)
+
+    # Add test cases
+    for result in results.get("eval_results", []):
+        testcase = ET.SubElement(testsuite, "testcase")
+        testcase.set("name", result.get("name", "Unknown"))
+        testcase.set("classname", "fi.evals")
+
+        runtime_ms = result.get("runtime", 0)
+        runtime_sec = runtime_ms / 1000 if runtime_ms else 0
+        testcase.set("time", f"{runtime_sec:.3f}")
+
+        result_output = result.get("output")
+        reason = result.get("reason", "")
+
+        # Check if this is a failure
+        is_failure = False
+        if isinstance(result_output, bool) and result_output is False:
+            is_failure = True
+        elif isinstance(result_output, (int, float)) and result_output < 0.5:
+            # Consider scores below 0.5 as failures for JUnit purposes
+            is_failure = True
+
+        if is_failure:
+            failure = ET.SubElement(testcase, "failure")
+            failure.set("message", f"Evaluation failed: {result_output}")
+            failure.text = reason
+
+    # Pretty print XML
+    xml_str = ET.tostring(testsuites, encoding="unicode")
+    dom = minidom.parseString(xml_str)
+    pretty_xml = dom.toprettyxml(indent="  ")
+
+    # Remove extra blank lines
+    lines = [line for line in pretty_xml.split("\n") if line.strip()]
+    pretty_xml = "\n".join(lines)
+
+    with open(output, "w") as f:
+        f.write(pretty_xml)

fi/cli/commands/init.py

@@ -0,0 +1,112 @@
+"""Init command for creating new evaluation projects."""
+
+from pathlib import Path
+from typing import Optional
+
+import typer
+
+from fi.cli.config.defaults import TEMPLATES, SAMPLE_DATA
+from fi.cli.utils.console import console, print_success, print_error, print_warning
+
+
+def init_project(
+    directory: Optional[Path] = typer.Argument(
+        None,
+        help="Directory to initialize (default: current directory)",
+    ),
+    template: str = typer.Option(
+        "basic",
+        "--template", "-t",
+        help="Template to use: basic, rag, safety, agent",
+    ),
+    force: bool = typer.Option(
+        False,
+        "--force", "-f",
+        help="Overwrite existing configuration file",
+    ),
+) -> None:
+    """
+    Initialize a new AI evaluation project.
+
+    Creates a fi-evaluation.yaml configuration file and sample data files.
+    """
+    # Determine target directory
+    if directory is None:
+        target_dir = Path.cwd()
+    else:
+        target_dir = Path(directory).resolve()
+
+    # Validate template
+    if template not in TEMPLATES:
+        print_error(
+            f"Unknown template: {template}\n"
+            f"Available templates: {', '.join(TEMPLATES.keys())}"
+        )
+        raise typer.Exit(1)
+
+    # Create directory if it doesn't exist
+    if not target_dir.exists():
+        target_dir.mkdir(parents=True)
+        console.print(f"[dim]Created directory: {target_dir}[/dim]")
+
+    # Check for existing config
+    config_path = target_dir / "fi-evaluation.yaml"
+    if config_path.exists() and not force:
+        print_warning(
+            f"Configuration file already exists: {config_path}\n"
+            "Use --force to overwrite."
+        )
+        raise typer.Exit(1)
+
+    # Create data directory
+    data_dir = target_dir / "data"
+    data_dir.mkdir(exist_ok=True)
+
+    # Create results directory
+    results_dir = target_dir / "results"
+    results_dir.mkdir(exist_ok=True)
+
+    # Write configuration file
+    config_content = TEMPLATES[template]
+    with open(config_path, "w") as f:
+        f.write(config_content)
+    console.print(f"[dim]Created: {config_path}[/dim]")
+
+    # Write sample data
+    sample_data = SAMPLE_DATA.get(template, SAMPLE_DATA["basic"])
+    data_filename = _get_data_filename(template)
+    data_path = data_dir / data_filename
+    with open(data_path, "w") as f:
+        f.write(sample_data)
+    console.print(f"[dim]Created: {data_path}[/dim]")
+
+    # Create .gitignore for results
+    gitignore_path = results_dir / ".gitignore"
+    with open(gitignore_path, "w") as f:
+        f.write("# Ignore evaluation results\n*\n!.gitignore\n")
+
+    # Success message
+    print_success(
+        f"Initialized evaluation project with '{template}' template!\n\n"
+        f"Project structure:\n"
+        f"  {target_dir}/\n"
+        f"  ├── fi-evaluation.yaml  # Configuration file\n"
+        f"  ├── data/\n"
+        f"  │   └── {data_filename}  # Sample test data\n"
+        f"  └── results/            # Evaluation results\n\n"
+        f"Next steps:\n"
+        f"  1. Edit fi-evaluation.yaml to configure your evaluations\n"
+        f"  2. Add your test data to the data/ directory\n"
+        f"  3. Run 'fi run' to execute evaluations"
+    )
+
+
+def _get_data_filename(template: str) -> str:
+    """Get the appropriate data filename for the template."""
+    filenames = {
+        "basic": "test_cases.json",
+        "rag": "rag_test_cases.json",
+        "safety": "safety_test_cases.json",
+        "agent": "agent_test_cases.json",
+    }
+    return filenames.get(template, "test_cases.json")

fi/cli/commands/list_cmd.py

@@ -0,0 +1,213 @@
+"""List command for displaying available resources."""
+
+import json
+from typing import Optional
+
+import typer
+from rich.table import Table
+
+from fi.cli.utils.console import console, print_error
+from fi.evals.templates import EvalTemplate
+
+
+# Template categories for filtering
+TEMPLATE_CATEGORIES = {
+    "conversation": [
+        "conversation_coherence",
+        "conversation_resolution",
+    ],
+    "rag": [
+        "context_adherence",
+        "context_relevance",
+        "completeness",
+        "chunk_attribution",
+        "chunk_utilization",
+        "groundedness",
+    ],
+    "safety": [
+        "content_moderation",
+        "pii",
+        "toxicity",
+        "prompt_injection",
+        "content_safety_violation",
+        "no_harmful_therapeutic_guidance",
+        "is_harmful_advice",
+    ],
+    "bias": [
+        "no_racial_bias",
+        "no_gender_bias",
+        "no_age_bias",
+        "bias_detection",
+        "sexist",
+    ],
+    "quality": [
+        "factual_accuracy",
+        "summary_quality",
+        "is_good_summary",
+        "is_factually_consistent",
+        "completeness",
+    ],
+    "format": [
+        "is_json",
+        "is_csv",
+        "is_code",
+        "is_email",
+        "one_line",
+        "contains_valid_link",
+        "no_valid_links",
+    ],
+    "tone": [
+        "tone",
+        "is_polite",
+        "is_concise",
+        "is_helpful",
+        "is_informal_tone",
+        "clinically_inappropriate_tone",
+    ],
+    "translation": [
+        "translation_accuracy",
+        "cultural_sensitivity",
+    ],
+    "audio": [
+        "audio_transcription",
+        "audio_quality",
+    ],
+    "function_calling": [
+        "llm_function_calling",
+        "evaluate_function_calling",
+    ],
+    "hallucination": [
+        "detect_hallucination_missing_info",
+        "caption_hallucination",
+    ],
+}
+
+
+def list_resources(
+    resource: str = typer.Argument(
+        "templates",
+        help="Resource to list: templates, categories",
+    ),
+    format: str = typer.Option(
+        "table",
+        "--format", "-f",
+        help="Output format: table, json",
+    ),
+    category: Optional[str] = typer.Option(
+        None,
+        "--category", "-c",
+        help="Filter templates by category",
+    ),
+) -> None:
+    """
+    List available evaluation resources.
+
+    Resources:
+        templates - List all available evaluation templates
+        categories - List template categories
+    """
+    if resource == "templates":
+        _list_templates(format, category)
+    elif resource == "categories":
+        _list_categories(format)
+    else:
+        print_error(f"Unknown resource: {resource}\nAvailable: templates, categories")
+        raise typer.Exit(1)
+
+
+def _list_templates(format: str, category: Optional[str] = None) -> None:
+    """List all available evaluation templates."""
+    # Get all template classes from templates module
+    from fi.evals import templates as templates_module
+
+    template_list = []
+    for name in dir(templates_module):
+        obj = getattr(templates_module, name)
+        if (
+            isinstance(obj, type)
+            and issubclass(obj, EvalTemplate)
+            and obj is not EvalTemplate
+            and hasattr(obj, "eval_name")
+        ):
+            eval_name = obj.eval_name
+            # Filter by category if specified
+            if category:
+                if category not in TEMPLATE_CATEGORIES:
+                    print_error(
+                        f"Unknown category: {category}\n"
+                        f"Available: {', '.join(TEMPLATE_CATEGORIES.keys())}"
+                    )
+                    raise typer.Exit(1)
+                if eval_name not in TEMPLATE_CATEGORIES[category]:
+                    continue
+
+            # Determine category for template
+            template_category = _get_template_category(eval_name)
+
+            template_list.append({
+                "name": eval_name,
+                "class": name,
+                "category": template_category,
+            })
+
+    # Sort by name
+    template_list.sort(key=lambda x: x["name"])
+
+    if format == "json":
+        console.print(json.dumps(template_list, indent=2))
+    else:
+        _print_templates_table(template_list)
+
+
+def _list_categories(format: str) -> None:
+    """List all template categories."""
+    categories = [
+        {"name": cat, "count": len(templates)}
+        for cat, templates in TEMPLATE_CATEGORIES.items()
+    ]
+
+    if format == "json":
+        console.print(json.dumps(categories, indent=2))
+    else:
+        table = Table(
+            title="Template Categories",
+            show_header=True,
+            header_style="bold cyan",
+        )
+        table.add_column("Category", style="green")
+        table.add_column("Templates", justify="right", style="blue")
+
+        for cat in categories:
+            table.add_row(cat["name"], str(cat["count"]))
+
+        console.print(table)
+
+
+def _print_templates_table(templates: list) -> None:
+    """Print templates as a Rich table."""
+    table = Table(
+        title=f"Available Evaluation Templates ({len(templates)} total)",
+        show_header=True,
+        header_style="bold cyan",
+    )
+
+    table.add_column("Template Name", style="green")
+    table.add_column("Category", style="blue")
+    table.add_column("Class", style="dim")
+
+    for template in templates:
+        table.add_row(
+            template["name"],
+            template["category"],
+            template["class"],
+        )
+
+    console.print(table)
+
+
+def _get_template_category(eval_name: str) -> str:
+    """Get the category for a template."""
+    for category, templates in TEMPLATE_CATEGORIES.items():
+        if eval_name in templates:
+            return category
+    return "other"