agent-learning-kit 0.1.0__py3-none-any.whl

fi/cli/storage/run_history.py

@@ -0,0 +1,249 @@
+"""Run history storage for tracking evaluation runs."""
+
+import json
+import uuid
+from dataclasses import asdict, dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from fi.evals.types import BatchRunResult
+
+
+@dataclass
+class RunRecord:
+    """Record of a single evaluation run."""
+
+    run_id: str
+    timestamp: str
+    config_file: Optional[str]
+    templates: List[str]
+    total_evaluations: int
+    successful: int
+    failed: int
+    pass_rate: Optional[float]
+    avg_score: Optional[float]
+    results_file: str
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "RunRecord":
+        """Create a RunRecord from a dictionary."""
+        return cls(**data)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary."""
+        return asdict(self)
+
+
+class RunHistory:
+    """Manages evaluation run history."""
+
+    def __init__(self, storage_dir: Optional[Path] = None):
+        """
+        Initialize run history storage.
+
+        Args:
+            storage_dir: Directory to store runs. Defaults to .fi/runs in cwd.
+        """
+        if storage_dir is None:
+            storage_dir = Path.cwd() / ".fi" / "runs"
+
+        self.storage_dir = Path(storage_dir)
+        self.history_file = self.storage_dir / "history.json"
+        self._ensure_storage_dir()
+
+    def _ensure_storage_dir(self) -> None:
+        """Create storage directory if it doesn't exist."""
+        self.storage_dir.mkdir(parents=True, exist_ok=True)
+
+        # Create .gitignore to exclude results
+        gitignore_path = self.storage_dir.parent / ".gitignore"
+        if not gitignore_path.exists():
+            gitignore_path.write_text("# Ignore evaluation runs\nruns/\n")
+
+    def _load_history(self) -> List[Dict[str, Any]]:
+        """Load run history from file."""
+        if not self.history_file.exists():
+            return []
+
+        try:
+            with open(self.history_file, "r") as f:
+                return json.load(f)
+        except (json.JSONDecodeError, IOError):
+            return []
+
+    def _save_history(self, history: List[Dict[str, Any]]) -> None:
+        """Save run history to file."""
+        with open(self.history_file, "w") as f:
+            json.dump(history, f, indent=2)
+
+    def save_run(
+        self,
+        results: BatchRunResult,
+        config_file: Optional[str] = None,
+        templates: Optional[List[str]] = None,
+    ) -> RunRecord:
+        """
+        Save an evaluation run to history.
+
+        Args:
+            results: BatchRunResult from the evaluation
+            config_file: Path to the config file used (if any)
+            templates: List of template names used
+
+        Returns:
+            RunRecord with the saved run info
+        """
+        run_id = self._generate_run_id()
+        timestamp = datetime.now().isoformat()
+
+        # Calculate statistics
+        total = len(results.eval_results)
+        successful = len([r for r in results.eval_results if r is not None])
+        failed = total - successful
+
+        # Calculate pass rate for boolean outputs
+        boolean_results = [
+            r for r in results.eval_results
+            if r is not None and isinstance(r.output, bool)
+        ]
+        passed = len([r for r in boolean_results if r.output is True])
+        pass_rate = (passed / len(boolean_results) * 100) if boolean_results else None
+
+        # Calculate average score for numeric outputs
+        score_results = [
+            r for r in results.eval_results
+            if r is not None and isinstance(r.output, (int, float))
+        ]
+        avg_score = (
+            sum(r.output for r in score_results) / len(score_results)
+            if score_results else None
+        )
+
+        # Get unique template names
+        if templates is None:
+            templates = list(set(
+                r.name for r in results.eval_results if r is not None
+            ))
+
+        # Save results to file
+        results_file = self.storage_dir / f"{run_id}.json"
+        results_data = {
+            "run_id": run_id,
+            "timestamp": timestamp,
+            "eval_results": [
+                {
+                    "name": r.name,
+                    "output": r.output,
+                    "reason": r.reason,
+                    "runtime": r.runtime,
+                    "output_type": r.output_type,
+                    "eval_id": r.eval_id,
+                }
+                for r in results.eval_results
+                if r is not None
+            ],
+        }
+
+        with open(results_file, "w") as f:
+            json.dump(results_data, f, indent=2)
+
+        # Create record
+        record = RunRecord(
+            run_id=run_id,
+            timestamp=timestamp,
+            config_file=config_file,
+            templates=templates,
+            total_evaluations=total,
+            successful=successful,
+            failed=failed,
+            pass_rate=pass_rate,
+            avg_score=avg_score,
+            results_file=str(results_file),
+        )
+
+        # Add to history
+        history = self._load_history()
+        history.insert(0, record.to_dict())  # Most recent first
+
+        # Keep only last 100 runs
+        history = history[:100]
+
+        self._save_history(history)
+
+        return record
+
+    def get_run(self, run_id: str) -> Optional[RunRecord]:
+        """Get a specific run by ID."""
+        history = self._load_history()
+        for entry in history:
+            if entry["run_id"] == run_id:
+                return RunRecord.from_dict(entry)
+        return None
+
+    def get_latest_run(self) -> Optional[RunRecord]:
+        """Get the most recent run."""
+        history = self._load_history()
+        if history:
+            return RunRecord.from_dict(history[0])
+        return None
+
+    def list_runs(self, limit: int = 10) -> List[RunRecord]:
+        """List recent runs."""
+        history = self._load_history()
+        return [RunRecord.from_dict(entry) for entry in history[:limit]]
+
+    def load_results(self, run_id: str) -> Optional[Dict[str, Any]]:
+        """Load full results for a run."""
+        record = self.get_run(run_id)
+        if record is None:
+            return None
+
+        results_file = Path(record.results_file)
+        if not results_file.exists():
+            return None
+
+        with open(results_file, "r") as f:
+            return json.load(f)
+
+    def delete_run(self, run_id: str) -> bool:
+        """Delete a run from history."""
+        history = self._load_history()
+
+        # Find and remove the run
+        new_history = [entry for entry in history if entry["run_id"] != run_id]
+
+        if len(new_history) == len(history):
+            return False  # Run not found
+
+        # Delete results file
+        for entry in history:
+            if entry["run_id"] == run_id:
+                results_file = Path(entry["results_file"])
+                if results_file.exists():
+                    results_file.unlink()
+                break
+
+        self._save_history(new_history)
+        return True
+
+    def clear_history(self) -> int:
+        """Clear all run history. Returns number of runs deleted."""
+        history = self._load_history()
+        count = len(history)
+
+        # Delete all results files
+        for entry in history:
+            results_file = Path(entry["results_file"])
+            if results_file.exists():
+                results_file.unlink()
+
+        self._save_history([])
+        return count
+
+    def _generate_run_id(self) -> str:
+        """Generate a unique run ID."""
+        # Use timestamp + short UUID for readability
+        timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+        short_uuid = str(uuid.uuid4())[:8]
+        return f"{timestamp}-{short_uuid}"

fi/cli/utils/__init__.py

@@ -0,0 +1,5 @@
+"""CLI Utilities Package"""
+
+from fi.cli.utils.console import console, print_error, print_success, print_warning
+
+__all__ = ["console", "print_error", "print_success", "print_warning"]

fi/cli/utils/console.py

@@ -0,0 +1,44 @@
+"""Console utilities for CLI output."""
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.text import Text
+
+# Global console instance
+console = Console()
+
+
+def print_error(message: str, title: str = "Error") -> None:
+    """Print an error message with red styling."""
+    console.print(Panel(
+        Text(message, style="red"),
+        title=f"[red]{title}[/red]",
+        border_style="red"
+    ))
+
+
+def print_success(message: str, title: str = "Success") -> None:
+    """Print a success message with green styling."""
+    console.print(Panel(
+        Text(message, style="green"),
+        title=f"[green]{title}[/green]",
+        border_style="green"
+    ))
+
+
+def print_warning(message: str, title: str = "Warning") -> None:
+    """Print a warning message with yellow styling."""
+    console.print(Panel(
+        Text(message, style="yellow"),
+        title=f"[yellow]{title}[/yellow]",
+        border_style="yellow"
+    ))
+
+
+def print_info(message: str, title: str = "Info") -> None:
+    """Print an info message with blue styling."""
+    console.print(Panel(
+        Text(message, style="blue"),
+        title=f"[blue]{title}[/blue]",
+        border_style="blue"
+    ))

fi/evals/__init__.py

@@ -0,0 +1,131 @@
+import inspect
+import warnings
+
+# Suppress Pydantic field-shadowing warnings from our models and the installed fi.api package
+warnings.filterwarnings("ignore", message='Field name "json" in .* shadows an attribute in parent')
+warnings.filterwarnings("ignore", message='Field name "schema" in .* shadows an attribute in parent')
+
+# ---------------------------------------------------------------------------
+# Unified evaluate() API (new)
+# ---------------------------------------------------------------------------
+from .core import evaluate, EvalResult, BatchResult, Turing  # noqa: E402, F401
+
+# ---------------------------------------------------------------------------
+# Execution handles (async eval / composite polling)
+# ---------------------------------------------------------------------------
+from .execution import Execution, ExecutionError  # noqa: E402, F401
+
+# ---------------------------------------------------------------------------
+# Cloud Evaluator + Protect (existing)
+# ---------------------------------------------------------------------------
+try:
+    from .evaluator import Evaluator, list_evaluations  # noqa: F401
+    from .manager import EvalTemplateManager  # noqa: F401
+    from .protect import Protect, protect  # noqa: F401
+    from .templates import *  # noqa: F403, F401
+    _evaluator_available = True
+except (ImportError, ModuleNotFoundError):
+    _evaluator_available = False
+    Evaluator = None
+    EvalTemplateManager = None
+    list_evaluations = None
+    Protect = None
+    protect = None
+
+# ---------------------------------------------------------------------------
+# Framework (evaluation orchestration + distributed backends)
+# ---------------------------------------------------------------------------
+try:
+    from .framework import (  # noqa: E402, F401
+        FrameworkEvaluator,
+        ExecutionMode,
+        blocking_evaluator,
+        async_evaluator,
+        distributed_evaluator,
+        resilient_evaluator,
+        register_current_span,
+        BaseEvaluation,
+        register_evaluation,
+        custom_eval,
+        simple_eval,
+        EvalBuilder,
+    )
+    _framework_available = True
+except (ImportError, ModuleNotFoundError):
+    _framework_available = False
+
+# ---------------------------------------------------------------------------
+# Streaming
+# ---------------------------------------------------------------------------
+from .streaming import (  # noqa: E402, F401
+    StreamingEvaluator,
+    StreamingConfig,
+    StreamingEvalResult,
+    ChunkResult,
+    EarlyStopPolicy,
+    EarlyStopReason,
+    StreamingState,
+)
+
+# ---------------------------------------------------------------------------
+# __all__
+# ---------------------------------------------------------------------------
+_globals = globals()
+evaluation_template_names = []
+if _evaluator_available:
+    evaluation_template_names = [
+        name
+        for name, obj in _globals.items()
+        if inspect.isclass(obj) and obj.__module__ == "fi.evals.templates"
+    ]
+
+# New unified API
+new_api_names = ["evaluate", "EvalResult", "BatchResult", "Turing"]
+
+# Execution handles
+execution_names = ["Execution", "ExecutionError"]
+
+# Existing clients
+client_names = [
+    "Evaluator",
+    "EvalTemplateManager",
+    "Protect",
+    "protect",
+    "list_evaluations",
+]
+
+# Framework exports
+framework_names = [
+    "FrameworkEvaluator",
+    "ExecutionMode",
+    "blocking_evaluator",
+    "async_evaluator",
+    "distributed_evaluator",
+    "resilient_evaluator",
+    "register_current_span",
+    "BaseEvaluation",
+    "register_evaluation",
+    "custom_eval",
+    "simple_eval",
+    "EvalBuilder",
+] if _framework_available else []
+
+# Streaming exports
+streaming_names = [
+    "StreamingEvaluator",
+    "StreamingConfig",
+    "StreamingEvalResult",
+    "ChunkResult",
+    "EarlyStopPolicy",
+    "EarlyStopReason",
+    "StreamingState",
+]
+
+__all__ = sorted(
+    new_api_names
+    + execution_names
+    + evaluation_template_names
+    + client_names
+    + framework_names
+    + streaming_names
+)

fi/evals/autoeval/__init__.py

@@ -0,0 +1,137 @@
+"""AutoEval - Automatic Evaluation Pipeline Builder.
+
+AutoEval automatically designs evaluation pipelines from natural language
+application descriptions. It analyzes the app type, risk level, and domain
+sensitivity to recommend appropriate evaluations and scanners.
+
+Example:
+    # From natural language description
+    from fi.evals.autoeval import AutoEvalPipeline
+
+    pipeline = AutoEvalPipeline.from_description(
+        "A RAG-based customer support chatbot for healthcare. "
+        "Retrieves patient records and answers questions about appointments."
+    )
+
+    result = pipeline.evaluate({
+        "query": "When is my appointment?",
+        "response": "Your appointment is Monday at 2pm.",
+        "context": "Patient has appointment on 2024-01-15 14:00",
+    })
+
+    print(result.passed)  # True/False
+    print(pipeline.explain())  # Detailed breakdown
+
+    # From pre-built template
+    pipeline = AutoEvalPipeline.from_template("rag_system")
+
+    # Export for version control
+    pipeline.export_yaml("eval_config.yaml")
+
+Available Templates:
+    - customer_support: Customer service chatbots
+    - rag_system: RAG-based document Q&A
+    - code_assistant: Code generation and review
+    - content_moderation: Content filtering and safety
+    - agent_workflow: Autonomous agents with tool use
+    - healthcare: Healthcare applications (HIPAA)
+    - financial: Financial services
+"""
+
+# Core types
+from .types import (
+    AppCategory,
+    RiskLevel,
+    DomainSensitivity,
+    AppRequirement,
+    AppAnalysis,
+    AutoEvalResult,
+)
+
+# Configuration
+from .config import (
+    EvalConfig,
+    ScannerConfig,
+    AutoEvalConfig,
+)
+
+# Main pipeline
+from .pipeline import (
+    AutoEvalPipeline,
+    register_eval_class,
+    register_scanner_class,
+)
+
+# Templates
+from .templates import (
+    get_template,
+    list_templates,
+    get_template_names,
+    TEMPLATES,
+)
+
+# Analysis components
+from .analyzer import AppAnalyzer
+from .recommender import EvalRecommender
+from .rules import RuleBasedAnalyzer
+
+# Export/Import
+from .export import (
+    export_yaml,
+    export_json,
+    load_yaml,
+    load_json,
+    load_config,
+    to_yaml_string,
+    to_json_string,
+    from_yaml_string,
+    from_json_string,
+)
+
+# Interactive mode
+from .interactive import (
+    InteractiveConfigurator,
+    InteractiveSession,
+    ClarificationQuestion,
+)
+
+__all__ = [
+    # Types
+    "AppCategory",
+    "RiskLevel",
+    "DomainSensitivity",
+    "AppRequirement",
+    "AppAnalysis",
+    "AutoEvalResult",
+    # Configuration
+    "EvalConfig",
+    "ScannerConfig",
+    "AutoEvalConfig",
+    # Pipeline
+    "AutoEvalPipeline",
+    "register_eval_class",
+    "register_scanner_class",
+    # Templates
+    "get_template",
+    "list_templates",
+    "get_template_names",
+    "TEMPLATES",
+    # Analyzers
+    "AppAnalyzer",
+    "EvalRecommender",
+    "RuleBasedAnalyzer",
+    # Export/Import
+    "export_yaml",
+    "export_json",
+    "load_yaml",
+    "load_json",
+    "load_config",
+    "to_yaml_string",
+    "to_json_string",
+    "from_yaml_string",
+    "from_json_string",
+    # Interactive
+    "InteractiveConfigurator",
+    "InteractiveSession",
+    "ClarificationQuestion",
+]