agent-learning-kit 0.1.0__py3-none-any.whl

fi/evals/autoeval/analyzer.py

@@ -0,0 +1,211 @@
+"""Application analyzer for AutoEval.
+
+Analyzes application descriptions to understand evaluation needs.
+Supports both LLM-powered and rule-based analysis.
+"""
+
+import json
+import logging
+from typing import Optional, Any, Protocol, runtime_checkable
+
+from .types import (
+    AppAnalysis,
+    AppCategory,
+    RiskLevel,
+    DomainSensitivity,
+    AppRequirement,
+)
+from .rules import RuleBasedAnalyzer
+from .prompts import ANALYSIS_SYSTEM_PROMPT, ANALYSIS_USER_PROMPT
+
+logger = logging.getLogger(__name__)
+
+
+@runtime_checkable
+class LLMProvider(Protocol):
+    """Protocol for LLM providers."""
+
+    def complete(
+        self,
+        prompt: str,
+        system_prompt: Optional[str] = None,
+        **kwargs: Any,
+    ) -> str:
+        """Generate a completion for the given prompt."""
+        ...
+
+
+class AppAnalyzer:
+    """
+    Analyzes application descriptions to understand evaluation needs.
+
+    Supports both LLM-powered and rule-based analysis. When LLM analysis
+    fails, automatically falls back to rule-based analysis.
+
+    Example:
+        # With LLM provider
+        from fi.evals.llm import LiteLLMProvider
+        analyzer = AppAnalyzer(llm_provider=LiteLLMProvider())
+        analysis = analyzer.analyze(
+            "A customer support chatbot for a healthcare company. "
+            "It retrieves patient information and answers questions."
+        )
+
+        # Without LLM (rule-based only)
+        analyzer = AppAnalyzer(use_llm=False)
+        analysis = analyzer.analyze("A RAG-based document Q&A system.")
+    """
+
+    def __init__(
+        self,
+        llm_provider: Optional[Any] = None,
+        model: str = "gpt-4o-mini",
+        use_llm: bool = True,
+        temperature: float = 0.1,
+    ):
+        """
+        Initialize the analyzer.
+
+        Args:
+            llm_provider: LLM provider for intelligent analysis (optional)
+            model: Model to use for LLM analysis
+            use_llm: If False, always use rule-based fallback
+            temperature: Temperature for LLM generation (lower = more deterministic)
+        """
+        self.llm_provider = llm_provider
+        self.model = model
+        self.use_llm = use_llm and llm_provider is not None
+        self.temperature = temperature
+        self.rule_analyzer = RuleBasedAnalyzer()
+
+    def analyze(self, description: str) -> AppAnalysis:
+        """
+        Analyze an application description.
+
+        Args:
+            description: Natural language description of the application
+
+        Returns:
+            AppAnalysis with detected requirements and recommendations
+        """
+        if not description or not description.strip():
+            return AppAnalysis(
+                category=AppCategory.UNKNOWN,
+                risk_level=RiskLevel.MEDIUM,
+                domain_sensitivity=DomainSensitivity.GENERAL,
+                requirements=[],
+                detected_features=[],
+                confidence=0.0,
+                explanation="Empty description provided.",
+            )
+
+        if self.use_llm:
+            try:
+                return self._analyze_with_llm(description)
+            except Exception as e:
+                logger.warning(f"LLM analysis failed, falling back to rules: {e}")
+                return self._analyze_with_rules(description)
+        else:
+            return self._analyze_with_rules(description)
+
+    def _analyze_with_llm(self, description: str) -> AppAnalysis:
+        """Use LLM to analyze the description."""
+        if not self.llm_provider:
+            raise ValueError("LLM provider not configured")
+
+        prompt = ANALYSIS_USER_PROMPT.format(description=description)
+
+        # Try different ways to call the LLM provider
+        try:
+            if hasattr(self.llm_provider, "complete"):
+                response = self.llm_provider.complete(
+                    prompt=prompt,
+                    system_prompt=ANALYSIS_SYSTEM_PROMPT,
+                    model=self.model,
+                    temperature=self.temperature,
+                )
+            elif hasattr(self.llm_provider, "generate"):
+                response = self.llm_provider.generate(
+                    prompt=prompt,
+                    system_prompt=ANALYSIS_SYSTEM_PROMPT,
+                    model=self.model,
+                    temperature=self.temperature,
+                )
+            elif callable(self.llm_provider):
+                response = self.llm_provider(
+                    prompt=prompt,
+                    system_prompt=ANALYSIS_SYSTEM_PROMPT,
+                )
+            else:
+                raise ValueError(f"Unknown LLM provider type: {type(self.llm_provider)}")
+        except TypeError as e:
+            # Only retry with simpler args if it's a signature mismatch
+            if "unexpected keyword argument" in str(e) or "positional argument" in str(e):
+                if hasattr(self.llm_provider, "complete"):
+                    response = self.llm_provider.complete(prompt)
+                else:
+                    raise
+            else:
+                raise
+
+        # Parse JSON response
+        return self._parse_llm_response(response, description)
+
+    def _parse_llm_response(self, response: str, description: str) -> AppAnalysis:
+        """Parse LLM response into AppAnalysis."""
+        # Clean up response - remove markdown code blocks if present
+        response = response.strip()
+        if response.startswith("```"):
+            lines = response.split("\n")
+            response = "\n".join(lines[1:-1] if lines[-1] == "```" else lines[1:])
+
+        try:
+            data = json.loads(response)
+        except json.JSONDecodeError as e:
+            logger.warning(f"Failed to parse LLM response as JSON: {e}")
+            # Fall back to rule-based
+            return self._analyze_with_rules(description)
+
+        # Parse with defaults
+        try:
+            category = AppCategory(data.get("category", "unknown"))
+        except ValueError:
+            category = AppCategory.UNKNOWN
+
+        try:
+            risk_level = RiskLevel(data.get("risk_level", "medium"))
+        except ValueError:
+            risk_level = RiskLevel.MEDIUM
+
+        try:
+            domain_sensitivity = DomainSensitivity(
+                data.get("domain_sensitivity", "general")
+            )
+        except ValueError:
+            domain_sensitivity = DomainSensitivity.GENERAL
+
+        requirements = []
+        for req_data in data.get("requirements", []):
+            requirements.append(
+                AppRequirement(
+                    category=req_data.get("category", ""),
+                    importance=req_data.get("importance", "recommended"),
+                    reason=req_data.get("reason", ""),
+                    suggested_evals=req_data.get("suggested_evals", []),
+                    suggested_scanners=req_data.get("suggested_scanners", []),
+                )
+            )
+
+        return AppAnalysis(
+            category=category,
+            risk_level=risk_level,
+            domain_sensitivity=domain_sensitivity,
+            requirements=requirements,
+            detected_features=data.get("detected_features", []),
+            confidence=0.85,  # LLM analysis has higher confidence
+            explanation=data.get("explanation", "Analysis performed by LLM."),
+        )
+
+    def _analyze_with_rules(self, description: str) -> AppAnalysis:
+        """Use rule-based analysis."""
+        return self.rule_analyzer.analyze(description)

fi/evals/autoeval/config.py

@@ -0,0 +1,244 @@
+"""Configuration classes for AutoEval."""
+
+from dataclasses import dataclass, field
+from typing import List, Dict, Any, Optional
+from copy import deepcopy
+
+
+@dataclass
+class EvalConfig:
+    """Configuration for a single evaluation."""
+
+    name: str
+    enabled: bool = True
+    threshold: float = 0.7
+    weight: float = 1.0
+    params: Dict[str, Any] = field(default_factory=dict)
+    model: Optional[str] = None  # LLM model for augmented metrics
+    augment: bool = False  # Enable local→LLM augmentation
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary."""
+        result = {
+            "name": self.name,
+            "enabled": self.enabled,
+            "threshold": self.threshold,
+            "weight": self.weight,
+        }
+        if self.params:
+            result["params"] = self.params
+        if self.model:
+            result["model"] = self.model
+        if self.augment:
+            result["augment"] = self.augment
+        return result
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "EvalConfig":
+        """Create from dictionary."""
+        return cls(
+            name=data["name"],
+            enabled=data.get("enabled", True),
+            threshold=data.get("threshold", 0.7),
+            weight=data.get("weight", 1.0),
+            params=data.get("params", {}),
+            model=data.get("model"),
+            augment=data.get("augment", False),
+        )
+
+    def copy(self) -> "EvalConfig":
+        """Create a deep copy."""
+        return EvalConfig(
+            name=self.name,
+            enabled=self.enabled,
+            threshold=self.threshold,
+            weight=self.weight,
+            params=deepcopy(self.params),
+            model=self.model,
+            augment=self.augment,
+        )
+
+
+@dataclass
+class ScannerConfig:
+    """Configuration for a single scanner."""
+
+    name: str
+    enabled: bool = True
+    threshold: float = 0.7
+    action: str = "block"  # block, flag, warn, redact
+    params: Dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary."""
+        result = {
+            "name": self.name,
+            "enabled": self.enabled,
+            "threshold": self.threshold,
+            "action": self.action,
+        }
+        if self.params:
+            result["params"] = self.params
+        return result
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "ScannerConfig":
+        """Create from dictionary."""
+        return cls(
+            name=data["name"],
+            enabled=data.get("enabled", True),
+            threshold=data.get("threshold", 0.7),
+            action=data.get("action", "block"),
+            params=data.get("params", {}),
+        )
+
+    def copy(self) -> "ScannerConfig":
+        """Create a deep copy."""
+        return ScannerConfig(
+            name=self.name,
+            enabled=self.enabled,
+            threshold=self.threshold,
+            action=self.action,
+            params=deepcopy(self.params),
+        )
+
+
+@dataclass
+class AutoEvalConfig:
+    """Complete configuration for an auto-generated evaluation pipeline."""
+
+    name: str
+    description: str = ""
+    version: str = "1.0.0"
+
+    # Analysis metadata
+    app_category: str = "unknown"
+    risk_level: str = "medium"
+    domain_sensitivity: str = "general"
+
+    # Evaluations
+    evaluations: List[EvalConfig] = field(default_factory=list)
+
+    # Scanners
+    scanners: List[ScannerConfig] = field(default_factory=list)
+
+    # Execution settings
+    execution_mode: str = "non_blocking"  # blocking, non_blocking, distributed
+    parallel_workers: int = 4
+    timeout_seconds: int = 30
+    fail_fast: bool = False
+
+    # Thresholds
+    global_pass_rate: float = 0.8
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for export."""
+        return {
+            "version": self.version,
+            "name": self.name,
+            "description": self.description,
+            "metadata": {
+                "app_category": self.app_category,
+                "risk_level": self.risk_level,
+                "domain_sensitivity": self.domain_sensitivity,
+                "generated_by": "autoeval",
+            },
+            "execution": {
+                "mode": self.execution_mode,
+                "parallel_workers": self.parallel_workers,
+                "timeout_seconds": self.timeout_seconds,
+                "fail_fast": self.fail_fast,
+            },
+            "thresholds": {
+                "global_pass_rate": self.global_pass_rate,
+            },
+            "evaluations": [e.to_dict() for e in self.evaluations],
+            "scanners": [s.to_dict() for s in self.scanners],
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "AutoEvalConfig":
+        """Create from dictionary (YAML import)."""
+        metadata = data.get("metadata", {})
+        execution = data.get("execution", {})
+        thresholds = data.get("thresholds", {})
+
+        return cls(
+            name=data.get("name", "unnamed"),
+            description=data.get("description", ""),
+            version=data.get("version", "1.0.0"),
+            app_category=metadata.get("app_category", "unknown"),
+            risk_level=metadata.get("risk_level", "medium"),
+            domain_sensitivity=metadata.get("domain_sensitivity", "general"),
+            evaluations=[
+                EvalConfig.from_dict(e) for e in data.get("evaluations", [])
+            ],
+            scanners=[
+                ScannerConfig.from_dict(s) for s in data.get("scanners", [])
+            ],
+            execution_mode=execution.get("mode", "non_blocking"),
+            parallel_workers=execution.get("parallel_workers", 4),
+            timeout_seconds=execution.get("timeout_seconds", 30),
+            fail_fast=execution.get("fail_fast", False),
+            global_pass_rate=thresholds.get("global_pass_rate", 0.8),
+        )
+
+    def copy(self) -> "AutoEvalConfig":
+        """Create a deep copy of the configuration."""
+        return AutoEvalConfig(
+            name=self.name,
+            description=self.description,
+            version=self.version,
+            app_category=self.app_category,
+            risk_level=self.risk_level,
+            domain_sensitivity=self.domain_sensitivity,
+            evaluations=[e.copy() for e in self.evaluations],
+            scanners=[s.copy() for s in self.scanners],
+            execution_mode=self.execution_mode,
+            parallel_workers=self.parallel_workers,
+            timeout_seconds=self.timeout_seconds,
+            fail_fast=self.fail_fast,
+            global_pass_rate=self.global_pass_rate,
+        )
+
+    def get_eval(self, name: str) -> Optional[EvalConfig]:
+        """Get evaluation config by name."""
+        for e in self.evaluations:
+            if e.name == name:
+                return e
+        return None
+
+    def get_scanner(self, name: str) -> Optional[ScannerConfig]:
+        """Get scanner config by name."""
+        for s in self.scanners:
+            if s.name == name:
+                return s
+        return None
+
+    def summary(self) -> str:
+        """Get a summary of the configuration."""
+        lines = [
+            f"AutoEval Config: {self.name}",
+            f"  Category: {self.app_category}",
+            f"  Risk Level: {self.risk_level}",
+            f"  Domain: {self.domain_sensitivity}",
+            "",
+            f"Evaluations ({len(self.evaluations)}):",
+        ]
+
+        for e in self.evaluations:
+            status = "enabled" if e.enabled else "disabled"
+            lines.append(f"  - {e.name} (threshold: {e.threshold}, {status})")
+
+        lines.append("")
+        lines.append(f"Scanners ({len(self.scanners)}):")
+
+        for s in self.scanners:
+            status = "enabled" if s.enabled else "disabled"
+            lines.append(f"  - {s.name} (action: {s.action}, {status})")
+
+        lines.append("")
+        lines.append(f"Execution: {self.execution_mode}, workers: {self.parallel_workers}")
+        lines.append(f"Pass Rate Threshold: {self.global_pass_rate:.0%}")
+
+        return "\n".join(lines)

fi/evals/autoeval/export.py

@@ -0,0 +1,213 @@
+"""Export and import functionality for AutoEval configurations.
+
+Supports YAML and JSON formats for version-controlled configurations.
+"""
+
+import json
+from pathlib import Path
+from typing import Union
+
+from .config import AutoEvalConfig
+
+
+def export_yaml(config: AutoEvalConfig, path: Union[str, Path]) -> None:
+    """
+    Export configuration to YAML file.
+
+    Args:
+        config: AutoEvalConfig to export
+        path: Path to write YAML file
+
+    Example:
+        export_yaml(config, "eval_config.yaml")
+    """
+    try:
+        import yaml
+    except ImportError:
+        raise ImportError("PyYAML is required for YAML export. Install with: pip install pyyaml")
+
+    data = config.to_dict()
+    path = Path(path)
+
+    with open(path, "w") as f:
+        yaml.dump(data, f, default_flow_style=False, sort_keys=False, allow_unicode=True)
+
+
+def export_json(config: AutoEvalConfig, path: Union[str, Path], indent: int = 2) -> None:
+    """
+    Export configuration to JSON file.
+
+    Args:
+        config: AutoEvalConfig to export
+        path: Path to write JSON file
+        indent: Indentation level for pretty printing
+
+    Example:
+        export_json(config, "eval_config.json")
+    """
+    data = config.to_dict()
+    path = Path(path)
+
+    with open(path, "w") as f:
+        json.dump(data, f, indent=indent)
+
+
+def load_yaml(path: Union[str, Path]) -> AutoEvalConfig:
+    """
+    Load configuration from YAML file.
+
+    Args:
+        path: Path to YAML configuration file
+
+    Returns:
+        AutoEvalConfig instance
+
+    Example:
+        config = load_yaml("eval_config.yaml")
+    """
+    try:
+        import yaml
+    except ImportError:
+        raise ImportError("PyYAML is required for YAML import. Install with: pip install pyyaml")
+
+    path = Path(path)
+
+    with open(path, "r") as f:
+        data = yaml.safe_load(f)
+
+    return AutoEvalConfig.from_dict(data)
+
+
+def load_json(path: Union[str, Path]) -> AutoEvalConfig:
+    """
+    Load configuration from JSON file.
+
+    Args:
+        path: Path to JSON configuration file
+
+    Returns:
+        AutoEvalConfig instance
+
+    Example:
+        config = load_json("eval_config.json")
+    """
+    path = Path(path)
+
+    with open(path, "r") as f:
+        data = json.load(f)
+
+    return AutoEvalConfig.from_dict(data)
+
+
+def load_config(path: Union[str, Path]) -> AutoEvalConfig:
+    """
+    Load configuration from file (auto-detect format).
+
+    Determines format from file extension (.yaml, .yml, .json).
+
+    Args:
+        path: Path to configuration file
+
+    Returns:
+        AutoEvalConfig instance
+
+    Raises:
+        ValueError: If file extension not recognized
+
+    Example:
+        config = load_config("eval_config.yaml")
+        config = load_config("eval_config.json")
+    """
+    path = Path(path)
+    suffix = path.suffix.lower()
+
+    if suffix in {".yaml", ".yml"}:
+        return load_yaml(path)
+    elif suffix == ".json":
+        return load_json(path)
+    else:
+        raise ValueError(
+            f"Unknown file format: {suffix}. "
+            "Supported formats: .yaml, .yml, .json"
+        )
+
+
+def to_yaml_string(config: AutoEvalConfig) -> str:
+    """
+    Convert configuration to YAML string.
+
+    Args:
+        config: AutoEvalConfig to convert
+
+    Returns:
+        YAML string representation
+
+    Example:
+        yaml_str = to_yaml_string(config)
+        print(yaml_str)
+    """
+    try:
+        import yaml
+    except ImportError:
+        raise ImportError("PyYAML is required. Install with: pip install pyyaml")
+
+    data = config.to_dict()
+    return yaml.dump(data, default_flow_style=False, sort_keys=False, allow_unicode=True)
+
+
+def to_json_string(config: AutoEvalConfig, indent: int = 2) -> str:
+    """
+    Convert configuration to JSON string.
+
+    Args:
+        config: AutoEvalConfig to convert
+        indent: Indentation level for pretty printing
+
+    Returns:
+        JSON string representation
+
+    Example:
+        json_str = to_json_string(config)
+        print(json_str)
+    """
+    data = config.to_dict()
+    return json.dumps(data, indent=indent)
+
+
+def from_yaml_string(yaml_str: str) -> AutoEvalConfig:
+    """
+    Load configuration from YAML string.
+
+    Args:
+        yaml_str: YAML string to parse
+
+    Returns:
+        AutoEvalConfig instance
+
+    Example:
+        config = from_yaml_string(yaml_content)
+    """
+    try:
+        import yaml
+    except ImportError:
+        raise ImportError("PyYAML is required. Install with: pip install pyyaml")
+
+    data = yaml.safe_load(yaml_str)
+    return AutoEvalConfig.from_dict(data)
+
+
+def from_json_string(json_str: str) -> AutoEvalConfig:
+    """
+    Load configuration from JSON string.
+
+    Args:
+        json_str: JSON string to parse
+
+    Returns:
+        AutoEvalConfig instance
+
+    Example:
+        config = from_json_string(json_content)
+    """
+    data = json.loads(json_str)
+    return AutoEvalConfig.from_dict(data)