agent-learning-kit 0.1.0__py3-none-any.whl

fi/evals/streaming/types.py

@@ -0,0 +1,238 @@
+"""Type definitions for Streaming Evaluation.
+
+Provides types for evaluating LLM outputs in real-time as tokens are generated.
+"""
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Dict, List, Optional, Callable
+from datetime import datetime, timezone
+
+
+class EarlyStopReason(Enum):
+    """Reasons for early stopping during streaming evaluation."""
+
+    NONE = "none"  # No early stop
+    TOXICITY = "toxicity"  # Toxic content detected
+    SAFETY = "safety"  # Safety violation detected
+    PII = "pii"  # PII detected
+    JAILBREAK = "jailbreak"  # Jailbreak attempt detected
+    MAX_TOKENS = "max_tokens"  # Maximum token limit reached
+    MAX_CHARS = "max_chars"  # Maximum character limit reached
+    THRESHOLD = "threshold"  # Score dropped below threshold
+    CUSTOM = "custom"  # Custom stop condition triggered
+    TIMEOUT = "timeout"  # Evaluation timeout
+    ERROR = "error"  # Evaluation error
+
+
+class StreamingState(Enum):
+    """State of the streaming evaluation."""
+
+    IDLE = "idle"  # Not started
+    STREAMING = "streaming"  # Actively processing chunks
+    PAUSED = "paused"  # Temporarily paused
+    STOPPED = "stopped"  # Early stopped
+    COMPLETED = "completed"  # Finished normally
+    ERROR = "error"  # Error occurred
+
+
+@dataclass
+class ChunkResult:
+    """Result of evaluating a single chunk."""
+
+    chunk_index: int
+    chunk_text: str
+    cumulative_text: str
+    scores: Dict[str, float]  # eval_name -> score
+    flags: Dict[str, bool]  # eval_name -> passed
+    should_stop: bool = False
+    stop_reason: EarlyStopReason = EarlyStopReason.NONE
+    latency_ms: float = 0.0
+    timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def all_passed(self) -> bool:
+        """Check if all evaluations passed for this chunk."""
+        return all(self.flags.values()) if self.flags else True
+
+    @property
+    def min_score(self) -> float:
+        """Get the minimum score across all evaluations."""
+        return min(self.scores.values()) if self.scores else 1.0
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "chunk_index": self.chunk_index,
+            "chunk_text": self.chunk_text,
+            "scores": self.scores,
+            "flags": self.flags,
+            "should_stop": self.should_stop,
+            "stop_reason": self.stop_reason.value,
+            "latency_ms": self.latency_ms,
+            "timestamp": self.timestamp.isoformat(),
+        }
+
+
+@dataclass
+class StreamingEvalResult:
+    """Final result of streaming evaluation."""
+
+    passed: bool
+    final_text: str
+    total_chunks: int
+    chunk_results: List[ChunkResult]
+    final_scores: Dict[str, float]  # eval_name -> final score
+    early_stopped: bool = False
+    stop_reason: EarlyStopReason = EarlyStopReason.NONE
+    stopped_at_chunk: Optional[int] = None
+    total_latency_ms: float = 0.0
+    state: StreamingState = StreamingState.COMPLETED
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def average_chunk_latency_ms(self) -> float:
+        """Average latency per chunk."""
+        if not self.chunk_results:
+            return 0.0
+        return sum(c.latency_ms for c in self.chunk_results) / len(self.chunk_results)
+
+    @property
+    def min_score_history(self) -> List[float]:
+        """Get minimum score at each chunk for plotting."""
+        return [c.min_score for c in self.chunk_results]
+
+    @property
+    def score_by_eval(self) -> Dict[str, List[float]]:
+        """Get score history by evaluation name."""
+        result: Dict[str, List[float]] = {}
+        for chunk in self.chunk_results:
+            for eval_name, score in chunk.scores.items():
+                if eval_name not in result:
+                    result[eval_name] = []
+                result[eval_name].append(score)
+        return result
+
+    def summary(self) -> str:
+        """Generate human-readable summary."""
+        lines = [
+            f"Streaming Evaluation: {'PASSED' if self.passed else 'FAILED'}",
+            f"  Total Chunks: {self.total_chunks}",
+            f"  Final Text Length: {len(self.final_text)} chars",
+            f"  Total Latency: {self.total_latency_ms:.2f}ms",
+            f"  Avg Chunk Latency: {self.average_chunk_latency_ms:.2f}ms",
+        ]
+
+        if self.early_stopped:
+            lines.append(f"  Early Stopped: Yes (at chunk {self.stopped_at_chunk})")
+            lines.append(f"  Stop Reason: {self.stop_reason.value}")
+
+        lines.append("  Final Scores:")
+        for eval_name, score in self.final_scores.items():
+            lines.append(f"    {eval_name}: {score:.3f}")
+
+        return "\n".join(lines)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "passed": self.passed,
+            "final_text": self.final_text,
+            "total_chunks": self.total_chunks,
+            "chunk_results": [c.to_dict() for c in self.chunk_results],
+            "final_scores": self.final_scores,
+            "early_stopped": self.early_stopped,
+            "stop_reason": self.stop_reason.value,
+            "stopped_at_chunk": self.stopped_at_chunk,
+            "total_latency_ms": self.total_latency_ms,
+            "state": self.state.value,
+        }
+
+
+@dataclass
+class StreamingConfig:
+    """Configuration for streaming evaluation."""
+
+    # Chunk processing
+    min_chunk_size: int = 1  # Minimum characters to trigger evaluation
+    max_chunk_size: int = 100  # Maximum characters per chunk before forced eval
+    eval_interval_ms: int = 100  # Minimum ms between evaluations
+
+    # Token limits
+    max_tokens: Optional[int] = None  # Maximum tokens before stopping
+    max_chars: Optional[int] = None  # Maximum characters before stopping
+
+    # Timeouts
+    chunk_timeout_ms: int = 5000  # Timeout for single chunk evaluation
+    total_timeout_ms: int = 60000  # Total timeout for streaming
+
+    # Early stopping
+    enable_early_stop: bool = True
+    stop_on_first_failure: bool = False  # Stop immediately on any failure
+
+    # Thresholds for early stopping
+    toxicity_threshold: float = 0.7  # Stop if toxicity score exceeds this
+    safety_threshold: float = 0.3  # Stop if safety score drops below this
+
+    # Evaluation frequency
+    eval_every_n_chunks: int = 1  # Run evaluation every N chunks
+    eval_on_sentence_end: bool = True  # Also evaluate at sentence boundaries
+
+    # Callbacks
+    on_chunk_callback: Optional[Callable[[ChunkResult], None]] = None
+    on_stop_callback: Optional[Callable[[EarlyStopReason, str], None]] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary (excluding callbacks)."""
+        return {
+            "min_chunk_size": self.min_chunk_size,
+            "max_chunk_size": self.max_chunk_size,
+            "eval_interval_ms": self.eval_interval_ms,
+            "max_tokens": self.max_tokens,
+            "max_chars": self.max_chars,
+            "chunk_timeout_ms": self.chunk_timeout_ms,
+            "total_timeout_ms": self.total_timeout_ms,
+            "enable_early_stop": self.enable_early_stop,
+            "stop_on_first_failure": self.stop_on_first_failure,
+            "toxicity_threshold": self.toxicity_threshold,
+            "safety_threshold": self.safety_threshold,
+            "eval_every_n_chunks": self.eval_every_n_chunks,
+            "eval_on_sentence_end": self.eval_on_sentence_end,
+        }
+
+
+@dataclass
+class EarlyStopCondition:
+    """Defines a condition for early stopping."""
+
+    name: str
+    eval_name: str  # Which evaluation to check
+    threshold: float  # Threshold value
+    comparison: str = "below"  # "below" or "above"
+    consecutive_chunks: int = 1  # How many consecutive chunks must fail
+    enabled: bool = True
+
+    def check(self, score: float, consecutive_count: int) -> bool:
+        """Check if this condition triggers early stop."""
+        if not self.enabled:
+            return False
+
+        if consecutive_count < self.consecutive_chunks:
+            return False
+
+        if self.comparison == "below":
+            return score < self.threshold
+        else:  # above
+            return score > self.threshold
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "name": self.name,
+            "eval_name": self.eval_name,
+            "threshold": self.threshold,
+            "comparison": self.comparison,
+            "consecutive_chunks": self.consecutive_chunks,
+            "enabled": self.enabled,
+        }

fi/evals/templates.py

@@ -0,0 +1,472 @@
+"""
+Template class shells for cloud evals.
+
+These classes exist purely for IDE autocomplete and backward compatibility
+with code that does ``from fi.evals import Toxicity``. Input schemas are
+no longer hardcoded here — the cloud registry (``fi.evals.core.cloud_registry``)
+fetches ``required_keys`` from the api and maps user inputs dynamically.
+
+To call an eval by name without importing a class, just pass the string::
+
+    evaluate("customer_agent_query_handling", conversation=[...], model="turing_flash")
+
+Renamed templates have a module-level alias from the old name to the new
+class (e.g. ``NoOpenAIReference = NoLLMReference``) so existing user code
+keeps working seamlessly. Templates that were removed outright are gone —
+an ``ImportError`` is a clearer signal than a silent runtime 400.
+"""
+from typing import Any, Dict, List, Optional
+
+from fi.utils.errors import MissingRequiredConfigForEvalTemplate
+
+
+# ---------------------------------------------------------------------------
+# EvalTemplate base class
+# ---------------------------------------------------------------------------
+
+class EvalTemplate:
+    """Lightweight marker for cloud eval templates.
+
+    Input filtering and validation happen dynamically via the cloud
+    registry — no hardcoded Input schema.
+    """
+
+    eval_id: str = ""
+    eval_name: str = ""
+    description: str = ""
+    eval_tags: List[str] = []
+    required_keys: List[str] = []
+    output: str = ""
+    eval_type_id: str = ""
+    config_schema: Dict[str, Any] = {}
+    criteria: str = ""
+    choices: List[str] = []
+    multi_choice: bool = False
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:
+        self.config = config or {}
+
+    def __repr__(self) -> str:
+        return f"EvalTemplate(name={self.eval_name})"
+
+    def validate_config(self, config: Dict[str, Any]) -> None:
+        for key in self.config_schema:
+            if key not in config:
+                raise MissingRequiredConfigForEvalTemplate(key, self.eval_name)
+
+
+# ---------------------------------------------------------------------------
+# Conversation
+# ---------------------------------------------------------------------------
+
+class ConversationCoherence(EvalTemplate):
+    eval_name = "conversation_coherence"
+    eval_id = "1"
+
+
+class ConversationResolution(EvalTemplate):
+    eval_name = "conversation_resolution"
+    eval_id = "2"
+
+
+# ---------------------------------------------------------------------------
+# Content moderation & safety
+# ---------------------------------------------------------------------------
+
+class ContentModeration(EvalTemplate):
+    eval_name = "content_moderation"
+    eval_id = "4"
+
+
+class PII(EvalTemplate):
+    eval_name = "pii"
+    eval_id = "14"
+
+
+class Toxicity(EvalTemplate):
+    eval_name = "toxicity"
+    eval_id = "15"
+
+
+class Sexist(EvalTemplate):
+    eval_name = "sexist"
+    eval_id = "17"
+
+
+class PromptInjection(EvalTemplate):
+    eval_name = "prompt_injection"
+    eval_id = "18"
+
+
+class DataPrivacyCompliance(EvalTemplate):
+    eval_name = "data_privacy_compliance"
+    eval_id = "22"
+
+
+class NoRacialBias(EvalTemplate):
+    eval_name = "no_racial_bias"
+    eval_id = "77"
+
+
+class NoGenderBias(EvalTemplate):
+    eval_name = "no_gender_bias"
+    eval_id = "78"
+
+
+class NoAgeBias(EvalTemplate):
+    eval_name = "no_age_bias"
+    eval_id = "79"
+
+
+class NoLLMReference(EvalTemplate):
+    """Checks that the output doesn't reference the underlying LLM (e.g. 'I am GPT…').
+
+    Replaced the old ``NoOpenAIReference`` template in the api revamp.
+    """
+
+    eval_name = "no_llm_reference"
+
+
+# Alias: old class name → current eval. Import-level backward compatibility.
+NoOpenAIReference = NoLLMReference
+
+
+class NoApologies(EvalTemplate):
+    eval_name = "no_apologies"
+    eval_id = "81"
+
+
+class ContentSafety(EvalTemplate):
+    eval_name = "content_safety_violation"
+    eval_id = "93"
+
+
+class NoHarmfulTherapeuticGuidance(EvalTemplate):
+    eval_name = "no_harmful_therapeutic_guidance"
+    eval_id = "90"
+
+
+class ClinicallyInappropriateTone(EvalTemplate):
+    eval_name = "clinically_inappropriate_tone"
+    eval_id = "91"
+
+
+class IsHarmfulAdvice(EvalTemplate):
+    eval_name = "is_harmful_advice"
+    eval_id = "92"
+
+
+# ---------------------------------------------------------------------------
+# RAG / Grounding
+# ---------------------------------------------------------------------------
+
+class ContextAdherence(EvalTemplate):
+    eval_name = "context_adherence"
+    eval_id = "5"
+
+
+class ContextRelevance(EvalTemplate):
+    eval_name = "context_relevance"
+    eval_id = "9"
+
+
+class Completeness(EvalTemplate):
+    eval_name = "completeness"
+    eval_id = "10"
+
+
+class ChunkAttribution(EvalTemplate):
+    eval_name = "chunk_attribution"
+    eval_id = "11"
+
+
+class ChunkUtilization(EvalTemplate):
+    eval_name = "chunk_utilization"
+    eval_id = "12"
+
+
+class Groundedness(EvalTemplate):
+    eval_name = "groundedness"
+    eval_id = "47"
+
+
+class FactualAccuracy(EvalTemplate):
+    eval_name = "factual_accuracy"
+    eval_id = "66"
+
+
+class DetectHallucination(EvalTemplate):
+    """Detects hallucinated or unsupported claims in the output.
+
+    Replaced the old ``DetectHallucinationMissingInfo`` template in the revamp.
+    """
+
+    eval_name = "detect_hallucination"
+
+
+# Alias: old class name → current eval.
+DetectHallucinationMissingInfo = DetectHallucination
+
+
+class IsFactuallyConsistent(EvalTemplate):
+    eval_name = "is_factually_consistent"
+    eval_id = "95"
+
+
+# ---------------------------------------------------------------------------
+# Tone & quality
+# ---------------------------------------------------------------------------
+
+class Tone(EvalTemplate):
+    eval_name = "tone"
+    eval_id = "16"
+
+
+class PromptAdherence(EvalTemplate):
+    eval_name = "prompt_adherence"
+    eval_id = "65"
+
+
+class IsPolite(EvalTemplate):
+    eval_name = "is_polite"
+    eval_id = "82"
+
+
+class IsConcise(EvalTemplate):
+    eval_name = "is_concise"
+    eval_id = "83"
+
+
+class IsHelpful(EvalTemplate):
+    eval_name = "is_helpful"
+    eval_id = "84"
+
+
+class IsInformalTone(EvalTemplate):
+    eval_name = "is_informal_tone"
+    eval_id = "97"
+
+
+class AnswerRefusal(EvalTemplate):
+    eval_name = "answer_refusal"
+    eval_id = "88"
+
+
+class TaskCompletion(EvalTemplate):
+    eval_name = "task_completion"
+    eval_id = "99"
+
+
+# ---------------------------------------------------------------------------
+# Format / structure
+# ---------------------------------------------------------------------------
+
+class IsJson(EvalTemplate):
+    eval_name = "is_json"
+    eval_id = "23"
+
+
+class OneLine(EvalTemplate):
+    eval_name = "one_line"
+    eval_id = "38"
+
+
+class ContainsValidLink(EvalTemplate):
+    eval_name = "contains_valid_link"
+    eval_id = "39"
+
+
+class IsEmail(EvalTemplate):
+    eval_name = "is_email"
+    eval_id = "40"
+
+
+class ContainsCode(EvalTemplate):
+    """New in the api revamp."""
+
+    eval_name = "contains_code"
+
+
+# ---------------------------------------------------------------------------
+# Comparison / matching
+# ---------------------------------------------------------------------------
+
+class Ranking(EvalTemplate):
+    eval_name = "eval_ranking"
+    eval_id = "61"
+
+
+class SummaryQuality(EvalTemplate):
+    eval_name = "summary_quality"
+    eval_id = "64"
+
+
+class IsGoodSummary(EvalTemplate):
+    eval_name = "is_good_summary"
+    eval_id = "94"
+
+
+class FuzzyMatch(EvalTemplate):
+    eval_name = "fuzzy_match"
+    eval_id = "87"
+
+
+class BleuScore(EvalTemplate):
+    eval_name = "bleu_score"
+    eval_id = "101"
+
+
+class TranslationAccuracy(EvalTemplate):
+    eval_name = "translation_accuracy"
+    eval_id = "67"
+
+
+class CulturalSensitivity(EvalTemplate):
+    eval_name = "cultural_sensitivity"
+    eval_id = "68"
+
+
+class BiasDetection(EvalTemplate):
+    eval_name = "bias_detection"
+    eval_id = "69"
+
+
+class GroundTruthMatch(EvalTemplate):
+    """New in the api revamp."""
+
+    eval_name = "ground_truth_match"
+
+
+# ---------------------------------------------------------------------------
+# Function calling
+# ---------------------------------------------------------------------------
+
+class EvaluateFunctionCalling(EvalTemplate):
+    eval_name = "evaluate_function_calling"
+    eval_id = "98"
+
+
+# Alias: old class name → current eval.
+LLMFunctionCalling = EvaluateFunctionCalling
+
+
+# ---------------------------------------------------------------------------
+# Domain / agent / multimodal
+# ---------------------------------------------------------------------------
+
+class TextToSQL(EvalTemplate):
+    """New in the api revamp."""
+
+    eval_name = "text_to_sql"
+
+
+class PromptInstructionAdherence(EvalTemplate):
+    """New in the api revamp."""
+
+    eval_name = "prompt_instruction_adherence"
+
+
+class ProtectFlash(EvalTemplate):
+    """New in the api revamp — lightweight prompt-injection check."""
+
+    eval_name = "protect_flash"
+
+
+class ImageInstructionAdherence(EvalTemplate):
+    """New in the api revamp."""
+
+    eval_name = "image_instruction_adherence"
+
+
+class SyntheticImageEvaluator(EvalTemplate):
+    """New in the api revamp."""
+
+    eval_name = "synthetic_image_evaluator"
+
+
+class OCREvaluation(EvalTemplate):
+    """New in the api revamp."""
+
+    eval_name = "ocr_evaluation"
+
+
+class ASRAccuracy(EvalTemplate):
+    """Speech-to-text accuracy eval.
+
+    Replaced the old ``AudioTranscriptionEvaluator`` in the revamp.
+    """
+
+    eval_name = "ASR/STT_accuracy"
+
+
+# Alias: old class name → current eval.
+AudioTranscriptionEvaluator = ASRAccuracy
+
+
+class TTSAccuracy(EvalTemplate):
+    """New in the api revamp."""
+
+    eval_name = "TTS_accuracy"
+
+
+class AudioQualityEvaluator(EvalTemplate):
+    eval_name = "audio_quality"
+    eval_id = "75"
+
+
+class CaptionHallucination(EvalTemplate):
+    eval_name = "caption_hallucination"
+    eval_id = "100"
+
+
+class IsCompliant(EvalTemplate):
+    eval_name = "is_compliant"
+    eval_id = "96"
+
+
+# ---------------------------------------------------------------------------
+# Customer-agent family (new in revamp)
+# ---------------------------------------------------------------------------
+
+class CustomerAgentClarificationSeeking(EvalTemplate):
+    eval_name = "customer_agent_clarification_seeking"
+
+
+class CustomerAgentContextRetention(EvalTemplate):
+    eval_name = "customer_agent_context_retention"
+
+
+class CustomerAgentConversationQuality(EvalTemplate):
+    eval_name = "customer_agent_conversation_quality"
+
+
+class CustomerAgentHumanEscalation(EvalTemplate):
+    eval_name = "customer_agent_human_escalation"
+
+
+class CustomerAgentInterruptionHandling(EvalTemplate):
+    eval_name = "customer_agent_interruption_handling"
+
+
+class CustomerAgentLanguageHandling(EvalTemplate):
+    eval_name = "customer_agent_language_handling"
+
+
+class CustomerAgentLoopDetection(EvalTemplate):
+    eval_name = "customer_agent_loop_detection"
+
+
+class CustomerAgentObjectionHandling(EvalTemplate):
+    eval_name = "customer_agent_objection_handling"
+
+
+class CustomerAgentPromptConformance(EvalTemplate):
+    eval_name = "customer_agent_prompt_conformance"
+
+
+class CustomerAgentQueryHandling(EvalTemplate):
+    eval_name = "customer_agent_query_handling"
+
+
+class CustomerAgentTerminationHandling(EvalTemplate):
+    eval_name = "customer_agent_termination_handling"