agent-learning-kit 0.1.0__py3-none-any.whl

fi/evals/protect_input_adapter.py

@@ -0,0 +1,154 @@
+import base64
+import os
+from typing import ClassVar, Set
+from urllib.parse import urlparse
+
+from pydantic import BaseModel, field_validator
+
+# Keep this class minimal & fast: no network calls; only local file -> data URI.
+# The backend will detect input type and do any heavy lifting.
+
+class ProtectInputAdapter(BaseModel):
+    """
+    Minimal, production-safe input wrapper for Protect.
+
+    - Accepts text, http(s) URLs, data URIs, and local audio/image files.
+    - Rejects known HTML 'viewer' URLs (e.g., GitHub blob) to avoid 400s downstream.
+    - Converts local media files to 'data:' URIs (cheap; no networking).
+    - Leaves type detection to the backend.
+    """
+    input: str
+    call_type: str = "protect"
+
+    AUDIO_EXTENSIONS: ClassVar[Set[str]] = {
+        ".mp3", ".wav",
+    }
+    IMAGE_EXTENSIONS: ClassVar[Set[str]] = {
+        ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".tiff", ".tif", ".svg"
+    }
+    # Optional cap for local files (bytes). Keep small to protect latency/footguns.
+    MAX_LOCAL_BYTES: ClassVar[int] = int(os.getenv("FI_PROTECT_MAX_LOCAL_BYTES", "20000000"))  # 20MB
+
+    # --- Pydantic v2: run after parsing ---
+    def model_post_init(self, __context) -> None:
+        try:
+            s = (self.input or "").strip()
+            if not s:
+                raise ValueError("Input cannot be empty or whitespace")
+
+            # 1) Data URIs: validate & allow
+            if s.startswith("data:"):
+                mime = s.split(";", 1)[0].split(":", 1)[-1].lower()
+                if not (mime.startswith("audio/") or mime.startswith("image/")):
+                    # Treat non-media data URIs as text – Protect supports only text/image/audio.
+                    raise ValueError("Unsupported data URI mime; only audio/* or image/* allowed")
+                # Pass as-is
+                return
+
+            p = urlparse(s)
+
+            # 2) HTTP(S) URL: allow pass-through, but block known viewer pages
+            if p.scheme in ("http", "https"):
+                if self._is_blocked_url(s, p):
+                    raise ValueError(
+                        "This link looks like a preview page, not a direct file. "
+                        "Use a direct download URL (e.g., raw.githubusercontent.com for GitHub; "
+                        "export=download for Google Drive; dl.dropboxusercontent.com for Dropbox)."
+                    )
+                # Do not sniff/transform: backend will handle it
+                return
+
+            # 3) Local file path → convert to data URI (fast, no network)
+            if self._looks_like_local_path(s):
+                ext = os.path.splitext(s)[1].lower()
+                if ext in self.AUDIO_EXTENSIONS:
+                    self.input = self._file_to_data_uri(s, self._audio_mime(ext))
+                    return
+                if ext in self.IMAGE_EXTENSIONS:
+                    self.input = self._file_to_data_uri(s, self._image_mime(ext))
+                    return
+                # Not a supported media extension
+                allowed_audio = ", ".join(sorted(self.AUDIO_EXTENSIONS))
+                allowed_img = ", ".join(sorted(self.IMAGE_EXTENSIONS))
+                raise ValueError(
+                    f"Unsupported local file type '{ext}'. Supported audio: {allowed_audio}. "
+                    f"Supported image: {allowed_img}."
+                )
+
+            # 4) Otherwise: treat as plain text (backend handles as text)
+            # Nothing to change.
+        except Exception as e:
+            # Fail fast with a clean message. Protect supports only text/image/audio.
+            raise ValueError(f"Invalid Protect input: {e}") from e
+
+    @staticmethod
+    def _looks_like_local_path(s: str) -> bool:
+        # Absolute or relative file path that exists
+        try:
+            return os.path.exists(s)
+        except Exception:
+            return False
+
+    @staticmethod
+    def _is_blocked_url(s: str, p) -> bool:
+        host = (p.netloc or "").lower()
+        path = (p.path or "").lower()
+
+        # Known viewer / HTML pages that won't return raw bytes:
+        # GitHub blob pages
+        if host == "github.com" and "/blob/" in path:
+            return True
+        # Google Drive viewers
+        if host in ("drive.google.com", "docs.google.com") and ("/file/" in path or "/uc" in path) and "export=download" not in s:
+            return True
+        # Dropbox share pages (use dl.dropboxusercontent.com for direct)
+        if host == "www.dropbox.com" and "/s/" in path:
+            return True
+        # OneDrive viewer links
+        if host == "onedrive.live.com" and "redir" in path:
+            return True
+
+        return False
+
+    @staticmethod
+    def _audio_mime(ext: str) -> str:
+        return {
+            ".mp3": "audio/mp3",
+            ".wav": "audio/wav",
+        }.get(ext, "audio/mp3")
+
+    @staticmethod
+    def _image_mime(ext: str) -> str:
+        return {
+            ".jpg": "image/jpeg",
+            ".jpeg": "image/jpeg",
+            ".png": "image/png",
+            ".gif": "image/gif",
+            ".webp": "image/webp",
+            ".bmp": "image/bmp",
+            ".tiff": "image/tiff",
+            ".tif": "image/tiff",
+            ".svg": "image/svg+xml",
+        }.get(ext, "image/jpeg")
+
+    def _file_to_data_uri(self, path: str, mime: str) -> str:
+        # Small, local-only work; no network. Guard against giant files.
+        size = os.path.getsize(path)
+        if size > self.MAX_LOCAL_BYTES:
+            raise ValueError(
+                f"Local file too large ({size} bytes). Max allowed is {self.MAX_LOCAL_BYTES}."
+            )
+        try:
+            with open(path, "rb") as f:
+                b64 = base64.b64encode(f.read()).decode("utf-8")
+            return f"data:{mime};base64,{b64}"
+        except Exception as e:
+            raise ValueError(f"Failed to read local file: {e}")
+
+    # An optional lightweight validator so callers get a nice error earlier if input is not a string.
+    @field_validator("input")
+    @classmethod
+    def _validate_input_is_str(cls, v: str) -> str:
+        if not isinstance(v, str):
+            raise TypeError("Protect input must be a string")
+        return v

fi/evals/streaming/__init__.py

@@ -0,0 +1,88 @@
+"""Streaming Evaluation Module.
+
+Provides real-time evaluation of LLM outputs as tokens stream in,
+with support for early stopping based on configurable policies.
+
+Example:
+    from fi.evals.streaming import StreamingEvaluator, StreamingConfig, EarlyStopPolicy
+
+    # Create evaluator
+    evaluator = StreamingEvaluator(
+        config=StreamingConfig(
+            min_chunk_size=10,
+            max_chunk_size=100,
+            enable_early_stop=True,
+        ),
+        policy=EarlyStopPolicy.default(),
+    )
+
+    # Add evaluation functions
+    evaluator.add_eval("toxicity", toxicity_scorer, threshold=0.7, pass_above=False)
+
+    # Process stream
+    for token in llm_stream:
+        result = evaluator.process_token(token)
+        if result and result.should_stop:
+            print(f"Early stop: {result.stop_reason}")
+            break
+
+    # Get final results
+    final_result = evaluator.finalize()
+    print(final_result.summary())
+"""
+
+from .types import (
+    ChunkResult,
+    EarlyStopCondition,
+    EarlyStopReason,
+    StreamingConfig,
+    StreamingEvalResult,
+    StreamingState,
+)
+from .buffer import BufferState, ChunkBuffer
+from .policy import EarlyStopPolicy, PolicyState
+from .evaluator import EvalSpec, StreamingEvaluator
+from .scorers import (
+    toxicity_scorer,
+    safety_scorer,
+    pii_scorer,
+    jailbreak_scorer,
+    coherence_scorer,
+    quality_scorer,
+    safety_composite_scorer,
+    quality_composite_scorer,
+    create_keyword_scorer,
+    create_pattern_scorer,
+    CompositeScorer,
+)
+
+__all__ = [
+    # Types
+    "ChunkResult",
+    "EarlyStopCondition",
+    "EarlyStopReason",
+    "StreamingConfig",
+    "StreamingEvalResult",
+    "StreamingState",
+    # Buffer
+    "BufferState",
+    "ChunkBuffer",
+    # Policy
+    "EarlyStopPolicy",
+    "PolicyState",
+    # Evaluator
+    "EvalSpec",
+    "StreamingEvaluator",
+    # Scorers
+    "toxicity_scorer",
+    "safety_scorer",
+    "pii_scorer",
+    "jailbreak_scorer",
+    "coherence_scorer",
+    "quality_scorer",
+    "safety_composite_scorer",
+    "quality_composite_scorer",
+    "create_keyword_scorer",
+    "create_pattern_scorer",
+    "CompositeScorer",
+]

fi/evals/streaming/buffer.py

@@ -0,0 +1,213 @@
+"""Chunk buffer for streaming evaluation.
+
+Accumulates tokens/chunks and manages when to trigger evaluations.
+"""
+
+import re
+import time
+from dataclasses import dataclass
+from typing import Optional, Tuple
+from .types import StreamingConfig
+
+
+@dataclass
+class BufferState:
+    """Current state of the chunk buffer."""
+
+    total_text: str = ""
+    pending_text: str = ""
+    chunk_count: int = 0
+    token_count: int = 0
+    last_eval_time: float = 0.0
+    last_eval_position: int = 0
+
+
+class ChunkBuffer:
+    """
+    Buffer that accumulates streaming tokens and determines when to evaluate.
+
+    Manages the accumulation of tokens from a streaming LLM response and
+    decides when enough content has been collected to trigger an evaluation.
+
+    Example:
+        buffer = ChunkBuffer(config)
+
+        for token in stream:
+            buffer.add(token)
+            if buffer.should_evaluate():
+                chunk = buffer.get_chunk()
+                result = evaluator.evaluate_chunk(chunk, buffer.get_cumulative())
+                buffer.mark_evaluated()
+    """
+
+    # Sentence ending patterns
+    SENTENCE_ENDINGS = re.compile(r'[.!?]\s*$')
+    PARTIAL_SENTENCE = re.compile(r'[.!?]\s+')
+
+    def __init__(self, config: Optional[StreamingConfig] = None):
+        """
+        Initialize the buffer.
+
+        Args:
+            config: Streaming configuration (uses defaults if None)
+        """
+        self.config = config or StreamingConfig()
+        self._state = BufferState()
+        self._start_time = time.perf_counter()
+
+    def add(self, token: str) -> None:
+        """
+        Add a token to the buffer.
+
+        Args:
+            token: The token text to add
+        """
+        self._state.total_text += token
+        self._state.pending_text += token
+        self._state.token_count += 1
+
+    def add_chunk(self, chunk: str) -> None:
+        """
+        Add a larger chunk of text to the buffer.
+
+        Args:
+            chunk: The chunk text to add
+        """
+        self._state.total_text += chunk
+        self._state.pending_text += chunk
+        # Estimate token count (rough approximation)
+        self._state.token_count += len(chunk.split())
+
+    def should_evaluate(self) -> bool:
+        """
+        Check if we should trigger an evaluation.
+
+        Returns:
+            True if evaluation should be triggered
+        """
+        pending_len = len(self._state.pending_text)
+
+        # Check minimum chunk size
+        if pending_len < self.config.min_chunk_size:
+            return False
+
+        # Check time interval
+        current_time = time.perf_counter()
+        time_since_last = (current_time - self._state.last_eval_time) * 1000
+
+        # Force evaluation if max chunk size reached
+        if pending_len >= self.config.max_chunk_size:
+            return True
+
+        # Check time-based interval
+        if time_since_last < self.config.eval_interval_ms:
+            return False
+
+        # Check sentence boundary if enabled
+        if self.config.eval_on_sentence_end:
+            if self.SENTENCE_ENDINGS.search(self._state.pending_text):
+                return True
+
+        # Check chunk count interval
+        if self.config.eval_every_n_chunks > 1:
+            # Only evaluate every N chunks
+            next_chunk = self._state.chunk_count + 1
+            if next_chunk % self.config.eval_every_n_chunks != 0:
+                return pending_len >= self.config.max_chunk_size
+
+        return pending_len >= self.config.min_chunk_size
+
+    def get_chunk(self) -> str:
+        """
+        Get the pending chunk for evaluation.
+
+        Returns:
+            The pending text that should be evaluated
+        """
+        return self._state.pending_text
+
+    def get_cumulative(self) -> str:
+        """
+        Get all accumulated text so far.
+
+        Returns:
+            The complete accumulated text
+        """
+        return self._state.total_text
+
+    def mark_evaluated(self) -> None:
+        """Mark the current pending content as evaluated."""
+        self._state.chunk_count += 1
+        self._state.last_eval_time = time.perf_counter()
+        self._state.last_eval_position = len(self._state.total_text)
+        self._state.pending_text = ""
+
+    def get_chunk_index(self) -> int:
+        """Get the current chunk index."""
+        return self._state.chunk_count
+
+    def get_token_count(self) -> int:
+        """Get the total token count."""
+        return self._state.token_count
+
+    def get_char_count(self) -> int:
+        """Get the total character count."""
+        return len(self._state.total_text)
+
+    def should_stop_for_limits(self) -> Tuple[bool, str]:
+        """
+        Check if we should stop due to limits.
+
+        Returns:
+            Tuple of (should_stop, reason)
+        """
+        # Check token limit
+        if self.config.max_tokens and self._state.token_count >= self.config.max_tokens:
+            return True, "max_tokens"
+
+        # Check character limit
+        if self.config.max_chars and len(self._state.total_text) >= self.config.max_chars:
+            return True, "max_chars"
+
+        # Check total timeout
+        elapsed_ms = (time.perf_counter() - self._start_time) * 1000
+        if elapsed_ms >= self.config.total_timeout_ms:
+            return True, "timeout"
+
+        return False, ""
+
+    def reset(self) -> None:
+        """Reset the buffer to initial state."""
+        self._state = BufferState()
+        self._start_time = time.perf_counter()
+
+    @property
+    def state(self) -> BufferState:
+        """Get the current buffer state."""
+        return self._state
+
+    @property
+    def is_empty(self) -> bool:
+        """Check if buffer has no content."""
+        return len(self._state.total_text) == 0
+
+    @property
+    def has_pending(self) -> bool:
+        """Check if there's pending unevaluated content."""
+        return len(self._state.pending_text) > 0
+
+    def get_stats(self) -> dict:
+        """Get buffer statistics."""
+        elapsed_ms = (time.perf_counter() - self._start_time) * 1000
+        return {
+            "total_chars": len(self._state.total_text),
+            "pending_chars": len(self._state.pending_text),
+            "chunk_count": self._state.chunk_count,
+            "token_count": self._state.token_count,
+            "elapsed_ms": elapsed_ms,
+            "avg_chunk_size": (
+                len(self._state.total_text) / self._state.chunk_count
+                if self._state.chunk_count > 0
+                else 0
+            ),
+        }