agent-learning-kit 0.1.0__py3-none-any.whl

fi/evals/feedback/retriever.py

@@ -0,0 +1,128 @@
+"""Feedback retrieval and few-shot formatting.
+
+Retrieves semantically similar feedback from the store and formats it
+as few-shot examples for the LLM judge pipeline.
+"""
+
+import json
+import logging
+from typing import Any, Dict, List, Optional
+
+from .store import FeedbackStore
+
+logger = logging.getLogger(__name__)
+
+
+class FeedbackRetriever:
+    """Retrieves semantically similar feedback and formats as few-shot examples.
+
+    This is the bridge between the feedback store and the LLM judge pipeline.
+    When a metric is run with a feedback store, the retriever:
+
+    1. Builds an embedding query from the current inputs
+    2. Searches the store for similar past feedback entries
+    3. Converts matching entries to the few_shot_examples format
+       expected by CustomLLMJudge's Jinja2 template
+
+    Args:
+        store: The FeedbackStore to search.
+        max_examples: Maximum number of few-shot examples to inject. Default 3.
+    """
+
+    def __init__(
+        self,
+        store: FeedbackStore,
+        max_examples: int = 3,
+    ):
+        self.store = store
+        self.max_examples = max_examples
+
+    def build_query_text(self, metric_name: str, inputs: Dict[str, Any]) -> str:
+        """Build a query string from inputs for semantic search.
+
+        Uses the same concatenation strategy as FeedbackEntry.to_embedding_text()
+        to ensure query-document alignment.
+        """
+        parts = [f"metric: {metric_name}"]
+        for key in ("output", "response", "context", "input", "query"):
+            val = inputs.get(key)
+            if val:
+                text = val if isinstance(val, str) else json.dumps(val, default=str)
+                parts.append(f"{key}: {text[:500]}")
+        return "\n".join(parts)
+
+    def retrieve_few_shot_examples(
+        self,
+        metric_name: str,
+        inputs: Dict[str, Any],
+    ) -> List[Dict[str, Any]]:
+        """Retrieve few-shot examples from feedback store.
+
+        Returns a list of dicts in the format expected by
+        CustomLLMJudge's config["few_shot_examples"]:
+
+            [{"inputs": {...}, "output": '{"score": 0.8, "reason": "..."}'}]
+
+        Args:
+            metric_name: The metric being run.
+            inputs: The current inputs.
+
+        Returns:
+            List of few-shot example dicts, possibly empty if no feedback exists.
+        """
+        if self.store.count(metric_name) == 0:
+            return []
+
+        query_text = self.build_query_text(metric_name, inputs)
+
+        similar_entries = self.store.query_similar(
+            metric_name=metric_name,
+            text=query_text,
+            n_results=self.max_examples,
+        )
+
+        if not similar_entries:
+            return []
+
+        examples = []
+        for entry in similar_entries:
+            # Only include entries where the developer provided a correction
+            if entry.correct_score is None and not entry.correct_reason:
+                continue
+            examples.append(entry.to_few_shot())
+
+        if examples:
+            logger.debug(
+                f"Retrieved {len(examples)} feedback examples for '{metric_name}'"
+            )
+
+        return examples
+
+    def inject_into_config(
+        self,
+        metric_name: str,
+        inputs: Dict[str, Any],
+        config: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        """Retrieve feedback and merge into a config dict for LLMEngine/CustomLLMJudge.
+
+        This is the primary integration point. Call this before passing config
+        to LLMEngine.run() to inject few-shot examples.
+
+        Args:
+            metric_name: Metric name.
+            inputs: Current inputs.
+            config: Existing config dict (will not be mutated).
+
+        Returns:
+            New config dict with few_shot_examples populated.
+        """
+        config = dict(config or {})
+
+        examples = self.retrieve_few_shot_examples(metric_name, inputs)
+        if examples:
+            # Merge with any existing few-shot examples
+            existing = config.get("few_shot_examples", [])
+            config["few_shot_examples"] = existing + examples
+
+        return config

fi/evals/feedback/store.py

@@ -0,0 +1,272 @@
+"""Feedback storage backends.
+
+Provides abstract FeedbackStore and two implementations:
+- InMemoryFeedbackStore: for testing and small-scale usage
+- ChromaFeedbackStore: for production with semantic vector search
+"""
+
+import json
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+from .types import FeedbackEntry
+
+logger = logging.getLogger(__name__)
+
+
+class FeedbackStore(ABC):
+    """Abstract base class for feedback persistence."""
+
+    @abstractmethod
+    def add(self, entry: FeedbackEntry) -> str:
+        """Store a feedback entry. Returns the entry ID."""
+        ...
+
+    @abstractmethod
+    def query_similar(
+        self,
+        metric_name: str,
+        text: str,
+        n_results: int = 5,
+    ) -> List[FeedbackEntry]:
+        """Find feedback entries semantically similar to the given text,
+        filtered by metric_name."""
+        ...
+
+    @abstractmethod
+    def get_by_metric(self, metric_name: str, limit: int = 100) -> List[FeedbackEntry]:
+        """Get all feedback entries for a specific metric."""
+        ...
+
+    @abstractmethod
+    def count(self, metric_name: Optional[str] = None) -> int:
+        """Count entries, optionally filtered by metric_name."""
+        ...
+
+    @abstractmethod
+    def delete(self, entry_id: str) -> bool:
+        """Delete a feedback entry by ID."""
+        ...
+
+
+class InMemoryFeedbackStore(FeedbackStore):
+    """In-memory feedback store for testing and small-scale usage.
+
+    No vector search -- falls back to recency-based retrieval.
+    Suitable for unit tests and quick experimentation.
+    """
+
+    def __init__(self):
+        self._entries: Dict[str, FeedbackEntry] = {}
+
+    def add(self, entry: FeedbackEntry) -> str:
+        self._entries[entry.id] = entry
+        return entry.id
+
+    def query_similar(
+        self,
+        metric_name: str,
+        text: str,
+        n_results: int = 5,
+    ) -> List[FeedbackEntry]:
+        # No semantic search -- return most recent entries for this metric
+        filtered = [
+            e for e in self._entries.values()
+            if e.eval_name == metric_name
+        ]
+        filtered.sort(key=lambda e: e.created_at, reverse=True)
+        return filtered[:n_results]
+
+    def get_by_metric(self, metric_name: str, limit: int = 100) -> List[FeedbackEntry]:
+        return [
+            e for e in self._entries.values()
+            if e.eval_name == metric_name
+        ][:limit]
+
+    def count(self, metric_name: Optional[str] = None) -> int:
+        if metric_name:
+            return sum(1 for e in self._entries.values() if e.eval_name == metric_name)
+        return len(self._entries)
+
+    def delete(self, entry_id: str) -> bool:
+        return self._entries.pop(entry_id, None) is not None
+
+
+class ChromaFeedbackStore(FeedbackStore):
+    """ChromaDB-backed feedback store with semantic vector search.
+
+    Supports two modes:
+    - Local: in-process persistent ChromaDB (default)
+    - Service: connects to a remote ChromaDB server (e.g. Docker container)
+
+    Embedding is handled by ChromaDB's built-in default embedding function
+    (all-MiniLM-L6-v2 via sentence-transformers) OR via a LiteLLM embedding
+    function for API-based embeddings.
+
+    Args:
+        host: ChromaDB server host. None = local persistent mode.
+        port: ChromaDB server port. Default 8000.
+        persist_directory: Local storage path (local mode only).
+            Default "~/.fi/feedback/chroma".
+        collection_prefix: Prefix for ChromaDB collection names.
+        embedding_model: LiteLLM model string for embeddings.
+            None = use ChromaDB's default (sentence-transformers).
+    """
+
+    def __init__(
+        self,
+        host: Optional[str] = None,
+        port: int = 8000,
+        persist_directory: Optional[str] = None,
+        collection_prefix: str = "fi_feedback",
+        embedding_model: Optional[str] = None,
+    ):
+        try:
+            import chromadb
+        except ImportError:
+            raise ImportError(
+                "chromadb is required for ChromaFeedbackStore. "
+                "Install it with: pip install ai-evaluation[feedback]"
+            )
+
+        self._collection_prefix = collection_prefix
+        self._embedding_model = embedding_model
+
+        # Initialize ChromaDB client
+        if host:
+            self._client = chromadb.HttpClient(host=host, port=port)
+            logger.info(f"Connected to ChromaDB server at {host}:{port}")
+        else:
+            import os
+            path = persist_directory or os.path.expanduser("~/.fi/feedback/chroma")
+            os.makedirs(path, exist_ok=True)
+            self._client = chromadb.PersistentClient(path=path)
+            logger.info(f"Using local ChromaDB at {path}")
+
+        # Set up embedding function
+        self._embedding_fn = None
+        if embedding_model:
+            self._embedding_fn = self._make_litellm_embedding_fn(embedding_model)
+
+    @staticmethod
+    def _make_litellm_embedding_fn(model: str):
+        """Create a ChromaDB-compatible embedding function using LiteLLM."""
+        from chromadb.api.types import EmbeddingFunction, Documents, Embeddings
+        import litellm
+
+        class LiteLLMEmbedding(EmbeddingFunction):
+            def __call__(self, input: Documents) -> Embeddings:
+                response = litellm.embedding(model=model, input=input)
+                return [item["embedding"] for item in response.data]
+
+        return LiteLLMEmbedding()
+
+    def _get_collection(self, metric_name: str):
+        """Get or create a ChromaDB collection for a specific metric."""
+        name = f"{self._collection_prefix}_{metric_name}".replace(".", "_")
+        kwargs: Dict[str, Any] = {"name": name}
+        if self._embedding_fn:
+            kwargs["embedding_function"] = self._embedding_fn
+        return self._client.get_or_create_collection(**kwargs)
+
+    def add(self, entry: FeedbackEntry) -> str:
+        collection = self._get_collection(entry.eval_name)
+        collection.add(
+            ids=[entry.id],
+            documents=[entry.to_embedding_text()],
+            metadatas=[{
+                "metric_name": entry.eval_name,
+                "original_score": entry.original_score or 0.0,
+                "correct_score": entry.correct_score if entry.correct_score is not None else -1.0,
+                "correct_reason": entry.correct_reason[:1000],
+                "inputs_json": json.dumps(entry.inputs, default=str)[:4000],
+                "original_reason": entry.original_reason[:1000],
+                "created_at": entry.created_at.isoformat(),
+            }],
+        )
+        return entry.id
+
+    def query_similar(
+        self,
+        metric_name: str,
+        text: str,
+        n_results: int = 5,
+    ) -> List[FeedbackEntry]:
+        collection = self._get_collection(metric_name)
+
+        if collection.count() == 0:
+            return []
+
+        actual_n = min(n_results, collection.count())
+
+        results = collection.query(
+            query_texts=[text],
+            n_results=actual_n,
+        )
+
+        entries = []
+        for i, meta in enumerate(results["metadatas"][0]):
+            inputs = {}
+            try:
+                inputs = json.loads(meta.get("inputs_json", "{}"))
+            except (json.JSONDecodeError, TypeError):
+                pass
+
+            correct_score_val = meta.get("correct_score", -1.0)
+            entry = FeedbackEntry(
+                id=results["ids"][0][i],
+                eval_name=meta.get("metric_name", metric_name),
+                inputs=inputs,
+                original_score=meta.get("original_score"),
+                original_reason=meta.get("original_reason", ""),
+                correct_score=correct_score_val if correct_score_val >= 0 else None,
+                correct_reason=meta.get("correct_reason", ""),
+            )
+            entries.append(entry)
+
+        return entries
+
+    def get_by_metric(self, metric_name: str, limit: int = 100) -> List[FeedbackEntry]:
+        collection = self._get_collection(metric_name)
+        if collection.count() == 0:
+            return []
+        results = collection.get(limit=limit)
+        entries = []
+        for i, meta in enumerate(results["metadatas"]):
+            inputs = {}
+            try:
+                inputs = json.loads(meta.get("inputs_json", "{}"))
+            except (json.JSONDecodeError, TypeError):
+                pass
+            correct_score_val = meta.get("correct_score", -1.0)
+            entry = FeedbackEntry(
+                id=results["ids"][i],
+                eval_name=meta.get("metric_name", metric_name),
+                inputs=inputs,
+                original_score=meta.get("original_score"),
+                original_reason=meta.get("original_reason", ""),
+                correct_score=correct_score_val if correct_score_val >= 0 else None,
+                correct_reason=meta.get("correct_reason", ""),
+            )
+            entries.append(entry)
+        return entries
+
+    def count(self, metric_name: Optional[str] = None) -> int:
+        if metric_name:
+            return self._get_collection(metric_name).count()
+        total = 0
+        for col in self._client.list_collections():
+            if col.name.startswith(self._collection_prefix):
+                total += col.count()
+        return total
+
+    def delete(self, entry_id: str) -> bool:
+        for col in self._client.list_collections():
+            if col.name.startswith(self._collection_prefix):
+                try:
+                    col.delete(ids=[entry_id])
+                    return True
+                except Exception:
+                    continue
+        return False

fi/evals/feedback/types.py

@@ -0,0 +1,99 @@
+"""Type definitions for the Feedback Loop system.
+
+Provides dataclasses for storing developer feedback on evaluation results,
+calibration profiles, and aggregate statistics.
+"""
+
+import json
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class FeedbackEntry:
+    """A single piece of developer feedback on an evaluation result."""
+
+    id: str = field(default_factory=lambda: str(uuid.uuid4()))
+
+    # What was evaluated
+    eval_name: str = ""
+    inputs: Dict[str, Any] = field(default_factory=dict)
+
+    # What the system produced
+    original_score: Optional[float] = None
+    original_reason: str = ""
+    original_passed: Optional[bool] = None
+
+    # What the developer says is correct
+    correct_score: Optional[float] = None
+    correct_passed: Optional[bool] = None
+    correct_reason: str = ""
+
+    # Metadata
+    tags: List[str] = field(default_factory=list)
+    created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def to_few_shot(self) -> Dict[str, Any]:
+        """Convert to the format expected by CustomLLMJudge few_shot_examples.
+
+        Returns dict matching the template schema:
+            {"inputs": {...}, "output": "<json with score/reason>"}
+        """
+        output_dict = {
+            "score": self.correct_score if self.correct_score is not None else self.original_score,
+            "reason": self.correct_reason or self.original_reason,
+        }
+        return {
+            "inputs": self.inputs,
+            "output": json.dumps(output_dict),
+        }
+
+    def to_embedding_text(self) -> str:
+        """Create a text representation for embedding.
+
+        Concatenates key input fields into a string suitable for
+        semantic embedding. Prioritizes output, context, and input fields.
+        """
+        parts = [f"metric: {self.eval_name}"]
+        for key in ("output", "response", "context", "input", "query"):
+            val = self.inputs.get(key)
+            if val:
+                text = val if isinstance(val, str) else json.dumps(val, default=str)
+                parts.append(f"{key}: {text[:500]}")
+        return "\n".join(parts)
+
+
+@dataclass
+class CalibrationProfile:
+    """Optimized threshold settings for a metric based on feedback."""
+
+    eval_name: str
+    optimal_threshold: float
+    sample_size: int
+    accuracy_at_threshold: float  # % of feedback entries that agree at this threshold
+
+    # Distribution stats
+    score_mean: float = 0.0
+    score_std: float = 0.0
+
+    # Confusion matrix at the optimal threshold
+    true_positives: int = 0
+    false_positives: int = 0
+    true_negatives: int = 0
+    false_negatives: int = 0
+
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class FeedbackStats:
+    """Aggregate statistics for feedback on a given metric."""
+
+    eval_name: str
+    total_entries: int = 0
+    agreement_rate: float = 0.0  # How often original == correct
+    avg_score_delta: float = 0.0  # avg(correct_score - original_score)
+    score_distribution: Dict[str, int] = field(default_factory=dict)  # buckets

fi/evals/framework/README.md

@@ -0,0 +1,79 @@
+# Evaluation Framework
+
+A scalable evaluation infrastructure for AI systems with support for blocking, non-blocking, and distributed execution modes.
+
+## Quick Start
+
+```python
+from fi.evals import FrameworkEvaluator, ExecutionMode
+from fi.evals.framework.evals import CoherenceEval, ActionSafetyEval
+
+# Create an evaluator with multiple evaluations
+evaluator = FrameworkEvaluator(
+    evaluations=[
+        CoherenceEval(),
+        ActionSafetyEval(),
+    ],
+    mode=ExecutionMode.BLOCKING,
+)
+
+# Run evaluations
+result = evaluator.run({
+    "response": "Paris is the capital of France. It is in Western Europe.",
+    "trajectory": [
+        {"type": "tool_call", "tool": "search", "args": "Paris facts"},
+    ],
+})
+
+# Check results
+for r in result.results:
+    print(f"{r.eval_name}: score={r.value.score:.2f}, passed={r.value.passed}")
+```
+
+## Execution Modes
+
+| Mode | Use Case | Latency Impact |
+|------|----------|----------------|
+| `BLOCKING` | Development, testing, sync workflows | Full evaluation time |
+| `NON_BLOCKING` | Production, real-time applications | Zero (async) |
+| `DISTRIBUTED` | Batch processing, high throughput | Zero (remote) |
+
+## Available Evaluations
+
+### Semantic Evaluations
+- `CoherenceEval` - Check text coherence
+
+### Agentic Evaluations
+- `ActionSafetyEval` - Safety scanning
+- `ReasoningQualityEval` - Reasoning quality
+
+### Custom Evaluation Builders
+- `EvalBuilder` - Fluent builder pattern
+- `@custom_eval` - Decorator for functions
+- `simple_eval()` - Score-based evaluation
+- `comparison_eval()` - Compare two fields
+- `threshold_eval()` - Min/max thresholds
+- `pattern_match_eval()` - Regex patterns
+
+## OpenTelemetry Integration
+
+All evaluations automatically generate span attributes compatible with OpenTelemetry:
+
+```python
+from fi.evals.framework import register_current_span, async_evaluator
+
+# Register span for cross-thread enrichment
+with tracer.start_as_current_span("llm_call") as span:
+    register_current_span()
+
+    response = llm.complete(prompt)
+    evaluator.run({"response": response})  # Enriches span automatically
+
+    return response
+
+# Span attributes include:
+#   eval.coherence.score
+#   eval.coherence.passed
+#   eval.action_safety.score
+#   etc.
+```