agent-learning-kit 0.1.0__py3-none-any.whl

fi/evals/execution.py

@@ -0,0 +1,168 @@
+"""
+Execution handles for async eval and composite runs.
+
+An ``Execution`` is a lightweight view into a (possibly still-running) eval
+on the backend (for single evals) or a background thread (for composite
+evals, which the backend runs synchronously).
+
+Typical usage::
+
+    from fi.evals import Evaluator, EvalTemplateManager
+
+    ev = Evaluator()
+    mgr = EvalTemplateManager()
+
+    # --- Single eval: real backend async via is_async=True ---
+    handle = ev.submit("tone", {"output": "I love this!"})
+    handle.wait()                # polls until completion
+    print(handle.result.output)  # -> "love"
+
+    # --- Composite eval: SDK-side threaded execution ---
+    handle = mgr.submit_composite(composite_id, mapping={"output": "Hi!"})
+    handle.wait()
+    print(handle.result["aggregate_score"])
+
+    # --- Resumable by ID (single eval only) ---
+    other_handle = ev.get_execution(handle.id)
+    other_handle.wait()
+
+Note on composite executions: the handle lives in a background thread
+inside the calling process. If the process dies or the handle is dropped,
+the in-flight work is lost. Use single-eval async submissions when you
+need cross-process resumability.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, Optional
+
+
+class ExecutionError(Exception):
+    """Raised when an Execution that finished in the ``failed`` state is awaited."""
+
+
+# Backend eval_status → SDK-normalized status.
+_STATUS_MAP = {
+    "pending": "pending",
+    "PENDING": "pending",
+    "processing": "processing",
+    "PROCESSING": "processing",
+    "running": "processing",
+    "completed": "completed",
+    "COMPLETED": "completed",
+    "failed": "failed",
+    "FAILED": "failed",
+}
+
+
+def _normalize_status(raw: Optional[str]) -> str:
+    if not raw:
+        return "pending"
+    return _STATUS_MAP.get(raw, raw.lower())
+
+
+@dataclass
+class Execution:
+    """
+    Handle to an in-flight or completed eval execution.
+
+    Attributes:
+        id: Execution identifier. For single evals this is the server-side
+            ``eval_id`` (a UUID) and is resumable from any process via
+            :py:meth:`fi.evals.Evaluator.get_execution`. For composite
+            evals this is a client-side UUID — see the module docstring
+            for the caveat.
+        kind: ``"eval"`` for a single eval execution, ``"composite"`` for
+            a composite one.
+        status: ``"pending"`` | ``"processing"`` | ``"completed"`` |
+            ``"failed"``.
+        result: Populated once ``status == "completed"``. An ``EvalResult``
+            for single evals; a dict matching the composite execute
+            response for composite ones.
+        error_message: Populated when the execution failed.
+        error_localizer: Populated for single evals when error
+            localization was enabled and the analysis is available.
+    """
+
+    id: str
+    kind: str
+    status: str = "pending"
+    result: Any = None
+    error_message: Optional[str] = None
+    error_localizer: Optional[Dict[str, Any]] = None
+
+    # Closure that (re)fetches the latest state. Set by the factory
+    # method on Evaluator / EvalTemplateManager. Excluded from repr.
+    _refresher: Optional[Callable[[], "Execution"]] = field(
+        default=None, repr=False, compare=False
+    )
+
+    def is_done(self) -> bool:
+        """Return True if status is terminal (``completed`` or ``failed``)."""
+        return self.status in ("completed", "failed")
+
+    def refresh(self) -> "Execution":
+        """
+        Re-fetch the latest state from the source (backend for single
+        evals, background thread for composites). Returns ``self``.
+        """
+        if self._refresher is None:
+            return self
+        updated = self._refresher()
+        self.status = updated.status
+        self.result = updated.result
+        self.error_message = updated.error_message
+        self.error_localizer = updated.error_localizer
+        return self
+
+    def wait(
+        self,
+        *,
+        timeout: float = 300.0,
+        poll_interval: float = 2.0,
+        raise_on_failure: bool = True,
+    ) -> "Execution":
+        """
+        Block until the execution reaches a terminal state, refreshing
+        every ``poll_interval`` seconds.
+
+        Args:
+            timeout: Maximum number of seconds to wait before giving up.
+            poll_interval: Seconds between refreshes.
+            raise_on_failure: If True (default) raise
+                :class:`ExecutionError` when the run finished in
+                ``"failed"`` state. If False, return the handle so the
+                caller can inspect ``error_message`` themselves.
+
+        Raises:
+            TimeoutError: If the execution did not reach a terminal
+                state within ``timeout`` seconds.
+            ExecutionError: If the execution failed and
+                ``raise_on_failure=True``.
+        """
+        if self.is_done():
+            if self.status == "failed" and raise_on_failure:
+                raise ExecutionError(
+                    f"Execution {self.id} failed: {self.error_message}"
+                )
+            return self
+
+        deadline = time.monotonic() + float(timeout)
+        while True:
+            time.sleep(poll_interval)
+            self.refresh()
+            if self.is_done():
+                break
+            if time.monotonic() > deadline:
+                raise TimeoutError(
+                    f"Execution {self.id} did not complete within {timeout}s "
+                    f"(last status: {self.status})"
+                )
+
+        if self.status == "failed" and raise_on_failure:
+            raise ExecutionError(
+                f"Execution {self.id} failed: {self.error_message}"
+            )
+        return self

fi/evals/feedback/__init__.py

@@ -0,0 +1,32 @@
+"""Feedback Loop system for improving evaluations over time.
+
+Store developer feedback on metric results, retrieve similar past feedback
+as few-shot examples for LLM judges, and calibrate thresholds statistically.
+"""
+
+from .types import FeedbackEntry, CalibrationProfile, FeedbackStats
+from .store import FeedbackStore, InMemoryFeedbackStore
+from .collector import FeedbackCollector
+from .retriever import FeedbackRetriever
+from .calibrator import ThresholdCalibrator
+from .hooks import configure_feedback, get_default_store
+
+# ChromaFeedbackStore requires chromadb — import conditionally
+try:
+    from .store import ChromaFeedbackStore
+except Exception:
+    pass
+
+__all__ = [
+    "FeedbackEntry",
+    "CalibrationProfile",
+    "FeedbackStats",
+    "FeedbackStore",
+    "InMemoryFeedbackStore",
+    "ChromaFeedbackStore",
+    "FeedbackCollector",
+    "FeedbackRetriever",
+    "ThresholdCalibrator",
+    "configure_feedback",
+    "get_default_store",
+]

fi/evals/feedback/calibrator.py

@@ -0,0 +1,160 @@
+"""Statistical threshold calibration based on feedback.
+
+Optimizes pass/fail thresholds by computing confusion matrices against
+developer-provided correct labels across a range of threshold values.
+"""
+
+import logging
+import math
+from typing import List, Tuple
+
+from .store import FeedbackStore
+from .types import CalibrationProfile, FeedbackEntry
+
+logger = logging.getLogger(__name__)
+
+
+class ThresholdCalibrator:
+    """Optimizes pass/fail thresholds based on accumulated feedback.
+
+    For each candidate threshold, computes a confusion matrix against
+    the developer's correct_passed labels, and selects the threshold
+    that maximizes agreement (accuracy) or F1 score.
+
+    Args:
+        store: FeedbackStore containing feedback entries.
+        optimize_for: Metric to optimize. "accuracy" (default) or "f1".
+    """
+
+    def __init__(
+        self,
+        store: FeedbackStore,
+        optimize_for: str = "accuracy",
+    ):
+        self.store = store
+        self.optimize_for = optimize_for
+
+    def calibrate(
+        self,
+        metric_name: str,
+        threshold_range: Tuple[float, float] = (0.3, 0.9),
+        steps: int = 13,
+    ) -> CalibrationProfile:
+        """Find the optimal pass/fail threshold for a metric.
+
+        Args:
+            metric_name: The metric to calibrate.
+            threshold_range: (min_threshold, max_threshold) to search.
+            steps: Number of evenly-spaced thresholds to try.
+
+        Returns:
+            CalibrationProfile with the optimal threshold and stats.
+
+        Raises:
+            ValueError: If insufficient feedback (< 5 entries with corrections).
+        """
+        entries = self.store.get_by_metric(metric_name)
+
+        # Filter to entries with both a correct label and a score
+        usable = [
+            e for e in entries
+            if e.correct_score is not None
+            and e.original_score is not None
+        ]
+
+        if len(usable) < 5:
+            raise ValueError(
+                f"Need at least 5 feedback entries with corrections to calibrate "
+                f"'{metric_name}', but only {len(usable)} found. Submit more feedback first."
+            )
+
+        # Derive correct_passed if not explicitly set
+        for e in usable:
+            if e.correct_passed is None:
+                e.correct_passed = e.correct_score >= 0.5
+
+        # Search over threshold space
+        min_t, max_t = threshold_range
+        best_score = -1.0
+        best_threshold = 0.5
+        best_matrix = (0, 0, 0, 0)
+
+        for i in range(steps):
+            t = min_t + (max_t - min_t) * i / (steps - 1) if steps > 1 else (min_t + max_t) / 2
+            tp, fp, tn, fn = self._confusion_matrix(usable, t)
+
+            if self.optimize_for == "f1":
+                score = self._f1(tp, fp, fn)
+            else:
+                total = tp + fp + tn + fn
+                score = (tp + tn) / total if total > 0 else 0.0
+
+            if score > best_score:
+                best_score = score
+                best_threshold = t
+                best_matrix = (tp, fp, tn, fn)
+
+        # Compute score statistics
+        scores = [e.correct_score for e in usable if e.correct_score is not None]
+        mean = sum(scores) / len(scores) if scores else 0.0
+        variance = sum((s - mean) ** 2 for s in scores) / len(scores) if scores else 0.0
+
+        tp, fp, tn, fn = best_matrix
+
+        profile = CalibrationProfile(
+            eval_name=metric_name,
+            optimal_threshold=round(best_threshold, 3),
+            sample_size=len(usable),
+            accuracy_at_threshold=best_score,
+            score_mean=round(mean, 4),
+            score_std=round(math.sqrt(variance), 4),
+            true_positives=tp,
+            false_positives=fp,
+            true_negatives=tn,
+            false_negatives=fn,
+        )
+
+        logger.info(
+            f"Calibrated '{metric_name}': threshold={profile.optimal_threshold} "
+            f"accuracy={profile.accuracy_at_threshold:.1%} (n={profile.sample_size})"
+        )
+
+        return profile
+
+    @staticmethod
+    def _confusion_matrix(
+        entries: List[FeedbackEntry],
+        threshold: float,
+    ) -> Tuple[int, int, int, int]:
+        """Compute confusion matrix at a given threshold.
+
+        Predicted positive = original_score >= threshold
+        Actual positive = correct_passed is True
+
+        Returns:
+            (true_positives, false_positives, true_negatives, false_negatives)
+        """
+        tp = fp = tn = fn = 0
+        for e in entries:
+            predicted_pass = e.original_score >= threshold
+            actual_pass = e.correct_passed
+
+            if predicted_pass and actual_pass:
+                tp += 1
+            elif predicted_pass and not actual_pass:
+                fp += 1
+            elif not predicted_pass and not actual_pass:
+                tn += 1
+            else:
+                fn += 1
+
+        return tp, fp, tn, fn
+
+    @staticmethod
+    def _f1(tp: int, fp: int, fn: int) -> float:
+        """Compute F1 score from confusion matrix components."""
+        precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+        recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+        if precision + recall == 0:
+            return 0.0
+        return 2 * precision * recall / (precision + recall)

fi/evals/feedback/collector.py

@@ -0,0 +1,214 @@
+"""User-facing API for the feedback loop system.
+
+Provides a clean interface for submitting feedback, querying statistics,
+calibrating thresholds, and creating retrievers.
+"""
+
+import logging
+from typing import Any, Dict, List, Optional
+
+from ..core.result import EvalResult
+from .store import FeedbackStore
+from .types import FeedbackEntry, FeedbackStats
+from .retriever import FeedbackRetriever
+from .calibrator import ThresholdCalibrator
+
+logger = logging.getLogger(__name__)
+
+
+class FeedbackCollector:
+    """Main user-facing class for the feedback loop system.
+
+    Provides a clean API for:
+    - Submitting feedback on metric results
+    - Retrieving statistics on accumulated feedback
+    - Calibrating thresholds based on feedback
+    - Creating a retriever for pipeline integration
+
+    Usage:
+        from fi.evals.feedback import FeedbackCollector, InMemoryFeedbackStore
+
+        store = InMemoryFeedbackStore()  # or ChromaFeedbackStore()
+        feedback = FeedbackCollector(store)
+
+        # After running a metric that gave wrong results:
+        result = run_metric("faithfulness", output="...", context="...")
+
+        # Submit correction
+        feedback.submit(
+            result,
+            inputs={"output": "...", "context": "..."},
+            correct_score=0.9,
+            correct_reason="The response IS faithful via semantic equivalence.",
+        )
+
+        # Later, get a retriever for pipeline integration
+        retriever = feedback.get_retriever()
+
+    Args:
+        store: The FeedbackStore backend.
+    """
+
+    def __init__(self, store: FeedbackStore):
+        self.store = store
+
+    def submit(
+        self,
+        result: EvalResult,
+        *,
+        inputs: Dict[str, Any],
+        correct_score: Optional[float] = None,
+        correct_passed: Optional[bool] = None,
+        correct_reason: str = "",
+        tags: Optional[List[str]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> FeedbackEntry:
+        """Submit feedback on a metric result.
+
+        Args:
+            result: The EvalResult that needs correction.
+            inputs: The original inputs that were used.
+            correct_score: What the score SHOULD have been (0.0-1.0).
+            correct_passed: What the pass/fail SHOULD have been.
+            correct_reason: Why the original result was wrong.
+            tags: Optional tags for organizing feedback.
+            metadata: Optional metadata dict.
+
+        Returns:
+            The stored FeedbackEntry.
+
+        Raises:
+            ValueError: If neither correct_score nor correct_reason is provided.
+        """
+        if correct_score is None and not correct_reason:
+            raise ValueError(
+                "Feedback must include at least one of: correct_score, correct_reason. "
+                "If the result was correct, use confirm() instead."
+            )
+
+        entry = FeedbackEntry(
+            eval_name=result.eval_name,
+            inputs=inputs,
+            original_score=result.score,
+            original_reason=result.reason,
+            original_passed=result.passed,
+            correct_score=correct_score,
+            correct_passed=correct_passed,
+            correct_reason=correct_reason,
+            tags=tags or [],
+            metadata=metadata or {},
+        )
+
+        self.store.add(entry)
+        logger.info(
+            f"Feedback submitted for '{result.eval_name}': "
+            f"original={result.score} -> corrected={correct_score}"
+        )
+        return entry
+
+    def confirm(
+        self,
+        result: EvalResult,
+        *,
+        inputs: Dict[str, Any],
+        reason: str = "",
+    ) -> FeedbackEntry:
+        """Confirm that a metric result was correct.
+
+        Records that the system got it right, which helps calibration
+        accuracy calculations. These entries are stored but NOT injected
+        as few-shot examples (since they don't correct anything).
+
+        Args:
+            result: The correct EvalResult.
+            inputs: The original inputs.
+            reason: Optional note on why this was correct.
+
+        Returns:
+            The stored FeedbackEntry.
+        """
+        entry = FeedbackEntry(
+            eval_name=result.eval_name,
+            inputs=inputs,
+            original_score=result.score,
+            original_reason=result.reason,
+            original_passed=result.passed,
+            correct_score=result.score,  # Same as original = confirmed correct
+            correct_passed=result.passed,
+            correct_reason=reason or "Confirmed correct by developer.",
+            tags=["confirmed"],
+        )
+        self.store.add(entry)
+        return entry
+
+    def stats(self, metric_name: str) -> FeedbackStats:
+        """Get aggregate statistics for feedback on a metric.
+
+        Args:
+            metric_name: The metric to get stats for.
+
+        Returns:
+            FeedbackStats with counts and agreement rates.
+        """
+        entries = self.store.get_by_metric(metric_name)
+
+        if not entries:
+            return FeedbackStats(eval_name=metric_name)
+
+        total = len(entries)
+        agreements = 0
+        score_deltas = []
+
+        for e in entries:
+            if e.correct_score is not None and e.original_score is not None:
+                delta = e.correct_score - e.original_score
+                score_deltas.append(delta)
+                # Agreement = within 0.1 of each other
+                if abs(delta) < 0.1:
+                    agreements += 1
+
+        # Score distribution in 0.1 buckets
+        distribution: Dict[str, int] = {}
+        for e in entries:
+            score = e.correct_score if e.correct_score is not None else e.original_score
+            if score is not None:
+                bucket = f"{int(score * 10) / 10:.1f}"
+                distribution[bucket] = distribution.get(bucket, 0) + 1
+
+        return FeedbackStats(
+            eval_name=metric_name,
+            total_entries=total,
+            agreement_rate=agreements / total if total > 0 else 0.0,
+            avg_score_delta=sum(score_deltas) / len(score_deltas) if score_deltas else 0.0,
+            score_distribution=distribution,
+        )
+
+    def get_retriever(self, max_examples: int = 3) -> FeedbackRetriever:
+        """Create a FeedbackRetriever wired to this collector's store.
+
+        Args:
+            max_examples: Max few-shot examples to retrieve per query.
+
+        Returns:
+            A FeedbackRetriever instance.
+        """
+        return FeedbackRetriever(store=self.store, max_examples=max_examples)
+
+    def calibrate(
+        self,
+        metric_name: str,
+        threshold_range: tuple = (0.3, 0.9),
+        steps: int = 13,
+    ):
+        """Run threshold calibration for a metric.
+
+        Args:
+            metric_name: Metric to calibrate.
+            threshold_range: (min, max) thresholds to search.
+            steps: Number of threshold steps to try.
+
+        Returns:
+            CalibrationProfile with optimal threshold.
+        """
+        calibrator = ThresholdCalibrator(self.store)
+        return calibrator.calibrate(metric_name, threshold_range, steps)

fi/evals/feedback/hooks.py

@@ -0,0 +1,81 @@
+"""Integration hooks for wiring feedback into the pipeline.
+
+These functions are called from the augmentation flow when a feedback_store
+is provided. Kept in a separate module to avoid circular imports.
+"""
+
+import logging
+from typing import Any, Dict, Optional
+
+from .store import FeedbackStore
+from .retriever import FeedbackRetriever
+
+logger = logging.getLogger(__name__)
+
+# Module-level default store (set via configure_feedback)
+_default_store: Optional[FeedbackStore] = None
+_default_max_examples: int = 3
+
+
+def configure_feedback(
+    store: FeedbackStore,
+    max_examples: int = 3,
+) -> None:
+    """Set a global default feedback store for all augmented metric runs.
+
+    After calling this, all augmented runs will automatically retrieve
+    feedback examples -- no need to pass feedback_store= every time.
+
+    Args:
+        store: The FeedbackStore to use globally.
+        max_examples: Max few-shot examples per query.
+
+    Usage:
+        from fi.evals.feedback import ChromaFeedbackStore, configure_feedback
+
+        store = ChromaFeedbackStore()
+        configure_feedback(store)
+
+        # Now all augmented runs automatically use feedback
+        result = run_metric("faithfulness", ..., augment=True, model="gemini/...")
+    """
+    global _default_store, _default_max_examples
+    _default_store = store
+    _default_max_examples = max_examples
+    logger.info(f"Feedback configured globally (max_examples={max_examples})")
+
+
+def get_default_store() -> Optional[FeedbackStore]:
+    """Get the globally configured feedback store, if any."""
+    return _default_store
+
+
+def retrieve_feedback_config(
+    metric_name: str,
+    inputs: Dict[str, Any],
+    store: Optional[FeedbackStore] = None,
+    config: Optional[Dict[str, Any]] = None,
+    max_examples: Optional[int] = None,
+) -> Dict[str, Any]:
+    """Retrieve feedback examples and inject into config dict.
+
+    Called from the augmentation flow. Can also be called directly.
+
+    Args:
+        metric_name: Metric being run.
+        inputs: Current inputs.
+        store: Explicit store override. Falls back to global default.
+        config: Existing config dict to merge into.
+        max_examples: Override for max examples.
+
+    Returns:
+        Config dict with few_shot_examples populated (or unchanged if
+        no store is configured / no feedback found).
+    """
+    effective_store = store or _default_store
+    if effective_store is None:
+        return dict(config or {})
+
+    n = max_examples or _default_max_examples
+    retriever = FeedbackRetriever(store=effective_store, max_examples=n)
+    return retriever.inject_into_config(metric_name, inputs, config)