agent-learning-kit 0.1.0__py3-none-any.whl

fi/evals/framework/__init__.py

@@ -0,0 +1,267 @@
+"""
+Evaluation Framework - Scalable evaluation infrastructure.
+
+This module provides a unified framework for running evaluations in different modes:
+- Blocking (synchronous): For development and when results are needed immediately
+- Non-blocking (async): For production with zero latency impact
+- Distributed: For batch processing at scale via pluggable backends
+
+Key Features:
+- Trace context propagation across threads/processes
+- Automatic span enrichment with evaluation results
+- Pluggable backends (thread pool built-in, Temporal/Celery/Ray extensible)
+- Write evaluation logic once, run anywhere
+
+Quick Start:
+    from fi.evals.framework import Evaluator, ExecutionMode, register_evaluation
+
+    # Define an evaluation
+    @register_evaluation
+    class MyEval:
+        name = "my_eval"
+        version = "1.0.0"
+
+        def evaluate(self, inputs):
+            score = compute_score(inputs["response"])
+            return {"score": score, "passed": score > 0.7}
+
+        def get_span_attributes(self, result):
+            return {"score": result["score"], "passed": result["passed"]}
+
+    # Run blocking (development)
+    evaluator = Evaluator([MyEval()], mode=ExecutionMode.BLOCKING)
+    result = evaluator.run({"response": "..."})
+    print(result.results[0].value)
+
+    # Run non-blocking (production - zero latency)
+    evaluator = Evaluator([MyEval()], mode=ExecutionMode.NON_BLOCKING)
+    result = evaluator.run({"response": "..."})  # Returns immediately
+    batch = result.wait()  # Get results when needed
+
+Factory Functions:
+    # Convenience functions for common configurations
+    from fi.evals.framework import blocking_evaluator, async_evaluator
+
+    # Blocking
+    evaluator = blocking_evaluator(MyEval())
+    result = evaluator.run(inputs)
+
+    # Async (non-blocking)
+    evaluator = async_evaluator(MyEval(), max_workers=4)
+    result = evaluator.run(inputs)
+    batch = result.wait()
+
+Example with OpenTelemetry:
+    from fi.evals.framework import async_evaluator, register_current_span
+
+    evaluator = async_evaluator(ToxicityEval(), BiasEval())
+
+    with tracer.start_as_current_span("llm_call") as span:
+        register_current_span()  # Enable cross-thread enrichment
+        response = llm.complete(prompt)
+        evaluator.run({"response": response})  # Zero latency
+        return response
+
+    # Span automatically enriched with:
+    #   eval.toxicity.score, eval.toxicity.status
+    #   eval.bias.score, eval.bias.status
+"""
+
+__version__ = "1.0.0"
+
+# Core types
+from .types import (
+    ExecutionMode,
+    EvalStatus,
+    FrameworkEvalResult,
+    BatchEvalResult,
+    EvalInputs,
+    SpanAttributes,
+)
+
+# Context
+from .context import (
+    EvalContext,
+    get_current_context,
+    create_standalone_context,
+)
+
+# Protocols
+from .protocols import (
+    BaseEvaluation,
+    EvalRegistry,
+    register_evaluation,
+    create_evaluation,
+)
+
+# Enrichment
+from .enrichment import (
+    enrich_current_span,
+    enrich_span,
+    add_eval_event,
+    get_current_span,
+    is_span_recording,
+    flatten_attributes,
+    SpanEnricher,
+)
+
+# Registry
+from .registry import (
+    SpanRegistry,
+    register_span,
+    get_span,
+    unregister_span,
+    get_registry,
+    register_current_span,
+)
+
+# Propagation
+from .propagation import (
+    SpanContextPropagator,
+    enrich_span_by_context,
+    enrich_span_by_ids,
+    add_event_by_context,
+    ContextCarrier,
+    propagate_context,
+    propagate_context_lazy,
+)
+
+# Evaluators
+from .evaluators import (
+    BlockingEvaluator,
+    blocking_evaluate,
+    NonBlockingEvaluator,
+    non_blocking_evaluate,
+    EvalFuture,
+    BatchEvalFuture,
+    EvalResultAggregator,
+)
+
+# Backends
+from .backends import (
+    Backend,
+    BackendConfig,
+    TaskHandle,
+    TaskStatus,
+    ThreadPoolBackend,
+    ThreadPoolConfig,
+)
+
+# Resilience
+from .resilience import (
+    ResilientBackend,
+    ResilienceConfig,
+    CircuitBreakerConfig,
+    RateLimitConfig,
+    RetryConfig,
+    DegradationConfig,
+    HealthCheckConfig,
+    wrap_backend,
+)
+
+# Built-in evals + builder
+from .evals import (
+    custom_eval,
+    simple_eval,
+    EvalBuilder,
+)
+
+# Unified API
+from .evaluator import (
+    FrameworkEvaluator,
+    EvaluatorResult,
+    blocking_evaluator,
+    async_evaluator,
+    distributed_evaluator,
+    resilient_evaluator,
+)
+
+# Backwards-compatible alias used by tests and examples
+Evaluator = FrameworkEvaluator
+
+__all__ = [
+    # Version
+    "__version__",
+
+    # Types
+    "ExecutionMode",
+    "EvalStatus",
+    "FrameworkEvalResult",
+    "BatchEvalResult",
+    "EvalInputs",
+    "SpanAttributes",
+
+    # Context
+    "EvalContext",
+    "get_current_context",
+    "create_standalone_context",
+
+    # Protocols
+    "BaseEvaluation",
+    "EvalRegistry",
+    "register_evaluation",
+    "create_evaluation",
+
+    # Enrichment
+    "enrich_current_span",
+    "enrich_span",
+    "add_eval_event",
+    "get_current_span",
+    "is_span_recording",
+    "flatten_attributes",
+    "SpanEnricher",
+
+    # Registry
+    "SpanRegistry",
+    "register_span",
+    "get_span",
+    "unregister_span",
+    "get_registry",
+    "register_current_span",
+
+    # Propagation
+    "SpanContextPropagator",
+    "enrich_span_by_context",
+    "enrich_span_by_ids",
+    "add_event_by_context",
+    "ContextCarrier",
+    "propagate_context",
+    "propagate_context_lazy",
+
+    # Evaluators - Blocking
+    "BlockingEvaluator",
+    "blocking_evaluate",
+    # Evaluators - Non-Blocking
+    "NonBlockingEvaluator",
+    "non_blocking_evaluate",
+    "EvalFuture",
+    "BatchEvalFuture",
+    "EvalResultAggregator",
+    # Backends
+    "Backend",
+    "BackendConfig",
+    "TaskHandle",
+    "TaskStatus",
+    "ThreadPoolBackend",
+    "ThreadPoolConfig",
+    # Resilience
+    "ResilientBackend",
+    "ResilienceConfig",
+    "CircuitBreakerConfig",
+    "RateLimitConfig",
+    "RetryConfig",
+    "DegradationConfig",
+    "HealthCheckConfig",
+    "wrap_backend",
+    # Built-in evals + builder
+    "custom_eval",
+    "simple_eval",
+    "EvalBuilder",
+    # Unified API
+    "FrameworkEvaluator",
+    "EvaluatorResult",
+    "blocking_evaluator",
+    "async_evaluator",
+    "distributed_evaluator",
+    "resilient_evaluator",
+]

fi/evals/framework/backends/Dockerfile.eval-runner

@@ -0,0 +1,33 @@
+# Eval runner image for the Kubernetes backend.
+#
+# This image is used by KubernetesBackend to execute serialized evaluation
+# tasks inside Kubernetes Jobs. It contains cloudpickle for deserialization
+# and a minimal Python runtime.
+#
+# Build:
+#   docker build -f Dockerfile.eval-runner -t fi-eval-runner:latest .
+#
+# Usage:
+#   from fi.evals.framework.backends import KubernetesBackend, KubernetesConfig
+#
+#   backend = KubernetesBackend(KubernetesConfig(
+#       image="fi-eval-runner:latest",
+#   ))
+
+FROM python:3.11-slim
+
+LABEL maintainer="Future AGI" \
+      description="Eval runner for fi-evals Kubernetes backend"
+
+# Install only the runtime dependency needed for task deserialization.
+# cloudpickle is the industry-standard serializer used by Kubeflow, Ray, etc.
+# It is only used here in trusted evaluation environments — never for untrusted input.
+RUN pip install --no-cache-dir cloudpickle>=3.0
+
+# Drop to non-root user for safety
+RUN useradd --create-home evalrunner
+USER evalrunner
+WORKDIR /home/evalrunner
+
+# The actual task code is injected via the EVAL_PAYLOAD env var at runtime.
+# The KubernetesBackend sets the container command to execute the runner script.

fi/evals/framework/backends/__init__.py

@@ -0,0 +1,99 @@
+"""
+Backend implementations for distributed evaluation.
+
+Provides pluggable backends for running evaluations:
+- ThreadPoolBackend: Local execution using thread pool (default)
+- TemporalBackend: Durable workflows via Temporal (optional)
+- CeleryBackend: Distributed tasks via Celery (optional)
+- RayBackend: Distributed computing via Ray (optional)
+- KubernetesBackend: Cloud-native jobs via Kubernetes (optional)
+
+Container utilities for custom backends:
+- serialize_task / parse_result_from_logs: serialization protocol
+- RUNNER_SCRIPT / RUNNER_COMMAND / DEFAULT_IMAGE: container constants
+- Dockerfile.eval-runner: pre-built eval runner image
+
+Optional backends are lazily imported to avoid requiring their dependencies.
+Install optional dependencies with:
+    pip install fi-evals[temporal]    # For Temporal
+    pip install fi-evals[celery]      # For Celery
+    pip install fi-evals[ray]         # For Ray
+    pip install fi-evals[kubernetes]  # For Kubernetes
+"""
+
+from typing import TYPE_CHECKING
+
+from .base import (
+    Backend,
+    BackendConfig,
+    TaskHandle,
+    TaskStatus,
+)
+from .thread_pool import ThreadPoolBackend, ThreadPoolConfig
+from ._container import (
+    DEFAULT_IMAGE,
+    EVAL_PAYLOAD_ENV,
+    RUNNER_COMMAND,
+    RUNNER_SCRIPT,
+    parse_result_from_logs,
+    serialize_task,
+)
+
+# Type hints for lazy imports (only used by type checkers)
+if TYPE_CHECKING:
+    from .temporal import TemporalBackend, TemporalConfig
+    from .celery_backend import CeleryBackend, CeleryConfig
+    from .ray_backend import RayBackend, RayConfig
+    from .kubernetes_backend import KubernetesBackend, KubernetesConfig
+
+__all__ = [
+    # Base
+    "Backend",
+    "BackendConfig",
+    "TaskHandle",
+    "TaskStatus",
+    # Thread Pool
+    "ThreadPoolBackend",
+    "ThreadPoolConfig",
+    # Temporal (optional)
+    "TemporalBackend",
+    "TemporalConfig",
+    # Celery (optional)
+    "CeleryBackend",
+    "CeleryConfig",
+    # Ray (optional)
+    "RayBackend",
+    "RayConfig",
+    # Kubernetes (optional)
+    "KubernetesBackend",
+    "KubernetesConfig",
+    # Container utilities (for custom backends)
+    "DEFAULT_IMAGE",
+    "EVAL_PAYLOAD_ENV",
+    "RUNNER_COMMAND",
+    "RUNNER_SCRIPT",
+    "serialize_task",
+    "parse_result_from_logs",
+]
+
+# Lazy imports for optional backends
+_LAZY_IMPORTS = {
+    "TemporalBackend": (".temporal", "TemporalBackend"),
+    "TemporalConfig": (".temporal", "TemporalConfig"),
+    "CeleryBackend": (".celery_backend", "CeleryBackend"),
+    "CeleryConfig": (".celery_backend", "CeleryConfig"),
+    "RayBackend": (".ray_backend", "RayBackend"),
+    "RayConfig": (".ray_backend", "RayConfig"),
+    "KubernetesBackend": (".kubernetes_backend", "KubernetesBackend"),
+    "KubernetesConfig": (".kubernetes_backend", "KubernetesConfig"),
+}
+
+
+def __getattr__(name: str):
+    """Lazy import for optional backend dependencies."""
+    if name in _LAZY_IMPORTS:
+        module_name, attr_name = _LAZY_IMPORTS[name]
+        import importlib
+        module = importlib.import_module(module_name, __package__)
+        return getattr(module, attr_name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

fi/evals/framework/backends/_container.py

@@ -0,0 +1,141 @@
+"""
+Shared container utilities for container-based backends.
+
+Provides the serialization protocol, runner script, and result parsing
+used by any backend that executes eval tasks inside containers (Kubernetes,
+Docker, ECS, Nomad, etc.).
+
+Custom backends can reuse these utilities::
+
+    from fi.evals.framework.backends._container import (
+        serialize_task,
+        parse_result_from_logs,
+        RUNNER_SCRIPT,
+        EVAL_PAYLOAD_ENV,
+        DEFAULT_IMAGE,
+    )
+
+    # Serialize (fn, args, kwargs) into a base64 string
+    payload = serialize_task(my_func, (arg1,), {"key": "val"})
+
+    # Pass payload as env var EVAL_PAYLOAD into any container that runs RUNNER_SCRIPT
+
+    # After container finishes, parse the JSON result from its stdout
+    result = parse_result_from_logs(container_stdout)
+
+Security note:
+    cloudpickle is the industry-standard serializer used by Kubeflow Pipelines,
+    Ray, Dask, etc. It is only used here in trusted evaluation environments
+    (your own code running on your own infrastructure) — never for untrusted input.
+"""
+
+import base64
+import json
+import logging
+from typing import Any, Callable, Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+#: Default eval runner image. Build from ``Dockerfile.eval-runner``.
+DEFAULT_IMAGE: str = "fi-eval-runner:latest"
+
+#: Environment variable name for the serialized task payload.
+EVAL_PAYLOAD_ENV: str = "EVAL_PAYLOAD"
+
+#: Python bootstrap script to inject into containers.
+#: Reads EVAL_PAYLOAD env var, deserializes with cloudpickle, runs the
+#: function, and prints a JSON result line to stdout.
+RUNNER_SCRIPT: str = """
+import base64, cloudpickle, json, sys, traceback
+try:
+    import os
+    payload = base64.b64decode(os.environ["EVAL_PAYLOAD"])
+    fn, args, kwargs = cloudpickle.loads(payload)
+    result = fn(*args, **kwargs)
+    print(json.dumps({"status": "success", "result": result}))
+except Exception:
+    tb = traceback.format_exc()
+    print(json.dumps({"status": "error", "error": tb}))
+    sys.exit(1)
+"""
+
+#: Container command that executes the runner script.
+RUNNER_COMMAND: list = ["python", "-c", RUNNER_SCRIPT]
+
+
+# ---------------------------------------------------------------------------
+# Serialization
+# ---------------------------------------------------------------------------
+
+def serialize_task(
+    fn: Callable,
+    args: tuple = (),
+    kwargs: Optional[Dict[str, Any]] = None,
+) -> str:
+    """
+    Serialize a task (function + arguments) into a base64-encoded string.
+
+    The result is safe to pass as an environment variable to a container
+    running ``RUNNER_SCRIPT``.
+
+    Args:
+        fn: The function to execute.
+        args: Positional arguments.
+        kwargs: Keyword arguments.
+
+    Returns:
+        Base64-encoded string of the cloudpickle-serialized payload.
+
+    Raises:
+        ImportError: If cloudpickle is not installed.
+    """
+    import cloudpickle
+
+    kwargs = kwargs or {}
+    payload = cloudpickle.dumps((fn, args, kwargs))
+    return base64.b64encode(payload).decode("utf-8")
+
+
+# ---------------------------------------------------------------------------
+# Result parsing
+# ---------------------------------------------------------------------------
+
+def parse_result_from_logs(logs: str) -> Any:
+    """
+    Parse the task result from container stdout logs.
+
+    The runner script prints a JSON line as its last output. This function
+    walks the log lines in reverse to find and parse that JSON.
+
+    Args:
+        logs: The full stdout text from the container.
+
+    Returns:
+        The deserialized result value.
+
+    Raises:
+        RuntimeError: If the logs contain an error result, are empty,
+                      or don't contain a valid JSON result line.
+    """
+    lines = logs.strip().splitlines()
+    if not lines:
+        raise RuntimeError("Empty container logs — no result found")
+
+    for line in reversed(lines):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            data = json.loads(line)
+            if data.get("status") == "success":
+                return data["result"]
+            elif data.get("status") == "error":
+                raise RuntimeError(data["error"])
+        except json.JSONDecodeError:
+            continue
+
+    raise RuntimeError("No JSON result found in container logs")

fi/evals/framework/backends/_utils.py

@@ -0,0 +1,145 @@
+"""
+Utilities for optional dependency handling.
+
+Provides helpers for gracefully handling missing optional dependencies
+with clear error messages guiding users to install them.
+"""
+
+from typing import Any, Optional
+
+
+class OptionalDependency:
+    """
+    Wrapper for optional imports with helpful error messages.
+
+    Example:
+        TEMPORAL = OptionalDependency("temporalio", "pip install temporalio")
+
+        # Check availability
+        if TEMPORAL.is_available():
+            client = TEMPORAL.require()
+
+        # Or just require (raises if not available)
+        temporalio = TEMPORAL.require()
+    """
+
+    def __init__(
+        self,
+        module_name: str,
+        install_hint: str,
+        extra_name: Optional[str] = None,
+    ):
+        """
+        Initialize optional dependency wrapper.
+
+        Args:
+            module_name: The Python module name to import
+            install_hint: Installation command to show in error message
+            extra_name: Optional extra name for pip install (e.g., 'temporal')
+        """
+        self.module_name = module_name
+        self.install_hint = install_hint
+        self.extra_name = extra_name or module_name
+        self._module: Optional[Any] = None
+        self._checked = False
+        self._available = False
+
+    def is_available(self) -> bool:
+        """Check if the dependency is installed."""
+        if not self._checked:
+            try:
+                self._module = __import__(self.module_name)
+                self._available = True
+            except ImportError:
+                self._available = False
+            self._checked = True
+        return self._available
+
+    def require(self) -> Any:
+        """
+        Require the dependency, raising ImportError if not available.
+
+        Returns:
+            The imported module
+
+        Raises:
+            ImportError: If dependency is not installed
+        """
+        if not self.is_available():
+            raise ImportError(
+                f"'{self.module_name}' is required but not installed. "
+                f"Install it with: {self.install_hint}\n"
+                f"Or install the extra: pip install fi-evals[{self.extra_name}]"
+            )
+        return self._module
+
+    def import_from(self, *names: str) -> tuple:
+        """
+        Import specific names from the module.
+
+        Args:
+            *names: Names to import from the module
+
+        Returns:
+            Tuple of imported objects
+
+        Raises:
+            ImportError: If dependency is not installed
+        """
+        module = self.require()
+        result = []
+        for name in names:
+            parts = name.split(".")
+            obj = module
+            for part in parts:
+                obj = getattr(obj, part)
+            result.append(obj)
+        return tuple(result) if len(result) > 1 else result[0]
+
+
+# Pre-configured optional dependencies
+TEMPORAL = OptionalDependency(
+    "temporalio",
+    "pip install temporalio",
+    "temporal",
+)
+
+CELERY = OptionalDependency(
+    "celery",
+    "pip install 'celery[redis]'",
+    "celery",
+)
+
+RAY = OptionalDependency(
+    "ray",
+    "pip install 'ray[default]'",
+    "ray",
+)
+
+KUBERNETES = OptionalDependency(
+    "kubernetes",
+    "pip install kubernetes",
+    "kubernetes",
+)
+
+
+def check_dependency(name: str) -> bool:
+    """
+    Check if a named dependency is available.
+
+    Args:
+        name: Dependency name ('temporal', 'celery', 'ray', 'kubernetes')
+
+    Returns:
+        True if available, False otherwise
+    """
+    deps = {
+        "temporal": TEMPORAL,
+        "celery": CELERY,
+        "ray": RAY,
+        "kubernetes": KUBERNETES,
+    }
+    dep = deps.get(name.lower())
+    if dep:
+        return dep.is_available()
+    return False