agent-learning-kit 0.1.0__py3-none-any.whl

fi/evals/core/evaluate.py

@@ -0,0 +1,319 @@
+"""
+evaluate() — the unified entrypoint for all evaluations.
+
+Usage:
+    from fi.evals import evaluate
+
+    # Local metric (auto-detected)
+    result = evaluate("contains", output="hello world", keyword="hello")
+
+    # Cloud template (turing model → auto-routes to Turing)
+    result = evaluate("toxicity", output="hello world", model="turing_flash")
+
+    # Custom prompt on any LLM (use engine="llm", not "turing")
+    result = evaluate(
+        prompt="Rate the clarity of: {output}",
+        output="ML is a subset of AI.",
+        engine="llm",
+        model="gemini/gemini-2.0-flash",
+    )
+
+    # LLM-augmented: local heuristic first, then LLM refines
+    result = evaluate(
+        "faithfulness",
+        output="The capital of France is Paris.",
+        context="Paris is the capital of France.",
+        model="gemini/gemini-2.5-flash",
+        augment=True,
+    )
+
+    # Multiple evals
+    results = evaluate(
+        ["toxicity", "factual_accuracy"],
+        output="Paris is the capital of France",
+        model="turing_flash",
+    )
+"""
+
+import warnings
+from typing import Any, Dict, List, Optional, Union
+
+from .registry import resolve_engine as _resolve_engine, is_turing_model
+from .result import BatchResult, EvalResult
+from .engines import LocalEngine, TuringEngine, LLMEngine, Engine
+
+
+def evaluate(
+    eval_name: Optional[Union[str, List[str]]] = None,
+    *,
+    prompt: Optional[str] = None,
+    engine: Optional[str] = None,
+    model: Optional[str] = None,
+    augment: Optional[bool] = None,
+    config: Optional[Dict[str, Any]] = None,
+    feedback_store: Optional[Any] = None,
+    generate_prompt: bool = False,
+    # Turing credentials (optional overrides)
+    fi_api_key: Optional[str] = None,
+    fi_secret_key: Optional[str] = None,
+    fi_base_url: Optional[str] = None,
+    **inputs,
+) -> Union[EvalResult, BatchResult]:
+    """Run one or more evaluations with automatic engine routing.
+
+    Args:
+        eval_name: Metric/template name or list of names. Can be None when
+                   using a custom prompt.
+        prompt: Custom evaluation prompt with {output}/{context}/{input}
+                placeholders. Requires explicit engine or a model hint.
+                Note: custom prompts only work with engine='llm'.
+        engine: Force a specific engine — "local", "turing", or "llm".
+        model: Model to use. Turing models (e.g. "turing_flash") auto-route
+               to the Turing engine. Other model strings (e.g.
+               "gemini/gemini-2.0-flash") auto-route to the LLM engine.
+        augment: When True, run the local heuristic first, then pass its
+                 scores + reasoning to the LLM (specified by model=) for
+                 refinement. Requires model= and a metric that supports
+                 LLM augmentation (supports_llm_judge = True).
+        config: Optional metric/judge config dict.
+        feedback_store: Optional FeedbackStore for retrieving few-shot
+                       examples from developer feedback. When provided with
+                       augment=True, similar past feedback is injected into
+                       the LLM judge prompt.
+        generate_prompt: When True, treat `prompt` as a short description
+                        and auto-generate detailed grading criteria via LLM.
+                        Requires model= and prompt=.
+        fi_api_key: Override FI_API_KEY for Turing engine.
+        fi_secret_key: Override FI_SECRET_KEY for Turing engine.
+        fi_base_url: Override FI_BASE_URL for Turing engine.
+        **inputs: Evaluation inputs (output, context, input, keyword, …).
+
+    Returns:
+        EvalResult for a single eval, BatchResult for multiple.
+    """
+    # --- Batch case: list of eval names --------------------------------
+    if isinstance(eval_name, list):
+        results = []
+        for name in eval_name:
+            r = evaluate(
+                name,
+                prompt=prompt,
+                engine=engine,
+                model=model,
+                augment=augment,
+                config=config,
+                feedback_store=feedback_store,
+                generate_prompt=generate_prompt,
+                fi_api_key=fi_api_key,
+                fi_secret_key=fi_secret_key,
+                fi_base_url=fi_base_url,
+                **inputs,
+            )
+            results.append(r)
+        return BatchResult(results=results)
+
+    # --- Single eval ---------------------------------------------------
+    # Auto-generate grading criteria from a short description
+    if generate_prompt and prompt:
+        if not model:
+            raise ValueError(
+                "generate_prompt=True requires a model= parameter "
+                "(e.g. model='gemini/gemini-2.5-flash')."
+            )
+        from .prompt_generator import generate_grading_criteria
+        prompt = generate_grading_criteria(prompt, model, inputs)
+
+    # Custom prompt with no eval_name
+    effective_name = eval_name or "custom_prompt"
+
+    # When augment=True, force local engine for initial run — model is for LLM step
+    resolved_engine = _resolve_engine(
+        eval_name,
+        model=None if augment else model,
+        prompt=prompt,
+        engine=engine,
+    )
+
+    if resolved_engine is None:
+        raise ValueError(
+            f"Cannot auto-detect engine for '{eval_name}'. "
+            "Specify engine='local', engine='turing', or engine='llm', "
+            "or provide a model (turing models → turing, others → llm)."
+        )
+
+    eng = _get_engine(
+        resolved_engine,
+        fi_api_key=fi_api_key,
+        fi_secret_key=fi_secret_key,
+        fi_base_url=fi_base_url,
+    )
+
+    # Run with OTEL span if tracing is enabled
+    result = _run_with_tracing(
+        eng, effective_name, inputs,
+        model=model, prompt=prompt, config=config,
+        engine_type=resolved_engine,
+    )
+
+    # LLM augmentation: only when explicitly requested via augment=True
+    if augment:
+        result = _augment_with_llm(
+            result,
+            effective_name=effective_name,
+            inputs=inputs,
+            model=model,
+            resolved_engine=resolved_engine,
+            feedback_store=feedback_store,
+        )
+
+    return result
+
+
+_ENGINE_FACTORIES = {
+    "local": lambda **_: LocalEngine(),
+    "turing": lambda **kw: TuringEngine(
+        fi_api_key=kw.get("fi_api_key"),
+        fi_secret_key=kw.get("fi_secret_key"),
+        fi_base_url=kw.get("fi_base_url"),
+    ),
+    "llm": lambda **_: LLMEngine(),
+}
+
+
+def _get_engine(
+    engine_type: str,
+    *,
+    fi_api_key: Optional[str] = None,
+    fi_secret_key: Optional[str] = None,
+    fi_base_url: Optional[str] = None,
+) -> Engine:
+    factory = _ENGINE_FACTORIES.get(engine_type.lower())
+    if factory is None:
+        raise ValueError(
+            f"Unknown engine: '{engine_type}'. "
+            f"Use one of: {', '.join(sorted(_ENGINE_FACTORIES))}."
+        )
+    return factory(fi_api_key=fi_api_key, fi_secret_key=fi_secret_key, fi_base_url=fi_base_url)
+
+
+def _run_with_tracing(
+    eng: Engine,
+    eval_name: str,
+    inputs: Dict[str, Any],
+    *,
+    model: Optional[str] = None,
+    prompt: Optional[str] = None,
+    config: Optional[Dict[str, Any]] = None,
+    engine_type: str = "",
+) -> EvalResult:
+    """Run an engine with optional OTEL tracing."""
+    try:
+        from fi.evals.otel.enrichment import (
+            is_auto_enrichment_enabled,
+            create_evaluation_span,
+            enrich_span_with_evaluation,
+        )
+        if not is_auto_enrichment_enabled():
+            raise ImportError  # fall through to untraced path
+
+        with create_evaluation_span(eval_name) as span:
+            result = eng.run(eval_name, inputs, model=model, prompt=prompt, config=config)
+            # Enrich the span with the result
+            if hasattr(span, "set_attribute"):
+                span.set_attribute("gen_ai.span.kind", "EVALUATOR")
+                span.set_attribute("gen_ai.evaluation.name", eval_name)
+            enrich_span_with_evaluation(
+                metric_name=result.eval_name,
+                score=result.score if result.score is not None else 0.0,
+                reason=result.reason,
+                latency_ms=result.latency_ms,
+                span=span if hasattr(span, "set_attribute") else None,
+            )
+            return result
+    except ImportError:
+        pass
+
+    return eng.run(eval_name, inputs, model=model, prompt=prompt, config=config)
+
+
+def _augment_with_llm(
+    result: EvalResult,
+    *,
+    effective_name: str,
+    inputs: Dict[str, Any],
+    model: Optional[str],
+    resolved_engine: str,
+    feedback_store: Optional[Any] = None,
+) -> EvalResult:
+    """Augment a local heuristic result with LLM judgment.
+
+    Called only when augment=True. Validates preconditions and raises
+    clear errors instead of silently skipping.
+    """
+    if not model:
+        raise ValueError(
+            "augment=True requires a model= parameter "
+            "(e.g. model='gemini/gemini-2.5-flash')."
+        )
+    if is_turing_model(model):
+        raise ValueError(
+            f"augment=True is not compatible with Turing models (got '{model}'). "
+            "Use a LiteLLM model string like 'gemini/gemini-2.5-flash'."
+        )
+    if resolved_engine != "local":
+        raise ValueError(
+            f"augment=True only works with local metrics, but engine resolved "
+            f"to '{resolved_engine}'. Remove engine= or set engine='local'."
+        )
+
+    from ..local.registry import get_registry
+    metric_cls = get_registry().get(effective_name)
+
+    if metric_cls is None or not getattr(metric_cls, "supports_llm_judge", False):
+        raise ValueError(
+            f"Metric '{effective_name}' does not support LLM augmentation "
+            f"(supports_llm_judge is False). Only judgment metrics like "
+            f"faithfulness, hallucination_score, task_completion, etc. can be augmented."
+        )
+
+    if result.status != "completed":
+        result.metadata["engine"] = "local"
+        return result
+
+    description = getattr(metric_cls, "judge_description", "") or ""
+
+    from .judge_prompt import build_judge_prompt
+    judge_prompt = build_judge_prompt(effective_name, description, inputs, result)
+
+    # Retrieve feedback-based few-shot examples if available
+    llm_config = None
+    try:
+        from ..feedback.hooks import retrieve_feedback_config
+        llm_config = retrieve_feedback_config(
+            metric_name=effective_name,
+            inputs=inputs,
+            store=feedback_store,
+        )
+    except ImportError:
+        pass  # feedback module not installed / not configured
+
+    llm_eng = LLMEngine()
+    try:
+        augmented = llm_eng.run(
+            effective_name, inputs,
+            model=model, prompt=judge_prompt, config=llm_config,
+        )
+        augmented.metadata["engine"] = "local+llm"
+        if llm_config and llm_config.get("few_shot_examples"):
+            augmented.metadata["feedback_examples_used"] = len(llm_config["few_shot_examples"])
+        return augmented
+    except Exception as exc:
+        warnings.warn(
+            f"LLM augmentation failed for '{effective_name}', "
+            f"falling back to local heuristic result: {exc}",
+            RuntimeWarning,
+            stacklevel=3,
+        )
+        result.metadata["engine"] = "local"
+        result.metadata["augment_error"] = str(exc)
+        return result

fi/evals/core/judge_prompt.py

@@ -0,0 +1,90 @@
+"""
+Generic LLM judge prompt builder for augmenting local metric results.
+
+When a local metric supports LLM augmentation (supports_llm_judge = True)
+and the user passes augment=True, the local heuristic runs first, then
+this module builds a prompt that feeds the heuristic scores + reasoning
+to the LLM for refinement.
+"""
+
+import json
+from typing import Any, Dict
+
+from .result import EvalResult
+
+
+_PROMPT_TEMPLATE = """\
+You are an expert AI evaluator. Your task is to evaluate: **{metric_name}**
+
+## What this metric measures
+{description}
+
+## Local analysis (heuristic pre-screening)
+The following analysis was produced by a fast, deterministic heuristic.
+Use it as a starting point — it may be accurate, but it cannot reason
+about semantics the way you can.
+
+Score: {local_score}
+Reasoning: {local_reason}
+
+## Raw data
+{formatted_inputs}
+
+## Instructions
+Using the local analysis as a starting point and the raw data for verification,
+provide your refined judgment.
+
+- If the heuristic score seems correct, confirm it with your own reasoning.
+- If you find the heuristic missed something or was too harsh/lenient, adjust.
+- Score from 0.0 (worst) to 1.0 (best).
+
+Return ONLY a JSON object: {{"score": <float>, "reason": "<brief explanation>"}}\
+"""
+
+
+def _format_inputs(inputs: Dict[str, Any]) -> str:
+    """Format evaluation inputs for the prompt, keeping it concise."""
+    parts = []
+    for key, value in inputs.items():
+        if value is None:
+            continue
+        if isinstance(value, str):
+            display = value if len(value) <= 1000 else value[:1000] + "..."
+            parts.append(f"**{key}**:\n{display}")
+        elif isinstance(value, (list, dict)):
+            try:
+                dumped = json.dumps(value, indent=2, default=str)
+                if len(dumped) > 1500:
+                    dumped = dumped[:1500] + "\n..."
+                parts.append(f"**{key}**:\n```json\n{dumped}\n```")
+            except (TypeError, ValueError):
+                parts.append(f"**{key}**: {str(value)[:500]}")
+        else:
+            parts.append(f"**{key}**: {value}")
+    return "\n\n".join(parts) if parts else "(no inputs)"
+
+
+def build_judge_prompt(
+    metric_name: str,
+    description: str,
+    inputs: Dict[str, Any],
+    local_result: EvalResult,
+) -> str:
+    """Build an LLM judge prompt that includes local heuristic results.
+
+    Args:
+        metric_name: The metric being evaluated (e.g. "faithfulness").
+        description: What this metric measures (from metric_cls.judge_description).
+        inputs: The raw evaluation inputs (output, context, trajectory, etc.).
+        local_result: The EvalResult from the local heuristic engine.
+
+    Returns:
+        A formatted prompt string ready for the LLM engine.
+    """
+    return _PROMPT_TEMPLATE.format(
+        metric_name=metric_name,
+        description=description or f"Evaluate the quality of the output for '{metric_name}'.",
+        local_score=local_result.score if local_result.score is not None else "N/A",
+        local_reason=local_result.reason or "No reasoning provided.",
+        formatted_inputs=_format_inputs(inputs),
+    )

fi/evals/core/prompt_generator.py

@@ -0,0 +1,83 @@
+"""
+Auto-generate grading criteria from a short description.
+
+Usage:
+    from fi.evals import evaluate
+
+    # Instead of writing a detailed rubric yourself:
+    result = evaluate(
+        prompt="product description accuracy for e-commerce images",
+        output="A red cotton t-shirt with v-neck",
+        image_url="https://example.com/tshirt.jpg",
+        engine="llm",
+        model="gemini/gemini-2.5-flash",
+        generate_prompt=True,
+    )
+"""
+
+import hashlib
+from typing import Any, Dict
+
+_CACHE: Dict[str, str] = {}
+
+_META_PROMPT = """\
+You are an expert prompt engineer specializing in LLM evaluation rubrics.
+
+Given a short description of what to evaluate, generate a detailed grading \
+criteria that an LLM judge can use to score inputs on a 0.0–1.0 scale.
+
+The criteria MUST:
+- Be specific and actionable (not vague)
+- Define what 1.0, 0.5, and 0.0 look like
+- Reference the input fields the judge will receive: {input_keys}
+- Be 4-8 sentences maximum
+
+Description of what to evaluate:
+{description}
+
+Return ONLY the grading criteria text. No JSON, no markdown, no preamble.\
+"""
+
+
+def generate_grading_criteria(
+    description: str,
+    model: str,
+    inputs: Dict[str, Any],
+    *,
+    cache: bool = True,
+) -> str:
+    """Generate a detailed grading criteria from a short description.
+
+    Args:
+        description: Short description of what to evaluate
+            (e.g. "product description accuracy for images").
+        model: LiteLLM model string (e.g. "gemini/gemini-2.5-flash").
+        inputs: The evaluation inputs dict — used to tell the generator
+            which fields the judge will receive.
+        cache: Cache results per (description, model) for the session.
+
+    Returns:
+        A detailed grading criteria string.
+    """
+    cache_key = hashlib.md5(f"{description}:{model}".encode()).hexdigest()
+    if cache and cache_key in _CACHE:
+        return _CACHE[cache_key]
+
+    input_keys = ", ".join(sorted(inputs.keys())) or "output"
+
+    prompt = _META_PROMPT.format(
+        description=description,
+        input_keys=input_keys,
+    )
+
+    import litellm
+    response = litellm.completion(
+        model=model,
+        messages=[{"role": "user", "content": prompt}],
+    )
+    criteria = response.choices[0].message.content.strip()
+
+    if cache:
+        _CACHE[cache_key] = criteria
+
+    return criteria

fi/evals/core/registry.py

@@ -0,0 +1,57 @@
+"""
+Unified registry — resolves eval names to engines.
+
+Routing is purely based on user-provided kwargs:
+    1. Explicit engine= → use that
+    2. Turing model → "turing"
+    3. Any other model → "llm"
+    4. No model → "local" (default)
+"""
+
+from enum import Enum
+from typing import Optional
+
+
+class Turing(str, Enum):
+    """Model options for the Turing (FutureAGI) cloud engine."""
+
+    FLASH = "turing_flash"
+    SMALL = "turing_small"
+    LARGE = "turing_large"
+
+
+TURING_MODEL_PREFIXES = ("turing",)
+
+
+def is_turing_model(model: Optional[str]) -> bool:
+    """Check if the model string indicates a Turing platform model."""
+    if not model:
+        return False
+    return model.lower().startswith(TURING_MODEL_PREFIXES)
+
+
+def resolve_engine(
+    name: Optional[str] = None,
+    *,
+    model: Optional[str] = None,
+    prompt: Optional[str] = None,
+    engine: Optional[str] = None,
+) -> str:
+    """Auto-detect which engine based on user-provided kwargs.
+
+    Priority:
+        1. Explicit engine kwarg
+        2. Turing model → "turing"
+        3. Any other model → "llm"
+        4. No model → "local" (default, fails gracefully if metric not found)
+    """
+    if engine:
+        return engine
+
+    if is_turing_model(model):
+        return "turing"
+
+    if model:
+        return "llm"
+
+    return "local"

fi/evals/core/result.py

@@ -0,0 +1,55 @@
+"""
+Unified result types for all evaluations.
+
+EvalResult is the ONE result type returned by evaluate() and all engines.
+BatchResult wraps multiple EvalResults when running several evals at once.
+"""
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, Iterator, List, Optional
+
+
+@dataclass
+class EvalResult:
+    """The unified result type returned by evaluate() and all engines."""
+
+    eval_name: str
+    score: Optional[float] = None
+    passed: Optional[bool] = None
+    reason: str = ""
+    latency_ms: float = 0.0
+    status: str = "completed"
+    error: Optional[str] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self):
+        # Auto-derive passed from score if not explicitly set
+        if self.passed is None and self.score is not None:
+            self.passed = self.score >= 0.5
+
+
+@dataclass
+class BatchResult:
+    """Returned when multiple evals are run via evaluate()."""
+
+    results: List[EvalResult] = field(default_factory=list)
+
+    @property
+    def success_rate(self) -> float:
+        if not self.results:
+            return 0.0
+        completed = sum(1 for r in self.results if r.status == "completed")
+        return completed / len(self.results)
+
+    def get(self, name: str) -> Optional[EvalResult]:
+        """Get result by eval name."""
+        for r in self.results:
+            if r.eval_name == name:
+                return r
+        return None
+
+    def __iter__(self) -> Iterator[EvalResult]:
+        return iter(self.results)
+
+    def __len__(self) -> int:
+        return len(self.results)