agent-learning 0.4.1__py3-none-any.whl

agent_learning/scorers/__init__.py

@@ -0,0 +1,187 @@
+"""Public scorer interface for the agent-learning SDK.
+
+Two selectors on :class:`agent_learning.config.ScoreRuntimeConfig`
+decide which backend ``build_scorers`` returns:
+
+- ``cfg.tier`` (preferred, new). One of:
+        - ``"stdlib"``: Tier 1, pure-stdlib text scorers. Zero external
+      dependencies.
+        - ``"nlp"``: Tier 2, in-SDK feature-based scorers over
+      ``(phi, action_id)``. Currently routed to the existing
+      :mod:`.nlp` package.
+    - ``"slm"``: Tier 3, Microsoft Phi-4-mini-instruct via the
+      ``[slm]`` extra. Stub raises until the extra is wired.
+    - ``"llm"``: Tier 4, ``azure-ai-evaluation`` evaluators behind
+      Azure OpenAI. Requires the ``[llm]`` extra.
+- ``cfg.mode`` (legacy). Used only when ``cfg.tier`` is None. Values
+  ``"nlp"`` and ``"llm"`` map to the same backends Tier 2 and Tier 4
+  resolve to, preserving the v0.1 API.
+
+Callers never branch on the tier themselves; the factory hides the
+choice so the reward shaper and learner stay backend-agnostic.
+"""
+
+from __future__ import annotations
+
+from typing import Tuple
+
+from ..config import ScoreRuntimeConfig
+from .base import Scorer, ScoreResult
+
+
+def _build_stdlib(cfg: ScoreRuntimeConfig) -> Tuple[Scorer, Scorer, Scorer]:
+    from .stdlib import (
+        StdlibAdherenceScorer,
+        StdlibCompletionScorer,
+        StdlibIntentScorer,
+    )
+    return (
+        StdlibIntentScorer.load_or_default(
+            cfg.stdlib.snapshot_dir,
+            feature_dim=cfg.stdlib.feature_dim,
+            pass_threshold=cfg.stdlib.pass_threshold,
+        ),
+        StdlibAdherenceScorer.load_or_default(
+            cfg.stdlib.snapshot_dir,
+            pass_threshold=cfg.stdlib.pass_threshold,
+        ),
+        StdlibCompletionScorer.load_or_default(
+            cfg.stdlib.snapshot_dir,
+            pass_threshold=cfg.stdlib.pass_threshold,
+        ),
+    )
+
+
+def _build_nlp(cfg: ScoreRuntimeConfig) -> Tuple[Scorer, Scorer, Scorer]:
+    """Tier 2: TF-IDF + scikit-learn scorers over query/response text.
+
+    Requires the ``[nlp]`` extra. The scorers are imported lazily so
+    callers that never select ``tier="nlp"`` don't need scikit-learn
+    installed.
+    """
+    from .nlp_text import (
+        NlpTextAdherenceScorer,
+        NlpTextCompletionScorer,
+        NlpTextIntentScorer,
+    )
+    nlp_text_cfg = cfg.nlp_text
+    return (
+        NlpTextIntentScorer.load_or_default(
+            nlp_text_cfg.snapshot_dir,
+            pass_threshold=nlp_text_cfg.pass_threshold,
+            max_features=nlp_text_cfg.max_features,
+            ngram_min=nlp_text_cfg.ngram_min,
+            ngram_max=nlp_text_cfg.ngram_max,
+        ),
+        NlpTextAdherenceScorer.load_or_default(
+            nlp_text_cfg.snapshot_dir,
+            pass_threshold=nlp_text_cfg.pass_threshold,
+            max_features=nlp_text_cfg.max_features,
+            ngram_min=nlp_text_cfg.ngram_min,
+            ngram_max=nlp_text_cfg.ngram_max,
+        ),
+        NlpTextCompletionScorer.load_or_default(
+            nlp_text_cfg.snapshot_dir,
+            pass_threshold=nlp_text_cfg.pass_threshold,
+            max_features=nlp_text_cfg.max_features,
+            ngram_min=nlp_text_cfg.ngram_min,
+            ngram_max=nlp_text_cfg.ngram_max,
+        ),
+    )
+
+
+def _build_nlp_legacy(cfg: ScoreRuntimeConfig) -> Tuple[Scorer, Scorer, Scorer]:
+    """Back-compat path for callers using ``mode="nlp"``.
+
+    Routes to the original :class:`agent_learning.classifiers.scorers.BinaryScorer`
+    stack over ``(phi, action_id)``. Preserves the v0.1 API for callers
+    that haven't migrated to the tier-based selector yet.
+    """
+    from .nlp import (
+        NlpAdherenceScorer,
+        NlpCompletionScorer,
+        NlpIntentScorer,
+    )
+    return (
+        NlpIntentScorer.load_or_default(cfg.nlp),
+        NlpAdherenceScorer.load_or_default(cfg.nlp),
+        NlpCompletionScorer.load_or_default(cfg.nlp),
+    )
+
+
+def _build_slm(cfg: ScoreRuntimeConfig) -> Tuple[Scorer, Scorer, Scorer]:
+    """Tier 3: Phi-4-mini-instruct INT4 ONNX scorers.
+
+    Requires the ``[slm]`` extra and a local copy of the
+    Phi-4-mini-instruct INT4 ONNX bundle. Scorers are imported lazily so
+    callers that never select ``tier="slm"`` don't need
+    ``onnxruntime-genai`` installed.
+    """
+    from .slm import (
+        SlmAdherenceScorer,
+        SlmCompletionScorer,
+        SlmIntentScorer,
+    )
+    slm_cfg = cfg.slm
+    return (
+        SlmIntentScorer.load_or_default(
+            slm_cfg.model_dir,
+            pass_threshold=slm_cfg.pass_threshold,
+            max_new_tokens=slm_cfg.max_new_tokens,
+            temperature=slm_cfg.temperature,
+        ),
+        SlmAdherenceScorer.load_or_default(
+            slm_cfg.model_dir,
+            pass_threshold=slm_cfg.pass_threshold,
+            max_new_tokens=slm_cfg.max_new_tokens,
+            temperature=slm_cfg.temperature,
+        ),
+        SlmCompletionScorer.load_or_default(
+            slm_cfg.model_dir,
+            pass_threshold=slm_cfg.pass_threshold,
+            max_new_tokens=slm_cfg.max_new_tokens,
+            temperature=slm_cfg.temperature,
+        ),
+    )
+
+
+def _build_llm(cfg: ScoreRuntimeConfig) -> Tuple[Scorer, Scorer, Scorer]:
+    from .llm import (
+        LlmAdherenceScorer,
+        LlmCompletionScorer,
+        LlmIntentScorer,
+    )
+    return (
+        LlmIntentScorer(cfg.llm),
+        LlmAdherenceScorer(cfg.llm),
+        LlmCompletionScorer(cfg.llm),
+    )
+
+
+def build_scorers(cfg: ScoreRuntimeConfig) -> Tuple[Scorer, Scorer, Scorer]:
+    """Return the ``(intent, adherence, completion)`` scorer trio.
+
+    Routing order: ``cfg.tier`` if set, else ``cfg.mode``.
+    """
+    tier = cfg.tier
+    if tier is None:
+        # Legacy mode fallback. mode="nlp" routes to the BinaryScorer
+        # stack, NOT the new TF-IDF scorers (those are reachable via
+        # tier="nlp").
+        if cfg.mode == "nlp":
+            return _build_nlp_legacy(cfg)
+        if cfg.mode == "llm":
+            return _build_llm(cfg)
+        raise ValueError(f"unknown score_mode: {cfg.mode!r}")
+    if tier == "stdlib":
+        return _build_stdlib(cfg)
+    if tier == "nlp":
+        return _build_nlp(cfg)
+    if tier == "slm":
+        return _build_slm(cfg)
+    if tier == "llm":
+        return _build_llm(cfg)
+    raise ValueError(f"unknown score tier: {tier!r}")
+
+
+__all__ = ["Scorer", "ScoreResult", "build_scorers"]

agent_learning/scorers/base.py

@@ -0,0 +1,49 @@
+"""Shared types for the scoring backends.
+
+Each backend produces the same :class:`ScoreResult` shape so the reward
+shaper and learner stay backend-agnostic.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Dict, Protocol, runtime_checkable
+
+
+@dataclass(frozen=True)
+class ScoreResult:
+    """The contract every scoring backend honors.
+
+    Attributes:
+        label: ``"pass"`` or ``"fail"``.
+        confidence: probability of the predicted label in ``[0, 1]``.
+        normalized: a continuous score in ``[0, 1]`` suitable for the
+            reward shaper. Equals ``confidence`` when ``label == "pass"``
+            and ``1.0 - confidence`` when ``label == "fail"``.
+        features: optional debugging payload. Never consumed by the
+            reward shaper or the learner.
+    """
+
+    label: str
+    confidence: float
+    normalized: float
+    features: Dict[str, object] = field(default_factory=dict)
+
+
+@runtime_checkable
+class Scorer(Protocol):
+    """Structural type for both NLP and LLM scorers."""
+
+    name: str
+
+    def score(self, **kwargs: object) -> ScoreResult:
+        """Score one episode.
+
+        NLP backends typically require ``phi`` and ``action_id``.
+        LLM backends typically require ``request`` and ``response``.
+        Unused keyword arguments are ignored by each backend so the
+        same call site works regardless of mode.
+        """
+
+
+__all__ = ["Scorer", "ScoreResult"]

agent_learning/scorers/llm/__init__.py

@@ -0,0 +1,19 @@
+"""LLM-backed scorers (Tier 4, requires ``azure-ai-evaluation``).
+
+These wrappers stay shallow on purpose: each one defers to the
+matching evaluator from the ``azure-ai-evaluation`` package and
+projects the evaluator's verdict onto the SDK-neutral
+:class:`agent_learning.scorers.base.ScoreResult` shape.
+
+The evaluator import is deferred until first call so installing the
+SDK without ``azure-ai-evaluation`` stays inexpensive when callers
+only use the NLP backend.
+"""
+
+from __future__ import annotations
+
+from .adherence import LlmAdherenceScorer
+from .completion import LlmCompletionScorer
+from .intent import LlmIntentScorer
+
+__all__ = ["LlmAdherenceScorer", "LlmCompletionScorer", "LlmIntentScorer"]

agent_learning/scorers/llm/_base.py

@@ -0,0 +1,126 @@
+"""Shared wrapper that adapts an ``azure-ai-evaluation`` evaluator to
+the SDK :class:`ScoreResult` contract.
+
+The evaluator import is lazy: the ``azure-ai-evaluation`` package is
+only imported the first time :meth:`_evaluator` is invoked, so callers
+who never use the LLM backend pay no import cost.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+from ...config import ScoreConfig
+from ..base import ScoreResult
+
+
+@dataclass
+class _LlmScorerWrapper:
+    """Common base for the three concrete LLM scorers."""
+
+    cfg: ScoreConfig
+    name: str = "llm-scorer"
+    label_name: str = "llm-scorer"
+    evaluator_attr: str = ""
+    _cached_evaluator: Optional[Any] = field(default=None, repr=False)
+
+    def _evaluator(self) -> Any:
+        if self._cached_evaluator is not None:
+            return self._cached_evaluator
+        try:
+            module = __import__("azure.ai.evaluation", fromlist=[self.evaluator_attr])
+        except ImportError as exc:
+            raise ImportError(
+                "LLM scorers require the optional 'azure-ai-evaluation' package. "
+                "Install it with: pip install azure-ai-evaluation"
+            ) from exc
+        evaluator_cls = getattr(module, self.evaluator_attr)
+        self._cached_evaluator = evaluator_cls(model_config=self.cfg.to_model_config())
+        return self._cached_evaluator
+
+    def score(
+        self,
+        *,
+        query: Optional[str] = None,
+        response: Optional[str] = None,
+        request: Optional[str] = None,
+        **kwargs: object,
+    ) -> ScoreResult:
+        """Run the underlying evaluator and project to a ScoreResult.
+
+        Either ``query`` or ``request`` may be used to supply the
+        prompt. Extra keyword arguments are forwarded to the evaluator
+        so callers can pass evaluator-specific fields like ``context``
+        or ``tool_calls`` without the SDK needing to learn them.
+        """
+        if query is None:
+            query = request
+        if query is None or response is None:
+            raise ValueError(
+                f"{self.name} LLM scorer requires query (or request) and response"
+            )
+        payload = {"query": query, "response": response, **kwargs}
+        # Drop NLP-specific kwargs that would confuse the evaluator.
+        payload.pop("phi", None)
+        payload.pop("action_id", None)
+        evaluator = self._evaluator()
+        result = evaluator(**payload)
+        return _project_to_score(result, threshold=self.cfg.threshold, name=self.name)
+
+
+def _project_to_score(
+    result: Any, *, threshold: float, name: str
+) -> ScoreResult:
+    """Map an evaluator return value onto a :class:`ScoreResult`.
+
+    ``azure-ai-evaluation`` evaluators return a mapping with a numeric
+    score keyed by either ``"score"`` or ``"<evaluator>_score"``. The
+    helper accepts either shape so the SDK works across evaluator
+    versions.
+    """
+    if not isinstance(result, dict):
+        raise TypeError(
+            f"{name} evaluator returned {type(result).__name__}, expected mapping"
+        )
+    raw: Optional[float] = None
+    for key in ("score", f"{name}_score", "result"):
+        if key in result and isinstance(result[key], (int, float)):
+            raw = float(result[key])
+            break
+    if raw is None:
+        # Fall back to the first numeric value in the result mapping.
+        for value in result.values():
+            if isinstance(value, (int, float)):
+                raw = float(value)
+                break
+    if raw is None:
+        raise ValueError(
+            f"{name} evaluator returned no numeric score; got keys {list(result)!r}"
+        )
+    normalized = _normalize(raw)
+    label = "pass" if normalized >= threshold else "fail"
+    confidence = normalized if label == "pass" else 1.0 - normalized
+    return ScoreResult(
+        label=label,
+        confidence=confidence,
+        normalized=normalized,
+        features={"raw": raw, "evaluator_result": result},
+    )
+
+
+def _normalize(raw: float) -> float:
+    """Project a raw evaluator score into ``[0, 1]``.
+
+    Common ``azure-ai-evaluation`` evaluators emit scores on a 1-5
+    Likert scale. Scores already in ``[0, 1]`` are passed through
+    unchanged.
+    """
+    if 0.0 <= raw <= 1.0:
+        return raw
+    if 1.0 <= raw <= 5.0:
+        return (raw - 1.0) / 4.0
+    return max(0.0, min(1.0, raw))
+
+
+__all__ = ["_LlmScorerWrapper"]

agent_learning/scorers/llm/adherence.py

@@ -0,0 +1,21 @@
+"""LLM-backed task-adherence scorer."""
+
+from __future__ import annotations
+
+from ...config import ScoreConfig
+from ._base import _LlmScorerWrapper
+
+
+class LlmAdherenceScorer(_LlmScorerWrapper):
+    """Defers to ``azure.ai.evaluation.TaskAdherenceEvaluator``."""
+
+    def __init__(self, cfg: ScoreConfig):
+        super().__init__(
+            cfg=cfg,
+            name="adherence",
+            label_name="adherence",
+            evaluator_attr="TaskAdherenceEvaluator",
+        )
+
+
+__all__ = ["LlmAdherenceScorer"]

agent_learning/scorers/llm/completion.py

@@ -0,0 +1,21 @@
+"""LLM-backed task-completion scorer."""
+
+from __future__ import annotations
+
+from ...config import ScoreConfig
+from ._base import _LlmScorerWrapper
+
+
+class LlmCompletionScorer(_LlmScorerWrapper):
+    """Defers to ``azure.ai.evaluation.TaskCompletionEvaluator``."""
+
+    def __init__(self, cfg: ScoreConfig):
+        super().__init__(
+            cfg=cfg,
+            name="completion",
+            label_name="completion",
+            evaluator_attr="TaskCompletionEvaluator",
+        )
+
+
+__all__ = ["LlmCompletionScorer"]

agent_learning/scorers/llm/intent.py

@@ -0,0 +1,21 @@
+"""LLM-backed intent-resolution scorer."""
+
+from __future__ import annotations
+
+from ...config import ScoreConfig
+from ._base import _LlmScorerWrapper
+
+
+class LlmIntentScorer(_LlmScorerWrapper):
+    """Defers to ``azure.ai.evaluation.IntentResolutionEvaluator``."""
+
+    def __init__(self, cfg: ScoreConfig):
+        super().__init__(
+            cfg=cfg,
+            name="intent",
+            label_name="intent",
+            evaluator_attr="IntentResolutionEvaluator",
+        )
+
+
+__all__ = ["LlmIntentScorer"]

agent_learning/scorers/nlp/__init__.py

@@ -0,0 +1,16 @@
+"""Native, in-process scorers (Tier 0, pure standard library).
+
+These wrap the existing :class:`agent_learning.classifiers.scorers.BinaryScorer`
+implementations so the SDK can ship a working scorer stack with no
+external service dependencies. The wrappers translate the underlying
+:class:`agent_learning.classifiers.base.ClassifierResult` into the
+backend-neutral :class:`agent_learning.scorers.base.ScoreResult`.
+"""
+
+from __future__ import annotations
+
+from .adherence import NlpAdherenceScorer
+from .completion import NlpCompletionScorer
+from .intent import NlpIntentScorer
+
+__all__ = ["NlpAdherenceScorer", "NlpCompletionScorer", "NlpIntentScorer"]

agent_learning/scorers/nlp/_base.py

@@ -0,0 +1,99 @@
+"""Shared wrapper that adapts a :class:`BinaryScorer` to the SDK
+:class:`ScoreResult` contract.
+
+The wrapper reads a snapshot from ``cfg.snapshot_dir/{name}.json`` at
+load time. When no snapshot is present the scorer stays in its unfitted
+"always-fail with zero confidence" state, which keeps the reward
+shaper safe to call but encourages operators to train the scorers.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from typing import List, Optional
+
+from ...classifiers.scorers._base import BinaryScorer
+from ...config import NlpScoreConfig
+from ..base import ScoreResult
+
+
+@dataclass
+class _NlpScorerWrapper:
+    """Common base for the three concrete NLP scorers."""
+
+    name: str
+    label_name: str
+    _impl: BinaryScorer
+    _pass_threshold: float = 0.5
+
+    @classmethod
+    def _build(cls, name: str, cfg: NlpScoreConfig) -> "_NlpScorerWrapper":
+        impl = BinaryScorer(label_name=name)
+        snapshot_path = os.path.join(cfg.snapshot_dir, f"{name}.json")
+        if os.path.isfile(snapshot_path):
+            with open(snapshot_path, "r", encoding="utf-8") as fh:
+                impl = BinaryScorer.from_snapshot(json.load(fh))
+                impl.label_name = name
+        return cls(
+            name=name,
+            label_name=name,
+            _impl=impl,
+            _pass_threshold=float(cfg.pass_threshold),
+        )
+
+    def fit(self, training_rows: List[dict]) -> "_NlpScorerWrapper":
+        """Fit the underlying logistic-regression head.
+
+        Training rows must look like ``{"phi": [...], "action_id": str, "label": 0|1}``.
+        """
+        self._impl.fit(training_rows)
+        return self
+
+    def save(self, snapshot_dir: str) -> str:
+        """Persist the fitted weights to ``{snapshot_dir}/{name}.json``."""
+        os.makedirs(snapshot_dir, exist_ok=True)
+        path = os.path.join(snapshot_dir, f"{self.name}.json")
+        with open(path, "w", encoding="utf-8") as fh:
+            json.dump(self._impl.to_snapshot(), fh)
+        return path
+
+    def score(
+        self,
+        *,
+        phi: Optional[List[float]] = None,
+        action_id: Optional[str] = None,
+        **_: object,
+    ) -> ScoreResult:
+        """Score one episode given a context vector and chosen action.
+
+        Extra keyword arguments are ignored so the same call site works
+        when the LLM backend is swapped in.
+        """
+        if phi is None or action_id is None:
+            raise ValueError(
+                f"{self.name} NLP scorer requires phi and action_id"
+            )
+        result = self._impl.score(phi=list(phi), action_id=str(action_id))
+        probability = float(result.features.get("probability", result.confidence))
+        label = "pass" if probability >= self._pass_threshold else "fail"
+        if label == "pass":
+            confidence = probability
+            normalized = probability
+        else:
+            confidence = 1.0 - probability
+            normalized = probability  # keep the raw "pass probability" for shaping
+        features = {
+            "probability": probability,
+            "fitted": bool(self._impl.weights),
+        }
+        return ScoreResult(
+            label=label,
+            confidence=confidence,
+            normalized=normalized,
+            features=features,
+        )
+
+
+__all__ = ["_NlpScorerWrapper"]

agent_learning/scorers/nlp/adherence.py

@@ -0,0 +1,17 @@
+"""NLP task-adherence scorer."""
+
+from __future__ import annotations
+
+from ...config import NlpScoreConfig
+from ._base import _NlpScorerWrapper
+
+
+class NlpAdherenceScorer(_NlpScorerWrapper):
+    """Predict whether the response adheres to the requested task contract."""
+
+    @classmethod
+    def load_or_default(cls, cfg: NlpScoreConfig) -> "NlpAdherenceScorer":
+        return cls._build("adherence", cfg)  # type: ignore[return-value]
+
+
+__all__ = ["NlpAdherenceScorer"]

agent_learning/scorers/nlp/completion.py

@@ -0,0 +1,17 @@
+"""NLP task-completion scorer."""
+
+from __future__ import annotations
+
+from ...config import NlpScoreConfig
+from ._base import _NlpScorerWrapper
+
+
+class NlpCompletionScorer(_NlpScorerWrapper):
+    """Predict whether the response completes the requested task."""
+
+    @classmethod
+    def load_or_default(cls, cfg: NlpScoreConfig) -> "NlpCompletionScorer":
+        return cls._build("completion", cfg)  # type: ignore[return-value]
+
+
+__all__ = ["NlpCompletionScorer"]

agent_learning/scorers/nlp/intent.py

@@ -0,0 +1,17 @@
+"""NLP intent-resolution scorer."""
+
+from __future__ import annotations
+
+from ...config import NlpScoreConfig
+from ._base import _NlpScorerWrapper
+
+
+class NlpIntentScorer(_NlpScorerWrapper):
+    """Predict whether the chosen action addresses the requester's intent."""
+
+    @classmethod
+    def load_or_default(cls, cfg: NlpScoreConfig) -> "NlpIntentScorer":
+        return cls._build("intent", cfg)  # type: ignore[return-value]
+
+
+__all__ = ["NlpIntentScorer"]

agent_learning/scorers/nlp_text/__init__.py

@@ -0,0 +1,26 @@
+"""Tier 2 NLP scorers (TF-IDF + scikit-learn) over raw query/response text.
+
+These scorers operate on the same ``(query, response)`` pair the Tier 1
+stdlib scorers accept and the same Tier 4 LLM scorers accept, but use a
+TF-IDF vectorizer + logistic-regression head from scikit-learn for the
+intent and adherence/completion classifiers. The adherence scorer also
+applies the same deterministic rule engine the stdlib backend uses;
+the rule-engine score is combined with the learned probability.
+
+Requires the ``[nlp]`` extra (``pip install
+agent-learning[nlp]``). All scikit-learn imports are lazy so
+the package can still be imported without the extra; calling
+:meth:`fit` or :meth:`score` raises a helpful ``ImportError`` then.
+"""
+
+from __future__ import annotations
+
+from .adherence import NlpTextAdherenceScorer
+from .completion import NlpTextCompletionScorer
+from .intent import NlpTextIntentScorer
+
+__all__ = [
+    "NlpTextAdherenceScorer",
+    "NlpTextCompletionScorer",
+    "NlpTextIntentScorer",
+]