agent-learning 0.4.1__py3-none-any.whl

agent_learning/scorers/nlp_text/_base.py

@@ -0,0 +1,234 @@
+"""Shared TF-IDF + scikit-learn logistic-regression pipeline for the
+Tier 2 NLP text scorers.
+
+The class wraps three concrete sub-classes (intent / adherence /
+completion). It defers the scikit-learn import to first use so the
+package stays importable without the ``[nlp]`` extra installed.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+
+from ..base import ScoreResult
+
+
+_SKLEARN_IMPORT_ERROR = (
+    "Tier 2 NLP text scorers require the [nlp] extra. Install with: "
+    "pip install agent-learning[nlp]"
+)
+
+
+def _require_sklearn() -> Tuple[Any, Any]:
+    """Lazy-import scikit-learn. Raises a friendly error if missing."""
+    try:
+        from sklearn.feature_extraction.text import TfidfVectorizer
+        from sklearn.linear_model import LogisticRegression
+    except ImportError as exc:  # pragma: no cover - exercised via tests/mocks
+        raise ImportError(_SKLEARN_IMPORT_ERROR) from exc
+    return TfidfVectorizer, LogisticRegression
+
+
+def _require_joblib() -> Any:
+    try:
+        import joblib
+    except ImportError as exc:  # pragma: no cover
+        raise ImportError(_SKLEARN_IMPORT_ERROR) from exc
+    return joblib
+
+
+def _join_text(*parts: Optional[str]) -> str:
+    """Join non-empty text parts with ``" [SEP] "``."""
+    cleaned = [p.strip() for p in parts if p]
+    return " [SEP] ".join(cleaned)
+
+
+@dataclass
+class _NlpTextScorerBase:
+    """Base class for the three Tier 2 NLP text scorers.
+
+    Sub-classes override :attr:`name` and :meth:`_pair_text` to control
+    which inputs feed the vectorizer (intent uses query+response;
+    adherence and completion use response only).
+    """
+
+    name: str = "nlp_text"
+    pass_threshold: float = 0.5
+    max_features: int = 20000
+    ngram_min: int = 1
+    ngram_max: int = 2
+    C: float = 1.0
+    min_df: int = 1
+    vectorizer: Any = None  # sklearn TfidfVectorizer or None
+    classifier: Any = None  # sklearn LogisticRegression or None
+
+    # ----- inputs -----
+    def _pair_text(self, *, query: Optional[str], response: str) -> str:
+        return _join_text(query, response)
+
+    # ----- training -----
+    def fit(self, training_rows: Sequence[Dict[str, Any]]) -> "_NlpTextScorerBase":
+        """Fit TF-IDF + logistic regression on labeled rows.
+
+        Each row must look like ``{"query": str?, "response": str,
+        "label": 0|1}``. Rows missing ``response`` or ``label`` are
+        skipped. When fewer than two distinct labels are seen the
+        scorer stays unfitted.
+        """
+        TfidfVectorizer, LogisticRegression = _require_sklearn()
+        texts: List[str] = []
+        labels: List[int] = []
+        for row in training_rows:
+            if not isinstance(row, dict):
+                continue
+            response = row.get("response")
+            label = row.get("label")
+            if not response or label is None:
+                continue
+            try:
+                lbl = int(label)
+            except (TypeError, ValueError):
+                continue
+            texts.append(self._pair_text(query=row.get("query"), response=str(response)))
+            labels.append(1 if lbl > 0 else 0)
+        if len(set(labels)) < 2:
+            self.vectorizer = None
+            self.classifier = None
+            return self
+        self.vectorizer = TfidfVectorizer(
+            max_features=self.max_features,
+            ngram_range=(self.ngram_min, self.ngram_max),
+            min_df=self.min_df,
+            lowercase=True,
+        )
+        X = self.vectorizer.fit_transform(texts)
+        self.classifier = LogisticRegression(
+            C=self.C,
+            max_iter=1000,
+            solver="liblinear",
+        )
+        self.classifier.fit(X, labels)
+        return self
+
+    # ----- prediction -----
+    @property
+    def fitted(self) -> bool:
+        return self.vectorizer is not None and self.classifier is not None
+
+    def _predict_probability(
+        self, *, query: Optional[str], response: str
+    ) -> float:
+        if not self.fitted:
+            return float(self.pass_threshold)
+        text = self._pair_text(query=query, response=response)
+        X = self.vectorizer.transform([text])
+        proba = self.classifier.predict_proba(X)[0]
+        # Find the index of the positive class (label == 1).
+        classes = list(self.classifier.classes_)
+        if 1 in classes:
+            idx = classes.index(1)
+        else:
+            # Degenerate single-class fit; fall back to the last column.
+            idx = len(classes) - 1
+        return float(proba[idx])
+
+    def _to_score(
+        self,
+        probability: float,
+        *,
+        features: Optional[Dict[str, Any]] = None,
+    ) -> ScoreResult:
+        if probability >= self.pass_threshold:
+            label = "pass"
+            confidence = probability
+        else:
+            label = "fail"
+            confidence = 1.0 - probability
+        feat: Dict[str, Any] = {
+            "probability": probability,
+            "fitted": self.fitted,
+        }
+        if features:
+            feat.update(features)
+        return ScoreResult(
+            label=label,
+            confidence=confidence,
+            normalized=probability,
+            features=feat,
+        )
+
+    # ----- snapshots -----
+    def to_snapshot(self) -> Dict[str, Any]:
+        """Return a JSON-serialisable header. The fitted sklearn state
+        is dumped separately to a joblib file.
+        """
+        return {
+            "name": self.name,
+            "pass_threshold": float(self.pass_threshold),
+            "max_features": int(self.max_features),
+            "ngram_min": int(self.ngram_min),
+            "ngram_max": int(self.ngram_max),
+            "C": float(self.C),
+            "min_df": int(self.min_df),
+            "fitted": self.fitted,
+        }
+
+    def save(self, snapshot_dir: str) -> str:
+        os.makedirs(snapshot_dir, exist_ok=True)
+        header_path = os.path.join(snapshot_dir, f"{self.name}.nlp_text.json")
+        with open(header_path, "w", encoding="utf-8") as fh:
+            json.dump(self.to_snapshot(), fh)
+        if self.fitted:
+            joblib = _require_joblib()
+            blob_path = os.path.join(
+                snapshot_dir, f"{self.name}.nlp_text.joblib"
+            )
+            joblib.dump(
+                {
+                    "vectorizer": self.vectorizer,
+                    "classifier": self.classifier,
+                },
+                blob_path,
+            )
+        return header_path
+
+    @classmethod
+    def _load_into(
+        cls,
+        instance: "_NlpTextScorerBase",
+        snapshot_dir: Optional[str],
+    ) -> "_NlpTextScorerBase":
+        if not snapshot_dir:
+            return instance
+        header_path = os.path.join(
+            snapshot_dir, f"{instance.name}.nlp_text.json"
+        )
+        blob_path = os.path.join(
+            snapshot_dir, f"{instance.name}.nlp_text.joblib"
+        )
+        if not os.path.isfile(header_path):
+            return instance
+        with open(header_path, "r", encoding="utf-8") as fh:
+            header = json.load(fh)
+        instance.pass_threshold = float(
+            header.get("pass_threshold", instance.pass_threshold)
+        )
+        instance.max_features = int(
+            header.get("max_features", instance.max_features)
+        )
+        instance.ngram_min = int(header.get("ngram_min", instance.ngram_min))
+        instance.ngram_max = int(header.get("ngram_max", instance.ngram_max))
+        instance.C = float(header.get("C", instance.C))
+        instance.min_df = int(header.get("min_df", instance.min_df))
+        if header.get("fitted") and os.path.isfile(blob_path):
+            joblib = _require_joblib()
+            blob = joblib.load(blob_path)
+            instance.vectorizer = blob["vectorizer"]
+            instance.classifier = blob["classifier"]
+        return instance
+
+
+__all__ = ["_NlpTextScorerBase", "_join_text", "_require_sklearn"]

agent_learning/scorers/nlp_text/adherence.py

@@ -0,0 +1,94 @@
+"""Tier 2 NLP text adherence scorer.
+
+Combines the deterministic rule engine that ships in
+:class:`agent_learning.scorers.stdlib.StdlibAdherenceScorer` with a
+TF-IDF + logistic-regression probability over the response text. The
+final probability is the mean of:
+
+- the rule-engine ratio (clauses satisfied / clauses total), and
+- the classifier's predicted probability of the positive class.
+
+When the classifier is unfitted the rule-engine ratio is used
+unchanged. When no contract is supplied the rule-engine ratio is 1.0
+and the score degenerates to the learned probability.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+from ..base import ScoreResult
+from ..stdlib.adherence import StdlibAdherenceScorer
+from ._base import _NlpTextScorerBase
+
+
+@dataclass
+class NlpTextAdherenceScorer(_NlpTextScorerBase):
+    """Adherence scorer backed by TF-IDF + a deterministic rule engine."""
+
+    name: str = "adherence"
+
+    @classmethod
+    def load_or_default(
+        cls,
+        snapshot_dir: Optional[str] = None,
+        *,
+        pass_threshold: float = 0.5,
+        max_features: int = 20000,
+        ngram_min: int = 1,
+        ngram_max: int = 2,
+    ) -> "NlpTextAdherenceScorer":
+        instance = cls(
+            pass_threshold=pass_threshold,
+            max_features=max_features,
+            ngram_min=ngram_min,
+            ngram_max=ngram_max,
+        )
+        cls._load_into(instance, snapshot_dir)
+        return instance  # type: ignore[return-value]
+
+    def _pair_text(
+        self, *, query: Optional[str], response: str
+    ) -> str:
+        # Adherence is about the response itself; ignore the query.
+        return response.strip()
+
+    def score(
+        self,
+        *,
+        response: Optional[str] = None,
+        contract: Optional[Dict[str, Any]] = None,
+        query: Optional[str] = None,
+        **_: object,
+    ) -> ScoreResult:
+        if response is None:
+            raise ValueError(
+                "adherence NLP text scorer requires a response"
+            )
+        # Rule engine via the Tier 1 backend.
+        rule_scorer = StdlibAdherenceScorer(pass_threshold=self.pass_threshold)
+        rule_score = rule_scorer.score(response=response, contract=contract)
+        rule_probability = float(rule_score.normalized)
+
+        # Classifier signal.
+        learned_probability: Optional[float]
+        if self.fitted:
+            learned_probability = self._predict_probability(
+                query=query, response=response
+            )
+            combined = 0.5 * (rule_probability + learned_probability)
+        else:
+            learned_probability = None
+            combined = rule_probability
+
+        features: Dict[str, Any] = {
+            "rule_probability": rule_probability,
+            "learned_probability": learned_probability,
+            "violations": rule_score.features.get("violations", []),
+            "clauses_total": rule_score.features.get("clauses_total", 0),
+        }
+        return self._to_score(combined, features=features)
+
+
+__all__ = ["NlpTextAdherenceScorer"]

agent_learning/scorers/nlp_text/completion.py

@@ -0,0 +1,91 @@
+"""Tier 2 NLP text completion scorer.
+
+Combines the deterministic token-coverage scorer from
+:class:`agent_learning.scorers.stdlib.StdlibCompletionScorer` with a
+TF-IDF + logistic-regression probability over the response text. The
+final probability is the mean of the two signals (rule + classifier)
+when both are available; otherwise the available signal is used.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict, Optional, Sequence
+
+from ..base import ScoreResult
+from ..stdlib.completion import StdlibCompletionScorer
+from ._base import _NlpTextScorerBase
+
+
+@dataclass
+class NlpTextCompletionScorer(_NlpTextScorerBase):
+    """Completion scorer backed by TF-IDF + token-coverage."""
+
+    name: str = "completion"
+
+    @classmethod
+    def load_or_default(
+        cls,
+        snapshot_dir: Optional[str] = None,
+        *,
+        pass_threshold: float = 0.5,
+        max_features: int = 20000,
+        ngram_min: int = 1,
+        ngram_max: int = 2,
+    ) -> "NlpTextCompletionScorer":
+        instance = cls(
+            pass_threshold=pass_threshold,
+            max_features=max_features,
+            ngram_min=ngram_min,
+            ngram_max=ngram_max,
+        )
+        cls._load_into(instance, snapshot_dir)
+        return instance  # type: ignore[return-value]
+
+    def _pair_text(
+        self, *, query: Optional[str], response: str
+    ) -> str:
+        return response.strip()
+
+    def score(
+        self,
+        *,
+        response: Optional[str] = None,
+        expected_tokens: Optional[Sequence[str]] = None,
+        contract: Optional[Dict[str, Any]] = None,
+        query: Optional[str] = None,
+        **_: object,
+    ) -> ScoreResult:
+        if response is None:
+            raise ValueError(
+                "completion NLP text scorer requires a response"
+            )
+        rule_scorer = StdlibCompletionScorer(pass_threshold=self.pass_threshold)
+        rule_score = rule_scorer.score(
+            response=response,
+            expected_tokens=expected_tokens,
+            contract=contract,
+        )
+        rule_probability = float(rule_score.normalized)
+
+        learned_probability: Optional[float]
+        if self.fitted:
+            learned_probability = self._predict_probability(
+                query=query, response=response
+            )
+            combined = 0.5 * (rule_probability + learned_probability)
+        else:
+            learned_probability = None
+            combined = rule_probability
+
+        features: Dict[str, Any] = {
+            "rule_probability": rule_probability,
+            "learned_probability": learned_probability,
+            "hits": rule_score.features.get("hits", 0),
+            "misses": rule_score.features.get("misses", []),
+            "total_targets": rule_score.features.get("total_targets", 0),
+        }
+        return self._to_score(combined, features=features)
+
+
+__all__ = ["NlpTextCompletionScorer"]

agent_learning/scorers/nlp_text/intent.py

@@ -0,0 +1,59 @@
+"""Tier 2 NLP text intent scorer."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+from ..base import ScoreResult
+from ._base import _NlpTextScorerBase
+
+
+@dataclass
+class NlpTextIntentScorer(_NlpTextScorerBase):
+    """Predicts whether the response addresses the requester's intent.
+
+    Feeds TF-IDF a concatenation of ``query`` and ``response`` so the
+    classifier can learn topical alignment between the question and
+    the agent's answer.
+    """
+
+    name: str = "intent"
+
+    @classmethod
+    def load_or_default(
+        cls,
+        snapshot_dir: Optional[str] = None,
+        *,
+        pass_threshold: float = 0.5,
+        max_features: int = 20000,
+        ngram_min: int = 1,
+        ngram_max: int = 2,
+    ) -> "NlpTextIntentScorer":
+        instance = cls(
+            pass_threshold=pass_threshold,
+            max_features=max_features,
+            ngram_min=ngram_min,
+            ngram_max=ngram_max,
+        )
+        cls._load_into(instance, snapshot_dir)
+        return instance  # type: ignore[return-value]
+
+    def score(
+        self,
+        *,
+        query: Optional[str] = None,
+        response: Optional[str] = None,
+        **_: object,
+    ) -> ScoreResult:
+        if not query or not response:
+            raise ValueError(
+                "intent NLP text scorer requires both query and response"
+            )
+        probability = self._predict_probability(
+            query=query, response=response
+        )
+        return self._to_score(probability)
+
+
+__all__ = ["NlpTextIntentScorer"]

agent_learning/scorers/slm/__init__.py

@@ -0,0 +1,25 @@
+"""Tier 3 small-language-model scorers.
+
+The package wraps a locally-hosted instance of
+`Microsoft Phi-4-mini-instruct
+<https://huggingface.co/microsoft/Phi-4-mini-instruct-onnx>`_ (3.8 B
+parameters, 4-bit ONNX) and exposes three pass/fail evaluators with the
+same surface as the Tier 1 and Tier 2 scorers. The model is loaded once
+per process via :class:`SlmRunner` and reused across calls.
+
+Install:
+
+.. code-block:: bash
+
+    pip install agent-learning[slm]
+"""
+
+from .adherence import SlmAdherenceScorer
+from .completion import SlmCompletionScorer
+from .intent import SlmIntentScorer
+
+__all__ = [
+    "SlmAdherenceScorer",
+    "SlmCompletionScorer",
+    "SlmIntentScorer",
+]