fiddler-evals 0.8.0__tar.gz → 0.9.0.dev1__tar.gz

{fiddler_evals-0.8.0/fiddler_evals.egg-info → fiddler_evals-0.9.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fiddler-evals
-Version: 0.8.0
+Version: 0.9.0.dev1
 Summary: Python SDK for evaluating LLM Applications
 Author-email: Fiddler AI <support@fiddler.ai>
 Maintainer-email: Fiddler AI <support@fiddler.ai>

fiddler_evals-0.9.0.dev1/fiddler_evals/VERSION

@@ -0,0 +1 @@
+0.9.0.dev1

{fiddler_evals-0.8.0 → fiddler_evals-0.9.0.dev1}/fiddler_evals/evaluators/__init__.py

@@ -1,4 +1,5 @@
 from fiddler_evals.evaluators.answer_relevance import AnswerRelevance
+from fiddler_evals.evaluators.centor_pii_phi import CentorPiiPhi
 from fiddler_evals.evaluators.coherence import Coherence
 from fiddler_evals.evaluators.conciseness import Conciseness
 from fiddler_evals.evaluators.context_relevance import ContextRelevance
@@ -22,6 +23,7 @@ __all__ = [
     'RegexSearch',
     'RegexMatch',
     'AnswerRelevance',
+    'CentorPiiPhi',
     'Coherence',
     'Conciseness',
     'ContextRelevance',

fiddler_evals-0.9.0.dev1/fiddler_evals/evaluators/centor_pii_phi.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+from fiddler_evals.evaluators.base import FiddlerEvaluator
+from fiddler_evals.pydantic_models.score import Score
+
+
+class CentorPiiPhi(FiddlerEvaluator):
+    """Evaluator for sensitive information detection using the Centor PII/PHI model.
+
+    Uses the GLiNER-based Centor Model for PII/PHI to detect sensitive
+    information in text. Returns a boolean detection flag plus one score per
+    detected entity type with instance count and per-instance detail.
+
+    Key Features:
+        - **GLiNER-Based Detection**: Uses Fiddler's Centor PII/PHI model for
+          entity recognition across PII, PHI, and custom entity types
+        - **Per-Entity-Type Detail**: Returns one score per distinct entity type
+          with count and per-instance confidence/position
+        - **Configurable Threshold**: Control precision/recall tradeoff via the
+          ``threshold`` parameter (default 0.5)
+        - **Custom Entities**: Support zero-shot detection of user-defined entity
+          types via ``custom_entities``
+
+    Entity Categories:
+        - **PII**: Names, emails, phone numbers, addresses, SSNs, credit cards,
+          driver's licenses
+        - **PII_ALL**: Extended PII including less common entity types
+        - **PHI**: Protected Health Information — patient IDs, diagnoses,
+          medications, lab results
+
+    Scores Returned:
+        - **has_pii_phi**: Boolean flag (``1.0`` = detected, ``0.0`` = clean)
+        - **<entity_type>**: One score per distinct entity type found (e.g.,
+          ``person``, ``email``, ``social security number``).
+          ``value`` = count of instances, ``reasoning`` = JSON array of
+          per-instance detail (``start_idx``, ``end_idx``, ``score``).
+          Raw detected text is never included.
+
+    Args:
+        entity_categories: Entity categories to detect. Defaults to
+            ``['PII']``. Options: ``'PII'``, ``'PII_ALL'``, ``'PHI'``.
+        custom_entities: Custom entity types for zero-shot detection.
+            When provided, ``'Custom Entities'`` is automatically added
+            to the category list.
+        threshold: Minimum confidence score (0.1--1.0) for a span to be
+            included. Default ``0.5`` is calibrated for balanced
+            precision/recall. Higher values increase precision; lower
+            values increase recall.
+        score_name_prefix: Optional prefix prepended to score names.
+
+    Returns:
+        list[Score]: A list of :class:`~fiddler_evals.pydantic_models.score.Score`
+            objects: one ``has_pii_phi`` boolean, plus one per detected entity type.
+
+    Raises:
+        ValueError: If the text is empty or None.
+
+    Example:
+        >>> from fiddler_evals.evaluators import CentorPiiPhi
+        >>> evaluator = CentorPiiPhi()
+
+        # Detect PII
+        >>> scores = evaluator.score("John Smith lives at 123 Main St, SSN 123-45-6789")
+        >>> has_pii = next(s for s in scores if s.name == "has_pii_phi")
+        >>> print(has_pii.value)
+        1.0
+
+        # Per-entity-type detail
+        >>> for s in scores:
+        ...     if s.name != "has_pii_phi":
+        ...         detail = json.loads(s.reasoning)
+        ...         print(f"{s.name}: {int(s.value)} instance(s)")
+        ...         for d in detail:
+        ...             print(f"  position {d['start_idx']}-{d['end_idx']}, confidence {d['score']}")
+        # person: 1 instance(s)
+        #   position 0-10, confidence 0.99
+        # social security number: 1 instance(s)
+        #   position 35-46, confidence 0.99
+        # address: 1 instance(s)
+        #   position 20-31, confidence 0.85
+
+        # Configure for PHI with higher threshold
+        >>> evaluator = CentorPiiPhi(entity_categories=["PHI"], threshold=0.8)
+        >>> scores = evaluator.score("Patient Jane Doe, MRN-123456, diagnosed Type 2 Diabetes")
+
+        # Clean content
+        >>> scores = evaluator.score("What is the capital of France?")
+        >>> next(s for s in scores if s.name == "has_pii_phi").value
+        0.0
+    """
+
+    name = 'centor_pii_phi'
+    """Evaluator identifier, recorded as ``evaluator_name`` on every
+    ``Score`` this evaluator produces."""
+
+    def __init__(
+        self,
+        *,
+        entity_categories: list[str] | None = None,
+        custom_entities: list[str] | None = None,
+        threshold: float = 0.5,
+        score_name_prefix: str | None = None,
+    ):
+        super().__init__(score_name_prefix=score_name_prefix)
+        self._entity_categories = entity_categories
+        self._custom_entities = custom_entities
+        self._threshold = threshold
+
+    def score(self, text: str) -> list[Score]:  # pylint: disable=arguments-differ
+        """Score text for sensitive information.
+
+        Args:
+            text (str): The text to scan.
+
+        Returns:
+            list[Score]: A ``has_pii_phi`` boolean score, plus one score per
+                detected entity type with count and per-instance detail.
+        """
+        text = text.strip() if text else ''
+
+        if not text:
+            raise ValueError('text is required for sensitive information detection')
+
+        parameters: dict = {}
+        if self._entity_categories is not None:
+            parameters['entity_categories'] = self._entity_categories
+        if self._custom_entities is not None:
+            parameters['custom_entities'] = self._custom_entities
+        if self._threshold != 0.5:
+            parameters['threshold'] = self._threshold
+
+        payload = {
+            'evaluator_name': self.name,
+            'parameters': parameters,
+            'inputs': {'text': text},
+        }
+
+        return self._parse_scores(data=self.make_call(payload))

{fiddler_evals-0.8.0 → fiddler_evals-0.9.0.dev1/fiddler_evals.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fiddler-evals
-Version: 0.8.0
+Version: 0.9.0.dev1
 Summary: Python SDK for evaluating LLM Applications
 Author-email: Fiddler AI <support@fiddler.ai>
 Maintainer-email: Fiddler AI <support@fiddler.ai>

{fiddler_evals-0.8.0 → fiddler_evals-0.9.0.dev1}/fiddler_evals.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ fiddler_evals/entities/tests/test_project.py
 fiddler_evals/evaluators/__init__.py
 fiddler_evals/evaluators/answer_relevance.py
 fiddler_evals/evaluators/base.py
+fiddler_evals/evaluators/centor_pii_phi.py
 fiddler_evals/evaluators/coherence.py
 fiddler_evals/evaluators/conciseness.py
 fiddler_evals/evaluators/context_relevance.py

{fiddler_evals-0.8.0 → fiddler_evals-0.9.0.dev1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fiddler-evals"
-version = "0.8.0"
+version = "0.9.0.dev1"
 authors = [
   { name = "Fiddler AI", email = "support@fiddler.ai" },
 ]

fiddler_evals-0.8.0/fiddler_evals/VERSION

@@ -1 +0,0 @@
-0.8.0