edgepii 0.1.0__py3-none-any.whl

edgepii/__init__.py

@@ -0,0 +1,16 @@
+"""Locate PII as typed character spans on the source string."""
+
+from edgepii.errors import DetectorLoadError, OverlapError, UnknownType
+from edgepii.locator import Locator
+from edgepii.redact import redact
+from edgepii.span import LocateResult, Span
+
+__all__ = [
+    "DetectorLoadError",
+    "LocateResult",
+    "Locator",
+    "OverlapError",
+    "Span",
+    "UnknownType",
+    "redact",
+]

edgepii/bridges/presidio.py

@@ -0,0 +1,17 @@
+"""Convert spans to the Presidio recognizer-result shape. Core does not import Presidio."""
+
+from __future__ import annotations
+
+from edgepii.span import Span
+
+
+def to_presidio(spans: tuple[Span, ...] | list[Span]) -> list[dict]:
+    return [
+        {
+            "entity_type": span.type,
+            "start": span.start,
+            "end": span.end,
+            "score": 0.0 if span.score is None else span.score,
+        }
+        for span in spans
+    ]

edgepii/check.py

@@ -0,0 +1,14 @@
+"""Slice check. A failed hit is dropped. The locator does not search for a repair."""
+
+from __future__ import annotations
+
+from edgepii.protocol import Hit
+
+
+def slice_text(source: str, hit: Hit) -> str | None:
+    if hit.start < 0 or hit.end <= hit.start or hit.end > len(source):
+        return None
+    text = source[hit.start : hit.end]
+    if hit.claimed_text is not None and hit.claimed_text != text:
+        return None
+    return text

edgepii/detectors/gliner.py

@@ -0,0 +1,58 @@
+"""GLiNER v1 adapter for nvidia/gliner-PII and the urchade PII checkpoint."""
+
+from __future__ import annotations
+
+from edgepii.errors import DetectorLoadError
+from edgepii.protocol import Hit
+
+
+class GlinerDetector:
+    max_chars = 8_000
+    overlap_chars = 64
+
+    def __init__(
+        self,
+        repo_id: str,
+        revision: str,
+        device: str = "cpu",
+        candidate_floor: float | None = None,
+    ) -> None:
+        self.repo_id = repo_id
+        self.revision = revision
+        self.device = device
+        self.candidate_floor = candidate_floor
+        try:
+            from gliner import GLiNER
+        except ImportError as exc:
+            raise DetectorLoadError(
+                f"extra=gliner repo={repo_id} revision={revision}"
+            ) from exc
+        try:
+            self._model = GLiNER.from_pretrained(repo_id, revision=revision)
+        except TypeError:
+            self._model = GLiNER.from_pretrained(repo_id)
+        except Exception as exc:
+            raise DetectorLoadError(
+                f"extra=gliner repo={repo_id} revision={revision}"
+            ) from exc
+
+    def locate(self, source: str, labels: list[str]) -> tuple[Hit, ...]:
+        if source == "" or not labels:
+            return ()
+        options: dict = {}
+        # The checkpoint sets its own boundary. Pass no second, unmeasured one.
+        if self.candidate_floor is not None:
+            options["threshold"] = self.candidate_floor
+        rows = self._model.predict_entities(source, labels, **options)
+        hits: list[Hit] = []
+        for row in rows:
+            hits.append(
+                Hit(
+                    raw_label=str(row.get("label") or ""),
+                    start=int(row["start"]),
+                    end=int(row["end"]),
+                    score=None if row.get("score") is None else float(row["score"]),
+                    detector_id=f"gliner.{self.repo_id}",
+                )
+            )
+        return tuple(hits)

edgepii/detectors/gliner2.py

@@ -0,0 +1,77 @@
+"""GLiNER2 open-schema adapter. The type list is an input to the model."""
+
+from __future__ import annotations
+
+from edgepii.errors import DetectorLoadError
+from edgepii.protocol import Hit
+
+
+class Gliner2Detector:
+    max_chars = 8_000
+    overlap_chars = 64
+
+    def __init__(
+        self,
+        repo_id: str,
+        revision: str,
+        device: str = "cpu",
+        candidate_floor: float | None = None,
+    ) -> None:
+        self.repo_id = repo_id
+        self.revision = revision
+        self.device = device
+        self.candidate_floor = candidate_floor
+        try:
+            from gliner2 import GLiNER2
+        except ImportError as exc:
+            raise DetectorLoadError(
+                f"extra=gliner2 repo={repo_id} revision={revision}"
+            ) from exc
+        try:
+            self._model = GLiNER2.from_pretrained(repo_id, revision=revision)
+        except TypeError:
+            self._model = GLiNER2.from_pretrained(repo_id)
+        except Exception as exc:
+            raise DetectorLoadError(
+                f"extra=gliner2 repo={repo_id} revision={revision}"
+            ) from exc
+
+    def locate(self, source: str, labels: list[str]) -> tuple[Hit, ...]:
+        if source == "" or not labels:
+            return ()
+        options: dict = {
+            "include_confidence": True,
+            "include_spans": True,
+        }
+        # The checkpoint author chose its own decision boundary. This adapter
+        # adds no second one. A caller that measured a floor passes one.
+        if self.candidate_floor is not None:
+            options["threshold"] = self.candidate_floor
+        result = self._model.extract_entities(source, labels, **options)
+        return tuple(_hits_from_tree(result, f"gliner2.{self.repo_id}"))
+
+
+def _hits_from_tree(node: object, detector_id: str, label: str | None = None) -> list[Hit]:
+    hits: list[Hit] = []
+    if isinstance(node, dict) and "start" in node and "end" in node:
+        raw = str(node.get("label") or node.get("type") or label or "")
+        score = node.get("confidence", node.get("score"))
+        hits.append(
+            Hit(
+                raw_label=raw,
+                start=int(node["start"]),
+                end=int(node["end"]),
+                score=None if score is None else float(score),
+                detector_id=detector_id,
+            )
+        )
+        return hits
+    if isinstance(node, dict):
+        for key, value in node.items():
+            next_label = key if isinstance(key, str) else label
+            hits.extend(_hits_from_tree(value, detector_id, next_label))
+        return hits
+    if isinstance(node, list):
+        for item in node:
+            hits.extend(_hits_from_tree(item, detector_id, label))
+    return hits

edgepii/detectors/hf_token.py

@@ -0,0 +1,70 @@
+"""Hugging Face token classifiers. BERT and ModernBERT share this class."""
+
+from __future__ import annotations
+
+from edgepii.errors import DetectorLoadError
+from edgepii.protocol import Hit
+
+
+class HfTokenDetector:
+    """A window is 75 percent of the checkpoint max position to stay under it."""
+
+    default_context = 512
+    context_fraction = 0.75
+    overlap_chars = 64
+
+    def __init__(self, repo_id: str, revision: str, device: str = "cpu") -> None:
+        self.repo_id = repo_id
+        self.revision = revision
+        self.device = device
+        try:
+            from transformers import pipeline
+        except ImportError as exc:
+            raise DetectorLoadError(
+                f"extra=hf repo={repo_id} revision={revision}"
+            ) from exc
+        device_index = 0 if device == "cuda" else -1
+        try:
+            self._pipe = pipeline(
+                "token-classification",
+                model=repo_id,
+                revision=revision,
+                aggregation_strategy="simple",
+                device=device_index,
+            )
+        except Exception as exc:
+            raise DetectorLoadError(
+                f"extra=hf repo={repo_id} revision={revision}"
+            ) from exc
+        self.max_chars = self._window_from_context()
+
+    def _window_from_context(self) -> int:
+        """Read the checkpoint limit. Characters here are a safe lower bound."""
+
+        config = getattr(self._pipe.model, "config", None)
+        positions = getattr(config, "max_position_embeddings", None)
+        if not isinstance(positions, int) or positions <= 0:
+            positions = self.default_context
+        return max(1, int(positions * self.context_fraction))
+
+    def locate(self, source: str, labels: list[str]) -> tuple[Hit, ...]:
+        del labels
+        if source == "":
+            return ()
+        rows = self._pipe(source)
+        hits: list[Hit] = []
+        for row in rows:
+            start = int(row["start"])
+            end = int(row["end"])
+            raw = str(row.get("entity_group") or row.get("entity") or "")
+            score = float(row["score"])
+            hits.append(
+                Hit(
+                    raw_label=raw,
+                    start=start,
+                    end=end,
+                    score=score,
+                    detector_id=f"hf.{self.repo_id}",
+                )
+            )
+        return tuple(hits)

edgepii/detectors/rules.py

@@ -0,0 +1,228 @@
+"""Structured forms with a named validator.
+
+Every rule here answers one question: does a published grammar or checksum
+confirm this slice? A rule that a validator confirms scores 1.0. A rule that
+only matches a shape scores null. No rule invents a probability.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import re
+
+import phonenumbers
+from phonenumbers import PhoneNumberMatcher, ValidationResult
+
+from edgepii.protocol import Hit
+
+# RFC 5321 section 4.5.3.1: local part at most 64 octets, path at most 256.
+# The domain must hold at least one dot and a two-letter top label.
+_EMAIL = re.compile(
+    r"(?<![A-Za-z0-9._%+\-])"
+    r"([A-Za-z0-9!#$%&'*+/=?^_`{|}~\-]{1,64}"
+    r"(?:\.[A-Za-z0-9!#$%&'*+/=?^_`{|}~\-]{1,64})*)"
+    r"@"
+    r"((?:[A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z]{2,63})"
+    r"(?![A-Za-z0-9\-])"
+)
+
+# ISO/IEC 7812-1: a PAN holds 8 to 19 digits and starts with a major industry
+# identifier in the range 1 to 9. Separators are spaces or hyphens.
+_CARD = re.compile(r"(?<![A-Za-z0-9])\d(?:[ \-]?\d){7,18}(?![A-Za-z0-9])")
+
+# ISO 13616: two country letters, two check digits, then 11 to 30 alphanumerics.
+_IBAN = re.compile(r"(?<![A-Za-z0-9])[A-Z]{2}\d{2}[A-Z0-9]{11,30}(?![A-Za-z0-9])")
+
+# RFC 3986 section 3.2.2 for IPv4 text form. Octet ranges are checked by value.
+# A trailing sentence period is allowed. A trailing digit or dotted octet is not.
+_IPV4 = re.compile(r"(?<![0-9.])(?:\d{1,3}\.){3}\d{1,3}(?![0-9])(?!\.[0-9])")
+
+# RFC 4291 section 2.2 text forms. The stdlib validator decides membership.
+_IPV6 = re.compile(r"(?<![0-9A-Za-z:])(?:[0-9A-Fa-f]{0,4}:){2,7}[0-9A-Fa-f]{0,4}(?![0-9A-Za-z:])")
+
+# A cue word, then one of two forms.
+#   1. Assignment: cue, separator, value. The separator marks the value.
+#   2. Mention: cue, space, value, where the value holds a digit or a symbol.
+# Form 2 exists because prose says "the password hunter2". A bare word after a
+# cue is prose, so form 2 refuses it. That is a grammar rule, not a word list.
+_CUE_ASSIGN = re.compile(
+    r"(?i)\b(?:password|passcode|passwd|api[_\-]?key|access[_\-]?token|secret[_\-]?key)\b"
+    r"\s*[:=]\s*"
+    r"([^\s,.;:=\"'()<>\[\]]{1,256})"
+)
+_CUE_MENTION = re.compile(
+    r"(?i)\b(?:password|passcode|passwd|api[_\-]?key|access[_\-]?token|secret[_\-]?key)\b"
+    r"\s+"
+    r"([^\s,.;:=\"'()<>\[\]]{1,256})"
+)
+_SECRET_SHAPE = re.compile(r"[0-9!-/:-@\[-`{-~]")
+
+# ISO 3166 regions that the phone rule asks libphonenumber to try.
+PHONE_REGIONS = ("US", "CN", "HK", "GB")
+
+
+def _luhn_ok(digits: str) -> bool:
+    """ISO/IEC 7812-1 check digit."""
+
+    if not 8 <= len(digits) <= 19:
+        return False
+    if digits[0] == "0":
+        return False
+    total = 0
+    for index, char in enumerate(reversed(digits)):
+        value = ord(char) - 48
+        if index % 2 == 1:
+            value *= 2
+            if value > 9:
+                value -= 9
+        total += value
+    return total % 10 == 0
+
+
+def _iban_ok(raw: str) -> bool:
+    """ISO 13616 with the ISO 7064 MOD-97-10 check."""
+
+    compact = raw.replace(" ", "").upper()
+    if not 15 <= len(compact) <= 34:
+        return False
+    rearranged = compact[4:] + compact[:4]
+    number = ""
+    for char in rearranged:
+        if char.isdigit():
+            number += char
+        elif "A" <= char <= "Z":
+            number += str(ord(char) - 55)
+        else:
+            return False
+    return int(number) % 97 == 1
+
+
+def _ip_ok(raw: str) -> bool:
+    try:
+        ipaddress.ip_address(raw)
+    except ValueError:
+        return False
+    return True
+
+
+class RuleDetector:
+    """Validators and grammars. Person names are not this detector's job."""
+
+    max_chars = 10_000_000
+    overlap_chars = 0
+    detector_kind = "rules"
+
+    def locate(self, source: str, labels: list[str]) -> tuple[Hit, ...]:
+        allowed = {_norm(label) for label in labels}
+        hits: list[Hit] = []
+        if _wants(allowed, "email"):
+            hits.extend(_emails(source))
+        if _wants(allowed, "phone", "phone_number", "phonenumber"):
+            hits.extend(_phones(source))
+        if _wants(allowed, "card_number", "payment_card", "creditcardnumber", "credit_debit_card", "iban", "bank_account"):
+            hits.extend(_payments(source, allowed))
+        if _wants(allowed, "ip", "ip_address", "ipv4", "ipv6"):
+            hits.extend(_ips(source))
+        if _wants(allowed, "password", "passcode", "secret", "api_key", "access_token", "secret_key"):
+            hits.extend(_secrets(source))
+        return tuple(hits)
+
+
+def _norm(label: str) -> str:
+    return label.strip().casefold().replace(" ", "_")
+
+
+def _wants(allowed: set[str], *names: str) -> bool:
+    return any(name in allowed for name in names)
+
+
+def _emails(source: str) -> list[Hit]:
+    """RFC 5322 address in its common ASCII form. Shape only, so score is null."""
+
+    hits = []
+    for match in _EMAIL.finditer(source):
+        if len(match.group()) > 254:
+            continue
+        hits.append(Hit("email", match.start(), match.end(), None, "rules.email"))
+    return hits
+
+
+def _phones(source: str) -> list[Hit]:
+    """libphonenumber decides. Two accept tests, one per documented form.
+
+    A number that libphonenumber calls valid is a phone. A number that starts
+    with an explicit international prefix and has a plan-correct length is a
+    phone. Nothing else is. The region list only supplies a default region.
+    """
+
+    found: dict[tuple[int, int], float | None] = {}
+    for region in PHONE_REGIONS:
+        matcher = PhoneNumberMatcher(
+            source, region, leniency=phonenumbers.Leniency.POSSIBLE
+        )
+        for match in matcher:
+            key = (match.start, match.end)
+            text = source[match.start : match.end]
+            if phonenumbers.is_valid_number(match.number):
+                found[key] = 1.0
+                continue
+            if not text.startswith("+"):
+                continue
+            reason = phonenumbers.is_possible_number_with_reason(match.number)
+            if reason == ValidationResult.IS_POSSIBLE:
+                found.setdefault(key, None)
+    return [
+        Hit("phone", start, end, score, "rules.phone")
+        for (start, end), score in sorted(found.items())
+    ]
+
+
+def _payments(source: str, allowed: set[str]) -> list[Hit]:
+    hits: list[Hit] = []
+    iban_spans: list[tuple[int, int]] = []
+    if _wants(allowed, "iban", "bank_account"):
+        for match in _IBAN.finditer(source):
+            if not _iban_ok(match.group()):
+                continue
+            iban_spans.append((match.start(), match.end()))
+            hits.append(Hit("iban", match.start(), match.end(), 1.0, "rules.iban"))
+    if _wants(allowed, "card_number", "payment_card", "creditcardnumber", "credit_debit_card"):
+        for match in _CARD.finditer(source):
+            if _inside(match.start(), match.end(), iban_spans):
+                continue
+            raw = match.group()
+            digits = "".join(char for char in raw if char.isdigit())
+            if not _luhn_ok(digits):
+                continue
+            hits.append(Hit("card_number", match.start(), match.end(), 1.0, "rules.card"))
+    return hits
+
+
+def _inside(start: int, end: int, spans: list[tuple[int, int]]) -> bool:
+    return any(left <= start and end <= right for left, right in spans)
+
+
+def _ips(source: str) -> list[Hit]:
+    hits: list[Hit] = []
+    for pattern in (_IPV4, _IPV6):
+        for match in pattern.finditer(source):
+            if not _ip_ok(match.group()):
+                continue
+            hits.append(Hit("ip_address", match.start(), match.end(), 1.0, "rules.ip"))
+    return hits
+
+
+def _secrets(source: str) -> list[Hit]:
+    """A cue names a value. An assignment always counts, a mention needs a shape."""
+
+    spans: dict[tuple[int, int], Hit] = {}
+    for match in _CUE_ASSIGN.finditer(source):
+        key = (match.start(1), match.end(1))
+        spans[key] = Hit("password", key[0], key[1], None, "rules.secret")
+    for match in _CUE_MENTION.finditer(source):
+        value = match.group(1)
+        if _SECRET_SHAPE.search(value) is None:
+            continue
+        key = (match.start(1), match.end(1))
+        spans.setdefault(key, Hit("password", key[0], key[1], None, "rules.secret"))
+    return [spans[key] for key in sorted(spans)]

edgepii/detectors/uie.py

@@ -0,0 +1,107 @@
+"""ModelScope Siamese-UIE. A hit is public only when the slice equals the surface."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from edgepii.errors import DetectorLoadError
+from edgepii.protocol import Hit
+
+
+class UieDetector:
+    max_chars = 2_000
+    overlap_chars = 64
+
+    def __init__(
+        self,
+        repo_id: str,
+        revision: str,
+        schema: dict[str, str],
+        device: str = "cpu",
+        runner: Callable | None = None,
+    ) -> None:
+        self.repo_id = repo_id
+        self.revision = revision
+        self.schema = schema
+        self.device = device
+        self._runner = runner
+        if runner is not None:
+            return
+        try:
+            from modelscope.pipelines import pipeline
+            from modelscope.utils.constant import Tasks
+        except ImportError as exc:
+            raise DetectorLoadError(
+                f"extra=ms repo={repo_id} revision={revision}"
+            ) from exc
+        try:
+            self._pipe = pipeline(Tasks.siamese_uie, model=repo_id, model_revision=revision)
+        except Exception as exc:
+            raise DetectorLoadError(
+                f"extra=ms repo={repo_id} revision={revision}"
+            ) from exc
+
+    def locate(self, source: str, labels: list[str]) -> tuple[Hit, ...]:
+        if source == "":
+            return ()
+        allowed = {label.strip() for label in labels}
+        schema = {
+            name: None
+            for name in self.schema.values()
+            if name in allowed
+        }
+        if not schema:
+            return ()
+        if self._runner is not None:
+            raw = self._runner(source, schema)
+        else:
+            raw = self._pipe(source, schema=schema)
+        hits: list[Hit] = []
+        for label, items in _pairs(raw):
+            if not isinstance(items, list):
+                continue
+            for item in items:
+                hit = _one(source, label, item, f"uie.{self.repo_id}")
+                if hit is not None:
+                    hits.append(hit)
+        return tuple(hits)
+
+
+def _pairs(raw: object) -> list[tuple[str, object]]:
+    if isinstance(raw, dict) and "output" in raw and isinstance(raw["output"], dict):
+        raw = raw["output"]
+    if isinstance(raw, dict):
+        return [(str(key), value) for key, value in raw.items()]
+    return []
+
+
+def _one(source: str, label: str, item: object, detector_id: str) -> Hit | None:
+    if not isinstance(item, dict):
+        return None
+    surface = str(item.get("span") or item.get("text") or "")
+    start, end = _bounds(item)
+    if start is None or end is None or surface == "":
+        return None
+    if source[start:end] == surface:
+        chosen = end
+    elif end + 1 <= len(source) and source[start : end + 1] == surface:
+        chosen = end + 1
+    else:
+        return None
+    score = item.get("prob", item.get("probs", item.get("score")))
+    return Hit(
+        raw_label=str(item.get("type") or label),
+        start=start,
+        end=chosen,
+        score=None if score is None else float(score),
+        detector_id=detector_id,
+    )
+
+
+def _bounds(item: dict) -> tuple[int | None, int | None]:
+    if "start" in item and "end" in item:
+        return int(item["start"]), int(item["end"])
+    offset = item.get("offset")
+    if isinstance(offset, (list, tuple)) and len(offset) == 2:
+        return int(offset[0]), int(offset[1])
+    return None, None

edgepii/errors.py

@@ -0,0 +1,22 @@
+"""Errors for the public locator. Messages do not include the source string."""
+
+
+class UnknownType(ValueError):
+    """The caller named a type that is not in the active taxonomy."""
+
+    def __init__(self, name: str) -> None:
+        self.name = name
+        super().__init__(name)
+
+
+class DetectorLoadError(RuntimeError):
+    """Weights or an extra failed to load. The message has no source text."""
+
+
+class OverlapError(ValueError):
+    """redact saw two spans that share a character."""
+
+    def __init__(self, start: int, end: int) -> None:
+        self.start = start
+        self.end = end
+        super().__init__(f"{start}:{end}")

edgepii/eval/__init__.py

@@ -0,0 +1,32 @@
+"""Scoring. Exact-span F1, the pinned public corpora, and the run report.
+
+`score` holds the metric. `corpora` holds the pins and the label fold. `runner`
+holds the per-document pass. `cli` is the entry point:
+
+    python -m edgepii.eval.cli --preset rules --corpus nemotron-pii
+"""
+
+from edgepii.eval.corpora import (
+    CorpusError,
+    CorpusSpec,
+    GoldDiagnostics,
+    corpus_keys,
+    load_spec,
+)
+from edgepii.eval.runner import RunReport, SliceScore, score_corpus, score_docs
+from edgepii.eval.score import ScoreReport, exact_span_f1, leakage_count
+
+__all__ = [
+    "CorpusError",
+    "CorpusSpec",
+    "GoldDiagnostics",
+    "RunReport",
+    "ScoreReport",
+    "SliceScore",
+    "corpus_keys",
+    "exact_span_f1",
+    "leakage_count",
+    "load_spec",
+    "score_corpus",
+    "score_docs",
+]