credgate 0.1.0__py3-none-any.whl

credgate/__init__.py

@@ -0,0 +1,47 @@
+"""CredGate: governance layer for LLM requests with Indian PII detection.
+
+Detection rules come from the shared spec (see :func:`spec_path`), so this
+package and the R package find exactly the same matches.
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("credgate")
+except PackageNotFoundError:  # running from a source checkout
+    __version__ = "0.1.0"
+
+from ._spec import spec_path
+from .checksums import (
+    gstin_check_char,
+    gstin_valid,
+    luhn_check_digit,
+    luhn_valid,
+    verhoeff_check_digit,
+    verhoeff_valid,
+)
+from .detect import Span, detect_pii, mask_value
+from .detectors import Detector, bounded, detectors, register_detector, reset_detectors
+from .normalise import Normalised, normalise
+from .synth import synth_indian_pii
+
+__all__ = [
+    "Detector",
+    "Normalised",
+    "Span",
+    "bounded",
+    "detect_pii",
+    "detectors",
+    "gstin_check_char",
+    "gstin_valid",
+    "luhn_check_digit",
+    "luhn_valid",
+    "mask_value",
+    "normalise",
+    "register_detector",
+    "reset_detectors",
+    "spec_path",
+    "synth_indian_pii",
+    "verhoeff_check_digit",
+    "verhoeff_valid",
+]

credgate/_spec.py

@@ -0,0 +1,64 @@
+"""Reading the shared detection spec shipped inside the package.
+
+The spec (``spec/`` in the repository, copied here by ``tools/sync-spec.sh``)
+is shared with the R package; see its README.md for the schema.
+"""
+
+from __future__ import annotations
+
+import functools
+from importlib import resources
+from pathlib import Path
+from typing import Any
+
+import regex
+import yaml
+
+SPEC_VERSION_SUPPORTED = 1
+
+
+def spec_path() -> Path:
+    """Path of the ``spec`` directory shipped with the package."""
+    return Path(str(resources.files("credgate") / "spec"))
+
+
+def read_spec_file(name: str, directory: Path | None = None) -> dict[str, Any]:
+    path = (directory or spec_path()) / name
+    with open(path, encoding="utf-8") as f:
+        data = yaml.safe_load(f)
+    if data.get("spec_version") != SPEC_VERSION_SUPPORTED:
+        raise ValueError(
+            f"Unsupported spec_version {data.get('spec_version')} in {name} "
+            f"(this credgate supports {SPEC_VERSION_SUPPORTED})."
+        )
+    return data
+
+
+def _code_points(values: list[str]) -> frozenset[int]:
+    return frozenset(int(v.removeprefix("U+"), 16) for v in values)
+
+
+@functools.cache
+def engine() -> dict[str, Any]:
+    """Engine parameters from engine.yaml, with code points parsed."""
+    spec = read_spec_file("engine.yaml")
+    norm = spec["normalisation"]
+    return {
+        "digit_zeros": sorted(_code_points(norm["digit_zeros"])),
+        "remove": _code_points(norm["remove"]),
+        "dash": _code_points(norm["dash"]),
+        "boundary": spec["boundary"],
+        "separators": regex.compile(spec["separators"]),
+        "scoring": spec["scoring"],
+    }
+
+
+@functools.cache
+def detector_spec() -> dict[str, Any]:
+    """The parsed detectors.yaml."""
+    spec = read_spec_file("detectors.yaml")
+    spec["lists"] = {k: [str(x) for x in v] for k, v in spec.get("lists", {}).items()}
+    spec["tables"] = {
+        k: {str(key): str(val) for key, val in v.items()} for k, v in spec.get("tables", {}).items()
+    }
+    return spec

credgate/checksums.py

@@ -0,0 +1,136 @@
+"""Checksum algorithms used to validate identifier candidates.
+
+All functions ignore spaces and hyphens and treat anything else that is not
+the expected characters as invalid.
+"""
+
+from __future__ import annotations
+
+import regex
+
+from ._spec import detector_spec, engine
+
+_VERHOEFF_D = (
+    (0, 1, 2, 3, 4, 5, 6, 7, 8, 9),
+    (1, 2, 3, 4, 0, 6, 7, 8, 9, 5),
+    (2, 3, 4, 0, 1, 7, 8, 9, 5, 6),
+    (3, 4, 0, 1, 2, 8, 9, 5, 6, 7),
+    (4, 0, 1, 2, 3, 9, 5, 6, 7, 8),
+    (5, 9, 8, 7, 6, 0, 4, 3, 2, 1),
+    (6, 5, 9, 8, 7, 1, 0, 4, 3, 2),
+    (7, 6, 5, 9, 8, 2, 1, 0, 4, 3),
+    (8, 7, 6, 5, 9, 3, 2, 1, 0, 4),
+    (9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
+)
+_VERHOEFF_P = (
+    (0, 1, 2, 3, 4, 5, 6, 7, 8, 9),
+    (1, 5, 7, 6, 2, 8, 3, 0, 9, 4),
+    (5, 8, 0, 3, 7, 9, 6, 1, 4, 2),
+    (8, 9, 1, 6, 0, 4, 3, 5, 2, 7),
+    (9, 4, 5, 3, 1, 2, 6, 8, 7, 0),
+    (4, 2, 8, 6, 5, 7, 3, 9, 0, 1),
+    (2, 7, 9, 3, 8, 0, 6, 4, 1, 5),
+    (7, 0, 4, 6, 9, 1, 3, 2, 5, 8),
+)
+_VERHOEFF_INV = (0, 4, 3, 2, 1, 5, 6, 7, 8, 9)
+
+_DIGITS = regex.compile(r"[0-9]+")
+_GSTIN_BODY = regex.compile(r"[0-9A-Z]{14}")
+_GSTIN_FULL = regex.compile(r"[0-9]{2}[A-Z]{5}[0-9]{4}[A-Z][1-9A-Z]Z[0-9A-Z]")
+_GSTIN_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+
+def strip_separators(x: str) -> str:
+    """Remove the spec's separator characters (spaces and hyphens)."""
+    return engine()["separators"].sub("", x)
+
+
+def _digits(x: str | None) -> str | None:
+    if x is None:
+        return None
+    d = strip_separators(x)
+    return d if _DIGITS.fullmatch(d) else None
+
+
+def _verhoeff_accumulate(digits: str, offset: int) -> int:
+    c = 0
+    for i, ch in enumerate(reversed(digits)):
+        c = _VERHOEFF_D[c][_VERHOEFF_P[(i + offset) % 8][int(ch)]]
+    return c
+
+
+def verhoeff_valid(x: str | None) -> bool:
+    """Whether a number (with its final check digit) passes Verhoeff.
+
+    Used by Aadhaar and Aadhaar Virtual ID.
+    """
+    d = _digits(x)
+    return d is not None and _verhoeff_accumulate(d, 0) == 0
+
+
+def verhoeff_check_digit(x: str | None) -> str | None:
+    """The Verhoeff check digit to append to ``x``, or ``None`` if invalid."""
+    d = _digits(x)
+    if d is None:
+        return None
+    return str(_VERHOEFF_INV[_verhoeff_accumulate(d, 1)])
+
+
+def _luhn_sum(digits: str, with_check: bool) -> int:
+    # With the check digit present, every second digit from the right is
+    # doubled; when computing a check digit it is the first, third, ...
+    doubled = 1 if with_check else 0
+    total = 0
+    for i, ch in enumerate(reversed(digits)):
+        d = int(ch)
+        if i % 2 == doubled:
+            d *= 2
+            if d > 9:
+                d -= 9
+        total += d
+    return total
+
+
+def luhn_valid(x: str | None) -> bool:
+    """Whether a number passes the Luhn checksum (payment cards)."""
+    d = _digits(x)
+    return d is not None and _luhn_sum(d, with_check=True) % 10 == 0
+
+
+def luhn_check_digit(x: str | None) -> str | None:
+    """The Luhn check digit to append to ``x``, or ``None`` if invalid."""
+    d = _digits(x)
+    if d is None:
+        return None
+    return str((10 - _luhn_sum(d, with_check=False) % 10) % 10)
+
+
+def gstin_check_char(x: str | None) -> str | None:
+    """The mod-36 check character for the first 14 characters of a GSTIN."""
+    if x is None:
+        return None
+    x = x.upper()
+    if not _GSTIN_BODY.fullmatch(x):
+        return None
+    total = 0
+    for i, ch in enumerate(x, start=1):
+        # Counting from the right, the last character has factor 2, then 1, 2...
+        addend = _GSTIN_ALPHABET.index(ch) * (2 if i % 2 == 0 else 1)
+        total += addend // 36 + addend % 36
+    return _GSTIN_ALPHABET[(36 - total % 36) % 36]
+
+
+def gstin_valid(x: str | None) -> bool:
+    """Whether ``x`` is a structurally valid GSTIN with a correct check character.
+
+    A GSTIN is a 2-digit state code, the holder's PAN, an entity number, the
+    letter ``Z`` and the check character.
+    """
+    if x is None:
+        return False
+    x = x.upper()
+    return (
+        _GSTIN_FULL.fullmatch(x) is not None
+        and x[:2] in detector_spec()["lists"]["gst_state_codes"]
+        and x[14] == gstin_check_char(x[:14])
+    )

credgate/cli.py

@@ -0,0 +1,80 @@
+"""Command-line interface: ``credgate detect``, ``credgate detectors``."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections.abc import Sequence
+
+from . import __version__
+from ._spec import spec_path
+from .detect import detect_pii, mask_value
+from .detectors import detectors
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="credgate",
+        description="Detect Indian PII before it leaves the machine.",
+    )
+    parser.add_argument("--version", action="version", version=f"credgate {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p = sub.add_parser("detect", help="detect PII in files or standard input")
+    p.add_argument("files", nargs="*", help="files to scan (default: standard input)")
+    p.add_argument("--lines", action="store_true", help="treat each line as a separate document")
+    p.add_argument("--json", action="store_true", help="write one JSON object per match")
+    p.add_argument(
+        "--reveal",
+        action="store_true",
+        help="show detected values and canonical forms (masked by default)",
+    )
+    p.add_argument("--min-confidence", type=float, default=0.0, metavar="X")
+    p.add_argument("--context", default=None, help="extra context applied to every match")
+    p.add_argument(
+        "--fail",
+        action="store_true",
+        help="exit with status 1 if anything is detected (for CI and pre-commit hooks)",
+    )
+
+    sub.add_parser("detectors", help="list the registered detectors")
+    sub.add_parser("spec-path", help="print the path of the shipped detection spec")
+
+    args = parser.parse_args(argv)
+    if args.command == "detectors":
+        for d in detectors():
+            print(f"{d['id']:<16} {d['class']:<16} {d['label']}")
+        return 0
+    if args.command == "spec-path":
+        print(spec_path())
+        return 0
+    return _detect(args)
+
+
+def _detect(args: argparse.Namespace) -> int:
+    sources = args.files or ["-"]
+    found = 0
+    for source in sources:
+        if source == "-":
+            content = sys.stdin.read()
+        else:
+            with open(source, encoding="utf-8") as f:
+                content = f.read()
+        docs = content.splitlines() if args.lines else [content]
+        spans = detect_pii(docs, context=args.context, min_confidence=args.min_confidence)
+        found += len(spans)
+        for s in spans:
+            location = s.doc + 1 if args.lines else None
+            if args.json:
+                record = {"source": source, "line": location, **s.to_dict(reveal=args.reveal)}
+                print(json.dumps(record, ensure_ascii=False))
+            else:
+                where = f"{source}:{location}" if location else source
+                value = s.value if args.reveal else mask_value(s.value)
+                print(f"{where}\t{s.start}-{s.end}\t{s.type}\t{s.confidence:.2f}\t{value}")
+    return 1 if args.fail and found else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

credgate/detect.py

@@ -0,0 +1,203 @@
+"""PII detection over text."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import asdict, dataclass
+from typing import Any
+
+import regex
+
+from ._spec import engine
+from .detectors import Detector, get_detectors
+from .normalise import Normalised, normalise
+
+
+@dataclass(frozen=True)
+class Span:
+    """One detected PII match.
+
+    ``start`` and ``end`` are 0-based, end-exclusive code-point offsets into
+    the original text of document ``doc``. The representation masks
+    ``value``; read the attribute to see it.
+    """
+
+    doc: int
+    start: int
+    end: int
+    type: str
+    data_class: str
+    value: str
+    canonical: str
+    confidence: float
+    reasons: str
+
+    def __repr__(self) -> str:
+        return (
+            f"Span(doc={self.doc}, start={self.start}, end={self.end}, type={self.type!r}, "
+            f"value={mask_value(self.value)!r}, confidence={self.confidence})"
+        )
+
+    def to_dict(self, reveal: bool = False) -> dict[str, Any]:
+        """A JSON-ready dict; ``value`` is masked and ``canonical`` omitted unless ``reveal``."""
+        d = asdict(self)
+        d["class"] = d.pop("data_class")
+        if not reveal:
+            d["value"] = mask_value(self.value)
+            del d["canonical"]
+        return d
+
+
+def detect_pii(
+    text: str | Sequence[str | None],
+    detectors: Sequence[str] | None = None,
+    context: str | Sequence[str | None] | None = None,
+    min_confidence: float = 0.0,
+    window: int | None = None,
+) -> list[Span]:
+    """Detect PII in one or more documents.
+
+    Runs every registered detector over normalised text, validates
+    candidates (checksums, code tables), scores them from nearby context and
+    resolves overlapping matches (the more confident, then the longer, span
+    wins - so a GSTIN is reported once, not also as the PAN inside it).
+
+    Args:
+        text: A string, or a sequence of strings (``None`` counts as empty).
+        detectors: Detector ids to run; ``None`` runs all registered ones.
+        context: Extra context per document (one string for all), treated as
+            if it appeared next to every match - e.g. a column name, so a
+            column called ``voter_id`` supports Voter ID matches.
+        min_confidence: Drop matches scoring below this value.
+        window: Characters either side of a match searched for context
+            keywords; ``None`` uses the spec's ``context_window``.
+
+    Returns:
+        Spans ordered by document and position.
+    """
+    docs = [text] if isinstance(text, str) else ["" if t is None else t for t in text]
+    if context is None or isinstance(context, str):
+        contexts = [context or ""] * len(docs)
+    else:
+        contexts = ["" if c is None else c for c in context]
+        if len(contexts) != len(docs):
+            raise ValueError("`context` must be one string or one per document.")
+    if window is None:
+        window = engine()["scoring"]["context_window"]
+
+    norms = [normalise(t) for t in docs]
+    spans: list[Span] = []
+    for det in get_detectors(detectors):
+        spans.extend(_run_detector(det, docs, norms, contexts, window))
+    spans = _resolve_overlaps(spans)
+    spans = [s for s in spans if s.confidence >= min_confidence]
+    spans.sort(key=lambda s: (s.doc, s.start))
+    return spans
+
+
+def _run_detector(
+    det: Detector,
+    docs: list[str],
+    norms: list[Normalised],
+    contexts: list[str],
+    window: int,
+) -> list[Span]:
+    patterns, context_re, negative_re = det._compiled
+    scoring = engine()["scoring"]
+    out: list[Span] = []
+    for doc, (original, norm) in enumerate(zip(docs, norms, strict=True)):
+        ntext = norm.text
+        for pattern in patterns:
+            for m in pattern.finditer(ntext):
+                s, e = m.span(det.capture) if det.capture else m.span()
+                if s < 0:
+                    continue
+                matched = ntext[s:e]
+                if det.validate is not None and not det.validate(matched):
+                    continue
+
+                around = f"{ntext[max(0, s - window) : s]} {ntext[e : e + window]} {contexts[doc]}"
+                ctx_hit = _keyword(context_re, around)
+                neg_hit = _keyword(negative_re, around)
+                if det.context_required and ctx_hit is None:
+                    continue
+
+                score = det.base(matched) if callable(det.base) else det.base
+                if ctx_hit is not None:
+                    score += (1 - score) * scoring["context_boost"]
+                if neg_hit is not None:
+                    score *= scoring["negative_factor"]
+
+                reasons = f"matched {det.label} pattern"
+                if det.checks:
+                    reasons += f"; {det.checks}"
+                if ctx_hit is not None:
+                    reasons += f"; context '{ctx_hit}'"
+                if neg_hit is not None:
+                    reasons += f"; negative context '{neg_hit}'"
+                if det.describe is not None:
+                    reasons += f"; {det.describe(matched)}"
+
+                if norm.map is None:
+                    start, end = s, e
+                else:
+                    start, end = norm.map[s], norm.map[e - 1] + 1
+                out.append(
+                    Span(
+                        doc=doc,
+                        start=start,
+                        end=end,
+                        type=det.id,
+                        data_class=det.data_class,
+                        value=original[start:end],
+                        canonical=det.canonical(matched),
+                        confidence=round(score, scoring["digits"]),
+                        reasons=reasons,
+                    )
+                )
+    return out
+
+
+def _keyword(pattern: regex.Pattern[str] | None, text: str) -> str | None:
+    if pattern is None:
+        return None
+    m = pattern.search(text)
+    return m.group(0) if m else None
+
+
+def _resolve_overlaps(spans: list[Span]) -> list[Span]:
+    """Within each cluster of overlapping spans, keep the most confident, then
+    the longest, span; repeat with what remains."""
+    spans = sorted(spans, key=lambda s: (s.doc, s.start))
+    kept: list[Span] = []
+    cluster: list[Span] = []
+    running_end = -1
+    for s in [*spans, None]:
+        if s is not None and cluster and s.doc == cluster[0].doc and s.start < running_end:
+            cluster.append(s)
+            running_end = max(running_end, s.end)
+            continue
+        if len(cluster) == 1:
+            kept.extend(cluster)
+        elif cluster:
+            taken: list[Span] = []
+            for c in sorted(cluster, key=lambda c: (-c.confidence, -(c.end - c.start))):
+                if not any(c.start < t.end and c.end > t.start for t in taken):
+                    taken.append(c)
+            kept.extend(taken)
+        if s is not None:
+            cluster = [s]
+            running_end = s.end
+    return kept
+
+
+_ALNUM = regex.compile(r"[\p{L}\p{N}]")
+
+
+def mask_value(x: str) -> str:
+    """Replace letters and digits with ``X``, keeping the last 4 characters
+    (2 for values shorter than 8) - the same convention as UIDAI's masked
+    Aadhaar."""
+    keep = 4 if len(x) >= 8 else 2
+    cut = max(0, len(x) - keep)
+    return _ALNUM.sub("X", x[:cut]) + x[cut:]