deidkit 0.1.0__py3-none-any.whl

deidkit/dates.py

@@ -0,0 +1,229 @@
+"""Interval-preserving date shifting (HIPAA-style date jitter).
+
+Every date belonging to one entity (default: one ``patient_id``) is shifted by
+the *same* deterministic offset, drawn from ``[-max_days, +max_days]``. Because
+the offset is identical for all of a patient's dates:
+
+* intervals are preserved exactly — length of stay, time-to-result, gaps between
+  encounters, and age (encounter minus birth) are unchanged;
+* the absolute calendar date is obscured, and offsets differ per patient so you
+  cannot line two patients up on the real timeline.
+
+This is the "shift, don't reduce fidelity" alternative to truncating dates to
+the year. Fidelity is fully retained; only the anchor moves.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from datetime import date, datetime, timedelta
+from typing import Optional, Tuple
+
+import pandas as pd
+
+from .secret import digest_int
+
+# Month names + abbreviations, Spanish AND English -> number.
+_MONTHS = {
+    "ene": 1, "enero": 1, "jan": 1, "january": 1,
+    "feb": 2, "febrero": 2, "february": 2,
+    "mar": 3, "marzo": 3, "march": 3,
+    "abr": 4, "abril": 4, "apr": 4, "april": 4,
+    "may": 5, "mayo": 5,
+    "jun": 6, "junio": 6, "june": 6,
+    "jul": 7, "julio": 7, "july": 7,
+    "ago": 8, "agosto": 8, "aug": 8, "august": 8,
+    "sep": 9, "sept": 9, "set": 9, "septiembre": 9, "setiembre": 9, "september": 9,
+    "oct": 10, "octubre": 10, "october": 10,
+    "nov": 11, "noviembre": 11, "november": 11,
+    "dic": 12, "diciembre": 12, "dec": 12, "december": 12,
+}
+_MONTHS_ES_REV = {
+    1: "enero", 2: "febrero", 3: "marzo", 4: "abril", 5: "mayo", 6: "junio",
+    7: "julio", 8: "agosto", 9: "septiembre", 10: "octubre", 11: "noviembre",
+    12: "diciembre",
+}
+
+_NUM_DMY = re.compile(r"^(\d{1,2})([/\-.])(\d{1,2})([/\-.])(\d{2,4})$")
+_NUM_YMD = re.compile(r"^(\d{4})([/\-.])(\d{1,2})([/\-.])(\d{1,2})$")
+# ISO/other date followed by a time component -> shift the date, keep the time.
+_DATETIME = re.compile(
+    r"^(\d{4}[/\-.]\d{1,2}[/\-.]\d{1,2}|\d{1,2}[/\-.]\d{1,2}[/\-.]\d{2,4})"
+    r"([ T])(\d{1,2}:\d{2}(?::\d{2})?)$"
+)
+_LONG_ES = re.compile(
+    r"^(\d{1,2})\s+de\s+([A-Za-zÁÉÍÓÚáéíóúñ]+)\s+(?:de\s+)?(\d{4})$", re.IGNORECASE
+)
+# 05-ABR-1988 / 5 March 1988 / 5th March 2021 (day [ordinal] monthname year)
+_DMY_ALPHA = re.compile(
+    r"^(\d{1,2})(?:st|nd|rd|th)?[-/ ]([A-Za-zÁÉÍÓÚáéíóúñ]{3,})[-/ ](\d{2,4})$",
+    re.IGNORECASE,
+)
+# March 5, 2021 / Mar 5 2021 / March 5th, 2021 (monthname day [ordinal] year)
+_MDY_ALPHA = re.compile(
+    r"^([A-Za-zÁÉÍÓÚáéíóúñ]{3,})\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(\d{4})$",
+    re.IGNORECASE,
+)
+
+
+def _pivot_year(y: int) -> int:
+    if y >= 100:
+        return y
+    return 2000 + y if y <= 69 else 1900 + y
+
+
+class DateShifter:
+    def __init__(self, secret: bytes, max_days: int = 365):
+        self.secret = secret
+        self.max_days = max(int(max_days), 1)
+        self._cache = {}
+
+    # ------------------------------------------------------------------ #
+    def offset_days(self, entity_id) -> int:
+        """Deterministic offset in ``[-max_days, +max_days] \\ {0}`` for an entity."""
+        if entity_id is None or (isinstance(entity_id, float) and math.isnan(entity_id)):
+            key = "__GLOBAL__"
+        else:
+            key = str(entity_id)
+        cached = self._cache.get(key)
+        if cached is not None:
+            return cached
+        val = digest_int(self.secret, "dateshift|", key)
+        span = 2 * self.max_days + 1
+        off = val % span - self.max_days
+        if off == 0:  # never leave a date unmoved
+            off = 1 if (val >> 3) % 2 else -1
+        self._cache[key] = off
+        return off
+
+    # ------------------------------------------------------------------ #
+    def shift_scalar(self, value, entity_id) -> Tuple[object, bool]:
+        """Shift a single value; return ``(new_value, changed)``."""
+        if value is None or (isinstance(value, float) and math.isnan(value)):
+            return value, False
+        try:
+            if pd.isna(value):
+                return value, False
+        except (TypeError, ValueError):
+            pass
+        off = self.offset_days(entity_id)
+        delta = timedelta(days=off)
+
+        if isinstance(value, pd.Timestamp):
+            return value + delta, True
+        if isinstance(value, datetime):
+            return value + delta, True
+        if isinstance(value, date):
+            return value + delta, True
+        if isinstance(value, str):
+            shifted = self._shift_string(value.strip(), off)
+            if shifted is not None:
+                return shifted, True
+            return value, False
+        return value, False
+
+    # ------------------------------------------------------------------ #
+    def shift_series(self, series: pd.Series, entity_ids: pd.Series) -> pd.Series:
+        """Shift a whole column. Returns a new Series (same index).
+
+        * real ``datetime64`` dtype -> vectorized add (fast path).
+        * string/object -> per-element, FORMAT-PRESERVING shift via
+          :meth:`_shift_string` (handles ISO, ``dd/mm/yyyy``, datetime-with-time,
+          Spanish/English month names & abbreviations). Element-wise avoids the
+          single-format inference that used to silently drop odd rows.
+        """
+        if pd.api.types.is_datetime64_any_dtype(series):
+            offs = entity_ids.map(self.offset_days).astype("float64")
+            return series + pd.to_timedelta(offs, unit="D")
+
+        return pd.Series(
+            [self.shift_scalar(v, e)[0] for v, e in zip(series, entity_ids)],
+            index=series.index,
+        )
+
+    @staticmethod
+    def _parse_datetime(series: pd.Series) -> pd.Series:
+        import warnings
+
+        # Decide day-first vs year-first from the data. ISO (YYYY-MM-DD) MUST be
+        # parsed with dayfirst=False — pandas mis-swaps month/day otherwise. Only
+        # dd/mm/yyyy strings need dayfirst=True.
+        sample = series.dropna().astype(str).head(50)
+        if len(sample):
+            year_first = sample.str.match(r"^\d{4}[-/.]").mean()
+        else:
+            year_first = 0.0
+        dayfirst = year_first < 0.5
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            return pd.to_datetime(series, errors="coerce", dayfirst=dayfirst)
+
+    # ------------------------------------------------------------------ #
+    def shift_fragment(self, fragment: str, entity_id) -> Optional[str]:
+        """Shift a date substring found in free text; ``None`` if unparseable."""
+        return self._shift_string(fragment.strip(), self.offset_days(entity_id))
+
+    # ------------------------------------------------------------------ #
+    def _shift_string(self, s: str, off: int) -> Optional[str]:
+        # datetime with a time component: shift the date, keep the time.
+        m = _DATETIME.match(s)
+        if m:
+            date_part, sep, time_part = m.groups()
+            shifted = self._shift_string(date_part, off)
+            return f"{shifted}{sep}{time_part}" if shifted else None
+        m = _NUM_YMD.match(s)
+        if m:
+            y, s1, mo, s2, d = m.groups()
+            dt = self._safe_date(int(y), int(mo), int(d))
+            if dt:
+                dt += timedelta(days=off)
+                return f"{dt.year:04d}{s1}{dt.month:02d}{s2}{dt.day:02d}"
+            return None
+        m = _NUM_DMY.match(s)
+        if m:
+            d, s1, mo, s2, y = m.groups()
+            dt = self._safe_date(_pivot_year(int(y)), int(mo), int(d))
+            if dt:
+                dt += timedelta(days=off)
+                yout = f"{dt.year % 100:02d}" if len(y) == 2 else f"{dt.year:04d}"
+                return f"{dt.day:02d}{s1}{dt.month:02d}{s2}{yout}"
+            return None
+        m = _LONG_ES.match(s)
+        if m:
+            d, mon, y = m.groups()
+            mo = _MONTHS.get(mon.lower())
+            if mo:
+                dt = self._safe_date(int(y), mo, int(d))
+                if dt:
+                    dt += timedelta(days=off)
+                    return f"{dt.day} de {_MONTHS_ES_REV[dt.month]} de {dt.year}"
+            return None
+        m = _DMY_ALPHA.match(s)  # 05-ABR-1988 / 5 March 1988
+        if m:
+            d, mon, y = m.groups()
+            mo = _MONTHS.get(mon.lower())
+            if mo:
+                dt = self._safe_date(_pivot_year(int(y)), mo, int(d))
+                if dt:
+                    dt += timedelta(days=off)
+                    return f"{dt.year:04d}-{dt.month:02d}-{dt.day:02d}"
+            return None
+        m = _MDY_ALPHA.match(s)  # March 5, 2021
+        if m:
+            mon, d, y = m.groups()
+            mo = _MONTHS.get(mon.lower())
+            if mo:
+                dt = self._safe_date(int(y), mo, int(d))
+                if dt:
+                    dt += timedelta(days=off)
+                    return f"{dt.year:04d}-{dt.month:02d}-{dt.day:02d}"
+            return None
+        return None
+
+    @staticmethod
+    def _safe_date(y: int, m: int, d: int) -> Optional[datetime]:
+        try:
+            return datetime(y, m, d)
+        except ValueError:
+            return None

deidkit/detect/__init__.py

@@ -0,0 +1,19 @@
+"""Multi-stage PII detection for free text.
+
+Stages (combined by :class:`~deidkit.detect.pipeline.Detector`):
+
+1. ``patterns``   - regex for structured identifiers (email, phone, national ID,
+   medical-record number, URL, in-text dates). High precision, always accepted.
+2. ``context``    - honorific / role-anchored person names ("Dr. X Y",
+   "paciente: X Y"). Captures names structurally, even out-of-gazetteer.
+3. ``gazetteer``  - dictionary match against curated given-name + surname lists.
+   A lone hit is only *proposed*; it is accepted when corroborated.
+4. ``spacy_ner``  - optional statistical PERSON model, used as an independent
+   detector and to corroborate gazetteer hits.
+
+A voting gate then decides which detections are confident enough to replace and
+which go to the human review queue.
+"""
+
+from .types import Detection, entity  # noqa: F401
+from .pipeline import Detector  # noqa: F401

deidkit/detect/checksums.py

@@ -0,0 +1,160 @@
+"""National-ID checksum validators + opt-in discovery.
+
+Most national IDs carry a check digit. Validating it means a *bare* ID (no cue
+word) can be detected with near-zero false positives — a random number almost
+never satisfies a mod-11 / mod-23 check. This both closes a recall gap (bare
+cédula/CPF in a note) and keeps precision high.
+
+Opt-in per region via ``Policy.id_checksums`` (e.g. ``["CPF","CNPJ","CNS"]`` for
+Brazil, ``["DNI","NIE"]`` for Spain, ``["NHS"]`` for the UK). Off by default so a
+10-digit phone is never mistaken for a UK NHS number in a Colombian dataset.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Dict, List, Tuple
+
+from .types import Detection, NATIONAL_ID
+
+_DNI_LETTERS = "TRWAGMYFPDXBNJZSQVHLCKE"
+
+
+def _all_same(s: str) -> bool:
+    return len(set(s)) == 1
+
+
+# --- validators (return True if the checksum/format holds) ----------------- #
+def valid_cpf(s: str) -> bool:
+    d = [int(c) for c in s if c.isdigit()]
+    if len(d) != 11 or _all_same("".join(map(str, d))):
+        return False
+    for n in (9, 10):
+        r = sum(d[i] * (n + 1 - i) for i in range(n)) % 11
+        dv = 0 if r < 2 else 11 - r
+        if d[n] != dv:
+            return False
+    return True
+
+
+def valid_cnpj(s: str) -> bool:
+    d = [int(c) for c in s if c.isdigit()]
+    if len(d) != 14 or _all_same("".join(map(str, d))):
+        return False
+    w1 = [5, 4, 3, 2, 9, 8, 7, 6, 5, 4, 3, 2]
+    w2 = [6] + w1
+    for n, w in ((12, w1), (13, w2)):
+        r = sum(d[i] * w[i] for i in range(n)) % 11
+        dv = 0 if r < 2 else 11 - r
+        if d[n] != dv:
+            return False
+    return True
+
+
+def valid_cns(s: str) -> bool:
+    d = [int(c) for c in s if c.isdigit()]
+    if len(d) != 15:
+        return False
+    return sum(d[i] * (15 - i) for i in range(15)) % 11 == 0
+
+
+def valid_nhs(s: str) -> bool:
+    d = [int(c) for c in s if c.isdigit()]
+    if len(d) != 10:
+        return False
+    total = sum(d[i] * (10 - i) for i in range(9))
+    check = 11 - (total % 11)
+    check = 0 if check == 11 else check
+    return check != 10 and check == d[9]
+
+
+def valid_dni_es(s: str) -> bool:
+    m = re.fullmatch(r"(\d{8})([A-Za-z])", s.strip().upper())
+    if not m:
+        return False
+    return _DNI_LETTERS[int(m.group(1)) % 23] == m.group(2)
+
+
+def valid_nie_es(s: str) -> bool:
+    m = re.fullmatch(r"([XYZ])(\d{7})([A-Za-z])", s.strip().upper())
+    if not m:
+        return False
+    num = str("XYZ".index(m.group(1))) + m.group(2)
+    return _DNI_LETTERS[int(num) % 23] == m.group(3)
+
+
+def valid_curp(s: str) -> bool:
+    s = s.strip().upper()
+    if not re.fullmatch(r"[A-Z]{4}\d{6}[HM][A-Z]{5}[0-9A-Z]\d", s):
+        return False
+    alphabet = "0123456789ABCDEFGHIJKLMNÑOPQRSTUVWXYZ"
+    total = 0
+    for i, ch in enumerate(s[:17]):
+        total += alphabet.index(ch) * (18 - i)
+    return (10 - total % 10) % 10 == int(s[17])
+
+
+def valid_rfc(s: str) -> bool:  # format only (RFC's homoclave check is optional)
+    return bool(re.fullmatch(r"[A-ZÑ&]{3,4}\d{6}[A-Z0-9]{3}", s.strip().upper()))
+
+
+def valid_ssn_us(s: str) -> bool:  # range/format only (SSN has no checksum)
+    m = re.fullmatch(r"(\d{3})[-. ]?(\d{2})[-. ]?(\d{4})", s.strip())
+    if not m:
+        return False
+    a, g, ser = m.group(1), m.group(2), m.group(3)
+    return a not in ("000", "666") and not a.startswith("9") and g != "00" and ser != "0000"
+
+
+VALIDATORS = {
+    "CPF": (valid_cpf, "num", 11),
+    "CNPJ": (valid_cnpj, "num", 14),
+    "CNS": (valid_cns, "num", 15),
+    "NHS": (valid_nhs, "num", 10),
+    "SSN": (valid_ssn_us, "num", 9),
+    "DNI": (valid_dni_es, "alnum", 9),
+    "NIE": (valid_nie_es, "alnum", 9),
+    "CURP": (valid_curp, "alnum", 18),
+    "RFC": (valid_rfc, "alnum", 13),
+}
+
+_NUM_CAND = re.compile(r"(?<![\w.])\d[\d.\-/ ]{6,17}\d(?![\w])")
+_ALNUM_CAND = re.compile(r"(?<![\w])[A-Za-z0-9]{8,18}(?![\w])")
+
+
+def find_valid_ids(text: str, enabled: List[str]) -> List[Detection]:
+    """Return NATIONAL_ID detections for substrings that pass an enabled check."""
+    if not enabled:
+        return []
+    enabled = [e.upper() for e in enabled]
+    if "ALL" in enabled:
+        enabled = list(VALIDATORS)
+    num_types = [e for e in enabled if e in VALIDATORS and VALIDATORS[e][1] == "num"]
+    alnum_types = [e for e in enabled if e in VALIDATORS and VALIDATORS[e][1] == "alnum"]
+    out: List[Detection] = []
+
+    for m in _NUM_CAND.finditer(text):
+        digits = "".join(c for c in m.group(0) if c.isdigit())
+        for t in num_types:
+            fn, _, ln = VALIDATORS[t]
+            if len(digits) == ln and fn(digits):
+                out.append(Detection(m.start(), m.end(), m.group(0), NATIONAL_ID,
+                                     {"checksum"}, confidence=0.98, accepted=True,
+                                     meta={"id_type": t}))
+                break
+    for m in _ALNUM_CAND.finditer(text):
+        tok = m.group(0)
+        for t in alnum_types:
+            fn, _, _ = VALIDATORS[t]
+            if fn(tok):
+                out.append(Detection(m.start(), m.end(), tok, NATIONAL_ID,
+                                     {"checksum"}, confidence=0.98, accepted=True,
+                                     meta={"id_type": t}))
+                break
+    return out
+
+
+def validate(value: str, id_type: str) -> bool:
+    """Public helper: does ``value`` satisfy ``id_type``'s check?"""
+    entry = VALIDATORS.get(id_type.upper())
+    return bool(entry and entry[0](value))

deidkit/detect/context.py

@@ -0,0 +1,137 @@
+"""Context stage: honorific / role-anchored person names.
+
+Captures a name by the words *around* it, so it works even when the name is not
+in the gazetteer. Anchors are graded by how reliably they precede a person, to
+keep precision high:
+
+* **strong**  — honorifics (``Dr.``, ``Dra.``, ``Sr.``) and name/signature cues
+  (``nombre``, ``firma``, ``atentamente``). A following capitalized token is
+  taken directly.
+* **naming**  — relational/role words that almost always precede a person
+  (``madre``, ``acompañante``, ``paciente``, ``mother`` ...). Accepted when the
+  following run looks like a name (>=2 tokens or a known name token).
+* **verb**    — verbs that often precede findings, not names (``refiere``,
+  ``solicita``, ``valorado`` ...). Only fire when a known name token follows, so
+  "valorado Ecografía Abdominal" is NOT mistaken for a person.
+"""
+
+from __future__ import annotations
+
+from typing import List
+
+from ..textnorm import Token, is_titlecase
+from .gazetteer import CONNECTORS, Gazetteer
+from .types import Detection, PERSON
+
+# Strong cues: a following capitalized token is a name with high confidence.
+STRONG_EXTRA = {
+    "nombre", "nombres", "apellido", "apellidos", "firma", "firmado", "atentamente",
+    "cordialmente", "suscribe", "signed", "signature", "sincerely", "name", "surname",
+}
+
+# Naming anchors: relational / role NOUNS that reliably precede a person name.
+# Deliberately excludes participle verbs (valorado, atendido, remitido) — those
+# are usually followed by "por <name>" and stay verb-class to avoid grabbing a
+# capitalized procedure ("Valorado Ecografía Abdominal").
+NAMING_TRIGGERS = {
+    "madre", "padre", "hijo", "hija", "esposo", "esposa", "acompanante", "responsable",
+    "familiar", "cuidador", "cuidadora", "tutor", "tutora", "informante", "senor",
+    "senora", "senorita", "paciente", "usuario", "asegurado", "afiliado", "beneficiario",
+    "cotizante", "mother", "father", "son", "daughter", "spouse", "guardian",
+    "companion", "caregiver", "relative", "patient",
+}
+
+
+def _is_name_like(tok: Token, gaz: Gazetteer) -> bool:
+    """A token that can belong to an anchored name run.
+
+    Title-case is the usual signal, but after an explicit anchor (``Dr.``,
+    ``paciente:`` ...) clinical staff often type the name in lowercase — so a
+    *known* gazetteer given/surname token counts even when it is not title-cased.
+    Generic (un-anchored) gazetteer detection stays title-case-only elsewhere, so
+    this relaxation cannot broaden false positives across the whole document.
+    """
+    if gaz.is_boundary(tok.norm):
+        return False
+    return is_titlecase(tok.text) or gaz.is_name_token(tok.norm)
+
+
+def _grab_following_name(tokens: List[Token], start_idx: int, gaz: Gazetteer, max_tokens: int = 4):
+    """Collect up to ``max_tokens`` name tokens after ``start_idx`` (title-case, or
+    a known lowercase name token straight after an anchor)."""
+    run: List[Token] = []
+    j = start_idx
+    n = len(tokens)
+    while j < n and len(run) < max_tokens:
+        tj = tokens[j]
+        if _is_name_like(tj, gaz):
+            run.append(tj)
+            j += 1
+        elif (
+            tj.norm in CONNECTORS
+            and run
+            and j + 1 < n
+            and _is_name_like(tokens[j + 1], gaz)
+        ):
+            run.append(tj)
+            j += 1
+        else:
+            break
+    while run and run[-1].norm in CONNECTORS:
+        run = run[:-1]
+    return run
+
+
+def _classify(nrm: str, gaz: Gazetteer) -> str:
+    if gaz.is_honorific(nrm) or nrm in STRONG_EXTRA:
+        return "strong"
+    if nrm in NAMING_TRIGGERS:
+        return "naming"
+    if gaz.is_trigger(nrm):
+        return "verb"
+    return ""
+
+
+def find(text: str, tokens: List[Token], gaz: Gazetteer) -> List[Detection]:
+    dets: List[Detection] = []
+    for idx, tok in enumerate(tokens):
+        strength = _classify(tok.norm, gaz)
+        if not strength:
+            continue
+        run = _grab_following_name(tokens, idx + 1, gaz)
+        if not run:
+            continue
+        known = sum(1 for r in run if gaz.is_name_token(r.norm))
+        name_len = sum(1 for r in run if r.norm not in CONNECTORS)
+
+        if strength == "strong":
+            accept = True
+        elif strength == "naming":
+            accept = name_len >= 2 or known >= 1
+        else:  # verb: only when a known name token actually follows
+            accept = known >= 1
+        if not accept:
+            continue
+
+        gender = "u"
+        for r in run:
+            if gaz.is_given(r.norm):
+                gender = gaz.gender(r.norm)
+                break
+        dets.append(
+            Detection(
+                start=run[0].start,
+                end=run[-1].end,
+                text=text[run[0].start : run[-1].end],
+                entity_type=PERSON,
+                methods={"context"},
+                meta={
+                    "anchor": tok.norm,
+                    "strength": strength,
+                    "known": known,
+                    "run_len": name_len,
+                    "gender": gender,
+                },
+            )
+        )
+    return dets