deidkit 0.1.0__py3-none-any.whl

deidkit/engine.py

@@ -0,0 +1,644 @@
+"""The orchestrator: apply a :class:`~deidkit.config.Policy` to tabular data.
+
+One :class:`Deidentifier` instance carries the shared state that guarantees
+**referential integrity across the whole dataset**:
+
+* one :class:`~deidkit.mapping.Mapping` (same real value -> same surrogate in
+  every column and every table),
+* one :class:`~deidkit.dates.DateShifter` (same patient shifted by the same
+  offset in every table, so intervals line up across tables),
+* one :class:`~deidkit.generators.NameGenerator`.
+
+Process a single DataFrame with :meth:`run_table`, or a whole set of tables with
+:meth:`run_dataset`. Every change is recorded for the before/after audit
+(:mod:`deidkit.report`).
+"""
+
+from __future__ import annotations
+
+import math
+import re
+import unicodedata
+from collections import defaultdict
+from typing import Any, Dict, List, Optional
+
+import pandas as pd
+
+from . import config, schema
+from .config import (
+    AGE_BAND,
+    DATE_SHIFT,
+    DROP,
+    FREETEXT,
+    GENERALIZE_YEAR,
+    IDENTIFIER,
+    PASSTHROUGH,
+    Policy,
+    REDACT,
+    SYNTHETIC_NAME,
+)
+from .dates import DateShifter
+from .detect import Detector
+from .detect.types import DATE as DATE_ENTITY
+from .detect.types import NATIONAL_ID, PERSON, Detection
+from .generators import NameGenerator, SurrogateGenerator
+from .mapping import Mapping
+from .secret import resolve_secret
+from .textnorm import norm
+
+
+class Deidentifier:
+    def __init__(
+        self,
+        policy: Policy,
+        secret: Optional[str] = None,
+        secret_file: Optional[str] = None,
+        mapping_path: Optional[str] = None,
+        dict_index: Optional[Dict[str, Dict[str, Any]]] = None,
+        max_records: int = 200_000,
+        max_structured_samples: int = 2_000,
+    ):
+        self.policy = policy
+        self.secret_str = secret
+        try:
+            self.secret = resolve_secret(secret, secret_file)
+        except ValueError:
+            # Friendly for notebooks: generate an ephemeral secret rather than
+            # erroring. Persist ``deid.secret_str`` to reproduce/extend the map.
+            import warnings
+
+            from .secret import generate_secret
+
+            self.secret_str = generate_secret()
+            self.secret = self.secret_str.encode("utf-8")
+            warnings.warn(
+                "deidkit: no secret provided — generated an ephemeral one. "
+                "Save `deid.secret_str` (or pass secret=/secret_file=) to keep "
+                "the mapping reproducible across runs.",
+                stacklevel=2,
+            )
+        self.mapping_path = mapping_path
+        self.mapping = Mapping.load_or_new(mapping_path)
+        self.dict_index = dict_index or {}
+
+        self.dates = DateShifter(self.secret, policy.date_max_days)
+        self.names = NameGenerator(self.secret, policy.lang, policy.extra_name_files)
+        self.surrogates = SurrogateGenerator(self.secret)
+        self.detector = Detector(
+            policy.lang,
+            policy.mode,
+            policy.spacy_model,
+            policy.extra_name_files,
+            policy.extra_stoplist_files,
+            policy.id_checksums,
+            policy.use_medical_vocab,
+        )
+
+        # audit state
+        self.changes: List[Dict[str, Any]] = []       # detailed before/after rows
+        self.review: List[Dict[str, Any]] = []         # detections not auto-replaced
+        self.plans: Dict[str, Dict[str, schema.PlanItem]] = {}
+        self.stats: Dict[str, int] = defaultdict(int)
+        self._structured_counts: Dict[tuple, int] = defaultdict(int)
+        self.max_records = max_records
+        self.max_structured_samples = max_structured_samples
+        self.truncated = False
+
+    # ------------------------------------------------------------------ #
+    def plan_for(self, table: str, columns: List[str], dtypes: Dict[str, str]) -> Dict[str, schema.PlanItem]:
+        plan = schema.plan_table(self.policy, table, columns, dtypes, self.dict_index)
+        self.plans[table] = plan
+        return plan
+
+    def plan_dataframe(self, df: pd.DataFrame, table: str) -> Dict[str, schema.PlanItem]:
+        """Plan every column, then content-sniff undetermined object columns for
+        dates (catches ``encounter_start`` etc. that name heuristics miss)."""
+        dtypes = dict(zip(df.columns, (str(t) for t in df.dtypes)))
+        plan = schema.plan_table(self.policy, table, list(df.columns), dtypes, self.dict_index)
+        for col, item in plan.items():
+            if item.source == "default" and dtypes.get(col) in ("object", "string"):
+                col_data = df[col]
+                if isinstance(col_data, pd.DataFrame):  # duplicate column name
+                    continue
+                if not self.policy.is_ignored(col) and self._looks_like_dates(col_data):
+                    plan[col] = schema.PlanItem(col, DATE_SHIFT, "content:date")
+        self.plans[table] = plan
+        return plan
+
+    _YEAR_ONLY = r"\d{4}"
+    _TIME_ONLY = r"\d{1,2}:\d{2}(:\d{2})?(\s?[APap]\.?[Mm]\.?)?"
+    _DATE_SHAPE = (
+        r"\d{1,4}[-/.]\d{1,2}[-/.]\d{1,4}"                 # 2021-06-15 / 15/06/2021
+        r"|\d{1,2}[-/ ][A-Za-z]{3,}[-/ ]\d{2,4}"            # 05-ABR-1988
+        r"|[A-Za-z]{3,}\.?\s+\d{1,2},?\s+\d{4}"             # March 5, 2021
+        r"|\bde\s+[A-Za-zÁÉÍÓÚáéíóúñ]+\s+(?:de\s+)?\d{4}"   # 5 de marzo de 2021
+    )
+
+    @staticmethod
+    def _looks_like_dates(series: pd.Series) -> bool:
+        sample = series.dropna().astype(str).str.strip()
+        sample = sample[sample != ""].head(200)
+        if len(sample) < 1:
+            return False
+        # NOT a date column: year-only ints (birth_year) or time-only (dose_time)
+        if sample.str.fullmatch(Deidentifier._YEAR_ONLY).mean() > 0.8:
+            return False
+        if sample.str.fullmatch(Deidentifier._TIME_ONLY).mean() > 0.8:
+            return False
+        # must actually look like calendar dates (not just contain a separator)
+        if sample.str.contains(Deidentifier._DATE_SHAPE, regex=True).mean() < 0.7:
+            return False
+        parsed = DateShifter._parse_datetime(sample)
+        return bool(parsed.notna().mean() >= 0.6)
+
+    # ------------------------------------------------------------------ #
+    def run_table(self, df: pd.DataFrame, table: str = "table") -> pd.DataFrame:
+        """Return a de-identified copy of ``df`` and record all changes."""
+        dups = df.columns[df.columns.duplicated()].unique().tolist()
+        if dups:
+            raise ValueError(
+                f"table {table!r} has duplicate column names {dups}; "
+                f"rename them before de-identifying (they are ambiguous)."
+            )
+        plan = self.plan_dataframe(df, table)
+
+        # Work on a positionally-indexed copy so a duplicated DataFrame index
+        # (e.g. from pd.concat) can't make `.at[label]` return a Series and crash.
+        orig_index = df.index
+        out = df.copy()
+        out.index = pd.RangeIndex(len(out))
+        entity_key = self.policy.entity_key
+        if entity_key in out.columns:
+            entity_ids = out[entity_key]
+        else:
+            entity_ids = pd.Series(out.index, index=out.index)
+
+        # Known-PHI propagation: snapshot the ORIGINAL known-name/id columns
+        # (positionally aligned to `out`) BEFORE they get pseudonymized.
+        kn_cols = [c for c in self.policy.known_name_columns if c in df.columns]
+        ki_cols = [c for c in self.policy.known_id_columns if c in df.columns]
+        known_names_df = df[kn_cols].reset_index(drop=True) if kn_cols else None
+        known_ids_df = df[ki_cols].reset_index(drop=True) if ki_cols else None
+
+        drop_cols: List[str] = []
+        for col, item in plan.items():
+            strat = item.strategy
+            if strat in (PASSTHROUGH,):
+                continue
+            self.stats[f"columns:{strat}"] += 1
+            if strat == DROP:
+                drop_cols.append(col)
+                continue
+            if strat == DATE_SHIFT:
+                out[col] = self._apply_date_shift(out[col], entity_ids, table, col)
+            elif strat == SYNTHETIC_NAME:
+                out[col] = self._apply_name(out[col], table, col)
+            elif strat == IDENTIFIER:
+                out[col] = self._apply_identifier(out[col], table, col)
+            elif strat == FREETEXT:
+                out[col] = self._apply_freetext(
+                    out[col], entity_ids, table, col, known_names_df, known_ids_df
+                )
+            elif strat == REDACT:
+                out[col] = self._apply_redact(out[col], table, col)
+            elif strat == GENERALIZE_YEAR:
+                out[col] = self._apply_generalize_year(out[col], table, col)
+            elif strat == AGE_BAND:
+                out[col] = self._apply_age_band(out[col], item.options, table, col)
+
+        if drop_cols:
+            out = out.drop(columns=drop_cols)
+        out.index = orig_index  # restore the caller's original index
+        self.stats[f"rows:{table}"] += len(df)
+        return out
+
+    def run_dataset(self, tables: Dict[str, pd.DataFrame]) -> Dict[str, pd.DataFrame]:
+        return {name: self.run_table(df, name) for name, df in tables.items()}
+
+    # ------------------------------------------------------------------ #
+    # strategy implementations
+    # ------------------------------------------------------------------ #
+    def _record(self, table, column, row_id, strategy, entity_type, before, after,
+                methods="", confidence=""):
+        self.stats["cells_changed"] += 1
+        self.stats[f"entity:{entity_type}"] += 1
+        if len(self.changes) >= self.max_records:
+            self.truncated = True
+            return
+        self.changes.append(
+            {
+                "table": table,
+                "column": column,
+                "row_id": row_id,
+                "strategy": strategy,
+                "entity_type": entity_type,
+                "before": before,
+                "after": after,
+                "methods": methods,
+                "confidence": confidence,
+            }
+        )
+
+    def _log_structured(self, table, column) -> bool:
+        """Rate-limit structured before/after samples per column (report size)."""
+        key = (table, column)
+        self._structured_counts[key] += 1
+        return self._structured_counts[key] <= self.max_structured_samples
+
+    @staticmethod
+    def _isna(v) -> bool:
+        if v is None:
+            return True
+        try:
+            return bool(pd.isna(v))
+        except (TypeError, ValueError):
+            return False
+
+    # --- date shift --------------------------------------------------- #
+    def _apply_date_shift(self, series, entity_ids, table, col):
+        # Year-only (birth_year) and time-only (dose_time) columns must NOT be
+        # day-shifted — that fabricates/corrupts. If one reached here via a name
+        # heuristic, leave it untouched.
+        if not pd.api.types.is_datetime64_any_dtype(series):
+            s = series.dropna().astype(str).str.strip()
+            s = s[s != ""].head(200)
+            if len(s) and (
+                s.str.fullmatch(self._YEAR_ONLY).mean() > 0.8
+                or s.str.fullmatch(self._TIME_ONLY).mean() > 0.8
+            ):
+                return series
+
+        new = self.dates.shift_series(series, entity_ids)
+        out = []
+        for o, n, e in zip(series.tolist(), new.tolist(), entity_ids.tolist()):
+            if self._isna(o):
+                out.append(o)
+                continue
+            o_s = str(o)
+            if not self._isna(n) and o_s != str(n):
+                out.append(n)
+                self.stats["cells_changed"] += 1
+                self.stats["entity:DATE"] += 1
+                if self._log_structured(table, col) and len(self.changes) < self.max_records:
+                    self.changes.append({
+                        "table": table, "column": col, "row_id": e,
+                        "strategy": DATE_SHIFT, "entity_type": "DATE",
+                        "before": o_s, "after": str(n), "methods": "date-offset", "confidence": "",
+                    })
+            else:
+                # non-empty value in a DATE column we could not parse -> REDACT.
+                # Passing it through could leak a real date in an odd format.
+                out.append("[DATE]")
+                self.stats["cells_changed"] += 1
+                self.stats["entity:DATE_REDACTED"] += 1
+                if self._log_structured(table, col) and len(self.changes) < self.max_records:
+                    self.changes.append({
+                        "table": table, "column": col, "row_id": e,
+                        "strategy": DATE_SHIFT, "entity_type": "DATE",
+                        "before": o_s, "after": "[DATE]", "methods": "redact-unparsed", "confidence": "",
+                    })
+        return pd.Series(out, index=series.index)
+
+    # --- person surrogate (shared by structured names AND free text) --- #
+    def _person_surrogate(self, source: str, gender: str = "u") -> str:
+        """Return the pseudonym for a person, consistent across every surface.
+
+        One namespace (``PERSON``) is used for structured name columns and for
+        names found in free text, so the same real person gets the SAME synthetic
+        identity whether they appear in a name field or inside a note
+        (referential integrity). The visible rendering follows ``name_style``.
+        """
+        from .config import NAME_PLACEHOLDER, NAME_REALISTIC, NAME_TAGGED, NAME_TOKEN, TAG_MARKERS
+        from .secret import digest_int, short_token
+
+        lang = self.policy.primary_lang
+        style = self.policy.name_style
+        key = norm(source)
+
+        # placeholder / token are allocated THROUGH the injective mapping (attempt
+        # perturbation breaks any hash collision) so two different people can
+        # never share a surrogate.
+        if style == NAME_TOKEN:
+            return self.mapping.get_or_create(
+                "PERSON_TOKEN", key,
+                lambda nv, att: f"[NOMBRE_{short_token(self.secret, 'TOK|', nv, '|', str(att), length=6)}]",
+            )
+        if style == NAME_PLACEHOLDER:
+            role = {"es": "Persona", "en": "Person"}.get(lang, "Persona")
+            return self.mapping.get_or_create(
+                "PERSON_PLACEHOLDER", key,
+                lambda nv, att: f"{role} {digest_int(self.secret, 'CODE|', nv, '|', str(att)) % 9000000 + 1000000}",
+            )
+
+        # tagged / realistic: a stable, injective realistic base name identity.
+        base = self.mapping.get_or_create(
+            "PERSON", key, lambda nv, att: self.names.generate_base(nv, gender, att)
+        )
+        display = NameGenerator.case_like(source, base)
+        if style == NAME_TAGGED:
+            return f"{display} {TAG_MARKERS.get(lang, '[SINTÉTICO]')}"
+        return display  # NAME_REALISTIC
+
+    def _apply_name(self, series, table, col):
+        out = []
+        for idx, v in series.items():
+            if self._isna(v) or str(v).strip() == "":
+                out.append(v)
+                continue
+            src = str(v)
+            new_v = self._person_surrogate(src, "u")
+            out.append(new_v)
+            if new_v != src:
+                self._record(table, col, idx, SYNTHETIC_NAME, "PERSON", src, new_v)
+        return pd.Series(out, index=series.index)
+
+    # --- identifier column ------------------------------------------- #
+    @staticmethod
+    def _mostly_numeric(src: str) -> bool:
+        s = src.strip()
+        digits = sum(c.isdigit() for c in s)
+        return digits >= 4 and digits >= len(s) - 2
+
+    def _apply_identifier(self, series, table, col):
+        out = []
+        for idx, v in series.items():
+            if self._isna(v) or str(v).strip() == "":
+                out.append(v)
+                continue
+            src = str(v)
+            if self._mostly_numeric(src):
+                # unify with the free-text NATIONAL_ID namespace + digit key, so
+                # a cédula maps to the SAME surrogate in `cedula`, `documento`,
+                # dotted form, and inside a note (joins survive).
+                digits = "".join(c for c in src if c.isdigit())
+                new_v = self.mapping.get_or_create(
+                    "NATIONAL_ID", digits, lambda nv, att: self.surrogates.national_id(nv, att)
+                )
+                etype = "NATIONAL_ID"
+            else:
+                new_v = self.mapping.get_or_create(
+                    "ID_GENERIC", src, lambda nv, att: self.surrogates.generic_id(nv, col, att)
+                )
+                etype = "ID"
+            out.append(new_v)
+            if new_v != src:
+                self._record(table, col, idx, IDENTIFIER, etype, src, new_v)
+        return pd.Series(out, index=series.index)
+
+    # --- free text (the multi-stage detector + known-PHI) ------------- #
+    def _apply_freetext(self, series, entity_ids, table, col,
+                        known_names_df=None, known_ids_df=None):
+        extra_names = self.policy.extra_known_names
+        out = []
+        for idx, v in series.items():
+            if self._isna(v) or str(v).strip() == "":
+                out.append(v)
+                continue
+            known_names = list(extra_names)
+            if known_names_df is not None:
+                known_names += [str(x) for x in known_names_df.iloc[idx]
+                                if not self._isna(x) and str(x).strip()]
+            known_ids = []
+            if known_ids_df is not None:
+                known_ids += [str(x) for x in known_ids_df.iloc[idx]
+                              if not self._isna(x) and str(x).strip()]
+            out.append(self._scrub_text(
+                str(v), entity_ids.at[idx], table, col, idx, known_names, known_ids))
+        return pd.Series(out, index=series.index)
+
+    def _known_spans(self, text, known_names, known_ids) -> List[Detection]:
+        """Deterministic spans for values we already KNOW belong to this row
+        (from structured columns / a registry) — near-100% recall, no guessing."""
+        spans: List[Detection] = []
+        stop = self.detector.gaz.stoplist
+        for raw in known_names:
+            name = raw.strip()
+            if len(name) < 2:
+                continue
+            toks = name.split()
+            # full name (flexible whitespace, word boundaries, case-insensitive)
+            full = r"(?<!\w)" + r"\s+".join(re.escape(t) for t in toks) + r"(?!\w)"
+            for m in re.finditer(full, text, re.IGNORECASE):
+                spans.append(Detection(m.start(), m.end(), m.group(0), PERSON,
+                                       {"known-pii"}, confidence=1.0, accepted=True))
+            # lone tokens: only long, non-stopword ones (avoid over-redacting
+            # short/common surnames like "Cruz"/"León"). Each lone token maps to
+            # the CORRESPONDING token of this person's full surrogate, so partial
+            # mentions stay consistent and keep the right given/surname shape.
+            full_sur = self._person_surrogate(name)
+            surr_toks = full_sur.split()
+            token_level = self.policy.name_style in ("realistic", "tagged")
+            for i, tok in enumerate(toks):
+                if len(tok) < self.policy.known_token_min_len or norm(tok) in stop:
+                    continue
+                if token_level and i < len(toks):
+                    repl = surr_toks[i] if i < len(surr_toks) else (surr_toks[-1] if surr_toks else tok)
+                else:  # placeholder / token: a lone mention -> the whole identity
+                    repl = full_sur
+                for m in re.finditer(r"(?<!\w)" + re.escape(tok) + r"(?!\w)", text, re.IGNORECASE):
+                    spans.append(Detection(m.start(), m.end(), m.group(0), PERSON,
+                                           {"known-pii"}, confidence=1.0, accepted=True,
+                                           meta={"surrogate": repl}))
+        for raw in known_ids:
+            digits = "".join(c for c in raw if c.isdigit())
+            if len(digits) < 4:
+                continue
+            variants = {raw.strip(), digits}
+            grouped = _group_thousands(digits)  # 12345678 -> 12.345.678
+            if grouped:
+                variants.add(grouped)
+            for v in variants:
+                for m in re.finditer(r"(?<!\d)" + re.escape(v) + r"(?!\d)", text):
+                    spans.append(Detection(m.start(), m.end(), m.group(0), NATIONAL_ID,
+                                           {"known-pii"}, confidence=1.0, accepted=True))
+        return spans
+
+    def _scrub_text(self, text, entity_id, table, col, row_id, known_names=(), known_ids=()):
+        # Match the detector's NFC normalization so replacement offsets align and
+        # decomposed (NFD) accents cannot leave a leaked fragment.
+        text = unicodedata.normalize("NFC", text)
+        dets = self.detector.detect(text)
+        accepted, review = [], []
+        for d in dets:
+            if d.entity_type == DATE_ENTITY and not self.policy.shift_dates_in_text:
+                continue
+            (accepted if d.accepted else review).append(d)
+
+        for d in review:
+            self.stats["review_detections"] += 1
+            if len(self.review) < self.max_records:
+                self.review.append({
+                    "table": table, "column": col, "row_id": row_id,
+                    "entity_type": d.entity_type, "text": d.text,
+                    "confidence": round(d.confidence, 2),
+                    "methods": "+".join(sorted(d.methods)),
+                    "context": _snippet(text, d.start, d.end),
+                })
+
+        accepted += self._known_spans(text, known_names, known_ids)
+        if not accepted:
+            return text
+
+        # resolve overlaps: earliest start, longest span, and known-PHI wins ties
+        # (it is authoritative and its surrogate stays consistent with the full name).
+        accepted.sort(key=lambda d: (d.start, -(d.end - d.start),
+                                     0 if "known-pii" in d.methods else 1))
+        kept, occupied = [], -1
+        for d in accepted:
+            if d.start >= occupied:
+                kept.append(d)
+                occupied = d.end
+
+        new_text = text
+        for d in sorted(kept, key=lambda x: x.start, reverse=True):
+            surrogate = self._surrogate_for_text(d, entity_id)
+            if surrogate is None:
+                # A DATE we detected but could not shift (odd format) must never be
+                # left in the clear -> redact it. Other unresolved spans are left.
+                if d.entity_type == DATE_ENTITY:
+                    surrogate = "[DATE]"
+                else:
+                    continue
+            new_text = new_text[: d.start] + surrogate + new_text[d.end :]
+            self._record(
+                table, col, row_id, FREETEXT, d.entity_type, d.text, surrogate,
+                methods="+".join(sorted(d.methods)), confidence=round(d.confidence, 2),
+            )
+        return new_text
+
+    def _surrogate_for_text(self, d, entity_id):
+        et = d.entity_type
+        surr = self.surrogates
+        if d.meta.get("surrogate") is not None:  # precomputed (known-PHI lone token)
+            return d.meta["surrogate"]
+        if et == "PERSON":
+            return self._person_surrogate(d.text, d.meta.get("gender", "u"))
+        if et == "EMAIL":
+            return self.mapping.get_or_create("EMAIL", d.text.lower(), surr.email)
+        if et == "PHONE":
+            return self.mapping.get_or_create("PHONE", _digits(d.text), surr.phone)
+        if et == "NATIONAL_ID":
+            return self.mapping.get_or_create("NATIONAL_ID", _digits(d.text), surr.national_id)
+        if et == "MRN":
+            return self.mapping.get_or_create("MRN", _digits(d.text), surr.mrn)
+        if et == "URL":
+            return self.mapping.get_or_create("URL", d.text, surr.url)
+        if et == "IP":
+            return self.mapping.get_or_create("IP", d.text, surr.ip)
+        if et == "DATE":
+            return self.dates.shift_fragment(d.text, entity_id)  # None if unparseable
+        return None
+
+    # --- simple transforms ------------------------------------------- #
+    def _apply_redact(self, series, table, col):
+        out = []
+        for idx, v in series.items():
+            if self._isna(v):
+                out.append(v)
+                continue
+            out.append("[REDACTED]")
+            if self._log_structured(table, col):
+                self._record(table, col, idx, REDACT, "REDACT", str(v), "[REDACTED]")
+        return pd.Series(out, index=series.index)
+
+    def _apply_generalize_year(self, series, table, col):
+        s = pd.to_datetime(series, errors="coerce")
+        out = []
+        for idx, (orig, dt) in enumerate(zip(series, s)):
+            if pd.isna(dt):
+                out.append(orig)
+                continue
+            year = str(int(dt.year))  # string: mixed int/str breaks Parquet writes
+            out.append(year)
+            if self._log_structured(table, col):
+                self._record(table, col, series.index[idx], GENERALIZE_YEAR, "DATE", str(orig), year)
+        return pd.Series(out, index=series.index)
+
+    def _apply_age_band(self, series, options, table, col):
+        cap = int(options.get("cap", 90))
+        width = int(options.get("width", 10))
+        out = []
+        for idx, v in series.items():
+            if self._isna(v):
+                out.append(v)
+                continue
+            try:
+                fv = float(v)
+                if not math.isfinite(fv):  # 'inf'/'nan'/'1e400' -> not an age
+                    out.append(v)
+                    continue
+                age = int(fv)
+            except (TypeError, ValueError, OverflowError):
+                out.append(v)
+                continue
+            if age >= cap:
+                band = f"{cap}+"
+            else:
+                lo = (age // width) * width
+                band = f"{lo}-{lo + width - 1}"
+            out.append(band)
+            if self._log_structured(table, col):
+                self._record(table, col, idx, AGE_BAND, "AGE", str(v), band)
+        return pd.Series(out, index=series.index)
+
+    # ------------------------------------------------------------------ #
+    def save_mapping(self, path: Optional[str] = None) -> None:
+        target = path or self.mapping_path
+        if target:
+            self.mapping.save(target)
+
+    def summary(self) -> Dict[str, Any]:
+        return {
+            "cells_changed": self.stats.get("cells_changed", 0),
+            "review_detections": self.stats.get("review_detections", 0),
+            "mapping_entries": self.mapping.size(),
+            "mapping_counts": self.mapping.counts(),
+            "entity_counts": {
+                k.split(":", 1)[1]: v for k, v in self.stats.items() if k.startswith("entity:")
+            },
+            "detailed_records_truncated": self.truncated,
+            "ner_status": self.detector.ner_status,
+        }
+
+    def write_report(self, path: str) -> None:
+        from . import report
+
+        report.write_report(path, self)
+
+    def write_review(self, path: str) -> int:
+        """Write the review queue as an editable CSV. A human sets the first
+        column to ``y`` (real PII -> learn as a name) or ``n`` (false positive ->
+        learn as a stopword); ``deidkit learn`` then folds those decisions back
+        in. Returns the number of rows written."""
+        import csv
+
+        with open(path, "w", newline="", encoding="utf-8") as fh:
+            w = csv.writer(fh)
+            w.writerow(["decision(y/n)", "entity_type", "candidate", "context",
+                        "table", "column", "row_id", "confidence", "detectors"])
+            for r in self.review:
+                w.writerow(["", r["entity_type"], r["text"], r["context"], r["table"],
+                            r["column"], r["row_id"], r["confidence"], r["methods"]])
+        return len(self.review)
+
+
+def _digits(s: str) -> str:
+    return "".join(c for c in str(s) if c.isdigit()) or str(s)
+
+
+def _group_thousands(digits: str) -> Optional[str]:
+    """12345678 -> '12.345.678' (the dotted cédula form), else None."""
+    if len(digits) <= 3:
+        return None
+    out = []
+    for i, ch in enumerate(reversed(digits)):
+        if i and i % 3 == 0:
+            out.append(".")
+        out.append(ch)
+    return "".join(reversed(out))
+
+
+def _snippet(text: str, start: int, end: int, pad: int = 25) -> str:
+    a = max(0, start - pad)
+    b = min(len(text), end + pad)
+    return ("…" if a > 0 else "") + text[a:b].replace("\n", " ") + ("…" if b < len(text) else "")

deidkit/generators/__init__.py

@@ -0,0 +1,4 @@
+"""Surrogate value generators (deterministic, keyed by the secret)."""
+
+from .names import NameGenerator  # noqa: F401
+from .surrogates import SurrogateGenerator  # noqa: F401