deidkit 0.1.0__py3-none-any.whl

deidkit/cli.py

@@ -0,0 +1,399 @@
+"""Command-line interface.
+
+    deidkit init-secret --out secret.key
+    deidkit plan  --input data/ --schema dict.json
+    deidkit run   --input data/ --out out/ --schema dict.json \
+                  --secret-file secret.key --report audit.xlsx --ignore blood_type
+    deidkit scan  --text "Paciente Juan Pérez, CC 1.234.567.890"
+
+See ``deidkit <command> --help`` for all options.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from typing import List, Optional
+
+from . import io
+from .config import (
+    ALL_MODES,
+    FieldRule,
+    Policy,
+    SYNTHETIC_NAME,
+    normalize_only_ignore,
+)
+from .engine import Deidentifier
+from .schema import load_dictionary
+from .secret import ENV_VAR, generate_secret
+from .version import __version__
+
+
+def _split_csv(value: Optional[str]) -> Optional[List[str]]:
+    if not value:
+        return None
+    return [v.strip() for v in value.split(",") if v.strip()]
+
+
+def build_policy(args) -> Policy:
+    policy = Policy.from_yaml(args.policy) if getattr(args, "policy", None) else Policy()
+    if getattr(args, "lang", None):
+        policy.lang = args.lang
+    if getattr(args, "mode", None):
+        policy.mode = args.mode
+    if getattr(args, "name_style", None):
+        policy.name_style = args.name_style
+    if getattr(args, "entity_key", None):
+        policy.entity_key = args.entity_key
+    if getattr(args, "date_max_days", None) is not None:
+        policy.date_max_days = args.date_max_days
+    if getattr(args, "spacy_model", None):
+        policy.spacy_model = args.spacy_model
+    if getattr(args, "names_token", False):
+        policy.names_synthetic = False
+    if getattr(args, "no_date_text", False):
+        policy.shift_dates_in_text = False
+    if getattr(args, "no_medical_vocab", False):
+        policy.use_medical_vocab = False
+    frag = normalize_only_ignore(_split_csv(getattr(args, "only", None)),
+                                 _split_csv(getattr(args, "ignore", None)))
+    if "only" in frag:
+        policy.only = frag["only"]
+    if "ignore" in frag:
+        policy.ignore |= frag["ignore"]
+    # --name-column NAME can be repeated to force synthetic-name treatment
+    for col in getattr(args, "name_column", None) or []:
+        policy.rules.insert(0, FieldRule(column=col, strategy=SYNTHETIC_NAME))
+    policy.known_name_columns += getattr(args, "known_name_column", None) or []
+    policy.known_id_columns += getattr(args, "known_id_column", None) or []
+    policy.extra_known_names += getattr(args, "known_name", None) or []
+    policy.id_checksums += _split_csv(getattr(args, "id_checksums", None)) or []
+    policy.extra_name_files += getattr(args, "extra_name_file", None) or []
+    policy.extra_stoplist_files += getattr(args, "extra_stoplist_file", None) or []
+    learn_dir = getattr(args, "learn_dir", None)
+    if learn_dir:
+        from .learn import learned_files
+
+        name_files, stop_files = learned_files(learn_dir)
+        policy.extra_name_files += name_files
+        policy.extra_stoplist_files += stop_files
+    policy.__post_init__()
+    return policy
+
+
+# --------------------------------------------------------------------------- #
+def cmd_init_secret(args) -> int:
+    secret = generate_secret()
+    if args.out:
+        with open(args.out, "w", encoding="utf-8") as fh:
+            fh.write(secret + "\n")
+        os.chmod(args.out, 0o600)
+        print(f"Wrote a new secret to {args.out} (chmod 600).")
+        print("Keep it private and stored separately from the delivered data.")
+    else:
+        print(secret)
+    return 0
+
+
+def cmd_plan(args) -> int:
+    policy = build_policy(args)
+    dict_index = load_dictionary(args.schema) if args.schema else {}
+    deid = Deidentifier(policy, secret="plan-only-not-used", dict_index=dict_index)
+    input_path = args.input or args.input_flag
+    if not input_path:
+        print("error: give an input path, e.g. `deidkit plan data/`", file=sys.stderr)
+        return 2
+    tables = io.read_tables(input_path)
+    print(f"{'table':<16} {'column':<32} {'strategy':<16} decided_by")
+    print("-" * 88)
+    for name, df in tables.items():
+        plan = deid.plan_dataframe(df, name)
+        for col, item in plan.items():
+            flag = "" if item.strategy == "passthrough" else "  *"
+            print(f"{name:<16} {col:<32} {item.strategy:<16} {item.source}{flag}")
+    return 0
+
+
+def cmd_scan(args) -> int:
+    policy = build_policy(args)
+    deid = Deidentifier(policy, secret="scan-only")
+    if args.text:
+        text = args.text
+    elif args.file:
+        with open(args.file, encoding="utf-8") as fh:
+            text = fh.read()
+    else:
+        text = sys.stdin.read()
+    dets = deid.detector.detect(text)
+    print(f"NER: {deid.detector.ner_status}")
+    print(f"{'accepted':<9} {'type':<12} {'conf':<5} {'detectors':<22} text")
+    print("-" * 80)
+    for d in sorted(dets, key=lambda x: x.start):
+        mark = "REPLACE" if d.accepted else "review"
+        print(f"{mark:<9} {d.entity_type:<12} {d.confidence:<5.2f} "
+              f"{'+'.join(sorted(d.methods)):<22} {d.text!r}")
+    return 0
+
+
+def cmd_map(args) -> int:
+    """Surface the private value->surrogate mapping (the re-identification key)."""
+    from .mapping import Mapping
+
+    if not os.path.exists(args.mapping):
+        print(f"error: mapping file not found: {args.mapping}", file=sys.stderr)
+        return 2
+    try:
+        mp = Mapping.load(args.mapping)
+    except (ValueError, OSError) as exc:
+        print(f"error: could not read mapping {args.mapping!r}: {exc}", file=sys.stderr)
+        return 2
+    rows = []
+    for etype in mp.entity_types():
+        for original, surrogate in mp.items(etype):
+            rows.append((etype, original, surrogate))
+    if args.out:
+        import csv
+
+        with open(args.out, "w", newline="", encoding="utf-8") as fh:
+            w = csv.writer(fh)
+            w.writerow(["entity_type", "original", "surrogate"])
+            w.writerows(rows)
+        print(f"PRIVATE mapping ({len(rows)} entries) -> {args.out}")
+        print("This is the re-identification key. Keep it out of any deliverable.")
+    else:
+        print("# PRIVATE re-identification key — do not share with the data")
+        print(f"{'entity_type':<16}{'original':<32}surrogate")
+        for etype, original, surrogate in rows:
+            print(f"{etype:<16}{str(original):<32}{surrogate}")
+        print(f"\n{len(rows)} entries")
+    return 0
+
+
+def _default_out(input_path: str) -> str:
+    p = input_path.rstrip("/\\")
+    if os.path.isdir(p):
+        return p + "_deid"
+    if any(ch in p for ch in "*?["):
+        return "deid-output"
+    return os.path.splitext(p)[0] + "_deid"
+
+
+def cmd_learn(args) -> int:
+    """Fold a human-edited review.csv into the learned dictionaries."""
+    from .learn import learn_from_review
+
+    if not os.path.exists(args.review):
+        print(f"error: review file not found: {args.review}", file=sys.stderr)
+        return 2
+    n_names, n_stop = learn_from_review(args.review, args.learn_dir)
+    print(f"learned {n_names} new name(s) and {n_stop} new stopword(s) -> {args.learn_dir}/")
+    print("Re-run with `--learn-dir " + args.learn_dir + "` to apply them.")
+    return 0
+
+
+def cmd_run_db(args) -> int:
+    from .api import deidentify_database
+
+    try:
+        result = deidentify_database(
+            args.source, args.out_db,
+            tables=_split_csv(args.tables),
+            schema=args.schema,
+            out_schema=args.out_schema,
+            if_exists=args.if_exists,
+            policy=build_policy(args),
+            secret=args.secret,
+            secret_file=args.secret_file,
+            mapping=args.mapping,
+            private_dir=args.private_dir,
+        )
+    except (ValueError, TypeError, ImportError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+
+    summ = result.summary
+    for name, df in result.tables.items():
+        print(f"  {name:<20} {len(df):,} rows -> output DB")
+    print(f"\n  OUTPUT DB   -> {args.out_db}  (de-identified, same structure)")
+    print(f"  KEEP PRIVATE-> {result.private_dir}/  (secret + mapping + audit — NEVER share)")
+    print(f"  cells changed : {summ['cells_changed']:,} | to review: {summ['review_detections']:,}")
+    return 0
+
+
+def cmd_run(args) -> int:
+    from .api import deidentify
+
+    input_path = args.input or args.input_flag
+    if not input_path:
+        print("error: give an input path, e.g. `deidkit run data/`", file=sys.stderr)
+        return 2
+    out = args.out or args.out_flag or _default_out(input_path)
+
+    try:
+        result = deidentify(
+            input_path,
+            out=out,
+            policy=build_policy(args),
+            schema=args.schema,
+            secret=args.secret,
+            secret_file=args.secret_file,
+            mapping=args.mapping,
+            report=args.report,
+            fmt=args.format,
+            quiet=True,
+        )
+    except (FileNotFoundError, ValueError, TypeError, ImportError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+
+    summ = result.summary
+    for name, df in result.tables.items():
+        print(f"  {name:<16} {len(df):,} rows")
+    print(f"\n  SHIP THIS   -> {result.out}/   (de-identified data only)")
+    print(f"  KEEP PRIVATE-> {result.private_dir}/   (secret + mapping + audit — NEVER share)")
+    print(f"      audit   : {os.path.basename(result.report_path or '')}")
+    print(f"      mapping : {os.path.basename(result.mapping_path or '')} (re-identification key)")
+    print(f"      review  : review.csv (mark y/n, then `deidkit learn`)")
+    if result.secret:
+        print(f"      secret  : deidkit-secret.key (KEEP — needed to re-run consistently)")
+    print(f"\n  cells changed      : {summ['cells_changed']:,}")
+    print(f"  to review          : {summ['review_detections']:,}")
+    print(f"  detection          : {summ['ner_status']} (deterministic rules; no AI by default)")
+    for etype, cnt in sorted(summ["entity_counts"].items(), key=lambda x: -x[1]):
+        print(f"      {etype:<14} {cnt:,}")
+    return 0
+
+
+# --------------------------------------------------------------------------- #
+def _add_policy_args(p):
+    p.add_argument("--policy", help="YAML policy file (CLI flags override it).")
+    p.add_argument("--lang", choices=["es", "en", "multi"],
+                   help="Language(s) for names/detection ('multi' = ES+EN).")
+    p.add_argument("--mode", choices=sorted(ALL_MODES),
+                   help="Free-text detection aggressiveness (default conservative).")
+    p.add_argument("--name-style", choices=["tagged", "placeholder", "token", "realistic"],
+                   help="How fake names look (default tagged = obviously synthetic).")
+    p.add_argument("--only", help="Comma list: transform ONLY these columns.")
+    p.add_argument("--ignore", help="Comma list: never transform these columns.")
+    p.add_argument("--name-column", action="append",
+                   help="Force a column to synthetic-name treatment (repeatable).")
+    p.add_argument("--entity-key", help="Column that groups date shifts (default patient_id).")
+    p.add_argument("--date-max-days", type=int, help="Date shift window in days (default 365).")
+    p.add_argument("--names-token", action="store_true",
+                   help="Use [PERSON_x] tokens instead of realistic synthetic names.")
+    p.add_argument("--no-date-text", action="store_true",
+                   help="Do not shift date strings found inside free text.")
+    p.add_argument("--spacy-model",
+                   help="Enable spaCy NER stage with this model (e.g. es_core_news_lg).")
+    p.add_argument("--known-name-column", action="append",
+                   help="Column whose value is a known person name to force-scrub "
+                        "from that row's free text (repeatable).")
+    p.add_argument("--known-id-column", action="append",
+                   help="Column whose value is a known identifier to force-scrub "
+                        "from that row's free text (repeatable).")
+    p.add_argument("--known-name", action="append",
+                   help="A known name (e.g. clinic staff) to scrub from ALL free text (repeatable).")
+    p.add_argument("--id-checksums",
+                   help="Comma list of checksum-validated ID types to detect (bare, no cue): "
+                        "CPF,CNPJ,CNS (Brazil), DNI,NIE (Spain), NHS (UK), CURP,RFC (Mexico), SSN, or ALL.")
+    p.add_argument("--extra-name-file", action="append",
+                   help="Extra gazetteer file (one name per line) to merge (repeatable).")
+    p.add_argument("--extra-stoplist-file", action="append",
+                   help="Extra stoplist file (words never treated as names) (repeatable).")
+    p.add_argument("--learn-dir",
+                   help="Directory of learned dictionaries from `deidkit learn`; "
+                        "auto-loads learned_names.txt + learned_stoplist.txt if present.")
+    p.add_argument("--no-medical-vocab", action="store_true",
+                   help="Do not load the bundled medical safe-vocabulary (drug/terminology terms).")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="deidkit",
+        description="Schema-driven pseudonymization for clinical / tabular datasets.",
+    )
+    parser.add_argument("--version", action="version", version=f"deidkit {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    ps = sub.add_parser("init-secret", help="Generate a secret key.")
+    ps.add_argument("--out", help="Write the secret to this file (else print it).")
+    ps.set_defaults(func=cmd_init_secret)
+
+    pp = sub.add_parser("plan", help="Show the resolved field plan without transforming.")
+    pp.add_argument("input", nargs="?", help="File, glob, or directory of tables.")
+    pp.add_argument("--input", dest="input_flag", help=argparse.SUPPRESS)
+    pp.add_argument("--schema", help="Data-dictionary JSON.")
+    _add_policy_args(pp)
+    pp.set_defaults(func=cmd_plan)
+
+    pr = sub.add_parser(
+        "run", help="De-identify a dataset.",
+        description="De-identify a dataset. Simplest form: `deidkit run data/` "
+                    "(auto output dir, auto secret, auto audit report).",
+    )
+    pr.add_argument("input", nargs="?", help="File, glob, or directory of tables.")
+    pr.add_argument("out", nargs="?", help="Output directory (default: <input>_deid).")
+    pr.add_argument("--input", dest="input_flag", help=argparse.SUPPRESS)
+    pr.add_argument("--out", dest="out_flag", help=argparse.SUPPRESS)
+    pr.add_argument("--schema", help="Data-dictionary JSON (auto-detected if omitted).")
+    pr.add_argument("--secret", help="Secret string (else --secret-file, $%s, or auto)." % ENV_VAR)
+    pr.add_argument("--secret-file", help=f"File holding the secret (or set {ENV_VAR}).")
+    pr.add_argument("--mapping", help="Private mapping JSON (default <out>/mapping.private.json).")
+    pr.add_argument("--report", help="Audit workbook path (default <out>/deid_audit.xlsx).")
+    pr.add_argument("--format", choices=["csv", "parquet", "tsv", "json"],
+                    help="Output format (default csv).")
+    _add_policy_args(pr)
+    pr.set_defaults(func=cmd_run)
+
+    pl = sub.add_parser("learn", help="Fold a reviewed review.csv into learned dictionaries.")
+    pl.add_argument("--review", required=True, help="The human-edited review.csv (y/n column).")
+    pl.add_argument("--learn-dir", required=True, help="Directory to write/append learned dicts.")
+    pl.set_defaults(func=cmd_learn)
+
+    pm = sub.add_parser("map", help="Dump the private value->surrogate mapping.")
+    pm.add_argument("--mapping", required=True, help="Mapping JSON produced by `run`.")
+    pm.add_argument("--out", help="Write to this CSV (else print).")
+    pm.set_defaults(func=cmd_map)
+
+    pd_ = sub.add_parser(
+        "run-db", help="De-identify a SQL database into a SEPARATE output DB.",
+        description="Reads the source DB (read-only) and writes the de-identified result "
+                    "to a SEPARATE output DB that mirrors the source structure. The output "
+                    "database must be provisioned by YOU (an empty DB you create) and passed "
+                    "as --out-db; the run fails with instructions if it is missing or "
+                    "unreachable. deidkit never edits the source in place.")
+    pd_.add_argument("--source", required=True, help="Source DB SQLAlchemy URL (read-only).")
+    pd_.add_argument("--out-db", required=True,
+                     help="Output DB SQLAlchemy URL you created (must differ from source). "
+                          "REQUIRED — the run fails without it.")
+    pd_.add_argument("--tables", help="Comma list of tables (default: all).")
+    pd_.add_argument("--schema", help="Data-dictionary JSON (drives the field plan).")
+    pd_.add_argument("--out-schema", help="Schema/namespace in the output DB.")
+    pd_.add_argument("--if-exists", choices=["fail", "replace", "append"], default="fail",
+                     help="Behaviour if an output table already exists (default fail).")
+    pd_.add_argument("--secret", help="Secret string (else --secret-file, env, or auto).")
+    pd_.add_argument("--secret-file", help=f"File holding the secret (or set {ENV_VAR}).")
+    pd_.add_argument("--mapping", help="Private mapping JSON (default <private>/mapping.private.json).")
+    pd_.add_argument("--private-dir", help="Dir for secret+mapping+audit (default ./deidkit-db-PRIVATE).")
+    _add_policy_args(pd_)
+    pd_.set_defaults(func=cmd_run_db)
+
+    px = sub.add_parser("scan", help="Show detections for a text (tuning/debug).")
+    g = px.add_mutually_exclusive_group()
+    g.add_argument("--text", help="Inline text to scan.")
+    g.add_argument("--file", help="Read text from a file.")
+    _add_policy_args(px)
+    px.set_defaults(func=cmd_scan)
+
+    return parser
+
+
+def main(argv=None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

deidkit/config.py

@@ -0,0 +1,285 @@
+"""Policy configuration: how each field is transformed.
+
+A :class:`Policy` is the single object that describes what the tool should do.
+It combines:
+
+* global knobs (language, date-shift window, detection mode),
+* an allow / ignore list of columns,
+* explicit per-column :class:`FieldRule` overrides.
+
+Everything else (which strategy a column gets when there is no explicit rule)
+is derived automatically from the schema / data dictionary in :mod:`deidkit.schema`.
+Policies can be built in code, loaded from YAML, or seeded from a data-dictionary
+JSON file.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Sequence
+
+# --------------------------------------------------------------------------- #
+# Strategy vocabulary
+# --------------------------------------------------------------------------- #
+# Each value column in a table is handled by exactly one strategy.
+
+PASSTHROUGH = "passthrough"          # leave value untouched (safe default)
+DATE_SHIFT = "date_shift"            # shift by a per-entity offset; preserves intervals
+SYNTHETIC_NAME = "synthetic_name"    # replace a structured name value with a fake one
+IDENTIFIER = "identifier"            # replace an ID-like value with a stable surrogate
+FREETEXT = "freetext"                # run the multi-stage detector over free text
+GENERALIZE_YEAR = "generalize_year"  # reduce a date to its year only
+AGE_BAND = "age_band"                # bucket an age into a band (e.g. 10-year bands, 90+)
+REDACT = "redact"                    # blank the value out
+DROP = "drop"                        # drop the whole column from the output
+
+ALL_STRATEGIES = frozenset(
+    {
+        PASSTHROUGH,
+        DATE_SHIFT,
+        SYNTHETIC_NAME,
+        IDENTIFIER,
+        FREETEXT,
+        GENERALIZE_YEAR,
+        AGE_BAND,
+        REDACT,
+        DROP,
+    }
+)
+
+# Detection aggressiveness for the free-text stage.
+MODE_CONSERVATIVE = "conservative"  # only replace when clearly a name (default)
+MODE_BALANCED = "balanced"          # also accept strong single-signal name hits
+MODE_AGGRESSIVE = "aggressive"      # accept any plausible person span (higher recall)
+ALL_MODES = frozenset({MODE_CONSERVATIVE, MODE_BALANCED, MODE_AGGRESSIVE})
+
+# How replacement person names look. All are OBVIOUSLY synthetic except
+# "realistic" (which reads like a real name and should be used with care).
+NAME_TAGGED = "tagged"            # realistic name + visible marker: "Juan Gómez [SINTÉTICO]"
+NAME_PLACEHOLDER = "placeholder"  # role + stable code: "Persona 481502" / "Person 481502"
+NAME_TOKEN = "token"              # machine token: "[NOMBRE_9F5287]"
+NAME_REALISTIC = "realistic"      # realistic, UNMARKED fake name (looks real)
+ALL_NAME_STYLES = frozenset({NAME_TAGGED, NAME_PLACEHOLDER, NAME_TOKEN, NAME_REALISTIC})
+
+# Visible markers for the "tagged" style, by primary language.
+TAG_MARKERS = {"es": "[SINTÉTICO]", "en": "[SYNTHETIC]"}
+
+
+@dataclass
+class FieldRule:
+    """An explicit override for one or more columns.
+
+    ``column`` and ``table`` support ``fnmatch`` globs (``*`` etc.). ``table``
+    of ``None`` matches every table. The first matching rule wins, so put more
+    specific rules first.
+    """
+
+    column: str
+    strategy: str
+    table: Optional[str] = None
+    options: Dict[str, Any] = field(default_factory=dict)
+
+    def matches(self, table: Optional[str], column: str) -> bool:
+        if self.table is not None and table is not None:
+            if not fnmatch.fnmatch(table, self.table):
+                return False
+        return fnmatch.fnmatch(column, self.column)
+
+    def __post_init__(self) -> None:
+        if self.strategy not in ALL_STRATEGIES:
+            raise ValueError(
+                f"Unknown strategy {self.strategy!r}. "
+                f"Valid: {sorted(ALL_STRATEGIES)}"
+            )
+
+
+@dataclass
+class Policy:
+    """The full transformation policy.
+
+    Attributes
+    ----------
+    entity_key:
+        Column used to group date shifts and to seed per-patient offsets.
+        Every date column of a given patient is shifted by the *same* number of
+        days, so intervals (length of stay, time-to-result, age) are preserved.
+    date_max_days:
+        Date offsets are drawn deterministically from ``[-date_max_days,
+        +date_max_days]``. The magnitude does not affect interval fidelity; it
+        only controls how far the absolute calendar date moves.
+    lang:
+        Default language for name generation / detection (``"es"`` or ``"en"``).
+    mode:
+        Free-text detection aggressiveness (see ``MODE_*``).
+    name_style:
+        How replacement names look (see ``NAME_*``). Default ``"tagged"`` yields
+        OBVIOUSLY-synthetic names ("Juan Gómez [SINTÉTICO]"). ``"placeholder"``
+        -> "Persona 481502"; ``"token"`` -> "[NOMBRE_9F5287]"; ``"realistic"`` ->
+        an unmarked realistic fake (looks real — use with care).
+    names_synthetic:
+        Legacy switch. ``False`` forces ``name_style="token"``.
+    shift_dates_in_text:
+        Shift date-like strings found inside free text by the row's offset.
+    only:
+        If set, ONLY these columns are transformed; everything else passes
+        through. (Applied by exact column name.)
+    ignore:
+        These columns are never transformed. Takes precedence over everything.
+    rules:
+        Explicit per-column overrides evaluated before the schema auto-plan.
+    spacy_model:
+        If set (e.g. ``"es_core_news_lg"``) the optional statistical NER stage
+        is enabled as a corroborating detector.
+    """
+
+    entity_key: str = "patient_id"
+    date_max_days: int = 365
+    lang: str = "es"
+    mode: str = MODE_CONSERVATIVE
+    name_style: str = NAME_TAGGED
+    names_synthetic: bool = True
+    shift_dates_in_text: bool = True
+    only: Optional[set] = None
+    ignore: set = field(default_factory=set)
+    rules: List[FieldRule] = field(default_factory=list)
+    spacy_model: Optional[str] = None
+    # Advanced: extra gazetteer/stoplist files to merge on top of the bundled ones.
+    extra_name_files: List[str] = field(default_factory=list)
+    extra_stoplist_files: List[str] = field(default_factory=list)
+    # Known-PHI propagation: values in these per-row columns are KNOWN PII and are
+    # force-scrubbed from that row's free text (near-100% recall, no guessing).
+    known_name_columns: List[str] = field(default_factory=list)
+    known_id_columns: List[str] = field(default_factory=list)
+    # Enable checksum-validated detection of bare national IDs, per region, e.g.
+    # ["CPF","CNPJ","CNS"] (Brazil), ["DNI","NIE"] (Spain), ["NHS"] (UK),
+    # ["CURP","RFC"] (Mexico), or ["ALL"]. Off by default (see detect/checksums).
+    id_checksums: List[str] = field(default_factory=list)
+    # Load the bundled medical safe-vocabulary (drug/terminology tokens) so
+    # capitalized medical terms are never flagged as person names (Philter-style).
+    use_medical_vocab: bool = True
+    # Global registry of known names (e.g. clinic staff) scrubbed from all free text.
+    extra_known_names: List[str] = field(default_factory=list)
+    # Only force-scrub a LONE known name token if it is this long (guards against
+    # over-redacting short/common surname words like "Cruz"/"León").
+    known_token_min_len: int = 5
+
+    def __post_init__(self) -> None:
+        if self.mode not in ALL_MODES:
+            raise ValueError(f"mode must be one of {sorted(ALL_MODES)}")
+        if not self.names_synthetic:  # legacy switch -> tokens
+            self.name_style = NAME_TOKEN
+        if self.name_style not in ALL_NAME_STYLES:
+            raise ValueError(f"name_style must be one of {sorted(ALL_NAME_STYLES)}")
+        if self.only is not None:
+            self.only = set(self.only)
+        self.ignore = set(self.ignore)
+
+    @property
+    def primary_lang(self) -> str:
+        """First language of a (possibly multilingual) spec, for name rendering."""
+        lg = self.lang
+        if isinstance(lg, (list, tuple)):
+            return lg[0] if lg else "es"
+        if lg in ("multi", "es+en", "en+es", "both"):
+            return "es"
+        return lg
+
+    # --- rule resolution --------------------------------------------------- #
+    def explicit_rule(self, table: Optional[str], column: str) -> Optional[FieldRule]:
+        """Return the first explicit rule matching ``table``/``column``."""
+        for rule in self.rules:
+            if rule.matches(table, column):
+                return rule
+        return None
+
+    def is_ignored(self, column: str) -> bool:
+        if column in self.ignore:
+            return True
+        if self.only is not None and column not in self.only:
+            return True
+        return False
+
+    # --- serialization ----------------------------------------------------- #
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "Policy":
+        data = dict(data)
+        rules = [
+            FieldRule(
+                column=r["column"],
+                strategy=r["strategy"],
+                table=r.get("table"),
+                options=r.get("options", {}) or {},
+            )
+            for r in data.pop("rules", []) or []
+        ]
+        only = data.pop("only", None)
+        ignore = data.pop("ignore", []) or []
+        known = {
+            "entity_key",
+            "date_max_days",
+            "lang",
+            "mode",
+            "name_style",
+            "names_synthetic",
+            "shift_dates_in_text",
+            "spacy_model",
+            "extra_name_files",
+            "extra_stoplist_files",
+            "known_name_columns",
+            "known_id_columns",
+            "extra_known_names",
+            "known_token_min_len",
+            "id_checksums",
+            "use_medical_vocab",
+        }
+        kwargs = {k: v for k, v in data.items() if k in known}
+        return cls(
+            rules=rules,
+            only=set(only) if only else None,
+            ignore=set(ignore),
+            **kwargs,
+        )
+
+    @classmethod
+    def from_yaml(cls, path: str) -> "Policy":
+        import yaml
+
+        with open(path, "r", encoding="utf-8") as fh:
+            data = yaml.safe_load(fh) or {}
+        return cls.from_dict(data)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "entity_key": self.entity_key,
+            "date_max_days": self.date_max_days,
+            "lang": self.lang,
+            "mode": self.mode,
+            "name_style": self.name_style,
+            "names_synthetic": self.names_synthetic,
+            "shift_dates_in_text": self.shift_dates_in_text,
+            "spacy_model": self.spacy_model,
+            "only": sorted(self.only) if self.only else None,
+            "ignore": sorted(self.ignore),
+            "rules": [
+                {
+                    "column": r.column,
+                    "strategy": r.strategy,
+                    "table": r.table,
+                    "options": r.options,
+                }
+                for r in self.rules
+            ],
+        }
+
+
+def normalize_only_ignore(
+    only: Optional[Sequence[str]], ignore: Optional[Sequence[str]]
+) -> Dict[str, Any]:
+    """Helper for the CLI: turn comma-lists into a dict fragment for a Policy."""
+    frag: Dict[str, Any] = {}
+    if only:
+        frag["only"] = set(only)
+    if ignore:
+        frag["ignore"] = set(ignore)
+    return frag