deidkit 0.1.0__py3-none-any.whl

deidkit/__init__.py

@@ -0,0 +1,74 @@
+"""deidkit — schema-driven pseudonymization for clinical / tabular datasets.
+
+Quick start (library)::
+
+    import deidkit as dk
+
+    policy = dk.Policy(lang="es", mode="conservative", ignore=["blood_type"])
+    deid = dk.Deidentifier(
+        policy,
+        secret="keep-this-secret",
+        dict_index=dk.load_dictionary("diccionario_columnas_dataset.json"),
+        mapping_path="private_map.json",
+    )
+
+    clean = deid.run_table(df, table="tbl_notes")   # a pandas DataFrame in, out
+    deid.save_mapping()
+    deid.write_report("audit.xlsx")
+
+What it does, per the policy:
+
+* replaces person names (structured columns *and* names inside free text) with
+  realistic, gender/structure-preserving synthetic names,
+* shifts every date by a per-patient offset that preserves all intervals,
+* replaces identifiers (cédula, phone, email, MRN ...) with stable surrogates,
+* records a full before/after audit of every change.
+"""
+
+from .version import __version__
+from .config import (
+    FieldRule,
+    Policy,
+    AGE_BAND,
+    DATE_SHIFT,
+    DROP,
+    FREETEXT,
+    GENERALIZE_YEAR,
+    IDENTIFIER,
+    PASSTHROUGH,
+    REDACT,
+    SYNTHETIC_NAME,
+)
+from .engine import Deidentifier
+from .detect import Detector
+from .mapping import Mapping
+from .schema import load_dictionary, parse_spark_schema
+from .secret import generate_secret
+from .api import deidentify, deidentify_dataframe, deidentify_database, Result
+from . import io
+
+__all__ = [
+    "__version__",
+    "deidentify",
+    "deidentify_dataframe",
+    "deidentify_database",
+    "Result",
+    "Policy",
+    "FieldRule",
+    "Deidentifier",
+    "Detector",
+    "Mapping",
+    "io",
+    "load_dictionary",
+    "parse_spark_schema",
+    "generate_secret",
+    "PASSTHROUGH",
+    "DATE_SHIFT",
+    "SYNTHETIC_NAME",
+    "IDENTIFIER",
+    "FREETEXT",
+    "GENERALIZE_YEAR",
+    "AGE_BAND",
+    "REDACT",
+    "DROP",
+]

deidkit/__main__.py

@@ -0,0 +1,6 @@
+"""Enable ``python -m deidkit``."""
+
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

deidkit/api.py

@@ -0,0 +1,316 @@
+"""One-call convenience API.
+
+The whole tool in one function::
+
+    import deidkit
+
+    result = deidkit.deidentify("data/", out="deid/")   # files in, files out
+    result = deidkit.deidentify(df)                       # -> Result; result.table is the clean df
+    df2 = deidkit.deidentify_dataframe(df)               # DataFrame in, DataFrame out
+
+``deidentify`` returns a :class:`Result` (``.tables`` / ``.table`` / ``.summary``
+/ ``.report_path`` / ``.secret``); ``deidentify_dataframe`` unwraps to the single
+DataFrame. Both handle secrets, schema discovery, the mapping and the audit
+report for you, with safe defaults. Everything is still overridable.
+"""
+
+from __future__ import annotations
+
+import glob
+import os
+import warnings
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Union
+
+from . import io
+from .config import Policy
+from .engine import Deidentifier
+from .schema import load_dictionary
+
+try:
+    import pandas as pd
+except ImportError:  # pragma: no cover
+    pd = None
+
+
+@dataclass
+class Result:
+    """What :func:`deidentify` returns."""
+
+    tables: Dict[str, "pd.DataFrame"]
+    deid: Deidentifier
+    out: Optional[str]
+    report_path: Optional[str]
+    mapping_path: Optional[str]
+    secret: Optional[str]  # set only when auto-generated, so you can save it
+    private_dir: Optional[str] = None  # secret + mapping + audit (NEVER ship)
+
+    @property
+    def table(self):
+        """The single output DataFrame (convenience when one table went in)."""
+        if len(self.tables) == 1:
+            return next(iter(self.tables.values()))
+        raise ValueError(f"{len(self.tables)} tables — use .tables[name].")
+
+    @property
+    def summary(self) -> Dict[str, Any]:
+        return self.deid.summary()
+
+    def __repr__(self) -> str:  # pragma: no cover
+        s = self.deid.summary()
+        return (f"<deidkit.Result tables={list(self.tables)} "
+                f"cells_changed={s['cells_changed']} review={s['review_detections']}>")
+
+
+def _find_dictionary(input_path) -> Optional[str]:
+    """Look for a data-dictionary JSON next to / inside the input."""
+    if not isinstance(input_path, str):
+        return None
+    search_dir = input_path if os.path.isdir(input_path) else os.path.dirname(input_path) or "."
+    for path in sorted(glob.glob(os.path.join(search_dir, "*.json"))):
+        try:
+            idx = load_dictionary(path)
+            if idx:
+                return path
+        except Exception:
+            continue
+    return None
+
+
+_POLICY_KEYS = {
+    "entity_key", "date_max_days", "lang", "mode", "name_style", "names_synthetic",
+    "shift_dates_in_text", "only", "ignore", "rules", "spacy_model",
+    "extra_name_files", "extra_stoplist_files",
+    "known_name_columns", "known_id_columns", "extra_known_names",
+    "known_token_min_len", "id_checksums", "use_medical_vocab",
+}
+
+
+def deidentify(
+    data: Union[str, "pd.DataFrame", Dict[str, "pd.DataFrame"]],
+    out: Optional[str] = None,
+    *,
+    policy: Optional[Policy] = None,
+    schema: Optional[str] = None,
+    secret: Optional[str] = None,
+    secret_file: Optional[str] = None,
+    mapping: Optional[str] = None,
+    report: Optional[str] = None,
+    private_dir: Optional[str] = None,
+    fmt: Optional[str] = None,
+    table: str = "table",
+    quiet: bool = True,
+    **policy_kwargs,
+) -> Result:
+    """De-identify files, a DataFrame, or a dict of DataFrames — in one call.
+
+    Parameters
+    ----------
+    data:
+        A path / glob / directory of tables, a single DataFrame, or a
+        ``{name: DataFrame}`` dict.
+    out:
+        Output directory. If given, writes de-identified tables, the audit
+        workbook, and the private mapping there. If omitted, nothing is written
+        and you just use ``result.tables`` / ``result.table``.
+    schema:
+        Data-dictionary JSON. Auto-discovered from the input folder if omitted.
+    secret / secret_file:
+        The keying secret. If none is available and ``out`` is set, a fresh
+        secret is generated and saved to ``<out>/deidkit-secret.key``. If no
+        ``out`` either, an ephemeral secret is generated (returned in
+        ``result.secret`` so you can persist it).
+    **policy_kwargs:
+        Any :class:`~deidkit.Policy` field, e.g. ``lang="multi"``,
+        ``mode="balanced"``, ``ignore=["blood_type"]``, ``name_style="placeholder"``.
+    """
+    if policy is None:
+        bad = set(policy_kwargs) - _POLICY_KEYS
+        if bad:
+            raise TypeError(f"Unknown option(s): {sorted(bad)}. "
+                            f"Valid Policy fields: {sorted(_POLICY_KEYS)}")
+        policy = Policy(**policy_kwargs)
+
+    # --- resolve input into {name: DataFrame} -------------------------- #
+    if pd is not None and isinstance(data, pd.DataFrame):
+        tables = {table: data}
+    elif isinstance(data, dict):
+        tables = data
+    elif isinstance(data, str):
+        tables = io.read_tables(data)
+        if not tables:
+            raise FileNotFoundError(f"No tables found at {data!r}")
+    else:  # pragma: no cover
+        raise TypeError("data must be a path, a DataFrame, or a dict of DataFrames.")
+
+    # --- schema auto-discovery ----------------------------------------- #
+    if schema is None:
+        schema = _find_dictionary(data)
+    dict_index = load_dictionary(schema) if schema else {}
+
+    # --- private dir: the secret, mapping and raw-PII audit live here, NEVER
+    #     inside `out/` (which is the shippable de-identified data). Shipping
+    #     `out/` must never leak the re-identification key. -------------------- #
+    if out and private_dir is None:
+        private_dir = out.rstrip("/\\") + "-PRIVATE"
+    if private_dir:
+        os.makedirs(private_dir, exist_ok=True)
+        try:
+            os.chmod(private_dir, 0o700)
+        except OSError:
+            pass
+
+    # --- secret: reuse, or generate a persisted/ephemeral one ---------- #
+    generated_secret = None
+    have_secret = bool(secret or secret_file or os.environ.get("DEIDKIT_SECRET"))
+    if not have_secret:
+        from .secret import generate_secret
+
+        generated_secret = generate_secret()
+        secret = generated_secret
+        if private_dir:
+            key_path = os.path.join(private_dir, "deidkit-secret.key")
+            with open(key_path, "w", encoding="utf-8") as fh:
+                fh.write(generated_secret + "\n")
+            os.chmod(key_path, 0o600)
+
+    if mapping is None and private_dir:
+        mapping = os.path.join(private_dir, "mapping.private.json")
+
+    deid = Deidentifier(
+        policy, secret=secret, secret_file=secret_file,
+        mapping_path=mapping, dict_index=dict_index,
+    )
+
+    clean = deid.run_dataset(tables)
+
+    report_path = None
+    if out:
+        if (fmt or "csv").lower() in ("parquet", "pq"):
+            io._require_pyarrow("write")  # fail early, before writing anything
+        os.makedirs(out, exist_ok=True)
+        for name, df in clean.items():
+            ext = fmt or "csv"
+            io.write_table(df, os.path.join(out, f"{name}.{ext}"), fmt=ext)
+        deid.save_mapping()
+        # audit + review queue contain raw PII -> private dir, not the ship dir
+        report_path = report or os.path.join(private_dir or out, "deid_audit.PRIVATE.xlsx")
+        deid.write_report(report_path)
+        if private_dir:
+            deid.write_review(os.path.join(private_dir, "review.csv"))
+        if not quiet:
+            s = deid.summary()
+            print(f"[deidkit] {len(clean)} table(s), {s['cells_changed']} cells changed, "
+                  f"{s['review_detections']} to review -> {out}")
+
+    return Result(
+        tables=clean, deid=deid, out=out, report_path=report_path,
+        mapping_path=mapping, secret=generated_secret, private_dir=private_dir,
+    )
+
+
+def deidentify_dataframe(df, table: str = "table", **kwargs):
+    """De-identify a single DataFrame and return the clean DataFrame."""
+    return deidentify(df, table=table, **kwargs).table
+
+
+def deidentify_database(
+    source: str,
+    out_db: str,
+    *,
+    tables=None,
+    schema: Optional[str] = None,
+    out_schema: Optional[str] = None,
+    if_exists: str = "fail",
+    policy: Optional[Policy] = None,
+    secret: Optional[str] = None,
+    secret_file: Optional[str] = None,
+    mapping: Optional[str] = None,
+    private_dir: Optional[str] = None,
+    report: Optional[str] = None,
+    quiet: bool = True,
+    **policy_kwargs,
+) -> Result:
+    """Read a SQL source DB, de-identify it, and write to a SEPARATE output DB
+    with the same structure — never editing the source in place.
+
+    ``source`` / ``out_db`` are SQLAlchemy URLs (e.g. a Fabric SQL endpoint for
+    the source and any writable DB for the output). The secret, mapping and audit
+    go to ``private_dir`` (default ``./deidkit-db-PRIVATE``), never the output DB.
+    Requires ``pip install deidkit[sql]``.
+    """
+    sa = io._require_sqlalchemy()
+    if not out_db:
+        raise ValueError(
+            "an output database is REQUIRED — deidkit never edits the source in "
+            "place. Create an empty output DB and pass its URL as out_db/--out-db."
+        )
+    if str(source) == str(out_db):
+        raise ValueError("source and output database must differ — deidkit never edits in place.")
+    src_engine = sa.create_engine(source)
+
+    # The output DB is an environment YOU provision. Fail early with the process
+    # if it isn't reachable (SQLite auto-creates a local file; a server DB does not).
+    # create_engine itself imports the driver, so it is inside the guard too.
+    try:
+        out_engine = sa.create_engine(out_db)
+        with out_engine.connect() as conn:
+            conn.execute(sa.text("SELECT 1"))
+    except Exception as exc:  # noqa: BLE001
+        raise ValueError(
+            f"cannot reach the output database.\n  {exc}\n\n"
+            "The output database must be created by YOU first (deidkit never writes\n"
+            "back to the source). Process:\n"
+            "  1. Create an EMPTY database on your server:  CREATE DATABASE deid_db;\n"
+            "  2. Create a user/role with write access to it.\n"
+            "  3. Pass its SQLAlchemy URL as --out-db, e.g.\n"
+            "       postgresql://user:pw@host:5432/deid_db\n"
+            "       mssql+pyodbc://user:pw@host/deid_db?driver=ODBC+Driver+18+for+SQL+Server\n"
+            "deidkit then creates the tables (mirroring the source structure) inside it."
+        ) from exc
+
+    dataframes, columns_meta = io.read_sql_all(src_engine, tables, schema)
+    if not dataframes:
+        raise ValueError("no tables found in the source database.")
+
+    if policy is None:
+        bad = set(policy_kwargs) - _POLICY_KEYS
+        if bad:
+            raise TypeError(f"Unknown option(s): {sorted(bad)}.")
+        policy = Policy(**policy_kwargs)
+
+    dict_index = load_dictionary(schema) if (schema and os.path.exists(str(schema))) else {}
+
+    private_dir = private_dir or "deidkit-db-PRIVATE"
+    os.makedirs(private_dir, exist_ok=True)
+    try:
+        os.chmod(private_dir, 0o700)
+    except OSError:
+        pass
+
+    generated_secret = None
+    if not (secret or secret_file or os.environ.get("DEIDKIT_SECRET")):
+        from .secret import generate_secret
+
+        generated_secret = generate_secret()
+        secret = generated_secret
+        key_path = os.path.join(private_dir, "deidkit-secret.key")
+        with open(key_path, "w", encoding="utf-8") as fh:
+            fh.write(generated_secret + "\n")
+        os.chmod(key_path, 0o600)
+    if mapping is None:
+        mapping = os.path.join(private_dir, "mapping.private.json")
+
+    deid = Deidentifier(policy, secret=secret, secret_file=secret_file,
+                        mapping_path=mapping, dict_index=dict_index)
+    clean = deid.run_dataset(dataframes)
+    for name, df in clean.items():
+        io.write_sql_mirrored(out_engine, name, df, columns_meta[name], out_schema, if_exists)
+
+    deid.save_mapping()
+    report_path = report or os.path.join(private_dir, "deid_audit.PRIVATE.xlsx")
+    deid.write_report(report_path)
+    deid.write_review(os.path.join(private_dir, "review.csv"))
+
+    return Result(tables=clean, deid=deid, out=out_db, report_path=report_path,
+                  mapping_path=mapping, secret=generated_secret, private_dir=private_dir)