deidkit 0.1.1__tar.gz → 0.1.2__tar.gz

{deidkit-0.1.1/src/deidkit.egg-info → deidkit-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deidkit
-Version: 0.1.1
+Version: 0.1.2
 Summary: Schema-driven pseudonymization for clinical/tabular datasets: synthetic names, interval-preserving date shifting, multi-stage free-text PII detection, and before/after audit exports.
 Author: Meridian Data
 License: MIT

{deidkit-0.1.1 → deidkit-0.1.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deidkit"
-version = "0.1.1"
+version = "0.1.2"
 description = "Schema-driven pseudonymization for clinical/tabular datasets: synthetic names, interval-preserving date shifting, multi-stage free-text PII detection, and before/after audit exports."
 readme = "README.md"
 requires-python = ">=3.9"

{deidkit-0.1.1 → deidkit-0.1.2}/src/deidkit/api.py

@@ -62,6 +62,26 @@ class Result:
                 f"cells_changed={s['cells_changed']} review={s['review_detections']}>")
 
 
+def _safe_table_name(name) -> str:
+    """A table name becomes an output filename, so it must be a bare name — never
+    a path. Reject separators / ``..`` / absolute paths so a crafted table name
+    (e.g. a dict key like ``"../escaped"``) cannot write outside the ship folder."""
+    n = str(name)
+    if (
+        n in ("", ".", "..")
+        or "/" in n
+        or "\\" in n
+        or "\x00" in n
+        or os.path.isabs(n)
+        or os.path.basename(n) != n
+    ):
+        raise ValueError(
+            f"unsafe table name {name!r}: table names become output filenames and "
+            f"must not contain path separators, '..', or be absolute."
+        )
+    return n
+
+
 def _find_dictionary(input_path) -> Optional[str]:
     """Look for a data-dictionary JSON next to / inside the input."""
     if not isinstance(input_path, str):
@@ -191,7 +211,7 @@ def deidentify(
         os.makedirs(out, exist_ok=True)
         for name, df in clean.items():
             ext = fmt or "csv"
-            io.write_table(df, os.path.join(out, f"{name}.{ext}"), fmt=ext)
+            io.write_table(df, os.path.join(out, f"{_safe_table_name(name)}.{ext}"), fmt=ext)
         deid.save_mapping()
         # audit + review queue contain raw PII -> private dir, not the ship dir
         report_path = report or os.path.join(private_dir or out, "deid_audit.PRIVATE.xlsx")

{deidkit-0.1.1 → deidkit-0.1.2}/src/deidkit/detect/pipeline.py

@@ -101,7 +101,7 @@ class Detector:
         persons = [d for d in raw if d.entity_type == PERSON]
 
         accepted_idents = self._resolve_identifiers(idents)
-        person_dets = self._resolve_persons(persons, accepted_idents)
+        person_dets = self._resolve_persons(persons, accepted_idents, text)
 
         final = accepted_idents + person_dets
         final.sort(key=lambda d: (d.start, -(d.end - d.start)))
@@ -140,9 +140,9 @@ class Detector:
         return kept
 
     def _resolve_persons(
-        self, persons: List[Detection], idents: List[Detection]
+        self, persons: List[Detection], idents: List[Detection], text: str
     ) -> List[Detection]:
-        clusters = _merge_person_spans(persons)
+        clusters = _merge_person_spans(persons, text)
         out: List[Detection] = []
         for c in clusters:
             # suppress person spans overlapping an accepted identifier
@@ -170,8 +170,14 @@ class Detector:
 
 
 # --------------------------------------------------------------------------- #
-def _merge_person_spans(persons: List[Detection]) -> List[Detection]:
-    """Merge overlapping person detections into clusters (union span + meta)."""
+def _merge_person_spans(persons: List[Detection], text: str) -> List[Detection]:
+    """Merge overlapping person detections into clusters (union span + meta).
+
+    ``.text`` is recomputed from ``text`` for the *final* span of each cluster, so
+    a merged span (e.g. 0..10 built from 'Juan' + 'Juan Pérez') reports the whole
+    'Juan Pérez', not the first fragment. Downstream code keys the surrogate and
+    the audit "before" on ``.text``, so a stale fragment would break referential
+    consistency and the audit."""
     if not persons:
         return []
     persons = sorted(persons, key=lambda d: (d.start, d.end))
@@ -180,12 +186,13 @@ def _merge_person_spans(persons: List[Detection]) -> List[Detection]:
     for d in persons[1:]:
         if d.start < cur.end:  # overlap
             cur.end = max(cur.end, d.end)
-            cur.text += ""  # placeholder; text re-set below
             _merge_meta(cur, d)
         else:
             clusters.append(cur)
             cur = _clone(d)
     clusters.append(cur)
+    for c in clusters:  # align .text with the (possibly widened) span
+        c.text = text[c.start : c.end]
     return clusters
 
 

{deidkit-0.1.1 → deidkit-0.1.2}/src/deidkit/io.py

@@ -27,9 +27,14 @@ def _require_pyarrow(action: str) -> None:
 
 
 def _read_csv(path: str, sep: str, **kwargs) -> pd.DataFrame:
-    """Read a CSV/TSV, tolerating non-UTF-8 clinical exports (latin-1/cp1252)."""
+    """Read a CSV/TSV, tolerating non-UTF-8 clinical exports (cp1252 / latin-1).
+
+    cp1252 is tried before latin-1 because Windows exports use its 0x80-0x9F range
+    for smart quotes / dashes; latin-1 would silently mis-decode those (it never
+    raises), so ordering it last keeps it only as the can't-fail fallback.
+    """
     kwargs.setdefault("dtype", str)
-    for enc in ("utf-8-sig", "latin-1"):
+    for enc in ("utf-8-sig", "cp1252", "latin-1"):
         try:
             return pd.read_csv(path, sep=sep, encoding=enc, **kwargs)
         except UnicodeDecodeError:
@@ -180,5 +185,11 @@ def read_sql_table(connection_string: str, table: str, schema: Optional[str] = N
             "`pip install --force-reinstall deidkit`."
         ) from exc
     engine = create_engine(connection_string)
-    ident = f"{schema}.{table}" if schema else table
+    # Quote identifiers through the dialect so reserved words, dots, spaces, and
+    # quote characters in a table/schema name are escaped safely (never raw-
+    # interpolated) — robust and injection-safe for arbitrary names.
+    prep = engine.dialect.identifier_preparer
+    ident = prep.quote(table)
+    if schema:
+        ident = f"{prep.quote(schema)}.{ident}"
     return pd.read_sql(f"SELECT * FROM {ident}", engine)

{deidkit-0.1.1 → deidkit-0.1.2}/src/deidkit/version.py

@@ -1,3 +1,3 @@
 """Single source of truth for the package version."""
 
-__version__ = "0.1.1"
+__version__ = "0.1.2"

{deidkit-0.1.1 → deidkit-0.1.2/src/deidkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deidkit
-Version: 0.1.1
+Version: 0.1.2
 Summary: Schema-driven pseudonymization for clinical/tabular datasets: synthetic names, interval-preserving date shifting, multi-stage free-text PII detection, and before/after audit exports.
 Author: Meridian Data
 License: MIT

{deidkit-0.1.1 → deidkit-0.1.2}/src/deidkit.egg-info/SOURCES.txt

@@ -65,4 +65,5 @@ tests/test_medical_vocab.py
 tests/test_names.py
 tests/test_qa_fixes.py
 tests/test_qa_freetext_leaks.py
+tests/test_qa_hardening.py
 tests/test_schema_lang.py

deidkit-0.1.2/tests/test_qa_hardening.py

@@ -0,0 +1,89 @@
+"""Regression tests for the code-review hardening pass (2026-07-06).
+
+1. dict table names must not path-traverse out of the ship folder (api.py).
+2. read_sql_table must quote identifiers (reserved/special names) safely (io.py).
+3. merged person spans must report the full span text, not the first fragment
+   (pipeline.py) — surrogate + audit key on .text.
+4. CSV reader must decode cp1252 smart punctuation correctly (io.py).
+"""
+
+import os
+
+import pandas as pd
+import pytest
+import sqlalchemy as sa
+
+import deidkit as dk
+from deidkit.detect.pipeline import _merge_person_spans
+from deidkit.detect.types import Detection, PERSON
+
+
+# --- 1. path traversal ----------------------------------------------------- #
+@pytest.mark.parametrize("bad", ["../escaped", "a/b", "..", "/abs", "x\\y"])
+def test_dict_table_name_traversal_rejected(tmp_path, bad):
+    df = pd.DataFrame({"patient_id": ["P1"], "note_full": ["hi"]})
+    with pytest.raises(ValueError):
+        dk.deidentify({bad: df}, out=str(tmp_path / "out"), secret="s")
+    # nothing was written outside the (never-created) out dir
+    assert not (tmp_path / "escaped.csv").exists()
+
+
+def test_normal_table_name_still_writes(tmp_path):
+    df = pd.DataFrame({"patient_id": ["P1"], "note_full": ["hi"]})
+    dk.deidentify({"tbl_notes": df}, out=str(tmp_path / "out"), secret="s")
+    assert (tmp_path / "out" / "tbl_notes.csv").exists()
+
+
+# --- 2. SQL identifier quoting -------------------------------------------- #
+def test_read_sql_table_quotes_reserved_and_normal_names(tmp_path):
+    url = f"sqlite:///{tmp_path/'t.db'}"
+    eng = sa.create_engine(url)
+    with eng.begin() as c:
+        c.execute(sa.text("CREATE TABLE tbl_notes (patient_id TEXT, note TEXT)"))
+        c.execute(sa.text("INSERT INTO tbl_notes VALUES ('P1','hi')"))
+        c.execute(sa.text('CREATE TABLE "Order" (id TEXT)'))   # reserved keyword
+        c.execute(sa.text("INSERT INTO \"Order\" VALUES ('x')"))
+    assert len(dk.io.read_sql_table(url, "tbl_notes")) == 1
+    assert len(dk.io.read_sql_table(url, "Order")) == 1        # raw f-string would fail
+
+
+# --- 3. merged person span text ------------------------------------------- #
+def test_merged_span_text_matches_full_span():
+    src = "Juan Perez fue visto"
+    merged = _merge_person_spans(
+        [Detection(0, 4, "Juan", PERSON, {"gazetteer"}),
+         Detection(0, 10, "Juan Perez", PERSON, {"ner"})],
+        src,
+    )[0]
+    assert (merged.start, merged.end) == (0, 10)
+    assert merged.text == "Juan Perez"          # not the stale "Juan"
+
+
+def test_single_span_text_unchanged():
+    src = "hola Juan adios"
+    out = _merge_person_spans([Detection(5, 9, "Juan", PERSON, {"gazetteer"})], src)[0]
+    assert out.text == "Juan"
+
+
+def test_merged_span_surrogate_and_audit_use_full_name():
+    """End-to-end: when spans merge, the replaced span, the surrogate identity,
+    and the audit 'before' are all the full name — internally consistent."""
+    deid = dk.Deidentifier(dk.Policy(lang="es", mode="conservative"),
+                           secret="qa-hardening")
+    df = pd.DataFrame({"patient_id": ["P1"],
+                       "note_full": ["Paciente Juan Pérez García consulta."]})
+    out = deid.run_table(df, "t")["note_full"].iloc[0]
+    assert "Juan Pérez García" not in out                 # full name scrubbed
+    # every freetext PERSON audit row's "before" equals the text that was replaced
+    for ch in deid.changes:
+        if ch["column"] == "note_full" and ch["entity_type"] == "PERSON":
+            assert ch["before"] in "Paciente Juan Pérez García consulta."
+
+
+# --- 4. cp1252 decoding ---------------------------------------------------- #
+def test_cp1252_smart_quotes_decode(tmp_path):
+    p = tmp_path / "cp.csv"
+    # 0x93/0x94 are curly double quotes in cp1252; latin-1 would mangle them
+    p.write_bytes(b"patient_id,note\nP1,\x93hola\x94\n")
+    val = dk.io.read_table(str(p))["note"].iloc[0]
+    assert val == "“hola”"      # “hola”