deidkit 0.1.2__tar.gz → 0.1.3__tar.gz

{deidkit-0.1.2/src/deidkit.egg-info → deidkit-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deidkit
-Version: 0.1.2
+Version: 0.1.3
 Summary: Schema-driven pseudonymization for clinical/tabular datasets: synthetic names, interval-preserving date shifting, multi-stage free-text PII detection, and before/after audit exports.
 Author: Meridian Data
 License: MIT
@@ -420,6 +420,18 @@ extra_stoplist_files=["learned/learned_stoplist.txt"])`.
 Start conservative, work the review queue down, and turn on the NER stage only if
 you need more recall — the audit is how you prove the result.
 
+**Enabling NER** is a one-time model install (the model is ~500 MB and ships
+separately from the package, not bundled):
+
+```bash
+pip install "deidkit[ner]"                 # installs spaCy
+deidkit download-model es_core_news_lg      # fetches the model (once)
+deidkit run <input> deid_out --spacy-model es_core_news_lg
+```
+
+If the model isn't installed, deidkit still runs on the deterministic rules and
+prints a clear warning — it never fails, but free-text name recall is lower.
+
 *In code*, the same knobs are `Policy` fields:
 
 ```python

{deidkit-0.1.2 → deidkit-0.1.3}/README.md

@@ -376,6 +376,18 @@ extra_stoplist_files=["learned/learned_stoplist.txt"])`.
 Start conservative, work the review queue down, and turn on the NER stage only if
 you need more recall — the audit is how you prove the result.
 
+**Enabling NER** is a one-time model install (the model is ~500 MB and ships
+separately from the package, not bundled):
+
+```bash
+pip install "deidkit[ner]"                 # installs spaCy
+deidkit download-model es_core_news_lg      # fetches the model (once)
+deidkit run <input> deid_out --spacy-model es_core_news_lg
+```
+
+If the model isn't installed, deidkit still runs on the deterministic rules and
+prints a clear warning — it never fails, but free-text name recall is lower.
+
 *In code*, the same knobs are `Policy` fields:
 
 ```python

{deidkit-0.1.2 → deidkit-0.1.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deidkit"
-version = "0.1.2"
+version = "0.1.3"
 description = "Schema-driven pseudonymization for clinical/tabular datasets: synthetic names, interval-preserving date shifting, multi-stage free-text PII detection, and before/after audit exports."
 readme = "README.md"
 requires-python = ">=3.9"

{deidkit-0.1.2 → deidkit-0.1.3}/src/deidkit/cli.py

@@ -221,6 +221,29 @@ def cmd_run_db(args) -> int:
     return 0
 
 
+def cmd_download_model(args) -> int:
+    """Download the spaCy model the optional NER stage uses (one-time, ~500 MB)."""
+    model = args.model
+    try:
+        import spacy  # noqa: F401
+        from spacy.cli import download
+    except ImportError:
+        print('error: spaCy is not installed. First run:  pip install "deidkit[ner]"',
+              file=sys.stderr)
+        return 2
+    print(f"Downloading spaCy model '{model}' (~500 MB, one-time)…")
+    try:
+        download(model)
+    except SystemExit as exc:  # spacy.cli.download exits non-zero on failure
+        if exc.code:
+            print(f"error: could not download '{model}'. Check the model name and "
+                  f"your network.", file=sys.stderr)
+            return 2
+    print(f"Done. Enable NER with:  --spacy-model {model}  "
+          f"(or spacy_model='{model}' in code).")
+    return 0
+
+
 def cmd_run(args) -> int:
     from .api import deidentify
 
@@ -379,6 +402,13 @@ def build_parser() -> argparse.ArgumentParser:
     _add_policy_args(pd_)
     pd_.set_defaults(func=cmd_run_db)
 
+    pdm = sub.add_parser(
+        "download-model",
+        help="Download the spaCy NER model for the optional NER stage (needs deidkit[ner]).")
+    pdm.add_argument("model", nargs="?", default="es_core_news_lg",
+                     help="spaCy model name (default es_core_news_lg).")
+    pdm.set_defaults(func=cmd_download_model)
+
     px = sub.add_parser("scan", help="Show detections for a text (tuning/debug).")
     g = px.add_mutually_exclusive_group()
     g.add_argument("--text", help="Inline text to scan.")

{deidkit-0.1.2 → deidkit-0.1.3}/src/deidkit/detect/spacy_ner.py

@@ -32,6 +32,16 @@ class SpacyNER:
             self.available = True
         except Exception as exc:  # ImportError or OSError (model not found)
             self.error = str(exc)
+            import warnings
+
+            warnings.warn(
+                f"NER requested but spaCy model {model!r} is not available "
+                f"({type(exc).__name__}). The NER stage is OFF, so free-text name "
+                f"recall is reduced — names not in the gazetteer/anchored may be "
+                f"missed. Install it once with:  deidkit download-model {model}  "
+                f'(after: pip install "deidkit[ner]").',
+                stacklevel=3,
+            )
 
     def find(self, text: str) -> List[Detection]:
         if not self.available or not text:

{deidkit-0.1.2 → deidkit-0.1.3}/src/deidkit/version.py

@@ -1,3 +1,3 @@
 """Single source of truth for the package version."""
 
-__version__ = "0.1.2"
+__version__ = "0.1.3"

{deidkit-0.1.2 → deidkit-0.1.3/src/deidkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deidkit
-Version: 0.1.2
+Version: 0.1.3
 Summary: Schema-driven pseudonymization for clinical/tabular datasets: synthetic names, interval-preserving date shifting, multi-stage free-text PII detection, and before/after audit exports.
 Author: Meridian Data
 License: MIT
@@ -420,6 +420,18 @@ extra_stoplist_files=["learned/learned_stoplist.txt"])`.
 Start conservative, work the review queue down, and turn on the NER stage only if
 you need more recall — the audit is how you prove the result.
 
+**Enabling NER** is a one-time model install (the model is ~500 MB and ships
+separately from the package, not bundled):
+
+```bash
+pip install "deidkit[ner]"                 # installs spaCy
+deidkit download-model es_core_news_lg      # fetches the model (once)
+deidkit run <input> deid_out --spacy-model es_core_news_lg
+```
+
+If the model isn't installed, deidkit still runs on the deterministic rules and
+prints a clear warning — it never fails, but free-text name recall is lower.
+
 *In code*, the same knobs are `Policy` fields:
 
 ```python

{deidkit-0.1.2 → deidkit-0.1.3}/src/deidkit.egg-info/SOURCES.txt

@@ -63,6 +63,7 @@ tests/test_known_phi.py
 tests/test_learn.py
 tests/test_medical_vocab.py
 tests/test_names.py
+tests/test_ner_model_helpers.py
 tests/test_qa_fixes.py
 tests/test_qa_freetext_leaks.py
 tests/test_qa_hardening.py

deidkit-0.1.3/tests/test_ner_model_helpers.py

@@ -0,0 +1,30 @@
+"""The NER model is separate from the package (too big for PyPI); it must be a
+one-command install, and a missing model must warn loudly (not silently reduce
+recall). These tests cover the helper command and the warning — without ever
+downloading a model."""
+
+import warnings
+
+from deidkit.cli import build_parser, cmd_download_model
+from deidkit.detect.spacy_ner import SpacyNER
+
+
+def test_missing_model_warns_and_disables():
+    with warnings.catch_warnings(record=True) as caught:
+        warnings.simplefilter("always")
+        ner = SpacyNER("definitely_not_a_real_model_zzz")
+    assert ner.available is False
+    assert ner.find("Paciente Juan Perez consulta.") == []      # degrades, no crash
+    msgs = " ".join(str(w.message) for w in caught)
+    assert "download-model" in msgs and "NER" in msgs           # loud + actionable
+
+
+def test_cli_wires_download_model_with_default():
+    args = build_parser().parse_args(["download-model"])
+    assert args.func is cmd_download_model
+    assert args.model == "es_core_news_lg"
+
+
+def test_cli_download_model_accepts_explicit_name():
+    args = build_parser().parse_args(["download-model", "en_core_web_sm"])
+    assert args.model == "en_core_web_sm"