idscrub 2.0.0__py3-none-any.whl → 2.0.1__py3-none-any.whl

idscrub/scrub.py

@@ -634,7 +634,7 @@ class IDScrub:
         """
 
         nlp = self.get_spacy_model(model_name)
-        stripped_texts = [s.strip() if s.isspace() else s for s in texts]
+        stripped_texts = ["" if s.strip() == "" else s for s in texts]
         docs = nlp.pipe(stripped_texts, n_process=n_process, batch_size=batch_size)
 
         idents = []
@@ -825,9 +825,11 @@ class IDScrub:
 
         analyzer = AnalyzerEngine(nlp_engine=loaded_nlp_engine)
 
+        stripped_texts = ["" if s.strip() == "" else s for s in texts]
+
         idents = []
 
-        for text, text_id in zip(texts, text_ids):
+        for text, text_id in zip(stripped_texts, text_ids):
             results = analyzer.analyze(text=text, language="en", entities=entity_types)
             for res in results:
                 if res.entity_type not in entity_types:

{idscrub-2.0.0.dist-info → idscrub-2.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: idscrub
-Version: 2.0.0
+Version: 2.0.1
 Author: Department for Business and Trade
 Classifier: Development Status :: 3 - Alpha
 Requires-Python: >=3.12

{idscrub-2.0.0.dist-info → idscrub-2.0.1.dist-info}/RECORD

@@ -1,7 +1,7 @@
 idscrub/__init__.py,sha256=cRugJv27q1q--bl-VNLpfiScJb_ROlUxyLFhaF55S1w,38
 idscrub/locations.py,sha256=7fMNOcGMYe7sX8TrfhMW6oYGAlc1WVYVQKQbpxE3pqo,217
-idscrub/scrub.py,sha256=ow5gDejdchXxi3iN1vEqlCaJGGkaEnvFJ8nAau6w9XI,45308
-idscrub-2.0.0.dist-info/licenses/LICENSE,sha256=JJnuf10NSx7YXglte1oH_N9ZP3AcWR_Y8irvQb_wnsg,1090
+idscrub/scrub.py,sha256=38r5pbsXkl3s_XLlstRbUm4e59zrN_CUwY9WtMnlcC0,45386
+idscrub-2.0.1.dist-info/licenses/LICENSE,sha256=JJnuf10NSx7YXglte1oH_N9ZP3AcWR_Y8irvQb_wnsg,1090
 notebooks/basic_usage.ipynb,sha256=kfCgdN8DnjIUd9wZs93mpLKMT5NA3E8ty8a6mQUAZGo,40195
 test/conftest.py,sha256=CD_fYlo-qjkDgsW-ZRMZjW5Famqpt8fzN2HOcJ2MBDU,1450
 test/test_dataframe.py,sha256=MWHQVJ_lCKLR86lnGGz9Nlke-pPQwvim11Cs5INxmh8,6395
@@ -13,12 +13,12 @@ test/test_id.py,sha256=z2Za6sk-Km-r2FN6i03_q2IY5I21OkwfOtc4p9sRvJg,1108
 test/test_label.py,sha256=uiUF72nZFt5GWE7HWO2ura3XU4hSKfe6ZTOyc7oRyzg,1053
 test/test_overlap.py,sha256=kM6jePx94evk-VRkMzeH_UiuYPR7FFe63oBgLsvfvZw,2380
 test/test_phonenumbers.py,sha256=LqAUzSYdYoQQyO_mAHYQkkRCgh4uArX4oxxKVhMi548,661
-test/test_presidio.py,sha256=Y6kgPOZ_ie3shVThbNfJ2i7w1yifS5ukI4eAo1Lgm6k,1658
+test/test_presidio.py,sha256=qm5zrqjK4Qdg0Ha7Oo63s_BvMmZvC3lgixlHTAZV0G4,1989
 test/test_regex.py,sha256=Z_D8ttPYnRCr4gl13W9IEb7vS6QtWMReAc9Rufbtv50,6789
 test/test_scrub.py,sha256=aelzLpF2S_5F754VcZtWeEzQIcUZ0ex0m6A8mTAnWQg,1591
 test/test_scrub_text.py,sha256=WadP66U3jrSqXlbUrIrguDYSfhRKFJW73lR6eudC22o,605
-test/test_spacy.py,sha256=5wIPnHirs6fx1Msz9IDDF8dxyh3t1asWaFEvSXeiBvU,1910
-idscrub-2.0.0.dist-info/METADATA,sha256=ARod5-Azd--n-lWnZ3yHF4jt9MSjpPrcnMePzmpp8BM,8537
-idscrub-2.0.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-idscrub-2.0.0.dist-info/top_level.txt,sha256=D4EEodXGCjGiX35ObiBTmjjBAdouN-eCvH-LezGGtks,23
-idscrub-2.0.0.dist-info/RECORD,,
+test/test_spacy.py,sha256=sf3dtpdZibUVc3BG5v7gGaADusNDek_loCDLj-_mQTc,1914
+idscrub-2.0.1.dist-info/METADATA,sha256=WYIHhgM7Ln7B62i0-A-0MOEcMhrqanorgTHMf1jAms8,8537
+idscrub-2.0.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+idscrub-2.0.1.dist-info/top_level.txt,sha256=D4EEodXGCjGiX35ObiBTmjjBAdouN-eCvH-LezGGtks,23
+idscrub-2.0.1.dist-info/RECORD,,

test/test_presidio.py

@@ -49,3 +49,11 @@ def test_presidio_get_data():
     )
 
     assert_frame_equal(df, expected_df)
+
+
+def test_presidio_empty():
+    scrub = IDScrub([" ", "  John Smith", ""])
+    scrubbed = scrub.scrub(pipeline=[{"method": "presidio_entities", "entity_types": ["PERSON"]}])
+
+    assert scrubbed == [" ", "  [PERSON]", ""]
+    assert_frame_equal(scrub.get_scrubbed_data(), pd.DataFrame({"text_id": 2, "person": [["John Smith"]]}))

test/test_spacy.py

@@ -18,10 +18,10 @@ def test_spacy_error():
 
 
 def test_spacy_empty():
-    scrub = IDScrub([" ", "John Smith", ""])
+    scrub = IDScrub([" ", "  John Smith", ""])
     scrubbed = scrub.scrub(pipeline=[{"method": "spacy_entities"}])
 
-    assert scrubbed == [" ", "[PERSON]", ""]
+    assert scrubbed == [" ", "  [PERSON]", ""]
     assert_frame_equal(scrub.get_scrubbed_data(), pd.DataFrame({"text_id": 2, "person": [["John Smith"]]}))