idscrub 2.0.0__tar.gz → 2.0.1__tar.gz

{idscrub-2.0.0 → idscrub-2.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: idscrub
-Version: 2.0.0
+Version: 2.0.1
 Author: Department for Business and Trade
 Classifier: Development Status :: 3 - Alpha
 Requires-Python: >=3.12

{idscrub-2.0.0 → idscrub-2.0.1}/idscrub/scrub.py

@@ -634,7 +634,7 @@ class IDScrub:
         """
 
         nlp = self.get_spacy_model(model_name)
-        stripped_texts = [s.strip() if s.isspace() else s for s in texts]
+        stripped_texts = ["" if s.strip() == "" else s for s in texts]
         docs = nlp.pipe(stripped_texts, n_process=n_process, batch_size=batch_size)
 
         idents = []
@@ -825,9 +825,11 @@ class IDScrub:
 
         analyzer = AnalyzerEngine(nlp_engine=loaded_nlp_engine)
 
+        stripped_texts = ["" if s.strip() == "" else s for s in texts]
+
         idents = []
 
-        for text, text_id in zip(texts, text_ids):
+        for text, text_id in zip(stripped_texts, text_ids):
             results = analyzer.analyze(text=text, language="en", entities=entity_types)
             for res in results:
                 if res.entity_type not in entity_types:

{idscrub-2.0.0 → idscrub-2.0.1}/idscrub.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: idscrub
-Version: 2.0.0
+Version: 2.0.1
 Author: Department for Business and Trade
 Classifier: Development Status :: 3 - Alpha
 Requires-Python: >=3.12

{idscrub-2.0.0 → idscrub-2.0.1}/test/test_presidio.py

@@ -49,3 +49,11 @@ def test_presidio_get_data():
     )
 
     assert_frame_equal(df, expected_df)
+
+
+def test_presidio_empty():
+    scrub = IDScrub([" ", "  John Smith", ""])
+    scrubbed = scrub.scrub(pipeline=[{"method": "presidio_entities", "entity_types": ["PERSON"]}])
+
+    assert scrubbed == [" ", "  [PERSON]", ""]
+    assert_frame_equal(scrub.get_scrubbed_data(), pd.DataFrame({"text_id": 2, "person": [["John Smith"]]}))

{idscrub-2.0.0 → idscrub-2.0.1}/test/test_spacy.py

@@ -18,10 +18,10 @@ def test_spacy_error():
 
 
 def test_spacy_empty():
-    scrub = IDScrub([" ", "John Smith", ""])
+    scrub = IDScrub([" ", "  John Smith", ""])
     scrubbed = scrub.scrub(pipeline=[{"method": "spacy_entities"}])
 
-    assert scrubbed == [" ", "[PERSON]", ""]
+    assert scrubbed == [" ", "  [PERSON]", ""]
     assert_frame_equal(scrub.get_scrubbed_data(), pd.DataFrame({"text_id": 2, "person": [["John Smith"]]}))