idscrub 1.1.2__py3-none-any.whl → 2.0.0__py3-none-any.whl

test/test_scrub.py

@@ -3,9 +3,10 @@ from idscrub import IDScrub
 from pandas.testing import assert_frame_equal
 
 
-# Note: These tests will fail if the kernel has not been restarted since the SpaCy model was downloaded.
 def test_scrub(scrub_object):
-    scrubbed = scrub_object.scrub(scrub_methods=["spacy_entities", "uk_phone_numbers", "uk_postcodes"])
+    scrubbed = scrub_object.scrub(
+        pipeline=[{"method": "spacy_entities"}, {"method": "uk_phone_numbers"}, {"method": "uk_postcodes"}]
+    )
     assert scrubbed == [
         "Our names are [PERSON], [PERSON], and [PERSON].",
         "My number is [PHONENO] and I live at [POSTCODE].",
@@ -15,7 +16,7 @@ def test_scrub(scrub_object):
 def test_scrub_text_id():
     scrub = IDScrub(["Our names are Hamish McDonald, L. Salah, and Elena Suárez."] * 10)
 
-    scrub.scrub(scrub_methods=["spacy_entities"])
+    scrub.scrub(pipeline=[{"method": "spacy_entities"}])
 
     df = scrub.get_scrubbed_data()
 
@@ -24,7 +25,7 @@ def test_scrub_text_id():
 
 
 def test_scrub_get_scrubbed_data(scrub_object):
-    scrub_object.scrub(scrub_methods=["uk_postcodes"])
+    scrub_object.scrub(pipeline=[{"method": "uk_postcodes"}])
     df = scrub_object.get_scrubbed_data()
 
     expected_df = pd.DataFrame(
@@ -37,12 +38,21 @@ def test_scrub_get_scrubbed_data(scrub_object):
     assert_frame_equal(df, expected_df)
 
 
-def test_scrub_order(scrub_object):
-    scrub_object.scrub(scrub_methods=["uk_postcodes", "uk_phone_numbers", "spacy_entities"])
+def test_scrub_get_all_identified_data(scrub_object):
+    scrub_object.scrub(pipeline=[{"method": "uk_postcodes"}])
+    df = scrub_object.get_all_identified_data()
 
-    assert scrub_object.get_scrubbed_data().columns.to_list() == [
-        "text_id",
-        "uk_postcode",
-        "uk_phone_number",
-        "person",
-    ]
+    expected_df = pd.DataFrame(
+        {
+            "text_id": {0: 2},
+            "text": {0: "AA11 1AA"},
+            "start": {0: 41},
+            "end": {0: 49},
+            "label": {0: "uk_postcode"},
+            "replacement": {0: "[POSTCODE]"},
+            "priority": {0: 0.5},
+            "source": {0: "regex"},
+        }
+    )
+
+    assert_frame_equal(df, expected_df)

test/test_scrub_text.py

@@ -0,0 +1,22 @@
+from idscrub import IDScrub
+
+
+def test_scrub_text(scrub_object):
+    scrub_object.scrub(pipeline=[{"method": "uk_postcodes"}])
+
+    assert scrub_object.idents == [
+        IDScrub.IDEnt(
+            text_id=2,
+            text="AA11 1AA",
+            start=41,
+            end=49,
+            label="uk_postcode",
+            replacement="[POSTCODE]",
+            priority=0.5,
+            source="regex",
+        )
+    ]
+    assert scrub_object.scrub_text() == [
+        "Our names are Hamish McDonald, L. Salah, and Elena Suárez.",
+        "My number is +441111111111 and I live at [POSTCODE].",
+    ]

test/test_spacy.py

@@ -4,10 +4,9 @@ from idscrub import IDScrub
 from pandas.testing import assert_frame_equal
 
 
-# Note: This test will fail if the kernel has not been restarted since the SpaCy model was downloaded.
 def test_spacy():
     scrub = IDScrub(texts=["Our names are Hamish McDonald, L. Salah, and Elena Suárez."])
-    scrubbed = scrub.spacy_entities(entities=["PERSON"], model_name="en_core_web_trf")
+    scrubbed = scrub.scrub(pipeline=[{"method": "spacy_entities"}])
     assert scrubbed == ["Our names are [PERSON], [PERSON], and [PERSON]."]
 
 
@@ -15,12 +14,12 @@ def test_spacy_error():
     scrub = IDScrub(texts=["Our names are Hamish McDonald, L. Salah, and Elena Suárez."])
 
     with pytest.raises(ValueError):
-        scrub.spacy_entities(model_name="not_a_model")
+        scrub.scrub(pipeline=[{"method": "spacy_entities", "model_name": "not_a_model"}])
 
 
 def test_spacy_empty():
     scrub = IDScrub([" ", "John Smith", ""])
-    scrubbed = scrub.spacy_entities()
+    scrubbed = scrub.scrub(pipeline=[{"method": "spacy_entities"}])
 
     assert scrubbed == [" ", "[PERSON]", ""]
     assert_frame_equal(scrub.get_scrubbed_data(), pd.DataFrame({"text_id": 2, "person": [["John Smith"]]}))
@@ -28,17 +27,22 @@ def test_spacy_empty():
 
 def test_spacy_map():
     scrub = IDScrub(["Our names are Hamish McDonald, L. Salah, and Elena Suárez.", "My company code is NASA."])
-    scrubbed_texts = scrub.spacy_entities(
-        entities=["PERSON", "ORG"], replacement_map={"PERSON": "[PHELLO]", "ORG": "[SPACE]"}
+    scrubbed = scrub.scrub(
+        pipeline=[
+            {
+                "method": "spacy_entities",
+                "entity_types": ["PERSON", "ORG"],
+                "replacement_map": {"PERSON": "[PHELLO]", "ORG": "[SPACE]"},
+            }
+        ]
     )
+    assert scrubbed == ["Our names are [PHELLO], [PHELLO], and [PHELLO].", "My company code is [SPACE]."]
 
-    assert scrubbed_texts == ["Our names are [PHELLO], [PHELLO], and [PHELLO].", "My company code is [SPACE]."]
 
-
-def test_spacy_get_data():
+def test_presidio_get_data():
     scrub = IDScrub(["Our names are Hamish McDonald, L. Salah, and Elena Suárez.", "My company code is NASA."])
 
-    scrub.spacy_entities(entities=["PERSON", "ORG"])
+    scrub.scrub(pipeline=[{"method": "spacy_entities"}])
 
     df = scrub.get_scrubbed_data()
 

idscrub-1.1.2.dist-info/RECORD

@@ -1,22 +0,0 @@
-idscrub/__init__.py,sha256=cRugJv27q1q--bl-VNLpfiScJb_ROlUxyLFhaF55S1w,38
-idscrub/locations.py,sha256=7fMNOcGMYe7sX8TrfhMW6oYGAlc1WVYVQKQbpxE3pqo,217
-idscrub/scrub.py,sha256=PPTKWW-RQxZ5NixRow8nrnX9KjfyZa3tPAP9Jgwnn_M,36631
-idscrub-1.1.2.dist-info/licenses/LICENSE,sha256=JJnuf10NSx7YXglte1oH_N9ZP3AcWR_Y8irvQb_wnsg,1090
-notebooks/basic_usage.ipynb,sha256=V62Bz88a9Zo3LO_VxXF4sLw8-MP51ZdVRRNS-zjtNqw,42664
-test/conftest.py,sha256=y-pwGXpdg7bbFc36HtE3wQtZkeI0JM77fcMYjej5veY,557
-test/test_all.py,sha256=ifuXAI0Hq3ETNXzdITjNGCnuFyozhN5TpJC2hOtA2bM,1103
-test/test_chain.py,sha256=YbJeA11EBjDNcq5ZZjG4lIIyngrRQZknNsX3Oo0jPMc,1810
-test/test_dataframe.py,sha256=1LhtkQQpXblQ18ppI1s1nNyse0YCwGHbhtrKGkdppBw,6413
-test/test_huggingface.py,sha256=RTkp8Xsy4w9WoXq2IQ2YOJof41snbOQkM7CVtiVVD0U,839
-test/test_id.py,sha256=TPsvz4Kw1z_Fiek2BV79Hc2q3N37xU3oQra6Y7Ke11Q,989
-test/test_label.py,sha256=aNkIxJ-_YkBnW8QrBfRxjSsRZWeh5hn_iM7Rk1wrfPU,652
-test/test_log.py,sha256=tGAGOv4aeHT4E_pB9rq_nNA1CDHNoINpkVrCKaP4d3U,645
-test/test_phonenumbers.py,sha256=hZsXgwhn5R-7426TTWwCH9gWQwhyHtjLUstN10jnX6c,607
-test/test_presidio.py,sha256=BOGghcTWLSQPBhQxO014rO3RG-IL5XEbAaKuGN677pU,1558
-test/test_regex.py,sha256=foc2N4UCi7mGL0EIfp1t-ivgujkXMrmbsnsU77sbWZ0,5424
-test/test_scrub.py,sha256=tMYrIhbyXXKqt24tS1U_kAJT_vZfhOD4DAsf5ZFbEvU,1380
-test/test_spacy.py,sha256=gxJrNpV5B3HydUfoMsbmzRUoiKNs3_zwdSXqbPeW0qA,1846
-idscrub-1.1.2.dist-info/METADATA,sha256=2cgKI6cNWw-6zmTgiRoGt9fStRtGdSTLY7YnYBqHKm0,7242
-idscrub-1.1.2.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-idscrub-1.1.2.dist-info/top_level.txt,sha256=D4EEodXGCjGiX35ObiBTmjjBAdouN-eCvH-LezGGtks,23
-idscrub-1.1.2.dist-info/RECORD,,

test/test_all.py

@@ -1,39 +0,0 @@
-import pandas as pd
-from idscrub import IDScrub
-from pandas.testing import assert_frame_equal
-
-
-# Note: These tests will fail if the kernel has not been restarted since the SpaCy model was downloaded.
-def test_all(scrub_object):
-    scrubbed = scrub_object.all()
-    assert scrubbed == [
-        "Our names are [PERSON], [PERSON], and [PERSON].",
-        "My number is [PHONENO] and I live at [POSTCODE].",
-    ]
-
-
-def test_text_id():
-    scrub = IDScrub(["Our names are Hamish McDonald, L. Salah, and Elena Suárez."] * 10)
-
-    scrub.all()
-
-    df = scrub.get_scrubbed_data()
-
-    assert df["text_id"].max() == 10
-    assert len(df["text_id"]) == 10
-
-
-def test_get_scrubbed_data(scrub_object):
-    scrub_object.all()
-    df = scrub_object.get_scrubbed_data()
-
-    expected_df = pd.DataFrame(
-        {
-            "text_id": {0: 1, 1: 2},
-            "person": {0: ["Hamish McDonald", "L. Salah", "Elena Suárez"], 1: None},
-            "uk_phone_number": {0: None, 1: ["+441111111111"]},
-            "uk_postcode": {0: None, 1: ["AA11 1AA"]},
-        }
-    )
-
-    assert_frame_equal(df, expected_df)

test/test_chain.py

@@ -1,54 +0,0 @@
-import pandas as pd
-from pandas.testing import assert_frame_equal
-
-
-# Note: These tests will fail if the kernel has not been restarted since the SpaCy model was downloaded.
-def test_chain(scrub_object):
-    scrub_object.uk_phone_numbers()
-    scrub_object.uk_postcodes()
-    scrubbed = scrub_object.spacy_entities()
-
-    assert scrubbed == [
-        "Our names are [PERSON], [PERSON], and [PERSON].",
-        "My number is [PHONENO] and I live at [POSTCODE].",
-    ]
-
-
-def test_chain_order(scrub_object):
-    scrubbed = scrub_object.uk_phone_numbers()
-
-    assert scrubbed == [
-        "Our names are Hamish McDonald, L. Salah, and Elena Suárez.",
-        "My number is [PHONENO] and I live at AA11 1AA.",
-    ]
-
-    assert scrub_object.get_scrubbed_data()["uk_phone_number"].to_list() == [["+441111111111"]]
-    assert "uk_postcode" not in scrub_object.get_scrubbed_data().columns
-
-    scrubbed = scrub_object.uk_postcodes()
-
-    assert scrubbed == [
-        "Our names are Hamish McDonald, L. Salah, and Elena Suárez.",
-        "My number is [PHONENO] and I live at [POSTCODE].",
-    ]
-    assert scrub_object.get_scrubbed_data()["uk_phone_number"].to_list() == [["+441111111111"]]
-    assert scrub_object.get_scrubbed_data()["uk_postcode"].to_list() == [["AA11 1AA"]]
-
-
-def test_get_scrubbed_data_chain(scrub_object):
-    scrub_object.uk_phone_numbers()
-    scrub_object.uk_postcodes()
-    scrub_object.spacy_entities()
-
-    df = scrub_object.get_scrubbed_data()
-
-    expected_df = pd.DataFrame(
-        {
-            "text_id": {0: 1, 1: 2},
-            "uk_phone_number": {0: None, 1: ["+441111111111"]},
-            "uk_postcode": {0: None, 1: ["AA11 1AA"]},
-            "person": {0: ["Hamish McDonald", "L. Salah", "Elena Suárez"], 1: None},
-        }
-    )
-
-    assert_frame_equal(df, expected_df)

test/test_log.py

@@ -1,17 +0,0 @@
-from idscrub import IDScrub
-
-
-def test_log_message():
-    scrub = IDScrub(texts=["My name is Dr Strangelove. Dr. Strangelove is my name", "My name is Professor Oppenheimer"])
-    scrub.titles()
-    count = scrub.log_message("title")
-    assert count == 3
-
-
-def test_log_message_custom_regex():
-    scrub = IDScrub(texts=["My name is Dr Strangelove. Dr. Strangelove is my name", "My name is Professor Oppenheimer"])
-    scrub.custom_regex([r"Strangelove", r"Oppenheimer"], ["[DR]", "[PROFESSOR]"])
-    count_1 = scrub.log_message("custom_regex_1")
-    count_2 = scrub.log_message("custom_regex_2")
-    assert count_1 == 2
-    assert count_2 == 1