SqueakyCleanText 0.8.3__tar.gz → 0.8.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/PKG-INFO +9 -5
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/README.md +8 -4
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/PKG-INFO +9 -5
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/__init__.py +1 -1
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/streaming.py +19 -7
- squeakycleantext-0.8.4/tests/test_streaming.py +175 -0
- squeakycleantext-0.8.3/tests/test_streaming.py +0 -77
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/LICENSE +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/SOURCES.txt +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/dependency_links.txt +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/entry_points.txt +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/requires.txt +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/top_level.txt +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/pyproject.toml +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/adapters.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/browser.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/cli.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/config.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/evaluation.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/integrations/__init__.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/integrations/_runtime.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/integrations/datatrove.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/integrations/haystack.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/integrations/hf.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/integrations/langchain.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/mcp_server.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/model_benchmark.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/model_candidates.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/ocr.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/presets.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/privacy.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/py.typed +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/report_cli.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/risk.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/score.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/sct.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/signals.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/__init__.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/anonymization_map.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/atomic_file.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/constants.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/contact.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/datetime.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/gliclass_adapter.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/gliner_adapter.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/ner.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/ner_chunking.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/ner_spans.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/normtext.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/onnx_pipeline.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/process_result.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/resources.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/special.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/stopwords.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/synthetic.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/torch_pipeline.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/setup.cfg +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_adapters.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_address_leftovers.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_atomic_file.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_batching.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_benchmark_adapters.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_benchmark_loaders.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_benchmark_release.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_benchmarks_compare.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_benchmarks_round2.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_benchmarks_round2_tools.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_benchmarks_score.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_browser.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_chunking.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_cjk_width.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_cleaner_lifecycle.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_cli.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_config_state.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_contracts.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_date_forms.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_dates_cjk_styled.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_dates_default.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_device_provenance.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_email_forms.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_escaped_markup.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_evaluation.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_fork_safety.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_functional.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_fuzz_styled_pii.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_integrations_frameworks.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_integrations_hf.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_integrations_runtime.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_keep_scripts.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_language_all.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_linux_workers.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_mcp_path.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_mcp_server.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_model_benchmarks.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_multilingual_eval_set.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_ner_sees_tags.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_ner_span_repair.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_ner_spans.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_ner_windows.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_number_identifiers.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_ocr.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_onnx_concurrency.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_onnx_external_data.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_onnx_model_cache.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_phone_forms.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_pipeline_text_steps.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_predicted_spans_monotone.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_presidio_tags_and_fakes.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_presidio_type_names.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_privacy.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_prose_punctuation_kept.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_public_view.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_regex_boundaries.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_regex_counts.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_repo_scripts.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_report_cli.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_reversible_markers.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_risk.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_score.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_sct.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_security_leftovers.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_stream_guard_shared_view.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_stream_keeps_values_whole.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_streaming_split_stability.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_styled_pii.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_surface_contracts.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_synthetic_determinism.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_synthetic_per_document.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_torch_ner_backend.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_training_preset.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_type_names.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_url_forms.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_user_terms.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SqueakyCleanText
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.4
|
|
4
4
|
Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
|
|
5
5
|
Author: Rehan Fazal
|
|
6
6
|
License-Expression: MIT
|
|
@@ -96,9 +96,11 @@ Dynamic: license-file
|
|
|
96
96
|
[](https://pypi.org/project/squeakycleantext/)
|
|
97
97
|
[](https://github.com/rhnfzl/SqueakyCleanText/blob/main/LICENSE)
|
|
98
98
|
|
|
99
|
-
**
|
|
99
|
+
**Stop hunting for personal details by hand. Mask detected names and contact details before sharing your text dataset.**
|
|
100
100
|
|
|
101
|
-
SqueakyCleanText
|
|
101
|
+
SqueakyCleanText helps people prepare text datasets for sharing, LLM training, fine-tuning or RAG, with language detection in 75 languages.
|
|
102
|
+
It masks the personal details it finds and repairs broken text.
|
|
103
|
+
Works with JSONL files, Hugging Face datasets and DataTrove jobs, on CPU or GPU.
|
|
102
104
|
|
|
103
105
|
```text
|
|
104
106
|
in: I’m Anna Berg & my mail is anna@example.se. Order 4711 ships on 2024-05-01.
|
|
@@ -110,6 +112,8 @@ out: Bel <PHONE> voor <PERSON>.
|
|
|
110
112
|
sct_language=DUTCH sct_pii_count=2 sct_pii_types=['PERSON', 'PHONE']
|
|
111
113
|
```
|
|
112
114
|
|
|
115
|
+
Review cleaned text before sharing sensitive data; some personal details can be missed.
|
|
116
|
+
|
|
113
117
|
Documentation: <https://rhnfzl.github.io/SqueakyCleanText/>
|
|
114
118
|
|
|
115
119
|
## What it does
|
|
@@ -318,7 +322,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
|
|
|
318
322
|
- **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
|
|
319
323
|
<!-- --8<-- [end:limits] -->
|
|
320
324
|
|
|
321
|
-
Detailed known limits of 0.8.
|
|
325
|
+
Detailed known limits of 0.8.4: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
|
|
322
326
|
|
|
323
327
|
## Documentation
|
|
324
328
|
|
|
@@ -350,7 +354,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
|
|
|
350
354
|
author = {Fazal, Rehan},
|
|
351
355
|
title = {SqueakyCleanText},
|
|
352
356
|
url = {https://github.com/rhnfzl/SqueakyCleanText},
|
|
353
|
-
version = {0.8.
|
|
357
|
+
version = {0.8.4},
|
|
354
358
|
year = {2026}
|
|
355
359
|
}
|
|
356
360
|
```
|
|
@@ -5,9 +5,11 @@
|
|
|
5
5
|
[](https://pypi.org/project/squeakycleantext/)
|
|
6
6
|
[](https://github.com/rhnfzl/SqueakyCleanText/blob/main/LICENSE)
|
|
7
7
|
|
|
8
|
-
**
|
|
8
|
+
**Stop hunting for personal details by hand. Mask detected names and contact details before sharing your text dataset.**
|
|
9
9
|
|
|
10
|
-
SqueakyCleanText
|
|
10
|
+
SqueakyCleanText helps people prepare text datasets for sharing, LLM training, fine-tuning or RAG, with language detection in 75 languages.
|
|
11
|
+
It masks the personal details it finds and repairs broken text.
|
|
12
|
+
Works with JSONL files, Hugging Face datasets and DataTrove jobs, on CPU or GPU.
|
|
11
13
|
|
|
12
14
|
```text
|
|
13
15
|
in: I’m Anna Berg & my mail is anna@example.se. Order 4711 ships on 2024-05-01.
|
|
@@ -19,6 +21,8 @@ out: Bel <PHONE> voor <PERSON>.
|
|
|
19
21
|
sct_language=DUTCH sct_pii_count=2 sct_pii_types=['PERSON', 'PHONE']
|
|
20
22
|
```
|
|
21
23
|
|
|
24
|
+
Review cleaned text before sharing sensitive data; some personal details can be missed.
|
|
25
|
+
|
|
22
26
|
Documentation: <https://rhnfzl.github.io/SqueakyCleanText/>
|
|
23
27
|
|
|
24
28
|
## What it does
|
|
@@ -227,7 +231,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
|
|
|
227
231
|
- **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
|
|
228
232
|
<!-- --8<-- [end:limits] -->
|
|
229
233
|
|
|
230
|
-
Detailed known limits of 0.8.
|
|
234
|
+
Detailed known limits of 0.8.4: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
|
|
231
235
|
|
|
232
236
|
## Documentation
|
|
233
237
|
|
|
@@ -259,7 +263,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
|
|
|
259
263
|
author = {Fazal, Rehan},
|
|
260
264
|
title = {SqueakyCleanText},
|
|
261
265
|
url = {https://github.com/rhnfzl/SqueakyCleanText},
|
|
262
|
-
version = {0.8.
|
|
266
|
+
version = {0.8.4},
|
|
263
267
|
year = {2026}
|
|
264
268
|
}
|
|
265
269
|
```
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SqueakyCleanText
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.4
|
|
4
4
|
Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
|
|
5
5
|
Author: Rehan Fazal
|
|
6
6
|
License-Expression: MIT
|
|
@@ -96,9 +96,11 @@ Dynamic: license-file
|
|
|
96
96
|
[](https://pypi.org/project/squeakycleantext/)
|
|
97
97
|
[](https://github.com/rhnfzl/SqueakyCleanText/blob/main/LICENSE)
|
|
98
98
|
|
|
99
|
-
**
|
|
99
|
+
**Stop hunting for personal details by hand. Mask detected names and contact details before sharing your text dataset.**
|
|
100
100
|
|
|
101
|
-
SqueakyCleanText
|
|
101
|
+
SqueakyCleanText helps people prepare text datasets for sharing, LLM training, fine-tuning or RAG, with language detection in 75 languages.
|
|
102
|
+
It masks the personal details it finds and repairs broken text.
|
|
103
|
+
Works with JSONL files, Hugging Face datasets and DataTrove jobs, on CPU or GPU.
|
|
102
104
|
|
|
103
105
|
```text
|
|
104
106
|
in: I’m Anna Berg & my mail is anna@example.se. Order 4711 ships on 2024-05-01.
|
|
@@ -110,6 +112,8 @@ out: Bel <PHONE> voor <PERSON>.
|
|
|
110
112
|
sct_language=DUTCH sct_pii_count=2 sct_pii_types=['PERSON', 'PHONE']
|
|
111
113
|
```
|
|
112
114
|
|
|
115
|
+
Review cleaned text before sharing sensitive data; some personal details can be missed.
|
|
116
|
+
|
|
113
117
|
Documentation: <https://rhnfzl.github.io/SqueakyCleanText/>
|
|
114
118
|
|
|
115
119
|
## What it does
|
|
@@ -318,7 +322,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
|
|
|
318
322
|
- **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
|
|
319
323
|
<!-- --8<-- [end:limits] -->
|
|
320
324
|
|
|
321
|
-
Detailed known limits of 0.8.
|
|
325
|
+
Detailed known limits of 0.8.4: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
|
|
322
326
|
|
|
323
327
|
## Documentation
|
|
324
328
|
|
|
@@ -350,7 +354,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
|
|
|
350
354
|
author = {Fazal, Rehan},
|
|
351
355
|
title = {SqueakyCleanText},
|
|
352
356
|
url = {https://github.com/rhnfzl/SqueakyCleanText},
|
|
353
|
-
version = {0.8.
|
|
357
|
+
version = {0.8.4},
|
|
354
358
|
year = {2026}
|
|
355
359
|
}
|
|
356
360
|
```
|
|
@@ -51,7 +51,7 @@ from sct.streaming import (
|
|
|
51
51
|
from sct.utils.anonymization_map import AnonymizationMap, MapEntry
|
|
52
52
|
from sct.utils.process_result import ProcessResult
|
|
53
53
|
|
|
54
|
-
__version__ = "0.8.
|
|
54
|
+
__version__ = "0.8.4"
|
|
55
55
|
__all__ = [
|
|
56
56
|
"TextCleaner", "TextCleanerConfig", "CustomPatternTimeout", "ModelLoadError", "PRESETS", "preset_config",
|
|
57
57
|
"PII_LABELS", "PII_LABEL_MAP",
|
|
@@ -50,11 +50,16 @@ class BufferedStreamingRedactor:
|
|
|
50
50
|
return ""
|
|
51
51
|
|
|
52
52
|
def finish(self) -> str:
|
|
53
|
-
"""
|
|
53
|
+
"""Return the processed document; a processing failure leaves the stream open for retry."""
|
|
54
54
|
if self._finished:
|
|
55
55
|
raise RuntimeError("Stream is already finished")
|
|
56
|
+
# Reject callback writes while processing a snapshot of the input.
|
|
56
57
|
self._finished = True
|
|
57
|
-
|
|
58
|
+
try:
|
|
59
|
+
return self._cleaner.process("".join(self._chunks)).lm_text
|
|
60
|
+
except BaseException:
|
|
61
|
+
self._finished = False
|
|
62
|
+
raise
|
|
58
63
|
|
|
59
64
|
|
|
60
65
|
def _sub_fuzzy_dates(cleaner, score_cutoff, language, repl, text):
|
|
@@ -229,13 +234,14 @@ class WindowedStreamingRedactor:
|
|
|
229
234
|
self._overlap_chars = overlap_chars
|
|
230
235
|
self._buffer = ""
|
|
231
236
|
self._pending = "" # the separator after the last emitted text, held until more text follows it
|
|
232
|
-
self._ready: list[str] = [] # processed windows not yet returned; they outlive a
|
|
237
|
+
self._ready: list[str] = [] # processed windows not yet returned; they outlive a failed feed or finish
|
|
233
238
|
self._finished = False
|
|
234
239
|
|
|
235
240
|
def feed(self, chunk: str) -> str:
|
|
236
241
|
"""Process stable prefixes until the buffer fits max_buffer_chars, retaining a trailing overlap window.
|
|
237
242
|
|
|
238
|
-
Text
|
|
243
|
+
Text retained after an error comes back from the next successful feed or finish.
|
|
244
|
+
Retry with feed("") to process retained input without adding the failed call's chunk again.
|
|
239
245
|
"""
|
|
240
246
|
if self._finished:
|
|
241
247
|
raise RuntimeError("Cannot feed a finished stream")
|
|
@@ -287,13 +293,19 @@ class WindowedStreamingRedactor:
|
|
|
287
293
|
prefix = self._buffer[:split_at]
|
|
288
294
|
# the cleaner's whitespace step turns any space (\u00a0, \u202f) into " "; the separator follows it
|
|
289
295
|
separator = " " if cfg.check_normalize_whitespace else separator
|
|
296
|
+
out = self._emit(self._cleaner.process(prefix).lm_text, separator)
|
|
290
297
|
self._buffer = self._buffer[split_at + 1:]
|
|
291
|
-
return
|
|
298
|
+
return out
|
|
292
299
|
|
|
293
300
|
def finish(self) -> str:
|
|
294
|
-
"""Process the final retained window."""
|
|
301
|
+
"""Process the final retained window; a processing failure leaves the stream open for retry."""
|
|
295
302
|
if self._finished:
|
|
296
303
|
raise RuntimeError("Stream is already finished")
|
|
304
|
+
# Reject callback writes while processing a snapshot of the input.
|
|
297
305
|
self._finished = True
|
|
298
|
-
|
|
306
|
+
try:
|
|
307
|
+
out, self._ready = "".join(self._ready) + self._emit(self._cleaner.process(self._buffer).lm_text, ""), []
|
|
308
|
+
except BaseException:
|
|
309
|
+
self._finished = False
|
|
310
|
+
raise
|
|
299
311
|
return out
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
from functools import partial
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from sct import TextCleaner, TextCleanerConfig
|
|
6
|
+
from sct.streaming import BufferedStreamingRedactor, WindowedStreamingRedactor, _masking_steps, _sub_fuzzy_dates
|
|
7
|
+
from sct.utils import constants, contact
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _cleaner_failing_on_call(fail_at, normalize_whitespace=True):
|
|
11
|
+
attempts = 0
|
|
12
|
+
|
|
13
|
+
def fail_once(text):
|
|
14
|
+
nonlocal attempts
|
|
15
|
+
attempts += 1
|
|
16
|
+
if attempts == fail_at:
|
|
17
|
+
raise ValueError("transient processing failure")
|
|
18
|
+
return text
|
|
19
|
+
|
|
20
|
+
return TextCleaner(TextCleanerConfig(
|
|
21
|
+
check_ner_process=False,
|
|
22
|
+
check_detect_language=False,
|
|
23
|
+
check_statistical_model_processing=False,
|
|
24
|
+
check_normalize_whitespace=normalize_whitespace,
|
|
25
|
+
custom_pipeline_steps=(fail_once,),
|
|
26
|
+
))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@pytest.mark.parametrize("redactor", [BufferedStreamingRedactor, WindowedStreamingRedactor])
|
|
30
|
+
@pytest.mark.parametrize("append_after_failure", [False, True])
|
|
31
|
+
def test_finish_can_retry_after_processing_fails(redactor, append_after_failure):
|
|
32
|
+
stream = redactor(_cleaner_failing_on_call(1))
|
|
33
|
+
stream.feed("Mail anna@")
|
|
34
|
+
stream.feed("example.com now")
|
|
35
|
+
|
|
36
|
+
with pytest.raises(ValueError, match="transient processing failure"):
|
|
37
|
+
stream.finish()
|
|
38
|
+
|
|
39
|
+
if append_after_failure:
|
|
40
|
+
assert stream.feed(" please") == ""
|
|
41
|
+
assert stream.finish() == "Mail <EMAIL> now" + (" please" if append_after_failure else "")
|
|
42
|
+
with pytest.raises(RuntimeError, match="finished"):
|
|
43
|
+
stream.finish()
|
|
44
|
+
with pytest.raises(RuntimeError, match="finished"):
|
|
45
|
+
stream.feed("another chunk")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@pytest.mark.parametrize("redactor", [BufferedStreamingRedactor, WindowedStreamingRedactor])
|
|
49
|
+
@pytest.mark.parametrize("operation", ["feed", "finish"])
|
|
50
|
+
def test_finish_rejects_reentrant_stream_calls(redactor, operation):
|
|
51
|
+
reenter = True
|
|
52
|
+
|
|
53
|
+
def callback(_metrics):
|
|
54
|
+
nonlocal reenter
|
|
55
|
+
if reenter:
|
|
56
|
+
reenter = False
|
|
57
|
+
if operation == "feed":
|
|
58
|
+
stream.feed(" extra")
|
|
59
|
+
else:
|
|
60
|
+
stream.finish()
|
|
61
|
+
|
|
62
|
+
cleaner = TextCleaner(TextCleanerConfig(
|
|
63
|
+
check_ner_process=False,
|
|
64
|
+
check_detect_language=False,
|
|
65
|
+
check_statistical_model_processing=False,
|
|
66
|
+
), metrics_callback=callback)
|
|
67
|
+
stream = redactor(cleaner)
|
|
68
|
+
stream.feed("Mail anna@example.com now")
|
|
69
|
+
|
|
70
|
+
with pytest.raises(RuntimeError, match="finished"):
|
|
71
|
+
stream.finish()
|
|
72
|
+
|
|
73
|
+
assert stream.finish() == "Mail <EMAIL> now"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@pytest.mark.parametrize("fail_at", [1, 2])
|
|
77
|
+
@pytest.mark.parametrize("normalize_whitespace", [False, True])
|
|
78
|
+
@pytest.mark.parametrize("retry_feed", [False, True])
|
|
79
|
+
def test_failed_window_keeps_text_for_retry(fail_at, normalize_whitespace, retry_feed):
|
|
80
|
+
cleaner = _cleaner_failing_on_call(fail_at, normalize_whitespace=normalize_whitespace)
|
|
81
|
+
stream = WindowedStreamingRedactor(cleaner, max_buffer_chars=40, overlap_chars=16)
|
|
82
|
+
text = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron anna@example.com end"
|
|
83
|
+
prefix = "Earlier words that were emitted before the failure. " if fail_at == 2 else ""
|
|
84
|
+
out = stream.feed(prefix)
|
|
85
|
+
|
|
86
|
+
with pytest.raises(ValueError, match="transient processing failure"):
|
|
87
|
+
stream.feed(text)
|
|
88
|
+
|
|
89
|
+
if retry_feed:
|
|
90
|
+
out += stream.feed("")
|
|
91
|
+
assert out + stream.finish() == prefix + ("alpha beta " if normalize_whitespace else "alpha beta ") + (
|
|
92
|
+
"gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron <EMAIL> end")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@pytest.mark.parametrize("normalize_whitespace", [False, True])
|
|
96
|
+
def test_finish_retry_keeps_output_queued_before_a_failed_window(normalize_whitespace):
|
|
97
|
+
cleaner = _cleaner_failing_on_call(2, normalize_whitespace=normalize_whitespace)
|
|
98
|
+
stream = WindowedStreamingRedactor(cleaner, max_buffer_chars=40, overlap_chars=16)
|
|
99
|
+
|
|
100
|
+
with pytest.raises(BufferError):
|
|
101
|
+
stream.feed("Mail anna@example.com before " + "z" * 200)
|
|
102
|
+
with pytest.raises(ValueError, match="transient processing failure"):
|
|
103
|
+
stream.finish()
|
|
104
|
+
|
|
105
|
+
assert stream.finish() == "Mail <EMAIL> before " + "z" * 200
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def test_streaming_redactors_refuse_reversible_and_keep_placeholder():
|
|
109
|
+
def cleaner(mode):
|
|
110
|
+
return TextCleaner(cfg=TextCleanerConfig(check_ner_process=False, replacement_mode=mode))
|
|
111
|
+
|
|
112
|
+
for redactor in (BufferedStreamingRedactor, WindowedStreamingRedactor):
|
|
113
|
+
with pytest.raises(ValueError, match="retain its token map"):
|
|
114
|
+
redactor(cleaner("reversible"))
|
|
115
|
+
stream = redactor(cleaner("placeholder"))
|
|
116
|
+
assert stream.feed("mail a@b.com now") + stream.finish() == "mail <EMAIL> now"
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def test_buffered_streaming_redactor_handles_entities_split_across_chunks():
|
|
120
|
+
cleaner = TextCleaner(cfg=TextCleanerConfig(
|
|
121
|
+
check_ner_process=False,
|
|
122
|
+
check_detect_language=False,
|
|
123
|
+
check_statistical_model_processing=False,
|
|
124
|
+
))
|
|
125
|
+
redactor = BufferedStreamingRedactor(cleaner)
|
|
126
|
+
|
|
127
|
+
assert redactor.feed("Email maria@exam") == ""
|
|
128
|
+
assert redactor.feed("ple.com now") == ""
|
|
129
|
+
assert redactor.finish() == "Email <EMAIL> now"
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def test_windowed_streaming_redactor_retains_split_token():
|
|
133
|
+
cleaner = TextCleaner(cfg=TextCleanerConfig(
|
|
134
|
+
check_ner_process=False,
|
|
135
|
+
check_detect_language=False,
|
|
136
|
+
check_statistical_model_processing=False,
|
|
137
|
+
))
|
|
138
|
+
redactor = WindowedStreamingRedactor(
|
|
139
|
+
cleaner,
|
|
140
|
+
max_buffer_chars=24,
|
|
141
|
+
overlap_chars=12,
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
first = redactor.feed("Prefix words maria@exam")
|
|
145
|
+
second = redactor.feed("ple.com suffix words")
|
|
146
|
+
final = redactor.finish()
|
|
147
|
+
|
|
148
|
+
assert first + second + final == "Prefix words <EMAIL> suffix words"
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def test_stream_guard_follows_the_masking_step_order():
|
|
152
|
+
# sct.streaming._masking_steps restates these steps: changing them here means changing it too.
|
|
153
|
+
cfg = TextCleanerConfig(check_ner_process=False, check_fuzzy_replace_dates=True, check_replace_identifiers=True,
|
|
154
|
+
custom_deny_terms={"PROJECT": ("acme",)})
|
|
155
|
+
cleaner = TextCleaner(cfg)
|
|
156
|
+
names = [step.__name__ for step in cleaner._pipeline]
|
|
157
|
+
masking = names[names.index("_replace_user_terms"):names.index("_replace_phone_numbers") + 1]
|
|
158
|
+
assert [name for name in masking if name != "_restore_brackets"] == [
|
|
159
|
+
"_replace_user_terms", "_replace_html", "_replace_urls", "_replace_emails", "_replace_ips", "_replace_dates",
|
|
160
|
+
"_replace_fuzzy_dates", "_replace_iban_cards", "_replace_identifiers", "_replace_phone_numbers",
|
|
161
|
+
]
|
|
162
|
+
# One guard step for each, and the IBAN/card step reads two patterns.
|
|
163
|
+
steps = _masking_steps(cleaner, cfg, "text")
|
|
164
|
+
owners = [s.func if isinstance(s, partial) else s.__self__ for s in steps]
|
|
165
|
+
assert len(owners) == 11
|
|
166
|
+
assert owners[0] == cleaner._sub_user_rule
|
|
167
|
+
for owner, expected in zip(owners[1:6], (cleaner._html_regex, cleaner._url_regex, constants.EMAIL_REGEX,
|
|
168
|
+
contact.IP_VIEW, cleaner._date_view), strict=True):
|
|
169
|
+
assert owner is expected
|
|
170
|
+
assert owners[6] is _sub_fuzzy_dates
|
|
171
|
+
for owner, expected in zip(owners[7:10], (contact.IBAN_VIEW, contact.CARD_VIEW, constants.IDENTIFIER_REGEX),
|
|
172
|
+
strict=True):
|
|
173
|
+
assert owner is expected
|
|
174
|
+
assert isinstance(owners[10], contact.RejectView)
|
|
175
|
+
assert owners[10].view is cleaner._phone_folded_view
|
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
from functools import partial
|
|
2
|
-
|
|
3
|
-
import pytest
|
|
4
|
-
|
|
5
|
-
from sct import TextCleaner, TextCleanerConfig
|
|
6
|
-
from sct.streaming import BufferedStreamingRedactor, WindowedStreamingRedactor, _masking_steps, _sub_fuzzy_dates
|
|
7
|
-
from sct.utils import constants, contact
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def test_streaming_redactors_refuse_reversible_and_keep_placeholder():
|
|
11
|
-
def cleaner(mode):
|
|
12
|
-
return TextCleaner(cfg=TextCleanerConfig(check_ner_process=False, replacement_mode=mode))
|
|
13
|
-
|
|
14
|
-
for redactor in (BufferedStreamingRedactor, WindowedStreamingRedactor):
|
|
15
|
-
with pytest.raises(ValueError, match="retain its token map"):
|
|
16
|
-
redactor(cleaner("reversible"))
|
|
17
|
-
stream = redactor(cleaner("placeholder"))
|
|
18
|
-
assert stream.feed("mail a@b.com now") + stream.finish() == "mail <EMAIL> now"
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def test_buffered_streaming_redactor_handles_entities_split_across_chunks():
|
|
22
|
-
cleaner = TextCleaner(cfg=TextCleanerConfig(
|
|
23
|
-
check_ner_process=False,
|
|
24
|
-
check_detect_language=False,
|
|
25
|
-
check_statistical_model_processing=False,
|
|
26
|
-
))
|
|
27
|
-
redactor = BufferedStreamingRedactor(cleaner)
|
|
28
|
-
|
|
29
|
-
assert redactor.feed("Email maria@exam") == ""
|
|
30
|
-
assert redactor.feed("ple.com now") == ""
|
|
31
|
-
assert redactor.finish() == "Email <EMAIL> now"
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
def test_windowed_streaming_redactor_retains_split_token():
|
|
35
|
-
cleaner = TextCleaner(cfg=TextCleanerConfig(
|
|
36
|
-
check_ner_process=False,
|
|
37
|
-
check_detect_language=False,
|
|
38
|
-
check_statistical_model_processing=False,
|
|
39
|
-
))
|
|
40
|
-
redactor = WindowedStreamingRedactor(
|
|
41
|
-
cleaner,
|
|
42
|
-
max_buffer_chars=24,
|
|
43
|
-
overlap_chars=12,
|
|
44
|
-
)
|
|
45
|
-
|
|
46
|
-
first = redactor.feed("Prefix words maria@exam")
|
|
47
|
-
second = redactor.feed("ple.com suffix words")
|
|
48
|
-
final = redactor.finish()
|
|
49
|
-
|
|
50
|
-
assert first + second + final == "Prefix words <EMAIL> suffix words"
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
def test_stream_guard_follows_the_masking_step_order():
|
|
54
|
-
# sct.streaming._masking_steps restates these steps: changing them here means changing it too.
|
|
55
|
-
cfg = TextCleanerConfig(check_ner_process=False, check_fuzzy_replace_dates=True, check_replace_identifiers=True,
|
|
56
|
-
custom_deny_terms={"PROJECT": ("acme",)})
|
|
57
|
-
cleaner = TextCleaner(cfg)
|
|
58
|
-
names = [step.__name__ for step in cleaner._pipeline]
|
|
59
|
-
masking = names[names.index("_replace_user_terms"):names.index("_replace_phone_numbers") + 1]
|
|
60
|
-
assert [name for name in masking if name != "_restore_brackets"] == [
|
|
61
|
-
"_replace_user_terms", "_replace_html", "_replace_urls", "_replace_emails", "_replace_ips", "_replace_dates",
|
|
62
|
-
"_replace_fuzzy_dates", "_replace_iban_cards", "_replace_identifiers", "_replace_phone_numbers",
|
|
63
|
-
]
|
|
64
|
-
# One guard step for each, and the IBAN/card step reads two patterns.
|
|
65
|
-
steps = _masking_steps(cleaner, cfg, "text")
|
|
66
|
-
owners = [s.func if isinstance(s, partial) else s.__self__ for s in steps]
|
|
67
|
-
assert len(owners) == 11
|
|
68
|
-
assert owners[0] == cleaner._sub_user_rule
|
|
69
|
-
for owner, expected in zip(owners[1:6], (cleaner._html_regex, cleaner._url_regex, constants.EMAIL_REGEX,
|
|
70
|
-
contact.IP_VIEW, cleaner._date_view), strict=True):
|
|
71
|
-
assert owner is expected
|
|
72
|
-
assert owners[6] is _sub_fuzzy_dates
|
|
73
|
-
for owner, expected in zip(owners[7:10], (contact.IBAN_VIEW, contact.CARD_VIEW, constants.IDENTIFIER_REGEX),
|
|
74
|
-
strict=True):
|
|
75
|
-
assert owner is expected
|
|
76
|
-
assert isinstance(owners[10], contact.RejectView)
|
|
77
|
-
assert owners[10].view is cleaner._phone_folded_view
|
|
File without changes
|
|
File without changes
|
{squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|