SqueakyCleanText 0.8.3__tar.gz → 0.8.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/PKG-INFO +9 -5
  2. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/README.md +8 -4
  3. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/PKG-INFO +9 -5
  4. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/__init__.py +1 -1
  5. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/streaming.py +19 -7
  6. squeakycleantext-0.8.4/tests/test_streaming.py +175 -0
  7. squeakycleantext-0.8.3/tests/test_streaming.py +0 -77
  8. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/LICENSE +0 -0
  9. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/SOURCES.txt +0 -0
  10. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/dependency_links.txt +0 -0
  11. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/entry_points.txt +0 -0
  12. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/requires.txt +0 -0
  13. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/top_level.txt +0 -0
  14. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/pyproject.toml +0 -0
  15. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/adapters.py +0 -0
  16. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/browser.py +0 -0
  17. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/cli.py +0 -0
  18. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/config.py +0 -0
  19. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/evaluation.py +0 -0
  20. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/integrations/__init__.py +0 -0
  21. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/integrations/_runtime.py +0 -0
  22. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/integrations/datatrove.py +0 -0
  23. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/integrations/haystack.py +0 -0
  24. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/integrations/hf.py +0 -0
  25. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/integrations/langchain.py +0 -0
  26. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/mcp_server.py +0 -0
  27. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/model_benchmark.py +0 -0
  28. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/model_candidates.py +0 -0
  29. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/ocr.py +0 -0
  30. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/presets.py +0 -0
  31. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/privacy.py +0 -0
  32. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/py.typed +0 -0
  33. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/report_cli.py +0 -0
  34. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/risk.py +0 -0
  35. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/score.py +0 -0
  36. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/sct.py +0 -0
  37. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/signals.py +0 -0
  38. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/__init__.py +0 -0
  39. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/anonymization_map.py +0 -0
  40. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/atomic_file.py +0 -0
  41. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/constants.py +0 -0
  42. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/contact.py +0 -0
  43. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/datetime.py +0 -0
  44. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/gliclass_adapter.py +0 -0
  45. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/gliner_adapter.py +0 -0
  46. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/ner.py +0 -0
  47. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/ner_chunking.py +0 -0
  48. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/ner_spans.py +0 -0
  49. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/normtext.py +0 -0
  50. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/onnx_pipeline.py +0 -0
  51. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/process_result.py +0 -0
  52. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/resources.py +0 -0
  53. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/special.py +0 -0
  54. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/stopwords.py +0 -0
  55. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/synthetic.py +0 -0
  56. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/sct/utils/torch_pipeline.py +0 -0
  57. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/setup.cfg +0 -0
  58. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_adapters.py +0 -0
  59. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_address_leftovers.py +0 -0
  60. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_atomic_file.py +0 -0
  61. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_batching.py +0 -0
  62. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_benchmark_adapters.py +0 -0
  63. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_benchmark_loaders.py +0 -0
  64. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_benchmark_release.py +0 -0
  65. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_benchmarks_compare.py +0 -0
  66. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_benchmarks_round2.py +0 -0
  67. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_benchmarks_round2_tools.py +0 -0
  68. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_benchmarks_score.py +0 -0
  69. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_browser.py +0 -0
  70. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_chunking.py +0 -0
  71. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_cjk_width.py +0 -0
  72. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_cleaner_lifecycle.py +0 -0
  73. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_cli.py +0 -0
  74. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_config_state.py +0 -0
  75. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_contracts.py +0 -0
  76. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_date_forms.py +0 -0
  77. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_dates_cjk_styled.py +0 -0
  78. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_dates_default.py +0 -0
  79. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_device_provenance.py +0 -0
  80. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_email_forms.py +0 -0
  81. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_escaped_markup.py +0 -0
  82. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_evaluation.py +0 -0
  83. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_fork_safety.py +0 -0
  84. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_functional.py +0 -0
  85. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_fuzz_styled_pii.py +0 -0
  86. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_integrations_frameworks.py +0 -0
  87. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_integrations_hf.py +0 -0
  88. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_integrations_runtime.py +0 -0
  89. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_keep_scripts.py +0 -0
  90. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_language_all.py +0 -0
  91. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_linux_workers.py +0 -0
  92. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_mcp_path.py +0 -0
  93. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_mcp_server.py +0 -0
  94. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_model_benchmarks.py +0 -0
  95. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_multilingual_eval_set.py +0 -0
  96. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_ner_sees_tags.py +0 -0
  97. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_ner_span_repair.py +0 -0
  98. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_ner_spans.py +0 -0
  99. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_ner_windows.py +0 -0
  100. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_number_identifiers.py +0 -0
  101. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_ocr.py +0 -0
  102. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_onnx_concurrency.py +0 -0
  103. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_onnx_external_data.py +0 -0
  104. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_onnx_model_cache.py +0 -0
  105. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_phone_forms.py +0 -0
  106. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_pipeline_text_steps.py +0 -0
  107. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_predicted_spans_monotone.py +0 -0
  108. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_presidio_tags_and_fakes.py +0 -0
  109. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_presidio_type_names.py +0 -0
  110. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_privacy.py +0 -0
  111. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_prose_punctuation_kept.py +0 -0
  112. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_public_view.py +0 -0
  113. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_regex_boundaries.py +0 -0
  114. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_regex_counts.py +0 -0
  115. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_repo_scripts.py +0 -0
  116. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_report_cli.py +0 -0
  117. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_reversible_markers.py +0 -0
  118. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_risk.py +0 -0
  119. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_score.py +0 -0
  120. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_sct.py +0 -0
  121. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_security_leftovers.py +0 -0
  122. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_stream_guard_shared_view.py +0 -0
  123. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_stream_keeps_values_whole.py +0 -0
  124. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_streaming_split_stability.py +0 -0
  125. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_styled_pii.py +0 -0
  126. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_surface_contracts.py +0 -0
  127. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_synthetic_determinism.py +0 -0
  128. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_synthetic_per_document.py +0 -0
  129. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_torch_ner_backend.py +0 -0
  130. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_training_preset.py +0 -0
  131. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_type_names.py +0 -0
  132. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_url_forms.py +0 -0
  133. {squeakycleantext-0.8.3 → squeakycleantext-0.8.4}/tests/test_user_terms.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SqueakyCleanText
3
- Version: 0.8.3
3
+ Version: 0.8.4
4
4
  Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
5
5
  Author: Rehan Fazal
6
6
  License-Expression: MIT
@@ -96,9 +96,11 @@ Dynamic: license-file
96
96
  [![Python Versions](https://img.shields.io/badge/Python-3.12%20|%203.13%20|%203.14-blue)](https://pypi.org/project/squeakycleantext/)
97
97
  [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/rhnfzl/SqueakyCleanText/blob/main/LICENSE)
98
98
 
99
- **Clean, privacy-safe text datasets for LLM training, fine-tuning and RAG, in 75 languages.**
99
+ **Stop hunting for personal details by hand. Mask detected names and contact details before sharing your text dataset.**
100
100
 
101
- SqueakyCleanText (SCT) is a text processing pipeline for datasets. A JSONL file, a Hugging Face dataset or a DataTrove job goes in. Cleaned text comes out, plus `sct_*` columns and a data card that describe each record without storing personal data. It runs on CPU or GPU.
101
+ SqueakyCleanText helps people prepare text datasets for sharing, LLM training, fine-tuning or RAG, with language detection in 75 languages.
102
+ It masks the personal details it finds and repairs broken text.
103
+ Works with JSONL files, Hugging Face datasets and DataTrove jobs, on CPU or GPU.
102
104
 
103
105
  ```text
104
106
  in: I’m Anna Berg & my mail is anna@example.se. Order 4711 ships on 2024-05-01.
@@ -110,6 +112,8 @@ out: Bel <PHONE> voor <PERSON>.
110
112
  sct_language=DUTCH sct_pii_count=2 sct_pii_types=['PERSON', 'PHONE']
111
113
  ```
112
114
 
115
+ Review cleaned text before sharing sensitive data; some personal details can be missed.
116
+
113
117
  Documentation: <https://rhnfzl.github.io/SqueakyCleanText/>
114
118
 
115
119
  ## What it does
@@ -318,7 +322,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
318
322
  - **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
319
323
  <!-- --8<-- [end:limits] -->
320
324
 
321
- Detailed known limits of 0.8.3: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
325
+ Detailed known limits of 0.8.4: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
322
326
 
323
327
  ## Documentation
324
328
 
@@ -350,7 +354,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
350
354
  author = {Fazal, Rehan},
351
355
  title = {SqueakyCleanText},
352
356
  url = {https://github.com/rhnfzl/SqueakyCleanText},
353
- version = {0.8.3},
357
+ version = {0.8.4},
354
358
  year = {2026}
355
359
  }
356
360
  ```
@@ -5,9 +5,11 @@
5
5
  [![Python Versions](https://img.shields.io/badge/Python-3.12%20|%203.13%20|%203.14-blue)](https://pypi.org/project/squeakycleantext/)
6
6
  [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/rhnfzl/SqueakyCleanText/blob/main/LICENSE)
7
7
 
8
- **Clean, privacy-safe text datasets for LLM training, fine-tuning and RAG, in 75 languages.**
8
+ **Stop hunting for personal details by hand. Mask detected names and contact details before sharing your text dataset.**
9
9
 
10
- SqueakyCleanText (SCT) is a text processing pipeline for datasets. A JSONL file, a Hugging Face dataset or a DataTrove job goes in. Cleaned text comes out, plus `sct_*` columns and a data card that describe each record without storing personal data. It runs on CPU or GPU.
10
+ SqueakyCleanText helps people prepare text datasets for sharing, LLM training, fine-tuning or RAG, with language detection in 75 languages.
11
+ It masks the personal details it finds and repairs broken text.
12
+ Works with JSONL files, Hugging Face datasets and DataTrove jobs, on CPU or GPU.
11
13
 
12
14
  ```text
13
15
  in: I’m Anna Berg &amp; my mail is anna@example.se. Order 4711 ships on 2024-05-01.
@@ -19,6 +21,8 @@ out: Bel <PHONE> voor <PERSON>.
19
21
  sct_language=DUTCH sct_pii_count=2 sct_pii_types=['PERSON', 'PHONE']
20
22
  ```
21
23
 
24
+ Review cleaned text before sharing sensitive data; some personal details can be missed.
25
+
22
26
  Documentation: <https://rhnfzl.github.io/SqueakyCleanText/>
23
27
 
24
28
  ## What it does
@@ -227,7 +231,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
227
231
  - **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
228
232
  <!-- --8<-- [end:limits] -->
229
233
 
230
- Detailed known limits of 0.8.3: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
234
+ Detailed known limits of 0.8.4: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
231
235
 
232
236
  ## Documentation
233
237
 
@@ -259,7 +263,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
259
263
  author = {Fazal, Rehan},
260
264
  title = {SqueakyCleanText},
261
265
  url = {https://github.com/rhnfzl/SqueakyCleanText},
262
- version = {0.8.3},
266
+ version = {0.8.4},
263
267
  year = {2026}
264
268
  }
265
269
  ```
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SqueakyCleanText
3
- Version: 0.8.3
3
+ Version: 0.8.4
4
4
  Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
5
5
  Author: Rehan Fazal
6
6
  License-Expression: MIT
@@ -96,9 +96,11 @@ Dynamic: license-file
96
96
  [![Python Versions](https://img.shields.io/badge/Python-3.12%20|%203.13%20|%203.14-blue)](https://pypi.org/project/squeakycleantext/)
97
97
  [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/rhnfzl/SqueakyCleanText/blob/main/LICENSE)
98
98
 
99
- **Clean, privacy-safe text datasets for LLM training, fine-tuning and RAG, in 75 languages.**
99
+ **Stop hunting for personal details by hand. Mask detected names and contact details before sharing your text dataset.**
100
100
 
101
- SqueakyCleanText (SCT) is a text processing pipeline for datasets. A JSONL file, a Hugging Face dataset or a DataTrove job goes in. Cleaned text comes out, plus `sct_*` columns and a data card that describe each record without storing personal data. It runs on CPU or GPU.
101
+ SqueakyCleanText helps people prepare text datasets for sharing, LLM training, fine-tuning or RAG, with language detection in 75 languages.
102
+ It masks the personal details it finds and repairs broken text.
103
+ Works with JSONL files, Hugging Face datasets and DataTrove jobs, on CPU or GPU.
102
104
 
103
105
  ```text
104
106
  in: I’m Anna Berg &amp; my mail is anna@example.se. Order 4711 ships on 2024-05-01.
@@ -110,6 +112,8 @@ out: Bel <PHONE> voor <PERSON>.
110
112
  sct_language=DUTCH sct_pii_count=2 sct_pii_types=['PERSON', 'PHONE']
111
113
  ```
112
114
 
115
+ Review cleaned text before sharing sensitive data; some personal details can be missed.
116
+
113
117
  Documentation: <https://rhnfzl.github.io/SqueakyCleanText/>
114
118
 
115
119
  ## What it does
@@ -318,7 +322,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
318
322
  - **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
319
323
  <!-- --8<-- [end:limits] -->
320
324
 
321
- Detailed known limits of 0.8.3: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
325
+ Detailed known limits of 0.8.4: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
322
326
 
323
327
  ## Documentation
324
328
 
@@ -350,7 +354,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
350
354
  author = {Fazal, Rehan},
351
355
  title = {SqueakyCleanText},
352
356
  url = {https://github.com/rhnfzl/SqueakyCleanText},
353
- version = {0.8.3},
357
+ version = {0.8.4},
354
358
  year = {2026}
355
359
  }
356
360
  ```
@@ -51,7 +51,7 @@ from sct.streaming import (
51
51
  from sct.utils.anonymization_map import AnonymizationMap, MapEntry
52
52
  from sct.utils.process_result import ProcessResult
53
53
 
54
- __version__ = "0.8.3"
54
+ __version__ = "0.8.4"
55
55
  __all__ = [
56
56
  "TextCleaner", "TextCleanerConfig", "CustomPatternTimeout", "ModelLoadError", "PRESETS", "preset_config",
57
57
  "PII_LABELS", "PII_LABEL_MAP",
@@ -50,11 +50,16 @@ class BufferedStreamingRedactor:
50
50
  return ""
51
51
 
52
52
  def finish(self) -> str:
53
- """Process and return the complete buffered document."""
53
+ """Return the processed document; a processing failure leaves the stream open for retry."""
54
54
  if self._finished:
55
55
  raise RuntimeError("Stream is already finished")
56
+ # Reject callback writes while processing a snapshot of the input.
56
57
  self._finished = True
57
- return self._cleaner.process("".join(self._chunks)).lm_text
58
+ try:
59
+ return self._cleaner.process("".join(self._chunks)).lm_text
60
+ except BaseException:
61
+ self._finished = False
62
+ raise
58
63
 
59
64
 
60
65
  def _sub_fuzzy_dates(cleaner, score_cutoff, language, repl, text):
@@ -229,13 +234,14 @@ class WindowedStreamingRedactor:
229
234
  self._overlap_chars = overlap_chars
230
235
  self._buffer = ""
231
236
  self._pending = "" # the separator after the last emitted text, held until more text follows it
232
- self._ready: list[str] = [] # processed windows not yet returned; they outlive a BufferError
237
+ self._ready: list[str] = [] # processed windows not yet returned; they outlive a failed feed or finish
233
238
  self._finished = False
234
239
 
235
240
  def feed(self, chunk: str) -> str:
236
241
  """Process stable prefixes until the buffer fits max_buffer_chars, retaining a trailing overlap window.
237
242
 
238
- Text processed before a BufferError comes back from the next successful feed or finish.
243
+ Text retained after an error comes back from the next successful feed or finish.
244
+ Retry with feed("") to process retained input without adding the failed call's chunk again.
239
245
  """
240
246
  if self._finished:
241
247
  raise RuntimeError("Cannot feed a finished stream")
@@ -287,13 +293,19 @@ class WindowedStreamingRedactor:
287
293
  prefix = self._buffer[:split_at]
288
294
  # the cleaner's whitespace step turns any space (\u00a0, \u202f) into " "; the separator follows it
289
295
  separator = " " if cfg.check_normalize_whitespace else separator
296
+ out = self._emit(self._cleaner.process(prefix).lm_text, separator)
290
297
  self._buffer = self._buffer[split_at + 1:]
291
- return self._emit(self._cleaner.process(prefix).lm_text, separator)
298
+ return out
292
299
 
293
300
  def finish(self) -> str:
294
- """Process the final retained window."""
301
+ """Process the final retained window; a processing failure leaves the stream open for retry."""
295
302
  if self._finished:
296
303
  raise RuntimeError("Stream is already finished")
304
+ # Reject callback writes while processing a snapshot of the input.
297
305
  self._finished = True
298
- out, self._ready = "".join(self._ready) + self._emit(self._cleaner.process(self._buffer).lm_text, ""), []
306
+ try:
307
+ out, self._ready = "".join(self._ready) + self._emit(self._cleaner.process(self._buffer).lm_text, ""), []
308
+ except BaseException:
309
+ self._finished = False
310
+ raise
299
311
  return out
@@ -0,0 +1,175 @@
1
+ from functools import partial
2
+
3
+ import pytest
4
+
5
+ from sct import TextCleaner, TextCleanerConfig
6
+ from sct.streaming import BufferedStreamingRedactor, WindowedStreamingRedactor, _masking_steps, _sub_fuzzy_dates
7
+ from sct.utils import constants, contact
8
+
9
+
10
+ def _cleaner_failing_on_call(fail_at, normalize_whitespace=True):
11
+ attempts = 0
12
+
13
+ def fail_once(text):
14
+ nonlocal attempts
15
+ attempts += 1
16
+ if attempts == fail_at:
17
+ raise ValueError("transient processing failure")
18
+ return text
19
+
20
+ return TextCleaner(TextCleanerConfig(
21
+ check_ner_process=False,
22
+ check_detect_language=False,
23
+ check_statistical_model_processing=False,
24
+ check_normalize_whitespace=normalize_whitespace,
25
+ custom_pipeline_steps=(fail_once,),
26
+ ))
27
+
28
+
29
+ @pytest.mark.parametrize("redactor", [BufferedStreamingRedactor, WindowedStreamingRedactor])
30
+ @pytest.mark.parametrize("append_after_failure", [False, True])
31
+ def test_finish_can_retry_after_processing_fails(redactor, append_after_failure):
32
+ stream = redactor(_cleaner_failing_on_call(1))
33
+ stream.feed("Mail anna@")
34
+ stream.feed("example.com now")
35
+
36
+ with pytest.raises(ValueError, match="transient processing failure"):
37
+ stream.finish()
38
+
39
+ if append_after_failure:
40
+ assert stream.feed(" please") == ""
41
+ assert stream.finish() == "Mail <EMAIL> now" + (" please" if append_after_failure else "")
42
+ with pytest.raises(RuntimeError, match="finished"):
43
+ stream.finish()
44
+ with pytest.raises(RuntimeError, match="finished"):
45
+ stream.feed("another chunk")
46
+
47
+
48
+ @pytest.mark.parametrize("redactor", [BufferedStreamingRedactor, WindowedStreamingRedactor])
49
+ @pytest.mark.parametrize("operation", ["feed", "finish"])
50
+ def test_finish_rejects_reentrant_stream_calls(redactor, operation):
51
+ reenter = True
52
+
53
+ def callback(_metrics):
54
+ nonlocal reenter
55
+ if reenter:
56
+ reenter = False
57
+ if operation == "feed":
58
+ stream.feed(" extra")
59
+ else:
60
+ stream.finish()
61
+
62
+ cleaner = TextCleaner(TextCleanerConfig(
63
+ check_ner_process=False,
64
+ check_detect_language=False,
65
+ check_statistical_model_processing=False,
66
+ ), metrics_callback=callback)
67
+ stream = redactor(cleaner)
68
+ stream.feed("Mail anna@example.com now")
69
+
70
+ with pytest.raises(RuntimeError, match="finished"):
71
+ stream.finish()
72
+
73
+ assert stream.finish() == "Mail <EMAIL> now"
74
+
75
+
76
+ @pytest.mark.parametrize("fail_at", [1, 2])
77
+ @pytest.mark.parametrize("normalize_whitespace", [False, True])
78
+ @pytest.mark.parametrize("retry_feed", [False, True])
79
+ def test_failed_window_keeps_text_for_retry(fail_at, normalize_whitespace, retry_feed):
80
+ cleaner = _cleaner_failing_on_call(fail_at, normalize_whitespace=normalize_whitespace)
81
+ stream = WindowedStreamingRedactor(cleaner, max_buffer_chars=40, overlap_chars=16)
82
+ text = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron anna@example.com end"
83
+ prefix = "Earlier words that were emitted before the failure. " if fail_at == 2 else ""
84
+ out = stream.feed(prefix)
85
+
86
+ with pytest.raises(ValueError, match="transient processing failure"):
87
+ stream.feed(text)
88
+
89
+ if retry_feed:
90
+ out += stream.feed("")
91
+ assert out + stream.finish() == prefix + ("alpha beta " if normalize_whitespace else "alpha beta ") + (
92
+ "gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron <EMAIL> end")
93
+
94
+
95
+ @pytest.mark.parametrize("normalize_whitespace", [False, True])
96
+ def test_finish_retry_keeps_output_queued_before_a_failed_window(normalize_whitespace):
97
+ cleaner = _cleaner_failing_on_call(2, normalize_whitespace=normalize_whitespace)
98
+ stream = WindowedStreamingRedactor(cleaner, max_buffer_chars=40, overlap_chars=16)
99
+
100
+ with pytest.raises(BufferError):
101
+ stream.feed("Mail anna@example.com before " + "z" * 200)
102
+ with pytest.raises(ValueError, match="transient processing failure"):
103
+ stream.finish()
104
+
105
+ assert stream.finish() == "Mail <EMAIL> before " + "z" * 200
106
+
107
+
108
+ def test_streaming_redactors_refuse_reversible_and_keep_placeholder():
109
+ def cleaner(mode):
110
+ return TextCleaner(cfg=TextCleanerConfig(check_ner_process=False, replacement_mode=mode))
111
+
112
+ for redactor in (BufferedStreamingRedactor, WindowedStreamingRedactor):
113
+ with pytest.raises(ValueError, match="retain its token map"):
114
+ redactor(cleaner("reversible"))
115
+ stream = redactor(cleaner("placeholder"))
116
+ assert stream.feed("mail a@b.com now") + stream.finish() == "mail <EMAIL> now"
117
+
118
+
119
+ def test_buffered_streaming_redactor_handles_entities_split_across_chunks():
120
+ cleaner = TextCleaner(cfg=TextCleanerConfig(
121
+ check_ner_process=False,
122
+ check_detect_language=False,
123
+ check_statistical_model_processing=False,
124
+ ))
125
+ redactor = BufferedStreamingRedactor(cleaner)
126
+
127
+ assert redactor.feed("Email maria@exam") == ""
128
+ assert redactor.feed("ple.com now") == ""
129
+ assert redactor.finish() == "Email <EMAIL> now"
130
+
131
+
132
+ def test_windowed_streaming_redactor_retains_split_token():
133
+ cleaner = TextCleaner(cfg=TextCleanerConfig(
134
+ check_ner_process=False,
135
+ check_detect_language=False,
136
+ check_statistical_model_processing=False,
137
+ ))
138
+ redactor = WindowedStreamingRedactor(
139
+ cleaner,
140
+ max_buffer_chars=24,
141
+ overlap_chars=12,
142
+ )
143
+
144
+ first = redactor.feed("Prefix words maria@exam")
145
+ second = redactor.feed("ple.com suffix words")
146
+ final = redactor.finish()
147
+
148
+ assert first + second + final == "Prefix words <EMAIL> suffix words"
149
+
150
+
151
+ def test_stream_guard_follows_the_masking_step_order():
152
+ # sct.streaming._masking_steps restates these steps: changing them here means changing it too.
153
+ cfg = TextCleanerConfig(check_ner_process=False, check_fuzzy_replace_dates=True, check_replace_identifiers=True,
154
+ custom_deny_terms={"PROJECT": ("acme",)})
155
+ cleaner = TextCleaner(cfg)
156
+ names = [step.__name__ for step in cleaner._pipeline]
157
+ masking = names[names.index("_replace_user_terms"):names.index("_replace_phone_numbers") + 1]
158
+ assert [name for name in masking if name != "_restore_brackets"] == [
159
+ "_replace_user_terms", "_replace_html", "_replace_urls", "_replace_emails", "_replace_ips", "_replace_dates",
160
+ "_replace_fuzzy_dates", "_replace_iban_cards", "_replace_identifiers", "_replace_phone_numbers",
161
+ ]
162
+ # One guard step for each, and the IBAN/card step reads two patterns.
163
+ steps = _masking_steps(cleaner, cfg, "text")
164
+ owners = [s.func if isinstance(s, partial) else s.__self__ for s in steps]
165
+ assert len(owners) == 11
166
+ assert owners[0] == cleaner._sub_user_rule
167
+ for owner, expected in zip(owners[1:6], (cleaner._html_regex, cleaner._url_regex, constants.EMAIL_REGEX,
168
+ contact.IP_VIEW, cleaner._date_view), strict=True):
169
+ assert owner is expected
170
+ assert owners[6] is _sub_fuzzy_dates
171
+ for owner, expected in zip(owners[7:10], (contact.IBAN_VIEW, contact.CARD_VIEW, constants.IDENTIFIER_REGEX),
172
+ strict=True):
173
+ assert owner is expected
174
+ assert isinstance(owners[10], contact.RejectView)
175
+ assert owners[10].view is cleaner._phone_folded_view
@@ -1,77 +0,0 @@
1
- from functools import partial
2
-
3
- import pytest
4
-
5
- from sct import TextCleaner, TextCleanerConfig
6
- from sct.streaming import BufferedStreamingRedactor, WindowedStreamingRedactor, _masking_steps, _sub_fuzzy_dates
7
- from sct.utils import constants, contact
8
-
9
-
10
- def test_streaming_redactors_refuse_reversible_and_keep_placeholder():
11
- def cleaner(mode):
12
- return TextCleaner(cfg=TextCleanerConfig(check_ner_process=False, replacement_mode=mode))
13
-
14
- for redactor in (BufferedStreamingRedactor, WindowedStreamingRedactor):
15
- with pytest.raises(ValueError, match="retain its token map"):
16
- redactor(cleaner("reversible"))
17
- stream = redactor(cleaner("placeholder"))
18
- assert stream.feed("mail a@b.com now") + stream.finish() == "mail <EMAIL> now"
19
-
20
-
21
- def test_buffered_streaming_redactor_handles_entities_split_across_chunks():
22
- cleaner = TextCleaner(cfg=TextCleanerConfig(
23
- check_ner_process=False,
24
- check_detect_language=False,
25
- check_statistical_model_processing=False,
26
- ))
27
- redactor = BufferedStreamingRedactor(cleaner)
28
-
29
- assert redactor.feed("Email maria@exam") == ""
30
- assert redactor.feed("ple.com now") == ""
31
- assert redactor.finish() == "Email <EMAIL> now"
32
-
33
-
34
- def test_windowed_streaming_redactor_retains_split_token():
35
- cleaner = TextCleaner(cfg=TextCleanerConfig(
36
- check_ner_process=False,
37
- check_detect_language=False,
38
- check_statistical_model_processing=False,
39
- ))
40
- redactor = WindowedStreamingRedactor(
41
- cleaner,
42
- max_buffer_chars=24,
43
- overlap_chars=12,
44
- )
45
-
46
- first = redactor.feed("Prefix words maria@exam")
47
- second = redactor.feed("ple.com suffix words")
48
- final = redactor.finish()
49
-
50
- assert first + second + final == "Prefix words <EMAIL> suffix words"
51
-
52
-
53
- def test_stream_guard_follows_the_masking_step_order():
54
- # sct.streaming._masking_steps restates these steps: changing them here means changing it too.
55
- cfg = TextCleanerConfig(check_ner_process=False, check_fuzzy_replace_dates=True, check_replace_identifiers=True,
56
- custom_deny_terms={"PROJECT": ("acme",)})
57
- cleaner = TextCleaner(cfg)
58
- names = [step.__name__ for step in cleaner._pipeline]
59
- masking = names[names.index("_replace_user_terms"):names.index("_replace_phone_numbers") + 1]
60
- assert [name for name in masking if name != "_restore_brackets"] == [
61
- "_replace_user_terms", "_replace_html", "_replace_urls", "_replace_emails", "_replace_ips", "_replace_dates",
62
- "_replace_fuzzy_dates", "_replace_iban_cards", "_replace_identifiers", "_replace_phone_numbers",
63
- ]
64
- # One guard step for each, and the IBAN/card step reads two patterns.
65
- steps = _masking_steps(cleaner, cfg, "text")
66
- owners = [s.func if isinstance(s, partial) else s.__self__ for s in steps]
67
- assert len(owners) == 11
68
- assert owners[0] == cleaner._sub_user_rule
69
- for owner, expected in zip(owners[1:6], (cleaner._html_regex, cleaner._url_regex, constants.EMAIL_REGEX,
70
- contact.IP_VIEW, cleaner._date_view), strict=True):
71
- assert owner is expected
72
- assert owners[6] is _sub_fuzzy_dates
73
- for owner, expected in zip(owners[7:10], (contact.IBAN_VIEW, contact.CARD_VIEW, constants.IDENTIFIER_REGEX),
74
- strict=True):
75
- assert owner is expected
76
- assert isinstance(owners[10], contact.RejectView)
77
- assert owners[10].view is cleaner._phone_folded_view