SqueakyCleanText 0.8.2__tar.gz → 0.8.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/PKG-INFO +22 -16
  2. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/README.md +14 -10
  3. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/PKG-INFO +22 -16
  4. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/SOURCES.txt +30 -44
  5. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/requires.txt +9 -5
  6. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/pyproject.toml +19 -6
  7. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/__init__.py +3 -2
  8. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/adapters.py +86 -71
  9. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/browser.py +55 -86
  10. squeakycleantext-0.8.4/sct/cli.py +127 -0
  11. squeakycleantext-0.8.4/sct/config.py +938 -0
  12. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/evaluation.py +45 -30
  13. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/integrations/_runtime.py +40 -34
  14. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/integrations/datatrove.py +2 -2
  15. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/integrations/haystack.py +10 -10
  16. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/integrations/hf.py +13 -13
  17. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/integrations/langchain.py +3 -2
  18. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/mcp_server.py +114 -52
  19. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/model_benchmark.py +3 -2
  20. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/ocr.py +13 -6
  21. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/presets.py +4 -2
  22. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/privacy.py +7 -6
  23. squeakycleantext-0.8.4/sct/report_cli.py +179 -0
  24. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/risk.py +7 -4
  25. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/score.py +31 -21
  26. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/sct.py +549 -535
  27. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/signals.py +56 -35
  28. squeakycleantext-0.8.4/sct/streaming.py +311 -0
  29. squeakycleantext-0.8.4/sct/utils/__init__.py +0 -0
  30. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/anonymization_map.py +15 -6
  31. squeakycleantext-0.8.4/sct/utils/atomic_file.py +44 -0
  32. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/constants.py +132 -76
  33. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/contact.py +37 -95
  34. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/datetime.py +14 -14
  35. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/gliclass_adapter.py +6 -6
  36. squeakycleantext-0.8.4/sct/utils/gliner_adapter.py +227 -0
  37. squeakycleantext-0.8.4/sct/utils/ner.py +1056 -0
  38. squeakycleantext-0.8.4/sct/utils/ner_chunking.py +196 -0
  39. squeakycleantext-0.8.4/sct/utils/ner_spans.py +162 -0
  40. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/normtext.py +33 -49
  41. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/onnx_pipeline.py +165 -156
  42. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/process_result.py +33 -8
  43. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/resources.py +8 -32
  44. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/special.py +15 -18
  45. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/stopwords.py +10 -14
  46. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/synthetic.py +112 -52
  47. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/torch_pipeline.py +10 -14
  48. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_adapters.py +89 -1
  49. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_address_leftovers.py +4 -15
  50. squeakycleantext-0.8.4/tests/test_atomic_file.py +131 -0
  51. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_batching.py +62 -14
  52. squeakycleantext-0.8.4/tests/test_benchmark_adapters.py +519 -0
  53. squeakycleantext-0.8.4/tests/test_benchmark_loaders.py +240 -0
  54. squeakycleantext-0.8.4/tests/test_benchmark_release.py +287 -0
  55. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_benchmarks_compare.py +110 -7
  56. squeakycleantext-0.8.4/tests/test_benchmarks_round2.py +204 -0
  57. squeakycleantext-0.8.4/tests/test_benchmarks_round2_tools.py +349 -0
  58. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_benchmarks_score.py +4 -4
  59. squeakycleantext-0.8.4/tests/test_browser.py +101 -0
  60. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_chunking.py +46 -120
  61. squeakycleantext-0.8.4/tests/test_cleaner_lifecycle.py +185 -0
  62. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_cli.py +21 -0
  63. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_config_state.py +39 -1
  64. squeakycleantext-0.8.4/tests/test_contracts.py +222 -0
  65. squeakycleantext-0.8.4/tests/test_date_forms.py +91 -0
  66. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_dates_cjk_styled.py +13 -11
  67. squeakycleantext-0.8.4/tests/test_device_provenance.py +437 -0
  68. squeakycleantext-0.8.4/tests/test_email_forms.py +409 -0
  69. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_escaped_markup.py +10 -7
  70. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_evaluation.py +23 -1
  71. squeakycleantext-0.8.4/tests/test_fork_safety.py +87 -0
  72. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_functional.py +5 -3
  73. squeakycleantext-0.8.4/tests/test_integrations_frameworks.py +169 -0
  74. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_integrations_hf.py +2 -2
  75. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_integrations_runtime.py +56 -26
  76. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_keep_scripts.py +3 -3
  77. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_language_all.py +12 -21
  78. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_linux_workers.py +2 -2
  79. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_mcp_server.py +98 -14
  80. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_ner_sees_tags.py +54 -3
  81. squeakycleantext-0.8.4/tests/test_ner_span_repair.py +406 -0
  82. squeakycleantext-0.8.4/tests/test_ner_spans.py +129 -0
  83. squeakycleantext-0.8.4/tests/test_ner_windows.py +360 -0
  84. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_number_identifiers.py +3 -4
  85. squeakycleantext-0.8.4/tests/test_ocr.py +170 -0
  86. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_onnx_concurrency.py +23 -18
  87. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_onnx_external_data.py +40 -0
  88. squeakycleantext-0.8.4/tests/test_onnx_model_cache.py +104 -0
  89. squeakycleantext-0.8.4/tests/test_phone_forms.py +202 -0
  90. squeakycleantext-0.8.4/tests/test_pipeline_text_steps.py +162 -0
  91. squeakycleantext-0.8.4/tests/test_predicted_spans_monotone.py +136 -0
  92. squeakycleantext-0.8.4/tests/test_presidio_tags_and_fakes.py +363 -0
  93. squeakycleantext-0.8.4/tests/test_presidio_type_names.py +118 -0
  94. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_privacy.py +41 -39
  95. squeakycleantext-0.8.4/tests/test_prose_punctuation_kept.py +76 -0
  96. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_public_view.py +2 -2
  97. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_regex_boundaries.py +16 -17
  98. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_regex_counts.py +4 -4
  99. squeakycleantext-0.8.4/tests/test_repo_scripts.py +254 -0
  100. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_report_cli.py +54 -1
  101. squeakycleantext-0.8.4/tests/test_reversible_markers.py +184 -0
  102. squeakycleantext-0.8.4/tests/test_risk.py +42 -0
  103. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_score.py +6 -0
  104. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_sct.py +92 -110
  105. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_security_leftovers.py +3 -15
  106. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_stream_guard_shared_view.py +24 -2
  107. squeakycleantext-0.8.4/tests/test_stream_keeps_values_whole.py +263 -0
  108. squeakycleantext-0.8.4/tests/test_streaming.py +175 -0
  109. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_streaming_split_stability.py +14 -7
  110. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_styled_pii.py +1 -1
  111. squeakycleantext-0.8.4/tests/test_surface_contracts.py +226 -0
  112. squeakycleantext-0.8.4/tests/test_synthetic_determinism.py +138 -0
  113. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_synthetic_per_document.py +6 -6
  114. squeakycleantext-0.8.4/tests/test_torch_ner_backend.py +104 -0
  115. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_training_preset.py +3 -2
  116. squeakycleantext-0.8.4/tests/test_url_forms.py +338 -0
  117. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_user_terms.py +4 -6
  118. squeakycleantext-0.8.2/sct/cli.py +0 -117
  119. squeakycleantext-0.8.2/sct/config.py +0 -1080
  120. squeakycleantext-0.8.2/sct/report_cli.py +0 -138
  121. squeakycleantext-0.8.2/sct/streaming.py +0 -275
  122. squeakycleantext-0.8.2/sct/utils/gliner_adapter.py +0 -227
  123. squeakycleantext-0.8.2/sct/utils/ner.py +0 -1411
  124. squeakycleantext-0.8.2/setup.py +0 -2
  125. squeakycleantext-0.8.2/tests/test_benchmark_release.py +0 -97
  126. squeakycleantext-0.8.2/tests/test_browser.py +0 -37
  127. squeakycleantext-0.8.2/tests/test_codex_benchmark_round1.py +0 -84
  128. squeakycleantext-0.8.2/tests/test_codex_full_round1.py +0 -166
  129. squeakycleantext-0.8.2/tests/test_codex_full_round13.py +0 -74
  130. squeakycleantext-0.8.2/tests/test_codex_full_round14.py +0 -73
  131. squeakycleantext-0.8.2/tests/test_codex_full_round15.py +0 -154
  132. squeakycleantext-0.8.2/tests/test_codex_full_round16.py +0 -90
  133. squeakycleantext-0.8.2/tests/test_codex_full_round17.py +0 -78
  134. squeakycleantext-0.8.2/tests/test_codex_full_round18.py +0 -79
  135. squeakycleantext-0.8.2/tests/test_codex_full_round19.py +0 -32
  136. squeakycleantext-0.8.2/tests/test_codex_full_round2.py +0 -81
  137. squeakycleantext-0.8.2/tests/test_codex_full_round20.py +0 -87
  138. squeakycleantext-0.8.2/tests/test_codex_full_round21.py +0 -168
  139. squeakycleantext-0.8.2/tests/test_codex_full_round22.py +0 -52
  140. squeakycleantext-0.8.2/tests/test_codex_full_round23.py +0 -54
  141. squeakycleantext-0.8.2/tests/test_codex_full_round24.py +0 -82
  142. squeakycleantext-0.8.2/tests/test_codex_full_round25.py +0 -72
  143. squeakycleantext-0.8.2/tests/test_codex_full_round27.py +0 -85
  144. squeakycleantext-0.8.2/tests/test_codex_full_round28.py +0 -28
  145. squeakycleantext-0.8.2/tests/test_codex_full_round29.py +0 -44
  146. squeakycleantext-0.8.2/tests/test_codex_full_round3.py +0 -104
  147. squeakycleantext-0.8.2/tests/test_codex_full_round30.py +0 -53
  148. squeakycleantext-0.8.2/tests/test_codex_full_round31.py +0 -82
  149. squeakycleantext-0.8.2/tests/test_codex_full_round32.py +0 -52
  150. squeakycleantext-0.8.2/tests/test_codex_full_round35.py +0 -59
  151. squeakycleantext-0.8.2/tests/test_codex_full_round37.py +0 -41
  152. squeakycleantext-0.8.2/tests/test_codex_full_round38.py +0 -57
  153. squeakycleantext-0.8.2/tests/test_codex_full_round4.py +0 -44
  154. squeakycleantext-0.8.2/tests/test_codex_full_round5.py +0 -106
  155. squeakycleantext-0.8.2/tests/test_codex_full_round6.py +0 -80
  156. squeakycleantext-0.8.2/tests/test_codex_full_round7.py +0 -66
  157. squeakycleantext-0.8.2/tests/test_codex_full_round8.py +0 -35
  158. squeakycleantext-0.8.2/tests/test_codex_full_round9.py +0 -58
  159. squeakycleantext-0.8.2/tests/test_contracts.py +0 -143
  160. squeakycleantext-0.8.2/tests/test_device_provenance.py +0 -211
  161. squeakycleantext-0.8.2/tests/test_final_check_fixes.py +0 -273
  162. squeakycleantext-0.8.2/tests/test_full_test_round2.py +0 -145
  163. squeakycleantext-0.8.2/tests/test_fuzz_round1.py +0 -72
  164. squeakycleantext-0.8.2/tests/test_fuzz_round2.py +0 -31
  165. squeakycleantext-0.8.2/tests/test_fuzz_round3.py +0 -52
  166. squeakycleantext-0.8.2/tests/test_fuzz_round5.py +0 -51
  167. squeakycleantext-0.8.2/tests/test_fuzz_round6.py +0 -51
  168. squeakycleantext-0.8.2/tests/test_integrations_frameworks.py +0 -54
  169. squeakycleantext-0.8.2/tests/test_ocr.py +0 -87
  170. squeakycleantext-0.8.2/tests/test_review_round1.py +0 -159
  171. squeakycleantext-0.8.2/tests/test_review_round2.py +0 -69
  172. squeakycleantext-0.8.2/tests/test_review_round3.py +0 -101
  173. squeakycleantext-0.8.2/tests/test_review_round4.py +0 -68
  174. squeakycleantext-0.8.2/tests/test_risk.py +0 -21
  175. squeakycleantext-0.8.2/tests/test_streaming.py +0 -34
  176. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/LICENSE +0 -0
  177. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/dependency_links.txt +0 -0
  178. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/entry_points.txt +0 -0
  179. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/top_level.txt +0 -0
  180. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/integrations/__init__.py +0 -0
  181. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/model_candidates.py +0 -0
  182. /squeakycleantext-0.8.2/sct/utils/__init__.py → /squeakycleantext-0.8.4/sct/py.typed +0 -0
  183. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/setup.cfg +0 -0
  184. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_cjk_width.py +0 -0
  185. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_dates_default.py +0 -0
  186. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_fuzz_styled_pii.py +0 -0
  187. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_mcp_path.py +0 -0
  188. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_model_benchmarks.py +0 -0
  189. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_multilingual_eval_set.py +0 -0
  190. {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_type_names.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SqueakyCleanText
3
- Version: 0.8.2
3
+ Version: 0.8.4
4
4
  Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
5
5
  Author: Rehan Fazal
6
6
  License-Expression: MIT
@@ -44,7 +44,7 @@ Provides-Extra: fuzzy
44
44
  Requires-Dist: rapidfuzz>=3.12; extra == "fuzzy"
45
45
  Provides-Extra: torch
46
46
  Requires-Dist: torch>=2.6.0; extra == "torch"
47
- Requires-Dist: transformers>=4.48; extra == "torch"
47
+ Requires-Dist: transformers>=5.0; extra == "torch"
48
48
  Provides-Extra: gliner
49
49
  Requires-Dist: gliner<1.0,>=0.2.25; extra == "gliner"
50
50
  Provides-Extra: gliner2
@@ -53,13 +53,13 @@ Provides-Extra: synthetic
53
53
  Requires-Dist: faker>=33.0; extra == "synthetic"
54
54
  Provides-Extra: presidio
55
55
  Requires-Dist: presidio-analyzer>=2.2.360; extra == "presidio"
56
+ Requires-Dist: gliner<1.0,>=0.2.25; extra == "presidio"
56
57
  Provides-Extra: classify
57
58
  Requires-Dist: gliclass>=0.1.16; extra == "classify"
58
59
  Provides-Extra: all-ner
59
- Requires-Dist: torch>=2.6.0; extra == "all-ner"
60
- Requires-Dist: transformers>=4.48; extra == "all-ner"
61
- Requires-Dist: gliner<1.0,>=0.2.25; extra == "all-ner"
62
- Requires-Dist: gliner2<2.0,>=1.0; extra == "all-ner"
60
+ Requires-Dist: SqueakyCleanText[gliner,gliner2,torch]; extra == "all-ner"
61
+ Provides-Extra: quantize
62
+ Requires-Dist: onnx>=1.19; extra == "quantize"
63
63
  Provides-Extra: dev
64
64
  Requires-Dist: hypothesis>=6.130; extra == "dev"
65
65
  Requires-Dist: faker>=33.0; extra == "dev"
@@ -74,6 +74,8 @@ Provides-Extra: mcp
74
74
  Requires-Dist: fastmcp>=4.0.5; extra == "mcp"
75
75
  Provides-Extra: score
76
76
  Requires-Dist: datasketch>=1.6; extra == "score"
77
+ Provides-Extra: parquet
78
+ Requires-Dist: pyarrow>=17; extra == "parquet"
77
79
  Provides-Extra: datasets
78
80
  Requires-Dist: datasets>=3.0; extra == "datasets"
79
81
  Provides-Extra: datatrove
@@ -94,9 +96,11 @@ Dynamic: license-file
94
96
  [![Python Versions](https://img.shields.io/badge/Python-3.12%20|%203.13%20|%203.14-blue)](https://pypi.org/project/squeakycleantext/)
95
97
  [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/rhnfzl/SqueakyCleanText/blob/main/LICENSE)
96
98
 
97
- **Clean, privacy-safe text datasets for LLM training, fine-tuning and RAG, in 75 languages.**
99
+ **Stop hunting for personal details by hand. Mask detected names and contact details before sharing your text dataset.**
98
100
 
99
- SqueakyCleanText (SCT) is a text processing pipeline for datasets. A JSONL file, a Hugging Face dataset or a DataTrove job goes in. Cleaned text comes out, plus `sct_*` columns and a data card that describe each record without storing personal data. It runs on CPU or GPU.
101
+ SqueakyCleanText helps people prepare text datasets for sharing, LLM training, fine-tuning or RAG, with language detection in 75 languages.
102
+ It masks the personal details it finds and repairs broken text.
103
+ Works with JSONL files, Hugging Face datasets and DataTrove jobs, on CPU or GPU.
100
104
 
101
105
  ```text
102
106
  in: I’m Anna Berg &amp; my mail is anna@example.se. Order 4711 ships on 2024-05-01.
@@ -108,6 +112,8 @@ out: Bel <PHONE> voor <PERSON>.
108
112
  sct_language=DUTCH sct_pii_count=2 sct_pii_types=['PERSON', 'PHONE']
109
113
  ```
110
114
 
115
+ Review cleaned text before sharing sensitive data; some personal details can be missed.
116
+
111
117
  Documentation: <https://rhnfzl.github.io/SqueakyCleanText/>
112
118
 
113
119
  ## What it does
@@ -228,7 +234,7 @@ if __name__ == "__main__":
228
234
  | Data card for a file | `sct-report data.jsonl --text-column text --out report.json --card card.md` | [SCORE](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md) |
229
235
  | AI assistants | `sct-mcp` (MCP server, `mcp` extra) | [MCP](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MCP.md) |
230
236
 
231
- The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
237
+ The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
232
238
 
233
239
  ## How it works
234
240
 
@@ -258,7 +264,7 @@ Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/r
258
264
  - `lm_text`: the cleaned text, sentences intact, for LLMs, RAG and training.
259
265
  - `stat_text`: lower-cased, without stopwords and punctuation, for classic ML models.
260
266
  - `language`: the detected language.
261
- - `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive`, warnings and the NER models that ran. Safe to log or store next to the text.
267
+ - `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive` (`pii_count > 0`), warnings and the NER models that ran. Safe to log or store next to the text.
262
268
  - `metadata`: the same counts, plus raw values only when you ask for them (the reversible map, or `findings` with `include_entities=True`). Keep those out of logs.
263
269
 
264
270
  `score_record` turns a result into flat `sct_*` columns for filtering and data cards. `ScoreConfig` adds an optional quality score and a MinHash signature. Fields: [docs/API.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/API.md), columns: [docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md).
@@ -276,17 +282,17 @@ Modes, privacy policies and more PII types (`ner_mode="pii"`): [docs/REPLACEMENT
276
282
 
277
283
  ## Speed and hardware
278
284
 
279
- Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, AMD Threadripper PRO 7985WX and RTX 4090, median of 3 runs. 0.8.1 runs at the same speed.
285
+ Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, on a workstation CPU and high-end GPUs, median of 3 runs. 0.8.1 runs at the same speed.
280
286
 
281
287
  | Workload | Hardware | Docs/s |
282
288
  |---|---|---|
283
289
  | `process`, one document at a time | 16 CPU cores | 5.0 |
284
290
  | `process_batch`, short documents, 16 workers | 16 CPU cores | 15.5 |
285
291
  | `process_batch`, short documents, 16 workers | 64 CPU cores | 29.2 |
286
- | `process_batch`, short documents, full ensemble | 3x RTX 4090 (models split across GPUs) | 130 |
287
- | `process_batch`, short documents, English only (2 models) | 1x RTX 4090 | 646 |
292
+ | `process_batch`, short documents, full ensemble | 3 GPUs (models split across them) | 130 |
293
+ | `process_batch`, short documents, English only (2 models) | 1 GPU | 646 |
288
294
 
289
- The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one RTX 4090, about 2 hours with the full ensemble on 3 GPUs, and about 10 hours on 64 CPU cores.
295
+ The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one GPU, about 2 hours with the full ensemble on 3 GPUs, and about 10 hours on 64 CPU cores.
290
296
 
291
297
  NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cleaning, or limit `language=` to load fewer models. Hardware advice and GPU setup: [docs/PERFORMANCE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PERFORMANCE.md).
292
298
 
@@ -316,7 +322,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
316
322
  - **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
317
323
  <!-- --8<-- [end:limits] -->
318
324
 
319
- Detailed known limits of 0.8.2: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
325
+ Detailed known limits of 0.8.4: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
320
326
 
321
327
  ## Documentation
322
328
 
@@ -348,7 +354,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
348
354
  author = {Fazal, Rehan},
349
355
  title = {SqueakyCleanText},
350
356
  url = {https://github.com/rhnfzl/SqueakyCleanText},
351
- version = {0.8.1},
357
+ version = {0.8.4},
352
358
  year = {2026}
353
359
  }
354
360
  ```
@@ -5,9 +5,11 @@
5
5
  [![Python Versions](https://img.shields.io/badge/Python-3.12%20|%203.13%20|%203.14-blue)](https://pypi.org/project/squeakycleantext/)
6
6
  [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/rhnfzl/SqueakyCleanText/blob/main/LICENSE)
7
7
 
8
- **Clean, privacy-safe text datasets for LLM training, fine-tuning and RAG, in 75 languages.**
8
+ **Stop hunting for personal details by hand. Mask detected names and contact details before sharing your text dataset.**
9
9
 
10
- SqueakyCleanText (SCT) is a text processing pipeline for datasets. A JSONL file, a Hugging Face dataset or a DataTrove job goes in. Cleaned text comes out, plus `sct_*` columns and a data card that describe each record without storing personal data. It runs on CPU or GPU.
10
+ SqueakyCleanText helps people prepare text datasets for sharing, LLM training, fine-tuning or RAG, with language detection in 75 languages.
11
+ It masks the personal details it finds and repairs broken text.
12
+ Works with JSONL files, Hugging Face datasets and DataTrove jobs, on CPU or GPU.
11
13
 
12
14
  ```text
13
15
  in: I’m Anna Berg &amp; my mail is anna@example.se. Order 4711 ships on 2024-05-01.
@@ -19,6 +21,8 @@ out: Bel <PHONE> voor <PERSON>.
19
21
  sct_language=DUTCH sct_pii_count=2 sct_pii_types=['PERSON', 'PHONE']
20
22
  ```
21
23
 
24
+ Review cleaned text before sharing sensitive data; some personal details can be missed.
25
+
22
26
  Documentation: <https://rhnfzl.github.io/SqueakyCleanText/>
23
27
 
24
28
  ## What it does
@@ -139,7 +143,7 @@ if __name__ == "__main__":
139
143
  | Data card for a file | `sct-report data.jsonl --text-column text --out report.json --card card.md` | [SCORE](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md) |
140
144
  | AI assistants | `sct-mcp` (MCP server, `mcp` extra) | [MCP](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MCP.md) |
141
145
 
142
- The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
146
+ The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
143
147
 
144
148
  ## How it works
145
149
 
@@ -169,7 +173,7 @@ Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/r
169
173
  - `lm_text`: the cleaned text, sentences intact, for LLMs, RAG and training.
170
174
  - `stat_text`: lower-cased, without stopwords and punctuation, for classic ML models.
171
175
  - `language`: the detected language.
172
- - `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive`, warnings and the NER models that ran. Safe to log or store next to the text.
176
+ - `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive` (`pii_count > 0`), warnings and the NER models that ran. Safe to log or store next to the text.
173
177
  - `metadata`: the same counts, plus raw values only when you ask for them (the reversible map, or `findings` with `include_entities=True`). Keep those out of logs.
174
178
 
175
179
  `score_record` turns a result into flat `sct_*` columns for filtering and data cards. `ScoreConfig` adds an optional quality score and a MinHash signature. Fields: [docs/API.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/API.md), columns: [docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md).
@@ -187,17 +191,17 @@ Modes, privacy policies and more PII types (`ner_mode="pii"`): [docs/REPLACEMENT
187
191
 
188
192
  ## Speed and hardware
189
193
 
190
- Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, AMD Threadripper PRO 7985WX and RTX 4090, median of 3 runs. 0.8.1 runs at the same speed.
194
+ Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, on a workstation CPU and high-end GPUs, median of 3 runs. 0.8.1 runs at the same speed.
191
195
 
192
196
  | Workload | Hardware | Docs/s |
193
197
  |---|---|---|
194
198
  | `process`, one document at a time | 16 CPU cores | 5.0 |
195
199
  | `process_batch`, short documents, 16 workers | 16 CPU cores | 15.5 |
196
200
  | `process_batch`, short documents, 16 workers | 64 CPU cores | 29.2 |
197
- | `process_batch`, short documents, full ensemble | 3x RTX 4090 (models split across GPUs) | 130 |
198
- | `process_batch`, short documents, English only (2 models) | 1x RTX 4090 | 646 |
201
+ | `process_batch`, short documents, full ensemble | 3 GPUs (models split across them) | 130 |
202
+ | `process_batch`, short documents, English only (2 models) | 1 GPU | 646 |
199
203
 
200
- The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one RTX 4090, about 2 hours with the full ensemble on 3 GPUs, and about 10 hours on 64 CPU cores.
204
+ The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one GPU, about 2 hours with the full ensemble on 3 GPUs, and about 10 hours on 64 CPU cores.
201
205
 
202
206
  NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cleaning, or limit `language=` to load fewer models. Hardware advice and GPU setup: [docs/PERFORMANCE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PERFORMANCE.md).
203
207
 
@@ -227,7 +231,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
227
231
  - **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
228
232
  <!-- --8<-- [end:limits] -->
229
233
 
230
- Detailed known limits of 0.8.2: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
234
+ Detailed known limits of 0.8.4: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
231
235
 
232
236
  ## Documentation
233
237
 
@@ -259,7 +263,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
259
263
  author = {Fazal, Rehan},
260
264
  title = {SqueakyCleanText},
261
265
  url = {https://github.com/rhnfzl/SqueakyCleanText},
262
- version = {0.8.1},
266
+ version = {0.8.4},
263
267
  year = {2026}
264
268
  }
265
269
  ```
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SqueakyCleanText
3
- Version: 0.8.2
3
+ Version: 0.8.4
4
4
  Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
5
5
  Author: Rehan Fazal
6
6
  License-Expression: MIT
@@ -44,7 +44,7 @@ Provides-Extra: fuzzy
44
44
  Requires-Dist: rapidfuzz>=3.12; extra == "fuzzy"
45
45
  Provides-Extra: torch
46
46
  Requires-Dist: torch>=2.6.0; extra == "torch"
47
- Requires-Dist: transformers>=4.48; extra == "torch"
47
+ Requires-Dist: transformers>=5.0; extra == "torch"
48
48
  Provides-Extra: gliner
49
49
  Requires-Dist: gliner<1.0,>=0.2.25; extra == "gliner"
50
50
  Provides-Extra: gliner2
@@ -53,13 +53,13 @@ Provides-Extra: synthetic
53
53
  Requires-Dist: faker>=33.0; extra == "synthetic"
54
54
  Provides-Extra: presidio
55
55
  Requires-Dist: presidio-analyzer>=2.2.360; extra == "presidio"
56
+ Requires-Dist: gliner<1.0,>=0.2.25; extra == "presidio"
56
57
  Provides-Extra: classify
57
58
  Requires-Dist: gliclass>=0.1.16; extra == "classify"
58
59
  Provides-Extra: all-ner
59
- Requires-Dist: torch>=2.6.0; extra == "all-ner"
60
- Requires-Dist: transformers>=4.48; extra == "all-ner"
61
- Requires-Dist: gliner<1.0,>=0.2.25; extra == "all-ner"
62
- Requires-Dist: gliner2<2.0,>=1.0; extra == "all-ner"
60
+ Requires-Dist: SqueakyCleanText[gliner,gliner2,torch]; extra == "all-ner"
61
+ Provides-Extra: quantize
62
+ Requires-Dist: onnx>=1.19; extra == "quantize"
63
63
  Provides-Extra: dev
64
64
  Requires-Dist: hypothesis>=6.130; extra == "dev"
65
65
  Requires-Dist: faker>=33.0; extra == "dev"
@@ -74,6 +74,8 @@ Provides-Extra: mcp
74
74
  Requires-Dist: fastmcp>=4.0.5; extra == "mcp"
75
75
  Provides-Extra: score
76
76
  Requires-Dist: datasketch>=1.6; extra == "score"
77
+ Provides-Extra: parquet
78
+ Requires-Dist: pyarrow>=17; extra == "parquet"
77
79
  Provides-Extra: datasets
78
80
  Requires-Dist: datasets>=3.0; extra == "datasets"
79
81
  Provides-Extra: datatrove
@@ -94,9 +96,11 @@ Dynamic: license-file
94
96
  [![Python Versions](https://img.shields.io/badge/Python-3.12%20|%203.13%20|%203.14-blue)](https://pypi.org/project/squeakycleantext/)
95
97
  [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/rhnfzl/SqueakyCleanText/blob/main/LICENSE)
96
98
 
97
- **Clean, privacy-safe text datasets for LLM training, fine-tuning and RAG, in 75 languages.**
99
+ **Stop hunting for personal details by hand. Mask detected names and contact details before sharing your text dataset.**
98
100
 
99
- SqueakyCleanText (SCT) is a text processing pipeline for datasets. A JSONL file, a Hugging Face dataset or a DataTrove job goes in. Cleaned text comes out, plus `sct_*` columns and a data card that describe each record without storing personal data. It runs on CPU or GPU.
101
+ SqueakyCleanText helps people prepare text datasets for sharing, LLM training, fine-tuning or RAG, with language detection in 75 languages.
102
+ It masks the personal details it finds and repairs broken text.
103
+ Works with JSONL files, Hugging Face datasets and DataTrove jobs, on CPU or GPU.
100
104
 
101
105
  ```text
102
106
  in: I’m Anna Berg &amp; my mail is anna@example.se. Order 4711 ships on 2024-05-01.
@@ -108,6 +112,8 @@ out: Bel <PHONE> voor <PERSON>.
108
112
  sct_language=DUTCH sct_pii_count=2 sct_pii_types=['PERSON', 'PHONE']
109
113
  ```
110
114
 
115
+ Review cleaned text before sharing sensitive data; some personal details can be missed.
116
+
111
117
  Documentation: <https://rhnfzl.github.io/SqueakyCleanText/>
112
118
 
113
119
  ## What it does
@@ -228,7 +234,7 @@ if __name__ == "__main__":
228
234
  | Data card for a file | `sct-report data.jsonl --text-column text --out report.json --card card.md` | [SCORE](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md) |
229
235
  | AI assistants | `sct-mcp` (MCP server, `mcp` extra) | [MCP](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MCP.md) |
230
236
 
231
- The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
237
+ The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
232
238
 
233
239
  ## How it works
234
240
 
@@ -258,7 +264,7 @@ Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/r
258
264
  - `lm_text`: the cleaned text, sentences intact, for LLMs, RAG and training.
259
265
  - `stat_text`: lower-cased, without stopwords and punctuation, for classic ML models.
260
266
  - `language`: the detected language.
261
- - `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive`, warnings and the NER models that ran. Safe to log or store next to the text.
267
+ - `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive` (`pii_count > 0`), warnings and the NER models that ran. Safe to log or store next to the text.
262
268
  - `metadata`: the same counts, plus raw values only when you ask for them (the reversible map, or `findings` with `include_entities=True`). Keep those out of logs.
263
269
 
264
270
  `score_record` turns a result into flat `sct_*` columns for filtering and data cards. `ScoreConfig` adds an optional quality score and a MinHash signature. Fields: [docs/API.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/API.md), columns: [docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md).
@@ -276,17 +282,17 @@ Modes, privacy policies and more PII types (`ner_mode="pii"`): [docs/REPLACEMENT
276
282
 
277
283
  ## Speed and hardware
278
284
 
279
- Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, AMD Threadripper PRO 7985WX and RTX 4090, median of 3 runs. 0.8.1 runs at the same speed.
285
+ Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, on a workstation CPU and high-end GPUs, median of 3 runs. 0.8.1 runs at the same speed.
280
286
 
281
287
  | Workload | Hardware | Docs/s |
282
288
  |---|---|---|
283
289
  | `process`, one document at a time | 16 CPU cores | 5.0 |
284
290
  | `process_batch`, short documents, 16 workers | 16 CPU cores | 15.5 |
285
291
  | `process_batch`, short documents, 16 workers | 64 CPU cores | 29.2 |
286
- | `process_batch`, short documents, full ensemble | 3x RTX 4090 (models split across GPUs) | 130 |
287
- | `process_batch`, short documents, English only (2 models) | 1x RTX 4090 | 646 |
292
+ | `process_batch`, short documents, full ensemble | 3 GPUs (models split across them) | 130 |
293
+ | `process_batch`, short documents, English only (2 models) | 1 GPU | 646 |
288
294
 
289
- The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one RTX 4090, about 2 hours with the full ensemble on 3 GPUs, and about 10 hours on 64 CPU cores.
295
+ The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one GPU, about 2 hours with the full ensemble on 3 GPUs, and about 10 hours on 64 CPU cores.
290
296
 
291
297
  NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cleaning, or limit `language=` to load fewer models. Hardware advice and GPU setup: [docs/PERFORMANCE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PERFORMANCE.md).
292
298
 
@@ -316,7 +322,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
316
322
  - **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
317
323
  <!-- --8<-- [end:limits] -->
318
324
 
319
- Detailed known limits of 0.8.2: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
325
+ Detailed known limits of 0.8.4: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
320
326
 
321
327
  ## Documentation
322
328
 
@@ -348,7 +354,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
348
354
  author = {Fazal, Rehan},
349
355
  title = {SqueakyCleanText},
350
356
  url = {https://github.com/rhnfzl/SqueakyCleanText},
351
- version = {0.8.1},
357
+ version = {0.8.4},
352
358
  year = {2026}
353
359
  }
354
360
  ```
@@ -1,7 +1,6 @@
1
1
  LICENSE
2
2
  README.md
3
3
  pyproject.toml
4
- setup.py
5
4
  SqueakyCleanText.egg-info/PKG-INFO
6
5
  SqueakyCleanText.egg-info/SOURCES.txt
7
6
  SqueakyCleanText.egg-info/dependency_links.txt
@@ -20,6 +19,7 @@ sct/model_candidates.py
20
19
  sct/ocr.py
21
20
  sct/presets.py
22
21
  sct/privacy.py
22
+ sct/py.typed
23
23
  sct/report_cli.py
24
24
  sct/risk.py
25
25
  sct/score.py
@@ -34,12 +34,15 @@ sct/integrations/hf.py
34
34
  sct/integrations/langchain.py
35
35
  sct/utils/__init__.py
36
36
  sct/utils/anonymization_map.py
37
+ sct/utils/atomic_file.py
37
38
  sct/utils/constants.py
38
39
  sct/utils/contact.py
39
40
  sct/utils/datetime.py
40
41
  sct/utils/gliclass_adapter.py
41
42
  sct/utils/gliner_adapter.py
42
43
  sct/utils/ner.py
44
+ sct/utils/ner_chunking.py
45
+ sct/utils/ner_spans.py
43
46
  sct/utils/normtext.py
44
47
  sct/utils/onnx_pipeline.py
45
48
  sct/utils/process_result.py
@@ -50,61 +53,31 @@ sct/utils/synthetic.py
50
53
  sct/utils/torch_pipeline.py
51
54
  tests/test_adapters.py
52
55
  tests/test_address_leftovers.py
56
+ tests/test_atomic_file.py
53
57
  tests/test_batching.py
58
+ tests/test_benchmark_adapters.py
59
+ tests/test_benchmark_loaders.py
54
60
  tests/test_benchmark_release.py
55
61
  tests/test_benchmarks_compare.py
62
+ tests/test_benchmarks_round2.py
63
+ tests/test_benchmarks_round2_tools.py
56
64
  tests/test_benchmarks_score.py
57
65
  tests/test_browser.py
58
66
  tests/test_chunking.py
59
67
  tests/test_cjk_width.py
68
+ tests/test_cleaner_lifecycle.py
60
69
  tests/test_cli.py
61
- tests/test_codex_benchmark_round1.py
62
- tests/test_codex_full_round1.py
63
- tests/test_codex_full_round13.py
64
- tests/test_codex_full_round14.py
65
- tests/test_codex_full_round15.py
66
- tests/test_codex_full_round16.py
67
- tests/test_codex_full_round17.py
68
- tests/test_codex_full_round18.py
69
- tests/test_codex_full_round19.py
70
- tests/test_codex_full_round2.py
71
- tests/test_codex_full_round20.py
72
- tests/test_codex_full_round21.py
73
- tests/test_codex_full_round22.py
74
- tests/test_codex_full_round23.py
75
- tests/test_codex_full_round24.py
76
- tests/test_codex_full_round25.py
77
- tests/test_codex_full_round27.py
78
- tests/test_codex_full_round28.py
79
- tests/test_codex_full_round29.py
80
- tests/test_codex_full_round3.py
81
- tests/test_codex_full_round30.py
82
- tests/test_codex_full_round31.py
83
- tests/test_codex_full_round32.py
84
- tests/test_codex_full_round35.py
85
- tests/test_codex_full_round37.py
86
- tests/test_codex_full_round38.py
87
- tests/test_codex_full_round4.py
88
- tests/test_codex_full_round5.py
89
- tests/test_codex_full_round6.py
90
- tests/test_codex_full_round7.py
91
- tests/test_codex_full_round8.py
92
- tests/test_codex_full_round9.py
93
70
  tests/test_config_state.py
94
71
  tests/test_contracts.py
72
+ tests/test_date_forms.py
95
73
  tests/test_dates_cjk_styled.py
96
74
  tests/test_dates_default.py
97
75
  tests/test_device_provenance.py
76
+ tests/test_email_forms.py
98
77
  tests/test_escaped_markup.py
99
78
  tests/test_evaluation.py
100
- tests/test_final_check_fixes.py
101
- tests/test_full_test_round2.py
79
+ tests/test_fork_safety.py
102
80
  tests/test_functional.py
103
- tests/test_fuzz_round1.py
104
- tests/test_fuzz_round2.py
105
- tests/test_fuzz_round3.py
106
- tests/test_fuzz_round5.py
107
- tests/test_fuzz_round6.py
108
81
  tests/test_fuzz_styled_pii.py
109
82
  tests/test_integrations_frameworks.py
110
83
  tests/test_integrations_hf.py
@@ -117,28 +90,41 @@ tests/test_mcp_server.py
117
90
  tests/test_model_benchmarks.py
118
91
  tests/test_multilingual_eval_set.py
119
92
  tests/test_ner_sees_tags.py
93
+ tests/test_ner_span_repair.py
94
+ tests/test_ner_spans.py
95
+ tests/test_ner_windows.py
120
96
  tests/test_number_identifiers.py
121
97
  tests/test_ocr.py
122
98
  tests/test_onnx_concurrency.py
123
99
  tests/test_onnx_external_data.py
100
+ tests/test_onnx_model_cache.py
101
+ tests/test_phone_forms.py
102
+ tests/test_pipeline_text_steps.py
103
+ tests/test_predicted_spans_monotone.py
104
+ tests/test_presidio_tags_and_fakes.py
105
+ tests/test_presidio_type_names.py
124
106
  tests/test_privacy.py
107
+ tests/test_prose_punctuation_kept.py
125
108
  tests/test_public_view.py
126
109
  tests/test_regex_boundaries.py
127
110
  tests/test_regex_counts.py
111
+ tests/test_repo_scripts.py
128
112
  tests/test_report_cli.py
129
- tests/test_review_round1.py
130
- tests/test_review_round2.py
131
- tests/test_review_round3.py
132
- tests/test_review_round4.py
113
+ tests/test_reversible_markers.py
133
114
  tests/test_risk.py
134
115
  tests/test_score.py
135
116
  tests/test_sct.py
136
117
  tests/test_security_leftovers.py
137
118
  tests/test_stream_guard_shared_view.py
119
+ tests/test_stream_keeps_values_whole.py
138
120
  tests/test_streaming.py
139
121
  tests/test_streaming_split_stability.py
140
122
  tests/test_styled_pii.py
123
+ tests/test_surface_contracts.py
124
+ tests/test_synthetic_determinism.py
141
125
  tests/test_synthetic_per_document.py
126
+ tests/test_torch_ner_backend.py
142
127
  tests/test_training_preset.py
143
128
  tests/test_type_names.py
129
+ tests/test_url_forms.py
144
130
  tests/test_user_terms.py
@@ -11,10 +11,7 @@ presidio_anonymizer>=2.2.360
11
11
  regex>=2024.11.6
12
12
 
13
13
  [all-ner]
14
- torch>=2.6.0
15
- transformers>=4.48
16
- gliner<1.0,>=0.2.25
17
- gliner2<2.0,>=1.0
14
+ SqueakyCleanText[gliner,gliner2,torch]
18
15
 
19
16
  [classify]
20
17
  gliclass>=0.1.16
@@ -58,8 +55,15 @@ fastmcp>=4.0.5
58
55
  pillow>=12.3.0
59
56
  pytesseract>=0.3.13
60
57
 
58
+ [parquet]
59
+ pyarrow>=17
60
+
61
61
  [presidio]
62
62
  presidio-analyzer>=2.2.360
63
+ gliner<1.0,>=0.2.25
64
+
65
+ [quantize]
66
+ onnx>=1.19
63
67
 
64
68
  [score]
65
69
  datasketch>=1.6
@@ -73,4 +77,4 @@ pytest-cov>=6.0
73
77
 
74
78
  [torch]
75
79
  torch>=2.6.0
76
- transformers>=4.48
80
+ transformers>=5.0
@@ -66,7 +66,7 @@ fuzzy = [
66
66
  ]
67
67
  torch = [
68
68
  "torch>=2.6.0",
69
- "transformers>=4.48",
69
+ "transformers>=5.0",
70
70
  ]
71
71
  gliner = [
72
72
  "gliner>=0.2.25,<1.0",
@@ -79,15 +79,16 @@ synthetic = [
79
79
  ]
80
80
  presidio = [
81
81
  "presidio-analyzer>=2.2.360",
82
+ "gliner>=0.2.25,<1.0",
82
83
  ]
83
84
  classify = [
84
85
  "gliclass>=0.1.16",
85
86
  ]
86
87
  all-ner = [
87
- "torch>=2.6.0",
88
- "transformers>=4.48",
89
- "gliner>=0.2.25,<1.0",
90
- "gliner2>=1.0,<2.0",
88
+ "SqueakyCleanText[torch,gliner,gliner2]",
89
+ ]
90
+ quantize = [
91
+ "onnx>=1.19",
91
92
  ]
92
93
  dev = [
93
94
  "hypothesis>=6.130",
@@ -107,6 +108,9 @@ mcp = [
107
108
  score = [
108
109
  "datasketch>=1.6",
109
110
  ]
111
+ parquet = [
112
+ "pyarrow>=17",
113
+ ]
110
114
  datasets = [
111
115
  "datasets>=3.0",
112
116
  ]
@@ -131,12 +135,21 @@ version = {attr = "sct.__version__"}
131
135
  include = ["sct*"]
132
136
  exclude = ["tests*"]
133
137
 
138
+ [tool.setuptools.package-data]
139
+ sct = ["py.typed"]
140
+
134
141
  [tool.pytest.ini_options]
135
142
  timeout = 1200
143
+ markers = ["needs_model: loads a real Hugging Face model"]
136
144
 
137
145
  [tool.ruff]
138
146
  line-length = 127
139
147
 
140
148
  [tool.ruff.lint]
141
- select = ["E", "F", "W", "S"]
149
+ select = ["E", "F", "W", "S", "UP"]
142
150
  ignore = ["S101"]
151
+
152
+ [tool.mypy]
153
+ files = ["sct"]
154
+ ignore_missing_imports = true
155
+ check_untyped_defs = true
@@ -11,6 +11,7 @@ from sct.privacy import (
11
11
  TokenStore,
12
12
  )
13
13
  from sct.sct import CustomPatternTimeout, TextCleaner
14
+ from sct.utils.ner import ModelLoadError
14
15
  from sct.presets import PRESETS, preset_config
15
16
  from sct.adapters import (
16
17
  DataFrameResult,
@@ -50,9 +51,9 @@ from sct.streaming import (
50
51
  from sct.utils.anonymization_map import AnonymizationMap, MapEntry
51
52
  from sct.utils.process_result import ProcessResult
52
53
 
53
- __version__ = "0.8.2"
54
+ __version__ = "0.8.4"
54
55
  __all__ = [
55
- "TextCleaner", "TextCleanerConfig", "CustomPatternTimeout", "PRESETS", "preset_config",
56
+ "TextCleaner", "TextCleanerConfig", "CustomPatternTimeout", "ModelLoadError", "PRESETS", "preset_config",
56
57
  "PII_LABELS", "PII_LABEL_MAP",
57
58
  "AnonymizationMap", "MapEntry", "ProcessResult",
58
59
  "DetectedEntity", "EntityFinding", "EntityRule",