SqueakyCleanText 0.8.2__tar.gz → 0.8.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/PKG-INFO +16 -14
  2. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/README.md +8 -8
  3. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/SqueakyCleanText.egg-info/PKG-INFO +16 -14
  4. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/SqueakyCleanText.egg-info/SOURCES.txt +30 -44
  5. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/SqueakyCleanText.egg-info/requires.txt +9 -5
  6. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/pyproject.toml +19 -6
  7. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/__init__.py +3 -2
  8. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/adapters.py +86 -71
  9. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/browser.py +55 -86
  10. squeakycleantext-0.8.3/sct/cli.py +127 -0
  11. squeakycleantext-0.8.3/sct/config.py +938 -0
  12. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/evaluation.py +45 -30
  13. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/integrations/_runtime.py +40 -34
  14. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/integrations/datatrove.py +2 -2
  15. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/integrations/haystack.py +10 -10
  16. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/integrations/hf.py +13 -13
  17. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/integrations/langchain.py +3 -2
  18. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/mcp_server.py +114 -52
  19. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/model_benchmark.py +3 -2
  20. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/ocr.py +13 -6
  21. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/presets.py +4 -2
  22. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/privacy.py +7 -6
  23. squeakycleantext-0.8.3/sct/report_cli.py +179 -0
  24. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/risk.py +7 -4
  25. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/score.py +31 -21
  26. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/sct.py +549 -535
  27. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/signals.py +56 -35
  28. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/streaming.py +136 -112
  29. squeakycleantext-0.8.3/sct/utils/__init__.py +0 -0
  30. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/anonymization_map.py +15 -6
  31. squeakycleantext-0.8.3/sct/utils/atomic_file.py +44 -0
  32. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/constants.py +132 -76
  33. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/contact.py +37 -95
  34. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/datetime.py +14 -14
  35. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/gliclass_adapter.py +6 -6
  36. squeakycleantext-0.8.3/sct/utils/gliner_adapter.py +227 -0
  37. squeakycleantext-0.8.3/sct/utils/ner.py +1056 -0
  38. squeakycleantext-0.8.3/sct/utils/ner_chunking.py +196 -0
  39. squeakycleantext-0.8.3/sct/utils/ner_spans.py +162 -0
  40. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/normtext.py +33 -49
  41. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/onnx_pipeline.py +165 -156
  42. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/process_result.py +33 -8
  43. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/resources.py +8 -32
  44. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/special.py +15 -18
  45. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/stopwords.py +10 -14
  46. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/synthetic.py +112 -52
  47. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/torch_pipeline.py +10 -14
  48. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_adapters.py +89 -1
  49. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_address_leftovers.py +4 -15
  50. squeakycleantext-0.8.3/tests/test_atomic_file.py +131 -0
  51. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_batching.py +62 -14
  52. squeakycleantext-0.8.3/tests/test_benchmark_adapters.py +519 -0
  53. squeakycleantext-0.8.3/tests/test_benchmark_loaders.py +240 -0
  54. squeakycleantext-0.8.3/tests/test_benchmark_release.py +287 -0
  55. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_benchmarks_compare.py +110 -7
  56. squeakycleantext-0.8.3/tests/test_benchmarks_round2.py +204 -0
  57. squeakycleantext-0.8.3/tests/test_benchmarks_round2_tools.py +349 -0
  58. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_benchmarks_score.py +4 -4
  59. squeakycleantext-0.8.3/tests/test_browser.py +101 -0
  60. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_chunking.py +46 -120
  61. squeakycleantext-0.8.3/tests/test_cleaner_lifecycle.py +185 -0
  62. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_cli.py +21 -0
  63. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_config_state.py +39 -1
  64. squeakycleantext-0.8.3/tests/test_contracts.py +222 -0
  65. squeakycleantext-0.8.3/tests/test_date_forms.py +91 -0
  66. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_dates_cjk_styled.py +13 -11
  67. squeakycleantext-0.8.3/tests/test_device_provenance.py +437 -0
  68. squeakycleantext-0.8.3/tests/test_email_forms.py +409 -0
  69. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_escaped_markup.py +10 -7
  70. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_evaluation.py +23 -1
  71. squeakycleantext-0.8.3/tests/test_fork_safety.py +87 -0
  72. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_functional.py +5 -3
  73. squeakycleantext-0.8.3/tests/test_integrations_frameworks.py +169 -0
  74. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_integrations_hf.py +2 -2
  75. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_integrations_runtime.py +56 -26
  76. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_keep_scripts.py +3 -3
  77. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_language_all.py +12 -21
  78. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_linux_workers.py +2 -2
  79. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_mcp_server.py +98 -14
  80. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_ner_sees_tags.py +54 -3
  81. squeakycleantext-0.8.3/tests/test_ner_span_repair.py +406 -0
  82. squeakycleantext-0.8.3/tests/test_ner_spans.py +129 -0
  83. squeakycleantext-0.8.3/tests/test_ner_windows.py +360 -0
  84. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_number_identifiers.py +3 -4
  85. squeakycleantext-0.8.3/tests/test_ocr.py +170 -0
  86. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_onnx_concurrency.py +23 -18
  87. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_onnx_external_data.py +40 -0
  88. squeakycleantext-0.8.3/tests/test_onnx_model_cache.py +104 -0
  89. squeakycleantext-0.8.3/tests/test_phone_forms.py +202 -0
  90. squeakycleantext-0.8.3/tests/test_pipeline_text_steps.py +162 -0
  91. squeakycleantext-0.8.3/tests/test_predicted_spans_monotone.py +136 -0
  92. squeakycleantext-0.8.3/tests/test_presidio_tags_and_fakes.py +363 -0
  93. squeakycleantext-0.8.3/tests/test_presidio_type_names.py +118 -0
  94. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_privacy.py +41 -39
  95. squeakycleantext-0.8.3/tests/test_prose_punctuation_kept.py +76 -0
  96. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_public_view.py +2 -2
  97. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_regex_boundaries.py +16 -17
  98. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_regex_counts.py +4 -4
  99. squeakycleantext-0.8.3/tests/test_repo_scripts.py +254 -0
  100. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_report_cli.py +54 -1
  101. squeakycleantext-0.8.3/tests/test_reversible_markers.py +184 -0
  102. squeakycleantext-0.8.3/tests/test_risk.py +42 -0
  103. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_score.py +6 -0
  104. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_sct.py +92 -110
  105. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_security_leftovers.py +3 -15
  106. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_stream_guard_shared_view.py +24 -2
  107. squeakycleantext-0.8.3/tests/test_stream_keeps_values_whole.py +263 -0
  108. squeakycleantext-0.8.3/tests/test_streaming.py +77 -0
  109. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_streaming_split_stability.py +14 -7
  110. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_styled_pii.py +1 -1
  111. squeakycleantext-0.8.3/tests/test_surface_contracts.py +226 -0
  112. squeakycleantext-0.8.3/tests/test_synthetic_determinism.py +138 -0
  113. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_synthetic_per_document.py +6 -6
  114. squeakycleantext-0.8.3/tests/test_torch_ner_backend.py +104 -0
  115. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_training_preset.py +3 -2
  116. squeakycleantext-0.8.3/tests/test_url_forms.py +338 -0
  117. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_user_terms.py +4 -6
  118. squeakycleantext-0.8.2/sct/cli.py +0 -117
  119. squeakycleantext-0.8.2/sct/config.py +0 -1080
  120. squeakycleantext-0.8.2/sct/report_cli.py +0 -138
  121. squeakycleantext-0.8.2/sct/utils/gliner_adapter.py +0 -227
  122. squeakycleantext-0.8.2/sct/utils/ner.py +0 -1411
  123. squeakycleantext-0.8.2/setup.py +0 -2
  124. squeakycleantext-0.8.2/tests/test_benchmark_release.py +0 -97
  125. squeakycleantext-0.8.2/tests/test_browser.py +0 -37
  126. squeakycleantext-0.8.2/tests/test_codex_benchmark_round1.py +0 -84
  127. squeakycleantext-0.8.2/tests/test_codex_full_round1.py +0 -166
  128. squeakycleantext-0.8.2/tests/test_codex_full_round13.py +0 -74
  129. squeakycleantext-0.8.2/tests/test_codex_full_round14.py +0 -73
  130. squeakycleantext-0.8.2/tests/test_codex_full_round15.py +0 -154
  131. squeakycleantext-0.8.2/tests/test_codex_full_round16.py +0 -90
  132. squeakycleantext-0.8.2/tests/test_codex_full_round17.py +0 -78
  133. squeakycleantext-0.8.2/tests/test_codex_full_round18.py +0 -79
  134. squeakycleantext-0.8.2/tests/test_codex_full_round19.py +0 -32
  135. squeakycleantext-0.8.2/tests/test_codex_full_round2.py +0 -81
  136. squeakycleantext-0.8.2/tests/test_codex_full_round20.py +0 -87
  137. squeakycleantext-0.8.2/tests/test_codex_full_round21.py +0 -168
  138. squeakycleantext-0.8.2/tests/test_codex_full_round22.py +0 -52
  139. squeakycleantext-0.8.2/tests/test_codex_full_round23.py +0 -54
  140. squeakycleantext-0.8.2/tests/test_codex_full_round24.py +0 -82
  141. squeakycleantext-0.8.2/tests/test_codex_full_round25.py +0 -72
  142. squeakycleantext-0.8.2/tests/test_codex_full_round27.py +0 -85
  143. squeakycleantext-0.8.2/tests/test_codex_full_round28.py +0 -28
  144. squeakycleantext-0.8.2/tests/test_codex_full_round29.py +0 -44
  145. squeakycleantext-0.8.2/tests/test_codex_full_round3.py +0 -104
  146. squeakycleantext-0.8.2/tests/test_codex_full_round30.py +0 -53
  147. squeakycleantext-0.8.2/tests/test_codex_full_round31.py +0 -82
  148. squeakycleantext-0.8.2/tests/test_codex_full_round32.py +0 -52
  149. squeakycleantext-0.8.2/tests/test_codex_full_round35.py +0 -59
  150. squeakycleantext-0.8.2/tests/test_codex_full_round37.py +0 -41
  151. squeakycleantext-0.8.2/tests/test_codex_full_round38.py +0 -57
  152. squeakycleantext-0.8.2/tests/test_codex_full_round4.py +0 -44
  153. squeakycleantext-0.8.2/tests/test_codex_full_round5.py +0 -106
  154. squeakycleantext-0.8.2/tests/test_codex_full_round6.py +0 -80
  155. squeakycleantext-0.8.2/tests/test_codex_full_round7.py +0 -66
  156. squeakycleantext-0.8.2/tests/test_codex_full_round8.py +0 -35
  157. squeakycleantext-0.8.2/tests/test_codex_full_round9.py +0 -58
  158. squeakycleantext-0.8.2/tests/test_contracts.py +0 -143
  159. squeakycleantext-0.8.2/tests/test_device_provenance.py +0 -211
  160. squeakycleantext-0.8.2/tests/test_final_check_fixes.py +0 -273
  161. squeakycleantext-0.8.2/tests/test_full_test_round2.py +0 -145
  162. squeakycleantext-0.8.2/tests/test_fuzz_round1.py +0 -72
  163. squeakycleantext-0.8.2/tests/test_fuzz_round2.py +0 -31
  164. squeakycleantext-0.8.2/tests/test_fuzz_round3.py +0 -52
  165. squeakycleantext-0.8.2/tests/test_fuzz_round5.py +0 -51
  166. squeakycleantext-0.8.2/tests/test_fuzz_round6.py +0 -51
  167. squeakycleantext-0.8.2/tests/test_integrations_frameworks.py +0 -54
  168. squeakycleantext-0.8.2/tests/test_ocr.py +0 -87
  169. squeakycleantext-0.8.2/tests/test_review_round1.py +0 -159
  170. squeakycleantext-0.8.2/tests/test_review_round2.py +0 -69
  171. squeakycleantext-0.8.2/tests/test_review_round3.py +0 -101
  172. squeakycleantext-0.8.2/tests/test_review_round4.py +0 -68
  173. squeakycleantext-0.8.2/tests/test_risk.py +0 -21
  174. squeakycleantext-0.8.2/tests/test_streaming.py +0 -34
  175. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/LICENSE +0 -0
  176. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/SqueakyCleanText.egg-info/dependency_links.txt +0 -0
  177. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/SqueakyCleanText.egg-info/entry_points.txt +0 -0
  178. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/SqueakyCleanText.egg-info/top_level.txt +0 -0
  179. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/integrations/__init__.py +0 -0
  180. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/model_candidates.py +0 -0
  181. /squeakycleantext-0.8.2/sct/utils/__init__.py → /squeakycleantext-0.8.3/sct/py.typed +0 -0
  182. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/setup.cfg +0 -0
  183. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_cjk_width.py +0 -0
  184. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_dates_default.py +0 -0
  185. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_fuzz_styled_pii.py +0 -0
  186. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_mcp_path.py +0 -0
  187. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_model_benchmarks.py +0 -0
  188. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_multilingual_eval_set.py +0 -0
  189. {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_type_names.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SqueakyCleanText
3
- Version: 0.8.2
3
+ Version: 0.8.3
4
4
  Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
5
5
  Author: Rehan Fazal
6
6
  License-Expression: MIT
@@ -44,7 +44,7 @@ Provides-Extra: fuzzy
44
44
  Requires-Dist: rapidfuzz>=3.12; extra == "fuzzy"
45
45
  Provides-Extra: torch
46
46
  Requires-Dist: torch>=2.6.0; extra == "torch"
47
- Requires-Dist: transformers>=4.48; extra == "torch"
47
+ Requires-Dist: transformers>=5.0; extra == "torch"
48
48
  Provides-Extra: gliner
49
49
  Requires-Dist: gliner<1.0,>=0.2.25; extra == "gliner"
50
50
  Provides-Extra: gliner2
@@ -53,13 +53,13 @@ Provides-Extra: synthetic
53
53
  Requires-Dist: faker>=33.0; extra == "synthetic"
54
54
  Provides-Extra: presidio
55
55
  Requires-Dist: presidio-analyzer>=2.2.360; extra == "presidio"
56
+ Requires-Dist: gliner<1.0,>=0.2.25; extra == "presidio"
56
57
  Provides-Extra: classify
57
58
  Requires-Dist: gliclass>=0.1.16; extra == "classify"
58
59
  Provides-Extra: all-ner
59
- Requires-Dist: torch>=2.6.0; extra == "all-ner"
60
- Requires-Dist: transformers>=4.48; extra == "all-ner"
61
- Requires-Dist: gliner<1.0,>=0.2.25; extra == "all-ner"
62
- Requires-Dist: gliner2<2.0,>=1.0; extra == "all-ner"
60
+ Requires-Dist: SqueakyCleanText[gliner,gliner2,torch]; extra == "all-ner"
61
+ Provides-Extra: quantize
62
+ Requires-Dist: onnx>=1.19; extra == "quantize"
63
63
  Provides-Extra: dev
64
64
  Requires-Dist: hypothesis>=6.130; extra == "dev"
65
65
  Requires-Dist: faker>=33.0; extra == "dev"
@@ -74,6 +74,8 @@ Provides-Extra: mcp
74
74
  Requires-Dist: fastmcp>=4.0.5; extra == "mcp"
75
75
  Provides-Extra: score
76
76
  Requires-Dist: datasketch>=1.6; extra == "score"
77
+ Provides-Extra: parquet
78
+ Requires-Dist: pyarrow>=17; extra == "parquet"
77
79
  Provides-Extra: datasets
78
80
  Requires-Dist: datasets>=3.0; extra == "datasets"
79
81
  Provides-Extra: datatrove
@@ -228,7 +230,7 @@ if __name__ == "__main__":
228
230
  | Data card for a file | `sct-report data.jsonl --text-column text --out report.json --card card.md` | [SCORE](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md) |
229
231
  | AI assistants | `sct-mcp` (MCP server, `mcp` extra) | [MCP](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MCP.md) |
230
232
 
231
- The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
233
+ The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
232
234
 
233
235
  ## How it works
234
236
 
@@ -258,7 +260,7 @@ Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/r
258
260
  - `lm_text`: the cleaned text, sentences intact, for LLMs, RAG and training.
259
261
  - `stat_text`: lower-cased, without stopwords and punctuation, for classic ML models.
260
262
  - `language`: the detected language.
261
- - `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive`, warnings and the NER models that ran. Safe to log or store next to the text.
263
+ - `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive` (`pii_count > 0`), warnings and the NER models that ran. Safe to log or store next to the text.
262
264
  - `metadata`: the same counts, plus raw values only when you ask for them (the reversible map, or `findings` with `include_entities=True`). Keep those out of logs.
263
265
 
264
266
  `score_record` turns a result into flat `sct_*` columns for filtering and data cards. `ScoreConfig` adds an optional quality score and a MinHash signature. Fields: [docs/API.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/API.md), columns: [docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md).
@@ -276,17 +278,17 @@ Modes, privacy policies and more PII types (`ner_mode="pii"`): [docs/REPLACEMENT
276
278
 
277
279
  ## Speed and hardware
278
280
 
279
- Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, AMD Threadripper PRO 7985WX and RTX 4090, median of 3 runs. 0.8.1 runs at the same speed.
281
+ Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, on a workstation CPU and high-end GPUs, median of 3 runs. 0.8.1 runs at the same speed.
280
282
 
281
283
  | Workload | Hardware | Docs/s |
282
284
  |---|---|---|
283
285
  | `process`, one document at a time | 16 CPU cores | 5.0 |
284
286
  | `process_batch`, short documents, 16 workers | 16 CPU cores | 15.5 |
285
287
  | `process_batch`, short documents, 16 workers | 64 CPU cores | 29.2 |
286
- | `process_batch`, short documents, full ensemble | 3x RTX 4090 (models split across GPUs) | 130 |
287
- | `process_batch`, short documents, English only (2 models) | 1x RTX 4090 | 646 |
288
+ | `process_batch`, short documents, full ensemble | 3 GPUs (models split across them) | 130 |
289
+ | `process_batch`, short documents, English only (2 models) | 1 GPU | 646 |
288
290
 
289
- The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one RTX 4090, about 2 hours with the full ensemble on 3 GPUs, and about 10 hours on 64 CPU cores.
291
+ The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one GPU, about 2 hours with the full ensemble on 3 GPUs, and about 10 hours on 64 CPU cores.
290
292
 
291
293
  NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cleaning, or limit `language=` to load fewer models. Hardware advice and GPU setup: [docs/PERFORMANCE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PERFORMANCE.md).
292
294
 
@@ -316,7 +318,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
316
318
  - **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
317
319
  <!-- --8<-- [end:limits] -->
318
320
 
319
- Detailed known limits of 0.8.2: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
321
+ Detailed known limits of 0.8.3: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
320
322
 
321
323
  ## Documentation
322
324
 
@@ -348,7 +350,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
348
350
  author = {Fazal, Rehan},
349
351
  title = {SqueakyCleanText},
350
352
  url = {https://github.com/rhnfzl/SqueakyCleanText},
351
- version = {0.8.1},
353
+ version = {0.8.3},
352
354
  year = {2026}
353
355
  }
354
356
  ```
@@ -139,7 +139,7 @@ if __name__ == "__main__":
139
139
  | Data card for a file | `sct-report data.jsonl --text-column text --out report.json --card card.md` | [SCORE](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md) |
140
140
  | AI assistants | `sct-mcp` (MCP server, `mcp` extra) | [MCP](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MCP.md) |
141
141
 
142
- The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
142
+ The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
143
143
 
144
144
  ## How it works
145
145
 
@@ -169,7 +169,7 @@ Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/r
169
169
  - `lm_text`: the cleaned text, sentences intact, for LLMs, RAG and training.
170
170
  - `stat_text`: lower-cased, without stopwords and punctuation, for classic ML models.
171
171
  - `language`: the detected language.
172
- - `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive`, warnings and the NER models that ran. Safe to log or store next to the text.
172
+ - `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive` (`pii_count > 0`), warnings and the NER models that ran. Safe to log or store next to the text.
173
173
  - `metadata`: the same counts, plus raw values only when you ask for them (the reversible map, or `findings` with `include_entities=True`). Keep those out of logs.
174
174
 
175
175
  `score_record` turns a result into flat `sct_*` columns for filtering and data cards. `ScoreConfig` adds an optional quality score and a MinHash signature. Fields: [docs/API.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/API.md), columns: [docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md).
@@ -187,17 +187,17 @@ Modes, privacy policies and more PII types (`ner_mode="pii"`): [docs/REPLACEMENT
187
187
 
188
188
  ## Speed and hardware
189
189
 
190
- Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, AMD Threadripper PRO 7985WX and RTX 4090, median of 3 runs. 0.8.1 runs at the same speed.
190
+ Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, on a workstation CPU and high-end GPUs, median of 3 runs. 0.8.1 runs at the same speed.
191
191
 
192
192
  | Workload | Hardware | Docs/s |
193
193
  |---|---|---|
194
194
  | `process`, one document at a time | 16 CPU cores | 5.0 |
195
195
  | `process_batch`, short documents, 16 workers | 16 CPU cores | 15.5 |
196
196
  | `process_batch`, short documents, 16 workers | 64 CPU cores | 29.2 |
197
- | `process_batch`, short documents, full ensemble | 3x RTX 4090 (models split across GPUs) | 130 |
198
- | `process_batch`, short documents, English only (2 models) | 1x RTX 4090 | 646 |
197
+ | `process_batch`, short documents, full ensemble | 3 GPUs (models split across them) | 130 |
198
+ | `process_batch`, short documents, English only (2 models) | 1 GPU | 646 |
199
199
 
200
- The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one RTX 4090, about 2 hours with the full ensemble on 3 GPUs, and about 10 hours on 64 CPU cores.
200
+ The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one GPU, about 2 hours with the full ensemble on 3 GPUs, and about 10 hours on 64 CPU cores.
201
201
 
202
202
  NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cleaning, or limit `language=` to load fewer models. Hardware advice and GPU setup: [docs/PERFORMANCE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PERFORMANCE.md).
203
203
 
@@ -227,7 +227,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
227
227
  - **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
228
228
  <!-- --8<-- [end:limits] -->
229
229
 
230
- Detailed known limits of 0.8.2: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
230
+ Detailed known limits of 0.8.3: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
231
231
 
232
232
  ## Documentation
233
233
 
@@ -259,7 +259,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
259
259
  author = {Fazal, Rehan},
260
260
  title = {SqueakyCleanText},
261
261
  url = {https://github.com/rhnfzl/SqueakyCleanText},
262
- version = {0.8.1},
262
+ version = {0.8.3},
263
263
  year = {2026}
264
264
  }
265
265
  ```
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SqueakyCleanText
3
- Version: 0.8.2
3
+ Version: 0.8.3
4
4
  Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
5
5
  Author: Rehan Fazal
6
6
  License-Expression: MIT
@@ -44,7 +44,7 @@ Provides-Extra: fuzzy
44
44
  Requires-Dist: rapidfuzz>=3.12; extra == "fuzzy"
45
45
  Provides-Extra: torch
46
46
  Requires-Dist: torch>=2.6.0; extra == "torch"
47
- Requires-Dist: transformers>=4.48; extra == "torch"
47
+ Requires-Dist: transformers>=5.0; extra == "torch"
48
48
  Provides-Extra: gliner
49
49
  Requires-Dist: gliner<1.0,>=0.2.25; extra == "gliner"
50
50
  Provides-Extra: gliner2
@@ -53,13 +53,13 @@ Provides-Extra: synthetic
53
53
  Requires-Dist: faker>=33.0; extra == "synthetic"
54
54
  Provides-Extra: presidio
55
55
  Requires-Dist: presidio-analyzer>=2.2.360; extra == "presidio"
56
+ Requires-Dist: gliner<1.0,>=0.2.25; extra == "presidio"
56
57
  Provides-Extra: classify
57
58
  Requires-Dist: gliclass>=0.1.16; extra == "classify"
58
59
  Provides-Extra: all-ner
59
- Requires-Dist: torch>=2.6.0; extra == "all-ner"
60
- Requires-Dist: transformers>=4.48; extra == "all-ner"
61
- Requires-Dist: gliner<1.0,>=0.2.25; extra == "all-ner"
62
- Requires-Dist: gliner2<2.0,>=1.0; extra == "all-ner"
60
+ Requires-Dist: SqueakyCleanText[gliner,gliner2,torch]; extra == "all-ner"
61
+ Provides-Extra: quantize
62
+ Requires-Dist: onnx>=1.19; extra == "quantize"
63
63
  Provides-Extra: dev
64
64
  Requires-Dist: hypothesis>=6.130; extra == "dev"
65
65
  Requires-Dist: faker>=33.0; extra == "dev"
@@ -74,6 +74,8 @@ Provides-Extra: mcp
74
74
  Requires-Dist: fastmcp>=4.0.5; extra == "mcp"
75
75
  Provides-Extra: score
76
76
  Requires-Dist: datasketch>=1.6; extra == "score"
77
+ Provides-Extra: parquet
78
+ Requires-Dist: pyarrow>=17; extra == "parquet"
77
79
  Provides-Extra: datasets
78
80
  Requires-Dist: datasets>=3.0; extra == "datasets"
79
81
  Provides-Extra: datatrove
@@ -228,7 +230,7 @@ if __name__ == "__main__":
228
230
  | Data card for a file | `sct-report data.jsonl --text-column text --out report.json --card card.md` | [SCORE](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md) |
229
231
  | AI assistants | `sct-mcp` (MCP server, `mcp` extra) | [MCP](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MCP.md) |
230
232
 
231
- The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
233
+ The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
232
234
 
233
235
  ## How it works
234
236
 
@@ -258,7 +260,7 @@ Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/r
258
260
  - `lm_text`: the cleaned text, sentences intact, for LLMs, RAG and training.
259
261
  - `stat_text`: lower-cased, without stopwords and punctuation, for classic ML models.
260
262
  - `language`: the detected language.
261
- - `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive`, warnings and the NER models that ran. Safe to log or store next to the text.
263
+ - `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive` (`pii_count > 0`), warnings and the NER models that ran. Safe to log or store next to the text.
262
264
  - `metadata`: the same counts, plus raw values only when you ask for them (the reversible map, or `findings` with `include_entities=True`). Keep those out of logs.
263
265
 
264
266
  `score_record` turns a result into flat `sct_*` columns for filtering and data cards. `ScoreConfig` adds an optional quality score and a MinHash signature. Fields: [docs/API.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/API.md), columns: [docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md).
@@ -276,17 +278,17 @@ Modes, privacy policies and more PII types (`ner_mode="pii"`): [docs/REPLACEMENT
276
278
 
277
279
  ## Speed and hardware
278
280
 
279
- Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, AMD Threadripper PRO 7985WX and RTX 4090, median of 3 runs. 0.8.1 runs at the same speed.
281
+ Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, on a workstation CPU and high-end GPUs, median of 3 runs. 0.8.1 runs at the same speed.
280
282
 
281
283
  | Workload | Hardware | Docs/s |
282
284
  |---|---|---|
283
285
  | `process`, one document at a time | 16 CPU cores | 5.0 |
284
286
  | `process_batch`, short documents, 16 workers | 16 CPU cores | 15.5 |
285
287
  | `process_batch`, short documents, 16 workers | 64 CPU cores | 29.2 |
286
- | `process_batch`, short documents, full ensemble | 3x RTX 4090 (models split across GPUs) | 130 |
287
- | `process_batch`, short documents, English only (2 models) | 1x RTX 4090 | 646 |
288
+ | `process_batch`, short documents, full ensemble | 3 GPUs (models split across them) | 130 |
289
+ | `process_batch`, short documents, English only (2 models) | 1 GPU | 646 |
288
290
 
289
- The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one RTX 4090, about 2 hours with the full ensemble on 3 GPUs, and about 10 hours on 64 CPU cores.
291
+ The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one GPU, about 2 hours with the full ensemble on 3 GPUs, and about 10 hours on 64 CPU cores.
290
292
 
291
293
  NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cleaning, or limit `language=` to load fewer models. Hardware advice and GPU setup: [docs/PERFORMANCE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PERFORMANCE.md).
292
294
 
@@ -316,7 +318,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
316
318
  - **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
317
319
  <!-- --8<-- [end:limits] -->
318
320
 
319
- Detailed known limits of 0.8.2: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
321
+ Detailed known limits of 0.8.3: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
320
322
 
321
323
  ## Documentation
322
324
 
@@ -348,7 +350,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
348
350
  author = {Fazal, Rehan},
349
351
  title = {SqueakyCleanText},
350
352
  url = {https://github.com/rhnfzl/SqueakyCleanText},
351
- version = {0.8.1},
353
+ version = {0.8.3},
352
354
  year = {2026}
353
355
  }
354
356
  ```
@@ -1,7 +1,6 @@
1
1
  LICENSE
2
2
  README.md
3
3
  pyproject.toml
4
- setup.py
5
4
  SqueakyCleanText.egg-info/PKG-INFO
6
5
  SqueakyCleanText.egg-info/SOURCES.txt
7
6
  SqueakyCleanText.egg-info/dependency_links.txt
@@ -20,6 +19,7 @@ sct/model_candidates.py
20
19
  sct/ocr.py
21
20
  sct/presets.py
22
21
  sct/privacy.py
22
+ sct/py.typed
23
23
  sct/report_cli.py
24
24
  sct/risk.py
25
25
  sct/score.py
@@ -34,12 +34,15 @@ sct/integrations/hf.py
34
34
  sct/integrations/langchain.py
35
35
  sct/utils/__init__.py
36
36
  sct/utils/anonymization_map.py
37
+ sct/utils/atomic_file.py
37
38
  sct/utils/constants.py
38
39
  sct/utils/contact.py
39
40
  sct/utils/datetime.py
40
41
  sct/utils/gliclass_adapter.py
41
42
  sct/utils/gliner_adapter.py
42
43
  sct/utils/ner.py
44
+ sct/utils/ner_chunking.py
45
+ sct/utils/ner_spans.py
43
46
  sct/utils/normtext.py
44
47
  sct/utils/onnx_pipeline.py
45
48
  sct/utils/process_result.py
@@ -50,61 +53,31 @@ sct/utils/synthetic.py
50
53
  sct/utils/torch_pipeline.py
51
54
  tests/test_adapters.py
52
55
  tests/test_address_leftovers.py
56
+ tests/test_atomic_file.py
53
57
  tests/test_batching.py
58
+ tests/test_benchmark_adapters.py
59
+ tests/test_benchmark_loaders.py
54
60
  tests/test_benchmark_release.py
55
61
  tests/test_benchmarks_compare.py
62
+ tests/test_benchmarks_round2.py
63
+ tests/test_benchmarks_round2_tools.py
56
64
  tests/test_benchmarks_score.py
57
65
  tests/test_browser.py
58
66
  tests/test_chunking.py
59
67
  tests/test_cjk_width.py
68
+ tests/test_cleaner_lifecycle.py
60
69
  tests/test_cli.py
61
- tests/test_codex_benchmark_round1.py
62
- tests/test_codex_full_round1.py
63
- tests/test_codex_full_round13.py
64
- tests/test_codex_full_round14.py
65
- tests/test_codex_full_round15.py
66
- tests/test_codex_full_round16.py
67
- tests/test_codex_full_round17.py
68
- tests/test_codex_full_round18.py
69
- tests/test_codex_full_round19.py
70
- tests/test_codex_full_round2.py
71
- tests/test_codex_full_round20.py
72
- tests/test_codex_full_round21.py
73
- tests/test_codex_full_round22.py
74
- tests/test_codex_full_round23.py
75
- tests/test_codex_full_round24.py
76
- tests/test_codex_full_round25.py
77
- tests/test_codex_full_round27.py
78
- tests/test_codex_full_round28.py
79
- tests/test_codex_full_round29.py
80
- tests/test_codex_full_round3.py
81
- tests/test_codex_full_round30.py
82
- tests/test_codex_full_round31.py
83
- tests/test_codex_full_round32.py
84
- tests/test_codex_full_round35.py
85
- tests/test_codex_full_round37.py
86
- tests/test_codex_full_round38.py
87
- tests/test_codex_full_round4.py
88
- tests/test_codex_full_round5.py
89
- tests/test_codex_full_round6.py
90
- tests/test_codex_full_round7.py
91
- tests/test_codex_full_round8.py
92
- tests/test_codex_full_round9.py
93
70
  tests/test_config_state.py
94
71
  tests/test_contracts.py
72
+ tests/test_date_forms.py
95
73
  tests/test_dates_cjk_styled.py
96
74
  tests/test_dates_default.py
97
75
  tests/test_device_provenance.py
76
+ tests/test_email_forms.py
98
77
  tests/test_escaped_markup.py
99
78
  tests/test_evaluation.py
100
- tests/test_final_check_fixes.py
101
- tests/test_full_test_round2.py
79
+ tests/test_fork_safety.py
102
80
  tests/test_functional.py
103
- tests/test_fuzz_round1.py
104
- tests/test_fuzz_round2.py
105
- tests/test_fuzz_round3.py
106
- tests/test_fuzz_round5.py
107
- tests/test_fuzz_round6.py
108
81
  tests/test_fuzz_styled_pii.py
109
82
  tests/test_integrations_frameworks.py
110
83
  tests/test_integrations_hf.py
@@ -117,28 +90,41 @@ tests/test_mcp_server.py
117
90
  tests/test_model_benchmarks.py
118
91
  tests/test_multilingual_eval_set.py
119
92
  tests/test_ner_sees_tags.py
93
+ tests/test_ner_span_repair.py
94
+ tests/test_ner_spans.py
95
+ tests/test_ner_windows.py
120
96
  tests/test_number_identifiers.py
121
97
  tests/test_ocr.py
122
98
  tests/test_onnx_concurrency.py
123
99
  tests/test_onnx_external_data.py
100
+ tests/test_onnx_model_cache.py
101
+ tests/test_phone_forms.py
102
+ tests/test_pipeline_text_steps.py
103
+ tests/test_predicted_spans_monotone.py
104
+ tests/test_presidio_tags_and_fakes.py
105
+ tests/test_presidio_type_names.py
124
106
  tests/test_privacy.py
107
+ tests/test_prose_punctuation_kept.py
125
108
  tests/test_public_view.py
126
109
  tests/test_regex_boundaries.py
127
110
  tests/test_regex_counts.py
111
+ tests/test_repo_scripts.py
128
112
  tests/test_report_cli.py
129
- tests/test_review_round1.py
130
- tests/test_review_round2.py
131
- tests/test_review_round3.py
132
- tests/test_review_round4.py
113
+ tests/test_reversible_markers.py
133
114
  tests/test_risk.py
134
115
  tests/test_score.py
135
116
  tests/test_sct.py
136
117
  tests/test_security_leftovers.py
137
118
  tests/test_stream_guard_shared_view.py
119
+ tests/test_stream_keeps_values_whole.py
138
120
  tests/test_streaming.py
139
121
  tests/test_streaming_split_stability.py
140
122
  tests/test_styled_pii.py
123
+ tests/test_surface_contracts.py
124
+ tests/test_synthetic_determinism.py
141
125
  tests/test_synthetic_per_document.py
126
+ tests/test_torch_ner_backend.py
142
127
  tests/test_training_preset.py
143
128
  tests/test_type_names.py
129
+ tests/test_url_forms.py
144
130
  tests/test_user_terms.py
@@ -11,10 +11,7 @@ presidio_anonymizer>=2.2.360
11
11
  regex>=2024.11.6
12
12
 
13
13
  [all-ner]
14
- torch>=2.6.0
15
- transformers>=4.48
16
- gliner<1.0,>=0.2.25
17
- gliner2<2.0,>=1.0
14
+ SqueakyCleanText[gliner,gliner2,torch]
18
15
 
19
16
  [classify]
20
17
  gliclass>=0.1.16
@@ -58,8 +55,15 @@ fastmcp>=4.0.5
58
55
  pillow>=12.3.0
59
56
  pytesseract>=0.3.13
60
57
 
58
+ [parquet]
59
+ pyarrow>=17
60
+
61
61
  [presidio]
62
62
  presidio-analyzer>=2.2.360
63
+ gliner<1.0,>=0.2.25
64
+
65
+ [quantize]
66
+ onnx>=1.19
63
67
 
64
68
  [score]
65
69
  datasketch>=1.6
@@ -73,4 +77,4 @@ pytest-cov>=6.0
73
77
 
74
78
  [torch]
75
79
  torch>=2.6.0
76
- transformers>=4.48
80
+ transformers>=5.0
@@ -66,7 +66,7 @@ fuzzy = [
66
66
  ]
67
67
  torch = [
68
68
  "torch>=2.6.0",
69
- "transformers>=4.48",
69
+ "transformers>=5.0",
70
70
  ]
71
71
  gliner = [
72
72
  "gliner>=0.2.25,<1.0",
@@ -79,15 +79,16 @@ synthetic = [
79
79
  ]
80
80
  presidio = [
81
81
  "presidio-analyzer>=2.2.360",
82
+ "gliner>=0.2.25,<1.0",
82
83
  ]
83
84
  classify = [
84
85
  "gliclass>=0.1.16",
85
86
  ]
86
87
  all-ner = [
87
- "torch>=2.6.0",
88
- "transformers>=4.48",
89
- "gliner>=0.2.25,<1.0",
90
- "gliner2>=1.0,<2.0",
88
+ "SqueakyCleanText[torch,gliner,gliner2]",
89
+ ]
90
+ quantize = [
91
+ "onnx>=1.19",
91
92
  ]
92
93
  dev = [
93
94
  "hypothesis>=6.130",
@@ -107,6 +108,9 @@ mcp = [
107
108
  score = [
108
109
  "datasketch>=1.6",
109
110
  ]
111
+ parquet = [
112
+ "pyarrow>=17",
113
+ ]
110
114
  datasets = [
111
115
  "datasets>=3.0",
112
116
  ]
@@ -131,12 +135,21 @@ version = {attr = "sct.__version__"}
131
135
  include = ["sct*"]
132
136
  exclude = ["tests*"]
133
137
 
138
+ [tool.setuptools.package-data]
139
+ sct = ["py.typed"]
140
+
134
141
  [tool.pytest.ini_options]
135
142
  timeout = 1200
143
+ markers = ["needs_model: loads a real Hugging Face model"]
136
144
 
137
145
  [tool.ruff]
138
146
  line-length = 127
139
147
 
140
148
  [tool.ruff.lint]
141
- select = ["E", "F", "W", "S"]
149
+ select = ["E", "F", "W", "S", "UP"]
142
150
  ignore = ["S101"]
151
+
152
+ [tool.mypy]
153
+ files = ["sct"]
154
+ ignore_missing_imports = true
155
+ check_untyped_defs = true
@@ -11,6 +11,7 @@ from sct.privacy import (
11
11
  TokenStore,
12
12
  )
13
13
  from sct.sct import CustomPatternTimeout, TextCleaner
14
+ from sct.utils.ner import ModelLoadError
14
15
  from sct.presets import PRESETS, preset_config
15
16
  from sct.adapters import (
16
17
  DataFrameResult,
@@ -50,9 +51,9 @@ from sct.streaming import (
50
51
  from sct.utils.anonymization_map import AnonymizationMap, MapEntry
51
52
  from sct.utils.process_result import ProcessResult
52
53
 
53
- __version__ = "0.8.2"
54
+ __version__ = "0.8.3"
54
55
  __all__ = [
55
- "TextCleaner", "TextCleanerConfig", "CustomPatternTimeout", "PRESETS", "preset_config",
56
+ "TextCleaner", "TextCleanerConfig", "CustomPatternTimeout", "ModelLoadError", "PRESETS", "preset_config",
56
57
  "PII_LABELS", "PII_LABEL_MAP",
57
58
  "AnonymizationMap", "MapEntry", "ProcessResult",
58
59
  "DetectedEntity", "EntityFinding", "EntityRule",