SqueakyCleanText 0.8.3__tar.gz → 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/PKG-INFO +21 -5
  2. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/README.md +20 -4
  3. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/SqueakyCleanText.egg-info/PKG-INFO +21 -5
  4. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/SqueakyCleanText.egg-info/SOURCES.txt +5 -0
  5. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/__init__.py +1 -1
  6. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/config.py +31 -132
  7. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/score.py +5 -3
  8. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/sct.py +49 -71
  9. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/streaming.py +20 -37
  10. squeakycleantext-0.9.0/sct/utils/markers.py +7 -0
  11. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/ner.py +21 -101
  12. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/ner_chunking.py +74 -1
  13. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/special.py +2 -3
  14. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_benchmark_adapters.py +20 -20
  15. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_benchmark_release.py +3 -1
  16. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_benchmarks_compare.py +69 -40
  17. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_benchmarks_round2_tools.py +10 -21
  18. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_chunking.py +3 -7
  19. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_cjk_width.py +0 -10
  20. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_cleaner_lifecycle.py +0 -17
  21. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_config_state.py +2 -33
  22. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_contracts.py +2 -15
  23. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_device_provenance.py +2 -8
  24. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_functional.py +0 -11
  25. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_keep_scripts.py +0 -7
  26. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_language_all.py +0 -21
  27. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_ner_sees_tags.py +0 -11
  28. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_ner_span_repair.py +4 -38
  29. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_ner_windows.py +28 -0
  30. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_number_identifiers.py +2 -26
  31. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_onnx_concurrency.py +0 -8
  32. squeakycleantext-0.9.0/tests/test_pii_social_cards.py +55 -0
  33. squeakycleantext-0.9.0/tests/test_plot_pii_comparison.py +62 -0
  34. squeakycleantext-0.9.0/tests/test_plot_version_comparison.py +36 -0
  35. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_presidio_tags_and_fakes.py +10 -0
  36. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_privacy.py +1 -1
  37. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_regex_boundaries.py +0 -26
  38. squeakycleantext-0.9.0/tests/test_release_comparison_watcher.py +74 -0
  39. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_repo_scripts.py +31 -0
  40. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_sct.py +1 -52
  41. squeakycleantext-0.9.0/tests/test_streaming.py +145 -0
  42. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_styled_pii.py +0 -10
  43. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_synthetic_per_document.py +1 -27
  44. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_training_preset.py +1 -8
  45. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_type_names.py +1 -10
  46. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_user_terms.py +1 -18
  47. squeakycleantext-0.8.3/tests/test_streaming.py +0 -77
  48. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/LICENSE +0 -0
  49. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/SqueakyCleanText.egg-info/dependency_links.txt +0 -0
  50. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/SqueakyCleanText.egg-info/entry_points.txt +0 -0
  51. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/SqueakyCleanText.egg-info/requires.txt +0 -0
  52. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/SqueakyCleanText.egg-info/top_level.txt +0 -0
  53. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/pyproject.toml +0 -0
  54. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/adapters.py +0 -0
  55. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/browser.py +0 -0
  56. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/cli.py +0 -0
  57. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/evaluation.py +0 -0
  58. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/integrations/__init__.py +0 -0
  59. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/integrations/_runtime.py +0 -0
  60. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/integrations/datatrove.py +0 -0
  61. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/integrations/haystack.py +0 -0
  62. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/integrations/hf.py +0 -0
  63. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/integrations/langchain.py +0 -0
  64. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/mcp_server.py +0 -0
  65. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/model_benchmark.py +0 -0
  66. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/model_candidates.py +0 -0
  67. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/ocr.py +0 -0
  68. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/presets.py +0 -0
  69. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/privacy.py +0 -0
  70. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/py.typed +0 -0
  71. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/report_cli.py +0 -0
  72. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/risk.py +0 -0
  73. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/signals.py +0 -0
  74. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/__init__.py +0 -0
  75. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/anonymization_map.py +0 -0
  76. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/atomic_file.py +0 -0
  77. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/constants.py +0 -0
  78. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/contact.py +0 -0
  79. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/datetime.py +0 -0
  80. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/gliclass_adapter.py +0 -0
  81. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/gliner_adapter.py +0 -0
  82. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/ner_spans.py +0 -0
  83. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/normtext.py +0 -0
  84. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/onnx_pipeline.py +0 -0
  85. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/process_result.py +0 -0
  86. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/resources.py +0 -0
  87. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/stopwords.py +0 -0
  88. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/synthetic.py +0 -0
  89. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/torch_pipeline.py +0 -0
  90. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/setup.cfg +0 -0
  91. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_adapters.py +0 -0
  92. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_address_leftovers.py +0 -0
  93. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_atomic_file.py +0 -0
  94. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_batching.py +0 -0
  95. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_benchmark_loaders.py +0 -0
  96. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_benchmarks_round2.py +0 -0
  97. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_benchmarks_score.py +0 -0
  98. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_browser.py +0 -0
  99. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_cli.py +0 -0
  100. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_date_forms.py +0 -0
  101. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_dates_cjk_styled.py +0 -0
  102. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_dates_default.py +0 -0
  103. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_email_forms.py +0 -0
  104. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_escaped_markup.py +0 -0
  105. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_evaluation.py +0 -0
  106. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_fork_safety.py +0 -0
  107. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_fuzz_styled_pii.py +0 -0
  108. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_integrations_frameworks.py +0 -0
  109. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_integrations_hf.py +0 -0
  110. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_integrations_runtime.py +0 -0
  111. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_linux_workers.py +0 -0
  112. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_mcp_path.py +0 -0
  113. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_mcp_server.py +0 -0
  114. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_model_benchmarks.py +0 -0
  115. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_multilingual_eval_set.py +0 -0
  116. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_ner_spans.py +0 -0
  117. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_ocr.py +0 -0
  118. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_onnx_external_data.py +0 -0
  119. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_onnx_model_cache.py +0 -0
  120. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_phone_forms.py +0 -0
  121. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_pipeline_text_steps.py +0 -0
  122. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_predicted_spans_monotone.py +0 -0
  123. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_presidio_type_names.py +0 -0
  124. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_prose_punctuation_kept.py +0 -0
  125. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_public_view.py +0 -0
  126. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_regex_counts.py +0 -0
  127. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_report_cli.py +0 -0
  128. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_reversible_markers.py +0 -0
  129. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_risk.py +0 -0
  130. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_score.py +0 -0
  131. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_security_leftovers.py +0 -0
  132. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_stream_guard_shared_view.py +0 -0
  133. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_stream_keeps_values_whole.py +0 -0
  134. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_streaming_split_stability.py +0 -0
  135. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_surface_contracts.py +0 -0
  136. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_synthetic_determinism.py +0 -0
  137. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_torch_ner_backend.py +0 -0
  138. {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_url_forms.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SqueakyCleanText
3
- Version: 0.8.3
3
+ Version: 0.9.0
4
4
  Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
5
5
  Author: Rehan Fazal
6
6
  License-Expression: MIT
@@ -96,9 +96,11 @@ Dynamic: license-file
96
96
  [![Python Versions](https://img.shields.io/badge/Python-3.12%20|%203.13%20|%203.14-blue)](https://pypi.org/project/squeakycleantext/)
97
97
  [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/rhnfzl/SqueakyCleanText/blob/main/LICENSE)
98
98
 
99
- **Clean, privacy-safe text datasets for LLM training, fine-tuning and RAG, in 75 languages.**
99
+ **Stop hunting for personal details by hand. Mask detected names and contact details before sharing your text dataset.**
100
100
 
101
- SqueakyCleanText (SCT) is a text processing pipeline for datasets. A JSONL file, a Hugging Face dataset or a DataTrove job goes in. Cleaned text comes out, plus `sct_*` columns and a data card that describe each record without storing personal data. It runs on CPU or GPU.
101
+ SqueakyCleanText helps people prepare text datasets for sharing, LLM training, fine-tuning or RAG, with language detection in 75 languages.
102
+ It masks the personal details it finds and repairs broken text.
103
+ Works with JSONL files, Hugging Face datasets and DataTrove jobs, on CPU or GPU.
102
104
 
103
105
  ```text
104
106
  in: I’m Anna Berg & my mail is anna@example.se. Order 4711 ships on 2024-05-01.
@@ -110,6 +112,8 @@ out: Bel <PHONE> voor <PERSON>.
110
112
  sct_language=DUTCH sct_pii_count=2 sct_pii_types=['PERSON', 'PHONE']
111
113
  ```
112
114
 
115
+ Review cleaned text before sharing sensitive data; some personal details can be missed.
116
+
113
117
  Documentation: <https://rhnfzl.github.io/SqueakyCleanText/>
114
118
 
115
119
  ## What it does
@@ -252,6 +256,9 @@ The framework adapters use the `training` preset unless you pass a config, so th
252
256
 
253
257
  Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CONFIG.md)). The diagrams are built by [scripts/diagrams/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/scripts/diagrams); open the `.excalidraw` files in [resources/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/resources) to view them in Excalidraw.
254
258
 
259
+ From 0.9.0, configure cleaners with `TextCleanerConfig` and set document concurrency with `max_workers`.
260
+ Deprecated config globals, `ner_models_list`, the worker `batch_size` alias and cleaner helper properties are removed.
261
+
255
262
  <!-- --8<-- [start:results] -->
256
263
  ## What you get back
257
264
 
@@ -296,6 +303,15 @@ NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cl
296
303
 
297
304
  Measured on open datasets with [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks).
298
305
 
306
+ The [SCT v0.8.2 comparison with other PII tools](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-04-visuals/README.md)
307
+ shows paired recall, precision and exposure results, with uncertainty intervals.
308
+ It measures PII configurations rather than the whole processing pipeline. The tested version is v0.8.2, not v0.8.3.
309
+
310
+ The [released v0.7.1 versus v0.8.3 comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-0.7.1-0.8.3/README.md)
311
+ includes all 12 PII/long-document datasets, repeated CPU speed runs, and charts.
312
+ Strict span F1 improves on every dataset. Unsupported-language false answers and TAB direct-identifier coverage worsen.
313
+ Runtime versions differ, so the speed results do not isolate SCT code changes.
314
+
299
315
  - **Cleaning** (FineWeb-2 with injected noise, default config): 100% of HTML, 99.6% of mojibake and 100% of URLs removed.
300
316
  - **Language detection** (WiLI): 95.1% accuracy over 7,200 documents.
301
317
  - **PII** (12 open PII datasets, about 40 languages): the 0.8.1 release branch leaked fewer values than 0.8.0 on every dataset, for example 18.1% to 8.2% on REDACT ([comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/comparison-0.8.0-unreleased.md), pre-release commit `2078e8c4`).
@@ -318,7 +334,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
318
334
  - **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
319
335
  <!-- --8<-- [end:limits] -->
320
336
 
321
- Detailed known limits of 0.8.3: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
337
+ Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
322
338
 
323
339
  ## Documentation
324
340
 
@@ -350,7 +366,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
350
366
  author = {Fazal, Rehan},
351
367
  title = {SqueakyCleanText},
352
368
  url = {https://github.com/rhnfzl/SqueakyCleanText},
353
- version = {0.8.3},
369
+ version = {0.9.0},
354
370
  year = {2026}
355
371
  }
356
372
  ```
@@ -5,9 +5,11 @@
5
5
  [![Python Versions](https://img.shields.io/badge/Python-3.12%20|%203.13%20|%203.14-blue)](https://pypi.org/project/squeakycleantext/)
6
6
  [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/rhnfzl/SqueakyCleanText/blob/main/LICENSE)
7
7
 
8
- **Clean, privacy-safe text datasets for LLM training, fine-tuning and RAG, in 75 languages.**
8
+ **Stop hunting for personal details by hand. Mask detected names and contact details before sharing your text dataset.**
9
9
 
10
- SqueakyCleanText (SCT) is a text processing pipeline for datasets. A JSONL file, a Hugging Face dataset or a DataTrove job goes in. Cleaned text comes out, plus `sct_*` columns and a data card that describe each record without storing personal data. It runs on CPU or GPU.
10
+ SqueakyCleanText helps people prepare text datasets for sharing, LLM training, fine-tuning or RAG, with language detection in 75 languages.
11
+ It masks the personal details it finds and repairs broken text.
12
+ Works with JSONL files, Hugging Face datasets and DataTrove jobs, on CPU or GPU.
11
13
 
12
14
  ```text
13
15
  in: I’m Anna Berg &amp; my mail is anna@example.se. Order 4711 ships on 2024-05-01.
@@ -19,6 +21,8 @@ out: Bel <PHONE> voor <PERSON>.
19
21
  sct_language=DUTCH sct_pii_count=2 sct_pii_types=['PERSON', 'PHONE']
20
22
  ```
21
23
 
24
+ Review cleaned text before sharing sensitive data; some personal details can be missed.
25
+
22
26
  Documentation: <https://rhnfzl.github.io/SqueakyCleanText/>
23
27
 
24
28
  ## What it does
@@ -161,6 +165,9 @@ The framework adapters use the `training` preset unless you pass a config, so th
161
165
 
162
166
  Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CONFIG.md)). The diagrams are built by [scripts/diagrams/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/scripts/diagrams); open the `.excalidraw` files in [resources/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/resources) to view them in Excalidraw.
163
167
 
168
+ From 0.9.0, configure cleaners with `TextCleanerConfig` and set document concurrency with `max_workers`.
169
+ Deprecated config globals, `ner_models_list`, the worker `batch_size` alias and cleaner helper properties are removed.
170
+
164
171
  <!-- --8<-- [start:results] -->
165
172
  ## What you get back
166
173
 
@@ -205,6 +212,15 @@ NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cl
205
212
 
206
213
  Measured on open datasets with [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks).
207
214
 
215
+ The [SCT v0.8.2 comparison with other PII tools](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-04-visuals/README.md)
216
+ shows paired recall, precision and exposure results, with uncertainty intervals.
217
+ It measures PII configurations rather than the whole processing pipeline. The tested version is v0.8.2, not v0.8.3.
218
+
219
+ The [released v0.7.1 versus v0.8.3 comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-0.7.1-0.8.3/README.md)
220
+ includes all 12 PII/long-document datasets, repeated CPU speed runs, and charts.
221
+ Strict span F1 improves on every dataset. Unsupported-language false answers and TAB direct-identifier coverage worsen.
222
+ Runtime versions differ, so the speed results do not isolate SCT code changes.
223
+
208
224
  - **Cleaning** (FineWeb-2 with injected noise, default config): 100% of HTML, 99.6% of mojibake and 100% of URLs removed.
209
225
  - **Language detection** (WiLI): 95.1% accuracy over 7,200 documents.
210
226
  - **PII** (12 open PII datasets, about 40 languages): the 0.8.1 release branch leaked fewer values than 0.8.0 on every dataset, for example 18.1% to 8.2% on REDACT ([comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/comparison-0.8.0-unreleased.md), pre-release commit `2078e8c4`).
@@ -227,7 +243,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
227
243
  - **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
228
244
  <!-- --8<-- [end:limits] -->
229
245
 
230
- Detailed known limits of 0.8.3: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
246
+ Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
231
247
 
232
248
  ## Documentation
233
249
 
@@ -259,7 +275,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
259
275
  author = {Fazal, Rehan},
260
276
  title = {SqueakyCleanText},
261
277
  url = {https://github.com/rhnfzl/SqueakyCleanText},
262
- version = {0.8.3},
278
+ version = {0.9.0},
263
279
  year = {2026}
264
280
  }
265
281
  ```
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SqueakyCleanText
3
- Version: 0.8.3
3
+ Version: 0.9.0
4
4
  Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
5
5
  Author: Rehan Fazal
6
6
  License-Expression: MIT
@@ -96,9 +96,11 @@ Dynamic: license-file
96
96
  [![Python Versions](https://img.shields.io/badge/Python-3.12%20|%203.13%20|%203.14-blue)](https://pypi.org/project/squeakycleantext/)
97
97
  [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/rhnfzl/SqueakyCleanText/blob/main/LICENSE)
98
98
 
99
- **Clean, privacy-safe text datasets for LLM training, fine-tuning and RAG, in 75 languages.**
99
+ **Stop hunting for personal details by hand. Mask detected names and contact details before sharing your text dataset.**
100
100
 
101
- SqueakyCleanText (SCT) is a text processing pipeline for datasets. A JSONL file, a Hugging Face dataset or a DataTrove job goes in. Cleaned text comes out, plus `sct_*` columns and a data card that describe each record without storing personal data. It runs on CPU or GPU.
101
+ SqueakyCleanText helps people prepare text datasets for sharing, LLM training, fine-tuning or RAG, with language detection in 75 languages.
102
+ It masks the personal details it finds and repairs broken text.
103
+ Works with JSONL files, Hugging Face datasets and DataTrove jobs, on CPU or GPU.
102
104
 
103
105
  ```text
104
106
  in: I’m Anna Berg &amp; my mail is anna@example.se. Order 4711 ships on 2024-05-01.
@@ -110,6 +112,8 @@ out: Bel <PHONE> voor <PERSON>.
110
112
  sct_language=DUTCH sct_pii_count=2 sct_pii_types=['PERSON', 'PHONE']
111
113
  ```
112
114
 
115
+ Review cleaned text before sharing sensitive data; some personal details can be missed.
116
+
113
117
  Documentation: <https://rhnfzl.github.io/SqueakyCleanText/>
114
118
 
115
119
  ## What it does
@@ -252,6 +256,9 @@ The framework adapters use the `training` preset unless you pass a config, so th
252
256
 
253
257
  Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CONFIG.md)). The diagrams are built by [scripts/diagrams/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/scripts/diagrams); open the `.excalidraw` files in [resources/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/resources) to view them in Excalidraw.
254
258
 
259
+ From 0.9.0, configure cleaners with `TextCleanerConfig` and set document concurrency with `max_workers`.
260
+ Deprecated config globals, `ner_models_list`, the worker `batch_size` alias and cleaner helper properties are removed.
261
+
255
262
  <!-- --8<-- [start:results] -->
256
263
  ## What you get back
257
264
 
@@ -296,6 +303,15 @@ NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cl
296
303
 
297
304
  Measured on open datasets with [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks).
298
305
 
306
+ The [SCT v0.8.2 comparison with other PII tools](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-04-visuals/README.md)
307
+ shows paired recall, precision and exposure results, with uncertainty intervals.
308
+ It measures PII configurations rather than the whole processing pipeline. The tested version is v0.8.2, not v0.8.3.
309
+
310
+ The [released v0.7.1 versus v0.8.3 comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-0.7.1-0.8.3/README.md)
311
+ includes all 12 PII/long-document datasets, repeated CPU speed runs, and charts.
312
+ Strict span F1 improves on every dataset. Unsupported-language false answers and TAB direct-identifier coverage worsen.
313
+ Runtime versions differ, so the speed results do not isolate SCT code changes.
314
+
299
315
  - **Cleaning** (FineWeb-2 with injected noise, default config): 100% of HTML, 99.6% of mojibake and 100% of URLs removed.
300
316
  - **Language detection** (WiLI): 95.1% accuracy over 7,200 documents.
301
317
  - **PII** (12 open PII datasets, about 40 languages): the 0.8.1 release branch leaked fewer values than 0.8.0 on every dataset, for example 18.1% to 8.2% on REDACT ([comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/comparison-0.8.0-unreleased.md), pre-release commit `2078e8c4`).
@@ -318,7 +334,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
318
334
  - **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
319
335
  <!-- --8<-- [end:limits] -->
320
336
 
321
- Detailed known limits of 0.8.3: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
337
+ Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
322
338
 
323
339
  ## Documentation
324
340
 
@@ -350,7 +366,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
350
366
  author = {Fazal, Rehan},
351
367
  title = {SqueakyCleanText},
352
368
  url = {https://github.com/rhnfzl/SqueakyCleanText},
353
- version = {0.8.3},
369
+ version = {0.9.0},
354
370
  year = {2026}
355
371
  }
356
372
  ```
@@ -40,6 +40,7 @@ sct/utils/contact.py
40
40
  sct/utils/datetime.py
41
41
  sct/utils/gliclass_adapter.py
42
42
  sct/utils/gliner_adapter.py
43
+ sct/utils/markers.py
43
44
  sct/utils/ner.py
44
45
  sct/utils/ner_chunking.py
45
46
  sct/utils/ner_spans.py
@@ -99,7 +100,10 @@ tests/test_onnx_concurrency.py
99
100
  tests/test_onnx_external_data.py
100
101
  tests/test_onnx_model_cache.py
101
102
  tests/test_phone_forms.py
103
+ tests/test_pii_social_cards.py
102
104
  tests/test_pipeline_text_steps.py
105
+ tests/test_plot_pii_comparison.py
106
+ tests/test_plot_version_comparison.py
103
107
  tests/test_predicted_spans_monotone.py
104
108
  tests/test_presidio_tags_and_fakes.py
105
109
  tests/test_presidio_type_names.py
@@ -108,6 +112,7 @@ tests/test_prose_punctuation_kept.py
108
112
  tests/test_public_view.py
109
113
  tests/test_regex_boundaries.py
110
114
  tests/test_regex_counts.py
115
+ tests/test_release_comparison_watcher.py
111
116
  tests/test_repo_scripts.py
112
117
  tests/test_report_cli.py
113
118
  tests/test_reversible_markers.py
@@ -51,7 +51,7 @@ from sct.streaming import (
51
51
  from sct.utils.anonymization_map import AnonymizationMap, MapEntry
52
52
  from sct.utils.process_result import ProcessResult
53
53
 
54
- __version__ = "0.8.3"
54
+ __version__ = "0.9.0"
55
55
  __all__ = [
56
56
  "TextCleaner", "TextCleanerConfig", "CustomPatternTimeout", "ModelLoadError", "PRESETS", "preset_config",
57
57
  "PII_LABELS", "PII_LABEL_MAP",
@@ -1,28 +1,22 @@
1
1
  """
2
2
  Configuration for the SqueakyCleanText pipeline.
3
3
 
4
- New API (thread-safe, recommended):
4
+ Pass an immutable config to each cleaner:
5
5
  from sct.config import TextCleanerConfig
6
6
  cfg = TextCleanerConfig(check_ner_process=False)
7
7
  cleaner = TextCleaner(cfg)
8
-
9
- Legacy API (backward compatible):
10
- from sct import config
11
- config.CHECK_NER_PROCESS = False
12
- cleaner = TextCleaner()
13
8
  """
14
9
 
15
10
  import re
16
- import sys
17
- import warnings
18
11
  from collections.abc import Iterable
19
- from dataclasses import dataclass, fields
12
+ from dataclasses import dataclass
20
13
  from datetime import date
21
14
  from types import MappingProxyType
22
15
  from typing import Any, Literal, get_args
23
16
  from collections.abc import Mapping
24
17
 
25
18
  from sct.utils.resources import DEFAULT_LANGUAGES, resolve_language
19
+ from sct.utils.markers import LABEL_PATTERN
26
20
 
27
21
 
28
22
  LANG_KEYS = ('ENGLISH', 'DUTCH', 'GERMAN', 'SPANISH', 'MULTILINGUAL')
@@ -44,51 +38,27 @@ ONNX_BACKENDS = frozenset({'onnx', 'ensemble_onnx'})
44
38
  TORCH_BACKENDS = frozenset({'torch', 'ensemble_torch'})
45
39
  TOKEN_MODEL_BACKENDS = ONNX_BACKENDS | TORCH_BACKENDS
46
40
 
47
- DEFAULT_NER_MODELS: dict[str, str] = {
48
- 'ENGLISH': 'rhnfzl/xlm-roberta-large-conll03-english-onnx',
49
- 'DUTCH': 'rhnfzl/xlm-roberta-large-conll02-dutch-onnx',
50
- 'GERMAN': 'rhnfzl/xlm-roberta-large-conll03-german-onnx',
51
- 'SPANISH': 'rhnfzl/xlm-roberta-large-conll02-spanish-onnx',
52
- # FR/PT/IT: no dedicated ONNX model available; map to MULTILINGUAL fallback
53
- 'FRENCH': 'rhnfzl/wikineural-multilingual-ner-onnx',
54
- 'PORTUGUESE': 'rhnfzl/wikineural-multilingual-ner-onnx',
55
- 'ITALIAN': 'rhnfzl/wikineural-multilingual-ner-onnx',
56
- 'MULTILINGUAL': 'rhnfzl/wikineural-multilingual-ner-onnx',
57
- }
58
-
59
- DEFAULT_NER_REVISIONS: dict[str, str] = {
60
- 'ENGLISH': '2c10955af9538d6ff8a68316c5dbfcd0b968761e',
61
- 'DUTCH': '381e34ba4c222bc221e9fd02832f52a179ed840d',
62
- 'GERMAN': 'fe4aecfa6dc376244ef37da37e6df3b1b0911d71',
63
- 'SPANISH': 'ef52a7f08172203c763d51fe36660d36985bb388',
64
- 'FRENCH': '74191432b21c956b5b04b4778c0e92e569b1f80d',
65
- 'PORTUGUESE': '74191432b21c956b5b04b4778c0e92e569b1f80d',
66
- 'ITALIAN': '74191432b21c956b5b04b4778c0e92e569b1f80d',
67
- 'MULTILINGUAL': '74191432b21c956b5b04b4778c0e92e569b1f80d',
68
- }
69
-
70
- DEFAULT_TORCH_NER_MODELS: dict[str, str] = {
71
- 'ENGLISH': 'FacebookAI/xlm-roberta-large-finetuned-conll03-english',
72
- 'DUTCH': 'FacebookAI/xlm-roberta-large-finetuned-conll02-dutch',
73
- 'GERMAN': 'FacebookAI/xlm-roberta-large-finetuned-conll03-german',
74
- 'SPANISH': 'FacebookAI/xlm-roberta-large-finetuned-conll02-spanish',
75
- # FR/PT/IT: no dedicated Torch model available; map to MULTILINGUAL fallback
76
- 'FRENCH': 'Babelscape/wikineural-multilingual-ner',
77
- 'PORTUGUESE': 'Babelscape/wikineural-multilingual-ner',
78
- 'ITALIAN': 'Babelscape/wikineural-multilingual-ner',
79
- 'MULTILINGUAL': 'Babelscape/wikineural-multilingual-ner',
80
- }
81
-
82
- DEFAULT_TORCH_NER_REVISIONS: dict[str, str] = {
83
- 'ENGLISH': '18f95e9924f3f452df09cc90945073906ef18f1e',
84
- 'DUTCH': '630d5d48d08071704d9a2719b045082019b6ac12',
85
- 'GERMAN': '1fbcc7a00a69ce5ab754623154a8e9cc6ba868e2',
86
- 'SPANISH': 'a7c5f08c766adcbd6c22f343145fa65d38b3c1d5',
87
- 'FRENCH': 'bed6ee7a45d2827b6c90a4fd7983f0241ae0a5c1',
88
- 'PORTUGUESE': 'bed6ee7a45d2827b6c90a4fd7983f0241ae0a5c1',
89
- 'ITALIAN': 'bed6ee7a45d2827b6c90a4fd7983f0241ae0a5c1',
90
- 'MULTILINGUAL': 'bed6ee7a45d2827b6c90a4fd7983f0241ae0a5c1',
91
- }
41
+ _ONNX_CHECKPOINTS = (
42
+ (('ENGLISH',), 'rhnfzl/xlm-roberta-large-conll03-english-onnx', '2c10955af9538d6ff8a68316c5dbfcd0b968761e'),
43
+ (('DUTCH',), 'rhnfzl/xlm-roberta-large-conll02-dutch-onnx', '381e34ba4c222bc221e9fd02832f52a179ed840d'),
44
+ (('GERMAN',), 'rhnfzl/xlm-roberta-large-conll03-german-onnx', 'fe4aecfa6dc376244ef37da37e6df3b1b0911d71'),
45
+ (('SPANISH',), 'rhnfzl/xlm-roberta-large-conll02-spanish-onnx', 'ef52a7f08172203c763d51fe36660d36985bb388'),
46
+ (('FRENCH', 'PORTUGUESE', 'ITALIAN', 'MULTILINGUAL'),
47
+ 'rhnfzl/wikineural-multilingual-ner-onnx', '74191432b21c956b5b04b4778c0e92e569b1f80d'),
48
+ )
49
+ _TORCH_CHECKPOINTS = (
50
+ (('ENGLISH',), 'FacebookAI/xlm-roberta-large-finetuned-conll03-english', '18f95e9924f3f452df09cc90945073906ef18f1e'),
51
+ (('DUTCH',), 'FacebookAI/xlm-roberta-large-finetuned-conll02-dutch', '630d5d48d08071704d9a2719b045082019b6ac12'),
52
+ (('GERMAN',), 'FacebookAI/xlm-roberta-large-finetuned-conll03-german', '1fbcc7a00a69ce5ab754623154a8e9cc6ba868e2'),
53
+ (('SPANISH',), 'FacebookAI/xlm-roberta-large-finetuned-conll02-spanish', 'a7c5f08c766adcbd6c22f343145fa65d38b3c1d5'),
54
+ (('FRENCH', 'PORTUGUESE', 'ITALIAN', 'MULTILINGUAL'),
55
+ 'Babelscape/wikineural-multilingual-ner', 'bed6ee7a45d2827b6c90a4fd7983f0241ae0a5c1'),
56
+ )
57
+ DEFAULT_NER_MODELS = {key: model for keys, model, _ in _ONNX_CHECKPOINTS for key in keys}
58
+ DEFAULT_NER_REVISIONS = {key: revision for keys, _, revision in _ONNX_CHECKPOINTS for key in keys}
59
+ DEFAULT_TORCH_NER_MODELS = {key: model for keys, model, _ in _TORCH_CHECKPOINTS for key in keys}
60
+ DEFAULT_TORCH_NER_REVISIONS = {key: revision for keys, _, revision in _TORCH_CHECKPOINTS for key in keys}
61
+ _DEFAULT_CONFIG_NER_MODELS = {key: DEFAULT_NER_MODELS[key] for key in LANG_KEYS}
92
62
 
93
63
  # NER ensemble: ordered model keys to run per input language.
94
64
  # Keys must exist in the resolved ner_models dict.
@@ -168,11 +138,6 @@ MODERNBERT_NER_MODELS: dict[str, str] = {
168
138
  GLICLASS_DEFAULT_MODEL = 'knowledgator/gliclass-edge-v3.0' # 32.7M params, 131MB
169
139
  GLICLASS_DEFAULT_REVISION = 'df03993a2ed98e5e4a0d2dd7efbbd105abe874cf'
170
140
 
171
- # Placeholder constants for future multilingual NER models (blocked: models not yet published)
172
- OTTER_NER_MODELS: dict[str, str] = {} # arXiv:2601.06347 — checkpoints not on HF Hub
173
- MMBERT_NER_MODELS: dict[str, str] = {} # GLiNER bug #332 blocks mmBERT as backbone
174
-
175
-
176
141
  @dataclass(frozen=True)
177
142
  class TextCleanerConfig:
178
143
  """Thread-safe, immutable configuration for the text cleaning pipeline.
@@ -260,10 +225,6 @@ class TextCleanerConfig:
260
225
  # ensemble inference for unmapped languages.
261
226
  ner_ensemble_default_keys: tuple[str, ...] | None = None
262
227
 
263
- # Deprecated: positional tuple (English, Dutch, German, Spanish, Multilingual).
264
- # Use ner_models dict instead. Kept for backward compatibility.
265
- ner_models_list: tuple[str, ...] = tuple(DEFAULT_NER_MODELS[k] for k in LANG_KEYS)
266
-
267
228
  # Torch backend models (only used when ner_backend is 'torch' or 'ensemble_torch')
268
229
  torch_ner_models: Mapping[str, str] | None = None
269
230
  torch_ner_model_revisions: Mapping[str, str] | None = None
@@ -516,7 +477,7 @@ class TextCleanerConfig:
516
477
  object.__setattr__(self, 'gliner_label_map', _PII_MAPS[self.legacy_type_names])
517
478
 
518
479
 
519
- _USER_LABEL = re.compile(r'[A-Z][A-Z0-9_]*')
480
+ _USER_LABEL = re.compile(LABEL_PATTERN)
520
481
 
521
482
 
522
483
  def _user_terms(name, terms):
@@ -601,8 +562,6 @@ def _freeze_sequences(cfg):
601
562
  # Convert mutable collections to immutable for frozen safety
602
563
  if isinstance(cfg.positional_tags, list):
603
564
  object.__setattr__(cfg, 'positional_tags', tuple(cfg.positional_tags))
604
- if isinstance(cfg.ner_models_list, list):
605
- object.__setattr__(cfg, 'ner_models_list', tuple(cfg.ner_models_list))
606
565
  if isinstance(cfg.custom_pipeline_steps, list):
607
566
  object.__setattr__(cfg, 'custom_pipeline_steps', tuple(cfg.custom_pipeline_steps))
608
567
  for step in cfg.custom_pipeline_steps:
@@ -780,42 +739,21 @@ def _reconcile_torch_models(cfg):
780
739
 
781
740
 
782
741
  def _reconcile_ner_models(cfg):
783
- # Reconcile ner_models (dict, preferred) and ner_models_list (tuple, deprecated).
784
- # After this block, both fields are guaranteed populated.
785
- if cfg.ner_models is not None:
786
- # Dict API used — validate required keys, fill defaults, freeze
742
+ if cfg.ner_models is None:
743
+ merged = dict(_DEFAULT_CONFIG_NER_MODELS)
744
+ else:
787
745
  missing = REQUIRED_NER_KEYS - set(cfg.ner_models)
788
746
  if missing:
789
747
  raise ValueError(
790
748
  f"ner_models must include {sorted(REQUIRED_NER_KEYS)}, "
791
749
  f"missing: {sorted(missing)}"
792
750
  )
793
- # A map of exactly LANG_KEYS matching ner_models_list is a normalized
794
- # legacy/default config (replace(), copy): keep it exact. Decided by
795
- # content only, so a dict and a mapping proxy give the same config.
796
- derived = (
797
- set(cfg.ner_models) == set(LANG_KEYS)
798
- and tuple(cfg.ner_models[k] for k in LANG_KEYS) == cfg.ner_models_list
799
- )
751
+ # A copied default config keeps its four-language routing.
800
752
  merged = (
801
- dict(cfg.ner_models) if derived
753
+ dict(cfg.ner_models) if cfg.ner_models == _DEFAULT_CONFIG_NER_MODELS
802
754
  else {**DEFAULT_NER_MODELS, **cfg.ner_models}
803
755
  )
804
- object.__setattr__(cfg, 'ner_models', MappingProxyType(merged))
805
- object.__setattr__(
806
- cfg, 'ner_models_list',
807
- tuple(merged[k] for k in LANG_KEYS),
808
- )
809
- else:
810
- # No dict provided — derive from ner_models_list (may be default or custom)
811
- if cfg.ner_models_list != TextCleanerConfig.ner_models_list:
812
- warnings.warn(
813
- "ner_models_list is deprecated. Use ner_models dict instead.",
814
- DeprecationWarning,
815
- stacklevel=3,
816
- )
817
- models_dict = dict(zip(LANG_KEYS, cfg.ner_models_list))
818
- object.__setattr__(cfg, 'ner_models', MappingProxyType(models_dict))
756
+ object.__setattr__(cfg, 'ner_models', MappingProxyType(merged))
819
757
 
820
758
  pinned_revisions = {
821
759
  key: revision
@@ -890,12 +828,6 @@ def _compute_supported_languages(cfg):
890
828
  object.__setattr__(cfg, 'supported_languages', frozenset(langs))
891
829
 
892
830
 
893
- # Fields without a plain UPPER_CASE global: ner_models_list is fed by the NER_MODELS_LIST global through
894
- # ner_models, supported_languages is computed, and NER_ENSEMBLE_DEFAULT_KEYS is also the built-in fallback.
895
- _NO_PLAIN_GLOBAL = frozenset({'ner_models_list', 'supported_languages', 'ner_ensemble_default_keys'})
896
- _BUILTIN_ENSEMBLE_DEFAULT_KEYS = NER_ENSEMBLE_DEFAULT_KEYS
897
-
898
-
899
831
  def require_adapter_mode(cfg: TextCleanerConfig) -> None:
900
832
  """Refuse reversible mode on a surface that has no way to hand back each text's map."""
901
833
  if cfg.replacement_mode == "reversible":
@@ -903,36 +835,3 @@ def require_adapter_mode(cfg: TextCleanerConfig) -> None:
903
835
  "Adapters do not support reversible mode. "
904
836
  "Process each text directly and retain its token map."
905
837
  )
906
-
907
-
908
- def _config_from_module_globals() -> TextCleanerConfig:
909
- """Build a TextCleanerConfig from the current module-level variable state.
910
-
911
- Supports backward compatibility with the legacy config pattern.
912
- """
913
- m = sys.modules[__name__]
914
- kwargs: dict[str, Any] = {
915
- f.name: getattr(m, f.name.upper(), f.default)
916
- for f in fields(TextCleanerConfig)
917
- if f.name not in _NO_PLAIN_GLOBAL
918
- }
919
- kwargs['positional_tags'] = tuple(m.POSITIONAL_TAGS)
920
- if kwargs['ner_models'] is None:
921
- kwargs['ner_models'] = dict(zip(LANG_KEYS, m.NER_MODELS_LIST))
922
- if m.NER_ENSEMBLE_DEFAULT_KEYS != _BUILTIN_ENSEMBLE_DEFAULT_KEYS:
923
- kwargs['ner_ensemble_default_keys'] = m.NER_ENSEMBLE_DEFAULT_KEYS
924
- return TextCleanerConfig(**kwargs)
925
-
926
-
927
- # ---------------------------------------------------------------------------
928
- # Backward-compatible module-level variables: one per field, named in upper case, set to the field's
929
- # default. Users can still do: config.CHECK_NER_PROCESS = False
930
- # ---------------------------------------------------------------------------
931
- for _field in fields(TextCleanerConfig):
932
- if _field.name not in _NO_PLAIN_GLOBAL:
933
- globals()[_field.name.upper()] = _field.default
934
-
935
- # Lists, as they always were, so config.POSITIONAL_TAGS.append('X') keeps working.
936
- POSITIONAL_TAGS = list(TextCleanerConfig.positional_tags)
937
- # Order: English, Dutch, German, Spanish, Multilingual
938
- NER_MODELS_LIST = list(TextCleanerConfig.ner_models_list)
@@ -17,6 +17,8 @@ from collections.abc import Iterable, Sequence
17
17
 
18
18
  import regex
19
19
 
20
+ from sct.utils.markers import LABEL_PATTERN, PLACEHOLDER_PATTERN
21
+
20
22
  if TYPE_CHECKING:
21
23
  from sct.utils.process_result import ProcessResult
22
24
 
@@ -29,9 +31,9 @@ SCORE_A_COLUMNS = tuple(SCORE_A_TYPES)
29
31
  SCORE_B_COLUMNS = ("sct_quality", "sct_minhash")
30
32
 
31
33
  # Placeholders count as one token; reversible ``<PERSON_3>`` normalises to ``<PERSON>``.
32
- _PLACEHOLDER = regex.compile(r"<([A-Z][A-Z0-9_]*?)(?:_\d+)?>")
33
- _WORD_TOKEN = regex.compile(r"<[A-Z][A-Z0-9_]*>|\w+")
34
- _CHAR_TOKEN = regex.compile(r"<[A-Z][A-Z0-9_]*>|\S")
34
+ _PLACEHOLDER = regex.compile(rf"<({LABEL_PATTERN}?)(?:_\d+)?>")
35
+ _WORD_TOKEN = regex.compile(rf"{PLACEHOLDER_PATTERN}|\w+")
36
+ _CHAR_TOKEN = regex.compile(rf"{PLACEHOLDER_PATTERN}|\S")
35
37
  # Scripts written without spaces between words get character shingles.
36
38
  _NO_SPACE_SCRIPT = regex.compile(
37
39
  r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Thai}\p{Lao}\p{Khmer}\p{Myanmar}]"