SqueakyCleanText 0.8.3__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/PKG-INFO +21 -5
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/README.md +20 -4
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/SqueakyCleanText.egg-info/PKG-INFO +21 -5
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/SqueakyCleanText.egg-info/SOURCES.txt +5 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/__init__.py +1 -1
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/config.py +31 -132
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/score.py +5 -3
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/sct.py +49 -71
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/streaming.py +20 -37
- squeakycleantext-0.9.0/sct/utils/markers.py +7 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/ner.py +21 -101
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/ner_chunking.py +74 -1
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/special.py +2 -3
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_benchmark_adapters.py +20 -20
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_benchmark_release.py +3 -1
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_benchmarks_compare.py +69 -40
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_benchmarks_round2_tools.py +10 -21
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_chunking.py +3 -7
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_cjk_width.py +0 -10
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_cleaner_lifecycle.py +0 -17
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_config_state.py +2 -33
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_contracts.py +2 -15
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_device_provenance.py +2 -8
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_functional.py +0 -11
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_keep_scripts.py +0 -7
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_language_all.py +0 -21
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_ner_sees_tags.py +0 -11
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_ner_span_repair.py +4 -38
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_ner_windows.py +28 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_number_identifiers.py +2 -26
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_onnx_concurrency.py +0 -8
- squeakycleantext-0.9.0/tests/test_pii_social_cards.py +55 -0
- squeakycleantext-0.9.0/tests/test_plot_pii_comparison.py +62 -0
- squeakycleantext-0.9.0/tests/test_plot_version_comparison.py +36 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_presidio_tags_and_fakes.py +10 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_privacy.py +1 -1
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_regex_boundaries.py +0 -26
- squeakycleantext-0.9.0/tests/test_release_comparison_watcher.py +74 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_repo_scripts.py +31 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_sct.py +1 -52
- squeakycleantext-0.9.0/tests/test_streaming.py +145 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_styled_pii.py +0 -10
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_synthetic_per_document.py +1 -27
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_training_preset.py +1 -8
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_type_names.py +1 -10
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_user_terms.py +1 -18
- squeakycleantext-0.8.3/tests/test_streaming.py +0 -77
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/LICENSE +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/SqueakyCleanText.egg-info/dependency_links.txt +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/SqueakyCleanText.egg-info/entry_points.txt +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/SqueakyCleanText.egg-info/requires.txt +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/SqueakyCleanText.egg-info/top_level.txt +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/pyproject.toml +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/adapters.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/browser.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/cli.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/evaluation.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/integrations/__init__.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/integrations/_runtime.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/integrations/datatrove.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/integrations/haystack.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/integrations/hf.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/integrations/langchain.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/mcp_server.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/model_benchmark.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/model_candidates.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/ocr.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/presets.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/privacy.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/py.typed +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/report_cli.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/risk.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/signals.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/__init__.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/anonymization_map.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/atomic_file.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/constants.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/contact.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/datetime.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/gliclass_adapter.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/gliner_adapter.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/ner_spans.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/normtext.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/onnx_pipeline.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/process_result.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/resources.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/stopwords.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/synthetic.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/sct/utils/torch_pipeline.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/setup.cfg +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_adapters.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_address_leftovers.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_atomic_file.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_batching.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_benchmark_loaders.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_benchmarks_round2.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_benchmarks_score.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_browser.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_cli.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_date_forms.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_dates_cjk_styled.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_dates_default.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_email_forms.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_escaped_markup.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_evaluation.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_fork_safety.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_fuzz_styled_pii.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_integrations_frameworks.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_integrations_hf.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_integrations_runtime.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_linux_workers.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_mcp_path.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_mcp_server.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_model_benchmarks.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_multilingual_eval_set.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_ner_spans.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_ocr.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_onnx_external_data.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_onnx_model_cache.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_phone_forms.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_pipeline_text_steps.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_predicted_spans_monotone.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_presidio_type_names.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_prose_punctuation_kept.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_public_view.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_regex_counts.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_report_cli.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_reversible_markers.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_risk.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_score.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_security_leftovers.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_stream_guard_shared_view.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_stream_keeps_values_whole.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_streaming_split_stability.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_surface_contracts.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_synthetic_determinism.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_torch_ner_backend.py +0 -0
- {squeakycleantext-0.8.3 → squeakycleantext-0.9.0}/tests/test_url_forms.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SqueakyCleanText
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
|
|
5
5
|
Author: Rehan Fazal
|
|
6
6
|
License-Expression: MIT
|
|
@@ -96,9 +96,11 @@ Dynamic: license-file
|
|
|
96
96
|
[](https://pypi.org/project/squeakycleantext/)
|
|
97
97
|
[](https://github.com/rhnfzl/SqueakyCleanText/blob/main/LICENSE)
|
|
98
98
|
|
|
99
|
-
**
|
|
99
|
+
**Stop hunting for personal details by hand. Mask detected names and contact details before sharing your text dataset.**
|
|
100
100
|
|
|
101
|
-
SqueakyCleanText
|
|
101
|
+
SqueakyCleanText helps people prepare text datasets for sharing, LLM training, fine-tuning or RAG, with language detection in 75 languages.
|
|
102
|
+
It masks the personal details it finds and repairs broken text.
|
|
103
|
+
Works with JSONL files, Hugging Face datasets and DataTrove jobs, on CPU or GPU.
|
|
102
104
|
|
|
103
105
|
```text
|
|
104
106
|
in: I’m Anna Berg & my mail is anna@example.se. Order 4711 ships on 2024-05-01.
|
|
@@ -110,6 +112,8 @@ out: Bel <PHONE> voor <PERSON>.
|
|
|
110
112
|
sct_language=DUTCH sct_pii_count=2 sct_pii_types=['PERSON', 'PHONE']
|
|
111
113
|
```
|
|
112
114
|
|
|
115
|
+
Review cleaned text before sharing sensitive data; some personal details can be missed.
|
|
116
|
+
|
|
113
117
|
Documentation: <https://rhnfzl.github.io/SqueakyCleanText/>
|
|
114
118
|
|
|
115
119
|
## What it does
|
|
@@ -252,6 +256,9 @@ The framework adapters use the `training` preset unless you pass a config, so th
|
|
|
252
256
|
|
|
253
257
|
Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CONFIG.md)). The diagrams are built by [scripts/diagrams/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/scripts/diagrams); open the `.excalidraw` files in [resources/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/resources) to view them in Excalidraw.
|
|
254
258
|
|
|
259
|
+
From 0.9.0, configure cleaners with `TextCleanerConfig` and set document concurrency with `max_workers`.
|
|
260
|
+
Deprecated config globals, `ner_models_list`, the worker `batch_size` alias and cleaner helper properties are removed.
|
|
261
|
+
|
|
255
262
|
<!-- --8<-- [start:results] -->
|
|
256
263
|
## What you get back
|
|
257
264
|
|
|
@@ -296,6 +303,15 @@ NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cl
|
|
|
296
303
|
|
|
297
304
|
Measured on open datasets with [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks).
|
|
298
305
|
|
|
306
|
+
The [SCT v0.8.2 comparison with other PII tools](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-04-visuals/README.md)
|
|
307
|
+
shows paired recall, precision and exposure results, with uncertainty intervals.
|
|
308
|
+
It measures PII configurations rather than the whole processing pipeline. The tested version is v0.8.2, not v0.8.3.
|
|
309
|
+
|
|
310
|
+
The [released v0.7.1 versus v0.8.3 comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-0.7.1-0.8.3/README.md)
|
|
311
|
+
includes all 12 PII/long-document datasets, repeated CPU speed runs, and charts.
|
|
312
|
+
Strict span F1 improves on every dataset. Unsupported-language false answers and TAB direct-identifier coverage worsen.
|
|
313
|
+
Runtime versions differ, so the speed results do not isolate SCT code changes.
|
|
314
|
+
|
|
299
315
|
- **Cleaning** (FineWeb-2 with injected noise, default config): 100% of HTML, 99.6% of mojibake and 100% of URLs removed.
|
|
300
316
|
- **Language detection** (WiLI): 95.1% accuracy over 7,200 documents.
|
|
301
317
|
- **PII** (12 open PII datasets, about 40 languages): the 0.8.1 release branch leaked fewer values than 0.8.0 on every dataset, for example 18.1% to 8.2% on REDACT ([comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/comparison-0.8.0-unreleased.md), pre-release commit `2078e8c4`).
|
|
@@ -318,7 +334,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
|
|
|
318
334
|
- **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
|
|
319
335
|
<!-- --8<-- [end:limits] -->
|
|
320
336
|
|
|
321
|
-
Detailed known limits of 0.
|
|
337
|
+
Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
|
|
322
338
|
|
|
323
339
|
## Documentation
|
|
324
340
|
|
|
@@ -350,7 +366,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
|
|
|
350
366
|
author = {Fazal, Rehan},
|
|
351
367
|
title = {SqueakyCleanText},
|
|
352
368
|
url = {https://github.com/rhnfzl/SqueakyCleanText},
|
|
353
|
-
version = {0.
|
|
369
|
+
version = {0.9.0},
|
|
354
370
|
year = {2026}
|
|
355
371
|
}
|
|
356
372
|
```
|
|
@@ -5,9 +5,11 @@
|
|
|
5
5
|
[](https://pypi.org/project/squeakycleantext/)
|
|
6
6
|
[](https://github.com/rhnfzl/SqueakyCleanText/blob/main/LICENSE)
|
|
7
7
|
|
|
8
|
-
**
|
|
8
|
+
**Stop hunting for personal details by hand. Mask detected names and contact details before sharing your text dataset.**
|
|
9
9
|
|
|
10
|
-
SqueakyCleanText
|
|
10
|
+
SqueakyCleanText helps people prepare text datasets for sharing, LLM training, fine-tuning or RAG, with language detection in 75 languages.
|
|
11
|
+
It masks the personal details it finds and repairs broken text.
|
|
12
|
+
Works with JSONL files, Hugging Face datasets and DataTrove jobs, on CPU or GPU.
|
|
11
13
|
|
|
12
14
|
```text
|
|
13
15
|
in: I’m Anna Berg & my mail is anna@example.se. Order 4711 ships on 2024-05-01.
|
|
@@ -19,6 +21,8 @@ out: Bel <PHONE> voor <PERSON>.
|
|
|
19
21
|
sct_language=DUTCH sct_pii_count=2 sct_pii_types=['PERSON', 'PHONE']
|
|
20
22
|
```
|
|
21
23
|
|
|
24
|
+
Review cleaned text before sharing sensitive data; some personal details can be missed.
|
|
25
|
+
|
|
22
26
|
Documentation: <https://rhnfzl.github.io/SqueakyCleanText/>
|
|
23
27
|
|
|
24
28
|
## What it does
|
|
@@ -161,6 +165,9 @@ The framework adapters use the `training` preset unless you pass a config, so th
|
|
|
161
165
|
|
|
162
166
|
Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CONFIG.md)). The diagrams are built by [scripts/diagrams/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/scripts/diagrams); open the `.excalidraw` files in [resources/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/resources) to view them in Excalidraw.
|
|
163
167
|
|
|
168
|
+
From 0.9.0, configure cleaners with `TextCleanerConfig` and set document concurrency with `max_workers`.
|
|
169
|
+
Deprecated config globals, `ner_models_list`, the worker `batch_size` alias and cleaner helper properties are removed.
|
|
170
|
+
|
|
164
171
|
<!-- --8<-- [start:results] -->
|
|
165
172
|
## What you get back
|
|
166
173
|
|
|
@@ -205,6 +212,15 @@ NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cl
|
|
|
205
212
|
|
|
206
213
|
Measured on open datasets with [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks).
|
|
207
214
|
|
|
215
|
+
The [SCT v0.8.2 comparison with other PII tools](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-04-visuals/README.md)
|
|
216
|
+
shows paired recall, precision and exposure results, with uncertainty intervals.
|
|
217
|
+
It measures PII configurations rather than the whole processing pipeline. The tested version is v0.8.2, not v0.8.3.
|
|
218
|
+
|
|
219
|
+
The [released v0.7.1 versus v0.8.3 comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-0.7.1-0.8.3/README.md)
|
|
220
|
+
includes all 12 PII/long-document datasets, repeated CPU speed runs, and charts.
|
|
221
|
+
Strict span F1 improves on every dataset. Unsupported-language false answers and TAB direct-identifier coverage worsen.
|
|
222
|
+
Runtime versions differ, so the speed results do not isolate SCT code changes.
|
|
223
|
+
|
|
208
224
|
- **Cleaning** (FineWeb-2 with injected noise, default config): 100% of HTML, 99.6% of mojibake and 100% of URLs removed.
|
|
209
225
|
- **Language detection** (WiLI): 95.1% accuracy over 7,200 documents.
|
|
210
226
|
- **PII** (12 open PII datasets, about 40 languages): the 0.8.1 release branch leaked fewer values than 0.8.0 on every dataset, for example 18.1% to 8.2% on REDACT ([comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/comparison-0.8.0-unreleased.md), pre-release commit `2078e8c4`).
|
|
@@ -227,7 +243,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
|
|
|
227
243
|
- **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
|
|
228
244
|
<!-- --8<-- [end:limits] -->
|
|
229
245
|
|
|
230
|
-
Detailed known limits of 0.
|
|
246
|
+
Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
|
|
231
247
|
|
|
232
248
|
## Documentation
|
|
233
249
|
|
|
@@ -259,7 +275,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
|
|
|
259
275
|
author = {Fazal, Rehan},
|
|
260
276
|
title = {SqueakyCleanText},
|
|
261
277
|
url = {https://github.com/rhnfzl/SqueakyCleanText},
|
|
262
|
-
version = {0.
|
|
278
|
+
version = {0.9.0},
|
|
263
279
|
year = {2026}
|
|
264
280
|
}
|
|
265
281
|
```
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SqueakyCleanText
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
|
|
5
5
|
Author: Rehan Fazal
|
|
6
6
|
License-Expression: MIT
|
|
@@ -96,9 +96,11 @@ Dynamic: license-file
|
|
|
96
96
|
[](https://pypi.org/project/squeakycleantext/)
|
|
97
97
|
[](https://github.com/rhnfzl/SqueakyCleanText/blob/main/LICENSE)
|
|
98
98
|
|
|
99
|
-
**
|
|
99
|
+
**Stop hunting for personal details by hand. Mask detected names and contact details before sharing your text dataset.**
|
|
100
100
|
|
|
101
|
-
SqueakyCleanText
|
|
101
|
+
SqueakyCleanText helps people prepare text datasets for sharing, LLM training, fine-tuning or RAG, with language detection in 75 languages.
|
|
102
|
+
It masks the personal details it finds and repairs broken text.
|
|
103
|
+
Works with JSONL files, Hugging Face datasets and DataTrove jobs, on CPU or GPU.
|
|
102
104
|
|
|
103
105
|
```text
|
|
104
106
|
in: I’m Anna Berg & my mail is anna@example.se. Order 4711 ships on 2024-05-01.
|
|
@@ -110,6 +112,8 @@ out: Bel <PHONE> voor <PERSON>.
|
|
|
110
112
|
sct_language=DUTCH sct_pii_count=2 sct_pii_types=['PERSON', 'PHONE']
|
|
111
113
|
```
|
|
112
114
|
|
|
115
|
+
Review cleaned text before sharing sensitive data; some personal details can be missed.
|
|
116
|
+
|
|
113
117
|
Documentation: <https://rhnfzl.github.io/SqueakyCleanText/>
|
|
114
118
|
|
|
115
119
|
## What it does
|
|
@@ -252,6 +256,9 @@ The framework adapters use the `training` preset unless you pass a config, so th
|
|
|
252
256
|
|
|
253
257
|
Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CONFIG.md)). The diagrams are built by [scripts/diagrams/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/scripts/diagrams); open the `.excalidraw` files in [resources/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/resources) to view them in Excalidraw.
|
|
254
258
|
|
|
259
|
+
From 0.9.0, configure cleaners with `TextCleanerConfig` and set document concurrency with `max_workers`.
|
|
260
|
+
Deprecated config globals, `ner_models_list`, the worker `batch_size` alias and cleaner helper properties are removed.
|
|
261
|
+
|
|
255
262
|
<!-- --8<-- [start:results] -->
|
|
256
263
|
## What you get back
|
|
257
264
|
|
|
@@ -296,6 +303,15 @@ NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cl
|
|
|
296
303
|
|
|
297
304
|
Measured on open datasets with [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks).
|
|
298
305
|
|
|
306
|
+
The [SCT v0.8.2 comparison with other PII tools](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-04-visuals/README.md)
|
|
307
|
+
shows paired recall, precision and exposure results, with uncertainty intervals.
|
|
308
|
+
It measures PII configurations rather than the whole processing pipeline. The tested version is v0.8.2, not v0.8.3.
|
|
309
|
+
|
|
310
|
+
The [released v0.7.1 versus v0.8.3 comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-0.7.1-0.8.3/README.md)
|
|
311
|
+
includes all 12 PII/long-document datasets, repeated CPU speed runs, and charts.
|
|
312
|
+
Strict span F1 improves on every dataset. Unsupported-language false answers and TAB direct-identifier coverage worsen.
|
|
313
|
+
Runtime versions differ, so the speed results do not isolate SCT code changes.
|
|
314
|
+
|
|
299
315
|
- **Cleaning** (FineWeb-2 with injected noise, default config): 100% of HTML, 99.6% of mojibake and 100% of URLs removed.
|
|
300
316
|
- **Language detection** (WiLI): 95.1% accuracy over 7,200 documents.
|
|
301
317
|
- **PII** (12 open PII datasets, about 40 languages): the 0.8.1 release branch leaked fewer values than 0.8.0 on every dataset, for example 18.1% to 8.2% on REDACT ([comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/comparison-0.8.0-unreleased.md), pre-release commit `2078e8c4`).
|
|
@@ -318,7 +334,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
|
|
|
318
334
|
- **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
|
|
319
335
|
<!-- --8<-- [end:limits] -->
|
|
320
336
|
|
|
321
|
-
Detailed known limits of 0.
|
|
337
|
+
Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
|
|
322
338
|
|
|
323
339
|
## Documentation
|
|
324
340
|
|
|
@@ -350,7 +366,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
|
|
|
350
366
|
author = {Fazal, Rehan},
|
|
351
367
|
title = {SqueakyCleanText},
|
|
352
368
|
url = {https://github.com/rhnfzl/SqueakyCleanText},
|
|
353
|
-
version = {0.
|
|
369
|
+
version = {0.9.0},
|
|
354
370
|
year = {2026}
|
|
355
371
|
}
|
|
356
372
|
```
|
|
@@ -40,6 +40,7 @@ sct/utils/contact.py
|
|
|
40
40
|
sct/utils/datetime.py
|
|
41
41
|
sct/utils/gliclass_adapter.py
|
|
42
42
|
sct/utils/gliner_adapter.py
|
|
43
|
+
sct/utils/markers.py
|
|
43
44
|
sct/utils/ner.py
|
|
44
45
|
sct/utils/ner_chunking.py
|
|
45
46
|
sct/utils/ner_spans.py
|
|
@@ -99,7 +100,10 @@ tests/test_onnx_concurrency.py
|
|
|
99
100
|
tests/test_onnx_external_data.py
|
|
100
101
|
tests/test_onnx_model_cache.py
|
|
101
102
|
tests/test_phone_forms.py
|
|
103
|
+
tests/test_pii_social_cards.py
|
|
102
104
|
tests/test_pipeline_text_steps.py
|
|
105
|
+
tests/test_plot_pii_comparison.py
|
|
106
|
+
tests/test_plot_version_comparison.py
|
|
103
107
|
tests/test_predicted_spans_monotone.py
|
|
104
108
|
tests/test_presidio_tags_and_fakes.py
|
|
105
109
|
tests/test_presidio_type_names.py
|
|
@@ -108,6 +112,7 @@ tests/test_prose_punctuation_kept.py
|
|
|
108
112
|
tests/test_public_view.py
|
|
109
113
|
tests/test_regex_boundaries.py
|
|
110
114
|
tests/test_regex_counts.py
|
|
115
|
+
tests/test_release_comparison_watcher.py
|
|
111
116
|
tests/test_repo_scripts.py
|
|
112
117
|
tests/test_report_cli.py
|
|
113
118
|
tests/test_reversible_markers.py
|
|
@@ -51,7 +51,7 @@ from sct.streaming import (
|
|
|
51
51
|
from sct.utils.anonymization_map import AnonymizationMap, MapEntry
|
|
52
52
|
from sct.utils.process_result import ProcessResult
|
|
53
53
|
|
|
54
|
-
__version__ = "0.
|
|
54
|
+
__version__ = "0.9.0"
|
|
55
55
|
__all__ = [
|
|
56
56
|
"TextCleaner", "TextCleanerConfig", "CustomPatternTimeout", "ModelLoadError", "PRESETS", "preset_config",
|
|
57
57
|
"PII_LABELS", "PII_LABEL_MAP",
|
|
@@ -1,28 +1,22 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Configuration for the SqueakyCleanText pipeline.
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
Pass an immutable config to each cleaner:
|
|
5
5
|
from sct.config import TextCleanerConfig
|
|
6
6
|
cfg = TextCleanerConfig(check_ner_process=False)
|
|
7
7
|
cleaner = TextCleaner(cfg)
|
|
8
|
-
|
|
9
|
-
Legacy API (backward compatible):
|
|
10
|
-
from sct import config
|
|
11
|
-
config.CHECK_NER_PROCESS = False
|
|
12
|
-
cleaner = TextCleaner()
|
|
13
8
|
"""
|
|
14
9
|
|
|
15
10
|
import re
|
|
16
|
-
import sys
|
|
17
|
-
import warnings
|
|
18
11
|
from collections.abc import Iterable
|
|
19
|
-
from dataclasses import dataclass
|
|
12
|
+
from dataclasses import dataclass
|
|
20
13
|
from datetime import date
|
|
21
14
|
from types import MappingProxyType
|
|
22
15
|
from typing import Any, Literal, get_args
|
|
23
16
|
from collections.abc import Mapping
|
|
24
17
|
|
|
25
18
|
from sct.utils.resources import DEFAULT_LANGUAGES, resolve_language
|
|
19
|
+
from sct.utils.markers import LABEL_PATTERN
|
|
26
20
|
|
|
27
21
|
|
|
28
22
|
LANG_KEYS = ('ENGLISH', 'DUTCH', 'GERMAN', 'SPANISH', 'MULTILINGUAL')
|
|
@@ -44,51 +38,27 @@ ONNX_BACKENDS = frozenset({'onnx', 'ensemble_onnx'})
|
|
|
44
38
|
TORCH_BACKENDS = frozenset({'torch', 'ensemble_torch'})
|
|
45
39
|
TOKEN_MODEL_BACKENDS = ONNX_BACKENDS | TORCH_BACKENDS
|
|
46
40
|
|
|
47
|
-
|
|
48
|
-
'ENGLISH'
|
|
49
|
-
'DUTCH'
|
|
50
|
-
'GERMAN'
|
|
51
|
-
'SPANISH'
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
'
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
'
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
DEFAULT_TORCH_NER_MODELS: dict[str, str] = {
|
|
71
|
-
'ENGLISH': 'FacebookAI/xlm-roberta-large-finetuned-conll03-english',
|
|
72
|
-
'DUTCH': 'FacebookAI/xlm-roberta-large-finetuned-conll02-dutch',
|
|
73
|
-
'GERMAN': 'FacebookAI/xlm-roberta-large-finetuned-conll03-german',
|
|
74
|
-
'SPANISH': 'FacebookAI/xlm-roberta-large-finetuned-conll02-spanish',
|
|
75
|
-
# FR/PT/IT: no dedicated Torch model available; map to MULTILINGUAL fallback
|
|
76
|
-
'FRENCH': 'Babelscape/wikineural-multilingual-ner',
|
|
77
|
-
'PORTUGUESE': 'Babelscape/wikineural-multilingual-ner',
|
|
78
|
-
'ITALIAN': 'Babelscape/wikineural-multilingual-ner',
|
|
79
|
-
'MULTILINGUAL': 'Babelscape/wikineural-multilingual-ner',
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
DEFAULT_TORCH_NER_REVISIONS: dict[str, str] = {
|
|
83
|
-
'ENGLISH': '18f95e9924f3f452df09cc90945073906ef18f1e',
|
|
84
|
-
'DUTCH': '630d5d48d08071704d9a2719b045082019b6ac12',
|
|
85
|
-
'GERMAN': '1fbcc7a00a69ce5ab754623154a8e9cc6ba868e2',
|
|
86
|
-
'SPANISH': 'a7c5f08c766adcbd6c22f343145fa65d38b3c1d5',
|
|
87
|
-
'FRENCH': 'bed6ee7a45d2827b6c90a4fd7983f0241ae0a5c1',
|
|
88
|
-
'PORTUGUESE': 'bed6ee7a45d2827b6c90a4fd7983f0241ae0a5c1',
|
|
89
|
-
'ITALIAN': 'bed6ee7a45d2827b6c90a4fd7983f0241ae0a5c1',
|
|
90
|
-
'MULTILINGUAL': 'bed6ee7a45d2827b6c90a4fd7983f0241ae0a5c1',
|
|
91
|
-
}
|
|
41
|
+
_ONNX_CHECKPOINTS = (
|
|
42
|
+
(('ENGLISH',), 'rhnfzl/xlm-roberta-large-conll03-english-onnx', '2c10955af9538d6ff8a68316c5dbfcd0b968761e'),
|
|
43
|
+
(('DUTCH',), 'rhnfzl/xlm-roberta-large-conll02-dutch-onnx', '381e34ba4c222bc221e9fd02832f52a179ed840d'),
|
|
44
|
+
(('GERMAN',), 'rhnfzl/xlm-roberta-large-conll03-german-onnx', 'fe4aecfa6dc376244ef37da37e6df3b1b0911d71'),
|
|
45
|
+
(('SPANISH',), 'rhnfzl/xlm-roberta-large-conll02-spanish-onnx', 'ef52a7f08172203c763d51fe36660d36985bb388'),
|
|
46
|
+
(('FRENCH', 'PORTUGUESE', 'ITALIAN', 'MULTILINGUAL'),
|
|
47
|
+
'rhnfzl/wikineural-multilingual-ner-onnx', '74191432b21c956b5b04b4778c0e92e569b1f80d'),
|
|
48
|
+
)
|
|
49
|
+
_TORCH_CHECKPOINTS = (
|
|
50
|
+
(('ENGLISH',), 'FacebookAI/xlm-roberta-large-finetuned-conll03-english', '18f95e9924f3f452df09cc90945073906ef18f1e'),
|
|
51
|
+
(('DUTCH',), 'FacebookAI/xlm-roberta-large-finetuned-conll02-dutch', '630d5d48d08071704d9a2719b045082019b6ac12'),
|
|
52
|
+
(('GERMAN',), 'FacebookAI/xlm-roberta-large-finetuned-conll03-german', '1fbcc7a00a69ce5ab754623154a8e9cc6ba868e2'),
|
|
53
|
+
(('SPANISH',), 'FacebookAI/xlm-roberta-large-finetuned-conll02-spanish', 'a7c5f08c766adcbd6c22f343145fa65d38b3c1d5'),
|
|
54
|
+
(('FRENCH', 'PORTUGUESE', 'ITALIAN', 'MULTILINGUAL'),
|
|
55
|
+
'Babelscape/wikineural-multilingual-ner', 'bed6ee7a45d2827b6c90a4fd7983f0241ae0a5c1'),
|
|
56
|
+
)
|
|
57
|
+
DEFAULT_NER_MODELS = {key: model for keys, model, _ in _ONNX_CHECKPOINTS for key in keys}
|
|
58
|
+
DEFAULT_NER_REVISIONS = {key: revision for keys, _, revision in _ONNX_CHECKPOINTS for key in keys}
|
|
59
|
+
DEFAULT_TORCH_NER_MODELS = {key: model for keys, model, _ in _TORCH_CHECKPOINTS for key in keys}
|
|
60
|
+
DEFAULT_TORCH_NER_REVISIONS = {key: revision for keys, _, revision in _TORCH_CHECKPOINTS for key in keys}
|
|
61
|
+
_DEFAULT_CONFIG_NER_MODELS = {key: DEFAULT_NER_MODELS[key] for key in LANG_KEYS}
|
|
92
62
|
|
|
93
63
|
# NER ensemble: ordered model keys to run per input language.
|
|
94
64
|
# Keys must exist in the resolved ner_models dict.
|
|
@@ -168,11 +138,6 @@ MODERNBERT_NER_MODELS: dict[str, str] = {
|
|
|
168
138
|
GLICLASS_DEFAULT_MODEL = 'knowledgator/gliclass-edge-v3.0' # 32.7M params, 131MB
|
|
169
139
|
GLICLASS_DEFAULT_REVISION = 'df03993a2ed98e5e4a0d2dd7efbbd105abe874cf'
|
|
170
140
|
|
|
171
|
-
# Placeholder constants for future multilingual NER models (blocked: models not yet published)
|
|
172
|
-
OTTER_NER_MODELS: dict[str, str] = {} # arXiv:2601.06347 — checkpoints not on HF Hub
|
|
173
|
-
MMBERT_NER_MODELS: dict[str, str] = {} # GLiNER bug #332 blocks mmBERT as backbone
|
|
174
|
-
|
|
175
|
-
|
|
176
141
|
@dataclass(frozen=True)
|
|
177
142
|
class TextCleanerConfig:
|
|
178
143
|
"""Thread-safe, immutable configuration for the text cleaning pipeline.
|
|
@@ -260,10 +225,6 @@ class TextCleanerConfig:
|
|
|
260
225
|
# ensemble inference for unmapped languages.
|
|
261
226
|
ner_ensemble_default_keys: tuple[str, ...] | None = None
|
|
262
227
|
|
|
263
|
-
# Deprecated: positional tuple (English, Dutch, German, Spanish, Multilingual).
|
|
264
|
-
# Use ner_models dict instead. Kept for backward compatibility.
|
|
265
|
-
ner_models_list: tuple[str, ...] = tuple(DEFAULT_NER_MODELS[k] for k in LANG_KEYS)
|
|
266
|
-
|
|
267
228
|
# Torch backend models (only used when ner_backend is 'torch' or 'ensemble_torch')
|
|
268
229
|
torch_ner_models: Mapping[str, str] | None = None
|
|
269
230
|
torch_ner_model_revisions: Mapping[str, str] | None = None
|
|
@@ -516,7 +477,7 @@ class TextCleanerConfig:
|
|
|
516
477
|
object.__setattr__(self, 'gliner_label_map', _PII_MAPS[self.legacy_type_names])
|
|
517
478
|
|
|
518
479
|
|
|
519
|
-
_USER_LABEL = re.compile(
|
|
480
|
+
_USER_LABEL = re.compile(LABEL_PATTERN)
|
|
520
481
|
|
|
521
482
|
|
|
522
483
|
def _user_terms(name, terms):
|
|
@@ -601,8 +562,6 @@ def _freeze_sequences(cfg):
|
|
|
601
562
|
# Convert mutable collections to immutable for frozen safety
|
|
602
563
|
if isinstance(cfg.positional_tags, list):
|
|
603
564
|
object.__setattr__(cfg, 'positional_tags', tuple(cfg.positional_tags))
|
|
604
|
-
if isinstance(cfg.ner_models_list, list):
|
|
605
|
-
object.__setattr__(cfg, 'ner_models_list', tuple(cfg.ner_models_list))
|
|
606
565
|
if isinstance(cfg.custom_pipeline_steps, list):
|
|
607
566
|
object.__setattr__(cfg, 'custom_pipeline_steps', tuple(cfg.custom_pipeline_steps))
|
|
608
567
|
for step in cfg.custom_pipeline_steps:
|
|
@@ -780,42 +739,21 @@ def _reconcile_torch_models(cfg):
|
|
|
780
739
|
|
|
781
740
|
|
|
782
741
|
def _reconcile_ner_models(cfg):
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
# Dict API used — validate required keys, fill defaults, freeze
|
|
742
|
+
if cfg.ner_models is None:
|
|
743
|
+
merged = dict(_DEFAULT_CONFIG_NER_MODELS)
|
|
744
|
+
else:
|
|
787
745
|
missing = REQUIRED_NER_KEYS - set(cfg.ner_models)
|
|
788
746
|
if missing:
|
|
789
747
|
raise ValueError(
|
|
790
748
|
f"ner_models must include {sorted(REQUIRED_NER_KEYS)}, "
|
|
791
749
|
f"missing: {sorted(missing)}"
|
|
792
750
|
)
|
|
793
|
-
# A
|
|
794
|
-
# legacy/default config (replace(), copy): keep it exact. Decided by
|
|
795
|
-
# content only, so a dict and a mapping proxy give the same config.
|
|
796
|
-
derived = (
|
|
797
|
-
set(cfg.ner_models) == set(LANG_KEYS)
|
|
798
|
-
and tuple(cfg.ner_models[k] for k in LANG_KEYS) == cfg.ner_models_list
|
|
799
|
-
)
|
|
751
|
+
# A copied default config keeps its four-language routing.
|
|
800
752
|
merged = (
|
|
801
|
-
dict(cfg.ner_models) if
|
|
753
|
+
dict(cfg.ner_models) if cfg.ner_models == _DEFAULT_CONFIG_NER_MODELS
|
|
802
754
|
else {**DEFAULT_NER_MODELS, **cfg.ner_models}
|
|
803
755
|
)
|
|
804
|
-
|
|
805
|
-
object.__setattr__(
|
|
806
|
-
cfg, 'ner_models_list',
|
|
807
|
-
tuple(merged[k] for k in LANG_KEYS),
|
|
808
|
-
)
|
|
809
|
-
else:
|
|
810
|
-
# No dict provided — derive from ner_models_list (may be default or custom)
|
|
811
|
-
if cfg.ner_models_list != TextCleanerConfig.ner_models_list:
|
|
812
|
-
warnings.warn(
|
|
813
|
-
"ner_models_list is deprecated. Use ner_models dict instead.",
|
|
814
|
-
DeprecationWarning,
|
|
815
|
-
stacklevel=3,
|
|
816
|
-
)
|
|
817
|
-
models_dict = dict(zip(LANG_KEYS, cfg.ner_models_list))
|
|
818
|
-
object.__setattr__(cfg, 'ner_models', MappingProxyType(models_dict))
|
|
756
|
+
object.__setattr__(cfg, 'ner_models', MappingProxyType(merged))
|
|
819
757
|
|
|
820
758
|
pinned_revisions = {
|
|
821
759
|
key: revision
|
|
@@ -890,12 +828,6 @@ def _compute_supported_languages(cfg):
|
|
|
890
828
|
object.__setattr__(cfg, 'supported_languages', frozenset(langs))
|
|
891
829
|
|
|
892
830
|
|
|
893
|
-
# Fields without a plain UPPER_CASE global: ner_models_list is fed by the NER_MODELS_LIST global through
|
|
894
|
-
# ner_models, supported_languages is computed, and NER_ENSEMBLE_DEFAULT_KEYS is also the built-in fallback.
|
|
895
|
-
_NO_PLAIN_GLOBAL = frozenset({'ner_models_list', 'supported_languages', 'ner_ensemble_default_keys'})
|
|
896
|
-
_BUILTIN_ENSEMBLE_DEFAULT_KEYS = NER_ENSEMBLE_DEFAULT_KEYS
|
|
897
|
-
|
|
898
|
-
|
|
899
831
|
def require_adapter_mode(cfg: TextCleanerConfig) -> None:
|
|
900
832
|
"""Refuse reversible mode on a surface that has no way to hand back each text's map."""
|
|
901
833
|
if cfg.replacement_mode == "reversible":
|
|
@@ -903,36 +835,3 @@ def require_adapter_mode(cfg: TextCleanerConfig) -> None:
|
|
|
903
835
|
"Adapters do not support reversible mode. "
|
|
904
836
|
"Process each text directly and retain its token map."
|
|
905
837
|
)
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
def _config_from_module_globals() -> TextCleanerConfig:
|
|
909
|
-
"""Build a TextCleanerConfig from the current module-level variable state.
|
|
910
|
-
|
|
911
|
-
Supports backward compatibility with the legacy config pattern.
|
|
912
|
-
"""
|
|
913
|
-
m = sys.modules[__name__]
|
|
914
|
-
kwargs: dict[str, Any] = {
|
|
915
|
-
f.name: getattr(m, f.name.upper(), f.default)
|
|
916
|
-
for f in fields(TextCleanerConfig)
|
|
917
|
-
if f.name not in _NO_PLAIN_GLOBAL
|
|
918
|
-
}
|
|
919
|
-
kwargs['positional_tags'] = tuple(m.POSITIONAL_TAGS)
|
|
920
|
-
if kwargs['ner_models'] is None:
|
|
921
|
-
kwargs['ner_models'] = dict(zip(LANG_KEYS, m.NER_MODELS_LIST))
|
|
922
|
-
if m.NER_ENSEMBLE_DEFAULT_KEYS != _BUILTIN_ENSEMBLE_DEFAULT_KEYS:
|
|
923
|
-
kwargs['ner_ensemble_default_keys'] = m.NER_ENSEMBLE_DEFAULT_KEYS
|
|
924
|
-
return TextCleanerConfig(**kwargs)
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
# ---------------------------------------------------------------------------
|
|
928
|
-
# Backward-compatible module-level variables: one per field, named in upper case, set to the field's
|
|
929
|
-
# default. Users can still do: config.CHECK_NER_PROCESS = False
|
|
930
|
-
# ---------------------------------------------------------------------------
|
|
931
|
-
for _field in fields(TextCleanerConfig):
|
|
932
|
-
if _field.name not in _NO_PLAIN_GLOBAL:
|
|
933
|
-
globals()[_field.name.upper()] = _field.default
|
|
934
|
-
|
|
935
|
-
# Lists, as they always were, so config.POSITIONAL_TAGS.append('X') keeps working.
|
|
936
|
-
POSITIONAL_TAGS = list(TextCleanerConfig.positional_tags)
|
|
937
|
-
# Order: English, Dutch, German, Spanish, Multilingual
|
|
938
|
-
NER_MODELS_LIST = list(TextCleanerConfig.ner_models_list)
|
|
@@ -17,6 +17,8 @@ from collections.abc import Iterable, Sequence
|
|
|
17
17
|
|
|
18
18
|
import regex
|
|
19
19
|
|
|
20
|
+
from sct.utils.markers import LABEL_PATTERN, PLACEHOLDER_PATTERN
|
|
21
|
+
|
|
20
22
|
if TYPE_CHECKING:
|
|
21
23
|
from sct.utils.process_result import ProcessResult
|
|
22
24
|
|
|
@@ -29,9 +31,9 @@ SCORE_A_COLUMNS = tuple(SCORE_A_TYPES)
|
|
|
29
31
|
SCORE_B_COLUMNS = ("sct_quality", "sct_minhash")
|
|
30
32
|
|
|
31
33
|
# Placeholders count as one token; reversible ``<PERSON_3>`` normalises to ``<PERSON>``.
|
|
32
|
-
_PLACEHOLDER = regex.compile(
|
|
33
|
-
_WORD_TOKEN = regex.compile(
|
|
34
|
-
_CHAR_TOKEN = regex.compile(
|
|
34
|
+
_PLACEHOLDER = regex.compile(rf"<({LABEL_PATTERN}?)(?:_\d+)?>")
|
|
35
|
+
_WORD_TOKEN = regex.compile(rf"{PLACEHOLDER_PATTERN}|\w+")
|
|
36
|
+
_CHAR_TOKEN = regex.compile(rf"{PLACEHOLDER_PATTERN}|\S")
|
|
35
37
|
# Scripts written without spaces between words get character shingles.
|
|
36
38
|
_NO_SPACE_SCRIPT = regex.compile(
|
|
37
39
|
r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Thai}\p{Lao}\p{Khmer}\p{Myanmar}]"
|