SqueakyCleanText 0.8.4__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/PKG-INFO +15 -3
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/README.md +14 -2
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/SqueakyCleanText.egg-info/PKG-INFO +15 -3
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/SqueakyCleanText.egg-info/SOURCES.txt +5 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/__init__.py +1 -1
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/config.py +31 -132
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/score.py +5 -3
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/sct.py +49 -71
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/streaming.py +1 -30
- squeakycleantext-0.9.0/sct/utils/markers.py +7 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/ner.py +21 -101
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/ner_chunking.py +74 -1
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/special.py +2 -3
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_benchmark_adapters.py +20 -20
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_benchmark_release.py +3 -1
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_benchmarks_compare.py +69 -40
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_benchmarks_round2_tools.py +10 -21
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_chunking.py +3 -7
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_cjk_width.py +0 -10
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_cleaner_lifecycle.py +0 -17
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_config_state.py +2 -33
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_contracts.py +2 -15
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_device_provenance.py +2 -8
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_functional.py +0 -11
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_keep_scripts.py +0 -7
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_language_all.py +0 -21
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_ner_sees_tags.py +0 -11
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_ner_span_repair.py +4 -38
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_ner_windows.py +28 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_number_identifiers.py +2 -26
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_onnx_concurrency.py +0 -8
- squeakycleantext-0.9.0/tests/test_pii_social_cards.py +55 -0
- squeakycleantext-0.9.0/tests/test_plot_pii_comparison.py +62 -0
- squeakycleantext-0.9.0/tests/test_plot_version_comparison.py +36 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_presidio_tags_and_fakes.py +10 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_privacy.py +1 -1
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_regex_boundaries.py +0 -26
- squeakycleantext-0.9.0/tests/test_release_comparison_watcher.py +74 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_repo_scripts.py +31 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_sct.py +1 -52
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_streaming.py +1 -31
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_styled_pii.py +0 -10
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_synthetic_per_document.py +1 -27
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_training_preset.py +1 -8
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_type_names.py +1 -10
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_user_terms.py +1 -18
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/LICENSE +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/SqueakyCleanText.egg-info/dependency_links.txt +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/SqueakyCleanText.egg-info/entry_points.txt +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/SqueakyCleanText.egg-info/requires.txt +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/SqueakyCleanText.egg-info/top_level.txt +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/pyproject.toml +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/adapters.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/browser.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/cli.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/evaluation.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/integrations/__init__.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/integrations/_runtime.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/integrations/datatrove.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/integrations/haystack.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/integrations/hf.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/integrations/langchain.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/mcp_server.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/model_benchmark.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/model_candidates.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/ocr.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/presets.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/privacy.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/py.typed +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/report_cli.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/risk.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/signals.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/__init__.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/anonymization_map.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/atomic_file.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/constants.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/contact.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/datetime.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/gliclass_adapter.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/gliner_adapter.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/ner_spans.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/normtext.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/onnx_pipeline.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/process_result.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/resources.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/stopwords.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/synthetic.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/sct/utils/torch_pipeline.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/setup.cfg +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_adapters.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_address_leftovers.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_atomic_file.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_batching.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_benchmark_loaders.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_benchmarks_round2.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_benchmarks_score.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_browser.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_cli.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_date_forms.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_dates_cjk_styled.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_dates_default.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_email_forms.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_escaped_markup.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_evaluation.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_fork_safety.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_fuzz_styled_pii.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_integrations_frameworks.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_integrations_hf.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_integrations_runtime.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_linux_workers.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_mcp_path.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_mcp_server.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_model_benchmarks.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_multilingual_eval_set.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_ner_spans.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_ocr.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_onnx_external_data.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_onnx_model_cache.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_phone_forms.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_pipeline_text_steps.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_predicted_spans_monotone.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_presidio_type_names.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_prose_punctuation_kept.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_public_view.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_regex_counts.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_report_cli.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_reversible_markers.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_risk.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_score.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_security_leftovers.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_stream_guard_shared_view.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_stream_keeps_values_whole.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_streaming_split_stability.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_surface_contracts.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_synthetic_determinism.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_torch_ner_backend.py +0 -0
- {squeakycleantext-0.8.4 → squeakycleantext-0.9.0}/tests/test_url_forms.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SqueakyCleanText
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
|
|
5
5
|
Author: Rehan Fazal
|
|
6
6
|
License-Expression: MIT
|
|
@@ -256,6 +256,9 @@ The framework adapters use the `training` preset unless you pass a config, so th
|
|
|
256
256
|
|
|
257
257
|
Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CONFIG.md)). The diagrams are built by [scripts/diagrams/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/scripts/diagrams); open the `.excalidraw` files in [resources/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/resources) to view them in Excalidraw.
|
|
258
258
|
|
|
259
|
+
From 0.9.0, configure cleaners with `TextCleanerConfig` and set document concurrency with `max_workers`.
|
|
260
|
+
Deprecated config globals, `ner_models_list`, the worker `batch_size` alias and cleaner helper properties are removed.
|
|
261
|
+
|
|
259
262
|
<!-- --8<-- [start:results] -->
|
|
260
263
|
## What you get back
|
|
261
264
|
|
|
@@ -300,6 +303,15 @@ NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cl
|
|
|
300
303
|
|
|
301
304
|
Measured on open datasets with [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks).
|
|
302
305
|
|
|
306
|
+
The [SCT v0.8.2 comparison with other PII tools](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-04-visuals/README.md)
|
|
307
|
+
shows paired recall, precision and exposure results, with uncertainty intervals.
|
|
308
|
+
It measures PII configurations rather than the whole processing pipeline. The tested version is v0.8.2, not v0.8.3.
|
|
309
|
+
|
|
310
|
+
The [released v0.7.1 versus v0.8.3 comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-0.7.1-0.8.3/README.md)
|
|
311
|
+
includes all 12 PII/long-document datasets, repeated CPU speed runs, and charts.
|
|
312
|
+
Strict span F1 improves on every dataset. Unsupported-language false answers and TAB direct-identifier coverage worsen.
|
|
313
|
+
Runtime versions differ, so the speed results do not isolate SCT code changes.
|
|
314
|
+
|
|
303
315
|
- **Cleaning** (FineWeb-2 with injected noise, default config): 100% of HTML, 99.6% of mojibake and 100% of URLs removed.
|
|
304
316
|
- **Language detection** (WiLI): 95.1% accuracy over 7,200 documents.
|
|
305
317
|
- **PII** (12 open PII datasets, about 40 languages): the 0.8.1 release branch leaked fewer values than 0.8.0 on every dataset, for example 18.1% to 8.2% on REDACT ([comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/comparison-0.8.0-unreleased.md), pre-release commit `2078e8c4`).
|
|
@@ -322,7 +334,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
|
|
|
322
334
|
- **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
|
|
323
335
|
<!-- --8<-- [end:limits] -->
|
|
324
336
|
|
|
325
|
-
Detailed known limits of 0.
|
|
337
|
+
Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
|
|
326
338
|
|
|
327
339
|
## Documentation
|
|
328
340
|
|
|
@@ -354,7 +366,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
|
|
|
354
366
|
author = {Fazal, Rehan},
|
|
355
367
|
title = {SqueakyCleanText},
|
|
356
368
|
url = {https://github.com/rhnfzl/SqueakyCleanText},
|
|
357
|
-
version = {0.
|
|
369
|
+
version = {0.9.0},
|
|
358
370
|
year = {2026}
|
|
359
371
|
}
|
|
360
372
|
```
|
|
@@ -165,6 +165,9 @@ The framework adapters use the `training` preset unless you pass a config, so th
|
|
|
165
165
|
|
|
166
166
|
Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CONFIG.md)). The diagrams are built by [scripts/diagrams/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/scripts/diagrams); open the `.excalidraw` files in [resources/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/resources) to view them in Excalidraw.
|
|
167
167
|
|
|
168
|
+
From 0.9.0, configure cleaners with `TextCleanerConfig` and set document concurrency with `max_workers`.
|
|
169
|
+
Deprecated config globals, `ner_models_list`, the worker `batch_size` alias and cleaner helper properties are removed.
|
|
170
|
+
|
|
168
171
|
<!-- --8<-- [start:results] -->
|
|
169
172
|
## What you get back
|
|
170
173
|
|
|
@@ -209,6 +212,15 @@ NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cl
|
|
|
209
212
|
|
|
210
213
|
Measured on open datasets with [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks).
|
|
211
214
|
|
|
215
|
+
The [SCT v0.8.2 comparison with other PII tools](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-04-visuals/README.md)
|
|
216
|
+
shows paired recall, precision and exposure results, with uncertainty intervals.
|
|
217
|
+
It measures PII configurations rather than the whole processing pipeline. The tested version is v0.8.2, not v0.8.3.
|
|
218
|
+
|
|
219
|
+
The [released v0.7.1 versus v0.8.3 comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-0.7.1-0.8.3/README.md)
|
|
220
|
+
includes all 12 PII/long-document datasets, repeated CPU speed runs, and charts.
|
|
221
|
+
Strict span F1 improves on every dataset. Unsupported-language false answers and TAB direct-identifier coverage worsen.
|
|
222
|
+
Runtime versions differ, so the speed results do not isolate SCT code changes.
|
|
223
|
+
|
|
212
224
|
- **Cleaning** (FineWeb-2 with injected noise, default config): 100% of HTML, 99.6% of mojibake and 100% of URLs removed.
|
|
213
225
|
- **Language detection** (WiLI): 95.1% accuracy over 7,200 documents.
|
|
214
226
|
- **PII** (12 open PII datasets, about 40 languages): the 0.8.1 release branch leaked fewer values than 0.8.0 on every dataset, for example 18.1% to 8.2% on REDACT ([comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/comparison-0.8.0-unreleased.md), pre-release commit `2078e8c4`).
|
|
@@ -231,7 +243,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
|
|
|
231
243
|
- **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
|
|
232
244
|
<!-- --8<-- [end:limits] -->
|
|
233
245
|
|
|
234
|
-
Detailed known limits of 0.
|
|
246
|
+
Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
|
|
235
247
|
|
|
236
248
|
## Documentation
|
|
237
249
|
|
|
@@ -263,7 +275,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
|
|
|
263
275
|
author = {Fazal, Rehan},
|
|
264
276
|
title = {SqueakyCleanText},
|
|
265
277
|
url = {https://github.com/rhnfzl/SqueakyCleanText},
|
|
266
|
-
version = {0.
|
|
278
|
+
version = {0.9.0},
|
|
267
279
|
year = {2026}
|
|
268
280
|
}
|
|
269
281
|
```
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SqueakyCleanText
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
|
|
5
5
|
Author: Rehan Fazal
|
|
6
6
|
License-Expression: MIT
|
|
@@ -256,6 +256,9 @@ The framework adapters use the `training` preset unless you pass a config, so th
|
|
|
256
256
|
|
|
257
257
|
Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CONFIG.md)). The diagrams are built by [scripts/diagrams/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/scripts/diagrams); open the `.excalidraw` files in [resources/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/resources) to view them in Excalidraw.
|
|
258
258
|
|
|
259
|
+
From 0.9.0, configure cleaners with `TextCleanerConfig` and set document concurrency with `max_workers`.
|
|
260
|
+
Deprecated config globals, `ner_models_list`, the worker `batch_size` alias and cleaner helper properties are removed.
|
|
261
|
+
|
|
259
262
|
<!-- --8<-- [start:results] -->
|
|
260
263
|
## What you get back
|
|
261
264
|
|
|
@@ -300,6 +303,15 @@ NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cl
|
|
|
300
303
|
|
|
301
304
|
Measured on open datasets with [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks).
|
|
302
305
|
|
|
306
|
+
The [SCT v0.8.2 comparison with other PII tools](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-04-visuals/README.md)
|
|
307
|
+
shows paired recall, precision and exposure results, with uncertainty intervals.
|
|
308
|
+
It measures PII configurations rather than the whole processing pipeline. The tested version is v0.8.2, not v0.8.3.
|
|
309
|
+
|
|
310
|
+
The [released v0.7.1 versus v0.8.3 comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-0.7.1-0.8.3/README.md)
|
|
311
|
+
includes all 12 PII/long-document datasets, repeated CPU speed runs, and charts.
|
|
312
|
+
Strict span F1 improves on every dataset. Unsupported-language false answers and TAB direct-identifier coverage worsen.
|
|
313
|
+
Runtime versions differ, so the speed results do not isolate SCT code changes.
|
|
314
|
+
|
|
303
315
|
- **Cleaning** (FineWeb-2 with injected noise, default config): 100% of HTML, 99.6% of mojibake and 100% of URLs removed.
|
|
304
316
|
- **Language detection** (WiLI): 95.1% accuracy over 7,200 documents.
|
|
305
317
|
- **PII** (12 open PII datasets, about 40 languages): the 0.8.1 release branch leaked fewer values than 0.8.0 on every dataset, for example 18.1% to 8.2% on REDACT ([comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/comparison-0.8.0-unreleased.md), pre-release commit `2078e8c4`).
|
|
@@ -322,7 +334,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
|
|
|
322
334
|
- **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
|
|
323
335
|
<!-- --8<-- [end:limits] -->
|
|
324
336
|
|
|
325
|
-
Detailed known limits of 0.
|
|
337
|
+
Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
|
|
326
338
|
|
|
327
339
|
## Documentation
|
|
328
340
|
|
|
@@ -354,7 +366,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
|
|
|
354
366
|
author = {Fazal, Rehan},
|
|
355
367
|
title = {SqueakyCleanText},
|
|
356
368
|
url = {https://github.com/rhnfzl/SqueakyCleanText},
|
|
357
|
-
version = {0.
|
|
369
|
+
version = {0.9.0},
|
|
358
370
|
year = {2026}
|
|
359
371
|
}
|
|
360
372
|
```
|
|
@@ -40,6 +40,7 @@ sct/utils/contact.py
|
|
|
40
40
|
sct/utils/datetime.py
|
|
41
41
|
sct/utils/gliclass_adapter.py
|
|
42
42
|
sct/utils/gliner_adapter.py
|
|
43
|
+
sct/utils/markers.py
|
|
43
44
|
sct/utils/ner.py
|
|
44
45
|
sct/utils/ner_chunking.py
|
|
45
46
|
sct/utils/ner_spans.py
|
|
@@ -99,7 +100,10 @@ tests/test_onnx_concurrency.py
|
|
|
99
100
|
tests/test_onnx_external_data.py
|
|
100
101
|
tests/test_onnx_model_cache.py
|
|
101
102
|
tests/test_phone_forms.py
|
|
103
|
+
tests/test_pii_social_cards.py
|
|
102
104
|
tests/test_pipeline_text_steps.py
|
|
105
|
+
tests/test_plot_pii_comparison.py
|
|
106
|
+
tests/test_plot_version_comparison.py
|
|
103
107
|
tests/test_predicted_spans_monotone.py
|
|
104
108
|
tests/test_presidio_tags_and_fakes.py
|
|
105
109
|
tests/test_presidio_type_names.py
|
|
@@ -108,6 +112,7 @@ tests/test_prose_punctuation_kept.py
|
|
|
108
112
|
tests/test_public_view.py
|
|
109
113
|
tests/test_regex_boundaries.py
|
|
110
114
|
tests/test_regex_counts.py
|
|
115
|
+
tests/test_release_comparison_watcher.py
|
|
111
116
|
tests/test_repo_scripts.py
|
|
112
117
|
tests/test_report_cli.py
|
|
113
118
|
tests/test_reversible_markers.py
|
|
@@ -51,7 +51,7 @@ from sct.streaming import (
|
|
|
51
51
|
from sct.utils.anonymization_map import AnonymizationMap, MapEntry
|
|
52
52
|
from sct.utils.process_result import ProcessResult
|
|
53
53
|
|
|
54
|
-
__version__ = "0.
|
|
54
|
+
__version__ = "0.9.0"
|
|
55
55
|
__all__ = [
|
|
56
56
|
"TextCleaner", "TextCleanerConfig", "CustomPatternTimeout", "ModelLoadError", "PRESETS", "preset_config",
|
|
57
57
|
"PII_LABELS", "PII_LABEL_MAP",
|
|
@@ -1,28 +1,22 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Configuration for the SqueakyCleanText pipeline.
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
Pass an immutable config to each cleaner:
|
|
5
5
|
from sct.config import TextCleanerConfig
|
|
6
6
|
cfg = TextCleanerConfig(check_ner_process=False)
|
|
7
7
|
cleaner = TextCleaner(cfg)
|
|
8
|
-
|
|
9
|
-
Legacy API (backward compatible):
|
|
10
|
-
from sct import config
|
|
11
|
-
config.CHECK_NER_PROCESS = False
|
|
12
|
-
cleaner = TextCleaner()
|
|
13
8
|
"""
|
|
14
9
|
|
|
15
10
|
import re
|
|
16
|
-
import sys
|
|
17
|
-
import warnings
|
|
18
11
|
from collections.abc import Iterable
|
|
19
|
-
from dataclasses import dataclass
|
|
12
|
+
from dataclasses import dataclass
|
|
20
13
|
from datetime import date
|
|
21
14
|
from types import MappingProxyType
|
|
22
15
|
from typing import Any, Literal, get_args
|
|
23
16
|
from collections.abc import Mapping
|
|
24
17
|
|
|
25
18
|
from sct.utils.resources import DEFAULT_LANGUAGES, resolve_language
|
|
19
|
+
from sct.utils.markers import LABEL_PATTERN
|
|
26
20
|
|
|
27
21
|
|
|
28
22
|
LANG_KEYS = ('ENGLISH', 'DUTCH', 'GERMAN', 'SPANISH', 'MULTILINGUAL')
|
|
@@ -44,51 +38,27 @@ ONNX_BACKENDS = frozenset({'onnx', 'ensemble_onnx'})
|
|
|
44
38
|
TORCH_BACKENDS = frozenset({'torch', 'ensemble_torch'})
|
|
45
39
|
TOKEN_MODEL_BACKENDS = ONNX_BACKENDS | TORCH_BACKENDS
|
|
46
40
|
|
|
47
|
-
|
|
48
|
-
'ENGLISH'
|
|
49
|
-
'DUTCH'
|
|
50
|
-
'GERMAN'
|
|
51
|
-
'SPANISH'
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
'
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
'
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
DEFAULT_TORCH_NER_MODELS: dict[str, str] = {
|
|
71
|
-
'ENGLISH': 'FacebookAI/xlm-roberta-large-finetuned-conll03-english',
|
|
72
|
-
'DUTCH': 'FacebookAI/xlm-roberta-large-finetuned-conll02-dutch',
|
|
73
|
-
'GERMAN': 'FacebookAI/xlm-roberta-large-finetuned-conll03-german',
|
|
74
|
-
'SPANISH': 'FacebookAI/xlm-roberta-large-finetuned-conll02-spanish',
|
|
75
|
-
# FR/PT/IT: no dedicated Torch model available; map to MULTILINGUAL fallback
|
|
76
|
-
'FRENCH': 'Babelscape/wikineural-multilingual-ner',
|
|
77
|
-
'PORTUGUESE': 'Babelscape/wikineural-multilingual-ner',
|
|
78
|
-
'ITALIAN': 'Babelscape/wikineural-multilingual-ner',
|
|
79
|
-
'MULTILINGUAL': 'Babelscape/wikineural-multilingual-ner',
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
DEFAULT_TORCH_NER_REVISIONS: dict[str, str] = {
|
|
83
|
-
'ENGLISH': '18f95e9924f3f452df09cc90945073906ef18f1e',
|
|
84
|
-
'DUTCH': '630d5d48d08071704d9a2719b045082019b6ac12',
|
|
85
|
-
'GERMAN': '1fbcc7a00a69ce5ab754623154a8e9cc6ba868e2',
|
|
86
|
-
'SPANISH': 'a7c5f08c766adcbd6c22f343145fa65d38b3c1d5',
|
|
87
|
-
'FRENCH': 'bed6ee7a45d2827b6c90a4fd7983f0241ae0a5c1',
|
|
88
|
-
'PORTUGUESE': 'bed6ee7a45d2827b6c90a4fd7983f0241ae0a5c1',
|
|
89
|
-
'ITALIAN': 'bed6ee7a45d2827b6c90a4fd7983f0241ae0a5c1',
|
|
90
|
-
'MULTILINGUAL': 'bed6ee7a45d2827b6c90a4fd7983f0241ae0a5c1',
|
|
91
|
-
}
|
|
41
|
+
_ONNX_CHECKPOINTS = (
|
|
42
|
+
(('ENGLISH',), 'rhnfzl/xlm-roberta-large-conll03-english-onnx', '2c10955af9538d6ff8a68316c5dbfcd0b968761e'),
|
|
43
|
+
(('DUTCH',), 'rhnfzl/xlm-roberta-large-conll02-dutch-onnx', '381e34ba4c222bc221e9fd02832f52a179ed840d'),
|
|
44
|
+
(('GERMAN',), 'rhnfzl/xlm-roberta-large-conll03-german-onnx', 'fe4aecfa6dc376244ef37da37e6df3b1b0911d71'),
|
|
45
|
+
(('SPANISH',), 'rhnfzl/xlm-roberta-large-conll02-spanish-onnx', 'ef52a7f08172203c763d51fe36660d36985bb388'),
|
|
46
|
+
(('FRENCH', 'PORTUGUESE', 'ITALIAN', 'MULTILINGUAL'),
|
|
47
|
+
'rhnfzl/wikineural-multilingual-ner-onnx', '74191432b21c956b5b04b4778c0e92e569b1f80d'),
|
|
48
|
+
)
|
|
49
|
+
_TORCH_CHECKPOINTS = (
|
|
50
|
+
(('ENGLISH',), 'FacebookAI/xlm-roberta-large-finetuned-conll03-english', '18f95e9924f3f452df09cc90945073906ef18f1e'),
|
|
51
|
+
(('DUTCH',), 'FacebookAI/xlm-roberta-large-finetuned-conll02-dutch', '630d5d48d08071704d9a2719b045082019b6ac12'),
|
|
52
|
+
(('GERMAN',), 'FacebookAI/xlm-roberta-large-finetuned-conll03-german', '1fbcc7a00a69ce5ab754623154a8e9cc6ba868e2'),
|
|
53
|
+
(('SPANISH',), 'FacebookAI/xlm-roberta-large-finetuned-conll02-spanish', 'a7c5f08c766adcbd6c22f343145fa65d38b3c1d5'),
|
|
54
|
+
(('FRENCH', 'PORTUGUESE', 'ITALIAN', 'MULTILINGUAL'),
|
|
55
|
+
'Babelscape/wikineural-multilingual-ner', 'bed6ee7a45d2827b6c90a4fd7983f0241ae0a5c1'),
|
|
56
|
+
)
|
|
57
|
+
DEFAULT_NER_MODELS = {key: model for keys, model, _ in _ONNX_CHECKPOINTS for key in keys}
|
|
58
|
+
DEFAULT_NER_REVISIONS = {key: revision for keys, _, revision in _ONNX_CHECKPOINTS for key in keys}
|
|
59
|
+
DEFAULT_TORCH_NER_MODELS = {key: model for keys, model, _ in _TORCH_CHECKPOINTS for key in keys}
|
|
60
|
+
DEFAULT_TORCH_NER_REVISIONS = {key: revision for keys, _, revision in _TORCH_CHECKPOINTS for key in keys}
|
|
61
|
+
_DEFAULT_CONFIG_NER_MODELS = {key: DEFAULT_NER_MODELS[key] for key in LANG_KEYS}
|
|
92
62
|
|
|
93
63
|
# NER ensemble: ordered model keys to run per input language.
|
|
94
64
|
# Keys must exist in the resolved ner_models dict.
|
|
@@ -168,11 +138,6 @@ MODERNBERT_NER_MODELS: dict[str, str] = {
|
|
|
168
138
|
GLICLASS_DEFAULT_MODEL = 'knowledgator/gliclass-edge-v3.0' # 32.7M params, 131MB
|
|
169
139
|
GLICLASS_DEFAULT_REVISION = 'df03993a2ed98e5e4a0d2dd7efbbd105abe874cf'
|
|
170
140
|
|
|
171
|
-
# Placeholder constants for future multilingual NER models (blocked: models not yet published)
|
|
172
|
-
OTTER_NER_MODELS: dict[str, str] = {} # arXiv:2601.06347 — checkpoints not on HF Hub
|
|
173
|
-
MMBERT_NER_MODELS: dict[str, str] = {} # GLiNER bug #332 blocks mmBERT as backbone
|
|
174
|
-
|
|
175
|
-
|
|
176
141
|
@dataclass(frozen=True)
|
|
177
142
|
class TextCleanerConfig:
|
|
178
143
|
"""Thread-safe, immutable configuration for the text cleaning pipeline.
|
|
@@ -260,10 +225,6 @@ class TextCleanerConfig:
|
|
|
260
225
|
# ensemble inference for unmapped languages.
|
|
261
226
|
ner_ensemble_default_keys: tuple[str, ...] | None = None
|
|
262
227
|
|
|
263
|
-
# Deprecated: positional tuple (English, Dutch, German, Spanish, Multilingual).
|
|
264
|
-
# Use ner_models dict instead. Kept for backward compatibility.
|
|
265
|
-
ner_models_list: tuple[str, ...] = tuple(DEFAULT_NER_MODELS[k] for k in LANG_KEYS)
|
|
266
|
-
|
|
267
228
|
# Torch backend models (only used when ner_backend is 'torch' or 'ensemble_torch')
|
|
268
229
|
torch_ner_models: Mapping[str, str] | None = None
|
|
269
230
|
torch_ner_model_revisions: Mapping[str, str] | None = None
|
|
@@ -516,7 +477,7 @@ class TextCleanerConfig:
|
|
|
516
477
|
object.__setattr__(self, 'gliner_label_map', _PII_MAPS[self.legacy_type_names])
|
|
517
478
|
|
|
518
479
|
|
|
519
|
-
_USER_LABEL = re.compile(
|
|
480
|
+
_USER_LABEL = re.compile(LABEL_PATTERN)
|
|
520
481
|
|
|
521
482
|
|
|
522
483
|
def _user_terms(name, terms):
|
|
@@ -601,8 +562,6 @@ def _freeze_sequences(cfg):
|
|
|
601
562
|
# Convert mutable collections to immutable for frozen safety
|
|
602
563
|
if isinstance(cfg.positional_tags, list):
|
|
603
564
|
object.__setattr__(cfg, 'positional_tags', tuple(cfg.positional_tags))
|
|
604
|
-
if isinstance(cfg.ner_models_list, list):
|
|
605
|
-
object.__setattr__(cfg, 'ner_models_list', tuple(cfg.ner_models_list))
|
|
606
565
|
if isinstance(cfg.custom_pipeline_steps, list):
|
|
607
566
|
object.__setattr__(cfg, 'custom_pipeline_steps', tuple(cfg.custom_pipeline_steps))
|
|
608
567
|
for step in cfg.custom_pipeline_steps:
|
|
@@ -780,42 +739,21 @@ def _reconcile_torch_models(cfg):
|
|
|
780
739
|
|
|
781
740
|
|
|
782
741
|
def _reconcile_ner_models(cfg):
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
# Dict API used — validate required keys, fill defaults, freeze
|
|
742
|
+
if cfg.ner_models is None:
|
|
743
|
+
merged = dict(_DEFAULT_CONFIG_NER_MODELS)
|
|
744
|
+
else:
|
|
787
745
|
missing = REQUIRED_NER_KEYS - set(cfg.ner_models)
|
|
788
746
|
if missing:
|
|
789
747
|
raise ValueError(
|
|
790
748
|
f"ner_models must include {sorted(REQUIRED_NER_KEYS)}, "
|
|
791
749
|
f"missing: {sorted(missing)}"
|
|
792
750
|
)
|
|
793
|
-
# A
|
|
794
|
-
# legacy/default config (replace(), copy): keep it exact. Decided by
|
|
795
|
-
# content only, so a dict and a mapping proxy give the same config.
|
|
796
|
-
derived = (
|
|
797
|
-
set(cfg.ner_models) == set(LANG_KEYS)
|
|
798
|
-
and tuple(cfg.ner_models[k] for k in LANG_KEYS) == cfg.ner_models_list
|
|
799
|
-
)
|
|
751
|
+
# A copied default config keeps its four-language routing.
|
|
800
752
|
merged = (
|
|
801
|
-
dict(cfg.ner_models) if
|
|
753
|
+
dict(cfg.ner_models) if cfg.ner_models == _DEFAULT_CONFIG_NER_MODELS
|
|
802
754
|
else {**DEFAULT_NER_MODELS, **cfg.ner_models}
|
|
803
755
|
)
|
|
804
|
-
|
|
805
|
-
object.__setattr__(
|
|
806
|
-
cfg, 'ner_models_list',
|
|
807
|
-
tuple(merged[k] for k in LANG_KEYS),
|
|
808
|
-
)
|
|
809
|
-
else:
|
|
810
|
-
# No dict provided — derive from ner_models_list (may be default or custom)
|
|
811
|
-
if cfg.ner_models_list != TextCleanerConfig.ner_models_list:
|
|
812
|
-
warnings.warn(
|
|
813
|
-
"ner_models_list is deprecated. Use ner_models dict instead.",
|
|
814
|
-
DeprecationWarning,
|
|
815
|
-
stacklevel=3,
|
|
816
|
-
)
|
|
817
|
-
models_dict = dict(zip(LANG_KEYS, cfg.ner_models_list))
|
|
818
|
-
object.__setattr__(cfg, 'ner_models', MappingProxyType(models_dict))
|
|
756
|
+
object.__setattr__(cfg, 'ner_models', MappingProxyType(merged))
|
|
819
757
|
|
|
820
758
|
pinned_revisions = {
|
|
821
759
|
key: revision
|
|
@@ -890,12 +828,6 @@ def _compute_supported_languages(cfg):
|
|
|
890
828
|
object.__setattr__(cfg, 'supported_languages', frozenset(langs))
|
|
891
829
|
|
|
892
830
|
|
|
893
|
-
# Fields without a plain UPPER_CASE global: ner_models_list is fed by the NER_MODELS_LIST global through
|
|
894
|
-
# ner_models, supported_languages is computed, and NER_ENSEMBLE_DEFAULT_KEYS is also the built-in fallback.
|
|
895
|
-
_NO_PLAIN_GLOBAL = frozenset({'ner_models_list', 'supported_languages', 'ner_ensemble_default_keys'})
|
|
896
|
-
_BUILTIN_ENSEMBLE_DEFAULT_KEYS = NER_ENSEMBLE_DEFAULT_KEYS
|
|
897
|
-
|
|
898
|
-
|
|
899
831
|
def require_adapter_mode(cfg: TextCleanerConfig) -> None:
|
|
900
832
|
"""Refuse reversible mode on a surface that has no way to hand back each text's map."""
|
|
901
833
|
if cfg.replacement_mode == "reversible":
|
|
@@ -903,36 +835,3 @@ def require_adapter_mode(cfg: TextCleanerConfig) -> None:
|
|
|
903
835
|
"Adapters do not support reversible mode. "
|
|
904
836
|
"Process each text directly and retain its token map."
|
|
905
837
|
)
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
def _config_from_module_globals() -> TextCleanerConfig:
|
|
909
|
-
"""Build a TextCleanerConfig from the current module-level variable state.
|
|
910
|
-
|
|
911
|
-
Supports backward compatibility with the legacy config pattern.
|
|
912
|
-
"""
|
|
913
|
-
m = sys.modules[__name__]
|
|
914
|
-
kwargs: dict[str, Any] = {
|
|
915
|
-
f.name: getattr(m, f.name.upper(), f.default)
|
|
916
|
-
for f in fields(TextCleanerConfig)
|
|
917
|
-
if f.name not in _NO_PLAIN_GLOBAL
|
|
918
|
-
}
|
|
919
|
-
kwargs['positional_tags'] = tuple(m.POSITIONAL_TAGS)
|
|
920
|
-
if kwargs['ner_models'] is None:
|
|
921
|
-
kwargs['ner_models'] = dict(zip(LANG_KEYS, m.NER_MODELS_LIST))
|
|
922
|
-
if m.NER_ENSEMBLE_DEFAULT_KEYS != _BUILTIN_ENSEMBLE_DEFAULT_KEYS:
|
|
923
|
-
kwargs['ner_ensemble_default_keys'] = m.NER_ENSEMBLE_DEFAULT_KEYS
|
|
924
|
-
return TextCleanerConfig(**kwargs)
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
# ---------------------------------------------------------------------------
|
|
928
|
-
# Backward-compatible module-level variables: one per field, named in upper case, set to the field's
|
|
929
|
-
# default. Users can still do: config.CHECK_NER_PROCESS = False
|
|
930
|
-
# ---------------------------------------------------------------------------
|
|
931
|
-
for _field in fields(TextCleanerConfig):
|
|
932
|
-
if _field.name not in _NO_PLAIN_GLOBAL:
|
|
933
|
-
globals()[_field.name.upper()] = _field.default
|
|
934
|
-
|
|
935
|
-
# Lists, as they always were, so config.POSITIONAL_TAGS.append('X') keeps working.
|
|
936
|
-
POSITIONAL_TAGS = list(TextCleanerConfig.positional_tags)
|
|
937
|
-
# Order: English, Dutch, German, Spanish, Multilingual
|
|
938
|
-
NER_MODELS_LIST = list(TextCleanerConfig.ner_models_list)
|
|
@@ -17,6 +17,8 @@ from collections.abc import Iterable, Sequence
|
|
|
17
17
|
|
|
18
18
|
import regex
|
|
19
19
|
|
|
20
|
+
from sct.utils.markers import LABEL_PATTERN, PLACEHOLDER_PATTERN
|
|
21
|
+
|
|
20
22
|
if TYPE_CHECKING:
|
|
21
23
|
from sct.utils.process_result import ProcessResult
|
|
22
24
|
|
|
@@ -29,9 +31,9 @@ SCORE_A_COLUMNS = tuple(SCORE_A_TYPES)
|
|
|
29
31
|
SCORE_B_COLUMNS = ("sct_quality", "sct_minhash")
|
|
30
32
|
|
|
31
33
|
# Placeholders count as one token; reversible ``<PERSON_3>`` normalises to ``<PERSON>``.
|
|
32
|
-
_PLACEHOLDER = regex.compile(
|
|
33
|
-
_WORD_TOKEN = regex.compile(
|
|
34
|
-
_CHAR_TOKEN = regex.compile(
|
|
34
|
+
_PLACEHOLDER = regex.compile(rf"<({LABEL_PATTERN}?)(?:_\d+)?>")
|
|
35
|
+
_WORD_TOKEN = regex.compile(rf"{PLACEHOLDER_PATTERN}|\w+")
|
|
36
|
+
_CHAR_TOKEN = regex.compile(rf"{PLACEHOLDER_PATTERN}|\S")
|
|
35
37
|
# Scripts written without spaces between words get character shingles.
|
|
36
38
|
_NO_SPACE_SCRIPT = regex.compile(
|
|
37
39
|
r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Thai}\p{Lao}\p{Khmer}\p{Myanmar}]"
|