SqueakyCleanText 0.8.0__tar.gz → 0.8.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/PKG-INFO +13 -4
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/README.md +12 -3
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/SqueakyCleanText.egg-info/PKG-INFO +13 -4
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/SqueakyCleanText.egg-info/SOURCES.txt +2 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/__init__.py +1 -1
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/browser.py +13 -7
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/config.py +90 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/model_benchmark.py +1 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/presets.py +2 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/sct.py +52 -19
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/signals.py +1 -1
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/streaming.py +4 -2
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/constants.py +102 -58
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/contact.py +61 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/datetime.py +7 -2
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/ner.py +130 -8
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/normtext.py +1 -1
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/resources.py +13 -0
- squeakycleantext-0.8.1/sct/utils/special.py +127 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/synthetic.py +1 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_benchmark_release.py +10 -0
- squeakycleantext-0.8.1/tests/test_benchmarks_score.py +255 -0
- squeakycleantext-0.8.1/tests/test_codex_benchmark_round1.py +84 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_dates_default.py +40 -0
- squeakycleantext-0.8.1/tests/test_final_check_fixes.py +273 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_functional.py +19 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_regex_boundaries.py +142 -0
- squeakycleantext-0.8.0/sct/utils/special.py +0 -66
- squeakycleantext-0.8.0/tests/test_final_check_fixes.py +0 -119
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/LICENSE +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/SqueakyCleanText.egg-info/dependency_links.txt +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/SqueakyCleanText.egg-info/entry_points.txt +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/SqueakyCleanText.egg-info/requires.txt +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/SqueakyCleanText.egg-info/top_level.txt +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/pyproject.toml +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/adapters.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/cli.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/evaluation.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/integrations/__init__.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/integrations/_runtime.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/integrations/datatrove.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/integrations/haystack.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/integrations/hf.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/integrations/langchain.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/mcp_server.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/model_candidates.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/ocr.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/privacy.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/report_cli.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/risk.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/score.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/__init__.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/anonymization_map.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/gliclass_adapter.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/gliner_adapter.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/onnx_pipeline.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/process_result.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/stopwords.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/torch_pipeline.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/setup.cfg +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/setup.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_adapters.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_address_leftovers.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_batching.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_browser.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_chunking.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_cjk_width.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_cli.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round1.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round13.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round14.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round15.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round16.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round17.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round18.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round19.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round2.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round20.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round21.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round22.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round23.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round24.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round25.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round27.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round28.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round29.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round3.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round30.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round31.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round32.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round35.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round37.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round38.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round4.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round5.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round6.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round7.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round8.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round9.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_config_state.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_contracts.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_dates_cjk_styled.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_device_provenance.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_escaped_markup.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_evaluation.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_full_test_round2.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_fuzz_round1.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_fuzz_round2.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_fuzz_round3.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_fuzz_round5.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_fuzz_round6.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_fuzz_styled_pii.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_integrations_frameworks.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_integrations_hf.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_integrations_runtime.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_keep_scripts.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_language_all.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_linux_workers.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_mcp_path.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_mcp_server.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_model_benchmarks.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_multilingual_eval_set.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_ner_sees_tags.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_ocr.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_onnx_concurrency.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_onnx_external_data.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_privacy.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_public_view.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_regex_counts.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_report_cli.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_review_round1.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_review_round2.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_review_round3.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_review_round4.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_risk.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_score.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_sct.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_security_leftovers.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_stream_guard_shared_view.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_streaming.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_styled_pii.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_synthetic_per_document.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_training_preset.py +0 -0
- {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_type_names.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SqueakyCleanText
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.1
|
|
4
4
|
Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
|
|
5
5
|
Author: Rehan Fazal
|
|
6
6
|
License-Expression: MIT
|
|
@@ -277,6 +277,7 @@ Source and method: [docs/PERFORMANCE.md](https://github.com/rhnfzl/SqueakyCleanT
|
|
|
277
277
|
|
|
278
278
|
## Quality and evaluation
|
|
279
279
|
|
|
280
|
+
- Open-dataset benchmark (12 PII datasets, about 30 languages, long documents, language ID, cleaning, speed): [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks), scores per release in `benchmarks/results/`.
|
|
280
281
|
- Release benchmark (30 documents, 84 entities, 9 languages): exact-span F1 0.99 and 0 of 67 PII values leaked in 0.8.0. Run `python -m scripts.benchmark_release`.
|
|
281
282
|
- Multilingual PII eval (220 synthetic records, 11 languages): the default ensemble scores 0.93 recall and 0.95 precision. Run `python -m scripts.run_multilingual_eval` ([docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md)).
|
|
282
283
|
- Score columns and data cards: `sct-report data.jsonl --text-column text --out report.json` writes counts, shares and hashes, never text ([docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md)).
|
|
@@ -295,7 +296,7 @@ Language detection covers 75 languages. Full NER models cover 4 (English, Dutch,
|
|
|
295
296
|
- **Synthetic data is not anonymisation.** Fake values keep the data useful; they do not make it anonymous in a legal sense.
|
|
296
297
|
|
|
297
298
|
<details>
|
|
298
|
-
<summary>Known limits of 0.8.
|
|
299
|
+
<summary>Known limits of 0.8.1 (details in <a href="CHANGELOG.md">CHANGELOG.md</a>)</summary>
|
|
299
300
|
|
|
300
301
|
- Dates: only full dates become one `<DATE>`. `1990年1月`, `1月15日`, Japanese era years and `1990.1.15.` do not; the year and number steps still change them (`1990年1月` becomes `<YEAR>年<NUMBER>月`, `生日1990.1.15谢谢` becomes `<YEAR><NUMBER>.15`). A circled number glued to a unit character (`1月⑮日`) stays a list marker. Thai month names need `custom_month_names`.
|
|
301
302
|
- In ensembles with GLiNER, windows are sized for the token-classification models, not for GLiNER's own word limit. GLiNER2 runs one chunk per call; torch models and GLiNER keep one lock per model, so they do not run in parallel across threads.
|
|
@@ -304,7 +305,7 @@ Language detection covers 75 languages. Full NER models cover 4 (English, Dutch,
|
|
|
304
305
|
- `06-12345678` becomes `06-<PHONE>`. Adversarial strings such as `+000000000(000)0` can leave `(000)0` after phone replacement.
|
|
305
306
|
- Two addresses glued with no separator in a script written without spaces can be read as one address or split at the wrong place.
|
|
306
307
|
- ORGANISATION is the weakest NER label (0.75 recall in the multilingual eval); Bengali and Tamil name spans can be a few characters off.
|
|
307
|
-
- A URL
|
|
308
|
+
- A URL path that switches from Latin letters or digits straight into a script written without spaces is replaced only up to the switch (`https://x.org/a東京secret` becomes `<URL>東京secret`). Rare styled URL hosts and mail addresses (`https://ab。⑧․cd.ex.com/...`, `INFO@COL·LEGI。CAT`) are not fully replaced.
|
|
308
309
|
- Score B's quality head was tested with a small ONNX graph only; GLiNER PII mode was not run end to end for this release. CI does not install the `mcp` extra; FastMCP-only tests skip there.
|
|
309
310
|
- An unpinned NER model makes one Hub request per worker when the parent never loaded it before forking; pin `ner_model_revisions` or set `HF_HUB_OFFLINE=1`.
|
|
310
311
|
- With an empty regex marker (`replace_with_email=""`), NER in synthetic or reversible mode can find other entities than in placeholder mode.
|
|
@@ -328,6 +329,14 @@ Language detection covers 75 languages. Full NER models cover 4 (English, Dutch,
|
|
|
328
329
|
| Languages and adding a language | [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md), [docs/ADDING_LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/ADDING_LANGUAGES.md) |
|
|
329
330
|
| The whole API in one file, for AI coding assistants | [llms.txt](https://github.com/rhnfzl/SqueakyCleanText/blob/main/llms.txt) |
|
|
330
331
|
|
|
332
|
+
## What's new in 0.8.1
|
|
333
|
+
|
|
334
|
+
Fixes from the open-dataset benchmark ([benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks)). Leak falls on all 12 PII datasets (REDACT 18.1% to 8.2%, long documents 42.4% to 11.7%); speed is unchanged.
|
|
335
|
+
|
|
336
|
+
- NER keeps a span when its best model passes the cut; other copies of a masked name are masked too.
|
|
337
|
+
- IBAN and card numbers get their own types; JSON and longer bracketed text survive cleanup.
|
|
338
|
+
- `Month/YY` dates, `Name <email>`, URLs before a full stop, and span edges inside words are fixed.
|
|
339
|
+
|
|
331
340
|
## What's new in 0.8.0
|
|
332
341
|
|
|
333
342
|
See [CHANGELOG.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/CHANGELOG.md) (many breaking changes). Highlights of 0.8.0:
|
|
@@ -356,7 +365,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
|
|
|
356
365
|
author = {Fazal, Rehan},
|
|
357
366
|
title = {SqueakyCleanText},
|
|
358
367
|
url = {https://github.com/rhnfzl/SqueakyCleanText},
|
|
359
|
-
version = {0.8.
|
|
368
|
+
version = {0.8.1},
|
|
360
369
|
year = {2026}
|
|
361
370
|
}
|
|
362
371
|
```
|
|
@@ -189,6 +189,7 @@ Source and method: [docs/PERFORMANCE.md](https://github.com/rhnfzl/SqueakyCleanT
|
|
|
189
189
|
|
|
190
190
|
## Quality and evaluation
|
|
191
191
|
|
|
192
|
+
- Open-dataset benchmark (12 PII datasets, about 30 languages, long documents, language ID, cleaning, speed): [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks), scores per release in `benchmarks/results/`.
|
|
192
193
|
- Release benchmark (30 documents, 84 entities, 9 languages): exact-span F1 0.99 and 0 of 67 PII values leaked in 0.8.0. Run `python -m scripts.benchmark_release`.
|
|
193
194
|
- Multilingual PII eval (220 synthetic records, 11 languages): the default ensemble scores 0.93 recall and 0.95 precision. Run `python -m scripts.run_multilingual_eval` ([docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md)).
|
|
194
195
|
- Score columns and data cards: `sct-report data.jsonl --text-column text --out report.json` writes counts, shares and hashes, never text ([docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md)).
|
|
@@ -207,7 +208,7 @@ Language detection covers 75 languages. Full NER models cover 4 (English, Dutch,
|
|
|
207
208
|
- **Synthetic data is not anonymisation.** Fake values keep the data useful; they do not make it anonymous in a legal sense.
|
|
208
209
|
|
|
209
210
|
<details>
|
|
210
|
-
<summary>Known limits of 0.8.
|
|
211
|
+
<summary>Known limits of 0.8.1 (details in <a href="CHANGELOG.md">CHANGELOG.md</a>)</summary>
|
|
211
212
|
|
|
212
213
|
- Dates: only full dates become one `<DATE>`. `1990年1月`, `1月15日`, Japanese era years and `1990.1.15.` do not; the year and number steps still change them (`1990年1月` becomes `<YEAR>年<NUMBER>月`, `生日1990.1.15谢谢` becomes `<YEAR><NUMBER>.15`). A circled number glued to a unit character (`1月⑮日`) stays a list marker. Thai month names need `custom_month_names`.
|
|
213
214
|
- In ensembles with GLiNER, windows are sized for the token-classification models, not for GLiNER's own word limit. GLiNER2 runs one chunk per call; torch models and GLiNER keep one lock per model, so they do not run in parallel across threads.
|
|
@@ -216,7 +217,7 @@ Language detection covers 75 languages. Full NER models cover 4 (English, Dutch,
|
|
|
216
217
|
- `06-12345678` becomes `06-<PHONE>`. Adversarial strings such as `+000000000(000)0` can leave `(000)0` after phone replacement.
|
|
217
218
|
- Two addresses glued with no separator in a script written without spaces can be read as one address or split at the wrong place.
|
|
218
219
|
- ORGANISATION is the weakest NER label (0.75 recall in the multilingual eval); Bengali and Tamil name spans can be a few characters off.
|
|
219
|
-
- A URL
|
|
220
|
+
- A URL path that switches from Latin letters or digits straight into a script written without spaces is replaced only up to the switch (`https://x.org/a東京secret` becomes `<URL>東京secret`). Rare styled URL hosts and mail addresses (`https://ab。⑧․cd.ex.com/...`, `INFO@COL·LEGI。CAT`) are not fully replaced.
|
|
220
221
|
- Score B's quality head was tested with a small ONNX graph only; GLiNER PII mode was not run end to end for this release. CI does not install the `mcp` extra; FastMCP-only tests skip there.
|
|
221
222
|
- An unpinned NER model makes one Hub request per worker when the parent never loaded it before forking; pin `ner_model_revisions` or set `HF_HUB_OFFLINE=1`.
|
|
222
223
|
- With an empty regex marker (`replace_with_email=""`), NER in synthetic or reversible mode can find other entities than in placeholder mode.
|
|
@@ -240,6 +241,14 @@ Language detection covers 75 languages. Full NER models cover 4 (English, Dutch,
|
|
|
240
241
|
| Languages and adding a language | [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md), [docs/ADDING_LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/ADDING_LANGUAGES.md) |
|
|
241
242
|
| The whole API in one file, for AI coding assistants | [llms.txt](https://github.com/rhnfzl/SqueakyCleanText/blob/main/llms.txt) |
|
|
242
243
|
|
|
244
|
+
## What's new in 0.8.1
|
|
245
|
+
|
|
246
|
+
Fixes from the open-dataset benchmark ([benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks)). Leak falls on all 12 PII datasets (REDACT 18.1% to 8.2%, long documents 42.4% to 11.7%); speed is unchanged.
|
|
247
|
+
|
|
248
|
+
- NER keeps a span when its best model passes the cut; other copies of a masked name are masked too.
|
|
249
|
+
- IBAN and card numbers get their own types; JSON and longer bracketed text survive cleanup.
|
|
250
|
+
- `Month/YY` dates, `Name <email>`, URLs before a full stop, and span edges inside words are fixed.
|
|
251
|
+
|
|
243
252
|
## What's new in 0.8.0
|
|
244
253
|
|
|
245
254
|
See [CHANGELOG.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/CHANGELOG.md) (many breaking changes). Highlights of 0.8.0:
|
|
@@ -268,7 +277,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
|
|
|
268
277
|
author = {Fazal, Rehan},
|
|
269
278
|
title = {SqueakyCleanText},
|
|
270
279
|
url = {https://github.com/rhnfzl/SqueakyCleanText},
|
|
271
|
-
version = {0.8.
|
|
280
|
+
version = {0.8.1},
|
|
272
281
|
year = {2026}
|
|
273
282
|
}
|
|
274
283
|
```
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SqueakyCleanText
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.1
|
|
4
4
|
Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
|
|
5
5
|
Author: Rehan Fazal
|
|
6
6
|
License-Expression: MIT
|
|
@@ -277,6 +277,7 @@ Source and method: [docs/PERFORMANCE.md](https://github.com/rhnfzl/SqueakyCleanT
|
|
|
277
277
|
|
|
278
278
|
## Quality and evaluation
|
|
279
279
|
|
|
280
|
+
- Open-dataset benchmark (12 PII datasets, about 30 languages, long documents, language ID, cleaning, speed): [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks), scores per release in `benchmarks/results/`.
|
|
280
281
|
- Release benchmark (30 documents, 84 entities, 9 languages): exact-span F1 0.99 and 0 of 67 PII values leaked in 0.8.0. Run `python -m scripts.benchmark_release`.
|
|
281
282
|
- Multilingual PII eval (220 synthetic records, 11 languages): the default ensemble scores 0.93 recall and 0.95 precision. Run `python -m scripts.run_multilingual_eval` ([docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md)).
|
|
282
283
|
- Score columns and data cards: `sct-report data.jsonl --text-column text --out report.json` writes counts, shares and hashes, never text ([docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md)).
|
|
@@ -295,7 +296,7 @@ Language detection covers 75 languages. Full NER models cover 4 (English, Dutch,
|
|
|
295
296
|
- **Synthetic data is not anonymisation.** Fake values keep the data useful; they do not make it anonymous in a legal sense.
|
|
296
297
|
|
|
297
298
|
<details>
|
|
298
|
-
<summary>Known limits of 0.8.
|
|
299
|
+
<summary>Known limits of 0.8.1 (details in <a href="CHANGELOG.md">CHANGELOG.md</a>)</summary>
|
|
299
300
|
|
|
300
301
|
- Dates: only full dates become one `<DATE>`. `1990年1月`, `1月15日`, Japanese era years and `1990.1.15.` do not; the year and number steps still change them (`1990年1月` becomes `<YEAR>年<NUMBER>月`, `生日1990.1.15谢谢` becomes `<YEAR><NUMBER>.15`). A circled number glued to a unit character (`1月⑮日`) stays a list marker. Thai month names need `custom_month_names`.
|
|
301
302
|
- In ensembles with GLiNER, windows are sized for the token-classification models, not for GLiNER's own word limit. GLiNER2 runs one chunk per call; torch models and GLiNER keep one lock per model, so they do not run in parallel across threads.
|
|
@@ -304,7 +305,7 @@ Language detection covers 75 languages. Full NER models cover 4 (English, Dutch,
|
|
|
304
305
|
- `06-12345678` becomes `06-<PHONE>`. Adversarial strings such as `+000000000(000)0` can leave `(000)0` after phone replacement.
|
|
305
306
|
- Two addresses glued with no separator in a script written without spaces can be read as one address or split at the wrong place.
|
|
306
307
|
- ORGANISATION is the weakest NER label (0.75 recall in the multilingual eval); Bengali and Tamil name spans can be a few characters off.
|
|
307
|
-
- A URL
|
|
308
|
+
- A URL path that switches from Latin letters or digits straight into a script written without spaces is replaced only up to the switch (`https://x.org/a東京secret` becomes `<URL>東京secret`). Rare styled URL hosts and mail addresses (`https://ab。⑧․cd.ex.com/...`, `INFO@COL·LEGI。CAT`) are not fully replaced.
|
|
308
309
|
- Score B's quality head was tested with a small ONNX graph only; GLiNER PII mode was not run end to end for this release. CI does not install the `mcp` extra; FastMCP-only tests skip there.
|
|
309
310
|
- An unpinned NER model makes one Hub request per worker when the parent never loaded it before forking; pin `ner_model_revisions` or set `HF_HUB_OFFLINE=1`.
|
|
310
311
|
- With an empty regex marker (`replace_with_email=""`), NER in synthetic or reversible mode can find other entities than in placeholder mode.
|
|
@@ -328,6 +329,14 @@ Language detection covers 75 languages. Full NER models cover 4 (English, Dutch,
|
|
|
328
329
|
| Languages and adding a language | [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md), [docs/ADDING_LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/ADDING_LANGUAGES.md) |
|
|
329
330
|
| The whole API in one file, for AI coding assistants | [llms.txt](https://github.com/rhnfzl/SqueakyCleanText/blob/main/llms.txt) |
|
|
330
331
|
|
|
332
|
+
## What's new in 0.8.1
|
|
333
|
+
|
|
334
|
+
Fixes from the open-dataset benchmark ([benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks)). Leak falls on all 12 PII datasets (REDACT 18.1% to 8.2%, long documents 42.4% to 11.7%); speed is unchanged.
|
|
335
|
+
|
|
336
|
+
- NER keeps a span when its best model passes the cut; other copies of a masked name are masked too.
|
|
337
|
+
- IBAN and card numbers get their own types; JSON and longer bracketed text survive cleanup.
|
|
338
|
+
- `Month/YY` dates, `Name <email>`, URLs before a full stop, and span edges inside words are fixed.
|
|
339
|
+
|
|
331
340
|
## What's new in 0.8.0
|
|
332
341
|
|
|
333
342
|
See [CHANGELOG.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/CHANGELOG.md) (many breaking changes). Highlights of 0.8.0:
|
|
@@ -356,7 +365,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
|
|
|
356
365
|
author = {Fazal, Rehan},
|
|
357
366
|
title = {SqueakyCleanText},
|
|
358
367
|
url = {https://github.com/rhnfzl/SqueakyCleanText},
|
|
359
|
-
version = {0.8.
|
|
368
|
+
version = {0.8.1},
|
|
360
369
|
year = {2026}
|
|
361
370
|
}
|
|
362
371
|
```
|
|
@@ -52,10 +52,12 @@ tests/test_adapters.py
|
|
|
52
52
|
tests/test_address_leftovers.py
|
|
53
53
|
tests/test_batching.py
|
|
54
54
|
tests/test_benchmark_release.py
|
|
55
|
+
tests/test_benchmarks_score.py
|
|
55
56
|
tests/test_browser.py
|
|
56
57
|
tests/test_chunking.py
|
|
57
58
|
tests/test_cjk_width.py
|
|
58
59
|
tests/test_cli.py
|
|
60
|
+
tests/test_codex_benchmark_round1.py
|
|
59
61
|
tests/test_codex_full_round1.py
|
|
60
62
|
tests/test_codex_full_round13.py
|
|
61
63
|
tests/test_codex_full_round14.py
|
|
@@ -50,7 +50,7 @@ from sct.streaming import (
|
|
|
50
50
|
from sct.utils.anonymization_map import AnonymizationMap, MapEntry
|
|
51
51
|
from sct.utils.process_result import ProcessResult
|
|
52
52
|
|
|
53
|
-
__version__ = "0.8.
|
|
53
|
+
__version__ = "0.8.1"
|
|
54
54
|
__all__ = [
|
|
55
55
|
"TextCleaner", "TextCleanerConfig", "PRESETS", "preset_config",
|
|
56
56
|
"PII_LABELS", "PII_LABEL_MAP",
|
|
@@ -6,7 +6,7 @@ from pathlib import Path
|
|
|
6
6
|
from typing import Any
|
|
7
7
|
|
|
8
8
|
from sct.config import TextCleanerConfig
|
|
9
|
-
from sct.utils import constants
|
|
9
|
+
from sct.utils import constants, resources
|
|
10
10
|
from sct.utils.contact import numbers_regex
|
|
11
11
|
|
|
12
12
|
|
|
@@ -54,12 +54,17 @@ def build_browser_manifest(config: TextCleanerConfig) -> dict[str, Any]:
|
|
|
54
54
|
|
|
55
55
|
# Same reason for any token a later enabled rule would match
|
|
56
56
|
# (an HTML token holding an email or URL, a URL token holding an email).
|
|
57
|
+
html_regex = constants.HTML_REGEX if config.html_tag_strict else constants.HTML_REGEX_ANY
|
|
58
|
+
url_regex = constants.URL_REGEX if config.url_trailing_punct_outside else constants.URL_REGEX_LEGACY
|
|
59
|
+
month_names = resources.configured_month_names(config)
|
|
60
|
+
date_regex = (constants.DATE_REGEX if month_names is None
|
|
61
|
+
else constants.build_date_regex(month_names, slash_year=config.date_month_name_slash_year))
|
|
57
62
|
ordered = [
|
|
58
|
-
("html",
|
|
59
|
-
("url",
|
|
63
|
+
("html", html_regex, config.replace_with_html, config.check_replace_html),
|
|
64
|
+
("url", url_regex, config.replace_with_url, config.check_replace_urls),
|
|
60
65
|
("email", constants.EMAIL_REGEX, config.replace_with_email, config.check_replace_emails),
|
|
61
66
|
("ip", constants.IP_REGEX, config.replace_with_ips, config.check_replace_ips),
|
|
62
|
-
("date",
|
|
67
|
+
("date", date_regex, config.replace_with_dates, config.check_replace_dates),
|
|
63
68
|
("phone", constants.PHONE_REGEX, config.replace_with_phone_numbers,
|
|
64
69
|
config.check_replace_phone_numbers),
|
|
65
70
|
("year", constants.YEAR_REGEX, config.replace_with_years, config.check_replace_years),
|
|
@@ -105,12 +110,12 @@ def build_browser_manifest(config: TextCleanerConfig) -> dict[str, Any]:
|
|
|
105
110
|
],
|
|
106
111
|
"rules": {
|
|
107
112
|
"html": _rule(
|
|
108
|
-
|
|
113
|
+
html_regex,
|
|
109
114
|
config.replace_with_html,
|
|
110
115
|
config.check_replace_html,
|
|
111
116
|
),
|
|
112
117
|
"url": _rule(
|
|
113
|
-
|
|
118
|
+
url_regex,
|
|
114
119
|
config.replace_with_url,
|
|
115
120
|
config.check_replace_urls,
|
|
116
121
|
),
|
|
@@ -125,7 +130,7 @@ def build_browser_manifest(config: TextCleanerConfig) -> dict[str, Any]:
|
|
|
125
130
|
config.check_replace_ips,
|
|
126
131
|
), "fold_decimal_digits": True},
|
|
127
132
|
"date": {**_rule(
|
|
128
|
-
|
|
133
|
+
date_regex,
|
|
129
134
|
config.replace_with_dates,
|
|
130
135
|
config.check_replace_dates,
|
|
131
136
|
), "fold_decimal_digits": True},
|
|
@@ -155,6 +160,7 @@ def build_browser_manifest(config: TextCleanerConfig) -> dict[str, Any]:
|
|
|
155
160
|
"Regular expressions require JavaScript compatibility tests.",
|
|
156
161
|
"Model files remain external and must keep their recorded revisions.",
|
|
157
162
|
"Reversible and synthetic replacement are server-side features.",
|
|
163
|
+
"IBAN/card checksum typing and identifier tokens are server-side rules.",
|
|
158
164
|
"Rules with fold_decimal_digits match on a copy of the text where every Unicode "
|
|
159
165
|
"decimal digit (Nd, such as ٠٦ or ०६) is its ASCII digit, one code point for one, "
|
|
160
166
|
"and replace the same offsets in the original text (JavaScript \\d is ASCII only).",
|
|
@@ -357,6 +357,47 @@ class TextCleanerConfig:
|
|
|
357
357
|
# synthetic: ISO date that date fakes (DATE, DATE_OF_BIRTH) end on, so a document gets the
|
|
358
358
|
# same dates on any day; None = today, from the clock (earlier 0.8.0)
|
|
359
359
|
synthetic_reference_date: Optional[str] = '2026-01-01'
|
|
360
|
+
# NER ensemble vote per span: 'max' keeps a span when its best model score passes
|
|
361
|
+
# ner_confidence_threshold, so a weak second model never hides a strong hit; 'mean' averages
|
|
362
|
+
# the model scores (0.8.0)
|
|
363
|
+
ner_ensemble_vote: str = 'max'
|
|
364
|
+
# mask every other case-sensitive whole-word copy of a masked PER/ORG/LOC value in the same
|
|
365
|
+
# document (the value comes from the model's own span); False = 0.8.0 output
|
|
366
|
+
ner_propagate_masked: bool = True
|
|
367
|
+
# score a span needs before its value is propagated (0..1); None = the effective NER cut.
|
|
368
|
+
# Raising it spreads fewer wrong tags at the cost of more visible repeats
|
|
369
|
+
ner_propagate_min_score: Optional[float] = None
|
|
370
|
+
# IBANs (mod-97 checksum) and payment cards (Luhn checksum) become one <IBAN> / <CARD> each,
|
|
371
|
+
# before the phone step; False = 0.8.0 (the phone/number steps take pieces of them)
|
|
372
|
+
check_replace_iban_cards: bool = True
|
|
373
|
+
replace_with_iban: str = "<IBAN>"
|
|
374
|
+
replace_with_cards: str = "<CARD>"
|
|
375
|
+
# Identifier tokens: 6+ uppercase letters and digits with 2+ digits and a letter (RBCON68T08).
|
|
376
|
+
# Off by default: the shape also takes product and reference codes (RTX4090, COVID19)
|
|
377
|
+
check_replace_identifiers: bool = False
|
|
378
|
+
replace_with_identifiers: str = "<ID>"
|
|
379
|
+
# Bracket/brace removal only takes innermost spans of at most this many characters and
|
|
380
|
+
# whitespace-separated words, so prose clauses stay; 0 = no limit (both 0 = 0.8.0 matching)
|
|
381
|
+
bracket_content_max_chars: int = 32
|
|
382
|
+
bracket_content_max_words: int = 3
|
|
383
|
+
# A {...} that parses as a JSON object (json.raw_decode, nested ones included) is never removed
|
|
384
|
+
keep_json_braces: bool = True
|
|
385
|
+
# A month name, "/" and a 2- or 4-digit year is one <DATE> (October/55, Oct/2019)
|
|
386
|
+
date_month_name_slash_year: bool = True
|
|
387
|
+
# ISO codes of the built-in month data dates use (en nl de es fr pt it th); None = all
|
|
388
|
+
date_builtin_month_languages: Optional[Tuple[str, ...]] = None
|
|
389
|
+
# HTML rule reads only tag shapes (<b>, </a>, <br/>, <!-- -->); False = 0.8.0 "<.*?>" (eats "<a@b.com>")
|
|
390
|
+
html_tag_strict: bool = True
|
|
391
|
+
# join two LOCATION spans whose gap has <= N chars (placeholders not counted), no line break
|
|
392
|
+
# and no sentence end, so an address is one <LOCATION>; 0 = off (0.8.0 output)
|
|
393
|
+
ner_merge_location_gap: int = 0
|
|
394
|
+
# NER span edges cut inside a word (letters and marks, not wide CJK) move out to the word
|
|
395
|
+
# edges on every backend ("Salut" -> "Salute"), within the ner_subword_widen cap and
|
|
396
|
+
# same-script rule; False = 0.8.0 spans
|
|
397
|
+
ner_snap_word_edges: bool = True
|
|
398
|
+
# . , ; : ! ? ' end a URL path or query unless more URL text follows ("https://x.com/a. Next");
|
|
399
|
+
# False = 0.8.0, where the URL takes the sentence's full stop or comma
|
|
400
|
+
url_trailing_punct_outside: bool = True
|
|
360
401
|
|
|
361
402
|
def __post_init__(self):
|
|
362
403
|
if self.synthetic_reference_date is not None:
|
|
@@ -475,6 +516,12 @@ class TextCleanerConfig:
|
|
|
475
516
|
value = getattr(self, name)
|
|
476
517
|
if value is not None and not 0.0 <= value <= 1.0:
|
|
477
518
|
raise ValueError(f"{name} must be None or in [0, 1], got {value}")
|
|
519
|
+
if self.ner_ensemble_vote not in ('max', 'mean'):
|
|
520
|
+
raise ValueError(f"ner_ensemble_vote must be 'max' or 'mean', got {self.ner_ensemble_vote!r}")
|
|
521
|
+
if self.ner_merge_location_gap < 0:
|
|
522
|
+
raise ValueError(f"ner_merge_location_gap must be >= 0, got {self.ner_merge_location_gap}")
|
|
523
|
+
if self.ner_propagate_min_score is not None and not 0.0 <= self.ner_propagate_min_score <= 1.0:
|
|
524
|
+
raise ValueError(f"ner_propagate_min_score must be None or in [0, 1], got {self.ner_propagate_min_score}")
|
|
478
525
|
if self.ner_subword_widen is not None and self.ner_subword_widen < 0:
|
|
479
526
|
raise ValueError(f"ner_subword_widen must be None or >= 0, got {self.ner_subword_widen}")
|
|
480
527
|
if not 0.0 <= self.language_confidence_warning <= 1.0:
|
|
@@ -614,6 +661,15 @@ class TextCleanerConfig:
|
|
|
614
661
|
object.__setattr__(self, 'custom_stopwords',
|
|
615
662
|
MappingProxyType(resolved_sw))
|
|
616
663
|
|
|
664
|
+
if self.date_builtin_month_languages is not None:
|
|
665
|
+
from sct.utils.constants import MONTH_NAMES_MULTILINGUAL
|
|
666
|
+
codes = tuple(self.date_builtin_month_languages)
|
|
667
|
+
unknown = set(codes) - set().union(*MONTH_NAMES_MULTILINGUAL.values())
|
|
668
|
+
if isinstance(self.date_builtin_month_languages, str) or unknown:
|
|
669
|
+
raise ValueError(f"date_builtin_month_languages must be a tuple of built-in month ISO codes, "
|
|
670
|
+
f"got {self.date_builtin_month_languages!r}")
|
|
671
|
+
object.__setattr__(self, 'date_builtin_month_languages', codes)
|
|
672
|
+
|
|
617
673
|
# Validate and freeze custom_month_names
|
|
618
674
|
if self.custom_month_names:
|
|
619
675
|
resolved_mn = {resolve_language(k): tuple(v) for k, v in self.custom_month_names.items()}
|
|
@@ -726,6 +782,23 @@ def _config_from_module_globals() -> TextCleanerConfig:
|
|
|
726
782
|
synthetic_low_confidence_locale=getattr(m, 'SYNTHETIC_LOW_CONFIDENCE_LOCALE', 'en_US'),
|
|
727
783
|
synthetic_collapse_period=getattr(m, 'SYNTHETIC_COLLAPSE_PERIOD', True),
|
|
728
784
|
synthetic_reference_date=getattr(m, 'SYNTHETIC_REFERENCE_DATE', '2026-01-01'),
|
|
785
|
+
ner_ensemble_vote=getattr(m, 'NER_ENSEMBLE_VOTE', 'max'),
|
|
786
|
+
ner_propagate_masked=getattr(m, 'NER_PROPAGATE_MASKED', True),
|
|
787
|
+
ner_propagate_min_score=getattr(m, 'NER_PROPAGATE_MIN_SCORE', None),
|
|
788
|
+
check_replace_iban_cards=getattr(m, 'CHECK_REPLACE_IBAN_CARDS', True),
|
|
789
|
+
replace_with_iban=getattr(m, 'REPLACE_WITH_IBAN', '<IBAN>'),
|
|
790
|
+
replace_with_cards=getattr(m, 'REPLACE_WITH_CARDS', '<CARD>'),
|
|
791
|
+
check_replace_identifiers=getattr(m, 'CHECK_REPLACE_IDENTIFIERS', False),
|
|
792
|
+
replace_with_identifiers=getattr(m, 'REPLACE_WITH_IDENTIFIERS', '<ID>'),
|
|
793
|
+
bracket_content_max_chars=getattr(m, 'BRACKET_CONTENT_MAX_CHARS', 32),
|
|
794
|
+
bracket_content_max_words=getattr(m, 'BRACKET_CONTENT_MAX_WORDS', 3),
|
|
795
|
+
keep_json_braces=getattr(m, 'KEEP_JSON_BRACES', True),
|
|
796
|
+
date_month_name_slash_year=getattr(m, 'DATE_MONTH_NAME_SLASH_YEAR', True),
|
|
797
|
+
date_builtin_month_languages=getattr(m, 'DATE_BUILTIN_MONTH_LANGUAGES', None),
|
|
798
|
+
html_tag_strict=getattr(m, 'HTML_TAG_STRICT', True),
|
|
799
|
+
ner_merge_location_gap=getattr(m, 'NER_MERGE_LOCATION_GAP', 0),
|
|
800
|
+
ner_snap_word_edges=getattr(m, 'NER_SNAP_WORD_EDGES', True),
|
|
801
|
+
url_trailing_punct_outside=getattr(m, 'URL_TRAILING_PUNCT_OUTSIDE', True),
|
|
729
802
|
check_replace_currency_symbols=m.CHECK_REPLACE_CURRENCY_SYMBOLS,
|
|
730
803
|
check_ner_process=m.CHECK_NER_PROCESS,
|
|
731
804
|
check_remove_isolated_letters=m.CHECK_REMOVE_ISOLATED_LETTERS,
|
|
@@ -834,6 +907,23 @@ NER_SEES_TAGS = True
|
|
|
834
907
|
SYNTHETIC_LOW_CONFIDENCE_LOCALE = 'en_US'
|
|
835
908
|
SYNTHETIC_COLLAPSE_PERIOD = True
|
|
836
909
|
SYNTHETIC_REFERENCE_DATE = '2026-01-01'
|
|
910
|
+
NER_ENSEMBLE_VOTE = 'max'
|
|
911
|
+
NER_PROPAGATE_MASKED = True
|
|
912
|
+
NER_PROPAGATE_MIN_SCORE = None
|
|
913
|
+
CHECK_REPLACE_IBAN_CARDS = True
|
|
914
|
+
REPLACE_WITH_IBAN = "<IBAN>"
|
|
915
|
+
REPLACE_WITH_CARDS = "<CARD>"
|
|
916
|
+
CHECK_REPLACE_IDENTIFIERS = False
|
|
917
|
+
REPLACE_WITH_IDENTIFIERS = "<ID>"
|
|
918
|
+
BRACKET_CONTENT_MAX_CHARS = 32
|
|
919
|
+
BRACKET_CONTENT_MAX_WORDS = 3
|
|
920
|
+
KEEP_JSON_BRACES = True
|
|
921
|
+
DATE_MONTH_NAME_SLASH_YEAR = True
|
|
922
|
+
DATE_BUILTIN_MONTH_LANGUAGES = None
|
|
923
|
+
HTML_TAG_STRICT = True
|
|
924
|
+
NER_MERGE_LOCATION_GAP = 0
|
|
925
|
+
NER_SNAP_WORD_EDGES = True
|
|
926
|
+
URL_TRAILING_PUNCT_OUTSIDE = True
|
|
837
927
|
GLINER_ONNX = False
|
|
838
928
|
LANGUAGE = None
|
|
839
929
|
|
|
@@ -60,6 +60,7 @@ def candidate_config(candidate: ModelCandidate) -> TextCleanerConfig:
|
|
|
60
60
|
"check_replace_years": False,
|
|
61
61
|
"check_replace_dates": False,
|
|
62
62
|
"check_replace_phone_numbers": False,
|
|
63
|
+
"check_replace_iban_cards": False,
|
|
63
64
|
"check_replace_numbers": False,
|
|
64
65
|
"check_replace_currency_symbols": False,
|
|
65
66
|
"check_remove_isolated_letters": False,
|
|
@@ -17,12 +17,14 @@ _TRAINING = {
|
|
|
17
17
|
'check_replace_emails': True,
|
|
18
18
|
'check_replace_phone_numbers': True,
|
|
19
19
|
'check_replace_ips': True,
|
|
20
|
+
'check_replace_iban_cards': True,
|
|
20
21
|
'check_ner_process': True,
|
|
21
22
|
'check_fix_bad_unicode': True,
|
|
22
23
|
'check_detect_language': True,
|
|
23
24
|
# Off
|
|
24
25
|
'check_to_ascii_unicode': False,
|
|
25
26
|
'check_replace_numbers': False,
|
|
27
|
+
'check_replace_identifiers': False, # takes product/reference codes too; not measured on training data
|
|
26
28
|
'check_replace_years': False,
|
|
27
29
|
'check_replace_dates': False,
|
|
28
30
|
'check_fuzzy_replace_dates': False,
|
|
@@ -62,6 +62,8 @@ class TextCleaner:
|
|
|
62
62
|
'DATE': self.cfg.replace_with_dates, 'YEAR': self.cfg.replace_with_years,
|
|
63
63
|
self._phone_type: self.cfg.replace_with_phone_numbers,
|
|
64
64
|
'NUMBER': self.cfg.replace_with_numbers, 'CURRENCY': self.cfg.replace_with_currency_symbols,
|
|
65
|
+
'IBAN': self.cfg.replace_with_iban, 'CARD': self.cfg.replace_with_cards,
|
|
66
|
+
'ID': self.cfg.replace_with_identifiers,
|
|
65
67
|
}
|
|
66
68
|
# Markers that are not <TAG>-shaped ("EMAIL"): NER protects them like <EMAIL> (ner.regex_markers).
|
|
67
69
|
self._custom_markers = tuple(sorted({
|
|
@@ -86,17 +88,15 @@ class TextCleaner:
|
|
|
86
88
|
)
|
|
87
89
|
|
|
88
90
|
# Instance-scoped datetime (with custom month names -> per-instance regex)
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
mn_dict = constants.build_month_names_dict(extra)
|
|
95
|
-
date_regex = constants.build_date_regex(mn_dict)
|
|
91
|
+
custom_months = self.cfg.custom_month_names or {}
|
|
92
|
+
slash_year = self.cfg.date_month_name_slash_year
|
|
93
|
+
mn_dict = resources.configured_month_names(self.cfg)
|
|
94
|
+
if mn_dict is not None:
|
|
95
|
+
date_regex = constants.build_date_regex(mn_dict, slash_year=slash_year)
|
|
96
96
|
fuzzy_all, fuzzy_by_lang = constants.build_fuzzy_vocabulary(
|
|
97
97
|
mn_dict,
|
|
98
98
|
{**constants._LANG_TO_VOCAB,
|
|
99
|
-
**{lang: resources.language_to_iso(lang) for lang in
|
|
99
|
+
**{lang: resources.language_to_iso(lang) for lang in custom_months}},
|
|
100
100
|
)
|
|
101
101
|
self.ProcessDateTime = datetime.ProcessDateTime(
|
|
102
102
|
date_regex=date_regex,
|
|
@@ -108,10 +108,13 @@ class TextCleaner:
|
|
|
108
108
|
# fold_styled_characters: dates, like phones, read any space as " " and any dash as "-" (1987\u201105\u201112),
|
|
109
109
|
# any dot form as "." (1990﹒ 1﹒ 15﹒), and a month-name date's day may be a circled number (⑮ January 1990).
|
|
110
110
|
self._date_view = (
|
|
111
|
-
contact.DigitView(constants.build_date_regex(mn_dict, styled=True
|
|
111
|
+
contact.DigitView(constants.build_date_regex(mn_dict, styled=True, slash_year=slash_year),
|
|
112
|
+
contact.DATE_SEPARATOR_VIEW)
|
|
112
113
|
if self.cfg.fold_styled_characters else self.ProcessDateTime._date_regex)
|
|
113
114
|
|
|
114
115
|
self.ProcessContacts = contact.ProcessContacts()
|
|
116
|
+
self._html_regex = constants.HTML_REGEX if self.cfg.html_tag_strict else constants.HTML_REGEX_ANY
|
|
117
|
+
self._url_regex = constants.URL_REGEX if self.cfg.url_trailing_punct_outside else constants.URL_REGEX_LEGACY
|
|
115
118
|
self.ProcessSpecialSymbols = special.ProcessSpecialSymbols()
|
|
116
119
|
self.NormaliseText = normtext.NormaliseText()
|
|
117
120
|
|
|
@@ -190,6 +193,11 @@ class TextCleaner:
|
|
|
190
193
|
onnx_intra_op_threads=self.cfg.onnx_intra_op_threads,
|
|
191
194
|
onnx_inter_op_threads=self.cfg.onnx_inter_op_threads,
|
|
192
195
|
ner_chunk_overlap=self.cfg.ner_chunk_overlap,
|
|
196
|
+
ner_ensemble_vote=self.cfg.ner_ensemble_vote,
|
|
197
|
+
ner_propagate_masked=self.cfg.ner_propagate_masked,
|
|
198
|
+
ner_propagate_min_score=self.cfg.ner_propagate_min_score,
|
|
199
|
+
ner_merge_location_gap=self.cfg.ner_merge_location_gap,
|
|
200
|
+
ner_snap_word_edges=self.cfg.ner_snap_word_edges,
|
|
193
201
|
)
|
|
194
202
|
# GLiClass document-level pre-classification (optional, lazy-loaded)
|
|
195
203
|
self._gliclass: Any = None
|
|
@@ -245,6 +253,11 @@ class TextCleaner:
|
|
|
245
253
|
# fuzzy_replace_dates runs here (between replace_dates and replace_years)
|
|
246
254
|
# but is called explicitly in _process_single with language context.
|
|
247
255
|
# Phones before years, so a year step cannot split "+31 6 1234 2019".
|
|
256
|
+
# Checksum-valid IBANs/cards and identifier tokens before phones, so a phone cannot take their digits.
|
|
257
|
+
if cfg.check_replace_iban_cards:
|
|
258
|
+
self._post_fuzzy_pipeline.append(self._replace_iban_cards)
|
|
259
|
+
if cfg.check_replace_identifiers:
|
|
260
|
+
self._post_fuzzy_pipeline.append(self._replace_identifiers)
|
|
248
261
|
if cfg.check_replace_phone_numbers:
|
|
249
262
|
self._post_fuzzy_pipeline.append(self._replace_phone_numbers)
|
|
250
263
|
if cfg.check_replace_years:
|
|
@@ -843,32 +856,30 @@ class TextCleaner:
|
|
|
843
856
|
def _replace_html(self, text):
|
|
844
857
|
reversible = self._replace_reversible(
|
|
845
858
|
text,
|
|
846
|
-
|
|
859
|
+
self._html_regex,
|
|
847
860
|
'HTML',
|
|
848
861
|
)
|
|
849
862
|
if reversible is not None:
|
|
850
863
|
return reversible
|
|
851
|
-
return self.
|
|
852
|
-
text, replace_with=lambda m: self._emit('HTML', m.group(), self.cfg.replace_with_html),
|
|
853
|
-
)
|
|
864
|
+
return self._html_regex.sub(lambda m: self._emit('HTML', m.group(), self.cfg.replace_with_html), text)
|
|
854
865
|
|
|
855
866
|
def _replace_urls(self, text):
|
|
856
867
|
reversible = self._replace_reversible(
|
|
857
868
|
text,
|
|
858
|
-
|
|
869
|
+
self._url_regex,
|
|
859
870
|
'URL',
|
|
860
871
|
)
|
|
861
872
|
if reversible is not None:
|
|
862
873
|
return reversible
|
|
863
874
|
if self._synthetic_replacer:
|
|
864
|
-
return self.
|
|
865
|
-
|
|
875
|
+
return self._url_regex.sub(
|
|
876
|
+
lambda m: self._emit('URL', m.group(), self._synthetic_replacer.get_replacement('URL', m.group())), text,
|
|
866
877
|
)
|
|
867
|
-
return self.
|
|
868
|
-
text, lambda orig: self._emit('URL', orig, self.cfg.replace_with_url),
|
|
869
|
-
)
|
|
878
|
+
return self._url_regex.sub(lambda m: self._emit('URL', m.group(), self.cfg.replace_with_url), text)
|
|
870
879
|
|
|
871
880
|
def _replace_emails(self, text):
|
|
881
|
+
if not constants.EMAIL_GATE.search(text): # no separator, no email: skip the slow regex
|
|
882
|
+
return text
|
|
872
883
|
reversible = self._replace_reversible(
|
|
873
884
|
text,
|
|
874
885
|
constants.EMAIL_REGEX,
|
|
@@ -950,6 +961,25 @@ class TextCleaner:
|
|
|
950
961
|
view=phone_view,
|
|
951
962
|
)
|
|
952
963
|
|
|
964
|
+
def _replace_typed(self, text, pattern, entity_type):
|
|
965
|
+
"""Replace *pattern* matches as *entity_type* in every replacement mode."""
|
|
966
|
+
reversible = self._replace_reversible(text, pattern, entity_type)
|
|
967
|
+
if reversible is not None:
|
|
968
|
+
return reversible
|
|
969
|
+
if self._synthetic_replacer:
|
|
970
|
+
def value(orig):
|
|
971
|
+
return self._synthetic_replacer.get_replacement(entity_type, orig)
|
|
972
|
+
else:
|
|
973
|
+
def value(orig):
|
|
974
|
+
return self._tags[entity_type]
|
|
975
|
+
return pattern.sub(lambda m: self._emit(entity_type, m.group(), value(m.group())), text)
|
|
976
|
+
|
|
977
|
+
def _replace_iban_cards(self, text):
|
|
978
|
+
return self._replace_typed(self._replace_typed(text, contact.IBAN_VIEW, 'IBAN'), contact.CARD_VIEW, 'CARD')
|
|
979
|
+
|
|
980
|
+
def _replace_identifiers(self, text):
|
|
981
|
+
return self._replace_typed(text, constants.IDENTIFIER_REGEX, 'ID')
|
|
982
|
+
|
|
953
983
|
def _replace_numbers(self, text):
|
|
954
984
|
anon_map = getattr(self._document_context, 'anon_map', None)
|
|
955
985
|
if anon_map is not None:
|
|
@@ -1006,6 +1036,9 @@ class TextCleaner:
|
|
|
1006
1036
|
text,
|
|
1007
1037
|
remove_brackets=self.cfg.check_remove_bracket_content,
|
|
1008
1038
|
remove_braces=self.cfg.check_remove_brace_content,
|
|
1039
|
+
max_chars=self.cfg.bracket_content_max_chars,
|
|
1040
|
+
max_words=self.cfg.bracket_content_max_words,
|
|
1041
|
+
keep_json=self.cfg.keep_json_braces,
|
|
1009
1042
|
)
|
|
1010
1043
|
|
|
1011
1044
|
def _normalize_whitespace(self, text):
|
|
@@ -12,7 +12,7 @@ PUBLIC_VIEW_KEYS: tuple[str, ...] = (
|
|
|
12
12
|
"regex_counts", "entity_counts", "sensitive", "warnings", "models",
|
|
13
13
|
)
|
|
14
14
|
# IP_ADDRESS / PHONE_NUMBER: the names under legacy_type_names=True.
|
|
15
|
-
PII_REGEX_TYPES = frozenset({"EMAIL", "URL", "IP", "PHONE", "IP_ADDRESS", "PHONE_NUMBER"})
|
|
15
|
+
PII_REGEX_TYPES = frozenset({"EMAIL", "URL", "IP", "PHONE", "IP_ADDRESS", "PHONE_NUMBER", "IBAN", "CARD", "ID"})
|
|
16
16
|
WARNING_CODES = frozenset({
|
|
17
17
|
"language_unknown", "language_fallback", "language_low_confidence",
|
|
18
18
|
"ner_fallback_models", "ner_off", "cuda_fallback", "empty_input",
|
|
@@ -84,13 +84,15 @@ class WindowedStreamingRedactor:
|
|
|
84
84
|
# document gives it.
|
|
85
85
|
cutoff = len(buffer) - self._overlap_chars
|
|
86
86
|
phone_view = contact.PHONE_VIEW if cfg.fold_styled_characters else contact.PHONE_DIGIT_VIEW
|
|
87
|
-
views = [
|
|
87
|
+
views = [cleaner._url_regex, constants.EMAIL_REGEX, contact.IP_VIEW, phone_view]
|
|
88
88
|
if cfg.check_replace_html: # a tag holding a space ('<a title="x y">')
|
|
89
|
-
views.append(
|
|
89
|
+
views.append(cleaner._html_regex)
|
|
90
90
|
if cfg.check_replace_dates:
|
|
91
91
|
views.append(cleaner._date_view)
|
|
92
92
|
if cfg.check_fuzzy_replace_dates: # every candidate the fuzzy step may mask ("15 Janury 1990")
|
|
93
93
|
views.append(constants.FUZZY_DATE_CANDIDATE_REGEX)
|
|
94
|
+
if cfg.check_replace_iban_cards: # "NL91 ABNA 0417 1643 00"
|
|
95
|
+
views += [constants.IBAN_REGEX, constants.CARD_REGEX]
|
|
94
96
|
if cfg.check_replace_numbers: # a grouped number ("5 379") is one token too
|
|
95
97
|
views.append(contact.numbers_regex(cfg.replace_numbers_in_words))
|
|
96
98
|
# (start, end) of view text a split must not cut: start < split < end
|