SqueakyCleanText 0.8.0__tar.gz → 0.8.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/PKG-INFO +13 -4
  2. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/README.md +12 -3
  3. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/SqueakyCleanText.egg-info/PKG-INFO +13 -4
  4. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/SqueakyCleanText.egg-info/SOURCES.txt +2 -0
  5. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/__init__.py +1 -1
  6. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/browser.py +13 -7
  7. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/config.py +90 -0
  8. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/model_benchmark.py +1 -0
  9. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/presets.py +2 -0
  10. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/sct.py +52 -19
  11. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/signals.py +1 -1
  12. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/streaming.py +4 -2
  13. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/constants.py +102 -58
  14. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/contact.py +61 -0
  15. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/datetime.py +7 -2
  16. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/ner.py +130 -8
  17. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/normtext.py +1 -1
  18. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/resources.py +13 -0
  19. squeakycleantext-0.8.1/sct/utils/special.py +127 -0
  20. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/synthetic.py +1 -0
  21. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_benchmark_release.py +10 -0
  22. squeakycleantext-0.8.1/tests/test_benchmarks_score.py +255 -0
  23. squeakycleantext-0.8.1/tests/test_codex_benchmark_round1.py +84 -0
  24. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_dates_default.py +40 -0
  25. squeakycleantext-0.8.1/tests/test_final_check_fixes.py +273 -0
  26. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_functional.py +19 -0
  27. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_regex_boundaries.py +142 -0
  28. squeakycleantext-0.8.0/sct/utils/special.py +0 -66
  29. squeakycleantext-0.8.0/tests/test_final_check_fixes.py +0 -119
  30. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/LICENSE +0 -0
  31. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/SqueakyCleanText.egg-info/dependency_links.txt +0 -0
  32. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/SqueakyCleanText.egg-info/entry_points.txt +0 -0
  33. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/SqueakyCleanText.egg-info/requires.txt +0 -0
  34. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/SqueakyCleanText.egg-info/top_level.txt +0 -0
  35. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/pyproject.toml +0 -0
  36. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/adapters.py +0 -0
  37. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/cli.py +0 -0
  38. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/evaluation.py +0 -0
  39. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/integrations/__init__.py +0 -0
  40. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/integrations/_runtime.py +0 -0
  41. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/integrations/datatrove.py +0 -0
  42. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/integrations/haystack.py +0 -0
  43. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/integrations/hf.py +0 -0
  44. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/integrations/langchain.py +0 -0
  45. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/mcp_server.py +0 -0
  46. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/model_candidates.py +0 -0
  47. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/ocr.py +0 -0
  48. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/privacy.py +0 -0
  49. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/report_cli.py +0 -0
  50. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/risk.py +0 -0
  51. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/score.py +0 -0
  52. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/__init__.py +0 -0
  53. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/anonymization_map.py +0 -0
  54. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/gliclass_adapter.py +0 -0
  55. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/gliner_adapter.py +0 -0
  56. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/onnx_pipeline.py +0 -0
  57. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/process_result.py +0 -0
  58. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/stopwords.py +0 -0
  59. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/sct/utils/torch_pipeline.py +0 -0
  60. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/setup.cfg +0 -0
  61. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/setup.py +0 -0
  62. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_adapters.py +0 -0
  63. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_address_leftovers.py +0 -0
  64. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_batching.py +0 -0
  65. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_browser.py +0 -0
  66. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_chunking.py +0 -0
  67. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_cjk_width.py +0 -0
  68. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_cli.py +0 -0
  69. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round1.py +0 -0
  70. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round13.py +0 -0
  71. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round14.py +0 -0
  72. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round15.py +0 -0
  73. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round16.py +0 -0
  74. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round17.py +0 -0
  75. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round18.py +0 -0
  76. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round19.py +0 -0
  77. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round2.py +0 -0
  78. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round20.py +0 -0
  79. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round21.py +0 -0
  80. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round22.py +0 -0
  81. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round23.py +0 -0
  82. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round24.py +0 -0
  83. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round25.py +0 -0
  84. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round27.py +0 -0
  85. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round28.py +0 -0
  86. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round29.py +0 -0
  87. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round3.py +0 -0
  88. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round30.py +0 -0
  89. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round31.py +0 -0
  90. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round32.py +0 -0
  91. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round35.py +0 -0
  92. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round37.py +0 -0
  93. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round38.py +0 -0
  94. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round4.py +0 -0
  95. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round5.py +0 -0
  96. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round6.py +0 -0
  97. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round7.py +0 -0
  98. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round8.py +0 -0
  99. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_codex_full_round9.py +0 -0
  100. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_config_state.py +0 -0
  101. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_contracts.py +0 -0
  102. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_dates_cjk_styled.py +0 -0
  103. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_device_provenance.py +0 -0
  104. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_escaped_markup.py +0 -0
  105. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_evaluation.py +0 -0
  106. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_full_test_round2.py +0 -0
  107. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_fuzz_round1.py +0 -0
  108. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_fuzz_round2.py +0 -0
  109. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_fuzz_round3.py +0 -0
  110. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_fuzz_round5.py +0 -0
  111. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_fuzz_round6.py +0 -0
  112. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_fuzz_styled_pii.py +0 -0
  113. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_integrations_frameworks.py +0 -0
  114. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_integrations_hf.py +0 -0
  115. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_integrations_runtime.py +0 -0
  116. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_keep_scripts.py +0 -0
  117. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_language_all.py +0 -0
  118. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_linux_workers.py +0 -0
  119. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_mcp_path.py +0 -0
  120. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_mcp_server.py +0 -0
  121. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_model_benchmarks.py +0 -0
  122. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_multilingual_eval_set.py +0 -0
  123. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_ner_sees_tags.py +0 -0
  124. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_ocr.py +0 -0
  125. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_onnx_concurrency.py +0 -0
  126. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_onnx_external_data.py +0 -0
  127. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_privacy.py +0 -0
  128. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_public_view.py +0 -0
  129. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_regex_counts.py +0 -0
  130. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_report_cli.py +0 -0
  131. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_review_round1.py +0 -0
  132. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_review_round2.py +0 -0
  133. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_review_round3.py +0 -0
  134. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_review_round4.py +0 -0
  135. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_risk.py +0 -0
  136. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_score.py +0 -0
  137. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_sct.py +0 -0
  138. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_security_leftovers.py +0 -0
  139. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_stream_guard_shared_view.py +0 -0
  140. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_streaming.py +0 -0
  141. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_styled_pii.py +0 -0
  142. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_synthetic_per_document.py +0 -0
  143. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_training_preset.py +0 -0
  144. {squeakycleantext-0.8.0 → squeakycleantext-0.8.1}/tests/test_type_names.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SqueakyCleanText
3
- Version: 0.8.0
3
+ Version: 0.8.1
4
4
  Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
5
5
  Author: Rehan Fazal
6
6
  License-Expression: MIT
@@ -277,6 +277,7 @@ Source and method: [docs/PERFORMANCE.md](https://github.com/rhnfzl/SqueakyCleanT
277
277
 
278
278
  ## Quality and evaluation
279
279
 
280
+ - Open-dataset benchmark (12 PII datasets, about 30 languages, long documents, language ID, cleaning, speed): [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks), scores per release in `benchmarks/results/`.
280
281
  - Release benchmark (30 documents, 84 entities, 9 languages): exact-span F1 0.99 and 0 of 67 PII values leaked in 0.8.0. Run `python -m scripts.benchmark_release`.
281
282
  - Multilingual PII eval (220 synthetic records, 11 languages): the default ensemble scores 0.93 recall and 0.95 precision. Run `python -m scripts.run_multilingual_eval` ([docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md)).
282
283
  - Score columns and data cards: `sct-report data.jsonl --text-column text --out report.json` writes counts, shares and hashes, never text ([docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md)).
@@ -295,7 +296,7 @@ Language detection covers 75 languages. Full NER models cover 4 (English, Dutch,
295
296
  - **Synthetic data is not anonymisation.** Fake values keep the data useful; they do not make it anonymous in a legal sense.
296
297
 
297
298
  <details>
298
- <summary>Known limits of 0.8.0 (details in <a href="CHANGELOG.md">CHANGELOG.md</a>)</summary>
299
+ <summary>Known limits of 0.8.1 (details in <a href="CHANGELOG.md">CHANGELOG.md</a>)</summary>
299
300
 
300
301
  - Dates: only full dates become one `<DATE>`. `1990年1月`, `1月15日`, Japanese era years and `1990.1.15.` do not; the year and number steps still change them (`1990年1月` becomes `<YEAR>年<NUMBER>月`, `生日1990.1.15谢谢` becomes `<YEAR><NUMBER>.15`). A circled number glued to a unit character (`1月⑮日`) stays a list marker. Thai month names need `custom_month_names`.
301
302
  - In ensembles with GLiNER, windows are sized for the token-classification models, not for GLiNER's own word limit. GLiNER2 runs one chunk per call; torch models and GLiNER keep one lock per model, so they do not run in parallel across threads.
@@ -304,7 +305,7 @@ Language detection covers 75 languages. Full NER models cover 4 (English, Dutch,
304
305
  - `06-12345678` becomes `06-<PHONE>`. Adversarial strings such as `+000000000(000)0` can leave `(000)0` after phone replacement.
305
306
  - Two addresses glued with no separator in a script written without spaces can be read as one address or split at the wrong place.
306
307
  - ORGANISATION is the weakest NER label (0.75 recall in the multilingual eval); Bengali and Tamil name spans can be a few characters off.
307
- - A URL with a path swallows a trailing sentence period (`see https://x.org/a.` becomes `see <URL>`). A URL path that switches from Latin letters or digits straight into a script written without spaces is replaced only up to the switch (`https://x.org/a東京secret` becomes `<URL>東京secret`). Rare styled URL hosts and mail addresses (`https://ab。⑧․cd.ex.com/...`, `INFO@COL·LEGI。CAT`) are not fully replaced.
308
+ - A URL path that switches from Latin letters or digits straight into a script written without spaces is replaced only up to the switch (`https://x.org/a東京secret` becomes `<URL>東京secret`). Rare styled URL hosts and mail addresses (`https://ab。⑧․cd.ex.com/...`, `INFO@COL·LEGI。CAT`) are not fully replaced.
308
309
  - Score B's quality head was tested with a small ONNX graph only; GLiNER PII mode was not run end to end for this release. CI does not install the `mcp` extra; FastMCP-only tests skip there.
309
310
  - An unpinned NER model makes one Hub request per worker when the parent never loaded it before forking; pin `ner_model_revisions` or set `HF_HUB_OFFLINE=1`.
310
311
  - With an empty regex marker (`replace_with_email=""`), NER in synthetic or reversible mode can find other entities than in placeholder mode.
@@ -328,6 +329,14 @@ Language detection covers 75 languages. Full NER models cover 4 (English, Dutch,
328
329
  | Languages and adding a language | [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md), [docs/ADDING_LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/ADDING_LANGUAGES.md) |
329
330
  | The whole API in one file, for AI coding assistants | [llms.txt](https://github.com/rhnfzl/SqueakyCleanText/blob/main/llms.txt) |
330
331
 
332
+ ## What's new in 0.8.1
333
+
334
+ Fixes from the open-dataset benchmark ([benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks)). Leak falls on all 12 PII datasets (REDACT 18.1% to 8.2%, long documents 42.4% to 11.7%); speed is unchanged.
335
+
336
+ - NER keeps a span when its best model passes the cut; other copies of a masked name are masked too.
337
+ - IBAN and card numbers get their own types; JSON and longer bracketed text survive cleanup.
338
+ - `Month/YY` dates, `Name <email>`, URLs before a full stop, and span edges inside words are fixed.
339
+
331
340
  ## What's new in 0.8.0
332
341
 
333
342
  See [CHANGELOG.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/CHANGELOG.md) (many breaking changes). Highlights of 0.8.0:
@@ -356,7 +365,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
356
365
  author = {Fazal, Rehan},
357
366
  title = {SqueakyCleanText},
358
367
  url = {https://github.com/rhnfzl/SqueakyCleanText},
359
- version = {0.8.0},
368
+ version = {0.8.1},
360
369
  year = {2026}
361
370
  }
362
371
  ```
@@ -189,6 +189,7 @@ Source and method: [docs/PERFORMANCE.md](https://github.com/rhnfzl/SqueakyCleanT
189
189
 
190
190
  ## Quality and evaluation
191
191
 
192
+ - Open-dataset benchmark (12 PII datasets, about 30 languages, long documents, language ID, cleaning, speed): [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks), scores per release in `benchmarks/results/`.
192
193
  - Release benchmark (30 documents, 84 entities, 9 languages): exact-span F1 0.99 and 0 of 67 PII values leaked in 0.8.0. Run `python -m scripts.benchmark_release`.
193
194
  - Multilingual PII eval (220 synthetic records, 11 languages): the default ensemble scores 0.93 recall and 0.95 precision. Run `python -m scripts.run_multilingual_eval` ([docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md)).
194
195
  - Score columns and data cards: `sct-report data.jsonl --text-column text --out report.json` writes counts, shares and hashes, never text ([docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md)).
@@ -207,7 +208,7 @@ Language detection covers 75 languages. Full NER models cover 4 (English, Dutch,
207
208
  - **Synthetic data is not anonymisation.** Fake values keep the data useful; they do not make it anonymous in a legal sense.
208
209
 
209
210
  <details>
210
- <summary>Known limits of 0.8.0 (details in <a href="CHANGELOG.md">CHANGELOG.md</a>)</summary>
211
+ <summary>Known limits of 0.8.1 (details in <a href="CHANGELOG.md">CHANGELOG.md</a>)</summary>
211
212
 
212
213
  - Dates: only full dates become one `<DATE>`. `1990年1月`, `1月15日`, Japanese era years and `1990.1.15.` do not; the year and number steps still change them (`1990年1月` becomes `<YEAR>年<NUMBER>月`, `生日1990.1.15谢谢` becomes `<YEAR><NUMBER>.15`). A circled number glued to a unit character (`1月⑮日`) stays a list marker. Thai month names need `custom_month_names`.
213
214
  - In ensembles with GLiNER, windows are sized for the token-classification models, not for GLiNER's own word limit. GLiNER2 runs one chunk per call; torch models and GLiNER keep one lock per model, so they do not run in parallel across threads.
@@ -216,7 +217,7 @@ Language detection covers 75 languages. Full NER models cover 4 (English, Dutch,
216
217
  - `06-12345678` becomes `06-<PHONE>`. Adversarial strings such as `+000000000(000)0` can leave `(000)0` after phone replacement.
217
218
  - Two addresses glued with no separator in a script written without spaces can be read as one address or split at the wrong place.
218
219
  - ORGANISATION is the weakest NER label (0.75 recall in the multilingual eval); Bengali and Tamil name spans can be a few characters off.
219
- - A URL with a path swallows a trailing sentence period (`see https://x.org/a.` becomes `see <URL>`). A URL path that switches from Latin letters or digits straight into a script written without spaces is replaced only up to the switch (`https://x.org/a東京secret` becomes `<URL>東京secret`). Rare styled URL hosts and mail addresses (`https://ab。⑧․cd.ex.com/...`, `INFO@COL·LEGI。CAT`) are not fully replaced.
220
+ - A URL path that switches from Latin letters or digits straight into a script written without spaces is replaced only up to the switch (`https://x.org/a東京secret` becomes `<URL>東京secret`). Rare styled URL hosts and mail addresses (`https://ab。⑧․cd.ex.com/...`, `INFO@COL·LEGI。CAT`) are not fully replaced.
220
221
  - Score B's quality head was tested with a small ONNX graph only; GLiNER PII mode was not run end to end for this release. CI does not install the `mcp` extra; FastMCP-only tests skip there.
221
222
  - An unpinned NER model makes one Hub request per worker when the parent never loaded it before forking; pin `ner_model_revisions` or set `HF_HUB_OFFLINE=1`.
222
223
  - With an empty regex marker (`replace_with_email=""`), NER in synthetic or reversible mode can find other entities than in placeholder mode.
@@ -240,6 +241,14 @@ Language detection covers 75 languages. Full NER models cover 4 (English, Dutch,
240
241
  | Languages and adding a language | [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md), [docs/ADDING_LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/ADDING_LANGUAGES.md) |
241
242
  | The whole API in one file, for AI coding assistants | [llms.txt](https://github.com/rhnfzl/SqueakyCleanText/blob/main/llms.txt) |
242
243
 
244
+ ## What's new in 0.8.1
245
+
246
+ Fixes from the open-dataset benchmark ([benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks)). Leak falls on all 12 PII datasets (REDACT 18.1% to 8.2%, long documents 42.4% to 11.7%); speed is unchanged.
247
+
248
+ - NER keeps a span when its best model passes the cut; other copies of a masked name are masked too.
249
+ - IBAN and card numbers get their own types; JSON and longer bracketed text survive cleanup.
250
+ - `Month/YY` dates, `Name <email>`, URLs before a full stop, and span edges inside words are fixed.
251
+
243
252
  ## What's new in 0.8.0
244
253
 
245
254
  See [CHANGELOG.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/CHANGELOG.md) (many breaking changes). Highlights of 0.8.0:
@@ -268,7 +277,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
268
277
  author = {Fazal, Rehan},
269
278
  title = {SqueakyCleanText},
270
279
  url = {https://github.com/rhnfzl/SqueakyCleanText},
271
- version = {0.8.0},
280
+ version = {0.8.1},
272
281
  year = {2026}
273
282
  }
274
283
  ```
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SqueakyCleanText
3
- Version: 0.8.0
3
+ Version: 0.8.1
4
4
  Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
5
5
  Author: Rehan Fazal
6
6
  License-Expression: MIT
@@ -277,6 +277,7 @@ Source and method: [docs/PERFORMANCE.md](https://github.com/rhnfzl/SqueakyCleanT
277
277
 
278
278
  ## Quality and evaluation
279
279
 
280
+ - Open-dataset benchmark (12 PII datasets, about 30 languages, long documents, language ID, cleaning, speed): [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks), scores per release in `benchmarks/results/`.
280
281
  - Release benchmark (30 documents, 84 entities, 9 languages): exact-span F1 0.99 and 0 of 67 PII values leaked in 0.8.0. Run `python -m scripts.benchmark_release`.
281
282
  - Multilingual PII eval (220 synthetic records, 11 languages): the default ensemble scores 0.93 recall and 0.95 precision. Run `python -m scripts.run_multilingual_eval` ([docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md)).
282
283
  - Score columns and data cards: `sct-report data.jsonl --text-column text --out report.json` writes counts, shares and hashes, never text ([docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md)).
@@ -295,7 +296,7 @@ Language detection covers 75 languages. Full NER models cover 4 (English, Dutch,
295
296
  - **Synthetic data is not anonymisation.** Fake values keep the data useful; they do not make it anonymous in a legal sense.
296
297
 
297
298
  <details>
298
- <summary>Known limits of 0.8.0 (details in <a href="CHANGELOG.md">CHANGELOG.md</a>)</summary>
299
+ <summary>Known limits of 0.8.1 (details in <a href="CHANGELOG.md">CHANGELOG.md</a>)</summary>
299
300
 
300
301
  - Dates: only full dates become one `<DATE>`. `1990年1月`, `1月15日`, Japanese era years and `1990.1.15.` do not; the year and number steps still change them (`1990年1月` becomes `<YEAR>年<NUMBER>月`, `生日1990.1.15谢谢` becomes `<YEAR><NUMBER>.15`). A circled number glued to a unit character (`1月⑮日`) stays a list marker. Thai month names need `custom_month_names`.
301
302
  - In ensembles with GLiNER, windows are sized for the token-classification models, not for GLiNER's own word limit. GLiNER2 runs one chunk per call; torch models and GLiNER keep one lock per model, so they do not run in parallel across threads.
@@ -304,7 +305,7 @@ Language detection covers 75 languages. Full NER models cover 4 (English, Dutch,
304
305
  - `06-12345678` becomes `06-<PHONE>`. Adversarial strings such as `+000000000(000)0` can leave `(000)0` after phone replacement.
305
306
  - Two addresses glued with no separator in a script written without spaces can be read as one address or split at the wrong place.
306
307
  - ORGANISATION is the weakest NER label (0.75 recall in the multilingual eval); Bengali and Tamil name spans can be a few characters off.
307
- - A URL with a path swallows a trailing sentence period (`see https://x.org/a.` becomes `see <URL>`). A URL path that switches from Latin letters or digits straight into a script written without spaces is replaced only up to the switch (`https://x.org/a東京secret` becomes `<URL>東京secret`). Rare styled URL hosts and mail addresses (`https://ab。⑧․cd.ex.com/...`, `INFO@COL·LEGI。CAT`) are not fully replaced.
308
+ - A URL path that switches from Latin letters or digits straight into a script written without spaces is replaced only up to the switch (`https://x.org/a東京secret` becomes `<URL>東京secret`). Rare styled URL hosts and mail addresses (`https://ab。⑧․cd.ex.com/...`, `INFO@COL·LEGI。CAT`) are not fully replaced.
308
309
  - Score B's quality head was tested with a small ONNX graph only; GLiNER PII mode was not run end to end for this release. CI does not install the `mcp` extra; FastMCP-only tests skip there.
309
310
  - An unpinned NER model makes one Hub request per worker when the parent never loaded it before forking; pin `ner_model_revisions` or set `HF_HUB_OFFLINE=1`.
310
311
  - With an empty regex marker (`replace_with_email=""`), NER in synthetic or reversible mode can find other entities than in placeholder mode.
@@ -328,6 +329,14 @@ Language detection covers 75 languages. Full NER models cover 4 (English, Dutch,
328
329
  | Languages and adding a language | [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md), [docs/ADDING_LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/ADDING_LANGUAGES.md) |
329
330
  | The whole API in one file, for AI coding assistants | [llms.txt](https://github.com/rhnfzl/SqueakyCleanText/blob/main/llms.txt) |
330
331
 
332
+ ## What's new in 0.8.1
333
+
334
+ Fixes from the open-dataset benchmark ([benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks)). Leak falls on all 12 PII datasets (REDACT 18.1% to 8.2%, long documents 42.4% to 11.7%); speed is unchanged.
335
+
336
+ - NER keeps a span when its best model passes the cut; other copies of a masked name are masked too.
337
+ - IBAN and card numbers get their own types; JSON and longer bracketed text survive cleanup.
338
+ - `Month/YY` dates, `Name <email>`, URLs before a full stop, and span edges inside words are fixed.
339
+
331
340
  ## What's new in 0.8.0
332
341
 
333
342
  See [CHANGELOG.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/CHANGELOG.md) (many breaking changes). Highlights of 0.8.0:
@@ -356,7 +365,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
356
365
  author = {Fazal, Rehan},
357
366
  title = {SqueakyCleanText},
358
367
  url = {https://github.com/rhnfzl/SqueakyCleanText},
359
- version = {0.8.0},
368
+ version = {0.8.1},
360
369
  year = {2026}
361
370
  }
362
371
  ```
@@ -52,10 +52,12 @@ tests/test_adapters.py
52
52
  tests/test_address_leftovers.py
53
53
  tests/test_batching.py
54
54
  tests/test_benchmark_release.py
55
+ tests/test_benchmarks_score.py
55
56
  tests/test_browser.py
56
57
  tests/test_chunking.py
57
58
  tests/test_cjk_width.py
58
59
  tests/test_cli.py
60
+ tests/test_codex_benchmark_round1.py
59
61
  tests/test_codex_full_round1.py
60
62
  tests/test_codex_full_round13.py
61
63
  tests/test_codex_full_round14.py
@@ -50,7 +50,7 @@ from sct.streaming import (
50
50
  from sct.utils.anonymization_map import AnonymizationMap, MapEntry
51
51
  from sct.utils.process_result import ProcessResult
52
52
 
53
- __version__ = "0.8.0"
53
+ __version__ = "0.8.1"
54
54
  __all__ = [
55
55
  "TextCleaner", "TextCleanerConfig", "PRESETS", "preset_config",
56
56
  "PII_LABELS", "PII_LABEL_MAP",
@@ -6,7 +6,7 @@ from pathlib import Path
6
6
  from typing import Any
7
7
 
8
8
  from sct.config import TextCleanerConfig
9
- from sct.utils import constants
9
+ from sct.utils import constants, resources
10
10
  from sct.utils.contact import numbers_regex
11
11
 
12
12
 
@@ -54,12 +54,17 @@ def build_browser_manifest(config: TextCleanerConfig) -> dict[str, Any]:
54
54
 
55
55
  # Same reason for any token a later enabled rule would match
56
56
  # (an HTML token holding an email or URL, a URL token holding an email).
57
+ html_regex = constants.HTML_REGEX if config.html_tag_strict else constants.HTML_REGEX_ANY
58
+ url_regex = constants.URL_REGEX if config.url_trailing_punct_outside else constants.URL_REGEX_LEGACY
59
+ month_names = resources.configured_month_names(config)
60
+ date_regex = (constants.DATE_REGEX if month_names is None
61
+ else constants.build_date_regex(month_names, slash_year=config.date_month_name_slash_year))
57
62
  ordered = [
58
- ("html", constants.HTML_REGEX, config.replace_with_html, config.check_replace_html),
59
- ("url", constants.URL_REGEX, config.replace_with_url, config.check_replace_urls),
63
+ ("html", html_regex, config.replace_with_html, config.check_replace_html),
64
+ ("url", url_regex, config.replace_with_url, config.check_replace_urls),
60
65
  ("email", constants.EMAIL_REGEX, config.replace_with_email, config.check_replace_emails),
61
66
  ("ip", constants.IP_REGEX, config.replace_with_ips, config.check_replace_ips),
62
- ("date", constants.DATE_REGEX, config.replace_with_dates, config.check_replace_dates),
67
+ ("date", date_regex, config.replace_with_dates, config.check_replace_dates),
63
68
  ("phone", constants.PHONE_REGEX, config.replace_with_phone_numbers,
64
69
  config.check_replace_phone_numbers),
65
70
  ("year", constants.YEAR_REGEX, config.replace_with_years, config.check_replace_years),
@@ -105,12 +110,12 @@ def build_browser_manifest(config: TextCleanerConfig) -> dict[str, Any]:
105
110
  ],
106
111
  "rules": {
107
112
  "html": _rule(
108
- constants.HTML_REGEX,
113
+ html_regex,
109
114
  config.replace_with_html,
110
115
  config.check_replace_html,
111
116
  ),
112
117
  "url": _rule(
113
- constants.URL_REGEX,
118
+ url_regex,
114
119
  config.replace_with_url,
115
120
  config.check_replace_urls,
116
121
  ),
@@ -125,7 +130,7 @@ def build_browser_manifest(config: TextCleanerConfig) -> dict[str, Any]:
125
130
  config.check_replace_ips,
126
131
  ), "fold_decimal_digits": True},
127
132
  "date": {**_rule(
128
- constants.DATE_REGEX,
133
+ date_regex,
129
134
  config.replace_with_dates,
130
135
  config.check_replace_dates,
131
136
  ), "fold_decimal_digits": True},
@@ -155,6 +160,7 @@ def build_browser_manifest(config: TextCleanerConfig) -> dict[str, Any]:
155
160
  "Regular expressions require JavaScript compatibility tests.",
156
161
  "Model files remain external and must keep their recorded revisions.",
157
162
  "Reversible and synthetic replacement are server-side features.",
163
+ "IBAN/card checksum typing and identifier tokens are server-side rules.",
158
164
  "Rules with fold_decimal_digits match on a copy of the text where every Unicode "
159
165
  "decimal digit (Nd, such as ٠٦ or ०६) is its ASCII digit, one code point for one, "
160
166
  "and replace the same offsets in the original text (JavaScript \\d is ASCII only).",
@@ -357,6 +357,47 @@ class TextCleanerConfig:
357
357
  # synthetic: ISO date that date fakes (DATE, DATE_OF_BIRTH) end on, so a document gets the
358
358
  # same dates on any day; None = today, from the clock (earlier 0.8.0)
359
359
  synthetic_reference_date: Optional[str] = '2026-01-01'
360
+ # NER ensemble vote per span: 'max' keeps a span when its best model score passes
361
+ # ner_confidence_threshold, so a weak second model never hides a strong hit; 'mean' averages
362
+ # the model scores (0.8.0)
363
+ ner_ensemble_vote: str = 'max'
364
+ # mask every other case-sensitive whole-word copy of a masked PER/ORG/LOC value in the same
365
+ # document (the value comes from the model's own span); False = 0.8.0 output
366
+ ner_propagate_masked: bool = True
367
+ # score a span needs before its value is propagated (0..1); None = the effective NER cut.
368
+ # Raising it spreads fewer wrong tags at the cost of more visible repeats
369
+ ner_propagate_min_score: Optional[float] = None
370
+ # IBANs (mod-97 checksum) and payment cards (Luhn checksum) become one <IBAN> / <CARD> each,
371
+ # before the phone step; False = 0.8.0 (the phone/number steps take pieces of them)
372
+ check_replace_iban_cards: bool = True
373
+ replace_with_iban: str = "<IBAN>"
374
+ replace_with_cards: str = "<CARD>"
375
+ # Identifier tokens: 6+ uppercase letters and digits with 2+ digits and a letter (RBCON68T08).
376
+ # Off by default: the shape also takes product and reference codes (RTX4090, COVID19)
377
+ check_replace_identifiers: bool = False
378
+ replace_with_identifiers: str = "<ID>"
379
+ # Bracket/brace removal only takes innermost spans of at most this many characters and
380
+ # whitespace-separated words, so prose clauses stay; 0 = no limit (both 0 = 0.8.0 matching)
381
+ bracket_content_max_chars: int = 32
382
+ bracket_content_max_words: int = 3
383
+ # A {...} that parses as a JSON object (json.raw_decode, nested ones included) is never removed
384
+ keep_json_braces: bool = True
385
+ # A month name, "/" and a 2- or 4-digit year is one <DATE> (October/55, Oct/2019)
386
+ date_month_name_slash_year: bool = True
387
+ # ISO codes of the built-in month data dates use (en nl de es fr pt it th); None = all
388
+ date_builtin_month_languages: Optional[Tuple[str, ...]] = None
389
+ # HTML rule reads only tag shapes (<b>, </a>, <br/>, <!-- -->); False = 0.8.0 "<.*?>" (eats "<a@b.com>")
390
+ html_tag_strict: bool = True
391
+ # join two LOCATION spans whose gap has <= N chars (placeholders not counted), no line break
392
+ # and no sentence end, so an address is one <LOCATION>; 0 = off (0.8.0 output)
393
+ ner_merge_location_gap: int = 0
394
+ # NER span edges cut inside a word (letters and marks, not wide CJK) move out to the word
395
+ # edges on every backend ("Salut" -> "Salute"), within the ner_subword_widen cap and
396
+ # same-script rule; False = 0.8.0 spans
397
+ ner_snap_word_edges: bool = True
398
+ # . , ; : ! ? ' end a URL path or query unless more URL text follows ("https://x.com/a. Next");
399
+ # False = 0.8.0, where the URL takes the sentence's full stop or comma
400
+ url_trailing_punct_outside: bool = True
360
401
 
361
402
  def __post_init__(self):
362
403
  if self.synthetic_reference_date is not None:
@@ -475,6 +516,12 @@ class TextCleanerConfig:
475
516
  value = getattr(self, name)
476
517
  if value is not None and not 0.0 <= value <= 1.0:
477
518
  raise ValueError(f"{name} must be None or in [0, 1], got {value}")
519
+ if self.ner_ensemble_vote not in ('max', 'mean'):
520
+ raise ValueError(f"ner_ensemble_vote must be 'max' or 'mean', got {self.ner_ensemble_vote!r}")
521
+ if self.ner_merge_location_gap < 0:
522
+ raise ValueError(f"ner_merge_location_gap must be >= 0, got {self.ner_merge_location_gap}")
523
+ if self.ner_propagate_min_score is not None and not 0.0 <= self.ner_propagate_min_score <= 1.0:
524
+ raise ValueError(f"ner_propagate_min_score must be None or in [0, 1], got {self.ner_propagate_min_score}")
478
525
  if self.ner_subword_widen is not None and self.ner_subword_widen < 0:
479
526
  raise ValueError(f"ner_subword_widen must be None or >= 0, got {self.ner_subword_widen}")
480
527
  if not 0.0 <= self.language_confidence_warning <= 1.0:
@@ -614,6 +661,15 @@ class TextCleanerConfig:
614
661
  object.__setattr__(self, 'custom_stopwords',
615
662
  MappingProxyType(resolved_sw))
616
663
 
664
+ if self.date_builtin_month_languages is not None:
665
+ from sct.utils.constants import MONTH_NAMES_MULTILINGUAL
666
+ codes = tuple(self.date_builtin_month_languages)
667
+ unknown = set(codes) - set().union(*MONTH_NAMES_MULTILINGUAL.values())
668
+ if isinstance(self.date_builtin_month_languages, str) or unknown:
669
+ raise ValueError(f"date_builtin_month_languages must be a tuple of built-in month ISO codes, "
670
+ f"got {self.date_builtin_month_languages!r}")
671
+ object.__setattr__(self, 'date_builtin_month_languages', codes)
672
+
617
673
  # Validate and freeze custom_month_names
618
674
  if self.custom_month_names:
619
675
  resolved_mn = {resolve_language(k): tuple(v) for k, v in self.custom_month_names.items()}
@@ -726,6 +782,23 @@ def _config_from_module_globals() -> TextCleanerConfig:
726
782
  synthetic_low_confidence_locale=getattr(m, 'SYNTHETIC_LOW_CONFIDENCE_LOCALE', 'en_US'),
727
783
  synthetic_collapse_period=getattr(m, 'SYNTHETIC_COLLAPSE_PERIOD', True),
728
784
  synthetic_reference_date=getattr(m, 'SYNTHETIC_REFERENCE_DATE', '2026-01-01'),
785
+ ner_ensemble_vote=getattr(m, 'NER_ENSEMBLE_VOTE', 'max'),
786
+ ner_propagate_masked=getattr(m, 'NER_PROPAGATE_MASKED', True),
787
+ ner_propagate_min_score=getattr(m, 'NER_PROPAGATE_MIN_SCORE', None),
788
+ check_replace_iban_cards=getattr(m, 'CHECK_REPLACE_IBAN_CARDS', True),
789
+ replace_with_iban=getattr(m, 'REPLACE_WITH_IBAN', '<IBAN>'),
790
+ replace_with_cards=getattr(m, 'REPLACE_WITH_CARDS', '<CARD>'),
791
+ check_replace_identifiers=getattr(m, 'CHECK_REPLACE_IDENTIFIERS', False),
792
+ replace_with_identifiers=getattr(m, 'REPLACE_WITH_IDENTIFIERS', '<ID>'),
793
+ bracket_content_max_chars=getattr(m, 'BRACKET_CONTENT_MAX_CHARS', 32),
794
+ bracket_content_max_words=getattr(m, 'BRACKET_CONTENT_MAX_WORDS', 3),
795
+ keep_json_braces=getattr(m, 'KEEP_JSON_BRACES', True),
796
+ date_month_name_slash_year=getattr(m, 'DATE_MONTH_NAME_SLASH_YEAR', True),
797
+ date_builtin_month_languages=getattr(m, 'DATE_BUILTIN_MONTH_LANGUAGES', None),
798
+ html_tag_strict=getattr(m, 'HTML_TAG_STRICT', True),
799
+ ner_merge_location_gap=getattr(m, 'NER_MERGE_LOCATION_GAP', 0),
800
+ ner_snap_word_edges=getattr(m, 'NER_SNAP_WORD_EDGES', True),
801
+ url_trailing_punct_outside=getattr(m, 'URL_TRAILING_PUNCT_OUTSIDE', True),
729
802
  check_replace_currency_symbols=m.CHECK_REPLACE_CURRENCY_SYMBOLS,
730
803
  check_ner_process=m.CHECK_NER_PROCESS,
731
804
  check_remove_isolated_letters=m.CHECK_REMOVE_ISOLATED_LETTERS,
@@ -834,6 +907,23 @@ NER_SEES_TAGS = True
834
907
  SYNTHETIC_LOW_CONFIDENCE_LOCALE = 'en_US'
835
908
  SYNTHETIC_COLLAPSE_PERIOD = True
836
909
  SYNTHETIC_REFERENCE_DATE = '2026-01-01'
910
+ NER_ENSEMBLE_VOTE = 'max'
911
+ NER_PROPAGATE_MASKED = True
912
+ NER_PROPAGATE_MIN_SCORE = None
913
+ CHECK_REPLACE_IBAN_CARDS = True
914
+ REPLACE_WITH_IBAN = "<IBAN>"
915
+ REPLACE_WITH_CARDS = "<CARD>"
916
+ CHECK_REPLACE_IDENTIFIERS = False
917
+ REPLACE_WITH_IDENTIFIERS = "<ID>"
918
+ BRACKET_CONTENT_MAX_CHARS = 32
919
+ BRACKET_CONTENT_MAX_WORDS = 3
920
+ KEEP_JSON_BRACES = True
921
+ DATE_MONTH_NAME_SLASH_YEAR = True
922
+ DATE_BUILTIN_MONTH_LANGUAGES = None
923
+ HTML_TAG_STRICT = True
924
+ NER_MERGE_LOCATION_GAP = 0
925
+ NER_SNAP_WORD_EDGES = True
926
+ URL_TRAILING_PUNCT_OUTSIDE = True
837
927
  GLINER_ONNX = False
838
928
  LANGUAGE = None
839
929
 
@@ -60,6 +60,7 @@ def candidate_config(candidate: ModelCandidate) -> TextCleanerConfig:
60
60
  "check_replace_years": False,
61
61
  "check_replace_dates": False,
62
62
  "check_replace_phone_numbers": False,
63
+ "check_replace_iban_cards": False,
63
64
  "check_replace_numbers": False,
64
65
  "check_replace_currency_symbols": False,
65
66
  "check_remove_isolated_letters": False,
@@ -17,12 +17,14 @@ _TRAINING = {
17
17
  'check_replace_emails': True,
18
18
  'check_replace_phone_numbers': True,
19
19
  'check_replace_ips': True,
20
+ 'check_replace_iban_cards': True,
20
21
  'check_ner_process': True,
21
22
  'check_fix_bad_unicode': True,
22
23
  'check_detect_language': True,
23
24
  # Off
24
25
  'check_to_ascii_unicode': False,
25
26
  'check_replace_numbers': False,
27
+ 'check_replace_identifiers': False, # takes product/reference codes too; not measured on training data
26
28
  'check_replace_years': False,
27
29
  'check_replace_dates': False,
28
30
  'check_fuzzy_replace_dates': False,
@@ -62,6 +62,8 @@ class TextCleaner:
62
62
  'DATE': self.cfg.replace_with_dates, 'YEAR': self.cfg.replace_with_years,
63
63
  self._phone_type: self.cfg.replace_with_phone_numbers,
64
64
  'NUMBER': self.cfg.replace_with_numbers, 'CURRENCY': self.cfg.replace_with_currency_symbols,
65
+ 'IBAN': self.cfg.replace_with_iban, 'CARD': self.cfg.replace_with_cards,
66
+ 'ID': self.cfg.replace_with_identifiers,
65
67
  }
66
68
  # Markers that are not <TAG>-shaped ("EMAIL"): NER protects them like <EMAIL> (ner.regex_markers).
67
69
  self._custom_markers = tuple(sorted({
@@ -86,17 +88,15 @@ class TextCleaner:
86
88
  )
87
89
 
88
90
  # Instance-scoped datetime (with custom month names -> per-instance regex)
89
- mn_dict = None
90
- if self.cfg.custom_month_names:
91
- extra = {}
92
- for lang, names in self.cfg.custom_month_names.items():
93
- extra[resources.language_to_iso(lang)] = names
94
- mn_dict = constants.build_month_names_dict(extra)
95
- date_regex = constants.build_date_regex(mn_dict)
91
+ custom_months = self.cfg.custom_month_names or {}
92
+ slash_year = self.cfg.date_month_name_slash_year
93
+ mn_dict = resources.configured_month_names(self.cfg)
94
+ if mn_dict is not None:
95
+ date_regex = constants.build_date_regex(mn_dict, slash_year=slash_year)
96
96
  fuzzy_all, fuzzy_by_lang = constants.build_fuzzy_vocabulary(
97
97
  mn_dict,
98
98
  {**constants._LANG_TO_VOCAB,
99
- **{lang: resources.language_to_iso(lang) for lang in self.cfg.custom_month_names}},
99
+ **{lang: resources.language_to_iso(lang) for lang in custom_months}},
100
100
  )
101
101
  self.ProcessDateTime = datetime.ProcessDateTime(
102
102
  date_regex=date_regex,
@@ -108,10 +108,13 @@ class TextCleaner:
108
108
  # fold_styled_characters: dates, like phones, read any space as " " and any dash as "-" (1987\u201105\u201112),
109
109
  # any dot form as "." (1990﹒ 1﹒ 15﹒), and a month-name date's day may be a circled number (⑮ January 1990).
110
110
  self._date_view = (
111
- contact.DigitView(constants.build_date_regex(mn_dict, styled=True), contact.DATE_SEPARATOR_VIEW)
111
+ contact.DigitView(constants.build_date_regex(mn_dict, styled=True, slash_year=slash_year),
112
+ contact.DATE_SEPARATOR_VIEW)
112
113
  if self.cfg.fold_styled_characters else self.ProcessDateTime._date_regex)
113
114
 
114
115
  self.ProcessContacts = contact.ProcessContacts()
116
+ self._html_regex = constants.HTML_REGEX if self.cfg.html_tag_strict else constants.HTML_REGEX_ANY
117
+ self._url_regex = constants.URL_REGEX if self.cfg.url_trailing_punct_outside else constants.URL_REGEX_LEGACY
115
118
  self.ProcessSpecialSymbols = special.ProcessSpecialSymbols()
116
119
  self.NormaliseText = normtext.NormaliseText()
117
120
 
@@ -190,6 +193,11 @@ class TextCleaner:
190
193
  onnx_intra_op_threads=self.cfg.onnx_intra_op_threads,
191
194
  onnx_inter_op_threads=self.cfg.onnx_inter_op_threads,
192
195
  ner_chunk_overlap=self.cfg.ner_chunk_overlap,
196
+ ner_ensemble_vote=self.cfg.ner_ensemble_vote,
197
+ ner_propagate_masked=self.cfg.ner_propagate_masked,
198
+ ner_propagate_min_score=self.cfg.ner_propagate_min_score,
199
+ ner_merge_location_gap=self.cfg.ner_merge_location_gap,
200
+ ner_snap_word_edges=self.cfg.ner_snap_word_edges,
193
201
  )
194
202
  # GLiClass document-level pre-classification (optional, lazy-loaded)
195
203
  self._gliclass: Any = None
@@ -245,6 +253,11 @@ class TextCleaner:
245
253
  # fuzzy_replace_dates runs here (between replace_dates and replace_years)
246
254
  # but is called explicitly in _process_single with language context.
247
255
  # Phones before years, so a year step cannot split "+31 6 1234 2019".
256
+ # Checksum-valid IBANs/cards and identifier tokens before phones, so a phone cannot take their digits.
257
+ if cfg.check_replace_iban_cards:
258
+ self._post_fuzzy_pipeline.append(self._replace_iban_cards)
259
+ if cfg.check_replace_identifiers:
260
+ self._post_fuzzy_pipeline.append(self._replace_identifiers)
248
261
  if cfg.check_replace_phone_numbers:
249
262
  self._post_fuzzy_pipeline.append(self._replace_phone_numbers)
250
263
  if cfg.check_replace_years:
@@ -843,32 +856,30 @@ class TextCleaner:
843
856
  def _replace_html(self, text):
844
857
  reversible = self._replace_reversible(
845
858
  text,
846
- constants.HTML_REGEX,
859
+ self._html_regex,
847
860
  'HTML',
848
861
  )
849
862
  if reversible is not None:
850
863
  return reversible
851
- return self.ProcessContacts.replace_html(
852
- text, replace_with=lambda m: self._emit('HTML', m.group(), self.cfg.replace_with_html),
853
- )
864
+ return self._html_regex.sub(lambda m: self._emit('HTML', m.group(), self.cfg.replace_with_html), text)
854
865
 
855
866
  def _replace_urls(self, text):
856
867
  reversible = self._replace_reversible(
857
868
  text,
858
- constants.URL_REGEX,
869
+ self._url_regex,
859
870
  'URL',
860
871
  )
861
872
  if reversible is not None:
862
873
  return reversible
863
874
  if self._synthetic_replacer:
864
- return self.ProcessContacts.replace_urls_with_fn(
865
- text, lambda orig: self._emit('URL', orig, self._synthetic_replacer.get_replacement('URL', orig)),
875
+ return self._url_regex.sub(
876
+ lambda m: self._emit('URL', m.group(), self._synthetic_replacer.get_replacement('URL', m.group())), text,
866
877
  )
867
- return self.ProcessContacts.replace_urls_with_fn(
868
- text, lambda orig: self._emit('URL', orig, self.cfg.replace_with_url),
869
- )
878
+ return self._url_regex.sub(lambda m: self._emit('URL', m.group(), self.cfg.replace_with_url), text)
870
879
 
871
880
  def _replace_emails(self, text):
881
+ if not constants.EMAIL_GATE.search(text): # no separator, no email: skip the slow regex
882
+ return text
872
883
  reversible = self._replace_reversible(
873
884
  text,
874
885
  constants.EMAIL_REGEX,
@@ -950,6 +961,25 @@ class TextCleaner:
950
961
  view=phone_view,
951
962
  )
952
963
 
964
+ def _replace_typed(self, text, pattern, entity_type):
965
+ """Replace *pattern* matches as *entity_type* in every replacement mode."""
966
+ reversible = self._replace_reversible(text, pattern, entity_type)
967
+ if reversible is not None:
968
+ return reversible
969
+ if self._synthetic_replacer:
970
+ def value(orig):
971
+ return self._synthetic_replacer.get_replacement(entity_type, orig)
972
+ else:
973
+ def value(orig):
974
+ return self._tags[entity_type]
975
+ return pattern.sub(lambda m: self._emit(entity_type, m.group(), value(m.group())), text)
976
+
977
+ def _replace_iban_cards(self, text):
978
+ return self._replace_typed(self._replace_typed(text, contact.IBAN_VIEW, 'IBAN'), contact.CARD_VIEW, 'CARD')
979
+
980
+ def _replace_identifiers(self, text):
981
+ return self._replace_typed(text, constants.IDENTIFIER_REGEX, 'ID')
982
+
953
983
  def _replace_numbers(self, text):
954
984
  anon_map = getattr(self._document_context, 'anon_map', None)
955
985
  if anon_map is not None:
@@ -1006,6 +1036,9 @@ class TextCleaner:
1006
1036
  text,
1007
1037
  remove_brackets=self.cfg.check_remove_bracket_content,
1008
1038
  remove_braces=self.cfg.check_remove_brace_content,
1039
+ max_chars=self.cfg.bracket_content_max_chars,
1040
+ max_words=self.cfg.bracket_content_max_words,
1041
+ keep_json=self.cfg.keep_json_braces,
1009
1042
  )
1010
1043
 
1011
1044
  def _normalize_whitespace(self, text):
@@ -12,7 +12,7 @@ PUBLIC_VIEW_KEYS: tuple[str, ...] = (
12
12
  "regex_counts", "entity_counts", "sensitive", "warnings", "models",
13
13
  )
14
14
  # IP_ADDRESS / PHONE_NUMBER: the names under legacy_type_names=True.
15
- PII_REGEX_TYPES = frozenset({"EMAIL", "URL", "IP", "PHONE", "IP_ADDRESS", "PHONE_NUMBER"})
15
+ PII_REGEX_TYPES = frozenset({"EMAIL", "URL", "IP", "PHONE", "IP_ADDRESS", "PHONE_NUMBER", "IBAN", "CARD", "ID"})
16
16
  WARNING_CODES = frozenset({
17
17
  "language_unknown", "language_fallback", "language_low_confidence",
18
18
  "ner_fallback_models", "ner_off", "cuda_fallback", "empty_input",
@@ -84,13 +84,15 @@ class WindowedStreamingRedactor:
84
84
  # document gives it.
85
85
  cutoff = len(buffer) - self._overlap_chars
86
86
  phone_view = contact.PHONE_VIEW if cfg.fold_styled_characters else contact.PHONE_DIGIT_VIEW
87
- views = [constants.URL_REGEX, constants.EMAIL_REGEX, contact.IP_VIEW, phone_view]
87
+ views = [cleaner._url_regex, constants.EMAIL_REGEX, contact.IP_VIEW, phone_view]
88
88
  if cfg.check_replace_html: # a tag holding a space ('<a title="x y">')
89
- views.append(constants.HTML_REGEX)
89
+ views.append(cleaner._html_regex)
90
90
  if cfg.check_replace_dates:
91
91
  views.append(cleaner._date_view)
92
92
  if cfg.check_fuzzy_replace_dates: # every candidate the fuzzy step may mask ("15 Janury 1990")
93
93
  views.append(constants.FUZZY_DATE_CANDIDATE_REGEX)
94
+ if cfg.check_replace_iban_cards: # "NL91 ABNA 0417 1643 00"
95
+ views += [constants.IBAN_REGEX, constants.CARD_REGEX]
94
96
  if cfg.check_replace_numbers: # a grouped number ("5 379") is one token too
95
97
  views.append(contact.numbers_regex(cfg.replace_numbers_in_words))
96
98
  # (start, end) of view text a split must not cut: start < split < end