SqueakyCleanText 0.8.2__tar.gz → 0.8.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/PKG-INFO +16 -14
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/README.md +8 -8
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/SqueakyCleanText.egg-info/PKG-INFO +16 -14
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/SqueakyCleanText.egg-info/SOURCES.txt +30 -44
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/SqueakyCleanText.egg-info/requires.txt +9 -5
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/pyproject.toml +19 -6
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/__init__.py +3 -2
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/adapters.py +86 -71
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/browser.py +55 -86
- squeakycleantext-0.8.3/sct/cli.py +127 -0
- squeakycleantext-0.8.3/sct/config.py +938 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/evaluation.py +45 -30
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/integrations/_runtime.py +40 -34
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/integrations/datatrove.py +2 -2
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/integrations/haystack.py +10 -10
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/integrations/hf.py +13 -13
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/integrations/langchain.py +3 -2
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/mcp_server.py +114 -52
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/model_benchmark.py +3 -2
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/ocr.py +13 -6
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/presets.py +4 -2
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/privacy.py +7 -6
- squeakycleantext-0.8.3/sct/report_cli.py +179 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/risk.py +7 -4
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/score.py +31 -21
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/sct.py +549 -535
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/signals.py +56 -35
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/streaming.py +136 -112
- squeakycleantext-0.8.3/sct/utils/__init__.py +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/anonymization_map.py +15 -6
- squeakycleantext-0.8.3/sct/utils/atomic_file.py +44 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/constants.py +132 -76
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/contact.py +37 -95
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/datetime.py +14 -14
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/gliclass_adapter.py +6 -6
- squeakycleantext-0.8.3/sct/utils/gliner_adapter.py +227 -0
- squeakycleantext-0.8.3/sct/utils/ner.py +1056 -0
- squeakycleantext-0.8.3/sct/utils/ner_chunking.py +196 -0
- squeakycleantext-0.8.3/sct/utils/ner_spans.py +162 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/normtext.py +33 -49
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/onnx_pipeline.py +165 -156
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/process_result.py +33 -8
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/resources.py +8 -32
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/special.py +15 -18
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/stopwords.py +10 -14
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/synthetic.py +112 -52
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/utils/torch_pipeline.py +10 -14
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_adapters.py +89 -1
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_address_leftovers.py +4 -15
- squeakycleantext-0.8.3/tests/test_atomic_file.py +131 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_batching.py +62 -14
- squeakycleantext-0.8.3/tests/test_benchmark_adapters.py +519 -0
- squeakycleantext-0.8.3/tests/test_benchmark_loaders.py +240 -0
- squeakycleantext-0.8.3/tests/test_benchmark_release.py +287 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_benchmarks_compare.py +110 -7
- squeakycleantext-0.8.3/tests/test_benchmarks_round2.py +204 -0
- squeakycleantext-0.8.3/tests/test_benchmarks_round2_tools.py +349 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_benchmarks_score.py +4 -4
- squeakycleantext-0.8.3/tests/test_browser.py +101 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_chunking.py +46 -120
- squeakycleantext-0.8.3/tests/test_cleaner_lifecycle.py +185 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_cli.py +21 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_config_state.py +39 -1
- squeakycleantext-0.8.3/tests/test_contracts.py +222 -0
- squeakycleantext-0.8.3/tests/test_date_forms.py +91 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_dates_cjk_styled.py +13 -11
- squeakycleantext-0.8.3/tests/test_device_provenance.py +437 -0
- squeakycleantext-0.8.3/tests/test_email_forms.py +409 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_escaped_markup.py +10 -7
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_evaluation.py +23 -1
- squeakycleantext-0.8.3/tests/test_fork_safety.py +87 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_functional.py +5 -3
- squeakycleantext-0.8.3/tests/test_integrations_frameworks.py +169 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_integrations_hf.py +2 -2
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_integrations_runtime.py +56 -26
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_keep_scripts.py +3 -3
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_language_all.py +12 -21
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_linux_workers.py +2 -2
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_mcp_server.py +98 -14
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_ner_sees_tags.py +54 -3
- squeakycleantext-0.8.3/tests/test_ner_span_repair.py +406 -0
- squeakycleantext-0.8.3/tests/test_ner_spans.py +129 -0
- squeakycleantext-0.8.3/tests/test_ner_windows.py +360 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_number_identifiers.py +3 -4
- squeakycleantext-0.8.3/tests/test_ocr.py +170 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_onnx_concurrency.py +23 -18
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_onnx_external_data.py +40 -0
- squeakycleantext-0.8.3/tests/test_onnx_model_cache.py +104 -0
- squeakycleantext-0.8.3/tests/test_phone_forms.py +202 -0
- squeakycleantext-0.8.3/tests/test_pipeline_text_steps.py +162 -0
- squeakycleantext-0.8.3/tests/test_predicted_spans_monotone.py +136 -0
- squeakycleantext-0.8.3/tests/test_presidio_tags_and_fakes.py +363 -0
- squeakycleantext-0.8.3/tests/test_presidio_type_names.py +118 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_privacy.py +41 -39
- squeakycleantext-0.8.3/tests/test_prose_punctuation_kept.py +76 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_public_view.py +2 -2
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_regex_boundaries.py +16 -17
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_regex_counts.py +4 -4
- squeakycleantext-0.8.3/tests/test_repo_scripts.py +254 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_report_cli.py +54 -1
- squeakycleantext-0.8.3/tests/test_reversible_markers.py +184 -0
- squeakycleantext-0.8.3/tests/test_risk.py +42 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_score.py +6 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_sct.py +92 -110
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_security_leftovers.py +3 -15
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_stream_guard_shared_view.py +24 -2
- squeakycleantext-0.8.3/tests/test_stream_keeps_values_whole.py +263 -0
- squeakycleantext-0.8.3/tests/test_streaming.py +77 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_streaming_split_stability.py +14 -7
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_styled_pii.py +1 -1
- squeakycleantext-0.8.3/tests/test_surface_contracts.py +226 -0
- squeakycleantext-0.8.3/tests/test_synthetic_determinism.py +138 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_synthetic_per_document.py +6 -6
- squeakycleantext-0.8.3/tests/test_torch_ner_backend.py +104 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_training_preset.py +3 -2
- squeakycleantext-0.8.3/tests/test_url_forms.py +338 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_user_terms.py +4 -6
- squeakycleantext-0.8.2/sct/cli.py +0 -117
- squeakycleantext-0.8.2/sct/config.py +0 -1080
- squeakycleantext-0.8.2/sct/report_cli.py +0 -138
- squeakycleantext-0.8.2/sct/utils/gliner_adapter.py +0 -227
- squeakycleantext-0.8.2/sct/utils/ner.py +0 -1411
- squeakycleantext-0.8.2/setup.py +0 -2
- squeakycleantext-0.8.2/tests/test_benchmark_release.py +0 -97
- squeakycleantext-0.8.2/tests/test_browser.py +0 -37
- squeakycleantext-0.8.2/tests/test_codex_benchmark_round1.py +0 -84
- squeakycleantext-0.8.2/tests/test_codex_full_round1.py +0 -166
- squeakycleantext-0.8.2/tests/test_codex_full_round13.py +0 -74
- squeakycleantext-0.8.2/tests/test_codex_full_round14.py +0 -73
- squeakycleantext-0.8.2/tests/test_codex_full_round15.py +0 -154
- squeakycleantext-0.8.2/tests/test_codex_full_round16.py +0 -90
- squeakycleantext-0.8.2/tests/test_codex_full_round17.py +0 -78
- squeakycleantext-0.8.2/tests/test_codex_full_round18.py +0 -79
- squeakycleantext-0.8.2/tests/test_codex_full_round19.py +0 -32
- squeakycleantext-0.8.2/tests/test_codex_full_round2.py +0 -81
- squeakycleantext-0.8.2/tests/test_codex_full_round20.py +0 -87
- squeakycleantext-0.8.2/tests/test_codex_full_round21.py +0 -168
- squeakycleantext-0.8.2/tests/test_codex_full_round22.py +0 -52
- squeakycleantext-0.8.2/tests/test_codex_full_round23.py +0 -54
- squeakycleantext-0.8.2/tests/test_codex_full_round24.py +0 -82
- squeakycleantext-0.8.2/tests/test_codex_full_round25.py +0 -72
- squeakycleantext-0.8.2/tests/test_codex_full_round27.py +0 -85
- squeakycleantext-0.8.2/tests/test_codex_full_round28.py +0 -28
- squeakycleantext-0.8.2/tests/test_codex_full_round29.py +0 -44
- squeakycleantext-0.8.2/tests/test_codex_full_round3.py +0 -104
- squeakycleantext-0.8.2/tests/test_codex_full_round30.py +0 -53
- squeakycleantext-0.8.2/tests/test_codex_full_round31.py +0 -82
- squeakycleantext-0.8.2/tests/test_codex_full_round32.py +0 -52
- squeakycleantext-0.8.2/tests/test_codex_full_round35.py +0 -59
- squeakycleantext-0.8.2/tests/test_codex_full_round37.py +0 -41
- squeakycleantext-0.8.2/tests/test_codex_full_round38.py +0 -57
- squeakycleantext-0.8.2/tests/test_codex_full_round4.py +0 -44
- squeakycleantext-0.8.2/tests/test_codex_full_round5.py +0 -106
- squeakycleantext-0.8.2/tests/test_codex_full_round6.py +0 -80
- squeakycleantext-0.8.2/tests/test_codex_full_round7.py +0 -66
- squeakycleantext-0.8.2/tests/test_codex_full_round8.py +0 -35
- squeakycleantext-0.8.2/tests/test_codex_full_round9.py +0 -58
- squeakycleantext-0.8.2/tests/test_contracts.py +0 -143
- squeakycleantext-0.8.2/tests/test_device_provenance.py +0 -211
- squeakycleantext-0.8.2/tests/test_final_check_fixes.py +0 -273
- squeakycleantext-0.8.2/tests/test_full_test_round2.py +0 -145
- squeakycleantext-0.8.2/tests/test_fuzz_round1.py +0 -72
- squeakycleantext-0.8.2/tests/test_fuzz_round2.py +0 -31
- squeakycleantext-0.8.2/tests/test_fuzz_round3.py +0 -52
- squeakycleantext-0.8.2/tests/test_fuzz_round5.py +0 -51
- squeakycleantext-0.8.2/tests/test_fuzz_round6.py +0 -51
- squeakycleantext-0.8.2/tests/test_integrations_frameworks.py +0 -54
- squeakycleantext-0.8.2/tests/test_ocr.py +0 -87
- squeakycleantext-0.8.2/tests/test_review_round1.py +0 -159
- squeakycleantext-0.8.2/tests/test_review_round2.py +0 -69
- squeakycleantext-0.8.2/tests/test_review_round3.py +0 -101
- squeakycleantext-0.8.2/tests/test_review_round4.py +0 -68
- squeakycleantext-0.8.2/tests/test_risk.py +0 -21
- squeakycleantext-0.8.2/tests/test_streaming.py +0 -34
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/LICENSE +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/SqueakyCleanText.egg-info/dependency_links.txt +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/SqueakyCleanText.egg-info/entry_points.txt +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/SqueakyCleanText.egg-info/top_level.txt +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/integrations/__init__.py +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/sct/model_candidates.py +0 -0
- /squeakycleantext-0.8.2/sct/utils/__init__.py → /squeakycleantext-0.8.3/sct/py.typed +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/setup.cfg +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_cjk_width.py +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_dates_default.py +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_fuzz_styled_pii.py +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_mcp_path.py +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_model_benchmarks.py +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_multilingual_eval_set.py +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.3}/tests/test_type_names.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SqueakyCleanText
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.3
|
|
4
4
|
Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
|
|
5
5
|
Author: Rehan Fazal
|
|
6
6
|
License-Expression: MIT
|
|
@@ -44,7 +44,7 @@ Provides-Extra: fuzzy
|
|
|
44
44
|
Requires-Dist: rapidfuzz>=3.12; extra == "fuzzy"
|
|
45
45
|
Provides-Extra: torch
|
|
46
46
|
Requires-Dist: torch>=2.6.0; extra == "torch"
|
|
47
|
-
Requires-Dist: transformers>=
|
|
47
|
+
Requires-Dist: transformers>=5.0; extra == "torch"
|
|
48
48
|
Provides-Extra: gliner
|
|
49
49
|
Requires-Dist: gliner<1.0,>=0.2.25; extra == "gliner"
|
|
50
50
|
Provides-Extra: gliner2
|
|
@@ -53,13 +53,13 @@ Provides-Extra: synthetic
|
|
|
53
53
|
Requires-Dist: faker>=33.0; extra == "synthetic"
|
|
54
54
|
Provides-Extra: presidio
|
|
55
55
|
Requires-Dist: presidio-analyzer>=2.2.360; extra == "presidio"
|
|
56
|
+
Requires-Dist: gliner<1.0,>=0.2.25; extra == "presidio"
|
|
56
57
|
Provides-Extra: classify
|
|
57
58
|
Requires-Dist: gliclass>=0.1.16; extra == "classify"
|
|
58
59
|
Provides-Extra: all-ner
|
|
59
|
-
Requires-Dist: torch
|
|
60
|
-
|
|
61
|
-
Requires-Dist:
|
|
62
|
-
Requires-Dist: gliner2<2.0,>=1.0; extra == "all-ner"
|
|
60
|
+
Requires-Dist: SqueakyCleanText[gliner,gliner2,torch]; extra == "all-ner"
|
|
61
|
+
Provides-Extra: quantize
|
|
62
|
+
Requires-Dist: onnx>=1.19; extra == "quantize"
|
|
63
63
|
Provides-Extra: dev
|
|
64
64
|
Requires-Dist: hypothesis>=6.130; extra == "dev"
|
|
65
65
|
Requires-Dist: faker>=33.0; extra == "dev"
|
|
@@ -74,6 +74,8 @@ Provides-Extra: mcp
|
|
|
74
74
|
Requires-Dist: fastmcp>=4.0.5; extra == "mcp"
|
|
75
75
|
Provides-Extra: score
|
|
76
76
|
Requires-Dist: datasketch>=1.6; extra == "score"
|
|
77
|
+
Provides-Extra: parquet
|
|
78
|
+
Requires-Dist: pyarrow>=17; extra == "parquet"
|
|
77
79
|
Provides-Extra: datasets
|
|
78
80
|
Requires-Dist: datasets>=3.0; extra == "datasets"
|
|
79
81
|
Provides-Extra: datatrove
|
|
@@ -228,7 +230,7 @@ if __name__ == "__main__":
|
|
|
228
230
|
| Data card for a file | `sct-report data.jsonl --text-column text --out report.json --card card.md` | [SCORE](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md) |
|
|
229
231
|
| AI assistants | `sct-mcp` (MCP server, `mcp` extra) | [MCP](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MCP.md) |
|
|
230
232
|
|
|
231
|
-
The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
|
|
233
|
+
The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
|
|
232
234
|
|
|
233
235
|
## How it works
|
|
234
236
|
|
|
@@ -258,7 +260,7 @@ Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/r
|
|
|
258
260
|
- `lm_text`: the cleaned text, sentences intact, for LLMs, RAG and training.
|
|
259
261
|
- `stat_text`: lower-cased, without stopwords and punctuation, for classic ML models.
|
|
260
262
|
- `language`: the detected language.
|
|
261
|
-
- `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive
|
|
263
|
+
- `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive` (`pii_count > 0`), warnings and the NER models that ran. Safe to log or store next to the text.
|
|
262
264
|
- `metadata`: the same counts, plus raw values only when you ask for them (the reversible map, or `findings` with `include_entities=True`). Keep those out of logs.
|
|
263
265
|
|
|
264
266
|
`score_record` turns a result into flat `sct_*` columns for filtering and data cards. `ScoreConfig` adds an optional quality score and a MinHash signature. Fields: [docs/API.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/API.md), columns: [docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md).
|
|
@@ -276,17 +278,17 @@ Modes, privacy policies and more PII types (`ner_mode="pii"`): [docs/REPLACEMENT
|
|
|
276
278
|
|
|
277
279
|
## Speed and hardware
|
|
278
280
|
|
|
279
|
-
Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0,
|
|
281
|
+
Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, on a workstation CPU and high-end GPUs, median of 3 runs. 0.8.1 runs at the same speed.
|
|
280
282
|
|
|
281
283
|
| Workload | Hardware | Docs/s |
|
|
282
284
|
|---|---|---|
|
|
283
285
|
| `process`, one document at a time | 16 CPU cores | 5.0 |
|
|
284
286
|
| `process_batch`, short documents, 16 workers | 16 CPU cores | 15.5 |
|
|
285
287
|
| `process_batch`, short documents, 16 workers | 64 CPU cores | 29.2 |
|
|
286
|
-
| `process_batch`, short documents, full ensemble |
|
|
287
|
-
| `process_batch`, short documents, English only (2 models) |
|
|
288
|
+
| `process_batch`, short documents, full ensemble | 3 GPUs (models split across them) | 130 |
|
|
289
|
+
| `process_batch`, short documents, English only (2 models) | 1 GPU | 646 |
|
|
288
290
|
|
|
289
|
-
The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one
|
|
291
|
+
The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one GPU, about 2 hours with the full ensemble on 3 GPUs, and about 10 hours on 64 CPU cores.
|
|
290
292
|
|
|
291
293
|
NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cleaning, or limit `language=` to load fewer models. Hardware advice and GPU setup: [docs/PERFORMANCE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PERFORMANCE.md).
|
|
292
294
|
|
|
@@ -316,7 +318,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
|
|
|
316
318
|
- **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
|
|
317
319
|
<!-- --8<-- [end:limits] -->
|
|
318
320
|
|
|
319
|
-
Detailed known limits of 0.8.
|
|
321
|
+
Detailed known limits of 0.8.3: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
|
|
320
322
|
|
|
321
323
|
## Documentation
|
|
322
324
|
|
|
@@ -348,7 +350,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
|
|
|
348
350
|
author = {Fazal, Rehan},
|
|
349
351
|
title = {SqueakyCleanText},
|
|
350
352
|
url = {https://github.com/rhnfzl/SqueakyCleanText},
|
|
351
|
-
version = {0.8.
|
|
353
|
+
version = {0.8.3},
|
|
352
354
|
year = {2026}
|
|
353
355
|
}
|
|
354
356
|
```
|
|
@@ -139,7 +139,7 @@ if __name__ == "__main__":
|
|
|
139
139
|
| Data card for a file | `sct-report data.jsonl --text-column text --out report.json --card card.md` | [SCORE](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md) |
|
|
140
140
|
| AI assistants | `sct-mcp` (MCP server, `mcp` extra) | [MCP](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MCP.md) |
|
|
141
141
|
|
|
142
|
-
The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
|
|
142
|
+
The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
|
|
143
143
|
|
|
144
144
|
## How it works
|
|
145
145
|
|
|
@@ -169,7 +169,7 @@ Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/r
|
|
|
169
169
|
- `lm_text`: the cleaned text, sentences intact, for LLMs, RAG and training.
|
|
170
170
|
- `stat_text`: lower-cased, without stopwords and punctuation, for classic ML models.
|
|
171
171
|
- `language`: the detected language.
|
|
172
|
-
- `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive
|
|
172
|
+
- `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive` (`pii_count > 0`), warnings and the NER models that ran. Safe to log or store next to the text.
|
|
173
173
|
- `metadata`: the same counts, plus raw values only when you ask for them (the reversible map, or `findings` with `include_entities=True`). Keep those out of logs.
|
|
174
174
|
|
|
175
175
|
`score_record` turns a result into flat `sct_*` columns for filtering and data cards. `ScoreConfig` adds an optional quality score and a MinHash signature. Fields: [docs/API.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/API.md), columns: [docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md).
|
|
@@ -187,17 +187,17 @@ Modes, privacy policies and more PII types (`ner_mode="pii"`): [docs/REPLACEMENT
|
|
|
187
187
|
|
|
188
188
|
## Speed and hardware
|
|
189
189
|
|
|
190
|
-
Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0,
|
|
190
|
+
Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, on a workstation CPU and high-end GPUs, median of 3 runs. 0.8.1 runs at the same speed.
|
|
191
191
|
|
|
192
192
|
| Workload | Hardware | Docs/s |
|
|
193
193
|
|---|---|---|
|
|
194
194
|
| `process`, one document at a time | 16 CPU cores | 5.0 |
|
|
195
195
|
| `process_batch`, short documents, 16 workers | 16 CPU cores | 15.5 |
|
|
196
196
|
| `process_batch`, short documents, 16 workers | 64 CPU cores | 29.2 |
|
|
197
|
-
| `process_batch`, short documents, full ensemble |
|
|
198
|
-
| `process_batch`, short documents, English only (2 models) |
|
|
197
|
+
| `process_batch`, short documents, full ensemble | 3 GPUs (models split across them) | 130 |
|
|
198
|
+
| `process_batch`, short documents, English only (2 models) | 1 GPU | 646 |
|
|
199
199
|
|
|
200
|
-
The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one
|
|
200
|
+
The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one GPU, about 2 hours with the full ensemble on 3 GPUs, and about 10 hours on 64 CPU cores.
|
|
201
201
|
|
|
202
202
|
NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cleaning, or limit `language=` to load fewer models. Hardware advice and GPU setup: [docs/PERFORMANCE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PERFORMANCE.md).
|
|
203
203
|
|
|
@@ -227,7 +227,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
|
|
|
227
227
|
- **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
|
|
228
228
|
<!-- --8<-- [end:limits] -->
|
|
229
229
|
|
|
230
|
-
Detailed known limits of 0.8.
|
|
230
|
+
Detailed known limits of 0.8.3: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
|
|
231
231
|
|
|
232
232
|
## Documentation
|
|
233
233
|
|
|
@@ -259,7 +259,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
|
|
|
259
259
|
author = {Fazal, Rehan},
|
|
260
260
|
title = {SqueakyCleanText},
|
|
261
261
|
url = {https://github.com/rhnfzl/SqueakyCleanText},
|
|
262
|
-
version = {0.8.
|
|
262
|
+
version = {0.8.3},
|
|
263
263
|
year = {2026}
|
|
264
264
|
}
|
|
265
265
|
```
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SqueakyCleanText
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.3
|
|
4
4
|
Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
|
|
5
5
|
Author: Rehan Fazal
|
|
6
6
|
License-Expression: MIT
|
|
@@ -44,7 +44,7 @@ Provides-Extra: fuzzy
|
|
|
44
44
|
Requires-Dist: rapidfuzz>=3.12; extra == "fuzzy"
|
|
45
45
|
Provides-Extra: torch
|
|
46
46
|
Requires-Dist: torch>=2.6.0; extra == "torch"
|
|
47
|
-
Requires-Dist: transformers>=
|
|
47
|
+
Requires-Dist: transformers>=5.0; extra == "torch"
|
|
48
48
|
Provides-Extra: gliner
|
|
49
49
|
Requires-Dist: gliner<1.0,>=0.2.25; extra == "gliner"
|
|
50
50
|
Provides-Extra: gliner2
|
|
@@ -53,13 +53,13 @@ Provides-Extra: synthetic
|
|
|
53
53
|
Requires-Dist: faker>=33.0; extra == "synthetic"
|
|
54
54
|
Provides-Extra: presidio
|
|
55
55
|
Requires-Dist: presidio-analyzer>=2.2.360; extra == "presidio"
|
|
56
|
+
Requires-Dist: gliner<1.0,>=0.2.25; extra == "presidio"
|
|
56
57
|
Provides-Extra: classify
|
|
57
58
|
Requires-Dist: gliclass>=0.1.16; extra == "classify"
|
|
58
59
|
Provides-Extra: all-ner
|
|
59
|
-
Requires-Dist: torch
|
|
60
|
-
|
|
61
|
-
Requires-Dist:
|
|
62
|
-
Requires-Dist: gliner2<2.0,>=1.0; extra == "all-ner"
|
|
60
|
+
Requires-Dist: SqueakyCleanText[gliner,gliner2,torch]; extra == "all-ner"
|
|
61
|
+
Provides-Extra: quantize
|
|
62
|
+
Requires-Dist: onnx>=1.19; extra == "quantize"
|
|
63
63
|
Provides-Extra: dev
|
|
64
64
|
Requires-Dist: hypothesis>=6.130; extra == "dev"
|
|
65
65
|
Requires-Dist: faker>=33.0; extra == "dev"
|
|
@@ -74,6 +74,8 @@ Provides-Extra: mcp
|
|
|
74
74
|
Requires-Dist: fastmcp>=4.0.5; extra == "mcp"
|
|
75
75
|
Provides-Extra: score
|
|
76
76
|
Requires-Dist: datasketch>=1.6; extra == "score"
|
|
77
|
+
Provides-Extra: parquet
|
|
78
|
+
Requires-Dist: pyarrow>=17; extra == "parquet"
|
|
77
79
|
Provides-Extra: datasets
|
|
78
80
|
Requires-Dist: datasets>=3.0; extra == "datasets"
|
|
79
81
|
Provides-Extra: datatrove
|
|
@@ -228,7 +230,7 @@ if __name__ == "__main__":
|
|
|
228
230
|
| Data card for a file | `sct-report data.jsonl --text-column text --out report.json --card card.md` | [SCORE](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md) |
|
|
229
231
|
| AI assistants | `sct-mcp` (MCP server, `mcp` extra) | [MCP](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MCP.md) |
|
|
230
232
|
|
|
231
|
-
The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
|
|
233
|
+
The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
|
|
232
234
|
|
|
233
235
|
## How it works
|
|
234
236
|
|
|
@@ -258,7 +260,7 @@ Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/r
|
|
|
258
260
|
- `lm_text`: the cleaned text, sentences intact, for LLMs, RAG and training.
|
|
259
261
|
- `stat_text`: lower-cased, without stopwords and punctuation, for classic ML models.
|
|
260
262
|
- `language`: the detected language.
|
|
261
|
-
- `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive
|
|
263
|
+
- `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive` (`pii_count > 0`), warnings and the NER models that ran. Safe to log or store next to the text.
|
|
262
264
|
- `metadata`: the same counts, plus raw values only when you ask for them (the reversible map, or `findings` with `include_entities=True`). Keep those out of logs.
|
|
263
265
|
|
|
264
266
|
`score_record` turns a result into flat `sct_*` columns for filtering and data cards. `ScoreConfig` adds an optional quality score and a MinHash signature. Fields: [docs/API.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/API.md), columns: [docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md).
|
|
@@ -276,17 +278,17 @@ Modes, privacy policies and more PII types (`ner_mode="pii"`): [docs/REPLACEMENT
|
|
|
276
278
|
|
|
277
279
|
## Speed and hardware
|
|
278
280
|
|
|
279
|
-
Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0,
|
|
281
|
+
Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, on a workstation CPU and high-end GPUs, median of 3 runs. 0.8.1 runs at the same speed.
|
|
280
282
|
|
|
281
283
|
| Workload | Hardware | Docs/s |
|
|
282
284
|
|---|---|---|
|
|
283
285
|
| `process`, one document at a time | 16 CPU cores | 5.0 |
|
|
284
286
|
| `process_batch`, short documents, 16 workers | 16 CPU cores | 15.5 |
|
|
285
287
|
| `process_batch`, short documents, 16 workers | 64 CPU cores | 29.2 |
|
|
286
|
-
| `process_batch`, short documents, full ensemble |
|
|
287
|
-
| `process_batch`, short documents, English only (2 models) |
|
|
288
|
+
| `process_batch`, short documents, full ensemble | 3 GPUs (models split across them) | 130 |
|
|
289
|
+
| `process_batch`, short documents, English only (2 models) | 1 GPU | 646 |
|
|
288
290
|
|
|
289
|
-
The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one
|
|
291
|
+
The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one GPU, about 2 hours with the full ensemble on 3 GPUs, and about 10 hours on 64 CPU cores.
|
|
290
292
|
|
|
291
293
|
NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cleaning, or limit `language=` to load fewer models. Hardware advice and GPU setup: [docs/PERFORMANCE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PERFORMANCE.md).
|
|
292
294
|
|
|
@@ -316,7 +318,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
|
|
|
316
318
|
- **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
|
|
317
319
|
<!-- --8<-- [end:limits] -->
|
|
318
320
|
|
|
319
|
-
Detailed known limits of 0.8.
|
|
321
|
+
Detailed known limits of 0.8.3: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
|
|
320
322
|
|
|
321
323
|
## Documentation
|
|
322
324
|
|
|
@@ -348,7 +350,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
|
|
|
348
350
|
author = {Fazal, Rehan},
|
|
349
351
|
title = {SqueakyCleanText},
|
|
350
352
|
url = {https://github.com/rhnfzl/SqueakyCleanText},
|
|
351
|
-
version = {0.8.
|
|
353
|
+
version = {0.8.3},
|
|
352
354
|
year = {2026}
|
|
353
355
|
}
|
|
354
356
|
```
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
LICENSE
|
|
2
2
|
README.md
|
|
3
3
|
pyproject.toml
|
|
4
|
-
setup.py
|
|
5
4
|
SqueakyCleanText.egg-info/PKG-INFO
|
|
6
5
|
SqueakyCleanText.egg-info/SOURCES.txt
|
|
7
6
|
SqueakyCleanText.egg-info/dependency_links.txt
|
|
@@ -20,6 +19,7 @@ sct/model_candidates.py
|
|
|
20
19
|
sct/ocr.py
|
|
21
20
|
sct/presets.py
|
|
22
21
|
sct/privacy.py
|
|
22
|
+
sct/py.typed
|
|
23
23
|
sct/report_cli.py
|
|
24
24
|
sct/risk.py
|
|
25
25
|
sct/score.py
|
|
@@ -34,12 +34,15 @@ sct/integrations/hf.py
|
|
|
34
34
|
sct/integrations/langchain.py
|
|
35
35
|
sct/utils/__init__.py
|
|
36
36
|
sct/utils/anonymization_map.py
|
|
37
|
+
sct/utils/atomic_file.py
|
|
37
38
|
sct/utils/constants.py
|
|
38
39
|
sct/utils/contact.py
|
|
39
40
|
sct/utils/datetime.py
|
|
40
41
|
sct/utils/gliclass_adapter.py
|
|
41
42
|
sct/utils/gliner_adapter.py
|
|
42
43
|
sct/utils/ner.py
|
|
44
|
+
sct/utils/ner_chunking.py
|
|
45
|
+
sct/utils/ner_spans.py
|
|
43
46
|
sct/utils/normtext.py
|
|
44
47
|
sct/utils/onnx_pipeline.py
|
|
45
48
|
sct/utils/process_result.py
|
|
@@ -50,61 +53,31 @@ sct/utils/synthetic.py
|
|
|
50
53
|
sct/utils/torch_pipeline.py
|
|
51
54
|
tests/test_adapters.py
|
|
52
55
|
tests/test_address_leftovers.py
|
|
56
|
+
tests/test_atomic_file.py
|
|
53
57
|
tests/test_batching.py
|
|
58
|
+
tests/test_benchmark_adapters.py
|
|
59
|
+
tests/test_benchmark_loaders.py
|
|
54
60
|
tests/test_benchmark_release.py
|
|
55
61
|
tests/test_benchmarks_compare.py
|
|
62
|
+
tests/test_benchmarks_round2.py
|
|
63
|
+
tests/test_benchmarks_round2_tools.py
|
|
56
64
|
tests/test_benchmarks_score.py
|
|
57
65
|
tests/test_browser.py
|
|
58
66
|
tests/test_chunking.py
|
|
59
67
|
tests/test_cjk_width.py
|
|
68
|
+
tests/test_cleaner_lifecycle.py
|
|
60
69
|
tests/test_cli.py
|
|
61
|
-
tests/test_codex_benchmark_round1.py
|
|
62
|
-
tests/test_codex_full_round1.py
|
|
63
|
-
tests/test_codex_full_round13.py
|
|
64
|
-
tests/test_codex_full_round14.py
|
|
65
|
-
tests/test_codex_full_round15.py
|
|
66
|
-
tests/test_codex_full_round16.py
|
|
67
|
-
tests/test_codex_full_round17.py
|
|
68
|
-
tests/test_codex_full_round18.py
|
|
69
|
-
tests/test_codex_full_round19.py
|
|
70
|
-
tests/test_codex_full_round2.py
|
|
71
|
-
tests/test_codex_full_round20.py
|
|
72
|
-
tests/test_codex_full_round21.py
|
|
73
|
-
tests/test_codex_full_round22.py
|
|
74
|
-
tests/test_codex_full_round23.py
|
|
75
|
-
tests/test_codex_full_round24.py
|
|
76
|
-
tests/test_codex_full_round25.py
|
|
77
|
-
tests/test_codex_full_round27.py
|
|
78
|
-
tests/test_codex_full_round28.py
|
|
79
|
-
tests/test_codex_full_round29.py
|
|
80
|
-
tests/test_codex_full_round3.py
|
|
81
|
-
tests/test_codex_full_round30.py
|
|
82
|
-
tests/test_codex_full_round31.py
|
|
83
|
-
tests/test_codex_full_round32.py
|
|
84
|
-
tests/test_codex_full_round35.py
|
|
85
|
-
tests/test_codex_full_round37.py
|
|
86
|
-
tests/test_codex_full_round38.py
|
|
87
|
-
tests/test_codex_full_round4.py
|
|
88
|
-
tests/test_codex_full_round5.py
|
|
89
|
-
tests/test_codex_full_round6.py
|
|
90
|
-
tests/test_codex_full_round7.py
|
|
91
|
-
tests/test_codex_full_round8.py
|
|
92
|
-
tests/test_codex_full_round9.py
|
|
93
70
|
tests/test_config_state.py
|
|
94
71
|
tests/test_contracts.py
|
|
72
|
+
tests/test_date_forms.py
|
|
95
73
|
tests/test_dates_cjk_styled.py
|
|
96
74
|
tests/test_dates_default.py
|
|
97
75
|
tests/test_device_provenance.py
|
|
76
|
+
tests/test_email_forms.py
|
|
98
77
|
tests/test_escaped_markup.py
|
|
99
78
|
tests/test_evaluation.py
|
|
100
|
-
tests/
|
|
101
|
-
tests/test_full_test_round2.py
|
|
79
|
+
tests/test_fork_safety.py
|
|
102
80
|
tests/test_functional.py
|
|
103
|
-
tests/test_fuzz_round1.py
|
|
104
|
-
tests/test_fuzz_round2.py
|
|
105
|
-
tests/test_fuzz_round3.py
|
|
106
|
-
tests/test_fuzz_round5.py
|
|
107
|
-
tests/test_fuzz_round6.py
|
|
108
81
|
tests/test_fuzz_styled_pii.py
|
|
109
82
|
tests/test_integrations_frameworks.py
|
|
110
83
|
tests/test_integrations_hf.py
|
|
@@ -117,28 +90,41 @@ tests/test_mcp_server.py
|
|
|
117
90
|
tests/test_model_benchmarks.py
|
|
118
91
|
tests/test_multilingual_eval_set.py
|
|
119
92
|
tests/test_ner_sees_tags.py
|
|
93
|
+
tests/test_ner_span_repair.py
|
|
94
|
+
tests/test_ner_spans.py
|
|
95
|
+
tests/test_ner_windows.py
|
|
120
96
|
tests/test_number_identifiers.py
|
|
121
97
|
tests/test_ocr.py
|
|
122
98
|
tests/test_onnx_concurrency.py
|
|
123
99
|
tests/test_onnx_external_data.py
|
|
100
|
+
tests/test_onnx_model_cache.py
|
|
101
|
+
tests/test_phone_forms.py
|
|
102
|
+
tests/test_pipeline_text_steps.py
|
|
103
|
+
tests/test_predicted_spans_monotone.py
|
|
104
|
+
tests/test_presidio_tags_and_fakes.py
|
|
105
|
+
tests/test_presidio_type_names.py
|
|
124
106
|
tests/test_privacy.py
|
|
107
|
+
tests/test_prose_punctuation_kept.py
|
|
125
108
|
tests/test_public_view.py
|
|
126
109
|
tests/test_regex_boundaries.py
|
|
127
110
|
tests/test_regex_counts.py
|
|
111
|
+
tests/test_repo_scripts.py
|
|
128
112
|
tests/test_report_cli.py
|
|
129
|
-
tests/
|
|
130
|
-
tests/test_review_round2.py
|
|
131
|
-
tests/test_review_round3.py
|
|
132
|
-
tests/test_review_round4.py
|
|
113
|
+
tests/test_reversible_markers.py
|
|
133
114
|
tests/test_risk.py
|
|
134
115
|
tests/test_score.py
|
|
135
116
|
tests/test_sct.py
|
|
136
117
|
tests/test_security_leftovers.py
|
|
137
118
|
tests/test_stream_guard_shared_view.py
|
|
119
|
+
tests/test_stream_keeps_values_whole.py
|
|
138
120
|
tests/test_streaming.py
|
|
139
121
|
tests/test_streaming_split_stability.py
|
|
140
122
|
tests/test_styled_pii.py
|
|
123
|
+
tests/test_surface_contracts.py
|
|
124
|
+
tests/test_synthetic_determinism.py
|
|
141
125
|
tests/test_synthetic_per_document.py
|
|
126
|
+
tests/test_torch_ner_backend.py
|
|
142
127
|
tests/test_training_preset.py
|
|
143
128
|
tests/test_type_names.py
|
|
129
|
+
tests/test_url_forms.py
|
|
144
130
|
tests/test_user_terms.py
|
|
@@ -11,10 +11,7 @@ presidio_anonymizer>=2.2.360
|
|
|
11
11
|
regex>=2024.11.6
|
|
12
12
|
|
|
13
13
|
[all-ner]
|
|
14
|
-
torch
|
|
15
|
-
transformers>=4.48
|
|
16
|
-
gliner<1.0,>=0.2.25
|
|
17
|
-
gliner2<2.0,>=1.0
|
|
14
|
+
SqueakyCleanText[gliner,gliner2,torch]
|
|
18
15
|
|
|
19
16
|
[classify]
|
|
20
17
|
gliclass>=0.1.16
|
|
@@ -58,8 +55,15 @@ fastmcp>=4.0.5
|
|
|
58
55
|
pillow>=12.3.0
|
|
59
56
|
pytesseract>=0.3.13
|
|
60
57
|
|
|
58
|
+
[parquet]
|
|
59
|
+
pyarrow>=17
|
|
60
|
+
|
|
61
61
|
[presidio]
|
|
62
62
|
presidio-analyzer>=2.2.360
|
|
63
|
+
gliner<1.0,>=0.2.25
|
|
64
|
+
|
|
65
|
+
[quantize]
|
|
66
|
+
onnx>=1.19
|
|
63
67
|
|
|
64
68
|
[score]
|
|
65
69
|
datasketch>=1.6
|
|
@@ -73,4 +77,4 @@ pytest-cov>=6.0
|
|
|
73
77
|
|
|
74
78
|
[torch]
|
|
75
79
|
torch>=2.6.0
|
|
76
|
-
transformers>=
|
|
80
|
+
transformers>=5.0
|
|
@@ -66,7 +66,7 @@ fuzzy = [
|
|
|
66
66
|
]
|
|
67
67
|
torch = [
|
|
68
68
|
"torch>=2.6.0",
|
|
69
|
-
"transformers>=
|
|
69
|
+
"transformers>=5.0",
|
|
70
70
|
]
|
|
71
71
|
gliner = [
|
|
72
72
|
"gliner>=0.2.25,<1.0",
|
|
@@ -79,15 +79,16 @@ synthetic = [
|
|
|
79
79
|
]
|
|
80
80
|
presidio = [
|
|
81
81
|
"presidio-analyzer>=2.2.360",
|
|
82
|
+
"gliner>=0.2.25,<1.0",
|
|
82
83
|
]
|
|
83
84
|
classify = [
|
|
84
85
|
"gliclass>=0.1.16",
|
|
85
86
|
]
|
|
86
87
|
all-ner = [
|
|
87
|
-
"torch
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
"
|
|
88
|
+
"SqueakyCleanText[torch,gliner,gliner2]",
|
|
89
|
+
]
|
|
90
|
+
quantize = [
|
|
91
|
+
"onnx>=1.19",
|
|
91
92
|
]
|
|
92
93
|
dev = [
|
|
93
94
|
"hypothesis>=6.130",
|
|
@@ -107,6 +108,9 @@ mcp = [
|
|
|
107
108
|
score = [
|
|
108
109
|
"datasketch>=1.6",
|
|
109
110
|
]
|
|
111
|
+
parquet = [
|
|
112
|
+
"pyarrow>=17",
|
|
113
|
+
]
|
|
110
114
|
datasets = [
|
|
111
115
|
"datasets>=3.0",
|
|
112
116
|
]
|
|
@@ -131,12 +135,21 @@ version = {attr = "sct.__version__"}
|
|
|
131
135
|
include = ["sct*"]
|
|
132
136
|
exclude = ["tests*"]
|
|
133
137
|
|
|
138
|
+
[tool.setuptools.package-data]
|
|
139
|
+
sct = ["py.typed"]
|
|
140
|
+
|
|
134
141
|
[tool.pytest.ini_options]
|
|
135
142
|
timeout = 1200
|
|
143
|
+
markers = ["needs_model: loads a real Hugging Face model"]
|
|
136
144
|
|
|
137
145
|
[tool.ruff]
|
|
138
146
|
line-length = 127
|
|
139
147
|
|
|
140
148
|
[tool.ruff.lint]
|
|
141
|
-
select = ["E", "F", "W", "S"]
|
|
149
|
+
select = ["E", "F", "W", "S", "UP"]
|
|
142
150
|
ignore = ["S101"]
|
|
151
|
+
|
|
152
|
+
[tool.mypy]
|
|
153
|
+
files = ["sct"]
|
|
154
|
+
ignore_missing_imports = true
|
|
155
|
+
check_untyped_defs = true
|
|
@@ -11,6 +11,7 @@ from sct.privacy import (
|
|
|
11
11
|
TokenStore,
|
|
12
12
|
)
|
|
13
13
|
from sct.sct import CustomPatternTimeout, TextCleaner
|
|
14
|
+
from sct.utils.ner import ModelLoadError
|
|
14
15
|
from sct.presets import PRESETS, preset_config
|
|
15
16
|
from sct.adapters import (
|
|
16
17
|
DataFrameResult,
|
|
@@ -50,9 +51,9 @@ from sct.streaming import (
|
|
|
50
51
|
from sct.utils.anonymization_map import AnonymizationMap, MapEntry
|
|
51
52
|
from sct.utils.process_result import ProcessResult
|
|
52
53
|
|
|
53
|
-
__version__ = "0.8.
|
|
54
|
+
__version__ = "0.8.3"
|
|
54
55
|
__all__ = [
|
|
55
|
-
"TextCleaner", "TextCleanerConfig", "CustomPatternTimeout", "PRESETS", "preset_config",
|
|
56
|
+
"TextCleaner", "TextCleanerConfig", "CustomPatternTimeout", "ModelLoadError", "PRESETS", "preset_config",
|
|
56
57
|
"PII_LABELS", "PII_LABEL_MAP",
|
|
57
58
|
"AnonymizationMap", "MapEntry", "ProcessResult",
|
|
58
59
|
"DetectedEntity", "EntityFinding", "EntityRule",
|