SqueakyCleanText 0.8.2__tar.gz → 0.8.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/PKG-INFO +22 -16
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/README.md +14 -10
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/PKG-INFO +22 -16
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/SOURCES.txt +30 -44
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/requires.txt +9 -5
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/pyproject.toml +19 -6
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/__init__.py +3 -2
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/adapters.py +86 -71
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/browser.py +55 -86
- squeakycleantext-0.8.4/sct/cli.py +127 -0
- squeakycleantext-0.8.4/sct/config.py +938 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/evaluation.py +45 -30
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/integrations/_runtime.py +40 -34
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/integrations/datatrove.py +2 -2
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/integrations/haystack.py +10 -10
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/integrations/hf.py +13 -13
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/integrations/langchain.py +3 -2
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/mcp_server.py +114 -52
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/model_benchmark.py +3 -2
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/ocr.py +13 -6
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/presets.py +4 -2
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/privacy.py +7 -6
- squeakycleantext-0.8.4/sct/report_cli.py +179 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/risk.py +7 -4
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/score.py +31 -21
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/sct.py +549 -535
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/signals.py +56 -35
- squeakycleantext-0.8.4/sct/streaming.py +311 -0
- squeakycleantext-0.8.4/sct/utils/__init__.py +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/anonymization_map.py +15 -6
- squeakycleantext-0.8.4/sct/utils/atomic_file.py +44 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/constants.py +132 -76
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/contact.py +37 -95
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/datetime.py +14 -14
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/gliclass_adapter.py +6 -6
- squeakycleantext-0.8.4/sct/utils/gliner_adapter.py +227 -0
- squeakycleantext-0.8.4/sct/utils/ner.py +1056 -0
- squeakycleantext-0.8.4/sct/utils/ner_chunking.py +196 -0
- squeakycleantext-0.8.4/sct/utils/ner_spans.py +162 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/normtext.py +33 -49
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/onnx_pipeline.py +165 -156
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/process_result.py +33 -8
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/resources.py +8 -32
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/special.py +15 -18
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/stopwords.py +10 -14
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/synthetic.py +112 -52
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/utils/torch_pipeline.py +10 -14
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_adapters.py +89 -1
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_address_leftovers.py +4 -15
- squeakycleantext-0.8.4/tests/test_atomic_file.py +131 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_batching.py +62 -14
- squeakycleantext-0.8.4/tests/test_benchmark_adapters.py +519 -0
- squeakycleantext-0.8.4/tests/test_benchmark_loaders.py +240 -0
- squeakycleantext-0.8.4/tests/test_benchmark_release.py +287 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_benchmarks_compare.py +110 -7
- squeakycleantext-0.8.4/tests/test_benchmarks_round2.py +204 -0
- squeakycleantext-0.8.4/tests/test_benchmarks_round2_tools.py +349 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_benchmarks_score.py +4 -4
- squeakycleantext-0.8.4/tests/test_browser.py +101 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_chunking.py +46 -120
- squeakycleantext-0.8.4/tests/test_cleaner_lifecycle.py +185 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_cli.py +21 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_config_state.py +39 -1
- squeakycleantext-0.8.4/tests/test_contracts.py +222 -0
- squeakycleantext-0.8.4/tests/test_date_forms.py +91 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_dates_cjk_styled.py +13 -11
- squeakycleantext-0.8.4/tests/test_device_provenance.py +437 -0
- squeakycleantext-0.8.4/tests/test_email_forms.py +409 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_escaped_markup.py +10 -7
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_evaluation.py +23 -1
- squeakycleantext-0.8.4/tests/test_fork_safety.py +87 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_functional.py +5 -3
- squeakycleantext-0.8.4/tests/test_integrations_frameworks.py +169 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_integrations_hf.py +2 -2
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_integrations_runtime.py +56 -26
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_keep_scripts.py +3 -3
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_language_all.py +12 -21
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_linux_workers.py +2 -2
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_mcp_server.py +98 -14
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_ner_sees_tags.py +54 -3
- squeakycleantext-0.8.4/tests/test_ner_span_repair.py +406 -0
- squeakycleantext-0.8.4/tests/test_ner_spans.py +129 -0
- squeakycleantext-0.8.4/tests/test_ner_windows.py +360 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_number_identifiers.py +3 -4
- squeakycleantext-0.8.4/tests/test_ocr.py +170 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_onnx_concurrency.py +23 -18
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_onnx_external_data.py +40 -0
- squeakycleantext-0.8.4/tests/test_onnx_model_cache.py +104 -0
- squeakycleantext-0.8.4/tests/test_phone_forms.py +202 -0
- squeakycleantext-0.8.4/tests/test_pipeline_text_steps.py +162 -0
- squeakycleantext-0.8.4/tests/test_predicted_spans_monotone.py +136 -0
- squeakycleantext-0.8.4/tests/test_presidio_tags_and_fakes.py +363 -0
- squeakycleantext-0.8.4/tests/test_presidio_type_names.py +118 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_privacy.py +41 -39
- squeakycleantext-0.8.4/tests/test_prose_punctuation_kept.py +76 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_public_view.py +2 -2
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_regex_boundaries.py +16 -17
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_regex_counts.py +4 -4
- squeakycleantext-0.8.4/tests/test_repo_scripts.py +254 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_report_cli.py +54 -1
- squeakycleantext-0.8.4/tests/test_reversible_markers.py +184 -0
- squeakycleantext-0.8.4/tests/test_risk.py +42 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_score.py +6 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_sct.py +92 -110
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_security_leftovers.py +3 -15
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_stream_guard_shared_view.py +24 -2
- squeakycleantext-0.8.4/tests/test_stream_keeps_values_whole.py +263 -0
- squeakycleantext-0.8.4/tests/test_streaming.py +175 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_streaming_split_stability.py +14 -7
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_styled_pii.py +1 -1
- squeakycleantext-0.8.4/tests/test_surface_contracts.py +226 -0
- squeakycleantext-0.8.4/tests/test_synthetic_determinism.py +138 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_synthetic_per_document.py +6 -6
- squeakycleantext-0.8.4/tests/test_torch_ner_backend.py +104 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_training_preset.py +3 -2
- squeakycleantext-0.8.4/tests/test_url_forms.py +338 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_user_terms.py +4 -6
- squeakycleantext-0.8.2/sct/cli.py +0 -117
- squeakycleantext-0.8.2/sct/config.py +0 -1080
- squeakycleantext-0.8.2/sct/report_cli.py +0 -138
- squeakycleantext-0.8.2/sct/streaming.py +0 -275
- squeakycleantext-0.8.2/sct/utils/gliner_adapter.py +0 -227
- squeakycleantext-0.8.2/sct/utils/ner.py +0 -1411
- squeakycleantext-0.8.2/setup.py +0 -2
- squeakycleantext-0.8.2/tests/test_benchmark_release.py +0 -97
- squeakycleantext-0.8.2/tests/test_browser.py +0 -37
- squeakycleantext-0.8.2/tests/test_codex_benchmark_round1.py +0 -84
- squeakycleantext-0.8.2/tests/test_codex_full_round1.py +0 -166
- squeakycleantext-0.8.2/tests/test_codex_full_round13.py +0 -74
- squeakycleantext-0.8.2/tests/test_codex_full_round14.py +0 -73
- squeakycleantext-0.8.2/tests/test_codex_full_round15.py +0 -154
- squeakycleantext-0.8.2/tests/test_codex_full_round16.py +0 -90
- squeakycleantext-0.8.2/tests/test_codex_full_round17.py +0 -78
- squeakycleantext-0.8.2/tests/test_codex_full_round18.py +0 -79
- squeakycleantext-0.8.2/tests/test_codex_full_round19.py +0 -32
- squeakycleantext-0.8.2/tests/test_codex_full_round2.py +0 -81
- squeakycleantext-0.8.2/tests/test_codex_full_round20.py +0 -87
- squeakycleantext-0.8.2/tests/test_codex_full_round21.py +0 -168
- squeakycleantext-0.8.2/tests/test_codex_full_round22.py +0 -52
- squeakycleantext-0.8.2/tests/test_codex_full_round23.py +0 -54
- squeakycleantext-0.8.2/tests/test_codex_full_round24.py +0 -82
- squeakycleantext-0.8.2/tests/test_codex_full_round25.py +0 -72
- squeakycleantext-0.8.2/tests/test_codex_full_round27.py +0 -85
- squeakycleantext-0.8.2/tests/test_codex_full_round28.py +0 -28
- squeakycleantext-0.8.2/tests/test_codex_full_round29.py +0 -44
- squeakycleantext-0.8.2/tests/test_codex_full_round3.py +0 -104
- squeakycleantext-0.8.2/tests/test_codex_full_round30.py +0 -53
- squeakycleantext-0.8.2/tests/test_codex_full_round31.py +0 -82
- squeakycleantext-0.8.2/tests/test_codex_full_round32.py +0 -52
- squeakycleantext-0.8.2/tests/test_codex_full_round35.py +0 -59
- squeakycleantext-0.8.2/tests/test_codex_full_round37.py +0 -41
- squeakycleantext-0.8.2/tests/test_codex_full_round38.py +0 -57
- squeakycleantext-0.8.2/tests/test_codex_full_round4.py +0 -44
- squeakycleantext-0.8.2/tests/test_codex_full_round5.py +0 -106
- squeakycleantext-0.8.2/tests/test_codex_full_round6.py +0 -80
- squeakycleantext-0.8.2/tests/test_codex_full_round7.py +0 -66
- squeakycleantext-0.8.2/tests/test_codex_full_round8.py +0 -35
- squeakycleantext-0.8.2/tests/test_codex_full_round9.py +0 -58
- squeakycleantext-0.8.2/tests/test_contracts.py +0 -143
- squeakycleantext-0.8.2/tests/test_device_provenance.py +0 -211
- squeakycleantext-0.8.2/tests/test_final_check_fixes.py +0 -273
- squeakycleantext-0.8.2/tests/test_full_test_round2.py +0 -145
- squeakycleantext-0.8.2/tests/test_fuzz_round1.py +0 -72
- squeakycleantext-0.8.2/tests/test_fuzz_round2.py +0 -31
- squeakycleantext-0.8.2/tests/test_fuzz_round3.py +0 -52
- squeakycleantext-0.8.2/tests/test_fuzz_round5.py +0 -51
- squeakycleantext-0.8.2/tests/test_fuzz_round6.py +0 -51
- squeakycleantext-0.8.2/tests/test_integrations_frameworks.py +0 -54
- squeakycleantext-0.8.2/tests/test_ocr.py +0 -87
- squeakycleantext-0.8.2/tests/test_review_round1.py +0 -159
- squeakycleantext-0.8.2/tests/test_review_round2.py +0 -69
- squeakycleantext-0.8.2/tests/test_review_round3.py +0 -101
- squeakycleantext-0.8.2/tests/test_review_round4.py +0 -68
- squeakycleantext-0.8.2/tests/test_risk.py +0 -21
- squeakycleantext-0.8.2/tests/test_streaming.py +0 -34
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/LICENSE +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/dependency_links.txt +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/entry_points.txt +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/SqueakyCleanText.egg-info/top_level.txt +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/integrations/__init__.py +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/sct/model_candidates.py +0 -0
- /squeakycleantext-0.8.2/sct/utils/__init__.py → /squeakycleantext-0.8.4/sct/py.typed +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/setup.cfg +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_cjk_width.py +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_dates_default.py +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_fuzz_styled_pii.py +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_mcp_path.py +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_model_benchmarks.py +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_multilingual_eval_set.py +0 -0
- {squeakycleantext-0.8.2 → squeakycleantext-0.8.4}/tests/test_type_names.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SqueakyCleanText
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.4
|
|
4
4
|
Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
|
|
5
5
|
Author: Rehan Fazal
|
|
6
6
|
License-Expression: MIT
|
|
@@ -44,7 +44,7 @@ Provides-Extra: fuzzy
|
|
|
44
44
|
Requires-Dist: rapidfuzz>=3.12; extra == "fuzzy"
|
|
45
45
|
Provides-Extra: torch
|
|
46
46
|
Requires-Dist: torch>=2.6.0; extra == "torch"
|
|
47
|
-
Requires-Dist: transformers>=
|
|
47
|
+
Requires-Dist: transformers>=5.0; extra == "torch"
|
|
48
48
|
Provides-Extra: gliner
|
|
49
49
|
Requires-Dist: gliner<1.0,>=0.2.25; extra == "gliner"
|
|
50
50
|
Provides-Extra: gliner2
|
|
@@ -53,13 +53,13 @@ Provides-Extra: synthetic
|
|
|
53
53
|
Requires-Dist: faker>=33.0; extra == "synthetic"
|
|
54
54
|
Provides-Extra: presidio
|
|
55
55
|
Requires-Dist: presidio-analyzer>=2.2.360; extra == "presidio"
|
|
56
|
+
Requires-Dist: gliner<1.0,>=0.2.25; extra == "presidio"
|
|
56
57
|
Provides-Extra: classify
|
|
57
58
|
Requires-Dist: gliclass>=0.1.16; extra == "classify"
|
|
58
59
|
Provides-Extra: all-ner
|
|
59
|
-
Requires-Dist: torch
|
|
60
|
-
|
|
61
|
-
Requires-Dist:
|
|
62
|
-
Requires-Dist: gliner2<2.0,>=1.0; extra == "all-ner"
|
|
60
|
+
Requires-Dist: SqueakyCleanText[gliner,gliner2,torch]; extra == "all-ner"
|
|
61
|
+
Provides-Extra: quantize
|
|
62
|
+
Requires-Dist: onnx>=1.19; extra == "quantize"
|
|
63
63
|
Provides-Extra: dev
|
|
64
64
|
Requires-Dist: hypothesis>=6.130; extra == "dev"
|
|
65
65
|
Requires-Dist: faker>=33.0; extra == "dev"
|
|
@@ -74,6 +74,8 @@ Provides-Extra: mcp
|
|
|
74
74
|
Requires-Dist: fastmcp>=4.0.5; extra == "mcp"
|
|
75
75
|
Provides-Extra: score
|
|
76
76
|
Requires-Dist: datasketch>=1.6; extra == "score"
|
|
77
|
+
Provides-Extra: parquet
|
|
78
|
+
Requires-Dist: pyarrow>=17; extra == "parquet"
|
|
77
79
|
Provides-Extra: datasets
|
|
78
80
|
Requires-Dist: datasets>=3.0; extra == "datasets"
|
|
79
81
|
Provides-Extra: datatrove
|
|
@@ -94,9 +96,11 @@ Dynamic: license-file
|
|
|
94
96
|
[](https://pypi.org/project/squeakycleantext/)
|
|
95
97
|
[](https://github.com/rhnfzl/SqueakyCleanText/blob/main/LICENSE)
|
|
96
98
|
|
|
97
|
-
**
|
|
99
|
+
**Stop hunting for personal details by hand. Mask detected names and contact details before sharing your text dataset.**
|
|
98
100
|
|
|
99
|
-
SqueakyCleanText
|
|
101
|
+
SqueakyCleanText helps people prepare text datasets for sharing, LLM training, fine-tuning or RAG, with language detection in 75 languages.
|
|
102
|
+
It masks the personal details it finds and repairs broken text.
|
|
103
|
+
Works with JSONL files, Hugging Face datasets and DataTrove jobs, on CPU or GPU.
|
|
100
104
|
|
|
101
105
|
```text
|
|
102
106
|
in: I’m Anna Berg & my mail is anna@example.se. Order 4711 ships on 2024-05-01.
|
|
@@ -108,6 +112,8 @@ out: Bel <PHONE> voor <PERSON>.
|
|
|
108
112
|
sct_language=DUTCH sct_pii_count=2 sct_pii_types=['PERSON', 'PHONE']
|
|
109
113
|
```
|
|
110
114
|
|
|
115
|
+
Review cleaned text before sharing sensitive data; some personal details can be missed.
|
|
116
|
+
|
|
111
117
|
Documentation: <https://rhnfzl.github.io/SqueakyCleanText/>
|
|
112
118
|
|
|
113
119
|
## What it does
|
|
@@ -228,7 +234,7 @@ if __name__ == "__main__":
|
|
|
228
234
|
| Data card for a file | `sct-report data.jsonl --text-column text --out report.json --card card.md` | [SCORE](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md) |
|
|
229
235
|
| AI assistants | `sct-mcp` (MCP server, `mcp` extra) | [MCP](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MCP.md) |
|
|
230
236
|
|
|
231
|
-
The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
|
|
237
|
+
The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
|
|
232
238
|
|
|
233
239
|
## How it works
|
|
234
240
|
|
|
@@ -258,7 +264,7 @@ Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/r
|
|
|
258
264
|
- `lm_text`: the cleaned text, sentences intact, for LLMs, RAG and training.
|
|
259
265
|
- `stat_text`: lower-cased, without stopwords and punctuation, for classic ML models.
|
|
260
266
|
- `language`: the detected language.
|
|
261
|
-
- `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive
|
|
267
|
+
- `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive` (`pii_count > 0`), warnings and the NER models that ran. Safe to log or store next to the text.
|
|
262
268
|
- `metadata`: the same counts, plus raw values only when you ask for them (the reversible map, or `findings` with `include_entities=True`). Keep those out of logs.
|
|
263
269
|
|
|
264
270
|
`score_record` turns a result into flat `sct_*` columns for filtering and data cards. `ScoreConfig` adds an optional quality score and a MinHash signature. Fields: [docs/API.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/API.md), columns: [docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md).
|
|
@@ -276,17 +282,17 @@ Modes, privacy policies and more PII types (`ner_mode="pii"`): [docs/REPLACEMENT
|
|
|
276
282
|
|
|
277
283
|
## Speed and hardware
|
|
278
284
|
|
|
279
|
-
Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0,
|
|
285
|
+
Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, on a workstation CPU and high-end GPUs, median of 3 runs. 0.8.1 runs at the same speed.
|
|
280
286
|
|
|
281
287
|
| Workload | Hardware | Docs/s |
|
|
282
288
|
|---|---|---|
|
|
283
289
|
| `process`, one document at a time | 16 CPU cores | 5.0 |
|
|
284
290
|
| `process_batch`, short documents, 16 workers | 16 CPU cores | 15.5 |
|
|
285
291
|
| `process_batch`, short documents, 16 workers | 64 CPU cores | 29.2 |
|
|
286
|
-
| `process_batch`, short documents, full ensemble |
|
|
287
|
-
| `process_batch`, short documents, English only (2 models) |
|
|
292
|
+
| `process_batch`, short documents, full ensemble | 3 GPUs (models split across them) | 130 |
|
|
293
|
+
| `process_batch`, short documents, English only (2 models) | 1 GPU | 646 |
|
|
288
294
|
|
|
289
|
-
The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one
|
|
295
|
+
The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one GPU, about 2 hours with the full ensemble on 3 GPUs, and about 10 hours on 64 CPU cores.
|
|
290
296
|
|
|
291
297
|
NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cleaning, or limit `language=` to load fewer models. Hardware advice and GPU setup: [docs/PERFORMANCE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PERFORMANCE.md).
|
|
292
298
|
|
|
@@ -316,7 +322,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
|
|
|
316
322
|
- **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
|
|
317
323
|
<!-- --8<-- [end:limits] -->
|
|
318
324
|
|
|
319
|
-
Detailed known limits of 0.8.
|
|
325
|
+
Detailed known limits of 0.8.4: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
|
|
320
326
|
|
|
321
327
|
## Documentation
|
|
322
328
|
|
|
@@ -348,7 +354,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
|
|
|
348
354
|
author = {Fazal, Rehan},
|
|
349
355
|
title = {SqueakyCleanText},
|
|
350
356
|
url = {https://github.com/rhnfzl/SqueakyCleanText},
|
|
351
|
-
version = {0.8.
|
|
357
|
+
version = {0.8.4},
|
|
352
358
|
year = {2026}
|
|
353
359
|
}
|
|
354
360
|
```
|
|
@@ -5,9 +5,11 @@
|
|
|
5
5
|
[](https://pypi.org/project/squeakycleantext/)
|
|
6
6
|
[](https://github.com/rhnfzl/SqueakyCleanText/blob/main/LICENSE)
|
|
7
7
|
|
|
8
|
-
**
|
|
8
|
+
**Stop hunting for personal details by hand. Mask detected names and contact details before sharing your text dataset.**
|
|
9
9
|
|
|
10
|
-
SqueakyCleanText
|
|
10
|
+
SqueakyCleanText helps people prepare text datasets for sharing, LLM training, fine-tuning or RAG, with language detection in 75 languages.
|
|
11
|
+
It masks the personal details it finds and repairs broken text.
|
|
12
|
+
Works with JSONL files, Hugging Face datasets and DataTrove jobs, on CPU or GPU.
|
|
11
13
|
|
|
12
14
|
```text
|
|
13
15
|
in: I’m Anna Berg & my mail is anna@example.se. Order 4711 ships on 2024-05-01.
|
|
@@ -19,6 +21,8 @@ out: Bel <PHONE> voor <PERSON>.
|
|
|
19
21
|
sct_language=DUTCH sct_pii_count=2 sct_pii_types=['PERSON', 'PHONE']
|
|
20
22
|
```
|
|
21
23
|
|
|
24
|
+
Review cleaned text before sharing sensitive data; some personal details can be missed.
|
|
25
|
+
|
|
22
26
|
Documentation: <https://rhnfzl.github.io/SqueakyCleanText/>
|
|
23
27
|
|
|
24
28
|
## What it does
|
|
@@ -139,7 +143,7 @@ if __name__ == "__main__":
|
|
|
139
143
|
| Data card for a file | `sct-report data.jsonl --text-column text --out report.json --card card.md` | [SCORE](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md) |
|
|
140
144
|
| AI assistants | `sct-mcp` (MCP server, `mcp` extra) | [MCP](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MCP.md) |
|
|
141
145
|
|
|
142
|
-
The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
|
|
146
|
+
The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
|
|
143
147
|
|
|
144
148
|
## How it works
|
|
145
149
|
|
|
@@ -169,7 +173,7 @@ Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/r
|
|
|
169
173
|
- `lm_text`: the cleaned text, sentences intact, for LLMs, RAG and training.
|
|
170
174
|
- `stat_text`: lower-cased, without stopwords and punctuation, for classic ML models.
|
|
171
175
|
- `language`: the detected language.
|
|
172
|
-
- `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive
|
|
176
|
+
- `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive` (`pii_count > 0`), warnings and the NER models that ran. Safe to log or store next to the text.
|
|
173
177
|
- `metadata`: the same counts, plus raw values only when you ask for them (the reversible map, or `findings` with `include_entities=True`). Keep those out of logs.
|
|
174
178
|
|
|
175
179
|
`score_record` turns a result into flat `sct_*` columns for filtering and data cards. `ScoreConfig` adds an optional quality score and a MinHash signature. Fields: [docs/API.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/API.md), columns: [docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md).
|
|
@@ -187,17 +191,17 @@ Modes, privacy policies and more PII types (`ner_mode="pii"`): [docs/REPLACEMENT
|
|
|
187
191
|
|
|
188
192
|
## Speed and hardware
|
|
189
193
|
|
|
190
|
-
Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0,
|
|
194
|
+
Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, on a workstation CPU and high-end GPUs, median of 3 runs. 0.8.1 runs at the same speed.
|
|
191
195
|
|
|
192
196
|
| Workload | Hardware | Docs/s |
|
|
193
197
|
|---|---|---|
|
|
194
198
|
| `process`, one document at a time | 16 CPU cores | 5.0 |
|
|
195
199
|
| `process_batch`, short documents, 16 workers | 16 CPU cores | 15.5 |
|
|
196
200
|
| `process_batch`, short documents, 16 workers | 64 CPU cores | 29.2 |
|
|
197
|
-
| `process_batch`, short documents, full ensemble |
|
|
198
|
-
| `process_batch`, short documents, English only (2 models) |
|
|
201
|
+
| `process_batch`, short documents, full ensemble | 3 GPUs (models split across them) | 130 |
|
|
202
|
+
| `process_batch`, short documents, English only (2 models) | 1 GPU | 646 |
|
|
199
203
|
|
|
200
|
-
The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one
|
|
204
|
+
The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one GPU, about 2 hours with the full ensemble on 3 GPUs, and about 10 hours on 64 CPU cores.
|
|
201
205
|
|
|
202
206
|
NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cleaning, or limit `language=` to load fewer models. Hardware advice and GPU setup: [docs/PERFORMANCE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PERFORMANCE.md).
|
|
203
207
|
|
|
@@ -227,7 +231,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
|
|
|
227
231
|
- **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
|
|
228
232
|
<!-- --8<-- [end:limits] -->
|
|
229
233
|
|
|
230
|
-
Detailed known limits of 0.8.
|
|
234
|
+
Detailed known limits of 0.8.4: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
|
|
231
235
|
|
|
232
236
|
## Documentation
|
|
233
237
|
|
|
@@ -259,7 +263,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
|
|
|
259
263
|
author = {Fazal, Rehan},
|
|
260
264
|
title = {SqueakyCleanText},
|
|
261
265
|
url = {https://github.com/rhnfzl/SqueakyCleanText},
|
|
262
|
-
version = {0.8.
|
|
266
|
+
version = {0.8.4},
|
|
263
267
|
year = {2026}
|
|
264
268
|
}
|
|
265
269
|
```
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SqueakyCleanText
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.4
|
|
4
4
|
Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
|
|
5
5
|
Author: Rehan Fazal
|
|
6
6
|
License-Expression: MIT
|
|
@@ -44,7 +44,7 @@ Provides-Extra: fuzzy
|
|
|
44
44
|
Requires-Dist: rapidfuzz>=3.12; extra == "fuzzy"
|
|
45
45
|
Provides-Extra: torch
|
|
46
46
|
Requires-Dist: torch>=2.6.0; extra == "torch"
|
|
47
|
-
Requires-Dist: transformers>=
|
|
47
|
+
Requires-Dist: transformers>=5.0; extra == "torch"
|
|
48
48
|
Provides-Extra: gliner
|
|
49
49
|
Requires-Dist: gliner<1.0,>=0.2.25; extra == "gliner"
|
|
50
50
|
Provides-Extra: gliner2
|
|
@@ -53,13 +53,13 @@ Provides-Extra: synthetic
|
|
|
53
53
|
Requires-Dist: faker>=33.0; extra == "synthetic"
|
|
54
54
|
Provides-Extra: presidio
|
|
55
55
|
Requires-Dist: presidio-analyzer>=2.2.360; extra == "presidio"
|
|
56
|
+
Requires-Dist: gliner<1.0,>=0.2.25; extra == "presidio"
|
|
56
57
|
Provides-Extra: classify
|
|
57
58
|
Requires-Dist: gliclass>=0.1.16; extra == "classify"
|
|
58
59
|
Provides-Extra: all-ner
|
|
59
|
-
Requires-Dist: torch
|
|
60
|
-
|
|
61
|
-
Requires-Dist:
|
|
62
|
-
Requires-Dist: gliner2<2.0,>=1.0; extra == "all-ner"
|
|
60
|
+
Requires-Dist: SqueakyCleanText[gliner,gliner2,torch]; extra == "all-ner"
|
|
61
|
+
Provides-Extra: quantize
|
|
62
|
+
Requires-Dist: onnx>=1.19; extra == "quantize"
|
|
63
63
|
Provides-Extra: dev
|
|
64
64
|
Requires-Dist: hypothesis>=6.130; extra == "dev"
|
|
65
65
|
Requires-Dist: faker>=33.0; extra == "dev"
|
|
@@ -74,6 +74,8 @@ Provides-Extra: mcp
|
|
|
74
74
|
Requires-Dist: fastmcp>=4.0.5; extra == "mcp"
|
|
75
75
|
Provides-Extra: score
|
|
76
76
|
Requires-Dist: datasketch>=1.6; extra == "score"
|
|
77
|
+
Provides-Extra: parquet
|
|
78
|
+
Requires-Dist: pyarrow>=17; extra == "parquet"
|
|
77
79
|
Provides-Extra: datasets
|
|
78
80
|
Requires-Dist: datasets>=3.0; extra == "datasets"
|
|
79
81
|
Provides-Extra: datatrove
|
|
@@ -94,9 +96,11 @@ Dynamic: license-file
|
|
|
94
96
|
[](https://pypi.org/project/squeakycleantext/)
|
|
95
97
|
[](https://github.com/rhnfzl/SqueakyCleanText/blob/main/LICENSE)
|
|
96
98
|
|
|
97
|
-
**
|
|
99
|
+
**Stop hunting for personal details by hand. Mask detected names and contact details before sharing your text dataset.**
|
|
98
100
|
|
|
99
|
-
SqueakyCleanText
|
|
101
|
+
SqueakyCleanText helps people prepare text datasets for sharing, LLM training, fine-tuning or RAG, with language detection in 75 languages.
|
|
102
|
+
It masks the personal details it finds and repairs broken text.
|
|
103
|
+
Works with JSONL files, Hugging Face datasets and DataTrove jobs, on CPU or GPU.
|
|
100
104
|
|
|
101
105
|
```text
|
|
102
106
|
in: I’m Anna Berg & my mail is anna@example.se. Order 4711 ships on 2024-05-01.
|
|
@@ -108,6 +112,8 @@ out: Bel <PHONE> voor <PERSON>.
|
|
|
108
112
|
sct_language=DUTCH sct_pii_count=2 sct_pii_types=['PERSON', 'PHONE']
|
|
109
113
|
```
|
|
110
114
|
|
|
115
|
+
Review cleaned text before sharing sensitive data; some personal details can be missed.
|
|
116
|
+
|
|
111
117
|
Documentation: <https://rhnfzl.github.io/SqueakyCleanText/>
|
|
112
118
|
|
|
113
119
|
## What it does
|
|
@@ -228,7 +234,7 @@ if __name__ == "__main__":
|
|
|
228
234
|
| Data card for a file | `sct-report data.jsonl --text-column text --out report.json --card card.md` | [SCORE](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md) |
|
|
229
235
|
| AI assistants | `sct-mcp` (MCP server, `mcp` extra) | [MCP](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MCP.md) |
|
|
230
236
|
|
|
231
|
-
The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
|
|
237
|
+
The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
|
|
232
238
|
|
|
233
239
|
## How it works
|
|
234
240
|
|
|
@@ -258,7 +264,7 @@ Every step is a `TextCleanerConfig` field ([docs/CONFIG.md](https://github.com/r
|
|
|
258
264
|
- `lm_text`: the cleaned text, sentences intact, for LLMs, RAG and training.
|
|
259
265
|
- `stat_text`: lower-cased, without stopwords and punctuation, for classic ML models.
|
|
260
266
|
- `language`: the detected language.
|
|
261
|
-
- `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive
|
|
267
|
+
- `signals`: a value-free view with the same keys for every input: language and confidence, `pii_count`, `pii_share`, `pii_types`, `sensitive` (`pii_count > 0`), warnings and the NER models that ran. Safe to log or store next to the text.
|
|
262
268
|
- `metadata`: the same counts, plus raw values only when you ask for them (the reversible map, or `findings` with `include_entities=True`). Keep those out of logs.
|
|
263
269
|
|
|
264
270
|
`score_record` turns a result into flat `sct_*` columns for filtering and data cards. `ScoreConfig` adds an optional quality score and a MinHash signature. Fields: [docs/API.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/API.md), columns: [docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md).
|
|
@@ -276,17 +282,17 @@ Modes, privacy policies and more PII types (`ner_mode="pii"`): [docs/REPLACEMENT
|
|
|
276
282
|
|
|
277
283
|
## Speed and hardware
|
|
278
284
|
|
|
279
|
-
Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0,
|
|
285
|
+
Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, on a workstation CPU and high-end GPUs, median of 3 runs. 0.8.1 runs at the same speed.
|
|
280
286
|
|
|
281
287
|
| Workload | Hardware | Docs/s |
|
|
282
288
|
|---|---|---|
|
|
283
289
|
| `process`, one document at a time | 16 CPU cores | 5.0 |
|
|
284
290
|
| `process_batch`, short documents, 16 workers | 16 CPU cores | 15.5 |
|
|
285
291
|
| `process_batch`, short documents, 16 workers | 64 CPU cores | 29.2 |
|
|
286
|
-
| `process_batch`, short documents, full ensemble |
|
|
287
|
-
| `process_batch`, short documents, English only (2 models) |
|
|
292
|
+
| `process_batch`, short documents, full ensemble | 3 GPUs (models split across them) | 130 |
|
|
293
|
+
| `process_batch`, short documents, English only (2 models) | 1 GPU | 646 |
|
|
288
294
|
|
|
289
|
-
The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one
|
|
295
|
+
The full ensemble needs about 12 to 16 GB of free memory on one GPU; English only needs 3 to 5 GB. As a rough estimate from this table, 1 million short documents take about 26 minutes English-only on one GPU, about 2 hours with the full ensemble on 3 GPUs, and about 10 hours on 64 CPU cores.
|
|
290
296
|
|
|
291
297
|
NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cleaning, or limit `language=` to load fewer models. Hardware advice and GPU setup: [docs/PERFORMANCE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PERFORMANCE.md).
|
|
292
298
|
|
|
@@ -316,7 +322,7 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
|
|
|
316
322
|
- **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
|
|
317
323
|
<!-- --8<-- [end:limits] -->
|
|
318
324
|
|
|
319
|
-
Detailed known limits of 0.8.
|
|
325
|
+
Detailed known limits of 0.8.4: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
|
|
320
326
|
|
|
321
327
|
## Documentation
|
|
322
328
|
|
|
@@ -348,7 +354,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
|
|
|
348
354
|
author = {Fazal, Rehan},
|
|
349
355
|
title = {SqueakyCleanText},
|
|
350
356
|
url = {https://github.com/rhnfzl/SqueakyCleanText},
|
|
351
|
-
version = {0.8.
|
|
357
|
+
version = {0.8.4},
|
|
352
358
|
year = {2026}
|
|
353
359
|
}
|
|
354
360
|
```
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
LICENSE
|
|
2
2
|
README.md
|
|
3
3
|
pyproject.toml
|
|
4
|
-
setup.py
|
|
5
4
|
SqueakyCleanText.egg-info/PKG-INFO
|
|
6
5
|
SqueakyCleanText.egg-info/SOURCES.txt
|
|
7
6
|
SqueakyCleanText.egg-info/dependency_links.txt
|
|
@@ -20,6 +19,7 @@ sct/model_candidates.py
|
|
|
20
19
|
sct/ocr.py
|
|
21
20
|
sct/presets.py
|
|
22
21
|
sct/privacy.py
|
|
22
|
+
sct/py.typed
|
|
23
23
|
sct/report_cli.py
|
|
24
24
|
sct/risk.py
|
|
25
25
|
sct/score.py
|
|
@@ -34,12 +34,15 @@ sct/integrations/hf.py
|
|
|
34
34
|
sct/integrations/langchain.py
|
|
35
35
|
sct/utils/__init__.py
|
|
36
36
|
sct/utils/anonymization_map.py
|
|
37
|
+
sct/utils/atomic_file.py
|
|
37
38
|
sct/utils/constants.py
|
|
38
39
|
sct/utils/contact.py
|
|
39
40
|
sct/utils/datetime.py
|
|
40
41
|
sct/utils/gliclass_adapter.py
|
|
41
42
|
sct/utils/gliner_adapter.py
|
|
42
43
|
sct/utils/ner.py
|
|
44
|
+
sct/utils/ner_chunking.py
|
|
45
|
+
sct/utils/ner_spans.py
|
|
43
46
|
sct/utils/normtext.py
|
|
44
47
|
sct/utils/onnx_pipeline.py
|
|
45
48
|
sct/utils/process_result.py
|
|
@@ -50,61 +53,31 @@ sct/utils/synthetic.py
|
|
|
50
53
|
sct/utils/torch_pipeline.py
|
|
51
54
|
tests/test_adapters.py
|
|
52
55
|
tests/test_address_leftovers.py
|
|
56
|
+
tests/test_atomic_file.py
|
|
53
57
|
tests/test_batching.py
|
|
58
|
+
tests/test_benchmark_adapters.py
|
|
59
|
+
tests/test_benchmark_loaders.py
|
|
54
60
|
tests/test_benchmark_release.py
|
|
55
61
|
tests/test_benchmarks_compare.py
|
|
62
|
+
tests/test_benchmarks_round2.py
|
|
63
|
+
tests/test_benchmarks_round2_tools.py
|
|
56
64
|
tests/test_benchmarks_score.py
|
|
57
65
|
tests/test_browser.py
|
|
58
66
|
tests/test_chunking.py
|
|
59
67
|
tests/test_cjk_width.py
|
|
68
|
+
tests/test_cleaner_lifecycle.py
|
|
60
69
|
tests/test_cli.py
|
|
61
|
-
tests/test_codex_benchmark_round1.py
|
|
62
|
-
tests/test_codex_full_round1.py
|
|
63
|
-
tests/test_codex_full_round13.py
|
|
64
|
-
tests/test_codex_full_round14.py
|
|
65
|
-
tests/test_codex_full_round15.py
|
|
66
|
-
tests/test_codex_full_round16.py
|
|
67
|
-
tests/test_codex_full_round17.py
|
|
68
|
-
tests/test_codex_full_round18.py
|
|
69
|
-
tests/test_codex_full_round19.py
|
|
70
|
-
tests/test_codex_full_round2.py
|
|
71
|
-
tests/test_codex_full_round20.py
|
|
72
|
-
tests/test_codex_full_round21.py
|
|
73
|
-
tests/test_codex_full_round22.py
|
|
74
|
-
tests/test_codex_full_round23.py
|
|
75
|
-
tests/test_codex_full_round24.py
|
|
76
|
-
tests/test_codex_full_round25.py
|
|
77
|
-
tests/test_codex_full_round27.py
|
|
78
|
-
tests/test_codex_full_round28.py
|
|
79
|
-
tests/test_codex_full_round29.py
|
|
80
|
-
tests/test_codex_full_round3.py
|
|
81
|
-
tests/test_codex_full_round30.py
|
|
82
|
-
tests/test_codex_full_round31.py
|
|
83
|
-
tests/test_codex_full_round32.py
|
|
84
|
-
tests/test_codex_full_round35.py
|
|
85
|
-
tests/test_codex_full_round37.py
|
|
86
|
-
tests/test_codex_full_round38.py
|
|
87
|
-
tests/test_codex_full_round4.py
|
|
88
|
-
tests/test_codex_full_round5.py
|
|
89
|
-
tests/test_codex_full_round6.py
|
|
90
|
-
tests/test_codex_full_round7.py
|
|
91
|
-
tests/test_codex_full_round8.py
|
|
92
|
-
tests/test_codex_full_round9.py
|
|
93
70
|
tests/test_config_state.py
|
|
94
71
|
tests/test_contracts.py
|
|
72
|
+
tests/test_date_forms.py
|
|
95
73
|
tests/test_dates_cjk_styled.py
|
|
96
74
|
tests/test_dates_default.py
|
|
97
75
|
tests/test_device_provenance.py
|
|
76
|
+
tests/test_email_forms.py
|
|
98
77
|
tests/test_escaped_markup.py
|
|
99
78
|
tests/test_evaluation.py
|
|
100
|
-
tests/
|
|
101
|
-
tests/test_full_test_round2.py
|
|
79
|
+
tests/test_fork_safety.py
|
|
102
80
|
tests/test_functional.py
|
|
103
|
-
tests/test_fuzz_round1.py
|
|
104
|
-
tests/test_fuzz_round2.py
|
|
105
|
-
tests/test_fuzz_round3.py
|
|
106
|
-
tests/test_fuzz_round5.py
|
|
107
|
-
tests/test_fuzz_round6.py
|
|
108
81
|
tests/test_fuzz_styled_pii.py
|
|
109
82
|
tests/test_integrations_frameworks.py
|
|
110
83
|
tests/test_integrations_hf.py
|
|
@@ -117,28 +90,41 @@ tests/test_mcp_server.py
|
|
|
117
90
|
tests/test_model_benchmarks.py
|
|
118
91
|
tests/test_multilingual_eval_set.py
|
|
119
92
|
tests/test_ner_sees_tags.py
|
|
93
|
+
tests/test_ner_span_repair.py
|
|
94
|
+
tests/test_ner_spans.py
|
|
95
|
+
tests/test_ner_windows.py
|
|
120
96
|
tests/test_number_identifiers.py
|
|
121
97
|
tests/test_ocr.py
|
|
122
98
|
tests/test_onnx_concurrency.py
|
|
123
99
|
tests/test_onnx_external_data.py
|
|
100
|
+
tests/test_onnx_model_cache.py
|
|
101
|
+
tests/test_phone_forms.py
|
|
102
|
+
tests/test_pipeline_text_steps.py
|
|
103
|
+
tests/test_predicted_spans_monotone.py
|
|
104
|
+
tests/test_presidio_tags_and_fakes.py
|
|
105
|
+
tests/test_presidio_type_names.py
|
|
124
106
|
tests/test_privacy.py
|
|
107
|
+
tests/test_prose_punctuation_kept.py
|
|
125
108
|
tests/test_public_view.py
|
|
126
109
|
tests/test_regex_boundaries.py
|
|
127
110
|
tests/test_regex_counts.py
|
|
111
|
+
tests/test_repo_scripts.py
|
|
128
112
|
tests/test_report_cli.py
|
|
129
|
-
tests/
|
|
130
|
-
tests/test_review_round2.py
|
|
131
|
-
tests/test_review_round3.py
|
|
132
|
-
tests/test_review_round4.py
|
|
113
|
+
tests/test_reversible_markers.py
|
|
133
114
|
tests/test_risk.py
|
|
134
115
|
tests/test_score.py
|
|
135
116
|
tests/test_sct.py
|
|
136
117
|
tests/test_security_leftovers.py
|
|
137
118
|
tests/test_stream_guard_shared_view.py
|
|
119
|
+
tests/test_stream_keeps_values_whole.py
|
|
138
120
|
tests/test_streaming.py
|
|
139
121
|
tests/test_streaming_split_stability.py
|
|
140
122
|
tests/test_styled_pii.py
|
|
123
|
+
tests/test_surface_contracts.py
|
|
124
|
+
tests/test_synthetic_determinism.py
|
|
141
125
|
tests/test_synthetic_per_document.py
|
|
126
|
+
tests/test_torch_ner_backend.py
|
|
142
127
|
tests/test_training_preset.py
|
|
143
128
|
tests/test_type_names.py
|
|
129
|
+
tests/test_url_forms.py
|
|
144
130
|
tests/test_user_terms.py
|
|
@@ -11,10 +11,7 @@ presidio_anonymizer>=2.2.360
|
|
|
11
11
|
regex>=2024.11.6
|
|
12
12
|
|
|
13
13
|
[all-ner]
|
|
14
|
-
torch
|
|
15
|
-
transformers>=4.48
|
|
16
|
-
gliner<1.0,>=0.2.25
|
|
17
|
-
gliner2<2.0,>=1.0
|
|
14
|
+
SqueakyCleanText[gliner,gliner2,torch]
|
|
18
15
|
|
|
19
16
|
[classify]
|
|
20
17
|
gliclass>=0.1.16
|
|
@@ -58,8 +55,15 @@ fastmcp>=4.0.5
|
|
|
58
55
|
pillow>=12.3.0
|
|
59
56
|
pytesseract>=0.3.13
|
|
60
57
|
|
|
58
|
+
[parquet]
|
|
59
|
+
pyarrow>=17
|
|
60
|
+
|
|
61
61
|
[presidio]
|
|
62
62
|
presidio-analyzer>=2.2.360
|
|
63
|
+
gliner<1.0,>=0.2.25
|
|
64
|
+
|
|
65
|
+
[quantize]
|
|
66
|
+
onnx>=1.19
|
|
63
67
|
|
|
64
68
|
[score]
|
|
65
69
|
datasketch>=1.6
|
|
@@ -73,4 +77,4 @@ pytest-cov>=6.0
|
|
|
73
77
|
|
|
74
78
|
[torch]
|
|
75
79
|
torch>=2.6.0
|
|
76
|
-
transformers>=
|
|
80
|
+
transformers>=5.0
|
|
@@ -66,7 +66,7 @@ fuzzy = [
|
|
|
66
66
|
]
|
|
67
67
|
torch = [
|
|
68
68
|
"torch>=2.6.0",
|
|
69
|
-
"transformers>=
|
|
69
|
+
"transformers>=5.0",
|
|
70
70
|
]
|
|
71
71
|
gliner = [
|
|
72
72
|
"gliner>=0.2.25,<1.0",
|
|
@@ -79,15 +79,16 @@ synthetic = [
|
|
|
79
79
|
]
|
|
80
80
|
presidio = [
|
|
81
81
|
"presidio-analyzer>=2.2.360",
|
|
82
|
+
"gliner>=0.2.25,<1.0",
|
|
82
83
|
]
|
|
83
84
|
classify = [
|
|
84
85
|
"gliclass>=0.1.16",
|
|
85
86
|
]
|
|
86
87
|
all-ner = [
|
|
87
|
-
"torch
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
"
|
|
88
|
+
"SqueakyCleanText[torch,gliner,gliner2]",
|
|
89
|
+
]
|
|
90
|
+
quantize = [
|
|
91
|
+
"onnx>=1.19",
|
|
91
92
|
]
|
|
92
93
|
dev = [
|
|
93
94
|
"hypothesis>=6.130",
|
|
@@ -107,6 +108,9 @@ mcp = [
|
|
|
107
108
|
score = [
|
|
108
109
|
"datasketch>=1.6",
|
|
109
110
|
]
|
|
111
|
+
parquet = [
|
|
112
|
+
"pyarrow>=17",
|
|
113
|
+
]
|
|
110
114
|
datasets = [
|
|
111
115
|
"datasets>=3.0",
|
|
112
116
|
]
|
|
@@ -131,12 +135,21 @@ version = {attr = "sct.__version__"}
|
|
|
131
135
|
include = ["sct*"]
|
|
132
136
|
exclude = ["tests*"]
|
|
133
137
|
|
|
138
|
+
[tool.setuptools.package-data]
|
|
139
|
+
sct = ["py.typed"]
|
|
140
|
+
|
|
134
141
|
[tool.pytest.ini_options]
|
|
135
142
|
timeout = 1200
|
|
143
|
+
markers = ["needs_model: loads a real Hugging Face model"]
|
|
136
144
|
|
|
137
145
|
[tool.ruff]
|
|
138
146
|
line-length = 127
|
|
139
147
|
|
|
140
148
|
[tool.ruff.lint]
|
|
141
|
-
select = ["E", "F", "W", "S"]
|
|
149
|
+
select = ["E", "F", "W", "S", "UP"]
|
|
142
150
|
ignore = ["S101"]
|
|
151
|
+
|
|
152
|
+
[tool.mypy]
|
|
153
|
+
files = ["sct"]
|
|
154
|
+
ignore_missing_imports = true
|
|
155
|
+
check_untyped_defs = true
|
|
@@ -11,6 +11,7 @@ from sct.privacy import (
|
|
|
11
11
|
TokenStore,
|
|
12
12
|
)
|
|
13
13
|
from sct.sct import CustomPatternTimeout, TextCleaner
|
|
14
|
+
from sct.utils.ner import ModelLoadError
|
|
14
15
|
from sct.presets import PRESETS, preset_config
|
|
15
16
|
from sct.adapters import (
|
|
16
17
|
DataFrameResult,
|
|
@@ -50,9 +51,9 @@ from sct.streaming import (
|
|
|
50
51
|
from sct.utils.anonymization_map import AnonymizationMap, MapEntry
|
|
51
52
|
from sct.utils.process_result import ProcessResult
|
|
52
53
|
|
|
53
|
-
__version__ = "0.8.
|
|
54
|
+
__version__ = "0.8.4"
|
|
54
55
|
__all__ = [
|
|
55
|
-
"TextCleaner", "TextCleanerConfig", "CustomPatternTimeout", "PRESETS", "preset_config",
|
|
56
|
+
"TextCleaner", "TextCleanerConfig", "CustomPatternTimeout", "ModelLoadError", "PRESETS", "preset_config",
|
|
56
57
|
"PII_LABELS", "PII_LABEL_MAP",
|
|
57
58
|
"AnonymizationMap", "MapEntry", "ProcessResult",
|
|
58
59
|
"DetectedEntity", "EntityFinding", "EntityRule",
|