SqueakyCleanText 0.9.0__tar.gz → 0.10.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (187) hide show
  1. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/PKG-INFO +93 -5
  2. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/README.md +81 -4
  3. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/SqueakyCleanText.egg-info/PKG-INFO +93 -5
  4. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/SqueakyCleanText.egg-info/SOURCES.txt +48 -0
  5. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/SqueakyCleanText.egg-info/requires.txt +15 -0
  6. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/pyproject.toml +7 -0
  7. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/__init__.py +13 -1
  8. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/adapters.py +96 -22
  9. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/browser.py +2 -0
  10. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/cli.py +29 -6
  11. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/config.py +42 -0
  12. squeakycleantext-0.10.0/sct/coordinates.py +111 -0
  13. squeakycleantext-0.10.0/sct/corpus/__init__.py +11 -0
  14. squeakycleantext-0.10.0/sct/corpus/datatrove.py +361 -0
  15. squeakycleantext-0.10.0/sct/corpus/manifest.py +174 -0
  16. squeakycleantext-0.10.0/sct/corpus/runner.py +463 -0
  17. squeakycleantext-0.10.0/sct/corpus/selection.py +292 -0
  18. squeakycleantext-0.10.0/sct/corpus/types.py +69 -0
  19. squeakycleantext-0.10.0/sct/detectors.py +182 -0
  20. squeakycleantext-0.10.0/sct/detectors_builtin.py +110 -0
  21. squeakycleantext-0.10.0/sct/detectors_datafog.py +63 -0
  22. squeakycleantext-0.10.0/sct/documents.py +170 -0
  23. squeakycleantext-0.10.0/sct/findings.py +90 -0
  24. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/integrations/_runtime.py +68 -10
  25. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/integrations/datatrove.py +10 -2
  26. squeakycleantext-0.10.0/sct/integrations/docling.py +176 -0
  27. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/integrations/haystack.py +33 -4
  28. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/integrations/hf.py +23 -5
  29. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/integrations/langchain.py +10 -2
  30. squeakycleantext-0.10.0/sct/language_policy.py +31 -0
  31. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/mcp_server.py +52 -13
  32. squeakycleantext-0.10.0/sct/policy_v2.py +71 -0
  33. squeakycleantext-0.10.0/sct/routing.py +218 -0
  34. squeakycleantext-0.10.0/sct/schema.py +120 -0
  35. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/score.py +4 -2
  36. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/sct.py +144 -30
  37. squeakycleantext-0.10.0/sct/signals.py +241 -0
  38. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/streaming.py +4 -0
  39. squeakycleantext-0.10.0/sct/token_stores/__init__.py +5 -0
  40. squeakycleantext-0.10.0/sct/token_stores/sqlite.py +228 -0
  41. squeakycleantext-0.10.0/sct/unified.py +376 -0
  42. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/datetime.py +14 -10
  43. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/gliner_adapter.py +5 -0
  44. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/ner.py +25 -84
  45. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/ner_spans.py +92 -0
  46. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/process_result.py +17 -1
  47. squeakycleantext-0.10.0/tests/test_baseline_measurement.py +71 -0
  48. squeakycleantext-0.10.0/tests/test_builtin_detections.py +137 -0
  49. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_cleaner_lifecycle.py +8 -0
  50. squeakycleantext-0.10.0/tests/test_coordinates.py +101 -0
  51. squeakycleantext-0.10.0/tests/test_corpus_decontamination.py +122 -0
  52. squeakycleantext-0.10.0/tests/test_corpus_failure_boundaries.py +174 -0
  53. squeakycleantext-0.10.0/tests/test_corpus_identity_boundaries.py +242 -0
  54. squeakycleantext-0.10.0/tests/test_corpus_manifest.py +291 -0
  55. squeakycleantext-0.10.0/tests/test_corpus_resume.py +225 -0
  56. squeakycleantext-0.10.0/tests/test_corpus_selection.py +312 -0
  57. squeakycleantext-0.10.0/tests/test_detections.py +56 -0
  58. squeakycleantext-0.10.0/tests/test_detector_registry.py +283 -0
  59. squeakycleantext-0.10.0/tests/test_detector_routing.py +435 -0
  60. squeakycleantext-0.10.0/tests/test_docling_integration.py +446 -0
  61. squeakycleantext-0.10.0/tests/test_gliner_recipe_identity.py +86 -0
  62. squeakycleantext-0.10.0/tests/test_gliner_revision.py +51 -0
  63. squeakycleantext-0.10.0/tests/test_improvements_end_to_end.py +188 -0
  64. squeakycleantext-0.10.0/tests/test_integrations_hf.py +177 -0
  65. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_language_all.py +21 -0
  66. squeakycleantext-0.10.0/tests/test_language_policy.py +280 -0
  67. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_linux_workers.py +33 -0
  68. squeakycleantext-0.10.0/tests/test_optional_profiles.py +75 -0
  69. squeakycleantext-0.10.0/tests/test_processing_compatibility.py +59 -0
  70. squeakycleantext-0.10.0/tests/test_quality_export.py +109 -0
  71. squeakycleantext-0.10.0/tests/test_routing_audit.py +551 -0
  72. squeakycleantext-0.10.0/tests/test_routing_calibration.py +257 -0
  73. squeakycleantext-0.10.0/tests/test_schema_context.py +564 -0
  74. squeakycleantext-0.10.0/tests/test_secret_detector.py +227 -0
  75. squeakycleantext-0.10.0/tests/test_sqlite_token_store.py +199 -0
  76. squeakycleantext-0.10.0/tests/test_token_store_failure_boundaries.py +135 -0
  77. squeakycleantext-0.10.0/tests/test_token_store_operations.py +344 -0
  78. squeakycleantext-0.10.0/tests/test_unified_processing.py +537 -0
  79. squeakycleantext-0.9.0/sct/signals.py +0 -130
  80. squeakycleantext-0.9.0/tests/test_integrations_hf.py +0 -70
  81. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/LICENSE +0 -0
  82. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/SqueakyCleanText.egg-info/dependency_links.txt +0 -0
  83. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/SqueakyCleanText.egg-info/entry_points.txt +0 -0
  84. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/SqueakyCleanText.egg-info/top_level.txt +0 -0
  85. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/evaluation.py +0 -0
  86. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/integrations/__init__.py +0 -0
  87. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/model_benchmark.py +0 -0
  88. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/model_candidates.py +0 -0
  89. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/ocr.py +0 -0
  90. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/presets.py +0 -0
  91. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/privacy.py +0 -0
  92. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/py.typed +0 -0
  93. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/report_cli.py +0 -0
  94. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/risk.py +0 -0
  95. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/__init__.py +0 -0
  96. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/anonymization_map.py +0 -0
  97. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/atomic_file.py +0 -0
  98. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/constants.py +0 -0
  99. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/contact.py +0 -0
  100. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/gliclass_adapter.py +0 -0
  101. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/markers.py +0 -0
  102. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/ner_chunking.py +0 -0
  103. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/normtext.py +0 -0
  104. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/onnx_pipeline.py +0 -0
  105. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/resources.py +0 -0
  106. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/special.py +0 -0
  107. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/stopwords.py +0 -0
  108. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/synthetic.py +0 -0
  109. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/sct/utils/torch_pipeline.py +0 -0
  110. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/setup.cfg +0 -0
  111. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_adapters.py +0 -0
  112. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_address_leftovers.py +0 -0
  113. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_atomic_file.py +0 -0
  114. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_batching.py +0 -0
  115. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_benchmark_adapters.py +0 -0
  116. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_benchmark_loaders.py +0 -0
  117. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_benchmark_release.py +0 -0
  118. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_benchmarks_compare.py +0 -0
  119. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_benchmarks_round2.py +0 -0
  120. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_benchmarks_round2_tools.py +0 -0
  121. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_benchmarks_score.py +0 -0
  122. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_browser.py +0 -0
  123. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_chunking.py +0 -0
  124. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_cjk_width.py +0 -0
  125. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_cli.py +0 -0
  126. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_config_state.py +0 -0
  127. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_contracts.py +0 -0
  128. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_date_forms.py +0 -0
  129. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_dates_cjk_styled.py +0 -0
  130. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_dates_default.py +0 -0
  131. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_device_provenance.py +0 -0
  132. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_email_forms.py +0 -0
  133. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_escaped_markup.py +0 -0
  134. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_evaluation.py +0 -0
  135. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_fork_safety.py +0 -0
  136. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_functional.py +0 -0
  137. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_fuzz_styled_pii.py +0 -0
  138. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_integrations_frameworks.py +0 -0
  139. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_integrations_runtime.py +0 -0
  140. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_keep_scripts.py +0 -0
  141. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_mcp_path.py +0 -0
  142. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_mcp_server.py +0 -0
  143. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_model_benchmarks.py +0 -0
  144. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_multilingual_eval_set.py +0 -0
  145. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_ner_sees_tags.py +0 -0
  146. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_ner_span_repair.py +0 -0
  147. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_ner_spans.py +0 -0
  148. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_ner_windows.py +0 -0
  149. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_number_identifiers.py +0 -0
  150. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_ocr.py +0 -0
  151. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_onnx_concurrency.py +0 -0
  152. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_onnx_external_data.py +0 -0
  153. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_onnx_model_cache.py +0 -0
  154. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_phone_forms.py +0 -0
  155. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_pii_social_cards.py +0 -0
  156. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_pipeline_text_steps.py +0 -0
  157. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_plot_pii_comparison.py +0 -0
  158. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_plot_version_comparison.py +0 -0
  159. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_predicted_spans_monotone.py +0 -0
  160. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_presidio_tags_and_fakes.py +0 -0
  161. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_presidio_type_names.py +0 -0
  162. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_privacy.py +0 -0
  163. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_prose_punctuation_kept.py +0 -0
  164. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_public_view.py +0 -0
  165. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_regex_boundaries.py +0 -0
  166. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_regex_counts.py +0 -0
  167. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_release_comparison_watcher.py +0 -0
  168. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_repo_scripts.py +0 -0
  169. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_report_cli.py +0 -0
  170. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_reversible_markers.py +0 -0
  171. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_risk.py +0 -0
  172. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_score.py +0 -0
  173. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_sct.py +0 -0
  174. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_security_leftovers.py +0 -0
  175. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_stream_guard_shared_view.py +0 -0
  176. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_stream_keeps_values_whole.py +0 -0
  177. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_streaming.py +0 -0
  178. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_streaming_split_stability.py +0 -0
  179. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_styled_pii.py +0 -0
  180. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_surface_contracts.py +0 -0
  181. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_synthetic_determinism.py +0 -0
  182. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_synthetic_per_document.py +0 -0
  183. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_torch_ner_backend.py +0 -0
  184. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_training_preset.py +0 -0
  185. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_type_names.py +0 -0
  186. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_url_forms.py +0 -0
  187. {squeakycleantext-0.9.0 → squeakycleantext-0.10.0}/tests/test_user_terms.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SqueakyCleanText
3
- Version: 0.9.0
3
+ Version: 0.10.0
4
4
  Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
5
5
  Author: Rehan Fazal
6
6
  License-Expression: MIT
@@ -38,6 +38,17 @@ Requires-Dist: huggingface-hub>=1.5.0
38
38
  Requires-Dist: numpy>=2.2.0
39
39
  Requires-Dist: presidio_anonymizer>=2.2.360
40
40
  Requires-Dist: regex>=2024.11.6
41
+ Provides-Extra: secrets
42
+ Requires-Dist: datafog-core==1.0.0; extra == "secrets"
43
+ Provides-Extra: documents
44
+ Requires-Dist: docling==2.148.0; extra == "documents"
45
+ Provides-Extra: token-store
46
+ Requires-Dist: cryptography==48.0.1; extra == "token-store"
47
+ Provides-Extra: corpus
48
+ Requires-Dist: datasketch>=1.6; extra == "corpus"
49
+ Requires-Dist: datatrove==0.10.1; extra == "corpus"
50
+ Requires-Dist: orjson>=3.10; extra == "corpus"
51
+ Requires-Dist: spacy==3.8.16; extra == "corpus"
41
52
  Provides-Extra: gpu
42
53
  Requires-Dist: onnxruntime-gpu>=1.24.2; extra == "gpu"
43
54
  Provides-Extra: fuzzy
@@ -226,6 +237,7 @@ if __name__ == "__main__":
226
237
 
227
238
  | Where | Usage | Docs |
228
239
  |---|---|---|
240
+ | PDF extraction | `extract_document` and `clean_document` | [DOCUMENTS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/DOCUMENTS.md) |
229
241
  | LangChain | `SCTDocumentTransformer().transform_documents(docs)` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
230
242
  | Haystack | `pipe.add_component("sct", SCTDocumentCleaner())` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
231
243
  | Ray, Spark, Polars, DuckDB, dlt, Daft | `clean_columns(handle, batch)` | [RECIPES](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/RECIPES.md) |
@@ -283,6 +295,62 @@ Deprecated config globals, `ner_models_list`, the worker `batch_size` alias and
283
295
  Modes, privacy policies and more PII types (`ner_mode="pii"`): [docs/REPLACEMENT_MODES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/REPLACEMENT_MODES.md), [docs/PRIVACY.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PRIVACY.md), [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md).
284
296
  <!-- --8<-- [end:results] -->
285
297
 
298
+ ## One policy across detectors (opt-in)
299
+
300
+ Unified processing lets you apply the same handling rules to format matches and model findings. Existing callers keep `pii_processing="legacy"` by default.
301
+
302
+ ```python
303
+ from sct import TextCleaner, TextCleanerConfig, UnifiedPrivacyPolicy, HandlingRule
304
+
305
+ cleaner = TextCleaner(
306
+ TextCleanerConfig(pii_processing="unified", check_ner_process=False),
307
+ unified_policy=UnifiedPrivacyPolicy("contact-policy", "1", {
308
+ "EMAIL": HandlingRule(action="redact"),
309
+ }),
310
+ )
311
+ result = cleaner.process("Contact alice@example.com")
312
+ assert "alice@example.com" not in result.lm_text
313
+ ```
314
+
315
+ All selected detectors inspect the same canonical text before PII replacement. `use_mode` follows the placeholder, synthetic or reversible mode. `keep` affects PII replacement only. `redact` emits `<REDACTED>` and never saves that value in the reversible map. Protection wins over overlapping keep decisions; redaction wins over other protection. Unknown scores protect by default, or fail with `unknown_score_action="error"`.
316
+
317
+ A nonempty `detector_specs` tuple selects the complete detector set from an application-owned `DetectorRegistry`. Empty specs select the configured built-in regex rules and native NER when `check_ner_process=True`. Native registry loading requires `allow_network_detectors=True` because model artifacts may need downloading. External-only selection does not initialize native NER. Separately requested document classification still runs and also requires network permission in unified mode. Legacy model-loading behavior stays unchanged.
318
+
319
+ `metadata["findings_v2"]` contains value-free canonical positions, original-input ranges, mapping quality and policy decisions. `counts_v2` separates detector hits after configured allow-term filtering from resolved replacements; `provenance_v2` records detector and policy identities. Legacy counters stay empty in unified mode. Unified signals add `version=2`, `counts_v2`, `coverage_v2` and `provenance_v2`. CLI/MCP summaries distinguish detector hits from rendered replacements. `coverage_v2` reports source coverage quality and lower/upper bounds. Non-exact coverage leaves `pii_chars` and `pii_share` as `None`; score summaries place these shares in an explicit unknown bucket. Only actual built-in regex formatting findings receive the non-PII format exclusion; native, external and caller-declared dates remain sensitive.
320
+
321
+ Source mappings use Unicode code-point offsets and report `exact`, `covering` or `unavailable`. Complex built-in repairs can cover the whole input. Opaque custom callbacks make mapping unavailable, even when they return unchanged text. `source_mapping="require_source"` rejects those callbacks. These ranges do not describe original PDF bytes.
322
+
323
+ `process(..., context=DetectionContext(...))`, `process_batch(..., contexts=...)` and `aprocess_batch(..., contexts=...)` accept caller context in unified mode. Batch contexts must match input count and order. Buffered streaming uses this whole-document path; windowed streaming and browser manifests reject unified mode.
324
+
325
+ `include_entities=True` explicitly adds sensitive `findings_v2_values`, linked by `finding_index` to canonical findings. This includes values whose output was redacted. Keep this audit payload and reversible maps out of logs. They are absent by default, apart from the map explicitly requested through reversible mode.
326
+
327
+ Custom callbacks need migration review. Legacy callbacks run after regex replacement and before NER.
328
+ Unified callbacks run before all detection, so callbacks that depend on `<EMAIL>` or other rendered markers receive different input.
329
+ See [configuration and rollback](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CONFIG.md#unified-processing-and-migration).
330
+
331
+ ## Caller-declared fields (opt-in)
332
+
333
+ Declare exact fields when a record needs protection beyond text detection. SCT does not infer meaning from field names.
334
+
335
+ ```python
336
+ from sct import FieldRule, FieldSchema, process_json
337
+
338
+ schema = FieldSchema("contacts", "1", (
339
+ FieldRule(("name",), ("PERSON",), "protect_value"),
340
+ ))
341
+ assert process_json(cleaner, {"name": "May", "status": "May"}, schema=schema).value == {
342
+ "name": "<PERSON>", "status": "May",
343
+ }
344
+ ```
345
+
346
+ Schemas require unified processing. Paths select original keys and list positions. Missing fields and `None` remain unchanged; selected containers fail. Whole-value protection also accepts finite numeric and boolean scalars. Default non-string handling stays unchanged. Protection uses the existing policy and replacement modes. Conflicting keep decisions, score errors, allow terms or disabled tags fail. Unknown scores with `unknown_score_action="protect"` remain compatible with thresholds. Explicit field types win equal-span ties after action priority.
347
+
348
+ Hints require providers that support context and the declared types. Presidio receives explicit declared types as context; ONNX-only providers reject required hints. DataFrame paths use `(column_label,)` across a column or `(row_label, column_label)` for a cell. Conflicting overlapping declarations fail. Framework paths use `(column_name,)` per record. RAG paths use `("text",)` or `("metadata", ...)`. Selected missing DataFrame cells remain unchanged. Masking numeric cells changes only the copied output column to object dtype. Unsupported path shapes fail.
349
+
350
+ Unified findings use positional references without original keys or labels. Schema-match paths and caller context stay out of signals and provenance. MCP full detail excludes sensitive value payloads. Framework constructors accept a trusted module-level `registry_factory`; workers pickle its reference and rebuild per process. Haystack data resolves only application-registered factory references. Protect serialized pipeline configuration, which can contain caller schema/context identifiers.
351
+
352
+ Applications register schemas through CLI `main(..., schemas=...)` or `create_mcp_server(..., schemas=...)`. Requests supply only exactly registered data-only schemas. CLI exposes `--processing unified --schema '<JSON>'`; `--signals` adds versioned counts and a summary. Requests cannot import Python factories. Existing adapter restrictions on reversible mode remain.
353
+
286
354
  ## Speed and hardware
287
355
 
288
356
  Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, on a workstation CPU and high-end GPUs, median of 3 runs. 0.8.1 runs at the same speed.
@@ -328,13 +396,32 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
328
396
 
329
397
  - **Safe defaults, every choice configurable.** Each behaviour is a config field; nothing is forced.
330
398
  - **No guarantee that all PII is found.** Models miss entities, mostly organisations and non-Latin scripts. Review output before you publish it.
331
- - **Free text, not tables.** For tables with known PII columns, drop or hash those columns first.
399
+ - **Declared fields need explicit rules.** Use a caller-owned schema to protect known PII fields; unselected fields remain unchanged.
332
400
  - **Reversible maps are personal data.** Anyone with the map can restore the values. Store maps apart from the cleaned data, with access control.
333
401
  - **Synthetic data is not anonymisation.** Fake values keep the data useful; they do not make it anonymous in a legal sense.
334
- - **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
402
+ - **Offsets name their text stage.** Legacy findings use post-regex positions. Unified findings use canonical positions and explicit source mapping quality.
335
403
  <!-- --8<-- [end:limits] -->
336
404
 
337
- Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
405
+ Detailed known limits: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
406
+
407
+ ## Opt-in document and corpus workflows
408
+
409
+ Users can combine extraction, unified detection, encrypted references and corpus selection without changing legacy defaults.
410
+ Document and framework adapters use non-reversible output; direct text processing can return encrypted map references.
411
+ The original documents and private corpus checkpoints remain caller-owned sensitive data.
412
+
413
+ | Capability | Entry point and limit |
414
+ |---|---|
415
+ | Encrypted references | `EncryptedSQLiteTokenStore` from `sct.token_stores`; keys and authorization remain application-owned |
416
+ | Corpus selection | `analyse_records` and `select_records` from `sct.corpus`; report-only by default, explicit basis required for removal |
417
+ | Native replay | `curate_resumable`; identity/checksum validation and private checkpoints, without exactly-once guarantees |
418
+ | PDF extraction | `extract_document` and `clean_document`; prepared local Docling assets, extracted-text protection only |
419
+ | Credential formats | Optional `secrets` detector; four syntactic categories, not credential validity |
420
+ | Language uncertainty | `language_policy='abstain'`; explicit thresholds and unknown-language action |
421
+ | Detector routing | `routing_policy='always_run'` stays default; calibrated cascade remains experimental |
422
+
423
+ See the [combined acceptance evidence](https://github.com/rhnfzl/SqueakyCleanText/blob/main/evaluation/improvements/ACCEPTANCE.md) for the tested profiles.
424
+ Trained Torch and ONNX scores agree on the parity controls. The synthetic gibberish control scored highest, so quality-filter accuracy remains unverified.
338
425
 
339
426
  ## Documentation
340
427
 
@@ -347,6 +434,7 @@ Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/Squea
347
434
  | JSON, DataFrame, RAG, CLI, streaming, OCR | [docs/ADAPTERS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/ADAPTERS.md) |
348
435
  | Score columns, quality, MinHash, data cards, `sct-report` | [docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md) |
349
436
  | NER backends, GLiNER, PII mode | [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md) |
437
+ | Encrypted local token maps | [docs/TOKEN_STORES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/TOKEN_STORES.md) |
350
438
  | Replacement modes, privacy policies, `TokenStore` | [docs/REPLACEMENT_MODES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/REPLACEMENT_MODES.md), [docs/PRIVACY.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PRIVACY.md) |
351
439
  | Evaluation | [docs/EVALUATION.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/EVALUATION.md), [docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md) |
352
440
  | Languages and adding a language | [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md), [docs/ADDING_LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/ADDING_LANGUAGES.md) |
@@ -366,7 +454,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
366
454
  author = {Fazal, Rehan},
367
455
  title = {SqueakyCleanText},
368
456
  url = {https://github.com/rhnfzl/SqueakyCleanText},
369
- version = {0.9.0},
457
+ version = {0.10.0},
370
458
  year = {2026}
371
459
  }
372
460
  ```
@@ -135,6 +135,7 @@ if __name__ == "__main__":
135
135
 
136
136
  | Where | Usage | Docs |
137
137
  |---|---|---|
138
+ | PDF extraction | `extract_document` and `clean_document` | [DOCUMENTS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/DOCUMENTS.md) |
138
139
  | LangChain | `SCTDocumentTransformer().transform_documents(docs)` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
139
140
  | Haystack | `pipe.add_component("sct", SCTDocumentCleaner())` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
140
141
  | Ray, Spark, Polars, DuckDB, dlt, Daft | `clean_columns(handle, batch)` | [RECIPES](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/RECIPES.md) |
@@ -192,6 +193,62 @@ Deprecated config globals, `ner_models_list`, the worker `batch_size` alias and
192
193
  Modes, privacy policies and more PII types (`ner_mode="pii"`): [docs/REPLACEMENT_MODES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/REPLACEMENT_MODES.md), [docs/PRIVACY.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PRIVACY.md), [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md).
193
194
  <!-- --8<-- [end:results] -->
194
195
 
196
+ ## One policy across detectors (opt-in)
197
+
198
+ Unified processing lets you apply the same handling rules to format matches and model findings. Existing callers keep `pii_processing="legacy"` by default.
199
+
200
+ ```python
201
+ from sct import TextCleaner, TextCleanerConfig, UnifiedPrivacyPolicy, HandlingRule
202
+
203
+ cleaner = TextCleaner(
204
+ TextCleanerConfig(pii_processing="unified", check_ner_process=False),
205
+ unified_policy=UnifiedPrivacyPolicy("contact-policy", "1", {
206
+ "EMAIL": HandlingRule(action="redact"),
207
+ }),
208
+ )
209
+ result = cleaner.process("Contact alice@example.com")
210
+ assert "alice@example.com" not in result.lm_text
211
+ ```
212
+
213
+ All selected detectors inspect the same canonical text before PII replacement. `use_mode` follows the placeholder, synthetic or reversible mode. `keep` affects PII replacement only. `redact` emits `<REDACTED>` and never saves that value in the reversible map. Protection wins over overlapping keep decisions; redaction wins over other protection. Unknown scores protect by default, or fail with `unknown_score_action="error"`.
214
+
215
+ A nonempty `detector_specs` tuple selects the complete detector set from an application-owned `DetectorRegistry`. Empty specs select the configured built-in regex rules and native NER when `check_ner_process=True`. Native registry loading requires `allow_network_detectors=True` because model artifacts may need downloading. External-only selection does not initialize native NER. Separately requested document classification still runs and also requires network permission in unified mode. Legacy model-loading behavior stays unchanged.
216
+
217
+ `metadata["findings_v2"]` contains value-free canonical positions, original-input ranges, mapping quality and policy decisions. `counts_v2` separates detector hits after configured allow-term filtering from resolved replacements; `provenance_v2` records detector and policy identities. Legacy counters stay empty in unified mode. Unified signals add `version=2`, `counts_v2`, `coverage_v2` and `provenance_v2`. CLI/MCP summaries distinguish detector hits from rendered replacements. `coverage_v2` reports source coverage quality and lower/upper bounds. Non-exact coverage leaves `pii_chars` and `pii_share` as `None`; score summaries place these shares in an explicit unknown bucket. Only actual built-in regex formatting findings receive the non-PII format exclusion; native, external and caller-declared dates remain sensitive.
218
+
219
+ Source mappings use Unicode code-point offsets and report `exact`, `covering` or `unavailable`. Complex built-in repairs can cover the whole input. Opaque custom callbacks make mapping unavailable, even when they return unchanged text. `source_mapping="require_source"` rejects those callbacks. These ranges do not describe original PDF bytes.
220
+
221
+ `process(..., context=DetectionContext(...))`, `process_batch(..., contexts=...)` and `aprocess_batch(..., contexts=...)` accept caller context in unified mode. Batch contexts must match input count and order. Buffered streaming uses this whole-document path; windowed streaming and browser manifests reject unified mode.
222
+
223
+ `include_entities=True` explicitly adds sensitive `findings_v2_values`, linked by `finding_index` to canonical findings. This includes values whose output was redacted. Keep this audit payload and reversible maps out of logs. They are absent by default, apart from the map explicitly requested through reversible mode.
224
+
225
+ Custom callbacks need migration review. Legacy callbacks run after regex replacement and before NER.
226
+ Unified callbacks run before all detection, so callbacks that depend on `<EMAIL>` or other rendered markers receive different input.
227
+ See [configuration and rollback](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CONFIG.md#unified-processing-and-migration).
228
+
229
+ ## Caller-declared fields (opt-in)
230
+
231
+ Declare exact fields when a record needs protection beyond text detection. SCT does not infer meaning from field names.
232
+
233
+ ```python
234
+ from sct import FieldRule, FieldSchema, process_json
235
+
236
+ schema = FieldSchema("contacts", "1", (
237
+ FieldRule(("name",), ("PERSON",), "protect_value"),
238
+ ))
239
+ assert process_json(cleaner, {"name": "May", "status": "May"}, schema=schema).value == {
240
+ "name": "<PERSON>", "status": "May",
241
+ }
242
+ ```
243
+
244
+ Schemas require unified processing. Paths select original keys and list positions. Missing fields and `None` remain unchanged; selected containers fail. Whole-value protection also accepts finite numeric and boolean scalars. Default non-string handling stays unchanged. Protection uses the existing policy and replacement modes. Conflicting keep decisions, score errors, allow terms or disabled tags fail. Unknown scores with `unknown_score_action="protect"` remain compatible with thresholds. Explicit field types win equal-span ties after action priority.
245
+
246
+ Hints require providers that support context and the declared types. Presidio receives explicit declared types as context; ONNX-only providers reject required hints. DataFrame paths use `(column_label,)` across a column or `(row_label, column_label)` for a cell. Conflicting overlapping declarations fail. Framework paths use `(column_name,)` per record. RAG paths use `("text",)` or `("metadata", ...)`. Selected missing DataFrame cells remain unchanged. Masking numeric cells changes only the copied output column to object dtype. Unsupported path shapes fail.
247
+
248
+ Unified findings use positional references without original keys or labels. Schema-match paths and caller context stay out of signals and provenance. MCP full detail excludes sensitive value payloads. Framework constructors accept a trusted module-level `registry_factory`; workers pickle its reference and rebuild per process. Haystack data resolves only application-registered factory references. Protect serialized pipeline configuration, which can contain caller schema/context identifiers.
249
+
250
+ Applications register schemas through CLI `main(..., schemas=...)` or `create_mcp_server(..., schemas=...)`. Requests supply only exactly registered data-only schemas. CLI exposes `--processing unified --schema '<JSON>'`; `--signals` adds versioned counts and a summary. Requests cannot import Python factories. Existing adapter restrictions on reversible mode remain.
251
+
195
252
  ## Speed and hardware
196
253
 
197
254
  Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, on a workstation CPU and high-end GPUs, median of 3 runs. 0.8.1 runs at the same speed.
@@ -237,13 +294,32 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
237
294
 
238
295
  - **Safe defaults, every choice configurable.** Each behaviour is a config field; nothing is forced.
239
296
  - **No guarantee that all PII is found.** Models miss entities, mostly organisations and non-Latin scripts. Review output before you publish it.
240
- - **Free text, not tables.** For tables with known PII columns, drop or hash those columns first.
297
+ - **Declared fields need explicit rules.** Use a caller-owned schema to protect known PII fields; unselected fields remain unchanged.
241
298
  - **Reversible maps are personal data.** Anyone with the map can restore the values. Store maps apart from the cleaned data, with access control.
242
299
  - **Synthetic data is not anonymisation.** Fake values keep the data useful; they do not make it anonymous in a legal sense.
243
- - **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
300
+ - **Offsets name their text stage.** Legacy findings use post-regex positions. Unified findings use canonical positions and explicit source mapping quality.
244
301
  <!-- --8<-- [end:limits] -->
245
302
 
246
- Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
303
+ Detailed known limits: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
304
+
305
+ ## Opt-in document and corpus workflows
306
+
307
+ Users can combine extraction, unified detection, encrypted references and corpus selection without changing legacy defaults.
308
+ Document and framework adapters use non-reversible output; direct text processing can return encrypted map references.
309
+ The original documents and private corpus checkpoints remain caller-owned sensitive data.
310
+
311
+ | Capability | Entry point and limit |
312
+ |---|---|
313
+ | Encrypted references | `EncryptedSQLiteTokenStore` from `sct.token_stores`; keys and authorization remain application-owned |
314
+ | Corpus selection | `analyse_records` and `select_records` from `sct.corpus`; report-only by default, explicit basis required for removal |
315
+ | Native replay | `curate_resumable`; identity/checksum validation and private checkpoints, without exactly-once guarantees |
316
+ | PDF extraction | `extract_document` and `clean_document`; prepared local Docling assets, extracted-text protection only |
317
+ | Credential formats | Optional `secrets` detector; four syntactic categories, not credential validity |
318
+ | Language uncertainty | `language_policy='abstain'`; explicit thresholds and unknown-language action |
319
+ | Detector routing | `routing_policy='always_run'` stays default; calibrated cascade remains experimental |
320
+
321
+ See the [combined acceptance evidence](https://github.com/rhnfzl/SqueakyCleanText/blob/main/evaluation/improvements/ACCEPTANCE.md) for the tested profiles.
322
+ Trained Torch and ONNX scores agree on the parity controls. The synthetic gibberish control scored highest, so quality-filter accuracy remains unverified.
247
323
 
248
324
  ## Documentation
249
325
 
@@ -256,6 +332,7 @@ Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/Squea
256
332
  | JSON, DataFrame, RAG, CLI, streaming, OCR | [docs/ADAPTERS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/ADAPTERS.md) |
257
333
  | Score columns, quality, MinHash, data cards, `sct-report` | [docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md) |
258
334
  | NER backends, GLiNER, PII mode | [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md) |
335
+ | Encrypted local token maps | [docs/TOKEN_STORES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/TOKEN_STORES.md) |
259
336
  | Replacement modes, privacy policies, `TokenStore` | [docs/REPLACEMENT_MODES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/REPLACEMENT_MODES.md), [docs/PRIVACY.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PRIVACY.md) |
260
337
  | Evaluation | [docs/EVALUATION.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/EVALUATION.md), [docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md) |
261
338
  | Languages and adding a language | [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md), [docs/ADDING_LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/ADDING_LANGUAGES.md) |
@@ -275,7 +352,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
275
352
  author = {Fazal, Rehan},
276
353
  title = {SqueakyCleanText},
277
354
  url = {https://github.com/rhnfzl/SqueakyCleanText},
278
- version = {0.9.0},
355
+ version = {0.10.0},
279
356
  year = {2026}
280
357
  }
281
358
  ```
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SqueakyCleanText
3
- Version: 0.9.0
3
+ Version: 0.10.0
4
4
  Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
5
5
  Author: Rehan Fazal
6
6
  License-Expression: MIT
@@ -38,6 +38,17 @@ Requires-Dist: huggingface-hub>=1.5.0
38
38
  Requires-Dist: numpy>=2.2.0
39
39
  Requires-Dist: presidio_anonymizer>=2.2.360
40
40
  Requires-Dist: regex>=2024.11.6
41
+ Provides-Extra: secrets
42
+ Requires-Dist: datafog-core==1.0.0; extra == "secrets"
43
+ Provides-Extra: documents
44
+ Requires-Dist: docling==2.148.0; extra == "documents"
45
+ Provides-Extra: token-store
46
+ Requires-Dist: cryptography==48.0.1; extra == "token-store"
47
+ Provides-Extra: corpus
48
+ Requires-Dist: datasketch>=1.6; extra == "corpus"
49
+ Requires-Dist: datatrove==0.10.1; extra == "corpus"
50
+ Requires-Dist: orjson>=3.10; extra == "corpus"
51
+ Requires-Dist: spacy==3.8.16; extra == "corpus"
41
52
  Provides-Extra: gpu
42
53
  Requires-Dist: onnxruntime-gpu>=1.24.2; extra == "gpu"
43
54
  Provides-Extra: fuzzy
@@ -226,6 +237,7 @@ if __name__ == "__main__":
226
237
 
227
238
  | Where | Usage | Docs |
228
239
  |---|---|---|
240
+ | PDF extraction | `extract_document` and `clean_document` | [DOCUMENTS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/DOCUMENTS.md) |
229
241
  | LangChain | `SCTDocumentTransformer().transform_documents(docs)` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
230
242
  | Haystack | `pipe.add_component("sct", SCTDocumentCleaner())` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
231
243
  | Ray, Spark, Polars, DuckDB, dlt, Daft | `clean_columns(handle, batch)` | [RECIPES](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/RECIPES.md) |
@@ -283,6 +295,62 @@ Deprecated config globals, `ner_models_list`, the worker `batch_size` alias and
283
295
  Modes, privacy policies and more PII types (`ner_mode="pii"`): [docs/REPLACEMENT_MODES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/REPLACEMENT_MODES.md), [docs/PRIVACY.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PRIVACY.md), [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md).
284
296
  <!-- --8<-- [end:results] -->
285
297
 
298
+ ## One policy across detectors (opt-in)
299
+
300
+ Unified processing lets you apply the same handling rules to format matches and model findings. Existing callers keep `pii_processing="legacy"` by default.
301
+
302
+ ```python
303
+ from sct import TextCleaner, TextCleanerConfig, UnifiedPrivacyPolicy, HandlingRule
304
+
305
+ cleaner = TextCleaner(
306
+ TextCleanerConfig(pii_processing="unified", check_ner_process=False),
307
+ unified_policy=UnifiedPrivacyPolicy("contact-policy", "1", {
308
+ "EMAIL": HandlingRule(action="redact"),
309
+ }),
310
+ )
311
+ result = cleaner.process("Contact alice@example.com")
312
+ assert "alice@example.com" not in result.lm_text
313
+ ```
314
+
315
+ All selected detectors inspect the same canonical text before PII replacement. `use_mode` follows the placeholder, synthetic or reversible mode. `keep` affects PII replacement only. `redact` emits `<REDACTED>` and never saves that value in the reversible map. Protection wins over overlapping keep decisions; redaction wins over other protection. Unknown scores protect by default, or fail with `unknown_score_action="error"`.
316
+
317
+ A nonempty `detector_specs` tuple selects the complete detector set from an application-owned `DetectorRegistry`. Empty specs select the configured built-in regex rules and native NER when `check_ner_process=True`. Native registry loading requires `allow_network_detectors=True` because model artifacts may need downloading. External-only selection does not initialize native NER. Separately requested document classification still runs and also requires network permission in unified mode. Legacy model-loading behavior stays unchanged.
318
+
319
+ `metadata["findings_v2"]` contains value-free canonical positions, original-input ranges, mapping quality and policy decisions. `counts_v2` separates detector hits after configured allow-term filtering from resolved replacements; `provenance_v2` records detector and policy identities. Legacy counters stay empty in unified mode. Unified signals add `version=2`, `counts_v2`, `coverage_v2` and `provenance_v2`. CLI/MCP summaries distinguish detector hits from rendered replacements. `coverage_v2` reports source coverage quality and lower/upper bounds. Non-exact coverage leaves `pii_chars` and `pii_share` as `None`; score summaries place these shares in an explicit unknown bucket. Only actual built-in regex formatting findings receive the non-PII format exclusion; native, external and caller-declared dates remain sensitive.
320
+
321
+ Source mappings use Unicode code-point offsets and report `exact`, `covering` or `unavailable`. Complex built-in repairs can cover the whole input. Opaque custom callbacks make mapping unavailable, even when they return unchanged text. `source_mapping="require_source"` rejects those callbacks. These ranges do not describe original PDF bytes.
322
+
323
+ `process(..., context=DetectionContext(...))`, `process_batch(..., contexts=...)` and `aprocess_batch(..., contexts=...)` accept caller context in unified mode. Batch contexts must match input count and order. Buffered streaming uses this whole-document path; windowed streaming and browser manifests reject unified mode.
324
+
325
+ `include_entities=True` explicitly adds sensitive `findings_v2_values`, linked by `finding_index` to canonical findings. This includes values whose output was redacted. Keep this audit payload and reversible maps out of logs. They are absent by default, apart from the map explicitly requested through reversible mode.
326
+
327
+ Custom callbacks need migration review. Legacy callbacks run after regex replacement and before NER.
328
+ Unified callbacks run before all detection, so callbacks that depend on `<EMAIL>` or other rendered markers receive different input.
329
+ See [configuration and rollback](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CONFIG.md#unified-processing-and-migration).
330
+
331
+ ## Caller-declared fields (opt-in)
332
+
333
+ Declare exact fields when a record needs protection beyond text detection. SCT does not infer meaning from field names.
334
+
335
+ ```python
336
+ from sct import FieldRule, FieldSchema, process_json
337
+
338
+ schema = FieldSchema("contacts", "1", (
339
+ FieldRule(("name",), ("PERSON",), "protect_value"),
340
+ ))
341
+ assert process_json(cleaner, {"name": "May", "status": "May"}, schema=schema).value == {
342
+ "name": "<PERSON>", "status": "May",
343
+ }
344
+ ```
345
+
346
+ Schemas require unified processing. Paths select original keys and list positions. Missing fields and `None` remain unchanged; selected containers fail. Whole-value protection also accepts finite numeric and boolean scalars. Default non-string handling stays unchanged. Protection uses the existing policy and replacement modes. Conflicting keep decisions, score errors, allow terms or disabled tags fail. Unknown scores with `unknown_score_action="protect"` remain compatible with thresholds. Explicit field types win equal-span ties after action priority.
347
+
348
+ Hints require providers that support context and the declared types. Presidio receives explicit declared types as context; ONNX-only providers reject required hints. DataFrame paths use `(column_label,)` across a column or `(row_label, column_label)` for a cell. Conflicting overlapping declarations fail. Framework paths use `(column_name,)` per record. RAG paths use `("text",)` or `("metadata", ...)`. Selected missing DataFrame cells remain unchanged. Masking numeric cells changes only the copied output column to object dtype. Unsupported path shapes fail.
349
+
350
+ Unified findings use positional references without original keys or labels. Schema-match paths and caller context stay out of signals and provenance. MCP full detail excludes sensitive value payloads. Framework constructors accept a trusted module-level `registry_factory`; workers pickle its reference and rebuild per process. Haystack data resolves only application-registered factory references. Protect serialized pipeline configuration, which can contain caller schema/context identifiers.
351
+
352
+ Applications register schemas through CLI `main(..., schemas=...)` or `create_mcp_server(..., schemas=...)`. Requests supply only exactly registered data-only schemas. CLI exposes `--processing unified --schema '<JSON>'`; `--signals` adds versioned counts and a summary. Requests cannot import Python factories. Existing adapter restrictions on reversible mode remain.
353
+
286
354
  ## Speed and hardware
287
355
 
288
356
  Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, on a workstation CPU and high-end GPUs, median of 3 runs. 0.8.1 runs at the same speed.
@@ -328,13 +396,32 @@ Language detection covers 75 languages. Full NER models cover English, Dutch, Ge
328
396
 
329
397
  - **Safe defaults, every choice configurable.** Each behaviour is a config field; nothing is forced.
330
398
  - **No guarantee that all PII is found.** Models miss entities, mostly organisations and non-Latin scripts. Review output before you publish it.
331
- - **Free text, not tables.** For tables with known PII columns, drop or hash those columns first.
399
+ - **Declared fields need explicit rules.** Use a caller-owned schema to protect known PII fields; unselected fields remain unchanged.
332
400
  - **Reversible maps are personal data.** Anyone with the map can restore the values. Store maps apart from the cleaned data, with access control.
333
401
  - **Synthetic data is not anonymisation.** Fake values keep the data useful; they do not make it anonymous in a legal sense.
334
- - **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
402
+ - **Offsets name their text stage.** Legacy findings use post-regex positions. Unified findings use canonical positions and explicit source mapping quality.
335
403
  <!-- --8<-- [end:limits] -->
336
404
 
337
- Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
405
+ Detailed known limits: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
406
+
407
+ ## Opt-in document and corpus workflows
408
+
409
+ Users can combine extraction, unified detection, encrypted references and corpus selection without changing legacy defaults.
410
+ Document and framework adapters use non-reversible output; direct text processing can return encrypted map references.
411
+ The original documents and private corpus checkpoints remain caller-owned sensitive data.
412
+
413
+ | Capability | Entry point and limit |
414
+ |---|---|
415
+ | Encrypted references | `EncryptedSQLiteTokenStore` from `sct.token_stores`; keys and authorization remain application-owned |
416
+ | Corpus selection | `analyse_records` and `select_records` from `sct.corpus`; report-only by default, explicit basis required for removal |
417
+ | Native replay | `curate_resumable`; identity/checksum validation and private checkpoints, without exactly-once guarantees |
418
+ | PDF extraction | `extract_document` and `clean_document`; prepared local Docling assets, extracted-text protection only |
419
+ | Credential formats | Optional `secrets` detector; four syntactic categories, not credential validity |
420
+ | Language uncertainty | `language_policy='abstain'`; explicit thresholds and unknown-language action |
421
+ | Detector routing | `routing_policy='always_run'` stays default; calibrated cascade remains experimental |
422
+
423
+ See the [combined acceptance evidence](https://github.com/rhnfzl/SqueakyCleanText/blob/main/evaluation/improvements/ACCEPTANCE.md) for the tested profiles.
424
+ Trained Torch and ONNX scores agree on the parity controls. The synthetic gibberish control scored highest, so quality-filter accuracy remains unverified.
338
425
 
339
426
  ## Documentation
340
427
 
@@ -347,6 +434,7 @@ Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/Squea
347
434
  | JSON, DataFrame, RAG, CLI, streaming, OCR | [docs/ADAPTERS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/ADAPTERS.md) |
348
435
  | Score columns, quality, MinHash, data cards, `sct-report` | [docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md) |
349
436
  | NER backends, GLiNER, PII mode | [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md) |
437
+ | Encrypted local token maps | [docs/TOKEN_STORES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/TOKEN_STORES.md) |
350
438
  | Replacement modes, privacy policies, `TokenStore` | [docs/REPLACEMENT_MODES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/REPLACEMENT_MODES.md), [docs/PRIVACY.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PRIVACY.md) |
351
439
  | Evaluation | [docs/EVALUATION.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/EVALUATION.md), [docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md) |
352
440
  | Languages and adding a language | [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md), [docs/ADDING_LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/ADDING_LANGUAGES.md) |
@@ -366,7 +454,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
366
454
  author = {Fazal, Rehan},
367
455
  title = {SqueakyCleanText},
368
456
  url = {https://github.com/rhnfzl/SqueakyCleanText},
369
- version = {0.9.0},
457
+ version = {0.10.0},
370
458
  year = {2026}
371
459
  }
372
460
  ```
@@ -12,26 +12,46 @@ sct/adapters.py
12
12
  sct/browser.py
13
13
  sct/cli.py
14
14
  sct/config.py
15
+ sct/coordinates.py
16
+ sct/detectors.py
17
+ sct/detectors_builtin.py
18
+ sct/detectors_datafog.py
19
+ sct/documents.py
15
20
  sct/evaluation.py
21
+ sct/findings.py
22
+ sct/language_policy.py
16
23
  sct/mcp_server.py
17
24
  sct/model_benchmark.py
18
25
  sct/model_candidates.py
19
26
  sct/ocr.py
27
+ sct/policy_v2.py
20
28
  sct/presets.py
21
29
  sct/privacy.py
22
30
  sct/py.typed
23
31
  sct/report_cli.py
24
32
  sct/risk.py
33
+ sct/routing.py
34
+ sct/schema.py
25
35
  sct/score.py
26
36
  sct/sct.py
27
37
  sct/signals.py
28
38
  sct/streaming.py
39
+ sct/unified.py
40
+ sct/corpus/__init__.py
41
+ sct/corpus/datatrove.py
42
+ sct/corpus/manifest.py
43
+ sct/corpus/runner.py
44
+ sct/corpus/selection.py
45
+ sct/corpus/types.py
29
46
  sct/integrations/__init__.py
30
47
  sct/integrations/_runtime.py
31
48
  sct/integrations/datatrove.py
49
+ sct/integrations/docling.py
32
50
  sct/integrations/haystack.py
33
51
  sct/integrations/hf.py
34
52
  sct/integrations/langchain.py
53
+ sct/token_stores/__init__.py
54
+ sct/token_stores/sqlite.py
35
55
  sct/utils/__init__.py
36
56
  sct/utils/anonymization_map.py
37
57
  sct/utils/atomic_file.py
@@ -55,6 +75,7 @@ sct/utils/torch_pipeline.py
55
75
  tests/test_adapters.py
56
76
  tests/test_address_leftovers.py
57
77
  tests/test_atomic_file.py
78
+ tests/test_baseline_measurement.py
58
79
  tests/test_batching.py
59
80
  tests/test_benchmark_adapters.py
60
81
  tests/test_benchmark_loaders.py
@@ -64,27 +85,43 @@ tests/test_benchmarks_round2.py
64
85
  tests/test_benchmarks_round2_tools.py
65
86
  tests/test_benchmarks_score.py
66
87
  tests/test_browser.py
88
+ tests/test_builtin_detections.py
67
89
  tests/test_chunking.py
68
90
  tests/test_cjk_width.py
69
91
  tests/test_cleaner_lifecycle.py
70
92
  tests/test_cli.py
71
93
  tests/test_config_state.py
72
94
  tests/test_contracts.py
95
+ tests/test_coordinates.py
96
+ tests/test_corpus_decontamination.py
97
+ tests/test_corpus_failure_boundaries.py
98
+ tests/test_corpus_identity_boundaries.py
99
+ tests/test_corpus_manifest.py
100
+ tests/test_corpus_resume.py
101
+ tests/test_corpus_selection.py
73
102
  tests/test_date_forms.py
74
103
  tests/test_dates_cjk_styled.py
75
104
  tests/test_dates_default.py
105
+ tests/test_detections.py
106
+ tests/test_detector_registry.py
107
+ tests/test_detector_routing.py
76
108
  tests/test_device_provenance.py
109
+ tests/test_docling_integration.py
77
110
  tests/test_email_forms.py
78
111
  tests/test_escaped_markup.py
79
112
  tests/test_evaluation.py
80
113
  tests/test_fork_safety.py
81
114
  tests/test_functional.py
82
115
  tests/test_fuzz_styled_pii.py
116
+ tests/test_gliner_recipe_identity.py
117
+ tests/test_gliner_revision.py
118
+ tests/test_improvements_end_to_end.py
83
119
  tests/test_integrations_frameworks.py
84
120
  tests/test_integrations_hf.py
85
121
  tests/test_integrations_runtime.py
86
122
  tests/test_keep_scripts.py
87
123
  tests/test_language_all.py
124
+ tests/test_language_policy.py
88
125
  tests/test_linux_workers.py
89
126
  tests/test_mcp_path.py
90
127
  tests/test_mcp_server.py
@@ -99,6 +136,7 @@ tests/test_ocr.py
99
136
  tests/test_onnx_concurrency.py
100
137
  tests/test_onnx_external_data.py
101
138
  tests/test_onnx_model_cache.py
139
+ tests/test_optional_profiles.py
102
140
  tests/test_phone_forms.py
103
141
  tests/test_pii_social_cards.py
104
142
  tests/test_pipeline_text_steps.py
@@ -108,8 +146,10 @@ tests/test_predicted_spans_monotone.py
108
146
  tests/test_presidio_tags_and_fakes.py
109
147
  tests/test_presidio_type_names.py
110
148
  tests/test_privacy.py
149
+ tests/test_processing_compatibility.py
111
150
  tests/test_prose_punctuation_kept.py
112
151
  tests/test_public_view.py
152
+ tests/test_quality_export.py
113
153
  tests/test_regex_boundaries.py
114
154
  tests/test_regex_counts.py
115
155
  tests/test_release_comparison_watcher.py
@@ -117,9 +157,14 @@ tests/test_repo_scripts.py
117
157
  tests/test_report_cli.py
118
158
  tests/test_reversible_markers.py
119
159
  tests/test_risk.py
160
+ tests/test_routing_audit.py
161
+ tests/test_routing_calibration.py
162
+ tests/test_schema_context.py
120
163
  tests/test_score.py
121
164
  tests/test_sct.py
165
+ tests/test_secret_detector.py
122
166
  tests/test_security_leftovers.py
167
+ tests/test_sqlite_token_store.py
123
168
  tests/test_stream_guard_shared_view.py
124
169
  tests/test_stream_keeps_values_whole.py
125
170
  tests/test_streaming.py
@@ -128,8 +173,11 @@ tests/test_styled_pii.py
128
173
  tests/test_surface_contracts.py
129
174
  tests/test_synthetic_determinism.py
130
175
  tests/test_synthetic_per_document.py
176
+ tests/test_token_store_failure_boundaries.py
177
+ tests/test_token_store_operations.py
131
178
  tests/test_torch_ner_backend.py
132
179
  tests/test_training_preset.py
133
180
  tests/test_type_names.py
181
+ tests/test_unified_processing.py
134
182
  tests/test_url_forms.py
135
183
  tests/test_user_terms.py
@@ -16,6 +16,12 @@ SqueakyCleanText[gliner,gliner2,torch]
16
16
  [classify]
17
17
  gliclass>=0.1.16
18
18
 
19
+ [corpus]
20
+ datasketch>=1.6
21
+ datatrove==0.10.1
22
+ orjson>=3.10
23
+ spacy==3.8.16
24
+
19
25
  [datasets]
20
26
  datasets>=3.0
21
27
 
@@ -30,6 +36,9 @@ pytest>=8.3
30
36
  pytest-timeout>=2.3
31
37
  rapidfuzz>=3.12
32
38
 
39
+ [documents]
40
+ docling==2.148.0
41
+
33
42
  [fuzzy]
34
43
  rapidfuzz>=3.12
35
44
 
@@ -68,6 +77,9 @@ onnx>=1.19
68
77
  [score]
69
78
  datasketch>=1.6
70
79
 
80
+ [secrets]
81
+ datafog-core==1.0.0
82
+
71
83
  [synthetic]
72
84
  faker>=33.0
73
85
 
@@ -75,6 +87,9 @@ faker>=33.0
75
87
  coverage>=7.8
76
88
  pytest-cov>=6.0
77
89
 
90
+ [token-store]
91
+ cryptography==48.0.1
92
+
78
93
  [torch]
79
94
  torch>=2.6.0
80
95
  transformers>=5.0
@@ -58,6 +58,10 @@ sct-mcp = "sct.mcp_server:main"
58
58
  sct-report = "sct.report_cli:main"
59
59
 
60
60
  [project.optional-dependencies]
61
+ secrets = ["datafog-core==1.0.0"]
62
+ documents = ["docling==2.148.0"]
63
+ token-store = ["cryptography==48.0.1"]
64
+ corpus = ["datasketch>=1.6", "datatrove==0.10.1", "orjson>=3.10", "spacy==3.8.16"]
61
65
  gpu = [
62
66
  "onnxruntime-gpu>=1.24.2",
63
67
  ]
@@ -142,6 +146,9 @@ sct = ["py.typed"]
142
146
  timeout = 1200
143
147
  markers = ["needs_model: loads a real Hugging Face model"]
144
148
 
149
+ [tool.coverage.report]
150
+ precision = 2
151
+
145
152
  [tool.ruff]
146
153
  line-length = 127
147
154