SqueakyCleanText 0.9.0__tar.gz → 0.11.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/PKG-INFO +121 -6
  2. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/README.md +106 -5
  3. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/SqueakyCleanText.egg-info/PKG-INFO +121 -6
  4. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/SqueakyCleanText.egg-info/SOURCES.txt +54 -0
  5. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/SqueakyCleanText.egg-info/requires.txt +19 -0
  6. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/pyproject.toml +8 -0
  7. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/__init__.py +19 -1
  8. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/adapters.py +96 -22
  9. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/browser.py +2 -0
  10. squeakycleantext-0.11.0/sct/chunked_document.py +127 -0
  11. squeakycleantext-0.11.0/sct/chunkers_builtin.py +97 -0
  12. squeakycleantext-0.11.0/sct/chunking.py +227 -0
  13. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/cli.py +29 -6
  14. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/config.py +77 -2
  15. squeakycleantext-0.11.0/sct/coordinates.py +111 -0
  16. squeakycleantext-0.11.0/sct/corpus/__init__.py +11 -0
  17. squeakycleantext-0.11.0/sct/corpus/datatrove.py +365 -0
  18. squeakycleantext-0.11.0/sct/corpus/manifest.py +174 -0
  19. squeakycleantext-0.11.0/sct/corpus/runner.py +463 -0
  20. squeakycleantext-0.11.0/sct/corpus/selection.py +292 -0
  21. squeakycleantext-0.11.0/sct/corpus/types.py +69 -0
  22. squeakycleantext-0.11.0/sct/detector_windows.py +96 -0
  23. squeakycleantext-0.11.0/sct/detectors.py +200 -0
  24. squeakycleantext-0.11.0/sct/detectors_builtin.py +119 -0
  25. squeakycleantext-0.11.0/sct/detectors_datafog.py +63 -0
  26. squeakycleantext-0.11.0/sct/documents.py +181 -0
  27. squeakycleantext-0.11.0/sct/findings.py +93 -0
  28. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/integrations/_runtime.py +68 -10
  29. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/integrations/datatrove.py +10 -2
  30. squeakycleantext-0.11.0/sct/integrations/docling.py +176 -0
  31. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/integrations/haystack.py +33 -4
  32. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/integrations/hf.py +23 -5
  33. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/integrations/langchain.py +10 -2
  34. squeakycleantext-0.11.0/sct/language_policy.py +31 -0
  35. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/mcp_server.py +52 -13
  36. squeakycleantext-0.11.0/sct/policy_v2.py +71 -0
  37. squeakycleantext-0.11.0/sct/routing.py +221 -0
  38. squeakycleantext-0.11.0/sct/schema.py +120 -0
  39. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/score.py +4 -2
  40. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/sct.py +150 -30
  41. squeakycleantext-0.11.0/sct/signals.py +241 -0
  42. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/streaming.py +4 -0
  43. squeakycleantext-0.11.0/sct/token_stores/__init__.py +5 -0
  44. squeakycleantext-0.11.0/sct/token_stores/sqlite.py +228 -0
  45. squeakycleantext-0.11.0/sct/unified.py +389 -0
  46. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/datetime.py +14 -10
  47. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/gliner_adapter.py +5 -0
  48. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/ner.py +29 -87
  49. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/ner_chunking.py +21 -9
  50. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/ner_spans.py +99 -0
  51. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/process_result.py +17 -1
  52. squeakycleantext-0.11.0/tests/test_baseline_measurement.py +71 -0
  53. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_benchmark_loaders.py +18 -0
  54. squeakycleantext-0.11.0/tests/test_builtin_detections.py +137 -0
  55. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_chunking.py +30 -0
  56. squeakycleantext-0.11.0/tests/test_chunking_router.py +238 -0
  57. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_cleaner_lifecycle.py +8 -0
  58. squeakycleantext-0.11.0/tests/test_coordinates.py +101 -0
  59. squeakycleantext-0.11.0/tests/test_corpus_decontamination.py +122 -0
  60. squeakycleantext-0.11.0/tests/test_corpus_failure_boundaries.py +174 -0
  61. squeakycleantext-0.11.0/tests/test_corpus_identity_boundaries.py +242 -0
  62. squeakycleantext-0.11.0/tests/test_corpus_manifest.py +291 -0
  63. squeakycleantext-0.11.0/tests/test_corpus_resume.py +225 -0
  64. squeakycleantext-0.11.0/tests/test_corpus_selection.py +312 -0
  65. squeakycleantext-0.11.0/tests/test_detections.py +56 -0
  66. squeakycleantext-0.11.0/tests/test_detector_registry.py +283 -0
  67. squeakycleantext-0.11.0/tests/test_detector_routing.py +439 -0
  68. squeakycleantext-0.11.0/tests/test_docling_integration.py +446 -0
  69. squeakycleantext-0.11.0/tests/test_gliner_recipe_identity.py +86 -0
  70. squeakycleantext-0.11.0/tests/test_gliner_revision.py +51 -0
  71. squeakycleantext-0.11.0/tests/test_improvements_end_to_end.py +188 -0
  72. squeakycleantext-0.11.0/tests/test_integrations_hf.py +177 -0
  73. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_language_all.py +21 -0
  74. squeakycleantext-0.11.0/tests/test_language_policy.py +280 -0
  75. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_linux_workers.py +33 -0
  76. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_ner_span_repair.py +24 -0
  77. squeakycleantext-0.11.0/tests/test_optional_profiles.py +75 -0
  78. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_pii_social_cards.py +4 -0
  79. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_plot_pii_comparison.py +2 -3
  80. squeakycleantext-0.11.0/tests/test_plot_pii_social_release.py +43 -0
  81. squeakycleantext-0.11.0/tests/test_processing_compatibility.py +59 -0
  82. squeakycleantext-0.11.0/tests/test_quality_export.py +109 -0
  83. squeakycleantext-0.11.0/tests/test_routing_audit.py +551 -0
  84. squeakycleantext-0.11.0/tests/test_routing_calibration.py +257 -0
  85. squeakycleantext-0.11.0/tests/test_schema_context.py +564 -0
  86. squeakycleantext-0.11.0/tests/test_secret_detector.py +227 -0
  87. squeakycleantext-0.11.0/tests/test_sqlite_token_store.py +199 -0
  88. squeakycleantext-0.11.0/tests/test_token_store_failure_boundaries.py +135 -0
  89. squeakycleantext-0.11.0/tests/test_token_store_operations.py +344 -0
  90. squeakycleantext-0.11.0/tests/test_unified_processing.py +537 -0
  91. squeakycleantext-0.9.0/sct/signals.py +0 -130
  92. squeakycleantext-0.9.0/tests/test_integrations_hf.py +0 -70
  93. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/LICENSE +0 -0
  94. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/SqueakyCleanText.egg-info/dependency_links.txt +0 -0
  95. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/SqueakyCleanText.egg-info/entry_points.txt +0 -0
  96. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/SqueakyCleanText.egg-info/top_level.txt +0 -0
  97. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/evaluation.py +0 -0
  98. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/integrations/__init__.py +0 -0
  99. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/model_benchmark.py +0 -0
  100. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/model_candidates.py +0 -0
  101. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/ocr.py +0 -0
  102. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/presets.py +0 -0
  103. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/privacy.py +0 -0
  104. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/py.typed +0 -0
  105. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/report_cli.py +0 -0
  106. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/risk.py +0 -0
  107. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/__init__.py +0 -0
  108. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/anonymization_map.py +0 -0
  109. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/atomic_file.py +0 -0
  110. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/constants.py +0 -0
  111. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/contact.py +0 -0
  112. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/gliclass_adapter.py +0 -0
  113. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/markers.py +0 -0
  114. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/normtext.py +0 -0
  115. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/onnx_pipeline.py +0 -0
  116. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/resources.py +0 -0
  117. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/special.py +0 -0
  118. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/stopwords.py +0 -0
  119. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/synthetic.py +0 -0
  120. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/sct/utils/torch_pipeline.py +0 -0
  121. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/setup.cfg +0 -0
  122. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_adapters.py +0 -0
  123. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_address_leftovers.py +0 -0
  124. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_atomic_file.py +0 -0
  125. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_batching.py +0 -0
  126. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_benchmark_adapters.py +0 -0
  127. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_benchmark_release.py +0 -0
  128. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_benchmarks_compare.py +0 -0
  129. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_benchmarks_round2.py +0 -0
  130. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_benchmarks_round2_tools.py +0 -0
  131. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_benchmarks_score.py +0 -0
  132. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_browser.py +0 -0
  133. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_cjk_width.py +0 -0
  134. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_cli.py +0 -0
  135. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_config_state.py +0 -0
  136. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_contracts.py +0 -0
  137. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_date_forms.py +0 -0
  138. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_dates_cjk_styled.py +0 -0
  139. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_dates_default.py +0 -0
  140. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_device_provenance.py +0 -0
  141. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_email_forms.py +0 -0
  142. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_escaped_markup.py +0 -0
  143. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_evaluation.py +0 -0
  144. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_fork_safety.py +0 -0
  145. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_functional.py +0 -0
  146. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_fuzz_styled_pii.py +0 -0
  147. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_integrations_frameworks.py +0 -0
  148. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_integrations_runtime.py +0 -0
  149. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_keep_scripts.py +0 -0
  150. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_mcp_path.py +0 -0
  151. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_mcp_server.py +0 -0
  152. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_model_benchmarks.py +0 -0
  153. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_multilingual_eval_set.py +0 -0
  154. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_ner_sees_tags.py +0 -0
  155. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_ner_spans.py +0 -0
  156. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_ner_windows.py +0 -0
  157. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_number_identifiers.py +0 -0
  158. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_ocr.py +0 -0
  159. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_onnx_concurrency.py +0 -0
  160. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_onnx_external_data.py +0 -0
  161. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_onnx_model_cache.py +0 -0
  162. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_phone_forms.py +0 -0
  163. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_pipeline_text_steps.py +0 -0
  164. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_plot_version_comparison.py +0 -0
  165. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_predicted_spans_monotone.py +0 -0
  166. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_presidio_tags_and_fakes.py +0 -0
  167. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_presidio_type_names.py +0 -0
  168. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_privacy.py +0 -0
  169. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_prose_punctuation_kept.py +0 -0
  170. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_public_view.py +0 -0
  171. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_regex_boundaries.py +0 -0
  172. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_regex_counts.py +0 -0
  173. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_release_comparison_watcher.py +0 -0
  174. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_repo_scripts.py +0 -0
  175. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_report_cli.py +0 -0
  176. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_reversible_markers.py +0 -0
  177. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_risk.py +0 -0
  178. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_score.py +0 -0
  179. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_sct.py +0 -0
  180. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_security_leftovers.py +0 -0
  181. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_stream_guard_shared_view.py +0 -0
  182. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_stream_keeps_values_whole.py +0 -0
  183. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_streaming.py +0 -0
  184. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_streaming_split_stability.py +0 -0
  185. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_styled_pii.py +0 -0
  186. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_surface_contracts.py +0 -0
  187. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_synthetic_determinism.py +0 -0
  188. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_synthetic_per_document.py +0 -0
  189. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_torch_ner_backend.py +0 -0
  190. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_training_preset.py +0 -0
  191. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_type_names.py +0 -0
  192. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_url_forms.py +0 -0
  193. {squeakycleantext-0.9.0 → squeakycleantext-0.11.0}/tests/test_user_terms.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SqueakyCleanText
3
- Version: 0.9.0
3
+ Version: 0.11.0
4
4
  Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
5
5
  Author: Rehan Fazal
6
6
  License-Expression: MIT
@@ -38,6 +38,20 @@ Requires-Dist: huggingface-hub>=1.5.0
38
38
  Requires-Dist: numpy>=2.2.0
39
39
  Requires-Dist: presidio_anonymizer>=2.2.360
40
40
  Requires-Dist: regex>=2024.11.6
41
+ Provides-Extra: secrets
42
+ Requires-Dist: datafog-core==1.0.0; extra == "secrets"
43
+ Provides-Extra: documents
44
+ Requires-Dist: docling==2.148.0; extra == "documents"
45
+ Provides-Extra: chunking
46
+ Requires-Dist: semchunk<4,>=3.2.5; extra == "chunking"
47
+ Requires-Dist: chonkie<2,>=1.7.0; extra == "chunking"
48
+ Provides-Extra: token-store
49
+ Requires-Dist: cryptography==48.0.1; extra == "token-store"
50
+ Provides-Extra: corpus
51
+ Requires-Dist: datasketch>=1.6; extra == "corpus"
52
+ Requires-Dist: datatrove==0.10.1; extra == "corpus"
53
+ Requires-Dist: orjson>=3.10; extra == "corpus"
54
+ Requires-Dist: spacy==3.8.16; extra == "corpus"
41
55
  Provides-Extra: gpu
42
56
  Requires-Dist: onnxruntime-gpu>=1.24.2; extra == "gpu"
43
57
  Provides-Extra: fuzzy
@@ -226,6 +240,8 @@ if __name__ == "__main__":
226
240
 
227
241
  | Where | Usage | Docs |
228
242
  |---|---|---|
243
+ | PDF extraction | `extract_document` and `clean_document` | [DOCUMENTS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/DOCUMENTS.md) |
244
+ | Clean and chunk a document | `chunk_document(cleaner, text, max_size=1024)` | [CHUNKING](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CHUNKING.md) |
229
245
  | LangChain | `SCTDocumentTransformer().transform_documents(docs)` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
230
246
  | Haystack | `pipe.add_component("sct", SCTDocumentCleaner())` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
231
247
  | Ray, Spark, Polars, DuckDB, dlt, Daft | `clean_columns(handle, batch)` | [RECIPES](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/RECIPES.md) |
@@ -236,6 +252,23 @@ if __name__ == "__main__":
236
252
 
237
253
  The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
238
254
 
255
+ ### Clean a document and produce chunks
256
+
257
+ Use `chunk_document()` to clean the full document before you split it for search, training, or export.
258
+
259
+ ```python
260
+ from sct import TextCleaner, TextCleanerConfig, chunk_document
261
+
262
+ cleaner = TextCleaner(TextCleanerConfig(check_ner_process=False))
263
+ result = chunk_document(cleaner, "Contact alice@example.com for details.", max_size=24)
264
+ texts = [chunk.text for chunk in result.chunks]
265
+ ```
266
+
267
+ SCT can use Markdown headings or cleaned document elements as boundaries. Install
268
+ `SqueakyCleanText[chunking]` for Semchunk and Chonkie, or register your own splitter.
269
+ In unified PII processing, you can also choose a chunker for each compatible detector.
270
+ See [chunking providers and routing](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CHUNKING.md).
271
+
239
272
  ## How it works
240
273
 
241
274
  <img src="https://raw.githubusercontent.com/rhnfzl/SqueakyCleanText/main/resources/sct_flow.png" alt="SqueakyCleanText pipeline: any entry point runs 1 detect language, 2 fix the text, 3 regex PII and cleanup, 4 NER, 5 replacement mode (placeholder tags are written in steps 3 and 4; synthetic and reversible swap the values in after NER), 6 output and signals. Dashed steps are off by default." width="900">
@@ -283,6 +316,62 @@ Deprecated config globals, `ner_models_list`, the worker `batch_size` alias and
283
316
  Modes, privacy policies and more PII types (`ner_mode="pii"`): [docs/REPLACEMENT_MODES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/REPLACEMENT_MODES.md), [docs/PRIVACY.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PRIVACY.md), [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md).
284
317
  <!-- --8<-- [end:results] -->
285
318
 
319
+ ## One policy across detectors (opt-in)
320
+
321
+ Unified processing lets you apply the same handling rules to format matches and model findings. Existing callers keep `pii_processing="legacy"` by default.
322
+
323
+ ```python
324
+ from sct import TextCleaner, TextCleanerConfig, UnifiedPrivacyPolicy, HandlingRule
325
+
326
+ cleaner = TextCleaner(
327
+ TextCleanerConfig(pii_processing="unified", check_ner_process=False),
328
+ unified_policy=UnifiedPrivacyPolicy("contact-policy", "1", {
329
+ "EMAIL": HandlingRule(action="redact"),
330
+ }),
331
+ )
332
+ result = cleaner.process("Contact alice@example.com")
333
+ assert "alice@example.com" not in result.lm_text
334
+ ```
335
+
336
+ All selected detectors inspect the same canonical text before PII replacement. `use_mode` follows the placeholder, synthetic or reversible mode. `keep` affects PII replacement only. `redact` emits `<REDACTED>` and never saves that value in the reversible map. Protection wins over overlapping keep decisions; redaction wins over other protection. Unknown scores protect by default, or fail with `unknown_score_action="error"`.
337
+
338
+ A nonempty `detector_specs` tuple selects the complete detector set from an application-owned `DetectorRegistry`. Empty specs select the configured built-in regex rules and native NER when `check_ner_process=True`. Native registry loading requires `allow_network_detectors=True` because model artifacts may need downloading. External-only selection does not initialize native NER. Separately requested document classification still runs and also requires network permission in unified mode. Legacy model-loading behavior stays unchanged.
339
+
340
+ `metadata["findings_v2"]` contains value-free canonical positions, original-input ranges, mapping quality and policy decisions. `counts_v2` separates detector hits after configured allow-term filtering from resolved replacements; `provenance_v2` records detector and policy identities. Legacy counters stay empty in unified mode. Unified signals add `version=2`, `counts_v2`, `coverage_v2` and `provenance_v2`. CLI/MCP summaries distinguish detector hits from rendered replacements. `coverage_v2` reports source coverage quality and lower/upper bounds. Non-exact coverage leaves `pii_chars` and `pii_share` as `None`; score summaries place these shares in an explicit unknown bucket. Only actual built-in regex formatting findings receive the non-PII format exclusion; native, external and caller-declared dates remain sensitive.
341
+
342
+ Source mappings use Unicode code-point offsets and report `exact`, `covering` or `unavailable`. Complex built-in repairs can cover the whole input. Opaque custom callbacks make mapping unavailable, even when they return unchanged text. `source_mapping="require_source"` rejects those callbacks. These ranges do not describe original PDF bytes.
343
+
344
+ `process(..., context=DetectionContext(...))`, `process_batch(..., contexts=...)` and `aprocess_batch(..., contexts=...)` accept caller context in unified mode. Batch contexts must match input count and order. Buffered streaming uses this whole-document path; windowed streaming and browser manifests reject unified mode.
345
+
346
+ `include_entities=True` explicitly adds sensitive `findings_v2_values`, linked by `finding_index` to canonical findings. This includes values whose output was redacted. Keep this audit payload and reversible maps out of logs. They are absent by default, apart from the map explicitly requested through reversible mode.
347
+
348
+ Custom callbacks need migration review. Legacy callbacks run after regex replacement and before NER.
349
+ Unified callbacks run before all detection, so callbacks that depend on `<EMAIL>` or other rendered markers receive different input.
350
+ See [configuration and rollback](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CONFIG.md#unified-processing-and-migration).
351
+
352
+ ## Caller-declared fields (opt-in)
353
+
354
+ Declare exact fields when a record needs protection beyond text detection. SCT does not infer meaning from field names.
355
+
356
+ ```python
357
+ from sct import FieldRule, FieldSchema, process_json
358
+
359
+ schema = FieldSchema("contacts", "1", (
360
+ FieldRule(("name",), ("PERSON",), "protect_value"),
361
+ ))
362
+ assert process_json(cleaner, {"name": "May", "status": "May"}, schema=schema).value == {
363
+ "name": "<PERSON>", "status": "May",
364
+ }
365
+ ```
366
+
367
+ Schemas require unified processing. Paths select original keys and list positions. Missing fields and `None` remain unchanged; selected containers fail. Whole-value protection also accepts finite numeric and boolean scalars. Default non-string handling stays unchanged. Protection uses the existing policy and replacement modes. Conflicting keep decisions, score errors, allow terms or disabled tags fail. Unknown scores with `unknown_score_action="protect"` remain compatible with thresholds. Explicit field types win equal-span ties after action priority.
368
+
369
+ Hints require providers that support context and the declared types. Presidio receives explicit declared types as context; ONNX-only providers reject required hints. DataFrame paths use `(column_label,)` across a column or `(row_label, column_label)` for a cell. Conflicting overlapping declarations fail. Framework paths use `(column_name,)` per record. RAG paths use `("text",)` or `("metadata", ...)`. Selected missing DataFrame cells remain unchanged. Masking numeric cells changes only the copied output column to object dtype. Unsupported path shapes fail.
370
+
371
+ Unified findings use positional references without original keys or labels. Schema-match paths and caller context stay out of signals and provenance. MCP full detail excludes sensitive value payloads. Framework constructors accept a trusted module-level `registry_factory`; workers pickle its reference and rebuild per process. Haystack data resolves only application-registered factory references. Protect serialized pipeline configuration, which can contain caller schema/context identifiers.
372
+
373
+ Applications register schemas through CLI `main(..., schemas=...)` or `create_mcp_server(..., schemas=...)`. Requests supply only exactly registered data-only schemas. CLI exposes `--processing unified --schema '<JSON>'`; `--signals` adds versioned counts and a summary. Requests cannot import Python factories. Existing adapter restrictions on reversible mode remain.
374
+
286
375
  ## Speed and hardware
287
376
 
288
377
  Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, on a workstation CPU and high-end GPUs, median of 3 runs. 0.8.1 runs at the same speed.
@@ -303,6 +392,10 @@ NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cl
303
392
 
304
393
  Measured on open datasets with [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks).
305
394
 
395
+ The [SCT v0.10.0 comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-12-v010/README.md)
396
+ adds fresh SCT results over 20,817 documents, six charts, and repeated CPU measurements.
397
+ Other-tool versions remain historical pins. Shared-label and full-label results differ because label coverage affects the comparison.
398
+
306
399
  The [SCT v0.8.2 comparison with other PII tools](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-04-visuals/README.md)
307
400
  shows paired recall, precision and exposure results, with uncertainty intervals.
308
401
  It measures PII configurations rather than the whole processing pipeline. The tested version is v0.8.2, not v0.8.3.
@@ -317,24 +410,45 @@ Runtime versions differ, so the speed results do not isolate SCT code changes.
317
410
  - **PII** (12 open PII datasets, about 40 languages): the 0.8.1 release branch leaked fewer values than 0.8.0 on every dataset, for example 18.1% to 8.2% on REDACT ([comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/comparison-0.8.0-unreleased.md), pre-release commit `2078e8c4`).
318
411
  - **Multilingual PII eval** (220 records, 11 languages): 0.93 recall, 0.95 precision ([docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md)).
319
412
 
413
+ **Dataset credits.** These scores use open datasets, most under CC BY or other attribution licences. Creators, sources, licences and citations are in [benchmarks/DATASET_CREDITS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/DATASET_CREDITS.md).
414
+
320
415
  Test your own data or model with the evaluation harness ([docs/EVALUATION.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/EVALUATION.md)).
321
416
 
322
417
  ## Languages
323
418
 
324
- Language detection covers 75 languages. Full NER models cover English, Dutch, German and Spanish. The other languages use the multilingual WikiNEural model with the English model, or a model you bring with `ner_models`. Full list: [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md).
419
+ Language detection covers 75 languages. Full NER models cover English, Dutch, German and Spanish. The other languages use the multilingual WikiNEural model with the English model, or a model you bring with `ner_models`. The WikiNEural model is CC BY-NC-SA 4.0, for non-commercial research only, and SCT loads it by default. [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md#model-licences) shows how to run without it. Full list: [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md).
325
420
 
326
421
  <!-- --8<-- [start:limits] -->
327
422
  ## Limits and design principles
328
423
 
329
424
  - **Safe defaults, every choice configurable.** Each behaviour is a config field; nothing is forced.
330
425
  - **No guarantee that all PII is found.** Models miss entities, mostly organisations and non-Latin scripts. Review output before you publish it.
331
- - **Free text, not tables.** For tables with known PII columns, drop or hash those columns first.
426
+ - **Declared fields need explicit rules.** Use a caller-owned schema to protect known PII fields; unselected fields remain unchanged.
332
427
  - **Reversible maps are personal data.** Anyone with the map can restore the values. Store maps apart from the cleaned data, with access control.
333
428
  - **Synthetic data is not anonymisation.** Fake values keep the data useful; they do not make it anonymous in a legal sense.
334
- - **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
429
+ - **Offsets name their text stage.** Legacy findings use post-regex positions. Unified findings use canonical positions and explicit source mapping quality.
335
430
  <!-- --8<-- [end:limits] -->
336
431
 
337
- Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
432
+ Detailed known limits: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
433
+
434
+ ## Opt-in document and corpus workflows
435
+
436
+ Users can combine extraction, unified detection, encrypted references and corpus selection without changing legacy defaults.
437
+ Document and framework adapters use non-reversible output; direct text processing can return encrypted map references.
438
+ The original documents and private corpus checkpoints remain caller-owned sensitive data.
439
+
440
+ | Capability | Entry point and limit |
441
+ |---|---|
442
+ | Encrypted references | `EncryptedSQLiteTokenStore` from `sct.token_stores`; keys and authorization remain application-owned |
443
+ | Corpus selection | `analyse_records` and `select_records` from `sct.corpus`; report-only by default, explicit basis required for removal |
444
+ | Native replay | `curate_resumable`; identity/checksum validation and private checkpoints, without exactly-once guarantees |
445
+ | PDF extraction | `extract_document` and `clean_document`; prepared local Docling assets, extracted-text protection only |
446
+ | Credential formats | Optional `secrets` detector; four syntactic categories, not credential validity |
447
+ | Language uncertainty | `language_policy='abstain'`; explicit thresholds and unknown-language action |
448
+ | Detector routing | `routing_policy='always_run'` stays default; calibrated cascade remains experimental |
449
+
450
+ See the [combined acceptance evidence](https://github.com/rhnfzl/SqueakyCleanText/blob/main/evaluation/improvements/ACCEPTANCE.md) for the tested profiles.
451
+ Trained Torch and ONNX scores agree on the parity controls. The synthetic gibberish control scored highest, so quality-filter accuracy remains unverified.
338
452
 
339
453
  ## Documentation
340
454
 
@@ -347,6 +461,7 @@ Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/Squea
347
461
  | JSON, DataFrame, RAG, CLI, streaming, OCR | [docs/ADAPTERS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/ADAPTERS.md) |
348
462
  | Score columns, quality, MinHash, data cards, `sct-report` | [docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md) |
349
463
  | NER backends, GLiNER, PII mode | [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md) |
464
+ | Encrypted local token maps | [docs/TOKEN_STORES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/TOKEN_STORES.md) |
350
465
  | Replacement modes, privacy policies, `TokenStore` | [docs/REPLACEMENT_MODES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/REPLACEMENT_MODES.md), [docs/PRIVACY.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PRIVACY.md) |
351
466
  | Evaluation | [docs/EVALUATION.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/EVALUATION.md), [docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md) |
352
467
  | Languages and adding a language | [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md), [docs/ADDING_LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/ADDING_LANGUAGES.md) |
@@ -366,7 +481,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
366
481
  author = {Fazal, Rehan},
367
482
  title = {SqueakyCleanText},
368
483
  url = {https://github.com/rhnfzl/SqueakyCleanText},
369
- version = {0.9.0},
484
+ version = {0.10.0},
370
485
  year = {2026}
371
486
  }
372
487
  ```
@@ -135,6 +135,8 @@ if __name__ == "__main__":
135
135
 
136
136
  | Where | Usage | Docs |
137
137
  |---|---|---|
138
+ | PDF extraction | `extract_document` and `clean_document` | [DOCUMENTS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/DOCUMENTS.md) |
139
+ | Clean and chunk a document | `chunk_document(cleaner, text, max_size=1024)` | [CHUNKING](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CHUNKING.md) |
138
140
  | LangChain | `SCTDocumentTransformer().transform_documents(docs)` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
139
141
  | Haystack | `pipe.add_component("sct", SCTDocumentCleaner())` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
140
142
  | Ray, Spark, Polars, DuckDB, dlt, Daft | `clean_columns(handle, batch)` | [RECIPES](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/RECIPES.md) |
@@ -145,6 +147,23 @@ if __name__ == "__main__":
145
147
 
146
148
  The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
147
149
 
150
+ ### Clean a document and produce chunks
151
+
152
+ Use `chunk_document()` to clean the full document before you split it for search, training, or export.
153
+
154
+ ```python
155
+ from sct import TextCleaner, TextCleanerConfig, chunk_document
156
+
157
+ cleaner = TextCleaner(TextCleanerConfig(check_ner_process=False))
158
+ result = chunk_document(cleaner, "Contact alice@example.com for details.", max_size=24)
159
+ texts = [chunk.text for chunk in result.chunks]
160
+ ```
161
+
162
+ SCT can use Markdown headings or cleaned document elements as boundaries. Install
163
+ `SqueakyCleanText[chunking]` for Semchunk and Chonkie, or register your own splitter.
164
+ In unified PII processing, you can also choose a chunker for each compatible detector.
165
+ See [chunking providers and routing](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CHUNKING.md).
166
+
148
167
  ## How it works
149
168
 
150
169
  <img src="https://raw.githubusercontent.com/rhnfzl/SqueakyCleanText/main/resources/sct_flow.png" alt="SqueakyCleanText pipeline: any entry point runs 1 detect language, 2 fix the text, 3 regex PII and cleanup, 4 NER, 5 replacement mode (placeholder tags are written in steps 3 and 4; synthetic and reversible swap the values in after NER), 6 output and signals. Dashed steps are off by default." width="900">
@@ -192,6 +211,62 @@ Deprecated config globals, `ner_models_list`, the worker `batch_size` alias and
192
211
  Modes, privacy policies and more PII types (`ner_mode="pii"`): [docs/REPLACEMENT_MODES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/REPLACEMENT_MODES.md), [docs/PRIVACY.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PRIVACY.md), [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md).
193
212
  <!-- --8<-- [end:results] -->
194
213
 
214
+ ## One policy across detectors (opt-in)
215
+
216
+ Unified processing lets you apply the same handling rules to format matches and model findings. Existing callers keep `pii_processing="legacy"` by default.
217
+
218
+ ```python
219
+ from sct import TextCleaner, TextCleanerConfig, UnifiedPrivacyPolicy, HandlingRule
220
+
221
+ cleaner = TextCleaner(
222
+ TextCleanerConfig(pii_processing="unified", check_ner_process=False),
223
+ unified_policy=UnifiedPrivacyPolicy("contact-policy", "1", {
224
+ "EMAIL": HandlingRule(action="redact"),
225
+ }),
226
+ )
227
+ result = cleaner.process("Contact alice@example.com")
228
+ assert "alice@example.com" not in result.lm_text
229
+ ```
230
+
231
+ All selected detectors inspect the same canonical text before PII replacement. `use_mode` follows the placeholder, synthetic or reversible mode. `keep` affects PII replacement only. `redact` emits `<REDACTED>` and never saves that value in the reversible map. Protection wins over overlapping keep decisions; redaction wins over other protection. Unknown scores protect by default, or fail with `unknown_score_action="error"`.
232
+
233
+ A nonempty `detector_specs` tuple selects the complete detector set from an application-owned `DetectorRegistry`. Empty specs select the configured built-in regex rules and native NER when `check_ner_process=True`. Native registry loading requires `allow_network_detectors=True` because model artifacts may need downloading. External-only selection does not initialize native NER. Separately requested document classification still runs and also requires network permission in unified mode. Legacy model-loading behavior stays unchanged.
234
+
235
+ `metadata["findings_v2"]` contains value-free canonical positions, original-input ranges, mapping quality and policy decisions. `counts_v2` separates detector hits after configured allow-term filtering from resolved replacements; `provenance_v2` records detector and policy identities. Legacy counters stay empty in unified mode. Unified signals add `version=2`, `counts_v2`, `coverage_v2` and `provenance_v2`. CLI/MCP summaries distinguish detector hits from rendered replacements. `coverage_v2` reports source coverage quality and lower/upper bounds. Non-exact coverage leaves `pii_chars` and `pii_share` as `None`; score summaries place these shares in an explicit unknown bucket. Only actual built-in regex formatting findings receive the non-PII format exclusion; native, external and caller-declared dates remain sensitive.
236
+
237
+ Source mappings use Unicode code-point offsets and report `exact`, `covering` or `unavailable`. Complex built-in repairs can cover the whole input. Opaque custom callbacks make mapping unavailable, even when they return unchanged text. `source_mapping="require_source"` rejects those callbacks. These ranges do not describe original PDF bytes.
238
+
239
+ `process(..., context=DetectionContext(...))`, `process_batch(..., contexts=...)` and `aprocess_batch(..., contexts=...)` accept caller context in unified mode. Batch contexts must match input count and order. Buffered streaming uses this whole-document path; windowed streaming and browser manifests reject unified mode.
240
+
241
+ `include_entities=True` explicitly adds sensitive `findings_v2_values`, linked by `finding_index` to canonical findings. This includes values whose output was redacted. Keep this audit payload and reversible maps out of logs. They are absent by default, apart from the map explicitly requested through reversible mode.
242
+
243
+ Custom callbacks need migration review. Legacy callbacks run after regex replacement and before NER.
244
+ Unified callbacks run before all detection, so callbacks that depend on `<EMAIL>` or other rendered markers receive different input.
245
+ See [configuration and rollback](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CONFIG.md#unified-processing-and-migration).
246
+
247
+ ## Caller-declared fields (opt-in)
248
+
249
+ Declare exact fields when a record needs protection beyond text detection. SCT does not infer meaning from field names.
250
+
251
+ ```python
252
+ from sct import FieldRule, FieldSchema, process_json
253
+
254
+ schema = FieldSchema("contacts", "1", (
255
+ FieldRule(("name",), ("PERSON",), "protect_value"),
256
+ ))
257
+ assert process_json(cleaner, {"name": "May", "status": "May"}, schema=schema).value == {
258
+ "name": "<PERSON>", "status": "May",
259
+ }
260
+ ```
261
+
262
+ Schemas require unified processing. Paths select original keys and list positions. Missing fields and `None` remain unchanged; selected containers fail. Whole-value protection also accepts finite numeric and boolean scalars. Default non-string handling stays unchanged. Protection uses the existing policy and replacement modes. Conflicting keep decisions, score errors, allow terms or disabled tags fail. Unknown scores with `unknown_score_action="protect"` remain compatible with thresholds. Explicit field types win equal-span ties after action priority.
263
+
264
+ Hints require providers that support context and the declared types. Presidio receives explicit declared types as context; ONNX-only providers reject required hints. DataFrame paths use `(column_label,)` across a column or `(row_label, column_label)` for a cell. Conflicting overlapping declarations fail. Framework paths use `(column_name,)` per record. RAG paths use `("text",)` or `("metadata", ...)`. Selected missing DataFrame cells remain unchanged. Masking numeric cells changes only the copied output column to object dtype. Unsupported path shapes fail.
265
+
266
+ Unified findings use positional references without original keys or labels. Schema-match paths and caller context stay out of signals and provenance. MCP full detail excludes sensitive value payloads. Framework constructors accept a trusted module-level `registry_factory`; workers pickle its reference and rebuild per process. Haystack data resolves only application-registered factory references. Protect serialized pipeline configuration, which can contain caller schema/context identifiers.
267
+
268
+ Applications register schemas through CLI `main(..., schemas=...)` or `create_mcp_server(..., schemas=...)`. Requests supply only exactly registered data-only schemas. CLI exposes `--processing unified --schema '<JSON>'`; `--signals` adds versioned counts and a summary. Requests cannot import Python factories. Existing adapter restrictions on reversible mode remain.
269
+
195
270
  ## Speed and hardware
196
271
 
197
272
  Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, on a workstation CPU and high-end GPUs, median of 3 runs. 0.8.1 runs at the same speed.
@@ -212,6 +287,10 @@ NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cl
212
287
 
213
288
  Measured on open datasets with [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks).
214
289
 
290
+ The [SCT v0.10.0 comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-12-v010/README.md)
291
+ adds fresh SCT results over 20,817 documents, six charts, and repeated CPU measurements.
292
+ Other-tool versions remain historical pins. Shared-label and full-label results differ because label coverage affects the comparison.
293
+
215
294
  The [SCT v0.8.2 comparison with other PII tools](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-04-visuals/README.md)
216
295
  shows paired recall, precision and exposure results, with uncertainty intervals.
217
296
  It measures PII configurations rather than the whole processing pipeline. The tested version is v0.8.2, not v0.8.3.
@@ -226,24 +305,45 @@ Runtime versions differ, so the speed results do not isolate SCT code changes.
226
305
  - **PII** (12 open PII datasets, about 40 languages): the 0.8.1 release branch leaked fewer values than 0.8.0 on every dataset, for example 18.1% to 8.2% on REDACT ([comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/comparison-0.8.0-unreleased.md), pre-release commit `2078e8c4`).
227
306
  - **Multilingual PII eval** (220 records, 11 languages): 0.93 recall, 0.95 precision ([docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md)).
228
307
 
308
+ **Dataset credits.** These scores use open datasets, most under CC BY or other attribution licences. Creators, sources, licences and citations are in [benchmarks/DATASET_CREDITS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/DATASET_CREDITS.md).
309
+
229
310
  Test your own data or model with the evaluation harness ([docs/EVALUATION.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/EVALUATION.md)).
230
311
 
231
312
  ## Languages
232
313
 
233
- Language detection covers 75 languages. Full NER models cover English, Dutch, German and Spanish. The other languages use the multilingual WikiNEural model with the English model, or a model you bring with `ner_models`. Full list: [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md).
314
+ Language detection covers 75 languages. Full NER models cover English, Dutch, German and Spanish. The other languages use the multilingual WikiNEural model with the English model, or a model you bring with `ner_models`. The WikiNEural model is CC BY-NC-SA 4.0, for non-commercial research only, and SCT loads it by default. [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md#model-licences) shows how to run without it. Full list: [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md).
234
315
 
235
316
  <!-- --8<-- [start:limits] -->
236
317
  ## Limits and design principles
237
318
 
238
319
  - **Safe defaults, every choice configurable.** Each behaviour is a config field; nothing is forced.
239
320
  - **No guarantee that all PII is found.** Models miss entities, mostly organisations and non-Latin scripts. Review output before you publish it.
240
- - **Free text, not tables.** For tables with known PII columns, drop or hash those columns first.
321
+ - **Declared fields need explicit rules.** Use a caller-owned schema to protect known PII fields; unselected fields remain unchanged.
241
322
  - **Reversible maps are personal data.** Anyone with the map can restore the values. Store maps apart from the cleaned data, with access control.
242
323
  - **Synthetic data is not anonymisation.** Fake values keep the data useful; they do not make it anonymous in a legal sense.
243
- - **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
324
+ - **Offsets name their text stage.** Legacy findings use post-regex positions. Unified findings use canonical positions and explicit source mapping quality.
244
325
  <!-- --8<-- [end:limits] -->
245
326
 
246
- Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
327
+ Detailed known limits: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
328
+
329
+ ## Opt-in document and corpus workflows
330
+
331
+ Users can combine extraction, unified detection, encrypted references and corpus selection without changing legacy defaults.
332
+ Document and framework adapters use non-reversible output; direct text processing can return encrypted map references.
333
+ The original documents and private corpus checkpoints remain caller-owned sensitive data.
334
+
335
+ | Capability | Entry point and limit |
336
+ |---|---|
337
+ | Encrypted references | `EncryptedSQLiteTokenStore` from `sct.token_stores`; keys and authorization remain application-owned |
338
+ | Corpus selection | `analyse_records` and `select_records` from `sct.corpus`; report-only by default, explicit basis required for removal |
339
+ | Native replay | `curate_resumable`; identity/checksum validation and private checkpoints, without exactly-once guarantees |
340
+ | PDF extraction | `extract_document` and `clean_document`; prepared local Docling assets, extracted-text protection only |
341
+ | Credential formats | Optional `secrets` detector; four syntactic categories, not credential validity |
342
+ | Language uncertainty | `language_policy='abstain'`; explicit thresholds and unknown-language action |
343
+ | Detector routing | `routing_policy='always_run'` stays default; calibrated cascade remains experimental |
344
+
345
+ See the [combined acceptance evidence](https://github.com/rhnfzl/SqueakyCleanText/blob/main/evaluation/improvements/ACCEPTANCE.md) for the tested profiles.
346
+ Trained Torch and ONNX scores agree on the parity controls. The synthetic gibberish control scored highest, so quality-filter accuracy remains unverified.
247
347
 
248
348
  ## Documentation
249
349
 
@@ -256,6 +356,7 @@ Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/Squea
256
356
  | JSON, DataFrame, RAG, CLI, streaming, OCR | [docs/ADAPTERS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/ADAPTERS.md) |
257
357
  | Score columns, quality, MinHash, data cards, `sct-report` | [docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md) |
258
358
  | NER backends, GLiNER, PII mode | [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md) |
359
+ | Encrypted local token maps | [docs/TOKEN_STORES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/TOKEN_STORES.md) |
259
360
  | Replacement modes, privacy policies, `TokenStore` | [docs/REPLACEMENT_MODES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/REPLACEMENT_MODES.md), [docs/PRIVACY.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PRIVACY.md) |
260
361
  | Evaluation | [docs/EVALUATION.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/EVALUATION.md), [docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md) |
261
362
  | Languages and adding a language | [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md), [docs/ADDING_LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/ADDING_LANGUAGES.md) |
@@ -275,7 +376,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
275
376
  author = {Fazal, Rehan},
276
377
  title = {SqueakyCleanText},
277
378
  url = {https://github.com/rhnfzl/SqueakyCleanText},
278
- version = {0.9.0},
379
+ version = {0.10.0},
279
380
  year = {2026}
280
381
  }
281
382
  ```
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SqueakyCleanText
3
- Version: 0.9.0
3
+ Version: 0.11.0
4
4
  Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
5
5
  Author: Rehan Fazal
6
6
  License-Expression: MIT
@@ -38,6 +38,20 @@ Requires-Dist: huggingface-hub>=1.5.0
38
38
  Requires-Dist: numpy>=2.2.0
39
39
  Requires-Dist: presidio_anonymizer>=2.2.360
40
40
  Requires-Dist: regex>=2024.11.6
41
+ Provides-Extra: secrets
42
+ Requires-Dist: datafog-core==1.0.0; extra == "secrets"
43
+ Provides-Extra: documents
44
+ Requires-Dist: docling==2.148.0; extra == "documents"
45
+ Provides-Extra: chunking
46
+ Requires-Dist: semchunk<4,>=3.2.5; extra == "chunking"
47
+ Requires-Dist: chonkie<2,>=1.7.0; extra == "chunking"
48
+ Provides-Extra: token-store
49
+ Requires-Dist: cryptography==48.0.1; extra == "token-store"
50
+ Provides-Extra: corpus
51
+ Requires-Dist: datasketch>=1.6; extra == "corpus"
52
+ Requires-Dist: datatrove==0.10.1; extra == "corpus"
53
+ Requires-Dist: orjson>=3.10; extra == "corpus"
54
+ Requires-Dist: spacy==3.8.16; extra == "corpus"
41
55
  Provides-Extra: gpu
42
56
  Requires-Dist: onnxruntime-gpu>=1.24.2; extra == "gpu"
43
57
  Provides-Extra: fuzzy
@@ -226,6 +240,8 @@ if __name__ == "__main__":
226
240
 
227
241
  | Where | Usage | Docs |
228
242
  |---|---|---|
243
+ | PDF extraction | `extract_document` and `clean_document` | [DOCUMENTS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/DOCUMENTS.md) |
244
+ | Clean and chunk a document | `chunk_document(cleaner, text, max_size=1024)` | [CHUNKING](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CHUNKING.md) |
229
245
  | LangChain | `SCTDocumentTransformer().transform_documents(docs)` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
230
246
  | Haystack | `pipe.add_component("sct", SCTDocumentCleaner())` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
231
247
  | Ray, Spark, Polars, DuckDB, dlt, Daft | `clean_columns(handle, batch)` | [RECIPES](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/RECIPES.md) |
@@ -236,6 +252,23 @@ if __name__ == "__main__":
236
252
 
237
253
  The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
238
254
 
255
+ ### Clean a document and produce chunks
256
+
257
+ Use `chunk_document()` to clean the full document before you split it for search, training, or export.
258
+
259
+ ```python
260
+ from sct import TextCleaner, TextCleanerConfig, chunk_document
261
+
262
+ cleaner = TextCleaner(TextCleanerConfig(check_ner_process=False))
263
+ result = chunk_document(cleaner, "Contact alice@example.com for details.", max_size=24)
264
+ texts = [chunk.text for chunk in result.chunks]
265
+ ```
266
+
267
+ SCT can use Markdown headings or cleaned document elements as boundaries. Install
268
+ `SqueakyCleanText[chunking]` for Semchunk and Chonkie, or register your own splitter.
269
+ In unified PII processing, you can also choose a chunker for each compatible detector.
270
+ See [chunking providers and routing](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CHUNKING.md).
271
+
239
272
  ## How it works
240
273
 
241
274
  <img src="https://raw.githubusercontent.com/rhnfzl/SqueakyCleanText/main/resources/sct_flow.png" alt="SqueakyCleanText pipeline: any entry point runs 1 detect language, 2 fix the text, 3 regex PII and cleanup, 4 NER, 5 replacement mode (placeholder tags are written in steps 3 and 4; synthetic and reversible swap the values in after NER), 6 output and signals. Dashed steps are off by default." width="900">
@@ -283,6 +316,62 @@ Deprecated config globals, `ner_models_list`, the worker `batch_size` alias and
283
316
  Modes, privacy policies and more PII types (`ner_mode="pii"`): [docs/REPLACEMENT_MODES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/REPLACEMENT_MODES.md), [docs/PRIVACY.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PRIVACY.md), [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md).
284
317
  <!-- --8<-- [end:results] -->
285
318
 
319
+ ## One policy across detectors (opt-in)
320
+
321
+ Unified processing lets you apply the same handling rules to format matches and model findings. Existing callers keep `pii_processing="legacy"` by default.
322
+
323
+ ```python
324
+ from sct import TextCleaner, TextCleanerConfig, UnifiedPrivacyPolicy, HandlingRule
325
+
326
+ cleaner = TextCleaner(
327
+ TextCleanerConfig(pii_processing="unified", check_ner_process=False),
328
+ unified_policy=UnifiedPrivacyPolicy("contact-policy", "1", {
329
+ "EMAIL": HandlingRule(action="redact"),
330
+ }),
331
+ )
332
+ result = cleaner.process("Contact alice@example.com")
333
+ assert "alice@example.com" not in result.lm_text
334
+ ```
335
+
336
+ All selected detectors inspect the same canonical text before PII replacement. `use_mode` follows the placeholder, synthetic or reversible mode. `keep` affects PII replacement only. `redact` emits `<REDACTED>` and never saves that value in the reversible map. Protection wins over overlapping keep decisions; redaction wins over other protection. Unknown scores protect by default, or fail with `unknown_score_action="error"`.
337
+
338
+ A nonempty `detector_specs` tuple selects the complete detector set from an application-owned `DetectorRegistry`. Empty specs select the configured built-in regex rules and native NER when `check_ner_process=True`. Native registry loading requires `allow_network_detectors=True` because model artifacts may need downloading. External-only selection does not initialize native NER. Separately requested document classification still runs and also requires network permission in unified mode. Legacy model-loading behavior stays unchanged.
339
+
340
+ `metadata["findings_v2"]` contains value-free canonical positions, original-input ranges, mapping quality and policy decisions. `counts_v2` separates detector hits after configured allow-term filtering from resolved replacements; `provenance_v2` records detector and policy identities. Legacy counters stay empty in unified mode. Unified signals add `version=2`, `counts_v2`, `coverage_v2` and `provenance_v2`. CLI/MCP summaries distinguish detector hits from rendered replacements. `coverage_v2` reports source coverage quality and lower/upper bounds. Non-exact coverage leaves `pii_chars` and `pii_share` as `None`; score summaries place these shares in an explicit unknown bucket. Only actual built-in regex formatting findings receive the non-PII format exclusion; native, external and caller-declared dates remain sensitive.
341
+
342
+ Source mappings use Unicode code-point offsets and report `exact`, `covering` or `unavailable`. Complex built-in repairs can cover the whole input. Opaque custom callbacks make mapping unavailable, even when they return unchanged text. `source_mapping="require_source"` rejects those callbacks. These ranges do not describe original PDF bytes.
343
+
344
+ `process(..., context=DetectionContext(...))`, `process_batch(..., contexts=...)` and `aprocess_batch(..., contexts=...)` accept caller context in unified mode. Batch contexts must match input count and order. Buffered streaming uses this whole-document path; windowed streaming and browser manifests reject unified mode.
345
+
346
+ `include_entities=True` explicitly adds sensitive `findings_v2_values`, linked by `finding_index` to canonical findings. This includes values whose output was redacted. Keep this audit payload and reversible maps out of logs. They are absent by default, apart from the map explicitly requested through reversible mode.
347
+
348
+ Custom callbacks need migration review. Legacy callbacks run after regex replacement and before NER.
349
+ Unified callbacks run before all detection, so callbacks that depend on `<EMAIL>` or other rendered markers receive different input.
350
+ See [configuration and rollback](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CONFIG.md#unified-processing-and-migration).
351
+
352
+ ## Caller-declared fields (opt-in)
353
+
354
+ Declare exact fields when a record needs protection beyond text detection. SCT does not infer meaning from field names.
355
+
356
+ ```python
357
+ from sct import FieldRule, FieldSchema, process_json
358
+
359
+ schema = FieldSchema("contacts", "1", (
360
+ FieldRule(("name",), ("PERSON",), "protect_value"),
361
+ ))
362
+ assert process_json(cleaner, {"name": "May", "status": "May"}, schema=schema).value == {
363
+ "name": "<PERSON>", "status": "May",
364
+ }
365
+ ```
366
+
367
+ Schemas require unified processing. Paths select original keys and list positions. Missing fields and `None` remain unchanged; selected containers fail. Whole-value protection also accepts finite numeric and boolean scalars. Default non-string handling stays unchanged. Protection uses the existing policy and replacement modes. Conflicting keep decisions, score errors, allow terms or disabled tags fail. Unknown scores with `unknown_score_action="protect"` remain compatible with thresholds. Explicit field types win equal-span ties after action priority.
368
+
369
+ Hints require providers that support context and the declared types. Presidio receives explicit declared types as context; ONNX-only providers reject required hints. DataFrame paths use `(column_label,)` across a column or `(row_label, column_label)` for a cell. Conflicting overlapping declarations fail. Framework paths use `(column_name,)` per record. RAG paths use `("text",)` or `("metadata", ...)`. Selected missing DataFrame cells remain unchanged. Masking numeric cells changes only the copied output column to object dtype. Unsupported path shapes fail.
370
+
371
+ Unified findings use positional references without original keys or labels. Schema-match paths and caller context stay out of signals and provenance. MCP full detail excludes sensitive value payloads. Framework constructors accept a trusted module-level `registry_factory`; workers pickle its reference and rebuild per process. Haystack data resolves only application-registered factory references. Protect serialized pipeline configuration, which can contain caller schema/context identifiers.
372
+
373
+ Applications register schemas through CLI `main(..., schemas=...)` or `create_mcp_server(..., schemas=...)`. Requests supply only exactly registered data-only schemas. CLI exposes `--processing unified --schema '<JSON>'`; `--signals` adds versioned counts and a summary. Requests cannot import Python factories. Existing adapter restrictions on reversible mode remain.
374
+
286
375
  ## Speed and hardware
287
376
 
288
377
  Default config (full multilingual NER ensemble), release benchmark documents (up to about 900 characters), SCT 0.8.0, on a workstation CPU and high-end GPUs, median of 3 runs. 0.8.1 runs at the same speed.
@@ -303,6 +392,10 @@ NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cl
303
392
 
304
393
  Measured on open datasets with [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks).
305
394
 
395
+ The [SCT v0.10.0 comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-12-v010/README.md)
396
+ adds fresh SCT results over 20,817 documents, six charts, and repeated CPU measurements.
397
+ Other-tool versions remain historical pins. Shared-label and full-label results differ because label coverage affects the comparison.
398
+
306
399
  The [SCT v0.8.2 comparison with other PII tools](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-04-visuals/README.md)
307
400
  shows paired recall, precision and exposure results, with uncertainty intervals.
308
401
  It measures PII configurations rather than the whole processing pipeline. The tested version is v0.8.2, not v0.8.3.
@@ -317,24 +410,45 @@ Runtime versions differ, so the speed results do not isolate SCT code changes.
317
410
  - **PII** (12 open PII datasets, about 40 languages): the 0.8.1 release branch leaked fewer values than 0.8.0 on every dataset, for example 18.1% to 8.2% on REDACT ([comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/comparison-0.8.0-unreleased.md), pre-release commit `2078e8c4`).
318
411
  - **Multilingual PII eval** (220 records, 11 languages): 0.93 recall, 0.95 precision ([docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md)).
319
412
 
413
+ **Dataset credits.** These scores use open datasets, most under CC BY or other attribution licences. Creators, sources, licences and citations are in [benchmarks/DATASET_CREDITS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/DATASET_CREDITS.md).
414
+
320
415
  Test your own data or model with the evaluation harness ([docs/EVALUATION.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/EVALUATION.md)).
321
416
 
322
417
  ## Languages
323
418
 
324
- Language detection covers 75 languages. Full NER models cover English, Dutch, German and Spanish. The other languages use the multilingual WikiNEural model with the English model, or a model you bring with `ner_models`. Full list: [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md).
419
+ Language detection covers 75 languages. Full NER models cover English, Dutch, German and Spanish. The other languages use the multilingual WikiNEural model with the English model, or a model you bring with `ner_models`. The WikiNEural model is CC BY-NC-SA 4.0, for non-commercial research only, and SCT loads it by default. [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md#model-licences) shows how to run without it. Full list: [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md).
325
420
 
326
421
  <!-- --8<-- [start:limits] -->
327
422
  ## Limits and design principles
328
423
 
329
424
  - **Safe defaults, every choice configurable.** Each behaviour is a config field; nothing is forced.
330
425
  - **No guarantee that all PII is found.** Models miss entities, mostly organisations and non-Latin scripts. Review output before you publish it.
331
- - **Free text, not tables.** For tables with known PII columns, drop or hash those columns first.
426
+ - **Declared fields need explicit rules.** Use a caller-owned schema to protect known PII fields; unselected fields remain unchanged.
332
427
  - **Reversible maps are personal data.** Anyone with the map can restore the values. Store maps apart from the cleaned data, with access control.
333
428
  - **Synthetic data is not anonymisation.** Fake values keep the data useful; they do not make it anonymous in a legal sense.
334
- - **Offsets are post-regex.** Finding offsets point into the text after the regex step, not into the raw input.
429
+ - **Offsets name their text stage.** Legacy findings use post-regex positions. Unified findings use canonical positions and explicit source mapping quality.
335
430
  <!-- --8<-- [end:limits] -->
336
431
 
337
- Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
432
+ Detailed known limits: [docs/limits.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/limits.md).
433
+
434
+ ## Opt-in document and corpus workflows
435
+
436
+ Users can combine extraction, unified detection, encrypted references and corpus selection without changing legacy defaults.
437
+ Document and framework adapters use non-reversible output; direct text processing can return encrypted map references.
438
+ The original documents and private corpus checkpoints remain caller-owned sensitive data.
439
+
440
+ | Capability | Entry point and limit |
441
+ |---|---|
442
+ | Encrypted references | `EncryptedSQLiteTokenStore` from `sct.token_stores`; keys and authorization remain application-owned |
443
+ | Corpus selection | `analyse_records` and `select_records` from `sct.corpus`; report-only by default, explicit basis required for removal |
444
+ | Native replay | `curate_resumable`; identity/checksum validation and private checkpoints, without exactly-once guarantees |
445
+ | PDF extraction | `extract_document` and `clean_document`; prepared local Docling assets, extracted-text protection only |
446
+ | Credential formats | Optional `secrets` detector; four syntactic categories, not credential validity |
447
+ | Language uncertainty | `language_policy='abstain'`; explicit thresholds and unknown-language action |
448
+ | Detector routing | `routing_policy='always_run'` stays default; calibrated cascade remains experimental |
449
+
450
+ See the [combined acceptance evidence](https://github.com/rhnfzl/SqueakyCleanText/blob/main/evaluation/improvements/ACCEPTANCE.md) for the tested profiles.
451
+ Trained Torch and ONNX scores agree on the parity controls. The synthetic gibberish control scored highest, so quality-filter accuracy remains unverified.
338
452
 
339
453
  ## Documentation
340
454
 
@@ -347,6 +461,7 @@ Detailed known limits of 0.9.0: [docs/limits.md](https://github.com/rhnfzl/Squea
347
461
  | JSON, DataFrame, RAG, CLI, streaming, OCR | [docs/ADAPTERS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/ADAPTERS.md) |
348
462
  | Score columns, quality, MinHash, data cards, `sct-report` | [docs/SCORE.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/SCORE.md) |
349
463
  | NER backends, GLiNER, PII mode | [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md) |
464
+ | Encrypted local token maps | [docs/TOKEN_STORES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/TOKEN_STORES.md) |
350
465
  | Replacement modes, privacy policies, `TokenStore` | [docs/REPLACEMENT_MODES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/REPLACEMENT_MODES.md), [docs/PRIVACY.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/PRIVACY.md) |
351
466
  | Evaluation | [docs/EVALUATION.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/EVALUATION.md), [docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md) |
352
467
  | Languages and adding a language | [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md), [docs/ADDING_LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/ADDING_LANGUAGES.md) |
@@ -366,7 +481,7 @@ Issues and pull requests are welcome. Read [AGENTS.md](https://github.com/rhnfzl
366
481
  author = {Fazal, Rehan},
367
482
  title = {SqueakyCleanText},
368
483
  url = {https://github.com/rhnfzl/SqueakyCleanText},
369
- version = {0.9.0},
484
+ version = {0.10.0},
370
485
  year = {2026}
371
486
  }
372
487
  ```