SqueakyCleanText 0.10.0__tar.gz → 0.11.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (191) hide show
  1. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/PKG-INFO +29 -2
  2. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/README.md +25 -1
  3. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/SqueakyCleanText.egg-info/PKG-INFO +29 -2
  4. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/SqueakyCleanText.egg-info/SOURCES.txt +6 -0
  5. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/SqueakyCleanText.egg-info/requires.txt +4 -0
  6. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/pyproject.toml +1 -0
  7. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/__init__.py +8 -2
  8. squeakycleantext-0.11.0/sct/chunked_document.py +127 -0
  9. squeakycleantext-0.11.0/sct/chunkers_builtin.py +97 -0
  10. squeakycleantext-0.11.0/sct/chunking.py +227 -0
  11. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/config.py +36 -3
  12. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/corpus/datatrove.py +6 -2
  13. squeakycleantext-0.11.0/sct/detector_windows.py +96 -0
  14. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/detectors.py +18 -0
  15. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/detectors_builtin.py +9 -0
  16. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/documents.py +13 -2
  17. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/findings.py +3 -0
  18. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/routing.py +6 -3
  19. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/sct.py +6 -0
  20. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/unified.py +24 -11
  21. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/ner.py +6 -5
  22. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/ner_chunking.py +21 -9
  23. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/ner_spans.py +7 -0
  24. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_benchmark_loaders.py +18 -0
  25. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_chunking.py +30 -0
  26. squeakycleantext-0.11.0/tests/test_chunking_router.py +238 -0
  27. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_detector_routing.py +4 -0
  28. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_ner_span_repair.py +24 -0
  29. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_pii_social_cards.py +4 -0
  30. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_plot_pii_comparison.py +2 -3
  31. squeakycleantext-0.11.0/tests/test_plot_pii_social_release.py +43 -0
  32. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/LICENSE +0 -0
  33. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/SqueakyCleanText.egg-info/dependency_links.txt +0 -0
  34. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/SqueakyCleanText.egg-info/entry_points.txt +0 -0
  35. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/SqueakyCleanText.egg-info/top_level.txt +0 -0
  36. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/adapters.py +0 -0
  37. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/browser.py +0 -0
  38. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/cli.py +0 -0
  39. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/coordinates.py +0 -0
  40. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/corpus/__init__.py +0 -0
  41. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/corpus/manifest.py +0 -0
  42. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/corpus/runner.py +0 -0
  43. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/corpus/selection.py +0 -0
  44. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/corpus/types.py +0 -0
  45. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/detectors_datafog.py +0 -0
  46. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/evaluation.py +0 -0
  47. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/integrations/__init__.py +0 -0
  48. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/integrations/_runtime.py +0 -0
  49. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/integrations/datatrove.py +0 -0
  50. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/integrations/docling.py +0 -0
  51. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/integrations/haystack.py +0 -0
  52. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/integrations/hf.py +0 -0
  53. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/integrations/langchain.py +0 -0
  54. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/language_policy.py +0 -0
  55. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/mcp_server.py +0 -0
  56. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/model_benchmark.py +0 -0
  57. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/model_candidates.py +0 -0
  58. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/ocr.py +0 -0
  59. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/policy_v2.py +0 -0
  60. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/presets.py +0 -0
  61. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/privacy.py +0 -0
  62. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/py.typed +0 -0
  63. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/report_cli.py +0 -0
  64. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/risk.py +0 -0
  65. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/schema.py +0 -0
  66. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/score.py +0 -0
  67. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/signals.py +0 -0
  68. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/streaming.py +0 -0
  69. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/token_stores/__init__.py +0 -0
  70. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/token_stores/sqlite.py +0 -0
  71. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/__init__.py +0 -0
  72. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/anonymization_map.py +0 -0
  73. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/atomic_file.py +0 -0
  74. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/constants.py +0 -0
  75. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/contact.py +0 -0
  76. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/datetime.py +0 -0
  77. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/gliclass_adapter.py +0 -0
  78. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/gliner_adapter.py +0 -0
  79. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/markers.py +0 -0
  80. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/normtext.py +0 -0
  81. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/onnx_pipeline.py +0 -0
  82. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/process_result.py +0 -0
  83. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/resources.py +0 -0
  84. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/special.py +0 -0
  85. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/stopwords.py +0 -0
  86. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/synthetic.py +0 -0
  87. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/torch_pipeline.py +0 -0
  88. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/setup.cfg +0 -0
  89. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_adapters.py +0 -0
  90. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_address_leftovers.py +0 -0
  91. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_atomic_file.py +0 -0
  92. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_baseline_measurement.py +0 -0
  93. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_batching.py +0 -0
  94. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_benchmark_adapters.py +0 -0
  95. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_benchmark_release.py +0 -0
  96. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_benchmarks_compare.py +0 -0
  97. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_benchmarks_round2.py +0 -0
  98. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_benchmarks_round2_tools.py +0 -0
  99. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_benchmarks_score.py +0 -0
  100. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_browser.py +0 -0
  101. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_builtin_detections.py +0 -0
  102. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_cjk_width.py +0 -0
  103. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_cleaner_lifecycle.py +0 -0
  104. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_cli.py +0 -0
  105. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_config_state.py +0 -0
  106. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_contracts.py +0 -0
  107. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_coordinates.py +0 -0
  108. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_corpus_decontamination.py +0 -0
  109. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_corpus_failure_boundaries.py +0 -0
  110. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_corpus_identity_boundaries.py +0 -0
  111. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_corpus_manifest.py +0 -0
  112. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_corpus_resume.py +0 -0
  113. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_corpus_selection.py +0 -0
  114. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_date_forms.py +0 -0
  115. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_dates_cjk_styled.py +0 -0
  116. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_dates_default.py +0 -0
  117. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_detections.py +0 -0
  118. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_detector_registry.py +0 -0
  119. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_device_provenance.py +0 -0
  120. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_docling_integration.py +0 -0
  121. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_email_forms.py +0 -0
  122. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_escaped_markup.py +0 -0
  123. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_evaluation.py +0 -0
  124. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_fork_safety.py +0 -0
  125. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_functional.py +0 -0
  126. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_fuzz_styled_pii.py +0 -0
  127. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_gliner_recipe_identity.py +0 -0
  128. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_gliner_revision.py +0 -0
  129. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_improvements_end_to_end.py +0 -0
  130. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_integrations_frameworks.py +0 -0
  131. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_integrations_hf.py +0 -0
  132. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_integrations_runtime.py +0 -0
  133. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_keep_scripts.py +0 -0
  134. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_language_all.py +0 -0
  135. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_language_policy.py +0 -0
  136. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_linux_workers.py +0 -0
  137. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_mcp_path.py +0 -0
  138. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_mcp_server.py +0 -0
  139. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_model_benchmarks.py +0 -0
  140. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_multilingual_eval_set.py +0 -0
  141. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_ner_sees_tags.py +0 -0
  142. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_ner_spans.py +0 -0
  143. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_ner_windows.py +0 -0
  144. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_number_identifiers.py +0 -0
  145. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_ocr.py +0 -0
  146. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_onnx_concurrency.py +0 -0
  147. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_onnx_external_data.py +0 -0
  148. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_onnx_model_cache.py +0 -0
  149. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_optional_profiles.py +0 -0
  150. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_phone_forms.py +0 -0
  151. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_pipeline_text_steps.py +0 -0
  152. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_plot_version_comparison.py +0 -0
  153. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_predicted_spans_monotone.py +0 -0
  154. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_presidio_tags_and_fakes.py +0 -0
  155. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_presidio_type_names.py +0 -0
  156. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_privacy.py +0 -0
  157. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_processing_compatibility.py +0 -0
  158. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_prose_punctuation_kept.py +0 -0
  159. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_public_view.py +0 -0
  160. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_quality_export.py +0 -0
  161. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_regex_boundaries.py +0 -0
  162. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_regex_counts.py +0 -0
  163. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_release_comparison_watcher.py +0 -0
  164. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_repo_scripts.py +0 -0
  165. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_report_cli.py +0 -0
  166. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_reversible_markers.py +0 -0
  167. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_risk.py +0 -0
  168. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_routing_audit.py +0 -0
  169. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_routing_calibration.py +0 -0
  170. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_schema_context.py +0 -0
  171. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_score.py +0 -0
  172. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_sct.py +0 -0
  173. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_secret_detector.py +0 -0
  174. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_security_leftovers.py +0 -0
  175. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_sqlite_token_store.py +0 -0
  176. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_stream_guard_shared_view.py +0 -0
  177. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_stream_keeps_values_whole.py +0 -0
  178. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_streaming.py +0 -0
  179. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_streaming_split_stability.py +0 -0
  180. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_styled_pii.py +0 -0
  181. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_surface_contracts.py +0 -0
  182. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_synthetic_determinism.py +0 -0
  183. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_synthetic_per_document.py +0 -0
  184. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_token_store_failure_boundaries.py +0 -0
  185. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_token_store_operations.py +0 -0
  186. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_torch_ner_backend.py +0 -0
  187. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_training_preset.py +0 -0
  188. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_type_names.py +0 -0
  189. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_unified_processing.py +0 -0
  190. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_url_forms.py +0 -0
  191. {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_user_terms.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SqueakyCleanText
3
- Version: 0.10.0
3
+ Version: 0.11.0
4
4
  Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
5
5
  Author: Rehan Fazal
6
6
  License-Expression: MIT
@@ -42,6 +42,9 @@ Provides-Extra: secrets
42
42
  Requires-Dist: datafog-core==1.0.0; extra == "secrets"
43
43
  Provides-Extra: documents
44
44
  Requires-Dist: docling==2.148.0; extra == "documents"
45
+ Provides-Extra: chunking
46
+ Requires-Dist: semchunk<4,>=3.2.5; extra == "chunking"
47
+ Requires-Dist: chonkie<2,>=1.7.0; extra == "chunking"
45
48
  Provides-Extra: token-store
46
49
  Requires-Dist: cryptography==48.0.1; extra == "token-store"
47
50
  Provides-Extra: corpus
@@ -238,6 +241,7 @@ if __name__ == "__main__":
238
241
  | Where | Usage | Docs |
239
242
  |---|---|---|
240
243
  | PDF extraction | `extract_document` and `clean_document` | [DOCUMENTS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/DOCUMENTS.md) |
244
+ | Clean and chunk a document | `chunk_document(cleaner, text, max_size=1024)` | [CHUNKING](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CHUNKING.md) |
241
245
  | LangChain | `SCTDocumentTransformer().transform_documents(docs)` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
242
246
  | Haystack | `pipe.add_component("sct", SCTDocumentCleaner())` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
243
247
  | Ray, Spark, Polars, DuckDB, dlt, Daft | `clean_columns(handle, batch)` | [RECIPES](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/RECIPES.md) |
@@ -248,6 +252,23 @@ if __name__ == "__main__":
248
252
 
249
253
  The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
250
254
 
255
+ ### Clean a document and produce chunks
256
+
257
+ Use `chunk_document()` to clean the full document before you split it for search, training, or export.
258
+
259
+ ```python
260
+ from sct import TextCleaner, TextCleanerConfig, chunk_document
261
+
262
+ cleaner = TextCleaner(TextCleanerConfig(check_ner_process=False))
263
+ result = chunk_document(cleaner, "Contact alice@example.com for details.", max_size=24)
264
+ texts = [chunk.text for chunk in result.chunks]
265
+ ```
266
+
267
+ SCT can use Markdown headings or cleaned document elements as boundaries. Install
268
+ `SqueakyCleanText[chunking]` for Semchunk and Chonkie, or register your own splitter.
269
+ In unified PII processing, you can also choose a chunker for each compatible detector.
270
+ See [chunking providers and routing](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CHUNKING.md).
271
+
251
272
  ## How it works
252
273
 
253
274
  <img src="https://raw.githubusercontent.com/rhnfzl/SqueakyCleanText/main/resources/sct_flow.png" alt="SqueakyCleanText pipeline: any entry point runs 1 detect language, 2 fix the text, 3 regex PII and cleanup, 4 NER, 5 replacement mode (placeholder tags are written in steps 3 and 4; synthetic and reversible swap the values in after NER), 6 output and signals. Dashed steps are off by default." width="900">
@@ -371,6 +392,10 @@ NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cl
371
392
 
372
393
  Measured on open datasets with [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks).
373
394
 
395
+ The [SCT v0.10.0 comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-12-v010/README.md)
396
+ adds fresh SCT results over 20,817 documents, six charts, and repeated CPU measurements.
397
+ Other-tool versions remain historical pins. Shared-label and full-label results differ because label coverage affects the comparison.
398
+
374
399
  The [SCT v0.8.2 comparison with other PII tools](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-04-visuals/README.md)
375
400
  shows paired recall, precision and exposure results, with uncertainty intervals.
376
401
  It measures PII configurations rather than the whole processing pipeline. The tested version is v0.8.2, not v0.8.3.
@@ -385,11 +410,13 @@ Runtime versions differ, so the speed results do not isolate SCT code changes.
385
410
  - **PII** (12 open PII datasets, about 40 languages): the 0.8.1 release branch leaked fewer values than 0.8.0 on every dataset, for example 18.1% to 8.2% on REDACT ([comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/comparison-0.8.0-unreleased.md), pre-release commit `2078e8c4`).
386
411
  - **Multilingual PII eval** (220 records, 11 languages): 0.93 recall, 0.95 precision ([docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md)).
387
412
 
413
+ **Dataset credits.** These scores use open datasets, most under CC BY or other attribution licences. Creators, sources, licences and citations are in [benchmarks/DATASET_CREDITS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/DATASET_CREDITS.md).
414
+
388
415
  Test your own data or model with the evaluation harness ([docs/EVALUATION.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/EVALUATION.md)).
389
416
 
390
417
  ## Languages
391
418
 
392
- Language detection covers 75 languages. Full NER models cover English, Dutch, German and Spanish. The other languages use the multilingual WikiNEural model with the English model, or a model you bring with `ner_models`. Full list: [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md).
419
+ Language detection covers 75 languages. Full NER models cover English, Dutch, German and Spanish. The other languages use the multilingual WikiNEural model with the English model, or a model you bring with `ner_models`. The WikiNEural model is CC BY-NC-SA 4.0, for non-commercial research only, and SCT loads it by default. [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md#model-licences) shows how to run without it. Full list: [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md).
393
420
 
394
421
  <!-- --8<-- [start:limits] -->
395
422
  ## Limits and design principles
@@ -136,6 +136,7 @@ if __name__ == "__main__":
136
136
  | Where | Usage | Docs |
137
137
  |---|---|---|
138
138
  | PDF extraction | `extract_document` and `clean_document` | [DOCUMENTS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/DOCUMENTS.md) |
139
+ | Clean and chunk a document | `chunk_document(cleaner, text, max_size=1024)` | [CHUNKING](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CHUNKING.md) |
139
140
  | LangChain | `SCTDocumentTransformer().transform_documents(docs)` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
140
141
  | Haystack | `pipe.add_component("sct", SCTDocumentCleaner())` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
141
142
  | Ray, Spark, Polars, DuckDB, dlt, Daft | `clean_columns(handle, batch)` | [RECIPES](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/RECIPES.md) |
@@ -146,6 +147,23 @@ if __name__ == "__main__":
146
147
 
147
148
  The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
148
149
 
150
+ ### Clean a document and produce chunks
151
+
152
+ Use `chunk_document()` to clean the full document before you split it for search, training, or export.
153
+
154
+ ```python
155
+ from sct import TextCleaner, TextCleanerConfig, chunk_document
156
+
157
+ cleaner = TextCleaner(TextCleanerConfig(check_ner_process=False))
158
+ result = chunk_document(cleaner, "Contact alice@example.com for details.", max_size=24)
159
+ texts = [chunk.text for chunk in result.chunks]
160
+ ```
161
+
162
+ SCT can use Markdown headings or cleaned document elements as boundaries. Install
163
+ `SqueakyCleanText[chunking]` for Semchunk and Chonkie, or register your own splitter.
164
+ In unified PII processing, you can also choose a chunker for each compatible detector.
165
+ See [chunking providers and routing](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CHUNKING.md).
166
+
149
167
  ## How it works
150
168
 
151
169
  <img src="https://raw.githubusercontent.com/rhnfzl/SqueakyCleanText/main/resources/sct_flow.png" alt="SqueakyCleanText pipeline: any entry point runs 1 detect language, 2 fix the text, 3 regex PII and cleanup, 4 NER, 5 replacement mode (placeholder tags are written in steps 3 and 4; synthetic and reversible swap the values in after NER), 6 output and signals. Dashed steps are off by default." width="900">
@@ -269,6 +287,10 @@ NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cl
269
287
 
270
288
  Measured on open datasets with [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks).
271
289
 
290
+ The [SCT v0.10.0 comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-12-v010/README.md)
291
+ adds fresh SCT results over 20,817 documents, six charts, and repeated CPU measurements.
292
+ Other-tool versions remain historical pins. Shared-label and full-label results differ because label coverage affects the comparison.
293
+
272
294
  The [SCT v0.8.2 comparison with other PII tools](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-04-visuals/README.md)
273
295
  shows paired recall, precision and exposure results, with uncertainty intervals.
274
296
  It measures PII configurations rather than the whole processing pipeline. The tested version is v0.8.2, not v0.8.3.
@@ -283,11 +305,13 @@ Runtime versions differ, so the speed results do not isolate SCT code changes.
283
305
  - **PII** (12 open PII datasets, about 40 languages): the 0.8.1 release branch leaked fewer values than 0.8.0 on every dataset, for example 18.1% to 8.2% on REDACT ([comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/comparison-0.8.0-unreleased.md), pre-release commit `2078e8c4`).
284
306
  - **Multilingual PII eval** (220 records, 11 languages): 0.93 recall, 0.95 precision ([docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md)).
285
307
 
308
+ **Dataset credits.** These scores use open datasets, most under CC BY or other attribution licences. Creators, sources, licences and citations are in [benchmarks/DATASET_CREDITS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/DATASET_CREDITS.md).
309
+
286
310
  Test your own data or model with the evaluation harness ([docs/EVALUATION.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/EVALUATION.md)).
287
311
 
288
312
  ## Languages
289
313
 
290
- Language detection covers 75 languages. Full NER models cover English, Dutch, German and Spanish. The other languages use the multilingual WikiNEural model with the English model, or a model you bring with `ner_models`. Full list: [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md).
314
+ Language detection covers 75 languages. Full NER models cover English, Dutch, German and Spanish. The other languages use the multilingual WikiNEural model with the English model, or a model you bring with `ner_models`. The WikiNEural model is CC BY-NC-SA 4.0, for non-commercial research only, and SCT loads it by default. [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md#model-licences) shows how to run without it. Full list: [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md).
291
315
 
292
316
  <!-- --8<-- [start:limits] -->
293
317
  ## Limits and design principles
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SqueakyCleanText
3
- Version: 0.10.0
3
+ Version: 0.11.0
4
4
  Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
5
5
  Author: Rehan Fazal
6
6
  License-Expression: MIT
@@ -42,6 +42,9 @@ Provides-Extra: secrets
42
42
  Requires-Dist: datafog-core==1.0.0; extra == "secrets"
43
43
  Provides-Extra: documents
44
44
  Requires-Dist: docling==2.148.0; extra == "documents"
45
+ Provides-Extra: chunking
46
+ Requires-Dist: semchunk<4,>=3.2.5; extra == "chunking"
47
+ Requires-Dist: chonkie<2,>=1.7.0; extra == "chunking"
45
48
  Provides-Extra: token-store
46
49
  Requires-Dist: cryptography==48.0.1; extra == "token-store"
47
50
  Provides-Extra: corpus
@@ -238,6 +241,7 @@ if __name__ == "__main__":
238
241
  | Where | Usage | Docs |
239
242
  |---|---|---|
240
243
  | PDF extraction | `extract_document` and `clean_document` | [DOCUMENTS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/DOCUMENTS.md) |
244
+ | Clean and chunk a document | `chunk_document(cleaner, text, max_size=1024)` | [CHUNKING](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CHUNKING.md) |
241
245
  | LangChain | `SCTDocumentTransformer().transform_documents(docs)` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
242
246
  | Haystack | `pipe.add_component("sct", SCTDocumentCleaner())` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
243
247
  | Ray, Spark, Polars, DuckDB, dlt, Daft | `clean_columns(handle, batch)` | [RECIPES](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/RECIPES.md) |
@@ -248,6 +252,23 @@ if __name__ == "__main__":
248
252
 
249
253
  The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
250
254
 
255
+ ### Clean a document and produce chunks
256
+
257
+ Use `chunk_document()` to clean the full document before you split it for search, training, or export.
258
+
259
+ ```python
260
+ from sct import TextCleaner, TextCleanerConfig, chunk_document
261
+
262
+ cleaner = TextCleaner(TextCleanerConfig(check_ner_process=False))
263
+ result = chunk_document(cleaner, "Contact alice@example.com for details.", max_size=24)
264
+ texts = [chunk.text for chunk in result.chunks]
265
+ ```
266
+
267
+ SCT can use Markdown headings or cleaned document elements as boundaries. Install
268
+ `SqueakyCleanText[chunking]` for Semchunk and Chonkie, or register your own splitter.
269
+ In unified PII processing, you can also choose a chunker for each compatible detector.
270
+ See [chunking providers and routing](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CHUNKING.md).
271
+
251
272
  ## How it works
252
273
 
253
274
  <img src="https://raw.githubusercontent.com/rhnfzl/SqueakyCleanText/main/resources/sct_flow.png" alt="SqueakyCleanText pipeline: any entry point runs 1 detect language, 2 fix the text, 3 regex PII and cleanup, 4 NER, 5 replacement mode (placeholder tags are written in steps 3 and 4; synthetic and reversible swap the values in after NER), 6 output and signals. Dashed steps are off by default." width="900">
@@ -371,6 +392,10 @@ NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cl
371
392
 
372
393
  Measured on open datasets with [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks).
373
394
 
395
+ The [SCT v0.10.0 comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-12-v010/README.md)
396
+ adds fresh SCT results over 20,817 documents, six charts, and repeated CPU measurements.
397
+ Other-tool versions remain historical pins. Shared-label and full-label results differ because label coverage affects the comparison.
398
+
374
399
  The [SCT v0.8.2 comparison with other PII tools](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-04-visuals/README.md)
375
400
  shows paired recall, precision and exposure results, with uncertainty intervals.
376
401
  It measures PII configurations rather than the whole processing pipeline. The tested version is v0.8.2, not v0.8.3.
@@ -385,11 +410,13 @@ Runtime versions differ, so the speed results do not isolate SCT code changes.
385
410
  - **PII** (12 open PII datasets, about 40 languages): the 0.8.1 release branch leaked fewer values than 0.8.0 on every dataset, for example 18.1% to 8.2% on REDACT ([comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/comparison-0.8.0-unreleased.md), pre-release commit `2078e8c4`).
386
411
  - **Multilingual PII eval** (220 records, 11 languages): 0.93 recall, 0.95 precision ([docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md)).
387
412
 
413
+ **Dataset credits.** These scores use open datasets, most under CC BY or other attribution licences. Creators, sources, licences and citations are in [benchmarks/DATASET_CREDITS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/DATASET_CREDITS.md).
414
+
388
415
  Test your own data or model with the evaluation harness ([docs/EVALUATION.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/EVALUATION.md)).
389
416
 
390
417
  ## Languages
391
418
 
392
- Language detection covers 75 languages. Full NER models cover English, Dutch, German and Spanish. The other languages use the multilingual WikiNEural model with the English model, or a model you bring with `ner_models`. Full list: [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md).
419
+ Language detection covers 75 languages. Full NER models cover English, Dutch, German and Spanish. The other languages use the multilingual WikiNEural model with the English model, or a model you bring with `ner_models`. The WikiNEural model is CC BY-NC-SA 4.0, for non-commercial research only, and SCT loads it by default. [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md#model-licences) shows how to run without it. Full list: [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md).
393
420
 
394
421
  <!-- --8<-- [start:limits] -->
395
422
  ## Limits and design principles
@@ -10,9 +10,13 @@ SqueakyCleanText.egg-info/top_level.txt
10
10
  sct/__init__.py
11
11
  sct/adapters.py
12
12
  sct/browser.py
13
+ sct/chunked_document.py
14
+ sct/chunkers_builtin.py
15
+ sct/chunking.py
13
16
  sct/cli.py
14
17
  sct/config.py
15
18
  sct/coordinates.py
19
+ sct/detector_windows.py
16
20
  sct/detectors.py
17
21
  sct/detectors_builtin.py
18
22
  sct/detectors_datafog.py
@@ -87,6 +91,7 @@ tests/test_benchmarks_score.py
87
91
  tests/test_browser.py
88
92
  tests/test_builtin_detections.py
89
93
  tests/test_chunking.py
94
+ tests/test_chunking_router.py
90
95
  tests/test_cjk_width.py
91
96
  tests/test_cleaner_lifecycle.py
92
97
  tests/test_cli.py
@@ -141,6 +146,7 @@ tests/test_phone_forms.py
141
146
  tests/test_pii_social_cards.py
142
147
  tests/test_pipeline_text_steps.py
143
148
  tests/test_plot_pii_comparison.py
149
+ tests/test_plot_pii_social_release.py
144
150
  tests/test_plot_version_comparison.py
145
151
  tests/test_predicted_spans_monotone.py
146
152
  tests/test_presidio_tags_and_fakes.py
@@ -13,6 +13,10 @@ regex>=2024.11.6
13
13
  [all-ner]
14
14
  SqueakyCleanText[gliner,gliner2,torch]
15
15
 
16
+ [chunking]
17
+ semchunk<4,>=3.2.5
18
+ chonkie<2,>=1.7.0
19
+
16
20
  [classify]
17
21
  gliclass>=0.1.16
18
22
 
@@ -60,6 +60,7 @@ sct-report = "sct.report_cli:main"
60
60
  [project.optional-dependencies]
61
61
  secrets = ["datafog-core==1.0.0"]
62
62
  documents = ["docling==2.148.0"]
63
+ chunking = ["semchunk>=3.2.5,<4", "chonkie>=1.7.0,<2"]
63
64
  token-store = ["cryptography==48.0.1"]
64
65
  corpus = ["datasketch>=1.6", "datatrove==0.10.1", "orjson>=3.10", "spacy==3.8.16"]
65
66
  gpu = [
@@ -1,7 +1,11 @@
1
1
  """SqueakyCleanText - Text cleaning and preprocessing pipeline for ML/NLP."""
2
2
 
3
+ from sct.chunked_document import ChunkedDocument, CleanedChunk, chunk_document
4
+ from sct.chunking import ChunkBudgetError, ChunkRequest, Chunker, ChunkerRegistry, ChunkingPolicy, ChunkingRule
3
5
  from sct.config import TextCleanerConfig, PII_LABELS, PII_LABEL_MAP
4
- from sct.detectors import Detector, DetectorCapabilities, DetectorRegistry, DetectorSpec, DetectorUnavailable
6
+ from sct.detectors import (
7
+ Detector, DetectorCapabilities, DetectorRegistry, DetectorSpec, DetectorUnavailable, DetectorWindowLimits,
8
+ )
5
9
  from sct.policy_v2 import UnifiedPrivacyPolicy, HandlingRule, FindingDecision
6
10
  from sct.documents import (
7
11
  DocumentElement, DocumentExtractor, ExtractedDocument, ProcessedDocument, clean_document, extract_document,
@@ -59,9 +63,11 @@ from sct.streaming import (
59
63
  from sct.utils.anonymization_map import AnonymizationMap, MapEntry
60
64
  from sct.utils.process_result import ProcessResult
61
65
 
62
- __version__ = "0.10.0"
66
+ __version__ = "0.11.0"
63
67
  __all__ = [
64
68
  "DocumentElement", "DocumentExtractor", "ExtractedDocument", "ProcessedDocument", "clean_document", "extract_document",
69
+ "ChunkBudgetError", "ChunkRequest", "Chunker", "ChunkerRegistry", "ChunkingPolicy", "ChunkingRule",
70
+ "ChunkedDocument", "CleanedChunk", "chunk_document", "DetectorWindowLimits",
65
71
  "UnifiedPrivacyPolicy", "HandlingRule", "FindingDecision", "Detection", "DetectionContext", "TextRange",
66
72
  "Detector", "DetectorCapabilities", "DetectorRegistry", "DetectorSpec", "DetectorUnavailable",
67
73
  "TextCleaner", "TextCleanerConfig", "CustomPatternTimeout", "ModelLoadError", "PRESETS", "preset_config",
@@ -0,0 +1,127 @@
1
+ """Clean an entire document and split only the cleaned result for downstream use."""
2
+ from __future__ import annotations
3
+
4
+ from bisect import bisect_left, bisect_right
5
+ from collections.abc import Callable, Iterable, Mapping
6
+ from dataclasses import dataclass, field, replace
7
+ import re
8
+ from typing import TYPE_CHECKING
9
+
10
+ from sct.chunking import ChunkRequest, ChunkingPolicy, ChunkingRule
11
+ from sct.documents import DocumentElement, ExtractedDocument, _clean_document
12
+ from sct.findings import DetectionContext, TextRange
13
+ from sct.utils.markers import PLACEHOLDER_TOKEN
14
+
15
+ if TYPE_CHECKING:
16
+ from sct.sct import TextCleaner
17
+ from sct.unified import RenderedEdit
18
+
19
+
20
+ @dataclass(frozen=True)
21
+ class CleanedChunk:
22
+ """A chunk and its half-open Unicode range in the cleaned document."""
23
+
24
+ text: str = field(repr=False)
25
+ range: TextRange
26
+ element_ids: tuple[str, ...] = ()
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class ChunkedDocument:
31
+ chunks: tuple[CleanedChunk, ...] = field(repr=False)
32
+ chunker: str
33
+ language: str | None
34
+
35
+
36
+ OUTPUT_CHUNKING = ChunkingPolicy(
37
+ 'auto', candidates=('text',), rules=(ChunkingRule('markdown', formats=('markdown', 'text/markdown')),),
38
+ )
39
+
40
+
41
+ def _element_ranges(elements: tuple[DocumentElement, ...]) -> tuple[tuple[TextRange, str], ...]:
42
+ start = 0
43
+ result = []
44
+ for item in elements:
45
+ end = start + len(item.text)
46
+ if end > start:
47
+ result.append((TextRange(start, end), item.id))
48
+ start = end + len(item.separator)
49
+ return tuple(result)
50
+
51
+
52
+ def _protected(text: str, edits: Iterable[tuple[int, int]], custom: tuple[str, ...]) -> tuple[TextRange, ...]:
53
+ boundaries = sorted((
54
+ *edits,
55
+ *(match.span() for match in PLACEHOLDER_TOKEN.finditer(text)),
56
+ *(match.span(1) for marker in custom for match in re.finditer(rf'(?=({re.escape(marker)}))', text)),
57
+ ))
58
+ spans: list[TextRange] = []
59
+ for start, end in boundaries:
60
+ if start == end:
61
+ continue
62
+ if spans and spans[-1].end > start:
63
+ prior = spans[-1]
64
+ spans[-1] = TextRange(prior.start, max(prior.end, end))
65
+ else:
66
+ spans.append(TextRange(start, end))
67
+ return tuple(spans)
68
+
69
+
70
+ def chunk_document(
71
+ cleaner: TextCleaner, document: str | ExtractedDocument, *,
72
+ chunking: ChunkingPolicy | str | Mapping | None = None,
73
+ max_size: int | None = None, count: Callable[[str], int] = len,
74
+ document_format: str | None = None, context: DetectionContext | None = None,
75
+ ) -> ChunkedDocument:
76
+ """Apply the cleaner once, then return chunks in the selected size units."""
77
+ if cleaner.cfg.replacement_mode in ('indexed', 'reversible'):
78
+ raise ValueError('Chunked output does not support indexed or reversible modes')
79
+ if cleaner.cfg.replacement_mode == 'synthetic' and cleaner.cfg.pii_processing != 'unified':
80
+ raise ValueError('Chunked synthetic output requires unified processing')
81
+ if context is not None and not isinstance(context, DetectionContext):
82
+ raise ValueError('Expected DetectionContext')
83
+ context = context or DetectionContext()
84
+ if document_format is not None:
85
+ context = replace(context, document_format=document_format)
86
+ document_format = context.document_format
87
+ policy = ChunkingPolicy.coerce(chunking) or cleaner.cfg.output_chunking
88
+ edits: tuple[RenderedEdit, ...] = ()
89
+ elements: tuple[tuple[TextRange, str], ...] = ()
90
+ if type(document) is ExtractedDocument:
91
+ if document_format is None:
92
+ document_format = 'document'
93
+ cleaned, processed = _clean_document(document, cleaner, context)
94
+ elements = _element_ranges(cleaned.elements)
95
+ result = processed.result
96
+ edits = processed.rendered_edits
97
+ policy = policy or ChunkingPolicy('elements')
98
+ elif isinstance(document, str):
99
+ if cleaner.cfg.pii_processing == 'unified':
100
+ processed = cleaner._process_unified_document(document, context)
101
+ result, edits = processed.result, processed.rendered_edits
102
+ else:
103
+ if replace(context, document_format=None) != DetectionContext():
104
+ raise ValueError('Detection context requires unified processing')
105
+ result = cleaner.process(document)
106
+ policy = policy or OUTPUT_CHUNKING
107
+ else:
108
+ raise ValueError('Expected a string or ExtractedDocument')
109
+ text = result.lm_text
110
+ ranges = tuple(span for span, _ in elements)
111
+ protected = _protected(text, ((edit.start, edit.end) for edit in edits), cleaner._custom_markers)
112
+ budget = cleaner.cfg.output_chunk_max_size if max_size is None else max_size
113
+ request = ChunkRequest(text, budget, count, 'output', result.language, document_format,
114
+ ranges, cleaner.cfg.allow_network_chunkers, protected)
115
+ registry = cleaner._chunker_registry
116
+ name = registry.resolve(policy, stage='output', language=result.language, document_format=document_format,
117
+ length=len(text), allow_network=request.allow_network)
118
+ if name == 'native':
119
+ name = 'elements' if type(document) is ExtractedDocument else 'text'
120
+ chosen = registry.plan(name, request) if text else ()
121
+ starts, ends = tuple(r.start for r in ranges), tuple(r.end for r in ranges)
122
+ return ChunkedDocument(tuple(
123
+ CleanedChunk(text[span.start:span.end], span,
124
+ tuple(identifier for _, identifier in elements[
125
+ bisect_right(ends, span.start):bisect_left(starts, span.end)]))
126
+ for span in chosen
127
+ ), name, result.language)
@@ -0,0 +1,97 @@
1
+ """Built-in slice planners that preserve text, replacements and structure."""
2
+ from bisect import bisect_right
3
+ from collections.abc import Iterable
4
+ from importlib.util import find_spec
5
+ import re
6
+
7
+ from sct.chunking import ChunkBudgetError, ChunkerRegistry, ChunkRequest, legal_cut
8
+ from sct.findings import TextRange
9
+
10
+ _MARKDOWN_HEADING = re.compile(r'^ {0,3}#{1,6}(?=\s)', re.MULTILINE)
11
+ _PARAGRAPHS = re.compile(r'\n[ \t]*\n')
12
+ _LINE_ENDS = re.compile(r'\r?\n')
13
+ _WHITESPACE = re.compile(r'\s+')
14
+
15
+
16
+ def _limit(request: ChunkRequest, start: int) -> int:
17
+ text, count, budget = request.text, request.count, request.max_size
18
+ end = len(text)
19
+ if count is len:
20
+ return min(start + budget, end)
21
+ # Bound the search from the current position to avoid recounting the whole tail.
22
+ lo, hi = start, min(end, start + budget)
23
+ while count(text[start:hi]) <= budget:
24
+ if hi == end:
25
+ return hi
26
+ lo, hi = hi, min(end, start + 2 * (hi - start))
27
+ while lo + 1 < hi:
28
+ mid = (lo + hi) // 2
29
+ if count(text[start:mid]) <= budget:
30
+ lo = mid
31
+ else:
32
+ hi = mid
33
+ return lo
34
+
35
+
36
+ def bounded_ranges(request: ChunkRequest, preferred: Iterable[int] = ()) -> Iterable[TextRange]:
37
+ text = request.text
38
+ protected_starts = tuple(span.start for span in request.protected)
39
+ boundaries = [
40
+ sorted({legal_cut(pos, request.protected, protected_starts) for pos in preferred if 0 < pos < len(text)}),
41
+ *[
42
+ sorted({legal_cut(match.end(), request.protected, protected_starts) for match in pattern.finditer(text)})
43
+ for pattern in (_PARAGRAPHS, _LINE_ENDS, _WHITESPACE)
44
+ ],
45
+ ]
46
+ start = 0
47
+ while start < len(text):
48
+ limit = legal_cut(_limit(request, start), request.protected, protected_starts)
49
+ if limit <= start:
50
+ raise ChunkBudgetError('The size budget cannot fit the next character or replacement')
51
+ end = limit
52
+ if limit < len(text):
53
+ for cuts in boundaries:
54
+ index = bisect_right(cuts, limit) - 1
55
+ if index >= 0 and cuts[index] > start and request.count(text[start:cuts[index]]) <= request.max_size:
56
+ end = cuts[index]
57
+ break
58
+ yield TextRange(start, end)
59
+ start = end
60
+
61
+
62
+ def _markdown(request: ChunkRequest) -> Iterable[TextRange]:
63
+ return bounded_ranges(request, (match.start() for match in _MARKDOWN_HEADING.finditer(request.text)))
64
+
65
+
66
+ def _elements(request: ChunkRequest) -> Iterable[TextRange]:
67
+ return bounded_ranges(request, (element.start for element in request.elements))
68
+
69
+
70
+ def _semchunk(request: ChunkRequest) -> Iterable[TextRange]:
71
+ from semchunk import chunk
72
+
73
+ chunks, offsets = chunk(request.text, request.max_size, request.count, offsets=True, memoize=False)
74
+ cuts = [
75
+ end for text, (start, end) in zip(chunks, offsets, strict=True)
76
+ if isinstance(start, int) and isinstance(end, int) and request.text[start:end] == text
77
+ ]
78
+ return bounded_ranges(request, cuts)
79
+
80
+
81
+ def _chonkie(request: ChunkRequest) -> Iterable[TextRange]:
82
+ from chonkie import RecursiveChunker
83
+
84
+ # Chonkie's offsets are hints. SCT copies and counts only slices of its own text.
85
+ chunks = RecursiveChunker(tokenizer='character', chunk_size=request.max_size).chunk(request.text)
86
+ cuts = [item.end_index for item in chunks
87
+ if request.text[item.start_index:item.end_index] == item.text]
88
+ return bounded_ranges(request, cuts)
89
+
90
+
91
+ def register_builtin_chunkers(registry: ChunkerRegistry) -> None:
92
+ registry.register('text', bounded_ranges, stages=('detection', 'output'))
93
+ registry.register('markdown', _markdown)
94
+ registry.register('elements', _elements)
95
+ registry.register('semchunk', _semchunk, stages=('detection', 'output'),
96
+ available=lambda: find_spec('semchunk') is not None)
97
+ registry.register('chonkie', _chonkie, available=lambda: find_spec('chonkie') is not None)