SqueakyCleanText 0.10.0__tar.gz → 0.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/PKG-INFO +29 -2
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/README.md +25 -1
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/SqueakyCleanText.egg-info/PKG-INFO +29 -2
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/SqueakyCleanText.egg-info/SOURCES.txt +6 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/SqueakyCleanText.egg-info/requires.txt +4 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/pyproject.toml +1 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/__init__.py +8 -2
- squeakycleantext-0.11.0/sct/chunked_document.py +127 -0
- squeakycleantext-0.11.0/sct/chunkers_builtin.py +97 -0
- squeakycleantext-0.11.0/sct/chunking.py +227 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/config.py +36 -3
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/corpus/datatrove.py +6 -2
- squeakycleantext-0.11.0/sct/detector_windows.py +96 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/detectors.py +18 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/detectors_builtin.py +9 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/documents.py +13 -2
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/findings.py +3 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/routing.py +6 -3
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/sct.py +6 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/unified.py +24 -11
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/ner.py +6 -5
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/ner_chunking.py +21 -9
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/ner_spans.py +7 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_benchmark_loaders.py +18 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_chunking.py +30 -0
- squeakycleantext-0.11.0/tests/test_chunking_router.py +238 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_detector_routing.py +4 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_ner_span_repair.py +24 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_pii_social_cards.py +4 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_plot_pii_comparison.py +2 -3
- squeakycleantext-0.11.0/tests/test_plot_pii_social_release.py +43 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/LICENSE +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/SqueakyCleanText.egg-info/dependency_links.txt +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/SqueakyCleanText.egg-info/entry_points.txt +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/SqueakyCleanText.egg-info/top_level.txt +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/adapters.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/browser.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/cli.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/coordinates.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/corpus/__init__.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/corpus/manifest.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/corpus/runner.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/corpus/selection.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/corpus/types.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/detectors_datafog.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/evaluation.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/integrations/__init__.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/integrations/_runtime.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/integrations/datatrove.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/integrations/docling.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/integrations/haystack.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/integrations/hf.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/integrations/langchain.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/language_policy.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/mcp_server.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/model_benchmark.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/model_candidates.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/ocr.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/policy_v2.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/presets.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/privacy.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/py.typed +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/report_cli.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/risk.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/schema.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/score.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/signals.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/streaming.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/token_stores/__init__.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/token_stores/sqlite.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/__init__.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/anonymization_map.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/atomic_file.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/constants.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/contact.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/datetime.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/gliclass_adapter.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/gliner_adapter.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/markers.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/normtext.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/onnx_pipeline.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/process_result.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/resources.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/special.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/stopwords.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/synthetic.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/sct/utils/torch_pipeline.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/setup.cfg +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_adapters.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_address_leftovers.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_atomic_file.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_baseline_measurement.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_batching.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_benchmark_adapters.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_benchmark_release.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_benchmarks_compare.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_benchmarks_round2.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_benchmarks_round2_tools.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_benchmarks_score.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_browser.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_builtin_detections.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_cjk_width.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_cleaner_lifecycle.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_cli.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_config_state.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_contracts.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_coordinates.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_corpus_decontamination.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_corpus_failure_boundaries.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_corpus_identity_boundaries.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_corpus_manifest.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_corpus_resume.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_corpus_selection.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_date_forms.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_dates_cjk_styled.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_dates_default.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_detections.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_detector_registry.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_device_provenance.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_docling_integration.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_email_forms.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_escaped_markup.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_evaluation.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_fork_safety.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_functional.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_fuzz_styled_pii.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_gliner_recipe_identity.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_gliner_revision.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_improvements_end_to_end.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_integrations_frameworks.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_integrations_hf.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_integrations_runtime.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_keep_scripts.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_language_all.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_language_policy.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_linux_workers.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_mcp_path.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_mcp_server.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_model_benchmarks.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_multilingual_eval_set.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_ner_sees_tags.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_ner_spans.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_ner_windows.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_number_identifiers.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_ocr.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_onnx_concurrency.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_onnx_external_data.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_onnx_model_cache.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_optional_profiles.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_phone_forms.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_pipeline_text_steps.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_plot_version_comparison.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_predicted_spans_monotone.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_presidio_tags_and_fakes.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_presidio_type_names.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_privacy.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_processing_compatibility.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_prose_punctuation_kept.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_public_view.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_quality_export.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_regex_boundaries.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_regex_counts.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_release_comparison_watcher.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_repo_scripts.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_report_cli.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_reversible_markers.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_risk.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_routing_audit.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_routing_calibration.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_schema_context.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_score.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_sct.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_secret_detector.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_security_leftovers.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_sqlite_token_store.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_stream_guard_shared_view.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_stream_keeps_values_whole.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_streaming.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_streaming_split_stability.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_styled_pii.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_surface_contracts.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_synthetic_determinism.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_synthetic_per_document.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_token_store_failure_boundaries.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_token_store_operations.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_torch_ner_backend.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_training_preset.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_type_names.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_unified_processing.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_url_forms.py +0 -0
- {squeakycleantext-0.10.0 → squeakycleantext-0.11.0}/tests/test_user_terms.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SqueakyCleanText
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.0
|
|
4
4
|
Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
|
|
5
5
|
Author: Rehan Fazal
|
|
6
6
|
License-Expression: MIT
|
|
@@ -42,6 +42,9 @@ Provides-Extra: secrets
|
|
|
42
42
|
Requires-Dist: datafog-core==1.0.0; extra == "secrets"
|
|
43
43
|
Provides-Extra: documents
|
|
44
44
|
Requires-Dist: docling==2.148.0; extra == "documents"
|
|
45
|
+
Provides-Extra: chunking
|
|
46
|
+
Requires-Dist: semchunk<4,>=3.2.5; extra == "chunking"
|
|
47
|
+
Requires-Dist: chonkie<2,>=1.7.0; extra == "chunking"
|
|
45
48
|
Provides-Extra: token-store
|
|
46
49
|
Requires-Dist: cryptography==48.0.1; extra == "token-store"
|
|
47
50
|
Provides-Extra: corpus
|
|
@@ -238,6 +241,7 @@ if __name__ == "__main__":
|
|
|
238
241
|
| Where | Usage | Docs |
|
|
239
242
|
|---|---|---|
|
|
240
243
|
| PDF extraction | `extract_document` and `clean_document` | [DOCUMENTS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/DOCUMENTS.md) |
|
|
244
|
+
| Clean and chunk a document | `chunk_document(cleaner, text, max_size=1024)` | [CHUNKING](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CHUNKING.md) |
|
|
241
245
|
| LangChain | `SCTDocumentTransformer().transform_documents(docs)` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
|
|
242
246
|
| Haystack | `pipe.add_component("sct", SCTDocumentCleaner())` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
|
|
243
247
|
| Ray, Spark, Polars, DuckDB, dlt, Daft | `clean_columns(handle, batch)` | [RECIPES](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/RECIPES.md) |
|
|
@@ -248,6 +252,23 @@ if __name__ == "__main__":
|
|
|
248
252
|
|
|
249
253
|
The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
|
|
250
254
|
|
|
255
|
+
### Clean a document and produce chunks
|
|
256
|
+
|
|
257
|
+
Use `chunk_document()` to clean the full document before you split it for search, training, or export.
|
|
258
|
+
|
|
259
|
+
```python
|
|
260
|
+
from sct import TextCleaner, TextCleanerConfig, chunk_document
|
|
261
|
+
|
|
262
|
+
cleaner = TextCleaner(TextCleanerConfig(check_ner_process=False))
|
|
263
|
+
result = chunk_document(cleaner, "Contact alice@example.com for details.", max_size=24)
|
|
264
|
+
texts = [chunk.text for chunk in result.chunks]
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
SCT can use Markdown headings or cleaned document elements as boundaries. Install
|
|
268
|
+
`SqueakyCleanText[chunking]` for Semchunk and Chonkie, or register your own splitter.
|
|
269
|
+
In unified PII processing, you can also choose a chunker for each compatible detector.
|
|
270
|
+
See [chunking providers and routing](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CHUNKING.md).
|
|
271
|
+
|
|
251
272
|
## How it works
|
|
252
273
|
|
|
253
274
|
<img src="https://raw.githubusercontent.com/rhnfzl/SqueakyCleanText/main/resources/sct_flow.png" alt="SqueakyCleanText pipeline: any entry point runs 1 detect language, 2 fix the text, 3 regex PII and cleanup, 4 NER, 5 replacement mode (placeholder tags are written in steps 3 and 4; synthetic and reversible swap the values in after NER), 6 output and signals. Dashed steps are off by default." width="900">
|
|
@@ -371,6 +392,10 @@ NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cl
|
|
|
371
392
|
|
|
372
393
|
Measured on open datasets with [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks).
|
|
373
394
|
|
|
395
|
+
The [SCT v0.10.0 comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-12-v010/README.md)
|
|
396
|
+
adds fresh SCT results over 20,817 documents, six charts, and repeated CPU measurements.
|
|
397
|
+
Other-tool versions remain historical pins. Shared-label and full-label results differ because label coverage affects the comparison.
|
|
398
|
+
|
|
374
399
|
The [SCT v0.8.2 comparison with other PII tools](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-04-visuals/README.md)
|
|
375
400
|
shows paired recall, precision and exposure results, with uncertainty intervals.
|
|
376
401
|
It measures PII configurations rather than the whole processing pipeline. The tested version is v0.8.2, not v0.8.3.
|
|
@@ -385,11 +410,13 @@ Runtime versions differ, so the speed results do not isolate SCT code changes.
|
|
|
385
410
|
- **PII** (12 open PII datasets, about 40 languages): the 0.8.1 release branch leaked fewer values than 0.8.0 on every dataset, for example 18.1% to 8.2% on REDACT ([comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/comparison-0.8.0-unreleased.md), pre-release commit `2078e8c4`).
|
|
386
411
|
- **Multilingual PII eval** (220 records, 11 languages): 0.93 recall, 0.95 precision ([docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md)).
|
|
387
412
|
|
|
413
|
+
**Dataset credits.** These scores use open datasets, most under CC BY or other attribution licences. Creators, sources, licences and citations are in [benchmarks/DATASET_CREDITS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/DATASET_CREDITS.md).
|
|
414
|
+
|
|
388
415
|
Test your own data or model with the evaluation harness ([docs/EVALUATION.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/EVALUATION.md)).
|
|
389
416
|
|
|
390
417
|
## Languages
|
|
391
418
|
|
|
392
|
-
Language detection covers 75 languages. Full NER models cover English, Dutch, German and Spanish. The other languages use the multilingual WikiNEural model with the English model, or a model you bring with `ner_models`. Full list: [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md).
|
|
419
|
+
Language detection covers 75 languages. Full NER models cover English, Dutch, German and Spanish. The other languages use the multilingual WikiNEural model with the English model, or a model you bring with `ner_models`. The WikiNEural model is CC BY-NC-SA 4.0, for non-commercial research only, and SCT loads it by default. [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md#model-licences) shows how to run without it. Full list: [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md).
|
|
393
420
|
|
|
394
421
|
<!-- --8<-- [start:limits] -->
|
|
395
422
|
## Limits and design principles
|
|
@@ -136,6 +136,7 @@ if __name__ == "__main__":
|
|
|
136
136
|
| Where | Usage | Docs |
|
|
137
137
|
|---|---|---|
|
|
138
138
|
| PDF extraction | `extract_document` and `clean_document` | [DOCUMENTS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/DOCUMENTS.md) |
|
|
139
|
+
| Clean and chunk a document | `chunk_document(cleaner, text, max_size=1024)` | [CHUNKING](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CHUNKING.md) |
|
|
139
140
|
| LangChain | `SCTDocumentTransformer().transform_documents(docs)` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
|
|
140
141
|
| Haystack | `pipe.add_component("sct", SCTDocumentCleaner())` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
|
|
141
142
|
| Ray, Spark, Polars, DuckDB, dlt, Daft | `clean_columns(handle, batch)` | [RECIPES](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/RECIPES.md) |
|
|
@@ -146,6 +147,23 @@ if __name__ == "__main__":
|
|
|
146
147
|
|
|
147
148
|
The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
|
|
148
149
|
|
|
150
|
+
### Clean a document and produce chunks
|
|
151
|
+
|
|
152
|
+
Use `chunk_document()` to clean the full document before you split it for search, training, or export.
|
|
153
|
+
|
|
154
|
+
```python
|
|
155
|
+
from sct import TextCleaner, TextCleanerConfig, chunk_document
|
|
156
|
+
|
|
157
|
+
cleaner = TextCleaner(TextCleanerConfig(check_ner_process=False))
|
|
158
|
+
result = chunk_document(cleaner, "Contact alice@example.com for details.", max_size=24)
|
|
159
|
+
texts = [chunk.text for chunk in result.chunks]
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
SCT can use Markdown headings or cleaned document elements as boundaries. Install
|
|
163
|
+
`SqueakyCleanText[chunking]` for Semchunk and Chonkie, or register your own splitter.
|
|
164
|
+
In unified PII processing, you can also choose a chunker for each compatible detector.
|
|
165
|
+
See [chunking providers and routing](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CHUNKING.md).
|
|
166
|
+
|
|
149
167
|
## How it works
|
|
150
168
|
|
|
151
169
|
<img src="https://raw.githubusercontent.com/rhnfzl/SqueakyCleanText/main/resources/sct_flow.png" alt="SqueakyCleanText pipeline: any entry point runs 1 detect language, 2 fix the text, 3 regex PII and cleanup, 4 NER, 5 replacement mode (placeholder tags are written in steps 3 and 4; synthetic and reversible swap the values in after NER), 6 output and signals. Dashed steps are off by default." width="900">
|
|
@@ -269,6 +287,10 @@ NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cl
|
|
|
269
287
|
|
|
270
288
|
Measured on open datasets with [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks).
|
|
271
289
|
|
|
290
|
+
The [SCT v0.10.0 comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-12-v010/README.md)
|
|
291
|
+
adds fresh SCT results over 20,817 documents, six charts, and repeated CPU measurements.
|
|
292
|
+
Other-tool versions remain historical pins. Shared-label and full-label results differ because label coverage affects the comparison.
|
|
293
|
+
|
|
272
294
|
The [SCT v0.8.2 comparison with other PII tools](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-04-visuals/README.md)
|
|
273
295
|
shows paired recall, precision and exposure results, with uncertainty intervals.
|
|
274
296
|
It measures PII configurations rather than the whole processing pipeline. The tested version is v0.8.2, not v0.8.3.
|
|
@@ -283,11 +305,13 @@ Runtime versions differ, so the speed results do not isolate SCT code changes.
|
|
|
283
305
|
- **PII** (12 open PII datasets, about 40 languages): the 0.8.1 release branch leaked fewer values than 0.8.0 on every dataset, for example 18.1% to 8.2% on REDACT ([comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/comparison-0.8.0-unreleased.md), pre-release commit `2078e8c4`).
|
|
284
306
|
- **Multilingual PII eval** (220 records, 11 languages): 0.93 recall, 0.95 precision ([docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md)).
|
|
285
307
|
|
|
308
|
+
**Dataset credits.** These scores use open datasets, most under CC BY or other attribution licences. Creators, sources, licences and citations are in [benchmarks/DATASET_CREDITS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/DATASET_CREDITS.md).
|
|
309
|
+
|
|
286
310
|
Test your own data or model with the evaluation harness ([docs/EVALUATION.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/EVALUATION.md)).
|
|
287
311
|
|
|
288
312
|
## Languages
|
|
289
313
|
|
|
290
|
-
Language detection covers 75 languages. Full NER models cover English, Dutch, German and Spanish. The other languages use the multilingual WikiNEural model with the English model, or a model you bring with `ner_models`. Full list: [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md).
|
|
314
|
+
Language detection covers 75 languages. Full NER models cover English, Dutch, German and Spanish. The other languages use the multilingual WikiNEural model with the English model, or a model you bring with `ner_models`. The WikiNEural model is CC BY-NC-SA 4.0, for non-commercial research only, and SCT loads it by default. [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md#model-licences) shows how to run without it. Full list: [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md).
|
|
291
315
|
|
|
292
316
|
<!-- --8<-- [start:limits] -->
|
|
293
317
|
## Limits and design principles
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SqueakyCleanText
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.0
|
|
4
4
|
Summary: Text preprocessing & PII anonymization pipeline for NLP/ML: ONNX NER ensemble, language detection, stopword removal, and configurable token replacement.
|
|
5
5
|
Author: Rehan Fazal
|
|
6
6
|
License-Expression: MIT
|
|
@@ -42,6 +42,9 @@ Provides-Extra: secrets
|
|
|
42
42
|
Requires-Dist: datafog-core==1.0.0; extra == "secrets"
|
|
43
43
|
Provides-Extra: documents
|
|
44
44
|
Requires-Dist: docling==2.148.0; extra == "documents"
|
|
45
|
+
Provides-Extra: chunking
|
|
46
|
+
Requires-Dist: semchunk<4,>=3.2.5; extra == "chunking"
|
|
47
|
+
Requires-Dist: chonkie<2,>=1.7.0; extra == "chunking"
|
|
45
48
|
Provides-Extra: token-store
|
|
46
49
|
Requires-Dist: cryptography==48.0.1; extra == "token-store"
|
|
47
50
|
Provides-Extra: corpus
|
|
@@ -238,6 +241,7 @@ if __name__ == "__main__":
|
|
|
238
241
|
| Where | Usage | Docs |
|
|
239
242
|
|---|---|---|
|
|
240
243
|
| PDF extraction | `extract_document` and `clean_document` | [DOCUMENTS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/DOCUMENTS.md) |
|
|
244
|
+
| Clean and chunk a document | `chunk_document(cleaner, text, max_size=1024)` | [CHUNKING](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CHUNKING.md) |
|
|
241
245
|
| LangChain | `SCTDocumentTransformer().transform_documents(docs)` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
|
|
242
246
|
| Haystack | `pipe.add_component("sct", SCTDocumentCleaner())` | [INTEGRATIONS](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/INTEGRATIONS.md) |
|
|
243
247
|
| Ray, Spark, Polars, DuckDB, dlt, Daft | `clean_columns(handle, batch)` | [RECIPES](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/RECIPES.md) |
|
|
@@ -248,6 +252,23 @@ if __name__ == "__main__":
|
|
|
248
252
|
|
|
249
253
|
The framework adapters use the `training` preset unless you pass a config, so their output keeps numbers, dates, HTML and case. They build one cleaner per worker process and reject reversible mode, as do the streaming redactors and the MCP server. Plan for 5 to 7 GB of RAM per worker with the default NER ensemble.
|
|
250
254
|
|
|
255
|
+
### Clean a document and produce chunks
|
|
256
|
+
|
|
257
|
+
Use `chunk_document()` to clean the full document before you split it for search, training, or export.
|
|
258
|
+
|
|
259
|
+
```python
|
|
260
|
+
from sct import TextCleaner, TextCleanerConfig, chunk_document
|
|
261
|
+
|
|
262
|
+
cleaner = TextCleaner(TextCleanerConfig(check_ner_process=False))
|
|
263
|
+
result = chunk_document(cleaner, "Contact alice@example.com for details.", max_size=24)
|
|
264
|
+
texts = [chunk.text for chunk in result.chunks]
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
SCT can use Markdown headings or cleaned document elements as boundaries. Install
|
|
268
|
+
`SqueakyCleanText[chunking]` for Semchunk and Chonkie, or register your own splitter.
|
|
269
|
+
In unified PII processing, you can also choose a chunker for each compatible detector.
|
|
270
|
+
See [chunking providers and routing](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/CHUNKING.md).
|
|
271
|
+
|
|
251
272
|
## How it works
|
|
252
273
|
|
|
253
274
|
<img src="https://raw.githubusercontent.com/rhnfzl/SqueakyCleanText/main/resources/sct_flow.png" alt="SqueakyCleanText pipeline: any entry point runs 1 detect language, 2 fix the text, 3 regex PII and cleanup, 4 NER, 5 replacement mode (placeholder tags are written in steps 3 and 4; synthetic and reversible swap the values in after NER), 6 output and signals. Dashed steps are off by default." width="900">
|
|
@@ -371,6 +392,10 @@ NER is the slow stage. Turn it off (`check_ner_process=False`) for regex-only cl
|
|
|
371
392
|
|
|
372
393
|
Measured on open datasets with [benchmarks/](https://github.com/rhnfzl/SqueakyCleanText/tree/main/benchmarks).
|
|
373
394
|
|
|
395
|
+
The [SCT v0.10.0 comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-12-v010/README.md)
|
|
396
|
+
adds fresh SCT results over 20,817 documents, six charts, and repeated CPU measurements.
|
|
397
|
+
Other-tool versions remain historical pins. Shared-label and full-label results differ because label coverage affects the comparison.
|
|
398
|
+
|
|
374
399
|
The [SCT v0.8.2 comparison with other PII tools](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/compare-2026-11-04-visuals/README.md)
|
|
375
400
|
shows paired recall, precision and exposure results, with uncertainty intervals.
|
|
376
401
|
It measures PII configurations rather than the whole processing pipeline. The tested version is v0.8.2, not v0.8.3.
|
|
@@ -385,11 +410,13 @@ Runtime versions differ, so the speed results do not isolate SCT code changes.
|
|
|
385
410
|
- **PII** (12 open PII datasets, about 40 languages): the 0.8.1 release branch leaked fewer values than 0.8.0 on every dataset, for example 18.1% to 8.2% on REDACT ([comparison](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/results/comparison-0.8.0-unreleased.md), pre-release commit `2078e8c4`).
|
|
386
411
|
- **Multilingual PII eval** (220 records, 11 languages): 0.93 recall, 0.95 precision ([docs/MULTILINGUAL_EVAL.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/MULTILINGUAL_EVAL.md)).
|
|
387
412
|
|
|
413
|
+
**Dataset credits.** These scores use open datasets, most under CC BY or other attribution licences. Creators, sources, licences and citations are in [benchmarks/DATASET_CREDITS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/benchmarks/DATASET_CREDITS.md).
|
|
414
|
+
|
|
388
415
|
Test your own data or model with the evaluation harness ([docs/EVALUATION.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/EVALUATION.md)).
|
|
389
416
|
|
|
390
417
|
## Languages
|
|
391
418
|
|
|
392
|
-
Language detection covers 75 languages. Full NER models cover English, Dutch, German and Spanish. The other languages use the multilingual WikiNEural model with the English model, or a model you bring with `ner_models`. Full list: [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md).
|
|
419
|
+
Language detection covers 75 languages. Full NER models cover English, Dutch, German and Spanish. The other languages use the multilingual WikiNEural model with the English model, or a model you bring with `ner_models`. The WikiNEural model is CC BY-NC-SA 4.0, for non-commercial research only, and SCT loads it by default. [docs/NER_BACKENDS.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/NER_BACKENDS.md#model-licences) shows how to run without it. Full list: [docs/LANGUAGES.md](https://github.com/rhnfzl/SqueakyCleanText/blob/main/docs/LANGUAGES.md).
|
|
393
420
|
|
|
394
421
|
<!-- --8<-- [start:limits] -->
|
|
395
422
|
## Limits and design principles
|
|
@@ -10,9 +10,13 @@ SqueakyCleanText.egg-info/top_level.txt
|
|
|
10
10
|
sct/__init__.py
|
|
11
11
|
sct/adapters.py
|
|
12
12
|
sct/browser.py
|
|
13
|
+
sct/chunked_document.py
|
|
14
|
+
sct/chunkers_builtin.py
|
|
15
|
+
sct/chunking.py
|
|
13
16
|
sct/cli.py
|
|
14
17
|
sct/config.py
|
|
15
18
|
sct/coordinates.py
|
|
19
|
+
sct/detector_windows.py
|
|
16
20
|
sct/detectors.py
|
|
17
21
|
sct/detectors_builtin.py
|
|
18
22
|
sct/detectors_datafog.py
|
|
@@ -87,6 +91,7 @@ tests/test_benchmarks_score.py
|
|
|
87
91
|
tests/test_browser.py
|
|
88
92
|
tests/test_builtin_detections.py
|
|
89
93
|
tests/test_chunking.py
|
|
94
|
+
tests/test_chunking_router.py
|
|
90
95
|
tests/test_cjk_width.py
|
|
91
96
|
tests/test_cleaner_lifecycle.py
|
|
92
97
|
tests/test_cli.py
|
|
@@ -141,6 +146,7 @@ tests/test_phone_forms.py
|
|
|
141
146
|
tests/test_pii_social_cards.py
|
|
142
147
|
tests/test_pipeline_text_steps.py
|
|
143
148
|
tests/test_plot_pii_comparison.py
|
|
149
|
+
tests/test_plot_pii_social_release.py
|
|
144
150
|
tests/test_plot_version_comparison.py
|
|
145
151
|
tests/test_predicted_spans_monotone.py
|
|
146
152
|
tests/test_presidio_tags_and_fakes.py
|
|
@@ -60,6 +60,7 @@ sct-report = "sct.report_cli:main"
|
|
|
60
60
|
[project.optional-dependencies]
|
|
61
61
|
secrets = ["datafog-core==1.0.0"]
|
|
62
62
|
documents = ["docling==2.148.0"]
|
|
63
|
+
chunking = ["semchunk>=3.2.5,<4", "chonkie>=1.7.0,<2"]
|
|
63
64
|
token-store = ["cryptography==48.0.1"]
|
|
64
65
|
corpus = ["datasketch>=1.6", "datatrove==0.10.1", "orjson>=3.10", "spacy==3.8.16"]
|
|
65
66
|
gpu = [
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
"""SqueakyCleanText - Text cleaning and preprocessing pipeline for ML/NLP."""
|
|
2
2
|
|
|
3
|
+
from sct.chunked_document import ChunkedDocument, CleanedChunk, chunk_document
|
|
4
|
+
from sct.chunking import ChunkBudgetError, ChunkRequest, Chunker, ChunkerRegistry, ChunkingPolicy, ChunkingRule
|
|
3
5
|
from sct.config import TextCleanerConfig, PII_LABELS, PII_LABEL_MAP
|
|
4
|
-
from sct.detectors import
|
|
6
|
+
from sct.detectors import (
|
|
7
|
+
Detector, DetectorCapabilities, DetectorRegistry, DetectorSpec, DetectorUnavailable, DetectorWindowLimits,
|
|
8
|
+
)
|
|
5
9
|
from sct.policy_v2 import UnifiedPrivacyPolicy, HandlingRule, FindingDecision
|
|
6
10
|
from sct.documents import (
|
|
7
11
|
DocumentElement, DocumentExtractor, ExtractedDocument, ProcessedDocument, clean_document, extract_document,
|
|
@@ -59,9 +63,11 @@ from sct.streaming import (
|
|
|
59
63
|
from sct.utils.anonymization_map import AnonymizationMap, MapEntry
|
|
60
64
|
from sct.utils.process_result import ProcessResult
|
|
61
65
|
|
|
62
|
-
__version__ = "0.
|
|
66
|
+
__version__ = "0.11.0"
|
|
63
67
|
__all__ = [
|
|
64
68
|
"DocumentElement", "DocumentExtractor", "ExtractedDocument", "ProcessedDocument", "clean_document", "extract_document",
|
|
69
|
+
"ChunkBudgetError", "ChunkRequest", "Chunker", "ChunkerRegistry", "ChunkingPolicy", "ChunkingRule",
|
|
70
|
+
"ChunkedDocument", "CleanedChunk", "chunk_document", "DetectorWindowLimits",
|
|
65
71
|
"UnifiedPrivacyPolicy", "HandlingRule", "FindingDecision", "Detection", "DetectionContext", "TextRange",
|
|
66
72
|
"Detector", "DetectorCapabilities", "DetectorRegistry", "DetectorSpec", "DetectorUnavailable",
|
|
67
73
|
"TextCleaner", "TextCleanerConfig", "CustomPatternTimeout", "ModelLoadError", "PRESETS", "preset_config",
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Clean an entire document and split only the cleaned result for downstream use."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from bisect import bisect_left, bisect_right
|
|
5
|
+
from collections.abc import Callable, Iterable, Mapping
|
|
6
|
+
from dataclasses import dataclass, field, replace
|
|
7
|
+
import re
|
|
8
|
+
from typing import TYPE_CHECKING
|
|
9
|
+
|
|
10
|
+
from sct.chunking import ChunkRequest, ChunkingPolicy, ChunkingRule
|
|
11
|
+
from sct.documents import DocumentElement, ExtractedDocument, _clean_document
|
|
12
|
+
from sct.findings import DetectionContext, TextRange
|
|
13
|
+
from sct.utils.markers import PLACEHOLDER_TOKEN
|
|
14
|
+
|
|
15
|
+
if TYPE_CHECKING:
|
|
16
|
+
from sct.sct import TextCleaner
|
|
17
|
+
from sct.unified import RenderedEdit
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True)
|
|
21
|
+
class CleanedChunk:
|
|
22
|
+
"""A chunk and its half-open Unicode range in the cleaned document."""
|
|
23
|
+
|
|
24
|
+
text: str = field(repr=False)
|
|
25
|
+
range: TextRange
|
|
26
|
+
element_ids: tuple[str, ...] = ()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class ChunkedDocument:
|
|
31
|
+
chunks: tuple[CleanedChunk, ...] = field(repr=False)
|
|
32
|
+
chunker: str
|
|
33
|
+
language: str | None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
OUTPUT_CHUNKING = ChunkingPolicy(
|
|
37
|
+
'auto', candidates=('text',), rules=(ChunkingRule('markdown', formats=('markdown', 'text/markdown')),),
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _element_ranges(elements: tuple[DocumentElement, ...]) -> tuple[tuple[TextRange, str], ...]:
|
|
42
|
+
start = 0
|
|
43
|
+
result = []
|
|
44
|
+
for item in elements:
|
|
45
|
+
end = start + len(item.text)
|
|
46
|
+
if end > start:
|
|
47
|
+
result.append((TextRange(start, end), item.id))
|
|
48
|
+
start = end + len(item.separator)
|
|
49
|
+
return tuple(result)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _protected(text: str, edits: Iterable[tuple[int, int]], custom: tuple[str, ...]) -> tuple[TextRange, ...]:
|
|
53
|
+
boundaries = sorted((
|
|
54
|
+
*edits,
|
|
55
|
+
*(match.span() for match in PLACEHOLDER_TOKEN.finditer(text)),
|
|
56
|
+
*(match.span(1) for marker in custom for match in re.finditer(rf'(?=({re.escape(marker)}))', text)),
|
|
57
|
+
))
|
|
58
|
+
spans: list[TextRange] = []
|
|
59
|
+
for start, end in boundaries:
|
|
60
|
+
if start == end:
|
|
61
|
+
continue
|
|
62
|
+
if spans and spans[-1].end > start:
|
|
63
|
+
prior = spans[-1]
|
|
64
|
+
spans[-1] = TextRange(prior.start, max(prior.end, end))
|
|
65
|
+
else:
|
|
66
|
+
spans.append(TextRange(start, end))
|
|
67
|
+
return tuple(spans)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def chunk_document(
|
|
71
|
+
cleaner: TextCleaner, document: str | ExtractedDocument, *,
|
|
72
|
+
chunking: ChunkingPolicy | str | Mapping | None = None,
|
|
73
|
+
max_size: int | None = None, count: Callable[[str], int] = len,
|
|
74
|
+
document_format: str | None = None, context: DetectionContext | None = None,
|
|
75
|
+
) -> ChunkedDocument:
|
|
76
|
+
"""Apply the cleaner once, then return chunks in the selected size units."""
|
|
77
|
+
if cleaner.cfg.replacement_mode in ('indexed', 'reversible'):
|
|
78
|
+
raise ValueError('Chunked output does not support indexed or reversible modes')
|
|
79
|
+
if cleaner.cfg.replacement_mode == 'synthetic' and cleaner.cfg.pii_processing != 'unified':
|
|
80
|
+
raise ValueError('Chunked synthetic output requires unified processing')
|
|
81
|
+
if context is not None and not isinstance(context, DetectionContext):
|
|
82
|
+
raise ValueError('Expected DetectionContext')
|
|
83
|
+
context = context or DetectionContext()
|
|
84
|
+
if document_format is not None:
|
|
85
|
+
context = replace(context, document_format=document_format)
|
|
86
|
+
document_format = context.document_format
|
|
87
|
+
policy = ChunkingPolicy.coerce(chunking) or cleaner.cfg.output_chunking
|
|
88
|
+
edits: tuple[RenderedEdit, ...] = ()
|
|
89
|
+
elements: tuple[tuple[TextRange, str], ...] = ()
|
|
90
|
+
if type(document) is ExtractedDocument:
|
|
91
|
+
if document_format is None:
|
|
92
|
+
document_format = 'document'
|
|
93
|
+
cleaned, processed = _clean_document(document, cleaner, context)
|
|
94
|
+
elements = _element_ranges(cleaned.elements)
|
|
95
|
+
result = processed.result
|
|
96
|
+
edits = processed.rendered_edits
|
|
97
|
+
policy = policy or ChunkingPolicy('elements')
|
|
98
|
+
elif isinstance(document, str):
|
|
99
|
+
if cleaner.cfg.pii_processing == 'unified':
|
|
100
|
+
processed = cleaner._process_unified_document(document, context)
|
|
101
|
+
result, edits = processed.result, processed.rendered_edits
|
|
102
|
+
else:
|
|
103
|
+
if replace(context, document_format=None) != DetectionContext():
|
|
104
|
+
raise ValueError('Detection context requires unified processing')
|
|
105
|
+
result = cleaner.process(document)
|
|
106
|
+
policy = policy or OUTPUT_CHUNKING
|
|
107
|
+
else:
|
|
108
|
+
raise ValueError('Expected a string or ExtractedDocument')
|
|
109
|
+
text = result.lm_text
|
|
110
|
+
ranges = tuple(span for span, _ in elements)
|
|
111
|
+
protected = _protected(text, ((edit.start, edit.end) for edit in edits), cleaner._custom_markers)
|
|
112
|
+
budget = cleaner.cfg.output_chunk_max_size if max_size is None else max_size
|
|
113
|
+
request = ChunkRequest(text, budget, count, 'output', result.language, document_format,
|
|
114
|
+
ranges, cleaner.cfg.allow_network_chunkers, protected)
|
|
115
|
+
registry = cleaner._chunker_registry
|
|
116
|
+
name = registry.resolve(policy, stage='output', language=result.language, document_format=document_format,
|
|
117
|
+
length=len(text), allow_network=request.allow_network)
|
|
118
|
+
if name == 'native':
|
|
119
|
+
name = 'elements' if type(document) is ExtractedDocument else 'text'
|
|
120
|
+
chosen = registry.plan(name, request) if text else ()
|
|
121
|
+
starts, ends = tuple(r.start for r in ranges), tuple(r.end for r in ranges)
|
|
122
|
+
return ChunkedDocument(tuple(
|
|
123
|
+
CleanedChunk(text[span.start:span.end], span,
|
|
124
|
+
tuple(identifier for _, identifier in elements[
|
|
125
|
+
bisect_right(ends, span.start):bisect_left(starts, span.end)]))
|
|
126
|
+
for span in chosen
|
|
127
|
+
), name, result.language)
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""Built-in slice planners that preserve text, replacements and structure."""
|
|
2
|
+
from bisect import bisect_right
|
|
3
|
+
from collections.abc import Iterable
|
|
4
|
+
from importlib.util import find_spec
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
from sct.chunking import ChunkBudgetError, ChunkerRegistry, ChunkRequest, legal_cut
|
|
8
|
+
from sct.findings import TextRange
|
|
9
|
+
|
|
10
|
+
_MARKDOWN_HEADING = re.compile(r'^ {0,3}#{1,6}(?=\s)', re.MULTILINE)
|
|
11
|
+
_PARAGRAPHS = re.compile(r'\n[ \t]*\n')
|
|
12
|
+
_LINE_ENDS = re.compile(r'\r?\n')
|
|
13
|
+
_WHITESPACE = re.compile(r'\s+')
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _limit(request: ChunkRequest, start: int) -> int:
|
|
17
|
+
text, count, budget = request.text, request.count, request.max_size
|
|
18
|
+
end = len(text)
|
|
19
|
+
if count is len:
|
|
20
|
+
return min(start + budget, end)
|
|
21
|
+
# Bound the search from the current position to avoid recounting the whole tail.
|
|
22
|
+
lo, hi = start, min(end, start + budget)
|
|
23
|
+
while count(text[start:hi]) <= budget:
|
|
24
|
+
if hi == end:
|
|
25
|
+
return hi
|
|
26
|
+
lo, hi = hi, min(end, start + 2 * (hi - start))
|
|
27
|
+
while lo + 1 < hi:
|
|
28
|
+
mid = (lo + hi) // 2
|
|
29
|
+
if count(text[start:mid]) <= budget:
|
|
30
|
+
lo = mid
|
|
31
|
+
else:
|
|
32
|
+
hi = mid
|
|
33
|
+
return lo
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def bounded_ranges(request: ChunkRequest, preferred: Iterable[int] = ()) -> Iterable[TextRange]:
|
|
37
|
+
text = request.text
|
|
38
|
+
protected_starts = tuple(span.start for span in request.protected)
|
|
39
|
+
boundaries = [
|
|
40
|
+
sorted({legal_cut(pos, request.protected, protected_starts) for pos in preferred if 0 < pos < len(text)}),
|
|
41
|
+
*[
|
|
42
|
+
sorted({legal_cut(match.end(), request.protected, protected_starts) for match in pattern.finditer(text)})
|
|
43
|
+
for pattern in (_PARAGRAPHS, _LINE_ENDS, _WHITESPACE)
|
|
44
|
+
],
|
|
45
|
+
]
|
|
46
|
+
start = 0
|
|
47
|
+
while start < len(text):
|
|
48
|
+
limit = legal_cut(_limit(request, start), request.protected, protected_starts)
|
|
49
|
+
if limit <= start:
|
|
50
|
+
raise ChunkBudgetError('The size budget cannot fit the next character or replacement')
|
|
51
|
+
end = limit
|
|
52
|
+
if limit < len(text):
|
|
53
|
+
for cuts in boundaries:
|
|
54
|
+
index = bisect_right(cuts, limit) - 1
|
|
55
|
+
if index >= 0 and cuts[index] > start and request.count(text[start:cuts[index]]) <= request.max_size:
|
|
56
|
+
end = cuts[index]
|
|
57
|
+
break
|
|
58
|
+
yield TextRange(start, end)
|
|
59
|
+
start = end
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _markdown(request: ChunkRequest) -> Iterable[TextRange]:
|
|
63
|
+
return bounded_ranges(request, (match.start() for match in _MARKDOWN_HEADING.finditer(request.text)))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _elements(request: ChunkRequest) -> Iterable[TextRange]:
|
|
67
|
+
return bounded_ranges(request, (element.start for element in request.elements))
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _semchunk(request: ChunkRequest) -> Iterable[TextRange]:
|
|
71
|
+
from semchunk import chunk
|
|
72
|
+
|
|
73
|
+
chunks, offsets = chunk(request.text, request.max_size, request.count, offsets=True, memoize=False)
|
|
74
|
+
cuts = [
|
|
75
|
+
end for text, (start, end) in zip(chunks, offsets, strict=True)
|
|
76
|
+
if isinstance(start, int) and isinstance(end, int) and request.text[start:end] == text
|
|
77
|
+
]
|
|
78
|
+
return bounded_ranges(request, cuts)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _chonkie(request: ChunkRequest) -> Iterable[TextRange]:
|
|
82
|
+
from chonkie import RecursiveChunker
|
|
83
|
+
|
|
84
|
+
# Chonkie's offsets are hints. SCT copies and counts only slices of its own text.
|
|
85
|
+
chunks = RecursiveChunker(tokenizer='character', chunk_size=request.max_size).chunk(request.text)
|
|
86
|
+
cuts = [item.end_index for item in chunks
|
|
87
|
+
if request.text[item.start_index:item.end_index] == item.text]
|
|
88
|
+
return bounded_ranges(request, cuts)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def register_builtin_chunkers(registry: ChunkerRegistry) -> None:
|
|
92
|
+
registry.register('text', bounded_ranges, stages=('detection', 'output'))
|
|
93
|
+
registry.register('markdown', _markdown)
|
|
94
|
+
registry.register('elements', _elements)
|
|
95
|
+
registry.register('semchunk', _semchunk, stages=('detection', 'output'),
|
|
96
|
+
available=lambda: find_spec('semchunk') is not None)
|
|
97
|
+
registry.register('chonkie', _chonkie, available=lambda: find_spec('chonkie') is not None)
|