classifyre-cli 0.4.70__tar.gz → 0.4.72__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/.turbo/turbo-build.log +1 -1
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/PKG-INFO +1 -1
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/package.json +1 -1
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/pyproject.toml +4 -1
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/pii/detector.py +126 -2
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/main.py +6 -1
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/models/generated_detectors.py +2 -2
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/models/generated_input.py +147 -97
- classifyre_cli-0.4.72/src/sources/dropbox/auth.py +254 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/dropbox/source.py +65 -9
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/kafka/source.py +194 -15
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/notion/source.py +11 -0
- classifyre_cli-0.4.72/src/sources/slack/source.py +953 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/utils/dependency_groups.py +1 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/utils/file_parser.py +70 -1
- classifyre_cli-0.4.72/tests/detectors/pii/test_pii_regional_entities.py +70 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_dependency_groups.py +7 -1
- classifyre_cli-0.4.72/tests/test_dropbox_auth_command.py +172 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_dropbox_source.py +75 -6
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_kafka_source.py +132 -9
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_notion_source.py +77 -0
- classifyre_cli-0.4.72/tests/test_slack_source.py +690 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_source_strategy_conformance.py +1 -10
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/utils/test_file_parser.py +93 -8
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/uv.lock +24 -11
- classifyre_cli-0.4.70/src/sources/slack/source.py +0 -574
- classifyre_cli-0.4.70/tests/test_slack_source.py +0 -208
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/.gitignore +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/.python-version +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/README.md +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/main.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/scripts/generate_models.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/config.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/base.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/broken_links/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/broken_links/detector.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/config.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/content/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/custom/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/custom/detector.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/custom/extractor.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/custom/runners/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/custom/runners/_base.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/custom/runners/_factory.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/custom/runners/_gliner2.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/custom/runners/_image_classification.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/custom/runners/_llm.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/custom/runners/_object_detection.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/custom/runners/_regex.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/custom/runners/_text_classification.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/custom/trainer.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/dependencies.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/pii/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/secrets/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/secrets/detector.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/threat/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/threat/code_security_detector.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/detectors/threat/yara_detector.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/file_evaluation/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/file_evaluation/runner.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/models/generated_single_asset_scan_results.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/outputs/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/outputs/base.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/outputs/console.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/outputs/factory.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/outputs/file.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/outputs/rest.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/pipeline/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/pipeline/content_provider.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/pipeline/detector_pipeline.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/pipeline/parsed_content_provider.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/pipeline/text_artifact.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/pipeline/worker_pool.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/asset_metadata.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/atlassian_common.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/azure_blob_storage/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/azure_blob_storage/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/base.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/confluence/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/confluence/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/databricks/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/databricks/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/delta_lake/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/delta_lake/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/dependencies.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/dropbox/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/elasticsearch/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/elasticsearch/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/email/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/email/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/google_cloud_storage/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/google_cloud_storage/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/google_workspace/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/google_workspace/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/hive/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/hive/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/iceberg/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/iceberg/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/jira/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/jira/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/kafka/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/kafka/rest.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/lakehouse_base.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/local_folder/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/local_folder/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/meilisearch/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/meilisearch/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/microsoft_365/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/microsoft_365/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/mongodb/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/mongodb/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/mssql/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/mssql/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/mysql/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/mysql/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/neo4j/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/neo4j/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/notion/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/notion/client.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/object_storage/base.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/opensearch/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/opensearch/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/oracle/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/oracle/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/postgresql/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/postgresql/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/powerbi/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/powerbi/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/recipe_normalizer.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/s3_client.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/s3_compatible_storage/README.md +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/s3_compatible_storage/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/s3_compatible_storage/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/sandbox/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/sandbox/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/search_engine_base.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/servicedesk/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/servicedesk/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/slack/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/snowflake/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/snowflake/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/sqlite/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/sqlite/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/tableau/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/tableau/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/tabular_base.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/tabular_utils.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/wordpress/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/wordpress/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/youtube/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/sources/youtube/source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/telemetry.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/utils/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/utils/archive_extraction.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/utils/content_extraction.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/utils/embedded_images.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/utils/file_metadata.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/utils/file_to_images.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/utils/hashing.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/utils/legacy_office.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/utils/resources.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/utils/transcription.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/utils/uv_sync.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/utils/validation.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/src/utils/video_processing.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/_lakehouse_fakes.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/conftest.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/broken_links/test_broken_links_detector.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/conftest.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/content/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/custom/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/custom/conftest.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/custom/test_gliner2_classification.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/custom/test_invoice_extraction.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/custom/test_llm_runner.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/custom/test_pipeline_integration.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/custom/test_regex_runner.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/custom/test_runner_hardening.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/custom/test_transformer_runners.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/pii/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/pii/conftest.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/pii/sample_invoice.pdf +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/pii/test_pii_chunking.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/pii/test_pii_detector.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/pii/test_pii_detector_extended.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/pii/test_pii_severity.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/secrets/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/secrets/test_secrets_detector.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/secrets/test_secrets_detector_extended.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/test_base_detector.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/test_custom_detector_examples_runtime.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/test_detector_catalog_commercial.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/test_detector_pipeline_types.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/test_detector_schema_examples.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/test_detector_types.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/test_phase2_detectors.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/test_registry.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/threat/__init__.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/threat/test_code_security_detector.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/detectors/threat/test_yara_detector.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/integration/test_wordpress_broken_links_detector.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/integration/test_wordpress_links_assets.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/pipeline/test_detector_outcomes.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/pipeline/test_detector_pipeline.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/pipeline/test_worker_pool.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_assets_metadata_catalog.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_azure_blob_storage_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_base_source_attachment.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_base_source_sampling.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_config.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_confluence_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_custom_extractor.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_databricks_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_delta_lake_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_elasticsearch_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_email_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_file_evaluation_runner.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_google_cloud_storage_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_google_workspace_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_hashing.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_hive_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_iceberg_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_jira_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_local_folder_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_main.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_meilisearch_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_microsoft_365_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_mongodb_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_mssql_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_mysql_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_neo4j_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_opensearch_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_oracle_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_outputs.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_postgresql_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_powerbi_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_recipe_normalizer.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_s3_compatible_storage_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_sampling_automatic.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_sandbox_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_servicedesk_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_snowflake_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_source_dependency_groups.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_sqlite_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_tableau_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_tabular_automatic_sampling.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_tabular_utils.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_uv_sync.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_wordpress_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_youtube_source.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/test_youtube_source_integration.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/utils/test_archive_extraction.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/utils/test_content_extraction.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/utils/test_embedded_images.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/utils/test_file_metadata.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/utils/test_file_to_images.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/utils/test_legacy_office.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/utils/test_transcription.py +0 -0
- {classifyre_cli-0.4.70 → classifyre_cli-0.4.72}/tests/utils/test_video_processing.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "classifyre-cli"
|
|
3
|
-
version = "0.4.
|
|
3
|
+
version = "0.4.72"
|
|
4
4
|
description = "Classifyre CLI — scan and classify unstructured data sources"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.12"
|
|
@@ -230,6 +230,9 @@ google-cloud-storage = [
|
|
|
230
230
|
dropbox = [
|
|
231
231
|
"dropbox>=12.2.1,<13.0.0",
|
|
232
232
|
]
|
|
233
|
+
slack = [
|
|
234
|
+
"slack-sdk>=3.43.0,<4.0.0",
|
|
235
|
+
]
|
|
233
236
|
email = [
|
|
234
237
|
"imap-tools>=1.10.0,<2.0.0",
|
|
235
238
|
]
|
|
@@ -107,7 +107,7 @@ class PIIDetector(BaseDetector):
|
|
|
107
107
|
# Italy
|
|
108
108
|
"IT_FISCAL_CODE",
|
|
109
109
|
"IT_DRIVER_LICENSE",
|
|
110
|
-
"
|
|
110
|
+
"IT_VAT_CODE",
|
|
111
111
|
"IT_PASSPORT",
|
|
112
112
|
"IT_IDENTITY_CARD",
|
|
113
113
|
# Singapore
|
|
@@ -314,6 +314,7 @@ class PIIDetector(BaseDetector):
|
|
|
314
314
|
else:
|
|
315
315
|
self.analyzer = AnalyzerEngine()
|
|
316
316
|
|
|
317
|
+
self._register_regional_recognizers(presidio_module)
|
|
317
318
|
self._register_custom_recognizers(presidio_module)
|
|
318
319
|
self._probe_phone_recognizer()
|
|
319
320
|
|
|
@@ -396,6 +397,129 @@ class PIIDetector(BaseDetector):
|
|
|
396
397
|
logger.debug("Loaded spaCy model '%s'", cfg_model)
|
|
397
398
|
return nlp_engine
|
|
398
399
|
|
|
400
|
+
# Regional recognizers Presidio ships but scopes to their own language, so an
|
|
401
|
+
# `supported_languages=["en"]` registry silently drops them (the "Recognizer
|
|
402
|
+
# not added to registry" warning is filtered out by _PresidioNoiseFilter).
|
|
403
|
+
# Every entity advertised in _ALL_SUPPORTED_ENTITIES has to be re-registered
|
|
404
|
+
# against "en" or it can never produce a finding, no matter what the preset
|
|
405
|
+
# asks for. Analysis always runs with language="en".
|
|
406
|
+
_REGIONAL_RECOGNIZER_CLASS_NAMES: ClassVar[tuple[str, ...]] = (
|
|
407
|
+
# Spain
|
|
408
|
+
"EsNifRecognizer",
|
|
409
|
+
"EsNieRecognizer",
|
|
410
|
+
# Italy
|
|
411
|
+
"ItFiscalCodeRecognizer",
|
|
412
|
+
"ItDriverLicenseRecognizer",
|
|
413
|
+
"ItVatCodeRecognizer",
|
|
414
|
+
"ItPassportRecognizer",
|
|
415
|
+
"ItIdentityCardRecognizer",
|
|
416
|
+
# Singapore
|
|
417
|
+
"SgFinRecognizer",
|
|
418
|
+
"SgUenRecognizer",
|
|
419
|
+
# Australia
|
|
420
|
+
"AuAbnRecognizer",
|
|
421
|
+
"AuAcnRecognizer",
|
|
422
|
+
"AuTfnRecognizer",
|
|
423
|
+
"AuMedicareRecognizer",
|
|
424
|
+
# India
|
|
425
|
+
"InPanRecognizer",
|
|
426
|
+
"InAadhaarRecognizer",
|
|
427
|
+
"InVehicleRegistrationRecognizer",
|
|
428
|
+
"InVoterRecognizer",
|
|
429
|
+
# Finland
|
|
430
|
+
"FiPersonalIdentityCodeRecognizer",
|
|
431
|
+
# Poland
|
|
432
|
+
"PlPeselRecognizer",
|
|
433
|
+
)
|
|
434
|
+
|
|
435
|
+
# DACH and EU identifiers Presidio has no recognizer for at all. Each pattern
|
|
436
|
+
# is deliberately anchored on a checkable structure and paired with context
|
|
437
|
+
# words, because a bare run of digits is far too common to flag on its own.
|
|
438
|
+
#
|
|
439
|
+
# Scores are chosen against Presidio's LemmaContextAwareEnhancer, which adds
|
|
440
|
+
# 0.35 when a context word sits near the match, and the detector's default
|
|
441
|
+
# confidence_threshold of 0.7. A structurally weak pattern therefore scores
|
|
442
|
+
# 0.4: on its own it stays below the threshold and is suppressed, and only a
|
|
443
|
+
# nearby context word lifts it to 0.75 and reports it. Anything lower could
|
|
444
|
+
# never surface at all; anything higher would fire on bare digit runs.
|
|
445
|
+
_DACH_PATTERN_SPECS: ClassVar[
|
|
446
|
+
tuple[tuple[str, str, tuple[tuple[str, str, float], ...], tuple[str, ...]], ...]
|
|
447
|
+
] = (
|
|
448
|
+
(
|
|
449
|
+
"AT_SVNR",
|
|
450
|
+
"Austrian social insurance number (SVNR)",
|
|
451
|
+
# NNNN DDMMYY — a 4-digit serial (last digit is a check digit)
|
|
452
|
+
# followed by the holder's date of birth.
|
|
453
|
+
(("svnr (needs context)", r"\b\d{4}[ /]?\d{2}[01]\d{3}\b", 0.4),),
|
|
454
|
+
("svnr", "sozialversicherung", "sozialversicherungsnummer", "versicherungsnummer"),
|
|
455
|
+
),
|
|
456
|
+
(
|
|
457
|
+
"CH_AHV",
|
|
458
|
+
"Swiss social security number (AHV/AVS)",
|
|
459
|
+
# Always begins with the 756 country prefix, so this one is safe to
|
|
460
|
+
# score highly on structure alone.
|
|
461
|
+
(("ahv (strong)", r"\b756[.\s]?\d{4}[.\s]?\d{4}[.\s]?\d{2}\b", 0.7),),
|
|
462
|
+
("ahv", "avs", "ahv-nummer", "sozialversicherung", "versichertennummer"),
|
|
463
|
+
),
|
|
464
|
+
(
|
|
465
|
+
"DE_TAX_ID",
|
|
466
|
+
"German tax identification number (Steuer-IdNr)",
|
|
467
|
+
# 11 digits, never leading zero.
|
|
468
|
+
(("steuer-idnr (needs context)", r"\b[1-9]\d{2}[ /]?\d{4}[ /]?\d{4}\b", 0.4),),
|
|
469
|
+
("steuer", "steuernummer", "steuer-idnr", "steueridentifikationsnummer", "tax id"),
|
|
470
|
+
),
|
|
471
|
+
(
|
|
472
|
+
"EU_NATIONAL_ID",
|
|
473
|
+
"Generic EU national identity number",
|
|
474
|
+
(("eu national id (needs context)", r"\b[A-Z]{2}[ -]?\d{6,12}\b", 0.4),),
|
|
475
|
+
("national id", "identity card", "personalausweis", "ausweisnummer", "id number"),
|
|
476
|
+
),
|
|
477
|
+
)
|
|
478
|
+
|
|
479
|
+
def _register_regional_recognizers(self, presidio_module: Any) -> None:
|
|
480
|
+
"""Re-register language-scoped and missing regional recognizers under "en"."""
|
|
481
|
+
if self.analyzer is None:
|
|
482
|
+
return
|
|
483
|
+
|
|
484
|
+
predefined = importlib.import_module("presidio_analyzer.predefined_recognizers")
|
|
485
|
+
already_supported = set(self.analyzer.get_supported_entities())
|
|
486
|
+
|
|
487
|
+
for class_name in self._REGIONAL_RECOGNIZER_CLASS_NAMES:
|
|
488
|
+
recognizer_class = getattr(predefined, class_name, None)
|
|
489
|
+
if recognizer_class is None:
|
|
490
|
+
logger.debug("Presidio recognizer %s not available; skipping", class_name)
|
|
491
|
+
continue
|
|
492
|
+
try:
|
|
493
|
+
recognizer = recognizer_class(supported_language="en")
|
|
494
|
+
except Exception as exc:
|
|
495
|
+
logger.debug("Could not instantiate %s: %s", class_name, exc)
|
|
496
|
+
continue
|
|
497
|
+
# Never double-register: a duplicate recognizer reports every match twice.
|
|
498
|
+
if already_supported.issuperset(recognizer.supported_entities):
|
|
499
|
+
continue
|
|
500
|
+
self.analyzer.registry.add_recognizer(recognizer)
|
|
501
|
+
already_supported.update(recognizer.supported_entities)
|
|
502
|
+
|
|
503
|
+
PatternRecognizer = presidio_module.PatternRecognizer # noqa: N806
|
|
504
|
+
Pattern = presidio_module.Pattern # noqa: N806
|
|
505
|
+
|
|
506
|
+
for entity, name, pattern_specs, context in self._DACH_PATTERN_SPECS:
|
|
507
|
+
if entity in already_supported:
|
|
508
|
+
continue
|
|
509
|
+
self.analyzer.registry.add_recognizer(
|
|
510
|
+
PatternRecognizer(
|
|
511
|
+
supported_entity=entity,
|
|
512
|
+
name=name,
|
|
513
|
+
supported_language="en",
|
|
514
|
+
patterns=[
|
|
515
|
+
Pattern(name=pattern_name, regex=regex, score=score)
|
|
516
|
+
for pattern_name, regex, score in pattern_specs
|
|
517
|
+
],
|
|
518
|
+
context=list(context),
|
|
519
|
+
)
|
|
520
|
+
)
|
|
521
|
+
already_supported.add(entity)
|
|
522
|
+
|
|
399
523
|
def _register_custom_recognizers(self, presidio_module: Any) -> None:
|
|
400
524
|
"""Add ad-hoc recognizers from config to the analyzer registry."""
|
|
401
525
|
custom_recognizers = getattr(self.config, "custom_recognizers", None) or []
|
|
@@ -826,7 +950,7 @@ class PIIDetector(BaseDetector):
|
|
|
826
950
|
"AU_ABN",
|
|
827
951
|
"AU_ACN",
|
|
828
952
|
"SG_UEN",
|
|
829
|
-
"
|
|
953
|
+
"IT_VAT_CODE",
|
|
830
954
|
"IN_VOTER",
|
|
831
955
|
"IN_VEHICLE_REGISTRATION",
|
|
832
956
|
}:
|
|
@@ -10,6 +10,7 @@ from typing import Any, cast
|
|
|
10
10
|
|
|
11
11
|
from .outputs import create_output_sink
|
|
12
12
|
from .sources import get_source, list_available_sources
|
|
13
|
+
from .sources.dropbox.auth import add_dropbox_auth_arguments, run_dropbox_auth_command
|
|
13
14
|
from .utils.validation import validate_input, validate_test_connection
|
|
14
15
|
|
|
15
16
|
logger = logging.getLogger(__name__)
|
|
@@ -586,7 +587,7 @@ def main() -> None:
|
|
|
586
587
|
parser = argparse.ArgumentParser(description="Classifyre Metadata Extraction CLI")
|
|
587
588
|
parser.add_argument(
|
|
588
589
|
"command",
|
|
589
|
-
choices=["test", "extract", "discover", "evaluate-file", "train"],
|
|
590
|
+
choices=["test", "extract", "discover", "evaluate-file", "train", "dropbox-auth"],
|
|
590
591
|
help="Command to run",
|
|
591
592
|
)
|
|
592
593
|
parser.add_argument(
|
|
@@ -667,6 +668,7 @@ def main() -> None:
|
|
|
667
668
|
default=None,
|
|
668
669
|
help="Max assets processed concurrently in Phase 2. Controls DB connection usage. Defaults to pool_workers*2 (env: CLASSIFYRE_MAX_CONCURRENT_ASSETS)",
|
|
669
670
|
)
|
|
671
|
+
add_dropbox_auth_arguments(parser)
|
|
670
672
|
|
|
671
673
|
args = parser.parse_args()
|
|
672
674
|
|
|
@@ -695,6 +697,9 @@ def main() -> None:
|
|
|
695
697
|
if args.debug:
|
|
696
698
|
logging.getLogger().setLevel(logging.DEBUG)
|
|
697
699
|
|
|
700
|
+
if args.command == "dropbox-auth":
|
|
701
|
+
sys.exit(run_dropbox_auth_command(args))
|
|
702
|
+
|
|
698
703
|
if args.command == "evaluate-file":
|
|
699
704
|
run_evaluate_file_command(args)
|
|
700
705
|
return
|
|
@@ -258,7 +258,7 @@ class SecretsEnabledPattern(StrEnum):
|
|
|
258
258
|
|
|
259
259
|
class PIIEnabledPattern(StrEnum):
|
|
260
260
|
"""
|
|
261
|
-
Presidio entity types for PII detection. Global: CREDIT_CARD, CRYPTO, DATE_TIME, EMAIL_ADDRESS, IBAN_CODE, IP_ADDRESS, NRP, LOCATION, PERSON, PHONE_NUMBER, MEDICAL_LICENSE, URL. USA: US_BANK_NUMBER, US_DRIVER_LICENSE, US_ITIN, US_PASSPORT, US_SSN. UK: UK_NHS. Spain: ES_NIF, ES_NIE. Italy: IT_FISCAL_CODE, IT_DRIVER_LICENSE,
|
|
261
|
+
Presidio entity types for PII detection. Global: CREDIT_CARD, CRYPTO, DATE_TIME, EMAIL_ADDRESS, IBAN_CODE, IP_ADDRESS, NRP, LOCATION, PERSON, PHONE_NUMBER, MEDICAL_LICENSE, URL. USA: US_BANK_NUMBER, US_DRIVER_LICENSE, US_ITIN, US_PASSPORT, US_SSN. UK: UK_NHS. Spain: ES_NIF, ES_NIE. Italy: IT_FISCAL_CODE, IT_DRIVER_LICENSE, IT_VAT_CODE, IT_PASSPORT, IT_IDENTITY_CARD. Singapore: SG_NRIC_FIN, SG_UEN. Australia: AU_ABN, AU_ACN, AU_TFN, AU_MEDICARE. India: IN_PAN, IN_AADHAAR, IN_VEHICLE_REGISTRATION, IN_VOTER. Finland: FI_PERSONAL_IDENTITY_CODE. Poland: PL_PESEL. DACH: AT_SVNR, CH_AHV, DE_TAX_ID, EU_NATIONAL_ID.
|
|
262
262
|
"""
|
|
263
263
|
|
|
264
264
|
CREDIT_CARD = 'CREDIT_CARD'
|
|
@@ -283,7 +283,7 @@ class PIIEnabledPattern(StrEnum):
|
|
|
283
283
|
ES_NIE = 'ES_NIE'
|
|
284
284
|
IT_FISCAL_CODE = 'IT_FISCAL_CODE'
|
|
285
285
|
IT_DRIVER_LICENSE = 'IT_DRIVER_LICENSE'
|
|
286
|
-
|
|
286
|
+
IT_VAT_CODE = 'IT_VAT_CODE'
|
|
287
287
|
IT_PASSPORT = 'IT_PASSPORT'
|
|
288
288
|
IT_IDENTITY_CARD = 'IT_IDENTITY_CARD'
|
|
289
289
|
SG_NRIC_FIN = 'SG_NRIC_FIN'
|
|
@@ -85,7 +85,7 @@ class PostStatus(StrEnum):
|
|
|
85
85
|
|
|
86
86
|
class SlackChannelType(StrEnum):
|
|
87
87
|
"""
|
|
88
|
-
Slack conversation types to include
|
|
88
|
+
Slack conversation types to include. public_channel and private_channel are named channels, mpim is a group direct message, im is a one-to-one direct message.
|
|
89
89
|
"""
|
|
90
90
|
|
|
91
91
|
public_channel = 'public_channel'
|
|
@@ -439,104 +439,91 @@ class SlackRequired(BaseModel):
|
|
|
439
439
|
extra='forbid',
|
|
440
440
|
)
|
|
441
441
|
workspace: str | None = Field(
|
|
442
|
-
None,
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
class SlackMaskedBotToken(BaseModel):
|
|
447
|
-
model_config = ConfigDict(
|
|
448
|
-
extra='forbid',
|
|
449
|
-
)
|
|
450
|
-
bot_token: str = Field(..., description='Slack bot token (starts with xoxb-)')
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
class SlackMaskedUserToken(BaseModel):
|
|
454
|
-
model_config = ConfigDict(
|
|
455
|
-
extra='forbid',
|
|
442
|
+
None,
|
|
443
|
+
description='Slack workspace name or domain, used for permalinks and stable asset IDs (example: acme or acme.slack.com)',
|
|
456
444
|
)
|
|
457
|
-
user_token: str = Field(..., description='Slack user token (starts with xoxp-)')
|
|
458
445
|
|
|
459
446
|
|
|
460
|
-
class
|
|
447
|
+
class SlackMasked(BaseModel):
|
|
461
448
|
"""
|
|
462
|
-
|
|
449
|
+
Bot-token authentication. Install the app into the workspace and copy its bot token; user tokens are not supported.
|
|
463
450
|
"""
|
|
464
451
|
|
|
465
|
-
bot = 'bot'
|
|
466
|
-
user = 'user'
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
class SlackMaskedToken(BaseModel):
|
|
470
452
|
model_config = ConfigDict(
|
|
471
453
|
extra='forbid',
|
|
472
454
|
)
|
|
473
|
-
|
|
474
|
-
|
|
455
|
+
bot_token: str = Field(
|
|
456
|
+
...,
|
|
457
|
+
description='Slack bot token, starts with xoxb-. Needs the scopes channels:read, groups:read, im:read, mpim:read, channels:history, groups:history, im:history, mpim:history, users:read and files:read.',
|
|
458
|
+
)
|
|
475
459
|
|
|
476
460
|
|
|
477
461
|
class SlackOptionalChannels(BaseModel):
|
|
478
462
|
"""
|
|
479
|
-
|
|
463
|
+
Which conversations to discover and scan.
|
|
480
464
|
"""
|
|
481
465
|
|
|
482
466
|
model_config = ConfigDict(
|
|
483
467
|
extra='forbid',
|
|
484
468
|
)
|
|
485
469
|
channel_types: list[SlackChannelType] | None = Field(
|
|
486
|
-
['public_channel'],
|
|
487
|
-
description='
|
|
470
|
+
['public_channel', 'private_channel', 'mpim', 'im'],
|
|
471
|
+
description='Conversation types to include when listing channels. The bot only sees private channels and group DMs it has been invited to.',
|
|
488
472
|
)
|
|
489
473
|
channel_ids: list[str] | None = Field(
|
|
490
474
|
None,
|
|
491
|
-
description='Explicit
|
|
475
|
+
description='Explicit conversation IDs to scan (example: C0123456789). When set, channel_types is ignored.',
|
|
492
476
|
)
|
|
493
477
|
exclude_archived: bool | None = Field(
|
|
494
|
-
True, description='
|
|
478
|
+
True, description='Skip archived channels when listing'
|
|
479
|
+
)
|
|
480
|
+
auto_join_public_channels: bool | None = Field(
|
|
481
|
+
False,
|
|
482
|
+
description='Join public channels the bot is not a member of before reading them. conversations.history returns not_in_channel otherwise. Requires the channels:join scope.',
|
|
495
483
|
)
|
|
496
484
|
|
|
497
485
|
|
|
498
486
|
class SlackOptionalTimeRange(BaseModel):
|
|
499
487
|
"""
|
|
500
|
-
|
|
488
|
+
Limit ingestion to messages inside a time window.
|
|
501
489
|
"""
|
|
502
490
|
|
|
503
491
|
model_config = ConfigDict(
|
|
504
492
|
extra='forbid',
|
|
505
493
|
)
|
|
506
494
|
oldest: str | None = Field(
|
|
507
|
-
None,
|
|
495
|
+
None,
|
|
496
|
+
description='Start of the range, as a Slack timestamp or an ISO 8601 datetime',
|
|
508
497
|
)
|
|
509
498
|
latest: str | None = Field(
|
|
510
|
-
None,
|
|
499
|
+
None,
|
|
500
|
+
description='End of the range, as a Slack timestamp or an ISO 8601 datetime',
|
|
511
501
|
)
|
|
512
502
|
|
|
513
503
|
|
|
514
504
|
class SlackOptionalIngestion(BaseModel):
|
|
515
505
|
"""
|
|
516
|
-
Throughput and
|
|
506
|
+
Throughput and message-content controls.
|
|
517
507
|
"""
|
|
518
508
|
|
|
519
509
|
model_config = ConfigDict(
|
|
520
510
|
extra='forbid',
|
|
521
511
|
)
|
|
522
512
|
batch_size: int | None = Field(
|
|
523
|
-
200,
|
|
513
|
+
200,
|
|
514
|
+
description='Messages requested per API call (Slack caps this at 200)',
|
|
515
|
+
ge=1,
|
|
516
|
+
le=200,
|
|
524
517
|
)
|
|
525
518
|
rate_limit_delay_seconds: float | None = Field(
|
|
526
|
-
1,
|
|
519
|
+
1,
|
|
520
|
+
description="Extra pause between API calls. Slack's 429 responses are retried automatically; this only smooths out sustained traffic.",
|
|
521
|
+
ge=0.0,
|
|
527
522
|
)
|
|
528
523
|
include_thread_replies: bool | None = Field(
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
class SlackOptional(BaseModel):
|
|
534
|
-
model_config = ConfigDict(
|
|
535
|
-
extra='forbid',
|
|
524
|
+
True,
|
|
525
|
+
description="Fetch each thread's replies and scan them alongside the parent message. Costs one extra API call per threaded message.",
|
|
536
526
|
)
|
|
537
|
-
channels: SlackOptionalChannels | None = None
|
|
538
|
-
time_range: SlackOptionalTimeRange | None = None
|
|
539
|
-
ingestion: SlackOptionalIngestion | None = None
|
|
540
527
|
|
|
541
528
|
|
|
542
529
|
class ObjectStorageOptionalScope(BaseModel):
|
|
@@ -2130,30 +2117,6 @@ class CoreInput(BaseModel):
|
|
|
2130
2117
|
)
|
|
2131
2118
|
|
|
2132
2119
|
|
|
2133
|
-
class SlackInput(CoreInput):
|
|
2134
|
-
type: Literal['SLACK'] | None = Field(
|
|
2135
|
-
None, description='Type of the asset or source'
|
|
2136
|
-
)
|
|
2137
|
-
required: SlackRequired
|
|
2138
|
-
masked: SlackMaskedBotToken | SlackMaskedUserToken | SlackMaskedToken = Field(
|
|
2139
|
-
..., title='SlackMasked'
|
|
2140
|
-
)
|
|
2141
|
-
optional: SlackOptional | None = None
|
|
2142
|
-
detectors: list[Detector] | None = Field(
|
|
2143
|
-
None, description='Detectors to run on ingested content'
|
|
2144
|
-
)
|
|
2145
|
-
custom_detectors: list[CustomDetectorSelection] | None = Field(
|
|
2146
|
-
None,
|
|
2147
|
-
description='Reusable custom detector IDs selected from the custom detector catalog.',
|
|
2148
|
-
)
|
|
2149
|
-
sampling: SamplingConfig
|
|
2150
|
-
resources: ResourceOverrides | None = None
|
|
2151
|
-
cleanup_removed_detector_findings: bool | None = Field(
|
|
2152
|
-
True,
|
|
2153
|
-
description='When enabled (default), findings produced by detectors that are no longer configured on this source (removed or disabled) are automatically resolved at the start of the next run, keeping the findings list in step with the current detector set.',
|
|
2154
|
-
)
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
2120
|
class EmailInput(CoreInput):
|
|
2158
2121
|
type: Literal['EMAIL'] | None = Field(
|
|
2159
2122
|
None, description='Type of the asset or source'
|
|
@@ -3461,27 +3424,31 @@ class SASLUsernamePassword1(BaseModel):
|
|
|
3461
3424
|
)
|
|
3462
3425
|
sasl_username: str = Field(..., description='SASL username')
|
|
3463
3426
|
sasl_password: str = Field(..., description='SASL password')
|
|
3464
|
-
|
|
3427
|
+
ca_certificate: str | None = Field(
|
|
3465
3428
|
None,
|
|
3466
|
-
description="PEM-encoded CA certificate used to verify the broker's TLS certificate. Required for private or self-signed CAs; leave empty to trust the system CA bundle.",
|
|
3429
|
+
description="PEM-encoded CA certificate used to verify the broker's TLS certificate — the provider's ca.pem. Required for private or self-signed CAs; leave empty to trust the system CA bundle.",
|
|
3467
3430
|
)
|
|
3468
3431
|
|
|
3469
3432
|
|
|
3470
3433
|
class ClientCertificateMTLS1(BaseModel):
|
|
3471
3434
|
"""
|
|
3472
|
-
mTLS credentials
|
|
3435
|
+
mTLS credentials, named as the providers name them: Aiven's access key / access certificate / CA certificate, Confluent's client key / client certificate. Paste each PEM block whole, including its BEGIN and END lines.
|
|
3473
3436
|
"""
|
|
3474
3437
|
|
|
3475
3438
|
model_config = ConfigDict(
|
|
3476
3439
|
extra='forbid',
|
|
3477
3440
|
)
|
|
3478
|
-
|
|
3479
|
-
|
|
3480
|
-
|
|
3441
|
+
access_key: str = Field(
|
|
3442
|
+
...,
|
|
3443
|
+
description='PEM-encoded client private key — the block starting with -----BEGIN PRIVATE KEY----- (Aiven: service.key)',
|
|
3481
3444
|
)
|
|
3482
|
-
|
|
3445
|
+
access_certificate: str = Field(
|
|
3446
|
+
...,
|
|
3447
|
+
description='PEM-encoded client certificate — the block starting with -----BEGIN CERTIFICATE----- (Aiven: service.cert)',
|
|
3448
|
+
)
|
|
3449
|
+
ca_certificate: str | None = Field(
|
|
3483
3450
|
None,
|
|
3484
|
-
description="PEM-encoded CA certificate used to verify the broker's TLS certificate. Required for private or self-signed CAs; leave empty to trust the system CA bundle.",
|
|
3451
|
+
description="PEM-encoded CA certificate used to verify the broker's TLS certificate — the provider's ca.pem. Required for private or self-signed CAs; leave empty to trust the system CA bundle.",
|
|
3485
3452
|
)
|
|
3486
3453
|
|
|
3487
3454
|
|
|
@@ -4221,39 +4188,43 @@ class GoogleWorkspaceInput(CoreInput):
|
|
|
4221
4188
|
)
|
|
4222
4189
|
|
|
4223
4190
|
|
|
4224
|
-
class
|
|
4191
|
+
class DropboxOAuthRefreshTokenRecommended(BaseModel):
|
|
4225
4192
|
"""
|
|
4226
|
-
|
|
4193
|
+
Recommended. An OAuth app key and secret plus a refresh token, which never expires — the client mints a fresh access token on every run. Obtain the refresh token once with `classifyre dropbox-auth --app-key <key> --app-secret <secret>`; it prints a URL you open in your own browser and exchanges the code it gives you.
|
|
4227
4194
|
"""
|
|
4228
4195
|
|
|
4229
4196
|
model_config = ConfigDict(
|
|
4230
4197
|
extra='forbid',
|
|
4231
4198
|
)
|
|
4232
|
-
auth_method: Literal['
|
|
4199
|
+
auth_method: Literal['oauth']
|
|
4200
|
+
app_key: str = Field(
|
|
4201
|
+
..., description='Dropbox app key (App Console → Settings → App key)'
|
|
4202
|
+
)
|
|
4233
4203
|
|
|
4234
4204
|
|
|
4235
|
-
class
|
|
4205
|
+
class DropboxOAuthWithPKCENoAppSecret(BaseModel):
|
|
4236
4206
|
"""
|
|
4237
|
-
|
|
4207
|
+
Same as the OAuth refresh token flow but for apps that hold no secret: the authorization is proved with PKCE, so only the app key is needed to refresh. Obtain the refresh token with `classifyre dropbox-auth --app-key <key> --pkce`.
|
|
4238
4208
|
"""
|
|
4239
4209
|
|
|
4240
4210
|
model_config = ConfigDict(
|
|
4241
4211
|
extra='forbid',
|
|
4242
4212
|
)
|
|
4243
|
-
auth_method: Literal['
|
|
4213
|
+
auth_method: Literal['oauth_pkce']
|
|
4244
4214
|
app_key: str = Field(
|
|
4245
|
-
...,
|
|
4246
|
-
description='Dropbox app key (client ID) of the OAuth app that issued the refresh token',
|
|
4215
|
+
..., description='Dropbox app key (App Console → Settings → App key)'
|
|
4247
4216
|
)
|
|
4248
4217
|
|
|
4249
4218
|
|
|
4250
|
-
class
|
|
4219
|
+
class DropboxAccessTokenShortLivedTestingOnly(BaseModel):
|
|
4220
|
+
"""
|
|
4221
|
+
Testing only. The token generated by the App Console button is short-lived and stops working about four hours later with `invalid_access_token`, so scheduled scans will fail. Use one of the refresh-token methods for anything recurring.
|
|
4222
|
+
"""
|
|
4223
|
+
|
|
4251
4224
|
model_config = ConfigDict(
|
|
4252
4225
|
extra='forbid',
|
|
4253
4226
|
)
|
|
4254
|
-
|
|
4255
|
-
..., description='Dropbox OAuth 2.0 access token (sl.… for short-lived tokens)'
|
|
4256
|
-
)
|
|
4227
|
+
auth_method: Literal['access_token']
|
|
4257
4228
|
|
|
4258
4229
|
|
|
4259
4230
|
class DropboxMaskedOAuth(BaseModel):
|
|
@@ -4261,12 +4232,31 @@ class DropboxMaskedOAuth(BaseModel):
|
|
|
4261
4232
|
extra='forbid',
|
|
4262
4233
|
)
|
|
4263
4234
|
app_secret: str = Field(
|
|
4235
|
+
..., description='Dropbox app secret (App Console → Settings → App secret)'
|
|
4236
|
+
)
|
|
4237
|
+
refresh_token: str = Field(
|
|
4264
4238
|
...,
|
|
4265
|
-
description='
|
|
4239
|
+
description='Long-lived refresh token issued with token_access_type=offline',
|
|
4240
|
+
)
|
|
4241
|
+
|
|
4242
|
+
|
|
4243
|
+
class DropboxMaskedOAuthPkce(BaseModel):
|
|
4244
|
+
model_config = ConfigDict(
|
|
4245
|
+
extra='forbid',
|
|
4266
4246
|
)
|
|
4267
4247
|
refresh_token: str = Field(
|
|
4268
4248
|
...,
|
|
4269
|
-
description='
|
|
4249
|
+
description='Long-lived refresh token issued by the PKCE flow with token_access_type=offline',
|
|
4250
|
+
)
|
|
4251
|
+
|
|
4252
|
+
|
|
4253
|
+
class DropboxMaskedAccessToken(BaseModel):
|
|
4254
|
+
model_config = ConfigDict(
|
|
4255
|
+
extra='forbid',
|
|
4256
|
+
)
|
|
4257
|
+
access_token: str = Field(
|
|
4258
|
+
...,
|
|
4259
|
+
description='Short-lived Dropbox access token (starts with sl.). Expires roughly four hours after it is generated.',
|
|
4270
4260
|
)
|
|
4271
4261
|
|
|
4272
4262
|
|
|
@@ -4386,11 +4376,13 @@ class DropboxInput(CoreInput):
|
|
|
4386
4376
|
type: Literal['DROPBOX'] | None = Field(
|
|
4387
4377
|
None, description='Type of the asset or source'
|
|
4388
4378
|
)
|
|
4389
|
-
required:
|
|
4390
|
-
|
|
4391
|
-
|
|
4392
|
-
|
|
4393
|
-
|
|
4379
|
+
required: (
|
|
4380
|
+
DropboxOAuthRefreshTokenRecommended
|
|
4381
|
+
| DropboxOAuthWithPKCENoAppSecret
|
|
4382
|
+
| DropboxAccessTokenShortLivedTestingOnly
|
|
4383
|
+
) = Field(..., title='DropboxRequired')
|
|
4384
|
+
masked: DropboxMaskedOAuth | DropboxMaskedOAuthPkce | DropboxMaskedAccessToken = (
|
|
4385
|
+
Field(..., title='DropboxMasked')
|
|
4394
4386
|
)
|
|
4395
4387
|
optional: DropboxOptional | None = None
|
|
4396
4388
|
detectors: list[Detector] | None = Field(
|
|
@@ -4408,6 +4400,32 @@ class DropboxInput(CoreInput):
|
|
|
4408
4400
|
)
|
|
4409
4401
|
|
|
4410
4402
|
|
|
4403
|
+
class SlackOptionalAttachments(BaseModel):
|
|
4404
|
+
"""
|
|
4405
|
+
Files shared in messages. Each file becomes its own asset, and its text is extracted with the same parser used for local files (PDF, Office, images via OCR, audio and video transcription).
|
|
4406
|
+
"""
|
|
4407
|
+
|
|
4408
|
+
model_config = ConfigDict(
|
|
4409
|
+
extra='forbid',
|
|
4410
|
+
)
|
|
4411
|
+
include_attachments: bool | None = Field(
|
|
4412
|
+
True, description='Emit files shared in messages as scannable assets'
|
|
4413
|
+
)
|
|
4414
|
+
max_attachment_bytes: int | None = Field(
|
|
4415
|
+
26214400,
|
|
4416
|
+
description='Skip files larger than this many bytes (default 25 MB)',
|
|
4417
|
+
ge=1024,
|
|
4418
|
+
le=524288000,
|
|
4419
|
+
)
|
|
4420
|
+
include_file_extensions: list[str] | None = Field(
|
|
4421
|
+
None,
|
|
4422
|
+
description='Only download files with these extensions (example: .pdf, .docx). Empty means all extensions.',
|
|
4423
|
+
)
|
|
4424
|
+
exclude_file_extensions: list[str] | None = Field(
|
|
4425
|
+
None, description='Skip files with these extensions (example: .mp4, .zip)'
|
|
4426
|
+
)
|
|
4427
|
+
|
|
4428
|
+
|
|
4411
4429
|
class YouTubeInput(CoreInput):
|
|
4412
4430
|
type: Literal['YOUTUBE'] | None = Field(
|
|
4413
4431
|
None, description='Type of the asset or source'
|
|
@@ -4430,6 +4448,38 @@ class YouTubeInput(CoreInput):
|
|
|
4430
4448
|
)
|
|
4431
4449
|
|
|
4432
4450
|
|
|
4451
|
+
class SlackOptional(BaseModel):
|
|
4452
|
+
model_config = ConfigDict(
|
|
4453
|
+
extra='forbid',
|
|
4454
|
+
)
|
|
4455
|
+
channels: SlackOptionalChannels | None = None
|
|
4456
|
+
time_range: SlackOptionalTimeRange | None = None
|
|
4457
|
+
ingestion: SlackOptionalIngestion | None = None
|
|
4458
|
+
attachments: SlackOptionalAttachments | None = None
|
|
4459
|
+
|
|
4460
|
+
|
|
4461
|
+
class SlackInput(CoreInput):
|
|
4462
|
+
type: Literal['SLACK'] | None = Field(
|
|
4463
|
+
None, description='Type of the asset or source'
|
|
4464
|
+
)
|
|
4465
|
+
required: SlackRequired
|
|
4466
|
+
masked: SlackMasked
|
|
4467
|
+
optional: SlackOptional | None = None
|
|
4468
|
+
detectors: list[Detector] | None = Field(
|
|
4469
|
+
None, description='Detectors to run on ingested content'
|
|
4470
|
+
)
|
|
4471
|
+
custom_detectors: list[CustomDetectorSelection] | None = Field(
|
|
4472
|
+
None,
|
|
4473
|
+
description='Reusable custom detector IDs selected from the custom detector catalog.',
|
|
4474
|
+
)
|
|
4475
|
+
sampling: SamplingConfig
|
|
4476
|
+
resources: ResourceOverrides | None = None
|
|
4477
|
+
cleanup_removed_detector_findings: bool | None = Field(
|
|
4478
|
+
True,
|
|
4479
|
+
description='When enabled (default), findings produced by detectors that are no longer configured on this source (removed or disabled) are automatically resolved at the start of the next run, keeping the findings list in step with the current detector set.',
|
|
4480
|
+
)
|
|
4481
|
+
|
|
4482
|
+
|
|
4433
4483
|
class DeltaLakeOptional(BaseModel):
|
|
4434
4484
|
model_config = ConfigDict(
|
|
4435
4485
|
extra='forbid',
|