classifyre-cli 0.4.45__tar.gz → 0.4.46__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- classifyre_cli-0.4.46/.turbo/turbo-build.log +3 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/PKG-INFO +1 -1
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/package.json +1 -1
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/pyproject.toml +1 -1
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/models/generated_input.py +84 -3
- classifyre_cli-0.4.46/src/sources/local_folder/__init__.py +3 -0
- classifyre_cli-0.4.46/src/sources/local_folder/source.py +146 -0
- classifyre_cli-0.4.46/tests/test_local_folder_source.py +335 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/uv.lock +2 -2
- classifyre_cli-0.4.45/.turbo/turbo-build.log +0 -3
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/.gitignore +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/.python-version +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/README.md +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/main.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/scripts/generate_models.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/config.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/base.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/broken_links/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/broken_links/detector.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/config.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/content/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/detector.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/extractor.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/_base.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/_factory.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/_feature_extraction.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/_gliner2.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/_image_classification.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/_llm.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/_object_detection.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/_regex.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/runners/_text_classification.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/custom/trainer.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/dependencies.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/pii/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/pii/detector.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/secrets/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/secrets/detector.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/threat/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/threat/code_security_detector.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/detectors/threat/yara_detector.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/main.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/models/generated_detectors.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/models/generated_single_asset_scan_results.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/outputs/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/outputs/base.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/outputs/console.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/outputs/factory.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/outputs/file.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/outputs/rest.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/pipeline/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/pipeline/content_provider.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/pipeline/detector_pipeline.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/pipeline/parsed_content_provider.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/pipeline/worker_pool.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sandbox/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sandbox/runner.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/asset_metadata.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/atlassian_common.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/azure_blob_storage/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/azure_blob_storage/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/base.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/confluence/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/confluence/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/databricks/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/databricks/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/delta_lake/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/delta_lake/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/dependencies.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/elasticsearch/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/elasticsearch/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/email/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/email/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/google_cloud_storage/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/google_cloud_storage/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/hive/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/hive/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/iceberg/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/iceberg/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/jira/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/jira/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/kafka/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/kafka/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/lakehouse_base.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/meilisearch/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/meilisearch/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/mongodb/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/mongodb/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/mssql/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/mssql/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/mysql/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/mysql/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/neo4j/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/neo4j/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/notion/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/notion/client.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/notion/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/object_storage/base.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/opensearch/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/opensearch/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/oracle/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/oracle/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/postgresql/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/postgresql/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/powerbi/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/powerbi/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/recipe_normalizer.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/s3_client.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/s3_compatible_storage/README.md +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/s3_compatible_storage/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/s3_compatible_storage/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/search_engine_base.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/servicedesk/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/servicedesk/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/slack/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/slack/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/snowflake/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/snowflake/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/sqlite/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/sqlite/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/tableau/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/tableau/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/tabular_base.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/tabular_utils.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/wordpress/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/wordpress/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/youtube/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/sources/youtube/source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/telemetry.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/content_extraction.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/dependency_groups.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/embedded_images.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/file_metadata.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/file_parser.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/file_to_images.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/hashing.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/resources.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/transcription.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/uv_sync.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/src/utils/validation.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/_lakehouse_fakes.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/conftest.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/broken_links/test_broken_links_detector.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/conftest.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/content/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/custom/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/custom/conftest.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/custom/test_invoice_extraction.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/custom/test_llm_runner.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/custom/test_pipeline_integration.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/custom/test_regex_runner.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/custom/test_transformer_runners.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/pii/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/pii/conftest.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/pii/sample_invoice.pdf +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/pii/test_pii_detector.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/pii/test_pii_detector_extended.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/secrets/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/secrets/test_secrets_detector.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/secrets/test_secrets_detector_extended.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/test_base_detector.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/test_custom_detector_examples_runtime.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/test_detector_catalog_commercial.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/test_detector_pipeline_types.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/test_detector_schema_examples.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/test_detector_types.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/test_phase2_detectors.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/test_registry.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/threat/__init__.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/threat/test_code_security_detector.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/detectors/threat/test_yara_detector.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/integration/test_wordpress_broken_links_detector.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/integration/test_wordpress_links_assets.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/pipeline/test_detector_pipeline.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/pipeline/test_worker_pool.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_assets_metadata_catalog.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_azure_blob_storage_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_base_source_attachment.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_base_source_sampling.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_config.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_confluence_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_custom_extractor.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_databricks_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_delta_lake_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_dependency_groups.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_elasticsearch_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_email_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_google_cloud_storage_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_hashing.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_hive_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_iceberg_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_jira_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_kafka_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_meilisearch_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_mongodb_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_mssql_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_mysql_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_neo4j_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_notion_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_opensearch_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_oracle_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_outputs.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_postgresql_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_powerbi_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_recipe_normalizer.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_s3_compatible_storage_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_sampling_automatic.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_sandbox_runner.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_servicedesk_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_slack_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_snowflake_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_source_dependency_groups.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_sqlite_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_tableau_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_tabular_automatic_sampling.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_tabular_utils.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_uv_sync.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_wordpress_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_youtube_source.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/test_youtube_source_integration.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/utils/test_content_extraction.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/utils/test_embedded_images.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/utils/test_file_metadata.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/utils/test_file_parser.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/utils/test_file_to_images.py +0 -0
- {classifyre_cli-0.4.45 → classifyre_cli-0.4.46}/tests/utils/test_transcription.py +0 -0
|
@@ -51,6 +51,7 @@ class AssetType(StrEnum):
|
|
|
51
51
|
ELASTICSEARCH = 'ELASTICSEARCH'
|
|
52
52
|
OPENSEARCH = 'OPENSEARCH'
|
|
53
53
|
MEILISEARCH = 'MEILISEARCH'
|
|
54
|
+
LOCAL_FOLDER = 'LOCAL_FOLDER'
|
|
54
55
|
|
|
55
56
|
|
|
56
57
|
class DetectorType(StrEnum):
|
|
@@ -355,6 +356,7 @@ class Type(StrEnum):
|
|
|
355
356
|
ELASTICSEARCH = 'ELASTICSEARCH'
|
|
356
357
|
OPENSEARCH = 'OPENSEARCH'
|
|
357
358
|
MEILISEARCH = 'MEILISEARCH'
|
|
359
|
+
LOCAL_FOLDER = 'LOCAL_FOLDER'
|
|
358
360
|
|
|
359
361
|
|
|
360
362
|
class YouTubeRequired(BaseModel):
|
|
@@ -644,6 +646,64 @@ class S3CompatibleStorageOptional(BaseModel):
|
|
|
644
646
|
scope: ObjectStorageOptionalScope | None = None
|
|
645
647
|
|
|
646
648
|
|
|
649
|
+
class LocalFolderRequired(BaseModel):
|
|
650
|
+
model_config = ConfigDict(
|
|
651
|
+
extra='forbid',
|
|
652
|
+
)
|
|
653
|
+
path: str = Field(
|
|
654
|
+
...,
|
|
655
|
+
description='Absolute path of the local folder to scan (for example, /Users/jane/Documents)',
|
|
656
|
+
)
|
|
657
|
+
|
|
658
|
+
|
|
659
|
+
class LocalFolderMasked(BaseModel):
|
|
660
|
+
"""
|
|
661
|
+
Local folder scans run with the desktop application's filesystem permissions; no credentials are required.
|
|
662
|
+
"""
|
|
663
|
+
|
|
664
|
+
model_config = ConfigDict(
|
|
665
|
+
extra='forbid',
|
|
666
|
+
)
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
class LocalFolderOptionalTraversal(BaseModel):
|
|
670
|
+
"""
|
|
671
|
+
Directory traversal controls.
|
|
672
|
+
"""
|
|
673
|
+
|
|
674
|
+
model_config = ConfigDict(
|
|
675
|
+
extra='forbid',
|
|
676
|
+
)
|
|
677
|
+
follow_symlinks: bool | None = Field(
|
|
678
|
+
False,
|
|
679
|
+
description='Follow symbolic links while traversing (loops are not detected; leave off unless needed)',
|
|
680
|
+
)
|
|
681
|
+
include_hidden: bool | None = Field(
|
|
682
|
+
False,
|
|
683
|
+
description='Include hidden files and dot-directories (names starting with a dot)',
|
|
684
|
+
)
|
|
685
|
+
max_depth: int | None = Field(
|
|
686
|
+
None,
|
|
687
|
+
description='Maximum directory depth below the selected folder (unset scans all levels)',
|
|
688
|
+
ge=1,
|
|
689
|
+
le=64,
|
|
690
|
+
)
|
|
691
|
+
max_file_bytes: int | None = Field(
|
|
692
|
+
10485760,
|
|
693
|
+
description='Maximum bytes read per file for MIME detection and text extraction',
|
|
694
|
+
ge=1024,
|
|
695
|
+
le=104857600,
|
|
696
|
+
)
|
|
697
|
+
|
|
698
|
+
|
|
699
|
+
class LocalFolderOptional(BaseModel):
|
|
700
|
+
model_config = ConfigDict(
|
|
701
|
+
extra='forbid',
|
|
702
|
+
)
|
|
703
|
+
traversal: LocalFolderOptionalTraversal | None = None
|
|
704
|
+
scope: ObjectStorageOptionalScope | None = None
|
|
705
|
+
|
|
706
|
+
|
|
647
707
|
class AzureBlobStorageRequired(BaseModel):
|
|
648
708
|
model_config = ConfigDict(
|
|
649
709
|
extra='forbid',
|
|
@@ -2100,6 +2160,24 @@ class S3CompatibleStorageInput(CoreInput):
|
|
|
2100
2160
|
resources: ResourceOverrides | None = None
|
|
2101
2161
|
|
|
2102
2162
|
|
|
2163
|
+
class LocalFolderInput(CoreInput):
|
|
2164
|
+
type: Literal['LOCAL_FOLDER'] | None = Field(
|
|
2165
|
+
None, description='Type of the asset or source'
|
|
2166
|
+
)
|
|
2167
|
+
required: LocalFolderRequired
|
|
2168
|
+
masked: LocalFolderMasked | None = None
|
|
2169
|
+
optional: LocalFolderOptional | None = None
|
|
2170
|
+
detectors: list[Detector] | None = Field(
|
|
2171
|
+
None, description='Detectors to run on ingested content'
|
|
2172
|
+
)
|
|
2173
|
+
custom_detectors: list[CustomDetectorSelection] | None = Field(
|
|
2174
|
+
None,
|
|
2175
|
+
description='Reusable custom detector IDs selected from the custom detector catalog.',
|
|
2176
|
+
)
|
|
2177
|
+
sampling: SamplingConfig
|
|
2178
|
+
resources: ResourceOverrides | None = None
|
|
2179
|
+
|
|
2180
|
+
|
|
2103
2181
|
class AzureBlobStorageInput(CoreInput):
|
|
2104
2182
|
type: Literal['AZURE_BLOB_STORAGE'] | None = Field(
|
|
2105
2183
|
None, description='Type of the asset or source'
|
|
@@ -2531,7 +2609,7 @@ class ConfluenceOptionalConnection(BaseModel):
|
|
|
2531
2609
|
)
|
|
2532
2610
|
|
|
2533
2611
|
|
|
2534
|
-
class
|
|
2612
|
+
class Type19(StrEnum):
|
|
2535
2613
|
"""
|
|
2536
2614
|
Filter spaces by space type
|
|
2537
2615
|
"""
|
|
@@ -2568,7 +2646,7 @@ class ConfluenceOptionalScopeSpaces(BaseModel):
|
|
|
2568
2646
|
keys: list[str] | None = Field(
|
|
2569
2647
|
None, description='Filter spaces by keys (up to 250)', max_length=250
|
|
2570
2648
|
)
|
|
2571
|
-
type:
|
|
2649
|
+
type: Type19 | None = Field(None, description='Filter spaces by space type')
|
|
2572
2650
|
status: Status | None = Field(None, description='Filter spaces by status')
|
|
2573
2651
|
labels: list[str] | None = Field(
|
|
2574
2652
|
None,
|
|
@@ -2834,7 +2912,7 @@ class ServiceDeskOptional(BaseModel):
|
|
|
2834
2912
|
content: ServiceDeskOptionalContent | None = None
|
|
2835
2913
|
|
|
2836
2914
|
|
|
2837
|
-
class
|
|
2915
|
+
class Type20(StrEnum):
|
|
2838
2916
|
"""
|
|
2839
2917
|
Type of the asset or source
|
|
2840
2918
|
"""
|
|
@@ -2868,6 +2946,7 @@ class Type19(StrEnum):
|
|
|
2868
2946
|
ELASTICSEARCH = 'ELASTICSEARCH'
|
|
2869
2947
|
OPENSEARCH = 'OPENSEARCH'
|
|
2870
2948
|
MEILISEARCH = 'MEILISEARCH'
|
|
2949
|
+
LOCAL_FOLDER = 'LOCAL_FOLDER'
|
|
2871
2950
|
|
|
2872
2951
|
|
|
2873
2952
|
class ConfluenceInput(CoreInput):
|
|
@@ -3682,6 +3761,7 @@ class SourceInput(
|
|
|
3682
3761
|
RootModel[
|
|
3683
3762
|
SlackInput
|
|
3684
3763
|
| S3CompatibleStorageInput
|
|
3764
|
+
| LocalFolderInput
|
|
3685
3765
|
| AzureBlobStorageInput
|
|
3686
3766
|
| GoogleCloudStorageInput
|
|
3687
3767
|
| PostgreSQLInput
|
|
@@ -3714,6 +3794,7 @@ class SourceInput(
|
|
|
3714
3794
|
root: (
|
|
3715
3795
|
SlackInput
|
|
3716
3796
|
| S3CompatibleStorageInput
|
|
3797
|
+
| LocalFolderInput
|
|
3717
3798
|
| AzureBlobStorageInput
|
|
3718
3799
|
| GoogleCloudStorageInput
|
|
3719
3800
|
| PostgreSQLInput
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
from collections.abc import Iterator
|
|
6
|
+
from datetime import UTC, datetime
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
from urllib.parse import quote
|
|
10
|
+
|
|
11
|
+
from ...models.generated_input import LocalFolderInput
|
|
12
|
+
from ..object_storage.base import ObjectRef, ObjectStorageSourceBase
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class LocalFolderSource(ObjectStorageSourceBase):
|
|
18
|
+
"""Scans a folder on the local filesystem (desktop application deployments).
|
|
19
|
+
|
|
20
|
+
Reuses the object-storage pipeline: each file becomes an ObjectRef whose key
|
|
21
|
+
is the path relative to the scanned root, so sampling, MIME resolution, text
|
|
22
|
+
extraction, file metadata, and embedded-image child assets all come from
|
|
23
|
+
ObjectStorageSourceBase.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
source_type = "local_folder"
|
|
27
|
+
provider_label = "LOCAL_FOLDER"
|
|
28
|
+
input_model = LocalFolderInput
|
|
29
|
+
|
|
30
|
+
DEFAULT_MAX_FILE_BYTES = 10 * 1024 * 1024
|
|
31
|
+
|
|
32
|
+
_resolved_root: Path | None = None
|
|
33
|
+
|
|
34
|
+
def _root(self) -> Path:
|
|
35
|
+
if self._resolved_root is not None:
|
|
36
|
+
return self._resolved_root
|
|
37
|
+
raw = str(self.config.required.path).strip()
|
|
38
|
+
if not raw:
|
|
39
|
+
raise ValueError("required.path must be set")
|
|
40
|
+
root = Path(raw).expanduser()
|
|
41
|
+
if not root.is_absolute():
|
|
42
|
+
raise ValueError(f"required.path must be an absolute path, got: {raw}")
|
|
43
|
+
if not root.exists():
|
|
44
|
+
raise ValueError(f"Folder does not exist: {root}")
|
|
45
|
+
if not root.is_dir():
|
|
46
|
+
raise ValueError(f"Path is not a folder: {root}")
|
|
47
|
+
self._resolved_root = root.resolve()
|
|
48
|
+
return self._resolved_root
|
|
49
|
+
|
|
50
|
+
def _traversal_option(self, key: str, default: Any = None) -> Any:
|
|
51
|
+
optional = self.config.optional
|
|
52
|
+
traversal = getattr(optional, "traversal", None) if optional else None
|
|
53
|
+
if traversal is not None:
|
|
54
|
+
value = getattr(traversal, key, None)
|
|
55
|
+
if value is not None:
|
|
56
|
+
return value
|
|
57
|
+
return default
|
|
58
|
+
|
|
59
|
+
def _follow_symlinks(self) -> bool:
|
|
60
|
+
return bool(self._traversal_option("follow_symlinks", False))
|
|
61
|
+
|
|
62
|
+
def _include_hidden(self) -> bool:
|
|
63
|
+
return bool(self._traversal_option("include_hidden", False))
|
|
64
|
+
|
|
65
|
+
def _max_depth(self) -> int | None:
|
|
66
|
+
value = self._traversal_option("max_depth")
|
|
67
|
+
try:
|
|
68
|
+
return int(value) if value is not None else None
|
|
69
|
+
except (TypeError, ValueError):
|
|
70
|
+
return None
|
|
71
|
+
|
|
72
|
+
def _max_file_bytes(self) -> int:
|
|
73
|
+
value = self._traversal_option("max_file_bytes", self.DEFAULT_MAX_FILE_BYTES)
|
|
74
|
+
try:
|
|
75
|
+
parsed = int(value)
|
|
76
|
+
except (TypeError, ValueError):
|
|
77
|
+
return self.DEFAULT_MAX_FILE_BYTES
|
|
78
|
+
return max(parsed, 1024)
|
|
79
|
+
|
|
80
|
+
def _is_hidden(self, name: str) -> bool:
|
|
81
|
+
return name.startswith(".")
|
|
82
|
+
|
|
83
|
+
def _walk(self, directory: Path, depth: int) -> Iterator[ObjectRef]:
|
|
84
|
+
max_depth = self._max_depth()
|
|
85
|
+
follow_symlinks = self._follow_symlinks()
|
|
86
|
+
include_hidden = self._include_hidden()
|
|
87
|
+
|
|
88
|
+
try:
|
|
89
|
+
entries = sorted(os.scandir(directory), key=lambda entry: entry.name)
|
|
90
|
+
except OSError as exc:
|
|
91
|
+
logger.warning("Skipping unreadable directory %s: %s", directory, exc)
|
|
92
|
+
return
|
|
93
|
+
|
|
94
|
+
for entry in entries:
|
|
95
|
+
if not include_hidden and self._is_hidden(entry.name):
|
|
96
|
+
continue
|
|
97
|
+
try:
|
|
98
|
+
if entry.is_dir(follow_symlinks=follow_symlinks):
|
|
99
|
+
if max_depth is None or depth < max_depth:
|
|
100
|
+
yield from self._walk(Path(entry.path), depth + 1)
|
|
101
|
+
continue
|
|
102
|
+
if not entry.is_file(follow_symlinks=follow_symlinks):
|
|
103
|
+
continue
|
|
104
|
+
stat = entry.stat(follow_symlinks=follow_symlinks)
|
|
105
|
+
except OSError as exc:
|
|
106
|
+
logger.warning("Skipping unreadable entry %s: %s", entry.path, exc)
|
|
107
|
+
continue
|
|
108
|
+
|
|
109
|
+
key = Path(entry.path).relative_to(self._root()).as_posix()
|
|
110
|
+
if not self._object_matches_extension_filters(key):
|
|
111
|
+
continue
|
|
112
|
+
prefix = self._prefix()
|
|
113
|
+
if prefix and not key.startswith(prefix):
|
|
114
|
+
continue
|
|
115
|
+
if stat.st_size == 0 and not self._include_empty_objects():
|
|
116
|
+
continue
|
|
117
|
+
|
|
118
|
+
yield ObjectRef(
|
|
119
|
+
key=key,
|
|
120
|
+
size=stat.st_size,
|
|
121
|
+
last_modified=datetime.fromtimestamp(stat.st_mtime, tz=UTC),
|
|
122
|
+
# Change signature: mtime+size, so edits produce a new checksum
|
|
123
|
+
# without hashing file bytes during discovery.
|
|
124
|
+
etag=f"{stat.st_mtime_ns:x}-{stat.st_size:x}",
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
def _list_objects(self) -> Iterator[ObjectRef]:
|
|
128
|
+
yield from self._walk(self._root(), depth=0)
|
|
129
|
+
|
|
130
|
+
def _download_object(self, ref: ObjectRef) -> tuple[bytes, str | None]:
|
|
131
|
+
file_path = self._root() / ref.key
|
|
132
|
+
max_bytes = self._max_file_bytes()
|
|
133
|
+
with open(file_path, "rb") as handle:
|
|
134
|
+
file_bytes = handle.read(max_bytes)
|
|
135
|
+
if ref.size > max_bytes:
|
|
136
|
+
logger.info(
|
|
137
|
+
"Truncated %s to %d of %d bytes for content extraction",
|
|
138
|
+
file_path,
|
|
139
|
+
max_bytes,
|
|
140
|
+
ref.size,
|
|
141
|
+
)
|
|
142
|
+
return file_bytes, None
|
|
143
|
+
|
|
144
|
+
def _external_url(self, key: str) -> str:
|
|
145
|
+
absolute = (self._root() / key).as_posix()
|
|
146
|
+
return f"file://{quote(absolute)}"
|
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
from src.models.generated_single_asset_scan_results import AssetType as OutputAssetType
|
|
9
|
+
from src.sources.local_folder.source import LocalFolderSource
|
|
10
|
+
|
|
11
|
+
# Minimal valid 1x1 PNG (8-byte signature + IHDR/IDAT/IEND chunks).
|
|
12
|
+
_PNG_BYTES = bytes.fromhex(
|
|
13
|
+
"89504e470d0a1a0a0000000d494844520000000100000001080600000"
|
|
14
|
+
"01f15c4890000000a49444154789c6300010000050001a5f645400000000049454e44ae426082"
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _recipe(path: Path, *, optional: dict | None = None, sampling: dict | None = None) -> dict:
|
|
19
|
+
return {
|
|
20
|
+
"type": "LOCAL_FOLDER",
|
|
21
|
+
"required": {"path": str(path)},
|
|
22
|
+
"masked": {},
|
|
23
|
+
"optional": optional or {},
|
|
24
|
+
"sampling": sampling or {"strategy": "ALL"},
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _make_source(path: Path, *, optional: dict | None = None, sampling: dict | None = None):
|
|
29
|
+
return LocalFolderSource(
|
|
30
|
+
_recipe(path, optional=optional, sampling=sampling),
|
|
31
|
+
source_id="s",
|
|
32
|
+
runner_id="r",
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
async def _extract_assets(source: LocalFolderSource) -> list:
|
|
37
|
+
assets = []
|
|
38
|
+
async for batch in source.extract_raw():
|
|
39
|
+
assets.extend(batch)
|
|
40
|
+
return assets
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _keys(assets: list) -> set[str]:
|
|
44
|
+
return {a.metadata["object_key"] for a in assets}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
# ---------------------------------------------------------------------------
|
|
48
|
+
# 1. Nested traversal
|
|
49
|
+
# ---------------------------------------------------------------------------
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@pytest.mark.asyncio
|
|
53
|
+
async def test_nested_traversal_returns_posix_relative_keys(tmp_path: Path):
|
|
54
|
+
(tmp_path / "top.txt").write_text("top level")
|
|
55
|
+
nested_dir = tmp_path / "a" / "b"
|
|
56
|
+
nested_dir.mkdir(parents=True)
|
|
57
|
+
(nested_dir / "deep.txt").write_text("deep file")
|
|
58
|
+
|
|
59
|
+
source = _make_source(tmp_path)
|
|
60
|
+
assets = await _extract_assets(source)
|
|
61
|
+
|
|
62
|
+
assert _keys(assets) == {"top.txt", "a/b/deep.txt"}
|
|
63
|
+
names = {a.name for a in assets}
|
|
64
|
+
assert names == {"top.txt", "deep.txt"}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# ---------------------------------------------------------------------------
|
|
68
|
+
# 2. Hidden files/dirs
|
|
69
|
+
# ---------------------------------------------------------------------------
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@pytest.mark.asyncio
|
|
73
|
+
async def test_hidden_files_skipped_by_default(tmp_path: Path):
|
|
74
|
+
(tmp_path / "visible.txt").write_text("visible")
|
|
75
|
+
(tmp_path / ".hidden.txt").write_text("hidden")
|
|
76
|
+
hidden_dir = tmp_path / ".hiddendir"
|
|
77
|
+
hidden_dir.mkdir()
|
|
78
|
+
(hidden_dir / "inside.txt").write_text("inside hidden dir")
|
|
79
|
+
|
|
80
|
+
source = _make_source(tmp_path)
|
|
81
|
+
assets = await _extract_assets(source)
|
|
82
|
+
|
|
83
|
+
assert _keys(assets) == {"visible.txt"}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@pytest.mark.asyncio
|
|
87
|
+
async def test_hidden_files_included_when_configured(tmp_path: Path):
|
|
88
|
+
(tmp_path / "visible.txt").write_text("visible")
|
|
89
|
+
(tmp_path / ".hidden.txt").write_text("hidden")
|
|
90
|
+
hidden_dir = tmp_path / ".hiddendir"
|
|
91
|
+
hidden_dir.mkdir()
|
|
92
|
+
(hidden_dir / "inside.txt").write_text("inside hidden dir")
|
|
93
|
+
|
|
94
|
+
source = _make_source(tmp_path, optional={"traversal": {"include_hidden": True}})
|
|
95
|
+
assets = await _extract_assets(source)
|
|
96
|
+
|
|
97
|
+
assert _keys(assets) == {"visible.txt", ".hidden.txt", ".hiddendir/inside.txt"}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# ---------------------------------------------------------------------------
|
|
101
|
+
# 3. include/exclude extensions
|
|
102
|
+
# ---------------------------------------------------------------------------
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@pytest.mark.asyncio
|
|
106
|
+
async def test_include_extensions_filters_to_allowlist(tmp_path: Path):
|
|
107
|
+
(tmp_path / "a.txt").write_text("a")
|
|
108
|
+
(tmp_path / "b.csv").write_text("b,c\n1,2\n")
|
|
109
|
+
(tmp_path / "c.md").write_text("# markdown")
|
|
110
|
+
|
|
111
|
+
source = _make_source(tmp_path, optional={"scope": {"include_extensions": [".txt", ".csv"]}})
|
|
112
|
+
assets = await _extract_assets(source)
|
|
113
|
+
|
|
114
|
+
assert _keys(assets) == {"a.txt", "b.csv"}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@pytest.mark.asyncio
|
|
118
|
+
async def test_exclude_extensions_filters_out_denylist(tmp_path: Path):
|
|
119
|
+
(tmp_path / "a.txt").write_text("a")
|
|
120
|
+
(tmp_path / "b.csv").write_text("b,c\n1,2\n")
|
|
121
|
+
(tmp_path / "c.md").write_text("# markdown")
|
|
122
|
+
|
|
123
|
+
source = _make_source(tmp_path, optional={"scope": {"exclude_extensions": [".md"]}})
|
|
124
|
+
assets = await _extract_assets(source)
|
|
125
|
+
|
|
126
|
+
assert _keys(assets) == {"a.txt", "b.csv"}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
# ---------------------------------------------------------------------------
|
|
130
|
+
# 4. Empty files
|
|
131
|
+
# ---------------------------------------------------------------------------
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@pytest.mark.asyncio
|
|
135
|
+
async def test_empty_files_skipped_by_default(tmp_path: Path):
|
|
136
|
+
(tmp_path / "empty.txt").touch()
|
|
137
|
+
(tmp_path / "nonempty.txt").write_text("content")
|
|
138
|
+
|
|
139
|
+
source = _make_source(tmp_path)
|
|
140
|
+
assets = await _extract_assets(source)
|
|
141
|
+
|
|
142
|
+
assert _keys(assets) == {"nonempty.txt"}
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@pytest.mark.asyncio
|
|
146
|
+
async def test_empty_files_included_when_configured(tmp_path: Path):
|
|
147
|
+
(tmp_path / "empty.txt").touch()
|
|
148
|
+
(tmp_path / "nonempty.txt").write_text("content")
|
|
149
|
+
|
|
150
|
+
source = _make_source(tmp_path, optional={"scope": {"include_empty_objects": True}})
|
|
151
|
+
assets = await _extract_assets(source)
|
|
152
|
+
|
|
153
|
+
assert _keys(assets) == {"empty.txt", "nonempty.txt"}
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
# ---------------------------------------------------------------------------
|
|
157
|
+
# 5. max_depth
|
|
158
|
+
# ---------------------------------------------------------------------------
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
@pytest.mark.asyncio
|
|
162
|
+
async def test_max_depth_limits_recursion(tmp_path: Path):
|
|
163
|
+
(tmp_path / "root.txt").write_text("root")
|
|
164
|
+
level1 = tmp_path / "l1"
|
|
165
|
+
level1.mkdir()
|
|
166
|
+
(level1 / "one.txt").write_text("one")
|
|
167
|
+
level2 = level1 / "l2"
|
|
168
|
+
level2.mkdir()
|
|
169
|
+
(level2 / "two.txt").write_text("two")
|
|
170
|
+
|
|
171
|
+
source = _make_source(tmp_path, optional={"traversal": {"max_depth": 1}})
|
|
172
|
+
assets = await _extract_assets(source)
|
|
173
|
+
|
|
174
|
+
assert _keys(assets) == {"root.txt", "l1/one.txt"}
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
# ---------------------------------------------------------------------------
|
|
178
|
+
# 6. max_file_bytes truncation
|
|
179
|
+
# ---------------------------------------------------------------------------
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def test_download_object_truncates_to_max_file_bytes(tmp_path: Path):
|
|
183
|
+
big_file = tmp_path / "big.bin"
|
|
184
|
+
big_file.write_bytes(b"x" * 3000)
|
|
185
|
+
|
|
186
|
+
source = _make_source(tmp_path, optional={"traversal": {"max_file_bytes": 1024}})
|
|
187
|
+
refs = list(source._list_objects())
|
|
188
|
+
assert len(refs) == 1
|
|
189
|
+
|
|
190
|
+
file_bytes, content_type_hint = source._download_object(refs[0])
|
|
191
|
+
|
|
192
|
+
assert len(file_bytes) == 1024
|
|
193
|
+
assert content_type_hint is None
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
# ---------------------------------------------------------------------------
|
|
197
|
+
# 7. external_url / hash determinism / etag & checksum change on edit
|
|
198
|
+
# ---------------------------------------------------------------------------
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
@pytest.mark.asyncio
|
|
202
|
+
async def test_external_url_and_hash_determinism(tmp_path: Path):
|
|
203
|
+
file_path = tmp_path / "note.txt"
|
|
204
|
+
file_path.write_text("hello")
|
|
205
|
+
|
|
206
|
+
source_a = _make_source(tmp_path)
|
|
207
|
+
assets_a = await _extract_assets(source_a)
|
|
208
|
+
assert len(assets_a) == 1
|
|
209
|
+
asset_a = assets_a[0]
|
|
210
|
+
|
|
211
|
+
assert asset_a.external_url.startswith("file://")
|
|
212
|
+
assert asset_a.external_url == f"file://{file_path.resolve().as_posix()}"
|
|
213
|
+
|
|
214
|
+
source_b = _make_source(tmp_path)
|
|
215
|
+
assets_b = await _extract_assets(source_b)
|
|
216
|
+
asset_b = assets_b[0]
|
|
217
|
+
|
|
218
|
+
assert asset_a.hash == asset_b.hash
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
@pytest.mark.asyncio
|
|
222
|
+
async def test_etag_and_checksum_change_on_content_and_mtime_change(tmp_path: Path):
|
|
223
|
+
file_path = tmp_path / "note.txt"
|
|
224
|
+
file_path.write_text("hello")
|
|
225
|
+
os.utime(file_path, (1_700_000_000, 1_700_000_000))
|
|
226
|
+
|
|
227
|
+
source_before = _make_source(tmp_path)
|
|
228
|
+
assets_before = await _extract_assets(source_before)
|
|
229
|
+
asset_before = assets_before[0]
|
|
230
|
+
|
|
231
|
+
file_path.write_text("hello world, now longer")
|
|
232
|
+
os.utime(file_path, (1_700_001_000, 1_700_001_000))
|
|
233
|
+
|
|
234
|
+
source_after = _make_source(tmp_path)
|
|
235
|
+
assets_after = await _extract_assets(source_after)
|
|
236
|
+
asset_after = assets_after[0]
|
|
237
|
+
|
|
238
|
+
# Hash is derived from the (unchanged) external_url/key, so it stays stable...
|
|
239
|
+
assert asset_before.hash == asset_after.hash
|
|
240
|
+
# ...but the etag (mtime+size signature) and checksum must reflect the edit.
|
|
241
|
+
assert asset_before.metadata["etag"] != asset_after.metadata["etag"]
|
|
242
|
+
assert asset_before.checksum != asset_after.checksum
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
# ---------------------------------------------------------------------------
|
|
246
|
+
# 8. asset_type mapping
|
|
247
|
+
# ---------------------------------------------------------------------------
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
@pytest.mark.asyncio
|
|
251
|
+
async def test_asset_type_mapping_by_extension(tmp_path: Path):
|
|
252
|
+
(tmp_path / "doc.txt").write_text("plain text")
|
|
253
|
+
(tmp_path / "table.csv").write_text("a,b\n1,2\n")
|
|
254
|
+
(tmp_path / "image.png").write_bytes(_PNG_BYTES)
|
|
255
|
+
|
|
256
|
+
source = _make_source(tmp_path)
|
|
257
|
+
assets = await _extract_assets(source)
|
|
258
|
+
by_key = {a.metadata["object_key"]: a for a in assets}
|
|
259
|
+
|
|
260
|
+
assert by_key["doc.txt"].asset_type == OutputAssetType.TXT
|
|
261
|
+
assert by_key["table.csv"].asset_type == OutputAssetType.TABLE
|
|
262
|
+
assert by_key["image.png"].asset_type == OutputAssetType.IMAGE
|
|
263
|
+
assert by_key["image.png"].asset_kind == "image"
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
# ---------------------------------------------------------------------------
|
|
267
|
+
# 9. Metadata contract (asset_kind + catalog-validated metadata)
|
|
268
|
+
# ---------------------------------------------------------------------------
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
@pytest.mark.asyncio
|
|
272
|
+
async def test_metadata_contract_for_file_asset(tmp_path: Path):
|
|
273
|
+
(tmp_path / "doc.txt").write_text("plain text")
|
|
274
|
+
|
|
275
|
+
source = _make_source(tmp_path)
|
|
276
|
+
assets = await _extract_assets(source)
|
|
277
|
+
asset = assets[0]
|
|
278
|
+
|
|
279
|
+
assert asset.asset_kind == "file"
|
|
280
|
+
assert asset.metadata["provider"] == "LOCAL_FOLDER"
|
|
281
|
+
assert asset.metadata["object_key"] == "doc.txt"
|
|
282
|
+
assert asset.metadata["size_bytes"] == len("plain text")
|
|
283
|
+
assert "mime_type" in asset.metadata
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
# ---------------------------------------------------------------------------
|
|
287
|
+
# 10. test_connection
|
|
288
|
+
# ---------------------------------------------------------------------------
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def test_connection_success_for_valid_directory(tmp_path: Path):
|
|
292
|
+
(tmp_path / "a.txt").write_text("a")
|
|
293
|
+
|
|
294
|
+
source = _make_source(tmp_path)
|
|
295
|
+
result = source.test_connection()
|
|
296
|
+
|
|
297
|
+
assert result["status"] == "SUCCESS"
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def test_connection_failure_for_missing_directory(tmp_path: Path):
|
|
301
|
+
missing = tmp_path / "does-not-exist"
|
|
302
|
+
|
|
303
|
+
source = _make_source(missing)
|
|
304
|
+
result = source.test_connection()
|
|
305
|
+
|
|
306
|
+
assert result["status"] == "FAILURE"
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
# ---------------------------------------------------------------------------
|
|
310
|
+
# 11. Sampling
|
|
311
|
+
# ---------------------------------------------------------------------------
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
@pytest.mark.asyncio
|
|
315
|
+
async def test_sampling_random_respects_rows_per_page(tmp_path: Path):
|
|
316
|
+
# SamplingConfig.rows_per_page has a hard floor of 10 (ge=10), so exercise the
|
|
317
|
+
# floor value against a larger population rather than requesting 1 row.
|
|
318
|
+
for i in range(15):
|
|
319
|
+
(tmp_path / f"file{i}.txt").write_text(f"content {i}")
|
|
320
|
+
|
|
321
|
+
source = _make_source(tmp_path, sampling={"strategy": "RANDOM", "rows_per_page": 10})
|
|
322
|
+
assets = await _extract_assets(source)
|
|
323
|
+
|
|
324
|
+
assert len(assets) == 10
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
@pytest.mark.asyncio
|
|
328
|
+
async def test_sampling_latest_respects_rows_per_page(tmp_path: Path):
|
|
329
|
+
for i in range(15):
|
|
330
|
+
(tmp_path / f"file{i}.txt").write_text(f"content {i}")
|
|
331
|
+
|
|
332
|
+
source = _make_source(tmp_path, sampling={"strategy": "LATEST", "rows_per_page": 10})
|
|
333
|
+
assets = await _extract_assets(source)
|
|
334
|
+
|
|
335
|
+
assert len(assets) == 10
|
|
@@ -721,7 +721,7 @@ wheels = [
|
|
|
721
721
|
|
|
722
722
|
[[package]]
|
|
723
723
|
name = "classifyre-cli"
|
|
724
|
-
version = "0.4.
|
|
724
|
+
version = "0.4.46"
|
|
725
725
|
source = { editable = "." }
|
|
726
726
|
dependencies = [
|
|
727
727
|
{ name = "beautifulsoup4" },
|
|
@@ -1151,7 +1151,7 @@ youtube = [
|
|
|
1151
1151
|
|
|
1152
1152
|
[[package]]
|
|
1153
1153
|
name = "classifyre-schemas"
|
|
1154
|
-
version = "0.4.
|
|
1154
|
+
version = "0.4.46"
|
|
1155
1155
|
source = { editable = "../../packages/schemas" }
|
|
1156
1156
|
dependencies = [
|
|
1157
1157
|
{ name = "fastjsonschema" },
|
|
File without changes
|