classifyre-cli 0.4.56__tar.gz → 0.4.57__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/.turbo/turbo-build.log +1 -1
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/PKG-INFO +3 -3
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/README.md +2 -2
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/package.json +1 -1
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/pyproject.toml +1 -1
- classifyre_cli-0.4.57/src/file_evaluation/__init__.py +5 -0
- {classifyre_cli-0.4.56/src/sandbox → classifyre_cli-0.4.57/src/file_evaluation}/runner.py +23 -17
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/main.py +15 -11
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/models/generated_detectors.py +1 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/models/generated_input.py +54 -5
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/object_storage/base.py +10 -5
- classifyre_cli-0.4.57/src/sources/sandbox/__init__.py +3 -0
- classifyre_cli-0.4.57/src/sources/sandbox/source.py +119 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/utils/embedded_images.py +1 -1
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/utils/file_parser.py +1 -1
- classifyre_cli-0.4.56/tests/test_sandbox_runner.py → classifyre_cli-0.4.57/tests/test_file_evaluation_runner.py +4 -4
- classifyre_cli-0.4.57/tests/test_sandbox_source.py +134 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/uv.lock +149 -157
- classifyre_cli-0.4.56/src/sandbox/__init__.py +0 -5
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/.gitignore +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/.python-version +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/main.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/scripts/generate_models.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/config.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/base.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/broken_links/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/broken_links/detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/config.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/content/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/custom/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/custom/detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/custom/extractor.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/custom/runners/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/custom/runners/_base.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/custom/runners/_factory.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/custom/runners/_gliner2.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/custom/runners/_image_classification.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/custom/runners/_llm.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/custom/runners/_object_detection.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/custom/runners/_regex.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/custom/runners/_text_classification.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/custom/trainer.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/dependencies.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/pii/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/pii/detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/secrets/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/secrets/detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/threat/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/threat/code_security_detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/detectors/threat/yara_detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/models/generated_single_asset_scan_results.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/outputs/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/outputs/base.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/outputs/console.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/outputs/factory.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/outputs/file.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/outputs/rest.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/pipeline/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/pipeline/content_provider.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/pipeline/detector_pipeline.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/pipeline/parsed_content_provider.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/pipeline/text_artifact.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/pipeline/worker_pool.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/asset_metadata.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/atlassian_common.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/azure_blob_storage/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/azure_blob_storage/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/base.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/confluence/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/confluence/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/databricks/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/databricks/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/delta_lake/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/delta_lake/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/dependencies.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/elasticsearch/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/elasticsearch/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/email/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/email/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/google_cloud_storage/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/google_cloud_storage/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/google_workspace/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/google_workspace/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/hive/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/hive/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/iceberg/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/iceberg/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/jira/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/jira/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/kafka/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/kafka/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/lakehouse_base.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/local_folder/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/local_folder/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/meilisearch/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/meilisearch/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/microsoft_365/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/microsoft_365/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/mongodb/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/mongodb/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/mssql/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/mssql/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/mysql/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/mysql/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/neo4j/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/neo4j/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/notion/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/notion/client.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/notion/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/opensearch/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/opensearch/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/oracle/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/oracle/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/postgresql/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/postgresql/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/powerbi/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/powerbi/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/recipe_normalizer.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/s3_client.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/s3_compatible_storage/README.md +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/s3_compatible_storage/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/s3_compatible_storage/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/search_engine_base.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/servicedesk/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/servicedesk/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/slack/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/slack/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/snowflake/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/snowflake/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/sqlite/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/sqlite/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/tableau/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/tableau/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/tabular_base.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/tabular_utils.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/wordpress/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/wordpress/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/youtube/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/sources/youtube/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/telemetry.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/utils/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/utils/content_extraction.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/utils/dependency_groups.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/utils/file_metadata.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/utils/file_to_images.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/utils/hashing.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/utils/resources.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/utils/transcription.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/utils/uv_sync.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/utils/validation.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/src/utils/video_processing.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/_lakehouse_fakes.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/conftest.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/broken_links/test_broken_links_detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/conftest.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/content/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/custom/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/custom/conftest.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/custom/test_gliner2_classification.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/custom/test_invoice_extraction.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/custom/test_llm_runner.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/custom/test_pipeline_integration.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/custom/test_regex_runner.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/custom/test_runner_hardening.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/custom/test_transformer_runners.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/pii/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/pii/conftest.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/pii/sample_invoice.pdf +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/pii/test_pii_chunking.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/pii/test_pii_detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/pii/test_pii_detector_extended.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/pii/test_pii_severity.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/secrets/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/secrets/test_secrets_detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/secrets/test_secrets_detector_extended.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/test_base_detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/test_custom_detector_examples_runtime.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/test_detector_catalog_commercial.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/test_detector_pipeline_types.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/test_detector_schema_examples.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/test_detector_types.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/test_phase2_detectors.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/test_registry.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/threat/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/threat/test_code_security_detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/detectors/threat/test_yara_detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/integration/test_wordpress_broken_links_detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/integration/test_wordpress_links_assets.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/pipeline/test_detector_outcomes.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/pipeline/test_detector_pipeline.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/pipeline/test_worker_pool.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_assets_metadata_catalog.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_azure_blob_storage_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_base_source_attachment.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_base_source_sampling.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_config.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_confluence_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_custom_extractor.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_databricks_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_delta_lake_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_dependency_groups.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_elasticsearch_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_email_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_google_cloud_storage_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_google_workspace_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_hashing.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_hive_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_iceberg_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_jira_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_kafka_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_local_folder_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_meilisearch_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_microsoft_365_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_mongodb_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_mssql_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_mysql_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_neo4j_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_notion_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_opensearch_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_oracle_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_outputs.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_postgresql_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_powerbi_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_recipe_normalizer.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_s3_compatible_storage_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_sampling_automatic.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_servicedesk_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_slack_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_snowflake_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_source_dependency_groups.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_source_strategy_conformance.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_sqlite_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_tableau_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_tabular_automatic_sampling.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_tabular_utils.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_uv_sync.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_wordpress_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_youtube_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/test_youtube_source_integration.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/utils/test_content_extraction.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/utils/test_embedded_images.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/utils/test_file_metadata.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/utils/test_file_parser.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/utils/test_file_to_images.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/utils/test_transcription.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.57}/tests/utils/test_video_processing.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: classifyre-cli
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.57
|
|
4
4
|
Summary: Classifyre CLI — scan and classify unstructured data sources
|
|
5
5
|
License: MIT
|
|
6
6
|
Keywords: data,ingestion,metadata,pii,secrets,unstructured
|
|
@@ -54,7 +54,7 @@ Commands:
|
|
|
54
54
|
- `test` - test source connection.
|
|
55
55
|
- `discover` - discover source resources.
|
|
56
56
|
- `extract` - run extraction and emit batched output.
|
|
57
|
-
- `
|
|
57
|
+
- `evaluate-file` - internally evaluate detectors against one local file.
|
|
58
58
|
|
|
59
59
|
## Extract Output Model
|
|
60
60
|
|
|
@@ -78,7 +78,7 @@ Default behavior:
|
|
|
78
78
|
Global/common:
|
|
79
79
|
|
|
80
80
|
- `--debug` - enable debug logging.
|
|
81
|
-
- `--detectors-file <path>` -
|
|
81
|
+
- `--detectors-file <path>` - file evaluation only.
|
|
82
82
|
|
|
83
83
|
Extract output options:
|
|
84
84
|
|
|
@@ -37,7 +37,7 @@ Commands:
|
|
|
37
37
|
- `test` - test source connection.
|
|
38
38
|
- `discover` - discover source resources.
|
|
39
39
|
- `extract` - run extraction and emit batched output.
|
|
40
|
-
- `
|
|
40
|
+
- `evaluate-file` - internally evaluate detectors against one local file.
|
|
41
41
|
|
|
42
42
|
## Extract Output Model
|
|
43
43
|
|
|
@@ -61,7 +61,7 @@ Default behavior:
|
|
|
61
61
|
Global/common:
|
|
62
62
|
|
|
63
63
|
- `--debug` - enable debug logging.
|
|
64
|
-
- `--detectors-file <path>` -
|
|
64
|
+
- `--detectors-file <path>` - file evaluation only.
|
|
65
65
|
|
|
66
66
|
Extract output options:
|
|
67
67
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""FileEvaluationRunner: run detectors on an ephemeral local file."""
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
@@ -15,7 +15,7 @@ from ..utils.file_parser import ParsedFile, parse_file
|
|
|
15
15
|
logger = logging.getLogger(__name__)
|
|
16
16
|
|
|
17
17
|
|
|
18
|
-
class
|
|
18
|
+
class FileEvaluationRunner:
|
|
19
19
|
"""Run a set of detectors against a single local file."""
|
|
20
20
|
|
|
21
21
|
def __init__(self, detectors_config: list[dict[str, Any]]) -> None:
|
|
@@ -41,7 +41,7 @@ class SandboxRunner:
|
|
|
41
41
|
|
|
42
42
|
detector = get_detector(detector_name, typed_config)
|
|
43
43
|
detectors.append(detector)
|
|
44
|
-
logger.info(f"Initialized
|
|
44
|
+
logger.info(f"Initialized file-evaluation detector: {detector_name}")
|
|
45
45
|
except Exception as e:
|
|
46
46
|
logger.error(f"Failed to initialize detector {detector_type}: {e}")
|
|
47
47
|
|
|
@@ -121,19 +121,23 @@ class SandboxRunner:
|
|
|
121
121
|
and self._is_binary_detector(detector)
|
|
122
122
|
and self._supports_mime(supported, mime_type)
|
|
123
123
|
):
|
|
124
|
-
logger.info(
|
|
124
|
+
logger.info(
|
|
125
|
+
"File evaluation: dispatching '%s' on file bytes (%s)", label, mime_type
|
|
126
|
+
)
|
|
125
127
|
tasks.append(detector.detect(_get_raw_bytes(), mime_type))
|
|
126
128
|
active_detectors.append(detector)
|
|
127
129
|
# Text delivery: text detectors get the extracted text layer.
|
|
128
130
|
elif "text/plain" in supported and text.strip():
|
|
129
131
|
logger.info(
|
|
130
|
-
"
|
|
132
|
+
"File evaluation: dispatching '%s' on extracted text (%d chars)",
|
|
133
|
+
label,
|
|
134
|
+
len(text),
|
|
131
135
|
)
|
|
132
136
|
tasks.append(detector.detect(text, "text/plain"))
|
|
133
137
|
active_detectors.append(detector)
|
|
134
138
|
else:
|
|
135
139
|
logger.info(
|
|
136
|
-
"
|
|
140
|
+
"File evaluation: '%s' not dispatched on whole file (supports=%s, file_mime=%s)",
|
|
137
141
|
label,
|
|
138
142
|
supported,
|
|
139
143
|
file_mime,
|
|
@@ -158,18 +162,20 @@ class SandboxRunner:
|
|
|
158
162
|
for detector, result in zip(active_detectors, results, strict=False):
|
|
159
163
|
label = self._detector_label(detector)
|
|
160
164
|
if isinstance(result, Exception):
|
|
161
|
-
logger.error("
|
|
165
|
+
logger.error("File evaluation: '%s' failed on whole file: %s", label, result)
|
|
162
166
|
continue
|
|
163
167
|
if isinstance(result, list):
|
|
164
168
|
detector_findings = [f for f in result if isinstance(f, DetectionResult)]
|
|
165
169
|
logger.info(
|
|
166
|
-
"
|
|
170
|
+
"File evaluation: '%s' on whole file → %d finding(s)",
|
|
171
|
+
label,
|
|
172
|
+
len(detector_findings),
|
|
167
173
|
)
|
|
168
174
|
for finding in detector_findings:
|
|
169
175
|
all_findings.append(
|
|
170
176
|
finding.model_copy(
|
|
171
177
|
update={
|
|
172
|
-
"runner_id": "
|
|
178
|
+
"runner_id": "file-evaluation",
|
|
173
179
|
"detected_at": detected_at,
|
|
174
180
|
}
|
|
175
181
|
)
|
|
@@ -203,14 +209,14 @@ class SandboxRunner:
|
|
|
203
209
|
binary_detectors = [d for d in detectors if self._is_binary_detector(d)]
|
|
204
210
|
if not binary_detectors:
|
|
205
211
|
logger.info(
|
|
206
|
-
"
|
|
212
|
+
"File evaluation: %s embeds images but no image-capable detectors are configured; "
|
|
207
213
|
"skipping embedded-image scan",
|
|
208
214
|
mime_type,
|
|
209
215
|
)
|
|
210
216
|
return []
|
|
211
217
|
|
|
212
218
|
logger.info(
|
|
213
|
-
"
|
|
219
|
+
"File evaluation: scanning images embedded in %s with image detectors: [%s]",
|
|
214
220
|
mime_type,
|
|
215
221
|
", ".join(self._detector_label(d) for d in binary_detectors),
|
|
216
222
|
)
|
|
@@ -227,14 +233,14 @@ class SandboxRunner:
|
|
|
227
233
|
]
|
|
228
234
|
if not compatible:
|
|
229
235
|
logger.info(
|
|
230
|
-
"
|
|
236
|
+
"File evaluation: embedded image %s (%s, %.0f KB) — no compatible detector",
|
|
231
237
|
image.location,
|
|
232
238
|
image.mime_type,
|
|
233
239
|
size_kb,
|
|
234
240
|
)
|
|
235
241
|
continue
|
|
236
242
|
logger.info(
|
|
237
|
-
"
|
|
243
|
+
"File evaluation: embedded image %s (%s, %.0f KB) → dispatching [%s]",
|
|
238
244
|
image.location,
|
|
239
245
|
image.mime_type,
|
|
240
246
|
size_kb,
|
|
@@ -246,7 +252,7 @@ class SandboxRunner:
|
|
|
246
252
|
result = await detector.detect(image.image_bytes, image.mime_type)
|
|
247
253
|
except Exception as exc:
|
|
248
254
|
logger.error(
|
|
249
|
-
"
|
|
255
|
+
"File evaluation: '%s' failed on embedded image %s: %s",
|
|
250
256
|
label,
|
|
251
257
|
image.location,
|
|
252
258
|
exc,
|
|
@@ -258,7 +264,7 @@ class SandboxRunner:
|
|
|
258
264
|
else []
|
|
259
265
|
)
|
|
260
266
|
logger.info(
|
|
261
|
-
"
|
|
267
|
+
"File evaluation: '%s' on embedded image %s → %d finding(s)",
|
|
262
268
|
label,
|
|
263
269
|
image.location,
|
|
264
270
|
len(detector_findings),
|
|
@@ -267,7 +273,7 @@ class SandboxRunner:
|
|
|
267
273
|
findings.append(self._tag_embedded(finding, image.location, detected_at))
|
|
268
274
|
|
|
269
275
|
logger.info(
|
|
270
|
-
"
|
|
276
|
+
"File evaluation: embedded-image scan complete — %d image(s) scanned, %d finding(s)",
|
|
271
277
|
image_count,
|
|
272
278
|
len(findings),
|
|
273
279
|
)
|
|
@@ -296,7 +302,7 @@ class SandboxRunner:
|
|
|
296
302
|
location = location.model_copy(update={"path": location_label})
|
|
297
303
|
return finding.model_copy(
|
|
298
304
|
update={
|
|
299
|
-
"runner_id": "
|
|
305
|
+
"runner_id": "file-evaluation",
|
|
300
306
|
"detected_at": detected_at,
|
|
301
307
|
"metadata": metadata,
|
|
302
308
|
"location": location,
|
|
@@ -131,6 +131,10 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
|
|
|
131
131
|
source_id = args.source_id or os.environ.get("SOURCE_ID")
|
|
132
132
|
if source_id:
|
|
133
133
|
os.environ["SOURCE_ID"] = source_id
|
|
134
|
+
if getattr(args, "output_rest_url", None):
|
|
135
|
+
# Sources such as SANDBOX use the same managed-job API base URL as the
|
|
136
|
+
# REST output sink to list and stream their input content.
|
|
137
|
+
os.environ["CLASSIFYRE_OUTPUT_REST_URL"] = args.output_rest_url
|
|
134
138
|
|
|
135
139
|
try:
|
|
136
140
|
try:
|
|
@@ -461,13 +465,13 @@ def run_train_command(args: argparse.Namespace) -> None:
|
|
|
461
465
|
print(json.dumps(result.to_dict()))
|
|
462
466
|
|
|
463
467
|
|
|
464
|
-
def
|
|
465
|
-
"""
|
|
466
|
-
from .
|
|
468
|
+
def run_evaluate_file_command(args: argparse.Namespace) -> None:
|
|
469
|
+
"""Evaluate detector configs against one ephemeral local file."""
|
|
470
|
+
from .file_evaluation import FileEvaluationRunner
|
|
467
471
|
|
|
468
472
|
file_path_str: str | None = args.recipe
|
|
469
473
|
if not file_path_str:
|
|
470
|
-
logger.error("
|
|
474
|
+
logger.error("evaluate-file requires a file path as the first argument")
|
|
471
475
|
sys.exit(1)
|
|
472
476
|
|
|
473
477
|
file_path = Path(file_path_str)
|
|
@@ -492,7 +496,7 @@ def run_sandbox_command(args: argparse.Namespace) -> None:
|
|
|
492
496
|
sys.exit(1)
|
|
493
497
|
|
|
494
498
|
try:
|
|
495
|
-
runner =
|
|
499
|
+
runner = FileEvaluationRunner(detectors)
|
|
496
500
|
parsed, findings = runner.run(file_path)
|
|
497
501
|
if parsed.parse_error:
|
|
498
502
|
logger.warning("File parse warning: %s", parsed.parse_error)
|
|
@@ -504,7 +508,7 @@ def run_sandbox_command(args: argparse.Namespace) -> None:
|
|
|
504
508
|
output["parse_error"] = parsed.parse_error
|
|
505
509
|
print(json.dumps(_sanitize_for_json(output), ensure_ascii=False))
|
|
506
510
|
except Exception as e:
|
|
507
|
-
logger.error("
|
|
511
|
+
logger.error("File evaluation failed: %s", e, exc_info=True)
|
|
508
512
|
sys.exit(1)
|
|
509
513
|
|
|
510
514
|
|
|
@@ -520,14 +524,14 @@ def main() -> None:
|
|
|
520
524
|
parser = argparse.ArgumentParser(description="Classifyre Metadata Extraction CLI")
|
|
521
525
|
parser.add_argument(
|
|
522
526
|
"command",
|
|
523
|
-
choices=["test", "extract", "discover", "
|
|
527
|
+
choices=["test", "extract", "discover", "evaluate-file", "train"],
|
|
524
528
|
help="Command to run",
|
|
525
529
|
)
|
|
526
530
|
parser.add_argument(
|
|
527
531
|
"recipe",
|
|
528
532
|
nargs="?",
|
|
529
533
|
default=None,
|
|
530
|
-
help="Path to recipe JSON (or file path for
|
|
534
|
+
help="Path to recipe JSON (or file path for evaluate-file)",
|
|
531
535
|
)
|
|
532
536
|
parser.add_argument("--debug", action="store_true", help="Enable debug logging")
|
|
533
537
|
parser.add_argument(
|
|
@@ -565,7 +569,7 @@ def main() -> None:
|
|
|
565
569
|
parser.add_argument(
|
|
566
570
|
"--detectors-file",
|
|
567
571
|
default=None,
|
|
568
|
-
help="Path to JSON file with detector configs (
|
|
572
|
+
help="Path to JSON file with detector configs (evaluate-file only)",
|
|
569
573
|
)
|
|
570
574
|
# train-command arguments
|
|
571
575
|
parser.add_argument(
|
|
@@ -629,8 +633,8 @@ def main() -> None:
|
|
|
629
633
|
if args.debug:
|
|
630
634
|
logging.getLogger().setLevel(logging.DEBUG)
|
|
631
635
|
|
|
632
|
-
if args.command == "
|
|
633
|
-
|
|
636
|
+
if args.command == "evaluate-file":
|
|
637
|
+
run_evaluate_file_command(args)
|
|
634
638
|
return
|
|
635
639
|
|
|
636
640
|
if args.command == "train":
|
|
@@ -54,6 +54,7 @@ class AssetType(StrEnum):
|
|
|
54
54
|
LOCAL_FOLDER = 'LOCAL_FOLDER'
|
|
55
55
|
MICROSOFT_365 = 'MICROSOFT_365'
|
|
56
56
|
GOOGLE_WORKSPACE = 'GOOGLE_WORKSPACE'
|
|
57
|
+
SANDBOX = 'SANDBOX'
|
|
57
58
|
|
|
58
59
|
|
|
59
60
|
class DetectorType(StrEnum):
|
|
@@ -353,6 +354,7 @@ class Type(StrEnum):
|
|
|
353
354
|
LOCAL_FOLDER = 'LOCAL_FOLDER'
|
|
354
355
|
MICROSOFT_365 = 'MICROSOFT_365'
|
|
355
356
|
GOOGLE_WORKSPACE = 'GOOGLE_WORKSPACE'
|
|
357
|
+
SANDBOX = 'SANDBOX'
|
|
356
358
|
|
|
357
359
|
|
|
358
360
|
class YouTubeRequired(BaseModel):
|
|
@@ -648,6 +650,32 @@ class LocalFolderRequired(BaseModel):
|
|
|
648
650
|
)
|
|
649
651
|
|
|
650
652
|
|
|
653
|
+
class SandboxRequired(BaseModel):
|
|
654
|
+
"""
|
|
655
|
+
Sandbox files are uploaded separately through the source files API.
|
|
656
|
+
"""
|
|
657
|
+
|
|
658
|
+
model_config = ConfigDict(
|
|
659
|
+
extra='forbid',
|
|
660
|
+
)
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
class SandboxMasked(BaseModel):
|
|
664
|
+
"""
|
|
665
|
+
Sandbox sources do not require credentials.
|
|
666
|
+
"""
|
|
667
|
+
|
|
668
|
+
model_config = ConfigDict(
|
|
669
|
+
extra='forbid',
|
|
670
|
+
)
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
class SandboxOptional(BaseModel):
|
|
674
|
+
model_config = ConfigDict(
|
|
675
|
+
extra='forbid',
|
|
676
|
+
)
|
|
677
|
+
|
|
678
|
+
|
|
651
679
|
class LocalFolderMasked(BaseModel):
|
|
652
680
|
"""
|
|
653
681
|
Local folder scans run with the desktop application's filesystem permissions; no credentials are required.
|
|
@@ -2170,6 +2198,24 @@ class LocalFolderInput(CoreInput):
|
|
|
2170
2198
|
resources: ResourceOverrides | None = None
|
|
2171
2199
|
|
|
2172
2200
|
|
|
2201
|
+
class SandboxInput(CoreInput):
|
|
2202
|
+
type: Literal['SANDBOX'] | None = Field(
|
|
2203
|
+
None, description='Type of the asset or source'
|
|
2204
|
+
)
|
|
2205
|
+
required: SandboxRequired
|
|
2206
|
+
masked: SandboxMasked | None = None
|
|
2207
|
+
optional: SandboxOptional | None = None
|
|
2208
|
+
detectors: list[Detector] | None = Field(
|
|
2209
|
+
None, description='Detectors to run on ingested content'
|
|
2210
|
+
)
|
|
2211
|
+
custom_detectors: list[CustomDetectorSelection] | None = Field(
|
|
2212
|
+
None,
|
|
2213
|
+
description='Reusable custom detector IDs selected from the custom detector catalog.',
|
|
2214
|
+
)
|
|
2215
|
+
sampling: SamplingConfig
|
|
2216
|
+
resources: ResourceOverrides | None = None
|
|
2217
|
+
|
|
2218
|
+
|
|
2173
2219
|
class AzureBlobStorageInput(CoreInput):
|
|
2174
2220
|
type: Literal['AZURE_BLOB_STORAGE'] | None = Field(
|
|
2175
2221
|
None, description='Type of the asset or source'
|
|
@@ -2601,7 +2647,7 @@ class ConfluenceOptionalConnection(BaseModel):
|
|
|
2601
2647
|
)
|
|
2602
2648
|
|
|
2603
2649
|
|
|
2604
|
-
class
|
|
2650
|
+
class Type20(StrEnum):
|
|
2605
2651
|
"""
|
|
2606
2652
|
Filter spaces by space type
|
|
2607
2653
|
"""
|
|
@@ -2638,7 +2684,7 @@ class ConfluenceOptionalScopeSpaces(BaseModel):
|
|
|
2638
2684
|
keys: list[str] | None = Field(
|
|
2639
2685
|
None, description='Filter spaces by keys (up to 250)', max_length=250
|
|
2640
2686
|
)
|
|
2641
|
-
type:
|
|
2687
|
+
type: Type20 | None = Field(None, description='Filter spaces by space type')
|
|
2642
2688
|
status: Status | None = Field(None, description='Filter spaces by status')
|
|
2643
2689
|
labels: list[str] | None = Field(
|
|
2644
2690
|
None,
|
|
@@ -2904,7 +2950,7 @@ class ServiceDeskOptional(BaseModel):
|
|
|
2904
2950
|
content: ServiceDeskOptionalContent | None = None
|
|
2905
2951
|
|
|
2906
2952
|
|
|
2907
|
-
class
|
|
2953
|
+
class Type21(StrEnum):
|
|
2908
2954
|
"""
|
|
2909
2955
|
Type of the asset or source
|
|
2910
2956
|
"""
|
|
@@ -2941,6 +2987,7 @@ class Type20(StrEnum):
|
|
|
2941
2987
|
LOCAL_FOLDER = 'LOCAL_FOLDER'
|
|
2942
2988
|
MICROSOFT_365 = 'MICROSOFT_365'
|
|
2943
2989
|
GOOGLE_WORKSPACE = 'GOOGLE_WORKSPACE'
|
|
2990
|
+
SANDBOX = 'SANDBOX'
|
|
2944
2991
|
|
|
2945
2992
|
|
|
2946
2993
|
class ConfluenceInput(CoreInput):
|
|
@@ -4075,7 +4122,8 @@ class IcebergInput(CoreInput):
|
|
|
4075
4122
|
|
|
4076
4123
|
class SourceInput(
|
|
4077
4124
|
RootModel[
|
|
4078
|
-
|
|
4125
|
+
SandboxInput
|
|
4126
|
+
| SlackInput
|
|
4079
4127
|
| S3CompatibleStorageInput
|
|
4080
4128
|
| LocalFolderInput
|
|
4081
4129
|
| AzureBlobStorageInput
|
|
@@ -4110,7 +4158,8 @@ class SourceInput(
|
|
|
4110
4158
|
]
|
|
4111
4159
|
):
|
|
4112
4160
|
root: (
|
|
4113
|
-
|
|
4161
|
+
SandboxInput
|
|
4162
|
+
| SlackInput
|
|
4114
4163
|
| S3CompatibleStorageInput
|
|
4115
4164
|
| LocalFolderInput
|
|
4116
4165
|
| AzureBlobStorageInput
|
|
@@ -303,6 +303,10 @@ class ObjectStorageSourceBase(BaseSource, ABC):
|
|
|
303
303
|
def _file_extension(self, key: str) -> str:
|
|
304
304
|
return PurePosixPath(key).suffix.lower()
|
|
305
305
|
|
|
306
|
+
def _object_file_name(self, ref: ObjectRef) -> str:
|
|
307
|
+
"""File name used for MIME inference and normalized metadata."""
|
|
308
|
+
return ref.key
|
|
309
|
+
|
|
306
310
|
def _asset_type_from_mime_or_key(self, mime_type: str | None, key: str) -> OutputAssetType:
|
|
307
311
|
normalized_mime = (mime_type or "").split(";", maxsplit=1)[0].strip().lower()
|
|
308
312
|
extension = self._file_extension(key)
|
|
@@ -364,7 +368,7 @@ class ObjectStorageSourceBase(BaseSource, ABC):
|
|
|
364
368
|
if self._discovery_only or not self._include_content_preview():
|
|
365
369
|
mime = (ref.content_type_hint or "").split(";", maxsplit=1)[0].strip().lower()
|
|
366
370
|
if not mime:
|
|
367
|
-
mime = infer_mime_type_from_file_name(ref
|
|
371
|
+
mime = infer_mime_type_from_file_name(self._object_file_name(ref))
|
|
368
372
|
return ContentSnapshot(
|
|
369
373
|
mime_type=mime or "application/octet-stream",
|
|
370
374
|
raw_content="",
|
|
@@ -389,7 +393,7 @@ class ObjectStorageSourceBase(BaseSource, ABC):
|
|
|
389
393
|
mime_type = resolve_mime_type(
|
|
390
394
|
file_bytes,
|
|
391
395
|
declared_mime_type=content_type_hint or ref.content_type_hint or "",
|
|
392
|
-
file_name=ref
|
|
396
|
+
file_name=self._object_file_name(ref),
|
|
393
397
|
)
|
|
394
398
|
normalized_mime = mime_type.split(";", 1)[0].strip().lower()
|
|
395
399
|
|
|
@@ -414,7 +418,8 @@ class ObjectStorageSourceBase(BaseSource, ABC):
|
|
|
414
418
|
asset_hash = self.generate_hash_id(external_url)
|
|
415
419
|
|
|
416
420
|
snapshot = self._build_snapshot(ref)
|
|
417
|
-
|
|
421
|
+
file_name = self._object_file_name(ref)
|
|
422
|
+
asset_type = self._asset_type_from_mime_or_key(snapshot.mime_type, file_name)
|
|
418
423
|
|
|
419
424
|
if snapshot.text_content:
|
|
420
425
|
self._content_cache[asset_hash] = (snapshot.raw_content, snapshot.text_content)
|
|
@@ -459,14 +464,14 @@ class ObjectStorageSourceBase(BaseSource, ABC):
|
|
|
459
464
|
file_meta = extract_file_metadata(
|
|
460
465
|
snapshot.raw_bytes,
|
|
461
466
|
snapshot.mime_type,
|
|
462
|
-
file_name=
|
|
467
|
+
file_name=file_name,
|
|
463
468
|
)
|
|
464
469
|
asset_metadata.update({k: v for k, v in file_meta.items() if v is not None})
|
|
465
470
|
|
|
466
471
|
asset = SingleAssetScanResults(
|
|
467
472
|
hash=asset_hash,
|
|
468
473
|
checksum=self.calculate_checksum(metadata),
|
|
469
|
-
name=
|
|
474
|
+
name=file_name.split("/")[-1] or file_name,
|
|
470
475
|
external_url=external_url,
|
|
471
476
|
links=[],
|
|
472
477
|
asset_type=asset_type,
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
from collections.abc import Iterator
|
|
6
|
+
from datetime import UTC, datetime
|
|
7
|
+
from typing import Any
|
|
8
|
+
from urllib.parse import quote
|
|
9
|
+
|
|
10
|
+
import requests
|
|
11
|
+
|
|
12
|
+
from ...models.generated_input import SandboxInput
|
|
13
|
+
from ..object_storage.base import ObjectRef, ObjectStorageSourceBase
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class SandboxSource(ObjectStorageSourceBase):
|
|
19
|
+
"""Uploaded files exposed by the API as a normal ingestion source."""
|
|
20
|
+
|
|
21
|
+
source_type = "sandbox"
|
|
22
|
+
provider_label = "SANDBOX"
|
|
23
|
+
input_model = SandboxInput
|
|
24
|
+
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
recipe: dict[str, Any],
|
|
28
|
+
source_id: str | None = None,
|
|
29
|
+
runner_id: str | None = None,
|
|
30
|
+
) -> None:
|
|
31
|
+
if not source_id:
|
|
32
|
+
raise ValueError("SANDBOX requires a source ID")
|
|
33
|
+
super().__init__(recipe, source_id=source_id, runner_id=runner_id)
|
|
34
|
+
self._api_url = (
|
|
35
|
+
os.environ.get("CLASSIFYRE_OUTPUT_REST_URL")
|
|
36
|
+
or os.environ.get("API_URL")
|
|
37
|
+
or "http://localhost:8000"
|
|
38
|
+
).rstrip("/")
|
|
39
|
+
self._session = requests.Session()
|
|
40
|
+
self._file_by_id: dict[str, dict[str, Any]] = {}
|
|
41
|
+
|
|
42
|
+
def _scope_option(self, key: str, default: Any = None) -> Any:
|
|
43
|
+
# SANDBOX intentionally has an empty optional config section; uploaded
|
|
44
|
+
# files are always in scope and always include their content.
|
|
45
|
+
return default
|
|
46
|
+
|
|
47
|
+
def _connection_option(self, key: str, default: Any = None) -> Any:
|
|
48
|
+
return default
|
|
49
|
+
|
|
50
|
+
def _max_object_bytes(self) -> int:
|
|
51
|
+
# Keep the extractor aligned with the API's per-upload limit instead
|
|
52
|
+
# of the object-storage connector family's conservative 5 MiB default.
|
|
53
|
+
return 50 * 1024 * 1024
|
|
54
|
+
|
|
55
|
+
def _request(self, method: str, path: str, *, stream: bool = False) -> requests.Response:
|
|
56
|
+
response = self._session.request(
|
|
57
|
+
method,
|
|
58
|
+
f"{self._api_url}{path}",
|
|
59
|
+
timeout=120,
|
|
60
|
+
stream=stream,
|
|
61
|
+
headers={"Connection": "close"},
|
|
62
|
+
)
|
|
63
|
+
response.raise_for_status()
|
|
64
|
+
return response
|
|
65
|
+
|
|
66
|
+
def _list_objects(self) -> Iterator[ObjectRef]:
|
|
67
|
+
response = self._request("GET", f"/sources/{self.source_id}/files")
|
|
68
|
+
payload = response.json()
|
|
69
|
+
if not isinstance(payload, list):
|
|
70
|
+
raise ValueError("Source files API returned a non-array response")
|
|
71
|
+
|
|
72
|
+
self._file_by_id = {}
|
|
73
|
+
for item in payload:
|
|
74
|
+
if not isinstance(item, dict) or not item.get("id"):
|
|
75
|
+
continue
|
|
76
|
+
file_id = str(item["id"])
|
|
77
|
+
self._file_by_id[file_id] = item
|
|
78
|
+
created_at = str(item.get("createdAt") or "").replace("Z", "+00:00")
|
|
79
|
+
try:
|
|
80
|
+
modified = datetime.fromisoformat(created_at)
|
|
81
|
+
if modified.tzinfo is None:
|
|
82
|
+
modified = modified.replace(tzinfo=UTC)
|
|
83
|
+
except ValueError:
|
|
84
|
+
modified = datetime.now(UTC)
|
|
85
|
+
yield ObjectRef(
|
|
86
|
+
key=file_id,
|
|
87
|
+
size=int(item.get("fileSizeBytes") or 0),
|
|
88
|
+
last_modified=modified,
|
|
89
|
+
etag=str(item.get("contentHash") or "") or None,
|
|
90
|
+
content_type_hint=str(item.get("declaredMimeType") or "application/octet-stream"),
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
def _download_object(self, ref: ObjectRef) -> tuple[bytes, str | None]:
|
|
94
|
+
response = self._request(
|
|
95
|
+
"GET",
|
|
96
|
+
f"/sources/{self.source_id}/files/{quote(ref.key, safe='')}/content",
|
|
97
|
+
stream=True,
|
|
98
|
+
)
|
|
99
|
+
chunks: list[bytes] = []
|
|
100
|
+
for chunk in response.iter_content(chunk_size=1024 * 1024):
|
|
101
|
+
if chunk:
|
|
102
|
+
chunks.append(chunk)
|
|
103
|
+
return b"".join(chunks), response.headers.get("Content-Type")
|
|
104
|
+
|
|
105
|
+
def _external_url(self, key: str) -> str:
|
|
106
|
+
return f"sandbox://{self.source_id}/{key}"
|
|
107
|
+
|
|
108
|
+
def _object_file_name(self, ref: ObjectRef) -> str:
|
|
109
|
+
metadata = self._file_by_id.get(ref.key, {})
|
|
110
|
+
return str(metadata.get("fileName") or ref.key)
|
|
111
|
+
|
|
112
|
+
def _file_name_for_asset_id(self, asset_id: str) -> str:
|
|
113
|
+
ref_name = super()._file_name_for_asset_id(asset_id)
|
|
114
|
+
metadata = self._file_by_id.get(ref_name)
|
|
115
|
+
return str(metadata.get("fileName") or ref_name) if metadata else ref_name
|
|
116
|
+
|
|
117
|
+
def cleanup(self) -> None:
|
|
118
|
+
self._session.close()
|
|
119
|
+
super().cleanup()
|
|
@@ -5,7 +5,7 @@ standalone assets — e.g. HuggingFace image datasets store an ``image`` column
|
|
|
5
5
|
``struct<bytes, path>``, and Office documents embed pictures under their ``media/``
|
|
6
6
|
folders. This module surfaces those images as raw bytes so the scan pipeline can
|
|
7
7
|
treat each one as its own child IMAGE asset (or run image detectors on it directly
|
|
8
|
-
in
|
|
8
|
+
in file evaluation), instead of dumping undecodable bytes into text detectors.
|
|
9
9
|
|
|
10
10
|
On any missing optional dependency or parse failure the iterators log a warning and
|
|
11
11
|
yield nothing, so callers degrade gracefully.
|
|
@@ -657,7 +657,7 @@ def parse_bytes(
|
|
|
657
657
|
"""
|
|
658
658
|
Parse in-memory bytes: resolve MIME type and extract raw/text content.
|
|
659
659
|
|
|
660
|
-
Used by
|
|
660
|
+
Used by file evaluation and any caller that needs a complete ParsedBytes in one shot.
|
|
661
661
|
Object-storage sources prefer resolve_mime_type() + iter_file_pages() to avoid
|
|
662
662
|
loading all content into memory before detector scanning.
|
|
663
663
|
"""
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Routing tests for
|
|
1
|
+
"""Routing tests for FileEvaluationRunner.
|
|
2
2
|
|
|
3
3
|
Verifies that file-capable detectors (e.g. a vision LLM detector) receive raw
|
|
4
4
|
bytes for images AND PDFs, while text detectors receive extracted text — without
|
|
@@ -13,7 +13,7 @@ from typing import Any
|
|
|
13
13
|
|
|
14
14
|
import pytest
|
|
15
15
|
|
|
16
|
-
from src.
|
|
16
|
+
from src.file_evaluation.runner import FileEvaluationRunner
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
class _RecordingDetector:
|
|
@@ -40,8 +40,8 @@ _VISION_TYPES = [
|
|
|
40
40
|
_TEXT_TYPES = ["text/plain", "text/html"]
|
|
41
41
|
|
|
42
42
|
|
|
43
|
-
def _runner_with(detector: _RecordingDetector) ->
|
|
44
|
-
runner =
|
|
43
|
+
def _runner_with(detector: _RecordingDetector) -> FileEvaluationRunner:
|
|
44
|
+
runner = FileEvaluationRunner([])
|
|
45
45
|
runner._build_detectors = lambda: [detector] # type: ignore[method-assign]
|
|
46
46
|
return runner
|
|
47
47
|
|