classifyre-cli 0.4.55__tar.gz → 0.4.57__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/.turbo/turbo-build.log +1 -1
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/PKG-INFO +3 -3
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/README.md +2 -2
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/package.json +1 -1
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/pyproject.toml +1 -3
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/custom/runners/__init__.py +0 -4
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/custom/runners/_factory.py +0 -5
- classifyre_cli-0.4.57/src/file_evaluation/__init__.py +5 -0
- {classifyre_cli-0.4.55/src/sandbox → classifyre_cli-0.4.57/src/file_evaluation}/runner.py +23 -17
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/main.py +20 -28
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/models/generated_detectors.py +1 -73
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/models/generated_input.py +54 -5
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/models/generated_single_asset_scan_results.py +17 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/outputs/rest.py +10 -0
- classifyre_cli-0.4.57/src/pipeline/__init__.py +25 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/pipeline/detector_pipeline.py +42 -11
- classifyre_cli-0.4.57/src/pipeline/text_artifact.py +54 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/pipeline/worker_pool.py +0 -1
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/object_storage/base.py +10 -5
- classifyre_cli-0.4.57/src/sources/sandbox/__init__.py +3 -0
- classifyre_cli-0.4.57/src/sources/sandbox/source.py +119 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/utils/embedded_images.py +1 -1
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/utils/file_parser.py +54 -6
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/utils/video_processing.py +28 -7
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/custom/test_transformer_runners.py +1 -69
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/test_custom_detector_examples_runtime.py +0 -2
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/test_detector_catalog_commercial.py +1 -3
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/pipeline/test_detector_outcomes.py +45 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/pipeline/test_detector_pipeline.py +3 -2
- classifyre_cli-0.4.55/tests/test_sandbox_runner.py → classifyre_cli-0.4.57/tests/test_file_evaluation_runner.py +4 -4
- classifyre_cli-0.4.57/tests/test_sandbox_source.py +134 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/utils/test_file_parser.py +21 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/utils/test_video_processing.py +45 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/uv.lock +188 -202
- classifyre_cli-0.4.55/src/detectors/custom/runners/_feature_extraction.py +0 -165
- classifyre_cli-0.4.55/src/pipeline/__init__.py +0 -7
- classifyre_cli-0.4.55/src/sandbox/__init__.py +0 -5
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/.gitignore +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/.python-version +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/main.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/scripts/generate_models.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/config.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/base.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/broken_links/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/broken_links/detector.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/config.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/content/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/custom/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/custom/detector.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/custom/extractor.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/custom/runners/_base.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/custom/runners/_gliner2.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/custom/runners/_image_classification.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/custom/runners/_llm.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/custom/runners/_object_detection.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/custom/runners/_regex.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/custom/runners/_text_classification.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/custom/trainer.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/dependencies.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/pii/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/pii/detector.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/secrets/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/secrets/detector.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/threat/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/threat/code_security_detector.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/detectors/threat/yara_detector.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/outputs/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/outputs/base.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/outputs/console.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/outputs/factory.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/outputs/file.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/pipeline/content_provider.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/pipeline/parsed_content_provider.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/asset_metadata.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/atlassian_common.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/azure_blob_storage/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/azure_blob_storage/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/base.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/confluence/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/confluence/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/databricks/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/databricks/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/delta_lake/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/delta_lake/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/dependencies.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/elasticsearch/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/elasticsearch/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/email/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/email/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/google_cloud_storage/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/google_cloud_storage/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/google_workspace/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/google_workspace/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/hive/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/hive/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/iceberg/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/iceberg/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/jira/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/jira/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/kafka/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/kafka/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/lakehouse_base.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/local_folder/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/local_folder/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/meilisearch/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/meilisearch/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/microsoft_365/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/microsoft_365/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/mongodb/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/mongodb/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/mssql/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/mssql/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/mysql/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/mysql/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/neo4j/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/neo4j/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/notion/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/notion/client.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/notion/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/opensearch/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/opensearch/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/oracle/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/oracle/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/postgresql/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/postgresql/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/powerbi/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/powerbi/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/recipe_normalizer.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/s3_client.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/s3_compatible_storage/README.md +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/s3_compatible_storage/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/s3_compatible_storage/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/search_engine_base.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/servicedesk/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/servicedesk/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/slack/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/slack/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/snowflake/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/snowflake/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/sqlite/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/sqlite/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/tableau/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/tableau/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/tabular_base.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/tabular_utils.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/wordpress/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/wordpress/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/youtube/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/sources/youtube/source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/telemetry.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/utils/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/utils/content_extraction.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/utils/dependency_groups.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/utils/file_metadata.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/utils/file_to_images.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/utils/hashing.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/utils/resources.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/utils/transcription.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/utils/uv_sync.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/utils/validation.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/_lakehouse_fakes.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/conftest.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/broken_links/test_broken_links_detector.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/conftest.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/content/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/custom/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/custom/conftest.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/custom/test_gliner2_classification.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/custom/test_invoice_extraction.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/custom/test_llm_runner.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/custom/test_pipeline_integration.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/custom/test_regex_runner.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/custom/test_runner_hardening.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/pii/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/pii/conftest.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/pii/sample_invoice.pdf +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/pii/test_pii_chunking.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/pii/test_pii_detector.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/pii/test_pii_detector_extended.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/pii/test_pii_severity.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/secrets/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/secrets/test_secrets_detector.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/secrets/test_secrets_detector_extended.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/test_base_detector.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/test_detector_pipeline_types.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/test_detector_schema_examples.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/test_detector_types.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/test_phase2_detectors.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/test_registry.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/threat/__init__.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/threat/test_code_security_detector.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/detectors/threat/test_yara_detector.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/integration/test_wordpress_broken_links_detector.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/integration/test_wordpress_links_assets.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/pipeline/test_worker_pool.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_assets_metadata_catalog.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_azure_blob_storage_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_base_source_attachment.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_base_source_sampling.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_config.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_confluence_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_custom_extractor.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_databricks_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_delta_lake_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_dependency_groups.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_elasticsearch_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_email_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_google_cloud_storage_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_google_workspace_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_hashing.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_hive_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_iceberg_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_jira_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_kafka_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_local_folder_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_meilisearch_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_microsoft_365_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_mongodb_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_mssql_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_mysql_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_neo4j_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_notion_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_opensearch_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_oracle_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_outputs.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_postgresql_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_powerbi_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_recipe_normalizer.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_s3_compatible_storage_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_sampling_automatic.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_servicedesk_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_slack_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_snowflake_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_source_dependency_groups.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_source_strategy_conformance.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_sqlite_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_tableau_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_tabular_automatic_sampling.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_tabular_utils.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_uv_sync.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_wordpress_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_youtube_source.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/test_youtube_source_integration.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/utils/test_content_extraction.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/utils/test_embedded_images.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/utils/test_file_metadata.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/utils/test_file_to_images.py +0 -0
- {classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/tests/utils/test_transcription.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: classifyre-cli
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.57
|
|
4
4
|
Summary: Classifyre CLI — scan and classify unstructured data sources
|
|
5
5
|
License: MIT
|
|
6
6
|
Keywords: data,ingestion,metadata,pii,secrets,unstructured
|
|
@@ -54,7 +54,7 @@ Commands:
|
|
|
54
54
|
- `test` - test source connection.
|
|
55
55
|
- `discover` - discover source resources.
|
|
56
56
|
- `extract` - run extraction and emit batched output.
|
|
57
|
-
- `
|
|
57
|
+
- `evaluate-file` - internally evaluate detectors against one local file.
|
|
58
58
|
|
|
59
59
|
## Extract Output Model
|
|
60
60
|
|
|
@@ -78,7 +78,7 @@ Default behavior:
|
|
|
78
78
|
Global/common:
|
|
79
79
|
|
|
80
80
|
- `--debug` - enable debug logging.
|
|
81
|
-
- `--detectors-file <path>` -
|
|
81
|
+
- `--detectors-file <path>` - file evaluation only.
|
|
82
82
|
|
|
83
83
|
Extract output options:
|
|
84
84
|
|
|
@@ -37,7 +37,7 @@ Commands:
|
|
|
37
37
|
- `test` - test source connection.
|
|
38
38
|
- `discover` - discover source resources.
|
|
39
39
|
- `extract` - run extraction and emit batched output.
|
|
40
|
-
- `
|
|
40
|
+
- `evaluate-file` - internally evaluate detectors against one local file.
|
|
41
41
|
|
|
42
42
|
## Extract Output Model
|
|
43
43
|
|
|
@@ -61,7 +61,7 @@ Default behavior:
|
|
|
61
61
|
Global/common:
|
|
62
62
|
|
|
63
63
|
- `--debug` - enable debug logging.
|
|
64
|
-
- `--detectors-file <path>` -
|
|
64
|
+
- `--detectors-file <path>` - file evaluation only.
|
|
65
65
|
|
|
66
66
|
Extract output options:
|
|
67
67
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "classifyre-cli"
|
|
3
|
-
version = "0.4.
|
|
3
|
+
version = "0.4.57"
|
|
4
4
|
description = "Classifyre CLI — scan and classify unstructured data sources"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.12"
|
|
@@ -83,7 +83,6 @@ content = [
|
|
|
83
83
|
]
|
|
84
84
|
quality = [
|
|
85
85
|
"datasketch>=1.10.0",
|
|
86
|
-
"sentence-transformers>=5.5.0",
|
|
87
86
|
"textstat>=0.7.13",
|
|
88
87
|
"transformers>=4.50.0,<6.0.0",
|
|
89
88
|
"torch>=2.12.0",
|
|
@@ -101,7 +100,6 @@ custom = [
|
|
|
101
100
|
"datasets>=4.8.5",
|
|
102
101
|
"scikit-learn>=1.7.2",
|
|
103
102
|
"gliner2>=1.3.1",
|
|
104
|
-
"sentence-transformers>=5.5.0",
|
|
105
103
|
# Direct declaration so the [tool.uv.sources] cpu-index pin applies (timm
|
|
106
104
|
# pulls torchvision transitively; see the ocr group note).
|
|
107
105
|
"torchvision>=0.27.0",
|
|
@@ -14,7 +14,6 @@ from ._base import (
|
|
|
14
14
|
_resolve_pipeline_severity,
|
|
15
15
|
)
|
|
16
16
|
from ._factory import create_runner
|
|
17
|
-
from ._feature_extraction import FeatureExtractionRunner, _chunk_text_with_offsets, _pool_hidden
|
|
18
17
|
from ._gliner2 import (
|
|
19
18
|
GLiNER2Runner,
|
|
20
19
|
_apply_classification_validation,
|
|
@@ -35,7 +34,6 @@ __all__ = [
|
|
|
35
34
|
"_IMAGE_CONTENT_TYPES",
|
|
36
35
|
"_TEXT_CONTENT_TYPES",
|
|
37
36
|
"BaseRunner",
|
|
38
|
-
"FeatureExtractionRunner",
|
|
39
37
|
"GLiNER2Runner",
|
|
40
38
|
"ImageClassificationRunner",
|
|
41
39
|
"LLMRunner",
|
|
@@ -45,12 +43,10 @@ __all__ = [
|
|
|
45
43
|
"_apply_classification_validation",
|
|
46
44
|
"_apply_entity_validation",
|
|
47
45
|
"_chunk_text",
|
|
48
|
-
"_chunk_text_with_offsets",
|
|
49
46
|
"_load_regex_engine",
|
|
50
47
|
"_normalise_classification_output",
|
|
51
48
|
"_normalise_entity_output",
|
|
52
49
|
"_normalise_span",
|
|
53
|
-
"_pool_hidden",
|
|
54
50
|
"_resolve_pipeline_severity",
|
|
55
51
|
"create_runner",
|
|
56
52
|
]
|
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
from ....models.generated_detectors import (
|
|
6
|
-
FeatureExtractionPipelineSchema,
|
|
7
6
|
GLiNER2PipelineSchema,
|
|
8
7
|
ImageClassificationPipelineSchema,
|
|
9
8
|
LLMPipelineSchema,
|
|
@@ -12,7 +11,6 @@ from ....models.generated_detectors import (
|
|
|
12
11
|
TextClassificationPipelineSchema,
|
|
13
12
|
)
|
|
14
13
|
from ._base import BaseRunner
|
|
15
|
-
from ._feature_extraction import FeatureExtractionRunner
|
|
16
14
|
from ._gliner2 import GLiNER2Runner
|
|
17
15
|
from ._image_classification import ImageClassificationRunner
|
|
18
16
|
from ._llm import LLMRunner
|
|
@@ -28,7 +26,6 @@ def create_runner(
|
|
|
28
26
|
| LLMPipelineSchema
|
|
29
27
|
| TextClassificationPipelineSchema
|
|
30
28
|
| ImageClassificationPipelineSchema
|
|
31
|
-
| FeatureExtractionPipelineSchema
|
|
32
29
|
| ObjectDetectionPipelineSchema
|
|
33
30
|
),
|
|
34
31
|
detector_key: str = "",
|
|
@@ -39,8 +36,6 @@ def create_runner(
|
|
|
39
36
|
return TextClassificationRunner(schema, detector_key, detector_name)
|
|
40
37
|
if isinstance(schema, ImageClassificationPipelineSchema):
|
|
41
38
|
return ImageClassificationRunner(schema, detector_key, detector_name)
|
|
42
|
-
if isinstance(schema, FeatureExtractionPipelineSchema):
|
|
43
|
-
return FeatureExtractionRunner(schema, detector_key, detector_name)
|
|
44
39
|
if isinstance(schema, ObjectDetectionPipelineSchema):
|
|
45
40
|
return ObjectDetectionRunner(schema, detector_key, detector_name)
|
|
46
41
|
if isinstance(schema, RegexPipelineSchema):
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""FileEvaluationRunner: run detectors on an ephemeral local file."""
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
@@ -15,7 +15,7 @@ from ..utils.file_parser import ParsedFile, parse_file
|
|
|
15
15
|
logger = logging.getLogger(__name__)
|
|
16
16
|
|
|
17
17
|
|
|
18
|
-
class
|
|
18
|
+
class FileEvaluationRunner:
|
|
19
19
|
"""Run a set of detectors against a single local file."""
|
|
20
20
|
|
|
21
21
|
def __init__(self, detectors_config: list[dict[str, Any]]) -> None:
|
|
@@ -41,7 +41,7 @@ class SandboxRunner:
|
|
|
41
41
|
|
|
42
42
|
detector = get_detector(detector_name, typed_config)
|
|
43
43
|
detectors.append(detector)
|
|
44
|
-
logger.info(f"Initialized
|
|
44
|
+
logger.info(f"Initialized file-evaluation detector: {detector_name}")
|
|
45
45
|
except Exception as e:
|
|
46
46
|
logger.error(f"Failed to initialize detector {detector_type}: {e}")
|
|
47
47
|
|
|
@@ -121,19 +121,23 @@ class SandboxRunner:
|
|
|
121
121
|
and self._is_binary_detector(detector)
|
|
122
122
|
and self._supports_mime(supported, mime_type)
|
|
123
123
|
):
|
|
124
|
-
logger.info(
|
|
124
|
+
logger.info(
|
|
125
|
+
"File evaluation: dispatching '%s' on file bytes (%s)", label, mime_type
|
|
126
|
+
)
|
|
125
127
|
tasks.append(detector.detect(_get_raw_bytes(), mime_type))
|
|
126
128
|
active_detectors.append(detector)
|
|
127
129
|
# Text delivery: text detectors get the extracted text layer.
|
|
128
130
|
elif "text/plain" in supported and text.strip():
|
|
129
131
|
logger.info(
|
|
130
|
-
"
|
|
132
|
+
"File evaluation: dispatching '%s' on extracted text (%d chars)",
|
|
133
|
+
label,
|
|
134
|
+
len(text),
|
|
131
135
|
)
|
|
132
136
|
tasks.append(detector.detect(text, "text/plain"))
|
|
133
137
|
active_detectors.append(detector)
|
|
134
138
|
else:
|
|
135
139
|
logger.info(
|
|
136
|
-
"
|
|
140
|
+
"File evaluation: '%s' not dispatched on whole file (supports=%s, file_mime=%s)",
|
|
137
141
|
label,
|
|
138
142
|
supported,
|
|
139
143
|
file_mime,
|
|
@@ -158,18 +162,20 @@ class SandboxRunner:
|
|
|
158
162
|
for detector, result in zip(active_detectors, results, strict=False):
|
|
159
163
|
label = self._detector_label(detector)
|
|
160
164
|
if isinstance(result, Exception):
|
|
161
|
-
logger.error("
|
|
165
|
+
logger.error("File evaluation: '%s' failed on whole file: %s", label, result)
|
|
162
166
|
continue
|
|
163
167
|
if isinstance(result, list):
|
|
164
168
|
detector_findings = [f for f in result if isinstance(f, DetectionResult)]
|
|
165
169
|
logger.info(
|
|
166
|
-
"
|
|
170
|
+
"File evaluation: '%s' on whole file → %d finding(s)",
|
|
171
|
+
label,
|
|
172
|
+
len(detector_findings),
|
|
167
173
|
)
|
|
168
174
|
for finding in detector_findings:
|
|
169
175
|
all_findings.append(
|
|
170
176
|
finding.model_copy(
|
|
171
177
|
update={
|
|
172
|
-
"runner_id": "
|
|
178
|
+
"runner_id": "file-evaluation",
|
|
173
179
|
"detected_at": detected_at,
|
|
174
180
|
}
|
|
175
181
|
)
|
|
@@ -203,14 +209,14 @@ class SandboxRunner:
|
|
|
203
209
|
binary_detectors = [d for d in detectors if self._is_binary_detector(d)]
|
|
204
210
|
if not binary_detectors:
|
|
205
211
|
logger.info(
|
|
206
|
-
"
|
|
212
|
+
"File evaluation: %s embeds images but no image-capable detectors are configured; "
|
|
207
213
|
"skipping embedded-image scan",
|
|
208
214
|
mime_type,
|
|
209
215
|
)
|
|
210
216
|
return []
|
|
211
217
|
|
|
212
218
|
logger.info(
|
|
213
|
-
"
|
|
219
|
+
"File evaluation: scanning images embedded in %s with image detectors: [%s]",
|
|
214
220
|
mime_type,
|
|
215
221
|
", ".join(self._detector_label(d) for d in binary_detectors),
|
|
216
222
|
)
|
|
@@ -227,14 +233,14 @@ class SandboxRunner:
|
|
|
227
233
|
]
|
|
228
234
|
if not compatible:
|
|
229
235
|
logger.info(
|
|
230
|
-
"
|
|
236
|
+
"File evaluation: embedded image %s (%s, %.0f KB) — no compatible detector",
|
|
231
237
|
image.location,
|
|
232
238
|
image.mime_type,
|
|
233
239
|
size_kb,
|
|
234
240
|
)
|
|
235
241
|
continue
|
|
236
242
|
logger.info(
|
|
237
|
-
"
|
|
243
|
+
"File evaluation: embedded image %s (%s, %.0f KB) → dispatching [%s]",
|
|
238
244
|
image.location,
|
|
239
245
|
image.mime_type,
|
|
240
246
|
size_kb,
|
|
@@ -246,7 +252,7 @@ class SandboxRunner:
|
|
|
246
252
|
result = await detector.detect(image.image_bytes, image.mime_type)
|
|
247
253
|
except Exception as exc:
|
|
248
254
|
logger.error(
|
|
249
|
-
"
|
|
255
|
+
"File evaluation: '%s' failed on embedded image %s: %s",
|
|
250
256
|
label,
|
|
251
257
|
image.location,
|
|
252
258
|
exc,
|
|
@@ -258,7 +264,7 @@ class SandboxRunner:
|
|
|
258
264
|
else []
|
|
259
265
|
)
|
|
260
266
|
logger.info(
|
|
261
|
-
"
|
|
267
|
+
"File evaluation: '%s' on embedded image %s → %d finding(s)",
|
|
262
268
|
label,
|
|
263
269
|
image.location,
|
|
264
270
|
len(detector_findings),
|
|
@@ -267,7 +273,7 @@ class SandboxRunner:
|
|
|
267
273
|
findings.append(self._tag_embedded(finding, image.location, detected_at))
|
|
268
274
|
|
|
269
275
|
logger.info(
|
|
270
|
-
"
|
|
276
|
+
"File evaluation: embedded-image scan complete — %d image(s) scanned, %d finding(s)",
|
|
271
277
|
image_count,
|
|
272
278
|
len(findings),
|
|
273
279
|
)
|
|
@@ -296,7 +302,7 @@ class SandboxRunner:
|
|
|
296
302
|
location = location.model_copy(update={"path": location_label})
|
|
297
303
|
return finding.model_copy(
|
|
298
304
|
update={
|
|
299
|
-
"runner_id": "
|
|
305
|
+
"runner_id": "file-evaluation",
|
|
300
306
|
"detected_at": detected_at,
|
|
301
307
|
"metadata": metadata,
|
|
302
308
|
"location": location,
|
|
@@ -131,6 +131,10 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
|
|
|
131
131
|
source_id = args.source_id or os.environ.get("SOURCE_ID")
|
|
132
132
|
if source_id:
|
|
133
133
|
os.environ["SOURCE_ID"] = source_id
|
|
134
|
+
if getattr(args, "output_rest_url", None):
|
|
135
|
+
# Sources such as SANDBOX use the same managed-job API base URL as the
|
|
136
|
+
# REST output sink to list and stream their input content.
|
|
137
|
+
os.environ["CLASSIFYRE_OUTPUT_REST_URL"] = args.output_rest_url
|
|
134
138
|
|
|
135
139
|
try:
|
|
136
140
|
try:
|
|
@@ -264,7 +268,7 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
|
|
|
264
268
|
logger.info("Phase 1 complete: %d assets discovered", total_assets)
|
|
265
269
|
|
|
266
270
|
# --- Phase 2: Processing ---
|
|
267
|
-
if
|
|
271
|
+
if all_stubs:
|
|
268
272
|
import asyncio as _asyncio
|
|
269
273
|
|
|
270
274
|
processed_count = 0
|
|
@@ -319,6 +323,10 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
|
|
|
319
323
|
)
|
|
320
324
|
payload = _asset_to_payload(result)
|
|
321
325
|
await sink.emit_batch([payload], skip_findings=False)
|
|
326
|
+
if hasattr(sink, "emit_text_chunks"):
|
|
327
|
+
artifact = pipeline.take_text_artifact(asset_hash)
|
|
328
|
+
if artifact is not None:
|
|
329
|
+
await sink.emit_text_chunks(asset_hash, artifact)
|
|
322
330
|
|
|
323
331
|
if hasattr(sink, "update_asset_status"):
|
|
324
332
|
f_total, f_by_sev, f_by_det = _compute_findings_counts(
|
|
@@ -353,22 +361,6 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
|
|
|
353
361
|
processed_count,
|
|
354
362
|
error_count,
|
|
355
363
|
)
|
|
356
|
-
elif all_stubs and hasattr(sink, "update_asset_status"):
|
|
357
|
-
# No detectors configured: mark discovered assets as PROCESSED
|
|
358
|
-
import asyncio as _asyncio
|
|
359
|
-
|
|
360
|
-
async def _mark_processed(asset: Any) -> None:
|
|
361
|
-
asset_hash = getattr(asset, "hash", None) or ""
|
|
362
|
-
await sink.update_asset_status(
|
|
363
|
-
asset_hash, "PROCESSED", findings_total=0
|
|
364
|
-
)
|
|
365
|
-
|
|
366
|
-
tasks = [_asyncio.create_task(_mark_processed(a)) for a in all_stubs]
|
|
367
|
-
await _asyncio.gather(*tasks, return_exceptions=True)
|
|
368
|
-
logger.info(
|
|
369
|
-
"Phase 2 skipped (no detectors): %d assets marked processed",
|
|
370
|
-
len(all_stubs),
|
|
371
|
-
)
|
|
372
364
|
|
|
373
365
|
# Persist the advanced AUTOMATIC sampling cursor (no-op for
|
|
374
366
|
# other strategies, which return None). Only on the normal
|
|
@@ -473,13 +465,13 @@ def run_train_command(args: argparse.Namespace) -> None:
|
|
|
473
465
|
print(json.dumps(result.to_dict()))
|
|
474
466
|
|
|
475
467
|
|
|
476
|
-
def
|
|
477
|
-
"""
|
|
478
|
-
from .
|
|
468
|
+
def run_evaluate_file_command(args: argparse.Namespace) -> None:
|
|
469
|
+
"""Evaluate detector configs against one ephemeral local file."""
|
|
470
|
+
from .file_evaluation import FileEvaluationRunner
|
|
479
471
|
|
|
480
472
|
file_path_str: str | None = args.recipe
|
|
481
473
|
if not file_path_str:
|
|
482
|
-
logger.error("
|
|
474
|
+
logger.error("evaluate-file requires a file path as the first argument")
|
|
483
475
|
sys.exit(1)
|
|
484
476
|
|
|
485
477
|
file_path = Path(file_path_str)
|
|
@@ -504,7 +496,7 @@ def run_sandbox_command(args: argparse.Namespace) -> None:
|
|
|
504
496
|
sys.exit(1)
|
|
505
497
|
|
|
506
498
|
try:
|
|
507
|
-
runner =
|
|
499
|
+
runner = FileEvaluationRunner(detectors)
|
|
508
500
|
parsed, findings = runner.run(file_path)
|
|
509
501
|
if parsed.parse_error:
|
|
510
502
|
logger.warning("File parse warning: %s", parsed.parse_error)
|
|
@@ -516,7 +508,7 @@ def run_sandbox_command(args: argparse.Namespace) -> None:
|
|
|
516
508
|
output["parse_error"] = parsed.parse_error
|
|
517
509
|
print(json.dumps(_sanitize_for_json(output), ensure_ascii=False))
|
|
518
510
|
except Exception as e:
|
|
519
|
-
logger.error("
|
|
511
|
+
logger.error("File evaluation failed: %s", e, exc_info=True)
|
|
520
512
|
sys.exit(1)
|
|
521
513
|
|
|
522
514
|
|
|
@@ -532,14 +524,14 @@ def main() -> None:
|
|
|
532
524
|
parser = argparse.ArgumentParser(description="Classifyre Metadata Extraction CLI")
|
|
533
525
|
parser.add_argument(
|
|
534
526
|
"command",
|
|
535
|
-
choices=["test", "extract", "discover", "
|
|
527
|
+
choices=["test", "extract", "discover", "evaluate-file", "train"],
|
|
536
528
|
help="Command to run",
|
|
537
529
|
)
|
|
538
530
|
parser.add_argument(
|
|
539
531
|
"recipe",
|
|
540
532
|
nargs="?",
|
|
541
533
|
default=None,
|
|
542
|
-
help="Path to recipe JSON (or file path for
|
|
534
|
+
help="Path to recipe JSON (or file path for evaluate-file)",
|
|
543
535
|
)
|
|
544
536
|
parser.add_argument("--debug", action="store_true", help="Enable debug logging")
|
|
545
537
|
parser.add_argument(
|
|
@@ -577,7 +569,7 @@ def main() -> None:
|
|
|
577
569
|
parser.add_argument(
|
|
578
570
|
"--detectors-file",
|
|
579
571
|
default=None,
|
|
580
|
-
help="Path to JSON file with detector configs (
|
|
572
|
+
help="Path to JSON file with detector configs (evaluate-file only)",
|
|
581
573
|
)
|
|
582
574
|
# train-command arguments
|
|
583
575
|
parser.add_argument(
|
|
@@ -641,8 +633,8 @@ def main() -> None:
|
|
|
641
633
|
if args.debug:
|
|
642
634
|
logging.getLogger().setLevel(logging.DEBUG)
|
|
643
635
|
|
|
644
|
-
if args.command == "
|
|
645
|
-
|
|
636
|
+
if args.command == "evaluate-file":
|
|
637
|
+
run_evaluate_file_command(args)
|
|
646
638
|
return
|
|
647
639
|
|
|
648
640
|
if args.command == "train":
|
|
@@ -75,6 +75,7 @@ class SourceAssetType(StrEnum):
|
|
|
75
75
|
ORACLE = 'ORACLE'
|
|
76
76
|
HIVE = 'HIVE'
|
|
77
77
|
POWERBI = 'POWERBI'
|
|
78
|
+
SANDBOX = 'SANDBOX'
|
|
78
79
|
|
|
79
80
|
|
|
80
81
|
class DetectorLifecycleStatus(StrEnum):
|
|
@@ -1255,78 +1256,6 @@ class ImageClassificationPipelineSchema(BaseModel):
|
|
|
1255
1256
|
|
|
1256
1257
|
|
|
1257
1258
|
class Type7(StrEnum):
|
|
1258
|
-
FEATURE_EXTRACTION = 'FEATURE_EXTRACTION'
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
class PoolingStrategy(StrEnum):
|
|
1262
|
-
"""
|
|
1263
|
-
How to aggregate per-token hidden states into a single embedding vector.
|
|
1264
|
-
"""
|
|
1265
|
-
|
|
1266
|
-
mean = 'mean'
|
|
1267
|
-
cls = 'cls'
|
|
1268
|
-
max = 'max'
|
|
1269
|
-
none = 'none'
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
class ChunkSize2(RootModel[int]):
|
|
1273
|
-
root: int = Field(
|
|
1274
|
-
None,
|
|
1275
|
-
description='Split text into chunks of this many characters before embedding. Each chunk produces its own finding.',
|
|
1276
|
-
ge=1,
|
|
1277
|
-
)
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
class FeatureExtractionPipelineSchema(BaseModel):
|
|
1281
|
-
"""
|
|
1282
|
-
Feature extraction (embedding) pipeline using a HuggingFace model. Runs a single encoder; create multiple custom detectors to run multiple embedding models.
|
|
1283
|
-
"""
|
|
1284
|
-
|
|
1285
|
-
model_config = ConfigDict(
|
|
1286
|
-
extra='forbid',
|
|
1287
|
-
)
|
|
1288
|
-
type: Literal['FEATURE_EXTRACTION']
|
|
1289
|
-
model: str = Field(
|
|
1290
|
-
...,
|
|
1291
|
-
description="HuggingFace hub ID (e.g. 'BAAI/bge-base-en-v1.5', 'sentence-transformers/all-MiniLM-L6-v2') or absolute local directory path.",
|
|
1292
|
-
)
|
|
1293
|
-
model_revision: str | None = Field(
|
|
1294
|
-
None,
|
|
1295
|
-
description='Git branch, tag, or commit hash when fetching from the HuggingFace hub.',
|
|
1296
|
-
)
|
|
1297
|
-
device: str | None = Field(
|
|
1298
|
-
'cpu',
|
|
1299
|
-
description="Inference device: 'cpu' (default), 'cuda', 'mps', or a CUDA device string like 'cuda:0'.",
|
|
1300
|
-
)
|
|
1301
|
-
pooling_strategy: PoolingStrategy | None = Field(
|
|
1302
|
-
'mean',
|
|
1303
|
-
description='How to aggregate per-token hidden states into a single embedding vector.',
|
|
1304
|
-
)
|
|
1305
|
-
normalize_embeddings: bool | None = Field(
|
|
1306
|
-
True,
|
|
1307
|
-
description='L2-normalise the final embedding vector. Recommended for cosine-similarity workloads.',
|
|
1308
|
-
)
|
|
1309
|
-
truncation: bool | None = Field(
|
|
1310
|
-
True, description="Truncate input to the model's maximum sequence length."
|
|
1311
|
-
)
|
|
1312
|
-
max_length: int | None = Field(
|
|
1313
|
-
None, description="Override the tokenizer's default maximum sequence length."
|
|
1314
|
-
)
|
|
1315
|
-
batch_size: int | None = Field(
|
|
1316
|
-
8, description='Number of texts to encode in a single forward pass.'
|
|
1317
|
-
)
|
|
1318
|
-
chunk_size: ChunkSize2 | None = Field(
|
|
1319
|
-
None,
|
|
1320
|
-
description='Split text into chunks of this many characters before embedding. Each chunk produces its own finding.',
|
|
1321
|
-
)
|
|
1322
|
-
chunk_overlap: ChunkOverlap1 | None = Field(
|
|
1323
|
-
0,
|
|
1324
|
-
description='Character overlap between consecutive chunks.',
|
|
1325
|
-
validate_default=True,
|
|
1326
|
-
)
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
class Type8(StrEnum):
|
|
1330
1259
|
OBJECT_DETECTION = 'OBJECT_DETECTION'
|
|
1331
1260
|
|
|
1332
1261
|
|
|
@@ -1407,7 +1336,6 @@ class CustomDetectorConfig(DetectorConfig):
|
|
|
1407
1336
|
| LLMPipelineSchema
|
|
1408
1337
|
| TextClassificationPipelineSchema
|
|
1409
1338
|
| ImageClassificationPipelineSchema
|
|
1410
|
-
| FeatureExtractionPipelineSchema
|
|
1411
1339
|
| ObjectDetectionPipelineSchema
|
|
1412
1340
|
| None
|
|
1413
1341
|
) = Field(None, discriminator='type', title='AnyPipelineSchema')
|
|
@@ -54,6 +54,7 @@ class AssetType(StrEnum):
|
|
|
54
54
|
LOCAL_FOLDER = 'LOCAL_FOLDER'
|
|
55
55
|
MICROSOFT_365 = 'MICROSOFT_365'
|
|
56
56
|
GOOGLE_WORKSPACE = 'GOOGLE_WORKSPACE'
|
|
57
|
+
SANDBOX = 'SANDBOX'
|
|
57
58
|
|
|
58
59
|
|
|
59
60
|
class DetectorType(StrEnum):
|
|
@@ -353,6 +354,7 @@ class Type(StrEnum):
|
|
|
353
354
|
LOCAL_FOLDER = 'LOCAL_FOLDER'
|
|
354
355
|
MICROSOFT_365 = 'MICROSOFT_365'
|
|
355
356
|
GOOGLE_WORKSPACE = 'GOOGLE_WORKSPACE'
|
|
357
|
+
SANDBOX = 'SANDBOX'
|
|
356
358
|
|
|
357
359
|
|
|
358
360
|
class YouTubeRequired(BaseModel):
|
|
@@ -648,6 +650,32 @@ class LocalFolderRequired(BaseModel):
|
|
|
648
650
|
)
|
|
649
651
|
|
|
650
652
|
|
|
653
|
+
class SandboxRequired(BaseModel):
|
|
654
|
+
"""
|
|
655
|
+
Sandbox files are uploaded separately through the source files API.
|
|
656
|
+
"""
|
|
657
|
+
|
|
658
|
+
model_config = ConfigDict(
|
|
659
|
+
extra='forbid',
|
|
660
|
+
)
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
class SandboxMasked(BaseModel):
|
|
664
|
+
"""
|
|
665
|
+
Sandbox sources do not require credentials.
|
|
666
|
+
"""
|
|
667
|
+
|
|
668
|
+
model_config = ConfigDict(
|
|
669
|
+
extra='forbid',
|
|
670
|
+
)
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
class SandboxOptional(BaseModel):
|
|
674
|
+
model_config = ConfigDict(
|
|
675
|
+
extra='forbid',
|
|
676
|
+
)
|
|
677
|
+
|
|
678
|
+
|
|
651
679
|
class LocalFolderMasked(BaseModel):
|
|
652
680
|
"""
|
|
653
681
|
Local folder scans run with the desktop application's filesystem permissions; no credentials are required.
|
|
@@ -2170,6 +2198,24 @@ class LocalFolderInput(CoreInput):
|
|
|
2170
2198
|
resources: ResourceOverrides | None = None
|
|
2171
2199
|
|
|
2172
2200
|
|
|
2201
|
+
class SandboxInput(CoreInput):
|
|
2202
|
+
type: Literal['SANDBOX'] | None = Field(
|
|
2203
|
+
None, description='Type of the asset or source'
|
|
2204
|
+
)
|
|
2205
|
+
required: SandboxRequired
|
|
2206
|
+
masked: SandboxMasked | None = None
|
|
2207
|
+
optional: SandboxOptional | None = None
|
|
2208
|
+
detectors: list[Detector] | None = Field(
|
|
2209
|
+
None, description='Detectors to run on ingested content'
|
|
2210
|
+
)
|
|
2211
|
+
custom_detectors: list[CustomDetectorSelection] | None = Field(
|
|
2212
|
+
None,
|
|
2213
|
+
description='Reusable custom detector IDs selected from the custom detector catalog.',
|
|
2214
|
+
)
|
|
2215
|
+
sampling: SamplingConfig
|
|
2216
|
+
resources: ResourceOverrides | None = None
|
|
2217
|
+
|
|
2218
|
+
|
|
2173
2219
|
class AzureBlobStorageInput(CoreInput):
|
|
2174
2220
|
type: Literal['AZURE_BLOB_STORAGE'] | None = Field(
|
|
2175
2221
|
None, description='Type of the asset or source'
|
|
@@ -2601,7 +2647,7 @@ class ConfluenceOptionalConnection(BaseModel):
|
|
|
2601
2647
|
)
|
|
2602
2648
|
|
|
2603
2649
|
|
|
2604
|
-
class
|
|
2650
|
+
class Type20(StrEnum):
|
|
2605
2651
|
"""
|
|
2606
2652
|
Filter spaces by space type
|
|
2607
2653
|
"""
|
|
@@ -2638,7 +2684,7 @@ class ConfluenceOptionalScopeSpaces(BaseModel):
|
|
|
2638
2684
|
keys: list[str] | None = Field(
|
|
2639
2685
|
None, description='Filter spaces by keys (up to 250)', max_length=250
|
|
2640
2686
|
)
|
|
2641
|
-
type:
|
|
2687
|
+
type: Type20 | None = Field(None, description='Filter spaces by space type')
|
|
2642
2688
|
status: Status | None = Field(None, description='Filter spaces by status')
|
|
2643
2689
|
labels: list[str] | None = Field(
|
|
2644
2690
|
None,
|
|
@@ -2904,7 +2950,7 @@ class ServiceDeskOptional(BaseModel):
|
|
|
2904
2950
|
content: ServiceDeskOptionalContent | None = None
|
|
2905
2951
|
|
|
2906
2952
|
|
|
2907
|
-
class
|
|
2953
|
+
class Type21(StrEnum):
|
|
2908
2954
|
"""
|
|
2909
2955
|
Type of the asset or source
|
|
2910
2956
|
"""
|
|
@@ -2941,6 +2987,7 @@ class Type20(StrEnum):
|
|
|
2941
2987
|
LOCAL_FOLDER = 'LOCAL_FOLDER'
|
|
2942
2988
|
MICROSOFT_365 = 'MICROSOFT_365'
|
|
2943
2989
|
GOOGLE_WORKSPACE = 'GOOGLE_WORKSPACE'
|
|
2990
|
+
SANDBOX = 'SANDBOX'
|
|
2944
2991
|
|
|
2945
2992
|
|
|
2946
2993
|
class ConfluenceInput(CoreInput):
|
|
@@ -4075,7 +4122,8 @@ class IcebergInput(CoreInput):
|
|
|
4075
4122
|
|
|
4076
4123
|
class SourceInput(
|
|
4077
4124
|
RootModel[
|
|
4078
|
-
|
|
4125
|
+
SandboxInput
|
|
4126
|
+
| SlackInput
|
|
4079
4127
|
| S3CompatibleStorageInput
|
|
4080
4128
|
| LocalFolderInput
|
|
4081
4129
|
| AzureBlobStorageInput
|
|
@@ -4110,7 +4158,8 @@ class SourceInput(
|
|
|
4110
4158
|
]
|
|
4111
4159
|
):
|
|
4112
4160
|
root: (
|
|
4113
|
-
|
|
4161
|
+
SandboxInput
|
|
4162
|
+
| SlackInput
|
|
4114
4163
|
| S3CompatibleStorageInput
|
|
4115
4164
|
| LocalFolderInput
|
|
4116
4165
|
| AzureBlobStorageInput
|
{classifyre_cli-0.4.55 → classifyre_cli-0.4.57}/src/models/generated_single_asset_scan_results.py
RENAMED
|
@@ -87,6 +87,19 @@ class Location(BaseModel):
|
|
|
87
87
|
end: int | None = Field(None, description='End offset (0-indexed)')
|
|
88
88
|
|
|
89
89
|
|
|
90
|
+
class TextExtractionStatus(StrEnum):
|
|
91
|
+
"""
|
|
92
|
+
Structured text coverage outcome. EXTRACTED means detector-visible text was produced. EMPTY means extraction completed without an error but yielded no text. ENGINE_UNAVAILABLE, ZERO_FRAMES, and FAILED are incomplete coverage states and must not be treated as an empty document.
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
NOT_APPLICABLE = 'NOT_APPLICABLE'
|
|
96
|
+
EXTRACTED = 'EXTRACTED'
|
|
97
|
+
EMPTY = 'EMPTY'
|
|
98
|
+
ENGINE_UNAVAILABLE = 'ENGINE_UNAVAILABLE'
|
|
99
|
+
ZERO_FRAMES = 'ZERO_FRAMES'
|
|
100
|
+
FAILED = 'FAILED'
|
|
101
|
+
|
|
102
|
+
|
|
90
103
|
class DetectionResult(BaseModel):
|
|
91
104
|
"""
|
|
92
105
|
Result from detector scan
|
|
@@ -221,6 +234,10 @@ class ScanStats(BaseModel):
|
|
|
221
234
|
None,
|
|
222
235
|
description="True when an asset that should carry text yielded none. Most significant for assets whose text is derived rather than native (OCR'd images and scanned PDFs, transcribed audio/video): empty derived text means the content may have been missed entirely, not that the document is blank. Distinct from an error — nothing failed — so it is reported as coverage, not as a run failure.",
|
|
223
236
|
)
|
|
237
|
+
text_extraction_status: TextExtractionStatus | None = Field(
|
|
238
|
+
None,
|
|
239
|
+
description='Structured text coverage outcome. EXTRACTED means detector-visible text was produced. EMPTY means extraction completed without an error but yielded no text. ENGINE_UNAVAILABLE, ZERO_FRAMES, and FAILED are incomplete coverage states and must not be treated as an empty document.',
|
|
240
|
+
)
|
|
224
241
|
findings_count: int | None = Field(
|
|
225
242
|
None, description='Total number of findings detected'
|
|
226
243
|
)
|