classifyre-cli 0.4.56__tar.gz → 0.4.58__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- classifyre_cli-0.4.58/.turbo/turbo-build.log +3 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/PKG-INFO +3 -3
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/README.md +2 -2
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/package.json +1 -1
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/pyproject.toml +16 -3
- classifyre_cli-0.4.58/src/file_evaluation/__init__.py +5 -0
- {classifyre_cli-0.4.56/src/sandbox → classifyre_cli-0.4.58/src/file_evaluation}/runner.py +28 -17
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/main.py +85 -19
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/models/generated_detectors.py +1 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/models/generated_input.py +54 -5
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/object_storage/base.py +151 -11
- classifyre_cli-0.4.58/src/sources/sandbox/__init__.py +3 -0
- classifyre_cli-0.4.58/src/sources/sandbox/source.py +119 -0
- classifyre_cli-0.4.58/src/utils/archive_extraction.py +281 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/utils/embedded_images.py +1 -1
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/utils/file_metadata.py +19 -1
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/utils/file_parser.py +347 -22
- classifyre_cli-0.4.58/src/utils/legacy_office.py +152 -0
- classifyre_cli-0.4.56/tests/test_sandbox_runner.py → classifyre_cli-0.4.58/tests/test_file_evaluation_runner.py +4 -4
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_local_folder_source.py +109 -0
- classifyre_cli-0.4.58/tests/test_main.py +36 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_s3_compatible_storage_source.py +45 -0
- classifyre_cli-0.4.58/tests/test_sandbox_source.py +134 -0
- classifyre_cli-0.4.58/tests/utils/test_archive_extraction.py +153 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/utils/test_file_parser.py +287 -0
- classifyre_cli-0.4.58/tests/utils/test_legacy_office.py +66 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/uv.lock +763 -581
- classifyre_cli-0.4.56/.turbo/turbo-build.log +0 -3
- classifyre_cli-0.4.56/src/sandbox/__init__.py +0 -5
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/.gitignore +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/.python-version +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/main.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/scripts/generate_models.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/config.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/base.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/broken_links/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/broken_links/detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/config.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/content/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/custom/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/custom/detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/custom/extractor.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/custom/runners/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/custom/runners/_base.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/custom/runners/_factory.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/custom/runners/_gliner2.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/custom/runners/_image_classification.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/custom/runners/_llm.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/custom/runners/_object_detection.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/custom/runners/_regex.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/custom/runners/_text_classification.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/custom/trainer.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/dependencies.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/pii/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/pii/detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/secrets/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/secrets/detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/threat/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/threat/code_security_detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/detectors/threat/yara_detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/models/generated_single_asset_scan_results.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/outputs/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/outputs/base.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/outputs/console.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/outputs/factory.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/outputs/file.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/outputs/rest.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/pipeline/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/pipeline/content_provider.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/pipeline/detector_pipeline.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/pipeline/parsed_content_provider.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/pipeline/text_artifact.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/pipeline/worker_pool.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/asset_metadata.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/atlassian_common.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/azure_blob_storage/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/azure_blob_storage/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/base.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/confluence/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/confluence/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/databricks/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/databricks/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/delta_lake/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/delta_lake/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/dependencies.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/elasticsearch/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/elasticsearch/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/email/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/email/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/google_cloud_storage/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/google_cloud_storage/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/google_workspace/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/google_workspace/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/hive/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/hive/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/iceberg/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/iceberg/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/jira/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/jira/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/kafka/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/kafka/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/lakehouse_base.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/local_folder/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/local_folder/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/meilisearch/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/meilisearch/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/microsoft_365/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/microsoft_365/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/mongodb/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/mongodb/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/mssql/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/mssql/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/mysql/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/mysql/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/neo4j/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/neo4j/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/notion/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/notion/client.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/notion/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/opensearch/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/opensearch/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/oracle/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/oracle/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/postgresql/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/postgresql/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/powerbi/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/powerbi/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/recipe_normalizer.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/s3_client.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/s3_compatible_storage/README.md +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/s3_compatible_storage/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/s3_compatible_storage/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/search_engine_base.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/servicedesk/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/servicedesk/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/slack/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/slack/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/snowflake/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/snowflake/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/sqlite/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/sqlite/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/tableau/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/tableau/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/tabular_base.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/tabular_utils.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/wordpress/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/wordpress/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/youtube/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/sources/youtube/source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/telemetry.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/utils/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/utils/content_extraction.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/utils/dependency_groups.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/utils/file_to_images.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/utils/hashing.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/utils/resources.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/utils/transcription.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/utils/uv_sync.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/utils/validation.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/src/utils/video_processing.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/_lakehouse_fakes.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/conftest.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/broken_links/test_broken_links_detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/conftest.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/content/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/custom/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/custom/conftest.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/custom/test_gliner2_classification.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/custom/test_invoice_extraction.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/custom/test_llm_runner.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/custom/test_pipeline_integration.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/custom/test_regex_runner.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/custom/test_runner_hardening.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/custom/test_transformer_runners.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/pii/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/pii/conftest.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/pii/sample_invoice.pdf +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/pii/test_pii_chunking.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/pii/test_pii_detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/pii/test_pii_detector_extended.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/pii/test_pii_severity.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/secrets/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/secrets/test_secrets_detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/secrets/test_secrets_detector_extended.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/test_base_detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/test_custom_detector_examples_runtime.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/test_detector_catalog_commercial.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/test_detector_pipeline_types.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/test_detector_schema_examples.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/test_detector_types.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/test_phase2_detectors.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/test_registry.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/threat/__init__.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/threat/test_code_security_detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/detectors/threat/test_yara_detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/integration/test_wordpress_broken_links_detector.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/integration/test_wordpress_links_assets.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/pipeline/test_detector_outcomes.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/pipeline/test_detector_pipeline.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/pipeline/test_worker_pool.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_assets_metadata_catalog.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_azure_blob_storage_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_base_source_attachment.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_base_source_sampling.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_config.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_confluence_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_custom_extractor.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_databricks_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_delta_lake_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_dependency_groups.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_elasticsearch_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_email_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_google_cloud_storage_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_google_workspace_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_hashing.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_hive_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_iceberg_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_jira_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_kafka_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_meilisearch_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_microsoft_365_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_mongodb_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_mssql_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_mysql_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_neo4j_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_notion_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_opensearch_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_oracle_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_outputs.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_postgresql_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_powerbi_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_recipe_normalizer.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_sampling_automatic.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_servicedesk_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_slack_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_snowflake_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_source_dependency_groups.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_source_strategy_conformance.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_sqlite_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_tableau_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_tabular_automatic_sampling.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_tabular_utils.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_uv_sync.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_wordpress_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_youtube_source.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/test_youtube_source_integration.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/utils/test_content_extraction.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/utils/test_embedded_images.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/utils/test_file_metadata.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/utils/test_file_to_images.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/utils/test_transcription.py +0 -0
- {classifyre_cli-0.4.56 → classifyre_cli-0.4.58}/tests/utils/test_video_processing.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: classifyre-cli
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.58
|
|
4
4
|
Summary: Classifyre CLI — scan and classify unstructured data sources
|
|
5
5
|
License: MIT
|
|
6
6
|
Keywords: data,ingestion,metadata,pii,secrets,unstructured
|
|
@@ -54,7 +54,7 @@ Commands:
|
|
|
54
54
|
- `test` - test source connection.
|
|
55
55
|
- `discover` - discover source resources.
|
|
56
56
|
- `extract` - run extraction and emit batched output.
|
|
57
|
-
- `
|
|
57
|
+
- `evaluate-file` - internally evaluate detectors against one local file.
|
|
58
58
|
|
|
59
59
|
## Extract Output Model
|
|
60
60
|
|
|
@@ -78,7 +78,7 @@ Default behavior:
|
|
|
78
78
|
Global/common:
|
|
79
79
|
|
|
80
80
|
- `--debug` - enable debug logging.
|
|
81
|
-
- `--detectors-file <path>` -
|
|
81
|
+
- `--detectors-file <path>` - file evaluation only.
|
|
82
82
|
|
|
83
83
|
Extract output options:
|
|
84
84
|
|
|
@@ -37,7 +37,7 @@ Commands:
|
|
|
37
37
|
- `test` - test source connection.
|
|
38
38
|
- `discover` - discover source resources.
|
|
39
39
|
- `extract` - run extraction and emit batched output.
|
|
40
|
-
- `
|
|
40
|
+
- `evaluate-file` - internally evaluate detectors against one local file.
|
|
41
41
|
|
|
42
42
|
## Extract Output Model
|
|
43
43
|
|
|
@@ -61,7 +61,7 @@ Default behavior:
|
|
|
61
61
|
Global/common:
|
|
62
62
|
|
|
63
63
|
- `--debug` - enable debug logging.
|
|
64
|
-
- `--detectors-file <path>` -
|
|
64
|
+
- `--detectors-file <path>` - file evaluation only.
|
|
65
65
|
|
|
66
66
|
Extract output options:
|
|
67
67
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "classifyre-cli"
|
|
3
|
-
version = "0.4.
|
|
3
|
+
version = "0.4.58"
|
|
4
4
|
description = "Classifyre CLI — scan and classify unstructured data sources"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.12"
|
|
@@ -60,7 +60,7 @@ privacy = [
|
|
|
60
60
|
# phonenumbers 9.x introduced lazy region loading via __import__ which fails
|
|
61
61
|
# mid-run in frozen/venv contexts. 8.x eagerly loads all data at import time,
|
|
62
62
|
# avoiding ModuleNotFoundError during Presidio phone number analysis.
|
|
63
|
-
"phonenumbers>=8.13.0,<
|
|
63
|
+
"phonenumbers>=8.13.0,<9.0.0",
|
|
64
64
|
"numpy>=1.26.0,<3.0.0",
|
|
65
65
|
]
|
|
66
66
|
security = [
|
|
@@ -135,6 +135,18 @@ file-processing = [
|
|
|
135
135
|
"openpyxl>=3.1.5",
|
|
136
136
|
"chardet>=7.4.3",
|
|
137
137
|
"pyarrow>=18.0.0",
|
|
138
|
+
# RTF text extraction (tiny, pure Python).
|
|
139
|
+
"striprtf>=0.0.32",
|
|
140
|
+
# Secure XML parsing (entity-expansion / XXE hardened stdlib drop-in).
|
|
141
|
+
"defusedxml>=0.7.1",
|
|
142
|
+
# Image decoding for GIF-frame extraction and HEIC/HEIF conversion before OCR.
|
|
143
|
+
"pillow>=12.2.0",
|
|
144
|
+
"pillow-heif>=1.4.0",
|
|
145
|
+
# Archive member extraction (zip/tar via stdlib). RAR additionally needs an
|
|
146
|
+
# external unrar/unar/bsdtar backend at runtime; rarfile degrades gracefully
|
|
147
|
+
# when none is installed.
|
|
148
|
+
"py7zr>=1.1.3",
|
|
149
|
+
"rarfile>=4.3",
|
|
138
150
|
# Table layer: Iceberg metadata resolution + SQL over Parquet without Spark.
|
|
139
151
|
"pyiceberg>=0.9.0",
|
|
140
152
|
"duckdb>=1.1.0",
|
|
@@ -152,7 +164,8 @@ video = [
|
|
|
152
164
|
]
|
|
153
165
|
ocr = [
|
|
154
166
|
{ include-group = "file-processing" },
|
|
155
|
-
|
|
167
|
+
# >=2.113 for the OpenDocument (odt/ods/odp) and email (eml/msg) backends.
|
|
168
|
+
"docling>=2.113.0",
|
|
156
169
|
"opencv-python-headless>=5.0.0.93,<6.0.0",
|
|
157
170
|
"rapidocr-onnxruntime>=1.4.0",
|
|
158
171
|
# torchvision is a transitive dep of docling-ibm-models, but [tool.uv.sources]
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""FileEvaluationRunner: run detectors on an ephemeral local file."""
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
@@ -15,11 +15,15 @@ from ..utils.file_parser import ParsedFile, parse_file
|
|
|
15
15
|
logger = logging.getLogger(__name__)
|
|
16
16
|
|
|
17
17
|
|
|
18
|
-
class
|
|
18
|
+
class FileEvaluationRunner:
|
|
19
19
|
"""Run a set of detectors against a single local file."""
|
|
20
20
|
|
|
21
21
|
def __init__(self, detectors_config: list[dict[str, Any]]) -> None:
|
|
22
22
|
self._config = detectors_config
|
|
23
|
+
# Detector construction failures, surfaced in the evaluate-file JSON
|
|
24
|
+
# output so callers can distinguish "misconfigured detector" from
|
|
25
|
+
# "no findings".
|
|
26
|
+
self.detector_errors: list[str] = []
|
|
23
27
|
|
|
24
28
|
def _build_detectors(self) -> list[Any]:
|
|
25
29
|
from ..detectors import get_detector
|
|
@@ -41,9 +45,10 @@ class SandboxRunner:
|
|
|
41
45
|
|
|
42
46
|
detector = get_detector(detector_name, typed_config)
|
|
43
47
|
detectors.append(detector)
|
|
44
|
-
logger.info(f"Initialized
|
|
48
|
+
logger.info(f"Initialized file-evaluation detector: {detector_name}")
|
|
45
49
|
except Exception as e:
|
|
46
50
|
logger.error(f"Failed to initialize detector {detector_type}: {e}")
|
|
51
|
+
self.detector_errors.append(f"{detector_type}: {e}")
|
|
47
52
|
|
|
48
53
|
return detectors
|
|
49
54
|
|
|
@@ -121,19 +126,23 @@ class SandboxRunner:
|
|
|
121
126
|
and self._is_binary_detector(detector)
|
|
122
127
|
and self._supports_mime(supported, mime_type)
|
|
123
128
|
):
|
|
124
|
-
logger.info(
|
|
129
|
+
logger.info(
|
|
130
|
+
"File evaluation: dispatching '%s' on file bytes (%s)", label, mime_type
|
|
131
|
+
)
|
|
125
132
|
tasks.append(detector.detect(_get_raw_bytes(), mime_type))
|
|
126
133
|
active_detectors.append(detector)
|
|
127
134
|
# Text delivery: text detectors get the extracted text layer.
|
|
128
135
|
elif "text/plain" in supported and text.strip():
|
|
129
136
|
logger.info(
|
|
130
|
-
"
|
|
137
|
+
"File evaluation: dispatching '%s' on extracted text (%d chars)",
|
|
138
|
+
label,
|
|
139
|
+
len(text),
|
|
131
140
|
)
|
|
132
141
|
tasks.append(detector.detect(text, "text/plain"))
|
|
133
142
|
active_detectors.append(detector)
|
|
134
143
|
else:
|
|
135
144
|
logger.info(
|
|
136
|
-
"
|
|
145
|
+
"File evaluation: '%s' not dispatched on whole file (supports=%s, file_mime=%s)",
|
|
137
146
|
label,
|
|
138
147
|
supported,
|
|
139
148
|
file_mime,
|
|
@@ -158,18 +167,20 @@ class SandboxRunner:
|
|
|
158
167
|
for detector, result in zip(active_detectors, results, strict=False):
|
|
159
168
|
label = self._detector_label(detector)
|
|
160
169
|
if isinstance(result, Exception):
|
|
161
|
-
logger.error("
|
|
170
|
+
logger.error("File evaluation: '%s' failed on whole file: %s", label, result)
|
|
162
171
|
continue
|
|
163
172
|
if isinstance(result, list):
|
|
164
173
|
detector_findings = [f for f in result if isinstance(f, DetectionResult)]
|
|
165
174
|
logger.info(
|
|
166
|
-
"
|
|
175
|
+
"File evaluation: '%s' on whole file → %d finding(s)",
|
|
176
|
+
label,
|
|
177
|
+
len(detector_findings),
|
|
167
178
|
)
|
|
168
179
|
for finding in detector_findings:
|
|
169
180
|
all_findings.append(
|
|
170
181
|
finding.model_copy(
|
|
171
182
|
update={
|
|
172
|
-
"runner_id": "
|
|
183
|
+
"runner_id": "file-evaluation",
|
|
173
184
|
"detected_at": detected_at,
|
|
174
185
|
}
|
|
175
186
|
)
|
|
@@ -203,14 +214,14 @@ class SandboxRunner:
|
|
|
203
214
|
binary_detectors = [d for d in detectors if self._is_binary_detector(d)]
|
|
204
215
|
if not binary_detectors:
|
|
205
216
|
logger.info(
|
|
206
|
-
"
|
|
217
|
+
"File evaluation: %s embeds images but no image-capable detectors are configured; "
|
|
207
218
|
"skipping embedded-image scan",
|
|
208
219
|
mime_type,
|
|
209
220
|
)
|
|
210
221
|
return []
|
|
211
222
|
|
|
212
223
|
logger.info(
|
|
213
|
-
"
|
|
224
|
+
"File evaluation: scanning images embedded in %s with image detectors: [%s]",
|
|
214
225
|
mime_type,
|
|
215
226
|
", ".join(self._detector_label(d) for d in binary_detectors),
|
|
216
227
|
)
|
|
@@ -227,14 +238,14 @@ class SandboxRunner:
|
|
|
227
238
|
]
|
|
228
239
|
if not compatible:
|
|
229
240
|
logger.info(
|
|
230
|
-
"
|
|
241
|
+
"File evaluation: embedded image %s (%s, %.0f KB) — no compatible detector",
|
|
231
242
|
image.location,
|
|
232
243
|
image.mime_type,
|
|
233
244
|
size_kb,
|
|
234
245
|
)
|
|
235
246
|
continue
|
|
236
247
|
logger.info(
|
|
237
|
-
"
|
|
248
|
+
"File evaluation: embedded image %s (%s, %.0f KB) → dispatching [%s]",
|
|
238
249
|
image.location,
|
|
239
250
|
image.mime_type,
|
|
240
251
|
size_kb,
|
|
@@ -246,7 +257,7 @@ class SandboxRunner:
|
|
|
246
257
|
result = await detector.detect(image.image_bytes, image.mime_type)
|
|
247
258
|
except Exception as exc:
|
|
248
259
|
logger.error(
|
|
249
|
-
"
|
|
260
|
+
"File evaluation: '%s' failed on embedded image %s: %s",
|
|
250
261
|
label,
|
|
251
262
|
image.location,
|
|
252
263
|
exc,
|
|
@@ -258,7 +269,7 @@ class SandboxRunner:
|
|
|
258
269
|
else []
|
|
259
270
|
)
|
|
260
271
|
logger.info(
|
|
261
|
-
"
|
|
272
|
+
"File evaluation: '%s' on embedded image %s → %d finding(s)",
|
|
262
273
|
label,
|
|
263
274
|
image.location,
|
|
264
275
|
len(detector_findings),
|
|
@@ -267,7 +278,7 @@ class SandboxRunner:
|
|
|
267
278
|
findings.append(self._tag_embedded(finding, image.location, detected_at))
|
|
268
279
|
|
|
269
280
|
logger.info(
|
|
270
|
-
"
|
|
281
|
+
"File evaluation: embedded-image scan complete — %d image(s) scanned, %d finding(s)",
|
|
271
282
|
image_count,
|
|
272
283
|
len(findings),
|
|
273
284
|
)
|
|
@@ -296,7 +307,7 @@ class SandboxRunner:
|
|
|
296
307
|
location = location.model_copy(update={"path": location_label})
|
|
297
308
|
return finding.model_copy(
|
|
298
309
|
update={
|
|
299
|
-
"runner_id": "
|
|
310
|
+
"runner_id": "file-evaluation",
|
|
300
311
|
"detected_at": detected_at,
|
|
301
312
|
"metadata": metadata,
|
|
302
313
|
"location": location,
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import argparse
|
|
2
|
+
import asyncio
|
|
2
3
|
import json
|
|
3
4
|
import logging
|
|
4
5
|
import os
|
|
@@ -17,6 +18,7 @@ _SURROGATE_RE = re.compile(r"[\ud800-\udfff]")
|
|
|
17
18
|
|
|
18
19
|
_TIMEOUT_PHRASES = ("timed out", "timeout", "connection reset", "errno 110", "connection refused")
|
|
19
20
|
_TIMEOUT_MYSQL_CODES = {2003, 2006, 2013}
|
|
21
|
+
_TEXT_CHUNK_EMIT_ATTEMPTS = 3
|
|
20
22
|
|
|
21
23
|
|
|
22
24
|
def _is_timeout_error(exc: BaseException) -> bool:
|
|
@@ -43,6 +45,34 @@ def _sanitize_for_json(value: Any) -> Any:
|
|
|
43
45
|
return value
|
|
44
46
|
|
|
45
47
|
|
|
48
|
+
async def _emit_text_chunks_with_retry(
|
|
49
|
+
sink: Any,
|
|
50
|
+
asset_hash: str,
|
|
51
|
+
artifact: Any,
|
|
52
|
+
*,
|
|
53
|
+
attempts: int = _TEXT_CHUNK_EMIT_ATTEMPTS,
|
|
54
|
+
) -> None:
|
|
55
|
+
"""Persist chunk provenance, retrying transient API failures.
|
|
56
|
+
|
|
57
|
+
The caller keeps the already-persisted asset successful, but propagates a
|
|
58
|
+
final failure at run level so missing chunks/embeddings cannot look healthy.
|
|
59
|
+
"""
|
|
60
|
+
for attempt in range(1, attempts + 1):
|
|
61
|
+
try:
|
|
62
|
+
await sink.emit_text_chunks(asset_hash, artifact)
|
|
63
|
+
return
|
|
64
|
+
except Exception:
|
|
65
|
+
if attempt == attempts:
|
|
66
|
+
raise
|
|
67
|
+
logger.warning(
|
|
68
|
+
"Text-chunk emission failed for asset %s (attempt %d/%d); retrying",
|
|
69
|
+
asset_hash,
|
|
70
|
+
attempt,
|
|
71
|
+
attempts,
|
|
72
|
+
)
|
|
73
|
+
await asyncio.sleep(0.25 * (2 ** (attempt - 1)))
|
|
74
|
+
|
|
75
|
+
|
|
46
76
|
def setup_logging() -> None:
|
|
47
77
|
"""Configure the logging facility."""
|
|
48
78
|
logging.basicConfig(
|
|
@@ -131,6 +161,10 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
|
|
|
131
161
|
source_id = args.source_id or os.environ.get("SOURCE_ID")
|
|
132
162
|
if source_id:
|
|
133
163
|
os.environ["SOURCE_ID"] = source_id
|
|
164
|
+
if getattr(args, "output_rest_url", None):
|
|
165
|
+
# Sources such as SANDBOX use the same managed-job API base URL as the
|
|
166
|
+
# REST output sink to list and stream their input content.
|
|
167
|
+
os.environ["CLASSIFYRE_OUTPUT_REST_URL"] = args.output_rest_url
|
|
134
168
|
|
|
135
169
|
try:
|
|
136
170
|
try:
|
|
@@ -246,13 +280,18 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
|
|
|
246
280
|
stub_batch = [_asset_to_payload(asset) for asset in raw_batch]
|
|
247
281
|
for stub in stub_batch:
|
|
248
282
|
stub["findings"] = []
|
|
249
|
-
await sink.emit_batch(stub_batch, skip_findings=True)
|
|
250
|
-
output_batch_count += 1
|
|
251
283
|
|
|
284
|
+
# Register BEFORE ingesting: the API stamps per-run
|
|
285
|
+
# change_type onto runner_assets rows during ingest, so
|
|
286
|
+
# the row must exist or the CREATED stamp is lost and
|
|
287
|
+
# first-run counters report every asset as unchanged.
|
|
252
288
|
hashes = [s["hash"] for s in stub_batch if s.get("hash")]
|
|
253
289
|
if hasattr(sink, "register_discovered_assets") and hashes:
|
|
254
290
|
await sink.register_discovered_assets(hashes)
|
|
255
291
|
|
|
292
|
+
await sink.emit_batch(stub_batch, skip_findings=True)
|
|
293
|
+
output_batch_count += 1
|
|
294
|
+
|
|
256
295
|
all_stubs.extend(raw_batch)
|
|
257
296
|
logger.info(
|
|
258
297
|
"Discovered %s assets (total: %s)",
|
|
@@ -265,13 +304,11 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
|
|
|
265
304
|
|
|
266
305
|
# --- Phase 2: Processing ---
|
|
267
306
|
if all_stubs:
|
|
268
|
-
import asyncio as _asyncio
|
|
269
|
-
|
|
270
307
|
processed_count = 0
|
|
271
308
|
_pw = worker_pool.max_workers if worker_pool else 4
|
|
272
309
|
max_concurrent = args.max_concurrent_assets or (_pw * 2)
|
|
273
310
|
max_concurrent = max(1, max_concurrent)
|
|
274
|
-
_asset_semaphore =
|
|
311
|
+
_asset_semaphore = asyncio.Semaphore(max_concurrent)
|
|
275
312
|
logger.info(
|
|
276
313
|
"Phase 2 starting: %d assets, pool_workers=%s, max_concurrent_assets=%d",
|
|
277
314
|
len(all_stubs),
|
|
@@ -279,6 +316,7 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
|
|
|
279
316
|
max_concurrent,
|
|
280
317
|
)
|
|
281
318
|
error_count = 0
|
|
319
|
+
chunk_errors: list[str] = []
|
|
282
320
|
|
|
283
321
|
async def _process_one(asset: Any) -> None:
|
|
284
322
|
async with _asset_semaphore:
|
|
@@ -322,7 +360,24 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
|
|
|
322
360
|
if hasattr(sink, "emit_text_chunks"):
|
|
323
361
|
artifact = pipeline.take_text_artifact(asset_hash)
|
|
324
362
|
if artifact is not None:
|
|
325
|
-
|
|
363
|
+
try:
|
|
364
|
+
await _emit_text_chunks_with_retry(
|
|
365
|
+
sink, asset_hash, artifact
|
|
366
|
+
)
|
|
367
|
+
except Exception as chunk_exc:
|
|
368
|
+
# Findings and the asset remain persisted,
|
|
369
|
+
# preserving B-2's graceful degradation.
|
|
370
|
+
# The run fails after phase 2 so operators
|
|
371
|
+
# cannot mistake missing embeddings for a
|
|
372
|
+
# healthy completed scan.
|
|
373
|
+
detail = f"{asset_hash}: {chunk_exc}"
|
|
374
|
+
chunk_errors.append(detail)
|
|
375
|
+
logger.error(
|
|
376
|
+
"Text-chunk emission exhausted retries"
|
|
377
|
+
" for asset %s: %s",
|
|
378
|
+
asset_hash,
|
|
379
|
+
chunk_exc,
|
|
380
|
+
)
|
|
326
381
|
|
|
327
382
|
if hasattr(sink, "update_asset_status"):
|
|
328
383
|
f_total, f_by_sev, f_by_det = _compute_findings_counts(
|
|
@@ -350,13 +405,22 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
|
|
|
350
405
|
except Exception:
|
|
351
406
|
pass
|
|
352
407
|
|
|
353
|
-
tasks = [
|
|
354
|
-
await
|
|
408
|
+
tasks = [asyncio.create_task(_process_one(a)) for a in all_stubs]
|
|
409
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
355
410
|
logger.info(
|
|
356
411
|
"Phase 2 complete: %d processed, %d errors",
|
|
357
412
|
processed_count,
|
|
358
413
|
error_count,
|
|
359
414
|
)
|
|
415
|
+
if chunk_errors:
|
|
416
|
+
preview = "; ".join(chunk_errors[:3])
|
|
417
|
+
remaining = len(chunk_errors) - 3
|
|
418
|
+
if remaining > 0:
|
|
419
|
+
preview += f"; and {remaining} more"
|
|
420
|
+
raise RuntimeError(
|
|
421
|
+
f"Text-chunk emission failed for {len(chunk_errors)}"
|
|
422
|
+
f" asset(s) after retries: {preview}"
|
|
423
|
+
)
|
|
360
424
|
|
|
361
425
|
# Persist the advanced AUTOMATIC sampling cursor (no-op for
|
|
362
426
|
# other strategies, which return None). Only on the normal
|
|
@@ -461,13 +525,13 @@ def run_train_command(args: argparse.Namespace) -> None:
|
|
|
461
525
|
print(json.dumps(result.to_dict()))
|
|
462
526
|
|
|
463
527
|
|
|
464
|
-
def
|
|
465
|
-
"""
|
|
466
|
-
from .
|
|
528
|
+
def run_evaluate_file_command(args: argparse.Namespace) -> None:
|
|
529
|
+
"""Evaluate detector configs against one ephemeral local file."""
|
|
530
|
+
from .file_evaluation import FileEvaluationRunner
|
|
467
531
|
|
|
468
532
|
file_path_str: str | None = args.recipe
|
|
469
533
|
if not file_path_str:
|
|
470
|
-
logger.error("
|
|
534
|
+
logger.error("evaluate-file requires a file path as the first argument")
|
|
471
535
|
sys.exit(1)
|
|
472
536
|
|
|
473
537
|
file_path = Path(file_path_str)
|
|
@@ -492,7 +556,7 @@ def run_sandbox_command(args: argparse.Namespace) -> None:
|
|
|
492
556
|
sys.exit(1)
|
|
493
557
|
|
|
494
558
|
try:
|
|
495
|
-
runner =
|
|
559
|
+
runner = FileEvaluationRunner(detectors)
|
|
496
560
|
parsed, findings = runner.run(file_path)
|
|
497
561
|
if parsed.parse_error:
|
|
498
562
|
logger.warning("File parse warning: %s", parsed.parse_error)
|
|
@@ -502,9 +566,11 @@ def run_sandbox_command(args: argparse.Namespace) -> None:
|
|
|
502
566
|
}
|
|
503
567
|
if parsed.parse_error:
|
|
504
568
|
output["parse_error"] = parsed.parse_error
|
|
569
|
+
if runner.detector_errors:
|
|
570
|
+
output["detector_errors"] = runner.detector_errors
|
|
505
571
|
print(json.dumps(_sanitize_for_json(output), ensure_ascii=False))
|
|
506
572
|
except Exception as e:
|
|
507
|
-
logger.error("
|
|
573
|
+
logger.error("File evaluation failed: %s", e, exc_info=True)
|
|
508
574
|
sys.exit(1)
|
|
509
575
|
|
|
510
576
|
|
|
@@ -520,14 +586,14 @@ def main() -> None:
|
|
|
520
586
|
parser = argparse.ArgumentParser(description="Classifyre Metadata Extraction CLI")
|
|
521
587
|
parser.add_argument(
|
|
522
588
|
"command",
|
|
523
|
-
choices=["test", "extract", "discover", "
|
|
589
|
+
choices=["test", "extract", "discover", "evaluate-file", "train"],
|
|
524
590
|
help="Command to run",
|
|
525
591
|
)
|
|
526
592
|
parser.add_argument(
|
|
527
593
|
"recipe",
|
|
528
594
|
nargs="?",
|
|
529
595
|
default=None,
|
|
530
|
-
help="Path to recipe JSON (or file path for
|
|
596
|
+
help="Path to recipe JSON (or file path for evaluate-file)",
|
|
531
597
|
)
|
|
532
598
|
parser.add_argument("--debug", action="store_true", help="Enable debug logging")
|
|
533
599
|
parser.add_argument(
|
|
@@ -565,7 +631,7 @@ def main() -> None:
|
|
|
565
631
|
parser.add_argument(
|
|
566
632
|
"--detectors-file",
|
|
567
633
|
default=None,
|
|
568
|
-
help="Path to JSON file with detector configs (
|
|
634
|
+
help="Path to JSON file with detector configs (evaluate-file only)",
|
|
569
635
|
)
|
|
570
636
|
# train-command arguments
|
|
571
637
|
parser.add_argument(
|
|
@@ -629,8 +695,8 @@ def main() -> None:
|
|
|
629
695
|
if args.debug:
|
|
630
696
|
logging.getLogger().setLevel(logging.DEBUG)
|
|
631
697
|
|
|
632
|
-
if args.command == "
|
|
633
|
-
|
|
698
|
+
if args.command == "evaluate-file":
|
|
699
|
+
run_evaluate_file_command(args)
|
|
634
700
|
return
|
|
635
701
|
|
|
636
702
|
if args.command == "train":
|
|
@@ -54,6 +54,7 @@ class AssetType(StrEnum):
|
|
|
54
54
|
LOCAL_FOLDER = 'LOCAL_FOLDER'
|
|
55
55
|
MICROSOFT_365 = 'MICROSOFT_365'
|
|
56
56
|
GOOGLE_WORKSPACE = 'GOOGLE_WORKSPACE'
|
|
57
|
+
SANDBOX = 'SANDBOX'
|
|
57
58
|
|
|
58
59
|
|
|
59
60
|
class DetectorType(StrEnum):
|
|
@@ -353,6 +354,7 @@ class Type(StrEnum):
|
|
|
353
354
|
LOCAL_FOLDER = 'LOCAL_FOLDER'
|
|
354
355
|
MICROSOFT_365 = 'MICROSOFT_365'
|
|
355
356
|
GOOGLE_WORKSPACE = 'GOOGLE_WORKSPACE'
|
|
357
|
+
SANDBOX = 'SANDBOX'
|
|
356
358
|
|
|
357
359
|
|
|
358
360
|
class YouTubeRequired(BaseModel):
|
|
@@ -648,6 +650,32 @@ class LocalFolderRequired(BaseModel):
|
|
|
648
650
|
)
|
|
649
651
|
|
|
650
652
|
|
|
653
|
+
class SandboxRequired(BaseModel):
|
|
654
|
+
"""
|
|
655
|
+
Sandbox files are uploaded separately through the source files API.
|
|
656
|
+
"""
|
|
657
|
+
|
|
658
|
+
model_config = ConfigDict(
|
|
659
|
+
extra='forbid',
|
|
660
|
+
)
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
class SandboxMasked(BaseModel):
|
|
664
|
+
"""
|
|
665
|
+
Sandbox sources do not require credentials.
|
|
666
|
+
"""
|
|
667
|
+
|
|
668
|
+
model_config = ConfigDict(
|
|
669
|
+
extra='forbid',
|
|
670
|
+
)
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
class SandboxOptional(BaseModel):
|
|
674
|
+
model_config = ConfigDict(
|
|
675
|
+
extra='forbid',
|
|
676
|
+
)
|
|
677
|
+
|
|
678
|
+
|
|
651
679
|
class LocalFolderMasked(BaseModel):
|
|
652
680
|
"""
|
|
653
681
|
Local folder scans run with the desktop application's filesystem permissions; no credentials are required.
|
|
@@ -2170,6 +2198,24 @@ class LocalFolderInput(CoreInput):
|
|
|
2170
2198
|
resources: ResourceOverrides | None = None
|
|
2171
2199
|
|
|
2172
2200
|
|
|
2201
|
+
class SandboxInput(CoreInput):
|
|
2202
|
+
type: Literal['SANDBOX'] | None = Field(
|
|
2203
|
+
None, description='Type of the asset or source'
|
|
2204
|
+
)
|
|
2205
|
+
required: SandboxRequired
|
|
2206
|
+
masked: SandboxMasked | None = None
|
|
2207
|
+
optional: SandboxOptional | None = None
|
|
2208
|
+
detectors: list[Detector] | None = Field(
|
|
2209
|
+
None, description='Detectors to run on ingested content'
|
|
2210
|
+
)
|
|
2211
|
+
custom_detectors: list[CustomDetectorSelection] | None = Field(
|
|
2212
|
+
None,
|
|
2213
|
+
description='Reusable custom detector IDs selected from the custom detector catalog.',
|
|
2214
|
+
)
|
|
2215
|
+
sampling: SamplingConfig
|
|
2216
|
+
resources: ResourceOverrides | None = None
|
|
2217
|
+
|
|
2218
|
+
|
|
2173
2219
|
class AzureBlobStorageInput(CoreInput):
|
|
2174
2220
|
type: Literal['AZURE_BLOB_STORAGE'] | None = Field(
|
|
2175
2221
|
None, description='Type of the asset or source'
|
|
@@ -2601,7 +2647,7 @@ class ConfluenceOptionalConnection(BaseModel):
|
|
|
2601
2647
|
)
|
|
2602
2648
|
|
|
2603
2649
|
|
|
2604
|
-
class
|
|
2650
|
+
class Type20(StrEnum):
|
|
2605
2651
|
"""
|
|
2606
2652
|
Filter spaces by space type
|
|
2607
2653
|
"""
|
|
@@ -2638,7 +2684,7 @@ class ConfluenceOptionalScopeSpaces(BaseModel):
|
|
|
2638
2684
|
keys: list[str] | None = Field(
|
|
2639
2685
|
None, description='Filter spaces by keys (up to 250)', max_length=250
|
|
2640
2686
|
)
|
|
2641
|
-
type:
|
|
2687
|
+
type: Type20 | None = Field(None, description='Filter spaces by space type')
|
|
2642
2688
|
status: Status | None = Field(None, description='Filter spaces by status')
|
|
2643
2689
|
labels: list[str] | None = Field(
|
|
2644
2690
|
None,
|
|
@@ -2904,7 +2950,7 @@ class ServiceDeskOptional(BaseModel):
|
|
|
2904
2950
|
content: ServiceDeskOptionalContent | None = None
|
|
2905
2951
|
|
|
2906
2952
|
|
|
2907
|
-
class
|
|
2953
|
+
class Type21(StrEnum):
|
|
2908
2954
|
"""
|
|
2909
2955
|
Type of the asset or source
|
|
2910
2956
|
"""
|
|
@@ -2941,6 +2987,7 @@ class Type20(StrEnum):
|
|
|
2941
2987
|
LOCAL_FOLDER = 'LOCAL_FOLDER'
|
|
2942
2988
|
MICROSOFT_365 = 'MICROSOFT_365'
|
|
2943
2989
|
GOOGLE_WORKSPACE = 'GOOGLE_WORKSPACE'
|
|
2990
|
+
SANDBOX = 'SANDBOX'
|
|
2944
2991
|
|
|
2945
2992
|
|
|
2946
2993
|
class ConfluenceInput(CoreInput):
|
|
@@ -4075,7 +4122,8 @@ class IcebergInput(CoreInput):
|
|
|
4075
4122
|
|
|
4076
4123
|
class SourceInput(
|
|
4077
4124
|
RootModel[
|
|
4078
|
-
|
|
4125
|
+
SandboxInput
|
|
4126
|
+
| SlackInput
|
|
4079
4127
|
| S3CompatibleStorageInput
|
|
4080
4128
|
| LocalFolderInput
|
|
4081
4129
|
| AzureBlobStorageInput
|
|
@@ -4110,7 +4158,8 @@ class SourceInput(
|
|
|
4110
4158
|
]
|
|
4111
4159
|
):
|
|
4112
4160
|
root: (
|
|
4113
|
-
|
|
4161
|
+
SandboxInput
|
|
4162
|
+
| SlackInput
|
|
4114
4163
|
| S3CompatibleStorageInput
|
|
4115
4164
|
| LocalFolderInput
|
|
4116
4165
|
| AzureBlobStorageInput
|