classifyre-cli 0.4.73__tar.gz → 0.4.74__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/.turbo/turbo-build.log +1 -1
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/PKG-INFO +30 -1
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/README.md +29 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/package.json +1 -1
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/pyproject.toml +1 -1
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/outputs/rest.py +14 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/utils/legacy_office.py +90 -19
- classifyre_cli-0.4.74/tests/utils/test_legacy_office.py +281 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/uv.lock +8 -8
- classifyre_cli-0.4.73/tests/utils/test_legacy_office.py +0 -66
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/.gitignore +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/.python-version +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/main.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/scripts/generate_models.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/config.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/base.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/broken_links/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/broken_links/detector.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/config.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/content/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/custom/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/custom/detector.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/custom/extractor.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/custom/runners/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/custom/runners/_base.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/custom/runners/_factory.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/custom/runners/_gliner2.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/custom/runners/_image_classification.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/custom/runners/_llm.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/custom/runners/_object_detection.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/custom/runners/_regex.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/custom/runners/_text_classification.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/custom/trainer.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/dependencies.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/pii/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/pii/detector.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/secrets/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/secrets/detector.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/threat/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/threat/code_security_detector.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/detectors/threat/yara_detector.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/file_evaluation/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/file_evaluation/runner.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/main.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/models/generated_detectors.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/models/generated_input.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/models/generated_single_asset_scan_results.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/outputs/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/outputs/base.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/outputs/console.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/outputs/factory.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/outputs/file.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/pipeline/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/pipeline/content_provider.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/pipeline/detector_pipeline.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/pipeline/parsed_content_provider.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/pipeline/text_artifact.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/pipeline/worker_pool.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/asset_metadata.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/atlassian_common.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/azure_blob_storage/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/azure_blob_storage/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/base.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/confluence/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/confluence/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/databricks/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/databricks/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/delta_lake/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/delta_lake/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/dependencies.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/dropbox/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/dropbox/auth.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/dropbox/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/elasticsearch/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/elasticsearch/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/email/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/email/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/google_cloud_storage/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/google_cloud_storage/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/google_workspace/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/google_workspace/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/hive/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/hive/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/iceberg/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/iceberg/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/jira/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/jira/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/kafka/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/kafka/rest.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/kafka/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/lakehouse_base.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/local_folder/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/local_folder/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/meilisearch/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/meilisearch/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/microsoft_365/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/microsoft_365/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/mongodb/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/mongodb/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/mssql/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/mssql/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/mysql/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/mysql/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/neo4j/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/neo4j/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/notion/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/notion/client.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/notion/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/object_storage/base.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/opensearch/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/opensearch/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/oracle/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/oracle/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/postgresql/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/postgresql/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/powerbi/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/powerbi/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/recipe_normalizer.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/s3_client.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/s3_compatible_storage/README.md +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/s3_compatible_storage/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/s3_compatible_storage/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/sandbox/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/sandbox/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/search_engine_base.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/servicedesk/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/servicedesk/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/slack/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/slack/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/snowflake/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/snowflake/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/sqlite/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/sqlite/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/tableau/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/tableau/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/tabular_base.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/tabular_utils.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/wordpress/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/wordpress/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/youtube/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/sources/youtube/source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/telemetry.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/utils/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/utils/archive_extraction.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/utils/content_extraction.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/utils/dependency_groups.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/utils/embedded_images.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/utils/file_metadata.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/utils/file_parser.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/utils/file_to_images.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/utils/hashing.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/utils/resources.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/utils/transcription.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/utils/uv_sync.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/utils/validation.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/src/utils/video_processing.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/_lakehouse_fakes.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/conftest.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/broken_links/test_broken_links_detector.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/conftest.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/content/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/custom/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/custom/conftest.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/custom/test_gliner2_classification.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/custom/test_invoice_extraction.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/custom/test_llm_runner.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/custom/test_pipeline_integration.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/custom/test_regex_runner.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/custom/test_runner_hardening.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/custom/test_transformer_runners.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/pii/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/pii/conftest.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/pii/sample_invoice.pdf +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/pii/test_pii_chunking.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/pii/test_pii_detector.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/pii/test_pii_detector_extended.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/pii/test_pii_regional_entities.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/pii/test_pii_severity.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/secrets/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/secrets/test_secrets_detector.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/secrets/test_secrets_detector_extended.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/test_base_detector.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/test_custom_detector_examples_runtime.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/test_detector_catalog_commercial.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/test_detector_pipeline_types.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/test_detector_schema_examples.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/test_detector_types.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/test_phase2_detectors.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/test_registry.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/threat/__init__.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/threat/test_code_security_detector.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/detectors/threat/test_yara_detector.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/integration/test_wordpress_broken_links_detector.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/integration/test_wordpress_links_assets.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/pipeline/test_detector_outcomes.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/pipeline/test_detector_pipeline.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/pipeline/test_worker_pool.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_assets_metadata_catalog.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_azure_blob_storage_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_base_source_attachment.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_base_source_sampling.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_config.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_confluence_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_custom_extractor.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_databricks_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_delta_lake_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_dependency_groups.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_dropbox_auth_command.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_dropbox_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_elasticsearch_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_email_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_file_evaluation_runner.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_google_cloud_storage_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_google_workspace_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_hashing.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_hive_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_iceberg_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_jira_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_kafka_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_local_folder_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_main.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_meilisearch_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_microsoft_365_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_mongodb_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_mssql_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_mysql_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_neo4j_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_notion_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_opensearch_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_oracle_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_outputs.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_postgresql_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_powerbi_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_recipe_normalizer.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_s3_compatible_storage_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_sampling_automatic.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_sandbox_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_servicedesk_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_slack_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_snowflake_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_source_dependency_groups.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_source_strategy_conformance.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_sqlite_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_tableau_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_tabular_automatic_sampling.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_tabular_utils.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_uv_sync.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_wordpress_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_youtube_source.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/test_youtube_source_integration.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/utils/test_archive_extraction.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/utils/test_content_extraction.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/utils/test_embedded_images.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/utils/test_file_metadata.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/utils/test_file_parser.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/utils/test_file_to_images.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/utils/test_transcription.py +0 -0
- {classifyre_cli-0.4.73 → classifyre_cli-0.4.74}/tests/utils/test_video_processing.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: classifyre-cli
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.74
|
|
4
4
|
Summary: Classifyre CLI — scan and classify unstructured data sources
|
|
5
5
|
License: MIT
|
|
6
6
|
Keywords: data,ingestion,metadata,pii,secrets,unstructured
|
|
@@ -35,6 +35,35 @@ uv sync --group detectors
|
|
|
35
35
|
# or specific groups: --group secrets --group pii --group threat ...
|
|
36
36
|
```
|
|
37
37
|
|
|
38
|
+
### System dependency: LibreOffice
|
|
39
|
+
|
|
40
|
+
Legacy Office formats (`.doc`, `.xls`, `.ppt`) have no usable pure-Python parser,
|
|
41
|
+
so `src/utils/legacy_office.py` shells out to `soffice --headless --convert-to
|
|
42
|
+
docx|xlsx|pptx` and feeds the result through the normal OOXML extractors. Without
|
|
43
|
+
it those files scan as *"no content available"* — a silent coverage gap, not an
|
|
44
|
+
error.
|
|
45
|
+
|
|
46
|
+
- **Kubernetes**: installed in the CLI image (`libreoffice-*-nogui`), so this
|
|
47
|
+
works out of the box.
|
|
48
|
+
- **Desktop and local dev**: install it yourself — `brew install --cask libreoffice`
|
|
49
|
+
(macOS), `apt install libreoffice-writer libreoffice-calc libreoffice-impress`
|
|
50
|
+
(Debian/Ubuntu), or libreoffice.org on Windows. The desktop app deliberately
|
|
51
|
+
does not bundle LibreOffice: it is ~550 MB even stripped, and slimming the
|
|
52
|
+
upstream bundle breaks its code signature, which Apple Silicon punishes by
|
|
53
|
+
killing the process on launch.
|
|
54
|
+
- **Non-standard install**: set `CLASSIFYRE_SOFFICE_PATH` to the binary. The CLI
|
|
55
|
+
checks that first, then `PATH`, then the standard macOS, Windows, and
|
|
56
|
+
Linux-tarball install locations.
|
|
57
|
+
|
|
58
|
+
When no binary is found the file is **not** silently reported as empty. The scan
|
|
59
|
+
raises an `ENGINE_UNAVAILABLE` text-extraction coverage error that fails that
|
|
60
|
+
asset's text scan, lands in `scan_stats.errors` and the run's text-coverage
|
|
61
|
+
counters, and carries a platform-specific message telling the user how to install
|
|
62
|
+
LibreOffice.
|
|
63
|
+
|
|
64
|
+
`tests/utils/test_legacy_office.py` skips its round-trip tests when no LibreOffice
|
|
65
|
+
is found, so a green local run does not by itself prove this path works.
|
|
66
|
+
|
|
38
67
|
## Command Syntax
|
|
39
68
|
|
|
40
69
|
Use the thin wrapper:
|
|
@@ -18,6 +18,35 @@ uv sync --group detectors
|
|
|
18
18
|
# or specific groups: --group secrets --group pii --group threat ...
|
|
19
19
|
```
|
|
20
20
|
|
|
21
|
+
### System dependency: LibreOffice
|
|
22
|
+
|
|
23
|
+
Legacy Office formats (`.doc`, `.xls`, `.ppt`) have no usable pure-Python parser,
|
|
24
|
+
so `src/utils/legacy_office.py` shells out to `soffice --headless --convert-to
|
|
25
|
+
docx|xlsx|pptx` and feeds the result through the normal OOXML extractors. Without
|
|
26
|
+
it those files scan as *"no content available"* — a silent coverage gap, not an
|
|
27
|
+
error.
|
|
28
|
+
|
|
29
|
+
- **Kubernetes**: installed in the CLI image (`libreoffice-*-nogui`), so this
|
|
30
|
+
works out of the box.
|
|
31
|
+
- **Desktop and local dev**: install it yourself — `brew install --cask libreoffice`
|
|
32
|
+
(macOS), `apt install libreoffice-writer libreoffice-calc libreoffice-impress`
|
|
33
|
+
(Debian/Ubuntu), or libreoffice.org on Windows. The desktop app deliberately
|
|
34
|
+
does not bundle LibreOffice: it is ~550 MB even stripped, and slimming the
|
|
35
|
+
upstream bundle breaks its code signature, which Apple Silicon punishes by
|
|
36
|
+
killing the process on launch.
|
|
37
|
+
- **Non-standard install**: set `CLASSIFYRE_SOFFICE_PATH` to the binary. The CLI
|
|
38
|
+
checks that first, then `PATH`, then the standard macOS, Windows, and
|
|
39
|
+
Linux-tarball install locations.
|
|
40
|
+
|
|
41
|
+
When no binary is found the file is **not** silently reported as empty. The scan
|
|
42
|
+
raises an `ENGINE_UNAVAILABLE` text-extraction coverage error that fails that
|
|
43
|
+
asset's text scan, lands in `scan_stats.errors` and the run's text-coverage
|
|
44
|
+
counters, and carries a platform-specific message telling the user how to install
|
|
45
|
+
LibreOffice.
|
|
46
|
+
|
|
47
|
+
`tests/utils/test_legacy_office.py` skips its round-trip tests when no LibreOffice
|
|
48
|
+
is found, so a green local run does not by itself prove this path works.
|
|
49
|
+
|
|
21
50
|
## Command Syntax
|
|
22
51
|
|
|
23
52
|
Use the thin wrapper:
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
|
+
import os
|
|
4
5
|
import random
|
|
5
6
|
from typing import Any, Literal, cast
|
|
6
7
|
from urllib.parse import urljoin
|
|
@@ -15,6 +16,18 @@ from .base import OutputRuntimeContext, OutputType
|
|
|
15
16
|
|
|
16
17
|
logger = logging.getLogger(__name__)
|
|
17
18
|
|
|
19
|
+
# Proves to the API that these write-backs come from a scan job it launched.
|
|
20
|
+
# The API has no user authentication, so the ingest endpoints are restricted to
|
|
21
|
+
# callers holding this shared secret; the API injects it into every CLI job's
|
|
22
|
+
# environment (Kubernetes job env, or inherited from the parent process on
|
|
23
|
+
# desktop). Absent locally, where the API leaves the restriction disabled.
|
|
24
|
+
INTERNAL_KEY_HEADER = "X-Classifyre-Internal-Key"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _internal_key_headers() -> dict[str, str]:
|
|
28
|
+
key = os.environ.get("CLASSIFYRE_INTERNAL_KEY", "").strip()
|
|
29
|
+
return {INTERNAL_KEY_HEADER: key} if key else {}
|
|
30
|
+
|
|
18
31
|
|
|
19
32
|
class _JitteredRetry(Retry):
|
|
20
33
|
"""urllib3 Retry subclass that adds ±25 % multiplicative jitter to the
|
|
@@ -164,6 +177,7 @@ class RestOutputSink:
|
|
|
164
177
|
# a pod restart or server-side keep-alive timeout. Each request opens
|
|
165
178
|
# a fresh TCP connection, which is cheap enough for our batch cadence.
|
|
166
179
|
self.session.headers.update({"Connection": "close"})
|
|
180
|
+
self.session.headers.update(_internal_key_headers())
|
|
167
181
|
adapter = HTTPAdapter(max_retries=_RETRY_POLICY)
|
|
168
182
|
self.session.mount("http://", adapter)
|
|
169
183
|
self.session.mount("https://", adapter)
|
|
@@ -6,16 +6,25 @@ is fed back through the existing docx/xlsx/pptx extraction paths:
|
|
|
6
6
|
|
|
7
7
|
.doc / .xls / .ppt → soffice --headless --convert-to → .docx / .xlsx / .pptx
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
The CLI container image ships LibreOffice (the ``libreoffice-*-nogui`` Debian
|
|
10
|
+
packages, which install ``/usr/bin/soffice``), so conversion works out of the box
|
|
11
|
+
under Kubernetes. The desktop app deliberately does NOT bundle it — ~550 MB even
|
|
12
|
+
stripped, and slimming the upstream bundle breaks its code signature — so there
|
|
13
|
+
it is a system dependency the user installs.
|
|
14
|
+
|
|
15
|
+
When no binary is found the file is **not** quietly reported as empty: the error
|
|
16
|
+
from :func:`soffice_missing_error` propagates as an ENGINE_UNAVAILABLE text
|
|
17
|
+
extraction coverage failure, which fails that asset's text scan and tells the
|
|
18
|
+
user, per platform, how to install LibreOffice.
|
|
12
19
|
"""
|
|
13
20
|
|
|
14
21
|
from __future__ import annotations
|
|
15
22
|
|
|
16
23
|
import logging
|
|
24
|
+
import os
|
|
17
25
|
import shutil
|
|
18
26
|
import subprocess
|
|
27
|
+
import sys
|
|
19
28
|
import tempfile
|
|
20
29
|
import threading
|
|
21
30
|
from functools import cache
|
|
@@ -46,13 +55,58 @@ LEGACY_OFFICE_MIME_TYPES = frozenset(_CONVERSION_TARGETS)
|
|
|
46
55
|
|
|
47
56
|
_SOFFICE_TIMEOUT_SECONDS = 120
|
|
48
57
|
|
|
49
|
-
#
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
"
|
|
54
|
-
"
|
|
55
|
-
|
|
58
|
+
# How to actually get LibreOffice, per platform. The desktop app does not bundle
|
|
59
|
+
# it, so this message is the entire remediation path a user gets — a generic
|
|
60
|
+
# "install LibreOffice" would leave them guessing which package to install.
|
|
61
|
+
_INSTALL_HINTS: dict[str, str] = {
|
|
62
|
+
"darwin": "install it with `brew install --cask libreoffice`, or from libreoffice.org",
|
|
63
|
+
"win32": "install it from https://www.libreoffice.org/download/",
|
|
64
|
+
"linux": (
|
|
65
|
+
"install it with "
|
|
66
|
+
"`apt install libreoffice-writer libreoffice-calc libreoffice-impress` "
|
|
67
|
+
"(or the equivalent for your distribution)"
|
|
68
|
+
),
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def soffice_missing_error() -> str:
|
|
73
|
+
"""Actionable message for a deployment with no LibreOffice.
|
|
74
|
+
|
|
75
|
+
The word "unavailable" is load-bearing: it is what makes iter_file_pages
|
|
76
|
+
classify this as ENGINE_UNAVAILABLE rather than a per-document FAILED — the
|
|
77
|
+
difference between "this install cannot read .doc at all" and "this one file
|
|
78
|
+
is broken". test_legacy_office.py pins that mapping so the two cannot drift.
|
|
79
|
+
"""
|
|
80
|
+
hint = _INSTALL_HINTS.get(sys.platform, "install LibreOffice")
|
|
81
|
+
return (
|
|
82
|
+
"LibreOffice is unavailable, so this .doc/.xls/.ppt file was NOT scanned. "
|
|
83
|
+
f"To scan legacy Office documents, {hint} — "
|
|
84
|
+
"or set CLASSIFYRE_SOFFICE_PATH to an existing soffice binary."
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
# Explicit override, checked before anything else. The desktop app rebuilds a
|
|
89
|
+
# minimal PATH for the processes it spawns, so an installation it discovered
|
|
90
|
+
# itself is handed down through this variable rather than through PATH.
|
|
91
|
+
SOFFICE_PATH_ENV = "CLASSIFYRE_SOFFICE_PATH"
|
|
92
|
+
|
|
93
|
+
# Install locations to check when PATH comes up empty, keyed by sys.platform.
|
|
94
|
+
# macOS and Windows always need these: their installers drop the binary inside an
|
|
95
|
+
# app bundle / Program Files without touching PATH, and the packaged desktop app
|
|
96
|
+
# rebuilds a minimal PATH that cannot see either. Linux only needs them for an
|
|
97
|
+
# upstream tarball install — the distro packages (including the
|
|
98
|
+
# libreoffice-*-nogui ones in the CLI image) all land on /usr/bin/soffice.
|
|
99
|
+
_SOFFICE_FALLBACK_PATHS: dict[str, tuple[str, ...]] = {
|
|
100
|
+
"darwin": ("/Applications/LibreOffice.app/Contents/MacOS/soffice",),
|
|
101
|
+
"linux": (
|
|
102
|
+
"/opt/libreoffice/program/soffice",
|
|
103
|
+
"/usr/lib/libreoffice/program/soffice",
|
|
104
|
+
),
|
|
105
|
+
"win32": (
|
|
106
|
+
"C:\\Program Files\\LibreOffice\\program\\soffice.exe",
|
|
107
|
+
"C:\\Program Files (x86)\\LibreOffice\\program\\soffice.exe",
|
|
108
|
+
),
|
|
109
|
+
}
|
|
56
110
|
|
|
57
111
|
# One conversion at a time: each soffice launch is a full process spinning up
|
|
58
112
|
# ~200 MB; serializing keeps peak memory bounded alongside detector workloads.
|
|
@@ -61,14 +115,36 @@ _conversion_lock = threading.Lock()
|
|
|
61
115
|
|
|
62
116
|
@cache
|
|
63
117
|
def find_soffice() -> str | None:
|
|
64
|
-
"""Locate the LibreOffice binary, or None when it is not installed.
|
|
118
|
+
"""Locate the LibreOffice binary, or None when it is not installed.
|
|
119
|
+
|
|
120
|
+
Resolution order, most explicit first:
|
|
121
|
+
|
|
122
|
+
1. ``CLASSIFYRE_SOFFICE_PATH`` — an operator/desktop-supplied absolute path.
|
|
123
|
+
2. ``PATH`` — the CLI container, Linux hosts, and dev shells.
|
|
124
|
+
3. Platform install locations — macOS/Windows only (see the table above).
|
|
125
|
+
|
|
126
|
+
Cached: the answer cannot change within a process. Tests that manipulate the
|
|
127
|
+
environment must call ``find_soffice.cache_clear()``.
|
|
128
|
+
"""
|
|
129
|
+
override = os.environ.get(SOFFICE_PATH_ENV, "").strip()
|
|
130
|
+
if override:
|
|
131
|
+
if Path(override).is_file():
|
|
132
|
+
return override
|
|
133
|
+
logger.warning(
|
|
134
|
+
"%s=%s does not point at an existing file; falling back to PATH lookup",
|
|
135
|
+
SOFFICE_PATH_ENV,
|
|
136
|
+
override,
|
|
137
|
+
)
|
|
138
|
+
|
|
65
139
|
for name in ("soffice", "libreoffice"):
|
|
66
140
|
path = shutil.which(name)
|
|
67
141
|
if path:
|
|
68
142
|
return path
|
|
69
|
-
|
|
70
|
-
|
|
143
|
+
|
|
144
|
+
for candidate in _SOFFICE_FALLBACK_PATHS.get(sys.platform, ()):
|
|
145
|
+
if Path(candidate).is_file():
|
|
71
146
|
return candidate
|
|
147
|
+
|
|
72
148
|
return None
|
|
73
149
|
|
|
74
150
|
|
|
@@ -90,12 +166,7 @@ def convert_legacy_office(
|
|
|
90
166
|
|
|
91
167
|
soffice = find_soffice()
|
|
92
168
|
if soffice is None:
|
|
93
|
-
return (
|
|
94
|
-
None,
|
|
95
|
-
target_mime,
|
|
96
|
-
"LibreOffice (soffice) not found; install LibreOffice to enable "
|
|
97
|
-
".doc/.xls/.ppt extraction",
|
|
98
|
-
)
|
|
169
|
+
return None, target_mime, soffice_missing_error()
|
|
99
170
|
|
|
100
171
|
try:
|
|
101
172
|
with tempfile.TemporaryDirectory(prefix="classifyre-soffice-") as temp_dir:
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
from src.utils import legacy_office
|
|
8
|
+
from src.utils.legacy_office import SOFFICE_PATH_ENV, convert_legacy_office, find_soffice
|
|
9
|
+
|
|
10
|
+
_DOCX_MIME = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
|
|
11
|
+
_XLSX_MIME = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
|
12
|
+
_PPTX_MIME = "application/vnd.openxmlformats-officedocument.presentationml.presentation"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@pytest.fixture(autouse=True)
|
|
16
|
+
def _clear_soffice_cache() -> object:
|
|
17
|
+
"""find_soffice() is @cache'd, so every environment tweak needs a reset."""
|
|
18
|
+
find_soffice.cache_clear()
|
|
19
|
+
yield
|
|
20
|
+
find_soffice.cache_clear()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class TestFindSoffice:
|
|
24
|
+
"""The resolution contract each deployment relies on.
|
|
25
|
+
|
|
26
|
+
Kubernetes finds /usr/bin/soffice on PATH (the libreoffice-*-nogui packages
|
|
27
|
+
baked into the CLI image); the packaged desktop app rebuilds a minimal PATH
|
|
28
|
+
that excludes /Applications, so macOS and Windows depend on the platform
|
|
29
|
+
fallback table; operators and the desktop override both with an env var.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def test_env_override_wins_over_path(
|
|
33
|
+
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
34
|
+
) -> None:
|
|
35
|
+
override = tmp_path / "custom-soffice"
|
|
36
|
+
override.write_text("#!/bin/sh\n")
|
|
37
|
+
monkeypatch.setattr(legacy_office.shutil, "which", lambda _name: "/usr/bin/soffice")
|
|
38
|
+
monkeypatch.setenv(SOFFICE_PATH_ENV, str(override))
|
|
39
|
+
|
|
40
|
+
assert find_soffice() == str(override)
|
|
41
|
+
|
|
42
|
+
def test_env_override_pointing_nowhere_falls_back_to_path(
|
|
43
|
+
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
44
|
+
) -> None:
|
|
45
|
+
monkeypatch.setattr(
|
|
46
|
+
legacy_office.shutil,
|
|
47
|
+
"which",
|
|
48
|
+
lambda name: "/usr/bin/soffice" if name == "soffice" else None,
|
|
49
|
+
)
|
|
50
|
+
monkeypatch.setenv(SOFFICE_PATH_ENV, str(tmp_path / "does-not-exist"))
|
|
51
|
+
|
|
52
|
+
assert find_soffice() == "/usr/bin/soffice"
|
|
53
|
+
|
|
54
|
+
def test_path_lookup_finds_container_install(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
55
|
+
"""The CLI image installs /usr/bin/soffice; no fallback should be needed."""
|
|
56
|
+
monkeypatch.delenv(SOFFICE_PATH_ENV, raising=False)
|
|
57
|
+
monkeypatch.setattr(
|
|
58
|
+
legacy_office.shutil,
|
|
59
|
+
"which",
|
|
60
|
+
lambda name: "/usr/bin/soffice" if name == "soffice" else None,
|
|
61
|
+
)
|
|
62
|
+
monkeypatch.setattr(legacy_office.sys, "platform", "linux")
|
|
63
|
+
|
|
64
|
+
assert find_soffice() == "/usr/bin/soffice"
|
|
65
|
+
|
|
66
|
+
@pytest.mark.parametrize("platform", ["darwin", "win32", "linux"])
|
|
67
|
+
def test_platform_fallback_used_when_path_is_scrubbed(
|
|
68
|
+
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, platform: str
|
|
69
|
+
) -> None:
|
|
70
|
+
"""Reproduces packaged-desktop mode: PATH holds no LibreOffice at all.
|
|
71
|
+
|
|
72
|
+
macOS/Windows installers never touch PATH, and the packaged app rebuilds
|
|
73
|
+
a minimal one; on Linux this covers an upstream tarball install in /opt.
|
|
74
|
+
"""
|
|
75
|
+
installed = tmp_path / "soffice"
|
|
76
|
+
installed.write_text("#!/bin/sh\n")
|
|
77
|
+
monkeypatch.delenv(SOFFICE_PATH_ENV, raising=False)
|
|
78
|
+
monkeypatch.setattr(legacy_office.shutil, "which", lambda _name: None)
|
|
79
|
+
monkeypatch.setattr(legacy_office.sys, "platform", platform)
|
|
80
|
+
monkeypatch.setitem(legacy_office._SOFFICE_FALLBACK_PATHS, platform, (str(installed),))
|
|
81
|
+
|
|
82
|
+
assert find_soffice() == str(installed)
|
|
83
|
+
|
|
84
|
+
@pytest.mark.parametrize("platform", ["darwin", "win32", "linux"])
|
|
85
|
+
def test_every_supported_platform_has_fallback_paths(self, platform: str) -> None:
|
|
86
|
+
"""A platform missing from the table silently loses discovery on hosts
|
|
87
|
+
whose installer does not touch PATH."""
|
|
88
|
+
assert legacy_office._SOFFICE_FALLBACK_PATHS.get(platform)
|
|
89
|
+
|
|
90
|
+
def test_fallback_paths_are_absolute(self) -> None:
|
|
91
|
+
for platform, candidates in legacy_office._SOFFICE_FALLBACK_PATHS.items():
|
|
92
|
+
for candidate in candidates:
|
|
93
|
+
assert candidate.startswith(("/", "C:\\")), f"{platform}: {candidate}"
|
|
94
|
+
|
|
95
|
+
def test_missing_everywhere_returns_none(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
96
|
+
monkeypatch.delenv(SOFFICE_PATH_ENV, raising=False)
|
|
97
|
+
monkeypatch.setattr(legacy_office.shutil, "which", lambda _name: None)
|
|
98
|
+
monkeypatch.setattr(legacy_office.sys, "platform", "darwin")
|
|
99
|
+
monkeypatch.setitem(legacy_office._SOFFICE_FALLBACK_PATHS, "darwin", ())
|
|
100
|
+
|
|
101
|
+
assert find_soffice() is None
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class TestConvertLegacyOffice:
|
|
105
|
+
def test_unsupported_mime_returns_error(self) -> None:
|
|
106
|
+
converted, target_mime, error = convert_legacy_office(b"data", "application/pdf")
|
|
107
|
+
assert converted is None
|
|
108
|
+
assert target_mime == ""
|
|
109
|
+
assert error is not None and "Unsupported" in error
|
|
110
|
+
|
|
111
|
+
def test_missing_soffice_returns_structured_error(
|
|
112
|
+
self, monkeypatch: pytest.MonkeyPatch
|
|
113
|
+
) -> None:
|
|
114
|
+
monkeypatch.setattr(legacy_office, "find_soffice", lambda: None)
|
|
115
|
+
converted, target_mime, error = convert_legacy_office(b"data", "application/msword")
|
|
116
|
+
assert converted is None
|
|
117
|
+
assert target_mime == _DOCX_MIME
|
|
118
|
+
assert error is not None and "LibreOffice" in error
|
|
119
|
+
|
|
120
|
+
def test_missing_soffice_is_reported_as_engine_unavailable(
|
|
121
|
+
self, monkeypatch: pytest.MonkeyPatch
|
|
122
|
+
) -> None:
|
|
123
|
+
"""A missing binary is a broken deployment, not a broken document.
|
|
124
|
+
|
|
125
|
+
iter_file_pages must raise — a silent empty result would let the asset be
|
|
126
|
+
recorded as scanned-and-empty, which is indistinguishable from a genuinely
|
|
127
|
+
empty file. The ENGINE_UNAVAILABLE code is what separates "this install
|
|
128
|
+
cannot read .doc at all" from "this one file failed".
|
|
129
|
+
"""
|
|
130
|
+
from src.utils.file_parser import (
|
|
131
|
+
TextExtractionCoverageCode,
|
|
132
|
+
TextExtractionCoverageError,
|
|
133
|
+
iter_file_pages,
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
monkeypatch.setattr(legacy_office, "find_soffice", lambda: None)
|
|
137
|
+
|
|
138
|
+
with pytest.raises(TextExtractionCoverageError) as excinfo:
|
|
139
|
+
list(iter_file_pages(b"\xd0\xcf\x11\xe0", "application/msword", file_name="x.doc"))
|
|
140
|
+
|
|
141
|
+
assert excinfo.value.code is TextExtractionCoverageCode.ENGINE_UNAVAILABLE
|
|
142
|
+
# The operator has to be able to act on it without reading our source.
|
|
143
|
+
assert "CLASSIFYRE_SOFFICE_PATH" in str(excinfo.value)
|
|
144
|
+
|
|
145
|
+
@pytest.mark.parametrize(
|
|
146
|
+
("platform", "expected_hint"),
|
|
147
|
+
[
|
|
148
|
+
("darwin", "brew install --cask libreoffice"),
|
|
149
|
+
("win32", "libreoffice.org/download"),
|
|
150
|
+
("linux", "apt install libreoffice-writer"),
|
|
151
|
+
],
|
|
152
|
+
)
|
|
153
|
+
def test_install_hint_matches_the_platform(
|
|
154
|
+
self, monkeypatch: pytest.MonkeyPatch, platform: str, expected_hint: str
|
|
155
|
+
) -> None:
|
|
156
|
+
"""The desktop app ships no LibreOffice, so this message is the user's
|
|
157
|
+
whole remediation path — a macOS user must not be told to run apt."""
|
|
158
|
+
monkeypatch.setattr(legacy_office.sys, "platform", platform)
|
|
159
|
+
|
|
160
|
+
message = legacy_office.soffice_missing_error()
|
|
161
|
+
|
|
162
|
+
assert expected_hint in message
|
|
163
|
+
# Keeps the ENGINE_UNAVAILABLE classification working (see iter_file_pages).
|
|
164
|
+
assert "unavailable" in message.casefold()
|
|
165
|
+
|
|
166
|
+
def test_install_hint_falls_back_on_unknown_platform(
|
|
167
|
+
self, monkeypatch: pytest.MonkeyPatch
|
|
168
|
+
) -> None:
|
|
169
|
+
monkeypatch.setattr(legacy_office.sys, "platform", "freebsd14")
|
|
170
|
+
|
|
171
|
+
message = legacy_office.soffice_missing_error()
|
|
172
|
+
|
|
173
|
+
assert "install LibreOffice" in message
|
|
174
|
+
assert "unavailable" in message.casefold()
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
# Fixture bodies chosen so LibreOffice can genuinely *import* them: Writer reads
|
|
178
|
+
# plain text, Calc reads CSV, Impress reads flat ODF. (Feeding .txt to Calc or
|
|
179
|
+
# Impress fails with "no export filter" and silently produces nothing.)
|
|
180
|
+
_SAMPLE_TEXT = "Legacy document body text."
|
|
181
|
+
_SAMPLE_CSV = "name,ssn\nAlice,123-45-6789\n"
|
|
182
|
+
_SAMPLE_FODP = """<?xml version="1.0" encoding="UTF-8"?>
|
|
183
|
+
<office:document xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0"
|
|
184
|
+
xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0"
|
|
185
|
+
xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0"
|
|
186
|
+
xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0"
|
|
187
|
+
office:version="1.2"
|
|
188
|
+
office:mimetype="application/vnd.oasis.opendocument.presentation">
|
|
189
|
+
<office:body><office:presentation>
|
|
190
|
+
<draw:page draw:name="page1">
|
|
191
|
+
<draw:frame svg:width="10cm" svg:height="2cm" svg:x="1cm" svg:y="1cm">
|
|
192
|
+
<draw:text-box><text:p>Legacy slide body text.</text:p></draw:text-box>
|
|
193
|
+
</draw:frame>
|
|
194
|
+
</draw:page>
|
|
195
|
+
</office:presentation></office:body>
|
|
196
|
+
</office:document>
|
|
197
|
+
"""
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _build_legacy_fixture(tmp_path: Path, file_name: str, body: str, legacy_ext: str) -> bytes:
|
|
201
|
+
"""Produce a genuine legacy binary (.doc/.xls/.ppt) with the same soffice."""
|
|
202
|
+
import subprocess
|
|
203
|
+
|
|
204
|
+
soffice = find_soffice()
|
|
205
|
+
assert soffice is not None
|
|
206
|
+
source = tmp_path / file_name
|
|
207
|
+
source.write_text(body)
|
|
208
|
+
subprocess.run(
|
|
209
|
+
[
|
|
210
|
+
soffice,
|
|
211
|
+
"--headless",
|
|
212
|
+
"--norestore",
|
|
213
|
+
f"-env:UserInstallation={(tmp_path / 'fixture-profile').as_uri()}",
|
|
214
|
+
"--convert-to",
|
|
215
|
+
legacy_ext,
|
|
216
|
+
"--outdir",
|
|
217
|
+
str(tmp_path),
|
|
218
|
+
str(source),
|
|
219
|
+
],
|
|
220
|
+
capture_output=True,
|
|
221
|
+
timeout=120,
|
|
222
|
+
check=True,
|
|
223
|
+
)
|
|
224
|
+
produced = tmp_path / f"{source.stem}.{legacy_ext}"
|
|
225
|
+
assert produced.is_file(), f"LibreOffice did not produce {produced.name}"
|
|
226
|
+
return produced.read_bytes()
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
@pytest.mark.skipif(find_soffice() is None, reason="LibreOffice not installed")
|
|
230
|
+
class TestLegacyOfficeRoundTrip:
|
|
231
|
+
"""Covers all three formats the CLI image installs an import filter for.
|
|
232
|
+
|
|
233
|
+
Each one needs its own libreoffice-*-nogui package, so a per-format test is
|
|
234
|
+
what catches a Dockerfile that dropped calc or impress.
|
|
235
|
+
"""
|
|
236
|
+
|
|
237
|
+
@pytest.mark.parametrize(
|
|
238
|
+
"case",
|
|
239
|
+
[
|
|
240
|
+
("sample.txt", _SAMPLE_TEXT, "doc", "application/msword", _DOCX_MIME),
|
|
241
|
+
("sample.csv", _SAMPLE_CSV, "xls", "application/vnd.ms-excel", _XLSX_MIME),
|
|
242
|
+
("sample.fodp", _SAMPLE_FODP, "ppt", "application/vnd.ms-powerpoint", _PPTX_MIME),
|
|
243
|
+
],
|
|
244
|
+
ids=["doc", "xls", "ppt"],
|
|
245
|
+
)
|
|
246
|
+
def test_legacy_binary_converts_to_ooxml(
|
|
247
|
+
self,
|
|
248
|
+
tmp_path: Path,
|
|
249
|
+
case: tuple[str, str, str, str, str],
|
|
250
|
+
) -> None:
|
|
251
|
+
file_name, body, legacy_ext, legacy_mime, target_mime = case
|
|
252
|
+
legacy_bytes = _build_legacy_fixture(tmp_path, file_name, body, legacy_ext)
|
|
253
|
+
|
|
254
|
+
converted, resolved_mime, error = convert_legacy_office(legacy_bytes, legacy_mime)
|
|
255
|
+
|
|
256
|
+
assert error is None
|
|
257
|
+
assert resolved_mime == target_mime
|
|
258
|
+
assert converted is not None and converted.startswith(b"PK\x03\x04")
|
|
259
|
+
|
|
260
|
+
def test_doc_text_reaches_detectors(self, tmp_path: Path) -> None:
|
|
261
|
+
from src.utils.file_parser import extract_text
|
|
262
|
+
|
|
263
|
+
doc_bytes = _build_legacy_fixture(tmp_path, "sample.txt", _SAMPLE_TEXT, "doc")
|
|
264
|
+
|
|
265
|
+
text, extract_error = extract_text(doc_bytes, "application/msword", file_name="sample.doc")
|
|
266
|
+
|
|
267
|
+
assert extract_error is None
|
|
268
|
+
assert _SAMPLE_TEXT in text
|
|
269
|
+
|
|
270
|
+
def test_xls_cell_values_reach_detectors(self, tmp_path: Path) -> None:
|
|
271
|
+
"""The Enron corpus failure case: a .xls scanned as "no content"."""
|
|
272
|
+
from src.utils.file_parser import extract_text
|
|
273
|
+
|
|
274
|
+
xls_bytes = _build_legacy_fixture(tmp_path, "sample.csv", _SAMPLE_CSV, "xls")
|
|
275
|
+
|
|
276
|
+
text, extract_error = extract_text(
|
|
277
|
+
xls_bytes, "application/vnd.ms-excel", file_name="sample.xls"
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
assert extract_error is None
|
|
281
|
+
assert "123-45-6789" in text
|
|
@@ -176,11 +176,11 @@ wheels = [
|
|
|
176
176
|
|
|
177
177
|
[[package]]
|
|
178
178
|
name = "annotated-doc"
|
|
179
|
-
version = "0.0.
|
|
179
|
+
version = "0.0.5"
|
|
180
180
|
source = { registry = "https://pypi.org/simple" }
|
|
181
|
-
sdist = { url = "https://files.pythonhosted.org/packages/
|
|
181
|
+
sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" }
|
|
182
182
|
wheels = [
|
|
183
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
183
|
+
{ url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" },
|
|
184
184
|
]
|
|
185
185
|
|
|
186
186
|
[[package]]
|
|
@@ -830,7 +830,7 @@ wheels = [
|
|
|
830
830
|
|
|
831
831
|
[[package]]
|
|
832
832
|
name = "classifyre-cli"
|
|
833
|
-
version = "0.4.
|
|
833
|
+
version = "0.4.74"
|
|
834
834
|
source = { editable = "." }
|
|
835
835
|
dependencies = [
|
|
836
836
|
{ name = "beautifulsoup4" },
|
|
@@ -1355,7 +1355,7 @@ youtube = [
|
|
|
1355
1355
|
|
|
1356
1356
|
[[package]]
|
|
1357
1357
|
name = "classifyre-schemas"
|
|
1358
|
-
version = "0.4.
|
|
1358
|
+
version = "0.4.74"
|
|
1359
1359
|
source = { editable = "../../packages/schemas" }
|
|
1360
1360
|
dependencies = [
|
|
1361
1361
|
{ name = "fastjsonschema" },
|
|
@@ -4040,7 +4040,7 @@ wheels = [
|
|
|
4040
4040
|
|
|
4041
4041
|
[[package]]
|
|
4042
4042
|
name = "peft"
|
|
4043
|
-
version = "0.
|
|
4043
|
+
version = "0.20.0"
|
|
4044
4044
|
source = { registry = "https://pypi.org/simple" }
|
|
4045
4045
|
dependencies = [
|
|
4046
4046
|
{ name = "accelerate" },
|
|
@@ -4056,9 +4056,9 @@ dependencies = [
|
|
|
4056
4056
|
{ name = "transformers", version = "5.8.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" },
|
|
4057
4057
|
{ name = "transformers", version = "5.14.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" },
|
|
4058
4058
|
]
|
|
4059
|
-
sdist = { url = "https://files.pythonhosted.org/packages/
|
|
4059
|
+
sdist = { url = "https://files.pythonhosted.org/packages/b4/08/02541a2c29be7c78698f73d438bc0e733b214f3f35ec5db79fca8da8fc61/peft-0.20.0.tar.gz", hash = "sha256:4769c8093a4ca145fd6fb3fd4dd50449675f5fe46434ad1e98b285a132d4b1d0", size = 880503, upload-time = "2026-07-28T13:46:01.85Z" }
|
|
4060
4060
|
wheels = [
|
|
4061
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
4061
|
+
{ url = "https://files.pythonhosted.org/packages/28/79/13bcabb8048126422d5c4b880575d40886c726f354db88cfeed4325525bb/peft-0.20.0-py3-none-any.whl", hash = "sha256:0fbba16ffebfad3de96e06f2da6860fd860292324b85b6141909fa1e26ea9233", size = 775777, upload-time = "2026-07-28T13:45:59.809Z" },
|
|
4062
4062
|
]
|
|
4063
4063
|
|
|
4064
4064
|
[[package]]
|