classifyre-cli 0.5.8__tar.gz → 0.5.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/.turbo/turbo-build.log +1 -1
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/PKG-INFO +1 -1
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/package.json +1 -1
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/pyproject.toml +1 -1
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/scripts/generate_sdk_completions.py +12 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/custom/runners/_tag.py +62 -7
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/graph/edges.py +28 -5
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/notebook/sdk.py +109 -7
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/pipeline/content_provider.py +4 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/pipeline/detector_pipeline.py +55 -5
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/pipeline/parsed_content_provider.py +6 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/base.py +19 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/custom/runner.py +12 -1
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/custom/source.py +68 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/test_tag_runner.py +84 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/pipeline/test_detector_pipeline.py +61 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_custom_source.py +46 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_custom_source_tags_end_to_end.py +65 -0
- classifyre_cli-0.5.10/tests/test_notebook_asset_tags.py +102 -0
- classifyre_cli-0.5.10/tests/test_notebook_log_levels.py +29 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_notebook_query_assets.py +10 -0
- classifyre_cli-0.5.10/tests/test_notebook_sdk_exports.py +43 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/uv.lock +218 -219
- classifyre_cli-0.5.8/tests/test_notebook_asset_tags.py +0 -50
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/.gitignore +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/.python-version +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/README.md +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/main.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/scripts/generate_models.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/scripts/generate_runtime_packages.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/augmentation/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/augmentation/contract.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/augmentation/preview.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/augmentation/runner.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/augmentation/sdk.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/augmentation/session.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/config.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/base.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/broken_links/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/broken_links/detector.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/config.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/content/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/custom/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/custom/detector.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/custom/extractor.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/custom/runners/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/custom/runners/_base.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/custom/runners/_factory.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/custom/runners/_gliner2.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/custom/runners/_image_classification.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/custom/runners/_llm.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/custom/runners/_object_detection.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/custom/runners/_regex.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/custom/runners/_text_classification.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/custom/trainer.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/dependencies.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/engine_version.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/errors.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/pii/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/pii/detector.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/secrets/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/secrets/detector.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/threat/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/threat/code_security_detector.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/detectors/threat/yara_detector.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/file_evaluation/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/file_evaluation/runner.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/graph/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/main.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/models/generated_detectors.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/models/generated_input.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/models/generated_single_asset_scan_results.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/notebook/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/notebook/child.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/notebook/cli.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/notebook/cohort.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/notebook/contract.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/notebook/execute.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/notebook/files.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/notebook/groups.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/notebook/outputs.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/notebook/packages.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/notebook/protocol.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/notebook/redact.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/notebook/scaffold.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/notebook/serialize.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/outputs/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/outputs/base.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/outputs/console.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/outputs/factory.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/outputs/file.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/outputs/rest.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/pipeline/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/pipeline/detector_breaker.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/pipeline/payload_window.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/pipeline/scan_cache.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/pipeline/text_artifact.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/pipeline/worker_pool.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/asset_metadata.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/atlassian_common.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/azure_blob_storage/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/azure_blob_storage/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/confluence/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/confluence/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/custom/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/custom/env.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/databricks/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/databricks/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/delta_lake/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/delta_lake/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/dependencies.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/dropbox/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/dropbox/auth.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/dropbox/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/elasticsearch/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/elasticsearch/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/email/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/email/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/git/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/git/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/google_cloud_storage/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/google_cloud_storage/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/google_workspace/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/google_workspace/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/hive/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/hive/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/hugging_face/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/hugging_face/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/iceberg/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/iceberg/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/jira/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/jira/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/kafka/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/kafka/rest.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/kafka/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/lakehouse_base.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/local_folder/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/local_folder/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/meilisearch/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/meilisearch/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/microsoft_365/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/microsoft_365/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/mongodb/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/mongodb/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/mssql/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/mssql/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/mysql/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/mysql/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/neo4j/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/neo4j/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/notion/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/notion/client.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/notion/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/object_storage/base.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/opensearch/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/opensearch/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/oracle/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/oracle/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/postgresql/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/postgresql/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/powerbi/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/powerbi/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/recipe_normalizer.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/reddit/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/reddit/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/s3_client.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/s3_compatible_storage/README.md +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/s3_compatible_storage/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/s3_compatible_storage/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/sandbox/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/sandbox/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/search_engine_base.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/servicedesk/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/servicedesk/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/slack/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/slack/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/snowflake/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/snowflake/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/sqlite/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/sqlite/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/tableau/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/tableau/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/tabular_base.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/tabular_utils.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/wordpress/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/wordpress/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/youtube/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/sources/youtube/source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/telemetry.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/archive_extraction.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/content_extraction.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/dependency_groups.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/embedded_files.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/external_video.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/file_metadata.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/file_parser.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/file_to_images.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/hashing.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/legacy_office.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/payload.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/range_reader.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/redaction.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/resources.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/source_files.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/sql_lineage.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/torch_runtime.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/transcription.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/urn.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/uv_sync.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/validation.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/src/utils/video_processing.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/_lakehouse_fakes.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/conftest.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/broken_links/test_broken_links_detector.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/conftest.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/content/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/custom/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/custom/conftest.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/custom/test_gliner2_classification.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/custom/test_invoice_extraction.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/custom/test_llm_runner.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/custom/test_pipeline_integration.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/custom/test_regex_runner.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/custom/test_runner_hardening.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/custom/test_transformer_runners.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/pii/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/pii/conftest.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/pii/sample_invoice.pdf +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/pii/test_pii_chunking.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/pii/test_pii_detector.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/pii/test_pii_detector_extended.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/pii/test_pii_regional_entities.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/pii/test_pii_severity.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/secrets/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/secrets/test_secrets_detector.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/secrets/test_secrets_detector_extended.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/test_base_detector.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/test_custom_detector_examples_runtime.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/test_detector_catalog_commercial.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/test_detector_pipeline_types.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/test_detector_schema_examples.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/test_detector_types.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/test_phase2_detectors.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/test_registry.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/threat/__init__.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/threat/test_code_security_detector.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/detectors/threat/test_yara_detector.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/integration/test_wordpress_broken_links_detector.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/integration/test_wordpress_links_assets.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/pipeline/test_detector_breaker.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/pipeline/test_detector_outcomes.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/pipeline/test_worker_pool.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_arrow_ipc_payloads.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_assets_metadata_catalog.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_augmentation_contract.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_augmentation_edges.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_augmentation_patch.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_augmentation_scan_cache.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_augmentation_sdk.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_augmentation_session.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_augmentation_tags_end_to_end.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_azure_blob_storage_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_base_source_attachment.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_base_source_sampling.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_byte_valued_fields.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_config.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_confluence_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_custom_extractor.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_custom_source_relationships.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_databricks_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_delta_lake_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_dependency_groups.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_detector_scope.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_dropbox_auth_command.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_dropbox_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_elasticsearch_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_email_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_file_evaluation_runner.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_git_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_google_cloud_storage_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_google_workspace_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_graph_edges.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_hashing.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_hive_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_hugging_face_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_iceberg_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_jira_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_kafka_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_lineage_loss_reporting.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_local_folder_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_main.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_meilisearch_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_microsoft_365_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_mongodb_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_mssql_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_mysql_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_neo4j_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_notebook_asset_extract.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_notebook_cohort.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_notebook_execute.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_notebook_files.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_notebook_groups.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_notebook_packages.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_notebook_serialize.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_notebook_templates.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_notion_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_object_storage_download_efficiency.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_opensearch_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_oracle_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_outputs.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_payload_window.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_payload_window_parquet.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_postgresql_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_powerbi_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_range_reading.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_recipe_normalizer.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_reddit_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_rest_backpressure.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_runtime_packages.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_s3_compatible_storage_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_sampling_automatic.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_sandbox_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_scan_cache.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_sdk_completions.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_secret_redaction.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_servicedesk_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_slack_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_snowflake_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_source_dependency_groups.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_source_strategy_conformance.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_sqlite_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_tableau_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_tabular_automatic_sampling.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_tabular_lineage.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_tabular_utils.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_torch_runtime.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_urn.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_uv_sync.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_wordpress_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_youtube_source.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/test_youtube_source_integration.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/utils/test_archive_extraction.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/utils/test_content_extraction.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/utils/test_embedded_files.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/utils/test_file_metadata.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/utils/test_file_parser.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/utils/test_file_to_images.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/utils/test_legacy_office.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/utils/test_payload.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/utils/test_transcription.py +0 -0
- {classifyre_cli-0.5.8 → classifyre_cli-0.5.10}/tests/utils/test_video_processing.py +0 -0
|
@@ -40,6 +40,7 @@ from src.notebook.sdk import ( # noqa: E402
|
|
|
40
40
|
NotebookFile,
|
|
41
41
|
ParsedContent,
|
|
42
42
|
Ref,
|
|
43
|
+
Tag,
|
|
43
44
|
contains,
|
|
44
45
|
flow,
|
|
45
46
|
pages,
|
|
@@ -251,6 +252,13 @@ def build() -> dict[str, Any]:
|
|
|
251
252
|
"documentation": _summary(Asset),
|
|
252
253
|
"insertText": "Asset",
|
|
253
254
|
},
|
|
255
|
+
{
|
|
256
|
+
"label": "Tag",
|
|
257
|
+
"kind": "class",
|
|
258
|
+
"detail": 'Tag(value, severity="HIGH")',
|
|
259
|
+
"documentation": _summary(Tag),
|
|
260
|
+
"insertText": "Tag",
|
|
261
|
+
},
|
|
254
262
|
{
|
|
255
263
|
"label": "Ref",
|
|
256
264
|
"kind": "class",
|
|
@@ -292,6 +300,10 @@ def build() -> dict[str, Any]:
|
|
|
292
300
|
"documentation": _summary(Asset),
|
|
293
301
|
"fields": _fields(Asset),
|
|
294
302
|
},
|
|
303
|
+
"Tag": {
|
|
304
|
+
"documentation": _summary(Tag),
|
|
305
|
+
"fields": _fields(Tag),
|
|
306
|
+
},
|
|
295
307
|
"AugmentedAsset": {
|
|
296
308
|
"documentation": _summary(augmentation_sdk.AugmentedAsset),
|
|
297
309
|
"members": _members(augmentation_sdk.AugmentedAsset),
|
|
@@ -28,6 +28,15 @@ from ._base import BaseRunner
|
|
|
28
28
|
#: from, so an unreadable one falls back here rather than being guessed at.
|
|
29
29
|
_DEFAULT_SEVERITY = Severity.medium
|
|
30
30
|
|
|
31
|
+
#: Most severe first, so a lower index is a higher severity.
|
|
32
|
+
_SEVERITY_ORDER: tuple[Severity, ...] = (
|
|
33
|
+
Severity.critical,
|
|
34
|
+
Severity.high,
|
|
35
|
+
Severity.medium,
|
|
36
|
+
Severity.low,
|
|
37
|
+
Severity.info,
|
|
38
|
+
)
|
|
39
|
+
|
|
31
40
|
|
|
32
41
|
class TagRunner(BaseRunner):
|
|
33
42
|
"""Placeholder pipeline: produces findings only from notebook-supplied tags."""
|
|
@@ -56,13 +65,39 @@ class TagRunner(BaseRunner):
|
|
|
56
65
|
"""No content type is supported, so the pipeline never schedules this."""
|
|
57
66
|
return []
|
|
58
67
|
|
|
59
|
-
def
|
|
60
|
-
"""
|
|
68
|
+
def bound_severity(self, requested: str | Severity | None) -> tuple[Severity, bool]:
|
|
69
|
+
"""The severity to use for one tag, and whether it had to be lowered.
|
|
70
|
+
|
|
71
|
+
The detector's own severity is a ceiling, not a default. A connector
|
|
72
|
+
that knows this instance is milder than usual may say so; one that
|
|
73
|
+
wants its findings to outrank what the operator configured may not,
|
|
74
|
+
because the detector is where that decision belongs. An unreadable or
|
|
75
|
+
over-severe request lands on the ceiling and the caller reports it,
|
|
76
|
+
rather than being applied or dropped in silence.
|
|
77
|
+
"""
|
|
78
|
+
if requested is None:
|
|
79
|
+
return self.severity, False
|
|
80
|
+
asked = _parse_severity(requested)
|
|
81
|
+
if asked is None:
|
|
82
|
+
return self.severity, True
|
|
83
|
+
if _SEVERITY_ORDER.index(asked) < _SEVERITY_ORDER.index(self.severity):
|
|
84
|
+
return self.severity, True
|
|
85
|
+
return asked, False
|
|
86
|
+
|
|
87
|
+
def tag_finding(self, value: str, severity: str | Severity | None = None) -> DetectionResult:
|
|
88
|
+
"""One finding for one value the notebook asserted under this key.
|
|
89
|
+
|
|
90
|
+
``severity`` is what the connector *asked for*, not an already-bounded
|
|
91
|
+
value: the finding records both what was applied and what was refused,
|
|
92
|
+
so a capped tag can be explained from the finding itself rather than
|
|
93
|
+
from a log line nobody kept.
|
|
94
|
+
"""
|
|
61
95
|
text = str(value).strip()
|
|
96
|
+
applied, lowered = self.bound_severity(severity)
|
|
62
97
|
return self._make_result(
|
|
63
98
|
finding_type=f"tag:{self.label}",
|
|
64
99
|
category="CLASSIFICATION",
|
|
65
|
-
severity=
|
|
100
|
+
severity=applied,
|
|
66
101
|
# Asserted by the connector, not inferred from content: there is
|
|
67
102
|
# nothing here for a confidence score to express.
|
|
68
103
|
confidence=1.0,
|
|
@@ -73,13 +108,33 @@ class TagRunner(BaseRunner):
|
|
|
73
108
|
"label": self.label,
|
|
74
109
|
"tag_key": self._detector_key,
|
|
75
110
|
"tag_value": text,
|
|
111
|
+
# Recorded so a finding whose severity differs from its
|
|
112
|
+
# detector's can be explained without re-reading the notebook.
|
|
113
|
+
**({"tag_severity": str(applied)} if applied != self.severity else {}),
|
|
114
|
+
# And when the connector asked for more than the detector
|
|
115
|
+
# allows, what it asked for. Without this the most interesting
|
|
116
|
+
# case -- a refused promotion -- left no trace on the finding
|
|
117
|
+
# at all, only a warning in a log.
|
|
118
|
+
**(
|
|
119
|
+
{"tag_severity_requested": str(severity).strip().upper()}
|
|
120
|
+
if lowered and severity is not None
|
|
121
|
+
else {}
|
|
122
|
+
),
|
|
76
123
|
},
|
|
77
124
|
)
|
|
78
125
|
|
|
79
126
|
|
|
80
|
-
def
|
|
127
|
+
def _parse_severity(value: object) -> Severity | None:
|
|
128
|
+
"""The severity this value names, or None when it names none."""
|
|
81
129
|
if isinstance(value, Severity):
|
|
82
130
|
return value
|
|
83
|
-
if isinstance(value, str)
|
|
84
|
-
|
|
85
|
-
|
|
131
|
+
if isinstance(value, str):
|
|
132
|
+
# The schema spells them lowercase; a notebook writes "HIGH".
|
|
133
|
+
name = value.strip().lower()
|
|
134
|
+
if name in Severity.__members__:
|
|
135
|
+
return Severity(name)
|
|
136
|
+
return None
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _coerce_severity(value: object) -> Severity:
|
|
140
|
+
return _parse_severity(value) or _DEFAULT_SEVERITY
|
|
@@ -385,7 +385,13 @@ def flow(
|
|
|
385
385
|
)
|
|
386
386
|
|
|
387
387
|
|
|
388
|
-
def contains(
|
|
388
|
+
def contains(
|
|
389
|
+
parent: Ref,
|
|
390
|
+
child: Ref,
|
|
391
|
+
*,
|
|
392
|
+
type: ContainmentType = ContainmentType.CONTAINS,
|
|
393
|
+
evidence: dict[str, Any] | None = None,
|
|
394
|
+
) -> Edge:
|
|
389
395
|
"""``child`` is a part of ``parent`` -- structural, not temporal."""
|
|
390
396
|
return Edge(
|
|
391
397
|
frm=parent,
|
|
@@ -394,18 +400,32 @@ def contains(parent: Ref, child: Ref, *, type: ContainmentType = ContainmentType
|
|
|
394
400
|
relation_type=str(type),
|
|
395
401
|
method=Method.SYSTEM_CATALOG,
|
|
396
402
|
confidence=1.0,
|
|
403
|
+
evidence=dict(evidence or {}),
|
|
397
404
|
)
|
|
398
405
|
|
|
399
406
|
|
|
400
|
-
def same_as(
|
|
401
|
-
|
|
407
|
+
def same_as(
|
|
408
|
+
a: Ref,
|
|
409
|
+
b: Ref,
|
|
410
|
+
*,
|
|
411
|
+
method: Method = Method.SYSTEM_CATALOG,
|
|
412
|
+
confidence: float | None = None,
|
|
413
|
+
evidence: dict[str, Any] | None = None,
|
|
414
|
+
) -> Edge:
|
|
415
|
+
"""The same real-world object, seen from two systems.
|
|
416
|
+
|
|
417
|
+
``evidence`` says why the match holds -- e.g. ``{"match": "alias", "checked_by": "hand"}``
|
|
418
|
+
for a crosswalk entry a person verified -- so a reader can weigh a MANUAL match
|
|
419
|
+
differently from a name heuristic.
|
|
420
|
+
"""
|
|
402
421
|
return Edge(
|
|
403
422
|
frm=a,
|
|
404
423
|
to=b,
|
|
405
424
|
edge_class=EdgeClass.IDENTITY,
|
|
406
425
|
relation_type="SAME_AS",
|
|
407
426
|
method=method,
|
|
408
|
-
confidence=_confidence(method,
|
|
427
|
+
confidence=_confidence(method, confidence),
|
|
428
|
+
evidence=dict(evidence or {}),
|
|
409
429
|
)
|
|
410
430
|
|
|
411
431
|
|
|
@@ -415,6 +435,8 @@ def references(
|
|
|
415
435
|
*,
|
|
416
436
|
type: ReferenceType = ReferenceType.REFERENCES,
|
|
417
437
|
method: Method = Method.SYSTEM_CATALOG,
|
|
438
|
+
confidence: float | None = None,
|
|
439
|
+
evidence: dict[str, Any] | None = None,
|
|
418
440
|
) -> Edge:
|
|
419
441
|
"""``frm`` points at ``to``. No data moves -- a foreign key is not lineage."""
|
|
420
442
|
return Edge(
|
|
@@ -423,7 +445,8 @@ def references(
|
|
|
423
445
|
edge_class=EdgeClass.REFERENCE,
|
|
424
446
|
relation_type=str(type),
|
|
425
447
|
method=method,
|
|
426
|
-
confidence=_confidence(method,
|
|
448
|
+
confidence=_confidence(method, confidence),
|
|
449
|
+
evidence=dict(evidence or {}),
|
|
427
450
|
)
|
|
428
451
|
|
|
429
452
|
|
|
@@ -31,10 +31,15 @@ from pathlib import Path
|
|
|
31
31
|
from typing import IO, Any
|
|
32
32
|
|
|
33
33
|
from ..graph.edges import (
|
|
34
|
+
ContainmentType,
|
|
34
35
|
Edge,
|
|
35
36
|
FieldMapping,
|
|
37
|
+
FieldTransform,
|
|
36
38
|
FlowType,
|
|
39
|
+
Method,
|
|
37
40
|
Ref,
|
|
41
|
+
ReferenceType,
|
|
42
|
+
UsageType,
|
|
38
43
|
contains,
|
|
39
44
|
flow,
|
|
40
45
|
references,
|
|
@@ -53,6 +58,48 @@ CONTENT_TYPES = ("TXT", "TABLE", "IMAGE", "VIDEO", "AUDIO", "URL", "BINARY", "OT
|
|
|
53
58
|
|
|
54
59
|
DEFAULT_KIND = "record"
|
|
55
60
|
|
|
61
|
+
#: Most severe first. A tag's severity is bounded by its detector's, so this is
|
|
62
|
+
#: also the order the bound is applied in.
|
|
63
|
+
TAG_SEVERITIES = ("CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class Tag:
|
|
68
|
+
"""A tag value that carries its own severity.
|
|
69
|
+
|
|
70
|
+
``tags={"insolvencies_yoy": "+34%"}`` gives every asset the Tag detector's
|
|
71
|
+
own severity. That is right when the fact itself is the finding, and wrong
|
|
72
|
+
when the *degree* is: an early-warning rule reading "HIGH at +30%, MEDIUM at
|
|
73
|
+
+20%" needed two detectors with duplicated descriptions, and every inquiry
|
|
74
|
+
and case had to name both keys (GENESIS field report P9). Written instead
|
|
75
|
+
as::
|
|
76
|
+
|
|
77
|
+
Asset(tags={"insolvencies_yoy": Tag("+34%", severity="HIGH")})
|
|
78
|
+
|
|
79
|
+
one detector covers the rule and the finding carries the band.
|
|
80
|
+
|
|
81
|
+
The detector's configured severity is the ceiling, never the default: a
|
|
82
|
+
connector can say "this instance matters less than usual" but cannot
|
|
83
|
+
promote its own findings past what the operator who created the detector
|
|
84
|
+
allowed. A severity above the ceiling is lowered to it and reported as a
|
|
85
|
+
scan warning rather than silently applied or dropped.
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
value: str
|
|
89
|
+
severity: str | None = None
|
|
90
|
+
|
|
91
|
+
def __post_init__(self) -> None:
|
|
92
|
+
object.__setattr__(self, "value", str(self.value).strip())
|
|
93
|
+
raw = self.severity
|
|
94
|
+
if raw is None:
|
|
95
|
+
return
|
|
96
|
+
normalized = str(raw).strip().upper()
|
|
97
|
+
if normalized not in TAG_SEVERITIES:
|
|
98
|
+
raise ValueError(
|
|
99
|
+
f"Tag severity must be one of {', '.join(TAG_SEVERITIES)}, got {raw!r}"
|
|
100
|
+
)
|
|
101
|
+
object.__setattr__(self, "severity", normalized)
|
|
102
|
+
|
|
56
103
|
|
|
57
104
|
@dataclass
|
|
58
105
|
class Asset:
|
|
@@ -95,7 +142,15 @@ class Asset:
|
|
|
95
142
|
#: when the source system already has the answer and running a classifier
|
|
96
143
|
#: over the content could only re-derive it less reliably. A key with no
|
|
97
144
|
#: matching Tag detector is reported as a scan warning and skipped.
|
|
145
|
+
#:
|
|
146
|
+
#: A value may also be a :class:`Tag`, which carries a severity for that one
|
|
147
|
+
#: assertion: ``tags={"insolvencies_yoy": Tag("+34%", severity="HIGH")}``.
|
|
98
148
|
tags: dict[str, str] = field(default_factory=dict)
|
|
149
|
+
#: Per-tag severity overrides, keyed exactly like ``tags``. Filled in from
|
|
150
|
+
#: any :class:`Tag` values above rather than written directly, so ``tags``
|
|
151
|
+
#: keeps its plain ``{key: value}`` shape for everything that already reads
|
|
152
|
+
#: it. Bounded by each detector's own severity when the finding is built.
|
|
153
|
+
tag_severities: dict[str, str] = field(default_factory=dict)
|
|
99
154
|
#: ``False`` records the asset without extracting or scanning its content:
|
|
100
155
|
#: ``Asset(id=..., metadata={...}, extract=False, reference_reason="...")``.
|
|
101
156
|
#: Use it for an artefact whose existence and metadata matter but whose
|
|
@@ -139,7 +194,7 @@ class Asset:
|
|
|
139
194
|
f"{type(self.content_bytes).__name__}); encode text first"
|
|
140
195
|
)
|
|
141
196
|
self.links = [str(link) for link in self.links if str(link).strip()]
|
|
142
|
-
self.tags = _normalize_tags(self.tags)
|
|
197
|
+
self.tags, self.tag_severities = _normalize_tags(self.tags)
|
|
143
198
|
if self.cohort is not None:
|
|
144
199
|
if not isinstance(self.cohort, CohortItem):
|
|
145
200
|
raise TypeError("Asset.cohort must be an item from ctx.cohort()")
|
|
@@ -166,15 +221,19 @@ class Asset:
|
|
|
166
221
|
)
|
|
167
222
|
|
|
168
223
|
|
|
169
|
-
def _normalize_tags(value: Any) -> dict[str, str]:
|
|
224
|
+
def _normalize_tags(value: Any) -> tuple[dict[str, str], dict[str, str]]:
|
|
170
225
|
"""Accept the two shapes an author reasonably writes, reject the rest.
|
|
171
226
|
|
|
172
227
|
A dict is the ordinary case. A sequence of pairs is accepted because it is
|
|
173
228
|
what someone reaches for to assert two values under one key, and silently
|
|
174
229
|
keeping only the last one would be worse than supporting it.
|
|
230
|
+
|
|
231
|
+
Returns the values and, separately, the severities any :class:`Tag` entries
|
|
232
|
+
asked for. Splitting them keeps ``Asset.tags`` a plain ``{key: value}`` map
|
|
233
|
+
for the adapter, the checksum and every reader that predates severities.
|
|
175
234
|
"""
|
|
176
235
|
if not value:
|
|
177
|
-
return {}
|
|
236
|
+
return {}, {}
|
|
178
237
|
if isinstance(value, Mapping):
|
|
179
238
|
items: Iterable[Any] = value.items()
|
|
180
239
|
elif isinstance(value, str | bytes):
|
|
@@ -191,6 +250,7 @@ def _normalize_tags(value: Any) -> dict[str, str]:
|
|
|
191
250
|
) from None
|
|
192
251
|
|
|
193
252
|
tags: dict[str, str] = {}
|
|
253
|
+
severities: dict[str, str] = {}
|
|
194
254
|
for item in items:
|
|
195
255
|
if isinstance(item, Mapping):
|
|
196
256
|
pair: Any = (item.get("key"), item.get("value"))
|
|
@@ -203,6 +263,10 @@ def _normalize_tags(value: Any) -> dict[str, str]:
|
|
|
203
263
|
f"Asset.tags entries must be (detector key, value) pairs; got {item!r}"
|
|
204
264
|
) from None
|
|
205
265
|
key = str(key or "").strip()
|
|
266
|
+
severity: str | None = None
|
|
267
|
+
if isinstance(tag_value, Tag):
|
|
268
|
+
severity = tag_value.severity
|
|
269
|
+
tag_value = tag_value.value
|
|
206
270
|
tag_value = str("" if tag_value is None else tag_value).strip()
|
|
207
271
|
if not key or not tag_value:
|
|
208
272
|
continue
|
|
@@ -211,7 +275,15 @@ def _normalize_tags(value: Any) -> dict[str, str]:
|
|
|
211
275
|
# one silently would lose the other.
|
|
212
276
|
existing = tags.get(key)
|
|
213
277
|
tags[key] = f"{existing}, {tag_value}" if existing else tag_value
|
|
214
|
-
|
|
278
|
+
if severity is None:
|
|
279
|
+
continue
|
|
280
|
+
# Joined values share one finding, so they share one severity: the most
|
|
281
|
+
# severe of the two, because lowering it would understate a fact the
|
|
282
|
+
# connector asserted.
|
|
283
|
+
current = severities.get(key)
|
|
284
|
+
if current is None or TAG_SEVERITIES.index(severity) < TAG_SEVERITIES.index(current):
|
|
285
|
+
severities[key] = severity
|
|
286
|
+
return tags, severities
|
|
215
287
|
|
|
216
288
|
|
|
217
289
|
def urn_for(platform: str, authority: str, *path: str) -> str:
|
|
@@ -629,6 +701,7 @@ class Context:
|
|
|
629
701
|
select: Iterable[str] = (),
|
|
630
702
|
limit: int = 1000,
|
|
631
703
|
cursor: str | None = None,
|
|
704
|
+
cohort: bool = True,
|
|
632
705
|
) -> AssetQueryPage:
|
|
633
706
|
"""Read assets another source in this namespace already wrote.
|
|
634
707
|
|
|
@@ -658,6 +731,13 @@ class Context:
|
|
|
658
731
|
Calling it declares this run's coverage partial, since a connector that
|
|
659
732
|
picks its cohort from a query never sees the rest of its source.
|
|
660
733
|
Available during scans only, not when running a cell in the editor.
|
|
734
|
+
|
|
735
|
+
Pass ``cohort=False`` when the query only supplies *inputs* and this run
|
|
736
|
+
still yields its whole universe -- e.g. Land aggregates computed from
|
|
737
|
+
every district profile. Coverage then stays complete, so an asset the
|
|
738
|
+
notebook stops yielding is retired. Without it such a source could never
|
|
739
|
+
retire anything (GENESIS field report P5). Fail the run rather than
|
|
740
|
+
yield a partial universe if the input is incomplete.
|
|
661
741
|
"""
|
|
662
742
|
if self._query_assets is None:
|
|
663
743
|
raise AssetQueryError(
|
|
@@ -701,7 +781,7 @@ class Context:
|
|
|
701
781
|
payload["cursor"] = str(cursor)
|
|
702
782
|
|
|
703
783
|
response = self._query_assets(payload) or {}
|
|
704
|
-
if not self._partial_coverage:
|
|
784
|
+
if cohort and not self._partial_coverage:
|
|
705
785
|
# Only on success: a refused query read nothing, so the run still
|
|
706
786
|
# covers what it covers. Never over the notebook's own reason.
|
|
707
787
|
self.set_partial_coverage("the notebook chose its cohort with ctx.query_assets()")
|
|
@@ -730,8 +810,12 @@ class Context:
|
|
|
730
810
|
"""
|
|
731
811
|
return bool(self._should_abort())
|
|
732
812
|
|
|
733
|
-
def log(self, *parts: Any) -> None:
|
|
734
|
-
|
|
813
|
+
def log(self, *parts: Any, level: str | None = None) -> None:
|
|
814
|
+
"""Write a line to the run log. ``level`` ("warning", "error", …) sets its level there."""
|
|
815
|
+
message = " ".join(str(part) for part in parts)
|
|
816
|
+
if level:
|
|
817
|
+
message = f"{level.strip().upper()}: {message}"
|
|
818
|
+
self._logger(message)
|
|
735
819
|
|
|
736
820
|
@staticmethod
|
|
737
821
|
def now() -> datetime:
|
|
@@ -743,9 +827,19 @@ def build_module(context: Context) -> types.ModuleType:
|
|
|
743
827
|
module = types.ModuleType(MODULE_NAME)
|
|
744
828
|
module.__doc__ = "Runtime helpers available to a Classifyre custom connector."
|
|
745
829
|
module.Asset = Asset # type: ignore[attr-defined]
|
|
830
|
+
# A tag value that carries its own severity, so one detector can cover a
|
|
831
|
+
# banded rule instead of one detector per band (field report P9).
|
|
832
|
+
module.Tag = Tag # type: ignore[attr-defined]
|
|
746
833
|
module.Ref = Ref # type: ignore[attr-defined]
|
|
747
834
|
module.FieldMapping = FieldMapping # type: ignore[attr-defined]
|
|
748
835
|
module.FlowType = FlowType # type: ignore[attr-defined]
|
|
836
|
+
# The enums the edge builders take. A string worked only because they are
|
|
837
|
+
# StrEnums; a typo in one failed at ingest, not at import (field report P11).
|
|
838
|
+
module.Method = Method # type: ignore[attr-defined]
|
|
839
|
+
module.ReferenceType = ReferenceType # type: ignore[attr-defined]
|
|
840
|
+
module.ContainmentType = ContainmentType # type: ignore[attr-defined]
|
|
841
|
+
module.FieldTransform = FieldTransform # type: ignore[attr-defined]
|
|
842
|
+
module.UsageType = UsageType # type: ignore[attr-defined]
|
|
749
843
|
module.flow = flow # type: ignore[attr-defined]
|
|
750
844
|
module.contains = contains # type: ignore[attr-defined]
|
|
751
845
|
module.references = references # type: ignore[attr-defined]
|
|
@@ -765,11 +859,17 @@ def build_module(context: Context) -> types.ModuleType:
|
|
|
765
859
|
module.__all__ = [ # type: ignore[attr-defined]
|
|
766
860
|
"Asset",
|
|
767
861
|
"Context",
|
|
862
|
+
"ContainmentType",
|
|
768
863
|
"FieldMapping",
|
|
864
|
+
"FieldTransform",
|
|
769
865
|
"FlowType",
|
|
866
|
+
"Method",
|
|
770
867
|
"NotebookFile",
|
|
771
868
|
"ParsedContent",
|
|
772
869
|
"Ref",
|
|
870
|
+
"ReferenceType",
|
|
871
|
+
"Tag",
|
|
872
|
+
"UsageType",
|
|
773
873
|
"contains",
|
|
774
874
|
"ctx",
|
|
775
875
|
"flow",
|
|
@@ -802,9 +902,11 @@ def namespace(context: Context) -> dict[str, Any]:
|
|
|
802
902
|
"__name__": "classifyre_notebook",
|
|
803
903
|
"__builtins__": __builtins__,
|
|
804
904
|
"Asset": Asset,
|
|
905
|
+
"Tag": Tag,
|
|
805
906
|
"Ref": Ref,
|
|
806
907
|
"FieldMapping": FieldMapping,
|
|
807
908
|
"FlowType": FlowType,
|
|
909
|
+
"Method": Method,
|
|
808
910
|
"flow": flow,
|
|
809
911
|
"contains": contains,
|
|
810
912
|
"references": references,
|
|
@@ -18,6 +18,10 @@ class ContentProvider(Protocol):
|
|
|
18
18
|
|
|
19
19
|
def asset_tags(self, asset_hash: str) -> Mapping[str, str]: ...
|
|
20
20
|
|
|
21
|
+
def asset_tag_severities(self, asset_hash: str) -> Mapping[str, str]: ...
|
|
22
|
+
|
|
23
|
+
def asserts_complete_tags(self, asset_hash: str) -> bool: ...
|
|
24
|
+
|
|
21
25
|
def enrich_finding_location(
|
|
22
26
|
self,
|
|
23
27
|
finding: DetectionResult,
|
|
@@ -1162,11 +1162,15 @@ class DetectorPipeline:
|
|
|
1162
1162
|
|
|
1163
1163
|
A tag is the one finding that does not come from reading anything. The
|
|
1164
1164
|
connector already knew the answer; the Tag detector named by the key
|
|
1165
|
-
supplies the label and severity, and the value the
|
|
1166
|
-
becomes the matched content.
|
|
1165
|
+
supplies the label and the severity ceiling, and the value the
|
|
1166
|
+
connector supplied becomes the matched content. A ``Tag(value,
|
|
1167
|
+
severity=)`` may lower the severity of its own finding within that
|
|
1168
|
+
ceiling (field report P9).
|
|
1167
1169
|
"""
|
|
1168
1170
|
tags = self.content_provider.asset_tags(str(asset.hash))
|
|
1169
|
-
|
|
1171
|
+
severities = self._asset_tag_severities(str(asset.hash))
|
|
1172
|
+
complete = self._asserts_complete_tags(str(asset.hash))
|
|
1173
|
+
if not tags and not complete:
|
|
1170
1174
|
return [], [], []
|
|
1171
1175
|
|
|
1172
1176
|
runners = self._tag_runners(detectors)
|
|
@@ -1181,20 +1185,66 @@ class DetectorPipeline:
|
|
|
1181
1185
|
# deleted. Silence would look like the tag simply not working,
|
|
1182
1186
|
# so name the key and what was available instead.
|
|
1183
1187
|
available = ", ".join(sorted(runners)) or "none configured"
|
|
1184
|
-
|
|
1188
|
+
msg = (
|
|
1185
1189
|
f"Tag '{key}' on asset '{asset.name}' matches no Tag detector "
|
|
1186
1190
|
f"and was skipped (available: {available})"
|
|
1187
1191
|
)
|
|
1192
|
+
logger.warning(msg)
|
|
1193
|
+
warnings.append(msg)
|
|
1188
1194
|
continue
|
|
1189
1195
|
detector, runner = entry
|
|
1190
|
-
|
|
1196
|
+
requested = severities.get(key)
|
|
1197
|
+
applied, lowered = runner.bound_severity(requested)
|
|
1198
|
+
if lowered:
|
|
1199
|
+
# The detector's severity is the ceiling. Saying so is the
|
|
1200
|
+
# point: a connector that thinks it raised a finding to
|
|
1201
|
+
# CRITICAL and silently did not would act on the wrong
|
|
1202
|
+
# severity for as long as nobody checked. Logged as well as
|
|
1203
|
+
# collected, because scan warnings live on the asset and the
|
|
1204
|
+
# runner log is where an operator actually looks.
|
|
1205
|
+
msg = (
|
|
1206
|
+
f"Tag '{key}' on asset '{asset.name}' asked for severity "
|
|
1207
|
+
f"{requested!r}, which its detector does not allow; "
|
|
1208
|
+
f"recorded as {str(applied).upper()}"
|
|
1209
|
+
)
|
|
1210
|
+
logger.warning(msg)
|
|
1211
|
+
warnings.append(msg)
|
|
1212
|
+
# The raw request, not `applied`: tag_finding bounds it again and
|
|
1213
|
+
# records what was refused on the finding itself.
|
|
1214
|
+
finding = runner.tag_finding(value, requested)
|
|
1191
1215
|
self.content_provider.enrich_finding_location(finding, asset, "")
|
|
1192
1216
|
findings.append(finding)
|
|
1193
1217
|
self._record_outcome(outcome_sink, detector, None)
|
|
1194
1218
|
types_run = self._merge_detector_types(types_run, [DetectorType.CUSTOM])
|
|
1195
1219
|
|
|
1220
|
+
if complete:
|
|
1221
|
+
# The source's tag set is complete, so an absent key is an answer:
|
|
1222
|
+
# this detector found nothing on this asset. Recording OK lets the
|
|
1223
|
+
# API resolve what it found in an earlier run. Without it a fact
|
|
1224
|
+
# the connector withdrew stayed an open finding for as long as the
|
|
1225
|
+
# asset lived (GENESIS field report P4).
|
|
1226
|
+
for key, (detector, _runner) in runners.items():
|
|
1227
|
+
if key not in tags:
|
|
1228
|
+
self._record_outcome(outcome_sink, detector, None)
|
|
1229
|
+
types_run = self._merge_detector_types(types_run, [DetectorType.CUSTOM])
|
|
1230
|
+
|
|
1196
1231
|
return findings, warnings, types_run
|
|
1197
1232
|
|
|
1233
|
+
def _asset_tag_severities(self, asset_hash: str) -> dict[str, str]:
|
|
1234
|
+
"""Per-tag severities this source asked for, or nothing.
|
|
1235
|
+
|
|
1236
|
+
Probed rather than called outright: a provider written before Tag
|
|
1237
|
+
severities existed still satisfies the pipeline's contract.
|
|
1238
|
+
"""
|
|
1239
|
+
probe = getattr(self.content_provider, "asset_tag_severities", None)
|
|
1240
|
+
if not callable(probe):
|
|
1241
|
+
return {}
|
|
1242
|
+
return dict(probe(asset_hash) or {})
|
|
1243
|
+
|
|
1244
|
+
def _asserts_complete_tags(self, asset_hash: str) -> bool:
|
|
1245
|
+
probe = getattr(self.content_provider, "asserts_complete_tags", None)
|
|
1246
|
+
return bool(probe(asset_hash)) if callable(probe) else False
|
|
1247
|
+
|
|
1198
1248
|
@staticmethod
|
|
1199
1249
|
def _tag_runners(detectors: list[BaseDetector]) -> dict[str, tuple[BaseDetector, Any]]:
|
|
1200
1250
|
"""Tag detectors in this run, keyed by the key a notebook writes."""
|
|
@@ -91,6 +91,12 @@ class ParsedContentProvider:
|
|
|
91
91
|
def asset_tags(self, asset_hash: str) -> Mapping[str, str]:
|
|
92
92
|
return self._source.asset_tags(asset_hash)
|
|
93
93
|
|
|
94
|
+
def asset_tag_severities(self, asset_hash: str) -> Mapping[str, str]:
|
|
95
|
+
return self._source.asset_tag_severities(asset_hash)
|
|
96
|
+
|
|
97
|
+
def asserts_complete_tags(self, asset_hash: str) -> bool:
|
|
98
|
+
return self._source.asserts_complete_tags(asset_hash)
|
|
99
|
+
|
|
94
100
|
def extracts_content(self, asset_hash: str) -> bool:
|
|
95
101
|
return self._source.extracts_content(asset_hash)
|
|
96
102
|
|
|
@@ -592,6 +592,25 @@ class BaseSource(ABC):
|
|
|
592
592
|
with self._augmentation_tags_lock:
|
|
593
593
|
return dict(self._augmentation_tags.get(asset_hash, {}))
|
|
594
594
|
|
|
595
|
+
def asset_tag_severities(self, asset_hash: str) -> Mapping[str, str]:
|
|
596
|
+
"""Per-tag severity overrides, for the keys a connector set one on.
|
|
597
|
+
|
|
598
|
+
Only the CUSTOM notebook can express one today (``Tag(value,
|
|
599
|
+
severity=)``); every other source leaves the Tag detector's own
|
|
600
|
+
severity in force, which is what an empty mapping means here.
|
|
601
|
+
"""
|
|
602
|
+
return {}
|
|
603
|
+
|
|
604
|
+
def asserts_complete_tags(self, asset_hash: str) -> bool:
|
|
605
|
+
"""Whether ``asset_tags`` is the *whole* truth about this asset.
|
|
606
|
+
|
|
607
|
+
When it is, a Tag detector whose key is absent has positively found
|
|
608
|
+
nothing, and a finding it produced earlier must resolve. Only a
|
|
609
|
+
connector that re-derives every tag on every run can promise that; an
|
|
610
|
+
augmentation notebook may tag a subset, so the default is False.
|
|
611
|
+
"""
|
|
612
|
+
return False
|
|
613
|
+
|
|
595
614
|
def enrich_finding_location(
|
|
596
615
|
self,
|
|
597
616
|
finding: DetectionResult,
|
|
@@ -17,6 +17,7 @@ import base64
|
|
|
17
17
|
import itertools
|
|
18
18
|
import json
|
|
19
19
|
import random
|
|
20
|
+
import re
|
|
20
21
|
import signal
|
|
21
22
|
import sys
|
|
22
23
|
from dataclasses import asdict
|
|
@@ -42,6 +43,11 @@ from ...notebook.serialize import (
|
|
|
42
43
|
to_module_source,
|
|
43
44
|
)
|
|
44
45
|
|
|
46
|
+
# The run log's level prefixes (runner-log-storage.service.ts matchLevelPrefix).
|
|
47
|
+
_LEVEL_PREFIX = re.compile(
|
|
48
|
+
r"^\s*\[?(TRACE|DEBUG|INFO|WARNING|WARN|ERROR|FATAL|CRITICAL)\]?\s*[:\-|\s]", re.I
|
|
49
|
+
)
|
|
50
|
+
|
|
45
51
|
ITEM = "item"
|
|
46
52
|
END = "end"
|
|
47
53
|
ERROR = "error"
|
|
@@ -159,7 +165,12 @@ class NotebookRuntime:
|
|
|
159
165
|
@staticmethod
|
|
160
166
|
def _log(message: str) -> None:
|
|
161
167
|
# ctx.log goes to stderr: stdout is the data channel, and a notebook
|
|
162
|
-
# that logs would otherwise corrupt the asset stream.
|
|
168
|
+
# that logs would otherwise corrupt the asset stream. The run log infers
|
|
169
|
+
# a line's level from its prefix; without one every notebook line was
|
|
170
|
+
# stored as UNKNOWN and vanished under any level filter (GENESIS field
|
|
171
|
+
# report P11). A line that already names its level keeps it.
|
|
172
|
+
if not _LEVEL_PREFIX.match(message):
|
|
173
|
+
message = f"INFO:notebook: {message}"
|
|
163
174
|
print(message, file=sys.stderr, flush=True)
|
|
164
175
|
|
|
165
176
|
def load(self) -> None:
|