voidaccess 1.8.2__tar.gz → 1.9.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {voidaccess-1.8.2/voidaccess.egg-info → voidaccess-1.9.1}/PKG-INFO +1 -1
- {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/entities.py +43 -11
- {voidaccess-1.8.2 → voidaccess-1.9.1}/config.py +11 -18
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/queries.py +2 -2
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/session.py +17 -4
- voidaccess-1.9.1/export/_entity_loading.py +46 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/export/ioc_package.py +26 -16
- {voidaccess-1.8.2 → voidaccess-1.9.1}/export/misp.py +8 -17
- {voidaccess-1.8.2 → voidaccess-1.9.1}/export/sigma.py +6 -16
- {voidaccess-1.8.2 → voidaccess-1.9.1}/export/snort_export.py +3 -12
- {voidaccess-1.8.2 → voidaccess-1.9.1}/export/stix.py +27 -62
- {voidaccess-1.8.2 → voidaccess-1.9.1}/export/yara_export.py +3 -13
- {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/confidence.py +31 -0
- voidaccess-1.9.1/extractor/identity.py +151 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/normalizer.py +34 -7
- {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/pipeline.py +5 -3
- {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/relationship_extract.py +12 -49
- {voidaccess-1.8.2 → voidaccess-1.9.1}/graph/builder.py +14 -5
- {voidaccess-1.8.2 → voidaccess-1.9.1}/graph/model.py +52 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/pyproject.toml +1 -1
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/dns_enrichment.py +2 -1
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_config.py +24 -1
- voidaccess-1.9.1/tests/test_consolidated_contracts.py +36 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_graph.py +23 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_llm.py +13 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_stix_export.py +73 -11
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess/config.py +4 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess/llm.py +26 -5
- {voidaccess-1.8.2 → voidaccess-1.9.1/voidaccess.egg-info}/PKG-INFO +1 -1
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess.egg-info/SOURCES.txt +3 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/__init__.py +1 -1
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/adapters/sqlite.py +9 -5
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/browser.py +20 -9
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/commands/enrich.py +9 -2
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/commands/investigate.py +140 -27
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/commands/show.py +27 -4
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/config.py +4 -4
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/display.py +33 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/main.py +5 -3
- {voidaccess-1.8.2 → voidaccess-1.9.1}/LICENSE +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/README.md +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/analysis/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/analysis/opsec.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/analysis/patterns.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/analysis/temporal.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/api/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/api/auth.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/api/errors.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/api/main.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/actors.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/admin.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/auth.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/export.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/investigations.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/monitors.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/search.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/settings.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/auth/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/auth/token_blacklist.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/crawler/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/crawler/dedup.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/crawler/frontier.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/crawler/spider.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/crawler/utils.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/env.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0001_initial_schema.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0002_add_investigation_status_column.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0002_add_missing_tables.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0003_add_canonical_value_and_entity_links.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0004_add_page_posted_at.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0005_add_extraction_method.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0006_add_monitor_alerts.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0007_add_actor_style_profiles.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0008_add_users_table.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0009_add_investigation_id_to_relationships.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0010_add_composite_index_entity_relationships.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0011_add_page_extraction_cache.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0013_add_graph_status.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0015_add_progress_fields.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0016_backfill_graph_status.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0017_add_user_api_keys.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0018_add_user_id_to_investigations.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0019_add_content_safety_log.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0020_add_entity_source_tracking.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0021_add_search_engine_stats.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0022_add_actor_profiles.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0023_add_investigation_metadata.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0024_add_investigation_step_metrics.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0025_add_cancellation_requested.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/models.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/db/search_engine_stats.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/export/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/entity_shape.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/gazetteer.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/llm_extract.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/ner.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/regex_patterns.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/fingerprint/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/fingerprint/calibration.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/fingerprint/profiler.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/fingerprint/stylometry.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/graph/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/graph/export.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/graph/queries.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/graph/visualize.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/i18n/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/i18n/detect.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/i18n/query_expand.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/i18n/translate.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/monitor/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/monitor/_db.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/monitor/alerts.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/monitor/config.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/monitor/diff.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/monitor/jobs.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/monitor/scheduler.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/scraper/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/scraper/scrape.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/scraper/scrape_js.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/search/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/search/circuit_breaker.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/search/engine_catalog.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/search/query_builder.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/search/search.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/setup.cfg +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/actor_profiles.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/blockchain.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/breach_lookup.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/cache.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/cisa.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/data/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/data/onion_seeds.json +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/domain_reputation.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/email_reputation.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/engines.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/enrichment.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/github_scraper.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/gitlab_scraper.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/hash_reputation.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/historical_intel.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/infostealer.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/ip_reputation.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/nvd.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/paste_scraper.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/pastes.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/proxy_client.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/rss_scraper.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/seed_manager.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/seeds.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/shodan.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/telegram.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/virustotal.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_analysis_opsec.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_analysis_stylometry.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_analysis_temporal.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_api.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_api_monitors.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_blockchain.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_breach_lookup.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_cli_proxy_config.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_confidence_calibration.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_crawler.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_db.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_dns_enrichment.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_domain_reputation.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_email_reputation.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_extractor_llm_pipeline.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_filter_e2e_manual.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_filter_parser_manual.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_fingerprint.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_github_scraper.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_gitlab_scraper.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_graph_integration.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_hash_reputation.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_i18n.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_infostealer.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_ip_reputation.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_llm_utils.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_model_singleton.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_monitor.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_monitor_integration.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_nvd.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_pagination.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_paste_scraper.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_proxy_client.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_ransomlook.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_regex_patterns.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_relationship_extract.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_rss_scraper.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_scrape_js.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_settings.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_snort_export.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_sources.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_sources_enrichment_new.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_vector.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_yara_export.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/async_utils.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/content_dedup.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/content_safety.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/defang.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/encryption.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/enrichment_cache.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/investigation_metrics.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/ioc_freshness.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/user_keys.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/vector/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/vector/embedder.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/vector/model_singleton.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/vector/search.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/vector/store.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess/llm_utils.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess.egg-info/dependency_links.txt +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess.egg-info/entry_points.txt +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess.egg-info/requires.txt +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess.egg-info/top_level.txt +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/adapters/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/commands/__init__.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/commands/actors.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/commands/configure.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/commands/export.py +0 -0
- {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/tor_detect.py +0 -0
|
@@ -22,6 +22,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query
|
|
|
22
22
|
from fastapi.responses import Response
|
|
23
23
|
from api.auth import CurrentUser, get_current_user
|
|
24
24
|
from api.errors import GENERIC_ERROR_MESSAGE, log_exception
|
|
25
|
+
from extractor.identity import entity_canonical_id
|
|
25
26
|
|
|
26
27
|
logger = logging.getLogger(__name__)
|
|
27
28
|
router = APIRouter()
|
|
@@ -321,7 +322,9 @@ async def get_stylometry_analysis(
|
|
|
321
322
|
if entity is None:
|
|
322
323
|
raise HTTPException(status_code=404, detail="Entity not found")
|
|
323
324
|
|
|
324
|
-
|
|
325
|
+
# Canonical dedup key via the shared identity module (was an inline
|
|
326
|
+
# `entity.canonical_value or entity.value.lower()` re-derivation).
|
|
327
|
+
canonical = entity_canonical_id(entity)
|
|
325
328
|
|
|
326
329
|
related = (
|
|
327
330
|
session.query(Entity)
|
|
@@ -406,7 +409,7 @@ async def get_stylometry_analysis(
|
|
|
406
409
|
similar_actors = await asyncio.to_thread(
|
|
407
410
|
_find_similar_actors,
|
|
408
411
|
profile=profile,
|
|
409
|
-
canonical_value=entity
|
|
412
|
+
canonical_value=entity_canonical_id(entity),
|
|
410
413
|
entity_type=entity.entity_type,
|
|
411
414
|
)
|
|
412
415
|
except Exception as e:
|
|
@@ -474,7 +477,9 @@ async def get_opsec_analysis(
|
|
|
474
477
|
if entity is None:
|
|
475
478
|
raise HTTPException(status_code=404, detail="Entity not found")
|
|
476
479
|
|
|
477
|
-
|
|
480
|
+
# Canonical dedup key via the shared identity module (was an inline
|
|
481
|
+
# `entity.canonical_value or entity.value.lower()` re-derivation).
|
|
482
|
+
canonical = entity_canonical_id(entity)
|
|
478
483
|
related = (
|
|
479
484
|
session.query(Entity)
|
|
480
485
|
.filter(
|
|
@@ -827,17 +832,44 @@ def _get_entity_value(entity_id: str) -> Optional[str]:
|
|
|
827
832
|
return None
|
|
828
833
|
|
|
829
834
|
|
|
830
|
-
def _resolve_graph_node_id(
|
|
831
|
-
|
|
835
|
+
def _resolve_graph_node_id(
|
|
836
|
+
graph, entity_value: str, entity_type: Optional[str] = None
|
|
837
|
+
) -> Optional[str]:
|
|
838
|
+
"""Resolve a graph node id for an entity value.
|
|
839
|
+
|
|
840
|
+
The graph is keyed by canonical identity (extractor.identity.entity_graph_id).
|
|
841
|
+
When ``entity_type`` is known, the lookup value is canonicalised through the
|
|
842
|
+
same shared module so it matches the node regardless of the casing the value
|
|
843
|
+
was observed in — closing the case-sensitivity gap the audit flagged.
|
|
844
|
+
Without a type it falls back to the historical exact-then-``handle@domain``
|
|
845
|
+
prefix behaviour.
|
|
846
|
+
"""
|
|
832
847
|
if graph is None:
|
|
833
848
|
return None
|
|
834
|
-
if graph.has_node(entity_value):
|
|
835
|
-
return entity_value
|
|
836
849
|
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
850
|
+
candidates = [entity_value]
|
|
851
|
+
if entity_type:
|
|
852
|
+
try:
|
|
853
|
+
from types import SimpleNamespace # noqa: PLC0415
|
|
854
|
+
|
|
855
|
+
canon = entity_canonical_id(
|
|
856
|
+
SimpleNamespace(entity_type=entity_type, value=entity_value)
|
|
857
|
+
)
|
|
858
|
+
if canon and canon not in candidates:
|
|
859
|
+
candidates.append(canon)
|
|
860
|
+
except Exception:
|
|
861
|
+
pass
|
|
862
|
+
|
|
863
|
+
# 1) exact match on any candidate
|
|
864
|
+
for cand in candidates:
|
|
865
|
+
if graph.has_node(cand):
|
|
866
|
+
return cand
|
|
867
|
+
# 2) handle@domain prefix match on any candidate
|
|
868
|
+
for cand in candidates:
|
|
869
|
+
prefix = f"{cand}@"
|
|
870
|
+
for node_id in graph.nodes:
|
|
871
|
+
if isinstance(node_id, str) and node_id.startswith(prefix):
|
|
872
|
+
return node_id
|
|
841
873
|
return None
|
|
842
874
|
|
|
843
875
|
|
|
@@ -5,7 +5,7 @@ from dotenv import load_dotenv
|
|
|
5
5
|
load_dotenv()
|
|
6
6
|
|
|
7
7
|
logger = logging.getLogger(__name__)
|
|
8
|
-
|
|
8
|
+
_OPTIONAL_CONFIG_WARNING_STATE = "_voidaccess_optional_config_warning_shown"
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
def _clean_env(name, default=None):
|
|
@@ -151,16 +151,12 @@ except ValueError:
|
|
|
151
151
|
BLOCKCYPHER_TOKEN = _clean_env("BLOCKCYPHER_TOKEN", "")
|
|
152
152
|
ETHERSCAN_API_KEY = _clean_env("ETHERSCAN_API_KEY", "")
|
|
153
153
|
|
|
154
|
-
# Auth — REQUIRED
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
"and set it as JWT_SECRET in your .env file. "
|
|
161
|
-
"Do NOT use a random secret — it will change on restart and invalidate all issued tokens."
|
|
162
|
-
)
|
|
163
|
-
JWT_SECRET = _jwt_secret
|
|
154
|
+
# Auth — REQUIRED when the API/auth stack is actually used. Do not validate
|
|
155
|
+
# this at import time: the CLI injects its runtime environment in its entry
|
|
156
|
+
# point before importing dependent modules, and `--help`/`--version` must work
|
|
157
|
+
# on a fresh install. `validate_config()` and the API startup path enforce it
|
|
158
|
+
# at the point where authentication is needed.
|
|
159
|
+
JWT_SECRET = _clean_env("JWT_SECRET")
|
|
164
160
|
|
|
165
161
|
# Token blacklist Redis (optional — omit to disable blacklist checks)
|
|
166
162
|
REDIS_URL = _clean_env("REDIS_URL")
|
|
@@ -210,7 +206,6 @@ OPTIONAL_KEYS = [
|
|
|
210
206
|
|
|
211
207
|
|
|
212
208
|
def validate_config():
|
|
213
|
-
global _OPTIONAL_CONFIG_WARNING_SHOWN
|
|
214
209
|
missing_required = []
|
|
215
210
|
for key in REQUIRED_KEYS:
|
|
216
211
|
if _clean_env(key) is None:
|
|
@@ -221,16 +216,14 @@ def validate_config():
|
|
|
221
216
|
key for key in OPTIONAL_KEYS
|
|
222
217
|
if (_clean_env(key) is None or _clean_env(key) == "")
|
|
223
218
|
]
|
|
224
|
-
if missing_optional and not
|
|
219
|
+
if missing_optional and not getattr(logging, _OPTIONAL_CONFIG_WARNING_STATE, False):
|
|
225
220
|
logger.warning(
|
|
226
221
|
"Optional configuration keys not set (%d): %s - related features will be disabled",
|
|
227
222
|
len(missing_optional),
|
|
228
223
|
", ".join(missing_optional),
|
|
229
224
|
)
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
validate_config()
|
|
234
|
-
|
|
225
|
+
# Store the guard on the process-wide logging module so compatibility
|
|
226
|
+
# wrappers/reloads cannot reset it and reintroduce warning spam.
|
|
227
|
+
setattr(logging, _OPTIONAL_CONFIG_WARNING_STATE, True)
|
|
235
228
|
|
|
236
229
|
|
|
@@ -348,8 +348,8 @@ def upsert_entity_canonical(
|
|
|
348
348
|
|
|
349
349
|
if existing:
|
|
350
350
|
# Update confidence if new extraction is more confident
|
|
351
|
-
|
|
352
|
-
|
|
351
|
+
from extractor.confidence import get_entity_confidence
|
|
352
|
+
existing.confidence = get_entity_confidence(existing, confidence)
|
|
353
353
|
# Update context if we have a better snippet
|
|
354
354
|
if context_snippet and len(context_snippet) > len(existing.context_snippet or ""):
|
|
355
355
|
existing.context_snippet = context_snippet
|
|
@@ -38,6 +38,7 @@ Usage (testing — pass an explicit URL to avoid needing DATABASE_URL in env)
|
|
|
38
38
|
from __future__ import annotations
|
|
39
39
|
|
|
40
40
|
from functools import lru_cache
|
|
41
|
+
import os
|
|
41
42
|
|
|
42
43
|
from contextlib import asynccontextmanager, contextmanager
|
|
43
44
|
from typing import AsyncGenerator, Generator, Optional
|
|
@@ -48,7 +49,19 @@ from sqlalchemy.orm import sessionmaker, Session
|
|
|
48
49
|
|
|
49
50
|
import config
|
|
50
51
|
|
|
52
|
+
# Kept as a compatibility/testing hook. Production lookups below use the
|
|
53
|
+
# current environment so apply_env() can run before the first DB operation.
|
|
51
54
|
DATABASE_URL = config.DATABASE_URL
|
|
55
|
+
_INITIAL_DATABASE_URL = DATABASE_URL
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _database_url() -> Optional[str]:
|
|
59
|
+
# Preserve the long-standing test/embedding hook of monkeypatching this
|
|
60
|
+
# module's DATABASE_URL. Otherwise, read the live environment so a caller
|
|
61
|
+
# can inject DATABASE_URL after db.session has been imported.
|
|
62
|
+
if DATABASE_URL != _INITIAL_DATABASE_URL:
|
|
63
|
+
return DATABASE_URL
|
|
64
|
+
return os.getenv("DATABASE_URL") or DATABASE_URL
|
|
52
65
|
_async_engine_cache: dict[str, "AsyncEngine"] = {}
|
|
53
66
|
|
|
54
67
|
|
|
@@ -104,7 +117,7 @@ def get_engine(url: Optional[str] = None) -> Engine:
|
|
|
104
117
|
PostgreSQL gets a connection pool tuned for the scraping workload.
|
|
105
118
|
SQLite skips pool parameters that only apply to QueuePool.
|
|
106
119
|
"""
|
|
107
|
-
target_url = url or
|
|
120
|
+
target_url = url or _database_url()
|
|
108
121
|
if not target_url:
|
|
109
122
|
raise RuntimeError(
|
|
110
123
|
"DATABASE_URL is not configured.\n"
|
|
@@ -122,7 +135,7 @@ def release_engine(url: Optional[str] = None) -> None:
|
|
|
122
135
|
Calls engine.dispose() to release connection pool resources and file handles,
|
|
123
136
|
then clears the cache. Use this in test teardown to prevent leaks.
|
|
124
137
|
"""
|
|
125
|
-
target_url = url or
|
|
138
|
+
target_url = url or _database_url()
|
|
126
139
|
if target_url:
|
|
127
140
|
try:
|
|
128
141
|
engine = get_engine(target_url)
|
|
@@ -139,7 +152,7 @@ def get_async_engine(url: Optional[str] = None) -> "AsyncEngine":
|
|
|
139
152
|
Converts postgresql:// to postgresql+asyncpg:// and sqlite:// to sqlite+aiosqlite://.
|
|
140
153
|
"""
|
|
141
154
|
|
|
142
|
-
target_url = url or
|
|
155
|
+
target_url = url or _database_url()
|
|
143
156
|
if not target_url:
|
|
144
157
|
raise RuntimeError(
|
|
145
158
|
"DATABASE_URL is not configured.\n"
|
|
@@ -185,7 +198,7 @@ def release_async_engine(url: Optional[str] = None) -> None:
|
|
|
185
198
|
|
|
186
199
|
Calls engine.dispose() to release connection pool resources and file handles.
|
|
187
200
|
"""
|
|
188
|
-
target_url = url or
|
|
201
|
+
target_url = url or _database_url()
|
|
189
202
|
if target_url in _async_engine_cache:
|
|
190
203
|
_async_engine_cache[target_url].dispose()
|
|
191
204
|
del _async_engine_cache[target_url]
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""
|
|
2
|
+
export/_entity_loading.py — one shared DB-row → NormalizedEntity mapping for
|
|
3
|
+
every exporter.
|
|
4
|
+
|
|
5
|
+
Previously STIX, MISP, and Sigma each reconstructed a ``NormalizedEntity`` from
|
|
6
|
+
a DB ``Entity`` row independently, and populated *different* fields — STIX set
|
|
7
|
+
``source_quality`` and ``extraction_method``, MISP and Sigma did not. A future
|
|
8
|
+
consumer reading ``entity.source_quality`` therefore got different behaviour
|
|
9
|
+
depending on which export path produced the entity (audit finding: STIX/MISP
|
|
10
|
+
field divergence). Centralising the construction here closes that drift: all
|
|
11
|
+
three exporters now produce identically-populated entities.
|
|
12
|
+
|
|
13
|
+
The value mapping is preserved exactly as it was in all three call sites
|
|
14
|
+
(``canonical_value or value``) so export outputs are unchanged; only the
|
|
15
|
+
previously-inconsistent auxiliary fields are now uniform.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
from extractor.normalizer import NormalizedEntity
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def normalized_entity_from_db_row(e: Any) -> NormalizedEntity:
|
|
26
|
+
"""Build a NormalizedEntity from a DB Entity row with consistent fields.
|
|
27
|
+
|
|
28
|
+
Used by every exporter (STIX / MISP / Sigma) so field population can never
|
|
29
|
+
drift between export paths again.
|
|
30
|
+
"""
|
|
31
|
+
source_url = ""
|
|
32
|
+
try:
|
|
33
|
+
if e.page:
|
|
34
|
+
source_url = e.page.url or ""
|
|
35
|
+
except Exception:
|
|
36
|
+
pass
|
|
37
|
+
return NormalizedEntity(
|
|
38
|
+
entity_type=e.entity_type,
|
|
39
|
+
value=e.canonical_value or e.value,
|
|
40
|
+
confidence=e.confidence,
|
|
41
|
+
source_url=source_url,
|
|
42
|
+
page_id=e.page_id,
|
|
43
|
+
context_snippet=e.context_snippet or "",
|
|
44
|
+
extraction_method="db",
|
|
45
|
+
source_quality=getattr(e, "source_quality", 1.0),
|
|
46
|
+
)
|
|
@@ -45,8 +45,11 @@ import logging
|
|
|
45
45
|
import re
|
|
46
46
|
import zipfile
|
|
47
47
|
from datetime import datetime, timezone
|
|
48
|
+
from types import SimpleNamespace
|
|
48
49
|
from typing import Any, Iterable, Optional
|
|
49
50
|
|
|
51
|
+
from extractor.identity import entity_canonical_id
|
|
52
|
+
|
|
50
53
|
logger = logging.getLogger(__name__)
|
|
51
54
|
|
|
52
55
|
# ---------------------------------------------------------------------------
|
|
@@ -54,7 +57,7 @@ logger = logging.getLogger(__name__)
|
|
|
54
57
|
# ---------------------------------------------------------------------------
|
|
55
58
|
|
|
56
59
|
PACKAGE_FORMAT = "voidaccess-ioc-v1"
|
|
57
|
-
PACKAGE_VERSION = "1.
|
|
60
|
+
PACKAGE_VERSION = "1.9.1"
|
|
58
61
|
SNORT_SID_BASE = 9000001 # 9xxxxxx reserved for VoidAccess-generated rules
|
|
59
62
|
|
|
60
63
|
# Map extractor upper-case entity types to the per-file bucket they belong to.
|
|
@@ -138,19 +141,28 @@ def _entity_type(e: Any) -> str:
|
|
|
138
141
|
|
|
139
142
|
|
|
140
143
|
def _entity_value(e: Any) -> str:
|
|
141
|
-
"""Return the entity's
|
|
144
|
+
"""Return the entity's canonical value via the shared identity module.
|
|
145
|
+
|
|
146
|
+
Migrated (audit finding 1.3 / 3.9 item 2) from the old
|
|
147
|
+
``entity.canonical_value or entity.value`` pattern to call
|
|
148
|
+
``extractor.identity.entity_canonical_id`` directly. Phase 1 already fixed
|
|
149
|
+
``NormalizedEntity.canonical_value`` so that old pattern no longer returned
|
|
150
|
+
the raw value for pipeline-stage entities — but this makes the choice
|
|
151
|
+
explicit at the call site (one shared function, not an assumption about
|
|
152
|
+
what a property returns), so DB-backed and pipeline-stage versions of the
|
|
153
|
+
same entity can never produce inconsistent casing here again.
|
|
154
|
+
"""
|
|
155
|
+
etype = _entity_type(e)
|
|
142
156
|
if isinstance(e, dict):
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
or ""
|
|
153
|
-
) or ""
|
|
157
|
+
raw = e.get("value") or e.get("canonical_value") or e.get("canonical") or ""
|
|
158
|
+
else:
|
|
159
|
+
raw = getattr(e, "value", None) or getattr(e, "canonical_value", None) or ""
|
|
160
|
+
if not raw:
|
|
161
|
+
return ""
|
|
162
|
+
try:
|
|
163
|
+
return entity_canonical_id(SimpleNamespace(entity_type=etype, value=raw))
|
|
164
|
+
except Exception:
|
|
165
|
+
return raw
|
|
154
166
|
|
|
155
167
|
|
|
156
168
|
def _entity_confidence(e: Any) -> float:
|
|
@@ -730,9 +742,7 @@ class _DictEntityAdapter:
|
|
|
730
742
|
|
|
731
743
|
@property
|
|
732
744
|
def canonical_value(self) -> str:
|
|
733
|
-
|
|
734
|
-
return self._d.get("canonical_value") or self._d.get("value") or ""
|
|
735
|
-
return ""
|
|
745
|
+
return entity_canonical_id(self._d)
|
|
736
746
|
|
|
737
747
|
@property
|
|
738
748
|
def confidence(self) -> float:
|
|
@@ -209,23 +209,14 @@ def _load_investigation_and_entities(
|
|
|
209
209
|
want = frozenset(filter_uuids)
|
|
210
210
|
db_entities = [e for e in db_entities if e.id in want]
|
|
211
211
|
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
ne = NormalizedEntity(
|
|
221
|
-
entity_type=e.entity_type,
|
|
222
|
-
value=e.canonical_value or e.value,
|
|
223
|
-
confidence=e.confidence,
|
|
224
|
-
source_url=source_url,
|
|
225
|
-
page_id=e.page_id,
|
|
226
|
-
context_snippet=e.context_snippet or "",
|
|
227
|
-
)
|
|
228
|
-
normalized.append(ne)
|
|
212
|
+
from export._entity_loading import normalized_entity_from_db_row # noqa: PLC0415
|
|
213
|
+
|
|
214
|
+
# Shared row→NormalizedEntity mapping (same one STIX/Sigma use), so
|
|
215
|
+
# auxiliary fields (source_quality, extraction_method) no longer
|
|
216
|
+
# differ between export paths.
|
|
217
|
+
normalized: list[NormalizedEntity] = [
|
|
218
|
+
normalized_entity_from_db_row(e) for e in db_entities
|
|
219
|
+
]
|
|
229
220
|
|
|
230
221
|
session.expunge_all()
|
|
231
222
|
return investigation, normalized
|
|
@@ -307,22 +307,12 @@ def _load_entities_for_investigation(investigation_id: Any) -> list[Any]:
|
|
|
307
307
|
.all()
|
|
308
308
|
)
|
|
309
309
|
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
except Exception:
|
|
317
|
-
pass
|
|
318
|
-
result.append(NormalizedEntity(
|
|
319
|
-
entity_type=e.entity_type,
|
|
320
|
-
value=e.canonical_value or e.value,
|
|
321
|
-
confidence=e.confidence,
|
|
322
|
-
source_url=source_url,
|
|
323
|
-
page_id=e.page_id,
|
|
324
|
-
context_snippet=e.context_snippet or "",
|
|
325
|
-
))
|
|
310
|
+
from export._entity_loading import normalized_entity_from_db_row # noqa: PLC0415
|
|
311
|
+
|
|
312
|
+
# Shared row→NormalizedEntity mapping (same one STIX/MISP use).
|
|
313
|
+
result: list[NormalizedEntity] = [
|
|
314
|
+
normalized_entity_from_db_row(e) for e in db_entities
|
|
315
|
+
]
|
|
326
316
|
return result
|
|
327
317
|
|
|
328
318
|
except Exception as exc:
|
|
@@ -44,6 +44,8 @@ from datetime import datetime, timezone
|
|
|
44
44
|
from typing import Any
|
|
45
45
|
from urllib.parse import urlparse
|
|
46
46
|
|
|
47
|
+
from extractor.identity import entity_canonical_id
|
|
48
|
+
|
|
47
49
|
# ---------------------------------------------------------------------------
|
|
48
50
|
# Constants
|
|
49
51
|
# ---------------------------------------------------------------------------
|
|
@@ -104,18 +106,7 @@ def _entity_type(e: Any) -> str:
|
|
|
104
106
|
|
|
105
107
|
|
|
106
108
|
def _entity_value(e: Any) -> str:
|
|
107
|
-
|
|
108
|
-
return (
|
|
109
|
-
e.get("canonical_value")
|
|
110
|
-
or e.get("value")
|
|
111
|
-
or e.get("canonical")
|
|
112
|
-
or ""
|
|
113
|
-
) or ""
|
|
114
|
-
return (
|
|
115
|
-
getattr(e, "canonical_value", None)
|
|
116
|
-
or getattr(e, "value", None)
|
|
117
|
-
or ""
|
|
118
|
-
) or ""
|
|
109
|
+
return entity_canonical_id(e)
|
|
119
110
|
|
|
120
111
|
|
|
121
112
|
def _entity_confidence(e: Any) -> float:
|
|
@@ -22,6 +22,9 @@ import warnings
|
|
|
22
22
|
from typing import Any, Optional
|
|
23
23
|
import uuid
|
|
24
24
|
|
|
25
|
+
from graph.model import RELATIONSHIP_TYPE_STIX
|
|
26
|
+
from extractor.confidence import get_entity_confidence
|
|
27
|
+
|
|
25
28
|
logger = logging.getLogger(__name__)
|
|
26
29
|
|
|
27
30
|
# MED-9: suppress SQLAlchemy SAWarning from ORM IN-clause coercion internals.
|
|
@@ -127,7 +130,7 @@ def entity_to_stix_indicator(entity: Any) -> Optional[Any]:
|
|
|
127
130
|
pattern=pattern,
|
|
128
131
|
pattern_type="stix",
|
|
129
132
|
indicator_types=indicator_types,
|
|
130
|
-
confidence=_to_stix_confidence(entity
|
|
133
|
+
confidence=_to_stix_confidence(get_entity_confidence(entity)),
|
|
131
134
|
allow_custom=True,
|
|
132
135
|
x_voidaccess_source_quality=getattr(entity, "source_quality", 1.0),
|
|
133
136
|
external_references=(
|
|
@@ -158,7 +161,7 @@ def entity_to_stix_malware(entity: Any) -> Optional[Any]:
|
|
|
158
161
|
malware = stix2.Malware(
|
|
159
162
|
name=entity.value,
|
|
160
163
|
is_family=True,
|
|
161
|
-
confidence=_to_stix_confidence(entity
|
|
164
|
+
confidence=_to_stix_confidence(get_entity_confidence(entity)),
|
|
162
165
|
allow_custom=True,
|
|
163
166
|
x_voidaccess_source_quality=getattr(entity, "source_quality", 1.0),
|
|
164
167
|
external_references=(
|
|
@@ -189,7 +192,7 @@ def entity_to_stix_threat_actor(entity: Any) -> Optional[Any]:
|
|
|
189
192
|
threat_actor = stix2.ThreatActor(
|
|
190
193
|
name=entity.value,
|
|
191
194
|
aliases=[entity.value],
|
|
192
|
-
confidence=_to_stix_confidence(entity
|
|
195
|
+
confidence=_to_stix_confidence(get_entity_confidence(entity)),
|
|
193
196
|
allow_custom=True,
|
|
194
197
|
x_voidaccess_source_quality=getattr(entity, "source_quality", 1.0),
|
|
195
198
|
external_references=(
|
|
@@ -241,30 +244,28 @@ def investigation_to_stix_bundle(
|
|
|
241
244
|
if not entities:
|
|
242
245
|
return _empty_bundle()
|
|
243
246
|
|
|
244
|
-
|
|
245
|
-
from graph.builder import _make_node_id # noqa: PLC0415
|
|
246
|
-
except Exception: # pragma: no cover — graph layer optional
|
|
247
|
-
def _make_node_id(entity_type, value, source_url): # type: ignore
|
|
248
|
-
return value
|
|
247
|
+
from extractor.identity import entity_canonical_id, entity_graph_id # noqa: PLC0415
|
|
249
248
|
|
|
250
249
|
stix_objects: list[Any] = []
|
|
251
|
-
# Maps BOTH the
|
|
252
|
-
# Graph edges are keyed by
|
|
253
|
-
# "
|
|
254
|
-
#
|
|
255
|
-
# "silently dropped relationships" failure
|
|
256
|
-
#
|
|
257
|
-
#
|
|
250
|
+
# Maps BOTH the graph node id AND the bare canonical value → stix_object.id.
|
|
251
|
+
# Graph edges are keyed by the graph node id (which for THREAT_ACTOR_HANDLE
|
|
252
|
+
# is "<canonical>@domain", not the bare value); registering only the bare
|
|
253
|
+
# value silently dropped every threat-actor relationship — the exact
|
|
254
|
+
# "silently dropped relationships" failure the CHANGELOG keeps re-fixing.
|
|
255
|
+
#
|
|
256
|
+
# Both keys are now derived through extractor.identity, the *same* module
|
|
257
|
+
# the graph builder uses to generate node ids. The two sides can no longer
|
|
258
|
+
# diverge (e.g. "LockBit" vs "lockbit") because they are literally the same
|
|
259
|
+
# function call, not two functions that happen to agree today.
|
|
258
260
|
stix_id_map: dict[str, str] = {}
|
|
259
261
|
|
|
260
262
|
def _register(entity: Any, stix_id: str) -> None:
|
|
261
|
-
stix_id_map.setdefault(entity.value, stix_id)
|
|
262
263
|
try:
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
)
|
|
264
|
+
stix_id_map.setdefault(entity_canonical_id(entity), stix_id)
|
|
265
|
+
except Exception:
|
|
266
|
+
stix_id_map.setdefault(entity.value, stix_id)
|
|
267
|
+
try:
|
|
268
|
+
node_id = entity_graph_id(entity)
|
|
268
269
|
except Exception:
|
|
269
270
|
node_id = entity.value
|
|
270
271
|
if node_id:
|
|
@@ -381,6 +382,7 @@ def _load_entities_for_investigation(
|
|
|
381
382
|
from db.queries import get_investigation_by_id_or_run # noqa: PLC0415
|
|
382
383
|
from db.models import Entity, InvestigationEntityLink # noqa: PLC0415
|
|
383
384
|
from extractor.normalizer import NormalizedEntity # noqa: PLC0415
|
|
385
|
+
from export._entity_loading import normalized_entity_from_db_row # noqa: PLC0415
|
|
384
386
|
|
|
385
387
|
inv_uuid = _coerce_uuid(investigation_id)
|
|
386
388
|
if inv_uuid is None:
|
|
@@ -407,24 +409,9 @@ def _load_entities_for_investigation(
|
|
|
407
409
|
want = frozenset(entity_ids)
|
|
408
410
|
db_entities = [e for e in db_entities if e.id in want]
|
|
409
411
|
|
|
410
|
-
result: list[NormalizedEntity] = [
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
try:
|
|
414
|
-
if e.page:
|
|
415
|
-
source_url = e.page.url or ""
|
|
416
|
-
except Exception:
|
|
417
|
-
pass
|
|
418
|
-
result.append(NormalizedEntity(
|
|
419
|
-
entity_type=e.entity_type,
|
|
420
|
-
value=e.canonical_value or e.value,
|
|
421
|
-
confidence=e.confidence,
|
|
422
|
-
source_url=source_url,
|
|
423
|
-
page_id=e.page_id,
|
|
424
|
-
context_snippet=e.context_snippet or "",
|
|
425
|
-
extraction_method="db",
|
|
426
|
-
source_quality=getattr(e, "source_quality", 1.0),
|
|
427
|
-
))
|
|
412
|
+
result: list[NormalizedEntity] = [
|
|
413
|
+
normalized_entity_from_db_row(e) for e in db_entities
|
|
414
|
+
]
|
|
428
415
|
return result
|
|
429
416
|
|
|
430
417
|
if session is not None:
|
|
@@ -501,29 +488,7 @@ def _build_stix_relationships(
|
|
|
501
488
|
|
|
502
489
|
def _edge_type_to_stix(edge_type: str) -> str:
|
|
503
490
|
"""Map VoidAccess graph edge types to STIX relationship type strings."""
|
|
504
|
-
|
|
505
|
-
"CO_APPEARED_ON": "related-to",
|
|
506
|
-
"CO_INVESTIGATION": "related-to",
|
|
507
|
-
"POSTED_BY": "attributed-to",
|
|
508
|
-
"LINKED_TO": "related-to",
|
|
509
|
-
"PAID_TO": "related-to",
|
|
510
|
-
"MEMBER_OF": "member-of",
|
|
511
|
-
"USES": "uses",
|
|
512
|
-
"CLAIMED": "attributed-to",
|
|
513
|
-
"LIKELY_SAME_ACTOR": "related-to",
|
|
514
|
-
"CONFIRMED_SAME_ACTOR": "related-to",
|
|
515
|
-
"FUNDED_BY": "related-to",
|
|
516
|
-
# Typed relationships from the LLM relationship-extraction pass. These
|
|
517
|
-
# map to documented STIX 2.1 common relationship types where one
|
|
518
|
-
# exists; CONTROLS has no standard STIX verb so it degrades to the
|
|
519
|
-
# safe "related-to" default rather than emitting a non-spec type.
|
|
520
|
-
"DROPS": "drops",
|
|
521
|
-
"TARGETS": "targets",
|
|
522
|
-
"EXPLOITS": "exploits",
|
|
523
|
-
"COMMUNICATES_WITH": "communicates-with",
|
|
524
|
-
"CONTROLS": "related-to",
|
|
525
|
-
}
|
|
526
|
-
return mapping.get(edge_type, "related-to")
|
|
491
|
+
return RELATIONSHIP_TYPE_STIX.get(edge_type, "related-to")
|
|
527
492
|
|
|
528
493
|
|
|
529
494
|
def _coerce_uuid(value: Any) -> Optional[uuid.UUID]:
|
|
@@ -35,6 +35,8 @@ import re
|
|
|
35
35
|
from datetime import datetime, timezone
|
|
36
36
|
from typing import Any, Optional
|
|
37
37
|
|
|
38
|
+
from extractor.identity import entity_canonical_id
|
|
39
|
+
|
|
38
40
|
# ---------------------------------------------------------------------------
|
|
39
41
|
# Constants
|
|
40
42
|
# ---------------------------------------------------------------------------
|
|
@@ -115,19 +117,7 @@ def _entity_type(e: Any) -> str:
|
|
|
115
117
|
|
|
116
118
|
|
|
117
119
|
def _entity_value(e: Any) -> str:
|
|
118
|
-
|
|
119
|
-
if isinstance(e, dict):
|
|
120
|
-
return (
|
|
121
|
-
e.get("canonical_value")
|
|
122
|
-
or e.get("value")
|
|
123
|
-
or e.get("canonical")
|
|
124
|
-
or ""
|
|
125
|
-
) or ""
|
|
126
|
-
return (
|
|
127
|
-
getattr(e, "canonical_value", None)
|
|
128
|
-
or getattr(e, "value", None)
|
|
129
|
-
or ""
|
|
130
|
-
) or ""
|
|
120
|
+
return entity_canonical_id(e)
|
|
131
121
|
|
|
132
122
|
|
|
133
123
|
def _entity_confidence(e: Any) -> float:
|