voidaccess 1.9.0__tar.gz → 1.9.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {voidaccess-1.9.0 → voidaccess-1.9.2}/PKG-INFO +12 -1
- {voidaccess-1.9.0 → voidaccess-1.9.2}/README.md +11 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/config.py +11 -18
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/session.py +17 -4
- {voidaccess-1.9.0 → voidaccess-1.9.2}/export/ioc_package.py +1 -1
- {voidaccess-1.9.0 → voidaccess-1.9.2}/export/misp.py +72 -19
- voidaccess-1.9.2/export/relationships.py +81 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/export/stix.py +45 -22
- {voidaccess-1.9.0 → voidaccess-1.9.2}/graph/builder.py +10 -2
- {voidaccess-1.9.0 → voidaccess-1.9.2}/pyproject.toml +1 -1
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_config.py +24 -1
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_llm.py +13 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/vector/model_singleton.py +5 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess/config.py +4 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess/llm.py +26 -5
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess.egg-info/PKG-INFO +12 -1
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess.egg-info/SOURCES.txt +1 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess_cli/__init__.py +1 -1
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess_cli/commands/configure.py +2 -3
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess_cli/commands/export.py +17 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess_cli/commands/investigate.py +58 -10
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess_cli/commands/show.py +2 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess_cli/config.py +71 -24
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess_cli/main.py +5 -3
- {voidaccess-1.9.0 → voidaccess-1.9.2}/LICENSE +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/analysis/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/analysis/opsec.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/analysis/patterns.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/analysis/temporal.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/api/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/api/auth.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/api/errors.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/api/main.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/api/routes/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/api/routes/actors.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/api/routes/admin.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/api/routes/auth.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/api/routes/entities.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/api/routes/export.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/api/routes/investigations.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/api/routes/monitors.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/api/routes/search.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/api/routes/settings.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/auth/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/auth/token_blacklist.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/crawler/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/crawler/dedup.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/crawler/frontier.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/crawler/spider.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/crawler/utils.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/env.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0001_initial_schema.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0002_add_investigation_status_column.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0002_add_missing_tables.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0003_add_canonical_value_and_entity_links.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0004_add_page_posted_at.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0005_add_extraction_method.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0006_add_monitor_alerts.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0007_add_actor_style_profiles.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0008_add_users_table.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0009_add_investigation_id_to_relationships.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0010_add_composite_index_entity_relationships.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0011_add_page_extraction_cache.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0013_add_graph_status.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0015_add_progress_fields.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0016_backfill_graph_status.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0017_add_user_api_keys.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0018_add_user_id_to_investigations.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0019_add_content_safety_log.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0020_add_entity_source_tracking.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0021_add_search_engine_stats.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0022_add_actor_profiles.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0023_add_investigation_metadata.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0024_add_investigation_step_metrics.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/migrations/versions/0025_add_cancellation_requested.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/models.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/queries.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/db/search_engine_stats.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/export/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/export/_entity_loading.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/export/sigma.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/export/snort_export.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/export/yara_export.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/extractor/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/extractor/confidence.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/extractor/entity_shape.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/extractor/gazetteer.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/extractor/identity.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/extractor/llm_extract.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/extractor/ner.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/extractor/normalizer.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/extractor/pipeline.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/extractor/regex_patterns.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/extractor/relationship_extract.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/fingerprint/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/fingerprint/calibration.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/fingerprint/profiler.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/fingerprint/stylometry.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/graph/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/graph/export.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/graph/model.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/graph/queries.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/graph/visualize.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/i18n/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/i18n/detect.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/i18n/query_expand.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/i18n/translate.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/monitor/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/monitor/_db.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/monitor/alerts.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/monitor/config.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/monitor/diff.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/monitor/jobs.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/monitor/scheduler.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/scraper/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/scraper/scrape.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/scraper/scrape_js.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/search/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/search/circuit_breaker.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/search/engine_catalog.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/search/query_builder.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/search/search.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/setup.cfg +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/actor_profiles.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/blockchain.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/breach_lookup.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/cache.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/cisa.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/data/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/data/onion_seeds.json +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/dns_enrichment.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/domain_reputation.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/email_reputation.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/engines.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/enrichment.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/github_scraper.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/gitlab_scraper.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/hash_reputation.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/historical_intel.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/infostealer.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/ip_reputation.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/nvd.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/paste_scraper.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/pastes.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/proxy_client.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/rss_scraper.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/seed_manager.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/seeds.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/shodan.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/telegram.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/sources/virustotal.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_analysis_opsec.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_analysis_stylometry.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_analysis_temporal.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_api.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_api_monitors.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_blockchain.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_breach_lookup.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_cli_proxy_config.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_confidence_calibration.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_consolidated_contracts.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_crawler.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_db.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_dns_enrichment.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_domain_reputation.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_email_reputation.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_extractor_llm_pipeline.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_filter_e2e_manual.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_filter_parser_manual.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_fingerprint.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_github_scraper.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_gitlab_scraper.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_graph.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_graph_integration.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_hash_reputation.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_i18n.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_infostealer.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_ip_reputation.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_llm_utils.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_model_singleton.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_monitor.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_monitor_integration.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_nvd.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_pagination.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_paste_scraper.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_proxy_client.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_ransomlook.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_regex_patterns.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_relationship_extract.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_rss_scraper.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_scrape_js.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_settings.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_snort_export.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_sources.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_sources_enrichment_new.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_stix_export.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_vector.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/tests/test_yara_export.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/utils/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/utils/async_utils.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/utils/content_dedup.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/utils/content_safety.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/utils/defang.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/utils/encryption.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/utils/enrichment_cache.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/utils/investigation_metrics.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/utils/ioc_freshness.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/utils/user_keys.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/vector/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/vector/embedder.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/vector/search.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/vector/store.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess/llm_utils.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess.egg-info/dependency_links.txt +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess.egg-info/entry_points.txt +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess.egg-info/requires.txt +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess.egg-info/top_level.txt +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess_cli/adapters/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess_cli/adapters/sqlite.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess_cli/browser.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess_cli/commands/__init__.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess_cli/commands/actors.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess_cli/commands/enrich.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess_cli/display.py +0 -0
- {voidaccess-1.9.0 → voidaccess-1.9.2}/voidaccess_cli/tor_detect.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: voidaccess
|
|
3
|
-
Version: 1.9.
|
|
3
|
+
Version: 1.9.2
|
|
4
4
|
Summary: Dark web OSINT CLI — automated threat intelligence from query to report
|
|
5
5
|
Author: VoidAccess
|
|
6
6
|
License-Expression: MIT
|
|
@@ -172,6 +172,17 @@ Full pipeline behavior, timeouts, recovery, and data flow are documented in [Arc
|
|
|
172
172
|
|
|
173
173
|
Sources that need API keys skip cleanly when their keys are absent. The complete key and configuration reference is in [Architecture](docs/architecture.md#13-configuration-reference).
|
|
174
174
|
|
|
175
|
+
### Optional vector embeddings
|
|
176
|
+
|
|
177
|
+
The default installation can run without PyTorch. When the embedding stack is
|
|
178
|
+
unavailable, VoidAccess logs that it is using a deterministic SHA-256 fallback
|
|
179
|
+
encoder. Install the optional NLP dependencies to enable full
|
|
180
|
+
sentence-transformer vector embeddings:
|
|
181
|
+
|
|
182
|
+
```bash
|
|
183
|
+
pip install "voidaccess[nlp]"
|
|
184
|
+
```
|
|
185
|
+
|
|
175
186
|
## LLM Providers
|
|
176
187
|
|
|
177
188
|
| Provider | Typical models | Notes |
|
|
@@ -87,6 +87,17 @@ Full pipeline behavior, timeouts, recovery, and data flow are documented in [Arc
|
|
|
87
87
|
|
|
88
88
|
Sources that need API keys skip cleanly when their keys are absent. The complete key and configuration reference is in [Architecture](docs/architecture.md#13-configuration-reference).
|
|
89
89
|
|
|
90
|
+
### Optional vector embeddings
|
|
91
|
+
|
|
92
|
+
The default installation can run without PyTorch. When the embedding stack is
|
|
93
|
+
unavailable, VoidAccess logs that it is using a deterministic SHA-256 fallback
|
|
94
|
+
encoder. Install the optional NLP dependencies to enable full
|
|
95
|
+
sentence-transformer vector embeddings:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
pip install "voidaccess[nlp]"
|
|
99
|
+
```
|
|
100
|
+
|
|
90
101
|
## LLM Providers
|
|
91
102
|
|
|
92
103
|
| Provider | Typical models | Notes |
|
|
@@ -5,7 +5,7 @@ from dotenv import load_dotenv
|
|
|
5
5
|
load_dotenv()
|
|
6
6
|
|
|
7
7
|
logger = logging.getLogger(__name__)
|
|
8
|
-
|
|
8
|
+
_OPTIONAL_CONFIG_WARNING_STATE = "_voidaccess_optional_config_warning_shown"
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
def _clean_env(name, default=None):
|
|
@@ -151,16 +151,12 @@ except ValueError:
|
|
|
151
151
|
BLOCKCYPHER_TOKEN = _clean_env("BLOCKCYPHER_TOKEN", "")
|
|
152
152
|
ETHERSCAN_API_KEY = _clean_env("ETHERSCAN_API_KEY", "")
|
|
153
153
|
|
|
154
|
-
# Auth — REQUIRED
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
"and set it as JWT_SECRET in your .env file. "
|
|
161
|
-
"Do NOT use a random secret — it will change on restart and invalidate all issued tokens."
|
|
162
|
-
)
|
|
163
|
-
JWT_SECRET = _jwt_secret
|
|
154
|
+
# Auth — REQUIRED when the API/auth stack is actually used. Do not validate
|
|
155
|
+
# this at import time: the CLI injects its runtime environment in its entry
|
|
156
|
+
# point before importing dependent modules, and `--help`/`--version` must work
|
|
157
|
+
# on a fresh install. `validate_config()` and the API startup path enforce it
|
|
158
|
+
# at the point where authentication is needed.
|
|
159
|
+
JWT_SECRET = _clean_env("JWT_SECRET")
|
|
164
160
|
|
|
165
161
|
# Token blacklist Redis (optional — omit to disable blacklist checks)
|
|
166
162
|
REDIS_URL = _clean_env("REDIS_URL")
|
|
@@ -210,7 +206,6 @@ OPTIONAL_KEYS = [
|
|
|
210
206
|
|
|
211
207
|
|
|
212
208
|
def validate_config():
|
|
213
|
-
global _OPTIONAL_CONFIG_WARNING_SHOWN
|
|
214
209
|
missing_required = []
|
|
215
210
|
for key in REQUIRED_KEYS:
|
|
216
211
|
if _clean_env(key) is None:
|
|
@@ -221,16 +216,14 @@ def validate_config():
|
|
|
221
216
|
key for key in OPTIONAL_KEYS
|
|
222
217
|
if (_clean_env(key) is None or _clean_env(key) == "")
|
|
223
218
|
]
|
|
224
|
-
if missing_optional and not
|
|
219
|
+
if missing_optional and not getattr(logging, _OPTIONAL_CONFIG_WARNING_STATE, False):
|
|
225
220
|
logger.warning(
|
|
226
221
|
"Optional configuration keys not set (%d): %s - related features will be disabled",
|
|
227
222
|
len(missing_optional),
|
|
228
223
|
", ".join(missing_optional),
|
|
229
224
|
)
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
validate_config()
|
|
234
|
-
|
|
225
|
+
# Store the guard on the process-wide logging module so compatibility
|
|
226
|
+
# wrappers/reloads cannot reset it and reintroduce warning spam.
|
|
227
|
+
setattr(logging, _OPTIONAL_CONFIG_WARNING_STATE, True)
|
|
235
228
|
|
|
236
229
|
|
|
@@ -38,6 +38,7 @@ Usage (testing — pass an explicit URL to avoid needing DATABASE_URL in env)
|
|
|
38
38
|
from __future__ import annotations
|
|
39
39
|
|
|
40
40
|
from functools import lru_cache
|
|
41
|
+
import os
|
|
41
42
|
|
|
42
43
|
from contextlib import asynccontextmanager, contextmanager
|
|
43
44
|
from typing import AsyncGenerator, Generator, Optional
|
|
@@ -48,7 +49,19 @@ from sqlalchemy.orm import sessionmaker, Session
|
|
|
48
49
|
|
|
49
50
|
import config
|
|
50
51
|
|
|
52
|
+
# Kept as a compatibility/testing hook. Production lookups below use the
|
|
53
|
+
# current environment so apply_env() can run before the first DB operation.
|
|
51
54
|
DATABASE_URL = config.DATABASE_URL
|
|
55
|
+
_INITIAL_DATABASE_URL = DATABASE_URL
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _database_url() -> Optional[str]:
|
|
59
|
+
# Preserve the long-standing test/embedding hook of monkeypatching this
|
|
60
|
+
# module's DATABASE_URL. Otherwise, read the live environment so a caller
|
|
61
|
+
# can inject DATABASE_URL after db.session has been imported.
|
|
62
|
+
if DATABASE_URL != _INITIAL_DATABASE_URL:
|
|
63
|
+
return DATABASE_URL
|
|
64
|
+
return os.getenv("DATABASE_URL") or DATABASE_URL
|
|
52
65
|
_async_engine_cache: dict[str, "AsyncEngine"] = {}
|
|
53
66
|
|
|
54
67
|
|
|
@@ -104,7 +117,7 @@ def get_engine(url: Optional[str] = None) -> Engine:
|
|
|
104
117
|
PostgreSQL gets a connection pool tuned for the scraping workload.
|
|
105
118
|
SQLite skips pool parameters that only apply to QueuePool.
|
|
106
119
|
"""
|
|
107
|
-
target_url = url or
|
|
120
|
+
target_url = url or _database_url()
|
|
108
121
|
if not target_url:
|
|
109
122
|
raise RuntimeError(
|
|
110
123
|
"DATABASE_URL is not configured.\n"
|
|
@@ -122,7 +135,7 @@ def release_engine(url: Optional[str] = None) -> None:
|
|
|
122
135
|
Calls engine.dispose() to release connection pool resources and file handles,
|
|
123
136
|
then clears the cache. Use this in test teardown to prevent leaks.
|
|
124
137
|
"""
|
|
125
|
-
target_url = url or
|
|
138
|
+
target_url = url or _database_url()
|
|
126
139
|
if target_url:
|
|
127
140
|
try:
|
|
128
141
|
engine = get_engine(target_url)
|
|
@@ -139,7 +152,7 @@ def get_async_engine(url: Optional[str] = None) -> "AsyncEngine":
|
|
|
139
152
|
Converts postgresql:// to postgresql+asyncpg:// and sqlite:// to sqlite+aiosqlite://.
|
|
140
153
|
"""
|
|
141
154
|
|
|
142
|
-
target_url = url or
|
|
155
|
+
target_url = url or _database_url()
|
|
143
156
|
if not target_url:
|
|
144
157
|
raise RuntimeError(
|
|
145
158
|
"DATABASE_URL is not configured.\n"
|
|
@@ -185,7 +198,7 @@ def release_async_engine(url: Optional[str] = None) -> None:
|
|
|
185
198
|
|
|
186
199
|
Calls engine.dispose() to release connection pool resources and file handles.
|
|
187
200
|
"""
|
|
188
|
-
target_url = url or
|
|
201
|
+
target_url = url or _database_url()
|
|
189
202
|
if target_url in _async_engine_cache:
|
|
190
203
|
_async_engine_cache[target_url].dispose()
|
|
191
204
|
del _async_engine_cache[target_url]
|
|
@@ -57,7 +57,7 @@ logger = logging.getLogger(__name__)
|
|
|
57
57
|
# ---------------------------------------------------------------------------
|
|
58
58
|
|
|
59
59
|
PACKAGE_FORMAT = "voidaccess-ioc-v1"
|
|
60
|
-
PACKAGE_VERSION = "1.9.
|
|
60
|
+
PACKAGE_VERSION = "1.9.2"
|
|
61
61
|
SNORT_SID_BASE = 9000001 # 9xxxxxx reserved for VoidAccess-generated rules
|
|
62
62
|
|
|
63
63
|
# Map extractor upper-case entity types to the per-file bucket they belong to.
|
|
@@ -16,6 +16,7 @@ from __future__ import annotations
|
|
|
16
16
|
import json
|
|
17
17
|
import logging
|
|
18
18
|
import os
|
|
19
|
+
import uuid
|
|
19
20
|
from datetime import datetime, timezone
|
|
20
21
|
from typing import Any, Optional
|
|
21
22
|
|
|
@@ -76,6 +77,11 @@ _MISP_ATTR_MAP: dict[str, dict] = {
|
|
|
76
77
|
"category": "Attribution",
|
|
77
78
|
"to_ids": False,
|
|
78
79
|
},
|
|
80
|
+
"ORGANIZATION_NAME": {
|
|
81
|
+
"type": "target-org",
|
|
82
|
+
"category": "Targeting data",
|
|
83
|
+
"to_ids": False,
|
|
84
|
+
},
|
|
79
85
|
}
|
|
80
86
|
|
|
81
87
|
|
|
@@ -94,7 +100,7 @@ def investigation_to_misp_event(
|
|
|
94
100
|
Returns a valid (but empty-attribute) event if the investigation is not found.
|
|
95
101
|
Never raises.
|
|
96
102
|
"""
|
|
97
|
-
investigation,
|
|
103
|
+
investigation, entity_rows = _load_investigation_and_entity_rows(
|
|
98
104
|
investigation_id, entity_ids=entity_ids
|
|
99
105
|
)
|
|
100
106
|
|
|
@@ -110,7 +116,14 @@ def investigation_to_misp_event(
|
|
|
110
116
|
query = getattr(investigation, "query", "") or ""
|
|
111
117
|
|
|
112
118
|
attributes: list[dict] = []
|
|
113
|
-
|
|
119
|
+
graph_endpoint_map: dict[str, str] = {}
|
|
120
|
+
objects: list[dict] = []
|
|
121
|
+
|
|
122
|
+
from extractor.identity import entity_graph_id # noqa: PLC0415
|
|
123
|
+
from export._entity_loading import normalized_entity_from_db_row # noqa: PLC0415
|
|
124
|
+
|
|
125
|
+
for row in entity_rows:
|
|
126
|
+
entity = normalized_entity_from_db_row(row)
|
|
114
127
|
mapping = _MISP_ATTR_MAP.get(entity.entity_type)
|
|
115
128
|
if mapping is None:
|
|
116
129
|
continue
|
|
@@ -123,6 +136,40 @@ def investigation_to_misp_event(
|
|
|
123
136
|
}
|
|
124
137
|
attributes.append(attr)
|
|
125
138
|
|
|
139
|
+
object_uuid = str(row.id)
|
|
140
|
+
graph_endpoint_map[entity_graph_id(entity)] = object_uuid
|
|
141
|
+
objects.append(
|
|
142
|
+
{
|
|
143
|
+
"name": "voidaccess-entity",
|
|
144
|
+
"meta-category": "threat-intelligence",
|
|
145
|
+
"uuid": object_uuid,
|
|
146
|
+
"Attribute": [dict(attr, uuid=str(uuid.uuid4()))],
|
|
147
|
+
"ObjectReference": [],
|
|
148
|
+
}
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
# Use the same graph relationship enumeration as STIX. This includes
|
|
152
|
+
# typed persisted edges and applies the same endpoint/deduplication rules.
|
|
153
|
+
from export.relationships import load_export_relationships # noqa: PLC0415
|
|
154
|
+
graph_relationships, relationship_warning = load_export_relationships(
|
|
155
|
+
investigation_id, graph_endpoint_map
|
|
156
|
+
)
|
|
157
|
+
if relationship_warning:
|
|
158
|
+
logger.warning("MISP relationships were not included: %s", relationship_warning)
|
|
159
|
+
|
|
160
|
+
object_by_uuid = {obj["uuid"]: obj for obj in objects}
|
|
161
|
+
for relationship in graph_relationships:
|
|
162
|
+
source_object = object_by_uuid.get(relationship["source_ref"])
|
|
163
|
+
if source_object is None or relationship["target_ref"] not in object_by_uuid:
|
|
164
|
+
continue
|
|
165
|
+
source_object["ObjectReference"].append(
|
|
166
|
+
{
|
|
167
|
+
"object_uuid": relationship["source_ref"],
|
|
168
|
+
"referenced_uuid": relationship["target_ref"],
|
|
169
|
+
"relationship_type": relationship["edge_type"],
|
|
170
|
+
}
|
|
171
|
+
)
|
|
172
|
+
|
|
126
173
|
return {
|
|
127
174
|
"Event": {
|
|
128
175
|
"info": f"VoidAccess Investigation: {query}",
|
|
@@ -131,6 +178,7 @@ def investigation_to_misp_event(
|
|
|
131
178
|
"analysis": "2", # Completed
|
|
132
179
|
"distribution": "0", # Your organisation only
|
|
133
180
|
"Attribute": attributes,
|
|
181
|
+
"Object": objects,
|
|
134
182
|
}
|
|
135
183
|
}
|
|
136
184
|
|
|
@@ -163,8 +211,21 @@ def _load_investigation_and_entities(
|
|
|
163
211
|
|
|
164
212
|
Returns (investigation, entities) or (None, []) on error / not found.
|
|
165
213
|
"""
|
|
166
|
-
|
|
214
|
+
investigation, rows = _load_investigation_and_entity_rows(
|
|
215
|
+
investigation_id, entity_ids=entity_ids
|
|
216
|
+
)
|
|
217
|
+
if investigation is None:
|
|
218
|
+
return None, []
|
|
167
219
|
|
|
220
|
+
from export._entity_loading import normalized_entity_from_db_row # noqa: PLC0415
|
|
221
|
+
return investigation, [normalized_entity_from_db_row(row) for row in rows]
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _load_investigation_and_entity_rows(
|
|
225
|
+
investigation_id: Any,
|
|
226
|
+
entity_ids: Optional[list[str]] = None,
|
|
227
|
+
):
|
|
228
|
+
"""Load an investigation and its ORM entity rows for export rendering."""
|
|
168
229
|
if not os.getenv("DATABASE_URL"):
|
|
169
230
|
return None, []
|
|
170
231
|
|
|
@@ -172,7 +233,7 @@ def _load_investigation_and_entities(
|
|
|
172
233
|
from db.session import get_session # noqa: PLC0415
|
|
173
234
|
from db.queries import get_investigation_by_id_or_run # noqa: PLC0415
|
|
174
235
|
from db.models import Entity, InvestigationEntityLink # noqa: PLC0415
|
|
175
|
-
from
|
|
236
|
+
from sqlalchemy.orm import joinedload # noqa: PLC0415
|
|
176
237
|
|
|
177
238
|
inv_uuid = _coerce_uuid(investigation_id)
|
|
178
239
|
if inv_uuid is None:
|
|
@@ -192,15 +253,16 @@ def _load_investigation_and_entities(
|
|
|
192
253
|
if investigation is None:
|
|
193
254
|
return None, []
|
|
194
255
|
|
|
195
|
-
linked_ids_select = (
|
|
196
|
-
session.query(InvestigationEntityLink.entity_id)
|
|
197
|
-
.filter(InvestigationEntityLink.investigation_id == investigation.id)
|
|
198
|
-
)
|
|
199
256
|
db_entities = (
|
|
200
257
|
session.query(Entity)
|
|
258
|
+
.outerjoin(
|
|
259
|
+
InvestigationEntityLink,
|
|
260
|
+
InvestigationEntityLink.entity_id == Entity.id,
|
|
261
|
+
)
|
|
262
|
+
.options(joinedload(Entity.page))
|
|
201
263
|
.filter(
|
|
202
264
|
(Entity.investigation_id == investigation.id)
|
|
203
|
-
|
|
|
265
|
+
| (InvestigationEntityLink.investigation_id == investigation.id)
|
|
204
266
|
)
|
|
205
267
|
.all()
|
|
206
268
|
)
|
|
@@ -209,17 +271,8 @@ def _load_investigation_and_entities(
|
|
|
209
271
|
want = frozenset(filter_uuids)
|
|
210
272
|
db_entities = [e for e in db_entities if e.id in want]
|
|
211
273
|
|
|
212
|
-
from export._entity_loading import normalized_entity_from_db_row # noqa: PLC0415
|
|
213
|
-
|
|
214
|
-
# Shared row→NormalizedEntity mapping (same one STIX/Sigma use), so
|
|
215
|
-
# auxiliary fields (source_quality, extraction_method) no longer
|
|
216
|
-
# differ between export paths.
|
|
217
|
-
normalized: list[NormalizedEntity] = [
|
|
218
|
-
normalized_entity_from_db_row(e) for e in db_entities
|
|
219
|
-
]
|
|
220
|
-
|
|
221
274
|
session.expunge_all()
|
|
222
|
-
return investigation,
|
|
275
|
+
return investigation, db_entities
|
|
223
276
|
|
|
224
277
|
except Exception as exc:
|
|
225
278
|
logger.warning("_load_investigation_and_entities failed: %s", exc)
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Shared relationship loading for structured exporters.
|
|
2
|
+
|
|
3
|
+
The graph builder is the canonical source for both persisted typed edges and
|
|
4
|
+
the bounded co-occurrence edges used by exports. Keeping the enumeration in
|
|
5
|
+
one place prevents one format from silently losing relationships that another
|
|
6
|
+
format preserves.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import logging
|
|
12
|
+
from typing import Any, Optional
|
|
13
|
+
import uuid
|
|
14
|
+
|
|
15
|
+
from graph.model import RELATIONSHIP_TYPE_STIX
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def load_export_relationships(
|
|
21
|
+
investigation_id: Any,
|
|
22
|
+
endpoint_map: dict[str, str],
|
|
23
|
+
) -> tuple[list[dict[str, Any]], Optional[str]]:
|
|
24
|
+
"""Return graph relationships whose endpoints exist in ``endpoint_map``.
|
|
25
|
+
|
|
26
|
+
Each result contains the original VoidAccess ``edge_type`` plus its STIX
|
|
27
|
+
spelling. Consumers can therefore share the exact same deduplication and
|
|
28
|
+
endpoint selection while rendering format-specific relationship records.
|
|
29
|
+
"""
|
|
30
|
+
try:
|
|
31
|
+
from graph.builder import build_graph_from_db # noqa: PLC0415
|
|
32
|
+
|
|
33
|
+
inv_uuid = _coerce_uuid(investigation_id)
|
|
34
|
+
graph = build_graph_from_db(investigation_id=inv_uuid)
|
|
35
|
+
relationships: list[dict[str, Any]] = []
|
|
36
|
+
seen: set[tuple[str, str, str]] = set()
|
|
37
|
+
|
|
38
|
+
for source_node, target_node, data in graph.edges(data=True):
|
|
39
|
+
source_ref = endpoint_map.get(source_node)
|
|
40
|
+
target_ref = endpoint_map.get(target_node)
|
|
41
|
+
if not source_ref or not target_ref:
|
|
42
|
+
continue
|
|
43
|
+
|
|
44
|
+
edge_type = _normalise_edge_type(data.get("edge_type", "related-to"))
|
|
45
|
+
stix_type = RELATIONSHIP_TYPE_STIX.get(edge_type, "related-to")
|
|
46
|
+
key = (stix_type, source_ref, target_ref)
|
|
47
|
+
if key in seen:
|
|
48
|
+
continue
|
|
49
|
+
seen.add(key)
|
|
50
|
+
relationships.append(
|
|
51
|
+
{
|
|
52
|
+
"source_ref": source_ref,
|
|
53
|
+
"target_ref": target_ref,
|
|
54
|
+
"edge_type": edge_type,
|
|
55
|
+
"stix_relationship_type": stix_type,
|
|
56
|
+
"confidence": data.get("confidence"),
|
|
57
|
+
}
|
|
58
|
+
)
|
|
59
|
+
return relationships, None
|
|
60
|
+
except Exception as exc:
|
|
61
|
+
logger.warning("load_export_relationships failed: %s", exc)
|
|
62
|
+
return [], f"load_export_relationships failed: {exc}"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _normalise_edge_type(edge_type: Any) -> str:
|
|
66
|
+
"""Normalise enum/string edge types without changing their meaning."""
|
|
67
|
+
value = getattr(edge_type, "value", edge_type)
|
|
68
|
+
text = str(value or "related-to").strip()
|
|
69
|
+
upper = text.upper()
|
|
70
|
+
if upper in RELATIONSHIP_TYPE_STIX:
|
|
71
|
+
return upper
|
|
72
|
+
return text
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _coerce_uuid(value: Any) -> Optional[uuid.UUID]:
|
|
76
|
+
if isinstance(value, uuid.UUID):
|
|
77
|
+
return value
|
|
78
|
+
try:
|
|
79
|
+
return uuid.UUID(str(value))
|
|
80
|
+
except (ValueError, AttributeError):
|
|
81
|
+
return None
|
|
@@ -22,7 +22,6 @@ import warnings
|
|
|
22
22
|
from typing import Any, Optional
|
|
23
23
|
import uuid
|
|
24
24
|
|
|
25
|
-
from graph.model import RELATIONSHIP_TYPE_STIX
|
|
26
25
|
from extractor.confidence import get_entity_confidence
|
|
27
26
|
|
|
28
27
|
logger = logging.getLogger(__name__)
|
|
@@ -161,6 +160,11 @@ def entity_to_stix_malware(entity: Any) -> Optional[Any]:
|
|
|
161
160
|
malware = stix2.Malware(
|
|
162
161
|
name=entity.value,
|
|
163
162
|
is_family=True,
|
|
163
|
+
malware_types=(
|
|
164
|
+
["ransomware"]
|
|
165
|
+
if entity.entity_type == "RANSOMWARE_GROUP"
|
|
166
|
+
else ["malware"]
|
|
167
|
+
),
|
|
164
168
|
confidence=_to_stix_confidence(get_entity_confidence(entity)),
|
|
165
169
|
allow_custom=True,
|
|
166
170
|
x_voidaccess_source_quality=getattr(entity, "source_quality", 1.0),
|
|
@@ -176,6 +180,28 @@ def entity_to_stix_malware(entity: Any) -> Optional[Any]:
|
|
|
176
180
|
return None
|
|
177
181
|
|
|
178
182
|
|
|
183
|
+
def entity_to_stix_identity(entity: Any) -> Optional[Any]:
|
|
184
|
+
"""Convert an organization entity to a STIX Identity object."""
|
|
185
|
+
if not _STIX2_AVAILABLE or entity.entity_type != "ORGANIZATION_NAME":
|
|
186
|
+
return None
|
|
187
|
+
try:
|
|
188
|
+
return stix2.Identity(
|
|
189
|
+
name=entity.value,
|
|
190
|
+
identity_class="organization",
|
|
191
|
+
confidence=_to_stix_confidence(get_entity_confidence(entity)),
|
|
192
|
+
allow_custom=True,
|
|
193
|
+
x_voidaccess_source_quality=getattr(entity, "source_quality", 1.0),
|
|
194
|
+
external_references=(
|
|
195
|
+
[{"source_name": "voidaccess", "url": entity.source_url}]
|
|
196
|
+
if entity.source_url
|
|
197
|
+
else []
|
|
198
|
+
),
|
|
199
|
+
)
|
|
200
|
+
except Exception as exc:
|
|
201
|
+
logger.warning("entity_to_stix_identity failed for %r: %s", entity.value, exc)
|
|
202
|
+
return None
|
|
203
|
+
|
|
204
|
+
|
|
179
205
|
def entity_to_stix_threat_actor(entity: Any) -> Optional[Any]:
|
|
180
206
|
"""
|
|
181
207
|
Convert a THREAT_ACTOR_HANDLE entity to a STIX 2.1 ThreatActor object.
|
|
@@ -287,6 +313,11 @@ def investigation_to_stix_bundle(
|
|
|
287
313
|
stix_objects.append(actor)
|
|
288
314
|
_register(entity, actor.id)
|
|
289
315
|
|
|
316
|
+
identity = entity_to_stix_identity(entity)
|
|
317
|
+
if identity:
|
|
318
|
+
stix_objects.append(identity)
|
|
319
|
+
_register(entity, identity.id)
|
|
320
|
+
|
|
290
321
|
if include_relationships and stix_objects:
|
|
291
322
|
relationships, relationship_warning = _build_stix_relationships(
|
|
292
323
|
investigation_id, stix_id_map
|
|
@@ -436,32 +467,22 @@ def _build_stix_relationships(
|
|
|
436
467
|
if not _STIX2_AVAILABLE:
|
|
437
468
|
return [], "stix2 is not installed"
|
|
438
469
|
try:
|
|
439
|
-
from
|
|
440
|
-
|
|
441
|
-
inv_uuid = _coerce_uuid(investigation_id)
|
|
442
|
-
graph = build_graph_from_db(investigation_id=inv_uuid)
|
|
470
|
+
from export.relationships import load_export_relationships # noqa: PLC0415
|
|
443
471
|
|
|
472
|
+
graph_relationships, warning = load_export_relationships(
|
|
473
|
+
investigation_id, stix_id_map
|
|
474
|
+
)
|
|
475
|
+
if warning:
|
|
476
|
+
return [], warning
|
|
444
477
|
relationships: list[Any] = []
|
|
445
|
-
|
|
446
|
-
for source_node, target_node, data in graph.edges(data=True):
|
|
447
|
-
src_stix_id = stix_id_map.get(source_node)
|
|
448
|
-
tgt_stix_id = stix_id_map.get(target_node)
|
|
449
|
-
if not src_stix_id or not tgt_stix_id:
|
|
450
|
-
continue
|
|
451
|
-
edge_type = data.get("edge_type", "related-to")
|
|
452
|
-
# Map VoidAccess edge types to STIX relationship types
|
|
453
|
-
rel_type = _edge_type_to_stix(edge_type)
|
|
454
|
-
relationship_key = (rel_type, src_stix_id, tgt_stix_id)
|
|
455
|
-
if relationship_key in seen_relationships:
|
|
456
|
-
continue
|
|
457
|
-
seen_relationships.add(relationship_key)
|
|
478
|
+
for data in graph_relationships:
|
|
458
479
|
# Carry the edge's own confidence onto the SRO. Typed relationships
|
|
459
480
|
# have a claim-specific confidence distinct from co-occurrence's
|
|
460
481
|
# flat value; STIX 2.1 confidence is an int 0-100.
|
|
461
482
|
rel_kwargs: dict[str, Any] = {
|
|
462
|
-
"relationship_type":
|
|
463
|
-
"source_ref":
|
|
464
|
-
"target_ref":
|
|
483
|
+
"relationship_type": data["stix_relationship_type"],
|
|
484
|
+
"source_ref": data["source_ref"],
|
|
485
|
+
"target_ref": data["target_ref"],
|
|
465
486
|
"allow_custom": True,
|
|
466
487
|
}
|
|
467
488
|
try:
|
|
@@ -488,7 +509,9 @@ def _build_stix_relationships(
|
|
|
488
509
|
|
|
489
510
|
def _edge_type_to_stix(edge_type: str) -> str:
|
|
490
511
|
"""Map VoidAccess graph edge types to STIX relationship type strings."""
|
|
491
|
-
|
|
512
|
+
from export.relationships import _normalise_edge_type # noqa: PLC0415
|
|
513
|
+
from graph.model import RELATIONSHIP_TYPE_STIX # noqa: PLC0415
|
|
514
|
+
return RELATIONSHIP_TYPE_STIX.get(_normalise_edge_type(edge_type), "related-to")
|
|
492
515
|
|
|
493
516
|
|
|
494
517
|
def _coerce_uuid(value: Any) -> Optional[uuid.UUID]:
|
|
@@ -789,8 +789,12 @@ def build_graph_from_db(
|
|
|
789
789
|
.join(
|
|
790
790
|
InvestigationEntityLink,
|
|
791
791
|
InvestigationEntityLink.entity_id == Entity.id,
|
|
792
|
+
isouter=True,
|
|
793
|
+
)
|
|
794
|
+
.filter(
|
|
795
|
+
(Entity.investigation_id == investigation_id)
|
|
796
|
+
| (InvestigationEntityLink.investigation_id == investigation_id)
|
|
792
797
|
)
|
|
793
|
-
.filter(InvestigationEntityLink.investigation_id == investigation_id)
|
|
794
798
|
.options(joinedload(Entity.page))
|
|
795
799
|
)
|
|
796
800
|
else:
|
|
@@ -1230,8 +1234,12 @@ def build_graph_from_db_cached(investigation_id: uuid.UUID) -> nx.MultiDiGraph:
|
|
|
1230
1234
|
.join(
|
|
1231
1235
|
InvestigationEntityLink,
|
|
1232
1236
|
InvestigationEntityLink.entity_id == Entity.id,
|
|
1237
|
+
isouter=True,
|
|
1238
|
+
)
|
|
1239
|
+
.filter(
|
|
1240
|
+
(Entity.investigation_id == investigation_id)
|
|
1241
|
+
| (InvestigationEntityLink.investigation_id == investigation_id)
|
|
1233
1242
|
)
|
|
1234
|
-
.filter(InvestigationEntityLink.investigation_id == investigation_id)
|
|
1235
1243
|
.options(joinedload(Entity.page))
|
|
1236
1244
|
.yield_per(500)
|
|
1237
1245
|
)
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import os
|
|
4
4
|
import sys
|
|
5
5
|
import unittest
|
|
6
|
+
import logging
|
|
6
7
|
from unittest.mock import patch, MagicMock
|
|
7
8
|
|
|
8
9
|
|
|
@@ -52,6 +53,28 @@ class TestConfigValidation(unittest.TestCase):
|
|
|
52
53
|
warning_calls = str(mock_logger.warning.call_args_list)
|
|
53
54
|
self.assertIn("OPENAI_API_KEY", warning_calls)
|
|
54
55
|
|
|
56
|
+
def test_optional_config_warning_is_once_per_process(self):
|
|
57
|
+
"""Repeated validation must not spam the same process with warnings."""
|
|
58
|
+
import config
|
|
59
|
+
|
|
60
|
+
with patch.dict(os.environ, {"JWT_SECRET": "test-secret-key-123"}, clear=True):
|
|
61
|
+
# The guard intentionally lives on logging so compatibility-module
|
|
62
|
+
# reloads cannot reset it; reset it explicitly for this isolated
|
|
63
|
+
# regression test.
|
|
64
|
+
setattr(logging, "_voidaccess_optional_config_warning_shown", False)
|
|
65
|
+
try:
|
|
66
|
+
with patch.object(config, "logger") as mock_logger:
|
|
67
|
+
config.validate_config()
|
|
68
|
+
config.validate_config()
|
|
69
|
+
|
|
70
|
+
warning_calls = [
|
|
71
|
+
call for call in mock_logger.warning.call_args_list
|
|
72
|
+
if call.args and "Optional configuration keys not set" in str(call.args[0])
|
|
73
|
+
]
|
|
74
|
+
self.assertEqual(len(warning_calls), 1)
|
|
75
|
+
finally:
|
|
76
|
+
setattr(logging, "_voidaccess_optional_config_warning_shown", False)
|
|
77
|
+
|
|
55
78
|
|
|
56
79
|
if __name__ == "__main__":
|
|
57
|
-
unittest.main()
|
|
80
|
+
unittest.main()
|
|
@@ -19,6 +19,19 @@ def test_select_relevant_pages_empty():
|
|
|
19
19
|
assert result == []
|
|
20
20
|
|
|
21
21
|
|
|
22
|
+
def test_get_llm_uses_runtime_provider_key_when_api_keys_omitted(monkeypatch):
|
|
23
|
+
"""A key injected after module import must reach the LangChain client."""
|
|
24
|
+
from unittest.mock import patch
|
|
25
|
+
from langchain_openai import ChatOpenAI
|
|
26
|
+
from voidaccess.llm import get_llm
|
|
27
|
+
|
|
28
|
+
monkeypatch.setenv("OPENROUTER_API_KEY", "runtime-openrouter-key")
|
|
29
|
+
with patch.object(ChatOpenAI, "__init__", return_value=None) as init:
|
|
30
|
+
get_llm("openrouter/test-model")
|
|
31
|
+
|
|
32
|
+
assert init.call_args.kwargs["api_key"] == "runtime-openrouter-key"
|
|
33
|
+
|
|
34
|
+
|
|
22
35
|
def test_select_relevant_pages_fits_budget():
|
|
23
36
|
"""Pages that fit in budget are all returned."""
|
|
24
37
|
pages = [make_page("a" * 100, f"http://{i}.onion") for i in range(5)]
|
|
@@ -82,6 +82,11 @@ def get_embedding_model() -> "SentenceTransformer | None":
|
|
|
82
82
|
logger.info("Loaded embedding model all-MiniLM-L6-v2 (singleton)")
|
|
83
83
|
except Exception as exc:
|
|
84
84
|
logger.warning("Failed to load embedding model: %s", exc)
|
|
85
|
+
logger.warning(
|
|
86
|
+
"Deterministic embedding fallback is active; install "
|
|
87
|
+
"voidaccess[nlp] (torch + sentence-transformers) for full "
|
|
88
|
+
"vector embeddings."
|
|
89
|
+
)
|
|
85
90
|
_model = _FallbackEmbeddingModel()
|
|
86
91
|
|
|
87
92
|
return _model
|
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
from importlib import import_module, reload
|
|
4
4
|
|
|
5
|
+
# Reload the compatibility target so explicit `importlib.reload` calls keep
|
|
6
|
+
# reflecting a test/application environment change. The warning guard itself
|
|
7
|
+
# lives on the process-wide logging module, so this reload cannot reintroduce
|
|
8
|
+
# warning spam.
|
|
5
9
|
_config = reload(import_module("config"))
|
|
6
10
|
|
|
7
11
|
for _name in dir(_config):
|