voidaccess 1.8.0__tar.gz → 1.8.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {voidaccess-1.8.0/voidaccess.egg-info → voidaccess-1.8.1}/PKG-INFO +1 -1
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/models.py +1 -1
- {voidaccess-1.8.0 → voidaccess-1.8.1}/export/ioc_package.py +1 -1
- {voidaccess-1.8.0 → voidaccess-1.8.1}/export/stix.py +1 -1
- {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/normalizer.py +29 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/relationship_extract.py +46 -10
- {voidaccess-1.8.0 → voidaccess-1.8.1}/graph/model.py +1 -1
- {voidaccess-1.8.0 → voidaccess-1.8.1}/pyproject.toml +1 -1
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/enrichment.py +40 -2
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_db.py +7 -6
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_graph.py +5 -5
- {voidaccess-1.8.0 → voidaccess-1.8.1/voidaccess.egg-info}/PKG-INFO +1 -1
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/__init__.py +1 -1
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/commands/investigate.py +85 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/LICENSE +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/README.md +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/analysis/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/analysis/opsec.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/analysis/patterns.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/analysis/temporal.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/api/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/api/auth.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/api/errors.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/api/main.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/actors.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/admin.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/auth.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/entities.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/export.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/investigations.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/monitors.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/search.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/settings.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/auth/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/auth/token_blacklist.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/config.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/crawler/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/crawler/dedup.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/crawler/frontier.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/crawler/spider.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/crawler/utils.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/env.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0001_initial_schema.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0002_add_investigation_status_column.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0002_add_missing_tables.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0003_add_canonical_value_and_entity_links.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0004_add_page_posted_at.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0005_add_extraction_method.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0006_add_monitor_alerts.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0007_add_actor_style_profiles.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0008_add_users_table.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0009_add_investigation_id_to_relationships.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0010_add_composite_index_entity_relationships.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0011_add_page_extraction_cache.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0013_add_graph_status.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0015_add_progress_fields.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0016_backfill_graph_status.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0017_add_user_api_keys.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0018_add_user_id_to_investigations.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0019_add_content_safety_log.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0020_add_entity_source_tracking.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0021_add_search_engine_stats.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0022_add_actor_profiles.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0023_add_investigation_metadata.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0024_add_investigation_step_metrics.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0025_add_cancellation_requested.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/queries.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/search_engine_stats.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/db/session.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/export/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/export/misp.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/export/sigma.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/export/snort_export.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/export/yara_export.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/confidence.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/entity_shape.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/gazetteer.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/llm_extract.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/ner.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/pipeline.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/regex_patterns.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/fingerprint/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/fingerprint/calibration.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/fingerprint/profiler.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/fingerprint/stylometry.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/graph/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/graph/builder.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/graph/export.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/graph/queries.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/graph/visualize.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/i18n/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/i18n/detect.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/i18n/query_expand.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/i18n/translate.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/monitor/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/monitor/_db.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/monitor/alerts.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/monitor/config.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/monitor/diff.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/monitor/jobs.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/monitor/scheduler.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/scraper/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/scraper/scrape.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/scraper/scrape_js.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/search/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/search/circuit_breaker.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/search/engine_catalog.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/search/query_builder.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/search/search.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/setup.cfg +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/actor_profiles.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/blockchain.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/breach_lookup.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/cache.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/cisa.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/data/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/data/onion_seeds.json +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/dns_enrichment.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/domain_reputation.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/email_reputation.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/engines.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/github_scraper.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/gitlab_scraper.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/hash_reputation.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/historical_intel.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/infostealer.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/ip_reputation.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/nvd.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/paste_scraper.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/pastes.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/proxy_client.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/rss_scraper.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/seed_manager.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/seeds.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/shodan.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/telegram.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/virustotal.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_analysis_opsec.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_analysis_stylometry.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_analysis_temporal.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_api.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_api_monitors.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_blockchain.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_breach_lookup.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_cli_proxy_config.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_confidence_calibration.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_config.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_crawler.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_dns_enrichment.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_domain_reputation.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_email_reputation.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_extractor_llm_pipeline.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_filter_e2e_manual.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_filter_parser_manual.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_fingerprint.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_github_scraper.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_gitlab_scraper.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_graph_integration.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_hash_reputation.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_i18n.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_infostealer.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_ip_reputation.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_llm.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_llm_utils.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_model_singleton.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_monitor.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_monitor_integration.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_nvd.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_pagination.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_paste_scraper.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_proxy_client.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_ransomlook.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_regex_patterns.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_rss_scraper.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_scrape_js.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_settings.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_snort_export.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_sources.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_sources_enrichment_new.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_stix_export.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_vector.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_yara_export.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/async_utils.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/content_dedup.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/content_safety.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/defang.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/encryption.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/enrichment_cache.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/investigation_metrics.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/ioc_freshness.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/user_keys.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/vector/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/vector/embedder.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/vector/model_singleton.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/vector/search.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/vector/store.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess/config.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess/llm.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess/llm_utils.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess.egg-info/SOURCES.txt +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess.egg-info/dependency_links.txt +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess.egg-info/entry_points.txt +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess.egg-info/requires.txt +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess.egg-info/top_level.txt +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/adapters/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/adapters/sqlite.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/browser.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/commands/__init__.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/commands/actors.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/commands/configure.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/commands/enrich.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/commands/export.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/commands/show.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/config.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/display.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/main.py +0 -0
- {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/tor_detect.py +0 -0
|
@@ -75,7 +75,7 @@ class RelationshipType(str, enum.Enum):
|
|
|
75
75
|
LINKED_TO = "LINKED_TO"
|
|
76
76
|
PAID_TO = "PAID_TO"
|
|
77
77
|
MEMBER_OF = "MEMBER_OF"
|
|
78
|
-
|
|
78
|
+
USES = "USES"
|
|
79
79
|
CLAIMED = "CLAIMED"
|
|
80
80
|
LIKELY_SAME_ACTOR = "LIKELY_SAME_ACTOR"
|
|
81
81
|
CONFIRMED_SAME_ACTOR = "CONFIRMED_SAME_ACTOR"
|
|
@@ -54,7 +54,7 @@ logger = logging.getLogger(__name__)
|
|
|
54
54
|
# ---------------------------------------------------------------------------
|
|
55
55
|
|
|
56
56
|
PACKAGE_FORMAT = "voidaccess-ioc-v1"
|
|
57
|
-
PACKAGE_VERSION = "1.8.
|
|
57
|
+
PACKAGE_VERSION = "1.8.1"
|
|
58
58
|
SNORT_SID_BASE = 9000001 # 9xxxxxx reserved for VoidAccess-generated rules
|
|
59
59
|
|
|
60
60
|
# Map extractor upper-case entity types to the per-file bucket they belong to.
|
|
@@ -508,7 +508,7 @@ def _edge_type_to_stix(edge_type: str) -> str:
|
|
|
508
508
|
"LINKED_TO": "related-to",
|
|
509
509
|
"PAID_TO": "related-to",
|
|
510
510
|
"MEMBER_OF": "member-of",
|
|
511
|
-
"
|
|
511
|
+
"USES": "uses",
|
|
512
512
|
"CLAIMED": "attributed-to",
|
|
513
513
|
"LIKELY_SAME_ACTOR": "related-to",
|
|
514
514
|
"CONFIRMED_SAME_ACTOR": "related-to",
|
|
@@ -887,6 +887,27 @@ _RFC5737_NETWORKS: list[tuple[str, str]] = [
|
|
|
887
887
|
("203.0.113.0", "255.255.255.0"), # 203.0.113.0/24
|
|
888
888
|
]
|
|
889
889
|
|
|
890
|
+
# Real, routable IPs that are never a meaningful indicator of compromise:
|
|
891
|
+
# the unspecified/null address, the broadcast address, and the well-known
|
|
892
|
+
# public DNS resolvers that appear constantly in technical content for
|
|
893
|
+
# reasons unrelated to being an IOC. Filtered like the RFC 5737 ranges so a
|
|
894
|
+
# user pulling ip_addresses.txt into a blocklist can't end up blocking a
|
|
895
|
+
# legitimate public resolver or a null route.
|
|
896
|
+
_NON_ACTIONABLE_IPS: frozenset[str] = frozenset({
|
|
897
|
+
"0.0.0.0", # unspecified / null route — never a real host
|
|
898
|
+
"255.255.255.255", # limited broadcast
|
|
899
|
+
"8.8.8.8", "8.8.4.4", # Google Public DNS
|
|
900
|
+
"1.1.1.1", "1.0.0.1", # Cloudflare Public DNS
|
|
901
|
+
"9.9.9.9", "149.112.112.112", # Quad9
|
|
902
|
+
"208.67.222.222", "208.67.220.220", # OpenDNS
|
|
903
|
+
"4.2.2.1", "4.2.2.2", # Level3 legacy resolvers
|
|
904
|
+
})
|
|
905
|
+
|
|
906
|
+
|
|
907
|
+
def _is_non_actionable_ip(value: str) -> bool:
|
|
908
|
+
"""Return True for real IPs that are never meaningful IOCs (null route, public DNS)."""
|
|
909
|
+
return value.strip() in _NON_ACTIONABLE_IPS
|
|
910
|
+
|
|
890
911
|
# RFC 2606 reserved domain names.
|
|
891
912
|
_RFC2606_DOMAINS: frozenset[str] = frozenset({
|
|
892
913
|
"example.com",
|
|
@@ -1563,6 +1584,14 @@ def normalize_entities(
|
|
|
1563
1584
|
)
|
|
1564
1585
|
continue
|
|
1565
1586
|
|
|
1587
|
+
if entity_type == "IP_ADDRESS" and _is_non_actionable_ip(canonical):
|
|
1588
|
+
source_quality = 0.0
|
|
1589
|
+
logger.debug(
|
|
1590
|
+
"Non-actionable IP rejected (null route / broadcast / public DNS resolver): %s",
|
|
1591
|
+
canonical,
|
|
1592
|
+
)
|
|
1593
|
+
continue
|
|
1594
|
+
|
|
1566
1595
|
if entity_type == "DOMAIN" and _is_rfc2606_domain(canonical):
|
|
1567
1596
|
source_quality = 0.0
|
|
1568
1597
|
logger.debug(
|
|
@@ -60,8 +60,8 @@ _DEFAULT_MAX_RELATIONSHIPS_PER_PAGE = 60
|
|
|
60
60
|
# EDGE_TYPES). Passive forms map to the same stored type but flip the edge
|
|
61
61
|
# direction (handled below) so the stored edge always reads source -> target.
|
|
62
62
|
_LLM_REL_VOCAB: dict[str, str] = {
|
|
63
|
-
"uses": "
|
|
64
|
-
"used": "
|
|
63
|
+
"uses": "USES",
|
|
64
|
+
"used": "USES",
|
|
65
65
|
"drops": "DROPS",
|
|
66
66
|
"controls": "CONTROLS",
|
|
67
67
|
"targets": "TARGETS",
|
|
@@ -144,17 +144,41 @@ def _parse_relationship_json(raw: str) -> list[dict]:
|
|
|
144
144
|
return rels if isinstance(rels, list) else []
|
|
145
145
|
|
|
146
146
|
|
|
147
|
+
class RelationshipLLMCallError(Exception):
|
|
148
|
+
"""Raised when the relationship-extraction LLM invocation itself fails.
|
|
149
|
+
|
|
150
|
+
Distinct from a call that *succeeds* and legitimately returns no
|
|
151
|
+
relationships — callers use this to log a visible warning so a broken LLM
|
|
152
|
+
integration can never masquerade as "the LLM found no relationships".
|
|
153
|
+
"""
|
|
154
|
+
|
|
155
|
+
|
|
147
156
|
async def _invoke_llm(prompt: str, llm) -> str:
|
|
148
|
-
"""Call the LLM with streaming/callbacks disabled (mirrors llm_extract).
|
|
149
|
-
|
|
157
|
+
"""Call the LLM with streaming/callbacks disabled (mirrors llm_extract).
|
|
158
|
+
|
|
159
|
+
Raises :class:`RelationshipLLMCallError` if the underlying invocation
|
|
160
|
+
fails, so the caller can distinguish a *failed* call from a successful
|
|
161
|
+
call that found nothing.
|
|
162
|
+
"""
|
|
163
|
+
# ``bind(callbacks=[])`` puts ``callbacks`` into the model's invocation
|
|
164
|
+
# kwargs, which LangChain forwards to ``agenerate_prompt`` as ``**kwargs``
|
|
165
|
+
# — while ``ainvoke`` ALSO passes ``callbacks`` explicitly, raising
|
|
166
|
+
# "agenerate_prompt() got multiple values for keyword argument 'callbacks'".
|
|
167
|
+
# ``with_config`` overrides callbacks for this call WITHOUT adding model
|
|
168
|
+
# kwargs, exactly as extractor/llm_extract._extract_chunk does.
|
|
169
|
+
if hasattr(llm, "with_config"):
|
|
170
|
+
silent_llm = llm.with_config({"callbacks": []})
|
|
171
|
+
else:
|
|
150
172
|
silent_llm = llm.bind(streaming=False, callbacks=[])
|
|
173
|
+
try:
|
|
174
|
+
# Guard stdout in case a legacy constructor callback still streams
|
|
175
|
+
# partial JSON tokens even with an empty per-call callback list.
|
|
151
176
|
with redirect_stdout(io.StringIO()):
|
|
152
177
|
response = await silent_llm.ainvoke(prompt)
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
return ""
|
|
178
|
+
except Exception as exc: # noqa: BLE001 — surface as a typed failure
|
|
179
|
+
raise RelationshipLLMCallError(str(exc)) from exc
|
|
180
|
+
content = response.content if hasattr(response, "content") else str(response)
|
|
181
|
+
return content.strip()
|
|
158
182
|
|
|
159
183
|
|
|
160
184
|
async def extract_relationships_for_page(
|
|
@@ -194,7 +218,19 @@ async def extract_relationships_for_page(
|
|
|
194
218
|
content = page_text[:max_chunk_chars]
|
|
195
219
|
prompt = _PROMPT_TEMPLATE.format(entities=entity_lines, content=content)
|
|
196
220
|
|
|
197
|
-
|
|
221
|
+
try:
|
|
222
|
+
raw = await _invoke_llm(prompt, llm)
|
|
223
|
+
except RelationshipLLMCallError as exc:
|
|
224
|
+
# The LLM call itself failed — this is NOT the same as the LLM running
|
|
225
|
+
# successfully and finding no relationship. Log loudly so a broken
|
|
226
|
+
# integration is visible in normal operation instead of silently
|
|
227
|
+
# degrading every page to zero typed relationships.
|
|
228
|
+
logger.warning(
|
|
229
|
+
"Relationship extraction: LLM call FAILED (%s) — this page yielded "
|
|
230
|
+
"NO typed relationships due to an error, not because none exist",
|
|
231
|
+
exc,
|
|
232
|
+
)
|
|
233
|
+
return []
|
|
198
234
|
rels = _parse_relationship_json(raw)
|
|
199
235
|
if not rels:
|
|
200
236
|
return []
|
|
@@ -77,7 +77,7 @@ class EDGE_TYPES:
|
|
|
77
77
|
POSTED_BY = "POSTED_BY" # content attributed to a handle
|
|
78
78
|
LINKED_TO = "LINKED_TO" # URL links to URL
|
|
79
79
|
MEMBER_OF = "MEMBER_OF" # handle to group/forum
|
|
80
|
-
|
|
80
|
+
USES = "USES" # actor uses a malware family
|
|
81
81
|
CLAIMED = "CLAIMED" # group claimed an attack
|
|
82
82
|
LIKELY_SAME_ACTOR = "LIKELY_SAME_ACTOR" # inferred, medium confidence
|
|
83
83
|
CONFIRMED_SAME_ACTOR = "CONFIRMED_SAME_ACTOR" # PGP key match, high confidence
|
|
@@ -565,6 +565,42 @@ def abusech_to_pages(
|
|
|
565
565
|
_RANSOMWARE_LIVE_BASE = "https://api.ransomware.live/v2"
|
|
566
566
|
_RANSOMWARE_LIVE_HEADERS = {"User-Agent": "VoidAccess-OSINT/1.0", "Accept": "application/json"}
|
|
567
567
|
|
|
568
|
+
# Generic words that carry no group-identity signal in a realistic analyst
|
|
569
|
+
# query ("LockBit ransomware leak site"). Stripped before matching so the
|
|
570
|
+
# meaningful token(s) — e.g. "lockbit" — are what we match on, rather than
|
|
571
|
+
# requiring the whole query string to appear literally in a group name.
|
|
572
|
+
_GROUP_QUERY_STOPWORDS: frozenset[str] = frozenset({
|
|
573
|
+
"ransomware", "ransom", "group", "gang", "crew", "team",
|
|
574
|
+
"leak", "leaks", "leaksite", "site", "blog", "portal",
|
|
575
|
+
"data", "the", "and", "actor", "threat", "malware",
|
|
576
|
+
})
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def _significant_group_tokens(query: str) -> list[str]:
|
|
580
|
+
"""Meaningful lowercase tokens from a group query, stopwords removed.
|
|
581
|
+
|
|
582
|
+
Falls back to all tokens if every token was a stopword, so a query made
|
|
583
|
+
entirely of generic words still matches on something rather than nothing.
|
|
584
|
+
"""
|
|
585
|
+
tokens = re.findall(r"[a-z0-9.]+", (query or "").lower())
|
|
586
|
+
tokens = [t.strip(".") for t in tokens if t.strip(".")]
|
|
587
|
+
significant = [t for t in tokens if t not in _GROUP_QUERY_STOPWORDS and len(t) >= 3]
|
|
588
|
+
return significant or tokens
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
def _group_name_matches_query(name: str, tokens: list[str]) -> bool:
|
|
592
|
+
"""Token-based match of a tracked group name against query tokens.
|
|
593
|
+
|
|
594
|
+
A token matches if it is a substring of the group name (so ``lockbit``
|
|
595
|
+
matches ``lockbit``, ``lockbit2``, ``lockbit3``) or the group name is a
|
|
596
|
+
substring of the token. This is how an analyst actually phrases a query,
|
|
597
|
+
versus the old "entire query string must appear in the name" rule.
|
|
598
|
+
"""
|
|
599
|
+
n = (name or "").lower()
|
|
600
|
+
if not n:
|
|
601
|
+
return False
|
|
602
|
+
return any(t and (t in n or n in t) for t in tokens)
|
|
603
|
+
|
|
568
604
|
|
|
569
605
|
def _rl_extract_onion_urls(group: dict) -> list[str]:
|
|
570
606
|
"""Extract .onion leak-site URLs from a group dict (available sites first)."""
|
|
@@ -609,10 +645,11 @@ async def fetch_ransomware_live(query: str) -> list[dict]:
|
|
|
609
645
|
return []
|
|
610
646
|
all_groups = await resp.json(content_type=None)
|
|
611
647
|
|
|
648
|
+
_tokens = _significant_group_tokens(query)
|
|
612
649
|
matched_summary: list[dict] = []
|
|
613
650
|
for g in (all_groups if isinstance(all_groups, list) else []):
|
|
614
651
|
name = (g.get("name") or "").lower()
|
|
615
|
-
if
|
|
652
|
+
if _group_name_matches_query(name, _tokens):
|
|
616
653
|
matched_summary.append(g)
|
|
617
654
|
|
|
618
655
|
if not matched_summary:
|
|
@@ -904,7 +941,8 @@ async def fetch_ransomlook(query: str) -> list[dict]:
|
|
|
904
941
|
all_groups = await resp.json(content_type=None)
|
|
905
942
|
|
|
906
943
|
group_names = [g for g in all_groups if isinstance(g, str)] if isinstance(all_groups, list) else []
|
|
907
|
-
|
|
944
|
+
_tokens = _significant_group_tokens(query)
|
|
945
|
+
matched = [name for name in group_names if _group_name_matches_query(name, _tokens)]
|
|
908
946
|
|
|
909
947
|
if not matched:
|
|
910
948
|
logger.info("ransomlook.io: no groups matched %r", query)
|
|
@@ -395,7 +395,7 @@ class TestEntityRelationship:
|
|
|
395
395
|
rel = EntityRelationship(
|
|
396
396
|
entity_a_id=e1.id,
|
|
397
397
|
entity_b_id=e2.id,
|
|
398
|
-
relationship_type=RelationshipType.
|
|
398
|
+
relationship_type=RelationshipType.USES.value,
|
|
399
399
|
source_page_id=page.id,
|
|
400
400
|
confidence=0.85,
|
|
401
401
|
)
|
|
@@ -404,7 +404,7 @@ class TestEntityRelationship:
|
|
|
404
404
|
|
|
405
405
|
assert rel.id is not None
|
|
406
406
|
assert rel.confidence == 0.85
|
|
407
|
-
assert rel.relationship_type == "
|
|
407
|
+
assert rel.relationship_type == "USES"
|
|
408
408
|
|
|
409
409
|
def test_bidirectional_orm_access(self, session):
|
|
410
410
|
from db.models import EntityRelationship
|
|
@@ -788,12 +788,12 @@ class TestQueries:
|
|
|
788
788
|
session,
|
|
789
789
|
entity_a_id=e1.id,
|
|
790
790
|
entity_b_id=e2.id,
|
|
791
|
-
relationship_type="
|
|
791
|
+
relationship_type="USES",
|
|
792
792
|
source_page_id=page.id,
|
|
793
793
|
confidence=0.9,
|
|
794
794
|
)
|
|
795
795
|
assert rel.id is not None
|
|
796
|
-
assert rel.relationship_type == "
|
|
796
|
+
assert rel.relationship_type == "USES"
|
|
797
797
|
assert rel.confidence == 0.9
|
|
798
798
|
|
|
799
799
|
def test_get_relationships_for_entity(self, session):
|
|
@@ -809,7 +809,7 @@ class TestQueries:
|
|
|
809
809
|
e2 = create_entity(session, page_id=page.id, entity_type="malware", value="Malware_Q")
|
|
810
810
|
e3 = create_entity(session, page_id=page.id, entity_type="handle", value="alias_q")
|
|
811
811
|
|
|
812
|
-
create_entity_relationship(session, e1.id, e2.id, "
|
|
812
|
+
create_entity_relationship(session, e1.id, e2.id, "USES", page.id)
|
|
813
813
|
create_entity_relationship(session, e3.id, e1.id, "LIKELY_SAME_ACTOR", page.id)
|
|
814
814
|
|
|
815
815
|
rels = get_relationships_for_entity(session, e1.id)
|
|
@@ -849,8 +849,9 @@ class TestEnums:
|
|
|
849
849
|
from db.models import RelationshipType
|
|
850
850
|
expected = {
|
|
851
851
|
"CO_APPEARED_ON", "POSTED_BY", "LINKED_TO", "PAID_TO",
|
|
852
|
-
"MEMBER_OF", "
|
|
852
|
+
"MEMBER_OF", "USES", "CLAIMED", "LIKELY_SAME_ACTOR",
|
|
853
853
|
"CONFIRMED_SAME_ACTOR", "FUNDED_BY", "POSSIBLE_SAME_AUTHOR",
|
|
854
|
+
"DROPS", "CONTROLS", "TARGETS", "EXPLOITS", "COMMUNICATES_WITH",
|
|
854
855
|
}
|
|
855
856
|
actual = {r.value for r in RelationshipType}
|
|
856
857
|
assert expected == actual
|
|
@@ -61,7 +61,7 @@ def _make_graph_with_nodes() -> nx.MultiDiGraph:
|
|
|
61
61
|
# Edges
|
|
62
62
|
g.add_edge("actor1", "wallet1", edge_type="CO_APPEARED_ON", confidence=1.0,
|
|
63
63
|
source_url="http://forum1.onion/page", timestamp=now, metadata={})
|
|
64
|
-
g.add_edge("actor1", "malware1", edge_type="
|
|
64
|
+
g.add_edge("actor1", "malware1", edge_type="USES", confidence=0.9,
|
|
65
65
|
source_url="http://forum1.onion/page", timestamp=now, metadata={})
|
|
66
66
|
g.add_edge("actor1", "forum1", edge_type="MEMBER_OF", confidence=1.0,
|
|
67
67
|
source_url="", timestamp=now, metadata={})
|
|
@@ -122,7 +122,7 @@ class TestModel(unittest.TestCase):
|
|
|
122
122
|
def test_edge_types_are_strings(self):
|
|
123
123
|
et = self.m.EDGE_TYPES
|
|
124
124
|
for attr in ("CO_APPEARED_ON", "POSTED_BY", "LINKED_TO", "MEMBER_OF",
|
|
125
|
-
"
|
|
125
|
+
"USES", "CLAIMED", "LIKELY_SAME_ACTOR", "CONFIRMED_SAME_ACTOR"):
|
|
126
126
|
self.assertIsInstance(getattr(et, attr), str, msg=f"EDGE_TYPES.{attr} is not str")
|
|
127
127
|
|
|
128
128
|
def test_node_types_no_duplicates(self):
|
|
@@ -138,7 +138,7 @@ class TestModel(unittest.TestCase):
|
|
|
138
138
|
et = self.m.EDGE_TYPES
|
|
139
139
|
values = [
|
|
140
140
|
et.CO_APPEARED_ON, et.POSTED_BY, et.LINKED_TO, et.MEMBER_OF,
|
|
141
|
-
et.
|
|
141
|
+
et.USES, et.CLAIMED, et.LIKELY_SAME_ACTOR, et.CONFIRMED_SAME_ACTOR,
|
|
142
142
|
]
|
|
143
143
|
self.assertEqual(len(values), len(set(values)), "Duplicate EDGE_TYPES values detected")
|
|
144
144
|
|
|
@@ -700,9 +700,9 @@ class TestQueries(unittest.TestCase):
|
|
|
700
700
|
def test_get_neighbors_edge_type_filter(self):
|
|
701
701
|
"""get_neighbors with edge_type filter only traverses matching edges."""
|
|
702
702
|
from graph.queries import get_neighbors
|
|
703
|
-
result = get_neighbors(self.graph, "actor1", hops=2, edge_types=["
|
|
703
|
+
result = get_neighbors(self.graph, "actor1", hops=2, edge_types=["USES"])
|
|
704
704
|
|
|
705
|
-
# Only malware1 should be reachable via
|
|
705
|
+
# Only malware1 should be reachable via USES edge
|
|
706
706
|
hop1_ids = {n.node_id for n in result.get("1", [])}
|
|
707
707
|
self.assertIn("malware1", hop1_ids)
|
|
708
708
|
# wallet1 should NOT be in hop 1 (it's connected via CO_APPEARED_ON)
|
|
@@ -80,6 +80,27 @@ def run(
|
|
|
80
80
|
|
|
81
81
|
cli_config.apply_env()
|
|
82
82
|
|
|
83
|
+
# Validate the query BEFORE any network activity. An empty/whitespace-only
|
|
84
|
+
# or trivially short query cannot yield real intelligence — accepting it
|
|
85
|
+
# silently produces a report that looks identical to a genuine one but is
|
|
86
|
+
# just noise from an essentially meaningless search. Reject up front.
|
|
87
|
+
MIN_QUERY_LENGTH = 3
|
|
88
|
+
stripped_query = (query or "").strip()
|
|
89
|
+
if not stripped_query:
|
|
90
|
+
console.print(
|
|
91
|
+
"[red]Empty query.[/red] Provide something to investigate, e.g. "
|
|
92
|
+
'[bold]voidaccess investigate "LockBit ransomware"[/bold].'
|
|
93
|
+
)
|
|
94
|
+
raise typer.Exit(code=2)
|
|
95
|
+
if len(stripped_query) < MIN_QUERY_LENGTH:
|
|
96
|
+
console.print(
|
|
97
|
+
f"[red]Query too short.[/red] '{stripped_query}' is only "
|
|
98
|
+
f"{len(stripped_query)} character(s); provide at least {MIN_QUERY_LENGTH} "
|
|
99
|
+
"characters of meaningful context so results reflect a real query, "
|
|
100
|
+
"not generic noise."
|
|
101
|
+
)
|
|
102
|
+
raise typer.Exit(code=2)
|
|
103
|
+
|
|
83
104
|
# v1.6.2 — --use-scraping-api and --use-proxies are one-shot CLI flags
|
|
84
105
|
# for activating the two independent clearnet transports in-process
|
|
85
106
|
# only. The REST API flag sets VOIDACCESS_USE_PROXIES=true; the
|
|
@@ -352,10 +373,21 @@ async def _run_investigation(
|
|
|
352
373
|
pipeline_metrics = InvestigationMetrics(inv_uuid)
|
|
353
374
|
set_current(pipeline_metrics)
|
|
354
375
|
|
|
376
|
+
# Step-cost instrumentation. ``start``/``finish`` bracket each phase so
|
|
377
|
+
# duration_ms is recorded, and — critically — so ``record_llm_call`` (which
|
|
378
|
+
# only credits steps that are currently *active*) attributes LLM calls to
|
|
379
|
+
# the right step. Mirrors the API pipeline's _metric_start/_metric_finish.
|
|
380
|
+
def _metric_start(step_name: str) -> None:
|
|
381
|
+
pipeline_metrics.start(step_name)
|
|
382
|
+
|
|
383
|
+
def _metric_finish(step_name: str) -> None:
|
|
384
|
+
pipeline_metrics.finish(step_name)
|
|
385
|
+
|
|
355
386
|
sources_used: dict[str, dict[str, Any]] = {}
|
|
356
387
|
page_count_by_url: dict[str, dict[str, Any]] = {}
|
|
357
388
|
|
|
358
389
|
# --- Step 1 — refine query -------------------------------------------
|
|
390
|
+
_metric_start("query_refinement")
|
|
359
391
|
display.update_step("Refining query", "active")
|
|
360
392
|
refined = query
|
|
361
393
|
if llm is not None:
|
|
@@ -370,8 +402,10 @@ async def _run_investigation(
|
|
|
370
402
|
else:
|
|
371
403
|
display.update_step("Refining query", "skip", "--no-llm")
|
|
372
404
|
sqlite_adapter.update_investigation(investigation_id, {"refined_query": refined})
|
|
405
|
+
_metric_finish("query_refinement")
|
|
373
406
|
|
|
374
407
|
# --- Step 2 — search fan-out -----------------------------------------
|
|
408
|
+
_metric_start("source_gathering")
|
|
375
409
|
display.update_step("Searching dark web", "active")
|
|
376
410
|
search_links: list[dict] = []
|
|
377
411
|
paste_pages: list[dict] = []
|
|
@@ -412,6 +446,14 @@ async def _run_investigation(
|
|
|
412
446
|
return []
|
|
413
447
|
|
|
414
448
|
async def _onionsearch():
|
|
449
|
+
# --no-tor means NO Tor traffic for the entire run. Torch and
|
|
450
|
+
# Haystack are .onion search engines fetched over the Tor SOCKS proxy,
|
|
451
|
+
# and they return real .onion result URLs that would otherwise be
|
|
452
|
+
# appended to the scrape list and routed back through Tor. Skipping
|
|
453
|
+
# the lookup entirely is what makes "zero Tor traffic" a direct
|
|
454
|
+
# consequence rather than a downstream filter after Tor was contacted.
|
|
455
|
+
if no_tor:
|
|
456
|
+
return []
|
|
415
457
|
from sources.engines import search_onionsearch
|
|
416
458
|
return await search_onionsearch(refined)
|
|
417
459
|
|
|
@@ -441,6 +483,10 @@ async def _run_investigation(
|
|
|
441
483
|
if not os.getenv("TELEGRAM_API_ID", "").strip() or not os.getenv("TELEGRAM_API_HASH", "").strip():
|
|
442
484
|
sources_used["telegram"] = {"status": "skipped_no_key"}
|
|
443
485
|
for _engine_name in ("Torch", "Haystack"):
|
|
486
|
+
if no_tor:
|
|
487
|
+
# Onion search engines are never contacted in --no-tor mode.
|
|
488
|
+
sources_used[_engine_name.lower()] = {"status": "skipped", "count": 0}
|
|
489
|
+
continue
|
|
444
490
|
_count = sum(1 for item in onion_pages if item.get("source") == _engine_name)
|
|
445
491
|
from sources.engines import get_last_onionsearch_status
|
|
446
492
|
_engine_status = get_last_onionsearch_status().get(_engine_name, "ok")
|
|
@@ -449,6 +495,25 @@ async def _run_investigation(
|
|
|
449
495
|
"count": _count,
|
|
450
496
|
}
|
|
451
497
|
|
|
498
|
+
# --no-tor defence-in-depth: even though the Tor search branch and the
|
|
499
|
+
# onion search engines are both skipped above, strip any .onion URL that
|
|
500
|
+
# may have reached the link list from ANY other side-source before it can
|
|
501
|
+
# be handed to the scraper (the scraper correctly routes .onion links
|
|
502
|
+
# through Tor, which is exactly what --no-tor must prevent). This runs
|
|
503
|
+
# BEFORE any scrape, so no Tor connection is ever attempted.
|
|
504
|
+
if no_tor:
|
|
505
|
+
_before = len(search_links)
|
|
506
|
+
search_links = [
|
|
507
|
+
_l for _l in search_links
|
|
508
|
+
if ".onion" not in str(_l.get("link", "")).lower()
|
|
509
|
+
]
|
|
510
|
+
_stripped = _before - len(search_links)
|
|
511
|
+
if _stripped:
|
|
512
|
+
logger.warning(
|
|
513
|
+
"[no-tor] Stripped %d .onion link(s) from search results before scraping",
|
|
514
|
+
_stripped,
|
|
515
|
+
)
|
|
516
|
+
|
|
452
517
|
if not no_tor and search_summary:
|
|
453
518
|
display.update_step(
|
|
454
519
|
"Searching dark web",
|
|
@@ -495,7 +560,10 @@ async def _run_investigation(
|
|
|
495
560
|
filtered_links = (search_links or [])[:filter_top_n]
|
|
496
561
|
display.update_step("Filtering results", "skip" if no_llm else "ok", f"{len(filtered_links)} kept")
|
|
497
562
|
|
|
563
|
+
_metric_finish("source_gathering")
|
|
564
|
+
|
|
498
565
|
# --- Step 4 — scrape pages -------------------------------------------
|
|
566
|
+
_metric_start("scraping")
|
|
499
567
|
display.update_step("Scraping pages", "active")
|
|
500
568
|
scraped_pages: list[dict] = []
|
|
501
569
|
if filtered_links:
|
|
@@ -521,11 +589,18 @@ async def _run_investigation(
|
|
|
521
589
|
for url, text in results.items():
|
|
522
590
|
if text:
|
|
523
591
|
scraped_pages.append({"url": url, "text": text, "source": "tor_search"})
|
|
592
|
+
_fetched = sum(1 for _t in results.values() if _t)
|
|
593
|
+
pipeline_metrics.record_scraping(
|
|
594
|
+
attempted=len(filtered_links),
|
|
595
|
+
fetched=_fetched,
|
|
596
|
+
cache_hits=0,
|
|
597
|
+
)
|
|
524
598
|
display.update_step("Scraping pages", "ok", f"{len(scraped_pages)} pages")
|
|
525
599
|
except Exception as exc:
|
|
526
600
|
display.update_step("Scraping pages", "fail", str(exc))
|
|
527
601
|
else:
|
|
528
602
|
display.update_step("Scraping pages", "skip", "no links")
|
|
603
|
+
_metric_finish("scraping")
|
|
529
604
|
|
|
530
605
|
# --- Step 4.1 — discover seeds from scraped pages --------------------
|
|
531
606
|
# scrape_multiple already submits discovered seeds fire-and-forget, but
|
|
@@ -576,6 +651,7 @@ async def _run_investigation(
|
|
|
576
651
|
page_count_by_url = {p["url"]: p for p in scraped_pages}
|
|
577
652
|
|
|
578
653
|
# --- Step 5 — extract entities ---------------------------------------
|
|
654
|
+
_metric_start("entity_extraction")
|
|
579
655
|
display.update_step("Extracting entities", "active")
|
|
580
656
|
extraction_results = []
|
|
581
657
|
try:
|
|
@@ -591,8 +667,10 @@ async def _run_investigation(
|
|
|
591
667
|
display.update_step("Extracting entities", "ok", f"{total_entities} entities")
|
|
592
668
|
except Exception as exc:
|
|
593
669
|
display.update_step("Extracting entities", "fail", str(exc))
|
|
670
|
+
_metric_finish("entity_extraction")
|
|
594
671
|
|
|
595
672
|
# --- Step 6 — enrich intelligence (OTX + IP) ---------------------------
|
|
673
|
+
_metric_start("enrichment")
|
|
596
674
|
display.update_step("Enriching intelligence", "active")
|
|
597
675
|
enrichment_pages: list[dict] = []
|
|
598
676
|
try:
|
|
@@ -735,6 +813,7 @@ async def _run_investigation(
|
|
|
735
813
|
)
|
|
736
814
|
except Exception as exc:
|
|
737
815
|
console.print(f"[grey50]Enrichment extraction failed: {exc}[/grey50]")
|
|
816
|
+
_metric_finish("enrichment")
|
|
738
817
|
|
|
739
818
|
# --- Step 6.9 — Update persistent actor profiles (non-blocking) -------
|
|
740
819
|
# Aggregate THREAT_ACTOR_HANDLE / RANSOMWARE_GROUP entities from the
|
|
@@ -770,6 +849,7 @@ async def _run_investigation(
|
|
|
770
849
|
# --- Step 7 — build graph (co-occurrence) ----------------------------
|
|
771
850
|
# Phase 6.2 — capped by CLI_PHASE_TIMEOUTS["graph_build"] so a large
|
|
772
851
|
# entity table doesn't pin the CLI thread indefinitely.
|
|
852
|
+
_metric_start("graph_build")
|
|
773
853
|
display.update_step("Building graph", "active")
|
|
774
854
|
try:
|
|
775
855
|
edges_written = await asyncio.wait_for(
|
|
@@ -798,10 +878,12 @@ async def _run_investigation(
|
|
|
798
878
|
)
|
|
799
879
|
except Exception as exc:
|
|
800
880
|
logger.debug("Community detection skipped: %s", exc)
|
|
881
|
+
_metric_finish("graph_build")
|
|
801
882
|
|
|
802
883
|
# --- Step 8 — summary -------------------------------------------------
|
|
803
884
|
# Phase 6.2 — capped by CLI_PHASE_TIMEOUTS["summary"] so a slow LLM
|
|
804
885
|
# call never holds the CLI process open past its budget.
|
|
886
|
+
_metric_start("summary_generation")
|
|
805
887
|
display.update_step("Generating summary", "active")
|
|
806
888
|
summary_text = ""
|
|
807
889
|
if llm is not None:
|
|
@@ -826,8 +908,10 @@ async def _run_investigation(
|
|
|
826
908
|
display.update_step("Generating summary", "fail", str(exc))
|
|
827
909
|
else:
|
|
828
910
|
display.update_step("Generating summary", "skip", "--no-llm")
|
|
911
|
+
_metric_finish("summary_generation")
|
|
829
912
|
|
|
830
913
|
# --- Step 9 — finalize & write outputs --------------------------------
|
|
914
|
+
_metric_start("finalization")
|
|
831
915
|
display.update_step("Finalizing results", "active")
|
|
832
916
|
final_entities = sqlite_adapter.get_entities(investigation_id)
|
|
833
917
|
final_relationships = sqlite_adapter.get_relationships(investigation_id)
|
|
@@ -886,6 +970,7 @@ async def _run_investigation(
|
|
|
886
970
|
md_path.write_text(_render_markdown(payload), encoding="utf-8")
|
|
887
971
|
|
|
888
972
|
display.update_step("Finalizing results", "ok")
|
|
973
|
+
_metric_finish("finalization")
|
|
889
974
|
|
|
890
975
|
# Drain any background tasks scheduled during the pipeline (e.g. the
|
|
891
976
|
# actor-profile aggregator). This keeps the "fire-and-forget"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0002_add_investigation_status_column.py
RENAMED
|
File without changes
|