voidaccess 1.8.0__tar.gz → 1.8.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (224) hide show
  1. {voidaccess-1.8.0/voidaccess.egg-info → voidaccess-1.8.1}/PKG-INFO +1 -1
  2. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/models.py +1 -1
  3. {voidaccess-1.8.0 → voidaccess-1.8.1}/export/ioc_package.py +1 -1
  4. {voidaccess-1.8.0 → voidaccess-1.8.1}/export/stix.py +1 -1
  5. {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/normalizer.py +29 -0
  6. {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/relationship_extract.py +46 -10
  7. {voidaccess-1.8.0 → voidaccess-1.8.1}/graph/model.py +1 -1
  8. {voidaccess-1.8.0 → voidaccess-1.8.1}/pyproject.toml +1 -1
  9. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/enrichment.py +40 -2
  10. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_db.py +7 -6
  11. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_graph.py +5 -5
  12. {voidaccess-1.8.0 → voidaccess-1.8.1/voidaccess.egg-info}/PKG-INFO +1 -1
  13. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/__init__.py +1 -1
  14. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/commands/investigate.py +85 -0
  15. {voidaccess-1.8.0 → voidaccess-1.8.1}/LICENSE +0 -0
  16. {voidaccess-1.8.0 → voidaccess-1.8.1}/README.md +0 -0
  17. {voidaccess-1.8.0 → voidaccess-1.8.1}/analysis/__init__.py +0 -0
  18. {voidaccess-1.8.0 → voidaccess-1.8.1}/analysis/opsec.py +0 -0
  19. {voidaccess-1.8.0 → voidaccess-1.8.1}/analysis/patterns.py +0 -0
  20. {voidaccess-1.8.0 → voidaccess-1.8.1}/analysis/temporal.py +0 -0
  21. {voidaccess-1.8.0 → voidaccess-1.8.1}/api/__init__.py +0 -0
  22. {voidaccess-1.8.0 → voidaccess-1.8.1}/api/auth.py +0 -0
  23. {voidaccess-1.8.0 → voidaccess-1.8.1}/api/errors.py +0 -0
  24. {voidaccess-1.8.0 → voidaccess-1.8.1}/api/main.py +0 -0
  25. {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/__init__.py +0 -0
  26. {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/actors.py +0 -0
  27. {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/admin.py +0 -0
  28. {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/auth.py +0 -0
  29. {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/entities.py +0 -0
  30. {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/export.py +0 -0
  31. {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/investigations.py +0 -0
  32. {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/monitors.py +0 -0
  33. {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/search.py +0 -0
  34. {voidaccess-1.8.0 → voidaccess-1.8.1}/api/routes/settings.py +0 -0
  35. {voidaccess-1.8.0 → voidaccess-1.8.1}/auth/__init__.py +0 -0
  36. {voidaccess-1.8.0 → voidaccess-1.8.1}/auth/token_blacklist.py +0 -0
  37. {voidaccess-1.8.0 → voidaccess-1.8.1}/config.py +0 -0
  38. {voidaccess-1.8.0 → voidaccess-1.8.1}/crawler/__init__.py +0 -0
  39. {voidaccess-1.8.0 → voidaccess-1.8.1}/crawler/dedup.py +0 -0
  40. {voidaccess-1.8.0 → voidaccess-1.8.1}/crawler/frontier.py +0 -0
  41. {voidaccess-1.8.0 → voidaccess-1.8.1}/crawler/spider.py +0 -0
  42. {voidaccess-1.8.0 → voidaccess-1.8.1}/crawler/utils.py +0 -0
  43. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/__init__.py +0 -0
  44. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/__init__.py +0 -0
  45. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/env.py +0 -0
  46. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0001_initial_schema.py +0 -0
  47. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0002_add_investigation_status_column.py +0 -0
  48. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0002_add_missing_tables.py +0 -0
  49. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0003_add_canonical_value_and_entity_links.py +0 -0
  50. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0004_add_page_posted_at.py +0 -0
  51. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0005_add_extraction_method.py +0 -0
  52. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0006_add_monitor_alerts.py +0 -0
  53. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0007_add_actor_style_profiles.py +0 -0
  54. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0008_add_users_table.py +0 -0
  55. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0009_add_investigation_id_to_relationships.py +0 -0
  56. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0010_add_composite_index_entity_relationships.py +0 -0
  57. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0011_add_page_extraction_cache.py +0 -0
  58. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0013_add_graph_status.py +0 -0
  59. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0015_add_progress_fields.py +0 -0
  60. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0016_backfill_graph_status.py +0 -0
  61. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0017_add_user_api_keys.py +0 -0
  62. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0018_add_user_id_to_investigations.py +0 -0
  63. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0019_add_content_safety_log.py +0 -0
  64. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0020_add_entity_source_tracking.py +0 -0
  65. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0021_add_search_engine_stats.py +0 -0
  66. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0022_add_actor_profiles.py +0 -0
  67. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0023_add_investigation_metadata.py +0 -0
  68. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0024_add_investigation_step_metrics.py +0 -0
  69. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/migrations/versions/0025_add_cancellation_requested.py +0 -0
  70. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/queries.py +0 -0
  71. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/search_engine_stats.py +0 -0
  72. {voidaccess-1.8.0 → voidaccess-1.8.1}/db/session.py +0 -0
  73. {voidaccess-1.8.0 → voidaccess-1.8.1}/export/__init__.py +0 -0
  74. {voidaccess-1.8.0 → voidaccess-1.8.1}/export/misp.py +0 -0
  75. {voidaccess-1.8.0 → voidaccess-1.8.1}/export/sigma.py +0 -0
  76. {voidaccess-1.8.0 → voidaccess-1.8.1}/export/snort_export.py +0 -0
  77. {voidaccess-1.8.0 → voidaccess-1.8.1}/export/yara_export.py +0 -0
  78. {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/__init__.py +0 -0
  79. {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/confidence.py +0 -0
  80. {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/entity_shape.py +0 -0
  81. {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/gazetteer.py +0 -0
  82. {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/llm_extract.py +0 -0
  83. {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/ner.py +0 -0
  84. {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/pipeline.py +0 -0
  85. {voidaccess-1.8.0 → voidaccess-1.8.1}/extractor/regex_patterns.py +0 -0
  86. {voidaccess-1.8.0 → voidaccess-1.8.1}/fingerprint/__init__.py +0 -0
  87. {voidaccess-1.8.0 → voidaccess-1.8.1}/fingerprint/calibration.py +0 -0
  88. {voidaccess-1.8.0 → voidaccess-1.8.1}/fingerprint/profiler.py +0 -0
  89. {voidaccess-1.8.0 → voidaccess-1.8.1}/fingerprint/stylometry.py +0 -0
  90. {voidaccess-1.8.0 → voidaccess-1.8.1}/graph/__init__.py +0 -0
  91. {voidaccess-1.8.0 → voidaccess-1.8.1}/graph/builder.py +0 -0
  92. {voidaccess-1.8.0 → voidaccess-1.8.1}/graph/export.py +0 -0
  93. {voidaccess-1.8.0 → voidaccess-1.8.1}/graph/queries.py +0 -0
  94. {voidaccess-1.8.0 → voidaccess-1.8.1}/graph/visualize.py +0 -0
  95. {voidaccess-1.8.0 → voidaccess-1.8.1}/i18n/__init__.py +0 -0
  96. {voidaccess-1.8.0 → voidaccess-1.8.1}/i18n/detect.py +0 -0
  97. {voidaccess-1.8.0 → voidaccess-1.8.1}/i18n/query_expand.py +0 -0
  98. {voidaccess-1.8.0 → voidaccess-1.8.1}/i18n/translate.py +0 -0
  99. {voidaccess-1.8.0 → voidaccess-1.8.1}/monitor/__init__.py +0 -0
  100. {voidaccess-1.8.0 → voidaccess-1.8.1}/monitor/_db.py +0 -0
  101. {voidaccess-1.8.0 → voidaccess-1.8.1}/monitor/alerts.py +0 -0
  102. {voidaccess-1.8.0 → voidaccess-1.8.1}/monitor/config.py +0 -0
  103. {voidaccess-1.8.0 → voidaccess-1.8.1}/monitor/diff.py +0 -0
  104. {voidaccess-1.8.0 → voidaccess-1.8.1}/monitor/jobs.py +0 -0
  105. {voidaccess-1.8.0 → voidaccess-1.8.1}/monitor/scheduler.py +0 -0
  106. {voidaccess-1.8.0 → voidaccess-1.8.1}/scraper/__init__.py +0 -0
  107. {voidaccess-1.8.0 → voidaccess-1.8.1}/scraper/scrape.py +0 -0
  108. {voidaccess-1.8.0 → voidaccess-1.8.1}/scraper/scrape_js.py +0 -0
  109. {voidaccess-1.8.0 → voidaccess-1.8.1}/search/__init__.py +0 -0
  110. {voidaccess-1.8.0 → voidaccess-1.8.1}/search/circuit_breaker.py +0 -0
  111. {voidaccess-1.8.0 → voidaccess-1.8.1}/search/engine_catalog.py +0 -0
  112. {voidaccess-1.8.0 → voidaccess-1.8.1}/search/query_builder.py +0 -0
  113. {voidaccess-1.8.0 → voidaccess-1.8.1}/search/search.py +0 -0
  114. {voidaccess-1.8.0 → voidaccess-1.8.1}/setup.cfg +0 -0
  115. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/__init__.py +0 -0
  116. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/actor_profiles.py +0 -0
  117. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/blockchain.py +0 -0
  118. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/breach_lookup.py +0 -0
  119. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/cache.py +0 -0
  120. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/cisa.py +0 -0
  121. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/data/__init__.py +0 -0
  122. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/data/onion_seeds.json +0 -0
  123. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/dns_enrichment.py +0 -0
  124. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/domain_reputation.py +0 -0
  125. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/email_reputation.py +0 -0
  126. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/engines.py +0 -0
  127. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/github_scraper.py +0 -0
  128. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/gitlab_scraper.py +0 -0
  129. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/hash_reputation.py +0 -0
  130. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/historical_intel.py +0 -0
  131. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/infostealer.py +0 -0
  132. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/ip_reputation.py +0 -0
  133. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/nvd.py +0 -0
  134. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/paste_scraper.py +0 -0
  135. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/pastes.py +0 -0
  136. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/proxy_client.py +0 -0
  137. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/rss_scraper.py +0 -0
  138. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/seed_manager.py +0 -0
  139. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/seeds.py +0 -0
  140. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/shodan.py +0 -0
  141. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/telegram.py +0 -0
  142. {voidaccess-1.8.0 → voidaccess-1.8.1}/sources/virustotal.py +0 -0
  143. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_analysis_opsec.py +0 -0
  144. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_analysis_stylometry.py +0 -0
  145. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_analysis_temporal.py +0 -0
  146. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_api.py +0 -0
  147. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_api_monitors.py +0 -0
  148. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_blockchain.py +0 -0
  149. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_breach_lookup.py +0 -0
  150. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_cli_proxy_config.py +0 -0
  151. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_confidence_calibration.py +0 -0
  152. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_config.py +0 -0
  153. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_crawler.py +0 -0
  154. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_dns_enrichment.py +0 -0
  155. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_domain_reputation.py +0 -0
  156. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_email_reputation.py +0 -0
  157. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_extractor_llm_pipeline.py +0 -0
  158. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_filter_e2e_manual.py +0 -0
  159. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_filter_parser_manual.py +0 -0
  160. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_fingerprint.py +0 -0
  161. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_github_scraper.py +0 -0
  162. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_gitlab_scraper.py +0 -0
  163. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_graph_integration.py +0 -0
  164. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_hash_reputation.py +0 -0
  165. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_i18n.py +0 -0
  166. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_infostealer.py +0 -0
  167. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_ip_reputation.py +0 -0
  168. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_llm.py +0 -0
  169. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_llm_utils.py +0 -0
  170. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_model_singleton.py +0 -0
  171. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_monitor.py +0 -0
  172. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_monitor_integration.py +0 -0
  173. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_nvd.py +0 -0
  174. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_pagination.py +0 -0
  175. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_paste_scraper.py +0 -0
  176. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_proxy_client.py +0 -0
  177. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_ransomlook.py +0 -0
  178. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_regex_patterns.py +0 -0
  179. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_rss_scraper.py +0 -0
  180. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_scrape_js.py +0 -0
  181. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_settings.py +0 -0
  182. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_snort_export.py +0 -0
  183. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_sources.py +0 -0
  184. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_sources_enrichment_new.py +0 -0
  185. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_stix_export.py +0 -0
  186. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_vector.py +0 -0
  187. {voidaccess-1.8.0 → voidaccess-1.8.1}/tests/test_yara_export.py +0 -0
  188. {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/__init__.py +0 -0
  189. {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/async_utils.py +0 -0
  190. {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/content_dedup.py +0 -0
  191. {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/content_safety.py +0 -0
  192. {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/defang.py +0 -0
  193. {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/encryption.py +0 -0
  194. {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/enrichment_cache.py +0 -0
  195. {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/investigation_metrics.py +0 -0
  196. {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/ioc_freshness.py +0 -0
  197. {voidaccess-1.8.0 → voidaccess-1.8.1}/utils/user_keys.py +0 -0
  198. {voidaccess-1.8.0 → voidaccess-1.8.1}/vector/__init__.py +0 -0
  199. {voidaccess-1.8.0 → voidaccess-1.8.1}/vector/embedder.py +0 -0
  200. {voidaccess-1.8.0 → voidaccess-1.8.1}/vector/model_singleton.py +0 -0
  201. {voidaccess-1.8.0 → voidaccess-1.8.1}/vector/search.py +0 -0
  202. {voidaccess-1.8.0 → voidaccess-1.8.1}/vector/store.py +0 -0
  203. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess/__init__.py +0 -0
  204. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess/config.py +0 -0
  205. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess/llm.py +0 -0
  206. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess/llm_utils.py +0 -0
  207. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess.egg-info/SOURCES.txt +0 -0
  208. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess.egg-info/dependency_links.txt +0 -0
  209. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess.egg-info/entry_points.txt +0 -0
  210. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess.egg-info/requires.txt +0 -0
  211. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess.egg-info/top_level.txt +0 -0
  212. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/adapters/__init__.py +0 -0
  213. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/adapters/sqlite.py +0 -0
  214. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/browser.py +0 -0
  215. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/commands/__init__.py +0 -0
  216. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/commands/actors.py +0 -0
  217. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/commands/configure.py +0 -0
  218. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/commands/enrich.py +0 -0
  219. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/commands/export.py +0 -0
  220. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/commands/show.py +0 -0
  221. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/config.py +0 -0
  222. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/display.py +0 -0
  223. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/main.py +0 -0
  224. {voidaccess-1.8.0 → voidaccess-1.8.1}/voidaccess_cli/tor_detect.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: voidaccess
3
- Version: 1.8.0
3
+ Version: 1.8.1
4
4
  Summary: Dark web OSINT CLI — automated threat intelligence from query to report
5
5
  Author: VoidAccess
6
6
  License-Expression: MIT
@@ -75,7 +75,7 @@ class RelationshipType(str, enum.Enum):
75
75
  LINKED_TO = "LINKED_TO"
76
76
  PAID_TO = "PAID_TO"
77
77
  MEMBER_OF = "MEMBER_OF"
78
- USED = "USED"
78
+ USES = "USES"
79
79
  CLAIMED = "CLAIMED"
80
80
  LIKELY_SAME_ACTOR = "LIKELY_SAME_ACTOR"
81
81
  CONFIRMED_SAME_ACTOR = "CONFIRMED_SAME_ACTOR"
@@ -54,7 +54,7 @@ logger = logging.getLogger(__name__)
54
54
  # ---------------------------------------------------------------------------
55
55
 
56
56
  PACKAGE_FORMAT = "voidaccess-ioc-v1"
57
- PACKAGE_VERSION = "1.8.0"
57
+ PACKAGE_VERSION = "1.8.1"
58
58
  SNORT_SID_BASE = 9000001 # 9xxxxxx reserved for VoidAccess-generated rules
59
59
 
60
60
  # Map extractor upper-case entity types to the per-file bucket they belong to.
@@ -508,7 +508,7 @@ def _edge_type_to_stix(edge_type: str) -> str:
508
508
  "LINKED_TO": "related-to",
509
509
  "PAID_TO": "related-to",
510
510
  "MEMBER_OF": "member-of",
511
- "USED": "uses",
511
+ "USES": "uses",
512
512
  "CLAIMED": "attributed-to",
513
513
  "LIKELY_SAME_ACTOR": "related-to",
514
514
  "CONFIRMED_SAME_ACTOR": "related-to",
@@ -887,6 +887,27 @@ _RFC5737_NETWORKS: list[tuple[str, str]] = [
887
887
  ("203.0.113.0", "255.255.255.0"), # 203.0.113.0/24
888
888
  ]
889
889
 
890
+ # Real, routable IPs that are never a meaningful indicator of compromise:
891
+ # the unspecified/null address, the broadcast address, and the well-known
892
+ # public DNS resolvers that appear constantly in technical content for
893
+ # reasons unrelated to being an IOC. Filtered like the RFC 5737 ranges so a
894
+ # user pulling ip_addresses.txt into a blocklist can't end up blocking a
895
+ # legitimate public resolver or a null route.
896
+ _NON_ACTIONABLE_IPS: frozenset[str] = frozenset({
897
+ "0.0.0.0", # unspecified / null route — never a real host
898
+ "255.255.255.255", # limited broadcast
899
+ "8.8.8.8", "8.8.4.4", # Google Public DNS
900
+ "1.1.1.1", "1.0.0.1", # Cloudflare Public DNS
901
+ "9.9.9.9", "149.112.112.112", # Quad9
902
+ "208.67.222.222", "208.67.220.220", # OpenDNS
903
+ "4.2.2.1", "4.2.2.2", # Level3 legacy resolvers
904
+ })
905
+
906
+
907
+ def _is_non_actionable_ip(value: str) -> bool:
908
+ """Return True for real IPs that are never meaningful IOCs (null route, public DNS)."""
909
+ return value.strip() in _NON_ACTIONABLE_IPS
910
+
890
911
  # RFC 2606 reserved domain names.
891
912
  _RFC2606_DOMAINS: frozenset[str] = frozenset({
892
913
  "example.com",
@@ -1563,6 +1584,14 @@ def normalize_entities(
1563
1584
  )
1564
1585
  continue
1565
1586
 
1587
+ if entity_type == "IP_ADDRESS" and _is_non_actionable_ip(canonical):
1588
+ source_quality = 0.0
1589
+ logger.debug(
1590
+ "Non-actionable IP rejected (null route / broadcast / public DNS resolver): %s",
1591
+ canonical,
1592
+ )
1593
+ continue
1594
+
1566
1595
  if entity_type == "DOMAIN" and _is_rfc2606_domain(canonical):
1567
1596
  source_quality = 0.0
1568
1597
  logger.debug(
@@ -60,8 +60,8 @@ _DEFAULT_MAX_RELATIONSHIPS_PER_PAGE = 60
60
60
  # EDGE_TYPES). Passive forms map to the same stored type but flip the edge
61
61
  # direction (handled below) so the stored edge always reads source -> target.
62
62
  _LLM_REL_VOCAB: dict[str, str] = {
63
- "uses": "USED",
64
- "used": "USED",
63
+ "uses": "USES",
64
+ "used": "USES",
65
65
  "drops": "DROPS",
66
66
  "controls": "CONTROLS",
67
67
  "targets": "TARGETS",
@@ -144,17 +144,41 @@ def _parse_relationship_json(raw: str) -> list[dict]:
144
144
  return rels if isinstance(rels, list) else []
145
145
 
146
146
 
147
+ class RelationshipLLMCallError(Exception):
148
+ """Raised when the relationship-extraction LLM invocation itself fails.
149
+
150
+ Distinct from a call that *succeeds* and legitimately returns no
151
+ relationships — callers use this to log a visible warning so a broken LLM
152
+ integration can never masquerade as "the LLM found no relationships".
153
+ """
154
+
155
+
147
156
  async def _invoke_llm(prompt: str, llm) -> str:
148
- """Call the LLM with streaming/callbacks disabled (mirrors llm_extract)."""
149
- try:
157
+ """Call the LLM with streaming/callbacks disabled (mirrors llm_extract).
158
+
159
+ Raises :class:`RelationshipLLMCallError` if the underlying invocation
160
+ fails, so the caller can distinguish a *failed* call from a successful
161
+ call that found nothing.
162
+ """
163
+ # ``bind(callbacks=[])`` puts ``callbacks`` into the model's invocation
164
+ # kwargs, which LangChain forwards to ``agenerate_prompt`` as ``**kwargs``
165
+ # — while ``ainvoke`` ALSO passes ``callbacks`` explicitly, raising
166
+ # "agenerate_prompt() got multiple values for keyword argument 'callbacks'".
167
+ # ``with_config`` overrides callbacks for this call WITHOUT adding model
168
+ # kwargs, exactly as extractor/llm_extract._extract_chunk does.
169
+ if hasattr(llm, "with_config"):
170
+ silent_llm = llm.with_config({"callbacks": []})
171
+ else:
150
172
  silent_llm = llm.bind(streaming=False, callbacks=[])
173
+ try:
174
+ # Guard stdout in case a legacy constructor callback still streams
175
+ # partial JSON tokens even with an empty per-call callback list.
151
176
  with redirect_stdout(io.StringIO()):
152
177
  response = await silent_llm.ainvoke(prompt)
153
- content = response.content if hasattr(response, "content") else str(response)
154
- return content.strip()
155
- except Exception as exc: # noqa: BLE001 — never let one page break the pass
156
- logger.warning("Relationship LLM call failed: %s", exc)
157
- return ""
178
+ except Exception as exc: # noqa: BLE001 — surface as a typed failure
179
+ raise RelationshipLLMCallError(str(exc)) from exc
180
+ content = response.content if hasattr(response, "content") else str(response)
181
+ return content.strip()
158
182
 
159
183
 
160
184
  async def extract_relationships_for_page(
@@ -194,7 +218,19 @@ async def extract_relationships_for_page(
194
218
  content = page_text[:max_chunk_chars]
195
219
  prompt = _PROMPT_TEMPLATE.format(entities=entity_lines, content=content)
196
220
 
197
- raw = await _invoke_llm(prompt, llm)
221
+ try:
222
+ raw = await _invoke_llm(prompt, llm)
223
+ except RelationshipLLMCallError as exc:
224
+ # The LLM call itself failed — this is NOT the same as the LLM running
225
+ # successfully and finding no relationship. Log loudly so a broken
226
+ # integration is visible in normal operation instead of silently
227
+ # degrading every page to zero typed relationships.
228
+ logger.warning(
229
+ "Relationship extraction: LLM call FAILED (%s) — this page yielded "
230
+ "NO typed relationships due to an error, not because none exist",
231
+ exc,
232
+ )
233
+ return []
198
234
  rels = _parse_relationship_json(raw)
199
235
  if not rels:
200
236
  return []
@@ -77,7 +77,7 @@ class EDGE_TYPES:
77
77
  POSTED_BY = "POSTED_BY" # content attributed to a handle
78
78
  LINKED_TO = "LINKED_TO" # URL links to URL
79
79
  MEMBER_OF = "MEMBER_OF" # handle to group/forum
80
- USED = "USED" # actor used a malware family
80
+ USES = "USES" # actor uses a malware family
81
81
  CLAIMED = "CLAIMED" # group claimed an attack
82
82
  LIKELY_SAME_ACTOR = "LIKELY_SAME_ACTOR" # inferred, medium confidence
83
83
  CONFIRMED_SAME_ACTOR = "CONFIRMED_SAME_ACTOR" # PGP key match, high confidence
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "voidaccess"
7
- version = "1.8.0"
7
+ version = "1.8.1"
8
8
  description = "Dark web OSINT CLI — automated threat intelligence from query to report"
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -565,6 +565,42 @@ def abusech_to_pages(
565
565
  _RANSOMWARE_LIVE_BASE = "https://api.ransomware.live/v2"
566
566
  _RANSOMWARE_LIVE_HEADERS = {"User-Agent": "VoidAccess-OSINT/1.0", "Accept": "application/json"}
567
567
 
568
+ # Generic words that carry no group-identity signal in a realistic analyst
569
+ # query ("LockBit ransomware leak site"). Stripped before matching so the
570
+ # meaningful token(s) — e.g. "lockbit" — are what we match on, rather than
571
+ # requiring the whole query string to appear literally in a group name.
572
+ _GROUP_QUERY_STOPWORDS: frozenset[str] = frozenset({
573
+ "ransomware", "ransom", "group", "gang", "crew", "team",
574
+ "leak", "leaks", "leaksite", "site", "blog", "portal",
575
+ "data", "the", "and", "actor", "threat", "malware",
576
+ })
577
+
578
+
579
+ def _significant_group_tokens(query: str) -> list[str]:
580
+ """Meaningful lowercase tokens from a group query, stopwords removed.
581
+
582
+ Falls back to all tokens if every token was a stopword, so a query made
583
+ entirely of generic words still matches on something rather than nothing.
584
+ """
585
+ tokens = re.findall(r"[a-z0-9.]+", (query or "").lower())
586
+ tokens = [t.strip(".") for t in tokens if t.strip(".")]
587
+ significant = [t for t in tokens if t not in _GROUP_QUERY_STOPWORDS and len(t) >= 3]
588
+ return significant or tokens
589
+
590
+
591
+ def _group_name_matches_query(name: str, tokens: list[str]) -> bool:
592
+ """Token-based match of a tracked group name against query tokens.
593
+
594
+ A token matches if it is a substring of the group name (so ``lockbit``
595
+ matches ``lockbit``, ``lockbit2``, ``lockbit3``) or the group name is a
596
+ substring of the token. This is how an analyst actually phrases a query,
597
+ versus the old "entire query string must appear in the name" rule.
598
+ """
599
+ n = (name or "").lower()
600
+ if not n:
601
+ return False
602
+ return any(t and (t in n or n in t) for t in tokens)
603
+
568
604
 
569
605
  def _rl_extract_onion_urls(group: dict) -> list[str]:
570
606
  """Extract .onion leak-site URLs from a group dict (available sites first)."""
@@ -609,10 +645,11 @@ async def fetch_ransomware_live(query: str) -> list[dict]:
609
645
  return []
610
646
  all_groups = await resp.json(content_type=None)
611
647
 
648
+ _tokens = _significant_group_tokens(query)
612
649
  matched_summary: list[dict] = []
613
650
  for g in (all_groups if isinstance(all_groups, list) else []):
614
651
  name = (g.get("name") or "").lower()
615
- if q in name:
652
+ if _group_name_matches_query(name, _tokens):
616
653
  matched_summary.append(g)
617
654
 
618
655
  if not matched_summary:
@@ -904,7 +941,8 @@ async def fetch_ransomlook(query: str) -> list[dict]:
904
941
  all_groups = await resp.json(content_type=None)
905
942
 
906
943
  group_names = [g for g in all_groups if isinstance(g, str)] if isinstance(all_groups, list) else []
907
- matched = [name for name in group_names if q in name.lower()]
944
+ _tokens = _significant_group_tokens(query)
945
+ matched = [name for name in group_names if _group_name_matches_query(name, _tokens)]
908
946
 
909
947
  if not matched:
910
948
  logger.info("ransomlook.io: no groups matched %r", query)
@@ -395,7 +395,7 @@ class TestEntityRelationship:
395
395
  rel = EntityRelationship(
396
396
  entity_a_id=e1.id,
397
397
  entity_b_id=e2.id,
398
- relationship_type=RelationshipType.USED.value,
398
+ relationship_type=RelationshipType.USES.value,
399
399
  source_page_id=page.id,
400
400
  confidence=0.85,
401
401
  )
@@ -404,7 +404,7 @@ class TestEntityRelationship:
404
404
 
405
405
  assert rel.id is not None
406
406
  assert rel.confidence == 0.85
407
- assert rel.relationship_type == "USED"
407
+ assert rel.relationship_type == "USES"
408
408
 
409
409
  def test_bidirectional_orm_access(self, session):
410
410
  from db.models import EntityRelationship
@@ -788,12 +788,12 @@ class TestQueries:
788
788
  session,
789
789
  entity_a_id=e1.id,
790
790
  entity_b_id=e2.id,
791
- relationship_type="USED",
791
+ relationship_type="USES",
792
792
  source_page_id=page.id,
793
793
  confidence=0.9,
794
794
  )
795
795
  assert rel.id is not None
796
- assert rel.relationship_type == "USED"
796
+ assert rel.relationship_type == "USES"
797
797
  assert rel.confidence == 0.9
798
798
 
799
799
  def test_get_relationships_for_entity(self, session):
@@ -809,7 +809,7 @@ class TestQueries:
809
809
  e2 = create_entity(session, page_id=page.id, entity_type="malware", value="Malware_Q")
810
810
  e3 = create_entity(session, page_id=page.id, entity_type="handle", value="alias_q")
811
811
 
812
- create_entity_relationship(session, e1.id, e2.id, "USED", page.id)
812
+ create_entity_relationship(session, e1.id, e2.id, "USES", page.id)
813
813
  create_entity_relationship(session, e3.id, e1.id, "LIKELY_SAME_ACTOR", page.id)
814
814
 
815
815
  rels = get_relationships_for_entity(session, e1.id)
@@ -849,8 +849,9 @@ class TestEnums:
849
849
  from db.models import RelationshipType
850
850
  expected = {
851
851
  "CO_APPEARED_ON", "POSTED_BY", "LINKED_TO", "PAID_TO",
852
- "MEMBER_OF", "USED", "CLAIMED", "LIKELY_SAME_ACTOR",
852
+ "MEMBER_OF", "USES", "CLAIMED", "LIKELY_SAME_ACTOR",
853
853
  "CONFIRMED_SAME_ACTOR", "FUNDED_BY", "POSSIBLE_SAME_AUTHOR",
854
+ "DROPS", "CONTROLS", "TARGETS", "EXPLOITS", "COMMUNICATES_WITH",
854
855
  }
855
856
  actual = {r.value for r in RelationshipType}
856
857
  assert expected == actual
@@ -61,7 +61,7 @@ def _make_graph_with_nodes() -> nx.MultiDiGraph:
61
61
  # Edges
62
62
  g.add_edge("actor1", "wallet1", edge_type="CO_APPEARED_ON", confidence=1.0,
63
63
  source_url="http://forum1.onion/page", timestamp=now, metadata={})
64
- g.add_edge("actor1", "malware1", edge_type="USED", confidence=0.9,
64
+ g.add_edge("actor1", "malware1", edge_type="USES", confidence=0.9,
65
65
  source_url="http://forum1.onion/page", timestamp=now, metadata={})
66
66
  g.add_edge("actor1", "forum1", edge_type="MEMBER_OF", confidence=1.0,
67
67
  source_url="", timestamp=now, metadata={})
@@ -122,7 +122,7 @@ class TestModel(unittest.TestCase):
122
122
  def test_edge_types_are_strings(self):
123
123
  et = self.m.EDGE_TYPES
124
124
  for attr in ("CO_APPEARED_ON", "POSTED_BY", "LINKED_TO", "MEMBER_OF",
125
- "USED", "CLAIMED", "LIKELY_SAME_ACTOR", "CONFIRMED_SAME_ACTOR"):
125
+ "USES", "CLAIMED", "LIKELY_SAME_ACTOR", "CONFIRMED_SAME_ACTOR"):
126
126
  self.assertIsInstance(getattr(et, attr), str, msg=f"EDGE_TYPES.{attr} is not str")
127
127
 
128
128
  def test_node_types_no_duplicates(self):
@@ -138,7 +138,7 @@ class TestModel(unittest.TestCase):
138
138
  et = self.m.EDGE_TYPES
139
139
  values = [
140
140
  et.CO_APPEARED_ON, et.POSTED_BY, et.LINKED_TO, et.MEMBER_OF,
141
- et.USED, et.CLAIMED, et.LIKELY_SAME_ACTOR, et.CONFIRMED_SAME_ACTOR,
141
+ et.USES, et.CLAIMED, et.LIKELY_SAME_ACTOR, et.CONFIRMED_SAME_ACTOR,
142
142
  ]
143
143
  self.assertEqual(len(values), len(set(values)), "Duplicate EDGE_TYPES values detected")
144
144
 
@@ -700,9 +700,9 @@ class TestQueries(unittest.TestCase):
700
700
  def test_get_neighbors_edge_type_filter(self):
701
701
  """get_neighbors with edge_type filter only traverses matching edges."""
702
702
  from graph.queries import get_neighbors
703
- result = get_neighbors(self.graph, "actor1", hops=2, edge_types=["USED"])
703
+ result = get_neighbors(self.graph, "actor1", hops=2, edge_types=["USES"])
704
704
 
705
- # Only malware1 should be reachable via USED edge
705
+ # Only malware1 should be reachable via USES edge
706
706
  hop1_ids = {n.node_id for n in result.get("1", [])}
707
707
  self.assertIn("malware1", hop1_ids)
708
708
  # wallet1 should NOT be in hop 1 (it's connected via CO_APPEARED_ON)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: voidaccess
3
- Version: 1.8.0
3
+ Version: 1.8.1
4
4
  Summary: Dark web OSINT CLI — automated threat intelligence from query to report
5
5
  Author: VoidAccess
6
6
  License-Expression: MIT
@@ -1,3 +1,3 @@
1
1
  """voidaccess CLI — dark-web OSINT command-line interface."""
2
2
 
3
- __version__ = "1.8.0"
3
+ __version__ = "1.8.1"
@@ -80,6 +80,27 @@ def run(
80
80
 
81
81
  cli_config.apply_env()
82
82
 
83
+ # Validate the query BEFORE any network activity. An empty/whitespace-only
84
+ # or trivially short query cannot yield real intelligence — accepting it
85
+ # silently produces a report that looks identical to a genuine one but is
86
+ # just noise from an essentially meaningless search. Reject up front.
87
+ MIN_QUERY_LENGTH = 3
88
+ stripped_query = (query or "").strip()
89
+ if not stripped_query:
90
+ console.print(
91
+ "[red]Empty query.[/red] Provide something to investigate, e.g. "
92
+ '[bold]voidaccess investigate "LockBit ransomware"[/bold].'
93
+ )
94
+ raise typer.Exit(code=2)
95
+ if len(stripped_query) < MIN_QUERY_LENGTH:
96
+ console.print(
97
+ f"[red]Query too short.[/red] '{stripped_query}' is only "
98
+ f"{len(stripped_query)} character(s); provide at least {MIN_QUERY_LENGTH} "
99
+ "characters of meaningful context so results reflect a real query, "
100
+ "not generic noise."
101
+ )
102
+ raise typer.Exit(code=2)
103
+
83
104
  # v1.6.2 — --use-scraping-api and --use-proxies are one-shot CLI flags
84
105
  # for activating the two independent clearnet transports in-process
85
106
  # only. The REST API flag sets VOIDACCESS_USE_PROXIES=true; the
@@ -352,10 +373,21 @@ async def _run_investigation(
352
373
  pipeline_metrics = InvestigationMetrics(inv_uuid)
353
374
  set_current(pipeline_metrics)
354
375
 
376
+ # Step-cost instrumentation. ``start``/``finish`` bracket each phase so
377
+ # duration_ms is recorded, and — critically — so ``record_llm_call`` (which
378
+ # only credits steps that are currently *active*) attributes LLM calls to
379
+ # the right step. Mirrors the API pipeline's _metric_start/_metric_finish.
380
+ def _metric_start(step_name: str) -> None:
381
+ pipeline_metrics.start(step_name)
382
+
383
+ def _metric_finish(step_name: str) -> None:
384
+ pipeline_metrics.finish(step_name)
385
+
355
386
  sources_used: dict[str, dict[str, Any]] = {}
356
387
  page_count_by_url: dict[str, dict[str, Any]] = {}
357
388
 
358
389
  # --- Step 1 — refine query -------------------------------------------
390
+ _metric_start("query_refinement")
359
391
  display.update_step("Refining query", "active")
360
392
  refined = query
361
393
  if llm is not None:
@@ -370,8 +402,10 @@ async def _run_investigation(
370
402
  else:
371
403
  display.update_step("Refining query", "skip", "--no-llm")
372
404
  sqlite_adapter.update_investigation(investigation_id, {"refined_query": refined})
405
+ _metric_finish("query_refinement")
373
406
 
374
407
  # --- Step 2 — search fan-out -----------------------------------------
408
+ _metric_start("source_gathering")
375
409
  display.update_step("Searching dark web", "active")
376
410
  search_links: list[dict] = []
377
411
  paste_pages: list[dict] = []
@@ -412,6 +446,14 @@ async def _run_investigation(
412
446
  return []
413
447
 
414
448
  async def _onionsearch():
449
+ # --no-tor means NO Tor traffic for the entire run. Torch and
450
+ # Haystack are .onion search engines fetched over the Tor SOCKS proxy,
451
+ # and they return real .onion result URLs that would otherwise be
452
+ # appended to the scrape list and routed back through Tor. Skipping
453
+ # the lookup entirely is what makes "zero Tor traffic" a direct
454
+ # consequence rather than a downstream filter after Tor was contacted.
455
+ if no_tor:
456
+ return []
415
457
  from sources.engines import search_onionsearch
416
458
  return await search_onionsearch(refined)
417
459
 
@@ -441,6 +483,10 @@ async def _run_investigation(
441
483
  if not os.getenv("TELEGRAM_API_ID", "").strip() or not os.getenv("TELEGRAM_API_HASH", "").strip():
442
484
  sources_used["telegram"] = {"status": "skipped_no_key"}
443
485
  for _engine_name in ("Torch", "Haystack"):
486
+ if no_tor:
487
+ # Onion search engines are never contacted in --no-tor mode.
488
+ sources_used[_engine_name.lower()] = {"status": "skipped", "count": 0}
489
+ continue
444
490
  _count = sum(1 for item in onion_pages if item.get("source") == _engine_name)
445
491
  from sources.engines import get_last_onionsearch_status
446
492
  _engine_status = get_last_onionsearch_status().get(_engine_name, "ok")
@@ -449,6 +495,25 @@ async def _run_investigation(
449
495
  "count": _count,
450
496
  }
451
497
 
498
+ # --no-tor defence-in-depth: even though the Tor search branch and the
499
+ # onion search engines are both skipped above, strip any .onion URL that
500
+ # may have reached the link list from ANY other side-source before it can
501
+ # be handed to the scraper (the scraper correctly routes .onion links
502
+ # through Tor, which is exactly what --no-tor must prevent). This runs
503
+ # BEFORE any scrape, so no Tor connection is ever attempted.
504
+ if no_tor:
505
+ _before = len(search_links)
506
+ search_links = [
507
+ _l for _l in search_links
508
+ if ".onion" not in str(_l.get("link", "")).lower()
509
+ ]
510
+ _stripped = _before - len(search_links)
511
+ if _stripped:
512
+ logger.warning(
513
+ "[no-tor] Stripped %d .onion link(s) from search results before scraping",
514
+ _stripped,
515
+ )
516
+
452
517
  if not no_tor and search_summary:
453
518
  display.update_step(
454
519
  "Searching dark web",
@@ -495,7 +560,10 @@ async def _run_investigation(
495
560
  filtered_links = (search_links or [])[:filter_top_n]
496
561
  display.update_step("Filtering results", "skip" if no_llm else "ok", f"{len(filtered_links)} kept")
497
562
 
563
+ _metric_finish("source_gathering")
564
+
498
565
  # --- Step 4 — scrape pages -------------------------------------------
566
+ _metric_start("scraping")
499
567
  display.update_step("Scraping pages", "active")
500
568
  scraped_pages: list[dict] = []
501
569
  if filtered_links:
@@ -521,11 +589,18 @@ async def _run_investigation(
521
589
  for url, text in results.items():
522
590
  if text:
523
591
  scraped_pages.append({"url": url, "text": text, "source": "tor_search"})
592
+ _fetched = sum(1 for _t in results.values() if _t)
593
+ pipeline_metrics.record_scraping(
594
+ attempted=len(filtered_links),
595
+ fetched=_fetched,
596
+ cache_hits=0,
597
+ )
524
598
  display.update_step("Scraping pages", "ok", f"{len(scraped_pages)} pages")
525
599
  except Exception as exc:
526
600
  display.update_step("Scraping pages", "fail", str(exc))
527
601
  else:
528
602
  display.update_step("Scraping pages", "skip", "no links")
603
+ _metric_finish("scraping")
529
604
 
530
605
  # --- Step 4.1 — discover seeds from scraped pages --------------------
531
606
  # scrape_multiple already submits discovered seeds fire-and-forget, but
@@ -576,6 +651,7 @@ async def _run_investigation(
576
651
  page_count_by_url = {p["url"]: p for p in scraped_pages}
577
652
 
578
653
  # --- Step 5 — extract entities ---------------------------------------
654
+ _metric_start("entity_extraction")
579
655
  display.update_step("Extracting entities", "active")
580
656
  extraction_results = []
581
657
  try:
@@ -591,8 +667,10 @@ async def _run_investigation(
591
667
  display.update_step("Extracting entities", "ok", f"{total_entities} entities")
592
668
  except Exception as exc:
593
669
  display.update_step("Extracting entities", "fail", str(exc))
670
+ _metric_finish("entity_extraction")
594
671
 
595
672
  # --- Step 6 — enrich intelligence (OTX + IP) ---------------------------
673
+ _metric_start("enrichment")
596
674
  display.update_step("Enriching intelligence", "active")
597
675
  enrichment_pages: list[dict] = []
598
676
  try:
@@ -735,6 +813,7 @@ async def _run_investigation(
735
813
  )
736
814
  except Exception as exc:
737
815
  console.print(f"[grey50]Enrichment extraction failed: {exc}[/grey50]")
816
+ _metric_finish("enrichment")
738
817
 
739
818
  # --- Step 6.9 — Update persistent actor profiles (non-blocking) -------
740
819
  # Aggregate THREAT_ACTOR_HANDLE / RANSOMWARE_GROUP entities from the
@@ -770,6 +849,7 @@ async def _run_investigation(
770
849
  # --- Step 7 — build graph (co-occurrence) ----------------------------
771
850
  # Phase 6.2 — capped by CLI_PHASE_TIMEOUTS["graph_build"] so a large
772
851
  # entity table doesn't pin the CLI thread indefinitely.
852
+ _metric_start("graph_build")
773
853
  display.update_step("Building graph", "active")
774
854
  try:
775
855
  edges_written = await asyncio.wait_for(
@@ -798,10 +878,12 @@ async def _run_investigation(
798
878
  )
799
879
  except Exception as exc:
800
880
  logger.debug("Community detection skipped: %s", exc)
881
+ _metric_finish("graph_build")
801
882
 
802
883
  # --- Step 8 — summary -------------------------------------------------
803
884
  # Phase 6.2 — capped by CLI_PHASE_TIMEOUTS["summary"] so a slow LLM
804
885
  # call never holds the CLI process open past its budget.
886
+ _metric_start("summary_generation")
805
887
  display.update_step("Generating summary", "active")
806
888
  summary_text = ""
807
889
  if llm is not None:
@@ -826,8 +908,10 @@ async def _run_investigation(
826
908
  display.update_step("Generating summary", "fail", str(exc))
827
909
  else:
828
910
  display.update_step("Generating summary", "skip", "--no-llm")
911
+ _metric_finish("summary_generation")
829
912
 
830
913
  # --- Step 9 — finalize & write outputs --------------------------------
914
+ _metric_start("finalization")
831
915
  display.update_step("Finalizing results", "active")
832
916
  final_entities = sqlite_adapter.get_entities(investigation_id)
833
917
  final_relationships = sqlite_adapter.get_relationships(investigation_id)
@@ -886,6 +970,7 @@ async def _run_investigation(
886
970
  md_path.write_text(_render_markdown(payload), encoding="utf-8")
887
971
 
888
972
  display.update_step("Finalizing results", "ok")
973
+ _metric_finish("finalization")
889
974
 
890
975
  # Drain any background tasks scheduled during the pipeline (e.g. the
891
976
  # actor-profile aggregator). This keeps the "fire-and-forget"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes