voidaccess 1.8.2__tar.gz → 1.9.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. {voidaccess-1.8.2/voidaccess.egg-info → voidaccess-1.9.1}/PKG-INFO +1 -1
  2. {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/entities.py +43 -11
  3. {voidaccess-1.8.2 → voidaccess-1.9.1}/config.py +11 -18
  4. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/queries.py +2 -2
  5. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/session.py +17 -4
  6. voidaccess-1.9.1/export/_entity_loading.py +46 -0
  7. {voidaccess-1.8.2 → voidaccess-1.9.1}/export/ioc_package.py +26 -16
  8. {voidaccess-1.8.2 → voidaccess-1.9.1}/export/misp.py +8 -17
  9. {voidaccess-1.8.2 → voidaccess-1.9.1}/export/sigma.py +6 -16
  10. {voidaccess-1.8.2 → voidaccess-1.9.1}/export/snort_export.py +3 -12
  11. {voidaccess-1.8.2 → voidaccess-1.9.1}/export/stix.py +27 -62
  12. {voidaccess-1.8.2 → voidaccess-1.9.1}/export/yara_export.py +3 -13
  13. {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/confidence.py +31 -0
  14. voidaccess-1.9.1/extractor/identity.py +151 -0
  15. {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/normalizer.py +34 -7
  16. {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/pipeline.py +5 -3
  17. {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/relationship_extract.py +12 -49
  18. {voidaccess-1.8.2 → voidaccess-1.9.1}/graph/builder.py +14 -5
  19. {voidaccess-1.8.2 → voidaccess-1.9.1}/graph/model.py +52 -0
  20. {voidaccess-1.8.2 → voidaccess-1.9.1}/pyproject.toml +1 -1
  21. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/dns_enrichment.py +2 -1
  22. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_config.py +24 -1
  23. voidaccess-1.9.1/tests/test_consolidated_contracts.py +36 -0
  24. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_graph.py +23 -0
  25. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_llm.py +13 -0
  26. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_stix_export.py +73 -11
  27. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess/config.py +4 -0
  28. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess/llm.py +26 -5
  29. {voidaccess-1.8.2 → voidaccess-1.9.1/voidaccess.egg-info}/PKG-INFO +1 -1
  30. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess.egg-info/SOURCES.txt +3 -0
  31. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/__init__.py +1 -1
  32. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/adapters/sqlite.py +9 -5
  33. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/browser.py +20 -9
  34. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/commands/enrich.py +9 -2
  35. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/commands/investigate.py +140 -27
  36. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/commands/show.py +27 -4
  37. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/config.py +4 -4
  38. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/display.py +33 -0
  39. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/main.py +5 -3
  40. {voidaccess-1.8.2 → voidaccess-1.9.1}/LICENSE +0 -0
  41. {voidaccess-1.8.2 → voidaccess-1.9.1}/README.md +0 -0
  42. {voidaccess-1.8.2 → voidaccess-1.9.1}/analysis/__init__.py +0 -0
  43. {voidaccess-1.8.2 → voidaccess-1.9.1}/analysis/opsec.py +0 -0
  44. {voidaccess-1.8.2 → voidaccess-1.9.1}/analysis/patterns.py +0 -0
  45. {voidaccess-1.8.2 → voidaccess-1.9.1}/analysis/temporal.py +0 -0
  46. {voidaccess-1.8.2 → voidaccess-1.9.1}/api/__init__.py +0 -0
  47. {voidaccess-1.8.2 → voidaccess-1.9.1}/api/auth.py +0 -0
  48. {voidaccess-1.8.2 → voidaccess-1.9.1}/api/errors.py +0 -0
  49. {voidaccess-1.8.2 → voidaccess-1.9.1}/api/main.py +0 -0
  50. {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/__init__.py +0 -0
  51. {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/actors.py +0 -0
  52. {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/admin.py +0 -0
  53. {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/auth.py +0 -0
  54. {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/export.py +0 -0
  55. {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/investigations.py +0 -0
  56. {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/monitors.py +0 -0
  57. {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/search.py +0 -0
  58. {voidaccess-1.8.2 → voidaccess-1.9.1}/api/routes/settings.py +0 -0
  59. {voidaccess-1.8.2 → voidaccess-1.9.1}/auth/__init__.py +0 -0
  60. {voidaccess-1.8.2 → voidaccess-1.9.1}/auth/token_blacklist.py +0 -0
  61. {voidaccess-1.8.2 → voidaccess-1.9.1}/crawler/__init__.py +0 -0
  62. {voidaccess-1.8.2 → voidaccess-1.9.1}/crawler/dedup.py +0 -0
  63. {voidaccess-1.8.2 → voidaccess-1.9.1}/crawler/frontier.py +0 -0
  64. {voidaccess-1.8.2 → voidaccess-1.9.1}/crawler/spider.py +0 -0
  65. {voidaccess-1.8.2 → voidaccess-1.9.1}/crawler/utils.py +0 -0
  66. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/__init__.py +0 -0
  67. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/__init__.py +0 -0
  68. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/env.py +0 -0
  69. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0001_initial_schema.py +0 -0
  70. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0002_add_investigation_status_column.py +0 -0
  71. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0002_add_missing_tables.py +0 -0
  72. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0003_add_canonical_value_and_entity_links.py +0 -0
  73. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0004_add_page_posted_at.py +0 -0
  74. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0005_add_extraction_method.py +0 -0
  75. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0006_add_monitor_alerts.py +0 -0
  76. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0007_add_actor_style_profiles.py +0 -0
  77. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0008_add_users_table.py +0 -0
  78. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0009_add_investigation_id_to_relationships.py +0 -0
  79. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0010_add_composite_index_entity_relationships.py +0 -0
  80. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0011_add_page_extraction_cache.py +0 -0
  81. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0013_add_graph_status.py +0 -0
  82. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0015_add_progress_fields.py +0 -0
  83. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0016_backfill_graph_status.py +0 -0
  84. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0017_add_user_api_keys.py +0 -0
  85. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0018_add_user_id_to_investigations.py +0 -0
  86. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0019_add_content_safety_log.py +0 -0
  87. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0020_add_entity_source_tracking.py +0 -0
  88. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0021_add_search_engine_stats.py +0 -0
  89. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0022_add_actor_profiles.py +0 -0
  90. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0023_add_investigation_metadata.py +0 -0
  91. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0024_add_investigation_step_metrics.py +0 -0
  92. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/migrations/versions/0025_add_cancellation_requested.py +0 -0
  93. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/models.py +0 -0
  94. {voidaccess-1.8.2 → voidaccess-1.9.1}/db/search_engine_stats.py +0 -0
  95. {voidaccess-1.8.2 → voidaccess-1.9.1}/export/__init__.py +0 -0
  96. {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/__init__.py +0 -0
  97. {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/entity_shape.py +0 -0
  98. {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/gazetteer.py +0 -0
  99. {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/llm_extract.py +0 -0
  100. {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/ner.py +0 -0
  101. {voidaccess-1.8.2 → voidaccess-1.9.1}/extractor/regex_patterns.py +0 -0
  102. {voidaccess-1.8.2 → voidaccess-1.9.1}/fingerprint/__init__.py +0 -0
  103. {voidaccess-1.8.2 → voidaccess-1.9.1}/fingerprint/calibration.py +0 -0
  104. {voidaccess-1.8.2 → voidaccess-1.9.1}/fingerprint/profiler.py +0 -0
  105. {voidaccess-1.8.2 → voidaccess-1.9.1}/fingerprint/stylometry.py +0 -0
  106. {voidaccess-1.8.2 → voidaccess-1.9.1}/graph/__init__.py +0 -0
  107. {voidaccess-1.8.2 → voidaccess-1.9.1}/graph/export.py +0 -0
  108. {voidaccess-1.8.2 → voidaccess-1.9.1}/graph/queries.py +0 -0
  109. {voidaccess-1.8.2 → voidaccess-1.9.1}/graph/visualize.py +0 -0
  110. {voidaccess-1.8.2 → voidaccess-1.9.1}/i18n/__init__.py +0 -0
  111. {voidaccess-1.8.2 → voidaccess-1.9.1}/i18n/detect.py +0 -0
  112. {voidaccess-1.8.2 → voidaccess-1.9.1}/i18n/query_expand.py +0 -0
  113. {voidaccess-1.8.2 → voidaccess-1.9.1}/i18n/translate.py +0 -0
  114. {voidaccess-1.8.2 → voidaccess-1.9.1}/monitor/__init__.py +0 -0
  115. {voidaccess-1.8.2 → voidaccess-1.9.1}/monitor/_db.py +0 -0
  116. {voidaccess-1.8.2 → voidaccess-1.9.1}/monitor/alerts.py +0 -0
  117. {voidaccess-1.8.2 → voidaccess-1.9.1}/monitor/config.py +0 -0
  118. {voidaccess-1.8.2 → voidaccess-1.9.1}/monitor/diff.py +0 -0
  119. {voidaccess-1.8.2 → voidaccess-1.9.1}/monitor/jobs.py +0 -0
  120. {voidaccess-1.8.2 → voidaccess-1.9.1}/monitor/scheduler.py +0 -0
  121. {voidaccess-1.8.2 → voidaccess-1.9.1}/scraper/__init__.py +0 -0
  122. {voidaccess-1.8.2 → voidaccess-1.9.1}/scraper/scrape.py +0 -0
  123. {voidaccess-1.8.2 → voidaccess-1.9.1}/scraper/scrape_js.py +0 -0
  124. {voidaccess-1.8.2 → voidaccess-1.9.1}/search/__init__.py +0 -0
  125. {voidaccess-1.8.2 → voidaccess-1.9.1}/search/circuit_breaker.py +0 -0
  126. {voidaccess-1.8.2 → voidaccess-1.9.1}/search/engine_catalog.py +0 -0
  127. {voidaccess-1.8.2 → voidaccess-1.9.1}/search/query_builder.py +0 -0
  128. {voidaccess-1.8.2 → voidaccess-1.9.1}/search/search.py +0 -0
  129. {voidaccess-1.8.2 → voidaccess-1.9.1}/setup.cfg +0 -0
  130. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/__init__.py +0 -0
  131. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/actor_profiles.py +0 -0
  132. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/blockchain.py +0 -0
  133. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/breach_lookup.py +0 -0
  134. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/cache.py +0 -0
  135. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/cisa.py +0 -0
  136. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/data/__init__.py +0 -0
  137. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/data/onion_seeds.json +0 -0
  138. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/domain_reputation.py +0 -0
  139. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/email_reputation.py +0 -0
  140. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/engines.py +0 -0
  141. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/enrichment.py +0 -0
  142. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/github_scraper.py +0 -0
  143. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/gitlab_scraper.py +0 -0
  144. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/hash_reputation.py +0 -0
  145. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/historical_intel.py +0 -0
  146. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/infostealer.py +0 -0
  147. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/ip_reputation.py +0 -0
  148. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/nvd.py +0 -0
  149. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/paste_scraper.py +0 -0
  150. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/pastes.py +0 -0
  151. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/proxy_client.py +0 -0
  152. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/rss_scraper.py +0 -0
  153. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/seed_manager.py +0 -0
  154. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/seeds.py +0 -0
  155. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/shodan.py +0 -0
  156. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/telegram.py +0 -0
  157. {voidaccess-1.8.2 → voidaccess-1.9.1}/sources/virustotal.py +0 -0
  158. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_analysis_opsec.py +0 -0
  159. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_analysis_stylometry.py +0 -0
  160. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_analysis_temporal.py +0 -0
  161. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_api.py +0 -0
  162. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_api_monitors.py +0 -0
  163. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_blockchain.py +0 -0
  164. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_breach_lookup.py +0 -0
  165. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_cli_proxy_config.py +0 -0
  166. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_confidence_calibration.py +0 -0
  167. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_crawler.py +0 -0
  168. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_db.py +0 -0
  169. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_dns_enrichment.py +0 -0
  170. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_domain_reputation.py +0 -0
  171. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_email_reputation.py +0 -0
  172. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_extractor_llm_pipeline.py +0 -0
  173. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_filter_e2e_manual.py +0 -0
  174. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_filter_parser_manual.py +0 -0
  175. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_fingerprint.py +0 -0
  176. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_github_scraper.py +0 -0
  177. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_gitlab_scraper.py +0 -0
  178. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_graph_integration.py +0 -0
  179. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_hash_reputation.py +0 -0
  180. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_i18n.py +0 -0
  181. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_infostealer.py +0 -0
  182. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_ip_reputation.py +0 -0
  183. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_llm_utils.py +0 -0
  184. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_model_singleton.py +0 -0
  185. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_monitor.py +0 -0
  186. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_monitor_integration.py +0 -0
  187. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_nvd.py +0 -0
  188. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_pagination.py +0 -0
  189. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_paste_scraper.py +0 -0
  190. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_proxy_client.py +0 -0
  191. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_ransomlook.py +0 -0
  192. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_regex_patterns.py +0 -0
  193. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_relationship_extract.py +0 -0
  194. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_rss_scraper.py +0 -0
  195. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_scrape_js.py +0 -0
  196. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_settings.py +0 -0
  197. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_snort_export.py +0 -0
  198. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_sources.py +0 -0
  199. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_sources_enrichment_new.py +0 -0
  200. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_vector.py +0 -0
  201. {voidaccess-1.8.2 → voidaccess-1.9.1}/tests/test_yara_export.py +0 -0
  202. {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/__init__.py +0 -0
  203. {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/async_utils.py +0 -0
  204. {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/content_dedup.py +0 -0
  205. {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/content_safety.py +0 -0
  206. {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/defang.py +0 -0
  207. {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/encryption.py +0 -0
  208. {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/enrichment_cache.py +0 -0
  209. {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/investigation_metrics.py +0 -0
  210. {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/ioc_freshness.py +0 -0
  211. {voidaccess-1.8.2 → voidaccess-1.9.1}/utils/user_keys.py +0 -0
  212. {voidaccess-1.8.2 → voidaccess-1.9.1}/vector/__init__.py +0 -0
  213. {voidaccess-1.8.2 → voidaccess-1.9.1}/vector/embedder.py +0 -0
  214. {voidaccess-1.8.2 → voidaccess-1.9.1}/vector/model_singleton.py +0 -0
  215. {voidaccess-1.8.2 → voidaccess-1.9.1}/vector/search.py +0 -0
  216. {voidaccess-1.8.2 → voidaccess-1.9.1}/vector/store.py +0 -0
  217. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess/__init__.py +0 -0
  218. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess/llm_utils.py +0 -0
  219. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess.egg-info/dependency_links.txt +0 -0
  220. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess.egg-info/entry_points.txt +0 -0
  221. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess.egg-info/requires.txt +0 -0
  222. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess.egg-info/top_level.txt +0 -0
  223. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/adapters/__init__.py +0 -0
  224. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/commands/__init__.py +0 -0
  225. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/commands/actors.py +0 -0
  226. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/commands/configure.py +0 -0
  227. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/commands/export.py +0 -0
  228. {voidaccess-1.8.2 → voidaccess-1.9.1}/voidaccess_cli/tor_detect.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: voidaccess
3
- Version: 1.8.2
3
+ Version: 1.9.1
4
4
  Summary: Dark web OSINT CLI — automated threat intelligence from query to report
5
5
  Author: VoidAccess
6
6
  License-Expression: MIT
@@ -22,6 +22,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query
22
22
  from fastapi.responses import Response
23
23
  from api.auth import CurrentUser, get_current_user
24
24
  from api.errors import GENERIC_ERROR_MESSAGE, log_exception
25
+ from extractor.identity import entity_canonical_id
25
26
 
26
27
  logger = logging.getLogger(__name__)
27
28
  router = APIRouter()
@@ -321,7 +322,9 @@ async def get_stylometry_analysis(
321
322
  if entity is None:
322
323
  raise HTTPException(status_code=404, detail="Entity not found")
323
324
 
324
- canonical = entity.canonical_value or entity.value.lower()
325
+ # Canonical dedup key via the shared identity module (was an inline
326
+ # `entity.canonical_value or entity.value.lower()` re-derivation).
327
+ canonical = entity_canonical_id(entity)
325
328
 
326
329
  related = (
327
330
  session.query(Entity)
@@ -406,7 +409,7 @@ async def get_stylometry_analysis(
406
409
  similar_actors = await asyncio.to_thread(
407
410
  _find_similar_actors,
408
411
  profile=profile,
409
- canonical_value=entity.canonical_value or entity.value.lower(),
412
+ canonical_value=entity_canonical_id(entity),
410
413
  entity_type=entity.entity_type,
411
414
  )
412
415
  except Exception as e:
@@ -474,7 +477,9 @@ async def get_opsec_analysis(
474
477
  if entity is None:
475
478
  raise HTTPException(status_code=404, detail="Entity not found")
476
479
 
477
- canonical = entity.canonical_value or entity.value.lower()
480
+ # Canonical dedup key via the shared identity module (was an inline
481
+ # `entity.canonical_value or entity.value.lower()` re-derivation).
482
+ canonical = entity_canonical_id(entity)
478
483
  related = (
479
484
  session.query(Entity)
480
485
  .filter(
@@ -827,17 +832,44 @@ def _get_entity_value(entity_id: str) -> Optional[str]:
827
832
  return None
828
833
 
829
834
 
830
- def _resolve_graph_node_id(graph, entity_value: str) -> Optional[str]:
831
- """Resolve graph node by exact value, then by handle@domain prefix."""
835
+ def _resolve_graph_node_id(
836
+ graph, entity_value: str, entity_type: Optional[str] = None
837
+ ) -> Optional[str]:
838
+ """Resolve a graph node id for an entity value.
839
+
840
+ The graph is keyed by canonical identity (extractor.identity.entity_graph_id).
841
+ When ``entity_type`` is known, the lookup value is canonicalised through the
842
+ same shared module so it matches the node regardless of the casing the value
843
+ was observed in — closing the case-sensitivity gap the audit flagged.
844
+ Without a type it falls back to the historical exact-then-``handle@domain``
845
+ prefix behaviour.
846
+ """
832
847
  if graph is None:
833
848
  return None
834
- if graph.has_node(entity_value):
835
- return entity_value
836
849
 
837
- prefix = f"{entity_value}@"
838
- for node_id in graph.nodes:
839
- if isinstance(node_id, str) and node_id.startswith(prefix):
840
- return node_id
850
+ candidates = [entity_value]
851
+ if entity_type:
852
+ try:
853
+ from types import SimpleNamespace # noqa: PLC0415
854
+
855
+ canon = entity_canonical_id(
856
+ SimpleNamespace(entity_type=entity_type, value=entity_value)
857
+ )
858
+ if canon and canon not in candidates:
859
+ candidates.append(canon)
860
+ except Exception:
861
+ pass
862
+
863
+ # 1) exact match on any candidate
864
+ for cand in candidates:
865
+ if graph.has_node(cand):
866
+ return cand
867
+ # 2) handle@domain prefix match on any candidate
868
+ for cand in candidates:
869
+ prefix = f"{cand}@"
870
+ for node_id in graph.nodes:
871
+ if isinstance(node_id, str) and node_id.startswith(prefix):
872
+ return node_id
841
873
  return None
842
874
 
843
875
 
@@ -5,7 +5,7 @@ from dotenv import load_dotenv
5
5
  load_dotenv()
6
6
 
7
7
  logger = logging.getLogger(__name__)
8
- _OPTIONAL_CONFIG_WARNING_SHOWN = False
8
+ _OPTIONAL_CONFIG_WARNING_STATE = "_voidaccess_optional_config_warning_shown"
9
9
 
10
10
 
11
11
  def _clean_env(name, default=None):
@@ -151,16 +151,12 @@ except ValueError:
151
151
  BLOCKCYPHER_TOKEN = _clean_env("BLOCKCYPHER_TOKEN", "")
152
152
  ETHERSCAN_API_KEY = _clean_env("ETHERSCAN_API_KEY", "")
153
153
 
154
- # Auth — REQUIRED in production. Generate with: python -c "import secrets; print(secrets.token_hex(32))"
155
- _jwt_secret = _clean_env("JWT_SECRET")
156
- if _jwt_secret is None:
157
- raise RuntimeError(
158
- "JWT_SECRET environment variable is not set. "
159
- "Generate a secure secret with: python -c \"import secrets; print(secrets.token_hex(32))\" "
160
- "and set it as JWT_SECRET in your .env file. "
161
- "Do NOT use a random secret — it will change on restart and invalidate all issued tokens."
162
- )
163
- JWT_SECRET = _jwt_secret
154
+ # Auth — REQUIRED when the API/auth stack is actually used. Do not validate
155
+ # this at import time: the CLI injects its runtime environment in its entry
156
+ # point before importing dependent modules, and `--help`/`--version` must work
157
+ # on a fresh install. `validate_config()` and the API startup path enforce it
158
+ # at the point where authentication is needed.
159
+ JWT_SECRET = _clean_env("JWT_SECRET")
164
160
 
165
161
  # Token blacklist Redis (optional — omit to disable blacklist checks)
166
162
  REDIS_URL = _clean_env("REDIS_URL")
@@ -210,7 +206,6 @@ OPTIONAL_KEYS = [
210
206
 
211
207
 
212
208
  def validate_config():
213
- global _OPTIONAL_CONFIG_WARNING_SHOWN
214
209
  missing_required = []
215
210
  for key in REQUIRED_KEYS:
216
211
  if _clean_env(key) is None:
@@ -221,16 +216,14 @@ def validate_config():
221
216
  key for key in OPTIONAL_KEYS
222
217
  if (_clean_env(key) is None or _clean_env(key) == "")
223
218
  ]
224
- if missing_optional and not _OPTIONAL_CONFIG_WARNING_SHOWN:
219
+ if missing_optional and not getattr(logging, _OPTIONAL_CONFIG_WARNING_STATE, False):
225
220
  logger.warning(
226
221
  "Optional configuration keys not set (%d): %s - related features will be disabled",
227
222
  len(missing_optional),
228
223
  ", ".join(missing_optional),
229
224
  )
230
- _OPTIONAL_CONFIG_WARNING_SHOWN = True
231
-
232
-
233
- validate_config()
234
-
225
+ # Store the guard on the process-wide logging module so compatibility
226
+ # wrappers/reloads cannot reset it and reintroduce warning spam.
227
+ setattr(logging, _OPTIONAL_CONFIG_WARNING_STATE, True)
235
228
 
236
229
 
@@ -348,8 +348,8 @@ def upsert_entity_canonical(
348
348
 
349
349
  if existing:
350
350
  # Update confidence if new extraction is more confident
351
- if confidence > existing.confidence:
352
- existing.confidence = confidence
351
+ from extractor.confidence import get_entity_confidence
352
+ existing.confidence = get_entity_confidence(existing, confidence)
353
353
  # Update context if we have a better snippet
354
354
  if context_snippet and len(context_snippet) > len(existing.context_snippet or ""):
355
355
  existing.context_snippet = context_snippet
@@ -38,6 +38,7 @@ Usage (testing — pass an explicit URL to avoid needing DATABASE_URL in env)
38
38
  from __future__ import annotations
39
39
 
40
40
  from functools import lru_cache
41
+ import os
41
42
 
42
43
  from contextlib import asynccontextmanager, contextmanager
43
44
  from typing import AsyncGenerator, Generator, Optional
@@ -48,7 +49,19 @@ from sqlalchemy.orm import sessionmaker, Session
48
49
 
49
50
  import config
50
51
 
52
+ # Kept as a compatibility/testing hook. Production lookups below use the
53
+ # current environment so apply_env() can run before the first DB operation.
51
54
  DATABASE_URL = config.DATABASE_URL
55
+ _INITIAL_DATABASE_URL = DATABASE_URL
56
+
57
+
58
+ def _database_url() -> Optional[str]:
59
+ # Preserve the long-standing test/embedding hook of monkeypatching this
60
+ # module's DATABASE_URL. Otherwise, read the live environment so a caller
61
+ # can inject DATABASE_URL after db.session has been imported.
62
+ if DATABASE_URL != _INITIAL_DATABASE_URL:
63
+ return DATABASE_URL
64
+ return os.getenv("DATABASE_URL") or DATABASE_URL
52
65
  _async_engine_cache: dict[str, "AsyncEngine"] = {}
53
66
 
54
67
 
@@ -104,7 +117,7 @@ def get_engine(url: Optional[str] = None) -> Engine:
104
117
  PostgreSQL gets a connection pool tuned for the scraping workload.
105
118
  SQLite skips pool parameters that only apply to QueuePool.
106
119
  """
107
- target_url = url or DATABASE_URL
120
+ target_url = url or _database_url()
108
121
  if not target_url:
109
122
  raise RuntimeError(
110
123
  "DATABASE_URL is not configured.\n"
@@ -122,7 +135,7 @@ def release_engine(url: Optional[str] = None) -> None:
122
135
  Calls engine.dispose() to release connection pool resources and file handles,
123
136
  then clears the cache. Use this in test teardown to prevent leaks.
124
137
  """
125
- target_url = url or DATABASE_URL
138
+ target_url = url or _database_url()
126
139
  if target_url:
127
140
  try:
128
141
  engine = get_engine(target_url)
@@ -139,7 +152,7 @@ def get_async_engine(url: Optional[str] = None) -> "AsyncEngine":
139
152
  Converts postgresql:// to postgresql+asyncpg:// and sqlite:// to sqlite+aiosqlite://.
140
153
  """
141
154
 
142
- target_url = url or DATABASE_URL
155
+ target_url = url or _database_url()
143
156
  if not target_url:
144
157
  raise RuntimeError(
145
158
  "DATABASE_URL is not configured.\n"
@@ -185,7 +198,7 @@ def release_async_engine(url: Optional[str] = None) -> None:
185
198
 
186
199
  Calls engine.dispose() to release connection pool resources and file handles.
187
200
  """
188
- target_url = url or DATABASE_URL
201
+ target_url = url or _database_url()
189
202
  if target_url in _async_engine_cache:
190
203
  _async_engine_cache[target_url].dispose()
191
204
  del _async_engine_cache[target_url]
@@ -0,0 +1,46 @@
1
+ """
2
+ export/_entity_loading.py — one shared DB-row → NormalizedEntity mapping for
3
+ every exporter.
4
+
5
+ Previously STIX, MISP, and Sigma each reconstructed a ``NormalizedEntity`` from
6
+ a DB ``Entity`` row independently, and populated *different* fields — STIX set
7
+ ``source_quality`` and ``extraction_method``, MISP and Sigma did not. A future
8
+ consumer reading ``entity.source_quality`` therefore got different behaviour
9
+ depending on which export path produced the entity (audit finding: STIX/MISP
10
+ field divergence). Centralising the construction here closes that drift: all
11
+ three exporters now produce identically-populated entities.
12
+
13
+ The value mapping is preserved exactly as it was in all three call sites
14
+ (``canonical_value or value``) so export outputs are unchanged; only the
15
+ previously-inconsistent auxiliary fields are now uniform.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ from typing import Any
21
+
22
+ from extractor.normalizer import NormalizedEntity
23
+
24
+
25
+ def normalized_entity_from_db_row(e: Any) -> NormalizedEntity:
26
+ """Build a NormalizedEntity from a DB Entity row with consistent fields.
27
+
28
+ Used by every exporter (STIX / MISP / Sigma) so field population can never
29
+ drift between export paths again.
30
+ """
31
+ source_url = ""
32
+ try:
33
+ if e.page:
34
+ source_url = e.page.url or ""
35
+ except Exception:
36
+ pass
37
+ return NormalizedEntity(
38
+ entity_type=e.entity_type,
39
+ value=e.canonical_value or e.value,
40
+ confidence=e.confidence,
41
+ source_url=source_url,
42
+ page_id=e.page_id,
43
+ context_snippet=e.context_snippet or "",
44
+ extraction_method="db",
45
+ source_quality=getattr(e, "source_quality", 1.0),
46
+ )
@@ -45,8 +45,11 @@ import logging
45
45
  import re
46
46
  import zipfile
47
47
  from datetime import datetime, timezone
48
+ from types import SimpleNamespace
48
49
  from typing import Any, Iterable, Optional
49
50
 
51
+ from extractor.identity import entity_canonical_id
52
+
50
53
  logger = logging.getLogger(__name__)
51
54
 
52
55
  # ---------------------------------------------------------------------------
@@ -54,7 +57,7 @@ logger = logging.getLogger(__name__)
54
57
  # ---------------------------------------------------------------------------
55
58
 
56
59
  PACKAGE_FORMAT = "voidaccess-ioc-v1"
57
- PACKAGE_VERSION = "1.8.2"
60
+ PACKAGE_VERSION = "1.9.1"
58
61
  SNORT_SID_BASE = 9000001 # 9xxxxxx reserved for VoidAccess-generated rules
59
62
 
60
63
  # Map extractor upper-case entity types to the per-file bucket they belong to.
@@ -138,19 +141,28 @@ def _entity_type(e: Any) -> str:
138
141
 
139
142
 
140
143
  def _entity_value(e: Any) -> str:
141
- """Return the entity's primary value, preferring canonical_value."""
144
+ """Return the entity's canonical value via the shared identity module.
145
+
146
+ Migrated (audit finding 1.3 / 3.9 item 2) from the old
147
+ ``entity.canonical_value or entity.value`` pattern to call
148
+ ``extractor.identity.entity_canonical_id`` directly. Phase 1 already fixed
149
+ ``NormalizedEntity.canonical_value`` so that old pattern no longer returned
150
+ the raw value for pipeline-stage entities — but this makes the choice
151
+ explicit at the call site (one shared function, not an assumption about
152
+ what a property returns), so DB-backed and pipeline-stage versions of the
153
+ same entity can never produce inconsistent casing here again.
154
+ """
155
+ etype = _entity_type(e)
142
156
  if isinstance(e, dict):
143
- return (
144
- e.get("canonical_value")
145
- or e.get("value")
146
- or e.get("canonical")
147
- or ""
148
- ) or ""
149
- return (
150
- getattr(e, "canonical_value", None)
151
- or getattr(e, "value", None)
152
- or ""
153
- ) or ""
157
+ raw = e.get("value") or e.get("canonical_value") or e.get("canonical") or ""
158
+ else:
159
+ raw = getattr(e, "value", None) or getattr(e, "canonical_value", None) or ""
160
+ if not raw:
161
+ return ""
162
+ try:
163
+ return entity_canonical_id(SimpleNamespace(entity_type=etype, value=raw))
164
+ except Exception:
165
+ return raw
154
166
 
155
167
 
156
168
  def _entity_confidence(e: Any) -> float:
@@ -730,9 +742,7 @@ class _DictEntityAdapter:
730
742
 
731
743
  @property
732
744
  def canonical_value(self) -> str:
733
- if isinstance(self._d, dict):
734
- return self._d.get("canonical_value") or self._d.get("value") or ""
735
- return ""
745
+ return entity_canonical_id(self._d)
736
746
 
737
747
  @property
738
748
  def confidence(self) -> float:
@@ -209,23 +209,14 @@ def _load_investigation_and_entities(
209
209
  want = frozenset(filter_uuids)
210
210
  db_entities = [e for e in db_entities if e.id in want]
211
211
 
212
- normalized: list[NormalizedEntity] = []
213
- for e in db_entities:
214
- source_url = ""
215
- try:
216
- if e.page:
217
- source_url = e.page.url or ""
218
- except Exception:
219
- pass
220
- ne = NormalizedEntity(
221
- entity_type=e.entity_type,
222
- value=e.canonical_value or e.value,
223
- confidence=e.confidence,
224
- source_url=source_url,
225
- page_id=e.page_id,
226
- context_snippet=e.context_snippet or "",
227
- )
228
- normalized.append(ne)
212
+ from export._entity_loading import normalized_entity_from_db_row # noqa: PLC0415
213
+
214
+ # Shared row→NormalizedEntity mapping (same one STIX/Sigma use), so
215
+ # auxiliary fields (source_quality, extraction_method) no longer
216
+ # differ between export paths.
217
+ normalized: list[NormalizedEntity] = [
218
+ normalized_entity_from_db_row(e) for e in db_entities
219
+ ]
229
220
 
230
221
  session.expunge_all()
231
222
  return investigation, normalized
@@ -307,22 +307,12 @@ def _load_entities_for_investigation(investigation_id: Any) -> list[Any]:
307
307
  .all()
308
308
  )
309
309
 
310
- result: list[NormalizedEntity] = []
311
- for e in db_entities:
312
- source_url = ""
313
- try:
314
- if e.page:
315
- source_url = e.page.url or ""
316
- except Exception:
317
- pass
318
- result.append(NormalizedEntity(
319
- entity_type=e.entity_type,
320
- value=e.canonical_value or e.value,
321
- confidence=e.confidence,
322
- source_url=source_url,
323
- page_id=e.page_id,
324
- context_snippet=e.context_snippet or "",
325
- ))
310
+ from export._entity_loading import normalized_entity_from_db_row # noqa: PLC0415
311
+
312
+ # Shared row→NormalizedEntity mapping (same one STIX/MISP use).
313
+ result: list[NormalizedEntity] = [
314
+ normalized_entity_from_db_row(e) for e in db_entities
315
+ ]
326
316
  return result
327
317
 
328
318
  except Exception as exc:
@@ -44,6 +44,8 @@ from datetime import datetime, timezone
44
44
  from typing import Any
45
45
  from urllib.parse import urlparse
46
46
 
47
+ from extractor.identity import entity_canonical_id
48
+
47
49
  # ---------------------------------------------------------------------------
48
50
  # Constants
49
51
  # ---------------------------------------------------------------------------
@@ -104,18 +106,7 @@ def _entity_type(e: Any) -> str:
104
106
 
105
107
 
106
108
  def _entity_value(e: Any) -> str:
107
- if isinstance(e, dict):
108
- return (
109
- e.get("canonical_value")
110
- or e.get("value")
111
- or e.get("canonical")
112
- or ""
113
- ) or ""
114
- return (
115
- getattr(e, "canonical_value", None)
116
- or getattr(e, "value", None)
117
- or ""
118
- ) or ""
109
+ return entity_canonical_id(e)
119
110
 
120
111
 
121
112
  def _entity_confidence(e: Any) -> float:
@@ -22,6 +22,9 @@ import warnings
22
22
  from typing import Any, Optional
23
23
  import uuid
24
24
 
25
+ from graph.model import RELATIONSHIP_TYPE_STIX
26
+ from extractor.confidence import get_entity_confidence
27
+
25
28
  logger = logging.getLogger(__name__)
26
29
 
27
30
  # MED-9: suppress SQLAlchemy SAWarning from ORM IN-clause coercion internals.
@@ -127,7 +130,7 @@ def entity_to_stix_indicator(entity: Any) -> Optional[Any]:
127
130
  pattern=pattern,
128
131
  pattern_type="stix",
129
132
  indicator_types=indicator_types,
130
- confidence=_to_stix_confidence(entity.confidence),
133
+ confidence=_to_stix_confidence(get_entity_confidence(entity)),
131
134
  allow_custom=True,
132
135
  x_voidaccess_source_quality=getattr(entity, "source_quality", 1.0),
133
136
  external_references=(
@@ -158,7 +161,7 @@ def entity_to_stix_malware(entity: Any) -> Optional[Any]:
158
161
  malware = stix2.Malware(
159
162
  name=entity.value,
160
163
  is_family=True,
161
- confidence=_to_stix_confidence(entity.confidence),
164
+ confidence=_to_stix_confidence(get_entity_confidence(entity)),
162
165
  allow_custom=True,
163
166
  x_voidaccess_source_quality=getattr(entity, "source_quality", 1.0),
164
167
  external_references=(
@@ -189,7 +192,7 @@ def entity_to_stix_threat_actor(entity: Any) -> Optional[Any]:
189
192
  threat_actor = stix2.ThreatActor(
190
193
  name=entity.value,
191
194
  aliases=[entity.value],
192
- confidence=_to_stix_confidence(entity.confidence),
195
+ confidence=_to_stix_confidence(get_entity_confidence(entity)),
193
196
  allow_custom=True,
194
197
  x_voidaccess_source_quality=getattr(entity, "source_quality", 1.0),
195
198
  external_references=(
@@ -241,30 +244,28 @@ def investigation_to_stix_bundle(
241
244
  if not entities:
242
245
  return _empty_bundle()
243
246
 
244
- try:
245
- from graph.builder import _make_node_id # noqa: PLC0415
246
- except Exception: # pragma: no cover — graph layer optional
247
- def _make_node_id(entity_type, value, source_url): # type: ignore
248
- return value
247
+ from extractor.identity import entity_canonical_id, entity_graph_id # noqa: PLC0415
249
248
 
250
249
  stix_objects: list[Any] = []
251
- # Maps BOTH the raw entity.value AND the graph node_id → stix_object.id.
252
- # Graph edges are keyed by node_id (which for THREAT_ACTOR_HANDLE is
253
- # "value@domain", not the bare value), so registering only entity.value
254
- # here silently dropped every threat-actor relationship — the exact
255
- # "silently dropped relationships" failure mode the CHANGELOG keeps
256
- # re-fixing. Registering the node_id alias lets typed edges (which very
257
- # often have a threat-actor endpoint) survive into the bundle.
250
+ # Maps BOTH the graph node id AND the bare canonical value → stix_object.id.
251
+ # Graph edges are keyed by the graph node id (which for THREAT_ACTOR_HANDLE
252
+ # is "<canonical>@domain", not the bare value); registering only the bare
253
+ # value silently dropped every threat-actor relationship — the exact
254
+ # "silently dropped relationships" failure the CHANGELOG keeps re-fixing.
255
+ #
256
+ # Both keys are now derived through extractor.identity, the *same* module
257
+ # the graph builder uses to generate node ids. The two sides can no longer
258
+ # diverge (e.g. "LockBit" vs "lockbit") because they are literally the same
259
+ # function call, not two functions that happen to agree today.
258
260
  stix_id_map: dict[str, str] = {}
259
261
 
260
262
  def _register(entity: Any, stix_id: str) -> None:
261
- stix_id_map.setdefault(entity.value, stix_id)
262
263
  try:
263
- node_id = _make_node_id(
264
- entity.entity_type,
265
- entity.value,
266
- getattr(entity, "source_url", "") or "",
267
- )
264
+ stix_id_map.setdefault(entity_canonical_id(entity), stix_id)
265
+ except Exception:
266
+ stix_id_map.setdefault(entity.value, stix_id)
267
+ try:
268
+ node_id = entity_graph_id(entity)
268
269
  except Exception:
269
270
  node_id = entity.value
270
271
  if node_id:
@@ -381,6 +382,7 @@ def _load_entities_for_investigation(
381
382
  from db.queries import get_investigation_by_id_or_run # noqa: PLC0415
382
383
  from db.models import Entity, InvestigationEntityLink # noqa: PLC0415
383
384
  from extractor.normalizer import NormalizedEntity # noqa: PLC0415
385
+ from export._entity_loading import normalized_entity_from_db_row # noqa: PLC0415
384
386
 
385
387
  inv_uuid = _coerce_uuid(investigation_id)
386
388
  if inv_uuid is None:
@@ -407,24 +409,9 @@ def _load_entities_for_investigation(
407
409
  want = frozenset(entity_ids)
408
410
  db_entities = [e for e in db_entities if e.id in want]
409
411
 
410
- result: list[NormalizedEntity] = []
411
- for e in db_entities:
412
- source_url = ""
413
- try:
414
- if e.page:
415
- source_url = e.page.url or ""
416
- except Exception:
417
- pass
418
- result.append(NormalizedEntity(
419
- entity_type=e.entity_type,
420
- value=e.canonical_value or e.value,
421
- confidence=e.confidence,
422
- source_url=source_url,
423
- page_id=e.page_id,
424
- context_snippet=e.context_snippet or "",
425
- extraction_method="db",
426
- source_quality=getattr(e, "source_quality", 1.0),
427
- ))
412
+ result: list[NormalizedEntity] = [
413
+ normalized_entity_from_db_row(e) for e in db_entities
414
+ ]
428
415
  return result
429
416
 
430
417
  if session is not None:
@@ -501,29 +488,7 @@ def _build_stix_relationships(
501
488
 
502
489
  def _edge_type_to_stix(edge_type: str) -> str:
503
490
  """Map VoidAccess graph edge types to STIX relationship type strings."""
504
- mapping = {
505
- "CO_APPEARED_ON": "related-to",
506
- "CO_INVESTIGATION": "related-to",
507
- "POSTED_BY": "attributed-to",
508
- "LINKED_TO": "related-to",
509
- "PAID_TO": "related-to",
510
- "MEMBER_OF": "member-of",
511
- "USES": "uses",
512
- "CLAIMED": "attributed-to",
513
- "LIKELY_SAME_ACTOR": "related-to",
514
- "CONFIRMED_SAME_ACTOR": "related-to",
515
- "FUNDED_BY": "related-to",
516
- # Typed relationships from the LLM relationship-extraction pass. These
517
- # map to documented STIX 2.1 common relationship types where one
518
- # exists; CONTROLS has no standard STIX verb so it degrades to the
519
- # safe "related-to" default rather than emitting a non-spec type.
520
- "DROPS": "drops",
521
- "TARGETS": "targets",
522
- "EXPLOITS": "exploits",
523
- "COMMUNICATES_WITH": "communicates-with",
524
- "CONTROLS": "related-to",
525
- }
526
- return mapping.get(edge_type, "related-to")
491
+ return RELATIONSHIP_TYPE_STIX.get(edge_type, "related-to")
527
492
 
528
493
 
529
494
  def _coerce_uuid(value: Any) -> Optional[uuid.UUID]:
@@ -35,6 +35,8 @@ import re
35
35
  from datetime import datetime, timezone
36
36
  from typing import Any, Optional
37
37
 
38
+ from extractor.identity import entity_canonical_id
39
+
38
40
  # ---------------------------------------------------------------------------
39
41
  # Constants
40
42
  # ---------------------------------------------------------------------------
@@ -115,19 +117,7 @@ def _entity_type(e: Any) -> str:
115
117
 
116
118
 
117
119
  def _entity_value(e: Any) -> str:
118
- """Return the entity's primary value, preferring canonical_value."""
119
- if isinstance(e, dict):
120
- return (
121
- e.get("canonical_value")
122
- or e.get("value")
123
- or e.get("canonical")
124
- or ""
125
- ) or ""
126
- return (
127
- getattr(e, "canonical_value", None)
128
- or getattr(e, "value", None)
129
- or ""
130
- ) or ""
120
+ return entity_canonical_id(e)
131
121
 
132
122
 
133
123
  def _entity_confidence(e: Any) -> float: