voidaccess 1.4.0__tar.gz → 1.4.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. {voidaccess-1.4.0 → voidaccess-1.4.3}/PKG-INFO +2 -1
  2. {voidaccess-1.4.0 → voidaccess-1.4.3}/auth/token_blacklist.py +10 -3
  3. {voidaccess-1.4.0 → voidaccess-1.4.3}/pyproject.toml +2 -1
  4. {voidaccess-1.4.0 → voidaccess-1.4.3}/search/circuit_breaker.py +10 -3
  5. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/enrichment.py +25 -0
  6. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/hash_reputation.py +18 -4
  7. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess.egg-info/PKG-INFO +2 -1
  8. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess.egg-info/requires.txt +1 -0
  9. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess_cli/__init__.py +1 -1
  10. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess_cli/commands/configure.py +16 -19
  11. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess_cli/commands/export.py +7 -3
  12. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess_cli/commands/investigate.py +40 -0
  13. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess_cli/commands/show.py +22 -0
  14. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess_cli/config.py +71 -5
  15. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess_cli/main.py +22 -10
  16. {voidaccess-1.4.0 → voidaccess-1.4.3}/LICENSE +0 -0
  17. {voidaccess-1.4.0 → voidaccess-1.4.3}/README.md +0 -0
  18. {voidaccess-1.4.0 → voidaccess-1.4.3}/analysis/__init__.py +0 -0
  19. {voidaccess-1.4.0 → voidaccess-1.4.3}/analysis/opsec.py +0 -0
  20. {voidaccess-1.4.0 → voidaccess-1.4.3}/analysis/patterns.py +0 -0
  21. {voidaccess-1.4.0 → voidaccess-1.4.3}/analysis/temporal.py +0 -0
  22. {voidaccess-1.4.0 → voidaccess-1.4.3}/api/__init__.py +0 -0
  23. {voidaccess-1.4.0 → voidaccess-1.4.3}/api/auth.py +0 -0
  24. {voidaccess-1.4.0 → voidaccess-1.4.3}/api/main.py +0 -0
  25. {voidaccess-1.4.0 → voidaccess-1.4.3}/api/routes/__init__.py +0 -0
  26. {voidaccess-1.4.0 → voidaccess-1.4.3}/api/routes/admin.py +0 -0
  27. {voidaccess-1.4.0 → voidaccess-1.4.3}/api/routes/auth.py +0 -0
  28. {voidaccess-1.4.0 → voidaccess-1.4.3}/api/routes/entities.py +0 -0
  29. {voidaccess-1.4.0 → voidaccess-1.4.3}/api/routes/export.py +0 -0
  30. {voidaccess-1.4.0 → voidaccess-1.4.3}/api/routes/investigations.py +0 -0
  31. {voidaccess-1.4.0 → voidaccess-1.4.3}/api/routes/monitors.py +0 -0
  32. {voidaccess-1.4.0 → voidaccess-1.4.3}/api/routes/search.py +0 -0
  33. {voidaccess-1.4.0 → voidaccess-1.4.3}/api/routes/settings.py +0 -0
  34. {voidaccess-1.4.0 → voidaccess-1.4.3}/auth/__init__.py +0 -0
  35. {voidaccess-1.4.0 → voidaccess-1.4.3}/config.py +0 -0
  36. {voidaccess-1.4.0 → voidaccess-1.4.3}/crawler/__init__.py +0 -0
  37. {voidaccess-1.4.0 → voidaccess-1.4.3}/crawler/dedup.py +0 -0
  38. {voidaccess-1.4.0 → voidaccess-1.4.3}/crawler/frontier.py +0 -0
  39. {voidaccess-1.4.0 → voidaccess-1.4.3}/crawler/spider.py +0 -0
  40. {voidaccess-1.4.0 → voidaccess-1.4.3}/crawler/utils.py +0 -0
  41. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/__init__.py +0 -0
  42. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/__init__.py +0 -0
  43. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/env.py +0 -0
  44. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0001_initial_schema.py +0 -0
  45. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0002_add_investigation_status_column.py +0 -0
  46. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0002_add_missing_tables.py +0 -0
  47. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0003_add_canonical_value_and_entity_links.py +0 -0
  48. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0004_add_page_posted_at.py +0 -0
  49. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0005_add_extraction_method.py +0 -0
  50. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0006_add_monitor_alerts.py +0 -0
  51. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0007_add_actor_style_profiles.py +0 -0
  52. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0008_add_users_table.py +0 -0
  53. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0009_add_investigation_id_to_relationships.py +0 -0
  54. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0010_add_composite_index_entity_relationships.py +0 -0
  55. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0011_add_page_extraction_cache.py +0 -0
  56. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0013_add_graph_status.py +0 -0
  57. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0015_add_progress_fields.py +0 -0
  58. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0016_backfill_graph_status.py +0 -0
  59. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0017_add_user_api_keys.py +0 -0
  60. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0018_add_user_id_to_investigations.py +0 -0
  61. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0019_add_content_safety_log.py +0 -0
  62. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/migrations/versions/0020_add_entity_source_tracking.py +0 -0
  63. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/models.py +0 -0
  64. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/queries.py +0 -0
  65. {voidaccess-1.4.0 → voidaccess-1.4.3}/db/session.py +0 -0
  66. {voidaccess-1.4.0 → voidaccess-1.4.3}/export/__init__.py +0 -0
  67. {voidaccess-1.4.0 → voidaccess-1.4.3}/export/misp.py +0 -0
  68. {voidaccess-1.4.0 → voidaccess-1.4.3}/export/sigma.py +0 -0
  69. {voidaccess-1.4.0 → voidaccess-1.4.3}/export/stix.py +0 -0
  70. {voidaccess-1.4.0 → voidaccess-1.4.3}/extractor/__init__.py +0 -0
  71. {voidaccess-1.4.0 → voidaccess-1.4.3}/extractor/llm_extract.py +0 -0
  72. {voidaccess-1.4.0 → voidaccess-1.4.3}/extractor/ner.py +0 -0
  73. {voidaccess-1.4.0 → voidaccess-1.4.3}/extractor/normalizer.py +0 -0
  74. {voidaccess-1.4.0 → voidaccess-1.4.3}/extractor/pipeline.py +0 -0
  75. {voidaccess-1.4.0 → voidaccess-1.4.3}/extractor/regex_patterns.py +0 -0
  76. {voidaccess-1.4.0 → voidaccess-1.4.3}/fingerprint/__init__.py +0 -0
  77. {voidaccess-1.4.0 → voidaccess-1.4.3}/fingerprint/profiler.py +0 -0
  78. {voidaccess-1.4.0 → voidaccess-1.4.3}/fingerprint/stylometry.py +0 -0
  79. {voidaccess-1.4.0 → voidaccess-1.4.3}/graph/__init__.py +0 -0
  80. {voidaccess-1.4.0 → voidaccess-1.4.3}/graph/builder.py +0 -0
  81. {voidaccess-1.4.0 → voidaccess-1.4.3}/graph/export.py +0 -0
  82. {voidaccess-1.4.0 → voidaccess-1.4.3}/graph/model.py +0 -0
  83. {voidaccess-1.4.0 → voidaccess-1.4.3}/graph/queries.py +0 -0
  84. {voidaccess-1.4.0 → voidaccess-1.4.3}/graph/visualize.py +0 -0
  85. {voidaccess-1.4.0 → voidaccess-1.4.3}/i18n/__init__.py +0 -0
  86. {voidaccess-1.4.0 → voidaccess-1.4.3}/i18n/detect.py +0 -0
  87. {voidaccess-1.4.0 → voidaccess-1.4.3}/i18n/query_expand.py +0 -0
  88. {voidaccess-1.4.0 → voidaccess-1.4.3}/i18n/translate.py +0 -0
  89. {voidaccess-1.4.0 → voidaccess-1.4.3}/monitor/__init__.py +0 -0
  90. {voidaccess-1.4.0 → voidaccess-1.4.3}/monitor/_db.py +0 -0
  91. {voidaccess-1.4.0 → voidaccess-1.4.3}/monitor/alerts.py +0 -0
  92. {voidaccess-1.4.0 → voidaccess-1.4.3}/monitor/config.py +0 -0
  93. {voidaccess-1.4.0 → voidaccess-1.4.3}/monitor/diff.py +0 -0
  94. {voidaccess-1.4.0 → voidaccess-1.4.3}/monitor/jobs.py +0 -0
  95. {voidaccess-1.4.0 → voidaccess-1.4.3}/monitor/scheduler.py +0 -0
  96. {voidaccess-1.4.0 → voidaccess-1.4.3}/scraper/__init__.py +0 -0
  97. {voidaccess-1.4.0 → voidaccess-1.4.3}/scraper/scrape.py +0 -0
  98. {voidaccess-1.4.0 → voidaccess-1.4.3}/scraper/scrape_js.py +0 -0
  99. {voidaccess-1.4.0 → voidaccess-1.4.3}/search/__init__.py +0 -0
  100. {voidaccess-1.4.0 → voidaccess-1.4.3}/search/search.py +0 -0
  101. {voidaccess-1.4.0 → voidaccess-1.4.3}/setup.cfg +0 -0
  102. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/__init__.py +0 -0
  103. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/blockchain.py +0 -0
  104. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/cache.py +0 -0
  105. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/cisa.py +0 -0
  106. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/dns_enrichment.py +0 -0
  107. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/domain_reputation.py +0 -0
  108. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/email_reputation.py +0 -0
  109. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/engines.py +0 -0
  110. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/github_scraper.py +0 -0
  111. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/gitlab_scraper.py +0 -0
  112. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/historical_intel.py +0 -0
  113. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/ip_reputation.py +0 -0
  114. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/paste_scraper.py +0 -0
  115. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/pastes.py +0 -0
  116. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/rss_scraper.py +0 -0
  117. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/seed_manager.py +0 -0
  118. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/seeds.py +0 -0
  119. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/shodan.py +0 -0
  120. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/telegram.py +0 -0
  121. {voidaccess-1.4.0 → voidaccess-1.4.3}/sources/virustotal.py +0 -0
  122. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_analysis_opsec.py +0 -0
  123. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_analysis_stylometry.py +0 -0
  124. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_analysis_temporal.py +0 -0
  125. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_api.py +0 -0
  126. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_api_monitors.py +0 -0
  127. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_blockchain.py +0 -0
  128. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_config.py +0 -0
  129. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_crawler.py +0 -0
  130. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_db.py +0 -0
  131. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_dns_enrichment.py +0 -0
  132. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_domain_reputation.py +0 -0
  133. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_email_reputation.py +0 -0
  134. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_fingerprint.py +0 -0
  135. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_github_scraper.py +0 -0
  136. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_gitlab_scraper.py +0 -0
  137. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_graph.py +0 -0
  138. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_hash_reputation.py +0 -0
  139. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_i18n.py +0 -0
  140. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_ip_reputation.py +0 -0
  141. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_llm.py +0 -0
  142. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_llm_utils.py +0 -0
  143. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_model_singleton.py +0 -0
  144. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_monitor.py +0 -0
  145. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_pagination.py +0 -0
  146. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_paste_scraper.py +0 -0
  147. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_rss_scraper.py +0 -0
  148. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_scrape_js.py +0 -0
  149. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_settings.py +0 -0
  150. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_sources.py +0 -0
  151. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_sources_enrichment_new.py +0 -0
  152. {voidaccess-1.4.0 → voidaccess-1.4.3}/tests/test_vector.py +0 -0
  153. {voidaccess-1.4.0 → voidaccess-1.4.3}/utils/__init__.py +0 -0
  154. {voidaccess-1.4.0 → voidaccess-1.4.3}/utils/async_utils.py +0 -0
  155. {voidaccess-1.4.0 → voidaccess-1.4.3}/utils/content_safety.py +0 -0
  156. {voidaccess-1.4.0 → voidaccess-1.4.3}/utils/defang.py +0 -0
  157. {voidaccess-1.4.0 → voidaccess-1.4.3}/utils/encryption.py +0 -0
  158. {voidaccess-1.4.0 → voidaccess-1.4.3}/utils/ioc_freshness.py +0 -0
  159. {voidaccess-1.4.0 → voidaccess-1.4.3}/utils/user_keys.py +0 -0
  160. {voidaccess-1.4.0 → voidaccess-1.4.3}/vector/__init__.py +0 -0
  161. {voidaccess-1.4.0 → voidaccess-1.4.3}/vector/embedder.py +0 -0
  162. {voidaccess-1.4.0 → voidaccess-1.4.3}/vector/model_singleton.py +0 -0
  163. {voidaccess-1.4.0 → voidaccess-1.4.3}/vector/search.py +0 -0
  164. {voidaccess-1.4.0 → voidaccess-1.4.3}/vector/store.py +0 -0
  165. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess/__init__.py +0 -0
  166. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess/llm.py +0 -0
  167. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess/llm_utils.py +0 -0
  168. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess.egg-info/SOURCES.txt +0 -0
  169. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess.egg-info/dependency_links.txt +0 -0
  170. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess.egg-info/entry_points.txt +0 -0
  171. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess.egg-info/top_level.txt +0 -0
  172. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess_cli/adapters/__init__.py +0 -0
  173. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess_cli/adapters/sqlite.py +0 -0
  174. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess_cli/browser.py +0 -0
  175. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess_cli/commands/__init__.py +0 -0
  176. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess_cli/commands/enrich.py +0 -0
  177. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess_cli/display.py +0 -0
  178. {voidaccess-1.4.0 → voidaccess-1.4.3}/voidaccess_cli/tor_detect.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: voidaccess
3
- Version: 1.4.0
3
+ Version: 1.4.3
4
4
  Summary: Dark web OSINT CLI — automated threat intelligence from query to report
5
5
  Author: VoidAccess
6
6
  License-Expression: MIT
@@ -28,6 +28,7 @@ Requires-Dist: langchain-openai>=0.1
28
28
  Requires-Dist: langchain-anthropic>=0.1
29
29
  Requires-Dist: langchain-google-genai>=1.0
30
30
  Requires-Dist: langchain-groq>=0.1
31
+ Requires-Dist: langchain-ollama>=0.1
31
32
  Requires-Dist: python-dotenv>=1.0
32
33
  Requires-Dist: httpx>=0.27
33
34
  Requires-Dist: spacy>=3.7
@@ -19,16 +19,22 @@ logger = logging.getLogger(__name__)
19
19
  _pool: Optional[redis.ConnectionPool] = None
20
20
  _redis_client: Optional[redis.Redis] = None
21
21
  _blacklist_enabled = False
22
+ _redis_unavailable = False
22
23
 
23
24
  BLACKLIST_PREFIX = "blacklist:"
24
25
 
25
26
 
26
27
  async def _get_redis() -> Optional[redis.Redis]:
27
- global _pool, _redis_client, _blacklist_enabled
28
+ global _pool, _redis_client, _blacklist_enabled, _redis_unavailable
29
+
30
+ if _redis_unavailable:
31
+ return None
28
32
 
29
33
  if REDIS_URL is None:
30
34
  _blacklist_enabled = False
31
- logger.warning("REDIS_URL not configured - token blacklist disabled")
35
+ if not _redis_unavailable:
36
+ logger.info("REDIS_URL not configured — token blacklist disabled")
37
+ _redis_unavailable = True
32
38
  return None
33
39
 
34
40
  if _redis_client is None:
@@ -42,9 +48,10 @@ async def _get_redis() -> Optional[redis.Redis]:
42
48
  _blacklist_enabled = True
43
49
  logger.info("Token blacklist enabled via Redis")
44
50
  except Exception as e:
45
- logger.warning(f"Failed to connect to Redis: %s - token blacklist disabled", e)
51
+ logger.warning("Failed to connect to Redis: %s token blacklist disabled", e)
46
52
  _redis_client = None
47
53
  _blacklist_enabled = False
54
+ _redis_unavailable = True
48
55
 
49
56
  return _redis_client
50
57
 
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "voidaccess"
7
- version = "1.4.0"
7
+ version = "1.4.3"
8
8
  description = "Dark web OSINT CLI — automated threat intelligence from query to report"
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -34,6 +34,7 @@ dependencies = [
34
34
  "langchain-anthropic>=0.1",
35
35
  "langchain-google-genai>=1.0",
36
36
  "langchain-groq>=0.1",
37
+ "langchain-ollama>=0.1",
37
38
  "python-dotenv>=1.0",
38
39
  "httpx>=0.27",
39
40
  "spacy>=3.7",
@@ -29,6 +29,7 @@ CIRCUIT_PREFIX = "circuit:"
29
29
  _pool: Optional[redis.ConnectionPool] = None
30
30
  _redis_client: Optional[redis.Redis] = None
31
31
  _circuit_breaker_enabled = False
32
+ _redis_unavailable = False # latched True once we decide Redis isn't reachable
32
33
 
33
34
  _engine_failures: dict[str, int] = {}
34
35
  _engine_last_success: dict[str, float] = {}
@@ -37,11 +38,16 @@ _engine_open_time: dict[str, float] = {}
37
38
 
38
39
 
39
40
  async def _get_redis() -> Optional[redis.Redis]:
40
- global _pool, _redis_client, _circuit_breaker_enabled
41
+ global _pool, _redis_client, _circuit_breaker_enabled, _redis_unavailable
42
+
43
+ if _redis_unavailable:
44
+ return None
41
45
 
42
46
  if REDIS_URL is None:
43
47
  _circuit_breaker_enabled = False
44
- logger.warning("REDIS_URL not configured - circuit breaker using in-memory fallback")
48
+ if not _redis_unavailable:
49
+ logger.info("REDIS_URL not configured — circuit breaker using in-memory fallback")
50
+ _redis_unavailable = True
45
51
  return None
46
52
 
47
53
  if _redis_client is None:
@@ -55,9 +61,10 @@ async def _get_redis() -> Optional[redis.Redis]:
55
61
  _circuit_breaker_enabled = True
56
62
  logger.info("Circuit breaker enabled via Redis")
57
63
  except Exception as e:
58
- logger.warning(f"Failed to connect to Redis: %s - circuit breaker using in-memory fallback", e)
64
+ logger.warning("Failed to connect to Redis: %s circuit breaker using in-memory fallback", e)
59
65
  _redis_client = None
60
66
  _circuit_breaker_enabled = False
67
+ _redis_unavailable = True
61
68
 
62
69
  return _redis_client
63
70
 
@@ -40,11 +40,30 @@ THREATFOX_URL = "https://threatfox-api.abuse.ch/api/v1/"
40
40
  # All HTTP calls use at most 30s client timeout (enforced per request).
41
41
 
42
42
 
43
+ _ABUSECH_WARNED = False
44
+
45
+
43
46
  def _abusech_headers() -> dict[str, str]:
44
47
  key = (os.environ.get("ABUSECH_API_KEY") or "").strip()
45
48
  return {"Auth-Key": key} if key else {}
46
49
 
47
50
 
51
+ def _abusech_enabled() -> bool:
52
+ """abuse.ch APIs (MalwareBazaar/ThreatFox/URLhaus) require an Auth-Key
53
+ since 2024. Return False (and log once) when the key is missing so we
54
+ skip the request entirely instead of spamming HTTP 401."""
55
+ global _ABUSECH_WARNED
56
+ if (os.environ.get("ABUSECH_API_KEY") or "").strip():
57
+ return True
58
+ if not _ABUSECH_WARNED:
59
+ logger.info(
60
+ "abuse.ch enrichment skipped — set ABUSECH_API_KEY "
61
+ "(free at https://auth.abuse.ch) to enable MalwareBazaar/ThreatFox/URLhaus."
62
+ )
63
+ _ABUSECH_WARNED = True
64
+ return False
65
+
66
+
48
67
  def is_onion_url(url: str) -> bool:
49
68
  """
50
69
  Return True if *url* looks like a Tor hidden service URL (.onion).
@@ -218,6 +237,8 @@ def otx_pulse_to_page(pulse: dict) -> dict:
218
237
 
219
238
  async def fetch_malwarebazaar(query: str, limit: int = 20) -> list[dict]:
220
239
  """Query MalwareBazaar by tag then by signature."""
240
+ if not _abusech_enabled():
241
+ return []
221
242
  results: list[dict] = []
222
243
  q = (query or "").strip()
223
244
  if not q:
@@ -312,6 +333,8 @@ async def fetch_malwarebazaar(query: str, limit: int = 20) -> list[dict]:
312
333
 
313
334
  async def fetch_threatfox(query: str, limit: int = 50) -> list[dict]:
314
335
  """Search ThreatFox IOCs by search term."""
336
+ if not _abusech_enabled():
337
+ return []
315
338
  results: list[dict] = []
316
339
  q = (query or "").strip()
317
340
  if not q:
@@ -397,6 +420,8 @@ async def fetch_threatfox(query: str, limit: int = 50) -> list[dict]:
397
420
 
398
421
  async def fetch_urlhaus(query: str, limit: int = 20) -> list[dict]:
399
422
  """Search URLhaus by tag."""
423
+ if not _abusech_enabled():
424
+ return []
400
425
  results: list[dict] = []
401
426
  q = (query or "").strip()
402
427
  if not q:
@@ -164,11 +164,18 @@ async def query_malwarebazaar(hash_value: str) -> dict[str, Any]:
164
164
  """
165
165
  POST get_info to MalwareBazaar for a file hash.
166
166
 
167
- No API key required. Returns malware family, file type, first seen date.
167
+ Requires ABUSECH_API_KEY (abuse.ch made auth-key mandatory in 2024).
168
+ Returns malware family, file type, first seen date.
168
169
  """
170
+ api_key = (os.environ.get("ABUSECH_API_KEY") or "").strip()
171
+ if not api_key:
172
+ return {"found": False, "source": "malwarebazaar_no_key"}
169
173
  try:
170
174
  timeout = aiohttp.ClientTimeout(total=15)
171
- headers = {"User-Agent": "VoidAccess-OSINT/1.1 (security research)"}
175
+ headers = {
176
+ "User-Agent": "VoidAccess-OSINT/1.1 (security research)",
177
+ "Auth-Key": api_key,
178
+ }
172
179
  async with aiohttp.ClientSession(timeout=timeout, headers=headers) as session:
173
180
  async with session.post(
174
181
  MALWAREBAZAAR_URL,
@@ -212,11 +219,18 @@ async def query_threatfox(hash_value: str) -> dict[str, Any]:
212
219
  """
213
220
  POST search_ioc to ThreatFox for a file hash.
214
221
 
215
- No API key required. Returns malware family and associated IOCs.
222
+ Requires ABUSECH_API_KEY (abuse.ch made auth-key mandatory in 2024).
223
+ Returns malware family and associated IOCs.
216
224
  """
225
+ api_key = (os.environ.get("ABUSECH_API_KEY") or "").strip()
226
+ if not api_key:
227
+ return {"found": False, "source": "threatfox_no_key"}
217
228
  try:
218
229
  timeout = aiohttp.ClientTimeout(total=15)
219
- headers = {"User-Agent": "VoidAccess-OSINT/1.1 (security research)"}
230
+ headers = {
231
+ "User-Agent": "VoidAccess-OSINT/1.1 (security research)",
232
+ "Auth-Key": api_key,
233
+ }
220
234
  async with aiohttp.ClientSession(timeout=timeout, headers=headers) as session:
221
235
  async with session.post(
222
236
  THREATFOX_URL,
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: voidaccess
3
- Version: 1.4.0
3
+ Version: 1.4.3
4
4
  Summary: Dark web OSINT CLI — automated threat intelligence from query to report
5
5
  Author: VoidAccess
6
6
  License-Expression: MIT
@@ -28,6 +28,7 @@ Requires-Dist: langchain-openai>=0.1
28
28
  Requires-Dist: langchain-anthropic>=0.1
29
29
  Requires-Dist: langchain-google-genai>=1.0
30
30
  Requires-Dist: langchain-groq>=0.1
31
+ Requires-Dist: langchain-ollama>=0.1
31
32
  Requires-Dist: python-dotenv>=1.0
32
33
  Requires-Dist: httpx>=0.27
33
34
  Requires-Dist: spacy>=3.7
@@ -10,6 +10,7 @@ langchain-openai>=0.1
10
10
  langchain-anthropic>=0.1
11
11
  langchain-google-genai>=1.0
12
12
  langchain-groq>=0.1
13
+ langchain-ollama>=0.1
13
14
  python-dotenv>=1.0
14
15
  httpx>=0.27
15
16
  spacy>=3.7
@@ -1,3 +1,3 @@
1
1
  """voidaccess CLI — dark-web OSINT command-line interface."""
2
2
 
3
- __version__ = "1.3.0"
3
+ __version__ = "1.4.1"
@@ -57,8 +57,23 @@ def _test_llm_key(provider: str, api_key: str, model: str) -> bool:
57
57
  return True
58
58
  try:
59
59
  from voidaccess.llm import get_llm
60
+ except ImportError as exc:
61
+ missing = str(exc).split("'")[-2] if "'" in str(exc) else str(exc)
62
+ console.print(
63
+ f"[yellow]Skipped validation:[/yellow] missing dependency [bold]{missing}[/bold]. "
64
+ f"Install with: [bold]pip install {missing.replace('_', '-')}[/bold]"
65
+ )
66
+ return False
67
+ try:
60
68
  get_llm(model, api_keys={cli_config.PROVIDER_ENV.get(provider, ""): api_key})
61
69
  return True
70
+ except ImportError as exc:
71
+ missing = str(exc).split("'")[-2] if "'" in str(exc) else str(exc)
72
+ console.print(
73
+ f"[yellow]Skipped validation:[/yellow] missing dependency [bold]{missing}[/bold]. "
74
+ f"Install with: [bold]pip install {missing.replace('_', '-')}[/bold]"
75
+ )
76
+ return False
62
77
  except Exception as exc:
63
78
  console.print(f"[yellow]Could not validate key:[/yellow] {exc}")
64
79
  return False
@@ -117,25 +132,7 @@ def _prompt_output_dir(cfg: dict) -> None:
117
132
 
118
133
 
119
134
  def _ensure_spacy_model() -> None:
120
- console.print("\n → Downloading spaCy NER model...")
121
- try:
122
- import subprocess
123
- import sys
124
-
125
- result = subprocess.run(
126
- [sys.executable, "-m", "spacy", "download", "en_core_web_sm"],
127
- capture_output=True,
128
- text=True,
129
- )
130
- if result.returncode == 0:
131
- console.print(" ✓ spaCy model ready")
132
- else:
133
- console.print(
134
- " ⚠ spaCy download failed — run manually: "
135
- "python -m spacy download en_core_web_sm"
136
- )
137
- except Exception as e:
138
- console.print(f" ⚠ spaCy: {e}")
135
+ cli_config.ensure_spacy_model()
139
136
 
140
137
 
141
138
  @app.callback()
@@ -39,7 +39,7 @@ def run(
39
39
  raise typer.Exit(code=1)
40
40
 
41
41
  payload, suffix = _render(fmt, inv_id, data)
42
- out_path = output or _default_out_path(target, suffix)
42
+ out_path = output or _default_out_path(target, suffix, fmt=fmt)
43
43
  out_path = Path(out_path).expanduser()
44
44
  out_path.parent.mkdir(parents=True, exist_ok=True)
45
45
  if isinstance(payload, bytes):
@@ -150,9 +150,13 @@ def _flatten_for_md(data: dict) -> dict:
150
150
  return data
151
151
 
152
152
 
153
- def _default_out_path(target: str, suffix: str) -> Path:
153
+ def _default_out_path(target: str, suffix: str, fmt: str = "") -> Path:
154
154
  p = Path(target).expanduser()
155
155
  if p.exists():
156
- return p.with_suffix(suffix)
156
+ candidate = p.with_suffix(suffix)
157
+ # Avoid overwriting input when suffix is the same (e.g. stix/misp .json)
158
+ if candidate == p and fmt and fmt not in ("json",):
159
+ return p.parent / f"{p.stem}-{fmt}{suffix}"
160
+ return candidate
157
161
  from voidaccess_cli import config as cli_config
158
162
  return cli_config.get_output_dir() / f"{target}{suffix}"
@@ -49,9 +49,31 @@ def run(
49
49
  from voidaccess_cli import config as cli_config
50
50
 
51
51
  cli_config.apply_env()
52
+
53
+ try:
54
+ import spacy
55
+ spacy.load("en_core_web_sm")
56
+ except Exception:
57
+ import subprocess, sys
58
+ from rich.console import Console
59
+ Console().print(
60
+ " [dim]→[/dim] Installing spaCy NER model (one-time)..."
61
+ )
62
+ subprocess.run(
63
+ [sys.executable, "-m", "spacy",
64
+ "download", "en_core_web_sm"],
65
+ capture_output=True
66
+ )
67
+
52
68
  if quiet:
53
69
  logging.getLogger().setLevel(logging.ERROR)
54
70
 
71
+ from utils.content_safety import is_blocked_query
72
+ blocked, reason = is_blocked_query(query)
73
+ if blocked:
74
+ console.print(f"[red]Query blocked:[/red] {reason}")
75
+ raise typer.Exit(code=1)
76
+
55
77
  if not cli_config.is_configured() and not no_llm:
56
78
  console.print("[yellow]No LLM configured.[/yellow] Run [bold]voidaccess configure[/bold] first, or pass --no-llm.")
57
79
  raise typer.Exit(code=2)
@@ -382,6 +404,7 @@ async def _run_investigation(
382
404
  "query": query,
383
405
  "refined_query": refined,
384
406
  "model_used": chosen_model if llm is not None else None,
407
+ "status": "completed" if final_entities or scraped_pages else "completed_no_results",
385
408
  "created_at": datetime.now(timezone.utc).isoformat(),
386
409
  "summary": summary_text,
387
410
  "sources_used": sources_used,
@@ -414,6 +437,23 @@ async def _run_investigation(
414
437
  }
415
438
  )
416
439
 
440
+ # Close any cached aiohttp sessions so the event loop exits cleanly
441
+ # (otherwise aiohttp prints "Unclosed client session" warnings).
442
+ await _close_cached_sessions()
443
+
444
+
445
+ async def _close_cached_sessions() -> None:
446
+ try:
447
+ from scraper.scrape import close_cached_sessions as _close_scrape
448
+ await _close_scrape()
449
+ except Exception:
450
+ pass
451
+ try:
452
+ from search import close_search_session as _close_search
453
+ await _close_search()
454
+ except Exception:
455
+ pass
456
+
417
457
 
418
458
  # ---------------------------------------------------------------------------
419
459
  # Side-source helpers (each gracefully degrades if module missing/disabled)
@@ -24,6 +24,7 @@ def run(
24
24
  target: Optional[str] = typer.Argument(
25
25
  None, help="Investigation id or path to a .json export"
26
26
  ),
27
+ no_tui: bool = typer.Option(False, "--no-tui", help="Print summary table without launching TUI (for scripted use)."),
27
28
  ) -> None:
28
29
  """Open the entity browser TUI."""
29
30
  from voidaccess_cli import config as cli_config
@@ -32,6 +33,9 @@ def run(
32
33
  data: Optional[dict] = None
33
34
 
34
35
  if target is None:
36
+ if no_tui:
37
+ console.print("[yellow]No target specified.[/yellow]")
38
+ raise typer.Exit(code=1)
35
39
  target = _pick_recent()
36
40
  if target is None:
37
41
  console.print("[yellow]No investigations found. Run `voidaccess investigate` first.[/yellow]")
@@ -49,11 +53,29 @@ def run(
49
53
  console.print(f"[red]Unknown investigation:[/red] {target}")
50
54
  raise typer.Exit(code=1)
51
55
 
56
+ if no_tui:
57
+ _print_summary(data)
58
+ return
59
+
52
60
  from voidaccess_cli.browser import EntityBrowserApp
53
61
  app = EntityBrowserApp(data=data)
54
62
  app.run()
55
63
 
56
64
 
65
+ def _print_summary(data: dict) -> None:
66
+ inv = data.get("investigation") or data
67
+ entities = data.get("entities", [])
68
+ table = Table(title="Investigation summary")
69
+ table.add_column("Field", style="bold")
70
+ table.add_column("Value")
71
+ table.add_row("Query", str(inv.get("query") or ""))
72
+ table.add_row("Status", str(inv.get("status") or ""))
73
+ table.add_row("Entities", str(len(entities)))
74
+ table.add_row("Created", str(inv.get("created_at") or "")[:19])
75
+ table.add_row("Summary", (str(inv.get("summary") or "—"))[:120])
76
+ console.print(table)
77
+
78
+
57
79
  def _pick_recent() -> Optional[str]:
58
80
  from voidaccess_cli.adapters import sqlite as sqlite_adapter
59
81
  sqlite_adapter.init_db()
@@ -139,6 +139,65 @@ def db_url() -> str:
139
139
  return f"sqlite:///{DB_PATH.as_posix()}"
140
140
 
141
141
 
142
+ def ensure_spacy_model(model_name: str = "en_core_web_sm") -> bool:
143
+ """
144
+ Ensure spaCy NER model is installed. Returns True if model is loadable
145
+ after this call. Handles PEP 668 (externally-managed-environment) on
146
+ Debian/Ubuntu/Kali by setting PIP_BREAK_SYSTEM_PACKAGES=1, and uses
147
+ PIP_USER=1 outside virtualenvs.
148
+
149
+ Prints progress via rich. Safe to call repeatedly.
150
+ """
151
+ import sys
152
+ import subprocess
153
+
154
+ try:
155
+ import spacy
156
+ spacy.load(model_name)
157
+ return True
158
+ except Exception:
159
+ pass
160
+
161
+ from rich.console import Console
162
+ con = Console()
163
+ con.print(f" [dim]→[/dim] Installing spaCy NER model [bold]{model_name}[/bold] (one-time)...")
164
+
165
+ env = dict(os.environ)
166
+ env["PIP_BREAK_SYSTEM_PACKAGES"] = "1"
167
+ in_venv = sys.prefix != getattr(sys, "base_prefix", sys.prefix)
168
+ if not in_venv:
169
+ env["PIP_USER"] = "1"
170
+
171
+ result = subprocess.run(
172
+ [sys.executable, "-m", "spacy", "download", model_name],
173
+ capture_output=True,
174
+ text=True,
175
+ env=env,
176
+ )
177
+
178
+ if result.returncode == 0:
179
+ try:
180
+ import importlib
181
+ import spacy as _spacy # noqa: F401
182
+ importlib.invalidate_caches()
183
+ import spacy as _spacy2
184
+ _spacy2.load(model_name)
185
+ con.print(f" [green]✓[/green] spaCy model ready")
186
+ return True
187
+ except Exception:
188
+ pass
189
+
190
+ err_tail = (result.stderr or result.stdout or "").strip().splitlines()[-3:]
191
+ con.print(f" [yellow]⚠[/yellow] spaCy install failed (exit {result.returncode}) — NER will be skipped.")
192
+ for line in err_tail:
193
+ con.print(f" [dim]{line}[/dim]")
194
+ con.print(
195
+ f" Run manually: [bold]PIP_BREAK_SYSTEM_PACKAGES=1 "
196
+ f"{os.path.basename(sys.executable)} -m spacy download {model_name}[/bold]"
197
+ )
198
+ return False
199
+
200
+
142
201
  def apply_env(config: Optional[dict[str, Any]] = None) -> None:
143
202
  """
144
203
  Push saved config into os.environ so that the existing voidaccess
@@ -154,11 +213,12 @@ def apply_env(config: Optional[dict[str, Any]] = None) -> None:
154
213
  os.environ.setdefault("PLAYWRIGHT_ENABLED", "false")
155
214
 
156
215
  def _set_env_if_present(key: str, value: Any, *, clear_if_empty: bool = False) -> None:
157
- text = str(value).strip() if value is not None else ""
158
- if text:
159
- os.environ[key] = text
160
- elif clear_if_empty:
161
- os.environ.pop(key, None)
216
+ text = str(value) if value is not None else ""
217
+ if not text or not text.strip():
218
+ if clear_if_empty:
219
+ os.environ.pop(key, None)
220
+ return
221
+ os.environ[key] = text.strip()
162
222
 
163
223
  # Tor proxy
164
224
  _set_env_if_present("TOR_PROXY_HOST", cfg.get("tor", {}).get("host", "127.0.0.1"))
@@ -178,3 +238,9 @@ def apply_env(config: Optional[dict[str, Any]] = None) -> None:
178
238
  # Enrichment keys
179
239
  for k, v in (cfg.get("enrichment_keys") or {}).items():
180
240
  _set_env_if_present(k, v, clear_if_empty=True)
241
+
242
+ # Keyless APIs (ThreatFox/URLhaus/MalwareBazaar/abuse.ch) must never
243
+ # receive an empty auth header — clear any empty env remnant.
244
+ for key in ("ABUSECH_API_KEY", "VT_API_KEY", "OTX_API_KEY"):
245
+ if not (os.environ.get(key) or "").strip():
246
+ os.environ.pop(key, None)
@@ -14,6 +14,7 @@ import sys
14
14
  # Force UTF-8 on Windows consoles so rich glyphs render reliably
15
15
  if sys.platform == "win32":
16
16
  os.environ.setdefault("PYTHONIOENCODING", "utf-8")
17
+ os.environ.setdefault("PYTHONUTF8", "1")
17
18
  try:
18
19
  sys.stdout.reconfigure(encoding="utf-8") # type: ignore[attr-defined]
19
20
  sys.stderr.reconfigure(encoding="utf-8") # type: ignore[attr-defined]
@@ -21,6 +22,7 @@ if sys.platform == "win32":
21
22
  pass
22
23
 
23
24
  import typer
25
+ from rich.align import Align
24
26
  from rich.console import Console
25
27
  from rich.table import Table
26
28
 
@@ -30,13 +32,17 @@ from voidaccess_cli.commands import configure, enrich, export, investigate, show
30
32
 
31
33
  console = Console()
32
34
  BANNER = """\
33
- [color(183)] ░░[color(141)]████████[color(183)]░░[/]
34
- [color(183)] ░░[color(141)]████████████[color(183)]░░[/]
35
- [color(183)] ░░[color(141)]██████████████[color(183)]░░[/]
36
- [color(183)] ░░[color(141)]████[/] [bright_white]void[/] [color(141)]████[color(183)]░░[/]
37
- [color(183)] ░░[color(141)]██████████████[color(183)]░░[/]
38
- [color(183)] ░░[color(141)]████████████[color(183)]░░[/]
39
- [color(183)] ░░[color(141)]████████[color(183)]░░[/]
35
+ [color(183)] ░░░░░[color(141)][color(183)]░░░░░[/]
36
+ [color(183)] ░░[color(141)]█████████████[color(183)]░░[/]
37
+ [color(183)][color(141)]█████████████████[color(183)][/]
38
+ [color(183)][color(141)]███████████████████[color(183)][/]
39
+ [color(183)][color(141)]███████████████████[color(183)][/]
40
+ [color(141)]██████[/] [bright_white]void[/] [color(141)]███████[/]
41
+ [color(183)][color(141)]███████████████████[color(183)][/]
42
+ [color(183)]░[color(141)]███████████████████[color(183)]░[/]
43
+ [color(183)] ░[color(141)]█████████████████[color(183)]░[/]
44
+ [color(183)] ░░[color(141)]█████████████[color(183)]░░[/]
45
+ [color(183)] ░░░░░[color(141)]█[color(183)]░░░░░[/]
40
46
  [dim white] dark web osint intelligence[/dim white]"""
41
47
 
42
48
  app = typer.Typer(
@@ -154,10 +160,16 @@ def version() -> None:
154
160
 
155
161
 
156
162
  def show_banner(console: Console) -> None:
157
- if not sys.stdout.isatty():
163
+ import shutil
164
+ if os.environ.get("TERM") == "dumb":
165
+ return
166
+ if not sys.stdout.isatty() and "PS1" not in os.environ and os.name != "nt":
158
167
  return
159
168
  console.print()
160
- console.print(BANNER, justify="center")
169
+ raw_line = " oooooXooooo " # widest line, 21 chars
170
+ pad = max(0, (console.width - len(raw_line)) // 2)
171
+ for line in BANNER.split("\n"):
172
+ console.print(" " * pad + line)
161
173
  console.print()
162
174
 
163
175
 
@@ -171,7 +183,7 @@ def main(
171
183
  ) -> None:
172
184
  """Set env vars and render banner before command execution."""
173
185
  cli_config.apply_env()
174
- if not no_banner and not ctx.invoked_subcommand:
186
+ if not no_banner and ctx.invoked_subcommand:
175
187
  show_banner(console)
176
188
 
177
189
 
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes