voidaccess 1.9.4__tar.gz → 2.0.0__tar.gz

{voidaccess-1.9.4/voidaccess.egg-info → voidaccess-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidaccess
-Version: 1.9.4
+Version: 2.0.0
 Summary: Dark web OSINT CLI — automated threat intelligence from query to report
 Author: VoidAccess
 License-Expression: MIT
@@ -45,6 +45,7 @@ Requires-Dist: networkx==3.4.2
 Requires-Dist: pydantic==2.13.3
 Requires-Dist: python-jose[cryptography]==3.5.0
 Requires-Dist: passlib[bcrypt]==1.7.4
+Requires-Dist: bcrypt<4.1,>=3.1.0
 Requires-Dist: slowapi==0.1.9
 Requires-Dist: redis==7.4.0
 Requires-Dist: pyyaml==6.0.3
@@ -55,6 +56,7 @@ Requires-Dist: langdetect==1.0.9
 Requires-Dist: pyvis==0.3.2
 Requires-Dist: urllib3==2.6.3
 Requires-Dist: trafilatura==2.0.0
+Requires-Dist: lxml_html_clean>=0.4.2
 Requires-Dist: requests==2.33.1
 Requires-Dist: python-socks[asyncio]==2.8.1
 Requires-Dist: tldextract==5.3.1
@@ -62,6 +64,7 @@ Requires-Dist: stix2==3.0.2
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-asyncio; extra == "dev"
+Requires-Dist: aioresponses>=0.7.6; extra == "dev"
 Provides-Extra: nlp
 Requires-Dist: sentence-transformers>=3.0; extra == "nlp"
 Requires-Dist: torch>=2.3; extra == "nlp"

voidaccess-2.0.0/alembic.ini

@@ -0,0 +1,58 @@
+# Alembic configuration for VoidAccess.
+# Run all alembic commands from the voidaccess/ directory.
+#
+# Common commands:
+#   alembic upgrade head          — apply all pending migrations
+#   alembic downgrade -1          — roll back one migration
+#   alembic revision --autogenerate -m "describe change"  — generate a new migration
+#   alembic history               — list applied migrations
+
+[alembic]
+script_location = db:migrations
+prepend_sys_path = .
+file_template = %%(rev)s_%%(slug)s
+timezone = UTC
+
+# sqlalchemy.url is intentionally NOT set here.
+# It is loaded from the DATABASE_URL environment variable inside db/migrations/env.py.
+# This prevents credentials from ever being committed to version control.
+
+[post_write_hooks]
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 88
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

{voidaccess-1.9.4 → voidaccess-2.0.0}/api/main.py

@@ -13,6 +13,7 @@ from __future__ import annotations
 import asyncio
 import logging
 import os
+import sys
 from contextlib import asynccontextmanager
 from datetime import datetime, timedelta, timezone
 from typing import Callable, Optional
@@ -491,6 +492,13 @@ def _run_migrations() -> None:
 
         project_root = pathlib.Path(__file__).resolve().parents[1]
         ini_path = project_root / "alembic.ini"
+        if not ini_path.exists():
+            # Wheel data-files are installed at the virtualenv prefix, while
+            # api/ and db/ live under site-packages.  Locate the shipped
+            # config there for non-editable/API deployments.
+            ini_path = pathlib.Path(sys.prefix) / "alembic.ini"
+        if not ini_path.exists():
+            raise FileNotFoundError(f"Alembic config not found: {ini_path}")
         alembic_cfg = Config(str(ini_path))
         alembic_cfg.set_main_option("sqlalchemy.url", os.environ["DATABASE_URL"])
         alembic_cfg.set_main_option("script_location", str(project_root / "db" / "migrations"))
@@ -620,6 +628,13 @@ async def tor_test(_=Depends(get_current_user)) -> dict:
         import aiohttp  # noqa: PLC0415
         from aiohttp_socks import ProxyConnector  # noqa: PLC0415
 
+        # Deliberately NOT the shared pool in scraper/tor_pool.py, and the one
+        # justified exception to the "no private Tor connector" rule: this
+        # endpoint exists to test whether the Tor transport itself works, so it
+        # must not report health through the very session cache whose health is
+        # in question — a pooled session would make a stale or broken cached
+        # connector look like a working Tor.  It also fetches one fixed clearnet
+        # URL, so no .onion stream isolation is at stake here.
         connector = ProxyConnector.from_url(f"socks5://{TOR_PROXY_HOST}:{TOR_PROXY_PORT}")
         timeout = aiohttp.ClientTimeout(total=10)
         async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:

{voidaccess-1.9.4 → voidaccess-2.0.0}/api/routes/investigations.py

@@ -95,6 +95,10 @@ router = APIRouter()
 # Populated during the pipeline run; read by the GET detail endpoint.
 _infra_cluster_cache: dict[str, list] = {}
 
+# In-process cache: investigation_id (str) → node-id/community-id map.
+# The DB metadata copy is the durable source of truth.
+_communities_cache: dict[str, dict[str, int]] = {}
+
 # In-process cache: investigation_id (str) → sources_used status dict.
 # Populated during the pipeline run; read by the GET detail endpoint.
 # Phase 6.1: this is now a *fast-path* cache — the DB metadata column is
@@ -124,6 +128,22 @@ def _set_infra_clusters(investigation_id: str, clusters: list) -> None:
         {"infrastructure_clusters": clusters},
     )
 
+
+def _set_communities(investigation_id: str, communities: dict[str, int]) -> None:
+    """Cache and persist the server-computed community partition."""
+    normalized = {
+        str(node_id): int(community_id)
+        for node_id, community_id in communities.items()
+    }
+    _communities_cache[investigation_id] = normalized
+    _update_investigation_metadata(
+        investigation_id,
+        {
+            "communities": normalized,
+            "community_count": len(set(normalized.values())) if normalized else 0,
+        },
+    )
+
 # ---------------------------------------------------------------------------
 # Phase 6.2 — per-phase timeouts (Phase 6.1: caches also persisted to DB)
 # ---------------------------------------------------------------------------
@@ -411,6 +431,26 @@ async def _build_graph_phase(
                 session.commit()
         except Exception as e:
             logger.info("[%s] Edge persistence failed (non-fatal): %s", inv_uuid, e)
+
+        # Match the CLI's pre-finalization community step.  Feed the helper
+        # the complete graph, before the graph endpoint's display truncation
+        # to 20 nodes/50 edges; using that response slice would create an
+        # incomplete partition and break CLI/API parity.
+        try:
+            communities = await asyncio.to_thread(
+                _detect_communities_for_graph, graph_obj
+            )
+            _set_communities(str(inv_uuid), communities)
+            logger.info(
+                "[%s] Community detection: %s communities (%s nodes)",
+                inv_uuid,
+                len(set(communities.values())) if communities else 0,
+                len(communities),
+            )
+        except Exception as exc:
+            logger.warning(
+                "[%s] Community detection failed (non-fatal): %s", inv_uuid, exc
+            )
     except Exception as exc:
         logger.exception("[%s] Graph building failed: %s", inv_uuid, str(exc))
         return None
@@ -725,6 +765,7 @@ def _get_db_investigation(investigation_id: str) -> Any:
 
             db_sources_used = db_metadata.get("sources_used") or {}
             db_infra_clusters = db_metadata.get("infrastructure_clusters") or []
+            db_communities = db_metadata.get("communities") or {}
 
             sources_used = (
                 _sources_used_cache.get(str(inv.id))
@@ -736,6 +777,15 @@ def _get_db_investigation(investigation_id: str) -> Any:
                 or _infra_cluster_cache.get(investigation_id)
                 or db_infra_clusters
             )
+            communities = (
+                _communities_cache.get(str(inv.id))
+                or _communities_cache.get(investigation_id)
+                or db_communities
+            )
+            communities = {
+                str(node_id): int(community_id)
+                for node_id, community_id in communities.items()
+            }
 
             return {
                 "id": str(inv.id),
@@ -759,6 +809,8 @@ def _get_db_investigation(investigation_id: str) -> Any:
                 "paste_sources_used": paste_sources_used,
                 "infrastructure_clusters": infrastructure_clusters,
                 "sources_used": sources_used,
+                "communities": communities,
+                "community_count": len(set(communities.values())) if communities else 0,
                 "seeds_discovered": _count_discovered_seeds_for_investigation(str(inv.id)),
             }
     except HTTPException:
@@ -912,19 +964,34 @@ async def _get_investigation_model_choice(model: Optional[str]) -> tuple[str, An
 # ---------------------------------------------------------------------------
 
 
+# LLM retry bounds.  Unlike an enrichment client's documented interval, 65 s here
+# is a guess ("outlast a 1-minute window"), so it is a fallback rather than a
+# floor — a provider naming a shorter wait is honoured down to the minimum.
+_LLM_RATE_LIMIT_FALLBACK = 65.0
+_LLM_RATE_LIMIT_MIN_WAIT = 5.0
+_LLM_RATE_LIMIT_MAX_WAIT = 300.0
+
+
 def _parse_rate_limit_reset(exc: Exception) -> float:
-    """Extract reset timestamp from a 429 error and return seconds to wait."""
-    import time
-    import re
-    exc_str = str(exc)
-    # OpenRouter returns X-RateLimit-Reset as epoch milliseconds in metadata
-    match = re.search(r"'X-RateLimit-Reset':\s*'?(\d{13})'?", exc_str)
-    if match:
-        reset_ms = int(match.group(1))
-        wait = (reset_ms / 1000.0) - time.time() + 1.0  # +1s buffer
-        return max(wait, 5.0)
-    # Fallback: 65s to outlast a 60s/1-min rate-limit window
-    return 65.0
+    """
+    Seconds to wait after an LLM 429, from the provider's own headers.
+
+    Delegates to ``pacing.retry_after_seconds`` so there is one implementation
+    of "the server told us how long to wait" shared with the enrichment API
+    clients, rather than two that can drift.  The exception object is passed
+    through directly: LangChain buries the 429 headers in the exception message,
+    which that helper handles via its string branch.
+
+    The 65 s fallback outlasts a 60 s/1-min window when no header is present.
+    """
+    import pacing
+
+    return pacing.retry_after_seconds(
+        exc,
+        fallback=_LLM_RATE_LIMIT_FALLBACK,
+        floor=_LLM_RATE_LIMIT_MIN_WAIT,
+        max_wait=_LLM_RATE_LIMIT_MAX_WAIT,
+    )
 
 
 async def _llm_with_backoff(fn, *args, max_retries: int = 4, investigation_id: "uuid.UUID | None" = None, **kwargs):
@@ -2835,7 +2902,7 @@ async def get_investigation_entities(
         raise HTTPException(status_code=503, detail="Database not configured")
     try:
         from db.session import get_session
-        from db.models import Entity, InvestigationEntityLink
+        from db.models import Entity, EntityRelationship, InvestigationEntityLink
         from db.queries import get_investigation_by_id_or_run
         from graph.builder import _make_node_id
         from utils.ioc_freshness import get_freshness_tag, get_freshness_display
@@ -2853,10 +2920,24 @@ async def get_investigation_entities(
                 sa_select(InvestigationEntityLink.entity_id)
                 .where(InvestigationEntityLink.investigation_id == inv.id)
             )
-            query = session.query(Entity).filter(
+            scoped_query = session.query(Entity).filter(
                 (Entity.investigation_id == inv.id)
                 | Entity.id.in_(linked_ids_select)
             )
+            # Scores are normalized against the complete investigation, not
+            # the current page or confidence/type filter.  This keeps ranking
+            # stable across pagination and makes it comparable to the CLI
+            # export for the same investigation.
+            score_entities = scoped_query.all()
+            relationships = (
+                session.query(EntityRelationship)
+                .filter(EntityRelationship.investigation_id == inv.id)
+                .all()
+            )
+            from utils.entity_priority import score_map
+            priority_by_id = score_map(score_entities, relationships)
+
+            query = scoped_query
             if entity_type:
                 query = query.filter(Entity.entity_type == entity_type)
             if min_confidence > 0.0:
@@ -2929,6 +3010,18 @@ async def get_investigation_entities(
                         "corroborating_sources": json.loads(e.corroborating_sources or '["dark_web_scrape"]'),
                         "cross_referenced": (getattr(e, "investigation_count", 1) or 1) > 1,
                         "graph_node_id": graph_node_id,
+                        "priority_score": priority_by_id.get(str(e.id), {}).get(
+                            "priority_score", 0.0
+                        ),
+                        "priority_score_components": priority_by_id.get(
+                            str(e.id), {}
+                        ).get("priority_score_components", {}),
+                        "priority_score_centrality_contribution": priority_by_id.get(
+                            str(e.id), {}
+                        ).get("priority_score_centrality_contribution", 0.0),
+                        "typed_relationship_degree": priority_by_id.get(
+                            str(e.id), {}
+                        ).get("typed_relationship_degree", 0),
                         "defanged": defang,
                     }
                 )
@@ -2961,8 +3054,9 @@ async def export_investigation_entities_csv(
 
     try:
         from db.session import get_session
-        from db.models import Entity, InvestigationEntityLink
+        from db.models import Entity, EntityRelationship, InvestigationEntityLink
         from db.queries import get_investigation_by_id_or_run
+        from utils.entity_priority import score_map
 
         with get_session() as session:
             inv = get_investigation_by_id_or_run(session, inv_uuid)
@@ -2983,6 +3077,12 @@ async def export_investigation_entities_csv(
                 )
                 .all()
             )
+            relationships = (
+                session.query(EntityRelationship)
+                .filter(EntityRelationship.investigation_id == inv.id)
+                .all()
+            )
+            priority_by_id = score_map(entities, relationships)
 
             output = io.StringIO()
             writer = csv.writer(output)
@@ -2990,6 +3090,7 @@ async def export_investigation_entities_csv(
                 "entity_type",
                 "canonical_value",
                 "confidence",
+                "priority_score",
                 "occurrence_count",
                 "first_seen_page",
                 "context_snippet",
@@ -3011,6 +3112,7 @@ async def export_investigation_entities_csv(
                     e.entity_type,
                     e.canonical_value or e.value,
                     e.confidence,
+                    priority_by_id.get(str(e.id), {}).get("priority_score", 0.0),
                     1,
                     source_url,
                     context[:500],
@@ -3080,6 +3182,27 @@ def _rebuild_networkx_graph(nodes: list, edges: list) -> "nx.DiGraph":
     return G
 
 
+def _detect_communities_for_graph(graph_obj) -> dict[str, int]:
+    """Run the shared detector over the complete API graph.
+
+    ``_rebuild_networkx_graph`` is the API representation adapter.  It is
+    correct here because it receives the full ``to_json`` graph, not the
+    endpoint's presentation slice.  Its directed graph is equivalent to the
+    CLI helper's simple undirected graph for detection: ``detect_communities``
+    projects either input to an undirected graph before running the algorithm.
+    """
+    from graph.builder import detect_communities
+    from graph.export import to_json
+
+    graph_data = to_json(graph_obj)
+    rebuilt = _rebuild_networkx_graph(graph_data["nodes"], graph_data["edges"])
+    partition = detect_communities(rebuilt)
+    return {
+        str(node_id): int(community_id)
+        for node_id, community_id in partition.items()
+    }
+
+
 @router.get("/{investigation_id}/graph")
 async def get_investigation_graph(
     investigation_id: str,
@@ -3106,6 +3229,20 @@ async def get_investigation_graph(
             if graph_status not in ("built", "complete", "skipped_overflow", "no_data"):
                 return {"status": "pending"}
             internal_id = inv.id
+            graph_communities = _communities_cache.get(str(internal_id), {})
+            if not graph_communities:
+                graph_metadata = getattr(inv, "metadata_json", None)
+                if isinstance(graph_metadata, str):
+                    try:
+                        graph_metadata = json.loads(graph_metadata)
+                    except (ValueError, TypeError):
+                        graph_metadata = {}
+                if isinstance(graph_metadata, dict):
+                    graph_communities = graph_metadata.get("communities") or {}
+            graph_communities = {
+                str(node_id): int(community_id)
+                for node_id, community_id in graph_communities.items()
+            }
 
         graph = build_graph_from_db_cached(investigation_id=internal_id) if not force_rebuild else build_graph_from_db(investigation_id=internal_id)
         graph_data = to_json(graph)
@@ -3119,6 +3256,12 @@ async def get_investigation_graph(
             "summary_stats": summary_stats(graph),
             "nodes": nodes,
             "edges": edges,
+            "communities": graph_communities,
+            "community_count": (
+                len(set(graph_communities.values()))
+                if graph_communities
+                else 0
+            ),
             "visualization_html": viz_html,
         }
     except HTTPException:

{voidaccess-1.9.4 → voidaccess-2.0.0}/config.py

@@ -37,6 +37,16 @@ DATABASE_URL = _clean_env("DATABASE_URL")
 TOR_PROXY_HOST = _clean_env("TOR_PROXY_HOST", "127.0.0.1")
 TOR_PROXY_PORT = _clean_env("TOR_PROXY_PORT", "9050")
 
+# Optional second SocksPort configured with `IsolateDestAddr`, used only by the
+# Playwright renderer.  Every aiohttp caller isolates by handing Tor a distinct
+# SOCKS credential per target, but Playwright's driver refuses SOCKS5
+# credentials outright, so that path needs Tor to do the keying itself.
+# docker/Dockerfile.tor opens 9250 for exactly this.  If the port is absent —
+# system tor, Tor Browser, any deployment not using our container — the JS
+# render path runs un-isolated and says so once; it is never silently assumed to
+# be isolated.  Set to an empty value to disable the probe entirely.
+TOR_ISOLATION_SOCKS_PORT = _clean_env("TOR_ISOLATION_SOCKS_PORT", "9250")
+
 # Phase 1D — expanded sources (all optional; missing vars disable that source)
 DARKSEARCH_API_KEY = _clean_env("DARKSEARCH_API_KEY")          # optional free-tier key
 TELEGRAM_API_ID   = _clean_env("TELEGRAM_API_ID")              # from my.telegram.org
@@ -80,6 +90,14 @@ else:
 OTX_API_KEY = _clean_env("OTX_API_KEY", "")  # AlienVault OTX — free at otx.alienvault.com
 VT_API_KEY = _clean_env("VT_API_KEY", "")    # VirusTotal — free tier at virustotal.com
 
+# VirusTotal pacing tier.  The public API allows 4 requests/minute (a 15 s gap);
+# premium keys have no per-minute limit at all, and a key string is
+# indistinguishable between the two, so a paying subscriber was being paced 25x
+# slower than their entitlement.  Set VT_API_TIER=premium to opt out of
+# free-tier pacing.  Unset or unrecognised means "public" — the safe default,
+# since guessing premium wrongly earns a 429.
+VT_API_TIER = (_clean_env("VT_API_TIER", "public") or "public").strip().lower()
+
 # IP Reputation Enrichment (all optional — features degrade gracefully without keys)
 ABUSEIPDB_API_KEY = _clean_env("ABUSEIPDB_API_KEY", "")   # Community IP abuse reports
 GREYNOISE_API_KEY = _clean_env("GREYNOISE_API_KEY", "")   # Suppresses benign scanner IPs
@@ -127,6 +145,10 @@ SCRAPINGANT_API_KEY = _clean_env("SCRAPINGANT_API_KEY", "")
 # UserApiKey registration on the API surface).
 SCRAPINGANT_PROXY_TYPE = _clean_env("SCRAPINGANT_PROXY_TYPE", "residential")
 
+# Shodan InternetDB publishes NO rate limit and needs no API key (verified
+# 2026-07-29) — this is VoidAccess's own courtesy pacing, not a quota.  It is
+# still routed through pacing.scale_delay_floor() at the call site in
+# sources/shodan.py so the mechanism matches every other enrichment client.
 SHODAN_RATE_LIMIT_DELAY = 1.0        # seconds between Shodan requests (InternetDB)
 MAX_IPS_PER_INVESTIGATION = 50      # max IPs to query Shodan per investigation
 MAX_HASHES_PER_INVESTIGATION = 20    # max file hashes to query VirusTotal per investigation