voidaccess 1.9.4__tar.gz → 2.0.1__tar.gz

{voidaccess-1.9.4/voidaccess.egg-info → voidaccess-2.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidaccess
-Version: 1.9.4
+Version: 2.0.1
 Summary: Dark web OSINT CLI — automated threat intelligence from query to report
 Author: VoidAccess
 License-Expression: MIT
@@ -45,6 +45,7 @@ Requires-Dist: networkx==3.4.2
 Requires-Dist: pydantic==2.13.3
 Requires-Dist: python-jose[cryptography]==3.5.0
 Requires-Dist: passlib[bcrypt]==1.7.4
+Requires-Dist: bcrypt<4.1,>=3.1.0
 Requires-Dist: slowapi==0.1.9
 Requires-Dist: redis==7.4.0
 Requires-Dist: pyyaml==6.0.3
@@ -55,6 +56,7 @@ Requires-Dist: langdetect==1.0.9
 Requires-Dist: pyvis==0.3.2
 Requires-Dist: urllib3==2.6.3
 Requires-Dist: trafilatura==2.0.0
+Requires-Dist: lxml_html_clean>=0.4.2
 Requires-Dist: requests==2.33.1
 Requires-Dist: python-socks[asyncio]==2.8.1
 Requires-Dist: tldextract==5.3.1
@@ -62,6 +64,7 @@ Requires-Dist: stix2==3.0.2
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-asyncio; extra == "dev"
+Requires-Dist: aioresponses>=0.7.6; extra == "dev"
 Provides-Extra: nlp
 Requires-Dist: sentence-transformers>=3.0; extra == "nlp"
 Requires-Dist: torch>=2.3; extra == "nlp"

voidaccess-2.0.1/alembic.ini

@@ -0,0 +1,58 @@
+# Alembic configuration for VoidAccess.
+# Run all alembic commands from the voidaccess/ directory.
+#
+# Common commands:
+#   alembic upgrade head          — apply all pending migrations
+#   alembic downgrade -1          — roll back one migration
+#   alembic revision --autogenerate -m "describe change"  — generate a new migration
+#   alembic history               — list applied migrations
+
+[alembic]
+script_location = db:migrations
+prepend_sys_path = .
+file_template = %%(rev)s_%%(slug)s
+timezone = UTC
+
+# sqlalchemy.url is intentionally NOT set here.
+# It is loaded from the DATABASE_URL environment variable inside db/migrations/env.py.
+# This prevents credentials from ever being committed to version control.
+
+[post_write_hooks]
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 88
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

{voidaccess-1.9.4 → voidaccess-2.0.1}/api/main.py

@@ -13,6 +13,7 @@ from __future__ import annotations
 import asyncio
 import logging
 import os
+import sys
 from contextlib import asynccontextmanager
 from datetime import datetime, timedelta, timezone
 from typing import Callable, Optional
@@ -331,7 +332,7 @@ async def _periodic_stuck_sweep() -> None:
 app = FastAPI(
     title="VoidAccess Intelligence API",
     description="VoidAccess: Dark Web Intelligence Platform",
-    version="2.0.0",
+    version="2.0.1",
     docs_url="/docs",
     redoc_url="/redoc",
     lifespan=lifespan,
@@ -491,6 +492,13 @@ def _run_migrations() -> None:
 
         project_root = pathlib.Path(__file__).resolve().parents[1]
         ini_path = project_root / "alembic.ini"
+        if not ini_path.exists():
+            # Wheel data-files are installed at the virtualenv prefix, while
+            # api/ and db/ live under site-packages.  Locate the shipped
+            # config there for non-editable/API deployments.
+            ini_path = pathlib.Path(sys.prefix) / "alembic.ini"
+        if not ini_path.exists():
+            raise FileNotFoundError(f"Alembic config not found: {ini_path}")
         alembic_cfg = Config(str(ini_path))
         alembic_cfg.set_main_option("sqlalchemy.url", os.environ["DATABASE_URL"])
         alembic_cfg.set_main_option("script_location", str(project_root / "db" / "migrations"))
@@ -620,6 +628,13 @@ async def tor_test(_=Depends(get_current_user)) -> dict:
         import aiohttp  # noqa: PLC0415
         from aiohttp_socks import ProxyConnector  # noqa: PLC0415
 
+        # Deliberately NOT the shared pool in scraper/tor_pool.py, and the one
+        # justified exception to the "no private Tor connector" rule: this
+        # endpoint exists to test whether the Tor transport itself works, so it
+        # must not report health through the very session cache whose health is
+        # in question — a pooled session would make a stale or broken cached
+        # connector look like a working Tor.  It also fetches one fixed clearnet
+        # URL, so no .onion stream isolation is at stake here.
         connector = ProxyConnector.from_url(f"socks5://{TOR_PROXY_HOST}:{TOR_PROXY_PORT}")
         timeout = aiohttp.ClientTimeout(total=10)
         async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:

{voidaccess-1.9.4 → voidaccess-2.0.1}/api/routes/entities.py

@@ -85,6 +85,10 @@ async def list_entities(
                 .limit(limit)
                 .all()
             )
+            # Score each row against its own investigation (cached per
+            # investigation id so a page of same-investigation entities builds
+            # the map once), matching the CLI export and /entities/{id} exactly.
+            priority_cache: dict = {}
             return {
                 "items": [
                     {
@@ -98,6 +102,7 @@ async def list_entities(
                         "investigation_id": str(e.investigation_id) if e.investigation_id else None,
                         "investigation_count": getattr(e, "investigation_count", 1) or 1,
                         "created_at": e.created_at.isoformat() if e.created_at else None,
+                        **_entity_priority_fields(session, e, priority_cache),
                     }
                     for e in entities
                 ],
@@ -150,7 +155,10 @@ async def export_entity_stix(
                     })
             except Exception as exc:
                 logger.warning("STIX export for entity %s failed, falling back to raw JSON: %s", entity_id, exc)
-                json_str = json.dumps(_entity_to_dict(entity), indent=2)
+                json_str = json.dumps(
+                    _entity_to_dict(entity, _entity_priority_fields(session, entity)),
+                    indent=2,
+                )
 
             filename = f"voidaccess_entity_{entity_id}_stix.json"
             return Response(
@@ -184,7 +192,7 @@ async def export_entity_json(
             _assert_entity_accessible(session, eid, current_user.user.id)
 
             appearances = get_entity_appearances(session, eid, current_user.user.id)
-            data = _entity_to_dict(entity)
+            data = _entity_to_dict(entity, _entity_priority_fields(session, entity))
             data["appearances"] = appearances
             json_str = json.dumps(data, indent=2, default=str)
 
@@ -629,8 +637,10 @@ async def get_entity(
                 if entity.context:
                     display_context = defang_text(entity.context)
 
+            priority = _entity_priority_fields(session, entity)
+
             return {
-                **_entity_to_dict(entity),
+                **_entity_to_dict(entity, priority),
                 "value": display_value,
                 "canonical_value": display_canonical,
                 "context": display_context,
@@ -798,7 +808,90 @@ def _assert_entity_accessible(session, entity_id: uuid.UUID, user_id: int) -> No
         raise HTTPException(status_code=404, detail="Entity not found")
 
 
-def _entity_to_dict(entity) -> dict:  # type: ignore[type-arg]
+# Default priority fields for an entity we could not (or chose not to) score.
+# Mirrors the zero-defaults the investigation entities endpoint uses so the two
+# API surfaces are shape-identical.
+_EMPTY_PRIORITY: dict = {
+    "priority_score": 0.0,
+    "priority_score_components": {},
+    "priority_score_centrality_contribution": 0.0,
+}
+
+
+def _investigation_priority_map(session, investigation_id) -> dict:
+    """Return ``score_map`` for a whole investigation, scoped identically to the
+    investigation entities endpoint and the CLI export.
+
+    Prioritization is normalized against the *complete* investigation (direct +
+    linked entities, all relationships), never a single entity or a filtered
+    page — that is what makes the value returned here byte-for-byte comparable
+    to ``GET /investigations/{id}/entities`` and to the CLI JSON/CSV/MD export
+    for the same entity (see ``api/routes/investigations.py`` and
+    ``voidaccess_cli/commands/export.py``).
+    """
+    from db.models import Entity, EntityRelationship, InvestigationEntityLink  # noqa: PLC0415
+    import sqlalchemy as sa  # noqa: PLC0415
+    from utils.entity_priority import score_map  # noqa: PLC0415
+
+    linked_ids = (
+        sa.select(InvestigationEntityLink.entity_id)
+        .where(InvestigationEntityLink.investigation_id == investigation_id)
+    )
+    scoped = (
+        session.query(Entity)
+        .filter(
+            (Entity.investigation_id == investigation_id)
+            | Entity.id.in_(linked_ids)
+        )
+        .all()
+    )
+    relationships = (
+        session.query(EntityRelationship)
+        .filter(EntityRelationship.investigation_id == investigation_id)
+        .all()
+    )
+    return score_map(scoped, relationships)
+
+
+def _entity_priority_fields(session, entity, cache: Optional[dict] = None) -> dict:
+    """Return the three ``priority_score`` fields for *entity*, computed with the
+    same function and the same investigation-wide input set the CLI export uses.
+
+    ``cache`` (an optional ``dict`` keyed by investigation id) lets a list
+    endpoint score many entities from the same investigation without rebuilding
+    the score map per row.  Any failure degrades to zero defaults rather than
+    breaking the response.
+    """
+    inv_id = getattr(entity, "investigation_id", None)
+    if inv_id is None:
+        return dict(_EMPTY_PRIORITY)
+    try:
+        if cache is not None and inv_id in cache:
+            pmap = cache[inv_id]
+        else:
+            pmap = _investigation_priority_map(session, inv_id)
+            if cache is not None:
+                cache[inv_id] = pmap
+        entry = pmap.get(str(entity.id), {})
+        return {
+            "priority_score": entry.get("priority_score", 0.0),
+            "priority_score_components": entry.get("priority_score_components", {}),
+            "priority_score_centrality_contribution": entry.get(
+                "priority_score_centrality_contribution", 0.0
+            ),
+        }
+    except Exception as exc:  # pragma: no cover - defensive
+        logger.warning("entity priority computation failed: %s", exc)
+        return dict(_EMPTY_PRIORITY)
+
+
+def _entity_to_dict(entity, priority: Optional[dict] = None) -> dict:  # type: ignore[type-arg]
+    # ``priority`` carries the three prioritization fields (see
+    # ``_entity_priority_fields``).  It is passed in by callers that hold an open
+    # session so the score matches the CLI export exactly; when absent we emit
+    # the same zero defaults as the investigation entities endpoint so the field
+    # is always present on the ``/entities`` surface.
+    priority = priority or _EMPTY_PRIORITY
     return {
         "id": str(entity.id),
         "entity_type": entity.entity_type,
@@ -816,6 +909,11 @@ def _entity_to_dict(entity) -> dict:  # type: ignore[type-arg]
         "source_count": getattr(entity, "source_count", 1) or 1,
         "investigation_count": getattr(entity, "investigation_count", 1) or 1,
         "corroborating_sources": getattr(entity, "corroborating_sources", None),
+        "priority_score": priority.get("priority_score", 0.0),
+        "priority_score_components": priority.get("priority_score_components", {}),
+        "priority_score_centrality_contribution": priority.get(
+            "priority_score_centrality_contribution", 0.0
+        ),
     }
 
 

{voidaccess-1.9.4 → voidaccess-2.0.1}/api/routes/export.py

@@ -16,6 +16,7 @@ POST /export/{investigation_id}/<fmt>/selected             — same exports rest
 from __future__ import annotations
 
 import io
+import json
 import logging
 import os
 import re
@@ -474,6 +475,21 @@ async def export_package(
                     investigation_dict["created_at"] = (
                         inv.created_at.isoformat() if inv.created_at else None
                     )
+                    # Backfill sources_used from investigation metadata, the
+                    # same place the CLI export reads it (adapters/sqlite.py
+                    # ``_investigation_row``). ``metadata_json`` is a dict on
+                    # Postgres and may be a JSON string on legacy SQLite rows.
+                    metadata_raw = getattr(inv, "metadata_json", None)
+                    if isinstance(metadata_raw, str):
+                        try:
+                            metadata_parsed = json.loads(metadata_raw) if metadata_raw.strip() else {}
+                        except (ValueError, TypeError):
+                            metadata_parsed = {}
+                    elif isinstance(metadata_raw, dict):
+                        metadata_parsed = metadata_raw
+                    else:
+                        metadata_parsed = {}
+                    investigation_dict["sources_used"] = metadata_parsed.get("sources_used") or {}
         except Exception as exc:
             logger.warning("export_package: investigation lookup failed: %s", exc)
 

{voidaccess-1.9.4 → voidaccess-2.0.1}/api/routes/investigations.py

@@ -95,6 +95,10 @@ router = APIRouter()
 # Populated during the pipeline run; read by the GET detail endpoint.
 _infra_cluster_cache: dict[str, list] = {}
 
+# In-process cache: investigation_id (str) → node-id/community-id map.
+# The DB metadata copy is the durable source of truth.
+_communities_cache: dict[str, dict[str, int]] = {}
+
 # In-process cache: investigation_id (str) → sources_used status dict.
 # Populated during the pipeline run; read by the GET detail endpoint.
 # Phase 6.1: this is now a *fast-path* cache — the DB metadata column is
@@ -124,6 +128,22 @@ def _set_infra_clusters(investigation_id: str, clusters: list) -> None:
         {"infrastructure_clusters": clusters},
     )
 
+
+def _set_communities(investigation_id: str, communities: dict[str, int]) -> None:
+    """Cache and persist the server-computed community partition."""
+    normalized = {
+        str(node_id): int(community_id)
+        for node_id, community_id in communities.items()
+    }
+    _communities_cache[investigation_id] = normalized
+    _update_investigation_metadata(
+        investigation_id,
+        {
+            "communities": normalized,
+            "community_count": len(set(normalized.values())) if normalized else 0,
+        },
+    )
+
 # ---------------------------------------------------------------------------
 # Phase 6.2 — per-phase timeouts (Phase 6.1: caches also persisted to DB)
 # ---------------------------------------------------------------------------
@@ -411,6 +431,26 @@ async def _build_graph_phase(
                 session.commit()
         except Exception as e:
             logger.info("[%s] Edge persistence failed (non-fatal): %s", inv_uuid, e)
+
+        # Match the CLI's pre-finalization community step.  Feed the helper
+        # the complete graph, before the graph endpoint's display truncation
+        # to 20 nodes/50 edges; using that response slice would create an
+        # incomplete partition and break CLI/API parity.
+        try:
+            communities = await asyncio.to_thread(
+                _detect_communities_for_graph, graph_obj
+            )
+            _set_communities(str(inv_uuid), communities)
+            logger.info(
+                "[%s] Community detection: %s communities (%s nodes)",
+                inv_uuid,
+                len(set(communities.values())) if communities else 0,
+                len(communities),
+            )
+        except Exception as exc:
+            logger.warning(
+                "[%s] Community detection failed (non-fatal): %s", inv_uuid, exc
+            )
     except Exception as exc:
         logger.exception("[%s] Graph building failed: %s", inv_uuid, str(exc))
         return None
@@ -725,6 +765,7 @@ def _get_db_investigation(investigation_id: str) -> Any:
 
             db_sources_used = db_metadata.get("sources_used") or {}
             db_infra_clusters = db_metadata.get("infrastructure_clusters") or []
+            db_communities = db_metadata.get("communities") or {}
 
             sources_used = (
                 _sources_used_cache.get(str(inv.id))
@@ -736,6 +777,15 @@ def _get_db_investigation(investigation_id: str) -> Any:
                 or _infra_cluster_cache.get(investigation_id)
                 or db_infra_clusters
             )
+            communities = (
+                _communities_cache.get(str(inv.id))
+                or _communities_cache.get(investigation_id)
+                or db_communities
+            )
+            communities = {
+                str(node_id): int(community_id)
+                for node_id, community_id in communities.items()
+            }
 
             return {
                 "id": str(inv.id),
@@ -759,6 +809,8 @@ def _get_db_investigation(investigation_id: str) -> Any:
                 "paste_sources_used": paste_sources_used,
                 "infrastructure_clusters": infrastructure_clusters,
                 "sources_used": sources_used,
+                "communities": communities,
+                "community_count": len(set(communities.values())) if communities else 0,
                 "seeds_discovered": _count_discovered_seeds_for_investigation(str(inv.id)),
             }
     except HTTPException:
@@ -912,19 +964,34 @@ async def _get_investigation_model_choice(model: Optional[str]) -> tuple[str, An
 # ---------------------------------------------------------------------------
 
 
+# LLM retry bounds.  Unlike an enrichment client's documented interval, 65 s here
+# is a guess ("outlast a 1-minute window"), so it is a fallback rather than a
+# floor — a provider naming a shorter wait is honoured down to the minimum.
+_LLM_RATE_LIMIT_FALLBACK = 65.0
+_LLM_RATE_LIMIT_MIN_WAIT = 5.0
+_LLM_RATE_LIMIT_MAX_WAIT = 300.0
+
+
 def _parse_rate_limit_reset(exc: Exception) -> float:
-    """Extract reset timestamp from a 429 error and return seconds to wait."""
-    import time
-    import re
-    exc_str = str(exc)
-    # OpenRouter returns X-RateLimit-Reset as epoch milliseconds in metadata
-    match = re.search(r"'X-RateLimit-Reset':\s*'?(\d{13})'?", exc_str)
-    if match:
-        reset_ms = int(match.group(1))
-        wait = (reset_ms / 1000.0) - time.time() + 1.0  # +1s buffer
-        return max(wait, 5.0)
-    # Fallback: 65s to outlast a 60s/1-min rate-limit window
-    return 65.0
+    """
+    Seconds to wait after an LLM 429, from the provider's own headers.
+
+    Delegates to ``pacing.retry_after_seconds`` so there is one implementation
+    of "the server told us how long to wait" shared with the enrichment API
+    clients, rather than two that can drift.  The exception object is passed
+    through directly: LangChain buries the 429 headers in the exception message,
+    which that helper handles via its string branch.
+
+    The 65 s fallback outlasts a 60 s/1-min window when no header is present.
+    """
+    import pacing
+
+    return pacing.retry_after_seconds(
+        exc,
+        fallback=_LLM_RATE_LIMIT_FALLBACK,
+        floor=_LLM_RATE_LIMIT_MIN_WAIT,
+        max_wait=_LLM_RATE_LIMIT_MAX_WAIT,
+    )
 
 
 async def _llm_with_backoff(fn, *args, max_retries: int = 4, investigation_id: "uuid.UUID | None" = None, **kwargs):
@@ -2835,7 +2902,7 @@ async def get_investigation_entities(
         raise HTTPException(status_code=503, detail="Database not configured")
     try:
         from db.session import get_session
-        from db.models import Entity, InvestigationEntityLink
+        from db.models import Entity, EntityRelationship, InvestigationEntityLink
         from db.queries import get_investigation_by_id_or_run
         from graph.builder import _make_node_id
         from utils.ioc_freshness import get_freshness_tag, get_freshness_display
@@ -2853,10 +2920,24 @@ async def get_investigation_entities(
                 sa_select(InvestigationEntityLink.entity_id)
                 .where(InvestigationEntityLink.investigation_id == inv.id)
             )
-            query = session.query(Entity).filter(
+            scoped_query = session.query(Entity).filter(
                 (Entity.investigation_id == inv.id)
                 | Entity.id.in_(linked_ids_select)
             )
+            # Scores are normalized against the complete investigation, not
+            # the current page or confidence/type filter.  This keeps ranking
+            # stable across pagination and makes it comparable to the CLI
+            # export for the same investigation.
+            score_entities = scoped_query.all()
+            relationships = (
+                session.query(EntityRelationship)
+                .filter(EntityRelationship.investigation_id == inv.id)
+                .all()
+            )
+            from utils.entity_priority import score_map
+            priority_by_id = score_map(score_entities, relationships)
+
+            query = scoped_query
             if entity_type:
                 query = query.filter(Entity.entity_type == entity_type)
             if min_confidence > 0.0:
@@ -2929,6 +3010,18 @@ async def get_investigation_entities(
                         "corroborating_sources": json.loads(e.corroborating_sources or '["dark_web_scrape"]'),
                         "cross_referenced": (getattr(e, "investigation_count", 1) or 1) > 1,
                         "graph_node_id": graph_node_id,
+                        "priority_score": priority_by_id.get(str(e.id), {}).get(
+                            "priority_score", 0.0
+                        ),
+                        "priority_score_components": priority_by_id.get(
+                            str(e.id), {}
+                        ).get("priority_score_components", {}),
+                        "priority_score_centrality_contribution": priority_by_id.get(
+                            str(e.id), {}
+                        ).get("priority_score_centrality_contribution", 0.0),
+                        "typed_relationship_degree": priority_by_id.get(
+                            str(e.id), {}
+                        ).get("typed_relationship_degree", 0),
                         "defanged": defang,
                     }
                 )
@@ -2961,8 +3054,9 @@ async def export_investigation_entities_csv(
 
     try:
         from db.session import get_session
-        from db.models import Entity, InvestigationEntityLink
+        from db.models import Entity, EntityRelationship, InvestigationEntityLink
         from db.queries import get_investigation_by_id_or_run
+        from utils.entity_priority import score_map
 
         with get_session() as session:
             inv = get_investigation_by_id_or_run(session, inv_uuid)
@@ -2983,6 +3077,12 @@ async def export_investigation_entities_csv(
                 )
                 .all()
             )
+            relationships = (
+                session.query(EntityRelationship)
+                .filter(EntityRelationship.investigation_id == inv.id)
+                .all()
+            )
+            priority_by_id = score_map(entities, relationships)
 
             output = io.StringIO()
             writer = csv.writer(output)
@@ -2990,6 +3090,7 @@ async def export_investigation_entities_csv(
                 "entity_type",
                 "canonical_value",
                 "confidence",
+                "priority_score",
                 "occurrence_count",
                 "first_seen_page",
                 "context_snippet",
@@ -3011,6 +3112,7 @@ async def export_investigation_entities_csv(
                     e.entity_type,
                     e.canonical_value or e.value,
                     e.confidence,
+                    priority_by_id.get(str(e.id), {}).get("priority_score", 0.0),
                     1,
                     source_url,
                     context[:500],
@@ -3080,6 +3182,27 @@ def _rebuild_networkx_graph(nodes: list, edges: list) -> "nx.DiGraph":
     return G
 
 
+def _detect_communities_for_graph(graph_obj) -> dict[str, int]:
+    """Run the shared detector over the complete API graph.
+
+    ``_rebuild_networkx_graph`` is the API representation adapter.  It is
+    correct here because it receives the full ``to_json`` graph, not the
+    endpoint's presentation slice.  Its directed graph is equivalent to the
+    CLI helper's simple undirected graph for detection: ``detect_communities``
+    projects either input to an undirected graph before running the algorithm.
+    """
+    from graph.builder import detect_communities
+    from graph.export import to_json
+
+    graph_data = to_json(graph_obj)
+    rebuilt = _rebuild_networkx_graph(graph_data["nodes"], graph_data["edges"])
+    partition = detect_communities(rebuilt)
+    return {
+        str(node_id): int(community_id)
+        for node_id, community_id in partition.items()
+    }
+
+
 @router.get("/{investigation_id}/graph")
 async def get_investigation_graph(
     investigation_id: str,
@@ -3106,6 +3229,20 @@ async def get_investigation_graph(
             if graph_status not in ("built", "complete", "skipped_overflow", "no_data"):
                 return {"status": "pending"}
             internal_id = inv.id
+            graph_communities = _communities_cache.get(str(internal_id), {})
+            if not graph_communities:
+                graph_metadata = getattr(inv, "metadata_json", None)
+                if isinstance(graph_metadata, str):
+                    try:
+                        graph_metadata = json.loads(graph_metadata)
+                    except (ValueError, TypeError):
+                        graph_metadata = {}
+                if isinstance(graph_metadata, dict):
+                    graph_communities = graph_metadata.get("communities") or {}
+            graph_communities = {
+                str(node_id): int(community_id)
+                for node_id, community_id in graph_communities.items()
+            }
 
         graph = build_graph_from_db_cached(investigation_id=internal_id) if not force_rebuild else build_graph_from_db(investigation_id=internal_id)
         graph_data = to_json(graph)
@@ -3119,6 +3256,12 @@ async def get_investigation_graph(
             "summary_stats": summary_stats(graph),
             "nodes": nodes,
             "edges": edges,
+            "communities": graph_communities,
+            "community_count": (
+                len(set(graph_communities.values()))
+                if graph_communities
+                else 0
+            ),
             "visualization_html": viz_html,
         }
     except HTTPException:

{voidaccess-1.9.4 → voidaccess-2.0.1}/config.py

@@ -37,6 +37,16 @@ DATABASE_URL = _clean_env("DATABASE_URL")
 TOR_PROXY_HOST = _clean_env("TOR_PROXY_HOST", "127.0.0.1")
 TOR_PROXY_PORT = _clean_env("TOR_PROXY_PORT", "9050")
 
+# Optional second SocksPort configured with `IsolateDestAddr`, used only by the
+# Playwright renderer.  Every aiohttp caller isolates by handing Tor a distinct
+# SOCKS credential per target, but Playwright's driver refuses SOCKS5
+# credentials outright, so that path needs Tor to do the keying itself.
+# docker/Dockerfile.tor opens 9250 for exactly this.  If the port is absent —
+# system tor, Tor Browser, any deployment not using our container — the JS
+# render path runs un-isolated and says so once; it is never silently assumed to
+# be isolated.  Set to an empty value to disable the probe entirely.
+TOR_ISOLATION_SOCKS_PORT = _clean_env("TOR_ISOLATION_SOCKS_PORT", "9250")
+
 # Phase 1D — expanded sources (all optional; missing vars disable that source)
 DARKSEARCH_API_KEY = _clean_env("DARKSEARCH_API_KEY")          # optional free-tier key
 TELEGRAM_API_ID   = _clean_env("TELEGRAM_API_ID")              # from my.telegram.org
@@ -80,6 +90,14 @@ else:
 OTX_API_KEY = _clean_env("OTX_API_KEY", "")  # AlienVault OTX — free at otx.alienvault.com
 VT_API_KEY = _clean_env("VT_API_KEY", "")    # VirusTotal — free tier at virustotal.com
 
+# VirusTotal pacing tier.  The public API allows 4 requests/minute (a 15 s gap);
+# premium keys have no per-minute limit at all, and a key string is
+# indistinguishable between the two, so a paying subscriber was being paced 25x
+# slower than their entitlement.  Set VT_API_TIER=premium to opt out of
+# free-tier pacing.  Unset or unrecognised means "public" — the safe default,
+# since guessing premium wrongly earns a 429.
+VT_API_TIER = (_clean_env("VT_API_TIER", "public") or "public").strip().lower()
+
 # IP Reputation Enrichment (all optional — features degrade gracefully without keys)
 ABUSEIPDB_API_KEY = _clean_env("ABUSEIPDB_API_KEY", "")   # Community IP abuse reports
 GREYNOISE_API_KEY = _clean_env("GREYNOISE_API_KEY", "")   # Suppresses benign scanner IPs
@@ -127,6 +145,10 @@ SCRAPINGANT_API_KEY = _clean_env("SCRAPINGANT_API_KEY", "")
 # UserApiKey registration on the API surface).
 SCRAPINGANT_PROXY_TYPE = _clean_env("SCRAPINGANT_PROXY_TYPE", "residential")
 
+# Shodan InternetDB publishes NO rate limit and needs no API key (verified
+# 2026-07-29) — this is VoidAccess's own courtesy pacing, not a quota.  It is
+# still routed through pacing.scale_delay_floor() at the call site in
+# sources/shodan.py so the mechanism matches every other enrichment client.
 SHODAN_RATE_LIMIT_DELAY = 1.0        # seconds between Shodan requests (InternetDB)
 MAX_IPS_PER_INVESTIGATION = 50      # max IPs to query Shodan per investigation
 MAX_HASHES_PER_INVESTIGATION = 20    # max file hashes to query VirusTotal per investigation