voidaccess 1.9.3__tar.gz → 1.9.4__tar.gz

{voidaccess-1.9.3 → voidaccess-1.9.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidaccess
-Version: 1.9.3
+Version: 1.9.4
 Summary: Dark web OSINT CLI — automated threat intelligence from query to report
 Author: VoidAccess
 License-Expression: MIT

{voidaccess-1.9.3 → voidaccess-1.9.4}/api/routes/entities.py

@@ -96,6 +96,7 @@ async def list_entities(
                         "context_snippet": e.context_snippet,
                         "context": e.context,
                         "investigation_id": str(e.investigation_id) if e.investigation_id else None,
+                        "investigation_count": getattr(e, "investigation_count", 1) or 1,
                         "created_at": e.created_at.isoformat() if e.created_at else None,
                     }
                     for e in entities
@@ -643,8 +644,9 @@ async def get_entity(
                 "freshness_label": freshness_display["label"],
                 "freshness_color": freshness_display["color"],
                 "source_count": entity.source_count or 1,
+                "investigation_count": getattr(entity, "investigation_count", 1) or 1,
                 "corroborating_sources": json.loads(entity.corroborating_sources or '["dark_web_scrape"]'),
-                "cross_referenced": (entity.source_count or 1) > 1,
+                "cross_referenced": (getattr(entity, "investigation_count", 1) or 1) > 1,
                 "defanged": defang,
                 "blockchain_data": {
                     "wallet_type": entity.entity_type if entity.entity_type in ("BITCOIN_ADDRESS", "ETHEREUM_ADDRESS", "MONERO_ADDRESS") else None,
@@ -811,6 +813,9 @@ def _entity_to_dict(entity) -> dict:  # type: ignore[type-arg]
         "investigation_id": str(entity.investigation_id) if entity.investigation_id else None,
         "created_at": entity.created_at.isoformat() if entity.created_at else None,
         "extraction_method": getattr(entity, "extraction_method", None),
+        "source_count": getattr(entity, "source_count", 1) or 1,
+        "investigation_count": getattr(entity, "investigation_count", 1) or 1,
+        "corroborating_sources": getattr(entity, "corroborating_sources", None),
     }
 
 

{voidaccess-1.9.3 → voidaccess-1.9.4}/api/routes/investigations.py

@@ -2925,8 +2925,9 @@ async def get_investigation_entities(
                         "freshness_label": freshness_display["label"],
                         "freshness_color": freshness_display["color"],
                         "source_count": e.source_count or 1,
+                        "investigation_count": getattr(e, "investigation_count", 1) or 1,
                         "corroborating_sources": json.loads(e.corroborating_sources or '["dark_web_scrape"]'),
-                        "cross_referenced": (e.source_count or 1) > 1,
+                        "cross_referenced": (getattr(e, "investigation_count", 1) or 1) > 1,
                         "graph_node_id": graph_node_id,
                         "defanged": defang,
                     }

{voidaccess-1.9.3 → voidaccess-1.9.4}/db/migrations/env.py

@@ -11,7 +11,7 @@ import sys
 from logging.config import fileConfig
 
 from alembic import context
-from sqlalchemy import engine_from_config, pool
+from sqlalchemy import engine_from_config, inspect, pool, text
 
 # Make sure the project root is on sys.path so `from db.models import Base`
 # resolves correctly regardless of where alembic is invoked from.
@@ -36,6 +36,33 @@ if config.config_file_name is not None:
 target_metadata = Base.metadata
 
 
+def _ensure_version_column_capacity(connection) -> None:
+    """Keep legacy PostgreSQL Alembic tables able to store current revision IDs.
+
+    The initial Alembic table used VARCHAR(32), but later revision identifiers
+    are longer than 32 characters.  Widen the metadata column before Alembic
+    writes the next revision; this touches only migration bookkeeping, never
+    application data.
+    """
+    if connection.dialect.name != "postgresql":
+        return
+    inspector = inspect(connection)
+    if "alembic_version" not in inspector.get_table_names():
+        return
+    version_column = next(
+        (c for c in inspector.get_columns("alembic_version") if c["name"] == "version_num"),
+        None,
+    )
+    current_length = getattr(version_column.get("type"), "length", None) if version_column else None
+    if current_length is not None and current_length < 255:
+        connection.execute(
+            text(
+                "ALTER TABLE alembic_version "
+                "ALTER COLUMN version_num TYPE VARCHAR(255)"
+            )
+        )
+
+
 def run_migrations_offline() -> None:
     """
     Run migrations without a live DB connection.
@@ -64,6 +91,7 @@ def run_migrations_online() -> None:
         poolclass=pool.NullPool,  # no pooling needed for one-shot migration runs
     )
     with connectable.connect() as connection:
+        _ensure_version_column_capacity(connection)
         context.configure(
             connection=connection,
             target_metadata=target_metadata,

voidaccess-1.9.4/db/migrations/versions/0026_add_entity_investigation_count.py

@@ -0,0 +1,46 @@
+"""Track distinct investigations independently from enrichment provenance."""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = "0026_add_entity_investigation_count"
+down_revision = "0025_add_cancellation_requested"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "entities",
+        sa.Column("investigation_count", sa.Integer(), server_default="1", nullable=False),
+    )
+
+    # Backfill from the existing cross-investigation links and include the
+    # owner column for rows created before links were consistently recorded.
+    conn = op.get_bind()
+    entity_rows = conn.execute(
+        sa.text("SELECT id, investigation_id FROM entities")
+    ).mappings().all()
+    link_rows = conn.execute(
+        sa.text("SELECT entity_id, investigation_id FROM investigation_entity_links")
+    ).mappings().all()
+    counts: dict[object, set[object]] = {}
+    for row in entity_rows:
+        counts[row["id"]] = (
+            {row["investigation_id"]}
+            if row["investigation_id"] is not None
+            else set()
+        )
+    for row in link_rows:
+        counts.setdefault(row["entity_id"], set()).add(row["investigation_id"])
+
+    for entity_id, investigation_ids in counts.items():
+        conn.execute(
+            sa.text("UPDATE entities SET investigation_count = :count WHERE id = :id"),
+            {"count": max(1, len(investigation_ids)), "id": entity_id},
+        )
+
+
+def downgrade() -> None:
+    op.drop_column("entities", "investigation_count")

{voidaccess-1.9.3 → voidaccess-1.9.4}/db/models.py

@@ -470,6 +470,11 @@ class Entity(Base):
     source_count: Mapped[int] = mapped_column(
         sa.Integer, server_default="1", default=1
     )
+    # Distinct investigations that independently surfaced this canonical
+    # entity. This is separate from enrichment-feed provenance below.
+    investigation_count: Mapped[int] = mapped_column(
+        sa.Integer, server_default="1", default=1
+    )
     corroborating_sources: Mapped[Optional[str]] = mapped_column(
         sa.Text, nullable=True
     )

{voidaccess-1.9.3 → voidaccess-1.9.4}/db/queries.py

@@ -281,7 +281,7 @@ def create_entity(
 
 def _link_entity_to_investigation(
     session: Session, entity_id: uuid.UUID, investigation_id: uuid.UUID
-) -> None:
+) -> bool:
     """Link an entity to an investigation via InvestigationEntityLink."""
     from db.models import InvestigationEntityLink
 
@@ -290,7 +290,7 @@ def _link_entity_to_investigation(
         entity_id=entity_id, investigation_id=investigation_id
     ).first()
     if existing:
-        return
+        return False
 
     # Also check pending (unflushed) objects — autoflush=False means they won't
     # appear in the query above, causing UNIQUE VIOLATION on batch flush.
@@ -300,13 +300,29 @@ def _link_entity_to_investigation(
             and obj.entity_id == entity_id
             and obj.investigation_id == investigation_id
         ):
-            return
+            return False
 
     link = InvestigationEntityLink(
         entity_id=entity_id,
         investigation_id=investigation_id
     )
     session.add(link)
+    return True
+
+
+def _entity_investigation_count(session: Session, entity: Entity) -> int:
+    """Count distinct investigations that surfaced *entity*, including owner."""
+    from db.models import InvestigationEntityLink
+
+    ids = {
+        row[0]
+        for row in session.query(InvestigationEntityLink.investigation_id)
+        .filter_by(entity_id=entity.id)
+        .all()
+    }
+    if entity.investigation_id:
+        ids.add(entity.investigation_id)
+    return max(1, len(ids))
 
 
 def upsert_entity_canonical(
@@ -359,9 +375,12 @@ def upsert_entity_canonical(
         existing.last_seen = datetime.now(timezone.utc)
         # Update last_seen_at for freshness tracking
         existing.last_seen_at = datetime.now(timezone.utc)
-        # Link to this investigation if not already linked
-        if existing.investigation_id != investigation_id:
-            _link_entity_to_investigation(session, existing.id, investigation_id)
+        # Link to this investigation if not already linked, including for
+        # legacy rows whose owner column predates the link table backfill.
+        linked = _link_entity_to_investigation(session, existing.id, investigation_id)
+        if linked:
+            session.flush()
+            existing.investigation_count = _entity_investigation_count(session, existing)
         return existing, False
     else:
         # Create new entity
@@ -380,6 +399,7 @@ def upsert_entity_canonical(
         session.add(entity)
         session.flush()  # populate entity.id before creating the link
         _link_entity_to_investigation(session, entity.id, investigation_id)
+        entity.investigation_count = 1
         return entity, True
 
 

{voidaccess-1.9.3 → voidaccess-1.9.4}/export/ioc_package.py

@@ -57,7 +57,7 @@ logger = logging.getLogger(__name__)
 # ---------------------------------------------------------------------------
 
 PACKAGE_FORMAT = "voidaccess-ioc-v1"
-PACKAGE_VERSION = "1.9.3"
+PACKAGE_VERSION = "1.9.4"
 SNORT_SID_BASE = 9000001  # 9xxxxxx reserved for VoidAccess-generated rules
 
 # Map extractor upper-case entity types to the per-file bucket they belong to.

{voidaccess-1.9.3 → voidaccess-1.9.4}/extractor/entity_shape.py

@@ -93,14 +93,7 @@ def _load_words() -> None:
 
 
 # Organisation suffixes that strongly signal a real org name.
-_ORG_SUFFIXES: frozenset[str] = frozenset({
-    "inc", "incorporated", "llc", "ltd", "limited", "corp", "corporation",
-    "gmbh", "ag", "sa", "srl", "bv", "plc", "co", "company", "group",
-    "holdings", "technologies", "systems", "solutions", "lab", "labs", "laboratories",
-    "networks", "software", "security", "capital", "partners", "ventures",
-    "bank", "university", "institute", "foundation", "agency", "bureau",
-    "industries", "international", "global", "media", "digital", "consulting",
-})
+_ORG_SUFFIXES = gazetteer.ORG_SUFFIXES
 
 _TOKEN_RE = re.compile(r"[A-Za-z0-9]+(?:['’][A-Za-z]+)?")
 _WORD_RE = re.compile(r"[A-Za-z][A-Za-z0-9]*")

{voidaccess-1.9.3 → voidaccess-1.9.4}/extractor/gazetteer.py

@@ -42,6 +42,24 @@ _DATA_PATH = os.path.join(
     "data",
     "threat_gazetteer.json",
 )
+_DOMAIN_STOPWORDS_PATH = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+    "data",
+    "domain_stopwords_en.txt",
+)
+
+# Organisation suffixes are also generic name qualifiers when they appear on
+# an actor/malware candidate (for example, ``LockBit Ransomware Group``).
+# Keep this set here so gazetteer matching and shape validation use the same
+# vocabulary without introducing a circular import through entity_shape.
+ORG_SUFFIXES: frozenset[str] = frozenset({
+    "inc", "incorporated", "llc", "ltd", "limited", "corp", "corporation",
+    "gmbh", "ag", "sa", "srl", "bv", "plc", "co", "company", "group",
+    "holdings", "technologies", "systems", "solutions", "lab", "labs", "laboratories",
+    "networks", "software", "security", "capital", "partners", "ventures",
+    "bank", "university", "institute", "foundation", "agency", "bureau",
+    "industries", "international", "global", "media", "digital", "consulting",
+})
 
 # Minimum canonical-key length — drops junk snapshot entries like "$$$" (which
 # canonicalise to "") or single stray characters that would over-match.
@@ -52,6 +70,7 @@ _actors: frozenset[str] = frozenset()
 _malware: frozenset[str] = frozenset()
 _ransomware: frozenset[str] = frozenset()
 _generated_at: Optional[str] = None
+_generic_terms: frozenset[str] = frozenset()
 
 
 def _canonical_key(name: str) -> str:
@@ -77,8 +96,38 @@ def _keyset(names) -> frozenset[str]:
     return frozenset(out)
 
 
+def _load_generic_terms() -> frozenset[str]:
+    """Load generic edge qualifiers used for suffix/prefix retry matching."""
+    terms = set(ORG_SUFFIXES)
+    try:
+        with open(_DOMAIN_STOPWORDS_PATH, encoding="utf-8") as fh:
+            terms.update(line.strip().lower() for line in fh if line.strip())
+    except FileNotFoundError:
+        logger.warning("Domain stopword lexicon not found at %s", _DOMAIN_STOPWORDS_PATH)
+    return frozenset(terms)
+
+
+def _lookup_keys(name: str) -> tuple[str, ...]:
+    """Return the original key and edge-stripped retry keys for *name*."""
+    original = _canonical_key(name)
+    if not name:
+        return (original,)
+
+    terms = _generic_terms
+    tokens = re.findall(r"[A-Za-z0-9]+", name.lower())
+    while tokens and tokens[0] in terms:
+        tokens.pop(0)
+    while tokens and tokens[-1] in terms:
+        tokens.pop()
+
+    stripped = _canonical_key(" ".join(tokens))
+    if stripped and stripped != original:
+        return (original, stripped)
+    return (original,)
+
+
 def _load() -> None:
-    global _loaded, _actors, _malware, _ransomware, _generated_at
+    global _loaded, _actors, _malware, _ransomware, _generated_at, _generic_terms
     if _loaded:
         return
     _loaded = True
@@ -89,6 +138,7 @@ def _load() -> None:
         _malware = _keyset(data.get("malware"))
         _ransomware = _keyset(data.get("ransomware"))
         _generated_at = data.get("generated_at")
+        _generic_terms = _load_generic_terms()
         logger.info(
             "Gazetteer loaded: %d actors, %d malware, %d ransomware (generated %s)",
             len(_actors), len(_malware), len(_ransomware), _generated_at,
@@ -106,17 +156,17 @@ def _load() -> None:
 
 def is_known_actor(name: str) -> bool:
     _load()
-    return _canonical_key(name) in _actors
+    return any(key in _actors for key in _lookup_keys(name))
 
 
 def is_known_malware(name: str) -> bool:
     _load()
-    return _canonical_key(name) in _malware
+    return any(key in _malware for key in _lookup_keys(name))
 
 
 def is_known_ransomware(name: str) -> bool:
     _load()
-    return _canonical_key(name) in _ransomware
+    return any(key in _ransomware for key in _lookup_keys(name))
 
 
 def is_known(name: str, category: str) -> bool:
@@ -132,12 +182,12 @@ def is_known(name: str, category: str) -> bool:
 def category_of(name: str) -> Optional[str]:
     """Return the most specific known category for *name*, or None."""
     _load()
-    key = _canonical_key(name)
-    if key in _ransomware:
+    keys = _lookup_keys(name)
+    if any(key in _ransomware for key in keys):
         return "ransomware"
-    if key in _malware:
+    if any(key in _malware for key in keys):
         return "malware"
-    if key in _actors:
+    if any(key in _actors for key in keys):
         return "threat_actor"
     return None
 

{voidaccess-1.9.3 → voidaccess-1.9.4}/extractor/ner.py

@@ -20,6 +20,7 @@ import logging
 import re
 
 from extractor import entity_shape as _shape
+from extractor import gazetteer
 
 logger = logging.getLogger(__name__)
 
@@ -493,12 +494,31 @@ def extract_named_entities(text: str) -> dict[str, list[str]]:
                 for ent in doc.ents:
                     if ent.label_ == "ORG":
                         candidate = ent.text.strip()
+                        # A known actor can be returned by spaCy as an ORG when
+                        # the surrounding generic qualifier is included in the
+                        # span (for example, "LockBit Ransomware Group").
+                        # Preserve the extracted value but emit the gazetteer
+                        # classification so downstream normalization does not
+                        # misfile it as ORGANIZATION_NAME.
+                        category = gazetteer.category_of(candidate)
+                        if category == "threat_actor":
+                            result[THREAT_ACTOR_HANDLE].append(candidate)
+                            continue
+                        if category == "ransomware":
+                            result[RANSOMWARE_GROUP].append(candidate)
+                            continue
+                        if category == "malware":
+                            result[MALWARE_FAMILY].append(candidate)
+                            continue
                         # Accept only if it has organisation shape (proper-noun
                         # structure, org suffix, brand casing) — not merely
                         # "absent from a denylist".
                         if _shape.evaluate(ORGANIZATION_NAME, candidate).accept:
                             orgs.append(candidate)
                 result[ORGANIZATION_NAME] = _dedup(orgs)
+                result[THREAT_ACTOR_HANDLE] = _dedup(result[THREAT_ACTOR_HANDLE])
+                result[MALWARE_FAMILY] = _dedup(result[MALWARE_FAMILY])
+                result[RANSOMWARE_GROUP] = _dedup(result[RANSOMWARE_GROUP])
 
     except Exception:
         logger.exception("extract_named_entities encountered an unexpected error")

{voidaccess-1.9.3 → voidaccess-1.9.4}/extractor/normalizer.py

@@ -30,6 +30,7 @@ from urllib.parse import urlparse
 
 from extractor import entity_shape as _shape
 from extractor import confidence as _conf
+from extractor import gazetteer as _gazetteer
 
 logger = logging.getLogger(__name__)
 
@@ -1463,11 +1464,25 @@ def normalize_entities(
 
     noise_count = 0
 
-    for entity_type, values in raw_entities.items():
+    for raw_entity_type, values in raw_entities.items():
         for raw_value in values:
             if not raw_value or not raw_value.strip():
                 continue
 
+            entity_type = raw_entity_type
+
+            # NER may label a known actor/malware span as ORGANIZATION_NAME
+            # when generic qualifiers are included in the span.  Let the
+            # gazetteer classification win after its suffix/prefix retry,
+            # while leaving unrelated organization candidates unchanged.
+            if entity_type == "ORGANIZATION_NAME":
+                known_category = _gazetteer.category_of(raw_value)
+                entity_type = {
+                    "threat_actor": "THREAT_ACTOR_HANDLE",
+                    "malware": "MALWARE_FAMILY",
+                    "ransomware": "RANSOMWARE_GROUP",
+                }.get(known_category, entity_type)
+
             if entity_type in ("FILE_HASH_MD5", "FILE_HASH_SHA1", "FILE_HASH_SHA256"):
                 if not _validate_hash_length(entity_type, raw_value):
                     logger.debug(

{voidaccess-1.9.3 → voidaccess-1.9.4}/graph/builder.py

@@ -39,6 +39,22 @@ logger = logging.getLogger(__name__)
 
 _CO_OCCURRENCE_MAX_ENTITIES_PER_PAGE = 40
 _CO_OCCURRENCE_MAX_EDGES_PER_PAGE = 120
+
+# ORGANIZATION_NAME is deliberately absent from both sets below, even though it
+# is the largest (or second-largest) bucket in most real runs.  These sets were
+# introduced by v1.7.2, the same release that filtered "ORGANIZATION_NAME NER
+# noise ... generic security/programming vocabulary false positives" — the
+# exclusion is that fix, not an oversight.  The bucket is still ~25-45% junk on
+# measured runs (values like POST, JavaScript, HTML, HTTPS, IP, SCADA, EDR sit
+# alongside genuine orgs), so admitting it here would attach hundreds of
+# meaningless edges to every real hub and drown the signal the semantic filter
+# exists to preserve.
+#
+# Dependency, not a TODO: genuinely-real organizations *should* eventually be
+# able to participate in co-occurrence edges.  That is gated on Phase 2's
+# noise-reduction work actually improving what lands in this bucket — do not
+# widen eligibility before then, and re-measure the bucket's noise rate rather
+# than assuming Phase 2 fixed it.
 _CO_OCCURRENCE_HUB_TYPES = frozenset({
     "THREAT_ACTOR_HANDLE",
     "RANSOMWARE_GROUP",

{voidaccess-1.9.3 → voidaccess-1.9.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "voidaccess"
-version = "1.9.3"
+version = "1.9.4"
 description = "Dark web OSINT CLI — automated threat intelligence from query to report"
 readme = "README.md"
 license = "MIT"

{voidaccess-1.9.3 → voidaccess-1.9.4}/search/__init__.py

@@ -23,10 +23,12 @@ from db.search_engine_stats import (
 from search.query_builder import diversify_query
 from utils.async_utils import run_async
 from search.engine_catalog import DEFAULT_SEARCH_ENGINES, SEARCH_ENGINES
+from search import config as search_config
 
 logger = logging.getLogger(__name__)
 
-ENGINE_TIMEOUT = 45
+# Compatibility exports; runtime reads use the shared config module below.
+ENGINE_TIMEOUT = search_config.ENGINE_TIMEOUT
 
 ENGINE_WEIGHTS = {
     "darksearch": 1.0,
@@ -57,8 +59,8 @@ USER_AGENTS = [
 _ONION_URL_RE = re.compile(r'https?:\/\/[a-z0-9.]+\.onion', re.IGNORECASE)
 
 MAX_CONCURRENT = 10
-SEARCH_TIMEOUT = 60
-ENGINE_RETRY_COUNT = 2
+SEARCH_TIMEOUT = search_config.SEARCH_TIMEOUT
+ENGINE_RETRY_COUNT = search_config.ENGINE_RETRY_COUNT
 
 _ENGINE_STATUS: dict[str, dict] = {}
 _LAST_SEARCH_SUMMARY: dict[str, int] = {}
@@ -109,7 +111,7 @@ def get_search_session() -> aiohttp.ClientSession:
         connector = _tor_aiohttp_connector()
         _search_session = aiohttp.ClientSession(
             connector=connector,
-            timeout=aiohttp.ClientTimeout(total=SEARCH_TIMEOUT),
+            timeout=aiohttp.ClientTimeout(total=search_config.SEARCH_TIMEOUT),
         )
     return _search_session
 
@@ -129,7 +131,7 @@ async def fetch_with_timeout(
     """Fetch a URL with timeout using the provided or cached session."""
     if session is None:
         session = get_search_session()
-    return await session.get(url, timeout=aiohttp.ClientTimeout(total=SEARCH_TIMEOUT))
+    return await session.get(url, timeout=aiohttp.ClientTimeout(total=search_config.SEARCH_TIMEOUT))
 
 
 async def _fetch_engine(
@@ -145,12 +147,12 @@ async def _fetch_engine(
     headers = {"User-Agent": random.choice(USER_AGENTS)}
 
     async with semaphore:
-        for attempt in range(ENGINE_RETRY_COUNT + 1):
+        for attempt in range(search_config.ENGINE_RETRY_COUNT + 1):
             try:
-                async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=SEARCH_TIMEOUT)) as resp:
+                async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=search_config.SEARCH_TIMEOUT)) as resp:
                     if resp.status != 200:
-                        if attempt < ENGINE_RETRY_COUNT:
-                            await asyncio.sleep(0.5 * (attempt + 1))
+                        if attempt < search_config.ENGINE_RETRY_COUNT:
+                            await asyncio.sleep(search_config.retry_backoff(attempt))
                             continue
                         return EngineResult(
                             name=name,
@@ -177,13 +179,13 @@ async def _fetch_engine(
                     return EngineResult(name=name, links=links)
 
             except asyncio.TimeoutError:
-                if attempt < ENGINE_RETRY_COUNT:
-                    await asyncio.sleep(0.5 * (attempt + 1))
+                if attempt < search_config.ENGINE_RETRY_COUNT:
+                    await asyncio.sleep(search_config.retry_backoff(attempt))
                     continue
                 return EngineResult(name=name, links=[], error="timeout")
             except Exception as e:
-                if attempt < ENGINE_RETRY_COUNT:
-                    await asyncio.sleep(0.5 * (attempt + 1))
+                if attempt < search_config.ENGINE_RETRY_COUNT:
+                    await asyncio.sleep(search_config.retry_backoff(attempt))
                     continue
                 return EngineResult(name=name, links=[], error=str(e))
 

voidaccess-1.9.4/search/config.py

@@ -0,0 +1,13 @@
+"""Shared search timeout, retry, and backoff configuration."""
+
+# Keep these values in one module so the CLI and API search paths cannot drift.
+# The API path's 30-second timeout is the common baseline.
+ENGINE_TIMEOUT = 30
+SEARCH_TIMEOUT = 30
+ENGINE_RETRY_COUNT = 2
+RETRY_BACKOFF_BASE = 0.5
+
+
+def retry_backoff(attempt: int) -> float:
+    """Return the delay before retry number *attempt* (zero-based)."""
+    return RETRY_BACKOFF_BASE * (attempt + 1)

{voidaccess-1.9.3 → voidaccess-1.9.4}/search/search.py

@@ -22,10 +22,12 @@ from db.search_engine_stats import (
 from search.query_builder import diversify_query
 from search.engine_catalog import SEARCH_ENGINES
 from utils.async_utils import run_async
+from search import config as search_config
 
 logger = logging.getLogger(__name__)
 
-ENGINE_TIMEOUT = 30
+# Compatibility exports; runtime reads use the shared config module below.
+ENGINE_TIMEOUT = search_config.ENGINE_TIMEOUT
 
 ENGINE_WEIGHTS = {
     "darksearch": 1.0,
@@ -56,8 +58,8 @@ USER_AGENTS = [
 _ONION_URL_RE = re.compile(r'https?://[a-z0-9._-]+\.onion(?:/[^\s"\'<>]*)?', re.IGNORECASE)
 
 MAX_CONCURRENT = 10
-SEARCH_TIMEOUT = 30
-ENGINE_RETRY_COUNT = 2
+SEARCH_TIMEOUT = search_config.SEARCH_TIMEOUT
+ENGINE_RETRY_COUNT = search_config.ENGINE_RETRY_COUNT
 
 _ENGINE_STATUS: dict[str, dict] = {}
 _LAST_SEARCH_SUMMARY: dict[str, int] = {}
@@ -99,7 +101,7 @@ async def fetch_with_timeout(
     session: aiohttp.ClientSession,
 ) -> aiohttp.ClientResponse:
     """Fetch a URL with timeout using the provided session."""
-    return await session.get(url, timeout=aiohttp.ClientTimeout(total=SEARCH_TIMEOUT))
+    return await session.get(url, timeout=aiohttp.ClientTimeout(total=search_config.SEARCH_TIMEOUT))
 
 
 async def _fetch_engine(
@@ -115,12 +117,12 @@ async def _fetch_engine(
     headers = {"User-Agent": random.choice(USER_AGENTS)}
     
     async with semaphore:
-        for attempt in range(ENGINE_RETRY_COUNT + 1):
+        for attempt in range(search_config.ENGINE_RETRY_COUNT + 1):
             try:
-                async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=SEARCH_TIMEOUT)) as resp:
+                async with session.get(url, headers=headers, timeout=aiohttp.ClientTimeout(total=search_config.SEARCH_TIMEOUT)) as resp:
                     if resp.status != 200:
-                        if attempt < ENGINE_RETRY_COUNT:
-                            await asyncio.sleep(0.5 * (attempt + 1))
+                        if attempt < search_config.ENGINE_RETRY_COUNT:
+                            await asyncio.sleep(search_config.retry_backoff(attempt))
                             continue
                         return EngineResult(
                             name=name,
@@ -147,13 +149,13 @@ async def _fetch_engine(
                     return EngineResult(name=name, links=links)
 
             except asyncio.TimeoutError:
-                if attempt < ENGINE_RETRY_COUNT:
-                    await asyncio.sleep(0.5 * (attempt + 1))
+                if attempt < search_config.ENGINE_RETRY_COUNT:
+                    await asyncio.sleep(search_config.retry_backoff(attempt))
                     continue
                 return EngineResult(name=name, links=[], error="timeout")
             except Exception as e:
-                if attempt < ENGINE_RETRY_COUNT:
-                    await asyncio.sleep(0.5 * (attempt + 1))
+                if attempt < search_config.ENGINE_RETRY_COUNT:
+                    await asyncio.sleep(search_config.retry_backoff(attempt))
                     continue
                 return EngineResult(name=name, links=[], error=str(e))
         
@@ -269,7 +271,7 @@ async def _search_async(
     connector = _tor_aiohttp_connector()
     async with aiohttp.ClientSession(
         connector=connector,
-        timeout=aiohttp.ClientTimeout(total=SEARCH_TIMEOUT),
+        timeout=aiohttp.ClientTimeout(total=search_config.SEARCH_TIMEOUT),
     ) as session:
 
         async def run_engine(engine: dict) -> EngineResult: