voidaccess 1.6.3__tar.gz → 1.7.0__tar.gz

{voidaccess-1.6.3/voidaccess.egg-info → voidaccess-1.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidaccess
-Version: 1.6.3
+Version: 1.7.0
 Summary: Dark web OSINT CLI — automated threat intelligence from query to report
 Author: VoidAccess
 License-Expression: MIT
@@ -20,10 +20,16 @@ Requires-Dist: typer>=0.12
 Requires-Dist: rich>=13
 Requires-Dist: textual>=0.60
 Requires-Dist: click>=8.0
+Requires-Dist: fastapi>=0.115
 Requires-Dist: aiohttp>=3.9
 Requires-Dist: aiohttp-socks>=0.8
+Requires-Dist: alembic>=1.13
+Requires-Dist: apscheduler>=3.10
+Requires-Dist: pillow>=10.0
 Requires-Dist: sqlalchemy>=2.0
 Requires-Dist: aiosqlite>=0.20
+Requires-Dist: filelock>=3.14
+Requires-Dist: numpy>=1.26
 Requires-Dist: langchain>=0.2
 Requires-Dist: langchain-openai>=0.1
 Requires-Dist: langchain-anthropic>=0.1
@@ -35,7 +41,19 @@ Requires-Dist: httpx>=0.27
 Requires-Dist: spacy>=3.7
 Requires-Dist: beautifulsoup4>=4.12
 Requires-Dist: feedparser>=6.0
+Requires-Dist: networkx>=3.0
+Requires-Dist: pydantic>=2.7
+Requires-Dist: python-jose[cryptography]>=3.3
+Requires-Dist: passlib[bcrypt]>=1.7
+Requires-Dist: slowapi>=0.1.9
+Requires-Dist: redis>=5.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: openai>=1.30
+Requires-Dist: cryptography>=42.0
 Requires-Dist: python-dateutil>=2.9
+Requires-Dist: langdetect>=1.0
+Requires-Dist: pyvis>=0.3
+Requires-Dist: urllib3>=2.2
 Requires-Dist: trafilatura>=1.6
 Requires-Dist: requests>=2.31
 Requires-Dist: python-socks>=2.4
@@ -44,6 +62,25 @@ Requires-Dist: stix2>=3.0.2
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-asyncio; extra == "dev"
+Provides-Extra: nlp
+Requires-Dist: sentence-transformers>=3.0; extra == "nlp"
+Requires-Dist: torch>=2.3; extra == "nlp"
+Requires-Dist: transformers>=4.42; extra == "nlp"
+Provides-Extra: vector
+Requires-Dist: chromadb>=0.5; extra == "vector"
+Requires-Dist: sentence-transformers>=3.0; extra == "vector"
+Requires-Dist: torch>=2.3; extra == "vector"
+Provides-Extra: telegram
+Requires-Dist: telethon>=1.36; extra == "telegram"
+Provides-Extra: js
+Requires-Dist: playwright>=1.45; extra == "js"
+Provides-Extra: all
+Requires-Dist: chromadb>=0.5; extra == "all"
+Requires-Dist: playwright>=1.45; extra == "all"
+Requires-Dist: sentence-transformers>=3.0; extra == "all"
+Requires-Dist: telethon>=1.36; extra == "all"
+Requires-Dist: torch>=2.3; extra == "all"
+Requires-Dist: transformers>=4.42; extra == "all"
 Dynamic: license-file
 
 <div align="center">
@@ -67,9 +104,9 @@ Commercial threat intelligence platforms often charge prohibitive annual fees fo
 
 ---
 
-## What's New in v1.6.3
+## What's New in v1.7.0
 
-- **Release candidate alignment** — release metadata, package metadata, and docs now all report `v1.6.3`.
+- **Dependency optional-extras split** — `sentence-transformers`, `torch`, `transformers`, `telethon`, and `playwright` are now optional extras: `voidaccess[nlp]`, `voidaccess[telegram]`, `voidaccess[js]`, and `voidaccess[all]`.
 - **Optional clearnet ScrapingAnt integration** — paste sites and RSS feeds can now be routed through ScrapingAnt. Affects clearnet scraping only; Tor, `.onion`, GitHub, and GitLab traffic are never affected.
 - **Three independent products** — Web Scraping API, Residential Proxy transport, and Datacenter Proxy transport are documented separately so the API key is not conflated with the proxy credentials.
 - **One-shot CLI flags** — `voidaccess investigate --use-scraping-api` and `voidaccess investigate --use-proxies` select the REST API or proxy transport for a single run.
@@ -79,7 +116,7 @@ Commercial threat intelligence platforms often charge prohibitive annual fees fo
 
 ---
 
-## What's New in v1.6.3
+## What's New in v1.6.4
 
 - 37 new entity types across crypto, credentials, messaging, and network/forensic indicators.
 - YARA, Snort, Suricata, and IOC package ZIP exports.
@@ -273,7 +310,7 @@ Export formats:
 
 ## Actor Intelligence
 
-VoidAccess v1.6.3 persists actor profiles across investigations in `actor_profiles`, with linked aliases and infrastructure in `actor_aliases` and `actor_infrastructure`. Profiles are populated from threat actor, ransomware group, and handle entities, then enriched with co-occurring infrastructure and timeline events.
+VoidAccess v1.7.0 persists actor profiles across investigations in `actor_profiles`, with linked aliases and infrastructure in `actor_aliases` and `actor_infrastructure`. Profiles are populated from threat actor, ransomware group, and handle entities, then enriched with co-occurring infrastructure and timeline events.
 
 Cross-alias resolution scores five signals: shared infrastructure, shared PGP, string similarity, temporal co-activity, and co-investigation. Use `voidaccess actors` to list profiles, `voidaccess actor <handle>` for the full profile, `voidaccess actor <handle> --timeline` for chronology, and `voidaccess actor <handle> --note "text"` for analyst notes.
 

{voidaccess-1.6.3 → voidaccess-1.7.0}/README.md

@@ -19,9 +19,9 @@ Commercial threat intelligence platforms often charge prohibitive annual fees fo
 
 ---
 
-## What's New in v1.6.3
+## What's New in v1.7.0
 
-- **Release candidate alignment** — release metadata, package metadata, and docs now all report `v1.6.3`.
+- **Dependency optional-extras split** — `sentence-transformers`, `torch`, `transformers`, `telethon`, and `playwright` are now optional extras: `voidaccess[nlp]`, `voidaccess[telegram]`, `voidaccess[js]`, and `voidaccess[all]`.
 - **Optional clearnet ScrapingAnt integration** — paste sites and RSS feeds can now be routed through ScrapingAnt. Affects clearnet scraping only; Tor, `.onion`, GitHub, and GitLab traffic are never affected.
 - **Three independent products** — Web Scraping API, Residential Proxy transport, and Datacenter Proxy transport are documented separately so the API key is not conflated with the proxy credentials.
 - **One-shot CLI flags** — `voidaccess investigate --use-scraping-api` and `voidaccess investigate --use-proxies` select the REST API or proxy transport for a single run.
@@ -31,7 +31,7 @@ Commercial threat intelligence platforms often charge prohibitive annual fees fo
 
 ---
 
-## What's New in v1.6.3
+## What's New in v1.6.4
 
 - 37 new entity types across crypto, credentials, messaging, and network/forensic indicators.
 - YARA, Snort, Suricata, and IOC package ZIP exports.
@@ -225,7 +225,7 @@ Export formats:
 
 ## Actor Intelligence
 
-VoidAccess v1.6.3 persists actor profiles across investigations in `actor_profiles`, with linked aliases and infrastructure in `actor_aliases` and `actor_infrastructure`. Profiles are populated from threat actor, ransomware group, and handle entities, then enriched with co-occurring infrastructure and timeline events.
+VoidAccess v1.7.0 persists actor profiles across investigations in `actor_profiles`, with linked aliases and infrastructure in `actor_aliases` and `actor_infrastructure`. Profiles are populated from threat actor, ransomware group, and handle entities, then enriched with co-occurring infrastructure and timeline events.
 
 Cross-alias resolution scores five signals: shared infrastructure, shared PGP, string similarity, temporal co-activity, and co-investigation. Use `voidaccess actors` to list profiles, `voidaccess actor <handle>` for the full profile, `voidaccess actor <handle> --timeline` for chronology, and `voidaccess actor <handle> --note "text"` for analyst notes.
 
@@ -491,7 +491,7 @@ To report a security vulnerability, see [docs/SECURITY.md](docs/SECURITY.md).
 ## License
 
 MIT License. See [LICENSE](LICENSE) for details.
-
-
-
-
+
+
+
+

{voidaccess-1.6.3 → voidaccess-1.7.0}/export/ioc_package.py

@@ -54,7 +54,7 @@ logger = logging.getLogger(__name__)
 # ---------------------------------------------------------------------------
 
 PACKAGE_FORMAT = "voidaccess-ioc-v1"
-PACKAGE_VERSION = "1.6.3"
+PACKAGE_VERSION = "1.7.0"
 SNORT_SID_BASE = 9000001  # 9xxxxxx reserved for VoidAccess-generated rules
 
 # Map extractor upper-case entity types to the per-file bucket they belong to.
@@ -461,7 +461,15 @@ def _write_emails(entities: Iterable[Any]) -> str:
             v = _entity_value(e).strip()
             if v:
                 items.append(v)
-    return "\n".join(items).rstrip() + ("\n" if items else "")
+    content = "\n".join(items).rstrip()
+    # v1.7 MED-4: if entities matched but all had empty values, write a
+    # header comment so the file is never 0 bytes (0-byte files inside a
+    # ZIP are technically valid but confusing to downstream tooling).
+    if items and not content:
+        return "# Email addresses extracted by VoidAccess\n"
+    if content:
+        return content + "\n"
+    return ""
 
 
 def _write_urls(entities: Iterable[Any]) -> str:
@@ -654,6 +662,16 @@ def _safe_stix_json(entities: list[Any], investigation_id: Any) -> str:
         # session.  When running with dict entities (e.g. CLI export), we
         # can't pass them through that path — fall back to the empty bundle.
         bundle = investigation_to_stix_bundle(investigation_id)
+        try:
+            from export.stix import get_last_relationship_warning
+            warning = get_last_relationship_warning()
+            if warning:
+                logger.warning(
+                    "STIX relationships were not included in IOC package: %s",
+                    warning,
+                )
+        except Exception:
+            pass
         return bundle_to_json(bundle)
     except Exception as exc:
         logger.warning("STIX generation for IOC package failed: %s", exc)

{voidaccess-1.6.3 → voidaccess-1.7.0}/export/stix.py

@@ -18,10 +18,11 @@ from __future__ import annotations
 import json
 import logging
 import os
-from typing import Any, Optional, Union
+from typing import Any, Optional
 import uuid
 
 logger = logging.getLogger(__name__)
+_LAST_RELATIONSHIP_WARNING: Optional[str] = None
 
 # ---------------------------------------------------------------------------
 # Graceful import of stix2
@@ -196,6 +197,7 @@ def investigation_to_stix_bundle(
     - investigation not found
     """
     if not _STIX2_AVAILABLE:
+        _set_relationship_warning("stix2 is not installed")
         return _empty_bundle()
 
     filter_uuids: Optional[list[uuid.UUID]] = None
@@ -233,7 +235,13 @@ def investigation_to_stix_bundle(
             stix_id_map.setdefault(entity.value, actor.id)
 
     if include_relationships and stix_objects:
-        stix_objects.extend(_build_stix_relationships(investigation_id, stix_id_map))
+        relationships, relationship_warning = _build_stix_relationships(
+            investigation_id, stix_id_map
+        )
+        stix_objects.extend(relationships)
+        _set_relationship_warning(relationship_warning)
+    else:
+        _set_relationship_warning(None)
 
     try:
         return stix2.Bundle(*stix_objects, allow_custom=True)
@@ -280,6 +288,16 @@ def _empty_bundle() -> Any:
         return stix2.Bundle()
 
 
+def _set_relationship_warning(message: Optional[str]) -> None:
+    global _LAST_RELATIONSHIP_WARNING
+    _LAST_RELATIONSHIP_WARNING = message
+
+
+def get_last_relationship_warning() -> Optional[str]:
+    """Return the last relationship-build warning, if any."""
+    return _LAST_RELATIONSHIP_WARNING
+
+
 def _load_entities_for_investigation(
     investigation_id: Any,
     entity_ids: Optional[list[uuid.UUID]] = None,
@@ -310,6 +328,11 @@ def _load_entities_for_investigation(
             if inv is None:
                 return []
 
+            # v1.7 MED-9: use .c.column explicitly for the IN clause subquery.
+            # In SQLAlchemy 2.x, passing a plain subquery object to Column.in_()
+            # triggers a "Subquery is not yet a select()" coercion warning.
+            # Using subquery.c.column accesses the explicit column object so
+            # the ORM knows exactly what to coerce — no suppression needed.
             linked_ids_subq = (
                 session.query(InvestigationEntityLink.entity_id)
                 .filter(InvestigationEntityLink.investigation_id == inv.id)
@@ -317,7 +340,7 @@ def _load_entities_for_investigation(
             )
             q = session.query(Entity).filter(
                 (Entity.investigation_id == inv.id)
-                | Entity.id.in_(linked_ids_subq)
+                | Entity.id.in_(linked_ids_subq.c.entity_id)
             )
             db_entities = q.all()
 
@@ -353,14 +376,14 @@ def _load_entities_for_investigation(
 def _build_stix_relationships(
     investigation_id: Any,
     stix_id_map: dict[str, str],
-) -> list[Any]:
+) -> tuple[list[Any], Optional[str]]:
     """
     Build STIX Relationship objects from graph edges for the investigation.
 
     Returns [] on any error.
     """
     if not _STIX2_AVAILABLE:
-        return []
+        return [], "stix2 is not installed"
     try:
         from graph.builder import build_graph_from_db  # noqa: PLC0415
 
@@ -385,10 +408,10 @@ def _build_stix_relationships(
                 relationships.append(rel)
             except Exception:
                 continue
-        return relationships
+        return relationships, None
     except Exception as exc:
         logger.warning("_build_stix_relationships failed: %s", exc)
-        return []
+        return [], f"_build_stix_relationships failed: {exc}"
 
 
 def _edge_type_to_stix(edge_type: str) -> str:

{voidaccess-1.6.3 → voidaccess-1.7.0}/extractor/normalizer.py

@@ -20,8 +20,7 @@ import os
 import re
 import unicodedata
 from dataclasses import dataclass, field
-from datetime import datetime, timezone
-from typing import Optional, List
+from typing import Optional
 import uuid
 
 logger = logging.getLogger(__name__)
@@ -734,6 +733,130 @@ def is_blocked_entity(entity_type: str, entity_value: str) -> bool:
     return False
 
 
+# ---------------------------------------------------------------------------
+# Reserved-range / placeholder blocklists  (v1.7 — Q-1/Q-2 fixes)
+# ---------------------------------------------------------------------------
+#
+# RFC 5737 documentation IP ranges — reserved for use in examples/docs,
+# never real routable addresses.
+#   192.0.2.0/24  (TEST-NET-1)
+#   198.51.100.0/24 (TEST-NET-2)
+#   203.0.113.0/24 (TEST-NET-3)
+_RFC5737_NETWORKS: list[tuple[str, str]] = [
+    ("192.0.2.0", "255.255.255.0"),    # 192.0.2.0/24
+    ("198.51.100.0", "255.255.255.0"),  # 198.51.100.0/24
+    ("203.0.113.0", "255.255.255.0"),   # 203.0.113.0/24
+]
+
+# RFC 2606 reserved domain names.
+_RFC2606_DOMAINS: frozenset[str] = frozenset({
+    "example.com",
+    "example.net",
+    "example.org",
+    "example.edu",
+    "test",
+})
+
+# Well-known placeholder email patterns that appear in examples/docs.
+_PLACEHOLDER_EMAIL_RE = re.compile(
+    r"(?i)^(test|user|admin|example|foo|bar|noreply|no.reply)"
+    r"@(example\.com|example\.net|example\.org|domain\.com|localhost)$"
+)
+
+
+def _is_rfc5737_ip(value: str) -> bool:
+    """Return True if value is an IPv4 address in an RFC 5737 documentation range."""
+    try:
+        ip = __import__("ipaddress").ip_address(value.strip())
+        if not isinstance(ip, __import__("ipaddress").IPv4Address):
+            return False
+        for net_str, mask_str in _RFC5737_NETWORKS:
+            network = __import__("ipaddress").IPv4Network(f"{net_str}/{mask_str}", strict=False)
+            if ip in network:
+                return True
+        return False
+    except Exception:
+        return False
+
+
+def _is_rfc2606_domain(value: str) -> bool:
+    """Return True if value is an RFC 2606 reserved domain."""
+    v = value.strip().lower()
+    # Exact match
+    if v in _RFC2606_DOMAINS:
+        return True
+    # Suffix match: "foo.example.com" etc.
+    parts = v.split(".")
+    for i in range(1, len(parts)):
+        suffix = ".".join(parts[i:])
+        if suffix in _RFC2606_DOMAINS:
+            return True
+    return False
+
+
+def _is_placeholder_email(value: str) -> bool:
+    """Return True if value looks like a documentation/example email address."""
+    return bool(_PLACEHOLDER_EMAIL_RE.match(value.strip()))
+
+
+# ---------------------------------------------------------------------------
+# Source quality assignment  (v1.7 — Q-2: GitHub source tier)
+# ---------------------------------------------------------------------------
+#
+# Known low-trust URL patterns that are documentation/example content, not
+# actual threat intelligence.  Entities extracted from these are given
+# source_quality=0.6 and are further de-prioritised in the entity cap.
+
+_LOW_TRUST_URL_PATH_RE = re.compile(
+    r"(?i)"
+    # README, CHANGELOG, LICENSE, CONTRIBUTING, SECURITY, INSTALL, docs/
+    # anything at the repo root that looks like documentation
+    r"(^|/)(README|CHANGELOG|LICENSE|CONTRIBUTING|SECURITY|"
+    r"INSTALL|USER_GUIDE|DOCUMENTATION"
+    r"(\.md|\.txt|\.rst|\.html)?"
+    r"|(^|/)(docs?|examples?|specs?|schemas?)"
+    r"|(\.github[/\\])"          # GitHub Actions / stored workflow files
+    r"|((^|/)examples?/)"         # example/ directory
+    r"|(/issues/[0-9]+)"          # GitHub issue comment — moderate signal
+    r"|(/pull/[0-9]+)"            # PR comment — moderate signal
+    r"|(\.git)"                   # git metadata URLs
+    r")"
+)
+
+
+def _source_quality_from_url(url: str) -> float:
+    """
+    Return a source-quality multiplier (0.0-1.0) for a given source URL.
+
+    Higher values = more trustworthy source.  Used to de-prioritise
+    documentation and example content vs. actual threat intel sources.
+    """
+    if not url:
+        return 1.0
+    url_lower = url.lower()
+
+    # GitHub
+    if "github.com" in url_lower:
+        if _LOW_TRUST_URL_PATH_RE.search(url):
+            return 0.6
+        # GitHub issue / PR comments — moderate value
+        if "/issues/" in url_lower or "/pull/" in url_lower:
+            return 0.7
+        # Default GitHub pages (raw blob from a repo)
+        return 0.6
+
+    # GitLab
+    if "gitlab.com" in url_lower:
+        if _LOW_TRUST_URL_PATH_RE.search(url):
+            return 0.6
+        if "/issues/" in url_lower or "/merge_requests/" in url_lower:
+            return 0.7
+        return 0.6
+
+    # Default: primary threat source
+    return 1.0
+
+
 _ORG_NOISE_RE = re.compile(
     r'\b(paid|bought|sold|hacked|leaked)\b'
     r'|\b(attempt|operations|disrupted)\b'
@@ -770,11 +893,22 @@ class NormalizedEntity:
     page_id: Optional[uuid.UUID]
     context_snippet: str = field(default="")
     extraction_method: str = field(default="")
+    # v1.7 — source_quality reflects the trustworthiness of the source.
+    # Higher = more credible.  Used to downgrade README / documentation sources.
+    # 1.0 = primary threat source (dark web, paste sites, RSS, forum posts)
+    # 0.6 = secondary source (GitHub README, GitLab repo files)
+    # 0.0 = placeholder / reserved range (RFC 5737 IPs, RFC 2606 domains)
+    source_quality: float = field(default=1.0)
 
     @property
     def canonical_value(self) -> str:
         return self.value
 
+    @property
+    def is_placeholder(self) -> bool:
+        """True when the entity is a known reserved-range value, not real intel."""
+        return self.source_quality == 0.0
+
     def get(self, key: str, default=None):
         return getattr(self, key, default)
 
@@ -1152,6 +1286,34 @@ def normalize_entities(
             if dedup_key in seen_values:
                 continue
             seen_values.add(dedup_key)
+
+            # v1.7 Q-1 — reserved-range / placeholder filtering.
+            # Assign source_quality=0.0 so these are excluded from the entity
+            # store (pipeline.py's apply_entity_cap drops confidence < 0.80,
+            # and source_quality=0.0 propagates a near-zero effective score).
+            source_quality = _source_quality_from_url(page_url)
+
+            if entity_type == "IP_ADDRESS" and _is_rfc5737_ip(canonical):
+                source_quality = 0.0
+                logger.debug(
+                    "RFC 5737 IP rejected (reserved documentation range): %s", canonical
+                )
+                continue
+
+            if entity_type == "DOMAIN" and _is_rfc2606_domain(canonical):
+                source_quality = 0.0
+                logger.debug(
+                    "RFC 2606 domain rejected (reserved documentation domain): %s", canonical
+                )
+                continue
+
+            if entity_type == "EMAIL_ADDRESS" and _is_placeholder_email(canonical):
+                source_quality = 0.0
+                logger.debug(
+                    "Placeholder email rejected (documentation/example address): %s", canonical
+                )
+                continue
+
             snip = _context_snippet(page_text, canonical) if page_text else ""
             result.append(
                 NormalizedEntity(
@@ -1162,6 +1324,7 @@ def normalize_entities(
                     page_id=page_id,
                     context_snippet=snip,
                     extraction_method=_extraction_method_for(entity_type),
+                    source_quality=source_quality,
                 )
             )
 

{voidaccess-1.6.3 → voidaccess-1.7.0}/extractor/pipeline.py

@@ -42,6 +42,13 @@ PER_TYPE_CAPS = {
     "PERSON_NAME": 30,
     "LOCATION": 20,
     "THREAT_ACTOR_HANDLE": 80,
+    # v1.7 MED-3: DATE entities from RSS changelogs and GitHub commit
+    # timestamps dominated one run (85/182 entities).  A hard sub-cap keeps
+    # genuinely relevant dates (ransomware leak announcement, CVE publication)
+    # while suppressing changelog/commit noise.  20 dates per investigation is
+    # far more than any analyst needs; the global entity_cap (400) handles
+    # the ceiling for all types combined.
+    "DATE": 20,
 }
 
 # ---------------------------------------------------------------------------
@@ -545,27 +552,40 @@ def apply_entity_cap(
     """
     original_count = len(entities)
 
-    # Step a: confidence filter
+    # Step a: confidence filter — also drops placeholder entities (source_quality=0.0)
     filtered = [e for e in entities if e.confidence >= 0.80]
-    removed_confidence = original_count - len(filtered)
+    placeholder_count = sum(1 for e in entities if getattr(e, "source_quality", 1.0) == 0.0)
+    filtered = [e for e in filtered if getattr(e, "source_quality", 1.0) != 0.0]
+    removed_confidence = original_count - placeholder_count - len(filtered)
     if removed_confidence:
-        logger.warning(f"Entity confidence filter removed {removed_confidence} low-confidence entities")
+        logger.warning(f"Entity confidence/placeholder filter removed {removed_confidence} low-confidence entities")
+    if placeholder_count:
+        logger.info(f"Entity placeholder filter dropped {placeholder_count} reserved-range entities")
 
     # Count occurrences per entity (by type+value) and boost confidence
+    # v1.7 Q-2: multiply the boost by source_quality so low-quality sources
+    # (GitHub README at 0.6) get a smaller boost than primary sources (1.0).
     total_pages = len(set(e.source_url for e in filtered)) or 1
     for ent in filtered:
         occ = _occurrence_count(ent, filtered)
         ent._occurrence = occ
         occurrence_ratio = occ / total_pages
-        confidence_boost = min(occurrence_ratio * 0.10, 0.10)
+        source_quality = getattr(ent, "source_quality", 1.0)
+        confidence_boost = min(occurrence_ratio * 0.10, 0.10) * source_quality
         ent.confidence = min(ent.confidence + confidence_boost, 1.0)
 
     # Step b: per-type sub-caps
     filtered = _apply_per_type_caps(filtered)
 
-    # Step c: sort and cap
+    # Step c: sort and cap — source_quality is a secondary sort key so
+    # GitHub/README entities with equal confidence are ranked below primary sources.
     if len(filtered) > cap:
-        filtered.sort(key=lambda e: (-e.confidence, _type_priority(e.entity_type), -e._occurrence))
+        filtered.sort(key=lambda e: (
+            -e.confidence,
+            -getattr(e, "source_quality", 1.0),  # higher quality = higher rank
+            _type_priority(e.entity_type),
+            -e._occurrence,
+        ))
         filtered = filtered[:cap]
         logger.warning(
             f"Entity cap applied: {original_count} entities reduced to {len(filtered)} "

{voidaccess-1.6.3 → voidaccess-1.7.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "voidaccess"
-version = "1.6.3"
+version = "1.7.0"
 description = "Dark web OSINT CLI — automated threat intelligence from query to report"
 readme = "README.md"
 license = "MIT"
@@ -26,10 +26,16 @@ dependencies = [
     "rich>=13",
     "textual>=0.60",
     "click>=8.0",
+    "fastapi>=0.115",
     "aiohttp>=3.9",
     "aiohttp-socks>=0.8",
+    "alembic>=1.13",
+    "apscheduler>=3.10",
+    "pillow>=10.0",
     "sqlalchemy>=2.0",
     "aiosqlite>=0.20",
+    "filelock>=3.14",
+    "numpy>=1.26",
     "langchain>=0.2",
     "langchain-openai>=0.1",
     "langchain-anthropic>=0.1",
@@ -41,7 +47,19 @@ dependencies = [
     "spacy>=3.7",
     "beautifulsoup4>=4.12",
     "feedparser>=6.0",
+    "networkx>=3.0",
+    "pydantic>=2.7",
+    "python-jose[cryptography]>=3.3",
+    "passlib[bcrypt]>=1.7",
+    "slowapi>=0.1.9",
+    "redis>=5.0",
+    "pyyaml>=6.0",
+    "openai>=1.30",
+    "cryptography>=42.0",
     "python-dateutil>=2.9",
+    "langdetect>=1.0",
+    "pyvis>=0.3",
+    "urllib3>=2.2",
     "trafilatura>=1.6",
     "requests>=2.31",
     "python-socks>=2.4",
@@ -51,6 +69,27 @@ dependencies = [
 
 [project.optional-dependencies]
 dev = ["pytest", "pytest-asyncio"]
+nlp = [
+    "sentence-transformers>=3.0",
+    "torch>=2.3",
+    "transformers>=4.42",
+]
+vector = [
+    "chromadb>=0.5",
+    "sentence-transformers>=3.0",
+    "torch>=2.3",
+]
+telegram = ["telethon>=1.36"]
+js = ["playwright>=1.45"]
+# Convenience: all optional packages at once
+all = [
+    "chromadb>=0.5",
+    "playwright>=1.45",
+    "sentence-transformers>=3.0",
+    "telethon>=1.36",
+    "torch>=2.3",
+    "transformers>=4.42",
+]
 
 [project.scripts]
 voidaccess = "voidaccess_cli.main:app"

{voidaccess-1.6.3 → voidaccess-1.7.0}/scraper/scrape.py

@@ -476,7 +476,7 @@ async def _fetch_one(
                     connector = _tor_aiohttp_connector() if is_onion_url(url) else _direct_tcp_connector()
                     async with aiohttp.ClientSession(
                         connector=connector,
-                        timeout=aiohttp.ClientTimeout(connect=5, sock_read=25 if not is_onion_url(url) else 5),
+                        timeout=aiohttp.ClientTimeout(connect=5, sock_read=25 if not is_onion_url(url) else 20),
                     ) as local_session:
                         async with local_session.get(url, headers=headers) as resp:
                             if resp.status in RETRYABLE_STATUS:
@@ -523,7 +523,10 @@ async def _fetch_one(
                 display_text = f"{title} - {db_text}" if db_text else title
 
                 # --- Playwright fallback for JS-rendered pages ---
-                if PLAYWRIGHT_ENABLED and db_text and len(db_text) < 300:
+                # Onion pages are more likely to be JS-rendered (Tor Browser default)
+                # and more likely to have slow initial loads, so use a lower threshold.
+                _pw_threshold = 500 if is_onion_url(url) else 300
+                if PLAYWRIGHT_ENABLED and db_text and len(db_text) < _pw_threshold:
                     # Import lazily to avoid import errors when playwright not installed
                     try:
                         from scraper.scrape_js import fetch_with_playwright, is_js_rendered
@@ -559,6 +562,15 @@ async def _fetch_one(
                         _logger.debug("Playwright fallback failed: %s", e)
                         pass
 
+                # HIGH-3: warn when a page fetched but yielded no extractable text —
+                # helps distinguish "fetch failed / timed out" from "genuinely empty".
+                # Onion pages are exempt because many Tor sites serve minimal content.
+                if not db_text and not is_onion_url(url):
+                    _logger.warning(
+                        "Page fetched but no text extracted: %s",
+                        url[:80] if len(url) > 80 else url,
+                    )
+
                 return url, display_text, raw_bytes, db_text, posted_at
 
             except (aiohttp.ClientError, asyncio.TimeoutError) as exc: