voidaccess 1.4.2__tar.gz → 1.4.3__tar.gz

{voidaccess-1.4.2 → voidaccess-1.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidaccess
-Version: 1.4.2
+Version: 1.4.3
 Summary: Dark web OSINT CLI — automated threat intelligence from query to report
 Author: VoidAccess
 License-Expression: MIT
@@ -28,6 +28,7 @@ Requires-Dist: langchain-openai>=0.1
 Requires-Dist: langchain-anthropic>=0.1
 Requires-Dist: langchain-google-genai>=1.0
 Requires-Dist: langchain-groq>=0.1
+Requires-Dist: langchain-ollama>=0.1
 Requires-Dist: python-dotenv>=1.0
 Requires-Dist: httpx>=0.27
 Requires-Dist: spacy>=3.7

{voidaccess-1.4.2 → voidaccess-1.4.3}/auth/token_blacklist.py

@@ -19,16 +19,22 @@ logger = logging.getLogger(__name__)
 _pool: Optional[redis.ConnectionPool] = None
 _redis_client: Optional[redis.Redis] = None
 _blacklist_enabled = False
+_redis_unavailable = False
 
 BLACKLIST_PREFIX = "blacklist:"
 
 
 async def _get_redis() -> Optional[redis.Redis]:
-    global _pool, _redis_client, _blacklist_enabled
+    global _pool, _redis_client, _blacklist_enabled, _redis_unavailable
+
+    if _redis_unavailable:
+        return None
 
     if REDIS_URL is None:
         _blacklist_enabled = False
-        logger.warning("REDIS_URL not configured - token blacklist disabled")
+        if not _redis_unavailable:
+            logger.info("REDIS_URL not configured — token blacklist disabled")
+        _redis_unavailable = True
         return None
 
     if _redis_client is None:
@@ -42,9 +48,10 @@ async def _get_redis() -> Optional[redis.Redis]:
             _blacklist_enabled = True
             logger.info("Token blacklist enabled via Redis")
         except Exception as e:
-            logger.warning(f"Failed to connect to Redis: %s - token blacklist disabled", e)
+            logger.warning("Failed to connect to Redis: %s — token blacklist disabled", e)
             _redis_client = None
             _blacklist_enabled = False
+            _redis_unavailable = True
 
     return _redis_client
 

{voidaccess-1.4.2 → voidaccess-1.4.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "voidaccess"
-version = "1.4.2"
+version = "1.4.3"
 description = "Dark web OSINT CLI — automated threat intelligence from query to report"
 readme = "README.md"
 license = "MIT"
@@ -34,6 +34,7 @@ dependencies = [
     "langchain-anthropic>=0.1",
     "langchain-google-genai>=1.0",
     "langchain-groq>=0.1",
+    "langchain-ollama>=0.1",
     "python-dotenv>=1.0",
     "httpx>=0.27",
     "spacy>=3.7",

{voidaccess-1.4.2 → voidaccess-1.4.3}/search/circuit_breaker.py

@@ -29,6 +29,7 @@ CIRCUIT_PREFIX = "circuit:"
 _pool: Optional[redis.ConnectionPool] = None
 _redis_client: Optional[redis.Redis] = None
 _circuit_breaker_enabled = False
+_redis_unavailable = False  # latched True once we decide Redis isn't reachable
 
 _engine_failures: dict[str, int] = {}
 _engine_last_success: dict[str, float] = {}
@@ -37,11 +38,16 @@ _engine_open_time: dict[str, float] = {}
 
 
 async def _get_redis() -> Optional[redis.Redis]:
-    global _pool, _redis_client, _circuit_breaker_enabled
+    global _pool, _redis_client, _circuit_breaker_enabled, _redis_unavailable
+
+    if _redis_unavailable:
+        return None
 
     if REDIS_URL is None:
         _circuit_breaker_enabled = False
-        logger.warning("REDIS_URL not configured - circuit breaker using in-memory fallback")
+        if not _redis_unavailable:
+            logger.info("REDIS_URL not configured — circuit breaker using in-memory fallback")
+        _redis_unavailable = True
         return None
 
     if _redis_client is None:
@@ -55,9 +61,10 @@ async def _get_redis() -> Optional[redis.Redis]:
             _circuit_breaker_enabled = True
             logger.info("Circuit breaker enabled via Redis")
         except Exception as e:
-            logger.warning(f"Failed to connect to Redis: %s - circuit breaker using in-memory fallback", e)
+            logger.warning("Failed to connect to Redis: %s — circuit breaker using in-memory fallback", e)
             _redis_client = None
             _circuit_breaker_enabled = False
+            _redis_unavailable = True
 
     return _redis_client
 

{voidaccess-1.4.2 → voidaccess-1.4.3}/sources/enrichment.py

@@ -40,11 +40,30 @@ THREATFOX_URL = "https://threatfox-api.abuse.ch/api/v1/"
 # All HTTP calls use at most 30s client timeout (enforced per request).
 
 
+_ABUSECH_WARNED = False
+
+
 def _abusech_headers() -> dict[str, str]:
     key = (os.environ.get("ABUSECH_API_KEY") or "").strip()
     return {"Auth-Key": key} if key else {}
 
 
+def _abusech_enabled() -> bool:
+    """abuse.ch APIs (MalwareBazaar/ThreatFox/URLhaus) require an Auth-Key
+    since 2024. Return False (and log once) when the key is missing so we
+    skip the request entirely instead of spamming HTTP 401."""
+    global _ABUSECH_WARNED
+    if (os.environ.get("ABUSECH_API_KEY") or "").strip():
+        return True
+    if not _ABUSECH_WARNED:
+        logger.info(
+            "abuse.ch enrichment skipped — set ABUSECH_API_KEY "
+            "(free at https://auth.abuse.ch) to enable MalwareBazaar/ThreatFox/URLhaus."
+        )
+        _ABUSECH_WARNED = True
+    return False
+
+
 def is_onion_url(url: str) -> bool:
     """
     Return True if *url* looks like a Tor hidden service URL (.onion).
@@ -218,6 +237,8 @@ def otx_pulse_to_page(pulse: dict) -> dict:
 
 async def fetch_malwarebazaar(query: str, limit: int = 20) -> list[dict]:
     """Query MalwareBazaar by tag then by signature."""
+    if not _abusech_enabled():
+        return []
     results: list[dict] = []
     q = (query or "").strip()
     if not q:
@@ -312,6 +333,8 @@ async def fetch_malwarebazaar(query: str, limit: int = 20) -> list[dict]:
 
 async def fetch_threatfox(query: str, limit: int = 50) -> list[dict]:
     """Search ThreatFox IOCs by search term."""
+    if not _abusech_enabled():
+        return []
     results: list[dict] = []
     q = (query or "").strip()
     if not q:
@@ -397,6 +420,8 @@ async def fetch_threatfox(query: str, limit: int = 50) -> list[dict]:
 
 async def fetch_urlhaus(query: str, limit: int = 20) -> list[dict]:
     """Search URLhaus by tag."""
+    if not _abusech_enabled():
+        return []
     results: list[dict] = []
     q = (query or "").strip()
     if not q:

{voidaccess-1.4.2 → voidaccess-1.4.3}/sources/hash_reputation.py

@@ -164,11 +164,18 @@ async def query_malwarebazaar(hash_value: str) -> dict[str, Any]:
     """
     POST get_info to MalwareBazaar for a file hash.
 
-    No API key required. Returns malware family, file type, first seen date.
+    Requires ABUSECH_API_KEY (abuse.ch made auth-key mandatory in 2024).
+    Returns malware family, file type, first seen date.
     """
+    api_key = (os.environ.get("ABUSECH_API_KEY") or "").strip()
+    if not api_key:
+        return {"found": False, "source": "malwarebazaar_no_key"}
     try:
         timeout = aiohttp.ClientTimeout(total=15)
-        headers = {"User-Agent": "VoidAccess-OSINT/1.1 (security research)"}
+        headers = {
+            "User-Agent": "VoidAccess-OSINT/1.1 (security research)",
+            "Auth-Key": api_key,
+        }
         async with aiohttp.ClientSession(timeout=timeout, headers=headers) as session:
             async with session.post(
                 MALWAREBAZAAR_URL,
@@ -212,11 +219,18 @@ async def query_threatfox(hash_value: str) -> dict[str, Any]:
     """
     POST search_ioc to ThreatFox for a file hash.
 
-    No API key required. Returns malware family and associated IOCs.
+    Requires ABUSECH_API_KEY (abuse.ch made auth-key mandatory in 2024).
+    Returns malware family and associated IOCs.
     """
+    api_key = (os.environ.get("ABUSECH_API_KEY") or "").strip()
+    if not api_key:
+        return {"found": False, "source": "threatfox_no_key"}
     try:
         timeout = aiohttp.ClientTimeout(total=15)
-        headers = {"User-Agent": "VoidAccess-OSINT/1.1 (security research)"}
+        headers = {
+            "User-Agent": "VoidAccess-OSINT/1.1 (security research)",
+            "Auth-Key": api_key,
+        }
         async with aiohttp.ClientSession(timeout=timeout, headers=headers) as session:
             async with session.post(
                 THREATFOX_URL,

{voidaccess-1.4.2 → voidaccess-1.4.3}/voidaccess.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voidaccess
-Version: 1.4.2
+Version: 1.4.3
 Summary: Dark web OSINT CLI — automated threat intelligence from query to report
 Author: VoidAccess
 License-Expression: MIT
@@ -28,6 +28,7 @@ Requires-Dist: langchain-openai>=0.1
 Requires-Dist: langchain-anthropic>=0.1
 Requires-Dist: langchain-google-genai>=1.0
 Requires-Dist: langchain-groq>=0.1
+Requires-Dist: langchain-ollama>=0.1
 Requires-Dist: python-dotenv>=1.0
 Requires-Dist: httpx>=0.27
 Requires-Dist: spacy>=3.7

{voidaccess-1.4.2 → voidaccess-1.4.3}/voidaccess.egg-info/requires.txt

@@ -10,6 +10,7 @@ langchain-openai>=0.1
 langchain-anthropic>=0.1
 langchain-google-genai>=1.0
 langchain-groq>=0.1
+langchain-ollama>=0.1
 python-dotenv>=1.0
 httpx>=0.27
 spacy>=3.7

{voidaccess-1.4.2 → voidaccess-1.4.3}/voidaccess_cli/commands/configure.py

@@ -57,8 +57,23 @@ def _test_llm_key(provider: str, api_key: str, model: str) -> bool:
         return True
     try:
         from voidaccess.llm import get_llm
+    except ImportError as exc:
+        missing = str(exc).split("'")[-2] if "'" in str(exc) else str(exc)
+        console.print(
+            f"[yellow]Skipped validation:[/yellow] missing dependency [bold]{missing}[/bold]. "
+            f"Install with: [bold]pip install {missing.replace('_', '-')}[/bold]"
+        )
+        return False
+    try:
         get_llm(model, api_keys={cli_config.PROVIDER_ENV.get(provider, ""): api_key})
         return True
+    except ImportError as exc:
+        missing = str(exc).split("'")[-2] if "'" in str(exc) else str(exc)
+        console.print(
+            f"[yellow]Skipped validation:[/yellow] missing dependency [bold]{missing}[/bold]. "
+            f"Install with: [bold]pip install {missing.replace('_', '-')}[/bold]"
+        )
+        return False
     except Exception as exc:
         console.print(f"[yellow]Could not validate key:[/yellow] {exc}")
         return False
@@ -117,25 +132,7 @@ def _prompt_output_dir(cfg: dict) -> None:
 
 
 def _ensure_spacy_model() -> None:
-    console.print("\n  → Downloading spaCy NER model...")
-    try:
-        import subprocess
-        import sys
-
-        result = subprocess.run(
-            [sys.executable, "-m", "spacy", "download", "en_core_web_sm"],
-            capture_output=True,
-            text=True,
-        )
-        if result.returncode == 0:
-            console.print("  ✓ spaCy model ready")
-        else:
-            console.print(
-                "  ⚠ spaCy download failed — run manually: "
-                "python -m spacy download en_core_web_sm"
-            )
-    except Exception as e:
-        console.print(f"  ⚠ spaCy: {e}")
+    cli_config.ensure_spacy_model()
 
 
 @app.callback()

{voidaccess-1.4.2 → voidaccess-1.4.3}/voidaccess_cli/commands/investigate.py

@@ -56,10 +56,13 @@ def run(
     except Exception:
         import subprocess, sys
         from rich.console import Console
-        Console().print("  [dim]→[/dim] Installing spaCy NER model (one-time)...")
+        Console().print(
+            "  [dim]→[/dim] Installing spaCy NER model (one-time)..."
+        )
         subprocess.run(
-            [sys.executable, "-m", "spacy", "download", "en_core_web_sm"],
-            capture_output=True,
+            [sys.executable, "-m", "spacy",
+             "download", "en_core_web_sm"],
+            capture_output=True
         )
 
     if quiet:
@@ -434,6 +437,23 @@ async def _run_investigation(
         }
     )
 
+    # Close any cached aiohttp sessions so the event loop exits cleanly
+    # (otherwise aiohttp prints "Unclosed client session" warnings).
+    await _close_cached_sessions()
+
+
+async def _close_cached_sessions() -> None:
+    try:
+        from scraper.scrape import close_cached_sessions as _close_scrape
+        await _close_scrape()
+    except Exception:
+        pass
+    try:
+        from search import close_search_session as _close_search
+        await _close_search()
+    except Exception:
+        pass
+
 
 # ---------------------------------------------------------------------------
 # Side-source helpers (each gracefully degrades if module missing/disabled)

{voidaccess-1.4.2 → voidaccess-1.4.3}/voidaccess_cli/config.py

@@ -139,6 +139,65 @@ def db_url() -> str:
     return f"sqlite:///{DB_PATH.as_posix()}"
 
 
+def ensure_spacy_model(model_name: str = "en_core_web_sm") -> bool:
+    """
+    Ensure spaCy NER model is installed. Returns True if model is loadable
+    after this call. Handles PEP 668 (externally-managed-environment) on
+    Debian/Ubuntu/Kali by setting PIP_BREAK_SYSTEM_PACKAGES=1, and uses
+    PIP_USER=1 outside virtualenvs.
+
+    Prints progress via rich. Safe to call repeatedly.
+    """
+    import sys
+    import subprocess
+
+    try:
+        import spacy
+        spacy.load(model_name)
+        return True
+    except Exception:
+        pass
+
+    from rich.console import Console
+    con = Console()
+    con.print(f"  [dim]→[/dim] Installing spaCy NER model [bold]{model_name}[/bold] (one-time)...")
+
+    env = dict(os.environ)
+    env["PIP_BREAK_SYSTEM_PACKAGES"] = "1"
+    in_venv = sys.prefix != getattr(sys, "base_prefix", sys.prefix)
+    if not in_venv:
+        env["PIP_USER"] = "1"
+
+    result = subprocess.run(
+        [sys.executable, "-m", "spacy", "download", model_name],
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+
+    if result.returncode == 0:
+        try:
+            import importlib
+            import spacy as _spacy  # noqa: F401
+            importlib.invalidate_caches()
+            import spacy as _spacy2
+            _spacy2.load(model_name)
+            con.print(f"  [green]✓[/green] spaCy model ready")
+            return True
+        except Exception:
+            pass
+
+    err_tail = (result.stderr or result.stdout or "").strip().splitlines()[-3:]
+    con.print(f"  [yellow]⚠[/yellow] spaCy install failed (exit {result.returncode}) — NER will be skipped.")
+    for line in err_tail:
+        con.print(f"    [dim]{line}[/dim]")
+    con.print(
+        f"  Run manually: [bold]PIP_BREAK_SYSTEM_PACKAGES=1 "
+        f"{os.path.basename(sys.executable)} -m spacy download {model_name}[/bold]"
+    )
+    return False
+
+
 def apply_env(config: Optional[dict[str, Any]] = None) -> None:
     """
     Push saved config into os.environ so that the existing voidaccess
@@ -154,11 +213,12 @@ def apply_env(config: Optional[dict[str, Any]] = None) -> None:
     os.environ.setdefault("PLAYWRIGHT_ENABLED", "false")
 
     def _set_env_if_present(key: str, value: Any, *, clear_if_empty: bool = False) -> None:
-        text = str(value).strip() if value is not None else ""
-        if text:
-            os.environ[key] = text
-        elif clear_if_empty:
-            os.environ.pop(key, None)
+        text = str(value) if value is not None else ""
+        if not text or not text.strip():
+            if clear_if_empty:
+                os.environ.pop(key, None)
+            return
+        os.environ[key] = text.strip()
 
     # Tor proxy
     _set_env_if_present("TOR_PROXY_HOST", cfg.get("tor", {}).get("host", "127.0.0.1"))
@@ -182,5 +242,5 @@ def apply_env(config: Optional[dict[str, Any]] = None) -> None:
     # Keyless APIs (ThreatFox/URLhaus/MalwareBazaar/abuse.ch) must never
     # receive an empty auth header — clear any empty env remnant.
     for key in ("ABUSECH_API_KEY", "VT_API_KEY", "OTX_API_KEY"):
-        if not os.environ.get(key):
+        if not (os.environ.get(key) or "").strip():
             os.environ.pop(key, None)