voidaccess 1.3.0__py3-none-any.whl

sources/virustotal.py

@@ -0,0 +1,113 @@
+"""
+sources/virustotal.py — VirusTotal hash enrichment (file hash lookup).
+
+Requires VT_API_KEY in config. Free tier: 4 requests/minute.
+Max 20 hashes per investigation.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Optional
+
+import aiohttp
+
+from config import VT_API_KEY
+
+logger = logging.getLogger(__name__)
+
+_VT_BASE = "https://www.virustotal.com/api/v3"
+_VT_HASH_LIMIT = 20
+_VT_RATE_LIMIT_DELAY = 15.0
+
+
+def _is_enabled() -> bool:
+    key = getattr(VT_API_KEY, "strip", lambda: "")()
+    return bool(key)
+
+
+async def _fetch_hash(hash_value: str, session: aiohttp.ClientSession) -> Optional[dict]:
+    try:
+        headers = {"x-apikey": VT_API_KEY.strip()}
+        timeout = aiohttp.ClientTimeout(total=15)
+        async with session.get(
+            f"{_VT_BASE}/files/{hash_value}", headers=headers, timeout=timeout
+        ) as resp:
+            if resp.status == 404:
+                return None
+            if resp.status == 401:
+                logger.warning("VirusTotal: invalid API key")
+                return None
+            if resp.status == 429:
+                logger.warning("VirusTotal: rate limited")
+                return None
+            if resp.status != 200:
+                return None
+            return await resp.json()
+    except asyncio.TimeoutError:
+        logger.warning("VirusTotal: timeout for hash %s", hash_value[:16])
+        return None
+    except Exception as e:
+        logger.warning("VirusTotal: error for hash %s: %s", hash_value[:16], e)
+        return None
+
+
+async def enrich_virustotal(entities: list[dict]) -> list[dict]:
+    """
+    For each FILE_HASH_MD5 / FILE_HASH_SHA1 / FILE_HASH_SHA256 entity,
+    query VirusTotal and return detection stats.
+    """
+    if not _is_enabled():
+        logger.debug("VirusTotal skipped — no API key configured")
+        return []
+
+    hash_type_map = {
+        "FILE_HASH_MD5": "md5",
+        "FILE_HASH_SHA1": "sha1",
+        "FILE_HASH_SHA256": "sha256",
+    }
+
+    hash_entities = [
+        e for e in entities
+        if (e.get("type") or e.get("entity_type", "")) in hash_type_map
+        and (e.get("value") or e.get("entity_value", ""))
+    ]
+
+    hashes_to_query = [
+        (e.get("value") or e.get("entity_value", ""), (e.get("type") or e.get("entity_type", "")))
+        for e in hash_entities
+    ][:_VT_HASH_LIMIT]
+
+    results: list[dict] = []
+    async with aiohttp.ClientSession() as session:
+        for hash_val, hash_type in hashes_to_query:
+            data = await _fetch_hash(hash_val, session)
+            if data is None:
+                await asyncio.sleep(_VT_RATE_LIMIT_DELAY)
+                continue
+
+            attr = data.get("data", {}).get("attributes", {})
+            stats = attr.get("last_analysis_stats", {})
+            mal = stats.get("malicious", 0)
+            total = sum(stats.values())
+            detection_ratio = mal / total if total > 0 else 0.0
+
+            results.append({
+                "source": "virustotal",
+                "entity_type": hash_type_map.get(hash_type, "FILE_HASH"),
+                "entity_value": hash_val,
+                "malicious_count": mal,
+                "total_engines": total,
+                "detection_ratio": detection_ratio,
+                "suggested_threat_label": attr.get("popular_threat_classification", {}).get("suggested_threat_label", ""),
+                "first_seen": attr.get("creation_date", ""),
+                "last_seen": attr.get("last_analysis_date", ""),
+                "confirmed_malicious": detection_ratio > 0.5,
+            })
+
+            await asyncio.sleep(_VT_RATE_LIMIT_DELAY)
+
+    if results:
+        logger.info("VirusTotal: %d results", len(results))
+    return results

utils/async_utils.py

@@ -0,0 +1,89 @@
+"""
+Async utilities for safely running coroutines in various contexts.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import threading
+from concurrent.futures import Future, ThreadPoolExecutor
+from typing import Any, Coroutine, TypeVar
+
+logger = logging.getLogger(__name__)
+
+_T = TypeVar("_T")
+
+_executor: ThreadPoolExecutor | None = None
+_executor_lock = threading.Lock()
+
+
+def _get_executor() -> ThreadPoolExecutor:
+    global _executor
+    if _executor is None:
+        with _executor_lock:
+            if _executor is None:
+                _executor = ThreadPoolExecutor(max_workers=4, thread_name_prefix="async_utils_")
+    return _executor
+
+
+def run_async(coro: Coroutine[Any, Any, _T]) -> _T:
+    """
+    Safely run a coroutine regardless of whether there's already a running event loop.
+
+    Uses a thread-isolated event loop when called from:
+    - An already-running event loop (e.g., inside APScheduler jobs, pytest-asyncio)
+    - A synchronous context
+
+    Args:
+        coro: The coroutine to run
+
+    Returns:
+        The result of the coroutine
+
+    Raises:
+        RuntimeError: If the coroutine fails to run
+    """
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        loop = None
+
+    if loop is not None:
+        return _run_in_thread(coro)
+
+    try:
+        return asyncio.run(coro)
+    except RuntimeError as e:
+        if "already running" in str(e).lower():
+            return _run_in_thread(coro)
+        raise
+
+
+def _run_in_thread(coro: Coroutine[Any, Any, _T]) -> _T:
+    """
+    Run a coroutine in a dedicated thread with its own event loop.
+    """
+    future: Future[_T] = Future()
+
+    def _run() -> None:
+        local_loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(local_loop)
+        try:
+            result = local_loop.run_until_complete(coro)
+            future.set_result(result)
+        except Exception as exc:
+            future.set_exception(exc)
+        finally:
+            local_loop.close()
+
+    executor = _get_executor()
+    executor.submit(_run)
+    return future.result()
+
+
+def run_async_optional(coro: Coroutine[Any, Any, _T] | None) -> _T | None:
+    """Run a coroutine if provided, otherwise return None."""
+    if coro is None:
+        return None
+    return run_async(coro)

utils/content_safety.py

@@ -0,0 +1,193 @@
+"""
+utils/content_safety.py — Mandatory content safety filters for VoidAccess.
+
+Operates at multiple layers: query intake, URL filtering, content scanning,
+and post-extraction entity value filtering.
+Never logs actual prohibited content — only event metadata.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import re
+from typing import Optional
+
+_logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Blocklists
+# ---------------------------------------------------------------------------
+
+BLOCKED_TERMS = [
+    # CSAM related
+    "child porn", "cp porn", "lolita", "pedo",
+    "pedophil", "childporn", "child sex", "minor sex",
+    "underage sex", "jailbait", "preteen sex",
+    "child abuse material", "csam", "child model",
+    "hurtcore", "daisy's destruction",
+    # Gore related
+    "gore site", "gore video", "snuff film",
+    "murder video", "execution video", "beheading video",
+    "torture video", "bestgore", "livegore",
+    "watchpeopledie", "realsnuff",
+]
+
+BLOCKED_PATTERNS = [
+    r'\bcp\b.{0,20}\bonion\b',      # "cp" near "onion"
+    r'\bchild.{0,10}\bnaked\b',
+    r'\bminor.{0,10}\bnaked\b',
+    r'\bkid.{0,10}\bporn\b',
+    r'\bteen.{0,10}\bporn\b',
+]
+
+BLOCKED_URL_TERMS = [
+    "pedo", "loli", "jailbait", "childporn",
+    "hurtcore", "csam", "bestgore", "livegore",
+    "watchpeople", "realsnuff", "daisy",
+]
+
+CONTENT_BLOCKLIST = [
+    "child pornography", "child porn",
+    "child sexual abuse", "csam",
+    "snuff film", "murder porn",
+]
+
+# ---------------------------------------------------------------------------
+# Entity value blocklist — applied after extraction, before DB storage
+# Only checked against text-based entity types (not technical IOCs)
+# ---------------------------------------------------------------------------
+
+ENTITY_VALUE_BLOCKLIST: list[str] = [
+    # Adult content categories
+    "porn", "blowjob", "bdsm", "hardcore",
+    "xxx", "nude", "nudes", "naked", "escort",
+    "onlyfans", "cam girl", "sex tape",
+    "adult content", "adult site",
+    # Gore/violence
+    "snuff", "gore", "murder video",
+    "execution video", "beheading",
+    # Exploitation
+    "jailbait", "pedo", "csam",
+    "child", "minor",
+]
+
+# Entity types where prohibited content can appear as names/labels.
+# Technical IOC types (hashes, IPs, CVEs, wallets, onion URLs) are
+# intentionally omitted — they cannot contain prohibited content.
+_TEXT_ENTITY_TYPES: frozenset[str] = frozenset({
+    "ORGANIZATION_NAME",
+    "THREAT_ACTOR_HANDLE",
+    "PERSON_NAME",
+    "MALWARE_FAMILY",
+})
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def is_blocked_query(query: str) -> tuple[bool, str]:
+    """
+    Check if a query should be blocked.
+    Returns (is_blocked, reason).
+    Never logs the actual query.
+    """
+    query_lower = query.lower()
+
+    for term in BLOCKED_TERMS:
+        if term in query_lower:
+            return True, "Query contains prohibited content"
+
+    for pattern in BLOCKED_PATTERNS:
+        if re.search(pattern, query_lower, re.IGNORECASE):
+            return True, "Query contains prohibited content"
+
+    return False, ""
+
+
+def is_blocked_entity_value(entity_type: str, value: str) -> bool:
+    """
+    Return True if an entity value should be dropped before storage.
+
+    Only applies to text-based entity types where prohibited content can
+    appear as organisation/actor names (ORGANIZATION_NAME, THREAT_ACTOR_HANDLE,
+    PERSON_NAME, MALWARE_FAMILY).
+
+    Never applies to technical IOC types such as FILE_HASH_*, IP_ADDRESS, CVE,
+    ONION_URL, or wallet addresses \u2014 these cannot contain prohibited content
+    by definition and are intentionally excluded.
+
+    The check is case-insensitive substring matching against
+    ENTITY_VALUE_BLOCKLIST.  The actual value is never logged.
+    """
+    if entity_type not in _TEXT_ENTITY_TYPES:
+        return False
+
+    value_lower = (value or "").lower()
+    for term in ENTITY_VALUE_BLOCKLIST:
+        if term in value_lower:
+            return True
+
+    return False
+
+
+def is_blocked_url(url: str) -> tuple[bool, str]:
+    """
+    Check if a URL should be blocked from scraping.
+    Returns (is_blocked, reason).
+    """
+    url_lower = url.lower()
+    for term in BLOCKED_URL_TERMS:
+        if term in url_lower:
+            return True, "URL blocked — prohibited content"
+    return False, ""
+
+
+def sanitize_content(text: str) -> tuple[str, bool]:
+    """
+    Scan scraped text for CSAM/gore indicators.
+    Returns (sanitized_text, was_flagged).
+    If flagged, returns empty string — the original text is never stored.
+    """
+    if not text:
+        return text, False
+
+    text_lower = text.lower()
+    for term in CONTENT_BLOCKLIST:
+        if term in text_lower:
+            return "", True
+
+    return text, False
+
+
+def log_content_safety_event(
+    event_type: str,
+    content_hash: Optional[str] = None,
+    user_id: Optional[int] = None,
+) -> None:
+    """
+    Persist a content safety block event to the DB for operator review.
+    Fails silently — never disrupts the calling pipeline.
+    event_type: one of "query_blocked", "url_blocked", "content_blocked"
+    content_hash: SHA-256 hex prefix (≤16 chars) of the blocked item, for correlation only.
+    """
+    try:
+        import os
+        if not os.getenv("DATABASE_URL"):
+            return
+        from db.session import get_session
+        from db.models import ContentSafetyEvent
+        from datetime import datetime, timezone
+
+        with get_session() as session:
+            event = ContentSafetyEvent(
+                event_type=event_type,
+                user_id=user_id,
+                content_hash=content_hash,
+                timestamp=datetime.now(timezone.utc),
+            )
+            session.add(event)
+            session.commit()
+    except Exception as exc:
+        _logger.debug("content_safety: DB log failed (non-critical): %s", exc)

utils/defang.py

@@ -0,0 +1,94 @@
+import re
+
+
+def defang_url(url: str) -> str:
+    """
+    Defang a URL for safe sharing in reports.
+    hxxp://example[.]com/path
+    """
+    if not url:
+        return url
+    url = url.replace("http://", "hxxp://")
+    url = url.replace("https://", "hxxps://")
+    url = url.replace("ftp://", "fxp://")
+    parts = url.split("/", 3)
+    if len(parts) >= 3:
+        parts[2] = parts[2].replace(".", "[.]")
+        url = "/".join(parts)
+    else:
+        url = url.replace(".", "[.]")
+    return url
+
+
+def defang_ip(ip: str) -> str:
+    """
+    Defang an IP address.
+    1.2.3.4 -> 1.2.3[.]4
+    """
+    if not ip:
+        return ip
+    parts = ip.rsplit(".", 1)
+    if len(parts) == 2:
+        return f"{parts[0]}[.]{parts[1]}"
+    return ip
+
+
+def defang_email(email: str) -> str:
+    """
+    Defang an email address.
+    user@example.com -> user[@]example[.]com
+    """
+    if not email:
+        return email
+    email = email.replace("@", "[@]")
+    parts = email.split("[@]", 1)
+    if len(parts) == 2:
+        parts[1] = parts[1].replace(".", "[.]")
+        email = "[@]".join(parts)
+    return email
+
+
+def defang_value(entity_type: str, value: str) -> str:
+    """
+    Defang an entity value based on its type.
+    Returns the defanged version for display.
+    """
+    if entity_type in (
+        "ONION_URL",
+        "DOMAIN",
+    ):
+        return defang_url(value)
+    elif entity_type == "IP_ADDRESS":
+        return defang_ip(value)
+    elif entity_type == "EMAIL_ADDRESS":
+        return defang_email(value)
+    else:
+        return value
+
+
+def defang_text(text: str) -> str:
+    """
+    Defang all URLs and IPs found in free text.
+    Use for report summaries and context snippets.
+    """
+    if not text:
+        return text
+
+    text = re.sub(
+        r'https?://',
+        lambda m: m.group().replace(
+            "http://", "hxxp://"
+        ).replace(
+            "https://", "hxxps://"
+        ),
+        text
+    )
+
+    text = re.sub(
+        r'\b(\d{1,3})\.(\d{1,3})\.(\d{1,3})'
+        r'\.(\d{1,3})\b',
+        r'\1.\2.\3[.]\4',
+        text,
+    )
+
+    return text

utils/encryption.py

@@ -0,0 +1,34 @@
+"""
+Encryption utilities for user API keys.
+
+Uses Fernet (AES-128-CBC) with a key derived from JWT_SECRET so that
+no new secret needs to be distributed — only the existing JWT_SECRET
+is required.
+"""
+
+import base64
+import hashlib
+
+from cryptography.fernet import Fernet
+
+
+def _get_fernet() -> Fernet:
+    from config import JWT_SECRET
+    key_bytes = hashlib.sha256(JWT_SECRET.encode()).digest()
+    fernet_key = base64.urlsafe_b64encode(key_bytes)
+    return Fernet(fernet_key)
+
+
+def encrypt_api_key(plaintext: str) -> str:
+    if not plaintext:
+        return ""
+    return _get_fernet().encrypt(plaintext.encode()).decode()
+
+
+def decrypt_api_key(ciphertext: str) -> str:
+    if not ciphertext:
+        return ""
+    try:
+        return _get_fernet().decrypt(ciphertext.encode()).decode()
+    except Exception:
+        return ""

utils/ioc_freshness.py

@@ -0,0 +1,124 @@
+from datetime import datetime, timedelta, timezone
+from enum import Enum
+
+
+class FreshnessTag(str, Enum):
+    FRESH = "fresh"
+    AGING = "aging"
+    STALE = "stale"
+    EXPIRED = "expired"
+    UNKNOWN = "unknown"
+
+
+FRESHNESS_THRESHOLDS = {
+    "IP_ADDRESS": {
+        "fresh": 14,
+        "aging": 30,
+        "stale": 90,
+    },
+    "DOMAIN": {
+        "fresh": 30,
+        "aging": 90,
+        "stale": 180,
+    },
+    "ONION_URL": {
+        "fresh": 60,
+        "aging": 180,
+        "stale": 365,
+    },
+    "FILE_HASH_MD5": {
+        "fresh": 365,
+        "aging": 730,
+        "stale": 1825,
+    },
+    "FILE_HASH_SHA256": {
+        "fresh": 365,
+        "aging": 730,
+        "stale": 1825,
+    },
+    "CVE": {
+        "fresh": 365,
+        "aging": 730,
+        "stale": 1825,
+    },
+    "BITCOIN_ADDRESS": {
+        "fresh": 90,
+        "aging": 180,
+        "stale": 365,
+    },
+    "THREAT_ACTOR": {
+        "fresh": 90,
+        "aging": 365,
+        "stale": 730,
+    },
+    "DEFAULT": {
+        "fresh": 30,
+        "aging": 90,
+        "stale": 180,
+    },
+}
+
+
+def get_freshness_tag(
+    entity_type: str,
+    last_seen_at: datetime | None,
+    first_seen_at: datetime | None = None,
+) -> FreshnessTag:
+    """
+    Calculate freshness tag for an entity based on its type and when it was last seen.
+    """
+    if not last_seen_at:
+        return FreshnessTag.UNKNOWN
+
+    thresholds = FRESHNESS_THRESHOLDS.get(
+        entity_type,
+        FRESHNESS_THRESHOLDS["DEFAULT"],
+    )
+
+    now = datetime.now(timezone.utc)
+    # Ensure last_seen_at is tz-aware before subtracting
+    if last_seen_at.tzinfo is None:
+        last_seen_at = last_seen_at.replace(tzinfo=timezone.utc)
+    days_since_seen = (now - last_seen_at).days
+
+    if days_since_seen <= thresholds["fresh"]:
+        return FreshnessTag.FRESH
+    elif days_since_seen <= thresholds["aging"]:
+        return FreshnessTag.AGING
+    elif days_since_seen <= thresholds["stale"]:
+        return FreshnessTag.STALE
+    else:
+        return FreshnessTag.EXPIRED
+
+
+def get_freshness_display(tag: FreshnessTag) -> dict:
+    """
+    Get display config for a freshness tag.
+    """
+    return {
+        FreshnessTag.FRESH: {
+            "label": "Fresh",
+            "color": "green",
+            "description": "Recently observed",
+        },
+        FreshnessTag.AGING: {
+            "label": "Aging",
+            "color": "yellow",
+            "description": "Observed 1-3 months ago",
+        },
+        FreshnessTag.STALE: {
+            "label": "Stale",
+            "color": "orange",
+            "description": "Observed 3-6 months ago — verify before use",
+        },
+        FreshnessTag.EXPIRED: {
+            "label": "Expired",
+            "color": "red",
+            "description": "Observed over 6 months ago — likely inactive",
+        },
+        FreshnessTag.UNKNOWN: {
+            "label": "Unknown",
+            "color": "gray",
+            "description": "No date information available",
+        },
+    }.get(tag, {"label": "Unknown", "color": "gray"})

utils/user_keys.py

@@ -0,0 +1,33 @@
+"""
+Per-user API key resolution with fallback chain.
+
+resolve_api_key checks the user's personal key first, then falls back to
+the server-level environment variable in config.py.
+"""
+
+from sqlalchemy import select as sa_select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from db.models import UserApiKey
+from utils.encryption import decrypt_api_key
+import config as _config
+
+
+async def get_user_key(user_id: int, key_name: str, session: AsyncSession) -> str | None:
+    result = await session.execute(
+        sa_select(UserApiKey).where(
+            UserApiKey.user_id == user_id,
+            UserApiKey.key_name == key_name,
+        )
+    )
+    record = result.scalar_one_or_none()
+    if record:
+        return decrypt_api_key(record.encrypted_value)
+    return None
+
+
+async def resolve_api_key(user_id: int, key_name: str, session: AsyncSession) -> str:
+    user_key = await get_user_key(user_id, key_name, session)
+    if user_key:
+        return user_key
+    return getattr(_config, key_name, "") or ""