voidaccess 1.3.0__py3-none-any.whl

sources/seeds.py

@@ -0,0 +1,368 @@
+"""
+sources/seeds.py — Curated seed URL list for the recursive crawler.
+
+SEED_URLS is a hardcoded list of known high-value .onion starting points —
+forums, indexes, directories, and paste sites that are commonly accessible
+and useful as entry points for threat-intelligence crawling.
+
+These are starting points only: the crawler follows their links recursively.
+None are assumed to have any particular content; they are known *link hubs*.
+
+Addresses were current at time of writing (2025).  .onion addresses change
+frequently; the crawler handles unreachable seeds gracefully.
+
+Public API:
+    SEED_URLS          — full list of seed dicts
+    get_seeds(category, language, query) -> list[dict]   — filtered view
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import List, Optional
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Curated seed list  (≥ 20 entries required by spec)
+# ---------------------------------------------------------------------------
+# Each entry: url, category, description, language
+# category: "search" | "index" | "forum" | "paste" | "market_index"
+# language: "en" | "ru" | "multi"
+
+SEED_URLS: List[dict] = [
+    # ── Search engines ──────────────────────────────────────────────────────
+    {
+        "url": "http://torchdeedp3i2jigzjdmfpn5ttjhthh5wbmda2rr3jvqjg5p77c54dqd.onion",
+        "category": "search",
+        "description": "Torch — one of the oldest and largest dark web search engines",
+        "language": "en",
+    },
+    {
+        "url": "http://haystak5njsmn2hqkewecpaxetahtwhsbsa64jom2k22z5afxhnpxfid.onion",
+        "category": "search",
+        "description": "Haystack — indexes millions of onion pages, fast results",
+        "language": "en",
+    },
+    {
+        "url": "http://duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion",
+        "category": "search",
+        "description": "DuckDuckGo official Tor hidden service — clearnet search over Tor",
+        "language": "en",
+    },
+    {
+        "url": "http://darksearch7bvmqn2sp7gokxbz7gvx5sflhkblekdxs5pfxypufksgfyd.onion",
+        "category": "search",
+        "description": "DarkSearch — dark web search engine with JSON API",
+        "language": "en",
+    },
+    # ── Indexes / Directories ────────────────────────────────────────────────
+    {
+        "url": "http://zqktlwiuavvvqqt4ybvgvi7tyo4hjl5xgfuvpdf6otjiycgwqbym2qad.onion/wiki/index.php/Main_Page",
+        "category": "index",
+        "description": "The Hidden Wiki — primary community .onion link directory",
+        "language": "en",
+    },
+    {
+        "url": "http://darkfailenbsdla5mal2mxn2uz66od5vtzd5qozslagrfzachha3f3id.onion",
+        "category": "index",
+        "description": "dark.fail — curated directory of verified, working onion sites",
+        "language": "en",
+    },
+    {
+        "url": "http://danielas3rtn54uwmofdo3x2bsdifr47huasnmbgqzfrec5ubupvtpid.onion",
+        "category": "index",
+        "description": "Daniel's Hosting — index of hundreds of hosted onion services",
+        "language": "en",
+    },
+    {
+        "url": "http://bbcnewsd73hkzno2ini43t4gblxvycyac5aw4gnv7t2rccijh7745uqd.onion",
+        "category": "index",
+        "description": "BBC News Tor mirror — official BBC onion service for censorship bypass",
+        "language": "en",
+    },
+    {
+        "url": "http://p53lf57qovyuvwsc6xnrppyply3vtqm7l6pcobkmyqsiofyeznfu5uqd.onion",
+        "category": "index",
+        "description": "ProPublica Tor mirror — investigative journalism, primary source links",
+        "language": "en",
+    },
+    {
+        "url": "http://sdolvtfhatvsysc6l34d65ymdwxcujausv7k5jk4cy5ttzhjoi6fzvyd.onion",
+        "category": "index",
+        "description": "SecureDrop directory — whistleblower submission platform index",
+        "language": "en",
+    },
+    # ── Forums ───────────────────────────────────────────────────────────────
+    {
+        "url": "http://dreadytofatroptsdj6io7l3xptbet6onoyno2yv7jicoxknyazubrad.onion",
+        "category": "forum",
+        "description": "Dread — dark web Reddit equivalent, hub for market discussion and news",
+        "language": "en",
+    },
+    {
+        "url": "http://enxx3byspwsdo446jujc52ucy2pf5urdbhqw3kbsfhlfjwmbpj5smdad.onion",
+        "category": "forum",
+        "description": "Endchan — decentralized imageboard, uncensored discussion boards",
+        "language": "en",
+    },
+    {
+        "url": "http://4usoivrpy52lmc4mgn2h34cmfiltslesthr56yttv2pxudd3dapqciyd.onion",
+        "category": "forum",
+        "description": "8chan/8kun — decentralized anonymous forum, various topic boards",
+        "language": "en",
+    },
+    {
+        "url": "http://crpxfhcgaaqxnpqgcmgrk2uupxrjyqrlc3dnlrgidcjbpq5zxkafbvid.onion",
+        "category": "forum",
+        "description": "CryptBB — cybercrime forum focusing on hacking and exploit trading",
+        "language": "en",
+    },
+    {
+        "url": "http://gg6zxtreajiijztyy5g6bt5o6l3qu32nrg7eulyemlnhbh6tl7r2vyad.onion",
+        "category": "forum",
+        "description": "XSS.is Tor mirror — Russian-language cybercrime and vulnerability forum",
+        "language": "ru",
+    },
+    {
+        "url": "http://exploitivzcm5dawzhe6c32bbylyggbjvh5dyvsvb5lkuz5ptmunkmqd.onion",
+        "category": "forum",
+        "description": "Exploit.in Tor mirror — Russian exploit marketplace and forum",
+        "language": "ru",
+    },
+    {
+        "url": "http://ransomwr3tsydeii.onion",
+        "category": "forum",
+        "description": "RansomWatch aggregator mirror — tracks ransomware group leak sites",
+        "language": "en",
+    },
+    # ── Paste sites ──────────────────────────────────────────────────────────
+    {
+        "url": "http://depastedihryjugl7sxhstlqjmqbedofrm3r5vynzw7rl7mwkv4zmcid.onion",
+        "category": "paste",
+        "description": "DeepPaste — dark web paste service, frequently used for leaks",
+        "language": "en",
+    },
+    {
+        "url": "http://zgjnkivynuasfwog7rkkphv5gdtyrcaxp4ihczgyuep2ulokhmuuduuqd.onion",
+        "category": "paste",
+        "description": "PrivateBin .onion instance — anonymous encrypted paste sharing",
+        "language": "en",
+    },
+    {
+        "url": "http://protonirockerxow.onion",
+        "category": "paste",
+        "description": "ProtonMail Tor mirror — encrypted email, often linked to paste leaks",
+        "language": "multi",
+    },
+    # ── Market indexes (aggregators only — not markets themselves) ────────────
+    {
+        "url": "http://darknetlidvrsli6iso7my54rjayjursyw637aypb6qambkoepmyq2yd.onion",
+        "category": "market_index",
+        "description": "Darknet market index — lists active markets and their mirror links",
+        "language": "en",
+    },
+    {
+        "url": "http://dark2web.com.onion",
+        "category": "market_index",
+        "description": "Dark2Web — aggregator that reviews and indexes dark web markets",
+        "language": "en",
+    },
+    {
+        "url": "http://dgdtaovql5oo7ait.onion",
+        "category": "market_index",
+        "description": "Tor Metrics onion — statistics on the Tor network and onion services",
+        "language": "en",
+    },
+    # ── Multi-language / Russian-language index ───────────────────────────────
+    {
+        "url": "http://rutorc6mqdinc4cz.onion",
+        "category": "index",
+        "description": "RuTor — Russian-language dark web link directory and index",
+        "language": "ru",
+    },
+    {
+        "url": "http://omgomgomg5j4yrr47fishp4rdwxkn3vkpbxbouys33ew74h6hq47qad.onion",
+        "category": "market_index",
+        "description": "OMG!OMG! market — large multi-language dark web marketplace index",
+        "language": "multi",
+    },
+]
+
+
+# ---------------------------------------------------------------------------
+# Query-aware topic seeds (verified-stable .onion only; small curated set)
+# ---------------------------------------------------------------------------
+
+TOPIC_SEEDS: dict[str, List[dict]] = {
+    "bitcoin": [
+        {
+            "url": "http://darkfailllnkf4vf.onion",
+            "category": "index",
+            "language": "en",
+            "description": "dark.fail index — query-aware bitcoin/crypto seed",
+        },
+    ],
+    "ransomware": [
+        {
+            "url": "http://darkfailllnkf4vf.onion",
+            "category": "index",
+            "language": "en",
+            "description": "dark.fail index — query-aware ransomware seed",
+        },
+        {
+            "url": "http://ransomwr3tsydeii.onion",
+            "category": "forum",
+            "language": "en",
+            "description": "RansomWatch — query-aware ransomware seed",
+        },
+    ],
+    "malware": [
+        {
+            "url": "http://darkfailllnkf4vf.onion",
+            "category": "index",
+            "language": "en",
+            "description": "dark.fail index — query-aware malware seed",
+        },
+    ],
+    "credentials": [
+        {
+            "url": "http://darkfailllnkf4vf.onion",
+            "category": "index",
+            "language": "en",
+            "description": "dark.fail index — query-aware credentials seed",
+        },
+    ],
+    "drugs": [
+        {
+            "url": "http://darkfailllnkf4vf.onion",
+            "category": "index",
+            "language": "en",
+            "description": "dark.fail index — query-aware seed (limited)",
+        },
+    ],
+    "hacking": [
+        {
+            "url": "http://darkfailllnkf4vf.onion",
+            "category": "index",
+            "language": "en",
+            "description": "dark.fail index — query-aware hacking seed",
+        },
+    ],
+    "fraud": [
+        {
+            "url": "http://darkfailllnkf4vf.onion",
+            "category": "index",
+            "language": "en",
+            "description": "dark.fail index — query-aware fraud seed",
+        },
+    ],
+}
+
+TOPIC_KEYWORDS: dict[str, List[str]] = {
+    "bitcoin": [
+        "bitcoin", "btc", "wallet", "crypto", "cryptocurrency", "blockchain",
+    ],
+    "ransomware": [
+        "ransomware", "lockbit", "alphv", "blackcat", "conti", "revil", "ryuk",
+        "extortion",
+    ],
+    "malware": [
+        "malware", "rat", "trojan", "backdoor", "botnet", "rootkit", "keylogger",
+        "stealer",
+    ],
+    "credentials": [
+        "credentials", "password", "login", "account", "breach", "leak", "dump",
+        "combo",
+    ],
+    "drugs": ["drug", "narcotic", "cannabis", "opioid"],
+    "hacking": [
+        "hacking", "exploit", "vulnerability", "cve", "0day", "zero-day", "shell",
+        "access",
+    ],
+    "fraud": [
+        "fraud", "carding", "cc", "credit card", "ssn", "identity", "fake",
+        "counterfeit",
+    ],
+}
+
+
+def detect_query_topics(query: str) -> List[str]:
+    """
+    Analyze a query string and return relevant topic categories.
+
+    Examples:
+        "lockbit ransomware bitcoin payments" → ["ransomware", "bitcoin"]
+        "CVE-2024-1234 exploit kit" → ["hacking"]
+        "stolen credentials combo list" → ["credentials"]
+    """
+    query_lower = query.lower()
+    detected_topics: List[str] = []
+
+    for topic, keywords in TOPIC_KEYWORDS.items():
+        if any(keyword in query_lower for keyword in keywords):
+            detected_topics.append(topic)
+
+    return detected_topics
+
+
+# ---------------------------------------------------------------------------
+# Public filter function
+# ---------------------------------------------------------------------------
+
+
+def get_seeds(
+    category: Optional[str] = None,
+    language: Optional[str] = None,
+    query: Optional[str] = None,
+) -> List[dict]:
+    """
+    Return the curated seed list, optionally filtered by *category* and/or *language*.
+
+    Args:
+        category:  one of "search", "index", "forum", "paste", "market_index",
+                   or None to return all categories.
+        language:  "en", "ru", "multi", or None to return all languages.
+        query:     optional investigation query; adds topic-specific seeds when
+                   keywords match.
+
+    Returns a new list; the original SEED_URLS is never mutated.
+    """
+    seeds = list(SEED_URLS)
+    if category is not None:
+        seeds = [s for s in seeds if s["category"] == category]
+    if language is not None:
+        seeds = [s for s in seeds if s["language"] == language]
+
+    if query:
+        detected_topics = detect_query_topics(query)
+        if detected_topics:
+            logger.warning("Query topics detected: %s", detected_topics)
+            topic_specific: List[dict] = []
+            for topic in detected_topics:
+                topic_seeds = list(TOPIC_SEEDS.get(topic, []))
+                if category is not None:
+                    topic_seeds = [
+                        s for s in topic_seeds if s.get("category") == category
+                    ]
+                topic_specific.extend(topic_seeds)
+
+            seen_topic_urls: set[str] = set()
+            topic_specific_deduped: List[dict] = []
+            for s in topic_specific:
+                u = s.get("url")
+                if not u or u in seen_topic_urls:
+                    continue
+                seen_topic_urls.add(u)
+                topic_specific_deduped.append(s)
+
+            existing_urls = {s["url"] for s in seeds}
+            new_seeds = [s for s in topic_specific_deduped if s["url"] not in existing_urls]
+            logger.warning(
+                "Adding %d topic-specific seeds for: %s",
+                len(new_seeds),
+                detected_topics,
+            )
+            seeds = seeds + new_seeds
+
+    return list(seeds)

sources/shodan.py

@@ -0,0 +1,103 @@
+"""
+sources/shodan.py — Shodan InternetDB integration for C2 infrastructure.
+
+No API key required. Queries https://internetdb.shodan.io/{ip} for each
+extracted IP_ADDRESS entity and returns open ports, vulnerabilities,
+tags, and hostnames. Tags are used to flag high-confidence C2.
+
+Rate-limited: max 1 request/second, max 50 IPs per investigation.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+
+import aiohttp
+
+from config import MAX_IPS_PER_INVESTIGATION, SHODAN_RATE_LIMIT_DELAY
+
+logger = logging.getLogger(__name__)
+
+_SHODAN_INTERNETDB = "https://internetdb.shodan.io"
+
+_C2_TAGS = {"c2", "cobalt-strike", "metasploit", "malware"}
+
+
+async def enrich_shodan_ip(ip_address: str, extracted_cves: set[str]) -> dict | None:
+    """
+    Query Shodan InternetDB for *ip_address*.
+
+    Returns a dict with open_ports, vulns, tags, hostnames, and
+    high_confidence_c2 flag, or None on error / no data.
+    """
+    try:
+        timeout = aiohttp.ClientTimeout(total=5)
+        async with aiohttp.ClientSession(timeout=timeout) as session:
+            async with session.get(f"{_SHODAN_INTERNETDB}/{ip_address}") as resp:
+                if resp.status == 404:
+                    return None
+                if resp.status != 200:
+                    logger.warning("Shodan InternetDB: HTTP %s for %s", resp.status, ip_address)
+                    return None
+                data = await resp.json()
+    except asyncio.TimeoutError:
+        logger.warning("Shodan InternetDB: timeout for %s", ip_address)
+        return None
+    except Exception as e:
+        logger.warning("Shodan InternetDB: error for %s: %s", ip_address, e)
+        return None
+
+    raw_tags = [t.lower() for t in data.get("tags") or []]
+    high_confidence_c2 = bool(raw_tags and set(raw_tags) & _C2_TAGS)
+
+    vulns = data.get("vulns") or {}
+    cve_set = set(vulns.keys())
+    correlated_cves = cve_set & extracted_cves if extracted_cves else set()
+
+    return {
+        "source": "shodan_internetdb",
+        "entity_type": "IP_ADDRESS",
+        "entity_value": ip_address,
+        "open_ports": data.get("ports") or [],
+        "vulns": list(cve_set),
+        "correlated_cves": list(correlated_cves),
+        "tags": raw_tags,
+        "hostnames": data.get("hostnames") or [],
+        "high_confidence_c2": high_confidence_c2,
+    }
+
+
+async def enrich_shodan(entities: list[dict]) -> list[dict]:
+    """
+    For each IP_ADDRESS entity in *entities*, query Shodan InternetDB.
+
+    Rate-limited to SHODAN_RATE_LIMIT_DELAY between requests.
+    Capped at MAX_IPS_PER_INVESTIGATION IPs.
+    """
+    ip_entities = [
+        e for e in entities
+        if (e.get("type") or e.get("entity_type", "")) == "IP_ADDRESS"
+        and (e.get("value") or e.get("entity_value", ""))
+    ]
+
+    extracted_cves: set[str] = {
+        e.get("value") or e.get("entity_value", "")
+        for e in entities
+        if (e.get("type") or e.get("entity_type", "")) == "CVE_NUMBER"
+        and (e.get("value") or e.get("entity_value", ""))
+    }
+
+    ips_to_query = [
+        ip_ent.get("value") or ip_ent.get("entity_value", "")
+        for ip_ent in ip_entities
+    ][:MAX_IPS_PER_INVESTIGATION]
+
+    results: list[dict] = []
+    for ip in ips_to_query:
+        result = await enrich_shodan_ip(ip, extracted_cves)
+        if result is not None:
+            results.append(result)
+        await asyncio.sleep(SHODAN_RATE_LIMIT_DELAY)
+
+    return results

sources/telegram.py

@@ -0,0 +1,199 @@
+"""
+sources/telegram.py — Telegram public channel monitor via Telethon.
+
+Telegram is clearnet (NOT routed through Tor) but carries enormous threat
+actor activity in public groups and channels.
+
+Credentials are loaded from config.py and treated as optional:
+  TELEGRAM_API_ID    integer app id from my.telegram.org
+  TELEGRAM_API_HASH  string hash from my.telegram.org
+  TELEGRAM_PHONE     E.164 phone number (for initial interactive session auth)
+
+If any credential is missing the function returns [] immediately with a
+warning — Telegram is always optional and must never block the pipeline.
+
+Initial session setup requires running an interactive auth once (Telethon
+sends a verification code to the phone).  Subsequent calls reuse the saved
+session file ("voidaccess_telegram.session" in the working directory).
+
+Public API:
+    async def fetch_telegram_messages(
+        channel_usernames, query, limit_per_channel=100
+    ) -> list[dict]
+
+Each result dict has: channel, message_id, text, date, url.
+Matching messages are also persisted to the DB pages table.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+from datetime import timezone
+from typing import List, Optional
+from urllib.parse import urlparse
+
+_logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Lazy Telethon import — keeps the module importable even if telethon is not
+# installed (tests can still mock it; real calls will fail with ImportError
+# which is caught and returns []).
+# ---------------------------------------------------------------------------
+
+def _import_telethon():
+    """Import Telethon; raise ImportError with a clear message if missing."""
+    try:
+        from telethon import TelegramClient
+        from telethon.errors import SessionPasswordNeededError
+        return TelegramClient, SessionPasswordNeededError
+    except ImportError as exc:
+        raise ImportError(
+            "telethon is required for Telegram integration. "
+            "Install it with: pip install telethon"
+        ) from exc
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _matches(text: str, query: str) -> bool:
+    """Case-insensitive: every whitespace-separated query term must appear."""
+    text_lower = text.lower()
+    return all(term in text_lower for term in query.lower().split())
+
+
+def _t_me_url(channel: str, message_id: int) -> str:
+    return f"https://t.me/{channel.lstrip('@')}/{message_id}"
+
+
+def _persist_message(url: str, text: str) -> None:
+    """Write a matching Telegram message to the DB pages table. Silent on failure."""
+    try:
+        from config import DATABASE_URL as _db_url
+        if not _db_url:
+            return
+        from db.queries import create_page, get_page_by_hash
+        from db.session import get_session
+    except ImportError:
+        return
+
+    content_hash = hashlib.sha256(text.encode("utf-8", errors="replace")).hexdigest()
+    try:
+        with get_session() as session:
+            if get_page_by_hash(session, content_hash):
+                return
+            # Telegram messages have no .onion source — source_id stays None
+            create_page(
+                session,
+                url=url,
+                source_id=None,
+                cleaned_text=text,
+                raw_content_hash=content_hash,
+                byte_size=len(text.encode("utf-8", errors="replace")),
+            )
+    except Exception as exc:
+        _logger.debug("Telegram DB persist failed url=%s: %s", url, exc)
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+async def fetch_telegram_messages(
+    channel_usernames: List[str],
+    query: str,
+    limit_per_channel: int = 100,
+) -> List[dict]:
+    """
+    Fetch recent messages from public Telegram channels/groups and return
+    those that keyword-match *query*.
+
+    Args:
+        channel_usernames:  list of "@handle" or "username" strings.
+        query:              investigation query; all space-separated terms
+                            must appear in the message text.
+        limit_per_channel:  max messages to fetch per channel before filtering.
+
+    Returns list[dict] with keys: channel, message_id, text, date, url.
+    Returns [] immediately (with a warning) when credentials are not set.
+    Telethon errors per channel are logged and skipped; the function never
+    raises.
+    """
+    # Lazy import credentials here so config changes in tests propagate
+    try:
+        from config import TELEGRAM_API_ID, TELEGRAM_API_HASH, TELEGRAM_PHONE
+    except ImportError:
+        _logger.warning("config.py not importable; skipping Telegram.")
+        return []
+
+    if not TELEGRAM_API_ID or not TELEGRAM_API_HASH:
+        _logger.warning(
+            "TELEGRAM_API_ID and TELEGRAM_API_HASH are required for Telegram "
+            "integration.  Set them in .env and restart."
+        )
+        return []
+
+    try:
+        api_id = int(TELEGRAM_API_ID)
+    except (ValueError, TypeError):
+        _logger.warning("TELEGRAM_API_ID must be an integer.  Skipping Telegram.")
+        return []
+
+    try:
+        TelegramClient, SessionPasswordNeededError = _import_telethon()
+    except ImportError as exc:
+        _logger.warning("%s", exc)
+        return []
+
+    results: List[dict] = []
+
+    try:
+        # "voidaccess_telegram" = session file name; StringSession("") = fresh in-memory
+        # For persistent auth: use "voidaccess_telegram" (creates voidaccess_telegram.session)
+        async with TelegramClient("voidaccess_telegram", api_id, TELEGRAM_API_HASH) as client:
+            if not await client.is_user_authorized():
+                _logger.warning(
+                    "Telegram session not authorized.  Run interactive auth once: "
+                    "the client will send a code to TELEGRAM_PHONE=%s",
+                    TELEGRAM_PHONE or "<not set>",
+                )
+                return []
+
+            for raw_channel in channel_usernames:
+                channel = raw_channel.lstrip("@")
+                try:
+                    async for msg in client.iter_messages(
+                        channel, limit=limit_per_channel
+                    ):
+                        text = msg.text or ""
+                        if not text or not _matches(text, query):
+                            continue
+
+                        url = _t_me_url(channel, msg.id)
+                        date = (
+                            msg.date.astimezone(timezone.utc)
+                            if msg.date
+                            else None
+                        )
+
+                        entry = {
+                            "channel": channel,
+                            "message_id": msg.id,
+                            "text": text,
+                            "date": date,
+                            "url": url,
+                        }
+                        results.append(entry)
+                        _persist_message(url, text)
+
+                except Exception as exc:
+                    _logger.debug(
+                        "Telegram channel %s fetch failed: %s", channel, exc
+                    )
+
+    except Exception as exc:
+        _logger.warning("Telegram client error: %s", exc)
+
+    return results