voidaccess 1.3.0__py3-none-any.whl

config.py

@@ -0,0 +1,180 @@
+import os
+import secrets
+import logging
+from dotenv import load_dotenv
+
+load_dotenv()
+
+logger = logging.getLogger(__name__)
+
+
+def _clean_env(name, default=None):
+	value = os.getenv(name, default)
+	if value is None:
+		return None
+	value = str(value).strip()
+	# Support accidentally quoted values copied into .env
+	if len(value) >= 2 and (
+		(value[0] == value[-1] == '"') or (value[0] == value[-1] == "'")
+	):
+		value = value[1:-1].strip()
+	return value
+
+# Configuration variables loaded from the .env file
+OPENAI_API_KEY = _clean_env("OPENAI_API_KEY")
+GOOGLE_API_KEY = _clean_env("GOOGLE_API_KEY")
+ANTHROPIC_API_KEY = _clean_env("ANTHROPIC_API_KEY")
+OLLAMA_BASE_URL = _clean_env("OLLAMA_BASE_URL")
+OPENROUTER_BASE_URL = _clean_env("OPENROUTER_BASE_URL", "https://openrouter.ai/api/v1")
+OPENROUTER_API_KEY = _clean_env("OPENROUTER_API_KEY")
+GROQ_API_KEY = _clean_env("GROQ_API_KEY", "")
+LLAMA_CPP_BASE_URL = _clean_env("LLAMA_CPP_BASE_URL")
+
+# Database
+DATABASE_URL = _clean_env("DATABASE_URL")
+
+# Tor proxy (configurable so Docker can point to a tor service by name)
+TOR_PROXY_HOST = _clean_env("TOR_PROXY_HOST", "127.0.0.1")
+TOR_PROXY_PORT = _clean_env("TOR_PROXY_PORT", "9050")
+
+# Phase 1D — expanded sources (all optional; missing vars disable that source)
+DARKSEARCH_API_KEY = _clean_env("DARKSEARCH_API_KEY")          # optional free-tier key
+TELEGRAM_API_ID   = _clean_env("TELEGRAM_API_ID")              # from my.telegram.org
+TELEGRAM_API_HASH = _clean_env("TELEGRAM_API_HASH")            # from my.telegram.org
+TELEGRAM_PHONE    = _clean_env("TELEGRAM_PHONE")               # E.164, e.g. +12025551234
+
+# Phase 4 — vector store + alert channels (all optional)
+CHROMA_PERSIST_DIR = _clean_env("CHROMA_PERSIST_DIR", "./chroma_db")
+TELEGRAM_BOT_TOKEN = _clean_env("TELEGRAM_BOT_TOKEN")          # one-way alerts; separate from Telethon API vars
+SMTP_HOST          = _clean_env("SMTP_HOST")
+SMTP_PORT          = _clean_env("SMTP_PORT", "587")
+SMTP_USER          = _clean_env("SMTP_USER")
+SMTP_PASS          = _clean_env("SMTP_PASS")
+
+# Phase 5 — REST API server
+API_HOST = _clean_env("API_HOST", "0.0.0.0")
+API_PORT = _clean_env("API_PORT", "8000")
+
+# Phase 6 — advanced capabilities
+DEEPL_API_KEY        = _clean_env("DEEPL_API_KEY")           # optional; translation
+STYLOMETRY_THRESHOLD = _clean_env("STYLOMETRY_THRESHOLD", "0.85")  # same-author detection
+
+# LLM extraction cache (optional — defaults to enabled)
+DISABLE_EXTRACTION_CACHE = _clean_env("DISABLE_EXTRACTION_CACHE")
+if DISABLE_EXTRACTION_CACHE is not None:
+    DISABLE_EXTRACTION_CACHE = DISABLE_EXTRACTION_CACHE.lower() == "true"
+else:
+    DISABLE_EXTRACTION_CACHE = False
+
+# i18n / Query Expansion
+# Languages to include in query expansion (comma-separated ISO 639-1 codes)
+# Default: en, ru, zh (English, Russian, Chinese)
+I18N_LANGUAGES = _clean_env("I18N_LANGUAGES", "en,ru,zh")
+if I18N_LANGUAGES and isinstance(I18N_LANGUAGES, str):
+    I18N_LANGUAGES = [lang.strip() for lang in I18N_LANGUAGES.split(",") if lang.strip()]
+else:
+    I18N_LANGUAGES = ["en", "ru", "zh"]
+
+# Threat Intelligence API Keys
+OTX_API_KEY = _clean_env("OTX_API_KEY", "")  # AlienVault OTX — free at otx.alienvault.com
+VT_API_KEY = _clean_env("VT_API_KEY", "")    # VirusTotal — free tier at virustotal.com
+
+# IP Reputation Enrichment (all optional — features degrade gracefully without keys)
+ABUSEIPDB_API_KEY = _clean_env("ABUSEIPDB_API_KEY", "")   # Community IP abuse reports
+GREYNOISE_API_KEY = _clean_env("GREYNOISE_API_KEY", "")   # Suppresses benign scanner IPs
+C2_FEED_CACHE_TTL = _clean_env("C2_FEED_CACHE_TTL", "24") # Hours between feed refreshes
+
+# Domain Reputation Enrichment
+URLSCAN_API_KEY        = _clean_env("URLSCAN_API_KEY", "")
+SECURITYTRAILS_API_KEY = _clean_env("SECURITYTRAILS_API_KEY", "")
+
+# Code Intelligence (GitHub / GitLab scraping)
+GITHUB_TOKEN = _clean_env("GITHUB_TOKEN", "")
+GITLAB_TOKEN = _clean_env("GITLAB_TOKEN", "")
+
+# Hash Reputation Enrichment
+HYBRID_ANALYSIS_API_KEY = _clean_env("HYBRID_ANALYSIS_API_KEY", "")
+
+# Email Reputation Enrichment
+HIBP_API_KEY      = _clean_env("HIBP_API_KEY", "")
+EMAILREP_API_KEY  = _clean_env("EMAILREP_API_KEY", "")
+
+SHODAN_RATE_LIMIT_DELAY = 1.0        # seconds between Shodan requests (InternetDB)
+MAX_IPS_PER_INVESTIGATION = 50      # max IPs to query Shodan per investigation
+MAX_HASHES_PER_INVESTIGATION = 20    # max file hashes to query VirusTotal per investigation
+
+# Blockchain API Keys (optional — free tiers work without)
+BLOCKCYPHER_TOKEN = _clean_env("BLOCKCYPHER_TOKEN", "")
+ETHERSCAN_API_KEY = _clean_env("ETHERSCAN_API_KEY", "")
+
+# Auth — REQUIRED in production. Generate with: python -c "import secrets; print(secrets.token_hex(32))"
+_jwt_secret = _clean_env("JWT_SECRET")
+if _jwt_secret is None:
+    raise RuntimeError(
+        "JWT_SECRET environment variable is not set. "
+        "Generate a secure secret with: python -c \"import secrets; print(secrets.token_hex(32))\" "
+        "and set it as JWT_SECRET in your .env file. "
+        "Do NOT use a random secret — it will change on restart and invalidate all issued tokens."
+    )
+JWT_SECRET = _jwt_secret
+
+# Token blacklist Redis (optional — omit to disable blacklist checks)
+REDIS_URL = _clean_env("REDIS_URL")
+
+# Playwright JS rendering for JavaScript-heavy .onion sites
+# Set to False to disable (faster startup, lower memory usage)
+# Requires: playwright installed + browsers downloaded
+PLAYWRIGHT_ENABLED = _clean_env("PLAYWRIGHT_ENABLED", "true")
+if PLAYWRIGHT_ENABLED is not None:
+    PLAYWRIGHT_ENABLED = PLAYWRIGHT_ENABLED.lower() == "true"
+else:
+    PLAYWRIGHT_ENABLED = True
+
+
+DEFAULT_MODELS = {
+    "openrouter": "deepseek/deepseek-chat",
+    "openai":     "gpt-4o-mini",
+    "anthropic":  "claude-haiku-4-5-20251001",
+    "google":     "gemini-1.5-flash",
+    "groq":       "llama-3.3-70b-versatile",
+    "ollama":     "llama3.2",
+}
+
+DEFAULT_MODEL = os.getenv(
+    "DEFAULT_MODEL",
+    "openrouter/deepseek/deepseek-chat"
+)
+
+REQUIRED_KEYS = [
+    "JWT_SECRET",
+]
+
+OPTIONAL_KEYS = [
+    "OPENAI_API_KEY",
+    "GOOGLE_API_KEY",
+    "ANTHROPIC_API_KEY",
+    "OTX_API_KEY",
+    "DEEPL_API_KEY",
+    "DARKSEARCH_API_KEY",
+    "OLLAMA_BASE_URL",
+    "OPENROUTER_API_KEY",
+    "LLAMA_CPP_BASE_URL",
+    "BLOCKCYPHER_TOKEN",
+    "ETHERSCAN_API_KEY",
+]
+
+
+def validate_config():
+    missing_required = []
+    for key in REQUIRED_KEYS:
+        if _clean_env(key) is None:
+            missing_required.append(key)
+    if missing_required:
+        raise RuntimeError(f"Missing required configuration keys: {', '.join(missing_required)}")
+    for key in OPTIONAL_KEYS:
+        val = _clean_env(key)
+        if val is None:
+            logger.debug(f"Optional configuration key '%s' is not set - related features will be disabled", key)
+
+
+validate_config()

crawler/__init__.py

@@ -0,0 +1,28 @@
+"""
+crawler — Phase 1C recursive .onion spider.
+
+Public interface:
+    CrawlResult  dataclass  — crawl statistics + scraped content
+    crawl()      async fn   — entry point; accepts seeds, query, and tuning params
+
+Example
+-------
+    import asyncio
+    from crawler import CrawlResult, crawl
+
+    result: CrawlResult = asyncio.run(crawl(
+        seed_urls=["http://exampleonionaddressv3aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.onion"],
+        query="ransomware affiliate recruitment",
+        max_depth=2,
+        max_pages=50,
+        min_relevance=0.3,
+    ))
+
+    print(result.pages_crawled, result.new_urls_discovered)
+    for page in result.results:
+        print(page["url"], page["content"][:200])
+"""
+
+from crawler.spider import CrawlResult, crawl
+
+__all__ = ["CrawlResult", "crawl"]

crawler/dedup.py

@@ -0,0 +1,97 @@
+"""
+crawler/dedup.py — Two-level deduplication for the .onion crawler.
+
+Level 1 — URL dedup:
+    In-memory set; never visits the same normalized URL twice within a run.
+
+Level 2 — Content dedup:
+    SHA-256 of extracted text is checked against the pages table before a DB
+    write.  If the hash already exists the write is skipped, but the URL is
+    still counted as visited (content was seen elsewhere — no need to store it
+    again, but crawling this URL was not wasted work).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+from typing import Optional
+
+_logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Level 1 — URL deduplication
+# ---------------------------------------------------------------------------
+
+class UrlDedup:
+    """
+    In-memory URL deduplication scoped to a single crawl run.
+
+    Thread-safety: designed for asyncio (single-threaded event loop).
+    All operations are synchronous and O(1).
+    """
+
+    def __init__(self) -> None:
+        self._seen: set[str] = set()
+
+    def is_new(self, url: str) -> bool:
+        """Return True if *url* has not been seen in this run."""
+        return url not in self._seen
+
+    def mark_seen(self, url: str) -> None:
+        """Mark *url* as seen so future is_new() calls return False."""
+        self._seen.add(url)
+
+    def __len__(self) -> int:
+        return len(self._seen)
+
+
+# ---------------------------------------------------------------------------
+# Level 2 — Content deduplication
+# ---------------------------------------------------------------------------
+
+class ContentDedup:
+    """
+    Content-hash deduplication backed by the database pages table.
+
+    Uses SHA-256 of raw download bytes (same convention as scrape.py's
+    _persist_pages so hashes are consistent across both pipelines).
+
+    Falls back to "not a duplicate" when DATABASE_URL is not set or the DB
+    is unreachable — this keeps the crawler running even without a database.
+    """
+
+    @staticmethod
+    def hash_bytes(data: bytes) -> str:
+        """Return SHA-256 hex digest of raw bytes."""
+        return hashlib.sha256(data).hexdigest()
+
+    @staticmethod
+    def hash_text(text: str) -> str:
+        """Return SHA-256 hex digest of text (UTF-8 encoded)."""
+        return hashlib.sha256(text.encode("utf-8", errors="replace")).hexdigest()
+
+    @staticmethod
+    def is_duplicate(content_hash: str, db_url: Optional[str] = None) -> bool:
+        """
+        Return True if *content_hash* already exists in the pages table.
+
+        *db_url* overrides DATABASE_URL — used in tests to point at the
+        test SQLite database without modifying the real config.
+
+        Any exception (DB unreachable, import error, etc.) is silently
+        swallowed and returns False so the crawler never crashes on dedup.
+        """
+        try:
+            from config import DATABASE_URL as _cfg_db_url
+            target = db_url or _cfg_db_url
+            if not target:
+                return False
+            from db.queries import get_page_by_hash
+            from db.session import get_session
+            with get_session(target) as session:
+                return get_page_by_hash(session, content_hash) is not None
+        except Exception as exc:
+            _logger.debug("ContentDedup DB check failed: %s", exc)
+            return False

crawler/frontier.py

@@ -0,0 +1,115 @@
+"""
+crawler/frontier.py — Priority-queue crawl frontier with relevance scoring.
+
+Each URL is scored by cosine similarity between "<url> <snippet>" and the
+investigation query using the sentence-transformers all-MiniLM-L6-v2 model.
+Higher score → popped sooner (max-priority implemented via negated scores on
+a min-heap).
+
+Uses vector.model_singleton for the shared SentenceTransformer instance.
+"""
+
+from __future__ import annotations
+
+import heapq
+import logging
+from typing import Optional, Tuple
+
+import numpy as np
+
+from vector.model_singleton import get_embedding_model
+
+_logger = logging.getLogger(__name__)
+
+
+def _get_model():
+    """Return the SentenceTransformer singleton."""
+    return get_embedding_model()
+
+
+# ---------------------------------------------------------------------------
+# Cosine similarity
+# ---------------------------------------------------------------------------
+
+def _cosine(a: np.ndarray, b: np.ndarray) -> float:
+    """Cosine similarity clipped to [0.0, 1.0]."""
+    denom = float(np.linalg.norm(a) * np.linalg.norm(b)) + 1e-10
+    return float(np.clip(np.dot(a, b) / denom, 0.0, 1.0))
+
+
+# ---------------------------------------------------------------------------
+# Frontier
+# ---------------------------------------------------------------------------
+
+class Frontier:
+    """
+    Min-heap priority queue where URLs with *higher* relevance scores are
+    popped first (achieved by storing negated scores as heap keys).
+
+    Not thread-safe — designed for a single asyncio event loop.
+    """
+
+    def __init__(self, query: str) -> None:
+        self._query = query
+        self._heap: list = []
+        self._counter = 0  # monotone tie-breaker so URLs are never compared
+        self._query_embedding: Optional[np.ndarray] = None
+
+    # ------------------------------------------------------------------
+    # Embedding helpers
+    # ------------------------------------------------------------------
+
+    def _query_emb(self) -> np.ndarray:
+        """Return (cached) embedding for the investigation query."""
+        if self._query_embedding is None:
+            self._query_embedding = _get_model().encode(
+                self._query, convert_to_numpy=True
+            )
+        return self._query_embedding
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def score(self, url: str, snippet: str = "") -> float:
+        """
+        Compute relevance score (0.0–1.0) for *url* + optional *snippet*.
+
+        The input text is "<url> <snippet>" embedded and compared to the
+        investigation query via cosine similarity.  Returns 0.5 on any
+        embedding failure so the crawler degrades gracefully.
+        """
+        text = f"{url} {snippet}".strip()
+        try:
+            model = _get_model()
+            if model is None:
+                return 0.5
+            emb = model.encode(text, convert_to_numpy=True)
+            return _cosine(emb, self._query_emb())
+        except Exception as exc:
+            _logger.debug("Frontier.score error: %s", exc)
+            return 0.5
+
+    def push(self, url: str, depth: int, score: float) -> None:
+        """
+        Add *url* at the given *depth* with pre-computed *score*.
+
+        Call Frontier.score() first to obtain the score; separating scoring
+        from pushing lets callers filter by min_relevance before enqueueing.
+        """
+        heapq.heappush(self._heap, (-score, self._counter, url, depth))
+        self._counter += 1
+
+    def pop(self) -> Tuple[str, int]:
+        """
+        Return (url, depth) for the highest-relevance item.
+        Raises IndexError if the frontier is empty.
+        """
+        _, _, url, depth = heapq.heappop(self._heap)
+        return url, depth
+
+    def empty(self) -> bool:
+        return not self._heap
+
+    def __len__(self) -> int:
+        return len(self._heap)