voidaccess 1.3.0__py3-none-any.whl

graph/visualize.py

@@ -0,0 +1,178 @@
+"""
+graph/visualize.py — Interactive graph visualisation using pyvis.
+
+Converts a NetworkX MultiDiGraph to a pyvis Network and exports it as a
+self-contained HTML file suitable for embedding in Streamlit via st.components.
+
+If pyvis is not installed, all functions log a warning and return None / empty
+string.  They never raise on a missing dependency.
+
+Public interface
+----------------
+build_pyvis_network(graph, max_nodes, highlight_node_id)  → Network | None
+export_html(network, filepath)                            → None
+get_html_string(network)                                  → str
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Optional
+
+import networkx as nx
+
+from graph.model import NODE_TYPES
+
+if TYPE_CHECKING:
+    pass  # pyvis.network.Network imported conditionally at runtime
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Node colour palette (hex strings)
+# ---------------------------------------------------------------------------
+
+_NODE_COLORS: dict[str, str] = {
+    NODE_TYPES.THREAT_ACTOR:      "#e74c3c",  # red
+    NODE_TYPES.CRYPTO_WALLET:     "#f39c12",  # gold
+    NODE_TYPES.MALWARE_FAMILY:    "#9b59b6",  # purple
+    NODE_TYPES.RANSOMWARE_GROUP:  "#9b59b6",  # purple
+    NODE_TYPES.ONION_URL:         "#3498db",  # blue
+    NODE_TYPES.FORUM:             "#3498db",  # blue
+    NODE_TYPES.CVE:               "#e67e22",  # orange
+    NODE_TYPES.EMAIL_ADDRESS:     "#2ecc71",  # green
+    NODE_TYPES.PGP_KEY:           "#2ecc71",  # green
+    NODE_TYPES.PASTE:             "#95a5a6",  # grey
+}
+
+_DEFAULT_COLOR = "#bdc3c7"  # light grey fallback
+_HIGHLIGHT_BORDER = "#f1c40f"  # yellow
+
+# Edge width thresholds mapped to pyvis ``width`` values
+_EDGE_WIDTH_THIN   = 1.0   # confidence < 0.4
+_EDGE_WIDTH_MEDIUM = 3.0   # 0.4 <= confidence < 0.7
+_EDGE_WIDTH_THICK  = 5.0   # confidence >= 0.7
+
+
+def _confidence_to_width(confidence: float) -> float:
+    if confidence < 0.4:
+        return _EDGE_WIDTH_THIN
+    if confidence < 0.7:
+        return _EDGE_WIDTH_MEDIUM
+    return _EDGE_WIDTH_THICK
+
+
+# ---------------------------------------------------------------------------
+# Public functions
+# ---------------------------------------------------------------------------
+
+
+def build_pyvis_network(
+    graph: nx.MultiDiGraph,
+    max_nodes: int = 200,
+    highlight_node_id: Optional[str] = None,
+) -> "Optional[object]":
+    """
+    Convert *graph* into a pyvis Network.
+
+    If the graph has more than *max_nodes* nodes, only the highest-degree
+    nodes are retained.
+
+    If *highlight_node_id* is given, that node receives a yellow border.
+
+    Returns the pyvis Network, or None if pyvis is not installed.
+    """
+    try:
+        from pyvis.network import Network  # noqa: PLC0415
+    except ImportError:
+        logger.warning(
+            "pyvis is not installed — graph visualisation is unavailable. "
+            "Install it with: pip install pyvis"
+        )
+        return None
+
+    # Trim graph to max_nodes highest-degree nodes if necessary
+    if graph.number_of_nodes() > max_nodes:
+        top_nodes = sorted(
+            graph.nodes(), key=lambda n: graph.degree(n), reverse=True
+        )[:max_nodes]
+        subgraph = graph.subgraph(top_nodes)
+    else:
+        subgraph = graph
+
+    net = Network(
+        height="750px",
+        width="100%",
+        directed=True,
+        notebook=False,
+    )
+    net.force_atlas_2based()
+
+    for node_id, data in subgraph.nodes(data=True):
+        node_type = data.get("node_type", "")
+        color = _NODE_COLORS.get(node_type, _DEFAULT_COLOR)
+
+        node_kwargs: dict = {
+            "label": node_id,
+            "color": color,
+            "title": f"Type: {node_type}\nID: {node_id}",
+        }
+
+        if node_id == highlight_node_id:
+            node_kwargs["color"] = {
+                "background": color,
+                "border": _HIGHLIGHT_BORDER,
+            }
+            node_kwargs["borderWidth"] = 3
+
+        net.add_node(node_id, **node_kwargs)
+
+    for src, tgt, data in subgraph.edges(data=True):
+        confidence = data.get("confidence", 0.5)
+        edge_type = data.get("edge_type", "")
+        net.add_edge(
+            src,
+            tgt,
+            title=f"{edge_type} (confidence={confidence:.2f})",
+            width=_confidence_to_width(confidence),
+        )
+
+    return net
+
+
+def export_html(network: "object", filepath: str) -> None:
+    """
+    Save the pyvis Network as a self-contained HTML file.
+
+    Does nothing if *network* is None (pyvis not installed).
+    """
+    if network is None:
+        logger.warning("export_html called with None network — skipping.")
+        return
+    try:
+        network.save_graph(filepath)  # type: ignore[attr-defined]
+    except Exception as exc:
+        logger.warning("export_html failed: %s", exc)
+
+
+def get_html_string(network: "object") -> str:
+    """
+    Return the full interactive HTML as a string.
+
+    Used for embedding in Streamlit via ``st.components.v1.html()``.
+    Returns an empty string if *network* is None or pyvis is not installed.
+    """
+    if network is None:
+        return ""
+    try:
+        return network.generate_html()  # type: ignore[attr-defined]
+    except AttributeError:
+        # Older pyvis versions use get_network_html
+        try:
+            return network.get_network_html()  # type: ignore[attr-defined]
+        except Exception as exc:
+            logger.warning("get_html_string failed: %s", exc)
+            return ""
+    except Exception as exc:
+        logger.warning("get_html_string failed: %s", exc)
+        return ""

i18n/__init__.py

@@ -0,0 +1,24 @@
+"""
+i18n — Multilingual intelligence: language detection, translation pipeline,
+and query expansion for broader dark web coverage.
+
+Public interface
+---------------
+from i18n.detect       import detect_language, detect_language_batch, is_non_english
+from i18n.translate    import translate_to_english, translate_batch
+from i18n.query_expand import expand_query, get_multilingual_search_terms
+"""
+
+from i18n.detect import detect_language, detect_language_batch, is_non_english
+from i18n.query_expand import expand_query, get_multilingual_search_terms
+from i18n.translate import translate_batch, translate_to_english
+
+__all__ = [
+    "detect_language",
+    "detect_language_batch",
+    "is_non_english",
+    "translate_to_english",
+    "translate_batch",
+    "expand_query",
+    "get_multilingual_search_terms",
+]

i18n/detect.py

@@ -0,0 +1,76 @@
+"""
+i18n/detect.py — Language detection for scraped content.
+
+Uses the langdetect library for per-text language identification.
+Returns ISO 639-1 language codes ("en", "ru", "zh", "ar", "es", "pt", etc.).
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+def detect_language(text: str) -> Optional[str]:
+    """
+    Return the ISO 639-1 language code for *text*.
+
+    Returns None for very short text (<50 chars), undetectable text, or
+    when langdetect is not installed. Never raises.
+    """
+    try:
+        if not text or len(text) < 50:
+            return None
+
+        from langdetect import detect as ld_detect, LangDetectException  # type: ignore
+
+        try:
+            return ld_detect(text)
+        except Exception:
+            return None
+
+    except ImportError:
+        logger.debug("detect_language: langdetect not installed")
+        return None
+    except Exception:
+        return None
+
+
+def detect_language_batch(texts: list[str]) -> list[Optional[str]]:
+    """
+    Batch language detection. More efficient than calling detect_language
+    in a loop for large datasets.
+    """
+    try:
+        from langdetect import detect as ld_detect  # type: ignore
+
+        results: list[Optional[str]] = []
+        for text in texts:
+            if not text or len(text) < 50:
+                results.append(None)
+                continue
+            try:
+                results.append(ld_detect(text))
+            except Exception:
+                results.append(None)
+        return results
+
+    except ImportError:
+        logger.debug("detect_language_batch: langdetect not installed")
+        return [None] * len(texts)
+    except Exception:
+        return [None] * len(texts)
+
+
+def is_non_english(text: str) -> bool:
+    """
+    Return True if the detected language is not English (or detection fails).
+
+    Used as a quick gate before running translation.
+    """
+    lang = detect_language(text)
+    if lang is None:
+        return True
+    return lang != "en"

i18n/query_expand.py

@@ -0,0 +1,72 @@
+"""
+i18n/query_expand.py — Expands a search query into multiple languages for
+broader dark web coverage.
+
+Russian, Chinese, and Arabic dark web communities contain high-value
+intelligence that is almost entirely missed by English-only tools.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_TARGET_LANGUAGES = ["ru", "zh", "ar", "es", "de", "fr", "pt"]
+
+
+def expand_query(
+    query: str,
+    target_languages: Optional[list[str]] = None,
+) -> dict[str, str]:
+    """
+    Translate the query into multiple languages.
+
+    Default target languages: ru, zh, ar, es, de, fr, pt.
+    Returns dict: {"en": original_query, "ru": translated, ...}
+    Skips languages where translation fails — never returns None values.
+
+    Args:
+        query: Original English query
+        target_languages: List of ISO 639-1 codes. If None, uses I18N_LANGUAGES
+                         from config.py, or falls back to default languages.
+    """
+    from i18n.translate import _translate_from_english
+
+    if target_languages is None:
+        try:
+            from config import I18N_LANGUAGES
+            if I18N_LANGUAGES:
+                target_languages = I18N_LANGUAGES
+        except ImportError:
+            pass
+        if not target_languages:
+            target_languages = ["en"] + _DEFAULT_TARGET_LANGUAGES
+
+    result: dict[str, str] = {"en": query}
+
+    for lang in target_languages:
+        if lang == "en":
+            continue
+        try:
+            translated = _translate_from_english(query, lang)
+            if translated is not None and translated != query:
+                result[lang] = translated
+        except Exception as exc:
+            logger.debug("expand_query: skipping lang=%s (%s)", lang, exc)
+
+    return result
+
+
+def get_multilingual_search_terms(
+    query: str,
+    target_languages: Optional[list[str]] = None,
+) -> list[str]:
+    """
+    Return a flat list of all query translations (including original English).
+
+    Used by search.py to fan out searches in multiple languages.
+    """
+    translations = expand_query(query, target_languages)
+    return list(translations.values())

i18n/translate.py

@@ -0,0 +1,210 @@
+"""
+i18n/translate.py — Translation pipeline.
+
+Strategy (tried in order, falls back on failure):
+  1. DeepL API if DEEPL_API_KEY is set
+  2. Helsinki-NLP/opus-mt local model if transformers is available
+  3. Returns None if both are unavailable
+
+Text longer than 2000 chars is split into sentences, each translated,
+then rejoined.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+# Module-level cache: {(src_lang, tgt_lang): (tokenizer, model)}
+_model_cache: dict[tuple[str, str], tuple] = {}
+
+_SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?])\s+")
+
+
+def _split_sentences(text: str) -> list[str]:
+    parts = _SENTENCE_SPLIT_RE.split(text.strip())
+    return [p for p in parts if p.strip()]
+
+
+def _deepl_translate(
+    text: str, target_lang: str, source_lang: Optional[str] = None
+) -> Optional[str]:
+    """Call the DeepL API to translate *text* to *target_lang*."""
+    api_key = os.getenv("DEEPL_API_KEY", "")
+    if not api_key:
+        return None
+
+    try:
+        import requests  # type: ignore
+
+        params: dict = {
+            "auth_key": api_key,
+            "text": text,
+            "target_lang": target_lang.upper(),
+        }
+        if source_lang:
+            params["source_lang"] = source_lang.upper()
+
+        resp = requests.post(
+            "https://api-free.deepl.com/v2/translate",
+            data=params,
+            timeout=10,
+        )
+        resp.raise_for_status()
+        return resp.json()["translations"][0]["text"]
+    except Exception as exc:
+        logger.debug("DeepL translate failed: %s", exc)
+        return None
+
+
+def _helsinki_translate(
+    text: str, src_lang: str, tgt_lang: str = "en"
+) -> Optional[str]:
+    """Translate using a Helsinki-NLP/opus-mt local model."""
+    try:
+        from transformers import MarianMTModel, MarianTokenizer  # type: ignore
+
+        cache_key = (src_lang, tgt_lang)
+        if cache_key not in _model_cache:
+            model_name = f"Helsinki-NLP/opus-mt-{src_lang}-{tgt_lang}"
+            tokenizer = MarianTokenizer.from_pretrained(model_name)
+            model = MarianMTModel.from_pretrained(model_name)
+            _model_cache[cache_key] = (tokenizer, model)
+
+        tokenizer, model = _model_cache[cache_key]
+        inputs = tokenizer([text], return_tensors="pt", padding=True, truncation=True)
+        translated = model.generate(**inputs)
+        return tokenizer.decode(translated[0], skip_special_tokens=True)
+    except Exception as exc:
+        logger.debug("Helsinki-NLP translate failed: %s", exc)
+        return None
+
+
+def _translate_long_text(
+    text: str,
+    translate_fn,
+    *args,
+    **kwargs,
+) -> Optional[str]:
+    """Split text into sentences, translate each, rejoin."""
+    sentences = _split_sentences(text)
+    if not sentences:
+        return translate_fn(text, *args, **kwargs)
+
+    translated_parts: list[str] = []
+    for sentence in sentences:
+        result = translate_fn(sentence, *args, **kwargs)
+        if result is None:
+            return None
+        translated_parts.append(result)
+    return " ".join(translated_parts)
+
+
+def translate_to_english(
+    text: str,
+    source_lang: Optional[str] = None,
+) -> Optional[str]:
+    """
+    Translate *text* to English.
+
+    source_lang: ISO 639-1 code, or None to auto-detect.
+    Returns None on complete failure. Never raises.
+    """
+    try:
+        if not text:
+            return None
+
+        api_key = os.getenv("DEEPL_API_KEY", "")
+
+        # Split long texts at sentence boundaries
+        use_chunking = len(text) > 2000
+
+        # Strategy 1: DeepL
+        if api_key:
+            if use_chunking:
+                result = _translate_long_text(
+                    text, _deepl_translate, "EN", source_lang
+                )
+            else:
+                result = _deepl_translate(text, "EN", source_lang)
+            if result is not None:
+                return result
+
+        # Strategy 2: Helsinki-NLP local model
+        if source_lang:
+            if use_chunking:
+                result = _translate_long_text(
+                    text, _helsinki_translate, source_lang, "en"
+                )
+            else:
+                result = _helsinki_translate(text, source_lang, "en")
+            if result is not None:
+                return result
+
+        return None
+
+    except Exception as exc:
+        logger.debug("translate_to_english: unexpected error (%s)", exc)
+        return None
+
+
+def translate_batch(
+    texts: list[str],
+    source_lang: Optional[str] = None,
+) -> list[Optional[str]]:
+    """
+    Translate a list of texts to English.
+
+    English texts are returned as-is. Returns list of same length.
+    """
+    try:
+        from i18n.detect import detect_language
+    except ImportError:
+        detect_language = lambda t: None  # noqa: E731
+
+    results: list[Optional[str]] = []
+    for text in texts:
+        if not text:
+            results.append(text)
+            continue
+        detected = detect_language(text) if not source_lang else source_lang
+        if detected == "en":
+            results.append(text)
+        else:
+            results.append(translate_to_english(text, detected))
+    return results
+
+
+def _translate_from_english(text: str, target_lang: str) -> Optional[str]:
+    """
+    Translate English text to *target_lang*.
+
+    Mirrors translate_to_english but in reverse direction.
+    Used by query_expand.py.
+    """
+    try:
+        if not text:
+            return None
+
+        api_key = os.getenv("DEEPL_API_KEY", "")
+
+        # Strategy 1: DeepL (EN → target)
+        if api_key:
+            result = _deepl_translate(text, target_lang, "EN")
+            if result is not None:
+                return result
+
+        # Strategy 2: Helsinki-NLP (en → target_lang)
+        result = _helsinki_translate(text, "en", target_lang)
+        if result is not None:
+            return result
+
+        return None
+
+    except Exception as exc:
+        logger.debug("_translate_from_english: error (%s)", exc)
+        return None

monitor/__init__.py

@@ -0,0 +1,27 @@
+"""
+monitor — Phase 4 continuous monitoring, diffing, and alerts.
+"""
+
+from monitor.alerts import dispatch_alerts
+from monitor.config import load_watches
+from monitor.diff import compute_diff, is_significant_change
+from monitor.jobs import run_keyword_watch, run_url_watch
+from monitor.scheduler import (
+    get_job_status,
+    start_scheduler,
+    stop_scheduler,
+    trigger_job_now,
+)
+
+__all__ = [
+    "load_watches",
+    "run_keyword_watch",
+    "run_url_watch",
+    "compute_diff",
+    "is_significant_change",
+    "dispatch_alerts",
+    "start_scheduler",
+    "stop_scheduler",
+    "get_job_status",
+    "trigger_job_now",
+]

monitor/_db.py

@@ -0,0 +1,74 @@
+"""
+Database helpers for monitor jobs (URL watch state). Not imported by tests as pipeline mocks.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from urllib.parse import urlparse
+
+from db.models import SourceType
+
+logger = logging.getLogger(__name__)
+
+
+def _onion_key_for_url(url: str) -> str:
+    """Stable key for Source.onion_address (max 255)."""
+    u = (url or "").strip()
+    if not u:
+        return ""
+    try:
+        p = urlparse(u)
+        host = (p.netloc or p.path or "").split("/")[0]
+        if host:
+            return host[:255]
+    except Exception:
+        pass
+    return u[:255]
+
+
+def get_last_cleaned_text_for_url(url: str) -> str:
+    """Return latest cleaned_text for *url* from pages table, or ''. Never raises."""
+    if not os.getenv("DATABASE_URL"):
+        return ""
+    try:
+        from db.queries import get_page_by_url  # noqa: PLC0415
+        from db.session import get_session  # noqa: PLC0415
+
+        with get_session() as session:
+            page = get_page_by_url(session, url)
+            if page is None or not page.cleaned_text:
+                return ""
+            return str(page.cleaned_text)
+    except Exception as exc:
+        logger.warning("get_last_cleaned_text_for_url failed: %s", exc)
+        return ""
+
+
+def update_source_watch_fingerprint(url: str, content_hash_hex: str) -> None:
+    """
+    Upsert Source row and store a short content fingerprint in status (VARCHAR(20)).
+    """
+    if not os.getenv("DATABASE_URL"):
+        return
+    fp = (content_hash_hex or "")[:20]
+    if not fp:
+        return
+    key = _onion_key_for_url(url)
+    if not key:
+        return
+    try:
+        from db.queries import get_or_create_source, update_source_status  # noqa: PLC0415
+        from db.session import get_session  # noqa: PLC0415
+
+        with get_session() as session:
+            src, _created = get_or_create_source(
+                session,
+                onion_address=key,
+                source_type=SourceType.CRAWLED.value,
+            )
+            session.flush()
+            update_source_status(session, src.id, fp)
+    except Exception as exc:
+        logger.warning("update_source_watch_fingerprint failed: %s", exc)