voidaccess 1.3.0__py3-none-any.whl

analysis/__init__.py

@@ -0,0 +1,49 @@
+"""
+analysis — Temporal, behavioral pattern, and OPSEC failure analysis.
+
+Public interface
+---------------
+from analysis.temporal  import build_activity_timeline, compute_activity_stats
+from analysis.temporal  import detect_anomalies, detect_silence_breaks
+from analysis.patterns  import check_exit_scam_pattern, check_law_enforcement_pattern
+from analysis.patterns  import check_new_actor_pattern, run_all_patterns
+from analysis.opsec     import detect_timezone_leak, detect_language_switch
+from analysis.opsec     import detect_clearnet_slip, detect_pgp_reuse
+from analysis.opsec     import run_full_opsec_analysis
+"""
+
+from analysis.opsec import (
+    detect_clearnet_slip,
+    detect_language_switch,
+    detect_pgp_reuse,
+    detect_timezone_leak,
+    run_full_opsec_analysis,
+)
+from analysis.patterns import (
+    check_exit_scam_pattern,
+    check_law_enforcement_pattern,
+    check_new_actor_pattern,
+    run_all_patterns,
+)
+from analysis.temporal import (
+    build_activity_timeline,
+    compute_activity_stats,
+    detect_anomalies,
+    detect_silence_breaks,
+)
+
+__all__ = [
+    "build_activity_timeline",
+    "compute_activity_stats",
+    "detect_anomalies",
+    "detect_silence_breaks",
+    "check_exit_scam_pattern",
+    "check_law_enforcement_pattern",
+    "check_new_actor_pattern",
+    "run_all_patterns",
+    "detect_timezone_leak",
+    "detect_language_switch",
+    "detect_clearnet_slip",
+    "detect_pgp_reuse",
+    "run_full_opsec_analysis",
+]

analysis/opsec.py

@@ -0,0 +1,454 @@
+"""
+analysis/opsec.py — Detects operational security failures in threat actor
+communications that inadvertently reveal real-world identity or location.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from collections import Counter
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+# Known clearnet URL regex — captures domain from http(s) URLs
+_HTTP_URL_RE = re.compile(
+    r"https?://([a-zA-Z0-9][-a-zA-Z0-9.]*\.[a-zA-Z]{2,})(?:[/?#][^\s]*)?",
+    re.IGNORECASE,
+)
+
+# Structured data patterns to strip before language detection
+_BITCOIN_RE = re.compile(r"\b[13][a-km-zA-HJ-NP-Z1-9]{25,34}\b")
+_ETH_RE = re.compile(r"\b0x[a-fA-F0-9]{40}\b")
+_CVE_RE = re.compile(r"\bCVE-\d{4}-\d{4,7}\b")
+_URL_RE = re.compile(r"https?://\S+")
+_ONION_RE = re.compile(r"\b[a-z2-7]{56}\.onion\b", re.IGNORECASE)
+
+
+def _strip_non_linguistic(text: str) -> str:
+    """Remove URLs, wallet addresses, CVE IDs, and .onion addresses before language detection."""
+    text = _URL_RE.sub(" ", text)
+    text = _BITCOIN_RE.sub(" ", text)
+    text = _ETH_RE.sub(" ", text)
+    text = _CVE_RE.sub(" ", text)
+    text = _ONION_RE.sub(" ", text)
+    return text
+
+
+def detect_timezone_leak(texts_with_timestamps: list[dict]) -> dict:
+    """
+    Analyze posting time distribution to infer actor timezone.
+
+    Input: list of {"text": str, "timestamp": datetime}
+    If 80%+ of posts fall within a 6-hour window: infer timezone.
+
+    Returns:
+        detected: bool
+        probable_timezone_offset: str | None  (e.g. "UTC+3")
+        confidence: float
+        posting_hours: list[int]
+        peak_window: str | None  (e.g. "09:00-15:00 UTC")
+    """
+    if not texts_with_timestamps:
+        return {
+            "detected": False,
+            "probable_timezone_offset": None,
+            "confidence": 0.0,
+            "posting_hours": [],
+            "peak_window": None,
+        }
+
+    posting_hours: list[int] = []
+    for entry in texts_with_timestamps:
+        ts = entry.get("timestamp")
+        if ts is None:
+            continue
+        if hasattr(ts, "utcoffset") and ts.utcoffset() is not None:
+            # Convert to UTC
+            utc_ts = ts.astimezone(tz=None).replace(tzinfo=None)
+            posting_hours.append(utc_ts.hour)
+        else:
+            posting_hours.append(ts.hour)
+
+    if not posting_hours:
+        return {
+            "detected": False,
+            "probable_timezone_offset": None,
+            "confidence": 0.0,
+            "posting_hours": [],
+            "peak_window": None,
+        }
+
+    total = len(posting_hours)
+
+    # Sliding 6-hour window to find best coverage
+    best_start = 0
+    best_count = 0
+    for h in range(24):
+        window_hours = {(h + offset) % 24 for offset in range(6)}
+        count = sum(1 for hour in posting_hours if hour in window_hours)
+        if count > best_count:
+            best_count = count
+            best_start = h
+
+    coverage = best_count / total
+
+    if coverage >= 0.80:
+        end_hour = (best_start + 6) % 24
+        peak_window = f"{best_start:02d}:00-{end_hour:02d}:00 UTC"
+
+        # Infer timezone: assume actor is active during 09:00-17:00 local.
+        # Window center → local noon assumption (midpoint at ~13:00 local)
+        window_center_utc = (best_start + 3) % 24
+        offset_raw = 13 - window_center_utc
+        if offset_raw > 12:
+            offset_raw -= 24
+        elif offset_raw < -12:
+            offset_raw += 24
+
+        if offset_raw >= 0:
+            tz_str = f"UTC+{offset_raw}"
+        else:
+            tz_str = f"UTC{offset_raw}"
+
+        return {
+            "detected": True,
+            "probable_timezone_offset": tz_str,
+            "confidence": round(coverage, 3),
+            "posting_hours": posting_hours,
+            "peak_window": peak_window,
+        }
+
+    return {
+        "detected": False,
+        "probable_timezone_offset": None,
+        "confidence": round(coverage, 3),
+        "posting_hours": posting_hours,
+        "peak_window": None,
+    }
+
+
+def detect_language_switch(texts: list[str]) -> dict:
+    """
+    Detect if an actor switches between languages across posts.
+
+    Returns:
+        detected: bool
+        languages_found: list[str]  (ISO 639-1 codes)
+        primary_language: str | None
+        switch_count: int
+        switched_texts_indices: list[int]
+    """
+    try:
+        from langdetect import detect as ld_detect
+    except ImportError:
+        return {"detected": False}
+
+    if not texts:
+        return {
+            "detected": False,
+            "languages_found": [],
+            "primary_language": None,
+            "switch_count": 0,
+            "switched_texts_indices": [],
+        }
+
+    detected_langs: list[Optional[str]] = []
+    for text in texts:
+        if not text:
+            detected_langs.append(None)
+            continue
+        clean_text = _strip_non_linguistic(text)
+        if len(clean_text) < 50:
+            detected_langs.append(None)
+            continue
+        try:
+            detected_langs.append(ld_detect(clean_text))
+        except Exception:
+            detected_langs.append(None)
+
+    valid_langs = [lang for lang in detected_langs if lang is not None]
+    if not valid_langs:
+        return {
+            "detected": False,
+            "languages_found": [],
+            "primary_language": None,
+            "switch_count": 0,
+            "switched_texts_indices": [],
+        }
+
+    counter = Counter(valid_langs)
+    primary_lang, _ = counter.most_common(1)[0]
+    languages_found = list(counter.keys())
+
+    switched_indices = [
+        i
+        for i, lang in enumerate(detected_langs)
+        if lang is not None and lang != primary_lang
+    ]
+
+    detected = len(switched_indices) > 0
+
+    return {
+        "detected": detected,
+        "languages_found": languages_found,
+        "primary_language": primary_lang,
+        "switch_count": len(switched_indices),
+        "switched_texts_indices": switched_indices,
+    }
+
+
+def detect_clearnet_slip(texts: list[str]) -> dict:
+    """
+    Find clearnet URLs accidentally posted in a dark web context.
+
+    Clearnet = any URL whose domain does not end in .onion.
+
+    Returns:
+        detected: bool
+        clearnet_urls: list[str]
+        platforms: list[str]  (e.g. ["youtube.com", "reddit.com"])
+    """
+    clearnet_urls: list[str] = []
+    platforms: set[str] = set()
+
+    for text in texts:
+        if not text:
+            continue
+        for match in _HTTP_URL_RE.finditer(text):
+            domain = match.group(1).lower()
+            full_url = match.group(0)
+            if not domain.endswith(".onion"):
+                clearnet_urls.append(full_url)
+                # Extract base domain (last two parts)
+                parts = domain.rstrip(".").split(".")
+                if len(parts) >= 2:
+                    platforms.add(".".join(parts[-2:]))
+                else:
+                    platforms.add(domain)
+
+    return {
+        "detected": len(clearnet_urls) > 0,
+        "clearnet_urls": clearnet_urls,
+        "platforms": sorted(platforms),
+    }
+
+
+def detect_pgp_reuse(
+    pgp_fingerprints: list[str],
+    sources: Optional[list[str]] = None,
+) -> dict:
+    """
+    Check if the same PGP fingerprint appears across multiple source domains,
+    or multiple times in the fingerprint list.
+
+    When *sources* is provided with the same length as *pgp_fingerprints*,
+    reuse is detected if the same fingerprint maps to more than one source.
+
+    When *sources* is omitted or length mismatches, reuse is detected when
+    any fingerprint appears more than once in *pgp_fingerprints*.
+
+    Returns:
+        detected: bool
+        reused_fingerprints: list[str]
+        cross_platform_exposure: list[dict]
+        forum_count: int
+        fingerprint: str | None
+    """
+    if not pgp_fingerprints:
+        return {
+            "detected": False,
+            "reused_fingerprints": [],
+            "cross_platform_exposure": [],
+            "forum_count": 0,
+            "fingerprint": None,
+        }
+
+    normalized = [fp.strip() for fp in pgp_fingerprints if fp and str(fp).strip()]
+
+    if sources is not None and len(sources) == len(pgp_fingerprints):
+        fp_to_sources: dict[str, set[str]] = {}
+        for fp, src in zip(normalized, sources):
+            if fp not in fp_to_sources:
+                fp_to_sources[fp] = set()
+            fp_to_sources[fp].add(src or "")
+
+        reused: list[str] = []
+        cross_platform: list[dict] = []
+
+        for fp, srcs in fp_to_sources.items():
+            if len(srcs) > 1:
+                reused.append(fp)
+                cross_platform.append({"fingerprint": fp, "sources": sorted(srcs)})
+
+        return {
+            "detected": len(reused) > 0,
+            "reused_fingerprints": reused,
+            "cross_platform_exposure": cross_platform,
+            "forum_count": max((len(s) for s in fp_to_sources.values()), default=0),
+            "fingerprint": reused[0] if reused else None,
+        }
+
+    cnt = Counter(normalized)
+    dupes = [fp for fp, n in cnt.items() if n > 1]
+    return {
+        "detected": len(dupes) > 0,
+        "reused_fingerprints": dupes,
+        "cross_platform_exposure": [],
+        "forum_count": 2,
+        "fingerprint": dupes[0] if dupes else None,
+    }
+
+
+def run_full_opsec_analysis(
+    handle: str,
+    texts_with_timestamps: list[dict],
+    pgp_fingerprints: Optional[list[str]] = None,
+    pgp_sources: Optional[list[str]] = None,
+) -> dict:
+    """
+    Run all OPSEC checks for a given actor.
+
+    Returns combined report with findings, opsec_score (100 = best), risk_level,
+    and legacy keys timezone_leak / language_switch / clearnet_slips for callers.
+    """
+    texts = [entry.get("text", "") for entry in texts_with_timestamps]
+
+    tz_result = detect_timezone_leak(texts_with_timestamps)
+    lang_result = detect_language_switch(texts)
+    clearnet_result = detect_clearnet_slip(texts)
+
+    if tz_result.get("detected"):
+        primary_language = lang_result.get("primary_language", "unknown")
+        data_points = len(texts_with_timestamps)
+        original_conf = float(tz_result.get("confidence", 0.5))
+        tz_result["data_points"] = data_points
+        tz_result["primary_language_correlation"] = primary_language
+        if primary_language == "en" or data_points < 20:
+            tz_result["confidence"] = round(original_conf * 0.5, 3)
+            tz_result["confidence_level"] = "low"
+            tz_result["note"] = (
+                "Insufficient data for reliable timezone inference"
+                if data_points < 20
+                else "Timezone leak is LOW confidence for English content"
+            )
+        else:
+            tz_result["confidence_level"] = "high" if original_conf >= 0.85 else "medium"
+
+    findings: list[dict] = []
+    score = 100
+
+    if tz_result.get("detected"):
+        conf = float(tz_result.get("confidence", 0.5))
+        severity = tz_result.get("confidence_level", "high") if conf >= 0.4 else "low"
+        findings.append(
+            {
+                "type": "timezone_leak",
+                "severity": severity,
+                "description": (
+                    f"Timezone leak: probable {tz_result.get('probable_timezone_offset', 'unknown')}"
+                ),
+                "evidence": (
+                    f"Activity window: {tz_result.get('peak_window', 'unknown')} "
+                    f"(confidence {conf:.0%})"
+                ),
+                "first_detected": None,
+            }
+        )
+        score -= 25
+
+    if lang_result.get("detected"):
+        langs = lang_result.get("languages_found", [])
+        findings.append(
+            {
+                "type": "language_switch",
+                "severity": "medium",
+                "description": (
+                    f"{lang_result.get('switch_count', 0)} language switch(es) detected"
+                ),
+                "evidence": (
+                    f"Primary: {lang_result.get('primary_language', 'unknown')}. "
+                    f"Also: {', '.join(str(l) for l in langs if l != lang_result.get('primary_language'))}"
+                ),
+                "first_detected": None,
+            }
+        )
+        score -= 15
+
+    if clearnet_result.get("detected"):
+        platforms = clearnet_result.get("platforms", [])
+        findings.append(
+            {
+                "type": "clearnet_slip",
+                "severity": "high",
+                "description": (
+                    f"{len(clearnet_result.get('clearnet_urls', []))} clearnet URL(s) posted"
+                ),
+                "evidence": f"Platforms: {', '.join(str(p) for p in platforms[:5])}",
+                "first_detected": None,
+            }
+        )
+        score -= 15
+
+    pgp_result: dict = {"detected": False}
+    if pgp_fingerprints and len(pgp_fingerprints) > 1:
+        pgp_result = detect_pgp_reuse(pgp_fingerprints, pgp_sources)
+        if pgp_result.get("detected"):
+            fp_short = (pgp_result.get("fingerprint") or "")[:16]
+            findings.append(
+                {
+                    "type": "pgp_reuse",
+                    "severity": "high",
+                    "description": (
+                        f"Same PGP key used across {pgp_result.get('forum_count', 2)} forums"
+                    ),
+                    "evidence": f"Key {fp_short}... reused",
+                    "first_detected": None,
+                }
+            )
+            score -= 20
+
+    score = max(0, score)
+
+    if score >= 80:
+        risk_level = "LOW"
+    elif score >= 60:
+        risk_level = "MEDIUM"
+    elif score >= 40:
+        risk_level = "HIGH"
+    else:
+        risk_level = "CRITICAL"
+
+    # Legacy normalized risk_score (0–1), higher = worse — for backward compatibility
+    legacy_scores: list[float] = []
+    if tz_result.get("detected"):
+        legacy_scores.append(float(tz_result.get("confidence", 0.5)))
+    else:
+        legacy_scores.append(0.0)
+    if lang_result.get("detected"):
+        n_texts = len(texts)
+        n_switched = lang_result.get("switch_count", 0)
+        legacy_scores.append(min(1.0, n_switched / max(n_texts, 1)))
+    else:
+        legacy_scores.append(0.0)
+    if clearnet_result.get("detected"):
+        legacy_scores.append(1.0)
+    else:
+        legacy_scores.append(0.0)
+
+    risk_score = sum(legacy_scores) / len(legacy_scores) if legacy_scores else 0.0
+    if pgp_result.get("detected"):
+        risk_score = min(1.0, risk_score + 0.2)
+
+    return {
+        "handle": handle,
+        "timezone_leak": tz_result,
+        "language_switch": lang_result,
+        "clearnet_slips": clearnet_result,
+        "pgp_reuse": pgp_result,
+        "findings": findings,
+        "opsec_score": score,
+        "risk_level": risk_level,
+        "risk_score": round(risk_score, 3),
+    }

analysis/patterns.py

@@ -0,0 +1,202 @@
+"""
+analysis/patterns.py — Pattern library for known behavioral signatures.
+
+Heuristic rules derived from threat intelligence research for detecting
+exit scams, law enforcement actions, and new actor emergence.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import date, timedelta
+from typing import Any
+
+from analysis.temporal import (  # noqa: E402 — imported at module level for patchability
+    build_activity_timeline,
+    compute_activity_stats,
+    detect_anomalies,
+    detect_silence_breaks,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def check_exit_scam_pattern(timeline: list[dict]) -> dict:
+    """
+    Check if a marketplace/forum shows exit scam warning signs.
+
+    Criteria: activity drops >60% over the last 14 days vs prior 14-day average.
+    Returns {"risk": "high"|"medium"|"low", "confidence": float, "reason": str}.
+    """
+    if not timeline:
+        return {"risk": "low", "confidence": 0.0, "reason": "Insufficient data"}
+
+    sorted_entries = sorted(timeline, key=lambda x: x["date"])
+
+    if len(sorted_entries) < 2:
+        return {"risk": "low", "confidence": 0.1, "reason": "Insufficient historical data"}
+
+    last_date = sorted_entries[-1]["date"]
+    cutoff_recent = last_date - timedelta(days=14)
+    cutoff_prior = cutoff_recent - timedelta(days=14)
+
+    recent_counts = [
+        e["count"] for e in sorted_entries if e["date"] > cutoff_recent
+    ]
+    prior_counts = [
+        e["count"]
+        for e in sorted_entries
+        if cutoff_prior < e["date"] <= cutoff_recent
+    ]
+
+    if not prior_counts:
+        return {
+            "risk": "low",
+            "confidence": 0.1,
+            "reason": "No prior 14-day baseline available",
+        }
+
+    recent_avg = sum(recent_counts) / max(len(recent_counts), 1) if recent_counts else 0.0
+    prior_avg = sum(prior_counts) / len(prior_counts)
+
+    if prior_avg == 0.0:
+        return {"risk": "low", "confidence": 0.2, "reason": "No prior activity baseline"}
+
+    drop_ratio = 1.0 - (recent_avg / prior_avg)
+    confidence = min(
+        1.0,
+        len(sorted_entries) / 30.0,  # more data → higher confidence
+    )
+
+    if drop_ratio > 0.60:
+        return {
+            "risk": "high",
+            "confidence": round(confidence, 3),
+            "reason": (
+                f"Activity dropped {drop_ratio:.0%} over the last 14 days "
+                f"(from {prior_avg:.1f} to {recent_avg:.1f} posts/day)"
+            ),
+        }
+    elif drop_ratio > 0.30:
+        return {
+            "risk": "medium",
+            "confidence": round(confidence * 0.7, 3),
+            "reason": (
+                f"Moderate activity decline of {drop_ratio:.0%} over the last 14 days"
+            ),
+        }
+    else:
+        return {
+            "risk": "low",
+            "confidence": round(confidence * 0.5, 3),
+            "reason": "Activity levels are stable",
+        }
+
+
+def check_law_enforcement_pattern(timeline: list[dict]) -> dict:
+    """
+    Check for sudden complete silence after sustained activity.
+
+    Criteria: zero activity for 7+ consecutive days after a period of daily activity.
+    Returns {"risk": "high"|"medium"|"low", "confidence": float, "reason": str}.
+    """
+    if not timeline or len(timeline) < 2:
+        return {"risk": "low", "confidence": 0.0, "reason": "Insufficient data"}
+
+    sorted_entries = sorted(timeline, key=lambda x: x["date"])
+
+    # Check for silence in the last N calendar days relative to last seen
+    last_date = sorted_entries[-1]["date"]
+    today = date.today()
+    days_since_last = (today - last_date).days
+
+    # Check if there was sustained prior activity (at least 5 data points)
+    has_sustained = len(sorted_entries) >= 5
+    confidence_base = min(1.0, len(sorted_entries) / 20.0)
+
+    if days_since_last >= 7 and has_sustained:
+        return {
+            "risk": "high",
+            "confidence": round(min(confidence_base + 0.3, 1.0), 3),
+            "reason": (
+                f"Complete silence for {days_since_last} days after sustained activity "
+                f"({len(sorted_entries)} active days on record)"
+            ),
+        }
+    elif days_since_last >= 3 and has_sustained:
+        return {
+            "risk": "medium",
+            "confidence": round(confidence_base * 0.6, 3),
+            "reason": f"Reduced activity for {days_since_last} days",
+        }
+    else:
+        return {
+            "risk": "low",
+            "confidence": round(confidence_base * 0.3, 3),
+            "reason": "Activity pattern appears normal",
+        }
+
+
+def check_new_actor_pattern(
+    entity_value: str,
+    entity_type: str,
+) -> dict:
+    """
+    Check if this entity has appeared for the first time in the last 7 days.
+
+    Returns {"is_new": bool, "first_seen": date | None, "days_active": int}.
+    """
+    try:
+        timeline = build_activity_timeline(entity_value, entity_type)
+        if not timeline:
+            return {"is_new": False, "first_seen": None, "days_active": 0}
+
+        stats = compute_activity_stats(timeline)
+        first_seen = stats.get("first_seen")
+        days_active = int(stats.get("active_days", 0))
+
+        is_new = False
+        if first_seen is not None:
+            days_since_first = (date.today() - first_seen).days
+            is_new = days_since_first < 7
+
+        return {
+            "is_new": is_new,
+            "first_seen": first_seen,
+            "days_active": days_active,
+        }
+
+    except Exception as exc:
+        logger.debug("check_new_actor_pattern: error (%s)", exc)
+        return {"is_new": False, "first_seen": None, "days_active": 0}
+
+
+def run_all_patterns(
+    entity_value: str,
+    entity_type: str,
+) -> dict:
+    """
+    Run all pattern checks for an entity.
+
+    Returns combined dict with keys: exit_scam, law_enforcement, new_actor,
+    anomalies, silence_breaks.
+    """
+    try:
+        timeline = build_activity_timeline(entity_value, entity_type)
+
+        return {
+            "exit_scam": check_exit_scam_pattern(timeline),
+            "law_enforcement": check_law_enforcement_pattern(timeline),
+            "new_actor": check_new_actor_pattern(entity_value, entity_type),
+            "anomalies": detect_anomalies(timeline),
+            "silence_breaks": detect_silence_breaks(timeline),
+        }
+    except Exception as exc:
+        logger.debug("run_all_patterns: error (%s)", exc)
+        return {
+            "exit_scam": {"risk": "low", "confidence": 0.0, "reason": "Error"},
+            "law_enforcement": {"risk": "low", "confidence": 0.0, "reason": "Error"},
+            "new_actor": {"is_new": False, "first_seen": None, "days_active": 0},
+            "anomalies": [],
+            "silence_breaks": [],
+        }