voidaccess 1.3.0__py3-none-any.whl

sources/ip_reputation.py

@@ -0,0 +1,521 @@
+"""
+sources/ip_reputation.py — IP reputation enrichment.
+
+Checks extracted IP addresses against four sources:
+  - Feodo Tracker (abuse.ch): confirmed C2 IPs for banking trojans/ransomware loaders
+  - C2IntelFeeds (montysecurity/C2-Tracker): framework-specific C2 IPs
+  - AbuseIPDB: community abuse reports (requires ABUSEIPDB_API_KEY)
+  - GreyNoise: scanner classification (requires GREYNOISE_API_KEY)
+
+GreyNoise "benign" IPs (known legitimate scanners) are SUPPRESSED from results.
+All other sources run without API keys — Feodo and C2IntelFeeds are fully public.
+
+Public interface
+----------------
+async load_feodo_feed()                    → dict[ip, malware_family]
+async load_c2_feeds()                      → dict[framework, set[ip]]
+async check_ip_reputation(ip, base_conf)   → dict with suppress/tags/threat_confidence
+async enrich_ip_entities(extraction_results, investigation_id) → (results, stats)
+"""
+
+from __future__ import annotations
+
+import asyncio
+import csv
+import ipaddress
+import json
+import logging
+import os
+import time
+from typing import Any
+
+import aiohttp
+
+logger = logging.getLogger(__name__)
+
+MAX_IPS = 50
+
+FEODO_CSV_URL = "https://feodotracker.abuse.ch/downloads/ipblocklist.csv"
+
+C2_FEED_URLS: dict[str, str] = {
+    "cobalt_strike": "https://raw.githubusercontent.com/montysecurity/C2-Tracker/main/data/Cobalt%20Strike%20C2%20IPs.txt",
+    "sliver":        "https://raw.githubusercontent.com/montysecurity/C2-Tracker/main/data/Sliver%20C2%20IPs.txt",
+    "metasploit":    "https://raw.githubusercontent.com/montysecurity/C2-Tracker/main/data/Metasploit%20Framework%20C2%20IPs.txt",
+    "brute_ratel":   "https://raw.githubusercontent.com/montysecurity/C2-Tracker/main/data/Brute%20Ratel%20C4%20IPs.txt",
+    "posh_c2":       "https://raw.githubusercontent.com/montysecurity/C2-Tracker/main/data/Posh%20C2%20IPs.txt",
+    "havoc":         "https://raw.githubusercontent.com/montysecurity/C2-Tracker/main/data/Havoc%20C2%20IPs.txt",
+}
+
+# In-memory feed caches (module-level singletons, refreshed on TTL expiry)
+_feed_cache: dict[str, dict] = {
+    "feodo":   {"ips": {}, "loaded_at": 0.0},
+    "c2feeds": {"ips": {}, "loaded_at": 0.0},
+}
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _feed_ttl_seconds() -> float:
+    try:
+        hours = float(os.getenv("C2_FEED_CACHE_TTL", "24"))
+    except ValueError:
+        hours = 24.0
+    return hours * 3600.0
+
+
+def is_private_ip(ip: str) -> bool:
+    """Return True if *ip* is private, loopback, link-local, or reserved."""
+    try:
+        addr = ipaddress.ip_address(ip.strip())
+        return (
+            addr.is_private
+            or addr.is_loopback
+            or addr.is_reserved
+            or addr.is_link_local
+            or addr.is_multicast
+            or addr.is_unspecified
+        )
+    except ValueError:
+        return False
+
+
+def _parse_feodo_csv(csv_text: str) -> dict[str, str]:
+    """Parse Feodo Tracker ipblocklist.csv → {ip: malware_family}."""
+    result: dict[str, str] = {}
+    # Strip comment lines; the first non-comment line is the CSV header
+    lines = [
+        line for line in csv_text.splitlines()
+        if line.strip() and not line.strip().startswith("#")
+    ]
+    if not lines:
+        return result
+    try:
+        reader = csv.DictReader(lines)
+        for row in reader:
+            ip = (row.get("dst_ip") or row.get("ip_address") or "").strip()
+            malware = (row.get("malware") or row.get("malware_family") or "").strip()
+            if ip:
+                result[ip] = malware or "unknown"
+    except Exception as exc:
+        logger.warning("ip_reputation: Feodo CSV parse error: %s", exc)
+    return result
+
+
+def _parse_c2_txt(text: str) -> set[str]:
+    """Parse a plain-text C2 IP list (one entry per line, optional comments)."""
+    ips: set[str] = set()
+    for line in text.splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        # Strip port suffix (1.2.3.4:8080 → 1.2.3.4) and CIDR (/32)
+        ip = line.split(":")[0].split("/")[0].strip()
+        if ip:
+            ips.add(ip)
+    return ips
+
+
+# ---------------------------------------------------------------------------
+# Feed loaders (cached, refreshed on TTL expiry)
+# ---------------------------------------------------------------------------
+
+async def load_feodo_feed() -> dict[str, str]:
+    """Fetch and cache Feodo Tracker blocklist. Returns {ip: malware_family}."""
+    cache = _feed_cache["feodo"]
+    if time.time() - cache["loaded_at"] < _feed_ttl_seconds() and cache["ips"]:
+        return cache["ips"]  # type: ignore[return-value]
+
+    logger.info("ip_reputation: Refreshing Feodo Tracker feed")
+    try:
+        timeout = aiohttp.ClientTimeout(total=30)
+        async with aiohttp.ClientSession(timeout=timeout) as session:
+            async with session.get(FEODO_CSV_URL) as resp:
+                if resp.status != 200:
+                    logger.warning("ip_reputation: Feodo returned HTTP %s", resp.status)
+                    return dict(cache["ips"])
+                text = await resp.text()
+
+        parsed = _parse_feodo_csv(text)
+        cache["ips"] = parsed
+        cache["loaded_at"] = time.time()
+        logger.info("ip_reputation: Feodo Tracker: %d C2 IPs loaded", len(parsed))
+        return parsed
+    except Exception as exc:
+        logger.warning("ip_reputation: Feodo fetch failed: %s", exc)
+        return dict(cache["ips"])
+
+
+async def load_c2_feeds() -> dict[str, set[str]]:
+    """Fetch and cache all C2IntelFeeds. Returns {framework: set_of_ips}."""
+    cache = _feed_cache["c2feeds"]
+    if time.time() - cache["loaded_at"] < _feed_ttl_seconds() and cache["ips"]:
+        return cache["ips"]  # type: ignore[return-value]
+
+    logger.info("ip_reputation: Refreshing C2IntelFeeds")
+
+    async def _fetch_one(framework: str, url: str) -> tuple[str, set[str]]:
+        try:
+            timeout = aiohttp.ClientTimeout(total=20)
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                async with session.get(url) as resp:
+                    if resp.status != 200:
+                        logger.debug("ip_reputation: C2Feed %s → HTTP %s", framework, resp.status)
+                        return framework, set()
+                    text = await resp.text()
+            return framework, _parse_c2_txt(text)
+        except Exception as exc:
+            logger.debug("ip_reputation: C2Feed %s fetch failed: %s", framework, exc)
+            return framework, set()
+
+    fetched = await asyncio.gather(*[_fetch_one(fw, url) for fw, url in C2_FEED_URLS.items()])
+
+    results: dict[str, set[str]] = {}
+    for framework, ips in fetched:
+        results[framework] = ips
+        logger.info("ip_reputation: C2Feed %-14s %d IPs", framework, len(ips))
+
+    cache["ips"] = results
+    cache["loaded_at"] = time.time()
+    return results
+
+
+# ---------------------------------------------------------------------------
+# External API checks
+# ---------------------------------------------------------------------------
+
+async def _check_abuseipdb(ip: str, api_key: str) -> dict:
+    """Query AbuseIPDB v2 /check. Returns parsed response or {}."""
+    try:
+        headers = {"Key": api_key, "Accept": "application/json"}
+        params = {"ipAddress": ip, "maxAgeInDays": 90}
+        timeout = aiohttp.ClientTimeout(total=15)
+        async with aiohttp.ClientSession(timeout=timeout) as session:
+            async with session.get(
+                "https://api.abuseipdb.com/api/v2/check",
+                headers=headers,
+                params=params,
+            ) as resp:
+                if resp.status != 200:
+                    logger.debug("ip_reputation: AbuseIPDB → HTTP %s for %s", resp.status, ip)
+                    return {}
+                return await resp.json()
+    except Exception as exc:
+        logger.debug("ip_reputation: AbuseIPDB check failed for %s: %s", ip, exc)
+        return {}
+
+
+async def _check_greynoise(ip: str, api_key: str) -> dict:
+    """Query GreyNoise community API. Returns parsed response or {}."""
+    try:
+        headers = {"key": api_key}
+        timeout = aiohttp.ClientTimeout(total=15)
+        async with aiohttp.ClientSession(timeout=timeout) as session:
+            async with session.get(
+                f"https://api.greynoise.io/v3/community/{ip}",
+                headers=headers,
+            ) as resp:
+                if resp.status == 404:
+                    return {"classification": "unknown"}
+                if resp.status != 200:
+                    logger.debug("ip_reputation: GreyNoise → HTTP %s for %s", resp.status, ip)
+                    return {}
+                return await resp.json()
+    except Exception as exc:
+        logger.debug("ip_reputation: GreyNoise check failed for %s: %s", ip, exc)
+        return {}
+
+
+# ---------------------------------------------------------------------------
+# Core reputation check
+# ---------------------------------------------------------------------------
+
+async def check_ip_reputation(
+    ip: str,
+    base_confidence: float = 1.0,
+) -> dict[str, Any]:
+    """
+    Run all four reputation checks for a single IP address.
+
+    Returns a dict with keys:
+        ip, feodo_hit, feodo_malware, c2feed_hit, c2feed_framework,
+        abuseipdb_score, abuseipdb_categories, greynoise_classification,
+        suppress, tags, threat_confidence
+    """
+    result: dict[str, Any] = {
+        "ip": ip,
+        "feodo_hit": False,
+        "feodo_malware": None,
+        "c2feed_hit": False,
+        "c2feed_framework": None,
+        "abuseipdb_score": None,
+        "abuseipdb_categories": [],
+        "greynoise_classification": None,
+        "suppress": False,
+        "tags": [],
+        "threat_confidence": base_confidence,
+    }
+
+    if is_private_ip(ip):
+        return result
+
+    abuseipdb_key = (os.getenv("ABUSEIPDB_API_KEY") or "").strip()
+    greynoise_key = (os.getenv("GREYNOISE_API_KEY") or "").strip()
+
+    # Load local feeds (both are cached — near-instant after first load)
+    feodo_data, c2feeds_data = await asyncio.gather(
+        load_feodo_feed(),
+        load_c2_feeds(),
+    )
+
+    # --- Feodo Tracker check ---
+    if ip in feodo_data:
+        malware = feodo_data[ip]
+        result["feodo_hit"] = True
+        result["feodo_malware"] = malware
+        result["tags"].append("confirmed_c2")
+        if malware and malware.lower() != "unknown":
+            slug = malware.lower().replace(" ", "_").replace("-", "_")
+            result["tags"].append(f"confirmed_c2_{slug}")
+
+    # --- C2IntelFeeds check ---
+    for framework, ips in c2feeds_data.items():
+        if ip in ips:
+            result["c2feed_hit"] = True
+            result["c2feed_framework"] = framework
+            if "confirmed_c2" not in result["tags"]:
+                result["tags"].append("confirmed_c2")
+            result["tags"].append(f"confirmed_c2_{framework}")
+            break
+
+    # --- AbuseIPDB check ---
+    if abuseipdb_key:
+        abuse_resp = await _check_abuseipdb(ip, abuseipdb_key)
+        if abuse_resp:
+            data = abuse_resp.get("data", {})
+            score = data.get("abuseConfidenceScore")
+            result["abuseipdb_score"] = score
+            # usageType is a string; categories come from individual reports
+            usage = data.get("usageType")
+            result["abuseipdb_categories"] = [usage] if usage else []
+            if score is not None and score > 50:
+                result["tags"].append("abuse_confirmed")
+    else:
+        logger.debug("ip_reputation: AbuseIPDB skipped — no API key")
+
+    # --- GreyNoise check ---
+    if greynoise_key:
+        gn_resp = await _check_greynoise(ip, greynoise_key)
+        if gn_resp:
+            classification = gn_resp.get("classification", "unknown")
+            result["greynoise_classification"] = classification
+
+            if classification == "benign":
+                result["suppress"] = True
+                logger.info("IP %s suppressed — GreyNoise benign scanner", ip)
+                return result
+
+            if classification == "malicious":
+                result["tags"].append("greynoise_malicious")
+                for gn_tag in gn_resp.get("tags") or []:
+                    slug = str(gn_tag).lower().replace(" ", "_")
+                    result["tags"].append(f"greynoise_{slug}")
+    else:
+        logger.debug("ip_reputation: GreyNoise skipped — no API key")
+
+    # --- Threat confidence calculation ---
+    conf = base_confidence
+    if result["feodo_hit"]:
+        conf += 0.15
+    if result["c2feed_hit"]:
+        conf += 0.15
+    score = result["abuseipdb_score"]
+    if score is not None:
+        if score > 80:
+            conf += 0.10
+        elif score >= 50:
+            conf += 0.05
+    if "greynoise_malicious" in result["tags"]:
+        conf += 0.10
+    result["threat_confidence"] = min(conf, 1.0)
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# DB helpers (sync — called via asyncio.to_thread or direct from sync context)
+# ---------------------------------------------------------------------------
+
+def _suppress_entities_in_db(suppressed_ips: set[str], investigation_id: Any) -> None:
+    """Remove suppressed IPs from an investigation's entity pool in the DB."""
+    if not os.getenv("DATABASE_URL") or not suppressed_ips:
+        return
+    try:
+        from db.session import get_session
+        from db.models import Entity, InvestigationEntityLink
+
+        with get_session() as session:
+            entity_ids = [
+                row[0]
+                for row in session.query(Entity.id).filter(
+                    Entity.entity_type == "IP_ADDRESS",
+                    Entity.value.in_(suppressed_ips),
+                ).all()
+            ]
+            if not entity_ids:
+                return
+
+            session.query(InvestigationEntityLink).filter(
+                InvestigationEntityLink.investigation_id == investigation_id,
+                InvestigationEntityLink.entity_id.in_(entity_ids),
+            ).delete(synchronize_session=False)
+
+            session.query(Entity).filter(
+                Entity.investigation_id == investigation_id,
+                Entity.id.in_(entity_ids),
+            ).update({"investigation_id": None}, synchronize_session=False)
+
+            session.commit()
+            logger.info(
+                "ip_reputation: Suppressed %d IP(s) from investigation %s",
+                len(entity_ids),
+                investigation_id,
+            )
+    except Exception as exc:
+        logger.warning("ip_reputation: DB suppression failed: %s", exc)
+
+
+def _update_entity_reputations(
+    updates: list[tuple[str, float, list[str]]],
+) -> None:
+    """
+    Update confidence and corroborating_sources for non-suppressed IP entities.
+
+    *updates* is a list of (ip_value, new_confidence, tags).
+    """
+    if not os.getenv("DATABASE_URL") or not updates:
+        return
+    try:
+        from db.session import get_session
+        from db.models import Entity
+
+        with get_session() as session:
+            for ip_val, confidence, tags in updates:
+                db_entity = session.query(Entity).filter(
+                    Entity.entity_type == "IP_ADDRESS",
+                    Entity.value == ip_val,
+                ).first()
+                if db_entity is None:
+                    continue
+                if confidence > (db_entity.confidence or 0.0):
+                    db_entity.confidence = confidence
+                if tags:
+                    existing: list = json.loads(db_entity.corroborating_sources or "[]")
+                    for tag in tags:
+                        if tag not in existing:
+                            existing.append(tag)
+                    db_entity.corroborating_sources = json.dumps(existing)
+            session.commit()
+    except Exception as exc:
+        logger.warning("ip_reputation: DB reputation update failed: %s", exc)
+
+
+# ---------------------------------------------------------------------------
+# Pipeline integration
+# ---------------------------------------------------------------------------
+
+async def enrich_ip_entities(
+    extraction_results: list,
+    investigation_id: Any,
+) -> tuple[list, dict]:
+    """
+    Post-extraction IP reputation enrichment step.
+
+    - Collects IP_ADDRESS entities from *extraction_results*
+    - Limits to MAX_IPS unique IPs per investigation
+    - Runs all four checks concurrently
+    - Suppresses benign scanner IPs (GreyNoise benign): removes from results + DB
+    - Updates confidence and corroborating_sources for remaining IPs
+
+    Returns (filtered_extraction_results, stats_dict).
+    """
+    # Collect unique IPs → (first_entity, confidence)
+    seen: dict[str, float] = {}
+    for exr in extraction_results:
+        for entity in getattr(exr, "entities", []):
+            if getattr(entity, "entity_type", "") == "IP_ADDRESS":
+                ip = entity.value
+                if ip not in seen:
+                    seen[ip] = getattr(entity, "confidence", 1.0)
+
+    unique_ips = list(seen.keys())
+    if not unique_ips:
+        return extraction_results, {"ip_reputation": "ok_0_ips"}
+
+    if len(unique_ips) > MAX_IPS:
+        logger.info(
+            "ip_reputation: capping to %d of %d unique IPs",
+            MAX_IPS, len(unique_ips),
+        )
+        unique_ips = unique_ips[:MAX_IPS]
+
+    logger.info("ip_reputation: checking %d unique IP(s)", len(unique_ips))
+
+    # Run all checks concurrently
+    rep_list = await asyncio.gather(
+        *[check_ip_reputation(ip, base_confidence=seen[ip]) for ip in unique_ips],
+        return_exceptions=True,
+    )
+
+    suppressed_ips: set[str] = set()
+    db_updates: list[tuple[str, float, list[str]]] = []
+    stats = {
+        "checked": len(unique_ips),
+        "suppressed": 0,
+        "c2_confirmed": 0,
+        "abuse_confirmed": 0,
+    }
+
+    for ip, rep in zip(unique_ips, rep_list):
+        if isinstance(rep, Exception):
+            logger.debug("ip_reputation: check raised for %s: %s", ip, rep)
+            continue
+        if rep["suppress"]:
+            suppressed_ips.add(ip)
+            stats["suppressed"] += 1
+            continue
+        if rep["c2feed_hit"] or rep["feodo_hit"]:
+            stats["c2_confirmed"] += 1
+        if (rep["abuseipdb_score"] or 0) > 50:
+            stats["abuse_confirmed"] += 1
+        if rep["tags"] or rep["threat_confidence"] > seen[ip]:
+            db_updates.append((ip, rep["threat_confidence"], rep["tags"]))
+
+    # Apply suppression to in-memory extraction results
+    if suppressed_ips:
+        for exr in extraction_results:
+            exr.entities = [
+                e for e in getattr(exr, "entities", [])
+                if not (
+                    getattr(e, "entity_type", "") == "IP_ADDRESS"
+                    and e.value in suppressed_ips
+                )
+            ]
+            exr.entity_count = len(exr.entities)
+        await asyncio.to_thread(_suppress_entities_in_db, suppressed_ips, investigation_id)
+
+    # Update DB for non-suppressed IPs
+    if db_updates:
+        await asyncio.to_thread(_update_entity_reputations, db_updates)
+
+    checked = stats["checked"]
+    sup = stats["suppressed"]
+    status = f"ok_{checked}_ips" + (f"_{sup}_suppressed" if sup else "")
+
+    logger.info(
+        "ip_reputation: done — %d checked, %d suppressed, %d C2, %d abuse",
+        checked, sup, stats["c2_confirmed"], stats["abuse_confirmed"],
+    )
+
+    return extraction_results, {"ip_reputation": status, **stats}