voidaccess 1.3.0__py3-none-any.whl

sources/blockchain.py

@@ -0,0 +1,444 @@
+"""
+Blockchain transaction lookup for extracted wallet addresses.
+Queries free APIs to get transaction history and connected addresses.
+
+Supports: Bitcoin (BlockCypher), Ethereum (Etherscan)
+Monero: privacy coin, no public lookup possible
+"""
+
+import asyncio
+import aiohttp
+import logging
+import uuid
+from typing import Optional, List, Dict, Any
+from datetime import datetime, timezone
+
+from db.models import Entity, EntityRelationship, RelationshipType
+from db.queries import upsert_entity_canonical
+
+logger = logging.getLogger(__name__)
+
+BLOCKCYPHER_BASE = "https://api.blockcypher.com/v1/btc/main"
+ETHERSCAN_BASE = "https://api.etherscan.io/api"
+
+# Reasonable caps to avoid hammering free APIs
+MAX_TRANSACTIONS_PER_WALLET = 50
+MAX_CONNECTED_ADDRESSES = 10  # How many counterparty addresses to extract
+
+# Entity type constants to match extractor/regex_patterns.py
+BITCOIN_ADDRESS = "BITCOIN_ADDRESS"
+ETHEREUM_ADDRESS = "ETHEREUM_ADDRESS"
+MONERO_ADDRESS = "MONERO_ADDRESS"
+
+
+def detect_wallet_type(address: str) -> Optional[str]:
+    """
+    Detect cryptocurrency type from address format.
+    
+    Returns: BITCOIN_ADDRESS, ETHEREUM_ADDRESS, MONERO_ADDRESS, or None
+    """
+    address = address.strip()
+    
+    # Bitcoin: starts with 1, 3, or bc1
+    if address.startswith(("1", "3")) and 25 <= len(address) <= 34:
+        return BITCOIN_ADDRESS
+    if address.startswith("bc1") and len(address) >= 42:
+        return BITCOIN_ADDRESS
+    
+    # Ethereum: 0x prefix, 42 chars total
+    if address.startswith("0x") and len(address) == 42:
+        return ETHEREUM_ADDRESS
+    
+    # Monero: starts with 4, 95-106 chars
+    if address.startswith("4") and 95 <= len(address) <= 106:
+        return MONERO_ADDRESS
+    
+    return None
+
+
+async def lookup_bitcoin_address(
+    address: str,
+    api_token: str = "",
+) -> dict:
+    """
+    Look up Bitcoin address via BlockCypher API.
+    
+    Returns dict with financial metadata and connected addresses.
+    """
+    result = {
+        "address": address,
+        "wallet_type": BITCOIN_ADDRESS,
+        "total_received_btc": 0.0,
+        "total_sent_btc": 0.0,
+        "balance_btc": 0.0,
+        "transaction_count": 0,
+        "first_seen": None,
+        "last_seen": None,
+        "connected_addresses": [],
+        "lookup_successful": False,
+        "error": None,
+    }
+    
+    params = {"limit": MAX_TRANSACTIONS_PER_WALLET}
+    if api_token:
+        params["token"] = api_token
+    
+    try:
+        connector = aiohttp.TCPConnector(ssl=True)
+        timeout = aiohttp.ClientTimeout(total=15)
+        async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
+            url = f"{BLOCKCYPHER_BASE}/addrs/{address}"
+            async with session.get(url, params=params) as resp:
+                
+                if resp.status == 429:
+                    logger.warning(f"BlockCypher rate limited for {address[:12]}...")
+                    result["error"] = "rate_limited"
+                    return result
+                
+                if resp.status == 404:
+                    # Valid address with no transactions
+                    result["lookup_successful"] = True
+                    result["error"] = "no_transactions"
+                    return result
+                
+                if resp.status != 200:
+                    result["error"] = f"http_{resp.status}"
+                    return result
+                
+                data = await resp.json()
+                
+                # Satoshis to BTC conversion
+                sat_to_btc = 1 / 100_000_000
+                
+                result["total_received_btc"] = data.get("total_received", 0) * sat_to_btc
+                result["total_sent_btc"] = data.get("total_sent", 0) * sat_to_btc
+                result["balance_btc"] = data.get("final_balance", 0) * sat_to_btc
+                result["transaction_count"] = data.get("n_tx", 0)
+                result["lookup_successful"] = True
+                
+                # Extract connected addresses from transaction refs
+                txrefs = data.get("txrefs", []) + data.get("unconfirmed_txrefs", [])
+                
+                # Sort by time to get first/last seen
+                confirmed_txs = [t for t in txrefs if t.get("confirmed")]
+                if confirmed_txs:
+                    times = [t.get("confirmed") for t in confirmed_txs if t.get("confirmed")]
+                    if times:
+                        result["first_seen"] = min(times)
+                        result["last_seen"] = max(times)
+                
+                # Extract counterparty addresses from inputs/outputs
+                # BlockCypher address endpoint gives txrefs but not full tx details
+                # We mark incoming as FUNDED_BY, outgoing as PAID_TO
+                connected = {}
+                for tx in txrefs[:MAX_TRANSACTIONS_PER_WALLET]:
+                    value_btc = tx.get("value", 0) * sat_to_btc
+                    tx_hash = tx.get("tx_hash", "")
+                    
+                    # tx_input_n >= 0 means this address was an input (sending)
+                    if tx.get("tx_input_n", -1) >= 0:
+                        direction = "sent"  # We sent to someone
+                    else:
+                        direction = "received"  # Someone sent to us
+                    
+                    # Note: Getting actual counterparty addresses requires
+                    # fetching the full transaction. For free tier, we'll simple-link
+                    if value_btc > 0.001 and tx_hash:  # Only significant transactions
+                        connected[tx_hash] = {
+                            "direction": direction,
+                            "amount": round(value_btc, 8),
+                            "tx_hash": tx_hash,
+                            "confirmed": tx.get("confirmed", ""),
+                        }
+                
+                result["connected_addresses"] = list(connected.values())[:MAX_CONNECTED_ADDRESSES]
+    
+    except asyncio.TimeoutError:
+        result["error"] = "timeout"
+        logger.warning(f"BlockCypher timeout for {address[:12]}...")
+    except Exception as e:
+        result["error"] = str(e)[:100]
+        logger.warning(f"BlockCypher lookup failed for {address[:12]}...: {e}")
+    
+    return result
+
+
+async def lookup_ethereum_address(
+    address: str,
+    api_key: str = "",
+) -> dict:
+    """
+    Look up Ethereum address via Etherscan API.
+    """
+    result = {
+        "address": address,
+        "wallet_type": ETHEREUM_ADDRESS,
+        "balance_eth": 0.0,
+        "transaction_count": 0,
+        "first_seen": None,
+        "last_seen": None,
+        "connected_addresses": [],
+        "lookup_successful": False,
+        "error": None,
+    }
+    
+    if not api_key:
+        result["error"] = "no_api_key"
+        return result
+    
+    try:
+        connector = aiohttp.TCPConnector(ssl=True)
+        timeout = aiohttp.ClientTimeout(total=15)
+        async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
+            
+            # Get balance
+            async with session.get(
+                ETHERSCAN_BASE,
+                params={
+                    "module": "account",
+                    "action": "balance",
+                    "address": address,
+                    "tag": "latest",
+                    "apikey": api_key,
+                },
+            ) as resp:
+                if resp.status == 200:
+                    data = await resp.json()
+                    if data.get("status") == "1":
+                        wei_to_eth = 1 / 10**18
+                        result["balance_eth"] = int(data.get("result", 0)) * wei_to_eth
+            
+            # Get transactions
+            async with session.get(
+                ETHERSCAN_BASE,
+                params={
+                    "module": "account",
+                    "action": "txlist",
+                    "address": address,
+                    "startblock": 0,
+                    "endblock": 99999999,
+                    "page": 1,
+                    "offset": MAX_TRANSACTIONS_PER_WALLET,
+                    "sort": "desc",
+                    "apikey": api_key,
+                },
+            ) as resp:
+                if resp.status == 200:
+                    data = await resp.json()
+                    if data.get("status") == "1":
+                        txs = data.get("result", [])
+                        result["transaction_count"] = len(txs)
+                        result["lookup_successful"] = True
+                        
+                        wei_to_eth = 1 / 10**18
+                        connected = []
+                        
+                        for tx in txs:
+                            is_incoming = tx.get("to", "").lower() == address.lower()
+                            counterparty = tx.get("from") if is_incoming else tx.get("to")
+                            value_eth = int(tx.get("value", 0)) * wei_to_eth
+                            
+                            if counterparty and counterparty.lower() != address.lower() and value_eth > 0.001:
+                                connected.append({
+                                    "address": counterparty,
+                                    "direction": "received" if is_incoming else "sent",
+                                    "amount": round(value_eth, 6),
+                                    "tx_hash": tx.get("hash", ""),
+                                    "confirmed": tx.get("timeStamp", ""),
+                                })
+                        
+                        result["connected_addresses"] = connected[:MAX_CONNECTED_ADDRESSES]
+                        
+                        if txs:
+                            # Sorted desc, so first is newest
+                            result["last_seen"] = txs[0].get("timeStamp", "")
+                            result["first_seen"] = txs[-1].get("timeStamp", "")
+    
+    except asyncio.TimeoutError:
+        result["error"] = "timeout"
+    except Exception as e:
+        result["error"] = str(e)[:100]
+        logger.warning(f"Etherscan lookup failed for {address[:12]}...: {e}")
+    
+    return result
+
+
+async def lookup_wallet(
+    address: str,
+    blockcypher_token: str = "",
+    etherscan_key: str = "",
+) -> dict:
+    """
+    Unified wallet lookup. Detects type and routes to correct API.
+    """
+    wallet_type = detect_wallet_type(address)
+    
+    if wallet_type == BITCOIN_ADDRESS:
+        return await lookup_bitcoin_address(address, blockcypher_token)
+    elif wallet_type == ETHEREUM_ADDRESS:
+        return await lookup_ethereum_address(address, etherscan_key)
+    elif wallet_type == MONERO_ADDRESS:
+        return {
+            "address": address,
+            "wallet_type": MONERO_ADDRESS,
+            "lookup_successful": False,
+            "error": "monero_privacy_coin",
+            "note": "Monero transactions are private and cannot be looked up without view key",
+        }
+    else:
+        return {
+            "address": address,
+            "wallet_type": "unknown",
+            "lookup_successful": False,
+            "error": "unrecognized_format",
+        }
+
+
+async def enrich_wallets_for_investigation(
+    investigation_id: uuid.UUID,
+    session: Any,
+    blockcypher_token: str = "",
+    etherscan_key: str = "",
+    max_wallets: int = 10,
+) -> dict:
+    """
+    For all crypto wallet entities in an investigation:
+    1. Look up transaction data from blockchain APIs
+    2. Store transaction metadata on the entity
+    3. Create connected address entities and PAID_TO/FUNDED_BY relationships
+    """
+    stats = {
+        "wallets_looked_up": 0,
+        "successful_lookups": 0,
+        "edges_created": 0,
+        "connected_wallets_found": 0,
+        "errors": 0,
+    }
+    
+    # Get all wallet entities for this investigation
+    wallets = (
+        session.query(Entity)
+        .filter(
+            Entity.investigation_id == investigation_id,
+            Entity.entity_type.in_([BITCOIN_ADDRESS, ETHEREUM_ADDRESS, MONERO_ADDRESS]),
+            Entity.value.isnot(None),
+        )
+        .limit(max_wallets)
+        .all()
+    )
+    
+    if not wallets:
+        return stats
+    
+    logger.warning(f"Blockchain enrichment: {len(wallets)} wallets to process")
+    
+    for wallet_entity in wallets:
+        address = wallet_entity.value.strip()
+        stats["wallets_looked_up"] += 1
+        
+        try:
+            lookup_result = await lookup_wallet(
+                address=address,
+                blockcypher_token=blockcypher_token,
+                etherscan_key=etherscan_key,
+            )
+            
+            if not lookup_result.get("lookup_successful"):
+                if lookup_result.get("error") != "monero_privacy_coin":
+                    stats["errors"] += 1
+                continue
+            
+            stats["successful_lookups"] += 1
+            
+            # Update entity historical_context with financial summary
+            wallet_type = lookup_result.get("wallet_type", "")
+            tx_count = lookup_result.get("transaction_count", 0)
+            
+            if wallet_type == BITCOIN_ADDRESS:
+                balance = lookup_result.get("balance_btc", 0)
+                summary = f"BTC Balance: {balance:.4f} BTC, Transactions: {tx_count}"
+            else:
+                balance = lookup_result.get("balance_eth", 0)
+                summary = f"ETH Balance: {balance:.4f} ETH, Transactions: {tx_count}"
+            
+            if not wallet_entity.historical_context:
+                wallet_entity.historical_context = summary
+            
+            # Update first_seen if available
+            first_seen_val = lookup_result.get("first_seen")
+            if first_seen_val and not wallet_entity.first_seen:
+                try:
+                    if isinstance(first_seen_val, int) or (isinstance(first_seen_val, str) and first_seen_val.isdigit()):
+                        wallet_entity.first_seen = datetime.fromtimestamp(int(first_seen_val), tz=timezone.utc)
+                    elif isinstance(first_seen_val, str):
+                        # BlockCypher ISO format
+                        wallet_entity.first_seen = datetime.fromisoformat(first_seen_val.replace("Z", "+00:00"))
+                except Exception:
+                    pass
+            
+            # Process connected addresses
+            connected = lookup_result.get("connected_addresses", [])
+            for conn in connected:
+                conn_address = conn.get("address")
+                if not conn_address or conn_address.lower() == address.lower():
+                    continue
+                
+                # Detect type for counterparty
+                conn_type = detect_wallet_type(conn_address) or wallet_type
+                
+                # Create counterparty entity
+                conn_entity, _ = upsert_entity_canonical(
+                    session=session,
+                    investigation_id=investigation_id,
+                    entity_type=conn_type,
+                    entity_value=conn_address,
+                    confidence=0.95,
+                    context_snippet=f"Related to {address[:12]} via blockchain transaction",
+                    extraction_method="blockchain_api",
+                )
+                stats["connected_wallets_found"] += 1
+                
+                # Build Relationship
+                direction = conn.get("direction", "sent")
+                if direction == "received":
+                    # conn -> us (FUNDED_BY) or conn PAID_TO us
+                    source_id = conn_entity.id
+                    target_id = wallet_entity.id
+                else:
+                    # us -> conn (PAID_TO)
+                    source_id = wallet_entity.id
+                    target_id = conn_entity.id
+                
+                # Check duplication
+                existing = session.query(EntityRelationship).filter_by(
+                    entity_a_id=source_id,
+                    entity_b_id=target_id,
+                    relationship_type=RelationshipType.PAID_TO.value
+                ).first()
+                
+                if not existing:
+                    rel = EntityRelationship(
+                        entity_a_id=source_id,
+                        entity_b_id=target_id,
+                        relationship_type=RelationshipType.PAID_TO.value,
+                        confidence=0.95,
+                        metadata_json={
+                            "amount": conn.get("amount"),
+                            "currency": "BTC" if wallet_type == BITCOIN_ADDRESS else "ETH",
+                            "tx_hash": conn.get("tx_hash"),
+                        } if hasattr(EntityRelationship, "metadata_json") else None
+                    )
+                    session.add(rel)
+                    stats["edges_created"] += 1
+            
+            session.flush()
+            
+        except Exception as e:
+            stats["errors"] += 1
+            logger.warning(f"Wallet enrichment failed for {address[:12]}: {e}")
+        
+        # Respect rate limits
+        await asyncio.sleep(0.4)
+        
+    session.commit()
+    return stats

sources/cache.py

@@ -0,0 +1,93 @@
+"""
+sources/cache.py — Simple file-based TTL cache for external feeds.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import time
+from pathlib import Path
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+_memory_cache: dict[str, tuple[Any, float]] = {}
+
+
+class CachedFeed:
+    """
+    Fetch a remote JSON feed and cache it to *cache_path* for *ttl_seconds*.
+
+    On every call to :meth:`fetch`:
+      - If a fresh cache file exists (mtime < ttl_seconds), return cached data.
+      - If stale or missing: fetch from *url*, save, and return.
+      - If fetch fails but stale cache exists: return stale cache with a warning.
+      - If fetch fails and no cache exists: log error and return None.
+    """
+
+    def __init__(self, url: str, cache_path: str, ttl_seconds: int):
+        self.url = url
+        self.cache_path = Path(cache_path)
+        self.ttl_seconds = ttl_seconds
+
+    def _is_fresh(self) -> bool:
+        if not self.cache_path.exists():
+            return False
+        mtime = self.cache_path.stat().st_mtime
+        return (time.time() - mtime) < self.ttl_seconds
+
+    async def fetch(self) -> Optional[dict | list]:
+        import aiohttp
+
+        now = time.time()
+        cache_key = self.url
+
+        if cache_key in _memory_cache:
+            cached_data, timestamp = _memory_cache[cache_key]
+            if (now - timestamp) < self.ttl_seconds:
+                return cached_data
+
+        if self._is_fresh():
+            try:
+                with self.cache_path.open("r", encoding="utf-8") as f:
+                    data = json.load(f)
+                    _memory_cache[cache_key] = (data, now)
+                    return data
+            except Exception:
+                pass
+
+        try:
+            timeout = aiohttp.ClientTimeout(total=30)
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                async with session.get(self.url) as resp:
+                    if resp.status != 200:
+                        logger.warning("CachedFeed: HTTP %s for %s", resp.status, self.url)
+                        stale = self._stale_cache()
+                        if stale is not None and cache_key in _memory_cache:
+                            return stale
+                        return stale
+                    data = await resp.json(content_type=None)
+                    self.cache_path.parent.mkdir(parents=True, exist_ok=True)
+                    with self.cache_path.open("w", encoding="utf-8") as f:
+                        json.dump(data, f)
+                    _memory_cache[cache_key] = (data, now)
+                    return data
+        except Exception as e:
+            logger.warning("CachedFeed: fetch failed for %s: %s", self.url, e)
+            if cache_key in _memory_cache:
+                cached_data, timestamp = _memory_cache[cache_key]
+                logger.warning("CachedFeed: returning stale memory cache for %s", self.url)
+                return cached_data
+            return self._stale_cache()
+
+    def _stale_cache(self) -> Optional[dict | list]:
+        if self.cache_path.exists():
+            try:
+                with self.cache_path.open("r", encoding="utf-8") as f:
+                    logger.warning("CachedFeed: falling back to stale cache %s", self.cache_path)
+                    return json.load(f)
+            except Exception:
+                pass
+        return None

sources/cisa.py

@@ -0,0 +1,108 @@
+"""
+sources/cisa.py — CISA KEV catalog + CISA advisories feed enrichment.
+
+Fetches two CISA feeds (clearnet, not through Tor):
+  1. Known Exploited Vulnerabilities (KEV) catalog — 24-hour TTL cache
+  2. Cybersecurity advisories index — 6-hour TTL cache
+
+For CVE entities: checks if they appear in the KEV catalog and marks them
+as actively exploited.
+For THREAT_ACTOR / RANSOMWARE_GROUP / MALWARE_FAMILY entities: searches
+advisory titles and tags for name matches.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from sources.cache import CachedFeed
+
+logger = logging.getLogger(__name__)
+
+CISA_KEV_URL = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
+CISA_ADVISORIES_URL = "https://www.cisa.gov/cybersecurity-advisories/all.json"
+
+_KEV_CACHE = "/tmp/voidaccess_cisa_kev.json"
+_ADVISORIES_CACHE = "/tmp/voidaccess_cisa_advisories.json"
+
+_kev_feed = CachedFeed(CISA_KEV_URL, _KEV_CACHE, ttl_seconds=86400)
+_adv_feed = CachedFeed(CISA_ADVISORIES_URL, _ADVISORIES_CACHE, ttl_seconds=21600)
+
+
+async def enrich_cisa_cve(cve_id: str) -> list[dict]:
+    """
+    Check if *cve_id* appears in the CISA KEV catalog.
+
+    Returns a list with one EnrichmentResult dict if found, empty list otherwise.
+    """
+    data = await _kev_feed.fetch()
+    if data is None:
+        return []
+
+    kev_list = data if isinstance(data, list) else data.get("vulnerabilities", [])
+    for entry in kev_list:
+        if (entry.get("cveID") or "").upper() == cve_id.upper():
+            return [{
+                "source": "cisa_kev",
+                "entity_type": "CVE_NUMBER",
+                "entity_value": cve_id,
+                "is_actively_exploited": True,
+                "vendor_project": entry.get("vendorProject", ""),
+                "product": entry.get("product", ""),
+                "vulnerability_name": entry.get("vulnerabilityName", ""),
+                "date_added": entry.get("dateAdded", ""),
+                "short_description": entry.get("shortDescription", ""),
+            }]
+    return []
+
+
+async def enrich_cisa_advisories(entity_value: str, entity_type: str) -> list[dict]:
+    """
+    Search CISA advisories for *entity_value* matching THREAT_ACTOR,
+    RANSOMWARE_GROUP, or MALWARE_FAMILY.
+    """
+    if entity_type not in ("THREAT_ACTOR", "RANSOMWARE_GROUP", "MALWARE_FAMILY"):
+        return []
+
+    data = await _adv_feed.fetch()
+    if data is None:
+        return []
+
+    advisories = data if isinstance(data, list) else data.get("items", [])
+    results = []
+    q = entity_value.lower()
+
+    for adv in advisories:
+        title = (adv.get("title") or "").lower()
+        tags = " ".join(adv.get("tags") or []).lower()
+        if q in title or q in tags:
+            results.append({
+                "source": "cisa_advisory",
+                "entity_type": entity_type,
+                "entity_value": entity_value,
+                "advisory_title": adv.get("title", ""),
+                "advisory_url": adv.get("url", ""),
+                "advisory_date": adv.get("datePublished", ""),
+            })
+    return results
+
+
+async def enrich_cisa(query: str, entities: list[dict]) -> list[dict]:
+    """
+    Main entry point for CISA enrichment.
+
+    For each CVE entity, checks KEV.
+    For each THREAT_ACTOR / RANSOMWARE_GROUP / MALWARE_FAMILY entity, searches advisories.
+    """
+    results: list[dict] = []
+    for ent in entities:
+        et = ent.get("type") or ent.get("entity_type", "")
+        ev = ent.get("value") or ent.get("entity_value", "")
+
+        if et == "CVE_NUMBER" and ev:
+            results.extend(await enrich_cisa_cve(ev))
+        elif et in ("THREAT_ACTOR", "RANSOMWARE_GROUP", "MALWARE_FAMILY") and ev:
+            results.extend(await enrich_cisa_advisories(ev, et))
+
+    return results