voidaccess 1.3.0__py3-none-any.whl

graph/export.py

@@ -0,0 +1,225 @@
+"""
+graph/export.py — Export the relationship graph to external formats.
+
+All functions accept an nx.MultiDiGraph as their first argument.
+to_graphml and to_gephi_csv write files; to_json and summary_stats return data.
+
+Public interface
+----------------
+to_graphml(graph, filepath)                          → None
+to_json(graph)                                       → dict
+to_gephi_csv(graph, nodes_path, edges_path)          → None
+summary_stats(graph)                                 → dict
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+import logging
+from collections import defaultdict
+from datetime import datetime
+from typing import Any
+
+import networkx as nx
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _serialize_value(v: Any) -> Any:
+    """Convert a value to a JSON-serializable form."""
+    if isinstance(v, datetime):
+        return v.isoformat()
+    if isinstance(v, (list, dict)):
+        return v  # already JSON-native (contents may recurse in to_json)
+    return v
+
+
+def _node_to_dict(node_id: str, data: dict) -> dict:
+    raw_label = data.get("label") or node_id.split("@")[0]
+    label = raw_label[:50] if len(raw_label) > 50 else raw_label
+    return {
+        "id": node_id,
+        "label": label,
+        "type": data.get("node_type", ""),
+        "confidence": data.get("confidence", 0.0),
+        "first_seen": _serialize_value(data.get("first_seen")),
+        "last_seen": _serialize_value(data.get("last_seen")),
+        "source_urls": data.get("source_urls", []),
+        "metadata": data.get("metadata", {}),
+    }
+
+
+def _edge_to_dict(source: str, target: str, data: dict) -> dict:
+    return {
+        "source": source,
+        "target": target,
+        "type": data.get("edge_type", ""),
+        "confidence": data.get("confidence", 0.0),
+        "source_url": data.get("source_url", ""),
+        "timestamp": _serialize_value(data.get("timestamp")),
+        "metadata": data.get("metadata", {}),
+    }
+
+
+# ---------------------------------------------------------------------------
+# Helpers for GraphML (NetworkX only supports basic scalar types)
+# ---------------------------------------------------------------------------
+
+
+def _flatten_for_graphml(data: dict) -> dict:
+    """
+    Convert node/edge attributes to GraphML-compatible scalars.
+
+    NetworkX's GraphML writer supports str, int, float, bool — not datetime,
+    list, or dict.  This function converts everything else to its JSON or
+    ISO-8601 string representation.
+    """
+    flat: dict = {}
+    for key, value in data.items():
+        if isinstance(value, datetime):
+            flat[key] = value.isoformat()
+        elif isinstance(value, (list, dict)):
+            flat[key] = json.dumps(value)
+        elif isinstance(value, (str, int, float, bool)) or value is None:
+            flat[key] = value if value is not None else ""
+        else:
+            flat[key] = str(value)
+    return flat
+
+
+# ---------------------------------------------------------------------------
+# Public functions
+# ---------------------------------------------------------------------------
+
+
+def to_graphml(graph: nx.MultiDiGraph, filepath: str) -> None:
+    """
+    Export *graph* to GraphML format (opens in Gephi, yEd, Cytoscape).
+
+    All node/edge attributes are serialised to GraphML-compatible scalar types
+    before writing.
+    """
+    serialised = nx.MultiDiGraph()
+
+    for node_id, data in graph.nodes(data=True):
+        serialised.add_node(node_id, **_flatten_for_graphml(data))
+
+    for src, tgt, key, data in graph.edges(data=True, keys=True):
+        serialised.add_edge(src, tgt, key=key, **_flatten_for_graphml(data))
+
+    nx.write_graphml(serialised, filepath)
+
+
+def to_json(graph: nx.MultiDiGraph) -> dict:
+    """
+    Return a JSON-serialisable dict representing the graph.
+
+    Schema:
+        {
+            "nodes": [{"id": ..., "type": ..., "first_seen": <ISO-8601>, ...}],
+            "edges": [{"source": ..., "target": ..., "type": ..., ...}],
+        }
+
+    All datetime values are serialised as ISO 8601 strings.
+    """
+    nodes = [
+        _node_to_dict(nid, data)
+        for nid, data in graph.nodes(data=True)
+    ]
+    edges = [
+        _edge_to_dict(src, tgt, data)
+        for src, tgt, data in graph.edges(data=True)
+    ]
+    return {"nodes": nodes, "edges": edges}
+
+
+def to_gephi_csv(
+    graph: nx.MultiDiGraph,
+    nodes_path: str,
+    edges_path: str,
+) -> None:
+    """
+    Export the graph to two CSV files in Gephi node/edge table format.
+
+    nodes.csv columns: Id, Label, Type, FirstSeen, LastSeen
+    edges.csv columns: Source, Target, Type, Confidence, SourceUrl
+    """
+    with open(nodes_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(
+            f, fieldnames=["Id", "Label", "Type", "FirstSeen", "LastSeen"]
+        )
+        writer.writeheader()
+        for node_id, data in graph.nodes(data=True):
+            first_seen = data.get("first_seen")
+            last_seen = data.get("last_seen")
+            writer.writerow({
+                "Id": node_id,
+                "Label": node_id,
+                "Type": data.get("node_type", ""),
+                "FirstSeen": first_seen.isoformat() if isinstance(first_seen, datetime) else str(first_seen or ""),
+                "LastSeen": last_seen.isoformat() if isinstance(last_seen, datetime) else str(last_seen or ""),
+            })
+
+    with open(edges_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(
+            f, fieldnames=["Source", "Target", "Type", "Confidence", "SourceUrl"]
+        )
+        writer.writeheader()
+        for src, tgt, data in graph.edges(data=True):
+            writer.writerow({
+                "Source": src,
+                "Target": tgt,
+                "Type": data.get("edge_type", ""),
+                "Confidence": data.get("confidence", ""),
+                "SourceUrl": data.get("source_url", ""),
+            })
+
+
+def summary_stats(graph: nx.MultiDiGraph) -> dict:
+    """
+    Return aggregate statistics about the graph.
+
+    Schema:
+        {
+            "total_nodes": int,
+            "total_edges": int,
+            "nodes_by_type": {"ThreatActor": N, ...},
+            "edges_by_type": {"CO_APPEARED_ON": N, ...},
+            "most_connected": [{"node_id": ..., "degree": N}],  # top 5
+        }
+    """
+    nodes_by_type: dict[str, int] = defaultdict(int)
+    for _, data in graph.nodes(data=True):
+        ntype = data.get("node_type", "")
+        if ntype:
+            nodes_by_type[ntype] += 1
+
+    edges_by_type: dict[str, int] = defaultdict(int)
+    for _, _, data in graph.edges(data=True):
+        etype = data.get("edge_type", "")
+        if etype:
+            edges_by_type[etype] += 1
+
+    degree_list = sorted(
+        ((nid, graph.degree(nid)) for nid in graph.nodes()),
+        key=lambda t: t[1],
+        reverse=True,
+    )
+    most_connected = [
+        {"node_id": nid, "degree": deg}
+        for nid, deg in degree_list[:5]
+    ]
+
+    return {
+        "total_nodes": graph.number_of_nodes(),
+        "total_edges": graph.number_of_edges(),
+        "nodes_by_type": dict(nodes_by_type),
+        "edges_by_type": dict(edges_by_type),
+        "most_connected": most_connected,
+    }

graph/model.py

@@ -0,0 +1,83 @@
+"""
+graph/model.py — Pure data definitions for the VoidAccess graph layer.
+
+No graph logic here — only node/edge type constants and dataclasses.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+
+# ---------------------------------------------------------------------------
+# Node type constants
+# ---------------------------------------------------------------------------
+
+
+class NODE_TYPES:
+    THREAT_ACTOR = "ThreatActor"
+    CRYPTO_WALLET = "CryptoWallet"
+    ONION_URL = "OnionURL"
+    FORUM = "Forum"
+    MALWARE_FAMILY = "MalwareFamily"
+    RANSOMWARE_GROUP = "RansomwareGroup"
+    PGP_KEY = "PGPKey"
+    EMAIL_ADDRESS = "EmailAddress"
+    CVE = "CVE"
+    PASTE = "Paste"
+    IP_ADDRESS = "IPAddress"
+    PHONE_NUMBER = "PhoneNumber"
+    ORGANIZATION = "Organization"
+    DATE = "Date"
+
+
+# ---------------------------------------------------------------------------
+# Edge type constants
+# ---------------------------------------------------------------------------
+
+
+class EDGE_TYPES:
+    CO_APPEARED_ON = "CO_APPEARED_ON"           # two entities on the same page
+    POSTED_BY = "POSTED_BY"                     # content attributed to a handle
+    LINKED_TO = "LINKED_TO"                     # URL links to URL
+    MEMBER_OF = "MEMBER_OF"                     # handle to group/forum
+    USED = "USED"                               # actor used a malware family
+    CLAIMED = "CLAIMED"                         # group claimed an attack
+    LIKELY_SAME_ACTOR = "LIKELY_SAME_ACTOR"     # inferred, medium confidence
+    CONFIRMED_SAME_ACTOR = "CONFIRMED_SAME_ACTOR"  # PGP key match, high confidence
+    CO_INVESTIGATION = "CO_INVESTIGATION"       # Entities found in same investigation across multiple pages
+    PAID_TO = "PAID_TO"                         # financial transaction
+    FUNDED_BY = "FUNDED_BY"                     # financial transaction
+
+
+
+# ---------------------------------------------------------------------------
+# Dataclasses
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class GraphNode:
+    """Represents a single entity node in the relationship graph."""
+
+    node_id: str                        # canonical value (wallet address, handle, etc.)
+    node_type: str                      # one of NODE_TYPES constants
+    first_seen: datetime
+    last_seen: datetime
+    source_urls: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class GraphEdge:
+    """Represents a directed relationship edge in the graph."""
+
+    source_id: str                      # node_id of the source node
+    target_id: str                      # node_id of the target node
+    edge_type: str                      # one of EDGE_TYPES constants
+    confidence: float                   # 0.0–1.0
+    source_url: str                     # page where the relationship was observed
+    timestamp: datetime
+    metadata: dict[str, Any] = field(default_factory=dict)

graph/queries.py

@@ -0,0 +1,297 @@
+"""
+graph/queries.py — Named query functions that operate on a NetworkX graph.
+
+All functions are pure (no side effects, no DB calls).
+All accept a graph as their first argument and return data.
+None of these functions modify the graph.
+
+Public interface
+----------------
+get_neighbors(graph, node_id, hops, edge_types)  → dict[str, list[GraphNode]]
+find_nodes_by_type(graph, node_type)             → list[GraphNode]
+find_co_occurring_entities(graph, node_id)       → list[tuple[GraphNode, int]]
+get_new_nodes_since(graph, since)                → list[GraphNode]
+find_high_degree_nodes(graph, top_n, node_type)  → list[tuple[GraphNode, int]]
+get_shortest_path(graph, source_id, target_id)   → list[GraphNode] | None
+get_actor_profile(graph, actor_node_id)          → dict
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict, deque
+from datetime import datetime
+from typing import Optional
+
+import networkx as nx
+
+from graph.model import EDGE_TYPES, NODE_TYPES, GraphNode
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_graphnode(node_id: str, data: dict) -> GraphNode:
+    """Reconstruct a GraphNode from raw NetworkX node attribute dict."""
+    return GraphNode(
+        node_id=node_id,
+        node_type=data.get("node_type", ""),
+        first_seen=data.get("first_seen", datetime.utcnow()),
+        last_seen=data.get("last_seen", datetime.utcnow()),
+        source_urls=list(data.get("source_urls", [])),
+        metadata=dict(data.get("metadata", {})),
+    )
+
+
+def _has_node(graph: nx.MultiDiGraph, node_id: str) -> bool:
+    return graph.has_node(node_id)
+
+
+# ---------------------------------------------------------------------------
+# Public functions
+# ---------------------------------------------------------------------------
+
+
+def get_neighbors(
+    graph: nx.MultiDiGraph,
+    node_id: str,
+    hops: int = 2,
+    edge_types: Optional[list[str]] = None,
+) -> dict[str, list[GraphNode]]:
+    """
+    Return all nodes reachable within *hops* steps from *node_id*.
+
+    Traversal is bidirectional (follows both outgoing and incoming edges) so
+    that CO_APPEARED_ON and similar undirected-semantics edges are fully
+    explored regardless of which direction they were stored.
+
+    If *edge_types* is provided, only edges whose ``edge_type`` attribute
+    matches one of the listed types are traversed.
+
+    Returns a dict keyed by hop distance (as a string):
+        {"1": [GraphNode, ...], "2": [GraphNode, ...], ...}
+    """
+    if not _has_node(graph, node_id):
+        return {}
+
+    visited: dict[str, int] = {node_id: 0}  # node_id → hop at which it was reached
+    queue: deque[tuple[str, int]] = deque([(node_id, 0)])
+    result: dict[str, list[GraphNode]] = {}
+
+    while queue:
+        current, depth = queue.popleft()
+        if depth >= hops:
+            continue
+
+        # Collect adjacent nodes (both directions)
+        neighbors: set[str] = set()
+
+        for _, nbr, edge_data in graph.out_edges(current, data=True):
+            if edge_types is None or edge_data.get("edge_type") in edge_types:
+                neighbors.add(nbr)
+
+        for pred, _, edge_data in graph.in_edges(current, data=True):
+            if edge_types is None or edge_data.get("edge_type") in edge_types:
+                neighbors.add(pred)
+
+        for nbr in neighbors:
+            if nbr == node_id:
+                continue
+            if nbr not in visited:
+                hop = depth + 1
+                visited[nbr] = hop
+                key = str(hop)
+                result.setdefault(key, [])
+                result[key].append(_make_graphnode(nbr, graph.nodes[nbr]))
+                queue.append((nbr, hop))
+
+    return result
+
+
+def find_nodes_by_type(
+    graph: nx.MultiDiGraph,
+    node_type: str,
+) -> list[GraphNode]:
+    """Return all nodes in *graph* whose node_type equals *node_type*."""
+    return [
+        _make_graphnode(nid, data)
+        for nid, data in graph.nodes(data=True)
+        if data.get("node_type") == node_type
+    ]
+
+
+def find_co_occurring_entities(
+    graph: nx.MultiDiGraph,
+    node_id: str,
+) -> list[tuple[GraphNode, int]]:
+    """
+    Return a list of (GraphNode, co_occurrence_count) for every node that
+    co-occurs with *node_id* via CO_APPEARED_ON edges.
+
+    Co-occurrence count = number of CO_APPEARED_ON edges connecting the pair
+    (in either direction).  Results are sorted by count descending.
+    """
+    if not _has_node(graph, node_id):
+        return []
+
+    counts: dict[str, int] = defaultdict(int)
+
+    for _, nbr, data in graph.out_edges(node_id, data=True):
+        if data.get("edge_type") == EDGE_TYPES.CO_APPEARED_ON:
+            counts[nbr] += 1
+
+    for pred, _, data in graph.in_edges(node_id, data=True):
+        if data.get("edge_type") == EDGE_TYPES.CO_APPEARED_ON:
+            counts[pred] += 1
+
+    result = [
+        (_make_graphnode(nid, graph.nodes[nid]), count)
+        for nid, count in counts.items()
+    ]
+    result.sort(key=lambda t: t[1], reverse=True)
+    return result
+
+
+def get_new_nodes_since(
+    graph: nx.MultiDiGraph,
+    since: datetime,
+) -> list[GraphNode]:
+    """Return all nodes where first_seen >= *since*."""
+    nodes = []
+    for nid, data in graph.nodes(data=True):
+        first_seen = data.get("first_seen")
+        if first_seen is not None and first_seen >= since:
+            nodes.append(_make_graphnode(nid, data))
+    return nodes
+
+
+def find_high_degree_nodes(
+    graph: nx.MultiDiGraph,
+    top_n: int = 10,
+    node_type: Optional[str] = None,
+) -> list[tuple[GraphNode, int]]:
+    """
+    Return the *top_n* most-connected nodes by total degree (in + out).
+
+    If *node_type* is provided, only consider nodes of that type.
+    Results are sorted by degree descending.
+    """
+    candidates = [
+        (nid, data)
+        for nid, data in graph.nodes(data=True)
+        if node_type is None or data.get("node_type") == node_type
+    ]
+
+    scored = [
+        (_make_graphnode(nid, data), graph.degree(nid))
+        for nid, data in candidates
+    ]
+    scored.sort(key=lambda t: t[1], reverse=True)
+    return scored[:top_n]
+
+
+def get_shortest_path(
+    graph: nx.MultiDiGraph,
+    source_id: str,
+    target_id: str,
+) -> Optional[list[GraphNode]]:
+    """
+    Return the shortest path between *source_id* and *target_id*.
+
+    Uses an undirected view of the graph so paths are found regardless of
+    edge direction.  Returns None if no path exists or either node is absent.
+    """
+    if not _has_node(graph, source_id) or not _has_node(graph, target_id):
+        return None
+
+    try:
+        undirected = graph.to_undirected()
+        path_ids: list[str] = nx.shortest_path(undirected, source_id, target_id)
+        return [_make_graphnode(nid, graph.nodes[nid]) for nid in path_ids]
+    except nx.NetworkXNoPath:
+        return None
+    except nx.NodeNotFound:
+        return None
+    except Exception:
+        return None
+
+
+def get_actor_profile(
+    graph: nx.MultiDiGraph,
+    actor_node_id: str,
+) -> dict:
+    """
+    Return a structured profile dict for a ThreatActor node.
+
+    Keys:
+        node               — GraphNode for the actor
+        connected_wallets  — list[GraphNode] of CryptoWallet neighbours
+        connected_malware  — list[GraphNode] of MalwareFamily/RansomwareGroup
+        connected_forums   — list[GraphNode] of Forum/OnionURL neighbours
+        co_actors          — list[GraphNode] connected via LIKELY/CONFIRMED_SAME_ACTOR
+        total_pages_appeared — number of unique source_urls on the node
+        first_seen         — datetime
+        last_seen          — datetime
+    """
+    if not _has_node(graph, actor_node_id):
+        return {}
+
+    node_data = graph.nodes[actor_node_id]
+    actor_node = _make_graphnode(actor_node_id, node_data)
+
+    _same_actor_types = {EDGE_TYPES.LIKELY_SAME_ACTOR, EDGE_TYPES.CONFIRMED_SAME_ACTOR}
+
+    connected_wallets: list[GraphNode] = []
+    connected_malware: list[GraphNode] = []
+    connected_forums: list[GraphNode] = []
+    co_actors: list[GraphNode] = []
+
+    # Collect all adjacent nodes across all edges
+    all_adjacent: set[str] = set()
+    for _, nbr in graph.out_edges(actor_node_id):
+        all_adjacent.add(nbr)
+    for pred, _ in graph.in_edges(actor_node_id):
+        all_adjacent.add(pred)
+
+    for nbr_id in all_adjacent:
+        if nbr_id == actor_node_id:
+            continue
+        nbr_data = graph.nodes.get(nbr_id, {})
+        nbr_type = nbr_data.get("node_type", "")
+        nbr_node = _make_graphnode(nbr_id, nbr_data)
+
+        if nbr_type == NODE_TYPES.CRYPTO_WALLET:
+            connected_wallets.append(nbr_node)
+        elif nbr_type in (NODE_TYPES.MALWARE_FAMILY, NODE_TYPES.RANSOMWARE_GROUP):
+            connected_malware.append(nbr_node)
+        elif nbr_type in (NODE_TYPES.FORUM, NODE_TYPES.ONION_URL):
+            connected_forums.append(nbr_node)
+
+    # Co-actors: nodes connected specifically via same-actor edge types
+    for _, nbr, data in graph.out_edges(actor_node_id, data=True):
+        if data.get("edge_type") in _same_actor_types:
+            co_actors.append(_make_graphnode(nbr, graph.nodes[nbr]))
+
+    for pred, _, data in graph.in_edges(actor_node_id, data=True):
+        if data.get("edge_type") in _same_actor_types:
+            co_actors.append(_make_graphnode(pred, graph.nodes[pred]))
+
+    # Deduplicate co_actors by node_id
+    seen_co: set[str] = set()
+    deduped_co: list[GraphNode] = []
+    for n in co_actors:
+        if n.node_id not in seen_co:
+            seen_co.add(n.node_id)
+            deduped_co.append(n)
+
+    return {
+        "node": actor_node,
+        "connected_wallets": connected_wallets,
+        "connected_malware": connected_malware,
+        "connected_forums": connected_forums,
+        "co_actors": deduped_co,
+        "total_pages_appeared": len(node_data.get("source_urls", [])),
+        "first_seen": node_data.get("first_seen"),
+        "last_seen": node_data.get("last_seen"),
+    }