icdev 0.0.3__py3-none-any.whl

tools/knowledge_graph/text_network.py

@@ -0,0 +1,832 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""Text Network Analysis Engine — InfraNodus-style co-occurrence graph builder.
+
+Converts free text into a weighted co-occurrence graph, detects topical
+communities (Louvain), ranks concepts by betweenness centrality, and
+identifies structural gaps between weakly-connected clusters.
+
+Architecture decisions:
+  - D-KG-1: Pure Python stdlib + minimal deps (air-gap safe)
+  - D-KG-2: SQL adjacency list storage (matches D27 pattern)
+  - D-KG-3: Louvain community detection (deterministic, no LLM)
+  - D-KG-4: 4-gram sliding window for co-occurrence (InfraNodus standard)
+  - D-KG-5: Structural gap = high-distance cross-cluster node pairs
+
+Usage:
+    python tools/knowledge_graph/text_network.py --text "your text" --json
+    python tools/knowledge_graph/text_network.py --graph-id <id> --stats --json
+    python tools/knowledge_graph/text_network.py --graph-id <id> --gaps --json
+    python tools/knowledge_graph/text_network.py --list --json
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+import re
+import sys
+import uuid
+from collections import defaultdict
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+BASE_DIR = Path(__file__).resolve().parent.parent.parent
+sys.path.insert(0, str(BASE_DIR))
+
+from tools.db.storage import get_connection  # noqa: E402
+
+# ── Stop words (English, expandable) ─────────────────────────────
+STOP_WORDS = frozenset(
+    "a an the and or but in on at to for of is it was were be been "
+    "being have has had do does did will would shall should may might "
+    "can could this that these those i me my we our you your he him "
+    "his she her they them their its not no nor so if then than too "
+    "very just also with from by as are am about into through during "
+    "before after above below between out up down off over under again "
+    "further once here there when where why how all each every both "
+    "few more most other some such only own same which what while "
+    "because until whom whose".split()
+)
+
+# ── Lemmatization (lightweight rule-based, no NLTK needed) ───────
+_SUFFIX_RULES = [
+    (r"ies$", "y"), (r"sses$", "ss"), (r"([^s])s$", r"\1"),
+    (r"eed$", "eed"), (r"ed$", ""), (r"ing$", ""),
+    (r"tion$", "tion"), (r"ment$", "ment"), (r"ness$", "ness"),
+    (r"ful$", "ful"), (r"able$", "able"), (r"ible$", "ible"),
+    (r"ly$", ""), (r"er$", ""), (r"est$", ""),
+]
+
+
+def _simple_lemma(word: str) -> str:
+    """Rule-based lemmatization (no external deps)."""
+    if len(word) <= 3:
+        return word
+    for pattern, replacement in _SUFFIX_RULES:
+        candidate = re.sub(pattern, replacement, word)
+        if candidate != word and len(candidate) >= 3:
+            return candidate
+    return word
+
+
+# ── Tokenizer ────────────────────────────────────────────────────
+def tokenize(text: str, *, lemmatize: bool = True,
+             min_length: int = 3) -> list[str]:
+    """Tokenize text: lowercase, remove punctuation, filter stops, lemmatize."""
+    words = re.findall(r"[a-z][a-z0-9_'-]*[a-z0-9]|[a-z]", text.lower())
+    tokens = [w for w in words if w not in STOP_WORDS and len(w) >= min_length]
+    if lemmatize:
+        tokens = [_simple_lemma(w) for w in tokens]
+    return tokens
+
+
+# ── Co-occurrence graph builder ──────────────────────────────────
+def build_cooccurrence_graph(
+    tokens: list[str],
+    window_size: int = 4,
+) -> tuple[dict[str, int], dict[tuple[str, str], float]]:
+    """Build weighted co-occurrence graph from token stream.
+
+    Uses a sliding window of `window_size` tokens (InfraNodus default=4).
+    Closer tokens get higher edge weight (1/distance).
+
+    Returns:
+        nodes: {word: total_occurrence_count}
+        edges: {(word_a, word_b): cumulative_weight}  (a < b always)
+    """
+    nodes: dict[str, int] = defaultdict(int)
+    edges: dict[tuple[str, str], float] = defaultdict(float)
+
+    for i, token in enumerate(tokens):
+        nodes[token] += 1
+        for j in range(i + 1, min(i + window_size, len(tokens))):
+            neighbor = tokens[j]
+            if token == neighbor:
+                continue
+            distance = j - i
+            weight = 1.0 / distance  # closer = stronger
+            pair = tuple(sorted([token, neighbor]))
+            edges[pair] += weight
+
+    return dict(nodes), dict(edges)
+
+
+# ── Betweenness centrality (Brandes algorithm, O(VE)) ───────────
+def _betweenness_centrality(
+    adj: dict[str, dict[str, float]],
+) -> dict[str, float]:
+    """Brandes algorithm for betweenness centrality."""
+    centrality: dict[str, float] = {v: 0.0 for v in adj}
+    nodes = list(adj.keys())
+
+    for s in nodes:
+        # BFS/Dijkstra from s
+        stack: list[str] = []
+        pred: dict[str, list[str]] = {v: [] for v in nodes}
+        sigma: dict[str, float] = {v: 0.0 for v in nodes}
+        sigma[s] = 1.0
+        dist: dict[str, float] = {v: float("inf") for v in nodes}
+        dist[s] = 0.0
+
+        # BFS (unweighted for simplicity — matches InfraNodus approach)
+        queue = [s]
+        while queue:
+            v = queue.pop(0)
+            stack.append(v)
+            for w in adj.get(v, {}):
+                if dist[w] == float("inf"):
+                    dist[w] = dist[v] + 1
+                    queue.append(w)
+                if dist[w] == dist[v] + 1:
+                    sigma[w] += sigma[v]
+                    pred[w].append(v)
+
+        delta: dict[str, float] = {v: 0.0 for v in nodes}
+        while stack:
+            w = stack.pop()
+            for v in pred[w]:
+                delta[v] += (sigma[v] / sigma[w]) * (1 + delta[w])
+            if w != s:
+                centrality[w] += delta[w]
+
+    # Normalize
+    n = len(nodes)
+    if n > 2:
+        norm = 1.0 / ((n - 1) * (n - 2))
+        centrality = {v: c * norm for v, c in centrality.items()}
+
+    return centrality
+
+
+# ── Louvain community detection (simplified, pure Python) ────────
+def _louvain_communities(
+    adj: dict[str, dict[str, float]],
+) -> dict[str, int]:
+    """Simplified Louvain for community detection.
+
+    Returns {node: community_id} mapping.
+    """
+    # Total edge weight
+    m = sum(sum(neighbors.values()) for neighbors in adj.values()) / 2.0
+    if m == 0:
+        return {v: i for i, v in enumerate(adj)}
+
+    # Initial: each node in its own community
+    community: dict[str, int] = {v: i for i, v in enumerate(adj)}
+    node_strength: dict[str, float] = {
+        v: sum(neighbors.values()) for v, neighbors in adj.items()
+    }
+
+    improved = True
+    max_iterations = 20
+    iteration = 0
+
+    while improved and iteration < max_iterations:
+        improved = False
+        iteration += 1
+
+        for v in adj:
+            current_comm = community[v]
+
+            # Compute neighbor community weights
+            comm_weights: dict[int, float] = defaultdict(float)
+            for w, weight in adj[v].items():
+                comm_weights[community[w]] += weight
+
+            # Strength of current community (excluding v)
+            best_comm = current_comm
+            best_delta = 0.0
+
+            for c, w_c in comm_weights.items():
+                if c == current_comm:
+                    continue
+                # Simplified modularity gain
+                sigma_c = sum(
+                    node_strength[u] for u in adj if community[u] == c
+                )
+                sigma_cur = sum(
+                    node_strength[u] for u in adj
+                    if community[u] == current_comm and u != v
+                )
+                k_v = node_strength[v]
+
+                delta = (w_c - comm_weights.get(current_comm, 0.0)) / m
+                delta -= k_v * (sigma_c - sigma_cur) / (2.0 * m * m)
+
+                if delta > best_delta:
+                    best_delta = delta
+                    best_comm = c
+
+            if best_comm != current_comm:
+                community[v] = best_comm
+                improved = True
+
+    # Renumber communities sequentially
+    unique = sorted(set(community.values()))
+    remap = {old: new for new, old in enumerate(unique)}
+    return {v: remap[c] for v, c in community.items()}
+
+
+# ── Structural gap detection ─────────────────────────────────────
+def detect_structural_gaps(
+    adj: dict[str, dict[str, float]],
+    communities: dict[str, int],
+    centrality: dict[str, float],
+    top_n: int = 10,
+) -> list[dict[str, Any]]:
+    """Find structural gaps — weakly connected cluster pairs.
+
+    A structural gap is a pair of communities with few or no direct edges,
+    representing topics that are discussed but not connected — the "blind
+    spots" in discourse (InfraNodus's key differentiator).
+
+    Returns top_n gap pairs ranked by potential insight value.
+    """
+    # Build community adjacency
+    comm_nodes: dict[int, list[str]] = defaultdict(list)
+    for node, comm in communities.items():
+        comm_nodes[comm].append(node)
+
+    num_communities = len(comm_nodes)
+    if num_communities < 2:
+        return []
+
+    gaps: list[dict[str, Any]] = []
+
+    comm_ids = sorted(comm_nodes.keys())
+    for i, c1 in enumerate(comm_ids):
+        for c2 in comm_ids[i + 1:]:
+            # Count cross-community edges and total weight
+            cross_weight = 0.0
+            cross_count = 0
+            for node in comm_nodes[c1]:
+                for neighbor, weight in adj.get(node, {}).items():
+                    if communities.get(neighbor) == c2:
+                        cross_weight += weight
+                        cross_count += 1
+
+            # Maximum possible cross-edges
+            max_possible = len(comm_nodes[c1]) * len(comm_nodes[c2])
+            density = cross_count / max_possible if max_possible > 0 else 0.0
+
+            # Gap score: lower density = bigger gap = more insight potential
+            gap_score = 1.0 - density
+
+            # Top bridge candidates: highest centrality nodes in each cluster
+            top_c1 = sorted(
+                comm_nodes[c1], key=lambda n: centrality.get(n, 0), reverse=True
+            )[:3]
+            top_c2 = sorted(
+                comm_nodes[c2], key=lambda n: centrality.get(n, 0), reverse=True
+            )[:3]
+
+            gaps.append({
+                "community_a": c1,
+                "community_b": c2,
+                "community_a_size": len(comm_nodes[c1]),
+                "community_b_size": len(comm_nodes[c2]),
+                "community_a_top_concepts": top_c1,
+                "community_b_top_concepts": top_c2,
+                "cross_edge_count": cross_count,
+                "cross_edge_weight": round(cross_weight, 4),
+                "density": round(density, 4),
+                "gap_score": round(gap_score, 4),
+            })
+
+    gaps.sort(key=lambda g: g["gap_score"], reverse=True)
+    return gaps[:top_n]
+
+
+# ── Adjacency dict builder ───────────────────────────────────────
+def _build_adjacency(
+    edges: dict[tuple[str, str], float],
+) -> dict[str, dict[str, float]]:
+    """Convert edge dict to bidirectional adjacency dict."""
+    adj: dict[str, dict[str, float]] = defaultdict(dict)
+    for (a, b), weight in edges.items():
+        adj[a][b] = weight
+        adj[b][a] = weight
+    return dict(adj)
+
+
+# ── Full analysis pipeline ───────────────────────────────────────
+def analyze_text(
+    text: str,
+    *,
+    window_size: int = 4,
+    lemmatize: bool = True,
+    min_word_length: int = 3,
+) -> dict[str, Any]:
+    """Full text network analysis pipeline.
+
+    1. Tokenize (lowercase, stop removal, lemmatization)
+    2. Build co-occurrence graph (sliding window)
+    3. Compute betweenness centrality (Brandes algorithm)
+    4. Detect communities (Louvain)
+    5. Find structural gaps
+
+    Returns complete graph + metrics as JSON-serializable dict.
+    """
+    tokens = tokenize(text, lemmatize=lemmatize, min_length=min_word_length)
+
+    if len(tokens) < 2:
+        return {
+            "error": "Insufficient tokens for analysis (need at least 2)",
+            "token_count": len(tokens),
+        }
+
+    nodes, edges = build_cooccurrence_graph(tokens, window_size=window_size)
+    adj = _build_adjacency(edges)
+
+    # Ensure all nodes appear in adjacency (isolated nodes)
+    for node in nodes:
+        if node not in adj:
+            adj[node] = {}
+
+    centrality = _betweenness_centrality(adj)
+    communities = _louvain_communities(adj)
+    gaps = detect_structural_gaps(adj, communities, centrality)
+
+    # Build node list with metrics
+    node_list = []
+    for word, count in sorted(nodes.items(), key=lambda x: -x[1]):
+        node_list.append({
+            "id": word,
+            "label": word,
+            "occurrences": count,
+            "centrality": round(centrality.get(word, 0.0), 6),
+            "community": communities.get(word, 0),
+            "degree": len(adj.get(word, {})),
+        })
+
+    # Build edge list
+    edge_list = []
+    for (a, b), weight in sorted(edges.items(), key=lambda x: -x[1]):
+        edge_list.append({
+            "source": a,
+            "target": b,
+            "weight": round(weight, 4),
+        })
+
+    # Community summary
+    comm_summary: dict[int, dict] = defaultdict(
+        lambda: {"id": 0, "size": 0, "top_concepts": [], "total_centrality": 0.0}
+    )
+    for n in node_list:
+        c = n["community"]
+        cs = comm_summary[c]
+        cs["id"] = c
+        cs["size"] += 1
+        cs["total_centrality"] += n["centrality"]
+        cs["top_concepts"].append((n["id"], n["centrality"]))
+
+    for cs in comm_summary.values():
+        cs["top_concepts"] = [
+            w for w, _ in sorted(cs["top_concepts"], key=lambda x: -x[1])[:5]
+        ]
+        cs["total_centrality"] = round(cs["total_centrality"], 6)
+
+    # Modularity estimate
+    num_communities = len(comm_summary)
+    total_nodes = len(node_list)
+    total_edges = len(edge_list)
+
+    return {
+        "nodes": node_list,
+        "edges": edge_list,
+        "communities": list(comm_summary.values()),
+        "structural_gaps": gaps,
+        "stats": {
+            "total_tokens": len(tokens),
+            "unique_concepts": total_nodes,
+            "total_edges": total_edges,
+            "num_communities": num_communities,
+            "avg_degree": round(
+                sum(n["degree"] for n in node_list) / max(total_nodes, 1), 2
+            ),
+            "density": round(
+                2 * total_edges / (total_nodes * (total_nodes - 1))
+                if total_nodes > 1 else 0.0,
+                4,
+            ),
+            "top_concepts_by_centrality": [
+                n["id"] for n in sorted(
+                    node_list, key=lambda x: -x["centrality"]
+                )[:10]
+            ],
+        },
+    }
+
+
+# ── Database persistence ─────────────────────────────────────────
+def _ensure_tables():
+    """Create knowledge graph tables if they don't exist."""
+    with get_connection() as conn:
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS kg_graphs (
+                id TEXT PRIMARY KEY,
+                project_id TEXT NOT NULL,
+                name TEXT NOT NULL,
+                description TEXT,
+                source_type TEXT DEFAULT 'text',
+                source_ref TEXT,
+                config_json TEXT,
+                stats_json TEXT,
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS kg_nodes (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                graph_id TEXT NOT NULL,
+                word TEXT NOT NULL,
+                occurrences INTEGER DEFAULT 1,
+                centrality REAL DEFAULT 0.0,
+                community INTEGER DEFAULT 0,
+                degree INTEGER DEFAULT 0,
+                metadata_json TEXT,
+                UNIQUE(graph_id, word)
+            )
+        """)
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS kg_edges (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                graph_id TEXT NOT NULL,
+                source_word TEXT NOT NULL,
+                target_word TEXT NOT NULL,
+                weight REAL DEFAULT 1.0,
+                UNIQUE(graph_id, source_word, target_word)
+            )
+        """)
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS kg_gaps (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                graph_id TEXT NOT NULL,
+                community_a INTEGER NOT NULL,
+                community_b INTEGER NOT NULL,
+                gap_score REAL DEFAULT 0.0,
+                details_json TEXT,
+                ai_questions TEXT,
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS kg_ingestions (
+                id TEXT PRIMARY KEY,
+                graph_id TEXT NOT NULL,
+                source_type TEXT NOT NULL,
+                source_ref TEXT,
+                token_count INTEGER DEFAULT 0,
+                status TEXT DEFAULT 'completed',
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+        conn.execute("""
+            CREATE INDEX IF NOT EXISTS idx_kg_nodes_graph ON kg_nodes(graph_id)
+        """)
+        conn.execute("""
+            CREATE INDEX IF NOT EXISTS idx_kg_edges_graph ON kg_edges(graph_id)
+        """)
+        conn.execute("""
+            CREATE INDEX IF NOT EXISTS idx_kg_gaps_graph ON kg_gaps(graph_id)
+        """)
+        conn.commit()
+
+
+def save_graph(
+    project_id: str,
+    name: str,
+    analysis: dict[str, Any],
+    *,
+    description: str = "",
+    source_type: str = "text",
+    source_ref: str = "",
+    config: dict | None = None,
+) -> str:
+    """Persist a text network analysis to the database.
+
+    Returns the graph_id.
+    """
+    _ensure_tables()
+    graph_id = f"kg-{uuid.uuid4().hex[:12]}"
+    now = datetime.now(timezone.utc).isoformat()
+
+    with get_connection() as conn:
+        conn.execute(
+            """INSERT INTO kg_graphs
+               (id, project_id, name, description, source_type, source_ref,
+                config_json, stats_json, created_at, updated_at)
+               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+            (graph_id, project_id, name, description, source_type, source_ref,
+             json.dumps(config or {}), json.dumps(analysis.get("stats", {})),
+             now, now),
+        )
+
+        # Nodes
+        for node in analysis.get("nodes", []):
+            conn.execute(
+                """INSERT OR REPLACE INTO kg_nodes
+                   (graph_id, word, occurrences, centrality, community, degree)
+                   VALUES (?, ?, ?, ?, ?, ?)""",
+                (graph_id, node["id"], node["occurrences"],
+                 node["centrality"], node["community"], node["degree"]),
+            )
+
+        # Edges
+        for edge in analysis.get("edges", []):
+            conn.execute(
+                """INSERT OR REPLACE INTO kg_edges
+                   (graph_id, source_word, target_word, weight)
+                   VALUES (?, ?, ?, ?)""",
+                (graph_id, edge["source"], edge["target"], edge["weight"]),
+            )
+
+        # Gaps
+        for gap in analysis.get("structural_gaps", []):
+            conn.execute(
+                """INSERT INTO kg_gaps
+                   (graph_id, community_a, community_b, gap_score, details_json)
+                   VALUES (?, ?, ?, ?, ?)""",
+                (graph_id, gap["community_a"], gap["community_b"],
+                 gap["gap_score"], json.dumps(gap)),
+            )
+
+        # Ingestion log
+        conn.execute(
+            """INSERT INTO kg_ingestions
+               (id, graph_id, source_type, source_ref, token_count, status)
+               VALUES (?, ?, ?, ?, ?, 'completed')""",
+            (f"ing-{uuid.uuid4().hex[:12]}", graph_id, source_type,
+             source_ref, analysis.get("stats", {}).get("total_tokens", 0)),
+        )
+
+        conn.commit()
+
+    return graph_id
+
+
+def load_graph(graph_id: str) -> dict[str, Any] | None:
+    """Load a saved graph from the database."""
+    _ensure_tables()
+    with get_connection() as conn:
+        row = conn.execute(
+            "SELECT * FROM kg_graphs WHERE id = ?", (graph_id,)
+        ).fetchone()
+        if not row:
+            return None
+
+        nodes = conn.execute(
+            "SELECT word, occurrences, centrality, community, degree "
+            "FROM kg_nodes WHERE graph_id = ? ORDER BY centrality DESC",
+            (graph_id,),
+        ).fetchall()
+
+        edges = conn.execute(
+            "SELECT source_word, target_word, weight "
+            "FROM kg_edges WHERE graph_id = ? ORDER BY weight DESC",
+            (graph_id,),
+        ).fetchall()
+
+        gaps = conn.execute(
+            "SELECT community_a, community_b, gap_score, details_json, ai_questions "
+            "FROM kg_gaps WHERE graph_id = ? ORDER BY gap_score DESC",
+            (graph_id,),
+        ).fetchall()
+
+        return {
+            "graph": {
+                "id": row["id"],
+                "project_id": row["project_id"],
+                "name": row["name"],
+                "description": row["description"],
+                "source_type": row["source_type"],
+                "source_ref": row["source_ref"],
+                "stats": json.loads(row["stats_json"] or "{}"),
+                "created_at": row["created_at"],
+            },
+            "nodes": [
+                {"id": r["word"], "label": r["word"],
+                 "occurrences": r["occurrences"],
+                 "centrality": r["centrality"],
+                 "community": r["community"],
+                 "degree": r["degree"]}
+                for r in nodes
+            ],
+            "edges": [
+                {"source": r["source_word"], "target": r["target_word"],
+                 "weight": r["weight"]}
+                for r in edges
+            ],
+            "structural_gaps": [
+                {**json.loads(r["details_json"] or "{}"),
+                 "ai_questions": r["ai_questions"]}
+                for r in gaps
+            ],
+        }
+
+
+def list_graphs(project_id: str = "sparkpilot") -> list[dict]:
+    """List all knowledge graphs for a project."""
+    _ensure_tables()
+    with get_connection() as conn:
+        rows = conn.execute(
+            "SELECT id, name, description, source_type, stats_json, created_at "
+            "FROM kg_graphs WHERE project_id = ? ORDER BY created_at DESC",
+            (project_id,),
+        ).fetchall()
+        return [
+            {"id": r["id"], "name": r["name"], "description": r["description"],
+             "source_type": r["source_type"],
+             "stats": json.loads(r["stats_json"] or "{}"),
+             "created_at": r["created_at"]}
+            for r in rows
+        ]
+
+
+def append_to_graph(graph_id: str, text: str, *,
+                    source_type: str = "text",
+                    source_ref: str = "",
+                    window_size: int = 4) -> dict[str, Any]:
+    """Incrementally add text to an existing graph and re-analyze.
+
+    Loads existing nodes/edges, merges new co-occurrences, re-computes
+    centrality and communities.
+    """
+    _ensure_tables()
+    existing = load_graph(graph_id)
+    if not existing:
+        return {"error": f"Graph {graph_id} not found"}
+
+    # Build existing data structures
+    nodes: dict[str, int] = {}
+    edges: dict[tuple[str, str], float] = {}
+    for n in existing["nodes"]:
+        nodes[n["id"]] = n["occurrences"]
+    for e in existing["edges"]:
+        pair = tuple(sorted([e["source"], e["target"]]))
+        edges[pair] = e["weight"]
+
+    # Tokenize new text and merge
+    tokens = tokenize(text, lemmatize=True, min_length=3)
+    new_nodes, new_edges = build_cooccurrence_graph(tokens, window_size)
+
+    for word, count in new_nodes.items():
+        nodes[word] = nodes.get(word, 0) + count
+    for pair, weight in new_edges.items():
+        edges[pair] = edges.get(pair, 0.0) + weight
+
+    # Re-analyze
+    adj = _build_adjacency(edges)
+    for node in nodes:
+        if node not in adj:
+            adj[node] = {}
+
+    centrality = _betweenness_centrality(adj)
+    communities = _louvain_communities(adj)
+    gaps = detect_structural_gaps(adj, communities, centrality)
+
+    # Build result
+    node_list = [
+        {"id": w, "label": w, "occurrences": c,
+         "centrality": round(centrality.get(w, 0), 6),
+         "community": communities.get(w, 0),
+         "degree": len(adj.get(w, {}))}
+        for w, c in sorted(nodes.items(), key=lambda x: -x[1])
+    ]
+    edge_list = [
+        {"source": a, "target": b, "weight": round(w, 4)}
+        for (a, b), w in sorted(edges.items(), key=lambda x: -x[1])
+    ]
+
+    analysis = {
+        "nodes": node_list,
+        "edges": edge_list,
+        "structural_gaps": gaps,
+        "stats": {
+            "total_tokens": sum(n["occurrences"] for n in node_list),
+            "unique_concepts": len(node_list),
+            "total_edges": len(edge_list),
+            "num_communities": len(set(communities.values())),
+        },
+    }
+
+    # Update database
+    now = datetime.now(timezone.utc).isoformat()
+    with get_connection() as conn:
+        conn.execute("DELETE FROM kg_nodes WHERE graph_id = ?", (graph_id,))
+        conn.execute("DELETE FROM kg_edges WHERE graph_id = ?", (graph_id,))
+        conn.execute("DELETE FROM kg_gaps WHERE graph_id = ?", (graph_id,))
+
+        for n in node_list:
+            conn.execute(
+                "INSERT INTO kg_nodes (graph_id, word, occurrences, centrality, community, degree) "
+                "VALUES (?, ?, ?, ?, ?, ?)",
+                (graph_id, n["id"], n["occurrences"], n["centrality"],
+                 n["community"], n["degree"]),
+            )
+        for e in edge_list:
+            conn.execute(
+                "INSERT INTO kg_edges (graph_id, source_word, target_word, weight) "
+                "VALUES (?, ?, ?, ?)",
+                (graph_id, e["source"], e["target"], e["weight"]),
+            )
+        for gap in gaps:
+            conn.execute(
+                "INSERT INTO kg_gaps (graph_id, community_a, community_b, gap_score, details_json) "
+                "VALUES (?, ?, ?, ?, ?)",
+                (graph_id, gap["community_a"], gap["community_b"],
+                 gap["gap_score"], json.dumps(gap)),
+            )
+        # Ingestion log
+        conn.execute(
+            "INSERT INTO kg_ingestions (id, graph_id, source_type, source_ref, token_count) "
+            "VALUES (?, ?, ?, ?, ?)",
+            (f"ing-{uuid.uuid4().hex[:12]}", graph_id, source_type,
+             source_ref, len(tokens)),
+        )
+        conn.execute(
+            "UPDATE kg_graphs SET stats_json = ?, updated_at = ? WHERE id = ?",
+            (json.dumps(analysis["stats"]), now, graph_id),
+        )
+        conn.commit()
+
+    analysis["graph_id"] = graph_id
+    return analysis
+
+
+# ── CLI ──────────────────────────────────────────────────────────
+def main():
+    parser = argparse.ArgumentParser(description="Text Network Analysis Engine")
+    parser.add_argument("--text", help="Text to analyze")
+    parser.add_argument("--file", help="File to analyze")
+    parser.add_argument("--graph-id", help="Graph ID for operations")
+    parser.add_argument("--project-id", default="sparkpilot")
+    parser.add_argument("--name", default="Untitled Graph")
+    parser.add_argument("--save", action="store_true", help="Save to database")
+    parser.add_argument("--append", action="store_true", help="Append to existing graph")
+    parser.add_argument("--list", action="store_true", help="List saved graphs")
+    parser.add_argument("--load", action="store_true", help="Load saved graph")
+    parser.add_argument("--stats", action="store_true", help="Show graph stats")
+    parser.add_argument("--gaps", action="store_true", help="Show structural gaps")
+    parser.add_argument("--window-size", type=int, default=4)
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+
+    if args.list:
+        graphs = list_graphs(args.project_id)
+        result = {"graphs": graphs, "count": len(graphs),
+                  "classification": "CUI // SP-CTI"}
+        print(json.dumps(result, indent=2) if args.json else str(result))
+        return
+
+    if args.load and args.graph_id:
+        graph = load_graph(args.graph_id)
+        if graph:
+            graph["classification"] = "CUI // SP-CTI"
+            print(json.dumps(graph, indent=2) if args.json else str(graph))
+        else:
+            print(json.dumps({"error": "Graph not found"}, indent=2))
+        return
+
+    # Get text
+    text = args.text
+    if args.file:
+        text = Path(args.file).read_text(encoding="utf-8")
+
+    if args.append and args.graph_id:
+        if not text:
+            print(json.dumps({"error": "No text provided for append"}))
+            return
+        result = append_to_graph(args.graph_id, text,
+                                 window_size=args.window_size)
+        result["classification"] = "CUI // SP-CTI"
+        print(json.dumps(result, indent=2) if args.json else str(result))
+        return
+
+    if not text:
+        parser.print_help()
+        return
+
+    analysis = analyze_text(text, window_size=args.window_size)
+    analysis["classification"] = "CUI // SP-CTI"
+
+    if args.save:
+        graph_id = save_graph(args.project_id, args.name, analysis)
+        analysis["graph_id"] = graph_id
+
+    if args.stats:
+        print(json.dumps(analysis["stats"], indent=2) if args.json
+              else str(analysis["stats"]))
+    elif args.gaps:
+        print(json.dumps(analysis["structural_gaps"], indent=2) if args.json
+              else str(analysis["structural_gaps"]))
+    else:
+        print(json.dumps(analysis, indent=2) if args.json else str(analysis))
+
+
+if __name__ == "__main__":
+    main()