ckgraphify 0.1.0__py3-none-any.whl

graphify/benchmark.py

@@ -0,0 +1,152 @@
+"""Token-reduction benchmark - measures how much context graphify saves vs naive full-corpus approach."""
+from __future__ import annotations
+import json
+import sys
+from pathlib import Path
+import networkx as nx
+from networkx.readwrite import json_graph
+
+from graphify.build import edge_data
+
+
+_CHARS_PER_TOKEN = 4  # standard approximation
+
+
+def _safe(unicode_char: str, ascii_fallback: str) -> str:
+    """Return unicode_char if stdout can encode it, else ascii_fallback.
+
+    Windows consoles often default to cp1252 which cannot encode box-drawing
+    or arrow glyphs; printing them raises UnicodeEncodeError mid-output.
+    """
+    encoding = getattr(sys.stdout, "encoding", None) or ""
+    try:
+        unicode_char.encode(encoding)
+        return unicode_char
+    except (UnicodeEncodeError, LookupError):
+        return ascii_fallback
+
+
+def _hr(width: int = 50) -> str:
+    """Horizontal rule that survives non-UTF-8 stdout (e.g. Windows cp1252 console)."""
+    return _safe("─", "-") * width
+
+
+def _estimate_tokens(text: str) -> int:
+    return max(1, len(text) // _CHARS_PER_TOKEN)
+
+
+def _query_subgraph_tokens(G: nx.Graph, question: str, depth: int = 3) -> int:
+    """Run BFS from best-matching nodes and return estimated tokens in the subgraph context."""
+    terms = [t.lower() for t in question.split() if len(t) > 2]
+    scored = []
+    for nid, data in G.nodes(data=True):
+        label = data.get("label", "").lower()
+        score = sum(1 for t in terms if t in label)
+        if score > 0:
+            scored.append((score, nid))
+    scored.sort(reverse=True)
+    start_nodes = [nid for _, nid in scored[:3]]
+    if not start_nodes:
+        return 0
+
+    visited: set[str] = set(start_nodes)
+    frontier = set(start_nodes)
+    edges_seen: list[tuple] = []
+    for _ in range(depth):
+        next_frontier: set[str] = set()
+        for n in frontier:
+            for neighbor in G.neighbors(n):
+                if neighbor not in visited:
+                    next_frontier.add(neighbor)
+                    edges_seen.append((n, neighbor))
+        visited.update(next_frontier)
+        frontier = next_frontier
+
+    lines = []
+    for nid in visited:
+        d = G.nodes[nid]
+        lines.append(f"NODE {d.get('label', nid)} src={d.get('source_file', '')} loc={d.get('source_location', '')}")
+    for u, v in edges_seen:
+        if u in visited and v in visited:
+            d = edge_data(G, u, v)
+            lines.append(f"EDGE {G.nodes[u].get('label', u)} --{d.get('relation', '')}--> {G.nodes[v].get('label', v)}")
+
+    return _estimate_tokens("\n".join(lines))
+
+
+_SAMPLE_QUESTIONS = [
+    "how does authentication work",
+    "what is the main entry point",
+    "how are errors handled",
+    "what connects the data layer to the api",
+    "what are the core abstractions",
+]
+
+
+def run_benchmark(
+    graph_path: str = "graphify-out/graph.json",
+    corpus_words: int | None = None,
+    questions: list[str] | None = None,
+) -> dict:
+    """Measure token reduction: corpus tokens vs graphify query tokens.
+
+    Args:
+        graph_path: path to the built graph
+        corpus_words: total word count from detect() output; if None, estimated from graph
+        questions: list of questions to benchmark; defaults to _SAMPLE_QUESTIONS
+
+    Returns dict with: corpus_tokens, avg_query_tokens, reduction_ratio, per_question
+    """
+    data = json.loads(Path(graph_path).read_text(encoding="utf-8"))
+    try:
+        G = json_graph.node_link_graph(data, edges="links")
+    except TypeError:
+        G = json_graph.node_link_graph(data)
+
+    if corpus_words is None:
+        # Rough estimate: each node label is ~3 words, plus source context
+        corpus_words = G.number_of_nodes() * 50
+
+    corpus_tokens = corpus_words * 100 // 75  # words → tokens (100 words ≈ 133 tokens)
+
+    qs = questions or _SAMPLE_QUESTIONS
+    per_question = []
+    for q in qs:
+        qt = _query_subgraph_tokens(G, q)
+        if qt > 0:
+            per_question.append({"question": q, "query_tokens": qt, "reduction": round(corpus_tokens / qt, 1)})
+
+    if not per_question:
+        return {"error": "No matching nodes found for sample questions. Build the graph first."}
+
+    avg_query_tokens = sum(p["query_tokens"] for p in per_question) // len(per_question)
+    reduction_ratio = round(corpus_tokens / avg_query_tokens, 1) if avg_query_tokens > 0 else 0
+
+    return {
+        "corpus_tokens": corpus_tokens,
+        "corpus_words": corpus_words,
+        "nodes": G.number_of_nodes(),
+        "edges": G.number_of_edges(),
+        "avg_query_tokens": avg_query_tokens,
+        "reduction_ratio": reduction_ratio,
+        "per_question": per_question,
+    }
+
+
+def print_benchmark(result: dict) -> None:
+    """Print a human-readable benchmark report."""
+    if "error" in result:
+        print(f"Benchmark error: {result['error']}")
+        return
+
+    print(f"\ngraphify token reduction benchmark")
+    print(_hr(50))
+    arrow = _safe("→", "->")
+    print(f"  Corpus:          {result['corpus_words']:,} words {arrow} ~{result['corpus_tokens']:,} tokens (naive)")
+    print(f"  Graph:           {result['nodes']:,} nodes, {result['edges']:,} edges")
+    print(f"  Avg query cost:  ~{result['avg_query_tokens']:,} tokens")
+    print(f"  Reduction:       {result['reduction_ratio']}x fewer tokens per query")
+    print(f"\n  Per question:")
+    for p in result["per_question"]:
+        print(f"    [{p['reduction']}x] {p['question'][:55]}")
+    print()

graphify/bridge_mtop.py

@@ -0,0 +1,517 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable
+
+
+_FRONTEND_EXTS = {".js", ".jsx", ".ts", ".tsx", ".vue", ".svelte", ".mjs", ".cjs"}
+_BACKEND_EXTS = {".java", ".kt", ".go", ".py", ".php", ".rb", ".cs", ".scala"}
+_SKIP_DIRS = {
+    ".git",
+    "node_modules",
+    "dist",
+    "build",
+    "target",
+    ".next",
+    ".nuxt",
+    ".idea",
+    ".vscode",
+    "__pycache__",
+    ".venv",
+    "venv",
+}
+
+# Strong signal: literal mtop.* token.
+_MTOP_LITERAL_RE = re.compile(r"\b(mtop\.[A-Za-z0-9_.-]{3,})\b")
+# Fallback signal: api/method assignment with dotted value.
+_API_ASSIGN_RE = re.compile(
+    r"(?i)\b(?:api|apiname|method|mtopapi)\b\s*[:=]\s*['\"]([A-Za-z0-9_.-]+)['\"]"
+)
+
+
+@dataclass(frozen=True)
+class ApiFact:
+    repo: str
+    rel_path: str
+    line: int
+    api: str
+    source: str  # literal|assign
+
+
+@dataclass(frozen=True)
+class BridgeStats:
+    repos_scanned: int
+    files_scanned: int
+    frontend_facts: int
+    backend_facts: int
+    api_nodes_added: int
+    edge_calls_added: int
+    edge_impl_added: int
+    unresolved_frontend: int
+    unresolved_backend: int
+
+
+def _norm_path(p: str | None) -> str:
+    return (p or "").replace("\\", "/")
+
+
+def _norm_api(raw: str) -> str:
+    v = raw.strip().strip("'\"")
+    if not v:
+        return ""
+    if v.lower().startswith("mtop."):
+        return v
+    # Heuristic: dotted API names usually represent method paths.
+    if v.count(".") >= 2:
+        return f"mtop.{v}"
+    return ""
+
+
+def _edge_key(e: dict) -> tuple[str, str, str, str]:
+    return (
+        str(e.get("source", "")),
+        str(e.get("target", "")),
+        str(e.get("relation", "")),
+        _norm_path(str(e.get("source_file", ""))),
+    )
+
+
+def _load_graph(path: Path) -> tuple[dict, str]:
+    data = json.loads(path.read_text(encoding="utf-8"))
+    if "nodes" not in data or not isinstance(data["nodes"], list):
+        raise ValueError("graph json missing 'nodes' list")
+    edge_key = "links" if "links" in data else "edges"
+    if edge_key not in data or not isinstance(data[edge_key], list):
+        raise ValueError("graph json missing 'links'/'edges' list")
+    return data, edge_key
+
+
+def _save_graph(path: Path, data: dict) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
+
+
+def _repo_tags_from_graph(nodes: list[dict]) -> list[str]:
+    tags = sorted({str(n.get("repo", "")).strip() for n in nodes if str(n.get("repo", "")).strip()})
+    if tags:
+        return tags
+    # Fallback for non-prefixed graphs: no repo partition available.
+    return ["default"]
+
+
+def _thin_graph_data(data: dict, edge_key: str, *, hops: int = 1) -> dict:
+    """Return a thin subgraph centered on mtop bridge chain nodes.
+
+    Keep:
+    - mtop API nodes (bridge_kind=mtop_api)
+    - repo navigation nodes (bridge_kind=repo)
+    - bridge edge endpoints (calls_mtop / implemented_by / depends_on_repo)
+    - optional neighborhood expansion by `hops` along existing edges.
+    """
+    nodes: list[dict] = data.get("nodes", [])
+    edges: list[dict] = data.get(edge_key, [])
+    by_id = {str(n.get("id", "")): n for n in nodes}
+
+    core_ids: set[str] = set()
+    for n in nodes:
+        nid = str(n.get("id", ""))
+        bk = str(n.get("bridge_kind", ""))
+        if bk in {"mtop_api", "repo"}:
+            core_ids.add(nid)
+
+    bridge_rels = {"calls_mtop", "implemented_by", "depends_on_repo"}
+    for e in edges:
+        if str(e.get("relation", "")) in bridge_rels:
+            core_ids.add(str(e.get("source", "")))
+            core_ids.add(str(e.get("target", "")))
+
+    # Expand by small neighborhood to preserve local navigability.
+    if hops > 0 and core_ids:
+        adj: dict[str, set[str]] = {}
+        for e in edges:
+            s = str(e.get("source", ""))
+            t = str(e.get("target", ""))
+            if not s or not t:
+                continue
+            adj.setdefault(s, set()).add(t)
+            adj.setdefault(t, set()).add(s)
+        frontier = set(core_ids)
+        keep_ids = set(core_ids)
+        for _ in range(hops):
+            nxt: set[str] = set()
+            for nid in frontier:
+                nxt.update(adj.get(nid, set()))
+            nxt -= keep_ids
+            if not nxt:
+                break
+            keep_ids.update(nxt)
+            frontier = nxt
+    else:
+        keep_ids = core_ids
+
+    thin_nodes = [by_id[nid] for nid in keep_ids if nid in by_id]
+    thin_edges = []
+    for e in edges:
+        s = str(e.get("source", ""))
+        t = str(e.get("target", ""))
+        if s in keep_ids and t in keep_ids:
+            thin_edges.append(e)
+
+    out = dict(data)
+    out["nodes"] = thin_nodes
+    out[edge_key] = thin_edges
+    out["bridge_thin"] = {
+        "enabled": True,
+        "hops": hops,
+        "seed_nodes": len(core_ids),
+        "nodes": len(thin_nodes),
+        "edges": len(thin_edges),
+    }
+    return out
+
+
+def _iter_code_files(repo_path: Path, *, max_file_size: int) -> Iterable[Path]:
+    for root, dirs, files in os.walk(repo_path):
+        dirs[:] = [d for d in dirs if d not in _SKIP_DIRS and not d.startswith(".cache")]
+        root_path = Path(root)
+        for name in files:
+            p = root_path / name
+            if p.suffix.lower() not in (_FRONTEND_EXTS | _BACKEND_EXTS):
+                continue
+            try:
+                if p.stat().st_size > max_file_size:
+                    continue
+            except OSError:
+                continue
+            yield p
+
+
+def _extract_api_facts(repo: str, repo_path: Path, file_path: Path) -> tuple[list[ApiFact], list[ApiFact]]:
+    rel = _norm_path(str(file_path.relative_to(repo_path)))
+    ext = file_path.suffix.lower()
+    is_frontend = ext in _FRONTEND_EXTS
+    is_backend = ext in _BACKEND_EXTS
+    if not is_frontend and not is_backend:
+        return [], []
+
+    try:
+        text = file_path.read_text(encoding="utf-8", errors="ignore")
+    except OSError:
+        return [], []
+
+    frontend: list[ApiFact] = []
+    backend: list[ApiFact] = []
+
+    for lineno, line in enumerate(text.splitlines(), 1):
+        for m in _MTOP_LITERAL_RE.finditer(line):
+            api = _norm_api(m.group(1))
+            if not api:
+                continue
+            fact = ApiFact(repo=repo, rel_path=rel, line=lineno, api=api, source="literal")
+            if is_frontend:
+                frontend.append(fact)
+            if is_backend:
+                backend.append(fact)
+
+        # Assignment-based fallback (weaker than literal mtop token).
+        for m in _API_ASSIGN_RE.finditer(line):
+            api = _norm_api(m.group(1))
+            if not api:
+                continue
+            fact = ApiFact(repo=repo, rel_path=rel, line=lineno, api=api, source="assign")
+            if is_frontend:
+                frontend.append(fact)
+            if is_backend:
+                backend.append(fact)
+
+    return frontend, backend
+
+
+def _parse_line_num(source_location: object) -> int | None:
+    if not isinstance(source_location, str):
+        return None
+    m = re.search(r"L(\d+)", source_location)
+    if not m:
+        return None
+    try:
+        return int(m.group(1))
+    except ValueError:
+        return None
+
+
+def _pick_symbol_node(
+    nodes: list[dict],
+    by_repo_source: dict[tuple[str, str], list[int]],
+    repo: str,
+    rel_path: str,
+    line: int,
+) -> str | None:
+    key = (repo, _norm_path(rel_path))
+    idxs = by_repo_source.get(key, [])
+    if not idxs:
+        return None
+
+    basename = Path(rel_path).name
+
+    def score(idx: int) -> tuple[int, int, int]:
+        n = nodes[idx]
+        s = 0
+        if n.get("file_type") == "code":
+            s += 50
+        label = str(n.get("label", ""))
+        # Prefer symbols over file-name wrapper nodes.
+        if label and label != basename:
+            s += 30
+        ln = _parse_line_num(n.get("source_location"))
+        if ln is not None:
+            s += 10
+            if ln <= line:
+                s += 5
+            dist = abs(line - ln)
+        else:
+            dist = 1_000_000
+        # Higher score first, then closer line.
+        return (s, -dist, -idx)
+
+    best_idx = max(idxs, key=score)
+    return str(nodes[best_idx].get("id", "")) or None
+
+
+def _api_node_id(api: str) -> str:
+    return "bridge_mtop_api__" + re.sub(r"[^a-z0-9]+", "_", api.lower()).strip("_")
+
+
+def _repo_node_id(repo: str) -> str:
+    return "bridge_repo__" + re.sub(r"[^a-z0-9]+", "_", repo.lower()).strip("_")
+
+
+def _ensure_api_node(nodes: list[dict], node_ids: set[str], api: str) -> tuple[str, bool]:
+    nid = _api_node_id(api)
+    if nid in node_ids:
+        return nid, False
+    node_ids.add(nid)
+    nodes.append(
+        {
+            "id": nid,
+            "label": api,
+            "norm_label": api.lower(),
+            "file_type": "concept",
+            "source_file": "bridge-mtop://derived",
+            "source_location": None,
+            "bridge_kind": "mtop_api",
+            "bridge_source": "bridge-mtop",
+        }
+    )
+    return nid, True
+
+
+def _ensure_repo_node(nodes: list[dict], node_ids: set[str], repo: str) -> tuple[str, bool]:
+    nid = _repo_node_id(repo)
+    if nid in node_ids:
+        return nid, False
+    node_ids.add(nid)
+    nodes.append(
+        {
+            "id": nid,
+            "label": f"repo:{repo}",
+            "norm_label": f"repo:{repo}".lower(),
+            "file_type": "concept",
+            "source_file": "bridge-mtop://derived",
+            "source_location": None,
+            "bridge_kind": "repo",
+            "bridge_source": "bridge-mtop",
+            "repo": repo,
+        }
+    )
+    return nid, True
+
+
+def bridge_mtop(
+    *,
+    graph_in: Path,
+    graph_out: Path,
+    repos_root: Path,
+    report_path: Path | None = None,
+    max_file_size: int = 1_000_000,
+    repo_tags: list[str] | None = None,
+    thin_out_path: Path | None = None,
+    thin_hops: int = 1,
+) -> BridgeStats:
+    data, edge_key = _load_graph(graph_in)
+    nodes: list[dict] = data["nodes"]
+    edges: list[dict] = data[edge_key]
+
+    node_ids = {str(n.get("id", "")) for n in nodes}
+
+    # Index nodes by (repo, source_file).
+    by_repo_source: dict[tuple[str, str], list[int]] = {}
+    for i, n in enumerate(nodes):
+        repo = str(n.get("repo", "")).strip() or "default"
+        src = _norm_path(str(n.get("source_file", "")))
+        if not src:
+            continue
+        by_repo_source.setdefault((repo, src), []).append(i)
+
+    tags = repo_tags or _repo_tags_from_graph(nodes)
+    frontend_facts: list[ApiFact] = []
+    backend_facts: list[ApiFact] = []
+    files_scanned = 0
+    repos_scanned = 0
+
+    for tag in tags:
+        repo_path = repos_root / tag if tag != "default" else repos_root
+        if not repo_path.exists():
+            continue
+        repos_scanned += 1
+        for p in _iter_code_files(repo_path, max_file_size=max_file_size):
+            files_scanned += 1
+            fe, be = _extract_api_facts(tag, repo_path, p)
+            frontend_facts.extend(fe)
+            backend_facts.extend(be)
+
+    # De-duplicate facts by (repo, rel_path, line, api, source).
+    frontend_facts = list(dict.fromkeys(frontend_facts))
+    backend_facts = list(dict.fromkeys(backend_facts))
+
+    existing_edges = {_edge_key(e) for e in edges}
+
+    api_nodes_added = 0
+    edge_calls_added = 0
+    edge_impl_added = 0
+    unresolved_frontend = 0
+    unresolved_backend = 0
+    repo_pair_counts: dict[tuple[str, str], int] = {}
+
+    backend_by_api: dict[str, list[str]] = {}
+    for fact in backend_facts:
+        target = _pick_symbol_node(nodes, by_repo_source, fact.repo, fact.rel_path, fact.line)
+        if not target:
+            unresolved_backend += 1
+            continue
+        backend_by_api.setdefault(fact.api, []).append(target)
+
+    for fact in frontend_facts:
+        source = _pick_symbol_node(nodes, by_repo_source, fact.repo, fact.rel_path, fact.line)
+        if not source:
+            unresolved_frontend += 1
+            continue
+
+        api_nid, created = _ensure_api_node(nodes, node_ids, fact.api)
+        if created:
+            api_nodes_added += 1
+
+        e = {
+            "source": source,
+            "target": api_nid,
+            "relation": "calls_mtop",
+            "context": "mtop_call",
+            "confidence": "EXTRACTED" if fact.source == "literal" else "INFERRED",
+            "confidence_score": 1.0 if fact.source == "literal" else 0.8,
+            "source_file": _norm_path(f"{fact.repo}/{fact.rel_path}"),
+            "source_location": f"L{fact.line}",
+            "bridge_source": "bridge-mtop",
+        }
+        k = _edge_key(e)
+        if k not in existing_edges:
+            edges.append(e)
+            existing_edges.add(k)
+            edge_calls_added += 1
+
+    # One API can be implemented in multiple repos/services.
+    for api, impl_nodes in backend_by_api.items():
+        api_nid, created = _ensure_api_node(nodes, node_ids, api)
+        if created:
+            api_nodes_added += 1
+        for impl in set(impl_nodes):
+            e = {
+                "source": api_nid,
+                "target": impl,
+                "relation": "implemented_by",
+                "context": "mtop_impl",
+                "confidence": "EXTRACTED",
+                "confidence_score": 1.0,
+                "source_file": "bridge-mtop://derived",
+                "source_location": None,
+                "bridge_source": "bridge-mtop",
+            }
+            k = _edge_key(e)
+            if k not in existing_edges:
+                edges.append(e)
+                existing_edges.add(k)
+                edge_impl_added += 1
+
+    # Optional repo-level navigation edges derived from API bridges.
+    # A->B means frontend repo A calls an mtop API implemented in backend repo B.
+    impl_repo_by_api: dict[str, set[str]] = {}
+    for fact in backend_facts:
+        if fact.api in backend_by_api:
+            impl_repo_by_api.setdefault(fact.api, set()).add(fact.repo)
+    for fact in frontend_facts:
+        for be_repo in impl_repo_by_api.get(fact.api, set()):
+            if be_repo == fact.repo:
+                continue
+            repo_pair_counts[(fact.repo, be_repo)] = repo_pair_counts.get((fact.repo, be_repo), 0) + 1
+    for (fe_repo, be_repo), count in repo_pair_counts.items():
+        src_repo_nid, _ = _ensure_repo_node(nodes, node_ids, fe_repo)
+        tgt_repo_nid, _ = _ensure_repo_node(nodes, node_ids, be_repo)
+        e = {
+            "source": src_repo_nid,
+            "target": tgt_repo_nid,
+            "relation": "depends_on_repo",
+            "context": "cross_repo_mtop",
+            "confidence": "EXTRACTED",
+            "confidence_score": 1.0,
+            "source_file": "bridge-mtop://derived",
+            "source_location": None,
+            "bridge_source": "bridge-mtop",
+            "weight": float(count),
+        }
+        k = _edge_key(e)
+        if k not in existing_edges:
+            edges.append(e)
+            existing_edges.add(k)
+
+    stats = BridgeStats(
+        repos_scanned=repos_scanned,
+        files_scanned=files_scanned,
+        frontend_facts=len(frontend_facts),
+        backend_facts=len(backend_facts),
+        api_nodes_added=api_nodes_added,
+        edge_calls_added=edge_calls_added,
+        edge_impl_added=edge_impl_added,
+        unresolved_frontend=unresolved_frontend,
+        unresolved_backend=unresolved_backend,
+    )
+
+    data["nodes"] = nodes
+    data[edge_key] = edges
+    _save_graph(graph_out, data)
+    if thin_out_path is not None:
+        _save_graph(thin_out_path, _thin_graph_data(data, edge_key, hops=max(0, thin_hops)))
+
+    if report_path is not None:
+        report_path.parent.mkdir(parents=True, exist_ok=True)
+        report_path.write_text(
+            json.dumps(
+                {
+                    "repos_scanned": stats.repos_scanned,
+                    "files_scanned": stats.files_scanned,
+                    "frontend_facts": stats.frontend_facts,
+                    "backend_facts": stats.backend_facts,
+                    "api_nodes_added": stats.api_nodes_added,
+                    "edge_calls_added": stats.edge_calls_added,
+                    "edge_impl_added": stats.edge_impl_added,
+                    "unresolved_frontend": stats.unresolved_frontend,
+                    "unresolved_backend": stats.unresolved_backend,
+                },
+                indent=2,
+                ensure_ascii=False,
+            ),
+            encoding="utf-8",
+        )
+
+    return stats