codegraph-network 0.1.0__py3-none-any.whl

codegraph/indexer.py

@@ -0,0 +1,190 @@
+"""Incremental repository indexer."""
+from __future__ import annotations
+
+import fnmatch
+import hashlib
+import os
+import sys
+from concurrent.futures import ProcessPoolExecutor
+from pathlib import Path
+
+from . import community, embed, links
+from .docs import DOC_EXT, DOC_LANGS, parse_doc
+from .parser import EXT, parse
+from .store import SCHEMA_VERSION, Store, recompute_rank
+
+SKIP_DIRS = {"node_modules", "venv", "__pycache__", "dist", "build", "target", "vendor", "site-packages"}
+MAX_BYTES = 1_000_000
+# bump when parser output changes; forces a full re-index
+PARSER_VERSION = 6  # 6: edge confidence, docs/SQL/PDF symbols
+IGNORE_FILE = ".codegraphignore"
+PARALLEL_MIN = 48  # below this, starting worker processes costs more than it saves
+ALL_EXT = {**EXT, **DOC_EXT}
+
+
+def db_path(root: Path) -> Path:
+    return root / ".codegraph" / "graph.db"
+
+
+def load_ignore(root: Path) -> list[tuple[str, bool, bool, bool]]:
+    """Parse `.codegraphignore` (gitignore-style, minus `**` and escapes) into
+    (pattern, negated, dir_only, anchored) rules. Later rules win, as in gitignore."""
+    try:
+        text = (root / IGNORE_FILE).read_text(errors="replace")
+    except OSError:
+        return []
+    rules = []
+    for line in text.splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        neg = line.startswith("!")
+        line = line.lstrip("!")
+        dir_only = line.endswith("/")
+        line = line.strip("/") if dir_only else line
+        anchored = line.startswith("/") or "/" in line
+        if line:
+            rules.append((line.lstrip("/"), neg, dir_only, anchored))
+    return rules
+
+
+def is_ignored(rel: str, is_dir: bool, rules) -> bool:
+    segs = rel.split("/")
+    hit = False
+    for pat, neg, dir_only, anchored in rules:
+        if anchored:
+            # the path itself, or any directory prefix of it
+            cands = ["/".join(segs[:i]) for i in range(1, len(segs) + 1)]
+            kinds = [True] * (len(segs) - 1) + [is_dir]
+        else:
+            cands, kinds = segs, [True] * (len(segs) - 1) + [is_dir]
+        if any(fnmatch.fnmatchcase(c, pat) and (k or not dir_only) for c, k in zip(cands, kinds, strict=True)):
+            hit = not neg
+    return hit
+
+
+def _scan(root: Path) -> dict[str, tuple[Path, str]]:
+    seen: dict[str, tuple[Path, str]] = {}
+    rules = load_ignore(root)
+    for dirpath, dirs, files in os.walk(root):
+        rel_dir = Path(dirpath).relative_to(root).as_posix()
+        rel_dir = "" if rel_dir == "." else rel_dir + "/"
+        dirs[:] = [d for d in dirs if d not in SKIP_DIRS and not d.startswith(".")
+                   and not (rules and is_ignored(rel_dir + d, True, rules))]
+        for f in files:
+            lang = ALL_EXT.get(Path(f).suffix.lower())
+            if not lang or (rules and is_ignored(rel_dir + f, False, rules)):
+                continue
+            p = Path(dirpath) / f
+            try:
+                if p.stat().st_size > MAX_BYTES:
+                    continue
+            except OSError:
+                continue
+            seen[p.relative_to(root).as_posix()] = (p, lang)
+    return seen
+
+
+def _parse_one(job):
+    """Worker: parse one file. Module level so it can be pickled into a process pool."""
+    data, lang, stem = job
+    try:
+        return (parse_doc if lang in DOC_LANGS else parse)(data, lang, stem), None
+    except Exception as e:  # a broken file must not kill the index
+        return None, str(e)
+
+
+def _parse_all(jobs: list) -> list:
+    """Parse in parallel across processes: tree-sitter plus the Python tree walk is CPU-bound, so
+    threads would not help. Falls back to serial if a pool cannot be started."""
+    if len(jobs) >= PARALLEL_MIN and (os.cpu_count() or 1) > 1:
+        try:
+            with ProcessPoolExecutor(max_workers=min(os.cpu_count() or 1, 8)) as pool:
+                return list(pool.map(_parse_one, jobs, chunksize=16))
+        except Exception as e:  # sandboxes without fork/semaphores, broken pool, ...
+            print(f"codegraph: parallel parse unavailable ({e}); parsing serially", file=sys.stderr)
+    return [_parse_one(j) for j in jobs]
+
+
+def index_repo(root: str | Path, force: bool = False, progress=None) -> dict:
+    """Update the index for `root`. Only changed files are re-parsed, and only the edges that
+    those changes can affect are re-resolved."""
+    root = Path(root).resolve()
+    store = Store(db_path(root))
+
+    def note(msg: str):
+        if progress is not None:
+            progress(msg)
+
+    # a database written by an older parser, or by an older schema, is rebuilt from scratch
+    if store.get_meta("parser_version") != str(PARSER_VERSION):
+        note("parser version changed, rebuilding index")
+        force = True
+
+    if force:
+        store.clear()  # one bulk delete instead of one per file
+    seen = _scan(root)
+    old = store.file_hashes()
+    removed = set(old) - set(seen)
+
+    # hash first, so we know the full dirty set before mutating anything
+    to_parse: list[tuple[str, Path, str, bytes, str]] = []
+    for rel, (p, lang) in sorted(seen.items()):
+        try:
+            data = p.read_bytes()
+        except OSError:
+            continue
+        h = hashlib.sha1(data).hexdigest()
+        if not force and old.get(rel) == h:
+            continue
+        to_parse.append((rel, p, lang, data, h))
+
+    dirty_files = set(removed) | {rel for rel, *_ in to_parse}
+    # names are captured before deletion: a definition that disappears can invalidate edges
+    # elsewhere that referred to it by name
+    dirty_names = store.names_in(dirty_files) if dirty_files else set()
+    # ...and importers must be captured before delete_file drops the rows that point at them,
+    # otherwise their import edges would never be recomputed and would stay missing
+    if dirty_files:
+        dirty_files |= store.importers_of(dirty_files)
+
+    for r in sorted(removed):
+        store.delete_file(r)
+    for rel, *_ in to_parse:
+        if rel in old:  # a new file has nothing to delete
+            store.delete_file(rel)
+    changed = 0
+    parsed_all = _parse_all([(data, lang, Path(rel).stem) for rel, _p, lang, data, _h in to_parse])
+    for (rel, _p, lang, data, h), (parsed, err) in zip(to_parse, parsed_all, strict=True):
+        if parsed is None:
+            print(f"codegraph: skipped {rel}: {err}", file=sys.stderr)  # stdout is the MCP channel
+            continue
+        store.add_file(rel, lang, h, parsed, data.count(b"\n") + 1)
+        dirty_names |= {s.name for s in parsed.syms}
+        changed += 1
+        if changed % 200 == 0:
+            note(f"parsed {changed} files")
+
+    dirty_names -= {None}
+    resolve_stats: dict = {}
+    rank_stats: dict = {}
+    link_stats: dict = {}
+    if changed or removed or force or store.migrated:
+        resolve_stats = store.resolve(full=force or not dirty_names,
+                                      dirty_files=dirty_files,
+                                      dirty_names=dirty_names)
+        store.set_meta("parser_version", str(PARSER_VERSION))
+        store.set_meta("schema_version", str(SCHEMA_VERSION))
+        rank_stats = recompute_rank(store)
+        link_stats.update(links.link_static(store.db))
+        link_stats["communities"] = community.detect(store.db)
+    link_stats["embed"] = embed.update(store, note)
+    link_stats["cochange"] = links.link_cochange(store, root, force)  # -1: unchanged, 0: no git
+    store.commit()
+    db = store.db
+    n_sym = db.execute("SELECT COUNT(*) FROM symbols").fetchone()[0]
+    n_edge = db.execute("SELECT COUNT(*) FROM edges").fetchone()[0]
+    store.close()
+    return {"files": len(seen), "changed": changed, "removed": len(removed),
+            "symbols": n_sym, "edges": n_edge,
+            "resolve": resolve_stats, "rank": rank_stats, "links": link_stats}

codegraph/install.py

@@ -0,0 +1,219 @@
+"""`codegraph install` and `codegraph hook`: wire the graph into coding assistants and into git.
+
+Everything written here is idempotent and reversible: instruction text lives between markers inside
+files the user may also edit by hand, JSON configs are merged key by key, and git hooks are appended to
+rather than replaced.
+"""
+from __future__ import annotations
+
+import json
+import os
+import stat
+import subprocess
+from pathlib import Path
+
+START, END = "<!-- codegraph:start -->", "<!-- codegraph:end -->"
+SH_START, SH_END = "# >>> codegraph >>>", "# <<< codegraph <<<"
+GIT_HOOKS = ("post-commit", "post-checkout", "post-merge", "post-rewrite")
+NUDGE_MARK = "codegraph-nudge"  # lets us find our own hook entry again
+PLATFORMS = ("claude", "cursor", "gemini", "agents")
+
+GUIDE = """\
+## Code navigation: use codegraph before reading files
+
+This repository is indexed by codegraph (symbols, call graph, imports, docs, SQL). Ask the graph instead
+of reading whole files or grepping:
+
+1. `repo_map` for an overview (pass `focus=` to personalise it), `search` to find a symbol by name or meaning.
+2. `context <symbol>` for its source, what it uses and who uses it, in one reply.
+3. `impact <symbol>` before changing anything; `review` before opening a PR (what the diff touches,
+   blast radius, untested changes).
+4. `callers` / `callees` / `related` / `deps` to walk the graph.
+
+Edges tagged `[inferred]` or `[ambiguous]` (or `?` in `impact`) are name-based guesses, so treat them as
+upper bounds and confirm by reading the code. Read files directly only for exact surrounding code.
+
+No MCP server connected? The same tools run from the shell: `codegraph context <symbol>`,
+`codegraph impact <symbol>`, `codegraph review`. Exit code 3 means the name is ambiguous, 4 that it was
+not found.
+"""
+
+SKILL = f"""---
+name: codegraph
+description: Navigate this codebase through its code graph (symbols, callers, impact, docs, SQL) instead of reading whole files. Use before searching or reading code, before editing a function, and before opening a PR.
+---
+
+{GUIDE}"""
+
+NUDGE = ("codegraph is available for this repo: try repo_map / search / context before grepping or "
+         "reading whole files (CLI: codegraph search <query>).")
+
+
+def _write(path: Path, text: str, report: list[str], what: str):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if path.exists() and path.read_text() == text:
+        report.append(f"unchanged  {what}")
+        return
+    verb = "updated" if path.exists() else "created"
+    path.write_text(text)
+    report.append(f"{verb:<10} {what}")
+
+
+def _block(text: str) -> str:
+    return f"{START}\n{text.rstrip()}\n{END}\n"
+
+
+def _upsert_block(path: Path, text: str, report: list[str], what: str):
+    """Put `text` between markers in a file that may hold other content; replace it if already there."""
+    block = _block(text)
+    cur = path.read_text() if path.exists() else ""
+    if START in cur and END in cur:
+        a, b = cur.index(START), cur.index(END) + len(END)
+        new = cur[:a] + block.rstrip("\n") + cur[b:]
+    else:
+        new = cur + ("\n" if cur and not cur.endswith("\n\n") else "") + block
+    _write(path, new, report, what)
+
+
+def _load_json(path: Path) -> dict:
+    if not path.exists():
+        return {}
+    try:
+        data = json.loads(path.read_text() or "{}")
+    except json.JSONDecodeError as e:
+        raise SystemExit(f"codegraph: {path} is not valid JSON ({e}); fix it or move it aside, then re-run") from e
+    if not isinstance(data, dict):
+        raise SystemExit(f"codegraph: {path} must hold a JSON object")
+    return data
+
+
+def _merge_mcp(path: Path, report: list[str], what: str):
+    data = _load_json(path)
+    servers = data.setdefault("mcpServers", {})
+    # relative root: MCP clients start servers in the project directory, so the config stays portable
+    servers["codegraph"] = {"command": "codegraph", "args": ["-r", ".", "serve"]}
+    _write(path, json.dumps(data, indent=2) + "\n", report, what)
+
+
+def _claude_hook(root: Path, report: list[str]):
+    path = root / ".claude" / "settings.json"
+    data = _load_json(path)
+    payload = json.dumps({"hookSpecificOutput": {"hookEventName": "PreToolUse", "additionalContext": NUDGE}})
+    entry = {"matcher": "Glob|Grep", "hooks": [{"type": "command", "command": f"echo '{payload}' # {NUDGE_MARK}"}]}
+    pre = data.setdefault("hooks", {}).setdefault("PreToolUse", [])
+    pre[:] = [e for e in pre if NUDGE_MARK not in json.dumps(e)] + [entry]
+    _write(path, json.dumps(data, indent=2) + "\n", report, ".claude/settings.json (Glob|Grep hint)")
+
+
+def install(root: str | Path, platform: str = "all", hooks: bool = True) -> list[str]:
+    root = Path(root).resolve()
+    chosen = PLATFORMS if platform == "all" else (platform,)
+    unknown = [p for p in chosen if p not in PLATFORMS]
+    if unknown:
+        raise SystemExit(f"codegraph: unknown platform {unknown[0]!r}; choose from {', '.join(PLATFORMS)}, all")
+    report: list[str] = []
+    for p in chosen:
+        if p == "claude":
+            _write(root / ".claude" / "skills" / "codegraph" / "SKILL.md", SKILL, report,
+                   ".claude/skills/codegraph/SKILL.md")
+            _merge_mcp(root / ".mcp.json", report, ".mcp.json (MCP server)")
+            _upsert_block(root / "CLAUDE.md", GUIDE, report, "CLAUDE.md (guidance block)")
+            if hooks:
+                _claude_hook(root, report)
+        elif p == "cursor":
+            rule = ("---\ndescription: Navigate the codebase with codegraph before reading files\n"
+                    f"alwaysApply: true\n---\n\n{GUIDE}")
+            _write(root / ".cursor" / "rules" / "codegraph.mdc", rule, report, ".cursor/rules/codegraph.mdc")
+            _merge_mcp(root / ".cursor" / "mcp.json", report, ".cursor/mcp.json (MCP server)")
+        elif p == "gemini":
+            _upsert_block(root / "GEMINI.md", GUIDE, report, "GEMINI.md (guidance block)")
+            _merge_mcp(root / ".gemini" / "settings.json", report, ".gemini/settings.json (MCP server)")
+        elif p == "agents":  # Codex, opencode, Aider (via --read), Copilot and others read AGENTS.md
+            _upsert_block(root / "AGENTS.md", GUIDE, report, "AGENTS.md (guidance block)")
+    return report
+
+
+def uninstall(root: str | Path) -> list[str]:
+    """Remove what `install` wrote. Files that held only our block are deleted."""
+    root = Path(root).resolve()
+    report: list[str] = []
+    for rel in ("CLAUDE.md", "GEMINI.md", "AGENTS.md"):
+        path = root / rel
+        if path.exists() and START in (cur := path.read_text()) and END in cur:
+            rest = (cur[:cur.index(START)] + cur[cur.index(END) + len(END):]).strip()
+            if rest:
+                path.write_text(rest + "\n")
+            else:
+                path.unlink()
+            report.append(f"cleaned    {rel}")
+    for rel in (".claude/skills/codegraph/SKILL.md", ".cursor/rules/codegraph.mdc"):
+        if (root / rel).exists():
+            (root / rel).unlink()
+            report.append(f"removed    {rel}")
+    for rel in (".mcp.json", ".cursor/mcp.json", ".gemini/settings.json"):
+        path = root / rel
+        if path.exists() and "codegraph" in _load_json(path).get("mcpServers", {}):
+            data = _load_json(path)
+            del data["mcpServers"]["codegraph"]
+            path.write_text(json.dumps(data, indent=2) + "\n")
+            report.append(f"cleaned    {rel}")
+    path = root / ".claude" / "settings.json"
+    if path.exists() and NUDGE_MARK in path.read_text():
+        data = _load_json(path)
+        pre = data.get("hooks", {}).get("PreToolUse", [])
+        pre[:] = [e for e in pre if NUDGE_MARK not in json.dumps(e)]
+        path.write_text(json.dumps(data, indent=2) + "\n")
+        report.append("cleaned    .claude/settings.json")
+    return report or ["nothing to remove"]
+
+
+# ---- git hooks ----
+
+def _hooks_dir(root: Path) -> Path:
+    p = subprocess.run(["git", "-C", str(root), "rev-parse", "--git-path", "hooks"],
+                       capture_output=True, text=True)
+    if p.returncode != 0:
+        raise SystemExit("codegraph: not a git repository")
+    d = Path(p.stdout.strip())
+    return d if d.is_absolute() else root / d
+
+
+def _hook_block() -> str:
+    # backgrounded: a commit or checkout must never wait for, or fail because of, the indexer
+    return (f"{SH_START}\ncommand -v codegraph >/dev/null 2>&1 && "
+            f'(codegraph -r "$(git rev-parse --show-toplevel)" index >/dev/null 2>&1 &)\n{SH_END}\n')
+
+
+def hook_install(root: str | Path) -> list[str]:
+    root = Path(root).resolve()
+    d = _hooks_dir(root)
+    d.mkdir(parents=True, exist_ok=True)
+    report = []
+    for name in GIT_HOOKS:
+        path = d / name
+        cur = path.read_text() if path.exists() else "#!/bin/sh\n"
+        if SH_START in cur:
+            a, b = cur.index(SH_START), cur.index(SH_END) + len(SH_END)
+            new = cur[:a] + _hook_block().rstrip("\n") + cur[b:]
+        else:
+            new = cur + ("" if cur.endswith("\n") else "\n") + _hook_block()
+        path.write_text(new)
+        path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+        report.append(f"installed  {os.path.relpath(path, root)}")
+    return report
+
+
+def hook_uninstall(root: str | Path) -> list[str]:
+    root = Path(root).resolve()
+    d, report = _hooks_dir(root), []
+    for name in GIT_HOOKS:
+        path = d / name
+        if not path.exists() or SH_START not in (cur := path.read_text()):
+            continue
+        rest = (cur[:cur.index(SH_START)] + cur[cur.index(SH_END) + len(SH_END):]).strip()
+        if rest in ("", "#!/bin/sh"):
+            path.unlink()
+        else:
+            path.write_text(rest + "\n")
+        report.append(f"removed    {os.path.relpath(path, root)}")
+    return report or ["no codegraph hooks found"]

codegraph/keys.py

@@ -0,0 +1,94 @@
+"""String-keyed hidden links: routes, event names and config keys.
+
+Two symbols that mention the same string literal in a recognisable role (a route path registered
+here and requested there, an event emitted here and listened to there) are related even though
+no call edge connects them. Recognition is table-driven: add a `Pattern` row to support a new
+framework. The parser feeds this module the callee text and the first string argument of every
+call; resolution (matching roles across files) lives in links.py.
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+PY = frozenset({"python"})
+JS = frozenset({"javascript", "typescript", "tsx"})
+ANY = PY | JS
+
+_HTTP = "get|post|put|patch|delete|head|options"
+_STRING = re.compile(r"""^[rbfuRBFU]{0,2}(\"\"\"|'''|"|'|`)(.*)\1$""", re.DOTALL)
+# events every Node/DOM program emits and listens to: linking on them would connect unrelated code
+COMMON_EVENTS = frozenset(
+    "error data end close exit open message connect finish drain readable line abort timeout "
+    "click change input submit keydown keyup load resize sigint sigterm uncaughtexception "
+    "unhandledrejection".split())
+
+
+def _route(k: str) -> bool:
+    return k.startswith("/") and len(k) > 1 and " " not in k
+
+
+def _event(k: str) -> bool:
+    return len(k) >= 3 and k.lower() not in COMMON_EVENTS and re.fullmatch(r"[\w.:/@-]+", k) is not None
+
+
+def _config(k: str) -> bool:
+    return len(k) >= 3 and re.fullmatch(r"[\w.:-]+", k) is not None
+
+
+@dataclass(frozen=True)
+class Pattern:
+    langs: frozenset
+    callee: re.Pattern  # matched against the whole callee text, e.g. `app.get`
+    ns: str  # namespace of the key: keys only match inside one namespace
+    role: str  # see links.KEY_ROLES for how roles pair up
+    valid: object  # str -> bool, rejects keys that are too generic to mean anything
+    decorator: bool | None = None  # True: only as a decorator, False: only as a plain call
+
+
+def _p(langs, callee, ns, role, valid, decorator=None) -> Pattern:
+    return Pattern(langs, re.compile(callee), ns, role, valid, decorator)
+
+
+# Add rows here to teach codegraph a new framework. Roles per namespace: see links.KEY_ROLES.
+PATTERNS: tuple[Pattern, ...] = (
+    # routes: registered (provides) vs requested (uses)
+    _p(PY, rf"^\w+\.({_HTTP}|route|websocket|api_route)$", "route", "provides", _route, decorator=True),
+    _p(PY, rf"^(?:\w+\.)*(client|requests|httpx|session|http|api)\.({_HTTP})$", "route", "uses", _route,
+       decorator=False),
+    _p(JS, rf"^(app|router|server|\w+Router)\.({_HTTP}|all)$", "route", "provides", _route),
+    _p(JS, rf"^(fetch|axios|ky|\$http)$|^(?:\w+\.)*(axios|client|http|api|request|supertest)\.({_HTTP})$",
+       "route", "uses", _route),
+    # events: emitted vs listened to
+    _p(ANY, r"(?:^|\.)(emit|dispatch|dispatchEvent|trigger|publish)$", "event", "emits", _event),
+    _p(ANY, r"(?:^|\.)(on|once|addListener|addEventListener|subscribe|prependListener)$",
+       "event", "listens", _event),
+    # config keys: everything that reads the same key is related
+    _p(PY, r"(?:^|\.)(getenv|environ\.get|environ\.setdefault)$", "config", "reads", _config),
+    _p(ANY, r"^(?:self\.|this\.)?(config|settings|cfg|conf|env)\.(get|getint|getboolean|require)$",
+       "config", "reads", _config),
+)
+
+
+def string_value(text: str) -> str | None:
+    """The contents of a plain string literal, or None for anything computed or interpolated."""
+    m = _STRING.match(text.strip())
+    if not m:
+        return None
+    body = m.group(2)
+    if m.group(1) == "`" and "${" in body:
+        return None
+    if text.lstrip()[:1] in "fF" and "{" in body:  # f-string: not a constant
+        return None
+    return body
+
+
+def match(lang: str, callee: str, key: str, decorator: bool) -> list[tuple[str, str, str]]:
+    """Every (namespace, key, role) a call `callee("key", ...)` contributes."""
+    out = []
+    for p in PATTERNS:
+        if lang not in p.langs or (p.decorator is not None and p.decorator != decorator):
+            continue
+        if p.callee.search(callee) and p.valid(key):
+            out.append((p.ns, key, p.role))
+    return out