codmap 0.0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codemap/__init__.py +10 -0
- codemap/apidiff.py +208 -0
- codemap/arch.py +190 -0
- codemap/cli.py +718 -0
- codemap/diagnostics.py +256 -0
- codemap/extract/__init__.py +10 -0
- codemap/extract/attrflow.py +230 -0
- codemap/extract/behavior.py +771 -0
- codemap/extract/dataflow.py +97 -0
- codemap/extract/dispatch.py +248 -0
- codemap/extract/griffe_extractor.py +496 -0
- codemap/extract/gsource.py +83 -0
- codemap/extract/roots.py +427 -0
- codemap/freshness.py +94 -0
- codemap/incremental.py +195 -0
- codemap/integrations/__init__.py +51 -0
- codemap/integrations/base.py +196 -0
- codemap/integrations/cocoindex.py +78 -0
- codemap/integrations/gate.py +58 -0
- codemap/integrations/gitnexus.py +93 -0
- codemap/integrations/registry.py +69 -0
- codemap/integrations/transport.py +46 -0
- codemap/model.py +178 -0
- codemap/provenance.py +248 -0
- codemap/query.py +1164 -0
- codemap/scope.py +212 -0
- codemap/serve/__init__.py +26 -0
- codemap/serve/_scip_pb2.py +100 -0
- codemap/serve/api_surface.py +60 -0
- codemap/serve/apidiff.py +83 -0
- codemap/serve/architecture.py +101 -0
- codemap/serve/audit.py +176 -0
- codemap/serve/check.py +80 -0
- codemap/serve/ctags.py +203 -0
- codemap/serve/impact.py +84 -0
- codemap/serve/livingdocs.py +174 -0
- codemap/serve/mcp_server.py +278 -0
- codemap/serve/mermaid.py +120 -0
- codemap/serve/pack.py +93 -0
- codemap/serve/rag.py +142 -0
- codemap/serve/review.py +197 -0
- codemap/serve/scip.py +183 -0
- codemap/serve/semantic.py +71 -0
- codemap/serve/server.py +43 -0
- codemap/serve/session.py +482 -0
- codemap/serve/subsystems.py +85 -0
- codemap/serve/vault.py +156 -0
- codemap/store.py +28 -0
- codemap/tomlio.py +59 -0
- codmap-0.0.3.dist-info/METADATA +245 -0
- codmap-0.0.3.dist-info/RECORD +55 -0
- codmap-0.0.3.dist-info/WHEEL +5 -0
- codmap-0.0.3.dist-info/entry_points.txt +2 -0
- codmap-0.0.3.dist-info/licenses/LICENSE +21 -0
- codmap-0.0.3.dist-info/top_level.txt +1 -0
codemap/serve/rag.py
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""RAG export — consumer A (DESIGN §1-A, §4, M2.1).
|
|
2
|
+
|
|
3
|
+
One self-contained *chunk* per symbol for retrieval-augmented generation: what it
|
|
4
|
+
is, where it lives, and its graph neighborhood (calls / callers / bases /
|
|
5
|
+
subclasses / module). Emits JSONL (one chunk per line — the standard RAG ingest
|
|
6
|
+
format); the ``text`` field is a compact rendering ready to embed.
|
|
7
|
+
|
|
8
|
+
The neighborhood is exactly what a plain source read does NOT cheaply give an AI:
|
|
9
|
+
"what calls this", "what this returns and who consumes it", "its subclasses" —
|
|
10
|
+
resolved across the whole package, not one file.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
|
|
17
|
+
from codemap.query import Query
|
|
18
|
+
|
|
19
|
+
_SYMBOL_KINDS = {"class", "function"}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def build_chunks(query: Query) -> list[dict]:
|
|
23
|
+
"""Structured RAG chunks for every class/function symbol, sorted by id."""
|
|
24
|
+
graph = query.graph
|
|
25
|
+
chunks = []
|
|
26
|
+
for node in sorted(graph.nodes.values(), key=lambda n: n.id):
|
|
27
|
+
if node.kind not in _SYMBOL_KINDS:
|
|
28
|
+
continue
|
|
29
|
+
module = node.id.rsplit(".", 1)[0]
|
|
30
|
+
neighbors = _neighbors(query, node)
|
|
31
|
+
chunk = {
|
|
32
|
+
"id": node.id,
|
|
33
|
+
"kind": node.kind,
|
|
34
|
+
"module": module,
|
|
35
|
+
"file": node.file,
|
|
36
|
+
"lines": [node.lineno, node.endlineno],
|
|
37
|
+
"signature": node.signature,
|
|
38
|
+
"docstring": node.docstring,
|
|
39
|
+
"deprecated": node.is_deprecated or None,
|
|
40
|
+
"neighbors": neighbors,
|
|
41
|
+
"text": _embed_text(node, module, neighbors),
|
|
42
|
+
}
|
|
43
|
+
chunks.append({k: v for k, v in chunk.items() if v not in (None, {}, [])})
|
|
44
|
+
return chunks
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def render_rag(query: Query) -> str:
|
|
48
|
+
"""RAG chunks as JSONL (one JSON object per line)."""
|
|
49
|
+
return "".join(
|
|
50
|
+
json.dumps(c, ensure_ascii=False) + "\n" for c in build_chunks(query)
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _neighbors(query: Query, node) -> dict:
|
|
55
|
+
n: dict = {}
|
|
56
|
+
if node.kind == "function":
|
|
57
|
+
callees = query.callees(node.id)
|
|
58
|
+
callers = query.callers(node.id)
|
|
59
|
+
if callees:
|
|
60
|
+
n["calls"] = callees
|
|
61
|
+
if callers:
|
|
62
|
+
n["called_by"] = callers
|
|
63
|
+
ret = node.extras.get("returns")
|
|
64
|
+
if ret:
|
|
65
|
+
n["returns"] = ret
|
|
66
|
+
if node.kind == "class":
|
|
67
|
+
bases = query.bases(node.id)
|
|
68
|
+
subs = query.subclasses(node.id)
|
|
69
|
+
if bases:
|
|
70
|
+
n["bases"] = bases
|
|
71
|
+
if subs:
|
|
72
|
+
n["subclasses"] = subs
|
|
73
|
+
reg = node.extras.get("registry")
|
|
74
|
+
if reg:
|
|
75
|
+
n["registered_as"] = reg.get("key")
|
|
76
|
+
# M9/F4: registry family — the Protocol satisfied and its implementers,
|
|
77
|
+
# neither of which is reachable via inheritance (structural typing).
|
|
78
|
+
impls = query.implements(node.id)
|
|
79
|
+
implers = query.implementers(node.id)
|
|
80
|
+
if impls:
|
|
81
|
+
n["implements"] = impls
|
|
82
|
+
if implers:
|
|
83
|
+
n["implementers"] = implers
|
|
84
|
+
# M10/F3: aggregate the call-neighbours of the class' own methods so the
|
|
85
|
+
# class chunk is self-sufficient. A class' behaviour (e.g. a deprecated
|
|
86
|
+
# wrapper delegating to `analyze_zones`) lives on its methods; a retriever
|
|
87
|
+
# pulling the class chunk otherwise never sees that delegation seam.
|
|
88
|
+
via = _methods_calls(query, node.id)
|
|
89
|
+
if via:
|
|
90
|
+
n["calls_via_methods"] = via
|
|
91
|
+
return n
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _methods_calls(query: Query, class_id: str) -> list[dict]:
|
|
95
|
+
"""Distinct call targets of the class' methods, each tagged with one via-method."""
|
|
96
|
+
prefix = class_id + "."
|
|
97
|
+
seen: dict[str, str] = {} # target -> via-method (first, deterministic)
|
|
98
|
+
for mid in sorted(query.graph.nodes):
|
|
99
|
+
if not mid.startswith(prefix):
|
|
100
|
+
continue
|
|
101
|
+
if query.graph.nodes[mid].kind != "function":
|
|
102
|
+
continue
|
|
103
|
+
for tgt in query.callees(mid):
|
|
104
|
+
if tgt.startswith(prefix):
|
|
105
|
+
continue # sibling method — internal, not an outward seam
|
|
106
|
+
seen.setdefault(tgt, mid)
|
|
107
|
+
return [{"target": t, "via": seen[t]} for t in sorted(seen)]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _embed_text(node, module, neighbors) -> str:
|
|
111
|
+
"""A compact, self-contained string for the embedding model."""
|
|
112
|
+
name = node.id.rsplit(".", 1)[-1]
|
|
113
|
+
parts = [f"{node.kind} {name} in {module}."]
|
|
114
|
+
if node.signature:
|
|
115
|
+
parts.append(f"Signature: {node.signature}.")
|
|
116
|
+
if node.docstring:
|
|
117
|
+
first = node.docstring.strip().splitlines()[0].strip()
|
|
118
|
+
if first:
|
|
119
|
+
parts.append(first)
|
|
120
|
+
if neighbors.get("bases"):
|
|
121
|
+
parts.append("Inherits: " + ", ".join(_short(b) for b in neighbors["bases"]) + ".")
|
|
122
|
+
if neighbors.get("calls"):
|
|
123
|
+
parts.append("Calls: " + ", ".join(_short(c) for c in neighbors["calls"][:8]) + ".")
|
|
124
|
+
if neighbors.get("called_by"):
|
|
125
|
+
parts.append("Called by: " + ", ".join(_short(c) for c in neighbors["called_by"][:8]) + ".")
|
|
126
|
+
if neighbors.get("registered_as"):
|
|
127
|
+
parts.append(f"Registered as '{neighbors['registered_as']}'.")
|
|
128
|
+
if neighbors.get("implements"):
|
|
129
|
+
parts.append("Implements: " + ", ".join(_short(p) for p in neighbors["implements"]) + ".")
|
|
130
|
+
if neighbors.get("implementers"):
|
|
131
|
+
parts.append("Implemented by: " + ", ".join(_short(c) for c in neighbors["implementers"][:8]) + ".")
|
|
132
|
+
if neighbors.get("calls_via_methods"):
|
|
133
|
+
seams = ", ".join(
|
|
134
|
+
f"{_short(v['target'])} (via {_short(v['via'])})"
|
|
135
|
+
for v in neighbors["calls_via_methods"][:8]
|
|
136
|
+
)
|
|
137
|
+
parts.append("Methods call: " + seams + ".")
|
|
138
|
+
return " ".join(parts)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _short(node_id: str) -> str:
|
|
142
|
+
return node_id.rsplit(".", 1)[-1]
|
codemap/serve/review.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"""Change-set review — turn a diff into a consolidated review dossier (M15/F17).
|
|
2
|
+
|
|
3
|
+
The reviewer's real input is a **diff** (files + changed line ranges), not a symbol
|
|
4
|
+
name. A11 dogfood found the pieces existed (impact, call_contract, columns_of,
|
|
5
|
+
references) but nothing stitched them: 4 changed symbols meant ~20 manual op-calls.
|
|
6
|
+
|
|
7
|
+
``build_review`` resolves hunks → symbols (via ``Query.symbols_in_range``), then per
|
|
8
|
+
symbol assembles callers / signature-change contract / touched columns / cross-root
|
|
9
|
+
consumers, plus a synthesized **risk** rank (the signals are all present — F17/R3),
|
|
10
|
+
a union blast-radius summary, and a risk-sorted order. ``render_review`` is the
|
|
11
|
+
human markdown. No schema change — pure aggregation over existing edges.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from codemap.query import Query
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def resolve_hunks(query: Query, hunks: list[dict]) -> list[str]:
|
|
20
|
+
"""[{file, ranges: [[start,end], …]}] → distinct changed symbol ids."""
|
|
21
|
+
changed: set[str] = set()
|
|
22
|
+
for h in hunks:
|
|
23
|
+
file = h["file"]
|
|
24
|
+
for rng in h.get("ranges", []):
|
|
25
|
+
start, end = int(rng[0]), int(rng[-1])
|
|
26
|
+
changed.update(query.symbols_in_range(file, start, end))
|
|
27
|
+
return sorted(changed)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _consumers_by_root(query: Query, sid: str) -> dict[str, int]:
|
|
31
|
+
by_root: dict[str, int] = {}
|
|
32
|
+
for ref in query.references_to(sid):
|
|
33
|
+
by_root[ref["root"]] = by_root.get(ref["root"], 0) + 1
|
|
34
|
+
return by_root
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _risk(caller_n: int, consumers: dict[str, int], contract_sites: int,
|
|
38
|
+
touches_columns: bool) -> tuple[str, int]:
|
|
39
|
+
"""Synthesize a review-priority label from the blast signals (F17/R3).
|
|
40
|
+
|
|
41
|
+
Heuristic, honest and documented — not a proof: more inbound reach, more
|
|
42
|
+
external (cross-root) consumers, more distinct call-site shapes, and dataflow
|
|
43
|
+
contact all raise how much a change here warrants scrutiny.
|
|
44
|
+
"""
|
|
45
|
+
external = sum(v for r, v in consumers.items() if r != "core")
|
|
46
|
+
score = caller_n + 2 * external + contract_sites + (1 if touches_columns else 0)
|
|
47
|
+
if external or score >= 5:
|
|
48
|
+
return "high", score
|
|
49
|
+
if score >= 2:
|
|
50
|
+
return "medium", score
|
|
51
|
+
return "low", score
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def build_symbol_dossier(query: Query, sid: str) -> dict:
|
|
55
|
+
"""Everything a reviewer needs about one changed symbol (F17)."""
|
|
56
|
+
node = query.graph.nodes.get(sid)
|
|
57
|
+
kind = node.kind if node else None
|
|
58
|
+
callers = query.callers(sid)
|
|
59
|
+
contract = query.call_contract(sid)
|
|
60
|
+
cols = query.columns_of(sid) if kind == "function" else {"reads": [], "writes": []}
|
|
61
|
+
consumers = _consumers_by_root(query, sid)
|
|
62
|
+
touches = bool(cols["reads"] or cols["writes"])
|
|
63
|
+
risk, score = _risk(len(callers), consumers, len(contract), touches)
|
|
64
|
+
return {
|
|
65
|
+
"symbol": sid,
|
|
66
|
+
"kind": kind,
|
|
67
|
+
"file": node.file if node else None,
|
|
68
|
+
"lines": [node.lineno, node.endlineno] if node else None,
|
|
69
|
+
"callers": callers,
|
|
70
|
+
"callees": query.callees(sid) if kind == "function" else [],
|
|
71
|
+
"call_contract": contract, # signature-change surface (F7)
|
|
72
|
+
"columns": cols, # dataflow contact (F6/F11)
|
|
73
|
+
"consumers_by_root": consumers, # cross-root blast (F1)
|
|
74
|
+
"risk": risk,
|
|
75
|
+
"risk_score": score,
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def build_review(query: Query, *, hunks: list[dict] | None = None,
|
|
80
|
+
symbols: list[str] | None = None, base_graph=None) -> dict:
|
|
81
|
+
"""Consolidated change-set review from diff hunks and/or explicit symbols.
|
|
82
|
+
|
|
83
|
+
Returns ``{changed: [dossier…], summary}`` with dossiers risk-sorted (highest
|
|
84
|
+
first). ``summary`` carries the symbol count, union blast-radius by root, and
|
|
85
|
+
the count of unresolved hunks (module-level / unknown files) so nothing is
|
|
86
|
+
silently dropped.
|
|
87
|
+
|
|
88
|
+
``base_graph`` (R1-C5): a 'before' graph to API-diff against. Hunk-based review
|
|
89
|
+
sees only *modified* lines; the diff adds the symbols hunks miss — **removed**
|
|
90
|
+
and **added** public symbols and **breaking** signature changes — under
|
|
91
|
+
``api_diff``.
|
|
92
|
+
"""
|
|
93
|
+
ids = set(symbols or [])
|
|
94
|
+
unresolved = []
|
|
95
|
+
if hunks:
|
|
96
|
+
for h in hunks:
|
|
97
|
+
file = h["file"]
|
|
98
|
+
for rng in h.get("ranges", []):
|
|
99
|
+
start, end = int(rng[0]), int(rng[-1])
|
|
100
|
+
got = query.symbols_in_range(file, start, end)
|
|
101
|
+
if got:
|
|
102
|
+
ids.update(got)
|
|
103
|
+
else:
|
|
104
|
+
unresolved.append({"file": file, "range": [start, end]})
|
|
105
|
+
ids = {query.canonical(i) or i for i in ids}
|
|
106
|
+
dossiers = [build_symbol_dossier(query, sid) for sid in sorted(ids)]
|
|
107
|
+
dossiers.sort(key=lambda d: (-d["risk_score"], d["symbol"]))
|
|
108
|
+
|
|
109
|
+
union_by_root: dict[str, int] = {}
|
|
110
|
+
for d in dossiers:
|
|
111
|
+
for r, v in d["consumers_by_root"].items():
|
|
112
|
+
union_by_root[r] = union_by_root.get(r, 0) + v
|
|
113
|
+
summary = {
|
|
114
|
+
"changed_symbols": len(dossiers),
|
|
115
|
+
"blast_by_root": union_by_root,
|
|
116
|
+
"high_risk": [d["symbol"] for d in dossiers if d["risk"] == "high"],
|
|
117
|
+
"unresolved_hunks": unresolved,
|
|
118
|
+
}
|
|
119
|
+
result = {"changed": dossiers, "summary": summary}
|
|
120
|
+
if base_graph is not None:
|
|
121
|
+
from codemap.serve.apidiff import build_apidiff
|
|
122
|
+
result["api_diff"] = build_apidiff(base_graph, query.graph)
|
|
123
|
+
return result
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def render_review(query: Query, *, hunks: list[dict] | None = None,
|
|
127
|
+
symbols: list[str] | None = None, base_graph=None) -> str:
|
|
128
|
+
"""Human markdown for a change-set review (highest risk first)."""
|
|
129
|
+
rv = build_review(query, hunks=hunks, symbols=symbols, base_graph=base_graph)
|
|
130
|
+
s = rv["summary"]
|
|
131
|
+
out = ["# Change-set review", ""]
|
|
132
|
+
out.append(f"**{s['changed_symbols']} changed symbol(s)**; "
|
|
133
|
+
f"blast by root: {s['blast_by_root'] or '—'}.")
|
|
134
|
+
if "api_diff" in rv:
|
|
135
|
+
ad = rv["api_diff"]["summary"]
|
|
136
|
+
out.append(f"**API diff vs baseline:** {ad['breaking_total']} breaking, "
|
|
137
|
+
f"{ad['removed']} removed, {ad['added']} added.")
|
|
138
|
+
if s["high_risk"]:
|
|
139
|
+
out.append(f"**Review first (high risk):** {', '.join(s['high_risk'])}")
|
|
140
|
+
if s["unresolved_hunks"]:
|
|
141
|
+
out.append(f"_Unresolved hunks (module-level / unknown file): "
|
|
142
|
+
f"{len(s['unresolved_hunks'])} — inspect manually._")
|
|
143
|
+
out.append("")
|
|
144
|
+
if "api_diff" in rv:
|
|
145
|
+
ad = rv["api_diff"]
|
|
146
|
+
breaking = [c for c in ad["changes"] if c["severity"] == "breaking"]
|
|
147
|
+
if ad["removed"] or breaking:
|
|
148
|
+
out.append("## API breaking changes (not in the hunks)")
|
|
149
|
+
for sym in ad["removed"][:20]:
|
|
150
|
+
out.append(f"- **removed** `{sym}`")
|
|
151
|
+
for c in breaking[:20]:
|
|
152
|
+
out.append(f"- **{c['kind']}** `{c['symbol']}` — {c['detail']}")
|
|
153
|
+
out.append("")
|
|
154
|
+
for d in rv["changed"]:
|
|
155
|
+
loc = f" ({d['file']}:{d['lines'][0]})" if d["file"] and d["lines"] else ""
|
|
156
|
+
out.append(f"## [{d['risk']}] {d['symbol']}{loc}")
|
|
157
|
+
if d["consumers_by_root"]:
|
|
158
|
+
out.append(f"- Consumers by root: {d['consumers_by_root']}")
|
|
159
|
+
if d["callers"]:
|
|
160
|
+
out.append(f"- Callers ({len(d['callers'])}): "
|
|
161
|
+
f"{', '.join(d['callers'][:8])}{' …' if len(d['callers']) > 8 else ''}")
|
|
162
|
+
if d["call_contract"]:
|
|
163
|
+
out.append(f"- Call-site contract: {len(d['call_contract'])} site(s) "
|
|
164
|
+
f"— check on signature change")
|
|
165
|
+
cols = d["columns"]
|
|
166
|
+
if cols["reads"] or cols["writes"]:
|
|
167
|
+
out.append(f"- Columns: reads {cols['reads'] or '—'}, writes {cols['writes'] or '—'}")
|
|
168
|
+
out.append("")
|
|
169
|
+
return "\n".join(out).rstrip() + "\n"
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
# -- unified-diff hunk parsing (for CLI `codemap review <diff>`) --------------
|
|
173
|
+
|
|
174
|
+
def parse_unified_diff(text: str) -> list[dict]:
|
|
175
|
+
"""Parse a unified diff → [{file, ranges: [[start,end], …]}] (new-file lines).
|
|
176
|
+
|
|
177
|
+
Reads ``+++ b/<path>`` targets and ``@@ -a,b +c,d @@`` hunk headers, taking the
|
|
178
|
+
**new-file** side (``+c,d`` → lines ``c … c+d-1``). Deletions (``d == 0``)
|
|
179
|
+
anchor a zero-width range at ``c`` so a pure deletion still maps to its site.
|
|
180
|
+
"""
|
|
181
|
+
import re
|
|
182
|
+
hdr = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@")
|
|
183
|
+
files: dict[str, list] = {}
|
|
184
|
+
current = None
|
|
185
|
+
for ln in text.splitlines():
|
|
186
|
+
if ln.startswith("+++ "):
|
|
187
|
+
path = ln[4:].strip().split("\t")[0]
|
|
188
|
+
if path.startswith("b/"):
|
|
189
|
+
path = path[2:]
|
|
190
|
+
current = None if path == "/dev/null" else path
|
|
191
|
+
files.setdefault(current, [])
|
|
192
|
+
elif current and (m := hdr.match(ln)):
|
|
193
|
+
start = int(m.group(1))
|
|
194
|
+
count = int(m.group(2)) if m.group(2) is not None else 1
|
|
195
|
+
end = start + count - 1 if count else start
|
|
196
|
+
files[current].append([start, max(start, end)])
|
|
197
|
+
return [{"file": f, "ranges": r} for f, r in files.items() if f and r]
|
codemap/serve/scip.py
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""SCIP export — codemap's graph as a SCIP index (R1-C1).
|
|
2
|
+
|
|
3
|
+
SCIP (Sourcegraph Code Intelligence Protocol) is the ascendant open interchange
|
|
4
|
+
format for precise code intelligence: emit one ``index.scip`` and Sourcegraph,
|
|
5
|
+
Glean and any SCIP consumer light up go-to-definition, symbol search and type
|
|
6
|
+
hierarchy across the ecosystem. The research landscape (``research/00_landscape.md``)
|
|
7
|
+
ranked this the highest-value interop move.
|
|
8
|
+
|
|
9
|
+
**Honest scope.** codemap's graph is *symbol-level*: edges relate symbol→symbol
|
|
10
|
+
but carry no call-site coordinates. SCIP occurrences are *location-based*. So this
|
|
11
|
+
exporter emits exactly what codemap knows precisely:
|
|
12
|
+
|
|
13
|
+
- **Definition occurrences** — one per node with a file location (module / class /
|
|
14
|
+
function / attribute) → go-to-definition + symbol search.
|
|
15
|
+
- **SymbolInformation** — kind, docstring, and ``inherits`` / ``implements`` as SCIP
|
|
16
|
+
``relationships`` (``is_implementation``) → type hierarchy.
|
|
17
|
+
|
|
18
|
+
It does **not** emit reference occurrences (find-references): that needs token
|
|
19
|
+
positions codemap does not track. Emitting fake positions would be worse than
|
|
20
|
+
omitting them, so we omit them and say so — consistent with codemap's other
|
|
21
|
+
lower-bound disclaimers.
|
|
22
|
+
|
|
23
|
+
``protobuf`` is an **optional** dependency (``pip install codemap[scip]``); the
|
|
24
|
+
vendored bindings (``_scip_pb2.py``) are generated from the official ``scip.proto``.
|
|
25
|
+
The import is lazy so codemap works without it.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import os
|
|
31
|
+
|
|
32
|
+
from codemap.model import Graph
|
|
33
|
+
from codemap.query import Query
|
|
34
|
+
|
|
35
|
+
# Descriptor suffix per node kind (SCIP symbol grammar): namespace ``/``, type
|
|
36
|
+
# ``#``, method ``().``, term ``.``. Functions and methods both use the method
|
|
37
|
+
# descriptor (as scip-python does). Unknown intermediates default to namespace.
|
|
38
|
+
_SUFFIX = {"module": "/", "class": "#", "function": "().", "attribute": "."}
|
|
39
|
+
|
|
40
|
+
# Names outside this set must be backtick-escaped in a SCIP symbol (grammar).
|
|
41
|
+
_SIMPLE = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_+$")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _load_pb2():
|
|
45
|
+
"""Import the vendored SCIP bindings, with a friendly error if protobuf is absent."""
|
|
46
|
+
try:
|
|
47
|
+
from codemap.serve import _scip_pb2 # noqa: PLC0415
|
|
48
|
+
except ImportError as exc: # pragma: no cover - exercised via the extra
|
|
49
|
+
raise RuntimeError(
|
|
50
|
+
"SCIP export needs the optional 'scip' extra — install it with "
|
|
51
|
+
"`pip install codemap[scip]` (adds protobuf)."
|
|
52
|
+
) from exc
|
|
53
|
+
return _scip_pb2
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _escape(name: str) -> str:
|
|
57
|
+
"""Escape a descriptor name per the SCIP symbol grammar (backtick-wrap if needed)."""
|
|
58
|
+
if name and all(c in _SIMPLE for c in name):
|
|
59
|
+
return name
|
|
60
|
+
return "`" + name.replace("`", "``") + "`"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _symbol(nodes: dict, node_id: str, prefix: str) -> str:
|
|
64
|
+
"""Build the SCIP symbol string for ``node_id`` from its ancestor chain.
|
|
65
|
+
|
|
66
|
+
Each dotted segment becomes a descriptor whose suffix is chosen by the kind of
|
|
67
|
+
the node at that prefix (looked up in the graph); missing intermediates default
|
|
68
|
+
to a namespace descriptor.
|
|
69
|
+
"""
|
|
70
|
+
parts = node_id.split(".")
|
|
71
|
+
out = []
|
|
72
|
+
for i in range(1, len(parts) + 1):
|
|
73
|
+
pid = ".".join(parts[:i])
|
|
74
|
+
node = nodes.get(pid)
|
|
75
|
+
last = i == len(parts)
|
|
76
|
+
kind = node.kind if node else ("attribute" if last else "module")
|
|
77
|
+
out.append(_escape(parts[i - 1]) + _SUFFIX.get(kind, "/"))
|
|
78
|
+
return prefix + "".join(out)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _kind(pb2, kind: str, parent_kind: str | None):
|
|
82
|
+
K = pb2.SymbolInformation.Kind
|
|
83
|
+
if kind == "module":
|
|
84
|
+
return K.Module
|
|
85
|
+
if kind == "class":
|
|
86
|
+
return K.Class
|
|
87
|
+
if kind == "function":
|
|
88
|
+
return K.Method if parent_kind == "class" else K.Function
|
|
89
|
+
if kind == "attribute":
|
|
90
|
+
return K.Field if parent_kind == "class" else K.Variable
|
|
91
|
+
return K.UnspecifiedKind
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _language(file: str, pb2) -> str:
|
|
95
|
+
return "Python" if file.endswith(".py") else ""
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def build_scip(
|
|
99
|
+
query: Query,
|
|
100
|
+
*,
|
|
101
|
+
project_root: str,
|
|
102
|
+
package: str | None = None,
|
|
103
|
+
version: str = ".",
|
|
104
|
+
tool_version: str = "0.0.1",
|
|
105
|
+
scheme: str = "codemap",
|
|
106
|
+
manager: str = "python",
|
|
107
|
+
):
|
|
108
|
+
"""Build a ``scip_pb2.Index`` from the graph (definitions + symbol info)."""
|
|
109
|
+
pb2 = _load_pb2()
|
|
110
|
+
graph: Graph = query.graph
|
|
111
|
+
nodes = graph.nodes
|
|
112
|
+
package = package or graph.target
|
|
113
|
+
prefix = f"{scheme} {manager} {package} {version} "
|
|
114
|
+
|
|
115
|
+
# Index edges by source for relationship lookup (inherits / implements).
|
|
116
|
+
rels_by_src: dict[str, list[tuple[str, str]]] = {}
|
|
117
|
+
for e in graph.edges:
|
|
118
|
+
if e.type in ("inherits", "implements"):
|
|
119
|
+
rels_by_src.setdefault(e.source, []).append((e.type, e.target))
|
|
120
|
+
|
|
121
|
+
# Group definition occurrences + symbol info by file.
|
|
122
|
+
by_file: dict[str, dict] = {}
|
|
123
|
+
for nid in sorted(nodes):
|
|
124
|
+
node = nodes[nid]
|
|
125
|
+
if node.kind not in _SUFFIX or not node.file:
|
|
126
|
+
continue # skip synthetic column nodes, doc nodes, locationless overlays
|
|
127
|
+
sym = _symbol(nodes, nid, prefix)
|
|
128
|
+
line0 = (node.lineno - 1) if node.lineno else 0
|
|
129
|
+
parent_kind = None
|
|
130
|
+
if "." in nid:
|
|
131
|
+
parent = nodes.get(nid.rsplit(".", 1)[0])
|
|
132
|
+
parent_kind = parent.kind if parent else None
|
|
133
|
+
|
|
134
|
+
occ = pb2.Occurrence(symbol=sym, symbol_roles=pb2.SymbolRole.Definition)
|
|
135
|
+
occ.single_line_range.line = line0
|
|
136
|
+
occ.single_line_range.start_character = 0
|
|
137
|
+
occ.single_line_range.end_character = 0
|
|
138
|
+
# Test-code provenance (honest signal to consumers).
|
|
139
|
+
if node.extras.get("root") == "tests":
|
|
140
|
+
occ.symbol_roles |= pb2.SymbolRole.Test
|
|
141
|
+
# Enclosing range = the full definition span (proto guidance for defs).
|
|
142
|
+
if node.endlineno and node.endlineno - 1 != line0:
|
|
143
|
+
occ.multi_line_enclosing_range.start_line = line0
|
|
144
|
+
occ.multi_line_enclosing_range.start_character = 0
|
|
145
|
+
occ.multi_line_enclosing_range.end_line = node.endlineno - 1
|
|
146
|
+
occ.multi_line_enclosing_range.end_character = 0
|
|
147
|
+
|
|
148
|
+
info = pb2.SymbolInformation(symbol=sym, kind=_kind(pb2, node.kind, parent_kind))
|
|
149
|
+
if node.docstring:
|
|
150
|
+
info.documentation.append(node.docstring)
|
|
151
|
+
for _etype, target in sorted(rels_by_src.get(nid, [])):
|
|
152
|
+
if target in nodes: # only relate to symbols we can name canonically
|
|
153
|
+
info.relationships.append(
|
|
154
|
+
pb2.Relationship(symbol=_symbol(nodes, target, prefix),
|
|
155
|
+
is_implementation=True))
|
|
156
|
+
|
|
157
|
+
bucket = by_file.setdefault(node.file, {"occ": [], "sym": []})
|
|
158
|
+
bucket["occ"].append(occ)
|
|
159
|
+
bucket["sym"].append(info)
|
|
160
|
+
|
|
161
|
+
index = pb2.Index()
|
|
162
|
+
index.metadata.version = pb2.ProtocolVersion.UnspecifiedProtocolVersion
|
|
163
|
+
index.metadata.tool_info.name = "codemap"
|
|
164
|
+
index.metadata.tool_info.version = tool_version
|
|
165
|
+
index.metadata.project_root = "file://" + os.path.abspath(project_root)
|
|
166
|
+
index.metadata.text_document_encoding = pb2.TextEncoding.UTF8
|
|
167
|
+
|
|
168
|
+
for rel_path in sorted(by_file):
|
|
169
|
+
bucket = by_file[rel_path]
|
|
170
|
+
doc = pb2.Document(
|
|
171
|
+
relative_path=rel_path,
|
|
172
|
+
language=_language(rel_path, pb2),
|
|
173
|
+
position_encoding=pb2.PositionEncoding.UTF8CodeUnitOffsetFromLineStart,
|
|
174
|
+
)
|
|
175
|
+
doc.occurrences.extend(sorted(bucket["occ"], key=lambda o: o.symbol))
|
|
176
|
+
doc.symbols.extend(sorted(bucket["sym"], key=lambda s: s.symbol))
|
|
177
|
+
index.documents.append(doc)
|
|
178
|
+
return index
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def write_scip(index) -> bytes:
|
|
182
|
+
"""Serialize a SCIP ``Index`` to deterministic protobuf bytes."""
|
|
183
|
+
return index.SerializeToString(deterministic=True)
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Semantic search via a retrieval adapter, enriched to codemap symbols (R1-C16).
|
|
2
|
+
|
|
3
|
+
The codemap-native half of a retrieval adapter. Lives in ``serve`` (not
|
|
4
|
+
``integrations``) because it needs the query layer: the ``integrations`` layer is a
|
|
5
|
+
near-leaf opt-in gate and may not depend on ``query`` (enforced by codemap's own
|
|
6
|
+
architecture contract). So the adapter returns raw location hits (no graph), and
|
|
7
|
+
this module — allowed to use both ``query`` and ``integrations`` — resolves each
|
|
8
|
+
``(file, line)`` to the **exact codemap symbol** at that location. The external tool
|
|
9
|
+
supplies fuzzy relevance; codemap supplies exact structure — the composition (fuzzy
|
|
10
|
+
retrieval → codemap symbols) neither gives alone.
|
|
11
|
+
|
|
12
|
+
Adapter-mode only (``mode=ADAPTER``): a router can only forward its answer as-is, so
|
|
13
|
+
it can't be enriched — for a router-only semantic tool (e.g. GitNexus, NC-licensed),
|
|
14
|
+
use ``codemap route semantic-search`` instead. The core works without any of this:
|
|
15
|
+
no enabled+installed adapter → ``{resolver: None, hits: []}``, never an error.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
from codemap.integrations import (
|
|
23
|
+
IntegrationConfig, IntegrationMode, SemanticHit, is_permissive, load_config, resolve,
|
|
24
|
+
)
|
|
25
|
+
from codemap.query import Query
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def semantic_search(query: Query, text: str, *, root: str = ".",
|
|
29
|
+
config: IntegrationConfig | None = None,
|
|
30
|
+
limit: int = 10) -> dict[str, Any]:
|
|
31
|
+
"""Resolve a semantic-search adapter, run it, enrich hits to codemap symbols.
|
|
32
|
+
|
|
33
|
+
Returns ``{resolver, disclaimer, hits}`` where ``hits`` is a list of
|
|
34
|
+
:class:`~codemap.integrations.base.SemanticHit` dicts sorted by score (each
|
|
35
|
+
de-duplicated to one entry per resolved symbol, keeping its best-scoring chunk).
|
|
36
|
+
``resolver`` is None when no adapter is enabled+installed — the caller degrades.
|
|
37
|
+
"""
|
|
38
|
+
cfg = config if config is not None else load_config(root)
|
|
39
|
+
adapter = resolve("semantic-search", config=cfg, root=root,
|
|
40
|
+
mode=IntegrationMode.ADAPTER)
|
|
41
|
+
if adapter is None:
|
|
42
|
+
return {"resolver": None, "disclaimer": None, "hits": []}
|
|
43
|
+
|
|
44
|
+
raw = adapter.search("semantic-search", text, root=root, limit=limit)
|
|
45
|
+
hits: list[SemanticHit] = []
|
|
46
|
+
seen: dict[str, int] = {} # symbol id → index in `hits` (dedup, keep best score)
|
|
47
|
+
for r in raw:
|
|
48
|
+
file, line = r["file"], r["start_line"]
|
|
49
|
+
symbol = query.symbol_at(file, line)
|
|
50
|
+
hit = SemanticHit(
|
|
51
|
+
file=file, start_line=line, end_line=r.get("end_line"),
|
|
52
|
+
score=r["score"], symbol=symbol,
|
|
53
|
+
resolution="symbol" if symbol else "unresolved",
|
|
54
|
+
)
|
|
55
|
+
if symbol is not None and symbol in seen:
|
|
56
|
+
if hit.score > hits[seen[symbol]].score: # same symbol → keep higher score
|
|
57
|
+
hits[seen[symbol]] = hit
|
|
58
|
+
continue
|
|
59
|
+
if symbol is not None:
|
|
60
|
+
seen[symbol] = len(hits)
|
|
61
|
+
hits.append(hit)
|
|
62
|
+
|
|
63
|
+
hits.sort(key=lambda h: (-h.score, h.file, h.start_line))
|
|
64
|
+
# Adapters are permissive by policy, so disclaimer is normally None; kept for
|
|
65
|
+
# uniformity with the router path (and any future edge case).
|
|
66
|
+
disclaimer = None if is_permissive(adapter.license) else adapter.disclaimer()
|
|
67
|
+
return {
|
|
68
|
+
"resolver": adapter.name,
|
|
69
|
+
"disclaimer": disclaimer,
|
|
70
|
+
"hits": [h.to_dict() for h in hits],
|
|
71
|
+
}
|
codemap/serve/server.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Line-delimited JSON stdio loop over a warm ``Session`` (DESIGN §14.4, M3.1).
|
|
2
|
+
|
|
3
|
+
Transport for the resident process: one JSON request per input line, one JSON
|
|
4
|
+
response per output line. Zero startup per call — the graph is loaded once by the
|
|
5
|
+
caller and reused for the life of the process. Transport-neutral by design: an
|
|
6
|
+
MCP adapter can wrap the same ``Session.handle`` later without touching this loop.
|
|
7
|
+
|
|
8
|
+
$ codemap serve --graph graph.json
|
|
9
|
+
{"op": "query", "args": {"name": "analyze_zones"}}
|
|
10
|
+
{"ok": true, "result": {...}}
|
|
11
|
+
{"op": "column", "args": {"name": "macd_hist"}}
|
|
12
|
+
{"ok": true, "result": {"writes": [...], "reads": [...]}}
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import sys
|
|
19
|
+
|
|
20
|
+
from codemap.serve.session import Session
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def serve_stdio(session: Session, stdin=None, stdout=None) -> int:
|
|
24
|
+
"""Read JSON requests line-by-line, write JSON responses. EOF ends the loop.
|
|
25
|
+
|
|
26
|
+
A blank line is skipped; a malformed line yields an error response rather than
|
|
27
|
+
crashing the resident process. Returns 0 at EOF.
|
|
28
|
+
"""
|
|
29
|
+
stdin = stdin if stdin is not None else sys.stdin
|
|
30
|
+
stdout = stdout if stdout is not None else sys.stdout
|
|
31
|
+
for line in stdin:
|
|
32
|
+
line = line.strip()
|
|
33
|
+
if not line:
|
|
34
|
+
continue
|
|
35
|
+
try:
|
|
36
|
+
request = json.loads(line)
|
|
37
|
+
except json.JSONDecodeError as exc:
|
|
38
|
+
response = {"ok": False, "error": f"invalid JSON: {exc}"}
|
|
39
|
+
else:
|
|
40
|
+
response = session.handle(request)
|
|
41
|
+
stdout.write(json.dumps(response, ensure_ascii=False) + "\n")
|
|
42
|
+
stdout.flush()
|
|
43
|
+
return 0
|