codepulse-engine 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codepulse/__init__.py +5 -0
- codepulse/api/app.py +1336 -0
- codepulse/api/errors.py +172 -0
- codepulse/api/security.py +189 -0
- codepulse/auth.py +406 -0
- codepulse/cli.py +1052 -0
- codepulse/config.py +475 -0
- codepulse/data_location.py +78 -0
- codepulse/embeddings/chunker.py +138 -0
- codepulse/embeddings/embedder.py +120 -0
- codepulse/git_diff.py +201 -0
- codepulse/indexer/engine.py +2655 -0
- codepulse/indexer/ignore.py +120 -0
- codepulse/ingestion_safety.py +169 -0
- codepulse/jobs.py +257 -0
- codepulse/mcp_server.py +269 -0
- codepulse/models.py +140 -0
- codepulse/observability.py +130 -0
- codepulse/parser/__init__.py +8 -0
- codepulse/parser/base.py +58 -0
- codepulse/parser/factory.py +37 -0
- codepulse/parser/graph.py +502 -0
- codepulse/parser/javascript.py +428 -0
- codepulse/parser/module_graph.py +232 -0
- codepulse/parser/python.py +331 -0
- codepulse/rate_limit.py +142 -0
- codepulse/repository_identity.py +83 -0
- codepulse/search/bm25.py +342 -0
- codepulse/search/ranker.py +382 -0
- codepulse/search/tokenizer.py +80 -0
- codepulse/search/vector.py +357 -0
- codepulse/storage/db.py +526 -0
- codepulse/storage/migrations.py +347 -0
- codepulse/storage/schema_migrations/README +1 -0
- codepulse/storage/schema_migrations/env.py +70 -0
- codepulse/storage/schema_migrations/script.py.mako +28 -0
- codepulse/storage/schema_migrations/versions/135a7b064ea7_add_base_classes_and_relationship_type.py +82 -0
- codepulse/storage/schema_migrations/versions/20fc2d0607e2_add_file_content_hash_and_natural_key_.py +106 -0
- codepulse/storage/schema_migrations/versions/21a7cb7f1a85_initial_schema.py +105 -0
- codepulse/storage/schema_migrations/versions/2cd38eb12a7e_add_call_edge_repo_id_isolation_fix.py +112 -0
- codepulse/storage/schema_migrations/versions/3ba396e9c78f_add_symbol_natural_key_uniqueness.py +80 -0
- codepulse/storage/schema_migrations/versions/49572f834366_add_embedding_persistence_fields.py +102 -0
- codepulse/storage/schema_migrations/versions/7c3e5a91d2f4_add_module_edges.py +61 -0
- codepulse/storage/schema_migrations/versions/a41f0d2c6b8e_canonicalize_repository_urls.py +94 -0
- codepulse/storage/schema_migrations/versions/b0bdd63949f4_add_api_key_model.py +71 -0
- codepulse/storage/schema_migrations/versions/b9014c049e33_add_api_key_repository_role.py +65 -0
- codepulse/storage/schema_migrations/versions/c286d277ea21_add_repository_snapshot_model.py +62 -0
- codepulse/storage/schema_migrations/versions/de22df2fc3ce_add_index_job_model.py +59 -0
- codepulse/storage/schema_migrations/versions/f6e481353553_rename_dependency_to_call_edge_add_confidence.py +80 -0
- codepulse/symbol_identity.py +29 -0
- codepulse_engine-0.1.0.dist-info/METADATA +191 -0
- codepulse_engine-0.1.0.dist-info/RECORD +56 -0
- codepulse_engine-0.1.0.dist-info/WHEEL +5 -0
- codepulse_engine-0.1.0.dist-info/entry_points.txt +2 -0
- codepulse_engine-0.1.0.dist-info/licenses/LICENSE +21 -0
- codepulse_engine-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,428 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
from typing import Any
|
|
5
|
+
from tree_sitter_languages import get_parser
|
|
6
|
+
|
|
7
|
+
from codepulse.models import CallReference, ImportItem, ImportStatement, Symbol, SymbolType
|
|
8
|
+
from codepulse.parser.base import LanguageParser
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class JavaScriptParser(LanguageParser):
|
|
12
|
+
"""AST parser for JavaScript and TypeScript source code using Tree-sitter."""
|
|
13
|
+
|
|
14
|
+
def __init__(self, is_typescript: bool = False) -> None:
|
|
15
|
+
self._is_typescript = is_typescript
|
|
16
|
+
try:
|
|
17
|
+
self._parser = get_parser("typescript" if is_typescript else "javascript")
|
|
18
|
+
except Exception:
|
|
19
|
+
# Fallback to javascript parser
|
|
20
|
+
self._parser = get_parser("javascript")
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def language(self) -> str:
|
|
24
|
+
return "typescript" if self._is_typescript else "javascript"
|
|
25
|
+
|
|
26
|
+
def extract_symbols(self, source_code: str, file_path: str = "") -> list[Symbol]:
|
|
27
|
+
source_bytes = source_code.encode("utf-8")
|
|
28
|
+
tree = self._parser.parse(source_bytes)
|
|
29
|
+
symbols: list[Symbol] = []
|
|
30
|
+
|
|
31
|
+
def walk(node: Any, parent_class: str | None = None) -> None:
|
|
32
|
+
# 1. Function Declarations: function foo() {}
|
|
33
|
+
if node.type in ("function_declaration", "generator_function_declaration"):
|
|
34
|
+
sym = self._parse_function_declaration(node, source_bytes, file_path, parent_class)
|
|
35
|
+
if sym:
|
|
36
|
+
symbols.append(sym)
|
|
37
|
+
body_node = node.child_by_field_name("body")
|
|
38
|
+
if body_node:
|
|
39
|
+
for child in body_node.children:
|
|
40
|
+
walk(child, parent_class=sym.name if sym else parent_class)
|
|
41
|
+
return
|
|
42
|
+
|
|
43
|
+
# 2. Variable declarations with arrow functions / function expressions: const foo = () => {}
|
|
44
|
+
elif node.type == "lexical_declaration" or node.type == "variable_declaration":
|
|
45
|
+
syms = self._parse_variable_functions(node, source_bytes, file_path, parent_class)
|
|
46
|
+
symbols.extend(syms)
|
|
47
|
+
for child in node.children:
|
|
48
|
+
walk(child, parent_class)
|
|
49
|
+
return
|
|
50
|
+
|
|
51
|
+
# 3. Class Declarations: class MyClass {}
|
|
52
|
+
elif node.type == "class_declaration":
|
|
53
|
+
sym = self._parse_class_declaration(node, source_bytes, file_path, parent_class)
|
|
54
|
+
if sym:
|
|
55
|
+
symbols.append(sym)
|
|
56
|
+
body_node = node.child_by_field_name("body")
|
|
57
|
+
if body_node:
|
|
58
|
+
for child in body_node.children:
|
|
59
|
+
walk(child, parent_class=sym.name if sym else parent_class)
|
|
60
|
+
return
|
|
61
|
+
|
|
62
|
+
# 4. Method Definitions inside class body: foo() {}
|
|
63
|
+
elif node.type == "method_definition":
|
|
64
|
+
sym = self._parse_method_definition(node, source_bytes, file_path, parent_class)
|
|
65
|
+
if sym:
|
|
66
|
+
symbols.append(sym)
|
|
67
|
+
body_node = node.child_by_field_name("body")
|
|
68
|
+
if body_node:
|
|
69
|
+
for child in body_node.children:
|
|
70
|
+
walk(child, parent_class=sym.name if sym else parent_class)
|
|
71
|
+
return
|
|
72
|
+
|
|
73
|
+
for child in node.children:
|
|
74
|
+
walk(child, parent_class)
|
|
75
|
+
|
|
76
|
+
walk(tree.root_node)
|
|
77
|
+
return symbols
|
|
78
|
+
|
|
79
|
+
def _parse_function_declaration(
|
|
80
|
+
self, node: Any, source_bytes: bytes, file_path: str, parent_class: str | None
|
|
81
|
+
) -> Symbol | None:
|
|
82
|
+
name_node = node.child_by_field_name("name")
|
|
83
|
+
if not name_node:
|
|
84
|
+
return None
|
|
85
|
+
|
|
86
|
+
name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace")
|
|
87
|
+
start_line = node.start_point[0] + 1
|
|
88
|
+
end_line = node.end_point[0] + 1
|
|
89
|
+
body_text = source_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
|
|
90
|
+
docstring = self._extract_jsdoc(node, source_bytes)
|
|
91
|
+
signature = self._extract_signature(node, source_bytes)
|
|
92
|
+
complexity = self._calculate_complexity(node)
|
|
93
|
+
|
|
94
|
+
return Symbol(
|
|
95
|
+
name=name,
|
|
96
|
+
type=SymbolType.METHOD if parent_class else SymbolType.FUNCTION,
|
|
97
|
+
file_path=file_path,
|
|
98
|
+
start_line=start_line,
|
|
99
|
+
end_line=end_line,
|
|
100
|
+
signature=signature,
|
|
101
|
+
docstring=docstring,
|
|
102
|
+
body=body_text,
|
|
103
|
+
body_hash=hashlib.sha256(body_text.strip().encode("utf-8")).hexdigest(),
|
|
104
|
+
language=self.language,
|
|
105
|
+
parent_symbol=parent_class,
|
|
106
|
+
complexity=complexity,
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
def _parse_variable_functions(
|
|
110
|
+
self, node: Any, source_bytes: bytes, file_path: str, parent_class: str | None
|
|
111
|
+
) -> list[Symbol]:
|
|
112
|
+
results: list[Symbol] = []
|
|
113
|
+
for child in node.children:
|
|
114
|
+
if child.type == "variable_declarator":
|
|
115
|
+
name_n = child.child_by_field_name("name")
|
|
116
|
+
value_n = child.child_by_field_name("value")
|
|
117
|
+
if name_n and value_n and value_n.type in ("arrow_function", "function_expression"):
|
|
118
|
+
name = source_bytes[name_n.start_byte:name_n.end_byte].decode("utf-8", errors="replace")
|
|
119
|
+
start_line = node.start_point[0] + 1
|
|
120
|
+
end_line = node.end_point[0] + 1
|
|
121
|
+
body_text = source_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
|
|
122
|
+
docstring = self._extract_jsdoc(node, source_bytes)
|
|
123
|
+
signature = self._extract_signature(node, source_bytes)
|
|
124
|
+
complexity = self._calculate_complexity(value_n)
|
|
125
|
+
|
|
126
|
+
results.append(
|
|
127
|
+
Symbol(
|
|
128
|
+
name=name,
|
|
129
|
+
type=SymbolType.METHOD if parent_class else SymbolType.FUNCTION,
|
|
130
|
+
file_path=file_path,
|
|
131
|
+
start_line=start_line,
|
|
132
|
+
end_line=end_line,
|
|
133
|
+
signature=signature,
|
|
134
|
+
docstring=docstring,
|
|
135
|
+
body=body_text,
|
|
136
|
+
body_hash=hashlib.sha256(body_text.strip().encode("utf-8")).hexdigest(),
|
|
137
|
+
language=self.language,
|
|
138
|
+
parent_symbol=parent_class,
|
|
139
|
+
complexity=complexity,
|
|
140
|
+
)
|
|
141
|
+
)
|
|
142
|
+
return results
|
|
143
|
+
|
|
144
|
+
def _parse_class_declaration(
|
|
145
|
+
self, node: Any, source_bytes: bytes, file_path: str, parent_class: str | None
|
|
146
|
+
) -> Symbol | None:
|
|
147
|
+
name_node = node.child_by_field_name("name")
|
|
148
|
+
if not name_node:
|
|
149
|
+
return None
|
|
150
|
+
|
|
151
|
+
name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace")
|
|
152
|
+
start_line = node.start_point[0] + 1
|
|
153
|
+
end_line = node.end_point[0] + 1
|
|
154
|
+
body_text = source_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
|
|
155
|
+
docstring = self._extract_jsdoc(node, source_bytes)
|
|
156
|
+
signature = self._extract_signature(node, source_bytes)
|
|
157
|
+
base_classes = self._extract_base_classes(node, source_bytes)
|
|
158
|
+
|
|
159
|
+
return Symbol(
|
|
160
|
+
name=name,
|
|
161
|
+
type=SymbolType.CLASS,
|
|
162
|
+
file_path=file_path,
|
|
163
|
+
start_line=start_line,
|
|
164
|
+
end_line=end_line,
|
|
165
|
+
signature=signature,
|
|
166
|
+
docstring=docstring,
|
|
167
|
+
body=body_text,
|
|
168
|
+
body_hash=hashlib.sha256(body_text.strip().encode("utf-8")).hexdigest(),
|
|
169
|
+
language=self.language,
|
|
170
|
+
parent_symbol=parent_class,
|
|
171
|
+
complexity=1,
|
|
172
|
+
base_classes=base_classes,
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
def _extract_base_classes(self, node: Any, source_bytes: bytes) -> list[str]:
|
|
176
|
+
"""
|
|
177
|
+
Phase 4.1 (roadmap §37 item 4.1: "Base-class/inheritance
|
|
178
|
+
extraction (Python + JS/TS)"). Confirmed via direct tree-sitter
|
|
179
|
+
grammar probes during this milestone's own design audit that the
|
|
180
|
+
PLAIN JAVASCRIPT and TYPESCRIPT grammars shape `class_heritage`
|
|
181
|
+
differently:
|
|
182
|
+
|
|
183
|
+
- Plain JavaScript: `class_declaration -> class_heritage ->
|
|
184
|
+
(extends, <base expression>)` -- the base expression is a
|
|
185
|
+
DIRECT child of `class_heritage` (no wrapper node).
|
|
186
|
+
- TypeScript: `class_declaration -> class_heritage ->
|
|
187
|
+
extends_clause -> (extends, <base expression>)`, with an
|
|
188
|
+
optional SIBLING `implements_clause` (interface
|
|
189
|
+
implementation) inside `class_heritage` alongside
|
|
190
|
+
`extends_clause`. `implements_clause` is DELIBERATELY never
|
|
191
|
+
inspected here -- interface implementation is a different
|
|
192
|
+
relationship than base-class inheritance, out of this
|
|
193
|
+
milestone's exact scope (roadmap §37 item 4.1 names
|
|
194
|
+
"base-class/inheritance extraction," not "implements");
|
|
195
|
+
only ever looking WITHIN `extends_clause` when present is
|
|
196
|
+
what keeps `implements_clause` categorically unreachable
|
|
197
|
+
here, not an extra exclusion check.
|
|
198
|
+
|
|
199
|
+
`<base expression>` is either an `identifier` (`Base`) or a
|
|
200
|
+
`member_expression` (`ns.Base`, a dotted/namespace-qualified
|
|
201
|
+
base) -- the only two node types that can actually name a base
|
|
202
|
+
class in either grammar. Returns the raw text exactly as
|
|
203
|
+
written; resolution happens separately in `CallGraphBuilder`.
|
|
204
|
+
"""
|
|
205
|
+
heritage = None
|
|
206
|
+
for child in node.children:
|
|
207
|
+
if child.type == "class_heritage":
|
|
208
|
+
heritage = child
|
|
209
|
+
break
|
|
210
|
+
if heritage is None:
|
|
211
|
+
return []
|
|
212
|
+
|
|
213
|
+
extends_clause = None
|
|
214
|
+
for child in heritage.children:
|
|
215
|
+
if child.type == "extends_clause":
|
|
216
|
+
extends_clause = child
|
|
217
|
+
break
|
|
218
|
+
|
|
219
|
+
# TypeScript wraps the base expression in extends_clause (and
|
|
220
|
+
# that is the ONLY place searched, so implements_clause -- a
|
|
221
|
+
# sibling of extends_clause, never a descendant of it -- is
|
|
222
|
+
# never reached). Plain JavaScript has no such wrapper -- the
|
|
223
|
+
# base expression is a direct child of class_heritage itself.
|
|
224
|
+
search_within = extends_clause if extends_clause is not None else heritage
|
|
225
|
+
|
|
226
|
+
bases: list[str] = []
|
|
227
|
+
for child in search_within.children:
|
|
228
|
+
if child.type in ("identifier", "member_expression", "type_identifier"):
|
|
229
|
+
bases.append(
|
|
230
|
+
source_bytes[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
|
|
231
|
+
)
|
|
232
|
+
return bases
|
|
233
|
+
|
|
234
|
+
def _parse_method_definition(
|
|
235
|
+
self, node: Any, source_bytes: bytes, file_path: str, parent_class: str | None
|
|
236
|
+
) -> Symbol | None:
|
|
237
|
+
name_node = node.child_by_field_name("name")
|
|
238
|
+
if not name_node:
|
|
239
|
+
return None
|
|
240
|
+
|
|
241
|
+
name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace")
|
|
242
|
+
start_line = node.start_point[0] + 1
|
|
243
|
+
end_line = node.end_point[0] + 1
|
|
244
|
+
body_text = source_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
|
|
245
|
+
docstring = self._extract_jsdoc(node, source_bytes)
|
|
246
|
+
signature = self._extract_signature(node, source_bytes)
|
|
247
|
+
complexity = self._calculate_complexity(node)
|
|
248
|
+
|
|
249
|
+
return Symbol(
|
|
250
|
+
name=name,
|
|
251
|
+
type=SymbolType.METHOD,
|
|
252
|
+
file_path=file_path,
|
|
253
|
+
start_line=start_line,
|
|
254
|
+
end_line=end_line,
|
|
255
|
+
signature=signature,
|
|
256
|
+
docstring=docstring,
|
|
257
|
+
body=body_text,
|
|
258
|
+
body_hash=hashlib.sha256(body_text.strip().encode("utf-8")).hexdigest(),
|
|
259
|
+
language=self.language,
|
|
260
|
+
parent_symbol=parent_class,
|
|
261
|
+
complexity=complexity,
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
def _extract_jsdoc(self, node: Any, source_bytes: bytes) -> str | None:
|
|
265
|
+
# Check previous sibling for comment
|
|
266
|
+
prev = node.prev_sibling
|
|
267
|
+
if prev and prev.type == "comment":
|
|
268
|
+
cmt = source_bytes[prev.start_byte:prev.end_byte].decode("utf-8", errors="replace")
|
|
269
|
+
if cmt.startswith("/**") and cmt.endswith("*/"):
|
|
270
|
+
lines = [l.strip().lstrip("/*").rstrip("*/").lstrip("*").strip() for l in cmt.splitlines()]
|
|
271
|
+
clean_lines = [l for l in lines if l]
|
|
272
|
+
return "\n".join(clean_lines) if clean_lines else None
|
|
273
|
+
elif cmt.startswith("//"):
|
|
274
|
+
return cmt.lstrip("/").strip()
|
|
275
|
+
return None
|
|
276
|
+
|
|
277
|
+
def _extract_signature(self, node: Any, source_bytes: bytes) -> str:
|
|
278
|
+
body_node = node.child_by_field_name("body")
|
|
279
|
+
if body_node:
|
|
280
|
+
sig_bytes = source_bytes[node.start_byte:body_node.start_byte]
|
|
281
|
+
sig_text = sig_bytes.decode("utf-8", errors="replace").strip()
|
|
282
|
+
return " ".join(sig_text.split())
|
|
283
|
+
|
|
284
|
+
full_text = source_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
|
|
285
|
+
return full_text.splitlines()[0].strip().rstrip("{").strip()
|
|
286
|
+
|
|
287
|
+
def _calculate_complexity(self, node: Any) -> int:
|
|
288
|
+
complexity = 1
|
|
289
|
+
branching_types = {
|
|
290
|
+
"if_statement", "for_statement", "for_in_statement", "while_statement",
|
|
291
|
+
"do_statement", "catch_clause", "switch_case", "ternary_expression",
|
|
292
|
+
"binary_expression" # Logical && and ||
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
def count_branches(n: Any) -> None:
|
|
296
|
+
nonlocal complexity
|
|
297
|
+
if n.type in branching_types:
|
|
298
|
+
complexity += 1
|
|
299
|
+
for child in n.children:
|
|
300
|
+
count_branches(child)
|
|
301
|
+
|
|
302
|
+
count_branches(node)
|
|
303
|
+
return complexity
|
|
304
|
+
|
|
305
|
+
def extract_imports(self, source_code: str, file_path: str = "") -> list[ImportStatement]:
|
|
306
|
+
source_bytes = source_code.encode("utf-8")
|
|
307
|
+
tree = self._parser.parse(source_bytes)
|
|
308
|
+
imports: list[ImportStatement] = []
|
|
309
|
+
|
|
310
|
+
def walk(node: Any) -> None:
|
|
311
|
+
# ES6: import { a, b as c } from './module';
|
|
312
|
+
# ES6: import React from 'react';
|
|
313
|
+
# ES6: import * as path from 'path';
|
|
314
|
+
if node.type == "import_statement":
|
|
315
|
+
line_num = node.start_point[0] + 1
|
|
316
|
+
source_node = node.child_by_field_name("source")
|
|
317
|
+
module_name = ""
|
|
318
|
+
if source_node:
|
|
319
|
+
module_name = source_bytes[source_node.start_byte:source_node.end_byte].decode("utf-8").strip("'\"")
|
|
320
|
+
|
|
321
|
+
is_relative = module_name.startswith(".")
|
|
322
|
+
level = 1 if module_name.startswith("./") else (2 if module_name.startswith("../") else 0)
|
|
323
|
+
|
|
324
|
+
items: list[ImportItem] = []
|
|
325
|
+
for child in node.children:
|
|
326
|
+
if child.type == "import_clause":
|
|
327
|
+
for sub in child.children:
|
|
328
|
+
if sub.type == "identifier":
|
|
329
|
+
default_name = source_bytes[sub.start_byte:sub.end_byte].decode("utf-8")
|
|
330
|
+
items.append(ImportItem(name=default_name, alias="default"))
|
|
331
|
+
elif sub.type == "named_imports":
|
|
332
|
+
for spec in sub.children:
|
|
333
|
+
if spec.type == "import_specifier":
|
|
334
|
+
name_n = spec.child_by_field_name("name")
|
|
335
|
+
alias_n = spec.child_by_field_name("alias")
|
|
336
|
+
if name_n:
|
|
337
|
+
item_name = source_bytes[name_n.start_byte:name_n.end_byte].decode("utf-8")
|
|
338
|
+
alias = source_bytes[alias_n.start_byte:alias_n.end_byte].decode("utf-8") if alias_n else None
|
|
339
|
+
items.append(ImportItem(name=item_name, alias=alias))
|
|
340
|
+
elif sub.type == "namespace_import":
|
|
341
|
+
for ns_child in sub.children:
|
|
342
|
+
if ns_child.type == "identifier":
|
|
343
|
+
alias = source_bytes[ns_child.start_byte:ns_child.end_byte].decode("utf-8")
|
|
344
|
+
items.append(ImportItem(name="*", alias=alias))
|
|
345
|
+
|
|
346
|
+
imports.append(
|
|
347
|
+
ImportStatement(
|
|
348
|
+
module=module_name,
|
|
349
|
+
items=items,
|
|
350
|
+
is_relative=is_relative,
|
|
351
|
+
level=level,
|
|
352
|
+
line_number=line_num,
|
|
353
|
+
file_path=file_path,
|
|
354
|
+
)
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
# CommonJS: const foo = require('foo')
|
|
358
|
+
elif node.type == "call_expression":
|
|
359
|
+
func_n = node.child_by_field_name("function")
|
|
360
|
+
args_n = node.child_by_field_name("arguments")
|
|
361
|
+
if func_n and args_n:
|
|
362
|
+
fn_name = source_bytes[func_n.start_byte:func_n.end_byte].decode("utf-8", errors="replace")
|
|
363
|
+
if fn_name == "require" and args_n.children:
|
|
364
|
+
for arg in args_n.children:
|
|
365
|
+
if arg.type == "string":
|
|
366
|
+
mod_name = source_bytes[arg.start_byte:arg.end_byte].decode("utf-8").strip("'\"")
|
|
367
|
+
line_num = node.start_point[0] + 1
|
|
368
|
+
is_rel = mod_name.startswith(".")
|
|
369
|
+
imports.append(
|
|
370
|
+
ImportStatement(
|
|
371
|
+
module=mod_name,
|
|
372
|
+
items=[],
|
|
373
|
+
is_relative=is_rel,
|
|
374
|
+
level=1 if mod_name.startswith("./") else (2 if mod_name.startswith("../") else 0),
|
|
375
|
+
line_number=line_num,
|
|
376
|
+
file_path=file_path,
|
|
377
|
+
)
|
|
378
|
+
)
|
|
379
|
+
|
|
380
|
+
for child in node.children:
|
|
381
|
+
walk(child)
|
|
382
|
+
|
|
383
|
+
walk(tree.root_node)
|
|
384
|
+
return imports
|
|
385
|
+
|
|
386
|
+
def extract_calls(
|
|
387
|
+
self, source_code: str, file_path: str = "", symbols: list[Symbol] | None = None
|
|
388
|
+
) -> list[CallReference]:
|
|
389
|
+
if symbols is None:
|
|
390
|
+
symbols = self.extract_symbols(source_code, file_path=file_path)
|
|
391
|
+
|
|
392
|
+
source_bytes = source_code.encode("utf-8")
|
|
393
|
+
tree = self._parser.parse(source_bytes)
|
|
394
|
+
calls: list[CallReference] = []
|
|
395
|
+
|
|
396
|
+
def find_enclosing_symbol(line: int) -> str:
|
|
397
|
+
matching = [s for s in symbols if s.start_line <= line <= s.end_line]
|
|
398
|
+
if not matching:
|
|
399
|
+
return "<module>"
|
|
400
|
+
matching.sort(key=lambda s: (s.end_line - s.start_line))
|
|
401
|
+
return matching[0].name
|
|
402
|
+
|
|
403
|
+
def walk(node: Any) -> None:
|
|
404
|
+
if node.type == "call_expression":
|
|
405
|
+
func_node = node.child_by_field_name("function")
|
|
406
|
+
if func_node:
|
|
407
|
+
callee_name = source_bytes[func_node.start_byte:func_node.end_byte].decode("utf-8", errors="replace")
|
|
408
|
+
if func_node.type == "member_expression":
|
|
409
|
+
prop_node = func_node.child_by_field_name("property")
|
|
410
|
+
if prop_node:
|
|
411
|
+
callee_name = source_bytes[prop_node.start_byte:prop_node.end_byte].decode("utf-8", errors="replace")
|
|
412
|
+
|
|
413
|
+
line_num = node.start_point[0] + 1
|
|
414
|
+
caller = find_enclosing_symbol(line_num)
|
|
415
|
+
calls.append(
|
|
416
|
+
CallReference(
|
|
417
|
+
caller_symbol=caller,
|
|
418
|
+
callee_name=callee_name,
|
|
419
|
+
line_number=line_num,
|
|
420
|
+
file_path=file_path,
|
|
421
|
+
)
|
|
422
|
+
)
|
|
423
|
+
|
|
424
|
+
for child in node.children:
|
|
425
|
+
walk(child)
|
|
426
|
+
|
|
427
|
+
walk(tree.root_node)
|
|
428
|
+
return calls
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import networkx as nx
|
|
6
|
+
|
|
7
|
+
from codepulse.models import FileParseResult
|
|
8
|
+
from codepulse.parser.graph import ImportResolver
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ModuleGraphBuilder:
|
|
12
|
+
"""
|
|
13
|
+
Phase 4.3 (roadmap §37 item 4.3: "Deterministic module-clustering
|
|
14
|
+
via import graph"; §11 "Architecture discovery": "module/dependency-
|
|
15
|
+
cluster detection via the existing import graph... to identify
|
|
16
|
+
likely 'modules' and 'entry points'"; §45 backlog B18: "New
|
|
17
|
+
graph-analysis module").
|
|
18
|
+
|
|
19
|
+
Deliberately kept SEPARATE from `CallGraphBuilder` (roadmap §37
|
|
20
|
+
item 4.3's own instruction) -- this class operates on the
|
|
21
|
+
FILE-LEVEL import graph, not the symbol-level call/inheritance
|
|
22
|
+
graph. It reuses `ImportResolver.resolve_import_path` exactly as
|
|
23
|
+
written (no second import resolver, no duplicated resolution
|
|
24
|
+
logic) purely to turn each file's own `ImportStatement`s into
|
|
25
|
+
file-to-file edges.
|
|
26
|
+
|
|
27
|
+
Library/shared-core only, per this milestone's explicit approved
|
|
28
|
+
scope -- no REST route, CLI command, or MCP tool consumes this yet.
|
|
29
|
+
A future one can, through `IndexingEngine.analyze_module_structure`,
|
|
30
|
+
without duplicating any of this construction logic.
|
|
31
|
+
|
|
32
|
+
2026-10-12 (user-approved, CODEPULSE_FUTURE_CHANGES.md item 15): the
|
|
33
|
+
resolved file-to-file edges are now persisted (`module_edges`) and a
|
|
34
|
+
graph can be rebuilt from them with `from_edges`, so module structure
|
|
35
|
+
survives a restart. `snapshot` computes just the files and edges (no
|
|
36
|
+
clustering) for the index-time write; `from_edges` then produces a
|
|
37
|
+
graph identical, in every public structure, to one built from parse
|
|
38
|
+
results.
|
|
39
|
+
|
|
40
|
+
Graph semantics (approved, roadmap §37 item 4.3 audit):
|
|
41
|
+
- Nodes are every file this repository's parser actually parsed
|
|
42
|
+
(`ImportResolver.files`), INCLUDING files with zero imports in
|
|
43
|
+
either direction -- isolated files are never silently omitted.
|
|
44
|
+
- An edge `source_file -> target_file` exists if `source_file`
|
|
45
|
+
contains at least one import statement that
|
|
46
|
+
`ImportResolver.resolve_import_path` resolves to `target_file`
|
|
47
|
+
(a real, repository-local file). Multiple import statements in
|
|
48
|
+
`source_file` resolving to the SAME `target_file` collapse to
|
|
49
|
+
exactly one edge -- this graph represents file-to-file
|
|
50
|
+
dependency EXISTENCE, not a raw import-statement count (unlike
|
|
51
|
+
`CallGraphEdge.call_count`, which deliberately does count).
|
|
52
|
+
- An import that does not resolve to a real file in this
|
|
53
|
+
repository (an external package, a stdlib module, or a genuine
|
|
54
|
+
parser gap) never becomes an edge and never creates a node for
|
|
55
|
+
the unresolved target -- `ImportResolver.resolve_import_path`
|
|
56
|
+
is the single source of truth for "does this import point at
|
|
57
|
+
one of this repository's own files," reused unmodified.
|
|
58
|
+
- A file importing itself is not recorded as a self-loop edge.
|
|
59
|
+
|
|
60
|
+
`in_degree`/`out_degree` are computed from this exact edge set (the
|
|
61
|
+
distinct-file-pair notion above), never from raw import-statement
|
|
62
|
+
counts. They are exposed as RAW, uninterpreted signals -- no
|
|
63
|
+
"importance"/"centrality"/"API probability" score is computed or
|
|
64
|
+
stored anywhere in this class. §11's own literal heuristic (kept
|
|
65
|
+
exactly as worded, not silently reversed to match conventional
|
|
66
|
+
"entry point" intuition): a file with many inbound and few outbound
|
|
67
|
+
edges is read by the roadmap as looking like an API/entry point; a
|
|
68
|
+
file with many outbound and few inbound edges is read as looking
|
|
69
|
+
like a utility/leaf module. `rank_by_inbound`/`rank_by_outbound`
|
|
70
|
+
below surface the raw degree data sorted for each of those two
|
|
71
|
+
readings -- they attach no label, score, or threshold of any kind.
|
|
72
|
+
|
|
73
|
+
Clustering (approved, roadmap §37 item 4.3 audit -- empirically
|
|
74
|
+
evaluated against CodePulse's own repository before this decision
|
|
75
|
+
was made, not assumed): plain connected components over this same
|
|
76
|
+
file graph was tried first and produced one 27-file component plus
|
|
77
|
+
one singleton on this project's own `codepulse/` package -- not
|
|
78
|
+
"coherent module clusters" by any reasonable reading, and a
|
|
79
|
+
measured failure of B18's own self-referential DoD. Seeded Louvain
|
|
80
|
+
community detection (`networkx.algorithms.community.
|
|
81
|
+
louvain_communities`) over an UNDIRECTED view of the same edge set
|
|
82
|
+
produced multiple genuinely coherent clusters on the same
|
|
83
|
+
repository and is the approved algorithm. `_LOUVAIN_SEED` is fixed
|
|
84
|
+
and NOT exposed as a caller-supplied parameter this milestone --
|
|
85
|
+
empirically verified (same audit) that the identical seed produces
|
|
86
|
+
byte-identical output across repeated runs and across independent
|
|
87
|
+
processes with different `PYTHONHASHSEED` values, while a
|
|
88
|
+
different seed can produce a different partition of the same
|
|
89
|
+
graph. The seed is therefore part of this class's deterministic
|
|
90
|
+
contract, not an incidental default. Clusters are NEVER read as an
|
|
91
|
+
objective "correct" decomposition of the codebase -- they are a
|
|
92
|
+
deterministic, graph-derived grouping, reproducible for a given
|
|
93
|
+
snapshot and seed, nothing more.
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
_LOUVAIN_SEED = 0
|
|
97
|
+
|
|
98
|
+
def __init__(self, parse_results: dict[str, FileParseResult] | list[FileParseResult]) -> None:
|
|
99
|
+
self.resolver = ImportResolver(parse_results)
|
|
100
|
+
self.files: list[str] = sorted(self.resolver.files.keys())
|
|
101
|
+
|
|
102
|
+
self.adjacency: dict[str, set[str]] = {f: set() for f in self.files}
|
|
103
|
+
self.reverse_adjacency: dict[str, set[str]] = {f: set() for f in self.files}
|
|
104
|
+
|
|
105
|
+
for file_path, target in self._resolve_edges(self.resolver):
|
|
106
|
+
self.adjacency[file_path].add(target)
|
|
107
|
+
self.reverse_adjacency[target].add(file_path)
|
|
108
|
+
|
|
109
|
+
self.in_degree: dict[str, int] = {f: len(self.reverse_adjacency[f]) for f in self.files}
|
|
110
|
+
self.out_degree: dict[str, int] = {f: len(self.adjacency[f]) for f in self.files}
|
|
111
|
+
|
|
112
|
+
self.clusters: list[dict[str, Any]] = self._compute_clusters()
|
|
113
|
+
|
|
114
|
+
@staticmethod
|
|
115
|
+
def _resolve_edges(resolver: ImportResolver) -> list[tuple[str, str]]:
|
|
116
|
+
"""
|
|
117
|
+
The single place import statements become file-to-file edges, shared
|
|
118
|
+
by a full build (`__init__`) and `snapshot` so the two can never
|
|
119
|
+
disagree about what an edge is.
|
|
120
|
+
"""
|
|
121
|
+
edges: list[tuple[str, str]] = []
|
|
122
|
+
for file_path, res in resolver.files.items():
|
|
123
|
+
for imp in res.imports:
|
|
124
|
+
target = resolver.resolve_import_path(file_path, imp)
|
|
125
|
+
# `resolve_import_path` only ever returns a key already
|
|
126
|
+
# present in `resolver.files` -- the `in resolver.files`
|
|
127
|
+
# check is a defensive, explicit restatement of "only
|
|
128
|
+
# resolved, repository-local imports become edges," not
|
|
129
|
+
# a case this can actually fail today.
|
|
130
|
+
if target and target != file_path and target in resolver.files:
|
|
131
|
+
edges.append((file_path, target))
|
|
132
|
+
return edges
|
|
133
|
+
|
|
134
|
+
@classmethod
|
|
135
|
+
def snapshot(
|
|
136
|
+
cls, parse_results: dict[str, FileParseResult] | list[FileParseResult]
|
|
137
|
+
) -> tuple[list[str], list[tuple[str, str]]]:
|
|
138
|
+
"""
|
|
139
|
+
`(files, edges)` for these parse results: every parsed file
|
|
140
|
+
(sorted, isolated ones included) and the distinct, sorted
|
|
141
|
+
`(source_file, target_file)` resolved edges -- WITHOUT computing
|
|
142
|
+
clusters/degrees. This is what the indexer persists on every run,
|
|
143
|
+
so it must stay cheap; `from_edges` accepts exactly this pair.
|
|
144
|
+
"""
|
|
145
|
+
resolver = ImportResolver(parse_results)
|
|
146
|
+
return sorted(resolver.files.keys()), sorted(set(cls._resolve_edges(resolver)))
|
|
147
|
+
|
|
148
|
+
@property
|
|
149
|
+
def edges(self) -> list[tuple[str, str]]:
|
|
150
|
+
"""Sorted `(source_file, target_file)` pairs; the exact set `from_edges` accepts."""
|
|
151
|
+
return sorted((src, tgt) for src, targets in self.adjacency.items() for tgt in targets)
|
|
152
|
+
|
|
153
|
+
@classmethod
|
|
154
|
+
def from_edges(cls, files: list[str], edges: list[tuple[str, str]]) -> "ModuleGraphBuilder":
|
|
155
|
+
"""
|
|
156
|
+
Rebuilds the graph from the persisted node list (every parsed file,
|
|
157
|
+
including isolated ones) and resolved edges, with no parse results.
|
|
158
|
+
Every public structure -- adjacency, degrees, clusters, rankings --
|
|
159
|
+
is identical to a graph built from parse results with the same
|
|
160
|
+
files and edges (clustering is deterministic for a given graph and
|
|
161
|
+
seed). An edge naming a file that is not in `files` is ignored,
|
|
162
|
+
never allowed to invent a node.
|
|
163
|
+
|
|
164
|
+
`resolver` is empty on a restored graph; nothing public reads it.
|
|
165
|
+
"""
|
|
166
|
+
graph = cls([])
|
|
167
|
+
graph.files = sorted(set(files))
|
|
168
|
+
known = set(graph.files)
|
|
169
|
+
graph.adjacency = {f: set() for f in graph.files}
|
|
170
|
+
graph.reverse_adjacency = {f: set() for f in graph.files}
|
|
171
|
+
for src, tgt in edges:
|
|
172
|
+
if src in known and tgt in known and src != tgt:
|
|
173
|
+
graph.adjacency[src].add(tgt)
|
|
174
|
+
graph.reverse_adjacency[tgt].add(src)
|
|
175
|
+
graph.in_degree = {f: len(graph.reverse_adjacency[f]) for f in graph.files}
|
|
176
|
+
graph.out_degree = {f: len(graph.adjacency[f]) for f in graph.files}
|
|
177
|
+
graph.clusters = graph._compute_clusters()
|
|
178
|
+
return graph
|
|
179
|
+
|
|
180
|
+
def _compute_clusters(self) -> list[dict[str, Any]]:
|
|
181
|
+
graph = nx.Graph()
|
|
182
|
+
graph.add_nodes_from(self.files)
|
|
183
|
+
for src, targets in self.adjacency.items():
|
|
184
|
+
for tgt in targets:
|
|
185
|
+
graph.add_edge(src, tgt)
|
|
186
|
+
|
|
187
|
+
raw_communities = nx.algorithms.community.louvain_communities(
|
|
188
|
+
graph, seed=self._LOUVAIN_SEED
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
# NetworkX's own community container order is arbitrary/internal
|
|
192
|
+
# -- never used as the public id. Each community's own files are
|
|
193
|
+
# sorted lexicographically first, then communities themselves
|
|
194
|
+
# are sorted by (size descending, lexicographically-smallest
|
|
195
|
+
# member), and only THEN assigned sequential ids in that order,
|
|
196
|
+
# so the same graph and seed always produce the same clusters in
|
|
197
|
+
# the same order with the same ids, regardless of container
|
|
198
|
+
# iteration order.
|
|
199
|
+
sorted_file_lists = sorted(
|
|
200
|
+
(sorted(community) for community in raw_communities),
|
|
201
|
+
key=lambda files: (-len(files), files[0]),
|
|
202
|
+
)
|
|
203
|
+
return [{"id": idx, "files": files} for idx, files in enumerate(sorted_file_lists)]
|
|
204
|
+
|
|
205
|
+
def rank_by_inbound(self) -> list[dict[str, Any]]:
|
|
206
|
+
"""
|
|
207
|
+
Raw per-file degree data, sorted by `(-in_degree, out_degree,
|
|
208
|
+
file_path)` -- roadmap §11's literal "many inbound, few
|
|
209
|
+
outbound" heuristic reading (files ranked at the top here are
|
|
210
|
+
what the roadmap describes as looking like APIs/entry points;
|
|
211
|
+
see this class's own docstring for why that wording is kept
|
|
212
|
+
exactly as written rather than reversed to match conventional
|
|
213
|
+
"entry point" intuition). No score, label, or threshold is
|
|
214
|
+
attached -- only the measured `in_degree`/`out_degree` values.
|
|
215
|
+
"""
|
|
216
|
+
return [
|
|
217
|
+
{"file": f, "in_degree": self.in_degree[f], "out_degree": self.out_degree[f]}
|
|
218
|
+
for f in sorted(self.files, key=lambda f: (-self.in_degree[f], self.out_degree[f], f))
|
|
219
|
+
]
|
|
220
|
+
|
|
221
|
+
def rank_by_outbound(self) -> list[dict[str, Any]]:
|
|
222
|
+
"""
|
|
223
|
+
Raw per-file degree data, sorted by `(-out_degree, in_degree,
|
|
224
|
+
file_path)` -- roadmap §11's literal "many outbound, few
|
|
225
|
+
inbound" heuristic reading (files ranked at the top here are
|
|
226
|
+
what the roadmap describes as looking like utility/leaf
|
|
227
|
+
modules). No score, label, or threshold is attached.
|
|
228
|
+
"""
|
|
229
|
+
return [
|
|
230
|
+
{"file": f, "in_degree": self.in_degree[f], "out_degree": self.out_degree[f]}
|
|
231
|
+
for f in sorted(self.files, key=lambda f: (-self.out_degree[f], self.in_degree[f], f))
|
|
232
|
+
]
|