codepulse-engine 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. codepulse/__init__.py +5 -0
  2. codepulse/api/app.py +1336 -0
  3. codepulse/api/errors.py +172 -0
  4. codepulse/api/security.py +189 -0
  5. codepulse/auth.py +406 -0
  6. codepulse/cli.py +1052 -0
  7. codepulse/config.py +475 -0
  8. codepulse/data_location.py +78 -0
  9. codepulse/embeddings/chunker.py +138 -0
  10. codepulse/embeddings/embedder.py +120 -0
  11. codepulse/git_diff.py +201 -0
  12. codepulse/indexer/engine.py +2655 -0
  13. codepulse/indexer/ignore.py +120 -0
  14. codepulse/ingestion_safety.py +169 -0
  15. codepulse/jobs.py +257 -0
  16. codepulse/mcp_server.py +269 -0
  17. codepulse/models.py +140 -0
  18. codepulse/observability.py +130 -0
  19. codepulse/parser/__init__.py +8 -0
  20. codepulse/parser/base.py +58 -0
  21. codepulse/parser/factory.py +37 -0
  22. codepulse/parser/graph.py +502 -0
  23. codepulse/parser/javascript.py +428 -0
  24. codepulse/parser/module_graph.py +232 -0
  25. codepulse/parser/python.py +331 -0
  26. codepulse/rate_limit.py +142 -0
  27. codepulse/repository_identity.py +83 -0
  28. codepulse/search/bm25.py +342 -0
  29. codepulse/search/ranker.py +382 -0
  30. codepulse/search/tokenizer.py +80 -0
  31. codepulse/search/vector.py +357 -0
  32. codepulse/storage/db.py +526 -0
  33. codepulse/storage/migrations.py +347 -0
  34. codepulse/storage/schema_migrations/README +1 -0
  35. codepulse/storage/schema_migrations/env.py +70 -0
  36. codepulse/storage/schema_migrations/script.py.mako +28 -0
  37. codepulse/storage/schema_migrations/versions/135a7b064ea7_add_base_classes_and_relationship_type.py +82 -0
  38. codepulse/storage/schema_migrations/versions/20fc2d0607e2_add_file_content_hash_and_natural_key_.py +106 -0
  39. codepulse/storage/schema_migrations/versions/21a7cb7f1a85_initial_schema.py +105 -0
  40. codepulse/storage/schema_migrations/versions/2cd38eb12a7e_add_call_edge_repo_id_isolation_fix.py +112 -0
  41. codepulse/storage/schema_migrations/versions/3ba396e9c78f_add_symbol_natural_key_uniqueness.py +80 -0
  42. codepulse/storage/schema_migrations/versions/49572f834366_add_embedding_persistence_fields.py +102 -0
  43. codepulse/storage/schema_migrations/versions/7c3e5a91d2f4_add_module_edges.py +61 -0
  44. codepulse/storage/schema_migrations/versions/a41f0d2c6b8e_canonicalize_repository_urls.py +94 -0
  45. codepulse/storage/schema_migrations/versions/b0bdd63949f4_add_api_key_model.py +71 -0
  46. codepulse/storage/schema_migrations/versions/b9014c049e33_add_api_key_repository_role.py +65 -0
  47. codepulse/storage/schema_migrations/versions/c286d277ea21_add_repository_snapshot_model.py +62 -0
  48. codepulse/storage/schema_migrations/versions/de22df2fc3ce_add_index_job_model.py +59 -0
  49. codepulse/storage/schema_migrations/versions/f6e481353553_rename_dependency_to_call_edge_add_confidence.py +80 -0
  50. codepulse/symbol_identity.py +29 -0
  51. codepulse_engine-0.1.0.dist-info/METADATA +191 -0
  52. codepulse_engine-0.1.0.dist-info/RECORD +56 -0
  53. codepulse_engine-0.1.0.dist-info/WHEEL +5 -0
  54. codepulse_engine-0.1.0.dist-info/entry_points.txt +2 -0
  55. codepulse_engine-0.1.0.dist-info/licenses/LICENSE +21 -0
  56. codepulse_engine-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,428 @@
1
+ from __future__ import annotations
2
+
3
+ import hashlib
4
+ from typing import Any
5
+ from tree_sitter_languages import get_parser
6
+
7
+ from codepulse.models import CallReference, ImportItem, ImportStatement, Symbol, SymbolType
8
+ from codepulse.parser.base import LanguageParser
9
+
10
+
11
+ class JavaScriptParser(LanguageParser):
12
+ """AST parser for JavaScript and TypeScript source code using Tree-sitter."""
13
+
14
+ def __init__(self, is_typescript: bool = False) -> None:
15
+ self._is_typescript = is_typescript
16
+ try:
17
+ self._parser = get_parser("typescript" if is_typescript else "javascript")
18
+ except Exception:
19
+ # Fallback to javascript parser
20
+ self._parser = get_parser("javascript")
21
+
22
+ @property
23
+ def language(self) -> str:
24
+ return "typescript" if self._is_typescript else "javascript"
25
+
26
+ def extract_symbols(self, source_code: str, file_path: str = "") -> list[Symbol]:
27
+ source_bytes = source_code.encode("utf-8")
28
+ tree = self._parser.parse(source_bytes)
29
+ symbols: list[Symbol] = []
30
+
31
+ def walk(node: Any, parent_class: str | None = None) -> None:
32
+ # 1. Function Declarations: function foo() {}
33
+ if node.type in ("function_declaration", "generator_function_declaration"):
34
+ sym = self._parse_function_declaration(node, source_bytes, file_path, parent_class)
35
+ if sym:
36
+ symbols.append(sym)
37
+ body_node = node.child_by_field_name("body")
38
+ if body_node:
39
+ for child in body_node.children:
40
+ walk(child, parent_class=sym.name if sym else parent_class)
41
+ return
42
+
43
+ # 2. Variable declarations with arrow functions / function expressions: const foo = () => {}
44
+ elif node.type == "lexical_declaration" or node.type == "variable_declaration":
45
+ syms = self._parse_variable_functions(node, source_bytes, file_path, parent_class)
46
+ symbols.extend(syms)
47
+ for child in node.children:
48
+ walk(child, parent_class)
49
+ return
50
+
51
+ # 3. Class Declarations: class MyClass {}
52
+ elif node.type == "class_declaration":
53
+ sym = self._parse_class_declaration(node, source_bytes, file_path, parent_class)
54
+ if sym:
55
+ symbols.append(sym)
56
+ body_node = node.child_by_field_name("body")
57
+ if body_node:
58
+ for child in body_node.children:
59
+ walk(child, parent_class=sym.name if sym else parent_class)
60
+ return
61
+
62
+ # 4. Method Definitions inside class body: foo() {}
63
+ elif node.type == "method_definition":
64
+ sym = self._parse_method_definition(node, source_bytes, file_path, parent_class)
65
+ if sym:
66
+ symbols.append(sym)
67
+ body_node = node.child_by_field_name("body")
68
+ if body_node:
69
+ for child in body_node.children:
70
+ walk(child, parent_class=sym.name if sym else parent_class)
71
+ return
72
+
73
+ for child in node.children:
74
+ walk(child, parent_class)
75
+
76
+ walk(tree.root_node)
77
+ return symbols
78
+
79
+ def _parse_function_declaration(
80
+ self, node: Any, source_bytes: bytes, file_path: str, parent_class: str | None
81
+ ) -> Symbol | None:
82
+ name_node = node.child_by_field_name("name")
83
+ if not name_node:
84
+ return None
85
+
86
+ name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace")
87
+ start_line = node.start_point[0] + 1
88
+ end_line = node.end_point[0] + 1
89
+ body_text = source_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
90
+ docstring = self._extract_jsdoc(node, source_bytes)
91
+ signature = self._extract_signature(node, source_bytes)
92
+ complexity = self._calculate_complexity(node)
93
+
94
+ return Symbol(
95
+ name=name,
96
+ type=SymbolType.METHOD if parent_class else SymbolType.FUNCTION,
97
+ file_path=file_path,
98
+ start_line=start_line,
99
+ end_line=end_line,
100
+ signature=signature,
101
+ docstring=docstring,
102
+ body=body_text,
103
+ body_hash=hashlib.sha256(body_text.strip().encode("utf-8")).hexdigest(),
104
+ language=self.language,
105
+ parent_symbol=parent_class,
106
+ complexity=complexity,
107
+ )
108
+
109
+ def _parse_variable_functions(
110
+ self, node: Any, source_bytes: bytes, file_path: str, parent_class: str | None
111
+ ) -> list[Symbol]:
112
+ results: list[Symbol] = []
113
+ for child in node.children:
114
+ if child.type == "variable_declarator":
115
+ name_n = child.child_by_field_name("name")
116
+ value_n = child.child_by_field_name("value")
117
+ if name_n and value_n and value_n.type in ("arrow_function", "function_expression"):
118
+ name = source_bytes[name_n.start_byte:name_n.end_byte].decode("utf-8", errors="replace")
119
+ start_line = node.start_point[0] + 1
120
+ end_line = node.end_point[0] + 1
121
+ body_text = source_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
122
+ docstring = self._extract_jsdoc(node, source_bytes)
123
+ signature = self._extract_signature(node, source_bytes)
124
+ complexity = self._calculate_complexity(value_n)
125
+
126
+ results.append(
127
+ Symbol(
128
+ name=name,
129
+ type=SymbolType.METHOD if parent_class else SymbolType.FUNCTION,
130
+ file_path=file_path,
131
+ start_line=start_line,
132
+ end_line=end_line,
133
+ signature=signature,
134
+ docstring=docstring,
135
+ body=body_text,
136
+ body_hash=hashlib.sha256(body_text.strip().encode("utf-8")).hexdigest(),
137
+ language=self.language,
138
+ parent_symbol=parent_class,
139
+ complexity=complexity,
140
+ )
141
+ )
142
+ return results
143
+
144
+ def _parse_class_declaration(
145
+ self, node: Any, source_bytes: bytes, file_path: str, parent_class: str | None
146
+ ) -> Symbol | None:
147
+ name_node = node.child_by_field_name("name")
148
+ if not name_node:
149
+ return None
150
+
151
+ name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace")
152
+ start_line = node.start_point[0] + 1
153
+ end_line = node.end_point[0] + 1
154
+ body_text = source_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
155
+ docstring = self._extract_jsdoc(node, source_bytes)
156
+ signature = self._extract_signature(node, source_bytes)
157
+ base_classes = self._extract_base_classes(node, source_bytes)
158
+
159
+ return Symbol(
160
+ name=name,
161
+ type=SymbolType.CLASS,
162
+ file_path=file_path,
163
+ start_line=start_line,
164
+ end_line=end_line,
165
+ signature=signature,
166
+ docstring=docstring,
167
+ body=body_text,
168
+ body_hash=hashlib.sha256(body_text.strip().encode("utf-8")).hexdigest(),
169
+ language=self.language,
170
+ parent_symbol=parent_class,
171
+ complexity=1,
172
+ base_classes=base_classes,
173
+ )
174
+
175
+ def _extract_base_classes(self, node: Any, source_bytes: bytes) -> list[str]:
176
+ """
177
+ Phase 4.1 (roadmap §37 item 4.1: "Base-class/inheritance
178
+ extraction (Python + JS/TS)"). Confirmed via direct tree-sitter
179
+ grammar probes during this milestone's own design audit that the
180
+ PLAIN JAVASCRIPT and TYPESCRIPT grammars shape `class_heritage`
181
+ differently:
182
+
183
+ - Plain JavaScript: `class_declaration -> class_heritage ->
184
+ (extends, <base expression>)` -- the base expression is a
185
+ DIRECT child of `class_heritage` (no wrapper node).
186
+ - TypeScript: `class_declaration -> class_heritage ->
187
+ extends_clause -> (extends, <base expression>)`, with an
188
+ optional SIBLING `implements_clause` (interface
189
+ implementation) inside `class_heritage` alongside
190
+ `extends_clause`. `implements_clause` is DELIBERATELY never
191
+ inspected here -- interface implementation is a different
192
+ relationship than base-class inheritance, out of this
193
+ milestone's exact scope (roadmap §37 item 4.1 names
194
+ "base-class/inheritance extraction," not "implements");
195
+ only ever looking WITHIN `extends_clause` when present is
196
+ what keeps `implements_clause` categorically unreachable
197
+ here, not an extra exclusion check.
198
+
199
+ `<base expression>` is either an `identifier` (`Base`) or a
200
+ `member_expression` (`ns.Base`, a dotted/namespace-qualified
201
+ base) -- the only two node types that can actually name a base
202
+ class in either grammar. Returns the raw text exactly as
203
+ written; resolution happens separately in `CallGraphBuilder`.
204
+ """
205
+ heritage = None
206
+ for child in node.children:
207
+ if child.type == "class_heritage":
208
+ heritage = child
209
+ break
210
+ if heritage is None:
211
+ return []
212
+
213
+ extends_clause = None
214
+ for child in heritage.children:
215
+ if child.type == "extends_clause":
216
+ extends_clause = child
217
+ break
218
+
219
+ # TypeScript wraps the base expression in extends_clause (and
220
+ # that is the ONLY place searched, so implements_clause -- a
221
+ # sibling of extends_clause, never a descendant of it -- is
222
+ # never reached). Plain JavaScript has no such wrapper -- the
223
+ # base expression is a direct child of class_heritage itself.
224
+ search_within = extends_clause if extends_clause is not None else heritage
225
+
226
+ bases: list[str] = []
227
+ for child in search_within.children:
228
+ if child.type in ("identifier", "member_expression", "type_identifier"):
229
+ bases.append(
230
+ source_bytes[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
231
+ )
232
+ return bases
233
+
234
+ def _parse_method_definition(
235
+ self, node: Any, source_bytes: bytes, file_path: str, parent_class: str | None
236
+ ) -> Symbol | None:
237
+ name_node = node.child_by_field_name("name")
238
+ if not name_node:
239
+ return None
240
+
241
+ name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace")
242
+ start_line = node.start_point[0] + 1
243
+ end_line = node.end_point[0] + 1
244
+ body_text = source_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
245
+ docstring = self._extract_jsdoc(node, source_bytes)
246
+ signature = self._extract_signature(node, source_bytes)
247
+ complexity = self._calculate_complexity(node)
248
+
249
+ return Symbol(
250
+ name=name,
251
+ type=SymbolType.METHOD,
252
+ file_path=file_path,
253
+ start_line=start_line,
254
+ end_line=end_line,
255
+ signature=signature,
256
+ docstring=docstring,
257
+ body=body_text,
258
+ body_hash=hashlib.sha256(body_text.strip().encode("utf-8")).hexdigest(),
259
+ language=self.language,
260
+ parent_symbol=parent_class,
261
+ complexity=complexity,
262
+ )
263
+
264
+ def _extract_jsdoc(self, node: Any, source_bytes: bytes) -> str | None:
265
+ # Check previous sibling for comment
266
+ prev = node.prev_sibling
267
+ if prev and prev.type == "comment":
268
+ cmt = source_bytes[prev.start_byte:prev.end_byte].decode("utf-8", errors="replace")
269
+ if cmt.startswith("/**") and cmt.endswith("*/"):
270
+ lines = [l.strip().lstrip("/*").rstrip("*/").lstrip("*").strip() for l in cmt.splitlines()]
271
+ clean_lines = [l for l in lines if l]
272
+ return "\n".join(clean_lines) if clean_lines else None
273
+ elif cmt.startswith("//"):
274
+ return cmt.lstrip("/").strip()
275
+ return None
276
+
277
+ def _extract_signature(self, node: Any, source_bytes: bytes) -> str:
278
+ body_node = node.child_by_field_name("body")
279
+ if body_node:
280
+ sig_bytes = source_bytes[node.start_byte:body_node.start_byte]
281
+ sig_text = sig_bytes.decode("utf-8", errors="replace").strip()
282
+ return " ".join(sig_text.split())
283
+
284
+ full_text = source_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
285
+ return full_text.splitlines()[0].strip().rstrip("{").strip()
286
+
287
+ def _calculate_complexity(self, node: Any) -> int:
288
+ complexity = 1
289
+ branching_types = {
290
+ "if_statement", "for_statement", "for_in_statement", "while_statement",
291
+ "do_statement", "catch_clause", "switch_case", "ternary_expression",
292
+ "binary_expression" # Logical && and ||
293
+ }
294
+
295
+ def count_branches(n: Any) -> None:
296
+ nonlocal complexity
297
+ if n.type in branching_types:
298
+ complexity += 1
299
+ for child in n.children:
300
+ count_branches(child)
301
+
302
+ count_branches(node)
303
+ return complexity
304
+
305
+ def extract_imports(self, source_code: str, file_path: str = "") -> list[ImportStatement]:
306
+ source_bytes = source_code.encode("utf-8")
307
+ tree = self._parser.parse(source_bytes)
308
+ imports: list[ImportStatement] = []
309
+
310
+ def walk(node: Any) -> None:
311
+ # ES6: import { a, b as c } from './module';
312
+ # ES6: import React from 'react';
313
+ # ES6: import * as path from 'path';
314
+ if node.type == "import_statement":
315
+ line_num = node.start_point[0] + 1
316
+ source_node = node.child_by_field_name("source")
317
+ module_name = ""
318
+ if source_node:
319
+ module_name = source_bytes[source_node.start_byte:source_node.end_byte].decode("utf-8").strip("'\"")
320
+
321
+ is_relative = module_name.startswith(".")
322
+ level = 1 if module_name.startswith("./") else (2 if module_name.startswith("../") else 0)
323
+
324
+ items: list[ImportItem] = []
325
+ for child in node.children:
326
+ if child.type == "import_clause":
327
+ for sub in child.children:
328
+ if sub.type == "identifier":
329
+ default_name = source_bytes[sub.start_byte:sub.end_byte].decode("utf-8")
330
+ items.append(ImportItem(name=default_name, alias="default"))
331
+ elif sub.type == "named_imports":
332
+ for spec in sub.children:
333
+ if spec.type == "import_specifier":
334
+ name_n = spec.child_by_field_name("name")
335
+ alias_n = spec.child_by_field_name("alias")
336
+ if name_n:
337
+ item_name = source_bytes[name_n.start_byte:name_n.end_byte].decode("utf-8")
338
+ alias = source_bytes[alias_n.start_byte:alias_n.end_byte].decode("utf-8") if alias_n else None
339
+ items.append(ImportItem(name=item_name, alias=alias))
340
+ elif sub.type == "namespace_import":
341
+ for ns_child in sub.children:
342
+ if ns_child.type == "identifier":
343
+ alias = source_bytes[ns_child.start_byte:ns_child.end_byte].decode("utf-8")
344
+ items.append(ImportItem(name="*", alias=alias))
345
+
346
+ imports.append(
347
+ ImportStatement(
348
+ module=module_name,
349
+ items=items,
350
+ is_relative=is_relative,
351
+ level=level,
352
+ line_number=line_num,
353
+ file_path=file_path,
354
+ )
355
+ )
356
+
357
+ # CommonJS: const foo = require('foo')
358
+ elif node.type == "call_expression":
359
+ func_n = node.child_by_field_name("function")
360
+ args_n = node.child_by_field_name("arguments")
361
+ if func_n and args_n:
362
+ fn_name = source_bytes[func_n.start_byte:func_n.end_byte].decode("utf-8", errors="replace")
363
+ if fn_name == "require" and args_n.children:
364
+ for arg in args_n.children:
365
+ if arg.type == "string":
366
+ mod_name = source_bytes[arg.start_byte:arg.end_byte].decode("utf-8").strip("'\"")
367
+ line_num = node.start_point[0] + 1
368
+ is_rel = mod_name.startswith(".")
369
+ imports.append(
370
+ ImportStatement(
371
+ module=mod_name,
372
+ items=[],
373
+ is_relative=is_rel,
374
+ level=1 if mod_name.startswith("./") else (2 if mod_name.startswith("../") else 0),
375
+ line_number=line_num,
376
+ file_path=file_path,
377
+ )
378
+ )
379
+
380
+ for child in node.children:
381
+ walk(child)
382
+
383
+ walk(tree.root_node)
384
+ return imports
385
+
386
+ def extract_calls(
387
+ self, source_code: str, file_path: str = "", symbols: list[Symbol] | None = None
388
+ ) -> list[CallReference]:
389
+ if symbols is None:
390
+ symbols = self.extract_symbols(source_code, file_path=file_path)
391
+
392
+ source_bytes = source_code.encode("utf-8")
393
+ tree = self._parser.parse(source_bytes)
394
+ calls: list[CallReference] = []
395
+
396
+ def find_enclosing_symbol(line: int) -> str:
397
+ matching = [s for s in symbols if s.start_line <= line <= s.end_line]
398
+ if not matching:
399
+ return "<module>"
400
+ matching.sort(key=lambda s: (s.end_line - s.start_line))
401
+ return matching[0].name
402
+
403
+ def walk(node: Any) -> None:
404
+ if node.type == "call_expression":
405
+ func_node = node.child_by_field_name("function")
406
+ if func_node:
407
+ callee_name = source_bytes[func_node.start_byte:func_node.end_byte].decode("utf-8", errors="replace")
408
+ if func_node.type == "member_expression":
409
+ prop_node = func_node.child_by_field_name("property")
410
+ if prop_node:
411
+ callee_name = source_bytes[prop_node.start_byte:prop_node.end_byte].decode("utf-8", errors="replace")
412
+
413
+ line_num = node.start_point[0] + 1
414
+ caller = find_enclosing_symbol(line_num)
415
+ calls.append(
416
+ CallReference(
417
+ caller_symbol=caller,
418
+ callee_name=callee_name,
419
+ line_number=line_num,
420
+ file_path=file_path,
421
+ )
422
+ )
423
+
424
+ for child in node.children:
425
+ walk(child)
426
+
427
+ walk(tree.root_node)
428
+ return calls
@@ -0,0 +1,232 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import Any
4
+
5
+ import networkx as nx
6
+
7
+ from codepulse.models import FileParseResult
8
+ from codepulse.parser.graph import ImportResolver
9
+
10
+
11
+ class ModuleGraphBuilder:
12
+ """
13
+ Phase 4.3 (roadmap §37 item 4.3: "Deterministic module-clustering
14
+ via import graph"; §11 "Architecture discovery": "module/dependency-
15
+ cluster detection via the existing import graph... to identify
16
+ likely 'modules' and 'entry points'"; §45 backlog B18: "New
17
+ graph-analysis module").
18
+
19
+ Deliberately kept SEPARATE from `CallGraphBuilder` (roadmap §37
20
+ item 4.3's own instruction) -- this class operates on the
21
+ FILE-LEVEL import graph, not the symbol-level call/inheritance
22
+ graph. It reuses `ImportResolver.resolve_import_path` exactly as
23
+ written (no second import resolver, no duplicated resolution
24
+ logic) purely to turn each file's own `ImportStatement`s into
25
+ file-to-file edges.
26
+
27
+ Library/shared-core only, per this milestone's explicit approved
28
+ scope -- no REST route, CLI command, or MCP tool consumes this yet.
29
+ A future one can, through `IndexingEngine.analyze_module_structure`,
30
+ without duplicating any of this construction logic.
31
+
32
+ 2026-10-12 (user-approved, CODEPULSE_FUTURE_CHANGES.md item 15): the
33
+ resolved file-to-file edges are now persisted (`module_edges`) and a
34
+ graph can be rebuilt from them with `from_edges`, so module structure
35
+ survives a restart. `snapshot` computes just the files and edges (no
36
+ clustering) for the index-time write; `from_edges` then produces a
37
+ graph identical, in every public structure, to one built from parse
38
+ results.
39
+
40
+ Graph semantics (approved, roadmap §37 item 4.3 audit):
41
+ - Nodes are every file this repository's parser actually parsed
42
+ (`ImportResolver.files`), INCLUDING files with zero imports in
43
+ either direction -- isolated files are never silently omitted.
44
+ - An edge `source_file -> target_file` exists if `source_file`
45
+ contains at least one import statement that
46
+ `ImportResolver.resolve_import_path` resolves to `target_file`
47
+ (a real, repository-local file). Multiple import statements in
48
+ `source_file` resolving to the SAME `target_file` collapse to
49
+ exactly one edge -- this graph represents file-to-file
50
+ dependency EXISTENCE, not a raw import-statement count (unlike
51
+ `CallGraphEdge.call_count`, which deliberately does count).
52
+ - An import that does not resolve to a real file in this
53
+ repository (an external package, a stdlib module, or a genuine
54
+ parser gap) never becomes an edge and never creates a node for
55
+ the unresolved target -- `ImportResolver.resolve_import_path`
56
+ is the single source of truth for "does this import point at
57
+ one of this repository's own files," reused unmodified.
58
+ - A file importing itself is not recorded as a self-loop edge.
59
+
60
+ `in_degree`/`out_degree` are computed from this exact edge set (the
61
+ distinct-file-pair notion above), never from raw import-statement
62
+ counts. They are exposed as RAW, uninterpreted signals -- no
63
+ "importance"/"centrality"/"API probability" score is computed or
64
+ stored anywhere in this class. §11's own literal heuristic (kept
65
+ exactly as worded, not silently reversed to match conventional
66
+ "entry point" intuition): a file with many inbound and few outbound
67
+ edges is read by the roadmap as looking like an API/entry point; a
68
+ file with many outbound and few inbound edges is read as looking
69
+ like a utility/leaf module. `rank_by_inbound`/`rank_by_outbound`
70
+ below surface the raw degree data sorted for each of those two
71
+ readings -- they attach no label, score, or threshold of any kind.
72
+
73
+ Clustering (approved, roadmap §37 item 4.3 audit -- empirically
74
+ evaluated against CodePulse's own repository before this decision
75
+ was made, not assumed): plain connected components over this same
76
+ file graph was tried first and produced one 27-file component plus
77
+ one singleton on this project's own `codepulse/` package -- not
78
+ "coherent module clusters" by any reasonable reading, and a
79
+ measured failure of B18's own self-referential DoD. Seeded Louvain
80
+ community detection (`networkx.algorithms.community.
81
+ louvain_communities`) over an UNDIRECTED view of the same edge set
82
+ produced multiple genuinely coherent clusters on the same
83
+ repository and is the approved algorithm. `_LOUVAIN_SEED` is fixed
84
+ and NOT exposed as a caller-supplied parameter this milestone --
85
+ empirically verified (same audit) that the identical seed produces
86
+ byte-identical output across repeated runs and across independent
87
+ processes with different `PYTHONHASHSEED` values, while a
88
+ different seed can produce a different partition of the same
89
+ graph. The seed is therefore part of this class's deterministic
90
+ contract, not an incidental default. Clusters are NEVER read as an
91
+ objective "correct" decomposition of the codebase -- they are a
92
+ deterministic, graph-derived grouping, reproducible for a given
93
+ snapshot and seed, nothing more.
94
+ """
95
+
96
+ _LOUVAIN_SEED = 0
97
+
98
+ def __init__(self, parse_results: dict[str, FileParseResult] | list[FileParseResult]) -> None:
99
+ self.resolver = ImportResolver(parse_results)
100
+ self.files: list[str] = sorted(self.resolver.files.keys())
101
+
102
+ self.adjacency: dict[str, set[str]] = {f: set() for f in self.files}
103
+ self.reverse_adjacency: dict[str, set[str]] = {f: set() for f in self.files}
104
+
105
+ for file_path, target in self._resolve_edges(self.resolver):
106
+ self.adjacency[file_path].add(target)
107
+ self.reverse_adjacency[target].add(file_path)
108
+
109
+ self.in_degree: dict[str, int] = {f: len(self.reverse_adjacency[f]) for f in self.files}
110
+ self.out_degree: dict[str, int] = {f: len(self.adjacency[f]) for f in self.files}
111
+
112
+ self.clusters: list[dict[str, Any]] = self._compute_clusters()
113
+
114
+ @staticmethod
115
+ def _resolve_edges(resolver: ImportResolver) -> list[tuple[str, str]]:
116
+ """
117
+ The single place import statements become file-to-file edges, shared
118
+ by a full build (`__init__`) and `snapshot` so the two can never
119
+ disagree about what an edge is.
120
+ """
121
+ edges: list[tuple[str, str]] = []
122
+ for file_path, res in resolver.files.items():
123
+ for imp in res.imports:
124
+ target = resolver.resolve_import_path(file_path, imp)
125
+ # `resolve_import_path` only ever returns a key already
126
+ # present in `resolver.files` -- the `in resolver.files`
127
+ # check is a defensive, explicit restatement of "only
128
+ # resolved, repository-local imports become edges," not
129
+ # a case this can actually fail today.
130
+ if target and target != file_path and target in resolver.files:
131
+ edges.append((file_path, target))
132
+ return edges
133
+
134
+ @classmethod
135
+ def snapshot(
136
+ cls, parse_results: dict[str, FileParseResult] | list[FileParseResult]
137
+ ) -> tuple[list[str], list[tuple[str, str]]]:
138
+ """
139
+ `(files, edges)` for these parse results: every parsed file
140
+ (sorted, isolated ones included) and the distinct, sorted
141
+ `(source_file, target_file)` resolved edges -- WITHOUT computing
142
+ clusters/degrees. This is what the indexer persists on every run,
143
+ so it must stay cheap; `from_edges` accepts exactly this pair.
144
+ """
145
+ resolver = ImportResolver(parse_results)
146
+ return sorted(resolver.files.keys()), sorted(set(cls._resolve_edges(resolver)))
147
+
148
+ @property
149
+ def edges(self) -> list[tuple[str, str]]:
150
+ """Sorted `(source_file, target_file)` pairs; the exact set `from_edges` accepts."""
151
+ return sorted((src, tgt) for src, targets in self.adjacency.items() for tgt in targets)
152
+
153
+ @classmethod
154
+ def from_edges(cls, files: list[str], edges: list[tuple[str, str]]) -> "ModuleGraphBuilder":
155
+ """
156
+ Rebuilds the graph from the persisted node list (every parsed file,
157
+ including isolated ones) and resolved edges, with no parse results.
158
+ Every public structure -- adjacency, degrees, clusters, rankings --
159
+ is identical to a graph built from parse results with the same
160
+ files and edges (clustering is deterministic for a given graph and
161
+ seed). An edge naming a file that is not in `files` is ignored,
162
+ never allowed to invent a node.
163
+
164
+ `resolver` is empty on a restored graph; nothing public reads it.
165
+ """
166
+ graph = cls([])
167
+ graph.files = sorted(set(files))
168
+ known = set(graph.files)
169
+ graph.adjacency = {f: set() for f in graph.files}
170
+ graph.reverse_adjacency = {f: set() for f in graph.files}
171
+ for src, tgt in edges:
172
+ if src in known and tgt in known and src != tgt:
173
+ graph.adjacency[src].add(tgt)
174
+ graph.reverse_adjacency[tgt].add(src)
175
+ graph.in_degree = {f: len(graph.reverse_adjacency[f]) for f in graph.files}
176
+ graph.out_degree = {f: len(graph.adjacency[f]) for f in graph.files}
177
+ graph.clusters = graph._compute_clusters()
178
+ return graph
179
+
180
+ def _compute_clusters(self) -> list[dict[str, Any]]:
181
+ graph = nx.Graph()
182
+ graph.add_nodes_from(self.files)
183
+ for src, targets in self.adjacency.items():
184
+ for tgt in targets:
185
+ graph.add_edge(src, tgt)
186
+
187
+ raw_communities = nx.algorithms.community.louvain_communities(
188
+ graph, seed=self._LOUVAIN_SEED
189
+ )
190
+
191
+ # NetworkX's own community container order is arbitrary/internal
192
+ # -- never used as the public id. Each community's own files are
193
+ # sorted lexicographically first, then communities themselves
194
+ # are sorted by (size descending, lexicographically-smallest
195
+ # member), and only THEN assigned sequential ids in that order,
196
+ # so the same graph and seed always produce the same clusters in
197
+ # the same order with the same ids, regardless of container
198
+ # iteration order.
199
+ sorted_file_lists = sorted(
200
+ (sorted(community) for community in raw_communities),
201
+ key=lambda files: (-len(files), files[0]),
202
+ )
203
+ return [{"id": idx, "files": files} for idx, files in enumerate(sorted_file_lists)]
204
+
205
+ def rank_by_inbound(self) -> list[dict[str, Any]]:
206
+ """
207
+ Raw per-file degree data, sorted by `(-in_degree, out_degree,
208
+ file_path)` -- roadmap §11's literal "many inbound, few
209
+ outbound" heuristic reading (files ranked at the top here are
210
+ what the roadmap describes as looking like APIs/entry points;
211
+ see this class's own docstring for why that wording is kept
212
+ exactly as written rather than reversed to match conventional
213
+ "entry point" intuition). No score, label, or threshold is
214
+ attached -- only the measured `in_degree`/`out_degree` values.
215
+ """
216
+ return [
217
+ {"file": f, "in_degree": self.in_degree[f], "out_degree": self.out_degree[f]}
218
+ for f in sorted(self.files, key=lambda f: (-self.in_degree[f], self.out_degree[f], f))
219
+ ]
220
+
221
+ def rank_by_outbound(self) -> list[dict[str, Any]]:
222
+ """
223
+ Raw per-file degree data, sorted by `(-out_degree, in_degree,
224
+ file_path)` -- roadmap §11's literal "many outbound, few
225
+ inbound" heuristic reading (files ranked at the top here are
226
+ what the roadmap describes as looking like utility/leaf
227
+ modules). No score, label, or threshold is attached.
228
+ """
229
+ return [
230
+ {"file": f, "in_degree": self.in_degree[f], "out_degree": self.out_degree[f]}
231
+ for f in sorted(self.files, key=lambda f: (-self.out_degree[f], self.in_degree[f], f))
232
+ ]