java-codebase-rag 0.12.0__py3-none-any.whl → 0.12.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. java_codebase_rag-0.12.2.dist-info/METADATA +35 -0
  2. java_codebase_rag-0.12.2.dist-info/RECORD +4 -0
  3. {java_codebase_rag-0.12.0.dist-info → java_codebase_rag-0.12.2.dist-info}/WHEEL +1 -1
  4. java_codebase_rag/_deprecation.py +0 -103
  5. java_codebase_rag/_fdlimit.py +0 -56
  6. java_codebase_rag/_stdio.py +0 -32
  7. java_codebase_rag/_version.py +0 -35
  8. java_codebase_rag/absence/__init__.py +0 -0
  9. java_codebase_rag/absence/absence_diagnosis.py +0 -700
  10. java_codebase_rag/absence/absence_types.py +0 -124
  11. java_codebase_rag/absence/absence_vocab.py +0 -460
  12. java_codebase_rag/analysis/__init__.py +0 -0
  13. java_codebase_rag/analysis/pr_analysis.py +0 -563
  14. java_codebase_rag/analysis/resolve_service.py +0 -740
  15. java_codebase_rag/ast/__init__.py +0 -0
  16. java_codebase_rag/ast/ast_java.py +0 -2847
  17. java_codebase_rag/ast/ast_kotlin.py +0 -1794
  18. java_codebase_rag/ast/brownfield_events.py +0 -58
  19. java_codebase_rag/ast/chunk_heuristics.py +0 -83
  20. java_codebase_rag/ast/language.py +0 -117
  21. java_codebase_rag/cli.py +0 -1215
  22. java_codebase_rag/cli_dispatch.py +0 -251
  23. java_codebase_rag/cli_format.py +0 -85
  24. java_codebase_rag/cli_progress.py +0 -94
  25. java_codebase_rag/config.py +0 -833
  26. java_codebase_rag/eval/__init__.py +0 -1
  27. java_codebase_rag/eval/ground_truth.py +0 -100
  28. java_codebase_rag/eval/metrics.py +0 -107
  29. java_codebase_rag/eval/runner.py +0 -556
  30. java_codebase_rag/graph/__init__.py +0 -0
  31. java_codebase_rag/graph/build_ast_graph.py +0 -4593
  32. java_codebase_rag/graph/graph_enrich.py +0 -1940
  33. java_codebase_rag/graph/graph_types.py +0 -224
  34. java_codebase_rag/graph/java_ontology.py +0 -465
  35. java_codebase_rag/graph/ladybug_queries.py +0 -2213
  36. java_codebase_rag/graph/path_filtering.py +0 -509
  37. java_codebase_rag/index/__init__.py +0 -0
  38. java_codebase_rag/index/java_index_flow_lancedb.py +0 -879
  39. java_codebase_rag/index/java_index_v1_common.py +0 -33
  40. java_codebase_rag/install_data/__init__.py +0 -0
  41. java_codebase_rag/install_data/agents/explorer-rag-cli.md +0 -110
  42. java_codebase_rag/install_data/agents/explorer-rag-enhanced.md +0 -152
  43. java_codebase_rag/install_data/skills/explore-codebase/SKILL.md +0 -165
  44. java_codebase_rag/install_data/skills/explore-codebase-cli/SKILL.md +0 -107
  45. java_codebase_rag/installer.py +0 -2188
  46. java_codebase_rag/jrag.py +0 -4545
  47. java_codebase_rag/jrag_envelope.py +0 -1107
  48. java_codebase_rag/jrag_hints.py +0 -204
  49. java_codebase_rag/jrag_render.py +0 -926
  50. java_codebase_rag/lance_optimize.py +0 -264
  51. java_codebase_rag/mcp/__init__.py +0 -0
  52. java_codebase_rag/mcp/mcp_hints.py +0 -932
  53. java_codebase_rag/mcp/mcp_v2.py +0 -1916
  54. java_codebase_rag/mcp/server.py +0 -886
  55. java_codebase_rag/pipeline.py +0 -531
  56. java_codebase_rag/progress.py +0 -570
  57. java_codebase_rag/read_payloads.py +0 -781
  58. java_codebase_rag/search/__init__.py +0 -0
  59. java_codebase_rag/search/index_common.py +0 -10
  60. java_codebase_rag/search/search_lancedb.py +0 -1296
  61. java_codebase_rag/search/search_lexical.py +0 -449
  62. java_codebase_rag/search/search_scoring.py +0 -537
  63. java_codebase_rag/watch/__init__.py +0 -0
  64. java_codebase_rag/watch/client.py +0 -230
  65. java_codebase_rag/watch/daemon.py +0 -396
  66. java_codebase_rag/watch/lock.py +0 -201
  67. java_codebase_rag/watch/paths.py +0 -76
  68. java_codebase_rag/watch/protocol.py +0 -122
  69. java_codebase_rag/watch/server.py +0 -273
  70. java_codebase_rag/watch/warm.py +0 -105
  71. java_codebase_rag/watch/watcher.py +0 -394
  72. java_codebase_rag-0.12.0.dist-info/METADATA +0 -340
  73. java_codebase_rag-0.12.0.dist-info/RECORD +0 -75
  74. java_codebase_rag-0.12.0.dist-info/entry_points.txt +0 -5
  75. java_codebase_rag-0.12.0.dist-info/licenses/LICENSE +0 -21
  76. java_codebase_rag-0.12.0.dist-info/top_level.txt +0 -1
  77. /java_codebase_rag/__init__.py → /java_codebase_rag-0.12.2.dist-info/top_level.txt +0 -0
@@ -1,4593 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Four-pass AST-derived Knowledge Base builder (LadybugDB).
3
-
4
- Walks a Java source tree with `tree_sitter_java`, writes a deterministic graph of:
5
- Symbol nodes: package, file, class, interface, enum, record, annotation, method, constructor
6
- Route nodes: declaration-site routes (Spring MVC/WebFlux, Feign, Kafka, …)
7
- Rel tables: EXTENDS, IMPLEMENTS, INJECTS, DECLARES, OVERRIDES, CALLS, EXPOSES
8
-
9
- Pass 1 builds every node and in-memory resolution indexes.
10
- Pass 2 resolves each extends/implements/injection target using Java's lookup order
11
- (same file → explicit import → same package → wildcard import → java.lang → phantom).
12
- Pass 3 resolves static call sites into confidence-scored CALLS edges and DECLARES.
13
- Pass 4 emits Route rows plus Symbol→Route EXPOSES edges from literal annotation metadata.
14
-
15
- Usage:
16
- build_ast_graph.py --source-root <repo> [--ladybug-path <path>] [--verbose]
17
-
18
- Default LadybugDB database path resolution order:
19
- --ladybug-path CLI arg (path passed to ladybug.Database(...))
20
- JAVA_CODEBASE_RAG_INDEX_DIR/code_graph.lbug (if set and local)
21
- ./.java-codebase-rag/code_graph.lbug under cwd
22
-
23
- The LadybugDB DB is dropped and rebuilt on every run (Phase 1 is a full rebuild).
24
- """
25
- from __future__ import annotations
26
-
27
- import argparse
28
- import contextlib
29
- import hashlib
30
- import json
31
- import logging
32
- import os
33
- import re
34
- import sys
35
- import threading
36
- import time
37
- from collections import defaultdict
38
- from dataclasses import asdict, dataclass, field, replace
39
- from pathlib import Path
40
-
41
- import ladybug
42
- import pyarrow as pa
43
-
44
- from java_codebase_rag.ast.ast_java import (
45
- ONTOLOGY_VERSION,
46
- CallSite,
47
- JavaFileAst,
48
- MethodDecl,
49
- OutgoingCallDecl,
50
- TypeDecl,
51
- injection_annotation_names,
52
- lombok_required_args_annotations,
53
- )
54
- from java_codebase_rag.ast.language import backend_for
55
-
56
- # Kotlin multifile-facade merge (Task 9). Conditionally imported: on installs
57
- # without the ``tree-sitter-kotlin`` grammar wheel (Intel-Mac graph-only image),
58
- # ``ast_kotlin`` raises ImportError at module load. ``merge_multifile_facades``
59
- # is a no-op on Java ASTs (they have no facades), so absent grammar → merge is
60
- # skipped and Java indexing is byte-identical. Used in pass1 AFTER parsing all
61
- # files but BEFORE ``_register_type`` (unmerged ``@file:JvmMultifileClass``
62
- # facades share one FQN → ``tables.types[fqn] = entry`` would overwrite and one
63
- # file's top-level functions would silently vanish from resolution).
64
- try: # pragma: no cover - branch depends on whether the wheel is installed
65
- from java_codebase_rag.ast.ast_kotlin import (
66
- merge_multifile_facades as _merge_multifile_facades,
67
- )
68
- except ImportError: # grammar wheel absent — no .kt files to merge.
69
- _merge_multifile_facades = None # type: ignore[assignment]
70
-
71
- from java_codebase_rag.graph.graph_enrich import (
72
- _load_config_cross_service_resolution,
73
- classify_java_file,
74
- collect_annotation_meta_chain,
75
- load_brownfield_overrides,
76
- load_generated_detection,
77
- microservice_for_path,
78
- module_for_path,
79
- phantom_id,
80
- resolve_async_producer_for_method,
81
- resolve_http_client_for_method,
82
- resolve_role_and_capabilities,
83
- resolve_routes_for_method,
84
- symbol_id,
85
- )
86
- from java_codebase_rag.graph.path_filtering import LayeredIgnore, iter_source_files
87
- from java_codebase_rag.search.search_scoring import SYMBOL_FTS_INDEX as _SYMBOL_FTS_INDEX, _split_identifier
88
- from java_codebase_rag.graph.java_ontology import (
89
- CLIENT_KIND_FEIGN_METHOD,
90
- CLIENT_KIND_REST_TEMPLATE,
91
- VALID_CLIENT_KINDS,
92
- VALID_HTTP_CALL_MATCHES,
93
- VALID_PRODUCER_KINDS,
94
- )
95
-
96
- log = logging.getLogger(__name__)
97
-
98
- _VERBOSE_STDERR_LOCK = threading.Lock()
99
-
100
- _PASS1_START = "[graph] pass 1 · parsing Java files"
101
- _PASS2_START = "[graph] pass 2 · emitting EXTENDS / IMPLEMENTS / DECLARES rows"
102
- _PASS3_START = "[graph] pass 3 · call resolution (outgoing calls per site)"
103
- _PASS4_START = "[graph] pass 4 · route and EXPOSES extraction"
104
- _PASS5_START = "[graph] pass 5 · imperative HTTP_CALLS / ASYNC_CALLS edges"
105
- _PASS6_START = "[graph] pass 6 · cross-service call-edge matching"
106
- _WRITE_START = "[graph] writing · LadybugDB graph to disk"
107
-
108
-
109
- def _verbose_stderr_line(content: str) -> None:
110
- with _VERBOSE_STDERR_LOCK:
111
- print(content, file=sys.stderr, flush=True)
112
-
113
-
114
- def _emit_graph_progress(parts: dict[str, object], *, verbose: bool) -> None:
115
- """Emit one ``JCIRAG_PROGRESS kind=graph …`` line to stderr (gated by verbose).
116
-
117
- The parent process (``pipeline.run_build_ast_graph`` /
118
- ``run_incremental_graph``) passes ``--verbose`` in default AND verbose modes
119
- (only suppressed for ``--quiet``), so this structured progress surfaces in
120
- default mode (where the parent renders it) and verbose mode (raw relay). In
121
- ``--quiet`` the builder is never invoked with ``--verbose`` so nothing is
122
- emitted. Field order is fixed so the parser and tests can pin substrings.
123
- """
124
- if not verbose:
125
- return
126
- fields = ["kind=graph"]
127
- for key in ("pass", "done", "total", "status", "elapsed_s"):
128
- if key in parts:
129
- fields.append(f"{key}={parts[key]}")
130
- line = "JCIRAG_PROGRESS " + " ".join(fields)
131
- _verbose_stderr_line(line)
132
-
133
-
134
- # Pass-1 per-file tick cadence: bound stderr volume on huge trees without making
135
- # the bar feel stale. A final tick on pass completion carries status=done.
136
- _PASS1_TICK_EVERY = 25
137
-
138
-
139
- @contextlib.contextmanager
140
- def _graph_pass_progress(pass_label: str, *, verbose: bool):
141
- """Emit ``pass=N/6 status=running`` on entry and ``status=done elapsed_s=…``
142
- on exit for passes 2–6 (each advances the rendered bar by 1/6).
143
-
144
- Usage: ``with _graph_pass_progress("2/6", verbose=verbose): …``
145
- """
146
- if not verbose:
147
- yield
148
- return
149
- _emit_graph_progress({"pass": pass_label, "status": "running"}, verbose=verbose)
150
- t0 = time.time()
151
- try:
152
- yield
153
- finally:
154
- elapsed = time.time() - t0
155
- _emit_graph_progress(
156
- {"pass": pass_label, "status": "done", "elapsed_s": f"{elapsed:.2f}"},
157
- verbose=verbose,
158
- )
159
-
160
-
161
- class _VerbosePassHeartbeats:
162
- """Emit ``[tag] running … Ns elapsed`` every 5s on stderr while in scope (verbose only)."""
163
-
164
- def __init__(self, tag: str, *, verbose: bool) -> None:
165
- self._tag = tag
166
- self._verbose = verbose
167
- self._thr: threading.Thread | None = None
168
- self._stop: threading.Event | None = None
169
-
170
- def __enter__(self) -> None:
171
- if not self._verbose:
172
- return None
173
- self._stop = threading.Event()
174
- stop = self._stop
175
- tag = self._tag
176
-
177
- def worker() -> None:
178
- t0 = time.monotonic()
179
- while not stop.wait(timeout=5.0):
180
- elapsed = int(time.monotonic() - t0)
181
- _verbose_stderr_line(f"{tag} · {elapsed}s elapsed")
182
-
183
- self._thr = threading.Thread(target=worker, name=f"hb-{tag}", daemon=True)
184
- self._thr.start()
185
- return None
186
-
187
- def __exit__(self, exc_type, exc, tb) -> bool:
188
- if self._thr is not None and self._stop is not None:
189
- self._stop.set()
190
- self._thr.join(timeout=2.0)
191
- return False
192
-
193
-
194
- _JAVA_LANG_SIMPLE = frozenset({
195
- "Object", "String", "Integer", "Long", "Short", "Byte", "Boolean", "Double",
196
- "Float", "Character", "Number", "Void", "Class", "Enum", "Record",
197
- "Throwable", "Exception", "RuntimeException", "Error", "Thread", "Runnable",
198
- "Iterable", "Comparable", "CharSequence", "StringBuilder", "StringBuffer",
199
- "Math", "System", "AutoCloseable", "Cloneable",
200
- })
201
-
202
-
203
- # Kotlin default-import simple names → deterministic stdlib FQN (Task 13).
204
- # Kotlin implicitly imports kotlin.*, kotlin.collections.*, kotlin.sequences.*,
205
- # kotlin.ranges.*, kotlin.text.*, kotlin.comparisons.*, kotlin.annotation.*,
206
- # kotlin.reflect.*, kotlin.jvm.*, kotlin.io.*, kotlin.math.*, kotlin.contracts.*.
207
- # Types from these (e.g. ``List``, ``Map``, ``Sequence``, ``Pair``) referenced as
208
- # supertypes would otherwise collapse to bare-name phantoms (``fqn="List"``). This
209
- # map gives the phantom fallback a deterministic FQN, mirroring ``_JAVA_LANG_SIMPLE``
210
- # for Java. Consulted ONLY when ``ast.language == "kotlin"`` (Kotlin-gated), so the
211
- # Java resolution path is byte-identical. Curated to the commonly-referenced stdlib
212
- # types; unknown simples keep falling through to the bare-name guess.
213
- _KOTLIN_DEFAULT_SIMPLE: dict[str, str] = {
214
- # kotlin.*
215
- "Any": "kotlin.Any", "Unit": "kotlin.Unit", "Nothing": "kotlin.Nothing",
216
- "Int": "kotlin.Int", "Long": "kotlin.Long", "Short": "kotlin.Short",
217
- "Byte": "kotlin.Byte", "Double": "kotlin.Double", "Float": "kotlin.Float",
218
- "Boolean": "kotlin.Boolean", "Char": "kotlin.Char", "String": "kotlin.String",
219
- "Array": "kotlin.Array", "Pair": "kotlin.Pair", "Triple": "kotlin.Triple",
220
- "Result": "kotlin.Result", "Enum": "kotlin.Enum", "Annotation": "kotlin.Annotation",
221
- "Throwable": "kotlin.Throwable", "Exception": "kotlin.Exception",
222
- "Error": "kotlin.Error", "Lazy": "kotlin.Lazy",
223
- # kotlin.collections.*
224
- "List": "kotlin.collections.List", "MutableList": "kotlin.collections.MutableList",
225
- "Set": "kotlin.collections.Set", "MutableSet": "kotlin.collections.MutableSet",
226
- "Map": "kotlin.collections.Map", "MutableMap": "kotlin.collections.MutableMap",
227
- "Collection": "kotlin.collections.Collection",
228
- "MutableCollection": "kotlin.collections.MutableCollection",
229
- "Iterable": "kotlin.collections.Iterable",
230
- "MutableIterable": "kotlin.collections.MutableIterable",
231
- "ArrayList": "kotlin.collections.ArrayList", "HashMap": "kotlin.collections.HashMap",
232
- "HashSet": "kotlin.collections.HashSet",
233
- "LinkedHashMap": "kotlin.collections.LinkedHashMap",
234
- "LinkedHashSet": "kotlin.collections.LinkedHashSet",
235
- "Grouping": "kotlin.collections.Grouping",
236
- "Iterator": "kotlin.collections.Iterator",
237
- "MutableIterator": "kotlin.collections.MutableIterator",
238
- "ListIterator": "kotlin.collections.ListIterator",
239
- "MutableListIterator": "kotlin.collections.MutableListIterator",
240
- "Comparator": "kotlin.Comparator",
241
- # kotlin.sequences.*
242
- "Sequence": "kotlin.sequences.Sequence",
243
- # kotlin.ranges.*
244
- "IntRange": "kotlin.ranges.IntRange", "LongRange": "kotlin.ranges.LongRange",
245
- "CharRange": "kotlin.ranges.CharRange", "ClosedRange": "kotlin.ranges.ClosedRange",
246
- "OpenEndRange": "kotlin.ranges.OpenEndRange",
247
- # kotlin.text.*
248
- "Regex": "kotlin.text.Regex", "Appendable": "kotlin.text.Appendable",
249
- "MatchResult": "kotlin.text.MatchResult",
250
- # kotlin.reflect.*
251
- "KClass": "kotlin.reflect.KClass", "KCallable": "kotlin.reflect.KCallable",
252
- "KProperty": "kotlin.reflect.KProperty", "KFunction": "kotlin.reflect.KFunction",
253
- "KType": "kotlin.reflect.KType", "KParameter": "kotlin.reflect.KParameter",
254
- }
255
-
256
-
257
- # ---------- dataclasses ----------
258
-
259
-
260
- @dataclass
261
- class TypeIndexEntry:
262
- """Pass-1 record for a type declaration + any methods/constructors inside it."""
263
- decl: TypeDecl
264
- file_path: str
265
- module: str
266
- microservice: str
267
- package: str
268
- outer_fqn: str | None
269
- node_id: str
270
- # True when this entry was loaded from the existing graph by
271
- # `_load_existing_types` (an unchanged-file stub used only for cross-file
272
- # resolution). Its `decl` is a placeholder (no annotations/methods), so its
273
- # recomputed role/capabilities must never be written back over the real
274
- # stored values. See `_write_nodes_impl`.
275
- loaded_from_db: bool = False
276
-
277
-
278
- @dataclass
279
- class MemberEntry:
280
- kind: str # method | constructor
281
- decl: MethodDecl
282
- parent_id: str
283
- parent_fqn: str
284
- file_path: str
285
- module: str
286
- microservice: str
287
- node_id: str
288
- # True when loaded from the existing graph by `_load_existing_members`
289
- # (an unchanged-file stub used only for cross-file call resolution). Its
290
- # DECLARES edge already persists in the graph, so it must not be re-emitted
291
- # by `_populate_declares_rows` (REL tables have no PK → would duplicate).
292
- loaded_from_db: bool = False
293
-
294
-
295
- @dataclass
296
- class EdgeRow:
297
- src_id: str
298
- dst_id: str
299
- dst_name: str
300
- dst_fqn: str
301
- resolved: bool
302
-
303
-
304
- @dataclass
305
- class InjectsRow(EdgeRow):
306
- mechanism: str = ""
307
- annotation: str = ""
308
- field_or_param: str = ""
309
-
310
-
311
- @dataclass
312
- class CallsRow:
313
- src_id: str
314
- dst_id: str
315
- call_site_line: int = 0
316
- call_site_byte: int = 0
317
- arg_count: int = 0
318
- confidence: float = 0.0
319
- strategy: str = "phantom"
320
- source: str = "static"
321
- resolved: bool = True
322
- callee_declaring_role: str = "OTHER"
323
-
324
-
325
- @dataclass
326
- class UnresolvedCallSiteRow:
327
- id: str
328
- caller_id: str
329
- call_site_line: int
330
- call_site_byte: int
331
- arg_count: int
332
- callee_simple: str
333
- receiver_expr: str
334
- reason: str
335
-
336
-
337
- @dataclass
338
- class DeclaresRow:
339
- src_id: str
340
- dst_id: str
341
-
342
-
343
- @dataclass
344
- class CallResolutionStats:
345
- total: int = 0
346
- by_strategy: dict[str, int] = field(default_factory=lambda: defaultdict(int))
347
- phantom_chained: int = 0
348
- phantom_other: int = 0
349
- callee_unresolved: int = 0
350
- skipped_cross_service: int = 0
351
-
352
-
353
- @dataclass
354
- class RouteRow:
355
- id: str
356
- kind: str
357
- framework: str
358
- method: str
359
- path: str
360
- path_template: str
361
- path_regex: str
362
- topic: str
363
- broker: str
364
- feign_name: str
365
- feign_url: str
366
- microservice: str
367
- module: str
368
- filename: str
369
- start_line: int
370
- end_line: int
371
- resolved: bool
372
- # B2a brownfield composition (PR-A3); not persisted on LadybugDB `Route` nodes.
373
- source_layer: str = "builtin"
374
-
375
-
376
- @dataclass
377
- class ExposesRow:
378
- symbol_id: str
379
- route_id: str
380
- confidence: float
381
- strategy: str
382
-
383
-
384
- @dataclass
385
- class RouteExtractionStats:
386
- routes_skipped_unresolved: int = 0
387
- by_framework: dict[str, int] = field(default_factory=lambda: defaultdict(int))
388
- by_kind: dict[str, int] = field(default_factory=lambda: defaultdict(int))
389
- routes_resolved_pct: float = 100.0
390
- # Percentage of emitted `Route` rows whose `source_layer` is not `builtin`.
391
- # Brownfield layers: `layer_b_ann`, `layer_a_meta`, `layer_c_source`, `layer_b_fqn`.
392
- routes_from_brownfield_pct: float = 0.0
393
- routes_by_layer: dict[str, int] = field(default_factory=dict)
394
- exposes_suppressed_feign: int = 0
395
-
396
-
397
- @dataclass
398
- class HttpCallRow:
399
- client_id: str
400
- route_id: str
401
- confidence: float
402
- strategy: str
403
- method_call: str
404
- raw_uri: str
405
- match: str
406
-
407
-
408
- @dataclass
409
- class AsyncCallRow:
410
- producer_id: str
411
- route_id: str
412
- confidence: float
413
- strategy: str
414
- direction: str
415
- raw_topic: str
416
- match: str
417
-
418
-
419
- @dataclass
420
- class ClientRow:
421
- id: str
422
- client_kind: str
423
- target_service: str
424
- path: str
425
- path_template: str
426
- path_regex: str
427
- method: str
428
- member_fqn: str
429
- member_id: str
430
- microservice: str
431
- module: str
432
- filename: str
433
- start_line: int
434
- end_line: int
435
- resolved: bool
436
- source_layer: str
437
-
438
-
439
- @dataclass
440
- class DeclaresClientRow:
441
- symbol_id: str
442
- client_id: str
443
- confidence: float
444
- strategy: str
445
-
446
-
447
- @dataclass
448
- class ProducerRow:
449
- id: str
450
- producer_kind: str
451
- topic: str
452
- broker: str
453
- direction: str
454
- member_fqn: str
455
- member_id: str
456
- microservice: str
457
- module: str
458
- filename: str
459
- start_line: int
460
- end_line: int
461
- resolved: bool
462
- source_layer: str
463
-
464
-
465
- @dataclass
466
- class DeclaresProducerRow:
467
- symbol_id: str
468
- producer_id: str
469
- confidence: float
470
- strategy: str
471
-
472
-
473
- @dataclass
474
- class ClientExtractionStats:
475
- clients_total: int = 0
476
- declares_client_total: int = 0
477
- clients_by_kind: dict[str, int] = field(default_factory=lambda: defaultdict(int))
478
-
479
-
480
- @dataclass
481
- class ProducerExtractionStats:
482
- producers_total: int = 0
483
- declares_producer_total: int = 0
484
- producers_by_kind: dict[str, int] = field(default_factory=lambda: defaultdict(int))
485
-
486
-
487
- @dataclass
488
- class CallEdgeStats:
489
- http_calls_total: int = 0
490
- async_calls_total: int = 0
491
- http_calls_by_client_kind: dict[str, int] = field(default_factory=lambda: defaultdict(int))
492
- async_calls_by_client_kind: dict[str, int] = field(default_factory=lambda: defaultdict(int))
493
- http_calls_by_strategy: dict[str, int] = field(default_factory=lambda: defaultdict(int))
494
- async_calls_by_strategy: dict[str, int] = field(default_factory=lambda: defaultdict(int))
495
- http_calls_skipped_unresolved: int = 0
496
- async_calls_skipped_unresolved: int = 0
497
- http_clients_from_brownfield_pct: float = 0.0
498
- async_producers_from_brownfield_pct: float = 0.0
499
- http_calls_match_breakdown: dict[str, int] = field(default_factory=lambda: defaultdict(int))
500
- async_calls_match_breakdown: dict[str, int] = field(default_factory=lambda: defaultdict(int))
501
- cross_service_calls_total: int = 0
502
-
503
-
504
- @dataclass
505
- class GraphTables:
506
- types: dict[str, TypeIndexEntry] = field(default_factory=dict) # fqn -> entry
507
- by_simple_name: dict[str, list[TypeIndexEntry]] = field(default_factory=dict)
508
- by_package: dict[str, list[TypeIndexEntry]] = field(default_factory=dict)
509
- files: dict[str, str] = field(default_factory=dict) # path -> node id
510
- packages: dict[str, str] = field(default_factory=dict) # pkg -> node id
511
- members: list[MemberEntry] = field(default_factory=list)
512
- phantoms: dict[str, dict] = field(default_factory=dict) # id -> row
513
- extends_rows: list[EdgeRow] = field(default_factory=list)
514
- implements_rows: list[EdgeRow] = field(default_factory=list)
515
- injects_rows: list[InjectsRow] = field(default_factory=list)
516
- calls_rows: list[CallsRow] = field(default_factory=list)
517
- unresolved_call_site_rows: list[UnresolvedCallSiteRow] = field(default_factory=list)
518
- declares_rows: list[DeclaresRow] = field(default_factory=list)
519
- routes_rows: list[RouteRow] = field(default_factory=list)
520
- exposes_rows: list[ExposesRow] = field(default_factory=list)
521
- http_call_rows: list[HttpCallRow] = field(default_factory=list)
522
- async_call_rows: list[AsyncCallRow] = field(default_factory=list)
523
- client_rows: list[ClientRow] = field(default_factory=list)
524
- declares_client_rows: list[DeclaresClientRow] = field(default_factory=list)
525
- producer_rows: list[ProducerRow] = field(default_factory=list)
526
- declares_producer_rows: list[DeclaresProducerRow] = field(default_factory=list)
527
- overrides_rows: list[DeclaresRow] = field(default_factory=list)
528
- route_stats: RouteExtractionStats = field(default_factory=RouteExtractionStats)
529
- call_edge_stats: CallEdgeStats = field(default_factory=CallEdgeStats)
530
- client_stats: ClientExtractionStats = field(default_factory=ClientExtractionStats)
531
- producer_stats: ProducerExtractionStats = field(default_factory=ProducerExtractionStats)
532
- methods_by_type: dict[str, list[MemberEntry]] = field(default_factory=dict)
533
- parse_errors: int = 0
534
- skipped_files: int = 0
535
- pass3_skipped_cross_service: int = 0
536
- pass3_unresolved_phantom_receiver: int = 0
537
- pass3_unresolved_chained: int = 0
538
- cross_service_resolution: str = "auto"
539
- # Populated in _write_nodes (same overrides + meta_chain as Symbol.role).
540
- type_role_by_node_id: dict[str, str] = field(default_factory=dict)
541
- # Populated in pass 1 (classify_java_file) and _load_existing_types for incremental rebuilds.
542
- type_generated_by_node_id: dict[str, tuple[bool, str | None]] = field(default_factory=dict)
543
- # Per-build dedup for the same-FQN cross-file collision warning in
544
- # ``_register_type`` (one warning per colliding FQN per build — avoids
545
- # spam when >2 files claim one FQN). See ``_register_type``.
546
- _warned_fqn_collisions: set[str] = field(default_factory=set)
547
-
548
-
549
- @dataclass
550
- class IncrementalResult:
551
- """Result of an incremental graph rebuild."""
552
- mode: str # "incremental" | "full_fallback"
553
- files_changed: int
554
- files_added: int
555
- files_removed: int
556
- dependents_reprocessed: int
557
- elapsed_sec: float
558
-
559
-
560
- # --- Builder-owned files in the index dir (single source of truth) ---------------
561
- # Every artifact the graph builder writes next to code_graph.lbug. The lifecycle
562
- # CLI's `erase` clears all of these from one list so the builder and erase cannot
563
- # drift (issues #349 / #350): previously erase hardcoded ".graph_hashes.json" only
564
- # and left the crash marker (.graph_increment_in_progress) and the atomic-write
565
- # temp (.graph_hashes.json.tmp) behind on disk.
566
- GRAPH_HASHES_FILENAME = ".graph_hashes.json"
567
- GRAPH_HASHES_TMP_FILENAME = ".graph_hashes.json.tmp"
568
- GRAPH_INCREMENT_MARKER_FILENAME = ".graph_increment_in_progress"
569
- BUILDER_OWNED_INDEX_FILES: tuple[str, ...] = (
570
- GRAPH_HASHES_FILENAME,
571
- GRAPH_HASHES_TMP_FILENAME,
572
- GRAPH_INCREMENT_MARKER_FILENAME,
573
- )
574
-
575
-
576
- class FileHashTracker:
577
- """Track content hashes for incremental graph rebuild."""
578
- def __init__(self, index_dir: Path):
579
- self._path = index_dir / GRAPH_HASHES_FILENAME
580
- self._hashes: dict[str, str] = {} # rel_path -> sha256_hex
581
-
582
- def load(self) -> None:
583
- """Load hashes from disk. No-op if file missing (first run)."""
584
- if not self._path.exists():
585
- return
586
- try:
587
- with open(self._path, "r", encoding="utf-8") as f:
588
- self._hashes = json.load(f)
589
- except (json.JSONDecodeError, OSError):
590
- # Corrupt or unreadable hash file; start fresh.
591
- self._hashes = {}
592
-
593
- def save(self) -> None:
594
- """Persist hashes to disk atomically (write .tmp, rename)."""
595
- tmp_path = self._path.parent / GRAPH_HASHES_TMP_FILENAME
596
- try:
597
- with open(tmp_path, "w", encoding="utf-8") as f:
598
- json.dump(self._hashes, f, sort_keys=True)
599
- os.replace(tmp_path, self._path)
600
- except OSError as e:
601
- # Fail gracefully; next run will treat as missing and rebuild.
602
- log.warning("Failed to save hash file %s: %s; next run will rebuild from scratch", self._path, e)
603
-
604
- def detect_changes(self, source_root: Path, ignore: LayeredIgnore) -> tuple[set[str], set[str], set[str]]:
605
- """Return (added, changed, removed) sets of relative POSIX paths."""
606
- current_files: set[str] = set()
607
- # Resolve source_root to handle symlinks
608
- source_root_resolved = source_root.resolve()
609
- for abs_path in iter_source_files(source_root, ignore=ignore):
610
- # Resolve the absolute path and compute relative path
611
- abs_path_resolved = abs_path.resolve()
612
- try:
613
- rel_path = abs_path_resolved.relative_to(source_root_resolved).as_posix()
614
- except ValueError:
615
- # Fallback to using the path as-is if it's not under source_root
616
- rel_path = abs_path.as_posix()
617
- current_files.add(rel_path)
618
-
619
- added: set[str] = set()
620
- changed: set[str] = set()
621
- removed: set[str] = set()
622
-
623
- # Detect added and changed files.
624
- for rel_path in current_files:
625
- abs_path = source_root / rel_path
626
- try:
627
- file_hash = _hash_file(abs_path)
628
- except FileNotFoundError:
629
- continue
630
- stored_hash = self._hashes.get(rel_path)
631
- if stored_hash is None:
632
- added.add(rel_path)
633
- elif stored_hash != file_hash:
634
- changed.add(rel_path)
635
-
636
- # Detect removed files.
637
- for rel_path in self._hashes:
638
- if rel_path not in current_files:
639
- removed.add(rel_path)
640
-
641
- return added, changed, removed
642
-
643
- def update(self, rel_paths: set[str], source_root: Path) -> None:
644
- """Compute and store hashes for the given paths."""
645
- for rel_path in rel_paths:
646
- abs_path = source_root / rel_path
647
- if abs_path.exists():
648
- self._hashes[rel_path] = _hash_file(abs_path)
649
-
650
-
651
- def _hash_file(abs_path: Path) -> str:
652
- """Compute SHA-256 hash of a file's raw bytes."""
653
- hasher = hashlib.sha256()
654
- with open(abs_path, "rb") as f:
655
- for chunk in iter(lambda: f.read(65536), b""):
656
- hasher.update(chunk)
657
- return hasher.hexdigest()
658
-
659
-
660
- # ---------- incremental rebuild helpers ----------
661
-
662
-
663
- def _load_existing_types(conn: ladybug.Connection, tables: GraphTables, exclude_files: set[str] | None = None) -> None:
664
- """Load type entries from existing LadybugDB graph into tables for cross-file resolution.
665
-
666
- When exclude_files is provided, only load types from files NOT in the set.
667
- """
668
- if exclude_files is not None and not exclude_files:
669
- return
670
-
671
- where = f"WHERE s.kind IN {list(_TYPE_KINDS)}"
672
- params: dict = {}
673
- if exclude_files:
674
- where += "\n AND NOT (s.filename IN $exclude_files)"
675
- params["exclude_files"] = list(exclude_files)
676
-
677
- query = f"""
678
- MATCH (s:Symbol)
679
- {where}
680
- RETURN s.kind, s.fqn, s.name, s.filename, s.module, s.microservice, s.id, s.role, s.generated, s.generated_by
681
- """
682
- result = conn.execute(query, params)
683
- while result.has_next():
684
- row = result.get_next()
685
- kind, fqn, name, filename = row[0], row[1], row[2], row[3]
686
- module = row[4] if len(row) > 4 else ""
687
- microservice = row[5] if len(row) > 5 else ""
688
- node_id = row[6] if len(row) > 6 else ""
689
- role = row[7] if len(row) > 7 else ""
690
- generated = row[8] if len(row) > 8 else False
691
- generated_by = row[9] if len(row) > 9 else ""
692
-
693
- decl = TypeDecl(name, kind, fqn)
694
- package = fqn[: -(len(name) + 1)] if fqn.endswith("." + name) else ""
695
-
696
- entry = TypeIndexEntry(
697
- decl=decl,
698
- file_path=filename,
699
- module=module,
700
- microservice=microservice,
701
- package=package,
702
- outer_fqn=None,
703
- node_id=node_id,
704
- loaded_from_db=True,
705
- )
706
- tables.types[fqn] = entry
707
- tables.by_simple_name.setdefault(name, []).append(entry)
708
- tables.by_package.setdefault(package, []).append(entry)
709
- # Seed the persisted role so the annotation-less stub is not recomputed to
710
- # the default during node staging (issue #352 divergence #2).
711
- if role:
712
- tables.type_role_by_node_id[node_id] = role
713
- # Seed the persisted generated/generated_by so stubs retain their values
714
- tables.type_generated_by_node_id[node_id] = (bool(generated), generated_by)
715
-
716
-
717
- def _load_existing_members(conn: ladybug.Connection, tables: GraphTables, exclude_files: set[str] | None = None) -> None:
718
- """Load member entries from existing LadybugDB graph into tables.members.
719
-
720
- When exclude_files is provided, only load members from files NOT in the set.
721
- """
722
- if exclude_files is not None and not exclude_files:
723
- return
724
-
725
- where = "WHERE s.kind IN ['method', 'constructor']"
726
- params: dict = {}
727
- if exclude_files:
728
- where += "\n AND NOT (s.filename IN $exclude_files)"
729
- params["exclude_files"] = list(exclude_files)
730
-
731
- query = f"""
732
- MATCH (s:Symbol)
733
- {where}
734
- RETURN s.kind, s.name, s.filename, s.signature, s.parent_id, s.fqn, s.id
735
- """
736
- result = conn.execute(query, params)
737
- while result.has_next():
738
- row = result.get_next()
739
- kind, name, filename = row[0], row[1], row[2]
740
- signature = row[3] if len(row) > 3 else ""
741
- parent_id = row[4] if len(row) > 4 else ""
742
- fqn = row[5] if len(row) > 5 else ""
743
- node_id = row[6] if len(row) > 6 else ""
744
-
745
- parent_fqn = fqn.split("#")[0] if "#" in fqn else ""
746
-
747
- decl = MethodDecl(name, "", kind == "constructor")
748
- decl.signature = signature
749
-
750
- tables.members.append(MemberEntry(
751
- kind=kind,
752
- decl=decl,
753
- parent_id=parent_id,
754
- parent_fqn=parent_fqn,
755
- file_path=filename,
756
- module="",
757
- microservice="",
758
- node_id=node_id,
759
- loaded_from_db=True,
760
- ))
761
-
762
-
763
- # Every Symbol->Symbol REL TABLE type in the graph schema. A Symbol node can
764
- # only have an INCOMING edge of one of these types, so `_find_dependents` MUST
765
- # walk all of them: that completeness is what makes the changed-node DETACH
766
- # DELETE in `_delete_file_scope` Phase 3 safe (every real caller of a changed
767
- # node is pulled into scope, so Phase 1 removes the edge before the node delete).
768
- # If you add a new Symbol->Symbol edge type to the schema, add it here too —
769
- # otherwise changed-node deletion would silently drop its surviving edges.
770
- _SYMBOL_TO_SYMBOL_EDGE_TYPES = (
771
- "EXTENDS", "IMPLEMENTS", "INJECTS", "CALLS", "DECLARES", "OVERRIDES",
772
- )
773
-
774
-
775
- def _find_dependents(conn: ladybug.Connection, changed_node_ids: set[str]) -> set[str]:
776
- """Find files whose nodes have edges pointing into changed nodes. Returns set of filenames."""
777
- dependent_files: set[str] = set()
778
-
779
- params = {"changed_ids": list(changed_node_ids)}
780
-
781
- for edge_type in _SYMBOL_TO_SYMBOL_EDGE_TYPES:
782
- query = f"""
783
- MATCH (src:Symbol)-[e:{edge_type}]->(dst:Symbol)
784
- WHERE dst.id IN $changed_ids
785
- RETURN DISTINCT src.filename
786
- """
787
- result = conn.execute(query, params)
788
- while result.has_next():
789
- row = result.get_next()
790
- filename = row[0]
791
- if filename: # Skip phantom nodes (filename = "")
792
- dependent_files.add(filename)
793
-
794
- return dependent_files
795
-
796
-
797
- def _find_annotation_dependents(conn: ladybug.Connection, changed_node_ids: set[str]) -> set[str]:
798
- """Find files that USE an annotation whose DEFINITION is among the changed nodes.
799
-
800
- Annotation usage is a node property (``annotations`` STRING[]), not a
801
- Symbol->Symbol edge, so `_find_dependents` — which walks edges — never pulls
802
- annotation users into scope. When an annotation definition changes (e.g.
803
- ``@interface Foo`` gains a meta-annotation that shifts the Layer-A chain in
804
- `resolve_role_and_capabilities`), every type carrying ``@Foo`` may need its
805
- ``role``/``capabilities`` recomputed or it goes stale until the next full
806
- rebuild. Return those users' files so the orchestrator treats them as
807
- dependents (re-parsed, role re-SET); the expansion cap bounds the scope.
808
-
809
- Scope is direct usage only: a user of an annotation that transitively
810
- composes the changed one (e.g. ``@A`` where ``@A`` is meta-annotated with the
811
- changed ``@B``) is NOT pulled in — that reverse-chain walk is left to a
812
- future hardening pass. The direct case covers the dominant real-world shape
813
- (a stereotype annotation applied directly to many types).
814
- """
815
- if not changed_node_ids:
816
- return set()
817
- # Changed annotation definitions → the simple names users reference them by.
818
- # Runs before `_delete_file_scope`, so the def nodes still exist.
819
- name_result = conn.execute(
820
- "MATCH (s:Symbol) WHERE s.id IN $ids AND s.kind = 'annotation' RETURN s.name",
821
- {"ids": list(changed_node_ids)},
822
- )
823
- names: list[str] = []
824
- while name_result.has_next():
825
- nm = name_result.get_next()[0]
826
- if nm:
827
- names.append(nm)
828
- if not names:
829
- return set()
830
- dependent_files: set[str] = set()
831
- for nm in names:
832
- user_result = conn.execute(
833
- "MATCH (s:Symbol) "
834
- "WHERE list_contains(s.annotations, $nm) AND s.filename <> '' "
835
- "RETURN DISTINCT s.filename",
836
- {"nm": nm},
837
- )
838
- while user_result.has_next():
839
- fn = user_result.get_next()[0]
840
- if fn:
841
- dependent_files.add(fn)
842
- return dependent_files
843
-
844
-
845
- def _delete_file_scope(
846
- conn: ladybug.Connection,
847
- changed_files: set[str],
848
- dependent_files: set[str],
849
- ) -> None:
850
- """Delete nodes and edges for a scope split into changed vs dependent files.
851
-
852
- ``changed_files`` are files whose content actually changed (added/modified/
853
- removed): their Symbol nodes are deleted (and re-created by ``_scoped_write``).
854
- ``dependent_files`` are files pulled in only to re-resolve their OUTGOING
855
- edges against the changed nodes; their node definitions did not change, so
856
- their nodes are deliberately PRESERVED (they re-MERGE in place on the same
857
- deterministic ``symbol_id``). Skipping phantom nodes (filename="").
858
-
859
- Why dependents are preserved (issue #305): the orchestrator computes
860
- dependents from the *changed* nodes only, so a dependent file's node can
861
- have an incoming CALLS edge from an out-of-scope caller. The ``source_file``
862
- on every Symbol->Symbol edge is the CALLER's file (pinned by
863
- ``test_source_file_value_matches_symbol_filename``), so Phase 1 below only
864
- deletes edges ORIGINATING in scope; incoming edges from out-of-scope callers
865
- survive. If we then tried to DELETE the dependent node, LadybugDB rejects it
866
- ("Node ... has connected edges in table CALLS in the bwd direction, ...
867
- Please delete the edges first or try DETACH DELETE") and the rebuild falls
868
- back to a full rebuild. A naive fix (DETACH DELETE on dependents, or an
869
- extra incoming-edge pass) would silence the crash but permanently drop those
870
- out-of-scope edges, corrupting the graph. Preserving dependent nodes keeps
871
- both the nodes and their incoming edges intact.
872
-
873
- Phase 1 deletes ALL edge types across the whole scope (changed + dependent)
874
- first to avoid LadybugDB "has connected edges" errors when edges from one
875
- file point to nodes in another file within the same scope. Route/Client/
876
- Producer nodes use DETACH DELETE as a safety net for any edges missed in
877
- Phase 1.
878
- """
879
- # Symbol DELETEs below maintain the FTS index, which needs the extension loaded on
880
- # this connection. Idempotent + best-effort (no-op when no index exists / FTS absent).
881
- try:
882
- conn.execute("LOAD EXTENSION FTS")
883
- except Exception:
884
- pass
885
- scope_files = changed_files | dependent_files
886
- scope_list = list(scope_files)
887
- changed_list = list(changed_files)
888
-
889
- # Phase 1: Delete ALL edges ORIGINATING from any scope file (changed +
890
- # dependent). Because `source_file` is the caller's file, this deletes edges
891
- # whose source is in scope (including dependents' outgoing edges to changed
892
- # nodes) while intentionally leaving incoming edges from out-of-scope callers
893
- # intact — those must survive so the dependent nodes below can be preserved.
894
- # This list is a superset of `_SYMBOL_TO_SYMBOL_EDGE_TYPES` (it also covers
895
- # Symbol->Route/Client/Producer/UCS and Client/Producer->Route edges); keep
896
- # both lists in sync with the schema.
897
- edge_tables = [
898
- "EXTENDS", "IMPLEMENTS", "INJECTS", "CALLS", "DECLARES", "OVERRIDES",
899
- "UNRESOLVED_AT", "EXPOSES", "DECLARES_CLIENT", "DECLARES_PRODUCER",
900
- "HTTP_CALLS", "ASYNC_CALLS",
901
- ]
902
- for edge_type in edge_tables:
903
- query = f"""
904
- MATCH (src)-[e:{edge_type}]->(dst)
905
- WHERE e.source_file IN $filenames
906
- DELETE e
907
- """
908
- conn.execute(query, {"filenames": scope_list})
909
-
910
- # Phase 2: Collect all Symbol node IDs for UnresolvedCallSite cleanup.
911
- symbol_ids: list[str] = []
912
- symbol_ids_query = """
913
- MATCH (s:Symbol)
914
- WHERE s.filename IN $filenames
915
- RETURN s.id
916
- """
917
- result = conn.execute(symbol_ids_query, {"filenames": scope_list})
918
- while result.has_next():
919
- row = result.get_next()
920
- symbol_ids.append(row[0])
921
-
922
- # Delete UnresolvedCallSite nodes whose caller_id is in the collected set.
923
- # These are children of scope symbols (including preserved dependents);
924
- # deleting them is safe because every scope file — dependents included — is
925
- # reprocessed and re-emits its UnresolvedCallSite nodes in `_scoped_write`.
926
- if symbol_ids:
927
- unresolved_query = """
928
- MATCH (u:UnresolvedCallSite)
929
- WHERE u.caller_id IN $symbol_ids
930
- DELETE u
931
- """
932
- conn.execute(unresolved_query, {"symbol_ids": symbol_ids})
933
-
934
- # Phase 3: Delete Symbol nodes ONLY for changed files (not dependents).
935
- # Dependent-file nodes are deliberately PRESERVED so their incoming edges
936
- # from out-of-scope callers survive; the dependents are re-MERGEd in place
937
- # by `_scoped_write` on the same deterministic node id. A changed node's
938
- # real incoming edges all come from dependent files (callers pulled into
939
- # scope by `_find_dependents`, which walks every type in
940
- # `_SYMBOL_TO_SYMBOL_EDGE_TYPES`), so Phase 1 already removed them and the
941
- # dependents re-emit them when reprocessed. DETACH DELETE is only a safety
942
- # net for the rare surviving edge whose source was NOT pulled into scope
943
- # (e.g. a phantom caller with filename="", which `_find_dependents` skips);
944
- # such an edge is stale once the node is recreated, so dropping it is fine.
945
- delete_symbols_query = """
946
- MATCH (s:Symbol)
947
- WHERE s.filename IN $filenames
948
- DETACH DELETE s
949
- """
950
- conn.execute(delete_symbols_query, {"filenames": changed_list})
951
-
952
- # Phase 4: Delete Route, Client, Producer nodes.
953
- # Use DETACH DELETE as a safety net in case any edges were missed in Phase 1.
954
- for label in ["Route", "Client", "Producer"]:
955
- conn.execute(
956
- f"MATCH (n:{label}) WHERE n.filename IN $filenames DETACH DELETE n",
957
- {"filenames": scope_list},
958
- )
959
-
960
-
961
- def _scoped_write(conn: ladybug.Connection, tables: GraphTables, *, project_root: Path, meta_chain: dict[str, frozenset[str]] | None) -> None:
962
- """Write nodes and edges to existing LadybugDB database without drop/create schema.
963
-
964
- Like write_ladybug() but without _drop_all()/_create_schema(). The caller is
965
- responsible for calling _populate_declares_rows() and _populate_overrides_rows()
966
- before invoking this function.
967
-
968
- Uses MERGE instead of CREATE to handle cases where nodes already exist.
969
- """
970
- t0 = time.time()
971
- _write_nodes_merge(
972
- conn,
973
- tables,
974
- project_root=project_root,
975
- meta_chain=meta_chain,
976
- )
977
- elapsed = time.time() - t0
978
- if elapsed > 0.1: # Only log if significant
979
- _verbose_stderr_line(f"[graph] scoped write · nodes written in {elapsed:.2f}s")
980
-
981
- t1 = time.time()
982
- _fbyid = _build_file_by_node_id(tables)
983
- _write_edges(conn, tables, _fbyid)
984
- elapsed = time.time() - t1
985
- if elapsed > 0.1:
986
- _verbose_stderr_line(f"[graph] scoped write · edges written in {elapsed:.2f}s")
987
-
988
- t2 = time.time()
989
- _write_routes_and_exposes(conn, tables, _fbyid)
990
- elapsed = time.time() - t2
991
- if elapsed > 0.1:
992
- _verbose_stderr_line(f"[graph] scoped write · routes/exposes written in {elapsed:.2f}s")
993
-
994
-
995
- def _write_nodes_merge(
996
- conn: ladybug.Connection,
997
- tables: GraphTables,
998
- *,
999
- project_root: Path,
1000
- meta_chain: dict[str, frozenset[str]] | None,
1001
- ) -> None:
1002
- """Write nodes to existing LadybugDB database using bulk COPY FROM."""
1003
- _write_nodes_impl(conn, tables, project_root=project_root, meta_chain=meta_chain)
1004
-
1005
-
1006
- # ---------- file walk (see `path_filtering.iter_source_files`) ----------
1007
-
1008
-
1009
- # ---------- pass 1 ----------
1010
-
1011
-
1012
- def _register_type(
1013
- tables: GraphTables,
1014
- decl: TypeDecl,
1015
- *,
1016
- file_path: str,
1017
- module: str,
1018
- microservice: str,
1019
- outer_fqn: str | None,
1020
- ) -> TypeIndexEntry:
1021
- package = decl.fqn.rsplit(".", 1)[0] if "." in decl.fqn and outer_fqn is None else (
1022
- outer_fqn.rsplit(".", 1)[0] if outer_fqn and "." in outer_fqn else ""
1023
- )
1024
- # top-level: package = fqn - name; nested: inherit from outer
1025
- if outer_fqn is None:
1026
- package = decl.fqn[: -(len(decl.name) + 1)] if decl.fqn.endswith("." + decl.name) else ""
1027
- else:
1028
- # walk outward to find a top-level fqn; package is everything before its simple name
1029
- top = outer_fqn
1030
- while top in tables.types and tables.types[top].outer_fqn:
1031
- top = tables.types[top].outer_fqn # type: ignore[assignment]
1032
- package = top[: top.rfind(".")] if "." in top else ""
1033
-
1034
- node_id = symbol_id(decl.kind, decl.fqn, file_path, decl.start_byte)
1035
- entry = TypeIndexEntry(
1036
- decl=decl,
1037
- file_path=file_path,
1038
- module=module,
1039
- microservice=microservice,
1040
- package=package,
1041
- outer_fqn=outer_fqn,
1042
- node_id=node_id,
1043
- )
1044
- # Warn on a same-FQN collision across DISTINCT files (the Kotlin+Java
1045
- # mixed-repo case, e.g. ``com.example.Foo`` in both ``Foo.java`` and
1046
- # ``Foo.kt``). Registration stays last-wins (silent before Task 16); this
1047
- # only surfaces the collision. Deduped per FQN per build so a pathological
1048
- # N-way collision warns once, not N-1 times. A same-file re-register
1049
- # (incremental re-parse) does not warn.
1050
- existing = tables.types.get(decl.fqn)
1051
- if (
1052
- existing is not None
1053
- and existing.file_path != file_path
1054
- and decl.fqn not in tables._warned_fqn_collisions
1055
- ):
1056
- tables._warned_fqn_collisions.add(decl.fqn)
1057
- log.warning(
1058
- "same-FQN type registered from two distinct files (last wins): "
1059
- "%s in %s and %s",
1060
- decl.fqn, existing.file_path, file_path,
1061
- )
1062
- tables.types[decl.fqn] = entry
1063
- tables.by_simple_name.setdefault(decl.name, []).append(entry)
1064
- tables.by_package.setdefault(package, []).append(entry)
1065
-
1066
- for m in decl.methods:
1067
- kind = "constructor" if m.is_constructor else "method"
1068
- mid = symbol_id(kind, f"{decl.fqn}#{m.signature}", file_path, m.start_byte)
1069
- tables.members.append(MemberEntry(
1070
- kind=kind, decl=m, parent_id=node_id, parent_fqn=decl.fqn,
1071
- file_path=file_path, module=module, microservice=microservice,
1072
- node_id=mid,
1073
- ))
1074
-
1075
- for nested in decl.nested:
1076
- _register_type(
1077
- tables, nested, file_path=file_path,
1078
- module=module, microservice=microservice, outer_fqn=decl.fqn,
1079
- )
1080
-
1081
- return entry
1082
-
1083
-
1084
- def pass1_parse(
1085
- root: Path,
1086
- tables: GraphTables,
1087
- *,
1088
- verbose: bool,
1089
- scope_files: set[str] | None = None,
1090
- removed_files: set[str] | None = None,
1091
- ) -> dict[str, JavaFileAst]:
1092
- """Walk files, parse them, populate node indexes. Returns path -> AST.
1093
-
1094
- Args:
1095
- root: Source root directory.
1096
- tables: GraphTables to populate.
1097
- verbose: Whether to emit progress output.
1098
- scope_files: Optional set of relative POSIX paths to parse. If None, parse all files.
1099
- removed_files: Optional set of relative POSIX paths that no longer exist
1100
- on disk (incremental deletions). These are members of ``scope_files``
1101
- (they were deleted, so they participate in scoped deletion) but are
1102
- never visited by the parse walk, so they must be excluded from the
1103
- pass-1 total to keep ``done`` from undercounting then two-way-clamping.
1104
- """
1105
- asts: dict[str, JavaFileAst] = {}
1106
- ignore = LayeredIgnore(root)
1107
- t0 = time.time()
1108
- n_files = 0
1109
- if verbose:
1110
- _verbose_stderr_line(_PASS1_START)
1111
- # Count-first: one filtered walk (no parsing) to set the EXACT total before
1112
- # the parse loop ticks. Single-layer ignore → the count is exact, so the
1113
- # rendered bar is determinate. For a scoped (incremental) parse the total is
1114
- # the number of files that will actually be visited: scope minus any removed
1115
- # files (which are members of scope for deletion but gone from disk, so the
1116
- # parse walk never ticks them); for a full rebuild it is the non-ignored
1117
- # .java count.
1118
- if verbose:
1119
- if scope_files is not None:
1120
- removed = removed_files if removed_files is not None else set()
1121
- pass1_total = len(scope_files - removed)
1122
- else:
1123
- pass1_total = sum(1 for _ in iter_source_files(root, ignore=ignore))
1124
- _emit_graph_progress(
1125
- {"pass": "1/6", "done": 0, "total": pass1_total, "status": "running"},
1126
- verbose=verbose,
1127
- )
1128
- slow_sec = 0.0
1129
- raw_slow = os.environ.get("JAVA_CODEBASE_RAG_TEST_GRAPH_SLOW_SEC", "").strip()
1130
- if raw_slow:
1131
- try:
1132
- slow_sec = float(raw_slow)
1133
- except ValueError:
1134
- slow_sec = 0.0
1135
- with _VerbosePassHeartbeats("[graph] pass 1", verbose=verbose):
1136
- if verbose and slow_sec > 0:
1137
- time.sleep(slow_sec)
1138
- # Per-file generated classification, carried through to the registration
1139
- # loop below (registration is deferred until after the multifile-facade
1140
- # merge so merged facades win their FQN slot in tables.types).
1141
- file_meta: dict[str, tuple[str, str, bool, str | None]] = {}
1142
- for p in iter_source_files(root, ignore=ignore):
1143
- # Skip files not in scope (if scope is provided)
1144
- try:
1145
- rel = p.resolve().relative_to(root.resolve()).as_posix()
1146
- except ValueError:
1147
- rel = p.as_posix()
1148
- if scope_files is not None and rel not in scope_files:
1149
- continue
1150
- n_files += 1
1151
- if verbose and (n_files % _PASS1_TICK_EVERY == 0):
1152
- _emit_graph_progress(
1153
- {"pass": "1/6", "done": n_files, "status": "running"},
1154
- verbose=verbose,
1155
- )
1156
- try:
1157
- content = p.read_bytes()
1158
- except OSError:
1159
- tables.skipped_files += 1
1160
- continue
1161
- if not content.strip():
1162
- continue
1163
- backend = backend_for(rel)
1164
- if backend is None:
1165
- continue
1166
- try:
1167
- ast = backend.parse(content, filename=rel, verbose=verbose)
1168
- except Exception:
1169
- tables.parse_errors += 1
1170
- continue
1171
- if ast.parse_error:
1172
- tables.parse_errors += 1
1173
- # Still index what tree-sitter gave us; robust to syntax errors.
1174
- module = module_for_path(str(p), root)
1175
- microservice = microservice_for_path(str(p), root)
1176
- asts[rel] = ast
1177
-
1178
- # Classify the file once (generated or not, and which tool generated it)
1179
- generated_config = load_generated_detection(str(root))
1180
- file_generated, file_generated_by = classify_java_file(
1181
- content, ast, config=generated_config, project_root=root
1182
- )
1183
- file_meta[rel] = (module, microservice, file_generated, file_generated_by)
1184
-
1185
- # file node
1186
- file_id = symbol_id("file", rel, rel, 0)
1187
- tables.files[rel] = file_id
1188
-
1189
- # package node (created lazily; nodes deduped by id)
1190
- if ast.package and ast.package not in tables.packages:
1191
- tables.packages[ast.package] = symbol_id("package", ast.package, "", 0)
1192
-
1193
- # Merge Kotlin ``@file:JvmMultifileClass`` facades BEFORE type
1194
- # registration (Task 9 / Task 11). Multiple ``.kt`` files sharing
1195
- # ``@file:JvmName("X")`` + ``@file:JvmMultifileClass()`` compile into ONE
1196
- # JVM class ``pkg.X``; the per-file parse emits one facade per file, all
1197
- # with FQN ``pkg.X``, so registering them per-file would collide in
1198
- # ``tables.types[fqn]`` (last-write-wins) and silently drop every other
1199
- # file's top-level functions from resolution. The merge concatenates the
1200
- # members onto ONE retained facade and strips the duplicates. It is a
1201
- # no-op on Java ASTs (no facades) and on single-file modules, so Java
1202
- # registration output is byte-identical (same files, same walk order).
1203
- if _merge_multifile_facades is not None and asts:
1204
- _merge_multifile_facades(list(asts.values()))
1205
-
1206
- # Register types AFTER the merge so a merged facade wins its FQN slot.
1207
- # Iteration order is the dict's insertion order = walk order, identical
1208
- # to the former per-file registration, so Java output is unchanged.
1209
- for rel, ast in asts.items():
1210
- module, microservice, file_generated, file_generated_by = file_meta[rel]
1211
- for t in ast.top_level_types:
1212
- _register_type(
1213
- tables, t, file_path=rel,
1214
- module=module, microservice=microservice, outer_fqn=None,
1215
- )
1216
-
1217
- # Seed generated/generated_by for all types in this file (including nested)
1218
- for t in ast.all_types:
1219
- if t.fqn in tables.types:
1220
- node_id = tables.types[t.fqn].node_id
1221
- tables.type_generated_by_node_id[node_id] = (file_generated, file_generated_by)
1222
-
1223
- if verbose:
1224
- elapsed = time.time() - t0
1225
- _emit_graph_progress(
1226
- {"pass": "1/6", "done": n_files, "status": "done", "elapsed_s": f"{elapsed:.2f}"},
1227
- verbose=verbose,
1228
- )
1229
- _verbose_stderr_line(
1230
- f"[graph] pass 1 · parsed {n_files} files in {elapsed:.2f}s: "
1231
- f"{len(tables.types)} types, {len(tables.members)} members, "
1232
- f"{tables.parse_errors} parse errors, {tables.skipped_files} skipped",
1233
- )
1234
- return asts
1235
-
1236
-
1237
- # ---------- pass 2: resolution + edges ----------
1238
-
1239
-
1240
- def _resolve_simple(
1241
- name: str,
1242
- *,
1243
- current: TypeIndexEntry,
1244
- ast: JavaFileAst,
1245
- tables: GraphTables,
1246
- ) -> TypeIndexEntry | None:
1247
- """Java-ish name resolution. Returns a known TypeIndexEntry or None (phantom)."""
1248
- # Strip trailing generics the caller may have left in, defensively.
1249
- bare = name.split("<", 1)[0].strip()
1250
- if not bare:
1251
- return None
1252
-
1253
- # 0. Nested inside the same top-level hierarchy — try `Outer.Bare` fqn.
1254
- outer = current.outer_fqn
1255
- while outer is not None and outer in tables.types:
1256
- candidate = f"{outer}.{bare}"
1257
- if candidate in tables.types:
1258
- return tables.types[candidate]
1259
- outer = tables.types[outer].outer_fqn
1260
-
1261
- # 1. Same-file siblings (same outer as `current`).
1262
- same_outer = current.outer_fqn or current.package
1263
- for e in tables.by_simple_name.get(bare, ()):
1264
- e_parent = e.outer_fqn or e.package
1265
- if e.file_path == current.file_path and e_parent == same_outer:
1266
- return e
1267
-
1268
- # 2. Explicit import.
1269
- if bare in ast.explicit_imports:
1270
- fq = ast.explicit_imports[bare]
1271
- if fq in tables.types:
1272
- return tables.types[fq]
1273
- # Known FQN (outside our codebase) → unresolved; caller will phantom-ise.
1274
- return None
1275
-
1276
- # 3. Same package.
1277
- if current.package:
1278
- candidate = f"{current.package}.{bare}"
1279
- if candidate in tables.types:
1280
- return tables.types[candidate]
1281
-
1282
- # 4. Wildcard imports.
1283
- for wild in ast.wildcard_imports:
1284
- candidate = f"{wild}.{bare}"
1285
- if candidate in tables.types:
1286
- return tables.types[candidate]
1287
-
1288
- # 5. java.lang best-effort (unresolved but deterministic phantom).
1289
- return None
1290
-
1291
-
1292
- def _phantom_target(
1293
- tables: GraphTables,
1294
- simple: str,
1295
- ast: JavaFileAst,
1296
- *,
1297
- current: TypeIndexEntry,
1298
- ) -> tuple[str, str, str]:
1299
- """Produce (id, simple, fqn-or-best-guess) for an unresolved type reference.
1300
-
1301
- The fqn falls back through: explicit import → wildcard → java.lang → bare name.
1302
- """
1303
- bare = simple.split("<", 1)[0].strip()
1304
- guess_fqn = bare
1305
- if bare in ast.explicit_imports:
1306
- guess_fqn = ast.explicit_imports[bare]
1307
- elif bare in _JAVA_LANG_SIMPLE:
1308
- guess_fqn = f"java.lang.{bare}"
1309
- elif ast.language == "kotlin" and bare in _KOTLIN_DEFAULT_SIMPLE:
1310
- # Kotlin default-import stdlib type (kotlin.collections.List etc.) —
1311
- # Kotlin-gated so Java resolution is byte-identical. See Task 13.
1312
- guess_fqn = _KOTLIN_DEFAULT_SIMPLE[bare]
1313
- elif ast.wildcard_imports:
1314
- # Pick first wildcard as a hint (imperfect but useful for display).
1315
- guess_fqn = f"{ast.wildcard_imports[0]}.{bare}"
1316
-
1317
- pid = phantom_id(guess_fqn)
1318
- if pid not in tables.phantoms:
1319
- tables.phantoms[pid] = {
1320
- "id": pid,
1321
- "kind": "class",
1322
- "name": bare,
1323
- "fqn": guess_fqn,
1324
- "package": guess_fqn.rsplit(".", 1)[0] if "." in guess_fqn else "",
1325
- "module": "",
1326
- "microservice": "",
1327
- "filename": "",
1328
- "start_line": 0,
1329
- "end_line": 0,
1330
- "start_byte": 0,
1331
- "end_byte": 0,
1332
- "modifiers": [],
1333
- "annotations": [],
1334
- "capabilities": [],
1335
- "role": "OTHER",
1336
- "signature": "",
1337
- "parent_id": "",
1338
- "resolved": False,
1339
- }
1340
- return pid, bare, guess_fqn
1341
-
1342
-
1343
- def _edge_for(
1344
- *,
1345
- src: TypeIndexEntry,
1346
- target_simple: str,
1347
- ast: JavaFileAst,
1348
- tables: GraphTables,
1349
- ) -> tuple[str, str, str, bool]:
1350
- resolved = _resolve_simple(target_simple, current=src, ast=ast, tables=tables)
1351
- if resolved is not None:
1352
- return resolved.node_id, resolved.decl.name, resolved.decl.fqn, True
1353
- pid, simple, fqn = _phantom_target(tables, target_simple, ast, current=src)
1354
- return pid, simple, fqn, False
1355
-
1356
-
1357
- def _emit_extends_implements(
1358
- entry: TypeIndexEntry,
1359
- ast: JavaFileAst,
1360
- tables: GraphTables,
1361
- *,
1362
- seen_ext: set[tuple[str, str]],
1363
- seen_impl: set[tuple[str, str]],
1364
- ) -> None:
1365
- for name in entry.decl.extends:
1366
- dst_id, dst_simple, dst_fqn, ok = _edge_for(
1367
- src=entry, target_simple=name, ast=ast, tables=tables,
1368
- )
1369
- key = (entry.node_id, dst_id)
1370
- if key in seen_ext:
1371
- continue
1372
- seen_ext.add(key)
1373
- tables.extends_rows.append(EdgeRow(
1374
- src_id=entry.node_id, dst_id=dst_id,
1375
- dst_name=dst_simple, dst_fqn=dst_fqn, resolved=ok,
1376
- ))
1377
-
1378
- for name in entry.decl.implements:
1379
- dst_id, dst_simple, dst_fqn, ok = _edge_for(
1380
- src=entry, target_simple=name, ast=ast, tables=tables,
1381
- )
1382
- key = (entry.node_id, dst_id)
1383
- if key in seen_impl:
1384
- continue
1385
- seen_impl.add(key)
1386
- tables.implements_rows.append(EdgeRow(
1387
- src_id=entry.node_id, dst_id=dst_id,
1388
- dst_name=dst_simple, dst_fqn=dst_fqn, resolved=ok,
1389
- ))
1390
-
1391
-
1392
- def _emit_injects(
1393
- entry: TypeIndexEntry,
1394
- ast: JavaFileAst,
1395
- tables: GraphTables,
1396
- *,
1397
- seen: set[tuple[str, str, str, str]],
1398
- ) -> None:
1399
- if entry.decl.kind == "interface":
1400
- return
1401
-
1402
- ann_names = [a.name for a in entry.decl.annotations]
1403
- inject_set = injection_annotation_names()
1404
- lombok_rac = lombok_required_args_annotations()
1405
- has_lombok_rac = any(a in lombok_rac for a in ann_names)
1406
-
1407
- def _add(
1408
- target: str, mechanism: str, annotation: str, slot: str,
1409
- ) -> None:
1410
- dst_id, dst_simple, dst_fqn, ok = _edge_for(
1411
- src=entry, target_simple=target, ast=ast, tables=tables,
1412
- )
1413
- key = (entry.node_id, dst_id, mechanism, slot)
1414
- if key in seen:
1415
- return
1416
- seen.add(key)
1417
- tables.injects_rows.append(InjectsRow(
1418
- src_id=entry.node_id, dst_id=dst_id,
1419
- dst_name=dst_simple, dst_fqn=dst_fqn, resolved=ok,
1420
- mechanism=mechanism, annotation=annotation, field_or_param=slot,
1421
- ))
1422
-
1423
- # Field injection: @Autowired / @Inject / @Resource.
1424
- for f in entry.decl.fields:
1425
- annotated = next((a.name for a in f.annotations if a.name in inject_set), None)
1426
- if annotated:
1427
- _add(f.type_name, "field", annotated, f.name)
1428
-
1429
- # Lombok: @RequiredArgsConstructor -> each `final` non-static field becomes an injection;
1430
- # @AllArgsConstructor -> every non-static field.
1431
- if has_lombok_rac:
1432
- all_args = "AllArgsConstructor" in ann_names
1433
- for f in entry.decl.fields:
1434
- if "static" in f.modifiers:
1435
- continue
1436
- if not all_args and "final" not in f.modifiers:
1437
- continue
1438
- _add(f.type_name, "lombok_required_args",
1439
- "AllArgsConstructor" if all_args else "RequiredArgsConstructor",
1440
- f.name)
1441
-
1442
- # Constructor injection:
1443
- ctors = [m for m in entry.decl.methods if m.is_constructor]
1444
- if ctors:
1445
- chosen = None
1446
- autowired = [c for c in ctors if any(a.name == "Autowired" for a in c.annotations)]
1447
- if autowired:
1448
- chosen = autowired[0]
1449
- elif len(ctors) == 1 and ctors[0].parameters:
1450
- chosen = ctors[0]
1451
- if chosen is not None:
1452
- annotation = "Autowired" if any(a.name == "Autowired" for a in chosen.annotations) else ""
1453
- for p in chosen.parameters:
1454
- _add(p.type_name, "constructor", annotation, p.name)
1455
-
1456
- # Setter injection: setXxx annotated @Autowired with 1 parameter.
1457
- for m in entry.decl.methods:
1458
- if m.is_constructor or not m.name.startswith("set") or len(m.parameters) != 1:
1459
- continue
1460
- if any(a.name == "Autowired" for a in m.annotations):
1461
- _add(m.parameters[0].type_name, "setter", "Autowired",
1462
- m.parameters[0].name)
1463
-
1464
-
1465
- def pass2_edges(tables: GraphTables, asts: dict[str, JavaFileAst], *, verbose: bool) -> None:
1466
- t0 = time.time()
1467
- seen_ext: set[tuple[str, str]] = set()
1468
- seen_impl: set[tuple[str, str]] = set()
1469
- seen_inj: set[tuple[str, str, str, str]] = set()
1470
- if verbose:
1471
- _verbose_stderr_line(_PASS2_START)
1472
- with _graph_pass_progress("2/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 2", verbose=verbose):
1473
- for fqn, entry in tables.types.items():
1474
- ast = asts.get(entry.file_path)
1475
- if ast is None:
1476
- continue
1477
- _emit_extends_implements(entry, ast, tables, seen_ext=seen_ext, seen_impl=seen_impl)
1478
- _emit_injects(entry, ast, tables, seen=seen_inj)
1479
- if verbose:
1480
- elapsed = time.time() - t0
1481
- _verbose_stderr_line(
1482
- f"[graph] pass 2 · emitted {len(tables.extends_rows)} EXTENDS, "
1483
- f"{len(tables.implements_rows)} IMPLEMENTS, "
1484
- f"{len(tables.injects_rows)} INJECTS, "
1485
- f"{len(tables.phantoms)} phantoms in {elapsed:.2f}s",
1486
- )
1487
-
1488
-
1489
- # ---------- pass 3: call graph ----------
1490
-
1491
-
1492
- def _build_member_indexes(tables: GraphTables) -> None:
1493
- tables.methods_by_type = {}
1494
- for m in tables.members:
1495
- tables.methods_by_type.setdefault(m.parent_fqn, []).append(m)
1496
-
1497
-
1498
- def _direct_supertype_fqns(entry: TypeIndexEntry, tables: GraphTables) -> list[str]:
1499
- out: list[str] = []
1500
- for r in tables.extends_rows:
1501
- if r.src_id == entry.node_id and r.dst_fqn in tables.types:
1502
- out.append(r.dst_fqn)
1503
- for r in tables.implements_rows:
1504
- if r.src_id == entry.node_id and r.dst_fqn in tables.types:
1505
- out.append(r.dst_fqn)
1506
- return out
1507
-
1508
-
1509
- def _first_supertype_fqn(tables: GraphTables, type_fqn: str) -> str | None:
1510
- entry = tables.types.get(type_fqn)
1511
- if entry is None:
1512
- return None
1513
- for r in tables.extends_rows:
1514
- if r.src_id == entry.node_id and r.dst_fqn in tables.types:
1515
- return r.dst_fqn
1516
- for r in tables.implements_rows:
1517
- if r.src_id == entry.node_id and r.dst_fqn in tables.types:
1518
- return r.dst_fqn
1519
- return None
1520
-
1521
-
1522
- def _is_chained_receiver_text(receiver_expr: str) -> bool:
1523
- """Heuristic: call chain or complex expr (contains a completed call)."""
1524
- s = receiver_expr.strip()
1525
- return "(" in s and ")" in s
1526
-
1527
-
1528
- def _resolve_this_super_field_chain(
1529
- expr: str,
1530
- *,
1531
- member: MemberEntry,
1532
- ast: JavaFileAst,
1533
- tables: GraphTables,
1534
- ) -> str | None:
1535
- """Resolve `this.a.b` / `super.a` (no calls) to the final field's type FQN."""
1536
- s = expr.strip()
1537
- if "(" in s or ")" in s or "." not in s:
1538
- return None
1539
- entry = tables.types.get(member.parent_fqn)
1540
- if entry is None:
1541
- return None
1542
- parts = s.split(".")
1543
- if len(parts) < 2:
1544
- return None
1545
- if parts[0] == "this":
1546
- cur = entry
1547
- elif parts[0] == "super":
1548
- sup = _first_supertype_fqn(tables, member.parent_fqn)
1549
- if sup is None or sup not in tables.types:
1550
- return None
1551
- cur = tables.types[sup]
1552
- else:
1553
- return None
1554
- for fname in parts[1:]:
1555
- fld = next((f for f in cur.decl.fields if f.name == fname), None)
1556
- if fld is None:
1557
- return None
1558
- resolved = _resolve_simple(fld.type_name, current=cur, ast=ast, tables=tables)
1559
- if resolved is None:
1560
- return None
1561
- cur = resolved
1562
- return cur.decl.fqn
1563
-
1564
-
1565
- def _scope_table(member: MemberEntry, ast: JavaFileAst, tables: GraphTables) -> dict[str, str]:
1566
- """Map simple variable/field/param name -> resolved declaring type FQN."""
1567
- scope: dict[str, str] = {}
1568
- entry = tables.types.get(member.parent_fqn)
1569
- if entry is None:
1570
- return scope
1571
-
1572
- def add_fields(tentry: TypeIndexEntry) -> None:
1573
- for f in tentry.decl.fields:
1574
- resolved = _resolve_simple(f.type_name, current=tentry, ast=ast, tables=tables)
1575
- if resolved is not None:
1576
- scope[f.name] = resolved.decl.fqn
1577
-
1578
- add_fields(entry)
1579
- seen: set[str] = {member.parent_fqn}
1580
- queue = list(_direct_supertype_fqns(entry, tables))
1581
- while queue:
1582
- sup = queue.pop()
1583
- if sup in seen or sup not in tables.types:
1584
- continue
1585
- seen.add(sup)
1586
- te = tables.types[sup]
1587
- for f in te.decl.fields:
1588
- if f.name not in scope:
1589
- resolved = _resolve_simple(f.type_name, current=te, ast=ast, tables=tables)
1590
- if resolved is not None:
1591
- scope[f.name] = resolved.decl.fqn
1592
- queue.extend(_direct_supertype_fqns(te, tables))
1593
-
1594
- for p in member.decl.parameters:
1595
- resolved = _resolve_simple(p.type_name, current=entry, ast=ast, tables=tables)
1596
- if resolved is not None:
1597
- scope[p.name] = resolved.decl.fqn
1598
-
1599
- # Locals shadow fields and parameters (same simple name → local wins).
1600
- for name, t_simple in member.decl.local_vars:
1601
- resolved = _resolve_simple(t_simple, current=entry, ast=ast, tables=tables)
1602
- if resolved is not None:
1603
- scope[name] = resolved.decl.fqn
1604
-
1605
- return scope
1606
-
1607
-
1608
- def _lookup_method_candidates(
1609
- type_fqn: str,
1610
- callee_simple: str,
1611
- arg_count: int,
1612
- tables: GraphTables,
1613
- ast: JavaFileAst,
1614
- *,
1615
- visited: set[str] | None = None,
1616
- ) -> tuple[list[MemberEntry], bool]:
1617
- """Return (candidates, used_name_only_fallback). Walks type + supertypes.
1618
-
1619
- When ``used_name_only_fallback`` is true and ``len(candidates) == 1``, the
1620
- caller may reuse the receiver-resolution strategy (see ``_resolve_and_emit_call``)
1621
- instead of tagging ``overload_ambiguous``.
1622
- """
1623
- if visited is None:
1624
- visited = set()
1625
- exact: list[MemberEntry] = []
1626
- name_only: list[MemberEntry] = []
1627
-
1628
- def collect_on_type(tfqn: str) -> None:
1629
- nonlocal exact, name_only
1630
- for m in tables.methods_by_type.get(tfqn, ()):
1631
- if callee_simple == "<init>":
1632
- if not m.decl.is_constructor:
1633
- continue
1634
- np = len(m.decl.parameters)
1635
- if arg_count < 0:
1636
- name_only.append(m)
1637
- elif np == arg_count:
1638
- exact.append(m)
1639
- else:
1640
- name_only.append(m)
1641
- continue
1642
- if m.decl.is_constructor:
1643
- continue
1644
- if m.decl.name != callee_simple:
1645
- continue
1646
- np = len(m.decl.parameters)
1647
- if arg_count < 0:
1648
- name_only.append(m)
1649
- elif np == arg_count:
1650
- exact.append(m)
1651
- else:
1652
- name_only.append(m)
1653
-
1654
- queue = [type_fqn]
1655
- while queue:
1656
- tfqn = queue.pop(0)
1657
- if tfqn in visited or tfqn not in tables.types:
1658
- continue
1659
- visited.add(tfqn)
1660
- collect_on_type(tfqn)
1661
- te = tables.types[tfqn]
1662
- for sup in _direct_supertype_fqns(te, tables):
1663
- if sup not in visited:
1664
- queue.append(sup)
1665
- # Synthetic anonymous classes (`….<anon:byte>`): unqualified instance calls
1666
- # may target the lexically enclosing type (D3), e.g. `pingFromAnon()` from
1667
- # `NestedCalls` inside `new Runnable() { void run() { … } }`.
1668
- if ".<anon:" in tfqn and te.outer_fqn and te.outer_fqn not in visited:
1669
- queue.append(te.outer_fqn)
1670
-
1671
- if exact:
1672
- return exact, False
1673
- if name_only:
1674
- return name_only, True
1675
- return [], False
1676
-
1677
-
1678
- def _static_wildcard_resolve(
1679
- callee_simple: str,
1680
- ast: JavaFileAst,
1681
- tables: GraphTables,
1682
- current: TypeIndexEntry,
1683
- ) -> str | None:
1684
- for tw in ast.file_imports.static_wildcards:
1685
- if tw not in tables.types:
1686
- continue
1687
- for m in tables.methods_by_type.get(tw, ()):
1688
- if m.decl.name != callee_simple or m.decl.is_constructor:
1689
- continue
1690
- if "static" not in m.decl.modifiers:
1691
- continue
1692
- return tw
1693
- return None
1694
-
1695
-
1696
- def _unique_type_simple_resolve(simple: str, tables: GraphTables) -> str | None:
1697
- """Return the type FQN iff exactly one indexed type uses `simple` as `decl.name`.
1698
-
1699
- Used only for receiver / static-qualifier disambiguation. Do not use the
1700
- method index here: an unresolved identifier that equals some method's
1701
- simple name elsewhere in the project is not evidence about the receiver type.
1702
- """
1703
- hits = tables.by_simple_name.get(simple, [])
1704
- if len(hits) != 1:
1705
- return None
1706
- return hits[0].decl.fqn
1707
-
1708
-
1709
- def _suffix_resolve(receiver_simple: str, tables: GraphTables) -> str | None:
1710
- matches = [fq for fq in tables.types if fq.endswith("." + receiver_simple)]
1711
- if len(matches) != 1:
1712
- return None
1713
- return matches[0]
1714
-
1715
-
1716
- def _resolve_receiver_type(
1717
- call: CallSite,
1718
- *,
1719
- scope: dict[str, str],
1720
- member: MemberEntry,
1721
- ast: JavaFileAst,
1722
- tables: GraphTables,
1723
- ) -> tuple[str | None, str, float]:
1724
- """Returns (receiver_type_fqn_or_none, strategy, confidence)."""
1725
- expr = call.receiver_expr.strip()
1726
- callee = call.callee_simple
1727
-
1728
- effective_static = call.is_static_call
1729
- if call.is_static_call and expr and not _is_chained_receiver_text(expr):
1730
- bare_for_static = expr.split("<", 1)[0].strip()
1731
- if bare_for_static and "." not in bare_for_static and bare_for_static in scope:
1732
- effective_static = False
1733
-
1734
- if not expr and not call.is_static_call:
1735
- if callee in ast.file_imports.static_methods:
1736
- full = ast.file_imports.static_methods[callee]
1737
- if "." in full:
1738
- type_fqn = full.rsplit(".", 1)[0]
1739
- return type_fqn, "static_import", 0.95
1740
- sw = _static_wildcard_resolve(callee, ast, tables, tables.types[member.parent_fqn])
1741
- if sw is not None:
1742
- return sw, "static_import_wildcard", 0.85
1743
-
1744
- if effective_static and expr:
1745
- if _is_chained_receiver_text(expr):
1746
- return None, "chained_receiver", 0.0
1747
- entry = tables.types.get(member.parent_fqn)
1748
- if entry is None:
1749
- return None, "chained_receiver", 0.0
1750
- bare_static = expr.split("<", 1)[0].strip()
1751
- resolved = _resolve_simple(bare_static, current=entry, ast=ast, tables=tables)
1752
- if resolved is not None:
1753
- return resolved.decl.fqn, "import_map", 0.95
1754
- # External type not in the index but FQN is deterministic via an explicit import.
1755
- # e.g. `import java.util.Objects; Objects.requireNonNull(x)` — we know the FQN
1756
- # is "java.util.Objects" even though the type isn't indexed; return it so the
1757
- # edge carries the correct receiver-tier confidence rather than collapsing to phantom.
1758
- if bare_static in ast.explicit_imports:
1759
- return ast.explicit_imports[bare_static], "import_map", 0.95
1760
- uq = _unique_type_simple_resolve(expr, tables)
1761
- if uq is not None:
1762
- return uq, "unique_type_name", 0.75
1763
- sf = _suffix_resolve(expr, tables)
1764
- if sf is not None:
1765
- return sf, "suffix", 0.55
1766
- return None, "phantom", 0.0
1767
-
1768
- if expr in ("", "this"):
1769
- return member.parent_fqn, "this_super", 0.95
1770
-
1771
- if expr == "super":
1772
- sup = _first_supertype_fqn(tables, member.parent_fqn)
1773
- if sup is not None:
1774
- return sup, "this_super", 0.95
1775
- # No indexed supertype — implicit super to java.lang.Object.
1776
- # Keep strategy='implicit_super' and confidence=0.90 so this path is
1777
- # distinguishable from a genuinely unresolvable receiver.
1778
- return "java.lang.Object", "implicit_super", 0.90
1779
-
1780
- if _is_chained_receiver_text(expr):
1781
- return None, "chained_receiver", 0.0
1782
-
1783
- entry = tables.types.get(member.parent_fqn)
1784
- if entry is None:
1785
- return None, "phantom", 0.0
1786
-
1787
- bare = expr.split("<", 1)[0].strip()
1788
- if bare in scope:
1789
- return scope[bare], "import_map", 0.95
1790
-
1791
- chain = _resolve_this_super_field_chain(expr, member=member, ast=ast, tables=tables)
1792
- if chain is not None:
1793
- return chain, "import_map", 0.95
1794
-
1795
- resolved = _resolve_simple(bare, current=entry, ast=ast, tables=tables)
1796
- if resolved is not None:
1797
- return resolved.decl.fqn, "import_map", 0.95
1798
-
1799
- if entry.package:
1800
- cand = f"{entry.package}.{bare}"
1801
- if cand in tables.types:
1802
- return cand, "same_module", 0.90
1803
-
1804
- uq = _unique_type_simple_resolve(bare, tables)
1805
- if uq is not None:
1806
- return uq, "unique_type_name", 0.75
1807
-
1808
- sf = _suffix_resolve(bare, tables)
1809
- if sf is not None:
1810
- return sf, "suffix", 0.55
1811
-
1812
- return None, "phantom", 0.0
1813
-
1814
-
1815
- def _phantom_method_id(
1816
- tables: GraphTables,
1817
- *,
1818
- receiver_fqn: str | None,
1819
- receiver_expr: str,
1820
- callee: str,
1821
- arg_count: int,
1822
- ) -> str:
1823
- # Phantom node identity for a resolved receiver omits call-site arity so
1824
- # method references (arg_count=-1) and normal invocations share one Symbol
1825
- # per (receiver_fqn, callee) when the callee is not indexed (D1).
1826
- if receiver_fqn:
1827
- fqn = f"{receiver_fqn}#{callee}(?)"
1828
- sig = f"{callee}(?)"
1829
- else:
1830
- expr_short = (receiver_expr[:50] if receiver_expr else "?")
1831
- arity = "(?)" if arg_count < 0 else f"({arg_count})"
1832
- fqn = f"?{expr_short}#{callee}{arity}"
1833
- sig = f"{callee}{arity}"
1834
- pid = phantom_id(fqn)
1835
- if pid not in tables.phantoms:
1836
- tables.phantoms[pid] = {
1837
- "id": pid,
1838
- "kind": "method",
1839
- "name": callee,
1840
- "fqn": fqn,
1841
- "package": "",
1842
- "module": "",
1843
- "microservice": "",
1844
- "filename": "",
1845
- "start_line": 0,
1846
- "end_line": 0,
1847
- "start_byte": 0,
1848
- "end_byte": 0,
1849
- "modifiers": [],
1850
- "annotations": [],
1851
- "capabilities": [],
1852
- "role": "OTHER",
1853
- "signature": sig,
1854
- "parent_id": "",
1855
- "resolved": False,
1856
- }
1857
- return pid
1858
-
1859
-
1860
- def _method_signature_matches_call(member: MemberEntry, call: CallSite) -> bool:
1861
- if call.arg_count < 0:
1862
- return True
1863
- return len(member.decl.parameters) == call.arg_count
1864
-
1865
-
1866
- def _is_strict_supertype_of(tables: GraphTables, super_fqn: str, subtype_fqn: str) -> bool:
1867
- if super_fqn == subtype_fqn:
1868
- return False
1869
- entry = tables.types.get(subtype_fqn)
1870
- if entry is None:
1871
- return False
1872
- visited: set[str] = set()
1873
- queue = list(_direct_supertype_fqns(entry, tables))
1874
- while queue:
1875
- tfqn = queue.pop(0)
1876
- if tfqn == super_fqn:
1877
- return True
1878
- if tfqn in visited or tfqn not in tables.types:
1879
- continue
1880
- visited.add(tfqn)
1881
- queue.extend(_direct_supertype_fqns(tables.types[tfqn], tables))
1882
- return False
1883
-
1884
-
1885
- def _callee_declaring_role_at_write(
1886
- tables: GraphTables,
1887
- dst_id: str,
1888
- *,
1889
- member_by_id: dict[str, MemberEntry],
1890
- ) -> str:
1891
- """Match parent declaring-type Symbol.role (brownfield + meta_chain included)."""
1892
- if dst_id in tables.phantoms:
1893
- return "OTHER"
1894
- member = member_by_id.get(dst_id)
1895
- if member is None:
1896
- return "OTHER"
1897
- return tables.type_role_by_node_id.get(member.parent_id, "OTHER")
1898
-
1899
-
1900
- def _collapse_supertype_duplicates(
1901
- candidates: list[MemberEntry],
1902
- recv_type_fqn: str,
1903
- call: CallSite,
1904
- tables: GraphTables,
1905
- ) -> list[MemberEntry]:
1906
- """§3.3.1 supertype-walk dedup — collapse interface + concrete duplicate sites."""
1907
- if len(candidates) <= 1:
1908
- return candidates
1909
- concrete_on_receiver = [
1910
- c for c in candidates
1911
- if c.parent_fqn == recv_type_fqn and _method_signature_matches_call(c, call)
1912
- ]
1913
- if len(concrete_on_receiver) != 1:
1914
- return candidates
1915
- concrete = concrete_on_receiver[0]
1916
- supertypes = [
1917
- c for c in candidates
1918
- if c is not concrete
1919
- and _is_strict_supertype_of(tables, c.parent_fqn, recv_type_fqn)
1920
- and c.decl.signature == concrete.decl.signature
1921
- ]
1922
- if not supertypes:
1923
- return candidates
1924
- allowed_ids = {concrete.node_id, *(c.node_id for c in supertypes)}
1925
- if any(c.node_id not in allowed_ids for c in candidates):
1926
- return candidates
1927
- log.debug(
1928
- "pass3 supertype dedup %s -> %s",
1929
- [c.node_id for c in candidates],
1930
- concrete.node_id,
1931
- )
1932
- return [concrete]
1933
-
1934
-
1935
- def _unresolved_call_site_id(caller_id: str, call: CallSite) -> str:
1936
- return f"ucs:{caller_id}:{call.line}:{call.byte}"
1937
-
1938
-
1939
- def _emit_unresolved_call_site(
1940
- tables: GraphTables,
1941
- stats: CallResolutionStats,
1942
- *,
1943
- caller_id: str,
1944
- call: CallSite,
1945
- reason: str,
1946
- ) -> None:
1947
- tables.unresolved_call_site_rows.append(UnresolvedCallSiteRow(
1948
- id=_unresolved_call_site_id(caller_id, call),
1949
- caller_id=caller_id,
1950
- call_site_line=call.line,
1951
- call_site_byte=call.byte,
1952
- arg_count=call.arg_count,
1953
- callee_simple=call.callee_simple,
1954
- receiver_expr=call.receiver_expr or "",
1955
- reason=reason,
1956
- ))
1957
- if reason == "chained_receiver":
1958
- stats.phantom_chained += 1
1959
- else:
1960
- stats.phantom_other += 1
1961
-
1962
-
1963
- def _emit_call_edge(
1964
- tables: GraphTables,
1965
- stats: CallResolutionStats,
1966
- *,
1967
- src_id: str,
1968
- dst_id: str,
1969
- call: CallSite,
1970
- confidence: float,
1971
- strategy: str,
1972
- resolved: bool,
1973
- edge_arg_count: int | None = None,
1974
- ) -> None:
1975
- arity = call.arg_count if edge_arg_count is None else edge_arg_count
1976
- tables.calls_rows.append(CallsRow(
1977
- src_id=src_id,
1978
- dst_id=dst_id,
1979
- call_site_line=call.line,
1980
- call_site_byte=call.byte,
1981
- arg_count=arity,
1982
- confidence=confidence,
1983
- strategy=strategy,
1984
- source="static",
1985
- resolved=resolved,
1986
- ))
1987
- stats.total += 1
1988
- stats.by_strategy[strategy] += 1
1989
- if not resolved:
1990
- stats.callee_unresolved += 1
1991
-
1992
-
1993
- def _resolve_and_emit_call(
1994
- call: CallSite,
1995
- member: MemberEntry,
1996
- ast: JavaFileAst,
1997
- tables: GraphTables,
1998
- stats: CallResolutionStats,
1999
- *,
2000
- scope: dict[str, str],
2001
- ) -> None:
2002
- """Emit CALLS rows for one call site.
2003
-
2004
- Candidate selection uses ``_lookup_method_candidates`` (exact arity first, then
2005
- name-only fallback on the type + supertype walk).
2006
-
2007
- When ``used_name_only_fallback`` is true and exactly one name-only candidate
2008
- exists, the edge ``strategy`` reuses the receiver-resolution tier (``strat``)
2009
- rather than ``overload_ambiguous``: arity at the call site did not match any
2010
- overload, but only one method of that name exists — the callee is unambiguous.
2011
- """
2012
- recv_type, strat, conf = _resolve_receiver_type(call, scope=scope, member=member, ast=ast, tables=tables)
2013
-
2014
- if strat == "chained_receiver":
2015
- _emit_unresolved_call_site(
2016
- tables, stats, caller_id=member.node_id, call=call, reason="chained_receiver",
2017
- )
2018
- return
2019
-
2020
- if recv_type is None:
2021
- _emit_unresolved_call_site(
2022
- tables, stats,
2023
- caller_id=member.node_id,
2024
- call=call,
2025
- reason="phantom_unresolved_receiver",
2026
- )
2027
- return
2028
-
2029
- candidates, name_only_fb = _lookup_method_candidates(
2030
- recv_type, call.callee_simple, call.arg_count, tables, ast,
2031
- )
2032
-
2033
- # Guard relies on `_lookup_method_candidates` returning a same-ms candidate when one exists; revisit if pass3 scopes lookups per-microservice.
2034
- if member.microservice:
2035
- same_ms = [c for c in candidates if c.microservice == member.microservice]
2036
- if same_ms and len(same_ms) != len(candidates):
2037
- for c in candidates:
2038
- if c.microservice and c.microservice != member.microservice:
2039
- log.warning(
2040
- "skipping cross-microservice CALLS edge %s -> %s "
2041
- "(caller=%s, callee=%s)",
2042
- f"{member.parent_fqn}#{member.decl.signature}",
2043
- f"{c.parent_fqn}#{c.decl.signature}",
2044
- member.microservice, c.microservice,
2045
- )
2046
- stats.skipped_cross_service += 1
2047
- candidates = same_ms
2048
-
2049
- # Compute the call-shape strategy / confidence override BEFORE the
2050
- # empty-candidates check so they are preserved even when the callee cannot
2051
- # be located on the resolved receiver type (B3 fix).
2052
- edge_conf = conf
2053
- if call.arg_count < 0:
2054
- edge_strat = "method_reference"
2055
- elif call.callee_simple == "<init>" and call.receiver_expr == "super" and (
2056
- call.byte == member.decl.start_byte and call.line == member.decl.start_line
2057
- ):
2058
- # Synthesized implicit-super site from _parse_method.
2059
- edge_strat = "implicit_super"
2060
- edge_conf = 0.90
2061
- elif call.callee_simple == "<init>":
2062
- # new Foo(…), this(…), super(…) — confidence inherited from receiver tier.
2063
- edge_strat = "constructor"
2064
- elif name_only_fb and len(candidates) > 1:
2065
- edge_strat = "overload_ambiguous"
2066
- elif name_only_fb and len(candidates) == 1:
2067
- # Name-only fallback with a single candidate — not ambiguous.
2068
- edge_strat = strat
2069
- else:
2070
- edge_strat = strat
2071
-
2072
- if not candidates:
2073
- # Receiver was resolved but the callee method isn't indexed on that type
2074
- # (e.g. JDK / Spring / external library). Preserve the receiver-tier
2075
- # strategy and confidence — only resolved=False signals the phantom callee
2076
- # (B3 fix: do NOT downgrade to confidence=0.0 / strategy='phantom' here).
2077
- pid = _phantom_method_id(
2078
- tables, receiver_fqn=recv_type, receiver_expr=call.receiver_expr,
2079
- callee=call.callee_simple, arg_count=call.arg_count,
2080
- )
2081
- _emit_call_edge(
2082
- tables, stats, src_id=member.node_id, dst_id=pid, call=call,
2083
- confidence=edge_conf, strategy=edge_strat, resolved=False,
2084
- )
2085
- return
2086
-
2087
- if len(candidates) > 1 and edge_strat != "overload_ambiguous":
2088
- candidates = _collapse_supertype_duplicates(candidates, recv_type, call, tables)
2089
-
2090
- if len(candidates) == 1:
2091
- candidate = candidates[0]
2092
- ref_arity: int | None = None
2093
- if call.arg_count < 0:
2094
- ref_arity = len(candidate.decl.parameters)
2095
- _emit_call_edge(
2096
- tables, stats, src_id=member.node_id, dst_id=candidate.node_id, call=call,
2097
- confidence=edge_conf, strategy=edge_strat, resolved=True,
2098
- edge_arg_count=ref_arity,
2099
- )
2100
- return
2101
-
2102
- for c in candidates:
2103
- ref_arity_multi: int | None = len(c.decl.parameters) if call.arg_count < 0 else None
2104
- _emit_call_edge(
2105
- tables, stats, src_id=member.node_id, dst_id=c.node_id, call=call,
2106
- confidence=edge_conf, strategy="overload_ambiguous", resolved=True,
2107
- edge_arg_count=ref_arity_multi,
2108
- )
2109
-
2110
-
2111
- def _resolve_method_calls(
2112
- member: MemberEntry,
2113
- ast: JavaFileAst,
2114
- tables: GraphTables,
2115
- stats: CallResolutionStats,
2116
- ) -> None:
2117
- scope = _scope_table(member, ast, tables)
2118
- for call in member.decl.call_sites:
2119
- try:
2120
- _resolve_and_emit_call(call, member, ast, tables, stats, scope=scope)
2121
- except Exception as e:
2122
- log.warning("call resolution failed for %s: %s", member.decl.signature, e)
2123
-
2124
-
2125
- def _process_file_calls(
2126
- file_ast: JavaFileAst,
2127
- file_path: str,
2128
- tables: GraphTables,
2129
- stats: CallResolutionStats,
2130
- ) -> None:
2131
- for member in tables.members:
2132
- if member.file_path != file_path:
2133
- continue
2134
- try:
2135
- _resolve_method_calls(member, file_ast, tables, stats)
2136
- except Exception as e:
2137
- log.warning("Failed to extract calls from %s#%s: %s", member.parent_fqn, member.decl.signature, e)
2138
-
2139
-
2140
- def pass3_calls(tables: GraphTables, asts: dict[str, JavaFileAst], *, verbose: bool) -> None:
2141
- if verbose:
2142
- _verbose_stderr_line(_PASS3_START)
2143
- _build_member_indexes(tables)
2144
- stats = CallResolutionStats()
2145
- with _graph_pass_progress("3/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 3", verbose=verbose):
2146
- for rel_path, file_ast in asts.items():
2147
- try:
2148
- _process_file_calls(file_ast, rel_path, tables, stats)
2149
- except Exception as e:
2150
- log.error("Call extraction failed for %s: %s", rel_path, e)
2151
- denom_calls = max(1, stats.total)
2152
- denom_sites = max(1, stats.total + stats.phantom_chained + stats.phantom_other)
2153
- pct_chained = 100.0 * stats.phantom_chained / denom_sites
2154
- pct_callee_unres = 100.0 * stats.callee_unresolved / denom_calls
2155
- pct_phantom_recv = 100.0 * stats.phantom_other / denom_sites
2156
- tables.pass3_skipped_cross_service = int(stats.skipped_cross_service)
2157
- tables.pass3_unresolved_phantom_receiver = int(stats.phantom_other)
2158
- tables.pass3_unresolved_chained = int(stats.phantom_chained)
2159
- msg = (
2160
- f"Call resolution: {stats.total} CALLS rows, {stats.phantom_chained} chained unresolved "
2161
- f"({pct_chained:.1f}%), {stats.callee_unresolved} unresolved callee on CALLS "
2162
- f"({pct_callee_unres:.1f}%), {stats.phantom_other} phantom-receiver unresolved "
2163
- f"({pct_phantom_recv:.1f}%), {stats.skipped_cross_service} skipped cross-service, "
2164
- f"strategies: {dict(stats.by_strategy)}"
2165
- )
2166
- log.info(msg)
2167
- if verbose:
2168
- _verbose_stderr_line(f"[graph] pass 3 · {msg}")
2169
-
2170
-
2171
- _PATH_VAR_SEG = re.compile(r"^\{([^:{}]+)(?::([^}]*))?\}$") # whole path segment
2172
-
2173
-
2174
- def _normalize_path(raw_path: str) -> tuple[str, str]:
2175
- """Return `(path_template, path_regex)` for a servlet-style path pattern.
2176
-
2177
- `/api/users/{id}` → ``("/api/users/{}", "^/api/users/[^/]+/?$")``.
2178
- `{id:\\d+}` constraints strip to ``{}`` in the template while preserving the
2179
- regex constraint for that segment. Deterministic for shared use by B2b/B6.
2180
- """
2181
- raw_path = (raw_path or "").strip()
2182
- if not raw_path:
2183
- return "", ""
2184
- p = raw_path if raw_path.startswith("/") else "/" + raw_path
2185
- trimmed = p.rstrip("/")
2186
- if trimmed == "":
2187
- return "/", "^/?$"
2188
- segments = [s for s in trimmed.split("/") if s != ""]
2189
- tmpl_parts: list[str] = []
2190
- re_parts: list[str] = []
2191
- for seg in segments:
2192
- m = _PATH_VAR_SEG.fullmatch(seg)
2193
- if m:
2194
- tmpl_parts.append("{}")
2195
- constraint = m.group(2)
2196
- re_parts.append(constraint if constraint else "[^/]+")
2197
- else:
2198
- tmpl_parts.append(seg)
2199
- re_parts.append(re.escape(seg))
2200
- tmpl = "/" + "/".join(tmpl_parts)
2201
- body = "/".join(re_parts)
2202
- if not body.startswith("/"):
2203
- body = "/" + body
2204
- return tmpl, f"^{body}/?$"
2205
-
2206
-
2207
- def _route_id(
2208
- framework: str,
2209
- kind: str,
2210
- http_method: str,
2211
- path_template: str,
2212
- path_raw: str,
2213
- topic: str,
2214
- broker: str,
2215
- microservice: str,
2216
- ) -> str:
2217
- """Stable id; `path_raw` disambiguates HTTP routes when `path_template` is empty (SpEL / const)."""
2218
- path_key = path_template if path_template else path_raw
2219
- key = (
2220
- f"{framework}|{kind}|{http_method}|{path_key}|"
2221
- f"{topic}|{broker}|{microservice}"
2222
- )
2223
- return f"r:{hashlib.sha1(key.encode()).hexdigest()[:16]}"
2224
-
2225
-
2226
- def _client_id(
2227
- *,
2228
- microservice: str,
2229
- member_fqn: str,
2230
- client_kind: str,
2231
- path: str,
2232
- method: str,
2233
- ) -> str:
2234
- key = f"{microservice}|{member_fqn}|{client_kind}|{path}|{method}"
2235
- return f"c:{hashlib.sha1(key.encode()).hexdigest()[:16]}"
2236
-
2237
-
2238
- def _producer_id(
2239
- *,
2240
- microservice: str,
2241
- member_fqn: str,
2242
- producer_kind: str,
2243
- topic: str,
2244
- ) -> str:
2245
- # Topic-level identity per method+kind; broker is intentionally omitted so the same
2246
- # resolved topic on one method shares one Producer node across call sites.
2247
- key = f"{microservice}|{member_fqn}|{producer_kind}|{topic}"
2248
- return f"p:{hashlib.sha1(key.encode()).hexdigest()[:16]}"
2249
-
2250
-
2251
- # The four brownfield source layers — single source of truth. Consumed by the
2252
- # client/producer source-layer classifiers, the *_from_brownfield_pct stats
2253
- # (via brownfield_strategies), and the brownfield_only authoritativeness gate in
2254
- # _is_brownfield_sourced. codebase_client/codebase_producer are caller-side
2255
- # declaration strategies, not layers — they extend brownfield_strategies only.
2256
- _BROWNFIELD_LAYERS = frozenset({
2257
- "layer_a_meta",
2258
- "layer_b_ann",
2259
- "layer_b_fqn",
2260
- "layer_c_source",
2261
- })
2262
-
2263
-
2264
- def _client_source_layer(strategy: str) -> str:
2265
- if strategy in _BROWNFIELD_LAYERS:
2266
- return strategy
2267
- # Some caller extraction paths emit client kind as strategy; treat those
2268
- # as builtin-source declarations instead of warning on every row.
2269
- if strategy in VALID_CLIENT_KINDS:
2270
- return "builtin"
2271
- if strategy != "builtin":
2272
- log.warning("unknown client source strategy %r, falling back to builtin", strategy)
2273
- return "builtin"
2274
-
2275
-
2276
- def _producer_source_layer(strategy: str) -> str:
2277
- if strategy in _BROWNFIELD_LAYERS:
2278
- return strategy
2279
- if strategy in VALID_PRODUCER_KINDS:
2280
- return "builtin"
2281
- if strategy != "builtin":
2282
- log.warning("unknown producer source strategy %r, falling back to builtin", strategy)
2283
- return "builtin"
2284
-
2285
-
2286
- _ROUTE_LAYER_RANK: dict[str, int] = {
2287
- "builtin": 0,
2288
- "layer_b_ann": 1,
2289
- "layer_a_meta": 2,
2290
- "layer_c_source": 3,
2291
- "layer_b_fqn": 4,
2292
- }
2293
-
2294
-
2295
- def pass4_routes(
2296
- tables: GraphTables,
2297
- asts: dict[str, JavaFileAst],
2298
- *,
2299
- source_root: Path,
2300
- verbose: bool,
2301
- ) -> None:
2302
- stats = tables.route_stats
2303
- overrides = load_brownfield_overrides(source_root)
2304
- try:
2305
- prs = str(source_root.resolve())
2306
- except OSError:
2307
- prs = str(source_root)
2308
- tables.cross_service_resolution = _load_config_cross_service_resolution(prs)
2309
- meta_chain = collect_annotation_meta_chain(prs)
2310
- if verbose:
2311
- _verbose_stderr_line(_PASS4_START)
2312
- with _graph_pass_progress("4/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 4", verbose=verbose):
2313
-
2314
- for ast in asts.values():
2315
- stats.routes_skipped_unresolved += ast.routes_skipped_unresolved
2316
-
2317
- routes_by_id: dict[str, RouteRow] = {}
2318
- exposes_seen: set[tuple[str, str]] = set()
2319
-
2320
- http_kinds = frozenset({"http_endpoint", "http_consumer"})
2321
-
2322
- for member in sorted(tables.members, key=lambda m: m.node_id):
2323
- if member.decl.is_constructor:
2324
- continue
2325
- ast = asts.get(member.file_path)
2326
- if ast is None:
2327
- continue
2328
- type_decl = tables.types[member.parent_fqn].decl
2329
- final_routes = resolve_routes_for_method(
2330
- method_decl=member.decl,
2331
- enclosing_type=type_decl,
2332
- overrides=overrides,
2333
- meta_chain=meta_chain,
2334
- builtin_routes=member.decl.routes,
2335
- )
2336
- if not final_routes:
2337
- continue
2338
- for decl in final_routes:
2339
- path_template, path_regex = ("", "")
2340
- if decl.kind in http_kinds:
2341
- if decl.resolved and decl.resolution_strategy in (
2342
- "annotation",
2343
- "codebase_route",
2344
- ):
2345
- path_template, path_regex = _normalize_path(decl.path)
2346
- else:
2347
- path_template, path_regex = "", ""
2348
- rid = _route_id(
2349
- decl.framework,
2350
- decl.kind,
2351
- decl.http_method,
2352
- path_template,
2353
- decl.path,
2354
- decl.topic,
2355
- decl.broker,
2356
- member.microservice,
2357
- )
2358
- layer = decl.route_source_layer
2359
- if rid not in routes_by_id:
2360
- routes_by_id[rid] = RouteRow(
2361
- id=rid,
2362
- kind=decl.kind,
2363
- framework=decl.framework,
2364
- method=decl.http_method,
2365
- path=decl.path,
2366
- path_template=path_template,
2367
- path_regex=path_regex,
2368
- topic=decl.topic,
2369
- broker=decl.broker,
2370
- feign_name=decl.feign_name,
2371
- feign_url=decl.feign_url,
2372
- microservice=member.microservice,
2373
- module=member.module,
2374
- filename=decl.filename,
2375
- start_line=decl.start_line,
2376
- end_line=decl.end_line,
2377
- resolved=decl.resolved,
2378
- source_layer=layer,
2379
- )
2380
- else:
2381
- prev = routes_by_id[rid]
2382
- if _ROUTE_LAYER_RANK.get(layer, 0) > _ROUTE_LAYER_RANK.get(
2383
- prev.source_layer,
2384
- 0,
2385
- ):
2386
- routes_by_id[rid] = replace(prev, source_layer=layer)
2387
- ek = (member.node_id, rid)
2388
- if ek not in exposes_seen:
2389
- route_kind = routes_by_id[rid].kind
2390
- if route_kind == "http_consumer":
2391
- stats.exposes_suppressed_feign += 1
2392
- continue
2393
- exposes_seen.add(ek)
2394
- tables.exposes_rows.append(
2395
- ExposesRow(
2396
- symbol_id=member.node_id,
2397
- route_id=rid,
2398
- confidence=decl.confidence,
2399
- strategy=decl.resolution_strategy,
2400
- ),
2401
- )
2402
-
2403
- tables.routes_rows = sorted(routes_by_id.values(), key=lambda r: r.id)
2404
-
2405
- for row in tables.routes_rows:
2406
- stats.by_framework[row.framework] += 1
2407
- stats.by_kind[row.kind] += 1
2408
-
2409
- n_routes = len(tables.routes_rows)
2410
- if n_routes:
2411
- stats.routes_resolved_pct = 100.0 * sum(
2412
- 1 for r in tables.routes_rows if r.resolved
2413
- ) / n_routes
2414
- stats.routes_from_brownfield_pct = 100.0 * sum(
2415
- 1 for r in tables.routes_rows if r.source_layer != "builtin"
2416
- ) / n_routes
2417
- else:
2418
- stats.routes_resolved_pct = 100.0
2419
- stats.routes_from_brownfield_pct = 0.0
2420
-
2421
- by_layer: dict[str, int] = defaultdict(int)
2422
- for row in tables.routes_rows:
2423
- by_layer[row.source_layer] += 1
2424
- stats.routes_by_layer = dict(sorted(by_layer.items()))
2425
-
2426
- msg = (
2427
- f"Route extraction: emitted={n_routes}, exposes={len(tables.exposes_rows)}, "
2428
- f"exposes_suppressed_feign={stats.exposes_suppressed_feign}, "
2429
- f"skipped_unresolved={stats.routes_skipped_unresolved}, "
2430
- f"routes_resolved_pct={stats.routes_resolved_pct:.1f}, "
2431
- f"routes_from_brownfield_pct={stats.routes_from_brownfield_pct:.1f}, "
2432
- f"by_framework={dict(stats.by_framework)}"
2433
- )
2434
- log.info(msg)
2435
- if verbose:
2436
- _verbose_stderr_line(f"[graph] pass 4 · {msg}")
2437
-
2438
-
2439
- def pass5_imperative_edges(
2440
- tables: GraphTables,
2441
- asts: dict[str, JavaFileAst],
2442
- *,
2443
- source_root: Path,
2444
- verbose: bool,
2445
- ) -> None:
2446
- del asts
2447
- overrides = load_brownfield_overrides(source_root)
2448
- try:
2449
- prs = str(source_root.resolve())
2450
- except OSError:
2451
- prs = str(source_root)
2452
- tables.cross_service_resolution = _load_config_cross_service_resolution(prs)
2453
- meta_chain = collect_annotation_meta_chain(prs)
2454
- routes_by_id = {r.id: r for r in tables.routes_rows}
2455
- existing_route_ids = set(routes_by_id)
2456
- http_seen: set[tuple[str, str]] = set()
2457
- async_seen: set[tuple[str, str]] = set()
2458
- client_seen: set[str] = set()
2459
- producer_seen: set[str] = set()
2460
- declares_client_seen: set[tuple[str, str]] = set()
2461
- declares_producer_seen: set[tuple[str, str]] = set()
2462
- route_rows = list(tables.routes_rows)
2463
-
2464
- def _micro_factor(member: MemberEntry) -> float:
2465
- ms = microservice_for_path(member.file_path, source_root)
2466
- return 1.0 if ms else 0.85
2467
-
2468
- def _append_route(row: RouteRow) -> None:
2469
- if row.id in existing_route_ids:
2470
- return
2471
- existing_route_ids.add(row.id)
2472
- routes_by_id[row.id] = row
2473
- route_rows.append(row)
2474
-
2475
- def _phantom_http_route_id(call: OutgoingCallDecl) -> str:
2476
- if call.path_template_call and call.method_call:
2477
- return _route_id("", "http_endpoint", call.method_call, call.path_template_call, call.path_template_call, "", "", "")
2478
- uniq = hashlib.sha1(f"{call.filename}:{call.start_line}:{call.raw_uri}".encode()).hexdigest()[:12]
2479
- return f"r:phantom:{uniq}"
2480
-
2481
- def _phantom_async_route_id(call: OutgoingCallDecl) -> str:
2482
- if call.topic_call:
2483
- return _route_id("", "kafka_topic", "", "", "", call.topic_call, call.broker_call, "")
2484
- uniq = hashlib.sha1(f"{call.filename}:{call.start_line}:{call.raw_topic}".encode()).hexdigest()[:12]
2485
- return f"r:phantom:{uniq}"
2486
-
2487
- if verbose:
2488
- _verbose_stderr_line(_PASS5_START)
2489
- with _graph_pass_progress("5/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 5", verbose=verbose):
2490
- for member in sorted(tables.members, key=lambda x: x.node_id):
2491
- if member.decl.is_constructor:
2492
- continue
2493
- type_decl = tables.types[member.parent_fqn].decl
2494
- final_http_calls = resolve_http_client_for_method(
2495
- method_decl=member.decl,
2496
- enclosing_type=type_decl,
2497
- overrides=overrides,
2498
- meta_chain=meta_chain,
2499
- builtin_calls=member.decl.outgoing_calls,
2500
- )
2501
- final_async_calls = resolve_async_producer_for_method(
2502
- method_decl=member.decl,
2503
- enclosing_type=type_decl,
2504
- overrides=overrides,
2505
- meta_chain=meta_chain,
2506
- builtin_calls=member.decl.outgoing_calls,
2507
- )
2508
- micro_factor = _micro_factor(member)
2509
- for call in final_http_calls + final_async_calls:
2510
- if call.channel == "http":
2511
- client_path = (call.path_template_call or "").strip()
2512
- client_method = (call.method_call or "").strip().upper()
2513
- # Keep normalized path fields on Client now so LC3 filter semantics
2514
- # (`path_prefix`) can use persisted columns without extra transforms.
2515
- client_path_template = ""
2516
- client_path_regex = ""
2517
- if client_path:
2518
- client_path_template, client_path_regex = _normalize_path(client_path)
2519
- cid = _client_id(
2520
- microservice=member.microservice,
2521
- member_fqn=call.method_fqn,
2522
- client_kind=call.client_kind,
2523
- path=client_path,
2524
- method=client_method,
2525
- )
2526
- if cid not in client_seen:
2527
- client_seen.add(cid)
2528
- tables.client_rows.append(
2529
- ClientRow(
2530
- id=cid,
2531
- client_kind=call.client_kind,
2532
- target_service=call.feign_target_name,
2533
- path=client_path,
2534
- path_template=client_path_template,
2535
- path_regex=client_path_regex,
2536
- method=client_method,
2537
- member_fqn=call.method_fqn,
2538
- member_id=member.node_id,
2539
- microservice=member.microservice,
2540
- module=member.module,
2541
- filename=call.filename,
2542
- start_line=call.start_line,
2543
- end_line=call.end_line,
2544
- resolved=call.resolved,
2545
- source_layer=_client_source_layer(call.resolution_strategy),
2546
- ),
2547
- )
2548
- dkey = (member.node_id, cid)
2549
- if dkey not in declares_client_seen:
2550
- declares_client_seen.add(dkey)
2551
- tables.declares_client_rows.append(
2552
- DeclaresClientRow(
2553
- symbol_id=member.node_id,
2554
- client_id=cid,
2555
- confidence=call.confidence_base,
2556
- strategy=call.resolution_strategy,
2557
- ),
2558
- )
2559
- rid = ""
2560
- strategy = call.resolution_strategy
2561
- if call.client_kind == CLIENT_KIND_FEIGN_METHOD:
2562
- exposing = next((e for e in tables.exposes_rows if e.symbol_id == member.node_id), None)
2563
- if exposing is not None:
2564
- rid = exposing.route_id
2565
- if not rid:
2566
- rid = _phantom_http_route_id(call)
2567
- _append_route(
2568
- RouteRow(
2569
- id=rid,
2570
- kind="http_endpoint",
2571
- framework="",
2572
- method=call.method_call,
2573
- path=call.path_template_call,
2574
- path_template=call.path_template_call,
2575
- path_regex="",
2576
- topic="",
2577
- broker="",
2578
- feign_name=call.feign_target_name,
2579
- feign_url=call.feign_target_url,
2580
- microservice="",
2581
- module="",
2582
- filename=call.filename,
2583
- start_line=call.start_line,
2584
- end_line=call.end_line,
2585
- resolved=False,
2586
- source_layer="builtin",
2587
- )
2588
- )
2589
- key = (cid, rid)
2590
- if key in http_seen:
2591
- continue
2592
- http_seen.add(key)
2593
- conf = call.confidence_base * 0.3 * micro_factor
2594
- tables.http_call_rows.append(
2595
- HttpCallRow(
2596
- client_id=cid,
2597
- route_id=rid,
2598
- confidence=conf,
2599
- strategy=strategy,
2600
- method_call=call.method_call,
2601
- raw_uri=call.raw_uri,
2602
- match="unresolved",
2603
- )
2604
- )
2605
- tables.call_edge_stats.http_calls_total += 1
2606
- tables.call_edge_stats.http_calls_by_client_kind[call.client_kind] += 1
2607
- tables.call_edge_stats.http_calls_by_strategy[strategy] += 1
2608
- elif call.channel == "async":
2609
- topic_atom = (call.topic_call or "").strip()
2610
- pid = _producer_id(
2611
- microservice=member.microservice,
2612
- member_fqn=call.method_fqn,
2613
- producer_kind=call.client_kind,
2614
- topic=topic_atom,
2615
- )
2616
- if pid not in producer_seen:
2617
- producer_seen.add(pid)
2618
- tables.producer_rows.append(
2619
- ProducerRow(
2620
- id=pid,
2621
- producer_kind=call.client_kind,
2622
- topic=topic_atom,
2623
- broker=call.broker_call,
2624
- direction="producer",
2625
- member_fqn=call.method_fqn,
2626
- member_id=member.node_id,
2627
- microservice=member.microservice,
2628
- module=member.module,
2629
- filename=call.filename,
2630
- start_line=call.start_line,
2631
- end_line=call.end_line,
2632
- resolved=call.resolved,
2633
- source_layer=_producer_source_layer(call.resolution_strategy),
2634
- ),
2635
- )
2636
- dpkey = (member.node_id, pid)
2637
- if dpkey not in declares_producer_seen:
2638
- declares_producer_seen.add(dpkey)
2639
- tables.declares_producer_rows.append(
2640
- DeclaresProducerRow(
2641
- symbol_id=member.node_id,
2642
- producer_id=pid,
2643
- confidence=call.confidence_base,
2644
- strategy=call.resolution_strategy,
2645
- ),
2646
- )
2647
- rid = _phantom_async_route_id(call)
2648
- _append_route(
2649
- RouteRow(
2650
- id=rid,
2651
- kind="kafka_topic",
2652
- framework="",
2653
- method="",
2654
- path="",
2655
- path_template="",
2656
- path_regex="",
2657
- topic=call.topic_call,
2658
- broker=call.broker_call,
2659
- feign_name="",
2660
- feign_url="",
2661
- microservice="",
2662
- module="",
2663
- filename=call.filename,
2664
- start_line=call.start_line,
2665
- end_line=call.end_line,
2666
- resolved=False,
2667
- source_layer="builtin",
2668
- )
2669
- )
2670
- key = (pid, rid)
2671
- if key in async_seen:
2672
- continue
2673
- async_seen.add(key)
2674
- conf = call.confidence_base * 0.3 * micro_factor
2675
- strategy = call.resolution_strategy
2676
- tables.async_call_rows.append(
2677
- AsyncCallRow(
2678
- producer_id=pid,
2679
- route_id=rid,
2680
- confidence=conf,
2681
- strategy=strategy,
2682
- direction="producer",
2683
- raw_topic=call.raw_topic,
2684
- match="unresolved",
2685
- )
2686
- )
2687
- tables.call_edge_stats.async_calls_total += 1
2688
- tables.call_edge_stats.async_calls_by_client_kind[call.client_kind] += 1
2689
- tables.call_edge_stats.async_calls_by_strategy[strategy] += 1
2690
-
2691
- tables.routes_rows = sorted(route_rows, key=lambda r: r.id)
2692
- tables.client_rows = sorted(tables.client_rows, key=lambda c: c.id)
2693
- tables.declares_client_rows = sorted(
2694
- tables.declares_client_rows,
2695
- key=lambda e: (e.symbol_id, e.client_id),
2696
- )
2697
- tables.client_stats.clients_total = len(tables.client_rows)
2698
- tables.client_stats.declares_client_total = len(tables.declares_client_rows)
2699
- tables.client_stats.clients_by_kind = defaultdict(int)
2700
- for row in tables.client_rows:
2701
- tables.client_stats.clients_by_kind[row.client_kind] += 1
2702
- tables.producer_rows = sorted(tables.producer_rows, key=lambda p: p.id)
2703
- tables.declares_producer_rows = sorted(
2704
- tables.declares_producer_rows,
2705
- key=lambda e: (e.symbol_id, e.producer_id),
2706
- )
2707
- tables.producer_stats.producers_total = len(tables.producer_rows)
2708
- tables.producer_stats.declares_producer_total = len(tables.declares_producer_rows)
2709
- tables.producer_stats.producers_by_kind = defaultdict(int)
2710
- for row in tables.producer_rows:
2711
- tables.producer_stats.producers_by_kind[row.producer_kind] += 1
2712
- # brownfield_strategies = the four brownfield layers plus the two
2713
- # caller-side declaration strategies (@CodebaseHttpClient /
2714
- # @CodebaseProducer). These extend _BROWNFIELD_LAYERS deliberately:
2715
- # the *_from_brownfield_pct stats count annotation-declared callers as
2716
- # brownfield-sourced even though they are not "layers" and so do not
2717
- # gate brownfield_only authoritativeness in _is_brownfield_sourced.
2718
- brownfield_strategies = _BROWNFIELD_LAYERS | frozenset(
2719
- {"codebase_client", "codebase_producer"},
2720
- )
2721
- if tables.call_edge_stats.http_calls_total:
2722
- n_http = sum(
2723
- v for k, v in tables.call_edge_stats.http_calls_by_strategy.items()
2724
- if k in brownfield_strategies
2725
- )
2726
- tables.call_edge_stats.http_clients_from_brownfield_pct = (
2727
- 100.0 * float(n_http) / float(tables.call_edge_stats.http_calls_total)
2728
- )
2729
- if tables.call_edge_stats.async_calls_total:
2730
- n_async = sum(
2731
- v for k, v in tables.call_edge_stats.async_calls_by_strategy.items()
2732
- if k in brownfield_strategies
2733
- )
2734
- tables.call_edge_stats.async_producers_from_brownfield_pct = (
2735
- 100.0 * float(n_async) / float(tables.call_edge_stats.async_calls_total)
2736
- )
2737
- if verbose:
2738
- http_client = dict(sorted(tables.call_edge_stats.http_calls_by_client_kind.items()))
2739
- async_client = dict(sorted(tables.call_edge_stats.async_calls_by_client_kind.items()))
2740
- http_strategy = dict(sorted(tables.call_edge_stats.http_calls_by_strategy.items()))
2741
- async_strategy = dict(sorted(tables.call_edge_stats.async_calls_by_strategy.items()))
2742
- _verbose_stderr_line(
2743
- f"[graph] pass 5 · HTTP_CALLS: {len(tables.http_call_rows)} edges, "
2744
- f"ASYNC_CALLS: {len(tables.async_call_rows)} edges; "
2745
- f"http_by_client_kind={http_client}, async_by_client_kind={async_client}, "
2746
- f"http_by_strategy={http_strategy}, async_by_strategy={async_strategy}",
2747
- )
2748
-
2749
-
2750
- def _match_call_edge(
2751
- call: OutgoingCallDecl,
2752
- routes: list[RouteRow],
2753
- caller_microservice: str,
2754
- ) -> tuple[str, list[RouteRow]]:
2755
- """Return (match_outcome, candidate_routes) for an outgoing call."""
2756
- if (
2757
- (not call.resolved)
2758
- and call.path_template_call == ""
2759
- and call.topic_call == ""
2760
- ):
2761
- return "unresolved", []
2762
-
2763
- candidates: list[RouteRow] = []
2764
- if call.client_kind == CLIENT_KIND_FEIGN_METHOD:
2765
- # Prefer endpoint matching by target service + path/method for Feign declarations.
2766
- path_value = call.path_template_call
2767
- method_value = call.method_call
2768
- if path_value:
2769
- for r in routes:
2770
- if r.kind != "http_endpoint":
2771
- continue
2772
- if call.feign_target_name and r.microservice != call.feign_target_name:
2773
- continue
2774
- if not (r.method == "" or method_value == "" or r.method == method_value):
2775
- continue
2776
- if not r.path_regex:
2777
- continue
2778
- try:
2779
- if re.fullmatch(r.path_regex, path_value or "") is None:
2780
- continue
2781
- except re.error:
2782
- continue
2783
- candidates.append(r)
2784
- if not candidates:
2785
- # Fallback for legacy/manual routes that only expose Feign target names.
2786
- candidates = [
2787
- r for r in routes
2788
- if r.feign_name and call.feign_target_name and r.feign_name == call.feign_target_name
2789
- ]
2790
- elif call.channel == "http":
2791
- path_value = call.path_template_call
2792
- method_value = call.method_call
2793
- for r in routes:
2794
- if r.kind != "http_endpoint":
2795
- continue
2796
- if not (r.method == "" or method_value == "" or r.method == method_value):
2797
- continue
2798
- if not r.path_regex:
2799
- continue
2800
- try:
2801
- if re.fullmatch(r.path_regex, path_value or "") is None:
2802
- continue
2803
- except re.error:
2804
- continue
2805
- candidates.append(r)
2806
- elif call.channel == "async":
2807
- candidates = [
2808
- r for r in routes
2809
- if r.topic == call.topic_call and r.broker == call.broker_call
2810
- ]
2811
-
2812
- if not candidates:
2813
- return "phantom", []
2814
- if len(candidates) > 1:
2815
- return "ambiguous", candidates
2816
- if candidates[0].microservice and candidates[0].microservice == caller_microservice:
2817
- return "intra_service", candidates
2818
- return "cross_service", candidates
2819
-
2820
-
2821
- def _is_brownfield_sourced(
2822
- call_strategy: str,
2823
- candidates: list[RouteRow],
2824
- ) -> bool:
2825
- """Both sides must come from brownfield layers for an edge to count as
2826
- authoritative under brownfield_only mode."""
2827
- if not candidates:
2828
- return False
2829
- if call_strategy not in _BROWNFIELD_LAYERS:
2830
- return False
2831
- return all(
2832
- getattr(c, "source_layer", "builtin") in _BROWNFIELD_LAYERS
2833
- for c in candidates
2834
- )
2835
-
2836
-
2837
- def pass6_match_edges(
2838
- tables: GraphTables,
2839
- *,
2840
- verbose: bool,
2841
- ) -> None:
2842
- match_factor: dict[str, float] = {
2843
- "cross_service": 1.0,
2844
- "intra_service": 0.6,
2845
- "ambiguous": 0.5,
2846
- "phantom": 0.4,
2847
- "unresolved": 0.3,
2848
- }
2849
- route_by_id = {r.id: r for r in tables.routes_rows}
2850
- all_routes = [r for r in tables.routes_rows if r.microservice]
2851
- member_by_id = {m.node_id: m for m in tables.members}
2852
- clients_by_id = {c.id: c for c in tables.client_rows}
2853
- producers_by_id = {p.id: p for p in tables.producer_rows}
2854
- client_hints_by_member: dict[str, list[ClientRow]] = defaultdict(list)
2855
- for edge in tables.declares_client_rows:
2856
- client = clients_by_id.get(edge.client_id)
2857
- if client is None:
2858
- continue
2859
- # `DECLARES_CLIENT.symbol_id` targets `Symbol.id` for member symbols,
2860
- # and member symbols are emitted with `id == MemberEntry.node_id`.
2861
- client_hints_by_member[edge.symbol_id].append(client)
2862
- for member_symbol_id in list(client_hints_by_member.keys()):
2863
- # Deterministic fallback when a method carries multiple feign declarations.
2864
- client_hints_by_member[member_symbol_id].sort(key=lambda c: c.id)
2865
-
2866
- # Pass 6 is idempotent for full rebuilds: each run fully re-derives match outcomes.
2867
- # If incremental rebuild lands later (Tier-2 follow-up), this reset must remain pass-scoped.
2868
- tables.call_edge_stats.http_calls_match_breakdown.clear()
2869
- tables.call_edge_stats.async_calls_match_breakdown.clear()
2870
- tables.call_edge_stats.cross_service_calls_total = 0
2871
-
2872
- brownfield_only = tables.cross_service_resolution == "brownfield_only"
2873
- suppressed_auto_cross_http: list[str] = []
2874
- suppressed_auto_cross_async: list[str] = []
2875
- suppressed_auto_cross_count = 0
2876
-
2877
- def _micro_factor(member: MemberEntry | None) -> float:
2878
- return 1.0 if (member and member.microservice) else 0.85
2879
-
2880
- if verbose:
2881
- _verbose_stderr_line(_PASS6_START)
2882
- with _graph_pass_progress("6/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 6", verbose=verbose):
2883
- for row in tables.http_call_rows:
2884
- if row.match != "unresolved":
2885
- continue
2886
- client = clients_by_id.get(row.client_id)
2887
- member = member_by_id.get(client.member_id) if client else None
2888
- base = row.confidence / max(1e-9, (0.3 * _micro_factor(member)))
2889
- src_route = route_by_id.get(row.route_id)
2890
- if src_route is None and member is not None:
2891
- # Recover feign caller hints from persisted caller-side Client declarations.
2892
- for client in client_hints_by_member.get(member.node_id, ()):
2893
- if client.client_kind != CLIENT_KIND_FEIGN_METHOD:
2894
- continue
2895
- path_template, path_regex = _normalize_path(client.path)
2896
- src_route = RouteRow(
2897
- id="",
2898
- kind="http_consumer",
2899
- framework="feign",
2900
- method=client.method,
2901
- path=client.path,
2902
- path_template=path_template,
2903
- path_regex=path_regex,
2904
- topic="",
2905
- broker="",
2906
- feign_name=client.target_service,
2907
- # `Client` stores service-name hints, not feign URL; matcher keys off feign_name.
2908
- feign_url="",
2909
- microservice=member.microservice,
2910
- module=member.module,
2911
- filename=client.filename,
2912
- start_line=client.start_line,
2913
- end_line=client.end_line,
2914
- resolved=client.resolved,
2915
- source_layer=client.source_layer,
2916
- )
2917
- break
2918
- # Feign caller hints are synthesized as transient `http_consumer` routes in pass6;
2919
- # synthetic phantoms from imperative clients are `http_endpoint` even when `feign_name` is populated from
2920
- # `@CodebaseHttpClient.targetService` / YAML hints — those must path-match like RestTemplate.
2921
- _feign_like = (
2922
- src_route is not None
2923
- and src_route.kind == "http_consumer"
2924
- and bool(src_route.feign_name)
2925
- )
2926
- call = OutgoingCallDecl(
2927
- method_fqn=f"{member.parent_fqn}#{member.decl.signature}" if member else "",
2928
- method_sig=member.decl.signature if member else "",
2929
- client_kind=CLIENT_KIND_FEIGN_METHOD if _feign_like else CLIENT_KIND_REST_TEMPLATE,
2930
- channel="http",
2931
- feign_target_name=src_route.feign_name if src_route else "",
2932
- feign_target_url=src_route.feign_url if src_route else "",
2933
- path_template_call=src_route.path_template if src_route else "",
2934
- method_call=row.method_call,
2935
- topic_call="",
2936
- broker_call="",
2937
- raw_uri=row.raw_uri,
2938
- raw_topic="",
2939
- resolution_strategy=row.strategy,
2940
- confidence_base=base,
2941
- resolved=(row.strategy != "unresolved"),
2942
- filename=member.file_path if member else "",
2943
- start_line=member.decl.start_line if member else 0,
2944
- end_line=member.decl.end_line if member else 0,
2945
- )
2946
- outcome, candidates = _match_call_edge(call, all_routes, member.microservice if member else "")
2947
- if (
2948
- brownfield_only
2949
- and outcome == "cross_service"
2950
- and not _is_brownfield_sourced(row.strategy, candidates)
2951
- ):
2952
- outcome = "unresolved"
2953
- candidates = []
2954
- suppressed_auto_cross_count += 1
2955
- if len(suppressed_auto_cross_http) < 5:
2956
- suppressed_auto_cross_http.append(call.method_fqn)
2957
- if outcome in VALID_HTTP_CALL_MATCHES:
2958
- row.match = outcome
2959
- if outcome in ("cross_service", "intra_service") and len(candidates) == 1:
2960
- row.route_id = candidates[0].id
2961
- row.confidence = call.confidence_base * match_factor[row.match] * _micro_factor(member)
2962
- tables.call_edge_stats.http_calls_match_breakdown[row.match] += 1
2963
- if row.match == "cross_service":
2964
- tables.call_edge_stats.cross_service_calls_total += 1
2965
-
2966
- for row in tables.async_call_rows:
2967
- if row.match != "unresolved":
2968
- continue
2969
- producer = producers_by_id.get(row.producer_id)
2970
- member = member_by_id.get(producer.member_id) if producer else None
2971
- base = row.confidence / max(1e-9, (0.3 * _micro_factor(member)))
2972
- src_route = route_by_id.get(row.route_id)
2973
- async_kind = producer.producer_kind if producer else "kafka_send"
2974
- call = OutgoingCallDecl(
2975
- method_fqn=f"{member.parent_fqn}#{member.decl.signature}" if member else "",
2976
- method_sig=member.decl.signature if member else "",
2977
- client_kind=async_kind,
2978
- channel="async",
2979
- feign_target_name="",
2980
- feign_target_url="",
2981
- path_template_call="",
2982
- method_call="",
2983
- topic_call=src_route.topic if src_route else "",
2984
- broker_call=src_route.broker if src_route else "",
2985
- raw_uri="",
2986
- raw_topic=row.raw_topic,
2987
- resolution_strategy=row.strategy,
2988
- confidence_base=base,
2989
- resolved=(row.strategy != "unresolved"),
2990
- filename=member.file_path if member else "",
2991
- start_line=member.decl.start_line if member else 0,
2992
- end_line=member.decl.end_line if member else 0,
2993
- )
2994
- outcome, candidates = _match_call_edge(call, all_routes, member.microservice if member else "")
2995
- if (
2996
- brownfield_only
2997
- and outcome == "cross_service"
2998
- and not _is_brownfield_sourced(row.strategy, candidates)
2999
- ):
3000
- outcome = "unresolved"
3001
- candidates = []
3002
- suppressed_auto_cross_count += 1
3003
- if len(suppressed_auto_cross_async) < 5:
3004
- suppressed_auto_cross_async.append(call.method_fqn)
3005
- if outcome in VALID_HTTP_CALL_MATCHES:
3006
- row.match = outcome
3007
- if outcome in ("cross_service", "intra_service") and len(candidates) == 1:
3008
- row.route_id = candidates[0].id
3009
- row.confidence = call.confidence_base * match_factor[row.match] * _micro_factor(member)
3010
- tables.call_edge_stats.async_calls_match_breakdown[row.match] += 1
3011
- if row.match == "cross_service":
3012
- tables.call_edge_stats.cross_service_calls_total += 1
3013
-
3014
- inbound_route_ids = {r.route_id for r in tables.http_call_rows} | {r.route_id for r in tables.async_call_rows}
3015
- tables.routes_rows = sorted(
3016
- [
3017
- r for r in tables.routes_rows
3018
- if not (
3019
- (r.microservice == "")
3020
- and (r.framework == "")
3021
- and (not r.resolved)
3022
- and (r.id not in inbound_route_ids)
3023
- )
3024
- ],
3025
- key=lambda r: r.id,
3026
- )
3027
-
3028
- if verbose:
3029
- if brownfield_only:
3030
- n_bf = tables.call_edge_stats.cross_service_calls_total
3031
- first_http = ", ".join(suppressed_auto_cross_http)
3032
- first_async = ", ".join(suppressed_auto_cross_async)
3033
- _verbose_stderr_line(
3034
- f"[graph] pass 6 · cross_service_resolution=brownfield_only:\n"
3035
- f" {n_bf} cross_service edges from brownfield layers,\n"
3036
- f" {suppressed_auto_cross_count} auto-cross-service candidates suppressed -> unresolved\n"
3037
- f" (first 5 http: {first_http})\n"
3038
- f" (first 5 async: {first_async})",
3039
- )
3040
- _verbose_stderr_line(
3041
- f"[graph] pass 6 · http_match={dict(sorted(tables.call_edge_stats.http_calls_match_breakdown.items()))}, "
3042
- f"async_match={dict(sorted(tables.call_edge_stats.async_calls_match_breakdown.items()))}, "
3043
- f"cross_service_calls_total={tables.call_edge_stats.cross_service_calls_total}",
3044
- )
3045
-
3046
-
3047
- # ---------- LadybugDB write ----------
3048
-
3049
-
3050
- _SCHEMA_NODE = (
3051
- "CREATE NODE TABLE Symbol("
3052
- "id STRING PRIMARY KEY, "
3053
- "kind STRING, name STRING, fqn STRING, package STRING, "
3054
- "module STRING, microservice STRING, "
3055
- "filename STRING, start_line INT64, end_line INT64, "
3056
- "start_byte INT64, end_byte INT64, "
3057
- "modifiers STRING[], annotations STRING[], capabilities STRING[], "
3058
- "role STRING, signature STRING, parent_id STRING, resolved BOOLEAN, "
3059
- "generated BOOLEAN, generated_by STRING, "
3060
- "search_text STRING"
3061
- ")"
3062
- )
3063
-
3064
- _SCHEMA_META = (
3065
- "CREATE NODE TABLE GraphMeta("
3066
- "key STRING PRIMARY KEY, "
3067
- "ontology_version INT64, built_at INT64, source_root STRING, "
3068
- "counts_json STRING, parse_errors INT64, "
3069
- "routes_total INT64, exposes_total INT64, "
3070
- # JSON map {framework: count}; STRING avoids LadybugDB Python MAP↔STRUCT binder mismatch.
3071
- "routes_by_framework STRING, "
3072
- "routes_resolved_pct DOUBLE, "
3073
- "routes_from_brownfield_pct DOUBLE, "
3074
- "routes_by_layer STRING, "
3075
- "clients_total INT64, "
3076
- "declares_client_total INT64, "
3077
- "clients_by_kind STRING, "
3078
- "producers_total INT64, "
3079
- "declares_producer_total INT64, "
3080
- "producers_by_kind STRING, "
3081
- "http_calls_total INT64, "
3082
- "async_calls_total INT64, "
3083
- "http_calls_by_strategy STRING, "
3084
- "async_calls_by_strategy STRING, "
3085
- "http_calls_resolved_pct DOUBLE, "
3086
- "async_calls_resolved_pct DOUBLE, "
3087
- "http_clients_from_brownfield_pct DOUBLE, "
3088
- "async_producers_from_brownfield_pct DOUBLE, "
3089
- "http_calls_match_breakdown STRING, "
3090
- "async_calls_match_breakdown STRING, "
3091
- "cross_service_calls_total INT64, "
3092
- "pass3_skipped_cross_service INT64, "
3093
- "pass3_unresolved_phantom_receiver INT64, "
3094
- "pass3_unresolved_chained INT64, "
3095
- "pass4_exposes_suppressed_feign INT64, "
3096
- "cross_service_resolution STRING"
3097
- ")"
3098
- )
3099
-
3100
- _SCHEMA_ROUTE = (
3101
- "CREATE NODE TABLE Route("
3102
- "id STRING, kind STRING, framework STRING, "
3103
- "method STRING, path STRING, path_template STRING, path_regex STRING, "
3104
- "topic STRING, broker STRING, "
3105
- "feign_name STRING, feign_url STRING, "
3106
- "microservice STRING, module STRING, "
3107
- "filename STRING, start_line INT64, end_line INT64, "
3108
- "resolved BOOLEAN, "
3109
- "PRIMARY KEY(id))"
3110
- )
3111
-
3112
- _SCHEMA_CLIENT = (
3113
- "CREATE NODE TABLE Client("
3114
- "id STRING, client_kind STRING, target_service STRING, "
3115
- "path STRING, path_template STRING, path_regex STRING, method STRING, "
3116
- "member_fqn STRING, member_id STRING, "
3117
- "microservice STRING, module STRING, filename STRING, "
3118
- "start_line INT64, end_line INT64, resolved BOOLEAN, source_layer STRING, "
3119
- "PRIMARY KEY(id))"
3120
- )
3121
-
3122
- _SCHEMA_PRODUCER = (
3123
- "CREATE NODE TABLE Producer("
3124
- "id STRING, producer_kind STRING, topic STRING, broker STRING, direction STRING, "
3125
- "member_fqn STRING, member_id STRING, "
3126
- "microservice STRING, module STRING, filename STRING, "
3127
- "start_line INT64, end_line INT64, resolved BOOLEAN, source_layer STRING, "
3128
- "PRIMARY KEY(id))"
3129
- )
3130
-
3131
- _SCHEMA_EXTENDS = (
3132
- "CREATE REL TABLE EXTENDS(FROM Symbol TO Symbol, "
3133
- "source_file STRING, dst_name STRING, dst_fqn STRING, resolved BOOLEAN)"
3134
- )
3135
- _SCHEMA_IMPLEMENTS = (
3136
- "CREATE REL TABLE IMPLEMENTS(FROM Symbol TO Symbol, "
3137
- "source_file STRING, dst_name STRING, dst_fqn STRING, resolved BOOLEAN)"
3138
- )
3139
- _SCHEMA_INJECTS = (
3140
- "CREATE REL TABLE INJECTS(FROM Symbol TO Symbol, "
3141
- "source_file STRING, dst_name STRING, dst_fqn STRING, resolved BOOLEAN, "
3142
- "mechanism STRING, annotation STRING, field_or_param STRING)"
3143
- )
3144
- _SCHEMA_DECLARES = "CREATE REL TABLE DECLARES(FROM Symbol TO Symbol, source_file STRING)"
3145
- _SCHEMA_OVERRIDES = "CREATE REL TABLE OVERRIDES(FROM Symbol TO Symbol, source_file STRING)"
3146
- _SCHEMA_CALLS = (
3147
- "CREATE REL TABLE CALLS(FROM Symbol TO Symbol, "
3148
- "source_file STRING, call_site_line INT64, call_site_byte INT64, arg_count INT64, "
3149
- "confidence DOUBLE, strategy STRING, source STRING, resolved BOOLEAN, "
3150
- "callee_declaring_role STRING)"
3151
- )
3152
- _SCHEMA_UNRESOLVED_CALL_SITE = (
3153
- "CREATE NODE TABLE UnresolvedCallSite("
3154
- "id STRING, caller_id STRING, call_site_line INT64, call_site_byte INT64, "
3155
- "arg_count INT64, callee_simple STRING, receiver_expr STRING, reason STRING, "
3156
- "PRIMARY KEY(id))"
3157
- )
3158
- _SCHEMA_UNRESOLVED_AT = "CREATE REL TABLE UNRESOLVED_AT(FROM Symbol TO UnresolvedCallSite, source_file STRING)"
3159
- _SCHEMA_EXPOSES = (
3160
- "CREATE REL TABLE EXPOSES(FROM Symbol TO Route, "
3161
- "source_file STRING, confidence DOUBLE, strategy STRING)"
3162
- )
3163
- _SCHEMA_DECLARES_CLIENT = (
3164
- "CREATE REL TABLE DECLARES_CLIENT(FROM Symbol TO Client, "
3165
- "source_file STRING, confidence DOUBLE, strategy STRING)"
3166
- )
3167
- _SCHEMA_DECLARES_PRODUCER = (
3168
- "CREATE REL TABLE DECLARES_PRODUCER(FROM Symbol TO Producer, "
3169
- "source_file STRING, confidence DOUBLE, strategy STRING)"
3170
- )
3171
- _SCHEMA_HTTP_CALLS = (
3172
- "CREATE REL TABLE HTTP_CALLS(FROM Client TO Route, "
3173
- "source_file STRING, confidence DOUBLE, strategy STRING, "
3174
- "method_call STRING, raw_uri STRING, match STRING)"
3175
- )
3176
- _SCHEMA_ASYNC_CALLS = (
3177
- "CREATE REL TABLE ASYNC_CALLS(FROM Producer TO Route, "
3178
- "source_file STRING, confidence DOUBLE, strategy STRING, "
3179
- "direction STRING, raw_topic STRING, match STRING)"
3180
- )
3181
-
3182
-
3183
- def _drop_symbol_fts_index_if_present(conn: ladybug.Connection) -> None:
3184
- """Drop the Symbol FTS index so the table drop below succeeds.
3185
-
3186
- LadybugDB refuses ``DROP TABLE Symbol`` while an FTS index references it ("Cannot
3187
- delete node table ... referenced by index"), so the index must go first. No-op when
3188
- FTS is unavailable (offline first-run) or the index was never created — in those
3189
- cases nothing blocks the table drop. Best-effort: any failure is swallowed.
3190
- """
3191
- try:
3192
- conn.execute("LOAD EXTENSION FTS")
3193
- existing = conn.execute("CALL SHOW_INDEXES() RETURN index_name")
3194
- names: set[str] = set()
3195
- while existing.has_next():
3196
- names.add(existing.get_next()[0])
3197
- if _SYMBOL_FTS_INDEX in names:
3198
- conn.execute(f"CALL DROP_FTS_INDEX('Symbol', '{_SYMBOL_FTS_INDEX}')")
3199
- except Exception:
3200
- pass
3201
-
3202
-
3203
- def _drop_all(conn: ladybug.Connection) -> None:
3204
- _drop_symbol_fts_index_if_present(conn)
3205
- for stmt in (
3206
- "DROP TABLE IF EXISTS DECLARES_CLIENT",
3207
- "DROP TABLE IF EXISTS DECLARES_PRODUCER",
3208
- "DROP TABLE IF EXISTS HTTP_CALLS",
3209
- "DROP TABLE IF EXISTS ASYNC_CALLS",
3210
- "DROP TABLE IF EXISTS EXPOSES",
3211
- "DROP TABLE IF EXISTS UNRESOLVED_AT",
3212
- "DROP TABLE IF EXISTS EXTENDS",
3213
- "DROP TABLE IF EXISTS IMPLEMENTS",
3214
- "DROP TABLE IF EXISTS INJECTS",
3215
- "DROP TABLE IF EXISTS CALLS",
3216
- "DROP TABLE IF EXISTS OVERRIDES",
3217
- "DROP TABLE IF EXISTS DECLARES",
3218
- "DROP TABLE IF EXISTS UnresolvedCallSite",
3219
- "DROP TABLE IF EXISTS Symbol",
3220
- "DROP TABLE IF EXISTS Route",
3221
- "DROP TABLE IF EXISTS Client",
3222
- "DROP TABLE IF EXISTS Producer",
3223
- "DROP TABLE IF EXISTS GraphMeta",
3224
- ):
3225
- try:
3226
- conn.execute(stmt)
3227
- except Exception:
3228
- pass
3229
-
3230
-
3231
- def _create_schema(conn: ladybug.Connection) -> None:
3232
- for stmt in (
3233
- _SCHEMA_NODE,
3234
- _SCHEMA_UNRESOLVED_CALL_SITE,
3235
- _SCHEMA_ROUTE,
3236
- _SCHEMA_CLIENT,
3237
- _SCHEMA_PRODUCER,
3238
- _SCHEMA_META,
3239
- _SCHEMA_EXTENDS,
3240
- _SCHEMA_IMPLEMENTS,
3241
- _SCHEMA_INJECTS,
3242
- _SCHEMA_DECLARES,
3243
- _SCHEMA_OVERRIDES,
3244
- _SCHEMA_CALLS,
3245
- _SCHEMA_UNRESOLVED_AT,
3246
- _SCHEMA_EXPOSES,
3247
- _SCHEMA_DECLARES_CLIENT,
3248
- _SCHEMA_DECLARES_PRODUCER,
3249
- _SCHEMA_HTTP_CALLS,
3250
- _SCHEMA_ASYNC_CALLS,
3251
- ):
3252
- conn.execute(stmt)
3253
-
3254
-
3255
- _IDENT_RE = re.compile(r"[A-Za-z][A-Za-z0-9]*")
3256
-
3257
-
3258
- def _compute_symbol_search_text(
3259
- name: str,
3260
- fqn: str,
3261
- signature: str,
3262
- annotations: list[str],
3263
- capabilities: list[str],
3264
- package: str = "",
3265
- ) -> str:
3266
- """Build the BM25-indexable token soup for a Symbol (fork A).
3267
-
3268
- camelCase / snake_case identifiers are split into lowercase tokens via the SAME
3269
- ``_split_identifier`` the query path uses (index- and query-time tokenization
3270
- must agree), then space-joined into one STRING that LadybugDB FTS porter-stems.
3271
- Name + fqn carry the strongest discovery signal; signature / annotations /
3272
- capabilities / package segments are weaker corroborators. Members reach this via
3273
- the same ``_node_row`` constructor as types.
3274
- """
3275
- toks: list[str] = []
3276
- for field in (name, fqn, signature, package):
3277
- for m in _IDENT_RE.findall(str(field or "")):
3278
- toks.extend(_split_identifier(m))
3279
- for lst in (annotations, capabilities):
3280
- for item in (lst or []):
3281
- for m in _IDENT_RE.findall(str(item)):
3282
- toks.extend(_split_identifier(m))
3283
- seen: set[str] = set()
3284
- out: list[str] = []
3285
- for t in toks:
3286
- if len(t) >= 2 and t not in seen:
3287
- seen.add(t)
3288
- out.append(t)
3289
- return " ".join(out)
3290
-
3291
-
3292
- def _ensure_symbol_fts_index(conn: ladybug.Connection, *, verbose: bool) -> None:
3293
- """Best-effort create the Symbol FTS (Okapi BM25) index if absent (fork A).
3294
-
3295
- Idempotent: skips when the index already exists. The FTS extension is fetched
3296
- from extension.ladybugdb.com on first ``INSTALL FTS`` (cached locally after); an
3297
- offline first run fails softly — ``run_lexical_search`` then falls back to the
3298
- heuristic over the bare Symbol scan. The index auto-maintains on later
3299
- COPY / MERGE / DELETE (verified), so this only runs on full builds and as a
3300
- cheap self-heal at the end of an incremental rebuild.
3301
- """
3302
- try:
3303
- try:
3304
- conn.execute("INSTALL FTS") # already-installed raises; swallow
3305
- except Exception:
3306
- pass
3307
- conn.execute("LOAD EXTENSION FTS")
3308
- existing = conn.execute("CALL SHOW_INDEXES() RETURN index_name")
3309
- names: set[str] = set()
3310
- while existing.has_next():
3311
- names.add(existing.get_next()[0])
3312
- if _SYMBOL_FTS_INDEX not in names:
3313
- conn.execute(
3314
- f"CALL CREATE_FTS_INDEX('Symbol', '{_SYMBOL_FTS_INDEX}', "
3315
- "['search_text'], stemmer := 'porter')"
3316
- )
3317
- if verbose:
3318
- _verbose_stderr_line(
3319
- f"[graph] fts · created Symbol {_SYMBOL_FTS_INDEX} (BM25 over search_text)"
3320
- )
3321
- except Exception as e:
3322
- if verbose:
3323
- _verbose_stderr_line(
3324
- f"[graph] fts · unavailable ({type(e).__name__}: {e}); "
3325
- "lexical search will use the heuristic scan"
3326
- )
3327
-
3328
-
3329
- def _node_row(**kwargs) -> dict:
3330
- base = {
3331
- "kind": "", "name": "", "fqn": "", "package": "",
3332
- "module": "", "microservice": "",
3333
- "filename": "", "start_line": 0, "end_line": 0,
3334
- "start_byte": 0, "end_byte": 0,
3335
- "modifiers": [], "annotations": [], "capabilities": [],
3336
- "role": "OTHER", "signature": "", "parent_id": "", "resolved": True,
3337
- "generated": False, "generated_by": None, "search_text": "",
3338
- }
3339
- base.update(kwargs)
3340
- # Derive the BM25 token soup unless the caller set it explicitly.
3341
- if not base.get("search_text"):
3342
- base["search_text"] = _compute_symbol_search_text(
3343
- base.get("name", ""), base.get("fqn", ""), base.get("signature", ""),
3344
- base.get("annotations", []), base.get("capabilities", []),
3345
- base.get("package", ""),
3346
- )
3347
- return base
3348
-
3349
-
3350
- def _bulk_copy(conn: ladybug.Connection, table_name: str, columns: list[str], rows: list[dict]) -> None:
3351
- """Bulk-load rows into a node/rel table via in-memory pyarrow COPY FROM.
3352
-
3353
- `columns` fixes column order; for REL tables the first two MUST be the
3354
- FROM/TO node primary keys (kuzu requirement). Empty `rows` is a no-op.
3355
-
3356
- Spike result (PR-P1 step-1): REL `COPY FROM` expects columns named `FROM` and `TO`
3357
- for the endpoint node IDs, followed by property columns in the declared order.
3358
- `pa.Table.from_pylist(rows)` correctly infers types from the dict values, but
3359
- we must select columns in the exact order expected by the table schema.
3360
- """
3361
- if not rows:
3362
- return
3363
- tbl = pa.Table.from_pylist(rows)
3364
- # Select columns in the exact order expected by the table schema
3365
- tbl = tbl.select(columns)
3366
- conn.execute(f"COPY {table_name} FROM $rows", {"rows": tbl})
3367
-
3368
-
3369
- def _existing_node_ids(conn: ladybug.Connection) -> set[str]:
3370
- """Return every node id (Symbol, Route, Client, Producer) currently in the graph.
3371
-
3372
- Bulk ``COPY FROM`` enforces referential integrity: a REL row whose FROM/TO
3373
- endpoint isn't a loaded node raises ``Unable to find primary key value``. The
3374
- legacy per-row ``MERGE (a:Symbol {id:$src}),(b:Symbol {id:$dst})`` silently
3375
- dropped such edges (a ``MATCH`` against a missing endpoint creates nothing).
3376
- Edge writers filter edge rows against this set to reproduce that exactly.
3377
-
3378
- This queries the live DB rather than just ``tables`` because it is shared
3379
- with the incremental path, whose edges legitimately reference nodes written
3380
- in prior runs.
3381
- """
3382
- result = conn.execute("MATCH (n) RETURN n.id")
3383
- ids: set[str] = set()
3384
- while result.has_next():
3385
- ids.add(result.get_next()[0])
3386
- return ids
3387
-
3388
-
3389
- # Column-order constants for bulk COPY FROM.
3390
- # For REL tables, the first two entries are FROM/TO node primary keys (kuzu requirement).
3391
- # Order matches the corresponding _SCHEMA_* declarations above.
3392
- _NODE_COLUMNS = [
3393
- "id", "kind", "name", "fqn", "package", "module", "microservice",
3394
- "filename", "start_line", "end_line", "start_byte", "end_byte",
3395
- "modifiers", "annotations", "capabilities", "role", "signature", "parent_id", "resolved",
3396
- "generated", "generated_by", "search_text"
3397
- ]
3398
-
3399
- # Type declaration kinds. Tuple (not set) so the rendered SQL `IN` clause is
3400
- # deterministic. Used to (a) load type stubs for cross-file resolution and
3401
- # (b) scope the incremental property-refresh SET to type nodes.
3402
- _TYPE_KINDS: tuple[str, ...] = ("class", "interface", "enum", "annotation", "record")
3403
-
3404
- # Update every mutable Symbol field on an existing node by primary key. Used on
3405
- # the incremental path to refresh preserved dependent type nodes whose
3406
- # `role`/`capabilities` (and other project-wide-derived fields) can shift
3407
- # without their own source changing — restoring the upsert the legacy per-row
3408
- # `MERGE (n:Symbol {id:$id}) SET …` provided. Field list mirrors `_NODE_COLUMNS`
3409
- # minus `id`.
3410
- _SET_SYMBOL_BY_ID = (
3411
- "MATCH (n:Symbol {id: $id}) "
3412
- "SET n.kind = $kind, n.name = $name, n.fqn = $fqn, "
3413
- "n.package = $package, n.module = $module, n.microservice = $microservice, "
3414
- "n.filename = $filename, "
3415
- "n.start_line = $start_line, n.end_line = $end_line, "
3416
- "n.start_byte = $start_byte, n.end_byte = $end_byte, "
3417
- "n.modifiers = $modifiers, n.annotations = $annotations, "
3418
- "n.capabilities = $capabilities, n.role = $role, "
3419
- "n.signature = $signature, n.parent_id = $parent_id, n.resolved = $resolved, "
3420
- "n.generated = $generated, n.generated_by = $generated_by, n.search_text = $search_text"
3421
- )
3422
-
3423
- # Refresh every mutable Route field on an existing Route node by id. Mirrors the
3424
- # `_write_nodes_impl` Symbol pattern (bulk COPY new rows + per-row SET existing
3425
- # ones) so the global pass 5-6 step no longer needs a per-row MERGE upsert.
3426
- # Field list mirrors `_ROUTE_COLUMNS` minus `id`.
3427
- _SET_ROUTE_BY_ID = (
3428
- "MATCH (r:Route {id: $id}) "
3429
- "SET r.kind = $kind, r.framework = $framework, r.method = $method, "
3430
- "r.path = $path, r.path_template = $path_template, r.path_regex = $path_regex, "
3431
- "r.topic = $topic, r.broker = $broker, r.feign_name = $feign_name, r.feign_url = $feign_url, "
3432
- "r.microservice = $microservice, r.module = $module, r.filename = $filename, "
3433
- "r.start_line = $start_line, r.end_line = $end_line, r.resolved = $resolved"
3434
- )
3435
-
3436
- _REL_EXTENDS_COLUMNS = ["FROM", "TO", "source_file", "dst_name", "dst_fqn", "resolved"]
3437
- _REL_IMPLEMENTS_COLUMNS = ["FROM", "TO", "source_file", "dst_name", "dst_fqn", "resolved"]
3438
- _REL_INJECTS_COLUMNS = ["FROM", "TO", "source_file", "dst_name", "dst_fqn", "resolved", "mechanism", "annotation", "field_or_param"]
3439
- _REL_DECLARES_COLUMNS = ["FROM", "TO", "source_file"]
3440
- _REL_OVERRIDES_COLUMNS = ["FROM", "TO", "source_file"]
3441
- _REL_CALLS_COLUMNS = ["FROM", "TO", "source_file", "call_site_line", "call_site_byte", "arg_count", "confidence", "strategy", "source", "resolved", "callee_declaring_role"]
3442
-
3443
- _UNRESOLVED_CALL_SITE_COLUMNS = ["id", "caller_id", "call_site_line", "call_site_byte", "arg_count", "callee_simple", "receiver_expr", "reason"]
3444
- _REL_UNRESOLVED_AT_COLUMNS = ["FROM", "TO", "source_file"]
3445
-
3446
- # Node table column constants (for bulk COPY FROM)
3447
- _ROUTE_COLUMNS = ["id", "kind", "framework", "method", "path", "path_template", "path_regex", "topic", "broker", "feign_name", "feign_url", "microservice", "module", "filename", "start_line", "end_line", "resolved"]
3448
- _CLIENT_COLUMNS = ["id", "client_kind", "target_service", "path", "path_template", "path_regex", "method", "member_fqn", "member_id", "microservice", "module", "filename", "start_line", "end_line", "resolved", "source_layer"]
3449
- _PRODUCER_COLUMNS = ["id", "producer_kind", "topic", "broker", "direction", "member_fqn", "member_id", "microservice", "module", "filename", "start_line", "end_line", "resolved", "source_layer"]
3450
-
3451
- # REL table column constants for routes/clients/producers
3452
- _REL_EXPOSES_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy"]
3453
- _REL_DECLARES_CLIENT_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy"]
3454
- _REL_DECLARES_PRODUCER_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy"]
3455
- _REL_HTTP_CALLS_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy", "method_call", "raw_uri", "match"]
3456
- _REL_ASYNC_CALLS_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy", "direction", "raw_topic", "match"]
3457
-
3458
-
3459
- def _write_nodes_impl(
3460
- conn: ladybug.Connection,
3461
- tables: GraphTables,
3462
- *,
3463
- project_root: Path,
3464
- meta_chain: dict[str, frozenset[str]] | None,
3465
- ) -> None:
3466
- overrides = load_brownfield_overrides(project_root)
3467
- try:
3468
- prs = str(project_root.resolve())
3469
- except OSError:
3470
- prs = str(project_root)
3471
- tables.cross_service_resolution = _load_config_cross_service_resolution(prs)
3472
- mch = meta_chain
3473
-
3474
- # Stage all Symbol rows
3475
- rows: list[dict] = []
3476
- # Node ids loaded from the existing graph as resolution-only stubs
3477
- # (`_load_existing_types`); their staged rows carry placeholder values and
3478
- # must never be written back over the real nodes.
3479
- stub_ids: set[str] = set()
3480
-
3481
- # packages
3482
- for pkg, pid in tables.packages.items():
3483
- rows.append(_node_row(
3484
- id=pid, kind="package", name=pkg.rsplit(".", 1)[-1], fqn=pkg, package=pkg,
3485
- ))
3486
- # files
3487
- for path, fid in tables.files.items():
3488
- rows.append(_node_row(
3489
- id=fid, kind="file", name=Path(path).name, fqn=path, filename=path,
3490
- ))
3491
- # types
3492
- for entry in tables.types.values():
3493
- d = entry.decl
3494
- role, capabilities = resolve_role_and_capabilities(
3495
- d,
3496
- overrides=overrides,
3497
- meta_chain=mch,
3498
- )
3499
- # Read generated/generated_by from pass-1 classification or stub persistence
3500
- generated, generated_by = tables.type_generated_by_node_id.get(entry.node_id, (False, None))
3501
-
3502
- if entry.loaded_from_db:
3503
- stub_ids.add(entry.node_id)
3504
- # Out-of-scope stub: its annotation-less decl collapses role to the
3505
- # default. The real node's role was persisted at index time and seeded
3506
- # into type_role_by_node_id by _load_existing_types; trust it so CALLS
3507
- # edges into this type keep the correct callee_declaring_role (#352).
3508
- # The staged row is filtered out of the write via stub_ids, so its
3509
- # capabilities placeholder never reaches the graph.
3510
- role = tables.type_role_by_node_id.get(entry.node_id, role)
3511
- capabilities = []
3512
- # For stubs, trust the persisted generated/generated_by (seeded by _load_existing_types)
3513
- else:
3514
- tables.type_role_by_node_id[entry.node_id] = role
3515
- rows.append(_node_row(
3516
- id=entry.node_id, kind=d.kind, name=d.name, fqn=d.fqn,
3517
- package=entry.package,
3518
- module=entry.module, microservice=entry.microservice,
3519
- filename=entry.file_path,
3520
- start_line=d.start_line, end_line=d.end_line,
3521
- start_byte=d.start_byte, end_byte=d.end_byte,
3522
- modifiers=list(d.modifiers),
3523
- annotations=[a.name for a in d.annotations],
3524
- capabilities=capabilities,
3525
- role=role,
3526
- signature="",
3527
- parent_id=tables.types[entry.outer_fqn].node_id if entry.outer_fqn and entry.outer_fqn in tables.types else "",
3528
- generated=generated,
3529
- generated_by=generated_by,
3530
- ))
3531
- # members (methods / constructors)
3532
- for m in tables.members:
3533
- rows.append(_node_row(
3534
- id=m.node_id, kind=m.kind, name=m.decl.name,
3535
- fqn=f"{m.parent_fqn}#{m.decl.signature}",
3536
- package=tables.types[m.parent_fqn].package if m.parent_fqn in tables.types else "",
3537
- module=m.module, microservice=m.microservice,
3538
- filename=m.file_path,
3539
- start_line=m.decl.start_line, end_line=m.decl.end_line,
3540
- start_byte=m.decl.start_byte, end_byte=m.decl.end_byte,
3541
- modifiers=list(m.decl.modifiers),
3542
- annotations=[a.name for a in m.decl.annotations],
3543
- signature=m.decl.signature, parent_id=m.parent_id,
3544
- ))
3545
- # phantoms
3546
- for pid, row in tables.phantoms.items():
3547
- rows.append(row)
3548
-
3549
- # Bulk-load new Symbol rows. The full-rebuild path starts from an empty
3550
- # database (`_drop_all`), so every row is new. The incremental path reaches
3551
- # here with a populated database: changed-file nodes were deleted by
3552
- # `_delete_file_scope` (absent here → new), while dependent-file nodes are
3553
- # deliberately preserved (see `_delete_file_scope` / issue #305).
3554
- existing_ids = _existing_node_ids(conn)
3555
- new_rows = [row for row in rows if row["id"] not in existing_ids]
3556
- _bulk_copy(conn, "Symbol", _NODE_COLUMNS, new_rows)
3557
-
3558
- # Refresh mutable properties on preserved dependent TYPE nodes (incremental
3559
- # path only; `update_rows` is empty on the full path). `role`/`capabilities`
3560
- # — and any other field derived from project-wide inputs (meta-annotation
3561
- # chain, brownfield overrides) — can shift without the type's own source
3562
- # changing, so a preserved dependent must be re-SET to stay byte-equivalent
3563
- # with a full rebuild. The legacy per-row `_MERGE_SYMBOL` upserted every
3564
- # staged node and did this implicitly; bulk `COPY FROM` only appends, so the
3565
- # SET is explicit here. Stubs (`stub_ids`) are skipped: their decl is a
3566
- # placeholder and their stored values are authoritative. Non-type kinds
3567
- # carry no mutable role/capabilities, so they are skipped too.
3568
- update_rows = [
3569
- row for row in rows
3570
- if row["id"] in existing_ids
3571
- and row["id"] not in stub_ids
3572
- and row["kind"] in _TYPE_KINDS
3573
- ]
3574
- for row in update_rows:
3575
- conn.execute(_SET_SYMBOL_BY_ID, row)
3576
-
3577
-
3578
- def _write_nodes(
3579
- conn: ladybug.Connection,
3580
- tables: GraphTables,
3581
- *,
3582
- project_root: Path,
3583
- meta_chain: dict[str, frozenset[str]] | None,
3584
- ) -> None:
3585
- _write_nodes_impl(conn, tables, project_root=project_root, meta_chain=meta_chain)
3586
-
3587
-
3588
-
3589
-
3590
- def _populate_declares_rows(tables: GraphTables) -> None:
3591
- # Skip members loaded from the existing graph for cross-file resolution: a
3592
- # DECLARES edge for an unchanged-file member already persists (its
3593
- # source_file is out of scope, so `_delete_file_scope` left it), and
3594
- # re-emitting it would append a duplicate (REL tables carry no primary key).
3595
- # Full-rebuild never loads members, so this is a no-op there.
3596
- tables.declares_rows = [
3597
- DeclaresRow(src_id=m.parent_id, dst_id=m.node_id)
3598
- for m in tables.members
3599
- if not m.loaded_from_db
3600
- ]
3601
-
3602
-
3603
- def _direct_supertype_ids(tables: GraphTables, type_id: str) -> list[str]:
3604
- out: list[str] = []
3605
- for r in tables.extends_rows:
3606
- if r.src_id == type_id:
3607
- out.append(r.dst_id)
3608
- for r in tables.implements_rows:
3609
- if r.src_id == type_id:
3610
- out.append(r.dst_id)
3611
- return out
3612
-
3613
-
3614
- def _populate_overrides_rows(tables: GraphTables) -> None:
3615
- """Materialize (subtype_method)-[:OVERRIDES]->(supertype_method) for one supertype hop.
3616
-
3617
- Matches ``LadybugDBGraph.override_axis_rollup_for`` (direct ``IMPLEMENTS`` / ``EXTENDS``
3618
- only, same ``signature``, distinct method ids, non-static instance methods).
3619
- """
3620
- by_declaring_type: dict[str, list[MemberEntry]] = defaultdict(list)
3621
- for m in tables.members:
3622
- by_declaring_type[m.parent_id].append(m)
3623
- pairs: set[tuple[str, str]] = set()
3624
- for m in tables.members:
3625
- if m.kind != "method" or "static" in m.decl.modifiers:
3626
- continue
3627
- impl_tid = m.parent_id
3628
- for sup_id in _direct_supertype_ids(tables, impl_tid):
3629
- for other in by_declaring_type.get(sup_id, ()):
3630
- if other.kind != "method":
3631
- continue
3632
- if other.decl.signature != m.decl.signature:
3633
- continue
3634
- if other.node_id == m.node_id:
3635
- continue
3636
- pairs.add((m.node_id, other.node_id))
3637
- tables.overrides_rows = [
3638
- DeclaresRow(src_id=a, dst_id=b) for a, b in sorted(pairs)
3639
- ]
3640
-
3641
-
3642
- def _build_file_by_node_id(tables: GraphTables) -> dict[str, str]:
3643
- """Build node_id -> file_path lookup for source_file resolution."""
3644
- lookup: dict[str, str] = {}
3645
- for entry in tables.types.values():
3646
- lookup[entry.node_id] = entry.file_path
3647
- for m in tables.members:
3648
- lookup[m.node_id] = m.file_path
3649
- return lookup
3650
-
3651
-
3652
- def _write_edges(conn: ladybug.Connection, tables: GraphTables, _file_by_node_id: dict[str, str] | None = None) -> None:
3653
- # Build node_id -> file_path lookup for source_file resolution.
3654
- if _file_by_node_id is None:
3655
- _file_by_node_id = _build_file_by_node_id(tables)
3656
-
3657
- # Bulk COPY FROM enforces referential integrity — a REL row whose endpoint
3658
- # node isn't loaded raises "Unable to find primary key value". The legacy
3659
- # per-row MERGE silently skipped such edges; drop them here to preserve the
3660
- # per-row graph exactly. _existing_node_ids reads the live DB (not just
3661
- # `tables`) so the incremental path's references to prior-run nodes still hold.
3662
- valid_ids = _existing_node_ids(conn)
3663
-
3664
- # Stage EXTENDS rows
3665
- extends_rows = [
3666
- {
3667
- "FROM": r.src_id, "TO": r.dst_id,
3668
- "source_file": _file_by_node_id.get(r.src_id, ""),
3669
- "dst_name": r.dst_name, "dst_fqn": r.dst_fqn, "resolved": r.resolved,
3670
- }
3671
- for r in tables.extends_rows
3672
- if r.src_id in valid_ids and r.dst_id in valid_ids
3673
- ]
3674
- _bulk_copy(conn, "EXTENDS", _REL_EXTENDS_COLUMNS, extends_rows)
3675
-
3676
- # Stage IMPLEMENTS rows
3677
- implements_rows = [
3678
- {
3679
- "FROM": r.src_id, "TO": r.dst_id,
3680
- "source_file": _file_by_node_id.get(r.src_id, ""),
3681
- "dst_name": r.dst_name, "dst_fqn": r.dst_fqn, "resolved": r.resolved,
3682
- }
3683
- for r in tables.implements_rows
3684
- if r.src_id in valid_ids and r.dst_id in valid_ids
3685
- ]
3686
- _bulk_copy(conn, "IMPLEMENTS", _REL_IMPLEMENTS_COLUMNS, implements_rows)
3687
-
3688
- # Stage INJECTS rows
3689
- injects_rows = [
3690
- {
3691
- "FROM": r.src_id, "TO": r.dst_id,
3692
- "source_file": _file_by_node_id.get(r.src_id, ""),
3693
- "dst_name": r.dst_name, "dst_fqn": r.dst_fqn, "resolved": r.resolved,
3694
- "mechanism": r.mechanism, "annotation": r.annotation,
3695
- "field_or_param": r.field_or_param,
3696
- }
3697
- for r in tables.injects_rows
3698
- if r.src_id in valid_ids and r.dst_id in valid_ids
3699
- ]
3700
- _bulk_copy(conn, "INJECTS", _REL_INJECTS_COLUMNS, injects_rows)
3701
-
3702
- # Stage DECLARES rows
3703
- declares_rows = [
3704
- {
3705
- "FROM": row.src_id, "TO": row.dst_id,
3706
- "source_file": _file_by_node_id.get(row.src_id, ""),
3707
- }
3708
- for row in tables.declares_rows
3709
- if row.src_id in valid_ids and row.dst_id in valid_ids
3710
- ]
3711
- _bulk_copy(conn, "DECLARES", _REL_DECLARES_COLUMNS, declares_rows)
3712
-
3713
- # Stage OVERRIDES rows
3714
- overrides_rows = [
3715
- {
3716
- "FROM": row.src_id, "TO": row.dst_id,
3717
- "source_file": _file_by_node_id.get(row.src_id, ""),
3718
- }
3719
- for row in tables.overrides_rows
3720
- if row.src_id in valid_ids and row.dst_id in valid_ids
3721
- ]
3722
- _bulk_copy(conn, "OVERRIDES", _REL_OVERRIDES_COLUMNS, overrides_rows)
3723
-
3724
- # Stage CALLS rows with dedup and callee_declaring_role materialization
3725
- seen_calls: set[tuple[str, str, int, int, int]] = set()
3726
- calls_rows: list[dict] = []
3727
- member_by_id = {m.node_id: m for m in tables.members}
3728
- for row in tables.calls_rows:
3729
- if row.src_id not in valid_ids or row.dst_id not in valid_ids:
3730
- continue
3731
- # Include call_site_byte so two call sites of the same method on the same
3732
- # source line (same arg_count) are kept as distinct edges (issue #359).
3733
- key = (row.src_id, row.dst_id, row.arg_count, row.call_site_line, row.call_site_byte)
3734
- if key in seen_calls:
3735
- continue
3736
- seen_calls.add(key)
3737
- calls_rows.append({
3738
- "FROM": row.src_id, "TO": row.dst_id,
3739
- "source_file": _file_by_node_id.get(row.src_id, ""),
3740
- "call_site_line": row.call_site_line, "call_site_byte": row.call_site_byte,
3741
- "arg_count": row.arg_count, "confidence": row.confidence, "strategy": row.strategy,
3742
- "source": row.source, "resolved": row.resolved,
3743
- "callee_declaring_role": _callee_declaring_role_at_write(
3744
- tables, row.dst_id, member_by_id=member_by_id,
3745
- ),
3746
- })
3747
- _bulk_copy(conn, "CALLS", _REL_CALLS_COLUMNS, calls_rows)
3748
-
3749
- # Stage UnresolvedCallSite node rows (must load before UNRESOLVED_AT edges)
3750
- seen_ucs: set[str] = set()
3751
- ucs_rows: list[dict] = []
3752
- for row in tables.unresolved_call_site_rows:
3753
- if row.id in seen_ucs:
3754
- continue
3755
- seen_ucs.add(row.id)
3756
- ucs_rows.append({
3757
- "id": row.id,
3758
- "caller_id": row.caller_id,
3759
- "call_site_line": row.call_site_line,
3760
- "call_site_byte": row.call_site_byte,
3761
- "arg_count": row.arg_count,
3762
- "callee_simple": row.callee_simple,
3763
- "receiver_expr": row.receiver_expr,
3764
- "reason": row.reason,
3765
- })
3766
- _bulk_copy(conn, "UnresolvedCallSite", _UNRESOLVED_CALL_SITE_COLUMNS, ucs_rows)
3767
-
3768
- # Stage UNRESOLVED_AT edge rows (one per unique UnresolvedCallSite node)
3769
- # Use the same ucs_rows list to ensure 1:1 correspondence
3770
- unresolved_at_rows = [
3771
- {
3772
- "FROM": ucs_row["caller_id"], "TO": ucs_row["id"],
3773
- "source_file": _file_by_node_id.get(ucs_row["caller_id"], ""),
3774
- }
3775
- for ucs_row in ucs_rows
3776
- if ucs_row["caller_id"] in valid_ids
3777
- ]
3778
- _bulk_copy(conn, "UNRESOLVED_AT", _REL_UNRESOLVED_AT_COLUMNS, unresolved_at_rows)
3779
-
3780
-
3781
- def _write_routes_and_exposes(conn: ladybug.Connection, tables: GraphTables, _file_by_node_id: dict[str, str] | None = None) -> None:
3782
- # Build node_id -> file_path lookup for source_file resolution (for Symbol sources).
3783
- if _file_by_node_id is None:
3784
- _file_by_node_id = _build_file_by_node_id(tables)
3785
-
3786
- # Build client_id -> filename lookup for HTTP_CALLS source_file.
3787
- _file_by_client_id: dict[str, str] = {row.id: row.filename for row in tables.client_rows}
3788
-
3789
- # Build producer_id -> filename lookup for ASYNC_CALLS source_file.
3790
- _file_by_producer_id: dict[str, str] = {row.id: row.filename for row in tables.producer_rows}
3791
-
3792
- # Bulk COPY FROM enforces referential integrity — get all valid node IDs
3793
- valid_ids = _existing_node_ids(conn)
3794
-
3795
- # Stage Route node rows (bulk-load before edges that reference them)
3796
- route_rows = [
3797
- {
3798
- "id": row.id,
3799
- "kind": row.kind,
3800
- "framework": row.framework,
3801
- "method": row.method,
3802
- "path": row.path,
3803
- "path_template": row.path_template,
3804
- "path_regex": row.path_regex,
3805
- "topic": row.topic,
3806
- "broker": row.broker,
3807
- "feign_name": row.feign_name,
3808
- "feign_url": row.feign_url,
3809
- "microservice": row.microservice,
3810
- "module": row.module,
3811
- "filename": row.filename,
3812
- "start_line": row.start_line,
3813
- "end_line": row.end_line,
3814
- "resolved": row.resolved,
3815
- }
3816
- for row in tables.routes_rows
3817
- ]
3818
- _bulk_copy(conn, "Route", _ROUTE_COLUMNS, route_rows)
3819
-
3820
- # Stage Client node rows (bulk-load before edges that reference them)
3821
- client_rows = [asdict(row) for row in tables.client_rows]
3822
- _bulk_copy(conn, "Client", _CLIENT_COLUMNS, client_rows)
3823
-
3824
- # Stage Producer node rows (bulk-load before edges that reference them)
3825
- producer_rows = [asdict(row) for row in tables.producer_rows]
3826
- _bulk_copy(conn, "Producer", _PRODUCER_COLUMNS, producer_rows)
3827
-
3828
- # Re-fetch valid IDs after loading Route/Client/Producer nodes (now includes them)
3829
- valid_ids = _existing_node_ids(conn)
3830
-
3831
- # Stage EXPOSES edge rows (Symbol -> Route)
3832
- exposes_rows = [
3833
- {
3834
- "FROM": row.symbol_id,
3835
- "TO": row.route_id,
3836
- "source_file": _file_by_node_id.get(row.symbol_id, ""),
3837
- "confidence": row.confidence,
3838
- "strategy": row.strategy,
3839
- }
3840
- for row in tables.exposes_rows
3841
- if row.symbol_id in valid_ids and row.route_id in valid_ids
3842
- ]
3843
- _bulk_copy(conn, "EXPOSES", _REL_EXPOSES_COLUMNS, exposes_rows)
3844
-
3845
- # Stage DECLARES_CLIENT edge rows (Symbol -> Client)
3846
- declares_client_rows = [
3847
- {
3848
- "FROM": row.symbol_id,
3849
- "TO": row.client_id,
3850
- "source_file": _file_by_node_id.get(row.symbol_id, ""),
3851
- "confidence": row.confidence,
3852
- "strategy": row.strategy,
3853
- }
3854
- for row in tables.declares_client_rows
3855
- if row.symbol_id in valid_ids and row.client_id in valid_ids
3856
- ]
3857
- _bulk_copy(conn, "DECLARES_CLIENT", _REL_DECLARES_CLIENT_COLUMNS, declares_client_rows)
3858
-
3859
- # Stage DECLARES_PRODUCER edge rows (Symbol -> Producer)
3860
- declares_producer_rows = [
3861
- {
3862
- "FROM": row.symbol_id,
3863
- "TO": row.producer_id,
3864
- "source_file": _file_by_node_id.get(row.symbol_id, ""),
3865
- "confidence": row.confidence,
3866
- "strategy": row.strategy,
3867
- }
3868
- for row in tables.declares_producer_rows
3869
- if row.symbol_id in valid_ids and row.producer_id in valid_ids
3870
- ]
3871
- _bulk_copy(conn, "DECLARES_PRODUCER", _REL_DECLARES_PRODUCER_COLUMNS, declares_producer_rows)
3872
-
3873
- # Stage HTTP_CALLS edge rows (Client -> Route)
3874
- http_call_rows = [
3875
- {
3876
- "FROM": row.client_id,
3877
- "TO": row.route_id,
3878
- "source_file": _file_by_client_id.get(row.client_id, ""),
3879
- "confidence": row.confidence,
3880
- "strategy": row.strategy,
3881
- "method_call": row.method_call,
3882
- "raw_uri": row.raw_uri,
3883
- "match": row.match,
3884
- }
3885
- for row in tables.http_call_rows
3886
- if row.client_id in valid_ids and row.route_id in valid_ids
3887
- ]
3888
- _bulk_copy(conn, "HTTP_CALLS", _REL_HTTP_CALLS_COLUMNS, http_call_rows)
3889
-
3890
- # Stage ASYNC_CALLS edge rows (Producer -> Route)
3891
- async_call_rows = [
3892
- {
3893
- "FROM": row.producer_id,
3894
- "TO": row.route_id,
3895
- "source_file": _file_by_producer_id.get(row.producer_id, ""),
3896
- "confidence": row.confidence,
3897
- "strategy": row.strategy,
3898
- "direction": row.direction,
3899
- "raw_topic": row.raw_topic,
3900
- "match": row.match,
3901
- }
3902
- for row in tables.async_call_rows
3903
- if row.producer_id in valid_ids and row.route_id in valid_ids
3904
- ]
3905
- _bulk_copy(conn, "ASYNC_CALLS", _REL_ASYNC_CALLS_COLUMNS, async_call_rows)
3906
-
3907
-
3908
- def _write_meta(conn: ladybug.Connection, tables: GraphTables, source_root: Path) -> None:
3909
- # Dedup key MUST match _write_edges (build_ast_graph.py, _REL_CALLS writer): the
3910
- # 5-tuple includes call_site_byte so two call sites of the same method on the
3911
- # same source line are counted separately. A previous version used the 4-tuple
3912
- # here, which made counts['calls'] (678) diverge from the real CALLS edge count
3913
- # (684) that _write_edges actually persisted — describe/stats then undercounted.
3914
- seen_calls: set[tuple[str, str, int, int, int]] = set()
3915
- calls_unique = 0
3916
- for row in tables.calls_rows:
3917
- key = (row.src_id, row.dst_id, row.arg_count, row.call_site_line, row.call_site_byte)
3918
- if key not in seen_calls:
3919
- seen_calls.add(key)
3920
- calls_unique += 1
3921
- st = tables.route_stats
3922
- routes_fw = dict(sorted(st.by_framework.items()))
3923
- call_stats = tables.call_edge_stats
3924
- client_stats = tables.client_stats
3925
- producer_stats = tables.producer_stats
3926
- http_by_strategy = dict(sorted(call_stats.http_calls_by_strategy.items()))
3927
- async_by_strategy = dict(sorted(call_stats.async_calls_by_strategy.items()))
3928
- http_match = dict(sorted(call_stats.http_calls_match_breakdown.items()))
3929
- async_match = dict(sorted(call_stats.async_calls_match_breakdown.items()))
3930
- http_resolved_pct = 0.0
3931
- async_resolved_pct = 0.0
3932
- if call_stats.http_calls_total:
3933
- # PR-D1 definition: "resolved_pct" is strategy-based (strategy != 'unresolved'),
3934
- # not match-based (all PR-D1 edges keep match='unresolved').
3935
- resolved_http = sum(v for k, v in call_stats.http_calls_by_strategy.items() if k != "unresolved")
3936
- http_resolved_pct = float(resolved_http) / float(call_stats.http_calls_total)
3937
- if call_stats.async_calls_total:
3938
- resolved_async = sum(v for k, v in call_stats.async_calls_by_strategy.items() if k != "unresolved")
3939
- async_resolved_pct = float(resolved_async) / float(call_stats.async_calls_total)
3940
- counts = {
3941
- "packages": len(tables.packages),
3942
- "files": len(tables.files),
3943
- "types": len(tables.types),
3944
- "members": len(tables.members),
3945
- "phantoms": len(tables.phantoms),
3946
- "extends": len(tables.extends_rows),
3947
- "implements": len(tables.implements_rows),
3948
- "injects": len(tables.injects_rows),
3949
- "declares": len(tables.declares_rows),
3950
- "overrides": len(tables.overrides_rows),
3951
- "calls": calls_unique,
3952
- "routes": len(tables.routes_rows),
3953
- "exposes": len(tables.exposes_rows),
3954
- "clients": len(tables.client_rows),
3955
- "declares_client": len(tables.declares_client_rows),
3956
- "producers": len(tables.producer_rows),
3957
- "declares_producer": len(tables.declares_producer_rows),
3958
- "http_calls": len(tables.http_call_rows),
3959
- "async_calls": len(tables.async_call_rows),
3960
- }
3961
- routes_layer = dict(sorted(st.routes_by_layer.items()))
3962
- clients_by_kind = dict(sorted(client_stats.clients_by_kind.items()))
3963
- producers_by_kind = dict(sorted(producer_stats.producers_by_kind.items()))
3964
- conn.execute(
3965
- "MERGE (m:GraphMeta {key: $k}) "
3966
- "SET m.ontology_version = $ov, m.built_at = $t, "
3967
- "m.source_root = $sr, m.counts_json = $cj, m.parse_errors = $pe, "
3968
- "m.routes_total = $routes_total, m.exposes_total = $exposes_total, "
3969
- "m.routes_by_framework = $routes_by_framework, m.routes_resolved_pct = $routes_resolved_pct, "
3970
- "m.routes_from_brownfield_pct = $routes_from_brownfield_pct, m.routes_by_layer = $routes_by_layer, "
3971
- "m.clients_total = $clients_total, m.declares_client_total = $declares_client_total, "
3972
- "m.clients_by_kind = $clients_by_kind, "
3973
- "m.producers_total = $producers_total, m.declares_producer_total = $declares_producer_total, "
3974
- "m.producers_by_kind = $producers_by_kind, "
3975
- "m.http_calls_total = $http_calls_total, m.async_calls_total = $async_calls_total, "
3976
- "m.http_calls_by_strategy = $http_calls_by_strategy, m.async_calls_by_strategy = $async_calls_by_strategy, "
3977
- "m.http_calls_resolved_pct = $http_calls_resolved_pct, m.async_calls_resolved_pct = $async_calls_resolved_pct, "
3978
- "m.http_clients_from_brownfield_pct = $http_clients_from_brownfield_pct, "
3979
- "m.async_producers_from_brownfield_pct = $async_producers_from_brownfield_pct, "
3980
- "m.http_calls_match_breakdown = $http_calls_match_breakdown, "
3981
- "m.async_calls_match_breakdown = $async_calls_match_breakdown, "
3982
- "m.cross_service_calls_total = $cross_service_calls_total, "
3983
- "m.pass3_skipped_cross_service = $pass3_skipped_cross_service, "
3984
- "m.pass3_unresolved_phantom_receiver = $pass3_unresolved_phantom_receiver, "
3985
- "m.pass3_unresolved_chained = $pass3_unresolved_chained, "
3986
- "m.pass4_exposes_suppressed_feign = $pass4_exposes_suppressed_feign, "
3987
- "m.cross_service_resolution = $cross_service_resolution",
3988
- {
3989
- "k": "graph",
3990
- "ov": ONTOLOGY_VERSION,
3991
- "t": int(time.time()),
3992
- "sr": str(source_root.resolve()),
3993
- "cj": json.dumps(counts),
3994
- "pe": tables.parse_errors,
3995
- "routes_total": len(tables.routes_rows),
3996
- "exposes_total": len(tables.exposes_rows),
3997
- "routes_by_framework": json.dumps(routes_fw),
3998
- "routes_resolved_pct": float(st.routes_resolved_pct),
3999
- "routes_from_brownfield_pct": float(st.routes_from_brownfield_pct),
4000
- "routes_by_layer": json.dumps(routes_layer),
4001
- "clients_total": int(client_stats.clients_total),
4002
- "declares_client_total": int(client_stats.declares_client_total),
4003
- "clients_by_kind": json.dumps(clients_by_kind),
4004
- "producers_total": int(producer_stats.producers_total),
4005
- "declares_producer_total": int(producer_stats.declares_producer_total),
4006
- "producers_by_kind": json.dumps(producers_by_kind),
4007
- "http_calls_total": call_stats.http_calls_total,
4008
- "async_calls_total": call_stats.async_calls_total,
4009
- "http_calls_by_strategy": json.dumps(http_by_strategy),
4010
- "async_calls_by_strategy": json.dumps(async_by_strategy),
4011
- "http_calls_resolved_pct": http_resolved_pct,
4012
- "async_calls_resolved_pct": async_resolved_pct,
4013
- "http_clients_from_brownfield_pct": call_stats.http_clients_from_brownfield_pct,
4014
- "async_producers_from_brownfield_pct": call_stats.async_producers_from_brownfield_pct,
4015
- "http_calls_match_breakdown": json.dumps(http_match),
4016
- "async_calls_match_breakdown": json.dumps(async_match),
4017
- "cross_service_calls_total": int(call_stats.cross_service_calls_total),
4018
- "pass3_skipped_cross_service": int(tables.pass3_skipped_cross_service),
4019
- "pass3_unresolved_phantom_receiver": int(tables.pass3_unresolved_phantom_receiver),
4020
- "pass3_unresolved_chained": int(tables.pass3_unresolved_chained),
4021
- "pass4_exposes_suppressed_feign": int(st.exposes_suppressed_feign),
4022
- "cross_service_resolution": str(tables.cross_service_resolution),
4023
- },
4024
- )
4025
-
4026
-
4027
- def incremental_rebuild(
4028
- source_root: Path,
4029
- ladybug_path: Path,
4030
- *,
4031
- verbose: bool,
4032
- expansion_cap: int = 50,
4033
- ) -> IncrementalResult:
4034
- """Incrementally rebuild the LadybugDB graph, processing only changed files and their dependents.
4035
-
4036
- Returns IncrementalResult with statistics about the rebuild.
4037
- Falls back to full rebuild if:
4038
- - No previous graph exists
4039
- - Ontology version < ONTOLOGY_VERSION (stale schema; rebuild for current columns)
4040
- - Crash marker exists (previous incremental run failed)
4041
- - Dependent expansion exceeds expansion_cap
4042
- """
4043
- t_start = time.time()
4044
-
4045
- # Step 1: Load existing graph and detect changes
4046
- if not ladybug_path.exists():
4047
- if verbose:
4048
- _verbose_stderr_line("[increment] no existing graph; falling back to full rebuild")
4049
- # Fall back to full rebuild
4050
- tables = GraphTables()
4051
- asts = pass1_parse(source_root, tables, verbose=verbose)
4052
- pass2_edges(tables, asts, verbose=verbose)
4053
- pass3_calls(tables, asts, verbose=verbose)
4054
- pass4_routes(tables, asts, source_root=source_root, verbose=verbose)
4055
- pass5_imperative_edges(tables, asts, source_root=source_root, verbose=verbose)
4056
- pass6_match_edges(tables, verbose=verbose)
4057
- write_ladybug(ladybug_path, tables, source_root=source_root, verbose=verbose)
4058
-
4059
- return IncrementalResult(
4060
- mode="full_fallback",
4061
- files_changed=0,
4062
- files_added=0,
4063
- files_removed=0,
4064
- dependents_reprocessed=0,
4065
- elapsed_sec=time.time() - t_start,
4066
- )
4067
-
4068
- db = ladybug.Database(str(ladybug_path))
4069
- conn = ladybug.Connection(db)
4070
- # If a Symbol FTS index exists, the scoped DELETE/MERGE/COPY below must maintain it,
4071
- # which needs the FTS extension loaded on THIS connection. Best-effort: when FTS is
4072
- # unavailable no index exists, so there's nothing to maintain and DML proceeds fine.
4073
- try:
4074
- conn.execute("LOAD EXTENSION FTS")
4075
- except Exception:
4076
- pass
4077
-
4078
- # Check ontology version
4079
- try:
4080
- meta_result = conn.execute("MATCH (m:GraphMeta) RETURN m.ontology_version AS version")
4081
- if meta_result.has_next():
4082
- row = meta_result.get_next()
4083
- version = row[0] if row else 0
4084
- if version < ONTOLOGY_VERSION:
4085
- if verbose:
4086
- _verbose_stderr_line(f"[increment] ontology version {version} < {ONTOLOGY_VERSION}; falling back to full rebuild")
4087
- conn.close()
4088
- db.close()
4089
- del conn, db
4090
- return _fallback_to_full(source_root, ladybug_path, verbose, t_start)
4091
- except Exception as e:
4092
- if verbose:
4093
- _verbose_stderr_line(f"[increment] failed to read ontology version: {e}; falling back to full rebuild")
4094
- try:
4095
- conn.close()
4096
- db.close()
4097
- except Exception:
4098
- pass
4099
- del conn, db
4100
- return _fallback_to_full(source_root, ladybug_path, verbose, t_start)
4101
-
4102
- index_dir = ladybug_path.parent
4103
- tracker = FileHashTracker(index_dir)
4104
- tracker.load()
4105
-
4106
- ignore = LayeredIgnore(source_root)
4107
- added, changed, removed = tracker.detect_changes(source_root, ignore=ignore)
4108
-
4109
- changed_files = added | changed | removed
4110
-
4111
- if not changed_files:
4112
- if verbose:
4113
- _verbose_stderr_line("[increment] no changes detected; no-op")
4114
- conn.close()
4115
- db.close()
4116
- return IncrementalResult(
4117
- mode="incremental",
4118
- files_changed=0,
4119
- files_added=0,
4120
- files_removed=0,
4121
- dependents_reprocessed=0,
4122
- elapsed_sec=time.time() - t_start,
4123
- )
4124
-
4125
- if verbose:
4126
- _verbose_stderr_line(f"[increment] detected {len(added)} added, {len(changed)} changed, {len(removed)} removed files")
4127
-
4128
- # Step 2: Crash marker check
4129
- crash_marker_path = index_dir / GRAPH_INCREMENT_MARKER_FILENAME
4130
- if crash_marker_path.exists():
4131
- if verbose:
4132
- _verbose_stderr_line("[increment] crash marker exists; falling back to full rebuild")
4133
- conn.close()
4134
- db.close()
4135
- crash_marker_path.unlink(missing_ok=True)
4136
- return _fallback_to_full(source_root, ladybug_path, verbose, t_start)
4137
-
4138
- # Write crash marker
4139
- crash_marker_path.write_text("", encoding="utf-8")
4140
-
4141
- try:
4142
- # Step 3: Dependent expansion
4143
- # Collect node IDs for changed files (single query instead of N+1)
4144
- changed_node_ids: set[str] = set()
4145
- result = conn.execute(
4146
- "MATCH (s:Symbol) WHERE s.filename IN $filenames RETURN s.id",
4147
- {"filenames": list(changed_files)},
4148
- )
4149
- while result.has_next():
4150
- row = result.get_next()
4151
- changed_node_ids.add(row[0])
4152
-
4153
- # Find dependents
4154
- dependent_files = _find_dependents(conn, changed_node_ids)
4155
-
4156
- # Annotation-definition change: also pull in files that USE the changed
4157
- # annotation. Annotation usage is a node property, not a Symbol->Symbol
4158
- # edge, so `_find_dependents` misses them and their role (derived from
4159
- # the project-wide meta-chain) would go stale. See PR-P5b.
4160
- dependent_files |= _find_annotation_dependents(conn, changed_node_ids)
4161
-
4162
- # Union changed files with dependents
4163
- scope_files = changed_files | dependent_files
4164
-
4165
- if len(scope_files) > expansion_cap:
4166
- if verbose:
4167
- _verbose_stderr_line(f"[increment] dependent expansion cap ({expansion_cap}) exceeded ({len(scope_files)} files); falling back to full rebuild")
4168
- conn.close()
4169
- db.close()
4170
- crash_marker_path.unlink(missing_ok=True)
4171
- return _fallback_to_full(source_root, ladybug_path, verbose, t_start)
4172
-
4173
- if verbose:
4174
- _verbose_stderr_line(f"[increment] processing {len(scope_files)} files ({len(changed_files)} changed + {len(dependent_files)} dependents)")
4175
-
4176
- # Step 4: Scoped deletion
4177
- if verbose:
4178
- _verbose_stderr_line("[increment] deleting outdated nodes and edges")
4179
- _delete_file_scope(conn, changed_files, dependent_files)
4180
-
4181
- # Force deletion to be applied by running a dummy query
4182
- conn.execute("MATCH (s:Symbol) RETURN count(*)")
4183
-
4184
- # Step 5: Scoped pass 1-4
4185
- if verbose:
4186
- _verbose_stderr_line("[increment] rebuilding scoped files (passes 1-4)")
4187
-
4188
- tables = GraphTables()
4189
- asts = pass1_parse(
4190
- source_root, tables, verbose=verbose, scope_files=scope_files, removed_files=removed
4191
- )
4192
-
4193
- # Load existing types and members for cross-file resolution (only from unchanged files)
4194
- _load_existing_types(conn, tables, exclude_files=scope_files)
4195
- _load_existing_members(conn, tables, exclude_files=scope_files)
4196
-
4197
- pass2_edges(tables, asts, verbose=verbose)
4198
- pass3_calls(tables, asts, verbose=verbose)
4199
- pass4_routes(tables, asts, source_root=source_root, verbose=verbose)
4200
-
4201
- # Populate declares and overrides rows
4202
- _populate_declares_rows(tables)
4203
- _populate_overrides_rows(tables)
4204
-
4205
- # Write scoped nodes and edges
4206
- meta_chain = collect_annotation_meta_chain(str(source_root.resolve()))
4207
- _scoped_write(conn, tables, project_root=source_root, meta_chain=meta_chain)
4208
-
4209
- # Step 6: Global pass 5-6
4210
- if verbose:
4211
- _verbose_stderr_line("[increment] running global passes 5-6")
4212
-
4213
- # Rebuild full tables for global pass 5-6 (pass1 populates members from scratch)
4214
- tables_for_global = GraphTables()
4215
- global_asts = pass1_parse(source_root, tables_for_global, verbose=verbose)
4216
- # pass4 (routes/EXPOSES) must run on the global pass5/6 tables too (issue
4217
- # #352): pass5 links Feign HTTP_CALLS to routes via exposes_rows, and pass6
4218
- # matches against routes_rows. Without pass4 both stay empty and the
4219
- # HTTP_CALLS match outcome drifts from a full rebuild. Mirrors main().
4220
- pass4_routes(tables_for_global, global_asts, source_root=source_root, verbose=verbose)
4221
-
4222
- pass5_imperative_edges(tables_for_global, global_asts, source_root=source_root, verbose=verbose)
4223
-
4224
- # Delete existing Client, Producer, and their edges
4225
- conn.execute("MATCH (c:Client) DETACH DELETE c")
4226
- conn.execute("MATCH (p:Producer) DETACH DELETE p")
4227
-
4228
- pass6_match_edges(tables_for_global, verbose=verbose)
4229
-
4230
- # Write Client, Producer, and cross-service edges
4231
- _write_clients_producers_and_calls(conn, tables_for_global)
4232
-
4233
- # Step 7: Update hash store and metadata
4234
- if verbose:
4235
- _verbose_stderr_line("[increment] updating hash store and metadata")
4236
-
4237
- # Update hashes for processed files
4238
- tracker.update(scope_files, source_root)
4239
-
4240
- # Remove hashes for deleted files
4241
- for filename in removed:
4242
- if filename in tracker._hashes:
4243
- del tracker._hashes[filename]
4244
-
4245
- tracker.save()
4246
-
4247
- # Update GraphMeta
4248
- _write_meta(conn, tables_for_global, source_root)
4249
- _ensure_symbol_fts_index(conn, verbose=verbose)
4250
-
4251
- # Remove crash marker
4252
- crash_marker_path.unlink(missing_ok=True)
4253
-
4254
- conn.close()
4255
- db.close()
4256
-
4257
- elapsed = time.time() - t_start
4258
- if verbose:
4259
- _verbose_stderr_line(f"[increment] completed in {elapsed:.2f}s")
4260
-
4261
- return IncrementalResult(
4262
- mode="incremental",
4263
- files_changed=len(changed),
4264
- files_added=len(added),
4265
- files_removed=len(removed),
4266
- dependents_reprocessed=len(dependent_files),
4267
- elapsed_sec=elapsed,
4268
- )
4269
-
4270
- except Exception as e:
4271
- # On error, remove crash marker and fall back to full rebuild
4272
- if verbose:
4273
- _verbose_stderr_line(f"[increment] error during incremental rebuild: {e}; falling back to full rebuild")
4274
- conn.close()
4275
- db.close()
4276
- crash_marker_path.unlink(missing_ok=True)
4277
- return _fallback_to_full(source_root, ladybug_path, verbose, t_start)
4278
-
4279
-
4280
- def _init_hash_tracker(source_root: Path, ladybug_path: Path) -> int:
4281
- """Initialize hash tracker for all Java files. Returns number of files hashed.
4282
-
4283
- Called right after a full graph rebuild (``write_ladybug``), so the store must
4284
- mirror exactly the files that were just indexed. We deliberately do NOT
4285
- ``load()`` the existing store: ``update`` re-hashes every current file anyway,
4286
- and preserving old entries would leave stale hashes for files that no longer
4287
- exist (deleted or now-ignored). Those ghosts would be re-detected as "removed"
4288
- on every subsequent ``increment``, sustaining an endless full-rebuild loop.
4289
- """
4290
- index_dir = ladybug_path.parent
4291
- tracker = FileHashTracker(index_dir)
4292
- ignore = LayeredIgnore(source_root)
4293
- all_files: set[str] = set()
4294
- source_root_resolved = source_root.resolve()
4295
- for p in iter_source_files(source_root, ignore=ignore):
4296
- p_resolved = p.resolve()
4297
- try:
4298
- rel_path = p_resolved.relative_to(source_root_resolved).as_posix()
4299
- except ValueError:
4300
- rel_path = p.as_posix()
4301
- all_files.add(rel_path)
4302
- tracker.update(all_files, source_root)
4303
- tracker.save()
4304
- return len(all_files)
4305
-
4306
-
4307
- def _fallback_to_full(source_root: Path, ladybug_path: Path, verbose: bool, t_start: float) -> IncrementalResult:
4308
- """Fallback to full rebuild."""
4309
- tables = GraphTables()
4310
- asts = pass1_parse(source_root, tables, verbose=verbose)
4311
- pass2_edges(tables, asts, verbose=verbose)
4312
- pass3_calls(tables, asts, verbose=verbose)
4313
- pass4_routes(tables, asts, source_root=source_root, verbose=verbose)
4314
- pass5_imperative_edges(tables, asts, source_root=source_root, verbose=verbose)
4315
- pass6_match_edges(tables, verbose=verbose)
4316
- write_ladybug(ladybug_path, tables, source_root=source_root, verbose=verbose)
4317
-
4318
- return IncrementalResult(
4319
- mode="full_fallback",
4320
- files_changed=0,
4321
- files_added=0,
4322
- files_removed=0,
4323
- dependents_reprocessed=0,
4324
- elapsed_sec=time.time() - t_start,
4325
- )
4326
-
4327
-
4328
- def _write_clients_producers_and_calls(conn: ladybug.Connection, tables: GraphTables) -> None:
4329
- """Write Route, Client, Producer, and cross-service edges to LadybugDB.
4330
-
4331
- Used by the incremental rebuild's global pass 5-6 step. Writes phantom
4332
- Route nodes (created by pass5 for cross-service calls) that wouldn't
4333
- otherwise exist in LadybugDB.
4334
- """
4335
- # Bulk-write routes, mirroring `_write_nodes_impl`: COPY the rows that are new
4336
- # to the DB, and SET every mutable field on the routes already present (the
4337
- # caller does NOT delete routes, only Clients/Producers — see the global
4338
- # pass 5-6 block). `tables.routes_rows` is the full route set (pass4 routes +
4339
- # pass5 phantom routes), not just phantoms, so the SET keeps existing routes'
4340
- # properties in sync with pass5's cross-service enrichment while the COPY
4341
- # materializes phantoms that have no node yet. Replaces the last per-row
4342
- # graph write (MERGE upsert).
4343
- route_rows = [asdict(row) for row in tables.routes_rows]
4344
- existing_route_ids = _existing_node_ids(conn)
4345
- new_route_rows = [r for r in route_rows if r["id"] not in existing_route_ids]
4346
- _bulk_copy(conn, "Route", _ROUTE_COLUMNS, new_route_rows)
4347
- for r in route_rows:
4348
- if r["id"] in existing_route_ids:
4349
- conn.execute(_SET_ROUTE_BY_ID, r)
4350
-
4351
- # Build node_id lookup for members and types
4352
- member_by_id = {m.node_id: m for m in tables.members}
4353
-
4354
- # Build client_id and producer_id lookups for source_file resolution
4355
- client_by_id = {c.id: c for c in tables.client_rows}
4356
- producer_by_id = {p.id: p for p in tables.producer_rows}
4357
-
4358
- # Stage Client + Producer NODE rows unconditionally. The caller DETACH-DELETEs
4359
- # every Client/Producer node immediately before calling this (see the global
4360
- # pass 5-6 block), so none are in the DB yet and COPY-ing them fresh is safe.
4361
- # Do NOT filter node rows against the existing-id set — that is the EDGE filter
4362
- # pattern mis-applied to nodes, and would drop every node being created (the
4363
- # caller's delete makes the pre-load set empty by construction).
4364
- client_rows = [asdict(row) for row in tables.client_rows]
4365
- _bulk_copy(conn, "Client", _CLIENT_COLUMNS, client_rows)
4366
- producer_rows = [asdict(row) for row in tables.producer_rows]
4367
- _bulk_copy(conn, "Producer", _PRODUCER_COLUMNS, producer_rows)
4368
-
4369
- # Endpoint filtering applies to EDGES only: re-fetch the live node set (now
4370
- # includes the Clients/Producers just loaded) and drop edge rows whose
4371
- # endpoints still aren't materialized — reproduces per-row MERGE silent-drop.
4372
- valid_ids = _existing_node_ids(conn)
4373
-
4374
- # Stage DECLARES_CLIENT edge rows (Symbol -> Client)
4375
- # Build source_file lookup from member_by_id
4376
- declares_client_rows = [
4377
- {
4378
- "FROM": row.symbol_id,
4379
- "TO": row.client_id,
4380
- "source_file": member_by_id.get(row.symbol_id, MemberEntry(kind="", decl=None, parent_id="", parent_fqn="", file_path="", module="", microservice="", node_id="")).file_path,
4381
- "confidence": row.confidence,
4382
- "strategy": row.strategy,
4383
- }
4384
- for row in tables.declares_client_rows
4385
- if row.symbol_id in valid_ids and row.client_id in valid_ids
4386
- ]
4387
- _bulk_copy(conn, "DECLARES_CLIENT", _REL_DECLARES_CLIENT_COLUMNS, declares_client_rows)
4388
-
4389
- # Stage DECLARES_PRODUCER edge rows (Symbol -> Producer)
4390
- declares_producer_rows = [
4391
- {
4392
- "FROM": row.symbol_id,
4393
- "TO": row.producer_id,
4394
- "source_file": member_by_id.get(row.symbol_id, MemberEntry(kind="", decl=None, parent_id="", parent_fqn="", file_path="", module="", microservice="", node_id="")).file_path,
4395
- "confidence": row.confidence,
4396
- "strategy": row.strategy,
4397
- }
4398
- for row in tables.declares_producer_rows
4399
- if row.symbol_id in valid_ids and row.producer_id in valid_ids
4400
- ]
4401
- _bulk_copy(conn, "DECLARES_PRODUCER", _REL_DECLARES_PRODUCER_COLUMNS, declares_producer_rows)
4402
-
4403
- # Stage HTTP_CALLS edge rows (Client -> Route)
4404
- http_call_rows = [
4405
- {
4406
- "FROM": row.client_id,
4407
- "TO": row.route_id,
4408
- "source_file": client_by_id.get(row.client_id, ClientRow(id="", client_kind="", target_service="", path="", path_template="", path_regex="", method="", member_fqn="", member_id="", microservice="", module="", filename="", start_line=0, end_line=0, resolved=False, source_layer="")).filename,
4409
- "confidence": row.confidence,
4410
- "strategy": row.strategy,
4411
- "method_call": row.method_call,
4412
- "raw_uri": row.raw_uri,
4413
- "match": row.match,
4414
- }
4415
- for row in tables.http_call_rows
4416
- if row.client_id in valid_ids and row.route_id in valid_ids
4417
- ]
4418
- _bulk_copy(conn, "HTTP_CALLS", _REL_HTTP_CALLS_COLUMNS, http_call_rows)
4419
-
4420
- # Stage ASYNC_CALLS edge rows (Producer -> Route)
4421
- async_call_rows = [
4422
- {
4423
- "FROM": row.producer_id,
4424
- "TO": row.route_id,
4425
- "source_file": producer_by_id.get(row.producer_id, ProducerRow(id="", producer_kind="", topic="", broker="", direction="", member_fqn="", member_id="", microservice="", module="", filename="", start_line=0, end_line=0, resolved=False, source_layer="")).filename,
4426
- "confidence": row.confidence,
4427
- "strategy": row.strategy,
4428
- "direction": row.direction,
4429
- "raw_topic": row.raw_topic,
4430
- "match": row.match,
4431
- }
4432
- for row in tables.async_call_rows
4433
- if row.producer_id in valid_ids and row.route_id in valid_ids
4434
- ]
4435
- _bulk_copy(conn, "ASYNC_CALLS", _REL_ASYNC_CALLS_COLUMNS, async_call_rows)
4436
-
4437
-
4438
- def write_ladybug(
4439
- db_path: Path,
4440
- tables: GraphTables,
4441
- *,
4442
- source_root: Path,
4443
- verbose: bool,
4444
- meta_chain: dict[str, frozenset[str]] | None = None,
4445
- ) -> None:
4446
- if meta_chain is None:
4447
- meta_chain = collect_annotation_meta_chain(
4448
- str(source_root.resolve()),
4449
- )
4450
- if verbose:
4451
- _verbose_stderr_line(_WRITE_START)
4452
- with _VerbosePassHeartbeats("[graph] writing", verbose=verbose):
4453
- db_path.parent.mkdir(parents=True, exist_ok=True)
4454
- db = ladybug.Database(str(db_path))
4455
- conn = ladybug.Connection(db)
4456
- _drop_all(conn)
4457
- _create_schema(conn)
4458
- t0 = time.time()
4459
- _write_nodes(
4460
- conn,
4461
- tables,
4462
- project_root=source_root,
4463
- meta_chain=meta_chain,
4464
- )
4465
- if verbose:
4466
- _verbose_stderr_line(f"[graph] writing · nodes written in {time.time() - t0:.2f}s")
4467
- _populate_declares_rows(tables)
4468
- _populate_overrides_rows(tables)
4469
- t1 = time.time()
4470
- _fbyid = _build_file_by_node_id(tables)
4471
- _write_edges(conn, tables, _fbyid)
4472
- if verbose:
4473
- _verbose_stderr_line(f"[graph] writing · edges written in {time.time() - t1:.2f}s")
4474
- t2 = time.time()
4475
- _write_routes_and_exposes(conn, tables, _fbyid)
4476
- if verbose:
4477
- _verbose_stderr_line(f"[graph] writing · routes/exposes written in {time.time() - t2:.2f}s")
4478
- _write_meta(conn, tables, source_root)
4479
- _ensure_symbol_fts_index(conn, verbose=verbose)
4480
- conn.close()
4481
- db.close()
4482
-
4483
- # Build vocabulary index (best-effort, failure doesn't fail the graph build)
4484
- _try_build_vocabulary_index(db_path, source_root, verbose)
4485
- _init_hash_tracker(source_root, db_path)
4486
-
4487
-
4488
- def _try_build_vocabulary_index(db_path: Path, source_root: Path, verbose: bool) -> None:
4489
- """Build and save the vocabulary index as a sidecar (best-effort).
4490
-
4491
- This is called after write_ladybug() completes. A build failure must not
4492
- fail the graph build, so this is wrapped in try/except and logged.
4493
-
4494
- Args:
4495
- db_path: Path to the LadybugDB database file
4496
- source_root: Source repository root
4497
- verbose: Whether to emit verbose progress
4498
- """
4499
- try:
4500
- from java_codebase_rag.absence.absence_vocab import VocabularyIndex, VOCAB_INDEX_FILENAME
4501
- from java_codebase_rag.graph.ladybug_queries import LadybugGraph
4502
-
4503
- t0 = time.time()
4504
- if verbose:
4505
- _verbose_stderr_line("[vocab] building vocabulary index")
4506
-
4507
- # Open graph for reading
4508
- graph = LadybugGraph.get(str(db_path))
4509
-
4510
- # Read q from env var set by ResolvedOperatorConfig.subprocess_env()
4511
- raw_q = os.environ.get("JAVA_CODEBASE_RAG_ABSENCE_NGRAM_Q", "3").strip()
4512
- try:
4513
- q = int(raw_q) if raw_q else 3
4514
- except ValueError:
4515
- q = 3 # Invalid env value falls back to default
4516
- # Build index with configured q (or default 3)
4517
- index = VocabularyIndex.build(graph, q=q)
4518
-
4519
- # Save to sidecar next to the graph db
4520
- sidecar_path = Path(db_path).parent / VOCAB_INDEX_FILENAME
4521
- index.save(sidecar_path, ontology_version=ONTOLOGY_VERSION)
4522
-
4523
- if verbose:
4524
- _verbose_stderr_line(f"[vocab] index built with {index.symbol_count} symbols in {time.time() - t0:.2f}s")
4525
-
4526
- except Exception as e:
4527
- # Log but don't fail - graph build is the primary concern
4528
- log.warning(f"Vocabulary index build failed (non-critical): {e}")
4529
- if verbose:
4530
- _verbose_stderr_line(f"[vocab] build failed (graph still written): {e}")
4531
-
4532
-
4533
- # ---------- CLI ----------
4534
-
4535
-
4536
- def _default_ladybug_path() -> Path:
4537
- idx = os.environ.get("JAVA_CODEBASE_RAG_INDEX_DIR", "").strip()
4538
- if idx and not idx.startswith(("s3://", "gs://", "az://")):
4539
- return Path(os.path.expanduser(idx.rstrip("/"))) / "code_graph.lbug"
4540
- return Path.cwd() / ".java-codebase-rag" / "code_graph.lbug"
4541
-
4542
-
4543
- def main() -> int:
4544
- parser = argparse.ArgumentParser(description="Build an AST-derived LadybugDB graph for Java sources.")
4545
- parser.add_argument("--source-root", default=None, help="Repository / monorepo root to scan for .java (defaults to current working directory)")
4546
- parser.add_argument(
4547
- "--ladybug-path",
4548
- default=None,
4549
- help=(
4550
- "LadybugDB database path (file/dir as used by ladybug.Database; "
4551
- "default: $JAVA_CODEBASE_RAG_INDEX_DIR/code_graph.lbug or ./.java-codebase-rag/code_graph.lbug)"
4552
- ),
4553
- )
4554
- parser.add_argument("--verbose", action="store_true")
4555
- parser.add_argument("--incremental", action="store_true", help="Run incremental rebuild instead of full rebuild")
4556
- args = parser.parse_args()
4557
-
4558
- root = Path(args.source_root).expanduser().resolve() if args.source_root else Path.cwd().resolve()
4559
- if not root.is_dir():
4560
- print(f"source-root not a directory: {root}", file=sys.stderr)
4561
- return 2
4562
-
4563
- ladybug_path = Path(args.ladybug_path).expanduser() if args.ladybug_path else _default_ladybug_path()
4564
-
4565
- if args.incremental:
4566
- result = incremental_rebuild(root, ladybug_path, verbose=args.verbose)
4567
- print(json.dumps({
4568
- "mode": result.mode,
4569
- "files_changed": result.files_changed,
4570
- "files_added": result.files_added,
4571
- "files_removed": result.files_removed,
4572
- "dependents_reprocessed": result.dependents_reprocessed,
4573
- "elapsed_sec": result.elapsed_sec,
4574
- }))
4575
- if args.verbose:
4576
- _verbose_stderr_line(f"[graph] done · mode={result.mode} files_changed={result.files_changed} files_added={result.files_added} files_removed={result.files_removed} dependents={result.dependents_reprocessed} elapsed={result.elapsed_sec:.2f}s")
4577
- return 0
4578
-
4579
- tables = GraphTables()
4580
- asts = pass1_parse(root, tables, verbose=args.verbose)
4581
- pass2_edges(tables, asts, verbose=args.verbose)
4582
- pass3_calls(tables, asts, verbose=args.verbose)
4583
- pass4_routes(tables, asts, source_root=root, verbose=args.verbose)
4584
- pass5_imperative_edges(tables, asts, source_root=root, verbose=args.verbose)
4585
- pass6_match_edges(tables, verbose=args.verbose)
4586
- write_ladybug(ladybug_path, tables, source_root=root, verbose=args.verbose)
4587
- if args.verbose:
4588
- _verbose_stderr_line(f"[graph] done · ladybug at {ladybug_path}")
4589
- return 0
4590
-
4591
-
4592
- if __name__ == "__main__":
4593
- sys.exit(main())