java-codebase-rag 0.11.2__py3-none-any.whl → 0.12.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. java_codebase_rag-0.12.1.dist-info/METADATA +35 -0
  2. java_codebase_rag-0.12.1.dist-info/RECORD +4 -0
  3. {java_codebase_rag-0.11.2.dist-info → java_codebase_rag-0.12.1.dist-info}/WHEEL +1 -1
  4. java_codebase_rag/_fdlimit.py +0 -56
  5. java_codebase_rag/_stdio.py +0 -32
  6. java_codebase_rag/_version.py +0 -35
  7. java_codebase_rag/absence/__init__.py +0 -0
  8. java_codebase_rag/absence/absence_diagnosis.py +0 -700
  9. java_codebase_rag/absence/absence_types.py +0 -124
  10. java_codebase_rag/absence/absence_vocab.py +0 -460
  11. java_codebase_rag/analysis/__init__.py +0 -0
  12. java_codebase_rag/analysis/pr_analysis.py +0 -563
  13. java_codebase_rag/analysis/resolve_service.py +0 -740
  14. java_codebase_rag/ast/__init__.py +0 -0
  15. java_codebase_rag/ast/ast_java.py +0 -2825
  16. java_codebase_rag/ast/brownfield_events.py +0 -58
  17. java_codebase_rag/ast/chunk_heuristics.py +0 -62
  18. java_codebase_rag/cli.py +0 -1215
  19. java_codebase_rag/cli_format.py +0 -85
  20. java_codebase_rag/cli_progress.py +0 -94
  21. java_codebase_rag/config.py +0 -833
  22. java_codebase_rag/eval/__init__.py +0 -1
  23. java_codebase_rag/eval/ground_truth.py +0 -100
  24. java_codebase_rag/eval/metrics.py +0 -107
  25. java_codebase_rag/eval/runner.py +0 -556
  26. java_codebase_rag/graph/__init__.py +0 -0
  27. java_codebase_rag/graph/build_ast_graph.py +0 -4471
  28. java_codebase_rag/graph/graph_enrich.py +0 -1937
  29. java_codebase_rag/graph/graph_types.py +0 -224
  30. java_codebase_rag/graph/java_ontology.py +0 -465
  31. java_codebase_rag/graph/ladybug_queries.py +0 -2213
  32. java_codebase_rag/graph/path_filtering.py +0 -477
  33. java_codebase_rag/index/__init__.py +0 -0
  34. java_codebase_rag/index/java_index_flow_lancedb.py +0 -734
  35. java_codebase_rag/index/java_index_v1_common.py +0 -33
  36. java_codebase_rag/install_data/__init__.py +0 -0
  37. java_codebase_rag/install_data/agents/explorer-rag-cli.md +0 -108
  38. java_codebase_rag/install_data/agents/explorer-rag-enhanced.md +0 -152
  39. java_codebase_rag/install_data/skills/explore-codebase/SKILL.md +0 -165
  40. java_codebase_rag/install_data/skills/explore-codebase-cli/SKILL.md +0 -107
  41. java_codebase_rag/installer.py +0 -2188
  42. java_codebase_rag/jrag.py +0 -4531
  43. java_codebase_rag/jrag_envelope.py +0 -1107
  44. java_codebase_rag/jrag_hints.py +0 -204
  45. java_codebase_rag/jrag_render.py +0 -926
  46. java_codebase_rag/lance_optimize.py +0 -264
  47. java_codebase_rag/mcp/__init__.py +0 -0
  48. java_codebase_rag/mcp/mcp_hints.py +0 -932
  49. java_codebase_rag/mcp/mcp_v2.py +0 -1916
  50. java_codebase_rag/mcp/server.py +0 -884
  51. java_codebase_rag/pipeline.py +0 -531
  52. java_codebase_rag/progress.py +0 -570
  53. java_codebase_rag/read_payloads.py +0 -781
  54. java_codebase_rag/search/__init__.py +0 -0
  55. java_codebase_rag/search/index_common.py +0 -10
  56. java_codebase_rag/search/search_lancedb.py +0 -1296
  57. java_codebase_rag/search/search_lexical.py +0 -449
  58. java_codebase_rag/search/search_scoring.py +0 -523
  59. java_codebase_rag/watch/__init__.py +0 -0
  60. java_codebase_rag/watch/client.py +0 -230
  61. java_codebase_rag/watch/daemon.py +0 -396
  62. java_codebase_rag/watch/lock.py +0 -201
  63. java_codebase_rag/watch/paths.py +0 -76
  64. java_codebase_rag/watch/protocol.py +0 -122
  65. java_codebase_rag/watch/server.py +0 -273
  66. java_codebase_rag/watch/warm.py +0 -105
  67. java_codebase_rag/watch/watcher.py +0 -370
  68. java_codebase_rag-0.11.2.dist-info/METADATA +0 -331
  69. java_codebase_rag-0.11.2.dist-info/RECORD +0 -71
  70. java_codebase_rag-0.11.2.dist-info/entry_points.txt +0 -4
  71. java_codebase_rag-0.11.2.dist-info/licenses/LICENSE +0 -21
  72. java_codebase_rag-0.11.2.dist-info/top_level.txt +0 -1
  73. /java_codebase_rag/__init__.py → /java_codebase_rag-0.12.1.dist-info/top_level.txt +0 -0
@@ -1,4471 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Four-pass AST-derived Knowledge Base builder (LadybugDB).
3
-
4
- Walks a Java source tree with `tree_sitter_java`, writes a deterministic graph of:
5
- Symbol nodes: package, file, class, interface, enum, record, annotation, method, constructor
6
- Route nodes: declaration-site routes (Spring MVC/WebFlux, Feign, Kafka, …)
7
- Rel tables: EXTENDS, IMPLEMENTS, INJECTS, DECLARES, OVERRIDES, CALLS, EXPOSES
8
-
9
- Pass 1 builds every node and in-memory resolution indexes.
10
- Pass 2 resolves each extends/implements/injection target using Java's lookup order
11
- (same file → explicit import → same package → wildcard import → java.lang → phantom).
12
- Pass 3 resolves static call sites into confidence-scored CALLS edges and DECLARES.
13
- Pass 4 emits Route rows plus Symbol→Route EXPOSES edges from literal annotation metadata.
14
-
15
- Usage:
16
- build_ast_graph.py --source-root <repo> [--ladybug-path <path>] [--verbose]
17
-
18
- Default LadybugDB database path resolution order:
19
- --ladybug-path CLI arg (path passed to ladybug.Database(...))
20
- JAVA_CODEBASE_RAG_INDEX_DIR/code_graph.lbug (if set and local)
21
- ./.java-codebase-rag/code_graph.lbug under cwd
22
-
23
- The LadybugDB DB is dropped and rebuilt on every run (Phase 1 is a full rebuild).
24
- """
25
- from __future__ import annotations
26
-
27
- import argparse
28
- import contextlib
29
- import hashlib
30
- import json
31
- import logging
32
- import os
33
- import re
34
- import sys
35
- import threading
36
- import time
37
- from collections import defaultdict
38
- from dataclasses import asdict, dataclass, field, replace
39
- from pathlib import Path
40
-
41
- import ladybug
42
- import pyarrow as pa
43
-
44
- from java_codebase_rag.ast.ast_java import (
45
- ONTOLOGY_VERSION,
46
- CallSite,
47
- JavaFileAst,
48
- MethodDecl,
49
- OutgoingCallDecl,
50
- TypeDecl,
51
- injection_annotation_names,
52
- lombok_required_args_annotations,
53
- parse_java,
54
- )
55
- from java_codebase_rag.graph.graph_enrich import (
56
- _load_config_cross_service_resolution,
57
- classify_java_file,
58
- collect_annotation_meta_chain,
59
- load_brownfield_overrides,
60
- load_generated_detection,
61
- microservice_for_path,
62
- module_for_path,
63
- phantom_id,
64
- resolve_async_producer_for_method,
65
- resolve_http_client_for_method,
66
- resolve_role_and_capabilities,
67
- resolve_routes_for_method,
68
- symbol_id,
69
- )
70
- from java_codebase_rag.graph.path_filtering import LayeredIgnore, iter_java_source_files
71
- from java_codebase_rag.search.search_scoring import SYMBOL_FTS_INDEX as _SYMBOL_FTS_INDEX, _split_identifier
72
- from java_codebase_rag.graph.java_ontology import (
73
- CLIENT_KIND_FEIGN_METHOD,
74
- CLIENT_KIND_REST_TEMPLATE,
75
- VALID_CLIENT_KINDS,
76
- VALID_HTTP_CALL_MATCHES,
77
- VALID_PRODUCER_KINDS,
78
- )
79
-
80
- log = logging.getLogger(__name__)
81
-
82
- _VERBOSE_STDERR_LOCK = threading.Lock()
83
-
84
- _PASS1_START = "[graph] pass 1 · parsing Java files"
85
- _PASS2_START = "[graph] pass 2 · emitting EXTENDS / IMPLEMENTS / DECLARES rows"
86
- _PASS3_START = "[graph] pass 3 · call resolution (outgoing calls per site)"
87
- _PASS4_START = "[graph] pass 4 · route and EXPOSES extraction"
88
- _PASS5_START = "[graph] pass 5 · imperative HTTP_CALLS / ASYNC_CALLS edges"
89
- _PASS6_START = "[graph] pass 6 · cross-service call-edge matching"
90
- _WRITE_START = "[graph] writing · LadybugDB graph to disk"
91
-
92
-
93
- def _verbose_stderr_line(content: str) -> None:
94
- with _VERBOSE_STDERR_LOCK:
95
- print(content, file=sys.stderr, flush=True)
96
-
97
-
98
- def _emit_graph_progress(parts: dict[str, object], *, verbose: bool) -> None:
99
- """Emit one ``JCIRAG_PROGRESS kind=graph …`` line to stderr (gated by verbose).
100
-
101
- The parent process (``pipeline.run_build_ast_graph`` /
102
- ``run_incremental_graph``) passes ``--verbose`` in default AND verbose modes
103
- (only suppressed for ``--quiet``), so this structured progress surfaces in
104
- default mode (where the parent renders it) and verbose mode (raw relay). In
105
- ``--quiet`` the builder is never invoked with ``--verbose`` so nothing is
106
- emitted. Field order is fixed so the parser and tests can pin substrings.
107
- """
108
- if not verbose:
109
- return
110
- fields = ["kind=graph"]
111
- for key in ("pass", "done", "total", "status", "elapsed_s"):
112
- if key in parts:
113
- fields.append(f"{key}={parts[key]}")
114
- line = "JCIRAG_PROGRESS " + " ".join(fields)
115
- _verbose_stderr_line(line)
116
-
117
-
118
- # Pass-1 per-file tick cadence: bound stderr volume on huge trees without making
119
- # the bar feel stale. A final tick on pass completion carries status=done.
120
- _PASS1_TICK_EVERY = 25
121
-
122
-
123
- @contextlib.contextmanager
124
- def _graph_pass_progress(pass_label: str, *, verbose: bool):
125
- """Emit ``pass=N/6 status=running`` on entry and ``status=done elapsed_s=…``
126
- on exit for passes 2–6 (each advances the rendered bar by 1/6).
127
-
128
- Usage: ``with _graph_pass_progress("2/6", verbose=verbose): …``
129
- """
130
- if not verbose:
131
- yield
132
- return
133
- _emit_graph_progress({"pass": pass_label, "status": "running"}, verbose=verbose)
134
- t0 = time.time()
135
- try:
136
- yield
137
- finally:
138
- elapsed = time.time() - t0
139
- _emit_graph_progress(
140
- {"pass": pass_label, "status": "done", "elapsed_s": f"{elapsed:.2f}"},
141
- verbose=verbose,
142
- )
143
-
144
-
145
- class _VerbosePassHeartbeats:
146
- """Emit ``[tag] running … Ns elapsed`` every 5s on stderr while in scope (verbose only)."""
147
-
148
- def __init__(self, tag: str, *, verbose: bool) -> None:
149
- self._tag = tag
150
- self._verbose = verbose
151
- self._thr: threading.Thread | None = None
152
- self._stop: threading.Event | None = None
153
-
154
- def __enter__(self) -> None:
155
- if not self._verbose:
156
- return None
157
- self._stop = threading.Event()
158
- stop = self._stop
159
- tag = self._tag
160
-
161
- def worker() -> None:
162
- t0 = time.monotonic()
163
- while not stop.wait(timeout=5.0):
164
- elapsed = int(time.monotonic() - t0)
165
- _verbose_stderr_line(f"{tag} · {elapsed}s elapsed")
166
-
167
- self._thr = threading.Thread(target=worker, name=f"hb-{tag}", daemon=True)
168
- self._thr.start()
169
- return None
170
-
171
- def __exit__(self, exc_type, exc, tb) -> bool:
172
- if self._thr is not None and self._stop is not None:
173
- self._stop.set()
174
- self._thr.join(timeout=2.0)
175
- return False
176
-
177
-
178
- _JAVA_LANG_SIMPLE = frozenset({
179
- "Object", "String", "Integer", "Long", "Short", "Byte", "Boolean", "Double",
180
- "Float", "Character", "Number", "Void", "Class", "Enum", "Record",
181
- "Throwable", "Exception", "RuntimeException", "Error", "Thread", "Runnable",
182
- "Iterable", "Comparable", "CharSequence", "StringBuilder", "StringBuffer",
183
- "Math", "System", "AutoCloseable", "Cloneable",
184
- })
185
-
186
-
187
- # ---------- dataclasses ----------
188
-
189
-
190
- @dataclass
191
- class TypeIndexEntry:
192
- """Pass-1 record for a type declaration + any methods/constructors inside it."""
193
- decl: TypeDecl
194
- file_path: str
195
- module: str
196
- microservice: str
197
- package: str
198
- outer_fqn: str | None
199
- node_id: str
200
- # True when this entry was loaded from the existing graph by
201
- # `_load_existing_types` (an unchanged-file stub used only for cross-file
202
- # resolution). Its `decl` is a placeholder (no annotations/methods), so its
203
- # recomputed role/capabilities must never be written back over the real
204
- # stored values. See `_write_nodes_impl`.
205
- loaded_from_db: bool = False
206
-
207
-
208
- @dataclass
209
- class MemberEntry:
210
- kind: str # method | constructor
211
- decl: MethodDecl
212
- parent_id: str
213
- parent_fqn: str
214
- file_path: str
215
- module: str
216
- microservice: str
217
- node_id: str
218
- # True when loaded from the existing graph by `_load_existing_members`
219
- # (an unchanged-file stub used only for cross-file call resolution). Its
220
- # DECLARES edge already persists in the graph, so it must not be re-emitted
221
- # by `_populate_declares_rows` (REL tables have no PK → would duplicate).
222
- loaded_from_db: bool = False
223
-
224
-
225
- @dataclass
226
- class EdgeRow:
227
- src_id: str
228
- dst_id: str
229
- dst_name: str
230
- dst_fqn: str
231
- resolved: bool
232
-
233
-
234
- @dataclass
235
- class InjectsRow(EdgeRow):
236
- mechanism: str = ""
237
- annotation: str = ""
238
- field_or_param: str = ""
239
-
240
-
241
- @dataclass
242
- class CallsRow:
243
- src_id: str
244
- dst_id: str
245
- call_site_line: int = 0
246
- call_site_byte: int = 0
247
- arg_count: int = 0
248
- confidence: float = 0.0
249
- strategy: str = "phantom"
250
- source: str = "static"
251
- resolved: bool = True
252
- callee_declaring_role: str = "OTHER"
253
-
254
-
255
- @dataclass
256
- class UnresolvedCallSiteRow:
257
- id: str
258
- caller_id: str
259
- call_site_line: int
260
- call_site_byte: int
261
- arg_count: int
262
- callee_simple: str
263
- receiver_expr: str
264
- reason: str
265
-
266
-
267
- @dataclass
268
- class DeclaresRow:
269
- src_id: str
270
- dst_id: str
271
-
272
-
273
- @dataclass
274
- class CallResolutionStats:
275
- total: int = 0
276
- by_strategy: dict[str, int] = field(default_factory=lambda: defaultdict(int))
277
- phantom_chained: int = 0
278
- phantom_other: int = 0
279
- callee_unresolved: int = 0
280
- skipped_cross_service: int = 0
281
-
282
-
283
- @dataclass
284
- class RouteRow:
285
- id: str
286
- kind: str
287
- framework: str
288
- method: str
289
- path: str
290
- path_template: str
291
- path_regex: str
292
- topic: str
293
- broker: str
294
- feign_name: str
295
- feign_url: str
296
- microservice: str
297
- module: str
298
- filename: str
299
- start_line: int
300
- end_line: int
301
- resolved: bool
302
- # B2a brownfield composition (PR-A3); not persisted on LadybugDB `Route` nodes.
303
- source_layer: str = "builtin"
304
-
305
-
306
- @dataclass
307
- class ExposesRow:
308
- symbol_id: str
309
- route_id: str
310
- confidence: float
311
- strategy: str
312
-
313
-
314
- @dataclass
315
- class RouteExtractionStats:
316
- routes_skipped_unresolved: int = 0
317
- by_framework: dict[str, int] = field(default_factory=lambda: defaultdict(int))
318
- by_kind: dict[str, int] = field(default_factory=lambda: defaultdict(int))
319
- routes_resolved_pct: float = 100.0
320
- # Percentage of emitted `Route` rows whose `source_layer` is not `builtin`.
321
- # Brownfield layers: `layer_b_ann`, `layer_a_meta`, `layer_c_source`, `layer_b_fqn`.
322
- routes_from_brownfield_pct: float = 0.0
323
- routes_by_layer: dict[str, int] = field(default_factory=dict)
324
- exposes_suppressed_feign: int = 0
325
-
326
-
327
- @dataclass
328
- class HttpCallRow:
329
- client_id: str
330
- route_id: str
331
- confidence: float
332
- strategy: str
333
- method_call: str
334
- raw_uri: str
335
- match: str
336
-
337
-
338
- @dataclass
339
- class AsyncCallRow:
340
- producer_id: str
341
- route_id: str
342
- confidence: float
343
- strategy: str
344
- direction: str
345
- raw_topic: str
346
- match: str
347
-
348
-
349
- @dataclass
350
- class ClientRow:
351
- id: str
352
- client_kind: str
353
- target_service: str
354
- path: str
355
- path_template: str
356
- path_regex: str
357
- method: str
358
- member_fqn: str
359
- member_id: str
360
- microservice: str
361
- module: str
362
- filename: str
363
- start_line: int
364
- end_line: int
365
- resolved: bool
366
- source_layer: str
367
-
368
-
369
- @dataclass
370
- class DeclaresClientRow:
371
- symbol_id: str
372
- client_id: str
373
- confidence: float
374
- strategy: str
375
-
376
-
377
- @dataclass
378
- class ProducerRow:
379
- id: str
380
- producer_kind: str
381
- topic: str
382
- broker: str
383
- direction: str
384
- member_fqn: str
385
- member_id: str
386
- microservice: str
387
- module: str
388
- filename: str
389
- start_line: int
390
- end_line: int
391
- resolved: bool
392
- source_layer: str
393
-
394
-
395
- @dataclass
396
- class DeclaresProducerRow:
397
- symbol_id: str
398
- producer_id: str
399
- confidence: float
400
- strategy: str
401
-
402
-
403
- @dataclass
404
- class ClientExtractionStats:
405
- clients_total: int = 0
406
- declares_client_total: int = 0
407
- clients_by_kind: dict[str, int] = field(default_factory=lambda: defaultdict(int))
408
-
409
-
410
- @dataclass
411
- class ProducerExtractionStats:
412
- producers_total: int = 0
413
- declares_producer_total: int = 0
414
- producers_by_kind: dict[str, int] = field(default_factory=lambda: defaultdict(int))
415
-
416
-
417
- @dataclass
418
- class CallEdgeStats:
419
- http_calls_total: int = 0
420
- async_calls_total: int = 0
421
- http_calls_by_client_kind: dict[str, int] = field(default_factory=lambda: defaultdict(int))
422
- async_calls_by_client_kind: dict[str, int] = field(default_factory=lambda: defaultdict(int))
423
- http_calls_by_strategy: dict[str, int] = field(default_factory=lambda: defaultdict(int))
424
- async_calls_by_strategy: dict[str, int] = field(default_factory=lambda: defaultdict(int))
425
- http_calls_skipped_unresolved: int = 0
426
- async_calls_skipped_unresolved: int = 0
427
- http_clients_from_brownfield_pct: float = 0.0
428
- async_producers_from_brownfield_pct: float = 0.0
429
- http_calls_match_breakdown: dict[str, int] = field(default_factory=lambda: defaultdict(int))
430
- async_calls_match_breakdown: dict[str, int] = field(default_factory=lambda: defaultdict(int))
431
- cross_service_calls_total: int = 0
432
-
433
-
434
- @dataclass
435
- class GraphTables:
436
- types: dict[str, TypeIndexEntry] = field(default_factory=dict) # fqn -> entry
437
- by_simple_name: dict[str, list[TypeIndexEntry]] = field(default_factory=dict)
438
- by_package: dict[str, list[TypeIndexEntry]] = field(default_factory=dict)
439
- files: dict[str, str] = field(default_factory=dict) # path -> node id
440
- packages: dict[str, str] = field(default_factory=dict) # pkg -> node id
441
- members: list[MemberEntry] = field(default_factory=list)
442
- phantoms: dict[str, dict] = field(default_factory=dict) # id -> row
443
- extends_rows: list[EdgeRow] = field(default_factory=list)
444
- implements_rows: list[EdgeRow] = field(default_factory=list)
445
- injects_rows: list[InjectsRow] = field(default_factory=list)
446
- calls_rows: list[CallsRow] = field(default_factory=list)
447
- unresolved_call_site_rows: list[UnresolvedCallSiteRow] = field(default_factory=list)
448
- declares_rows: list[DeclaresRow] = field(default_factory=list)
449
- routes_rows: list[RouteRow] = field(default_factory=list)
450
- exposes_rows: list[ExposesRow] = field(default_factory=list)
451
- http_call_rows: list[HttpCallRow] = field(default_factory=list)
452
- async_call_rows: list[AsyncCallRow] = field(default_factory=list)
453
- client_rows: list[ClientRow] = field(default_factory=list)
454
- declares_client_rows: list[DeclaresClientRow] = field(default_factory=list)
455
- producer_rows: list[ProducerRow] = field(default_factory=list)
456
- declares_producer_rows: list[DeclaresProducerRow] = field(default_factory=list)
457
- overrides_rows: list[DeclaresRow] = field(default_factory=list)
458
- route_stats: RouteExtractionStats = field(default_factory=RouteExtractionStats)
459
- call_edge_stats: CallEdgeStats = field(default_factory=CallEdgeStats)
460
- client_stats: ClientExtractionStats = field(default_factory=ClientExtractionStats)
461
- producer_stats: ProducerExtractionStats = field(default_factory=ProducerExtractionStats)
462
- methods_by_type: dict[str, list[MemberEntry]] = field(default_factory=dict)
463
- parse_errors: int = 0
464
- skipped_files: int = 0
465
- pass3_skipped_cross_service: int = 0
466
- pass3_unresolved_phantom_receiver: int = 0
467
- pass3_unresolved_chained: int = 0
468
- cross_service_resolution: str = "auto"
469
- # Populated in _write_nodes (same overrides + meta_chain as Symbol.role).
470
- type_role_by_node_id: dict[str, str] = field(default_factory=dict)
471
- # Populated in pass 1 (classify_java_file) and _load_existing_types for incremental rebuilds.
472
- type_generated_by_node_id: dict[str, tuple[bool, str | None]] = field(default_factory=dict)
473
-
474
-
475
- @dataclass
476
- class IncrementalResult:
477
- """Result of an incremental graph rebuild."""
478
- mode: str # "incremental" | "full_fallback"
479
- files_changed: int
480
- files_added: int
481
- files_removed: int
482
- dependents_reprocessed: int
483
- elapsed_sec: float
484
-
485
-
486
- # --- Builder-owned files in the index dir (single source of truth) ---------------
487
- # Every artifact the graph builder writes next to code_graph.lbug. The lifecycle
488
- # CLI's `erase` clears all of these from one list so the builder and erase cannot
489
- # drift (issues #349 / #350): previously erase hardcoded ".graph_hashes.json" only
490
- # and left the crash marker (.graph_increment_in_progress) and the atomic-write
491
- # temp (.graph_hashes.json.tmp) behind on disk.
492
- GRAPH_HASHES_FILENAME = ".graph_hashes.json"
493
- GRAPH_HASHES_TMP_FILENAME = ".graph_hashes.json.tmp"
494
- GRAPH_INCREMENT_MARKER_FILENAME = ".graph_increment_in_progress"
495
- BUILDER_OWNED_INDEX_FILES: tuple[str, ...] = (
496
- GRAPH_HASHES_FILENAME,
497
- GRAPH_HASHES_TMP_FILENAME,
498
- GRAPH_INCREMENT_MARKER_FILENAME,
499
- )
500
-
501
-
502
- class FileHashTracker:
503
- """Track content hashes for incremental graph rebuild."""
504
- def __init__(self, index_dir: Path):
505
- self._path = index_dir / GRAPH_HASHES_FILENAME
506
- self._hashes: dict[str, str] = {} # rel_path -> sha256_hex
507
-
508
- def load(self) -> None:
509
- """Load hashes from disk. No-op if file missing (first run)."""
510
- if not self._path.exists():
511
- return
512
- try:
513
- with open(self._path, "r", encoding="utf-8") as f:
514
- self._hashes = json.load(f)
515
- except (json.JSONDecodeError, OSError):
516
- # Corrupt or unreadable hash file; start fresh.
517
- self._hashes = {}
518
-
519
- def save(self) -> None:
520
- """Persist hashes to disk atomically (write .tmp, rename)."""
521
- tmp_path = self._path.parent / GRAPH_HASHES_TMP_FILENAME
522
- try:
523
- with open(tmp_path, "w", encoding="utf-8") as f:
524
- json.dump(self._hashes, f, sort_keys=True)
525
- os.replace(tmp_path, self._path)
526
- except OSError as e:
527
- # Fail gracefully; next run will treat as missing and rebuild.
528
- log.warning("Failed to save hash file %s: %s; next run will rebuild from scratch", self._path, e)
529
-
530
- def detect_changes(self, source_root: Path, ignore: LayeredIgnore) -> tuple[set[str], set[str], set[str]]:
531
- """Return (added, changed, removed) sets of relative POSIX paths."""
532
- current_files: set[str] = set()
533
- # Resolve source_root to handle symlinks
534
- source_root_resolved = source_root.resolve()
535
- for abs_path in iter_java_source_files(source_root, ignore=ignore):
536
- # Resolve the absolute path and compute relative path
537
- abs_path_resolved = abs_path.resolve()
538
- try:
539
- rel_path = abs_path_resolved.relative_to(source_root_resolved).as_posix()
540
- except ValueError:
541
- # Fallback to using the path as-is if it's not under source_root
542
- rel_path = abs_path.as_posix()
543
- current_files.add(rel_path)
544
-
545
- added: set[str] = set()
546
- changed: set[str] = set()
547
- removed: set[str] = set()
548
-
549
- # Detect added and changed files.
550
- for rel_path in current_files:
551
- abs_path = source_root / rel_path
552
- try:
553
- file_hash = _hash_file(abs_path)
554
- except FileNotFoundError:
555
- continue
556
- stored_hash = self._hashes.get(rel_path)
557
- if stored_hash is None:
558
- added.add(rel_path)
559
- elif stored_hash != file_hash:
560
- changed.add(rel_path)
561
-
562
- # Detect removed files.
563
- for rel_path in self._hashes:
564
- if rel_path not in current_files:
565
- removed.add(rel_path)
566
-
567
- return added, changed, removed
568
-
569
- def update(self, rel_paths: set[str], source_root: Path) -> None:
570
- """Compute and store hashes for the given paths."""
571
- for rel_path in rel_paths:
572
- abs_path = source_root / rel_path
573
- if abs_path.exists():
574
- self._hashes[rel_path] = _hash_file(abs_path)
575
-
576
-
577
- def _hash_file(abs_path: Path) -> str:
578
- """Compute SHA-256 hash of a file's raw bytes."""
579
- hasher = hashlib.sha256()
580
- with open(abs_path, "rb") as f:
581
- for chunk in iter(lambda: f.read(65536), b""):
582
- hasher.update(chunk)
583
- return hasher.hexdigest()
584
-
585
-
586
- # ---------- incremental rebuild helpers ----------
587
-
588
-
589
- def _load_existing_types(conn: ladybug.Connection, tables: GraphTables, exclude_files: set[str] | None = None) -> None:
590
- """Load type entries from existing LadybugDB graph into tables for cross-file resolution.
591
-
592
- When exclude_files is provided, only load types from files NOT in the set.
593
- """
594
- if exclude_files is not None and not exclude_files:
595
- return
596
-
597
- where = f"WHERE s.kind IN {list(_TYPE_KINDS)}"
598
- params: dict = {}
599
- if exclude_files:
600
- where += "\n AND NOT (s.filename IN $exclude_files)"
601
- params["exclude_files"] = list(exclude_files)
602
-
603
- query = f"""
604
- MATCH (s:Symbol)
605
- {where}
606
- RETURN s.kind, s.fqn, s.name, s.filename, s.module, s.microservice, s.id, s.role, s.generated, s.generated_by
607
- """
608
- result = conn.execute(query, params)
609
- while result.has_next():
610
- row = result.get_next()
611
- kind, fqn, name, filename = row[0], row[1], row[2], row[3]
612
- module = row[4] if len(row) > 4 else ""
613
- microservice = row[5] if len(row) > 5 else ""
614
- node_id = row[6] if len(row) > 6 else ""
615
- role = row[7] if len(row) > 7 else ""
616
- generated = row[8] if len(row) > 8 else False
617
- generated_by = row[9] if len(row) > 9 else ""
618
-
619
- decl = TypeDecl(name, kind, fqn)
620
- package = fqn[: -(len(name) + 1)] if fqn.endswith("." + name) else ""
621
-
622
- entry = TypeIndexEntry(
623
- decl=decl,
624
- file_path=filename,
625
- module=module,
626
- microservice=microservice,
627
- package=package,
628
- outer_fqn=None,
629
- node_id=node_id,
630
- loaded_from_db=True,
631
- )
632
- tables.types[fqn] = entry
633
- tables.by_simple_name.setdefault(name, []).append(entry)
634
- tables.by_package.setdefault(package, []).append(entry)
635
- # Seed the persisted role so the annotation-less stub is not recomputed to
636
- # the default during node staging (issue #352 divergence #2).
637
- if role:
638
- tables.type_role_by_node_id[node_id] = role
639
- # Seed the persisted generated/generated_by so stubs retain their values
640
- tables.type_generated_by_node_id[node_id] = (bool(generated), generated_by)
641
-
642
-
643
- def _load_existing_members(conn: ladybug.Connection, tables: GraphTables, exclude_files: set[str] | None = None) -> None:
644
- """Load member entries from existing LadybugDB graph into tables.members.
645
-
646
- When exclude_files is provided, only load members from files NOT in the set.
647
- """
648
- if exclude_files is not None and not exclude_files:
649
- return
650
-
651
- where = "WHERE s.kind IN ['method', 'constructor']"
652
- params: dict = {}
653
- if exclude_files:
654
- where += "\n AND NOT (s.filename IN $exclude_files)"
655
- params["exclude_files"] = list(exclude_files)
656
-
657
- query = f"""
658
- MATCH (s:Symbol)
659
- {where}
660
- RETURN s.kind, s.name, s.filename, s.signature, s.parent_id, s.fqn, s.id
661
- """
662
- result = conn.execute(query, params)
663
- while result.has_next():
664
- row = result.get_next()
665
- kind, name, filename = row[0], row[1], row[2]
666
- signature = row[3] if len(row) > 3 else ""
667
- parent_id = row[4] if len(row) > 4 else ""
668
- fqn = row[5] if len(row) > 5 else ""
669
- node_id = row[6] if len(row) > 6 else ""
670
-
671
- parent_fqn = fqn.split("#")[0] if "#" in fqn else ""
672
-
673
- decl = MethodDecl(name, "", kind == "constructor")
674
- decl.signature = signature
675
-
676
- tables.members.append(MemberEntry(
677
- kind=kind,
678
- decl=decl,
679
- parent_id=parent_id,
680
- parent_fqn=parent_fqn,
681
- file_path=filename,
682
- module="",
683
- microservice="",
684
- node_id=node_id,
685
- loaded_from_db=True,
686
- ))
687
-
688
-
689
- # Every Symbol->Symbol REL TABLE type in the graph schema. A Symbol node can
690
- # only have an INCOMING edge of one of these types, so `_find_dependents` MUST
691
- # walk all of them: that completeness is what makes the changed-node DETACH
692
- # DELETE in `_delete_file_scope` Phase 3 safe (every real caller of a changed
693
- # node is pulled into scope, so Phase 1 removes the edge before the node delete).
694
- # If you add a new Symbol->Symbol edge type to the schema, add it here too —
695
- # otherwise changed-node deletion would silently drop its surviving edges.
696
- _SYMBOL_TO_SYMBOL_EDGE_TYPES = (
697
- "EXTENDS", "IMPLEMENTS", "INJECTS", "CALLS", "DECLARES", "OVERRIDES",
698
- )
699
-
700
-
701
- def _find_dependents(conn: ladybug.Connection, changed_node_ids: set[str]) -> set[str]:
702
- """Find files whose nodes have edges pointing into changed nodes. Returns set of filenames."""
703
- dependent_files: set[str] = set()
704
-
705
- params = {"changed_ids": list(changed_node_ids)}
706
-
707
- for edge_type in _SYMBOL_TO_SYMBOL_EDGE_TYPES:
708
- query = f"""
709
- MATCH (src:Symbol)-[e:{edge_type}]->(dst:Symbol)
710
- WHERE dst.id IN $changed_ids
711
- RETURN DISTINCT src.filename
712
- """
713
- result = conn.execute(query, params)
714
- while result.has_next():
715
- row = result.get_next()
716
- filename = row[0]
717
- if filename: # Skip phantom nodes (filename = "")
718
- dependent_files.add(filename)
719
-
720
- return dependent_files
721
-
722
-
723
- def _find_annotation_dependents(conn: ladybug.Connection, changed_node_ids: set[str]) -> set[str]:
724
- """Find files that USE an annotation whose DEFINITION is among the changed nodes.
725
-
726
- Annotation usage is a node property (``annotations`` STRING[]), not a
727
- Symbol->Symbol edge, so `_find_dependents` — which walks edges — never pulls
728
- annotation users into scope. When an annotation definition changes (e.g.
729
- ``@interface Foo`` gains a meta-annotation that shifts the Layer-A chain in
730
- `resolve_role_and_capabilities`), every type carrying ``@Foo`` may need its
731
- ``role``/``capabilities`` recomputed or it goes stale until the next full
732
- rebuild. Return those users' files so the orchestrator treats them as
733
- dependents (re-parsed, role re-SET); the expansion cap bounds the scope.
734
-
735
- Scope is direct usage only: a user of an annotation that transitively
736
- composes the changed one (e.g. ``@A`` where ``@A`` is meta-annotated with the
737
- changed ``@B``) is NOT pulled in — that reverse-chain walk is left to a
738
- future hardening pass. The direct case covers the dominant real-world shape
739
- (a stereotype annotation applied directly to many types).
740
- """
741
- if not changed_node_ids:
742
- return set()
743
- # Changed annotation definitions → the simple names users reference them by.
744
- # Runs before `_delete_file_scope`, so the def nodes still exist.
745
- name_result = conn.execute(
746
- "MATCH (s:Symbol) WHERE s.id IN $ids AND s.kind = 'annotation' RETURN s.name",
747
- {"ids": list(changed_node_ids)},
748
- )
749
- names: list[str] = []
750
- while name_result.has_next():
751
- nm = name_result.get_next()[0]
752
- if nm:
753
- names.append(nm)
754
- if not names:
755
- return set()
756
- dependent_files: set[str] = set()
757
- for nm in names:
758
- user_result = conn.execute(
759
- "MATCH (s:Symbol) "
760
- "WHERE list_contains(s.annotations, $nm) AND s.filename <> '' "
761
- "RETURN DISTINCT s.filename",
762
- {"nm": nm},
763
- )
764
- while user_result.has_next():
765
- fn = user_result.get_next()[0]
766
- if fn:
767
- dependent_files.add(fn)
768
- return dependent_files
769
-
770
-
771
- def _delete_file_scope(
772
- conn: ladybug.Connection,
773
- changed_files: set[str],
774
- dependent_files: set[str],
775
- ) -> None:
776
- """Delete nodes and edges for a scope split into changed vs dependent files.
777
-
778
- ``changed_files`` are files whose content actually changed (added/modified/
779
- removed): their Symbol nodes are deleted (and re-created by ``_scoped_write``).
780
- ``dependent_files`` are files pulled in only to re-resolve their OUTGOING
781
- edges against the changed nodes; their node definitions did not change, so
782
- their nodes are deliberately PRESERVED (they re-MERGE in place on the same
783
- deterministic ``symbol_id``). Skipping phantom nodes (filename="").
784
-
785
- Why dependents are preserved (issue #305): the orchestrator computes
786
- dependents from the *changed* nodes only, so a dependent file's node can
787
- have an incoming CALLS edge from an out-of-scope caller. The ``source_file``
788
- on every Symbol->Symbol edge is the CALLER's file (pinned by
789
- ``test_source_file_value_matches_symbol_filename``), so Phase 1 below only
790
- deletes edges ORIGINATING in scope; incoming edges from out-of-scope callers
791
- survive. If we then tried to DELETE the dependent node, LadybugDB rejects it
792
- ("Node ... has connected edges in table CALLS in the bwd direction, ...
793
- Please delete the edges first or try DETACH DELETE") and the rebuild falls
794
- back to a full rebuild. A naive fix (DETACH DELETE on dependents, or an
795
- extra incoming-edge pass) would silence the crash but permanently drop those
796
- out-of-scope edges, corrupting the graph. Preserving dependent nodes keeps
797
- both the nodes and their incoming edges intact.
798
-
799
- Phase 1 deletes ALL edge types across the whole scope (changed + dependent)
800
- first to avoid LadybugDB "has connected edges" errors when edges from one
801
- file point to nodes in another file within the same scope. Route/Client/
802
- Producer nodes use DETACH DELETE as a safety net for any edges missed in
803
- Phase 1.
804
- """
805
- # Symbol DELETEs below maintain the FTS index, which needs the extension loaded on
806
- # this connection. Idempotent + best-effort (no-op when no index exists / FTS absent).
807
- try:
808
- conn.execute("LOAD EXTENSION FTS")
809
- except Exception:
810
- pass
811
- scope_files = changed_files | dependent_files
812
- scope_list = list(scope_files)
813
- changed_list = list(changed_files)
814
-
815
- # Phase 1: Delete ALL edges ORIGINATING from any scope file (changed +
816
- # dependent). Because `source_file` is the caller's file, this deletes edges
817
- # whose source is in scope (including dependents' outgoing edges to changed
818
- # nodes) while intentionally leaving incoming edges from out-of-scope callers
819
- # intact — those must survive so the dependent nodes below can be preserved.
820
- # This list is a superset of `_SYMBOL_TO_SYMBOL_EDGE_TYPES` (it also covers
821
- # Symbol->Route/Client/Producer/UCS and Client/Producer->Route edges); keep
822
- # both lists in sync with the schema.
823
- edge_tables = [
824
- "EXTENDS", "IMPLEMENTS", "INJECTS", "CALLS", "DECLARES", "OVERRIDES",
825
- "UNRESOLVED_AT", "EXPOSES", "DECLARES_CLIENT", "DECLARES_PRODUCER",
826
- "HTTP_CALLS", "ASYNC_CALLS",
827
- ]
828
- for edge_type in edge_tables:
829
- query = f"""
830
- MATCH (src)-[e:{edge_type}]->(dst)
831
- WHERE e.source_file IN $filenames
832
- DELETE e
833
- """
834
- conn.execute(query, {"filenames": scope_list})
835
-
836
- # Phase 2: Collect all Symbol node IDs for UnresolvedCallSite cleanup.
837
- symbol_ids: list[str] = []
838
- symbol_ids_query = """
839
- MATCH (s:Symbol)
840
- WHERE s.filename IN $filenames
841
- RETURN s.id
842
- """
843
- result = conn.execute(symbol_ids_query, {"filenames": scope_list})
844
- while result.has_next():
845
- row = result.get_next()
846
- symbol_ids.append(row[0])
847
-
848
- # Delete UnresolvedCallSite nodes whose caller_id is in the collected set.
849
- # These are children of scope symbols (including preserved dependents);
850
- # deleting them is safe because every scope file — dependents included — is
851
- # reprocessed and re-emits its UnresolvedCallSite nodes in `_scoped_write`.
852
- if symbol_ids:
853
- unresolved_query = """
854
- MATCH (u:UnresolvedCallSite)
855
- WHERE u.caller_id IN $symbol_ids
856
- DELETE u
857
- """
858
- conn.execute(unresolved_query, {"symbol_ids": symbol_ids})
859
-
860
- # Phase 3: Delete Symbol nodes ONLY for changed files (not dependents).
861
- # Dependent-file nodes are deliberately PRESERVED so their incoming edges
862
- # from out-of-scope callers survive; the dependents are re-MERGEd in place
863
- # by `_scoped_write` on the same deterministic node id. A changed node's
864
- # real incoming edges all come from dependent files (callers pulled into
865
- # scope by `_find_dependents`, which walks every type in
866
- # `_SYMBOL_TO_SYMBOL_EDGE_TYPES`), so Phase 1 already removed them and the
867
- # dependents re-emit them when reprocessed. DETACH DELETE is only a safety
868
- # net for the rare surviving edge whose source was NOT pulled into scope
869
- # (e.g. a phantom caller with filename="", which `_find_dependents` skips);
870
- # such an edge is stale once the node is recreated, so dropping it is fine.
871
- delete_symbols_query = """
872
- MATCH (s:Symbol)
873
- WHERE s.filename IN $filenames
874
- DETACH DELETE s
875
- """
876
- conn.execute(delete_symbols_query, {"filenames": changed_list})
877
-
878
- # Phase 4: Delete Route, Client, Producer nodes.
879
- # Use DETACH DELETE as a safety net in case any edges were missed in Phase 1.
880
- for label in ["Route", "Client", "Producer"]:
881
- conn.execute(
882
- f"MATCH (n:{label}) WHERE n.filename IN $filenames DETACH DELETE n",
883
- {"filenames": scope_list},
884
- )
885
-
886
-
887
- def _scoped_write(conn: ladybug.Connection, tables: GraphTables, *, project_root: Path, meta_chain: dict[str, frozenset[str]] | None) -> None:
888
- """Write nodes and edges to existing LadybugDB database without drop/create schema.
889
-
890
- Like write_ladybug() but without _drop_all()/_create_schema(). The caller is
891
- responsible for calling _populate_declares_rows() and _populate_overrides_rows()
892
- before invoking this function.
893
-
894
- Uses MERGE instead of CREATE to handle cases where nodes already exist.
895
- """
896
- t0 = time.time()
897
- _write_nodes_merge(
898
- conn,
899
- tables,
900
- project_root=project_root,
901
- meta_chain=meta_chain,
902
- )
903
- elapsed = time.time() - t0
904
- if elapsed > 0.1: # Only log if significant
905
- _verbose_stderr_line(f"[graph] scoped write · nodes written in {elapsed:.2f}s")
906
-
907
- t1 = time.time()
908
- _fbyid = _build_file_by_node_id(tables)
909
- _write_edges(conn, tables, _fbyid)
910
- elapsed = time.time() - t1
911
- if elapsed > 0.1:
912
- _verbose_stderr_line(f"[graph] scoped write · edges written in {elapsed:.2f}s")
913
-
914
- t2 = time.time()
915
- _write_routes_and_exposes(conn, tables, _fbyid)
916
- elapsed = time.time() - t2
917
- if elapsed > 0.1:
918
- _verbose_stderr_line(f"[graph] scoped write · routes/exposes written in {elapsed:.2f}s")
919
-
920
-
921
- def _write_nodes_merge(
922
- conn: ladybug.Connection,
923
- tables: GraphTables,
924
- *,
925
- project_root: Path,
926
- meta_chain: dict[str, frozenset[str]] | None,
927
- ) -> None:
928
- """Write nodes to existing LadybugDB database using bulk COPY FROM."""
929
- _write_nodes_impl(conn, tables, project_root=project_root, meta_chain=meta_chain)
930
-
931
-
932
- # ---------- file walk (see `path_filtering.iter_java_source_files`) ----------
933
-
934
-
935
- # ---------- pass 1 ----------
936
-
937
-
938
- def _register_type(
939
- tables: GraphTables,
940
- decl: TypeDecl,
941
- *,
942
- file_path: str,
943
- module: str,
944
- microservice: str,
945
- outer_fqn: str | None,
946
- ) -> TypeIndexEntry:
947
- package = decl.fqn.rsplit(".", 1)[0] if "." in decl.fqn and outer_fqn is None else (
948
- outer_fqn.rsplit(".", 1)[0] if outer_fqn and "." in outer_fqn else ""
949
- )
950
- # top-level: package = fqn - name; nested: inherit from outer
951
- if outer_fqn is None:
952
- package = decl.fqn[: -(len(decl.name) + 1)] if decl.fqn.endswith("." + decl.name) else ""
953
- else:
954
- # walk outward to find a top-level fqn; package is everything before its simple name
955
- top = outer_fqn
956
- while top in tables.types and tables.types[top].outer_fqn:
957
- top = tables.types[top].outer_fqn # type: ignore[assignment]
958
- package = top[: top.rfind(".")] if "." in top else ""
959
-
960
- node_id = symbol_id(decl.kind, decl.fqn, file_path, decl.start_byte)
961
- entry = TypeIndexEntry(
962
- decl=decl,
963
- file_path=file_path,
964
- module=module,
965
- microservice=microservice,
966
- package=package,
967
- outer_fqn=outer_fqn,
968
- node_id=node_id,
969
- )
970
- tables.types[decl.fqn] = entry
971
- tables.by_simple_name.setdefault(decl.name, []).append(entry)
972
- tables.by_package.setdefault(package, []).append(entry)
973
-
974
- for m in decl.methods:
975
- kind = "constructor" if m.is_constructor else "method"
976
- mid = symbol_id(kind, f"{decl.fqn}#{m.signature}", file_path, m.start_byte)
977
- tables.members.append(MemberEntry(
978
- kind=kind, decl=m, parent_id=node_id, parent_fqn=decl.fqn,
979
- file_path=file_path, module=module, microservice=microservice,
980
- node_id=mid,
981
- ))
982
-
983
- for nested in decl.nested:
984
- _register_type(
985
- tables, nested, file_path=file_path,
986
- module=module, microservice=microservice, outer_fqn=decl.fqn,
987
- )
988
-
989
- return entry
990
-
991
-
992
- def pass1_parse(
993
- root: Path,
994
- tables: GraphTables,
995
- *,
996
- verbose: bool,
997
- scope_files: set[str] | None = None,
998
- removed_files: set[str] | None = None,
999
- ) -> dict[str, JavaFileAst]:
1000
- """Walk files, parse them, populate node indexes. Returns path -> AST.
1001
-
1002
- Args:
1003
- root: Source root directory.
1004
- tables: GraphTables to populate.
1005
- verbose: Whether to emit progress output.
1006
- scope_files: Optional set of relative POSIX paths to parse. If None, parse all files.
1007
- removed_files: Optional set of relative POSIX paths that no longer exist
1008
- on disk (incremental deletions). These are members of ``scope_files``
1009
- (they were deleted, so they participate in scoped deletion) but are
1010
- never visited by the parse walk, so they must be excluded from the
1011
- pass-1 total to keep ``done`` from undercounting then two-way-clamping.
1012
- """
1013
- asts: dict[str, JavaFileAst] = {}
1014
- ignore = LayeredIgnore(root)
1015
- t0 = time.time()
1016
- n_files = 0
1017
- if verbose:
1018
- _verbose_stderr_line(_PASS1_START)
1019
- # Count-first: one filtered walk (no parsing) to set the EXACT total before
1020
- # the parse loop ticks. Single-layer ignore → the count is exact, so the
1021
- # rendered bar is determinate. For a scoped (incremental) parse the total is
1022
- # the number of files that will actually be visited: scope minus any removed
1023
- # files (which are members of scope for deletion but gone from disk, so the
1024
- # parse walk never ticks them); for a full rebuild it is the non-ignored
1025
- # .java count.
1026
- if verbose:
1027
- if scope_files is not None:
1028
- removed = removed_files if removed_files is not None else set()
1029
- pass1_total = len(scope_files - removed)
1030
- else:
1031
- pass1_total = sum(1 for _ in iter_java_source_files(root, ignore=ignore))
1032
- _emit_graph_progress(
1033
- {"pass": "1/6", "done": 0, "total": pass1_total, "status": "running"},
1034
- verbose=verbose,
1035
- )
1036
- slow_sec = 0.0
1037
- raw_slow = os.environ.get("JAVA_CODEBASE_RAG_TEST_GRAPH_SLOW_SEC", "").strip()
1038
- if raw_slow:
1039
- try:
1040
- slow_sec = float(raw_slow)
1041
- except ValueError:
1042
- slow_sec = 0.0
1043
- with _VerbosePassHeartbeats("[graph] pass 1", verbose=verbose):
1044
- if verbose and slow_sec > 0:
1045
- time.sleep(slow_sec)
1046
- for p in iter_java_source_files(root, ignore=ignore):
1047
- # Skip files not in scope (if scope is provided)
1048
- try:
1049
- rel = p.resolve().relative_to(root.resolve()).as_posix()
1050
- except ValueError:
1051
- rel = p.as_posix()
1052
- if scope_files is not None and rel not in scope_files:
1053
- continue
1054
- n_files += 1
1055
- if verbose and (n_files % _PASS1_TICK_EVERY == 0):
1056
- _emit_graph_progress(
1057
- {"pass": "1/6", "done": n_files, "status": "running"},
1058
- verbose=verbose,
1059
- )
1060
- try:
1061
- content = p.read_bytes()
1062
- except OSError:
1063
- tables.skipped_files += 1
1064
- continue
1065
- if not content.strip():
1066
- continue
1067
- try:
1068
- ast = parse_java(content, filename=rel, verbose=verbose)
1069
- except Exception:
1070
- tables.parse_errors += 1
1071
- continue
1072
- if ast.parse_error:
1073
- tables.parse_errors += 1
1074
- # Still index what tree-sitter gave us; robust to syntax errors.
1075
- module = module_for_path(str(p), root)
1076
- microservice = microservice_for_path(str(p), root)
1077
- asts[rel] = ast
1078
-
1079
- # Classify the file once (generated or not, and which tool generated it)
1080
- generated_config = load_generated_detection(str(root))
1081
- file_generated, file_generated_by = classify_java_file(
1082
- content, ast, config=generated_config, project_root=root
1083
- )
1084
-
1085
- # file node
1086
- file_id = symbol_id("file", rel, rel, 0)
1087
- tables.files[rel] = file_id
1088
-
1089
- # package node (created lazily; nodes deduped by id)
1090
- if ast.package and ast.package not in tables.packages:
1091
- tables.packages[ast.package] = symbol_id("package", ast.package, "", 0)
1092
-
1093
- for t in ast.top_level_types:
1094
- _register_type(
1095
- tables, t, file_path=rel,
1096
- module=module, microservice=microservice, outer_fqn=None,
1097
- )
1098
-
1099
- # Seed generated/generated_by for all types in this file (including nested)
1100
- for t in ast.all_types:
1101
- if t.fqn in tables.types:
1102
- node_id = tables.types[t.fqn].node_id
1103
- tables.type_generated_by_node_id[node_id] = (file_generated, file_generated_by)
1104
-
1105
- if verbose:
1106
- elapsed = time.time() - t0
1107
- _emit_graph_progress(
1108
- {"pass": "1/6", "done": n_files, "status": "done", "elapsed_s": f"{elapsed:.2f}"},
1109
- verbose=verbose,
1110
- )
1111
- _verbose_stderr_line(
1112
- f"[graph] pass 1 · parsed {n_files} files in {elapsed:.2f}s: "
1113
- f"{len(tables.types)} types, {len(tables.members)} members, "
1114
- f"{tables.parse_errors} parse errors, {tables.skipped_files} skipped",
1115
- )
1116
- return asts
1117
-
1118
-
1119
- # ---------- pass 2: resolution + edges ----------
1120
-
1121
-
1122
- def _resolve_simple(
1123
- name: str,
1124
- *,
1125
- current: TypeIndexEntry,
1126
- ast: JavaFileAst,
1127
- tables: GraphTables,
1128
- ) -> TypeIndexEntry | None:
1129
- """Java-ish name resolution. Returns a known TypeIndexEntry or None (phantom)."""
1130
- # Strip trailing generics the caller may have left in, defensively.
1131
- bare = name.split("<", 1)[0].strip()
1132
- if not bare:
1133
- return None
1134
-
1135
- # 0. Nested inside the same top-level hierarchy — try `Outer.Bare` fqn.
1136
- outer = current.outer_fqn
1137
- while outer is not None and outer in tables.types:
1138
- candidate = f"{outer}.{bare}"
1139
- if candidate in tables.types:
1140
- return tables.types[candidate]
1141
- outer = tables.types[outer].outer_fqn
1142
-
1143
- # 1. Same-file siblings (same outer as `current`).
1144
- same_outer = current.outer_fqn or current.package
1145
- for e in tables.by_simple_name.get(bare, ()):
1146
- e_parent = e.outer_fqn or e.package
1147
- if e.file_path == current.file_path and e_parent == same_outer:
1148
- return e
1149
-
1150
- # 2. Explicit import.
1151
- if bare in ast.explicit_imports:
1152
- fq = ast.explicit_imports[bare]
1153
- if fq in tables.types:
1154
- return tables.types[fq]
1155
- # Known FQN (outside our codebase) → unresolved; caller will phantom-ise.
1156
- return None
1157
-
1158
- # 3. Same package.
1159
- if current.package:
1160
- candidate = f"{current.package}.{bare}"
1161
- if candidate in tables.types:
1162
- return tables.types[candidate]
1163
-
1164
- # 4. Wildcard imports.
1165
- for wild in ast.wildcard_imports:
1166
- candidate = f"{wild}.{bare}"
1167
- if candidate in tables.types:
1168
- return tables.types[candidate]
1169
-
1170
- # 5. java.lang best-effort (unresolved but deterministic phantom).
1171
- return None
1172
-
1173
-
1174
- def _phantom_target(
1175
- tables: GraphTables,
1176
- simple: str,
1177
- ast: JavaFileAst,
1178
- *,
1179
- current: TypeIndexEntry,
1180
- ) -> tuple[str, str, str]:
1181
- """Produce (id, simple, fqn-or-best-guess) for an unresolved type reference.
1182
-
1183
- The fqn falls back through: explicit import → wildcard → java.lang → bare name.
1184
- """
1185
- bare = simple.split("<", 1)[0].strip()
1186
- guess_fqn = bare
1187
- if bare in ast.explicit_imports:
1188
- guess_fqn = ast.explicit_imports[bare]
1189
- elif bare in _JAVA_LANG_SIMPLE:
1190
- guess_fqn = f"java.lang.{bare}"
1191
- elif ast.wildcard_imports:
1192
- # Pick first wildcard as a hint (imperfect but useful for display).
1193
- guess_fqn = f"{ast.wildcard_imports[0]}.{bare}"
1194
-
1195
- pid = phantom_id(guess_fqn)
1196
- if pid not in tables.phantoms:
1197
- tables.phantoms[pid] = {
1198
- "id": pid,
1199
- "kind": "class",
1200
- "name": bare,
1201
- "fqn": guess_fqn,
1202
- "package": guess_fqn.rsplit(".", 1)[0] if "." in guess_fqn else "",
1203
- "module": "",
1204
- "microservice": "",
1205
- "filename": "",
1206
- "start_line": 0,
1207
- "end_line": 0,
1208
- "start_byte": 0,
1209
- "end_byte": 0,
1210
- "modifiers": [],
1211
- "annotations": [],
1212
- "capabilities": [],
1213
- "role": "OTHER",
1214
- "signature": "",
1215
- "parent_id": "",
1216
- "resolved": False,
1217
- }
1218
- return pid, bare, guess_fqn
1219
-
1220
-
1221
- def _edge_for(
1222
- *,
1223
- src: TypeIndexEntry,
1224
- target_simple: str,
1225
- ast: JavaFileAst,
1226
- tables: GraphTables,
1227
- ) -> tuple[str, str, str, bool]:
1228
- resolved = _resolve_simple(target_simple, current=src, ast=ast, tables=tables)
1229
- if resolved is not None:
1230
- return resolved.node_id, resolved.decl.name, resolved.decl.fqn, True
1231
- pid, simple, fqn = _phantom_target(tables, target_simple, ast, current=src)
1232
- return pid, simple, fqn, False
1233
-
1234
-
1235
- def _emit_extends_implements(
1236
- entry: TypeIndexEntry,
1237
- ast: JavaFileAst,
1238
- tables: GraphTables,
1239
- *,
1240
- seen_ext: set[tuple[str, str]],
1241
- seen_impl: set[tuple[str, str]],
1242
- ) -> None:
1243
- for name in entry.decl.extends:
1244
- dst_id, dst_simple, dst_fqn, ok = _edge_for(
1245
- src=entry, target_simple=name, ast=ast, tables=tables,
1246
- )
1247
- key = (entry.node_id, dst_id)
1248
- if key in seen_ext:
1249
- continue
1250
- seen_ext.add(key)
1251
- tables.extends_rows.append(EdgeRow(
1252
- src_id=entry.node_id, dst_id=dst_id,
1253
- dst_name=dst_simple, dst_fqn=dst_fqn, resolved=ok,
1254
- ))
1255
-
1256
- for name in entry.decl.implements:
1257
- dst_id, dst_simple, dst_fqn, ok = _edge_for(
1258
- src=entry, target_simple=name, ast=ast, tables=tables,
1259
- )
1260
- key = (entry.node_id, dst_id)
1261
- if key in seen_impl:
1262
- continue
1263
- seen_impl.add(key)
1264
- tables.implements_rows.append(EdgeRow(
1265
- src_id=entry.node_id, dst_id=dst_id,
1266
- dst_name=dst_simple, dst_fqn=dst_fqn, resolved=ok,
1267
- ))
1268
-
1269
-
1270
- def _emit_injects(
1271
- entry: TypeIndexEntry,
1272
- ast: JavaFileAst,
1273
- tables: GraphTables,
1274
- *,
1275
- seen: set[tuple[str, str, str, str]],
1276
- ) -> None:
1277
- if entry.decl.kind == "interface":
1278
- return
1279
-
1280
- ann_names = [a.name for a in entry.decl.annotations]
1281
- inject_set = injection_annotation_names()
1282
- lombok_rac = lombok_required_args_annotations()
1283
- has_lombok_rac = any(a in lombok_rac for a in ann_names)
1284
-
1285
- def _add(
1286
- target: str, mechanism: str, annotation: str, slot: str,
1287
- ) -> None:
1288
- dst_id, dst_simple, dst_fqn, ok = _edge_for(
1289
- src=entry, target_simple=target, ast=ast, tables=tables,
1290
- )
1291
- key = (entry.node_id, dst_id, mechanism, slot)
1292
- if key in seen:
1293
- return
1294
- seen.add(key)
1295
- tables.injects_rows.append(InjectsRow(
1296
- src_id=entry.node_id, dst_id=dst_id,
1297
- dst_name=dst_simple, dst_fqn=dst_fqn, resolved=ok,
1298
- mechanism=mechanism, annotation=annotation, field_or_param=slot,
1299
- ))
1300
-
1301
- # Field injection: @Autowired / @Inject / @Resource.
1302
- for f in entry.decl.fields:
1303
- annotated = next((a.name for a in f.annotations if a.name in inject_set), None)
1304
- if annotated:
1305
- _add(f.type_name, "field", annotated, f.name)
1306
-
1307
- # Lombok: @RequiredArgsConstructor -> each `final` non-static field becomes an injection;
1308
- # @AllArgsConstructor -> every non-static field.
1309
- if has_lombok_rac:
1310
- all_args = "AllArgsConstructor" in ann_names
1311
- for f in entry.decl.fields:
1312
- if "static" in f.modifiers:
1313
- continue
1314
- if not all_args and "final" not in f.modifiers:
1315
- continue
1316
- _add(f.type_name, "lombok_required_args",
1317
- "AllArgsConstructor" if all_args else "RequiredArgsConstructor",
1318
- f.name)
1319
-
1320
- # Constructor injection:
1321
- ctors = [m for m in entry.decl.methods if m.is_constructor]
1322
- if ctors:
1323
- chosen = None
1324
- autowired = [c for c in ctors if any(a.name == "Autowired" for a in c.annotations)]
1325
- if autowired:
1326
- chosen = autowired[0]
1327
- elif len(ctors) == 1 and ctors[0].parameters:
1328
- chosen = ctors[0]
1329
- if chosen is not None:
1330
- annotation = "Autowired" if any(a.name == "Autowired" for a in chosen.annotations) else ""
1331
- for p in chosen.parameters:
1332
- _add(p.type_name, "constructor", annotation, p.name)
1333
-
1334
- # Setter injection: setXxx annotated @Autowired with 1 parameter.
1335
- for m in entry.decl.methods:
1336
- if m.is_constructor or not m.name.startswith("set") or len(m.parameters) != 1:
1337
- continue
1338
- if any(a.name == "Autowired" for a in m.annotations):
1339
- _add(m.parameters[0].type_name, "setter", "Autowired",
1340
- m.parameters[0].name)
1341
-
1342
-
1343
- def pass2_edges(tables: GraphTables, asts: dict[str, JavaFileAst], *, verbose: bool) -> None:
1344
- t0 = time.time()
1345
- seen_ext: set[tuple[str, str]] = set()
1346
- seen_impl: set[tuple[str, str]] = set()
1347
- seen_inj: set[tuple[str, str, str, str]] = set()
1348
- if verbose:
1349
- _verbose_stderr_line(_PASS2_START)
1350
- with _graph_pass_progress("2/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 2", verbose=verbose):
1351
- for fqn, entry in tables.types.items():
1352
- ast = asts.get(entry.file_path)
1353
- if ast is None:
1354
- continue
1355
- _emit_extends_implements(entry, ast, tables, seen_ext=seen_ext, seen_impl=seen_impl)
1356
- _emit_injects(entry, ast, tables, seen=seen_inj)
1357
- if verbose:
1358
- elapsed = time.time() - t0
1359
- _verbose_stderr_line(
1360
- f"[graph] pass 2 · emitted {len(tables.extends_rows)} EXTENDS, "
1361
- f"{len(tables.implements_rows)} IMPLEMENTS, "
1362
- f"{len(tables.injects_rows)} INJECTS, "
1363
- f"{len(tables.phantoms)} phantoms in {elapsed:.2f}s",
1364
- )
1365
-
1366
-
1367
- # ---------- pass 3: call graph ----------
1368
-
1369
-
1370
- def _build_member_indexes(tables: GraphTables) -> None:
1371
- tables.methods_by_type = {}
1372
- for m in tables.members:
1373
- tables.methods_by_type.setdefault(m.parent_fqn, []).append(m)
1374
-
1375
-
1376
- def _direct_supertype_fqns(entry: TypeIndexEntry, tables: GraphTables) -> list[str]:
1377
- out: list[str] = []
1378
- for r in tables.extends_rows:
1379
- if r.src_id == entry.node_id and r.dst_fqn in tables.types:
1380
- out.append(r.dst_fqn)
1381
- for r in tables.implements_rows:
1382
- if r.src_id == entry.node_id and r.dst_fqn in tables.types:
1383
- out.append(r.dst_fqn)
1384
- return out
1385
-
1386
-
1387
- def _first_supertype_fqn(tables: GraphTables, type_fqn: str) -> str | None:
1388
- entry = tables.types.get(type_fqn)
1389
- if entry is None:
1390
- return None
1391
- for r in tables.extends_rows:
1392
- if r.src_id == entry.node_id and r.dst_fqn in tables.types:
1393
- return r.dst_fqn
1394
- for r in tables.implements_rows:
1395
- if r.src_id == entry.node_id and r.dst_fqn in tables.types:
1396
- return r.dst_fqn
1397
- return None
1398
-
1399
-
1400
- def _is_chained_receiver_text(receiver_expr: str) -> bool:
1401
- """Heuristic: call chain or complex expr (contains a completed call)."""
1402
- s = receiver_expr.strip()
1403
- return "(" in s and ")" in s
1404
-
1405
-
1406
- def _resolve_this_super_field_chain(
1407
- expr: str,
1408
- *,
1409
- member: MemberEntry,
1410
- ast: JavaFileAst,
1411
- tables: GraphTables,
1412
- ) -> str | None:
1413
- """Resolve `this.a.b` / `super.a` (no calls) to the final field's type FQN."""
1414
- s = expr.strip()
1415
- if "(" in s or ")" in s or "." not in s:
1416
- return None
1417
- entry = tables.types.get(member.parent_fqn)
1418
- if entry is None:
1419
- return None
1420
- parts = s.split(".")
1421
- if len(parts) < 2:
1422
- return None
1423
- if parts[0] == "this":
1424
- cur = entry
1425
- elif parts[0] == "super":
1426
- sup = _first_supertype_fqn(tables, member.parent_fqn)
1427
- if sup is None or sup not in tables.types:
1428
- return None
1429
- cur = tables.types[sup]
1430
- else:
1431
- return None
1432
- for fname in parts[1:]:
1433
- fld = next((f for f in cur.decl.fields if f.name == fname), None)
1434
- if fld is None:
1435
- return None
1436
- resolved = _resolve_simple(fld.type_name, current=cur, ast=ast, tables=tables)
1437
- if resolved is None:
1438
- return None
1439
- cur = resolved
1440
- return cur.decl.fqn
1441
-
1442
-
1443
- def _scope_table(member: MemberEntry, ast: JavaFileAst, tables: GraphTables) -> dict[str, str]:
1444
- """Map simple variable/field/param name -> resolved declaring type FQN."""
1445
- scope: dict[str, str] = {}
1446
- entry = tables.types.get(member.parent_fqn)
1447
- if entry is None:
1448
- return scope
1449
-
1450
- def add_fields(tentry: TypeIndexEntry) -> None:
1451
- for f in tentry.decl.fields:
1452
- resolved = _resolve_simple(f.type_name, current=tentry, ast=ast, tables=tables)
1453
- if resolved is not None:
1454
- scope[f.name] = resolved.decl.fqn
1455
-
1456
- add_fields(entry)
1457
- seen: set[str] = {member.parent_fqn}
1458
- queue = list(_direct_supertype_fqns(entry, tables))
1459
- while queue:
1460
- sup = queue.pop()
1461
- if sup in seen or sup not in tables.types:
1462
- continue
1463
- seen.add(sup)
1464
- te = tables.types[sup]
1465
- for f in te.decl.fields:
1466
- if f.name not in scope:
1467
- resolved = _resolve_simple(f.type_name, current=te, ast=ast, tables=tables)
1468
- if resolved is not None:
1469
- scope[f.name] = resolved.decl.fqn
1470
- queue.extend(_direct_supertype_fqns(te, tables))
1471
-
1472
- for p in member.decl.parameters:
1473
- resolved = _resolve_simple(p.type_name, current=entry, ast=ast, tables=tables)
1474
- if resolved is not None:
1475
- scope[p.name] = resolved.decl.fqn
1476
-
1477
- # Locals shadow fields and parameters (same simple name → local wins).
1478
- for name, t_simple in member.decl.local_vars:
1479
- resolved = _resolve_simple(t_simple, current=entry, ast=ast, tables=tables)
1480
- if resolved is not None:
1481
- scope[name] = resolved.decl.fqn
1482
-
1483
- return scope
1484
-
1485
-
1486
- def _lookup_method_candidates(
1487
- type_fqn: str,
1488
- callee_simple: str,
1489
- arg_count: int,
1490
- tables: GraphTables,
1491
- ast: JavaFileAst,
1492
- *,
1493
- visited: set[str] | None = None,
1494
- ) -> tuple[list[MemberEntry], bool]:
1495
- """Return (candidates, used_name_only_fallback). Walks type + supertypes.
1496
-
1497
- When ``used_name_only_fallback`` is true and ``len(candidates) == 1``, the
1498
- caller may reuse the receiver-resolution strategy (see ``_resolve_and_emit_call``)
1499
- instead of tagging ``overload_ambiguous``.
1500
- """
1501
- if visited is None:
1502
- visited = set()
1503
- exact: list[MemberEntry] = []
1504
- name_only: list[MemberEntry] = []
1505
-
1506
- def collect_on_type(tfqn: str) -> None:
1507
- nonlocal exact, name_only
1508
- for m in tables.methods_by_type.get(tfqn, ()):
1509
- if callee_simple == "<init>":
1510
- if not m.decl.is_constructor:
1511
- continue
1512
- np = len(m.decl.parameters)
1513
- if arg_count < 0:
1514
- name_only.append(m)
1515
- elif np == arg_count:
1516
- exact.append(m)
1517
- else:
1518
- name_only.append(m)
1519
- continue
1520
- if m.decl.is_constructor:
1521
- continue
1522
- if m.decl.name != callee_simple:
1523
- continue
1524
- np = len(m.decl.parameters)
1525
- if arg_count < 0:
1526
- name_only.append(m)
1527
- elif np == arg_count:
1528
- exact.append(m)
1529
- else:
1530
- name_only.append(m)
1531
-
1532
- queue = [type_fqn]
1533
- while queue:
1534
- tfqn = queue.pop(0)
1535
- if tfqn in visited or tfqn not in tables.types:
1536
- continue
1537
- visited.add(tfqn)
1538
- collect_on_type(tfqn)
1539
- te = tables.types[tfqn]
1540
- for sup in _direct_supertype_fqns(te, tables):
1541
- if sup not in visited:
1542
- queue.append(sup)
1543
- # Synthetic anonymous classes (`….<anon:byte>`): unqualified instance calls
1544
- # may target the lexically enclosing type (D3), e.g. `pingFromAnon()` from
1545
- # `NestedCalls` inside `new Runnable() { void run() { … } }`.
1546
- if ".<anon:" in tfqn and te.outer_fqn and te.outer_fqn not in visited:
1547
- queue.append(te.outer_fqn)
1548
-
1549
- if exact:
1550
- return exact, False
1551
- if name_only:
1552
- return name_only, True
1553
- return [], False
1554
-
1555
-
1556
- def _static_wildcard_resolve(
1557
- callee_simple: str,
1558
- ast: JavaFileAst,
1559
- tables: GraphTables,
1560
- current: TypeIndexEntry,
1561
- ) -> str | None:
1562
- for tw in ast.file_imports.static_wildcards:
1563
- if tw not in tables.types:
1564
- continue
1565
- for m in tables.methods_by_type.get(tw, ()):
1566
- if m.decl.name != callee_simple or m.decl.is_constructor:
1567
- continue
1568
- if "static" not in m.decl.modifiers:
1569
- continue
1570
- return tw
1571
- return None
1572
-
1573
-
1574
- def _unique_type_simple_resolve(simple: str, tables: GraphTables) -> str | None:
1575
- """Return the type FQN iff exactly one indexed type uses `simple` as `decl.name`.
1576
-
1577
- Used only for receiver / static-qualifier disambiguation. Do not use the
1578
- method index here: an unresolved identifier that equals some method's
1579
- simple name elsewhere in the project is not evidence about the receiver type.
1580
- """
1581
- hits = tables.by_simple_name.get(simple, [])
1582
- if len(hits) != 1:
1583
- return None
1584
- return hits[0].decl.fqn
1585
-
1586
-
1587
- def _suffix_resolve(receiver_simple: str, tables: GraphTables) -> str | None:
1588
- matches = [fq for fq in tables.types if fq.endswith("." + receiver_simple)]
1589
- if len(matches) != 1:
1590
- return None
1591
- return matches[0]
1592
-
1593
-
1594
- def _resolve_receiver_type(
1595
- call: CallSite,
1596
- *,
1597
- scope: dict[str, str],
1598
- member: MemberEntry,
1599
- ast: JavaFileAst,
1600
- tables: GraphTables,
1601
- ) -> tuple[str | None, str, float]:
1602
- """Returns (receiver_type_fqn_or_none, strategy, confidence)."""
1603
- expr = call.receiver_expr.strip()
1604
- callee = call.callee_simple
1605
-
1606
- effective_static = call.is_static_call
1607
- if call.is_static_call and expr and not _is_chained_receiver_text(expr):
1608
- bare_for_static = expr.split("<", 1)[0].strip()
1609
- if bare_for_static and "." not in bare_for_static and bare_for_static in scope:
1610
- effective_static = False
1611
-
1612
- if not expr and not call.is_static_call:
1613
- if callee in ast.file_imports.static_methods:
1614
- full = ast.file_imports.static_methods[callee]
1615
- if "." in full:
1616
- type_fqn = full.rsplit(".", 1)[0]
1617
- return type_fqn, "static_import", 0.95
1618
- sw = _static_wildcard_resolve(callee, ast, tables, tables.types[member.parent_fqn])
1619
- if sw is not None:
1620
- return sw, "static_import_wildcard", 0.85
1621
-
1622
- if effective_static and expr:
1623
- if _is_chained_receiver_text(expr):
1624
- return None, "chained_receiver", 0.0
1625
- entry = tables.types.get(member.parent_fqn)
1626
- if entry is None:
1627
- return None, "chained_receiver", 0.0
1628
- bare_static = expr.split("<", 1)[0].strip()
1629
- resolved = _resolve_simple(bare_static, current=entry, ast=ast, tables=tables)
1630
- if resolved is not None:
1631
- return resolved.decl.fqn, "import_map", 0.95
1632
- # External type not in the index but FQN is deterministic via an explicit import.
1633
- # e.g. `import java.util.Objects; Objects.requireNonNull(x)` — we know the FQN
1634
- # is "java.util.Objects" even though the type isn't indexed; return it so the
1635
- # edge carries the correct receiver-tier confidence rather than collapsing to phantom.
1636
- if bare_static in ast.explicit_imports:
1637
- return ast.explicit_imports[bare_static], "import_map", 0.95
1638
- uq = _unique_type_simple_resolve(expr, tables)
1639
- if uq is not None:
1640
- return uq, "unique_type_name", 0.75
1641
- sf = _suffix_resolve(expr, tables)
1642
- if sf is not None:
1643
- return sf, "suffix", 0.55
1644
- return None, "phantom", 0.0
1645
-
1646
- if expr in ("", "this"):
1647
- return member.parent_fqn, "this_super", 0.95
1648
-
1649
- if expr == "super":
1650
- sup = _first_supertype_fqn(tables, member.parent_fqn)
1651
- if sup is not None:
1652
- return sup, "this_super", 0.95
1653
- # No indexed supertype — implicit super to java.lang.Object.
1654
- # Keep strategy='implicit_super' and confidence=0.90 so this path is
1655
- # distinguishable from a genuinely unresolvable receiver.
1656
- return "java.lang.Object", "implicit_super", 0.90
1657
-
1658
- if _is_chained_receiver_text(expr):
1659
- return None, "chained_receiver", 0.0
1660
-
1661
- entry = tables.types.get(member.parent_fqn)
1662
- if entry is None:
1663
- return None, "phantom", 0.0
1664
-
1665
- bare = expr.split("<", 1)[0].strip()
1666
- if bare in scope:
1667
- return scope[bare], "import_map", 0.95
1668
-
1669
- chain = _resolve_this_super_field_chain(expr, member=member, ast=ast, tables=tables)
1670
- if chain is not None:
1671
- return chain, "import_map", 0.95
1672
-
1673
- resolved = _resolve_simple(bare, current=entry, ast=ast, tables=tables)
1674
- if resolved is not None:
1675
- return resolved.decl.fqn, "import_map", 0.95
1676
-
1677
- if entry.package:
1678
- cand = f"{entry.package}.{bare}"
1679
- if cand in tables.types:
1680
- return cand, "same_module", 0.90
1681
-
1682
- uq = _unique_type_simple_resolve(bare, tables)
1683
- if uq is not None:
1684
- return uq, "unique_type_name", 0.75
1685
-
1686
- sf = _suffix_resolve(bare, tables)
1687
- if sf is not None:
1688
- return sf, "suffix", 0.55
1689
-
1690
- return None, "phantom", 0.0
1691
-
1692
-
1693
- def _phantom_method_id(
1694
- tables: GraphTables,
1695
- *,
1696
- receiver_fqn: str | None,
1697
- receiver_expr: str,
1698
- callee: str,
1699
- arg_count: int,
1700
- ) -> str:
1701
- # Phantom node identity for a resolved receiver omits call-site arity so
1702
- # method references (arg_count=-1) and normal invocations share one Symbol
1703
- # per (receiver_fqn, callee) when the callee is not indexed (D1).
1704
- if receiver_fqn:
1705
- fqn = f"{receiver_fqn}#{callee}(?)"
1706
- sig = f"{callee}(?)"
1707
- else:
1708
- expr_short = (receiver_expr[:50] if receiver_expr else "?")
1709
- arity = "(?)" if arg_count < 0 else f"({arg_count})"
1710
- fqn = f"?{expr_short}#{callee}{arity}"
1711
- sig = f"{callee}{arity}"
1712
- pid = phantom_id(fqn)
1713
- if pid not in tables.phantoms:
1714
- tables.phantoms[pid] = {
1715
- "id": pid,
1716
- "kind": "method",
1717
- "name": callee,
1718
- "fqn": fqn,
1719
- "package": "",
1720
- "module": "",
1721
- "microservice": "",
1722
- "filename": "",
1723
- "start_line": 0,
1724
- "end_line": 0,
1725
- "start_byte": 0,
1726
- "end_byte": 0,
1727
- "modifiers": [],
1728
- "annotations": [],
1729
- "capabilities": [],
1730
- "role": "OTHER",
1731
- "signature": sig,
1732
- "parent_id": "",
1733
- "resolved": False,
1734
- }
1735
- return pid
1736
-
1737
-
1738
- def _method_signature_matches_call(member: MemberEntry, call: CallSite) -> bool:
1739
- if call.arg_count < 0:
1740
- return True
1741
- return len(member.decl.parameters) == call.arg_count
1742
-
1743
-
1744
- def _is_strict_supertype_of(tables: GraphTables, super_fqn: str, subtype_fqn: str) -> bool:
1745
- if super_fqn == subtype_fqn:
1746
- return False
1747
- entry = tables.types.get(subtype_fqn)
1748
- if entry is None:
1749
- return False
1750
- visited: set[str] = set()
1751
- queue = list(_direct_supertype_fqns(entry, tables))
1752
- while queue:
1753
- tfqn = queue.pop(0)
1754
- if tfqn == super_fqn:
1755
- return True
1756
- if tfqn in visited or tfqn not in tables.types:
1757
- continue
1758
- visited.add(tfqn)
1759
- queue.extend(_direct_supertype_fqns(tables.types[tfqn], tables))
1760
- return False
1761
-
1762
-
1763
- def _callee_declaring_role_at_write(
1764
- tables: GraphTables,
1765
- dst_id: str,
1766
- *,
1767
- member_by_id: dict[str, MemberEntry],
1768
- ) -> str:
1769
- """Match parent declaring-type Symbol.role (brownfield + meta_chain included)."""
1770
- if dst_id in tables.phantoms:
1771
- return "OTHER"
1772
- member = member_by_id.get(dst_id)
1773
- if member is None:
1774
- return "OTHER"
1775
- return tables.type_role_by_node_id.get(member.parent_id, "OTHER")
1776
-
1777
-
1778
- def _collapse_supertype_duplicates(
1779
- candidates: list[MemberEntry],
1780
- recv_type_fqn: str,
1781
- call: CallSite,
1782
- tables: GraphTables,
1783
- ) -> list[MemberEntry]:
1784
- """§3.3.1 supertype-walk dedup — collapse interface + concrete duplicate sites."""
1785
- if len(candidates) <= 1:
1786
- return candidates
1787
- concrete_on_receiver = [
1788
- c for c in candidates
1789
- if c.parent_fqn == recv_type_fqn and _method_signature_matches_call(c, call)
1790
- ]
1791
- if len(concrete_on_receiver) != 1:
1792
- return candidates
1793
- concrete = concrete_on_receiver[0]
1794
- supertypes = [
1795
- c for c in candidates
1796
- if c is not concrete
1797
- and _is_strict_supertype_of(tables, c.parent_fqn, recv_type_fqn)
1798
- and c.decl.signature == concrete.decl.signature
1799
- ]
1800
- if not supertypes:
1801
- return candidates
1802
- allowed_ids = {concrete.node_id, *(c.node_id for c in supertypes)}
1803
- if any(c.node_id not in allowed_ids for c in candidates):
1804
- return candidates
1805
- log.debug(
1806
- "pass3 supertype dedup %s -> %s",
1807
- [c.node_id for c in candidates],
1808
- concrete.node_id,
1809
- )
1810
- return [concrete]
1811
-
1812
-
1813
- def _unresolved_call_site_id(caller_id: str, call: CallSite) -> str:
1814
- return f"ucs:{caller_id}:{call.line}:{call.byte}"
1815
-
1816
-
1817
- def _emit_unresolved_call_site(
1818
- tables: GraphTables,
1819
- stats: CallResolutionStats,
1820
- *,
1821
- caller_id: str,
1822
- call: CallSite,
1823
- reason: str,
1824
- ) -> None:
1825
- tables.unresolved_call_site_rows.append(UnresolvedCallSiteRow(
1826
- id=_unresolved_call_site_id(caller_id, call),
1827
- caller_id=caller_id,
1828
- call_site_line=call.line,
1829
- call_site_byte=call.byte,
1830
- arg_count=call.arg_count,
1831
- callee_simple=call.callee_simple,
1832
- receiver_expr=call.receiver_expr or "",
1833
- reason=reason,
1834
- ))
1835
- if reason == "chained_receiver":
1836
- stats.phantom_chained += 1
1837
- else:
1838
- stats.phantom_other += 1
1839
-
1840
-
1841
- def _emit_call_edge(
1842
- tables: GraphTables,
1843
- stats: CallResolutionStats,
1844
- *,
1845
- src_id: str,
1846
- dst_id: str,
1847
- call: CallSite,
1848
- confidence: float,
1849
- strategy: str,
1850
- resolved: bool,
1851
- edge_arg_count: int | None = None,
1852
- ) -> None:
1853
- arity = call.arg_count if edge_arg_count is None else edge_arg_count
1854
- tables.calls_rows.append(CallsRow(
1855
- src_id=src_id,
1856
- dst_id=dst_id,
1857
- call_site_line=call.line,
1858
- call_site_byte=call.byte,
1859
- arg_count=arity,
1860
- confidence=confidence,
1861
- strategy=strategy,
1862
- source="static",
1863
- resolved=resolved,
1864
- ))
1865
- stats.total += 1
1866
- stats.by_strategy[strategy] += 1
1867
- if not resolved:
1868
- stats.callee_unresolved += 1
1869
-
1870
-
1871
- def _resolve_and_emit_call(
1872
- call: CallSite,
1873
- member: MemberEntry,
1874
- ast: JavaFileAst,
1875
- tables: GraphTables,
1876
- stats: CallResolutionStats,
1877
- *,
1878
- scope: dict[str, str],
1879
- ) -> None:
1880
- """Emit CALLS rows for one call site.
1881
-
1882
- Candidate selection uses ``_lookup_method_candidates`` (exact arity first, then
1883
- name-only fallback on the type + supertype walk).
1884
-
1885
- When ``used_name_only_fallback`` is true and exactly one name-only candidate
1886
- exists, the edge ``strategy`` reuses the receiver-resolution tier (``strat``)
1887
- rather than ``overload_ambiguous``: arity at the call site did not match any
1888
- overload, but only one method of that name exists — the callee is unambiguous.
1889
- """
1890
- recv_type, strat, conf = _resolve_receiver_type(call, scope=scope, member=member, ast=ast, tables=tables)
1891
-
1892
- if strat == "chained_receiver":
1893
- _emit_unresolved_call_site(
1894
- tables, stats, caller_id=member.node_id, call=call, reason="chained_receiver",
1895
- )
1896
- return
1897
-
1898
- if recv_type is None:
1899
- _emit_unresolved_call_site(
1900
- tables, stats,
1901
- caller_id=member.node_id,
1902
- call=call,
1903
- reason="phantom_unresolved_receiver",
1904
- )
1905
- return
1906
-
1907
- candidates, name_only_fb = _lookup_method_candidates(
1908
- recv_type, call.callee_simple, call.arg_count, tables, ast,
1909
- )
1910
-
1911
- # Guard relies on `_lookup_method_candidates` returning a same-ms candidate when one exists; revisit if pass3 scopes lookups per-microservice.
1912
- if member.microservice:
1913
- same_ms = [c for c in candidates if c.microservice == member.microservice]
1914
- if same_ms and len(same_ms) != len(candidates):
1915
- for c in candidates:
1916
- if c.microservice and c.microservice != member.microservice:
1917
- log.warning(
1918
- "skipping cross-microservice CALLS edge %s -> %s "
1919
- "(caller=%s, callee=%s)",
1920
- f"{member.parent_fqn}#{member.decl.signature}",
1921
- f"{c.parent_fqn}#{c.decl.signature}",
1922
- member.microservice, c.microservice,
1923
- )
1924
- stats.skipped_cross_service += 1
1925
- candidates = same_ms
1926
-
1927
- # Compute the call-shape strategy / confidence override BEFORE the
1928
- # empty-candidates check so they are preserved even when the callee cannot
1929
- # be located on the resolved receiver type (B3 fix).
1930
- edge_conf = conf
1931
- if call.arg_count < 0:
1932
- edge_strat = "method_reference"
1933
- elif call.callee_simple == "<init>" and call.receiver_expr == "super" and (
1934
- call.byte == member.decl.start_byte and call.line == member.decl.start_line
1935
- ):
1936
- # Synthesized implicit-super site from _parse_method.
1937
- edge_strat = "implicit_super"
1938
- edge_conf = 0.90
1939
- elif call.callee_simple == "<init>":
1940
- # new Foo(…), this(…), super(…) — confidence inherited from receiver tier.
1941
- edge_strat = "constructor"
1942
- elif name_only_fb and len(candidates) > 1:
1943
- edge_strat = "overload_ambiguous"
1944
- elif name_only_fb and len(candidates) == 1:
1945
- # Name-only fallback with a single candidate — not ambiguous.
1946
- edge_strat = strat
1947
- else:
1948
- edge_strat = strat
1949
-
1950
- if not candidates:
1951
- # Receiver was resolved but the callee method isn't indexed on that type
1952
- # (e.g. JDK / Spring / external library). Preserve the receiver-tier
1953
- # strategy and confidence — only resolved=False signals the phantom callee
1954
- # (B3 fix: do NOT downgrade to confidence=0.0 / strategy='phantom' here).
1955
- pid = _phantom_method_id(
1956
- tables, receiver_fqn=recv_type, receiver_expr=call.receiver_expr,
1957
- callee=call.callee_simple, arg_count=call.arg_count,
1958
- )
1959
- _emit_call_edge(
1960
- tables, stats, src_id=member.node_id, dst_id=pid, call=call,
1961
- confidence=edge_conf, strategy=edge_strat, resolved=False,
1962
- )
1963
- return
1964
-
1965
- if len(candidates) > 1 and edge_strat != "overload_ambiguous":
1966
- candidates = _collapse_supertype_duplicates(candidates, recv_type, call, tables)
1967
-
1968
- if len(candidates) == 1:
1969
- candidate = candidates[0]
1970
- ref_arity: int | None = None
1971
- if call.arg_count < 0:
1972
- ref_arity = len(candidate.decl.parameters)
1973
- _emit_call_edge(
1974
- tables, stats, src_id=member.node_id, dst_id=candidate.node_id, call=call,
1975
- confidence=edge_conf, strategy=edge_strat, resolved=True,
1976
- edge_arg_count=ref_arity,
1977
- )
1978
- return
1979
-
1980
- for c in candidates:
1981
- ref_arity_multi: int | None = len(c.decl.parameters) if call.arg_count < 0 else None
1982
- _emit_call_edge(
1983
- tables, stats, src_id=member.node_id, dst_id=c.node_id, call=call,
1984
- confidence=edge_conf, strategy="overload_ambiguous", resolved=True,
1985
- edge_arg_count=ref_arity_multi,
1986
- )
1987
-
1988
-
1989
- def _resolve_method_calls(
1990
- member: MemberEntry,
1991
- ast: JavaFileAst,
1992
- tables: GraphTables,
1993
- stats: CallResolutionStats,
1994
- ) -> None:
1995
- scope = _scope_table(member, ast, tables)
1996
- for call in member.decl.call_sites:
1997
- try:
1998
- _resolve_and_emit_call(call, member, ast, tables, stats, scope=scope)
1999
- except Exception as e:
2000
- log.warning("call resolution failed for %s: %s", member.decl.signature, e)
2001
-
2002
-
2003
- def _process_file_calls(
2004
- file_ast: JavaFileAst,
2005
- file_path: str,
2006
- tables: GraphTables,
2007
- stats: CallResolutionStats,
2008
- ) -> None:
2009
- for member in tables.members:
2010
- if member.file_path != file_path:
2011
- continue
2012
- try:
2013
- _resolve_method_calls(member, file_ast, tables, stats)
2014
- except Exception as e:
2015
- log.warning("Failed to extract calls from %s#%s: %s", member.parent_fqn, member.decl.signature, e)
2016
-
2017
-
2018
- def pass3_calls(tables: GraphTables, asts: dict[str, JavaFileAst], *, verbose: bool) -> None:
2019
- if verbose:
2020
- _verbose_stderr_line(_PASS3_START)
2021
- _build_member_indexes(tables)
2022
- stats = CallResolutionStats()
2023
- with _graph_pass_progress("3/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 3", verbose=verbose):
2024
- for rel_path, file_ast in asts.items():
2025
- try:
2026
- _process_file_calls(file_ast, rel_path, tables, stats)
2027
- except Exception as e:
2028
- log.error("Call extraction failed for %s: %s", rel_path, e)
2029
- denom_calls = max(1, stats.total)
2030
- denom_sites = max(1, stats.total + stats.phantom_chained + stats.phantom_other)
2031
- pct_chained = 100.0 * stats.phantom_chained / denom_sites
2032
- pct_callee_unres = 100.0 * stats.callee_unresolved / denom_calls
2033
- pct_phantom_recv = 100.0 * stats.phantom_other / denom_sites
2034
- tables.pass3_skipped_cross_service = int(stats.skipped_cross_service)
2035
- tables.pass3_unresolved_phantom_receiver = int(stats.phantom_other)
2036
- tables.pass3_unresolved_chained = int(stats.phantom_chained)
2037
- msg = (
2038
- f"Call resolution: {stats.total} CALLS rows, {stats.phantom_chained} chained unresolved "
2039
- f"({pct_chained:.1f}%), {stats.callee_unresolved} unresolved callee on CALLS "
2040
- f"({pct_callee_unres:.1f}%), {stats.phantom_other} phantom-receiver unresolved "
2041
- f"({pct_phantom_recv:.1f}%), {stats.skipped_cross_service} skipped cross-service, "
2042
- f"strategies: {dict(stats.by_strategy)}"
2043
- )
2044
- log.info(msg)
2045
- if verbose:
2046
- _verbose_stderr_line(f"[graph] pass 3 · {msg}")
2047
-
2048
-
2049
- _PATH_VAR_SEG = re.compile(r"^\{([^:{}]+)(?::([^}]*))?\}$") # whole path segment
2050
-
2051
-
2052
- def _normalize_path(raw_path: str) -> tuple[str, str]:
2053
- """Return `(path_template, path_regex)` for a servlet-style path pattern.
2054
-
2055
- `/api/users/{id}` → ``("/api/users/{}", "^/api/users/[^/]+/?$")``.
2056
- `{id:\\d+}` constraints strip to ``{}`` in the template while preserving the
2057
- regex constraint for that segment. Deterministic for shared use by B2b/B6.
2058
- """
2059
- raw_path = (raw_path or "").strip()
2060
- if not raw_path:
2061
- return "", ""
2062
- p = raw_path if raw_path.startswith("/") else "/" + raw_path
2063
- trimmed = p.rstrip("/")
2064
- if trimmed == "":
2065
- return "/", "^/?$"
2066
- segments = [s for s in trimmed.split("/") if s != ""]
2067
- tmpl_parts: list[str] = []
2068
- re_parts: list[str] = []
2069
- for seg in segments:
2070
- m = _PATH_VAR_SEG.fullmatch(seg)
2071
- if m:
2072
- tmpl_parts.append("{}")
2073
- constraint = m.group(2)
2074
- re_parts.append(constraint if constraint else "[^/]+")
2075
- else:
2076
- tmpl_parts.append(seg)
2077
- re_parts.append(re.escape(seg))
2078
- tmpl = "/" + "/".join(tmpl_parts)
2079
- body = "/".join(re_parts)
2080
- if not body.startswith("/"):
2081
- body = "/" + body
2082
- return tmpl, f"^{body}/?$"
2083
-
2084
-
2085
- def _route_id(
2086
- framework: str,
2087
- kind: str,
2088
- http_method: str,
2089
- path_template: str,
2090
- path_raw: str,
2091
- topic: str,
2092
- broker: str,
2093
- microservice: str,
2094
- ) -> str:
2095
- """Stable id; `path_raw` disambiguates HTTP routes when `path_template` is empty (SpEL / const)."""
2096
- path_key = path_template if path_template else path_raw
2097
- key = (
2098
- f"{framework}|{kind}|{http_method}|{path_key}|"
2099
- f"{topic}|{broker}|{microservice}"
2100
- )
2101
- return f"r:{hashlib.sha1(key.encode()).hexdigest()[:16]}"
2102
-
2103
-
2104
- def _client_id(
2105
- *,
2106
- microservice: str,
2107
- member_fqn: str,
2108
- client_kind: str,
2109
- path: str,
2110
- method: str,
2111
- ) -> str:
2112
- key = f"{microservice}|{member_fqn}|{client_kind}|{path}|{method}"
2113
- return f"c:{hashlib.sha1(key.encode()).hexdigest()[:16]}"
2114
-
2115
-
2116
- def _producer_id(
2117
- *,
2118
- microservice: str,
2119
- member_fqn: str,
2120
- producer_kind: str,
2121
- topic: str,
2122
- ) -> str:
2123
- # Topic-level identity per method+kind; broker is intentionally omitted so the same
2124
- # resolved topic on one method shares one Producer node across call sites.
2125
- key = f"{microservice}|{member_fqn}|{producer_kind}|{topic}"
2126
- return f"p:{hashlib.sha1(key.encode()).hexdigest()[:16]}"
2127
-
2128
-
2129
- # The four brownfield source layers — single source of truth. Consumed by the
2130
- # client/producer source-layer classifiers, the *_from_brownfield_pct stats
2131
- # (via brownfield_strategies), and the brownfield_only authoritativeness gate in
2132
- # _is_brownfield_sourced. codebase_client/codebase_producer are caller-side
2133
- # declaration strategies, not layers — they extend brownfield_strategies only.
2134
- _BROWNFIELD_LAYERS = frozenset({
2135
- "layer_a_meta",
2136
- "layer_b_ann",
2137
- "layer_b_fqn",
2138
- "layer_c_source",
2139
- })
2140
-
2141
-
2142
- def _client_source_layer(strategy: str) -> str:
2143
- if strategy in _BROWNFIELD_LAYERS:
2144
- return strategy
2145
- # Some caller extraction paths emit client kind as strategy; treat those
2146
- # as builtin-source declarations instead of warning on every row.
2147
- if strategy in VALID_CLIENT_KINDS:
2148
- return "builtin"
2149
- if strategy != "builtin":
2150
- log.warning("unknown client source strategy %r, falling back to builtin", strategy)
2151
- return "builtin"
2152
-
2153
-
2154
- def _producer_source_layer(strategy: str) -> str:
2155
- if strategy in _BROWNFIELD_LAYERS:
2156
- return strategy
2157
- if strategy in VALID_PRODUCER_KINDS:
2158
- return "builtin"
2159
- if strategy != "builtin":
2160
- log.warning("unknown producer source strategy %r, falling back to builtin", strategy)
2161
- return "builtin"
2162
-
2163
-
2164
- _ROUTE_LAYER_RANK: dict[str, int] = {
2165
- "builtin": 0,
2166
- "layer_b_ann": 1,
2167
- "layer_a_meta": 2,
2168
- "layer_c_source": 3,
2169
- "layer_b_fqn": 4,
2170
- }
2171
-
2172
-
2173
- def pass4_routes(
2174
- tables: GraphTables,
2175
- asts: dict[str, JavaFileAst],
2176
- *,
2177
- source_root: Path,
2178
- verbose: bool,
2179
- ) -> None:
2180
- stats = tables.route_stats
2181
- overrides = load_brownfield_overrides(source_root)
2182
- try:
2183
- prs = str(source_root.resolve())
2184
- except OSError:
2185
- prs = str(source_root)
2186
- tables.cross_service_resolution = _load_config_cross_service_resolution(prs)
2187
- meta_chain = collect_annotation_meta_chain(prs)
2188
- if verbose:
2189
- _verbose_stderr_line(_PASS4_START)
2190
- with _graph_pass_progress("4/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 4", verbose=verbose):
2191
-
2192
- for ast in asts.values():
2193
- stats.routes_skipped_unresolved += ast.routes_skipped_unresolved
2194
-
2195
- routes_by_id: dict[str, RouteRow] = {}
2196
- exposes_seen: set[tuple[str, str]] = set()
2197
-
2198
- http_kinds = frozenset({"http_endpoint", "http_consumer"})
2199
-
2200
- for member in sorted(tables.members, key=lambda m: m.node_id):
2201
- if member.decl.is_constructor:
2202
- continue
2203
- ast = asts.get(member.file_path)
2204
- if ast is None:
2205
- continue
2206
- type_decl = tables.types[member.parent_fqn].decl
2207
- final_routes = resolve_routes_for_method(
2208
- method_decl=member.decl,
2209
- enclosing_type=type_decl,
2210
- overrides=overrides,
2211
- meta_chain=meta_chain,
2212
- builtin_routes=member.decl.routes,
2213
- )
2214
- if not final_routes:
2215
- continue
2216
- for decl in final_routes:
2217
- path_template, path_regex = ("", "")
2218
- if decl.kind in http_kinds:
2219
- if decl.resolved and decl.resolution_strategy in (
2220
- "annotation",
2221
- "codebase_route",
2222
- ):
2223
- path_template, path_regex = _normalize_path(decl.path)
2224
- else:
2225
- path_template, path_regex = "", ""
2226
- rid = _route_id(
2227
- decl.framework,
2228
- decl.kind,
2229
- decl.http_method,
2230
- path_template,
2231
- decl.path,
2232
- decl.topic,
2233
- decl.broker,
2234
- member.microservice,
2235
- )
2236
- layer = decl.route_source_layer
2237
- if rid not in routes_by_id:
2238
- routes_by_id[rid] = RouteRow(
2239
- id=rid,
2240
- kind=decl.kind,
2241
- framework=decl.framework,
2242
- method=decl.http_method,
2243
- path=decl.path,
2244
- path_template=path_template,
2245
- path_regex=path_regex,
2246
- topic=decl.topic,
2247
- broker=decl.broker,
2248
- feign_name=decl.feign_name,
2249
- feign_url=decl.feign_url,
2250
- microservice=member.microservice,
2251
- module=member.module,
2252
- filename=decl.filename,
2253
- start_line=decl.start_line,
2254
- end_line=decl.end_line,
2255
- resolved=decl.resolved,
2256
- source_layer=layer,
2257
- )
2258
- else:
2259
- prev = routes_by_id[rid]
2260
- if _ROUTE_LAYER_RANK.get(layer, 0) > _ROUTE_LAYER_RANK.get(
2261
- prev.source_layer,
2262
- 0,
2263
- ):
2264
- routes_by_id[rid] = replace(prev, source_layer=layer)
2265
- ek = (member.node_id, rid)
2266
- if ek not in exposes_seen:
2267
- route_kind = routes_by_id[rid].kind
2268
- if route_kind == "http_consumer":
2269
- stats.exposes_suppressed_feign += 1
2270
- continue
2271
- exposes_seen.add(ek)
2272
- tables.exposes_rows.append(
2273
- ExposesRow(
2274
- symbol_id=member.node_id,
2275
- route_id=rid,
2276
- confidence=decl.confidence,
2277
- strategy=decl.resolution_strategy,
2278
- ),
2279
- )
2280
-
2281
- tables.routes_rows = sorted(routes_by_id.values(), key=lambda r: r.id)
2282
-
2283
- for row in tables.routes_rows:
2284
- stats.by_framework[row.framework] += 1
2285
- stats.by_kind[row.kind] += 1
2286
-
2287
- n_routes = len(tables.routes_rows)
2288
- if n_routes:
2289
- stats.routes_resolved_pct = 100.0 * sum(
2290
- 1 for r in tables.routes_rows if r.resolved
2291
- ) / n_routes
2292
- stats.routes_from_brownfield_pct = 100.0 * sum(
2293
- 1 for r in tables.routes_rows if r.source_layer != "builtin"
2294
- ) / n_routes
2295
- else:
2296
- stats.routes_resolved_pct = 100.0
2297
- stats.routes_from_brownfield_pct = 0.0
2298
-
2299
- by_layer: dict[str, int] = defaultdict(int)
2300
- for row in tables.routes_rows:
2301
- by_layer[row.source_layer] += 1
2302
- stats.routes_by_layer = dict(sorted(by_layer.items()))
2303
-
2304
- msg = (
2305
- f"Route extraction: emitted={n_routes}, exposes={len(tables.exposes_rows)}, "
2306
- f"exposes_suppressed_feign={stats.exposes_suppressed_feign}, "
2307
- f"skipped_unresolved={stats.routes_skipped_unresolved}, "
2308
- f"routes_resolved_pct={stats.routes_resolved_pct:.1f}, "
2309
- f"routes_from_brownfield_pct={stats.routes_from_brownfield_pct:.1f}, "
2310
- f"by_framework={dict(stats.by_framework)}"
2311
- )
2312
- log.info(msg)
2313
- if verbose:
2314
- _verbose_stderr_line(f"[graph] pass 4 · {msg}")
2315
-
2316
-
2317
- def pass5_imperative_edges(
2318
- tables: GraphTables,
2319
- asts: dict[str, JavaFileAst],
2320
- *,
2321
- source_root: Path,
2322
- verbose: bool,
2323
- ) -> None:
2324
- del asts
2325
- overrides = load_brownfield_overrides(source_root)
2326
- try:
2327
- prs = str(source_root.resolve())
2328
- except OSError:
2329
- prs = str(source_root)
2330
- tables.cross_service_resolution = _load_config_cross_service_resolution(prs)
2331
- meta_chain = collect_annotation_meta_chain(prs)
2332
- routes_by_id = {r.id: r for r in tables.routes_rows}
2333
- existing_route_ids = set(routes_by_id)
2334
- http_seen: set[tuple[str, str]] = set()
2335
- async_seen: set[tuple[str, str]] = set()
2336
- client_seen: set[str] = set()
2337
- producer_seen: set[str] = set()
2338
- declares_client_seen: set[tuple[str, str]] = set()
2339
- declares_producer_seen: set[tuple[str, str]] = set()
2340
- route_rows = list(tables.routes_rows)
2341
-
2342
- def _micro_factor(member: MemberEntry) -> float:
2343
- ms = microservice_for_path(member.file_path, source_root)
2344
- return 1.0 if ms else 0.85
2345
-
2346
- def _append_route(row: RouteRow) -> None:
2347
- if row.id in existing_route_ids:
2348
- return
2349
- existing_route_ids.add(row.id)
2350
- routes_by_id[row.id] = row
2351
- route_rows.append(row)
2352
-
2353
- def _phantom_http_route_id(call: OutgoingCallDecl) -> str:
2354
- if call.path_template_call and call.method_call:
2355
- return _route_id("", "http_endpoint", call.method_call, call.path_template_call, call.path_template_call, "", "", "")
2356
- uniq = hashlib.sha1(f"{call.filename}:{call.start_line}:{call.raw_uri}".encode()).hexdigest()[:12]
2357
- return f"r:phantom:{uniq}"
2358
-
2359
- def _phantom_async_route_id(call: OutgoingCallDecl) -> str:
2360
- if call.topic_call:
2361
- return _route_id("", "kafka_topic", "", "", "", call.topic_call, call.broker_call, "")
2362
- uniq = hashlib.sha1(f"{call.filename}:{call.start_line}:{call.raw_topic}".encode()).hexdigest()[:12]
2363
- return f"r:phantom:{uniq}"
2364
-
2365
- if verbose:
2366
- _verbose_stderr_line(_PASS5_START)
2367
- with _graph_pass_progress("5/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 5", verbose=verbose):
2368
- for member in sorted(tables.members, key=lambda x: x.node_id):
2369
- if member.decl.is_constructor:
2370
- continue
2371
- type_decl = tables.types[member.parent_fqn].decl
2372
- final_http_calls = resolve_http_client_for_method(
2373
- method_decl=member.decl,
2374
- enclosing_type=type_decl,
2375
- overrides=overrides,
2376
- meta_chain=meta_chain,
2377
- builtin_calls=member.decl.outgoing_calls,
2378
- )
2379
- final_async_calls = resolve_async_producer_for_method(
2380
- method_decl=member.decl,
2381
- enclosing_type=type_decl,
2382
- overrides=overrides,
2383
- meta_chain=meta_chain,
2384
- builtin_calls=member.decl.outgoing_calls,
2385
- )
2386
- micro_factor = _micro_factor(member)
2387
- for call in final_http_calls + final_async_calls:
2388
- if call.channel == "http":
2389
- client_path = (call.path_template_call or "").strip()
2390
- client_method = (call.method_call or "").strip().upper()
2391
- # Keep normalized path fields on Client now so LC3 filter semantics
2392
- # (`path_prefix`) can use persisted columns without extra transforms.
2393
- client_path_template = ""
2394
- client_path_regex = ""
2395
- if client_path:
2396
- client_path_template, client_path_regex = _normalize_path(client_path)
2397
- cid = _client_id(
2398
- microservice=member.microservice,
2399
- member_fqn=call.method_fqn,
2400
- client_kind=call.client_kind,
2401
- path=client_path,
2402
- method=client_method,
2403
- )
2404
- if cid not in client_seen:
2405
- client_seen.add(cid)
2406
- tables.client_rows.append(
2407
- ClientRow(
2408
- id=cid,
2409
- client_kind=call.client_kind,
2410
- target_service=call.feign_target_name,
2411
- path=client_path,
2412
- path_template=client_path_template,
2413
- path_regex=client_path_regex,
2414
- method=client_method,
2415
- member_fqn=call.method_fqn,
2416
- member_id=member.node_id,
2417
- microservice=member.microservice,
2418
- module=member.module,
2419
- filename=call.filename,
2420
- start_line=call.start_line,
2421
- end_line=call.end_line,
2422
- resolved=call.resolved,
2423
- source_layer=_client_source_layer(call.resolution_strategy),
2424
- ),
2425
- )
2426
- dkey = (member.node_id, cid)
2427
- if dkey not in declares_client_seen:
2428
- declares_client_seen.add(dkey)
2429
- tables.declares_client_rows.append(
2430
- DeclaresClientRow(
2431
- symbol_id=member.node_id,
2432
- client_id=cid,
2433
- confidence=call.confidence_base,
2434
- strategy=call.resolution_strategy,
2435
- ),
2436
- )
2437
- rid = ""
2438
- strategy = call.resolution_strategy
2439
- if call.client_kind == CLIENT_KIND_FEIGN_METHOD:
2440
- exposing = next((e for e in tables.exposes_rows if e.symbol_id == member.node_id), None)
2441
- if exposing is not None:
2442
- rid = exposing.route_id
2443
- if not rid:
2444
- rid = _phantom_http_route_id(call)
2445
- _append_route(
2446
- RouteRow(
2447
- id=rid,
2448
- kind="http_endpoint",
2449
- framework="",
2450
- method=call.method_call,
2451
- path=call.path_template_call,
2452
- path_template=call.path_template_call,
2453
- path_regex="",
2454
- topic="",
2455
- broker="",
2456
- feign_name=call.feign_target_name,
2457
- feign_url=call.feign_target_url,
2458
- microservice="",
2459
- module="",
2460
- filename=call.filename,
2461
- start_line=call.start_line,
2462
- end_line=call.end_line,
2463
- resolved=False,
2464
- source_layer="builtin",
2465
- )
2466
- )
2467
- key = (cid, rid)
2468
- if key in http_seen:
2469
- continue
2470
- http_seen.add(key)
2471
- conf = call.confidence_base * 0.3 * micro_factor
2472
- tables.http_call_rows.append(
2473
- HttpCallRow(
2474
- client_id=cid,
2475
- route_id=rid,
2476
- confidence=conf,
2477
- strategy=strategy,
2478
- method_call=call.method_call,
2479
- raw_uri=call.raw_uri,
2480
- match="unresolved",
2481
- )
2482
- )
2483
- tables.call_edge_stats.http_calls_total += 1
2484
- tables.call_edge_stats.http_calls_by_client_kind[call.client_kind] += 1
2485
- tables.call_edge_stats.http_calls_by_strategy[strategy] += 1
2486
- elif call.channel == "async":
2487
- topic_atom = (call.topic_call or "").strip()
2488
- pid = _producer_id(
2489
- microservice=member.microservice,
2490
- member_fqn=call.method_fqn,
2491
- producer_kind=call.client_kind,
2492
- topic=topic_atom,
2493
- )
2494
- if pid not in producer_seen:
2495
- producer_seen.add(pid)
2496
- tables.producer_rows.append(
2497
- ProducerRow(
2498
- id=pid,
2499
- producer_kind=call.client_kind,
2500
- topic=topic_atom,
2501
- broker=call.broker_call,
2502
- direction="producer",
2503
- member_fqn=call.method_fqn,
2504
- member_id=member.node_id,
2505
- microservice=member.microservice,
2506
- module=member.module,
2507
- filename=call.filename,
2508
- start_line=call.start_line,
2509
- end_line=call.end_line,
2510
- resolved=call.resolved,
2511
- source_layer=_producer_source_layer(call.resolution_strategy),
2512
- ),
2513
- )
2514
- dpkey = (member.node_id, pid)
2515
- if dpkey not in declares_producer_seen:
2516
- declares_producer_seen.add(dpkey)
2517
- tables.declares_producer_rows.append(
2518
- DeclaresProducerRow(
2519
- symbol_id=member.node_id,
2520
- producer_id=pid,
2521
- confidence=call.confidence_base,
2522
- strategy=call.resolution_strategy,
2523
- ),
2524
- )
2525
- rid = _phantom_async_route_id(call)
2526
- _append_route(
2527
- RouteRow(
2528
- id=rid,
2529
- kind="kafka_topic",
2530
- framework="",
2531
- method="",
2532
- path="",
2533
- path_template="",
2534
- path_regex="",
2535
- topic=call.topic_call,
2536
- broker=call.broker_call,
2537
- feign_name="",
2538
- feign_url="",
2539
- microservice="",
2540
- module="",
2541
- filename=call.filename,
2542
- start_line=call.start_line,
2543
- end_line=call.end_line,
2544
- resolved=False,
2545
- source_layer="builtin",
2546
- )
2547
- )
2548
- key = (pid, rid)
2549
- if key in async_seen:
2550
- continue
2551
- async_seen.add(key)
2552
- conf = call.confidence_base * 0.3 * micro_factor
2553
- strategy = call.resolution_strategy
2554
- tables.async_call_rows.append(
2555
- AsyncCallRow(
2556
- producer_id=pid,
2557
- route_id=rid,
2558
- confidence=conf,
2559
- strategy=strategy,
2560
- direction="producer",
2561
- raw_topic=call.raw_topic,
2562
- match="unresolved",
2563
- )
2564
- )
2565
- tables.call_edge_stats.async_calls_total += 1
2566
- tables.call_edge_stats.async_calls_by_client_kind[call.client_kind] += 1
2567
- tables.call_edge_stats.async_calls_by_strategy[strategy] += 1
2568
-
2569
- tables.routes_rows = sorted(route_rows, key=lambda r: r.id)
2570
- tables.client_rows = sorted(tables.client_rows, key=lambda c: c.id)
2571
- tables.declares_client_rows = sorted(
2572
- tables.declares_client_rows,
2573
- key=lambda e: (e.symbol_id, e.client_id),
2574
- )
2575
- tables.client_stats.clients_total = len(tables.client_rows)
2576
- tables.client_stats.declares_client_total = len(tables.declares_client_rows)
2577
- tables.client_stats.clients_by_kind = defaultdict(int)
2578
- for row in tables.client_rows:
2579
- tables.client_stats.clients_by_kind[row.client_kind] += 1
2580
- tables.producer_rows = sorted(tables.producer_rows, key=lambda p: p.id)
2581
- tables.declares_producer_rows = sorted(
2582
- tables.declares_producer_rows,
2583
- key=lambda e: (e.symbol_id, e.producer_id),
2584
- )
2585
- tables.producer_stats.producers_total = len(tables.producer_rows)
2586
- tables.producer_stats.declares_producer_total = len(tables.declares_producer_rows)
2587
- tables.producer_stats.producers_by_kind = defaultdict(int)
2588
- for row in tables.producer_rows:
2589
- tables.producer_stats.producers_by_kind[row.producer_kind] += 1
2590
- # brownfield_strategies = the four brownfield layers plus the two
2591
- # caller-side declaration strategies (@CodebaseHttpClient /
2592
- # @CodebaseProducer). These extend _BROWNFIELD_LAYERS deliberately:
2593
- # the *_from_brownfield_pct stats count annotation-declared callers as
2594
- # brownfield-sourced even though they are not "layers" and so do not
2595
- # gate brownfield_only authoritativeness in _is_brownfield_sourced.
2596
- brownfield_strategies = _BROWNFIELD_LAYERS | frozenset(
2597
- {"codebase_client", "codebase_producer"},
2598
- )
2599
- if tables.call_edge_stats.http_calls_total:
2600
- n_http = sum(
2601
- v for k, v in tables.call_edge_stats.http_calls_by_strategy.items()
2602
- if k in brownfield_strategies
2603
- )
2604
- tables.call_edge_stats.http_clients_from_brownfield_pct = (
2605
- 100.0 * float(n_http) / float(tables.call_edge_stats.http_calls_total)
2606
- )
2607
- if tables.call_edge_stats.async_calls_total:
2608
- n_async = sum(
2609
- v for k, v in tables.call_edge_stats.async_calls_by_strategy.items()
2610
- if k in brownfield_strategies
2611
- )
2612
- tables.call_edge_stats.async_producers_from_brownfield_pct = (
2613
- 100.0 * float(n_async) / float(tables.call_edge_stats.async_calls_total)
2614
- )
2615
- if verbose:
2616
- http_client = dict(sorted(tables.call_edge_stats.http_calls_by_client_kind.items()))
2617
- async_client = dict(sorted(tables.call_edge_stats.async_calls_by_client_kind.items()))
2618
- http_strategy = dict(sorted(tables.call_edge_stats.http_calls_by_strategy.items()))
2619
- async_strategy = dict(sorted(tables.call_edge_stats.async_calls_by_strategy.items()))
2620
- _verbose_stderr_line(
2621
- f"[graph] pass 5 · HTTP_CALLS: {len(tables.http_call_rows)} edges, "
2622
- f"ASYNC_CALLS: {len(tables.async_call_rows)} edges; "
2623
- f"http_by_client_kind={http_client}, async_by_client_kind={async_client}, "
2624
- f"http_by_strategy={http_strategy}, async_by_strategy={async_strategy}",
2625
- )
2626
-
2627
-
2628
- def _match_call_edge(
2629
- call: OutgoingCallDecl,
2630
- routes: list[RouteRow],
2631
- caller_microservice: str,
2632
- ) -> tuple[str, list[RouteRow]]:
2633
- """Return (match_outcome, candidate_routes) for an outgoing call."""
2634
- if (
2635
- (not call.resolved)
2636
- and call.path_template_call == ""
2637
- and call.topic_call == ""
2638
- ):
2639
- return "unresolved", []
2640
-
2641
- candidates: list[RouteRow] = []
2642
- if call.client_kind == CLIENT_KIND_FEIGN_METHOD:
2643
- # Prefer endpoint matching by target service + path/method for Feign declarations.
2644
- path_value = call.path_template_call
2645
- method_value = call.method_call
2646
- if path_value:
2647
- for r in routes:
2648
- if r.kind != "http_endpoint":
2649
- continue
2650
- if call.feign_target_name and r.microservice != call.feign_target_name:
2651
- continue
2652
- if not (r.method == "" or method_value == "" or r.method == method_value):
2653
- continue
2654
- if not r.path_regex:
2655
- continue
2656
- try:
2657
- if re.fullmatch(r.path_regex, path_value or "") is None:
2658
- continue
2659
- except re.error:
2660
- continue
2661
- candidates.append(r)
2662
- if not candidates:
2663
- # Fallback for legacy/manual routes that only expose Feign target names.
2664
- candidates = [
2665
- r for r in routes
2666
- if r.feign_name and call.feign_target_name and r.feign_name == call.feign_target_name
2667
- ]
2668
- elif call.channel == "http":
2669
- path_value = call.path_template_call
2670
- method_value = call.method_call
2671
- for r in routes:
2672
- if r.kind != "http_endpoint":
2673
- continue
2674
- if not (r.method == "" or method_value == "" or r.method == method_value):
2675
- continue
2676
- if not r.path_regex:
2677
- continue
2678
- try:
2679
- if re.fullmatch(r.path_regex, path_value or "") is None:
2680
- continue
2681
- except re.error:
2682
- continue
2683
- candidates.append(r)
2684
- elif call.channel == "async":
2685
- candidates = [
2686
- r for r in routes
2687
- if r.topic == call.topic_call and r.broker == call.broker_call
2688
- ]
2689
-
2690
- if not candidates:
2691
- return "phantom", []
2692
- if len(candidates) > 1:
2693
- return "ambiguous", candidates
2694
- if candidates[0].microservice and candidates[0].microservice == caller_microservice:
2695
- return "intra_service", candidates
2696
- return "cross_service", candidates
2697
-
2698
-
2699
- def _is_brownfield_sourced(
2700
- call_strategy: str,
2701
- candidates: list[RouteRow],
2702
- ) -> bool:
2703
- """Both sides must come from brownfield layers for an edge to count as
2704
- authoritative under brownfield_only mode."""
2705
- if not candidates:
2706
- return False
2707
- if call_strategy not in _BROWNFIELD_LAYERS:
2708
- return False
2709
- return all(
2710
- getattr(c, "source_layer", "builtin") in _BROWNFIELD_LAYERS
2711
- for c in candidates
2712
- )
2713
-
2714
-
2715
- def pass6_match_edges(
2716
- tables: GraphTables,
2717
- *,
2718
- verbose: bool,
2719
- ) -> None:
2720
- match_factor: dict[str, float] = {
2721
- "cross_service": 1.0,
2722
- "intra_service": 0.6,
2723
- "ambiguous": 0.5,
2724
- "phantom": 0.4,
2725
- "unresolved": 0.3,
2726
- }
2727
- route_by_id = {r.id: r for r in tables.routes_rows}
2728
- all_routes = [r for r in tables.routes_rows if r.microservice]
2729
- member_by_id = {m.node_id: m for m in tables.members}
2730
- clients_by_id = {c.id: c for c in tables.client_rows}
2731
- producers_by_id = {p.id: p for p in tables.producer_rows}
2732
- client_hints_by_member: dict[str, list[ClientRow]] = defaultdict(list)
2733
- for edge in tables.declares_client_rows:
2734
- client = clients_by_id.get(edge.client_id)
2735
- if client is None:
2736
- continue
2737
- # `DECLARES_CLIENT.symbol_id` targets `Symbol.id` for member symbols,
2738
- # and member symbols are emitted with `id == MemberEntry.node_id`.
2739
- client_hints_by_member[edge.symbol_id].append(client)
2740
- for member_symbol_id in list(client_hints_by_member.keys()):
2741
- # Deterministic fallback when a method carries multiple feign declarations.
2742
- client_hints_by_member[member_symbol_id].sort(key=lambda c: c.id)
2743
-
2744
- # Pass 6 is idempotent for full rebuilds: each run fully re-derives match outcomes.
2745
- # If incremental rebuild lands later (Tier-2 follow-up), this reset must remain pass-scoped.
2746
- tables.call_edge_stats.http_calls_match_breakdown.clear()
2747
- tables.call_edge_stats.async_calls_match_breakdown.clear()
2748
- tables.call_edge_stats.cross_service_calls_total = 0
2749
-
2750
- brownfield_only = tables.cross_service_resolution == "brownfield_only"
2751
- suppressed_auto_cross_http: list[str] = []
2752
- suppressed_auto_cross_async: list[str] = []
2753
- suppressed_auto_cross_count = 0
2754
-
2755
- def _micro_factor(member: MemberEntry | None) -> float:
2756
- return 1.0 if (member and member.microservice) else 0.85
2757
-
2758
- if verbose:
2759
- _verbose_stderr_line(_PASS6_START)
2760
- with _graph_pass_progress("6/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 6", verbose=verbose):
2761
- for row in tables.http_call_rows:
2762
- if row.match != "unresolved":
2763
- continue
2764
- client = clients_by_id.get(row.client_id)
2765
- member = member_by_id.get(client.member_id) if client else None
2766
- base = row.confidence / max(1e-9, (0.3 * _micro_factor(member)))
2767
- src_route = route_by_id.get(row.route_id)
2768
- if src_route is None and member is not None:
2769
- # Recover feign caller hints from persisted caller-side Client declarations.
2770
- for client in client_hints_by_member.get(member.node_id, ()):
2771
- if client.client_kind != CLIENT_KIND_FEIGN_METHOD:
2772
- continue
2773
- path_template, path_regex = _normalize_path(client.path)
2774
- src_route = RouteRow(
2775
- id="",
2776
- kind="http_consumer",
2777
- framework="feign",
2778
- method=client.method,
2779
- path=client.path,
2780
- path_template=path_template,
2781
- path_regex=path_regex,
2782
- topic="",
2783
- broker="",
2784
- feign_name=client.target_service,
2785
- # `Client` stores service-name hints, not feign URL; matcher keys off feign_name.
2786
- feign_url="",
2787
- microservice=member.microservice,
2788
- module=member.module,
2789
- filename=client.filename,
2790
- start_line=client.start_line,
2791
- end_line=client.end_line,
2792
- resolved=client.resolved,
2793
- source_layer=client.source_layer,
2794
- )
2795
- break
2796
- # Feign caller hints are synthesized as transient `http_consumer` routes in pass6;
2797
- # synthetic phantoms from imperative clients are `http_endpoint` even when `feign_name` is populated from
2798
- # `@CodebaseHttpClient.targetService` / YAML hints — those must path-match like RestTemplate.
2799
- _feign_like = (
2800
- src_route is not None
2801
- and src_route.kind == "http_consumer"
2802
- and bool(src_route.feign_name)
2803
- )
2804
- call = OutgoingCallDecl(
2805
- method_fqn=f"{member.parent_fqn}#{member.decl.signature}" if member else "",
2806
- method_sig=member.decl.signature if member else "",
2807
- client_kind=CLIENT_KIND_FEIGN_METHOD if _feign_like else CLIENT_KIND_REST_TEMPLATE,
2808
- channel="http",
2809
- feign_target_name=src_route.feign_name if src_route else "",
2810
- feign_target_url=src_route.feign_url if src_route else "",
2811
- path_template_call=src_route.path_template if src_route else "",
2812
- method_call=row.method_call,
2813
- topic_call="",
2814
- broker_call="",
2815
- raw_uri=row.raw_uri,
2816
- raw_topic="",
2817
- resolution_strategy=row.strategy,
2818
- confidence_base=base,
2819
- resolved=(row.strategy != "unresolved"),
2820
- filename=member.file_path if member else "",
2821
- start_line=member.decl.start_line if member else 0,
2822
- end_line=member.decl.end_line if member else 0,
2823
- )
2824
- outcome, candidates = _match_call_edge(call, all_routes, member.microservice if member else "")
2825
- if (
2826
- brownfield_only
2827
- and outcome == "cross_service"
2828
- and not _is_brownfield_sourced(row.strategy, candidates)
2829
- ):
2830
- outcome = "unresolved"
2831
- candidates = []
2832
- suppressed_auto_cross_count += 1
2833
- if len(suppressed_auto_cross_http) < 5:
2834
- suppressed_auto_cross_http.append(call.method_fqn)
2835
- if outcome in VALID_HTTP_CALL_MATCHES:
2836
- row.match = outcome
2837
- if outcome in ("cross_service", "intra_service") and len(candidates) == 1:
2838
- row.route_id = candidates[0].id
2839
- row.confidence = call.confidence_base * match_factor[row.match] * _micro_factor(member)
2840
- tables.call_edge_stats.http_calls_match_breakdown[row.match] += 1
2841
- if row.match == "cross_service":
2842
- tables.call_edge_stats.cross_service_calls_total += 1
2843
-
2844
- for row in tables.async_call_rows:
2845
- if row.match != "unresolved":
2846
- continue
2847
- producer = producers_by_id.get(row.producer_id)
2848
- member = member_by_id.get(producer.member_id) if producer else None
2849
- base = row.confidence / max(1e-9, (0.3 * _micro_factor(member)))
2850
- src_route = route_by_id.get(row.route_id)
2851
- async_kind = producer.producer_kind if producer else "kafka_send"
2852
- call = OutgoingCallDecl(
2853
- method_fqn=f"{member.parent_fqn}#{member.decl.signature}" if member else "",
2854
- method_sig=member.decl.signature if member else "",
2855
- client_kind=async_kind,
2856
- channel="async",
2857
- feign_target_name="",
2858
- feign_target_url="",
2859
- path_template_call="",
2860
- method_call="",
2861
- topic_call=src_route.topic if src_route else "",
2862
- broker_call=src_route.broker if src_route else "",
2863
- raw_uri="",
2864
- raw_topic=row.raw_topic,
2865
- resolution_strategy=row.strategy,
2866
- confidence_base=base,
2867
- resolved=(row.strategy != "unresolved"),
2868
- filename=member.file_path if member else "",
2869
- start_line=member.decl.start_line if member else 0,
2870
- end_line=member.decl.end_line if member else 0,
2871
- )
2872
- outcome, candidates = _match_call_edge(call, all_routes, member.microservice if member else "")
2873
- if (
2874
- brownfield_only
2875
- and outcome == "cross_service"
2876
- and not _is_brownfield_sourced(row.strategy, candidates)
2877
- ):
2878
- outcome = "unresolved"
2879
- candidates = []
2880
- suppressed_auto_cross_count += 1
2881
- if len(suppressed_auto_cross_async) < 5:
2882
- suppressed_auto_cross_async.append(call.method_fqn)
2883
- if outcome in VALID_HTTP_CALL_MATCHES:
2884
- row.match = outcome
2885
- if outcome in ("cross_service", "intra_service") and len(candidates) == 1:
2886
- row.route_id = candidates[0].id
2887
- row.confidence = call.confidence_base * match_factor[row.match] * _micro_factor(member)
2888
- tables.call_edge_stats.async_calls_match_breakdown[row.match] += 1
2889
- if row.match == "cross_service":
2890
- tables.call_edge_stats.cross_service_calls_total += 1
2891
-
2892
- inbound_route_ids = {r.route_id for r in tables.http_call_rows} | {r.route_id for r in tables.async_call_rows}
2893
- tables.routes_rows = sorted(
2894
- [
2895
- r for r in tables.routes_rows
2896
- if not (
2897
- (r.microservice == "")
2898
- and (r.framework == "")
2899
- and (not r.resolved)
2900
- and (r.id not in inbound_route_ids)
2901
- )
2902
- ],
2903
- key=lambda r: r.id,
2904
- )
2905
-
2906
- if verbose:
2907
- if brownfield_only:
2908
- n_bf = tables.call_edge_stats.cross_service_calls_total
2909
- first_http = ", ".join(suppressed_auto_cross_http)
2910
- first_async = ", ".join(suppressed_auto_cross_async)
2911
- _verbose_stderr_line(
2912
- f"[graph] pass 6 · cross_service_resolution=brownfield_only:\n"
2913
- f" {n_bf} cross_service edges from brownfield layers,\n"
2914
- f" {suppressed_auto_cross_count} auto-cross-service candidates suppressed -> unresolved\n"
2915
- f" (first 5 http: {first_http})\n"
2916
- f" (first 5 async: {first_async})",
2917
- )
2918
- _verbose_stderr_line(
2919
- f"[graph] pass 6 · http_match={dict(sorted(tables.call_edge_stats.http_calls_match_breakdown.items()))}, "
2920
- f"async_match={dict(sorted(tables.call_edge_stats.async_calls_match_breakdown.items()))}, "
2921
- f"cross_service_calls_total={tables.call_edge_stats.cross_service_calls_total}",
2922
- )
2923
-
2924
-
2925
- # ---------- LadybugDB write ----------
2926
-
2927
-
2928
- _SCHEMA_NODE = (
2929
- "CREATE NODE TABLE Symbol("
2930
- "id STRING PRIMARY KEY, "
2931
- "kind STRING, name STRING, fqn STRING, package STRING, "
2932
- "module STRING, microservice STRING, "
2933
- "filename STRING, start_line INT64, end_line INT64, "
2934
- "start_byte INT64, end_byte INT64, "
2935
- "modifiers STRING[], annotations STRING[], capabilities STRING[], "
2936
- "role STRING, signature STRING, parent_id STRING, resolved BOOLEAN, "
2937
- "generated BOOLEAN, generated_by STRING, "
2938
- "search_text STRING"
2939
- ")"
2940
- )
2941
-
2942
- _SCHEMA_META = (
2943
- "CREATE NODE TABLE GraphMeta("
2944
- "key STRING PRIMARY KEY, "
2945
- "ontology_version INT64, built_at INT64, source_root STRING, "
2946
- "counts_json STRING, parse_errors INT64, "
2947
- "routes_total INT64, exposes_total INT64, "
2948
- # JSON map {framework: count}; STRING avoids LadybugDB Python MAP↔STRUCT binder mismatch.
2949
- "routes_by_framework STRING, "
2950
- "routes_resolved_pct DOUBLE, "
2951
- "routes_from_brownfield_pct DOUBLE, "
2952
- "routes_by_layer STRING, "
2953
- "clients_total INT64, "
2954
- "declares_client_total INT64, "
2955
- "clients_by_kind STRING, "
2956
- "producers_total INT64, "
2957
- "declares_producer_total INT64, "
2958
- "producers_by_kind STRING, "
2959
- "http_calls_total INT64, "
2960
- "async_calls_total INT64, "
2961
- "http_calls_by_strategy STRING, "
2962
- "async_calls_by_strategy STRING, "
2963
- "http_calls_resolved_pct DOUBLE, "
2964
- "async_calls_resolved_pct DOUBLE, "
2965
- "http_clients_from_brownfield_pct DOUBLE, "
2966
- "async_producers_from_brownfield_pct DOUBLE, "
2967
- "http_calls_match_breakdown STRING, "
2968
- "async_calls_match_breakdown STRING, "
2969
- "cross_service_calls_total INT64, "
2970
- "pass3_skipped_cross_service INT64, "
2971
- "pass3_unresolved_phantom_receiver INT64, "
2972
- "pass3_unresolved_chained INT64, "
2973
- "pass4_exposes_suppressed_feign INT64, "
2974
- "cross_service_resolution STRING"
2975
- ")"
2976
- )
2977
-
2978
- _SCHEMA_ROUTE = (
2979
- "CREATE NODE TABLE Route("
2980
- "id STRING, kind STRING, framework STRING, "
2981
- "method STRING, path STRING, path_template STRING, path_regex STRING, "
2982
- "topic STRING, broker STRING, "
2983
- "feign_name STRING, feign_url STRING, "
2984
- "microservice STRING, module STRING, "
2985
- "filename STRING, start_line INT64, end_line INT64, "
2986
- "resolved BOOLEAN, "
2987
- "PRIMARY KEY(id))"
2988
- )
2989
-
2990
- _SCHEMA_CLIENT = (
2991
- "CREATE NODE TABLE Client("
2992
- "id STRING, client_kind STRING, target_service STRING, "
2993
- "path STRING, path_template STRING, path_regex STRING, method STRING, "
2994
- "member_fqn STRING, member_id STRING, "
2995
- "microservice STRING, module STRING, filename STRING, "
2996
- "start_line INT64, end_line INT64, resolved BOOLEAN, source_layer STRING, "
2997
- "PRIMARY KEY(id))"
2998
- )
2999
-
3000
- _SCHEMA_PRODUCER = (
3001
- "CREATE NODE TABLE Producer("
3002
- "id STRING, producer_kind STRING, topic STRING, broker STRING, direction STRING, "
3003
- "member_fqn STRING, member_id STRING, "
3004
- "microservice STRING, module STRING, filename STRING, "
3005
- "start_line INT64, end_line INT64, resolved BOOLEAN, source_layer STRING, "
3006
- "PRIMARY KEY(id))"
3007
- )
3008
-
3009
- _SCHEMA_EXTENDS = (
3010
- "CREATE REL TABLE EXTENDS(FROM Symbol TO Symbol, "
3011
- "source_file STRING, dst_name STRING, dst_fqn STRING, resolved BOOLEAN)"
3012
- )
3013
- _SCHEMA_IMPLEMENTS = (
3014
- "CREATE REL TABLE IMPLEMENTS(FROM Symbol TO Symbol, "
3015
- "source_file STRING, dst_name STRING, dst_fqn STRING, resolved BOOLEAN)"
3016
- )
3017
- _SCHEMA_INJECTS = (
3018
- "CREATE REL TABLE INJECTS(FROM Symbol TO Symbol, "
3019
- "source_file STRING, dst_name STRING, dst_fqn STRING, resolved BOOLEAN, "
3020
- "mechanism STRING, annotation STRING, field_or_param STRING)"
3021
- )
3022
- _SCHEMA_DECLARES = "CREATE REL TABLE DECLARES(FROM Symbol TO Symbol, source_file STRING)"
3023
- _SCHEMA_OVERRIDES = "CREATE REL TABLE OVERRIDES(FROM Symbol TO Symbol, source_file STRING)"
3024
- _SCHEMA_CALLS = (
3025
- "CREATE REL TABLE CALLS(FROM Symbol TO Symbol, "
3026
- "source_file STRING, call_site_line INT64, call_site_byte INT64, arg_count INT64, "
3027
- "confidence DOUBLE, strategy STRING, source STRING, resolved BOOLEAN, "
3028
- "callee_declaring_role STRING)"
3029
- )
3030
- _SCHEMA_UNRESOLVED_CALL_SITE = (
3031
- "CREATE NODE TABLE UnresolvedCallSite("
3032
- "id STRING, caller_id STRING, call_site_line INT64, call_site_byte INT64, "
3033
- "arg_count INT64, callee_simple STRING, receiver_expr STRING, reason STRING, "
3034
- "PRIMARY KEY(id))"
3035
- )
3036
- _SCHEMA_UNRESOLVED_AT = "CREATE REL TABLE UNRESOLVED_AT(FROM Symbol TO UnresolvedCallSite, source_file STRING)"
3037
- _SCHEMA_EXPOSES = (
3038
- "CREATE REL TABLE EXPOSES(FROM Symbol TO Route, "
3039
- "source_file STRING, confidence DOUBLE, strategy STRING)"
3040
- )
3041
- _SCHEMA_DECLARES_CLIENT = (
3042
- "CREATE REL TABLE DECLARES_CLIENT(FROM Symbol TO Client, "
3043
- "source_file STRING, confidence DOUBLE, strategy STRING)"
3044
- )
3045
- _SCHEMA_DECLARES_PRODUCER = (
3046
- "CREATE REL TABLE DECLARES_PRODUCER(FROM Symbol TO Producer, "
3047
- "source_file STRING, confidence DOUBLE, strategy STRING)"
3048
- )
3049
- _SCHEMA_HTTP_CALLS = (
3050
- "CREATE REL TABLE HTTP_CALLS(FROM Client TO Route, "
3051
- "source_file STRING, confidence DOUBLE, strategy STRING, "
3052
- "method_call STRING, raw_uri STRING, match STRING)"
3053
- )
3054
- _SCHEMA_ASYNC_CALLS = (
3055
- "CREATE REL TABLE ASYNC_CALLS(FROM Producer TO Route, "
3056
- "source_file STRING, confidence DOUBLE, strategy STRING, "
3057
- "direction STRING, raw_topic STRING, match STRING)"
3058
- )
3059
-
3060
-
3061
- def _drop_symbol_fts_index_if_present(conn: ladybug.Connection) -> None:
3062
- """Drop the Symbol FTS index so the table drop below succeeds.
3063
-
3064
- LadybugDB refuses ``DROP TABLE Symbol`` while an FTS index references it ("Cannot
3065
- delete node table ... referenced by index"), so the index must go first. No-op when
3066
- FTS is unavailable (offline first-run) or the index was never created — in those
3067
- cases nothing blocks the table drop. Best-effort: any failure is swallowed.
3068
- """
3069
- try:
3070
- conn.execute("LOAD EXTENSION FTS")
3071
- existing = conn.execute("CALL SHOW_INDEXES() RETURN index_name")
3072
- names: set[str] = set()
3073
- while existing.has_next():
3074
- names.add(existing.get_next()[0])
3075
- if _SYMBOL_FTS_INDEX in names:
3076
- conn.execute(f"CALL DROP_FTS_INDEX('Symbol', '{_SYMBOL_FTS_INDEX}')")
3077
- except Exception:
3078
- pass
3079
-
3080
-
3081
- def _drop_all(conn: ladybug.Connection) -> None:
3082
- _drop_symbol_fts_index_if_present(conn)
3083
- for stmt in (
3084
- "DROP TABLE IF EXISTS DECLARES_CLIENT",
3085
- "DROP TABLE IF EXISTS DECLARES_PRODUCER",
3086
- "DROP TABLE IF EXISTS HTTP_CALLS",
3087
- "DROP TABLE IF EXISTS ASYNC_CALLS",
3088
- "DROP TABLE IF EXISTS EXPOSES",
3089
- "DROP TABLE IF EXISTS UNRESOLVED_AT",
3090
- "DROP TABLE IF EXISTS EXTENDS",
3091
- "DROP TABLE IF EXISTS IMPLEMENTS",
3092
- "DROP TABLE IF EXISTS INJECTS",
3093
- "DROP TABLE IF EXISTS CALLS",
3094
- "DROP TABLE IF EXISTS OVERRIDES",
3095
- "DROP TABLE IF EXISTS DECLARES",
3096
- "DROP TABLE IF EXISTS UnresolvedCallSite",
3097
- "DROP TABLE IF EXISTS Symbol",
3098
- "DROP TABLE IF EXISTS Route",
3099
- "DROP TABLE IF EXISTS Client",
3100
- "DROP TABLE IF EXISTS Producer",
3101
- "DROP TABLE IF EXISTS GraphMeta",
3102
- ):
3103
- try:
3104
- conn.execute(stmt)
3105
- except Exception:
3106
- pass
3107
-
3108
-
3109
- def _create_schema(conn: ladybug.Connection) -> None:
3110
- for stmt in (
3111
- _SCHEMA_NODE,
3112
- _SCHEMA_UNRESOLVED_CALL_SITE,
3113
- _SCHEMA_ROUTE,
3114
- _SCHEMA_CLIENT,
3115
- _SCHEMA_PRODUCER,
3116
- _SCHEMA_META,
3117
- _SCHEMA_EXTENDS,
3118
- _SCHEMA_IMPLEMENTS,
3119
- _SCHEMA_INJECTS,
3120
- _SCHEMA_DECLARES,
3121
- _SCHEMA_OVERRIDES,
3122
- _SCHEMA_CALLS,
3123
- _SCHEMA_UNRESOLVED_AT,
3124
- _SCHEMA_EXPOSES,
3125
- _SCHEMA_DECLARES_CLIENT,
3126
- _SCHEMA_DECLARES_PRODUCER,
3127
- _SCHEMA_HTTP_CALLS,
3128
- _SCHEMA_ASYNC_CALLS,
3129
- ):
3130
- conn.execute(stmt)
3131
-
3132
-
3133
- _IDENT_RE = re.compile(r"[A-Za-z][A-Za-z0-9]*")
3134
-
3135
-
3136
- def _compute_symbol_search_text(
3137
- name: str,
3138
- fqn: str,
3139
- signature: str,
3140
- annotations: list[str],
3141
- capabilities: list[str],
3142
- package: str = "",
3143
- ) -> str:
3144
- """Build the BM25-indexable token soup for a Symbol (fork A).
3145
-
3146
- camelCase / snake_case identifiers are split into lowercase tokens via the SAME
3147
- ``_split_identifier`` the query path uses (index- and query-time tokenization
3148
- must agree), then space-joined into one STRING that LadybugDB FTS porter-stems.
3149
- Name + fqn carry the strongest discovery signal; signature / annotations /
3150
- capabilities / package segments are weaker corroborators. Members reach this via
3151
- the same ``_node_row`` constructor as types.
3152
- """
3153
- toks: list[str] = []
3154
- for field in (name, fqn, signature, package):
3155
- for m in _IDENT_RE.findall(str(field or "")):
3156
- toks.extend(_split_identifier(m))
3157
- for lst in (annotations, capabilities):
3158
- for item in (lst or []):
3159
- for m in _IDENT_RE.findall(str(item)):
3160
- toks.extend(_split_identifier(m))
3161
- seen: set[str] = set()
3162
- out: list[str] = []
3163
- for t in toks:
3164
- if len(t) >= 2 and t not in seen:
3165
- seen.add(t)
3166
- out.append(t)
3167
- return " ".join(out)
3168
-
3169
-
3170
- def _ensure_symbol_fts_index(conn: ladybug.Connection, *, verbose: bool) -> None:
3171
- """Best-effort create the Symbol FTS (Okapi BM25) index if absent (fork A).
3172
-
3173
- Idempotent: skips when the index already exists. The FTS extension is fetched
3174
- from extension.ladybugdb.com on first ``INSTALL FTS`` (cached locally after); an
3175
- offline first run fails softly — ``run_lexical_search`` then falls back to the
3176
- heuristic over the bare Symbol scan. The index auto-maintains on later
3177
- COPY / MERGE / DELETE (verified), so this only runs on full builds and as a
3178
- cheap self-heal at the end of an incremental rebuild.
3179
- """
3180
- try:
3181
- try:
3182
- conn.execute("INSTALL FTS") # already-installed raises; swallow
3183
- except Exception:
3184
- pass
3185
- conn.execute("LOAD EXTENSION FTS")
3186
- existing = conn.execute("CALL SHOW_INDEXES() RETURN index_name")
3187
- names: set[str] = set()
3188
- while existing.has_next():
3189
- names.add(existing.get_next()[0])
3190
- if _SYMBOL_FTS_INDEX not in names:
3191
- conn.execute(
3192
- f"CALL CREATE_FTS_INDEX('Symbol', '{_SYMBOL_FTS_INDEX}', "
3193
- "['search_text'], stemmer := 'porter')"
3194
- )
3195
- if verbose:
3196
- _verbose_stderr_line(
3197
- f"[graph] fts · created Symbol {_SYMBOL_FTS_INDEX} (BM25 over search_text)"
3198
- )
3199
- except Exception as e:
3200
- if verbose:
3201
- _verbose_stderr_line(
3202
- f"[graph] fts · unavailable ({type(e).__name__}: {e}); "
3203
- "lexical search will use the heuristic scan"
3204
- )
3205
-
3206
-
3207
- def _node_row(**kwargs) -> dict:
3208
- base = {
3209
- "kind": "", "name": "", "fqn": "", "package": "",
3210
- "module": "", "microservice": "",
3211
- "filename": "", "start_line": 0, "end_line": 0,
3212
- "start_byte": 0, "end_byte": 0,
3213
- "modifiers": [], "annotations": [], "capabilities": [],
3214
- "role": "OTHER", "signature": "", "parent_id": "", "resolved": True,
3215
- "generated": False, "generated_by": None, "search_text": "",
3216
- }
3217
- base.update(kwargs)
3218
- # Derive the BM25 token soup unless the caller set it explicitly.
3219
- if not base.get("search_text"):
3220
- base["search_text"] = _compute_symbol_search_text(
3221
- base.get("name", ""), base.get("fqn", ""), base.get("signature", ""),
3222
- base.get("annotations", []), base.get("capabilities", []),
3223
- base.get("package", ""),
3224
- )
3225
- return base
3226
-
3227
-
3228
- def _bulk_copy(conn: ladybug.Connection, table_name: str, columns: list[str], rows: list[dict]) -> None:
3229
- """Bulk-load rows into a node/rel table via in-memory pyarrow COPY FROM.
3230
-
3231
- `columns` fixes column order; for REL tables the first two MUST be the
3232
- FROM/TO node primary keys (kuzu requirement). Empty `rows` is a no-op.
3233
-
3234
- Spike result (PR-P1 step-1): REL `COPY FROM` expects columns named `FROM` and `TO`
3235
- for the endpoint node IDs, followed by property columns in the declared order.
3236
- `pa.Table.from_pylist(rows)` correctly infers types from the dict values, but
3237
- we must select columns in the exact order expected by the table schema.
3238
- """
3239
- if not rows:
3240
- return
3241
- tbl = pa.Table.from_pylist(rows)
3242
- # Select columns in the exact order expected by the table schema
3243
- tbl = tbl.select(columns)
3244
- conn.execute(f"COPY {table_name} FROM $rows", {"rows": tbl})
3245
-
3246
-
3247
- def _existing_node_ids(conn: ladybug.Connection) -> set[str]:
3248
- """Return every node id (Symbol, Route, Client, Producer) currently in the graph.
3249
-
3250
- Bulk ``COPY FROM`` enforces referential integrity: a REL row whose FROM/TO
3251
- endpoint isn't a loaded node raises ``Unable to find primary key value``. The
3252
- legacy per-row ``MERGE (a:Symbol {id:$src}),(b:Symbol {id:$dst})`` silently
3253
- dropped such edges (a ``MATCH`` against a missing endpoint creates nothing).
3254
- Edge writers filter edge rows against this set to reproduce that exactly.
3255
-
3256
- This queries the live DB rather than just ``tables`` because it is shared
3257
- with the incremental path, whose edges legitimately reference nodes written
3258
- in prior runs.
3259
- """
3260
- result = conn.execute("MATCH (n) RETURN n.id")
3261
- ids: set[str] = set()
3262
- while result.has_next():
3263
- ids.add(result.get_next()[0])
3264
- return ids
3265
-
3266
-
3267
- # Column-order constants for bulk COPY FROM.
3268
- # For REL tables, the first two entries are FROM/TO node primary keys (kuzu requirement).
3269
- # Order matches the corresponding _SCHEMA_* declarations above.
3270
- _NODE_COLUMNS = [
3271
- "id", "kind", "name", "fqn", "package", "module", "microservice",
3272
- "filename", "start_line", "end_line", "start_byte", "end_byte",
3273
- "modifiers", "annotations", "capabilities", "role", "signature", "parent_id", "resolved",
3274
- "generated", "generated_by", "search_text"
3275
- ]
3276
-
3277
- # Type declaration kinds. Tuple (not set) so the rendered SQL `IN` clause is
3278
- # deterministic. Used to (a) load type stubs for cross-file resolution and
3279
- # (b) scope the incremental property-refresh SET to type nodes.
3280
- _TYPE_KINDS: tuple[str, ...] = ("class", "interface", "enum", "annotation", "record")
3281
-
3282
- # Update every mutable Symbol field on an existing node by primary key. Used on
3283
- # the incremental path to refresh preserved dependent type nodes whose
3284
- # `role`/`capabilities` (and other project-wide-derived fields) can shift
3285
- # without their own source changing — restoring the upsert the legacy per-row
3286
- # `MERGE (n:Symbol {id:$id}) SET …` provided. Field list mirrors `_NODE_COLUMNS`
3287
- # minus `id`.
3288
- _SET_SYMBOL_BY_ID = (
3289
- "MATCH (n:Symbol {id: $id}) "
3290
- "SET n.kind = $kind, n.name = $name, n.fqn = $fqn, "
3291
- "n.package = $package, n.module = $module, n.microservice = $microservice, "
3292
- "n.filename = $filename, "
3293
- "n.start_line = $start_line, n.end_line = $end_line, "
3294
- "n.start_byte = $start_byte, n.end_byte = $end_byte, "
3295
- "n.modifiers = $modifiers, n.annotations = $annotations, "
3296
- "n.capabilities = $capabilities, n.role = $role, "
3297
- "n.signature = $signature, n.parent_id = $parent_id, n.resolved = $resolved, "
3298
- "n.generated = $generated, n.generated_by = $generated_by, n.search_text = $search_text"
3299
- )
3300
-
3301
- # Refresh every mutable Route field on an existing Route node by id. Mirrors the
3302
- # `_write_nodes_impl` Symbol pattern (bulk COPY new rows + per-row SET existing
3303
- # ones) so the global pass 5-6 step no longer needs a per-row MERGE upsert.
3304
- # Field list mirrors `_ROUTE_COLUMNS` minus `id`.
3305
- _SET_ROUTE_BY_ID = (
3306
- "MATCH (r:Route {id: $id}) "
3307
- "SET r.kind = $kind, r.framework = $framework, r.method = $method, "
3308
- "r.path = $path, r.path_template = $path_template, r.path_regex = $path_regex, "
3309
- "r.topic = $topic, r.broker = $broker, r.feign_name = $feign_name, r.feign_url = $feign_url, "
3310
- "r.microservice = $microservice, r.module = $module, r.filename = $filename, "
3311
- "r.start_line = $start_line, r.end_line = $end_line, r.resolved = $resolved"
3312
- )
3313
-
3314
- _REL_EXTENDS_COLUMNS = ["FROM", "TO", "source_file", "dst_name", "dst_fqn", "resolved"]
3315
- _REL_IMPLEMENTS_COLUMNS = ["FROM", "TO", "source_file", "dst_name", "dst_fqn", "resolved"]
3316
- _REL_INJECTS_COLUMNS = ["FROM", "TO", "source_file", "dst_name", "dst_fqn", "resolved", "mechanism", "annotation", "field_or_param"]
3317
- _REL_DECLARES_COLUMNS = ["FROM", "TO", "source_file"]
3318
- _REL_OVERRIDES_COLUMNS = ["FROM", "TO", "source_file"]
3319
- _REL_CALLS_COLUMNS = ["FROM", "TO", "source_file", "call_site_line", "call_site_byte", "arg_count", "confidence", "strategy", "source", "resolved", "callee_declaring_role"]
3320
-
3321
- _UNRESOLVED_CALL_SITE_COLUMNS = ["id", "caller_id", "call_site_line", "call_site_byte", "arg_count", "callee_simple", "receiver_expr", "reason"]
3322
- _REL_UNRESOLVED_AT_COLUMNS = ["FROM", "TO", "source_file"]
3323
-
3324
- # Node table column constants (for bulk COPY FROM)
3325
- _ROUTE_COLUMNS = ["id", "kind", "framework", "method", "path", "path_template", "path_regex", "topic", "broker", "feign_name", "feign_url", "microservice", "module", "filename", "start_line", "end_line", "resolved"]
3326
- _CLIENT_COLUMNS = ["id", "client_kind", "target_service", "path", "path_template", "path_regex", "method", "member_fqn", "member_id", "microservice", "module", "filename", "start_line", "end_line", "resolved", "source_layer"]
3327
- _PRODUCER_COLUMNS = ["id", "producer_kind", "topic", "broker", "direction", "member_fqn", "member_id", "microservice", "module", "filename", "start_line", "end_line", "resolved", "source_layer"]
3328
-
3329
- # REL table column constants for routes/clients/producers
3330
- _REL_EXPOSES_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy"]
3331
- _REL_DECLARES_CLIENT_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy"]
3332
- _REL_DECLARES_PRODUCER_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy"]
3333
- _REL_HTTP_CALLS_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy", "method_call", "raw_uri", "match"]
3334
- _REL_ASYNC_CALLS_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy", "direction", "raw_topic", "match"]
3335
-
3336
-
3337
- def _write_nodes_impl(
3338
- conn: ladybug.Connection,
3339
- tables: GraphTables,
3340
- *,
3341
- project_root: Path,
3342
- meta_chain: dict[str, frozenset[str]] | None,
3343
- ) -> None:
3344
- overrides = load_brownfield_overrides(project_root)
3345
- try:
3346
- prs = str(project_root.resolve())
3347
- except OSError:
3348
- prs = str(project_root)
3349
- tables.cross_service_resolution = _load_config_cross_service_resolution(prs)
3350
- mch = meta_chain
3351
-
3352
- # Stage all Symbol rows
3353
- rows: list[dict] = []
3354
- # Node ids loaded from the existing graph as resolution-only stubs
3355
- # (`_load_existing_types`); their staged rows carry placeholder values and
3356
- # must never be written back over the real nodes.
3357
- stub_ids: set[str] = set()
3358
-
3359
- # packages
3360
- for pkg, pid in tables.packages.items():
3361
- rows.append(_node_row(
3362
- id=pid, kind="package", name=pkg.rsplit(".", 1)[-1], fqn=pkg, package=pkg,
3363
- ))
3364
- # files
3365
- for path, fid in tables.files.items():
3366
- rows.append(_node_row(
3367
- id=fid, kind="file", name=Path(path).name, fqn=path, filename=path,
3368
- ))
3369
- # types
3370
- for entry in tables.types.values():
3371
- d = entry.decl
3372
- role, capabilities = resolve_role_and_capabilities(
3373
- d,
3374
- overrides=overrides,
3375
- meta_chain=mch,
3376
- )
3377
- # Read generated/generated_by from pass-1 classification or stub persistence
3378
- generated, generated_by = tables.type_generated_by_node_id.get(entry.node_id, (False, None))
3379
-
3380
- if entry.loaded_from_db:
3381
- stub_ids.add(entry.node_id)
3382
- # Out-of-scope stub: its annotation-less decl collapses role to the
3383
- # default. The real node's role was persisted at index time and seeded
3384
- # into type_role_by_node_id by _load_existing_types; trust it so CALLS
3385
- # edges into this type keep the correct callee_declaring_role (#352).
3386
- # The staged row is filtered out of the write via stub_ids, so its
3387
- # capabilities placeholder never reaches the graph.
3388
- role = tables.type_role_by_node_id.get(entry.node_id, role)
3389
- capabilities = []
3390
- # For stubs, trust the persisted generated/generated_by (seeded by _load_existing_types)
3391
- else:
3392
- tables.type_role_by_node_id[entry.node_id] = role
3393
- rows.append(_node_row(
3394
- id=entry.node_id, kind=d.kind, name=d.name, fqn=d.fqn,
3395
- package=entry.package,
3396
- module=entry.module, microservice=entry.microservice,
3397
- filename=entry.file_path,
3398
- start_line=d.start_line, end_line=d.end_line,
3399
- start_byte=d.start_byte, end_byte=d.end_byte,
3400
- modifiers=list(d.modifiers),
3401
- annotations=[a.name for a in d.annotations],
3402
- capabilities=capabilities,
3403
- role=role,
3404
- signature="",
3405
- parent_id=tables.types[entry.outer_fqn].node_id if entry.outer_fqn and entry.outer_fqn in tables.types else "",
3406
- generated=generated,
3407
- generated_by=generated_by,
3408
- ))
3409
- # members (methods / constructors)
3410
- for m in tables.members:
3411
- rows.append(_node_row(
3412
- id=m.node_id, kind=m.kind, name=m.decl.name,
3413
- fqn=f"{m.parent_fqn}#{m.decl.signature}",
3414
- package=tables.types[m.parent_fqn].package if m.parent_fqn in tables.types else "",
3415
- module=m.module, microservice=m.microservice,
3416
- filename=m.file_path,
3417
- start_line=m.decl.start_line, end_line=m.decl.end_line,
3418
- start_byte=m.decl.start_byte, end_byte=m.decl.end_byte,
3419
- modifiers=list(m.decl.modifiers),
3420
- annotations=[a.name for a in m.decl.annotations],
3421
- signature=m.decl.signature, parent_id=m.parent_id,
3422
- ))
3423
- # phantoms
3424
- for pid, row in tables.phantoms.items():
3425
- rows.append(row)
3426
-
3427
- # Bulk-load new Symbol rows. The full-rebuild path starts from an empty
3428
- # database (`_drop_all`), so every row is new. The incremental path reaches
3429
- # here with a populated database: changed-file nodes were deleted by
3430
- # `_delete_file_scope` (absent here → new), while dependent-file nodes are
3431
- # deliberately preserved (see `_delete_file_scope` / issue #305).
3432
- existing_ids = _existing_node_ids(conn)
3433
- new_rows = [row for row in rows if row["id"] not in existing_ids]
3434
- _bulk_copy(conn, "Symbol", _NODE_COLUMNS, new_rows)
3435
-
3436
- # Refresh mutable properties on preserved dependent TYPE nodes (incremental
3437
- # path only; `update_rows` is empty on the full path). `role`/`capabilities`
3438
- # — and any other field derived from project-wide inputs (meta-annotation
3439
- # chain, brownfield overrides) — can shift without the type's own source
3440
- # changing, so a preserved dependent must be re-SET to stay byte-equivalent
3441
- # with a full rebuild. The legacy per-row `_MERGE_SYMBOL` upserted every
3442
- # staged node and did this implicitly; bulk `COPY FROM` only appends, so the
3443
- # SET is explicit here. Stubs (`stub_ids`) are skipped: their decl is a
3444
- # placeholder and their stored values are authoritative. Non-type kinds
3445
- # carry no mutable role/capabilities, so they are skipped too.
3446
- update_rows = [
3447
- row for row in rows
3448
- if row["id"] in existing_ids
3449
- and row["id"] not in stub_ids
3450
- and row["kind"] in _TYPE_KINDS
3451
- ]
3452
- for row in update_rows:
3453
- conn.execute(_SET_SYMBOL_BY_ID, row)
3454
-
3455
-
3456
- def _write_nodes(
3457
- conn: ladybug.Connection,
3458
- tables: GraphTables,
3459
- *,
3460
- project_root: Path,
3461
- meta_chain: dict[str, frozenset[str]] | None,
3462
- ) -> None:
3463
- _write_nodes_impl(conn, tables, project_root=project_root, meta_chain=meta_chain)
3464
-
3465
-
3466
-
3467
-
3468
- def _populate_declares_rows(tables: GraphTables) -> None:
3469
- # Skip members loaded from the existing graph for cross-file resolution: a
3470
- # DECLARES edge for an unchanged-file member already persists (its
3471
- # source_file is out of scope, so `_delete_file_scope` left it), and
3472
- # re-emitting it would append a duplicate (REL tables carry no primary key).
3473
- # Full-rebuild never loads members, so this is a no-op there.
3474
- tables.declares_rows = [
3475
- DeclaresRow(src_id=m.parent_id, dst_id=m.node_id)
3476
- for m in tables.members
3477
- if not m.loaded_from_db
3478
- ]
3479
-
3480
-
3481
- def _direct_supertype_ids(tables: GraphTables, type_id: str) -> list[str]:
3482
- out: list[str] = []
3483
- for r in tables.extends_rows:
3484
- if r.src_id == type_id:
3485
- out.append(r.dst_id)
3486
- for r in tables.implements_rows:
3487
- if r.src_id == type_id:
3488
- out.append(r.dst_id)
3489
- return out
3490
-
3491
-
3492
- def _populate_overrides_rows(tables: GraphTables) -> None:
3493
- """Materialize (subtype_method)-[:OVERRIDES]->(supertype_method) for one supertype hop.
3494
-
3495
- Matches ``LadybugDBGraph.override_axis_rollup_for`` (direct ``IMPLEMENTS`` / ``EXTENDS``
3496
- only, same ``signature``, distinct method ids, non-static instance methods).
3497
- """
3498
- by_declaring_type: dict[str, list[MemberEntry]] = defaultdict(list)
3499
- for m in tables.members:
3500
- by_declaring_type[m.parent_id].append(m)
3501
- pairs: set[tuple[str, str]] = set()
3502
- for m in tables.members:
3503
- if m.kind != "method" or "static" in m.decl.modifiers:
3504
- continue
3505
- impl_tid = m.parent_id
3506
- for sup_id in _direct_supertype_ids(tables, impl_tid):
3507
- for other in by_declaring_type.get(sup_id, ()):
3508
- if other.kind != "method":
3509
- continue
3510
- if other.decl.signature != m.decl.signature:
3511
- continue
3512
- if other.node_id == m.node_id:
3513
- continue
3514
- pairs.add((m.node_id, other.node_id))
3515
- tables.overrides_rows = [
3516
- DeclaresRow(src_id=a, dst_id=b) for a, b in sorted(pairs)
3517
- ]
3518
-
3519
-
3520
- def _build_file_by_node_id(tables: GraphTables) -> dict[str, str]:
3521
- """Build node_id -> file_path lookup for source_file resolution."""
3522
- lookup: dict[str, str] = {}
3523
- for entry in tables.types.values():
3524
- lookup[entry.node_id] = entry.file_path
3525
- for m in tables.members:
3526
- lookup[m.node_id] = m.file_path
3527
- return lookup
3528
-
3529
-
3530
- def _write_edges(conn: ladybug.Connection, tables: GraphTables, _file_by_node_id: dict[str, str] | None = None) -> None:
3531
- # Build node_id -> file_path lookup for source_file resolution.
3532
- if _file_by_node_id is None:
3533
- _file_by_node_id = _build_file_by_node_id(tables)
3534
-
3535
- # Bulk COPY FROM enforces referential integrity — a REL row whose endpoint
3536
- # node isn't loaded raises "Unable to find primary key value". The legacy
3537
- # per-row MERGE silently skipped such edges; drop them here to preserve the
3538
- # per-row graph exactly. _existing_node_ids reads the live DB (not just
3539
- # `tables`) so the incremental path's references to prior-run nodes still hold.
3540
- valid_ids = _existing_node_ids(conn)
3541
-
3542
- # Stage EXTENDS rows
3543
- extends_rows = [
3544
- {
3545
- "FROM": r.src_id, "TO": r.dst_id,
3546
- "source_file": _file_by_node_id.get(r.src_id, ""),
3547
- "dst_name": r.dst_name, "dst_fqn": r.dst_fqn, "resolved": r.resolved,
3548
- }
3549
- for r in tables.extends_rows
3550
- if r.src_id in valid_ids and r.dst_id in valid_ids
3551
- ]
3552
- _bulk_copy(conn, "EXTENDS", _REL_EXTENDS_COLUMNS, extends_rows)
3553
-
3554
- # Stage IMPLEMENTS rows
3555
- implements_rows = [
3556
- {
3557
- "FROM": r.src_id, "TO": r.dst_id,
3558
- "source_file": _file_by_node_id.get(r.src_id, ""),
3559
- "dst_name": r.dst_name, "dst_fqn": r.dst_fqn, "resolved": r.resolved,
3560
- }
3561
- for r in tables.implements_rows
3562
- if r.src_id in valid_ids and r.dst_id in valid_ids
3563
- ]
3564
- _bulk_copy(conn, "IMPLEMENTS", _REL_IMPLEMENTS_COLUMNS, implements_rows)
3565
-
3566
- # Stage INJECTS rows
3567
- injects_rows = [
3568
- {
3569
- "FROM": r.src_id, "TO": r.dst_id,
3570
- "source_file": _file_by_node_id.get(r.src_id, ""),
3571
- "dst_name": r.dst_name, "dst_fqn": r.dst_fqn, "resolved": r.resolved,
3572
- "mechanism": r.mechanism, "annotation": r.annotation,
3573
- "field_or_param": r.field_or_param,
3574
- }
3575
- for r in tables.injects_rows
3576
- if r.src_id in valid_ids and r.dst_id in valid_ids
3577
- ]
3578
- _bulk_copy(conn, "INJECTS", _REL_INJECTS_COLUMNS, injects_rows)
3579
-
3580
- # Stage DECLARES rows
3581
- declares_rows = [
3582
- {
3583
- "FROM": row.src_id, "TO": row.dst_id,
3584
- "source_file": _file_by_node_id.get(row.src_id, ""),
3585
- }
3586
- for row in tables.declares_rows
3587
- if row.src_id in valid_ids and row.dst_id in valid_ids
3588
- ]
3589
- _bulk_copy(conn, "DECLARES", _REL_DECLARES_COLUMNS, declares_rows)
3590
-
3591
- # Stage OVERRIDES rows
3592
- overrides_rows = [
3593
- {
3594
- "FROM": row.src_id, "TO": row.dst_id,
3595
- "source_file": _file_by_node_id.get(row.src_id, ""),
3596
- }
3597
- for row in tables.overrides_rows
3598
- if row.src_id in valid_ids and row.dst_id in valid_ids
3599
- ]
3600
- _bulk_copy(conn, "OVERRIDES", _REL_OVERRIDES_COLUMNS, overrides_rows)
3601
-
3602
- # Stage CALLS rows with dedup and callee_declaring_role materialization
3603
- seen_calls: set[tuple[str, str, int, int, int]] = set()
3604
- calls_rows: list[dict] = []
3605
- member_by_id = {m.node_id: m for m in tables.members}
3606
- for row in tables.calls_rows:
3607
- if row.src_id not in valid_ids or row.dst_id not in valid_ids:
3608
- continue
3609
- # Include call_site_byte so two call sites of the same method on the same
3610
- # source line (same arg_count) are kept as distinct edges (issue #359).
3611
- key = (row.src_id, row.dst_id, row.arg_count, row.call_site_line, row.call_site_byte)
3612
- if key in seen_calls:
3613
- continue
3614
- seen_calls.add(key)
3615
- calls_rows.append({
3616
- "FROM": row.src_id, "TO": row.dst_id,
3617
- "source_file": _file_by_node_id.get(row.src_id, ""),
3618
- "call_site_line": row.call_site_line, "call_site_byte": row.call_site_byte,
3619
- "arg_count": row.arg_count, "confidence": row.confidence, "strategy": row.strategy,
3620
- "source": row.source, "resolved": row.resolved,
3621
- "callee_declaring_role": _callee_declaring_role_at_write(
3622
- tables, row.dst_id, member_by_id=member_by_id,
3623
- ),
3624
- })
3625
- _bulk_copy(conn, "CALLS", _REL_CALLS_COLUMNS, calls_rows)
3626
-
3627
- # Stage UnresolvedCallSite node rows (must load before UNRESOLVED_AT edges)
3628
- seen_ucs: set[str] = set()
3629
- ucs_rows: list[dict] = []
3630
- for row in tables.unresolved_call_site_rows:
3631
- if row.id in seen_ucs:
3632
- continue
3633
- seen_ucs.add(row.id)
3634
- ucs_rows.append({
3635
- "id": row.id,
3636
- "caller_id": row.caller_id,
3637
- "call_site_line": row.call_site_line,
3638
- "call_site_byte": row.call_site_byte,
3639
- "arg_count": row.arg_count,
3640
- "callee_simple": row.callee_simple,
3641
- "receiver_expr": row.receiver_expr,
3642
- "reason": row.reason,
3643
- })
3644
- _bulk_copy(conn, "UnresolvedCallSite", _UNRESOLVED_CALL_SITE_COLUMNS, ucs_rows)
3645
-
3646
- # Stage UNRESOLVED_AT edge rows (one per unique UnresolvedCallSite node)
3647
- # Use the same ucs_rows list to ensure 1:1 correspondence
3648
- unresolved_at_rows = [
3649
- {
3650
- "FROM": ucs_row["caller_id"], "TO": ucs_row["id"],
3651
- "source_file": _file_by_node_id.get(ucs_row["caller_id"], ""),
3652
- }
3653
- for ucs_row in ucs_rows
3654
- if ucs_row["caller_id"] in valid_ids
3655
- ]
3656
- _bulk_copy(conn, "UNRESOLVED_AT", _REL_UNRESOLVED_AT_COLUMNS, unresolved_at_rows)
3657
-
3658
-
3659
- def _write_routes_and_exposes(conn: ladybug.Connection, tables: GraphTables, _file_by_node_id: dict[str, str] | None = None) -> None:
3660
- # Build node_id -> file_path lookup for source_file resolution (for Symbol sources).
3661
- if _file_by_node_id is None:
3662
- _file_by_node_id = _build_file_by_node_id(tables)
3663
-
3664
- # Build client_id -> filename lookup for HTTP_CALLS source_file.
3665
- _file_by_client_id: dict[str, str] = {row.id: row.filename for row in tables.client_rows}
3666
-
3667
- # Build producer_id -> filename lookup for ASYNC_CALLS source_file.
3668
- _file_by_producer_id: dict[str, str] = {row.id: row.filename for row in tables.producer_rows}
3669
-
3670
- # Bulk COPY FROM enforces referential integrity — get all valid node IDs
3671
- valid_ids = _existing_node_ids(conn)
3672
-
3673
- # Stage Route node rows (bulk-load before edges that reference them)
3674
- route_rows = [
3675
- {
3676
- "id": row.id,
3677
- "kind": row.kind,
3678
- "framework": row.framework,
3679
- "method": row.method,
3680
- "path": row.path,
3681
- "path_template": row.path_template,
3682
- "path_regex": row.path_regex,
3683
- "topic": row.topic,
3684
- "broker": row.broker,
3685
- "feign_name": row.feign_name,
3686
- "feign_url": row.feign_url,
3687
- "microservice": row.microservice,
3688
- "module": row.module,
3689
- "filename": row.filename,
3690
- "start_line": row.start_line,
3691
- "end_line": row.end_line,
3692
- "resolved": row.resolved,
3693
- }
3694
- for row in tables.routes_rows
3695
- ]
3696
- _bulk_copy(conn, "Route", _ROUTE_COLUMNS, route_rows)
3697
-
3698
- # Stage Client node rows (bulk-load before edges that reference them)
3699
- client_rows = [asdict(row) for row in tables.client_rows]
3700
- _bulk_copy(conn, "Client", _CLIENT_COLUMNS, client_rows)
3701
-
3702
- # Stage Producer node rows (bulk-load before edges that reference them)
3703
- producer_rows = [asdict(row) for row in tables.producer_rows]
3704
- _bulk_copy(conn, "Producer", _PRODUCER_COLUMNS, producer_rows)
3705
-
3706
- # Re-fetch valid IDs after loading Route/Client/Producer nodes (now includes them)
3707
- valid_ids = _existing_node_ids(conn)
3708
-
3709
- # Stage EXPOSES edge rows (Symbol -> Route)
3710
- exposes_rows = [
3711
- {
3712
- "FROM": row.symbol_id,
3713
- "TO": row.route_id,
3714
- "source_file": _file_by_node_id.get(row.symbol_id, ""),
3715
- "confidence": row.confidence,
3716
- "strategy": row.strategy,
3717
- }
3718
- for row in tables.exposes_rows
3719
- if row.symbol_id in valid_ids and row.route_id in valid_ids
3720
- ]
3721
- _bulk_copy(conn, "EXPOSES", _REL_EXPOSES_COLUMNS, exposes_rows)
3722
-
3723
- # Stage DECLARES_CLIENT edge rows (Symbol -> Client)
3724
- declares_client_rows = [
3725
- {
3726
- "FROM": row.symbol_id,
3727
- "TO": row.client_id,
3728
- "source_file": _file_by_node_id.get(row.symbol_id, ""),
3729
- "confidence": row.confidence,
3730
- "strategy": row.strategy,
3731
- }
3732
- for row in tables.declares_client_rows
3733
- if row.symbol_id in valid_ids and row.client_id in valid_ids
3734
- ]
3735
- _bulk_copy(conn, "DECLARES_CLIENT", _REL_DECLARES_CLIENT_COLUMNS, declares_client_rows)
3736
-
3737
- # Stage DECLARES_PRODUCER edge rows (Symbol -> Producer)
3738
- declares_producer_rows = [
3739
- {
3740
- "FROM": row.symbol_id,
3741
- "TO": row.producer_id,
3742
- "source_file": _file_by_node_id.get(row.symbol_id, ""),
3743
- "confidence": row.confidence,
3744
- "strategy": row.strategy,
3745
- }
3746
- for row in tables.declares_producer_rows
3747
- if row.symbol_id in valid_ids and row.producer_id in valid_ids
3748
- ]
3749
- _bulk_copy(conn, "DECLARES_PRODUCER", _REL_DECLARES_PRODUCER_COLUMNS, declares_producer_rows)
3750
-
3751
- # Stage HTTP_CALLS edge rows (Client -> Route)
3752
- http_call_rows = [
3753
- {
3754
- "FROM": row.client_id,
3755
- "TO": row.route_id,
3756
- "source_file": _file_by_client_id.get(row.client_id, ""),
3757
- "confidence": row.confidence,
3758
- "strategy": row.strategy,
3759
- "method_call": row.method_call,
3760
- "raw_uri": row.raw_uri,
3761
- "match": row.match,
3762
- }
3763
- for row in tables.http_call_rows
3764
- if row.client_id in valid_ids and row.route_id in valid_ids
3765
- ]
3766
- _bulk_copy(conn, "HTTP_CALLS", _REL_HTTP_CALLS_COLUMNS, http_call_rows)
3767
-
3768
- # Stage ASYNC_CALLS edge rows (Producer -> Route)
3769
- async_call_rows = [
3770
- {
3771
- "FROM": row.producer_id,
3772
- "TO": row.route_id,
3773
- "source_file": _file_by_producer_id.get(row.producer_id, ""),
3774
- "confidence": row.confidence,
3775
- "strategy": row.strategy,
3776
- "direction": row.direction,
3777
- "raw_topic": row.raw_topic,
3778
- "match": row.match,
3779
- }
3780
- for row in tables.async_call_rows
3781
- if row.producer_id in valid_ids and row.route_id in valid_ids
3782
- ]
3783
- _bulk_copy(conn, "ASYNC_CALLS", _REL_ASYNC_CALLS_COLUMNS, async_call_rows)
3784
-
3785
-
3786
- def _write_meta(conn: ladybug.Connection, tables: GraphTables, source_root: Path) -> None:
3787
- # Dedup key MUST match _write_edges (build_ast_graph.py, _REL_CALLS writer): the
3788
- # 5-tuple includes call_site_byte so two call sites of the same method on the
3789
- # same source line are counted separately. A previous version used the 4-tuple
3790
- # here, which made counts['calls'] (678) diverge from the real CALLS edge count
3791
- # (684) that _write_edges actually persisted — describe/stats then undercounted.
3792
- seen_calls: set[tuple[str, str, int, int, int]] = set()
3793
- calls_unique = 0
3794
- for row in tables.calls_rows:
3795
- key = (row.src_id, row.dst_id, row.arg_count, row.call_site_line, row.call_site_byte)
3796
- if key not in seen_calls:
3797
- seen_calls.add(key)
3798
- calls_unique += 1
3799
- st = tables.route_stats
3800
- routes_fw = dict(sorted(st.by_framework.items()))
3801
- call_stats = tables.call_edge_stats
3802
- client_stats = tables.client_stats
3803
- producer_stats = tables.producer_stats
3804
- http_by_strategy = dict(sorted(call_stats.http_calls_by_strategy.items()))
3805
- async_by_strategy = dict(sorted(call_stats.async_calls_by_strategy.items()))
3806
- http_match = dict(sorted(call_stats.http_calls_match_breakdown.items()))
3807
- async_match = dict(sorted(call_stats.async_calls_match_breakdown.items()))
3808
- http_resolved_pct = 0.0
3809
- async_resolved_pct = 0.0
3810
- if call_stats.http_calls_total:
3811
- # PR-D1 definition: "resolved_pct" is strategy-based (strategy != 'unresolved'),
3812
- # not match-based (all PR-D1 edges keep match='unresolved').
3813
- resolved_http = sum(v for k, v in call_stats.http_calls_by_strategy.items() if k != "unresolved")
3814
- http_resolved_pct = float(resolved_http) / float(call_stats.http_calls_total)
3815
- if call_stats.async_calls_total:
3816
- resolved_async = sum(v for k, v in call_stats.async_calls_by_strategy.items() if k != "unresolved")
3817
- async_resolved_pct = float(resolved_async) / float(call_stats.async_calls_total)
3818
- counts = {
3819
- "packages": len(tables.packages),
3820
- "files": len(tables.files),
3821
- "types": len(tables.types),
3822
- "members": len(tables.members),
3823
- "phantoms": len(tables.phantoms),
3824
- "extends": len(tables.extends_rows),
3825
- "implements": len(tables.implements_rows),
3826
- "injects": len(tables.injects_rows),
3827
- "declares": len(tables.declares_rows),
3828
- "overrides": len(tables.overrides_rows),
3829
- "calls": calls_unique,
3830
- "routes": len(tables.routes_rows),
3831
- "exposes": len(tables.exposes_rows),
3832
- "clients": len(tables.client_rows),
3833
- "declares_client": len(tables.declares_client_rows),
3834
- "producers": len(tables.producer_rows),
3835
- "declares_producer": len(tables.declares_producer_rows),
3836
- "http_calls": len(tables.http_call_rows),
3837
- "async_calls": len(tables.async_call_rows),
3838
- }
3839
- routes_layer = dict(sorted(st.routes_by_layer.items()))
3840
- clients_by_kind = dict(sorted(client_stats.clients_by_kind.items()))
3841
- producers_by_kind = dict(sorted(producer_stats.producers_by_kind.items()))
3842
- conn.execute(
3843
- "MERGE (m:GraphMeta {key: $k}) "
3844
- "SET m.ontology_version = $ov, m.built_at = $t, "
3845
- "m.source_root = $sr, m.counts_json = $cj, m.parse_errors = $pe, "
3846
- "m.routes_total = $routes_total, m.exposes_total = $exposes_total, "
3847
- "m.routes_by_framework = $routes_by_framework, m.routes_resolved_pct = $routes_resolved_pct, "
3848
- "m.routes_from_brownfield_pct = $routes_from_brownfield_pct, m.routes_by_layer = $routes_by_layer, "
3849
- "m.clients_total = $clients_total, m.declares_client_total = $declares_client_total, "
3850
- "m.clients_by_kind = $clients_by_kind, "
3851
- "m.producers_total = $producers_total, m.declares_producer_total = $declares_producer_total, "
3852
- "m.producers_by_kind = $producers_by_kind, "
3853
- "m.http_calls_total = $http_calls_total, m.async_calls_total = $async_calls_total, "
3854
- "m.http_calls_by_strategy = $http_calls_by_strategy, m.async_calls_by_strategy = $async_calls_by_strategy, "
3855
- "m.http_calls_resolved_pct = $http_calls_resolved_pct, m.async_calls_resolved_pct = $async_calls_resolved_pct, "
3856
- "m.http_clients_from_brownfield_pct = $http_clients_from_brownfield_pct, "
3857
- "m.async_producers_from_brownfield_pct = $async_producers_from_brownfield_pct, "
3858
- "m.http_calls_match_breakdown = $http_calls_match_breakdown, "
3859
- "m.async_calls_match_breakdown = $async_calls_match_breakdown, "
3860
- "m.cross_service_calls_total = $cross_service_calls_total, "
3861
- "m.pass3_skipped_cross_service = $pass3_skipped_cross_service, "
3862
- "m.pass3_unresolved_phantom_receiver = $pass3_unresolved_phantom_receiver, "
3863
- "m.pass3_unresolved_chained = $pass3_unresolved_chained, "
3864
- "m.pass4_exposes_suppressed_feign = $pass4_exposes_suppressed_feign, "
3865
- "m.cross_service_resolution = $cross_service_resolution",
3866
- {
3867
- "k": "graph",
3868
- "ov": ONTOLOGY_VERSION,
3869
- "t": int(time.time()),
3870
- "sr": str(source_root.resolve()),
3871
- "cj": json.dumps(counts),
3872
- "pe": tables.parse_errors,
3873
- "routes_total": len(tables.routes_rows),
3874
- "exposes_total": len(tables.exposes_rows),
3875
- "routes_by_framework": json.dumps(routes_fw),
3876
- "routes_resolved_pct": float(st.routes_resolved_pct),
3877
- "routes_from_brownfield_pct": float(st.routes_from_brownfield_pct),
3878
- "routes_by_layer": json.dumps(routes_layer),
3879
- "clients_total": int(client_stats.clients_total),
3880
- "declares_client_total": int(client_stats.declares_client_total),
3881
- "clients_by_kind": json.dumps(clients_by_kind),
3882
- "producers_total": int(producer_stats.producers_total),
3883
- "declares_producer_total": int(producer_stats.declares_producer_total),
3884
- "producers_by_kind": json.dumps(producers_by_kind),
3885
- "http_calls_total": call_stats.http_calls_total,
3886
- "async_calls_total": call_stats.async_calls_total,
3887
- "http_calls_by_strategy": json.dumps(http_by_strategy),
3888
- "async_calls_by_strategy": json.dumps(async_by_strategy),
3889
- "http_calls_resolved_pct": http_resolved_pct,
3890
- "async_calls_resolved_pct": async_resolved_pct,
3891
- "http_clients_from_brownfield_pct": call_stats.http_clients_from_brownfield_pct,
3892
- "async_producers_from_brownfield_pct": call_stats.async_producers_from_brownfield_pct,
3893
- "http_calls_match_breakdown": json.dumps(http_match),
3894
- "async_calls_match_breakdown": json.dumps(async_match),
3895
- "cross_service_calls_total": int(call_stats.cross_service_calls_total),
3896
- "pass3_skipped_cross_service": int(tables.pass3_skipped_cross_service),
3897
- "pass3_unresolved_phantom_receiver": int(tables.pass3_unresolved_phantom_receiver),
3898
- "pass3_unresolved_chained": int(tables.pass3_unresolved_chained),
3899
- "pass4_exposes_suppressed_feign": int(st.exposes_suppressed_feign),
3900
- "cross_service_resolution": str(tables.cross_service_resolution),
3901
- },
3902
- )
3903
-
3904
-
3905
- def incremental_rebuild(
3906
- source_root: Path,
3907
- ladybug_path: Path,
3908
- *,
3909
- verbose: bool,
3910
- expansion_cap: int = 50,
3911
- ) -> IncrementalResult:
3912
- """Incrementally rebuild the LadybugDB graph, processing only changed files and their dependents.
3913
-
3914
- Returns IncrementalResult with statistics about the rebuild.
3915
- Falls back to full rebuild if:
3916
- - No previous graph exists
3917
- - Ontology version < ONTOLOGY_VERSION (stale schema; rebuild for current columns)
3918
- - Crash marker exists (previous incremental run failed)
3919
- - Dependent expansion exceeds expansion_cap
3920
- """
3921
- t_start = time.time()
3922
-
3923
- # Step 1: Load existing graph and detect changes
3924
- if not ladybug_path.exists():
3925
- if verbose:
3926
- _verbose_stderr_line("[increment] no existing graph; falling back to full rebuild")
3927
- # Fall back to full rebuild
3928
- tables = GraphTables()
3929
- asts = pass1_parse(source_root, tables, verbose=verbose)
3930
- pass2_edges(tables, asts, verbose=verbose)
3931
- pass3_calls(tables, asts, verbose=verbose)
3932
- pass4_routes(tables, asts, source_root=source_root, verbose=verbose)
3933
- pass5_imperative_edges(tables, asts, source_root=source_root, verbose=verbose)
3934
- pass6_match_edges(tables, verbose=verbose)
3935
- write_ladybug(ladybug_path, tables, source_root=source_root, verbose=verbose)
3936
-
3937
- return IncrementalResult(
3938
- mode="full_fallback",
3939
- files_changed=0,
3940
- files_added=0,
3941
- files_removed=0,
3942
- dependents_reprocessed=0,
3943
- elapsed_sec=time.time() - t_start,
3944
- )
3945
-
3946
- db = ladybug.Database(str(ladybug_path))
3947
- conn = ladybug.Connection(db)
3948
- # If a Symbol FTS index exists, the scoped DELETE/MERGE/COPY below must maintain it,
3949
- # which needs the FTS extension loaded on THIS connection. Best-effort: when FTS is
3950
- # unavailable no index exists, so there's nothing to maintain and DML proceeds fine.
3951
- try:
3952
- conn.execute("LOAD EXTENSION FTS")
3953
- except Exception:
3954
- pass
3955
-
3956
- # Check ontology version
3957
- try:
3958
- meta_result = conn.execute("MATCH (m:GraphMeta) RETURN m.ontology_version AS version")
3959
- if meta_result.has_next():
3960
- row = meta_result.get_next()
3961
- version = row[0] if row else 0
3962
- if version < ONTOLOGY_VERSION:
3963
- if verbose:
3964
- _verbose_stderr_line(f"[increment] ontology version {version} < {ONTOLOGY_VERSION}; falling back to full rebuild")
3965
- conn.close()
3966
- db.close()
3967
- del conn, db
3968
- return _fallback_to_full(source_root, ladybug_path, verbose, t_start)
3969
- except Exception as e:
3970
- if verbose:
3971
- _verbose_stderr_line(f"[increment] failed to read ontology version: {e}; falling back to full rebuild")
3972
- try:
3973
- conn.close()
3974
- db.close()
3975
- except Exception:
3976
- pass
3977
- del conn, db
3978
- return _fallback_to_full(source_root, ladybug_path, verbose, t_start)
3979
-
3980
- index_dir = ladybug_path.parent
3981
- tracker = FileHashTracker(index_dir)
3982
- tracker.load()
3983
-
3984
- ignore = LayeredIgnore(source_root)
3985
- added, changed, removed = tracker.detect_changes(source_root, ignore=ignore)
3986
-
3987
- changed_files = added | changed | removed
3988
-
3989
- if not changed_files:
3990
- if verbose:
3991
- _verbose_stderr_line("[increment] no changes detected; no-op")
3992
- conn.close()
3993
- db.close()
3994
- return IncrementalResult(
3995
- mode="incremental",
3996
- files_changed=0,
3997
- files_added=0,
3998
- files_removed=0,
3999
- dependents_reprocessed=0,
4000
- elapsed_sec=time.time() - t_start,
4001
- )
4002
-
4003
- if verbose:
4004
- _verbose_stderr_line(f"[increment] detected {len(added)} added, {len(changed)} changed, {len(removed)} removed files")
4005
-
4006
- # Step 2: Crash marker check
4007
- crash_marker_path = index_dir / GRAPH_INCREMENT_MARKER_FILENAME
4008
- if crash_marker_path.exists():
4009
- if verbose:
4010
- _verbose_stderr_line("[increment] crash marker exists; falling back to full rebuild")
4011
- conn.close()
4012
- db.close()
4013
- crash_marker_path.unlink(missing_ok=True)
4014
- return _fallback_to_full(source_root, ladybug_path, verbose, t_start)
4015
-
4016
- # Write crash marker
4017
- crash_marker_path.write_text("", encoding="utf-8")
4018
-
4019
- try:
4020
- # Step 3: Dependent expansion
4021
- # Collect node IDs for changed files (single query instead of N+1)
4022
- changed_node_ids: set[str] = set()
4023
- result = conn.execute(
4024
- "MATCH (s:Symbol) WHERE s.filename IN $filenames RETURN s.id",
4025
- {"filenames": list(changed_files)},
4026
- )
4027
- while result.has_next():
4028
- row = result.get_next()
4029
- changed_node_ids.add(row[0])
4030
-
4031
- # Find dependents
4032
- dependent_files = _find_dependents(conn, changed_node_ids)
4033
-
4034
- # Annotation-definition change: also pull in files that USE the changed
4035
- # annotation. Annotation usage is a node property, not a Symbol->Symbol
4036
- # edge, so `_find_dependents` misses them and their role (derived from
4037
- # the project-wide meta-chain) would go stale. See PR-P5b.
4038
- dependent_files |= _find_annotation_dependents(conn, changed_node_ids)
4039
-
4040
- # Union changed files with dependents
4041
- scope_files = changed_files | dependent_files
4042
-
4043
- if len(scope_files) > expansion_cap:
4044
- if verbose:
4045
- _verbose_stderr_line(f"[increment] dependent expansion cap ({expansion_cap}) exceeded ({len(scope_files)} files); falling back to full rebuild")
4046
- conn.close()
4047
- db.close()
4048
- crash_marker_path.unlink(missing_ok=True)
4049
- return _fallback_to_full(source_root, ladybug_path, verbose, t_start)
4050
-
4051
- if verbose:
4052
- _verbose_stderr_line(f"[increment] processing {len(scope_files)} files ({len(changed_files)} changed + {len(dependent_files)} dependents)")
4053
-
4054
- # Step 4: Scoped deletion
4055
- if verbose:
4056
- _verbose_stderr_line("[increment] deleting outdated nodes and edges")
4057
- _delete_file_scope(conn, changed_files, dependent_files)
4058
-
4059
- # Force deletion to be applied by running a dummy query
4060
- conn.execute("MATCH (s:Symbol) RETURN count(*)")
4061
-
4062
- # Step 5: Scoped pass 1-4
4063
- if verbose:
4064
- _verbose_stderr_line("[increment] rebuilding scoped files (passes 1-4)")
4065
-
4066
- tables = GraphTables()
4067
- asts = pass1_parse(
4068
- source_root, tables, verbose=verbose, scope_files=scope_files, removed_files=removed
4069
- )
4070
-
4071
- # Load existing types and members for cross-file resolution (only from unchanged files)
4072
- _load_existing_types(conn, tables, exclude_files=scope_files)
4073
- _load_existing_members(conn, tables, exclude_files=scope_files)
4074
-
4075
- pass2_edges(tables, asts, verbose=verbose)
4076
- pass3_calls(tables, asts, verbose=verbose)
4077
- pass4_routes(tables, asts, source_root=source_root, verbose=verbose)
4078
-
4079
- # Populate declares and overrides rows
4080
- _populate_declares_rows(tables)
4081
- _populate_overrides_rows(tables)
4082
-
4083
- # Write scoped nodes and edges
4084
- meta_chain = collect_annotation_meta_chain(str(source_root.resolve()))
4085
- _scoped_write(conn, tables, project_root=source_root, meta_chain=meta_chain)
4086
-
4087
- # Step 6: Global pass 5-6
4088
- if verbose:
4089
- _verbose_stderr_line("[increment] running global passes 5-6")
4090
-
4091
- # Rebuild full tables for global pass 5-6 (pass1 populates members from scratch)
4092
- tables_for_global = GraphTables()
4093
- global_asts = pass1_parse(source_root, tables_for_global, verbose=verbose)
4094
- # pass4 (routes/EXPOSES) must run on the global pass5/6 tables too (issue
4095
- # #352): pass5 links Feign HTTP_CALLS to routes via exposes_rows, and pass6
4096
- # matches against routes_rows. Without pass4 both stay empty and the
4097
- # HTTP_CALLS match outcome drifts from a full rebuild. Mirrors main().
4098
- pass4_routes(tables_for_global, global_asts, source_root=source_root, verbose=verbose)
4099
-
4100
- pass5_imperative_edges(tables_for_global, global_asts, source_root=source_root, verbose=verbose)
4101
-
4102
- # Delete existing Client, Producer, and their edges
4103
- conn.execute("MATCH (c:Client) DETACH DELETE c")
4104
- conn.execute("MATCH (p:Producer) DETACH DELETE p")
4105
-
4106
- pass6_match_edges(tables_for_global, verbose=verbose)
4107
-
4108
- # Write Client, Producer, and cross-service edges
4109
- _write_clients_producers_and_calls(conn, tables_for_global)
4110
-
4111
- # Step 7: Update hash store and metadata
4112
- if verbose:
4113
- _verbose_stderr_line("[increment] updating hash store and metadata")
4114
-
4115
- # Update hashes for processed files
4116
- tracker.update(scope_files, source_root)
4117
-
4118
- # Remove hashes for deleted files
4119
- for filename in removed:
4120
- if filename in tracker._hashes:
4121
- del tracker._hashes[filename]
4122
-
4123
- tracker.save()
4124
-
4125
- # Update GraphMeta
4126
- _write_meta(conn, tables_for_global, source_root)
4127
- _ensure_symbol_fts_index(conn, verbose=verbose)
4128
-
4129
- # Remove crash marker
4130
- crash_marker_path.unlink(missing_ok=True)
4131
-
4132
- conn.close()
4133
- db.close()
4134
-
4135
- elapsed = time.time() - t_start
4136
- if verbose:
4137
- _verbose_stderr_line(f"[increment] completed in {elapsed:.2f}s")
4138
-
4139
- return IncrementalResult(
4140
- mode="incremental",
4141
- files_changed=len(changed),
4142
- files_added=len(added),
4143
- files_removed=len(removed),
4144
- dependents_reprocessed=len(dependent_files),
4145
- elapsed_sec=elapsed,
4146
- )
4147
-
4148
- except Exception as e:
4149
- # On error, remove crash marker and fall back to full rebuild
4150
- if verbose:
4151
- _verbose_stderr_line(f"[increment] error during incremental rebuild: {e}; falling back to full rebuild")
4152
- conn.close()
4153
- db.close()
4154
- crash_marker_path.unlink(missing_ok=True)
4155
- return _fallback_to_full(source_root, ladybug_path, verbose, t_start)
4156
-
4157
-
4158
- def _init_hash_tracker(source_root: Path, ladybug_path: Path) -> int:
4159
- """Initialize hash tracker for all Java files. Returns number of files hashed.
4160
-
4161
- Called right after a full graph rebuild (``write_ladybug``), so the store must
4162
- mirror exactly the files that were just indexed. We deliberately do NOT
4163
- ``load()`` the existing store: ``update`` re-hashes every current file anyway,
4164
- and preserving old entries would leave stale hashes for files that no longer
4165
- exist (deleted or now-ignored). Those ghosts would be re-detected as "removed"
4166
- on every subsequent ``increment``, sustaining an endless full-rebuild loop.
4167
- """
4168
- index_dir = ladybug_path.parent
4169
- tracker = FileHashTracker(index_dir)
4170
- ignore = LayeredIgnore(source_root)
4171
- all_files: set[str] = set()
4172
- source_root_resolved = source_root.resolve()
4173
- for p in iter_java_source_files(source_root, ignore=ignore):
4174
- p_resolved = p.resolve()
4175
- try:
4176
- rel_path = p_resolved.relative_to(source_root_resolved).as_posix()
4177
- except ValueError:
4178
- rel_path = p.as_posix()
4179
- all_files.add(rel_path)
4180
- tracker.update(all_files, source_root)
4181
- tracker.save()
4182
- return len(all_files)
4183
-
4184
-
4185
- def _fallback_to_full(source_root: Path, ladybug_path: Path, verbose: bool, t_start: float) -> IncrementalResult:
4186
- """Fallback to full rebuild."""
4187
- tables = GraphTables()
4188
- asts = pass1_parse(source_root, tables, verbose=verbose)
4189
- pass2_edges(tables, asts, verbose=verbose)
4190
- pass3_calls(tables, asts, verbose=verbose)
4191
- pass4_routes(tables, asts, source_root=source_root, verbose=verbose)
4192
- pass5_imperative_edges(tables, asts, source_root=source_root, verbose=verbose)
4193
- pass6_match_edges(tables, verbose=verbose)
4194
- write_ladybug(ladybug_path, tables, source_root=source_root, verbose=verbose)
4195
-
4196
- return IncrementalResult(
4197
- mode="full_fallback",
4198
- files_changed=0,
4199
- files_added=0,
4200
- files_removed=0,
4201
- dependents_reprocessed=0,
4202
- elapsed_sec=time.time() - t_start,
4203
- )
4204
-
4205
-
4206
- def _write_clients_producers_and_calls(conn: ladybug.Connection, tables: GraphTables) -> None:
4207
- """Write Route, Client, Producer, and cross-service edges to LadybugDB.
4208
-
4209
- Used by the incremental rebuild's global pass 5-6 step. Writes phantom
4210
- Route nodes (created by pass5 for cross-service calls) that wouldn't
4211
- otherwise exist in LadybugDB.
4212
- """
4213
- # Bulk-write routes, mirroring `_write_nodes_impl`: COPY the rows that are new
4214
- # to the DB, and SET every mutable field on the routes already present (the
4215
- # caller does NOT delete routes, only Clients/Producers — see the global
4216
- # pass 5-6 block). `tables.routes_rows` is the full route set (pass4 routes +
4217
- # pass5 phantom routes), not just phantoms, so the SET keeps existing routes'
4218
- # properties in sync with pass5's cross-service enrichment while the COPY
4219
- # materializes phantoms that have no node yet. Replaces the last per-row
4220
- # graph write (MERGE upsert).
4221
- route_rows = [asdict(row) for row in tables.routes_rows]
4222
- existing_route_ids = _existing_node_ids(conn)
4223
- new_route_rows = [r for r in route_rows if r["id"] not in existing_route_ids]
4224
- _bulk_copy(conn, "Route", _ROUTE_COLUMNS, new_route_rows)
4225
- for r in route_rows:
4226
- if r["id"] in existing_route_ids:
4227
- conn.execute(_SET_ROUTE_BY_ID, r)
4228
-
4229
- # Build node_id lookup for members and types
4230
- member_by_id = {m.node_id: m for m in tables.members}
4231
-
4232
- # Build client_id and producer_id lookups for source_file resolution
4233
- client_by_id = {c.id: c for c in tables.client_rows}
4234
- producer_by_id = {p.id: p for p in tables.producer_rows}
4235
-
4236
- # Stage Client + Producer NODE rows unconditionally. The caller DETACH-DELETEs
4237
- # every Client/Producer node immediately before calling this (see the global
4238
- # pass 5-6 block), so none are in the DB yet and COPY-ing them fresh is safe.
4239
- # Do NOT filter node rows against the existing-id set — that is the EDGE filter
4240
- # pattern mis-applied to nodes, and would drop every node being created (the
4241
- # caller's delete makes the pre-load set empty by construction).
4242
- client_rows = [asdict(row) for row in tables.client_rows]
4243
- _bulk_copy(conn, "Client", _CLIENT_COLUMNS, client_rows)
4244
- producer_rows = [asdict(row) for row in tables.producer_rows]
4245
- _bulk_copy(conn, "Producer", _PRODUCER_COLUMNS, producer_rows)
4246
-
4247
- # Endpoint filtering applies to EDGES only: re-fetch the live node set (now
4248
- # includes the Clients/Producers just loaded) and drop edge rows whose
4249
- # endpoints still aren't materialized — reproduces per-row MERGE silent-drop.
4250
- valid_ids = _existing_node_ids(conn)
4251
-
4252
- # Stage DECLARES_CLIENT edge rows (Symbol -> Client)
4253
- # Build source_file lookup from member_by_id
4254
- declares_client_rows = [
4255
- {
4256
- "FROM": row.symbol_id,
4257
- "TO": row.client_id,
4258
- "source_file": member_by_id.get(row.symbol_id, MemberEntry(kind="", decl=None, parent_id="", parent_fqn="", file_path="", module="", microservice="", node_id="")).file_path,
4259
- "confidence": row.confidence,
4260
- "strategy": row.strategy,
4261
- }
4262
- for row in tables.declares_client_rows
4263
- if row.symbol_id in valid_ids and row.client_id in valid_ids
4264
- ]
4265
- _bulk_copy(conn, "DECLARES_CLIENT", _REL_DECLARES_CLIENT_COLUMNS, declares_client_rows)
4266
-
4267
- # Stage DECLARES_PRODUCER edge rows (Symbol -> Producer)
4268
- declares_producer_rows = [
4269
- {
4270
- "FROM": row.symbol_id,
4271
- "TO": row.producer_id,
4272
- "source_file": member_by_id.get(row.symbol_id, MemberEntry(kind="", decl=None, parent_id="", parent_fqn="", file_path="", module="", microservice="", node_id="")).file_path,
4273
- "confidence": row.confidence,
4274
- "strategy": row.strategy,
4275
- }
4276
- for row in tables.declares_producer_rows
4277
- if row.symbol_id in valid_ids and row.producer_id in valid_ids
4278
- ]
4279
- _bulk_copy(conn, "DECLARES_PRODUCER", _REL_DECLARES_PRODUCER_COLUMNS, declares_producer_rows)
4280
-
4281
- # Stage HTTP_CALLS edge rows (Client -> Route)
4282
- http_call_rows = [
4283
- {
4284
- "FROM": row.client_id,
4285
- "TO": row.route_id,
4286
- "source_file": client_by_id.get(row.client_id, ClientRow(id="", client_kind="", target_service="", path="", path_template="", path_regex="", method="", member_fqn="", member_id="", microservice="", module="", filename="", start_line=0, end_line=0, resolved=False, source_layer="")).filename,
4287
- "confidence": row.confidence,
4288
- "strategy": row.strategy,
4289
- "method_call": row.method_call,
4290
- "raw_uri": row.raw_uri,
4291
- "match": row.match,
4292
- }
4293
- for row in tables.http_call_rows
4294
- if row.client_id in valid_ids and row.route_id in valid_ids
4295
- ]
4296
- _bulk_copy(conn, "HTTP_CALLS", _REL_HTTP_CALLS_COLUMNS, http_call_rows)
4297
-
4298
- # Stage ASYNC_CALLS edge rows (Producer -> Route)
4299
- async_call_rows = [
4300
- {
4301
- "FROM": row.producer_id,
4302
- "TO": row.route_id,
4303
- "source_file": producer_by_id.get(row.producer_id, ProducerRow(id="", producer_kind="", topic="", broker="", direction="", member_fqn="", member_id="", microservice="", module="", filename="", start_line=0, end_line=0, resolved=False, source_layer="")).filename,
4304
- "confidence": row.confidence,
4305
- "strategy": row.strategy,
4306
- "direction": row.direction,
4307
- "raw_topic": row.raw_topic,
4308
- "match": row.match,
4309
- }
4310
- for row in tables.async_call_rows
4311
- if row.producer_id in valid_ids and row.route_id in valid_ids
4312
- ]
4313
- _bulk_copy(conn, "ASYNC_CALLS", _REL_ASYNC_CALLS_COLUMNS, async_call_rows)
4314
-
4315
-
4316
- def write_ladybug(
4317
- db_path: Path,
4318
- tables: GraphTables,
4319
- *,
4320
- source_root: Path,
4321
- verbose: bool,
4322
- meta_chain: dict[str, frozenset[str]] | None = None,
4323
- ) -> None:
4324
- if meta_chain is None:
4325
- meta_chain = collect_annotation_meta_chain(
4326
- str(source_root.resolve()),
4327
- )
4328
- if verbose:
4329
- _verbose_stderr_line(_WRITE_START)
4330
- with _VerbosePassHeartbeats("[graph] writing", verbose=verbose):
4331
- db_path.parent.mkdir(parents=True, exist_ok=True)
4332
- db = ladybug.Database(str(db_path))
4333
- conn = ladybug.Connection(db)
4334
- _drop_all(conn)
4335
- _create_schema(conn)
4336
- t0 = time.time()
4337
- _write_nodes(
4338
- conn,
4339
- tables,
4340
- project_root=source_root,
4341
- meta_chain=meta_chain,
4342
- )
4343
- if verbose:
4344
- _verbose_stderr_line(f"[graph] writing · nodes written in {time.time() - t0:.2f}s")
4345
- _populate_declares_rows(tables)
4346
- _populate_overrides_rows(tables)
4347
- t1 = time.time()
4348
- _fbyid = _build_file_by_node_id(tables)
4349
- _write_edges(conn, tables, _fbyid)
4350
- if verbose:
4351
- _verbose_stderr_line(f"[graph] writing · edges written in {time.time() - t1:.2f}s")
4352
- t2 = time.time()
4353
- _write_routes_and_exposes(conn, tables, _fbyid)
4354
- if verbose:
4355
- _verbose_stderr_line(f"[graph] writing · routes/exposes written in {time.time() - t2:.2f}s")
4356
- _write_meta(conn, tables, source_root)
4357
- _ensure_symbol_fts_index(conn, verbose=verbose)
4358
- conn.close()
4359
- db.close()
4360
-
4361
- # Build vocabulary index (best-effort, failure doesn't fail the graph build)
4362
- _try_build_vocabulary_index(db_path, source_root, verbose)
4363
- _init_hash_tracker(source_root, db_path)
4364
-
4365
-
4366
- def _try_build_vocabulary_index(db_path: Path, source_root: Path, verbose: bool) -> None:
4367
- """Build and save the vocabulary index as a sidecar (best-effort).
4368
-
4369
- This is called after write_ladybug() completes. A build failure must not
4370
- fail the graph build, so this is wrapped in try/except and logged.
4371
-
4372
- Args:
4373
- db_path: Path to the LadybugDB database file
4374
- source_root: Source repository root
4375
- verbose: Whether to emit verbose progress
4376
- """
4377
- try:
4378
- from java_codebase_rag.absence.absence_vocab import VocabularyIndex, VOCAB_INDEX_FILENAME
4379
- from java_codebase_rag.graph.ladybug_queries import LadybugGraph
4380
-
4381
- t0 = time.time()
4382
- if verbose:
4383
- _verbose_stderr_line("[vocab] building vocabulary index")
4384
-
4385
- # Open graph for reading
4386
- graph = LadybugGraph.get(str(db_path))
4387
-
4388
- # Read q from env var set by ResolvedOperatorConfig.subprocess_env()
4389
- raw_q = os.environ.get("JAVA_CODEBASE_RAG_ABSENCE_NGRAM_Q", "3").strip()
4390
- try:
4391
- q = int(raw_q) if raw_q else 3
4392
- except ValueError:
4393
- q = 3 # Invalid env value falls back to default
4394
- # Build index with configured q (or default 3)
4395
- index = VocabularyIndex.build(graph, q=q)
4396
-
4397
- # Save to sidecar next to the graph db
4398
- sidecar_path = Path(db_path).parent / VOCAB_INDEX_FILENAME
4399
- index.save(sidecar_path, ontology_version=ONTOLOGY_VERSION)
4400
-
4401
- if verbose:
4402
- _verbose_stderr_line(f"[vocab] index built with {index.symbol_count} symbols in {time.time() - t0:.2f}s")
4403
-
4404
- except Exception as e:
4405
- # Log but don't fail - graph build is the primary concern
4406
- log.warning(f"Vocabulary index build failed (non-critical): {e}")
4407
- if verbose:
4408
- _verbose_stderr_line(f"[vocab] build failed (graph still written): {e}")
4409
-
4410
-
4411
- # ---------- CLI ----------
4412
-
4413
-
4414
- def _default_ladybug_path() -> Path:
4415
- idx = os.environ.get("JAVA_CODEBASE_RAG_INDEX_DIR", "").strip()
4416
- if idx and not idx.startswith(("s3://", "gs://", "az://")):
4417
- return Path(os.path.expanduser(idx.rstrip("/"))) / "code_graph.lbug"
4418
- return Path.cwd() / ".java-codebase-rag" / "code_graph.lbug"
4419
-
4420
-
4421
- def main() -> int:
4422
- parser = argparse.ArgumentParser(description="Build an AST-derived LadybugDB graph for Java sources.")
4423
- parser.add_argument("--source-root", default=None, help="Repository / monorepo root to scan for .java (defaults to current working directory)")
4424
- parser.add_argument(
4425
- "--ladybug-path",
4426
- default=None,
4427
- help=(
4428
- "LadybugDB database path (file/dir as used by ladybug.Database; "
4429
- "default: $JAVA_CODEBASE_RAG_INDEX_DIR/code_graph.lbug or ./.java-codebase-rag/code_graph.lbug)"
4430
- ),
4431
- )
4432
- parser.add_argument("--verbose", action="store_true")
4433
- parser.add_argument("--incremental", action="store_true", help="Run incremental rebuild instead of full rebuild")
4434
- args = parser.parse_args()
4435
-
4436
- root = Path(args.source_root).expanduser().resolve() if args.source_root else Path.cwd().resolve()
4437
- if not root.is_dir():
4438
- print(f"source-root not a directory: {root}", file=sys.stderr)
4439
- return 2
4440
-
4441
- ladybug_path = Path(args.ladybug_path).expanduser() if args.ladybug_path else _default_ladybug_path()
4442
-
4443
- if args.incremental:
4444
- result = incremental_rebuild(root, ladybug_path, verbose=args.verbose)
4445
- print(json.dumps({
4446
- "mode": result.mode,
4447
- "files_changed": result.files_changed,
4448
- "files_added": result.files_added,
4449
- "files_removed": result.files_removed,
4450
- "dependents_reprocessed": result.dependents_reprocessed,
4451
- "elapsed_sec": result.elapsed_sec,
4452
- }))
4453
- if args.verbose:
4454
- _verbose_stderr_line(f"[graph] done · mode={result.mode} files_changed={result.files_changed} files_added={result.files_added} files_removed={result.files_removed} dependents={result.dependents_reprocessed} elapsed={result.elapsed_sec:.2f}s")
4455
- return 0
4456
-
4457
- tables = GraphTables()
4458
- asts = pass1_parse(root, tables, verbose=args.verbose)
4459
- pass2_edges(tables, asts, verbose=args.verbose)
4460
- pass3_calls(tables, asts, verbose=args.verbose)
4461
- pass4_routes(tables, asts, source_root=root, verbose=args.verbose)
4462
- pass5_imperative_edges(tables, asts, source_root=root, verbose=args.verbose)
4463
- pass6_match_edges(tables, verbose=args.verbose)
4464
- write_ladybug(ladybug_path, tables, source_root=root, verbose=args.verbose)
4465
- if args.verbose:
4466
- _verbose_stderr_line(f"[graph] done · ladybug at {ladybug_path}")
4467
- return 0
4468
-
4469
-
4470
- if __name__ == "__main__":
4471
- sys.exit(main())