java-codebase-rag 0.11.2__py3-none-any.whl → 0.12.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- java_codebase_rag-0.12.1.dist-info/METADATA +35 -0
- java_codebase_rag-0.12.1.dist-info/RECORD +4 -0
- {java_codebase_rag-0.11.2.dist-info → java_codebase_rag-0.12.1.dist-info}/WHEEL +1 -1
- java_codebase_rag/_fdlimit.py +0 -56
- java_codebase_rag/_stdio.py +0 -32
- java_codebase_rag/_version.py +0 -35
- java_codebase_rag/absence/__init__.py +0 -0
- java_codebase_rag/absence/absence_diagnosis.py +0 -700
- java_codebase_rag/absence/absence_types.py +0 -124
- java_codebase_rag/absence/absence_vocab.py +0 -460
- java_codebase_rag/analysis/__init__.py +0 -0
- java_codebase_rag/analysis/pr_analysis.py +0 -563
- java_codebase_rag/analysis/resolve_service.py +0 -740
- java_codebase_rag/ast/__init__.py +0 -0
- java_codebase_rag/ast/ast_java.py +0 -2825
- java_codebase_rag/ast/brownfield_events.py +0 -58
- java_codebase_rag/ast/chunk_heuristics.py +0 -62
- java_codebase_rag/cli.py +0 -1215
- java_codebase_rag/cli_format.py +0 -85
- java_codebase_rag/cli_progress.py +0 -94
- java_codebase_rag/config.py +0 -833
- java_codebase_rag/eval/__init__.py +0 -1
- java_codebase_rag/eval/ground_truth.py +0 -100
- java_codebase_rag/eval/metrics.py +0 -107
- java_codebase_rag/eval/runner.py +0 -556
- java_codebase_rag/graph/__init__.py +0 -0
- java_codebase_rag/graph/build_ast_graph.py +0 -4471
- java_codebase_rag/graph/graph_enrich.py +0 -1937
- java_codebase_rag/graph/graph_types.py +0 -224
- java_codebase_rag/graph/java_ontology.py +0 -465
- java_codebase_rag/graph/ladybug_queries.py +0 -2213
- java_codebase_rag/graph/path_filtering.py +0 -477
- java_codebase_rag/index/__init__.py +0 -0
- java_codebase_rag/index/java_index_flow_lancedb.py +0 -734
- java_codebase_rag/index/java_index_v1_common.py +0 -33
- java_codebase_rag/install_data/__init__.py +0 -0
- java_codebase_rag/install_data/agents/explorer-rag-cli.md +0 -108
- java_codebase_rag/install_data/agents/explorer-rag-enhanced.md +0 -152
- java_codebase_rag/install_data/skills/explore-codebase/SKILL.md +0 -165
- java_codebase_rag/install_data/skills/explore-codebase-cli/SKILL.md +0 -107
- java_codebase_rag/installer.py +0 -2188
- java_codebase_rag/jrag.py +0 -4531
- java_codebase_rag/jrag_envelope.py +0 -1107
- java_codebase_rag/jrag_hints.py +0 -204
- java_codebase_rag/jrag_render.py +0 -926
- java_codebase_rag/lance_optimize.py +0 -264
- java_codebase_rag/mcp/__init__.py +0 -0
- java_codebase_rag/mcp/mcp_hints.py +0 -932
- java_codebase_rag/mcp/mcp_v2.py +0 -1916
- java_codebase_rag/mcp/server.py +0 -884
- java_codebase_rag/pipeline.py +0 -531
- java_codebase_rag/progress.py +0 -570
- java_codebase_rag/read_payloads.py +0 -781
- java_codebase_rag/search/__init__.py +0 -0
- java_codebase_rag/search/index_common.py +0 -10
- java_codebase_rag/search/search_lancedb.py +0 -1296
- java_codebase_rag/search/search_lexical.py +0 -449
- java_codebase_rag/search/search_scoring.py +0 -523
- java_codebase_rag/watch/__init__.py +0 -0
- java_codebase_rag/watch/client.py +0 -230
- java_codebase_rag/watch/daemon.py +0 -396
- java_codebase_rag/watch/lock.py +0 -201
- java_codebase_rag/watch/paths.py +0 -76
- java_codebase_rag/watch/protocol.py +0 -122
- java_codebase_rag/watch/server.py +0 -273
- java_codebase_rag/watch/warm.py +0 -105
- java_codebase_rag/watch/watcher.py +0 -370
- java_codebase_rag-0.11.2.dist-info/METADATA +0 -331
- java_codebase_rag-0.11.2.dist-info/RECORD +0 -71
- java_codebase_rag-0.11.2.dist-info/entry_points.txt +0 -4
- java_codebase_rag-0.11.2.dist-info/licenses/LICENSE +0 -21
- java_codebase_rag-0.11.2.dist-info/top_level.txt +0 -1
- /java_codebase_rag/__init__.py → /java_codebase_rag-0.12.1.dist-info/top_level.txt +0 -0
|
@@ -1,4471 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Four-pass AST-derived Knowledge Base builder (LadybugDB).
|
|
3
|
-
|
|
4
|
-
Walks a Java source tree with `tree_sitter_java`, writes a deterministic graph of:
|
|
5
|
-
Symbol nodes: package, file, class, interface, enum, record, annotation, method, constructor
|
|
6
|
-
Route nodes: declaration-site routes (Spring MVC/WebFlux, Feign, Kafka, …)
|
|
7
|
-
Rel tables: EXTENDS, IMPLEMENTS, INJECTS, DECLARES, OVERRIDES, CALLS, EXPOSES
|
|
8
|
-
|
|
9
|
-
Pass 1 builds every node and in-memory resolution indexes.
|
|
10
|
-
Pass 2 resolves each extends/implements/injection target using Java's lookup order
|
|
11
|
-
(same file → explicit import → same package → wildcard import → java.lang → phantom).
|
|
12
|
-
Pass 3 resolves static call sites into confidence-scored CALLS edges and DECLARES.
|
|
13
|
-
Pass 4 emits Route rows plus Symbol→Route EXPOSES edges from literal annotation metadata.
|
|
14
|
-
|
|
15
|
-
Usage:
|
|
16
|
-
build_ast_graph.py --source-root <repo> [--ladybug-path <path>] [--verbose]
|
|
17
|
-
|
|
18
|
-
Default LadybugDB database path resolution order:
|
|
19
|
-
--ladybug-path CLI arg (path passed to ladybug.Database(...))
|
|
20
|
-
JAVA_CODEBASE_RAG_INDEX_DIR/code_graph.lbug (if set and local)
|
|
21
|
-
./.java-codebase-rag/code_graph.lbug under cwd
|
|
22
|
-
|
|
23
|
-
The LadybugDB DB is dropped and rebuilt on every run (Phase 1 is a full rebuild).
|
|
24
|
-
"""
|
|
25
|
-
from __future__ import annotations
|
|
26
|
-
|
|
27
|
-
import argparse
|
|
28
|
-
import contextlib
|
|
29
|
-
import hashlib
|
|
30
|
-
import json
|
|
31
|
-
import logging
|
|
32
|
-
import os
|
|
33
|
-
import re
|
|
34
|
-
import sys
|
|
35
|
-
import threading
|
|
36
|
-
import time
|
|
37
|
-
from collections import defaultdict
|
|
38
|
-
from dataclasses import asdict, dataclass, field, replace
|
|
39
|
-
from pathlib import Path
|
|
40
|
-
|
|
41
|
-
import ladybug
|
|
42
|
-
import pyarrow as pa
|
|
43
|
-
|
|
44
|
-
from java_codebase_rag.ast.ast_java import (
|
|
45
|
-
ONTOLOGY_VERSION,
|
|
46
|
-
CallSite,
|
|
47
|
-
JavaFileAst,
|
|
48
|
-
MethodDecl,
|
|
49
|
-
OutgoingCallDecl,
|
|
50
|
-
TypeDecl,
|
|
51
|
-
injection_annotation_names,
|
|
52
|
-
lombok_required_args_annotations,
|
|
53
|
-
parse_java,
|
|
54
|
-
)
|
|
55
|
-
from java_codebase_rag.graph.graph_enrich import (
|
|
56
|
-
_load_config_cross_service_resolution,
|
|
57
|
-
classify_java_file,
|
|
58
|
-
collect_annotation_meta_chain,
|
|
59
|
-
load_brownfield_overrides,
|
|
60
|
-
load_generated_detection,
|
|
61
|
-
microservice_for_path,
|
|
62
|
-
module_for_path,
|
|
63
|
-
phantom_id,
|
|
64
|
-
resolve_async_producer_for_method,
|
|
65
|
-
resolve_http_client_for_method,
|
|
66
|
-
resolve_role_and_capabilities,
|
|
67
|
-
resolve_routes_for_method,
|
|
68
|
-
symbol_id,
|
|
69
|
-
)
|
|
70
|
-
from java_codebase_rag.graph.path_filtering import LayeredIgnore, iter_java_source_files
|
|
71
|
-
from java_codebase_rag.search.search_scoring import SYMBOL_FTS_INDEX as _SYMBOL_FTS_INDEX, _split_identifier
|
|
72
|
-
from java_codebase_rag.graph.java_ontology import (
|
|
73
|
-
CLIENT_KIND_FEIGN_METHOD,
|
|
74
|
-
CLIENT_KIND_REST_TEMPLATE,
|
|
75
|
-
VALID_CLIENT_KINDS,
|
|
76
|
-
VALID_HTTP_CALL_MATCHES,
|
|
77
|
-
VALID_PRODUCER_KINDS,
|
|
78
|
-
)
|
|
79
|
-
|
|
80
|
-
log = logging.getLogger(__name__)
|
|
81
|
-
|
|
82
|
-
_VERBOSE_STDERR_LOCK = threading.Lock()
|
|
83
|
-
|
|
84
|
-
_PASS1_START = "[graph] pass 1 · parsing Java files"
|
|
85
|
-
_PASS2_START = "[graph] pass 2 · emitting EXTENDS / IMPLEMENTS / DECLARES rows"
|
|
86
|
-
_PASS3_START = "[graph] pass 3 · call resolution (outgoing calls per site)"
|
|
87
|
-
_PASS4_START = "[graph] pass 4 · route and EXPOSES extraction"
|
|
88
|
-
_PASS5_START = "[graph] pass 5 · imperative HTTP_CALLS / ASYNC_CALLS edges"
|
|
89
|
-
_PASS6_START = "[graph] pass 6 · cross-service call-edge matching"
|
|
90
|
-
_WRITE_START = "[graph] writing · LadybugDB graph to disk"
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
def _verbose_stderr_line(content: str) -> None:
|
|
94
|
-
with _VERBOSE_STDERR_LOCK:
|
|
95
|
-
print(content, file=sys.stderr, flush=True)
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
def _emit_graph_progress(parts: dict[str, object], *, verbose: bool) -> None:
|
|
99
|
-
"""Emit one ``JCIRAG_PROGRESS kind=graph …`` line to stderr (gated by verbose).
|
|
100
|
-
|
|
101
|
-
The parent process (``pipeline.run_build_ast_graph`` /
|
|
102
|
-
``run_incremental_graph``) passes ``--verbose`` in default AND verbose modes
|
|
103
|
-
(only suppressed for ``--quiet``), so this structured progress surfaces in
|
|
104
|
-
default mode (where the parent renders it) and verbose mode (raw relay). In
|
|
105
|
-
``--quiet`` the builder is never invoked with ``--verbose`` so nothing is
|
|
106
|
-
emitted. Field order is fixed so the parser and tests can pin substrings.
|
|
107
|
-
"""
|
|
108
|
-
if not verbose:
|
|
109
|
-
return
|
|
110
|
-
fields = ["kind=graph"]
|
|
111
|
-
for key in ("pass", "done", "total", "status", "elapsed_s"):
|
|
112
|
-
if key in parts:
|
|
113
|
-
fields.append(f"{key}={parts[key]}")
|
|
114
|
-
line = "JCIRAG_PROGRESS " + " ".join(fields)
|
|
115
|
-
_verbose_stderr_line(line)
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
# Pass-1 per-file tick cadence: bound stderr volume on huge trees without making
|
|
119
|
-
# the bar feel stale. A final tick on pass completion carries status=done.
|
|
120
|
-
_PASS1_TICK_EVERY = 25
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
@contextlib.contextmanager
|
|
124
|
-
def _graph_pass_progress(pass_label: str, *, verbose: bool):
|
|
125
|
-
"""Emit ``pass=N/6 status=running`` on entry and ``status=done elapsed_s=…``
|
|
126
|
-
on exit for passes 2–6 (each advances the rendered bar by 1/6).
|
|
127
|
-
|
|
128
|
-
Usage: ``with _graph_pass_progress("2/6", verbose=verbose): …``
|
|
129
|
-
"""
|
|
130
|
-
if not verbose:
|
|
131
|
-
yield
|
|
132
|
-
return
|
|
133
|
-
_emit_graph_progress({"pass": pass_label, "status": "running"}, verbose=verbose)
|
|
134
|
-
t0 = time.time()
|
|
135
|
-
try:
|
|
136
|
-
yield
|
|
137
|
-
finally:
|
|
138
|
-
elapsed = time.time() - t0
|
|
139
|
-
_emit_graph_progress(
|
|
140
|
-
{"pass": pass_label, "status": "done", "elapsed_s": f"{elapsed:.2f}"},
|
|
141
|
-
verbose=verbose,
|
|
142
|
-
)
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
class _VerbosePassHeartbeats:
|
|
146
|
-
"""Emit ``[tag] running … Ns elapsed`` every 5s on stderr while in scope (verbose only)."""
|
|
147
|
-
|
|
148
|
-
def __init__(self, tag: str, *, verbose: bool) -> None:
|
|
149
|
-
self._tag = tag
|
|
150
|
-
self._verbose = verbose
|
|
151
|
-
self._thr: threading.Thread | None = None
|
|
152
|
-
self._stop: threading.Event | None = None
|
|
153
|
-
|
|
154
|
-
def __enter__(self) -> None:
|
|
155
|
-
if not self._verbose:
|
|
156
|
-
return None
|
|
157
|
-
self._stop = threading.Event()
|
|
158
|
-
stop = self._stop
|
|
159
|
-
tag = self._tag
|
|
160
|
-
|
|
161
|
-
def worker() -> None:
|
|
162
|
-
t0 = time.monotonic()
|
|
163
|
-
while not stop.wait(timeout=5.0):
|
|
164
|
-
elapsed = int(time.monotonic() - t0)
|
|
165
|
-
_verbose_stderr_line(f"{tag} · {elapsed}s elapsed")
|
|
166
|
-
|
|
167
|
-
self._thr = threading.Thread(target=worker, name=f"hb-{tag}", daemon=True)
|
|
168
|
-
self._thr.start()
|
|
169
|
-
return None
|
|
170
|
-
|
|
171
|
-
def __exit__(self, exc_type, exc, tb) -> bool:
|
|
172
|
-
if self._thr is not None and self._stop is not None:
|
|
173
|
-
self._stop.set()
|
|
174
|
-
self._thr.join(timeout=2.0)
|
|
175
|
-
return False
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
_JAVA_LANG_SIMPLE = frozenset({
|
|
179
|
-
"Object", "String", "Integer", "Long", "Short", "Byte", "Boolean", "Double",
|
|
180
|
-
"Float", "Character", "Number", "Void", "Class", "Enum", "Record",
|
|
181
|
-
"Throwable", "Exception", "RuntimeException", "Error", "Thread", "Runnable",
|
|
182
|
-
"Iterable", "Comparable", "CharSequence", "StringBuilder", "StringBuffer",
|
|
183
|
-
"Math", "System", "AutoCloseable", "Cloneable",
|
|
184
|
-
})
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
# ---------- dataclasses ----------
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
@dataclass
|
|
191
|
-
class TypeIndexEntry:
|
|
192
|
-
"""Pass-1 record for a type declaration + any methods/constructors inside it."""
|
|
193
|
-
decl: TypeDecl
|
|
194
|
-
file_path: str
|
|
195
|
-
module: str
|
|
196
|
-
microservice: str
|
|
197
|
-
package: str
|
|
198
|
-
outer_fqn: str | None
|
|
199
|
-
node_id: str
|
|
200
|
-
# True when this entry was loaded from the existing graph by
|
|
201
|
-
# `_load_existing_types` (an unchanged-file stub used only for cross-file
|
|
202
|
-
# resolution). Its `decl` is a placeholder (no annotations/methods), so its
|
|
203
|
-
# recomputed role/capabilities must never be written back over the real
|
|
204
|
-
# stored values. See `_write_nodes_impl`.
|
|
205
|
-
loaded_from_db: bool = False
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
@dataclass
|
|
209
|
-
class MemberEntry:
|
|
210
|
-
kind: str # method | constructor
|
|
211
|
-
decl: MethodDecl
|
|
212
|
-
parent_id: str
|
|
213
|
-
parent_fqn: str
|
|
214
|
-
file_path: str
|
|
215
|
-
module: str
|
|
216
|
-
microservice: str
|
|
217
|
-
node_id: str
|
|
218
|
-
# True when loaded from the existing graph by `_load_existing_members`
|
|
219
|
-
# (an unchanged-file stub used only for cross-file call resolution). Its
|
|
220
|
-
# DECLARES edge already persists in the graph, so it must not be re-emitted
|
|
221
|
-
# by `_populate_declares_rows` (REL tables have no PK → would duplicate).
|
|
222
|
-
loaded_from_db: bool = False
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
@dataclass
|
|
226
|
-
class EdgeRow:
|
|
227
|
-
src_id: str
|
|
228
|
-
dst_id: str
|
|
229
|
-
dst_name: str
|
|
230
|
-
dst_fqn: str
|
|
231
|
-
resolved: bool
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
@dataclass
|
|
235
|
-
class InjectsRow(EdgeRow):
|
|
236
|
-
mechanism: str = ""
|
|
237
|
-
annotation: str = ""
|
|
238
|
-
field_or_param: str = ""
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
@dataclass
|
|
242
|
-
class CallsRow:
|
|
243
|
-
src_id: str
|
|
244
|
-
dst_id: str
|
|
245
|
-
call_site_line: int = 0
|
|
246
|
-
call_site_byte: int = 0
|
|
247
|
-
arg_count: int = 0
|
|
248
|
-
confidence: float = 0.0
|
|
249
|
-
strategy: str = "phantom"
|
|
250
|
-
source: str = "static"
|
|
251
|
-
resolved: bool = True
|
|
252
|
-
callee_declaring_role: str = "OTHER"
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
@dataclass
|
|
256
|
-
class UnresolvedCallSiteRow:
|
|
257
|
-
id: str
|
|
258
|
-
caller_id: str
|
|
259
|
-
call_site_line: int
|
|
260
|
-
call_site_byte: int
|
|
261
|
-
arg_count: int
|
|
262
|
-
callee_simple: str
|
|
263
|
-
receiver_expr: str
|
|
264
|
-
reason: str
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
@dataclass
|
|
268
|
-
class DeclaresRow:
|
|
269
|
-
src_id: str
|
|
270
|
-
dst_id: str
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
@dataclass
|
|
274
|
-
class CallResolutionStats:
|
|
275
|
-
total: int = 0
|
|
276
|
-
by_strategy: dict[str, int] = field(default_factory=lambda: defaultdict(int))
|
|
277
|
-
phantom_chained: int = 0
|
|
278
|
-
phantom_other: int = 0
|
|
279
|
-
callee_unresolved: int = 0
|
|
280
|
-
skipped_cross_service: int = 0
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
@dataclass
|
|
284
|
-
class RouteRow:
|
|
285
|
-
id: str
|
|
286
|
-
kind: str
|
|
287
|
-
framework: str
|
|
288
|
-
method: str
|
|
289
|
-
path: str
|
|
290
|
-
path_template: str
|
|
291
|
-
path_regex: str
|
|
292
|
-
topic: str
|
|
293
|
-
broker: str
|
|
294
|
-
feign_name: str
|
|
295
|
-
feign_url: str
|
|
296
|
-
microservice: str
|
|
297
|
-
module: str
|
|
298
|
-
filename: str
|
|
299
|
-
start_line: int
|
|
300
|
-
end_line: int
|
|
301
|
-
resolved: bool
|
|
302
|
-
# B2a brownfield composition (PR-A3); not persisted on LadybugDB `Route` nodes.
|
|
303
|
-
source_layer: str = "builtin"
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
@dataclass
|
|
307
|
-
class ExposesRow:
|
|
308
|
-
symbol_id: str
|
|
309
|
-
route_id: str
|
|
310
|
-
confidence: float
|
|
311
|
-
strategy: str
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
@dataclass
|
|
315
|
-
class RouteExtractionStats:
|
|
316
|
-
routes_skipped_unresolved: int = 0
|
|
317
|
-
by_framework: dict[str, int] = field(default_factory=lambda: defaultdict(int))
|
|
318
|
-
by_kind: dict[str, int] = field(default_factory=lambda: defaultdict(int))
|
|
319
|
-
routes_resolved_pct: float = 100.0
|
|
320
|
-
# Percentage of emitted `Route` rows whose `source_layer` is not `builtin`.
|
|
321
|
-
# Brownfield layers: `layer_b_ann`, `layer_a_meta`, `layer_c_source`, `layer_b_fqn`.
|
|
322
|
-
routes_from_brownfield_pct: float = 0.0
|
|
323
|
-
routes_by_layer: dict[str, int] = field(default_factory=dict)
|
|
324
|
-
exposes_suppressed_feign: int = 0
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
@dataclass
|
|
328
|
-
class HttpCallRow:
|
|
329
|
-
client_id: str
|
|
330
|
-
route_id: str
|
|
331
|
-
confidence: float
|
|
332
|
-
strategy: str
|
|
333
|
-
method_call: str
|
|
334
|
-
raw_uri: str
|
|
335
|
-
match: str
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
@dataclass
|
|
339
|
-
class AsyncCallRow:
|
|
340
|
-
producer_id: str
|
|
341
|
-
route_id: str
|
|
342
|
-
confidence: float
|
|
343
|
-
strategy: str
|
|
344
|
-
direction: str
|
|
345
|
-
raw_topic: str
|
|
346
|
-
match: str
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
@dataclass
|
|
350
|
-
class ClientRow:
|
|
351
|
-
id: str
|
|
352
|
-
client_kind: str
|
|
353
|
-
target_service: str
|
|
354
|
-
path: str
|
|
355
|
-
path_template: str
|
|
356
|
-
path_regex: str
|
|
357
|
-
method: str
|
|
358
|
-
member_fqn: str
|
|
359
|
-
member_id: str
|
|
360
|
-
microservice: str
|
|
361
|
-
module: str
|
|
362
|
-
filename: str
|
|
363
|
-
start_line: int
|
|
364
|
-
end_line: int
|
|
365
|
-
resolved: bool
|
|
366
|
-
source_layer: str
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
@dataclass
|
|
370
|
-
class DeclaresClientRow:
|
|
371
|
-
symbol_id: str
|
|
372
|
-
client_id: str
|
|
373
|
-
confidence: float
|
|
374
|
-
strategy: str
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
@dataclass
|
|
378
|
-
class ProducerRow:
|
|
379
|
-
id: str
|
|
380
|
-
producer_kind: str
|
|
381
|
-
topic: str
|
|
382
|
-
broker: str
|
|
383
|
-
direction: str
|
|
384
|
-
member_fqn: str
|
|
385
|
-
member_id: str
|
|
386
|
-
microservice: str
|
|
387
|
-
module: str
|
|
388
|
-
filename: str
|
|
389
|
-
start_line: int
|
|
390
|
-
end_line: int
|
|
391
|
-
resolved: bool
|
|
392
|
-
source_layer: str
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
@dataclass
|
|
396
|
-
class DeclaresProducerRow:
|
|
397
|
-
symbol_id: str
|
|
398
|
-
producer_id: str
|
|
399
|
-
confidence: float
|
|
400
|
-
strategy: str
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
@dataclass
|
|
404
|
-
class ClientExtractionStats:
|
|
405
|
-
clients_total: int = 0
|
|
406
|
-
declares_client_total: int = 0
|
|
407
|
-
clients_by_kind: dict[str, int] = field(default_factory=lambda: defaultdict(int))
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
@dataclass
|
|
411
|
-
class ProducerExtractionStats:
|
|
412
|
-
producers_total: int = 0
|
|
413
|
-
declares_producer_total: int = 0
|
|
414
|
-
producers_by_kind: dict[str, int] = field(default_factory=lambda: defaultdict(int))
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
@dataclass
|
|
418
|
-
class CallEdgeStats:
|
|
419
|
-
http_calls_total: int = 0
|
|
420
|
-
async_calls_total: int = 0
|
|
421
|
-
http_calls_by_client_kind: dict[str, int] = field(default_factory=lambda: defaultdict(int))
|
|
422
|
-
async_calls_by_client_kind: dict[str, int] = field(default_factory=lambda: defaultdict(int))
|
|
423
|
-
http_calls_by_strategy: dict[str, int] = field(default_factory=lambda: defaultdict(int))
|
|
424
|
-
async_calls_by_strategy: dict[str, int] = field(default_factory=lambda: defaultdict(int))
|
|
425
|
-
http_calls_skipped_unresolved: int = 0
|
|
426
|
-
async_calls_skipped_unresolved: int = 0
|
|
427
|
-
http_clients_from_brownfield_pct: float = 0.0
|
|
428
|
-
async_producers_from_brownfield_pct: float = 0.0
|
|
429
|
-
http_calls_match_breakdown: dict[str, int] = field(default_factory=lambda: defaultdict(int))
|
|
430
|
-
async_calls_match_breakdown: dict[str, int] = field(default_factory=lambda: defaultdict(int))
|
|
431
|
-
cross_service_calls_total: int = 0
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
@dataclass
|
|
435
|
-
class GraphTables:
|
|
436
|
-
types: dict[str, TypeIndexEntry] = field(default_factory=dict) # fqn -> entry
|
|
437
|
-
by_simple_name: dict[str, list[TypeIndexEntry]] = field(default_factory=dict)
|
|
438
|
-
by_package: dict[str, list[TypeIndexEntry]] = field(default_factory=dict)
|
|
439
|
-
files: dict[str, str] = field(default_factory=dict) # path -> node id
|
|
440
|
-
packages: dict[str, str] = field(default_factory=dict) # pkg -> node id
|
|
441
|
-
members: list[MemberEntry] = field(default_factory=list)
|
|
442
|
-
phantoms: dict[str, dict] = field(default_factory=dict) # id -> row
|
|
443
|
-
extends_rows: list[EdgeRow] = field(default_factory=list)
|
|
444
|
-
implements_rows: list[EdgeRow] = field(default_factory=list)
|
|
445
|
-
injects_rows: list[InjectsRow] = field(default_factory=list)
|
|
446
|
-
calls_rows: list[CallsRow] = field(default_factory=list)
|
|
447
|
-
unresolved_call_site_rows: list[UnresolvedCallSiteRow] = field(default_factory=list)
|
|
448
|
-
declares_rows: list[DeclaresRow] = field(default_factory=list)
|
|
449
|
-
routes_rows: list[RouteRow] = field(default_factory=list)
|
|
450
|
-
exposes_rows: list[ExposesRow] = field(default_factory=list)
|
|
451
|
-
http_call_rows: list[HttpCallRow] = field(default_factory=list)
|
|
452
|
-
async_call_rows: list[AsyncCallRow] = field(default_factory=list)
|
|
453
|
-
client_rows: list[ClientRow] = field(default_factory=list)
|
|
454
|
-
declares_client_rows: list[DeclaresClientRow] = field(default_factory=list)
|
|
455
|
-
producer_rows: list[ProducerRow] = field(default_factory=list)
|
|
456
|
-
declares_producer_rows: list[DeclaresProducerRow] = field(default_factory=list)
|
|
457
|
-
overrides_rows: list[DeclaresRow] = field(default_factory=list)
|
|
458
|
-
route_stats: RouteExtractionStats = field(default_factory=RouteExtractionStats)
|
|
459
|
-
call_edge_stats: CallEdgeStats = field(default_factory=CallEdgeStats)
|
|
460
|
-
client_stats: ClientExtractionStats = field(default_factory=ClientExtractionStats)
|
|
461
|
-
producer_stats: ProducerExtractionStats = field(default_factory=ProducerExtractionStats)
|
|
462
|
-
methods_by_type: dict[str, list[MemberEntry]] = field(default_factory=dict)
|
|
463
|
-
parse_errors: int = 0
|
|
464
|
-
skipped_files: int = 0
|
|
465
|
-
pass3_skipped_cross_service: int = 0
|
|
466
|
-
pass3_unresolved_phantom_receiver: int = 0
|
|
467
|
-
pass3_unresolved_chained: int = 0
|
|
468
|
-
cross_service_resolution: str = "auto"
|
|
469
|
-
# Populated in _write_nodes (same overrides + meta_chain as Symbol.role).
|
|
470
|
-
type_role_by_node_id: dict[str, str] = field(default_factory=dict)
|
|
471
|
-
# Populated in pass 1 (classify_java_file) and _load_existing_types for incremental rebuilds.
|
|
472
|
-
type_generated_by_node_id: dict[str, tuple[bool, str | None]] = field(default_factory=dict)
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
@dataclass
|
|
476
|
-
class IncrementalResult:
|
|
477
|
-
"""Result of an incremental graph rebuild."""
|
|
478
|
-
mode: str # "incremental" | "full_fallback"
|
|
479
|
-
files_changed: int
|
|
480
|
-
files_added: int
|
|
481
|
-
files_removed: int
|
|
482
|
-
dependents_reprocessed: int
|
|
483
|
-
elapsed_sec: float
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
# --- Builder-owned files in the index dir (single source of truth) ---------------
|
|
487
|
-
# Every artifact the graph builder writes next to code_graph.lbug. The lifecycle
|
|
488
|
-
# CLI's `erase` clears all of these from one list so the builder and erase cannot
|
|
489
|
-
# drift (issues #349 / #350): previously erase hardcoded ".graph_hashes.json" only
|
|
490
|
-
# and left the crash marker (.graph_increment_in_progress) and the atomic-write
|
|
491
|
-
# temp (.graph_hashes.json.tmp) behind on disk.
|
|
492
|
-
GRAPH_HASHES_FILENAME = ".graph_hashes.json"
|
|
493
|
-
GRAPH_HASHES_TMP_FILENAME = ".graph_hashes.json.tmp"
|
|
494
|
-
GRAPH_INCREMENT_MARKER_FILENAME = ".graph_increment_in_progress"
|
|
495
|
-
BUILDER_OWNED_INDEX_FILES: tuple[str, ...] = (
|
|
496
|
-
GRAPH_HASHES_FILENAME,
|
|
497
|
-
GRAPH_HASHES_TMP_FILENAME,
|
|
498
|
-
GRAPH_INCREMENT_MARKER_FILENAME,
|
|
499
|
-
)
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
class FileHashTracker:
|
|
503
|
-
"""Track content hashes for incremental graph rebuild."""
|
|
504
|
-
def __init__(self, index_dir: Path):
|
|
505
|
-
self._path = index_dir / GRAPH_HASHES_FILENAME
|
|
506
|
-
self._hashes: dict[str, str] = {} # rel_path -> sha256_hex
|
|
507
|
-
|
|
508
|
-
def load(self) -> None:
|
|
509
|
-
"""Load hashes from disk. No-op if file missing (first run)."""
|
|
510
|
-
if not self._path.exists():
|
|
511
|
-
return
|
|
512
|
-
try:
|
|
513
|
-
with open(self._path, "r", encoding="utf-8") as f:
|
|
514
|
-
self._hashes = json.load(f)
|
|
515
|
-
except (json.JSONDecodeError, OSError):
|
|
516
|
-
# Corrupt or unreadable hash file; start fresh.
|
|
517
|
-
self._hashes = {}
|
|
518
|
-
|
|
519
|
-
def save(self) -> None:
|
|
520
|
-
"""Persist hashes to disk atomically (write .tmp, rename)."""
|
|
521
|
-
tmp_path = self._path.parent / GRAPH_HASHES_TMP_FILENAME
|
|
522
|
-
try:
|
|
523
|
-
with open(tmp_path, "w", encoding="utf-8") as f:
|
|
524
|
-
json.dump(self._hashes, f, sort_keys=True)
|
|
525
|
-
os.replace(tmp_path, self._path)
|
|
526
|
-
except OSError as e:
|
|
527
|
-
# Fail gracefully; next run will treat as missing and rebuild.
|
|
528
|
-
log.warning("Failed to save hash file %s: %s; next run will rebuild from scratch", self._path, e)
|
|
529
|
-
|
|
530
|
-
def detect_changes(self, source_root: Path, ignore: LayeredIgnore) -> tuple[set[str], set[str], set[str]]:
|
|
531
|
-
"""Return (added, changed, removed) sets of relative POSIX paths."""
|
|
532
|
-
current_files: set[str] = set()
|
|
533
|
-
# Resolve source_root to handle symlinks
|
|
534
|
-
source_root_resolved = source_root.resolve()
|
|
535
|
-
for abs_path in iter_java_source_files(source_root, ignore=ignore):
|
|
536
|
-
# Resolve the absolute path and compute relative path
|
|
537
|
-
abs_path_resolved = abs_path.resolve()
|
|
538
|
-
try:
|
|
539
|
-
rel_path = abs_path_resolved.relative_to(source_root_resolved).as_posix()
|
|
540
|
-
except ValueError:
|
|
541
|
-
# Fallback to using the path as-is if it's not under source_root
|
|
542
|
-
rel_path = abs_path.as_posix()
|
|
543
|
-
current_files.add(rel_path)
|
|
544
|
-
|
|
545
|
-
added: set[str] = set()
|
|
546
|
-
changed: set[str] = set()
|
|
547
|
-
removed: set[str] = set()
|
|
548
|
-
|
|
549
|
-
# Detect added and changed files.
|
|
550
|
-
for rel_path in current_files:
|
|
551
|
-
abs_path = source_root / rel_path
|
|
552
|
-
try:
|
|
553
|
-
file_hash = _hash_file(abs_path)
|
|
554
|
-
except FileNotFoundError:
|
|
555
|
-
continue
|
|
556
|
-
stored_hash = self._hashes.get(rel_path)
|
|
557
|
-
if stored_hash is None:
|
|
558
|
-
added.add(rel_path)
|
|
559
|
-
elif stored_hash != file_hash:
|
|
560
|
-
changed.add(rel_path)
|
|
561
|
-
|
|
562
|
-
# Detect removed files.
|
|
563
|
-
for rel_path in self._hashes:
|
|
564
|
-
if rel_path not in current_files:
|
|
565
|
-
removed.add(rel_path)
|
|
566
|
-
|
|
567
|
-
return added, changed, removed
|
|
568
|
-
|
|
569
|
-
def update(self, rel_paths: set[str], source_root: Path) -> None:
|
|
570
|
-
"""Compute and store hashes for the given paths."""
|
|
571
|
-
for rel_path in rel_paths:
|
|
572
|
-
abs_path = source_root / rel_path
|
|
573
|
-
if abs_path.exists():
|
|
574
|
-
self._hashes[rel_path] = _hash_file(abs_path)
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
def _hash_file(abs_path: Path) -> str:
|
|
578
|
-
"""Compute SHA-256 hash of a file's raw bytes."""
|
|
579
|
-
hasher = hashlib.sha256()
|
|
580
|
-
with open(abs_path, "rb") as f:
|
|
581
|
-
for chunk in iter(lambda: f.read(65536), b""):
|
|
582
|
-
hasher.update(chunk)
|
|
583
|
-
return hasher.hexdigest()
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
# ---------- incremental rebuild helpers ----------
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
def _load_existing_types(conn: ladybug.Connection, tables: GraphTables, exclude_files: set[str] | None = None) -> None:
|
|
590
|
-
"""Load type entries from existing LadybugDB graph into tables for cross-file resolution.
|
|
591
|
-
|
|
592
|
-
When exclude_files is provided, only load types from files NOT in the set.
|
|
593
|
-
"""
|
|
594
|
-
if exclude_files is not None and not exclude_files:
|
|
595
|
-
return
|
|
596
|
-
|
|
597
|
-
where = f"WHERE s.kind IN {list(_TYPE_KINDS)}"
|
|
598
|
-
params: dict = {}
|
|
599
|
-
if exclude_files:
|
|
600
|
-
where += "\n AND NOT (s.filename IN $exclude_files)"
|
|
601
|
-
params["exclude_files"] = list(exclude_files)
|
|
602
|
-
|
|
603
|
-
query = f"""
|
|
604
|
-
MATCH (s:Symbol)
|
|
605
|
-
{where}
|
|
606
|
-
RETURN s.kind, s.fqn, s.name, s.filename, s.module, s.microservice, s.id, s.role, s.generated, s.generated_by
|
|
607
|
-
"""
|
|
608
|
-
result = conn.execute(query, params)
|
|
609
|
-
while result.has_next():
|
|
610
|
-
row = result.get_next()
|
|
611
|
-
kind, fqn, name, filename = row[0], row[1], row[2], row[3]
|
|
612
|
-
module = row[4] if len(row) > 4 else ""
|
|
613
|
-
microservice = row[5] if len(row) > 5 else ""
|
|
614
|
-
node_id = row[6] if len(row) > 6 else ""
|
|
615
|
-
role = row[7] if len(row) > 7 else ""
|
|
616
|
-
generated = row[8] if len(row) > 8 else False
|
|
617
|
-
generated_by = row[9] if len(row) > 9 else ""
|
|
618
|
-
|
|
619
|
-
decl = TypeDecl(name, kind, fqn)
|
|
620
|
-
package = fqn[: -(len(name) + 1)] if fqn.endswith("." + name) else ""
|
|
621
|
-
|
|
622
|
-
entry = TypeIndexEntry(
|
|
623
|
-
decl=decl,
|
|
624
|
-
file_path=filename,
|
|
625
|
-
module=module,
|
|
626
|
-
microservice=microservice,
|
|
627
|
-
package=package,
|
|
628
|
-
outer_fqn=None,
|
|
629
|
-
node_id=node_id,
|
|
630
|
-
loaded_from_db=True,
|
|
631
|
-
)
|
|
632
|
-
tables.types[fqn] = entry
|
|
633
|
-
tables.by_simple_name.setdefault(name, []).append(entry)
|
|
634
|
-
tables.by_package.setdefault(package, []).append(entry)
|
|
635
|
-
# Seed the persisted role so the annotation-less stub is not recomputed to
|
|
636
|
-
# the default during node staging (issue #352 divergence #2).
|
|
637
|
-
if role:
|
|
638
|
-
tables.type_role_by_node_id[node_id] = role
|
|
639
|
-
# Seed the persisted generated/generated_by so stubs retain their values
|
|
640
|
-
tables.type_generated_by_node_id[node_id] = (bool(generated), generated_by)
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
def _load_existing_members(conn: ladybug.Connection, tables: GraphTables, exclude_files: set[str] | None = None) -> None:
|
|
644
|
-
"""Load member entries from existing LadybugDB graph into tables.members.
|
|
645
|
-
|
|
646
|
-
When exclude_files is provided, only load members from files NOT in the set.
|
|
647
|
-
"""
|
|
648
|
-
if exclude_files is not None and not exclude_files:
|
|
649
|
-
return
|
|
650
|
-
|
|
651
|
-
where = "WHERE s.kind IN ['method', 'constructor']"
|
|
652
|
-
params: dict = {}
|
|
653
|
-
if exclude_files:
|
|
654
|
-
where += "\n AND NOT (s.filename IN $exclude_files)"
|
|
655
|
-
params["exclude_files"] = list(exclude_files)
|
|
656
|
-
|
|
657
|
-
query = f"""
|
|
658
|
-
MATCH (s:Symbol)
|
|
659
|
-
{where}
|
|
660
|
-
RETURN s.kind, s.name, s.filename, s.signature, s.parent_id, s.fqn, s.id
|
|
661
|
-
"""
|
|
662
|
-
result = conn.execute(query, params)
|
|
663
|
-
while result.has_next():
|
|
664
|
-
row = result.get_next()
|
|
665
|
-
kind, name, filename = row[0], row[1], row[2]
|
|
666
|
-
signature = row[3] if len(row) > 3 else ""
|
|
667
|
-
parent_id = row[4] if len(row) > 4 else ""
|
|
668
|
-
fqn = row[5] if len(row) > 5 else ""
|
|
669
|
-
node_id = row[6] if len(row) > 6 else ""
|
|
670
|
-
|
|
671
|
-
parent_fqn = fqn.split("#")[0] if "#" in fqn else ""
|
|
672
|
-
|
|
673
|
-
decl = MethodDecl(name, "", kind == "constructor")
|
|
674
|
-
decl.signature = signature
|
|
675
|
-
|
|
676
|
-
tables.members.append(MemberEntry(
|
|
677
|
-
kind=kind,
|
|
678
|
-
decl=decl,
|
|
679
|
-
parent_id=parent_id,
|
|
680
|
-
parent_fqn=parent_fqn,
|
|
681
|
-
file_path=filename,
|
|
682
|
-
module="",
|
|
683
|
-
microservice="",
|
|
684
|
-
node_id=node_id,
|
|
685
|
-
loaded_from_db=True,
|
|
686
|
-
))
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
# Every Symbol->Symbol REL TABLE type in the graph schema. A Symbol node can
|
|
690
|
-
# only have an INCOMING edge of one of these types, so `_find_dependents` MUST
|
|
691
|
-
# walk all of them: that completeness is what makes the changed-node DETACH
|
|
692
|
-
# DELETE in `_delete_file_scope` Phase 3 safe (every real caller of a changed
|
|
693
|
-
# node is pulled into scope, so Phase 1 removes the edge before the node delete).
|
|
694
|
-
# If you add a new Symbol->Symbol edge type to the schema, add it here too —
|
|
695
|
-
# otherwise changed-node deletion would silently drop its surviving edges.
|
|
696
|
-
_SYMBOL_TO_SYMBOL_EDGE_TYPES = (
|
|
697
|
-
"EXTENDS", "IMPLEMENTS", "INJECTS", "CALLS", "DECLARES", "OVERRIDES",
|
|
698
|
-
)
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
def _find_dependents(conn: ladybug.Connection, changed_node_ids: set[str]) -> set[str]:
|
|
702
|
-
"""Find files whose nodes have edges pointing into changed nodes. Returns set of filenames."""
|
|
703
|
-
dependent_files: set[str] = set()
|
|
704
|
-
|
|
705
|
-
params = {"changed_ids": list(changed_node_ids)}
|
|
706
|
-
|
|
707
|
-
for edge_type in _SYMBOL_TO_SYMBOL_EDGE_TYPES:
|
|
708
|
-
query = f"""
|
|
709
|
-
MATCH (src:Symbol)-[e:{edge_type}]->(dst:Symbol)
|
|
710
|
-
WHERE dst.id IN $changed_ids
|
|
711
|
-
RETURN DISTINCT src.filename
|
|
712
|
-
"""
|
|
713
|
-
result = conn.execute(query, params)
|
|
714
|
-
while result.has_next():
|
|
715
|
-
row = result.get_next()
|
|
716
|
-
filename = row[0]
|
|
717
|
-
if filename: # Skip phantom nodes (filename = "")
|
|
718
|
-
dependent_files.add(filename)
|
|
719
|
-
|
|
720
|
-
return dependent_files
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
def _find_annotation_dependents(conn: ladybug.Connection, changed_node_ids: set[str]) -> set[str]:
|
|
724
|
-
"""Find files that USE an annotation whose DEFINITION is among the changed nodes.
|
|
725
|
-
|
|
726
|
-
Annotation usage is a node property (``annotations`` STRING[]), not a
|
|
727
|
-
Symbol->Symbol edge, so `_find_dependents` — which walks edges — never pulls
|
|
728
|
-
annotation users into scope. When an annotation definition changes (e.g.
|
|
729
|
-
``@interface Foo`` gains a meta-annotation that shifts the Layer-A chain in
|
|
730
|
-
`resolve_role_and_capabilities`), every type carrying ``@Foo`` may need its
|
|
731
|
-
``role``/``capabilities`` recomputed or it goes stale until the next full
|
|
732
|
-
rebuild. Return those users' files so the orchestrator treats them as
|
|
733
|
-
dependents (re-parsed, role re-SET); the expansion cap bounds the scope.
|
|
734
|
-
|
|
735
|
-
Scope is direct usage only: a user of an annotation that transitively
|
|
736
|
-
composes the changed one (e.g. ``@A`` where ``@A`` is meta-annotated with the
|
|
737
|
-
changed ``@B``) is NOT pulled in — that reverse-chain walk is left to a
|
|
738
|
-
future hardening pass. The direct case covers the dominant real-world shape
|
|
739
|
-
(a stereotype annotation applied directly to many types).
|
|
740
|
-
"""
|
|
741
|
-
if not changed_node_ids:
|
|
742
|
-
return set()
|
|
743
|
-
# Changed annotation definitions → the simple names users reference them by.
|
|
744
|
-
# Runs before `_delete_file_scope`, so the def nodes still exist.
|
|
745
|
-
name_result = conn.execute(
|
|
746
|
-
"MATCH (s:Symbol) WHERE s.id IN $ids AND s.kind = 'annotation' RETURN s.name",
|
|
747
|
-
{"ids": list(changed_node_ids)},
|
|
748
|
-
)
|
|
749
|
-
names: list[str] = []
|
|
750
|
-
while name_result.has_next():
|
|
751
|
-
nm = name_result.get_next()[0]
|
|
752
|
-
if nm:
|
|
753
|
-
names.append(nm)
|
|
754
|
-
if not names:
|
|
755
|
-
return set()
|
|
756
|
-
dependent_files: set[str] = set()
|
|
757
|
-
for nm in names:
|
|
758
|
-
user_result = conn.execute(
|
|
759
|
-
"MATCH (s:Symbol) "
|
|
760
|
-
"WHERE list_contains(s.annotations, $nm) AND s.filename <> '' "
|
|
761
|
-
"RETURN DISTINCT s.filename",
|
|
762
|
-
{"nm": nm},
|
|
763
|
-
)
|
|
764
|
-
while user_result.has_next():
|
|
765
|
-
fn = user_result.get_next()[0]
|
|
766
|
-
if fn:
|
|
767
|
-
dependent_files.add(fn)
|
|
768
|
-
return dependent_files
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
def _delete_file_scope(
|
|
772
|
-
conn: ladybug.Connection,
|
|
773
|
-
changed_files: set[str],
|
|
774
|
-
dependent_files: set[str],
|
|
775
|
-
) -> None:
|
|
776
|
-
"""Delete nodes and edges for a scope split into changed vs dependent files.
|
|
777
|
-
|
|
778
|
-
``changed_files`` are files whose content actually changed (added/modified/
|
|
779
|
-
removed): their Symbol nodes are deleted (and re-created by ``_scoped_write``).
|
|
780
|
-
``dependent_files`` are files pulled in only to re-resolve their OUTGOING
|
|
781
|
-
edges against the changed nodes; their node definitions did not change, so
|
|
782
|
-
their nodes are deliberately PRESERVED (they re-MERGE in place on the same
|
|
783
|
-
deterministic ``symbol_id``). Skipping phantom nodes (filename="").
|
|
784
|
-
|
|
785
|
-
Why dependents are preserved (issue #305): the orchestrator computes
|
|
786
|
-
dependents from the *changed* nodes only, so a dependent file's node can
|
|
787
|
-
have an incoming CALLS edge from an out-of-scope caller. The ``source_file``
|
|
788
|
-
on every Symbol->Symbol edge is the CALLER's file (pinned by
|
|
789
|
-
``test_source_file_value_matches_symbol_filename``), so Phase 1 below only
|
|
790
|
-
deletes edges ORIGINATING in scope; incoming edges from out-of-scope callers
|
|
791
|
-
survive. If we then tried to DELETE the dependent node, LadybugDB rejects it
|
|
792
|
-
("Node ... has connected edges in table CALLS in the bwd direction, ...
|
|
793
|
-
Please delete the edges first or try DETACH DELETE") and the rebuild falls
|
|
794
|
-
back to a full rebuild. A naive fix (DETACH DELETE on dependents, or an
|
|
795
|
-
extra incoming-edge pass) would silence the crash but permanently drop those
|
|
796
|
-
out-of-scope edges, corrupting the graph. Preserving dependent nodes keeps
|
|
797
|
-
both the nodes and their incoming edges intact.
|
|
798
|
-
|
|
799
|
-
Phase 1 deletes ALL edge types across the whole scope (changed + dependent)
|
|
800
|
-
first to avoid LadybugDB "has connected edges" errors when edges from one
|
|
801
|
-
file point to nodes in another file within the same scope. Route/Client/
|
|
802
|
-
Producer nodes use DETACH DELETE as a safety net for any edges missed in
|
|
803
|
-
Phase 1.
|
|
804
|
-
"""
|
|
805
|
-
# Symbol DELETEs below maintain the FTS index, which needs the extension loaded on
|
|
806
|
-
# this connection. Idempotent + best-effort (no-op when no index exists / FTS absent).
|
|
807
|
-
try:
|
|
808
|
-
conn.execute("LOAD EXTENSION FTS")
|
|
809
|
-
except Exception:
|
|
810
|
-
pass
|
|
811
|
-
scope_files = changed_files | dependent_files
|
|
812
|
-
scope_list = list(scope_files)
|
|
813
|
-
changed_list = list(changed_files)
|
|
814
|
-
|
|
815
|
-
# Phase 1: Delete ALL edges ORIGINATING from any scope file (changed +
|
|
816
|
-
# dependent). Because `source_file` is the caller's file, this deletes edges
|
|
817
|
-
# whose source is in scope (including dependents' outgoing edges to changed
|
|
818
|
-
# nodes) while intentionally leaving incoming edges from out-of-scope callers
|
|
819
|
-
# intact — those must survive so the dependent nodes below can be preserved.
|
|
820
|
-
# This list is a superset of `_SYMBOL_TO_SYMBOL_EDGE_TYPES` (it also covers
|
|
821
|
-
# Symbol->Route/Client/Producer/UCS and Client/Producer->Route edges); keep
|
|
822
|
-
# both lists in sync with the schema.
|
|
823
|
-
edge_tables = [
|
|
824
|
-
"EXTENDS", "IMPLEMENTS", "INJECTS", "CALLS", "DECLARES", "OVERRIDES",
|
|
825
|
-
"UNRESOLVED_AT", "EXPOSES", "DECLARES_CLIENT", "DECLARES_PRODUCER",
|
|
826
|
-
"HTTP_CALLS", "ASYNC_CALLS",
|
|
827
|
-
]
|
|
828
|
-
for edge_type in edge_tables:
|
|
829
|
-
query = f"""
|
|
830
|
-
MATCH (src)-[e:{edge_type}]->(dst)
|
|
831
|
-
WHERE e.source_file IN $filenames
|
|
832
|
-
DELETE e
|
|
833
|
-
"""
|
|
834
|
-
conn.execute(query, {"filenames": scope_list})
|
|
835
|
-
|
|
836
|
-
# Phase 2: Collect all Symbol node IDs for UnresolvedCallSite cleanup.
|
|
837
|
-
symbol_ids: list[str] = []
|
|
838
|
-
symbol_ids_query = """
|
|
839
|
-
MATCH (s:Symbol)
|
|
840
|
-
WHERE s.filename IN $filenames
|
|
841
|
-
RETURN s.id
|
|
842
|
-
"""
|
|
843
|
-
result = conn.execute(symbol_ids_query, {"filenames": scope_list})
|
|
844
|
-
while result.has_next():
|
|
845
|
-
row = result.get_next()
|
|
846
|
-
symbol_ids.append(row[0])
|
|
847
|
-
|
|
848
|
-
# Delete UnresolvedCallSite nodes whose caller_id is in the collected set.
|
|
849
|
-
# These are children of scope symbols (including preserved dependents);
|
|
850
|
-
# deleting them is safe because every scope file — dependents included — is
|
|
851
|
-
# reprocessed and re-emits its UnresolvedCallSite nodes in `_scoped_write`.
|
|
852
|
-
if symbol_ids:
|
|
853
|
-
unresolved_query = """
|
|
854
|
-
MATCH (u:UnresolvedCallSite)
|
|
855
|
-
WHERE u.caller_id IN $symbol_ids
|
|
856
|
-
DELETE u
|
|
857
|
-
"""
|
|
858
|
-
conn.execute(unresolved_query, {"symbol_ids": symbol_ids})
|
|
859
|
-
|
|
860
|
-
# Phase 3: Delete Symbol nodes ONLY for changed files (not dependents).
|
|
861
|
-
# Dependent-file nodes are deliberately PRESERVED so their incoming edges
|
|
862
|
-
# from out-of-scope callers survive; the dependents are re-MERGEd in place
|
|
863
|
-
# by `_scoped_write` on the same deterministic node id. A changed node's
|
|
864
|
-
# real incoming edges all come from dependent files (callers pulled into
|
|
865
|
-
# scope by `_find_dependents`, which walks every type in
|
|
866
|
-
# `_SYMBOL_TO_SYMBOL_EDGE_TYPES`), so Phase 1 already removed them and the
|
|
867
|
-
# dependents re-emit them when reprocessed. DETACH DELETE is only a safety
|
|
868
|
-
# net for the rare surviving edge whose source was NOT pulled into scope
|
|
869
|
-
# (e.g. a phantom caller with filename="", which `_find_dependents` skips);
|
|
870
|
-
# such an edge is stale once the node is recreated, so dropping it is fine.
|
|
871
|
-
delete_symbols_query = """
|
|
872
|
-
MATCH (s:Symbol)
|
|
873
|
-
WHERE s.filename IN $filenames
|
|
874
|
-
DETACH DELETE s
|
|
875
|
-
"""
|
|
876
|
-
conn.execute(delete_symbols_query, {"filenames": changed_list})
|
|
877
|
-
|
|
878
|
-
# Phase 4: Delete Route, Client, Producer nodes.
|
|
879
|
-
# Use DETACH DELETE as a safety net in case any edges were missed in Phase 1.
|
|
880
|
-
for label in ["Route", "Client", "Producer"]:
|
|
881
|
-
conn.execute(
|
|
882
|
-
f"MATCH (n:{label}) WHERE n.filename IN $filenames DETACH DELETE n",
|
|
883
|
-
{"filenames": scope_list},
|
|
884
|
-
)
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
def _scoped_write(conn: ladybug.Connection, tables: GraphTables, *, project_root: Path, meta_chain: dict[str, frozenset[str]] | None) -> None:
|
|
888
|
-
"""Write nodes and edges to existing LadybugDB database without drop/create schema.
|
|
889
|
-
|
|
890
|
-
Like write_ladybug() but without _drop_all()/_create_schema(). The caller is
|
|
891
|
-
responsible for calling _populate_declares_rows() and _populate_overrides_rows()
|
|
892
|
-
before invoking this function.
|
|
893
|
-
|
|
894
|
-
Uses MERGE instead of CREATE to handle cases where nodes already exist.
|
|
895
|
-
"""
|
|
896
|
-
t0 = time.time()
|
|
897
|
-
_write_nodes_merge(
|
|
898
|
-
conn,
|
|
899
|
-
tables,
|
|
900
|
-
project_root=project_root,
|
|
901
|
-
meta_chain=meta_chain,
|
|
902
|
-
)
|
|
903
|
-
elapsed = time.time() - t0
|
|
904
|
-
if elapsed > 0.1: # Only log if significant
|
|
905
|
-
_verbose_stderr_line(f"[graph] scoped write · nodes written in {elapsed:.2f}s")
|
|
906
|
-
|
|
907
|
-
t1 = time.time()
|
|
908
|
-
_fbyid = _build_file_by_node_id(tables)
|
|
909
|
-
_write_edges(conn, tables, _fbyid)
|
|
910
|
-
elapsed = time.time() - t1
|
|
911
|
-
if elapsed > 0.1:
|
|
912
|
-
_verbose_stderr_line(f"[graph] scoped write · edges written in {elapsed:.2f}s")
|
|
913
|
-
|
|
914
|
-
t2 = time.time()
|
|
915
|
-
_write_routes_and_exposes(conn, tables, _fbyid)
|
|
916
|
-
elapsed = time.time() - t2
|
|
917
|
-
if elapsed > 0.1:
|
|
918
|
-
_verbose_stderr_line(f"[graph] scoped write · routes/exposes written in {elapsed:.2f}s")
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
def _write_nodes_merge(
|
|
922
|
-
conn: ladybug.Connection,
|
|
923
|
-
tables: GraphTables,
|
|
924
|
-
*,
|
|
925
|
-
project_root: Path,
|
|
926
|
-
meta_chain: dict[str, frozenset[str]] | None,
|
|
927
|
-
) -> None:
|
|
928
|
-
"""Write nodes to existing LadybugDB database using bulk COPY FROM."""
|
|
929
|
-
_write_nodes_impl(conn, tables, project_root=project_root, meta_chain=meta_chain)
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
# ---------- file walk (see `path_filtering.iter_java_source_files`) ----------
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
# ---------- pass 1 ----------
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
def _register_type(
|
|
939
|
-
tables: GraphTables,
|
|
940
|
-
decl: TypeDecl,
|
|
941
|
-
*,
|
|
942
|
-
file_path: str,
|
|
943
|
-
module: str,
|
|
944
|
-
microservice: str,
|
|
945
|
-
outer_fqn: str | None,
|
|
946
|
-
) -> TypeIndexEntry:
|
|
947
|
-
package = decl.fqn.rsplit(".", 1)[0] if "." in decl.fqn and outer_fqn is None else (
|
|
948
|
-
outer_fqn.rsplit(".", 1)[0] if outer_fqn and "." in outer_fqn else ""
|
|
949
|
-
)
|
|
950
|
-
# top-level: package = fqn - name; nested: inherit from outer
|
|
951
|
-
if outer_fqn is None:
|
|
952
|
-
package = decl.fqn[: -(len(decl.name) + 1)] if decl.fqn.endswith("." + decl.name) else ""
|
|
953
|
-
else:
|
|
954
|
-
# walk outward to find a top-level fqn; package is everything before its simple name
|
|
955
|
-
top = outer_fqn
|
|
956
|
-
while top in tables.types and tables.types[top].outer_fqn:
|
|
957
|
-
top = tables.types[top].outer_fqn # type: ignore[assignment]
|
|
958
|
-
package = top[: top.rfind(".")] if "." in top else ""
|
|
959
|
-
|
|
960
|
-
node_id = symbol_id(decl.kind, decl.fqn, file_path, decl.start_byte)
|
|
961
|
-
entry = TypeIndexEntry(
|
|
962
|
-
decl=decl,
|
|
963
|
-
file_path=file_path,
|
|
964
|
-
module=module,
|
|
965
|
-
microservice=microservice,
|
|
966
|
-
package=package,
|
|
967
|
-
outer_fqn=outer_fqn,
|
|
968
|
-
node_id=node_id,
|
|
969
|
-
)
|
|
970
|
-
tables.types[decl.fqn] = entry
|
|
971
|
-
tables.by_simple_name.setdefault(decl.name, []).append(entry)
|
|
972
|
-
tables.by_package.setdefault(package, []).append(entry)
|
|
973
|
-
|
|
974
|
-
for m in decl.methods:
|
|
975
|
-
kind = "constructor" if m.is_constructor else "method"
|
|
976
|
-
mid = symbol_id(kind, f"{decl.fqn}#{m.signature}", file_path, m.start_byte)
|
|
977
|
-
tables.members.append(MemberEntry(
|
|
978
|
-
kind=kind, decl=m, parent_id=node_id, parent_fqn=decl.fqn,
|
|
979
|
-
file_path=file_path, module=module, microservice=microservice,
|
|
980
|
-
node_id=mid,
|
|
981
|
-
))
|
|
982
|
-
|
|
983
|
-
for nested in decl.nested:
|
|
984
|
-
_register_type(
|
|
985
|
-
tables, nested, file_path=file_path,
|
|
986
|
-
module=module, microservice=microservice, outer_fqn=decl.fqn,
|
|
987
|
-
)
|
|
988
|
-
|
|
989
|
-
return entry
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
def pass1_parse(
|
|
993
|
-
root: Path,
|
|
994
|
-
tables: GraphTables,
|
|
995
|
-
*,
|
|
996
|
-
verbose: bool,
|
|
997
|
-
scope_files: set[str] | None = None,
|
|
998
|
-
removed_files: set[str] | None = None,
|
|
999
|
-
) -> dict[str, JavaFileAst]:
|
|
1000
|
-
"""Walk files, parse them, populate node indexes. Returns path -> AST.
|
|
1001
|
-
|
|
1002
|
-
Args:
|
|
1003
|
-
root: Source root directory.
|
|
1004
|
-
tables: GraphTables to populate.
|
|
1005
|
-
verbose: Whether to emit progress output.
|
|
1006
|
-
scope_files: Optional set of relative POSIX paths to parse. If None, parse all files.
|
|
1007
|
-
removed_files: Optional set of relative POSIX paths that no longer exist
|
|
1008
|
-
on disk (incremental deletions). These are members of ``scope_files``
|
|
1009
|
-
(they were deleted, so they participate in scoped deletion) but are
|
|
1010
|
-
never visited by the parse walk, so they must be excluded from the
|
|
1011
|
-
pass-1 total to keep ``done`` from undercounting then two-way-clamping.
|
|
1012
|
-
"""
|
|
1013
|
-
asts: dict[str, JavaFileAst] = {}
|
|
1014
|
-
ignore = LayeredIgnore(root)
|
|
1015
|
-
t0 = time.time()
|
|
1016
|
-
n_files = 0
|
|
1017
|
-
if verbose:
|
|
1018
|
-
_verbose_stderr_line(_PASS1_START)
|
|
1019
|
-
# Count-first: one filtered walk (no parsing) to set the EXACT total before
|
|
1020
|
-
# the parse loop ticks. Single-layer ignore → the count is exact, so the
|
|
1021
|
-
# rendered bar is determinate. For a scoped (incremental) parse the total is
|
|
1022
|
-
# the number of files that will actually be visited: scope minus any removed
|
|
1023
|
-
# files (which are members of scope for deletion but gone from disk, so the
|
|
1024
|
-
# parse walk never ticks them); for a full rebuild it is the non-ignored
|
|
1025
|
-
# .java count.
|
|
1026
|
-
if verbose:
|
|
1027
|
-
if scope_files is not None:
|
|
1028
|
-
removed = removed_files if removed_files is not None else set()
|
|
1029
|
-
pass1_total = len(scope_files - removed)
|
|
1030
|
-
else:
|
|
1031
|
-
pass1_total = sum(1 for _ in iter_java_source_files(root, ignore=ignore))
|
|
1032
|
-
_emit_graph_progress(
|
|
1033
|
-
{"pass": "1/6", "done": 0, "total": pass1_total, "status": "running"},
|
|
1034
|
-
verbose=verbose,
|
|
1035
|
-
)
|
|
1036
|
-
slow_sec = 0.0
|
|
1037
|
-
raw_slow = os.environ.get("JAVA_CODEBASE_RAG_TEST_GRAPH_SLOW_SEC", "").strip()
|
|
1038
|
-
if raw_slow:
|
|
1039
|
-
try:
|
|
1040
|
-
slow_sec = float(raw_slow)
|
|
1041
|
-
except ValueError:
|
|
1042
|
-
slow_sec = 0.0
|
|
1043
|
-
with _VerbosePassHeartbeats("[graph] pass 1", verbose=verbose):
|
|
1044
|
-
if verbose and slow_sec > 0:
|
|
1045
|
-
time.sleep(slow_sec)
|
|
1046
|
-
for p in iter_java_source_files(root, ignore=ignore):
|
|
1047
|
-
# Skip files not in scope (if scope is provided)
|
|
1048
|
-
try:
|
|
1049
|
-
rel = p.resolve().relative_to(root.resolve()).as_posix()
|
|
1050
|
-
except ValueError:
|
|
1051
|
-
rel = p.as_posix()
|
|
1052
|
-
if scope_files is not None and rel not in scope_files:
|
|
1053
|
-
continue
|
|
1054
|
-
n_files += 1
|
|
1055
|
-
if verbose and (n_files % _PASS1_TICK_EVERY == 0):
|
|
1056
|
-
_emit_graph_progress(
|
|
1057
|
-
{"pass": "1/6", "done": n_files, "status": "running"},
|
|
1058
|
-
verbose=verbose,
|
|
1059
|
-
)
|
|
1060
|
-
try:
|
|
1061
|
-
content = p.read_bytes()
|
|
1062
|
-
except OSError:
|
|
1063
|
-
tables.skipped_files += 1
|
|
1064
|
-
continue
|
|
1065
|
-
if not content.strip():
|
|
1066
|
-
continue
|
|
1067
|
-
try:
|
|
1068
|
-
ast = parse_java(content, filename=rel, verbose=verbose)
|
|
1069
|
-
except Exception:
|
|
1070
|
-
tables.parse_errors += 1
|
|
1071
|
-
continue
|
|
1072
|
-
if ast.parse_error:
|
|
1073
|
-
tables.parse_errors += 1
|
|
1074
|
-
# Still index what tree-sitter gave us; robust to syntax errors.
|
|
1075
|
-
module = module_for_path(str(p), root)
|
|
1076
|
-
microservice = microservice_for_path(str(p), root)
|
|
1077
|
-
asts[rel] = ast
|
|
1078
|
-
|
|
1079
|
-
# Classify the file once (generated or not, and which tool generated it)
|
|
1080
|
-
generated_config = load_generated_detection(str(root))
|
|
1081
|
-
file_generated, file_generated_by = classify_java_file(
|
|
1082
|
-
content, ast, config=generated_config, project_root=root
|
|
1083
|
-
)
|
|
1084
|
-
|
|
1085
|
-
# file node
|
|
1086
|
-
file_id = symbol_id("file", rel, rel, 0)
|
|
1087
|
-
tables.files[rel] = file_id
|
|
1088
|
-
|
|
1089
|
-
# package node (created lazily; nodes deduped by id)
|
|
1090
|
-
if ast.package and ast.package not in tables.packages:
|
|
1091
|
-
tables.packages[ast.package] = symbol_id("package", ast.package, "", 0)
|
|
1092
|
-
|
|
1093
|
-
for t in ast.top_level_types:
|
|
1094
|
-
_register_type(
|
|
1095
|
-
tables, t, file_path=rel,
|
|
1096
|
-
module=module, microservice=microservice, outer_fqn=None,
|
|
1097
|
-
)
|
|
1098
|
-
|
|
1099
|
-
# Seed generated/generated_by for all types in this file (including nested)
|
|
1100
|
-
for t in ast.all_types:
|
|
1101
|
-
if t.fqn in tables.types:
|
|
1102
|
-
node_id = tables.types[t.fqn].node_id
|
|
1103
|
-
tables.type_generated_by_node_id[node_id] = (file_generated, file_generated_by)
|
|
1104
|
-
|
|
1105
|
-
if verbose:
|
|
1106
|
-
elapsed = time.time() - t0
|
|
1107
|
-
_emit_graph_progress(
|
|
1108
|
-
{"pass": "1/6", "done": n_files, "status": "done", "elapsed_s": f"{elapsed:.2f}"},
|
|
1109
|
-
verbose=verbose,
|
|
1110
|
-
)
|
|
1111
|
-
_verbose_stderr_line(
|
|
1112
|
-
f"[graph] pass 1 · parsed {n_files} files in {elapsed:.2f}s: "
|
|
1113
|
-
f"{len(tables.types)} types, {len(tables.members)} members, "
|
|
1114
|
-
f"{tables.parse_errors} parse errors, {tables.skipped_files} skipped",
|
|
1115
|
-
)
|
|
1116
|
-
return asts
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
# ---------- pass 2: resolution + edges ----------
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
def _resolve_simple(
|
|
1123
|
-
name: str,
|
|
1124
|
-
*,
|
|
1125
|
-
current: TypeIndexEntry,
|
|
1126
|
-
ast: JavaFileAst,
|
|
1127
|
-
tables: GraphTables,
|
|
1128
|
-
) -> TypeIndexEntry | None:
|
|
1129
|
-
"""Java-ish name resolution. Returns a known TypeIndexEntry or None (phantom)."""
|
|
1130
|
-
# Strip trailing generics the caller may have left in, defensively.
|
|
1131
|
-
bare = name.split("<", 1)[0].strip()
|
|
1132
|
-
if not bare:
|
|
1133
|
-
return None
|
|
1134
|
-
|
|
1135
|
-
# 0. Nested inside the same top-level hierarchy — try `Outer.Bare` fqn.
|
|
1136
|
-
outer = current.outer_fqn
|
|
1137
|
-
while outer is not None and outer in tables.types:
|
|
1138
|
-
candidate = f"{outer}.{bare}"
|
|
1139
|
-
if candidate in tables.types:
|
|
1140
|
-
return tables.types[candidate]
|
|
1141
|
-
outer = tables.types[outer].outer_fqn
|
|
1142
|
-
|
|
1143
|
-
# 1. Same-file siblings (same outer as `current`).
|
|
1144
|
-
same_outer = current.outer_fqn or current.package
|
|
1145
|
-
for e in tables.by_simple_name.get(bare, ()):
|
|
1146
|
-
e_parent = e.outer_fqn or e.package
|
|
1147
|
-
if e.file_path == current.file_path and e_parent == same_outer:
|
|
1148
|
-
return e
|
|
1149
|
-
|
|
1150
|
-
# 2. Explicit import.
|
|
1151
|
-
if bare in ast.explicit_imports:
|
|
1152
|
-
fq = ast.explicit_imports[bare]
|
|
1153
|
-
if fq in tables.types:
|
|
1154
|
-
return tables.types[fq]
|
|
1155
|
-
# Known FQN (outside our codebase) → unresolved; caller will phantom-ise.
|
|
1156
|
-
return None
|
|
1157
|
-
|
|
1158
|
-
# 3. Same package.
|
|
1159
|
-
if current.package:
|
|
1160
|
-
candidate = f"{current.package}.{bare}"
|
|
1161
|
-
if candidate in tables.types:
|
|
1162
|
-
return tables.types[candidate]
|
|
1163
|
-
|
|
1164
|
-
# 4. Wildcard imports.
|
|
1165
|
-
for wild in ast.wildcard_imports:
|
|
1166
|
-
candidate = f"{wild}.{bare}"
|
|
1167
|
-
if candidate in tables.types:
|
|
1168
|
-
return tables.types[candidate]
|
|
1169
|
-
|
|
1170
|
-
# 5. java.lang best-effort (unresolved but deterministic phantom).
|
|
1171
|
-
return None
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
def _phantom_target(
|
|
1175
|
-
tables: GraphTables,
|
|
1176
|
-
simple: str,
|
|
1177
|
-
ast: JavaFileAst,
|
|
1178
|
-
*,
|
|
1179
|
-
current: TypeIndexEntry,
|
|
1180
|
-
) -> tuple[str, str, str]:
|
|
1181
|
-
"""Produce (id, simple, fqn-or-best-guess) for an unresolved type reference.
|
|
1182
|
-
|
|
1183
|
-
The fqn falls back through: explicit import → wildcard → java.lang → bare name.
|
|
1184
|
-
"""
|
|
1185
|
-
bare = simple.split("<", 1)[0].strip()
|
|
1186
|
-
guess_fqn = bare
|
|
1187
|
-
if bare in ast.explicit_imports:
|
|
1188
|
-
guess_fqn = ast.explicit_imports[bare]
|
|
1189
|
-
elif bare in _JAVA_LANG_SIMPLE:
|
|
1190
|
-
guess_fqn = f"java.lang.{bare}"
|
|
1191
|
-
elif ast.wildcard_imports:
|
|
1192
|
-
# Pick first wildcard as a hint (imperfect but useful for display).
|
|
1193
|
-
guess_fqn = f"{ast.wildcard_imports[0]}.{bare}"
|
|
1194
|
-
|
|
1195
|
-
pid = phantom_id(guess_fqn)
|
|
1196
|
-
if pid not in tables.phantoms:
|
|
1197
|
-
tables.phantoms[pid] = {
|
|
1198
|
-
"id": pid,
|
|
1199
|
-
"kind": "class",
|
|
1200
|
-
"name": bare,
|
|
1201
|
-
"fqn": guess_fqn,
|
|
1202
|
-
"package": guess_fqn.rsplit(".", 1)[0] if "." in guess_fqn else "",
|
|
1203
|
-
"module": "",
|
|
1204
|
-
"microservice": "",
|
|
1205
|
-
"filename": "",
|
|
1206
|
-
"start_line": 0,
|
|
1207
|
-
"end_line": 0,
|
|
1208
|
-
"start_byte": 0,
|
|
1209
|
-
"end_byte": 0,
|
|
1210
|
-
"modifiers": [],
|
|
1211
|
-
"annotations": [],
|
|
1212
|
-
"capabilities": [],
|
|
1213
|
-
"role": "OTHER",
|
|
1214
|
-
"signature": "",
|
|
1215
|
-
"parent_id": "",
|
|
1216
|
-
"resolved": False,
|
|
1217
|
-
}
|
|
1218
|
-
return pid, bare, guess_fqn
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
def _edge_for(
|
|
1222
|
-
*,
|
|
1223
|
-
src: TypeIndexEntry,
|
|
1224
|
-
target_simple: str,
|
|
1225
|
-
ast: JavaFileAst,
|
|
1226
|
-
tables: GraphTables,
|
|
1227
|
-
) -> tuple[str, str, str, bool]:
|
|
1228
|
-
resolved = _resolve_simple(target_simple, current=src, ast=ast, tables=tables)
|
|
1229
|
-
if resolved is not None:
|
|
1230
|
-
return resolved.node_id, resolved.decl.name, resolved.decl.fqn, True
|
|
1231
|
-
pid, simple, fqn = _phantom_target(tables, target_simple, ast, current=src)
|
|
1232
|
-
return pid, simple, fqn, False
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
def _emit_extends_implements(
|
|
1236
|
-
entry: TypeIndexEntry,
|
|
1237
|
-
ast: JavaFileAst,
|
|
1238
|
-
tables: GraphTables,
|
|
1239
|
-
*,
|
|
1240
|
-
seen_ext: set[tuple[str, str]],
|
|
1241
|
-
seen_impl: set[tuple[str, str]],
|
|
1242
|
-
) -> None:
|
|
1243
|
-
for name in entry.decl.extends:
|
|
1244
|
-
dst_id, dst_simple, dst_fqn, ok = _edge_for(
|
|
1245
|
-
src=entry, target_simple=name, ast=ast, tables=tables,
|
|
1246
|
-
)
|
|
1247
|
-
key = (entry.node_id, dst_id)
|
|
1248
|
-
if key in seen_ext:
|
|
1249
|
-
continue
|
|
1250
|
-
seen_ext.add(key)
|
|
1251
|
-
tables.extends_rows.append(EdgeRow(
|
|
1252
|
-
src_id=entry.node_id, dst_id=dst_id,
|
|
1253
|
-
dst_name=dst_simple, dst_fqn=dst_fqn, resolved=ok,
|
|
1254
|
-
))
|
|
1255
|
-
|
|
1256
|
-
for name in entry.decl.implements:
|
|
1257
|
-
dst_id, dst_simple, dst_fqn, ok = _edge_for(
|
|
1258
|
-
src=entry, target_simple=name, ast=ast, tables=tables,
|
|
1259
|
-
)
|
|
1260
|
-
key = (entry.node_id, dst_id)
|
|
1261
|
-
if key in seen_impl:
|
|
1262
|
-
continue
|
|
1263
|
-
seen_impl.add(key)
|
|
1264
|
-
tables.implements_rows.append(EdgeRow(
|
|
1265
|
-
src_id=entry.node_id, dst_id=dst_id,
|
|
1266
|
-
dst_name=dst_simple, dst_fqn=dst_fqn, resolved=ok,
|
|
1267
|
-
))
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
def _emit_injects(
|
|
1271
|
-
entry: TypeIndexEntry,
|
|
1272
|
-
ast: JavaFileAst,
|
|
1273
|
-
tables: GraphTables,
|
|
1274
|
-
*,
|
|
1275
|
-
seen: set[tuple[str, str, str, str]],
|
|
1276
|
-
) -> None:
|
|
1277
|
-
if entry.decl.kind == "interface":
|
|
1278
|
-
return
|
|
1279
|
-
|
|
1280
|
-
ann_names = [a.name for a in entry.decl.annotations]
|
|
1281
|
-
inject_set = injection_annotation_names()
|
|
1282
|
-
lombok_rac = lombok_required_args_annotations()
|
|
1283
|
-
has_lombok_rac = any(a in lombok_rac for a in ann_names)
|
|
1284
|
-
|
|
1285
|
-
def _add(
|
|
1286
|
-
target: str, mechanism: str, annotation: str, slot: str,
|
|
1287
|
-
) -> None:
|
|
1288
|
-
dst_id, dst_simple, dst_fqn, ok = _edge_for(
|
|
1289
|
-
src=entry, target_simple=target, ast=ast, tables=tables,
|
|
1290
|
-
)
|
|
1291
|
-
key = (entry.node_id, dst_id, mechanism, slot)
|
|
1292
|
-
if key in seen:
|
|
1293
|
-
return
|
|
1294
|
-
seen.add(key)
|
|
1295
|
-
tables.injects_rows.append(InjectsRow(
|
|
1296
|
-
src_id=entry.node_id, dst_id=dst_id,
|
|
1297
|
-
dst_name=dst_simple, dst_fqn=dst_fqn, resolved=ok,
|
|
1298
|
-
mechanism=mechanism, annotation=annotation, field_or_param=slot,
|
|
1299
|
-
))
|
|
1300
|
-
|
|
1301
|
-
# Field injection: @Autowired / @Inject / @Resource.
|
|
1302
|
-
for f in entry.decl.fields:
|
|
1303
|
-
annotated = next((a.name for a in f.annotations if a.name in inject_set), None)
|
|
1304
|
-
if annotated:
|
|
1305
|
-
_add(f.type_name, "field", annotated, f.name)
|
|
1306
|
-
|
|
1307
|
-
# Lombok: @RequiredArgsConstructor -> each `final` non-static field becomes an injection;
|
|
1308
|
-
# @AllArgsConstructor -> every non-static field.
|
|
1309
|
-
if has_lombok_rac:
|
|
1310
|
-
all_args = "AllArgsConstructor" in ann_names
|
|
1311
|
-
for f in entry.decl.fields:
|
|
1312
|
-
if "static" in f.modifiers:
|
|
1313
|
-
continue
|
|
1314
|
-
if not all_args and "final" not in f.modifiers:
|
|
1315
|
-
continue
|
|
1316
|
-
_add(f.type_name, "lombok_required_args",
|
|
1317
|
-
"AllArgsConstructor" if all_args else "RequiredArgsConstructor",
|
|
1318
|
-
f.name)
|
|
1319
|
-
|
|
1320
|
-
# Constructor injection:
|
|
1321
|
-
ctors = [m for m in entry.decl.methods if m.is_constructor]
|
|
1322
|
-
if ctors:
|
|
1323
|
-
chosen = None
|
|
1324
|
-
autowired = [c for c in ctors if any(a.name == "Autowired" for a in c.annotations)]
|
|
1325
|
-
if autowired:
|
|
1326
|
-
chosen = autowired[0]
|
|
1327
|
-
elif len(ctors) == 1 and ctors[0].parameters:
|
|
1328
|
-
chosen = ctors[0]
|
|
1329
|
-
if chosen is not None:
|
|
1330
|
-
annotation = "Autowired" if any(a.name == "Autowired" for a in chosen.annotations) else ""
|
|
1331
|
-
for p in chosen.parameters:
|
|
1332
|
-
_add(p.type_name, "constructor", annotation, p.name)
|
|
1333
|
-
|
|
1334
|
-
# Setter injection: setXxx annotated @Autowired with 1 parameter.
|
|
1335
|
-
for m in entry.decl.methods:
|
|
1336
|
-
if m.is_constructor or not m.name.startswith("set") or len(m.parameters) != 1:
|
|
1337
|
-
continue
|
|
1338
|
-
if any(a.name == "Autowired" for a in m.annotations):
|
|
1339
|
-
_add(m.parameters[0].type_name, "setter", "Autowired",
|
|
1340
|
-
m.parameters[0].name)
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
def pass2_edges(tables: GraphTables, asts: dict[str, JavaFileAst], *, verbose: bool) -> None:
|
|
1344
|
-
t0 = time.time()
|
|
1345
|
-
seen_ext: set[tuple[str, str]] = set()
|
|
1346
|
-
seen_impl: set[tuple[str, str]] = set()
|
|
1347
|
-
seen_inj: set[tuple[str, str, str, str]] = set()
|
|
1348
|
-
if verbose:
|
|
1349
|
-
_verbose_stderr_line(_PASS2_START)
|
|
1350
|
-
with _graph_pass_progress("2/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 2", verbose=verbose):
|
|
1351
|
-
for fqn, entry in tables.types.items():
|
|
1352
|
-
ast = asts.get(entry.file_path)
|
|
1353
|
-
if ast is None:
|
|
1354
|
-
continue
|
|
1355
|
-
_emit_extends_implements(entry, ast, tables, seen_ext=seen_ext, seen_impl=seen_impl)
|
|
1356
|
-
_emit_injects(entry, ast, tables, seen=seen_inj)
|
|
1357
|
-
if verbose:
|
|
1358
|
-
elapsed = time.time() - t0
|
|
1359
|
-
_verbose_stderr_line(
|
|
1360
|
-
f"[graph] pass 2 · emitted {len(tables.extends_rows)} EXTENDS, "
|
|
1361
|
-
f"{len(tables.implements_rows)} IMPLEMENTS, "
|
|
1362
|
-
f"{len(tables.injects_rows)} INJECTS, "
|
|
1363
|
-
f"{len(tables.phantoms)} phantoms in {elapsed:.2f}s",
|
|
1364
|
-
)
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
# ---------- pass 3: call graph ----------
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
def _build_member_indexes(tables: GraphTables) -> None:
|
|
1371
|
-
tables.methods_by_type = {}
|
|
1372
|
-
for m in tables.members:
|
|
1373
|
-
tables.methods_by_type.setdefault(m.parent_fqn, []).append(m)
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
def _direct_supertype_fqns(entry: TypeIndexEntry, tables: GraphTables) -> list[str]:
|
|
1377
|
-
out: list[str] = []
|
|
1378
|
-
for r in tables.extends_rows:
|
|
1379
|
-
if r.src_id == entry.node_id and r.dst_fqn in tables.types:
|
|
1380
|
-
out.append(r.dst_fqn)
|
|
1381
|
-
for r in tables.implements_rows:
|
|
1382
|
-
if r.src_id == entry.node_id and r.dst_fqn in tables.types:
|
|
1383
|
-
out.append(r.dst_fqn)
|
|
1384
|
-
return out
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
def _first_supertype_fqn(tables: GraphTables, type_fqn: str) -> str | None:
|
|
1388
|
-
entry = tables.types.get(type_fqn)
|
|
1389
|
-
if entry is None:
|
|
1390
|
-
return None
|
|
1391
|
-
for r in tables.extends_rows:
|
|
1392
|
-
if r.src_id == entry.node_id and r.dst_fqn in tables.types:
|
|
1393
|
-
return r.dst_fqn
|
|
1394
|
-
for r in tables.implements_rows:
|
|
1395
|
-
if r.src_id == entry.node_id and r.dst_fqn in tables.types:
|
|
1396
|
-
return r.dst_fqn
|
|
1397
|
-
return None
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
def _is_chained_receiver_text(receiver_expr: str) -> bool:
|
|
1401
|
-
"""Heuristic: call chain or complex expr (contains a completed call)."""
|
|
1402
|
-
s = receiver_expr.strip()
|
|
1403
|
-
return "(" in s and ")" in s
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
def _resolve_this_super_field_chain(
|
|
1407
|
-
expr: str,
|
|
1408
|
-
*,
|
|
1409
|
-
member: MemberEntry,
|
|
1410
|
-
ast: JavaFileAst,
|
|
1411
|
-
tables: GraphTables,
|
|
1412
|
-
) -> str | None:
|
|
1413
|
-
"""Resolve `this.a.b` / `super.a` (no calls) to the final field's type FQN."""
|
|
1414
|
-
s = expr.strip()
|
|
1415
|
-
if "(" in s or ")" in s or "." not in s:
|
|
1416
|
-
return None
|
|
1417
|
-
entry = tables.types.get(member.parent_fqn)
|
|
1418
|
-
if entry is None:
|
|
1419
|
-
return None
|
|
1420
|
-
parts = s.split(".")
|
|
1421
|
-
if len(parts) < 2:
|
|
1422
|
-
return None
|
|
1423
|
-
if parts[0] == "this":
|
|
1424
|
-
cur = entry
|
|
1425
|
-
elif parts[0] == "super":
|
|
1426
|
-
sup = _first_supertype_fqn(tables, member.parent_fqn)
|
|
1427
|
-
if sup is None or sup not in tables.types:
|
|
1428
|
-
return None
|
|
1429
|
-
cur = tables.types[sup]
|
|
1430
|
-
else:
|
|
1431
|
-
return None
|
|
1432
|
-
for fname in parts[1:]:
|
|
1433
|
-
fld = next((f for f in cur.decl.fields if f.name == fname), None)
|
|
1434
|
-
if fld is None:
|
|
1435
|
-
return None
|
|
1436
|
-
resolved = _resolve_simple(fld.type_name, current=cur, ast=ast, tables=tables)
|
|
1437
|
-
if resolved is None:
|
|
1438
|
-
return None
|
|
1439
|
-
cur = resolved
|
|
1440
|
-
return cur.decl.fqn
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
def _scope_table(member: MemberEntry, ast: JavaFileAst, tables: GraphTables) -> dict[str, str]:
|
|
1444
|
-
"""Map simple variable/field/param name -> resolved declaring type FQN."""
|
|
1445
|
-
scope: dict[str, str] = {}
|
|
1446
|
-
entry = tables.types.get(member.parent_fqn)
|
|
1447
|
-
if entry is None:
|
|
1448
|
-
return scope
|
|
1449
|
-
|
|
1450
|
-
def add_fields(tentry: TypeIndexEntry) -> None:
|
|
1451
|
-
for f in tentry.decl.fields:
|
|
1452
|
-
resolved = _resolve_simple(f.type_name, current=tentry, ast=ast, tables=tables)
|
|
1453
|
-
if resolved is not None:
|
|
1454
|
-
scope[f.name] = resolved.decl.fqn
|
|
1455
|
-
|
|
1456
|
-
add_fields(entry)
|
|
1457
|
-
seen: set[str] = {member.parent_fqn}
|
|
1458
|
-
queue = list(_direct_supertype_fqns(entry, tables))
|
|
1459
|
-
while queue:
|
|
1460
|
-
sup = queue.pop()
|
|
1461
|
-
if sup in seen or sup not in tables.types:
|
|
1462
|
-
continue
|
|
1463
|
-
seen.add(sup)
|
|
1464
|
-
te = tables.types[sup]
|
|
1465
|
-
for f in te.decl.fields:
|
|
1466
|
-
if f.name not in scope:
|
|
1467
|
-
resolved = _resolve_simple(f.type_name, current=te, ast=ast, tables=tables)
|
|
1468
|
-
if resolved is not None:
|
|
1469
|
-
scope[f.name] = resolved.decl.fqn
|
|
1470
|
-
queue.extend(_direct_supertype_fqns(te, tables))
|
|
1471
|
-
|
|
1472
|
-
for p in member.decl.parameters:
|
|
1473
|
-
resolved = _resolve_simple(p.type_name, current=entry, ast=ast, tables=tables)
|
|
1474
|
-
if resolved is not None:
|
|
1475
|
-
scope[p.name] = resolved.decl.fqn
|
|
1476
|
-
|
|
1477
|
-
# Locals shadow fields and parameters (same simple name → local wins).
|
|
1478
|
-
for name, t_simple in member.decl.local_vars:
|
|
1479
|
-
resolved = _resolve_simple(t_simple, current=entry, ast=ast, tables=tables)
|
|
1480
|
-
if resolved is not None:
|
|
1481
|
-
scope[name] = resolved.decl.fqn
|
|
1482
|
-
|
|
1483
|
-
return scope
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
def _lookup_method_candidates(
|
|
1487
|
-
type_fqn: str,
|
|
1488
|
-
callee_simple: str,
|
|
1489
|
-
arg_count: int,
|
|
1490
|
-
tables: GraphTables,
|
|
1491
|
-
ast: JavaFileAst,
|
|
1492
|
-
*,
|
|
1493
|
-
visited: set[str] | None = None,
|
|
1494
|
-
) -> tuple[list[MemberEntry], bool]:
|
|
1495
|
-
"""Return (candidates, used_name_only_fallback). Walks type + supertypes.
|
|
1496
|
-
|
|
1497
|
-
When ``used_name_only_fallback`` is true and ``len(candidates) == 1``, the
|
|
1498
|
-
caller may reuse the receiver-resolution strategy (see ``_resolve_and_emit_call``)
|
|
1499
|
-
instead of tagging ``overload_ambiguous``.
|
|
1500
|
-
"""
|
|
1501
|
-
if visited is None:
|
|
1502
|
-
visited = set()
|
|
1503
|
-
exact: list[MemberEntry] = []
|
|
1504
|
-
name_only: list[MemberEntry] = []
|
|
1505
|
-
|
|
1506
|
-
def collect_on_type(tfqn: str) -> None:
|
|
1507
|
-
nonlocal exact, name_only
|
|
1508
|
-
for m in tables.methods_by_type.get(tfqn, ()):
|
|
1509
|
-
if callee_simple == "<init>":
|
|
1510
|
-
if not m.decl.is_constructor:
|
|
1511
|
-
continue
|
|
1512
|
-
np = len(m.decl.parameters)
|
|
1513
|
-
if arg_count < 0:
|
|
1514
|
-
name_only.append(m)
|
|
1515
|
-
elif np == arg_count:
|
|
1516
|
-
exact.append(m)
|
|
1517
|
-
else:
|
|
1518
|
-
name_only.append(m)
|
|
1519
|
-
continue
|
|
1520
|
-
if m.decl.is_constructor:
|
|
1521
|
-
continue
|
|
1522
|
-
if m.decl.name != callee_simple:
|
|
1523
|
-
continue
|
|
1524
|
-
np = len(m.decl.parameters)
|
|
1525
|
-
if arg_count < 0:
|
|
1526
|
-
name_only.append(m)
|
|
1527
|
-
elif np == arg_count:
|
|
1528
|
-
exact.append(m)
|
|
1529
|
-
else:
|
|
1530
|
-
name_only.append(m)
|
|
1531
|
-
|
|
1532
|
-
queue = [type_fqn]
|
|
1533
|
-
while queue:
|
|
1534
|
-
tfqn = queue.pop(0)
|
|
1535
|
-
if tfqn in visited or tfqn not in tables.types:
|
|
1536
|
-
continue
|
|
1537
|
-
visited.add(tfqn)
|
|
1538
|
-
collect_on_type(tfqn)
|
|
1539
|
-
te = tables.types[tfqn]
|
|
1540
|
-
for sup in _direct_supertype_fqns(te, tables):
|
|
1541
|
-
if sup not in visited:
|
|
1542
|
-
queue.append(sup)
|
|
1543
|
-
# Synthetic anonymous classes (`….<anon:byte>`): unqualified instance calls
|
|
1544
|
-
# may target the lexically enclosing type (D3), e.g. `pingFromAnon()` from
|
|
1545
|
-
# `NestedCalls` inside `new Runnable() { void run() { … } }`.
|
|
1546
|
-
if ".<anon:" in tfqn and te.outer_fqn and te.outer_fqn not in visited:
|
|
1547
|
-
queue.append(te.outer_fqn)
|
|
1548
|
-
|
|
1549
|
-
if exact:
|
|
1550
|
-
return exact, False
|
|
1551
|
-
if name_only:
|
|
1552
|
-
return name_only, True
|
|
1553
|
-
return [], False
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
def _static_wildcard_resolve(
|
|
1557
|
-
callee_simple: str,
|
|
1558
|
-
ast: JavaFileAst,
|
|
1559
|
-
tables: GraphTables,
|
|
1560
|
-
current: TypeIndexEntry,
|
|
1561
|
-
) -> str | None:
|
|
1562
|
-
for tw in ast.file_imports.static_wildcards:
|
|
1563
|
-
if tw not in tables.types:
|
|
1564
|
-
continue
|
|
1565
|
-
for m in tables.methods_by_type.get(tw, ()):
|
|
1566
|
-
if m.decl.name != callee_simple or m.decl.is_constructor:
|
|
1567
|
-
continue
|
|
1568
|
-
if "static" not in m.decl.modifiers:
|
|
1569
|
-
continue
|
|
1570
|
-
return tw
|
|
1571
|
-
return None
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
def _unique_type_simple_resolve(simple: str, tables: GraphTables) -> str | None:
|
|
1575
|
-
"""Return the type FQN iff exactly one indexed type uses `simple` as `decl.name`.
|
|
1576
|
-
|
|
1577
|
-
Used only for receiver / static-qualifier disambiguation. Do not use the
|
|
1578
|
-
method index here: an unresolved identifier that equals some method's
|
|
1579
|
-
simple name elsewhere in the project is not evidence about the receiver type.
|
|
1580
|
-
"""
|
|
1581
|
-
hits = tables.by_simple_name.get(simple, [])
|
|
1582
|
-
if len(hits) != 1:
|
|
1583
|
-
return None
|
|
1584
|
-
return hits[0].decl.fqn
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
def _suffix_resolve(receiver_simple: str, tables: GraphTables) -> str | None:
|
|
1588
|
-
matches = [fq for fq in tables.types if fq.endswith("." + receiver_simple)]
|
|
1589
|
-
if len(matches) != 1:
|
|
1590
|
-
return None
|
|
1591
|
-
return matches[0]
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
def _resolve_receiver_type(
|
|
1595
|
-
call: CallSite,
|
|
1596
|
-
*,
|
|
1597
|
-
scope: dict[str, str],
|
|
1598
|
-
member: MemberEntry,
|
|
1599
|
-
ast: JavaFileAst,
|
|
1600
|
-
tables: GraphTables,
|
|
1601
|
-
) -> tuple[str | None, str, float]:
|
|
1602
|
-
"""Returns (receiver_type_fqn_or_none, strategy, confidence)."""
|
|
1603
|
-
expr = call.receiver_expr.strip()
|
|
1604
|
-
callee = call.callee_simple
|
|
1605
|
-
|
|
1606
|
-
effective_static = call.is_static_call
|
|
1607
|
-
if call.is_static_call and expr and not _is_chained_receiver_text(expr):
|
|
1608
|
-
bare_for_static = expr.split("<", 1)[0].strip()
|
|
1609
|
-
if bare_for_static and "." not in bare_for_static and bare_for_static in scope:
|
|
1610
|
-
effective_static = False
|
|
1611
|
-
|
|
1612
|
-
if not expr and not call.is_static_call:
|
|
1613
|
-
if callee in ast.file_imports.static_methods:
|
|
1614
|
-
full = ast.file_imports.static_methods[callee]
|
|
1615
|
-
if "." in full:
|
|
1616
|
-
type_fqn = full.rsplit(".", 1)[0]
|
|
1617
|
-
return type_fqn, "static_import", 0.95
|
|
1618
|
-
sw = _static_wildcard_resolve(callee, ast, tables, tables.types[member.parent_fqn])
|
|
1619
|
-
if sw is not None:
|
|
1620
|
-
return sw, "static_import_wildcard", 0.85
|
|
1621
|
-
|
|
1622
|
-
if effective_static and expr:
|
|
1623
|
-
if _is_chained_receiver_text(expr):
|
|
1624
|
-
return None, "chained_receiver", 0.0
|
|
1625
|
-
entry = tables.types.get(member.parent_fqn)
|
|
1626
|
-
if entry is None:
|
|
1627
|
-
return None, "chained_receiver", 0.0
|
|
1628
|
-
bare_static = expr.split("<", 1)[0].strip()
|
|
1629
|
-
resolved = _resolve_simple(bare_static, current=entry, ast=ast, tables=tables)
|
|
1630
|
-
if resolved is not None:
|
|
1631
|
-
return resolved.decl.fqn, "import_map", 0.95
|
|
1632
|
-
# External type not in the index but FQN is deterministic via an explicit import.
|
|
1633
|
-
# e.g. `import java.util.Objects; Objects.requireNonNull(x)` — we know the FQN
|
|
1634
|
-
# is "java.util.Objects" even though the type isn't indexed; return it so the
|
|
1635
|
-
# edge carries the correct receiver-tier confidence rather than collapsing to phantom.
|
|
1636
|
-
if bare_static in ast.explicit_imports:
|
|
1637
|
-
return ast.explicit_imports[bare_static], "import_map", 0.95
|
|
1638
|
-
uq = _unique_type_simple_resolve(expr, tables)
|
|
1639
|
-
if uq is not None:
|
|
1640
|
-
return uq, "unique_type_name", 0.75
|
|
1641
|
-
sf = _suffix_resolve(expr, tables)
|
|
1642
|
-
if sf is not None:
|
|
1643
|
-
return sf, "suffix", 0.55
|
|
1644
|
-
return None, "phantom", 0.0
|
|
1645
|
-
|
|
1646
|
-
if expr in ("", "this"):
|
|
1647
|
-
return member.parent_fqn, "this_super", 0.95
|
|
1648
|
-
|
|
1649
|
-
if expr == "super":
|
|
1650
|
-
sup = _first_supertype_fqn(tables, member.parent_fqn)
|
|
1651
|
-
if sup is not None:
|
|
1652
|
-
return sup, "this_super", 0.95
|
|
1653
|
-
# No indexed supertype — implicit super to java.lang.Object.
|
|
1654
|
-
# Keep strategy='implicit_super' and confidence=0.90 so this path is
|
|
1655
|
-
# distinguishable from a genuinely unresolvable receiver.
|
|
1656
|
-
return "java.lang.Object", "implicit_super", 0.90
|
|
1657
|
-
|
|
1658
|
-
if _is_chained_receiver_text(expr):
|
|
1659
|
-
return None, "chained_receiver", 0.0
|
|
1660
|
-
|
|
1661
|
-
entry = tables.types.get(member.parent_fqn)
|
|
1662
|
-
if entry is None:
|
|
1663
|
-
return None, "phantom", 0.0
|
|
1664
|
-
|
|
1665
|
-
bare = expr.split("<", 1)[0].strip()
|
|
1666
|
-
if bare in scope:
|
|
1667
|
-
return scope[bare], "import_map", 0.95
|
|
1668
|
-
|
|
1669
|
-
chain = _resolve_this_super_field_chain(expr, member=member, ast=ast, tables=tables)
|
|
1670
|
-
if chain is not None:
|
|
1671
|
-
return chain, "import_map", 0.95
|
|
1672
|
-
|
|
1673
|
-
resolved = _resolve_simple(bare, current=entry, ast=ast, tables=tables)
|
|
1674
|
-
if resolved is not None:
|
|
1675
|
-
return resolved.decl.fqn, "import_map", 0.95
|
|
1676
|
-
|
|
1677
|
-
if entry.package:
|
|
1678
|
-
cand = f"{entry.package}.{bare}"
|
|
1679
|
-
if cand in tables.types:
|
|
1680
|
-
return cand, "same_module", 0.90
|
|
1681
|
-
|
|
1682
|
-
uq = _unique_type_simple_resolve(bare, tables)
|
|
1683
|
-
if uq is not None:
|
|
1684
|
-
return uq, "unique_type_name", 0.75
|
|
1685
|
-
|
|
1686
|
-
sf = _suffix_resolve(bare, tables)
|
|
1687
|
-
if sf is not None:
|
|
1688
|
-
return sf, "suffix", 0.55
|
|
1689
|
-
|
|
1690
|
-
return None, "phantom", 0.0
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
def _phantom_method_id(
|
|
1694
|
-
tables: GraphTables,
|
|
1695
|
-
*,
|
|
1696
|
-
receiver_fqn: str | None,
|
|
1697
|
-
receiver_expr: str,
|
|
1698
|
-
callee: str,
|
|
1699
|
-
arg_count: int,
|
|
1700
|
-
) -> str:
|
|
1701
|
-
# Phantom node identity for a resolved receiver omits call-site arity so
|
|
1702
|
-
# method references (arg_count=-1) and normal invocations share one Symbol
|
|
1703
|
-
# per (receiver_fqn, callee) when the callee is not indexed (D1).
|
|
1704
|
-
if receiver_fqn:
|
|
1705
|
-
fqn = f"{receiver_fqn}#{callee}(?)"
|
|
1706
|
-
sig = f"{callee}(?)"
|
|
1707
|
-
else:
|
|
1708
|
-
expr_short = (receiver_expr[:50] if receiver_expr else "?")
|
|
1709
|
-
arity = "(?)" if arg_count < 0 else f"({arg_count})"
|
|
1710
|
-
fqn = f"?{expr_short}#{callee}{arity}"
|
|
1711
|
-
sig = f"{callee}{arity}"
|
|
1712
|
-
pid = phantom_id(fqn)
|
|
1713
|
-
if pid not in tables.phantoms:
|
|
1714
|
-
tables.phantoms[pid] = {
|
|
1715
|
-
"id": pid,
|
|
1716
|
-
"kind": "method",
|
|
1717
|
-
"name": callee,
|
|
1718
|
-
"fqn": fqn,
|
|
1719
|
-
"package": "",
|
|
1720
|
-
"module": "",
|
|
1721
|
-
"microservice": "",
|
|
1722
|
-
"filename": "",
|
|
1723
|
-
"start_line": 0,
|
|
1724
|
-
"end_line": 0,
|
|
1725
|
-
"start_byte": 0,
|
|
1726
|
-
"end_byte": 0,
|
|
1727
|
-
"modifiers": [],
|
|
1728
|
-
"annotations": [],
|
|
1729
|
-
"capabilities": [],
|
|
1730
|
-
"role": "OTHER",
|
|
1731
|
-
"signature": sig,
|
|
1732
|
-
"parent_id": "",
|
|
1733
|
-
"resolved": False,
|
|
1734
|
-
}
|
|
1735
|
-
return pid
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
def _method_signature_matches_call(member: MemberEntry, call: CallSite) -> bool:
|
|
1739
|
-
if call.arg_count < 0:
|
|
1740
|
-
return True
|
|
1741
|
-
return len(member.decl.parameters) == call.arg_count
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
def _is_strict_supertype_of(tables: GraphTables, super_fqn: str, subtype_fqn: str) -> bool:
|
|
1745
|
-
if super_fqn == subtype_fqn:
|
|
1746
|
-
return False
|
|
1747
|
-
entry = tables.types.get(subtype_fqn)
|
|
1748
|
-
if entry is None:
|
|
1749
|
-
return False
|
|
1750
|
-
visited: set[str] = set()
|
|
1751
|
-
queue = list(_direct_supertype_fqns(entry, tables))
|
|
1752
|
-
while queue:
|
|
1753
|
-
tfqn = queue.pop(0)
|
|
1754
|
-
if tfqn == super_fqn:
|
|
1755
|
-
return True
|
|
1756
|
-
if tfqn in visited or tfqn not in tables.types:
|
|
1757
|
-
continue
|
|
1758
|
-
visited.add(tfqn)
|
|
1759
|
-
queue.extend(_direct_supertype_fqns(tables.types[tfqn], tables))
|
|
1760
|
-
return False
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
def _callee_declaring_role_at_write(
|
|
1764
|
-
tables: GraphTables,
|
|
1765
|
-
dst_id: str,
|
|
1766
|
-
*,
|
|
1767
|
-
member_by_id: dict[str, MemberEntry],
|
|
1768
|
-
) -> str:
|
|
1769
|
-
"""Match parent declaring-type Symbol.role (brownfield + meta_chain included)."""
|
|
1770
|
-
if dst_id in tables.phantoms:
|
|
1771
|
-
return "OTHER"
|
|
1772
|
-
member = member_by_id.get(dst_id)
|
|
1773
|
-
if member is None:
|
|
1774
|
-
return "OTHER"
|
|
1775
|
-
return tables.type_role_by_node_id.get(member.parent_id, "OTHER")
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
def _collapse_supertype_duplicates(
|
|
1779
|
-
candidates: list[MemberEntry],
|
|
1780
|
-
recv_type_fqn: str,
|
|
1781
|
-
call: CallSite,
|
|
1782
|
-
tables: GraphTables,
|
|
1783
|
-
) -> list[MemberEntry]:
|
|
1784
|
-
"""§3.3.1 supertype-walk dedup — collapse interface + concrete duplicate sites."""
|
|
1785
|
-
if len(candidates) <= 1:
|
|
1786
|
-
return candidates
|
|
1787
|
-
concrete_on_receiver = [
|
|
1788
|
-
c for c in candidates
|
|
1789
|
-
if c.parent_fqn == recv_type_fqn and _method_signature_matches_call(c, call)
|
|
1790
|
-
]
|
|
1791
|
-
if len(concrete_on_receiver) != 1:
|
|
1792
|
-
return candidates
|
|
1793
|
-
concrete = concrete_on_receiver[0]
|
|
1794
|
-
supertypes = [
|
|
1795
|
-
c for c in candidates
|
|
1796
|
-
if c is not concrete
|
|
1797
|
-
and _is_strict_supertype_of(tables, c.parent_fqn, recv_type_fqn)
|
|
1798
|
-
and c.decl.signature == concrete.decl.signature
|
|
1799
|
-
]
|
|
1800
|
-
if not supertypes:
|
|
1801
|
-
return candidates
|
|
1802
|
-
allowed_ids = {concrete.node_id, *(c.node_id for c in supertypes)}
|
|
1803
|
-
if any(c.node_id not in allowed_ids for c in candidates):
|
|
1804
|
-
return candidates
|
|
1805
|
-
log.debug(
|
|
1806
|
-
"pass3 supertype dedup %s -> %s",
|
|
1807
|
-
[c.node_id for c in candidates],
|
|
1808
|
-
concrete.node_id,
|
|
1809
|
-
)
|
|
1810
|
-
return [concrete]
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
def _unresolved_call_site_id(caller_id: str, call: CallSite) -> str:
|
|
1814
|
-
return f"ucs:{caller_id}:{call.line}:{call.byte}"
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
def _emit_unresolved_call_site(
|
|
1818
|
-
tables: GraphTables,
|
|
1819
|
-
stats: CallResolutionStats,
|
|
1820
|
-
*,
|
|
1821
|
-
caller_id: str,
|
|
1822
|
-
call: CallSite,
|
|
1823
|
-
reason: str,
|
|
1824
|
-
) -> None:
|
|
1825
|
-
tables.unresolved_call_site_rows.append(UnresolvedCallSiteRow(
|
|
1826
|
-
id=_unresolved_call_site_id(caller_id, call),
|
|
1827
|
-
caller_id=caller_id,
|
|
1828
|
-
call_site_line=call.line,
|
|
1829
|
-
call_site_byte=call.byte,
|
|
1830
|
-
arg_count=call.arg_count,
|
|
1831
|
-
callee_simple=call.callee_simple,
|
|
1832
|
-
receiver_expr=call.receiver_expr or "",
|
|
1833
|
-
reason=reason,
|
|
1834
|
-
))
|
|
1835
|
-
if reason == "chained_receiver":
|
|
1836
|
-
stats.phantom_chained += 1
|
|
1837
|
-
else:
|
|
1838
|
-
stats.phantom_other += 1
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
def _emit_call_edge(
|
|
1842
|
-
tables: GraphTables,
|
|
1843
|
-
stats: CallResolutionStats,
|
|
1844
|
-
*,
|
|
1845
|
-
src_id: str,
|
|
1846
|
-
dst_id: str,
|
|
1847
|
-
call: CallSite,
|
|
1848
|
-
confidence: float,
|
|
1849
|
-
strategy: str,
|
|
1850
|
-
resolved: bool,
|
|
1851
|
-
edge_arg_count: int | None = None,
|
|
1852
|
-
) -> None:
|
|
1853
|
-
arity = call.arg_count if edge_arg_count is None else edge_arg_count
|
|
1854
|
-
tables.calls_rows.append(CallsRow(
|
|
1855
|
-
src_id=src_id,
|
|
1856
|
-
dst_id=dst_id,
|
|
1857
|
-
call_site_line=call.line,
|
|
1858
|
-
call_site_byte=call.byte,
|
|
1859
|
-
arg_count=arity,
|
|
1860
|
-
confidence=confidence,
|
|
1861
|
-
strategy=strategy,
|
|
1862
|
-
source="static",
|
|
1863
|
-
resolved=resolved,
|
|
1864
|
-
))
|
|
1865
|
-
stats.total += 1
|
|
1866
|
-
stats.by_strategy[strategy] += 1
|
|
1867
|
-
if not resolved:
|
|
1868
|
-
stats.callee_unresolved += 1
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
def _resolve_and_emit_call(
|
|
1872
|
-
call: CallSite,
|
|
1873
|
-
member: MemberEntry,
|
|
1874
|
-
ast: JavaFileAst,
|
|
1875
|
-
tables: GraphTables,
|
|
1876
|
-
stats: CallResolutionStats,
|
|
1877
|
-
*,
|
|
1878
|
-
scope: dict[str, str],
|
|
1879
|
-
) -> None:
|
|
1880
|
-
"""Emit CALLS rows for one call site.
|
|
1881
|
-
|
|
1882
|
-
Candidate selection uses ``_lookup_method_candidates`` (exact arity first, then
|
|
1883
|
-
name-only fallback on the type + supertype walk).
|
|
1884
|
-
|
|
1885
|
-
When ``used_name_only_fallback`` is true and exactly one name-only candidate
|
|
1886
|
-
exists, the edge ``strategy`` reuses the receiver-resolution tier (``strat``)
|
|
1887
|
-
rather than ``overload_ambiguous``: arity at the call site did not match any
|
|
1888
|
-
overload, but only one method of that name exists — the callee is unambiguous.
|
|
1889
|
-
"""
|
|
1890
|
-
recv_type, strat, conf = _resolve_receiver_type(call, scope=scope, member=member, ast=ast, tables=tables)
|
|
1891
|
-
|
|
1892
|
-
if strat == "chained_receiver":
|
|
1893
|
-
_emit_unresolved_call_site(
|
|
1894
|
-
tables, stats, caller_id=member.node_id, call=call, reason="chained_receiver",
|
|
1895
|
-
)
|
|
1896
|
-
return
|
|
1897
|
-
|
|
1898
|
-
if recv_type is None:
|
|
1899
|
-
_emit_unresolved_call_site(
|
|
1900
|
-
tables, stats,
|
|
1901
|
-
caller_id=member.node_id,
|
|
1902
|
-
call=call,
|
|
1903
|
-
reason="phantom_unresolved_receiver",
|
|
1904
|
-
)
|
|
1905
|
-
return
|
|
1906
|
-
|
|
1907
|
-
candidates, name_only_fb = _lookup_method_candidates(
|
|
1908
|
-
recv_type, call.callee_simple, call.arg_count, tables, ast,
|
|
1909
|
-
)
|
|
1910
|
-
|
|
1911
|
-
# Guard relies on `_lookup_method_candidates` returning a same-ms candidate when one exists; revisit if pass3 scopes lookups per-microservice.
|
|
1912
|
-
if member.microservice:
|
|
1913
|
-
same_ms = [c for c in candidates if c.microservice == member.microservice]
|
|
1914
|
-
if same_ms and len(same_ms) != len(candidates):
|
|
1915
|
-
for c in candidates:
|
|
1916
|
-
if c.microservice and c.microservice != member.microservice:
|
|
1917
|
-
log.warning(
|
|
1918
|
-
"skipping cross-microservice CALLS edge %s -> %s "
|
|
1919
|
-
"(caller=%s, callee=%s)",
|
|
1920
|
-
f"{member.parent_fqn}#{member.decl.signature}",
|
|
1921
|
-
f"{c.parent_fqn}#{c.decl.signature}",
|
|
1922
|
-
member.microservice, c.microservice,
|
|
1923
|
-
)
|
|
1924
|
-
stats.skipped_cross_service += 1
|
|
1925
|
-
candidates = same_ms
|
|
1926
|
-
|
|
1927
|
-
# Compute the call-shape strategy / confidence override BEFORE the
|
|
1928
|
-
# empty-candidates check so they are preserved even when the callee cannot
|
|
1929
|
-
# be located on the resolved receiver type (B3 fix).
|
|
1930
|
-
edge_conf = conf
|
|
1931
|
-
if call.arg_count < 0:
|
|
1932
|
-
edge_strat = "method_reference"
|
|
1933
|
-
elif call.callee_simple == "<init>" and call.receiver_expr == "super" and (
|
|
1934
|
-
call.byte == member.decl.start_byte and call.line == member.decl.start_line
|
|
1935
|
-
):
|
|
1936
|
-
# Synthesized implicit-super site from _parse_method.
|
|
1937
|
-
edge_strat = "implicit_super"
|
|
1938
|
-
edge_conf = 0.90
|
|
1939
|
-
elif call.callee_simple == "<init>":
|
|
1940
|
-
# new Foo(…), this(…), super(…) — confidence inherited from receiver tier.
|
|
1941
|
-
edge_strat = "constructor"
|
|
1942
|
-
elif name_only_fb and len(candidates) > 1:
|
|
1943
|
-
edge_strat = "overload_ambiguous"
|
|
1944
|
-
elif name_only_fb and len(candidates) == 1:
|
|
1945
|
-
# Name-only fallback with a single candidate — not ambiguous.
|
|
1946
|
-
edge_strat = strat
|
|
1947
|
-
else:
|
|
1948
|
-
edge_strat = strat
|
|
1949
|
-
|
|
1950
|
-
if not candidates:
|
|
1951
|
-
# Receiver was resolved but the callee method isn't indexed on that type
|
|
1952
|
-
# (e.g. JDK / Spring / external library). Preserve the receiver-tier
|
|
1953
|
-
# strategy and confidence — only resolved=False signals the phantom callee
|
|
1954
|
-
# (B3 fix: do NOT downgrade to confidence=0.0 / strategy='phantom' here).
|
|
1955
|
-
pid = _phantom_method_id(
|
|
1956
|
-
tables, receiver_fqn=recv_type, receiver_expr=call.receiver_expr,
|
|
1957
|
-
callee=call.callee_simple, arg_count=call.arg_count,
|
|
1958
|
-
)
|
|
1959
|
-
_emit_call_edge(
|
|
1960
|
-
tables, stats, src_id=member.node_id, dst_id=pid, call=call,
|
|
1961
|
-
confidence=edge_conf, strategy=edge_strat, resolved=False,
|
|
1962
|
-
)
|
|
1963
|
-
return
|
|
1964
|
-
|
|
1965
|
-
if len(candidates) > 1 and edge_strat != "overload_ambiguous":
|
|
1966
|
-
candidates = _collapse_supertype_duplicates(candidates, recv_type, call, tables)
|
|
1967
|
-
|
|
1968
|
-
if len(candidates) == 1:
|
|
1969
|
-
candidate = candidates[0]
|
|
1970
|
-
ref_arity: int | None = None
|
|
1971
|
-
if call.arg_count < 0:
|
|
1972
|
-
ref_arity = len(candidate.decl.parameters)
|
|
1973
|
-
_emit_call_edge(
|
|
1974
|
-
tables, stats, src_id=member.node_id, dst_id=candidate.node_id, call=call,
|
|
1975
|
-
confidence=edge_conf, strategy=edge_strat, resolved=True,
|
|
1976
|
-
edge_arg_count=ref_arity,
|
|
1977
|
-
)
|
|
1978
|
-
return
|
|
1979
|
-
|
|
1980
|
-
for c in candidates:
|
|
1981
|
-
ref_arity_multi: int | None = len(c.decl.parameters) if call.arg_count < 0 else None
|
|
1982
|
-
_emit_call_edge(
|
|
1983
|
-
tables, stats, src_id=member.node_id, dst_id=c.node_id, call=call,
|
|
1984
|
-
confidence=edge_conf, strategy="overload_ambiguous", resolved=True,
|
|
1985
|
-
edge_arg_count=ref_arity_multi,
|
|
1986
|
-
)
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
def _resolve_method_calls(
|
|
1990
|
-
member: MemberEntry,
|
|
1991
|
-
ast: JavaFileAst,
|
|
1992
|
-
tables: GraphTables,
|
|
1993
|
-
stats: CallResolutionStats,
|
|
1994
|
-
) -> None:
|
|
1995
|
-
scope = _scope_table(member, ast, tables)
|
|
1996
|
-
for call in member.decl.call_sites:
|
|
1997
|
-
try:
|
|
1998
|
-
_resolve_and_emit_call(call, member, ast, tables, stats, scope=scope)
|
|
1999
|
-
except Exception as e:
|
|
2000
|
-
log.warning("call resolution failed for %s: %s", member.decl.signature, e)
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
def _process_file_calls(
|
|
2004
|
-
file_ast: JavaFileAst,
|
|
2005
|
-
file_path: str,
|
|
2006
|
-
tables: GraphTables,
|
|
2007
|
-
stats: CallResolutionStats,
|
|
2008
|
-
) -> None:
|
|
2009
|
-
for member in tables.members:
|
|
2010
|
-
if member.file_path != file_path:
|
|
2011
|
-
continue
|
|
2012
|
-
try:
|
|
2013
|
-
_resolve_method_calls(member, file_ast, tables, stats)
|
|
2014
|
-
except Exception as e:
|
|
2015
|
-
log.warning("Failed to extract calls from %s#%s: %s", member.parent_fqn, member.decl.signature, e)
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
def pass3_calls(tables: GraphTables, asts: dict[str, JavaFileAst], *, verbose: bool) -> None:
|
|
2019
|
-
if verbose:
|
|
2020
|
-
_verbose_stderr_line(_PASS3_START)
|
|
2021
|
-
_build_member_indexes(tables)
|
|
2022
|
-
stats = CallResolutionStats()
|
|
2023
|
-
with _graph_pass_progress("3/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 3", verbose=verbose):
|
|
2024
|
-
for rel_path, file_ast in asts.items():
|
|
2025
|
-
try:
|
|
2026
|
-
_process_file_calls(file_ast, rel_path, tables, stats)
|
|
2027
|
-
except Exception as e:
|
|
2028
|
-
log.error("Call extraction failed for %s: %s", rel_path, e)
|
|
2029
|
-
denom_calls = max(1, stats.total)
|
|
2030
|
-
denom_sites = max(1, stats.total + stats.phantom_chained + stats.phantom_other)
|
|
2031
|
-
pct_chained = 100.0 * stats.phantom_chained / denom_sites
|
|
2032
|
-
pct_callee_unres = 100.0 * stats.callee_unresolved / denom_calls
|
|
2033
|
-
pct_phantom_recv = 100.0 * stats.phantom_other / denom_sites
|
|
2034
|
-
tables.pass3_skipped_cross_service = int(stats.skipped_cross_service)
|
|
2035
|
-
tables.pass3_unresolved_phantom_receiver = int(stats.phantom_other)
|
|
2036
|
-
tables.pass3_unresolved_chained = int(stats.phantom_chained)
|
|
2037
|
-
msg = (
|
|
2038
|
-
f"Call resolution: {stats.total} CALLS rows, {stats.phantom_chained} chained unresolved "
|
|
2039
|
-
f"({pct_chained:.1f}%), {stats.callee_unresolved} unresolved callee on CALLS "
|
|
2040
|
-
f"({pct_callee_unres:.1f}%), {stats.phantom_other} phantom-receiver unresolved "
|
|
2041
|
-
f"({pct_phantom_recv:.1f}%), {stats.skipped_cross_service} skipped cross-service, "
|
|
2042
|
-
f"strategies: {dict(stats.by_strategy)}"
|
|
2043
|
-
)
|
|
2044
|
-
log.info(msg)
|
|
2045
|
-
if verbose:
|
|
2046
|
-
_verbose_stderr_line(f"[graph] pass 3 · {msg}")
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
_PATH_VAR_SEG = re.compile(r"^\{([^:{}]+)(?::([^}]*))?\}$") # whole path segment
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
def _normalize_path(raw_path: str) -> tuple[str, str]:
|
|
2053
|
-
"""Return `(path_template, path_regex)` for a servlet-style path pattern.
|
|
2054
|
-
|
|
2055
|
-
`/api/users/{id}` → ``("/api/users/{}", "^/api/users/[^/]+/?$")``.
|
|
2056
|
-
`{id:\\d+}` constraints strip to ``{}`` in the template while preserving the
|
|
2057
|
-
regex constraint for that segment. Deterministic for shared use by B2b/B6.
|
|
2058
|
-
"""
|
|
2059
|
-
raw_path = (raw_path or "").strip()
|
|
2060
|
-
if not raw_path:
|
|
2061
|
-
return "", ""
|
|
2062
|
-
p = raw_path if raw_path.startswith("/") else "/" + raw_path
|
|
2063
|
-
trimmed = p.rstrip("/")
|
|
2064
|
-
if trimmed == "":
|
|
2065
|
-
return "/", "^/?$"
|
|
2066
|
-
segments = [s for s in trimmed.split("/") if s != ""]
|
|
2067
|
-
tmpl_parts: list[str] = []
|
|
2068
|
-
re_parts: list[str] = []
|
|
2069
|
-
for seg in segments:
|
|
2070
|
-
m = _PATH_VAR_SEG.fullmatch(seg)
|
|
2071
|
-
if m:
|
|
2072
|
-
tmpl_parts.append("{}")
|
|
2073
|
-
constraint = m.group(2)
|
|
2074
|
-
re_parts.append(constraint if constraint else "[^/]+")
|
|
2075
|
-
else:
|
|
2076
|
-
tmpl_parts.append(seg)
|
|
2077
|
-
re_parts.append(re.escape(seg))
|
|
2078
|
-
tmpl = "/" + "/".join(tmpl_parts)
|
|
2079
|
-
body = "/".join(re_parts)
|
|
2080
|
-
if not body.startswith("/"):
|
|
2081
|
-
body = "/" + body
|
|
2082
|
-
return tmpl, f"^{body}/?$"
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
def _route_id(
|
|
2086
|
-
framework: str,
|
|
2087
|
-
kind: str,
|
|
2088
|
-
http_method: str,
|
|
2089
|
-
path_template: str,
|
|
2090
|
-
path_raw: str,
|
|
2091
|
-
topic: str,
|
|
2092
|
-
broker: str,
|
|
2093
|
-
microservice: str,
|
|
2094
|
-
) -> str:
|
|
2095
|
-
"""Stable id; `path_raw` disambiguates HTTP routes when `path_template` is empty (SpEL / const)."""
|
|
2096
|
-
path_key = path_template if path_template else path_raw
|
|
2097
|
-
key = (
|
|
2098
|
-
f"{framework}|{kind}|{http_method}|{path_key}|"
|
|
2099
|
-
f"{topic}|{broker}|{microservice}"
|
|
2100
|
-
)
|
|
2101
|
-
return f"r:{hashlib.sha1(key.encode()).hexdigest()[:16]}"
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
def _client_id(
|
|
2105
|
-
*,
|
|
2106
|
-
microservice: str,
|
|
2107
|
-
member_fqn: str,
|
|
2108
|
-
client_kind: str,
|
|
2109
|
-
path: str,
|
|
2110
|
-
method: str,
|
|
2111
|
-
) -> str:
|
|
2112
|
-
key = f"{microservice}|{member_fqn}|{client_kind}|{path}|{method}"
|
|
2113
|
-
return f"c:{hashlib.sha1(key.encode()).hexdigest()[:16]}"
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
def _producer_id(
|
|
2117
|
-
*,
|
|
2118
|
-
microservice: str,
|
|
2119
|
-
member_fqn: str,
|
|
2120
|
-
producer_kind: str,
|
|
2121
|
-
topic: str,
|
|
2122
|
-
) -> str:
|
|
2123
|
-
# Topic-level identity per method+kind; broker is intentionally omitted so the same
|
|
2124
|
-
# resolved topic on one method shares one Producer node across call sites.
|
|
2125
|
-
key = f"{microservice}|{member_fqn}|{producer_kind}|{topic}"
|
|
2126
|
-
return f"p:{hashlib.sha1(key.encode()).hexdigest()[:16]}"
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
# The four brownfield source layers — single source of truth. Consumed by the
|
|
2130
|
-
# client/producer source-layer classifiers, the *_from_brownfield_pct stats
|
|
2131
|
-
# (via brownfield_strategies), and the brownfield_only authoritativeness gate in
|
|
2132
|
-
# _is_brownfield_sourced. codebase_client/codebase_producer are caller-side
|
|
2133
|
-
# declaration strategies, not layers — they extend brownfield_strategies only.
|
|
2134
|
-
_BROWNFIELD_LAYERS = frozenset({
|
|
2135
|
-
"layer_a_meta",
|
|
2136
|
-
"layer_b_ann",
|
|
2137
|
-
"layer_b_fqn",
|
|
2138
|
-
"layer_c_source",
|
|
2139
|
-
})
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
def _client_source_layer(strategy: str) -> str:
|
|
2143
|
-
if strategy in _BROWNFIELD_LAYERS:
|
|
2144
|
-
return strategy
|
|
2145
|
-
# Some caller extraction paths emit client kind as strategy; treat those
|
|
2146
|
-
# as builtin-source declarations instead of warning on every row.
|
|
2147
|
-
if strategy in VALID_CLIENT_KINDS:
|
|
2148
|
-
return "builtin"
|
|
2149
|
-
if strategy != "builtin":
|
|
2150
|
-
log.warning("unknown client source strategy %r, falling back to builtin", strategy)
|
|
2151
|
-
return "builtin"
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
def _producer_source_layer(strategy: str) -> str:
|
|
2155
|
-
if strategy in _BROWNFIELD_LAYERS:
|
|
2156
|
-
return strategy
|
|
2157
|
-
if strategy in VALID_PRODUCER_KINDS:
|
|
2158
|
-
return "builtin"
|
|
2159
|
-
if strategy != "builtin":
|
|
2160
|
-
log.warning("unknown producer source strategy %r, falling back to builtin", strategy)
|
|
2161
|
-
return "builtin"
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
_ROUTE_LAYER_RANK: dict[str, int] = {
|
|
2165
|
-
"builtin": 0,
|
|
2166
|
-
"layer_b_ann": 1,
|
|
2167
|
-
"layer_a_meta": 2,
|
|
2168
|
-
"layer_c_source": 3,
|
|
2169
|
-
"layer_b_fqn": 4,
|
|
2170
|
-
}
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
def pass4_routes(
|
|
2174
|
-
tables: GraphTables,
|
|
2175
|
-
asts: dict[str, JavaFileAst],
|
|
2176
|
-
*,
|
|
2177
|
-
source_root: Path,
|
|
2178
|
-
verbose: bool,
|
|
2179
|
-
) -> None:
|
|
2180
|
-
stats = tables.route_stats
|
|
2181
|
-
overrides = load_brownfield_overrides(source_root)
|
|
2182
|
-
try:
|
|
2183
|
-
prs = str(source_root.resolve())
|
|
2184
|
-
except OSError:
|
|
2185
|
-
prs = str(source_root)
|
|
2186
|
-
tables.cross_service_resolution = _load_config_cross_service_resolution(prs)
|
|
2187
|
-
meta_chain = collect_annotation_meta_chain(prs)
|
|
2188
|
-
if verbose:
|
|
2189
|
-
_verbose_stderr_line(_PASS4_START)
|
|
2190
|
-
with _graph_pass_progress("4/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 4", verbose=verbose):
|
|
2191
|
-
|
|
2192
|
-
for ast in asts.values():
|
|
2193
|
-
stats.routes_skipped_unresolved += ast.routes_skipped_unresolved
|
|
2194
|
-
|
|
2195
|
-
routes_by_id: dict[str, RouteRow] = {}
|
|
2196
|
-
exposes_seen: set[tuple[str, str]] = set()
|
|
2197
|
-
|
|
2198
|
-
http_kinds = frozenset({"http_endpoint", "http_consumer"})
|
|
2199
|
-
|
|
2200
|
-
for member in sorted(tables.members, key=lambda m: m.node_id):
|
|
2201
|
-
if member.decl.is_constructor:
|
|
2202
|
-
continue
|
|
2203
|
-
ast = asts.get(member.file_path)
|
|
2204
|
-
if ast is None:
|
|
2205
|
-
continue
|
|
2206
|
-
type_decl = tables.types[member.parent_fqn].decl
|
|
2207
|
-
final_routes = resolve_routes_for_method(
|
|
2208
|
-
method_decl=member.decl,
|
|
2209
|
-
enclosing_type=type_decl,
|
|
2210
|
-
overrides=overrides,
|
|
2211
|
-
meta_chain=meta_chain,
|
|
2212
|
-
builtin_routes=member.decl.routes,
|
|
2213
|
-
)
|
|
2214
|
-
if not final_routes:
|
|
2215
|
-
continue
|
|
2216
|
-
for decl in final_routes:
|
|
2217
|
-
path_template, path_regex = ("", "")
|
|
2218
|
-
if decl.kind in http_kinds:
|
|
2219
|
-
if decl.resolved and decl.resolution_strategy in (
|
|
2220
|
-
"annotation",
|
|
2221
|
-
"codebase_route",
|
|
2222
|
-
):
|
|
2223
|
-
path_template, path_regex = _normalize_path(decl.path)
|
|
2224
|
-
else:
|
|
2225
|
-
path_template, path_regex = "", ""
|
|
2226
|
-
rid = _route_id(
|
|
2227
|
-
decl.framework,
|
|
2228
|
-
decl.kind,
|
|
2229
|
-
decl.http_method,
|
|
2230
|
-
path_template,
|
|
2231
|
-
decl.path,
|
|
2232
|
-
decl.topic,
|
|
2233
|
-
decl.broker,
|
|
2234
|
-
member.microservice,
|
|
2235
|
-
)
|
|
2236
|
-
layer = decl.route_source_layer
|
|
2237
|
-
if rid not in routes_by_id:
|
|
2238
|
-
routes_by_id[rid] = RouteRow(
|
|
2239
|
-
id=rid,
|
|
2240
|
-
kind=decl.kind,
|
|
2241
|
-
framework=decl.framework,
|
|
2242
|
-
method=decl.http_method,
|
|
2243
|
-
path=decl.path,
|
|
2244
|
-
path_template=path_template,
|
|
2245
|
-
path_regex=path_regex,
|
|
2246
|
-
topic=decl.topic,
|
|
2247
|
-
broker=decl.broker,
|
|
2248
|
-
feign_name=decl.feign_name,
|
|
2249
|
-
feign_url=decl.feign_url,
|
|
2250
|
-
microservice=member.microservice,
|
|
2251
|
-
module=member.module,
|
|
2252
|
-
filename=decl.filename,
|
|
2253
|
-
start_line=decl.start_line,
|
|
2254
|
-
end_line=decl.end_line,
|
|
2255
|
-
resolved=decl.resolved,
|
|
2256
|
-
source_layer=layer,
|
|
2257
|
-
)
|
|
2258
|
-
else:
|
|
2259
|
-
prev = routes_by_id[rid]
|
|
2260
|
-
if _ROUTE_LAYER_RANK.get(layer, 0) > _ROUTE_LAYER_RANK.get(
|
|
2261
|
-
prev.source_layer,
|
|
2262
|
-
0,
|
|
2263
|
-
):
|
|
2264
|
-
routes_by_id[rid] = replace(prev, source_layer=layer)
|
|
2265
|
-
ek = (member.node_id, rid)
|
|
2266
|
-
if ek not in exposes_seen:
|
|
2267
|
-
route_kind = routes_by_id[rid].kind
|
|
2268
|
-
if route_kind == "http_consumer":
|
|
2269
|
-
stats.exposes_suppressed_feign += 1
|
|
2270
|
-
continue
|
|
2271
|
-
exposes_seen.add(ek)
|
|
2272
|
-
tables.exposes_rows.append(
|
|
2273
|
-
ExposesRow(
|
|
2274
|
-
symbol_id=member.node_id,
|
|
2275
|
-
route_id=rid,
|
|
2276
|
-
confidence=decl.confidence,
|
|
2277
|
-
strategy=decl.resolution_strategy,
|
|
2278
|
-
),
|
|
2279
|
-
)
|
|
2280
|
-
|
|
2281
|
-
tables.routes_rows = sorted(routes_by_id.values(), key=lambda r: r.id)
|
|
2282
|
-
|
|
2283
|
-
for row in tables.routes_rows:
|
|
2284
|
-
stats.by_framework[row.framework] += 1
|
|
2285
|
-
stats.by_kind[row.kind] += 1
|
|
2286
|
-
|
|
2287
|
-
n_routes = len(tables.routes_rows)
|
|
2288
|
-
if n_routes:
|
|
2289
|
-
stats.routes_resolved_pct = 100.0 * sum(
|
|
2290
|
-
1 for r in tables.routes_rows if r.resolved
|
|
2291
|
-
) / n_routes
|
|
2292
|
-
stats.routes_from_brownfield_pct = 100.0 * sum(
|
|
2293
|
-
1 for r in tables.routes_rows if r.source_layer != "builtin"
|
|
2294
|
-
) / n_routes
|
|
2295
|
-
else:
|
|
2296
|
-
stats.routes_resolved_pct = 100.0
|
|
2297
|
-
stats.routes_from_brownfield_pct = 0.0
|
|
2298
|
-
|
|
2299
|
-
by_layer: dict[str, int] = defaultdict(int)
|
|
2300
|
-
for row in tables.routes_rows:
|
|
2301
|
-
by_layer[row.source_layer] += 1
|
|
2302
|
-
stats.routes_by_layer = dict(sorted(by_layer.items()))
|
|
2303
|
-
|
|
2304
|
-
msg = (
|
|
2305
|
-
f"Route extraction: emitted={n_routes}, exposes={len(tables.exposes_rows)}, "
|
|
2306
|
-
f"exposes_suppressed_feign={stats.exposes_suppressed_feign}, "
|
|
2307
|
-
f"skipped_unresolved={stats.routes_skipped_unresolved}, "
|
|
2308
|
-
f"routes_resolved_pct={stats.routes_resolved_pct:.1f}, "
|
|
2309
|
-
f"routes_from_brownfield_pct={stats.routes_from_brownfield_pct:.1f}, "
|
|
2310
|
-
f"by_framework={dict(stats.by_framework)}"
|
|
2311
|
-
)
|
|
2312
|
-
log.info(msg)
|
|
2313
|
-
if verbose:
|
|
2314
|
-
_verbose_stderr_line(f"[graph] pass 4 · {msg}")
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
def pass5_imperative_edges(
|
|
2318
|
-
tables: GraphTables,
|
|
2319
|
-
asts: dict[str, JavaFileAst],
|
|
2320
|
-
*,
|
|
2321
|
-
source_root: Path,
|
|
2322
|
-
verbose: bool,
|
|
2323
|
-
) -> None:
|
|
2324
|
-
del asts
|
|
2325
|
-
overrides = load_brownfield_overrides(source_root)
|
|
2326
|
-
try:
|
|
2327
|
-
prs = str(source_root.resolve())
|
|
2328
|
-
except OSError:
|
|
2329
|
-
prs = str(source_root)
|
|
2330
|
-
tables.cross_service_resolution = _load_config_cross_service_resolution(prs)
|
|
2331
|
-
meta_chain = collect_annotation_meta_chain(prs)
|
|
2332
|
-
routes_by_id = {r.id: r for r in tables.routes_rows}
|
|
2333
|
-
existing_route_ids = set(routes_by_id)
|
|
2334
|
-
http_seen: set[tuple[str, str]] = set()
|
|
2335
|
-
async_seen: set[tuple[str, str]] = set()
|
|
2336
|
-
client_seen: set[str] = set()
|
|
2337
|
-
producer_seen: set[str] = set()
|
|
2338
|
-
declares_client_seen: set[tuple[str, str]] = set()
|
|
2339
|
-
declares_producer_seen: set[tuple[str, str]] = set()
|
|
2340
|
-
route_rows = list(tables.routes_rows)
|
|
2341
|
-
|
|
2342
|
-
def _micro_factor(member: MemberEntry) -> float:
|
|
2343
|
-
ms = microservice_for_path(member.file_path, source_root)
|
|
2344
|
-
return 1.0 if ms else 0.85
|
|
2345
|
-
|
|
2346
|
-
def _append_route(row: RouteRow) -> None:
|
|
2347
|
-
if row.id in existing_route_ids:
|
|
2348
|
-
return
|
|
2349
|
-
existing_route_ids.add(row.id)
|
|
2350
|
-
routes_by_id[row.id] = row
|
|
2351
|
-
route_rows.append(row)
|
|
2352
|
-
|
|
2353
|
-
def _phantom_http_route_id(call: OutgoingCallDecl) -> str:
|
|
2354
|
-
if call.path_template_call and call.method_call:
|
|
2355
|
-
return _route_id("", "http_endpoint", call.method_call, call.path_template_call, call.path_template_call, "", "", "")
|
|
2356
|
-
uniq = hashlib.sha1(f"{call.filename}:{call.start_line}:{call.raw_uri}".encode()).hexdigest()[:12]
|
|
2357
|
-
return f"r:phantom:{uniq}"
|
|
2358
|
-
|
|
2359
|
-
def _phantom_async_route_id(call: OutgoingCallDecl) -> str:
|
|
2360
|
-
if call.topic_call:
|
|
2361
|
-
return _route_id("", "kafka_topic", "", "", "", call.topic_call, call.broker_call, "")
|
|
2362
|
-
uniq = hashlib.sha1(f"{call.filename}:{call.start_line}:{call.raw_topic}".encode()).hexdigest()[:12]
|
|
2363
|
-
return f"r:phantom:{uniq}"
|
|
2364
|
-
|
|
2365
|
-
if verbose:
|
|
2366
|
-
_verbose_stderr_line(_PASS5_START)
|
|
2367
|
-
with _graph_pass_progress("5/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 5", verbose=verbose):
|
|
2368
|
-
for member in sorted(tables.members, key=lambda x: x.node_id):
|
|
2369
|
-
if member.decl.is_constructor:
|
|
2370
|
-
continue
|
|
2371
|
-
type_decl = tables.types[member.parent_fqn].decl
|
|
2372
|
-
final_http_calls = resolve_http_client_for_method(
|
|
2373
|
-
method_decl=member.decl,
|
|
2374
|
-
enclosing_type=type_decl,
|
|
2375
|
-
overrides=overrides,
|
|
2376
|
-
meta_chain=meta_chain,
|
|
2377
|
-
builtin_calls=member.decl.outgoing_calls,
|
|
2378
|
-
)
|
|
2379
|
-
final_async_calls = resolve_async_producer_for_method(
|
|
2380
|
-
method_decl=member.decl,
|
|
2381
|
-
enclosing_type=type_decl,
|
|
2382
|
-
overrides=overrides,
|
|
2383
|
-
meta_chain=meta_chain,
|
|
2384
|
-
builtin_calls=member.decl.outgoing_calls,
|
|
2385
|
-
)
|
|
2386
|
-
micro_factor = _micro_factor(member)
|
|
2387
|
-
for call in final_http_calls + final_async_calls:
|
|
2388
|
-
if call.channel == "http":
|
|
2389
|
-
client_path = (call.path_template_call or "").strip()
|
|
2390
|
-
client_method = (call.method_call or "").strip().upper()
|
|
2391
|
-
# Keep normalized path fields on Client now so LC3 filter semantics
|
|
2392
|
-
# (`path_prefix`) can use persisted columns without extra transforms.
|
|
2393
|
-
client_path_template = ""
|
|
2394
|
-
client_path_regex = ""
|
|
2395
|
-
if client_path:
|
|
2396
|
-
client_path_template, client_path_regex = _normalize_path(client_path)
|
|
2397
|
-
cid = _client_id(
|
|
2398
|
-
microservice=member.microservice,
|
|
2399
|
-
member_fqn=call.method_fqn,
|
|
2400
|
-
client_kind=call.client_kind,
|
|
2401
|
-
path=client_path,
|
|
2402
|
-
method=client_method,
|
|
2403
|
-
)
|
|
2404
|
-
if cid not in client_seen:
|
|
2405
|
-
client_seen.add(cid)
|
|
2406
|
-
tables.client_rows.append(
|
|
2407
|
-
ClientRow(
|
|
2408
|
-
id=cid,
|
|
2409
|
-
client_kind=call.client_kind,
|
|
2410
|
-
target_service=call.feign_target_name,
|
|
2411
|
-
path=client_path,
|
|
2412
|
-
path_template=client_path_template,
|
|
2413
|
-
path_regex=client_path_regex,
|
|
2414
|
-
method=client_method,
|
|
2415
|
-
member_fqn=call.method_fqn,
|
|
2416
|
-
member_id=member.node_id,
|
|
2417
|
-
microservice=member.microservice,
|
|
2418
|
-
module=member.module,
|
|
2419
|
-
filename=call.filename,
|
|
2420
|
-
start_line=call.start_line,
|
|
2421
|
-
end_line=call.end_line,
|
|
2422
|
-
resolved=call.resolved,
|
|
2423
|
-
source_layer=_client_source_layer(call.resolution_strategy),
|
|
2424
|
-
),
|
|
2425
|
-
)
|
|
2426
|
-
dkey = (member.node_id, cid)
|
|
2427
|
-
if dkey not in declares_client_seen:
|
|
2428
|
-
declares_client_seen.add(dkey)
|
|
2429
|
-
tables.declares_client_rows.append(
|
|
2430
|
-
DeclaresClientRow(
|
|
2431
|
-
symbol_id=member.node_id,
|
|
2432
|
-
client_id=cid,
|
|
2433
|
-
confidence=call.confidence_base,
|
|
2434
|
-
strategy=call.resolution_strategy,
|
|
2435
|
-
),
|
|
2436
|
-
)
|
|
2437
|
-
rid = ""
|
|
2438
|
-
strategy = call.resolution_strategy
|
|
2439
|
-
if call.client_kind == CLIENT_KIND_FEIGN_METHOD:
|
|
2440
|
-
exposing = next((e for e in tables.exposes_rows if e.symbol_id == member.node_id), None)
|
|
2441
|
-
if exposing is not None:
|
|
2442
|
-
rid = exposing.route_id
|
|
2443
|
-
if not rid:
|
|
2444
|
-
rid = _phantom_http_route_id(call)
|
|
2445
|
-
_append_route(
|
|
2446
|
-
RouteRow(
|
|
2447
|
-
id=rid,
|
|
2448
|
-
kind="http_endpoint",
|
|
2449
|
-
framework="",
|
|
2450
|
-
method=call.method_call,
|
|
2451
|
-
path=call.path_template_call,
|
|
2452
|
-
path_template=call.path_template_call,
|
|
2453
|
-
path_regex="",
|
|
2454
|
-
topic="",
|
|
2455
|
-
broker="",
|
|
2456
|
-
feign_name=call.feign_target_name,
|
|
2457
|
-
feign_url=call.feign_target_url,
|
|
2458
|
-
microservice="",
|
|
2459
|
-
module="",
|
|
2460
|
-
filename=call.filename,
|
|
2461
|
-
start_line=call.start_line,
|
|
2462
|
-
end_line=call.end_line,
|
|
2463
|
-
resolved=False,
|
|
2464
|
-
source_layer="builtin",
|
|
2465
|
-
)
|
|
2466
|
-
)
|
|
2467
|
-
key = (cid, rid)
|
|
2468
|
-
if key in http_seen:
|
|
2469
|
-
continue
|
|
2470
|
-
http_seen.add(key)
|
|
2471
|
-
conf = call.confidence_base * 0.3 * micro_factor
|
|
2472
|
-
tables.http_call_rows.append(
|
|
2473
|
-
HttpCallRow(
|
|
2474
|
-
client_id=cid,
|
|
2475
|
-
route_id=rid,
|
|
2476
|
-
confidence=conf,
|
|
2477
|
-
strategy=strategy,
|
|
2478
|
-
method_call=call.method_call,
|
|
2479
|
-
raw_uri=call.raw_uri,
|
|
2480
|
-
match="unresolved",
|
|
2481
|
-
)
|
|
2482
|
-
)
|
|
2483
|
-
tables.call_edge_stats.http_calls_total += 1
|
|
2484
|
-
tables.call_edge_stats.http_calls_by_client_kind[call.client_kind] += 1
|
|
2485
|
-
tables.call_edge_stats.http_calls_by_strategy[strategy] += 1
|
|
2486
|
-
elif call.channel == "async":
|
|
2487
|
-
topic_atom = (call.topic_call or "").strip()
|
|
2488
|
-
pid = _producer_id(
|
|
2489
|
-
microservice=member.microservice,
|
|
2490
|
-
member_fqn=call.method_fqn,
|
|
2491
|
-
producer_kind=call.client_kind,
|
|
2492
|
-
topic=topic_atom,
|
|
2493
|
-
)
|
|
2494
|
-
if pid not in producer_seen:
|
|
2495
|
-
producer_seen.add(pid)
|
|
2496
|
-
tables.producer_rows.append(
|
|
2497
|
-
ProducerRow(
|
|
2498
|
-
id=pid,
|
|
2499
|
-
producer_kind=call.client_kind,
|
|
2500
|
-
topic=topic_atom,
|
|
2501
|
-
broker=call.broker_call,
|
|
2502
|
-
direction="producer",
|
|
2503
|
-
member_fqn=call.method_fqn,
|
|
2504
|
-
member_id=member.node_id,
|
|
2505
|
-
microservice=member.microservice,
|
|
2506
|
-
module=member.module,
|
|
2507
|
-
filename=call.filename,
|
|
2508
|
-
start_line=call.start_line,
|
|
2509
|
-
end_line=call.end_line,
|
|
2510
|
-
resolved=call.resolved,
|
|
2511
|
-
source_layer=_producer_source_layer(call.resolution_strategy),
|
|
2512
|
-
),
|
|
2513
|
-
)
|
|
2514
|
-
dpkey = (member.node_id, pid)
|
|
2515
|
-
if dpkey not in declares_producer_seen:
|
|
2516
|
-
declares_producer_seen.add(dpkey)
|
|
2517
|
-
tables.declares_producer_rows.append(
|
|
2518
|
-
DeclaresProducerRow(
|
|
2519
|
-
symbol_id=member.node_id,
|
|
2520
|
-
producer_id=pid,
|
|
2521
|
-
confidence=call.confidence_base,
|
|
2522
|
-
strategy=call.resolution_strategy,
|
|
2523
|
-
),
|
|
2524
|
-
)
|
|
2525
|
-
rid = _phantom_async_route_id(call)
|
|
2526
|
-
_append_route(
|
|
2527
|
-
RouteRow(
|
|
2528
|
-
id=rid,
|
|
2529
|
-
kind="kafka_topic",
|
|
2530
|
-
framework="",
|
|
2531
|
-
method="",
|
|
2532
|
-
path="",
|
|
2533
|
-
path_template="",
|
|
2534
|
-
path_regex="",
|
|
2535
|
-
topic=call.topic_call,
|
|
2536
|
-
broker=call.broker_call,
|
|
2537
|
-
feign_name="",
|
|
2538
|
-
feign_url="",
|
|
2539
|
-
microservice="",
|
|
2540
|
-
module="",
|
|
2541
|
-
filename=call.filename,
|
|
2542
|
-
start_line=call.start_line,
|
|
2543
|
-
end_line=call.end_line,
|
|
2544
|
-
resolved=False,
|
|
2545
|
-
source_layer="builtin",
|
|
2546
|
-
)
|
|
2547
|
-
)
|
|
2548
|
-
key = (pid, rid)
|
|
2549
|
-
if key in async_seen:
|
|
2550
|
-
continue
|
|
2551
|
-
async_seen.add(key)
|
|
2552
|
-
conf = call.confidence_base * 0.3 * micro_factor
|
|
2553
|
-
strategy = call.resolution_strategy
|
|
2554
|
-
tables.async_call_rows.append(
|
|
2555
|
-
AsyncCallRow(
|
|
2556
|
-
producer_id=pid,
|
|
2557
|
-
route_id=rid,
|
|
2558
|
-
confidence=conf,
|
|
2559
|
-
strategy=strategy,
|
|
2560
|
-
direction="producer",
|
|
2561
|
-
raw_topic=call.raw_topic,
|
|
2562
|
-
match="unresolved",
|
|
2563
|
-
)
|
|
2564
|
-
)
|
|
2565
|
-
tables.call_edge_stats.async_calls_total += 1
|
|
2566
|
-
tables.call_edge_stats.async_calls_by_client_kind[call.client_kind] += 1
|
|
2567
|
-
tables.call_edge_stats.async_calls_by_strategy[strategy] += 1
|
|
2568
|
-
|
|
2569
|
-
tables.routes_rows = sorted(route_rows, key=lambda r: r.id)
|
|
2570
|
-
tables.client_rows = sorted(tables.client_rows, key=lambda c: c.id)
|
|
2571
|
-
tables.declares_client_rows = sorted(
|
|
2572
|
-
tables.declares_client_rows,
|
|
2573
|
-
key=lambda e: (e.symbol_id, e.client_id),
|
|
2574
|
-
)
|
|
2575
|
-
tables.client_stats.clients_total = len(tables.client_rows)
|
|
2576
|
-
tables.client_stats.declares_client_total = len(tables.declares_client_rows)
|
|
2577
|
-
tables.client_stats.clients_by_kind = defaultdict(int)
|
|
2578
|
-
for row in tables.client_rows:
|
|
2579
|
-
tables.client_stats.clients_by_kind[row.client_kind] += 1
|
|
2580
|
-
tables.producer_rows = sorted(tables.producer_rows, key=lambda p: p.id)
|
|
2581
|
-
tables.declares_producer_rows = sorted(
|
|
2582
|
-
tables.declares_producer_rows,
|
|
2583
|
-
key=lambda e: (e.symbol_id, e.producer_id),
|
|
2584
|
-
)
|
|
2585
|
-
tables.producer_stats.producers_total = len(tables.producer_rows)
|
|
2586
|
-
tables.producer_stats.declares_producer_total = len(tables.declares_producer_rows)
|
|
2587
|
-
tables.producer_stats.producers_by_kind = defaultdict(int)
|
|
2588
|
-
for row in tables.producer_rows:
|
|
2589
|
-
tables.producer_stats.producers_by_kind[row.producer_kind] += 1
|
|
2590
|
-
# brownfield_strategies = the four brownfield layers plus the two
|
|
2591
|
-
# caller-side declaration strategies (@CodebaseHttpClient /
|
|
2592
|
-
# @CodebaseProducer). These extend _BROWNFIELD_LAYERS deliberately:
|
|
2593
|
-
# the *_from_brownfield_pct stats count annotation-declared callers as
|
|
2594
|
-
# brownfield-sourced even though they are not "layers" and so do not
|
|
2595
|
-
# gate brownfield_only authoritativeness in _is_brownfield_sourced.
|
|
2596
|
-
brownfield_strategies = _BROWNFIELD_LAYERS | frozenset(
|
|
2597
|
-
{"codebase_client", "codebase_producer"},
|
|
2598
|
-
)
|
|
2599
|
-
if tables.call_edge_stats.http_calls_total:
|
|
2600
|
-
n_http = sum(
|
|
2601
|
-
v for k, v in tables.call_edge_stats.http_calls_by_strategy.items()
|
|
2602
|
-
if k in brownfield_strategies
|
|
2603
|
-
)
|
|
2604
|
-
tables.call_edge_stats.http_clients_from_brownfield_pct = (
|
|
2605
|
-
100.0 * float(n_http) / float(tables.call_edge_stats.http_calls_total)
|
|
2606
|
-
)
|
|
2607
|
-
if tables.call_edge_stats.async_calls_total:
|
|
2608
|
-
n_async = sum(
|
|
2609
|
-
v for k, v in tables.call_edge_stats.async_calls_by_strategy.items()
|
|
2610
|
-
if k in brownfield_strategies
|
|
2611
|
-
)
|
|
2612
|
-
tables.call_edge_stats.async_producers_from_brownfield_pct = (
|
|
2613
|
-
100.0 * float(n_async) / float(tables.call_edge_stats.async_calls_total)
|
|
2614
|
-
)
|
|
2615
|
-
if verbose:
|
|
2616
|
-
http_client = dict(sorted(tables.call_edge_stats.http_calls_by_client_kind.items()))
|
|
2617
|
-
async_client = dict(sorted(tables.call_edge_stats.async_calls_by_client_kind.items()))
|
|
2618
|
-
http_strategy = dict(sorted(tables.call_edge_stats.http_calls_by_strategy.items()))
|
|
2619
|
-
async_strategy = dict(sorted(tables.call_edge_stats.async_calls_by_strategy.items()))
|
|
2620
|
-
_verbose_stderr_line(
|
|
2621
|
-
f"[graph] pass 5 · HTTP_CALLS: {len(tables.http_call_rows)} edges, "
|
|
2622
|
-
f"ASYNC_CALLS: {len(tables.async_call_rows)} edges; "
|
|
2623
|
-
f"http_by_client_kind={http_client}, async_by_client_kind={async_client}, "
|
|
2624
|
-
f"http_by_strategy={http_strategy}, async_by_strategy={async_strategy}",
|
|
2625
|
-
)
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
def _match_call_edge(
|
|
2629
|
-
call: OutgoingCallDecl,
|
|
2630
|
-
routes: list[RouteRow],
|
|
2631
|
-
caller_microservice: str,
|
|
2632
|
-
) -> tuple[str, list[RouteRow]]:
|
|
2633
|
-
"""Return (match_outcome, candidate_routes) for an outgoing call."""
|
|
2634
|
-
if (
|
|
2635
|
-
(not call.resolved)
|
|
2636
|
-
and call.path_template_call == ""
|
|
2637
|
-
and call.topic_call == ""
|
|
2638
|
-
):
|
|
2639
|
-
return "unresolved", []
|
|
2640
|
-
|
|
2641
|
-
candidates: list[RouteRow] = []
|
|
2642
|
-
if call.client_kind == CLIENT_KIND_FEIGN_METHOD:
|
|
2643
|
-
# Prefer endpoint matching by target service + path/method for Feign declarations.
|
|
2644
|
-
path_value = call.path_template_call
|
|
2645
|
-
method_value = call.method_call
|
|
2646
|
-
if path_value:
|
|
2647
|
-
for r in routes:
|
|
2648
|
-
if r.kind != "http_endpoint":
|
|
2649
|
-
continue
|
|
2650
|
-
if call.feign_target_name and r.microservice != call.feign_target_name:
|
|
2651
|
-
continue
|
|
2652
|
-
if not (r.method == "" or method_value == "" or r.method == method_value):
|
|
2653
|
-
continue
|
|
2654
|
-
if not r.path_regex:
|
|
2655
|
-
continue
|
|
2656
|
-
try:
|
|
2657
|
-
if re.fullmatch(r.path_regex, path_value or "") is None:
|
|
2658
|
-
continue
|
|
2659
|
-
except re.error:
|
|
2660
|
-
continue
|
|
2661
|
-
candidates.append(r)
|
|
2662
|
-
if not candidates:
|
|
2663
|
-
# Fallback for legacy/manual routes that only expose Feign target names.
|
|
2664
|
-
candidates = [
|
|
2665
|
-
r for r in routes
|
|
2666
|
-
if r.feign_name and call.feign_target_name and r.feign_name == call.feign_target_name
|
|
2667
|
-
]
|
|
2668
|
-
elif call.channel == "http":
|
|
2669
|
-
path_value = call.path_template_call
|
|
2670
|
-
method_value = call.method_call
|
|
2671
|
-
for r in routes:
|
|
2672
|
-
if r.kind != "http_endpoint":
|
|
2673
|
-
continue
|
|
2674
|
-
if not (r.method == "" or method_value == "" or r.method == method_value):
|
|
2675
|
-
continue
|
|
2676
|
-
if not r.path_regex:
|
|
2677
|
-
continue
|
|
2678
|
-
try:
|
|
2679
|
-
if re.fullmatch(r.path_regex, path_value or "") is None:
|
|
2680
|
-
continue
|
|
2681
|
-
except re.error:
|
|
2682
|
-
continue
|
|
2683
|
-
candidates.append(r)
|
|
2684
|
-
elif call.channel == "async":
|
|
2685
|
-
candidates = [
|
|
2686
|
-
r for r in routes
|
|
2687
|
-
if r.topic == call.topic_call and r.broker == call.broker_call
|
|
2688
|
-
]
|
|
2689
|
-
|
|
2690
|
-
if not candidates:
|
|
2691
|
-
return "phantom", []
|
|
2692
|
-
if len(candidates) > 1:
|
|
2693
|
-
return "ambiguous", candidates
|
|
2694
|
-
if candidates[0].microservice and candidates[0].microservice == caller_microservice:
|
|
2695
|
-
return "intra_service", candidates
|
|
2696
|
-
return "cross_service", candidates
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
def _is_brownfield_sourced(
|
|
2700
|
-
call_strategy: str,
|
|
2701
|
-
candidates: list[RouteRow],
|
|
2702
|
-
) -> bool:
|
|
2703
|
-
"""Both sides must come from brownfield layers for an edge to count as
|
|
2704
|
-
authoritative under brownfield_only mode."""
|
|
2705
|
-
if not candidates:
|
|
2706
|
-
return False
|
|
2707
|
-
if call_strategy not in _BROWNFIELD_LAYERS:
|
|
2708
|
-
return False
|
|
2709
|
-
return all(
|
|
2710
|
-
getattr(c, "source_layer", "builtin") in _BROWNFIELD_LAYERS
|
|
2711
|
-
for c in candidates
|
|
2712
|
-
)
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
def pass6_match_edges(
|
|
2716
|
-
tables: GraphTables,
|
|
2717
|
-
*,
|
|
2718
|
-
verbose: bool,
|
|
2719
|
-
) -> None:
|
|
2720
|
-
match_factor: dict[str, float] = {
|
|
2721
|
-
"cross_service": 1.0,
|
|
2722
|
-
"intra_service": 0.6,
|
|
2723
|
-
"ambiguous": 0.5,
|
|
2724
|
-
"phantom": 0.4,
|
|
2725
|
-
"unresolved": 0.3,
|
|
2726
|
-
}
|
|
2727
|
-
route_by_id = {r.id: r for r in tables.routes_rows}
|
|
2728
|
-
all_routes = [r for r in tables.routes_rows if r.microservice]
|
|
2729
|
-
member_by_id = {m.node_id: m for m in tables.members}
|
|
2730
|
-
clients_by_id = {c.id: c for c in tables.client_rows}
|
|
2731
|
-
producers_by_id = {p.id: p for p in tables.producer_rows}
|
|
2732
|
-
client_hints_by_member: dict[str, list[ClientRow]] = defaultdict(list)
|
|
2733
|
-
for edge in tables.declares_client_rows:
|
|
2734
|
-
client = clients_by_id.get(edge.client_id)
|
|
2735
|
-
if client is None:
|
|
2736
|
-
continue
|
|
2737
|
-
# `DECLARES_CLIENT.symbol_id` targets `Symbol.id` for member symbols,
|
|
2738
|
-
# and member symbols are emitted with `id == MemberEntry.node_id`.
|
|
2739
|
-
client_hints_by_member[edge.symbol_id].append(client)
|
|
2740
|
-
for member_symbol_id in list(client_hints_by_member.keys()):
|
|
2741
|
-
# Deterministic fallback when a method carries multiple feign declarations.
|
|
2742
|
-
client_hints_by_member[member_symbol_id].sort(key=lambda c: c.id)
|
|
2743
|
-
|
|
2744
|
-
# Pass 6 is idempotent for full rebuilds: each run fully re-derives match outcomes.
|
|
2745
|
-
# If incremental rebuild lands later (Tier-2 follow-up), this reset must remain pass-scoped.
|
|
2746
|
-
tables.call_edge_stats.http_calls_match_breakdown.clear()
|
|
2747
|
-
tables.call_edge_stats.async_calls_match_breakdown.clear()
|
|
2748
|
-
tables.call_edge_stats.cross_service_calls_total = 0
|
|
2749
|
-
|
|
2750
|
-
brownfield_only = tables.cross_service_resolution == "brownfield_only"
|
|
2751
|
-
suppressed_auto_cross_http: list[str] = []
|
|
2752
|
-
suppressed_auto_cross_async: list[str] = []
|
|
2753
|
-
suppressed_auto_cross_count = 0
|
|
2754
|
-
|
|
2755
|
-
def _micro_factor(member: MemberEntry | None) -> float:
|
|
2756
|
-
return 1.0 if (member and member.microservice) else 0.85
|
|
2757
|
-
|
|
2758
|
-
if verbose:
|
|
2759
|
-
_verbose_stderr_line(_PASS6_START)
|
|
2760
|
-
with _graph_pass_progress("6/6", verbose=verbose), _VerbosePassHeartbeats("[graph] pass 6", verbose=verbose):
|
|
2761
|
-
for row in tables.http_call_rows:
|
|
2762
|
-
if row.match != "unresolved":
|
|
2763
|
-
continue
|
|
2764
|
-
client = clients_by_id.get(row.client_id)
|
|
2765
|
-
member = member_by_id.get(client.member_id) if client else None
|
|
2766
|
-
base = row.confidence / max(1e-9, (0.3 * _micro_factor(member)))
|
|
2767
|
-
src_route = route_by_id.get(row.route_id)
|
|
2768
|
-
if src_route is None and member is not None:
|
|
2769
|
-
# Recover feign caller hints from persisted caller-side Client declarations.
|
|
2770
|
-
for client in client_hints_by_member.get(member.node_id, ()):
|
|
2771
|
-
if client.client_kind != CLIENT_KIND_FEIGN_METHOD:
|
|
2772
|
-
continue
|
|
2773
|
-
path_template, path_regex = _normalize_path(client.path)
|
|
2774
|
-
src_route = RouteRow(
|
|
2775
|
-
id="",
|
|
2776
|
-
kind="http_consumer",
|
|
2777
|
-
framework="feign",
|
|
2778
|
-
method=client.method,
|
|
2779
|
-
path=client.path,
|
|
2780
|
-
path_template=path_template,
|
|
2781
|
-
path_regex=path_regex,
|
|
2782
|
-
topic="",
|
|
2783
|
-
broker="",
|
|
2784
|
-
feign_name=client.target_service,
|
|
2785
|
-
# `Client` stores service-name hints, not feign URL; matcher keys off feign_name.
|
|
2786
|
-
feign_url="",
|
|
2787
|
-
microservice=member.microservice,
|
|
2788
|
-
module=member.module,
|
|
2789
|
-
filename=client.filename,
|
|
2790
|
-
start_line=client.start_line,
|
|
2791
|
-
end_line=client.end_line,
|
|
2792
|
-
resolved=client.resolved,
|
|
2793
|
-
source_layer=client.source_layer,
|
|
2794
|
-
)
|
|
2795
|
-
break
|
|
2796
|
-
# Feign caller hints are synthesized as transient `http_consumer` routes in pass6;
|
|
2797
|
-
# synthetic phantoms from imperative clients are `http_endpoint` even when `feign_name` is populated from
|
|
2798
|
-
# `@CodebaseHttpClient.targetService` / YAML hints — those must path-match like RestTemplate.
|
|
2799
|
-
_feign_like = (
|
|
2800
|
-
src_route is not None
|
|
2801
|
-
and src_route.kind == "http_consumer"
|
|
2802
|
-
and bool(src_route.feign_name)
|
|
2803
|
-
)
|
|
2804
|
-
call = OutgoingCallDecl(
|
|
2805
|
-
method_fqn=f"{member.parent_fqn}#{member.decl.signature}" if member else "",
|
|
2806
|
-
method_sig=member.decl.signature if member else "",
|
|
2807
|
-
client_kind=CLIENT_KIND_FEIGN_METHOD if _feign_like else CLIENT_KIND_REST_TEMPLATE,
|
|
2808
|
-
channel="http",
|
|
2809
|
-
feign_target_name=src_route.feign_name if src_route else "",
|
|
2810
|
-
feign_target_url=src_route.feign_url if src_route else "",
|
|
2811
|
-
path_template_call=src_route.path_template if src_route else "",
|
|
2812
|
-
method_call=row.method_call,
|
|
2813
|
-
topic_call="",
|
|
2814
|
-
broker_call="",
|
|
2815
|
-
raw_uri=row.raw_uri,
|
|
2816
|
-
raw_topic="",
|
|
2817
|
-
resolution_strategy=row.strategy,
|
|
2818
|
-
confidence_base=base,
|
|
2819
|
-
resolved=(row.strategy != "unresolved"),
|
|
2820
|
-
filename=member.file_path if member else "",
|
|
2821
|
-
start_line=member.decl.start_line if member else 0,
|
|
2822
|
-
end_line=member.decl.end_line if member else 0,
|
|
2823
|
-
)
|
|
2824
|
-
outcome, candidates = _match_call_edge(call, all_routes, member.microservice if member else "")
|
|
2825
|
-
if (
|
|
2826
|
-
brownfield_only
|
|
2827
|
-
and outcome == "cross_service"
|
|
2828
|
-
and not _is_brownfield_sourced(row.strategy, candidates)
|
|
2829
|
-
):
|
|
2830
|
-
outcome = "unresolved"
|
|
2831
|
-
candidates = []
|
|
2832
|
-
suppressed_auto_cross_count += 1
|
|
2833
|
-
if len(suppressed_auto_cross_http) < 5:
|
|
2834
|
-
suppressed_auto_cross_http.append(call.method_fqn)
|
|
2835
|
-
if outcome in VALID_HTTP_CALL_MATCHES:
|
|
2836
|
-
row.match = outcome
|
|
2837
|
-
if outcome in ("cross_service", "intra_service") and len(candidates) == 1:
|
|
2838
|
-
row.route_id = candidates[0].id
|
|
2839
|
-
row.confidence = call.confidence_base * match_factor[row.match] * _micro_factor(member)
|
|
2840
|
-
tables.call_edge_stats.http_calls_match_breakdown[row.match] += 1
|
|
2841
|
-
if row.match == "cross_service":
|
|
2842
|
-
tables.call_edge_stats.cross_service_calls_total += 1
|
|
2843
|
-
|
|
2844
|
-
for row in tables.async_call_rows:
|
|
2845
|
-
if row.match != "unresolved":
|
|
2846
|
-
continue
|
|
2847
|
-
producer = producers_by_id.get(row.producer_id)
|
|
2848
|
-
member = member_by_id.get(producer.member_id) if producer else None
|
|
2849
|
-
base = row.confidence / max(1e-9, (0.3 * _micro_factor(member)))
|
|
2850
|
-
src_route = route_by_id.get(row.route_id)
|
|
2851
|
-
async_kind = producer.producer_kind if producer else "kafka_send"
|
|
2852
|
-
call = OutgoingCallDecl(
|
|
2853
|
-
method_fqn=f"{member.parent_fqn}#{member.decl.signature}" if member else "",
|
|
2854
|
-
method_sig=member.decl.signature if member else "",
|
|
2855
|
-
client_kind=async_kind,
|
|
2856
|
-
channel="async",
|
|
2857
|
-
feign_target_name="",
|
|
2858
|
-
feign_target_url="",
|
|
2859
|
-
path_template_call="",
|
|
2860
|
-
method_call="",
|
|
2861
|
-
topic_call=src_route.topic if src_route else "",
|
|
2862
|
-
broker_call=src_route.broker if src_route else "",
|
|
2863
|
-
raw_uri="",
|
|
2864
|
-
raw_topic=row.raw_topic,
|
|
2865
|
-
resolution_strategy=row.strategy,
|
|
2866
|
-
confidence_base=base,
|
|
2867
|
-
resolved=(row.strategy != "unresolved"),
|
|
2868
|
-
filename=member.file_path if member else "",
|
|
2869
|
-
start_line=member.decl.start_line if member else 0,
|
|
2870
|
-
end_line=member.decl.end_line if member else 0,
|
|
2871
|
-
)
|
|
2872
|
-
outcome, candidates = _match_call_edge(call, all_routes, member.microservice if member else "")
|
|
2873
|
-
if (
|
|
2874
|
-
brownfield_only
|
|
2875
|
-
and outcome == "cross_service"
|
|
2876
|
-
and not _is_brownfield_sourced(row.strategy, candidates)
|
|
2877
|
-
):
|
|
2878
|
-
outcome = "unresolved"
|
|
2879
|
-
candidates = []
|
|
2880
|
-
suppressed_auto_cross_count += 1
|
|
2881
|
-
if len(suppressed_auto_cross_async) < 5:
|
|
2882
|
-
suppressed_auto_cross_async.append(call.method_fqn)
|
|
2883
|
-
if outcome in VALID_HTTP_CALL_MATCHES:
|
|
2884
|
-
row.match = outcome
|
|
2885
|
-
if outcome in ("cross_service", "intra_service") and len(candidates) == 1:
|
|
2886
|
-
row.route_id = candidates[0].id
|
|
2887
|
-
row.confidence = call.confidence_base * match_factor[row.match] * _micro_factor(member)
|
|
2888
|
-
tables.call_edge_stats.async_calls_match_breakdown[row.match] += 1
|
|
2889
|
-
if row.match == "cross_service":
|
|
2890
|
-
tables.call_edge_stats.cross_service_calls_total += 1
|
|
2891
|
-
|
|
2892
|
-
inbound_route_ids = {r.route_id for r in tables.http_call_rows} | {r.route_id for r in tables.async_call_rows}
|
|
2893
|
-
tables.routes_rows = sorted(
|
|
2894
|
-
[
|
|
2895
|
-
r for r in tables.routes_rows
|
|
2896
|
-
if not (
|
|
2897
|
-
(r.microservice == "")
|
|
2898
|
-
and (r.framework == "")
|
|
2899
|
-
and (not r.resolved)
|
|
2900
|
-
and (r.id not in inbound_route_ids)
|
|
2901
|
-
)
|
|
2902
|
-
],
|
|
2903
|
-
key=lambda r: r.id,
|
|
2904
|
-
)
|
|
2905
|
-
|
|
2906
|
-
if verbose:
|
|
2907
|
-
if brownfield_only:
|
|
2908
|
-
n_bf = tables.call_edge_stats.cross_service_calls_total
|
|
2909
|
-
first_http = ", ".join(suppressed_auto_cross_http)
|
|
2910
|
-
first_async = ", ".join(suppressed_auto_cross_async)
|
|
2911
|
-
_verbose_stderr_line(
|
|
2912
|
-
f"[graph] pass 6 · cross_service_resolution=brownfield_only:\n"
|
|
2913
|
-
f" {n_bf} cross_service edges from brownfield layers,\n"
|
|
2914
|
-
f" {suppressed_auto_cross_count} auto-cross-service candidates suppressed -> unresolved\n"
|
|
2915
|
-
f" (first 5 http: {first_http})\n"
|
|
2916
|
-
f" (first 5 async: {first_async})",
|
|
2917
|
-
)
|
|
2918
|
-
_verbose_stderr_line(
|
|
2919
|
-
f"[graph] pass 6 · http_match={dict(sorted(tables.call_edge_stats.http_calls_match_breakdown.items()))}, "
|
|
2920
|
-
f"async_match={dict(sorted(tables.call_edge_stats.async_calls_match_breakdown.items()))}, "
|
|
2921
|
-
f"cross_service_calls_total={tables.call_edge_stats.cross_service_calls_total}",
|
|
2922
|
-
)
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
# ---------- LadybugDB write ----------
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
_SCHEMA_NODE = (
|
|
2929
|
-
"CREATE NODE TABLE Symbol("
|
|
2930
|
-
"id STRING PRIMARY KEY, "
|
|
2931
|
-
"kind STRING, name STRING, fqn STRING, package STRING, "
|
|
2932
|
-
"module STRING, microservice STRING, "
|
|
2933
|
-
"filename STRING, start_line INT64, end_line INT64, "
|
|
2934
|
-
"start_byte INT64, end_byte INT64, "
|
|
2935
|
-
"modifiers STRING[], annotations STRING[], capabilities STRING[], "
|
|
2936
|
-
"role STRING, signature STRING, parent_id STRING, resolved BOOLEAN, "
|
|
2937
|
-
"generated BOOLEAN, generated_by STRING, "
|
|
2938
|
-
"search_text STRING"
|
|
2939
|
-
")"
|
|
2940
|
-
)
|
|
2941
|
-
|
|
2942
|
-
_SCHEMA_META = (
|
|
2943
|
-
"CREATE NODE TABLE GraphMeta("
|
|
2944
|
-
"key STRING PRIMARY KEY, "
|
|
2945
|
-
"ontology_version INT64, built_at INT64, source_root STRING, "
|
|
2946
|
-
"counts_json STRING, parse_errors INT64, "
|
|
2947
|
-
"routes_total INT64, exposes_total INT64, "
|
|
2948
|
-
# JSON map {framework: count}; STRING avoids LadybugDB Python MAP↔STRUCT binder mismatch.
|
|
2949
|
-
"routes_by_framework STRING, "
|
|
2950
|
-
"routes_resolved_pct DOUBLE, "
|
|
2951
|
-
"routes_from_brownfield_pct DOUBLE, "
|
|
2952
|
-
"routes_by_layer STRING, "
|
|
2953
|
-
"clients_total INT64, "
|
|
2954
|
-
"declares_client_total INT64, "
|
|
2955
|
-
"clients_by_kind STRING, "
|
|
2956
|
-
"producers_total INT64, "
|
|
2957
|
-
"declares_producer_total INT64, "
|
|
2958
|
-
"producers_by_kind STRING, "
|
|
2959
|
-
"http_calls_total INT64, "
|
|
2960
|
-
"async_calls_total INT64, "
|
|
2961
|
-
"http_calls_by_strategy STRING, "
|
|
2962
|
-
"async_calls_by_strategy STRING, "
|
|
2963
|
-
"http_calls_resolved_pct DOUBLE, "
|
|
2964
|
-
"async_calls_resolved_pct DOUBLE, "
|
|
2965
|
-
"http_clients_from_brownfield_pct DOUBLE, "
|
|
2966
|
-
"async_producers_from_brownfield_pct DOUBLE, "
|
|
2967
|
-
"http_calls_match_breakdown STRING, "
|
|
2968
|
-
"async_calls_match_breakdown STRING, "
|
|
2969
|
-
"cross_service_calls_total INT64, "
|
|
2970
|
-
"pass3_skipped_cross_service INT64, "
|
|
2971
|
-
"pass3_unresolved_phantom_receiver INT64, "
|
|
2972
|
-
"pass3_unresolved_chained INT64, "
|
|
2973
|
-
"pass4_exposes_suppressed_feign INT64, "
|
|
2974
|
-
"cross_service_resolution STRING"
|
|
2975
|
-
")"
|
|
2976
|
-
)
|
|
2977
|
-
|
|
2978
|
-
_SCHEMA_ROUTE = (
|
|
2979
|
-
"CREATE NODE TABLE Route("
|
|
2980
|
-
"id STRING, kind STRING, framework STRING, "
|
|
2981
|
-
"method STRING, path STRING, path_template STRING, path_regex STRING, "
|
|
2982
|
-
"topic STRING, broker STRING, "
|
|
2983
|
-
"feign_name STRING, feign_url STRING, "
|
|
2984
|
-
"microservice STRING, module STRING, "
|
|
2985
|
-
"filename STRING, start_line INT64, end_line INT64, "
|
|
2986
|
-
"resolved BOOLEAN, "
|
|
2987
|
-
"PRIMARY KEY(id))"
|
|
2988
|
-
)
|
|
2989
|
-
|
|
2990
|
-
_SCHEMA_CLIENT = (
|
|
2991
|
-
"CREATE NODE TABLE Client("
|
|
2992
|
-
"id STRING, client_kind STRING, target_service STRING, "
|
|
2993
|
-
"path STRING, path_template STRING, path_regex STRING, method STRING, "
|
|
2994
|
-
"member_fqn STRING, member_id STRING, "
|
|
2995
|
-
"microservice STRING, module STRING, filename STRING, "
|
|
2996
|
-
"start_line INT64, end_line INT64, resolved BOOLEAN, source_layer STRING, "
|
|
2997
|
-
"PRIMARY KEY(id))"
|
|
2998
|
-
)
|
|
2999
|
-
|
|
3000
|
-
_SCHEMA_PRODUCER = (
|
|
3001
|
-
"CREATE NODE TABLE Producer("
|
|
3002
|
-
"id STRING, producer_kind STRING, topic STRING, broker STRING, direction STRING, "
|
|
3003
|
-
"member_fqn STRING, member_id STRING, "
|
|
3004
|
-
"microservice STRING, module STRING, filename STRING, "
|
|
3005
|
-
"start_line INT64, end_line INT64, resolved BOOLEAN, source_layer STRING, "
|
|
3006
|
-
"PRIMARY KEY(id))"
|
|
3007
|
-
)
|
|
3008
|
-
|
|
3009
|
-
_SCHEMA_EXTENDS = (
|
|
3010
|
-
"CREATE REL TABLE EXTENDS(FROM Symbol TO Symbol, "
|
|
3011
|
-
"source_file STRING, dst_name STRING, dst_fqn STRING, resolved BOOLEAN)"
|
|
3012
|
-
)
|
|
3013
|
-
_SCHEMA_IMPLEMENTS = (
|
|
3014
|
-
"CREATE REL TABLE IMPLEMENTS(FROM Symbol TO Symbol, "
|
|
3015
|
-
"source_file STRING, dst_name STRING, dst_fqn STRING, resolved BOOLEAN)"
|
|
3016
|
-
)
|
|
3017
|
-
_SCHEMA_INJECTS = (
|
|
3018
|
-
"CREATE REL TABLE INJECTS(FROM Symbol TO Symbol, "
|
|
3019
|
-
"source_file STRING, dst_name STRING, dst_fqn STRING, resolved BOOLEAN, "
|
|
3020
|
-
"mechanism STRING, annotation STRING, field_or_param STRING)"
|
|
3021
|
-
)
|
|
3022
|
-
_SCHEMA_DECLARES = "CREATE REL TABLE DECLARES(FROM Symbol TO Symbol, source_file STRING)"
|
|
3023
|
-
_SCHEMA_OVERRIDES = "CREATE REL TABLE OVERRIDES(FROM Symbol TO Symbol, source_file STRING)"
|
|
3024
|
-
_SCHEMA_CALLS = (
|
|
3025
|
-
"CREATE REL TABLE CALLS(FROM Symbol TO Symbol, "
|
|
3026
|
-
"source_file STRING, call_site_line INT64, call_site_byte INT64, arg_count INT64, "
|
|
3027
|
-
"confidence DOUBLE, strategy STRING, source STRING, resolved BOOLEAN, "
|
|
3028
|
-
"callee_declaring_role STRING)"
|
|
3029
|
-
)
|
|
3030
|
-
_SCHEMA_UNRESOLVED_CALL_SITE = (
|
|
3031
|
-
"CREATE NODE TABLE UnresolvedCallSite("
|
|
3032
|
-
"id STRING, caller_id STRING, call_site_line INT64, call_site_byte INT64, "
|
|
3033
|
-
"arg_count INT64, callee_simple STRING, receiver_expr STRING, reason STRING, "
|
|
3034
|
-
"PRIMARY KEY(id))"
|
|
3035
|
-
)
|
|
3036
|
-
_SCHEMA_UNRESOLVED_AT = "CREATE REL TABLE UNRESOLVED_AT(FROM Symbol TO UnresolvedCallSite, source_file STRING)"
|
|
3037
|
-
_SCHEMA_EXPOSES = (
|
|
3038
|
-
"CREATE REL TABLE EXPOSES(FROM Symbol TO Route, "
|
|
3039
|
-
"source_file STRING, confidence DOUBLE, strategy STRING)"
|
|
3040
|
-
)
|
|
3041
|
-
_SCHEMA_DECLARES_CLIENT = (
|
|
3042
|
-
"CREATE REL TABLE DECLARES_CLIENT(FROM Symbol TO Client, "
|
|
3043
|
-
"source_file STRING, confidence DOUBLE, strategy STRING)"
|
|
3044
|
-
)
|
|
3045
|
-
_SCHEMA_DECLARES_PRODUCER = (
|
|
3046
|
-
"CREATE REL TABLE DECLARES_PRODUCER(FROM Symbol TO Producer, "
|
|
3047
|
-
"source_file STRING, confidence DOUBLE, strategy STRING)"
|
|
3048
|
-
)
|
|
3049
|
-
_SCHEMA_HTTP_CALLS = (
|
|
3050
|
-
"CREATE REL TABLE HTTP_CALLS(FROM Client TO Route, "
|
|
3051
|
-
"source_file STRING, confidence DOUBLE, strategy STRING, "
|
|
3052
|
-
"method_call STRING, raw_uri STRING, match STRING)"
|
|
3053
|
-
)
|
|
3054
|
-
_SCHEMA_ASYNC_CALLS = (
|
|
3055
|
-
"CREATE REL TABLE ASYNC_CALLS(FROM Producer TO Route, "
|
|
3056
|
-
"source_file STRING, confidence DOUBLE, strategy STRING, "
|
|
3057
|
-
"direction STRING, raw_topic STRING, match STRING)"
|
|
3058
|
-
)
|
|
3059
|
-
|
|
3060
|
-
|
|
3061
|
-
def _drop_symbol_fts_index_if_present(conn: ladybug.Connection) -> None:
|
|
3062
|
-
"""Drop the Symbol FTS index so the table drop below succeeds.
|
|
3063
|
-
|
|
3064
|
-
LadybugDB refuses ``DROP TABLE Symbol`` while an FTS index references it ("Cannot
|
|
3065
|
-
delete node table ... referenced by index"), so the index must go first. No-op when
|
|
3066
|
-
FTS is unavailable (offline first-run) or the index was never created — in those
|
|
3067
|
-
cases nothing blocks the table drop. Best-effort: any failure is swallowed.
|
|
3068
|
-
"""
|
|
3069
|
-
try:
|
|
3070
|
-
conn.execute("LOAD EXTENSION FTS")
|
|
3071
|
-
existing = conn.execute("CALL SHOW_INDEXES() RETURN index_name")
|
|
3072
|
-
names: set[str] = set()
|
|
3073
|
-
while existing.has_next():
|
|
3074
|
-
names.add(existing.get_next()[0])
|
|
3075
|
-
if _SYMBOL_FTS_INDEX in names:
|
|
3076
|
-
conn.execute(f"CALL DROP_FTS_INDEX('Symbol', '{_SYMBOL_FTS_INDEX}')")
|
|
3077
|
-
except Exception:
|
|
3078
|
-
pass
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
def _drop_all(conn: ladybug.Connection) -> None:
|
|
3082
|
-
_drop_symbol_fts_index_if_present(conn)
|
|
3083
|
-
for stmt in (
|
|
3084
|
-
"DROP TABLE IF EXISTS DECLARES_CLIENT",
|
|
3085
|
-
"DROP TABLE IF EXISTS DECLARES_PRODUCER",
|
|
3086
|
-
"DROP TABLE IF EXISTS HTTP_CALLS",
|
|
3087
|
-
"DROP TABLE IF EXISTS ASYNC_CALLS",
|
|
3088
|
-
"DROP TABLE IF EXISTS EXPOSES",
|
|
3089
|
-
"DROP TABLE IF EXISTS UNRESOLVED_AT",
|
|
3090
|
-
"DROP TABLE IF EXISTS EXTENDS",
|
|
3091
|
-
"DROP TABLE IF EXISTS IMPLEMENTS",
|
|
3092
|
-
"DROP TABLE IF EXISTS INJECTS",
|
|
3093
|
-
"DROP TABLE IF EXISTS CALLS",
|
|
3094
|
-
"DROP TABLE IF EXISTS OVERRIDES",
|
|
3095
|
-
"DROP TABLE IF EXISTS DECLARES",
|
|
3096
|
-
"DROP TABLE IF EXISTS UnresolvedCallSite",
|
|
3097
|
-
"DROP TABLE IF EXISTS Symbol",
|
|
3098
|
-
"DROP TABLE IF EXISTS Route",
|
|
3099
|
-
"DROP TABLE IF EXISTS Client",
|
|
3100
|
-
"DROP TABLE IF EXISTS Producer",
|
|
3101
|
-
"DROP TABLE IF EXISTS GraphMeta",
|
|
3102
|
-
):
|
|
3103
|
-
try:
|
|
3104
|
-
conn.execute(stmt)
|
|
3105
|
-
except Exception:
|
|
3106
|
-
pass
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
def _create_schema(conn: ladybug.Connection) -> None:
|
|
3110
|
-
for stmt in (
|
|
3111
|
-
_SCHEMA_NODE,
|
|
3112
|
-
_SCHEMA_UNRESOLVED_CALL_SITE,
|
|
3113
|
-
_SCHEMA_ROUTE,
|
|
3114
|
-
_SCHEMA_CLIENT,
|
|
3115
|
-
_SCHEMA_PRODUCER,
|
|
3116
|
-
_SCHEMA_META,
|
|
3117
|
-
_SCHEMA_EXTENDS,
|
|
3118
|
-
_SCHEMA_IMPLEMENTS,
|
|
3119
|
-
_SCHEMA_INJECTS,
|
|
3120
|
-
_SCHEMA_DECLARES,
|
|
3121
|
-
_SCHEMA_OVERRIDES,
|
|
3122
|
-
_SCHEMA_CALLS,
|
|
3123
|
-
_SCHEMA_UNRESOLVED_AT,
|
|
3124
|
-
_SCHEMA_EXPOSES,
|
|
3125
|
-
_SCHEMA_DECLARES_CLIENT,
|
|
3126
|
-
_SCHEMA_DECLARES_PRODUCER,
|
|
3127
|
-
_SCHEMA_HTTP_CALLS,
|
|
3128
|
-
_SCHEMA_ASYNC_CALLS,
|
|
3129
|
-
):
|
|
3130
|
-
conn.execute(stmt)
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
_IDENT_RE = re.compile(r"[A-Za-z][A-Za-z0-9]*")
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
def _compute_symbol_search_text(
|
|
3137
|
-
name: str,
|
|
3138
|
-
fqn: str,
|
|
3139
|
-
signature: str,
|
|
3140
|
-
annotations: list[str],
|
|
3141
|
-
capabilities: list[str],
|
|
3142
|
-
package: str = "",
|
|
3143
|
-
) -> str:
|
|
3144
|
-
"""Build the BM25-indexable token soup for a Symbol (fork A).
|
|
3145
|
-
|
|
3146
|
-
camelCase / snake_case identifiers are split into lowercase tokens via the SAME
|
|
3147
|
-
``_split_identifier`` the query path uses (index- and query-time tokenization
|
|
3148
|
-
must agree), then space-joined into one STRING that LadybugDB FTS porter-stems.
|
|
3149
|
-
Name + fqn carry the strongest discovery signal; signature / annotations /
|
|
3150
|
-
capabilities / package segments are weaker corroborators. Members reach this via
|
|
3151
|
-
the same ``_node_row`` constructor as types.
|
|
3152
|
-
"""
|
|
3153
|
-
toks: list[str] = []
|
|
3154
|
-
for field in (name, fqn, signature, package):
|
|
3155
|
-
for m in _IDENT_RE.findall(str(field or "")):
|
|
3156
|
-
toks.extend(_split_identifier(m))
|
|
3157
|
-
for lst in (annotations, capabilities):
|
|
3158
|
-
for item in (lst or []):
|
|
3159
|
-
for m in _IDENT_RE.findall(str(item)):
|
|
3160
|
-
toks.extend(_split_identifier(m))
|
|
3161
|
-
seen: set[str] = set()
|
|
3162
|
-
out: list[str] = []
|
|
3163
|
-
for t in toks:
|
|
3164
|
-
if len(t) >= 2 and t not in seen:
|
|
3165
|
-
seen.add(t)
|
|
3166
|
-
out.append(t)
|
|
3167
|
-
return " ".join(out)
|
|
3168
|
-
|
|
3169
|
-
|
|
3170
|
-
def _ensure_symbol_fts_index(conn: ladybug.Connection, *, verbose: bool) -> None:
|
|
3171
|
-
"""Best-effort create the Symbol FTS (Okapi BM25) index if absent (fork A).
|
|
3172
|
-
|
|
3173
|
-
Idempotent: skips when the index already exists. The FTS extension is fetched
|
|
3174
|
-
from extension.ladybugdb.com on first ``INSTALL FTS`` (cached locally after); an
|
|
3175
|
-
offline first run fails softly — ``run_lexical_search`` then falls back to the
|
|
3176
|
-
heuristic over the bare Symbol scan. The index auto-maintains on later
|
|
3177
|
-
COPY / MERGE / DELETE (verified), so this only runs on full builds and as a
|
|
3178
|
-
cheap self-heal at the end of an incremental rebuild.
|
|
3179
|
-
"""
|
|
3180
|
-
try:
|
|
3181
|
-
try:
|
|
3182
|
-
conn.execute("INSTALL FTS") # already-installed raises; swallow
|
|
3183
|
-
except Exception:
|
|
3184
|
-
pass
|
|
3185
|
-
conn.execute("LOAD EXTENSION FTS")
|
|
3186
|
-
existing = conn.execute("CALL SHOW_INDEXES() RETURN index_name")
|
|
3187
|
-
names: set[str] = set()
|
|
3188
|
-
while existing.has_next():
|
|
3189
|
-
names.add(existing.get_next()[0])
|
|
3190
|
-
if _SYMBOL_FTS_INDEX not in names:
|
|
3191
|
-
conn.execute(
|
|
3192
|
-
f"CALL CREATE_FTS_INDEX('Symbol', '{_SYMBOL_FTS_INDEX}', "
|
|
3193
|
-
"['search_text'], stemmer := 'porter')"
|
|
3194
|
-
)
|
|
3195
|
-
if verbose:
|
|
3196
|
-
_verbose_stderr_line(
|
|
3197
|
-
f"[graph] fts · created Symbol {_SYMBOL_FTS_INDEX} (BM25 over search_text)"
|
|
3198
|
-
)
|
|
3199
|
-
except Exception as e:
|
|
3200
|
-
if verbose:
|
|
3201
|
-
_verbose_stderr_line(
|
|
3202
|
-
f"[graph] fts · unavailable ({type(e).__name__}: {e}); "
|
|
3203
|
-
"lexical search will use the heuristic scan"
|
|
3204
|
-
)
|
|
3205
|
-
|
|
3206
|
-
|
|
3207
|
-
def _node_row(**kwargs) -> dict:
|
|
3208
|
-
base = {
|
|
3209
|
-
"kind": "", "name": "", "fqn": "", "package": "",
|
|
3210
|
-
"module": "", "microservice": "",
|
|
3211
|
-
"filename": "", "start_line": 0, "end_line": 0,
|
|
3212
|
-
"start_byte": 0, "end_byte": 0,
|
|
3213
|
-
"modifiers": [], "annotations": [], "capabilities": [],
|
|
3214
|
-
"role": "OTHER", "signature": "", "parent_id": "", "resolved": True,
|
|
3215
|
-
"generated": False, "generated_by": None, "search_text": "",
|
|
3216
|
-
}
|
|
3217
|
-
base.update(kwargs)
|
|
3218
|
-
# Derive the BM25 token soup unless the caller set it explicitly.
|
|
3219
|
-
if not base.get("search_text"):
|
|
3220
|
-
base["search_text"] = _compute_symbol_search_text(
|
|
3221
|
-
base.get("name", ""), base.get("fqn", ""), base.get("signature", ""),
|
|
3222
|
-
base.get("annotations", []), base.get("capabilities", []),
|
|
3223
|
-
base.get("package", ""),
|
|
3224
|
-
)
|
|
3225
|
-
return base
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
def _bulk_copy(conn: ladybug.Connection, table_name: str, columns: list[str], rows: list[dict]) -> None:
|
|
3229
|
-
"""Bulk-load rows into a node/rel table via in-memory pyarrow COPY FROM.
|
|
3230
|
-
|
|
3231
|
-
`columns` fixes column order; for REL tables the first two MUST be the
|
|
3232
|
-
FROM/TO node primary keys (kuzu requirement). Empty `rows` is a no-op.
|
|
3233
|
-
|
|
3234
|
-
Spike result (PR-P1 step-1): REL `COPY FROM` expects columns named `FROM` and `TO`
|
|
3235
|
-
for the endpoint node IDs, followed by property columns in the declared order.
|
|
3236
|
-
`pa.Table.from_pylist(rows)` correctly infers types from the dict values, but
|
|
3237
|
-
we must select columns in the exact order expected by the table schema.
|
|
3238
|
-
"""
|
|
3239
|
-
if not rows:
|
|
3240
|
-
return
|
|
3241
|
-
tbl = pa.Table.from_pylist(rows)
|
|
3242
|
-
# Select columns in the exact order expected by the table schema
|
|
3243
|
-
tbl = tbl.select(columns)
|
|
3244
|
-
conn.execute(f"COPY {table_name} FROM $rows", {"rows": tbl})
|
|
3245
|
-
|
|
3246
|
-
|
|
3247
|
-
def _existing_node_ids(conn: ladybug.Connection) -> set[str]:
|
|
3248
|
-
"""Return every node id (Symbol, Route, Client, Producer) currently in the graph.
|
|
3249
|
-
|
|
3250
|
-
Bulk ``COPY FROM`` enforces referential integrity: a REL row whose FROM/TO
|
|
3251
|
-
endpoint isn't a loaded node raises ``Unable to find primary key value``. The
|
|
3252
|
-
legacy per-row ``MERGE (a:Symbol {id:$src}),(b:Symbol {id:$dst})`` silently
|
|
3253
|
-
dropped such edges (a ``MATCH`` against a missing endpoint creates nothing).
|
|
3254
|
-
Edge writers filter edge rows against this set to reproduce that exactly.
|
|
3255
|
-
|
|
3256
|
-
This queries the live DB rather than just ``tables`` because it is shared
|
|
3257
|
-
with the incremental path, whose edges legitimately reference nodes written
|
|
3258
|
-
in prior runs.
|
|
3259
|
-
"""
|
|
3260
|
-
result = conn.execute("MATCH (n) RETURN n.id")
|
|
3261
|
-
ids: set[str] = set()
|
|
3262
|
-
while result.has_next():
|
|
3263
|
-
ids.add(result.get_next()[0])
|
|
3264
|
-
return ids
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
# Column-order constants for bulk COPY FROM.
|
|
3268
|
-
# For REL tables, the first two entries are FROM/TO node primary keys (kuzu requirement).
|
|
3269
|
-
# Order matches the corresponding _SCHEMA_* declarations above.
|
|
3270
|
-
_NODE_COLUMNS = [
|
|
3271
|
-
"id", "kind", "name", "fqn", "package", "module", "microservice",
|
|
3272
|
-
"filename", "start_line", "end_line", "start_byte", "end_byte",
|
|
3273
|
-
"modifiers", "annotations", "capabilities", "role", "signature", "parent_id", "resolved",
|
|
3274
|
-
"generated", "generated_by", "search_text"
|
|
3275
|
-
]
|
|
3276
|
-
|
|
3277
|
-
# Type declaration kinds. Tuple (not set) so the rendered SQL `IN` clause is
|
|
3278
|
-
# deterministic. Used to (a) load type stubs for cross-file resolution and
|
|
3279
|
-
# (b) scope the incremental property-refresh SET to type nodes.
|
|
3280
|
-
_TYPE_KINDS: tuple[str, ...] = ("class", "interface", "enum", "annotation", "record")
|
|
3281
|
-
|
|
3282
|
-
# Update every mutable Symbol field on an existing node by primary key. Used on
|
|
3283
|
-
# the incremental path to refresh preserved dependent type nodes whose
|
|
3284
|
-
# `role`/`capabilities` (and other project-wide-derived fields) can shift
|
|
3285
|
-
# without their own source changing — restoring the upsert the legacy per-row
|
|
3286
|
-
# `MERGE (n:Symbol {id:$id}) SET …` provided. Field list mirrors `_NODE_COLUMNS`
|
|
3287
|
-
# minus `id`.
|
|
3288
|
-
_SET_SYMBOL_BY_ID = (
|
|
3289
|
-
"MATCH (n:Symbol {id: $id}) "
|
|
3290
|
-
"SET n.kind = $kind, n.name = $name, n.fqn = $fqn, "
|
|
3291
|
-
"n.package = $package, n.module = $module, n.microservice = $microservice, "
|
|
3292
|
-
"n.filename = $filename, "
|
|
3293
|
-
"n.start_line = $start_line, n.end_line = $end_line, "
|
|
3294
|
-
"n.start_byte = $start_byte, n.end_byte = $end_byte, "
|
|
3295
|
-
"n.modifiers = $modifiers, n.annotations = $annotations, "
|
|
3296
|
-
"n.capabilities = $capabilities, n.role = $role, "
|
|
3297
|
-
"n.signature = $signature, n.parent_id = $parent_id, n.resolved = $resolved, "
|
|
3298
|
-
"n.generated = $generated, n.generated_by = $generated_by, n.search_text = $search_text"
|
|
3299
|
-
)
|
|
3300
|
-
|
|
3301
|
-
# Refresh every mutable Route field on an existing Route node by id. Mirrors the
|
|
3302
|
-
# `_write_nodes_impl` Symbol pattern (bulk COPY new rows + per-row SET existing
|
|
3303
|
-
# ones) so the global pass 5-6 step no longer needs a per-row MERGE upsert.
|
|
3304
|
-
# Field list mirrors `_ROUTE_COLUMNS` minus `id`.
|
|
3305
|
-
_SET_ROUTE_BY_ID = (
|
|
3306
|
-
"MATCH (r:Route {id: $id}) "
|
|
3307
|
-
"SET r.kind = $kind, r.framework = $framework, r.method = $method, "
|
|
3308
|
-
"r.path = $path, r.path_template = $path_template, r.path_regex = $path_regex, "
|
|
3309
|
-
"r.topic = $topic, r.broker = $broker, r.feign_name = $feign_name, r.feign_url = $feign_url, "
|
|
3310
|
-
"r.microservice = $microservice, r.module = $module, r.filename = $filename, "
|
|
3311
|
-
"r.start_line = $start_line, r.end_line = $end_line, r.resolved = $resolved"
|
|
3312
|
-
)
|
|
3313
|
-
|
|
3314
|
-
_REL_EXTENDS_COLUMNS = ["FROM", "TO", "source_file", "dst_name", "dst_fqn", "resolved"]
|
|
3315
|
-
_REL_IMPLEMENTS_COLUMNS = ["FROM", "TO", "source_file", "dst_name", "dst_fqn", "resolved"]
|
|
3316
|
-
_REL_INJECTS_COLUMNS = ["FROM", "TO", "source_file", "dst_name", "dst_fqn", "resolved", "mechanism", "annotation", "field_or_param"]
|
|
3317
|
-
_REL_DECLARES_COLUMNS = ["FROM", "TO", "source_file"]
|
|
3318
|
-
_REL_OVERRIDES_COLUMNS = ["FROM", "TO", "source_file"]
|
|
3319
|
-
_REL_CALLS_COLUMNS = ["FROM", "TO", "source_file", "call_site_line", "call_site_byte", "arg_count", "confidence", "strategy", "source", "resolved", "callee_declaring_role"]
|
|
3320
|
-
|
|
3321
|
-
_UNRESOLVED_CALL_SITE_COLUMNS = ["id", "caller_id", "call_site_line", "call_site_byte", "arg_count", "callee_simple", "receiver_expr", "reason"]
|
|
3322
|
-
_REL_UNRESOLVED_AT_COLUMNS = ["FROM", "TO", "source_file"]
|
|
3323
|
-
|
|
3324
|
-
# Node table column constants (for bulk COPY FROM)
|
|
3325
|
-
_ROUTE_COLUMNS = ["id", "kind", "framework", "method", "path", "path_template", "path_regex", "topic", "broker", "feign_name", "feign_url", "microservice", "module", "filename", "start_line", "end_line", "resolved"]
|
|
3326
|
-
_CLIENT_COLUMNS = ["id", "client_kind", "target_service", "path", "path_template", "path_regex", "method", "member_fqn", "member_id", "microservice", "module", "filename", "start_line", "end_line", "resolved", "source_layer"]
|
|
3327
|
-
_PRODUCER_COLUMNS = ["id", "producer_kind", "topic", "broker", "direction", "member_fqn", "member_id", "microservice", "module", "filename", "start_line", "end_line", "resolved", "source_layer"]
|
|
3328
|
-
|
|
3329
|
-
# REL table column constants for routes/clients/producers
|
|
3330
|
-
_REL_EXPOSES_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy"]
|
|
3331
|
-
_REL_DECLARES_CLIENT_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy"]
|
|
3332
|
-
_REL_DECLARES_PRODUCER_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy"]
|
|
3333
|
-
_REL_HTTP_CALLS_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy", "method_call", "raw_uri", "match"]
|
|
3334
|
-
_REL_ASYNC_CALLS_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy", "direction", "raw_topic", "match"]
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
def _write_nodes_impl(
|
|
3338
|
-
conn: ladybug.Connection,
|
|
3339
|
-
tables: GraphTables,
|
|
3340
|
-
*,
|
|
3341
|
-
project_root: Path,
|
|
3342
|
-
meta_chain: dict[str, frozenset[str]] | None,
|
|
3343
|
-
) -> None:
|
|
3344
|
-
overrides = load_brownfield_overrides(project_root)
|
|
3345
|
-
try:
|
|
3346
|
-
prs = str(project_root.resolve())
|
|
3347
|
-
except OSError:
|
|
3348
|
-
prs = str(project_root)
|
|
3349
|
-
tables.cross_service_resolution = _load_config_cross_service_resolution(prs)
|
|
3350
|
-
mch = meta_chain
|
|
3351
|
-
|
|
3352
|
-
# Stage all Symbol rows
|
|
3353
|
-
rows: list[dict] = []
|
|
3354
|
-
# Node ids loaded from the existing graph as resolution-only stubs
|
|
3355
|
-
# (`_load_existing_types`); their staged rows carry placeholder values and
|
|
3356
|
-
# must never be written back over the real nodes.
|
|
3357
|
-
stub_ids: set[str] = set()
|
|
3358
|
-
|
|
3359
|
-
# packages
|
|
3360
|
-
for pkg, pid in tables.packages.items():
|
|
3361
|
-
rows.append(_node_row(
|
|
3362
|
-
id=pid, kind="package", name=pkg.rsplit(".", 1)[-1], fqn=pkg, package=pkg,
|
|
3363
|
-
))
|
|
3364
|
-
# files
|
|
3365
|
-
for path, fid in tables.files.items():
|
|
3366
|
-
rows.append(_node_row(
|
|
3367
|
-
id=fid, kind="file", name=Path(path).name, fqn=path, filename=path,
|
|
3368
|
-
))
|
|
3369
|
-
# types
|
|
3370
|
-
for entry in tables.types.values():
|
|
3371
|
-
d = entry.decl
|
|
3372
|
-
role, capabilities = resolve_role_and_capabilities(
|
|
3373
|
-
d,
|
|
3374
|
-
overrides=overrides,
|
|
3375
|
-
meta_chain=mch,
|
|
3376
|
-
)
|
|
3377
|
-
# Read generated/generated_by from pass-1 classification or stub persistence
|
|
3378
|
-
generated, generated_by = tables.type_generated_by_node_id.get(entry.node_id, (False, None))
|
|
3379
|
-
|
|
3380
|
-
if entry.loaded_from_db:
|
|
3381
|
-
stub_ids.add(entry.node_id)
|
|
3382
|
-
# Out-of-scope stub: its annotation-less decl collapses role to the
|
|
3383
|
-
# default. The real node's role was persisted at index time and seeded
|
|
3384
|
-
# into type_role_by_node_id by _load_existing_types; trust it so CALLS
|
|
3385
|
-
# edges into this type keep the correct callee_declaring_role (#352).
|
|
3386
|
-
# The staged row is filtered out of the write via stub_ids, so its
|
|
3387
|
-
# capabilities placeholder never reaches the graph.
|
|
3388
|
-
role = tables.type_role_by_node_id.get(entry.node_id, role)
|
|
3389
|
-
capabilities = []
|
|
3390
|
-
# For stubs, trust the persisted generated/generated_by (seeded by _load_existing_types)
|
|
3391
|
-
else:
|
|
3392
|
-
tables.type_role_by_node_id[entry.node_id] = role
|
|
3393
|
-
rows.append(_node_row(
|
|
3394
|
-
id=entry.node_id, kind=d.kind, name=d.name, fqn=d.fqn,
|
|
3395
|
-
package=entry.package,
|
|
3396
|
-
module=entry.module, microservice=entry.microservice,
|
|
3397
|
-
filename=entry.file_path,
|
|
3398
|
-
start_line=d.start_line, end_line=d.end_line,
|
|
3399
|
-
start_byte=d.start_byte, end_byte=d.end_byte,
|
|
3400
|
-
modifiers=list(d.modifiers),
|
|
3401
|
-
annotations=[a.name for a in d.annotations],
|
|
3402
|
-
capabilities=capabilities,
|
|
3403
|
-
role=role,
|
|
3404
|
-
signature="",
|
|
3405
|
-
parent_id=tables.types[entry.outer_fqn].node_id if entry.outer_fqn and entry.outer_fqn in tables.types else "",
|
|
3406
|
-
generated=generated,
|
|
3407
|
-
generated_by=generated_by,
|
|
3408
|
-
))
|
|
3409
|
-
# members (methods / constructors)
|
|
3410
|
-
for m in tables.members:
|
|
3411
|
-
rows.append(_node_row(
|
|
3412
|
-
id=m.node_id, kind=m.kind, name=m.decl.name,
|
|
3413
|
-
fqn=f"{m.parent_fqn}#{m.decl.signature}",
|
|
3414
|
-
package=tables.types[m.parent_fqn].package if m.parent_fqn in tables.types else "",
|
|
3415
|
-
module=m.module, microservice=m.microservice,
|
|
3416
|
-
filename=m.file_path,
|
|
3417
|
-
start_line=m.decl.start_line, end_line=m.decl.end_line,
|
|
3418
|
-
start_byte=m.decl.start_byte, end_byte=m.decl.end_byte,
|
|
3419
|
-
modifiers=list(m.decl.modifiers),
|
|
3420
|
-
annotations=[a.name for a in m.decl.annotations],
|
|
3421
|
-
signature=m.decl.signature, parent_id=m.parent_id,
|
|
3422
|
-
))
|
|
3423
|
-
# phantoms
|
|
3424
|
-
for pid, row in tables.phantoms.items():
|
|
3425
|
-
rows.append(row)
|
|
3426
|
-
|
|
3427
|
-
# Bulk-load new Symbol rows. The full-rebuild path starts from an empty
|
|
3428
|
-
# database (`_drop_all`), so every row is new. The incremental path reaches
|
|
3429
|
-
# here with a populated database: changed-file nodes were deleted by
|
|
3430
|
-
# `_delete_file_scope` (absent here → new), while dependent-file nodes are
|
|
3431
|
-
# deliberately preserved (see `_delete_file_scope` / issue #305).
|
|
3432
|
-
existing_ids = _existing_node_ids(conn)
|
|
3433
|
-
new_rows = [row for row in rows if row["id"] not in existing_ids]
|
|
3434
|
-
_bulk_copy(conn, "Symbol", _NODE_COLUMNS, new_rows)
|
|
3435
|
-
|
|
3436
|
-
# Refresh mutable properties on preserved dependent TYPE nodes (incremental
|
|
3437
|
-
# path only; `update_rows` is empty on the full path). `role`/`capabilities`
|
|
3438
|
-
# — and any other field derived from project-wide inputs (meta-annotation
|
|
3439
|
-
# chain, brownfield overrides) — can shift without the type's own source
|
|
3440
|
-
# changing, so a preserved dependent must be re-SET to stay byte-equivalent
|
|
3441
|
-
# with a full rebuild. The legacy per-row `_MERGE_SYMBOL` upserted every
|
|
3442
|
-
# staged node and did this implicitly; bulk `COPY FROM` only appends, so the
|
|
3443
|
-
# SET is explicit here. Stubs (`stub_ids`) are skipped: their decl is a
|
|
3444
|
-
# placeholder and their stored values are authoritative. Non-type kinds
|
|
3445
|
-
# carry no mutable role/capabilities, so they are skipped too.
|
|
3446
|
-
update_rows = [
|
|
3447
|
-
row for row in rows
|
|
3448
|
-
if row["id"] in existing_ids
|
|
3449
|
-
and row["id"] not in stub_ids
|
|
3450
|
-
and row["kind"] in _TYPE_KINDS
|
|
3451
|
-
]
|
|
3452
|
-
for row in update_rows:
|
|
3453
|
-
conn.execute(_SET_SYMBOL_BY_ID, row)
|
|
3454
|
-
|
|
3455
|
-
|
|
3456
|
-
def _write_nodes(
|
|
3457
|
-
conn: ladybug.Connection,
|
|
3458
|
-
tables: GraphTables,
|
|
3459
|
-
*,
|
|
3460
|
-
project_root: Path,
|
|
3461
|
-
meta_chain: dict[str, frozenset[str]] | None,
|
|
3462
|
-
) -> None:
|
|
3463
|
-
_write_nodes_impl(conn, tables, project_root=project_root, meta_chain=meta_chain)
|
|
3464
|
-
|
|
3465
|
-
|
|
3466
|
-
|
|
3467
|
-
|
|
3468
|
-
def _populate_declares_rows(tables: GraphTables) -> None:
|
|
3469
|
-
# Skip members loaded from the existing graph for cross-file resolution: a
|
|
3470
|
-
# DECLARES edge for an unchanged-file member already persists (its
|
|
3471
|
-
# source_file is out of scope, so `_delete_file_scope` left it), and
|
|
3472
|
-
# re-emitting it would append a duplicate (REL tables carry no primary key).
|
|
3473
|
-
# Full-rebuild never loads members, so this is a no-op there.
|
|
3474
|
-
tables.declares_rows = [
|
|
3475
|
-
DeclaresRow(src_id=m.parent_id, dst_id=m.node_id)
|
|
3476
|
-
for m in tables.members
|
|
3477
|
-
if not m.loaded_from_db
|
|
3478
|
-
]
|
|
3479
|
-
|
|
3480
|
-
|
|
3481
|
-
def _direct_supertype_ids(tables: GraphTables, type_id: str) -> list[str]:
|
|
3482
|
-
out: list[str] = []
|
|
3483
|
-
for r in tables.extends_rows:
|
|
3484
|
-
if r.src_id == type_id:
|
|
3485
|
-
out.append(r.dst_id)
|
|
3486
|
-
for r in tables.implements_rows:
|
|
3487
|
-
if r.src_id == type_id:
|
|
3488
|
-
out.append(r.dst_id)
|
|
3489
|
-
return out
|
|
3490
|
-
|
|
3491
|
-
|
|
3492
|
-
def _populate_overrides_rows(tables: GraphTables) -> None:
|
|
3493
|
-
"""Materialize (subtype_method)-[:OVERRIDES]->(supertype_method) for one supertype hop.
|
|
3494
|
-
|
|
3495
|
-
Matches ``LadybugDBGraph.override_axis_rollup_for`` (direct ``IMPLEMENTS`` / ``EXTENDS``
|
|
3496
|
-
only, same ``signature``, distinct method ids, non-static instance methods).
|
|
3497
|
-
"""
|
|
3498
|
-
by_declaring_type: dict[str, list[MemberEntry]] = defaultdict(list)
|
|
3499
|
-
for m in tables.members:
|
|
3500
|
-
by_declaring_type[m.parent_id].append(m)
|
|
3501
|
-
pairs: set[tuple[str, str]] = set()
|
|
3502
|
-
for m in tables.members:
|
|
3503
|
-
if m.kind != "method" or "static" in m.decl.modifiers:
|
|
3504
|
-
continue
|
|
3505
|
-
impl_tid = m.parent_id
|
|
3506
|
-
for sup_id in _direct_supertype_ids(tables, impl_tid):
|
|
3507
|
-
for other in by_declaring_type.get(sup_id, ()):
|
|
3508
|
-
if other.kind != "method":
|
|
3509
|
-
continue
|
|
3510
|
-
if other.decl.signature != m.decl.signature:
|
|
3511
|
-
continue
|
|
3512
|
-
if other.node_id == m.node_id:
|
|
3513
|
-
continue
|
|
3514
|
-
pairs.add((m.node_id, other.node_id))
|
|
3515
|
-
tables.overrides_rows = [
|
|
3516
|
-
DeclaresRow(src_id=a, dst_id=b) for a, b in sorted(pairs)
|
|
3517
|
-
]
|
|
3518
|
-
|
|
3519
|
-
|
|
3520
|
-
def _build_file_by_node_id(tables: GraphTables) -> dict[str, str]:
|
|
3521
|
-
"""Build node_id -> file_path lookup for source_file resolution."""
|
|
3522
|
-
lookup: dict[str, str] = {}
|
|
3523
|
-
for entry in tables.types.values():
|
|
3524
|
-
lookup[entry.node_id] = entry.file_path
|
|
3525
|
-
for m in tables.members:
|
|
3526
|
-
lookup[m.node_id] = m.file_path
|
|
3527
|
-
return lookup
|
|
3528
|
-
|
|
3529
|
-
|
|
3530
|
-
def _write_edges(conn: ladybug.Connection, tables: GraphTables, _file_by_node_id: dict[str, str] | None = None) -> None:
|
|
3531
|
-
# Build node_id -> file_path lookup for source_file resolution.
|
|
3532
|
-
if _file_by_node_id is None:
|
|
3533
|
-
_file_by_node_id = _build_file_by_node_id(tables)
|
|
3534
|
-
|
|
3535
|
-
# Bulk COPY FROM enforces referential integrity — a REL row whose endpoint
|
|
3536
|
-
# node isn't loaded raises "Unable to find primary key value". The legacy
|
|
3537
|
-
# per-row MERGE silently skipped such edges; drop them here to preserve the
|
|
3538
|
-
# per-row graph exactly. _existing_node_ids reads the live DB (not just
|
|
3539
|
-
# `tables`) so the incremental path's references to prior-run nodes still hold.
|
|
3540
|
-
valid_ids = _existing_node_ids(conn)
|
|
3541
|
-
|
|
3542
|
-
# Stage EXTENDS rows
|
|
3543
|
-
extends_rows = [
|
|
3544
|
-
{
|
|
3545
|
-
"FROM": r.src_id, "TO": r.dst_id,
|
|
3546
|
-
"source_file": _file_by_node_id.get(r.src_id, ""),
|
|
3547
|
-
"dst_name": r.dst_name, "dst_fqn": r.dst_fqn, "resolved": r.resolved,
|
|
3548
|
-
}
|
|
3549
|
-
for r in tables.extends_rows
|
|
3550
|
-
if r.src_id in valid_ids and r.dst_id in valid_ids
|
|
3551
|
-
]
|
|
3552
|
-
_bulk_copy(conn, "EXTENDS", _REL_EXTENDS_COLUMNS, extends_rows)
|
|
3553
|
-
|
|
3554
|
-
# Stage IMPLEMENTS rows
|
|
3555
|
-
implements_rows = [
|
|
3556
|
-
{
|
|
3557
|
-
"FROM": r.src_id, "TO": r.dst_id,
|
|
3558
|
-
"source_file": _file_by_node_id.get(r.src_id, ""),
|
|
3559
|
-
"dst_name": r.dst_name, "dst_fqn": r.dst_fqn, "resolved": r.resolved,
|
|
3560
|
-
}
|
|
3561
|
-
for r in tables.implements_rows
|
|
3562
|
-
if r.src_id in valid_ids and r.dst_id in valid_ids
|
|
3563
|
-
]
|
|
3564
|
-
_bulk_copy(conn, "IMPLEMENTS", _REL_IMPLEMENTS_COLUMNS, implements_rows)
|
|
3565
|
-
|
|
3566
|
-
# Stage INJECTS rows
|
|
3567
|
-
injects_rows = [
|
|
3568
|
-
{
|
|
3569
|
-
"FROM": r.src_id, "TO": r.dst_id,
|
|
3570
|
-
"source_file": _file_by_node_id.get(r.src_id, ""),
|
|
3571
|
-
"dst_name": r.dst_name, "dst_fqn": r.dst_fqn, "resolved": r.resolved,
|
|
3572
|
-
"mechanism": r.mechanism, "annotation": r.annotation,
|
|
3573
|
-
"field_or_param": r.field_or_param,
|
|
3574
|
-
}
|
|
3575
|
-
for r in tables.injects_rows
|
|
3576
|
-
if r.src_id in valid_ids and r.dst_id in valid_ids
|
|
3577
|
-
]
|
|
3578
|
-
_bulk_copy(conn, "INJECTS", _REL_INJECTS_COLUMNS, injects_rows)
|
|
3579
|
-
|
|
3580
|
-
# Stage DECLARES rows
|
|
3581
|
-
declares_rows = [
|
|
3582
|
-
{
|
|
3583
|
-
"FROM": row.src_id, "TO": row.dst_id,
|
|
3584
|
-
"source_file": _file_by_node_id.get(row.src_id, ""),
|
|
3585
|
-
}
|
|
3586
|
-
for row in tables.declares_rows
|
|
3587
|
-
if row.src_id in valid_ids and row.dst_id in valid_ids
|
|
3588
|
-
]
|
|
3589
|
-
_bulk_copy(conn, "DECLARES", _REL_DECLARES_COLUMNS, declares_rows)
|
|
3590
|
-
|
|
3591
|
-
# Stage OVERRIDES rows
|
|
3592
|
-
overrides_rows = [
|
|
3593
|
-
{
|
|
3594
|
-
"FROM": row.src_id, "TO": row.dst_id,
|
|
3595
|
-
"source_file": _file_by_node_id.get(row.src_id, ""),
|
|
3596
|
-
}
|
|
3597
|
-
for row in tables.overrides_rows
|
|
3598
|
-
if row.src_id in valid_ids and row.dst_id in valid_ids
|
|
3599
|
-
]
|
|
3600
|
-
_bulk_copy(conn, "OVERRIDES", _REL_OVERRIDES_COLUMNS, overrides_rows)
|
|
3601
|
-
|
|
3602
|
-
# Stage CALLS rows with dedup and callee_declaring_role materialization
|
|
3603
|
-
seen_calls: set[tuple[str, str, int, int, int]] = set()
|
|
3604
|
-
calls_rows: list[dict] = []
|
|
3605
|
-
member_by_id = {m.node_id: m for m in tables.members}
|
|
3606
|
-
for row in tables.calls_rows:
|
|
3607
|
-
if row.src_id not in valid_ids or row.dst_id not in valid_ids:
|
|
3608
|
-
continue
|
|
3609
|
-
# Include call_site_byte so two call sites of the same method on the same
|
|
3610
|
-
# source line (same arg_count) are kept as distinct edges (issue #359).
|
|
3611
|
-
key = (row.src_id, row.dst_id, row.arg_count, row.call_site_line, row.call_site_byte)
|
|
3612
|
-
if key in seen_calls:
|
|
3613
|
-
continue
|
|
3614
|
-
seen_calls.add(key)
|
|
3615
|
-
calls_rows.append({
|
|
3616
|
-
"FROM": row.src_id, "TO": row.dst_id,
|
|
3617
|
-
"source_file": _file_by_node_id.get(row.src_id, ""),
|
|
3618
|
-
"call_site_line": row.call_site_line, "call_site_byte": row.call_site_byte,
|
|
3619
|
-
"arg_count": row.arg_count, "confidence": row.confidence, "strategy": row.strategy,
|
|
3620
|
-
"source": row.source, "resolved": row.resolved,
|
|
3621
|
-
"callee_declaring_role": _callee_declaring_role_at_write(
|
|
3622
|
-
tables, row.dst_id, member_by_id=member_by_id,
|
|
3623
|
-
),
|
|
3624
|
-
})
|
|
3625
|
-
_bulk_copy(conn, "CALLS", _REL_CALLS_COLUMNS, calls_rows)
|
|
3626
|
-
|
|
3627
|
-
# Stage UnresolvedCallSite node rows (must load before UNRESOLVED_AT edges)
|
|
3628
|
-
seen_ucs: set[str] = set()
|
|
3629
|
-
ucs_rows: list[dict] = []
|
|
3630
|
-
for row in tables.unresolved_call_site_rows:
|
|
3631
|
-
if row.id in seen_ucs:
|
|
3632
|
-
continue
|
|
3633
|
-
seen_ucs.add(row.id)
|
|
3634
|
-
ucs_rows.append({
|
|
3635
|
-
"id": row.id,
|
|
3636
|
-
"caller_id": row.caller_id,
|
|
3637
|
-
"call_site_line": row.call_site_line,
|
|
3638
|
-
"call_site_byte": row.call_site_byte,
|
|
3639
|
-
"arg_count": row.arg_count,
|
|
3640
|
-
"callee_simple": row.callee_simple,
|
|
3641
|
-
"receiver_expr": row.receiver_expr,
|
|
3642
|
-
"reason": row.reason,
|
|
3643
|
-
})
|
|
3644
|
-
_bulk_copy(conn, "UnresolvedCallSite", _UNRESOLVED_CALL_SITE_COLUMNS, ucs_rows)
|
|
3645
|
-
|
|
3646
|
-
# Stage UNRESOLVED_AT edge rows (one per unique UnresolvedCallSite node)
|
|
3647
|
-
# Use the same ucs_rows list to ensure 1:1 correspondence
|
|
3648
|
-
unresolved_at_rows = [
|
|
3649
|
-
{
|
|
3650
|
-
"FROM": ucs_row["caller_id"], "TO": ucs_row["id"],
|
|
3651
|
-
"source_file": _file_by_node_id.get(ucs_row["caller_id"], ""),
|
|
3652
|
-
}
|
|
3653
|
-
for ucs_row in ucs_rows
|
|
3654
|
-
if ucs_row["caller_id"] in valid_ids
|
|
3655
|
-
]
|
|
3656
|
-
_bulk_copy(conn, "UNRESOLVED_AT", _REL_UNRESOLVED_AT_COLUMNS, unresolved_at_rows)
|
|
3657
|
-
|
|
3658
|
-
|
|
3659
|
-
def _write_routes_and_exposes(conn: ladybug.Connection, tables: GraphTables, _file_by_node_id: dict[str, str] | None = None) -> None:
|
|
3660
|
-
# Build node_id -> file_path lookup for source_file resolution (for Symbol sources).
|
|
3661
|
-
if _file_by_node_id is None:
|
|
3662
|
-
_file_by_node_id = _build_file_by_node_id(tables)
|
|
3663
|
-
|
|
3664
|
-
# Build client_id -> filename lookup for HTTP_CALLS source_file.
|
|
3665
|
-
_file_by_client_id: dict[str, str] = {row.id: row.filename for row in tables.client_rows}
|
|
3666
|
-
|
|
3667
|
-
# Build producer_id -> filename lookup for ASYNC_CALLS source_file.
|
|
3668
|
-
_file_by_producer_id: dict[str, str] = {row.id: row.filename for row in tables.producer_rows}
|
|
3669
|
-
|
|
3670
|
-
# Bulk COPY FROM enforces referential integrity — get all valid node IDs
|
|
3671
|
-
valid_ids = _existing_node_ids(conn)
|
|
3672
|
-
|
|
3673
|
-
# Stage Route node rows (bulk-load before edges that reference them)
|
|
3674
|
-
route_rows = [
|
|
3675
|
-
{
|
|
3676
|
-
"id": row.id,
|
|
3677
|
-
"kind": row.kind,
|
|
3678
|
-
"framework": row.framework,
|
|
3679
|
-
"method": row.method,
|
|
3680
|
-
"path": row.path,
|
|
3681
|
-
"path_template": row.path_template,
|
|
3682
|
-
"path_regex": row.path_regex,
|
|
3683
|
-
"topic": row.topic,
|
|
3684
|
-
"broker": row.broker,
|
|
3685
|
-
"feign_name": row.feign_name,
|
|
3686
|
-
"feign_url": row.feign_url,
|
|
3687
|
-
"microservice": row.microservice,
|
|
3688
|
-
"module": row.module,
|
|
3689
|
-
"filename": row.filename,
|
|
3690
|
-
"start_line": row.start_line,
|
|
3691
|
-
"end_line": row.end_line,
|
|
3692
|
-
"resolved": row.resolved,
|
|
3693
|
-
}
|
|
3694
|
-
for row in tables.routes_rows
|
|
3695
|
-
]
|
|
3696
|
-
_bulk_copy(conn, "Route", _ROUTE_COLUMNS, route_rows)
|
|
3697
|
-
|
|
3698
|
-
# Stage Client node rows (bulk-load before edges that reference them)
|
|
3699
|
-
client_rows = [asdict(row) for row in tables.client_rows]
|
|
3700
|
-
_bulk_copy(conn, "Client", _CLIENT_COLUMNS, client_rows)
|
|
3701
|
-
|
|
3702
|
-
# Stage Producer node rows (bulk-load before edges that reference them)
|
|
3703
|
-
producer_rows = [asdict(row) for row in tables.producer_rows]
|
|
3704
|
-
_bulk_copy(conn, "Producer", _PRODUCER_COLUMNS, producer_rows)
|
|
3705
|
-
|
|
3706
|
-
# Re-fetch valid IDs after loading Route/Client/Producer nodes (now includes them)
|
|
3707
|
-
valid_ids = _existing_node_ids(conn)
|
|
3708
|
-
|
|
3709
|
-
# Stage EXPOSES edge rows (Symbol -> Route)
|
|
3710
|
-
exposes_rows = [
|
|
3711
|
-
{
|
|
3712
|
-
"FROM": row.symbol_id,
|
|
3713
|
-
"TO": row.route_id,
|
|
3714
|
-
"source_file": _file_by_node_id.get(row.symbol_id, ""),
|
|
3715
|
-
"confidence": row.confidence,
|
|
3716
|
-
"strategy": row.strategy,
|
|
3717
|
-
}
|
|
3718
|
-
for row in tables.exposes_rows
|
|
3719
|
-
if row.symbol_id in valid_ids and row.route_id in valid_ids
|
|
3720
|
-
]
|
|
3721
|
-
_bulk_copy(conn, "EXPOSES", _REL_EXPOSES_COLUMNS, exposes_rows)
|
|
3722
|
-
|
|
3723
|
-
# Stage DECLARES_CLIENT edge rows (Symbol -> Client)
|
|
3724
|
-
declares_client_rows = [
|
|
3725
|
-
{
|
|
3726
|
-
"FROM": row.symbol_id,
|
|
3727
|
-
"TO": row.client_id,
|
|
3728
|
-
"source_file": _file_by_node_id.get(row.symbol_id, ""),
|
|
3729
|
-
"confidence": row.confidence,
|
|
3730
|
-
"strategy": row.strategy,
|
|
3731
|
-
}
|
|
3732
|
-
for row in tables.declares_client_rows
|
|
3733
|
-
if row.symbol_id in valid_ids and row.client_id in valid_ids
|
|
3734
|
-
]
|
|
3735
|
-
_bulk_copy(conn, "DECLARES_CLIENT", _REL_DECLARES_CLIENT_COLUMNS, declares_client_rows)
|
|
3736
|
-
|
|
3737
|
-
# Stage DECLARES_PRODUCER edge rows (Symbol -> Producer)
|
|
3738
|
-
declares_producer_rows = [
|
|
3739
|
-
{
|
|
3740
|
-
"FROM": row.symbol_id,
|
|
3741
|
-
"TO": row.producer_id,
|
|
3742
|
-
"source_file": _file_by_node_id.get(row.symbol_id, ""),
|
|
3743
|
-
"confidence": row.confidence,
|
|
3744
|
-
"strategy": row.strategy,
|
|
3745
|
-
}
|
|
3746
|
-
for row in tables.declares_producer_rows
|
|
3747
|
-
if row.symbol_id in valid_ids and row.producer_id in valid_ids
|
|
3748
|
-
]
|
|
3749
|
-
_bulk_copy(conn, "DECLARES_PRODUCER", _REL_DECLARES_PRODUCER_COLUMNS, declares_producer_rows)
|
|
3750
|
-
|
|
3751
|
-
# Stage HTTP_CALLS edge rows (Client -> Route)
|
|
3752
|
-
http_call_rows = [
|
|
3753
|
-
{
|
|
3754
|
-
"FROM": row.client_id,
|
|
3755
|
-
"TO": row.route_id,
|
|
3756
|
-
"source_file": _file_by_client_id.get(row.client_id, ""),
|
|
3757
|
-
"confidence": row.confidence,
|
|
3758
|
-
"strategy": row.strategy,
|
|
3759
|
-
"method_call": row.method_call,
|
|
3760
|
-
"raw_uri": row.raw_uri,
|
|
3761
|
-
"match": row.match,
|
|
3762
|
-
}
|
|
3763
|
-
for row in tables.http_call_rows
|
|
3764
|
-
if row.client_id in valid_ids and row.route_id in valid_ids
|
|
3765
|
-
]
|
|
3766
|
-
_bulk_copy(conn, "HTTP_CALLS", _REL_HTTP_CALLS_COLUMNS, http_call_rows)
|
|
3767
|
-
|
|
3768
|
-
# Stage ASYNC_CALLS edge rows (Producer -> Route)
|
|
3769
|
-
async_call_rows = [
|
|
3770
|
-
{
|
|
3771
|
-
"FROM": row.producer_id,
|
|
3772
|
-
"TO": row.route_id,
|
|
3773
|
-
"source_file": _file_by_producer_id.get(row.producer_id, ""),
|
|
3774
|
-
"confidence": row.confidence,
|
|
3775
|
-
"strategy": row.strategy,
|
|
3776
|
-
"direction": row.direction,
|
|
3777
|
-
"raw_topic": row.raw_topic,
|
|
3778
|
-
"match": row.match,
|
|
3779
|
-
}
|
|
3780
|
-
for row in tables.async_call_rows
|
|
3781
|
-
if row.producer_id in valid_ids and row.route_id in valid_ids
|
|
3782
|
-
]
|
|
3783
|
-
_bulk_copy(conn, "ASYNC_CALLS", _REL_ASYNC_CALLS_COLUMNS, async_call_rows)
|
|
3784
|
-
|
|
3785
|
-
|
|
3786
|
-
def _write_meta(conn: ladybug.Connection, tables: GraphTables, source_root: Path) -> None:
|
|
3787
|
-
# Dedup key MUST match _write_edges (build_ast_graph.py, _REL_CALLS writer): the
|
|
3788
|
-
# 5-tuple includes call_site_byte so two call sites of the same method on the
|
|
3789
|
-
# same source line are counted separately. A previous version used the 4-tuple
|
|
3790
|
-
# here, which made counts['calls'] (678) diverge from the real CALLS edge count
|
|
3791
|
-
# (684) that _write_edges actually persisted — describe/stats then undercounted.
|
|
3792
|
-
seen_calls: set[tuple[str, str, int, int, int]] = set()
|
|
3793
|
-
calls_unique = 0
|
|
3794
|
-
for row in tables.calls_rows:
|
|
3795
|
-
key = (row.src_id, row.dst_id, row.arg_count, row.call_site_line, row.call_site_byte)
|
|
3796
|
-
if key not in seen_calls:
|
|
3797
|
-
seen_calls.add(key)
|
|
3798
|
-
calls_unique += 1
|
|
3799
|
-
st = tables.route_stats
|
|
3800
|
-
routes_fw = dict(sorted(st.by_framework.items()))
|
|
3801
|
-
call_stats = tables.call_edge_stats
|
|
3802
|
-
client_stats = tables.client_stats
|
|
3803
|
-
producer_stats = tables.producer_stats
|
|
3804
|
-
http_by_strategy = dict(sorted(call_stats.http_calls_by_strategy.items()))
|
|
3805
|
-
async_by_strategy = dict(sorted(call_stats.async_calls_by_strategy.items()))
|
|
3806
|
-
http_match = dict(sorted(call_stats.http_calls_match_breakdown.items()))
|
|
3807
|
-
async_match = dict(sorted(call_stats.async_calls_match_breakdown.items()))
|
|
3808
|
-
http_resolved_pct = 0.0
|
|
3809
|
-
async_resolved_pct = 0.0
|
|
3810
|
-
if call_stats.http_calls_total:
|
|
3811
|
-
# PR-D1 definition: "resolved_pct" is strategy-based (strategy != 'unresolved'),
|
|
3812
|
-
# not match-based (all PR-D1 edges keep match='unresolved').
|
|
3813
|
-
resolved_http = sum(v for k, v in call_stats.http_calls_by_strategy.items() if k != "unresolved")
|
|
3814
|
-
http_resolved_pct = float(resolved_http) / float(call_stats.http_calls_total)
|
|
3815
|
-
if call_stats.async_calls_total:
|
|
3816
|
-
resolved_async = sum(v for k, v in call_stats.async_calls_by_strategy.items() if k != "unresolved")
|
|
3817
|
-
async_resolved_pct = float(resolved_async) / float(call_stats.async_calls_total)
|
|
3818
|
-
counts = {
|
|
3819
|
-
"packages": len(tables.packages),
|
|
3820
|
-
"files": len(tables.files),
|
|
3821
|
-
"types": len(tables.types),
|
|
3822
|
-
"members": len(tables.members),
|
|
3823
|
-
"phantoms": len(tables.phantoms),
|
|
3824
|
-
"extends": len(tables.extends_rows),
|
|
3825
|
-
"implements": len(tables.implements_rows),
|
|
3826
|
-
"injects": len(tables.injects_rows),
|
|
3827
|
-
"declares": len(tables.declares_rows),
|
|
3828
|
-
"overrides": len(tables.overrides_rows),
|
|
3829
|
-
"calls": calls_unique,
|
|
3830
|
-
"routes": len(tables.routes_rows),
|
|
3831
|
-
"exposes": len(tables.exposes_rows),
|
|
3832
|
-
"clients": len(tables.client_rows),
|
|
3833
|
-
"declares_client": len(tables.declares_client_rows),
|
|
3834
|
-
"producers": len(tables.producer_rows),
|
|
3835
|
-
"declares_producer": len(tables.declares_producer_rows),
|
|
3836
|
-
"http_calls": len(tables.http_call_rows),
|
|
3837
|
-
"async_calls": len(tables.async_call_rows),
|
|
3838
|
-
}
|
|
3839
|
-
routes_layer = dict(sorted(st.routes_by_layer.items()))
|
|
3840
|
-
clients_by_kind = dict(sorted(client_stats.clients_by_kind.items()))
|
|
3841
|
-
producers_by_kind = dict(sorted(producer_stats.producers_by_kind.items()))
|
|
3842
|
-
conn.execute(
|
|
3843
|
-
"MERGE (m:GraphMeta {key: $k}) "
|
|
3844
|
-
"SET m.ontology_version = $ov, m.built_at = $t, "
|
|
3845
|
-
"m.source_root = $sr, m.counts_json = $cj, m.parse_errors = $pe, "
|
|
3846
|
-
"m.routes_total = $routes_total, m.exposes_total = $exposes_total, "
|
|
3847
|
-
"m.routes_by_framework = $routes_by_framework, m.routes_resolved_pct = $routes_resolved_pct, "
|
|
3848
|
-
"m.routes_from_brownfield_pct = $routes_from_brownfield_pct, m.routes_by_layer = $routes_by_layer, "
|
|
3849
|
-
"m.clients_total = $clients_total, m.declares_client_total = $declares_client_total, "
|
|
3850
|
-
"m.clients_by_kind = $clients_by_kind, "
|
|
3851
|
-
"m.producers_total = $producers_total, m.declares_producer_total = $declares_producer_total, "
|
|
3852
|
-
"m.producers_by_kind = $producers_by_kind, "
|
|
3853
|
-
"m.http_calls_total = $http_calls_total, m.async_calls_total = $async_calls_total, "
|
|
3854
|
-
"m.http_calls_by_strategy = $http_calls_by_strategy, m.async_calls_by_strategy = $async_calls_by_strategy, "
|
|
3855
|
-
"m.http_calls_resolved_pct = $http_calls_resolved_pct, m.async_calls_resolved_pct = $async_calls_resolved_pct, "
|
|
3856
|
-
"m.http_clients_from_brownfield_pct = $http_clients_from_brownfield_pct, "
|
|
3857
|
-
"m.async_producers_from_brownfield_pct = $async_producers_from_brownfield_pct, "
|
|
3858
|
-
"m.http_calls_match_breakdown = $http_calls_match_breakdown, "
|
|
3859
|
-
"m.async_calls_match_breakdown = $async_calls_match_breakdown, "
|
|
3860
|
-
"m.cross_service_calls_total = $cross_service_calls_total, "
|
|
3861
|
-
"m.pass3_skipped_cross_service = $pass3_skipped_cross_service, "
|
|
3862
|
-
"m.pass3_unresolved_phantom_receiver = $pass3_unresolved_phantom_receiver, "
|
|
3863
|
-
"m.pass3_unresolved_chained = $pass3_unresolved_chained, "
|
|
3864
|
-
"m.pass4_exposes_suppressed_feign = $pass4_exposes_suppressed_feign, "
|
|
3865
|
-
"m.cross_service_resolution = $cross_service_resolution",
|
|
3866
|
-
{
|
|
3867
|
-
"k": "graph",
|
|
3868
|
-
"ov": ONTOLOGY_VERSION,
|
|
3869
|
-
"t": int(time.time()),
|
|
3870
|
-
"sr": str(source_root.resolve()),
|
|
3871
|
-
"cj": json.dumps(counts),
|
|
3872
|
-
"pe": tables.parse_errors,
|
|
3873
|
-
"routes_total": len(tables.routes_rows),
|
|
3874
|
-
"exposes_total": len(tables.exposes_rows),
|
|
3875
|
-
"routes_by_framework": json.dumps(routes_fw),
|
|
3876
|
-
"routes_resolved_pct": float(st.routes_resolved_pct),
|
|
3877
|
-
"routes_from_brownfield_pct": float(st.routes_from_brownfield_pct),
|
|
3878
|
-
"routes_by_layer": json.dumps(routes_layer),
|
|
3879
|
-
"clients_total": int(client_stats.clients_total),
|
|
3880
|
-
"declares_client_total": int(client_stats.declares_client_total),
|
|
3881
|
-
"clients_by_kind": json.dumps(clients_by_kind),
|
|
3882
|
-
"producers_total": int(producer_stats.producers_total),
|
|
3883
|
-
"declares_producer_total": int(producer_stats.declares_producer_total),
|
|
3884
|
-
"producers_by_kind": json.dumps(producers_by_kind),
|
|
3885
|
-
"http_calls_total": call_stats.http_calls_total,
|
|
3886
|
-
"async_calls_total": call_stats.async_calls_total,
|
|
3887
|
-
"http_calls_by_strategy": json.dumps(http_by_strategy),
|
|
3888
|
-
"async_calls_by_strategy": json.dumps(async_by_strategy),
|
|
3889
|
-
"http_calls_resolved_pct": http_resolved_pct,
|
|
3890
|
-
"async_calls_resolved_pct": async_resolved_pct,
|
|
3891
|
-
"http_clients_from_brownfield_pct": call_stats.http_clients_from_brownfield_pct,
|
|
3892
|
-
"async_producers_from_brownfield_pct": call_stats.async_producers_from_brownfield_pct,
|
|
3893
|
-
"http_calls_match_breakdown": json.dumps(http_match),
|
|
3894
|
-
"async_calls_match_breakdown": json.dumps(async_match),
|
|
3895
|
-
"cross_service_calls_total": int(call_stats.cross_service_calls_total),
|
|
3896
|
-
"pass3_skipped_cross_service": int(tables.pass3_skipped_cross_service),
|
|
3897
|
-
"pass3_unresolved_phantom_receiver": int(tables.pass3_unresolved_phantom_receiver),
|
|
3898
|
-
"pass3_unresolved_chained": int(tables.pass3_unresolved_chained),
|
|
3899
|
-
"pass4_exposes_suppressed_feign": int(st.exposes_suppressed_feign),
|
|
3900
|
-
"cross_service_resolution": str(tables.cross_service_resolution),
|
|
3901
|
-
},
|
|
3902
|
-
)
|
|
3903
|
-
|
|
3904
|
-
|
|
3905
|
-
def incremental_rebuild(
|
|
3906
|
-
source_root: Path,
|
|
3907
|
-
ladybug_path: Path,
|
|
3908
|
-
*,
|
|
3909
|
-
verbose: bool,
|
|
3910
|
-
expansion_cap: int = 50,
|
|
3911
|
-
) -> IncrementalResult:
|
|
3912
|
-
"""Incrementally rebuild the LadybugDB graph, processing only changed files and their dependents.
|
|
3913
|
-
|
|
3914
|
-
Returns IncrementalResult with statistics about the rebuild.
|
|
3915
|
-
Falls back to full rebuild if:
|
|
3916
|
-
- No previous graph exists
|
|
3917
|
-
- Ontology version < ONTOLOGY_VERSION (stale schema; rebuild for current columns)
|
|
3918
|
-
- Crash marker exists (previous incremental run failed)
|
|
3919
|
-
- Dependent expansion exceeds expansion_cap
|
|
3920
|
-
"""
|
|
3921
|
-
t_start = time.time()
|
|
3922
|
-
|
|
3923
|
-
# Step 1: Load existing graph and detect changes
|
|
3924
|
-
if not ladybug_path.exists():
|
|
3925
|
-
if verbose:
|
|
3926
|
-
_verbose_stderr_line("[increment] no existing graph; falling back to full rebuild")
|
|
3927
|
-
# Fall back to full rebuild
|
|
3928
|
-
tables = GraphTables()
|
|
3929
|
-
asts = pass1_parse(source_root, tables, verbose=verbose)
|
|
3930
|
-
pass2_edges(tables, asts, verbose=verbose)
|
|
3931
|
-
pass3_calls(tables, asts, verbose=verbose)
|
|
3932
|
-
pass4_routes(tables, asts, source_root=source_root, verbose=verbose)
|
|
3933
|
-
pass5_imperative_edges(tables, asts, source_root=source_root, verbose=verbose)
|
|
3934
|
-
pass6_match_edges(tables, verbose=verbose)
|
|
3935
|
-
write_ladybug(ladybug_path, tables, source_root=source_root, verbose=verbose)
|
|
3936
|
-
|
|
3937
|
-
return IncrementalResult(
|
|
3938
|
-
mode="full_fallback",
|
|
3939
|
-
files_changed=0,
|
|
3940
|
-
files_added=0,
|
|
3941
|
-
files_removed=0,
|
|
3942
|
-
dependents_reprocessed=0,
|
|
3943
|
-
elapsed_sec=time.time() - t_start,
|
|
3944
|
-
)
|
|
3945
|
-
|
|
3946
|
-
db = ladybug.Database(str(ladybug_path))
|
|
3947
|
-
conn = ladybug.Connection(db)
|
|
3948
|
-
# If a Symbol FTS index exists, the scoped DELETE/MERGE/COPY below must maintain it,
|
|
3949
|
-
# which needs the FTS extension loaded on THIS connection. Best-effort: when FTS is
|
|
3950
|
-
# unavailable no index exists, so there's nothing to maintain and DML proceeds fine.
|
|
3951
|
-
try:
|
|
3952
|
-
conn.execute("LOAD EXTENSION FTS")
|
|
3953
|
-
except Exception:
|
|
3954
|
-
pass
|
|
3955
|
-
|
|
3956
|
-
# Check ontology version
|
|
3957
|
-
try:
|
|
3958
|
-
meta_result = conn.execute("MATCH (m:GraphMeta) RETURN m.ontology_version AS version")
|
|
3959
|
-
if meta_result.has_next():
|
|
3960
|
-
row = meta_result.get_next()
|
|
3961
|
-
version = row[0] if row else 0
|
|
3962
|
-
if version < ONTOLOGY_VERSION:
|
|
3963
|
-
if verbose:
|
|
3964
|
-
_verbose_stderr_line(f"[increment] ontology version {version} < {ONTOLOGY_VERSION}; falling back to full rebuild")
|
|
3965
|
-
conn.close()
|
|
3966
|
-
db.close()
|
|
3967
|
-
del conn, db
|
|
3968
|
-
return _fallback_to_full(source_root, ladybug_path, verbose, t_start)
|
|
3969
|
-
except Exception as e:
|
|
3970
|
-
if verbose:
|
|
3971
|
-
_verbose_stderr_line(f"[increment] failed to read ontology version: {e}; falling back to full rebuild")
|
|
3972
|
-
try:
|
|
3973
|
-
conn.close()
|
|
3974
|
-
db.close()
|
|
3975
|
-
except Exception:
|
|
3976
|
-
pass
|
|
3977
|
-
del conn, db
|
|
3978
|
-
return _fallback_to_full(source_root, ladybug_path, verbose, t_start)
|
|
3979
|
-
|
|
3980
|
-
index_dir = ladybug_path.parent
|
|
3981
|
-
tracker = FileHashTracker(index_dir)
|
|
3982
|
-
tracker.load()
|
|
3983
|
-
|
|
3984
|
-
ignore = LayeredIgnore(source_root)
|
|
3985
|
-
added, changed, removed = tracker.detect_changes(source_root, ignore=ignore)
|
|
3986
|
-
|
|
3987
|
-
changed_files = added | changed | removed
|
|
3988
|
-
|
|
3989
|
-
if not changed_files:
|
|
3990
|
-
if verbose:
|
|
3991
|
-
_verbose_stderr_line("[increment] no changes detected; no-op")
|
|
3992
|
-
conn.close()
|
|
3993
|
-
db.close()
|
|
3994
|
-
return IncrementalResult(
|
|
3995
|
-
mode="incremental",
|
|
3996
|
-
files_changed=0,
|
|
3997
|
-
files_added=0,
|
|
3998
|
-
files_removed=0,
|
|
3999
|
-
dependents_reprocessed=0,
|
|
4000
|
-
elapsed_sec=time.time() - t_start,
|
|
4001
|
-
)
|
|
4002
|
-
|
|
4003
|
-
if verbose:
|
|
4004
|
-
_verbose_stderr_line(f"[increment] detected {len(added)} added, {len(changed)} changed, {len(removed)} removed files")
|
|
4005
|
-
|
|
4006
|
-
# Step 2: Crash marker check
|
|
4007
|
-
crash_marker_path = index_dir / GRAPH_INCREMENT_MARKER_FILENAME
|
|
4008
|
-
if crash_marker_path.exists():
|
|
4009
|
-
if verbose:
|
|
4010
|
-
_verbose_stderr_line("[increment] crash marker exists; falling back to full rebuild")
|
|
4011
|
-
conn.close()
|
|
4012
|
-
db.close()
|
|
4013
|
-
crash_marker_path.unlink(missing_ok=True)
|
|
4014
|
-
return _fallback_to_full(source_root, ladybug_path, verbose, t_start)
|
|
4015
|
-
|
|
4016
|
-
# Write crash marker
|
|
4017
|
-
crash_marker_path.write_text("", encoding="utf-8")
|
|
4018
|
-
|
|
4019
|
-
try:
|
|
4020
|
-
# Step 3: Dependent expansion
|
|
4021
|
-
# Collect node IDs for changed files (single query instead of N+1)
|
|
4022
|
-
changed_node_ids: set[str] = set()
|
|
4023
|
-
result = conn.execute(
|
|
4024
|
-
"MATCH (s:Symbol) WHERE s.filename IN $filenames RETURN s.id",
|
|
4025
|
-
{"filenames": list(changed_files)},
|
|
4026
|
-
)
|
|
4027
|
-
while result.has_next():
|
|
4028
|
-
row = result.get_next()
|
|
4029
|
-
changed_node_ids.add(row[0])
|
|
4030
|
-
|
|
4031
|
-
# Find dependents
|
|
4032
|
-
dependent_files = _find_dependents(conn, changed_node_ids)
|
|
4033
|
-
|
|
4034
|
-
# Annotation-definition change: also pull in files that USE the changed
|
|
4035
|
-
# annotation. Annotation usage is a node property, not a Symbol->Symbol
|
|
4036
|
-
# edge, so `_find_dependents` misses them and their role (derived from
|
|
4037
|
-
# the project-wide meta-chain) would go stale. See PR-P5b.
|
|
4038
|
-
dependent_files |= _find_annotation_dependents(conn, changed_node_ids)
|
|
4039
|
-
|
|
4040
|
-
# Union changed files with dependents
|
|
4041
|
-
scope_files = changed_files | dependent_files
|
|
4042
|
-
|
|
4043
|
-
if len(scope_files) > expansion_cap:
|
|
4044
|
-
if verbose:
|
|
4045
|
-
_verbose_stderr_line(f"[increment] dependent expansion cap ({expansion_cap}) exceeded ({len(scope_files)} files); falling back to full rebuild")
|
|
4046
|
-
conn.close()
|
|
4047
|
-
db.close()
|
|
4048
|
-
crash_marker_path.unlink(missing_ok=True)
|
|
4049
|
-
return _fallback_to_full(source_root, ladybug_path, verbose, t_start)
|
|
4050
|
-
|
|
4051
|
-
if verbose:
|
|
4052
|
-
_verbose_stderr_line(f"[increment] processing {len(scope_files)} files ({len(changed_files)} changed + {len(dependent_files)} dependents)")
|
|
4053
|
-
|
|
4054
|
-
# Step 4: Scoped deletion
|
|
4055
|
-
if verbose:
|
|
4056
|
-
_verbose_stderr_line("[increment] deleting outdated nodes and edges")
|
|
4057
|
-
_delete_file_scope(conn, changed_files, dependent_files)
|
|
4058
|
-
|
|
4059
|
-
# Force deletion to be applied by running a dummy query
|
|
4060
|
-
conn.execute("MATCH (s:Symbol) RETURN count(*)")
|
|
4061
|
-
|
|
4062
|
-
# Step 5: Scoped pass 1-4
|
|
4063
|
-
if verbose:
|
|
4064
|
-
_verbose_stderr_line("[increment] rebuilding scoped files (passes 1-4)")
|
|
4065
|
-
|
|
4066
|
-
tables = GraphTables()
|
|
4067
|
-
asts = pass1_parse(
|
|
4068
|
-
source_root, tables, verbose=verbose, scope_files=scope_files, removed_files=removed
|
|
4069
|
-
)
|
|
4070
|
-
|
|
4071
|
-
# Load existing types and members for cross-file resolution (only from unchanged files)
|
|
4072
|
-
_load_existing_types(conn, tables, exclude_files=scope_files)
|
|
4073
|
-
_load_existing_members(conn, tables, exclude_files=scope_files)
|
|
4074
|
-
|
|
4075
|
-
pass2_edges(tables, asts, verbose=verbose)
|
|
4076
|
-
pass3_calls(tables, asts, verbose=verbose)
|
|
4077
|
-
pass4_routes(tables, asts, source_root=source_root, verbose=verbose)
|
|
4078
|
-
|
|
4079
|
-
# Populate declares and overrides rows
|
|
4080
|
-
_populate_declares_rows(tables)
|
|
4081
|
-
_populate_overrides_rows(tables)
|
|
4082
|
-
|
|
4083
|
-
# Write scoped nodes and edges
|
|
4084
|
-
meta_chain = collect_annotation_meta_chain(str(source_root.resolve()))
|
|
4085
|
-
_scoped_write(conn, tables, project_root=source_root, meta_chain=meta_chain)
|
|
4086
|
-
|
|
4087
|
-
# Step 6: Global pass 5-6
|
|
4088
|
-
if verbose:
|
|
4089
|
-
_verbose_stderr_line("[increment] running global passes 5-6")
|
|
4090
|
-
|
|
4091
|
-
# Rebuild full tables for global pass 5-6 (pass1 populates members from scratch)
|
|
4092
|
-
tables_for_global = GraphTables()
|
|
4093
|
-
global_asts = pass1_parse(source_root, tables_for_global, verbose=verbose)
|
|
4094
|
-
# pass4 (routes/EXPOSES) must run on the global pass5/6 tables too (issue
|
|
4095
|
-
# #352): pass5 links Feign HTTP_CALLS to routes via exposes_rows, and pass6
|
|
4096
|
-
# matches against routes_rows. Without pass4 both stay empty and the
|
|
4097
|
-
# HTTP_CALLS match outcome drifts from a full rebuild. Mirrors main().
|
|
4098
|
-
pass4_routes(tables_for_global, global_asts, source_root=source_root, verbose=verbose)
|
|
4099
|
-
|
|
4100
|
-
pass5_imperative_edges(tables_for_global, global_asts, source_root=source_root, verbose=verbose)
|
|
4101
|
-
|
|
4102
|
-
# Delete existing Client, Producer, and their edges
|
|
4103
|
-
conn.execute("MATCH (c:Client) DETACH DELETE c")
|
|
4104
|
-
conn.execute("MATCH (p:Producer) DETACH DELETE p")
|
|
4105
|
-
|
|
4106
|
-
pass6_match_edges(tables_for_global, verbose=verbose)
|
|
4107
|
-
|
|
4108
|
-
# Write Client, Producer, and cross-service edges
|
|
4109
|
-
_write_clients_producers_and_calls(conn, tables_for_global)
|
|
4110
|
-
|
|
4111
|
-
# Step 7: Update hash store and metadata
|
|
4112
|
-
if verbose:
|
|
4113
|
-
_verbose_stderr_line("[increment] updating hash store and metadata")
|
|
4114
|
-
|
|
4115
|
-
# Update hashes for processed files
|
|
4116
|
-
tracker.update(scope_files, source_root)
|
|
4117
|
-
|
|
4118
|
-
# Remove hashes for deleted files
|
|
4119
|
-
for filename in removed:
|
|
4120
|
-
if filename in tracker._hashes:
|
|
4121
|
-
del tracker._hashes[filename]
|
|
4122
|
-
|
|
4123
|
-
tracker.save()
|
|
4124
|
-
|
|
4125
|
-
# Update GraphMeta
|
|
4126
|
-
_write_meta(conn, tables_for_global, source_root)
|
|
4127
|
-
_ensure_symbol_fts_index(conn, verbose=verbose)
|
|
4128
|
-
|
|
4129
|
-
# Remove crash marker
|
|
4130
|
-
crash_marker_path.unlink(missing_ok=True)
|
|
4131
|
-
|
|
4132
|
-
conn.close()
|
|
4133
|
-
db.close()
|
|
4134
|
-
|
|
4135
|
-
elapsed = time.time() - t_start
|
|
4136
|
-
if verbose:
|
|
4137
|
-
_verbose_stderr_line(f"[increment] completed in {elapsed:.2f}s")
|
|
4138
|
-
|
|
4139
|
-
return IncrementalResult(
|
|
4140
|
-
mode="incremental",
|
|
4141
|
-
files_changed=len(changed),
|
|
4142
|
-
files_added=len(added),
|
|
4143
|
-
files_removed=len(removed),
|
|
4144
|
-
dependents_reprocessed=len(dependent_files),
|
|
4145
|
-
elapsed_sec=elapsed,
|
|
4146
|
-
)
|
|
4147
|
-
|
|
4148
|
-
except Exception as e:
|
|
4149
|
-
# On error, remove crash marker and fall back to full rebuild
|
|
4150
|
-
if verbose:
|
|
4151
|
-
_verbose_stderr_line(f"[increment] error during incremental rebuild: {e}; falling back to full rebuild")
|
|
4152
|
-
conn.close()
|
|
4153
|
-
db.close()
|
|
4154
|
-
crash_marker_path.unlink(missing_ok=True)
|
|
4155
|
-
return _fallback_to_full(source_root, ladybug_path, verbose, t_start)
|
|
4156
|
-
|
|
4157
|
-
|
|
4158
|
-
def _init_hash_tracker(source_root: Path, ladybug_path: Path) -> int:
|
|
4159
|
-
"""Initialize hash tracker for all Java files. Returns number of files hashed.
|
|
4160
|
-
|
|
4161
|
-
Called right after a full graph rebuild (``write_ladybug``), so the store must
|
|
4162
|
-
mirror exactly the files that were just indexed. We deliberately do NOT
|
|
4163
|
-
``load()`` the existing store: ``update`` re-hashes every current file anyway,
|
|
4164
|
-
and preserving old entries would leave stale hashes for files that no longer
|
|
4165
|
-
exist (deleted or now-ignored). Those ghosts would be re-detected as "removed"
|
|
4166
|
-
on every subsequent ``increment``, sustaining an endless full-rebuild loop.
|
|
4167
|
-
"""
|
|
4168
|
-
index_dir = ladybug_path.parent
|
|
4169
|
-
tracker = FileHashTracker(index_dir)
|
|
4170
|
-
ignore = LayeredIgnore(source_root)
|
|
4171
|
-
all_files: set[str] = set()
|
|
4172
|
-
source_root_resolved = source_root.resolve()
|
|
4173
|
-
for p in iter_java_source_files(source_root, ignore=ignore):
|
|
4174
|
-
p_resolved = p.resolve()
|
|
4175
|
-
try:
|
|
4176
|
-
rel_path = p_resolved.relative_to(source_root_resolved).as_posix()
|
|
4177
|
-
except ValueError:
|
|
4178
|
-
rel_path = p.as_posix()
|
|
4179
|
-
all_files.add(rel_path)
|
|
4180
|
-
tracker.update(all_files, source_root)
|
|
4181
|
-
tracker.save()
|
|
4182
|
-
return len(all_files)
|
|
4183
|
-
|
|
4184
|
-
|
|
4185
|
-
def _fallback_to_full(source_root: Path, ladybug_path: Path, verbose: bool, t_start: float) -> IncrementalResult:
|
|
4186
|
-
"""Fallback to full rebuild."""
|
|
4187
|
-
tables = GraphTables()
|
|
4188
|
-
asts = pass1_parse(source_root, tables, verbose=verbose)
|
|
4189
|
-
pass2_edges(tables, asts, verbose=verbose)
|
|
4190
|
-
pass3_calls(tables, asts, verbose=verbose)
|
|
4191
|
-
pass4_routes(tables, asts, source_root=source_root, verbose=verbose)
|
|
4192
|
-
pass5_imperative_edges(tables, asts, source_root=source_root, verbose=verbose)
|
|
4193
|
-
pass6_match_edges(tables, verbose=verbose)
|
|
4194
|
-
write_ladybug(ladybug_path, tables, source_root=source_root, verbose=verbose)
|
|
4195
|
-
|
|
4196
|
-
return IncrementalResult(
|
|
4197
|
-
mode="full_fallback",
|
|
4198
|
-
files_changed=0,
|
|
4199
|
-
files_added=0,
|
|
4200
|
-
files_removed=0,
|
|
4201
|
-
dependents_reprocessed=0,
|
|
4202
|
-
elapsed_sec=time.time() - t_start,
|
|
4203
|
-
)
|
|
4204
|
-
|
|
4205
|
-
|
|
4206
|
-
def _write_clients_producers_and_calls(conn: ladybug.Connection, tables: GraphTables) -> None:
|
|
4207
|
-
"""Write Route, Client, Producer, and cross-service edges to LadybugDB.
|
|
4208
|
-
|
|
4209
|
-
Used by the incremental rebuild's global pass 5-6 step. Writes phantom
|
|
4210
|
-
Route nodes (created by pass5 for cross-service calls) that wouldn't
|
|
4211
|
-
otherwise exist in LadybugDB.
|
|
4212
|
-
"""
|
|
4213
|
-
# Bulk-write routes, mirroring `_write_nodes_impl`: COPY the rows that are new
|
|
4214
|
-
# to the DB, and SET every mutable field on the routes already present (the
|
|
4215
|
-
# caller does NOT delete routes, only Clients/Producers — see the global
|
|
4216
|
-
# pass 5-6 block). `tables.routes_rows` is the full route set (pass4 routes +
|
|
4217
|
-
# pass5 phantom routes), not just phantoms, so the SET keeps existing routes'
|
|
4218
|
-
# properties in sync with pass5's cross-service enrichment while the COPY
|
|
4219
|
-
# materializes phantoms that have no node yet. Replaces the last per-row
|
|
4220
|
-
# graph write (MERGE upsert).
|
|
4221
|
-
route_rows = [asdict(row) for row in tables.routes_rows]
|
|
4222
|
-
existing_route_ids = _existing_node_ids(conn)
|
|
4223
|
-
new_route_rows = [r for r in route_rows if r["id"] not in existing_route_ids]
|
|
4224
|
-
_bulk_copy(conn, "Route", _ROUTE_COLUMNS, new_route_rows)
|
|
4225
|
-
for r in route_rows:
|
|
4226
|
-
if r["id"] in existing_route_ids:
|
|
4227
|
-
conn.execute(_SET_ROUTE_BY_ID, r)
|
|
4228
|
-
|
|
4229
|
-
# Build node_id lookup for members and types
|
|
4230
|
-
member_by_id = {m.node_id: m for m in tables.members}
|
|
4231
|
-
|
|
4232
|
-
# Build client_id and producer_id lookups for source_file resolution
|
|
4233
|
-
client_by_id = {c.id: c for c in tables.client_rows}
|
|
4234
|
-
producer_by_id = {p.id: p for p in tables.producer_rows}
|
|
4235
|
-
|
|
4236
|
-
# Stage Client + Producer NODE rows unconditionally. The caller DETACH-DELETEs
|
|
4237
|
-
# every Client/Producer node immediately before calling this (see the global
|
|
4238
|
-
# pass 5-6 block), so none are in the DB yet and COPY-ing them fresh is safe.
|
|
4239
|
-
# Do NOT filter node rows against the existing-id set — that is the EDGE filter
|
|
4240
|
-
# pattern mis-applied to nodes, and would drop every node being created (the
|
|
4241
|
-
# caller's delete makes the pre-load set empty by construction).
|
|
4242
|
-
client_rows = [asdict(row) for row in tables.client_rows]
|
|
4243
|
-
_bulk_copy(conn, "Client", _CLIENT_COLUMNS, client_rows)
|
|
4244
|
-
producer_rows = [asdict(row) for row in tables.producer_rows]
|
|
4245
|
-
_bulk_copy(conn, "Producer", _PRODUCER_COLUMNS, producer_rows)
|
|
4246
|
-
|
|
4247
|
-
# Endpoint filtering applies to EDGES only: re-fetch the live node set (now
|
|
4248
|
-
# includes the Clients/Producers just loaded) and drop edge rows whose
|
|
4249
|
-
# endpoints still aren't materialized — reproduces per-row MERGE silent-drop.
|
|
4250
|
-
valid_ids = _existing_node_ids(conn)
|
|
4251
|
-
|
|
4252
|
-
# Stage DECLARES_CLIENT edge rows (Symbol -> Client)
|
|
4253
|
-
# Build source_file lookup from member_by_id
|
|
4254
|
-
declares_client_rows = [
|
|
4255
|
-
{
|
|
4256
|
-
"FROM": row.symbol_id,
|
|
4257
|
-
"TO": row.client_id,
|
|
4258
|
-
"source_file": member_by_id.get(row.symbol_id, MemberEntry(kind="", decl=None, parent_id="", parent_fqn="", file_path="", module="", microservice="", node_id="")).file_path,
|
|
4259
|
-
"confidence": row.confidence,
|
|
4260
|
-
"strategy": row.strategy,
|
|
4261
|
-
}
|
|
4262
|
-
for row in tables.declares_client_rows
|
|
4263
|
-
if row.symbol_id in valid_ids and row.client_id in valid_ids
|
|
4264
|
-
]
|
|
4265
|
-
_bulk_copy(conn, "DECLARES_CLIENT", _REL_DECLARES_CLIENT_COLUMNS, declares_client_rows)
|
|
4266
|
-
|
|
4267
|
-
# Stage DECLARES_PRODUCER edge rows (Symbol -> Producer)
|
|
4268
|
-
declares_producer_rows = [
|
|
4269
|
-
{
|
|
4270
|
-
"FROM": row.symbol_id,
|
|
4271
|
-
"TO": row.producer_id,
|
|
4272
|
-
"source_file": member_by_id.get(row.symbol_id, MemberEntry(kind="", decl=None, parent_id="", parent_fqn="", file_path="", module="", microservice="", node_id="")).file_path,
|
|
4273
|
-
"confidence": row.confidence,
|
|
4274
|
-
"strategy": row.strategy,
|
|
4275
|
-
}
|
|
4276
|
-
for row in tables.declares_producer_rows
|
|
4277
|
-
if row.symbol_id in valid_ids and row.producer_id in valid_ids
|
|
4278
|
-
]
|
|
4279
|
-
_bulk_copy(conn, "DECLARES_PRODUCER", _REL_DECLARES_PRODUCER_COLUMNS, declares_producer_rows)
|
|
4280
|
-
|
|
4281
|
-
# Stage HTTP_CALLS edge rows (Client -> Route)
|
|
4282
|
-
http_call_rows = [
|
|
4283
|
-
{
|
|
4284
|
-
"FROM": row.client_id,
|
|
4285
|
-
"TO": row.route_id,
|
|
4286
|
-
"source_file": client_by_id.get(row.client_id, ClientRow(id="", client_kind="", target_service="", path="", path_template="", path_regex="", method="", member_fqn="", member_id="", microservice="", module="", filename="", start_line=0, end_line=0, resolved=False, source_layer="")).filename,
|
|
4287
|
-
"confidence": row.confidence,
|
|
4288
|
-
"strategy": row.strategy,
|
|
4289
|
-
"method_call": row.method_call,
|
|
4290
|
-
"raw_uri": row.raw_uri,
|
|
4291
|
-
"match": row.match,
|
|
4292
|
-
}
|
|
4293
|
-
for row in tables.http_call_rows
|
|
4294
|
-
if row.client_id in valid_ids and row.route_id in valid_ids
|
|
4295
|
-
]
|
|
4296
|
-
_bulk_copy(conn, "HTTP_CALLS", _REL_HTTP_CALLS_COLUMNS, http_call_rows)
|
|
4297
|
-
|
|
4298
|
-
# Stage ASYNC_CALLS edge rows (Producer -> Route)
|
|
4299
|
-
async_call_rows = [
|
|
4300
|
-
{
|
|
4301
|
-
"FROM": row.producer_id,
|
|
4302
|
-
"TO": row.route_id,
|
|
4303
|
-
"source_file": producer_by_id.get(row.producer_id, ProducerRow(id="", producer_kind="", topic="", broker="", direction="", member_fqn="", member_id="", microservice="", module="", filename="", start_line=0, end_line=0, resolved=False, source_layer="")).filename,
|
|
4304
|
-
"confidence": row.confidence,
|
|
4305
|
-
"strategy": row.strategy,
|
|
4306
|
-
"direction": row.direction,
|
|
4307
|
-
"raw_topic": row.raw_topic,
|
|
4308
|
-
"match": row.match,
|
|
4309
|
-
}
|
|
4310
|
-
for row in tables.async_call_rows
|
|
4311
|
-
if row.producer_id in valid_ids and row.route_id in valid_ids
|
|
4312
|
-
]
|
|
4313
|
-
_bulk_copy(conn, "ASYNC_CALLS", _REL_ASYNC_CALLS_COLUMNS, async_call_rows)
|
|
4314
|
-
|
|
4315
|
-
|
|
4316
|
-
def write_ladybug(
|
|
4317
|
-
db_path: Path,
|
|
4318
|
-
tables: GraphTables,
|
|
4319
|
-
*,
|
|
4320
|
-
source_root: Path,
|
|
4321
|
-
verbose: bool,
|
|
4322
|
-
meta_chain: dict[str, frozenset[str]] | None = None,
|
|
4323
|
-
) -> None:
|
|
4324
|
-
if meta_chain is None:
|
|
4325
|
-
meta_chain = collect_annotation_meta_chain(
|
|
4326
|
-
str(source_root.resolve()),
|
|
4327
|
-
)
|
|
4328
|
-
if verbose:
|
|
4329
|
-
_verbose_stderr_line(_WRITE_START)
|
|
4330
|
-
with _VerbosePassHeartbeats("[graph] writing", verbose=verbose):
|
|
4331
|
-
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
4332
|
-
db = ladybug.Database(str(db_path))
|
|
4333
|
-
conn = ladybug.Connection(db)
|
|
4334
|
-
_drop_all(conn)
|
|
4335
|
-
_create_schema(conn)
|
|
4336
|
-
t0 = time.time()
|
|
4337
|
-
_write_nodes(
|
|
4338
|
-
conn,
|
|
4339
|
-
tables,
|
|
4340
|
-
project_root=source_root,
|
|
4341
|
-
meta_chain=meta_chain,
|
|
4342
|
-
)
|
|
4343
|
-
if verbose:
|
|
4344
|
-
_verbose_stderr_line(f"[graph] writing · nodes written in {time.time() - t0:.2f}s")
|
|
4345
|
-
_populate_declares_rows(tables)
|
|
4346
|
-
_populate_overrides_rows(tables)
|
|
4347
|
-
t1 = time.time()
|
|
4348
|
-
_fbyid = _build_file_by_node_id(tables)
|
|
4349
|
-
_write_edges(conn, tables, _fbyid)
|
|
4350
|
-
if verbose:
|
|
4351
|
-
_verbose_stderr_line(f"[graph] writing · edges written in {time.time() - t1:.2f}s")
|
|
4352
|
-
t2 = time.time()
|
|
4353
|
-
_write_routes_and_exposes(conn, tables, _fbyid)
|
|
4354
|
-
if verbose:
|
|
4355
|
-
_verbose_stderr_line(f"[graph] writing · routes/exposes written in {time.time() - t2:.2f}s")
|
|
4356
|
-
_write_meta(conn, tables, source_root)
|
|
4357
|
-
_ensure_symbol_fts_index(conn, verbose=verbose)
|
|
4358
|
-
conn.close()
|
|
4359
|
-
db.close()
|
|
4360
|
-
|
|
4361
|
-
# Build vocabulary index (best-effort, failure doesn't fail the graph build)
|
|
4362
|
-
_try_build_vocabulary_index(db_path, source_root, verbose)
|
|
4363
|
-
_init_hash_tracker(source_root, db_path)
|
|
4364
|
-
|
|
4365
|
-
|
|
4366
|
-
def _try_build_vocabulary_index(db_path: Path, source_root: Path, verbose: bool) -> None:
|
|
4367
|
-
"""Build and save the vocabulary index as a sidecar (best-effort).
|
|
4368
|
-
|
|
4369
|
-
This is called after write_ladybug() completes. A build failure must not
|
|
4370
|
-
fail the graph build, so this is wrapped in try/except and logged.
|
|
4371
|
-
|
|
4372
|
-
Args:
|
|
4373
|
-
db_path: Path to the LadybugDB database file
|
|
4374
|
-
source_root: Source repository root
|
|
4375
|
-
verbose: Whether to emit verbose progress
|
|
4376
|
-
"""
|
|
4377
|
-
try:
|
|
4378
|
-
from java_codebase_rag.absence.absence_vocab import VocabularyIndex, VOCAB_INDEX_FILENAME
|
|
4379
|
-
from java_codebase_rag.graph.ladybug_queries import LadybugGraph
|
|
4380
|
-
|
|
4381
|
-
t0 = time.time()
|
|
4382
|
-
if verbose:
|
|
4383
|
-
_verbose_stderr_line("[vocab] building vocabulary index")
|
|
4384
|
-
|
|
4385
|
-
# Open graph for reading
|
|
4386
|
-
graph = LadybugGraph.get(str(db_path))
|
|
4387
|
-
|
|
4388
|
-
# Read q from env var set by ResolvedOperatorConfig.subprocess_env()
|
|
4389
|
-
raw_q = os.environ.get("JAVA_CODEBASE_RAG_ABSENCE_NGRAM_Q", "3").strip()
|
|
4390
|
-
try:
|
|
4391
|
-
q = int(raw_q) if raw_q else 3
|
|
4392
|
-
except ValueError:
|
|
4393
|
-
q = 3 # Invalid env value falls back to default
|
|
4394
|
-
# Build index with configured q (or default 3)
|
|
4395
|
-
index = VocabularyIndex.build(graph, q=q)
|
|
4396
|
-
|
|
4397
|
-
# Save to sidecar next to the graph db
|
|
4398
|
-
sidecar_path = Path(db_path).parent / VOCAB_INDEX_FILENAME
|
|
4399
|
-
index.save(sidecar_path, ontology_version=ONTOLOGY_VERSION)
|
|
4400
|
-
|
|
4401
|
-
if verbose:
|
|
4402
|
-
_verbose_stderr_line(f"[vocab] index built with {index.symbol_count} symbols in {time.time() - t0:.2f}s")
|
|
4403
|
-
|
|
4404
|
-
except Exception as e:
|
|
4405
|
-
# Log but don't fail - graph build is the primary concern
|
|
4406
|
-
log.warning(f"Vocabulary index build failed (non-critical): {e}")
|
|
4407
|
-
if verbose:
|
|
4408
|
-
_verbose_stderr_line(f"[vocab] build failed (graph still written): {e}")
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
# ---------- CLI ----------
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
def _default_ladybug_path() -> Path:
|
|
4415
|
-
idx = os.environ.get("JAVA_CODEBASE_RAG_INDEX_DIR", "").strip()
|
|
4416
|
-
if idx and not idx.startswith(("s3://", "gs://", "az://")):
|
|
4417
|
-
return Path(os.path.expanduser(idx.rstrip("/"))) / "code_graph.lbug"
|
|
4418
|
-
return Path.cwd() / ".java-codebase-rag" / "code_graph.lbug"
|
|
4419
|
-
|
|
4420
|
-
|
|
4421
|
-
def main() -> int:
|
|
4422
|
-
parser = argparse.ArgumentParser(description="Build an AST-derived LadybugDB graph for Java sources.")
|
|
4423
|
-
parser.add_argument("--source-root", default=None, help="Repository / monorepo root to scan for .java (defaults to current working directory)")
|
|
4424
|
-
parser.add_argument(
|
|
4425
|
-
"--ladybug-path",
|
|
4426
|
-
default=None,
|
|
4427
|
-
help=(
|
|
4428
|
-
"LadybugDB database path (file/dir as used by ladybug.Database; "
|
|
4429
|
-
"default: $JAVA_CODEBASE_RAG_INDEX_DIR/code_graph.lbug or ./.java-codebase-rag/code_graph.lbug)"
|
|
4430
|
-
),
|
|
4431
|
-
)
|
|
4432
|
-
parser.add_argument("--verbose", action="store_true")
|
|
4433
|
-
parser.add_argument("--incremental", action="store_true", help="Run incremental rebuild instead of full rebuild")
|
|
4434
|
-
args = parser.parse_args()
|
|
4435
|
-
|
|
4436
|
-
root = Path(args.source_root).expanduser().resolve() if args.source_root else Path.cwd().resolve()
|
|
4437
|
-
if not root.is_dir():
|
|
4438
|
-
print(f"source-root not a directory: {root}", file=sys.stderr)
|
|
4439
|
-
return 2
|
|
4440
|
-
|
|
4441
|
-
ladybug_path = Path(args.ladybug_path).expanduser() if args.ladybug_path else _default_ladybug_path()
|
|
4442
|
-
|
|
4443
|
-
if args.incremental:
|
|
4444
|
-
result = incremental_rebuild(root, ladybug_path, verbose=args.verbose)
|
|
4445
|
-
print(json.dumps({
|
|
4446
|
-
"mode": result.mode,
|
|
4447
|
-
"files_changed": result.files_changed,
|
|
4448
|
-
"files_added": result.files_added,
|
|
4449
|
-
"files_removed": result.files_removed,
|
|
4450
|
-
"dependents_reprocessed": result.dependents_reprocessed,
|
|
4451
|
-
"elapsed_sec": result.elapsed_sec,
|
|
4452
|
-
}))
|
|
4453
|
-
if args.verbose:
|
|
4454
|
-
_verbose_stderr_line(f"[graph] done · mode={result.mode} files_changed={result.files_changed} files_added={result.files_added} files_removed={result.files_removed} dependents={result.dependents_reprocessed} elapsed={result.elapsed_sec:.2f}s")
|
|
4455
|
-
return 0
|
|
4456
|
-
|
|
4457
|
-
tables = GraphTables()
|
|
4458
|
-
asts = pass1_parse(root, tables, verbose=args.verbose)
|
|
4459
|
-
pass2_edges(tables, asts, verbose=args.verbose)
|
|
4460
|
-
pass3_calls(tables, asts, verbose=args.verbose)
|
|
4461
|
-
pass4_routes(tables, asts, source_root=root, verbose=args.verbose)
|
|
4462
|
-
pass5_imperative_edges(tables, asts, source_root=root, verbose=args.verbose)
|
|
4463
|
-
pass6_match_edges(tables, verbose=args.verbose)
|
|
4464
|
-
write_ladybug(ladybug_path, tables, source_root=root, verbose=args.verbose)
|
|
4465
|
-
if args.verbose:
|
|
4466
|
-
_verbose_stderr_line(f"[graph] done · ladybug at {ladybug_path}")
|
|
4467
|
-
return 0
|
|
4468
|
-
|
|
4469
|
-
|
|
4470
|
-
if __name__ == "__main__":
|
|
4471
|
-
sys.exit(main())
|