java-codebase-rag 0.6.3__py3-none-any.whl → 0.6.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ast_java.py +28 -57
- build_ast_graph.py +486 -262
- graph_enrich.py +13 -35
- java_codebase_rag/cli.py +10 -10
- java_codebase_rag/install_data/agents/explorer-rag-enhanced.md +3 -3
- java_codebase_rag/install_data/skills/explore-codebase/SKILL.md +5 -5
- {java_codebase_rag-0.6.3.dist-info → java_codebase_rag-0.6.5.dist-info}/METADATA +10 -10
- {java_codebase_rag-0.6.3.dist-info → java_codebase_rag-0.6.5.dist-info}/RECORD +18 -18
- java_index_flow_lancedb.py +95 -21
- ladybug_queries.py +45 -51
- mcp_v2.py +9 -17
- path_filtering.py +24 -19
- pr_analysis.py +3 -4
- search_lancedb.py +2 -2
- {java_codebase_rag-0.6.3.dist-info → java_codebase_rag-0.6.5.dist-info}/WHEEL +0 -0
- {java_codebase_rag-0.6.3.dist-info → java_codebase_rag-0.6.5.dist-info}/entry_points.txt +0 -0
- {java_codebase_rag-0.6.3.dist-info → java_codebase_rag-0.6.5.dist-info}/licenses/LICENSE +0 -0
- {java_codebase_rag-0.6.3.dist-info → java_codebase_rag-0.6.5.dist-info}/top_level.txt +0 -0
build_ast_graph.py
CHANGED
|
@@ -39,6 +39,7 @@ from dataclasses import asdict, dataclass, field, replace
|
|
|
39
39
|
from pathlib import Path
|
|
40
40
|
|
|
41
41
|
import ladybug
|
|
42
|
+
import pyarrow as pa
|
|
42
43
|
|
|
43
44
|
from ast_java import (
|
|
44
45
|
ONTOLOGY_VERSION,
|
|
@@ -187,6 +188,12 @@ class TypeIndexEntry:
|
|
|
187
188
|
package: str
|
|
188
189
|
outer_fqn: str | None
|
|
189
190
|
node_id: str
|
|
191
|
+
# True when this entry was loaded from the existing graph by
|
|
192
|
+
# `_load_existing_types` (an unchanged-file stub used only for cross-file
|
|
193
|
+
# resolution). Its `decl` is a placeholder (no annotations/methods), so its
|
|
194
|
+
# recomputed role/capabilities must never be written back over the real
|
|
195
|
+
# stored values. See `_write_nodes_impl`.
|
|
196
|
+
loaded_from_db: bool = False
|
|
190
197
|
|
|
191
198
|
|
|
192
199
|
@dataclass
|
|
@@ -199,6 +206,11 @@ class MemberEntry:
|
|
|
199
206
|
module: str
|
|
200
207
|
microservice: str
|
|
201
208
|
node_id: str
|
|
209
|
+
# True when loaded from the existing graph by `_load_existing_members`
|
|
210
|
+
# (an unchanged-file stub used only for cross-file call resolution). Its
|
|
211
|
+
# DECLARES edge already persists in the graph, so it must not be re-emitted
|
|
212
|
+
# by `_populate_declares_rows` (REL tables have no PK → would duplicate).
|
|
213
|
+
loaded_from_db: bool = False
|
|
202
214
|
|
|
203
215
|
|
|
204
216
|
@dataclass
|
|
@@ -555,7 +567,7 @@ def _load_existing_types(conn: ladybug.Connection, tables: GraphTables, exclude_
|
|
|
555
567
|
if exclude_files is not None and not exclude_files:
|
|
556
568
|
return
|
|
557
569
|
|
|
558
|
-
where = "WHERE s.kind IN
|
|
570
|
+
where = f"WHERE s.kind IN {list(_TYPE_KINDS)}"
|
|
559
571
|
params: dict = {}
|
|
560
572
|
if exclude_files:
|
|
561
573
|
where += "\n AND NOT (s.filename IN $exclude_files)"
|
|
@@ -585,6 +597,7 @@ def _load_existing_types(conn: ladybug.Connection, tables: GraphTables, exclude_
|
|
|
585
597
|
package=package,
|
|
586
598
|
outer_fqn=None,
|
|
587
599
|
node_id=node_id,
|
|
600
|
+
loaded_from_db=True,
|
|
588
601
|
)
|
|
589
602
|
tables.types[fqn] = entry
|
|
590
603
|
tables.by_simple_name.setdefault(name, []).append(entry)
|
|
@@ -633,6 +646,7 @@ def _load_existing_members(conn: ladybug.Connection, tables: GraphTables, exclud
|
|
|
633
646
|
module="",
|
|
634
647
|
microservice="",
|
|
635
648
|
node_id=node_id,
|
|
649
|
+
loaded_from_db=True,
|
|
636
650
|
))
|
|
637
651
|
|
|
638
652
|
|
|
@@ -670,6 +684,54 @@ def _find_dependents(conn: ladybug.Connection, changed_node_ids: set[str]) -> se
|
|
|
670
684
|
return dependent_files
|
|
671
685
|
|
|
672
686
|
|
|
687
|
+
def _find_annotation_dependents(conn: ladybug.Connection, changed_node_ids: set[str]) -> set[str]:
|
|
688
|
+
"""Find files that USE an annotation whose DEFINITION is among the changed nodes.
|
|
689
|
+
|
|
690
|
+
Annotation usage is a node property (``annotations`` STRING[]), not a
|
|
691
|
+
Symbol->Symbol edge, so `_find_dependents` — which walks edges — never pulls
|
|
692
|
+
annotation users into scope. When an annotation definition changes (e.g.
|
|
693
|
+
``@interface Foo`` gains a meta-annotation that shifts the Layer-A chain in
|
|
694
|
+
`resolve_role_and_capabilities`), every type carrying ``@Foo`` may need its
|
|
695
|
+
``role``/``capabilities`` recomputed or it goes stale until the next full
|
|
696
|
+
rebuild. Return those users' files so the orchestrator treats them as
|
|
697
|
+
dependents (re-parsed, role re-SET); the expansion cap bounds the scope.
|
|
698
|
+
|
|
699
|
+
Scope is direct usage only: a user of an annotation that transitively
|
|
700
|
+
composes the changed one (e.g. ``@A`` where ``@A`` is meta-annotated with the
|
|
701
|
+
changed ``@B``) is NOT pulled in — that reverse-chain walk is left to a
|
|
702
|
+
future hardening pass. The direct case covers the dominant real-world shape
|
|
703
|
+
(a stereotype annotation applied directly to many types).
|
|
704
|
+
"""
|
|
705
|
+
if not changed_node_ids:
|
|
706
|
+
return set()
|
|
707
|
+
# Changed annotation definitions → the simple names users reference them by.
|
|
708
|
+
# Runs before `_delete_file_scope`, so the def nodes still exist.
|
|
709
|
+
name_result = conn.execute(
|
|
710
|
+
"MATCH (s:Symbol) WHERE s.id IN $ids AND s.kind = 'annotation' RETURN s.name",
|
|
711
|
+
{"ids": list(changed_node_ids)},
|
|
712
|
+
)
|
|
713
|
+
names: list[str] = []
|
|
714
|
+
while name_result.has_next():
|
|
715
|
+
nm = name_result.get_next()[0]
|
|
716
|
+
if nm:
|
|
717
|
+
names.append(nm)
|
|
718
|
+
if not names:
|
|
719
|
+
return set()
|
|
720
|
+
dependent_files: set[str] = set()
|
|
721
|
+
for nm in names:
|
|
722
|
+
user_result = conn.execute(
|
|
723
|
+
"MATCH (s:Symbol) "
|
|
724
|
+
"WHERE list_contains(s.annotations, $nm) AND s.filename <> '' "
|
|
725
|
+
"RETURN DISTINCT s.filename",
|
|
726
|
+
{"nm": nm},
|
|
727
|
+
)
|
|
728
|
+
while user_result.has_next():
|
|
729
|
+
fn = user_result.get_next()[0]
|
|
730
|
+
if fn:
|
|
731
|
+
dependent_files.add(fn)
|
|
732
|
+
return dependent_files
|
|
733
|
+
|
|
734
|
+
|
|
673
735
|
def _delete_file_scope(
|
|
674
736
|
conn: ladybug.Connection,
|
|
675
737
|
changed_files: set[str],
|
|
@@ -821,8 +883,8 @@ def _write_nodes_merge(
|
|
|
821
883
|
project_root: Path,
|
|
822
884
|
meta_chain: dict[str, frozenset[str]] | None,
|
|
823
885
|
) -> None:
|
|
824
|
-
"""Write nodes to existing LadybugDB database using
|
|
825
|
-
_write_nodes_impl(conn, tables, project_root=project_root, meta_chain=meta_chain
|
|
886
|
+
"""Write nodes to existing LadybugDB database using bulk COPY FROM."""
|
|
887
|
+
_write_nodes_impl(conn, tables, project_root=project_root, meta_chain=meta_chain)
|
|
826
888
|
|
|
827
889
|
|
|
828
890
|
# ---------- file walk (see `path_filtering.iter_java_source_files`) ----------
|
|
@@ -2010,8 +2072,21 @@ def _producer_id(
|
|
|
2010
2072
|
return f"p:{hashlib.sha1(key.encode()).hexdigest()[:16]}"
|
|
2011
2073
|
|
|
2012
2074
|
|
|
2075
|
+
# The four brownfield source layers — single source of truth. Consumed by the
|
|
2076
|
+
# client/producer source-layer classifiers, the *_from_brownfield_pct stats
|
|
2077
|
+
# (via brownfield_strategies), and the brownfield_only authoritativeness gate in
|
|
2078
|
+
# _is_brownfield_sourced. codebase_client/codebase_producer are caller-side
|
|
2079
|
+
# declaration strategies, not layers — they extend brownfield_strategies only.
|
|
2080
|
+
_BROWNFIELD_LAYERS = frozenset({
|
|
2081
|
+
"layer_a_meta",
|
|
2082
|
+
"layer_b_ann",
|
|
2083
|
+
"layer_b_fqn",
|
|
2084
|
+
"layer_c_source",
|
|
2085
|
+
})
|
|
2086
|
+
|
|
2087
|
+
|
|
2013
2088
|
def _client_source_layer(strategy: str) -> str:
|
|
2014
|
-
if strategy in
|
|
2089
|
+
if strategy in _BROWNFIELD_LAYERS:
|
|
2015
2090
|
return strategy
|
|
2016
2091
|
# Some caller extraction paths emit client kind as strategy; treat those
|
|
2017
2092
|
# as builtin-source declarations instead of warning on every row.
|
|
@@ -2023,7 +2098,7 @@ def _client_source_layer(strategy: str) -> str:
|
|
|
2023
2098
|
|
|
2024
2099
|
|
|
2025
2100
|
def _producer_source_layer(strategy: str) -> str:
|
|
2026
|
-
if strategy in
|
|
2101
|
+
if strategy in _BROWNFIELD_LAYERS:
|
|
2027
2102
|
return strategy
|
|
2028
2103
|
if strategy in VALID_PRODUCER_KINDS:
|
|
2029
2104
|
return "builtin"
|
|
@@ -2458,15 +2533,14 @@ def pass5_imperative_edges(
|
|
|
2458
2533
|
tables.producer_stats.producers_by_kind = defaultdict(int)
|
|
2459
2534
|
for row in tables.producer_rows:
|
|
2460
2535
|
tables.producer_stats.producers_by_kind[row.producer_kind] += 1
|
|
2461
|
-
brownfield_strategies =
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
),
|
|
2536
|
+
# brownfield_strategies = the four brownfield layers plus the two
|
|
2537
|
+
# caller-side declaration strategies (@CodebaseHttpClient /
|
|
2538
|
+
# @CodebaseProducer). These extend _BROWNFIELD_LAYERS deliberately:
|
|
2539
|
+
# the *_from_brownfield_pct stats count annotation-declared callers as
|
|
2540
|
+
# brownfield-sourced even though they are not "layers" and so do not
|
|
2541
|
+
# gate brownfield_only authoritativeness in _is_brownfield_sourced.
|
|
2542
|
+
brownfield_strategies = _BROWNFIELD_LAYERS | frozenset(
|
|
2543
|
+
{"codebase_client", "codebase_producer"},
|
|
2470
2544
|
)
|
|
2471
2545
|
if tables.call_edge_stats.http_calls_total:
|
|
2472
2546
|
n_http = sum(
|
|
@@ -2568,14 +2642,6 @@ def _match_call_edge(
|
|
|
2568
2642
|
return "cross_service", candidates
|
|
2569
2643
|
|
|
2570
2644
|
|
|
2571
|
-
_BROWNFIELD_LAYERS = frozenset({
|
|
2572
|
-
"layer_c_source",
|
|
2573
|
-
"layer_b_ann",
|
|
2574
|
-
"layer_b_fqn",
|
|
2575
|
-
"layer_a_meta",
|
|
2576
|
-
})
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
2645
|
def _is_brownfield_sourced(
|
|
2580
2646
|
call_strategy: str,
|
|
2581
2647
|
candidates: list[RouteRow],
|
|
@@ -3000,27 +3066,112 @@ def _node_row(**kwargs) -> dict:
|
|
|
3000
3066
|
return base
|
|
3001
3067
|
|
|
3002
3068
|
|
|
3003
|
-
|
|
3004
|
-
"
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
"start_byte: $start_byte, end_byte: $end_byte, "
|
|
3009
|
-
"modifiers: $modifiers, annotations: $annotations, capabilities: $capabilities, "
|
|
3010
|
-
"role: $role, signature: $signature, parent_id: $parent_id, resolved: $resolved})"
|
|
3011
|
-
)
|
|
3069
|
+
def _bulk_copy(conn: ladybug.Connection, table_name: str, columns: list[str], rows: list[dict]) -> None:
|
|
3070
|
+
"""Bulk-load rows into a node/rel table via in-memory pyarrow COPY FROM.
|
|
3071
|
+
|
|
3072
|
+
`columns` fixes column order; for REL tables the first two MUST be the
|
|
3073
|
+
FROM/TO node primary keys (kuzu requirement). Empty `rows` is a no-op.
|
|
3012
3074
|
|
|
3013
|
-
|
|
3014
|
-
|
|
3075
|
+
Spike result (PR-P1 step-1): REL `COPY FROM` expects columns named `FROM` and `TO`
|
|
3076
|
+
for the endpoint node IDs, followed by property columns in the declared order.
|
|
3077
|
+
`pa.Table.from_pylist(rows)` correctly infers types from the dict values, but
|
|
3078
|
+
we must select columns in the exact order expected by the table schema.
|
|
3079
|
+
"""
|
|
3080
|
+
if not rows:
|
|
3081
|
+
return
|
|
3082
|
+
tbl = pa.Table.from_pylist(rows)
|
|
3083
|
+
# Select columns in the exact order expected by the table schema
|
|
3084
|
+
tbl = tbl.select(columns)
|
|
3085
|
+
conn.execute(f"COPY {table_name} FROM $rows", {"rows": tbl})
|
|
3086
|
+
|
|
3087
|
+
|
|
3088
|
+
def _existing_node_ids(conn: ladybug.Connection) -> set[str]:
|
|
3089
|
+
"""Return every node id (Symbol, Route, Client, Producer) currently in the graph.
|
|
3090
|
+
|
|
3091
|
+
Bulk ``COPY FROM`` enforces referential integrity: a REL row whose FROM/TO
|
|
3092
|
+
endpoint isn't a loaded node raises ``Unable to find primary key value``. The
|
|
3093
|
+
legacy per-row ``MERGE (a:Symbol {id:$src}),(b:Symbol {id:$dst})`` silently
|
|
3094
|
+
dropped such edges (a ``MATCH`` against a missing endpoint creates nothing).
|
|
3095
|
+
Edge writers filter edge rows against this set to reproduce that exactly.
|
|
3096
|
+
|
|
3097
|
+
This queries the live DB rather than just ``tables`` because it is shared
|
|
3098
|
+
with the incremental path, whose edges legitimately reference nodes written
|
|
3099
|
+
in prior runs.
|
|
3100
|
+
"""
|
|
3101
|
+
result = conn.execute("MATCH (n) RETURN n.id")
|
|
3102
|
+
ids: set[str] = set()
|
|
3103
|
+
while result.has_next():
|
|
3104
|
+
ids.add(result.get_next()[0])
|
|
3105
|
+
return ids
|
|
3106
|
+
|
|
3107
|
+
|
|
3108
|
+
# Column-order constants for bulk COPY FROM.
|
|
3109
|
+
# For REL tables, the first two entries are FROM/TO node primary keys (kuzu requirement).
|
|
3110
|
+
# Order matches the corresponding _SCHEMA_* declarations above.
|
|
3111
|
+
_NODE_COLUMNS = [
|
|
3112
|
+
"id", "kind", "name", "fqn", "package", "module", "microservice",
|
|
3113
|
+
"filename", "start_line", "end_line", "start_byte", "end_byte",
|
|
3114
|
+
"modifiers", "annotations", "capabilities", "role", "signature", "parent_id", "resolved"
|
|
3115
|
+
]
|
|
3116
|
+
|
|
3117
|
+
# Type declaration kinds. Tuple (not set) so the rendered SQL `IN` clause is
|
|
3118
|
+
# deterministic. Used to (a) load type stubs for cross-file resolution and
|
|
3119
|
+
# (b) scope the incremental property-refresh SET to type nodes.
|
|
3120
|
+
_TYPE_KINDS: tuple[str, ...] = ("class", "interface", "enum", "annotation", "record")
|
|
3121
|
+
|
|
3122
|
+
# Update every mutable Symbol field on an existing node by primary key. Used on
|
|
3123
|
+
# the incremental path to refresh preserved dependent type nodes whose
|
|
3124
|
+
# `role`/`capabilities` (and other project-wide-derived fields) can shift
|
|
3125
|
+
# without their own source changing — restoring the upsert the legacy per-row
|
|
3126
|
+
# `MERGE (n:Symbol {id:$id}) SET …` provided. Field list mirrors `_NODE_COLUMNS`
|
|
3127
|
+
# minus `id`.
|
|
3128
|
+
_SET_SYMBOL_BY_ID = (
|
|
3129
|
+
"MATCH (n:Symbol {id: $id}) "
|
|
3015
3130
|
"SET n.kind = $kind, n.name = $name, n.fqn = $fqn, "
|
|
3016
3131
|
"n.package = $package, n.module = $module, n.microservice = $microservice, "
|
|
3017
3132
|
"n.filename = $filename, "
|
|
3018
3133
|
"n.start_line = $start_line, n.end_line = $end_line, "
|
|
3019
3134
|
"n.start_byte = $start_byte, n.end_byte = $end_byte, "
|
|
3020
|
-
"n.modifiers = $modifiers, n.annotations = $annotations,
|
|
3021
|
-
"n.
|
|
3135
|
+
"n.modifiers = $modifiers, n.annotations = $annotations, "
|
|
3136
|
+
"n.capabilities = $capabilities, n.role = $role, "
|
|
3137
|
+
"n.signature = $signature, n.parent_id = $parent_id, n.resolved = $resolved"
|
|
3138
|
+
)
|
|
3139
|
+
|
|
3140
|
+
# Refresh every mutable Route field on an existing Route node by id. Mirrors the
|
|
3141
|
+
# `_write_nodes_impl` Symbol pattern (bulk COPY new rows + per-row SET existing
|
|
3142
|
+
# ones) so the global pass 5-6 step no longer needs a per-row MERGE upsert.
|
|
3143
|
+
# Field list mirrors `_ROUTE_COLUMNS` minus `id`.
|
|
3144
|
+
_SET_ROUTE_BY_ID = (
|
|
3145
|
+
"MATCH (r:Route {id: $id}) "
|
|
3146
|
+
"SET r.kind = $kind, r.framework = $framework, r.method = $method, "
|
|
3147
|
+
"r.path = $path, r.path_template = $path_template, r.path_regex = $path_regex, "
|
|
3148
|
+
"r.topic = $topic, r.broker = $broker, r.feign_name = $feign_name, r.feign_url = $feign_url, "
|
|
3149
|
+
"r.microservice = $microservice, r.module = $module, r.filename = $filename, "
|
|
3150
|
+
"r.start_line = $start_line, r.end_line = $end_line, r.resolved = $resolved"
|
|
3022
3151
|
)
|
|
3023
3152
|
|
|
3153
|
+
_REL_EXTENDS_COLUMNS = ["FROM", "TO", "source_file", "dst_name", "dst_fqn", "resolved"]
|
|
3154
|
+
_REL_IMPLEMENTS_COLUMNS = ["FROM", "TO", "source_file", "dst_name", "dst_fqn", "resolved"]
|
|
3155
|
+
_REL_INJECTS_COLUMNS = ["FROM", "TO", "source_file", "dst_name", "dst_fqn", "resolved", "mechanism", "annotation", "field_or_param"]
|
|
3156
|
+
_REL_DECLARES_COLUMNS = ["FROM", "TO", "source_file"]
|
|
3157
|
+
_REL_OVERRIDES_COLUMNS = ["FROM", "TO", "source_file"]
|
|
3158
|
+
_REL_CALLS_COLUMNS = ["FROM", "TO", "source_file", "call_site_line", "call_site_byte", "arg_count", "confidence", "strategy", "source", "resolved", "callee_declaring_role"]
|
|
3159
|
+
|
|
3160
|
+
_UNRESOLVED_CALL_SITE_COLUMNS = ["id", "caller_id", "call_site_line", "call_site_byte", "arg_count", "callee_simple", "receiver_expr", "reason"]
|
|
3161
|
+
_REL_UNRESOLVED_AT_COLUMNS = ["FROM", "TO", "source_file"]
|
|
3162
|
+
|
|
3163
|
+
# Node table column constants (for bulk COPY FROM)
|
|
3164
|
+
_ROUTE_COLUMNS = ["id", "kind", "framework", "method", "path", "path_template", "path_regex", "topic", "broker", "feign_name", "feign_url", "microservice", "module", "filename", "start_line", "end_line", "resolved"]
|
|
3165
|
+
_CLIENT_COLUMNS = ["id", "client_kind", "target_service", "path", "path_template", "path_regex", "method", "member_fqn", "member_id", "microservice", "module", "filename", "start_line", "end_line", "resolved", "source_layer"]
|
|
3166
|
+
_PRODUCER_COLUMNS = ["id", "producer_kind", "topic", "broker", "direction", "member_fqn", "member_id", "microservice", "module", "filename", "start_line", "end_line", "resolved", "source_layer"]
|
|
3167
|
+
|
|
3168
|
+
# REL table column constants for routes/clients/producers
|
|
3169
|
+
_REL_EXPOSES_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy"]
|
|
3170
|
+
_REL_DECLARES_CLIENT_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy"]
|
|
3171
|
+
_REL_DECLARES_PRODUCER_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy"]
|
|
3172
|
+
_REL_HTTP_CALLS_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy", "method_call", "raw_uri", "match"]
|
|
3173
|
+
_REL_ASYNC_CALLS_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy", "direction", "raw_topic", "match"]
|
|
3174
|
+
|
|
3024
3175
|
|
|
3025
3176
|
def _write_nodes_impl(
|
|
3026
3177
|
conn: ladybug.Connection,
|
|
@@ -3028,7 +3179,6 @@ def _write_nodes_impl(
|
|
|
3028
3179
|
*,
|
|
3029
3180
|
project_root: Path,
|
|
3030
3181
|
meta_chain: dict[str, frozenset[str]] | None,
|
|
3031
|
-
symbol_query: str,
|
|
3032
3182
|
) -> None:
|
|
3033
3183
|
overrides = load_brownfield_overrides(project_root)
|
|
3034
3184
|
try:
|
|
@@ -3037,18 +3187,28 @@ def _write_nodes_impl(
|
|
|
3037
3187
|
prs = str(project_root)
|
|
3038
3188
|
tables.cross_service_resolution = _load_config_cross_service_resolution(prs)
|
|
3039
3189
|
mch = meta_chain
|
|
3190
|
+
|
|
3191
|
+
# Stage all Symbol rows
|
|
3192
|
+
rows: list[dict] = []
|
|
3193
|
+
# Node ids loaded from the existing graph as resolution-only stubs
|
|
3194
|
+
# (`_load_existing_types`); their staged rows carry placeholder values and
|
|
3195
|
+
# must never be written back over the real nodes.
|
|
3196
|
+
stub_ids: set[str] = set()
|
|
3197
|
+
|
|
3040
3198
|
# packages
|
|
3041
3199
|
for pkg, pid in tables.packages.items():
|
|
3042
|
-
|
|
3200
|
+
rows.append(_node_row(
|
|
3043
3201
|
id=pid, kind="package", name=pkg.rsplit(".", 1)[-1], fqn=pkg, package=pkg,
|
|
3044
3202
|
))
|
|
3045
3203
|
# files
|
|
3046
3204
|
for path, fid in tables.files.items():
|
|
3047
|
-
|
|
3205
|
+
rows.append(_node_row(
|
|
3048
3206
|
id=fid, kind="file", name=Path(path).name, fqn=path, filename=path,
|
|
3049
3207
|
))
|
|
3050
3208
|
# types
|
|
3051
3209
|
for entry in tables.types.values():
|
|
3210
|
+
if entry.loaded_from_db:
|
|
3211
|
+
stub_ids.add(entry.node_id)
|
|
3052
3212
|
d = entry.decl
|
|
3053
3213
|
role, capabilities = resolve_role_and_capabilities(
|
|
3054
3214
|
d,
|
|
@@ -3056,7 +3216,7 @@ def _write_nodes_impl(
|
|
|
3056
3216
|
meta_chain=mch,
|
|
3057
3217
|
)
|
|
3058
3218
|
tables.type_role_by_node_id[entry.node_id] = role
|
|
3059
|
-
|
|
3219
|
+
rows.append(_node_row(
|
|
3060
3220
|
id=entry.node_id, kind=d.kind, name=d.name, fqn=d.fqn,
|
|
3061
3221
|
package=entry.package,
|
|
3062
3222
|
module=entry.module, microservice=entry.microservice,
|
|
@@ -3072,7 +3232,7 @@ def _write_nodes_impl(
|
|
|
3072
3232
|
))
|
|
3073
3233
|
# members (methods / constructors)
|
|
3074
3234
|
for m in tables.members:
|
|
3075
|
-
|
|
3235
|
+
rows.append(_node_row(
|
|
3076
3236
|
id=m.node_id, kind=m.kind, name=m.decl.name,
|
|
3077
3237
|
fqn=f"{m.parent_fqn}#{m.decl.signature}",
|
|
3078
3238
|
package=tables.types[m.parent_fqn].package if m.parent_fqn in tables.types else "",
|
|
@@ -3086,7 +3246,35 @@ def _write_nodes_impl(
|
|
|
3086
3246
|
))
|
|
3087
3247
|
# phantoms
|
|
3088
3248
|
for pid, row in tables.phantoms.items():
|
|
3089
|
-
|
|
3249
|
+
rows.append(row)
|
|
3250
|
+
|
|
3251
|
+
# Bulk-load new Symbol rows. The full-rebuild path starts from an empty
|
|
3252
|
+
# database (`_drop_all`), so every row is new. The incremental path reaches
|
|
3253
|
+
# here with a populated database: changed-file nodes were deleted by
|
|
3254
|
+
# `_delete_file_scope` (absent here → new), while dependent-file nodes are
|
|
3255
|
+
# deliberately preserved (see `_delete_file_scope` / issue #305).
|
|
3256
|
+
existing_ids = _existing_node_ids(conn)
|
|
3257
|
+
new_rows = [row for row in rows if row["id"] not in existing_ids]
|
|
3258
|
+
_bulk_copy(conn, "Symbol", _NODE_COLUMNS, new_rows)
|
|
3259
|
+
|
|
3260
|
+
# Refresh mutable properties on preserved dependent TYPE nodes (incremental
|
|
3261
|
+
# path only; `update_rows` is empty on the full path). `role`/`capabilities`
|
|
3262
|
+
# — and any other field derived from project-wide inputs (meta-annotation
|
|
3263
|
+
# chain, brownfield overrides) — can shift without the type's own source
|
|
3264
|
+
# changing, so a preserved dependent must be re-SET to stay byte-equivalent
|
|
3265
|
+
# with a full rebuild. The legacy per-row `_MERGE_SYMBOL` upserted every
|
|
3266
|
+
# staged node and did this implicitly; bulk `COPY FROM` only appends, so the
|
|
3267
|
+
# SET is explicit here. Stubs (`stub_ids`) are skipped: their decl is a
|
|
3268
|
+
# placeholder and their stored values are authoritative. Non-type kinds
|
|
3269
|
+
# carry no mutable role/capabilities, so they are skipped too.
|
|
3270
|
+
update_rows = [
|
|
3271
|
+
row for row in rows
|
|
3272
|
+
if row["id"] in existing_ids
|
|
3273
|
+
and row["id"] not in stub_ids
|
|
3274
|
+
and row["kind"] in _TYPE_KINDS
|
|
3275
|
+
]
|
|
3276
|
+
for row in update_rows:
|
|
3277
|
+
conn.execute(_SET_SYMBOL_BY_ID, row)
|
|
3090
3278
|
|
|
3091
3279
|
|
|
3092
3280
|
def _write_nodes(
|
|
@@ -3096,95 +3284,21 @@ def _write_nodes(
|
|
|
3096
3284
|
project_root: Path,
|
|
3097
3285
|
meta_chain: dict[str, frozenset[str]] | None,
|
|
3098
3286
|
) -> None:
|
|
3099
|
-
_write_nodes_impl(conn, tables, project_root=project_root, meta_chain=meta_chain
|
|
3287
|
+
_write_nodes_impl(conn, tables, project_root=project_root, meta_chain=meta_chain)
|
|
3100
3288
|
|
|
3101
3289
|
|
|
3102
|
-
_CREATE_EXT = (
|
|
3103
|
-
"MATCH (a:Symbol {id: $src}), (b:Symbol {id: $dst}) "
|
|
3104
|
-
"CREATE (a)-[:EXTENDS {source_file: $source_file, dst_name: $dst_name, dst_fqn: $dst_fqn, resolved: $resolved}]->(b)"
|
|
3105
|
-
)
|
|
3106
|
-
_CREATE_IMPL = (
|
|
3107
|
-
"MATCH (a:Symbol {id: $src}), (b:Symbol {id: $dst}) "
|
|
3108
|
-
"CREATE (a)-[:IMPLEMENTS {source_file: $source_file, dst_name: $dst_name, dst_fqn: $dst_fqn, resolved: $resolved}]->(b)"
|
|
3109
|
-
)
|
|
3110
|
-
_CREATE_INJ = (
|
|
3111
|
-
"MATCH (a:Symbol {id: $src}), (b:Symbol {id: $dst}) "
|
|
3112
|
-
"CREATE (a)-[:INJECTS {source_file: $source_file, dst_name: $dst_name, dst_fqn: $dst_fqn, resolved: $resolved, "
|
|
3113
|
-
"mechanism: $mechanism, annotation: $annotation, field_or_param: $field_or_param}]->(b)"
|
|
3114
|
-
)
|
|
3115
|
-
_CREATE_DECL = (
|
|
3116
|
-
"MATCH (a:Symbol {id: $src}), (b:Symbol {id: $dst}) "
|
|
3117
|
-
"CREATE (a)-[:DECLARES {source_file: $source_file}]->(b)"
|
|
3118
|
-
)
|
|
3119
|
-
_CREATE_OVERRIDES = (
|
|
3120
|
-
"MATCH (a:Symbol {id: $src}), (b:Symbol {id: $dst}) "
|
|
3121
|
-
"CREATE (a)-[:OVERRIDES {source_file: $source_file}]->(b)"
|
|
3122
|
-
)
|
|
3123
|
-
_CREATE_CALL = (
|
|
3124
|
-
"MATCH (a:Symbol {id: $src}), (b:Symbol {id: $dst}) "
|
|
3125
|
-
"CREATE (a)-[:CALLS {"
|
|
3126
|
-
"source_file: $source_file, "
|
|
3127
|
-
"call_site_line: $line, call_site_byte: $byte, arg_count: $argc, "
|
|
3128
|
-
"confidence: $conf, strategy: $strat, source: $src_kind, resolved: $resolved, "
|
|
3129
|
-
"callee_declaring_role: $callee_declaring_role"
|
|
3130
|
-
"}]->(b)"
|
|
3131
|
-
)
|
|
3132
|
-
|
|
3133
|
-
_CREATE_ROUTE = (
|
|
3134
|
-
"CREATE (:Route {"
|
|
3135
|
-
"id: $id, kind: $kind, framework: $framework, method: $method, "
|
|
3136
|
-
"path: $path, path_template: $path_template, path_regex: $path_regex, "
|
|
3137
|
-
"topic: $topic, broker: $broker, feign_name: $feign_name, feign_url: $feign_url, "
|
|
3138
|
-
"microservice: $microservice, module: $module, filename: $filename, "
|
|
3139
|
-
"start_line: $start_line, end_line: $end_line, resolved: $resolved"
|
|
3140
|
-
"})"
|
|
3141
|
-
)
|
|
3142
|
-
_CREATE_CLIENT = (
|
|
3143
|
-
"CREATE (:Client {"
|
|
3144
|
-
"id: $id, client_kind: $client_kind, target_service: $target_service, "
|
|
3145
|
-
"path: $path, path_template: $path_template, path_regex: $path_regex, method: $method, "
|
|
3146
|
-
"member_fqn: $member_fqn, member_id: $member_id, "
|
|
3147
|
-
"microservice: $microservice, module: $module, filename: $filename, "
|
|
3148
|
-
"start_line: $start_line, end_line: $end_line, resolved: $resolved, source_layer: $source_layer"
|
|
3149
|
-
"})"
|
|
3150
|
-
)
|
|
3151
|
-
|
|
3152
|
-
_CREATE_EXPOSES = (
|
|
3153
|
-
"MATCH (s:Symbol {id: $sid}), (r:Route {id: $rid}) "
|
|
3154
|
-
"CREATE (s)-[:EXPOSES {source_file: $source_file, confidence: $confidence, strategy: $strategy}]->(r)"
|
|
3155
|
-
)
|
|
3156
|
-
_CREATE_DECLARES_CLIENT = (
|
|
3157
|
-
"MATCH (s:Symbol {id: $sid}), (c:Client {id: $cid}) "
|
|
3158
|
-
"CREATE (s)-[:DECLARES_CLIENT {source_file: $source_file, confidence: $confidence, strategy: $strategy}]->(c)"
|
|
3159
|
-
)
|
|
3160
|
-
_CREATE_PRODUCER = (
|
|
3161
|
-
"CREATE (:Producer {"
|
|
3162
|
-
"id: $id, producer_kind: $producer_kind, topic: $topic, broker: $broker, "
|
|
3163
|
-
"direction: $direction, member_fqn: $member_fqn, member_id: $member_id, "
|
|
3164
|
-
"microservice: $microservice, module: $module, filename: $filename, "
|
|
3165
|
-
"start_line: $start_line, end_line: $end_line, resolved: $resolved, "
|
|
3166
|
-
"source_layer: $source_layer"
|
|
3167
|
-
"})"
|
|
3168
|
-
)
|
|
3169
|
-
_CREATE_DECLARES_PRODUCER = (
|
|
3170
|
-
"MATCH (s:Symbol {id: $sid}), (p:Producer {id: $pid}) "
|
|
3171
|
-
"CREATE (s)-[:DECLARES_PRODUCER {source_file: $source_file, confidence: $confidence, strategy: $strategy}]->(p)"
|
|
3172
|
-
)
|
|
3173
|
-
_CREATE_HTTP_CALL = (
|
|
3174
|
-
"MATCH (c:Client {id: $cid}), (r:Route {id: $rid}) "
|
|
3175
|
-
"CREATE (c)-[:HTTP_CALLS {source_file: $source_file, confidence: $confidence, strategy: $strategy, "
|
|
3176
|
-
"method_call: $method_call, raw_uri: $raw_uri, match: $match}]->(r)"
|
|
3177
|
-
)
|
|
3178
|
-
_CREATE_ASYNC_CALL = (
|
|
3179
|
-
"MATCH (p:Producer {id: $pid}), (r:Route {id: $rid}) "
|
|
3180
|
-
"CREATE (p)-[:ASYNC_CALLS {source_file: $source_file, confidence: $confidence, strategy: $strategy, "
|
|
3181
|
-
"direction: $direction, raw_topic: $raw_topic, match: $match}]->(r)"
|
|
3182
|
-
)
|
|
3183
3290
|
|
|
3184
3291
|
|
|
3185
3292
|
def _populate_declares_rows(tables: GraphTables) -> None:
|
|
3293
|
+
# Skip members loaded from the existing graph for cross-file resolution: a
|
|
3294
|
+
# DECLARES edge for an unchanged-file member already persists (its
|
|
3295
|
+
# source_file is out of scope, so `_delete_file_scope` left it), and
|
|
3296
|
+
# re-emitting it would append a duplicate (REL tables carry no primary key).
|
|
3297
|
+
# Full-rebuild never loads members, so this is a no-op there.
|
|
3186
3298
|
tables.declares_rows = [
|
|
3187
|
-
DeclaresRow(src_id=m.parent_id, dst_id=m.node_id)
|
|
3299
|
+
DeclaresRow(src_id=m.parent_id, dst_id=m.node_id)
|
|
3300
|
+
for m in tables.members
|
|
3301
|
+
if not m.loaded_from_db
|
|
3188
3302
|
]
|
|
3189
3303
|
|
|
3190
3304
|
|
|
@@ -3242,93 +3356,126 @@ def _write_edges(conn: ladybug.Connection, tables: GraphTables, _file_by_node_id
|
|
|
3242
3356
|
if _file_by_node_id is None:
|
|
3243
3357
|
_file_by_node_id = _build_file_by_node_id(tables)
|
|
3244
3358
|
|
|
3245
|
-
|
|
3246
|
-
|
|
3247
|
-
|
|
3359
|
+
# Bulk COPY FROM enforces referential integrity — a REL row whose endpoint
|
|
3360
|
+
# node isn't loaded raises "Unable to find primary key value". The legacy
|
|
3361
|
+
# per-row MERGE silently skipped such edges; drop them here to preserve the
|
|
3362
|
+
# per-row graph exactly. _existing_node_ids reads the live DB (not just
|
|
3363
|
+
# `tables`) so the incremental path's references to prior-run nodes still hold.
|
|
3364
|
+
valid_ids = _existing_node_ids(conn)
|
|
3365
|
+
|
|
3366
|
+
# Stage EXTENDS rows
|
|
3367
|
+
extends_rows = [
|
|
3368
|
+
{
|
|
3369
|
+
"FROM": r.src_id, "TO": r.dst_id,
|
|
3248
3370
|
"source_file": _file_by_node_id.get(r.src_id, ""),
|
|
3249
3371
|
"dst_name": r.dst_name, "dst_fqn": r.dst_fqn, "resolved": r.resolved,
|
|
3250
|
-
}
|
|
3251
|
-
|
|
3252
|
-
|
|
3253
|
-
|
|
3372
|
+
}
|
|
3373
|
+
for r in tables.extends_rows
|
|
3374
|
+
if r.src_id in valid_ids and r.dst_id in valid_ids
|
|
3375
|
+
]
|
|
3376
|
+
_bulk_copy(conn, "EXTENDS", _REL_EXTENDS_COLUMNS, extends_rows)
|
|
3377
|
+
|
|
3378
|
+
# Stage IMPLEMENTS rows
|
|
3379
|
+
implements_rows = [
|
|
3380
|
+
{
|
|
3381
|
+
"FROM": r.src_id, "TO": r.dst_id,
|
|
3254
3382
|
"source_file": _file_by_node_id.get(r.src_id, ""),
|
|
3255
3383
|
"dst_name": r.dst_name, "dst_fqn": r.dst_fqn, "resolved": r.resolved,
|
|
3256
|
-
}
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3384
|
+
}
|
|
3385
|
+
for r in tables.implements_rows
|
|
3386
|
+
if r.src_id in valid_ids and r.dst_id in valid_ids
|
|
3387
|
+
]
|
|
3388
|
+
_bulk_copy(conn, "IMPLEMENTS", _REL_IMPLEMENTS_COLUMNS, implements_rows)
|
|
3389
|
+
|
|
3390
|
+
# Stage INJECTS rows
|
|
3391
|
+
injects_rows = [
|
|
3392
|
+
{
|
|
3393
|
+
"FROM": r.src_id, "TO": r.dst_id,
|
|
3260
3394
|
"source_file": _file_by_node_id.get(r.src_id, ""),
|
|
3261
3395
|
"dst_name": r.dst_name, "dst_fqn": r.dst_fqn, "resolved": r.resolved,
|
|
3262
3396
|
"mechanism": r.mechanism, "annotation": r.annotation,
|
|
3263
3397
|
"field_or_param": r.field_or_param,
|
|
3264
|
-
}
|
|
3398
|
+
}
|
|
3399
|
+
for r in tables.injects_rows
|
|
3400
|
+
if r.src_id in valid_ids and r.dst_id in valid_ids
|
|
3401
|
+
]
|
|
3402
|
+
_bulk_copy(conn, "INJECTS", _REL_INJECTS_COLUMNS, injects_rows)
|
|
3265
3403
|
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
|
|
3404
|
+
# Stage DECLARES rows
|
|
3405
|
+
declares_rows = [
|
|
3406
|
+
{
|
|
3407
|
+
"FROM": row.src_id, "TO": row.dst_id,
|
|
3269
3408
|
"source_file": _file_by_node_id.get(row.src_id, ""),
|
|
3270
|
-
}
|
|
3409
|
+
}
|
|
3410
|
+
for row in tables.declares_rows
|
|
3411
|
+
if row.src_id in valid_ids and row.dst_id in valid_ids
|
|
3412
|
+
]
|
|
3413
|
+
_bulk_copy(conn, "DECLARES", _REL_DECLARES_COLUMNS, declares_rows)
|
|
3271
3414
|
|
|
3272
|
-
|
|
3273
|
-
|
|
3274
|
-
|
|
3415
|
+
# Stage OVERRIDES rows
|
|
3416
|
+
overrides_rows = [
|
|
3417
|
+
{
|
|
3418
|
+
"FROM": row.src_id, "TO": row.dst_id,
|
|
3275
3419
|
"source_file": _file_by_node_id.get(row.src_id, ""),
|
|
3276
|
-
}
|
|
3420
|
+
}
|
|
3421
|
+
for row in tables.overrides_rows
|
|
3422
|
+
if row.src_id in valid_ids and row.dst_id in valid_ids
|
|
3423
|
+
]
|
|
3424
|
+
_bulk_copy(conn, "OVERRIDES", _REL_OVERRIDES_COLUMNS, overrides_rows)
|
|
3277
3425
|
|
|
3426
|
+
# Stage CALLS rows with dedup and callee_declaring_role materialization
|
|
3278
3427
|
seen_calls: set[tuple[str, str, int, int]] = set()
|
|
3279
|
-
|
|
3428
|
+
calls_rows: list[dict] = []
|
|
3429
|
+
member_by_id = {m.node_id: m for m in tables.members}
|
|
3280
3430
|
for row in tables.calls_rows:
|
|
3431
|
+
if row.src_id not in valid_ids or row.dst_id not in valid_ids:
|
|
3432
|
+
continue
|
|
3281
3433
|
key = (row.src_id, row.dst_id, row.arg_count, row.call_site_line)
|
|
3282
|
-
if key
|
|
3283
|
-
|
|
3284
|
-
|
|
3285
|
-
|
|
3286
|
-
|
|
3287
|
-
for row in unique_calls:
|
|
3288
|
-
conn.execute(_CREATE_CALL, {
|
|
3289
|
-
"src": row.src_id, "dst": row.dst_id,
|
|
3434
|
+
if key in seen_calls:
|
|
3435
|
+
continue
|
|
3436
|
+
seen_calls.add(key)
|
|
3437
|
+
calls_rows.append({
|
|
3438
|
+
"FROM": row.src_id, "TO": row.dst_id,
|
|
3290
3439
|
"source_file": _file_by_node_id.get(row.src_id, ""),
|
|
3291
|
-
"
|
|
3292
|
-
"
|
|
3293
|
-
"
|
|
3294
|
-
"conf": row.confidence,
|
|
3295
|
-
"strat": row.strategy,
|
|
3296
|
-
"src_kind": row.source,
|
|
3297
|
-
"resolved": row.resolved,
|
|
3440
|
+
"call_site_line": row.call_site_line, "call_site_byte": row.call_site_byte,
|
|
3441
|
+
"arg_count": row.arg_count, "confidence": row.confidence, "strategy": row.strategy,
|
|
3442
|
+
"source": row.source, "resolved": row.resolved,
|
|
3298
3443
|
"callee_declaring_role": _callee_declaring_role_at_write(
|
|
3299
3444
|
tables, row.dst_id, member_by_id=member_by_id,
|
|
3300
3445
|
),
|
|
3301
3446
|
})
|
|
3447
|
+
_bulk_copy(conn, "CALLS", _REL_CALLS_COLUMNS, calls_rows)
|
|
3302
3448
|
|
|
3303
|
-
|
|
3304
|
-
"CREATE (:UnresolvedCallSite {"
|
|
3305
|
-
"id: $id, caller_id: $caller_id, call_site_line: $line, call_site_byte: $byte, "
|
|
3306
|
-
"arg_count: $argc, callee_simple: $callee, receiver_expr: $recv, reason: $reason"
|
|
3307
|
-
"})"
|
|
3308
|
-
)
|
|
3309
|
-
_CREATE_UNRESOLVED_AT = (
|
|
3310
|
-
"MATCH (a:Symbol {id: $caller}), (u:UnresolvedCallSite {id: $ucs}) "
|
|
3311
|
-
"CREATE (a)-[:UNRESOLVED_AT {source_file: $source_file}]->(u)"
|
|
3312
|
-
)
|
|
3449
|
+
# Stage UnresolvedCallSite node rows (must load before UNRESOLVED_AT edges)
|
|
3313
3450
|
seen_ucs: set[str] = set()
|
|
3451
|
+
ucs_rows: list[dict] = []
|
|
3314
3452
|
for row in tables.unresolved_call_site_rows:
|
|
3315
3453
|
if row.id in seen_ucs:
|
|
3316
3454
|
continue
|
|
3317
3455
|
seen_ucs.add(row.id)
|
|
3318
|
-
|
|
3456
|
+
ucs_rows.append({
|
|
3319
3457
|
"id": row.id,
|
|
3320
3458
|
"caller_id": row.caller_id,
|
|
3321
|
-
"
|
|
3322
|
-
"
|
|
3323
|
-
"
|
|
3324
|
-
"
|
|
3325
|
-
"
|
|
3459
|
+
"call_site_line": row.call_site_line,
|
|
3460
|
+
"call_site_byte": row.call_site_byte,
|
|
3461
|
+
"arg_count": row.arg_count,
|
|
3462
|
+
"callee_simple": row.callee_simple,
|
|
3463
|
+
"receiver_expr": row.receiver_expr,
|
|
3326
3464
|
"reason": row.reason,
|
|
3327
3465
|
})
|
|
3328
|
-
|
|
3329
|
-
|
|
3330
|
-
|
|
3331
|
-
|
|
3466
|
+
_bulk_copy(conn, "UnresolvedCallSite", _UNRESOLVED_CALL_SITE_COLUMNS, ucs_rows)
|
|
3467
|
+
|
|
3468
|
+
# Stage UNRESOLVED_AT edge rows (one per unique UnresolvedCallSite node)
|
|
3469
|
+
# Use the same ucs_rows list to ensure 1:1 correspondence
|
|
3470
|
+
unresolved_at_rows = [
|
|
3471
|
+
{
|
|
3472
|
+
"FROM": ucs_row["caller_id"], "TO": ucs_row["id"],
|
|
3473
|
+
"source_file": _file_by_node_id.get(ucs_row["caller_id"], ""),
|
|
3474
|
+
}
|
|
3475
|
+
for ucs_row in ucs_rows
|
|
3476
|
+
if ucs_row["caller_id"] in valid_ids
|
|
3477
|
+
]
|
|
3478
|
+
_bulk_copy(conn, "UNRESOLVED_AT", _REL_UNRESOLVED_AT_COLUMNS, unresolved_at_rows)
|
|
3332
3479
|
|
|
3333
3480
|
|
|
3334
3481
|
def _write_routes_and_exposes(conn: ladybug.Connection, tables: GraphTables, _file_by_node_id: dict[str, str] | None = None) -> None:
|
|
@@ -3342,8 +3489,12 @@ def _write_routes_and_exposes(conn: ladybug.Connection, tables: GraphTables, _fi
|
|
|
3342
3489
|
# Build producer_id -> filename lookup for ASYNC_CALLS source_file.
|
|
3343
3490
|
_file_by_producer_id: dict[str, str] = {row.id: row.filename for row in tables.producer_rows}
|
|
3344
3491
|
|
|
3345
|
-
|
|
3346
|
-
|
|
3492
|
+
# Bulk COPY FROM enforces referential integrity — get all valid node IDs
|
|
3493
|
+
valid_ids = _existing_node_ids(conn)
|
|
3494
|
+
|
|
3495
|
+
# Stage Route node rows (bulk-load before edges that reference them)
|
|
3496
|
+
route_rows = [
|
|
3497
|
+
{
|
|
3347
3498
|
"id": row.id,
|
|
3348
3499
|
"kind": row.kind,
|
|
3349
3500
|
"framework": row.framework,
|
|
@@ -3361,57 +3512,97 @@ def _write_routes_and_exposes(conn: ladybug.Connection, tables: GraphTables, _fi
|
|
|
3361
3512
|
"start_line": row.start_line,
|
|
3362
3513
|
"end_line": row.end_line,
|
|
3363
3514
|
"resolved": row.resolved,
|
|
3364
|
-
}
|
|
3365
|
-
|
|
3366
|
-
|
|
3367
|
-
|
|
3368
|
-
|
|
3515
|
+
}
|
|
3516
|
+
for row in tables.routes_rows
|
|
3517
|
+
]
|
|
3518
|
+
_bulk_copy(conn, "Route", _ROUTE_COLUMNS, route_rows)
|
|
3519
|
+
|
|
3520
|
+
# Stage Client node rows (bulk-load before edges that reference them)
|
|
3521
|
+
client_rows = [asdict(row) for row in tables.client_rows]
|
|
3522
|
+
_bulk_copy(conn, "Client", _CLIENT_COLUMNS, client_rows)
|
|
3523
|
+
|
|
3524
|
+
# Stage Producer node rows (bulk-load before edges that reference them)
|
|
3525
|
+
producer_rows = [asdict(row) for row in tables.producer_rows]
|
|
3526
|
+
_bulk_copy(conn, "Producer", _PRODUCER_COLUMNS, producer_rows)
|
|
3527
|
+
|
|
3528
|
+
# Re-fetch valid IDs after loading Route/Client/Producer nodes (now includes them)
|
|
3529
|
+
valid_ids = _existing_node_ids(conn)
|
|
3530
|
+
|
|
3531
|
+
# Stage EXPOSES edge rows (Symbol -> Route)
|
|
3532
|
+
exposes_rows = [
|
|
3533
|
+
{
|
|
3534
|
+
"FROM": row.symbol_id,
|
|
3535
|
+
"TO": row.route_id,
|
|
3369
3536
|
"source_file": _file_by_node_id.get(row.symbol_id, ""),
|
|
3370
3537
|
"confidence": row.confidence,
|
|
3371
3538
|
"strategy": row.strategy,
|
|
3372
|
-
}
|
|
3373
|
-
|
|
3374
|
-
|
|
3375
|
-
|
|
3376
|
-
|
|
3377
|
-
|
|
3378
|
-
|
|
3539
|
+
}
|
|
3540
|
+
for row in tables.exposes_rows
|
|
3541
|
+
if row.symbol_id in valid_ids and row.route_id in valid_ids
|
|
3542
|
+
]
|
|
3543
|
+
_bulk_copy(conn, "EXPOSES", _REL_EXPOSES_COLUMNS, exposes_rows)
|
|
3544
|
+
|
|
3545
|
+
# Stage DECLARES_CLIENT edge rows (Symbol -> Client)
|
|
3546
|
+
declares_client_rows = [
|
|
3547
|
+
{
|
|
3548
|
+
"FROM": row.symbol_id,
|
|
3549
|
+
"TO": row.client_id,
|
|
3379
3550
|
"source_file": _file_by_node_id.get(row.symbol_id, ""),
|
|
3380
3551
|
"confidence": row.confidence,
|
|
3381
3552
|
"strategy": row.strategy,
|
|
3382
|
-
}
|
|
3383
|
-
|
|
3384
|
-
|
|
3385
|
-
|
|
3386
|
-
|
|
3387
|
-
|
|
3388
|
-
|
|
3553
|
+
}
|
|
3554
|
+
for row in tables.declares_client_rows
|
|
3555
|
+
if row.symbol_id in valid_ids and row.client_id in valid_ids
|
|
3556
|
+
]
|
|
3557
|
+
_bulk_copy(conn, "DECLARES_CLIENT", _REL_DECLARES_CLIENT_COLUMNS, declares_client_rows)
|
|
3558
|
+
|
|
3559
|
+
# Stage DECLARES_PRODUCER edge rows (Symbol -> Producer)
|
|
3560
|
+
declares_producer_rows = [
|
|
3561
|
+
{
|
|
3562
|
+
"FROM": row.symbol_id,
|
|
3563
|
+
"TO": row.producer_id,
|
|
3389
3564
|
"source_file": _file_by_node_id.get(row.symbol_id, ""),
|
|
3390
3565
|
"confidence": row.confidence,
|
|
3391
3566
|
"strategy": row.strategy,
|
|
3392
|
-
}
|
|
3393
|
-
|
|
3394
|
-
|
|
3395
|
-
|
|
3396
|
-
|
|
3567
|
+
}
|
|
3568
|
+
for row in tables.declares_producer_rows
|
|
3569
|
+
if row.symbol_id in valid_ids and row.producer_id in valid_ids
|
|
3570
|
+
]
|
|
3571
|
+
_bulk_copy(conn, "DECLARES_PRODUCER", _REL_DECLARES_PRODUCER_COLUMNS, declares_producer_rows)
|
|
3572
|
+
|
|
3573
|
+
# Stage HTTP_CALLS edge rows (Client -> Route)
|
|
3574
|
+
http_call_rows = [
|
|
3575
|
+
{
|
|
3576
|
+
"FROM": row.client_id,
|
|
3577
|
+
"TO": row.route_id,
|
|
3397
3578
|
"source_file": _file_by_client_id.get(row.client_id, ""),
|
|
3398
3579
|
"confidence": row.confidence,
|
|
3399
3580
|
"strategy": row.strategy,
|
|
3400
3581
|
"method_call": row.method_call,
|
|
3401
3582
|
"raw_uri": row.raw_uri,
|
|
3402
3583
|
"match": row.match,
|
|
3403
|
-
}
|
|
3404
|
-
|
|
3405
|
-
|
|
3406
|
-
|
|
3407
|
-
|
|
3584
|
+
}
|
|
3585
|
+
for row in tables.http_call_rows
|
|
3586
|
+
if row.client_id in valid_ids and row.route_id in valid_ids
|
|
3587
|
+
]
|
|
3588
|
+
_bulk_copy(conn, "HTTP_CALLS", _REL_HTTP_CALLS_COLUMNS, http_call_rows)
|
|
3589
|
+
|
|
3590
|
+
# Stage ASYNC_CALLS edge rows (Producer -> Route)
|
|
3591
|
+
async_call_rows = [
|
|
3592
|
+
{
|
|
3593
|
+
"FROM": row.producer_id,
|
|
3594
|
+
"TO": row.route_id,
|
|
3408
3595
|
"source_file": _file_by_producer_id.get(row.producer_id, ""),
|
|
3409
3596
|
"confidence": row.confidence,
|
|
3410
3597
|
"strategy": row.strategy,
|
|
3411
3598
|
"direction": row.direction,
|
|
3412
3599
|
"raw_topic": row.raw_topic,
|
|
3413
3600
|
"match": row.match,
|
|
3414
|
-
}
|
|
3601
|
+
}
|
|
3602
|
+
for row in tables.async_call_rows
|
|
3603
|
+
if row.producer_id in valid_ids and row.route_id in valid_ids
|
|
3604
|
+
]
|
|
3605
|
+
_bulk_copy(conn, "ASYNC_CALLS", _REL_ASYNC_CALLS_COLUMNS, async_call_rows)
|
|
3415
3606
|
|
|
3416
3607
|
|
|
3417
3608
|
def _write_meta(conn: ladybug.Connection, tables: GraphTables, source_root: Path) -> None:
|
|
@@ -3646,6 +3837,12 @@ def incremental_rebuild(
|
|
|
3646
3837
|
# Find dependents
|
|
3647
3838
|
dependent_files = _find_dependents(conn, changed_node_ids)
|
|
3648
3839
|
|
|
3840
|
+
# Annotation-definition change: also pull in files that USE the changed
|
|
3841
|
+
# annotation. Annotation usage is a node property, not a Symbol->Symbol
|
|
3842
|
+
# edge, so `_find_dependents` misses them and their role (derived from
|
|
3843
|
+
# the project-wide meta-chain) would go stale. See PR-P5b.
|
|
3844
|
+
dependent_files |= _find_annotation_dependents(conn, changed_node_ids)
|
|
3845
|
+
|
|
3649
3846
|
# Union changed files with dependents
|
|
3650
3847
|
scope_files = changed_files | dependent_files
|
|
3651
3848
|
|
|
@@ -3810,80 +4007,107 @@ def _write_clients_producers_and_calls(conn: ladybug.Connection, tables: GraphTa
|
|
|
3810
4007
|
Route nodes (created by pass5 for cross-service calls) that wouldn't
|
|
3811
4008
|
otherwise exist in LadybugDB.
|
|
3812
4009
|
"""
|
|
3813
|
-
#
|
|
3814
|
-
|
|
3815
|
-
|
|
3816
|
-
|
|
3817
|
-
|
|
3818
|
-
|
|
3819
|
-
|
|
3820
|
-
|
|
3821
|
-
|
|
3822
|
-
|
|
3823
|
-
|
|
3824
|
-
|
|
4010
|
+
# Bulk-write routes, mirroring `_write_nodes_impl`: COPY the rows that are new
|
|
4011
|
+
# to the DB, and SET every mutable field on the routes already present (the
|
|
4012
|
+
# caller does NOT delete routes, only Clients/Producers — see the global
|
|
4013
|
+
# pass 5-6 block). `tables.routes_rows` is the full route set (pass4 routes +
|
|
4014
|
+
# pass5 phantom routes), not just phantoms, so the SET keeps existing routes'
|
|
4015
|
+
# properties in sync with pass5's cross-service enrichment while the COPY
|
|
4016
|
+
# materializes phantoms that have no node yet. Replaces the last per-row
|
|
4017
|
+
# graph write (MERGE upsert).
|
|
4018
|
+
route_rows = [asdict(row) for row in tables.routes_rows]
|
|
4019
|
+
existing_route_ids = _existing_node_ids(conn)
|
|
4020
|
+
new_route_rows = [r for r in route_rows if r["id"] not in existing_route_ids]
|
|
4021
|
+
_bulk_copy(conn, "Route", _ROUTE_COLUMNS, new_route_rows)
|
|
4022
|
+
for r in route_rows:
|
|
4023
|
+
if r["id"] in existing_route_ids:
|
|
4024
|
+
conn.execute(_SET_ROUTE_BY_ID, r)
|
|
3825
4025
|
|
|
3826
4026
|
# Build node_id lookup for members and types
|
|
3827
4027
|
member_by_id = {m.node_id: m for m in tables.members}
|
|
3828
4028
|
|
|
3829
|
-
#
|
|
3830
|
-
for row in tables.client_rows:
|
|
3831
|
-
conn.execute(_CREATE_CLIENT, asdict(row))
|
|
3832
|
-
for row in tables.producer_rows:
|
|
3833
|
-
conn.execute(_CREATE_PRODUCER, asdict(row))
|
|
3834
|
-
|
|
4029
|
+
# Build client_id and producer_id lookups for source_file resolution
|
|
3835
4030
|
client_by_id = {c.id: c for c in tables.client_rows}
|
|
3836
4031
|
producer_by_id = {p.id: p for p in tables.producer_rows}
|
|
3837
4032
|
|
|
3838
|
-
#
|
|
3839
|
-
|
|
3840
|
-
|
|
3841
|
-
|
|
3842
|
-
|
|
3843
|
-
|
|
3844
|
-
|
|
4033
|
+
# Stage Client + Producer NODE rows unconditionally. The caller DETACH-DELETEs
|
|
4034
|
+
# every Client/Producer node immediately before calling this (see the global
|
|
4035
|
+
# pass 5-6 block), so none are in the DB yet and COPY-ing them fresh is safe.
|
|
4036
|
+
# Do NOT filter node rows against the existing-id set — that is the EDGE filter
|
|
4037
|
+
# pattern mis-applied to nodes, and would drop every node being created (the
|
|
4038
|
+
# caller's delete makes the pre-load set empty by construction).
|
|
4039
|
+
client_rows = [asdict(row) for row in tables.client_rows]
|
|
4040
|
+
_bulk_copy(conn, "Client", _CLIENT_COLUMNS, client_rows)
|
|
4041
|
+
producer_rows = [asdict(row) for row in tables.producer_rows]
|
|
4042
|
+
_bulk_copy(conn, "Producer", _PRODUCER_COLUMNS, producer_rows)
|
|
4043
|
+
|
|
4044
|
+
# Endpoint filtering applies to EDGES only: re-fetch the live node set (now
|
|
4045
|
+
# includes the Clients/Producers just loaded) and drop edge rows whose
|
|
4046
|
+
# endpoints still aren't materialized — reproduces per-row MERGE silent-drop.
|
|
4047
|
+
valid_ids = _existing_node_ids(conn)
|
|
4048
|
+
|
|
4049
|
+
# Stage DECLARES_CLIENT edge rows (Symbol -> Client)
|
|
4050
|
+
# Build source_file lookup from member_by_id
|
|
4051
|
+
declares_client_rows = [
|
|
4052
|
+
{
|
|
4053
|
+
"FROM": row.symbol_id,
|
|
4054
|
+
"TO": row.client_id,
|
|
4055
|
+
"source_file": member_by_id.get(row.symbol_id, MemberEntry(kind="", decl=None, parent_id="", parent_fqn="", file_path="", module="", microservice="", node_id="")).file_path,
|
|
3845
4056
|
"confidence": row.confidence,
|
|
3846
4057
|
"strategy": row.strategy,
|
|
3847
|
-
}
|
|
4058
|
+
}
|
|
4059
|
+
for row in tables.declares_client_rows
|
|
4060
|
+
if row.symbol_id in valid_ids and row.client_id in valid_ids
|
|
4061
|
+
]
|
|
4062
|
+
_bulk_copy(conn, "DECLARES_CLIENT", _REL_DECLARES_CLIENT_COLUMNS, declares_client_rows)
|
|
3848
4063
|
|
|
3849
|
-
#
|
|
3850
|
-
|
|
3851
|
-
|
|
3852
|
-
|
|
3853
|
-
"
|
|
3854
|
-
"
|
|
3855
|
-
"source_file": source_file,
|
|
4064
|
+
# Stage DECLARES_PRODUCER edge rows (Symbol -> Producer)
|
|
4065
|
+
declares_producer_rows = [
|
|
4066
|
+
{
|
|
4067
|
+
"FROM": row.symbol_id,
|
|
4068
|
+
"TO": row.producer_id,
|
|
4069
|
+
"source_file": member_by_id.get(row.symbol_id, MemberEntry(kind="", decl=None, parent_id="", parent_fqn="", file_path="", module="", microservice="", node_id="")).file_path,
|
|
3856
4070
|
"confidence": row.confidence,
|
|
3857
4071
|
"strategy": row.strategy,
|
|
3858
|
-
}
|
|
4072
|
+
}
|
|
4073
|
+
for row in tables.declares_producer_rows
|
|
4074
|
+
if row.symbol_id in valid_ids and row.producer_id in valid_ids
|
|
4075
|
+
]
|
|
4076
|
+
_bulk_copy(conn, "DECLARES_PRODUCER", _REL_DECLARES_PRODUCER_COLUMNS, declares_producer_rows)
|
|
3859
4077
|
|
|
3860
|
-
#
|
|
3861
|
-
|
|
3862
|
-
|
|
3863
|
-
|
|
3864
|
-
"
|
|
3865
|
-
"
|
|
3866
|
-
"source_file": client.filename if client else "",
|
|
4078
|
+
# Stage HTTP_CALLS edge rows (Client -> Route)
|
|
4079
|
+
http_call_rows = [
|
|
4080
|
+
{
|
|
4081
|
+
"FROM": row.client_id,
|
|
4082
|
+
"TO": row.route_id,
|
|
4083
|
+
"source_file": client_by_id.get(row.client_id, ClientRow(id="", client_kind="", target_service="", path="", path_template="", path_regex="", method="", member_fqn="", member_id="", microservice="", module="", filename="", start_line=0, end_line=0, resolved=False, source_layer="")).filename,
|
|
3867
4084
|
"confidence": row.confidence,
|
|
3868
4085
|
"strategy": row.strategy,
|
|
3869
4086
|
"method_call": row.method_call,
|
|
3870
4087
|
"raw_uri": row.raw_uri,
|
|
3871
4088
|
"match": row.match,
|
|
3872
|
-
}
|
|
4089
|
+
}
|
|
4090
|
+
for row in tables.http_call_rows
|
|
4091
|
+
if row.client_id in valid_ids and row.route_id in valid_ids
|
|
4092
|
+
]
|
|
4093
|
+
_bulk_copy(conn, "HTTP_CALLS", _REL_HTTP_CALLS_COLUMNS, http_call_rows)
|
|
3873
4094
|
|
|
3874
|
-
#
|
|
3875
|
-
|
|
3876
|
-
|
|
3877
|
-
|
|
3878
|
-
"
|
|
3879
|
-
"
|
|
3880
|
-
"source_file": producer.filename if producer else "",
|
|
4095
|
+
# Stage ASYNC_CALLS edge rows (Producer -> Route)
|
|
4096
|
+
async_call_rows = [
|
|
4097
|
+
{
|
|
4098
|
+
"FROM": row.producer_id,
|
|
4099
|
+
"TO": row.route_id,
|
|
4100
|
+
"source_file": producer_by_id.get(row.producer_id, ProducerRow(id="", producer_kind="", topic="", broker="", direction="", member_fqn="", member_id="", microservice="", module="", filename="", start_line=0, end_line=0, resolved=False, source_layer="")).filename,
|
|
3881
4101
|
"confidence": row.confidence,
|
|
3882
4102
|
"strategy": row.strategy,
|
|
3883
4103
|
"direction": row.direction,
|
|
3884
4104
|
"raw_topic": row.raw_topic,
|
|
3885
4105
|
"match": row.match,
|
|
3886
|
-
}
|
|
4106
|
+
}
|
|
4107
|
+
for row in tables.async_call_rows
|
|
4108
|
+
if row.producer_id in valid_ids and row.route_id in valid_ids
|
|
4109
|
+
]
|
|
4110
|
+
_bulk_copy(conn, "ASYNC_CALLS", _REL_ASYNC_CALLS_COLUMNS, async_call_rows)
|
|
3887
4111
|
|
|
3888
4112
|
|
|
3889
4113
|
def write_ladybug(
|