java-codebase-rag 0.6.3__py3-none-any.whl → 0.6.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
build_ast_graph.py CHANGED
@@ -39,6 +39,7 @@ from dataclasses import asdict, dataclass, field, replace
39
39
  from pathlib import Path
40
40
 
41
41
  import ladybug
42
+ import pyarrow as pa
42
43
 
43
44
  from ast_java import (
44
45
  ONTOLOGY_VERSION,
@@ -187,6 +188,12 @@ class TypeIndexEntry:
187
188
  package: str
188
189
  outer_fqn: str | None
189
190
  node_id: str
191
+ # True when this entry was loaded from the existing graph by
192
+ # `_load_existing_types` (an unchanged-file stub used only for cross-file
193
+ # resolution). Its `decl` is a placeholder (no annotations/methods), so its
194
+ # recomputed role/capabilities must never be written back over the real
195
+ # stored values. See `_write_nodes_impl`.
196
+ loaded_from_db: bool = False
190
197
 
191
198
 
192
199
  @dataclass
@@ -199,6 +206,11 @@ class MemberEntry:
199
206
  module: str
200
207
  microservice: str
201
208
  node_id: str
209
+ # True when loaded from the existing graph by `_load_existing_members`
210
+ # (an unchanged-file stub used only for cross-file call resolution). Its
211
+ # DECLARES edge already persists in the graph, so it must not be re-emitted
212
+ # by `_populate_declares_rows` (REL tables have no PK → would duplicate).
213
+ loaded_from_db: bool = False
202
214
 
203
215
 
204
216
  @dataclass
@@ -555,7 +567,7 @@ def _load_existing_types(conn: ladybug.Connection, tables: GraphTables, exclude_
555
567
  if exclude_files is not None and not exclude_files:
556
568
  return
557
569
 
558
- where = "WHERE s.kind IN ['class', 'interface', 'enum', 'annotation', 'record']"
570
+ where = f"WHERE s.kind IN {list(_TYPE_KINDS)}"
559
571
  params: dict = {}
560
572
  if exclude_files:
561
573
  where += "\n AND NOT (s.filename IN $exclude_files)"
@@ -585,6 +597,7 @@ def _load_existing_types(conn: ladybug.Connection, tables: GraphTables, exclude_
585
597
  package=package,
586
598
  outer_fqn=None,
587
599
  node_id=node_id,
600
+ loaded_from_db=True,
588
601
  )
589
602
  tables.types[fqn] = entry
590
603
  tables.by_simple_name.setdefault(name, []).append(entry)
@@ -633,6 +646,7 @@ def _load_existing_members(conn: ladybug.Connection, tables: GraphTables, exclud
633
646
  module="",
634
647
  microservice="",
635
648
  node_id=node_id,
649
+ loaded_from_db=True,
636
650
  ))
637
651
 
638
652
 
@@ -670,6 +684,54 @@ def _find_dependents(conn: ladybug.Connection, changed_node_ids: set[str]) -> se
670
684
  return dependent_files
671
685
 
672
686
 
687
+ def _find_annotation_dependents(conn: ladybug.Connection, changed_node_ids: set[str]) -> set[str]:
688
+ """Find files that USE an annotation whose DEFINITION is among the changed nodes.
689
+
690
+ Annotation usage is a node property (``annotations`` STRING[]), not a
691
+ Symbol->Symbol edge, so `_find_dependents` — which walks edges — never pulls
692
+ annotation users into scope. When an annotation definition changes (e.g.
693
+ ``@interface Foo`` gains a meta-annotation that shifts the Layer-A chain in
694
+ `resolve_role_and_capabilities`), every type carrying ``@Foo`` may need its
695
+ ``role``/``capabilities`` recomputed or it goes stale until the next full
696
+ rebuild. Return those users' files so the orchestrator treats them as
697
+ dependents (re-parsed, role re-SET); the expansion cap bounds the scope.
698
+
699
+ Scope is direct usage only: a user of an annotation that transitively
700
+ composes the changed one (e.g. ``@A`` where ``@A`` is meta-annotated with the
701
+ changed ``@B``) is NOT pulled in — that reverse-chain walk is left to a
702
+ future hardening pass. The direct case covers the dominant real-world shape
703
+ (a stereotype annotation applied directly to many types).
704
+ """
705
+ if not changed_node_ids:
706
+ return set()
707
+ # Changed annotation definitions → the simple names users reference them by.
708
+ # Runs before `_delete_file_scope`, so the def nodes still exist.
709
+ name_result = conn.execute(
710
+ "MATCH (s:Symbol) WHERE s.id IN $ids AND s.kind = 'annotation' RETURN s.name",
711
+ {"ids": list(changed_node_ids)},
712
+ )
713
+ names: list[str] = []
714
+ while name_result.has_next():
715
+ nm = name_result.get_next()[0]
716
+ if nm:
717
+ names.append(nm)
718
+ if not names:
719
+ return set()
720
+ dependent_files: set[str] = set()
721
+ for nm in names:
722
+ user_result = conn.execute(
723
+ "MATCH (s:Symbol) "
724
+ "WHERE list_contains(s.annotations, $nm) AND s.filename <> '' "
725
+ "RETURN DISTINCT s.filename",
726
+ {"nm": nm},
727
+ )
728
+ while user_result.has_next():
729
+ fn = user_result.get_next()[0]
730
+ if fn:
731
+ dependent_files.add(fn)
732
+ return dependent_files
733
+
734
+
673
735
  def _delete_file_scope(
674
736
  conn: ladybug.Connection,
675
737
  changed_files: set[str],
@@ -821,8 +883,8 @@ def _write_nodes_merge(
821
883
  project_root: Path,
822
884
  meta_chain: dict[str, frozenset[str]] | None,
823
885
  ) -> None:
824
- """Write nodes to existing LadybugDB database using MERGE to handle existing nodes."""
825
- _write_nodes_impl(conn, tables, project_root=project_root, meta_chain=meta_chain, symbol_query=_MERGE_SYMBOL)
886
+ """Write nodes to existing LadybugDB database using bulk COPY FROM."""
887
+ _write_nodes_impl(conn, tables, project_root=project_root, meta_chain=meta_chain)
826
888
 
827
889
 
828
890
  # ---------- file walk (see `path_filtering.iter_java_source_files`) ----------
@@ -2010,8 +2072,21 @@ def _producer_id(
2010
2072
  return f"p:{hashlib.sha1(key.encode()).hexdigest()[:16]}"
2011
2073
 
2012
2074
 
2075
+ # The four brownfield source layers — single source of truth. Consumed by the
2076
+ # client/producer source-layer classifiers, the *_from_brownfield_pct stats
2077
+ # (via brownfield_strategies), and the brownfield_only authoritativeness gate in
2078
+ # _is_brownfield_sourced. codebase_client/codebase_producer are caller-side
2079
+ # declaration strategies, not layers — they extend brownfield_strategies only.
2080
+ _BROWNFIELD_LAYERS = frozenset({
2081
+ "layer_a_meta",
2082
+ "layer_b_ann",
2083
+ "layer_b_fqn",
2084
+ "layer_c_source",
2085
+ })
2086
+
2087
+
2013
2088
  def _client_source_layer(strategy: str) -> str:
2014
- if strategy in {"layer_a_meta", "layer_b_ann", "layer_b_fqn", "layer_c_source"}:
2089
+ if strategy in _BROWNFIELD_LAYERS:
2015
2090
  return strategy
2016
2091
  # Some caller extraction paths emit client kind as strategy; treat those
2017
2092
  # as builtin-source declarations instead of warning on every row.
@@ -2023,7 +2098,7 @@ def _client_source_layer(strategy: str) -> str:
2023
2098
 
2024
2099
 
2025
2100
  def _producer_source_layer(strategy: str) -> str:
2026
- if strategy in {"layer_a_meta", "layer_b_ann", "layer_b_fqn", "layer_c_source"}:
2101
+ if strategy in _BROWNFIELD_LAYERS:
2027
2102
  return strategy
2028
2103
  if strategy in VALID_PRODUCER_KINDS:
2029
2104
  return "builtin"
@@ -2458,15 +2533,14 @@ def pass5_imperative_edges(
2458
2533
  tables.producer_stats.producers_by_kind = defaultdict(int)
2459
2534
  for row in tables.producer_rows:
2460
2535
  tables.producer_stats.producers_by_kind[row.producer_kind] += 1
2461
- brownfield_strategies = frozenset(
2462
- (
2463
- "layer_b_ann",
2464
- "layer_a_meta",
2465
- "layer_c_source",
2466
- "layer_b_fqn",
2467
- "codebase_client",
2468
- "codebase_producer",
2469
- ),
2536
+ # brownfield_strategies = the four brownfield layers plus the two
2537
+ # caller-side declaration strategies (@CodebaseHttpClient /
2538
+ # @CodebaseProducer). These extend _BROWNFIELD_LAYERS deliberately:
2539
+ # the *_from_brownfield_pct stats count annotation-declared callers as
2540
+ # brownfield-sourced even though they are not "layers" and so do not
2541
+ # gate brownfield_only authoritativeness in _is_brownfield_sourced.
2542
+ brownfield_strategies = _BROWNFIELD_LAYERS | frozenset(
2543
+ {"codebase_client", "codebase_producer"},
2470
2544
  )
2471
2545
  if tables.call_edge_stats.http_calls_total:
2472
2546
  n_http = sum(
@@ -2568,14 +2642,6 @@ def _match_call_edge(
2568
2642
  return "cross_service", candidates
2569
2643
 
2570
2644
 
2571
- _BROWNFIELD_LAYERS = frozenset({
2572
- "layer_c_source",
2573
- "layer_b_ann",
2574
- "layer_b_fqn",
2575
- "layer_a_meta",
2576
- })
2577
-
2578
-
2579
2645
  def _is_brownfield_sourced(
2580
2646
  call_strategy: str,
2581
2647
  candidates: list[RouteRow],
@@ -3000,27 +3066,112 @@ def _node_row(**kwargs) -> dict:
3000
3066
  return base
3001
3067
 
3002
3068
 
3003
- _CREATE_SYMBOL = (
3004
- "CREATE (:Symbol {id: $id, kind: $kind, name: $name, fqn: $fqn, "
3005
- "package: $package, module: $module, microservice: $microservice, "
3006
- "filename: $filename, "
3007
- "start_line: $start_line, end_line: $end_line, "
3008
- "start_byte: $start_byte, end_byte: $end_byte, "
3009
- "modifiers: $modifiers, annotations: $annotations, capabilities: $capabilities, "
3010
- "role: $role, signature: $signature, parent_id: $parent_id, resolved: $resolved})"
3011
- )
3069
+ def _bulk_copy(conn: ladybug.Connection, table_name: str, columns: list[str], rows: list[dict]) -> None:
3070
+ """Bulk-load rows into a node/rel table via in-memory pyarrow COPY FROM.
3071
+
3072
+ `columns` fixes column order; for REL tables the first two MUST be the
3073
+ FROM/TO node primary keys (kuzu requirement). Empty `rows` is a no-op.
3012
3074
 
3013
- _MERGE_SYMBOL = (
3014
- "MERGE (n:Symbol {id: $id}) "
3075
+ Spike result (PR-P1 step-1): REL `COPY FROM` expects columns named `FROM` and `TO`
3076
+ for the endpoint node IDs, followed by property columns in the declared order.
3077
+ `pa.Table.from_pylist(rows)` correctly infers types from the dict values, but
3078
+ we must select columns in the exact order expected by the table schema.
3079
+ """
3080
+ if not rows:
3081
+ return
3082
+ tbl = pa.Table.from_pylist(rows)
3083
+ # Select columns in the exact order expected by the table schema
3084
+ tbl = tbl.select(columns)
3085
+ conn.execute(f"COPY {table_name} FROM $rows", {"rows": tbl})
3086
+
3087
+
3088
+ def _existing_node_ids(conn: ladybug.Connection) -> set[str]:
3089
+ """Return every node id (Symbol, Route, Client, Producer) currently in the graph.
3090
+
3091
+ Bulk ``COPY FROM`` enforces referential integrity: a REL row whose FROM/TO
3092
+ endpoint isn't a loaded node raises ``Unable to find primary key value``. The
3093
+ legacy per-row ``MERGE (a:Symbol {id:$src}),(b:Symbol {id:$dst})`` silently
3094
+ dropped such edges (a ``MATCH`` against a missing endpoint creates nothing).
3095
+ Edge writers filter edge rows against this set to reproduce that exactly.
3096
+
3097
+ This queries the live DB rather than just ``tables`` because it is shared
3098
+ with the incremental path, whose edges legitimately reference nodes written
3099
+ in prior runs.
3100
+ """
3101
+ result = conn.execute("MATCH (n) RETURN n.id")
3102
+ ids: set[str] = set()
3103
+ while result.has_next():
3104
+ ids.add(result.get_next()[0])
3105
+ return ids
3106
+
3107
+
3108
+ # Column-order constants for bulk COPY FROM.
3109
+ # For REL tables, the first two entries are FROM/TO node primary keys (kuzu requirement).
3110
+ # Order matches the corresponding _SCHEMA_* declarations above.
3111
+ _NODE_COLUMNS = [
3112
+ "id", "kind", "name", "fqn", "package", "module", "microservice",
3113
+ "filename", "start_line", "end_line", "start_byte", "end_byte",
3114
+ "modifiers", "annotations", "capabilities", "role", "signature", "parent_id", "resolved"
3115
+ ]
3116
+
3117
+ # Type declaration kinds. Tuple (not set) so the rendered SQL `IN` clause is
3118
+ # deterministic. Used to (a) load type stubs for cross-file resolution and
3119
+ # (b) scope the incremental property-refresh SET to type nodes.
3120
+ _TYPE_KINDS: tuple[str, ...] = ("class", "interface", "enum", "annotation", "record")
3121
+
3122
+ # Update every mutable Symbol field on an existing node by primary key. Used on
3123
+ # the incremental path to refresh preserved dependent type nodes whose
3124
+ # `role`/`capabilities` (and other project-wide-derived fields) can shift
3125
+ # without their own source changing — restoring the upsert the legacy per-row
3126
+ # `MERGE (n:Symbol {id:$id}) SET …` provided. Field list mirrors `_NODE_COLUMNS`
3127
+ # minus `id`.
3128
+ _SET_SYMBOL_BY_ID = (
3129
+ "MATCH (n:Symbol {id: $id}) "
3015
3130
  "SET n.kind = $kind, n.name = $name, n.fqn = $fqn, "
3016
3131
  "n.package = $package, n.module = $module, n.microservice = $microservice, "
3017
3132
  "n.filename = $filename, "
3018
3133
  "n.start_line = $start_line, n.end_line = $end_line, "
3019
3134
  "n.start_byte = $start_byte, n.end_byte = $end_byte, "
3020
- "n.modifiers = $modifiers, n.annotations = $annotations, n.capabilities = $capabilities, "
3021
- "n.role = $role, n.signature = $signature, n.parent_id = $parent_id, n.resolved = $resolved"
3135
+ "n.modifiers = $modifiers, n.annotations = $annotations, "
3136
+ "n.capabilities = $capabilities, n.role = $role, "
3137
+ "n.signature = $signature, n.parent_id = $parent_id, n.resolved = $resolved"
3138
+ )
3139
+
3140
+ # Refresh every mutable Route field on an existing Route node by id. Mirrors the
3141
+ # `_write_nodes_impl` Symbol pattern (bulk COPY new rows + per-row SET existing
3142
+ # ones) so the global pass 5-6 step no longer needs a per-row MERGE upsert.
3143
+ # Field list mirrors `_ROUTE_COLUMNS` minus `id`.
3144
+ _SET_ROUTE_BY_ID = (
3145
+ "MATCH (r:Route {id: $id}) "
3146
+ "SET r.kind = $kind, r.framework = $framework, r.method = $method, "
3147
+ "r.path = $path, r.path_template = $path_template, r.path_regex = $path_regex, "
3148
+ "r.topic = $topic, r.broker = $broker, r.feign_name = $feign_name, r.feign_url = $feign_url, "
3149
+ "r.microservice = $microservice, r.module = $module, r.filename = $filename, "
3150
+ "r.start_line = $start_line, r.end_line = $end_line, r.resolved = $resolved"
3022
3151
  )
3023
3152
 
3153
+ _REL_EXTENDS_COLUMNS = ["FROM", "TO", "source_file", "dst_name", "dst_fqn", "resolved"]
3154
+ _REL_IMPLEMENTS_COLUMNS = ["FROM", "TO", "source_file", "dst_name", "dst_fqn", "resolved"]
3155
+ _REL_INJECTS_COLUMNS = ["FROM", "TO", "source_file", "dst_name", "dst_fqn", "resolved", "mechanism", "annotation", "field_or_param"]
3156
+ _REL_DECLARES_COLUMNS = ["FROM", "TO", "source_file"]
3157
+ _REL_OVERRIDES_COLUMNS = ["FROM", "TO", "source_file"]
3158
+ _REL_CALLS_COLUMNS = ["FROM", "TO", "source_file", "call_site_line", "call_site_byte", "arg_count", "confidence", "strategy", "source", "resolved", "callee_declaring_role"]
3159
+
3160
+ _UNRESOLVED_CALL_SITE_COLUMNS = ["id", "caller_id", "call_site_line", "call_site_byte", "arg_count", "callee_simple", "receiver_expr", "reason"]
3161
+ _REL_UNRESOLVED_AT_COLUMNS = ["FROM", "TO", "source_file"]
3162
+
3163
+ # Node table column constants (for bulk COPY FROM)
3164
+ _ROUTE_COLUMNS = ["id", "kind", "framework", "method", "path", "path_template", "path_regex", "topic", "broker", "feign_name", "feign_url", "microservice", "module", "filename", "start_line", "end_line", "resolved"]
3165
+ _CLIENT_COLUMNS = ["id", "client_kind", "target_service", "path", "path_template", "path_regex", "method", "member_fqn", "member_id", "microservice", "module", "filename", "start_line", "end_line", "resolved", "source_layer"]
3166
+ _PRODUCER_COLUMNS = ["id", "producer_kind", "topic", "broker", "direction", "member_fqn", "member_id", "microservice", "module", "filename", "start_line", "end_line", "resolved", "source_layer"]
3167
+
3168
+ # REL table column constants for routes/clients/producers
3169
+ _REL_EXPOSES_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy"]
3170
+ _REL_DECLARES_CLIENT_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy"]
3171
+ _REL_DECLARES_PRODUCER_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy"]
3172
+ _REL_HTTP_CALLS_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy", "method_call", "raw_uri", "match"]
3173
+ _REL_ASYNC_CALLS_COLUMNS = ["FROM", "TO", "source_file", "confidence", "strategy", "direction", "raw_topic", "match"]
3174
+
3024
3175
 
3025
3176
  def _write_nodes_impl(
3026
3177
  conn: ladybug.Connection,
@@ -3028,7 +3179,6 @@ def _write_nodes_impl(
3028
3179
  *,
3029
3180
  project_root: Path,
3030
3181
  meta_chain: dict[str, frozenset[str]] | None,
3031
- symbol_query: str,
3032
3182
  ) -> None:
3033
3183
  overrides = load_brownfield_overrides(project_root)
3034
3184
  try:
@@ -3037,18 +3187,28 @@ def _write_nodes_impl(
3037
3187
  prs = str(project_root)
3038
3188
  tables.cross_service_resolution = _load_config_cross_service_resolution(prs)
3039
3189
  mch = meta_chain
3190
+
3191
+ # Stage all Symbol rows
3192
+ rows: list[dict] = []
3193
+ # Node ids loaded from the existing graph as resolution-only stubs
3194
+ # (`_load_existing_types`); their staged rows carry placeholder values and
3195
+ # must never be written back over the real nodes.
3196
+ stub_ids: set[str] = set()
3197
+
3040
3198
  # packages
3041
3199
  for pkg, pid in tables.packages.items():
3042
- conn.execute(symbol_query, _node_row(
3200
+ rows.append(_node_row(
3043
3201
  id=pid, kind="package", name=pkg.rsplit(".", 1)[-1], fqn=pkg, package=pkg,
3044
3202
  ))
3045
3203
  # files
3046
3204
  for path, fid in tables.files.items():
3047
- conn.execute(symbol_query, _node_row(
3205
+ rows.append(_node_row(
3048
3206
  id=fid, kind="file", name=Path(path).name, fqn=path, filename=path,
3049
3207
  ))
3050
3208
  # types
3051
3209
  for entry in tables.types.values():
3210
+ if entry.loaded_from_db:
3211
+ stub_ids.add(entry.node_id)
3052
3212
  d = entry.decl
3053
3213
  role, capabilities = resolve_role_and_capabilities(
3054
3214
  d,
@@ -3056,7 +3216,7 @@ def _write_nodes_impl(
3056
3216
  meta_chain=mch,
3057
3217
  )
3058
3218
  tables.type_role_by_node_id[entry.node_id] = role
3059
- conn.execute(symbol_query, _node_row(
3219
+ rows.append(_node_row(
3060
3220
  id=entry.node_id, kind=d.kind, name=d.name, fqn=d.fqn,
3061
3221
  package=entry.package,
3062
3222
  module=entry.module, microservice=entry.microservice,
@@ -3072,7 +3232,7 @@ def _write_nodes_impl(
3072
3232
  ))
3073
3233
  # members (methods / constructors)
3074
3234
  for m in tables.members:
3075
- conn.execute(symbol_query, _node_row(
3235
+ rows.append(_node_row(
3076
3236
  id=m.node_id, kind=m.kind, name=m.decl.name,
3077
3237
  fqn=f"{m.parent_fqn}#{m.decl.signature}",
3078
3238
  package=tables.types[m.parent_fqn].package if m.parent_fqn in tables.types else "",
@@ -3086,7 +3246,35 @@ def _write_nodes_impl(
3086
3246
  ))
3087
3247
  # phantoms
3088
3248
  for pid, row in tables.phantoms.items():
3089
- conn.execute(symbol_query, row)
3249
+ rows.append(row)
3250
+
3251
+ # Bulk-load new Symbol rows. The full-rebuild path starts from an empty
3252
+ # database (`_drop_all`), so every row is new. The incremental path reaches
3253
+ # here with a populated database: changed-file nodes were deleted by
3254
+ # `_delete_file_scope` (absent here → new), while dependent-file nodes are
3255
+ # deliberately preserved (see `_delete_file_scope` / issue #305).
3256
+ existing_ids = _existing_node_ids(conn)
3257
+ new_rows = [row for row in rows if row["id"] not in existing_ids]
3258
+ _bulk_copy(conn, "Symbol", _NODE_COLUMNS, new_rows)
3259
+
3260
+ # Refresh mutable properties on preserved dependent TYPE nodes (incremental
3261
+ # path only; `update_rows` is empty on the full path). `role`/`capabilities`
3262
+ # — and any other field derived from project-wide inputs (meta-annotation
3263
+ # chain, brownfield overrides) — can shift without the type's own source
3264
+ # changing, so a preserved dependent must be re-SET to stay byte-equivalent
3265
+ # with a full rebuild. The legacy per-row `_MERGE_SYMBOL` upserted every
3266
+ # staged node and did this implicitly; bulk `COPY FROM` only appends, so the
3267
+ # SET is explicit here. Stubs (`stub_ids`) are skipped: their decl is a
3268
+ # placeholder and their stored values are authoritative. Non-type kinds
3269
+ # carry no mutable role/capabilities, so they are skipped too.
3270
+ update_rows = [
3271
+ row for row in rows
3272
+ if row["id"] in existing_ids
3273
+ and row["id"] not in stub_ids
3274
+ and row["kind"] in _TYPE_KINDS
3275
+ ]
3276
+ for row in update_rows:
3277
+ conn.execute(_SET_SYMBOL_BY_ID, row)
3090
3278
 
3091
3279
 
3092
3280
  def _write_nodes(
@@ -3096,95 +3284,21 @@ def _write_nodes(
3096
3284
  project_root: Path,
3097
3285
  meta_chain: dict[str, frozenset[str]] | None,
3098
3286
  ) -> None:
3099
- _write_nodes_impl(conn, tables, project_root=project_root, meta_chain=meta_chain, symbol_query=_CREATE_SYMBOL)
3287
+ _write_nodes_impl(conn, tables, project_root=project_root, meta_chain=meta_chain)
3100
3288
 
3101
3289
 
3102
- _CREATE_EXT = (
3103
- "MATCH (a:Symbol {id: $src}), (b:Symbol {id: $dst}) "
3104
- "CREATE (a)-[:EXTENDS {source_file: $source_file, dst_name: $dst_name, dst_fqn: $dst_fqn, resolved: $resolved}]->(b)"
3105
- )
3106
- _CREATE_IMPL = (
3107
- "MATCH (a:Symbol {id: $src}), (b:Symbol {id: $dst}) "
3108
- "CREATE (a)-[:IMPLEMENTS {source_file: $source_file, dst_name: $dst_name, dst_fqn: $dst_fqn, resolved: $resolved}]->(b)"
3109
- )
3110
- _CREATE_INJ = (
3111
- "MATCH (a:Symbol {id: $src}), (b:Symbol {id: $dst}) "
3112
- "CREATE (a)-[:INJECTS {source_file: $source_file, dst_name: $dst_name, dst_fqn: $dst_fqn, resolved: $resolved, "
3113
- "mechanism: $mechanism, annotation: $annotation, field_or_param: $field_or_param}]->(b)"
3114
- )
3115
- _CREATE_DECL = (
3116
- "MATCH (a:Symbol {id: $src}), (b:Symbol {id: $dst}) "
3117
- "CREATE (a)-[:DECLARES {source_file: $source_file}]->(b)"
3118
- )
3119
- _CREATE_OVERRIDES = (
3120
- "MATCH (a:Symbol {id: $src}), (b:Symbol {id: $dst}) "
3121
- "CREATE (a)-[:OVERRIDES {source_file: $source_file}]->(b)"
3122
- )
3123
- _CREATE_CALL = (
3124
- "MATCH (a:Symbol {id: $src}), (b:Symbol {id: $dst}) "
3125
- "CREATE (a)-[:CALLS {"
3126
- "source_file: $source_file, "
3127
- "call_site_line: $line, call_site_byte: $byte, arg_count: $argc, "
3128
- "confidence: $conf, strategy: $strat, source: $src_kind, resolved: $resolved, "
3129
- "callee_declaring_role: $callee_declaring_role"
3130
- "}]->(b)"
3131
- )
3132
-
3133
- _CREATE_ROUTE = (
3134
- "CREATE (:Route {"
3135
- "id: $id, kind: $kind, framework: $framework, method: $method, "
3136
- "path: $path, path_template: $path_template, path_regex: $path_regex, "
3137
- "topic: $topic, broker: $broker, feign_name: $feign_name, feign_url: $feign_url, "
3138
- "microservice: $microservice, module: $module, filename: $filename, "
3139
- "start_line: $start_line, end_line: $end_line, resolved: $resolved"
3140
- "})"
3141
- )
3142
- _CREATE_CLIENT = (
3143
- "CREATE (:Client {"
3144
- "id: $id, client_kind: $client_kind, target_service: $target_service, "
3145
- "path: $path, path_template: $path_template, path_regex: $path_regex, method: $method, "
3146
- "member_fqn: $member_fqn, member_id: $member_id, "
3147
- "microservice: $microservice, module: $module, filename: $filename, "
3148
- "start_line: $start_line, end_line: $end_line, resolved: $resolved, source_layer: $source_layer"
3149
- "})"
3150
- )
3151
-
3152
- _CREATE_EXPOSES = (
3153
- "MATCH (s:Symbol {id: $sid}), (r:Route {id: $rid}) "
3154
- "CREATE (s)-[:EXPOSES {source_file: $source_file, confidence: $confidence, strategy: $strategy}]->(r)"
3155
- )
3156
- _CREATE_DECLARES_CLIENT = (
3157
- "MATCH (s:Symbol {id: $sid}), (c:Client {id: $cid}) "
3158
- "CREATE (s)-[:DECLARES_CLIENT {source_file: $source_file, confidence: $confidence, strategy: $strategy}]->(c)"
3159
- )
3160
- _CREATE_PRODUCER = (
3161
- "CREATE (:Producer {"
3162
- "id: $id, producer_kind: $producer_kind, topic: $topic, broker: $broker, "
3163
- "direction: $direction, member_fqn: $member_fqn, member_id: $member_id, "
3164
- "microservice: $microservice, module: $module, filename: $filename, "
3165
- "start_line: $start_line, end_line: $end_line, resolved: $resolved, "
3166
- "source_layer: $source_layer"
3167
- "})"
3168
- )
3169
- _CREATE_DECLARES_PRODUCER = (
3170
- "MATCH (s:Symbol {id: $sid}), (p:Producer {id: $pid}) "
3171
- "CREATE (s)-[:DECLARES_PRODUCER {source_file: $source_file, confidence: $confidence, strategy: $strategy}]->(p)"
3172
- )
3173
- _CREATE_HTTP_CALL = (
3174
- "MATCH (c:Client {id: $cid}), (r:Route {id: $rid}) "
3175
- "CREATE (c)-[:HTTP_CALLS {source_file: $source_file, confidence: $confidence, strategy: $strategy, "
3176
- "method_call: $method_call, raw_uri: $raw_uri, match: $match}]->(r)"
3177
- )
3178
- _CREATE_ASYNC_CALL = (
3179
- "MATCH (p:Producer {id: $pid}), (r:Route {id: $rid}) "
3180
- "CREATE (p)-[:ASYNC_CALLS {source_file: $source_file, confidence: $confidence, strategy: $strategy, "
3181
- "direction: $direction, raw_topic: $raw_topic, match: $match}]->(r)"
3182
- )
3183
3290
 
3184
3291
 
3185
3292
  def _populate_declares_rows(tables: GraphTables) -> None:
3293
+ # Skip members loaded from the existing graph for cross-file resolution: a
3294
+ # DECLARES edge for an unchanged-file member already persists (its
3295
+ # source_file is out of scope, so `_delete_file_scope` left it), and
3296
+ # re-emitting it would append a duplicate (REL tables carry no primary key).
3297
+ # Full-rebuild never loads members, so this is a no-op there.
3186
3298
  tables.declares_rows = [
3187
- DeclaresRow(src_id=m.parent_id, dst_id=m.node_id) for m in tables.members
3299
+ DeclaresRow(src_id=m.parent_id, dst_id=m.node_id)
3300
+ for m in tables.members
3301
+ if not m.loaded_from_db
3188
3302
  ]
3189
3303
 
3190
3304
 
@@ -3242,93 +3356,126 @@ def _write_edges(conn: ladybug.Connection, tables: GraphTables, _file_by_node_id
3242
3356
  if _file_by_node_id is None:
3243
3357
  _file_by_node_id = _build_file_by_node_id(tables)
3244
3358
 
3245
- for r in tables.extends_rows:
3246
- conn.execute(_CREATE_EXT, {
3247
- "src": r.src_id, "dst": r.dst_id,
3359
+ # Bulk COPY FROM enforces referential integrity — a REL row whose endpoint
3360
+ # node isn't loaded raises "Unable to find primary key value". The legacy
3361
+ # per-row MERGE silently skipped such edges; drop them here to preserve the
3362
+ # per-row graph exactly. _existing_node_ids reads the live DB (not just
3363
+ # `tables`) so the incremental path's references to prior-run nodes still hold.
3364
+ valid_ids = _existing_node_ids(conn)
3365
+
3366
+ # Stage EXTENDS rows
3367
+ extends_rows = [
3368
+ {
3369
+ "FROM": r.src_id, "TO": r.dst_id,
3248
3370
  "source_file": _file_by_node_id.get(r.src_id, ""),
3249
3371
  "dst_name": r.dst_name, "dst_fqn": r.dst_fqn, "resolved": r.resolved,
3250
- })
3251
- for r in tables.implements_rows:
3252
- conn.execute(_CREATE_IMPL, {
3253
- "src": r.src_id, "dst": r.dst_id,
3372
+ }
3373
+ for r in tables.extends_rows
3374
+ if r.src_id in valid_ids and r.dst_id in valid_ids
3375
+ ]
3376
+ _bulk_copy(conn, "EXTENDS", _REL_EXTENDS_COLUMNS, extends_rows)
3377
+
3378
+ # Stage IMPLEMENTS rows
3379
+ implements_rows = [
3380
+ {
3381
+ "FROM": r.src_id, "TO": r.dst_id,
3254
3382
  "source_file": _file_by_node_id.get(r.src_id, ""),
3255
3383
  "dst_name": r.dst_name, "dst_fqn": r.dst_fqn, "resolved": r.resolved,
3256
- })
3257
- for r in tables.injects_rows:
3258
- conn.execute(_CREATE_INJ, {
3259
- "src": r.src_id, "dst": r.dst_id,
3384
+ }
3385
+ for r in tables.implements_rows
3386
+ if r.src_id in valid_ids and r.dst_id in valid_ids
3387
+ ]
3388
+ _bulk_copy(conn, "IMPLEMENTS", _REL_IMPLEMENTS_COLUMNS, implements_rows)
3389
+
3390
+ # Stage INJECTS rows
3391
+ injects_rows = [
3392
+ {
3393
+ "FROM": r.src_id, "TO": r.dst_id,
3260
3394
  "source_file": _file_by_node_id.get(r.src_id, ""),
3261
3395
  "dst_name": r.dst_name, "dst_fqn": r.dst_fqn, "resolved": r.resolved,
3262
3396
  "mechanism": r.mechanism, "annotation": r.annotation,
3263
3397
  "field_or_param": r.field_or_param,
3264
- })
3398
+ }
3399
+ for r in tables.injects_rows
3400
+ if r.src_id in valid_ids and r.dst_id in valid_ids
3401
+ ]
3402
+ _bulk_copy(conn, "INJECTS", _REL_INJECTS_COLUMNS, injects_rows)
3265
3403
 
3266
- for row in tables.declares_rows:
3267
- conn.execute(_CREATE_DECL, {
3268
- "src": row.src_id, "dst": row.dst_id,
3404
+ # Stage DECLARES rows
3405
+ declares_rows = [
3406
+ {
3407
+ "FROM": row.src_id, "TO": row.dst_id,
3269
3408
  "source_file": _file_by_node_id.get(row.src_id, ""),
3270
- })
3409
+ }
3410
+ for row in tables.declares_rows
3411
+ if row.src_id in valid_ids and row.dst_id in valid_ids
3412
+ ]
3413
+ _bulk_copy(conn, "DECLARES", _REL_DECLARES_COLUMNS, declares_rows)
3271
3414
 
3272
- for row in tables.overrides_rows:
3273
- conn.execute(_CREATE_OVERRIDES, {
3274
- "src": row.src_id, "dst": row.dst_id,
3415
+ # Stage OVERRIDES rows
3416
+ overrides_rows = [
3417
+ {
3418
+ "FROM": row.src_id, "TO": row.dst_id,
3275
3419
  "source_file": _file_by_node_id.get(row.src_id, ""),
3276
- })
3420
+ }
3421
+ for row in tables.overrides_rows
3422
+ if row.src_id in valid_ids and row.dst_id in valid_ids
3423
+ ]
3424
+ _bulk_copy(conn, "OVERRIDES", _REL_OVERRIDES_COLUMNS, overrides_rows)
3277
3425
 
3426
+ # Stage CALLS rows with dedup and callee_declaring_role materialization
3278
3427
  seen_calls: set[tuple[str, str, int, int]] = set()
3279
- unique_calls: list[CallsRow] = []
3428
+ calls_rows: list[dict] = []
3429
+ member_by_id = {m.node_id: m for m in tables.members}
3280
3430
  for row in tables.calls_rows:
3431
+ if row.src_id not in valid_ids or row.dst_id not in valid_ids:
3432
+ continue
3281
3433
  key = (row.src_id, row.dst_id, row.arg_count, row.call_site_line)
3282
- if key not in seen_calls:
3283
- seen_calls.add(key)
3284
- unique_calls.append(row)
3285
-
3286
- member_by_id = {m.node_id: m for m in tables.members}
3287
- for row in unique_calls:
3288
- conn.execute(_CREATE_CALL, {
3289
- "src": row.src_id, "dst": row.dst_id,
3434
+ if key in seen_calls:
3435
+ continue
3436
+ seen_calls.add(key)
3437
+ calls_rows.append({
3438
+ "FROM": row.src_id, "TO": row.dst_id,
3290
3439
  "source_file": _file_by_node_id.get(row.src_id, ""),
3291
- "line": row.call_site_line,
3292
- "byte": row.call_site_byte,
3293
- "argc": row.arg_count,
3294
- "conf": row.confidence,
3295
- "strat": row.strategy,
3296
- "src_kind": row.source,
3297
- "resolved": row.resolved,
3440
+ "call_site_line": row.call_site_line, "call_site_byte": row.call_site_byte,
3441
+ "arg_count": row.arg_count, "confidence": row.confidence, "strategy": row.strategy,
3442
+ "source": row.source, "resolved": row.resolved,
3298
3443
  "callee_declaring_role": _callee_declaring_role_at_write(
3299
3444
  tables, row.dst_id, member_by_id=member_by_id,
3300
3445
  ),
3301
3446
  })
3447
+ _bulk_copy(conn, "CALLS", _REL_CALLS_COLUMNS, calls_rows)
3302
3448
 
3303
- _CREATE_UNRESOLVED = (
3304
- "CREATE (:UnresolvedCallSite {"
3305
- "id: $id, caller_id: $caller_id, call_site_line: $line, call_site_byte: $byte, "
3306
- "arg_count: $argc, callee_simple: $callee, receiver_expr: $recv, reason: $reason"
3307
- "})"
3308
- )
3309
- _CREATE_UNRESOLVED_AT = (
3310
- "MATCH (a:Symbol {id: $caller}), (u:UnresolvedCallSite {id: $ucs}) "
3311
- "CREATE (a)-[:UNRESOLVED_AT {source_file: $source_file}]->(u)"
3312
- )
3449
+ # Stage UnresolvedCallSite node rows (must load before UNRESOLVED_AT edges)
3313
3450
  seen_ucs: set[str] = set()
3451
+ ucs_rows: list[dict] = []
3314
3452
  for row in tables.unresolved_call_site_rows:
3315
3453
  if row.id in seen_ucs:
3316
3454
  continue
3317
3455
  seen_ucs.add(row.id)
3318
- conn.execute(_CREATE_UNRESOLVED, {
3456
+ ucs_rows.append({
3319
3457
  "id": row.id,
3320
3458
  "caller_id": row.caller_id,
3321
- "line": row.call_site_line,
3322
- "byte": row.call_site_byte,
3323
- "argc": row.arg_count,
3324
- "callee": row.callee_simple,
3325
- "recv": row.receiver_expr,
3459
+ "call_site_line": row.call_site_line,
3460
+ "call_site_byte": row.call_site_byte,
3461
+ "arg_count": row.arg_count,
3462
+ "callee_simple": row.callee_simple,
3463
+ "receiver_expr": row.receiver_expr,
3326
3464
  "reason": row.reason,
3327
3465
  })
3328
- conn.execute(_CREATE_UNRESOLVED_AT, {
3329
- "caller": row.caller_id, "ucs": row.id,
3330
- "source_file": _file_by_node_id.get(row.caller_id, ""),
3331
- })
3466
+ _bulk_copy(conn, "UnresolvedCallSite", _UNRESOLVED_CALL_SITE_COLUMNS, ucs_rows)
3467
+
3468
+ # Stage UNRESOLVED_AT edge rows (one per unique UnresolvedCallSite node)
3469
+ # Use the same ucs_rows list to ensure 1:1 correspondence
3470
+ unresolved_at_rows = [
3471
+ {
3472
+ "FROM": ucs_row["caller_id"], "TO": ucs_row["id"],
3473
+ "source_file": _file_by_node_id.get(ucs_row["caller_id"], ""),
3474
+ }
3475
+ for ucs_row in ucs_rows
3476
+ if ucs_row["caller_id"] in valid_ids
3477
+ ]
3478
+ _bulk_copy(conn, "UNRESOLVED_AT", _REL_UNRESOLVED_AT_COLUMNS, unresolved_at_rows)
3332
3479
 
3333
3480
 
3334
3481
  def _write_routes_and_exposes(conn: ladybug.Connection, tables: GraphTables, _file_by_node_id: dict[str, str] | None = None) -> None:
@@ -3342,8 +3489,12 @@ def _write_routes_and_exposes(conn: ladybug.Connection, tables: GraphTables, _fi
3342
3489
  # Build producer_id -> filename lookup for ASYNC_CALLS source_file.
3343
3490
  _file_by_producer_id: dict[str, str] = {row.id: row.filename for row in tables.producer_rows}
3344
3491
 
3345
- for row in tables.routes_rows:
3346
- conn.execute(_CREATE_ROUTE, {
3492
+ # Bulk COPY FROM enforces referential integrity — get all valid node IDs
3493
+ valid_ids = _existing_node_ids(conn)
3494
+
3495
+ # Stage Route node rows (bulk-load before edges that reference them)
3496
+ route_rows = [
3497
+ {
3347
3498
  "id": row.id,
3348
3499
  "kind": row.kind,
3349
3500
  "framework": row.framework,
@@ -3361,57 +3512,97 @@ def _write_routes_and_exposes(conn: ladybug.Connection, tables: GraphTables, _fi
3361
3512
  "start_line": row.start_line,
3362
3513
  "end_line": row.end_line,
3363
3514
  "resolved": row.resolved,
3364
- })
3365
- for row in tables.exposes_rows:
3366
- conn.execute(_CREATE_EXPOSES, {
3367
- "sid": row.symbol_id,
3368
- "rid": row.route_id,
3515
+ }
3516
+ for row in tables.routes_rows
3517
+ ]
3518
+ _bulk_copy(conn, "Route", _ROUTE_COLUMNS, route_rows)
3519
+
3520
+ # Stage Client node rows (bulk-load before edges that reference them)
3521
+ client_rows = [asdict(row) for row in tables.client_rows]
3522
+ _bulk_copy(conn, "Client", _CLIENT_COLUMNS, client_rows)
3523
+
3524
+ # Stage Producer node rows (bulk-load before edges that reference them)
3525
+ producer_rows = [asdict(row) for row in tables.producer_rows]
3526
+ _bulk_copy(conn, "Producer", _PRODUCER_COLUMNS, producer_rows)
3527
+
3528
+ # Re-fetch valid IDs after loading Route/Client/Producer nodes (now includes them)
3529
+ valid_ids = _existing_node_ids(conn)
3530
+
3531
+ # Stage EXPOSES edge rows (Symbol -> Route)
3532
+ exposes_rows = [
3533
+ {
3534
+ "FROM": row.symbol_id,
3535
+ "TO": row.route_id,
3369
3536
  "source_file": _file_by_node_id.get(row.symbol_id, ""),
3370
3537
  "confidence": row.confidence,
3371
3538
  "strategy": row.strategy,
3372
- })
3373
- for row in tables.client_rows:
3374
- conn.execute(_CREATE_CLIENT, asdict(row))
3375
- for row in tables.declares_client_rows:
3376
- conn.execute(_CREATE_DECLARES_CLIENT, {
3377
- "sid": row.symbol_id,
3378
- "cid": row.client_id,
3539
+ }
3540
+ for row in tables.exposes_rows
3541
+ if row.symbol_id in valid_ids and row.route_id in valid_ids
3542
+ ]
3543
+ _bulk_copy(conn, "EXPOSES", _REL_EXPOSES_COLUMNS, exposes_rows)
3544
+
3545
+ # Stage DECLARES_CLIENT edge rows (Symbol -> Client)
3546
+ declares_client_rows = [
3547
+ {
3548
+ "FROM": row.symbol_id,
3549
+ "TO": row.client_id,
3379
3550
  "source_file": _file_by_node_id.get(row.symbol_id, ""),
3380
3551
  "confidence": row.confidence,
3381
3552
  "strategy": row.strategy,
3382
- })
3383
- for row in tables.producer_rows:
3384
- conn.execute(_CREATE_PRODUCER, asdict(row))
3385
- for row in tables.declares_producer_rows:
3386
- conn.execute(_CREATE_DECLARES_PRODUCER, {
3387
- "sid": row.symbol_id,
3388
- "pid": row.producer_id,
3553
+ }
3554
+ for row in tables.declares_client_rows
3555
+ if row.symbol_id in valid_ids and row.client_id in valid_ids
3556
+ ]
3557
+ _bulk_copy(conn, "DECLARES_CLIENT", _REL_DECLARES_CLIENT_COLUMNS, declares_client_rows)
3558
+
3559
+ # Stage DECLARES_PRODUCER edge rows (Symbol -> Producer)
3560
+ declares_producer_rows = [
3561
+ {
3562
+ "FROM": row.symbol_id,
3563
+ "TO": row.producer_id,
3389
3564
  "source_file": _file_by_node_id.get(row.symbol_id, ""),
3390
3565
  "confidence": row.confidence,
3391
3566
  "strategy": row.strategy,
3392
- })
3393
- for row in tables.http_call_rows:
3394
- conn.execute(_CREATE_HTTP_CALL, {
3395
- "cid": row.client_id,
3396
- "rid": row.route_id,
3567
+ }
3568
+ for row in tables.declares_producer_rows
3569
+ if row.symbol_id in valid_ids and row.producer_id in valid_ids
3570
+ ]
3571
+ _bulk_copy(conn, "DECLARES_PRODUCER", _REL_DECLARES_PRODUCER_COLUMNS, declares_producer_rows)
3572
+
3573
+ # Stage HTTP_CALLS edge rows (Client -> Route)
3574
+ http_call_rows = [
3575
+ {
3576
+ "FROM": row.client_id,
3577
+ "TO": row.route_id,
3397
3578
  "source_file": _file_by_client_id.get(row.client_id, ""),
3398
3579
  "confidence": row.confidence,
3399
3580
  "strategy": row.strategy,
3400
3581
  "method_call": row.method_call,
3401
3582
  "raw_uri": row.raw_uri,
3402
3583
  "match": row.match,
3403
- })
3404
- for row in tables.async_call_rows:
3405
- conn.execute(_CREATE_ASYNC_CALL, {
3406
- "pid": row.producer_id,
3407
- "rid": row.route_id,
3584
+ }
3585
+ for row in tables.http_call_rows
3586
+ if row.client_id in valid_ids and row.route_id in valid_ids
3587
+ ]
3588
+ _bulk_copy(conn, "HTTP_CALLS", _REL_HTTP_CALLS_COLUMNS, http_call_rows)
3589
+
3590
+ # Stage ASYNC_CALLS edge rows (Producer -> Route)
3591
+ async_call_rows = [
3592
+ {
3593
+ "FROM": row.producer_id,
3594
+ "TO": row.route_id,
3408
3595
  "source_file": _file_by_producer_id.get(row.producer_id, ""),
3409
3596
  "confidence": row.confidence,
3410
3597
  "strategy": row.strategy,
3411
3598
  "direction": row.direction,
3412
3599
  "raw_topic": row.raw_topic,
3413
3600
  "match": row.match,
3414
- })
3601
+ }
3602
+ for row in tables.async_call_rows
3603
+ if row.producer_id in valid_ids and row.route_id in valid_ids
3604
+ ]
3605
+ _bulk_copy(conn, "ASYNC_CALLS", _REL_ASYNC_CALLS_COLUMNS, async_call_rows)
3415
3606
 
3416
3607
 
3417
3608
  def _write_meta(conn: ladybug.Connection, tables: GraphTables, source_root: Path) -> None:
@@ -3646,6 +3837,12 @@ def incremental_rebuild(
3646
3837
  # Find dependents
3647
3838
  dependent_files = _find_dependents(conn, changed_node_ids)
3648
3839
 
3840
+ # Annotation-definition change: also pull in files that USE the changed
3841
+ # annotation. Annotation usage is a node property, not a Symbol->Symbol
3842
+ # edge, so `_find_dependents` misses them and their role (derived from
3843
+ # the project-wide meta-chain) would go stale. See PR-P5b.
3844
+ dependent_files |= _find_annotation_dependents(conn, changed_node_ids)
3845
+
3649
3846
  # Union changed files with dependents
3650
3847
  scope_files = changed_files | dependent_files
3651
3848
 
@@ -3810,80 +4007,107 @@ def _write_clients_producers_and_calls(conn: ladybug.Connection, tables: GraphTa
3810
4007
  Route nodes (created by pass5 for cross-service calls) that wouldn't
3811
4008
  otherwise exist in LadybugDB.
3812
4009
  """
3813
- # Write phantom routes that don't already exist (pass5 creates these for cross-service calls)
3814
- for row in tables.routes_rows:
3815
- # MERGE to avoid duplicates with routes written during scoped step
3816
- conn.execute(
3817
- "MERGE (r:Route {id: $id}) "
3818
- "SET r.kind = $kind, r.framework = $framework, r.method = $method, "
3819
- "r.path = $path, r.path_template = $path_template, r.path_regex = $path_regex, "
3820
- "r.topic = $topic, r.broker = $broker, r.feign_name = $feign_name, r.feign_url = $feign_url, "
3821
- "r.microservice = $microservice, r.module = $module, r.filename = $filename, "
3822
- "r.start_line = $start_line, r.end_line = $end_line, r.resolved = $resolved",
3823
- asdict(row),
3824
- )
4010
+ # Bulk-write routes, mirroring `_write_nodes_impl`: COPY the rows that are new
4011
+ # to the DB, and SET every mutable field on the routes already present (the
4012
+ # caller does NOT delete routes, only Clients/Producers see the global
4013
+ # pass 5-6 block). `tables.routes_rows` is the full route set (pass4 routes +
4014
+ # pass5 phantom routes), not just phantoms, so the SET keeps existing routes'
4015
+ # properties in sync with pass5's cross-service enrichment while the COPY
4016
+ # materializes phantoms that have no node yet. Replaces the last per-row
4017
+ # graph write (MERGE upsert).
4018
+ route_rows = [asdict(row) for row in tables.routes_rows]
4019
+ existing_route_ids = _existing_node_ids(conn)
4020
+ new_route_rows = [r for r in route_rows if r["id"] not in existing_route_ids]
4021
+ _bulk_copy(conn, "Route", _ROUTE_COLUMNS, new_route_rows)
4022
+ for r in route_rows:
4023
+ if r["id"] in existing_route_ids:
4024
+ conn.execute(_SET_ROUTE_BY_ID, r)
3825
4025
 
3826
4026
  # Build node_id lookup for members and types
3827
4027
  member_by_id = {m.node_id: m for m in tables.members}
3828
4028
 
3829
- # Write clients and producers using asdict (same pattern as _write_routes_and_exposes)
3830
- for row in tables.client_rows:
3831
- conn.execute(_CREATE_CLIENT, asdict(row))
3832
- for row in tables.producer_rows:
3833
- conn.execute(_CREATE_PRODUCER, asdict(row))
3834
-
4029
+ # Build client_id and producer_id lookups for source_file resolution
3835
4030
  client_by_id = {c.id: c for c in tables.client_rows}
3836
4031
  producer_by_id = {p.id: p for p in tables.producer_rows}
3837
4032
 
3838
- # Write declares_client edges
3839
- for row in tables.declares_client_rows:
3840
- source_file = member_by_id.get(row.symbol_id, MemberEntry(kind="", decl=None, parent_id="", parent_fqn="", file_path="", module="", microservice="", node_id="")).file_path
3841
- conn.execute(_CREATE_DECLARES_CLIENT, {
3842
- "sid": row.symbol_id,
3843
- "cid": row.client_id,
3844
- "source_file": source_file,
4033
+ # Stage Client + Producer NODE rows unconditionally. The caller DETACH-DELETEs
4034
+ # every Client/Producer node immediately before calling this (see the global
4035
+ # pass 5-6 block), so none are in the DB yet and COPY-ing them fresh is safe.
4036
+ # Do NOT filter node rows against the existing-id set — that is the EDGE filter
4037
+ # pattern mis-applied to nodes, and would drop every node being created (the
4038
+ # caller's delete makes the pre-load set empty by construction).
4039
+ client_rows = [asdict(row) for row in tables.client_rows]
4040
+ _bulk_copy(conn, "Client", _CLIENT_COLUMNS, client_rows)
4041
+ producer_rows = [asdict(row) for row in tables.producer_rows]
4042
+ _bulk_copy(conn, "Producer", _PRODUCER_COLUMNS, producer_rows)
4043
+
4044
+ # Endpoint filtering applies to EDGES only: re-fetch the live node set (now
4045
+ # includes the Clients/Producers just loaded) and drop edge rows whose
4046
+ # endpoints still aren't materialized — reproduces per-row MERGE silent-drop.
4047
+ valid_ids = _existing_node_ids(conn)
4048
+
4049
+ # Stage DECLARES_CLIENT edge rows (Symbol -> Client)
4050
+ # Build source_file lookup from member_by_id
4051
+ declares_client_rows = [
4052
+ {
4053
+ "FROM": row.symbol_id,
4054
+ "TO": row.client_id,
4055
+ "source_file": member_by_id.get(row.symbol_id, MemberEntry(kind="", decl=None, parent_id="", parent_fqn="", file_path="", module="", microservice="", node_id="")).file_path,
3845
4056
  "confidence": row.confidence,
3846
4057
  "strategy": row.strategy,
3847
- })
4058
+ }
4059
+ for row in tables.declares_client_rows
4060
+ if row.symbol_id in valid_ids and row.client_id in valid_ids
4061
+ ]
4062
+ _bulk_copy(conn, "DECLARES_CLIENT", _REL_DECLARES_CLIENT_COLUMNS, declares_client_rows)
3848
4063
 
3849
- # Write declares_producer edges
3850
- for row in tables.declares_producer_rows:
3851
- source_file = member_by_id.get(row.symbol_id, MemberEntry(kind="", decl=None, parent_id="", parent_fqn="", file_path="", module="", microservice="", node_id="")).file_path
3852
- conn.execute(_CREATE_DECLARES_PRODUCER, {
3853
- "sid": row.symbol_id,
3854
- "pid": row.producer_id,
3855
- "source_file": source_file,
4064
+ # Stage DECLARES_PRODUCER edge rows (Symbol -> Producer)
4065
+ declares_producer_rows = [
4066
+ {
4067
+ "FROM": row.symbol_id,
4068
+ "TO": row.producer_id,
4069
+ "source_file": member_by_id.get(row.symbol_id, MemberEntry(kind="", decl=None, parent_id="", parent_fqn="", file_path="", module="", microservice="", node_id="")).file_path,
3856
4070
  "confidence": row.confidence,
3857
4071
  "strategy": row.strategy,
3858
- })
4072
+ }
4073
+ for row in tables.declares_producer_rows
4074
+ if row.symbol_id in valid_ids and row.producer_id in valid_ids
4075
+ ]
4076
+ _bulk_copy(conn, "DECLARES_PRODUCER", _REL_DECLARES_PRODUCER_COLUMNS, declares_producer_rows)
3859
4077
 
3860
- # Write HTTP_CALLS edges
3861
- for row in tables.http_call_rows:
3862
- client = client_by_id.get(row.client_id)
3863
- conn.execute(_CREATE_HTTP_CALL, {
3864
- "cid": row.client_id,
3865
- "rid": row.route_id,
3866
- "source_file": client.filename if client else "",
4078
+ # Stage HTTP_CALLS edge rows (Client -> Route)
4079
+ http_call_rows = [
4080
+ {
4081
+ "FROM": row.client_id,
4082
+ "TO": row.route_id,
4083
+ "source_file": client_by_id.get(row.client_id, ClientRow(id="", client_kind="", target_service="", path="", path_template="", path_regex="", method="", member_fqn="", member_id="", microservice="", module="", filename="", start_line=0, end_line=0, resolved=False, source_layer="")).filename,
3867
4084
  "confidence": row.confidence,
3868
4085
  "strategy": row.strategy,
3869
4086
  "method_call": row.method_call,
3870
4087
  "raw_uri": row.raw_uri,
3871
4088
  "match": row.match,
3872
- })
4089
+ }
4090
+ for row in tables.http_call_rows
4091
+ if row.client_id in valid_ids and row.route_id in valid_ids
4092
+ ]
4093
+ _bulk_copy(conn, "HTTP_CALLS", _REL_HTTP_CALLS_COLUMNS, http_call_rows)
3873
4094
 
3874
- # Write ASYNC_CALLS edges
3875
- for row in tables.async_call_rows:
3876
- producer = producer_by_id.get(row.producer_id)
3877
- conn.execute(_CREATE_ASYNC_CALL, {
3878
- "pid": row.producer_id,
3879
- "rid": row.route_id,
3880
- "source_file": producer.filename if producer else "",
4095
+ # Stage ASYNC_CALLS edge rows (Producer -> Route)
4096
+ async_call_rows = [
4097
+ {
4098
+ "FROM": row.producer_id,
4099
+ "TO": row.route_id,
4100
+ "source_file": producer_by_id.get(row.producer_id, ProducerRow(id="", producer_kind="", topic="", broker="", direction="", member_fqn="", member_id="", microservice="", module="", filename="", start_line=0, end_line=0, resolved=False, source_layer="")).filename,
3881
4101
  "confidence": row.confidence,
3882
4102
  "strategy": row.strategy,
3883
4103
  "direction": row.direction,
3884
4104
  "raw_topic": row.raw_topic,
3885
4105
  "match": row.match,
3886
- })
4106
+ }
4107
+ for row in tables.async_call_rows
4108
+ if row.producer_id in valid_ids and row.route_id in valid_ids
4109
+ ]
4110
+ _bulk_copy(conn, "ASYNC_CALLS", _REL_ASYNC_CALLS_COLUMNS, async_call_rows)
3887
4111
 
3888
4112
 
3889
4113
  def write_ladybug(