java-codebase-rag 0.9.6__py3-none-any.whl → 0.10.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- java_codebase_rag/absence/absence_vocab.py +7 -2
- java_codebase_rag/analysis/pr_analysis.py +33 -3
- java_codebase_rag/ast/ast_java.py +2 -1
- java_codebase_rag/cli.py +23 -8
- java_codebase_rag/config.py +68 -1
- java_codebase_rag/graph/build_ast_graph.py +123 -4
- java_codebase_rag/graph/graph_types.py +109 -22
- java_codebase_rag/graph/ladybug_queries.py +45 -2
- java_codebase_rag/index/java_index_flow_lancedb.py +10 -16
- java_codebase_rag/install_data/agents/explorer-rag-cli.md +3 -1
- java_codebase_rag/install_data/skills/explore-codebase-cli/SKILL.md +3 -1
- java_codebase_rag/jrag.py +627 -661
- java_codebase_rag/jrag_envelope.py +13 -0
- java_codebase_rag/jrag_render.py +160 -3
- java_codebase_rag/lance_optimize.py +11 -12
- java_codebase_rag/mcp/mcp_v2.py +16 -1
- java_codebase_rag/pipeline.py +47 -1
- java_codebase_rag/read_payloads.py +781 -0
- java_codebase_rag/search/search_lancedb.py +138 -6
- java_codebase_rag/search/search_lexical.py +128 -30
- java_codebase_rag/search/search_scoring.py +82 -0
- java_codebase_rag/watch/__init__.py +0 -0
- java_codebase_rag/watch/client.py +230 -0
- java_codebase_rag/watch/daemon.py +368 -0
- java_codebase_rag/watch/lock.py +201 -0
- java_codebase_rag/watch/paths.py +76 -0
- java_codebase_rag/watch/protocol.py +122 -0
- java_codebase_rag/watch/server.py +273 -0
- java_codebase_rag/watch/warm.py +105 -0
- java_codebase_rag/watch/watcher.py +352 -0
- {java_codebase_rag-0.9.6.dist-info → java_codebase_rag-0.10.0.dist-info}/METADATA +30 -31
- java_codebase_rag-0.10.0.dist-info/RECORD +67 -0
- java_codebase_rag-0.9.6.dist-info/RECORD +0 -57
- {java_codebase_rag-0.9.6.dist-info → java_codebase_rag-0.10.0.dist-info}/WHEEL +0 -0
- {java_codebase_rag-0.9.6.dist-info → java_codebase_rag-0.10.0.dist-info}/entry_points.txt +0 -0
- {java_codebase_rag-0.9.6.dist-info → java_codebase_rag-0.10.0.dist-info}/licenses/LICENSE +0 -0
- {java_codebase_rag-0.9.6.dist-info → java_codebase_rag-0.10.0.dist-info}/top_level.txt +0 -0
|
@@ -375,8 +375,13 @@ def get_vocabulary_index(graph: Any, cfg: Any) -> VocabularyIndex:
|
|
|
375
375
|
# (JSONDecodeError/KeyError) — all subsumed by Exception; rebuild.
|
|
376
376
|
log.debug(f"Vocab index missing/stale/corrupt ({e}), rebuilding from graph")
|
|
377
377
|
|
|
378
|
-
# Build from graph
|
|
379
|
-
|
|
378
|
+
# Build from graph. Coerce q to int (mirror absence_diagnosis.py's
|
|
379
|
+
# int(getattr(cfg, ...)) pattern): a non-int cfg.absence_ngram_q (e.g. a
|
|
380
|
+
# MagicMock cfg from a leaked test mock, or a YAML string) would otherwise
|
|
381
|
+
# crash _qgrams at `len(text) < q` ('int < MagicMock'). Default 3 = the
|
|
382
|
+
# config default for absence_ngram_q.
|
|
383
|
+
q = int(getattr(cfg, "absence_ngram_q", 3) or 3)
|
|
384
|
+
index = VocabularyIndex.build(graph, q=q)
|
|
380
385
|
|
|
381
386
|
# Save to sidecar (best-effort)
|
|
382
387
|
try:
|
|
@@ -377,6 +377,31 @@ def _route_ids_for_symbol(graph: Any, symbol_id: str) -> list[str]:
|
|
|
377
377
|
return out
|
|
378
378
|
|
|
379
379
|
|
|
380
|
+
def _route_natural_id(graph: Any, rid: str) -> str:
|
|
381
|
+
"""Map a raw Route node id to its agent-facing natural identifier.
|
|
382
|
+
|
|
383
|
+
Mirrors the envelope contract (``METHOD path`` for HTTP endpoints,
|
|
384
|
+
``topic:<name>`` for kafka topics surfaced as :Route) so ``routes_touched``
|
|
385
|
+
in the PR risk report is readable instead of leaking raw graph ids like
|
|
386
|
+
``r:970ffaa960a4f65d``. Falls back to ``rid`` only if the route vanished.
|
|
387
|
+
"""
|
|
388
|
+
rows = graph._rows(
|
|
389
|
+
"MATCH (r:Route {id: $rid}) "
|
|
390
|
+
"RETURN r.method AS method, r.path_template AS path_template, "
|
|
391
|
+
"r.path AS path, r.topic AS topic LIMIT 1",
|
|
392
|
+
{"rid": rid},
|
|
393
|
+
)
|
|
394
|
+
if not rows:
|
|
395
|
+
return rid
|
|
396
|
+
r = rows[0]
|
|
397
|
+
method = str(r.get("method") or "")
|
|
398
|
+
path = str(r.get("path_template") or r.get("path") or "")
|
|
399
|
+
if method or path:
|
|
400
|
+
return f"{method} {path}".strip()
|
|
401
|
+
topic = str(r.get("topic") or "")
|
|
402
|
+
return f"topic:{topic}" if topic else rid
|
|
403
|
+
|
|
404
|
+
|
|
380
405
|
def compute_risk(graph: Any, changed: list[ChangedSymbol]) -> PrRiskReport:
|
|
381
406
|
"""Aggregate blast radius, routes, cross-service callers, and v1 risk score.
|
|
382
407
|
|
|
@@ -415,7 +440,9 @@ def compute_risk(graph: Any, changed: list[ChangedSymbol]) -> PrRiskReport:
|
|
|
415
440
|
needle = _impact_needle_for_changed(graph, fqn, cs.kind)
|
|
416
441
|
ia = graph.impact_analysis(needle, depth=2, limit=400)
|
|
417
442
|
n = len(ia)
|
|
418
|
-
|
|
443
|
+
# Key blast radius by the symbol's FQN (agent-facing identifier), not
|
|
444
|
+
# its raw graph id — the report is operator-facing JSON.
|
|
445
|
+
blast_by[fqn or cs.symbol_id] = n
|
|
419
446
|
blast_total += n
|
|
420
447
|
|
|
421
448
|
for e in graph.find_callers(cs.fqn, depth=2, limit=400):
|
|
@@ -429,8 +456,11 @@ def compute_risk(graph: Any, changed: list[ChangedSymbol]) -> PrRiskReport:
|
|
|
429
456
|
cs_cross_service = 0
|
|
430
457
|
route_ids = _route_ids_for_symbol(graph, cs.symbol_id)
|
|
431
458
|
for rid in route_ids:
|
|
432
|
-
|
|
433
|
-
|
|
459
|
+
# Record the route by its natural identifier (METHOD path /
|
|
460
|
+
# topic:name), not the raw graph id.
|
|
461
|
+
label = _route_natural_id(graph, rid)
|
|
462
|
+
if label and label not in routes:
|
|
463
|
+
routes.append(label)
|
|
434
464
|
callers = graph._rows(
|
|
435
465
|
"MATCH (s:Symbol)-[:DECLARES_CLIENT]->(c:Client)-[e:HTTP_CALLS]->(r:Route {id: $rid}) "
|
|
436
466
|
"WHERE e.match = 'cross_service' "
|
|
@@ -84,7 +84,8 @@ _DTO_LOMBOK_ANNOTATIONS: frozenset[str] = frozenset({
|
|
|
84
84
|
# Phase 11: `EDGE_SCHEMA` in `java_ontology.py` (canonical edge navigation schema; v14 re-index).
|
|
85
85
|
# Phase 12: CALLS `callee_declaring_role`, supertype-walk dedup, pass3 unresolved counters (v15 re-index).
|
|
86
86
|
# Bumps whenever extraction / enrichment semantics change.
|
|
87
|
-
|
|
87
|
+
# Phase 13: Symbol.search_text + LadybugDB FTS (Okapi BM25) index for lexical search (v19 re-index).
|
|
88
|
+
ONTOLOGY_VERSION = 19
|
|
88
89
|
|
|
89
90
|
ROLE_ANNOTATIONS: dict[str, str] = {
|
|
90
91
|
# Spring Web
|
java_codebase_rag/cli.py
CHANGED
|
@@ -846,7 +846,15 @@ def _cmd_unresolved_calls_list(args: argparse.Namespace) -> int:
|
|
|
846
846
|
callee_simple=args.callee_simple,
|
|
847
847
|
limit=int(args.limit),
|
|
848
848
|
)
|
|
849
|
-
|
|
849
|
+
# Drop the raw caller symbol id: the row already carries the agent-facing
|
|
850
|
+
# ``caller_fqn``, so ``caller_id`` is redundant noise in an operator-facing
|
|
851
|
+
# report. The call-site ``id`` (``ucs:``) is kept — it's each site's
|
|
852
|
+
# primary key, not a caller reference.
|
|
853
|
+
sites = [
|
|
854
|
+
{k: v for k, v in row.items() if k != "caller_id"}
|
|
855
|
+
for row in rows
|
|
856
|
+
]
|
|
857
|
+
_emit({"success": True, "count": len(sites), "sites": sites})
|
|
850
858
|
return 0
|
|
851
859
|
|
|
852
860
|
|
|
@@ -1060,12 +1068,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
1060
1068
|
)
|
|
1061
1069
|
_add_index_embedding_flags(erase)
|
|
1062
1070
|
erase.add_argument("--yes", action="store_true", help="Confirm destructive deletion (required in CI)")
|
|
1063
|
-
erase
|
|
1064
|
-
"--quiet", "-q",
|
|
1065
|
-
action="store_true",
|
|
1066
|
-
dest="quiet",
|
|
1067
|
-
help="Suppress stderr progress relay; stdout payload unchanged.",
|
|
1068
|
-
)
|
|
1071
|
+
_add_verbosity_flags(erase)
|
|
1069
1072
|
erase.set_defaults(handler=_cmd_erase)
|
|
1070
1073
|
|
|
1071
1074
|
meta = subparsers.add_parser("meta", help="Print graph meta and embedding resolution.")
|
|
@@ -1162,8 +1165,20 @@ def _console_script_main() -> None:
|
|
|
1162
1165
|
racy teardown — the command has already done its work and emitted its result.
|
|
1163
1166
|
``main()`` stays return-based so in-process test callers (``cli.main(...)``)
|
|
1164
1167
|
keep working.
|
|
1168
|
+
|
|
1169
|
+
``KeyboardInterrupt`` (Ctrl+C during a long indexing step) is caught here
|
|
1170
|
+
rather than left to propagate: an uncaught interrupt bypasses this function
|
|
1171
|
+
and runs full interpreter finalization (traceback + thread teardown),
|
|
1172
|
+
whereas routing it through the same flush + ``os._exit`` path gives a clean,
|
|
1173
|
+
immediate exit (code 130) and avoids the finalization-time SIGABRT noted
|
|
1174
|
+
above for commands that loaded lancedb.
|
|
1165
1175
|
"""
|
|
1166
|
-
|
|
1176
|
+
try:
|
|
1177
|
+
rc = main()
|
|
1178
|
+
except KeyboardInterrupt:
|
|
1179
|
+
sys.stderr.write("\nInterrupted.\n")
|
|
1180
|
+
sys.stderr.flush()
|
|
1181
|
+
rc = 130
|
|
1167
1182
|
sys.stdout.flush()
|
|
1168
1183
|
sys.stderr.flush()
|
|
1169
1184
|
os._exit(rc)
|
java_codebase_rag/config.py
CHANGED
|
@@ -378,6 +378,14 @@ class ResolvedOperatorConfig:
|
|
|
378
378
|
# used with no config file). Recorded into the index dir at index time so a
|
|
379
379
|
# later discovery run from a sibling/cwd can relocate this config.
|
|
380
380
|
yaml_config_path: Path | None = None
|
|
381
|
+
# ``watch:`` block knobs (jrag watch / watcher). Defaults make the block
|
|
382
|
+
# optional; no env vars are introduced for these (CLI flag > YAML > default).
|
|
383
|
+
watch_debounce_ms: int = 1500
|
|
384
|
+
watch_backend: str = "auto"
|
|
385
|
+
watch_poll_interval_ms: int = 2000
|
|
386
|
+
watch_debounce_ms_source: SettingSource = "default"
|
|
387
|
+
watch_backend_source: SettingSource = "default"
|
|
388
|
+
watch_poll_interval_ms_source: SettingSource = "default"
|
|
381
389
|
|
|
382
390
|
def apply_to_os_environ(self) -> None:
|
|
383
391
|
"""Make downstream modules (server, ladybug_queries, flows) see a consistent environment.
|
|
@@ -506,16 +514,20 @@ def _pick_float(
|
|
|
506
514
|
|
|
507
515
|
def _pick_int(
|
|
508
516
|
*,
|
|
517
|
+
cli_val: int | None = None,
|
|
509
518
|
env_key: str,
|
|
510
519
|
yaml_dict: dict[str, Any],
|
|
511
520
|
yaml_path: tuple[str, ...],
|
|
512
521
|
default: int,
|
|
513
522
|
) -> tuple[int, SettingSource]:
|
|
514
|
-
"""Pick an int setting from env (parsed via int(...)), YAML, or default.
|
|
523
|
+
"""Pick an int setting from CLI, env (parsed via int(...)), YAML, or default.
|
|
515
524
|
|
|
516
525
|
Precedence: CLI > env > YAML > default. Env values that fail to parse as int
|
|
517
526
|
fall back to the default (matching the brief's requirement for graceful degradation).
|
|
527
|
+
``cli_val`` defaults to ``None`` so existing callers are unaffected.
|
|
518
528
|
"""
|
|
529
|
+
if cli_val is not None:
|
|
530
|
+
return int(cli_val), "cli"
|
|
519
531
|
env_raw = os.environ.get(env_key, "").strip()
|
|
520
532
|
if env_raw:
|
|
521
533
|
try:
|
|
@@ -577,6 +589,9 @@ def resolve_operator_config(
|
|
|
577
589
|
cli_index_dir: str | None = None,
|
|
578
590
|
cli_embedding_model: str | None = None,
|
|
579
591
|
cli_embedding_device: str | None = None,
|
|
592
|
+
cli_watch_debounce_ms: int | None = None,
|
|
593
|
+
cli_watch_backend: str | None = None,
|
|
594
|
+
cli_watch_poll_interval_ms: int | None = None,
|
|
580
595
|
) -> ResolvedOperatorConfig:
|
|
581
596
|
# Phase 1: Find the config file directory
|
|
582
597
|
if source_root is not None:
|
|
@@ -671,6 +686,52 @@ def resolve_operator_config(
|
|
|
671
686
|
yaml_path=("absence", "diag_enabled"),
|
|
672
687
|
default=True,
|
|
673
688
|
)
|
|
689
|
+
# ``watch:`` block knobs. No env vars are introduced for watch (CLI > YAML >
|
|
690
|
+
# default), so an empty env_key is passed to the ``_pick_*`` helpers —
|
|
691
|
+
# ``os.environ.get("", "")`` never matches, leaving the env tier inert.
|
|
692
|
+
w_debounce, w_debounce_src = _pick_int(
|
|
693
|
+
cli_val=cli_watch_debounce_ms,
|
|
694
|
+
env_key="",
|
|
695
|
+
yaml_dict=yaml_dict,
|
|
696
|
+
yaml_path=("watch", "debounce_ms"),
|
|
697
|
+
default=1500,
|
|
698
|
+
)
|
|
699
|
+
w_backend, w_backend_src = _pick_str(
|
|
700
|
+
cli_val=cli_watch_backend,
|
|
701
|
+
env_key="",
|
|
702
|
+
yaml_dict=yaml_dict,
|
|
703
|
+
yaml_path=("watch", "backend"),
|
|
704
|
+
default="auto",
|
|
705
|
+
)
|
|
706
|
+
w_poll, w_poll_src = _pick_int(
|
|
707
|
+
cli_val=cli_watch_poll_interval_ms,
|
|
708
|
+
env_key="",
|
|
709
|
+
yaml_dict=yaml_dict,
|
|
710
|
+
yaml_path=("watch", "poll_interval_ms"),
|
|
711
|
+
default=2000,
|
|
712
|
+
)
|
|
713
|
+
# Inline floors/validation (mirror the existing graceful-degradation style).
|
|
714
|
+
if w_debounce < 100:
|
|
715
|
+
print(
|
|
716
|
+
f"java-codebase-rag: watch.debounce_ms={w_debounce} is below the 100 ms "
|
|
717
|
+
"floor; falling back to 1500.",
|
|
718
|
+
file=sys.stderr,
|
|
719
|
+
)
|
|
720
|
+
w_debounce, w_debounce_src = 1500, "default"
|
|
721
|
+
if w_backend not in ("auto", "watchdog", "polling"):
|
|
722
|
+
print(
|
|
723
|
+
f"java-codebase-rag: watch.backend={w_backend!r} is not one of "
|
|
724
|
+
"auto/watchdog/polling; falling back to 'auto'.",
|
|
725
|
+
file=sys.stderr,
|
|
726
|
+
)
|
|
727
|
+
w_backend, w_backend_src = "auto", "default"
|
|
728
|
+
if w_poll < 200:
|
|
729
|
+
print(
|
|
730
|
+
f"java-codebase-rag: watch.poll_interval_ms={w_poll} is below the 200 ms "
|
|
731
|
+
"floor; falling back to 2000.",
|
|
732
|
+
file=sys.stderr,
|
|
733
|
+
)
|
|
734
|
+
w_poll, w_poll_src = 2000, "default"
|
|
674
735
|
ku = index_dir / "code_graph.lbug"
|
|
675
736
|
coco = index_dir / "cocoindex.db"
|
|
676
737
|
return ResolvedOperatorConfig(
|
|
@@ -696,6 +757,12 @@ def resolve_operator_config(
|
|
|
696
757
|
absence_ngram_q_source=abs_q_src,
|
|
697
758
|
absence_diag_enabled_source=abs_diag_src,
|
|
698
759
|
yaml_config_path=find_yaml_config_file(config_dir),
|
|
760
|
+
watch_debounce_ms=w_debounce,
|
|
761
|
+
watch_backend=w_backend,
|
|
762
|
+
watch_poll_interval_ms=w_poll,
|
|
763
|
+
watch_debounce_ms_source=w_debounce_src,
|
|
764
|
+
watch_backend_source=w_backend_src,
|
|
765
|
+
watch_poll_interval_ms_source=w_poll_src,
|
|
699
766
|
)
|
|
700
767
|
|
|
701
768
|
|
|
@@ -68,6 +68,7 @@ from java_codebase_rag.graph.graph_enrich import (
|
|
|
68
68
|
symbol_id,
|
|
69
69
|
)
|
|
70
70
|
from java_codebase_rag.graph.path_filtering import LayeredIgnore, iter_java_source_files
|
|
71
|
+
from java_codebase_rag.search.search_scoring import SYMBOL_FTS_INDEX as _SYMBOL_FTS_INDEX, _split_identifier
|
|
71
72
|
from java_codebase_rag.graph.java_ontology import (
|
|
72
73
|
CLIENT_KIND_FEIGN_METHOD,
|
|
73
74
|
CLIENT_KIND_REST_TEMPLATE,
|
|
@@ -801,6 +802,12 @@ def _delete_file_scope(
|
|
|
801
802
|
Producer nodes use DETACH DELETE as a safety net for any edges missed in
|
|
802
803
|
Phase 1.
|
|
803
804
|
"""
|
|
805
|
+
# Symbol DELETEs below maintain the FTS index, which needs the extension loaded on
|
|
806
|
+
# this connection. Idempotent + best-effort (no-op when no index exists / FTS absent).
|
|
807
|
+
try:
|
|
808
|
+
conn.execute("LOAD EXTENSION FTS")
|
|
809
|
+
except Exception:
|
|
810
|
+
pass
|
|
804
811
|
scope_files = changed_files | dependent_files
|
|
805
812
|
scope_list = list(scope_files)
|
|
806
813
|
changed_list = list(changed_files)
|
|
@@ -2927,7 +2934,8 @@ _SCHEMA_NODE = (
|
|
|
2927
2934
|
"start_byte INT64, end_byte INT64, "
|
|
2928
2935
|
"modifiers STRING[], annotations STRING[], capabilities STRING[], "
|
|
2929
2936
|
"role STRING, signature STRING, parent_id STRING, resolved BOOLEAN, "
|
|
2930
|
-
"generated BOOLEAN, generated_by STRING"
|
|
2937
|
+
"generated BOOLEAN, generated_by STRING, "
|
|
2938
|
+
"search_text STRING"
|
|
2931
2939
|
")"
|
|
2932
2940
|
)
|
|
2933
2941
|
|
|
@@ -3050,7 +3058,28 @@ _SCHEMA_ASYNC_CALLS = (
|
|
|
3050
3058
|
)
|
|
3051
3059
|
|
|
3052
3060
|
|
|
3061
|
+
def _drop_symbol_fts_index_if_present(conn: ladybug.Connection) -> None:
|
|
3062
|
+
"""Drop the Symbol FTS index so the table drop below succeeds.
|
|
3063
|
+
|
|
3064
|
+
LadybugDB refuses ``DROP TABLE Symbol`` while an FTS index references it ("Cannot
|
|
3065
|
+
delete node table ... referenced by index"), so the index must go first. No-op when
|
|
3066
|
+
FTS is unavailable (offline first-run) or the index was never created — in those
|
|
3067
|
+
cases nothing blocks the table drop. Best-effort: any failure is swallowed.
|
|
3068
|
+
"""
|
|
3069
|
+
try:
|
|
3070
|
+
conn.execute("LOAD EXTENSION FTS")
|
|
3071
|
+
existing = conn.execute("CALL SHOW_INDEXES() RETURN index_name")
|
|
3072
|
+
names: set[str] = set()
|
|
3073
|
+
while existing.has_next():
|
|
3074
|
+
names.add(existing.get_next()[0])
|
|
3075
|
+
if _SYMBOL_FTS_INDEX in names:
|
|
3076
|
+
conn.execute(f"CALL DROP_FTS_INDEX('Symbol', '{_SYMBOL_FTS_INDEX}')")
|
|
3077
|
+
except Exception:
|
|
3078
|
+
pass
|
|
3079
|
+
|
|
3080
|
+
|
|
3053
3081
|
def _drop_all(conn: ladybug.Connection) -> None:
|
|
3082
|
+
_drop_symbol_fts_index_if_present(conn)
|
|
3054
3083
|
for stmt in (
|
|
3055
3084
|
"DROP TABLE IF EXISTS DECLARES_CLIENT",
|
|
3056
3085
|
"DROP TABLE IF EXISTS DECLARES_PRODUCER",
|
|
@@ -3101,6 +3130,80 @@ def _create_schema(conn: ladybug.Connection) -> None:
|
|
|
3101
3130
|
conn.execute(stmt)
|
|
3102
3131
|
|
|
3103
3132
|
|
|
3133
|
+
_IDENT_RE = re.compile(r"[A-Za-z][A-Za-z0-9]*")
|
|
3134
|
+
|
|
3135
|
+
|
|
3136
|
+
def _compute_symbol_search_text(
|
|
3137
|
+
name: str,
|
|
3138
|
+
fqn: str,
|
|
3139
|
+
signature: str,
|
|
3140
|
+
annotations: list[str],
|
|
3141
|
+
capabilities: list[str],
|
|
3142
|
+
package: str = "",
|
|
3143
|
+
) -> str:
|
|
3144
|
+
"""Build the BM25-indexable token soup for a Symbol (fork A).
|
|
3145
|
+
|
|
3146
|
+
camelCase / snake_case identifiers are split into lowercase tokens via the SAME
|
|
3147
|
+
``_split_identifier`` the query path uses (index- and query-time tokenization
|
|
3148
|
+
must agree), then space-joined into one STRING that LadybugDB FTS porter-stems.
|
|
3149
|
+
Name + fqn carry the strongest discovery signal; signature / annotations /
|
|
3150
|
+
capabilities / package segments are weaker corroborators. Members reach this via
|
|
3151
|
+
the same ``_node_row`` constructor as types.
|
|
3152
|
+
"""
|
|
3153
|
+
toks: list[str] = []
|
|
3154
|
+
for field in (name, fqn, signature, package):
|
|
3155
|
+
for m in _IDENT_RE.findall(str(field or "")):
|
|
3156
|
+
toks.extend(_split_identifier(m))
|
|
3157
|
+
for lst in (annotations, capabilities):
|
|
3158
|
+
for item in (lst or []):
|
|
3159
|
+
for m in _IDENT_RE.findall(str(item)):
|
|
3160
|
+
toks.extend(_split_identifier(m))
|
|
3161
|
+
seen: set[str] = set()
|
|
3162
|
+
out: list[str] = []
|
|
3163
|
+
for t in toks:
|
|
3164
|
+
if len(t) >= 2 and t not in seen:
|
|
3165
|
+
seen.add(t)
|
|
3166
|
+
out.append(t)
|
|
3167
|
+
return " ".join(out)
|
|
3168
|
+
|
|
3169
|
+
|
|
3170
|
+
def _ensure_symbol_fts_index(conn: ladybug.Connection, *, verbose: bool) -> None:
|
|
3171
|
+
"""Best-effort create the Symbol FTS (Okapi BM25) index if absent (fork A).
|
|
3172
|
+
|
|
3173
|
+
Idempotent: skips when the index already exists. The FTS extension is fetched
|
|
3174
|
+
from extension.ladybugdb.com on first ``INSTALL FTS`` (cached locally after); an
|
|
3175
|
+
offline first run fails softly — ``run_lexical_search`` then falls back to the
|
|
3176
|
+
heuristic over the bare Symbol scan. The index auto-maintains on later
|
|
3177
|
+
COPY / MERGE / DELETE (verified), so this only runs on full builds and as a
|
|
3178
|
+
cheap self-heal at the end of an incremental rebuild.
|
|
3179
|
+
"""
|
|
3180
|
+
try:
|
|
3181
|
+
try:
|
|
3182
|
+
conn.execute("INSTALL FTS") # already-installed raises; swallow
|
|
3183
|
+
except Exception:
|
|
3184
|
+
pass
|
|
3185
|
+
conn.execute("LOAD EXTENSION FTS")
|
|
3186
|
+
existing = conn.execute("CALL SHOW_INDEXES() RETURN index_name")
|
|
3187
|
+
names: set[str] = set()
|
|
3188
|
+
while existing.has_next():
|
|
3189
|
+
names.add(existing.get_next()[0])
|
|
3190
|
+
if _SYMBOL_FTS_INDEX not in names:
|
|
3191
|
+
conn.execute(
|
|
3192
|
+
f"CALL CREATE_FTS_INDEX('Symbol', '{_SYMBOL_FTS_INDEX}', "
|
|
3193
|
+
"['search_text'], stemmer := 'porter')"
|
|
3194
|
+
)
|
|
3195
|
+
if verbose:
|
|
3196
|
+
_verbose_stderr_line(
|
|
3197
|
+
f"[graph] fts · created Symbol {_SYMBOL_FTS_INDEX} (BM25 over search_text)"
|
|
3198
|
+
)
|
|
3199
|
+
except Exception as e:
|
|
3200
|
+
if verbose:
|
|
3201
|
+
_verbose_stderr_line(
|
|
3202
|
+
f"[graph] fts · unavailable ({type(e).__name__}: {e}); "
|
|
3203
|
+
"lexical search will use the heuristic scan"
|
|
3204
|
+
)
|
|
3205
|
+
|
|
3206
|
+
|
|
3104
3207
|
def _node_row(**kwargs) -> dict:
|
|
3105
3208
|
base = {
|
|
3106
3209
|
"kind": "", "name": "", "fqn": "", "package": "",
|
|
@@ -3109,9 +3212,16 @@ def _node_row(**kwargs) -> dict:
|
|
|
3109
3212
|
"start_byte": 0, "end_byte": 0,
|
|
3110
3213
|
"modifiers": [], "annotations": [], "capabilities": [],
|
|
3111
3214
|
"role": "OTHER", "signature": "", "parent_id": "", "resolved": True,
|
|
3112
|
-
"generated": False, "generated_by": None,
|
|
3215
|
+
"generated": False, "generated_by": None, "search_text": "",
|
|
3113
3216
|
}
|
|
3114
3217
|
base.update(kwargs)
|
|
3218
|
+
# Derive the BM25 token soup unless the caller set it explicitly.
|
|
3219
|
+
if not base.get("search_text"):
|
|
3220
|
+
base["search_text"] = _compute_symbol_search_text(
|
|
3221
|
+
base.get("name", ""), base.get("fqn", ""), base.get("signature", ""),
|
|
3222
|
+
base.get("annotations", []), base.get("capabilities", []),
|
|
3223
|
+
base.get("package", ""),
|
|
3224
|
+
)
|
|
3115
3225
|
return base
|
|
3116
3226
|
|
|
3117
3227
|
|
|
@@ -3161,7 +3271,7 @@ _NODE_COLUMNS = [
|
|
|
3161
3271
|
"id", "kind", "name", "fqn", "package", "module", "microservice",
|
|
3162
3272
|
"filename", "start_line", "end_line", "start_byte", "end_byte",
|
|
3163
3273
|
"modifiers", "annotations", "capabilities", "role", "signature", "parent_id", "resolved",
|
|
3164
|
-
"generated", "generated_by"
|
|
3274
|
+
"generated", "generated_by", "search_text"
|
|
3165
3275
|
]
|
|
3166
3276
|
|
|
3167
3277
|
# Type declaration kinds. Tuple (not set) so the rendered SQL `IN` clause is
|
|
@@ -3185,7 +3295,7 @@ _SET_SYMBOL_BY_ID = (
|
|
|
3185
3295
|
"n.modifiers = $modifiers, n.annotations = $annotations, "
|
|
3186
3296
|
"n.capabilities = $capabilities, n.role = $role, "
|
|
3187
3297
|
"n.signature = $signature, n.parent_id = $parent_id, n.resolved = $resolved, "
|
|
3188
|
-
"n.generated = $generated, n.generated_by = $generated_by"
|
|
3298
|
+
"n.generated = $generated, n.generated_by = $generated_by, n.search_text = $search_text"
|
|
3189
3299
|
)
|
|
3190
3300
|
|
|
3191
3301
|
# Refresh every mutable Route field on an existing Route node by id. Mirrors the
|
|
@@ -3835,6 +3945,13 @@ def incremental_rebuild(
|
|
|
3835
3945
|
|
|
3836
3946
|
db = ladybug.Database(str(ladybug_path))
|
|
3837
3947
|
conn = ladybug.Connection(db)
|
|
3948
|
+
# If a Symbol FTS index exists, the scoped DELETE/MERGE/COPY below must maintain it,
|
|
3949
|
+
# which needs the FTS extension loaded on THIS connection. Best-effort: when FTS is
|
|
3950
|
+
# unavailable no index exists, so there's nothing to maintain and DML proceeds fine.
|
|
3951
|
+
try:
|
|
3952
|
+
conn.execute("LOAD EXTENSION FTS")
|
|
3953
|
+
except Exception:
|
|
3954
|
+
pass
|
|
3838
3955
|
|
|
3839
3956
|
# Check ontology version
|
|
3840
3957
|
try:
|
|
@@ -4007,6 +4124,7 @@ def incremental_rebuild(
|
|
|
4007
4124
|
|
|
4008
4125
|
# Update GraphMeta
|
|
4009
4126
|
_write_meta(conn, tables_for_global, source_root)
|
|
4127
|
+
_ensure_symbol_fts_index(conn, verbose=verbose)
|
|
4010
4128
|
|
|
4011
4129
|
# Remove crash marker
|
|
4012
4130
|
crash_marker_path.unlink(missing_ok=True)
|
|
@@ -4236,6 +4354,7 @@ def write_ladybug(
|
|
|
4236
4354
|
if verbose:
|
|
4237
4355
|
_verbose_stderr_line(f"[graph] writing · routes/exposes written in {time.time() - t2:.2f}s")
|
|
4238
4356
|
_write_meta(conn, tables, source_root)
|
|
4357
|
+
_ensure_symbol_fts_index(conn, verbose=verbose)
|
|
4239
4358
|
conn.close()
|
|
4240
4359
|
db.close()
|
|
4241
4360
|
|
|
@@ -36,6 +36,24 @@ class NodeRef(BaseModel):
|
|
|
36
36
|
role: str | None = None
|
|
37
37
|
generated: bool | None = None
|
|
38
38
|
generated_by: str | None = None
|
|
39
|
+
# Identity-adjacent fields consumed by ``jrag_envelope.node_key`` (keying)
|
|
40
|
+
# and ``project_node`` allow-lists (rendering) so find filter-mode nodes
|
|
41
|
+
# carry the same rich shape as the dedicated listings (http-routes /
|
|
42
|
+
# http-clients / producers). All optional with default None: existing
|
|
43
|
+
# NodeRef constructors are unaffected and ``_drop_empty`` strips unset
|
|
44
|
+
# fields at the JSON boundary.
|
|
45
|
+
method: str | None = None
|
|
46
|
+
path: str | None = None
|
|
47
|
+
framework: str | None = None
|
|
48
|
+
member_fqn: str | None = None
|
|
49
|
+
target_service: str | None = None
|
|
50
|
+
client_kind: str | None = None
|
|
51
|
+
topic: str | None = None
|
|
52
|
+
broker: str | None = None
|
|
53
|
+
producer_kind: str | None = None
|
|
54
|
+
filename: str | None = None
|
|
55
|
+
start_line: int | None = None
|
|
56
|
+
resolved: bool | None = None
|
|
39
57
|
|
|
40
58
|
|
|
41
59
|
class StructuredHint(BaseModel):
|
|
@@ -98,38 +116,107 @@ def _resolve_node_kind(
|
|
|
98
116
|
|
|
99
117
|
|
|
100
118
|
def _node_ref_from_row(kind: Literal["symbol", "route", "client", "producer"], row: dict[str, Any]) -> NodeRef:
|
|
101
|
-
|
|
119
|
+
"""Map a graph store row to a :class:`NodeRef`.
|
|
120
|
+
|
|
121
|
+
``fqn`` is set to each kind's documented natural identifier (README
|
|
122
|
+
§"jrag — agent CLI") so ``jrag_envelope.node_key`` (which checks ``fqn``
|
|
123
|
+
first) keys nodes correctly and no raw graph id leaks:
|
|
124
|
+
|
|
125
|
+
* symbol -> symbol fqn
|
|
126
|
+
* route -> ``"METHOD path"`` (HTTP endpoint); ``"topic:<name>"`` when a
|
|
127
|
+
kafka topic surfaces as :Route; ``""`` for a phantom route
|
|
128
|
+
(node_key then falls through to the composed ``file``)
|
|
129
|
+
* client -> ``"member_fqn->target_service"``
|
|
130
|
+
* producer-> ``"topic:<name>"``
|
|
131
|
+
|
|
132
|
+
Rich detail (member_fqn / target_service / method / path / topic /
|
|
133
|
+
framework / filename / start_line / resolved) is populated alongside so
|
|
134
|
+
find filter-mode nodes render with the same shape as the dedicated
|
|
135
|
+
listings (``http-routes`` / ``http-clients`` / ``producers``) instead of
|
|
136
|
+
collapsing to ``{"kind": ...}`` after the envelope's projection+drop-empty.
|
|
137
|
+
"""
|
|
138
|
+
microservice = str(row.get("microservice") or "") or None
|
|
139
|
+
module = str(row.get("module") or "") or None
|
|
140
|
+
filename = str(row.get("filename") or "") or None
|
|
141
|
+
start_line = row.get("start_line")
|
|
142
|
+
try:
|
|
143
|
+
start_line = int(start_line) if start_line not in (None, "") else None
|
|
144
|
+
except (TypeError, ValueError):
|
|
145
|
+
start_line = None
|
|
146
|
+
resolved_raw = row.get("resolved")
|
|
147
|
+
resolved = bool(resolved_raw) if resolved_raw is not None else None
|
|
148
|
+
nid = str(row.get("id") or "")
|
|
149
|
+
|
|
102
150
|
if kind == "symbol":
|
|
103
151
|
fqn = str(row.get("fqn") or "")
|
|
104
152
|
role = str(row.get("role") or "") or None
|
|
105
153
|
symbol_kind_val = str(row.get("symbol_kind") or row.get("kind") or "").strip()
|
|
106
154
|
symbol_kind = symbol_kind_val or None
|
|
107
|
-
|
|
155
|
+
return NodeRef(
|
|
156
|
+
id=nid, kind="symbol", fqn=fqn,
|
|
157
|
+
name=str(row.get("name") or "") or None,
|
|
158
|
+
symbol_kind=symbol_kind,
|
|
159
|
+
microservice=microservice, module=module, role=role,
|
|
160
|
+
filename=filename, start_line=start_line,
|
|
161
|
+
generated=bool(row.get("generated")) if row.get("generated") is not None else None,
|
|
162
|
+
generated_by=str(row.get("generated_by")) if row.get("generated_by") else None,
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
if kind == "route":
|
|
108
166
|
method = str(row.get("method") or "")
|
|
109
167
|
path = str(row.get("path_template") or row.get("path") or "")
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
168
|
+
topic = str(row.get("topic") or "") or None
|
|
169
|
+
if method or path:
|
|
170
|
+
fqn = f"{method} {path}".strip()
|
|
171
|
+
elif topic:
|
|
172
|
+
fqn = f"topic:{topic}"
|
|
173
|
+
else:
|
|
174
|
+
fqn = ""
|
|
175
|
+
return NodeRef(
|
|
176
|
+
id=nid, kind="route", fqn=fqn, name=(path or topic),
|
|
177
|
+
method=method or None, path=path or None, topic=topic,
|
|
178
|
+
framework=str(row.get("framework") or "") or None,
|
|
179
|
+
microservice=microservice, module=module,
|
|
180
|
+
filename=filename, start_line=start_line, resolved=resolved,
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
if kind == "client":
|
|
184
|
+
mfqn = str(row.get("member_fqn") or "")
|
|
185
|
+
tgt = str(row.get("target_service") or "")
|
|
113
186
|
method = str(row.get("method") or "")
|
|
114
|
-
target = str(row.get("target_service") or "")
|
|
115
187
|
path = str(row.get("path_template") or row.get("path") or "")
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
188
|
+
member_fqn = mfqn or None
|
|
189
|
+
target_service = tgt or None
|
|
190
|
+
member_simple = (mfqn.rsplit(".", 1)[-1] if mfqn else "") or None
|
|
191
|
+
# Build the contract id ``member_fqn->target_service`` from the RAW
|
|
192
|
+
# strings: ``member_fqn`` was normalized to None above, so interpolating
|
|
193
|
+
# it would emit ``"None-><target>"`` for member-less (brownfield/meta)
|
|
194
|
+
# clients. Fall back to whichever half is present; never a dangling arrow.
|
|
195
|
+
fqn = f"{mfqn}->{tgt}" if (mfqn and tgt) else (mfqn or tgt)
|
|
196
|
+
return NodeRef(
|
|
197
|
+
id=nid, kind="client",
|
|
198
|
+
fqn=fqn,
|
|
199
|
+
name=member_simple,
|
|
200
|
+
member_fqn=member_fqn, target_service=target_service,
|
|
201
|
+
method=method or None, path=path or None,
|
|
202
|
+
client_kind=str(row.get("client_kind") or "") or None,
|
|
203
|
+
microservice=microservice, module=module,
|
|
204
|
+
filename=filename, start_line=start_line, resolved=resolved,
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
# producer
|
|
208
|
+
topic = str(row.get("topic") or "") or None
|
|
209
|
+
member_fqn = str(row.get("member_fqn") or "") or None
|
|
210
|
+
member_simple = (member_fqn.rsplit(".", 1)[-1] if member_fqn else "") or None
|
|
123
211
|
return NodeRef(
|
|
124
|
-
id=
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
generated_by=str(row.get("generated_by")) if row.get("generated_by") else None,
|
|
212
|
+
id=nid, kind="producer",
|
|
213
|
+
fqn=(f"topic:{topic}" if topic else ""),
|
|
214
|
+
name=(topic or member_simple),
|
|
215
|
+
topic=topic, broker=str(row.get("broker") or "") or None,
|
|
216
|
+
producer_kind=str(row.get("producer_kind") or "") or None,
|
|
217
|
+
member_fqn=member_fqn,
|
|
218
|
+
microservice=microservice, module=module,
|
|
219
|
+
filename=filename, start_line=start_line, resolved=resolved,
|
|
133
220
|
)
|
|
134
221
|
|
|
135
222
|
|
|
@@ -396,6 +396,22 @@ class LadybugGraph:
|
|
|
396
396
|
# Ladybug represents DB as a directory; allow file form too (single-file DBs).
|
|
397
397
|
return True
|
|
398
398
|
|
|
399
|
+
@classmethod
|
|
400
|
+
def reset_for_path(cls, db_path: str | None) -> None:
|
|
401
|
+
"""Drop the cached singleton so the next ``get`` reopens.
|
|
402
|
+
|
|
403
|
+
Clears the cache when ``db_path is None`` (unconditional) or when ``db_path``
|
|
404
|
+
resolves to the currently cached instance path (resolved the same way
|
|
405
|
+
``get`` resolves it, so ``~``-relative / non-normalized paths match); a
|
|
406
|
+
non-matching ``db_path`` is a no-op. The watch daemon uses this to manage
|
|
407
|
+
its copy-on-write graph snapshot lifecycle (drop the original reader
|
|
408
|
+
before a subprocess reindex writes it; drop the sidecar reader on commit).
|
|
409
|
+
"""
|
|
410
|
+
with cls._lock:
|
|
411
|
+
if db_path is None or cls._instance_path == resolve_ladybug_path(db_path):
|
|
412
|
+
cls._instance = None
|
|
413
|
+
cls._instance_path = None
|
|
414
|
+
|
|
399
415
|
# ---- low-level ----
|
|
400
416
|
|
|
401
417
|
def _rows(self, query: str, params: dict[str, Any] | None = None) -> list[dict[str, Any]]:
|
|
@@ -997,8 +1013,35 @@ class LadybugGraph:
|
|
|
997
1013
|
def find_by_name_or_fqn(self, name_or_fqn: str, *, kinds: list[str] | None = None,
|
|
998
1014
|
module: str | None = None,
|
|
999
1015
|
microservice: str | None = None,
|
|
1000
|
-
limit: int = 50
|
|
1001
|
-
|
|
1016
|
+
limit: int = 50,
|
|
1017
|
+
mode: str = "exact") -> list[SymbolHit]:
|
|
1018
|
+
# ``mode`` selects the name/FQN predicate. ``exact`` (default) preserves the
|
|
1019
|
+
# original ``s.name = $needle OR s.fqn = $needle``. ``prefix`` / ``contains``
|
|
1020
|
+
# use STARTS WITH / CONTAINS (Ladybug Cypher supports both — see
|
|
1021
|
+
# resolve_service.py); they back the ``find --fuzzy`` fallback (issue #375).
|
|
1022
|
+
# Fuzzy modes additionally exclude file/package Symbol nodes: their fqn is a
|
|
1023
|
+
# filesystem path, so a substring/prefix would leak filename rows (mirrors
|
|
1024
|
+
# the find_v2 fix, #411). Exact mode is unchanged for back-compat.
|
|
1025
|
+
# Empty needle: STARTS WITH '' / CONTAINS '' match every string, so a
|
|
1026
|
+
# fuzzy mode would silently return up to `limit` arbitrary Symbols. The
|
|
1027
|
+
# CLI guards this (query mode requires a positional), but keep the
|
|
1028
|
+
# backend safe-by-construction for any future caller.
|
|
1029
|
+
if mode != "exact" and not name_or_fqn:
|
|
1030
|
+
return []
|
|
1031
|
+
if mode == "exact":
|
|
1032
|
+
filters = ["(s.name = $needle OR s.fqn = $needle)"]
|
|
1033
|
+
elif mode == "prefix":
|
|
1034
|
+
filters = [
|
|
1035
|
+
"(s.name STARTS WITH $needle OR s.fqn STARTS WITH $needle)",
|
|
1036
|
+
"(s.kind <> 'file' AND s.kind <> 'package')",
|
|
1037
|
+
]
|
|
1038
|
+
elif mode == "contains":
|
|
1039
|
+
filters = [
|
|
1040
|
+
"(s.name CONTAINS $needle OR s.fqn CONTAINS $needle)",
|
|
1041
|
+
"(s.kind <> 'file' AND s.kind <> 'package')",
|
|
1042
|
+
]
|
|
1043
|
+
else:
|
|
1044
|
+
raise ValueError(f"unknown find_by_name_or_fqn mode: {mode!r}")
|
|
1002
1045
|
params: dict[str, Any] = {"needle": name_or_fqn}
|
|
1003
1046
|
if kinds:
|
|
1004
1047
|
params["kinds"] = kinds
|