java-codebase-rag 0.6.2__py3-none-any.whl → 0.6.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- build_ast_graph.py +95 -7
- java_codebase_rag/cli.py +87 -14
- java_codebase_rag/cli_format.py +8 -35
- java_codebase_rag/cli_progress.py +17 -21
- java_codebase_rag/installer.py +180 -43
- java_codebase_rag/lance_optimize.py +125 -51
- java_codebase_rag/pipeline.py +71 -23
- java_codebase_rag/progress.py +570 -0
- {java_codebase_rag-0.6.2.dist-info → java_codebase_rag-0.6.3.dist-info}/METADATA +4 -1
- {java_codebase_rag-0.6.2.dist-info → java_codebase_rag-0.6.3.dist-info}/RECORD +16 -15
- java_index_flow_lancedb.py +155 -0
- server.py +75 -9
- {java_codebase_rag-0.6.2.dist-info → java_codebase_rag-0.6.3.dist-info}/WHEEL +0 -0
- {java_codebase_rag-0.6.2.dist-info → java_codebase_rag-0.6.3.dist-info}/entry_points.txt +0 -0
- {java_codebase_rag-0.6.2.dist-info → java_codebase_rag-0.6.3.dist-info}/licenses/LICENSE +0 -0
- {java_codebase_rag-0.6.2.dist-info → java_codebase_rag-0.6.3.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,570 @@
|
|
|
1
|
+
"""JCIRAG_PROGRESS protocol: parse, render, and relay subprocess progress.
|
|
2
|
+
|
|
3
|
+
The five lifecycle commands (``init`` / ``increment`` / ``install`` /
|
|
4
|
+
``reprocess`` / ``update``) drive two subprocesses — ``cocoindex update``
|
|
5
|
+
(vectors → Lance) and ``build_ast_graph.py`` (graph → LadybugDB) — whose
|
|
6
|
+
stderr output is inconsistent and shows no real progress. This module is the
|
|
7
|
+
foundation of a unified progress surface.
|
|
8
|
+
|
|
9
|
+
A subprocess prints lines of the form::
|
|
10
|
+
|
|
11
|
+
JCIRAG_PROGRESS kind=vectors phase=embed done=842 total=1240
|
|
12
|
+
JCIRAG_PROGRESS kind=graph phase=build pass=3/6 done=120 total=600 status=running
|
|
13
|
+
JCIRAG_PROGRESS kind=optimize status=done elapsed_s=42.1 total=1240
|
|
14
|
+
|
|
15
|
+
to its **stderr**. The parent (``pipeline._LineFilter`` /
|
|
16
|
+
``cli_progress._AsyncLineFilter`` drain path) feeds each stderr chunk to a
|
|
17
|
+
:class:`ProgressRelay`, which:
|
|
18
|
+
|
|
19
|
+
1. buffers chunks and splits on ``\\n`` (mirrors the existing line filters),
|
|
20
|
+
2. parses each complete line with :func:`parse_progress_line`,
|
|
21
|
+
3. if it is a progress line → forwards a :class:`ProgressEvent` to an
|
|
22
|
+
:class:`IndexProgressRenderer` and **suppresses** the raw line, else
|
|
23
|
+
4. runs the existing noise matcher (``is_noise_line``) and routes the
|
|
24
|
+
surviving line to the active sink (console / raw stderr / drop).
|
|
25
|
+
|
|
26
|
+
On a TTY the renderer drives a ``rich.progress.Live`` region; off-TTY it
|
|
27
|
+
prints concise throttled lines (one per phase, ~every 5 s, plus a final
|
|
28
|
+
terminal line). This module is pure library — no production caller is wired
|
|
29
|
+
in yet.
|
|
30
|
+
"""
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import sys
|
|
34
|
+
import time
|
|
35
|
+
from dataclasses import dataclass
|
|
36
|
+
from typing import Callable, Literal
|
|
37
|
+
|
|
38
|
+
from rich.console import Console
|
|
39
|
+
from rich.live import Live
|
|
40
|
+
from rich.progress import (
|
|
41
|
+
BarColumn,
|
|
42
|
+
MofNCompleteColumn,
|
|
43
|
+
Progress,
|
|
44
|
+
SpinnerColumn,
|
|
45
|
+
TaskProgressColumn,
|
|
46
|
+
TextColumn,
|
|
47
|
+
TimeRemainingColumn,
|
|
48
|
+
)
|
|
49
|
+
from rich.text import Text
|
|
50
|
+
|
|
51
|
+
from java_codebase_rag.cli_format import is_noise_line
|
|
52
|
+
|
|
53
|
+
__all__ = [
|
|
54
|
+
"ProgressEvent",
|
|
55
|
+
"parse_progress_line",
|
|
56
|
+
"IndexProgressRenderer",
|
|
57
|
+
"ProgressRelay",
|
|
58
|
+
"CallbackRenderer",
|
|
59
|
+
"make_relay",
|
|
60
|
+
"build_index_progress_context",
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
ProgressKind = Literal["vectors", "graph", "optimize"]
|
|
64
|
+
ProgressStatus = Literal["running", "done", "failed"]
|
|
65
|
+
|
|
66
|
+
_PREFIX = "JCIRAG_PROGRESS"
|
|
67
|
+
# Non-TTY concise-line throttle window, in seconds, per phase.
|
|
68
|
+
_FALLBACK_THROTTLE_S = 5.0
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# ---------------------------------------------------------------------------
|
|
72
|
+
# Protocol
|
|
73
|
+
# ---------------------------------------------------------------------------
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@dataclass
|
|
77
|
+
class ProgressEvent:
|
|
78
|
+
"""A single parsed ``JCIRAG_PROGRESS`` line.
|
|
79
|
+
|
|
80
|
+
``pass_`` carries a multi-pass label like ``"3/6"`` (graph builder passes).
|
|
81
|
+
All optional fields are ``None`` when the emitting line omits the token.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
kind: ProgressKind
|
|
85
|
+
phase: str | None
|
|
86
|
+
pass_: str | None
|
|
87
|
+
done: int | None
|
|
88
|
+
total: int | None
|
|
89
|
+
status: ProgressStatus
|
|
90
|
+
elapsed_s: float | None
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def parse_progress_line(line: bytes) -> ProgressEvent | None:
|
|
94
|
+
"""Parse one raw stderr line into a :class:`ProgressEvent`.
|
|
95
|
+
|
|
96
|
+
Returns ``None`` (never raises) when the line:
|
|
97
|
+
|
|
98
|
+
* does not start (after stripping a leading ``\\r``) with the exact
|
|
99
|
+
``JCIRAG_PROGRESS`` prefix, or
|
|
100
|
+
* has the prefix but no usable tokens afterwards (malformed).
|
|
101
|
+
|
|
102
|
+
The remainder after the prefix is whitespace-separated ``key=value``
|
|
103
|
+
tokens. ``kind`` is required (omitting it → ``None``). ``status`` defaults
|
|
104
|
+
to ``"running"``. Double / extra spaces are tolerated.
|
|
105
|
+
"""
|
|
106
|
+
try:
|
|
107
|
+
text = line.decode("utf-8", errors="replace")
|
|
108
|
+
except Exception:
|
|
109
|
+
return None
|
|
110
|
+
# Tolerate a carriage-return rewind before the prefix.
|
|
111
|
+
stripped = text.lstrip("\r")
|
|
112
|
+
if not stripped.startswith(_PREFIX):
|
|
113
|
+
return None
|
|
114
|
+
tail = stripped[len(_PREFIX):]
|
|
115
|
+
# Tokenize on whitespace; empty/whitespace-only tails are malformed.
|
|
116
|
+
tokens = tail.split()
|
|
117
|
+
if not tokens:
|
|
118
|
+
return None
|
|
119
|
+
|
|
120
|
+
fields: dict[str, str] = {}
|
|
121
|
+
for tok in tokens:
|
|
122
|
+
if "=" not in tok:
|
|
123
|
+
# A bare token is not a key=value pair → malformed line.
|
|
124
|
+
continue
|
|
125
|
+
key, _, value = tok.partition("=")
|
|
126
|
+
key = key.strip()
|
|
127
|
+
value = value.strip()
|
|
128
|
+
if key:
|
|
129
|
+
fields[key] = value
|
|
130
|
+
|
|
131
|
+
if not fields:
|
|
132
|
+
return None
|
|
133
|
+
|
|
134
|
+
kind_raw = fields.get("kind")
|
|
135
|
+
if kind_raw not in ("vectors", "graph", "optimize"):
|
|
136
|
+
# kind is required and must be one of the known kinds.
|
|
137
|
+
return None
|
|
138
|
+
|
|
139
|
+
def _maybe_int(name: str) -> int | None:
|
|
140
|
+
raw = fields.get(name)
|
|
141
|
+
if raw is None or raw == "":
|
|
142
|
+
return None
|
|
143
|
+
try:
|
|
144
|
+
return int(raw)
|
|
145
|
+
except ValueError:
|
|
146
|
+
return None
|
|
147
|
+
|
|
148
|
+
def _maybe_float(name: str) -> float | None:
|
|
149
|
+
raw = fields.get(name)
|
|
150
|
+
if raw is None or raw == "":
|
|
151
|
+
return None
|
|
152
|
+
try:
|
|
153
|
+
return float(raw)
|
|
154
|
+
except ValueError:
|
|
155
|
+
return None
|
|
156
|
+
|
|
157
|
+
status_raw = fields.get("status", "running")
|
|
158
|
+
if status_raw not in ("running", "done", "failed"):
|
|
159
|
+
status_raw = "running"
|
|
160
|
+
|
|
161
|
+
return ProgressEvent(
|
|
162
|
+
kind=kind_raw, # type: ignore[arg-type]
|
|
163
|
+
phase=fields.get("phase") or None,
|
|
164
|
+
pass_=fields.get("pass") or None,
|
|
165
|
+
done=_maybe_int("done"),
|
|
166
|
+
total=_maybe_int("total"),
|
|
167
|
+
status=status_raw, # type: ignore[arg-type]
|
|
168
|
+
elapsed_s=_maybe_float("elapsed_s"),
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
# ---------------------------------------------------------------------------
|
|
173
|
+
# Renderer
|
|
174
|
+
# ---------------------------------------------------------------------------
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _build_progress(console: Console) -> Progress:
|
|
178
|
+
"""Construct the rich.Progress with the canonical column layout.
|
|
179
|
+
|
|
180
|
+
The label TextColumn reads ``task.fields["label"]`` so the phase name
|
|
181
|
+
shown in the bar is independent of the (mutable) description, which we
|
|
182
|
+
repurpose for status suffixes (``✗`` on failure).
|
|
183
|
+
"""
|
|
184
|
+
return Progress(
|
|
185
|
+
SpinnerColumn(),
|
|
186
|
+
TextColumn("[bold cyan]{task.fields[label]}"),
|
|
187
|
+
BarColumn(),
|
|
188
|
+
MofNCompleteColumn(),
|
|
189
|
+
TaskProgressColumn(),
|
|
190
|
+
TextColumn("{task.description}"),
|
|
191
|
+
TimeRemainingColumn(),
|
|
192
|
+
console=console,
|
|
193
|
+
transient=False,
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
class IndexProgressRenderer:
|
|
198
|
+
"""Owns the rich Progress region (TTY) or the concise-line fallback (non-TTY).
|
|
199
|
+
|
|
200
|
+
Construction registers one task per phase name, each ``total=None`` and
|
|
201
|
+
**invisible** (the "never spawned" invariant: no task is ``running`` until
|
|
202
|
+
its first :meth:`apply` arrives). The first event for a kind flips that
|
|
203
|
+
kind's task visible + started.
|
|
204
|
+
|
|
205
|
+
On a non-TTY console (``console.is_terminal is False``) no Live region is
|
|
206
|
+
used; instead concise lines are printed via :meth:`_fallback_print`,
|
|
207
|
+
throttled to once per ~5 s per phase, plus one terminal line per phase.
|
|
208
|
+
"""
|
|
209
|
+
|
|
210
|
+
def __init__(self, phases: list[str], *, console: Console | None = None) -> None:
|
|
211
|
+
self._console: Console = console if console is not None else Console(stderr=True)
|
|
212
|
+
self._phases: list[str] = list(phases)
|
|
213
|
+
self._fallback: bool = not self._console.is_terminal
|
|
214
|
+
self._progress: Progress = _build_progress(self._console)
|
|
215
|
+
# kind -> rich task id. Start every task invisible + not started so the
|
|
216
|
+
# "never spawned" invariant holds until the first event arrives.
|
|
217
|
+
self._task_ids: dict[str, int] = {}
|
|
218
|
+
for phase in self._phases:
|
|
219
|
+
tid = self._progress.add_task(
|
|
220
|
+
phase,
|
|
221
|
+
total=None,
|
|
222
|
+
visible=False,
|
|
223
|
+
start=False,
|
|
224
|
+
label=phase,
|
|
225
|
+
)
|
|
226
|
+
self._task_ids[phase] = tid
|
|
227
|
+
self._live: Live | None = None
|
|
228
|
+
# Non-TTY throttle bookkeeping (monotonic seconds of last concise print).
|
|
229
|
+
self._last_print_at: dict[str, float] = {phase: 0.0 for phase in self._phases}
|
|
230
|
+
# Non-TTY carry-forward: a minimal ``done`` event may omit total /
|
|
231
|
+
# elapsed_s; fall back to the last-seen values so the concise terminal
|
|
232
|
+
# line never degrades (e.g. stays ``vectors done · 1240 · 42.1s``).
|
|
233
|
+
self._last_total: dict[str, int | None] = {phase: None for phase in self._phases}
|
|
234
|
+
self._last_elapsed: dict[str, float | None] = {phase: None for phase in self._phases}
|
|
235
|
+
self._started: bool = False
|
|
236
|
+
|
|
237
|
+
# -- lifecycle -------------------------------------------------------
|
|
238
|
+
|
|
239
|
+
def start(self) -> None:
|
|
240
|
+
"""Enter the Live region (TTY) or no-op (non-TTY). Idempotent."""
|
|
241
|
+
if self._started:
|
|
242
|
+
return
|
|
243
|
+
self._started = True
|
|
244
|
+
if not self._fallback:
|
|
245
|
+
self._live = Live(
|
|
246
|
+
self._progress,
|
|
247
|
+
console=self._console,
|
|
248
|
+
refresh_per_second=10,
|
|
249
|
+
transient=False,
|
|
250
|
+
)
|
|
251
|
+
self._live.start()
|
|
252
|
+
|
|
253
|
+
def stop(self) -> None:
|
|
254
|
+
"""Exit the Live region (TTY) or no-op (non-TTY). Safe to call once."""
|
|
255
|
+
if not self._started:
|
|
256
|
+
return
|
|
257
|
+
self._started = False
|
|
258
|
+
if self._live is not None:
|
|
259
|
+
self._live.stop()
|
|
260
|
+
self._live = None
|
|
261
|
+
|
|
262
|
+
# -- routing ---------------------------------------------------------
|
|
263
|
+
|
|
264
|
+
def apply(self, ev: ProgressEvent) -> None:
|
|
265
|
+
"""Route a single :class:`ProgressEvent` to its matching phase task.
|
|
266
|
+
|
|
267
|
+
First event for a kind makes that task visible + started. ``total`` and
|
|
268
|
+
``done`` are applied directly when present. ``done`` is clamped to be
|
|
269
|
+
non-decreasing (producers should emit monotonically non-decreasing
|
|
270
|
+
``done``; this defensive clamp enforces it so a stray smaller value
|
|
271
|
+
can't rewind the bar). ``status == "done"`` unconditionally clamps
|
|
272
|
+
completed to the total in both directions (an approximate total can't
|
|
273
|
+
stall below 100%, nor can an approximate pre-walk over-count exceed it).
|
|
274
|
+
``status == "failed"`` halts the task and marks the description with a
|
|
275
|
+
red ``✗`` (rich renders the spinner stopped). On non-TTY consoles this
|
|
276
|
+
delegates to the throttled concise-line printer.
|
|
277
|
+
|
|
278
|
+
Safe to call after :meth:`stop`: once the Live region is torn down a
|
|
279
|
+
drain thread may still feed a trailing event; this is a no-op then so
|
|
280
|
+
the apply/stop pair is atomic from the caller's view (PR-1 review
|
|
281
|
+
Minor #6 / cross-PR risk #10).
|
|
282
|
+
"""
|
|
283
|
+
if not self._started:
|
|
284
|
+
# After stop() the Live region is gone and rich tasks are torn down;
|
|
285
|
+
# a trailing event from the drain thread must not mutate state.
|
|
286
|
+
return
|
|
287
|
+
if self._fallback:
|
|
288
|
+
self._fallback_apply(ev)
|
|
289
|
+
return
|
|
290
|
+
tid = self._task_ids.get(ev.kind)
|
|
291
|
+
if tid is None:
|
|
292
|
+
return
|
|
293
|
+
task = self._progress.tasks[tid]
|
|
294
|
+
# First event: promote from pending to running/visible.
|
|
295
|
+
if not task.started:
|
|
296
|
+
self._progress.start_task(tid)
|
|
297
|
+
self._progress.update(tid, visible=True)
|
|
298
|
+
if ev.total is not None:
|
|
299
|
+
self._progress.update(tid, total=ev.total)
|
|
300
|
+
if ev.done is not None:
|
|
301
|
+
# Set-based (not advance-based) for determinism: each event carries
|
|
302
|
+
# the absolute completed count, not a delta. Monotonic clamp: never
|
|
303
|
+
# let a smaller done rewind the bar.
|
|
304
|
+
new_completed = max(task.completed, ev.done)
|
|
305
|
+
self._progress.update(tid, completed=new_completed)
|
|
306
|
+
if ev.status == "done":
|
|
307
|
+
# Two-way clamp: completed must equal total on done. An approximate
|
|
308
|
+
# total can under-count (stall below 100%) or the propose's
|
|
309
|
+
# approximate pre-walk can over-count; both resolve to == total.
|
|
310
|
+
if task.total is not None and task.completed != task.total:
|
|
311
|
+
self._progress.update(tid, completed=task.total)
|
|
312
|
+
self._progress.update(tid, description=f"{ev.kind} ✓")
|
|
313
|
+
self._progress.stop_task(tid)
|
|
314
|
+
elif ev.status == "failed":
|
|
315
|
+
self._progress.update(tid, description=f"{ev.kind} ✗")
|
|
316
|
+
self._progress.stop_task(tid)
|
|
317
|
+
|
|
318
|
+
# -- non-TTY fallback ------------------------------------------------
|
|
319
|
+
|
|
320
|
+
def _now(self) -> float:
|
|
321
|
+
"""Indirection for tests; returns the monotonic clock in seconds."""
|
|
322
|
+
return time.monotonic()
|
|
323
|
+
|
|
324
|
+
def _fallback_apply(self, ev: ProgressEvent) -> None:
|
|
325
|
+
"""Concise-line path for non-TTY consoles."""
|
|
326
|
+
last = self._last_print_at.get(ev.kind, 0.0)
|
|
327
|
+
now = self._now()
|
|
328
|
+
terminal = ev.status in ("done", "failed")
|
|
329
|
+
throttle_ok = (now - last) >= _FALLBACK_THROTTLE_S
|
|
330
|
+
if not terminal and not throttle_ok and last != 0.0:
|
|
331
|
+
# Suppressed by the throttle window. (``last != 0.0`` lets the very
|
|
332
|
+
# first event for a phase print immediately.) Still track totals so
|
|
333
|
+
# a later terminal line can carry them forward.
|
|
334
|
+
if ev.total is not None:
|
|
335
|
+
self._last_total[ev.kind] = ev.total
|
|
336
|
+
if ev.elapsed_s is not None:
|
|
337
|
+
self._last_elapsed[ev.kind] = ev.elapsed_s
|
|
338
|
+
return
|
|
339
|
+
# Carry forward last-seen total / elapsed_s so a minimal ``done`` event
|
|
340
|
+
# that omits them still prints a complete terminal line.
|
|
341
|
+
carried_total = ev.total
|
|
342
|
+
if carried_total is None:
|
|
343
|
+
carried_total = self._last_total.get(ev.kind)
|
|
344
|
+
carried_elapsed = ev.elapsed_s
|
|
345
|
+
if carried_elapsed is None:
|
|
346
|
+
carried_elapsed = self._last_elapsed.get(ev.kind)
|
|
347
|
+
# Update the carry-forward state with whatever this event carried.
|
|
348
|
+
if ev.total is not None:
|
|
349
|
+
self._last_total[ev.kind] = ev.total
|
|
350
|
+
if ev.elapsed_s is not None:
|
|
351
|
+
self._last_elapsed[ev.kind] = ev.elapsed_s
|
|
352
|
+
self._last_print_at[ev.kind] = now
|
|
353
|
+
self._console.print(
|
|
354
|
+
self._format_concise(ev, total=carried_total, elapsed_s=carried_elapsed)
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
def _format_concise(
|
|
358
|
+
self,
|
|
359
|
+
ev: ProgressEvent,
|
|
360
|
+
*,
|
|
361
|
+
total: int | None = None,
|
|
362
|
+
elapsed_s: float | None = None,
|
|
363
|
+
) -> Text:
|
|
364
|
+
"""Render one concise line for the non-TTY fallback.
|
|
365
|
+
|
|
366
|
+
``total`` / ``elapsed_s`` are the already-carry-forwarded values to show
|
|
367
|
+
(the caller resolves event-vs-last-seen before formatting); ``ev``'s own
|
|
368
|
+
fields are only used for status / done / phase.
|
|
369
|
+
"""
|
|
370
|
+
kind = ev.kind
|
|
371
|
+
if ev.status == "done":
|
|
372
|
+
total_str = str(total) if total is not None else ""
|
|
373
|
+
elapsed = f"{elapsed_s:.1f}s" if elapsed_s is not None else ""
|
|
374
|
+
bits = [kind, "done"]
|
|
375
|
+
if total_str != "":
|
|
376
|
+
bits.append(total_str)
|
|
377
|
+
if elapsed:
|
|
378
|
+
bits.append(elapsed)
|
|
379
|
+
return Text(" · ".join(bits))
|
|
380
|
+
if ev.status == "failed":
|
|
381
|
+
return Text(f"{kind} failed")
|
|
382
|
+
if ev.done is not None and ev.total is not None and ev.total > 0:
|
|
383
|
+
pct = int(round(ev.done * 100.0 / ev.total))
|
|
384
|
+
return Text(f"{kind} {ev.done}/{ev.total} ({pct}%)")
|
|
385
|
+
if ev.done is not None:
|
|
386
|
+
return Text(f"{kind} {ev.done}")
|
|
387
|
+
return Text(kind)
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
class CallbackRenderer:
|
|
391
|
+
"""Adapter exposing a ``renderer.apply(ev)`` surface to :class:`ProgressRelay`.
|
|
392
|
+
|
|
393
|
+
``ProgressRelay`` calls ``renderer.apply(ev)`` for each parsed progress line.
|
|
394
|
+
This adapter forwards to a caller-supplied ``on_progress`` callback (used by
|
|
395
|
+
the sync/async subprocess drains in ``pipeline`` / ``cli_progress`` to route
|
|
396
|
+
progress events up to the command-level renderer without owning a Live region
|
|
397
|
+
themselves). It carries a ``_console`` attribute so the relay can route
|
|
398
|
+
non-progress lines through ``console.print`` while a Live region is up.
|
|
399
|
+
"""
|
|
400
|
+
|
|
401
|
+
def __init__(self, on_progress, console=None) -> None: # type: ignore[no-untyped-def]
|
|
402
|
+
self._on_progress = on_progress
|
|
403
|
+
self._console = console
|
|
404
|
+
|
|
405
|
+
def apply(self, ev: ProgressEvent) -> None:
|
|
406
|
+
self._on_progress(ev)
|
|
407
|
+
|
|
408
|
+
def start(self) -> None:
|
|
409
|
+
pass
|
|
410
|
+
|
|
411
|
+
def stop(self) -> None:
|
|
412
|
+
pass
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def make_relay(
|
|
416
|
+
on_progress: Callable[[ProgressEvent], None],
|
|
417
|
+
*,
|
|
418
|
+
console: object | None,
|
|
419
|
+
verbose: bool,
|
|
420
|
+
) -> ProgressRelay:
|
|
421
|
+
"""Build the standard drain-side :class:`ProgressRelay`.
|
|
422
|
+
|
|
423
|
+
Both subprocess drains (``pipeline._popen_capturing_stderr`` sync path and
|
|
424
|
+
``cli_progress.accumulate_and_relay_subprocess_streams`` async path) wire a
|
|
425
|
+
``CallbackRenderer`` that forwards parsed events to the command-level
|
|
426
|
+
``on_progress`` callback, and route non-progress lines through the same
|
|
427
|
+
caller-supplied ``console``. Centralizing the construction here keeps the
|
|
428
|
+
sync and async wiring identical (PR-3 forks the async path further).
|
|
429
|
+
"""
|
|
430
|
+
return ProgressRelay(
|
|
431
|
+
CallbackRenderer(on_progress, console),
|
|
432
|
+
console=console,
|
|
433
|
+
verbose=verbose,
|
|
434
|
+
)
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
# The canonical phase order shared by every lifecycle command that renders
|
|
438
|
+
# progress. The operator commands (init/increment/reprocess) and the installer
|
|
439
|
+
# sub-steps (install/update indexing) all render this same list so the
|
|
440
|
+
# Vectors → Optimize → Graph shape is uniform across the CLI.
|
|
441
|
+
_INDEX_PHASES = ["vectors", "optimize", "graph"]
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def build_index_progress_context(
|
|
445
|
+
phases: list[str] | None = None,
|
|
446
|
+
) -> tuple["IndexProgressRenderer", Callable[[ProgressEvent], None], "Console"]:
|
|
447
|
+
"""Construct the shared ``(renderer, on_progress, console)`` triple.
|
|
448
|
+
|
|
449
|
+
Both ``cli._run_with_pipeline_progress`` (operator commands, default TTY
|
|
450
|
+
mode) and the installer's indexing sub-step (``installer.run_init_if_needed``
|
|
451
|
+
/ ``run_update``) use this so the phase list, the callback wiring, and the
|
|
452
|
+
single-writer console are defined in exactly one place. The returned
|
|
453
|
+
``on_progress`` forwards each event to ``renderer.apply``; ``console`` is
|
|
454
|
+
the renderer's stderr ``rich.Console`` so the subprocess drain routes
|
|
455
|
+
non-progress lines through ``console.print`` while a Live region is up.
|
|
456
|
+
|
|
457
|
+
The caller owns ``renderer.start()``/``stop()`` lifecycle. In ``--quiet``
|
|
458
|
+
or ``--verbose`` mode the caller simply does not call this helper (quiet
|
|
459
|
+
is silent; verbose raw-relays).
|
|
460
|
+
"""
|
|
461
|
+
renderer = IndexProgressRenderer(phases if phases is not None else _INDEX_PHASES)
|
|
462
|
+
|
|
463
|
+
def on_progress(ev: ProgressEvent) -> None:
|
|
464
|
+
renderer.apply(ev)
|
|
465
|
+
|
|
466
|
+
return renderer, on_progress, renderer._console # noqa: SLF001 — shared console for the drain
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
# ---------------------------------------------------------------------------
|
|
470
|
+
# Relay
|
|
471
|
+
# ---------------------------------------------------------------------------
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
class ProgressRelay:
|
|
475
|
+
"""Single-writer bridge between a subprocess stderr drain and the renderer.
|
|
476
|
+
|
|
477
|
+
Mirrors the byte-buffering of ``cli_progress._AsyncLineFilter`` /
|
|
478
|
+
``pipeline._LineFilter``: accumulate chunks, split on ``\\n``, route each
|
|
479
|
+
complete line. When ``verbose`` is True (and no renderer is attached) the
|
|
480
|
+
relay writes raw bytes to ``sys.stderr.buffer`` (raw mode, no Live region).
|
|
481
|
+
"""
|
|
482
|
+
|
|
483
|
+
def __init__(
|
|
484
|
+
self,
|
|
485
|
+
renderer: IndexProgressRenderer | None,
|
|
486
|
+
*,
|
|
487
|
+
console: Console | None = None,
|
|
488
|
+
verbose: bool = False,
|
|
489
|
+
) -> None:
|
|
490
|
+
self._renderer = renderer
|
|
491
|
+
self._verbose = verbose
|
|
492
|
+
self._console: Console | None = console
|
|
493
|
+
self._buf = bytearray()
|
|
494
|
+
# Live region is only meaningful when a renderer is attached.
|
|
495
|
+
self._live_active: bool = renderer is not None
|
|
496
|
+
# Mirrors ``_LineFilter._suppress_next``: a noise header line (e.g. a
|
|
497
|
+
# ``FutureWarning:`` banner) suppresses the NEXT line too, which is its
|
|
498
|
+
# indented traceback frame(s). ``line[:1] in (b" ", b"\t")`` is the
|
|
499
|
+
# continuation signal. Progress lines reset this (they are consumed by
|
|
500
|
+
# the renderer, never noise).
|
|
501
|
+
self._suppress_next: bool = False
|
|
502
|
+
|
|
503
|
+
def feed(self, chunk: bytes) -> None:
|
|
504
|
+
"""Buffer ``chunk`` and route each complete (``\\n``-terminated) line."""
|
|
505
|
+
self._buf.extend(chunk)
|
|
506
|
+
while b"\n" in self._buf:
|
|
507
|
+
line, self._buf = self._buf.split(b"\n", 1)
|
|
508
|
+
line += b"\n"
|
|
509
|
+
self._route_line(line)
|
|
510
|
+
|
|
511
|
+
def flush(self) -> None:
|
|
512
|
+
"""Emit any trailing partial buffer (without a trailing newline)."""
|
|
513
|
+
if self._buf:
|
|
514
|
+
# Trailing partial line: continuation suppression does not apply
|
|
515
|
+
# (there is no following line). Route it directly.
|
|
516
|
+
self._route_line(bytes(self._buf))
|
|
517
|
+
self._buf.clear()
|
|
518
|
+
|
|
519
|
+
def _route_line(self, line: bytes) -> None:
|
|
520
|
+
ev = parse_progress_line(line)
|
|
521
|
+
if ev is not None and self._renderer is not None:
|
|
522
|
+
# Consumed by the protocol — never echoed to any sink. It is not
|
|
523
|
+
# noise, so it must not keep the suppression flag armed.
|
|
524
|
+
self._suppress_next = False
|
|
525
|
+
# Guard the render chain: a drain thread is a daemon, so an
|
|
526
|
+
# unhandled exception here dies silently and truncates the captured
|
|
527
|
+
# stderr, masking real bugs. Mirror the defensive ``except Exception``
|
|
528
|
+
# around the raw ``buffer.write`` path below: log a one-line note to
|
|
529
|
+
# real stderr and keep draining. Never re-raise.
|
|
530
|
+
try:
|
|
531
|
+
self._renderer.apply(ev)
|
|
532
|
+
except Exception as exc:
|
|
533
|
+
sys.stderr.write(
|
|
534
|
+
f"java-codebase-rag: progress renderer error: {exc}\n"
|
|
535
|
+
)
|
|
536
|
+
sys.stderr.flush()
|
|
537
|
+
return
|
|
538
|
+
if ev is not None and self._renderer is None:
|
|
539
|
+
# Parsed as progress but no renderer attached: still reset the flag
|
|
540
|
+
# (a progress line is never noise) and drop quietly.
|
|
541
|
+
self._suppress_next = False
|
|
542
|
+
return
|
|
543
|
+
# Non-progress line: noise path, with continuation suppression.
|
|
544
|
+
if is_noise_line(line):
|
|
545
|
+
self._suppress_next = True
|
|
546
|
+
return
|
|
547
|
+
if self._suppress_next and line[:1] in (b" ", b"\t"):
|
|
548
|
+
# Indented continuation of the preceding noise header (e.g. a
|
|
549
|
+
# traceback frame). Drop without disarming: a multi-frame traceback
|
|
550
|
+
# has several such lines in a row.
|
|
551
|
+
return
|
|
552
|
+
self._suppress_next = False
|
|
553
|
+
text = line.decode("utf-8", errors="replace")
|
|
554
|
+
if self._renderer is not None and self._live_active:
|
|
555
|
+
# The drains always construct the relay with the caller's console
|
|
556
|
+
# (see make_relay), so ``self._console`` is authoritative here.
|
|
557
|
+
assert self._console is not None # invariant: drains always supply it
|
|
558
|
+
# rich.Console over a Live region must suspend/resume to interleave
|
|
559
|
+
# a one-off line without corrupting the bar redraw; print() handles
|
|
560
|
+
# this correctly when the Live was started on the same console.
|
|
561
|
+
self._console.print(text, end="")
|
|
562
|
+
return
|
|
563
|
+
if self._verbose and self._renderer is None:
|
|
564
|
+
try:
|
|
565
|
+
sys.stderr.buffer.write(line)
|
|
566
|
+
sys.stderr.buffer.flush()
|
|
567
|
+
except Exception:
|
|
568
|
+
pass
|
|
569
|
+
return
|
|
570
|
+
# Neither verbose nor a renderer: drop quietly (quiet mode).
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: java-codebase-rag
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.3
|
|
4
4
|
Summary: MCP server for semantic + structural search over Java codebases
|
|
5
5
|
Author: HumanBean17
|
|
6
6
|
License-Expression: MIT
|
|
@@ -28,6 +28,7 @@ Requires-Dist: pyarrow<24,>=23.0.1
|
|
|
28
28
|
Requires-Dist: pydantic<3,>=2.0
|
|
29
29
|
Requires-Dist: PyYAML<7,>=6.0.3
|
|
30
30
|
Requires-Dist: questionary<3,>=2.0
|
|
31
|
+
Requires-Dist: rich<15,>=14
|
|
31
32
|
Requires-Dist: sentence-transformers<6,>=5.4.0
|
|
32
33
|
Requires-Dist: tree-sitter<0.26,>=0.25.2
|
|
33
34
|
Requires-Dist: tree-sitter-java<0.24,>=0.23.5
|
|
@@ -106,6 +107,8 @@ java-codebase-rag install --non-interactive --agent claude-code
|
|
|
106
107
|
|
|
107
108
|
After `pip install --upgrade java-codebase-rag`, run `java-codebase-rag update` to refresh shipped artifacts and catch up the index (Lance + graph).
|
|
108
109
|
|
|
110
|
+
All indexing lifecycle commands (`init`, `increment`, `reprocess`, `install`, `update`) show a unified `Vectors → Optimize → Graph` progress bar on stderr during the index build (powered by `rich`); pass `--quiet` to suppress it.
|
|
111
|
+
|
|
109
112
|
### Manual registration
|
|
110
113
|
|
|
111
114
|
If you prefer manual configuration, see [`docs/JAVA-CODEBASE-RAG-CLI.md`](./docs/JAVA-CODEBASE-RAG-CLI.md) for the full CLI reference.
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
ast_java.py,sha256=NQgZzstbsMq-PdowoD6r_ixJKxEEFzTP9xUzqDpiXeU,99661
|
|
2
2
|
brownfield_events.py,sha256=yxXkKDgMb3VPtaiakGzncHM_EGnda8xIue6w90yYp8s,2055
|
|
3
|
-
build_ast_graph.py,sha256=
|
|
3
|
+
build_ast_graph.py,sha256=LNU1rIHwgEpbrWnWoywg4JdqbwwT4UKcx1iDay_GemM,156776
|
|
4
4
|
chunk_heuristics.py,sha256=aQk2NOKxzUdqoUAJUO3G3LE0MN_bYZWNLQ0tkmj5uts,1813
|
|
5
5
|
graph_enrich.py,sha256=POT4LwSkTsrjUmP67bsm2UezUam70cunuPDYDh-v1Bs,63332
|
|
6
6
|
index_common.py,sha256=HT6FKHFJ084eFvd3fR1j8z8gf4eWoPHVW8GXLpw464I,285
|
|
7
|
-
java_index_flow_lancedb.py,sha256=
|
|
7
|
+
java_index_flow_lancedb.py,sha256=7CS-sdbzhCkUsHfpCZ85uM7vy4aoZI8n-RowLocfiKc,20267
|
|
8
8
|
java_index_v1_common.py,sha256=nF1KrSqboF_RRvWerG9knRRFmWwsrG_CvhgnsoZ8KqA,1154
|
|
9
9
|
java_ontology.py,sha256=71bCLDNvMy0SpZPzSR5apJ0qJXNd6y5ggkLdBEw_PFo,16682
|
|
10
10
|
ladybug_queries.py,sha256=912j9VAYDjcU4ReVorWQ6R4DZl0tteKic-Pqu0jyBS0,90837
|
|
@@ -13,21 +13,22 @@ mcp_v2.py,sha256=o94GJI7j6dLJDIA3R_1ZiQhjzQfMAEW3etdeZYnHOUc,80637
|
|
|
13
13
|
path_filtering.py,sha256=-oX16SYLWYwX9pcV1fu3vbVTIhY1GzFflT7J1E2tqPY,17122
|
|
14
14
|
pr_analysis.py,sha256=3-5L8_G5XupdJsl9RN73Lq-ejPoK11B3m_VzAx2fGG8,18413
|
|
15
15
|
search_lancedb.py,sha256=scG6HBUrsgIeSWFrGcLcGdhWv1qODOx4JOBMAlLDY_E,36793
|
|
16
|
-
server.py,sha256=
|
|
16
|
+
server.py,sha256=DcJwocSknBy0vwsUvYiC5hr-hrD_rRT-u9_DmuRTC9k,35035
|
|
17
17
|
java_codebase_rag/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
18
18
|
java_codebase_rag/_fdlimit.py,sha256=WroFdfSNbcriKok6q8znTf74dqlznxea_1Fd5bHl_3o,1930
|
|
19
|
-
java_codebase_rag/cli.py,sha256=
|
|
20
|
-
java_codebase_rag/cli_format.py,sha256=
|
|
21
|
-
java_codebase_rag/cli_progress.py,sha256=
|
|
19
|
+
java_codebase_rag/cli.py,sha256=uTCpgsE3gbDPSqPDNJeJ7hZu_MhyKYm4tqHIMkM67LM,38650
|
|
20
|
+
java_codebase_rag/cli_format.py,sha256=CT7-xdwZ0bMCdP68_UOwkvm-mnLluU3LutlM-mDNk60,1839
|
|
21
|
+
java_codebase_rag/cli_progress.py,sha256=q6Wh97yzLGs1B8UFk_WAKivfQu7Y5RnUUE-T2YHWkIs,3237
|
|
22
22
|
java_codebase_rag/config.py,sha256=bfwYI4R8PU9YV_M4r8-03iaUZ_0TW-qN_NuhIsDXy2M,18769
|
|
23
|
-
java_codebase_rag/installer.py,sha256=
|
|
24
|
-
java_codebase_rag/lance_optimize.py,sha256=
|
|
25
|
-
java_codebase_rag/pipeline.py,sha256=
|
|
23
|
+
java_codebase_rag/installer.py,sha256=sXsHPo24aoDFoTr0D_vYLg0MFdGAV2wdL05FqRaul6E,52861
|
|
24
|
+
java_codebase_rag/lance_optimize.py,sha256=25Rwj7HNO8F-35MxhFK6naqgbjd3H-T0zKb3pXB4H0s,9268
|
|
25
|
+
java_codebase_rag/pipeline.py,sha256=ydNktEGL1YniAjJsr37yKBo_bGV4cN_LTGVTmmrsrZw,14688
|
|
26
|
+
java_codebase_rag/progress.py,sha256=2IxdMALDM0wAQCyJrrfZ975zM_85C-4BfHxf4AtYifE,23212
|
|
26
27
|
java_codebase_rag/install_data/agents/explorer-rag-enhanced.md,sha256=APl9d-No12qZNZLjU7mwNRwxHIgnT3ZtQZiD4clWlyU,14413
|
|
27
28
|
java_codebase_rag/install_data/skills/explore-codebase/SKILL.md,sha256=pIM-Xdwq_fXkhhBJCdb-fA2nes5c_mMPcdUXb7Adyxo,12040
|
|
28
|
-
java_codebase_rag-0.6.
|
|
29
|
-
java_codebase_rag-0.6.
|
|
30
|
-
java_codebase_rag-0.6.
|
|
31
|
-
java_codebase_rag-0.6.
|
|
32
|
-
java_codebase_rag-0.6.
|
|
33
|
-
java_codebase_rag-0.6.
|
|
29
|
+
java_codebase_rag-0.6.3.dist-info/licenses/LICENSE,sha256=gxvtiHtuviR_q8ZAjWw-QTcF3DyPzg6ZY-lQrr8OPpw,1068
|
|
30
|
+
java_codebase_rag-0.6.3.dist-info/METADATA,sha256=zDwRolQN2GTSexo7J2ngIqB-RnqmsnTZLRQ-lWuqBL4,17197
|
|
31
|
+
java_codebase_rag-0.6.3.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
32
|
+
java_codebase_rag-0.6.3.dist-info/entry_points.txt,sha256=wsPZwot0Ui4JI3TIgW8LcbN8bNtKFbwQAlHAAJXfYgQ,117
|
|
33
|
+
java_codebase_rag-0.6.3.dist-info/top_level.txt,sha256=syQgi8XPBwY2ws_NZ1uRCxTf_s41NpshwEHNdcdnk3A,245
|
|
34
|
+
java_codebase_rag-0.6.3.dist-info/RECORD,,
|