SourceIndex 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sourceindex/__init__.py +30 -0
- sourceindex/build/__init__.py +592 -0
- sourceindex/build/indexer.py +403 -0
- sourceindex/build/linerange/__init__.py +24 -0
- sourceindex/build/linerange/python_ast.py +155 -0
- sourceindex/build/linerange/treesitter.py +397 -0
- sourceindex/build/prompts.py +76 -0
- sourceindex/build/state.py +261 -0
- sourceindex/build/walker.py +94 -0
- sourceindex/claudecode/__init__.py +0 -0
- sourceindex/claudecode/savings.py +325 -0
- sourceindex/claudecode/savings_summary.py +88 -0
- sourceindex/claudecode/statusline.py +116 -0
- sourceindex/cli/__init__.py +304 -0
- sourceindex/cli/__main__.py +10 -0
- sourceindex/cli/api_key.py +171 -0
- sourceindex/cli/commands.py +678 -0
- sourceindex/cli/install.py +378 -0
- sourceindex/daemon/__init__.py +83 -0
- sourceindex/daemon/client.py +141 -0
- sourceindex/daemon/crypto.py +48 -0
- sourceindex/daemon/keyring_store.py +129 -0
- sourceindex/daemon/lifecycle.py +237 -0
- sourceindex/daemon/protocol.py +134 -0
- sourceindex/daemon/server.py +389 -0
- sourceindex/daemon/store.py +426 -0
- sourceindex/lib/__init__.py +0 -0
- sourceindex/lib/backend.py +240 -0
- sourceindex/lib/cost.py +96 -0
- sourceindex/lib/env.py +30 -0
- sourceindex/lib/git.py +33 -0
- sourceindex/lib/languages.py +406 -0
- sourceindex/lib/llm.py +298 -0
- sourceindex/lib/log.py +228 -0
- sourceindex/lib/registry.py +75 -0
- sourceindex/lib/timing.py +10 -0
- sourceindex/search/__init__.py +251 -0
- sourceindex/search/experiments.py +276 -0
- sourceindex/search/imports.py +155 -0
- sourceindex/search/passes.py +51 -0
- sourceindex/search/prompts.py +379 -0
- sourceindex/search/roadmap.py +265 -0
- sourceindex/server.py +127 -0
- sourceindex-0.1.0.dist-info/METADATA +73 -0
- sourceindex-0.1.0.dist-info/RECORD +47 -0
- sourceindex-0.1.0.dist-info/WHEEL +4 -0
- sourceindex-0.1.0.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,678 @@
|
|
|
1
|
+
"""CLI subcommands. Heavy imports (build/search/server/claudecode) are
|
|
2
|
+
deferred into each ``cmd_*`` body so ``sourceindex statusline`` — re-run
|
|
3
|
+
on every Claude Code render tick — doesn't pay for ``litellm``."""
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import contextlib
|
|
7
|
+
import sys
|
|
8
|
+
import threading
|
|
9
|
+
from datetime import datetime, timezone
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from ..lib.log import get_logger
|
|
13
|
+
from .api_key import _persist_api_key, _prompt_api_key, _require_api_key
|
|
14
|
+
|
|
15
|
+
_log = get_logger(__name__)
|
|
16
|
+
from .install import (
|
|
17
|
+
CLAUDE_MD_MARKER_AGENT,
|
|
18
|
+
CLAUDE_MD_MARKER_MCP,
|
|
19
|
+
CLAUDE_MD_SECTION_AGENT,
|
|
20
|
+
CLAUDE_MD_SECTION_MCP,
|
|
21
|
+
_append_claude_md,
|
|
22
|
+
_ensure_gitignored,
|
|
23
|
+
_install_hooks,
|
|
24
|
+
_register_pretooluse_hook,
|
|
25
|
+
_write_mcp_config,
|
|
26
|
+
_write_restrict_hook,
|
|
27
|
+
_write_subagent_config,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _parse_languages(value: str | None):
|
|
32
|
+
if not value:
|
|
33
|
+
return None
|
|
34
|
+
from ..lib.languages import get_languages
|
|
35
|
+
keys = [k.strip() for k in value.split(",") if k.strip()]
|
|
36
|
+
return get_languages(keys) if keys else None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _stream_progress(progress_path: Path, stop: threading.Event) -> None:
|
|
40
|
+
"""Tail a single-line progress file the daemon updates on each tick.
|
|
41
|
+
Re-renders the bar to our own stderr — the daemon's stderr is /dev/null."""
|
|
42
|
+
last = ""
|
|
43
|
+
is_tty = sys.stderr.isatty()
|
|
44
|
+
landed_on_fresh_line = False
|
|
45
|
+
while not stop.wait(0.25):
|
|
46
|
+
try:
|
|
47
|
+
line = progress_path.read_text().rstrip("\n")
|
|
48
|
+
except (FileNotFoundError, OSError):
|
|
49
|
+
continue
|
|
50
|
+
if not line or line == last:
|
|
51
|
+
continue
|
|
52
|
+
if is_tty and not landed_on_fresh_line:
|
|
53
|
+
# Move below any log lines already printed before the build started.
|
|
54
|
+
sys.stderr.write("\n")
|
|
55
|
+
landed_on_fresh_line = True
|
|
56
|
+
last = line
|
|
57
|
+
if is_tty:
|
|
58
|
+
sys.stderr.write("\r\033[K" + line)
|
|
59
|
+
sys.stderr.flush()
|
|
60
|
+
else:
|
|
61
|
+
print(line, file=sys.stderr)
|
|
62
|
+
if is_tty and landed_on_fresh_line:
|
|
63
|
+
sys.stderr.write("\n")
|
|
64
|
+
sys.stderr.flush()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@contextlib.contextmanager
|
|
68
|
+
def _mirror_daemon_progress(repo_hash: str):
|
|
69
|
+
"""Mirror the daemon's progress side-channel onto our stderr for the
|
|
70
|
+
duration of the wrapped RPC. No-ops if the daemon writes nothing."""
|
|
71
|
+
from ..daemon import lifecycle
|
|
72
|
+
progress_path = lifecycle.progress_file(repo_hash)
|
|
73
|
+
stop = threading.Event()
|
|
74
|
+
thread = threading.Thread(
|
|
75
|
+
target=_stream_progress, args=(progress_path, stop), daemon=True,
|
|
76
|
+
)
|
|
77
|
+
thread.start()
|
|
78
|
+
try:
|
|
79
|
+
yield
|
|
80
|
+
finally:
|
|
81
|
+
stop.set()
|
|
82
|
+
thread.join(timeout=1)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def cmd_init(args: argparse.Namespace) -> None:
|
|
86
|
+
from ..daemon import Client, IndexDir
|
|
87
|
+
from ..lib.backend import apply_remote_config
|
|
88
|
+
from ..lib.llm import auth_error_to_systemexit
|
|
89
|
+
|
|
90
|
+
repo_root = Path(args.repo_root).resolve()
|
|
91
|
+
idx = IndexDir.for_repo(repo_root)
|
|
92
|
+
index_dir = idx.path
|
|
93
|
+
languages = _parse_languages(getattr(args, "languages", None))
|
|
94
|
+
mode = getattr(args, "mode", "agent")
|
|
95
|
+
allow_dump = bool(getattr(args, "allow_dump", False))
|
|
96
|
+
|
|
97
|
+
api_key = _prompt_api_key(args.api_key)
|
|
98
|
+
# First-run write of plaintext .env; the daemon migrates it on its
|
|
99
|
+
# next start. Writing into an encrypted store would corrupt the blob.
|
|
100
|
+
if not idx.is_encrypted_layout():
|
|
101
|
+
_persist_api_key(index_dir, api_key)
|
|
102
|
+
_ensure_gitignored(repo_root)
|
|
103
|
+
|
|
104
|
+
model = apply_remote_config(index_dir=index_dir, default_model=args.model)
|
|
105
|
+
|
|
106
|
+
_log.info("Installing git hooks")
|
|
107
|
+
_install_hooks(repo_root)
|
|
108
|
+
_ensure_autostart_installed()
|
|
109
|
+
# Register before any early-return paths so re-running init on an
|
|
110
|
+
# already-indexed repo still backfills its registry entry.
|
|
111
|
+
from ..lib import registry
|
|
112
|
+
registry.add(idx.repo_hash, str(repo_root))
|
|
113
|
+
|
|
114
|
+
if mode == "mcp":
|
|
115
|
+
_log.info("Writing MCP config")
|
|
116
|
+
_write_mcp_config(repo_root)
|
|
117
|
+
_log.info("Updating CLAUDE.md with workflow instructions...")
|
|
118
|
+
_append_claude_md(repo_root, CLAUDE_MD_SECTION_MCP, CLAUDE_MD_MARKER_MCP)
|
|
119
|
+
elif mode == "agent":
|
|
120
|
+
_log.info("Writing Claude subagent config")
|
|
121
|
+
_write_subagent_config(repo_root)
|
|
122
|
+
_log.info("Installing subagent single-shot gate (PreToolUse hook)")
|
|
123
|
+
_write_restrict_hook(repo_root)
|
|
124
|
+
_register_pretooluse_hook(repo_root)
|
|
125
|
+
_log.info("Updating CLAUDE.md with workflow instructions...")
|
|
126
|
+
_append_claude_md(repo_root, CLAUDE_MD_SECTION_AGENT, CLAUDE_MD_MARKER_AGENT)
|
|
127
|
+
else:
|
|
128
|
+
raise ValueError(f"Unknown init mode: {mode!r} (expected 'mcp' or 'agent')")
|
|
129
|
+
|
|
130
|
+
if idx.is_encrypted_layout() or idx.has_plaintext_index():
|
|
131
|
+
_log.info(
|
|
132
|
+
f"Index already exists at {index_dir}, skipping build. Use 'sourceindex update' to refresh.",
|
|
133
|
+
extra={"event": "init.index_exists", "index_dir": str(index_dir)},
|
|
134
|
+
)
|
|
135
|
+
return
|
|
136
|
+
|
|
137
|
+
_log.info(
|
|
138
|
+
f"Initializing index for {repo_root}...",
|
|
139
|
+
extra={"event": "init.start", "repo_root": str(repo_root), "mode": mode},
|
|
140
|
+
)
|
|
141
|
+
_preflight_indexing(repo_root, index_dir, languages, model)
|
|
142
|
+
client = Client(repo_root, allow_dump_on_spawn=allow_dump)
|
|
143
|
+
lang_strings = [l.key for l in languages] if languages else None
|
|
144
|
+
with auth_error_to_systemexit(index_dir=index_dir, api_key=api_key), \
|
|
145
|
+
_mirror_daemon_progress(idx.repo_hash):
|
|
146
|
+
result = client.init(
|
|
147
|
+
api_key=api_key, model=model,
|
|
148
|
+
languages=lang_strings, workers=args.workers,
|
|
149
|
+
)
|
|
150
|
+
_log.info(
|
|
151
|
+
f"Index written to {index_dir}",
|
|
152
|
+
extra={"event": "init.done", "index_dir": str(index_dir), **result},
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def cmd_install_hooks(args: argparse.Namespace) -> None:
|
|
157
|
+
repo_root = Path(args.repo_root).resolve()
|
|
158
|
+
_install_hooks(repo_root)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def cmd_update(args: argparse.Namespace) -> None:
|
|
162
|
+
from ..daemon import Client, IndexDir
|
|
163
|
+
from ..lib.backend import apply_remote_config
|
|
164
|
+
from ..lib.llm import auth_error_to_systemexit
|
|
165
|
+
|
|
166
|
+
repo_root = Path(args.repo_root).resolve()
|
|
167
|
+
idx = IndexDir.for_repo(repo_root)
|
|
168
|
+
index_dir = idx.path
|
|
169
|
+
languages = _parse_languages(getattr(args, "languages", None))
|
|
170
|
+
allow_dump = bool(getattr(args, "allow_dump", False))
|
|
171
|
+
|
|
172
|
+
api_key = _require_api_key(getattr(args, "api_key", None), daemon_managed=idx.is_encrypted_layout())
|
|
173
|
+
model = apply_remote_config(index_dir=index_dir, default_model=args.model)
|
|
174
|
+
client = Client(repo_root, allow_dump_on_spawn=allow_dump)
|
|
175
|
+
lang_strings = [l.key for l in languages] if languages else None
|
|
176
|
+
with auth_error_to_systemexit(index_dir=index_dir, api_key=api_key), \
|
|
177
|
+
_mirror_daemon_progress(idx.repo_hash):
|
|
178
|
+
if not (idx.is_encrypted_layout() or idx.has_plaintext_index()):
|
|
179
|
+
_log.info(
|
|
180
|
+
f"No index found. Building full index for {repo_root}...",
|
|
181
|
+
extra={"event": "update.cold_build", "repo_root": str(repo_root)},
|
|
182
|
+
)
|
|
183
|
+
result = client.init(
|
|
184
|
+
api_key=api_key, model=model,
|
|
185
|
+
languages=lang_strings, workers=args.workers,
|
|
186
|
+
)
|
|
187
|
+
_log.info(
|
|
188
|
+
f"Index written to {index_dir}",
|
|
189
|
+
extra={"event": "init.done", "index_dir": str(index_dir), **result},
|
|
190
|
+
)
|
|
191
|
+
else:
|
|
192
|
+
client.update(
|
|
193
|
+
api_key=api_key, model=model,
|
|
194
|
+
languages=lang_strings, workers=args.workers,
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def cmd_serve(args: argparse.Namespace) -> None:
|
|
199
|
+
from ..build import DEFAULT_MODEL
|
|
200
|
+
from ..daemon import IndexDir
|
|
201
|
+
from ..lib.backend import apply_remote_config
|
|
202
|
+
from ..server import run_server
|
|
203
|
+
|
|
204
|
+
repo_root = Path(args.repo_root).resolve()
|
|
205
|
+
index_dir = IndexDir.for_repo(repo_root, args.index_dir).path
|
|
206
|
+
|
|
207
|
+
model = apply_remote_config(index_dir=index_dir, default_model=DEFAULT_MODEL)
|
|
208
|
+
run_server(repo_root, index_dir, index_search_model=model)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
ROADMAP_FILENAME = "sourceindex-output.txt"
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def cmd_search(args: argparse.Namespace) -> None:
|
|
215
|
+
"""Run the search pipeline, write the roadmap into the index dir, and
|
|
216
|
+
print a one-line completion signal naming that path.
|
|
217
|
+
|
|
218
|
+
Routing the roadmap through a file (instead of stdout) keeps the calling
|
|
219
|
+
subagent a pure tool invoker — it never sees the roadmap text, so it
|
|
220
|
+
cannot summarize, paraphrase, or conjecture about it. The main agent
|
|
221
|
+
reads the file directly.
|
|
222
|
+
|
|
223
|
+
By default the file lives in the index dir (``<repo>/.sourceindex/``,
|
|
224
|
+
already gitignored) so it doesn't leave a stray artifact in the repo
|
|
225
|
+
root. ``--output`` overrides the destination — a caller that points one
|
|
226
|
+
shared index dir at concurrent searches (e.g. the sourceindex-bare eval
|
|
227
|
+
variant) uses it to write a per-run path and avoid clobbering.
|
|
228
|
+
"""
|
|
229
|
+
from ..daemon import Client, IndexDir
|
|
230
|
+
from ..lib.backend import apply_remote_config
|
|
231
|
+
from ..lib.llm import auth_error_to_systemexit
|
|
232
|
+
from ..search import search_index
|
|
233
|
+
|
|
234
|
+
repo_root = Path(args.repo_root).resolve()
|
|
235
|
+
idx = IndexDir.for_repo(repo_root, args.index_dir)
|
|
236
|
+
index_dir = idx.path
|
|
237
|
+
|
|
238
|
+
api_key = _require_api_key(getattr(args, "api_key", None), daemon_managed=idx.is_encrypted_layout())
|
|
239
|
+
model = apply_remote_config(index_dir=index_dir, default_model=args.index_search_model)
|
|
240
|
+
_preflight_search(index_dir, args.query, model)
|
|
241
|
+
with auth_error_to_systemexit(index_dir=index_dir, api_key=api_key):
|
|
242
|
+
if idx.is_encrypted_layout():
|
|
243
|
+
client = Client(repo_root)
|
|
244
|
+
result = client.search(args.query, model=model, api_key=api_key)["roadmap"]
|
|
245
|
+
else:
|
|
246
|
+
result = search_index(
|
|
247
|
+
query=args.query,
|
|
248
|
+
repo_root=repo_root,
|
|
249
|
+
index_dir=index_dir,
|
|
250
|
+
index_search_model=model,
|
|
251
|
+
mode=getattr(args, "mode", "implement"),
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
if result.startswith("ERROR:"):
|
|
255
|
+
print(result)
|
|
256
|
+
# Non-zero so callers (e.g. swe_bench_eval's prefix.py with check=True) see the
|
|
257
|
+
# failure — a 0 exit here used to surface later as a FileNotFoundError on --output.
|
|
258
|
+
raise SystemExit(2)
|
|
259
|
+
|
|
260
|
+
output_arg = getattr(args, "output", None)
|
|
261
|
+
output_path = Path(output_arg) if output_arg else index_dir / ROADMAP_FILENAME
|
|
262
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
263
|
+
output_path.write_text(result)
|
|
264
|
+
print(f"Roadmap written to {output_path}")
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _run_preflight(estimator) -> None:
|
|
268
|
+
# SystemExit propagates (budget overrun); other failures are swallowed
|
|
269
|
+
# so a backend hiccup never blocks the action.
|
|
270
|
+
try:
|
|
271
|
+
from ..lib.cost import abort_if_over_budget, fetch_budget
|
|
272
|
+
abort_if_over_budget(estimator(), fetch_budget())
|
|
273
|
+
except SystemExit:
|
|
274
|
+
raise
|
|
275
|
+
except Exception as e: # noqa: BLE001
|
|
276
|
+
_log.debug(
|
|
277
|
+
"pre-flight estimate failed",
|
|
278
|
+
extra={"event": "preflight.failed", "error": str(e)},
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _preflight_indexing(repo_root: Path, index_dir: Path, languages, model: str) -> None:
|
|
283
|
+
# Estimate only files that will actually hit the LLM — skip empties
|
|
284
|
+
# and files already covered by _summary_cache.json (cache hits don't
|
|
285
|
+
# incur an LLM call).
|
|
286
|
+
def estimate() -> float:
|
|
287
|
+
import json
|
|
288
|
+
from ..build.walker import collect_source_files, hash_file_content
|
|
289
|
+
from ..lib.cost import estimate_indexing_cost
|
|
290
|
+
from ..lib.languages import detect_languages, get_languages
|
|
291
|
+
|
|
292
|
+
cache_hashes: set[str] = set()
|
|
293
|
+
cache_path = index_dir / "_summary_cache.json"
|
|
294
|
+
if cache_path.exists():
|
|
295
|
+
try:
|
|
296
|
+
cache_hashes = set(json.loads(cache_path.read_text()).keys())
|
|
297
|
+
except (OSError, ValueError):
|
|
298
|
+
pass # unreadable cache → estimate as if nothing is cached
|
|
299
|
+
|
|
300
|
+
resolved = languages or detect_languages(repo_root) or get_languages()
|
|
301
|
+
sizes: list[int] = []
|
|
302
|
+
for rel in collect_source_files(repo_root, resolved):
|
|
303
|
+
try:
|
|
304
|
+
content = (repo_root / rel).read_text(errors="replace")
|
|
305
|
+
except OSError:
|
|
306
|
+
continue
|
|
307
|
+
if not content.strip():
|
|
308
|
+
continue # indexer skips empty files
|
|
309
|
+
if hash_file_content(content) in cache_hashes:
|
|
310
|
+
continue # cache hit — no LLM call
|
|
311
|
+
sizes.append(len(content.encode()))
|
|
312
|
+
return estimate_indexing_cost(sizes, model)[2]
|
|
313
|
+
_run_preflight(estimate)
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _preflight_search(index_dir: Path, query: str, model: str) -> None:
|
|
317
|
+
def estimate() -> float:
|
|
318
|
+
from ..lib.cost import estimate_search_cost
|
|
319
|
+
index_md = index_dir / "index.md"
|
|
320
|
+
details = index_dir / "details"
|
|
321
|
+
index_md_size = index_md.stat().st_size if index_md.exists() else 0
|
|
322
|
+
tier2_total = 0
|
|
323
|
+
if details.exists():
|
|
324
|
+
for p in details.rglob("*.md"):
|
|
325
|
+
try:
|
|
326
|
+
tier2_total += p.stat().st_size
|
|
327
|
+
except OSError:
|
|
328
|
+
continue
|
|
329
|
+
return estimate_search_cost(index_md_size, tier2_total, len(query), model)[2]
|
|
330
|
+
_run_preflight(estimate)
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def cmd_dump(args: argparse.Namespace) -> None:
|
|
334
|
+
from ..daemon import AdminSocketUnavailable, Client, IndexDir
|
|
335
|
+
|
|
336
|
+
repo_root = Path(args.repo_root).resolve()
|
|
337
|
+
idx = IndexDir.for_repo(repo_root)
|
|
338
|
+
out_dir = Path(args.out).resolve()
|
|
339
|
+
|
|
340
|
+
if not idx.is_encrypted_layout() and idx.has_plaintext_index():
|
|
341
|
+
# Plaintext layout: just copy. Gives eval a single dump entry
|
|
342
|
+
# point regardless of whether init was encrypted.
|
|
343
|
+
from ..daemon import PlaintextStore
|
|
344
|
+
store = PlaintextStore(idx.path)
|
|
345
|
+
files, total = store.plaintext_dump(out_dir)
|
|
346
|
+
_log.info(
|
|
347
|
+
f"Copied plaintext store to {out_dir} ({files} files, {total} bytes)",
|
|
348
|
+
extra={"event": "dump.plaintext_copy", "out_dir": str(out_dir),
|
|
349
|
+
"files": files, "bytes": total},
|
|
350
|
+
)
|
|
351
|
+
return
|
|
352
|
+
|
|
353
|
+
client = Client(repo_root, admin=True)
|
|
354
|
+
try:
|
|
355
|
+
result = client.dump_all(out_dir)
|
|
356
|
+
except AdminSocketUnavailable as e:
|
|
357
|
+
raise SystemExit(
|
|
358
|
+
f"[sourceindex] {e}\n"
|
|
359
|
+
"Hint: pass --allow-dump to `sourceindex init` or "
|
|
360
|
+
"`sourceindex daemon` to enable the admin socket."
|
|
361
|
+
)
|
|
362
|
+
_log.info(
|
|
363
|
+
f"Dumped encrypted store to {out_dir} "
|
|
364
|
+
f"({result.get('files_written', '?')} files, {result.get('bytes', '?')} bytes)",
|
|
365
|
+
extra={"event": "dump.done", "out_dir": str(out_dir), **result},
|
|
366
|
+
)
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _deinit_one(repo_root: Path, repo_hash: str) -> dict:
|
|
370
|
+
"""Drop the encryption key + runtime files for a single repo. Caller is
|
|
371
|
+
responsible for the safety check (deletion of a live <repo>/.sourceindex/
|
|
372
|
+
must be opt-in). Returns telemetry suitable for a log `extra` payload."""
|
|
373
|
+
import shutil
|
|
374
|
+
from ..daemon import IndexDir, lifecycle
|
|
375
|
+
from ..daemon.keyring_store import delete_key
|
|
376
|
+
from ..lib import registry
|
|
377
|
+
|
|
378
|
+
daemon_stopped = lifecycle.stop_daemon(repo_root)
|
|
379
|
+
delete_key(repo_hash)
|
|
380
|
+
lifecycle.cleanup_stale_files(repo_hash)
|
|
381
|
+
index_removed = False
|
|
382
|
+
idx_path = IndexDir.for_repo(repo_root).path
|
|
383
|
+
if idx_path.exists():
|
|
384
|
+
shutil.rmtree(idx_path)
|
|
385
|
+
index_removed = True
|
|
386
|
+
registry.remove(repo_hash)
|
|
387
|
+
return {
|
|
388
|
+
"daemon_stopped": daemon_stopped,
|
|
389
|
+
"index_removed": index_removed,
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def cmd_deinit(args: argparse.Namespace) -> None:
|
|
394
|
+
from ..daemon import IndexDir
|
|
395
|
+
from ..lib import registry
|
|
396
|
+
|
|
397
|
+
if getattr(args, "all", False):
|
|
398
|
+
# Login-time sweep: forget every registered repo whose path has gone
|
|
399
|
+
# missing. Live repos are skipped — they're not orphans.
|
|
400
|
+
swept = []
|
|
401
|
+
kept = 0
|
|
402
|
+
for repo_hash, repo_path in registry.read():
|
|
403
|
+
if Path(repo_path).exists():
|
|
404
|
+
kept += 1
|
|
405
|
+
continue
|
|
406
|
+
telemetry = _deinit_one(Path(repo_path), repo_hash)
|
|
407
|
+
swept.append((repo_hash, repo_path, telemetry))
|
|
408
|
+
_log.info(
|
|
409
|
+
"Sweep complete: %d orphan repo(s) cleaned, %d live repo(s) left alone",
|
|
410
|
+
len(swept), kept,
|
|
411
|
+
extra={
|
|
412
|
+
"event": "cli.deinit_all",
|
|
413
|
+
"swept": [{"repo_hash": h, "repo_path": p, **t} for h, p, t in swept],
|
|
414
|
+
"kept": kept,
|
|
415
|
+
},
|
|
416
|
+
)
|
|
417
|
+
return
|
|
418
|
+
|
|
419
|
+
repo_root = Path(args.repo_root).resolve()
|
|
420
|
+
idx = IndexDir.for_repo(repo_root)
|
|
421
|
+
|
|
422
|
+
# Safety: a live store is real data. Require --force so the user
|
|
423
|
+
# opts into deleting it rather than getting it wiped by surprise.
|
|
424
|
+
if not args.force and idx.path.exists():
|
|
425
|
+
_log.error(
|
|
426
|
+
"Refusing to deinit: %s still exists. Pass --force to remove "
|
|
427
|
+
"the encrypted index, drop the key, and stop the daemon.",
|
|
428
|
+
idx.path,
|
|
429
|
+
)
|
|
430
|
+
sys.exit(2)
|
|
431
|
+
|
|
432
|
+
telemetry = _deinit_one(repo_root, idx.repo_hash)
|
|
433
|
+
_log.info(
|
|
434
|
+
"Deinit complete for %s (repo_hash=%s, daemon_stopped=%s, "
|
|
435
|
+
"index_removed=%s)",
|
|
436
|
+
repo_root, idx.repo_hash, telemetry["daemon_stopped"],
|
|
437
|
+
telemetry["index_removed"],
|
|
438
|
+
extra={"event": "cli.deinit", "repo_hash": idx.repo_hash, **telemetry},
|
|
439
|
+
)
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
_SYSTEMD_UNIT_NAME = "sourceindex-sweep.service"
|
|
443
|
+
_SYSTEMD_UNIT_TEMPLATE = """\
|
|
444
|
+
[Unit]
|
|
445
|
+
Description=SourceIndex login-time sweep — remove orphan state for deleted repos
|
|
446
|
+
|
|
447
|
+
[Service]
|
|
448
|
+
Type=oneshot
|
|
449
|
+
ExecStart={binary} deinit --all
|
|
450
|
+
|
|
451
|
+
[Install]
|
|
452
|
+
WantedBy=default.target
|
|
453
|
+
"""
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def _ensure_autostart_installed() -> None:
|
|
457
|
+
"""Install and enable the systemd --user unit that runs deinit --all at
|
|
458
|
+
each login. Idempotent — re-runs of `sourceindex init` are no-ops once
|
|
459
|
+
the unit is present and current. Non-fatal: a missing systemctl just
|
|
460
|
+
logs a warning so the rest of init still completes."""
|
|
461
|
+
import shutil as _shutil
|
|
462
|
+
import subprocess
|
|
463
|
+
|
|
464
|
+
if sys.platform != "linux":
|
|
465
|
+
return
|
|
466
|
+
|
|
467
|
+
binary = _shutil.which("sourceindex") or sys.argv[0]
|
|
468
|
+
desired = _SYSTEMD_UNIT_TEMPLATE.format(binary=binary)
|
|
469
|
+
unit_dir = Path.home() / ".config" / "systemd" / "user"
|
|
470
|
+
unit_path = unit_dir / _SYSTEMD_UNIT_NAME
|
|
471
|
+
if unit_path.is_file() and unit_path.read_text() == desired:
|
|
472
|
+
return
|
|
473
|
+
|
|
474
|
+
unit_dir.mkdir(parents=True, exist_ok=True)
|
|
475
|
+
unit_path.write_text(desired)
|
|
476
|
+
try:
|
|
477
|
+
subprocess.run(["systemctl", "--user", "daemon-reload"], check=True, capture_output=True)
|
|
478
|
+
subprocess.run(["systemctl", "--user", "enable", _SYSTEMD_UNIT_NAME], check=True, capture_output=True)
|
|
479
|
+
except FileNotFoundError:
|
|
480
|
+
_log.warning(
|
|
481
|
+
"systemctl not found; wrote %s but couldn't enable it. "
|
|
482
|
+
"Reboot-then-delete orphans won't be cleaned automatically.",
|
|
483
|
+
unit_path,
|
|
484
|
+
)
|
|
485
|
+
return
|
|
486
|
+
except subprocess.CalledProcessError as e:
|
|
487
|
+
_log.warning(
|
|
488
|
+
"systemctl returned %s while enabling the login-time sweep: %s. "
|
|
489
|
+
"Unit file is at %s — enable it manually if you want auto-cleanup.",
|
|
490
|
+
e.returncode, e.stderr.decode(errors="replace").strip(), unit_path,
|
|
491
|
+
)
|
|
492
|
+
return
|
|
493
|
+
_log.info(
|
|
494
|
+
"Installed login-time orphan sweep at %s",
|
|
495
|
+
unit_path,
|
|
496
|
+
extra={"event": "init.autostart_installed", "unit_path": str(unit_path)},
|
|
497
|
+
)
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def cmd_daemon(args: argparse.Namespace) -> None:
|
|
501
|
+
from ..daemon import lifecycle, run_daemon
|
|
502
|
+
|
|
503
|
+
repo_root = Path(args.repo_root).resolve()
|
|
504
|
+
if args.stop:
|
|
505
|
+
stopped = lifecycle.stop_daemon(repo_root)
|
|
506
|
+
if stopped:
|
|
507
|
+
_log.info("Daemon stopped", extra={"event": "daemon.stopped"})
|
|
508
|
+
else:
|
|
509
|
+
_log.info("No daemon was running", extra={"event": "daemon.not_running"})
|
|
510
|
+
return
|
|
511
|
+
run_daemon(repo_root, allow_dump=bool(args.allow_dump))
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def cmd_usage(args: argparse.Namespace) -> None:
|
|
515
|
+
from ..lib.cost import fetch_budget
|
|
516
|
+
|
|
517
|
+
budget = fetch_budget()
|
|
518
|
+
if budget is None:
|
|
519
|
+
print(
|
|
520
|
+
"[sourceindex] Could not reach the hosted backend. "
|
|
521
|
+
"Check SOURCEINDEX_BACKEND_URL / SOURCEINDEX_API_KEY, "
|
|
522
|
+
"or set SOURCEINDEX_BACKEND_URL=\"\" to opt out of the backend."
|
|
523
|
+
)
|
|
524
|
+
return
|
|
525
|
+
if not budget.get("minted"):
|
|
526
|
+
print(
|
|
527
|
+
"[sourceindex] No usage this month yet — your virtual key "
|
|
528
|
+
"will be minted on your first indexing call."
|
|
529
|
+
)
|
|
530
|
+
return
|
|
531
|
+
|
|
532
|
+
month = budget.get("month", "?")
|
|
533
|
+
used = float(budget.get("used_usd") or 0)
|
|
534
|
+
limit = budget.get("limit_usd")
|
|
535
|
+
remaining = budget.get("remaining_usd")
|
|
536
|
+
lines = [f"[sourceindex] Month {month}", f"[sourceindex] Used: ${used:.4f}"]
|
|
537
|
+
if limit is not None:
|
|
538
|
+
lines.append(f"[sourceindex] Cap: ${limit:.2f}")
|
|
539
|
+
if remaining is not None:
|
|
540
|
+
lines.append(f"[sourceindex] Remaining: ${remaining:.4f}")
|
|
541
|
+
lines.append("[sourceindex] Resets: first of next month (UTC)")
|
|
542
|
+
print("\n".join(lines))
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
def cmd_report_savings(args: argparse.Namespace) -> None:
|
|
546
|
+
"""Stop-hook entry point — read payload JSON from stdin, persist savings ledger."""
|
|
547
|
+
from ..claudecode import savings
|
|
548
|
+
|
|
549
|
+
savings.main()
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def cmd_statusline(args: argparse.Namespace) -> None:
|
|
553
|
+
"""statusLine entry point — read payload JSON from stdin, render savings line."""
|
|
554
|
+
from ..claudecode import statusline
|
|
555
|
+
|
|
556
|
+
statusline.main()
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def cmd_savings_summary(args: argparse.Namespace) -> None:
|
|
560
|
+
"""Stop-hook entry point — print a per-turn multi-line savings breakdown
|
|
561
|
+
(session/week/month). Pairs with ``report-savings`` as a second Stop
|
|
562
|
+
hook: that one updates the ledger silently, this one renders detail."""
|
|
563
|
+
from ..claudecode import savings_summary
|
|
564
|
+
|
|
565
|
+
savings_summary.main()
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
def _local_timepart(ts: str) -> str:
|
|
569
|
+
"""Convert a UTC ``ts`` string (``...Z``) to a local ``HH:MM:SS`` time."""
|
|
570
|
+
if not ts:
|
|
571
|
+
return ts
|
|
572
|
+
try:
|
|
573
|
+
dt = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=timezone.utc)
|
|
574
|
+
return dt.astimezone().strftime("%H:%M:%S")
|
|
575
|
+
except (ValueError, TypeError):
|
|
576
|
+
return ts[11:19] if len(ts) >= 19 else ts
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def _format_log_record(rec: dict) -> str | None:
|
|
580
|
+
"""Render one JSONL log record as a compact summary line. Return ``None``
|
|
581
|
+
to suppress the record (covered by a sibling event, or pure debug noise)."""
|
|
582
|
+
ts = rec.get("ts") or ""
|
|
583
|
+
timepart = _local_timepart(ts)
|
|
584
|
+
event = rec.get("event") or ""
|
|
585
|
+
msg = rec.get("message") or ""
|
|
586
|
+
level = rec.get("level") or ""
|
|
587
|
+
|
|
588
|
+
if event == "cli.invocation":
|
|
589
|
+
return f"{timepart} cli sourceindex {rec.get('command', '?')}"
|
|
590
|
+
if event == "update.start":
|
|
591
|
+
return f"{timepart} update start: {rec.get('changed_count', '?')} files"
|
|
592
|
+
if event == "update.cache_hit":
|
|
593
|
+
return f"{timepart} update cached: {rec.get('path', '?')}"
|
|
594
|
+
if event == "update.done":
|
|
595
|
+
return (
|
|
596
|
+
f"{timepart} update done: {rec.get('files_indexed', 0)} files "
|
|
597
|
+
f"· ${rec.get('cost_usd', 0):.4f} · {rec.get('elapsed_s', 0):.1f}s"
|
|
598
|
+
)
|
|
599
|
+
if event == "search.start":
|
|
600
|
+
q = (rec.get("query") or "").strip().replace("\n", " ")
|
|
601
|
+
if len(q) > 60:
|
|
602
|
+
q = q[:57] + "..."
|
|
603
|
+
return f'{timepart} search start: "{q}"'
|
|
604
|
+
if event == "search.pass_1a":
|
|
605
|
+
return f"{timepart} search pass 1a: {rec.get('files_selected', '?')} files"
|
|
606
|
+
if event == "search.pass_1b":
|
|
607
|
+
return f"{timepart} search pass 1b: {rec.get('functions_selected', '?')} functions"
|
|
608
|
+
if event == "search.done":
|
|
609
|
+
return (
|
|
610
|
+
f"{timepart} search done: {rec.get('roadmap_bytes', 0)} bytes "
|
|
611
|
+
f"· {rec.get('latency_ms', 0) / 1000:.1f}s"
|
|
612
|
+
)
|
|
613
|
+
if event == "llm.call":
|
|
614
|
+
model = rec.get("model") or "?"
|
|
615
|
+
short = model.rsplit("/", 1)[-1]
|
|
616
|
+
cost = rec.get("cost_usd") or 0.0
|
|
617
|
+
latency = (rec.get("latency_ms") or 0) / 1000.0
|
|
618
|
+
return (
|
|
619
|
+
f"{timepart} llm {short} · "
|
|
620
|
+
f"in={rec.get('input_tokens', 0)} out={rec.get('output_tokens', 0)} · "
|
|
621
|
+
f"${cost:.4f} · {latency:.1f}s"
|
|
622
|
+
)
|
|
623
|
+
if event == "llm.cost_unknown":
|
|
624
|
+
tried = rec.get("tried")
|
|
625
|
+
tail = f" (tried: {', '.join(tried)})" if tried else ""
|
|
626
|
+
return f"{timepart} warn cost lookup failed for {rec.get('model', '?')}{tail}"
|
|
627
|
+
if event == "lazy_refresh.skipped":
|
|
628
|
+
return (
|
|
629
|
+
f"{timepart} warn refresh skipped: {rec.get('reason', '?')} "
|
|
630
|
+
f"({rec.get('changed_count', 0)} files changed)"
|
|
631
|
+
)
|
|
632
|
+
if event in (
|
|
633
|
+
"update.concurrent_indexing", "batch.dispatch", "batch.results",
|
|
634
|
+
"search.pass_1a.start", "search.pass_1b.start",
|
|
635
|
+
):
|
|
636
|
+
return None
|
|
637
|
+
if level in ("INFO", "WARNING", "ERROR"):
|
|
638
|
+
return f"{timepart} {level.lower():8s} {msg or event}"
|
|
639
|
+
return None
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
def cmd_log(args: argparse.Namespace) -> None:
|
|
643
|
+
"""Pretty-print the sourceindex JSONL log."""
|
|
644
|
+
import json
|
|
645
|
+
from ..daemon import IndexDir
|
|
646
|
+
|
|
647
|
+
repo_root = Path(args.repo_root).resolve()
|
|
648
|
+
log_path = (
|
|
649
|
+
Path(args.log_path).resolve()
|
|
650
|
+
if args.log_path
|
|
651
|
+
else IndexDir.for_repo(repo_root).path / "sourceindex.log"
|
|
652
|
+
)
|
|
653
|
+
if not log_path.is_file():
|
|
654
|
+
raise SystemExit(f"[sourceindex] No log file at {log_path}")
|
|
655
|
+
log_lines = log_path.read_text().splitlines()
|
|
656
|
+
|
|
657
|
+
filt = (args.filter or "").lower()
|
|
658
|
+
rendered: list[str] = []
|
|
659
|
+
for line in log_lines:
|
|
660
|
+
if not line.strip():
|
|
661
|
+
continue
|
|
662
|
+
try:
|
|
663
|
+
rec = json.loads(line)
|
|
664
|
+
except json.JSONDecodeError:
|
|
665
|
+
continue
|
|
666
|
+
if filt and filt not in (rec.get("event") or "").lower():
|
|
667
|
+
continue
|
|
668
|
+
out = _format_log_record(rec)
|
|
669
|
+
if out:
|
|
670
|
+
rendered.append(out)
|
|
671
|
+
|
|
672
|
+
# Slice after formatting so --tail N counts visible lines, not raw
|
|
673
|
+
# JSONL records (which can collapse 10 → 2 once suppressed events,
|
|
674
|
+
# batch chatter, and unrecognized DEBUG entries are dropped).
|
|
675
|
+
if args.tail and args.tail > 0:
|
|
676
|
+
rendered = rendered[-args.tail :]
|
|
677
|
+
for line in rendered:
|
|
678
|
+
print(line)
|