SourceIndex 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sourceindex/__init__.py +30 -0
- sourceindex/build/__init__.py +592 -0
- sourceindex/build/indexer.py +403 -0
- sourceindex/build/linerange/__init__.py +24 -0
- sourceindex/build/linerange/python_ast.py +155 -0
- sourceindex/build/linerange/treesitter.py +397 -0
- sourceindex/build/prompts.py +76 -0
- sourceindex/build/state.py +261 -0
- sourceindex/build/walker.py +94 -0
- sourceindex/claudecode/__init__.py +0 -0
- sourceindex/claudecode/savings.py +325 -0
- sourceindex/claudecode/savings_summary.py +88 -0
- sourceindex/claudecode/statusline.py +116 -0
- sourceindex/cli/__init__.py +304 -0
- sourceindex/cli/__main__.py +10 -0
- sourceindex/cli/api_key.py +171 -0
- sourceindex/cli/commands.py +678 -0
- sourceindex/cli/install.py +378 -0
- sourceindex/daemon/__init__.py +83 -0
- sourceindex/daemon/client.py +141 -0
- sourceindex/daemon/crypto.py +48 -0
- sourceindex/daemon/keyring_store.py +129 -0
- sourceindex/daemon/lifecycle.py +237 -0
- sourceindex/daemon/protocol.py +134 -0
- sourceindex/daemon/server.py +389 -0
- sourceindex/daemon/store.py +426 -0
- sourceindex/lib/__init__.py +0 -0
- sourceindex/lib/backend.py +240 -0
- sourceindex/lib/cost.py +96 -0
- sourceindex/lib/env.py +30 -0
- sourceindex/lib/git.py +33 -0
- sourceindex/lib/languages.py +406 -0
- sourceindex/lib/llm.py +298 -0
- sourceindex/lib/log.py +228 -0
- sourceindex/lib/registry.py +75 -0
- sourceindex/lib/timing.py +10 -0
- sourceindex/search/__init__.py +251 -0
- sourceindex/search/experiments.py +276 -0
- sourceindex/search/imports.py +155 -0
- sourceindex/search/passes.py +51 -0
- sourceindex/search/prompts.py +379 -0
- sourceindex/search/roadmap.py +265 -0
- sourceindex/server.py +127 -0
- sourceindex-0.1.0.dist-info/METADATA +73 -0
- sourceindex-0.1.0.dist-info/RECORD +47 -0
- sourceindex-0.1.0.dist-info/WHEEL +4 -0
- sourceindex-0.1.0.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
"""Persistent JSON state for the sourceindex cache directory.
|
|
2
|
+
|
|
3
|
+
Three artifacts share this module because they're all "what we know about
|
|
4
|
+
the index after the last run":
|
|
5
|
+
|
|
6
|
+
- ``_summary_cache.json`` — content-hash → {oneliner, functions} so unchanged
|
|
7
|
+
files skip re-summarization across builds.
|
|
8
|
+
- ``_file_index.json`` — rel_path → {mtime_ns, hash} so lazy refresh can
|
|
9
|
+
detect working-tree edits without re-hashing every file.
|
|
10
|
+
- ``build_meta.json`` — model + token + cost + timing accounting, merged
|
|
11
|
+
across init and update runs. Read by eval_summarize.py — schema is stable;
|
|
12
|
+
add new fields only at the end.
|
|
13
|
+
|
|
14
|
+
All read/write goes through a duck-typed ``store`` object (one of
|
|
15
|
+
``daemon.EncryptedStore`` / ``daemon.PlaintextStore``). State logic stays
|
|
16
|
+
ignorant of whether bytes on disk are encrypted — the store handles that.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
from datetime import datetime, timezone
|
|
21
|
+
|
|
22
|
+
from .. import __version__ as _SOURCEINDEX_VERSION
|
|
23
|
+
from ..lib.languages import Language
|
|
24
|
+
from ..lib.llm import LLMUsage
|
|
25
|
+
from ..lib.log import get_logger
|
|
26
|
+
|
|
27
|
+
_log = get_logger(__name__)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
SUMMARY_CACHE_FILENAME = "_summary_cache.json"
|
|
31
|
+
FILE_INDEX_FILENAME = "_file_index.json"
|
|
32
|
+
BUILD_META_FILENAME = "build_meta.json"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
# ── Summary cache ──────────────────────────────────────────────────────────
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def load_summary_cache(store) -> dict[str, dict[str, str]]:
|
|
39
|
+
"""Load {content_hash: {"oneliner": ..., "functions": ...}} from store."""
|
|
40
|
+
if not store.exists(SUMMARY_CACHE_FILENAME):
|
|
41
|
+
return {}
|
|
42
|
+
try:
|
|
43
|
+
data = json.loads(store.read_text(SUMMARY_CACHE_FILENAME))
|
|
44
|
+
except (json.JSONDecodeError, OSError, Exception) as e:
|
|
45
|
+
_log.warning(
|
|
46
|
+
f"Summary cache at {store.path_for(SUMMARY_CACHE_FILENAME)} is unreadable; starting from empty cache.",
|
|
47
|
+
extra={
|
|
48
|
+
"event": "cache.decode_failed",
|
|
49
|
+
"kind": "summary_cache",
|
|
50
|
+
"path": str(store.path_for(SUMMARY_CACHE_FILENAME)),
|
|
51
|
+
"error_type": type(e).__name__,
|
|
52
|
+
"error": str(e),
|
|
53
|
+
},
|
|
54
|
+
)
|
|
55
|
+
return {}
|
|
56
|
+
# Migrate legacy format: {hash: oneliner_string}
|
|
57
|
+
migrated: dict[str, dict[str, str]] = {}
|
|
58
|
+
for h, v in data.items():
|
|
59
|
+
if isinstance(v, str):
|
|
60
|
+
migrated[h] = {"oneliner": v, "functions": ""}
|
|
61
|
+
elif isinstance(v, dict):
|
|
62
|
+
migrated[h] = {
|
|
63
|
+
"oneliner": v.get("oneliner", ""),
|
|
64
|
+
"functions": v.get("functions", ""),
|
|
65
|
+
}
|
|
66
|
+
return migrated
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def save_summary_cache(store, cache: dict[str, dict[str, str]]) -> None:
|
|
70
|
+
"""Persist summary cache to store."""
|
|
71
|
+
store.write_text(
|
|
72
|
+
SUMMARY_CACHE_FILENAME,
|
|
73
|
+
json.dumps(cache, indent=1, sort_keys=True) + "\n",
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
# ── File index (path → mtime + hash) ───────────────────────────────────────
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def load_file_index(store) -> dict[str, dict]:
|
|
81
|
+
"""Load {rel_path: {"mtime_ns": int, "hash": str}} from disk.
|
|
82
|
+
|
|
83
|
+
Path-keyed companion to ``_summary_cache.json`` (which is keyed by
|
|
84
|
+
content hash for cross-file dedupe). Used by lazy refresh to detect
|
|
85
|
+
edits since the last index update.
|
|
86
|
+
"""
|
|
87
|
+
if not store.exists(FILE_INDEX_FILENAME):
|
|
88
|
+
return {}
|
|
89
|
+
try:
|
|
90
|
+
data = json.loads(store.read_text(FILE_INDEX_FILENAME))
|
|
91
|
+
except (json.JSONDecodeError, OSError, Exception) as e:
|
|
92
|
+
_log.warning(
|
|
93
|
+
f"File index at {store.path_for(FILE_INDEX_FILENAME)} is unreadable; treating as empty.",
|
|
94
|
+
extra={
|
|
95
|
+
"event": "cache.decode_failed",
|
|
96
|
+
"kind": "file_index",
|
|
97
|
+
"path": str(store.path_for(FILE_INDEX_FILENAME)),
|
|
98
|
+
"error_type": type(e).__name__,
|
|
99
|
+
"error": str(e),
|
|
100
|
+
},
|
|
101
|
+
)
|
|
102
|
+
return {}
|
|
103
|
+
if not isinstance(data, dict):
|
|
104
|
+
return {}
|
|
105
|
+
paths = data.get("paths", {})
|
|
106
|
+
return paths if isinstance(paths, dict) else {}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def save_file_index(store, paths: dict[str, dict]) -> None:
|
|
110
|
+
"""Persist path-keyed file index to store."""
|
|
111
|
+
payload = {"version": 1, "paths": paths}
|
|
112
|
+
store.write_text(
|
|
113
|
+
FILE_INDEX_FILENAME,
|
|
114
|
+
json.dumps(payload, indent=1, sort_keys=True) + "\n",
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def write_details(store, rel_path: str, functions_text: str) -> None:
|
|
119
|
+
"""Write tier-2 function index to details/<rel_path>.md."""
|
|
120
|
+
body = functions_text.strip() or "(LLM indexing failed)"
|
|
121
|
+
store.write_text(f"details/{rel_path}.md", f"# {rel_path}\n\n{body}\n")
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def details_exists(store, rel_path: str) -> bool:
|
|
125
|
+
return store.exists(f"details/{rel_path}.md")
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def try_summary_cache_hit(
|
|
129
|
+
summary_cache: dict[str, dict[str, str]],
|
|
130
|
+
content_hash: str,
|
|
131
|
+
lang: Language,
|
|
132
|
+
store,
|
|
133
|
+
rel_path: str,
|
|
134
|
+
) -> tuple[str, str] | None:
|
|
135
|
+
"""Return ``(oneliner, functions)`` if the summary cache + on-store
|
|
136
|
+
details are sufficient to skip an LLM call. Writes the details
|
|
137
|
+
file from cache if missing. Otherwise returns ``None``."""
|
|
138
|
+
cached = summary_cache.get(content_hash)
|
|
139
|
+
if not cached or not cached.get("oneliner"):
|
|
140
|
+
return None
|
|
141
|
+
oneliner = cached["oneliner"]
|
|
142
|
+
functions = cached.get("functions", "")
|
|
143
|
+
if not lang.build_tier2:
|
|
144
|
+
return oneliner, functions
|
|
145
|
+
if not (details_exists(store, rel_path) or functions):
|
|
146
|
+
return None
|
|
147
|
+
if not details_exists(store, rel_path):
|
|
148
|
+
write_details(store, rel_path, functions)
|
|
149
|
+
return oneliner, functions
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# ── Build meta accounting ──────────────────────────────────────────────────
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def utcnow_iso() -> str:
|
|
156
|
+
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _compose_meta(
|
|
160
|
+
base: dict,
|
|
161
|
+
*,
|
|
162
|
+
model: str,
|
|
163
|
+
files_indexed: int,
|
|
164
|
+
files_from_cache: int | None,
|
|
165
|
+
usage: LLMUsage,
|
|
166
|
+
elapsed_s: float,
|
|
167
|
+
started_at: str,
|
|
168
|
+
) -> dict:
|
|
169
|
+
"""Build the canonical ``build_meta.json`` dict.
|
|
170
|
+
|
|
171
|
+
``base`` is the previous meta (or ``{}`` for a fresh build). Numeric
|
|
172
|
+
fields sum ``base + new``, the model string is replaced with the
|
|
173
|
+
latest-used model, and ``started_at`` carries forward from the
|
|
174
|
+
original build. ``files_from_cache=None`` skips updating that field
|
|
175
|
+
(update-mode doesn't have a hit/miss split). Schema is stable —
|
|
176
|
+
eval_summarize.py reads it; add new fields only at the end so old
|
|
177
|
+
summarizers keep working."""
|
|
178
|
+
return {
|
|
179
|
+
"model": model,
|
|
180
|
+
"files_indexed": int(base.get("files_indexed", 0)) + files_indexed,
|
|
181
|
+
"files_from_cache": int(base.get("files_from_cache", 0))
|
|
182
|
+
if files_from_cache is None
|
|
183
|
+
else files_from_cache,
|
|
184
|
+
"input_tokens": int(base.get("input_tokens", 0)) + usage.input_tokens,
|
|
185
|
+
"output_tokens": int(base.get("output_tokens", 0)) + usage.output_tokens,
|
|
186
|
+
"cache_read_tokens": int(base.get("cache_read_tokens", 0)) + usage.cache_read_tokens,
|
|
187
|
+
"cache_creation_tokens": int(base.get("cache_creation_tokens", 0)) + usage.cache_creation_tokens,
|
|
188
|
+
"cost_usd": round(float(base.get("cost_usd", 0.0)) + usage.cost_usd, 6),
|
|
189
|
+
"elapsed_s": round(float(base.get("elapsed_s", 0.0)) + elapsed_s, 3),
|
|
190
|
+
"started_at": base.get("started_at", started_at),
|
|
191
|
+
"sourceindex_version": _SOURCEINDEX_VERSION,
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _read_existing_meta(store) -> dict:
|
|
196
|
+
if not store.exists(BUILD_META_FILENAME):
|
|
197
|
+
return {}
|
|
198
|
+
try:
|
|
199
|
+
return json.loads(store.read_text(BUILD_META_FILENAME))
|
|
200
|
+
except (json.JSONDecodeError, OSError, Exception) as e:
|
|
201
|
+
_log.warning(
|
|
202
|
+
f"Build meta at {store.path_for(BUILD_META_FILENAME)} is unreadable; treating as fresh.",
|
|
203
|
+
extra={
|
|
204
|
+
"event": "cache.decode_failed",
|
|
205
|
+
"kind": "build_meta",
|
|
206
|
+
"path": str(store.path_for(BUILD_META_FILENAME)),
|
|
207
|
+
"error_type": type(e).__name__,
|
|
208
|
+
"error": str(e),
|
|
209
|
+
},
|
|
210
|
+
)
|
|
211
|
+
return {}
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def write_build_meta(
|
|
215
|
+
store,
|
|
216
|
+
*,
|
|
217
|
+
model: str,
|
|
218
|
+
files_indexed: int,
|
|
219
|
+
files_from_cache: int,
|
|
220
|
+
usage: LLMUsage,
|
|
221
|
+
elapsed_s: float,
|
|
222
|
+
started_at: str,
|
|
223
|
+
) -> None:
|
|
224
|
+
"""Fresh-build write: replace any existing meta with new values."""
|
|
225
|
+
meta = _compose_meta(
|
|
226
|
+
base={},
|
|
227
|
+
model=model,
|
|
228
|
+
files_indexed=files_indexed,
|
|
229
|
+
files_from_cache=files_from_cache,
|
|
230
|
+
usage=usage,
|
|
231
|
+
elapsed_s=elapsed_s,
|
|
232
|
+
started_at=started_at,
|
|
233
|
+
)
|
|
234
|
+
store.write_text(BUILD_META_FILENAME, json.dumps(meta, indent=2) + "\n")
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def merge_build_meta(
|
|
238
|
+
store,
|
|
239
|
+
*,
|
|
240
|
+
model: str,
|
|
241
|
+
files_indexed: int,
|
|
242
|
+
usage: LLMUsage,
|
|
243
|
+
elapsed_s: float,
|
|
244
|
+
started_at: str,
|
|
245
|
+
) -> None:
|
|
246
|
+
"""Update-mode merge: sum tokens/cost into any existing meta and
|
|
247
|
+
append ``last_updated_at``. No-op when ``files_indexed == 0``."""
|
|
248
|
+
if files_indexed == 0:
|
|
249
|
+
return
|
|
250
|
+
base = _read_existing_meta(store)
|
|
251
|
+
meta = _compose_meta(
|
|
252
|
+
base=base,
|
|
253
|
+
model=model,
|
|
254
|
+
files_indexed=files_indexed,
|
|
255
|
+
files_from_cache=None,
|
|
256
|
+
usage=usage,
|
|
257
|
+
elapsed_s=elapsed_s,
|
|
258
|
+
started_at=started_at,
|
|
259
|
+
)
|
|
260
|
+
meta["last_updated_at"] = utcnow_iso()
|
|
261
|
+
store.write_text(BUILD_META_FILENAME, json.dumps(meta, indent=2) + "\n")
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""Filesystem traversal + content hashing + mtime-delta change detection.
|
|
2
|
+
|
|
3
|
+
Owns the question "which files in the working tree need (re-)indexing?"
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import hashlib
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from ..lib.languages import Language, all_extensions, all_skip_dirs
|
|
11
|
+
from ..lib.log import get_logger, log_file_error
|
|
12
|
+
from .state import load_file_index
|
|
13
|
+
|
|
14
|
+
_log = get_logger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def collect_source_files(root: str | Path, languages: list[Language]) -> list[str]:
|
|
18
|
+
"""Collect all source files for the given languages, pruning non-source dirs."""
|
|
19
|
+
root_path = Path(root)
|
|
20
|
+
skip = all_skip_dirs(languages)
|
|
21
|
+
exts = all_extensions(languages)
|
|
22
|
+
found: list[str] = []
|
|
23
|
+
for dirpath, dirnames, filenames in os.walk(root_path):
|
|
24
|
+
dirnames[:] = [d for d in dirnames if d not in skip]
|
|
25
|
+
for f in filenames:
|
|
26
|
+
if f.lower().endswith(exts):
|
|
27
|
+
found.append(str(Path(dirpath, f).relative_to(root_path)))
|
|
28
|
+
return sorted(found)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def hash_file_content(content: str) -> str:
|
|
32
|
+
"""SHA-256 prefix of file content for deduplication."""
|
|
33
|
+
return hashlib.sha256(content.encode()).hexdigest()[:16]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def detect_changes_by_mtime(
|
|
37
|
+
repo_root: Path,
|
|
38
|
+
store,
|
|
39
|
+
languages: list[Language],
|
|
40
|
+
) -> tuple[list[str], dict[str, dict], list[str], dict[str, dict]]:
|
|
41
|
+
"""Walk the working tree, diff each path's mtime against _file_index.json.
|
|
42
|
+
|
|
43
|
+
Returns ``(changed, updated_path_map, deleted, stored)``:
|
|
44
|
+
|
|
45
|
+
- ``changed``: paths whose content hash differs from what's stored
|
|
46
|
+
(i.e. need to be re-indexed). The caller routes these through
|
|
47
|
+
``update_index``, which itself short-circuits on summary_cache hits.
|
|
48
|
+
- ``updated_path_map``: complete path → {mtime_ns, hash} map for the
|
|
49
|
+
working tree as it stands now. Authoritative — caller persists this
|
|
50
|
+
verbatim. Includes touch-only and unchanged entries.
|
|
51
|
+
- ``deleted``: paths that were in the stored map but no longer exist on
|
|
52
|
+
disk (caller must remove them from index.md and details/).
|
|
53
|
+
- ``stored``: the previously persisted map. Returned so callers can
|
|
54
|
+
compare without re-reading the file.
|
|
55
|
+
|
|
56
|
+
Touch-only edits (mtime moved but content hash unchanged) skip the
|
|
57
|
+
re-index while still bumping the stored mtime so the next search
|
|
58
|
+
short-circuits without re-hashing.
|
|
59
|
+
"""
|
|
60
|
+
current_paths = set(collect_source_files(repo_root, languages))
|
|
61
|
+
stored = load_file_index(store)
|
|
62
|
+
|
|
63
|
+
updated: dict[str, dict] = {}
|
|
64
|
+
changed: list[str] = []
|
|
65
|
+
|
|
66
|
+
for rel in sorted(current_paths):
|
|
67
|
+
full_path = repo_root / rel
|
|
68
|
+
try:
|
|
69
|
+
mtime = full_path.stat().st_mtime_ns
|
|
70
|
+
except OSError as e:
|
|
71
|
+
log_file_error(_log, rel, e, phase="mtime_scan.stat")
|
|
72
|
+
continue
|
|
73
|
+
|
|
74
|
+
prev = stored.get(rel)
|
|
75
|
+
if prev is not None and prev.get("mtime_ns") == mtime:
|
|
76
|
+
updated[rel] = {"mtime_ns": mtime, "hash": prev.get("hash", "")}
|
|
77
|
+
continue
|
|
78
|
+
|
|
79
|
+
try:
|
|
80
|
+
content = full_path.read_text(errors="replace")
|
|
81
|
+
except OSError as e:
|
|
82
|
+
log_file_error(_log, rel, e, phase="mtime_scan.read")
|
|
83
|
+
continue
|
|
84
|
+
if not content.strip():
|
|
85
|
+
continue
|
|
86
|
+
|
|
87
|
+
h = hash_file_content(content)
|
|
88
|
+
updated[rel] = {"mtime_ns": mtime, "hash": h}
|
|
89
|
+
|
|
90
|
+
if prev is None or prev.get("hash") != h:
|
|
91
|
+
changed.append(rel)
|
|
92
|
+
|
|
93
|
+
deleted = [rel for rel in stored if rel not in current_paths]
|
|
94
|
+
return changed, updated, deleted, stored
|
|
File without changes
|
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
"""Report sourceindex savings for the current session (Stop-hook entry point).
|
|
2
|
+
|
|
3
|
+
Wired into Claude Code as a Stop hook: after every assistant turn, walks the
|
|
4
|
+
transcript JSONL referenced by `payload.transcript_path`, attributes cost
|
|
5
|
+
from the first sourceindex invocation onward (with `/compact` and `/clear`
|
|
6
|
+
resetting the credit window), and persists a per-session savings entry
|
|
7
|
+
plus a weekly ledger to ~/.claude/sourceindex_savings.json. The companion
|
|
8
|
+
`sourceindex statusline` subcommand reads that ledger and renders the
|
|
9
|
+
one-liner
|
|
10
|
+
|
|
11
|
+
SourceIndex saved $~0.18 this session · $~4.20 this week
|
|
12
|
+
|
|
13
|
+
into the Claude Code status line. The Stop hook itself no longer prints to
|
|
14
|
+
stdout — keeping the message out of the assistant's response area.
|
|
15
|
+
|
|
16
|
+
Attribution is "from the first sourceindex invocation onward, until the
|
|
17
|
+
roadmap is wiped." Once the agent invokes the sourceindex subagent, every
|
|
18
|
+
subsequent turn is credited — the check is purely structural (presence of
|
|
19
|
+
an Agent/Task tool_use with subagent_type="sourceindex"); we don't inspect
|
|
20
|
+
the tool_result text, so error paths and synthesized summaries still count.
|
|
21
|
+
Pre-invocation turns (recon the agent did before deciding to call
|
|
22
|
+
sourceindex) don't get credit.
|
|
23
|
+
|
|
24
|
+
Two events wipe the roadmap from context and reset the credit window:
|
|
25
|
+
* `/compact` (or auto-compact at context limit) — emits a record with
|
|
26
|
+
`isCompactSummary: true`. The verbose roadmap is exactly the detail
|
|
27
|
+
compaction strips out, so we conservatively require a new invocation.
|
|
28
|
+
* `/clear` — emits a user message whose content contains
|
|
29
|
+
`<command-name>/clear</command-name>`. Wipes context entirely.
|
|
30
|
+
|
|
31
|
+
After either marker, `active` resets to False until the next sourceindex
|
|
32
|
+
call re-activates it.
|
|
33
|
+
|
|
34
|
+
Pricing table is mirrored from inspect/lib/report_print.py:55-73. Savings
|
|
35
|
+
ratios come from reports/2026-05-06_haiku-vs-gpt5nano-cache.md (N=5, 100
|
|
36
|
+
runs; sourceindex-bare with gpt-5.4-nano cache). Those numbers measure the
|
|
37
|
+
bare variant, not the subagent variant detected here, but they share the
|
|
38
|
+
same Pass 1a/1b core — best-available proxy until subagent N=5 numbers land.
|
|
39
|
+
|
|
40
|
+
`SAVINGS_RATIO[m]` is the fraction of the *baseline* cost that sourceindex
|
|
41
|
+
saves. The transcript cost we measure is the *post*-reduction cost, so the
|
|
42
|
+
amount saved is `total_cost * ratio / (1 - ratio)`, not `total_cost * ratio`
|
|
43
|
+
— the latter under-reports because it scales the wrong denominator.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
from __future__ import annotations
|
|
47
|
+
|
|
48
|
+
import contextlib
|
|
49
|
+
import json
|
|
50
|
+
import os
|
|
51
|
+
import sys
|
|
52
|
+
import tempfile
|
|
53
|
+
from datetime import date, datetime, timedelta
|
|
54
|
+
from pathlib import Path
|
|
55
|
+
from typing import Any
|
|
56
|
+
|
|
57
|
+
STATE_PATH = Path.home() / ".claude" / "sourceindex_savings.json"
|
|
58
|
+
LOG_PATH = Path.home() / ".claude" / "sourceindex_savings.log"
|
|
59
|
+
|
|
60
|
+
PRICING: dict[str, tuple[float, float, float, float]] = {
|
|
61
|
+
# Source: https://platform.claude.com/docs/en/about-claude/pricing
|
|
62
|
+
# Tuples are (input, output, cache_read, cache_creation_5m) per MTok.
|
|
63
|
+
# Order matters: the substring matcher in `_match` returns the first key
|
|
64
|
+
# contained in the model name, so put more-specific keys first.
|
|
65
|
+
# cache_creation defaults to the 5-minute write rate (1.25x input);
|
|
66
|
+
# the 1-hour rate (2x input) costs 60% more but the transcript's
|
|
67
|
+
# `cache_creation_input_tokens` field doesn't distinguish the two.
|
|
68
|
+
# Opus 4.5+ is priced at 1/3 of Opus 4.0/4.1.
|
|
69
|
+
"claude-opus-4-7": ( 5.00, 25.00, 0.50, 6.25),
|
|
70
|
+
"claude-opus-4-6": ( 5.00, 25.00, 0.50, 6.25),
|
|
71
|
+
"claude-opus-4-5": ( 5.00, 25.00, 0.50, 6.25),
|
|
72
|
+
"claude-opus-4": (15.00, 75.00, 1.50, 18.75),
|
|
73
|
+
"claude-sonnet-4": ( 3.00, 15.00, 0.30, 3.75),
|
|
74
|
+
"claude-haiku-4-5": ( 1.00, 5.00, 0.10, 1.25),
|
|
75
|
+
"claude-haiku-4": ( 0.80, 4.00, 0.08, 1.00),
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
SAVINGS_RATIO: dict[str, float] = {
|
|
79
|
+
"claude-opus-4": 0.25,
|
|
80
|
+
"claude-sonnet-4": 0.25,
|
|
81
|
+
"claude-haiku-4": 0.05,
|
|
82
|
+
}
|
|
83
|
+
DEFAULT_RATIO = 0.20
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _match(table: dict[str, Any], model: str | None) -> Any:
|
|
87
|
+
m = (model or "").lower()
|
|
88
|
+
for key, val in table.items():
|
|
89
|
+
if key in m:
|
|
90
|
+
return val
|
|
91
|
+
return None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _line_cost(model: str | None, usage: Any) -> float:
|
|
95
|
+
rates = _match(PRICING, model)
|
|
96
|
+
if not rates or not isinstance(usage, dict):
|
|
97
|
+
return 0.0
|
|
98
|
+
inp = usage.get("input_tokens", 0) or 0
|
|
99
|
+
out = usage.get("output_tokens", 0) or 0
|
|
100
|
+
cr = usage.get("cache_read_input_tokens", 0) or 0
|
|
101
|
+
cc = usage.get("cache_creation_input_tokens", 0) or 0
|
|
102
|
+
in_r, out_r, cr_r, cc_r = rates
|
|
103
|
+
return (inp * in_r + out * out_r + cr * cr_r + cc * cc_r) / 1_000_000
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _sourceindex_call_id(blocks: Any) -> str | None:
|
|
107
|
+
"""If `blocks` contains a sourceindex Agent/Task tool_use, return its id."""
|
|
108
|
+
if not isinstance(blocks, list):
|
|
109
|
+
return None
|
|
110
|
+
for b in blocks:
|
|
111
|
+
if not isinstance(b, dict):
|
|
112
|
+
continue
|
|
113
|
+
if b.get("type") != "tool_use":
|
|
114
|
+
continue
|
|
115
|
+
if b.get("name") not in ("Agent", "Task"):
|
|
116
|
+
continue
|
|
117
|
+
if (b.get("input") or {}).get("subagent_type") == "sourceindex":
|
|
118
|
+
cid = b.get("id")
|
|
119
|
+
return cid if isinstance(cid, str) else None
|
|
120
|
+
return None
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
CLEAR_MARKER = "<command-name>/clear</command-name>"
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _is_reset_marker(rec: dict) -> bool:
|
|
127
|
+
"""True if `rec` is a /compact summary or a /clear slash-command."""
|
|
128
|
+
if rec.get("isCompactSummary"):
|
|
129
|
+
return True
|
|
130
|
+
msg = rec.get("message") or {}
|
|
131
|
+
if msg.get("role") != "user":
|
|
132
|
+
return False
|
|
133
|
+
content = msg.get("content")
|
|
134
|
+
if isinstance(content, str):
|
|
135
|
+
return CLEAR_MARKER in content
|
|
136
|
+
if isinstance(content, list):
|
|
137
|
+
return any(
|
|
138
|
+
isinstance(b, dict)
|
|
139
|
+
and isinstance(b.get("text"), str)
|
|
140
|
+
and CLEAR_MARKER in b["text"]
|
|
141
|
+
for b in content
|
|
142
|
+
)
|
|
143
|
+
return False
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def walk_transcript(path: str | os.PathLike) -> tuple[bool, float, str | None, bool]:
|
|
147
|
+
"""Return (used, total_cost, main_model, last_turn_invoked) from a transcript JSONL.
|
|
148
|
+
|
|
149
|
+
Cost runs from the first sourceindex invocation onward. The check is
|
|
150
|
+
purely structural — any assistant turn whose content contains an
|
|
151
|
+
Agent/Task tool_use with subagent_type="sourceindex" activates the
|
|
152
|
+
credit window. We do not inspect the tool_result, so error paths and
|
|
153
|
+
synthesized summaries still count. `/compact` and `/clear` reset the
|
|
154
|
+
active window — after either, a new sourceindex call is required.
|
|
155
|
+
|
|
156
|
+
`last_turn_invoked` is True iff the *chronologically last* assistant
|
|
157
|
+
turn carried a sourceindex tool_use. The Stop hook uses this to
|
|
158
|
+
decide whether to refresh `updated_at` — otherwise the statusline's
|
|
159
|
+
60s freshness window would re-arm on every turn, keeping the row lit
|
|
160
|
+
until the user goes idle. We want it to fade 60s after the actual
|
|
161
|
+
sourceindex invocation, not after the most recent assistant turn.
|
|
162
|
+
|
|
163
|
+
Skips sidechain (subagent) lines — we account parent cost only.
|
|
164
|
+
Unknown models (no PRICING substring match) contribute zero cost.
|
|
165
|
+
"""
|
|
166
|
+
total = 0.0
|
|
167
|
+
any_used = False
|
|
168
|
+
main_model: str | None = None
|
|
169
|
+
active = False
|
|
170
|
+
last_turn_invoked = False
|
|
171
|
+
|
|
172
|
+
try:
|
|
173
|
+
with open(path, "r", encoding="utf-8") as f:
|
|
174
|
+
for line in f:
|
|
175
|
+
line = line.strip()
|
|
176
|
+
if not line:
|
|
177
|
+
continue
|
|
178
|
+
try:
|
|
179
|
+
rec = json.loads(line)
|
|
180
|
+
except json.JSONDecodeError:
|
|
181
|
+
continue
|
|
182
|
+
if rec.get("isSidechain"):
|
|
183
|
+
continue
|
|
184
|
+
if _is_reset_marker(rec):
|
|
185
|
+
active = False
|
|
186
|
+
continue
|
|
187
|
+
|
|
188
|
+
msg = rec.get("message") or {}
|
|
189
|
+
if msg.get("role") != "assistant":
|
|
190
|
+
continue
|
|
191
|
+
model = msg.get("model")
|
|
192
|
+
if main_model is None and model:
|
|
193
|
+
main_model = model
|
|
194
|
+
|
|
195
|
+
this_turn_invoked = bool(_sourceindex_call_id(msg.get("content")))
|
|
196
|
+
last_turn_invoked = this_turn_invoked
|
|
197
|
+
if not active and this_turn_invoked:
|
|
198
|
+
active = True
|
|
199
|
+
any_used = True
|
|
200
|
+
if active:
|
|
201
|
+
total += _line_cost(model, msg.get("usage"))
|
|
202
|
+
except (OSError, IOError):
|
|
203
|
+
pass
|
|
204
|
+
return any_used, total, main_model, last_turn_invoked
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def load_state(path: Path | None = None) -> dict[str, dict[str, Any]]:
|
|
208
|
+
p = path if path is not None else STATE_PATH
|
|
209
|
+
try:
|
|
210
|
+
with open(p, "r", encoding="utf-8") as f:
|
|
211
|
+
data = json.load(f)
|
|
212
|
+
if isinstance(data, dict):
|
|
213
|
+
return data
|
|
214
|
+
except (OSError, json.JSONDecodeError):
|
|
215
|
+
pass
|
|
216
|
+
return {}
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def save_state(state: dict[str, Any], path: Path | None = None) -> None:
|
|
220
|
+
p = path if path is not None else STATE_PATH
|
|
221
|
+
try:
|
|
222
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
223
|
+
fd, tmp = tempfile.mkstemp(prefix=".sourceindex_savings.", dir=str(p.parent))
|
|
224
|
+
try:
|
|
225
|
+
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
|
226
|
+
json.dump(state, f)
|
|
227
|
+
os.replace(tmp, p)
|
|
228
|
+
except Exception:
|
|
229
|
+
with contextlib.suppress(OSError):
|
|
230
|
+
os.unlink(tmp)
|
|
231
|
+
raise
|
|
232
|
+
except OSError:
|
|
233
|
+
pass
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def prune(state: dict[str, Any], today: date, days: int = 7) -> dict[str, dict[str, Any]]:
|
|
237
|
+
cutoff = today - timedelta(days=days)
|
|
238
|
+
keep: dict[str, dict[str, Any]] = {}
|
|
239
|
+
for sid, entry in state.items():
|
|
240
|
+
if not isinstance(entry, dict):
|
|
241
|
+
continue
|
|
242
|
+
d = entry.get("date")
|
|
243
|
+
try:
|
|
244
|
+
ed = datetime.strptime(d, "%Y-%m-%d").date()
|
|
245
|
+
except (TypeError, ValueError):
|
|
246
|
+
continue
|
|
247
|
+
if ed >= cutoff:
|
|
248
|
+
keep[sid] = entry
|
|
249
|
+
return keep
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def format_message(saved_session: float, weekly: float) -> str:
|
|
253
|
+
return f"SourceIndex saved $~{saved_session:0.2f} this session · $~{weekly:0.2f} this week."
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def run(payload: dict[str, Any], *, stdout=sys.stdout, now: datetime | None = None) -> int:
|
|
257
|
+
"""Hook entry point. Returns process exit code (always 0).
|
|
258
|
+
|
|
259
|
+
`payload` is the parsed Stop-hook JSON. `now` is injectable for tests.
|
|
260
|
+
"""
|
|
261
|
+
if payload.get("stop_hook_active") is True:
|
|
262
|
+
return 0
|
|
263
|
+
|
|
264
|
+
transcript = payload.get("transcript_path")
|
|
265
|
+
session_id = payload.get("session_id") or "unknown-session"
|
|
266
|
+
if not transcript:
|
|
267
|
+
return 0
|
|
268
|
+
|
|
269
|
+
used, total_cost, main_model, last_turn_invoked = walk_transcript(transcript)
|
|
270
|
+
if not used:
|
|
271
|
+
return 0
|
|
272
|
+
|
|
273
|
+
ratio = _match(SAVINGS_RATIO, main_model)
|
|
274
|
+
if ratio is None:
|
|
275
|
+
ratio = DEFAULT_RATIO
|
|
276
|
+
saved_session = total_cost * ratio / (1.0 - ratio)
|
|
277
|
+
|
|
278
|
+
actual_now = now or datetime.now()
|
|
279
|
+
today = actual_now.date()
|
|
280
|
+
state = load_state()
|
|
281
|
+
# Only refresh updated_at when this turn invoked sourceindex — otherwise
|
|
282
|
+
# preserve the prior timestamp so the statusline's 60s freshness window
|
|
283
|
+
# is anchored to the invocation, not to every Stop-hook fire.
|
|
284
|
+
prior = state.get(session_id) if isinstance(state.get(session_id), dict) else None
|
|
285
|
+
if last_turn_invoked or prior is None or not prior.get("updated_at"):
|
|
286
|
+
updated_at = actual_now.isoformat(timespec="seconds")
|
|
287
|
+
else:
|
|
288
|
+
updated_at = prior["updated_at"]
|
|
289
|
+
state[session_id] = {
|
|
290
|
+
"date": today.isoformat(),
|
|
291
|
+
"saved": saved_session,
|
|
292
|
+
"updated_at": updated_at,
|
|
293
|
+
}
|
|
294
|
+
state = prune(state, today)
|
|
295
|
+
save_state(state)
|
|
296
|
+
|
|
297
|
+
weekly = sum(e.get("saved", 0.0) for e in state.values() if isinstance(e, dict))
|
|
298
|
+
msg = format_message(saved_session, weekly)
|
|
299
|
+
|
|
300
|
+
try:
|
|
301
|
+
LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
302
|
+
with open(LOG_PATH, "a", encoding="utf-8") as f:
|
|
303
|
+
f.write(f"{actual_now.isoformat(timespec='seconds')}\t{session_id}\t{msg}\n")
|
|
304
|
+
except OSError:
|
|
305
|
+
pass
|
|
306
|
+
|
|
307
|
+
return 0
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def main() -> int:
|
|
311
|
+
"""CLI entry point — read JSON payload from stdin and run."""
|
|
312
|
+
try:
|
|
313
|
+
payload = json.load(sys.stdin)
|
|
314
|
+
except (json.JSONDecodeError, ValueError):
|
|
315
|
+
return 0
|
|
316
|
+
if not isinstance(payload, dict):
|
|
317
|
+
return 0
|
|
318
|
+
try:
|
|
319
|
+
return run(payload)
|
|
320
|
+
except Exception:
|
|
321
|
+
return 0
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
if __name__ == "__main__":
|
|
325
|
+
sys.exit(main())
|