cortex-vec 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cortex_vec/__init__.py +3 -0
- cortex_vec/bm25.py +129 -0
- cortex_vec/cli.py +191 -0
- cortex_vec/config.py +82 -0
- cortex_vec/distill_plan.py +431 -0
- cortex_vec/distill_queue.py +171 -0
- cortex_vec/eval/__init__.py +0 -0
- cortex_vec/eval/adapters.py +83 -0
- cortex_vec/eval/corpus.py +36 -0
- cortex_vec/eval/report.py +44 -0
- cortex_vec/eval/run.py +130 -0
- cortex_vec/eval/score.py +62 -0
- cortex_vec/fusion.py +168 -0
- cortex_vec/graph.py +107 -0
- cortex_vec/parser.py +77 -0
- cortex_vec/raw_map.py +165 -0
- cortex_vec/raw_page.py +69 -0
- cortex_vec/raw_source.py +306 -0
- cortex_vec/raw_span.py +96 -0
- cortex_vec/raw_view.py +264 -0
- cortex_vec/reclaim.py +141 -0
- cortex_vec/rerank.py +52 -0
- cortex_vec/store.py +466 -0
- cortex_vec/synonyms.py +43 -0
- cortex_vec/tokenize.py +79 -0
- cortex_vec-0.7.0.dist-info/METADATA +68 -0
- cortex_vec-0.7.0.dist-info/RECORD +32 -0
- cortex_vec-0.7.0.dist-info/WHEEL +5 -0
- cortex_vec-0.7.0.dist-info/entry_points.txt +2 -0
- cortex_vec-0.7.0.dist-info/licenses/LICENSE +201 -0
- cortex_vec-0.7.0.dist-info/licenses/NOTICE +5 -0
- cortex_vec-0.7.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
"""Single-raw distill plan: coverage intervals, budget ledger, lifecycle.
|
|
2
|
+
|
|
3
|
+
Deterministic navigation state only — no semantic verdicts, no
|
|
4
|
+
auto-promote/auto-skip. State lives under $XDG_CACHE_HOME/cortex/
|
|
5
|
+
distill-plans/<source_sha256>.json with an active.json pointer enforcing
|
|
6
|
+
one active raw at a time. Atomic writes, user-only permissions,
|
|
7
|
+
fail-closed on corruption or identity drift.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
from datetime import datetime, timezone
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from . import raw_source as rs
|
|
17
|
+
from .raw_page import CONTROL_RESERVE, PageError
|
|
18
|
+
|
|
19
|
+
STATE_SCHEMA_VERSION = 1
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def plans_dir() -> Path:
|
|
23
|
+
root = os.environ.get("XDG_CACHE_HOME")
|
|
24
|
+
base = Path(root) if root else Path.home() / ".cache"
|
|
25
|
+
return base / "cortex" / "distill-plans"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _now() -> str:
|
|
29
|
+
return datetime.now(timezone.utc).isoformat()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _write_json(path: Path, obj) -> None:
|
|
33
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
34
|
+
os.chmod(path.parent, 0o700)
|
|
35
|
+
tmp = path.with_name(path.name + ".tmp")
|
|
36
|
+
with open(tmp, "w", encoding="utf-8") as f:
|
|
37
|
+
f.write(json.dumps(obj, ensure_ascii=False))
|
|
38
|
+
f.flush()
|
|
39
|
+
os.fsync(f.fileno())
|
|
40
|
+
os.chmod(tmp, 0o600)
|
|
41
|
+
os.replace(tmp, path)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _state_path(source_sha256: str) -> Path:
|
|
45
|
+
return plans_dir() / (source_sha256 + ".json")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _active_path() -> Path:
|
|
49
|
+
return plans_dir() / "active.json"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# --- interval helpers (half-open [lo, hi) over source chars) ----------------
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _merge(intervals: list, lo: int, hi: int) -> list:
|
|
56
|
+
out = []
|
|
57
|
+
for a, b in sorted(intervals + [[lo, hi]]):
|
|
58
|
+
if out and a <= out[-1][1]:
|
|
59
|
+
out[-1][1] = max(out[-1][1], b)
|
|
60
|
+
else:
|
|
61
|
+
out.append([a, b])
|
|
62
|
+
return out
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _covered(intervals: list, lo: int, hi: int) -> bool:
|
|
66
|
+
for a, b in intervals:
|
|
67
|
+
if a <= lo and hi <= b:
|
|
68
|
+
return True
|
|
69
|
+
return False
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _uncovered(target: list, reviewed: list) -> list:
|
|
73
|
+
out = []
|
|
74
|
+
for lo, hi in target:
|
|
75
|
+
pos = lo
|
|
76
|
+
for a, b in reviewed:
|
|
77
|
+
if b <= pos or a >= hi:
|
|
78
|
+
continue
|
|
79
|
+
if a > pos:
|
|
80
|
+
out.append([pos, min(a, hi)])
|
|
81
|
+
pos = max(pos, b)
|
|
82
|
+
if pos >= hi:
|
|
83
|
+
break
|
|
84
|
+
if pos < hi:
|
|
85
|
+
out.append([pos, hi])
|
|
86
|
+
return out
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# --- lifecycle ---------------------------------------------------------------
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _save(state: dict) -> None:
|
|
93
|
+
state["updated"] = _now()
|
|
94
|
+
_write_json(_state_path(state["identity"]["source_sha256"]), state)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _load_state_file(path: Path) -> dict:
|
|
98
|
+
try:
|
|
99
|
+
with open(path, encoding="utf-8") as f:
|
|
100
|
+
state = json.load(f)
|
|
101
|
+
except FileNotFoundError:
|
|
102
|
+
# A dangling reference (active.json points at a state file that
|
|
103
|
+
# teardown already removed) is a recoverable "no active plan", not a
|
|
104
|
+
# corruption -- do not fabricate a .corrupt rename for a missing file.
|
|
105
|
+
raise PageError("PLAN_NOT_ACTIVE", reason="state file missing",
|
|
106
|
+
state_file=str(path))
|
|
107
|
+
except (OSError, ValueError):
|
|
108
|
+
corrupt = path.with_name(path.name + ".corrupt")
|
|
109
|
+
if path.exists():
|
|
110
|
+
os.replace(path, corrupt)
|
|
111
|
+
raise PageError("PLAN_STATE_CORRUPT", state_file=str(path),
|
|
112
|
+
kept=str(corrupt))
|
|
113
|
+
if ("schema_version" in state
|
|
114
|
+
and state["schema_version"] != STATE_SCHEMA_VERSION):
|
|
115
|
+
raise PageError("SCHEMA_VERSION_MISMATCH",
|
|
116
|
+
found=state.get("schema_version"),
|
|
117
|
+
expected=STATE_SCHEMA_VERSION)
|
|
118
|
+
return state
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _read_active() -> dict:
|
|
122
|
+
p = _active_path()
|
|
123
|
+
if not p.exists():
|
|
124
|
+
raise PageError("PLAN_NOT_ACTIVE")
|
|
125
|
+
return _load_state_file(p)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def start_plan(raw_path: str, page_budget: int, session_budget: int) -> dict:
|
|
129
|
+
ident, text = rs.load(raw_path)
|
|
130
|
+
active = _active_path()
|
|
131
|
+
if active.exists():
|
|
132
|
+
try:
|
|
133
|
+
ptr = _load_state_file(active)
|
|
134
|
+
existing = _load_state_file(plans_dir() / ptr["state_file"])
|
|
135
|
+
except PageError as e:
|
|
136
|
+
# Only a genuinely unusable pointer is recoverable: a dangling
|
|
137
|
+
# reference (PLAN_NOT_ACTIVE) or an unreadable/corrupt state file
|
|
138
|
+
# (PLAN_STATE_CORRUPT, already renamed to .corrupt). Identity drift
|
|
139
|
+
# such as SCHEMA_VERSION_MISMATCH is a VALID plan and must stay
|
|
140
|
+
# fail-closed -- never silently discard or overwrite it.
|
|
141
|
+
if e.code not in ("PLAN_NOT_ACTIVE", "PLAN_STATE_CORRUPT"):
|
|
142
|
+
raise
|
|
143
|
+
if active.exists():
|
|
144
|
+
os.remove(active)
|
|
145
|
+
existing = None
|
|
146
|
+
if existing is not None:
|
|
147
|
+
if existing["identity"]["source_sha256"] != ident.source_sha256:
|
|
148
|
+
raise PageError("ANOTHER_PLAN_ACTIVE",
|
|
149
|
+
active_plan_id=existing["plan_id"],
|
|
150
|
+
active_raw=existing["identity"]["raw_path"])
|
|
151
|
+
existing["note"] = "already-active"
|
|
152
|
+
return existing
|
|
153
|
+
spans = rs.parse(text)
|
|
154
|
+
semantic = [[s.char_start, s.char_end] for s in spans
|
|
155
|
+
if s.kind in rs.SPAN_REVIEW_KINDS
|
|
156
|
+
or s.kind in rs.PREVIEW_REVIEW_KINDS]
|
|
157
|
+
ambiguous = [[s.char_start, s.char_end] for s in spans
|
|
158
|
+
if s.kind in ("ambiguous", "opaque")]
|
|
159
|
+
state = {
|
|
160
|
+
"schema_version": STATE_SCHEMA_VERSION,
|
|
161
|
+
"plan_id": ident.source_sha256[:12],
|
|
162
|
+
"identity": ident.to_json(),
|
|
163
|
+
"page_budget": page_budget,
|
|
164
|
+
"session_budget": session_budget,
|
|
165
|
+
"session_seq": 1,
|
|
166
|
+
"session_used_chars": 0,
|
|
167
|
+
"map_next_index": 0,
|
|
168
|
+
"span_count": len(spans),
|
|
169
|
+
"semantic": semantic,
|
|
170
|
+
"ambiguous": ambiguous,
|
|
171
|
+
"reviewed": [],
|
|
172
|
+
"evidence": [],
|
|
173
|
+
"status": "active",
|
|
174
|
+
"expected_outcome": None,
|
|
175
|
+
"created": _now(),
|
|
176
|
+
"updated": _now(),
|
|
177
|
+
}
|
|
178
|
+
_save(state)
|
|
179
|
+
_write_json(_active_path(),
|
|
180
|
+
{"plan_id": state["plan_id"],
|
|
181
|
+
"state_file": ident.source_sha256 + ".json"})
|
|
182
|
+
return state
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _load_by_plan_id(plan_id: str) -> dict:
|
|
186
|
+
ptr = _read_active()
|
|
187
|
+
if ptr["plan_id"] != plan_id:
|
|
188
|
+
raise PageError("PLAN_NOT_ACTIVE", active_plan_id=ptr["plan_id"])
|
|
189
|
+
return _load_state_file(plans_dir() / ptr["state_file"])
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def load_for_page(plan_id: str, raw_path: str):
|
|
193
|
+
"""Validate identity + status, reparse; returns (state, ident, text, spans)."""
|
|
194
|
+
state = _load_by_plan_id(plan_id)
|
|
195
|
+
if state["status"] == "sealed":
|
|
196
|
+
raise PageError("PLAN_SEALED", plan_id=plan_id)
|
|
197
|
+
ident, text = rs.load(raw_path)
|
|
198
|
+
rec = state["identity"]
|
|
199
|
+
if (ident.source_sha256 != rec["source_sha256"]
|
|
200
|
+
or ident.file_sha256 != rec["file_sha256"]):
|
|
201
|
+
raise PageError("RAW_CHANGED", plan_id=plan_id)
|
|
202
|
+
if (ident.schema_version != rec["schema_version"]
|
|
203
|
+
or ident.parser_version != rec["parser_version"]):
|
|
204
|
+
raise PageError("SCHEMA_VERSION_MISMATCH")
|
|
205
|
+
return state, ident, text, rs.parse(text)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def charge_and_save(state: dict, page_chars: int) -> None:
|
|
209
|
+
budget = state["session_budget"] - CONTROL_RESERVE
|
|
210
|
+
if state["session_used_chars"] + page_chars > budget:
|
|
211
|
+
raise PageError(
|
|
212
|
+
"BUDGET_EXHAUSTED", budget_status="continuation_required",
|
|
213
|
+
session_used_chars=state["session_used_chars"],
|
|
214
|
+
session_remaining_chars=max(
|
|
215
|
+
0, budget - state["session_used_chars"]),
|
|
216
|
+
resume="cortex-vec distill-plan resume --plan-id "
|
|
217
|
+
+ state["plan_id"] + " --new-session")
|
|
218
|
+
state["session_used_chars"] += page_chars
|
|
219
|
+
_save(state)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _check_reviewed_range(state: dict, lo: int, hi: int) -> None:
|
|
223
|
+
# Fail closed: the no-insight gate trusts state["reviewed"], so an
|
|
224
|
+
# out-of-bounds or inverted range must never silently widen coverage.
|
|
225
|
+
cap = state["identity"]["char_count"]
|
|
226
|
+
if not (0 <= lo < hi <= cap):
|
|
227
|
+
raise PageError("CURSOR_MISMATCH",
|
|
228
|
+
reason="reviewed range out of bounds",
|
|
229
|
+
char_start=lo, char_end=hi, char_count=cap)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def mark_map_progress(state: dict, next_index: int,
|
|
233
|
+
auto_reviewed: list) -> None:
|
|
234
|
+
if not (0 <= next_index <= state["span_count"]):
|
|
235
|
+
raise PageError("CURSOR_MISMATCH", reason="map index out of range",
|
|
236
|
+
next_index=next_index, span_count=state["span_count"])
|
|
237
|
+
for lo, hi in auto_reviewed:
|
|
238
|
+
_check_reviewed_range(state, lo, hi)
|
|
239
|
+
state["reviewed"] = _merge(state["reviewed"], lo, hi)
|
|
240
|
+
state["map_next_index"] = max(state["map_next_index"], next_index)
|
|
241
|
+
_save(state)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def mark_reviewed(state: dict, lo: int, hi: int) -> None:
|
|
245
|
+
_check_reviewed_range(state, lo, hi)
|
|
246
|
+
state["reviewed"] = _merge(state["reviewed"], lo, hi)
|
|
247
|
+
_save(state)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def add_evidence(state: dict, lo: int, hi: int, label: str) -> None:
|
|
251
|
+
if not (0 <= lo < hi <= state["identity"]["char_count"]):
|
|
252
|
+
raise PageError("CURSOR_MISMATCH", reason="evidence range invalid")
|
|
253
|
+
if not _covered(state["reviewed"], lo, hi):
|
|
254
|
+
raise PageError("UNREVIEWED_AMBIGUOUS_RANGE",
|
|
255
|
+
reason="evidence range not yet reviewed",
|
|
256
|
+
char_start=lo, char_end=hi)
|
|
257
|
+
state["evidence"].append(
|
|
258
|
+
{"char_start": lo, "char_end": hi, "label": label or ""})
|
|
259
|
+
_save(state)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def status_payload(state: dict) -> dict:
|
|
263
|
+
map_done = state["map_next_index"] >= state["span_count"]
|
|
264
|
+
un_sem = _uncovered(state["semantic"], state["reviewed"])
|
|
265
|
+
un_amb = _uncovered(state["ambiguous"], state["reviewed"])
|
|
266
|
+
sem_done = not un_sem
|
|
267
|
+
budget = state["session_budget"] - CONTROL_RESERVE
|
|
268
|
+
remaining = max(0, budget - state["session_used_chars"])
|
|
269
|
+
return {
|
|
270
|
+
"schema_version": STATE_SCHEMA_VERSION,
|
|
271
|
+
"plan_id": state["plan_id"],
|
|
272
|
+
"raw_path": state["identity"]["raw_path"],
|
|
273
|
+
"status": state["status"],
|
|
274
|
+
"session_seq": state["session_seq"],
|
|
275
|
+
"map_coverage_complete": map_done,
|
|
276
|
+
"semantic_review_coverage_complete": sem_done,
|
|
277
|
+
"unreviewed_semantic_ranges": un_sem[:20],
|
|
278
|
+
"unreviewed_semantic_count": len(un_sem),
|
|
279
|
+
"unreviewed_ambiguous_ranges": un_amb[:20],
|
|
280
|
+
"positive_candidate_allowed": state["status"] == "active",
|
|
281
|
+
"no_insight_candidate_allowed": bool(
|
|
282
|
+
map_done and sem_done and not un_amb
|
|
283
|
+
and state["status"] == "active"),
|
|
284
|
+
"evidence_count": len(state["evidence"]),
|
|
285
|
+
"session_used_chars": state["session_used_chars"],
|
|
286
|
+
"session_remaining_chars": remaining,
|
|
287
|
+
"budget_status": ("continuation_required" if remaining == 0
|
|
288
|
+
else "ok"),
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def resume_plan(plan_id: str, new_session: bool) -> dict:
|
|
293
|
+
state = _load_by_plan_id(plan_id)
|
|
294
|
+
if state["status"] != "sealed":
|
|
295
|
+
ident, _ = rs.load(state["identity"]["raw_path"])
|
|
296
|
+
rec = state["identity"]
|
|
297
|
+
if (ident.source_sha256 != rec["source_sha256"]
|
|
298
|
+
or ident.file_sha256 != rec["file_sha256"]):
|
|
299
|
+
raise PageError("RAW_CHANGED", plan_id=plan_id)
|
|
300
|
+
if new_session:
|
|
301
|
+
state["session_used_chars"] = 0
|
|
302
|
+
state["session_seq"] += 1
|
|
303
|
+
_save(state)
|
|
304
|
+
return state
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def seal_plan(state: dict, expected_outcome: str) -> None:
|
|
308
|
+
allowed = ("new", "pending-merge", "skip-routine", "no-insight")
|
|
309
|
+
if expected_outcome not in allowed:
|
|
310
|
+
raise PageError("CURSOR_MISMATCH",
|
|
311
|
+
reason="unknown expected outcome",
|
|
312
|
+
allowed=list(allowed))
|
|
313
|
+
if expected_outcome == "no-insight" \
|
|
314
|
+
and not status_payload(state)["no_insight_candidate_allowed"]:
|
|
315
|
+
raise PageError("NO_INSIGHT_COVERAGE_INCOMPLETE",
|
|
316
|
+
detail=status_payload(state))
|
|
317
|
+
state["status"] = "sealed"
|
|
318
|
+
state["expected_outcome"] = expected_outcome
|
|
319
|
+
_save(state)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def complete_plan(plan_id: str, raw_path: str) -> dict:
|
|
323
|
+
state = _load_by_plan_id(plan_id)
|
|
324
|
+
if state["status"] != "sealed":
|
|
325
|
+
raise PageError("PLAN_NOT_ACTIVE", reason="seal before complete")
|
|
326
|
+
ident, _ = rs.load(raw_path)
|
|
327
|
+
if ident.source_sha256 != state["identity"]["source_sha256"]:
|
|
328
|
+
raise PageError("RAW_CHANGED", reason="source content changed")
|
|
329
|
+
from .distill_queue import classify
|
|
330
|
+
outcome = classify(raw_path).outcome
|
|
331
|
+
if outcome != state["expected_outcome"]:
|
|
332
|
+
raise PageError("RAW_CHANGED",
|
|
333
|
+
reason="marker does not match sealed expectation",
|
|
334
|
+
found=outcome, expected=state["expected_outcome"])
|
|
335
|
+
# Remove the active pointer FIRST so a crash mid-teardown leaves at most an
|
|
336
|
+
# orphan state file (harmless; shown by `list`, overwritten on next start)
|
|
337
|
+
# and a clean PLAN_NOT_ACTIVE -- never a dangling pointer that wedges.
|
|
338
|
+
ap = _active_path()
|
|
339
|
+
if ap.exists():
|
|
340
|
+
os.remove(ap)
|
|
341
|
+
sf = _state_path(state["identity"]["source_sha256"])
|
|
342
|
+
if sf.exists():
|
|
343
|
+
os.remove(sf)
|
|
344
|
+
return {"plan_id": plan_id, "completed": True, "outcome": outcome}
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
# --- CLI dispatch (called from cli.py, before the heavy store import) --------
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def dispatch(args) -> None:
|
|
351
|
+
try:
|
|
352
|
+
_cli(args)
|
|
353
|
+
except PageError as e:
|
|
354
|
+
from .raw_page import emit_error
|
|
355
|
+
emit_error(e.code, **e.detail)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _cli(args) -> None:
|
|
359
|
+
from .config import get_map_config
|
|
360
|
+
from .raw_page import emit
|
|
361
|
+
|
|
362
|
+
action = args.action
|
|
363
|
+
if action == "start":
|
|
364
|
+
if not args.path:
|
|
365
|
+
raise PageError("PLAN_NOT_ACTIVE", reason="start needs a raw path")
|
|
366
|
+
mc = get_map_config()
|
|
367
|
+
state = start_plan(args.path,
|
|
368
|
+
args.page_budget or mc["page_budget"],
|
|
369
|
+
args.session_budget or mc["session_budget"])
|
|
370
|
+
emit(status_payload(state))
|
|
371
|
+
elif action == "status":
|
|
372
|
+
emit(status_payload(_load_by_plan_id(args.plan_id)))
|
|
373
|
+
elif action == "resume":
|
|
374
|
+
emit(status_payload(resume_plan(args.plan_id,
|
|
375
|
+
new_session=args.new_session)))
|
|
376
|
+
elif action == "evidence-add":
|
|
377
|
+
if args.char_start is None or args.char_end is None:
|
|
378
|
+
raise PageError("CURSOR_MISMATCH",
|
|
379
|
+
reason="evidence-add requires --char-start "
|
|
380
|
+
"and --char-end")
|
|
381
|
+
state = _load_by_plan_id(args.plan_id)
|
|
382
|
+
if state["status"] != "active":
|
|
383
|
+
raise PageError("PLAN_SEALED", plan_id=args.plan_id)
|
|
384
|
+
add_evidence(state, args.char_start, args.char_end, args.label)
|
|
385
|
+
emit(status_payload(state))
|
|
386
|
+
elif action == "seal":
|
|
387
|
+
state = _load_by_plan_id(args.plan_id)
|
|
388
|
+
seal_plan(state, args.expected_outcome)
|
|
389
|
+
emit(status_payload(state))
|
|
390
|
+
elif action == "complete":
|
|
391
|
+
state = _load_by_plan_id(args.plan_id)
|
|
392
|
+
emit(complete_plan(args.plan_id, state["identity"]["raw_path"]))
|
|
393
|
+
elif action == "list":
|
|
394
|
+
rows = []
|
|
395
|
+
d = plans_dir()
|
|
396
|
+
if d.is_dir():
|
|
397
|
+
for p in sorted(d.glob("*.json")):
|
|
398
|
+
if p.name == "active.json":
|
|
399
|
+
continue
|
|
400
|
+
try:
|
|
401
|
+
s = _load_state_file(p)
|
|
402
|
+
except PageError:
|
|
403
|
+
rows.append({"state_file": p.name,
|
|
404
|
+
"status": "corrupt"})
|
|
405
|
+
continue
|
|
406
|
+
rows.append({"plan_id": s["plan_id"],
|
|
407
|
+
"raw_path": s["identity"]["raw_path"],
|
|
408
|
+
"status": s["status"]})
|
|
409
|
+
emit({"plans": rows})
|
|
410
|
+
elif action == "clear":
|
|
411
|
+
ap = _active_path()
|
|
412
|
+
try:
|
|
413
|
+
ptr = _read_active() # pointer only; does not load the state file
|
|
414
|
+
except PageError:
|
|
415
|
+
ptr = None
|
|
416
|
+
if ptr is None:
|
|
417
|
+
raise PageError("PLAN_NOT_ACTIVE",
|
|
418
|
+
reason="no active plan to clear")
|
|
419
|
+
if ptr.get("plan_id") != args.plan_id:
|
|
420
|
+
raise PageError("PLAN_NOT_ACTIVE",
|
|
421
|
+
reason="plan-id does not match the active plan",
|
|
422
|
+
active_plan_id=ptr.get("plan_id"))
|
|
423
|
+
# Remove the pointer first, then the state file if it still exists.
|
|
424
|
+
if ap.exists():
|
|
425
|
+
os.remove(ap)
|
|
426
|
+
sf = plans_dir() / ptr["state_file"]
|
|
427
|
+
if sf.exists():
|
|
428
|
+
os.remove(sf)
|
|
429
|
+
emit({"plan_id": args.plan_id, "cleared": True})
|
|
430
|
+
else:
|
|
431
|
+
raise PageError("PLAN_NOT_ACTIVE", reason="unknown action")
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""Position-anchored detection of distilled state for Raw session records.
|
|
2
|
+
|
|
3
|
+
A real distilled marker is a bare whole-line ``<!-- distilled: ... -->`` written
|
|
4
|
+
by one of two writers, in one of two positions:
|
|
5
|
+
|
|
6
|
+
* the **header** — distill's Phase-1 stamp (and broadcast's Step-9 append) goes
|
|
7
|
+
into the top of the file, BEFORE the first conversation turn; or
|
|
8
|
+
* the **end of file** — session-end's ``(skip: meta-session)`` stamp is appended
|
|
9
|
+
as the last non-empty line.
|
|
10
|
+
|
|
11
|
+
A pipeline meta-session's *conversation body* also quotes the marker string many
|
|
12
|
+
times (it printfs markers onto OTHER files and every command + output lands in
|
|
13
|
+
the transcript), so a body-substring scan such as ``grep -rL '<!-- distilled:'``
|
|
14
|
+
is fooled. We anchor to position instead: a marker counts ONLY if it appears
|
|
15
|
+
before the first turn header, or as the last non-empty line. Markers strictly
|
|
16
|
+
inside the conversation are quotes and ignored.
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import re
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
# A real marker occupies a whole (stripped) line.
|
|
25
|
+
_TRAILER_RE = re.compile(r"^<!--\s*distilled:.*-->$")
|
|
26
|
+
# A broadcast resolution segment lives inside the same marker, after a `|`.
|
|
27
|
+
_RESOLVED_RE = re.compile(r"\|\s*(?:broadcast|merged|no-broadcast):")
|
|
28
|
+
# filter-transcript.py turn headers (see hooks/scripts/filter-transcript.py).
|
|
29
|
+
_TURN_HDRS = frozenset(("### User", "### Claude"))
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class RawState:
|
|
34
|
+
"""Distilled state derived from a Raw file's trailer.
|
|
35
|
+
|
|
36
|
+
outcome is one of: ``undistilled``, ``new``, ``pending-merge``,
|
|
37
|
+
``skip-routine``, ``skip-meta``, ``no-insight``, ``no-extractable-content``
|
|
38
|
+
(legacy), or ``unknown`` (trailer present but unparseable).
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
outcome: str
|
|
42
|
+
broadcast_resolved: bool = False
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _real_marker(path) -> str | None:
|
|
46
|
+
"""Return the authoritative marker line, or None if undistilled.
|
|
47
|
+
|
|
48
|
+
Single pass: the first whole-line marker seen before the first conversation
|
|
49
|
+
turn wins (header convention); otherwise the last non-empty line, if it is a
|
|
50
|
+
marker, wins (end-of-file convention). Markers inside the conversation are
|
|
51
|
+
quotes and ignored.
|
|
52
|
+
"""
|
|
53
|
+
header_marker = None
|
|
54
|
+
last_nonempty = None
|
|
55
|
+
in_conversation = False
|
|
56
|
+
try:
|
|
57
|
+
with Path(path).open(encoding="utf-8") as f:
|
|
58
|
+
for line in f:
|
|
59
|
+
stripped = line.strip()
|
|
60
|
+
if not stripped:
|
|
61
|
+
continue
|
|
62
|
+
last_nonempty = stripped
|
|
63
|
+
if not in_conversation:
|
|
64
|
+
if stripped in _TURN_HDRS:
|
|
65
|
+
in_conversation = True
|
|
66
|
+
elif header_marker is None and _TRAILER_RE.match(stripped):
|
|
67
|
+
header_marker = stripped
|
|
68
|
+
except OSError:
|
|
69
|
+
return None
|
|
70
|
+
if header_marker is not None:
|
|
71
|
+
return header_marker
|
|
72
|
+
if last_nonempty is not None and _TRAILER_RE.match(last_nonempty):
|
|
73
|
+
return last_nonempty
|
|
74
|
+
return None
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _outcome_from_trailer(line: str) -> str:
|
|
78
|
+
# line shape: <!-- distilled: DATE → OUTCOME [| resolution-seg]... -->
|
|
79
|
+
arrow = line.find("→")
|
|
80
|
+
if arrow == -1:
|
|
81
|
+
return "unknown"
|
|
82
|
+
rest = line[arrow + len("→") :].rstrip()
|
|
83
|
+
if rest.endswith("-->"):
|
|
84
|
+
rest = rest[:-3]
|
|
85
|
+
# OUTCOME is everything before the first resolution segment.
|
|
86
|
+
outcome = rest.split("|", 1)[0].strip()
|
|
87
|
+
low = outcome.lower()
|
|
88
|
+
if low.startswith("(skip: meta"):
|
|
89
|
+
return "skip-meta"
|
|
90
|
+
if low.startswith("(skip: routine"):
|
|
91
|
+
return "skip-routine"
|
|
92
|
+
if low.startswith("(no insight"):
|
|
93
|
+
return "no-insight"
|
|
94
|
+
if low.startswith("(no extractable content"):
|
|
95
|
+
return "no-extractable-content"
|
|
96
|
+
if low.startswith("pending-merge:"):
|
|
97
|
+
return "pending-merge"
|
|
98
|
+
if outcome: # a target path like Notes/X.md or Projects/Y/Z.md
|
|
99
|
+
return "new"
|
|
100
|
+
return "unknown"
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def classify(path) -> RawState:
|
|
104
|
+
"""Authoritative distilled state of a single Raw file."""
|
|
105
|
+
line = _real_marker(path)
|
|
106
|
+
if line is None:
|
|
107
|
+
return RawState("undistilled")
|
|
108
|
+
return RawState(_outcome_from_trailer(line), bool(_RESOLVED_RE.search(line)))
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def is_distilled(state: RawState) -> bool:
|
|
112
|
+
return state.outcome != "undistilled"
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def is_broadcast_eligible(state: RawState) -> bool:
|
|
116
|
+
"""Eligible iff Phase-1 outcome is new/pending-merge and not yet broadcast."""
|
|
117
|
+
return state.outcome in ("new", "pending-merge") and not state.broadcast_resolved
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _raw_files(root) -> list[Path]:
|
|
121
|
+
return sorted(Path(root).rglob("*.md"))
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def distill_queue(root) -> list[Path]:
|
|
125
|
+
"""Raw files still awaiting distillation (FIFO by path)."""
|
|
126
|
+
return [p for p in _raw_files(root) if classify(p).outcome == "undistilled"]
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def broadcast_queue(root) -> list[Path]:
|
|
130
|
+
"""Distilled Raw files eligible for broadcast (FIFO by path)."""
|
|
131
|
+
return [p for p in _raw_files(root) if is_broadcast_eligible(classify(p))]
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
# --- CLI dispatch (called from cli.py, before the heavy store import) --------
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def dispatch_raw_state(path) -> None:
|
|
138
|
+
state = classify(path)
|
|
139
|
+
suffix = " (broadcast-resolved)" if state.broadcast_resolved else ""
|
|
140
|
+
print(f"{state.outcome}{suffix}")
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def dispatch_distill_queue(root, stat=False, as_json=False) -> None:
|
|
144
|
+
files = distill_queue(root)
|
|
145
|
+
if not stat:
|
|
146
|
+
for p in files:
|
|
147
|
+
print(p)
|
|
148
|
+
return
|
|
149
|
+
import json
|
|
150
|
+
|
|
151
|
+
from .config import get_view_config
|
|
152
|
+
from .raw_view import stat_sizes
|
|
153
|
+
|
|
154
|
+
cfg = get_view_config()
|
|
155
|
+
rows = []
|
|
156
|
+
for p in files:
|
|
157
|
+
text = Path(p).read_text(encoding="utf-8", errors="replace")
|
|
158
|
+
s = stat_sizes(text, cfg)
|
|
159
|
+
rows.append({"file": str(p), **s})
|
|
160
|
+
if as_json:
|
|
161
|
+
print(json.dumps(rows, ensure_ascii=False))
|
|
162
|
+
else:
|
|
163
|
+
print(f"{'raw':>8}{'L1':>8}{'L2':>8}{'L3':>8}{'chosen':>8} file")
|
|
164
|
+
for r in rows:
|
|
165
|
+
print(f"{r['raw']//1024:>7}K{r['L1']//1024:>7}K{r['L2']//1024:>7}K"
|
|
166
|
+
f"{r['L3']//1024:>7}K{r['chosen']:>8} {r['file']}")
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def dispatch_broadcast_queue(root) -> None:
|
|
170
|
+
for p in broadcast_queue(root):
|
|
171
|
+
print(p)
|
|
File without changes
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Pluggable retrieval adapters for eval: grep / vector / bm25 / hybrid.
|
|
2
|
+
|
|
3
|
+
Each adapter exposes init(docs) / query(q, k) -> [(base_path, score)] / teardown().
|
|
4
|
+
`docs` is a list of dicts with id/title/body/summary/tags/repos/type/category.
|
|
5
|
+
"""
|
|
6
|
+
from .. import bm25, config, fusion, store
|
|
7
|
+
from ..tokenize import tokenize
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class GrepAdapter:
|
|
11
|
+
"""Zero-dependency keyword baseline: rank by query-term frequency in title+body."""
|
|
12
|
+
name = "grep"
|
|
13
|
+
|
|
14
|
+
def init(self, docs):
|
|
15
|
+
self._docs = [(d["id"], tokenize(f"{d.get('title','')}\n{d.get('body','')}")) for d in docs]
|
|
16
|
+
|
|
17
|
+
def query(self, q, k):
|
|
18
|
+
terms = set(tokenize(q))
|
|
19
|
+
scored = []
|
|
20
|
+
for doc_id, toks in self._docs:
|
|
21
|
+
score = sum(1 for t in toks if t in terms)
|
|
22
|
+
if score > 0:
|
|
23
|
+
scored.append((doc_id, float(score)))
|
|
24
|
+
scored.sort(key=lambda x: x[1], reverse=True)
|
|
25
|
+
return scored[:k]
|
|
26
|
+
|
|
27
|
+
def teardown(self):
|
|
28
|
+
self._docs = []
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class BM25Adapter:
|
|
32
|
+
"""Isolated BM25 index over the corpus docs."""
|
|
33
|
+
name = "bm25"
|
|
34
|
+
|
|
35
|
+
def init(self, docs):
|
|
36
|
+
self._idx = bm25.BM25Index(config.BM25_DIR.parent / "bm25_eval")
|
|
37
|
+
self._idx.build_from_docs(docs)
|
|
38
|
+
|
|
39
|
+
def query(self, q, k):
|
|
40
|
+
return [(h["id"], h["score"]) for h in self._idx.search(q, n=k)]
|
|
41
|
+
|
|
42
|
+
def teardown(self):
|
|
43
|
+
self._idx = None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class VectorAdapter:
|
|
47
|
+
"""Production vector stream (ChromaDB + OpenAI). Requires a built index.
|
|
48
|
+
|
|
49
|
+
init() is a no-op: it queries the live vector store the same way production does.
|
|
50
|
+
Use only against a vault whose vector index is already built.
|
|
51
|
+
"""
|
|
52
|
+
name = "vector"
|
|
53
|
+
|
|
54
|
+
def init(self, docs):
|
|
55
|
+
pass
|
|
56
|
+
|
|
57
|
+
def query(self, q, k):
|
|
58
|
+
return [(it["id"], it["score"]) for it in store.vector_stream(q, k)]
|
|
59
|
+
|
|
60
|
+
def teardown(self):
|
|
61
|
+
pass
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class HybridAdapter:
|
|
65
|
+
"""Full production fusion.search (vector + BM25, RRF)."""
|
|
66
|
+
name = "hybrid"
|
|
67
|
+
|
|
68
|
+
def init(self, docs):
|
|
69
|
+
pass
|
|
70
|
+
|
|
71
|
+
def query(self, q, k):
|
|
72
|
+
return [(it["id"], it["score"]) for it in fusion.search(q, n=k)]
|
|
73
|
+
|
|
74
|
+
def teardown(self):
|
|
75
|
+
pass
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
REGISTRY = {
|
|
79
|
+
"grep": GrepAdapter,
|
|
80
|
+
"bm25": BM25Adapter,
|
|
81
|
+
"vector": VectorAdapter,
|
|
82
|
+
"hybrid": HybridAdapter,
|
|
83
|
+
}
|