cortex-vec 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,431 @@
1
+ """Single-raw distill plan: coverage intervals, budget ledger, lifecycle.
2
+
3
+ Deterministic navigation state only — no semantic verdicts, no
4
+ auto-promote/auto-skip. State lives under $XDG_CACHE_HOME/cortex/
5
+ distill-plans/<source_sha256>.json with an active.json pointer enforcing
6
+ one active raw at a time. Atomic writes, user-only permissions,
7
+ fail-closed on corruption or identity drift.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ import os
13
+ from datetime import datetime, timezone
14
+ from pathlib import Path
15
+
16
+ from . import raw_source as rs
17
+ from .raw_page import CONTROL_RESERVE, PageError
18
+
19
+ STATE_SCHEMA_VERSION = 1
20
+
21
+
22
+ def plans_dir() -> Path:
23
+ root = os.environ.get("XDG_CACHE_HOME")
24
+ base = Path(root) if root else Path.home() / ".cache"
25
+ return base / "cortex" / "distill-plans"
26
+
27
+
28
+ def _now() -> str:
29
+ return datetime.now(timezone.utc).isoformat()
30
+
31
+
32
+ def _write_json(path: Path, obj) -> None:
33
+ path.parent.mkdir(parents=True, exist_ok=True)
34
+ os.chmod(path.parent, 0o700)
35
+ tmp = path.with_name(path.name + ".tmp")
36
+ with open(tmp, "w", encoding="utf-8") as f:
37
+ f.write(json.dumps(obj, ensure_ascii=False))
38
+ f.flush()
39
+ os.fsync(f.fileno())
40
+ os.chmod(tmp, 0o600)
41
+ os.replace(tmp, path)
42
+
43
+
44
+ def _state_path(source_sha256: str) -> Path:
45
+ return plans_dir() / (source_sha256 + ".json")
46
+
47
+
48
+ def _active_path() -> Path:
49
+ return plans_dir() / "active.json"
50
+
51
+
52
+ # --- interval helpers (half-open [lo, hi) over source chars) ----------------
53
+
54
+
55
+ def _merge(intervals: list, lo: int, hi: int) -> list:
56
+ out = []
57
+ for a, b in sorted(intervals + [[lo, hi]]):
58
+ if out and a <= out[-1][1]:
59
+ out[-1][1] = max(out[-1][1], b)
60
+ else:
61
+ out.append([a, b])
62
+ return out
63
+
64
+
65
+ def _covered(intervals: list, lo: int, hi: int) -> bool:
66
+ for a, b in intervals:
67
+ if a <= lo and hi <= b:
68
+ return True
69
+ return False
70
+
71
+
72
+ def _uncovered(target: list, reviewed: list) -> list:
73
+ out = []
74
+ for lo, hi in target:
75
+ pos = lo
76
+ for a, b in reviewed:
77
+ if b <= pos or a >= hi:
78
+ continue
79
+ if a > pos:
80
+ out.append([pos, min(a, hi)])
81
+ pos = max(pos, b)
82
+ if pos >= hi:
83
+ break
84
+ if pos < hi:
85
+ out.append([pos, hi])
86
+ return out
87
+
88
+
89
+ # --- lifecycle ---------------------------------------------------------------
90
+
91
+
92
+ def _save(state: dict) -> None:
93
+ state["updated"] = _now()
94
+ _write_json(_state_path(state["identity"]["source_sha256"]), state)
95
+
96
+
97
+ def _load_state_file(path: Path) -> dict:
98
+ try:
99
+ with open(path, encoding="utf-8") as f:
100
+ state = json.load(f)
101
+ except FileNotFoundError:
102
+ # A dangling reference (active.json points at a state file that
103
+ # teardown already removed) is a recoverable "no active plan", not a
104
+ # corruption -- do not fabricate a .corrupt rename for a missing file.
105
+ raise PageError("PLAN_NOT_ACTIVE", reason="state file missing",
106
+ state_file=str(path))
107
+ except (OSError, ValueError):
108
+ corrupt = path.with_name(path.name + ".corrupt")
109
+ if path.exists():
110
+ os.replace(path, corrupt)
111
+ raise PageError("PLAN_STATE_CORRUPT", state_file=str(path),
112
+ kept=str(corrupt))
113
+ if ("schema_version" in state
114
+ and state["schema_version"] != STATE_SCHEMA_VERSION):
115
+ raise PageError("SCHEMA_VERSION_MISMATCH",
116
+ found=state.get("schema_version"),
117
+ expected=STATE_SCHEMA_VERSION)
118
+ return state
119
+
120
+
121
+ def _read_active() -> dict:
122
+ p = _active_path()
123
+ if not p.exists():
124
+ raise PageError("PLAN_NOT_ACTIVE")
125
+ return _load_state_file(p)
126
+
127
+
128
+ def start_plan(raw_path: str, page_budget: int, session_budget: int) -> dict:
129
+ ident, text = rs.load(raw_path)
130
+ active = _active_path()
131
+ if active.exists():
132
+ try:
133
+ ptr = _load_state_file(active)
134
+ existing = _load_state_file(plans_dir() / ptr["state_file"])
135
+ except PageError as e:
136
+ # Only a genuinely unusable pointer is recoverable: a dangling
137
+ # reference (PLAN_NOT_ACTIVE) or an unreadable/corrupt state file
138
+ # (PLAN_STATE_CORRUPT, already renamed to .corrupt). Identity drift
139
+ # such as SCHEMA_VERSION_MISMATCH is a VALID plan and must stay
140
+ # fail-closed -- never silently discard or overwrite it.
141
+ if e.code not in ("PLAN_NOT_ACTIVE", "PLAN_STATE_CORRUPT"):
142
+ raise
143
+ if active.exists():
144
+ os.remove(active)
145
+ existing = None
146
+ if existing is not None:
147
+ if existing["identity"]["source_sha256"] != ident.source_sha256:
148
+ raise PageError("ANOTHER_PLAN_ACTIVE",
149
+ active_plan_id=existing["plan_id"],
150
+ active_raw=existing["identity"]["raw_path"])
151
+ existing["note"] = "already-active"
152
+ return existing
153
+ spans = rs.parse(text)
154
+ semantic = [[s.char_start, s.char_end] for s in spans
155
+ if s.kind in rs.SPAN_REVIEW_KINDS
156
+ or s.kind in rs.PREVIEW_REVIEW_KINDS]
157
+ ambiguous = [[s.char_start, s.char_end] for s in spans
158
+ if s.kind in ("ambiguous", "opaque")]
159
+ state = {
160
+ "schema_version": STATE_SCHEMA_VERSION,
161
+ "plan_id": ident.source_sha256[:12],
162
+ "identity": ident.to_json(),
163
+ "page_budget": page_budget,
164
+ "session_budget": session_budget,
165
+ "session_seq": 1,
166
+ "session_used_chars": 0,
167
+ "map_next_index": 0,
168
+ "span_count": len(spans),
169
+ "semantic": semantic,
170
+ "ambiguous": ambiguous,
171
+ "reviewed": [],
172
+ "evidence": [],
173
+ "status": "active",
174
+ "expected_outcome": None,
175
+ "created": _now(),
176
+ "updated": _now(),
177
+ }
178
+ _save(state)
179
+ _write_json(_active_path(),
180
+ {"plan_id": state["plan_id"],
181
+ "state_file": ident.source_sha256 + ".json"})
182
+ return state
183
+
184
+
185
+ def _load_by_plan_id(plan_id: str) -> dict:
186
+ ptr = _read_active()
187
+ if ptr["plan_id"] != plan_id:
188
+ raise PageError("PLAN_NOT_ACTIVE", active_plan_id=ptr["plan_id"])
189
+ return _load_state_file(plans_dir() / ptr["state_file"])
190
+
191
+
192
+ def load_for_page(plan_id: str, raw_path: str):
193
+ """Validate identity + status, reparse; returns (state, ident, text, spans)."""
194
+ state = _load_by_plan_id(plan_id)
195
+ if state["status"] == "sealed":
196
+ raise PageError("PLAN_SEALED", plan_id=plan_id)
197
+ ident, text = rs.load(raw_path)
198
+ rec = state["identity"]
199
+ if (ident.source_sha256 != rec["source_sha256"]
200
+ or ident.file_sha256 != rec["file_sha256"]):
201
+ raise PageError("RAW_CHANGED", plan_id=plan_id)
202
+ if (ident.schema_version != rec["schema_version"]
203
+ or ident.parser_version != rec["parser_version"]):
204
+ raise PageError("SCHEMA_VERSION_MISMATCH")
205
+ return state, ident, text, rs.parse(text)
206
+
207
+
208
+ def charge_and_save(state: dict, page_chars: int) -> None:
209
+ budget = state["session_budget"] - CONTROL_RESERVE
210
+ if state["session_used_chars"] + page_chars > budget:
211
+ raise PageError(
212
+ "BUDGET_EXHAUSTED", budget_status="continuation_required",
213
+ session_used_chars=state["session_used_chars"],
214
+ session_remaining_chars=max(
215
+ 0, budget - state["session_used_chars"]),
216
+ resume="cortex-vec distill-plan resume --plan-id "
217
+ + state["plan_id"] + " --new-session")
218
+ state["session_used_chars"] += page_chars
219
+ _save(state)
220
+
221
+
222
+ def _check_reviewed_range(state: dict, lo: int, hi: int) -> None:
223
+ # Fail closed: the no-insight gate trusts state["reviewed"], so an
224
+ # out-of-bounds or inverted range must never silently widen coverage.
225
+ cap = state["identity"]["char_count"]
226
+ if not (0 <= lo < hi <= cap):
227
+ raise PageError("CURSOR_MISMATCH",
228
+ reason="reviewed range out of bounds",
229
+ char_start=lo, char_end=hi, char_count=cap)
230
+
231
+
232
+ def mark_map_progress(state: dict, next_index: int,
233
+ auto_reviewed: list) -> None:
234
+ if not (0 <= next_index <= state["span_count"]):
235
+ raise PageError("CURSOR_MISMATCH", reason="map index out of range",
236
+ next_index=next_index, span_count=state["span_count"])
237
+ for lo, hi in auto_reviewed:
238
+ _check_reviewed_range(state, lo, hi)
239
+ state["reviewed"] = _merge(state["reviewed"], lo, hi)
240
+ state["map_next_index"] = max(state["map_next_index"], next_index)
241
+ _save(state)
242
+
243
+
244
+ def mark_reviewed(state: dict, lo: int, hi: int) -> None:
245
+ _check_reviewed_range(state, lo, hi)
246
+ state["reviewed"] = _merge(state["reviewed"], lo, hi)
247
+ _save(state)
248
+
249
+
250
+ def add_evidence(state: dict, lo: int, hi: int, label: str) -> None:
251
+ if not (0 <= lo < hi <= state["identity"]["char_count"]):
252
+ raise PageError("CURSOR_MISMATCH", reason="evidence range invalid")
253
+ if not _covered(state["reviewed"], lo, hi):
254
+ raise PageError("UNREVIEWED_AMBIGUOUS_RANGE",
255
+ reason="evidence range not yet reviewed",
256
+ char_start=lo, char_end=hi)
257
+ state["evidence"].append(
258
+ {"char_start": lo, "char_end": hi, "label": label or ""})
259
+ _save(state)
260
+
261
+
262
+ def status_payload(state: dict) -> dict:
263
+ map_done = state["map_next_index"] >= state["span_count"]
264
+ un_sem = _uncovered(state["semantic"], state["reviewed"])
265
+ un_amb = _uncovered(state["ambiguous"], state["reviewed"])
266
+ sem_done = not un_sem
267
+ budget = state["session_budget"] - CONTROL_RESERVE
268
+ remaining = max(0, budget - state["session_used_chars"])
269
+ return {
270
+ "schema_version": STATE_SCHEMA_VERSION,
271
+ "plan_id": state["plan_id"],
272
+ "raw_path": state["identity"]["raw_path"],
273
+ "status": state["status"],
274
+ "session_seq": state["session_seq"],
275
+ "map_coverage_complete": map_done,
276
+ "semantic_review_coverage_complete": sem_done,
277
+ "unreviewed_semantic_ranges": un_sem[:20],
278
+ "unreviewed_semantic_count": len(un_sem),
279
+ "unreviewed_ambiguous_ranges": un_amb[:20],
280
+ "positive_candidate_allowed": state["status"] == "active",
281
+ "no_insight_candidate_allowed": bool(
282
+ map_done and sem_done and not un_amb
283
+ and state["status"] == "active"),
284
+ "evidence_count": len(state["evidence"]),
285
+ "session_used_chars": state["session_used_chars"],
286
+ "session_remaining_chars": remaining,
287
+ "budget_status": ("continuation_required" if remaining == 0
288
+ else "ok"),
289
+ }
290
+
291
+
292
+ def resume_plan(plan_id: str, new_session: bool) -> dict:
293
+ state = _load_by_plan_id(plan_id)
294
+ if state["status"] != "sealed":
295
+ ident, _ = rs.load(state["identity"]["raw_path"])
296
+ rec = state["identity"]
297
+ if (ident.source_sha256 != rec["source_sha256"]
298
+ or ident.file_sha256 != rec["file_sha256"]):
299
+ raise PageError("RAW_CHANGED", plan_id=plan_id)
300
+ if new_session:
301
+ state["session_used_chars"] = 0
302
+ state["session_seq"] += 1
303
+ _save(state)
304
+ return state
305
+
306
+
307
+ def seal_plan(state: dict, expected_outcome: str) -> None:
308
+ allowed = ("new", "pending-merge", "skip-routine", "no-insight")
309
+ if expected_outcome not in allowed:
310
+ raise PageError("CURSOR_MISMATCH",
311
+ reason="unknown expected outcome",
312
+ allowed=list(allowed))
313
+ if expected_outcome == "no-insight" \
314
+ and not status_payload(state)["no_insight_candidate_allowed"]:
315
+ raise PageError("NO_INSIGHT_COVERAGE_INCOMPLETE",
316
+ detail=status_payload(state))
317
+ state["status"] = "sealed"
318
+ state["expected_outcome"] = expected_outcome
319
+ _save(state)
320
+
321
+
322
+ def complete_plan(plan_id: str, raw_path: str) -> dict:
323
+ state = _load_by_plan_id(plan_id)
324
+ if state["status"] != "sealed":
325
+ raise PageError("PLAN_NOT_ACTIVE", reason="seal before complete")
326
+ ident, _ = rs.load(raw_path)
327
+ if ident.source_sha256 != state["identity"]["source_sha256"]:
328
+ raise PageError("RAW_CHANGED", reason="source content changed")
329
+ from .distill_queue import classify
330
+ outcome = classify(raw_path).outcome
331
+ if outcome != state["expected_outcome"]:
332
+ raise PageError("RAW_CHANGED",
333
+ reason="marker does not match sealed expectation",
334
+ found=outcome, expected=state["expected_outcome"])
335
+ # Remove the active pointer FIRST so a crash mid-teardown leaves at most an
336
+ # orphan state file (harmless; shown by `list`, overwritten on next start)
337
+ # and a clean PLAN_NOT_ACTIVE -- never a dangling pointer that wedges.
338
+ ap = _active_path()
339
+ if ap.exists():
340
+ os.remove(ap)
341
+ sf = _state_path(state["identity"]["source_sha256"])
342
+ if sf.exists():
343
+ os.remove(sf)
344
+ return {"plan_id": plan_id, "completed": True, "outcome": outcome}
345
+
346
+
347
+ # --- CLI dispatch (called from cli.py, before the heavy store import) --------
348
+
349
+
350
+ def dispatch(args) -> None:
351
+ try:
352
+ _cli(args)
353
+ except PageError as e:
354
+ from .raw_page import emit_error
355
+ emit_error(e.code, **e.detail)
356
+
357
+
358
+ def _cli(args) -> None:
359
+ from .config import get_map_config
360
+ from .raw_page import emit
361
+
362
+ action = args.action
363
+ if action == "start":
364
+ if not args.path:
365
+ raise PageError("PLAN_NOT_ACTIVE", reason="start needs a raw path")
366
+ mc = get_map_config()
367
+ state = start_plan(args.path,
368
+ args.page_budget or mc["page_budget"],
369
+ args.session_budget or mc["session_budget"])
370
+ emit(status_payload(state))
371
+ elif action == "status":
372
+ emit(status_payload(_load_by_plan_id(args.plan_id)))
373
+ elif action == "resume":
374
+ emit(status_payload(resume_plan(args.plan_id,
375
+ new_session=args.new_session)))
376
+ elif action == "evidence-add":
377
+ if args.char_start is None or args.char_end is None:
378
+ raise PageError("CURSOR_MISMATCH",
379
+ reason="evidence-add requires --char-start "
380
+ "and --char-end")
381
+ state = _load_by_plan_id(args.plan_id)
382
+ if state["status"] != "active":
383
+ raise PageError("PLAN_SEALED", plan_id=args.plan_id)
384
+ add_evidence(state, args.char_start, args.char_end, args.label)
385
+ emit(status_payload(state))
386
+ elif action == "seal":
387
+ state = _load_by_plan_id(args.plan_id)
388
+ seal_plan(state, args.expected_outcome)
389
+ emit(status_payload(state))
390
+ elif action == "complete":
391
+ state = _load_by_plan_id(args.plan_id)
392
+ emit(complete_plan(args.plan_id, state["identity"]["raw_path"]))
393
+ elif action == "list":
394
+ rows = []
395
+ d = plans_dir()
396
+ if d.is_dir():
397
+ for p in sorted(d.glob("*.json")):
398
+ if p.name == "active.json":
399
+ continue
400
+ try:
401
+ s = _load_state_file(p)
402
+ except PageError:
403
+ rows.append({"state_file": p.name,
404
+ "status": "corrupt"})
405
+ continue
406
+ rows.append({"plan_id": s["plan_id"],
407
+ "raw_path": s["identity"]["raw_path"],
408
+ "status": s["status"]})
409
+ emit({"plans": rows})
410
+ elif action == "clear":
411
+ ap = _active_path()
412
+ try:
413
+ ptr = _read_active() # pointer only; does not load the state file
414
+ except PageError:
415
+ ptr = None
416
+ if ptr is None:
417
+ raise PageError("PLAN_NOT_ACTIVE",
418
+ reason="no active plan to clear")
419
+ if ptr.get("plan_id") != args.plan_id:
420
+ raise PageError("PLAN_NOT_ACTIVE",
421
+ reason="plan-id does not match the active plan",
422
+ active_plan_id=ptr.get("plan_id"))
423
+ # Remove the pointer first, then the state file if it still exists.
424
+ if ap.exists():
425
+ os.remove(ap)
426
+ sf = plans_dir() / ptr["state_file"]
427
+ if sf.exists():
428
+ os.remove(sf)
429
+ emit({"plan_id": args.plan_id, "cleared": True})
430
+ else:
431
+ raise PageError("PLAN_NOT_ACTIVE", reason="unknown action")
@@ -0,0 +1,171 @@
1
+ """Position-anchored detection of distilled state for Raw session records.
2
+
3
+ A real distilled marker is a bare whole-line ``<!-- distilled: ... -->`` written
4
+ by one of two writers, in one of two positions:
5
+
6
+ * the **header** — distill's Phase-1 stamp (and broadcast's Step-9 append) goes
7
+ into the top of the file, BEFORE the first conversation turn; or
8
+ * the **end of file** — session-end's ``(skip: meta-session)`` stamp is appended
9
+ as the last non-empty line.
10
+
11
+ A pipeline meta-session's *conversation body* also quotes the marker string many
12
+ times (it printfs markers onto OTHER files and every command + output lands in
13
+ the transcript), so a body-substring scan such as ``grep -rL '<!-- distilled:'``
14
+ is fooled. We anchor to position instead: a marker counts ONLY if it appears
15
+ before the first turn header, or as the last non-empty line. Markers strictly
16
+ inside the conversation are quotes and ignored.
17
+ """
18
+ from __future__ import annotations
19
+
20
+ import re
21
+ from dataclasses import dataclass
22
+ from pathlib import Path
23
+
24
+ # A real marker occupies a whole (stripped) line.
25
+ _TRAILER_RE = re.compile(r"^<!--\s*distilled:.*-->$")
26
+ # A broadcast resolution segment lives inside the same marker, after a `|`.
27
+ _RESOLVED_RE = re.compile(r"\|\s*(?:broadcast|merged|no-broadcast):")
28
+ # filter-transcript.py turn headers (see hooks/scripts/filter-transcript.py).
29
+ _TURN_HDRS = frozenset(("### User", "### Claude"))
30
+
31
+
32
+ @dataclass(frozen=True)
33
+ class RawState:
34
+ """Distilled state derived from a Raw file's trailer.
35
+
36
+ outcome is one of: ``undistilled``, ``new``, ``pending-merge``,
37
+ ``skip-routine``, ``skip-meta``, ``no-insight``, ``no-extractable-content``
38
+ (legacy), or ``unknown`` (trailer present but unparseable).
39
+ """
40
+
41
+ outcome: str
42
+ broadcast_resolved: bool = False
43
+
44
+
45
+ def _real_marker(path) -> str | None:
46
+ """Return the authoritative marker line, or None if undistilled.
47
+
48
+ Single pass: the first whole-line marker seen before the first conversation
49
+ turn wins (header convention); otherwise the last non-empty line, if it is a
50
+ marker, wins (end-of-file convention). Markers inside the conversation are
51
+ quotes and ignored.
52
+ """
53
+ header_marker = None
54
+ last_nonempty = None
55
+ in_conversation = False
56
+ try:
57
+ with Path(path).open(encoding="utf-8") as f:
58
+ for line in f:
59
+ stripped = line.strip()
60
+ if not stripped:
61
+ continue
62
+ last_nonempty = stripped
63
+ if not in_conversation:
64
+ if stripped in _TURN_HDRS:
65
+ in_conversation = True
66
+ elif header_marker is None and _TRAILER_RE.match(stripped):
67
+ header_marker = stripped
68
+ except OSError:
69
+ return None
70
+ if header_marker is not None:
71
+ return header_marker
72
+ if last_nonempty is not None and _TRAILER_RE.match(last_nonempty):
73
+ return last_nonempty
74
+ return None
75
+
76
+
77
+ def _outcome_from_trailer(line: str) -> str:
78
+ # line shape: <!-- distilled: DATE → OUTCOME [| resolution-seg]... -->
79
+ arrow = line.find("→")
80
+ if arrow == -1:
81
+ return "unknown"
82
+ rest = line[arrow + len("→") :].rstrip()
83
+ if rest.endswith("-->"):
84
+ rest = rest[:-3]
85
+ # OUTCOME is everything before the first resolution segment.
86
+ outcome = rest.split("|", 1)[0].strip()
87
+ low = outcome.lower()
88
+ if low.startswith("(skip: meta"):
89
+ return "skip-meta"
90
+ if low.startswith("(skip: routine"):
91
+ return "skip-routine"
92
+ if low.startswith("(no insight"):
93
+ return "no-insight"
94
+ if low.startswith("(no extractable content"):
95
+ return "no-extractable-content"
96
+ if low.startswith("pending-merge:"):
97
+ return "pending-merge"
98
+ if outcome: # a target path like Notes/X.md or Projects/Y/Z.md
99
+ return "new"
100
+ return "unknown"
101
+
102
+
103
+ def classify(path) -> RawState:
104
+ """Authoritative distilled state of a single Raw file."""
105
+ line = _real_marker(path)
106
+ if line is None:
107
+ return RawState("undistilled")
108
+ return RawState(_outcome_from_trailer(line), bool(_RESOLVED_RE.search(line)))
109
+
110
+
111
+ def is_distilled(state: RawState) -> bool:
112
+ return state.outcome != "undistilled"
113
+
114
+
115
+ def is_broadcast_eligible(state: RawState) -> bool:
116
+ """Eligible iff Phase-1 outcome is new/pending-merge and not yet broadcast."""
117
+ return state.outcome in ("new", "pending-merge") and not state.broadcast_resolved
118
+
119
+
120
+ def _raw_files(root) -> list[Path]:
121
+ return sorted(Path(root).rglob("*.md"))
122
+
123
+
124
+ def distill_queue(root) -> list[Path]:
125
+ """Raw files still awaiting distillation (FIFO by path)."""
126
+ return [p for p in _raw_files(root) if classify(p).outcome == "undistilled"]
127
+
128
+
129
+ def broadcast_queue(root) -> list[Path]:
130
+ """Distilled Raw files eligible for broadcast (FIFO by path)."""
131
+ return [p for p in _raw_files(root) if is_broadcast_eligible(classify(p))]
132
+
133
+
134
+ # --- CLI dispatch (called from cli.py, before the heavy store import) --------
135
+
136
+
137
+ def dispatch_raw_state(path) -> None:
138
+ state = classify(path)
139
+ suffix = " (broadcast-resolved)" if state.broadcast_resolved else ""
140
+ print(f"{state.outcome}{suffix}")
141
+
142
+
143
+ def dispatch_distill_queue(root, stat=False, as_json=False) -> None:
144
+ files = distill_queue(root)
145
+ if not stat:
146
+ for p in files:
147
+ print(p)
148
+ return
149
+ import json
150
+
151
+ from .config import get_view_config
152
+ from .raw_view import stat_sizes
153
+
154
+ cfg = get_view_config()
155
+ rows = []
156
+ for p in files:
157
+ text = Path(p).read_text(encoding="utf-8", errors="replace")
158
+ s = stat_sizes(text, cfg)
159
+ rows.append({"file": str(p), **s})
160
+ if as_json:
161
+ print(json.dumps(rows, ensure_ascii=False))
162
+ else:
163
+ print(f"{'raw':>8}{'L1':>8}{'L2':>8}{'L3':>8}{'chosen':>8} file")
164
+ for r in rows:
165
+ print(f"{r['raw']//1024:>7}K{r['L1']//1024:>7}K{r['L2']//1024:>7}K"
166
+ f"{r['L3']//1024:>7}K{r['chosen']:>8} {r['file']}")
167
+
168
+
169
+ def dispatch_broadcast_queue(root) -> None:
170
+ for p in broadcast_queue(root):
171
+ print(p)
File without changes
@@ -0,0 +1,83 @@
1
+ """Pluggable retrieval adapters for eval: grep / vector / bm25 / hybrid.
2
+
3
+ Each adapter exposes init(docs) / query(q, k) -> [(base_path, score)] / teardown().
4
+ `docs` is a list of dicts with id/title/body/summary/tags/repos/type/category.
5
+ """
6
+ from .. import bm25, config, fusion, store
7
+ from ..tokenize import tokenize
8
+
9
+
10
+ class GrepAdapter:
11
+ """Zero-dependency keyword baseline: rank by query-term frequency in title+body."""
12
+ name = "grep"
13
+
14
+ def init(self, docs):
15
+ self._docs = [(d["id"], tokenize(f"{d.get('title','')}\n{d.get('body','')}")) for d in docs]
16
+
17
+ def query(self, q, k):
18
+ terms = set(tokenize(q))
19
+ scored = []
20
+ for doc_id, toks in self._docs:
21
+ score = sum(1 for t in toks if t in terms)
22
+ if score > 0:
23
+ scored.append((doc_id, float(score)))
24
+ scored.sort(key=lambda x: x[1], reverse=True)
25
+ return scored[:k]
26
+
27
+ def teardown(self):
28
+ self._docs = []
29
+
30
+
31
+ class BM25Adapter:
32
+ """Isolated BM25 index over the corpus docs."""
33
+ name = "bm25"
34
+
35
+ def init(self, docs):
36
+ self._idx = bm25.BM25Index(config.BM25_DIR.parent / "bm25_eval")
37
+ self._idx.build_from_docs(docs)
38
+
39
+ def query(self, q, k):
40
+ return [(h["id"], h["score"]) for h in self._idx.search(q, n=k)]
41
+
42
+ def teardown(self):
43
+ self._idx = None
44
+
45
+
46
+ class VectorAdapter:
47
+ """Production vector stream (ChromaDB + OpenAI). Requires a built index.
48
+
49
+ init() is a no-op: it queries the live vector store the same way production does.
50
+ Use only against a vault whose vector index is already built.
51
+ """
52
+ name = "vector"
53
+
54
+ def init(self, docs):
55
+ pass
56
+
57
+ def query(self, q, k):
58
+ return [(it["id"], it["score"]) for it in store.vector_stream(q, k)]
59
+
60
+ def teardown(self):
61
+ pass
62
+
63
+
64
+ class HybridAdapter:
65
+ """Full production fusion.search (vector + BM25, RRF)."""
66
+ name = "hybrid"
67
+
68
+ def init(self, docs):
69
+ pass
70
+
71
+ def query(self, q, k):
72
+ return [(it["id"], it["score"]) for it in fusion.search(q, n=k)]
73
+
74
+ def teardown(self):
75
+ pass
76
+
77
+
78
+ REGISTRY = {
79
+ "grep": GrepAdapter,
80
+ "bm25": BM25Adapter,
81
+ "vector": VectorAdapter,
82
+ "hybrid": HybridAdapter,
83
+ }