cortex-vec 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
cortex_vec/raw_view.py ADDED
@@ -0,0 +1,264 @@
1
+ """Deterministic projection of a Raw session record for distill consumption.
2
+
3
+ Segments a Raw into meta / prose / tool blocks, then renders a budget-bounded
4
+ view (L0..L3) that keeps all analysis prose while eliding the verbatim tool
5
+ output that dominates byte-mass but carries little distill signal. Every
6
+ elision leaves a `(raw Lx-Ly)` anchor so the main session can Read the exact
7
+ span on demand. Pure functions only — no I/O, no config-file access.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import re
12
+ from dataclasses import dataclass
13
+
14
+ from . import raw_source as _src
15
+
16
+
17
+ @dataclass(frozen=True)
18
+ class Block:
19
+ kind: str # "meta" | "prose" | "tool"
20
+ lines: tuple[str, ...]
21
+ start: int # 1-based first source line
22
+ end: int # 1-based last source line (inclusive)
23
+ tool_name: str | None = None
24
+ header: str | None = None
25
+ out_lines: tuple[str, ...] | None = None
26
+ out_start: int | None = None # 1-based first output-body line
27
+ out_end: int | None = None # 1-based last output-body line
28
+
29
+
30
+ def _span_lines(text: str, s) -> list:
31
+ seg = text[s.char_start:s.char_end]
32
+ if seg.endswith("\n"):
33
+ seg = seg[:-1]
34
+ return seg.split("\n")
35
+
36
+
37
+ def parse_blocks(text: str) -> list:
38
+ """Legacy Block view rendered FROM the shared source model.
39
+
40
+ raw_source.parse is the single parsing authority; this adapter folds its
41
+ span partition back into the meta/prose/tool Block shape the L1-L3
42
+ renderer consumes. Wrapper fences and inter-tool blanks have no Block
43
+ representation (same as the historical parser) — coverage lives in the
44
+ span layer, not here.
45
+ """
46
+ spans = _src.parse(text)
47
+ blocks: list = []
48
+ prose: list = []
49
+ prose_start = None # 1-based line
50
+
51
+ def flush():
52
+ if prose:
53
+ blocks.append(Block("prose", tuple(prose), prose_start,
54
+ prose_start + len(prose) - 1))
55
+ prose.clear()
56
+
57
+ i = 0
58
+ n = len(spans)
59
+ # meta region: spans before the first turn_header / tool_header
60
+ meta_end = 0
61
+ while meta_end < n and spans[meta_end].kind not in ("turn_header",
62
+ "tool_header"):
63
+ meta_end += 1
64
+ if meta_end:
65
+ last = spans[meta_end - 1]
66
+ lines = []
67
+ for s in spans[:meta_end]:
68
+ lines.extend(_span_lines(text, s))
69
+ blocks.append(Block("meta", tuple(lines), 1, last.line_end))
70
+ i = meta_end
71
+
72
+ while i < n:
73
+ s = spans[i]
74
+ if s.kind == "tool_header":
75
+ flush()
76
+ out_lines = out_start = out_end = None
77
+ j = i + 1
78
+ # Skip any blank spans after the header (they have no Block
79
+ # representation, matching the historical parser), then look for
80
+ # the ```output wrapper at the advanced position.
81
+ while j < n and spans[j].kind == "blank":
82
+ j += 1
83
+ if j < n and spans[j].kind == "output_open":
84
+ j += 1
85
+ if j < n and spans[j].kind == "output_body":
86
+ out_lines = tuple(_span_lines(text, spans[j]))
87
+ out_start, out_end = spans[j].line_start, spans[j].line_end
88
+ j += 1
89
+ if j < n and spans[j].kind == "output_close":
90
+ j += 1
91
+ hdr = _span_lines(text, s)[0]
92
+ blocks.append(Block("tool", (hdr,), s.line_start, s.line_end,
93
+ tool_name=s.tool_name or "?", header=hdr,
94
+ out_lines=out_lines, out_start=out_start,
95
+ out_end=out_end))
96
+ i = j
97
+ elif s.kind == "opaque" and i and spans[i - 1].kind == "output_open":
98
+ # unterminated output: legacy shape treated the tail as body
99
+ prev = blocks[-1]
100
+ blocks[-1] = Block("tool", prev.lines, prev.start, prev.end,
101
+ tool_name=prev.tool_name, header=prev.header,
102
+ out_lines=tuple(_span_lines(text, s)),
103
+ out_start=s.line_start, out_end=s.line_end)
104
+ i += 1
105
+ else:
106
+ if not prose:
107
+ prose_start = s.line_start
108
+ prose.extend(_span_lines(text, s))
109
+ i += 1
110
+ flush()
111
+ return blocks
112
+
113
+
114
+ VIEW_DEFAULTS = {
115
+ "budget": 150000,
116
+ "keep_output_tools": ("Agent", "Task"),
117
+ "l3_prose_head_lines": 8,
118
+ "l3_collapse_tool_run": 4,
119
+ }
120
+
121
+
122
+ def _elide_anchor(start: int, end: int) -> str:
123
+ return f"[... elided {end - start + 1} lines (raw L{start}-L{end}) ...]"
124
+
125
+
126
+ def _tool_lines(b: Block, level: str, keep: set) -> list[str]:
127
+ out = [b.header] if level == "L1" else [f"> [tool] {b.tool_name}"]
128
+ if b.out_lines is not None:
129
+ if level == "L1" and b.tool_name in keep:
130
+ out.append("```output")
131
+ out.extend(b.out_lines)
132
+ out.append("```")
133
+ else:
134
+ out.append(_elide_anchor(b.out_start, b.out_end))
135
+ return out
136
+
137
+
138
+ def _is_special_prose(line: str) -> bool:
139
+ # ★ Insight callouts, table rows, and box-drawing separators are
140
+ # insight-dense — keep them verbatim even in the L3 skeleton.
141
+ return "★" in line or line.lstrip().startswith("|") or "─" in line
142
+
143
+
144
+ def _l3_prose_lines(b: Block, head: int) -> list[str]:
145
+ # parse_blocks does NOT split prose on blank lines, so a callout usually
146
+ # shares its Block with surrounding filler. Handle per-line: keep the first
147
+ # `head` lines, then keep special lines verbatim while eliding contiguous
148
+ # runs of non-special lines to anchors. (A block-level "keep whole if any
149
+ # special line" check would defeat head-limiting on exactly these blocks.)
150
+ n = len(b.lines)
151
+ if n <= head:
152
+ return list(b.lines)
153
+ out = list(b.lines[:head])
154
+ i = head
155
+ while i < n:
156
+ if _is_special_prose(b.lines[i]):
157
+ out.append(b.lines[i])
158
+ i += 1
159
+ continue
160
+ j = i
161
+ while j < n and not _is_special_prose(b.lines[j]):
162
+ j += 1
163
+ out.append(_elide_anchor(b.start + i, b.start + j - 1))
164
+ i = j
165
+ return out
166
+
167
+
168
+ def render(blocks: list[Block], level: str, cfg: dict) -> str:
169
+ keep = set(cfg["keep_output_tools"])
170
+ run_min = cfg["l3_collapse_tool_run"]
171
+ head = cfg["l3_prose_head_lines"]
172
+ out: list[str] = []
173
+ i = 0
174
+ while i < len(blocks):
175
+ b = blocks[i]
176
+ if b.kind == "meta":
177
+ out.extend(b.lines)
178
+ elif b.kind == "prose":
179
+ out.extend(_l3_prose_lines(b, head) if level == "L3" else b.lines)
180
+ else: # tool
181
+ if level == "L3" and run_min > 0:
182
+ end = i
183
+ while end + 1 < len(blocks) and blocks[end + 1].kind == "tool":
184
+ end += 1
185
+ run_len = end - i + 1
186
+ if run_len >= run_min:
187
+ a = blocks[i].start
188
+ z = blocks[end].out_end or blocks[end].end
189
+ out.append(f"[... {run_len} tool calls collapsed "
190
+ f"(raw L{a}-L{z}) ...]")
191
+ i = end + 1
192
+ continue
193
+ out.extend(_tool_lines(b, level, keep))
194
+ i += 1
195
+ return "\n".join(out)
196
+
197
+
198
+ # Room reserved for the L3* truncation marker so it survives the [:budget]
199
+ # clamp at real budgets. The final [:budget] still guarantees output <= budget
200
+ # for ANY budget (the marker itself is clipped when budget is tiny).
201
+ _HARD_MARK_RESERVE = 200
202
+
203
+
204
+ def choose_level(text: str, cfg: dict) -> tuple[str, str]:
205
+ budget = cfg["budget"]
206
+ if len(text) <= budget:
207
+ return "L0", text
208
+ blocks = parse_blocks(text)
209
+ rendered = ""
210
+ for lvl in ("L1", "L2", "L3"):
211
+ rendered = render(blocks, lvl, cfg)
212
+ if len(rendered) <= budget:
213
+ return lvl, rendered
214
+ # Even L3 exceeds budget → hard-truncate with a RECOVERABLE pointer so the
215
+ # dropped tail is not silently lost: cite the highest source line still
216
+ # covered by a surviving anchor and the file's total line count, so the
217
+ # operator can Read the un-projected tail directly.
218
+ cut = max(0, budget - _HARD_MARK_RESERVE)
219
+ body = rendered[:cut]
220
+ total = text.count("\n") + 1
221
+ covered = [int(m) for m in re.findall(r"raw L\d+-L(\d+)", body)]
222
+ if covered:
223
+ mark = (f"\n[... hard-truncated to budget: this projection covers "
224
+ f"through ~raw L{max(covered)} of {total}; the tail is NOT "
225
+ f"anchored — Read the source directly beyond ~L{max(covered)} ...]")
226
+ else:
227
+ mark = (f"\n[... hard-truncated to budget; source has {total} lines and "
228
+ f"this projection is not anchored — Read the source directly ...]")
229
+ return "L3*", (body + mark)[:budget]
230
+
231
+
232
+ def stat_sizes(text: str, cfg: dict) -> dict:
233
+ blocks = parse_blocks(text)
234
+ sizes = {"raw": len(text)}
235
+ for lvl in ("L1", "L2", "L3"):
236
+ sizes[lvl] = len(render(blocks, lvl, cfg))
237
+ lvl, out = choose_level(text, cfg)
238
+ sizes["chosen"] = lvl
239
+ sizes["chosen_chars"] = len(out)
240
+ return sizes
241
+
242
+
243
+ # --- CLI dispatch (called from cli.py, before the heavy store import) --------
244
+
245
+
246
+ def dispatch_raw_view(args) -> None:
247
+ import json
248
+
249
+ from .config import get_view_config
250
+
251
+ cfg = get_view_config()
252
+ if getattr(args, "budget", None):
253
+ cfg["budget"] = args.budget
254
+ with open(args.path, encoding="utf-8") as f:
255
+ text = f.read()
256
+ if getattr(args, "stat", False):
257
+ print(json.dumps(stat_sizes(text, cfg), ensure_ascii=False))
258
+ return
259
+ lvl = getattr(args, "level", None)
260
+ if lvl:
261
+ out = text if lvl == "L0" else render(parse_blocks(text), lvl, cfg)
262
+ else:
263
+ _, out = choose_level(text, cfg)
264
+ print(out)
cortex_vec/reclaim.py ADDED
@@ -0,0 +1,141 @@
1
+ """Reclaim superseded Raw snapshots — one conversation recorded more than once.
2
+
3
+ SessionEnd names each Raw by wall-clock (``HHMMSS_session_<repo>.md``), but it
4
+ fires more than once per conversation (``/clear``, exit + ``--resume``) and the
5
+ transcript it filters is one continuously growing jsonl. Every firing re-filters
6
+ the *whole* transcript, so the earlier files are strict prefixes of the latest
7
+ one — pure redundancy that would otherwise each sit in the distill queue as its
8
+ own entry, spending the distill budget several times over on the same
9
+ conversation and landing duplicate Notes.
10
+
11
+ Two safety properties make removal sound:
12
+
13
+ * the candidate set is exactly :func:`distill_queue`'s output, i.e. Raw files
14
+ nothing references yet. An already-distilled Raw carries a position-anchored
15
+ ``<!-- distilled: ... -->`` marker and is pointed at by a Note's ``source:``,
16
+ so it is never touched.
17
+ * a candidate must be a **prefix** of the survivor, so it carries no line the
18
+ survivor lacks. If the filter output ever diverges (its LLM residue
19
+ classifier is not deterministic) the match simply fails and nothing is
20
+ removed — the failure mode is "duplicate stays", never "content lost".
21
+
22
+ Comparison starts at the first conversation turn: the frontmatter holds the
23
+ wall-clock stamp that differs between recordings of the same session.
24
+ """
25
+ from __future__ import annotations
26
+
27
+ import subprocess
28
+ from pathlib import Path
29
+
30
+ from .distill_queue import _TURN_HDRS, distill_queue
31
+
32
+
33
+ def _body(path) -> list[str]:
34
+ """Conversation body: lines from the first turn header to EOF.
35
+
36
+ Returns ``[]`` for a file with no turn header (filter failure, truncation).
37
+ An empty body is a prefix of everything, so it must never match.
38
+ """
39
+ lines: list[str] = []
40
+ started = False
41
+ try:
42
+ with Path(path).open(encoding="utf-8") as f:
43
+ for line in f:
44
+ if not started:
45
+ if line.strip() not in _TURN_HDRS:
46
+ continue
47
+ started = True
48
+ lines.append(line.rstrip("\n"))
49
+ except OSError:
50
+ return []
51
+ return lines
52
+
53
+
54
+ def _covers(survivor: list[str], candidate: list[str]) -> bool:
55
+ """True iff candidate is a non-empty prefix of survivor."""
56
+ return (
57
+ bool(candidate)
58
+ and len(candidate) <= len(survivor)
59
+ and survivor[: len(candidate)] == candidate
60
+ )
61
+
62
+
63
+ def find_superseded(root, keep=None) -> list[Path]:
64
+ """Undistilled Raw files made redundant by a longer recording (FIFO order).
65
+
66
+ With ``keep``, only that file is treated as a survivor — the session-end
67
+ path, one body read per queued file. Without it, the whole queue is
68
+ compared pairwise (backlog cleanup). For two byte-identical bodies the
69
+ later path survives, so exactly one of the pair is reclaimed.
70
+ """
71
+ queue = distill_queue(root)
72
+ if keep is not None:
73
+ survivor = _body(keep)
74
+ if not survivor:
75
+ return []
76
+ keep_resolved = Path(keep).resolve()
77
+ return [
78
+ p for p in queue
79
+ if p.resolve() != keep_resolved and _covers(survivor, _body(p))
80
+ ]
81
+
82
+ bodies = {p: _body(p) for p in queue}
83
+ superseded = []
84
+ for cand, cand_body in bodies.items():
85
+ for other, other_body in bodies.items():
86
+ if other == cand or not _covers(other_body, cand_body):
87
+ continue
88
+ if len(cand_body) < len(other_body) or str(cand) < str(other):
89
+ superseded.append(cand)
90
+ break
91
+ return superseded
92
+
93
+
94
+ def _git_rm(vault, path) -> bool:
95
+ try:
96
+ proc = subprocess.run(
97
+ ["git", "-C", str(vault), "rm", "-q", "-f", "--", str(path)],
98
+ capture_output=True, text=True,
99
+ )
100
+ except OSError:
101
+ return False
102
+ return proc.returncode == 0 and not Path(path).exists()
103
+
104
+
105
+ def apply_reclaim(paths, vault=None) -> list[Path]:
106
+ """Remove the given Raw files; return the ones actually gone.
107
+
108
+ Prefers ``git rm`` so the deletion is staged for the vault's auto-commit
109
+ (and therefore recoverable from history); falls back to unlink for files
110
+ git does not track, e.g. when ``git.auto_commit`` is off.
111
+ """
112
+ removed = []
113
+ for path in paths:
114
+ path = Path(path)
115
+ if vault is not None and _git_rm(vault, path):
116
+ removed.append(path)
117
+ continue
118
+ try:
119
+ path.unlink()
120
+ except OSError:
121
+ continue
122
+ removed.append(path)
123
+ return removed
124
+
125
+
126
+ def dispatch(args) -> None:
127
+ root = getattr(args, "root", None)
128
+ if not root:
129
+ from .config import get_vault_path
130
+
131
+ root = get_vault_path() / "Raw"
132
+ found = find_superseded(root, keep=getattr(args, "keep", None))
133
+ if getattr(args, "apply", False):
134
+ vault = getattr(args, "vault", None)
135
+ if not vault:
136
+ from .config import get_vault_path
137
+
138
+ vault = get_vault_path()
139
+ found = apply_reclaim(found, vault=vault)
140
+ for path in found:
141
+ print(path)
cortex_vec/rerank.py ADDED
@@ -0,0 +1,52 @@
1
+ """Optional LLM reranker over the top window of results (reuses OpenAI).
2
+
3
+ Off by default. Any failure (no key, bad JSON, API error) returns the input
4
+ order unchanged — reranking must never make a search fail.
5
+ """
6
+ import json
7
+
8
+ _SYSTEM = (
9
+ "你是檢索結果重排器。給定查詢與若干候選筆記(每個有 index、title、summary),"
10
+ "依與查詢的相關性為每個候選打 0-10 分。只輸出 JSON 陣列,每個元素為 "
11
+ '{"index": <int>, "score": <number>},不要任何其他文字或 markdown。'
12
+ )
13
+
14
+
15
+ def _parse_scores(content):
16
+ """Parse the model's JSON array, tolerating ```json fences."""
17
+ text = content.strip()
18
+ if text.startswith("```"):
19
+ text = text.split("```", 2)[1]
20
+ if text.startswith("json"):
21
+ text = text[4:]
22
+ return json.loads(text)
23
+
24
+
25
+ def rerank(query, results, model, window=15):
26
+ """Reorder the top `window` of `results` by LLM relevance; tail unchanged."""
27
+ if not results:
28
+ return results
29
+ head = results[:window]
30
+ tail = results[window:]
31
+ try:
32
+ import openai
33
+ client = openai.OpenAI()
34
+ candidates = "\n".join(
35
+ f'{i}: {r.get("title", "")} — {r.get("summary", "")}' for i, r in enumerate(head)
36
+ )
37
+ resp = client.chat.completions.create(
38
+ model=model,
39
+ messages=[
40
+ {"role": "system", "content": _SYSTEM},
41
+ {"role": "user", "content": f"查詢:{query}\n\n候選:\n{candidates}"},
42
+ ],
43
+ max_completion_tokens=500,
44
+ reasoning_effort="none",
45
+ )
46
+ scored = _parse_scores(resp.choices[0].message.content)
47
+ order = {int(s["index"]): float(s["score"]) for s in scored}
48
+ ranked_idx = sorted(range(len(head)), key=lambda i: order.get(i, -1.0), reverse=True)
49
+ head = [head[i] for i in ranked_idx]
50
+ except Exception:
51
+ return results
52
+ return head + tail