cortex-vec 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cortex_vec/__init__.py +3 -0
- cortex_vec/bm25.py +129 -0
- cortex_vec/cli.py +191 -0
- cortex_vec/config.py +82 -0
- cortex_vec/distill_plan.py +431 -0
- cortex_vec/distill_queue.py +171 -0
- cortex_vec/eval/__init__.py +0 -0
- cortex_vec/eval/adapters.py +83 -0
- cortex_vec/eval/corpus.py +36 -0
- cortex_vec/eval/report.py +44 -0
- cortex_vec/eval/run.py +130 -0
- cortex_vec/eval/score.py +62 -0
- cortex_vec/fusion.py +168 -0
- cortex_vec/graph.py +107 -0
- cortex_vec/parser.py +77 -0
- cortex_vec/raw_map.py +165 -0
- cortex_vec/raw_page.py +69 -0
- cortex_vec/raw_source.py +306 -0
- cortex_vec/raw_span.py +96 -0
- cortex_vec/raw_view.py +264 -0
- cortex_vec/reclaim.py +141 -0
- cortex_vec/rerank.py +52 -0
- cortex_vec/store.py +466 -0
- cortex_vec/synonyms.py +43 -0
- cortex_vec/tokenize.py +79 -0
- cortex_vec-0.7.0.dist-info/METADATA +68 -0
- cortex_vec-0.7.0.dist-info/RECORD +32 -0
- cortex_vec-0.7.0.dist-info/WHEEL +5 -0
- cortex_vec-0.7.0.dist-info/entry_points.txt +2 -0
- cortex_vec-0.7.0.dist-info/licenses/LICENSE +201 -0
- cortex_vec-0.7.0.dist-info/licenses/NOTICE +5 -0
- cortex_vec-0.7.0.dist-info/top_level.txt +1 -0
cortex_vec/raw_view.py
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
"""Deterministic projection of a Raw session record for distill consumption.
|
|
2
|
+
|
|
3
|
+
Segments a Raw into meta / prose / tool blocks, then renders a budget-bounded
|
|
4
|
+
view (L0..L3) that keeps all analysis prose while eliding the verbatim tool
|
|
5
|
+
output that dominates byte-mass but carries little distill signal. Every
|
|
6
|
+
elision leaves a `(raw Lx-Ly)` anchor so the main session can Read the exact
|
|
7
|
+
span on demand. Pure functions only — no I/O, no config-file access.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
|
|
14
|
+
from . import raw_source as _src
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class Block:
|
|
19
|
+
kind: str # "meta" | "prose" | "tool"
|
|
20
|
+
lines: tuple[str, ...]
|
|
21
|
+
start: int # 1-based first source line
|
|
22
|
+
end: int # 1-based last source line (inclusive)
|
|
23
|
+
tool_name: str | None = None
|
|
24
|
+
header: str | None = None
|
|
25
|
+
out_lines: tuple[str, ...] | None = None
|
|
26
|
+
out_start: int | None = None # 1-based first output-body line
|
|
27
|
+
out_end: int | None = None # 1-based last output-body line
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _span_lines(text: str, s) -> list:
|
|
31
|
+
seg = text[s.char_start:s.char_end]
|
|
32
|
+
if seg.endswith("\n"):
|
|
33
|
+
seg = seg[:-1]
|
|
34
|
+
return seg.split("\n")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def parse_blocks(text: str) -> list:
|
|
38
|
+
"""Legacy Block view rendered FROM the shared source model.
|
|
39
|
+
|
|
40
|
+
raw_source.parse is the single parsing authority; this adapter folds its
|
|
41
|
+
span partition back into the meta/prose/tool Block shape the L1-L3
|
|
42
|
+
renderer consumes. Wrapper fences and inter-tool blanks have no Block
|
|
43
|
+
representation (same as the historical parser) — coverage lives in the
|
|
44
|
+
span layer, not here.
|
|
45
|
+
"""
|
|
46
|
+
spans = _src.parse(text)
|
|
47
|
+
blocks: list = []
|
|
48
|
+
prose: list = []
|
|
49
|
+
prose_start = None # 1-based line
|
|
50
|
+
|
|
51
|
+
def flush():
|
|
52
|
+
if prose:
|
|
53
|
+
blocks.append(Block("prose", tuple(prose), prose_start,
|
|
54
|
+
prose_start + len(prose) - 1))
|
|
55
|
+
prose.clear()
|
|
56
|
+
|
|
57
|
+
i = 0
|
|
58
|
+
n = len(spans)
|
|
59
|
+
# meta region: spans before the first turn_header / tool_header
|
|
60
|
+
meta_end = 0
|
|
61
|
+
while meta_end < n and spans[meta_end].kind not in ("turn_header",
|
|
62
|
+
"tool_header"):
|
|
63
|
+
meta_end += 1
|
|
64
|
+
if meta_end:
|
|
65
|
+
last = spans[meta_end - 1]
|
|
66
|
+
lines = []
|
|
67
|
+
for s in spans[:meta_end]:
|
|
68
|
+
lines.extend(_span_lines(text, s))
|
|
69
|
+
blocks.append(Block("meta", tuple(lines), 1, last.line_end))
|
|
70
|
+
i = meta_end
|
|
71
|
+
|
|
72
|
+
while i < n:
|
|
73
|
+
s = spans[i]
|
|
74
|
+
if s.kind == "tool_header":
|
|
75
|
+
flush()
|
|
76
|
+
out_lines = out_start = out_end = None
|
|
77
|
+
j = i + 1
|
|
78
|
+
# Skip any blank spans after the header (they have no Block
|
|
79
|
+
# representation, matching the historical parser), then look for
|
|
80
|
+
# the ```output wrapper at the advanced position.
|
|
81
|
+
while j < n and spans[j].kind == "blank":
|
|
82
|
+
j += 1
|
|
83
|
+
if j < n and spans[j].kind == "output_open":
|
|
84
|
+
j += 1
|
|
85
|
+
if j < n and spans[j].kind == "output_body":
|
|
86
|
+
out_lines = tuple(_span_lines(text, spans[j]))
|
|
87
|
+
out_start, out_end = spans[j].line_start, spans[j].line_end
|
|
88
|
+
j += 1
|
|
89
|
+
if j < n and spans[j].kind == "output_close":
|
|
90
|
+
j += 1
|
|
91
|
+
hdr = _span_lines(text, s)[0]
|
|
92
|
+
blocks.append(Block("tool", (hdr,), s.line_start, s.line_end,
|
|
93
|
+
tool_name=s.tool_name or "?", header=hdr,
|
|
94
|
+
out_lines=out_lines, out_start=out_start,
|
|
95
|
+
out_end=out_end))
|
|
96
|
+
i = j
|
|
97
|
+
elif s.kind == "opaque" and i and spans[i - 1].kind == "output_open":
|
|
98
|
+
# unterminated output: legacy shape treated the tail as body
|
|
99
|
+
prev = blocks[-1]
|
|
100
|
+
blocks[-1] = Block("tool", prev.lines, prev.start, prev.end,
|
|
101
|
+
tool_name=prev.tool_name, header=prev.header,
|
|
102
|
+
out_lines=tuple(_span_lines(text, s)),
|
|
103
|
+
out_start=s.line_start, out_end=s.line_end)
|
|
104
|
+
i += 1
|
|
105
|
+
else:
|
|
106
|
+
if not prose:
|
|
107
|
+
prose_start = s.line_start
|
|
108
|
+
prose.extend(_span_lines(text, s))
|
|
109
|
+
i += 1
|
|
110
|
+
flush()
|
|
111
|
+
return blocks
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
VIEW_DEFAULTS = {
|
|
115
|
+
"budget": 150000,
|
|
116
|
+
"keep_output_tools": ("Agent", "Task"),
|
|
117
|
+
"l3_prose_head_lines": 8,
|
|
118
|
+
"l3_collapse_tool_run": 4,
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _elide_anchor(start: int, end: int) -> str:
|
|
123
|
+
return f"[... elided {end - start + 1} lines (raw L{start}-L{end}) ...]"
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _tool_lines(b: Block, level: str, keep: set) -> list[str]:
|
|
127
|
+
out = [b.header] if level == "L1" else [f"> [tool] {b.tool_name}"]
|
|
128
|
+
if b.out_lines is not None:
|
|
129
|
+
if level == "L1" and b.tool_name in keep:
|
|
130
|
+
out.append("```output")
|
|
131
|
+
out.extend(b.out_lines)
|
|
132
|
+
out.append("```")
|
|
133
|
+
else:
|
|
134
|
+
out.append(_elide_anchor(b.out_start, b.out_end))
|
|
135
|
+
return out
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _is_special_prose(line: str) -> bool:
|
|
139
|
+
# ★ Insight callouts, table rows, and box-drawing separators are
|
|
140
|
+
# insight-dense — keep them verbatim even in the L3 skeleton.
|
|
141
|
+
return "★" in line or line.lstrip().startswith("|") or "─" in line
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _l3_prose_lines(b: Block, head: int) -> list[str]:
|
|
145
|
+
# parse_blocks does NOT split prose on blank lines, so a callout usually
|
|
146
|
+
# shares its Block with surrounding filler. Handle per-line: keep the first
|
|
147
|
+
# `head` lines, then keep special lines verbatim while eliding contiguous
|
|
148
|
+
# runs of non-special lines to anchors. (A block-level "keep whole if any
|
|
149
|
+
# special line" check would defeat head-limiting on exactly these blocks.)
|
|
150
|
+
n = len(b.lines)
|
|
151
|
+
if n <= head:
|
|
152
|
+
return list(b.lines)
|
|
153
|
+
out = list(b.lines[:head])
|
|
154
|
+
i = head
|
|
155
|
+
while i < n:
|
|
156
|
+
if _is_special_prose(b.lines[i]):
|
|
157
|
+
out.append(b.lines[i])
|
|
158
|
+
i += 1
|
|
159
|
+
continue
|
|
160
|
+
j = i
|
|
161
|
+
while j < n and not _is_special_prose(b.lines[j]):
|
|
162
|
+
j += 1
|
|
163
|
+
out.append(_elide_anchor(b.start + i, b.start + j - 1))
|
|
164
|
+
i = j
|
|
165
|
+
return out
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def render(blocks: list[Block], level: str, cfg: dict) -> str:
|
|
169
|
+
keep = set(cfg["keep_output_tools"])
|
|
170
|
+
run_min = cfg["l3_collapse_tool_run"]
|
|
171
|
+
head = cfg["l3_prose_head_lines"]
|
|
172
|
+
out: list[str] = []
|
|
173
|
+
i = 0
|
|
174
|
+
while i < len(blocks):
|
|
175
|
+
b = blocks[i]
|
|
176
|
+
if b.kind == "meta":
|
|
177
|
+
out.extend(b.lines)
|
|
178
|
+
elif b.kind == "prose":
|
|
179
|
+
out.extend(_l3_prose_lines(b, head) if level == "L3" else b.lines)
|
|
180
|
+
else: # tool
|
|
181
|
+
if level == "L3" and run_min > 0:
|
|
182
|
+
end = i
|
|
183
|
+
while end + 1 < len(blocks) and blocks[end + 1].kind == "tool":
|
|
184
|
+
end += 1
|
|
185
|
+
run_len = end - i + 1
|
|
186
|
+
if run_len >= run_min:
|
|
187
|
+
a = blocks[i].start
|
|
188
|
+
z = blocks[end].out_end or blocks[end].end
|
|
189
|
+
out.append(f"[... {run_len} tool calls collapsed "
|
|
190
|
+
f"(raw L{a}-L{z}) ...]")
|
|
191
|
+
i = end + 1
|
|
192
|
+
continue
|
|
193
|
+
out.extend(_tool_lines(b, level, keep))
|
|
194
|
+
i += 1
|
|
195
|
+
return "\n".join(out)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
# Room reserved for the L3* truncation marker so it survives the [:budget]
|
|
199
|
+
# clamp at real budgets. The final [:budget] still guarantees output <= budget
|
|
200
|
+
# for ANY budget (the marker itself is clipped when budget is tiny).
|
|
201
|
+
_HARD_MARK_RESERVE = 200
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def choose_level(text: str, cfg: dict) -> tuple[str, str]:
|
|
205
|
+
budget = cfg["budget"]
|
|
206
|
+
if len(text) <= budget:
|
|
207
|
+
return "L0", text
|
|
208
|
+
blocks = parse_blocks(text)
|
|
209
|
+
rendered = ""
|
|
210
|
+
for lvl in ("L1", "L2", "L3"):
|
|
211
|
+
rendered = render(blocks, lvl, cfg)
|
|
212
|
+
if len(rendered) <= budget:
|
|
213
|
+
return lvl, rendered
|
|
214
|
+
# Even L3 exceeds budget → hard-truncate with a RECOVERABLE pointer so the
|
|
215
|
+
# dropped tail is not silently lost: cite the highest source line still
|
|
216
|
+
# covered by a surviving anchor and the file's total line count, so the
|
|
217
|
+
# operator can Read the un-projected tail directly.
|
|
218
|
+
cut = max(0, budget - _HARD_MARK_RESERVE)
|
|
219
|
+
body = rendered[:cut]
|
|
220
|
+
total = text.count("\n") + 1
|
|
221
|
+
covered = [int(m) for m in re.findall(r"raw L\d+-L(\d+)", body)]
|
|
222
|
+
if covered:
|
|
223
|
+
mark = (f"\n[... hard-truncated to budget: this projection covers "
|
|
224
|
+
f"through ~raw L{max(covered)} of {total}; the tail is NOT "
|
|
225
|
+
f"anchored — Read the source directly beyond ~L{max(covered)} ...]")
|
|
226
|
+
else:
|
|
227
|
+
mark = (f"\n[... hard-truncated to budget; source has {total} lines and "
|
|
228
|
+
f"this projection is not anchored — Read the source directly ...]")
|
|
229
|
+
return "L3*", (body + mark)[:budget]
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def stat_sizes(text: str, cfg: dict) -> dict:
|
|
233
|
+
blocks = parse_blocks(text)
|
|
234
|
+
sizes = {"raw": len(text)}
|
|
235
|
+
for lvl in ("L1", "L2", "L3"):
|
|
236
|
+
sizes[lvl] = len(render(blocks, lvl, cfg))
|
|
237
|
+
lvl, out = choose_level(text, cfg)
|
|
238
|
+
sizes["chosen"] = lvl
|
|
239
|
+
sizes["chosen_chars"] = len(out)
|
|
240
|
+
return sizes
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
# --- CLI dispatch (called from cli.py, before the heavy store import) --------
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def dispatch_raw_view(args) -> None:
|
|
247
|
+
import json
|
|
248
|
+
|
|
249
|
+
from .config import get_view_config
|
|
250
|
+
|
|
251
|
+
cfg = get_view_config()
|
|
252
|
+
if getattr(args, "budget", None):
|
|
253
|
+
cfg["budget"] = args.budget
|
|
254
|
+
with open(args.path, encoding="utf-8") as f:
|
|
255
|
+
text = f.read()
|
|
256
|
+
if getattr(args, "stat", False):
|
|
257
|
+
print(json.dumps(stat_sizes(text, cfg), ensure_ascii=False))
|
|
258
|
+
return
|
|
259
|
+
lvl = getattr(args, "level", None)
|
|
260
|
+
if lvl:
|
|
261
|
+
out = text if lvl == "L0" else render(parse_blocks(text), lvl, cfg)
|
|
262
|
+
else:
|
|
263
|
+
_, out = choose_level(text, cfg)
|
|
264
|
+
print(out)
|
cortex_vec/reclaim.py
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""Reclaim superseded Raw snapshots — one conversation recorded more than once.
|
|
2
|
+
|
|
3
|
+
SessionEnd names each Raw by wall-clock (``HHMMSS_session_<repo>.md``), but it
|
|
4
|
+
fires more than once per conversation (``/clear``, exit + ``--resume``) and the
|
|
5
|
+
transcript it filters is one continuously growing jsonl. Every firing re-filters
|
|
6
|
+
the *whole* transcript, so the earlier files are strict prefixes of the latest
|
|
7
|
+
one — pure redundancy that would otherwise each sit in the distill queue as its
|
|
8
|
+
own entry, spending the distill budget several times over on the same
|
|
9
|
+
conversation and landing duplicate Notes.
|
|
10
|
+
|
|
11
|
+
Two safety properties make removal sound:
|
|
12
|
+
|
|
13
|
+
* the candidate set is exactly :func:`distill_queue`'s output, i.e. Raw files
|
|
14
|
+
nothing references yet. An already-distilled Raw carries a position-anchored
|
|
15
|
+
``<!-- distilled: ... -->`` marker and is pointed at by a Note's ``source:``,
|
|
16
|
+
so it is never touched.
|
|
17
|
+
* a candidate must be a **prefix** of the survivor, so it carries no line the
|
|
18
|
+
survivor lacks. If the filter output ever diverges (its LLM residue
|
|
19
|
+
classifier is not deterministic) the match simply fails and nothing is
|
|
20
|
+
removed — the failure mode is "duplicate stays", never "content lost".
|
|
21
|
+
|
|
22
|
+
Comparison starts at the first conversation turn: the frontmatter holds the
|
|
23
|
+
wall-clock stamp that differs between recordings of the same session.
|
|
24
|
+
"""
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import subprocess
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
|
|
30
|
+
from .distill_queue import _TURN_HDRS, distill_queue
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _body(path) -> list[str]:
|
|
34
|
+
"""Conversation body: lines from the first turn header to EOF.
|
|
35
|
+
|
|
36
|
+
Returns ``[]`` for a file with no turn header (filter failure, truncation).
|
|
37
|
+
An empty body is a prefix of everything, so it must never match.
|
|
38
|
+
"""
|
|
39
|
+
lines: list[str] = []
|
|
40
|
+
started = False
|
|
41
|
+
try:
|
|
42
|
+
with Path(path).open(encoding="utf-8") as f:
|
|
43
|
+
for line in f:
|
|
44
|
+
if not started:
|
|
45
|
+
if line.strip() not in _TURN_HDRS:
|
|
46
|
+
continue
|
|
47
|
+
started = True
|
|
48
|
+
lines.append(line.rstrip("\n"))
|
|
49
|
+
except OSError:
|
|
50
|
+
return []
|
|
51
|
+
return lines
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _covers(survivor: list[str], candidate: list[str]) -> bool:
|
|
55
|
+
"""True iff candidate is a non-empty prefix of survivor."""
|
|
56
|
+
return (
|
|
57
|
+
bool(candidate)
|
|
58
|
+
and len(candidate) <= len(survivor)
|
|
59
|
+
and survivor[: len(candidate)] == candidate
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def find_superseded(root, keep=None) -> list[Path]:
|
|
64
|
+
"""Undistilled Raw files made redundant by a longer recording (FIFO order).
|
|
65
|
+
|
|
66
|
+
With ``keep``, only that file is treated as a survivor — the session-end
|
|
67
|
+
path, one body read per queued file. Without it, the whole queue is
|
|
68
|
+
compared pairwise (backlog cleanup). For two byte-identical bodies the
|
|
69
|
+
later path survives, so exactly one of the pair is reclaimed.
|
|
70
|
+
"""
|
|
71
|
+
queue = distill_queue(root)
|
|
72
|
+
if keep is not None:
|
|
73
|
+
survivor = _body(keep)
|
|
74
|
+
if not survivor:
|
|
75
|
+
return []
|
|
76
|
+
keep_resolved = Path(keep).resolve()
|
|
77
|
+
return [
|
|
78
|
+
p for p in queue
|
|
79
|
+
if p.resolve() != keep_resolved and _covers(survivor, _body(p))
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
bodies = {p: _body(p) for p in queue}
|
|
83
|
+
superseded = []
|
|
84
|
+
for cand, cand_body in bodies.items():
|
|
85
|
+
for other, other_body in bodies.items():
|
|
86
|
+
if other == cand or not _covers(other_body, cand_body):
|
|
87
|
+
continue
|
|
88
|
+
if len(cand_body) < len(other_body) or str(cand) < str(other):
|
|
89
|
+
superseded.append(cand)
|
|
90
|
+
break
|
|
91
|
+
return superseded
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _git_rm(vault, path) -> bool:
|
|
95
|
+
try:
|
|
96
|
+
proc = subprocess.run(
|
|
97
|
+
["git", "-C", str(vault), "rm", "-q", "-f", "--", str(path)],
|
|
98
|
+
capture_output=True, text=True,
|
|
99
|
+
)
|
|
100
|
+
except OSError:
|
|
101
|
+
return False
|
|
102
|
+
return proc.returncode == 0 and not Path(path).exists()
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def apply_reclaim(paths, vault=None) -> list[Path]:
|
|
106
|
+
"""Remove the given Raw files; return the ones actually gone.
|
|
107
|
+
|
|
108
|
+
Prefers ``git rm`` so the deletion is staged for the vault's auto-commit
|
|
109
|
+
(and therefore recoverable from history); falls back to unlink for files
|
|
110
|
+
git does not track, e.g. when ``git.auto_commit`` is off.
|
|
111
|
+
"""
|
|
112
|
+
removed = []
|
|
113
|
+
for path in paths:
|
|
114
|
+
path = Path(path)
|
|
115
|
+
if vault is not None and _git_rm(vault, path):
|
|
116
|
+
removed.append(path)
|
|
117
|
+
continue
|
|
118
|
+
try:
|
|
119
|
+
path.unlink()
|
|
120
|
+
except OSError:
|
|
121
|
+
continue
|
|
122
|
+
removed.append(path)
|
|
123
|
+
return removed
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def dispatch(args) -> None:
|
|
127
|
+
root = getattr(args, "root", None)
|
|
128
|
+
if not root:
|
|
129
|
+
from .config import get_vault_path
|
|
130
|
+
|
|
131
|
+
root = get_vault_path() / "Raw"
|
|
132
|
+
found = find_superseded(root, keep=getattr(args, "keep", None))
|
|
133
|
+
if getattr(args, "apply", False):
|
|
134
|
+
vault = getattr(args, "vault", None)
|
|
135
|
+
if not vault:
|
|
136
|
+
from .config import get_vault_path
|
|
137
|
+
|
|
138
|
+
vault = get_vault_path()
|
|
139
|
+
found = apply_reclaim(found, vault=vault)
|
|
140
|
+
for path in found:
|
|
141
|
+
print(path)
|
cortex_vec/rerank.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Optional LLM reranker over the top window of results (reuses OpenAI).
|
|
2
|
+
|
|
3
|
+
Off by default. Any failure (no key, bad JSON, API error) returns the input
|
|
4
|
+
order unchanged — reranking must never make a search fail.
|
|
5
|
+
"""
|
|
6
|
+
import json
|
|
7
|
+
|
|
8
|
+
_SYSTEM = (
|
|
9
|
+
"你是檢索結果重排器。給定查詢與若干候選筆記(每個有 index、title、summary),"
|
|
10
|
+
"依與查詢的相關性為每個候選打 0-10 分。只輸出 JSON 陣列,每個元素為 "
|
|
11
|
+
'{"index": <int>, "score": <number>},不要任何其他文字或 markdown。'
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _parse_scores(content):
|
|
16
|
+
"""Parse the model's JSON array, tolerating ```json fences."""
|
|
17
|
+
text = content.strip()
|
|
18
|
+
if text.startswith("```"):
|
|
19
|
+
text = text.split("```", 2)[1]
|
|
20
|
+
if text.startswith("json"):
|
|
21
|
+
text = text[4:]
|
|
22
|
+
return json.loads(text)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def rerank(query, results, model, window=15):
|
|
26
|
+
"""Reorder the top `window` of `results` by LLM relevance; tail unchanged."""
|
|
27
|
+
if not results:
|
|
28
|
+
return results
|
|
29
|
+
head = results[:window]
|
|
30
|
+
tail = results[window:]
|
|
31
|
+
try:
|
|
32
|
+
import openai
|
|
33
|
+
client = openai.OpenAI()
|
|
34
|
+
candidates = "\n".join(
|
|
35
|
+
f'{i}: {r.get("title", "")} — {r.get("summary", "")}' for i, r in enumerate(head)
|
|
36
|
+
)
|
|
37
|
+
resp = client.chat.completions.create(
|
|
38
|
+
model=model,
|
|
39
|
+
messages=[
|
|
40
|
+
{"role": "system", "content": _SYSTEM},
|
|
41
|
+
{"role": "user", "content": f"查詢:{query}\n\n候選:\n{candidates}"},
|
|
42
|
+
],
|
|
43
|
+
max_completion_tokens=500,
|
|
44
|
+
reasoning_effort="none",
|
|
45
|
+
)
|
|
46
|
+
scored = _parse_scores(resp.choices[0].message.content)
|
|
47
|
+
order = {int(s["index"]): float(s["score"]) for s in scored}
|
|
48
|
+
ranked_idx = sorted(range(len(head)), key=lambda i: order.get(i, -1.0), reverse=True)
|
|
49
|
+
head = [head[i] for i in ranked_idx]
|
|
50
|
+
except Exception:
|
|
51
|
+
return results
|
|
52
|
+
return head + tail
|