cortex-vec 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cortex_vec/__init__.py +3 -0
- cortex_vec/bm25.py +129 -0
- cortex_vec/cli.py +191 -0
- cortex_vec/config.py +82 -0
- cortex_vec/distill_plan.py +431 -0
- cortex_vec/distill_queue.py +171 -0
- cortex_vec/eval/__init__.py +0 -0
- cortex_vec/eval/adapters.py +83 -0
- cortex_vec/eval/corpus.py +36 -0
- cortex_vec/eval/report.py +44 -0
- cortex_vec/eval/run.py +130 -0
- cortex_vec/eval/score.py +62 -0
- cortex_vec/fusion.py +168 -0
- cortex_vec/graph.py +107 -0
- cortex_vec/parser.py +77 -0
- cortex_vec/raw_map.py +165 -0
- cortex_vec/raw_page.py +69 -0
- cortex_vec/raw_source.py +306 -0
- cortex_vec/raw_span.py +96 -0
- cortex_vec/raw_view.py +264 -0
- cortex_vec/reclaim.py +141 -0
- cortex_vec/rerank.py +52 -0
- cortex_vec/store.py +466 -0
- cortex_vec/synonyms.py +43 -0
- cortex_vec/tokenize.py +79 -0
- cortex_vec-0.7.0.dist-info/METADATA +68 -0
- cortex_vec-0.7.0.dist-info/RECORD +32 -0
- cortex_vec-0.7.0.dist-info/WHEEL +5 -0
- cortex_vec-0.7.0.dist-info/entry_points.txt +2 -0
- cortex_vec-0.7.0.dist-info/licenses/LICENSE +201 -0
- cortex_vec-0.7.0.dist-info/licenses/NOTICE +5 -0
- cortex_vec-0.7.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Load and validate the hand-labeled eval corpus (queries.jsonl)."""
|
|
2
|
+
import json
|
|
3
|
+
|
|
4
|
+
REQUIRED_FIELDS = ("id", "query", "gold", "type")
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def load_queries(path):
|
|
8
|
+
"""Load queries.jsonl. Each line must have id/query/gold/type.
|
|
9
|
+
|
|
10
|
+
Returns list[dict]. Raises ValueError on a malformed entry.
|
|
11
|
+
"""
|
|
12
|
+
queries = []
|
|
13
|
+
with open(path, encoding="utf-8") as f:
|
|
14
|
+
for lineno, line in enumerate(f, 1):
|
|
15
|
+
line = line.strip()
|
|
16
|
+
if not line:
|
|
17
|
+
continue
|
|
18
|
+
row = json.loads(line)
|
|
19
|
+
for field in REQUIRED_FIELDS:
|
|
20
|
+
if field not in row:
|
|
21
|
+
raise ValueError(f"line {lineno}: missing required field '{field}'")
|
|
22
|
+
if not isinstance(row["gold"], list) or not row["gold"]:
|
|
23
|
+
raise ValueError(f"line {lineno}: 'gold' must be a non-empty list")
|
|
24
|
+
queries.append(row)
|
|
25
|
+
return queries
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def check_gold_paths(queries, existing_paths):
|
|
29
|
+
"""Return {query_id: [missing_gold_paths]} for gold entries not in existing_paths."""
|
|
30
|
+
existing = set(existing_paths)
|
|
31
|
+
missing = {}
|
|
32
|
+
for q in queries:
|
|
33
|
+
gone = [g for g in q["gold"] if g not in existing]
|
|
34
|
+
if gone:
|
|
35
|
+
missing[q["id"]] = gone
|
|
36
|
+
return missing
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Render an eval summary into a markdown scorecard."""
|
|
2
|
+
from datetime import date
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def _fmt(x):
|
|
6
|
+
return f"{x:.3f}" if isinstance(x, float) else str(x)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def render(summary, meta):
|
|
10
|
+
"""meta: dict with corpus, k, n (and optional commit/hardware)."""
|
|
11
|
+
lines = []
|
|
12
|
+
lines.append(f"# {date.today().isoformat()} — {meta.get('corpus', 'corpus')}")
|
|
13
|
+
lines.append("")
|
|
14
|
+
lines.append(f"- **Corpus:** {meta.get('corpus', '')}")
|
|
15
|
+
lines.append(f"- **N (queries):** {meta.get('n', '')}")
|
|
16
|
+
lines.append(f"- **K:** {meta.get('k', '')}")
|
|
17
|
+
if meta.get("commit"):
|
|
18
|
+
lines.append(f"- **Commit:** `{meta['commit']}`")
|
|
19
|
+
lines.append("")
|
|
20
|
+
|
|
21
|
+
lines.append("## Per-adapter")
|
|
22
|
+
lines.append("")
|
|
23
|
+
lines.append("| Adapter | P@K | R@K | MRR | Hit rate | p50 latency (ms) |")
|
|
24
|
+
lines.append("|---|---|---|---|---|---|")
|
|
25
|
+
for adapter, s in summary["by_adapter"].items():
|
|
26
|
+
lines.append(
|
|
27
|
+
f"| {adapter} | {_fmt(s['p'])} | {_fmt(s['r'])} | {_fmt(s['mrr'])} "
|
|
28
|
+
f"| {_fmt(s['hit_rate'])} | {_fmt(s['latency_p50'])} |"
|
|
29
|
+
)
|
|
30
|
+
lines.append("")
|
|
31
|
+
|
|
32
|
+
if summary.get("by_type"):
|
|
33
|
+
lines.append("## Per-adapter/type")
|
|
34
|
+
lines.append("")
|
|
35
|
+
lines.append("| Adapter/Type | n | P@K | R@K | MRR | Hit rate |")
|
|
36
|
+
lines.append("|---|---|---|---|---|---|")
|
|
37
|
+
for key, s in summary["by_type"].items():
|
|
38
|
+
lines.append(
|
|
39
|
+
f"| {key} | {s['n']} | {_fmt(s['p'])} | {_fmt(s['r'])} "
|
|
40
|
+
f"| {_fmt(s['mrr'])} | {_fmt(s['hit_rate'])} |"
|
|
41
|
+
)
|
|
42
|
+
lines.append("")
|
|
43
|
+
|
|
44
|
+
return "\n".join(lines)
|
cortex_vec/eval/run.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Eval runner: run adapters over a query set, score, aggregate, render."""
|
|
2
|
+
import json
|
|
3
|
+
import sys
|
|
4
|
+
import time
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from .. import store
|
|
8
|
+
from ..config import get_vault_path
|
|
9
|
+
from ..parser import parse_document
|
|
10
|
+
from . import adapters, corpus, report, score
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _load_corpus_docs():
|
|
14
|
+
"""Build adapter corpus docs from vault Notes/ + Projects/ (skip _archive)."""
|
|
15
|
+
vault = get_vault_path()
|
|
16
|
+
docs = []
|
|
17
|
+
for scan_dir in ("Notes", "Projects"):
|
|
18
|
+
base = vault / scan_dir
|
|
19
|
+
if not base.is_dir():
|
|
20
|
+
continue
|
|
21
|
+
for md in base.rglob("*.md"):
|
|
22
|
+
rel = str(md.relative_to(vault))
|
|
23
|
+
if "_archive" in rel:
|
|
24
|
+
continue
|
|
25
|
+
fm, body = parse_document(md.read_text(encoding="utf-8", errors="replace"))
|
|
26
|
+
docs.append(store.bm25_doc_from_fields(rel, fm, body))
|
|
27
|
+
return docs
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def run_adapter(adapter, queries, k):
|
|
31
|
+
"""Run one initialized adapter over all queries; return per-query score rows."""
|
|
32
|
+
rows = []
|
|
33
|
+
for q in queries:
|
|
34
|
+
t0 = time.perf_counter()
|
|
35
|
+
ranked = [doc_id for doc_id, _ in adapter.query(q["query"], k)]
|
|
36
|
+
latency_ms = (time.perf_counter() - t0) * 1000.0
|
|
37
|
+
sc = score.score_query(ranked, set(q["gold"]), k)
|
|
38
|
+
rows.append({
|
|
39
|
+
"query_id": q["id"],
|
|
40
|
+
"adapter": adapter.name,
|
|
41
|
+
"type": q["type"],
|
|
42
|
+
"latency_ms": latency_ms,
|
|
43
|
+
**sc,
|
|
44
|
+
})
|
|
45
|
+
return rows
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def dispatch(args):
|
|
49
|
+
if args.action == "propose":
|
|
50
|
+
_propose(args)
|
|
51
|
+
return
|
|
52
|
+
_run(args)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _run(args):
|
|
56
|
+
queries = corpus.load_queries(Path(args.queries))
|
|
57
|
+
docs = _load_corpus_docs()
|
|
58
|
+
existing = {d["id"] for d in docs}
|
|
59
|
+
missing = corpus.check_gold_paths(queries, existing)
|
|
60
|
+
if missing:
|
|
61
|
+
print(f"WARNING: gold paths missing from vault: {missing}", file=sys.stderr)
|
|
62
|
+
|
|
63
|
+
all_rows = []
|
|
64
|
+
for name in [a.strip() for a in args.adapters.split(",") if a.strip()]:
|
|
65
|
+
cls = adapters.REGISTRY.get(name)
|
|
66
|
+
if cls is None:
|
|
67
|
+
print(f"Unknown adapter: {name}", file=sys.stderr)
|
|
68
|
+
continue
|
|
69
|
+
adapter = cls()
|
|
70
|
+
adapter.init(docs)
|
|
71
|
+
all_rows.extend(run_adapter(adapter, queries, args.k))
|
|
72
|
+
adapter.teardown()
|
|
73
|
+
|
|
74
|
+
for row in all_rows:
|
|
75
|
+
print(json.dumps(row, ensure_ascii=False))
|
|
76
|
+
|
|
77
|
+
summary = score.aggregate(all_rows)
|
|
78
|
+
meta = {"corpus": Path(args.queries).stem, "k": args.k, "n": len(queries)}
|
|
79
|
+
md = report.render(summary, meta)
|
|
80
|
+
if args.out:
|
|
81
|
+
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
|
|
82
|
+
Path(args.out).write_text(md, encoding="utf-8")
|
|
83
|
+
print(f"Scorecard written to {args.out}", file=sys.stderr)
|
|
84
|
+
else:
|
|
85
|
+
print(md, file=sys.stderr)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _propose(args):
|
|
89
|
+
"""Use OpenAI to propose candidate queries from notes for the user to confirm.
|
|
90
|
+
|
|
91
|
+
Writes JSONL with gold pre-filled to the note path; the user edits/confirms.
|
|
92
|
+
"""
|
|
93
|
+
from openai import OpenAI
|
|
94
|
+
|
|
95
|
+
from ..config import SUMMARY_MODEL
|
|
96
|
+
|
|
97
|
+
docs = _load_corpus_docs()
|
|
98
|
+
client = OpenAI()
|
|
99
|
+
out_path = Path(args.queries)
|
|
100
|
+
proposed = []
|
|
101
|
+
for d in docs:
|
|
102
|
+
prompt = (
|
|
103
|
+
"根據以下技術筆記,生成 1 個使用者最可能用來查到這篇筆記的搜尋 query"
|
|
104
|
+
"(中英混合,貼近真實工程查詢,10 字內)。只輸出 query 文字。\n\n"
|
|
105
|
+
f"標題:{d['title']}\n摘要:{d['summary']}"
|
|
106
|
+
)
|
|
107
|
+
try:
|
|
108
|
+
resp = client.chat.completions.create(
|
|
109
|
+
model=SUMMARY_MODEL,
|
|
110
|
+
messages=[{"role": "user", "content": prompt}],
|
|
111
|
+
max_completion_tokens=40,
|
|
112
|
+
reasoning_effort="none",
|
|
113
|
+
)
|
|
114
|
+
query = resp.choices[0].message.content.strip()
|
|
115
|
+
except Exception as e: # noqa: BLE001
|
|
116
|
+
print(f"propose failed for {d['id']}: {e}", file=sys.stderr)
|
|
117
|
+
continue
|
|
118
|
+
proposed.append({
|
|
119
|
+
"id": f"q-{len(proposed) + 1:03d}",
|
|
120
|
+
"query": query,
|
|
121
|
+
"gold": [d["id"]],
|
|
122
|
+
"type": "single-note",
|
|
123
|
+
"note": "AUTO-PROPOSED — review query + gold before use",
|
|
124
|
+
})
|
|
125
|
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
126
|
+
out_path.write_text(
|
|
127
|
+
"\n".join(json.dumps(p, ensure_ascii=False) for p in proposed), encoding="utf-8"
|
|
128
|
+
)
|
|
129
|
+
print(f"Proposed {len(proposed)} queries to {out_path} — review before running eval",
|
|
130
|
+
file=sys.stderr)
|
cortex_vec/eval/score.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Retrieval metrics: P@k, R@k, MRR, hit, and aggregation."""
|
|
2
|
+
from statistics import median
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def score_query(ranked, gold, k):
|
|
6
|
+
"""Score one query's ranked base-path list against a gold set.
|
|
7
|
+
|
|
8
|
+
ranked: list[str] of base paths, best-first.
|
|
9
|
+
gold: set[str] of relevant base paths.
|
|
10
|
+
Returns dict with precision_at_k, recall_at_k, reciprocal_rank, hit.
|
|
11
|
+
"""
|
|
12
|
+
gold = set(gold)
|
|
13
|
+
top_k = ranked[:k]
|
|
14
|
+
hits = sum(1 for p in top_k if p in gold)
|
|
15
|
+
precision = hits / k if k else 0.0
|
|
16
|
+
recall = hits / len(gold) if gold else 0.0
|
|
17
|
+
|
|
18
|
+
reciprocal_rank = 0.0
|
|
19
|
+
for idx, p in enumerate(top_k):
|
|
20
|
+
if p in gold:
|
|
21
|
+
reciprocal_rank = 1.0 / (idx + 1)
|
|
22
|
+
break
|
|
23
|
+
|
|
24
|
+
return {
|
|
25
|
+
"precision_at_k": precision,
|
|
26
|
+
"recall_at_k": recall,
|
|
27
|
+
"reciprocal_rank": reciprocal_rank,
|
|
28
|
+
"hit": hits > 0,
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def aggregate(rows):
|
|
33
|
+
"""Aggregate per-query score rows into by_adapter and by_type summaries.
|
|
34
|
+
|
|
35
|
+
Each row must have: adapter, type, precision_at_k, recall_at_k,
|
|
36
|
+
reciprocal_rank, hit, latency_ms.
|
|
37
|
+
"""
|
|
38
|
+
by_adapter = {}
|
|
39
|
+
by_type = {}
|
|
40
|
+
|
|
41
|
+
def _bucket(d, key):
|
|
42
|
+
return d.setdefault(key, [])
|
|
43
|
+
|
|
44
|
+
for row in rows:
|
|
45
|
+
_bucket(by_adapter, row["adapter"]).append(row)
|
|
46
|
+
_bucket(by_type, (row["adapter"], row["type"])).append(row)
|
|
47
|
+
|
|
48
|
+
def _summary(group):
|
|
49
|
+
n = len(group)
|
|
50
|
+
return {
|
|
51
|
+
"n": n,
|
|
52
|
+
"p": sum(g["precision_at_k"] for g in group) / n,
|
|
53
|
+
"r": sum(g["recall_at_k"] for g in group) / n,
|
|
54
|
+
"mrr": sum(g["reciprocal_rank"] for g in group) / n,
|
|
55
|
+
"hit_rate": sum(1 for g in group if g["hit"]) / n,
|
|
56
|
+
"latency_p50": median(g["latency_ms"] for g in group),
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
return {
|
|
60
|
+
"by_adapter": {a: _summary(g) for a, g in by_adapter.items()},
|
|
61
|
+
"by_type": {f"{a}/{t}": _summary(g) for (a, t), g in by_type.items()},
|
|
62
|
+
}
|
cortex_vec/fusion.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""Hybrid retrieval: RRF fusion of vector + BM25 streams."""
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
from . import store
|
|
5
|
+
from .config import BM25_DIR, get_retrieval_config, get_vault_path
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def rrf_fuse(ranked, weights, k=60):
|
|
9
|
+
"""Reciprocal Rank Fusion over named streams.
|
|
10
|
+
|
|
11
|
+
ranked: {stream_name: [(doc_id, rank), ...]} (rank is 0-based, best=0)
|
|
12
|
+
weights: {stream_name: weight}
|
|
13
|
+
Returns [(doc_id, score)] sorted by score desc. Streams that are empty
|
|
14
|
+
have their weight redistributed across the present streams.
|
|
15
|
+
"""
|
|
16
|
+
present = [name for name, items in ranked.items() if items]
|
|
17
|
+
if not present:
|
|
18
|
+
return []
|
|
19
|
+
total = sum(weights.get(name, 0.0) for name in present) or 1.0
|
|
20
|
+
norm = {name: weights.get(name, 0.0) / total for name in present}
|
|
21
|
+
|
|
22
|
+
scores = {}
|
|
23
|
+
for name in present:
|
|
24
|
+
for doc_id, rank in ranked[name]:
|
|
25
|
+
scores[doc_id] = scores.get(doc_id, 0.0) + norm[name] * (1.0 / (k + rank + 1))
|
|
26
|
+
return sorted(scores.items(), key=lambda kv: kv[1], reverse=True)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _bm25_stream(query, n, where, synonym_weight=0.0):
|
|
30
|
+
"""Load the persisted BM25 index and search; return [] on any failure."""
|
|
31
|
+
from . import bm25
|
|
32
|
+
try:
|
|
33
|
+
idx = bm25.BM25Index(BM25_DIR)
|
|
34
|
+
idx.load()
|
|
35
|
+
return idx.search(query, n, where, synonym_weight=synonym_weight)
|
|
36
|
+
except Exception:
|
|
37
|
+
return []
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _vector_stream(query, n, where):
|
|
41
|
+
# No API key -> embeddings unavailable; skip vector so the query degrades to
|
|
42
|
+
# BM25-only. (The embedding fn hard-exits via sys.exit on a missing key, and
|
|
43
|
+
# SystemExit is not caught by `except Exception`, so we must check up front.)
|
|
44
|
+
if not os.environ.get("OPENAI_API_KEY"):
|
|
45
|
+
return []
|
|
46
|
+
try:
|
|
47
|
+
return store.vector_stream(query, n, where)
|
|
48
|
+
except Exception:
|
|
49
|
+
return []
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# Order matters: vector first so its display fields (e.g. summary) win on merge.
|
|
53
|
+
_STREAM_ORDER = ("vector", "bm25")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _graph_ranked(ranked, weights, rc, n):
|
|
57
|
+
"""Build the graph RRF stream: wikilink-neighbors of the top vector/bm25 hits.
|
|
58
|
+
|
|
59
|
+
Returns (ranked_list, display_map) where ranked_list is [(doc_id, rank)] for
|
|
60
|
+
rrf_fuse and display_map carries display dicts for any graph-introduced docs.
|
|
61
|
+
([], {}) on failure (e.g. vault unconfigured) or when graph yields nothing.
|
|
62
|
+
Rank-based (not additive), so it composes with the other streams via RRF
|
|
63
|
+
without the score-scale conflict that the old additive boost had.
|
|
64
|
+
"""
|
|
65
|
+
from . import graph as graph_mod
|
|
66
|
+
try:
|
|
67
|
+
pre = rrf_fuse(ranked, weights, k=rc["rrf_k"])
|
|
68
|
+
seeds = [doc_id for doc_id, _ in pre[:rc["graph_top_k"]]]
|
|
69
|
+
if not seeds:
|
|
70
|
+
return [], {}
|
|
71
|
+
adjacency, meta = graph_mod.build_graph(get_vault_path())
|
|
72
|
+
stream = graph_mod.graph_stream(
|
|
73
|
+
adjacency, seeds, hops=rc["graph_hops"], max_n=max(n * 2, 10)
|
|
74
|
+
)
|
|
75
|
+
gdisplay = {doc_id: meta[doc_id] for doc_id, _ in stream if doc_id in meta}
|
|
76
|
+
return stream, gdisplay
|
|
77
|
+
except (Exception, SystemExit):
|
|
78
|
+
return [], {}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _diversify(fused, display, max_per_repo):
|
|
82
|
+
"""Cap results per repo: keep best-`max_per_repo` per repo first, then append
|
|
83
|
+
the rest in original order (nothing dropped, only reordered). 0 = no-op.
|
|
84
|
+
Docs with an empty repo are never capped.
|
|
85
|
+
"""
|
|
86
|
+
if not max_per_repo:
|
|
87
|
+
return fused
|
|
88
|
+
counts = {}
|
|
89
|
+
primary, overflow = [], []
|
|
90
|
+
for doc_id, score in fused:
|
|
91
|
+
repo = (display.get(doc_id) or {}).get("repo", "")
|
|
92
|
+
if not repo:
|
|
93
|
+
primary.append((doc_id, score))
|
|
94
|
+
continue
|
|
95
|
+
if counts.get(repo, 0) < max_per_repo:
|
|
96
|
+
counts[repo] = counts.get(repo, 0) + 1
|
|
97
|
+
primary.append((doc_id, score))
|
|
98
|
+
else:
|
|
99
|
+
overflow.append((doc_id, score))
|
|
100
|
+
return primary + overflow
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, rerank=None):
|
|
104
|
+
"""Hybrid search entry point. Returns up to n display dicts (best-first).
|
|
105
|
+
|
|
106
|
+
Gracefully degrades: if a stream errors or is disabled, the other carries
|
|
107
|
+
the query (RRF weight is redistributed in rrf_fuse).
|
|
108
|
+
"""
|
|
109
|
+
rc = get_retrieval_config()
|
|
110
|
+
weights = {"vector": rc["w_vec"], "bm25": rc["w_bm25"], "graph": rc["w_graph"]}
|
|
111
|
+
|
|
112
|
+
streams = {}
|
|
113
|
+
if use_vector:
|
|
114
|
+
streams["vector"] = _vector_stream(query, n, where)
|
|
115
|
+
if use_bm25:
|
|
116
|
+
streams["bm25"] = _bm25_stream(query, n, where, synonym_weight=rc["synonym_weight"])
|
|
117
|
+
|
|
118
|
+
ranked = {}
|
|
119
|
+
display = {}
|
|
120
|
+
vec_score = {} # doc_id -> vector cosine similarity (interpretable, 0-1)
|
|
121
|
+
for name in _STREAM_ORDER:
|
|
122
|
+
items = streams.get(name) or []
|
|
123
|
+
for rank, item in enumerate(items):
|
|
124
|
+
ranked.setdefault(name, []).append((item["id"], rank))
|
|
125
|
+
if name == "vector":
|
|
126
|
+
vec_score[item["id"]] = item.get("score", 0.0)
|
|
127
|
+
disp = display.setdefault(item["id"], {})
|
|
128
|
+
for key, val in item.items():
|
|
129
|
+
if key == "score":
|
|
130
|
+
continue
|
|
131
|
+
if key not in disp or not disp.get(key):
|
|
132
|
+
disp[key] = val
|
|
133
|
+
|
|
134
|
+
use_graph = rc["graph"] if graph is None else graph
|
|
135
|
+
if use_graph:
|
|
136
|
+
g_ranked, g_display = _graph_ranked(ranked, weights, rc, n)
|
|
137
|
+
if g_ranked:
|
|
138
|
+
ranked["graph"] = g_ranked
|
|
139
|
+
for doc_id, disp in g_display.items():
|
|
140
|
+
d = display.setdefault(doc_id, {})
|
|
141
|
+
for key, val in disp.items():
|
|
142
|
+
if key == "score":
|
|
143
|
+
continue
|
|
144
|
+
if key not in d or not d.get(key):
|
|
145
|
+
d[key] = val
|
|
146
|
+
|
|
147
|
+
fused = rrf_fuse(ranked, weights, k=rc["rrf_k"])
|
|
148
|
+
fused = _diversify(fused, display, rc["max_per_repo"])
|
|
149
|
+
|
|
150
|
+
use_rerank = rc["rerank"] if rerank is None else rerank
|
|
151
|
+
take = max(n, rc["rerank_window"]) if use_rerank else n
|
|
152
|
+
|
|
153
|
+
out = []
|
|
154
|
+
for doc_id, _rrf in fused[:take]:
|
|
155
|
+
entry = dict(display.get(doc_id, {}))
|
|
156
|
+
entry["id"] = doc_id
|
|
157
|
+
# `score` reports the vector cosine similarity (0-1, interpretable) so
|
|
158
|
+
# absolute-threshold consumers (distill/broadcast dedup) keep working.
|
|
159
|
+
# RRF only determines ORDER (already baked into `fused`). bm25-only hits
|
|
160
|
+
# have no cosine score -> 0.0.
|
|
161
|
+
entry["score"] = round(vec_score.get(doc_id, 0.0), 4)
|
|
162
|
+
out.append(entry)
|
|
163
|
+
|
|
164
|
+
if use_rerank:
|
|
165
|
+
from . import rerank as rerank_mod
|
|
166
|
+
out = rerank_mod.rerank(query, out, model=rc["rerank_model"], window=rc["rerank_window"])
|
|
167
|
+
|
|
168
|
+
return out[:n]
|
cortex_vec/graph.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Wikilink graph over the vault, built lazily from markdown and cached per vault.
|
|
2
|
+
|
|
3
|
+
The vault's `[[Title]]` links form a human-curated graph. We resolve each link
|
|
4
|
+
target (a note title) to a base path via a title index (frontmatter `title`
|
|
5
|
+
plus the filename stem), then expose:
|
|
6
|
+
- `adjacency` {base_path: set(neighbor_base_path)} for traversal, and
|
|
7
|
+
- `meta` {base_path: display dict} so graph-introduced neighbors can be shown.
|
|
8
|
+
Unresolved links are skipped (a dangling link is not an error).
|
|
9
|
+
|
|
10
|
+
Graph participates in retrieval as a THIRD RRF stream (see fusion.py): a
|
|
11
|
+
rank-based list of wikilink-neighbors of the top hits. Rank-based fusion avoids
|
|
12
|
+
the scale conflict of adding a boost onto RRF's compressed score band.
|
|
13
|
+
"""
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from .parser import classify_path, extract_summary, extract_wikilinks, parse_document
|
|
17
|
+
|
|
18
|
+
_cache = {} # str(vault) -> (adjacency, meta)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _meta_for(rel, fm, body):
|
|
22
|
+
"""Display dict for a note (mirrors the shape store/bm25 streams emit)."""
|
|
23
|
+
doc_type, category = classify_path(rel)
|
|
24
|
+
repos_str = fm.get("repos", "")
|
|
25
|
+
repos = [r.strip() for r in repos_str.split(",") if r.strip()] if repos_str else []
|
|
26
|
+
if doc_type == "project" and category and category not in repos:
|
|
27
|
+
repos.insert(0, category)
|
|
28
|
+
return {
|
|
29
|
+
"id": rel,
|
|
30
|
+
"title": fm.get("title", rel.rsplit("/", 1)[-1].removesuffix(".md")),
|
|
31
|
+
"type": doc_type,
|
|
32
|
+
"repo": (repos or [""])[0],
|
|
33
|
+
"category": category,
|
|
34
|
+
"tags": fm.get("tags", ""),
|
|
35
|
+
"summary": extract_summary(body),
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def build_graph(vault):
|
|
40
|
+
"""Return (adjacency, meta) for the vault. Cached by vault path."""
|
|
41
|
+
key = str(vault)
|
|
42
|
+
if key in _cache:
|
|
43
|
+
return _cache[key]
|
|
44
|
+
|
|
45
|
+
vault = Path(vault)
|
|
46
|
+
title_to_path = {}
|
|
47
|
+
meta = {}
|
|
48
|
+
raw = [] # (base_path, [link targets])
|
|
49
|
+
|
|
50
|
+
for scan_dir in ("Notes", "Projects"):
|
|
51
|
+
base = vault / scan_dir
|
|
52
|
+
if not base.is_dir():
|
|
53
|
+
continue
|
|
54
|
+
for md in base.rglob("*.md"):
|
|
55
|
+
rel = str(md.relative_to(vault))
|
|
56
|
+
if "_archive" in rel:
|
|
57
|
+
continue
|
|
58
|
+
text = md.read_text(encoding="utf-8", errors="replace")
|
|
59
|
+
fm, body = parse_document(text)
|
|
60
|
+
stem = md.stem
|
|
61
|
+
title_to_path.setdefault(fm.get("title", stem), rel)
|
|
62
|
+
title_to_path.setdefault(stem, rel)
|
|
63
|
+
meta[rel] = _meta_for(rel, fm, body)
|
|
64
|
+
raw.append((rel, extract_wikilinks(body)))
|
|
65
|
+
|
|
66
|
+
adjacency = {rel: set() for rel, _ in raw}
|
|
67
|
+
for rel, targets in raw:
|
|
68
|
+
for t in targets:
|
|
69
|
+
dest = title_to_path.get(t)
|
|
70
|
+
if dest and dest != rel:
|
|
71
|
+
adjacency[rel].add(dest)
|
|
72
|
+
adjacency.setdefault(dest, set()).add(rel) # links are bidirectional
|
|
73
|
+
|
|
74
|
+
_cache[key] = (adjacency, meta)
|
|
75
|
+
return _cache[key]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _bfs_neighbors(adjacency, seeds, hops):
|
|
79
|
+
"""Return {base_path: distance} reachable within `hops` from seeds (excluding seeds)."""
|
|
80
|
+
frontier = set(seeds)
|
|
81
|
+
visited = set(seeds)
|
|
82
|
+
dist = {}
|
|
83
|
+
for d in range(1, hops + 1):
|
|
84
|
+
nxt = set()
|
|
85
|
+
for node in frontier:
|
|
86
|
+
for nb in adjacency.get(node, ()):
|
|
87
|
+
if nb not in visited:
|
|
88
|
+
visited.add(nb)
|
|
89
|
+
dist[nb] = d
|
|
90
|
+
nxt.add(nb)
|
|
91
|
+
frontier = nxt
|
|
92
|
+
if not frontier:
|
|
93
|
+
break
|
|
94
|
+
return dist
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def graph_stream(adjacency, seeds, hops=1, max_n=15):
|
|
98
|
+
"""A rank-based stream of wikilink-neighbors of `seeds`, nearest first.
|
|
99
|
+
|
|
100
|
+
Returns [(doc_id, rank)] (rank 0-based, capped at max_n), excluding seeds —
|
|
101
|
+
shaped for rrf_fuse as a third retrieval stream. Empty if no neighbors.
|
|
102
|
+
"""
|
|
103
|
+
dist = _bfs_neighbors(adjacency, seeds, hops)
|
|
104
|
+
if not dist:
|
|
105
|
+
return []
|
|
106
|
+
ordered = sorted(dist.items(), key=lambda kv: kv[1]) # nearest first
|
|
107
|
+
return [(doc_id, rank) for rank, (doc_id, _d) in enumerate(ordered[:max_n])]
|
cortex_vec/parser.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Frontmatter parsing and content classification."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import frontmatter
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def parse_document(text):
|
|
10
|
+
"""Parse frontmatter and body from markdown text.
|
|
11
|
+
|
|
12
|
+
Returns (dict, str) — metadata dict and body content.
|
|
13
|
+
Lists in tags/repos are normalized to comma-separated strings
|
|
14
|
+
(ChromaDB metadata only accepts scalar values).
|
|
15
|
+
"""
|
|
16
|
+
post = frontmatter.loads(text)
|
|
17
|
+
fm = dict(post.metadata)
|
|
18
|
+
|
|
19
|
+
for key in ("tags", "repos"):
|
|
20
|
+
if key in fm and isinstance(fm[key], list):
|
|
21
|
+
fm[key] = ",".join(str(v) for v in fm[key])
|
|
22
|
+
|
|
23
|
+
return fm, post.content
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def extract_summary(body, max_len=120):
|
|
27
|
+
"""Extract first non-empty, non-heading line from body."""
|
|
28
|
+
for line in body.splitlines():
|
|
29
|
+
stripped = line.strip()
|
|
30
|
+
if not stripped or stripped.startswith("#"):
|
|
31
|
+
continue
|
|
32
|
+
stripped = re.sub(r"^>\s*(\[![a-zA-Z]+\])?\s*", "", stripped)
|
|
33
|
+
if stripped:
|
|
34
|
+
return stripped[:max_len]
|
|
35
|
+
return ""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def classify_path(rel_path):
|
|
39
|
+
"""Determine (type, category) from vault-relative path.
|
|
40
|
+
|
|
41
|
+
Returns:
|
|
42
|
+
tuple: (doc_type, category) e.g. ("note", "Nginx"), ("project", "acme-core")
|
|
43
|
+
"""
|
|
44
|
+
parts = Path(rel_path).parts
|
|
45
|
+
if not parts:
|
|
46
|
+
return "unknown", ""
|
|
47
|
+
|
|
48
|
+
top = parts[0]
|
|
49
|
+
if top == "Notes":
|
|
50
|
+
category = parts[1] if len(parts) > 2 else ""
|
|
51
|
+
return "note", category
|
|
52
|
+
elif top == "Projects":
|
|
53
|
+
if len(parts) > 1 and parts[1] == "_archive":
|
|
54
|
+
return "archive", ""
|
|
55
|
+
repo = parts[1] if len(parts) > 1 else ""
|
|
56
|
+
return "project", repo
|
|
57
|
+
elif top == "Weekly":
|
|
58
|
+
return "weekly", ""
|
|
59
|
+
elif top == "Raw":
|
|
60
|
+
return "raw", ""
|
|
61
|
+
return "unknown", ""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
_WIKILINK_RE = re.compile(r"\[\[([^\]]+)\]\]")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def extract_wikilinks(text):
|
|
68
|
+
"""Return unique wikilink targets from text. Strips surrounding whitespace
|
|
69
|
+
and drops any `|alias` suffix (Obsidian alias syntax)."""
|
|
70
|
+
targets = []
|
|
71
|
+
seen = set()
|
|
72
|
+
for raw in _WIKILINK_RE.findall(text):
|
|
73
|
+
target = raw.split("|", 1)[0].strip()
|
|
74
|
+
if target and target not in seen:
|
|
75
|
+
seen.add(target)
|
|
76
|
+
targets.append(target)
|
|
77
|
+
return targets
|