cortex-vec 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cortex_vec/__init__.py +3 -0
- cortex_vec/bm25.py +129 -0
- cortex_vec/cli.py +191 -0
- cortex_vec/config.py +82 -0
- cortex_vec/distill_plan.py +431 -0
- cortex_vec/distill_queue.py +171 -0
- cortex_vec/eval/__init__.py +0 -0
- cortex_vec/eval/adapters.py +83 -0
- cortex_vec/eval/corpus.py +36 -0
- cortex_vec/eval/report.py +44 -0
- cortex_vec/eval/run.py +130 -0
- cortex_vec/eval/score.py +62 -0
- cortex_vec/fusion.py +168 -0
- cortex_vec/graph.py +107 -0
- cortex_vec/parser.py +77 -0
- cortex_vec/raw_map.py +165 -0
- cortex_vec/raw_page.py +69 -0
- cortex_vec/raw_source.py +306 -0
- cortex_vec/raw_span.py +96 -0
- cortex_vec/raw_view.py +264 -0
- cortex_vec/reclaim.py +141 -0
- cortex_vec/rerank.py +52 -0
- cortex_vec/store.py +466 -0
- cortex_vec/synonyms.py +43 -0
- cortex_vec/tokenize.py +79 -0
- cortex_vec-0.7.0.dist-info/METADATA +68 -0
- cortex_vec-0.7.0.dist-info/RECORD +32 -0
- cortex_vec-0.7.0.dist-info/WHEEL +5 -0
- cortex_vec-0.7.0.dist-info/entry_points.txt +2 -0
- cortex_vec-0.7.0.dist-info/licenses/LICENSE +201 -0
- cortex_vec-0.7.0.dist-info/licenses/NOTICE +5 -0
- cortex_vec-0.7.0.dist-info/top_level.txt +1 -0
cortex_vec/store.py
ADDED
|
@@ -0,0 +1,466 @@
|
|
|
1
|
+
"""ChromaDB vector store operations."""
|
|
2
|
+
|
|
3
|
+
# pysqlite3 patch: some systems' SQLite is too old for ChromaDB (needs >= 3.35).
|
|
4
|
+
# pysqlite3-binary only ships Linux wheels; on macOS the stdlib sqlite3 is new
|
|
5
|
+
# enough, so fall back to it when pysqlite3 is unavailable.
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
try:
|
|
9
|
+
__import__("pysqlite3")
|
|
10
|
+
sys.modules["sqlite3"] = sys.modules.pop("pysqlite3")
|
|
11
|
+
except ModuleNotFoundError:
|
|
12
|
+
pass
|
|
13
|
+
|
|
14
|
+
import os
|
|
15
|
+
|
|
16
|
+
import chromadb
|
|
17
|
+
from chromadb.utils.embedding_functions import OpenAIEmbeddingFunction
|
|
18
|
+
|
|
19
|
+
from .bm25 import BM25Index
|
|
20
|
+
from .config import BM25_DIR, COLLECTION_NAME, SUMMARY_MODEL, VECTORSTORE_DIR, get_vault_path
|
|
21
|
+
from .parser import classify_path, extract_summary, parse_document
|
|
22
|
+
|
|
23
|
+
EMBEDDING_MODEL = "text-embedding-3-small"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _get_embedding_function():
|
|
27
|
+
"""Return OpenAI embedding function."""
|
|
28
|
+
api_key = os.environ.get("OPENAI_API_KEY", "")
|
|
29
|
+
if not api_key:
|
|
30
|
+
print("Error: OPENAI_API_KEY not set.", file=sys.stderr)
|
|
31
|
+
sys.exit(1)
|
|
32
|
+
return OpenAIEmbeddingFunction(model_name=EMBEDDING_MODEL, api_key=api_key)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
_SUMMARY_PROMPT = """為這份技術筆記產生摘要,用於語意搜尋索引。
|
|
36
|
+
|
|
37
|
+
規則:
|
|
38
|
+
1. 用繁體中文寫 2-3 句摘要,描述核心知識
|
|
39
|
+
2. 重要概念同時列出中英文(例如:憑證 certificate)
|
|
40
|
+
3. 最後加「關鍵詞:」列出中英文關鍵詞
|
|
41
|
+
4. 總長度不超過 300 字
|
|
42
|
+
|
|
43
|
+
標題:{title}
|
|
44
|
+
標籤:{tags}
|
|
45
|
+
---
|
|
46
|
+
{body}"""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _generate_summary(title, tags, body):
|
|
50
|
+
"""Generate a bilingual summary using LLM. Falls back to title+tags on failure."""
|
|
51
|
+
try:
|
|
52
|
+
from openai import OpenAI
|
|
53
|
+
|
|
54
|
+
client = OpenAI()
|
|
55
|
+
resp = client.chat.completions.create(
|
|
56
|
+
model=SUMMARY_MODEL,
|
|
57
|
+
messages=[
|
|
58
|
+
{
|
|
59
|
+
"role": "user",
|
|
60
|
+
"content": _SUMMARY_PROMPT.format(title=title, tags=tags, body=body[:3000]),
|
|
61
|
+
}
|
|
62
|
+
],
|
|
63
|
+
max_completion_tokens=400,
|
|
64
|
+
reasoning_effort="none",
|
|
65
|
+
)
|
|
66
|
+
return resp.choices[0].message.content.strip()
|
|
67
|
+
except Exception:
|
|
68
|
+
return f"{title}. Tags: {tags}"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def get_client():
|
|
72
|
+
"""Return a persistent ChromaDB client."""
|
|
73
|
+
VECTORSTORE_DIR.mkdir(parents=True, exist_ok=True)
|
|
74
|
+
return chromadb.PersistentClient(path=str(VECTORSTORE_DIR))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def get_collection(client):
|
|
78
|
+
"""Get or create the cortex collection."""
|
|
79
|
+
return client.get_or_create_collection(
|
|
80
|
+
name=COLLECTION_NAME,
|
|
81
|
+
embedding_function=_get_embedding_function(),
|
|
82
|
+
metadata={"hnsw:space": "cosine"},
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _resolve_repos(doc_type, category, repos_str):
|
|
87
|
+
"""Build repo list from doc type, category, and frontmatter repos field."""
|
|
88
|
+
repos = []
|
|
89
|
+
if doc_type == "project" and category:
|
|
90
|
+
repos.append(category)
|
|
91
|
+
if repos_str:
|
|
92
|
+
for r in repos_str.split(","):
|
|
93
|
+
r = r.strip()
|
|
94
|
+
if r and r not in repos:
|
|
95
|
+
repos.append(r)
|
|
96
|
+
return repos if repos else [""]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _build_metadata(doc_type, category, title, tags, repos_str, status, source_path, repo):
|
|
100
|
+
"""Build metadata dict for a single ChromaDB entry."""
|
|
101
|
+
metadata = {
|
|
102
|
+
"type": doc_type,
|
|
103
|
+
"category": category,
|
|
104
|
+
"title": title,
|
|
105
|
+
"tags": tags,
|
|
106
|
+
"source_path": str(source_path),
|
|
107
|
+
}
|
|
108
|
+
if repo:
|
|
109
|
+
metadata["repo"] = repo
|
|
110
|
+
if repos_str:
|
|
111
|
+
metadata["repos"] = repos_str
|
|
112
|
+
if status:
|
|
113
|
+
metadata["status"] = status
|
|
114
|
+
return metadata
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _delete_stale_entries(collection, rel_path):
|
|
118
|
+
"""Remove all existing entries for a path (base + ::repo variants).
|
|
119
|
+
|
|
120
|
+
This is the critical fix for the stale repo association bug.
|
|
121
|
+
Must be called before inserting new entries for a document.
|
|
122
|
+
"""
|
|
123
|
+
all_ids = collection.get(include=[])["ids"]
|
|
124
|
+
stale = [i for i in all_ids if i == rel_path or i.startswith(f"{rel_path}::")]
|
|
125
|
+
if stale:
|
|
126
|
+
collection.delete(ids=stale)
|
|
127
|
+
return stale
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _base_path(doc_id):
|
|
131
|
+
"""Extract base file path from a doc ID, stripping ::repo and ::summary suffixes."""
|
|
132
|
+
return doc_id.split("::")[0]
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def cmd_status(_args):
|
|
136
|
+
"""Show index health."""
|
|
137
|
+
vault = get_vault_path()
|
|
138
|
+
client = get_client()
|
|
139
|
+
try:
|
|
140
|
+
col = client.get_collection(COLLECTION_NAME)
|
|
141
|
+
total = col.count()
|
|
142
|
+
all_metas = col.get(include=["metadatas"])["metadatas"]
|
|
143
|
+
summaries = sum(1 for m in all_metas if m.get("entry_type") == "summary")
|
|
144
|
+
bodies = total - summaries
|
|
145
|
+
except Exception:
|
|
146
|
+
total = 0
|
|
147
|
+
bodies = 0
|
|
148
|
+
summaries = 0
|
|
149
|
+
|
|
150
|
+
try:
|
|
151
|
+
bm25_index = BM25Index(BM25_DIR)
|
|
152
|
+
bm25_index.load()
|
|
153
|
+
bm25_count = bm25_index.count()
|
|
154
|
+
except Exception:
|
|
155
|
+
bm25_count = 0
|
|
156
|
+
print(f"BM25: {bm25_count} notes")
|
|
157
|
+
if total and not bm25_count:
|
|
158
|
+
print(" WARNING: BM25 index empty — run rebuild", file=sys.stderr)
|
|
159
|
+
print(f"Collection: {COLLECTION_NAME}")
|
|
160
|
+
print(f"Entries: {total} ({bodies} body + {summaries} summary)")
|
|
161
|
+
print(f"Embedding: {EMBEDDING_MODEL} (OpenAI)")
|
|
162
|
+
print(f"Summary: {SUMMARY_MODEL} (OpenAI)")
|
|
163
|
+
print(f"DB path: {VECTORSTORE_DIR}")
|
|
164
|
+
print(f"Vault: {vault}")
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _build_bm25_from_vault(vault):
|
|
168
|
+
"""Build the BM25 index from all Notes/ + Projects/ notes (skip _archive).
|
|
169
|
+
|
|
170
|
+
Returns the note count. Does NOT touch ChromaDB or call any embedding API —
|
|
171
|
+
safe and free to run when only the BM25 index is stale.
|
|
172
|
+
"""
|
|
173
|
+
bm25_docs = []
|
|
174
|
+
for scan_dir in ("Notes", "Projects"):
|
|
175
|
+
scan_path = vault / scan_dir
|
|
176
|
+
if not scan_path.is_dir():
|
|
177
|
+
continue
|
|
178
|
+
for md_file in scan_path.rglob("*.md"):
|
|
179
|
+
rel_path = str(md_file.relative_to(vault))
|
|
180
|
+
if "_archive" in rel_path:
|
|
181
|
+
continue
|
|
182
|
+
text = md_file.read_text(encoding="utf-8", errors="replace")
|
|
183
|
+
fm, body = parse_document(text)
|
|
184
|
+
bm25_docs.append(bm25_doc_from_fields(rel_path, fm, body))
|
|
185
|
+
bm25_index = BM25Index(BM25_DIR)
|
|
186
|
+
bm25_index.build_from_docs(bm25_docs)
|
|
187
|
+
bm25_index.save()
|
|
188
|
+
return bm25_index.count()
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def cmd_rebuild(args):
|
|
192
|
+
"""Rebuild from vault. `--bm25-only` rebuilds just the BM25 index (no re-embed)."""
|
|
193
|
+
vault = get_vault_path()
|
|
194
|
+
|
|
195
|
+
if getattr(args, "bm25_only", False):
|
|
196
|
+
count = _build_bm25_from_vault(vault)
|
|
197
|
+
print(f"BM25: {count} notes indexed (vector store untouched)")
|
|
198
|
+
return
|
|
199
|
+
|
|
200
|
+
client = get_client()
|
|
201
|
+
|
|
202
|
+
try:
|
|
203
|
+
client.delete_collection(COLLECTION_NAME)
|
|
204
|
+
except Exception:
|
|
205
|
+
pass
|
|
206
|
+
|
|
207
|
+
col = get_collection(client)
|
|
208
|
+
scan_dirs = ["Notes", "Projects"]
|
|
209
|
+
doc_count = 0
|
|
210
|
+
|
|
211
|
+
for scan_dir in scan_dirs:
|
|
212
|
+
scan_path = vault / scan_dir
|
|
213
|
+
if not scan_path.is_dir():
|
|
214
|
+
continue
|
|
215
|
+
|
|
216
|
+
for md_file in scan_path.rglob("*.md"):
|
|
217
|
+
rel_path = str(md_file.relative_to(vault))
|
|
218
|
+
if "_archive" in rel_path:
|
|
219
|
+
continue
|
|
220
|
+
|
|
221
|
+
text = md_file.read_text(encoding="utf-8", errors="replace")
|
|
222
|
+
fm, body = parse_document(text)
|
|
223
|
+
|
|
224
|
+
doc_type, category = classify_path(rel_path)
|
|
225
|
+
title = fm.get("title", md_file.stem)
|
|
226
|
+
status = fm.get("status", "")
|
|
227
|
+
tags = fm.get("tags", "")
|
|
228
|
+
repos_str = fm.get("repos", "")
|
|
229
|
+
|
|
230
|
+
repos = _resolve_repos(doc_type, category, repos_str)
|
|
231
|
+
embed_content = f"{title}\n\n{body}".strip()
|
|
232
|
+
if not embed_content:
|
|
233
|
+
continue
|
|
234
|
+
|
|
235
|
+
# Generate bilingual summary
|
|
236
|
+
summary_text = _generate_summary(title, tags, body)
|
|
237
|
+
|
|
238
|
+
for repo in repos:
|
|
239
|
+
doc_id = (
|
|
240
|
+
rel_path
|
|
241
|
+
if not repo or len(repos) == 1
|
|
242
|
+
else f"{rel_path}::{repo}"
|
|
243
|
+
)
|
|
244
|
+
metadata = _build_metadata(
|
|
245
|
+
doc_type, category, title, tags, repos_str, status,
|
|
246
|
+
str(md_file), repo,
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
# Full body vector
|
|
250
|
+
col.upsert(
|
|
251
|
+
ids=[doc_id],
|
|
252
|
+
documents=[embed_content],
|
|
253
|
+
metadatas=[metadata],
|
|
254
|
+
)
|
|
255
|
+
doc_count += 1
|
|
256
|
+
|
|
257
|
+
# Summary vector
|
|
258
|
+
summary_id = f"{doc_id}::summary"
|
|
259
|
+
summary_metadata = {**metadata, "entry_type": "summary"}
|
|
260
|
+
col.upsert(
|
|
261
|
+
ids=[summary_id],
|
|
262
|
+
documents=[summary_text],
|
|
263
|
+
metadatas=[summary_metadata],
|
|
264
|
+
)
|
|
265
|
+
doc_count += 1
|
|
266
|
+
|
|
267
|
+
bm25_count = _build_bm25_from_vault(vault)
|
|
268
|
+
print(f"BM25: {bm25_count} notes indexed")
|
|
269
|
+
print(f"Rebuilt: {doc_count} documents indexed")
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def cmd_upsert(args):
|
|
273
|
+
"""Add or update a single document with stale entry cleanup."""
|
|
274
|
+
vault = get_vault_path()
|
|
275
|
+
rel_path = args.path
|
|
276
|
+
full_path = vault / rel_path
|
|
277
|
+
|
|
278
|
+
if not full_path.exists():
|
|
279
|
+
print(f"Error: {full_path} not found.", file=sys.stderr)
|
|
280
|
+
sys.exit(1)
|
|
281
|
+
|
|
282
|
+
client = get_client()
|
|
283
|
+
col = get_collection(client)
|
|
284
|
+
|
|
285
|
+
# Critical fix: clean stale entries before reinserting
|
|
286
|
+
_delete_stale_entries(col, rel_path)
|
|
287
|
+
|
|
288
|
+
text = full_path.read_text(encoding="utf-8", errors="replace")
|
|
289
|
+
fm, body = parse_document(text)
|
|
290
|
+
|
|
291
|
+
doc_type, category = classify_path(rel_path)
|
|
292
|
+
title = fm.get("title", full_path.stem)
|
|
293
|
+
status = fm.get("status", "")
|
|
294
|
+
tags = fm.get("tags", "")
|
|
295
|
+
repos_str = fm.get("repos", "")
|
|
296
|
+
|
|
297
|
+
repos = _resolve_repos(doc_type, category, repos_str)
|
|
298
|
+
embed_content = f"{title}\n\n{body}".strip()
|
|
299
|
+
|
|
300
|
+
# Generate bilingual summary
|
|
301
|
+
summary_text = _generate_summary(title, tags, body)
|
|
302
|
+
|
|
303
|
+
for repo in repos:
|
|
304
|
+
doc_id = (
|
|
305
|
+
rel_path
|
|
306
|
+
if not repo or len(repos) == 1
|
|
307
|
+
else f"{rel_path}::{repo}"
|
|
308
|
+
)
|
|
309
|
+
metadata = _build_metadata(
|
|
310
|
+
doc_type, category, title, tags, repos_str, status,
|
|
311
|
+
str(full_path), repo,
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
# Full body vector
|
|
315
|
+
col.upsert(
|
|
316
|
+
ids=[doc_id],
|
|
317
|
+
documents=[embed_content],
|
|
318
|
+
metadatas=[metadata],
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
# Summary vector
|
|
322
|
+
summary_id = f"{doc_id}::summary"
|
|
323
|
+
summary_metadata = {**metadata, "entry_type": "summary"}
|
|
324
|
+
col.upsert(
|
|
325
|
+
ids=[summary_id],
|
|
326
|
+
documents=[summary_text],
|
|
327
|
+
metadatas=[summary_metadata],
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
bm25_index = BM25Index(BM25_DIR)
|
|
331
|
+
try:
|
|
332
|
+
bm25_index.load()
|
|
333
|
+
except FileNotFoundError:
|
|
334
|
+
bm25_index.build_from_docs([])
|
|
335
|
+
bm25_index.upsert(bm25_doc_from_fields(rel_path, fm, body))
|
|
336
|
+
bm25_index.save()
|
|
337
|
+
print(f"Upserted: {rel_path}")
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def cmd_delete(args):
|
|
341
|
+
"""Remove a document from the index."""
|
|
342
|
+
client = get_client()
|
|
343
|
+
col = get_collection(client)
|
|
344
|
+
rel_path = args.path
|
|
345
|
+
|
|
346
|
+
stale = _delete_stale_entries(col, rel_path)
|
|
347
|
+
if not stale:
|
|
348
|
+
print(f"Not found: {rel_path}", file=sys.stderr)
|
|
349
|
+
sys.exit(1)
|
|
350
|
+
|
|
351
|
+
bm25_index = BM25Index(BM25_DIR)
|
|
352
|
+
try:
|
|
353
|
+
bm25_index.load()
|
|
354
|
+
bm25_index.delete(rel_path)
|
|
355
|
+
bm25_index.save()
|
|
356
|
+
except FileNotFoundError:
|
|
357
|
+
pass
|
|
358
|
+
|
|
359
|
+
print(f"Deleted: {len(stale)} entries for {rel_path}")
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _build_where(repo=None, type=None, category=None):
|
|
363
|
+
clauses = []
|
|
364
|
+
if repo:
|
|
365
|
+
# Notes/ are cross-repo by design and must always pass the repo filter.
|
|
366
|
+
# See docs/specs/2026-05-27-distill-dedup-repo-filter-blindspot.md.
|
|
367
|
+
clauses.append({"$or": [{"repo": repo}, {"type": "note"}]})
|
|
368
|
+
if type:
|
|
369
|
+
clauses.append({"type": type})
|
|
370
|
+
if category:
|
|
371
|
+
clauses.append({"category": category})
|
|
372
|
+
if len(clauses) == 1:
|
|
373
|
+
return clauses[0]
|
|
374
|
+
if len(clauses) > 1:
|
|
375
|
+
return {"$and": clauses}
|
|
376
|
+
return None
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def bm25_doc_from_fields(rel_path, fm, body):
|
|
380
|
+
"""Build a BM25 doc record dict from a parsed note."""
|
|
381
|
+
doc_type, category = classify_path(rel_path)
|
|
382
|
+
repos_str = fm.get("repos", "")
|
|
383
|
+
repos = [r.strip() for r in repos_str.split(",") if r.strip()] if repos_str else []
|
|
384
|
+
if doc_type == "project" and category and category not in repos:
|
|
385
|
+
repos.insert(0, category)
|
|
386
|
+
return {
|
|
387
|
+
"id": rel_path,
|
|
388
|
+
"title": fm.get("title", rel_path.rsplit("/", 1)[-1].removesuffix(".md")),
|
|
389
|
+
"body": body,
|
|
390
|
+
"summary": extract_summary(body),
|
|
391
|
+
"tags": fm.get("tags", ""),
|
|
392
|
+
"repos": repos,
|
|
393
|
+
"type": doc_type,
|
|
394
|
+
"category": category,
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def vector_stream(query, n, where=None):
|
|
399
|
+
"""Vector retrieval stream: dedup by base path, return display dicts sorted desc.
|
|
400
|
+
|
|
401
|
+
Returns the full deduped list (not sliced to n) so the fusion layer can rank it.
|
|
402
|
+
"""
|
|
403
|
+
from pathlib import Path
|
|
404
|
+
|
|
405
|
+
client = get_client()
|
|
406
|
+
col = get_collection(client)
|
|
407
|
+
vault = get_vault_path()
|
|
408
|
+
|
|
409
|
+
kwargs = {
|
|
410
|
+
"query_texts": [query],
|
|
411
|
+
"n_results": n * 3,
|
|
412
|
+
"include": ["documents", "metadatas", "distances"],
|
|
413
|
+
}
|
|
414
|
+
if where:
|
|
415
|
+
kwargs["where"] = where
|
|
416
|
+
|
|
417
|
+
results = col.query(**kwargs)
|
|
418
|
+
docs = results["documents"][0]
|
|
419
|
+
metas = results["metadatas"][0]
|
|
420
|
+
dists = results["distances"][0]
|
|
421
|
+
|
|
422
|
+
seen = {}
|
|
423
|
+
for doc, meta, dist in zip(docs, metas, dists):
|
|
424
|
+
score = round(1 - dist, 4)
|
|
425
|
+
source = meta.get("source_path", "")
|
|
426
|
+
try:
|
|
427
|
+
rel_id = str(Path(source).relative_to(vault))
|
|
428
|
+
except (ValueError, TypeError):
|
|
429
|
+
rel_id = source
|
|
430
|
+
base = _base_path(rel_id)
|
|
431
|
+
if base not in seen or score > seen[base]["score"]:
|
|
432
|
+
seen[base] = {
|
|
433
|
+
"id": base,
|
|
434
|
+
"score": score,
|
|
435
|
+
"title": meta.get("title", ""),
|
|
436
|
+
"type": meta.get("type", ""),
|
|
437
|
+
"repo": meta.get("repo", ""),
|
|
438
|
+
"category": meta.get("category", ""),
|
|
439
|
+
"tags": meta.get("tags", ""),
|
|
440
|
+
"summary": extract_summary(doc),
|
|
441
|
+
}
|
|
442
|
+
return sorted(seen.values(), key=lambda x: -x["score"])
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def cmd_search(args):
|
|
446
|
+
"""Hybrid search across the vault (vector + BM25, RRF-fused)."""
|
|
447
|
+
import json
|
|
448
|
+
|
|
449
|
+
from . import fusion
|
|
450
|
+
|
|
451
|
+
where = _build_where(
|
|
452
|
+
repo=getattr(args, "repo", None),
|
|
453
|
+
type=getattr(args, "type", None),
|
|
454
|
+
category=getattr(args, "category", None),
|
|
455
|
+
)
|
|
456
|
+
results = fusion.search(
|
|
457
|
+
args.query,
|
|
458
|
+
n=args.n or 5,
|
|
459
|
+
where=where,
|
|
460
|
+
use_bm25=not getattr(args, "no_bm25", False),
|
|
461
|
+
use_vector=not getattr(args, "no_vector", False),
|
|
462
|
+
rerank=getattr(args, "rerank", False) or None,
|
|
463
|
+
graph=getattr(args, "graph", False) or None,
|
|
464
|
+
)
|
|
465
|
+
for entry in results:
|
|
466
|
+
print(json.dumps(entry, ensure_ascii=False))
|
cortex_vec/synonyms.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Static synonym groups (common zh/en tech terms).
|
|
2
|
+
|
|
3
|
+
Groups are written in human-readable form; at first use each term is run
|
|
4
|
+
through the same tokenizer as the index/query, so the synonym lookup lives in
|
|
5
|
+
the SAME token space as BM25 (lowercased, stemmed, jieba-segmented).
|
|
6
|
+
"""
|
|
7
|
+
from .tokenize import tokenize
|
|
8
|
+
|
|
9
|
+
SYNONYM_GROUPS = [
|
|
10
|
+
["套件", "package"],
|
|
11
|
+
["憑證", "certificate", "cert", "tls", "ssl"],
|
|
12
|
+
["週報", "weekly report"],
|
|
13
|
+
["登入", "login", "signin", "authentication", "auth"],
|
|
14
|
+
["記憶體", "memory", "ram"],
|
|
15
|
+
["效能", "performance", "perf"],
|
|
16
|
+
["編譯", "build", "compile"],
|
|
17
|
+
["測試", "test", "unittest"],
|
|
18
|
+
["設定", "config", "configuration", "設定檔"],
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
_index = None # token -> set of synonym tokens (built lazily)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _build_index():
|
|
25
|
+
global _index
|
|
26
|
+
_index = {}
|
|
27
|
+
for group in SYNONYM_GROUPS:
|
|
28
|
+
group_tokens = set()
|
|
29
|
+
for term in group:
|
|
30
|
+
group_tokens.update(tokenize(term))
|
|
31
|
+
for tok in group_tokens:
|
|
32
|
+
_index.setdefault(tok, set()).update(group_tokens - {tok})
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def synonyms_for(tokens):
|
|
36
|
+
"""Return extra synonym tokens for the given query tokens (sorted, originals excluded)."""
|
|
37
|
+
if _index is None:
|
|
38
|
+
_build_index()
|
|
39
|
+
originals = set(tokens)
|
|
40
|
+
extra = set()
|
|
41
|
+
for tok in originals:
|
|
42
|
+
extra.update(_index.get(tok, set()))
|
|
43
|
+
return sorted(extra - originals)
|
cortex_vec/tokenize.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""CJK-aware tokenizer for BM25.
|
|
2
|
+
|
|
3
|
+
Splits Han runs with jieba (soft-falls to whole-run if jieba missing),
|
|
4
|
+
stems Latin words with a Porter stemmer, and preserves Latin word
|
|
5
|
+
boundaries embedded inside CJK text (e.g. "機器學習ML技術").
|
|
6
|
+
"""
|
|
7
|
+
import re
|
|
8
|
+
import sys
|
|
9
|
+
|
|
10
|
+
# Han, Hiragana/Katakana, Hangul ranges. cortex content is zh+en; non-Han CJK
|
|
11
|
+
# runs are kept whole (jieba targets Han).
|
|
12
|
+
CJK_RUN_RE = re.compile(r"[㐀-鿿-ヿ가-]+")
|
|
13
|
+
HAN_RE = re.compile(r"[㐀-鿿]")
|
|
14
|
+
# Keep alphanumerics, CJK, and path-ish chars; everything else -> space.
|
|
15
|
+
_CLEAN_RE = re.compile(r"[^\w\s/.\\\-㐀-鿿-ヿ가-]", re.UNICODE)
|
|
16
|
+
|
|
17
|
+
_stemmer = None
|
|
18
|
+
_jieba_ok = None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _stem(word):
|
|
22
|
+
global _stemmer
|
|
23
|
+
if _stemmer is None:
|
|
24
|
+
import snowballstemmer
|
|
25
|
+
_stemmer = snowballstemmer.stemmer("porter")
|
|
26
|
+
return _stemmer.stemWord(word)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _seg_han(run):
|
|
30
|
+
global _jieba_ok
|
|
31
|
+
if _jieba_ok is None:
|
|
32
|
+
try:
|
|
33
|
+
import jieba # noqa: F401
|
|
34
|
+
_jieba_ok = True
|
|
35
|
+
except Exception:
|
|
36
|
+
_jieba_ok = False
|
|
37
|
+
print("cortex-vec: jieba not installed; CJK runs kept whole "
|
|
38
|
+
"(install jieba for word-level CJK recall)", file=sys.stderr)
|
|
39
|
+
if not _jieba_ok:
|
|
40
|
+
return [run]
|
|
41
|
+
import jieba
|
|
42
|
+
return [w for w in jieba.lcut(run, HMM=True) if w.strip()]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _segment_token(token):
|
|
46
|
+
"""Split one whitespace-delimited token that contains CJK into parts,
|
|
47
|
+
preserving embedded Latin runs."""
|
|
48
|
+
parts = []
|
|
49
|
+
cursor = 0
|
|
50
|
+
for m in CJK_RUN_RE.finditer(token):
|
|
51
|
+
if m.start() > cursor:
|
|
52
|
+
latin = token[cursor:m.start()].strip()
|
|
53
|
+
if latin:
|
|
54
|
+
parts.append(_stem(latin))
|
|
55
|
+
run = m.group()
|
|
56
|
+
if HAN_RE.search(run):
|
|
57
|
+
parts.extend(_seg_han(run))
|
|
58
|
+
else:
|
|
59
|
+
parts.append(run)
|
|
60
|
+
cursor = m.end()
|
|
61
|
+
if cursor < len(token):
|
|
62
|
+
latin = token[cursor:].strip()
|
|
63
|
+
if latin:
|
|
64
|
+
parts.append(_stem(latin))
|
|
65
|
+
return parts
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def tokenize(text):
|
|
69
|
+
"""Return a list of lowercased tokens for BM25 indexing/querying."""
|
|
70
|
+
cleaned = _CLEAN_RE.sub(" ", text.lower())
|
|
71
|
+
out = []
|
|
72
|
+
for raw in cleaned.split():
|
|
73
|
+
if not raw:
|
|
74
|
+
continue
|
|
75
|
+
if CJK_RUN_RE.search(raw):
|
|
76
|
+
out.extend(t for t in _segment_token(raw) if t)
|
|
77
|
+
else:
|
|
78
|
+
out.append(_stem(raw))
|
|
79
|
+
return out
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cortex-vec
|
|
3
|
+
Version: 0.7.0
|
|
4
|
+
Summary: Vector store CLI for the cortex vault — hybrid BM25 + embedding retrieval over a markdown knowledge vault
|
|
5
|
+
Author: XBlueSky
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://cortexes.pages.dev
|
|
8
|
+
Project-URL: Repository, https://github.com/XBlueSky/cortexes
|
|
9
|
+
Project-URL: Changelog, https://github.com/XBlueSky/cortexes/blob/plugin/CHANGELOG.md
|
|
10
|
+
Keywords: knowledge-base,semantic-search,bm25,chromadb,claude-code
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Topic :: Text Processing :: Indexing
|
|
17
|
+
Requires-Python: >=3.10
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
License-File: NOTICE
|
|
21
|
+
Requires-Dist: chromadb
|
|
22
|
+
Requires-Dist: openai
|
|
23
|
+
Requires-Dist: python-frontmatter
|
|
24
|
+
Requires-Dist: pysqlite3-binary; sys_platform == "linux"
|
|
25
|
+
Requires-Dist: rank-bm25
|
|
26
|
+
Requires-Dist: jieba
|
|
27
|
+
Requires-Dist: snowballstemmer
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# cortex-vec
|
|
33
|
+
|
|
34
|
+
Vector store CLI for the [cortex](https://github.com/XBlueSky/cortexes)
|
|
35
|
+
knowledge vault — hybrid retrieval over a plain-markdown vault, fusing a
|
|
36
|
+
persistent BM25 index with OpenAI-embedding vector search (ChromaDB) via
|
|
37
|
+
Reciprocal Rank Fusion. Handles mixed Chinese/English content.
|
|
38
|
+
|
|
39
|
+
The vault is the source of truth (markdown + git); everything this tool
|
|
40
|
+
builds is a rebuildable derived index.
|
|
41
|
+
|
|
42
|
+
## Install
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
# Recommended: isolated tool install via uv
|
|
46
|
+
uv tool install cortex-vec
|
|
47
|
+
|
|
48
|
+
# Or with pip
|
|
49
|
+
pip install cortex-vec
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Usage
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
cortex-vec rebuild # full rebuild (embeddings, needs OPENAI_API_KEY)
|
|
56
|
+
cortex-vec rebuild --bm25-only # rebuild just the BM25 index — free, no API key
|
|
57
|
+
cortex-vec search "nginx certificate" --n 5
|
|
58
|
+
cortex-vec status # index health: vector + BM25 entry counts
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Configuration lives in `~/.cortex/config.json` (created by the cortex
|
|
62
|
+
plugin's `/cortex:genesis` command). `OPENAI_API_KEY` is required for
|
|
63
|
+
embeddings; without it, `search` automatically degrades to BM25-only.
|
|
64
|
+
|
|
65
|
+
This package is the retrieval backend of the
|
|
66
|
+
[Cortex Claude Code plugin](https://github.com/XBlueSky/cortexes) — session
|
|
67
|
+
recording, memory distillation, and indexed retrieval for Claude Code. See
|
|
68
|
+
the repository for the full documentation.
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
cortex_vec/__init__.py,sha256=qNFZ6WC6uxC4s2Pj699SOe2PQti61950SnaKrNSvVb8,83
|
|
2
|
+
cortex_vec/bm25.py,sha256=b0ftAf9SnWwes3gGhZrFOEPJoxHlLRdlHSo5vJftHP4,4595
|
|
3
|
+
cortex_vec/cli.py,sha256=KRsTBaYQ6UJZyIWL4h-yp0ZRGyNDxCoZSKLbQ8PPrTg,8124
|
|
4
|
+
cortex_vec/config.py,sha256=DH0xfVrPbaaoCbJj5HU-KcRwr_2Ay8S3_P9SnhLokKU,2374
|
|
5
|
+
cortex_vec/distill_plan.py,sha256=-BlRzvgjaj_-krn4CwBNglCniWNzuwuSeEe87sYOWfI,16436
|
|
6
|
+
cortex_vec/distill_queue.py,sha256=kpycbKhMivx65MdTEtSewvq9_vylqS5q43dbwxt-SOo,6066
|
|
7
|
+
cortex_vec/fusion.py,sha256=6DhgdOcF5knHnjA5VTI06lMT5JmH61hT_29oJKj3fJw,6471
|
|
8
|
+
cortex_vec/graph.py,sha256=GJsk2-ThMYnli2EN29_M4C8BxunQZpIgMKBIlK9IjjQ,3911
|
|
9
|
+
cortex_vec/parser.py,sha256=Q8NAfFR6dAqfhX1MKnzfZxqf2FoM96Z1fhAOsJxCrZc,2203
|
|
10
|
+
cortex_vec/raw_map.py,sha256=MYS33qJzstRus6EKFK2OMKMfRbmBd3Er8sGHzLjrNfo,5914
|
|
11
|
+
cortex_vec/raw_page.py,sha256=tRK4sIzZ2Dx6SPIaLyRJakxmhTZccnV3DXEGFGhGRS4,2204
|
|
12
|
+
cortex_vec/raw_source.py,sha256=9Z1RTjgIikOmp8Alba_yfoKJz338DBwWXpxBoFDm6LY,10958
|
|
13
|
+
cortex_vec/raw_span.py,sha256=GX0kY8M2rvQch-1GkrRn2_GKHn2UP1S-WlBduq-W2sk,3563
|
|
14
|
+
cortex_vec/raw_view.py,sha256=lK0qdfKFTGAG-HJrIr591YQr_vzU79-i-pa45je2DOI,9748
|
|
15
|
+
cortex_vec/reclaim.py,sha256=SDtxkCxESKUXIo7xeofXg78Ph3aldea0uiNPY5f6jD8,5034
|
|
16
|
+
cortex_vec/rerank.py,sha256=3XuuNjkUnpVWDaKovSddH2RvLGpAkI7WQAZekC5_CIE,1941
|
|
17
|
+
cortex_vec/store.py,sha256=dqirQtMS2ozaQN2LyMvdvhys2u8e5NY34TzZuVdQmgA,14545
|
|
18
|
+
cortex_vec/synonyms.py,sha256=5E_J-inW4TYwQV1eLmUP4Q47nQC3zP0-2NGe7SUPXfU,1391
|
|
19
|
+
cortex_vec/tokenize.py,sha256=PAbQ2jDms9bY0ZRPL2MXrdxOiep-gXVTeEwkiT9Wqhg,2371
|
|
20
|
+
cortex_vec/eval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
|
+
cortex_vec/eval/adapters.py,sha256=oo8zAisyRcwnmpYqMcLqNdMXvD5iPF5ymxrsQzYQxC0,2211
|
|
22
|
+
cortex_vec/eval/corpus.py,sha256=O9BxCIFN12ivna8YHeSKrwSrY3GpZq5S1O2BlkJtXd4,1238
|
|
23
|
+
cortex_vec/eval/report.py,sha256=EbCdcuEUzMlmQOwxfCamq9wJ5dk1xJk8ZbKwO4E28V4,1596
|
|
24
|
+
cortex_vec/eval/run.py,sha256=Wj12-2mIGRyE3HBZVCMabE9a6jb2iJOcDnYW0dy1PyQ,4438
|
|
25
|
+
cortex_vec/eval/score.py,sha256=GE90OTqYyVWHAjRaQ0J1azGLaSVtqxgLtarU6QovqNA,1896
|
|
26
|
+
cortex_vec-0.7.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
27
|
+
cortex_vec-0.7.0.dist-info/licenses/NOTICE,sha256=qB2siBf9qWqLWILm0GqEsaUW4yhid19vl2tQgsAhJFg,146
|
|
28
|
+
cortex_vec-0.7.0.dist-info/METADATA,sha256=hhXmM0T5lR6cZR58W1zpjpGo-cv-qI6KHNMUOMNhpj4,2464
|
|
29
|
+
cortex_vec-0.7.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
|
|
30
|
+
cortex_vec-0.7.0.dist-info/entry_points.txt,sha256=P_nsYu2ZVqlQxyAW2Y5l7HTSHiaqw5m1z1oYRp_JGVo,51
|
|
31
|
+
cortex_vec-0.7.0.dist-info/top_level.txt,sha256=_LJ6RWHFOnKzYqbT2W9HvJa2fkEsiGCZ9YxCCQKfHAQ,11
|
|
32
|
+
cortex_vec-0.7.0.dist-info/RECORD,,
|