cortex-vec 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
cortex_vec/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """cortex-vec — Vector store CLI for the cortex vault."""
2
+
3
+ __version__ = "0.3.0"
cortex_vec/bm25.py ADDED
@@ -0,0 +1,129 @@
1
+ """Persistent BM25 index over vault notes (one entry per note base path)."""
2
+ import pickle
3
+ from pathlib import Path
4
+
5
+ from rank_bm25 import BM25Okapi
6
+
7
+ from .tokenize import tokenize
8
+
9
+ # Display/metadata fields carried per doc (everything except the raw body).
10
+ _META_FIELDS = ("id", "title", "summary", "tags", "repos", "type", "category")
11
+
12
+
13
+ def _doc_record(doc):
14
+ """Normalize an input doc into the stored record (tokens + metadata)."""
15
+ text = f"{doc.get('title', '')}\n\n{doc.get('body', '')}".strip()
16
+ rec = {f: doc.get(f) for f in _META_FIELDS}
17
+ rec["repos"] = list(doc.get("repos") or [])
18
+ rec["tokens"] = tokenize(text)
19
+ return rec
20
+
21
+
22
+ def _matches(rec, where):
23
+ if not where:
24
+ return True
25
+ # Notes/ are cross-repo by design and must always pass the repo filter.
26
+ # See docs/specs/2026-05-27-distill-dedup-repo-filter-blindspot.md.
27
+ if ("repo" in where and rec.get("type") != "note"
28
+ and where["repo"] not in rec.get("repos", [])):
29
+ return False
30
+ if "type" in where and rec.get("type") != where["type"]:
31
+ return False
32
+ if "category" in where and rec.get("category") != where["category"]:
33
+ return False
34
+ return True
35
+
36
+
37
+ class BM25Index:
38
+ """BM25 index persisted as a pickle of doc records; BM25Okapi rebuilt on load."""
39
+
40
+ def __init__(self, dir_path):
41
+ self.dir = Path(dir_path)
42
+ self.docs = [] # list of stored records
43
+ self._bm25 = None # BM25Okapi, lazily (re)built
44
+
45
+ @property
46
+ def _file(self):
47
+ return self.dir / "index.pkl"
48
+
49
+ def count(self):
50
+ return len(self.docs)
51
+
52
+ def _reindex(self):
53
+ corpus = [d["tokens"] for d in self.docs] or [[""]]
54
+ self._bm25 = BM25Okapi(corpus)
55
+
56
+ def build_from_docs(self, docs):
57
+ self.docs = [_doc_record(d) for d in docs]
58
+ self._reindex()
59
+
60
+ def upsert(self, doc):
61
+ rec = _doc_record(doc)
62
+ self.docs = [d for d in self.docs if d["id"] != rec["id"]]
63
+ self.docs.append(rec)
64
+ self._reindex()
65
+
66
+ def delete(self, base_path):
67
+ before = len(self.docs)
68
+ self.docs = [d for d in self.docs if d["id"] != base_path]
69
+ if len(self.docs) != before:
70
+ self._reindex()
71
+ return before - len(self.docs)
72
+
73
+ def save(self):
74
+ self.dir.mkdir(parents=True, exist_ok=True)
75
+ with open(self._file, "wb") as f:
76
+ pickle.dump(self.docs, f)
77
+
78
+ def load(self):
79
+ if not self._file.exists():
80
+ raise FileNotFoundError(f"BM25 index not found at {self._file}; run rebuild")
81
+ with open(self._file, "rb") as f:
82
+ self.docs = pickle.load(f)
83
+ self._reindex()
84
+
85
+ def search(self, query, n=5, where=None, synonym_weight=0.0):
86
+ """Return up to n display dicts, best-first, filtered by `where`.
87
+
88
+ If synonym_weight > 0, synonym tokens (from synonyms.synonyms_for) are
89
+ scored separately and added at the given weight, and are also admitted
90
+ to the token-overlap gate so synonym-only matches can surface.
91
+ """
92
+ if not self.docs:
93
+ return []
94
+ if self._bm25 is None:
95
+ self._reindex()
96
+ q_toks = tokenize(query)
97
+ q_tokens = set(q_toks)
98
+ scores = self._bm25.get_scores(q_toks)
99
+ if synonym_weight > 0:
100
+ from .synonyms import synonyms_for
101
+ syn_toks = synonyms_for(q_toks)
102
+ if syn_toks:
103
+ scores = scores + synonym_weight * self._bm25.get_scores(syn_toks)
104
+ q_tokens |= set(syn_toks)
105
+ ranked = sorted(
106
+ zip(self.docs, scores), key=lambda pair: pair[1], reverse=True
107
+ )
108
+ out = []
109
+ for rec, sc in ranked:
110
+ # Relevance gate by token overlap: BM25Okapi IDF can be <= 0 for terms
111
+ # that appear in >= half the corpus (and is degenerate on tiny corpora),
112
+ # which would zero out genuine matches. Token overlap is the reliable match test.
113
+ if not (set(rec["tokens"]) & q_tokens):
114
+ continue
115
+ if not _matches(rec, where):
116
+ continue
117
+ out.append({
118
+ "id": rec["id"],
119
+ "score": float(sc),
120
+ "title": rec.get("title") or "",
121
+ "type": rec.get("type") or "",
122
+ "repo": (rec.get("repos") or [""])[0],
123
+ "category": rec.get("category") or "",
124
+ "tags": rec.get("tags") or "",
125
+ "summary": rec.get("summary") or "",
126
+ })
127
+ if len(out) >= n:
128
+ break
129
+ return out
cortex_vec/cli.py ADDED
@@ -0,0 +1,191 @@
1
+ """CLI entry point for cortex-vec."""
2
+
3
+ import argparse
4
+ import sys
5
+
6
+
7
+ def main():
8
+ parser = argparse.ArgumentParser(
9
+ description="cortex-vec — Vector store CLI for cortex vault"
10
+ )
11
+ sub = parser.add_subparsers(dest="command")
12
+
13
+ sub.add_parser("status", help="Show index health")
14
+ p_rebuild = sub.add_parser("rebuild", help="Full rebuild from vault")
15
+ p_rebuild.add_argument(
16
+ "--bm25-only",
17
+ action="store_true",
18
+ help="Rebuild only the BM25 index from the vault (no re-embedding, free)",
19
+ )
20
+
21
+ p_upsert = sub.add_parser("upsert", help="Add/update a document")
22
+ p_upsert.add_argument("path", help="Relative path from vault root")
23
+
24
+ p_delete = sub.add_parser("delete", help="Remove a document")
25
+ p_delete.add_argument("path", help="Relative path from vault root")
26
+
27
+ p_search = sub.add_parser("search", help="Semantic search")
28
+ p_search.add_argument("query", help="Search query text")
29
+ p_search.add_argument("--repo", help="Filter by repo")
30
+ p_search.add_argument("--type", help="Filter by type (note/project/weekly)")
31
+ p_search.add_argument("--category", help="Filter by category")
32
+ p_search.add_argument("--n", type=int, default=5, help="Number of results")
33
+ p_search.add_argument("--no-bm25", action="store_true", help="Disable BM25 stream")
34
+ p_search.add_argument("--no-vector", action="store_true", help="Disable vector stream")
35
+ p_search.add_argument("--rerank", action="store_true", help="Enable LLM rerank of top results")
36
+ p_search.add_argument("--graph", action="store_true", help="Enable wikilink graph-boost")
37
+
38
+ p_eval = sub.add_parser("eval", help="Run retrieval eval / propose queries")
39
+ p_eval.add_argument("action", choices=["run", "propose"], help="run scorecard or propose queries")
40
+ p_eval.add_argument("--queries", required=True, help="Path to queries.jsonl")
41
+ p_eval.add_argument("--adapters", default="grep,vector,bm25,hybrid",
42
+ help="Comma-separated adapters to run")
43
+ p_eval.add_argument("--k", type=int, default=5, help="Cutoff K")
44
+ p_eval.add_argument("--out", help="Scorecard output path (markdown)")
45
+
46
+ # Trailer-anchored vault-maintenance queries (no vector store needed).
47
+ p_dq = sub.add_parser(
48
+ "distill-queue", help="List Raw files awaiting distillation"
49
+ )
50
+ p_dq.add_argument("--root", help="Raw directory (default: <vault>/Raw)")
51
+ p_dq.add_argument("--stat", action="store_true",
52
+ help="Show per-level projected sizes instead of paths")
53
+ p_dq.add_argument("--json", action="store_true",
54
+ help="With --stat: emit JSON rows")
55
+ p_bq = sub.add_parser(
56
+ "broadcast-queue", help="List distilled Raw eligible for broadcast"
57
+ )
58
+ p_bq.add_argument("--root", help="Raw directory (default: <vault>/Raw)")
59
+ p_rc = sub.add_parser(
60
+ "reclaim-superseded",
61
+ help="List (or remove) undistilled Raw snapshots that a longer "
62
+ "recording of the same session already covers",
63
+ )
64
+ p_rc.add_argument("--root", help="Raw directory (default: <vault>/Raw)")
65
+ p_rc.add_argument("--keep", help="Survivor Raw (default: scan queue pairwise)")
66
+ p_rc.add_argument("--apply", action="store_true",
67
+ help="Remove them (git rm, staged for the vault commit)")
68
+ p_rc.add_argument("--vault", help="Vault git root (default: config vault_path)")
69
+ p_rs = sub.add_parser(
70
+ "raw-state", help="Classify a single Raw file's distilled state"
71
+ )
72
+ p_rs.add_argument("path", help="Path to a Raw .md file")
73
+ p_rv = sub.add_parser(
74
+ "raw-view", help="Projected (budget-bounded) view of a Raw file"
75
+ )
76
+ p_rv.add_argument("path", help="Path to a Raw .md file")
77
+ p_rv.add_argument("--budget", type=int, help="Max output chars (default: config)")
78
+ p_rv.add_argument("--level", choices=["L0", "L1", "L2", "L3"],
79
+ help="Force a level (default: auto by budget)")
80
+ p_rv.add_argument("--stat", action="store_true",
81
+ help="Emit per-level size JSON instead of the view")
82
+
83
+ p_rm = sub.add_parser(
84
+ "raw-map", help="Bounded navigation cards over a Raw (map-first)"
85
+ )
86
+ p_rm.add_argument("path", help="Path to a Raw .md file")
87
+ p_rm.add_argument("--plan-id", required=True, dest="plan_id")
88
+ p_rm.add_argument("--cursor", help="Continuation cursor from a prior page")
89
+ p_rm.add_argument("--max-chars", type=int, dest="max_chars",
90
+ help="Page stdout cap (default: config, 12000)")
91
+ p_rm.add_argument("--find", help="Exact literal to locate in the Raw")
92
+
93
+ p_rsp = sub.add_parser(
94
+ "raw-span", help="Bounded original-source page of one span"
95
+ )
96
+ p_rsp.add_argument("path", help="Path to a Raw .md file")
97
+ p_rsp.add_argument("--plan-id", required=True, dest="plan_id")
98
+ p_rsp.add_argument("--span-id", type=int, dest="span_id")
99
+ p_rsp.add_argument("--cursor")
100
+ p_rsp.add_argument("--max-chars", type=int, dest="max_chars")
101
+
102
+ p_dp = sub.add_parser(
103
+ "distill-plan", help="Single-raw distill plan lifecycle"
104
+ )
105
+ p_dp.add_argument("action", choices=["start", "status", "resume",
106
+ "evidence-add", "seal", "complete",
107
+ "list", "clear"])
108
+ p_dp.add_argument("path", nargs="?", help="Raw path (start only)")
109
+ p_dp.add_argument("--plan-id", dest="plan_id")
110
+ p_dp.add_argument("--page-budget", type=int, dest="page_budget")
111
+ p_dp.add_argument("--session-budget", type=int, dest="session_budget")
112
+ p_dp.add_argument("--char-start", type=int, dest="char_start")
113
+ p_dp.add_argument("--char-end", type=int, dest="char_end")
114
+ p_dp.add_argument("--label", default="")
115
+ p_dp.add_argument("--expected-outcome", dest="expected_outcome",
116
+ choices=["new", "pending-merge", "skip-routine",
117
+ "no-insight"])
118
+ p_dp.add_argument("--new-session", action="store_true",
119
+ dest="new_session")
120
+
121
+ args = parser.parse_args()
122
+
123
+ if not args.command:
124
+ parser.print_help()
125
+ sys.exit(1)
126
+
127
+ # Fast path: trailer-anchored queue scans skip the heavy store import
128
+ # (chromadb ~2.7s). They read only each Raw's last non-empty line.
129
+ if args.command in ("distill-queue", "broadcast-queue", "raw-state",
130
+ "raw-view", "raw-map", "raw-span", "distill-plan",
131
+ "reclaim-superseded"):
132
+ from . import distill_queue as dq
133
+
134
+ if args.command == "reclaim-superseded":
135
+ from . import reclaim as rcl
136
+
137
+ rcl.dispatch(args)
138
+ return
139
+ if args.command == "raw-state":
140
+ dq.dispatch_raw_state(args.path)
141
+ return
142
+ if args.command == "raw-view":
143
+ from . import raw_view as rv
144
+
145
+ rv.dispatch_raw_view(args)
146
+ return
147
+ if args.command == "raw-map":
148
+ from . import raw_map as rm
149
+
150
+ rm.dispatch(args)
151
+ return
152
+ if args.command == "raw-span":
153
+ from . import raw_span as rsp
154
+
155
+ rsp.dispatch(args)
156
+ return
157
+ if args.command == "distill-plan":
158
+ from . import distill_plan as dpl
159
+
160
+ dpl.dispatch(args)
161
+ return
162
+ from .config import get_vault_path
163
+
164
+ root = args.root or (get_vault_path() / "Raw")
165
+ if args.command == "distill-queue":
166
+ dq.dispatch_distill_queue(root, stat=args.stat, as_json=args.json)
167
+ else:
168
+ dq.dispatch_broadcast_queue(root)
169
+ return
170
+
171
+ # Lazy import: chromadb (~2.7s) only loaded when store is needed
172
+ from . import store
173
+
174
+ def _dispatch_eval(args):
175
+ from .eval import run
176
+ run.dispatch(args)
177
+
178
+ commands = {
179
+ "status": store.cmd_status,
180
+ "rebuild": store.cmd_rebuild,
181
+ "upsert": store.cmd_upsert,
182
+ "delete": store.cmd_delete,
183
+ "search": store.cmd_search,
184
+ "eval": _dispatch_eval,
185
+ }
186
+
187
+ commands[args.command](args)
188
+
189
+
190
+ if __name__ == "__main__":
191
+ main()
cortex_vec/config.py ADDED
@@ -0,0 +1,82 @@
1
+ """Configuration loading and constants."""
2
+
3
+ import json
4
+ import sys
5
+ from pathlib import Path
6
+
7
+ CORTEX_CONFIG = Path.home() / ".cortex" / "config.json"
8
+ VECTORSTORE_DIR = Path.home() / ".cortex" / "vectorstore"
9
+ COLLECTION_NAME = "cortex"
10
+ SUMMARY_MODEL = "gpt-5.4-mini"
11
+ BM25_DIR = Path.home() / ".cortex" / "bm25"
12
+
13
+ _RETRIEVAL_DEFAULTS = {
14
+ # Plan 1
15
+ "rrf_k": 60,
16
+ "w_bm25": 0.4,
17
+ "w_vec": 0.6,
18
+ "max_per_repo": 0,
19
+ # Plan 2
20
+ "synonym_weight": 0.0, # 0 = synonym expansion off
21
+ "graph": False,
22
+ "graph_hops": 1,
23
+ "w_graph": 0.3, # graph stream weight in RRF (rank-based, not additive)
24
+ "graph_top_k": 5, # how many top hits to expand wikilink-neighbors from
25
+ "rerank": False,
26
+ "rerank_model": "gpt-5.4-mini",
27
+ "rerank_window": 15,
28
+ }
29
+
30
+
31
+ def load_config():
32
+ """Load ~/.cortex/config.json. Exit with error if not found."""
33
+ if not CORTEX_CONFIG.exists():
34
+ print(
35
+ "Error: ~/.cortex/config.json not found. Run /cortex:genesis first.",
36
+ file=sys.stderr,
37
+ )
38
+ sys.exit(1)
39
+ with open(CORTEX_CONFIG) as f:
40
+ return json.load(f)
41
+
42
+
43
+ def get_vault_path():
44
+ """Return the vault path from config. Exit with error if invalid."""
45
+ cfg = load_config()
46
+ vault = cfg.get("vault_path", "")
47
+ if not vault or not Path(vault).is_dir():
48
+ print(f"Error: vault_path '{vault}' not found.", file=sys.stderr)
49
+ sys.exit(1)
50
+ return Path(vault)
51
+
52
+
53
+ def get_retrieval_config():
54
+ """Return retrieval settings merged over defaults."""
55
+ cfg = load_config()
56
+ rc = dict(_RETRIEVAL_DEFAULTS)
57
+ rc.update(cfg.get("retrieval", {}))
58
+ return rc
59
+
60
+
61
+ def get_view_config():
62
+ """Return raw-view settings merged over defaults. Tolerates missing config."""
63
+ from .raw_view import VIEW_DEFAULTS
64
+ vc = dict(VIEW_DEFAULTS)
65
+ if CORTEX_CONFIG.exists():
66
+ with open(CORTEX_CONFIG) as f:
67
+ cfg = json.load(f)
68
+ vc.update(cfg.get("distill", {}).get("view", {}))
69
+ return vc
70
+
71
+
72
+ _MAP_DEFAULTS = {"page_budget": 12000, "session_budget": 100000}
73
+
74
+
75
+ def get_map_config():
76
+ """Return map-first page/session budgets merged over defaults."""
77
+ mc = dict(_MAP_DEFAULTS)
78
+ if CORTEX_CONFIG.exists():
79
+ with open(CORTEX_CONFIG) as f:
80
+ cfg = json.load(f)
81
+ mc.update(cfg.get("distill", {}).get("map", {}))
82
+ return mc