cortex-vec 0.9.0__tar.gz → 0.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cortex_vec-0.9.0/src/cortex_vec.egg-info → cortex_vec-0.11.0}/PKG-INFO +1 -1
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/pyproject.toml +1 -1
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/bm25.py +3 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/cli.py +24 -4
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/config.py +9 -10
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/distill_queue.py +12 -1
- cortex_vec-0.11.0/src/cortex_vec/errors.py +30 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/fusion.py +24 -10
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/raw_map.py +18 -7
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/store.py +77 -22
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/tokenize.py +11 -1
- {cortex_vec-0.9.0 → cortex_vec-0.11.0/src/cortex_vec.egg-info}/PKG-INFO +1 -1
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec.egg-info/SOURCES.txt +3 -0
- cortex_vec-0.11.0/tests/test_cli_errors.py +114 -0
- cortex_vec-0.11.0/tests/test_cli_json.py +138 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_cmd_search.py +33 -2
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_cmd_search_flags.py +1 -1
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_distill_queue.py +22 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_fusion_search.py +46 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_raw_map.py +44 -2
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_tokenize.py +16 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/LICENSE +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/NOTICE +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/README.md +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/setup.cfg +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/__init__.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/distill_plan.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/eval/__init__.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/eval/adapters.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/eval/corpus.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/eval/report.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/eval/run.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/eval/score.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/graph.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/parser.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/raw_page.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/raw_source.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/raw_span.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/raw_view.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/reclaim.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/rerank.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec/synonyms.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec.egg-info/dependency_links.txt +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec.egg-info/entry_points.txt +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec.egg-info/requires.txt +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/src/cortex_vec.egg-info/top_level.txt +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_adapters.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_bm25.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_bm25_synonym.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_config.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_config_plan2.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_corpus.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_cosine_backfill.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_distill_plan.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_fusion_diversify.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_fusion_graph.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_fusion_graph_filter.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_fusion_rerank.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_fusion_rrf.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_fusion_synonym.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_graph_build.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_graph_stream.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_lockstep.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_no_weekly_type.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_raw_page.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_raw_source.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_raw_span.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_raw_view.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_rebuild_bm25_only.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_reclaim.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_report.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_rerank.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_run.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_score.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_smoke.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_synonyms.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_vector_stream.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_where_shape_lockstep.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.11.0}/tests/test_wikilinks.py +0 -0
|
@@ -121,6 +121,9 @@ class BM25Index:
|
|
|
121
121
|
with open(self._file, "wb") as f:
|
|
122
122
|
pickle.dump(self.docs, f)
|
|
123
123
|
|
|
124
|
+
def exists(self):
|
|
125
|
+
return self._file.exists()
|
|
126
|
+
|
|
124
127
|
def load(self):
|
|
125
128
|
if not self._file.exists():
|
|
126
129
|
raise FileNotFoundError(f"BM25 index not found at {self._file}; run rebuild")
|
|
@@ -3,26 +3,40 @@
|
|
|
3
3
|
import argparse
|
|
4
4
|
import sys
|
|
5
5
|
|
|
6
|
+
from .errors import EXIT_CALLER, CliError
|
|
7
|
+
|
|
6
8
|
|
|
7
9
|
def main():
|
|
10
|
+
try:
|
|
11
|
+
_main()
|
|
12
|
+
except CliError as e:
|
|
13
|
+
print(e.render(), file=sys.stderr)
|
|
14
|
+
sys.exit(e.code)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _main():
|
|
8
18
|
parser = argparse.ArgumentParser(
|
|
9
19
|
description="cortex-vec — Vector store CLI for cortex vault"
|
|
10
20
|
)
|
|
11
21
|
sub = parser.add_subparsers(dest="command")
|
|
12
22
|
|
|
13
|
-
sub.add_parser("status", help="Show index health")
|
|
23
|
+
p_status = sub.add_parser("status", help="Show index health")
|
|
24
|
+
p_status.add_argument("--json", action="store_true", help="Emit one JSON document instead of human text")
|
|
14
25
|
p_rebuild = sub.add_parser("rebuild", help="Full rebuild from vault")
|
|
15
26
|
p_rebuild.add_argument(
|
|
16
27
|
"--bm25-only",
|
|
17
28
|
action="store_true",
|
|
18
29
|
help="Rebuild only the BM25 index from the vault (no re-embedding, free)",
|
|
19
30
|
)
|
|
31
|
+
p_rebuild.add_argument("--json", action="store_true", help="Emit one JSON document instead of human text")
|
|
20
32
|
|
|
21
33
|
p_upsert = sub.add_parser("upsert", help="Add/update a document")
|
|
22
34
|
p_upsert.add_argument("path", help="Relative path from vault root")
|
|
35
|
+
p_upsert.add_argument("--json", action="store_true", help="Emit one JSON document instead of human text")
|
|
23
36
|
|
|
24
37
|
p_delete = sub.add_parser("delete", help="Remove a document")
|
|
25
38
|
p_delete.add_argument("path", help="Relative path from vault root")
|
|
39
|
+
p_delete.add_argument("--json", action="store_true", help="Emit one JSON document instead of human text")
|
|
26
40
|
|
|
27
41
|
p_search = sub.add_parser("search", help="Semantic search")
|
|
28
42
|
p_search.add_argument("query", help="Search query text")
|
|
@@ -51,7 +65,7 @@ def main():
|
|
|
51
65
|
p_dq.add_argument("--stat", action="store_true",
|
|
52
66
|
help="Show per-level projected sizes instead of paths")
|
|
53
67
|
p_dq.add_argument("--json", action="store_true",
|
|
54
|
-
help="
|
|
68
|
+
help="Emit a JSON array (of paths, or of --stat rows)")
|
|
55
69
|
p_bq = sub.add_parser(
|
|
56
70
|
"broadcast-queue", help="List distilled Raw eligible for broadcast"
|
|
57
71
|
)
|
|
@@ -95,6 +109,12 @@ def main():
|
|
|
95
109
|
p_rm.add_argument("--find-only", action="store_true", dest="find_only",
|
|
96
110
|
help="With --find: emit only `find_matches` (no cards, "
|
|
97
111
|
"no map progress); costs one envelope, not a page")
|
|
112
|
+
p_rm.add_argument("--find-after", type=int, dest="find_after",
|
|
113
|
+
metavar="SPAN_ID",
|
|
114
|
+
help="With --find: report only hits in spans after this "
|
|
115
|
+
"id. `find_matches` holds at most 50 ids; when "
|
|
116
|
+
"`find_truncated` is true, pass the last id here "
|
|
117
|
+
"for the next 50 (`find_total` counts all hits)")
|
|
98
118
|
|
|
99
119
|
p_rsp = sub.add_parser(
|
|
100
120
|
"raw-span", help="Bounded original-source page of one span"
|
|
@@ -127,8 +147,8 @@ def main():
|
|
|
127
147
|
args = parser.parse_args()
|
|
128
148
|
|
|
129
149
|
if not args.command:
|
|
130
|
-
parser.print_help()
|
|
131
|
-
sys.exit(
|
|
150
|
+
parser.print_help(sys.stderr)
|
|
151
|
+
sys.exit(EXIT_CALLER)
|
|
132
152
|
|
|
133
153
|
# Fast path: trailer-anchored queue scans skip the heavy store import
|
|
134
154
|
# (chromadb ~2.7s). They read only each Raw's last non-empty line.
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
"""Configuration loading and constants."""
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
|
-
import sys
|
|
5
4
|
from pathlib import Path
|
|
6
5
|
|
|
6
|
+
from .errors import EXIT_SETUP, CliError
|
|
7
|
+
|
|
7
8
|
CORTEX_CONFIG = Path.home() / ".cortex" / "config.json"
|
|
8
9
|
VECTORSTORE_DIR = Path.home() / ".cortex" / "vectorstore"
|
|
9
10
|
COLLECTION_NAME = "cortex"
|
|
@@ -29,24 +30,22 @@ _RETRIEVAL_DEFAULTS = {
|
|
|
29
30
|
|
|
30
31
|
|
|
31
32
|
def load_config():
|
|
32
|
-
"""Load ~/.cortex/config.json
|
|
33
|
+
"""Load ~/.cortex/config.json; raise CliError if it is missing."""
|
|
33
34
|
if not CORTEX_CONFIG.exists():
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
file=sys.stderr,
|
|
37
|
-
)
|
|
38
|
-
sys.exit(1)
|
|
35
|
+
raise CliError(f"{CORTEX_CONFIG} not found", "run /cortexes:genesis first",
|
|
36
|
+
code=EXIT_SETUP)
|
|
39
37
|
with open(CORTEX_CONFIG) as f:
|
|
40
38
|
return json.load(f)
|
|
41
39
|
|
|
42
40
|
|
|
43
41
|
def get_vault_path():
|
|
44
|
-
"""Return the vault path from config
|
|
42
|
+
"""Return the vault path from config; raise CliError if it is invalid."""
|
|
45
43
|
cfg = load_config()
|
|
46
44
|
vault = cfg.get("vault_path", "")
|
|
47
45
|
if not vault or not Path(vault).is_dir():
|
|
48
|
-
|
|
49
|
-
|
|
46
|
+
raise CliError(f"vault_path '{vault}' is not a directory",
|
|
47
|
+
f"fix vault_path in {CORTEX_CONFIG} or rerun /cortexes:genesis",
|
|
48
|
+
code=EXIT_SETUP)
|
|
50
49
|
return Path(vault)
|
|
51
50
|
|
|
52
51
|
|
|
@@ -18,6 +18,7 @@ inside the conversation are quotes and ignored.
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
20
|
import re
|
|
21
|
+
import sys
|
|
21
22
|
from dataclasses import dataclass
|
|
22
23
|
from pathlib import Path
|
|
23
24
|
|
|
@@ -142,7 +143,14 @@ def dispatch_raw_state(path) -> None:
|
|
|
142
143
|
|
|
143
144
|
def dispatch_distill_queue(root, stat=False, as_json=False) -> None:
|
|
144
145
|
files = distill_queue(root)
|
|
146
|
+
if not stat and as_json:
|
|
147
|
+
import json
|
|
148
|
+
|
|
149
|
+
print(json.dumps([str(p) for p in files], ensure_ascii=False))
|
|
150
|
+
return
|
|
145
151
|
if not stat:
|
|
152
|
+
if not files:
|
|
153
|
+
print("cortex-vec: distill queue is empty", file=sys.stderr)
|
|
146
154
|
for p in files:
|
|
147
155
|
print(p)
|
|
148
156
|
return
|
|
@@ -167,5 +175,8 @@ def dispatch_distill_queue(root, stat=False, as_json=False) -> None:
|
|
|
167
175
|
|
|
168
176
|
|
|
169
177
|
def dispatch_broadcast_queue(root) -> None:
|
|
170
|
-
|
|
178
|
+
files = broadcast_queue(root)
|
|
179
|
+
if not files:
|
|
180
|
+
print("cortex-vec: broadcast queue is empty", file=sys.stderr)
|
|
181
|
+
for p in files:
|
|
171
182
|
print(p)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Agent-facing failure contract for the cortex-vec CLI.
|
|
2
|
+
|
|
3
|
+
Callers are mostly agents, which choose their next step from the exit code:
|
|
4
|
+
|
|
5
|
+
0 success, including an empty result
|
|
6
|
+
1 unexpected failure (an uncaught traceback exits 1 too)
|
|
7
|
+
2 caller error: fix the command and retry (argparse and raw-page errors
|
|
8
|
+
already use 2)
|
|
9
|
+
3 environment not set up: stop and tell the user, or run the named setup
|
|
10
|
+
|
|
11
|
+
`cli.main` catches CliError and prints it as one stderr line that names the
|
|
12
|
+
fix. The raw-map / raw-span family keeps its own JSON-on-stdout envelope.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
EXIT_CALLER = 2
|
|
16
|
+
EXIT_SETUP = 3
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class CliError(Exception):
|
|
20
|
+
"""A failure the caller can act on; `hint` says how."""
|
|
21
|
+
|
|
22
|
+
def __init__(self, message, hint="", code=EXIT_CALLER):
|
|
23
|
+
super().__init__(message)
|
|
24
|
+
self.message = message
|
|
25
|
+
self.hint = hint
|
|
26
|
+
self.code = code
|
|
27
|
+
|
|
28
|
+
def render(self):
|
|
29
|
+
line = f"cortex-vec: error: {self.message}"
|
|
30
|
+
return f"{line} ({self.hint})" if self.hint else line
|
|
@@ -26,26 +26,37 @@ def rrf_fuse(ranked, weights, k=60):
|
|
|
26
26
|
return sorted(scores.items(), key=lambda kv: kv[1], reverse=True)
|
|
27
27
|
|
|
28
28
|
|
|
29
|
-
def
|
|
29
|
+
def _note(notes, message):
|
|
30
|
+
"""Record a degradation for the caller to surface (None = nobody asked)."""
|
|
31
|
+
if notes is not None:
|
|
32
|
+
notes.append(message)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _bm25_stream(query, n, where, synonym_weight=0.0, notes=None):
|
|
30
36
|
"""Load the persisted BM25 index and search; return [] on any failure."""
|
|
31
37
|
from . import bm25
|
|
32
38
|
try:
|
|
33
39
|
idx = bm25.BM25Index(BM25_DIR)
|
|
34
40
|
idx.load()
|
|
35
41
|
return idx.search(query, n, where, synonym_weight=synonym_weight)
|
|
36
|
-
except Exception:
|
|
42
|
+
except Exception as e:
|
|
43
|
+
_note(notes, f"bm25 stream failed ({type(e).__name__}: {e}); "
|
|
44
|
+
"results are vector-only")
|
|
37
45
|
return []
|
|
38
46
|
|
|
39
47
|
|
|
40
|
-
def _vector_stream(query, n, where):
|
|
48
|
+
def _vector_stream(query, n, where, notes=None):
|
|
41
49
|
# No API key -> embeddings unavailable; skip vector so the query degrades to
|
|
42
|
-
# BM25-only
|
|
43
|
-
# SystemExit is not caught by `except Exception`, so we must check up front.)
|
|
50
|
+
# BM25-only without building the embedding fn (which raises CliError).
|
|
44
51
|
if not os.environ.get("OPENAI_API_KEY"):
|
|
52
|
+
_note(notes, "vector stream skipped (OPENAI_API_KEY not set); "
|
|
53
|
+
"results are BM25-only")
|
|
45
54
|
return []
|
|
46
55
|
try:
|
|
47
56
|
return store.vector_stream(query, n, where)
|
|
48
|
-
except Exception:
|
|
57
|
+
except Exception as e:
|
|
58
|
+
_note(notes, f"vector stream failed ({type(e).__name__}: {e}); "
|
|
59
|
+
"results are BM25-only")
|
|
49
60
|
return []
|
|
50
61
|
|
|
51
62
|
|
|
@@ -117,20 +128,23 @@ def _diversify(fused, display, max_per_repo):
|
|
|
117
128
|
return primary + overflow
|
|
118
129
|
|
|
119
130
|
|
|
120
|
-
def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, rerank=None
|
|
131
|
+
def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, rerank=None,
|
|
132
|
+
notes=None):
|
|
121
133
|
"""Hybrid search entry point. Returns up to n display dicts (best-first).
|
|
122
134
|
|
|
123
135
|
Gracefully degrades: if a stream errors or is disabled, the other carries
|
|
124
|
-
the query (RRF weight is redistributed in rrf_fuse).
|
|
136
|
+
the query (RRF weight is redistributed in rrf_fuse). Pass a list as
|
|
137
|
+
`notes` to receive one message per enabled stream that degraded.
|
|
125
138
|
"""
|
|
126
139
|
rc = get_retrieval_config()
|
|
127
140
|
weights = {"vector": rc["w_vec"], "bm25": rc["w_bm25"], "graph": rc["w_graph"]}
|
|
128
141
|
|
|
129
142
|
streams = {}
|
|
130
143
|
if use_vector:
|
|
131
|
-
streams["vector"] = _vector_stream(query, n, where)
|
|
144
|
+
streams["vector"] = _vector_stream(query, n, where, notes=notes)
|
|
132
145
|
if use_bm25:
|
|
133
|
-
streams["bm25"] = _bm25_stream(query, n, where,
|
|
146
|
+
streams["bm25"] = _bm25_stream(query, n, where,
|
|
147
|
+
synonym_weight=rc["synonym_weight"], notes=notes)
|
|
134
148
|
|
|
135
149
|
ranked = {}
|
|
136
150
|
display = {}
|
|
@@ -7,6 +7,7 @@ is complete); prose/output_body/ambiguous/opaque always need raw-span.
|
|
|
7
7
|
"""
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
|
+
import bisect
|
|
10
11
|
import re
|
|
11
12
|
|
|
12
13
|
from . import distill_plan as dp
|
|
@@ -16,6 +17,8 @@ from .raw_page import (PAGE_BUDGET_DEFAULT, PageError, decode_cursor,
|
|
|
16
17
|
render_page)
|
|
17
18
|
|
|
18
19
|
PREVIEW_CHARS = 160
|
|
20
|
+
# span ids per find envelope; keeps a find-only reply inside its page budget
|
|
21
|
+
FIND_LIMIT = 50
|
|
19
22
|
_ISSUE_RE = re.compile(r"\b[A-Z][A-Z0-9]{1,9}-\d{3,7}\b")
|
|
20
23
|
_URL_RE = re.compile(r"https?://\S+")
|
|
21
24
|
|
|
@@ -95,15 +98,21 @@ def _dispatch(args) -> None:
|
|
|
95
98
|
reason="--find-only needs --find <literal>")
|
|
96
99
|
find_matches = None
|
|
97
100
|
if find:
|
|
98
|
-
|
|
101
|
+
# Spans partition the text contiguously, so a bisect on char_start
|
|
102
|
+
# maps each occurrence to its span without scanning every span.
|
|
103
|
+
starts = [s.char_start for s in spans]
|
|
104
|
+
after = getattr(args, "find_after", None)
|
|
105
|
+
hits = []
|
|
99
106
|
pos = text.find(find)
|
|
100
|
-
while pos != -1
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
find_matches.append(s.id)
|
|
105
|
-
break
|
|
107
|
+
while pos != -1:
|
|
108
|
+
sid = spans[bisect.bisect_right(starts, pos) - 1].id
|
|
109
|
+
if not hits or hits[-1] != sid:
|
|
110
|
+
hits.append(sid)
|
|
106
111
|
pos = text.find(find, pos + 1)
|
|
112
|
+
later = [h for h in hits if after is None or h > after]
|
|
113
|
+
find_matches = later[:FIND_LIMIT]
|
|
114
|
+
find_total = len(hits)
|
|
115
|
+
find_truncated = len(later) > FIND_LIMIT
|
|
107
116
|
|
|
108
117
|
used_before = state["session_used_chars"]
|
|
109
118
|
budget = state["session_budget"] - dp.CONTROL_RESERVE
|
|
@@ -126,6 +135,8 @@ def _dispatch(args) -> None:
|
|
|
126
135
|
}
|
|
127
136
|
if find_matches is not None:
|
|
128
137
|
obj["find_matches"] = find_matches
|
|
138
|
+
obj["find_total"] = find_total
|
|
139
|
+
obj["find_truncated"] = find_truncated
|
|
129
140
|
if find_only:
|
|
130
141
|
# Locate only: no cards, no map progress, no continuation.
|
|
131
142
|
obj["find_only"] = True
|
|
@@ -11,6 +11,7 @@ try:
|
|
|
11
11
|
except ModuleNotFoundError:
|
|
12
12
|
pass
|
|
13
13
|
|
|
14
|
+
import json
|
|
14
15
|
import os
|
|
15
16
|
|
|
16
17
|
import chromadb
|
|
@@ -18,6 +19,7 @@ from chromadb.utils.embedding_functions import OpenAIEmbeddingFunction
|
|
|
18
19
|
|
|
19
20
|
from .bm25 import BM25Index
|
|
20
21
|
from .config import BM25_DIR, COLLECTION_NAME, SUMMARY_MODEL, VECTORSTORE_DIR, get_vault_path
|
|
22
|
+
from .errors import EXIT_SETUP, CliError
|
|
21
23
|
from .parser import classify_path, extract_summary, parse_document
|
|
22
24
|
|
|
23
25
|
EMBEDDING_MODEL = "text-embedding-3-small"
|
|
@@ -27,8 +29,11 @@ def _get_embedding_function():
|
|
|
27
29
|
"""Return OpenAI embedding function."""
|
|
28
30
|
api_key = os.environ.get("OPENAI_API_KEY", "")
|
|
29
31
|
if not api_key:
|
|
30
|
-
|
|
31
|
-
|
|
32
|
+
raise CliError(
|
|
33
|
+
"OPENAI_API_KEY not set",
|
|
34
|
+
"export it for embeddings; `cortex-vec rebuild --bm25-only` "
|
|
35
|
+
"works without it",
|
|
36
|
+
code=EXIT_SETUP)
|
|
32
37
|
return OpenAIEmbeddingFunction(model_name=EMBEDDING_MODEL, api_key=api_key)
|
|
33
38
|
|
|
34
39
|
|
|
@@ -132,7 +137,16 @@ def _base_path(doc_id):
|
|
|
132
137
|
return doc_id.split("::")[0]
|
|
133
138
|
|
|
134
139
|
|
|
135
|
-
def
|
|
140
|
+
def _emit(args, lines, doc):
|
|
141
|
+
"""Print `doc` as one JSON line under --json, else the human `lines`."""
|
|
142
|
+
if getattr(args, "json", False):
|
|
143
|
+
print(json.dumps(doc, ensure_ascii=False))
|
|
144
|
+
else:
|
|
145
|
+
for line in lines:
|
|
146
|
+
print(line)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def cmd_status(args):
|
|
136
150
|
"""Show index health."""
|
|
137
151
|
vault = get_vault_path()
|
|
138
152
|
client = get_client()
|
|
@@ -153,15 +167,32 @@ def cmd_status(_args):
|
|
|
153
167
|
bm25_count = bm25_index.count()
|
|
154
168
|
except Exception:
|
|
155
169
|
bm25_count = 0
|
|
156
|
-
|
|
170
|
+
warnings = []
|
|
157
171
|
if total and not bm25_count:
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
172
|
+
warnings.append("BM25 index empty — run `cortex-vec rebuild --bm25-only`")
|
|
173
|
+
if not getattr(args, "json", False):
|
|
174
|
+
for w in warnings:
|
|
175
|
+
print(f" WARNING: {w}", file=sys.stderr)
|
|
176
|
+
_emit(args, [
|
|
177
|
+
f"BM25: {bm25_count} notes",
|
|
178
|
+
f"Collection: {COLLECTION_NAME}",
|
|
179
|
+
f"Entries: {total} ({bodies} body + {summaries} summary)",
|
|
180
|
+
f"Embedding: {EMBEDDING_MODEL} (OpenAI)",
|
|
181
|
+
f"Summary: {SUMMARY_MODEL} (OpenAI)",
|
|
182
|
+
f"DB path: {VECTORSTORE_DIR}",
|
|
183
|
+
f"Vault: {vault}",
|
|
184
|
+
], {
|
|
185
|
+
"bm25_notes": bm25_count,
|
|
186
|
+
"collection": COLLECTION_NAME,
|
|
187
|
+
"entries": total,
|
|
188
|
+
"body_entries": bodies,
|
|
189
|
+
"summary_entries": summaries,
|
|
190
|
+
"embedding_model": EMBEDDING_MODEL,
|
|
191
|
+
"summary_model": SUMMARY_MODEL,
|
|
192
|
+
"db_path": str(VECTORSTORE_DIR),
|
|
193
|
+
"vault": str(vault),
|
|
194
|
+
"warnings": warnings,
|
|
195
|
+
})
|
|
165
196
|
|
|
166
197
|
|
|
167
198
|
def _build_bm25_from_vault(vault):
|
|
@@ -194,7 +225,8 @@ def cmd_rebuild(args):
|
|
|
194
225
|
|
|
195
226
|
if getattr(args, "bm25_only", False):
|
|
196
227
|
count = _build_bm25_from_vault(vault)
|
|
197
|
-
|
|
228
|
+
_emit(args, [f"BM25: {count} notes indexed (vector store untouched)"],
|
|
229
|
+
{"bm25_notes": count, "vector_entries": None})
|
|
198
230
|
return
|
|
199
231
|
|
|
200
232
|
client = get_client()
|
|
@@ -265,8 +297,9 @@ def cmd_rebuild(args):
|
|
|
265
297
|
doc_count += 1
|
|
266
298
|
|
|
267
299
|
bm25_count = _build_bm25_from_vault(vault)
|
|
268
|
-
|
|
269
|
-
|
|
300
|
+
_emit(args, [f"BM25: {bm25_count} notes indexed",
|
|
301
|
+
f"Rebuilt: {doc_count} documents indexed"],
|
|
302
|
+
{"bm25_notes": bm25_count, "vector_entries": doc_count})
|
|
270
303
|
|
|
271
304
|
|
|
272
305
|
def cmd_upsert(args):
|
|
@@ -276,8 +309,8 @@ def cmd_upsert(args):
|
|
|
276
309
|
full_path = vault / rel_path
|
|
277
310
|
|
|
278
311
|
if not full_path.exists():
|
|
279
|
-
|
|
280
|
-
|
|
312
|
+
raise CliError(f"{rel_path} not found",
|
|
313
|
+
f"paths are relative to the vault root {vault}")
|
|
281
314
|
|
|
282
315
|
client = get_client()
|
|
283
316
|
col = get_collection(client)
|
|
@@ -334,7 +367,7 @@ def cmd_upsert(args):
|
|
|
334
367
|
bm25_index.build_from_docs([])
|
|
335
368
|
bm25_index.upsert(bm25_doc_from_fields(rel_path, fm, body))
|
|
336
369
|
bm25_index.save()
|
|
337
|
-
|
|
370
|
+
_emit(args, [f"Upserted: {rel_path}"], {"upserted": rel_path})
|
|
338
371
|
|
|
339
372
|
|
|
340
373
|
def cmd_delete(args):
|
|
@@ -345,8 +378,11 @@ def cmd_delete(args):
|
|
|
345
378
|
|
|
346
379
|
stale = _delete_stale_entries(col, rel_path)
|
|
347
380
|
if not stale:
|
|
348
|
-
|
|
349
|
-
|
|
381
|
+
# Idempotent: the goal state (not indexed) already holds.
|
|
382
|
+
print(f"cortex-vec: {rel_path} not indexed; nothing to delete",
|
|
383
|
+
file=sys.stderr)
|
|
384
|
+
_emit(args, [], {"deleted": 0, "path": rel_path})
|
|
385
|
+
return
|
|
350
386
|
|
|
351
387
|
bm25_index = BM25Index(BM25_DIR)
|
|
352
388
|
try:
|
|
@@ -356,7 +392,8 @@ def cmd_delete(args):
|
|
|
356
392
|
except FileNotFoundError:
|
|
357
393
|
pass
|
|
358
394
|
|
|
359
|
-
|
|
395
|
+
_emit(args, [f"Deleted: {len(stale)} entries for {rel_path}"],
|
|
396
|
+
{"deleted": len(stale), "path": rel_path})
|
|
360
397
|
|
|
361
398
|
|
|
362
399
|
def cosine_for(query, doc_ids):
|
|
@@ -489,12 +526,23 @@ def vector_stream(query, n, where=None):
|
|
|
489
526
|
return sorted(seen.values(), key=lambda x: -x["score"])
|
|
490
527
|
|
|
491
528
|
|
|
529
|
+
def _describe_filters(args):
|
|
530
|
+
active = [f"{k}={getattr(args, k)}" for k in ("repo", "type", "category")
|
|
531
|
+
if getattr(args, k, None)]
|
|
532
|
+
return f" ({', '.join(active)})" if active else ""
|
|
533
|
+
|
|
534
|
+
|
|
492
535
|
def cmd_search(args):
|
|
493
536
|
"""Hybrid search across the vault (vector + BM25, RRF-fused)."""
|
|
494
|
-
import json
|
|
495
|
-
|
|
496
537
|
from . import fusion
|
|
497
538
|
|
|
539
|
+
if not getattr(args, "no_bm25", False) and not BM25Index(BM25_DIR).exists():
|
|
540
|
+
raise CliError("no search index found",
|
|
541
|
+
"run `cortex-vec rebuild`, or `cortex-vec rebuild "
|
|
542
|
+
"--bm25-only` without OPENAI_API_KEY",
|
|
543
|
+
code=EXIT_SETUP)
|
|
544
|
+
|
|
545
|
+
notes = []
|
|
498
546
|
where = _build_where(
|
|
499
547
|
repo=getattr(args, "repo", None),
|
|
500
548
|
type=getattr(args, "type", None),
|
|
@@ -508,6 +556,13 @@ def cmd_search(args):
|
|
|
508
556
|
use_vector=not getattr(args, "no_vector", False),
|
|
509
557
|
rerank=getattr(args, "rerank", False) or None,
|
|
510
558
|
graph=getattr(args, "graph", False) or None,
|
|
559
|
+
notes=notes,
|
|
511
560
|
)
|
|
561
|
+
# stdout carries only hits (one JSON line each); anything an agent needs
|
|
562
|
+
# to interpret them -- a degraded stream, an empty result -- goes to stderr.
|
|
563
|
+
for note in notes:
|
|
564
|
+
print(f"cortex-vec: {note}", file=sys.stderr)
|
|
565
|
+
if not results:
|
|
566
|
+
print(f"cortex-vec: 0 results{_describe_filters(args)}", file=sys.stderr)
|
|
512
567
|
for entry in results:
|
|
513
568
|
print(json.dumps(entry, ensure_ascii=False))
|
|
@@ -4,8 +4,10 @@ Splits Han runs with jieba (soft-falls to whole-run if jieba missing),
|
|
|
4
4
|
stems Latin words with a Porter stemmer, and preserves Latin word
|
|
5
5
|
boundaries embedded inside CJK text (e.g. "機器學習ML技術").
|
|
6
6
|
"""
|
|
7
|
+
import logging
|
|
7
8
|
import re
|
|
8
9
|
import sys
|
|
10
|
+
import warnings
|
|
9
11
|
|
|
10
12
|
# Han, Hiragana/Katakana, Hangul ranges. cortex content is zh+en; non-Han CJK
|
|
11
13
|
# runs are kept whole (jieba targets Han).
|
|
@@ -30,7 +32,15 @@ def _seg_han(run):
|
|
|
30
32
|
global _jieba_ok
|
|
31
33
|
if _jieba_ok is None:
|
|
32
34
|
try:
|
|
33
|
-
|
|
35
|
+
# jieba's import trips setuptools' pkg_resources deprecation and its
|
|
36
|
+
# first lcut logs dictionary loading, both to stderr; callers are
|
|
37
|
+
# mostly agents that read stderr, so keep it for real warnings.
|
|
38
|
+
with warnings.catch_warnings():
|
|
39
|
+
warnings.filterwarnings(
|
|
40
|
+
"ignore", message="pkg_resources is deprecated",
|
|
41
|
+
category=UserWarning)
|
|
42
|
+
import jieba
|
|
43
|
+
jieba.setLogLevel(logging.WARNING)
|
|
34
44
|
_jieba_ok = True
|
|
35
45
|
except Exception:
|
|
36
46
|
_jieba_ok = False
|
|
@@ -8,6 +8,7 @@ src/cortex_vec/cli.py
|
|
|
8
8
|
src/cortex_vec/config.py
|
|
9
9
|
src/cortex_vec/distill_plan.py
|
|
10
10
|
src/cortex_vec/distill_queue.py
|
|
11
|
+
src/cortex_vec/errors.py
|
|
11
12
|
src/cortex_vec/fusion.py
|
|
12
13
|
src/cortex_vec/graph.py
|
|
13
14
|
src/cortex_vec/parser.py
|
|
@@ -36,6 +37,8 @@ src/cortex_vec/eval/score.py
|
|
|
36
37
|
tests/test_adapters.py
|
|
37
38
|
tests/test_bm25.py
|
|
38
39
|
tests/test_bm25_synonym.py
|
|
40
|
+
tests/test_cli_errors.py
|
|
41
|
+
tests/test_cli_json.py
|
|
39
42
|
tests/test_cmd_search.py
|
|
40
43
|
tests/test_cmd_search_flags.py
|
|
41
44
|
tests/test_config.py
|