cortex-vec 0.9.0__tar.gz → 0.10.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cortex_vec-0.9.0/src/cortex_vec.egg-info → cortex_vec-0.10.0}/PKG-INFO +1 -1
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/pyproject.toml +1 -1
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/bm25.py +3 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/cli.py +18 -4
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/config.py +9 -10
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/distill_queue.py +12 -1
- cortex_vec-0.10.0/src/cortex_vec/errors.py +30 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/fusion.py +24 -10
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/store.py +77 -22
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/tokenize.py +11 -1
- {cortex_vec-0.9.0 → cortex_vec-0.10.0/src/cortex_vec.egg-info}/PKG-INFO +1 -1
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec.egg-info/SOURCES.txt +3 -0
- cortex_vec-0.10.0/tests/test_cli_errors.py +114 -0
- cortex_vec-0.10.0/tests/test_cli_json.py +138 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_cmd_search.py +33 -2
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_cmd_search_flags.py +1 -1
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_distill_queue.py +22 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_fusion_search.py +46 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_tokenize.py +16 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/LICENSE +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/NOTICE +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/README.md +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/setup.cfg +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/__init__.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/distill_plan.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/eval/__init__.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/eval/adapters.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/eval/corpus.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/eval/report.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/eval/run.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/eval/score.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/graph.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/parser.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/raw_map.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/raw_page.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/raw_source.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/raw_span.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/raw_view.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/reclaim.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/rerank.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec/synonyms.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec.egg-info/dependency_links.txt +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec.egg-info/entry_points.txt +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec.egg-info/requires.txt +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/src/cortex_vec.egg-info/top_level.txt +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_adapters.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_bm25.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_bm25_synonym.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_config.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_config_plan2.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_corpus.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_cosine_backfill.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_distill_plan.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_fusion_diversify.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_fusion_graph.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_fusion_graph_filter.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_fusion_rerank.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_fusion_rrf.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_fusion_synonym.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_graph_build.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_graph_stream.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_lockstep.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_no_weekly_type.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_raw_map.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_raw_page.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_raw_source.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_raw_span.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_raw_view.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_rebuild_bm25_only.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_reclaim.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_report.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_rerank.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_run.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_score.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_smoke.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_synonyms.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_vector_stream.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_where_shape_lockstep.py +0 -0
- {cortex_vec-0.9.0 → cortex_vec-0.10.0}/tests/test_wikilinks.py +0 -0
|
@@ -121,6 +121,9 @@ class BM25Index:
|
|
|
121
121
|
with open(self._file, "wb") as f:
|
|
122
122
|
pickle.dump(self.docs, f)
|
|
123
123
|
|
|
124
|
+
def exists(self):
|
|
125
|
+
return self._file.exists()
|
|
126
|
+
|
|
124
127
|
def load(self):
|
|
125
128
|
if not self._file.exists():
|
|
126
129
|
raise FileNotFoundError(f"BM25 index not found at {self._file}; run rebuild")
|
|
@@ -3,26 +3,40 @@
|
|
|
3
3
|
import argparse
|
|
4
4
|
import sys
|
|
5
5
|
|
|
6
|
+
from .errors import EXIT_CALLER, CliError
|
|
7
|
+
|
|
6
8
|
|
|
7
9
|
def main():
|
|
10
|
+
try:
|
|
11
|
+
_main()
|
|
12
|
+
except CliError as e:
|
|
13
|
+
print(e.render(), file=sys.stderr)
|
|
14
|
+
sys.exit(e.code)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _main():
|
|
8
18
|
parser = argparse.ArgumentParser(
|
|
9
19
|
description="cortex-vec — Vector store CLI for cortex vault"
|
|
10
20
|
)
|
|
11
21
|
sub = parser.add_subparsers(dest="command")
|
|
12
22
|
|
|
13
|
-
sub.add_parser("status", help="Show index health")
|
|
23
|
+
p_status = sub.add_parser("status", help="Show index health")
|
|
24
|
+
p_status.add_argument("--json", action="store_true", help="Emit one JSON document instead of human text")
|
|
14
25
|
p_rebuild = sub.add_parser("rebuild", help="Full rebuild from vault")
|
|
15
26
|
p_rebuild.add_argument(
|
|
16
27
|
"--bm25-only",
|
|
17
28
|
action="store_true",
|
|
18
29
|
help="Rebuild only the BM25 index from the vault (no re-embedding, free)",
|
|
19
30
|
)
|
|
31
|
+
p_rebuild.add_argument("--json", action="store_true", help="Emit one JSON document instead of human text")
|
|
20
32
|
|
|
21
33
|
p_upsert = sub.add_parser("upsert", help="Add/update a document")
|
|
22
34
|
p_upsert.add_argument("path", help="Relative path from vault root")
|
|
35
|
+
p_upsert.add_argument("--json", action="store_true", help="Emit one JSON document instead of human text")
|
|
23
36
|
|
|
24
37
|
p_delete = sub.add_parser("delete", help="Remove a document")
|
|
25
38
|
p_delete.add_argument("path", help="Relative path from vault root")
|
|
39
|
+
p_delete.add_argument("--json", action="store_true", help="Emit one JSON document instead of human text")
|
|
26
40
|
|
|
27
41
|
p_search = sub.add_parser("search", help="Semantic search")
|
|
28
42
|
p_search.add_argument("query", help="Search query text")
|
|
@@ -51,7 +65,7 @@ def main():
|
|
|
51
65
|
p_dq.add_argument("--stat", action="store_true",
|
|
52
66
|
help="Show per-level projected sizes instead of paths")
|
|
53
67
|
p_dq.add_argument("--json", action="store_true",
|
|
54
|
-
help="
|
|
68
|
+
help="Emit a JSON array (of paths, or of --stat rows)")
|
|
55
69
|
p_bq = sub.add_parser(
|
|
56
70
|
"broadcast-queue", help="List distilled Raw eligible for broadcast"
|
|
57
71
|
)
|
|
@@ -127,8 +141,8 @@ def main():
|
|
|
127
141
|
args = parser.parse_args()
|
|
128
142
|
|
|
129
143
|
if not args.command:
|
|
130
|
-
parser.print_help()
|
|
131
|
-
sys.exit(
|
|
144
|
+
parser.print_help(sys.stderr)
|
|
145
|
+
sys.exit(EXIT_CALLER)
|
|
132
146
|
|
|
133
147
|
# Fast path: trailer-anchored queue scans skip the heavy store import
|
|
134
148
|
# (chromadb ~2.7s). They read only each Raw's last non-empty line.
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
"""Configuration loading and constants."""
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
|
-
import sys
|
|
5
4
|
from pathlib import Path
|
|
6
5
|
|
|
6
|
+
from .errors import EXIT_SETUP, CliError
|
|
7
|
+
|
|
7
8
|
CORTEX_CONFIG = Path.home() / ".cortex" / "config.json"
|
|
8
9
|
VECTORSTORE_DIR = Path.home() / ".cortex" / "vectorstore"
|
|
9
10
|
COLLECTION_NAME = "cortex"
|
|
@@ -29,24 +30,22 @@ _RETRIEVAL_DEFAULTS = {
|
|
|
29
30
|
|
|
30
31
|
|
|
31
32
|
def load_config():
|
|
32
|
-
"""Load ~/.cortex/config.json
|
|
33
|
+
"""Load ~/.cortex/config.json; raise CliError if it is missing."""
|
|
33
34
|
if not CORTEX_CONFIG.exists():
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
file=sys.stderr,
|
|
37
|
-
)
|
|
38
|
-
sys.exit(1)
|
|
35
|
+
raise CliError(f"{CORTEX_CONFIG} not found", "run /cortexes:genesis first",
|
|
36
|
+
code=EXIT_SETUP)
|
|
39
37
|
with open(CORTEX_CONFIG) as f:
|
|
40
38
|
return json.load(f)
|
|
41
39
|
|
|
42
40
|
|
|
43
41
|
def get_vault_path():
|
|
44
|
-
"""Return the vault path from config
|
|
42
|
+
"""Return the vault path from config; raise CliError if it is invalid."""
|
|
45
43
|
cfg = load_config()
|
|
46
44
|
vault = cfg.get("vault_path", "")
|
|
47
45
|
if not vault or not Path(vault).is_dir():
|
|
48
|
-
|
|
49
|
-
|
|
46
|
+
raise CliError(f"vault_path '{vault}' is not a directory",
|
|
47
|
+
f"fix vault_path in {CORTEX_CONFIG} or rerun /cortexes:genesis",
|
|
48
|
+
code=EXIT_SETUP)
|
|
50
49
|
return Path(vault)
|
|
51
50
|
|
|
52
51
|
|
|
@@ -18,6 +18,7 @@ inside the conversation are quotes and ignored.
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
20
|
import re
|
|
21
|
+
import sys
|
|
21
22
|
from dataclasses import dataclass
|
|
22
23
|
from pathlib import Path
|
|
23
24
|
|
|
@@ -142,7 +143,14 @@ def dispatch_raw_state(path) -> None:
|
|
|
142
143
|
|
|
143
144
|
def dispatch_distill_queue(root, stat=False, as_json=False) -> None:
|
|
144
145
|
files = distill_queue(root)
|
|
146
|
+
if not stat and as_json:
|
|
147
|
+
import json
|
|
148
|
+
|
|
149
|
+
print(json.dumps([str(p) for p in files], ensure_ascii=False))
|
|
150
|
+
return
|
|
145
151
|
if not stat:
|
|
152
|
+
if not files:
|
|
153
|
+
print("cortex-vec: distill queue is empty", file=sys.stderr)
|
|
146
154
|
for p in files:
|
|
147
155
|
print(p)
|
|
148
156
|
return
|
|
@@ -167,5 +175,8 @@ def dispatch_distill_queue(root, stat=False, as_json=False) -> None:
|
|
|
167
175
|
|
|
168
176
|
|
|
169
177
|
def dispatch_broadcast_queue(root) -> None:
|
|
170
|
-
|
|
178
|
+
files = broadcast_queue(root)
|
|
179
|
+
if not files:
|
|
180
|
+
print("cortex-vec: broadcast queue is empty", file=sys.stderr)
|
|
181
|
+
for p in files:
|
|
171
182
|
print(p)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Agent-facing failure contract for the cortex-vec CLI.
|
|
2
|
+
|
|
3
|
+
Callers are mostly agents, which choose their next step from the exit code:
|
|
4
|
+
|
|
5
|
+
0 success, including an empty result
|
|
6
|
+
1 unexpected failure (an uncaught traceback exits 1 too)
|
|
7
|
+
2 caller error: fix the command and retry (argparse and raw-page errors
|
|
8
|
+
already use 2)
|
|
9
|
+
3 environment not set up: stop and tell the user, or run the named setup
|
|
10
|
+
|
|
11
|
+
`cli.main` catches CliError and prints it as one stderr line that names the
|
|
12
|
+
fix. The raw-map / raw-span family keeps its own JSON-on-stdout envelope.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
EXIT_CALLER = 2
|
|
16
|
+
EXIT_SETUP = 3
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class CliError(Exception):
|
|
20
|
+
"""A failure the caller can act on; `hint` says how."""
|
|
21
|
+
|
|
22
|
+
def __init__(self, message, hint="", code=EXIT_CALLER):
|
|
23
|
+
super().__init__(message)
|
|
24
|
+
self.message = message
|
|
25
|
+
self.hint = hint
|
|
26
|
+
self.code = code
|
|
27
|
+
|
|
28
|
+
def render(self):
|
|
29
|
+
line = f"cortex-vec: error: {self.message}"
|
|
30
|
+
return f"{line} ({self.hint})" if self.hint else line
|
|
@@ -26,26 +26,37 @@ def rrf_fuse(ranked, weights, k=60):
|
|
|
26
26
|
return sorted(scores.items(), key=lambda kv: kv[1], reverse=True)
|
|
27
27
|
|
|
28
28
|
|
|
29
|
-
def
|
|
29
|
+
def _note(notes, message):
|
|
30
|
+
"""Record a degradation for the caller to surface (None = nobody asked)."""
|
|
31
|
+
if notes is not None:
|
|
32
|
+
notes.append(message)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _bm25_stream(query, n, where, synonym_weight=0.0, notes=None):
|
|
30
36
|
"""Load the persisted BM25 index and search; return [] on any failure."""
|
|
31
37
|
from . import bm25
|
|
32
38
|
try:
|
|
33
39
|
idx = bm25.BM25Index(BM25_DIR)
|
|
34
40
|
idx.load()
|
|
35
41
|
return idx.search(query, n, where, synonym_weight=synonym_weight)
|
|
36
|
-
except Exception:
|
|
42
|
+
except Exception as e:
|
|
43
|
+
_note(notes, f"bm25 stream failed ({type(e).__name__}: {e}); "
|
|
44
|
+
"results are vector-only")
|
|
37
45
|
return []
|
|
38
46
|
|
|
39
47
|
|
|
40
|
-
def _vector_stream(query, n, where):
|
|
48
|
+
def _vector_stream(query, n, where, notes=None):
|
|
41
49
|
# No API key -> embeddings unavailable; skip vector so the query degrades to
|
|
42
|
-
# BM25-only
|
|
43
|
-
# SystemExit is not caught by `except Exception`, so we must check up front.)
|
|
50
|
+
# BM25-only without building the embedding fn (which raises CliError).
|
|
44
51
|
if not os.environ.get("OPENAI_API_KEY"):
|
|
52
|
+
_note(notes, "vector stream skipped (OPENAI_API_KEY not set); "
|
|
53
|
+
"results are BM25-only")
|
|
45
54
|
return []
|
|
46
55
|
try:
|
|
47
56
|
return store.vector_stream(query, n, where)
|
|
48
|
-
except Exception:
|
|
57
|
+
except Exception as e:
|
|
58
|
+
_note(notes, f"vector stream failed ({type(e).__name__}: {e}); "
|
|
59
|
+
"results are BM25-only")
|
|
49
60
|
return []
|
|
50
61
|
|
|
51
62
|
|
|
@@ -117,20 +128,23 @@ def _diversify(fused, display, max_per_repo):
|
|
|
117
128
|
return primary + overflow
|
|
118
129
|
|
|
119
130
|
|
|
120
|
-
def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, rerank=None
|
|
131
|
+
def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, rerank=None,
|
|
132
|
+
notes=None):
|
|
121
133
|
"""Hybrid search entry point. Returns up to n display dicts (best-first).
|
|
122
134
|
|
|
123
135
|
Gracefully degrades: if a stream errors or is disabled, the other carries
|
|
124
|
-
the query (RRF weight is redistributed in rrf_fuse).
|
|
136
|
+
the query (RRF weight is redistributed in rrf_fuse). Pass a list as
|
|
137
|
+
`notes` to receive one message per enabled stream that degraded.
|
|
125
138
|
"""
|
|
126
139
|
rc = get_retrieval_config()
|
|
127
140
|
weights = {"vector": rc["w_vec"], "bm25": rc["w_bm25"], "graph": rc["w_graph"]}
|
|
128
141
|
|
|
129
142
|
streams = {}
|
|
130
143
|
if use_vector:
|
|
131
|
-
streams["vector"] = _vector_stream(query, n, where)
|
|
144
|
+
streams["vector"] = _vector_stream(query, n, where, notes=notes)
|
|
132
145
|
if use_bm25:
|
|
133
|
-
streams["bm25"] = _bm25_stream(query, n, where,
|
|
146
|
+
streams["bm25"] = _bm25_stream(query, n, where,
|
|
147
|
+
synonym_weight=rc["synonym_weight"], notes=notes)
|
|
134
148
|
|
|
135
149
|
ranked = {}
|
|
136
150
|
display = {}
|
|
@@ -11,6 +11,7 @@ try:
|
|
|
11
11
|
except ModuleNotFoundError:
|
|
12
12
|
pass
|
|
13
13
|
|
|
14
|
+
import json
|
|
14
15
|
import os
|
|
15
16
|
|
|
16
17
|
import chromadb
|
|
@@ -18,6 +19,7 @@ from chromadb.utils.embedding_functions import OpenAIEmbeddingFunction
|
|
|
18
19
|
|
|
19
20
|
from .bm25 import BM25Index
|
|
20
21
|
from .config import BM25_DIR, COLLECTION_NAME, SUMMARY_MODEL, VECTORSTORE_DIR, get_vault_path
|
|
22
|
+
from .errors import EXIT_SETUP, CliError
|
|
21
23
|
from .parser import classify_path, extract_summary, parse_document
|
|
22
24
|
|
|
23
25
|
EMBEDDING_MODEL = "text-embedding-3-small"
|
|
@@ -27,8 +29,11 @@ def _get_embedding_function():
|
|
|
27
29
|
"""Return OpenAI embedding function."""
|
|
28
30
|
api_key = os.environ.get("OPENAI_API_KEY", "")
|
|
29
31
|
if not api_key:
|
|
30
|
-
|
|
31
|
-
|
|
32
|
+
raise CliError(
|
|
33
|
+
"OPENAI_API_KEY not set",
|
|
34
|
+
"export it for embeddings; `cortex-vec rebuild --bm25-only` "
|
|
35
|
+
"works without it",
|
|
36
|
+
code=EXIT_SETUP)
|
|
32
37
|
return OpenAIEmbeddingFunction(model_name=EMBEDDING_MODEL, api_key=api_key)
|
|
33
38
|
|
|
34
39
|
|
|
@@ -132,7 +137,16 @@ def _base_path(doc_id):
|
|
|
132
137
|
return doc_id.split("::")[0]
|
|
133
138
|
|
|
134
139
|
|
|
135
|
-
def
|
|
140
|
+
def _emit(args, lines, doc):
|
|
141
|
+
"""Print `doc` as one JSON line under --json, else the human `lines`."""
|
|
142
|
+
if getattr(args, "json", False):
|
|
143
|
+
print(json.dumps(doc, ensure_ascii=False))
|
|
144
|
+
else:
|
|
145
|
+
for line in lines:
|
|
146
|
+
print(line)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def cmd_status(args):
|
|
136
150
|
"""Show index health."""
|
|
137
151
|
vault = get_vault_path()
|
|
138
152
|
client = get_client()
|
|
@@ -153,15 +167,32 @@ def cmd_status(_args):
|
|
|
153
167
|
bm25_count = bm25_index.count()
|
|
154
168
|
except Exception:
|
|
155
169
|
bm25_count = 0
|
|
156
|
-
|
|
170
|
+
warnings = []
|
|
157
171
|
if total and not bm25_count:
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
172
|
+
warnings.append("BM25 index empty — run `cortex-vec rebuild --bm25-only`")
|
|
173
|
+
if not getattr(args, "json", False):
|
|
174
|
+
for w in warnings:
|
|
175
|
+
print(f" WARNING: {w}", file=sys.stderr)
|
|
176
|
+
_emit(args, [
|
|
177
|
+
f"BM25: {bm25_count} notes",
|
|
178
|
+
f"Collection: {COLLECTION_NAME}",
|
|
179
|
+
f"Entries: {total} ({bodies} body + {summaries} summary)",
|
|
180
|
+
f"Embedding: {EMBEDDING_MODEL} (OpenAI)",
|
|
181
|
+
f"Summary: {SUMMARY_MODEL} (OpenAI)",
|
|
182
|
+
f"DB path: {VECTORSTORE_DIR}",
|
|
183
|
+
f"Vault: {vault}",
|
|
184
|
+
], {
|
|
185
|
+
"bm25_notes": bm25_count,
|
|
186
|
+
"collection": COLLECTION_NAME,
|
|
187
|
+
"entries": total,
|
|
188
|
+
"body_entries": bodies,
|
|
189
|
+
"summary_entries": summaries,
|
|
190
|
+
"embedding_model": EMBEDDING_MODEL,
|
|
191
|
+
"summary_model": SUMMARY_MODEL,
|
|
192
|
+
"db_path": str(VECTORSTORE_DIR),
|
|
193
|
+
"vault": str(vault),
|
|
194
|
+
"warnings": warnings,
|
|
195
|
+
})
|
|
165
196
|
|
|
166
197
|
|
|
167
198
|
def _build_bm25_from_vault(vault):
|
|
@@ -194,7 +225,8 @@ def cmd_rebuild(args):
|
|
|
194
225
|
|
|
195
226
|
if getattr(args, "bm25_only", False):
|
|
196
227
|
count = _build_bm25_from_vault(vault)
|
|
197
|
-
|
|
228
|
+
_emit(args, [f"BM25: {count} notes indexed (vector store untouched)"],
|
|
229
|
+
{"bm25_notes": count, "vector_entries": None})
|
|
198
230
|
return
|
|
199
231
|
|
|
200
232
|
client = get_client()
|
|
@@ -265,8 +297,9 @@ def cmd_rebuild(args):
|
|
|
265
297
|
doc_count += 1
|
|
266
298
|
|
|
267
299
|
bm25_count = _build_bm25_from_vault(vault)
|
|
268
|
-
|
|
269
|
-
|
|
300
|
+
_emit(args, [f"BM25: {bm25_count} notes indexed",
|
|
301
|
+
f"Rebuilt: {doc_count} documents indexed"],
|
|
302
|
+
{"bm25_notes": bm25_count, "vector_entries": doc_count})
|
|
270
303
|
|
|
271
304
|
|
|
272
305
|
def cmd_upsert(args):
|
|
@@ -276,8 +309,8 @@ def cmd_upsert(args):
|
|
|
276
309
|
full_path = vault / rel_path
|
|
277
310
|
|
|
278
311
|
if not full_path.exists():
|
|
279
|
-
|
|
280
|
-
|
|
312
|
+
raise CliError(f"{rel_path} not found",
|
|
313
|
+
f"paths are relative to the vault root {vault}")
|
|
281
314
|
|
|
282
315
|
client = get_client()
|
|
283
316
|
col = get_collection(client)
|
|
@@ -334,7 +367,7 @@ def cmd_upsert(args):
|
|
|
334
367
|
bm25_index.build_from_docs([])
|
|
335
368
|
bm25_index.upsert(bm25_doc_from_fields(rel_path, fm, body))
|
|
336
369
|
bm25_index.save()
|
|
337
|
-
|
|
370
|
+
_emit(args, [f"Upserted: {rel_path}"], {"upserted": rel_path})
|
|
338
371
|
|
|
339
372
|
|
|
340
373
|
def cmd_delete(args):
|
|
@@ -345,8 +378,11 @@ def cmd_delete(args):
|
|
|
345
378
|
|
|
346
379
|
stale = _delete_stale_entries(col, rel_path)
|
|
347
380
|
if not stale:
|
|
348
|
-
|
|
349
|
-
|
|
381
|
+
# Idempotent: the goal state (not indexed) already holds.
|
|
382
|
+
print(f"cortex-vec: {rel_path} not indexed; nothing to delete",
|
|
383
|
+
file=sys.stderr)
|
|
384
|
+
_emit(args, [], {"deleted": 0, "path": rel_path})
|
|
385
|
+
return
|
|
350
386
|
|
|
351
387
|
bm25_index = BM25Index(BM25_DIR)
|
|
352
388
|
try:
|
|
@@ -356,7 +392,8 @@ def cmd_delete(args):
|
|
|
356
392
|
except FileNotFoundError:
|
|
357
393
|
pass
|
|
358
394
|
|
|
359
|
-
|
|
395
|
+
_emit(args, [f"Deleted: {len(stale)} entries for {rel_path}"],
|
|
396
|
+
{"deleted": len(stale), "path": rel_path})
|
|
360
397
|
|
|
361
398
|
|
|
362
399
|
def cosine_for(query, doc_ids):
|
|
@@ -489,12 +526,23 @@ def vector_stream(query, n, where=None):
|
|
|
489
526
|
return sorted(seen.values(), key=lambda x: -x["score"])
|
|
490
527
|
|
|
491
528
|
|
|
529
|
+
def _describe_filters(args):
|
|
530
|
+
active = [f"{k}={getattr(args, k)}" for k in ("repo", "type", "category")
|
|
531
|
+
if getattr(args, k, None)]
|
|
532
|
+
return f" ({', '.join(active)})" if active else ""
|
|
533
|
+
|
|
534
|
+
|
|
492
535
|
def cmd_search(args):
|
|
493
536
|
"""Hybrid search across the vault (vector + BM25, RRF-fused)."""
|
|
494
|
-
import json
|
|
495
|
-
|
|
496
537
|
from . import fusion
|
|
497
538
|
|
|
539
|
+
if not getattr(args, "no_bm25", False) and not BM25Index(BM25_DIR).exists():
|
|
540
|
+
raise CliError("no search index found",
|
|
541
|
+
"run `cortex-vec rebuild`, or `cortex-vec rebuild "
|
|
542
|
+
"--bm25-only` without OPENAI_API_KEY",
|
|
543
|
+
code=EXIT_SETUP)
|
|
544
|
+
|
|
545
|
+
notes = []
|
|
498
546
|
where = _build_where(
|
|
499
547
|
repo=getattr(args, "repo", None),
|
|
500
548
|
type=getattr(args, "type", None),
|
|
@@ -508,6 +556,13 @@ def cmd_search(args):
|
|
|
508
556
|
use_vector=not getattr(args, "no_vector", False),
|
|
509
557
|
rerank=getattr(args, "rerank", False) or None,
|
|
510
558
|
graph=getattr(args, "graph", False) or None,
|
|
559
|
+
notes=notes,
|
|
511
560
|
)
|
|
561
|
+
# stdout carries only hits (one JSON line each); anything an agent needs
|
|
562
|
+
# to interpret them -- a degraded stream, an empty result -- goes to stderr.
|
|
563
|
+
for note in notes:
|
|
564
|
+
print(f"cortex-vec: {note}", file=sys.stderr)
|
|
565
|
+
if not results:
|
|
566
|
+
print(f"cortex-vec: 0 results{_describe_filters(args)}", file=sys.stderr)
|
|
512
567
|
for entry in results:
|
|
513
568
|
print(json.dumps(entry, ensure_ascii=False))
|
|
@@ -4,8 +4,10 @@ Splits Han runs with jieba (soft-falls to whole-run if jieba missing),
|
|
|
4
4
|
stems Latin words with a Porter stemmer, and preserves Latin word
|
|
5
5
|
boundaries embedded inside CJK text (e.g. "機器學習ML技術").
|
|
6
6
|
"""
|
|
7
|
+
import logging
|
|
7
8
|
import re
|
|
8
9
|
import sys
|
|
10
|
+
import warnings
|
|
9
11
|
|
|
10
12
|
# Han, Hiragana/Katakana, Hangul ranges. cortex content is zh+en; non-Han CJK
|
|
11
13
|
# runs are kept whole (jieba targets Han).
|
|
@@ -30,7 +32,15 @@ def _seg_han(run):
|
|
|
30
32
|
global _jieba_ok
|
|
31
33
|
if _jieba_ok is None:
|
|
32
34
|
try:
|
|
33
|
-
|
|
35
|
+
# jieba's import trips setuptools' pkg_resources deprecation and its
|
|
36
|
+
# first lcut logs dictionary loading, both to stderr; callers are
|
|
37
|
+
# mostly agents that read stderr, so keep it for real warnings.
|
|
38
|
+
with warnings.catch_warnings():
|
|
39
|
+
warnings.filterwarnings(
|
|
40
|
+
"ignore", message="pkg_resources is deprecated",
|
|
41
|
+
category=UserWarning)
|
|
42
|
+
import jieba
|
|
43
|
+
jieba.setLogLevel(logging.WARNING)
|
|
34
44
|
_jieba_ok = True
|
|
35
45
|
except Exception:
|
|
36
46
|
_jieba_ok = False
|
|
@@ -8,6 +8,7 @@ src/cortex_vec/cli.py
|
|
|
8
8
|
src/cortex_vec/config.py
|
|
9
9
|
src/cortex_vec/distill_plan.py
|
|
10
10
|
src/cortex_vec/distill_queue.py
|
|
11
|
+
src/cortex_vec/errors.py
|
|
11
12
|
src/cortex_vec/fusion.py
|
|
12
13
|
src/cortex_vec/graph.py
|
|
13
14
|
src/cortex_vec/parser.py
|
|
@@ -36,6 +37,8 @@ src/cortex_vec/eval/score.py
|
|
|
36
37
|
tests/test_adapters.py
|
|
37
38
|
tests/test_bm25.py
|
|
38
39
|
tests/test_bm25_synonym.py
|
|
40
|
+
tests/test_cli_errors.py
|
|
41
|
+
tests/test_cli_json.py
|
|
39
42
|
tests/test_cmd_search.py
|
|
40
43
|
tests/test_cmd_search_flags.py
|
|
41
44
|
tests/test_config.py
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""The CLI's agent-facing failure contract.
|
|
2
|
+
|
|
3
|
+
An agent decides its next step from the exit code alone: 2 means "fix the
|
|
4
|
+
command and retry", 3 means "the environment is not set up — stop and tell
|
|
5
|
+
the user". Every error is one stderr line that names the fix.
|
|
6
|
+
"""
|
|
7
|
+
import json
|
|
8
|
+
import sys
|
|
9
|
+
|
|
10
|
+
import pytest
|
|
11
|
+
|
|
12
|
+
from cortex_vec import cli, config, store
|
|
13
|
+
|
|
14
|
+
# conftest replaces config.load_config with a stub for every test; keep the
|
|
15
|
+
# real one so these tests can exercise the missing/invalid-config paths.
|
|
16
|
+
_REAL_LOAD_CONFIG = config.load_config
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _run(monkeypatch, capsys, *argv):
|
|
20
|
+
monkeypatch.setattr(sys, "argv", ["cortex-vec", *argv])
|
|
21
|
+
with pytest.raises(SystemExit) as exc:
|
|
22
|
+
cli.main()
|
|
23
|
+
err = capsys.readouterr().err
|
|
24
|
+
return exc.value.code, err
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _run_ok(monkeypatch, capsys, *argv):
|
|
28
|
+
monkeypatch.setattr(sys, "argv", ["cortex-vec", *argv])
|
|
29
|
+
cli.main()
|
|
30
|
+
return capsys.readouterr()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@pytest.fixture
|
|
34
|
+
def vault(tmp_path, monkeypatch):
|
|
35
|
+
root = tmp_path / "vault"
|
|
36
|
+
root.mkdir()
|
|
37
|
+
monkeypatch.setattr(config, "load_config", lambda: {"vault_path": str(root)})
|
|
38
|
+
return root
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_missing_config_is_a_setup_error_naming_genesis(tmp_path, monkeypatch, capsys):
|
|
42
|
+
monkeypatch.setattr(config, "load_config", _REAL_LOAD_CONFIG)
|
|
43
|
+
monkeypatch.setattr(config, "CORTEX_CONFIG", tmp_path / "absent.json")
|
|
44
|
+
|
|
45
|
+
code, err = _run(monkeypatch, capsys, "distill-queue")
|
|
46
|
+
|
|
47
|
+
assert code == 3
|
|
48
|
+
assert err.startswith("cortex-vec: error: ")
|
|
49
|
+
assert "/cortexes:genesis" in err
|
|
50
|
+
assert err.count("\n") == 1
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_invalid_vault_path_is_a_setup_error(tmp_path, monkeypatch, capsys):
|
|
54
|
+
monkeypatch.setattr(config, "load_config",
|
|
55
|
+
lambda: {"vault_path": str(tmp_path / "gone")})
|
|
56
|
+
|
|
57
|
+
code, err = _run(monkeypatch, capsys, "distill-queue")
|
|
58
|
+
|
|
59
|
+
assert code == 3
|
|
60
|
+
assert "vault_path" in err
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_missing_api_key_is_a_setup_error_offering_bm25_only(vault, monkeypatch, capsys):
|
|
64
|
+
(vault / "Notes").mkdir()
|
|
65
|
+
(vault / "Notes" / "a.md").write_text("# a\n")
|
|
66
|
+
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
|
67
|
+
|
|
68
|
+
code, err = _run(monkeypatch, capsys, "upsert", "Notes/a.md")
|
|
69
|
+
|
|
70
|
+
assert code == 3
|
|
71
|
+
assert "OPENAI_API_KEY" in err
|
|
72
|
+
assert "--bm25-only" in err
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_upsert_of_a_missing_file_is_a_caller_error(vault, monkeypatch, capsys):
|
|
76
|
+
code, err = _run(monkeypatch, capsys, "upsert", "Notes/nope.md")
|
|
77
|
+
|
|
78
|
+
assert code == 2
|
|
79
|
+
assert "Notes/nope.md" in err
|
|
80
|
+
assert str(vault) in err # names the root the path is resolved against
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def test_delete_of_an_unindexed_path_succeeds(monkeypatch, capsys):
|
|
84
|
+
monkeypatch.setattr(store, "get_client", lambda: None)
|
|
85
|
+
monkeypatch.setattr(store, "get_collection", lambda client: None)
|
|
86
|
+
monkeypatch.setattr(store, "_delete_stale_entries", lambda col, rel: [])
|
|
87
|
+
|
|
88
|
+
out = _run_ok(monkeypatch, capsys, "delete", "Notes/never.md")
|
|
89
|
+
|
|
90
|
+
assert out.out == ""
|
|
91
|
+
assert "not indexed" in out.err
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def test_search_without_a_bm25_index_is_a_setup_error(tmp_path, monkeypatch, capsys):
|
|
95
|
+
monkeypatch.setattr(store, "BM25_DIR", tmp_path / "no-bm25")
|
|
96
|
+
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
|
97
|
+
|
|
98
|
+
code, err = _run(monkeypatch, capsys, "search", "anything")
|
|
99
|
+
|
|
100
|
+
assert code == 3
|
|
101
|
+
assert "cortex-vec rebuild" in err
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def test_raw_page_errors_keep_their_json_envelope(tmp_path, monkeypatch, capsys):
|
|
105
|
+
# raw-map/raw-span already speak JSON-on-stdout + exit 2; the new stderr
|
|
106
|
+
# contract must not wrap or duplicate them.
|
|
107
|
+
monkeypatch.setattr(sys, "argv", ["cortex-vec", "raw-span", str(tmp_path / "x.md"),
|
|
108
|
+
"--plan-id", "nope"])
|
|
109
|
+
with pytest.raises(SystemExit) as exc:
|
|
110
|
+
cli.main()
|
|
111
|
+
cap = capsys.readouterr()
|
|
112
|
+
assert exc.value.code == 2
|
|
113
|
+
assert "error" in json.loads(cap.out)
|
|
114
|
+
assert "cortex-vec: error:" not in cap.err
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""`--json` on the operational commands: one JSON document on stdout, nothing
|
|
2
|
+
an agent has to scrape out of aligned human text."""
|
|
3
|
+
import json
|
|
4
|
+
import sys
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
from cortex_vec import cli, config, store
|
|
9
|
+
from cortex_vec.bm25 import BM25Index
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class _FakeCollection:
|
|
13
|
+
def __init__(self, metas=()):
|
|
14
|
+
self._metas = list(metas)
|
|
15
|
+
|
|
16
|
+
def count(self):
|
|
17
|
+
return len(self._metas)
|
|
18
|
+
|
|
19
|
+
def get(self, include=None):
|
|
20
|
+
return {"metadatas": self._metas}
|
|
21
|
+
|
|
22
|
+
def upsert(self, **kw):
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class _FakeClient:
|
|
27
|
+
def __init__(self, col):
|
|
28
|
+
self._col = col
|
|
29
|
+
|
|
30
|
+
def get_collection(self, name):
|
|
31
|
+
return self._col
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@pytest.fixture
|
|
35
|
+
def vault(tmp_path, monkeypatch):
|
|
36
|
+
root = tmp_path / "vault"
|
|
37
|
+
(root / "Notes").mkdir(parents=True)
|
|
38
|
+
monkeypatch.setattr(config, "load_config", lambda: {"vault_path": str(root)})
|
|
39
|
+
return root
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _run(monkeypatch, capsys, *argv):
|
|
43
|
+
monkeypatch.setattr(sys, "argv", ["cortex-vec", *argv])
|
|
44
|
+
cli.main()
|
|
45
|
+
return capsys.readouterr()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_status_json_reports_counts(vault, monkeypatch, capsys):
|
|
49
|
+
col = _FakeCollection([{"entry_type": "body"}, {"entry_type": "summary"},
|
|
50
|
+
{"entry_type": "body"}])
|
|
51
|
+
monkeypatch.setattr(store, "get_client", lambda: _FakeClient(col))
|
|
52
|
+
idx = BM25Index(store.BM25_DIR)
|
|
53
|
+
idx.build_from_docs([{"id": "Notes/a.md", "title": "a", "body": "x"}])
|
|
54
|
+
idx.save()
|
|
55
|
+
|
|
56
|
+
cap = _run(monkeypatch, capsys, "status", "--json")
|
|
57
|
+
|
|
58
|
+
doc = json.loads(cap.out)
|
|
59
|
+
assert doc["bm25_notes"] == 1
|
|
60
|
+
assert doc["entries"] == 3
|
|
61
|
+
assert doc["body_entries"] == 2 and doc["summary_entries"] == 1
|
|
62
|
+
assert doc["vault"] == str(vault)
|
|
63
|
+
assert doc["warnings"] == []
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_status_json_carries_warnings_instead_of_stderr(vault, monkeypatch, capsys):
|
|
67
|
+
# vectors present but BM25 empty (conftest's isolated index has no docs)
|
|
68
|
+
col = _FakeCollection([{"entry_type": "body"}])
|
|
69
|
+
monkeypatch.setattr(store, "get_client", lambda: _FakeClient(col))
|
|
70
|
+
|
|
71
|
+
cap = _run(monkeypatch, capsys, "status", "--json")
|
|
72
|
+
|
|
73
|
+
doc = json.loads(cap.out)
|
|
74
|
+
assert len(doc["warnings"]) == 1 and "rebuild" in doc["warnings"][0]
|
|
75
|
+
assert cap.err == ""
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_upsert_json(vault, monkeypatch, capsys):
|
|
79
|
+
(vault / "Notes" / "a.md").write_text("# a\nbody\n")
|
|
80
|
+
monkeypatch.setattr(store, "get_client", lambda: None)
|
|
81
|
+
monkeypatch.setattr(store, "get_collection", lambda client: _FakeCollection())
|
|
82
|
+
monkeypatch.setattr(store, "_delete_stale_entries", lambda col, rel: [])
|
|
83
|
+
monkeypatch.setattr(store, "_generate_summary", lambda *a: "summary")
|
|
84
|
+
|
|
85
|
+
cap = _run(monkeypatch, capsys, "upsert", "Notes/a.md", "--json")
|
|
86
|
+
|
|
87
|
+
assert json.loads(cap.out) == {"upserted": "Notes/a.md"}
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_delete_json_counts_removed_entries(monkeypatch, capsys):
|
|
91
|
+
monkeypatch.setattr(store, "get_client", lambda: None)
|
|
92
|
+
monkeypatch.setattr(store, "get_collection", lambda client: None)
|
|
93
|
+
monkeypatch.setattr(store, "_delete_stale_entries",
|
|
94
|
+
lambda col, rel: ["Notes/a.md", "Notes/a.md::summary"])
|
|
95
|
+
|
|
96
|
+
cap = _run(monkeypatch, capsys, "delete", "Notes/a.md", "--json")
|
|
97
|
+
|
|
98
|
+
assert json.loads(cap.out) == {"deleted": 2, "path": "Notes/a.md"}
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def test_delete_json_of_unindexed_path_reports_zero(monkeypatch, capsys):
|
|
102
|
+
monkeypatch.setattr(store, "get_client", lambda: None)
|
|
103
|
+
monkeypatch.setattr(store, "get_collection", lambda client: None)
|
|
104
|
+
monkeypatch.setattr(store, "_delete_stale_entries", lambda col, rel: [])
|
|
105
|
+
|
|
106
|
+
cap = _run(monkeypatch, capsys, "delete", "Notes/never.md", "--json")
|
|
107
|
+
|
|
108
|
+
assert json.loads(cap.out) == {"deleted": 0, "path": "Notes/never.md"}
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def test_rebuild_bm25_only_json(vault, monkeypatch, capsys):
|
|
112
|
+
(vault / "Notes" / "a.md").write_text("# a\nbody\n")
|
|
113
|
+
(vault / "Notes" / "b.md").write_text("# b\nbody\n")
|
|
114
|
+
|
|
115
|
+
cap = _run(monkeypatch, capsys, "rebuild", "--bm25-only", "--json")
|
|
116
|
+
|
|
117
|
+
assert json.loads(cap.out) == {"bm25_notes": 2, "vector_entries": None}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def test_distill_queue_json_without_stat_lists_paths(tmp_path, monkeypatch, capsys):
|
|
121
|
+
raw = tmp_path / "Raw"
|
|
122
|
+
raw.mkdir()
|
|
123
|
+
(raw / "a.md").write_text("undistilled\n")
|
|
124
|
+
|
|
125
|
+
cap = _run(monkeypatch, capsys, "distill-queue", "--root", str(raw), "--json")
|
|
126
|
+
|
|
127
|
+
assert json.loads(cap.out) == [str(raw / "a.md")]
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def test_empty_distill_queue_json_is_an_empty_array_without_stderr(tmp_path, monkeypatch,
|
|
131
|
+
capsys):
|
|
132
|
+
raw = tmp_path / "Raw"
|
|
133
|
+
raw.mkdir()
|
|
134
|
+
|
|
135
|
+
cap = _run(monkeypatch, capsys, "distill-queue", "--root", str(raw), "--json")
|
|
136
|
+
|
|
137
|
+
assert json.loads(cap.out) == []
|
|
138
|
+
assert cap.err == ""
|
|
@@ -7,7 +7,7 @@ def test_cmd_search_uses_fusion(monkeypatch, capsys):
|
|
|
7
7
|
captured = {}
|
|
8
8
|
|
|
9
9
|
def fake_search(query, n=5, where=None, use_bm25=True, use_vector=True,
|
|
10
|
-
rerank=None, graph=None):
|
|
10
|
+
rerank=None, graph=None, notes=None):
|
|
11
11
|
captured["query"] = query
|
|
12
12
|
captured["where"] = where
|
|
13
13
|
captured["use_bm25"] = use_bm25
|
|
@@ -30,9 +30,40 @@ def test_cmd_search_no_bm25_flag(monkeypatch, capsys):
|
|
|
30
30
|
seen = {}
|
|
31
31
|
monkeypatch.setattr(fusion, "search",
|
|
32
32
|
lambda query, n=5, where=None, use_bm25=True, use_vector=True,
|
|
33
|
-
rerank=None, graph=None:
|
|
33
|
+
rerank=None, graph=None, notes=None:
|
|
34
34
|
seen.update(use_bm25=use_bm25) or [])
|
|
35
35
|
args = SimpleNamespace(query="x", repo=None, type=None, category=None,
|
|
36
36
|
n=5, no_bm25=True, no_vector=False)
|
|
37
37
|
store.cmd_search(args)
|
|
38
38
|
assert seen["use_bm25"] is False
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _args(**kw):
|
|
42
|
+
base = dict(query="x", repo=None, type=None, category=None, n=5,
|
|
43
|
+
no_bm25=False, no_vector=False)
|
|
44
|
+
base.update(kw)
|
|
45
|
+
return SimpleNamespace(**base)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_cmd_search_empty_result_is_named_on_stderr(monkeypatch, capsys):
|
|
49
|
+
monkeypatch.setattr(fusion, "search", lambda query, notes=None, **kw: [])
|
|
50
|
+
store.cmd_search(_args(repo="cortex"))
|
|
51
|
+
cap = capsys.readouterr()
|
|
52
|
+
assert cap.out == "" # stdout stays pure JSONL: line count == hit count
|
|
53
|
+
assert "0 results" in cap.err
|
|
54
|
+
assert "repo=cortex" in cap.err
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_cmd_search_relays_degradation_notes_to_stderr(monkeypatch, capsys):
|
|
58
|
+
hit = {"id": "Notes/a.md", "score": 0.3, "title": "a", "type": "note",
|
|
59
|
+
"repo": "", "category": "", "tags": "", "summary": ""}
|
|
60
|
+
|
|
61
|
+
def fake(query, notes=None, **kw):
|
|
62
|
+
notes.append("vector stream skipped: test reason; results are BM25-only")
|
|
63
|
+
return [hit]
|
|
64
|
+
|
|
65
|
+
monkeypatch.setattr(fusion, "search", fake)
|
|
66
|
+
store.cmd_search(_args())
|
|
67
|
+
cap = capsys.readouterr()
|
|
68
|
+
assert json.loads(cap.out)["id"] == "Notes/a.md"
|
|
69
|
+
assert "vector stream skipped: test reason" in cap.err
|
|
@@ -6,7 +6,7 @@ def test_cmd_search_threads_rerank_graph(monkeypatch):
|
|
|
6
6
|
captured = {}
|
|
7
7
|
monkeypatch.setattr(fusion, "search",
|
|
8
8
|
lambda query, n=5, where=None, use_bm25=True, use_vector=True,
|
|
9
|
-
rerank=None, graph=None:
|
|
9
|
+
rerank=None, graph=None, notes=None:
|
|
10
10
|
captured.update(rerank=rerank, graph=graph) or [])
|
|
11
11
|
args = SimpleNamespace(query="q", repo=None, type=None, category=None, n=5,
|
|
12
12
|
no_bm25=False, no_vector=False, rerank=True, graph=True)
|
|
@@ -207,3 +207,25 @@ def test_dispatch_distill_queue_stat_json(tmp_path, capsys):
|
|
|
207
207
|
rows = json.loads(capsys.readouterr().out)
|
|
208
208
|
assert rows[0]["file"].endswith("a.md")
|
|
209
209
|
assert rows[0]["raw"] > 0 and "chosen" in rows[0]
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def test_empty_distill_queue_is_named_on_stderr(tmp_path, capsys):
|
|
213
|
+
_write(tmp_path, "Raw/bb.md", "x\n<!-- distilled: 2026-06-10 → (skip: routine) -->\n")
|
|
214
|
+
dq.dispatch_distill_queue(str(tmp_path / "Raw"))
|
|
215
|
+
cap = capsys.readouterr()
|
|
216
|
+
assert cap.out == ""
|
|
217
|
+
assert "distill queue is empty" in cap.err
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def test_empty_broadcast_queue_is_named_on_stderr(tmp_path, capsys):
|
|
221
|
+
_write(tmp_path, "Raw/05.md", "body only, undistilled\n")
|
|
222
|
+
dq.dispatch_broadcast_queue(str(tmp_path / "Raw"))
|
|
223
|
+
cap = capsys.readouterr()
|
|
224
|
+
assert cap.out == ""
|
|
225
|
+
assert "broadcast queue is empty" in cap.err
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def test_non_empty_queue_writes_nothing_to_stderr(tmp_path, capsys):
|
|
229
|
+
_write(tmp_path, "Raw/aa.md", "undistilled body\n")
|
|
230
|
+
dq.dispatch_distill_queue(str(tmp_path / "Raw"))
|
|
231
|
+
assert capsys.readouterr().err == ""
|
|
@@ -88,3 +88,49 @@ def test_score_is_vector_cosine_not_rrf(monkeypatch):
|
|
|
88
88
|
assert by_id["Notes/Nginx/cert-renew.md"] == 0.9
|
|
89
89
|
# bm25-only hit reports a real cosine on the same 0-1 scale, not an RRF score
|
|
90
90
|
assert by_id["Notes/Linux/oom.md"] == 0.42
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
# --- degradation is reported, not silent --------------------------------------
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_missing_api_key_is_reported_as_bm25_only(monkeypatch):
|
|
97
|
+
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
|
98
|
+
monkeypatch.setattr(bm25, "BM25Index", _FakeBM25)
|
|
99
|
+
notes = []
|
|
100
|
+
out = fusion.search("oom", n=5, notes=notes)
|
|
101
|
+
assert out
|
|
102
|
+
assert len(notes) == 1
|
|
103
|
+
assert "OPENAI_API_KEY" in notes[0] and "BM25-only" in notes[0]
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def test_vector_failure_is_reported_with_its_cause(monkeypatch):
|
|
107
|
+
monkeypatch.setenv("OPENAI_API_KEY", "test-key")
|
|
108
|
+
def _boom(*a, **k):
|
|
109
|
+
raise RuntimeError("chroma exploded")
|
|
110
|
+
monkeypatch.setattr(store, "vector_stream", _boom)
|
|
111
|
+
monkeypatch.setattr(bm25, "BM25Index", _FakeBM25)
|
|
112
|
+
notes = []
|
|
113
|
+
fusion.search("oom", n=5, notes=notes)
|
|
114
|
+
assert len(notes) == 1
|
|
115
|
+
assert "chroma exploded" in notes[0] and "BM25-only" in notes[0]
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def test_bm25_failure_is_reported_with_its_cause(monkeypatch):
|
|
119
|
+
class _BrokenBM25(_FakeBM25):
|
|
120
|
+
def load(self):
|
|
121
|
+
raise ValueError("pickle truncated")
|
|
122
|
+
monkeypatch.setenv("OPENAI_API_KEY", "test-key")
|
|
123
|
+
monkeypatch.setattr(store, "vector_stream", lambda q, n, where=None: _vec_items())
|
|
124
|
+
monkeypatch.setattr(bm25, "BM25Index", _BrokenBM25)
|
|
125
|
+
notes = []
|
|
126
|
+
fusion.search("nginx", n=5, notes=notes)
|
|
127
|
+
assert len(notes) == 1
|
|
128
|
+
assert "pickle truncated" in notes[0] and "vector-only" in notes[0]
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def test_disabled_vector_stream_is_not_reported(monkeypatch):
|
|
132
|
+
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
|
133
|
+
monkeypatch.setattr(bm25, "BM25Index", _FakeBM25)
|
|
134
|
+
notes = []
|
|
135
|
+
fusion.search("oom", n=5, use_vector=False, notes=notes)
|
|
136
|
+
assert notes == []
|
|
@@ -29,3 +29,19 @@ def test_punctuation_stripped_but_path_chars_kept():
|
|
|
29
29
|
|
|
30
30
|
def test_short_query_nonempty():
|
|
31
31
|
assert tokenize.tokenize("PROJ-123456") # issue id should survive as tokens
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_first_cjk_tokenize_writes_nothing_to_stderr():
|
|
35
|
+
# jieba's import warns about pkg_resources and its first segmentation logs
|
|
36
|
+
# "Building prefix dict ..." to stderr; agents read stderr, so both are noise.
|
|
37
|
+
import os
|
|
38
|
+
import subprocess
|
|
39
|
+
import sys
|
|
40
|
+
|
|
41
|
+
env = {k: v for k, v in os.environ.items() if k != "PYTHONWARNINGS"}
|
|
42
|
+
proc = subprocess.run(
|
|
43
|
+
[sys.executable, "-c",
|
|
44
|
+
"from cortex_vec.tokenize import tokenize; tokenize('憑證自動更新')"],
|
|
45
|
+
capture_output=True, text=True, env=env, check=True,
|
|
46
|
+
)
|
|
47
|
+
assert proc.stderr == ""
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|