cortex-vec 0.8.0__tar.gz → 0.10.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cortex_vec-0.8.0/src/cortex_vec.egg-info → cortex_vec-0.10.0}/PKG-INFO +1 -1
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/pyproject.toml +1 -1
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/bm25.py +59 -10
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/cli.py +25 -5
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/config.py +9 -10
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/distill_plan.py +68 -2
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/distill_queue.py +12 -1
- cortex_vec-0.10.0/src/cortex_vec/errors.py +30 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/fusion.py +58 -17
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/graph.py +43 -2
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/raw_map.py +19 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/raw_source.py +32 -7
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/reclaim.py +111 -14
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/store.py +124 -22
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/tokenize.py +11 -1
- {cortex_vec-0.8.0 → cortex_vec-0.10.0/src/cortex_vec.egg-info}/PKG-INFO +1 -1
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec.egg-info/SOURCES.txt +6 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_bm25.py +8 -6
- cortex_vec-0.10.0/tests/test_cli_errors.py +114 -0
- cortex_vec-0.10.0/tests/test_cli_json.py +138 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_cmd_search.py +33 -2
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_cmd_search_flags.py +1 -1
- cortex_vec-0.10.0/tests/test_cosine_backfill.py +212 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_distill_plan.py +73 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_distill_queue.py +22 -0
- cortex_vec-0.10.0/tests/test_fusion_graph_filter.py +331 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_fusion_search.py +51 -2
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_raw_map.py +28 -2
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_raw_source.py +45 -14
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_reclaim.py +126 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_tokenize.py +16 -0
- cortex_vec-0.10.0/tests/test_where_shape_lockstep.py +180 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/LICENSE +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/NOTICE +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/README.md +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/setup.cfg +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/__init__.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/eval/__init__.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/eval/adapters.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/eval/corpus.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/eval/report.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/eval/run.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/eval/score.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/parser.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/raw_page.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/raw_span.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/raw_view.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/rerank.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/synonyms.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec.egg-info/dependency_links.txt +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec.egg-info/entry_points.txt +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec.egg-info/requires.txt +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec.egg-info/top_level.txt +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_adapters.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_bm25_synonym.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_config.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_config_plan2.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_corpus.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_fusion_diversify.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_fusion_graph.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_fusion_rerank.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_fusion_rrf.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_fusion_synonym.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_graph_build.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_graph_stream.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_lockstep.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_no_weekly_type.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_raw_page.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_raw_span.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_raw_view.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_rebuild_bm25_only.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_report.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_rerank.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_run.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_score.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_smoke.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_synonyms.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_vector_stream.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_wikilinks.py +0 -0
|
@@ -19,19 +19,65 @@ def _doc_record(doc):
|
|
|
19
19
|
return rec
|
|
20
20
|
|
|
21
21
|
|
|
22
|
+
# Fields this evaluator models, each paired with the predicate that evaluates it.
|
|
23
|
+
# ChromaDB accepts any metadata key, so a clause on a key absent here would be
|
|
24
|
+
# honoured by the vector stream and ignored by the two that run through
|
|
25
|
+
# `_matches` -- one query language per stream again, which is exactly the defect
|
|
26
|
+
# the spec below documents. Adding a filter field means adding it here;
|
|
27
|
+
# `test_where_shape_lockstep` fails until you do.
|
|
28
|
+
#
|
|
29
|
+
# The name list and the comparisons are ONE table on purpose. Keeping them apart
|
|
30
|
+
# let a field be whitelisted as "known" while no branch ever compared it, which
|
|
31
|
+
# passes every record silently -- the same fail-open, one level further in.
|
|
32
|
+
_FIELD_MATCHERS = {
|
|
33
|
+
"repo": lambda rec, want: want in rec.get("repos", []),
|
|
34
|
+
"type": lambda rec, want: rec.get("type") == want,
|
|
35
|
+
"category": lambda rec, want: rec.get("category") == want,
|
|
36
|
+
}
|
|
37
|
+
_KNOWN_FIELDS = frozenset(_FIELD_MATCHERS)
|
|
38
|
+
|
|
39
|
+
|
|
22
40
|
def _matches(rec, where):
|
|
41
|
+
"""Evaluate a Chroma-style `where` clause against a stored record.
|
|
42
|
+
|
|
43
|
+
Understands exactly the shapes `store._build_where` emits: flat field
|
|
44
|
+
equality plus `$and` / `$or` composition. All three retrieval streams
|
|
45
|
+
(BM25, wikilink graph, and -- via ChromaDB -- vector) must read the same
|
|
46
|
+
query language; a flat-only matcher silently ignored the nested payload
|
|
47
|
+
and dropped the filter entirely. See
|
|
48
|
+
docs/specs/2026-05-27-distill-dedup-repo-filter-blindspot.md.
|
|
49
|
+
|
|
50
|
+
The Notes/-are-cross-repo exemption lives in the `$or` branch that
|
|
51
|
+
`_build_where` emits, not in a `type == "note"` special case here.
|
|
52
|
+
|
|
53
|
+
Raises ValueError on a clause this evaluator does not model, rather than
|
|
54
|
+
guessing. Silently returning True for an unknown key is how a filter goes
|
|
55
|
+
missing without anyone noticing; the callers wrap this in the same
|
|
56
|
+
degrade-to-empty-stream contract they use for every other failure, so a
|
|
57
|
+
drift fails closed in production and loudly in the tests.
|
|
58
|
+
"""
|
|
23
59
|
if not where:
|
|
24
60
|
return True
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
if
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
61
|
+
if "$and" in where:
|
|
62
|
+
return all(_matches(rec, clause) for clause in where["$and"])
|
|
63
|
+
if "$or" in where:
|
|
64
|
+
return any(_matches(rec, clause) for clause in where["$or"])
|
|
65
|
+
|
|
66
|
+
unknown = set(where) - _KNOWN_FIELDS
|
|
67
|
+
if unknown:
|
|
68
|
+
raise ValueError(
|
|
69
|
+
f"_matches cannot evaluate {sorted(unknown)}: the filter producer and "
|
|
70
|
+
f"this evaluator have drifted. Add the field to _KNOWN_FIELDS and "
|
|
71
|
+
f"handle it below."
|
|
72
|
+
)
|
|
73
|
+
for field, expected in where.items():
|
|
74
|
+
if isinstance(expected, dict):
|
|
75
|
+
raise ValueError(
|
|
76
|
+
f"_matches does not implement operator form for {field!r}: "
|
|
77
|
+
f"{expected!r}. ChromaDB would honour it and this stream would not."
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
return all(_FIELD_MATCHERS[field](rec, want) for field, want in where.items())
|
|
35
81
|
|
|
36
82
|
|
|
37
83
|
class BM25Index:
|
|
@@ -75,6 +121,9 @@ class BM25Index:
|
|
|
75
121
|
with open(self._file, "wb") as f:
|
|
76
122
|
pickle.dump(self.docs, f)
|
|
77
123
|
|
|
124
|
+
def exists(self):
|
|
125
|
+
return self._file.exists()
|
|
126
|
+
|
|
78
127
|
def load(self):
|
|
79
128
|
if not self._file.exists():
|
|
80
129
|
raise FileNotFoundError(f"BM25 index not found at {self._file}; run rebuild")
|
|
@@ -3,26 +3,40 @@
|
|
|
3
3
|
import argparse
|
|
4
4
|
import sys
|
|
5
5
|
|
|
6
|
+
from .errors import EXIT_CALLER, CliError
|
|
7
|
+
|
|
6
8
|
|
|
7
9
|
def main():
|
|
10
|
+
try:
|
|
11
|
+
_main()
|
|
12
|
+
except CliError as e:
|
|
13
|
+
print(e.render(), file=sys.stderr)
|
|
14
|
+
sys.exit(e.code)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _main():
|
|
8
18
|
parser = argparse.ArgumentParser(
|
|
9
19
|
description="cortex-vec — Vector store CLI for cortex vault"
|
|
10
20
|
)
|
|
11
21
|
sub = parser.add_subparsers(dest="command")
|
|
12
22
|
|
|
13
|
-
sub.add_parser("status", help="Show index health")
|
|
23
|
+
p_status = sub.add_parser("status", help="Show index health")
|
|
24
|
+
p_status.add_argument("--json", action="store_true", help="Emit one JSON document instead of human text")
|
|
14
25
|
p_rebuild = sub.add_parser("rebuild", help="Full rebuild from vault")
|
|
15
26
|
p_rebuild.add_argument(
|
|
16
27
|
"--bm25-only",
|
|
17
28
|
action="store_true",
|
|
18
29
|
help="Rebuild only the BM25 index from the vault (no re-embedding, free)",
|
|
19
30
|
)
|
|
31
|
+
p_rebuild.add_argument("--json", action="store_true", help="Emit one JSON document instead of human text")
|
|
20
32
|
|
|
21
33
|
p_upsert = sub.add_parser("upsert", help="Add/update a document")
|
|
22
34
|
p_upsert.add_argument("path", help="Relative path from vault root")
|
|
35
|
+
p_upsert.add_argument("--json", action="store_true", help="Emit one JSON document instead of human text")
|
|
23
36
|
|
|
24
37
|
p_delete = sub.add_parser("delete", help="Remove a document")
|
|
25
38
|
p_delete.add_argument("path", help="Relative path from vault root")
|
|
39
|
+
p_delete.add_argument("--json", action="store_true", help="Emit one JSON document instead of human text")
|
|
26
40
|
|
|
27
41
|
p_search = sub.add_parser("search", help="Semantic search")
|
|
28
42
|
p_search.add_argument("query", help="Search query text")
|
|
@@ -51,7 +65,7 @@ def main():
|
|
|
51
65
|
p_dq.add_argument("--stat", action="store_true",
|
|
52
66
|
help="Show per-level projected sizes instead of paths")
|
|
53
67
|
p_dq.add_argument("--json", action="store_true",
|
|
54
|
-
help="
|
|
68
|
+
help="Emit a JSON array (of paths, or of --stat rows)")
|
|
55
69
|
p_bq = sub.add_parser(
|
|
56
70
|
"broadcast-queue", help="List distilled Raw eligible for broadcast"
|
|
57
71
|
)
|
|
@@ -88,7 +102,13 @@ def main():
|
|
|
88
102
|
p_rm.add_argument("--cursor", help="Continuation cursor from a prior page")
|
|
89
103
|
p_rm.add_argument("--max-chars", type=int, dest="max_chars",
|
|
90
104
|
help="Page stdout cap (default: config, 12000)")
|
|
91
|
-
p_rm.add_argument("--find",
|
|
105
|
+
p_rm.add_argument("--find",
|
|
106
|
+
help="Exact literal to locate: reports the matching span "
|
|
107
|
+
"ids in `find_matches`; does NOT move the page "
|
|
108
|
+
"(cards still start at 0 / --cursor)")
|
|
109
|
+
p_rm.add_argument("--find-only", action="store_true", dest="find_only",
|
|
110
|
+
help="With --find: emit only `find_matches` (no cards, "
|
|
111
|
+
"no map progress); costs one envelope, not a page")
|
|
92
112
|
|
|
93
113
|
p_rsp = sub.add_parser(
|
|
94
114
|
"raw-span", help="Bounded original-source page of one span"
|
|
@@ -121,8 +141,8 @@ def main():
|
|
|
121
141
|
args = parser.parse_args()
|
|
122
142
|
|
|
123
143
|
if not args.command:
|
|
124
|
-
parser.print_help()
|
|
125
|
-
sys.exit(
|
|
144
|
+
parser.print_help(sys.stderr)
|
|
145
|
+
sys.exit(EXIT_CALLER)
|
|
126
146
|
|
|
127
147
|
# Fast path: trailer-anchored queue scans skip the heavy store import
|
|
128
148
|
# (chromadb ~2.7s). They read only each Raw's last non-empty line.
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
"""Configuration loading and constants."""
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
|
-
import sys
|
|
5
4
|
from pathlib import Path
|
|
6
5
|
|
|
6
|
+
from .errors import EXIT_SETUP, CliError
|
|
7
|
+
|
|
7
8
|
CORTEX_CONFIG = Path.home() / ".cortex" / "config.json"
|
|
8
9
|
VECTORSTORE_DIR = Path.home() / ".cortex" / "vectorstore"
|
|
9
10
|
COLLECTION_NAME = "cortex"
|
|
@@ -29,24 +30,22 @@ _RETRIEVAL_DEFAULTS = {
|
|
|
29
30
|
|
|
30
31
|
|
|
31
32
|
def load_config():
|
|
32
|
-
"""Load ~/.cortex/config.json
|
|
33
|
+
"""Load ~/.cortex/config.json; raise CliError if it is missing."""
|
|
33
34
|
if not CORTEX_CONFIG.exists():
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
file=sys.stderr,
|
|
37
|
-
)
|
|
38
|
-
sys.exit(1)
|
|
35
|
+
raise CliError(f"{CORTEX_CONFIG} not found", "run /cortexes:genesis first",
|
|
36
|
+
code=EXIT_SETUP)
|
|
39
37
|
with open(CORTEX_CONFIG) as f:
|
|
40
38
|
return json.load(f)
|
|
41
39
|
|
|
42
40
|
|
|
43
41
|
def get_vault_path():
|
|
44
|
-
"""Return the vault path from config
|
|
42
|
+
"""Return the vault path from config; raise CliError if it is invalid."""
|
|
45
43
|
cfg = load_config()
|
|
46
44
|
vault = cfg.get("vault_path", "")
|
|
47
45
|
if not vault or not Path(vault).is_dir():
|
|
48
|
-
|
|
49
|
-
|
|
46
|
+
raise CliError(f"vault_path '{vault}' is not a directory",
|
|
47
|
+
f"fix vault_path in {CORTEX_CONFIG} or rerun /cortexes:genesis",
|
|
48
|
+
code=EXIT_SETUP)
|
|
50
49
|
return Path(vault)
|
|
51
50
|
|
|
52
51
|
|
|
@@ -8,6 +8,7 @@ fail-closed on corruption or identity drift.
|
|
|
8
8
|
"""
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
|
+
import hashlib
|
|
11
12
|
import json
|
|
12
13
|
import os
|
|
13
14
|
from datetime import datetime, timezone
|
|
@@ -319,13 +320,78 @@ def seal_plan(state: dict, expected_outcome: str) -> None:
|
|
|
319
320
|
_save(state)
|
|
320
321
|
|
|
321
322
|
|
|
323
|
+
def _normalization_variants(text: str) -> list:
|
|
324
|
+
"""Alternative marker strippings of ``text``, for attributing a mismatch.
|
|
325
|
+
|
|
326
|
+
Each entry is ``(variant_name, stripped_text)``. Only the error path calls
|
|
327
|
+
this, so a whole-file sha256 per variant is affordable; what costs
|
|
328
|
+
something is naming a rule that never existed, because that would
|
|
329
|
+
attribute a real edit to a phantom normalization. Every rule below either
|
|
330
|
+
shipped once or is something a writer could plausibly emit next:
|
|
331
|
+
|
|
332
|
+
* ``marker_line_only`` -- the marker line dropped, its blank separator
|
|
333
|
+
kept. This is exactly what the header branch did before 1.8.1, so a
|
|
334
|
+
regression of that wedge names itself instead of coming back
|
|
335
|
+
disguised as "source content changed".
|
|
336
|
+
* ``unstripped`` -- no marker removal at all, i.e. the plan recorded a
|
|
337
|
+
source that already carried a marker.
|
|
338
|
+
* ``final_newline_*`` -- the current strip with the trailing newline
|
|
339
|
+
added or dropped, for a writer that changes its mind about the last
|
|
340
|
+
byte (14 Raws were appended without one).
|
|
341
|
+
|
|
342
|
+
The marker scan here is deliberately position-blind: a quoted marker in
|
|
343
|
+
the conversation may be picked instead of the anchored one, which makes a
|
|
344
|
+
variant that simply fails to match -- never a wrong attribution.
|
|
345
|
+
"""
|
|
346
|
+
stripped = rs.strip_state_marker(text)
|
|
347
|
+
lines = text.split("\n")
|
|
348
|
+
first = next((i for i, ln in enumerate(lines) if rs.is_marker_line(ln)), None)
|
|
349
|
+
candidates = []
|
|
350
|
+
if first is not None:
|
|
351
|
+
candidates.append(("marker_line_only",
|
|
352
|
+
"\n".join(lines[:first] + lines[first + 1:])))
|
|
353
|
+
candidates.append(("unstripped", text))
|
|
354
|
+
candidates.append(("final_newline_added", stripped + "\n"))
|
|
355
|
+
candidates.append(("final_newline_dropped", stripped.rstrip("\n")))
|
|
356
|
+
seen = {stripped} # the rule in force already failed to match
|
|
357
|
+
out = []
|
|
358
|
+
for name, variant in candidates:
|
|
359
|
+
if variant not in seen:
|
|
360
|
+
seen.add(variant)
|
|
361
|
+
out.append((name, variant))
|
|
362
|
+
return out
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _diagnose_source_mismatch(text: str, rec: dict) -> dict:
|
|
366
|
+
"""Name why ``text`` no longer hashes to the plan's recorded source.
|
|
367
|
+
|
|
368
|
+
A plan stores hashes, not the pre-marker text, so a mismatch cannot be
|
|
369
|
+
diffed -- but it can be attributed. If some other stripping rule
|
|
370
|
+
reproduces the recorded source hash, the conversation is untouched and
|
|
371
|
+
only the marker accounting differs; if none does, the file really was
|
|
372
|
+
edited and "source content changed" is the honest answer. Conflating the
|
|
373
|
+
two is what sent a whole afternoon looking for an editor that had never
|
|
374
|
+
touched the file.
|
|
375
|
+
"""
|
|
376
|
+
for name, variant in _normalization_variants(text):
|
|
377
|
+
digest = hashlib.sha256(variant.encode("utf-8")).hexdigest()
|
|
378
|
+
if digest == rec["source_sha256"]:
|
|
379
|
+
return {"reason": "marker normalization mismatch",
|
|
380
|
+
"normalization": name,
|
|
381
|
+
"hint": "the marker block differs, not the conversation"}
|
|
382
|
+
return {"reason": "source content changed",
|
|
383
|
+
"chars_now": len(text),
|
|
384
|
+
"chars_at_plan_start": rec["char_count"]}
|
|
385
|
+
|
|
386
|
+
|
|
322
387
|
def complete_plan(plan_id: str, raw_path: str) -> dict:
|
|
323
388
|
state = _load_by_plan_id(plan_id)
|
|
324
389
|
if state["status"] != "sealed":
|
|
325
390
|
raise PageError("PLAN_NOT_ACTIVE", reason="seal before complete")
|
|
326
|
-
ident,
|
|
391
|
+
ident, text = rs.load(raw_path)
|
|
327
392
|
if ident.source_sha256 != state["identity"]["source_sha256"]:
|
|
328
|
-
raise PageError("RAW_CHANGED",
|
|
393
|
+
raise PageError("RAW_CHANGED",
|
|
394
|
+
**_diagnose_source_mismatch(text, state["identity"]))
|
|
329
395
|
from .distill_queue import classify
|
|
330
396
|
outcome = classify(raw_path).outcome
|
|
331
397
|
if outcome != state["expected_outcome"]:
|
|
@@ -18,6 +18,7 @@ inside the conversation are quotes and ignored.
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
20
|
import re
|
|
21
|
+
import sys
|
|
21
22
|
from dataclasses import dataclass
|
|
22
23
|
from pathlib import Path
|
|
23
24
|
|
|
@@ -142,7 +143,14 @@ def dispatch_raw_state(path) -> None:
|
|
|
142
143
|
|
|
143
144
|
def dispatch_distill_queue(root, stat=False, as_json=False) -> None:
|
|
144
145
|
files = distill_queue(root)
|
|
146
|
+
if not stat and as_json:
|
|
147
|
+
import json
|
|
148
|
+
|
|
149
|
+
print(json.dumps([str(p) for p in files], ensure_ascii=False))
|
|
150
|
+
return
|
|
145
151
|
if not stat:
|
|
152
|
+
if not files:
|
|
153
|
+
print("cortex-vec: distill queue is empty", file=sys.stderr)
|
|
146
154
|
for p in files:
|
|
147
155
|
print(p)
|
|
148
156
|
return
|
|
@@ -167,5 +175,8 @@ def dispatch_distill_queue(root, stat=False, as_json=False) -> None:
|
|
|
167
175
|
|
|
168
176
|
|
|
169
177
|
def dispatch_broadcast_queue(root) -> None:
|
|
170
|
-
|
|
178
|
+
files = broadcast_queue(root)
|
|
179
|
+
if not files:
|
|
180
|
+
print("cortex-vec: broadcast queue is empty", file=sys.stderr)
|
|
181
|
+
for p in files:
|
|
171
182
|
print(p)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Agent-facing failure contract for the cortex-vec CLI.
|
|
2
|
+
|
|
3
|
+
Callers are mostly agents, which choose their next step from the exit code:
|
|
4
|
+
|
|
5
|
+
0 success, including an empty result
|
|
6
|
+
1 unexpected failure (an uncaught traceback exits 1 too)
|
|
7
|
+
2 caller error: fix the command and retry (argparse and raw-page errors
|
|
8
|
+
already use 2)
|
|
9
|
+
3 environment not set up: stop and tell the user, or run the named setup
|
|
10
|
+
|
|
11
|
+
`cli.main` catches CliError and prints it as one stderr line that names the
|
|
12
|
+
fix. The raw-map / raw-span family keeps its own JSON-on-stdout envelope.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
EXIT_CALLER = 2
|
|
16
|
+
EXIT_SETUP = 3
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class CliError(Exception):
|
|
20
|
+
"""A failure the caller can act on; `hint` says how."""
|
|
21
|
+
|
|
22
|
+
def __init__(self, message, hint="", code=EXIT_CALLER):
|
|
23
|
+
super().__init__(message)
|
|
24
|
+
self.message = message
|
|
25
|
+
self.hint = hint
|
|
26
|
+
self.code = code
|
|
27
|
+
|
|
28
|
+
def render(self):
|
|
29
|
+
line = f"cortex-vec: error: {self.message}"
|
|
30
|
+
return f"{line} ({self.hint})" if self.hint else line
|
|
@@ -26,26 +26,37 @@ def rrf_fuse(ranked, weights, k=60):
|
|
|
26
26
|
return sorted(scores.items(), key=lambda kv: kv[1], reverse=True)
|
|
27
27
|
|
|
28
28
|
|
|
29
|
-
def
|
|
29
|
+
def _note(notes, message):
|
|
30
|
+
"""Record a degradation for the caller to surface (None = nobody asked)."""
|
|
31
|
+
if notes is not None:
|
|
32
|
+
notes.append(message)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _bm25_stream(query, n, where, synonym_weight=0.0, notes=None):
|
|
30
36
|
"""Load the persisted BM25 index and search; return [] on any failure."""
|
|
31
37
|
from . import bm25
|
|
32
38
|
try:
|
|
33
39
|
idx = bm25.BM25Index(BM25_DIR)
|
|
34
40
|
idx.load()
|
|
35
41
|
return idx.search(query, n, where, synonym_weight=synonym_weight)
|
|
36
|
-
except Exception:
|
|
42
|
+
except Exception as e:
|
|
43
|
+
_note(notes, f"bm25 stream failed ({type(e).__name__}: {e}); "
|
|
44
|
+
"results are vector-only")
|
|
37
45
|
return []
|
|
38
46
|
|
|
39
47
|
|
|
40
|
-
def _vector_stream(query, n, where):
|
|
48
|
+
def _vector_stream(query, n, where, notes=None):
|
|
41
49
|
# No API key -> embeddings unavailable; skip vector so the query degrades to
|
|
42
|
-
# BM25-only
|
|
43
|
-
# SystemExit is not caught by `except Exception`, so we must check up front.)
|
|
50
|
+
# BM25-only without building the embedding fn (which raises CliError).
|
|
44
51
|
if not os.environ.get("OPENAI_API_KEY"):
|
|
52
|
+
_note(notes, "vector stream skipped (OPENAI_API_KEY not set); "
|
|
53
|
+
"results are BM25-only")
|
|
45
54
|
return []
|
|
46
55
|
try:
|
|
47
56
|
return store.vector_stream(query, n, where)
|
|
48
|
-
except Exception:
|
|
57
|
+
except Exception as e:
|
|
58
|
+
_note(notes, f"vector stream failed ({type(e).__name__}: {e}); "
|
|
59
|
+
"results are BM25-only")
|
|
49
60
|
return []
|
|
50
61
|
|
|
51
62
|
|
|
@@ -53,7 +64,7 @@ def _vector_stream(query, n, where):
|
|
|
53
64
|
_STREAM_ORDER = ("vector", "bm25")
|
|
54
65
|
|
|
55
66
|
|
|
56
|
-
def _graph_ranked(ranked, weights, rc, n):
|
|
67
|
+
def _graph_ranked(ranked, weights, rc, n, where=None):
|
|
57
68
|
"""Build the graph RRF stream: wikilink-neighbors of the top vector/bm25 hits.
|
|
58
69
|
|
|
59
70
|
Returns (ranked_list, display_map) where ranked_list is [(doc_id, rank)] for
|
|
@@ -61,7 +72,14 @@ def _graph_ranked(ranked, weights, rc, n):
|
|
|
61
72
|
([], {}) on failure (e.g. vault unconfigured) or when graph yields nothing.
|
|
62
73
|
Rank-based (not additive), so it composes with the other streams via RRF
|
|
63
74
|
without the score-scale conflict that the old additive boost had.
|
|
75
|
+
|
|
76
|
+
`where` is evaluated with the same `bm25._matches` the keyword stream uses:
|
|
77
|
+
three streams, one filter language. This stream used to receive no `where`
|
|
78
|
+
at all, so wikilink neighbours entered the fused set past `--repo`/`--type`/
|
|
79
|
+
`--category`. A doc with no metadata is rejected rather than admitted — we
|
|
80
|
+
cannot show it passed the filter.
|
|
64
81
|
"""
|
|
82
|
+
from . import bm25 as bm25_mod
|
|
65
83
|
from . import graph as graph_mod
|
|
66
84
|
try:
|
|
67
85
|
pre = rrf_fuse(ranked, weights, k=rc["rrf_k"])
|
|
@@ -69,10 +87,20 @@ def _graph_ranked(ranked, weights, rc, n):
|
|
|
69
87
|
if not seeds:
|
|
70
88
|
return [], {}
|
|
71
89
|
adjacency, meta = graph_mod.build_graph(get_vault_path())
|
|
90
|
+
keep = None
|
|
91
|
+
if where:
|
|
92
|
+
def keep(doc_id):
|
|
93
|
+
rec = meta.get(doc_id)
|
|
94
|
+
return rec is not None and bm25_mod._matches(rec, where)
|
|
72
95
|
stream = graph_mod.graph_stream(
|
|
73
|
-
adjacency, seeds, hops=rc["graph_hops"], max_n=max(n * 2, 10)
|
|
96
|
+
adjacency, seeds, hops=rc["graph_hops"], max_n=max(n * 2, 10), keep=keep
|
|
74
97
|
)
|
|
75
|
-
|
|
98
|
+
# `repos` is a filter field, not a display field: the other streams do
|
|
99
|
+
# not emit it, so it must not reach the result dicts from this one.
|
|
100
|
+
gdisplay = {
|
|
101
|
+
doc_id: {k: v for k, v in meta[doc_id].items() if k != "repos"}
|
|
102
|
+
for doc_id, _ in stream if doc_id in meta
|
|
103
|
+
}
|
|
76
104
|
return stream, gdisplay
|
|
77
105
|
except (Exception, SystemExit):
|
|
78
106
|
return [], {}
|
|
@@ -100,20 +128,23 @@ def _diversify(fused, display, max_per_repo):
|
|
|
100
128
|
return primary + overflow
|
|
101
129
|
|
|
102
130
|
|
|
103
|
-
def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, rerank=None
|
|
131
|
+
def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, rerank=None,
|
|
132
|
+
notes=None):
|
|
104
133
|
"""Hybrid search entry point. Returns up to n display dicts (best-first).
|
|
105
134
|
|
|
106
135
|
Gracefully degrades: if a stream errors or is disabled, the other carries
|
|
107
|
-
the query (RRF weight is redistributed in rrf_fuse).
|
|
136
|
+
the query (RRF weight is redistributed in rrf_fuse). Pass a list as
|
|
137
|
+
`notes` to receive one message per enabled stream that degraded.
|
|
108
138
|
"""
|
|
109
139
|
rc = get_retrieval_config()
|
|
110
140
|
weights = {"vector": rc["w_vec"], "bm25": rc["w_bm25"], "graph": rc["w_graph"]}
|
|
111
141
|
|
|
112
142
|
streams = {}
|
|
113
143
|
if use_vector:
|
|
114
|
-
streams["vector"] = _vector_stream(query, n, where)
|
|
144
|
+
streams["vector"] = _vector_stream(query, n, where, notes=notes)
|
|
115
145
|
if use_bm25:
|
|
116
|
-
streams["bm25"] = _bm25_stream(query, n, where,
|
|
146
|
+
streams["bm25"] = _bm25_stream(query, n, where,
|
|
147
|
+
synonym_weight=rc["synonym_weight"], notes=notes)
|
|
117
148
|
|
|
118
149
|
ranked = {}
|
|
119
150
|
display = {}
|
|
@@ -133,7 +164,7 @@ def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, r
|
|
|
133
164
|
|
|
134
165
|
use_graph = rc["graph"] if graph is None else graph
|
|
135
166
|
if use_graph:
|
|
136
|
-
g_ranked, g_display = _graph_ranked(ranked, weights, rc, n)
|
|
167
|
+
g_ranked, g_display = _graph_ranked(ranked, weights, rc, n, where)
|
|
137
168
|
if g_ranked:
|
|
138
169
|
ranked["graph"] = g_ranked
|
|
139
170
|
for doc_id, disp in g_display.items():
|
|
@@ -150,14 +181,24 @@ def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, r
|
|
|
150
181
|
use_rerank = rc["rerank"] if rerank is None else rerank
|
|
151
182
|
take = max(n, rc["rerank_window"]) if use_rerank else n
|
|
152
183
|
|
|
184
|
+
head = fused[:take]
|
|
185
|
+
# Hits that arrived via bm25 or graph never passed through the vector
|
|
186
|
+
# top-n window, so they carry no cosine. Ask the collection for theirs
|
|
187
|
+
# rather than reporting 0.0, which reads as "no overlap" downstream.
|
|
188
|
+
missing = [doc_id for doc_id, _ in head if doc_id not in vec_score]
|
|
189
|
+
if missing and use_vector and os.environ.get("OPENAI_API_KEY"):
|
|
190
|
+
try:
|
|
191
|
+
vec_score.update(store.cosine_for(query, missing))
|
|
192
|
+
except (Exception, SystemExit):
|
|
193
|
+
pass # same degradation contract as the streams: score falls to 0.0
|
|
194
|
+
|
|
153
195
|
out = []
|
|
154
|
-
for doc_id, _rrf in
|
|
196
|
+
for doc_id, _rrf in head:
|
|
155
197
|
entry = dict(display.get(doc_id, {}))
|
|
156
198
|
entry["id"] = doc_id
|
|
157
199
|
# `score` reports the vector cosine similarity (0-1, interpretable) so
|
|
158
200
|
# absolute-threshold consumers (distill/broadcast dedup) keep working.
|
|
159
|
-
# RRF only determines ORDER (already baked into `fused`).
|
|
160
|
-
# have no cosine score -> 0.0.
|
|
201
|
+
# RRF only determines ORDER (already baked into `fused`).
|
|
161
202
|
entry["score"] = round(vec_score.get(doc_id, 0.0), 4)
|
|
162
203
|
out.append(entry)
|
|
163
204
|
|
|
@@ -11,6 +11,7 @@ Graph participates in retrieval as a THIRD RRF stream (see fusion.py): a
|
|
|
11
11
|
rank-based list of wikilink-neighbors of the top hits. Rank-based fusion avoids
|
|
12
12
|
the scale conflict of adding a boost onto RRF's compressed score band.
|
|
13
13
|
"""
|
|
14
|
+
import hashlib
|
|
14
15
|
from pathlib import Path
|
|
15
16
|
|
|
16
17
|
from .parser import classify_path, extract_summary, extract_wikilinks, parse_document
|
|
@@ -29,7 +30,11 @@ def _meta_for(rel, fm, body):
|
|
|
29
30
|
"id": rel,
|
|
30
31
|
"title": fm.get("title", rel.rsplit("/", 1)[-1].removesuffix(".md")),
|
|
31
32
|
"type": doc_type,
|
|
33
|
+
# `repo` is the display field (first membership); `repos` is the full
|
|
34
|
+
# membership list the `where` evaluator reads. Keeping only the singular
|
|
35
|
+
# form let a multi-repo page fail a filter naming its second repo.
|
|
32
36
|
"repo": (repos or [""])[0],
|
|
37
|
+
"repos": repos,
|
|
33
38
|
"category": category,
|
|
34
39
|
"tags": fm.get("tags", ""),
|
|
35
40
|
"summary": extract_summary(body),
|
|
@@ -75,6 +80,15 @@ def build_graph(vault):
|
|
|
75
80
|
return _cache[key]
|
|
76
81
|
|
|
77
82
|
|
|
83
|
+
def _tiebreak(doc_id):
|
|
84
|
+
"""Stable, path-insensitive ordering key for equidistant neighbours.
|
|
85
|
+
|
|
86
|
+
blake2b rather than `hash()`: the builtin is salted per process by
|
|
87
|
+
PYTHONHASHSEED, which is the non-determinism this exists to remove.
|
|
88
|
+
"""
|
|
89
|
+
return hashlib.blake2b(doc_id.encode("utf-8"), digest_size=8).digest()
|
|
90
|
+
|
|
91
|
+
|
|
78
92
|
def _bfs_neighbors(adjacency, seeds, hops):
|
|
79
93
|
"""Return {base_path: distance} reachable within `hops` from seeds (excluding seeds)."""
|
|
80
94
|
frontier = set(seeds)
|
|
@@ -94,14 +108,41 @@ def _bfs_neighbors(adjacency, seeds, hops):
|
|
|
94
108
|
return dist
|
|
95
109
|
|
|
96
110
|
|
|
97
|
-
def graph_stream(adjacency, seeds, hops=1, max_n=15):
|
|
111
|
+
def graph_stream(adjacency, seeds, hops=1, max_n=15, keep=None):
|
|
98
112
|
"""A rank-based stream of wikilink-neighbors of `seeds`, nearest first.
|
|
99
113
|
|
|
100
114
|
Returns [(doc_id, rank)] (rank 0-based, capped at max_n), excluding seeds —
|
|
101
115
|
shaped for rrf_fuse as a third retrieval stream. Empty if no neighbors.
|
|
116
|
+
|
|
117
|
+
`keep` is an optional predicate on doc_id applied BEFORE the max_n cut, so a
|
|
118
|
+
filtered query still gets a full window of admissible neighbours rather than
|
|
119
|
+
a window thinned by rejects. It carries the caller's `where` clause: without
|
|
120
|
+
it this stream was the one path into the fused set that no filter reached.
|
|
121
|
+
|
|
122
|
+
`keep` screens what is RETURNED, not what the walk may cross. With
|
|
123
|
+
`graph_hops > 1` a neighbour reachable only *through* a page the filter
|
|
124
|
+
rejects is still returned if it passes itself. That is deliberate — the
|
|
125
|
+
clause describes the answer set, not the path — but it means raising
|
|
126
|
+
`graph_hops` widens what a scoped query can reach. Moot at the current
|
|
127
|
+
`graph_hops = 1`, where there are no intermediate nodes.
|
|
102
128
|
"""
|
|
103
129
|
dist = _bfs_neighbors(adjacency, seeds, hops)
|
|
104
130
|
if not dist:
|
|
105
131
|
return []
|
|
106
|
-
|
|
132
|
+
# Tie-break deterministically, but NOT on the doc id itself. `dist` is
|
|
133
|
+
# populated by iterating adjacency *sets*, so equidistant neighbours arrive
|
|
134
|
+
# in an order that varies with PYTHONHASHSEED, and sorting on distance alone
|
|
135
|
+
# is stable -- it preserved that variation, leaving the stream irreproducible
|
|
136
|
+
# across processes. Ordering by doc id fixes that but buys a worse problem:
|
|
137
|
+
# at graph_hops=1 every neighbour is distance 1, so the id becomes the sole
|
|
138
|
+
# key and the max_n cut degenerates into "keep the lexicographically smallest
|
|
139
|
+
# N". `Notes/` sorts before `Projects/`, so scoped queries -- the ones that
|
|
140
|
+
# exist to surface a repo's Projects/ pages -- were the worst affected.
|
|
141
|
+
# Measured over 300 realistic 5-seed calls: truncation fired on 70% of them
|
|
142
|
+
# and lifted Notes/ from 46% of the candidates to 68% of the survivors.
|
|
143
|
+
# A keyed digest is reproducible across processes like the id, without
|
|
144
|
+
# correlating to the path prefix.
|
|
145
|
+
ordered = sorted(dist.items(), key=lambda kv: (kv[1], _tiebreak(kv[0]))) # nearest first
|
|
146
|
+
if keep is not None:
|
|
147
|
+
ordered = [item for item in ordered if keep(item[0])]
|
|
107
148
|
return [(doc_id, rank) for rank, (doc_id, _d) in enumerate(ordered[:max_n])]
|
|
@@ -89,6 +89,10 @@ def _dispatch(args) -> None:
|
|
|
89
89
|
raise PageError("CURSOR_MISMATCH", reason="cursor beyond span count")
|
|
90
90
|
|
|
91
91
|
find = getattr(args, "find", None)
|
|
92
|
+
find_only = bool(getattr(args, "find_only", False))
|
|
93
|
+
if find_only and not find:
|
|
94
|
+
raise PageError("FIND_ONLY_REQUIRES_FIND",
|
|
95
|
+
reason="--find-only needs --find <literal>")
|
|
92
96
|
find_matches = None
|
|
93
97
|
if find:
|
|
94
98
|
find_matches = []
|
|
@@ -122,8 +126,23 @@ def _dispatch(args) -> None:
|
|
|
122
126
|
}
|
|
123
127
|
if find_matches is not None:
|
|
124
128
|
obj["find_matches"] = find_matches
|
|
129
|
+
if find_only:
|
|
130
|
+
# Locate only: no cards, no map progress, no continuation.
|
|
131
|
+
obj["find_only"] = True
|
|
132
|
+
obj["next_cursor"] = None
|
|
125
133
|
return obj
|
|
126
134
|
|
|
135
|
+
if find_only:
|
|
136
|
+
obj, chars = finalize_page(
|
|
137
|
+
lambda pc, ua: envelope([], start, pc, ua), used_before)
|
|
138
|
+
if chars > max_chars:
|
|
139
|
+
raise PageError("PAGE_BUDGET_TOO_SMALL", max_chars=max_chars,
|
|
140
|
+
reason="find-only envelope exceeds budget")
|
|
141
|
+
dp.charge_and_save(state, chars)
|
|
142
|
+
assert len(render_page(obj)) + 1 == chars
|
|
143
|
+
emit(obj)
|
|
144
|
+
return
|
|
145
|
+
|
|
127
146
|
cards: list = []
|
|
128
147
|
idx = start
|
|
129
148
|
final_obj = None
|