cortex-vec 0.8.0__tar.gz → 0.10.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. {cortex_vec-0.8.0/src/cortex_vec.egg-info → cortex_vec-0.10.0}/PKG-INFO +1 -1
  2. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/pyproject.toml +1 -1
  3. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/bm25.py +59 -10
  4. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/cli.py +25 -5
  5. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/config.py +9 -10
  6. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/distill_plan.py +68 -2
  7. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/distill_queue.py +12 -1
  8. cortex_vec-0.10.0/src/cortex_vec/errors.py +30 -0
  9. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/fusion.py +58 -17
  10. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/graph.py +43 -2
  11. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/raw_map.py +19 -0
  12. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/raw_source.py +32 -7
  13. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/reclaim.py +111 -14
  14. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/store.py +124 -22
  15. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/tokenize.py +11 -1
  16. {cortex_vec-0.8.0 → cortex_vec-0.10.0/src/cortex_vec.egg-info}/PKG-INFO +1 -1
  17. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec.egg-info/SOURCES.txt +6 -0
  18. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_bm25.py +8 -6
  19. cortex_vec-0.10.0/tests/test_cli_errors.py +114 -0
  20. cortex_vec-0.10.0/tests/test_cli_json.py +138 -0
  21. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_cmd_search.py +33 -2
  22. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_cmd_search_flags.py +1 -1
  23. cortex_vec-0.10.0/tests/test_cosine_backfill.py +212 -0
  24. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_distill_plan.py +73 -0
  25. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_distill_queue.py +22 -0
  26. cortex_vec-0.10.0/tests/test_fusion_graph_filter.py +331 -0
  27. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_fusion_search.py +51 -2
  28. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_raw_map.py +28 -2
  29. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_raw_source.py +45 -14
  30. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_reclaim.py +126 -0
  31. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_tokenize.py +16 -0
  32. cortex_vec-0.10.0/tests/test_where_shape_lockstep.py +180 -0
  33. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/LICENSE +0 -0
  34. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/NOTICE +0 -0
  35. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/README.md +0 -0
  36. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/setup.cfg +0 -0
  37. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/__init__.py +0 -0
  38. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/eval/__init__.py +0 -0
  39. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/eval/adapters.py +0 -0
  40. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/eval/corpus.py +0 -0
  41. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/eval/report.py +0 -0
  42. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/eval/run.py +0 -0
  43. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/eval/score.py +0 -0
  44. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/parser.py +0 -0
  45. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/raw_page.py +0 -0
  46. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/raw_span.py +0 -0
  47. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/raw_view.py +0 -0
  48. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/rerank.py +0 -0
  49. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec/synonyms.py +0 -0
  50. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec.egg-info/dependency_links.txt +0 -0
  51. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec.egg-info/entry_points.txt +0 -0
  52. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec.egg-info/requires.txt +0 -0
  53. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/src/cortex_vec.egg-info/top_level.txt +0 -0
  54. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_adapters.py +0 -0
  55. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_bm25_synonym.py +0 -0
  56. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_config.py +0 -0
  57. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_config_plan2.py +0 -0
  58. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_corpus.py +0 -0
  59. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_fusion_diversify.py +0 -0
  60. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_fusion_graph.py +0 -0
  61. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_fusion_rerank.py +0 -0
  62. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_fusion_rrf.py +0 -0
  63. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_fusion_synonym.py +0 -0
  64. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_graph_build.py +0 -0
  65. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_graph_stream.py +0 -0
  66. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_lockstep.py +0 -0
  67. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_no_weekly_type.py +0 -0
  68. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_raw_page.py +0 -0
  69. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_raw_span.py +0 -0
  70. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_raw_view.py +0 -0
  71. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_rebuild_bm25_only.py +0 -0
  72. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_report.py +0 -0
  73. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_rerank.py +0 -0
  74. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_run.py +0 -0
  75. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_score.py +0 -0
  76. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_smoke.py +0 -0
  77. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_synonyms.py +0 -0
  78. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_vector_stream.py +0 -0
  79. {cortex_vec-0.8.0 → cortex_vec-0.10.0}/tests/test_wikilinks.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cortex-vec
3
- Version: 0.8.0
3
+ Version: 0.10.0
4
4
  Summary: Vector store CLI for the cortex vault — hybrid BM25 + embedding retrieval over a markdown knowledge vault
5
5
  Author: XBlueSky
6
6
  License-Expression: Apache-2.0
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "cortex-vec"
3
- version = "0.8.0"
3
+ version = "0.10.0"
4
4
  description = "Vector store CLI for the cortex vault — hybrid BM25 + embedding retrieval over a markdown knowledge vault"
5
5
  readme = "README.md"
6
6
  license = "Apache-2.0"
@@ -19,19 +19,65 @@ def _doc_record(doc):
19
19
  return rec
20
20
 
21
21
 
22
+ # Fields this evaluator models, each paired with the predicate that evaluates it.
23
+ # ChromaDB accepts any metadata key, so a clause on a key absent here would be
24
+ # honoured by the vector stream and ignored by the two that run through
25
+ # `_matches` -- one query language per stream again, which is exactly the defect
26
+ # the spec below documents. Adding a filter field means adding it here;
27
+ # `test_where_shape_lockstep` fails until you do.
28
+ #
29
+ # The name list and the comparisons are ONE table on purpose. Keeping them apart
30
+ # let a field be whitelisted as "known" while no branch ever compared it, which
31
+ # passes every record silently -- the same fail-open, one level further in.
32
+ _FIELD_MATCHERS = {
33
+ "repo": lambda rec, want: want in rec.get("repos", []),
34
+ "type": lambda rec, want: rec.get("type") == want,
35
+ "category": lambda rec, want: rec.get("category") == want,
36
+ }
37
+ _KNOWN_FIELDS = frozenset(_FIELD_MATCHERS)
38
+
39
+
22
40
  def _matches(rec, where):
41
+ """Evaluate a Chroma-style `where` clause against a stored record.
42
+
43
+ Understands exactly the shapes `store._build_where` emits: flat field
44
+ equality plus `$and` / `$or` composition. All three retrieval streams
45
+ (BM25, wikilink graph, and -- via ChromaDB -- vector) must read the same
46
+ query language; a flat-only matcher silently ignored the nested payload
47
+ and dropped the filter entirely. See
48
+ docs/specs/2026-05-27-distill-dedup-repo-filter-blindspot.md.
49
+
50
+ The Notes/-are-cross-repo exemption lives in the `$or` branch that
51
+ `_build_where` emits, not in a `type == "note"` special case here.
52
+
53
+ Raises ValueError on a clause this evaluator does not model, rather than
54
+ guessing. Silently returning True for an unknown key is how a filter goes
55
+ missing without anyone noticing; the callers wrap this in the same
56
+ degrade-to-empty-stream contract they use for every other failure, so a
57
+ drift fails closed in production and loudly in the tests.
58
+ """
23
59
  if not where:
24
60
  return True
25
- # Notes/ are cross-repo by design and must always pass the repo filter.
26
- # See docs/specs/2026-05-27-distill-dedup-repo-filter-blindspot.md.
27
- if ("repo" in where and rec.get("type") != "note"
28
- and where["repo"] not in rec.get("repos", [])):
29
- return False
30
- if "type" in where and rec.get("type") != where["type"]:
31
- return False
32
- if "category" in where and rec.get("category") != where["category"]:
33
- return False
34
- return True
61
+ if "$and" in where:
62
+ return all(_matches(rec, clause) for clause in where["$and"])
63
+ if "$or" in where:
64
+ return any(_matches(rec, clause) for clause in where["$or"])
65
+
66
+ unknown = set(where) - _KNOWN_FIELDS
67
+ if unknown:
68
+ raise ValueError(
69
+ f"_matches cannot evaluate {sorted(unknown)}: the filter producer and "
70
+ f"this evaluator have drifted. Add the field to _KNOWN_FIELDS and "
71
+ f"handle it below."
72
+ )
73
+ for field, expected in where.items():
74
+ if isinstance(expected, dict):
75
+ raise ValueError(
76
+ f"_matches does not implement operator form for {field!r}: "
77
+ f"{expected!r}. ChromaDB would honour it and this stream would not."
78
+ )
79
+
80
+ return all(_FIELD_MATCHERS[field](rec, want) for field, want in where.items())
35
81
 
36
82
 
37
83
  class BM25Index:
@@ -75,6 +121,9 @@ class BM25Index:
75
121
  with open(self._file, "wb") as f:
76
122
  pickle.dump(self.docs, f)
77
123
 
124
+ def exists(self):
125
+ return self._file.exists()
126
+
78
127
  def load(self):
79
128
  if not self._file.exists():
80
129
  raise FileNotFoundError(f"BM25 index not found at {self._file}; run rebuild")
@@ -3,26 +3,40 @@
3
3
  import argparse
4
4
  import sys
5
5
 
6
+ from .errors import EXIT_CALLER, CliError
7
+
6
8
 
7
9
  def main():
10
+ try:
11
+ _main()
12
+ except CliError as e:
13
+ print(e.render(), file=sys.stderr)
14
+ sys.exit(e.code)
15
+
16
+
17
+ def _main():
8
18
  parser = argparse.ArgumentParser(
9
19
  description="cortex-vec — Vector store CLI for cortex vault"
10
20
  )
11
21
  sub = parser.add_subparsers(dest="command")
12
22
 
13
- sub.add_parser("status", help="Show index health")
23
+ p_status = sub.add_parser("status", help="Show index health")
24
+ p_status.add_argument("--json", action="store_true", help="Emit one JSON document instead of human text")
14
25
  p_rebuild = sub.add_parser("rebuild", help="Full rebuild from vault")
15
26
  p_rebuild.add_argument(
16
27
  "--bm25-only",
17
28
  action="store_true",
18
29
  help="Rebuild only the BM25 index from the vault (no re-embedding, free)",
19
30
  )
31
+ p_rebuild.add_argument("--json", action="store_true", help="Emit one JSON document instead of human text")
20
32
 
21
33
  p_upsert = sub.add_parser("upsert", help="Add/update a document")
22
34
  p_upsert.add_argument("path", help="Relative path from vault root")
35
+ p_upsert.add_argument("--json", action="store_true", help="Emit one JSON document instead of human text")
23
36
 
24
37
  p_delete = sub.add_parser("delete", help="Remove a document")
25
38
  p_delete.add_argument("path", help="Relative path from vault root")
39
+ p_delete.add_argument("--json", action="store_true", help="Emit one JSON document instead of human text")
26
40
 
27
41
  p_search = sub.add_parser("search", help="Semantic search")
28
42
  p_search.add_argument("query", help="Search query text")
@@ -51,7 +65,7 @@ def main():
51
65
  p_dq.add_argument("--stat", action="store_true",
52
66
  help="Show per-level projected sizes instead of paths")
53
67
  p_dq.add_argument("--json", action="store_true",
54
- help="With --stat: emit JSON rows")
68
+ help="Emit a JSON array (of paths, or of --stat rows)")
55
69
  p_bq = sub.add_parser(
56
70
  "broadcast-queue", help="List distilled Raw eligible for broadcast"
57
71
  )
@@ -88,7 +102,13 @@ def main():
88
102
  p_rm.add_argument("--cursor", help="Continuation cursor from a prior page")
89
103
  p_rm.add_argument("--max-chars", type=int, dest="max_chars",
90
104
  help="Page stdout cap (default: config, 12000)")
91
- p_rm.add_argument("--find", help="Exact literal to locate in the Raw")
105
+ p_rm.add_argument("--find",
106
+ help="Exact literal to locate: reports the matching span "
107
+ "ids in `find_matches`; does NOT move the page "
108
+ "(cards still start at 0 / --cursor)")
109
+ p_rm.add_argument("--find-only", action="store_true", dest="find_only",
110
+ help="With --find: emit only `find_matches` (no cards, "
111
+ "no map progress); costs one envelope, not a page")
92
112
 
93
113
  p_rsp = sub.add_parser(
94
114
  "raw-span", help="Bounded original-source page of one span"
@@ -121,8 +141,8 @@ def main():
121
141
  args = parser.parse_args()
122
142
 
123
143
  if not args.command:
124
- parser.print_help()
125
- sys.exit(1)
144
+ parser.print_help(sys.stderr)
145
+ sys.exit(EXIT_CALLER)
126
146
 
127
147
  # Fast path: trailer-anchored queue scans skip the heavy store import
128
148
  # (chromadb ~2.7s). They read only each Raw's last non-empty line.
@@ -1,9 +1,10 @@
1
1
  """Configuration loading and constants."""
2
2
 
3
3
  import json
4
- import sys
5
4
  from pathlib import Path
6
5
 
6
+ from .errors import EXIT_SETUP, CliError
7
+
7
8
  CORTEX_CONFIG = Path.home() / ".cortex" / "config.json"
8
9
  VECTORSTORE_DIR = Path.home() / ".cortex" / "vectorstore"
9
10
  COLLECTION_NAME = "cortex"
@@ -29,24 +30,22 @@ _RETRIEVAL_DEFAULTS = {
29
30
 
30
31
 
31
32
  def load_config():
32
- """Load ~/.cortex/config.json. Exit with error if not found."""
33
+ """Load ~/.cortex/config.json; raise CliError if it is missing."""
33
34
  if not CORTEX_CONFIG.exists():
34
- print(
35
- "Error: ~/.cortex/config.json not found. Run /cortexes:genesis first.",
36
- file=sys.stderr,
37
- )
38
- sys.exit(1)
35
+ raise CliError(f"{CORTEX_CONFIG} not found", "run /cortexes:genesis first",
36
+ code=EXIT_SETUP)
39
37
  with open(CORTEX_CONFIG) as f:
40
38
  return json.load(f)
41
39
 
42
40
 
43
41
  def get_vault_path():
44
- """Return the vault path from config. Exit with error if invalid."""
42
+ """Return the vault path from config; raise CliError if it is invalid."""
45
43
  cfg = load_config()
46
44
  vault = cfg.get("vault_path", "")
47
45
  if not vault or not Path(vault).is_dir():
48
- print(f"Error: vault_path '{vault}' not found.", file=sys.stderr)
49
- sys.exit(1)
46
+ raise CliError(f"vault_path '{vault}' is not a directory",
47
+ f"fix vault_path in {CORTEX_CONFIG} or rerun /cortexes:genesis",
48
+ code=EXIT_SETUP)
50
49
  return Path(vault)
51
50
 
52
51
 
@@ -8,6 +8,7 @@ fail-closed on corruption or identity drift.
8
8
  """
9
9
  from __future__ import annotations
10
10
 
11
+ import hashlib
11
12
  import json
12
13
  import os
13
14
  from datetime import datetime, timezone
@@ -319,13 +320,78 @@ def seal_plan(state: dict, expected_outcome: str) -> None:
319
320
  _save(state)
320
321
 
321
322
 
323
+ def _normalization_variants(text: str) -> list:
324
+ """Alternative marker strippings of ``text``, for attributing a mismatch.
325
+
326
+ Each entry is ``(variant_name, stripped_text)``. Only the error path calls
327
+ this, so a whole-file sha256 per variant is affordable; what costs
328
+ something is naming a rule that never existed, because that would
329
+ attribute a real edit to a phantom normalization. Every rule below either
330
+ shipped once or is something a writer could plausibly emit next:
331
+
332
+ * ``marker_line_only`` -- the marker line dropped, its blank separator
333
+ kept. This is exactly what the header branch did before 1.8.1, so a
334
+ regression of that wedge names itself instead of coming back
335
+ disguised as "source content changed".
336
+ * ``unstripped`` -- no marker removal at all, i.e. the plan recorded a
337
+ source that already carried a marker.
338
+ * ``final_newline_*`` -- the current strip with the trailing newline
339
+ added or dropped, for a writer that changes its mind about the last
340
+ byte (14 Raws were appended without one).
341
+
342
+ The marker scan here is deliberately position-blind: a quoted marker in
343
+ the conversation may be picked instead of the anchored one, which makes a
344
+ variant that simply fails to match -- never a wrong attribution.
345
+ """
346
+ stripped = rs.strip_state_marker(text)
347
+ lines = text.split("\n")
348
+ first = next((i for i, ln in enumerate(lines) if rs.is_marker_line(ln)), None)
349
+ candidates = []
350
+ if first is not None:
351
+ candidates.append(("marker_line_only",
352
+ "\n".join(lines[:first] + lines[first + 1:])))
353
+ candidates.append(("unstripped", text))
354
+ candidates.append(("final_newline_added", stripped + "\n"))
355
+ candidates.append(("final_newline_dropped", stripped.rstrip("\n")))
356
+ seen = {stripped} # the rule in force already failed to match
357
+ out = []
358
+ for name, variant in candidates:
359
+ if variant not in seen:
360
+ seen.add(variant)
361
+ out.append((name, variant))
362
+ return out
363
+
364
+
365
+ def _diagnose_source_mismatch(text: str, rec: dict) -> dict:
366
+ """Name why ``text`` no longer hashes to the plan's recorded source.
367
+
368
+ A plan stores hashes, not the pre-marker text, so a mismatch cannot be
369
+ diffed -- but it can be attributed. If some other stripping rule
370
+ reproduces the recorded source hash, the conversation is untouched and
371
+ only the marker accounting differs; if none does, the file really was
372
+ edited and "source content changed" is the honest answer. Conflating the
373
+ two is what sent a whole afternoon looking for an editor that had never
374
+ touched the file.
375
+ """
376
+ for name, variant in _normalization_variants(text):
377
+ digest = hashlib.sha256(variant.encode("utf-8")).hexdigest()
378
+ if digest == rec["source_sha256"]:
379
+ return {"reason": "marker normalization mismatch",
380
+ "normalization": name,
381
+ "hint": "the marker block differs, not the conversation"}
382
+ return {"reason": "source content changed",
383
+ "chars_now": len(text),
384
+ "chars_at_plan_start": rec["char_count"]}
385
+
386
+
322
387
  def complete_plan(plan_id: str, raw_path: str) -> dict:
323
388
  state = _load_by_plan_id(plan_id)
324
389
  if state["status"] != "sealed":
325
390
  raise PageError("PLAN_NOT_ACTIVE", reason="seal before complete")
326
- ident, _ = rs.load(raw_path)
391
+ ident, text = rs.load(raw_path)
327
392
  if ident.source_sha256 != state["identity"]["source_sha256"]:
328
- raise PageError("RAW_CHANGED", reason="source content changed")
393
+ raise PageError("RAW_CHANGED",
394
+ **_diagnose_source_mismatch(text, state["identity"]))
329
395
  from .distill_queue import classify
330
396
  outcome = classify(raw_path).outcome
331
397
  if outcome != state["expected_outcome"]:
@@ -18,6 +18,7 @@ inside the conversation are quotes and ignored.
18
18
  from __future__ import annotations
19
19
 
20
20
  import re
21
+ import sys
21
22
  from dataclasses import dataclass
22
23
  from pathlib import Path
23
24
 
@@ -142,7 +143,14 @@ def dispatch_raw_state(path) -> None:
142
143
 
143
144
  def dispatch_distill_queue(root, stat=False, as_json=False) -> None:
144
145
  files = distill_queue(root)
146
+ if not stat and as_json:
147
+ import json
148
+
149
+ print(json.dumps([str(p) for p in files], ensure_ascii=False))
150
+ return
145
151
  if not stat:
152
+ if not files:
153
+ print("cortex-vec: distill queue is empty", file=sys.stderr)
146
154
  for p in files:
147
155
  print(p)
148
156
  return
@@ -167,5 +175,8 @@ def dispatch_distill_queue(root, stat=False, as_json=False) -> None:
167
175
 
168
176
 
169
177
  def dispatch_broadcast_queue(root) -> None:
170
- for p in broadcast_queue(root):
178
+ files = broadcast_queue(root)
179
+ if not files:
180
+ print("cortex-vec: broadcast queue is empty", file=sys.stderr)
181
+ for p in files:
171
182
  print(p)
@@ -0,0 +1,30 @@
1
+ """Agent-facing failure contract for the cortex-vec CLI.
2
+
3
+ Callers are mostly agents, which choose their next step from the exit code:
4
+
5
+ 0 success, including an empty result
6
+ 1 unexpected failure (an uncaught traceback exits 1 too)
7
+ 2 caller error: fix the command and retry (argparse and raw-page errors
8
+ already use 2)
9
+ 3 environment not set up: stop and tell the user, or run the named setup
10
+
11
+ `cli.main` catches CliError and prints it as one stderr line that names the
12
+ fix. The raw-map / raw-span family keeps its own JSON-on-stdout envelope.
13
+ """
14
+
15
+ EXIT_CALLER = 2
16
+ EXIT_SETUP = 3
17
+
18
+
19
+ class CliError(Exception):
20
+ """A failure the caller can act on; `hint` says how."""
21
+
22
+ def __init__(self, message, hint="", code=EXIT_CALLER):
23
+ super().__init__(message)
24
+ self.message = message
25
+ self.hint = hint
26
+ self.code = code
27
+
28
+ def render(self):
29
+ line = f"cortex-vec: error: {self.message}"
30
+ return f"{line} ({self.hint})" if self.hint else line
@@ -26,26 +26,37 @@ def rrf_fuse(ranked, weights, k=60):
26
26
  return sorted(scores.items(), key=lambda kv: kv[1], reverse=True)
27
27
 
28
28
 
29
- def _bm25_stream(query, n, where, synonym_weight=0.0):
29
+ def _note(notes, message):
30
+ """Record a degradation for the caller to surface (None = nobody asked)."""
31
+ if notes is not None:
32
+ notes.append(message)
33
+
34
+
35
+ def _bm25_stream(query, n, where, synonym_weight=0.0, notes=None):
30
36
  """Load the persisted BM25 index and search; return [] on any failure."""
31
37
  from . import bm25
32
38
  try:
33
39
  idx = bm25.BM25Index(BM25_DIR)
34
40
  idx.load()
35
41
  return idx.search(query, n, where, synonym_weight=synonym_weight)
36
- except Exception:
42
+ except Exception as e:
43
+ _note(notes, f"bm25 stream failed ({type(e).__name__}: {e}); "
44
+ "results are vector-only")
37
45
  return []
38
46
 
39
47
 
40
- def _vector_stream(query, n, where):
48
+ def _vector_stream(query, n, where, notes=None):
41
49
  # No API key -> embeddings unavailable; skip vector so the query degrades to
42
- # BM25-only. (The embedding fn hard-exits via sys.exit on a missing key, and
43
- # SystemExit is not caught by `except Exception`, so we must check up front.)
50
+ # BM25-only without building the embedding fn (which raises CliError).
44
51
  if not os.environ.get("OPENAI_API_KEY"):
52
+ _note(notes, "vector stream skipped (OPENAI_API_KEY not set); "
53
+ "results are BM25-only")
45
54
  return []
46
55
  try:
47
56
  return store.vector_stream(query, n, where)
48
- except Exception:
57
+ except Exception as e:
58
+ _note(notes, f"vector stream failed ({type(e).__name__}: {e}); "
59
+ "results are BM25-only")
49
60
  return []
50
61
 
51
62
 
@@ -53,7 +64,7 @@ def _vector_stream(query, n, where):
53
64
  _STREAM_ORDER = ("vector", "bm25")
54
65
 
55
66
 
56
- def _graph_ranked(ranked, weights, rc, n):
67
+ def _graph_ranked(ranked, weights, rc, n, where=None):
57
68
  """Build the graph RRF stream: wikilink-neighbors of the top vector/bm25 hits.
58
69
 
59
70
  Returns (ranked_list, display_map) where ranked_list is [(doc_id, rank)] for
@@ -61,7 +72,14 @@ def _graph_ranked(ranked, weights, rc, n):
61
72
  ([], {}) on failure (e.g. vault unconfigured) or when graph yields nothing.
62
73
  Rank-based (not additive), so it composes with the other streams via RRF
63
74
  without the score-scale conflict that the old additive boost had.
75
+
76
+ `where` is evaluated with the same `bm25._matches` the keyword stream uses:
77
+ three streams, one filter language. This stream used to receive no `where`
78
+ at all, so wikilink neighbours entered the fused set past `--repo`/`--type`/
79
+ `--category`. A doc with no metadata is rejected rather than admitted — we
80
+ cannot show it passed the filter.
64
81
  """
82
+ from . import bm25 as bm25_mod
65
83
  from . import graph as graph_mod
66
84
  try:
67
85
  pre = rrf_fuse(ranked, weights, k=rc["rrf_k"])
@@ -69,10 +87,20 @@ def _graph_ranked(ranked, weights, rc, n):
69
87
  if not seeds:
70
88
  return [], {}
71
89
  adjacency, meta = graph_mod.build_graph(get_vault_path())
90
+ keep = None
91
+ if where:
92
+ def keep(doc_id):
93
+ rec = meta.get(doc_id)
94
+ return rec is not None and bm25_mod._matches(rec, where)
72
95
  stream = graph_mod.graph_stream(
73
- adjacency, seeds, hops=rc["graph_hops"], max_n=max(n * 2, 10)
96
+ adjacency, seeds, hops=rc["graph_hops"], max_n=max(n * 2, 10), keep=keep
74
97
  )
75
- gdisplay = {doc_id: meta[doc_id] for doc_id, _ in stream if doc_id in meta}
98
+ # `repos` is a filter field, not a display field: the other streams do
99
+ # not emit it, so it must not reach the result dicts from this one.
100
+ gdisplay = {
101
+ doc_id: {k: v for k, v in meta[doc_id].items() if k != "repos"}
102
+ for doc_id, _ in stream if doc_id in meta
103
+ }
76
104
  return stream, gdisplay
77
105
  except (Exception, SystemExit):
78
106
  return [], {}
@@ -100,20 +128,23 @@ def _diversify(fused, display, max_per_repo):
100
128
  return primary + overflow
101
129
 
102
130
 
103
- def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, rerank=None):
131
+ def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, rerank=None,
132
+ notes=None):
104
133
  """Hybrid search entry point. Returns up to n display dicts (best-first).
105
134
 
106
135
  Gracefully degrades: if a stream errors or is disabled, the other carries
107
- the query (RRF weight is redistributed in rrf_fuse).
136
+ the query (RRF weight is redistributed in rrf_fuse). Pass a list as
137
+ `notes` to receive one message per enabled stream that degraded.
108
138
  """
109
139
  rc = get_retrieval_config()
110
140
  weights = {"vector": rc["w_vec"], "bm25": rc["w_bm25"], "graph": rc["w_graph"]}
111
141
 
112
142
  streams = {}
113
143
  if use_vector:
114
- streams["vector"] = _vector_stream(query, n, where)
144
+ streams["vector"] = _vector_stream(query, n, where, notes=notes)
115
145
  if use_bm25:
116
- streams["bm25"] = _bm25_stream(query, n, where, synonym_weight=rc["synonym_weight"])
146
+ streams["bm25"] = _bm25_stream(query, n, where,
147
+ synonym_weight=rc["synonym_weight"], notes=notes)
117
148
 
118
149
  ranked = {}
119
150
  display = {}
@@ -133,7 +164,7 @@ def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, r
133
164
 
134
165
  use_graph = rc["graph"] if graph is None else graph
135
166
  if use_graph:
136
- g_ranked, g_display = _graph_ranked(ranked, weights, rc, n)
167
+ g_ranked, g_display = _graph_ranked(ranked, weights, rc, n, where)
137
168
  if g_ranked:
138
169
  ranked["graph"] = g_ranked
139
170
  for doc_id, disp in g_display.items():
@@ -150,14 +181,24 @@ def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, r
150
181
  use_rerank = rc["rerank"] if rerank is None else rerank
151
182
  take = max(n, rc["rerank_window"]) if use_rerank else n
152
183
 
184
+ head = fused[:take]
185
+ # Hits that arrived via bm25 or graph never passed through the vector
186
+ # top-n window, so they carry no cosine. Ask the collection for theirs
187
+ # rather than reporting 0.0, which reads as "no overlap" downstream.
188
+ missing = [doc_id for doc_id, _ in head if doc_id not in vec_score]
189
+ if missing and use_vector and os.environ.get("OPENAI_API_KEY"):
190
+ try:
191
+ vec_score.update(store.cosine_for(query, missing))
192
+ except (Exception, SystemExit):
193
+ pass # same degradation contract as the streams: score falls to 0.0
194
+
153
195
  out = []
154
- for doc_id, _rrf in fused[:take]:
196
+ for doc_id, _rrf in head:
155
197
  entry = dict(display.get(doc_id, {}))
156
198
  entry["id"] = doc_id
157
199
  # `score` reports the vector cosine similarity (0-1, interpretable) so
158
200
  # absolute-threshold consumers (distill/broadcast dedup) keep working.
159
- # RRF only determines ORDER (already baked into `fused`). bm25-only hits
160
- # have no cosine score -> 0.0.
201
+ # RRF only determines ORDER (already baked into `fused`).
161
202
  entry["score"] = round(vec_score.get(doc_id, 0.0), 4)
162
203
  out.append(entry)
163
204
 
@@ -11,6 +11,7 @@ Graph participates in retrieval as a THIRD RRF stream (see fusion.py): a
11
11
  rank-based list of wikilink-neighbors of the top hits. Rank-based fusion avoids
12
12
  the scale conflict of adding a boost onto RRF's compressed score band.
13
13
  """
14
+ import hashlib
14
15
  from pathlib import Path
15
16
 
16
17
  from .parser import classify_path, extract_summary, extract_wikilinks, parse_document
@@ -29,7 +30,11 @@ def _meta_for(rel, fm, body):
29
30
  "id": rel,
30
31
  "title": fm.get("title", rel.rsplit("/", 1)[-1].removesuffix(".md")),
31
32
  "type": doc_type,
33
+ # `repo` is the display field (first membership); `repos` is the full
34
+ # membership list the `where` evaluator reads. Keeping only the singular
35
+ # form let a multi-repo page fail a filter naming its second repo.
32
36
  "repo": (repos or [""])[0],
37
+ "repos": repos,
33
38
  "category": category,
34
39
  "tags": fm.get("tags", ""),
35
40
  "summary": extract_summary(body),
@@ -75,6 +80,15 @@ def build_graph(vault):
75
80
  return _cache[key]
76
81
 
77
82
 
83
+ def _tiebreak(doc_id):
84
+ """Stable, path-insensitive ordering key for equidistant neighbours.
85
+
86
+ blake2b rather than `hash()`: the builtin is salted per process by
87
+ PYTHONHASHSEED, which is the non-determinism this exists to remove.
88
+ """
89
+ return hashlib.blake2b(doc_id.encode("utf-8"), digest_size=8).digest()
90
+
91
+
78
92
  def _bfs_neighbors(adjacency, seeds, hops):
79
93
  """Return {base_path: distance} reachable within `hops` from seeds (excluding seeds)."""
80
94
  frontier = set(seeds)
@@ -94,14 +108,41 @@ def _bfs_neighbors(adjacency, seeds, hops):
94
108
  return dist
95
109
 
96
110
 
97
- def graph_stream(adjacency, seeds, hops=1, max_n=15):
111
+ def graph_stream(adjacency, seeds, hops=1, max_n=15, keep=None):
98
112
  """A rank-based stream of wikilink-neighbors of `seeds`, nearest first.
99
113
 
100
114
  Returns [(doc_id, rank)] (rank 0-based, capped at max_n), excluding seeds —
101
115
  shaped for rrf_fuse as a third retrieval stream. Empty if no neighbors.
116
+
117
+ `keep` is an optional predicate on doc_id applied BEFORE the max_n cut, so a
118
+ filtered query still gets a full window of admissible neighbours rather than
119
+ a window thinned by rejects. It carries the caller's `where` clause: without
120
+ it this stream was the one path into the fused set that no filter reached.
121
+
122
+ `keep` screens what is RETURNED, not what the walk may cross. With
123
+ `graph_hops > 1` a neighbour reachable only *through* a page the filter
124
+ rejects is still returned if it passes itself. That is deliberate — the
125
+ clause describes the answer set, not the path — but it means raising
126
+ `graph_hops` widens what a scoped query can reach. Moot at the current
127
+ `graph_hops = 1`, where there are no intermediate nodes.
102
128
  """
103
129
  dist = _bfs_neighbors(adjacency, seeds, hops)
104
130
  if not dist:
105
131
  return []
106
- ordered = sorted(dist.items(), key=lambda kv: kv[1]) # nearest first
132
+ # Tie-break deterministically, but NOT on the doc id itself. `dist` is
133
+ # populated by iterating adjacency *sets*, so equidistant neighbours arrive
134
+ # in an order that varies with PYTHONHASHSEED, and sorting on distance alone
135
+ # is stable -- it preserved that variation, leaving the stream irreproducible
136
+ # across processes. Ordering by doc id fixes that but buys a worse problem:
137
+ # at graph_hops=1 every neighbour is distance 1, so the id becomes the sole
138
+ # key and the max_n cut degenerates into "keep the lexicographically smallest
139
+ # N". `Notes/` sorts before `Projects/`, so scoped queries -- the ones that
140
+ # exist to surface a repo's Projects/ pages -- were the worst affected.
141
+ # Measured over 300 realistic 5-seed calls: truncation fired on 70% of them
142
+ # and lifted Notes/ from 46% of the candidates to 68% of the survivors.
143
+ # A keyed digest is reproducible across processes like the id, without
144
+ # correlating to the path prefix.
145
+ ordered = sorted(dist.items(), key=lambda kv: (kv[1], _tiebreak(kv[0]))) # nearest first
146
+ if keep is not None:
147
+ ordered = [item for item in ordered if keep(item[0])]
107
148
  return [(doc_id, rank) for rank, (doc_id, _d) in enumerate(ordered[:max_n])]
@@ -89,6 +89,10 @@ def _dispatch(args) -> None:
89
89
  raise PageError("CURSOR_MISMATCH", reason="cursor beyond span count")
90
90
 
91
91
  find = getattr(args, "find", None)
92
+ find_only = bool(getattr(args, "find_only", False))
93
+ if find_only and not find:
94
+ raise PageError("FIND_ONLY_REQUIRES_FIND",
95
+ reason="--find-only needs --find <literal>")
92
96
  find_matches = None
93
97
  if find:
94
98
  find_matches = []
@@ -122,8 +126,23 @@ def _dispatch(args) -> None:
122
126
  }
123
127
  if find_matches is not None:
124
128
  obj["find_matches"] = find_matches
129
+ if find_only:
130
+ # Locate only: no cards, no map progress, no continuation.
131
+ obj["find_only"] = True
132
+ obj["next_cursor"] = None
125
133
  return obj
126
134
 
135
+ if find_only:
136
+ obj, chars = finalize_page(
137
+ lambda pc, ua: envelope([], start, pc, ua), used_before)
138
+ if chars > max_chars:
139
+ raise PageError("PAGE_BUDGET_TOO_SMALL", max_chars=max_chars,
140
+ reason="find-only envelope exceeds budget")
141
+ dp.charge_and_save(state, chars)
142
+ assert len(render_page(obj)) + 1 == chars
143
+ emit(obj)
144
+ return
145
+
127
146
  cards: list = []
128
147
  idx = start
129
148
  final_obj = None