cortex-vec 0.7.0__tar.gz → 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. {cortex_vec-0.7.0/src/cortex_vec.egg-info → cortex_vec-0.9.0}/PKG-INFO +5 -5
  2. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/README.md +4 -4
  3. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/pyproject.toml +1 -1
  4. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/bm25.py +56 -10
  5. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/cli.py +8 -2
  6. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/config.py +1 -1
  7. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/distill_plan.py +68 -2
  8. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/fusion.py +34 -7
  9. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/graph.py +43 -2
  10. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/parser.py +4 -2
  11. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/raw_map.py +19 -0
  12. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/raw_source.py +32 -7
  13. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/reclaim.py +111 -14
  14. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/store.py +47 -0
  15. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/synonyms.py +2 -0
  16. {cortex_vec-0.7.0 → cortex_vec-0.9.0/src/cortex_vec.egg-info}/PKG-INFO +5 -5
  17. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec.egg-info/SOURCES.txt +4 -0
  18. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_bm25.py +8 -6
  19. cortex_vec-0.9.0/tests/test_cosine_backfill.py +212 -0
  20. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_distill_plan.py +73 -0
  21. cortex_vec-0.9.0/tests/test_fusion_graph_filter.py +331 -0
  22. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_fusion_search.py +5 -2
  23. cortex_vec-0.9.0/tests/test_no_weekly_type.py +39 -0
  24. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_raw_map.py +28 -2
  25. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_raw_source.py +45 -14
  26. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_reclaim.py +126 -0
  27. cortex_vec-0.9.0/tests/test_where_shape_lockstep.py +180 -0
  28. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/LICENSE +0 -0
  29. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/NOTICE +0 -0
  30. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/setup.cfg +0 -0
  31. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/__init__.py +0 -0
  32. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/distill_queue.py +0 -0
  33. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/eval/__init__.py +0 -0
  34. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/eval/adapters.py +0 -0
  35. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/eval/corpus.py +0 -0
  36. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/eval/report.py +0 -0
  37. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/eval/run.py +0 -0
  38. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/eval/score.py +0 -0
  39. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/raw_page.py +0 -0
  40. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/raw_span.py +0 -0
  41. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/raw_view.py +0 -0
  42. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/rerank.py +0 -0
  43. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec/tokenize.py +0 -0
  44. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec.egg-info/dependency_links.txt +0 -0
  45. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec.egg-info/entry_points.txt +0 -0
  46. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec.egg-info/requires.txt +0 -0
  47. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/src/cortex_vec.egg-info/top_level.txt +0 -0
  48. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_adapters.py +0 -0
  49. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_bm25_synonym.py +0 -0
  50. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_cmd_search.py +0 -0
  51. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_cmd_search_flags.py +0 -0
  52. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_config.py +0 -0
  53. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_config_plan2.py +0 -0
  54. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_corpus.py +0 -0
  55. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_distill_queue.py +0 -0
  56. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_fusion_diversify.py +0 -0
  57. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_fusion_graph.py +0 -0
  58. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_fusion_rerank.py +0 -0
  59. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_fusion_rrf.py +0 -0
  60. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_fusion_synonym.py +0 -0
  61. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_graph_build.py +0 -0
  62. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_graph_stream.py +0 -0
  63. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_lockstep.py +0 -0
  64. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_raw_page.py +0 -0
  65. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_raw_span.py +0 -0
  66. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_raw_view.py +0 -0
  67. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_rebuild_bm25_only.py +0 -0
  68. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_report.py +0 -0
  69. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_rerank.py +0 -0
  70. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_run.py +0 -0
  71. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_score.py +0 -0
  72. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_smoke.py +0 -0
  73. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_synonyms.py +0 -0
  74. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_tokenize.py +0 -0
  75. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_vector_stream.py +0 -0
  76. {cortex_vec-0.7.0 → cortex_vec-0.9.0}/tests/test_wikilinks.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cortex-vec
3
- Version: 0.7.0
3
+ Version: 0.9.0
4
4
  Summary: Vector store CLI for the cortex vault — hybrid BM25 + embedding retrieval over a markdown knowledge vault
5
5
  Author: XBlueSky
6
6
  License-Expression: Apache-2.0
@@ -31,7 +31,7 @@ Dynamic: license-file
31
31
 
32
32
  # cortex-vec
33
33
 
34
- Vector store CLI for the [cortex](https://github.com/XBlueSky/cortexes)
34
+ Vector store CLI for the [cortexes](https://github.com/XBlueSky/cortexes)
35
35
  knowledge vault — hybrid retrieval over a plain-markdown vault, fusing a
36
36
  persistent BM25 index with OpenAI-embedding vector search (ChromaDB) via
37
37
  Reciprocal Rank Fusion. Handles mixed Chinese/English content.
@@ -58,11 +58,11 @@ cortex-vec search "nginx certificate" --n 5
58
58
  cortex-vec status # index health: vector + BM25 entry counts
59
59
  ```
60
60
 
61
- Configuration lives in `~/.cortex/config.json` (created by the cortex
62
- plugin's `/cortex:genesis` command). `OPENAI_API_KEY` is required for
61
+ Configuration lives in `~/.cortex/config.json` (created by the Cortexes
62
+ plugin's `/cortexes:genesis` command). `OPENAI_API_KEY` is required for
63
63
  embeddings; without it, `search` automatically degrades to BM25-only.
64
64
 
65
65
  This package is the retrieval backend of the
66
- [Cortex Claude Code plugin](https://github.com/XBlueSky/cortexes) — session
66
+ [Cortexes Claude Code plugin](https://github.com/XBlueSky/cortexes) — session
67
67
  recording, memory distillation, and indexed retrieval for Claude Code. See
68
68
  the repository for the full documentation.
@@ -1,6 +1,6 @@
1
1
  # cortex-vec
2
2
 
3
- Vector store CLI for the [cortex](https://github.com/XBlueSky/cortexes)
3
+ Vector store CLI for the [cortexes](https://github.com/XBlueSky/cortexes)
4
4
  knowledge vault — hybrid retrieval over a plain-markdown vault, fusing a
5
5
  persistent BM25 index with OpenAI-embedding vector search (ChromaDB) via
6
6
  Reciprocal Rank Fusion. Handles mixed Chinese/English content.
@@ -27,11 +27,11 @@ cortex-vec search "nginx certificate" --n 5
27
27
  cortex-vec status # index health: vector + BM25 entry counts
28
28
  ```
29
29
 
30
- Configuration lives in `~/.cortex/config.json` (created by the cortex
31
- plugin's `/cortex:genesis` command). `OPENAI_API_KEY` is required for
30
+ Configuration lives in `~/.cortex/config.json` (created by the Cortexes
31
+ plugin's `/cortexes:genesis` command). `OPENAI_API_KEY` is required for
32
32
  embeddings; without it, `search` automatically degrades to BM25-only.
33
33
 
34
34
  This package is the retrieval backend of the
35
- [Cortex Claude Code plugin](https://github.com/XBlueSky/cortexes) — session
35
+ [Cortexes Claude Code plugin](https://github.com/XBlueSky/cortexes) — session
36
36
  recording, memory distillation, and indexed retrieval for Claude Code. See
37
37
  the repository for the full documentation.
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "cortex-vec"
3
- version = "0.7.0"
3
+ version = "0.9.0"
4
4
  description = "Vector store CLI for the cortex vault — hybrid BM25 + embedding retrieval over a markdown knowledge vault"
5
5
  readme = "README.md"
6
6
  license = "Apache-2.0"
@@ -19,19 +19,65 @@ def _doc_record(doc):
19
19
  return rec
20
20
 
21
21
 
22
+ # Fields this evaluator models, each paired with the predicate that evaluates it.
23
+ # ChromaDB accepts any metadata key, so a clause on a key absent here would be
24
+ # honoured by the vector stream and ignored by the two that run through
25
+ # `_matches` -- one query language per stream again, which is exactly the defect
26
+ # the spec below documents. Adding a filter field means adding it here;
27
+ # `test_where_shape_lockstep` fails until you do.
28
+ #
29
+ # The name list and the comparisons are ONE table on purpose. Keeping them apart
30
+ # let a field be whitelisted as "known" while no branch ever compared it, which
31
+ # passes every record silently -- the same fail-open, one level further in.
32
+ _FIELD_MATCHERS = {
33
+ "repo": lambda rec, want: want in rec.get("repos", []),
34
+ "type": lambda rec, want: rec.get("type") == want,
35
+ "category": lambda rec, want: rec.get("category") == want,
36
+ }
37
+ _KNOWN_FIELDS = frozenset(_FIELD_MATCHERS)
38
+
39
+
22
40
  def _matches(rec, where):
41
+ """Evaluate a Chroma-style `where` clause against a stored record.
42
+
43
+ Understands exactly the shapes `store._build_where` emits: flat field
44
+ equality plus `$and` / `$or` composition. All three retrieval streams
45
+ (BM25, wikilink graph, and -- via ChromaDB -- vector) must read the same
46
+ query language; a flat-only matcher silently ignored the nested payload
47
+ and dropped the filter entirely. See
48
+ docs/specs/2026-05-27-distill-dedup-repo-filter-blindspot.md.
49
+
50
+ The Notes/-are-cross-repo exemption lives in the `$or` branch that
51
+ `_build_where` emits, not in a `type == "note"` special case here.
52
+
53
+ Raises ValueError on a clause this evaluator does not model, rather than
54
+ guessing. Silently returning True for an unknown key is how a filter goes
55
+ missing without anyone noticing; the callers wrap this in the same
56
+ degrade-to-empty-stream contract they use for every other failure, so a
57
+ drift fails closed in production and loudly in the tests.
58
+ """
23
59
  if not where:
24
60
  return True
25
- # Notes/ are cross-repo by design and must always pass the repo filter.
26
- # See docs/specs/2026-05-27-distill-dedup-repo-filter-blindspot.md.
27
- if ("repo" in where and rec.get("type") != "note"
28
- and where["repo"] not in rec.get("repos", [])):
29
- return False
30
- if "type" in where and rec.get("type") != where["type"]:
31
- return False
32
- if "category" in where and rec.get("category") != where["category"]:
33
- return False
34
- return True
61
+ if "$and" in where:
62
+ return all(_matches(rec, clause) for clause in where["$and"])
63
+ if "$or" in where:
64
+ return any(_matches(rec, clause) for clause in where["$or"])
65
+
66
+ unknown = set(where) - _KNOWN_FIELDS
67
+ if unknown:
68
+ raise ValueError(
69
+ f"_matches cannot evaluate {sorted(unknown)}: the filter producer and "
70
+ f"this evaluator have drifted. Add the field to _KNOWN_FIELDS and "
71
+ f"handle it below."
72
+ )
73
+ for field, expected in where.items():
74
+ if isinstance(expected, dict):
75
+ raise ValueError(
76
+ f"_matches does not implement operator form for {field!r}: "
77
+ f"{expected!r}. ChromaDB would honour it and this stream would not."
78
+ )
79
+
80
+ return all(_FIELD_MATCHERS[field](rec, want) for field, want in where.items())
35
81
 
36
82
 
37
83
  class BM25Index:
@@ -27,7 +27,7 @@ def main():
27
27
  p_search = sub.add_parser("search", help="Semantic search")
28
28
  p_search.add_argument("query", help="Search query text")
29
29
  p_search.add_argument("--repo", help="Filter by repo")
30
- p_search.add_argument("--type", help="Filter by type (note/project/weekly)")
30
+ p_search.add_argument("--type", help="Filter by type (note/project)")
31
31
  p_search.add_argument("--category", help="Filter by category")
32
32
  p_search.add_argument("--n", type=int, default=5, help="Number of results")
33
33
  p_search.add_argument("--no-bm25", action="store_true", help="Disable BM25 stream")
@@ -88,7 +88,13 @@ def main():
88
88
  p_rm.add_argument("--cursor", help="Continuation cursor from a prior page")
89
89
  p_rm.add_argument("--max-chars", type=int, dest="max_chars",
90
90
  help="Page stdout cap (default: config, 12000)")
91
- p_rm.add_argument("--find", help="Exact literal to locate in the Raw")
91
+ p_rm.add_argument("--find",
92
+ help="Exact literal to locate: reports the matching span "
93
+ "ids in `find_matches`; does NOT move the page "
94
+ "(cards still start at 0 / --cursor)")
95
+ p_rm.add_argument("--find-only", action="store_true", dest="find_only",
96
+ help="With --find: emit only `find_matches` (no cards, "
97
+ "no map progress); costs one envelope, not a page")
92
98
 
93
99
  p_rsp = sub.add_parser(
94
100
  "raw-span", help="Bounded original-source page of one span"
@@ -32,7 +32,7 @@ def load_config():
32
32
  """Load ~/.cortex/config.json. Exit with error if not found."""
33
33
  if not CORTEX_CONFIG.exists():
34
34
  print(
35
- "Error: ~/.cortex/config.json not found. Run /cortex:genesis first.",
35
+ "Error: ~/.cortex/config.json not found. Run /cortexes:genesis first.",
36
36
  file=sys.stderr,
37
37
  )
38
38
  sys.exit(1)
@@ -8,6 +8,7 @@ fail-closed on corruption or identity drift.
8
8
  """
9
9
  from __future__ import annotations
10
10
 
11
+ import hashlib
11
12
  import json
12
13
  import os
13
14
  from datetime import datetime, timezone
@@ -319,13 +320,78 @@ def seal_plan(state: dict, expected_outcome: str) -> None:
319
320
  _save(state)
320
321
 
321
322
 
323
+ def _normalization_variants(text: str) -> list:
324
+ """Alternative marker strippings of ``text``, for attributing a mismatch.
325
+
326
+ Each entry is ``(variant_name, stripped_text)``. Only the error path calls
327
+ this, so a whole-file sha256 per variant is affordable; what costs
328
+ something is naming a rule that never existed, because that would
329
+ attribute a real edit to a phantom normalization. Every rule below either
330
+ shipped once or is something a writer could plausibly emit next:
331
+
332
+ * ``marker_line_only`` -- the marker line dropped, its blank separator
333
+ kept. This is exactly what the header branch did before 1.8.1, so a
334
+ regression of that wedge names itself instead of coming back
335
+ disguised as "source content changed".
336
+ * ``unstripped`` -- no marker removal at all, i.e. the plan recorded a
337
+ source that already carried a marker.
338
+ * ``final_newline_*`` -- the current strip with the trailing newline
339
+ added or dropped, for a writer that changes its mind about the last
340
+ byte (14 Raws were appended without one).
341
+
342
+ The marker scan here is deliberately position-blind: a quoted marker in
343
+ the conversation may be picked instead of the anchored one, which makes a
344
+ variant that simply fails to match -- never a wrong attribution.
345
+ """
346
+ stripped = rs.strip_state_marker(text)
347
+ lines = text.split("\n")
348
+ first = next((i for i, ln in enumerate(lines) if rs.is_marker_line(ln)), None)
349
+ candidates = []
350
+ if first is not None:
351
+ candidates.append(("marker_line_only",
352
+ "\n".join(lines[:first] + lines[first + 1:])))
353
+ candidates.append(("unstripped", text))
354
+ candidates.append(("final_newline_added", stripped + "\n"))
355
+ candidates.append(("final_newline_dropped", stripped.rstrip("\n")))
356
+ seen = {stripped} # the rule in force already failed to match
357
+ out = []
358
+ for name, variant in candidates:
359
+ if variant not in seen:
360
+ seen.add(variant)
361
+ out.append((name, variant))
362
+ return out
363
+
364
+
365
+ def _diagnose_source_mismatch(text: str, rec: dict) -> dict:
366
+ """Name why ``text`` no longer hashes to the plan's recorded source.
367
+
368
+ A plan stores hashes, not the pre-marker text, so a mismatch cannot be
369
+ diffed -- but it can be attributed. If some other stripping rule
370
+ reproduces the recorded source hash, the conversation is untouched and
371
+ only the marker accounting differs; if none does, the file really was
372
+ edited and "source content changed" is the honest answer. Conflating the
373
+ two is what sent a whole afternoon looking for an editor that had never
374
+ touched the file.
375
+ """
376
+ for name, variant in _normalization_variants(text):
377
+ digest = hashlib.sha256(variant.encode("utf-8")).hexdigest()
378
+ if digest == rec["source_sha256"]:
379
+ return {"reason": "marker normalization mismatch",
380
+ "normalization": name,
381
+ "hint": "the marker block differs, not the conversation"}
382
+ return {"reason": "source content changed",
383
+ "chars_now": len(text),
384
+ "chars_at_plan_start": rec["char_count"]}
385
+
386
+
322
387
  def complete_plan(plan_id: str, raw_path: str) -> dict:
323
388
  state = _load_by_plan_id(plan_id)
324
389
  if state["status"] != "sealed":
325
390
  raise PageError("PLAN_NOT_ACTIVE", reason="seal before complete")
326
- ident, _ = rs.load(raw_path)
391
+ ident, text = rs.load(raw_path)
327
392
  if ident.source_sha256 != state["identity"]["source_sha256"]:
328
- raise PageError("RAW_CHANGED", reason="source content changed")
393
+ raise PageError("RAW_CHANGED",
394
+ **_diagnose_source_mismatch(text, state["identity"]))
329
395
  from .distill_queue import classify
330
396
  outcome = classify(raw_path).outcome
331
397
  if outcome != state["expected_outcome"]:
@@ -53,7 +53,7 @@ def _vector_stream(query, n, where):
53
53
  _STREAM_ORDER = ("vector", "bm25")
54
54
 
55
55
 
56
- def _graph_ranked(ranked, weights, rc, n):
56
+ def _graph_ranked(ranked, weights, rc, n, where=None):
57
57
  """Build the graph RRF stream: wikilink-neighbors of the top vector/bm25 hits.
58
58
 
59
59
  Returns (ranked_list, display_map) where ranked_list is [(doc_id, rank)] for
@@ -61,7 +61,14 @@ def _graph_ranked(ranked, weights, rc, n):
61
61
  ([], {}) on failure (e.g. vault unconfigured) or when graph yields nothing.
62
62
  Rank-based (not additive), so it composes with the other streams via RRF
63
63
  without the score-scale conflict that the old additive boost had.
64
+
65
+ `where` is evaluated with the same `bm25._matches` the keyword stream uses:
66
+ three streams, one filter language. This stream used to receive no `where`
67
+ at all, so wikilink neighbours entered the fused set past `--repo`/`--type`/
68
+ `--category`. A doc with no metadata is rejected rather than admitted — we
69
+ cannot show it passed the filter.
64
70
  """
71
+ from . import bm25 as bm25_mod
65
72
  from . import graph as graph_mod
66
73
  try:
67
74
  pre = rrf_fuse(ranked, weights, k=rc["rrf_k"])
@@ -69,10 +76,20 @@ def _graph_ranked(ranked, weights, rc, n):
69
76
  if not seeds:
70
77
  return [], {}
71
78
  adjacency, meta = graph_mod.build_graph(get_vault_path())
79
+ keep = None
80
+ if where:
81
+ def keep(doc_id):
82
+ rec = meta.get(doc_id)
83
+ return rec is not None and bm25_mod._matches(rec, where)
72
84
  stream = graph_mod.graph_stream(
73
- adjacency, seeds, hops=rc["graph_hops"], max_n=max(n * 2, 10)
85
+ adjacency, seeds, hops=rc["graph_hops"], max_n=max(n * 2, 10), keep=keep
74
86
  )
75
- gdisplay = {doc_id: meta[doc_id] for doc_id, _ in stream if doc_id in meta}
87
+ # `repos` is a filter field, not a display field: the other streams do
88
+ # not emit it, so it must not reach the result dicts from this one.
89
+ gdisplay = {
90
+ doc_id: {k: v for k, v in meta[doc_id].items() if k != "repos"}
91
+ for doc_id, _ in stream if doc_id in meta
92
+ }
76
93
  return stream, gdisplay
77
94
  except (Exception, SystemExit):
78
95
  return [], {}
@@ -133,7 +150,7 @@ def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, r
133
150
 
134
151
  use_graph = rc["graph"] if graph is None else graph
135
152
  if use_graph:
136
- g_ranked, g_display = _graph_ranked(ranked, weights, rc, n)
153
+ g_ranked, g_display = _graph_ranked(ranked, weights, rc, n, where)
137
154
  if g_ranked:
138
155
  ranked["graph"] = g_ranked
139
156
  for doc_id, disp in g_display.items():
@@ -150,14 +167,24 @@ def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, r
150
167
  use_rerank = rc["rerank"] if rerank is None else rerank
151
168
  take = max(n, rc["rerank_window"]) if use_rerank else n
152
169
 
170
+ head = fused[:take]
171
+ # Hits that arrived via bm25 or graph never passed through the vector
172
+ # top-n window, so they carry no cosine. Ask the collection for theirs
173
+ # rather than reporting 0.0, which reads as "no overlap" downstream.
174
+ missing = [doc_id for doc_id, _ in head if doc_id not in vec_score]
175
+ if missing and use_vector and os.environ.get("OPENAI_API_KEY"):
176
+ try:
177
+ vec_score.update(store.cosine_for(query, missing))
178
+ except (Exception, SystemExit):
179
+ pass # same degradation contract as the streams: score falls to 0.0
180
+
153
181
  out = []
154
- for doc_id, _rrf in fused[:take]:
182
+ for doc_id, _rrf in head:
155
183
  entry = dict(display.get(doc_id, {}))
156
184
  entry["id"] = doc_id
157
185
  # `score` reports the vector cosine similarity (0-1, interpretable) so
158
186
  # absolute-threshold consumers (distill/broadcast dedup) keep working.
159
- # RRF only determines ORDER (already baked into `fused`). bm25-only hits
160
- # have no cosine score -> 0.0.
187
+ # RRF only determines ORDER (already baked into `fused`).
161
188
  entry["score"] = round(vec_score.get(doc_id, 0.0), 4)
162
189
  out.append(entry)
163
190
 
@@ -11,6 +11,7 @@ Graph participates in retrieval as a THIRD RRF stream (see fusion.py): a
11
11
  rank-based list of wikilink-neighbors of the top hits. Rank-based fusion avoids
12
12
  the scale conflict of adding a boost onto RRF's compressed score band.
13
13
  """
14
+ import hashlib
14
15
  from pathlib import Path
15
16
 
16
17
  from .parser import classify_path, extract_summary, extract_wikilinks, parse_document
@@ -29,7 +30,11 @@ def _meta_for(rel, fm, body):
29
30
  "id": rel,
30
31
  "title": fm.get("title", rel.rsplit("/", 1)[-1].removesuffix(".md")),
31
32
  "type": doc_type,
33
+ # `repo` is the display field (first membership); `repos` is the full
34
+ # membership list the `where` evaluator reads. Keeping only the singular
35
+ # form let a multi-repo page fail a filter naming its second repo.
32
36
  "repo": (repos or [""])[0],
37
+ "repos": repos,
33
38
  "category": category,
34
39
  "tags": fm.get("tags", ""),
35
40
  "summary": extract_summary(body),
@@ -75,6 +80,15 @@ def build_graph(vault):
75
80
  return _cache[key]
76
81
 
77
82
 
83
+ def _tiebreak(doc_id):
84
+ """Stable, path-insensitive ordering key for equidistant neighbours.
85
+
86
+ blake2b rather than `hash()`: the builtin is salted per process by
87
+ PYTHONHASHSEED, which is the non-determinism this exists to remove.
88
+ """
89
+ return hashlib.blake2b(doc_id.encode("utf-8"), digest_size=8).digest()
90
+
91
+
78
92
  def _bfs_neighbors(adjacency, seeds, hops):
79
93
  """Return {base_path: distance} reachable within `hops` from seeds (excluding seeds)."""
80
94
  frontier = set(seeds)
@@ -94,14 +108,41 @@ def _bfs_neighbors(adjacency, seeds, hops):
94
108
  return dist
95
109
 
96
110
 
97
- def graph_stream(adjacency, seeds, hops=1, max_n=15):
111
+ def graph_stream(adjacency, seeds, hops=1, max_n=15, keep=None):
98
112
  """A rank-based stream of wikilink-neighbors of `seeds`, nearest first.
99
113
 
100
114
  Returns [(doc_id, rank)] (rank 0-based, capped at max_n), excluding seeds —
101
115
  shaped for rrf_fuse as a third retrieval stream. Empty if no neighbors.
116
+
117
+ `keep` is an optional predicate on doc_id applied BEFORE the max_n cut, so a
118
+ filtered query still gets a full window of admissible neighbours rather than
119
+ a window thinned by rejects. It carries the caller's `where` clause: without
120
+ it this stream was the one path into the fused set that no filter reached.
121
+
122
+ `keep` screens what is RETURNED, not what the walk may cross. With
123
+ `graph_hops > 1` a neighbour reachable only *through* a page the filter
124
+ rejects is still returned if it passes itself. That is deliberate — the
125
+ clause describes the answer set, not the path — but it means raising
126
+ `graph_hops` widens what a scoped query can reach. Moot at the current
127
+ `graph_hops = 1`, where there are no intermediate nodes.
102
128
  """
103
129
  dist = _bfs_neighbors(adjacency, seeds, hops)
104
130
  if not dist:
105
131
  return []
106
- ordered = sorted(dist.items(), key=lambda kv: kv[1]) # nearest first
132
+ # Tie-break deterministically, but NOT on the doc id itself. `dist` is
133
+ # populated by iterating adjacency *sets*, so equidistant neighbours arrive
134
+ # in an order that varies with PYTHONHASHSEED, and sorting on distance alone
135
+ # is stable -- it preserved that variation, leaving the stream irreproducible
136
+ # across processes. Ordering by doc id fixes that but buys a worse problem:
137
+ # at graph_hops=1 every neighbour is distance 1, so the id becomes the sole
138
+ # key and the max_n cut degenerates into "keep the lexicographically smallest
139
+ # N". `Notes/` sorts before `Projects/`, so scoped queries -- the ones that
140
+ # exist to surface a repo's Projects/ pages -- were the worst affected.
141
+ # Measured over 300 realistic 5-seed calls: truncation fired on 70% of them
142
+ # and lifted Notes/ from 46% of the candidates to 68% of the survivors.
143
+ # A keyed digest is reproducible across processes like the id, without
144
+ # correlating to the path prefix.
145
+ ordered = sorted(dist.items(), key=lambda kv: (kv[1], _tiebreak(kv[0]))) # nearest first
146
+ if keep is not None:
147
+ ordered = [item for item in ordered if keep(item[0])]
107
148
  return [(doc_id, rank) for rank, (doc_id, _d) in enumerate(ordered[:max_n])]
@@ -40,6 +40,10 @@ def classify_path(rel_path):
40
40
 
41
41
  Returns:
42
42
  tuple: (doc_type, category) e.g. ("note", "Nginx"), ("project", "acme-core")
43
+
44
+ `Weekly/` is no longer a Cortexes content type. A vault that still has the
45
+ directory keeps it on disk untouched; its pages simply classify as
46
+ ``unknown`` and are not indexed (rebuild only scans Notes/ and Projects/).
43
47
  """
44
48
  parts = Path(rel_path).parts
45
49
  if not parts:
@@ -54,8 +58,6 @@ def classify_path(rel_path):
54
58
  return "archive", ""
55
59
  repo = parts[1] if len(parts) > 1 else ""
56
60
  return "project", repo
57
- elif top == "Weekly":
58
- return "weekly", ""
59
61
  elif top == "Raw":
60
62
  return "raw", ""
61
63
  return "unknown", ""
@@ -89,6 +89,10 @@ def _dispatch(args) -> None:
89
89
  raise PageError("CURSOR_MISMATCH", reason="cursor beyond span count")
90
90
 
91
91
  find = getattr(args, "find", None)
92
+ find_only = bool(getattr(args, "find_only", False))
93
+ if find_only and not find:
94
+ raise PageError("FIND_ONLY_REQUIRES_FIND",
95
+ reason="--find-only needs --find <literal>")
92
96
  find_matches = None
93
97
  if find:
94
98
  find_matches = []
@@ -122,8 +126,23 @@ def _dispatch(args) -> None:
122
126
  }
123
127
  if find_matches is not None:
124
128
  obj["find_matches"] = find_matches
129
+ if find_only:
130
+ # Locate only: no cards, no map progress, no continuation.
131
+ obj["find_only"] = True
132
+ obj["next_cursor"] = None
125
133
  return obj
126
134
 
135
+ if find_only:
136
+ obj, chars = finalize_page(
137
+ lambda pc, ua: envelope([], start, pc, ua), used_before)
138
+ if chars > max_chars:
139
+ raise PageError("PAGE_BUDGET_TOO_SMALL", max_chars=max_chars,
140
+ reason="find-only envelope exceeds budget")
141
+ dp.charge_and_save(state, chars)
142
+ assert len(render_page(obj)) + 1 == chars
143
+ emit(obj)
144
+ return
145
+
127
146
  cards: list = []
128
147
  idx = start
129
148
  final_obj = None
@@ -76,8 +76,37 @@ class SourceSpan:
76
76
  tool_name: str | None = None
77
77
 
78
78
 
79
+ def is_marker_line(line: str) -> bool:
80
+ """True if ``line`` is a whole-line distilled marker, indent ignored.
81
+
82
+ Public so callers outside this module can recognise a marker without
83
+ reaching for the private pattern -- position anchoring stays here.
84
+ """
85
+ return bool(_MARKER_RE.match(line.strip()))
86
+
87
+
88
+ def _drop_marker_block(lines: list[str], i: int) -> str:
89
+ """Rejoin ``lines`` without the marker block anchored at index ``i``.
90
+
91
+ A marker is written as a BLOCK, not a line: the writers emit
92
+ ``"\\n<!-- distilled: ... -->\\n"``, so the marker arrives together with a
93
+ blank separator line. Removing only the marker leaves that blank behind and
94
+ the stripped text no longer hashes to the pre-marker source. Both anchoring
95
+ conventions share this helper precisely because they once drifted: the
96
+ header branch kept the blank, so a turnless Raw (no ``### User`` at all,
97
+ e.g. a vacuum session) fell through the header loop, got the header
98
+ treatment for its EOF marker, and stayed wedged in RAW_CHANGED forever.
99
+ """
100
+ head = lines[:i - 1] if i > 0 and not lines[i - 1].strip() else lines[:i]
101
+ tail = lines[i + 1:]
102
+ # An empty tail means the marker was appended without a trailing newline,
103
+ # so the split() carries no final empty element. Re-emit one, or the line
104
+ # above would silently lose its own newline.
105
+ return "\n".join(head + (tail or [""]))
106
+
107
+
79
108
  def strip_state_marker(text: str) -> str:
80
- """Remove the ONE position-anchored distilled marker line, if present.
109
+ """Remove the ONE position-anchored distilled marker block, if present.
81
110
 
82
111
  Same anchoring rules as distill_queue: the first whole-line marker BEFORE
83
112
  the first turn/tool header (header convention), else the last non-empty
@@ -92,17 +121,13 @@ def strip_state_marker(text: str) -> str:
92
121
  if s in TURN_HEADERS:
93
122
  break
94
123
  if _MARKER_RE.match(s):
95
- return "\n".join(lines[:i] + lines[i + 1:])
124
+ return _drop_marker_block(lines, i)
96
125
  for i in range(len(lines) - 1, -1, -1):
97
126
  s = lines[i].strip()
98
127
  if not s:
99
128
  continue
100
129
  if _MARKER_RE.match(s):
101
- # Remove marker and preceding blank line if present
102
- if i > 0 and not lines[i - 1].strip():
103
- return "\n".join(lines[:i - 1] + lines[i + 1:])
104
- else:
105
- return "\n".join(lines[:i] + lines[i + 1:])
130
+ return _drop_marker_block(lines, i)
106
131
  break
107
132
  return text
108
133