cortex-vec 0.8.0__tar.gz → 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. {cortex_vec-0.8.0/src/cortex_vec.egg-info → cortex_vec-0.9.0}/PKG-INFO +1 -1
  2. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/pyproject.toml +1 -1
  3. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/bm25.py +56 -10
  4. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/cli.py +7 -1
  5. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/distill_plan.py +68 -2
  6. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/fusion.py +34 -7
  7. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/graph.py +43 -2
  8. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/raw_map.py +19 -0
  9. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/raw_source.py +32 -7
  10. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/reclaim.py +111 -14
  11. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/store.py +47 -0
  12. {cortex_vec-0.8.0 → cortex_vec-0.9.0/src/cortex_vec.egg-info}/PKG-INFO +1 -1
  13. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec.egg-info/SOURCES.txt +3 -0
  14. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_bm25.py +8 -6
  15. cortex_vec-0.9.0/tests/test_cosine_backfill.py +212 -0
  16. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_distill_plan.py +73 -0
  17. cortex_vec-0.9.0/tests/test_fusion_graph_filter.py +331 -0
  18. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_fusion_search.py +5 -2
  19. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_raw_map.py +28 -2
  20. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_raw_source.py +45 -14
  21. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_reclaim.py +126 -0
  22. cortex_vec-0.9.0/tests/test_where_shape_lockstep.py +180 -0
  23. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/LICENSE +0 -0
  24. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/NOTICE +0 -0
  25. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/README.md +0 -0
  26. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/setup.cfg +0 -0
  27. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/__init__.py +0 -0
  28. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/config.py +0 -0
  29. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/distill_queue.py +0 -0
  30. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/eval/__init__.py +0 -0
  31. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/eval/adapters.py +0 -0
  32. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/eval/corpus.py +0 -0
  33. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/eval/report.py +0 -0
  34. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/eval/run.py +0 -0
  35. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/eval/score.py +0 -0
  36. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/parser.py +0 -0
  37. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/raw_page.py +0 -0
  38. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/raw_span.py +0 -0
  39. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/raw_view.py +0 -0
  40. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/rerank.py +0 -0
  41. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/synonyms.py +0 -0
  42. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/tokenize.py +0 -0
  43. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec.egg-info/dependency_links.txt +0 -0
  44. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec.egg-info/entry_points.txt +0 -0
  45. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec.egg-info/requires.txt +0 -0
  46. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec.egg-info/top_level.txt +0 -0
  47. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_adapters.py +0 -0
  48. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_bm25_synonym.py +0 -0
  49. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_cmd_search.py +0 -0
  50. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_cmd_search_flags.py +0 -0
  51. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_config.py +0 -0
  52. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_config_plan2.py +0 -0
  53. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_corpus.py +0 -0
  54. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_distill_queue.py +0 -0
  55. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_fusion_diversify.py +0 -0
  56. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_fusion_graph.py +0 -0
  57. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_fusion_rerank.py +0 -0
  58. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_fusion_rrf.py +0 -0
  59. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_fusion_synonym.py +0 -0
  60. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_graph_build.py +0 -0
  61. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_graph_stream.py +0 -0
  62. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_lockstep.py +0 -0
  63. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_no_weekly_type.py +0 -0
  64. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_raw_page.py +0 -0
  65. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_raw_span.py +0 -0
  66. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_raw_view.py +0 -0
  67. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_rebuild_bm25_only.py +0 -0
  68. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_report.py +0 -0
  69. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_rerank.py +0 -0
  70. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_run.py +0 -0
  71. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_score.py +0 -0
  72. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_smoke.py +0 -0
  73. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_synonyms.py +0 -0
  74. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_tokenize.py +0 -0
  75. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_vector_stream.py +0 -0
  76. {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_wikilinks.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cortex-vec
3
- Version: 0.8.0
3
+ Version: 0.9.0
4
4
  Summary: Vector store CLI for the cortex vault — hybrid BM25 + embedding retrieval over a markdown knowledge vault
5
5
  Author: XBlueSky
6
6
  License-Expression: Apache-2.0
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "cortex-vec"
3
- version = "0.8.0"
3
+ version = "0.9.0"
4
4
  description = "Vector store CLI for the cortex vault — hybrid BM25 + embedding retrieval over a markdown knowledge vault"
5
5
  readme = "README.md"
6
6
  license = "Apache-2.0"
@@ -19,19 +19,65 @@ def _doc_record(doc):
19
19
  return rec
20
20
 
21
21
 
22
+ # Fields this evaluator models, each paired with the predicate that evaluates it.
23
+ # ChromaDB accepts any metadata key, so a clause on a key absent here would be
24
+ # honoured by the vector stream and ignored by the two that run through
25
+ # `_matches` -- one query language per stream again, which is exactly the defect
26
+ # the spec below documents. Adding a filter field means adding it here;
27
+ # `test_where_shape_lockstep` fails until you do.
28
+ #
29
+ # The name list and the comparisons are ONE table on purpose. Keeping them apart
30
+ # let a field be whitelisted as "known" while no branch ever compared it, which
31
+ # passes every record silently -- the same fail-open, one level further in.
32
+ _FIELD_MATCHERS = {
33
+ "repo": lambda rec, want: want in rec.get("repos", []),
34
+ "type": lambda rec, want: rec.get("type") == want,
35
+ "category": lambda rec, want: rec.get("category") == want,
36
+ }
37
+ _KNOWN_FIELDS = frozenset(_FIELD_MATCHERS)
38
+
39
+
22
40
  def _matches(rec, where):
41
+ """Evaluate a Chroma-style `where` clause against a stored record.
42
+
43
+ Understands exactly the shapes `store._build_where` emits: flat field
44
+ equality plus `$and` / `$or` composition. All three retrieval streams
45
+ (BM25, wikilink graph, and -- via ChromaDB -- vector) must read the same
46
+ query language; a flat-only matcher silently ignored the nested payload
47
+ and dropped the filter entirely. See
48
+ docs/specs/2026-05-27-distill-dedup-repo-filter-blindspot.md.
49
+
50
+ The Notes/-are-cross-repo exemption lives in the `$or` branch that
51
+ `_build_where` emits, not in a `type == "note"` special case here.
52
+
53
+ Raises ValueError on a clause this evaluator does not model, rather than
54
+ guessing. Silently returning True for an unknown key is how a filter goes
55
+ missing without anyone noticing; the callers wrap this in the same
56
+ degrade-to-empty-stream contract they use for every other failure, so a
57
+ drift fails closed in production and loudly in the tests.
58
+ """
23
59
  if not where:
24
60
  return True
25
- # Notes/ are cross-repo by design and must always pass the repo filter.
26
- # See docs/specs/2026-05-27-distill-dedup-repo-filter-blindspot.md.
27
- if ("repo" in where and rec.get("type") != "note"
28
- and where["repo"] not in rec.get("repos", [])):
29
- return False
30
- if "type" in where and rec.get("type") != where["type"]:
31
- return False
32
- if "category" in where and rec.get("category") != where["category"]:
33
- return False
34
- return True
61
+ if "$and" in where:
62
+ return all(_matches(rec, clause) for clause in where["$and"])
63
+ if "$or" in where:
64
+ return any(_matches(rec, clause) for clause in where["$or"])
65
+
66
+ unknown = set(where) - _KNOWN_FIELDS
67
+ if unknown:
68
+ raise ValueError(
69
+ f"_matches cannot evaluate {sorted(unknown)}: the filter producer and "
70
+ f"this evaluator have drifted. Add the field to _KNOWN_FIELDS and "
71
+ f"handle it below."
72
+ )
73
+ for field, expected in where.items():
74
+ if isinstance(expected, dict):
75
+ raise ValueError(
76
+ f"_matches does not implement operator form for {field!r}: "
77
+ f"{expected!r}. ChromaDB would honour it and this stream would not."
78
+ )
79
+
80
+ return all(_FIELD_MATCHERS[field](rec, want) for field, want in where.items())
35
81
 
36
82
 
37
83
  class BM25Index:
@@ -88,7 +88,13 @@ def main():
88
88
  p_rm.add_argument("--cursor", help="Continuation cursor from a prior page")
89
89
  p_rm.add_argument("--max-chars", type=int, dest="max_chars",
90
90
  help="Page stdout cap (default: config, 12000)")
91
- p_rm.add_argument("--find", help="Exact literal to locate in the Raw")
91
+ p_rm.add_argument("--find",
92
+ help="Exact literal to locate: reports the matching span "
93
+ "ids in `find_matches`; does NOT move the page "
94
+ "(cards still start at 0 / --cursor)")
95
+ p_rm.add_argument("--find-only", action="store_true", dest="find_only",
96
+ help="With --find: emit only `find_matches` (no cards, "
97
+ "no map progress); costs one envelope, not a page")
92
98
 
93
99
  p_rsp = sub.add_parser(
94
100
  "raw-span", help="Bounded original-source page of one span"
@@ -8,6 +8,7 @@ fail-closed on corruption or identity drift.
8
8
  """
9
9
  from __future__ import annotations
10
10
 
11
+ import hashlib
11
12
  import json
12
13
  import os
13
14
  from datetime import datetime, timezone
@@ -319,13 +320,78 @@ def seal_plan(state: dict, expected_outcome: str) -> None:
319
320
  _save(state)
320
321
 
321
322
 
323
+ def _normalization_variants(text: str) -> list:
324
+ """Alternative marker strippings of ``text``, for attributing a mismatch.
325
+
326
+ Each entry is ``(variant_name, stripped_text)``. Only the error path calls
327
+ this, so a whole-file sha256 per variant is affordable; what costs
328
+ something is naming a rule that never existed, because that would
329
+ attribute a real edit to a phantom normalization. Every rule below either
330
+ shipped once or is something a writer could plausibly emit next:
331
+
332
+ * ``marker_line_only`` -- the marker line dropped, its blank separator
333
+ kept. This is exactly what the header branch did before 1.8.1, so a
334
+ regression of that wedge names itself instead of coming back
335
+ disguised as "source content changed".
336
+ * ``unstripped`` -- no marker removal at all, i.e. the plan recorded a
337
+ source that already carried a marker.
338
+ * ``final_newline_*`` -- the current strip with the trailing newline
339
+ added or dropped, for a writer that changes its mind about the last
340
+ byte (14 Raws were appended without one).
341
+
342
+ The marker scan here is deliberately position-blind: a quoted marker in
343
+ the conversation may be picked instead of the anchored one, which makes a
344
+ variant that simply fails to match -- never a wrong attribution.
345
+ """
346
+ stripped = rs.strip_state_marker(text)
347
+ lines = text.split("\n")
348
+ first = next((i for i, ln in enumerate(lines) if rs.is_marker_line(ln)), None)
349
+ candidates = []
350
+ if first is not None:
351
+ candidates.append(("marker_line_only",
352
+ "\n".join(lines[:first] + lines[first + 1:])))
353
+ candidates.append(("unstripped", text))
354
+ candidates.append(("final_newline_added", stripped + "\n"))
355
+ candidates.append(("final_newline_dropped", stripped.rstrip("\n")))
356
+ seen = {stripped} # the rule in force already failed to match
357
+ out = []
358
+ for name, variant in candidates:
359
+ if variant not in seen:
360
+ seen.add(variant)
361
+ out.append((name, variant))
362
+ return out
363
+
364
+
365
+ def _diagnose_source_mismatch(text: str, rec: dict) -> dict:
366
+ """Name why ``text`` no longer hashes to the plan's recorded source.
367
+
368
+ A plan stores hashes, not the pre-marker text, so a mismatch cannot be
369
+ diffed -- but it can be attributed. If some other stripping rule
370
+ reproduces the recorded source hash, the conversation is untouched and
371
+ only the marker accounting differs; if none does, the file really was
372
+ edited and "source content changed" is the honest answer. Conflating the
373
+ two is what sent a whole afternoon looking for an editor that had never
374
+ touched the file.
375
+ """
376
+ for name, variant in _normalization_variants(text):
377
+ digest = hashlib.sha256(variant.encode("utf-8")).hexdigest()
378
+ if digest == rec["source_sha256"]:
379
+ return {"reason": "marker normalization mismatch",
380
+ "normalization": name,
381
+ "hint": "the marker block differs, not the conversation"}
382
+ return {"reason": "source content changed",
383
+ "chars_now": len(text),
384
+ "chars_at_plan_start": rec["char_count"]}
385
+
386
+
322
387
  def complete_plan(plan_id: str, raw_path: str) -> dict:
323
388
  state = _load_by_plan_id(plan_id)
324
389
  if state["status"] != "sealed":
325
390
  raise PageError("PLAN_NOT_ACTIVE", reason="seal before complete")
326
- ident, _ = rs.load(raw_path)
391
+ ident, text = rs.load(raw_path)
327
392
  if ident.source_sha256 != state["identity"]["source_sha256"]:
328
- raise PageError("RAW_CHANGED", reason="source content changed")
393
+ raise PageError("RAW_CHANGED",
394
+ **_diagnose_source_mismatch(text, state["identity"]))
329
395
  from .distill_queue import classify
330
396
  outcome = classify(raw_path).outcome
331
397
  if outcome != state["expected_outcome"]:
@@ -53,7 +53,7 @@ def _vector_stream(query, n, where):
53
53
  _STREAM_ORDER = ("vector", "bm25")
54
54
 
55
55
 
56
- def _graph_ranked(ranked, weights, rc, n):
56
+ def _graph_ranked(ranked, weights, rc, n, where=None):
57
57
  """Build the graph RRF stream: wikilink-neighbors of the top vector/bm25 hits.
58
58
 
59
59
  Returns (ranked_list, display_map) where ranked_list is [(doc_id, rank)] for
@@ -61,7 +61,14 @@ def _graph_ranked(ranked, weights, rc, n):
61
61
  ([], {}) on failure (e.g. vault unconfigured) or when graph yields nothing.
62
62
  Rank-based (not additive), so it composes with the other streams via RRF
63
63
  without the score-scale conflict that the old additive boost had.
64
+
65
+ `where` is evaluated with the same `bm25._matches` the keyword stream uses:
66
+ three streams, one filter language. This stream used to receive no `where`
67
+ at all, so wikilink neighbours entered the fused set past `--repo`/`--type`/
68
+ `--category`. A doc with no metadata is rejected rather than admitted — we
69
+ cannot show it passed the filter.
64
70
  """
71
+ from . import bm25 as bm25_mod
65
72
  from . import graph as graph_mod
66
73
  try:
67
74
  pre = rrf_fuse(ranked, weights, k=rc["rrf_k"])
@@ -69,10 +76,20 @@ def _graph_ranked(ranked, weights, rc, n):
69
76
  if not seeds:
70
77
  return [], {}
71
78
  adjacency, meta = graph_mod.build_graph(get_vault_path())
79
+ keep = None
80
+ if where:
81
+ def keep(doc_id):
82
+ rec = meta.get(doc_id)
83
+ return rec is not None and bm25_mod._matches(rec, where)
72
84
  stream = graph_mod.graph_stream(
73
- adjacency, seeds, hops=rc["graph_hops"], max_n=max(n * 2, 10)
85
+ adjacency, seeds, hops=rc["graph_hops"], max_n=max(n * 2, 10), keep=keep
74
86
  )
75
- gdisplay = {doc_id: meta[doc_id] for doc_id, _ in stream if doc_id in meta}
87
+ # `repos` is a filter field, not a display field: the other streams do
88
+ # not emit it, so it must not reach the result dicts from this one.
89
+ gdisplay = {
90
+ doc_id: {k: v for k, v in meta[doc_id].items() if k != "repos"}
91
+ for doc_id, _ in stream if doc_id in meta
92
+ }
76
93
  return stream, gdisplay
77
94
  except (Exception, SystemExit):
78
95
  return [], {}
@@ -133,7 +150,7 @@ def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, r
133
150
 
134
151
  use_graph = rc["graph"] if graph is None else graph
135
152
  if use_graph:
136
- g_ranked, g_display = _graph_ranked(ranked, weights, rc, n)
153
+ g_ranked, g_display = _graph_ranked(ranked, weights, rc, n, where)
137
154
  if g_ranked:
138
155
  ranked["graph"] = g_ranked
139
156
  for doc_id, disp in g_display.items():
@@ -150,14 +167,24 @@ def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, r
150
167
  use_rerank = rc["rerank"] if rerank is None else rerank
151
168
  take = max(n, rc["rerank_window"]) if use_rerank else n
152
169
 
170
+ head = fused[:take]
171
+ # Hits that arrived via bm25 or graph never passed through the vector
172
+ # top-n window, so they carry no cosine. Ask the collection for theirs
173
+ # rather than reporting 0.0, which reads as "no overlap" downstream.
174
+ missing = [doc_id for doc_id, _ in head if doc_id not in vec_score]
175
+ if missing and use_vector and os.environ.get("OPENAI_API_KEY"):
176
+ try:
177
+ vec_score.update(store.cosine_for(query, missing))
178
+ except (Exception, SystemExit):
179
+ pass # same degradation contract as the streams: score falls to 0.0
180
+
153
181
  out = []
154
- for doc_id, _rrf in fused[:take]:
182
+ for doc_id, _rrf in head:
155
183
  entry = dict(display.get(doc_id, {}))
156
184
  entry["id"] = doc_id
157
185
  # `score` reports the vector cosine similarity (0-1, interpretable) so
158
186
  # absolute-threshold consumers (distill/broadcast dedup) keep working.
159
- # RRF only determines ORDER (already baked into `fused`). bm25-only hits
160
- # have no cosine score -> 0.0.
187
+ # RRF only determines ORDER (already baked into `fused`).
161
188
  entry["score"] = round(vec_score.get(doc_id, 0.0), 4)
162
189
  out.append(entry)
163
190
 
@@ -11,6 +11,7 @@ Graph participates in retrieval as a THIRD RRF stream (see fusion.py): a
11
11
  rank-based list of wikilink-neighbors of the top hits. Rank-based fusion avoids
12
12
  the scale conflict of adding a boost onto RRF's compressed score band.
13
13
  """
14
+ import hashlib
14
15
  from pathlib import Path
15
16
 
16
17
  from .parser import classify_path, extract_summary, extract_wikilinks, parse_document
@@ -29,7 +30,11 @@ def _meta_for(rel, fm, body):
29
30
  "id": rel,
30
31
  "title": fm.get("title", rel.rsplit("/", 1)[-1].removesuffix(".md")),
31
32
  "type": doc_type,
33
+ # `repo` is the display field (first membership); `repos` is the full
34
+ # membership list the `where` evaluator reads. Keeping only the singular
35
+ # form let a multi-repo page fail a filter naming its second repo.
32
36
  "repo": (repos or [""])[0],
37
+ "repos": repos,
33
38
  "category": category,
34
39
  "tags": fm.get("tags", ""),
35
40
  "summary": extract_summary(body),
@@ -75,6 +80,15 @@ def build_graph(vault):
75
80
  return _cache[key]
76
81
 
77
82
 
83
+ def _tiebreak(doc_id):
84
+ """Stable, path-insensitive ordering key for equidistant neighbours.
85
+
86
+ blake2b rather than `hash()`: the builtin is salted per process by
87
+ PYTHONHASHSEED, which is the non-determinism this exists to remove.
88
+ """
89
+ return hashlib.blake2b(doc_id.encode("utf-8"), digest_size=8).digest()
90
+
91
+
78
92
  def _bfs_neighbors(adjacency, seeds, hops):
79
93
  """Return {base_path: distance} reachable within `hops` from seeds (excluding seeds)."""
80
94
  frontier = set(seeds)
@@ -94,14 +108,41 @@ def _bfs_neighbors(adjacency, seeds, hops):
94
108
  return dist
95
109
 
96
110
 
97
- def graph_stream(adjacency, seeds, hops=1, max_n=15):
111
+ def graph_stream(adjacency, seeds, hops=1, max_n=15, keep=None):
98
112
  """A rank-based stream of wikilink-neighbors of `seeds`, nearest first.
99
113
 
100
114
  Returns [(doc_id, rank)] (rank 0-based, capped at max_n), excluding seeds —
101
115
  shaped for rrf_fuse as a third retrieval stream. Empty if no neighbors.
116
+
117
+ `keep` is an optional predicate on doc_id applied BEFORE the max_n cut, so a
118
+ filtered query still gets a full window of admissible neighbours rather than
119
+ a window thinned by rejects. It carries the caller's `where` clause: without
120
+ it this stream was the one path into the fused set that no filter reached.
121
+
122
+ `keep` screens what is RETURNED, not what the walk may cross. With
123
+ `graph_hops > 1` a neighbour reachable only *through* a page the filter
124
+ rejects is still returned if it passes itself. That is deliberate — the
125
+ clause describes the answer set, not the path — but it means raising
126
+ `graph_hops` widens what a scoped query can reach. Moot at the current
127
+ `graph_hops = 1`, where there are no intermediate nodes.
102
128
  """
103
129
  dist = _bfs_neighbors(adjacency, seeds, hops)
104
130
  if not dist:
105
131
  return []
106
- ordered = sorted(dist.items(), key=lambda kv: kv[1]) # nearest first
132
+ # Tie-break deterministically, but NOT on the doc id itself. `dist` is
133
+ # populated by iterating adjacency *sets*, so equidistant neighbours arrive
134
+ # in an order that varies with PYTHONHASHSEED, and sorting on distance alone
135
+ # is stable -- it preserved that variation, leaving the stream irreproducible
136
+ # across processes. Ordering by doc id fixes that but buys a worse problem:
137
+ # at graph_hops=1 every neighbour is distance 1, so the id becomes the sole
138
+ # key and the max_n cut degenerates into "keep the lexicographically smallest
139
+ # N". `Notes/` sorts before `Projects/`, so scoped queries -- the ones that
140
+ # exist to surface a repo's Projects/ pages -- were the worst affected.
141
+ # Measured over 300 realistic 5-seed calls: truncation fired on 70% of them
142
+ # and lifted Notes/ from 46% of the candidates to 68% of the survivors.
143
+ # A keyed digest is reproducible across processes like the id, without
144
+ # correlating to the path prefix.
145
+ ordered = sorted(dist.items(), key=lambda kv: (kv[1], _tiebreak(kv[0]))) # nearest first
146
+ if keep is not None:
147
+ ordered = [item for item in ordered if keep(item[0])]
107
148
  return [(doc_id, rank) for rank, (doc_id, _d) in enumerate(ordered[:max_n])]
@@ -89,6 +89,10 @@ def _dispatch(args) -> None:
89
89
  raise PageError("CURSOR_MISMATCH", reason="cursor beyond span count")
90
90
 
91
91
  find = getattr(args, "find", None)
92
+ find_only = bool(getattr(args, "find_only", False))
93
+ if find_only and not find:
94
+ raise PageError("FIND_ONLY_REQUIRES_FIND",
95
+ reason="--find-only needs --find <literal>")
92
96
  find_matches = None
93
97
  if find:
94
98
  find_matches = []
@@ -122,8 +126,23 @@ def _dispatch(args) -> None:
122
126
  }
123
127
  if find_matches is not None:
124
128
  obj["find_matches"] = find_matches
129
+ if find_only:
130
+ # Locate only: no cards, no map progress, no continuation.
131
+ obj["find_only"] = True
132
+ obj["next_cursor"] = None
125
133
  return obj
126
134
 
135
+ if find_only:
136
+ obj, chars = finalize_page(
137
+ lambda pc, ua: envelope([], start, pc, ua), used_before)
138
+ if chars > max_chars:
139
+ raise PageError("PAGE_BUDGET_TOO_SMALL", max_chars=max_chars,
140
+ reason="find-only envelope exceeds budget")
141
+ dp.charge_and_save(state, chars)
142
+ assert len(render_page(obj)) + 1 == chars
143
+ emit(obj)
144
+ return
145
+
127
146
  cards: list = []
128
147
  idx = start
129
148
  final_obj = None
@@ -76,8 +76,37 @@ class SourceSpan:
76
76
  tool_name: str | None = None
77
77
 
78
78
 
79
+ def is_marker_line(line: str) -> bool:
80
+ """True if ``line`` is a whole-line distilled marker, indent ignored.
81
+
82
+ Public so callers outside this module can recognise a marker without
83
+ reaching for the private pattern -- position anchoring stays here.
84
+ """
85
+ return bool(_MARKER_RE.match(line.strip()))
86
+
87
+
88
+ def _drop_marker_block(lines: list[str], i: int) -> str:
89
+ """Rejoin ``lines`` without the marker block anchored at index ``i``.
90
+
91
+ A marker is written as a BLOCK, not a line: the writers emit
92
+ ``"\\n<!-- distilled: ... -->\\n"``, so the marker arrives together with a
93
+ blank separator line. Removing only the marker leaves that blank behind and
94
+ the stripped text no longer hashes to the pre-marker source. Both anchoring
95
+ conventions share this helper precisely because they once drifted: the
96
+ header branch kept the blank, so a turnless Raw (no ``### User`` at all,
97
+ e.g. a vacuum session) fell through the header loop, got the header
98
+ treatment for its EOF marker, and stayed wedged in RAW_CHANGED forever.
99
+ """
100
+ head = lines[:i - 1] if i > 0 and not lines[i - 1].strip() else lines[:i]
101
+ tail = lines[i + 1:]
102
+ # An empty tail means the marker was appended without a trailing newline,
103
+ # so the split() carries no final empty element. Re-emit one, or the line
104
+ # above would silently lose its own newline.
105
+ return "\n".join(head + (tail or [""]))
106
+
107
+
79
108
  def strip_state_marker(text: str) -> str:
80
- """Remove the ONE position-anchored distilled marker line, if present.
109
+ """Remove the ONE position-anchored distilled marker block, if present.
81
110
 
82
111
  Same anchoring rules as distill_queue: the first whole-line marker BEFORE
83
112
  the first turn/tool header (header convention), else the last non-empty
@@ -92,17 +121,13 @@ def strip_state_marker(text: str) -> str:
92
121
  if s in TURN_HEADERS:
93
122
  break
94
123
  if _MARKER_RE.match(s):
95
- return "\n".join(lines[:i] + lines[i + 1:])
124
+ return _drop_marker_block(lines, i)
96
125
  for i in range(len(lines) - 1, -1, -1):
97
126
  s = lines[i].strip()
98
127
  if not s:
99
128
  continue
100
129
  if _MARKER_RE.match(s):
101
- # Remove marker and preceding blank line if present
102
- if i > 0 and not lines[i - 1].strip():
103
- return "\n".join(lines[:i - 1] + lines[i + 1:])
104
- else:
105
- return "\n".join(lines[:i] + lines[i + 1:])
130
+ return _drop_marker_block(lines, i)
106
131
  break
107
132
  return text
108
133
 
@@ -8,7 +8,7 @@ one — pure redundancy that would otherwise each sit in the distill queue as it
8
8
  own entry, spending the distill budget several times over on the same
9
9
  conversation and landing duplicate Notes.
10
10
 
11
- Two safety properties make removal sound:
11
+ Three safety properties make removal sound:
12
12
 
13
13
  * the candidate set is exactly :func:`distill_queue`'s output, i.e. Raw files
14
14
  nothing references yet. An already-distilled Raw carries a position-anchored
@@ -18,17 +18,30 @@ Two safety properties make removal sound:
18
18
  survivor lacks. If the filter output ever diverges (its LLM residue
19
19
  classifier is not deterministic) the match simply fails and nothing is
20
20
  removed — the failure mode is "duplicate stays", never "content lost".
21
+ * both files must carry the same ``repo:`` label. A prefix alone does not prove
22
+ redundancy: one conversation whose cwd changes mid-session produces two Raws
23
+ from the same growing transcript but under DIFFERENT repo labels, and the
24
+ earlier one may be that repo's only record, so reclaiming it would erase that
25
+ repo from the vault's history of the period entirely. Such a
26
+ pair is refused and reported (see :class:`CrossRepoPair`) rather than dropped
27
+ in silence — a cross-repo prefix pair is worth an operator's attention.
21
28
 
22
29
  Comparison starts at the first conversation turn: the frontmatter holds the
23
30
  wall-clock stamp that differs between recordings of the same session.
24
31
  """
25
32
  from __future__ import annotations
26
33
 
34
+ import re
27
35
  import subprocess
36
+ import sys
37
+ from dataclasses import dataclass, field
28
38
  from pathlib import Path
29
39
 
30
40
  from .distill_queue import _TURN_HDRS, distill_queue
31
41
 
42
+ # `repo: <name>` inside the leading frontmatter block, as session-end writes it.
43
+ _REPO_RE = re.compile(r"^repo:\s*(.*?)\s*$")
44
+
32
45
 
33
46
  def _body(path) -> list[str]:
34
47
  """Conversation body: lines from the first turn header to EOF.
@@ -51,6 +64,29 @@ def _body(path) -> list[str]:
51
64
  return lines
52
65
 
53
66
 
67
+ def _repo(path) -> str:
68
+ """The ``repo:`` label from the leading frontmatter block.
69
+
70
+ Returns ``""`` when the file has no frontmatter or no ``repo:`` key — an
71
+ unlabelled Raw therefore differs from every labelled one and is refused,
72
+ which is the safe direction.
73
+ """
74
+ try:
75
+ with Path(path).open(encoding="utf-8") as f:
76
+ if f.readline().rstrip("\n") != "---":
77
+ return ""
78
+ for line in f:
79
+ stripped = line.rstrip("\n")
80
+ if stripped == "---" or stripped.strip() in _TURN_HDRS:
81
+ break
82
+ m = _REPO_RE.match(stripped)
83
+ if m:
84
+ return m.group(1)
85
+ except OSError:
86
+ return ""
87
+ return ""
88
+
89
+
54
90
  def _covers(survivor: list[str], candidate: list[str]) -> bool:
55
91
  """True iff candidate is a non-empty prefix of survivor."""
56
92
  return (
@@ -60,35 +96,90 @@ def _covers(survivor: list[str], candidate: list[str]) -> bool:
60
96
  )
61
97
 
62
98
 
63
- def find_superseded(root, keep=None) -> list[Path]:
99
+ @dataclass(frozen=True)
100
+ class CrossRepoPair:
101
+ """A prefix pair reclaim refused because the two Raws name different repos."""
102
+
103
+ candidate: Path
104
+ survivor: Path
105
+ candidate_repo: str
106
+ survivor_repo: str
107
+
108
+ def describe(self) -> str:
109
+ return (
110
+ f"refused: {self.candidate} (repo: {self.candidate_repo or '?'}) is a "
111
+ f"prefix of {self.survivor} (repo: {self.survivor_repo or '?'}) — "
112
+ "different repo, kept"
113
+ )
114
+
115
+
116
+ @dataclass(frozen=True)
117
+ class ReclaimScan:
118
+ """Outcome of one scan: what may go, and what was refused and why."""
119
+
120
+ superseded: list[Path] = field(default_factory=list)
121
+ refused: list[CrossRepoPair] = field(default_factory=list)
122
+
123
+
124
+ def scan(root, keep=None) -> ReclaimScan:
64
125
  """Undistilled Raw files made redundant by a longer recording (FIFO order).
65
126
 
66
127
  With ``keep``, only that file is treated as a survivor — the session-end
67
128
  path, one body read per queued file. Without it, the whole queue is
68
129
  compared pairwise (backlog cleanup). For two byte-identical bodies the
69
130
  later path survives, so exactly one of the pair is reclaimed.
131
+
132
+ A prefix pair whose ``repo:`` labels differ lands in ``refused`` instead of
133
+ ``superseded``: the candidate stays on disk and the pair is reported.
70
134
  """
71
135
  queue = distill_queue(root)
72
136
  if keep is not None:
73
137
  survivor = _body(keep)
74
138
  if not survivor:
75
- return []
76
- keep_resolved = Path(keep).resolve()
77
- return [
78
- p for p in queue
79
- if p.resolve() != keep_resolved and _covers(survivor, _body(p))
80
- ]
139
+ return ReclaimScan()
140
+ keep_path = Path(keep)
141
+ keep_resolved = keep_path.resolve()
142
+ keep_repo = _repo(keep_path)
143
+ result = ReclaimScan()
144
+ for p in queue:
145
+ if p.resolve() == keep_resolved or not _covers(survivor, _body(p)):
146
+ continue
147
+ cand_repo = _repo(p)
148
+ if cand_repo != keep_repo:
149
+ result.refused.append(
150
+ CrossRepoPair(p, keep_path, cand_repo, keep_repo)
151
+ )
152
+ continue
153
+ result.superseded.append(p)
154
+ return result
81
155
 
82
156
  bodies = {p: _body(p) for p in queue}
83
- superseded = []
157
+ repos = {p: _repo(p) for p in queue}
158
+ result = ReclaimScan()
84
159
  for cand, cand_body in bodies.items():
160
+ pending: list[CrossRepoPair] = []
85
161
  for other, other_body in bodies.items():
86
162
  if other == cand or not _covers(other_body, cand_body):
87
163
  continue
88
- if len(cand_body) < len(other_body) or str(cand) < str(other):
89
- superseded.append(cand)
90
- break
91
- return superseded
164
+ if not (len(cand_body) < len(other_body) or str(cand) < str(other)):
165
+ continue
166
+ if repos[cand] != repos[other]:
167
+ pending.append(
168
+ CrossRepoPair(cand, other, repos[cand], repos[other])
169
+ )
170
+ continue
171
+ # A same-repo survivor settles it; the cross-repo near-misses seen
172
+ # on the way are not worth reporting once the file is going anyway.
173
+ result.superseded.append(cand)
174
+ pending = []
175
+ break
176
+ result.refused.extend(pending)
177
+ return result
178
+
179
+
180
+ def find_superseded(root, keep=None) -> list[Path]:
181
+ """The reclaimable subset of :func:`scan` (refusals dropped)."""
182
+ return scan(root, keep=keep).superseded
92
183
 
93
184
 
94
185
  def _git_rm(vault, path) -> bool:
@@ -129,7 +220,8 @@ def dispatch(args) -> None:
129
220
  from .config import get_vault_path
130
221
 
131
222
  root = get_vault_path() / "Raw"
132
- found = find_superseded(root, keep=getattr(args, "keep", None))
223
+ result = scan(root, keep=getattr(args, "keep", None))
224
+ found = result.superseded
133
225
  if getattr(args, "apply", False):
134
226
  vault = getattr(args, "vault", None)
135
227
  if not vault:
@@ -139,3 +231,8 @@ def dispatch(args) -> None:
139
231
  found = apply_reclaim(found, vault=vault)
140
232
  for path in found:
141
233
  print(path)
234
+ # Refusals go to stderr in BOTH modes: stdout is the machine-readable list
235
+ # of reclaimed paths (session-end-record.sh counts its lines for the commit
236
+ # message), so a refusal printed there would be miscounted as a removal.
237
+ for pair in result.refused:
238
+ print(pair.describe(), file=sys.stderr)