cortex-vec 0.8.0__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cortex_vec-0.8.0/src/cortex_vec.egg-info → cortex_vec-0.9.0}/PKG-INFO +1 -1
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/pyproject.toml +1 -1
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/bm25.py +56 -10
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/cli.py +7 -1
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/distill_plan.py +68 -2
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/fusion.py +34 -7
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/graph.py +43 -2
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/raw_map.py +19 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/raw_source.py +32 -7
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/reclaim.py +111 -14
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/store.py +47 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0/src/cortex_vec.egg-info}/PKG-INFO +1 -1
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec.egg-info/SOURCES.txt +3 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_bm25.py +8 -6
- cortex_vec-0.9.0/tests/test_cosine_backfill.py +212 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_distill_plan.py +73 -0
- cortex_vec-0.9.0/tests/test_fusion_graph_filter.py +331 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_fusion_search.py +5 -2
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_raw_map.py +28 -2
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_raw_source.py +45 -14
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_reclaim.py +126 -0
- cortex_vec-0.9.0/tests/test_where_shape_lockstep.py +180 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/LICENSE +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/NOTICE +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/README.md +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/setup.cfg +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/__init__.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/config.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/distill_queue.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/eval/__init__.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/eval/adapters.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/eval/corpus.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/eval/report.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/eval/run.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/eval/score.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/parser.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/raw_page.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/raw_span.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/raw_view.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/rerank.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/synonyms.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec/tokenize.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec.egg-info/dependency_links.txt +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec.egg-info/entry_points.txt +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec.egg-info/requires.txt +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/src/cortex_vec.egg-info/top_level.txt +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_adapters.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_bm25_synonym.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_cmd_search.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_cmd_search_flags.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_config.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_config_plan2.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_corpus.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_distill_queue.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_fusion_diversify.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_fusion_graph.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_fusion_rerank.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_fusion_rrf.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_fusion_synonym.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_graph_build.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_graph_stream.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_lockstep.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_no_weekly_type.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_raw_page.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_raw_span.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_raw_view.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_rebuild_bm25_only.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_report.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_rerank.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_run.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_score.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_smoke.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_synonyms.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_tokenize.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_vector_stream.py +0 -0
- {cortex_vec-0.8.0 → cortex_vec-0.9.0}/tests/test_wikilinks.py +0 -0
|
@@ -19,19 +19,65 @@ def _doc_record(doc):
|
|
|
19
19
|
return rec
|
|
20
20
|
|
|
21
21
|
|
|
22
|
+
# Fields this evaluator models, each paired with the predicate that evaluates it.
|
|
23
|
+
# ChromaDB accepts any metadata key, so a clause on a key absent here would be
|
|
24
|
+
# honoured by the vector stream and ignored by the two that run through
|
|
25
|
+
# `_matches` -- one query language per stream again, which is exactly the defect
|
|
26
|
+
# the spec below documents. Adding a filter field means adding it here;
|
|
27
|
+
# `test_where_shape_lockstep` fails until you do.
|
|
28
|
+
#
|
|
29
|
+
# The name list and the comparisons are ONE table on purpose. Keeping them apart
|
|
30
|
+
# let a field be whitelisted as "known" while no branch ever compared it, which
|
|
31
|
+
# passes every record silently -- the same fail-open, one level further in.
|
|
32
|
+
_FIELD_MATCHERS = {
|
|
33
|
+
"repo": lambda rec, want: want in rec.get("repos", []),
|
|
34
|
+
"type": lambda rec, want: rec.get("type") == want,
|
|
35
|
+
"category": lambda rec, want: rec.get("category") == want,
|
|
36
|
+
}
|
|
37
|
+
_KNOWN_FIELDS = frozenset(_FIELD_MATCHERS)
|
|
38
|
+
|
|
39
|
+
|
|
22
40
|
def _matches(rec, where):
|
|
41
|
+
"""Evaluate a Chroma-style `where` clause against a stored record.
|
|
42
|
+
|
|
43
|
+
Understands exactly the shapes `store._build_where` emits: flat field
|
|
44
|
+
equality plus `$and` / `$or` composition. All three retrieval streams
|
|
45
|
+
(BM25, wikilink graph, and -- via ChromaDB -- vector) must read the same
|
|
46
|
+
query language; a flat-only matcher silently ignored the nested payload
|
|
47
|
+
and dropped the filter entirely. See
|
|
48
|
+
docs/specs/2026-05-27-distill-dedup-repo-filter-blindspot.md.
|
|
49
|
+
|
|
50
|
+
The Notes/-are-cross-repo exemption lives in the `$or` branch that
|
|
51
|
+
`_build_where` emits, not in a `type == "note"` special case here.
|
|
52
|
+
|
|
53
|
+
Raises ValueError on a clause this evaluator does not model, rather than
|
|
54
|
+
guessing. Silently returning True for an unknown key is how a filter goes
|
|
55
|
+
missing without anyone noticing; the callers wrap this in the same
|
|
56
|
+
degrade-to-empty-stream contract they use for every other failure, so a
|
|
57
|
+
drift fails closed in production and loudly in the tests.
|
|
58
|
+
"""
|
|
23
59
|
if not where:
|
|
24
60
|
return True
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
if
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
61
|
+
if "$and" in where:
|
|
62
|
+
return all(_matches(rec, clause) for clause in where["$and"])
|
|
63
|
+
if "$or" in where:
|
|
64
|
+
return any(_matches(rec, clause) for clause in where["$or"])
|
|
65
|
+
|
|
66
|
+
unknown = set(where) - _KNOWN_FIELDS
|
|
67
|
+
if unknown:
|
|
68
|
+
raise ValueError(
|
|
69
|
+
f"_matches cannot evaluate {sorted(unknown)}: the filter producer and "
|
|
70
|
+
f"this evaluator have drifted. Add the field to _KNOWN_FIELDS and "
|
|
71
|
+
f"handle it below."
|
|
72
|
+
)
|
|
73
|
+
for field, expected in where.items():
|
|
74
|
+
if isinstance(expected, dict):
|
|
75
|
+
raise ValueError(
|
|
76
|
+
f"_matches does not implement operator form for {field!r}: "
|
|
77
|
+
f"{expected!r}. ChromaDB would honour it and this stream would not."
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
return all(_FIELD_MATCHERS[field](rec, want) for field, want in where.items())
|
|
35
81
|
|
|
36
82
|
|
|
37
83
|
class BM25Index:
|
|
@@ -88,7 +88,13 @@ def main():
|
|
|
88
88
|
p_rm.add_argument("--cursor", help="Continuation cursor from a prior page")
|
|
89
89
|
p_rm.add_argument("--max-chars", type=int, dest="max_chars",
|
|
90
90
|
help="Page stdout cap (default: config, 12000)")
|
|
91
|
-
p_rm.add_argument("--find",
|
|
91
|
+
p_rm.add_argument("--find",
|
|
92
|
+
help="Exact literal to locate: reports the matching span "
|
|
93
|
+
"ids in `find_matches`; does NOT move the page "
|
|
94
|
+
"(cards still start at 0 / --cursor)")
|
|
95
|
+
p_rm.add_argument("--find-only", action="store_true", dest="find_only",
|
|
96
|
+
help="With --find: emit only `find_matches` (no cards, "
|
|
97
|
+
"no map progress); costs one envelope, not a page")
|
|
92
98
|
|
|
93
99
|
p_rsp = sub.add_parser(
|
|
94
100
|
"raw-span", help="Bounded original-source page of one span"
|
|
@@ -8,6 +8,7 @@ fail-closed on corruption or identity drift.
|
|
|
8
8
|
"""
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
|
+
import hashlib
|
|
11
12
|
import json
|
|
12
13
|
import os
|
|
13
14
|
from datetime import datetime, timezone
|
|
@@ -319,13 +320,78 @@ def seal_plan(state: dict, expected_outcome: str) -> None:
|
|
|
319
320
|
_save(state)
|
|
320
321
|
|
|
321
322
|
|
|
323
|
+
def _normalization_variants(text: str) -> list:
|
|
324
|
+
"""Alternative marker strippings of ``text``, for attributing a mismatch.
|
|
325
|
+
|
|
326
|
+
Each entry is ``(variant_name, stripped_text)``. Only the error path calls
|
|
327
|
+
this, so a whole-file sha256 per variant is affordable; what costs
|
|
328
|
+
something is naming a rule that never existed, because that would
|
|
329
|
+
attribute a real edit to a phantom normalization. Every rule below either
|
|
330
|
+
shipped once or is something a writer could plausibly emit next:
|
|
331
|
+
|
|
332
|
+
* ``marker_line_only`` -- the marker line dropped, its blank separator
|
|
333
|
+
kept. This is exactly what the header branch did before 1.8.1, so a
|
|
334
|
+
regression of that wedge names itself instead of coming back
|
|
335
|
+
disguised as "source content changed".
|
|
336
|
+
* ``unstripped`` -- no marker removal at all, i.e. the plan recorded a
|
|
337
|
+
source that already carried a marker.
|
|
338
|
+
* ``final_newline_*`` -- the current strip with the trailing newline
|
|
339
|
+
added or dropped, for a writer that changes its mind about the last
|
|
340
|
+
byte (14 Raws were appended without one).
|
|
341
|
+
|
|
342
|
+
The marker scan here is deliberately position-blind: a quoted marker in
|
|
343
|
+
the conversation may be picked instead of the anchored one, which makes a
|
|
344
|
+
variant that simply fails to match -- never a wrong attribution.
|
|
345
|
+
"""
|
|
346
|
+
stripped = rs.strip_state_marker(text)
|
|
347
|
+
lines = text.split("\n")
|
|
348
|
+
first = next((i for i, ln in enumerate(lines) if rs.is_marker_line(ln)), None)
|
|
349
|
+
candidates = []
|
|
350
|
+
if first is not None:
|
|
351
|
+
candidates.append(("marker_line_only",
|
|
352
|
+
"\n".join(lines[:first] + lines[first + 1:])))
|
|
353
|
+
candidates.append(("unstripped", text))
|
|
354
|
+
candidates.append(("final_newline_added", stripped + "\n"))
|
|
355
|
+
candidates.append(("final_newline_dropped", stripped.rstrip("\n")))
|
|
356
|
+
seen = {stripped} # the rule in force already failed to match
|
|
357
|
+
out = []
|
|
358
|
+
for name, variant in candidates:
|
|
359
|
+
if variant not in seen:
|
|
360
|
+
seen.add(variant)
|
|
361
|
+
out.append((name, variant))
|
|
362
|
+
return out
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _diagnose_source_mismatch(text: str, rec: dict) -> dict:
|
|
366
|
+
"""Name why ``text`` no longer hashes to the plan's recorded source.
|
|
367
|
+
|
|
368
|
+
A plan stores hashes, not the pre-marker text, so a mismatch cannot be
|
|
369
|
+
diffed -- but it can be attributed. If some other stripping rule
|
|
370
|
+
reproduces the recorded source hash, the conversation is untouched and
|
|
371
|
+
only the marker accounting differs; if none does, the file really was
|
|
372
|
+
edited and "source content changed" is the honest answer. Conflating the
|
|
373
|
+
two is what sent a whole afternoon looking for an editor that had never
|
|
374
|
+
touched the file.
|
|
375
|
+
"""
|
|
376
|
+
for name, variant in _normalization_variants(text):
|
|
377
|
+
digest = hashlib.sha256(variant.encode("utf-8")).hexdigest()
|
|
378
|
+
if digest == rec["source_sha256"]:
|
|
379
|
+
return {"reason": "marker normalization mismatch",
|
|
380
|
+
"normalization": name,
|
|
381
|
+
"hint": "the marker block differs, not the conversation"}
|
|
382
|
+
return {"reason": "source content changed",
|
|
383
|
+
"chars_now": len(text),
|
|
384
|
+
"chars_at_plan_start": rec["char_count"]}
|
|
385
|
+
|
|
386
|
+
|
|
322
387
|
def complete_plan(plan_id: str, raw_path: str) -> dict:
|
|
323
388
|
state = _load_by_plan_id(plan_id)
|
|
324
389
|
if state["status"] != "sealed":
|
|
325
390
|
raise PageError("PLAN_NOT_ACTIVE", reason="seal before complete")
|
|
326
|
-
ident,
|
|
391
|
+
ident, text = rs.load(raw_path)
|
|
327
392
|
if ident.source_sha256 != state["identity"]["source_sha256"]:
|
|
328
|
-
raise PageError("RAW_CHANGED",
|
|
393
|
+
raise PageError("RAW_CHANGED",
|
|
394
|
+
**_diagnose_source_mismatch(text, state["identity"]))
|
|
329
395
|
from .distill_queue import classify
|
|
330
396
|
outcome = classify(raw_path).outcome
|
|
331
397
|
if outcome != state["expected_outcome"]:
|
|
@@ -53,7 +53,7 @@ def _vector_stream(query, n, where):
|
|
|
53
53
|
_STREAM_ORDER = ("vector", "bm25")
|
|
54
54
|
|
|
55
55
|
|
|
56
|
-
def _graph_ranked(ranked, weights, rc, n):
|
|
56
|
+
def _graph_ranked(ranked, weights, rc, n, where=None):
|
|
57
57
|
"""Build the graph RRF stream: wikilink-neighbors of the top vector/bm25 hits.
|
|
58
58
|
|
|
59
59
|
Returns (ranked_list, display_map) where ranked_list is [(doc_id, rank)] for
|
|
@@ -61,7 +61,14 @@ def _graph_ranked(ranked, weights, rc, n):
|
|
|
61
61
|
([], {}) on failure (e.g. vault unconfigured) or when graph yields nothing.
|
|
62
62
|
Rank-based (not additive), so it composes with the other streams via RRF
|
|
63
63
|
without the score-scale conflict that the old additive boost had.
|
|
64
|
+
|
|
65
|
+
`where` is evaluated with the same `bm25._matches` the keyword stream uses:
|
|
66
|
+
three streams, one filter language. This stream used to receive no `where`
|
|
67
|
+
at all, so wikilink neighbours entered the fused set past `--repo`/`--type`/
|
|
68
|
+
`--category`. A doc with no metadata is rejected rather than admitted — we
|
|
69
|
+
cannot show it passed the filter.
|
|
64
70
|
"""
|
|
71
|
+
from . import bm25 as bm25_mod
|
|
65
72
|
from . import graph as graph_mod
|
|
66
73
|
try:
|
|
67
74
|
pre = rrf_fuse(ranked, weights, k=rc["rrf_k"])
|
|
@@ -69,10 +76,20 @@ def _graph_ranked(ranked, weights, rc, n):
|
|
|
69
76
|
if not seeds:
|
|
70
77
|
return [], {}
|
|
71
78
|
adjacency, meta = graph_mod.build_graph(get_vault_path())
|
|
79
|
+
keep = None
|
|
80
|
+
if where:
|
|
81
|
+
def keep(doc_id):
|
|
82
|
+
rec = meta.get(doc_id)
|
|
83
|
+
return rec is not None and bm25_mod._matches(rec, where)
|
|
72
84
|
stream = graph_mod.graph_stream(
|
|
73
|
-
adjacency, seeds, hops=rc["graph_hops"], max_n=max(n * 2, 10)
|
|
85
|
+
adjacency, seeds, hops=rc["graph_hops"], max_n=max(n * 2, 10), keep=keep
|
|
74
86
|
)
|
|
75
|
-
|
|
87
|
+
# `repos` is a filter field, not a display field: the other streams do
|
|
88
|
+
# not emit it, so it must not reach the result dicts from this one.
|
|
89
|
+
gdisplay = {
|
|
90
|
+
doc_id: {k: v for k, v in meta[doc_id].items() if k != "repos"}
|
|
91
|
+
for doc_id, _ in stream if doc_id in meta
|
|
92
|
+
}
|
|
76
93
|
return stream, gdisplay
|
|
77
94
|
except (Exception, SystemExit):
|
|
78
95
|
return [], {}
|
|
@@ -133,7 +150,7 @@ def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, r
|
|
|
133
150
|
|
|
134
151
|
use_graph = rc["graph"] if graph is None else graph
|
|
135
152
|
if use_graph:
|
|
136
|
-
g_ranked, g_display = _graph_ranked(ranked, weights, rc, n)
|
|
153
|
+
g_ranked, g_display = _graph_ranked(ranked, weights, rc, n, where)
|
|
137
154
|
if g_ranked:
|
|
138
155
|
ranked["graph"] = g_ranked
|
|
139
156
|
for doc_id, disp in g_display.items():
|
|
@@ -150,14 +167,24 @@ def search(query, n=5, where=None, use_bm25=True, use_vector=True, graph=None, r
|
|
|
150
167
|
use_rerank = rc["rerank"] if rerank is None else rerank
|
|
151
168
|
take = max(n, rc["rerank_window"]) if use_rerank else n
|
|
152
169
|
|
|
170
|
+
head = fused[:take]
|
|
171
|
+
# Hits that arrived via bm25 or graph never passed through the vector
|
|
172
|
+
# top-n window, so they carry no cosine. Ask the collection for theirs
|
|
173
|
+
# rather than reporting 0.0, which reads as "no overlap" downstream.
|
|
174
|
+
missing = [doc_id for doc_id, _ in head if doc_id not in vec_score]
|
|
175
|
+
if missing and use_vector and os.environ.get("OPENAI_API_KEY"):
|
|
176
|
+
try:
|
|
177
|
+
vec_score.update(store.cosine_for(query, missing))
|
|
178
|
+
except (Exception, SystemExit):
|
|
179
|
+
pass # same degradation contract as the streams: score falls to 0.0
|
|
180
|
+
|
|
153
181
|
out = []
|
|
154
|
-
for doc_id, _rrf in
|
|
182
|
+
for doc_id, _rrf in head:
|
|
155
183
|
entry = dict(display.get(doc_id, {}))
|
|
156
184
|
entry["id"] = doc_id
|
|
157
185
|
# `score` reports the vector cosine similarity (0-1, interpretable) so
|
|
158
186
|
# absolute-threshold consumers (distill/broadcast dedup) keep working.
|
|
159
|
-
# RRF only determines ORDER (already baked into `fused`).
|
|
160
|
-
# have no cosine score -> 0.0.
|
|
187
|
+
# RRF only determines ORDER (already baked into `fused`).
|
|
161
188
|
entry["score"] = round(vec_score.get(doc_id, 0.0), 4)
|
|
162
189
|
out.append(entry)
|
|
163
190
|
|
|
@@ -11,6 +11,7 @@ Graph participates in retrieval as a THIRD RRF stream (see fusion.py): a
|
|
|
11
11
|
rank-based list of wikilink-neighbors of the top hits. Rank-based fusion avoids
|
|
12
12
|
the scale conflict of adding a boost onto RRF's compressed score band.
|
|
13
13
|
"""
|
|
14
|
+
import hashlib
|
|
14
15
|
from pathlib import Path
|
|
15
16
|
|
|
16
17
|
from .parser import classify_path, extract_summary, extract_wikilinks, parse_document
|
|
@@ -29,7 +30,11 @@ def _meta_for(rel, fm, body):
|
|
|
29
30
|
"id": rel,
|
|
30
31
|
"title": fm.get("title", rel.rsplit("/", 1)[-1].removesuffix(".md")),
|
|
31
32
|
"type": doc_type,
|
|
33
|
+
# `repo` is the display field (first membership); `repos` is the full
|
|
34
|
+
# membership list the `where` evaluator reads. Keeping only the singular
|
|
35
|
+
# form let a multi-repo page fail a filter naming its second repo.
|
|
32
36
|
"repo": (repos or [""])[0],
|
|
37
|
+
"repos": repos,
|
|
33
38
|
"category": category,
|
|
34
39
|
"tags": fm.get("tags", ""),
|
|
35
40
|
"summary": extract_summary(body),
|
|
@@ -75,6 +80,15 @@ def build_graph(vault):
|
|
|
75
80
|
return _cache[key]
|
|
76
81
|
|
|
77
82
|
|
|
83
|
+
def _tiebreak(doc_id):
|
|
84
|
+
"""Stable, path-insensitive ordering key for equidistant neighbours.
|
|
85
|
+
|
|
86
|
+
blake2b rather than `hash()`: the builtin is salted per process by
|
|
87
|
+
PYTHONHASHSEED, which is the non-determinism this exists to remove.
|
|
88
|
+
"""
|
|
89
|
+
return hashlib.blake2b(doc_id.encode("utf-8"), digest_size=8).digest()
|
|
90
|
+
|
|
91
|
+
|
|
78
92
|
def _bfs_neighbors(adjacency, seeds, hops):
|
|
79
93
|
"""Return {base_path: distance} reachable within `hops` from seeds (excluding seeds)."""
|
|
80
94
|
frontier = set(seeds)
|
|
@@ -94,14 +108,41 @@ def _bfs_neighbors(adjacency, seeds, hops):
|
|
|
94
108
|
return dist
|
|
95
109
|
|
|
96
110
|
|
|
97
|
-
def graph_stream(adjacency, seeds, hops=1, max_n=15):
|
|
111
|
+
def graph_stream(adjacency, seeds, hops=1, max_n=15, keep=None):
|
|
98
112
|
"""A rank-based stream of wikilink-neighbors of `seeds`, nearest first.
|
|
99
113
|
|
|
100
114
|
Returns [(doc_id, rank)] (rank 0-based, capped at max_n), excluding seeds —
|
|
101
115
|
shaped for rrf_fuse as a third retrieval stream. Empty if no neighbors.
|
|
116
|
+
|
|
117
|
+
`keep` is an optional predicate on doc_id applied BEFORE the max_n cut, so a
|
|
118
|
+
filtered query still gets a full window of admissible neighbours rather than
|
|
119
|
+
a window thinned by rejects. It carries the caller's `where` clause: without
|
|
120
|
+
it this stream was the one path into the fused set that no filter reached.
|
|
121
|
+
|
|
122
|
+
`keep` screens what is RETURNED, not what the walk may cross. With
|
|
123
|
+
`graph_hops > 1` a neighbour reachable only *through* a page the filter
|
|
124
|
+
rejects is still returned if it passes itself. That is deliberate — the
|
|
125
|
+
clause describes the answer set, not the path — but it means raising
|
|
126
|
+
`graph_hops` widens what a scoped query can reach. Moot at the current
|
|
127
|
+
`graph_hops = 1`, where there are no intermediate nodes.
|
|
102
128
|
"""
|
|
103
129
|
dist = _bfs_neighbors(adjacency, seeds, hops)
|
|
104
130
|
if not dist:
|
|
105
131
|
return []
|
|
106
|
-
|
|
132
|
+
# Tie-break deterministically, but NOT on the doc id itself. `dist` is
|
|
133
|
+
# populated by iterating adjacency *sets*, so equidistant neighbours arrive
|
|
134
|
+
# in an order that varies with PYTHONHASHSEED, and sorting on distance alone
|
|
135
|
+
# is stable -- it preserved that variation, leaving the stream irreproducible
|
|
136
|
+
# across processes. Ordering by doc id fixes that but buys a worse problem:
|
|
137
|
+
# at graph_hops=1 every neighbour is distance 1, so the id becomes the sole
|
|
138
|
+
# key and the max_n cut degenerates into "keep the lexicographically smallest
|
|
139
|
+
# N". `Notes/` sorts before `Projects/`, so scoped queries -- the ones that
|
|
140
|
+
# exist to surface a repo's Projects/ pages -- were the worst affected.
|
|
141
|
+
# Measured over 300 realistic 5-seed calls: truncation fired on 70% of them
|
|
142
|
+
# and lifted Notes/ from 46% of the candidates to 68% of the survivors.
|
|
143
|
+
# A keyed digest is reproducible across processes like the id, without
|
|
144
|
+
# correlating to the path prefix.
|
|
145
|
+
ordered = sorted(dist.items(), key=lambda kv: (kv[1], _tiebreak(kv[0]))) # nearest first
|
|
146
|
+
if keep is not None:
|
|
147
|
+
ordered = [item for item in ordered if keep(item[0])]
|
|
107
148
|
return [(doc_id, rank) for rank, (doc_id, _d) in enumerate(ordered[:max_n])]
|
|
@@ -89,6 +89,10 @@ def _dispatch(args) -> None:
|
|
|
89
89
|
raise PageError("CURSOR_MISMATCH", reason="cursor beyond span count")
|
|
90
90
|
|
|
91
91
|
find = getattr(args, "find", None)
|
|
92
|
+
find_only = bool(getattr(args, "find_only", False))
|
|
93
|
+
if find_only and not find:
|
|
94
|
+
raise PageError("FIND_ONLY_REQUIRES_FIND",
|
|
95
|
+
reason="--find-only needs --find <literal>")
|
|
92
96
|
find_matches = None
|
|
93
97
|
if find:
|
|
94
98
|
find_matches = []
|
|
@@ -122,8 +126,23 @@ def _dispatch(args) -> None:
|
|
|
122
126
|
}
|
|
123
127
|
if find_matches is not None:
|
|
124
128
|
obj["find_matches"] = find_matches
|
|
129
|
+
if find_only:
|
|
130
|
+
# Locate only: no cards, no map progress, no continuation.
|
|
131
|
+
obj["find_only"] = True
|
|
132
|
+
obj["next_cursor"] = None
|
|
125
133
|
return obj
|
|
126
134
|
|
|
135
|
+
if find_only:
|
|
136
|
+
obj, chars = finalize_page(
|
|
137
|
+
lambda pc, ua: envelope([], start, pc, ua), used_before)
|
|
138
|
+
if chars > max_chars:
|
|
139
|
+
raise PageError("PAGE_BUDGET_TOO_SMALL", max_chars=max_chars,
|
|
140
|
+
reason="find-only envelope exceeds budget")
|
|
141
|
+
dp.charge_and_save(state, chars)
|
|
142
|
+
assert len(render_page(obj)) + 1 == chars
|
|
143
|
+
emit(obj)
|
|
144
|
+
return
|
|
145
|
+
|
|
127
146
|
cards: list = []
|
|
128
147
|
idx = start
|
|
129
148
|
final_obj = None
|
|
@@ -76,8 +76,37 @@ class SourceSpan:
|
|
|
76
76
|
tool_name: str | None = None
|
|
77
77
|
|
|
78
78
|
|
|
79
|
+
def is_marker_line(line: str) -> bool:
|
|
80
|
+
"""True if ``line`` is a whole-line distilled marker, indent ignored.
|
|
81
|
+
|
|
82
|
+
Public so callers outside this module can recognise a marker without
|
|
83
|
+
reaching for the private pattern -- position anchoring stays here.
|
|
84
|
+
"""
|
|
85
|
+
return bool(_MARKER_RE.match(line.strip()))
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _drop_marker_block(lines: list[str], i: int) -> str:
|
|
89
|
+
"""Rejoin ``lines`` without the marker block anchored at index ``i``.
|
|
90
|
+
|
|
91
|
+
A marker is written as a BLOCK, not a line: the writers emit
|
|
92
|
+
``"\\n<!-- distilled: ... -->\\n"``, so the marker arrives together with a
|
|
93
|
+
blank separator line. Removing only the marker leaves that blank behind and
|
|
94
|
+
the stripped text no longer hashes to the pre-marker source. Both anchoring
|
|
95
|
+
conventions share this helper precisely because they once drifted: the
|
|
96
|
+
header branch kept the blank, so a turnless Raw (no ``### User`` at all,
|
|
97
|
+
e.g. a vacuum session) fell through the header loop, got the header
|
|
98
|
+
treatment for its EOF marker, and stayed wedged in RAW_CHANGED forever.
|
|
99
|
+
"""
|
|
100
|
+
head = lines[:i - 1] if i > 0 and not lines[i - 1].strip() else lines[:i]
|
|
101
|
+
tail = lines[i + 1:]
|
|
102
|
+
# An empty tail means the marker was appended without a trailing newline,
|
|
103
|
+
# so the split() carries no final empty element. Re-emit one, or the line
|
|
104
|
+
# above would silently lose its own newline.
|
|
105
|
+
return "\n".join(head + (tail or [""]))
|
|
106
|
+
|
|
107
|
+
|
|
79
108
|
def strip_state_marker(text: str) -> str:
|
|
80
|
-
"""Remove the ONE position-anchored distilled marker
|
|
109
|
+
"""Remove the ONE position-anchored distilled marker block, if present.
|
|
81
110
|
|
|
82
111
|
Same anchoring rules as distill_queue: the first whole-line marker BEFORE
|
|
83
112
|
the first turn/tool header (header convention), else the last non-empty
|
|
@@ -92,17 +121,13 @@ def strip_state_marker(text: str) -> str:
|
|
|
92
121
|
if s in TURN_HEADERS:
|
|
93
122
|
break
|
|
94
123
|
if _MARKER_RE.match(s):
|
|
95
|
-
return
|
|
124
|
+
return _drop_marker_block(lines, i)
|
|
96
125
|
for i in range(len(lines) - 1, -1, -1):
|
|
97
126
|
s = lines[i].strip()
|
|
98
127
|
if not s:
|
|
99
128
|
continue
|
|
100
129
|
if _MARKER_RE.match(s):
|
|
101
|
-
|
|
102
|
-
if i > 0 and not lines[i - 1].strip():
|
|
103
|
-
return "\n".join(lines[:i - 1] + lines[i + 1:])
|
|
104
|
-
else:
|
|
105
|
-
return "\n".join(lines[:i] + lines[i + 1:])
|
|
130
|
+
return _drop_marker_block(lines, i)
|
|
106
131
|
break
|
|
107
132
|
return text
|
|
108
133
|
|
|
@@ -8,7 +8,7 @@ one — pure redundancy that would otherwise each sit in the distill queue as it
|
|
|
8
8
|
own entry, spending the distill budget several times over on the same
|
|
9
9
|
conversation and landing duplicate Notes.
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
Three safety properties make removal sound:
|
|
12
12
|
|
|
13
13
|
* the candidate set is exactly :func:`distill_queue`'s output, i.e. Raw files
|
|
14
14
|
nothing references yet. An already-distilled Raw carries a position-anchored
|
|
@@ -18,17 +18,30 @@ Two safety properties make removal sound:
|
|
|
18
18
|
survivor lacks. If the filter output ever diverges (its LLM residue
|
|
19
19
|
classifier is not deterministic) the match simply fails and nothing is
|
|
20
20
|
removed — the failure mode is "duplicate stays", never "content lost".
|
|
21
|
+
* both files must carry the same ``repo:`` label. A prefix alone does not prove
|
|
22
|
+
redundancy: one conversation whose cwd changes mid-session produces two Raws
|
|
23
|
+
from the same growing transcript but under DIFFERENT repo labels, and the
|
|
24
|
+
earlier one may be that repo's only record, so reclaiming it would erase that
|
|
25
|
+
repo from the vault's history of the period entirely. Such a
|
|
26
|
+
pair is refused and reported (see :class:`CrossRepoPair`) rather than dropped
|
|
27
|
+
in silence — a cross-repo prefix pair is worth an operator's attention.
|
|
21
28
|
|
|
22
29
|
Comparison starts at the first conversation turn: the frontmatter holds the
|
|
23
30
|
wall-clock stamp that differs between recordings of the same session.
|
|
24
31
|
"""
|
|
25
32
|
from __future__ import annotations
|
|
26
33
|
|
|
34
|
+
import re
|
|
27
35
|
import subprocess
|
|
36
|
+
import sys
|
|
37
|
+
from dataclasses import dataclass, field
|
|
28
38
|
from pathlib import Path
|
|
29
39
|
|
|
30
40
|
from .distill_queue import _TURN_HDRS, distill_queue
|
|
31
41
|
|
|
42
|
+
# `repo: <name>` inside the leading frontmatter block, as session-end writes it.
|
|
43
|
+
_REPO_RE = re.compile(r"^repo:\s*(.*?)\s*$")
|
|
44
|
+
|
|
32
45
|
|
|
33
46
|
def _body(path) -> list[str]:
|
|
34
47
|
"""Conversation body: lines from the first turn header to EOF.
|
|
@@ -51,6 +64,29 @@ def _body(path) -> list[str]:
|
|
|
51
64
|
return lines
|
|
52
65
|
|
|
53
66
|
|
|
67
|
+
def _repo(path) -> str:
|
|
68
|
+
"""The ``repo:`` label from the leading frontmatter block.
|
|
69
|
+
|
|
70
|
+
Returns ``""`` when the file has no frontmatter or no ``repo:`` key — an
|
|
71
|
+
unlabelled Raw therefore differs from every labelled one and is refused,
|
|
72
|
+
which is the safe direction.
|
|
73
|
+
"""
|
|
74
|
+
try:
|
|
75
|
+
with Path(path).open(encoding="utf-8") as f:
|
|
76
|
+
if f.readline().rstrip("\n") != "---":
|
|
77
|
+
return ""
|
|
78
|
+
for line in f:
|
|
79
|
+
stripped = line.rstrip("\n")
|
|
80
|
+
if stripped == "---" or stripped.strip() in _TURN_HDRS:
|
|
81
|
+
break
|
|
82
|
+
m = _REPO_RE.match(stripped)
|
|
83
|
+
if m:
|
|
84
|
+
return m.group(1)
|
|
85
|
+
except OSError:
|
|
86
|
+
return ""
|
|
87
|
+
return ""
|
|
88
|
+
|
|
89
|
+
|
|
54
90
|
def _covers(survivor: list[str], candidate: list[str]) -> bool:
|
|
55
91
|
"""True iff candidate is a non-empty prefix of survivor."""
|
|
56
92
|
return (
|
|
@@ -60,35 +96,90 @@ def _covers(survivor: list[str], candidate: list[str]) -> bool:
|
|
|
60
96
|
)
|
|
61
97
|
|
|
62
98
|
|
|
63
|
-
|
|
99
|
+
@dataclass(frozen=True)
|
|
100
|
+
class CrossRepoPair:
|
|
101
|
+
"""A prefix pair reclaim refused because the two Raws name different repos."""
|
|
102
|
+
|
|
103
|
+
candidate: Path
|
|
104
|
+
survivor: Path
|
|
105
|
+
candidate_repo: str
|
|
106
|
+
survivor_repo: str
|
|
107
|
+
|
|
108
|
+
def describe(self) -> str:
|
|
109
|
+
return (
|
|
110
|
+
f"refused: {self.candidate} (repo: {self.candidate_repo or '?'}) is a "
|
|
111
|
+
f"prefix of {self.survivor} (repo: {self.survivor_repo or '?'}) — "
|
|
112
|
+
"different repo, kept"
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@dataclass(frozen=True)
|
|
117
|
+
class ReclaimScan:
|
|
118
|
+
"""Outcome of one scan: what may go, and what was refused and why."""
|
|
119
|
+
|
|
120
|
+
superseded: list[Path] = field(default_factory=list)
|
|
121
|
+
refused: list[CrossRepoPair] = field(default_factory=list)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def scan(root, keep=None) -> ReclaimScan:
|
|
64
125
|
"""Undistilled Raw files made redundant by a longer recording (FIFO order).
|
|
65
126
|
|
|
66
127
|
With ``keep``, only that file is treated as a survivor — the session-end
|
|
67
128
|
path, one body read per queued file. Without it, the whole queue is
|
|
68
129
|
compared pairwise (backlog cleanup). For two byte-identical bodies the
|
|
69
130
|
later path survives, so exactly one of the pair is reclaimed.
|
|
131
|
+
|
|
132
|
+
A prefix pair whose ``repo:`` labels differ lands in ``refused`` instead of
|
|
133
|
+
``superseded``: the candidate stays on disk and the pair is reported.
|
|
70
134
|
"""
|
|
71
135
|
queue = distill_queue(root)
|
|
72
136
|
if keep is not None:
|
|
73
137
|
survivor = _body(keep)
|
|
74
138
|
if not survivor:
|
|
75
|
-
return
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
139
|
+
return ReclaimScan()
|
|
140
|
+
keep_path = Path(keep)
|
|
141
|
+
keep_resolved = keep_path.resolve()
|
|
142
|
+
keep_repo = _repo(keep_path)
|
|
143
|
+
result = ReclaimScan()
|
|
144
|
+
for p in queue:
|
|
145
|
+
if p.resolve() == keep_resolved or not _covers(survivor, _body(p)):
|
|
146
|
+
continue
|
|
147
|
+
cand_repo = _repo(p)
|
|
148
|
+
if cand_repo != keep_repo:
|
|
149
|
+
result.refused.append(
|
|
150
|
+
CrossRepoPair(p, keep_path, cand_repo, keep_repo)
|
|
151
|
+
)
|
|
152
|
+
continue
|
|
153
|
+
result.superseded.append(p)
|
|
154
|
+
return result
|
|
81
155
|
|
|
82
156
|
bodies = {p: _body(p) for p in queue}
|
|
83
|
-
|
|
157
|
+
repos = {p: _repo(p) for p in queue}
|
|
158
|
+
result = ReclaimScan()
|
|
84
159
|
for cand, cand_body in bodies.items():
|
|
160
|
+
pending: list[CrossRepoPair] = []
|
|
85
161
|
for other, other_body in bodies.items():
|
|
86
162
|
if other == cand or not _covers(other_body, cand_body):
|
|
87
163
|
continue
|
|
88
|
-
if len(cand_body) < len(other_body) or str(cand) < str(other):
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
164
|
+
if not (len(cand_body) < len(other_body) or str(cand) < str(other)):
|
|
165
|
+
continue
|
|
166
|
+
if repos[cand] != repos[other]:
|
|
167
|
+
pending.append(
|
|
168
|
+
CrossRepoPair(cand, other, repos[cand], repos[other])
|
|
169
|
+
)
|
|
170
|
+
continue
|
|
171
|
+
# A same-repo survivor settles it; the cross-repo near-misses seen
|
|
172
|
+
# on the way are not worth reporting once the file is going anyway.
|
|
173
|
+
result.superseded.append(cand)
|
|
174
|
+
pending = []
|
|
175
|
+
break
|
|
176
|
+
result.refused.extend(pending)
|
|
177
|
+
return result
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def find_superseded(root, keep=None) -> list[Path]:
|
|
181
|
+
"""The reclaimable subset of :func:`scan` (refusals dropped)."""
|
|
182
|
+
return scan(root, keep=keep).superseded
|
|
92
183
|
|
|
93
184
|
|
|
94
185
|
def _git_rm(vault, path) -> bool:
|
|
@@ -129,7 +220,8 @@ def dispatch(args) -> None:
|
|
|
129
220
|
from .config import get_vault_path
|
|
130
221
|
|
|
131
222
|
root = get_vault_path() / "Raw"
|
|
132
|
-
|
|
223
|
+
result = scan(root, keep=getattr(args, "keep", None))
|
|
224
|
+
found = result.superseded
|
|
133
225
|
if getattr(args, "apply", False):
|
|
134
226
|
vault = getattr(args, "vault", None)
|
|
135
227
|
if not vault:
|
|
@@ -139,3 +231,8 @@ def dispatch(args) -> None:
|
|
|
139
231
|
found = apply_reclaim(found, vault=vault)
|
|
140
232
|
for path in found:
|
|
141
233
|
print(path)
|
|
234
|
+
# Refusals go to stderr in BOTH modes: stdout is the machine-readable list
|
|
235
|
+
# of reclaimed paths (session-end-record.sh counts its lines for the commit
|
|
236
|
+
# message), so a refusal printed there would be miscounted as a removal.
|
|
237
|
+
for pair in result.refused:
|
|
238
|
+
print(pair.describe(), file=sys.stderr)
|