cortex-vec 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cortex_vec/__init__.py +3 -0
- cortex_vec/bm25.py +129 -0
- cortex_vec/cli.py +191 -0
- cortex_vec/config.py +82 -0
- cortex_vec/distill_plan.py +431 -0
- cortex_vec/distill_queue.py +171 -0
- cortex_vec/eval/__init__.py +0 -0
- cortex_vec/eval/adapters.py +83 -0
- cortex_vec/eval/corpus.py +36 -0
- cortex_vec/eval/report.py +44 -0
- cortex_vec/eval/run.py +130 -0
- cortex_vec/eval/score.py +62 -0
- cortex_vec/fusion.py +168 -0
- cortex_vec/graph.py +107 -0
- cortex_vec/parser.py +77 -0
- cortex_vec/raw_map.py +165 -0
- cortex_vec/raw_page.py +69 -0
- cortex_vec/raw_source.py +306 -0
- cortex_vec/raw_span.py +96 -0
- cortex_vec/raw_view.py +264 -0
- cortex_vec/reclaim.py +141 -0
- cortex_vec/rerank.py +52 -0
- cortex_vec/store.py +466 -0
- cortex_vec/synonyms.py +43 -0
- cortex_vec/tokenize.py +79 -0
- cortex_vec-0.7.0.dist-info/METADATA +68 -0
- cortex_vec-0.7.0.dist-info/RECORD +32 -0
- cortex_vec-0.7.0.dist-info/WHEEL +5 -0
- cortex_vec-0.7.0.dist-info/entry_points.txt +2 -0
- cortex_vec-0.7.0.dist-info/licenses/LICENSE +201 -0
- cortex_vec-0.7.0.dist-info/licenses/NOTICE +5 -0
- cortex_vec-0.7.0.dist-info/top_level.txt +1 -0
cortex_vec/raw_map.py
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""raw-map: bounded navigation cards over the source partition.
|
|
2
|
+
|
|
3
|
+
Navigation, not judgment: cards expose kind/coordinates/size/preview/
|
|
4
|
+
deterministic lexical anchors. Emitting a card auto-reviews only the
|
|
5
|
+
structural classes (CARD_REVIEW_KINDS, and PREVIEW kinds when the preview
|
|
6
|
+
is complete); prose/output_body/ambiguous/opaque always need raw-span.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
|
|
12
|
+
from . import distill_plan as dp
|
|
13
|
+
from . import raw_source as rs
|
|
14
|
+
from .raw_page import (PAGE_BUDGET_DEFAULT, PageError, decode_cursor,
|
|
15
|
+
emit, emit_error, encode_cursor, finalize_page,
|
|
16
|
+
render_page)
|
|
17
|
+
|
|
18
|
+
PREVIEW_CHARS = 160
|
|
19
|
+
_ISSUE_RE = re.compile(r"\b[A-Z][A-Z0-9]{1,9}-\d{3,7}\b")
|
|
20
|
+
_URL_RE = re.compile(r"https?://\S+")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def lexical_anchors(text: str, limit: int = 8) -> list:
|
|
24
|
+
out, seen = [], set()
|
|
25
|
+
for rx in (_ISSUE_RE, _URL_RE):
|
|
26
|
+
for m in rx.finditer(text):
|
|
27
|
+
v = m.group(0)
|
|
28
|
+
if v not in seen:
|
|
29
|
+
seen.add(v)
|
|
30
|
+
out.append(v)
|
|
31
|
+
if len(out) >= limit:
|
|
32
|
+
return out
|
|
33
|
+
return out
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def build_card(span, text: str, preview_chars: int = PREVIEW_CHARS) -> dict:
|
|
37
|
+
seg = text[span.char_start:span.char_end]
|
|
38
|
+
if span.kind == "blank":
|
|
39
|
+
preview, complete = "", True
|
|
40
|
+
else:
|
|
41
|
+
preview = seg[:preview_chars]
|
|
42
|
+
complete = len(preview) == len(seg)
|
|
43
|
+
card = {
|
|
44
|
+
"span_id": span.id,
|
|
45
|
+
"kind": span.kind,
|
|
46
|
+
"confidence": span.confidence,
|
|
47
|
+
"char_start": span.char_start,
|
|
48
|
+
"char_end": span.char_end,
|
|
49
|
+
"line_start": span.line_start,
|
|
50
|
+
"line_end": span.line_end,
|
|
51
|
+
"source_chars": span.char_end - span.char_start,
|
|
52
|
+
"preview": preview,
|
|
53
|
+
"preview_complete": complete,
|
|
54
|
+
}
|
|
55
|
+
if span.tool_name:
|
|
56
|
+
card["tool_name"] = span.tool_name
|
|
57
|
+
ax = lexical_anchors(seg)
|
|
58
|
+
if ax:
|
|
59
|
+
card["lexical_anchors"] = ax
|
|
60
|
+
return card
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _auto_reviewed(span, card) -> bool:
|
|
64
|
+
if span.kind in rs.CARD_REVIEW_KINDS:
|
|
65
|
+
return True
|
|
66
|
+
return span.kind in rs.PREVIEW_REVIEW_KINDS and card["preview_complete"]
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _cursor(identity, next_index: int) -> str:
|
|
70
|
+
return encode_cursor({"k": "map", "s": identity.schema_version,
|
|
71
|
+
"p": identity.parser_version,
|
|
72
|
+
"f": identity.file_sha256[:16], "i": next_index})
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def dispatch(args) -> None:
|
|
76
|
+
try:
|
|
77
|
+
_dispatch(args)
|
|
78
|
+
except PageError as e:
|
|
79
|
+
emit_error(e.code, **e.detail)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _dispatch(args) -> None:
|
|
83
|
+
max_chars = getattr(args, "max_chars", None) or PAGE_BUDGET_DEFAULT
|
|
84
|
+
state, ident, text, spans = dp.load_for_page(args.plan_id, args.path)
|
|
85
|
+
start = 0
|
|
86
|
+
if getattr(args, "cursor", None):
|
|
87
|
+
start = decode_cursor(args.cursor, "map", ident)["i"]
|
|
88
|
+
if start > len(spans):
|
|
89
|
+
raise PageError("CURSOR_MISMATCH", reason="cursor beyond span count")
|
|
90
|
+
|
|
91
|
+
find = getattr(args, "find", None)
|
|
92
|
+
find_matches = None
|
|
93
|
+
if find:
|
|
94
|
+
find_matches = []
|
|
95
|
+
pos = text.find(find)
|
|
96
|
+
while pos != -1 and len(find_matches) < 50:
|
|
97
|
+
for s in spans:
|
|
98
|
+
if s.char_start <= pos < s.char_end \
|
|
99
|
+
and s.id not in find_matches:
|
|
100
|
+
find_matches.append(s.id)
|
|
101
|
+
break
|
|
102
|
+
pos = text.find(find, pos + 1)
|
|
103
|
+
|
|
104
|
+
used_before = state["session_used_chars"]
|
|
105
|
+
budget = state["session_budget"] - dp.CONTROL_RESERVE
|
|
106
|
+
|
|
107
|
+
def envelope(cards, next_index, page_chars, used_after):
|
|
108
|
+
obj = {
|
|
109
|
+
"schema_version": rs.SCHEMA_VERSION,
|
|
110
|
+
"raw_identity": ident.to_json(),
|
|
111
|
+
"plan_id": state["plan_id"],
|
|
112
|
+
"cards": cards,
|
|
113
|
+
"page_chars": page_chars,
|
|
114
|
+
"session_used_chars": used_after,
|
|
115
|
+
"session_remaining_chars": max(0, budget - used_after),
|
|
116
|
+
"map_coverage": {"next_index": max(state["map_next_index"],
|
|
117
|
+
next_index),
|
|
118
|
+
"span_count": len(spans)},
|
|
119
|
+
"next_cursor": (_cursor(ident, next_index)
|
|
120
|
+
if next_index < len(spans) else None),
|
|
121
|
+
"budget_status": "ok",
|
|
122
|
+
}
|
|
123
|
+
if find_matches is not None:
|
|
124
|
+
obj["find_matches"] = find_matches
|
|
125
|
+
return obj
|
|
126
|
+
|
|
127
|
+
cards: list = []
|
|
128
|
+
idx = start
|
|
129
|
+
final_obj = None
|
|
130
|
+
final_chars = None
|
|
131
|
+
while idx < len(spans):
|
|
132
|
+
cards.append(build_card(spans[idx], text))
|
|
133
|
+
try:
|
|
134
|
+
obj, chars = finalize_page(
|
|
135
|
+
lambda pc, ua, c=list(cards), nx=idx + 1:
|
|
136
|
+
envelope(c, nx, pc, ua), used_before)
|
|
137
|
+
except PageError:
|
|
138
|
+
obj, chars = None, max_chars + 1
|
|
139
|
+
if chars > max_chars:
|
|
140
|
+
cards.pop()
|
|
141
|
+
break
|
|
142
|
+
final_obj, final_chars = obj, chars
|
|
143
|
+
idx += 1
|
|
144
|
+
if final_obj is None:
|
|
145
|
+
if start >= len(spans):
|
|
146
|
+
# Empty tail page (empty Raw, or a cursor already at span_count):
|
|
147
|
+
# still bounded -- if even the empty envelope exceeds the budget,
|
|
148
|
+
# the budget is too small to represent a page at all.
|
|
149
|
+
obj, chars = finalize_page(
|
|
150
|
+
lambda pc, ua: envelope([], start, pc, ua), used_before)
|
|
151
|
+
if chars > max_chars:
|
|
152
|
+
raise PageError("PAGE_BUDGET_TOO_SMALL", max_chars=max_chars,
|
|
153
|
+
reason="empty page envelope exceeds budget")
|
|
154
|
+
final_obj, final_chars = obj, chars
|
|
155
|
+
else:
|
|
156
|
+
raise PageError("PAGE_BUDGET_TOO_SMALL",
|
|
157
|
+
span_id=spans[start].id, max_chars=max_chars)
|
|
158
|
+
|
|
159
|
+
dp.charge_and_save(state, final_chars)
|
|
160
|
+
auto = [[spans[k].char_start, spans[k].char_end]
|
|
161
|
+
for k in range(start, idx)
|
|
162
|
+
if _auto_reviewed(spans[k], build_card(spans[k], text))]
|
|
163
|
+
dp.mark_map_progress(state, idx, auto)
|
|
164
|
+
assert len(render_page(final_obj)) + 1 == final_chars
|
|
165
|
+
emit(final_obj)
|
cortex_vec/raw_page.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Bounded-page plumbing shared by raw-map / raw-span / distill-plan.
|
|
2
|
+
|
|
3
|
+
page_chars counts the FINAL emitted stdout including print's trailing
|
|
4
|
+
newline: len(render_page(obj)) + 1. Because the object embeds its own
|
|
5
|
+
page_chars and the post-charge session counter, the size is resolved by
|
|
6
|
+
fixpoint iteration (monotone in digit width, so it converges).
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import base64
|
|
11
|
+
import json
|
|
12
|
+
import sys
|
|
13
|
+
|
|
14
|
+
PAGE_BUDGET_DEFAULT = 12000
|
|
15
|
+
SESSION_BUDGET_DEFAULT = 100000
|
|
16
|
+
CONTROL_RESERVE = 2048
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class PageError(Exception):
|
|
20
|
+
def __init__(self, code: str, **detail):
|
|
21
|
+
self.code = code
|
|
22
|
+
self.detail = detail
|
|
23
|
+
super().__init__(code)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def render_page(obj) -> str:
|
|
27
|
+
return json.dumps(obj, ensure_ascii=False, separators=(",", ":"))
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def encode_cursor(payload: dict) -> str:
|
|
31
|
+
raw = json.dumps(payload, separators=(",", ":"), sort_keys=True)
|
|
32
|
+
return base64.urlsafe_b64encode(raw.encode("utf-8")).decode("ascii")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def decode_cursor(cursor: str, expect_kind: str, identity) -> dict:
|
|
36
|
+
try:
|
|
37
|
+
payload = json.loads(base64.urlsafe_b64decode(cursor.encode("ascii")))
|
|
38
|
+
except Exception:
|
|
39
|
+
raise PageError("CURSOR_MISMATCH", reason="undecodable cursor")
|
|
40
|
+
if not (isinstance(payload, dict)
|
|
41
|
+
and payload.get("k") == expect_kind
|
|
42
|
+
and payload.get("s") == identity.schema_version
|
|
43
|
+
and payload.get("p") == identity.parser_version
|
|
44
|
+
and payload.get("f") == identity.file_sha256[:16]):
|
|
45
|
+
raise PageError("CURSOR_MISMATCH",
|
|
46
|
+
reason="cursor is stale or belongs to another raw")
|
|
47
|
+
return payload
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def finalize_page(build, used_before: int):
|
|
51
|
+
"""Resolve build(page_chars, used_after) -> obj to a size fixpoint."""
|
|
52
|
+
guess = 0
|
|
53
|
+
for _ in range(8):
|
|
54
|
+
obj = build(guess, used_before + guess)
|
|
55
|
+
actual = len(render_page(obj)) + 1
|
|
56
|
+
if actual == guess:
|
|
57
|
+
return obj, actual
|
|
58
|
+
guess = actual
|
|
59
|
+
raise PageError("PAGE_BUDGET_TOO_SMALL",
|
|
60
|
+
reason="page accounting did not converge")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def emit(obj) -> None:
|
|
64
|
+
print(render_page(obj))
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def emit_error(code: str, **detail) -> None:
|
|
68
|
+
emit({"error": code, **detail})
|
|
69
|
+
sys.exit(2)
|
cortex_vec/raw_source.py
ADDED
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
"""Shared source model for Raw session records (map-first authority).
|
|
2
|
+
|
|
3
|
+
Every character of a Raw belongs to exactly one SourceSpan: a gap-free,
|
|
4
|
+
overlap-free partition of the decoded text. Content the parser cannot
|
|
5
|
+
classify safely becomes ambiguous/opaque spans instead of disappearing.
|
|
6
|
+
This module is the ONLY Raw parsing authority; raw_view renders its legacy
|
|
7
|
+
L1-L3 projections from these spans through an adapter.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import re
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
SCHEMA_VERSION = 1
|
|
17
|
+
PARSER_VERSION = 1
|
|
18
|
+
|
|
19
|
+
TURN_HEADERS = ("### User", "### Claude")
|
|
20
|
+
_TOOL_HDR = "> [tool] "
|
|
21
|
+
_TOOL_NAME_RE = re.compile(r"^> \[tool\] \*\*([^*]+)\*\*")
|
|
22
|
+
_MARKER_RE = re.compile(r"^<!--\s*distilled:.*-->$")
|
|
23
|
+
|
|
24
|
+
# Review-obligation classes (consumed by raw_map / distill_plan):
|
|
25
|
+
# card alone suffices / complete preview suffices / always needs raw-span.
|
|
26
|
+
CARD_REVIEW_KINDS = frozenset({"blank", "turn_header", "output_open",
|
|
27
|
+
"output_close"})
|
|
28
|
+
PREVIEW_REVIEW_KINDS = frozenset({"frontmatter", "tool_header",
|
|
29
|
+
"meta_trailer"})
|
|
30
|
+
SPAN_REVIEW_KINDS = frozenset({"prose", "output_body", "ambiguous", "opaque"})
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class DecodeError(ValueError):
|
|
34
|
+
def __init__(self, byte_offset: int):
|
|
35
|
+
self.byte_offset = byte_offset
|
|
36
|
+
super().__init__(f"invalid UTF-8 at byte {byte_offset}")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class PartitionError(AssertionError):
|
|
40
|
+
pass
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class RawIdentity:
|
|
45
|
+
raw_path: str
|
|
46
|
+
file_sha256: str
|
|
47
|
+
source_sha256: str
|
|
48
|
+
schema_version: int
|
|
49
|
+
parser_version: int
|
|
50
|
+
byte_count: int
|
|
51
|
+
char_count: int
|
|
52
|
+
line_count: int
|
|
53
|
+
|
|
54
|
+
def to_json(self) -> dict:
|
|
55
|
+
return {
|
|
56
|
+
"raw_path": self.raw_path,
|
|
57
|
+
"file_sha256": self.file_sha256,
|
|
58
|
+
"source_sha256": self.source_sha256,
|
|
59
|
+
"schema_version": self.schema_version,
|
|
60
|
+
"parser_version": self.parser_version,
|
|
61
|
+
"byte_count": self.byte_count,
|
|
62
|
+
"char_count": self.char_count,
|
|
63
|
+
"line_count": self.line_count,
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass(frozen=True)
|
|
68
|
+
class SourceSpan:
|
|
69
|
+
id: int
|
|
70
|
+
kind: str
|
|
71
|
+
char_start: int # inclusive
|
|
72
|
+
char_end: int # exclusive
|
|
73
|
+
line_start: int # 1-based inclusive
|
|
74
|
+
line_end: int # 1-based inclusive
|
|
75
|
+
confidence: str # "high" | "medium" | "low"
|
|
76
|
+
tool_name: str | None = None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def strip_state_marker(text: str) -> str:
|
|
80
|
+
"""Remove the ONE position-anchored distilled marker line, if present.
|
|
81
|
+
|
|
82
|
+
Same anchoring rules as distill_queue: the first whole-line marker BEFORE
|
|
83
|
+
the first turn/tool header (header convention), else the last non-empty
|
|
84
|
+
line if it is a marker (EOF convention). Markers quoted inside the
|
|
85
|
+
conversation stay put.
|
|
86
|
+
"""
|
|
87
|
+
lines = text.split("\n")
|
|
88
|
+
for i, ln in enumerate(lines):
|
|
89
|
+
s = ln.strip()
|
|
90
|
+
if not s:
|
|
91
|
+
continue
|
|
92
|
+
if s in TURN_HEADERS:
|
|
93
|
+
break
|
|
94
|
+
if _MARKER_RE.match(s):
|
|
95
|
+
return "\n".join(lines[:i] + lines[i + 1:])
|
|
96
|
+
for i in range(len(lines) - 1, -1, -1):
|
|
97
|
+
s = lines[i].strip()
|
|
98
|
+
if not s:
|
|
99
|
+
continue
|
|
100
|
+
if _MARKER_RE.match(s):
|
|
101
|
+
# Remove marker and preceding blank line if present
|
|
102
|
+
if i > 0 and not lines[i - 1].strip():
|
|
103
|
+
return "\n".join(lines[:i - 1] + lines[i + 1:])
|
|
104
|
+
else:
|
|
105
|
+
return "\n".join(lines[:i] + lines[i + 1:])
|
|
106
|
+
break
|
|
107
|
+
return text
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def load(path) -> tuple:
|
|
111
|
+
"""Read a Raw; return (RawIdentity, decoded_text). Strict UTF-8."""
|
|
112
|
+
data = Path(path).read_bytes()
|
|
113
|
+
try:
|
|
114
|
+
text = data.decode("utf-8")
|
|
115
|
+
except UnicodeDecodeError as e:
|
|
116
|
+
raise DecodeError(e.start) from e
|
|
117
|
+
source = strip_state_marker(text)
|
|
118
|
+
ident = RawIdentity(
|
|
119
|
+
raw_path=str(path),
|
|
120
|
+
file_sha256=hashlib.sha256(data).hexdigest(),
|
|
121
|
+
source_sha256=hashlib.sha256(source.encode("utf-8")).hexdigest(),
|
|
122
|
+
schema_version=SCHEMA_VERSION,
|
|
123
|
+
parser_version=PARSER_VERSION,
|
|
124
|
+
byte_count=len(data),
|
|
125
|
+
char_count=len(text),
|
|
126
|
+
line_count=text.count("\n") + 1,
|
|
127
|
+
)
|
|
128
|
+
return ident, text
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _line_table(text: str) -> list:
|
|
132
|
+
"""[(char_start, char_end)] per line; each line owns its trailing \\n.
|
|
133
|
+
A zero-width final line (text ending in \\n) is dropped so every span
|
|
134
|
+
is non-empty."""
|
|
135
|
+
table = []
|
|
136
|
+
pos = 0
|
|
137
|
+
while pos <= len(text):
|
|
138
|
+
nl = text.find("\n", pos)
|
|
139
|
+
if nl == -1:
|
|
140
|
+
if pos < len(text):
|
|
141
|
+
table.append((pos, len(text)))
|
|
142
|
+
break
|
|
143
|
+
table.append((pos, nl + 1))
|
|
144
|
+
pos = nl + 1
|
|
145
|
+
return table
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def validate_partition(spans, text) -> None:
|
|
149
|
+
pos = 0
|
|
150
|
+
for k, s in enumerate(spans):
|
|
151
|
+
if s.id != k or s.char_start != pos or s.char_end <= s.char_start:
|
|
152
|
+
raise PartitionError(f"span {k} breaks partition at char {pos}")
|
|
153
|
+
pos = s.char_end
|
|
154
|
+
if pos != len(text):
|
|
155
|
+
raise PartitionError(f"partition ends at {pos}, text is {len(text)}")
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def parse(text: str) -> list:
|
|
159
|
+
"""Parse a Raw into a validated gap-free/overlap-free span partition.
|
|
160
|
+
|
|
161
|
+
Rules (this docstring is the spec):
|
|
162
|
+
1. Lines own their trailing \\n; the partition is built line-by-line via
|
|
163
|
+
the line table, so gap-freeness holds by construction.
|
|
164
|
+
2. A leading ``---`` line is frontmatter only if a closing ``---`` is
|
|
165
|
+
found before any turn/tool header; otherwise it is not frontmatter.
|
|
166
|
+
3. Inside a ```output wrapper, turn/tool strings are literal. The close
|
|
167
|
+
fence is the first bare ``` line whose next non-blank line is a
|
|
168
|
+
turn/tool header or EOF. Passing a boundary-looking literal line
|
|
169
|
+
before the close degrades output_body confidence to medium.
|
|
170
|
+
4. No close fence found: everything after output_open to EOF is opaque
|
|
171
|
+
(confidence low).
|
|
172
|
+
5. A run of blank lines is always its own blank span; never swallowed.
|
|
173
|
+
6. A whole-line distilled marker outside the conversation is
|
|
174
|
+
meta_trailer; a marker quoted inside the conversation stays prose.
|
|
175
|
+
"""
|
|
176
|
+
lt = _line_table(text)
|
|
177
|
+
n = len(lt)
|
|
178
|
+
lines = []
|
|
179
|
+
for a, b in lt:
|
|
180
|
+
seg = text[a:b]
|
|
181
|
+
lines.append(seg[:-1] if seg.endswith("\n") else seg)
|
|
182
|
+
|
|
183
|
+
spans: list = []
|
|
184
|
+
|
|
185
|
+
def emit(kind, lo, hi, confidence="high", tool_name=None):
|
|
186
|
+
spans.append(SourceSpan(
|
|
187
|
+
id=len(spans), kind=kind,
|
|
188
|
+
char_start=lt[lo][0], char_end=lt[hi][1],
|
|
189
|
+
line_start=lo + 1, line_end=hi + 1,
|
|
190
|
+
confidence=confidence, tool_name=tool_name,
|
|
191
|
+
))
|
|
192
|
+
|
|
193
|
+
def is_blank(k):
|
|
194
|
+
return lines[k].strip() == ""
|
|
195
|
+
|
|
196
|
+
def is_turn(k):
|
|
197
|
+
return lines[k] in TURN_HEADERS
|
|
198
|
+
|
|
199
|
+
def is_tool(k):
|
|
200
|
+
return lines[k].startswith(_TOOL_HDR)
|
|
201
|
+
|
|
202
|
+
def parse_output(open_i: int) -> int:
|
|
203
|
+
"""```output wrapper starting at open_i; returns next line index.
|
|
204
|
+
|
|
205
|
+
Inside the wrapper, turn/tool strings are literal content. The close
|
|
206
|
+
fence is the FIRST bare ``` whose next non-blank line is a turn/tool
|
|
207
|
+
header or EOF. This is safe because of filter-transcript's emit
|
|
208
|
+
invariant: every assistant text block is written with its own
|
|
209
|
+
`### Claude` header (see render_transcript), and a tool chunk is
|
|
210
|
+
flushed with a trailing newline before the next non-tool chunk, so a
|
|
211
|
+
real close fence is ALWAYS followed (after blanks) by a `### Claude` /
|
|
212
|
+
`### User` / `> [tool]` boundary -- assistant commentary after a tool
|
|
213
|
+
result never trails the fence as bare prose. A boundary-looking literal
|
|
214
|
+
inside the body (a meta-session quoting another transcript) degrades
|
|
215
|
+
confidence to medium; no close at all makes the tail opaque. A
|
|
216
|
+
hand-edited Raw with bare prose directly after a close fence would fall
|
|
217
|
+
to one opaque span -- coverage-safe (validate_partition still holds),
|
|
218
|
+
just coarser navigation for that non-emittable region."""
|
|
219
|
+
body0 = open_i + 1
|
|
220
|
+
close = None
|
|
221
|
+
saw_literal_boundary = False
|
|
222
|
+
j = body0
|
|
223
|
+
while j < n:
|
|
224
|
+
if lines[j].strip() == "```":
|
|
225
|
+
k = j + 1
|
|
226
|
+
while k < n and is_blank(k):
|
|
227
|
+
k += 1
|
|
228
|
+
if k >= n or is_turn(k) or is_tool(k):
|
|
229
|
+
close = j
|
|
230
|
+
break
|
|
231
|
+
elif is_turn(j) or is_tool(j):
|
|
232
|
+
saw_literal_boundary = True
|
|
233
|
+
j += 1
|
|
234
|
+
emit("output_open", open_i, open_i)
|
|
235
|
+
if close is None:
|
|
236
|
+
if body0 < n:
|
|
237
|
+
emit("opaque", body0, n - 1, confidence="low")
|
|
238
|
+
return n
|
|
239
|
+
conf = "medium" if saw_literal_boundary else "high"
|
|
240
|
+
if close > body0:
|
|
241
|
+
emit("output_body", body0, close - 1, confidence=conf)
|
|
242
|
+
emit("output_close", close, close)
|
|
243
|
+
return close + 1
|
|
244
|
+
|
|
245
|
+
i = 0
|
|
246
|
+
in_conversation = False
|
|
247
|
+
|
|
248
|
+
if n and lines[0] == "---":
|
|
249
|
+
j = 1
|
|
250
|
+
while (j < n and lines[j] != "---"
|
|
251
|
+
and not is_turn(j) and not is_tool(j)):
|
|
252
|
+
j += 1
|
|
253
|
+
if j < n and lines[j] == "---":
|
|
254
|
+
emit("frontmatter", 0, j)
|
|
255
|
+
i = j + 1
|
|
256
|
+
|
|
257
|
+
while i < n:
|
|
258
|
+
if is_blank(i):
|
|
259
|
+
j = i
|
|
260
|
+
while j + 1 < n and is_blank(j + 1):
|
|
261
|
+
j += 1
|
|
262
|
+
emit("blank", i, j)
|
|
263
|
+
i = j + 1
|
|
264
|
+
elif is_turn(i):
|
|
265
|
+
in_conversation = True
|
|
266
|
+
emit("turn_header", i, i)
|
|
267
|
+
i += 1
|
|
268
|
+
elif is_tool(i):
|
|
269
|
+
in_conversation = True
|
|
270
|
+
m = _TOOL_NAME_RE.match(lines[i])
|
|
271
|
+
emit("tool_header", i, i, tool_name=m.group(1) if m else None)
|
|
272
|
+
i += 1
|
|
273
|
+
k = i
|
|
274
|
+
while k < n and is_blank(k):
|
|
275
|
+
k += 1
|
|
276
|
+
if k < n and lines[k] == "```output":
|
|
277
|
+
if k > i:
|
|
278
|
+
emit("blank", i, k - 1)
|
|
279
|
+
i = parse_output(k)
|
|
280
|
+
# no ```output: the blanks re-enter the main loop as blank spans
|
|
281
|
+
elif _MARKER_RE.match(lines[i].strip()) and not in_conversation:
|
|
282
|
+
emit("meta_trailer", i, i)
|
|
283
|
+
i += 1
|
|
284
|
+
else:
|
|
285
|
+
j = i
|
|
286
|
+
while (j + 1 < n and not is_blank(j + 1)
|
|
287
|
+
and not is_turn(j + 1) and not is_tool(j + 1)):
|
|
288
|
+
j += 1
|
|
289
|
+
emit("prose", i, j)
|
|
290
|
+
i = j + 1
|
|
291
|
+
|
|
292
|
+
# EOF marker convention: a whole-line marker as the very last span in
|
|
293
|
+
# conversation is the state trailer, not prose.
|
|
294
|
+
if spans and spans[-1].kind == "prose":
|
|
295
|
+
last = spans[-1]
|
|
296
|
+
seg_lines = text[last.char_start:last.char_end].split("\n")
|
|
297
|
+
seg_lines = [x for x in seg_lines if x.strip()]
|
|
298
|
+
if len(seg_lines) == 1 and _MARKER_RE.match(seg_lines[0].strip()):
|
|
299
|
+
spans[-1] = SourceSpan(
|
|
300
|
+
id=last.id, kind="meta_trailer",
|
|
301
|
+
char_start=last.char_start, char_end=last.char_end,
|
|
302
|
+
line_start=last.line_start, line_end=last.line_end,
|
|
303
|
+
confidence="high")
|
|
304
|
+
|
|
305
|
+
validate_partition(spans, text)
|
|
306
|
+
return spans
|
cortex_vec/raw_span.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""raw-span: the ONLY reader that returns Raw original text to the session.
|
|
2
|
+
|
|
3
|
+
Pages are exact substrings of the immutable Raw, hard-capped at max_chars
|
|
4
|
+
of emitted stdout, with character-offset continuation for spans (or single
|
|
5
|
+
lines) larger than one page. No summarization, no projection, no verdicts.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from . import distill_plan as dp
|
|
10
|
+
from . import raw_source as rs
|
|
11
|
+
from .raw_page import (PAGE_BUDGET_DEFAULT, PageError, decode_cursor,
|
|
12
|
+
emit, emit_error, encode_cursor, finalize_page,
|
|
13
|
+
render_page)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def dispatch(args) -> None:
|
|
17
|
+
try:
|
|
18
|
+
_dispatch(args)
|
|
19
|
+
except PageError as e:
|
|
20
|
+
emit_error(e.code, **e.detail)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _dispatch(args) -> None:
|
|
24
|
+
max_chars = getattr(args, "max_chars", None) or PAGE_BUDGET_DEFAULT
|
|
25
|
+
state, ident, text, spans = dp.load_for_page(args.plan_id, args.path)
|
|
26
|
+
|
|
27
|
+
if getattr(args, "cursor", None):
|
|
28
|
+
payload = decode_cursor(args.cursor, "span", ident)
|
|
29
|
+
span_id, offset = payload["id"], payload["o"]
|
|
30
|
+
elif getattr(args, "span_id", None) is not None:
|
|
31
|
+
span_id, offset = args.span_id, 0
|
|
32
|
+
else:
|
|
33
|
+
raise PageError("CURSOR_MISMATCH",
|
|
34
|
+
reason="need --span-id or --cursor")
|
|
35
|
+
if not (0 <= span_id < len(spans)):
|
|
36
|
+
raise PageError("CURSOR_MISMATCH", reason="unknown span id",
|
|
37
|
+
span_id=span_id)
|
|
38
|
+
span = spans[span_id]
|
|
39
|
+
seg = text[span.char_start:span.char_end]
|
|
40
|
+
if not (0 <= offset < len(seg)):
|
|
41
|
+
raise PageError("CURSOR_MISMATCH", reason="offset out of range")
|
|
42
|
+
|
|
43
|
+
used_before = state["session_used_chars"]
|
|
44
|
+
budget = state["session_budget"] - dp.CONTROL_RESERVE
|
|
45
|
+
|
|
46
|
+
def envelope(take, page_chars, used_after):
|
|
47
|
+
end = offset + take
|
|
48
|
+
return {
|
|
49
|
+
"schema_version": rs.SCHEMA_VERSION,
|
|
50
|
+
"raw_identity": ident.to_json(),
|
|
51
|
+
"plan_id": state["plan_id"],
|
|
52
|
+
"span_id": span.id,
|
|
53
|
+
"kind": span.kind,
|
|
54
|
+
"char_start": span.char_start + offset,
|
|
55
|
+
"char_end": span.char_start + end,
|
|
56
|
+
"line_start": span.line_start,
|
|
57
|
+
"line_end": span.line_end,
|
|
58
|
+
"source_chars": len(seg),
|
|
59
|
+
"content": seg[offset:end],
|
|
60
|
+
"page_chars": page_chars,
|
|
61
|
+
"session_used_chars": used_after,
|
|
62
|
+
"session_remaining_chars": max(0, budget - used_after),
|
|
63
|
+
"next_cursor": (encode_cursor(
|
|
64
|
+
{"k": "span", "s": ident.schema_version,
|
|
65
|
+
"p": ident.parser_version,
|
|
66
|
+
"f": ident.file_sha256[:16],
|
|
67
|
+
"id": span.id, "o": end}) if end < len(seg) else None),
|
|
68
|
+
"budget_status": "ok",
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
# binary-search the largest take whose FINAL stdout fits max_chars
|
|
72
|
+
lo, hi = 0, len(seg) - offset
|
|
73
|
+
best = None
|
|
74
|
+
while lo < hi:
|
|
75
|
+
mid = (lo + hi + 1) // 2
|
|
76
|
+
try:
|
|
77
|
+
obj, chars = finalize_page(
|
|
78
|
+
lambda pc, ua, t=mid: envelope(t, pc, ua), used_before)
|
|
79
|
+
except PageError:
|
|
80
|
+
hi = mid - 1
|
|
81
|
+
continue
|
|
82
|
+
if chars <= max_chars:
|
|
83
|
+
best = (obj, chars, mid)
|
|
84
|
+
lo = mid
|
|
85
|
+
else:
|
|
86
|
+
hi = mid - 1
|
|
87
|
+
if best is None:
|
|
88
|
+
raise PageError("PAGE_BUDGET_TOO_SMALL", span_id=span.id,
|
|
89
|
+
max_chars=max_chars)
|
|
90
|
+
obj, chars, take = best
|
|
91
|
+
|
|
92
|
+
dp.charge_and_save(state, chars)
|
|
93
|
+
dp.mark_reviewed(state, span.char_start + offset,
|
|
94
|
+
span.char_start + offset + take)
|
|
95
|
+
assert len(render_page(obj)) + 1 == chars
|
|
96
|
+
emit(obj)
|