codebase-explorer 2.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cbe/__init__.py +9 -0
- cbe/__main__.py +4 -0
- cbe/aggregation.py +183 -0
- cbe/api_completion.py +410 -0
- cbe/batching.py +283 -0
- cbe/catalog_query.py +376 -0
- cbe/checks.py +390 -0
- cbe/cli.py +434 -0
- cbe/concrete_facts.py +321 -0
- cbe/data/LICENSE.tiktoken +30 -0
- cbe/data/o200k_base.tiktoken +199998 -0
- cbe/declarations.py +537 -0
- cbe/docs.py +929 -0
- cbe/generate.py +316 -0
- cbe/grammars.py +741 -0
- cbe/graph.py +857 -0
- cbe/graph_backends/__init__.py +43 -0
- cbe/graph_backends/_calls.py +262 -0
- cbe/graph_backends/_paths.py +42 -0
- cbe/graph_backends/identity.py +93 -0
- cbe/graph_backends/imports.py +256 -0
- cbe/graph_backends/joern.py +1021 -0
- cbe/graph_backends/resolution.py +288 -0
- cbe/graph_backends/toolchain.py +396 -0
- cbe/grounding.py +57 -0
- cbe/host_run.py +1752 -0
- cbe/model_settings.py +80 -0
- cbe/projection.py +305 -0
- cbe/prompts.py +316 -0
- cbe/records.py +200 -0
- cbe/skeleton.py +474 -0
- cbe/source_context.py +709 -0
- cbe/symbols.py +136 -0
- cbe/token_budget.py +68 -0
- cbe/treename.py +398 -0
- cbe/usage.py +300 -0
- cbe_install/__init__.py +11 -0
- cbe_install/cli.py +434 -0
- cbe_install/resources/agents/cbe-writer.md +16 -0
- cbe_install/resources/codebase-explorer/SKILL.md +101 -0
- cbe_install/resources/codebase-explorer/references/claude-code.md +63 -0
- cbe_install/resources/codebase-explorer/references/codex.md +37 -0
- codebase_explorer-2.2.0.dist-info/METADATA +233 -0
- codebase_explorer-2.2.0.dist-info/RECORD +47 -0
- codebase_explorer-2.2.0.dist-info/WHEEL +4 -0
- codebase_explorer-2.2.0.dist-info/entry_points.txt +3 -0
- codebase_explorer-2.2.0.dist-info/licenses/LICENSE +21 -0
cbe/source_context.py
ADDED
|
@@ -0,0 +1,709 @@
|
|
|
1
|
+
"""Multi-language source chunking and the unified graph + source context.
|
|
2
|
+
|
|
3
|
+
This module is the source-context half of the public ``cbe-context/1``
|
|
4
|
+
contract consumed by the generation pipeline: byte-exact source chunks plus
|
|
5
|
+
an optional standard ``cbe-graph/1`` analysis graph. It turns a repository
|
|
6
|
+
into a byte-exact, non-overlapping partition of source chunks (functions,
|
|
7
|
+
classes, methods, types, module residue) and combines those chunks with a
|
|
8
|
+
standard graph object.
|
|
9
|
+
|
|
10
|
+
Design rules that the implementation follows:
|
|
11
|
+
|
|
12
|
+
* Chunks partition the original UTF-8 bytes exactly once. Concatenating the
|
|
13
|
+
chunks of one file in ``start_byte`` order reproduces the decoded source
|
|
14
|
+
byte-for-byte. A nested declaration is its own chunk, so a class chunk
|
|
15
|
+
never repeats its method bodies; ancestor names, signatures and the full
|
|
16
|
+
syntactic range stay in metadata.
|
|
17
|
+
* Oversized exclusive regions are split at grammar child boundaries only.
|
|
18
|
+
A segment boundary is always a real syntax-tree boundary, never a character
|
|
19
|
+
prefix cut.
|
|
20
|
+
* Unknown languages, a missing grammar package and parse failures retain their
|
|
21
|
+
per-file diagnostics and use whole-file chunks for non-blank UTF-8 source.
|
|
22
|
+
A UTF-8 decode failure yields ``decode_error`` and no chunks for that file;
|
|
23
|
+
other files still build. Syntax errors retain grammar coverage plus a diagnostic.
|
|
24
|
+
* Only tree-sitter grammars do the parsing. The glue here maps grammar node
|
|
25
|
+
kinds to chunk kinds and maps byte/char/line coordinates; it does not
|
|
26
|
+
re-implement any language analysis.
|
|
27
|
+
* Graphs are consumed, never built. A graph whose ``source_manifest`` drifts
|
|
28
|
+
from the current source is rejected with a locatable error.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import bisect
|
|
34
|
+
import json
|
|
35
|
+
from dataclasses import dataclass
|
|
36
|
+
from pathlib import Path, PurePosixPath
|
|
37
|
+
from typing import Any
|
|
38
|
+
|
|
39
|
+
from cbe.grammars import (
|
|
40
|
+
AMBIGUOUS_SUFFIXES,
|
|
41
|
+
DEFAULT_MAX_SEGMENT_BYTES,
|
|
42
|
+
MAX_FILE_BYTES,
|
|
43
|
+
SCHEMA_CONTEXT,
|
|
44
|
+
SCHEMA_GRAPH,
|
|
45
|
+
ContextError,
|
|
46
|
+
OffsetMap,
|
|
47
|
+
_EXTRA_SPECS,
|
|
48
|
+
_all_specs,
|
|
49
|
+
_grammar_for_suffix,
|
|
50
|
+
_iter_source_files,
|
|
51
|
+
_make_parser,
|
|
52
|
+
canonical_symbol_id,
|
|
53
|
+
language_name,
|
|
54
|
+
line_starts,
|
|
55
|
+
sha256_bytes,
|
|
56
|
+
sha256_text,
|
|
57
|
+
)
|
|
58
|
+
from cbe.declarations import _Declaration, _body_start, _extract_declarations
|
|
59
|
+
from cbe.graph import _prune_graph, check_graph_source
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# ---------------------------------------------------------------------------
|
|
63
|
+
# Chunk construction
|
|
64
|
+
# ---------------------------------------------------------------------------
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
_BODY_CONTAINERS = frozenset(
|
|
68
|
+
{
|
|
69
|
+
"block",
|
|
70
|
+
"statement_block",
|
|
71
|
+
"compound_statement",
|
|
72
|
+
"statement_list",
|
|
73
|
+
"class_body",
|
|
74
|
+
"declaration_list",
|
|
75
|
+
"field_declaration_list",
|
|
76
|
+
"constructor_body",
|
|
77
|
+
"interface_body",
|
|
78
|
+
"enum_body",
|
|
79
|
+
"interface_body",
|
|
80
|
+
"module",
|
|
81
|
+
"program",
|
|
82
|
+
"translation_unit",
|
|
83
|
+
"source_file",
|
|
84
|
+
"body_statement",
|
|
85
|
+
"function_body",
|
|
86
|
+
"protocol_body",
|
|
87
|
+
"statements",
|
|
88
|
+
"compilation_unit",
|
|
89
|
+
}
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _collect_named_boundaries(root: Any) -> tuple[list[int], list[int]]:
|
|
94
|
+
"""Return (all named-node boundaries, statement-level boundaries).
|
|
95
|
+
|
|
96
|
+
Statement-level boundaries are the starts/ends of nodes that sit directly
|
|
97
|
+
in a body/block, so an oversized declaration prefers to cut between
|
|
98
|
+
statements instead of inside an expression. Both lists are token
|
|
99
|
+
boundaries; splitting on either never cuts inside a lexeme.
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
all_boundaries: list[int] = [root.start_byte, root.end_byte]
|
|
103
|
+
statement_boundaries: list[int] = []
|
|
104
|
+
stack = [root]
|
|
105
|
+
while stack:
|
|
106
|
+
node = stack.pop()
|
|
107
|
+
all_boundaries.append(node.start_byte)
|
|
108
|
+
all_boundaries.append(node.end_byte)
|
|
109
|
+
if node.type in _BODY_CONTAINERS:
|
|
110
|
+
for child in node.named_children:
|
|
111
|
+
statement_boundaries.append(child.start_byte)
|
|
112
|
+
statement_boundaries.append(child.end_byte)
|
|
113
|
+
stack.extend(node.named_children)
|
|
114
|
+
return sorted(set(all_boundaries)), sorted(set(statement_boundaries))
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _subtract_ranges(start: int, end: int, holes: list[tuple[int, int]]) -> list[tuple[int, int]]:
|
|
118
|
+
kept = [(start, end)]
|
|
119
|
+
for hole_start, hole_end in sorted(holes):
|
|
120
|
+
next_kept: list[tuple[int, int]] = []
|
|
121
|
+
for seg_start, seg_end in kept:
|
|
122
|
+
if hole_end <= seg_start or hole_start >= seg_end:
|
|
123
|
+
next_kept.append((seg_start, seg_end))
|
|
124
|
+
continue
|
|
125
|
+
if seg_start < hole_start:
|
|
126
|
+
next_kept.append((seg_start, hole_start))
|
|
127
|
+
if hole_end < seg_end:
|
|
128
|
+
next_kept.append((hole_end, seg_end))
|
|
129
|
+
kept = next_kept
|
|
130
|
+
return [(seg_start, seg_end) for seg_start, seg_end in kept if seg_end > seg_start]
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _leftover_ranges(total_bytes: int, covered: list[tuple[int, int]]) -> list[tuple[int, int]]:
|
|
134
|
+
if total_bytes <= 0:
|
|
135
|
+
return []
|
|
136
|
+
return _subtract_ranges(0, total_bytes, covered)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _split_byte_span(
|
|
140
|
+
start: int,
|
|
141
|
+
end: int,
|
|
142
|
+
all_boundaries: list[int],
|
|
143
|
+
statement_boundaries: list[int],
|
|
144
|
+
max_bytes: int,
|
|
145
|
+
) -> tuple[list[tuple[int, int]], list[tuple[int, int]]]:
|
|
146
|
+
"""Split a byte span at grammar boundaries, never inside a token.
|
|
147
|
+
|
|
148
|
+
Everything here is in bytes, matching tree-sitter and the chunk contract.
|
|
149
|
+
Returns ``(segments, oversized)``; ``oversized`` holds segments that had
|
|
150
|
+
to exceed ``max_bytes`` because one indivisible grammar token does not
|
|
151
|
+
fit. The caller turns those into an explicit diagnostic instead of a
|
|
152
|
+
silently over-budget chunk.
|
|
153
|
+
"""
|
|
154
|
+
|
|
155
|
+
if max_bytes <= 0 or end - start <= max_bytes:
|
|
156
|
+
return [(start, end)], []
|
|
157
|
+
segments: list[tuple[int, int]] = []
|
|
158
|
+
oversized: list[tuple[int, int]] = []
|
|
159
|
+
current = start
|
|
160
|
+
|
|
161
|
+
def choose(boundaries: list[int], limit: int) -> int | None:
|
|
162
|
+
lo = bisect.bisect_left(boundaries, start)
|
|
163
|
+
hi = bisect.bisect_right(boundaries, end)
|
|
164
|
+
window = boundaries[lo:hi]
|
|
165
|
+
index = bisect.bisect_right(window, limit) - 1
|
|
166
|
+
while index >= 0:
|
|
167
|
+
candidate = window[index]
|
|
168
|
+
if candidate > current:
|
|
169
|
+
return candidate
|
|
170
|
+
index -= 1
|
|
171
|
+
return None
|
|
172
|
+
|
|
173
|
+
while current < end:
|
|
174
|
+
limit = current + max_bytes
|
|
175
|
+
nxt = choose(statement_boundaries, limit)
|
|
176
|
+
if nxt is None:
|
|
177
|
+
nxt = choose(all_boundaries, limit)
|
|
178
|
+
if nxt is None:
|
|
179
|
+
after = bisect.bisect_right(all_boundaries, current)
|
|
180
|
+
nxt = all_boundaries[after] if after < len(all_boundaries) else end
|
|
181
|
+
seg_end = min(nxt, end)
|
|
182
|
+
if seg_end - current > max_bytes:
|
|
183
|
+
oversized.append((current, seg_end))
|
|
184
|
+
segments.append((current, seg_end))
|
|
185
|
+
if seg_end >= end:
|
|
186
|
+
break
|
|
187
|
+
current = seg_end
|
|
188
|
+
return segments, oversized
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _line_of(starts: tuple[int, ...], char_index: int) -> int:
|
|
192
|
+
return bisect.bisect_right(starts, char_index)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
@dataclass
|
|
196
|
+
class _ChunkPlan:
|
|
197
|
+
path: str
|
|
198
|
+
language: str
|
|
199
|
+
kind: str
|
|
200
|
+
name: str
|
|
201
|
+
qualified_name: str
|
|
202
|
+
symbol_id: str
|
|
203
|
+
parent_id: str | None
|
|
204
|
+
signature: str
|
|
205
|
+
ancestors: tuple[str, ...]
|
|
206
|
+
start_byte: int
|
|
207
|
+
end_byte: int
|
|
208
|
+
full_start_byte: int
|
|
209
|
+
full_end_byte: int
|
|
210
|
+
segment_index: int
|
|
211
|
+
segment_count: int
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _plan_chunks(
|
|
215
|
+
offsets: OffsetMap,
|
|
216
|
+
declarations: list[_Declaration],
|
|
217
|
+
root: Any,
|
|
218
|
+
path: str,
|
|
219
|
+
language: str,
|
|
220
|
+
max_segment_bytes: int,
|
|
221
|
+
) -> tuple[list[_ChunkPlan], list[dict[str, Any]]]:
|
|
222
|
+
total_bytes = len(offsets.raw)
|
|
223
|
+
by_parent: dict[str | None, list[_Declaration]] = {}
|
|
224
|
+
for decl in declarations:
|
|
225
|
+
by_parent.setdefault(decl.parent_id, []).append(decl)
|
|
226
|
+
|
|
227
|
+
# Exclusive byte ranges: full span minus the spans of direct child
|
|
228
|
+
# declarations. Union over all declarations equals the union of the
|
|
229
|
+
# top-level declaration spans (no duplication).
|
|
230
|
+
exclusive: dict[str, list[tuple[int, int]]] = {}
|
|
231
|
+
for decl in declarations:
|
|
232
|
+
holes = [(child.start_byte, child.end_byte) for child in by_parent.get(decl.symbol_id, ())]
|
|
233
|
+
exclusive[decl.symbol_id] = _subtract_ranges(
|
|
234
|
+
decl.start_byte, decl.exclusive_end_byte or decl.end_byte, holes)
|
|
235
|
+
|
|
236
|
+
covered = [(decl.start_byte, decl.exclusive_end_byte or decl.end_byte)
|
|
237
|
+
for decl in declarations if decl.parent_id is None]
|
|
238
|
+
residual = _leftover_ranges(total_bytes, covered)
|
|
239
|
+
|
|
240
|
+
boundaries, statement_boundaries = _collect_named_boundaries(root)
|
|
241
|
+
max_bytes = max(1, int(max_segment_bytes))
|
|
242
|
+
plans: list[_ChunkPlan] = []
|
|
243
|
+
diagnostics: list[dict[str, Any]] = []
|
|
244
|
+
|
|
245
|
+
def record_oversized(ranges: list[tuple[int, int]]) -> None:
|
|
246
|
+
for seg_start, seg_end in ranges:
|
|
247
|
+
diagnostics.append(
|
|
248
|
+
{
|
|
249
|
+
"path": path,
|
|
250
|
+
"code": "oversized_token",
|
|
251
|
+
"message": (
|
|
252
|
+
f"segment [{seg_start},{seg_end}) is {seg_end - seg_start} bytes, above the "
|
|
253
|
+
f"{max_bytes}-byte target: one grammar token does not fit and no safe boundary exists"
|
|
254
|
+
),
|
|
255
|
+
}
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
def add_plan(decl: _Declaration, seg_start: int, seg_end: int, index: int, count: int) -> None:
|
|
259
|
+
plans.append(
|
|
260
|
+
_ChunkPlan(
|
|
261
|
+
path=path,
|
|
262
|
+
language=language,
|
|
263
|
+
kind=decl.kind,
|
|
264
|
+
name=decl.name,
|
|
265
|
+
qualified_name=decl.qualified_name,
|
|
266
|
+
symbol_id=decl.symbol_id,
|
|
267
|
+
parent_id=decl.parent_id,
|
|
268
|
+
signature=decl.signature,
|
|
269
|
+
ancestors=decl.ancestors,
|
|
270
|
+
start_byte=seg_start,
|
|
271
|
+
end_byte=seg_end,
|
|
272
|
+
full_start_byte=decl.start_byte,
|
|
273
|
+
full_end_byte=decl.end_byte,
|
|
274
|
+
segment_index=index,
|
|
275
|
+
segment_count=count,
|
|
276
|
+
)
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
for decl in declarations:
|
|
280
|
+
segments: list[tuple[int, int]] = []
|
|
281
|
+
oversized: list[tuple[int, int]] = []
|
|
282
|
+
for rng_start, rng_end in exclusive.get(decl.symbol_id, []):
|
|
283
|
+
part, over = _split_byte_span(rng_start, rng_end, boundaries, statement_boundaries, max_bytes)
|
|
284
|
+
segments.extend(part)
|
|
285
|
+
oversized.extend(over)
|
|
286
|
+
record_oversized(oversized)
|
|
287
|
+
count = len(segments)
|
|
288
|
+
for index, (seg_start, seg_end) in enumerate(segments):
|
|
289
|
+
add_plan(decl, seg_start, seg_end, index, count)
|
|
290
|
+
|
|
291
|
+
for rng_start, rng_end in residual:
|
|
292
|
+
segments, oversized = _split_byte_span(rng_start, rng_end, boundaries, statement_boundaries, max_bytes)
|
|
293
|
+
record_oversized(oversized)
|
|
294
|
+
count = len(segments)
|
|
295
|
+
residual_id = canonical_symbol_id(path, "module_residual", "", rng_start)
|
|
296
|
+
for index, (seg_start, seg_end) in enumerate(segments):
|
|
297
|
+
plans.append(
|
|
298
|
+
_ChunkPlan(
|
|
299
|
+
path=path,
|
|
300
|
+
language=language,
|
|
301
|
+
kind="module_residual",
|
|
302
|
+
name=PurePosixPath(path).name,
|
|
303
|
+
qualified_name="",
|
|
304
|
+
symbol_id=residual_id,
|
|
305
|
+
parent_id=None,
|
|
306
|
+
signature="",
|
|
307
|
+
ancestors=(),
|
|
308
|
+
start_byte=seg_start,
|
|
309
|
+
end_byte=seg_end,
|
|
310
|
+
full_start_byte=rng_start,
|
|
311
|
+
full_end_byte=rng_end,
|
|
312
|
+
segment_index=index,
|
|
313
|
+
segment_count=count,
|
|
314
|
+
)
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
plans.sort(key=lambda item: (item.start_byte, item.end_byte))
|
|
318
|
+
return plans, diagnostics
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def source_signatures(path: str, source: str, language: str) -> tuple[dict[str, str], dict[tuple[str, int], str]]:
|
|
322
|
+
"""Full signatures by canonical ID and (qualified name, decorated start line).
|
|
323
|
+
|
|
324
|
+
Prompt chunks intentionally cap their signatures. Published records reparse
|
|
325
|
+
verified source with the same grammar and declaration boundaries, and retain
|
|
326
|
+
the complete signature. No code executes. Unsupported grammars return empty
|
|
327
|
+
maps, letting legacy inventory records retain their available metadata.
|
|
328
|
+
"""
|
|
329
|
+
spec = next((s for s in _all_specs() if s.name == language), None)
|
|
330
|
+
if spec is None:
|
|
331
|
+
return {}, {}
|
|
332
|
+
try:
|
|
333
|
+
parser = _make_parser(spec)
|
|
334
|
+
except (ImportError, AttributeError):
|
|
335
|
+
return {}, {}
|
|
336
|
+
raw = source.encode("utf-8")
|
|
337
|
+
offsets = OffsetMap.from_bytes(raw)
|
|
338
|
+
declarations = _extract_declarations(spec, offsets, parser.parse(raw).root_node, path)
|
|
339
|
+
by_id, by_location = {}, {}
|
|
340
|
+
starts = line_starts(offsets.text)
|
|
341
|
+
for declaration in declarations:
|
|
342
|
+
end = _body_start(declaration.node)
|
|
343
|
+
if end < 0:
|
|
344
|
+
value = declaration.node.child_by_field_name("value")
|
|
345
|
+
if value is not None:
|
|
346
|
+
end = _body_start(value)
|
|
347
|
+
if end < 0:
|
|
348
|
+
end = declaration.end_byte
|
|
349
|
+
signature = raw[declaration.start_byte:end].decode("utf-8").strip()
|
|
350
|
+
if declaration.kind != "module_region" and declaration.signature and declaration.signature not in signature:
|
|
351
|
+
signature += "\n" + declaration.signature
|
|
352
|
+
by_id[declaration.symbol_id] = signature
|
|
353
|
+
start_line = _line_of(starts, offsets.char_of_byte(declaration.start_byte))
|
|
354
|
+
by_location[declaration.qualified_name, start_line] = signature
|
|
355
|
+
return by_id, by_location
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _chunk_from_plan(plan: _ChunkPlan, offsets: OffsetMap, starts: tuple[int, ...]) -> dict[str, Any]:
|
|
359
|
+
text = offsets.raw[plan.start_byte:plan.end_byte].decode("utf-8")
|
|
360
|
+
char_start = offsets.char_of_byte(plan.start_byte)
|
|
361
|
+
char_end = offsets.char_of_byte(plan.end_byte)
|
|
362
|
+
chunk_id = plan.symbol_id
|
|
363
|
+
if plan.segment_count > 1:
|
|
364
|
+
chunk_id = f"{chunk_id}::s{plan.segment_index}"
|
|
365
|
+
return {
|
|
366
|
+
"id": chunk_id,
|
|
367
|
+
"path": plan.path,
|
|
368
|
+
"language": plan.language,
|
|
369
|
+
"kind": plan.kind,
|
|
370
|
+
"name": plan.name,
|
|
371
|
+
"start_line": _line_of(starts, char_start),
|
|
372
|
+
"end_line": _line_of(starts, max(char_start, char_end - 1)),
|
|
373
|
+
"start_byte": plan.start_byte,
|
|
374
|
+
"end_byte": plan.end_byte,
|
|
375
|
+
"text": text,
|
|
376
|
+
"sha256": sha256_text(text),
|
|
377
|
+
# --- additive metadata (DESIGN allows it; required fields keep meaning)
|
|
378
|
+
"symbol_id": plan.symbol_id,
|
|
379
|
+
"parent_id": plan.parent_id,
|
|
380
|
+
"qualified_name": plan.qualified_name,
|
|
381
|
+
"signature": plan.signature,
|
|
382
|
+
"ancestors": list(plan.ancestors),
|
|
383
|
+
"segment_index": plan.segment_index,
|
|
384
|
+
"segment_count": plan.segment_count,
|
|
385
|
+
"full_start_byte": plan.full_start_byte,
|
|
386
|
+
"full_end_byte": plan.full_end_byte,
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
# ---------------------------------------------------------------------------
|
|
391
|
+
# build_context
|
|
392
|
+
# ---------------------------------------------------------------------------
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _checked_chunks(
|
|
396
|
+
path: str, language: str, raw: bytes, chunks: list[dict[str, Any]],
|
|
397
|
+
diagnostics: list[dict[str, Any]],
|
|
398
|
+
) -> list[dict[str, Any]]:
|
|
399
|
+
"""Check the frozen partition before consumers can schedule model work.
|
|
400
|
+
|
|
401
|
+
Valid partitions pass through unchanged, including symbol identities.
|
|
402
|
+
A broken partition becomes one lossless file region; never repair spans
|
|
403
|
+
while retaining misleading declaration identities.
|
|
404
|
+
"""
|
|
405
|
+
ordered = sorted(chunks, key=lambda chunk: chunk["start_byte"])
|
|
406
|
+
# No grammar output is a different condition from a broken partition.
|
|
407
|
+
# Keep the original diagnostic code as the reason shown to readers.
|
|
408
|
+
if not chunks:
|
|
409
|
+
try:
|
|
410
|
+
offsets = OffsetMap.from_bytes(raw)
|
|
411
|
+
except UnicodeDecodeError as exc:
|
|
412
|
+
if not any(row.get("path") == path and row.get("code") == "decode_error"
|
|
413
|
+
for row in diagnostics):
|
|
414
|
+
diagnostics.append({"path": path, "code": "decode_error",
|
|
415
|
+
"message": f"UTF-8 decode failed: {exc}"})
|
|
416
|
+
return []
|
|
417
|
+
if not offsets.text.strip():
|
|
418
|
+
# The host's ensure_chunk_coverage records known_empty without a unit.
|
|
419
|
+
return []
|
|
420
|
+
cause = next((row for row in diagnostics if row.get("path") == path
|
|
421
|
+
and row.get("code") in {"unsupported_language", "grammar_unavailable",
|
|
422
|
+
"parse_error", "file_too_large"}), None)
|
|
423
|
+
reason = cause["code"] if cause else "no_chunk_output"
|
|
424
|
+
diagnostic_code = "context_fallback"
|
|
425
|
+
message = (f"{reason}: {cause['message']}; " if cause else f"{reason}; ") + (
|
|
426
|
+
"froze one whole-file source chunk; declaration-level detail is unavailable")
|
|
427
|
+
else:
|
|
428
|
+
reason = ""
|
|
429
|
+
diagnostic_code = "source_reassembly_fallback"
|
|
430
|
+
message = ""
|
|
431
|
+
cursor = 0
|
|
432
|
+
for chunk in ordered:
|
|
433
|
+
start, end = chunk["start_byte"], chunk["end_byte"]
|
|
434
|
+
if start != cursor or not start <= end <= len(raw):
|
|
435
|
+
reason = f"non-contiguous chunk range [{start},{end}) at byte {cursor}"
|
|
436
|
+
break
|
|
437
|
+
cursor = end
|
|
438
|
+
if not reason and cursor != len(raw):
|
|
439
|
+
reason = f"chunk coverage ends at byte {cursor}, expected {len(raw)}"
|
|
440
|
+
assembled = b"".join(chunk["text"].encode("utf-8") for chunk in ordered)
|
|
441
|
+
if not reason and sha256_bytes(assembled) != sha256_bytes(raw):
|
|
442
|
+
reason = "reassembled source SHA-256 differs from the input file"
|
|
443
|
+
if not reason:
|
|
444
|
+
return chunks
|
|
445
|
+
|
|
446
|
+
try:
|
|
447
|
+
offsets = OffsetMap.from_bytes(raw)
|
|
448
|
+
except UnicodeDecodeError as exc:
|
|
449
|
+
raise ContextError("source_not_reassemblable",
|
|
450
|
+
f"{path}: cannot freeze byte-exact UTF-8 source: {exc}", path=path) from exc
|
|
451
|
+
diagnostics.append({
|
|
452
|
+
"path": path, "code": diagnostic_code, "reason": reason,
|
|
453
|
+
"message": message or (f"{reason}; CBE could not reassemble its declaration chunks; "
|
|
454
|
+
"froze one whole-file source chunk with coarser detail"),
|
|
455
|
+
})
|
|
456
|
+
if not raw:
|
|
457
|
+
return []
|
|
458
|
+
plan = _ChunkPlan(
|
|
459
|
+
path=path, language=language, kind="module_residual", name=PurePosixPath(path).name,
|
|
460
|
+
qualified_name="", symbol_id=canonical_symbol_id(path, "module_residual", "", 0),
|
|
461
|
+
parent_id=None, signature="", ancestors=(), start_byte=0, end_byte=len(raw),
|
|
462
|
+
full_start_byte=0, full_end_byte=len(raw), segment_index=0, segment_count=1,
|
|
463
|
+
)
|
|
464
|
+
fallback = _chunk_from_plan(plan, offsets, line_starts(offsets.text))
|
|
465
|
+
if sha256_text(fallback["text"]) != sha256_bytes(raw): # fail before any model calls
|
|
466
|
+
raise ContextError("source_not_reassemblable", f"{path}: whole-file fallback SHA-256 mismatch", path=path)
|
|
467
|
+
return [fallback]
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def _process_source_file(
|
|
471
|
+
relative: str,
|
|
472
|
+
raw: bytes,
|
|
473
|
+
digest: str,
|
|
474
|
+
files: list[dict[str, Any]],
|
|
475
|
+
chunks: list[dict[str, Any]],
|
|
476
|
+
diagnostics: list[dict[str, Any]],
|
|
477
|
+
max_segment_bytes: int,
|
|
478
|
+
) -> None:
|
|
479
|
+
spec = _grammar_for_suffix(PurePosixPath(relative).suffix)
|
|
480
|
+
language = language_name(relative)
|
|
481
|
+
|
|
482
|
+
if spec is None:
|
|
483
|
+
diagnostics.append(
|
|
484
|
+
{
|
|
485
|
+
"path": relative,
|
|
486
|
+
"code": "unsupported_language",
|
|
487
|
+
"message": f"no grammar registered for suffix {PurePosixPath(relative).suffix!r}",
|
|
488
|
+
}
|
|
489
|
+
)
|
|
490
|
+
files.append({"path": relative, "language": language, "sha256": digest})
|
|
491
|
+
return
|
|
492
|
+
|
|
493
|
+
if PurePosixPath(relative).suffix.lower() in AMBIGUOUS_SUFFIXES:
|
|
494
|
+
candidates = AMBIGUOUS_SUFFIXES[PurePosixPath(relative).suffix.lower()]
|
|
495
|
+
explicitly_mapped = any(".h" in extra.suffixes for extra in _EXTRA_SPECS.values())
|
|
496
|
+
if spec.name == candidates[0] and not explicitly_mapped:
|
|
497
|
+
diagnostics.append(
|
|
498
|
+
{
|
|
499
|
+
"path": relative,
|
|
500
|
+
"code": "ambiguous_language",
|
|
501
|
+
"message": (
|
|
502
|
+
f"{PurePosixPath(relative).suffix!r} is valid in {', '.join(candidates)}; "
|
|
503
|
+
f"parsed with {spec.name}. Register a mapping or use a unique extension "
|
|
504
|
+
"to remove the ambiguity."
|
|
505
|
+
),
|
|
506
|
+
}
|
|
507
|
+
)
|
|
508
|
+
|
|
509
|
+
if len(raw) > MAX_FILE_BYTES:
|
|
510
|
+
diagnostics.append(
|
|
511
|
+
{
|
|
512
|
+
"path": relative,
|
|
513
|
+
"code": "file_too_large",
|
|
514
|
+
"message": f"{len(raw)} bytes exceeds MAX_FILE_BYTES={MAX_FILE_BYTES}",
|
|
515
|
+
}
|
|
516
|
+
)
|
|
517
|
+
files.append({"path": relative, "language": language, "sha256": digest})
|
|
518
|
+
return
|
|
519
|
+
|
|
520
|
+
files.append({"path": relative, "language": language, "sha256": digest})
|
|
521
|
+
|
|
522
|
+
try:
|
|
523
|
+
offsets = OffsetMap.from_bytes(raw)
|
|
524
|
+
except UnicodeDecodeError as exc:
|
|
525
|
+
diagnostics.append(
|
|
526
|
+
{
|
|
527
|
+
"path": relative,
|
|
528
|
+
"code": "decode_error",
|
|
529
|
+
"message": f"UTF-8 decode failed: {exc}",
|
|
530
|
+
}
|
|
531
|
+
)
|
|
532
|
+
return
|
|
533
|
+
|
|
534
|
+
try:
|
|
535
|
+
parser = _make_parser(spec)
|
|
536
|
+
except Exception as exc:
|
|
537
|
+
diagnostics.append(
|
|
538
|
+
{
|
|
539
|
+
"path": relative,
|
|
540
|
+
"code": "grammar_unavailable",
|
|
541
|
+
"message": f"grammar {spec.module} is not importable: {exc}",
|
|
542
|
+
}
|
|
543
|
+
)
|
|
544
|
+
return
|
|
545
|
+
|
|
546
|
+
try:
|
|
547
|
+
tree = parser.parse(raw)
|
|
548
|
+
except Exception as exc: # pragma: no cover - grammar-level failure
|
|
549
|
+
diagnostics.append(
|
|
550
|
+
{"path": relative, "code": "parse_error", "message": f"{type(exc).__name__}: {exc}"}
|
|
551
|
+
)
|
|
552
|
+
return
|
|
553
|
+
|
|
554
|
+
root = tree.root_node
|
|
555
|
+
if getattr(root, "has_error", False):
|
|
556
|
+
line = root.start_point[0] + 1
|
|
557
|
+
diagnostics.append(
|
|
558
|
+
{
|
|
559
|
+
"path": relative,
|
|
560
|
+
"code": "syntax_error",
|
|
561
|
+
"message": f"tree contains an ERROR node near line {line}; chunk coverage is preserved",
|
|
562
|
+
}
|
|
563
|
+
)
|
|
564
|
+
|
|
565
|
+
declarations = _extract_declarations(spec, offsets, root, relative)
|
|
566
|
+
plans, plan_diagnostics = _plan_chunks(
|
|
567
|
+
offsets, declarations, root, relative, spec.name, max_segment_bytes
|
|
568
|
+
)
|
|
569
|
+
diagnostics.extend(plan_diagnostics)
|
|
570
|
+
starts = line_starts(offsets.text)
|
|
571
|
+
chunks.extend(_chunk_from_plan(plan, offsets, starts) for plan in plans)
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
def build_context(
|
|
575
|
+
repo: Path | str,
|
|
576
|
+
paths: list[str] | None = None,
|
|
577
|
+
*,
|
|
578
|
+
graph_path: Path | str | None = None,
|
|
579
|
+
max_segment_bytes: int = DEFAULT_MAX_SEGMENT_BYTES,
|
|
580
|
+
) -> dict[str, Any]:
|
|
581
|
+
"""Freeze one source snapshot and (optionally) a matching standard graph.
|
|
582
|
+
|
|
583
|
+
``paths`` must be repo-relative POSIX paths. They are validated so they
|
|
584
|
+
can neither escape the repository nor follow an external symlink.
|
|
585
|
+
``graph_path`` is read and its ``source_manifest`` is checked against the
|
|
586
|
+
current bytes; a mismatch raises :class:`GraphSourceMismatch`.
|
|
587
|
+
"""
|
|
588
|
+
|
|
589
|
+
repo_path = Path(repo)
|
|
590
|
+
if not repo_path.is_dir():
|
|
591
|
+
raise ContextError("invalid_repo", f"not a directory: {repo_path}", repo=str(repo_path))
|
|
592
|
+
|
|
593
|
+
selected, diagnostics = _iter_source_files(repo_path, paths)
|
|
594
|
+
|
|
595
|
+
files: list[dict[str, Any]] = []
|
|
596
|
+
chunks: list[dict[str, Any]] = []
|
|
597
|
+
source_hashes: dict[str, str] = {}
|
|
598
|
+
raw_sources: dict[str, bytes] = {}
|
|
599
|
+
|
|
600
|
+
for relative in selected:
|
|
601
|
+
full = repo_path / PurePosixPath(relative)
|
|
602
|
+
try:
|
|
603
|
+
raw = full.read_bytes()
|
|
604
|
+
except OSError as exc:
|
|
605
|
+
diagnostics.append({"path": relative, "code": "source_unavailable", "message": str(exc)})
|
|
606
|
+
continue
|
|
607
|
+
|
|
608
|
+
digest = sha256_bytes(raw)
|
|
609
|
+
raw_sources[relative] = raw
|
|
610
|
+
source_hashes[relative] = digest
|
|
611
|
+
_process_source_file(
|
|
612
|
+
relative, raw, digest, files, chunks, diagnostics, max_segment_bytes
|
|
613
|
+
)
|
|
614
|
+
|
|
615
|
+
by_path: dict[str, list[dict[str, Any]]] = {}
|
|
616
|
+
for chunk in chunks:
|
|
617
|
+
by_path.setdefault(chunk["path"], []).append(chunk)
|
|
618
|
+
chunks = [chunk for file in files for chunk in _checked_chunks(
|
|
619
|
+
file["path"], file["language"], raw_sources[file["path"]],
|
|
620
|
+
by_path.get(file["path"], []), diagnostics,
|
|
621
|
+
)]
|
|
622
|
+
|
|
623
|
+
graph: dict[str, Any] | None = None
|
|
624
|
+
graph_status = "not_provided"
|
|
625
|
+
if graph_path is not None:
|
|
626
|
+
graph = _load_graph(Path(graph_path), repo_path, source_hashes)
|
|
627
|
+
graph_status = "provided"
|
|
628
|
+
|
|
629
|
+
return {
|
|
630
|
+
"schema": SCHEMA_CONTEXT,
|
|
631
|
+
"files": files,
|
|
632
|
+
"chunks": chunks,
|
|
633
|
+
"graph": graph,
|
|
634
|
+
"graph_status": graph_status,
|
|
635
|
+
"diagnostics": diagnostics,
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
# ---------------------------------------------------------------------------
|
|
641
|
+
# Graph loading and pruning
|
|
642
|
+
# ---------------------------------------------------------------------------
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
def _load_graph(graph_path: Path, repo: Path, frozen_hashes: dict[str, str]) -> dict[str, Any]:
|
|
646
|
+
if not graph_path.exists():
|
|
647
|
+
raise ContextError("graph_missing", f"graph file does not exist: {graph_path}", path=str(graph_path))
|
|
648
|
+
try:
|
|
649
|
+
payload = json.loads(graph_path.read_text(encoding="utf-8"))
|
|
650
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
651
|
+
raise ContextError("graph_unreadable", f"cannot read graph {graph_path}: {exc}", path=str(graph_path)) from exc
|
|
652
|
+
if not isinstance(payload, dict):
|
|
653
|
+
raise ContextError("graph_schema_unsupported", "graph payload is not an object", path=str(graph_path))
|
|
654
|
+
if payload.get("schema") != SCHEMA_GRAPH:
|
|
655
|
+
raise ContextError(
|
|
656
|
+
"graph_schema_unsupported",
|
|
657
|
+
f"expected schema {SCHEMA_GRAPH!r}, got {payload.get('schema')!r}",
|
|
658
|
+
path=str(graph_path),
|
|
659
|
+
)
|
|
660
|
+
try:
|
|
661
|
+
check_graph_source(repo, payload, frozen_hashes)
|
|
662
|
+
except ContextError as exc:
|
|
663
|
+
# The shared check knows no file name; errors about the graph as a
|
|
664
|
+
# whole name the graph file, errors about one source file keep theirs.
|
|
665
|
+
if "path" not in exc.details and "paths" not in exc.details:
|
|
666
|
+
exc.details["path"] = str(graph_path)
|
|
667
|
+
raise
|
|
668
|
+
return payload
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
# ---------------------------------------------------------------------------
|
|
672
|
+
# context_for_paths
|
|
673
|
+
# ---------------------------------------------------------------------------
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
def context_for_paths(context: dict[str, Any], paths: list[str]) -> dict[str, Any]:
|
|
677
|
+
"""Select a local slice from a frozen context. Never reads the repository.
|
|
678
|
+
|
|
679
|
+
The slice keeps the chunks and files for the requested paths, prunes the
|
|
680
|
+
graph to edges incident to those paths plus their boundary/external
|
|
681
|
+
endpoints, and marks the result as partial so a consumer cannot mistake it
|
|
682
|
+
for a complete relationship set.
|
|
683
|
+
"""
|
|
684
|
+
|
|
685
|
+
if not isinstance(context, dict) or context.get("schema") != SCHEMA_CONTEXT:
|
|
686
|
+
raise ContextError(
|
|
687
|
+
"invalid_context",
|
|
688
|
+
f"expected a {SCHEMA_CONTEXT!r} context object",
|
|
689
|
+
)
|
|
690
|
+
selected: set[str] = set()
|
|
691
|
+
for raw in paths:
|
|
692
|
+
candidate = PurePosixPath(str(raw))
|
|
693
|
+
if candidate.is_absolute() or ".." in candidate.parts:
|
|
694
|
+
raise ContextError("invalid_path", f"path escapes the context scope: {raw}", path=str(raw))
|
|
695
|
+
selected.add(candidate.as_posix())
|
|
696
|
+
|
|
697
|
+
chunks = [chunk for chunk in context.get("chunks", []) if chunk.get("path") in selected]
|
|
698
|
+
files = [item for item in context.get("files", []) if item.get("path") in selected]
|
|
699
|
+
diagnostics = [item for item in context.get("diagnostics", []) if item.get("path") in selected]
|
|
700
|
+
|
|
701
|
+
return {
|
|
702
|
+
"schema": SCHEMA_CONTEXT,
|
|
703
|
+
"files": files,
|
|
704
|
+
"chunks": chunks,
|
|
705
|
+
"graph": _prune_graph(context.get("graph"), selected),
|
|
706
|
+
"graph_status": context.get("graph_status", "not_provided"),
|
|
707
|
+
"diagnostics": diagnostics,
|
|
708
|
+
"scope": {"paths": sorted(selected), "partial": True},
|
|
709
|
+
}
|