tokencut-proxy 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokencut/__init__.py +0 -0
- tokencut/cli.py +127 -0
- tokencut/contracts.py +148 -0
- tokencut/gateway/__init__.py +0 -0
- tokencut/gateway/__main__.py +14 -0
- tokencut/gateway/adapters/__init__.py +1 -0
- tokencut/gateway/adapters/anthropic.py +119 -0
- tokencut/gateway/adapters/common.py +95 -0
- tokencut/gateway/adapters/gemini.py +107 -0
- tokencut/gateway/adapters/openai.py +82 -0
- tokencut/gateway/app.py +366 -0
- tokencut/gateway/brief.py +101 -0
- tokencut/gateway/ccr.py +83 -0
- tokencut/gateway/classify.py +41 -0
- tokencut/gateway/codeview.py +139 -0
- tokencut/gateway/history.py +433 -0
- tokencut/gateway/live.html +135 -0
- tokencut/gateway/media.py +825 -0
- tokencut/gateway/note.py +71 -0
- tokencut/gateway/prompt_router.py +214 -0
- tokencut/gateway/records.py +312 -0
- tokencut/gateway/routing.py +131 -0
- tokencut/gateway/session.py +76 -0
- tokencut/gateway/task_router.py +157 -0
- tokencut/gateway/textcut.py +91 -0
- tokencut/gateway/trimmer.py +256 -0
- tokencut/gateway/usage.py +120 -0
- tokencut/gateway/virtual.py +273 -0
- tokencut/graph/__init__.py +0 -0
- tokencut/graph/builder.py +307 -0
- tokencut/graph/modules.py +149 -0
- tokencut/graph/queries.py +106 -0
- tokencut/ingestion/__init__.py +0 -0
- tokencut/ingestion/base.py +18 -0
- tokencut/ingestion/parser.py +36 -0
- tokencut/ingestion/python_parser.py +275 -0
- tokencut/ingestion/walker.py +68 -0
- tokencut/launch.py +104 -0
- tokencut/litellm_proxy.py +134 -0
- tokencut/pdftool/__init__.py +9 -0
- tokencut/pdftool/__main__.py +115 -0
- tokencut/pdftool/reader.py +97 -0
- tokencut/repoindex/__init__.py +0 -0
- tokencut/repoindex/__main__.py +67 -0
- tokencut/repoindex/bm25.py +63 -0
- tokencut/repoindex/embed.py +93 -0
- tokencut/repoindex/index.py +455 -0
- tokencut/repoindex/mcp.py +145 -0
- tokencut/tokens.py +19 -0
- tokencut/trimming/__init__.py +0 -0
- tokencut/trimming/packing.py +317 -0
- tokencut/trimming/query.py +259 -0
- tokencut/trimming/render.py +65 -0
- tokencut/trimming/scoring.py +117 -0
- tokencut/trimming/variants.py +61 -0
- tokencut/userconfig.py +107 -0
- tokencut/wizard.py +94 -0
- tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
- tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
- tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
- tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
- tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
"""Choose the 'full' pool and pack the four nested tiers (IMPLEMENTATION_PLAN C3, C4).
|
|
2
|
+
|
|
3
|
+
Tiers are packed aggressive -> moderate -> conservative -> full. Each tier starts
|
|
4
|
+
from everything the previous tier contained, so kept sets are nested and token
|
|
5
|
+
counts never decrease. Each tier is rendered and its real token count checked
|
|
6
|
+
against the budget; if the estimate was too low, the lowest-ranked chunk added
|
|
7
|
+
in this tier is removed and the text re-rendered until it fits.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
|
|
12
|
+
from tokencut.contracts import (
|
|
13
|
+
AGGRESSIVE_MIN_TOKENS,
|
|
14
|
+
SKELETON_TIERS,
|
|
15
|
+
TIER_BUDGET_FRACTION,
|
|
16
|
+
Chunk,
|
|
17
|
+
ChunkKind,
|
|
18
|
+
ScoredChunk,
|
|
19
|
+
Tier,
|
|
20
|
+
)
|
|
21
|
+
from tokencut.graph.queries import GraphIndex
|
|
22
|
+
from tokencut.tokens import count_tokens
|
|
23
|
+
from tokencut.trimming.render import file_header, render_variant, skeleton_text
|
|
24
|
+
|
|
25
|
+
TIER_ORDER = [Tier.AGGRESSIVE, Tier.MODERATE, Tier.CONSERVATIVE, Tier.FULL]
|
|
26
|
+
SEPARATOR_TOKENS = 1 # the blank line between two rendered chunks, roughly one token
|
|
27
|
+
_SKELETON_KINDS = (ChunkKind.CLASS, ChunkKind.FUNCTION, ChunkKind.METHOD)
|
|
28
|
+
# Dependency hops pulled in with each kept chunk, per tier (tunable). In the tight aggressive
|
|
29
|
+
# tier each chunk comes alone (its class header and file imports are still added), so the
|
|
30
|
+
# budget goes to more relevant chunks; larger tiers also bring the functions it calls.
|
|
31
|
+
CLOSURE_HOPS: dict[Tier, int] = {Tier.AGGRESSIVE: 0, Tier.MODERATE: 1, Tier.CONSERVATIVE: 1}
|
|
32
|
+
# Pool file ranking (tunable; chosen on the 40 eval tasks, checked on held-out django tasks).
|
|
33
|
+
FILE_TOP_K = 3 # a file's chunk score = mean of its 3 best chunk scores
|
|
34
|
+
FILE_TEXT_WEIGHT = 0.25 # weight of the whole-file BM25 score
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class PackedTier:
|
|
39
|
+
tier: Tier
|
|
40
|
+
budget_tokens: int
|
|
41
|
+
kept: set[str]
|
|
42
|
+
skeletons: set[str]
|
|
43
|
+
rendered_text: str
|
|
44
|
+
token_count: int
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def rank_files(
|
|
48
|
+
chunks: list[Chunk], scores: list[ScoredChunk], file_text: dict[str, float] | None = None
|
|
49
|
+
) -> list[str]:
|
|
50
|
+
"""Files by relevance: mean of the file's top FILE_TOP_K chunk scores (normalised), blended
|
|
51
|
+
with the whole-file BM25 score (weight FILE_TEXT_WEIGHT). Ties broken by path."""
|
|
52
|
+
score_by_id = {s.chunk_id: s.score for s in scores}
|
|
53
|
+
per_file: dict[str, list[float]] = {}
|
|
54
|
+
for c in chunks:
|
|
55
|
+
per_file.setdefault(c.file_path, []).append(score_by_id.get(c.id, 0.0))
|
|
56
|
+
top: dict[str, float] = {}
|
|
57
|
+
for file_path, values in per_file.items():
|
|
58
|
+
best = sorted(values, reverse=True)[:FILE_TOP_K]
|
|
59
|
+
top[file_path] = sum(best) / FILE_TOP_K
|
|
60
|
+
highest = max(top.values(), default=0.0)
|
|
61
|
+
file_text = file_text or {}
|
|
62
|
+
value: dict[str, float] = {}
|
|
63
|
+
for file_path in per_file:
|
|
64
|
+
chunk_part = top[file_path] / highest if highest > 0 else 0.0
|
|
65
|
+
value[file_path] = (1 - FILE_TEXT_WEIGHT) * chunk_part + FILE_TEXT_WEIGHT * file_text.get(
|
|
66
|
+
file_path, 0.0
|
|
67
|
+
)
|
|
68
|
+
return sorted(value, key=lambda f: (-value[f], f))
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def build_pool(
|
|
72
|
+
chunks: list[Chunk],
|
|
73
|
+
scores: list[ScoredChunk],
|
|
74
|
+
full_cap_tokens: int | None,
|
|
75
|
+
file_text: dict[str, float] | None = None,
|
|
76
|
+
) -> list[Chunk]:
|
|
77
|
+
"""The 'full' context (DECISIONS #4).
|
|
78
|
+
|
|
79
|
+
full_cap_tokens=None (gateway mode): every chunk. Otherwise (evaluation): whole files in
|
|
80
|
+
rank_files() order while the total fits the cap; a file that does not fit is skipped and
|
|
81
|
+
smaller ones are still tried.
|
|
82
|
+
"""
|
|
83
|
+
if full_cap_tokens is None:
|
|
84
|
+
return list(chunks)
|
|
85
|
+
file_cost: dict[str, int] = {}
|
|
86
|
+
for c in chunks:
|
|
87
|
+
file_cost[c.file_path] = file_cost.get(c.file_path, count_tokens(file_header(c.file_path)))
|
|
88
|
+
file_cost[c.file_path] += c.token_count + SEPARATOR_TOKENS
|
|
89
|
+
selected: set[str] = set()
|
|
90
|
+
total = 0
|
|
91
|
+
for file_path in rank_files(chunks, scores, file_text):
|
|
92
|
+
if total + file_cost[file_path] <= full_cap_tokens:
|
|
93
|
+
selected.add(file_path)
|
|
94
|
+
total += file_cost[file_path]
|
|
95
|
+
return [c for c in chunks if c.file_path in selected]
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def tier_budgets(
|
|
99
|
+
full_tokens: int, aggressive_min_tokens: int = AGGRESSIVE_MIN_TOKENS
|
|
100
|
+
) -> dict[Tier, int]:
|
|
101
|
+
"""Budget = fraction of full (DECISIONS #4), aggressive floor, capped at full.
|
|
102
|
+
|
|
103
|
+
A tier's budget is never below the previous tier's: on a small repo the aggressive
|
|
104
|
+
floor can exceed 35% of full, and moderate must not then get a smaller budget.
|
|
105
|
+
"""
|
|
106
|
+
budgets = {}
|
|
107
|
+
previous = 0
|
|
108
|
+
for tier in TIER_ORDER:
|
|
109
|
+
budget = round(TIER_BUDGET_FRACTION[tier] * full_tokens)
|
|
110
|
+
if tier == Tier.AGGRESSIVE:
|
|
111
|
+
budget = max(budget, aggressive_min_tokens)
|
|
112
|
+
budget = min(max(budget, previous), full_tokens)
|
|
113
|
+
budgets[tier] = budget
|
|
114
|
+
previous = budget
|
|
115
|
+
return budgets
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class TierPacker:
|
|
119
|
+
def __init__(self, pool: list[Chunk], scores: list[ScoredChunk], index: GraphIndex,
|
|
120
|
+
closure_hops: int | dict[Tier, int] | None = None,
|
|
121
|
+
tier_scores: dict[Tier, list[ScoredChunk]] | None = None):
|
|
122
|
+
"""tier_scores: optional different ranking for some tiers (e.g. a blended one for the
|
|
123
|
+
larger tiers); tiers not listed use `scores`."""
|
|
124
|
+
self.pool = pool
|
|
125
|
+
self.by_id = {c.id: c for c in pool}
|
|
126
|
+
self.scores = scores
|
|
127
|
+
self.tier_scores = tier_scores or {}
|
|
128
|
+
self._use_ranking(scores)
|
|
129
|
+
self.index = index
|
|
130
|
+
# How many hops of dependencies come along with each kept chunk (per tier or for all).
|
|
131
|
+
self.closure_hops = CLOSURE_HOPS if closure_hops is None else closure_hops
|
|
132
|
+
self._hops = 1
|
|
133
|
+
self._skeleton_tokens: dict[str, int] = {}
|
|
134
|
+
self._header_tokens: dict[str, int] = {}
|
|
135
|
+
|
|
136
|
+
def _use_ranking(self, scores: list[ScoredChunk]) -> None:
|
|
137
|
+
# Rank = position in score order (score desc, ties by id); pool chunks without a score go last.
|
|
138
|
+
ordered = [s.chunk_id for s in scores if s.chunk_id in self.by_id]
|
|
139
|
+
seen = set(ordered)
|
|
140
|
+
ordered += sorted(cid for cid in self.by_id if cid not in seen)
|
|
141
|
+
self.ranked = ordered
|
|
142
|
+
self.rank = {cid: i for i, cid in enumerate(ordered)}
|
|
143
|
+
|
|
144
|
+
# ---------- cost estimates ----------
|
|
145
|
+
|
|
146
|
+
def _skel_cost(self, cid: str) -> int:
|
|
147
|
+
if cid not in self._skeleton_tokens:
|
|
148
|
+
self._skeleton_tokens[cid] = (
|
|
149
|
+
count_tokens(skeleton_text(self.by_id[cid])) + SEPARATOR_TOKENS
|
|
150
|
+
)
|
|
151
|
+
return self._skeleton_tokens[cid]
|
|
152
|
+
|
|
153
|
+
def _full_cost(self, cid: str) -> int:
|
|
154
|
+
return self.by_id[cid].token_count + SEPARATOR_TOKENS
|
|
155
|
+
|
|
156
|
+
def _header_cost(self, file_path: str) -> int:
|
|
157
|
+
if file_path not in self._header_tokens:
|
|
158
|
+
# No separator here: the newline after the header is the first chunk's separator.
|
|
159
|
+
self._header_tokens[file_path] = count_tokens(file_header(file_path))
|
|
160
|
+
return self._header_tokens[file_path]
|
|
161
|
+
|
|
162
|
+
def _estimate(self, kept: set[str], skeletons: set[str]) -> int:
|
|
163
|
+
files = {self.by_id[cid].file_path for cid in kept | skeletons}
|
|
164
|
+
return (
|
|
165
|
+
sum(self._full_cost(cid) for cid in kept)
|
|
166
|
+
+ sum(self._skel_cost(cid) for cid in skeletons)
|
|
167
|
+
+ sum(self._header_cost(f) for f in files)
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
def _cost_to_keep(self, group: set[str], skeletons: set[str], files: set[str]) -> int:
|
|
171
|
+
"""Extra tokens if `group` becomes full code (a skeleton being upgraded costs the difference)."""
|
|
172
|
+
cost = 0
|
|
173
|
+
new_files = set()
|
|
174
|
+
for cid in group:
|
|
175
|
+
cost += self._full_cost(cid)
|
|
176
|
+
if cid in skeletons:
|
|
177
|
+
cost -= self._skel_cost(cid)
|
|
178
|
+
file_path = self.by_id[cid].file_path
|
|
179
|
+
if file_path not in files:
|
|
180
|
+
new_files.add(file_path)
|
|
181
|
+
return cost + sum(self._header_cost(f) for f in new_files)
|
|
182
|
+
|
|
183
|
+
def _parent(self, cid: str) -> str | None:
|
|
184
|
+
parent = self.by_id[cid].parent_id
|
|
185
|
+
return parent if parent in self.by_id else None
|
|
186
|
+
|
|
187
|
+
# ---------- packing steps ----------
|
|
188
|
+
|
|
189
|
+
def _add_full_code(self, kept: set[str], skeletons: set[str], budget: int) -> None:
|
|
190
|
+
remaining = budget - self._estimate(kept, skeletons)
|
|
191
|
+
files = {self.by_id[cid].file_path for cid in kept | skeletons}
|
|
192
|
+
for cid in self.ranked:
|
|
193
|
+
if cid in kept or self.by_id[cid].token_count == 0:
|
|
194
|
+
continue # an empty chunk (e.g. empty __init__.py) adds only a file header
|
|
195
|
+
group = (self.index.closure({cid}, max_hops=self._hops) & self.by_id.keys()) - kept
|
|
196
|
+
group.add(cid)
|
|
197
|
+
cost = self._cost_to_keep(group, skeletons, files)
|
|
198
|
+
if cost > remaining:
|
|
199
|
+
# Fall back to the chunk alone, plus what it needs to render correctly.
|
|
200
|
+
group = {cid}
|
|
201
|
+
parent = self._parent(cid)
|
|
202
|
+
if parent is not None:
|
|
203
|
+
group.add(parent)
|
|
204
|
+
module_id = self.index.module_of(cid)
|
|
205
|
+
if module_id is not None and module_id in self.by_id:
|
|
206
|
+
group.add(module_id)
|
|
207
|
+
group -= kept
|
|
208
|
+
cost = self._cost_to_keep(group, skeletons, files)
|
|
209
|
+
if cost > remaining:
|
|
210
|
+
continue # keep going: a smaller chunk further down may still fit
|
|
211
|
+
kept |= group
|
|
212
|
+
skeletons -= group
|
|
213
|
+
files |= {self.by_id[g].file_path for g in group}
|
|
214
|
+
remaining -= cost
|
|
215
|
+
|
|
216
|
+
def _add_skeletons(self, kept: set[str], skeletons: set[str], budget: int) -> None:
|
|
217
|
+
# DECISIONS #5: only dropped chunks directly connected to kept code become skeletons.
|
|
218
|
+
remaining = budget - self._estimate(kept, skeletons)
|
|
219
|
+
files = {self.by_id[cid].file_path for cid in kept | skeletons}
|
|
220
|
+
candidates = []
|
|
221
|
+
for cid in self.ranked:
|
|
222
|
+
if cid in kept or cid in skeletons or self.by_id[cid].kind not in _SKELETON_KINDS:
|
|
223
|
+
continue
|
|
224
|
+
if any(n in kept for n in self.index.neighbours(cid)):
|
|
225
|
+
candidates.append(cid)
|
|
226
|
+
for cid in candidates:
|
|
227
|
+
if cid in skeletons:
|
|
228
|
+
continue
|
|
229
|
+
group = [cid]
|
|
230
|
+
parent = self._parent(cid)
|
|
231
|
+
if parent is not None and parent not in kept and parent not in skeletons:
|
|
232
|
+
group.append(parent) # a method skeleton needs its class line above it
|
|
233
|
+
cost = sum(self._skel_cost(g) for g in group)
|
|
234
|
+
new_files = {self.by_id[g].file_path for g in group} - files
|
|
235
|
+
cost += sum(self._header_cost(f) for f in new_files)
|
|
236
|
+
if cost <= remaining:
|
|
237
|
+
skeletons.update(group)
|
|
238
|
+
files |= new_files
|
|
239
|
+
remaining -= cost
|
|
240
|
+
|
|
241
|
+
def _render(self, kept: set[str], skeletons: set[str]) -> tuple[str, int]:
|
|
242
|
+
text = render_variant(self.pool, kept, skeletons)
|
|
243
|
+
return text, count_tokens(text)
|
|
244
|
+
|
|
245
|
+
def _removable(
|
|
246
|
+
self, ids: set[str], protected: set[str], kept: set[str], skeletons: set[str]
|
|
247
|
+
) -> list[str]:
|
|
248
|
+
"""ids that can be removed without orphaning a method (its class must stay rendered)."""
|
|
249
|
+
has_children = {self._parent(cid) for cid in kept | skeletons}
|
|
250
|
+
return [cid for cid in ids if cid not in protected and cid not in has_children]
|
|
251
|
+
|
|
252
|
+
def _fit(
|
|
253
|
+
self,
|
|
254
|
+
kept: set[str],
|
|
255
|
+
skeletons: set[str],
|
|
256
|
+
base_kept: set[str],
|
|
257
|
+
base_skeletons: set[str],
|
|
258
|
+
budget: int,
|
|
259
|
+
) -> tuple[str, int]:
|
|
260
|
+
"""Remove lowest-ranked chunks added in this tier until the rendered text fits the budget.
|
|
261
|
+
|
|
262
|
+
Everything in the previous tier (base_*) is protected; the previous tier's text fit a
|
|
263
|
+
smaller budget, so this loop always ends within budget.
|
|
264
|
+
"""
|
|
265
|
+
while True:
|
|
266
|
+
text, tokens = self._render(kept, skeletons)
|
|
267
|
+
if tokens <= budget:
|
|
268
|
+
return text, tokens
|
|
269
|
+
victims = self._removable(skeletons, base_skeletons, kept, skeletons)
|
|
270
|
+
if not victims:
|
|
271
|
+
victims = self._removable(kept, base_kept, kept, skeletons)
|
|
272
|
+
if not victims:
|
|
273
|
+
return text, tokens # unreachable in practice: the base content already fit
|
|
274
|
+
victim = max(victims, key=lambda cid: self.rank[cid])
|
|
275
|
+
if victim in kept:
|
|
276
|
+
kept.discard(victim)
|
|
277
|
+
if victim in base_skeletons:
|
|
278
|
+
skeletons.add(victim) # it was a skeleton before: fall back to that
|
|
279
|
+
else:
|
|
280
|
+
skeletons.discard(victim)
|
|
281
|
+
|
|
282
|
+
def pack(self, budgets: dict[Tier, int]) -> list[PackedTier]:
|
|
283
|
+
results: list[PackedTier] = []
|
|
284
|
+
prev_kept: set[str] = set()
|
|
285
|
+
prev_skeletons: set[str] = set()
|
|
286
|
+
prev_text, prev_tokens = "", 0
|
|
287
|
+
|
|
288
|
+
for tier in TIER_ORDER:
|
|
289
|
+
budget = budgets[tier]
|
|
290
|
+
self._use_ranking(self.tier_scores.get(tier, self.scores))
|
|
291
|
+
hops = self.closure_hops
|
|
292
|
+
self._hops = hops.get(tier, 1) if isinstance(hops, dict) else hops
|
|
293
|
+
if tier == Tier.FULL:
|
|
294
|
+
kept, skeletons = set(self.by_id), set()
|
|
295
|
+
text, tokens = self._render(kept, skeletons)
|
|
296
|
+
else:
|
|
297
|
+
kept = set(prev_kept)
|
|
298
|
+
skeletons = set(prev_skeletons) if tier in SKELETON_TIERS else set()
|
|
299
|
+
self._add_full_code(kept, skeletons, budget)
|
|
300
|
+
if tier in SKELETON_TIERS:
|
|
301
|
+
self._add_skeletons(kept, skeletons, budget)
|
|
302
|
+
# In a skeleton tier, a previous skeleton upgraded to full code may fall back to a skeleton.
|
|
303
|
+
base_skeletons = prev_skeletons if tier in SKELETON_TIERS else set()
|
|
304
|
+
text, tokens = self._fit(kept, skeletons, prev_kept, base_skeletons, budget)
|
|
305
|
+
|
|
306
|
+
if tokens < prev_tokens and tier not in SKELETON_TIERS:
|
|
307
|
+
# Dropping the previous tier's skeletons made this tier smaller: bring them back.
|
|
308
|
+
skeletons = prev_skeletons - kept
|
|
309
|
+
text, tokens = self._fit(kept, skeletons, prev_kept, prev_skeletons, budget)
|
|
310
|
+
if tokens < prev_tokens:
|
|
311
|
+
# Safety net (token merging can shift counts by a token or two).
|
|
312
|
+
kept, skeletons = set(prev_kept), set(prev_skeletons)
|
|
313
|
+
text, tokens = prev_text, prev_tokens
|
|
314
|
+
|
|
315
|
+
results.append(PackedTier(tier, budget, kept, skeletons, text, tokens))
|
|
316
|
+
prev_kept, prev_skeletons, prev_text, prev_tokens = kept, skeletons, text, tokens
|
|
317
|
+
return results
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
"""Find which files and symbols the query mentions (IMPLEMENTATION_PLAN C1)."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
|
|
6
|
+
from tokencut.contracts import Chunk, ChunkKind
|
|
7
|
+
|
|
8
|
+
SYMBOL_WEIGHT = 1.0 # query names a function/class/method (or a traceback points into it)
|
|
9
|
+
FILE_WEIGHT = 0.5 # query only names the file
|
|
10
|
+
|
|
11
|
+
_FILE_RE = re.compile(r"[\w./-]+\.py\b")
|
|
12
|
+
_TRACEBACK_RE = re.compile(r'File "([^"]+\.py)", line (\d+)(?:, in (\w+))?')
|
|
13
|
+
_SYMBOL_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*")
|
|
14
|
+
MIN_SYMBOL_LENGTH = 3
|
|
15
|
+
|
|
16
|
+
STOP_WORDS = frozenset(
|
|
17
|
+
[
|
|
18
|
+
"the",
|
|
19
|
+
"and",
|
|
20
|
+
"for",
|
|
21
|
+
"are",
|
|
22
|
+
"but",
|
|
23
|
+
"not",
|
|
24
|
+
"you",
|
|
25
|
+
"all",
|
|
26
|
+
"any",
|
|
27
|
+
"can",
|
|
28
|
+
"had",
|
|
29
|
+
"has",
|
|
30
|
+
"her",
|
|
31
|
+
"was",
|
|
32
|
+
"one",
|
|
33
|
+
"our",
|
|
34
|
+
"out",
|
|
35
|
+
"day",
|
|
36
|
+
"get",
|
|
37
|
+
"him",
|
|
38
|
+
"his",
|
|
39
|
+
"how",
|
|
40
|
+
"man",
|
|
41
|
+
"new",
|
|
42
|
+
"now",
|
|
43
|
+
"old",
|
|
44
|
+
"see",
|
|
45
|
+
"two",
|
|
46
|
+
"way",
|
|
47
|
+
"who",
|
|
48
|
+
"its",
|
|
49
|
+
"did",
|
|
50
|
+
"let",
|
|
51
|
+
"put",
|
|
52
|
+
"say",
|
|
53
|
+
"she",
|
|
54
|
+
"too",
|
|
55
|
+
"use",
|
|
56
|
+
"with",
|
|
57
|
+
"that",
|
|
58
|
+
"this",
|
|
59
|
+
"from",
|
|
60
|
+
"have",
|
|
61
|
+
"they",
|
|
62
|
+
"will",
|
|
63
|
+
"what",
|
|
64
|
+
"when",
|
|
65
|
+
"where",
|
|
66
|
+
"which",
|
|
67
|
+
"while",
|
|
68
|
+
"would",
|
|
69
|
+
"there",
|
|
70
|
+
"their",
|
|
71
|
+
"them",
|
|
72
|
+
"then",
|
|
73
|
+
"than",
|
|
74
|
+
"been",
|
|
75
|
+
"were",
|
|
76
|
+
"into",
|
|
77
|
+
"some",
|
|
78
|
+
"such",
|
|
79
|
+
"only",
|
|
80
|
+
"also",
|
|
81
|
+
"just",
|
|
82
|
+
"like",
|
|
83
|
+
"more",
|
|
84
|
+
"most",
|
|
85
|
+
"other",
|
|
86
|
+
"very",
|
|
87
|
+
"should",
|
|
88
|
+
"could",
|
|
89
|
+
"about",
|
|
90
|
+
"after",
|
|
91
|
+
"before",
|
|
92
|
+
"because",
|
|
93
|
+
"does",
|
|
94
|
+
"doing",
|
|
95
|
+
"done",
|
|
96
|
+
"each",
|
|
97
|
+
"here",
|
|
98
|
+
"same",
|
|
99
|
+
"these",
|
|
100
|
+
"those",
|
|
101
|
+
"under",
|
|
102
|
+
"until",
|
|
103
|
+
"upon",
|
|
104
|
+
"your",
|
|
105
|
+
"yours",
|
|
106
|
+
"error",
|
|
107
|
+
"errors",
|
|
108
|
+
"issue",
|
|
109
|
+
"bug",
|
|
110
|
+
"fix",
|
|
111
|
+
"fixed",
|
|
112
|
+
"wrong",
|
|
113
|
+
"instead",
|
|
114
|
+
"expected",
|
|
115
|
+
"actual",
|
|
116
|
+
"should",
|
|
117
|
+
"none",
|
|
118
|
+
"true",
|
|
119
|
+
"false",
|
|
120
|
+
"self",
|
|
121
|
+
"cls",
|
|
122
|
+
"return",
|
|
123
|
+
"returns",
|
|
124
|
+
"raise",
|
|
125
|
+
"raises",
|
|
126
|
+
"import",
|
|
127
|
+
"def",
|
|
128
|
+
"class",
|
|
129
|
+
"pass",
|
|
130
|
+
"python",
|
|
131
|
+
"file",
|
|
132
|
+
"line",
|
|
133
|
+
"code",
|
|
134
|
+
"test",
|
|
135
|
+
"tests",
|
|
136
|
+
"value",
|
|
137
|
+
"values",
|
|
138
|
+
]
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@dataclass
|
|
143
|
+
class QueryMentions:
|
|
144
|
+
file_paths: list[str] = field(default_factory=list) # repo files the query names, sorted
|
|
145
|
+
symbols: list[str] = field(
|
|
146
|
+
default_factory=list
|
|
147
|
+
) # query words that matched a chunk name, sorted
|
|
148
|
+
chunk_ids: list[str] = field(default_factory=list) # every matched chunk, sorted
|
|
149
|
+
weights: dict[str, float] = field(
|
|
150
|
+
default_factory=dict
|
|
151
|
+
) # chunk id -> SYMBOL_WEIGHT / FILE_WEIGHT
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _path_matches(mention: str, file_path: str) -> bool:
|
|
155
|
+
"""True if one path is a suffix of the other on a '/' boundary ('cart.py' ~ 'shop/cart.py')."""
|
|
156
|
+
if mention == file_path:
|
|
157
|
+
return True
|
|
158
|
+
return file_path.endswith("/" + mention) or mention.endswith("/" + file_path)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def analyse_query(query: str, chunks: list[Chunk]) -> QueryMentions:
|
|
162
|
+
text = query.replace("\\", "/") # Windows paths in tracebacks
|
|
163
|
+
files = sorted({c.file_path for c in chunks})
|
|
164
|
+
chunks_by_file: dict[str, list[Chunk]] = {}
|
|
165
|
+
for c in chunks:
|
|
166
|
+
chunks_by_file.setdefault(c.file_path, []).append(c)
|
|
167
|
+
|
|
168
|
+
weights: dict[str, float] = {}
|
|
169
|
+
matched_files: set[str] = set()
|
|
170
|
+
matched_symbols: set[str] = set()
|
|
171
|
+
|
|
172
|
+
def give(chunk_id: str, weight: float) -> None:
|
|
173
|
+
weights[chunk_id] = max(weights.get(chunk_id, 0.0), weight)
|
|
174
|
+
|
|
175
|
+
# 1. Traceback lines: the innermost chunk around the line is a strong (symbol-level) signal.
|
|
176
|
+
for match in _TRACEBACK_RE.finditer(text):
|
|
177
|
+
path, line, func = match.group(1), int(match.group(2)), match.group(3)
|
|
178
|
+
for file_path in files:
|
|
179
|
+
if not _path_matches(path.lstrip("./"), file_path):
|
|
180
|
+
continue
|
|
181
|
+
matched_files.add(file_path)
|
|
182
|
+
target = _innermost(chunks_by_file[file_path], line)
|
|
183
|
+
if target is None and func:
|
|
184
|
+
target = next(
|
|
185
|
+
(
|
|
186
|
+
c
|
|
187
|
+
for c in chunks_by_file[file_path]
|
|
188
|
+
if c.qualified_name.split(".")[-1] == func
|
|
189
|
+
),
|
|
190
|
+
None,
|
|
191
|
+
)
|
|
192
|
+
if target is not None:
|
|
193
|
+
give(target.id, SYMBOL_WEIGHT)
|
|
194
|
+
|
|
195
|
+
# 2. Plain file mentions: every chunk of the file gets the weaker file weight.
|
|
196
|
+
for match in _FILE_RE.finditer(text):
|
|
197
|
+
mention = match.group(0).lstrip("./")
|
|
198
|
+
for file_path in files:
|
|
199
|
+
if _path_matches(mention, file_path):
|
|
200
|
+
matched_files.add(file_path)
|
|
201
|
+
for file_path in sorted(matched_files):
|
|
202
|
+
for c in chunks_by_file[file_path]:
|
|
203
|
+
give(c.id, FILE_WEIGHT)
|
|
204
|
+
|
|
205
|
+
# 3. Symbols: identifiers / dotted names matched against qualified names.
|
|
206
|
+
by_qualified: dict[str, list[str]] = {}
|
|
207
|
+
by_last: dict[str, list[str]] = {}
|
|
208
|
+
for c in chunks:
|
|
209
|
+
if c.kind == ChunkKind.MODULE:
|
|
210
|
+
continue
|
|
211
|
+
by_qualified.setdefault(c.qualified_name, []).append(c.id)
|
|
212
|
+
by_last.setdefault(c.qualified_name.split(".")[-1], []).append(c.id)
|
|
213
|
+
|
|
214
|
+
for match in _SYMBOL_RE.finditer(_FILE_RE.sub(" ", text)):
|
|
215
|
+
symbol = match.group(0).strip(".")
|
|
216
|
+
if len(symbol) < MIN_SYMBOL_LENGTH or symbol.lower() in STOP_WORDS:
|
|
217
|
+
continue
|
|
218
|
+
ids = _match_symbol(symbol, by_qualified, by_last)
|
|
219
|
+
if ids:
|
|
220
|
+
matched_symbols.add(symbol)
|
|
221
|
+
for chunk_id in ids:
|
|
222
|
+
give(chunk_id, SYMBOL_WEIGHT)
|
|
223
|
+
|
|
224
|
+
return QueryMentions(
|
|
225
|
+
file_paths=sorted(matched_files),
|
|
226
|
+
symbols=sorted(matched_symbols),
|
|
227
|
+
chunk_ids=sorted(weights),
|
|
228
|
+
weights=dict(sorted(weights.items())),
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _match_symbol(
|
|
233
|
+
symbol: str, by_qualified: dict[str, list[str]], by_last: dict[str, list[str]]
|
|
234
|
+
) -> list[str]:
|
|
235
|
+
"""'Cart.total' -> exact qualified name; 'total' -> any chunk whose last name is 'total'.
|
|
236
|
+
|
|
237
|
+
For a dotted symbol that is not an exact qualified name ('self.subtotal', 'shop.cart.Cart')
|
|
238
|
+
try its longest suffix that is one, then fall back to its last component.
|
|
239
|
+
"""
|
|
240
|
+
parts = symbol.split(".")
|
|
241
|
+
for i in range(len(parts) - 1): # suffixes with at least two parts, longest first
|
|
242
|
+
suffix = ".".join(parts[i:])
|
|
243
|
+
if suffix in by_qualified:
|
|
244
|
+
return by_qualified[suffix]
|
|
245
|
+
last = parts[-1]
|
|
246
|
+
if len(last) < MIN_SYMBOL_LENGTH or last.lower() in STOP_WORDS:
|
|
247
|
+
return []
|
|
248
|
+
return by_last.get(last, [])
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _innermost(chunks: list[Chunk], line: int) -> Chunk | None:
|
|
252
|
+
"""Smallest non-module chunk whose line range contains the line (METHOD beats CLASS)."""
|
|
253
|
+
best = None
|
|
254
|
+
for c in chunks:
|
|
255
|
+
if c.kind == ChunkKind.MODULE or not (c.start_line <= line <= c.end_line):
|
|
256
|
+
continue
|
|
257
|
+
if best is None or (c.end_line - c.start_line) < (best.end_line - best.start_line):
|
|
258
|
+
best = c
|
|
259
|
+
return best
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Turn a set of kept / skeleton chunks into the text sent to the model (IMPLEMENTATION_PLAN C5).
|
|
2
|
+
|
|
3
|
+
# File: shop/cart.py
|
|
4
|
+
<chunks of this file in start_line order: full code, or skeleton>
|
|
5
|
+
|
|
6
|
+
# File: shop/utils/pricing.py
|
|
7
|
+
...
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from tokencut.contracts import Chunk
|
|
11
|
+
|
|
12
|
+
INDENT = " "
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def file_header(file_path: str) -> str:
|
|
16
|
+
return f"# File: {file_path}"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _leading_whitespace(line: str) -> str:
|
|
20
|
+
return line[: len(line) - len(line.lstrip())]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def skeleton_text(chunk: Chunk) -> str:
|
|
24
|
+
"""Signature line(s) + docstring + '...', indented as in the source."""
|
|
25
|
+
code_lines = chunk.code.split("\n")
|
|
26
|
+
indent = _leading_whitespace(code_lines[0]) if code_lines else ""
|
|
27
|
+
|
|
28
|
+
# Use the real body indentation when we can see it (first code line after the header).
|
|
29
|
+
signature_lines = chunk.signature.split("\n") if chunk.signature else []
|
|
30
|
+
body_indent = indent + INDENT
|
|
31
|
+
for line in code_lines[len(signature_lines) :]:
|
|
32
|
+
if line.strip():
|
|
33
|
+
candidate = _leading_whitespace(line)
|
|
34
|
+
if len(candidate) > len(indent):
|
|
35
|
+
body_indent = candidate
|
|
36
|
+
break
|
|
37
|
+
|
|
38
|
+
lines = [indent + chunk.signature] if chunk.signature else []
|
|
39
|
+
if chunk.docstring:
|
|
40
|
+
doc = chunk.docstring.replace('"""', '\\"\\"\\"')
|
|
41
|
+
lines.append(f'{body_indent}"""{doc}"""')
|
|
42
|
+
lines.append(f"{body_indent}...")
|
|
43
|
+
return "\n".join(lines)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def render_variant(chunks: list[Chunk], kept_ids: set[str], skeleton_ids: set[str]) -> str:
|
|
47
|
+
"""Render the chosen chunks. `chunks` must contain every id in kept_ids and skeleton_ids."""
|
|
48
|
+
by_file: dict[str, list[Chunk]] = {}
|
|
49
|
+
for c in chunks:
|
|
50
|
+
if c.id in kept_ids or c.id in skeleton_ids:
|
|
51
|
+
by_file.setdefault(c.file_path, []).append(c)
|
|
52
|
+
|
|
53
|
+
blocks = []
|
|
54
|
+
for file_path in sorted(by_file):
|
|
55
|
+
parts = [file_header(file_path)]
|
|
56
|
+
file_chunks = sorted(by_file[file_path], key=lambda c: (c.start_line, c.id))
|
|
57
|
+
texts = []
|
|
58
|
+
for c in file_chunks:
|
|
59
|
+
text = c.code if c.id in kept_ids else skeleton_text(c)
|
|
60
|
+
if text.strip():
|
|
61
|
+
texts.append(text)
|
|
62
|
+
if texts:
|
|
63
|
+
parts.append("\n\n".join(texts))
|
|
64
|
+
blocks.append("\n".join(parts))
|
|
65
|
+
return "\n\n".join(blocks)
|