tokencut-proxy 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. tokencut/__init__.py +0 -0
  2. tokencut/cli.py +127 -0
  3. tokencut/contracts.py +148 -0
  4. tokencut/gateway/__init__.py +0 -0
  5. tokencut/gateway/__main__.py +14 -0
  6. tokencut/gateway/adapters/__init__.py +1 -0
  7. tokencut/gateway/adapters/anthropic.py +119 -0
  8. tokencut/gateway/adapters/common.py +95 -0
  9. tokencut/gateway/adapters/gemini.py +107 -0
  10. tokencut/gateway/adapters/openai.py +82 -0
  11. tokencut/gateway/app.py +366 -0
  12. tokencut/gateway/brief.py +101 -0
  13. tokencut/gateway/ccr.py +83 -0
  14. tokencut/gateway/classify.py +41 -0
  15. tokencut/gateway/codeview.py +139 -0
  16. tokencut/gateway/history.py +433 -0
  17. tokencut/gateway/live.html +135 -0
  18. tokencut/gateway/media.py +825 -0
  19. tokencut/gateway/note.py +71 -0
  20. tokencut/gateway/prompt_router.py +214 -0
  21. tokencut/gateway/records.py +312 -0
  22. tokencut/gateway/routing.py +131 -0
  23. tokencut/gateway/session.py +76 -0
  24. tokencut/gateway/task_router.py +157 -0
  25. tokencut/gateway/textcut.py +91 -0
  26. tokencut/gateway/trimmer.py +256 -0
  27. tokencut/gateway/usage.py +120 -0
  28. tokencut/gateway/virtual.py +273 -0
  29. tokencut/graph/__init__.py +0 -0
  30. tokencut/graph/builder.py +307 -0
  31. tokencut/graph/modules.py +149 -0
  32. tokencut/graph/queries.py +106 -0
  33. tokencut/ingestion/__init__.py +0 -0
  34. tokencut/ingestion/base.py +18 -0
  35. tokencut/ingestion/parser.py +36 -0
  36. tokencut/ingestion/python_parser.py +275 -0
  37. tokencut/ingestion/walker.py +68 -0
  38. tokencut/launch.py +104 -0
  39. tokencut/litellm_proxy.py +134 -0
  40. tokencut/pdftool/__init__.py +9 -0
  41. tokencut/pdftool/__main__.py +115 -0
  42. tokencut/pdftool/reader.py +97 -0
  43. tokencut/repoindex/__init__.py +0 -0
  44. tokencut/repoindex/__main__.py +67 -0
  45. tokencut/repoindex/bm25.py +63 -0
  46. tokencut/repoindex/embed.py +93 -0
  47. tokencut/repoindex/index.py +455 -0
  48. tokencut/repoindex/mcp.py +145 -0
  49. tokencut/tokens.py +19 -0
  50. tokencut/trimming/__init__.py +0 -0
  51. tokencut/trimming/packing.py +317 -0
  52. tokencut/trimming/query.py +259 -0
  53. tokencut/trimming/render.py +65 -0
  54. tokencut/trimming/scoring.py +117 -0
  55. tokencut/trimming/variants.py +61 -0
  56. tokencut/userconfig.py +107 -0
  57. tokencut/wizard.py +94 -0
  58. tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
  59. tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
  60. tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
  61. tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
  62. tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,317 @@
1
+ """Choose the 'full' pool and pack the four nested tiers (IMPLEMENTATION_PLAN C3, C4).
2
+
3
+ Tiers are packed aggressive -> moderate -> conservative -> full. Each tier starts
4
+ from everything the previous tier contained, so kept sets are nested and token
5
+ counts never decrease. Each tier is rendered and its real token count checked
6
+ against the budget; if the estimate was too low, the lowest-ranked chunk added
7
+ in this tier is removed and the text re-rendered until it fits.
8
+ """
9
+
10
+ from dataclasses import dataclass
11
+
12
+ from tokencut.contracts import (
13
+ AGGRESSIVE_MIN_TOKENS,
14
+ SKELETON_TIERS,
15
+ TIER_BUDGET_FRACTION,
16
+ Chunk,
17
+ ChunkKind,
18
+ ScoredChunk,
19
+ Tier,
20
+ )
21
+ from tokencut.graph.queries import GraphIndex
22
+ from tokencut.tokens import count_tokens
23
+ from tokencut.trimming.render import file_header, render_variant, skeleton_text
24
+
25
+ TIER_ORDER = [Tier.AGGRESSIVE, Tier.MODERATE, Tier.CONSERVATIVE, Tier.FULL]
26
+ SEPARATOR_TOKENS = 1 # the blank line between two rendered chunks, roughly one token
27
+ _SKELETON_KINDS = (ChunkKind.CLASS, ChunkKind.FUNCTION, ChunkKind.METHOD)
28
+ # Dependency hops pulled in with each kept chunk, per tier (tunable). In the tight aggressive
29
+ # tier each chunk comes alone (its class header and file imports are still added), so the
30
+ # budget goes to more relevant chunks; larger tiers also bring the functions it calls.
31
+ CLOSURE_HOPS: dict[Tier, int] = {Tier.AGGRESSIVE: 0, Tier.MODERATE: 1, Tier.CONSERVATIVE: 1}
32
+ # Pool file ranking (tunable; chosen on the 40 eval tasks, checked on held-out django tasks).
33
+ FILE_TOP_K = 3 # a file's chunk score = mean of its 3 best chunk scores
34
+ FILE_TEXT_WEIGHT = 0.25 # weight of the whole-file BM25 score
35
+
36
+
37
+ @dataclass
38
+ class PackedTier:
39
+ tier: Tier
40
+ budget_tokens: int
41
+ kept: set[str]
42
+ skeletons: set[str]
43
+ rendered_text: str
44
+ token_count: int
45
+
46
+
47
+ def rank_files(
48
+ chunks: list[Chunk], scores: list[ScoredChunk], file_text: dict[str, float] | None = None
49
+ ) -> list[str]:
50
+ """Files by relevance: mean of the file's top FILE_TOP_K chunk scores (normalised), blended
51
+ with the whole-file BM25 score (weight FILE_TEXT_WEIGHT). Ties broken by path."""
52
+ score_by_id = {s.chunk_id: s.score for s in scores}
53
+ per_file: dict[str, list[float]] = {}
54
+ for c in chunks:
55
+ per_file.setdefault(c.file_path, []).append(score_by_id.get(c.id, 0.0))
56
+ top: dict[str, float] = {}
57
+ for file_path, values in per_file.items():
58
+ best = sorted(values, reverse=True)[:FILE_TOP_K]
59
+ top[file_path] = sum(best) / FILE_TOP_K
60
+ highest = max(top.values(), default=0.0)
61
+ file_text = file_text or {}
62
+ value: dict[str, float] = {}
63
+ for file_path in per_file:
64
+ chunk_part = top[file_path] / highest if highest > 0 else 0.0
65
+ value[file_path] = (1 - FILE_TEXT_WEIGHT) * chunk_part + FILE_TEXT_WEIGHT * file_text.get(
66
+ file_path, 0.0
67
+ )
68
+ return sorted(value, key=lambda f: (-value[f], f))
69
+
70
+
71
+ def build_pool(
72
+ chunks: list[Chunk],
73
+ scores: list[ScoredChunk],
74
+ full_cap_tokens: int | None,
75
+ file_text: dict[str, float] | None = None,
76
+ ) -> list[Chunk]:
77
+ """The 'full' context (DECISIONS #4).
78
+
79
+ full_cap_tokens=None (gateway mode): every chunk. Otherwise (evaluation): whole files in
80
+ rank_files() order while the total fits the cap; a file that does not fit is skipped and
81
+ smaller ones are still tried.
82
+ """
83
+ if full_cap_tokens is None:
84
+ return list(chunks)
85
+ file_cost: dict[str, int] = {}
86
+ for c in chunks:
87
+ file_cost[c.file_path] = file_cost.get(c.file_path, count_tokens(file_header(c.file_path)))
88
+ file_cost[c.file_path] += c.token_count + SEPARATOR_TOKENS
89
+ selected: set[str] = set()
90
+ total = 0
91
+ for file_path in rank_files(chunks, scores, file_text):
92
+ if total + file_cost[file_path] <= full_cap_tokens:
93
+ selected.add(file_path)
94
+ total += file_cost[file_path]
95
+ return [c for c in chunks if c.file_path in selected]
96
+
97
+
98
+ def tier_budgets(
99
+ full_tokens: int, aggressive_min_tokens: int = AGGRESSIVE_MIN_TOKENS
100
+ ) -> dict[Tier, int]:
101
+ """Budget = fraction of full (DECISIONS #4), aggressive floor, capped at full.
102
+
103
+ A tier's budget is never below the previous tier's: on a small repo the aggressive
104
+ floor can exceed 35% of full, and moderate must not then get a smaller budget.
105
+ """
106
+ budgets = {}
107
+ previous = 0
108
+ for tier in TIER_ORDER:
109
+ budget = round(TIER_BUDGET_FRACTION[tier] * full_tokens)
110
+ if tier == Tier.AGGRESSIVE:
111
+ budget = max(budget, aggressive_min_tokens)
112
+ budget = min(max(budget, previous), full_tokens)
113
+ budgets[tier] = budget
114
+ previous = budget
115
+ return budgets
116
+
117
+
118
+ class TierPacker:
119
+ def __init__(self, pool: list[Chunk], scores: list[ScoredChunk], index: GraphIndex,
120
+ closure_hops: int | dict[Tier, int] | None = None,
121
+ tier_scores: dict[Tier, list[ScoredChunk]] | None = None):
122
+ """tier_scores: optional different ranking for some tiers (e.g. a blended one for the
123
+ larger tiers); tiers not listed use `scores`."""
124
+ self.pool = pool
125
+ self.by_id = {c.id: c for c in pool}
126
+ self.scores = scores
127
+ self.tier_scores = tier_scores or {}
128
+ self._use_ranking(scores)
129
+ self.index = index
130
+ # How many hops of dependencies come along with each kept chunk (per tier or for all).
131
+ self.closure_hops = CLOSURE_HOPS if closure_hops is None else closure_hops
132
+ self._hops = 1
133
+ self._skeleton_tokens: dict[str, int] = {}
134
+ self._header_tokens: dict[str, int] = {}
135
+
136
+ def _use_ranking(self, scores: list[ScoredChunk]) -> None:
137
+ # Rank = position in score order (score desc, ties by id); pool chunks without a score go last.
138
+ ordered = [s.chunk_id for s in scores if s.chunk_id in self.by_id]
139
+ seen = set(ordered)
140
+ ordered += sorted(cid for cid in self.by_id if cid not in seen)
141
+ self.ranked = ordered
142
+ self.rank = {cid: i for i, cid in enumerate(ordered)}
143
+
144
+ # ---------- cost estimates ----------
145
+
146
+ def _skel_cost(self, cid: str) -> int:
147
+ if cid not in self._skeleton_tokens:
148
+ self._skeleton_tokens[cid] = (
149
+ count_tokens(skeleton_text(self.by_id[cid])) + SEPARATOR_TOKENS
150
+ )
151
+ return self._skeleton_tokens[cid]
152
+
153
+ def _full_cost(self, cid: str) -> int:
154
+ return self.by_id[cid].token_count + SEPARATOR_TOKENS
155
+
156
+ def _header_cost(self, file_path: str) -> int:
157
+ if file_path not in self._header_tokens:
158
+ # No separator here: the newline after the header is the first chunk's separator.
159
+ self._header_tokens[file_path] = count_tokens(file_header(file_path))
160
+ return self._header_tokens[file_path]
161
+
162
+ def _estimate(self, kept: set[str], skeletons: set[str]) -> int:
163
+ files = {self.by_id[cid].file_path for cid in kept | skeletons}
164
+ return (
165
+ sum(self._full_cost(cid) for cid in kept)
166
+ + sum(self._skel_cost(cid) for cid in skeletons)
167
+ + sum(self._header_cost(f) for f in files)
168
+ )
169
+
170
+ def _cost_to_keep(self, group: set[str], skeletons: set[str], files: set[str]) -> int:
171
+ """Extra tokens if `group` becomes full code (a skeleton being upgraded costs the difference)."""
172
+ cost = 0
173
+ new_files = set()
174
+ for cid in group:
175
+ cost += self._full_cost(cid)
176
+ if cid in skeletons:
177
+ cost -= self._skel_cost(cid)
178
+ file_path = self.by_id[cid].file_path
179
+ if file_path not in files:
180
+ new_files.add(file_path)
181
+ return cost + sum(self._header_cost(f) for f in new_files)
182
+
183
+ def _parent(self, cid: str) -> str | None:
184
+ parent = self.by_id[cid].parent_id
185
+ return parent if parent in self.by_id else None
186
+
187
+ # ---------- packing steps ----------
188
+
189
+ def _add_full_code(self, kept: set[str], skeletons: set[str], budget: int) -> None:
190
+ remaining = budget - self._estimate(kept, skeletons)
191
+ files = {self.by_id[cid].file_path for cid in kept | skeletons}
192
+ for cid in self.ranked:
193
+ if cid in kept or self.by_id[cid].token_count == 0:
194
+ continue # an empty chunk (e.g. empty __init__.py) adds only a file header
195
+ group = (self.index.closure({cid}, max_hops=self._hops) & self.by_id.keys()) - kept
196
+ group.add(cid)
197
+ cost = self._cost_to_keep(group, skeletons, files)
198
+ if cost > remaining:
199
+ # Fall back to the chunk alone, plus what it needs to render correctly.
200
+ group = {cid}
201
+ parent = self._parent(cid)
202
+ if parent is not None:
203
+ group.add(parent)
204
+ module_id = self.index.module_of(cid)
205
+ if module_id is not None and module_id in self.by_id:
206
+ group.add(module_id)
207
+ group -= kept
208
+ cost = self._cost_to_keep(group, skeletons, files)
209
+ if cost > remaining:
210
+ continue # keep going: a smaller chunk further down may still fit
211
+ kept |= group
212
+ skeletons -= group
213
+ files |= {self.by_id[g].file_path for g in group}
214
+ remaining -= cost
215
+
216
+ def _add_skeletons(self, kept: set[str], skeletons: set[str], budget: int) -> None:
217
+ # DECISIONS #5: only dropped chunks directly connected to kept code become skeletons.
218
+ remaining = budget - self._estimate(kept, skeletons)
219
+ files = {self.by_id[cid].file_path for cid in kept | skeletons}
220
+ candidates = []
221
+ for cid in self.ranked:
222
+ if cid in kept or cid in skeletons or self.by_id[cid].kind not in _SKELETON_KINDS:
223
+ continue
224
+ if any(n in kept for n in self.index.neighbours(cid)):
225
+ candidates.append(cid)
226
+ for cid in candidates:
227
+ if cid in skeletons:
228
+ continue
229
+ group = [cid]
230
+ parent = self._parent(cid)
231
+ if parent is not None and parent not in kept and parent not in skeletons:
232
+ group.append(parent) # a method skeleton needs its class line above it
233
+ cost = sum(self._skel_cost(g) for g in group)
234
+ new_files = {self.by_id[g].file_path for g in group} - files
235
+ cost += sum(self._header_cost(f) for f in new_files)
236
+ if cost <= remaining:
237
+ skeletons.update(group)
238
+ files |= new_files
239
+ remaining -= cost
240
+
241
+ def _render(self, kept: set[str], skeletons: set[str]) -> tuple[str, int]:
242
+ text = render_variant(self.pool, kept, skeletons)
243
+ return text, count_tokens(text)
244
+
245
+ def _removable(
246
+ self, ids: set[str], protected: set[str], kept: set[str], skeletons: set[str]
247
+ ) -> list[str]:
248
+ """ids that can be removed without orphaning a method (its class must stay rendered)."""
249
+ has_children = {self._parent(cid) for cid in kept | skeletons}
250
+ return [cid for cid in ids if cid not in protected and cid not in has_children]
251
+
252
+ def _fit(
253
+ self,
254
+ kept: set[str],
255
+ skeletons: set[str],
256
+ base_kept: set[str],
257
+ base_skeletons: set[str],
258
+ budget: int,
259
+ ) -> tuple[str, int]:
260
+ """Remove lowest-ranked chunks added in this tier until the rendered text fits the budget.
261
+
262
+ Everything in the previous tier (base_*) is protected; the previous tier's text fit a
263
+ smaller budget, so this loop always ends within budget.
264
+ """
265
+ while True:
266
+ text, tokens = self._render(kept, skeletons)
267
+ if tokens <= budget:
268
+ return text, tokens
269
+ victims = self._removable(skeletons, base_skeletons, kept, skeletons)
270
+ if not victims:
271
+ victims = self._removable(kept, base_kept, kept, skeletons)
272
+ if not victims:
273
+ return text, tokens # unreachable in practice: the base content already fit
274
+ victim = max(victims, key=lambda cid: self.rank[cid])
275
+ if victim in kept:
276
+ kept.discard(victim)
277
+ if victim in base_skeletons:
278
+ skeletons.add(victim) # it was a skeleton before: fall back to that
279
+ else:
280
+ skeletons.discard(victim)
281
+
282
+ def pack(self, budgets: dict[Tier, int]) -> list[PackedTier]:
283
+ results: list[PackedTier] = []
284
+ prev_kept: set[str] = set()
285
+ prev_skeletons: set[str] = set()
286
+ prev_text, prev_tokens = "", 0
287
+
288
+ for tier in TIER_ORDER:
289
+ budget = budgets[tier]
290
+ self._use_ranking(self.tier_scores.get(tier, self.scores))
291
+ hops = self.closure_hops
292
+ self._hops = hops.get(tier, 1) if isinstance(hops, dict) else hops
293
+ if tier == Tier.FULL:
294
+ kept, skeletons = set(self.by_id), set()
295
+ text, tokens = self._render(kept, skeletons)
296
+ else:
297
+ kept = set(prev_kept)
298
+ skeletons = set(prev_skeletons) if tier in SKELETON_TIERS else set()
299
+ self._add_full_code(kept, skeletons, budget)
300
+ if tier in SKELETON_TIERS:
301
+ self._add_skeletons(kept, skeletons, budget)
302
+ # In a skeleton tier, a previous skeleton upgraded to full code may fall back to a skeleton.
303
+ base_skeletons = prev_skeletons if tier in SKELETON_TIERS else set()
304
+ text, tokens = self._fit(kept, skeletons, prev_kept, base_skeletons, budget)
305
+
306
+ if tokens < prev_tokens and tier not in SKELETON_TIERS:
307
+ # Dropping the previous tier's skeletons made this tier smaller: bring them back.
308
+ skeletons = prev_skeletons - kept
309
+ text, tokens = self._fit(kept, skeletons, prev_kept, prev_skeletons, budget)
310
+ if tokens < prev_tokens:
311
+ # Safety net (token merging can shift counts by a token or two).
312
+ kept, skeletons = set(prev_kept), set(prev_skeletons)
313
+ text, tokens = prev_text, prev_tokens
314
+
315
+ results.append(PackedTier(tier, budget, kept, skeletons, text, tokens))
316
+ prev_kept, prev_skeletons, prev_text, prev_tokens = kept, skeletons, text, tokens
317
+ return results
@@ -0,0 +1,259 @@
1
+ """Find which files and symbols the query mentions (IMPLEMENTATION_PLAN C1)."""
2
+
3
+ import re
4
+ from dataclasses import dataclass, field
5
+
6
+ from tokencut.contracts import Chunk, ChunkKind
7
+
8
+ SYMBOL_WEIGHT = 1.0 # query names a function/class/method (or a traceback points into it)
9
+ FILE_WEIGHT = 0.5 # query only names the file
10
+
11
+ _FILE_RE = re.compile(r"[\w./-]+\.py\b")
12
+ _TRACEBACK_RE = re.compile(r'File "([^"]+\.py)", line (\d+)(?:, in (\w+))?')
13
+ _SYMBOL_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*")
14
+ MIN_SYMBOL_LENGTH = 3
15
+
16
+ STOP_WORDS = frozenset(
17
+ [
18
+ "the",
19
+ "and",
20
+ "for",
21
+ "are",
22
+ "but",
23
+ "not",
24
+ "you",
25
+ "all",
26
+ "any",
27
+ "can",
28
+ "had",
29
+ "has",
30
+ "her",
31
+ "was",
32
+ "one",
33
+ "our",
34
+ "out",
35
+ "day",
36
+ "get",
37
+ "him",
38
+ "his",
39
+ "how",
40
+ "man",
41
+ "new",
42
+ "now",
43
+ "old",
44
+ "see",
45
+ "two",
46
+ "way",
47
+ "who",
48
+ "its",
49
+ "did",
50
+ "let",
51
+ "put",
52
+ "say",
53
+ "she",
54
+ "too",
55
+ "use",
56
+ "with",
57
+ "that",
58
+ "this",
59
+ "from",
60
+ "have",
61
+ "they",
62
+ "will",
63
+ "what",
64
+ "when",
65
+ "where",
66
+ "which",
67
+ "while",
68
+ "would",
69
+ "there",
70
+ "their",
71
+ "them",
72
+ "then",
73
+ "than",
74
+ "been",
75
+ "were",
76
+ "into",
77
+ "some",
78
+ "such",
79
+ "only",
80
+ "also",
81
+ "just",
82
+ "like",
83
+ "more",
84
+ "most",
85
+ "other",
86
+ "very",
87
+ "should",
88
+ "could",
89
+ "about",
90
+ "after",
91
+ "before",
92
+ "because",
93
+ "does",
94
+ "doing",
95
+ "done",
96
+ "each",
97
+ "here",
98
+ "same",
99
+ "these",
100
+ "those",
101
+ "under",
102
+ "until",
103
+ "upon",
104
+ "your",
105
+ "yours",
106
+ "error",
107
+ "errors",
108
+ "issue",
109
+ "bug",
110
+ "fix",
111
+ "fixed",
112
+ "wrong",
113
+ "instead",
114
+ "expected",
115
+ "actual",
116
+ "should",
117
+ "none",
118
+ "true",
119
+ "false",
120
+ "self",
121
+ "cls",
122
+ "return",
123
+ "returns",
124
+ "raise",
125
+ "raises",
126
+ "import",
127
+ "def",
128
+ "class",
129
+ "pass",
130
+ "python",
131
+ "file",
132
+ "line",
133
+ "code",
134
+ "test",
135
+ "tests",
136
+ "value",
137
+ "values",
138
+ ]
139
+ )
140
+
141
+
142
+ @dataclass
143
+ class QueryMentions:
144
+ file_paths: list[str] = field(default_factory=list) # repo files the query names, sorted
145
+ symbols: list[str] = field(
146
+ default_factory=list
147
+ ) # query words that matched a chunk name, sorted
148
+ chunk_ids: list[str] = field(default_factory=list) # every matched chunk, sorted
149
+ weights: dict[str, float] = field(
150
+ default_factory=dict
151
+ ) # chunk id -> SYMBOL_WEIGHT / FILE_WEIGHT
152
+
153
+
154
+ def _path_matches(mention: str, file_path: str) -> bool:
155
+ """True if one path is a suffix of the other on a '/' boundary ('cart.py' ~ 'shop/cart.py')."""
156
+ if mention == file_path:
157
+ return True
158
+ return file_path.endswith("/" + mention) or mention.endswith("/" + file_path)
159
+
160
+
161
+ def analyse_query(query: str, chunks: list[Chunk]) -> QueryMentions:
162
+ text = query.replace("\\", "/") # Windows paths in tracebacks
163
+ files = sorted({c.file_path for c in chunks})
164
+ chunks_by_file: dict[str, list[Chunk]] = {}
165
+ for c in chunks:
166
+ chunks_by_file.setdefault(c.file_path, []).append(c)
167
+
168
+ weights: dict[str, float] = {}
169
+ matched_files: set[str] = set()
170
+ matched_symbols: set[str] = set()
171
+
172
+ def give(chunk_id: str, weight: float) -> None:
173
+ weights[chunk_id] = max(weights.get(chunk_id, 0.0), weight)
174
+
175
+ # 1. Traceback lines: the innermost chunk around the line is a strong (symbol-level) signal.
176
+ for match in _TRACEBACK_RE.finditer(text):
177
+ path, line, func = match.group(1), int(match.group(2)), match.group(3)
178
+ for file_path in files:
179
+ if not _path_matches(path.lstrip("./"), file_path):
180
+ continue
181
+ matched_files.add(file_path)
182
+ target = _innermost(chunks_by_file[file_path], line)
183
+ if target is None and func:
184
+ target = next(
185
+ (
186
+ c
187
+ for c in chunks_by_file[file_path]
188
+ if c.qualified_name.split(".")[-1] == func
189
+ ),
190
+ None,
191
+ )
192
+ if target is not None:
193
+ give(target.id, SYMBOL_WEIGHT)
194
+
195
+ # 2. Plain file mentions: every chunk of the file gets the weaker file weight.
196
+ for match in _FILE_RE.finditer(text):
197
+ mention = match.group(0).lstrip("./")
198
+ for file_path in files:
199
+ if _path_matches(mention, file_path):
200
+ matched_files.add(file_path)
201
+ for file_path in sorted(matched_files):
202
+ for c in chunks_by_file[file_path]:
203
+ give(c.id, FILE_WEIGHT)
204
+
205
+ # 3. Symbols: identifiers / dotted names matched against qualified names.
206
+ by_qualified: dict[str, list[str]] = {}
207
+ by_last: dict[str, list[str]] = {}
208
+ for c in chunks:
209
+ if c.kind == ChunkKind.MODULE:
210
+ continue
211
+ by_qualified.setdefault(c.qualified_name, []).append(c.id)
212
+ by_last.setdefault(c.qualified_name.split(".")[-1], []).append(c.id)
213
+
214
+ for match in _SYMBOL_RE.finditer(_FILE_RE.sub(" ", text)):
215
+ symbol = match.group(0).strip(".")
216
+ if len(symbol) < MIN_SYMBOL_LENGTH or symbol.lower() in STOP_WORDS:
217
+ continue
218
+ ids = _match_symbol(symbol, by_qualified, by_last)
219
+ if ids:
220
+ matched_symbols.add(symbol)
221
+ for chunk_id in ids:
222
+ give(chunk_id, SYMBOL_WEIGHT)
223
+
224
+ return QueryMentions(
225
+ file_paths=sorted(matched_files),
226
+ symbols=sorted(matched_symbols),
227
+ chunk_ids=sorted(weights),
228
+ weights=dict(sorted(weights.items())),
229
+ )
230
+
231
+
232
+ def _match_symbol(
233
+ symbol: str, by_qualified: dict[str, list[str]], by_last: dict[str, list[str]]
234
+ ) -> list[str]:
235
+ """'Cart.total' -> exact qualified name; 'total' -> any chunk whose last name is 'total'.
236
+
237
+ For a dotted symbol that is not an exact qualified name ('self.subtotal', 'shop.cart.Cart')
238
+ try its longest suffix that is one, then fall back to its last component.
239
+ """
240
+ parts = symbol.split(".")
241
+ for i in range(len(parts) - 1): # suffixes with at least two parts, longest first
242
+ suffix = ".".join(parts[i:])
243
+ if suffix in by_qualified:
244
+ return by_qualified[suffix]
245
+ last = parts[-1]
246
+ if len(last) < MIN_SYMBOL_LENGTH or last.lower() in STOP_WORDS:
247
+ return []
248
+ return by_last.get(last, [])
249
+
250
+
251
+ def _innermost(chunks: list[Chunk], line: int) -> Chunk | None:
252
+ """Smallest non-module chunk whose line range contains the line (METHOD beats CLASS)."""
253
+ best = None
254
+ for c in chunks:
255
+ if c.kind == ChunkKind.MODULE or not (c.start_line <= line <= c.end_line):
256
+ continue
257
+ if best is None or (c.end_line - c.start_line) < (best.end_line - best.start_line):
258
+ best = c
259
+ return best
@@ -0,0 +1,65 @@
1
+ """Turn a set of kept / skeleton chunks into the text sent to the model (IMPLEMENTATION_PLAN C5).
2
+
3
+ # File: shop/cart.py
4
+ <chunks of this file in start_line order: full code, or skeleton>
5
+
6
+ # File: shop/utils/pricing.py
7
+ ...
8
+ """
9
+
10
+ from tokencut.contracts import Chunk
11
+
12
+ INDENT = " "
13
+
14
+
15
+ def file_header(file_path: str) -> str:
16
+ return f"# File: {file_path}"
17
+
18
+
19
+ def _leading_whitespace(line: str) -> str:
20
+ return line[: len(line) - len(line.lstrip())]
21
+
22
+
23
+ def skeleton_text(chunk: Chunk) -> str:
24
+ """Signature line(s) + docstring + '...', indented as in the source."""
25
+ code_lines = chunk.code.split("\n")
26
+ indent = _leading_whitespace(code_lines[0]) if code_lines else ""
27
+
28
+ # Use the real body indentation when we can see it (first code line after the header).
29
+ signature_lines = chunk.signature.split("\n") if chunk.signature else []
30
+ body_indent = indent + INDENT
31
+ for line in code_lines[len(signature_lines) :]:
32
+ if line.strip():
33
+ candidate = _leading_whitespace(line)
34
+ if len(candidate) > len(indent):
35
+ body_indent = candidate
36
+ break
37
+
38
+ lines = [indent + chunk.signature] if chunk.signature else []
39
+ if chunk.docstring:
40
+ doc = chunk.docstring.replace('"""', '\\"\\"\\"')
41
+ lines.append(f'{body_indent}"""{doc}"""')
42
+ lines.append(f"{body_indent}...")
43
+ return "\n".join(lines)
44
+
45
+
46
+ def render_variant(chunks: list[Chunk], kept_ids: set[str], skeleton_ids: set[str]) -> str:
47
+ """Render the chosen chunks. `chunks` must contain every id in kept_ids and skeleton_ids."""
48
+ by_file: dict[str, list[Chunk]] = {}
49
+ for c in chunks:
50
+ if c.id in kept_ids or c.id in skeleton_ids:
51
+ by_file.setdefault(c.file_path, []).append(c)
52
+
53
+ blocks = []
54
+ for file_path in sorted(by_file):
55
+ parts = [file_header(file_path)]
56
+ file_chunks = sorted(by_file[file_path], key=lambda c: (c.start_line, c.id))
57
+ texts = []
58
+ for c in file_chunks:
59
+ text = c.code if c.id in kept_ids else skeleton_text(c)
60
+ if text.strip():
61
+ texts.append(text)
62
+ if texts:
63
+ parts.append("\n\n".join(texts))
64
+ blocks.append("\n".join(parts))
65
+ return "\n\n".join(blocks)