tokencut-proxy 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (263) hide show
  1. tokencut_proxy-0.1.0/.gitattributes +2 -0
  2. tokencut_proxy-0.1.0/.github/workflows/release.yml +42 -0
  3. tokencut_proxy-0.1.0/.github/workflows/tests.yml +59 -0
  4. tokencut_proxy-0.1.0/.gitignore +37 -0
  5. tokencut_proxy-0.1.0/CLAUDE.md +395 -0
  6. tokencut_proxy-0.1.0/PKG-INFO +267 -0
  7. tokencut_proxy-0.1.0/README.md +236 -0
  8. tokencut_proxy-0.1.0/activate.bat +7 -0
  9. tokencut_proxy-0.1.0/api/__init__.py +0 -0
  10. tokencut_proxy-0.1.0/api/main.py +58 -0
  11. tokencut_proxy-0.1.0/deploy/litellm/config.yaml +41 -0
  12. tokencut_proxy-0.1.0/deploy/litellm/login.ps1 +7 -0
  13. tokencut_proxy-0.1.0/deploy/litellm/requirements.txt +1 -0
  14. tokencut_proxy-0.1.0/deploy/litellm/setup.ps1 +23 -0
  15. tokencut_proxy-0.1.0/deploy/litellm/setup.sh +16 -0
  16. tokencut_proxy-0.1.0/deploy/litellm/start.ps1 +8 -0
  17. tokencut_proxy-0.1.0/deploy/litellm/start.sh +7 -0
  18. tokencut_proxy-0.1.0/deploy/litellm/test.ps1 +23 -0
  19. tokencut_proxy-0.1.0/docs/ATTACHMENTS_BENCH.md +111 -0
  20. tokencut_proxy-0.1.0/docs/ATTACHMENTS_PLAN.md +228 -0
  21. tokencut_proxy-0.1.0/docs/DECISIONS.md +87 -0
  22. tokencut_proxy-0.1.0/docs/DEMO_DUAL_PLAN.md +125 -0
  23. tokencut_proxy-0.1.0/docs/FULL_RESULTS.md +559 -0
  24. tokencut_proxy-0.1.0/docs/IMPLEMENTATION_PLAN.md +412 -0
  25. tokencut_proxy-0.1.0/docs/LIVE_TEST_SCRIPT.md +252 -0
  26. tokencut_proxy-0.1.0/docs/PROGRESS_SUMMARY.md +119 -0
  27. tokencut_proxy-0.1.0/docs/PROMPT_ROUTER_LABELS.md +103 -0
  28. tokencut_proxy-0.1.0/docs/RESULTS_REPORT.md +1307 -0
  29. tokencut_proxy-0.1.0/docs/TEAMMATE_SETUP.md +161 -0
  30. tokencut_proxy-0.1.0/eval/__init__.py +0 -0
  31. tokencut_proxy-0.1.0/eval/attachments_bench.py +1127 -0
  32. tokencut_proxy-0.1.0/eval/attachments_datasets.py +188 -0
  33. tokencut_proxy-0.1.0/eval/benchmark.py +436 -0
  34. tokencut_proxy-0.1.0/eval/demo_data.py +155 -0
  35. tokencut_proxy-0.1.0/eval/demo_savings.py +79 -0
  36. tokencut_proxy-0.1.0/eval/embeddings.py +65 -0
  37. tokencut_proxy-0.1.0/eval/loader.py +185 -0
  38. tokencut_proxy-0.1.0/eval/metrics.py +167 -0
  39. tokencut_proxy-0.1.0/eval/mini_swe_replay.py +254 -0
  40. tokencut_proxy-0.1.0/eval/prepare.py +38 -0
  41. tokencut_proxy-0.1.0/eval/prompt_hint_eval.py +225 -0
  42. tokencut_proxy-0.1.0/eval/prompt_router/__init__.py +0 -0
  43. tokencut_proxy-0.1.0/eval/prompt_router/ccsbench.py +111 -0
  44. tokencut_proxy-0.1.0/eval/prompt_router/data.py +194 -0
  45. tokencut_proxy-0.1.0/eval/prompt_router/label.py +90 -0
  46. tokencut_proxy-0.1.0/eval/prompt_router/labels.jsonl +700 -0
  47. tokencut_proxy-0.1.0/eval/prompt_router/labels_extra.jsonl +192 -0
  48. tokencut_proxy-0.1.0/eval/prompt_router/labels_second_partial.jsonl +8 -0
  49. tokencut_proxy-0.1.0/eval/prompt_router/router_metrics.py +147 -0
  50. tokencut_proxy-0.1.0/eval/prompt_router/router_onnx_eval.py +109 -0
  51. tokencut_proxy-0.1.0/eval/prompt_router/run.py +225 -0
  52. tokencut_proxy-0.1.0/eval/prompt_router/setfit_kaggle.py +127 -0
  53. tokencut_proxy-0.1.0/eval/prompt_router/setfit_train.py +105 -0
  54. tokencut_proxy-0.1.0/eval/prompt_router/splits.json +708 -0
  55. tokencut_proxy-0.1.0/eval/prompt_router/swechat.py +99 -0
  56. tokencut_proxy-0.1.0/eval/prompt_router/swechat_prep.py +225 -0
  57. tokencut_proxy-0.1.0/eval/prompt_router/train_router_kaggle.py +320 -0
  58. tokencut_proxy-0.1.0/eval/replay_captures.py +387 -0
  59. tokencut_proxy-0.1.0/eval/repoindex_eval.py +193 -0
  60. tokencut_proxy-0.1.0/eval/results/benchmark_dev.csv +1681 -0
  61. tokencut_proxy-0.1.0/eval/results/benchmark_dev.md +82 -0
  62. tokencut_proxy-0.1.0/eval/results/benchmark_test.csv +2917 -0
  63. tokencut_proxy-0.1.0/eval/results/benchmark_test.md +73 -0
  64. tokencut_proxy-0.1.0/eval/results/demo/benchmark.json +1 -0
  65. tokencut_proxy-0.1.0/eval/results/demo/bugs.json +1 -0
  66. tokencut_proxy-0.1.0/eval/results/demo/data.json +1 -0
  67. tokencut_proxy-0.1.0/eval/results/demo/proofs.json +1 -0
  68. tokencut_proxy-0.1.0/eval/results/demo/repo_search.json +1 -0
  69. tokencut_proxy-0.1.0/eval/results/demo/requests.json +1 -0
  70. tokencut_proxy-0.1.0/eval/results/demo/techniques.json +1 -0
  71. tokencut_proxy-0.1.0/eval/results/iter1_results.csv +161 -0
  72. tokencut_proxy-0.1.0/eval/results/iter1_summary.md +49 -0
  73. tokencut_proxy-0.1.0/eval/results/mini_swe/requests.csv +8249 -0
  74. tokencut_proxy-0.1.0/eval/results/mini_swe/sessions.csv +151 -0
  75. tokencut_proxy-0.1.0/eval/results/mini_swe/summary.md +32 -0
  76. tokencut_proxy-0.1.0/eval/results/mini_swe/threshold_1000/requests.csv +8249 -0
  77. tokencut_proxy-0.1.0/eval/results/mini_swe/threshold_1000/sessions.csv +151 -0
  78. tokencut_proxy-0.1.0/eval/results/mini_swe/threshold_1000/summary.md +32 -0
  79. tokencut_proxy-0.1.0/eval/results/mini_swe/threshold_500/requests.csv +8249 -0
  80. tokencut_proxy-0.1.0/eval/results/mini_swe/threshold_500/sessions.csv +151 -0
  81. tokencut_proxy-0.1.0/eval/results/mini_swe/threshold_500/summary.md +31 -0
  82. tokencut_proxy-0.1.0/eval/results/prompt_hints/offline.json +4792 -0
  83. tokencut_proxy-0.1.0/eval/results/prompt_hints/stepd.json +101 -0
  84. tokencut_proxy-0.1.0/eval/results/prompt_hints/summary.md +25 -0
  85. tokencut_proxy-0.1.0/eval/results/prompt_router/devgpt_frozen_dev.json +28120 -0
  86. tokencut_proxy-0.1.0/eval/results/prompt_router/devgpt_frozen_dev.md +46 -0
  87. tokencut_proxy-0.1.0/eval/results/prompt_router/router_dev.json +391 -0
  88. tokencut_proxy-0.1.0/eval/results/prompt_router/router_dev.md +39 -0
  89. tokencut_proxy-0.1.0/eval/results/prompt_router/router_int8_dev.json +391 -0
  90. tokencut_proxy-0.1.0/eval/results/prompt_router/router_int8_dev.md +55 -0
  91. tokencut_proxy-0.1.0/eval/results/prompt_router/router_test.json +432 -0
  92. tokencut_proxy-0.1.0/eval/results/prompt_router/router_test.md +38 -0
  93. tokencut_proxy-0.1.0/eval/results/prompt_router/setfit_dev.md +76 -0
  94. tokencut_proxy-0.1.0/eval/results/prompt_router/swechat_eda.md +52 -0
  95. tokencut_proxy-0.1.0/eval/results/replay_L1/replay.json +735 -0
  96. tokencut_proxy-0.1.0/eval/results/replay_L1/replay.md +24 -0
  97. tokencut_proxy-0.1.0/eval/results/repoindex_dev.json +46178 -0
  98. tokencut_proxy-0.1.0/eval/results/repoindex_dev.md +22 -0
  99. tokencut_proxy-0.1.0/eval/results/repoindex_test.json +3842 -0
  100. tokencut_proxy-0.1.0/eval/results/repoindex_test.md +22 -0
  101. tokencut_proxy-0.1.0/eval/results/repro/benchmark_test.csv +1945 -0
  102. tokencut_proxy-0.1.0/eval/results/repro/benchmark_test.md +55 -0
  103. tokencut_proxy-0.1.0/eval/results/repro/diag.jsonl +162 -0
  104. tokencut_proxy-0.1.0/eval/results/repro/diag.py +87 -0
  105. tokencut_proxy-0.1.0/eval/results/repro/latency.json +125 -0
  106. tokencut_proxy-0.1.0/eval/results/repro/latency.py +169 -0
  107. tokencut_proxy-0.1.0/eval/results/repro_run.log +83 -0
  108. tokencut_proxy-0.1.0/eval/results/stepd/analysis.md +74 -0
  109. tokencut_proxy-0.1.0/eval/results/stepd/attempt1_no_warmup/requests.csv +78 -0
  110. tokencut_proxy-0.1.0/eval/results/stepd/attempt1_no_warmup/runs.csv +7 -0
  111. tokencut_proxy-0.1.0/eval/results/stepd/attempt1_no_warmup/state.json +10 -0
  112. tokencut_proxy-0.1.0/eval/results/stepd/attempt1_no_warmup/summary.md +17 -0
  113. tokencut_proxy-0.1.0/eval/results/stepd/attempt2_single_prompt/requests.csv +82 -0
  114. tokencut_proxy-0.1.0/eval/results/stepd/attempt2_single_prompt/runs.csv +7 -0
  115. tokencut_proxy-0.1.0/eval/results/stepd/attempt2_single_prompt/state.json +10 -0
  116. tokencut_proxy-0.1.0/eval/results/stepd/attempt2_single_prompt/summary.md +18 -0
  117. tokencut_proxy-0.1.0/eval/results/stepd/requests.csv +1562 -0
  118. tokencut_proxy-0.1.0/eval/results/stepd/requests_network_failed.csv +49 -0
  119. tokencut_proxy-0.1.0/eval/results/stepd/runs.csv +59 -0
  120. tokencut_proxy-0.1.0/eval/results/stepd/runs_network_failed.csv +2 -0
  121. tokencut_proxy-0.1.0/eval/results/stepd/state.json +62 -0
  122. tokencut_proxy-0.1.0/eval/results/stepd/summary.md +44 -0
  123. tokencut_proxy-0.1.0/eval/results/stepd/task_list.txt +1 -0
  124. tokencut_proxy-0.1.0/eval/results/stepd/validation.csv +13 -0
  125. tokencut_proxy-0.1.0/eval/results/stepd/validation_batch1.csv +13 -0
  126. tokencut_proxy-0.1.0/eval/results/stepd/validation_batch2.csv +23 -0
  127. tokencut_proxy-0.1.0/eval/results/stepd_p1/requests.csv +1628 -0
  128. tokencut_proxy-0.1.0/eval/results/stepd_p1/run.log +119 -0
  129. tokencut_proxy-0.1.0/eval/results/stepd_p1/runs.csv +59 -0
  130. tokencut_proxy-0.1.0/eval/results/stepd_p1/state.json +62 -0
  131. tokencut_proxy-0.1.0/eval/results/stepd_p1/validation.csv +30 -0
  132. tokencut_proxy-0.1.0/eval/results/stepd_p1_rerun/requests.csv +192 -0
  133. tokencut_proxy-0.1.0/eval/results/stepd_p1_rerun/run.log +9 -0
  134. tokencut_proxy-0.1.0/eval/results/stepd_p1_rerun/runs.csv +5 -0
  135. tokencut_proxy-0.1.0/eval/results/stepd_p1_rerun/state.json +8 -0
  136. tokencut_proxy-0.1.0/eval/run_eval.py +123 -0
  137. tokencut_proxy-0.1.0/eval/stepd_analyse.py +154 -0
  138. tokencut_proxy-0.1.0/eval/stepd_p1_run.py +201 -0
  139. tokencut_proxy-0.1.0/eval/stepd_run.py +532 -0
  140. tokencut_proxy-0.1.0/eval/stepd_validate.py +125 -0
  141. tokencut_proxy-0.1.0/eval/tasks_iter1.json +42 -0
  142. tokencut_proxy-0.1.0/eval/tasks_iter2.json +127 -0
  143. tokencut_proxy-0.1.0/eval/trim_viewer.py +146 -0
  144. tokencut_proxy-0.1.0/eval/verify_results.py +193 -0
  145. tokencut_proxy-0.1.0/fixtures/expected_chunks.json +422 -0
  146. tokencut_proxy-0.1.0/fixtures/expected_graph.json +180 -0
  147. tokencut_proxy-0.1.0/fixtures/sample_repo/README.md +16 -0
  148. tokencut_proxy-0.1.0/fixtures/sample_repo/config.yaml +2 -0
  149. tokencut_proxy-0.1.0/fixtures/sample_repo/shop/__init__.py +0 -0
  150. tokencut_proxy-0.1.0/fixtures/sample_repo/shop/cart.py +25 -0
  151. tokencut_proxy-0.1.0/fixtures/sample_repo/shop/logo.bin +0 -0
  152. tokencut_proxy-0.1.0/fixtures/sample_repo/shop/models.py +20 -0
  153. tokencut_proxy-0.1.0/fixtures/sample_repo/shop/payments/__init__.py +0 -0
  154. tokencut_proxy-0.1.0/fixtures/sample_repo/shop/payments/gateway.py +15 -0
  155. tokencut_proxy-0.1.0/fixtures/sample_repo/shop/payments/refunds.py +13 -0
  156. tokencut_proxy-0.1.0/fixtures/sample_repo/shop/utils/__init__.py +0 -0
  157. tokencut_proxy-0.1.0/fixtures/sample_repo/shop/utils/broken.py +13 -0
  158. tokencut_proxy-0.1.0/fixtures/sample_repo/shop/utils/pricing.py +11 -0
  159. tokencut_proxy-0.1.0/install.ps1 +21 -0
  160. tokencut_proxy-0.1.0/install.sh +25 -0
  161. tokencut_proxy-0.1.0/pyproject.toml +60 -0
  162. tokencut_proxy-0.1.0/requirements.txt +21 -0
  163. tokencut_proxy-0.1.0/results/README.md +9 -0
  164. tokencut_proxy-0.1.0/setup.cfg +4 -0
  165. tokencut_proxy-0.1.0/tests/__init__.py +0 -0
  166. tokencut_proxy-0.1.0/tests/conftest.py +27 -0
  167. tokencut_proxy-0.1.0/tests/test_api.py +49 -0
  168. tokencut_proxy-0.1.0/tests/test_attachments_bench.py +58 -0
  169. tokencut_proxy-0.1.0/tests/test_benchmark.py +51 -0
  170. tokencut_proxy-0.1.0/tests/test_cache_events.py +388 -0
  171. tokencut_proxy-0.1.0/tests/test_cli.py +61 -0
  172. tokencut_proxy-0.1.0/tests/test_end_to_end.py +36 -0
  173. tokencut_proxy-0.1.0/tests/test_eval.py +151 -0
  174. tokencut_proxy-0.1.0/tests/test_gateway_adapters.py +261 -0
  175. tokencut_proxy-0.1.0/tests/test_gateway_app.py +214 -0
  176. tokencut_proxy-0.1.0/tests/test_gateway_key.py +49 -0
  177. tokencut_proxy-0.1.0/tests/test_gateway_routing.py +242 -0
  178. tokencut_proxy-0.1.0/tests/test_gateway_trimming.py +565 -0
  179. tokencut_proxy-0.1.0/tests/test_gateway_virtual.py +277 -0
  180. tokencut_proxy-0.1.0/tests/test_graph.py +232 -0
  181. tokencut_proxy-0.1.0/tests/test_ingestion.py +236 -0
  182. tokencut_proxy-0.1.0/tests/test_litellm_proxy.py +78 -0
  183. tokencut_proxy-0.1.0/tests/test_media.py +362 -0
  184. tokencut_proxy-0.1.0/tests/test_media_pdf.py +314 -0
  185. tokencut_proxy-0.1.0/tests/test_media_resize.py +193 -0
  186. tokencut_proxy-0.1.0/tests/test_media_trim.py +173 -0
  187. tokencut_proxy-0.1.0/tests/test_pdftool.py +98 -0
  188. tokencut_proxy-0.1.0/tests/test_prompt_router.py +97 -0
  189. tokencut_proxy-0.1.0/tests/test_prompt_router_eval.py +104 -0
  190. tokencut_proxy-0.1.0/tests/test_repoindex.py +182 -0
  191. tokencut_proxy-0.1.0/tests/test_router_metrics.py +84 -0
  192. tokencut_proxy-0.1.0/tests/test_setup.py +74 -0
  193. tokencut_proxy-0.1.0/tests/test_swechat_prep.py +94 -0
  194. tokencut_proxy-0.1.0/tests/test_system_note.py +136 -0
  195. tokencut_proxy-0.1.0/tests/test_task_brief.py +238 -0
  196. tokencut_proxy-0.1.0/tests/test_trimming.py +381 -0
  197. tokencut_proxy-0.1.0/tests/test_userconfig.py +64 -0
  198. tokencut_proxy-0.1.0/tests/test_wizard.py +63 -0
  199. tokencut_proxy-0.1.0/tokencut/__init__.py +0 -0
  200. tokencut_proxy-0.1.0/tokencut/cli.py +127 -0
  201. tokencut_proxy-0.1.0/tokencut/contracts.py +148 -0
  202. tokencut_proxy-0.1.0/tokencut/gateway/__init__.py +0 -0
  203. tokencut_proxy-0.1.0/tokencut/gateway/__main__.py +14 -0
  204. tokencut_proxy-0.1.0/tokencut/gateway/adapters/__init__.py +1 -0
  205. tokencut_proxy-0.1.0/tokencut/gateway/adapters/anthropic.py +119 -0
  206. tokencut_proxy-0.1.0/tokencut/gateway/adapters/common.py +95 -0
  207. tokencut_proxy-0.1.0/tokencut/gateway/adapters/gemini.py +107 -0
  208. tokencut_proxy-0.1.0/tokencut/gateway/adapters/openai.py +82 -0
  209. tokencut_proxy-0.1.0/tokencut/gateway/app.py +366 -0
  210. tokencut_proxy-0.1.0/tokencut/gateway/brief.py +101 -0
  211. tokencut_proxy-0.1.0/tokencut/gateway/ccr.py +83 -0
  212. tokencut_proxy-0.1.0/tokencut/gateway/classify.py +41 -0
  213. tokencut_proxy-0.1.0/tokencut/gateway/codeview.py +139 -0
  214. tokencut_proxy-0.1.0/tokencut/gateway/history.py +433 -0
  215. tokencut_proxy-0.1.0/tokencut/gateway/live.html +135 -0
  216. tokencut_proxy-0.1.0/tokencut/gateway/media.py +825 -0
  217. tokencut_proxy-0.1.0/tokencut/gateway/note.py +71 -0
  218. tokencut_proxy-0.1.0/tokencut/gateway/prompt_router.py +214 -0
  219. tokencut_proxy-0.1.0/tokencut/gateway/records.py +312 -0
  220. tokencut_proxy-0.1.0/tokencut/gateway/routing.py +131 -0
  221. tokencut_proxy-0.1.0/tokencut/gateway/session.py +76 -0
  222. tokencut_proxy-0.1.0/tokencut/gateway/task_router.py +157 -0
  223. tokencut_proxy-0.1.0/tokencut/gateway/textcut.py +91 -0
  224. tokencut_proxy-0.1.0/tokencut/gateway/trimmer.py +256 -0
  225. tokencut_proxy-0.1.0/tokencut/gateway/usage.py +120 -0
  226. tokencut_proxy-0.1.0/tokencut/gateway/virtual.py +273 -0
  227. tokencut_proxy-0.1.0/tokencut/graph/__init__.py +0 -0
  228. tokencut_proxy-0.1.0/tokencut/graph/builder.py +307 -0
  229. tokencut_proxy-0.1.0/tokencut/graph/modules.py +149 -0
  230. tokencut_proxy-0.1.0/tokencut/graph/queries.py +106 -0
  231. tokencut_proxy-0.1.0/tokencut/ingestion/__init__.py +0 -0
  232. tokencut_proxy-0.1.0/tokencut/ingestion/base.py +18 -0
  233. tokencut_proxy-0.1.0/tokencut/ingestion/parser.py +36 -0
  234. tokencut_proxy-0.1.0/tokencut/ingestion/python_parser.py +275 -0
  235. tokencut_proxy-0.1.0/tokencut/ingestion/walker.py +68 -0
  236. tokencut_proxy-0.1.0/tokencut/launch.py +104 -0
  237. tokencut_proxy-0.1.0/tokencut/litellm_proxy.py +134 -0
  238. tokencut_proxy-0.1.0/tokencut/pdftool/__init__.py +9 -0
  239. tokencut_proxy-0.1.0/tokencut/pdftool/__main__.py +115 -0
  240. tokencut_proxy-0.1.0/tokencut/pdftool/reader.py +97 -0
  241. tokencut_proxy-0.1.0/tokencut/repoindex/__init__.py +0 -0
  242. tokencut_proxy-0.1.0/tokencut/repoindex/__main__.py +67 -0
  243. tokencut_proxy-0.1.0/tokencut/repoindex/bm25.py +63 -0
  244. tokencut_proxy-0.1.0/tokencut/repoindex/embed.py +93 -0
  245. tokencut_proxy-0.1.0/tokencut/repoindex/index.py +455 -0
  246. tokencut_proxy-0.1.0/tokencut/repoindex/mcp.py +145 -0
  247. tokencut_proxy-0.1.0/tokencut/tokens.py +19 -0
  248. tokencut_proxy-0.1.0/tokencut/trimming/__init__.py +0 -0
  249. tokencut_proxy-0.1.0/tokencut/trimming/packing.py +317 -0
  250. tokencut_proxy-0.1.0/tokencut/trimming/query.py +259 -0
  251. tokencut_proxy-0.1.0/tokencut/trimming/render.py +65 -0
  252. tokencut_proxy-0.1.0/tokencut/trimming/scoring.py +117 -0
  253. tokencut_proxy-0.1.0/tokencut/trimming/variants.py +61 -0
  254. tokencut_proxy-0.1.0/tokencut/userconfig.py +107 -0
  255. tokencut_proxy-0.1.0/tokencut/wizard.py +94 -0
  256. tokencut_proxy-0.1.0/tokencut_proxy.egg-info/PKG-INFO +267 -0
  257. tokencut_proxy-0.1.0/tokencut_proxy.egg-info/SOURCES.txt +261 -0
  258. tokencut_proxy-0.1.0/tokencut_proxy.egg-info/dependency_links.txt +1 -0
  259. tokencut_proxy-0.1.0/tokencut_proxy.egg-info/entry_points.txt +2 -0
  260. tokencut_proxy-0.1.0/tokencut_proxy.egg-info/requires.txt +24 -0
  261. tokencut_proxy-0.1.0/tokencut_proxy.egg-info/scm_file_list.json +257 -0
  262. tokencut_proxy-0.1.0/tokencut_proxy.egg-info/scm_version.json +8 -0
  263. tokencut_proxy-0.1.0/tokencut_proxy.egg-info/top_level.txt +1 -0
@@ -0,0 +1,2 @@
1
+ # Keep fixture files byte-identical on Windows and Linux (file_hash in expected_chunks.json).
2
+ fixtures/** -text
@@ -0,0 +1,42 @@
1
+ name: release
2
+
3
+ # Push a version tag to release: git tag v0.1.0 && git push origin v0.1.0
4
+ # 1. the full test + package matrix, 2. publish to PyPI (trusted publishing, no token
5
+ # stored in GitHub), 3. a GitHub Release with the wheel, sdist, install scripts and notes.
6
+ # Install link that always points to the newest release (works once the repo is public):
7
+ # https://github.com/Masooma-Ali/TokenCut/releases/latest/download/install.sh (or .ps1)
8
+ on:
9
+ push:
10
+ tags: ["v*"]
11
+
12
+ jobs:
13
+ checks:
14
+ uses: ./.github/workflows/tests.yml
15
+
16
+ pypi:
17
+ needs: checks
18
+ runs-on: ubuntu-latest
19
+ environment: pypi # one-time: add this repo as a trusted publisher of tokencut-proxy on PyPI
20
+ permissions:
21
+ id-token: write
22
+ steps:
23
+ - uses: actions/download-artifact@v4
24
+ with:
25
+ name: dist
26
+ path: dist/
27
+ - uses: pypa/gh-action-pypi-publish@release/v1
28
+
29
+ github-release:
30
+ needs: pypi
31
+ runs-on: ubuntu-latest
32
+ permissions:
33
+ contents: write
34
+ steps:
35
+ - uses: actions/checkout@v4
36
+ - uses: actions/download-artifact@v4
37
+ with:
38
+ name: dist
39
+ path: dist/
40
+ - env:
41
+ GH_TOKEN: ${{ github.token }}
42
+ run: gh release create "$GITHUB_REF_NAME" dist/* install.sh install.ps1 --generate-notes --title "TokenCut $GITHUB_REF_NAME"
@@ -0,0 +1,59 @@
1
+ name: tests
2
+
3
+ on:
4
+ push:
5
+ branches: ["**"] # tags run the same checks through release.yml
6
+ pull_request:
7
+ workflow_call: # release.yml runs the same checks before publishing
8
+
9
+ jobs:
10
+ test:
11
+ strategy:
12
+ fail-fast: false # report every system even if one fails
13
+ matrix:
14
+ os: [ubuntu-latest, windows-latest, macos-latest]
15
+ runs-on: ${{ matrix.os }}
16
+ steps:
17
+ - uses: actions/checkout@v4
18
+ - uses: actions/setup-python@v5
19
+ with:
20
+ python-version: "3.11"
21
+ cache: pip
22
+ # Poppler (pdftoppm): PDF page pictures (read_pdf); tests that need it skip without it.
23
+ - if: runner.os == 'Linux'
24
+ run: sudo apt-get update && sudo apt-get install -y poppler-utils
25
+ - run: pip install -r requirements.txt
26
+ - run: ruff check .
27
+ - run: pytest -v
28
+
29
+ package:
30
+ # Build the wheel, install it into a clean environment, check the command and the gateway.
31
+ strategy:
32
+ fail-fast: false
33
+ matrix:
34
+ os: [ubuntu-latest, windows-latest, macos-latest]
35
+ runs-on: ${{ matrix.os }}
36
+ steps:
37
+ - uses: actions/checkout@v4
38
+ with:
39
+ fetch-depth: 0 # the version comes from git tags
40
+ - uses: actions/setup-python@v5
41
+ with:
42
+ python-version: "3.12"
43
+ - run: pip install build && python -m build
44
+ - name: Install into a clean venv and smoke-test
45
+ shell: bash
46
+ run: |
47
+ python -m venv clean
48
+ if [ -d clean/Scripts ]; then bin=clean/Scripts; else bin=clean/bin; fi
49
+ "$bin/pip" install dist/*.whl
50
+ "$bin/tokencut" --version
51
+ TOKENCUT_HOME="$PWD/tchome" TOKENCUT_PORT=8799 "$bin/tokencut" gateway &
52
+ for i in $(seq 1 60); do curl -sf http://127.0.0.1:8799/tokencut/health && break; sleep 1; done
53
+ curl -sf http://127.0.0.1:8799/tokencut/health
54
+ curl -sf -o /dev/null http://127.0.0.1:8799/tokencut/
55
+ - uses: actions/upload-artifact@v4
56
+ if: matrix.os == 'ubuntu-latest'
57
+ with:
58
+ name: dist
59
+ path: dist/
@@ -0,0 +1,37 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *.egg-info/
5
+ .venv/
6
+ venv/
7
+ .pytest_cache/
8
+ .ruff_cache/
9
+ .coverage
10
+ htmlcov/
11
+
12
+ # Editors / OS
13
+ .vscode/
14
+ .idea/
15
+ .DS_Store
16
+ Thumbs.db
17
+
18
+ # Secrets
19
+ .env
20
+
21
+ # Large data (never commit)
22
+ data/
23
+ swebench_repos/
24
+ *.parquet
25
+ cache/
26
+
27
+ # Gateway logs and captured requests (contain prompts and code)
28
+ logs/
29
+
30
+ # LiteLLM secrets (live in ~/.tokencut and ~/.config/litellm; never commit)
31
+ litellm_master_key
32
+ auth.json
33
+
34
+ # Python packaging output
35
+ build/
36
+ dist/
37
+ *.egg-info/
@@ -0,0 +1,395 @@
1
+ # CLAUDE.md — TokenCut project guide for Claude Code
2
+
3
+ Read this whole file before doing anything in this repository. It tells you what TokenCut
4
+ is, what is already built, what the team has decided, **what your job is in the current
5
+ phase**, how to run everything, and how to report results.
6
+
7
+ ---
8
+
9
+ ## 0. Current phase (after the mid evaluation, updated 2026-11-21)
10
+
11
+ The mid evaluation has **passed**. The testing / evaluation phase (sections 6 and 8) is done;
12
+ its results are in `docs/RESULTS_REPORT.md`. The team now works on:
13
+
14
+ 1. **Installable product** (branch `feat/cli-install`, section 5a): one install command, a
15
+ `tokencut` command, `tokencut setup` for logins, `tokencut` opens Claude Code with
16
+ trimming, prompt enhancement and routing on; releases through a CD pipeline.
17
+ 2. **Prompt enhancement**: system note (`TOKENCUT_SYSTEM_NOTE`) and task briefs
18
+ (`TOKENCUT_PROMPT_BRIEF`) are merged but opt-in; a live A/B has not been run yet.
19
+ Planned next: **minimal-code instructions** (prompt text, measured by diff size).
20
+ 3. **ODE** (contextual bandit for model × effort per task): now in scope, not started.
21
+ Step D showed the tier hardly matters, so ODE's actions are the model (and effort).
22
+
23
+ How to work:
24
+ - Build what the user asks for, test-first, small PRs; plan first when the user asks for a
25
+ plan (do not start coding during a planning question).
26
+ - Ask before anything that spends API quota (live A/B, SWE-bench runs).
27
+ - Measure before claiming: tell the user what you measured, how, and what the numbers mean
28
+ in plain words. Never present a result you did not actually measure. If a run fails, say
29
+ so with the output. Honest negative results are as valuable as positive ones.
30
+ - Log every change of behaviour or tuning in the `docs/DECISIONS.md` change log.
31
+
32
+ ---
33
+
34
+ ## 1. What TokenCut is
35
+
36
+ TokenCut is a final-year project (FAST NUCES Islamabad): a **gateway** that sits between an
37
+ AI coding agent (Claude Code, Cline, Gemini CLI, any OpenAI-compatible client) and the model
38
+ provider (Anthropic, OpenAI, Gemini). It removes content the model does not need from each
39
+ request — mainly **tool output** (file contents, test logs, search results), and at the
40
+ right moments old history and unused tool definitions — so every request costs less,
41
+ **without breaking the agent, its edits, or the provider's prompt cache.**
42
+
43
+ Next, a contextual bandit called **ODE** will choose the model (and reasoning effort) per
44
+ task (in scope since the mid evaluation; not built yet).
45
+
46
+ ### One-sentence definition
47
+ *TokenCut is a cache-aware gateway that cuts the tool output coding agents feed
48
+ back to the model, keeping the code and errors that matter, so every request costs less
49
+ without breaking the agent.*
50
+
51
+ ### Plain-language picture
52
+ An editor stands between the agent and the model. Big file → keep the functions that matter,
53
+ leave a note for the rest ("lines 40–95 omitted, read again if needed"). Long test log → keep
54
+ failures and the summary. Never touch the user's messages or the model's replies. The model
55
+ remembers recent text cheaply (prompt cache), so old pages are never rewritten while that
56
+ memory is valid; when it is lost anyway (model switch, long pause, context nearly full), the
57
+ old pages are tidied too.
58
+
59
+ ---
60
+
61
+ ## 2. Team decisions you must respect
62
+
63
+ | Decision | Detail |
64
+ |---|---|
65
+ | **Phase focus** | After the mid evaluation: installable product, prompt enhancement, ODE (section 0). |
66
+ | **ODE / model routing** | In scope, not started. Decide per task (at a new typed user message), never mid-task: a model switch costs a cache rewrite. Actions = model (and effort); the tier stays `moderate` (it hardly mattered in step D). |
67
+ | **No local repo map in the gateway** | The gateway only trims what is inside requests. Do not add whole-repo indexing to the live gateway. |
68
+ | **Embeddings** | No embedding of **code chunks** in the gateway (too slow on a laptop CPU, ~13 chunks/s → 2–5 s per new large file); allowed offline (`eval/embeddings.py`). Exception (DECISIONS 2026-11-21): the ONNX prompt router embeds one short user message (median 14 ms). Models in the gateway are allowed in general. |
69
+ | **Cache safety is non-negotiable** | Between cache events, earlier messages must be sent byte-for-byte as before. There are tests for this; keep them green. |
70
+ | **Fail open** | Any trimming error → forward the original request. |
71
+ | **Privacy** | Logs contain numbers only (no prompts, code, headers or keys). Full bodies only with `TOKENCUT_CAPTURE=1`, local, gitignored. Never commit `logs/`, `cache/`, `swebench_repos/`. |
72
+ | **Deployment** | One install command; `tokencut setup` asks for the Claude and ChatGPT logins; **both Claude and ChatGPT go through LiteLLM**, which stays hidden from the user (private venv, generated config, child process). Claude Code is a prerequisite the user installs themselves. Router model is in the **public** HF repo `humataj/tokencut-router`. Releases via a CD pipeline (tag → tests → build → publish). |
73
+ | **Shared files need approval of the other two members** | `tokencut/contracts.py`, `tokencut/tokens.py`, `requirements.txt`, `pyproject.toml`, `docs/DECISIONS.md` (append to the change log is fine). |
74
+
75
+ Owners (README): ingestion + SWE-bench loader — Masooma Ali; graph + API — Huma Taj;
76
+ trimming + metrics — Fatima Shakir. In practice Claude implemented all parts; the team reviews.
77
+
78
+ ---
79
+
80
+ ## 3. What is built (current state)
81
+
82
+ ### Pipeline (offline, whole repo)
83
+ ```
84
+ repo ──► ingestion.parse_repo() ──► list[Chunk] (tree-sitter: module / class / function / method)
85
+ │
86
+ ▼
87
+ graph.build_graph() ──► CodeGraph (contains / imports / calls / inherits edges)
88
+ │
89
+ query ─────►▼
90
+ trimming.generate_variants() ──► 4 × ContextVariant (aggressive 15% / moderate 35% / conservative 60% / full)
91
+ ```
92
+ All shared data formats: `tokencut/contracts.py` (Chunk, Edge, CodeGraph, ScoredChunk,
93
+ ContextVariant, Tier, Segment, SegmentKind, ToolResultKind).
94
+
95
+ ### Gateway (live)
96
+ ```
97
+ agent ──► TokenCut gateway (python -m tokencut.gateway, 127.0.0.1:8787) ──► provider
98
+ 1. adapter: provider JSON → list[Segment] with JSON paths (gateway/adapters/)
99
+ 2. classify tool results: file_read / command_output / search (gateway/classify.py)
100
+ 3. on-arrival trim of results ≥ 2,000 tokens (gateway/trimmer.py)
101
+ Python code → codeview.py (Iteration 1 engine, verbatim lines + markers)
102
+ logs / search → textcut.py (tail, error lines, head)
103
+ 4. cache events: model switch to a cold model / cache expired / context nearly full
104
+ → re-trim old history, optional tool pruning (gateway/history.py, session.py)
105
+ 5. write back, forward, stream reply back, read billed usage (gateway/app.py, usage.py)
106
+ 6. log numbers, /tokencut/stats (gateway/records.py)
107
+ ```
108
+
109
+ ### Code map
110
+ | Path | What it does |
111
+ |---|---|
112
+ | `tokencut/ingestion/walker.py` | file list via `git ls-files`; skips non-.py, >1 MB, binary, `migrations/`, `vendor/` |
113
+ | `tokencut/ingestion/python_parser.py` | tree-sitter chunker; ids `path::Qualified.name`, `#2` for duplicates |
114
+ | `tokencut/graph/builder.py`, `modules.py`, `queries.py` | code graph; name-resolution rules (DECISIONS #7); `GraphIndex` closure / BFS |
115
+ | `tokencut/trimming/query.py` | file / symbol / traceback mentions in the query |
116
+ | `tokencut/trimming/scoring.py` | score = 0.5·structural (graph distance from mentions) + 0.5·BM25 |
117
+ | `tokencut/trimming/packing.py` | 32k pool (whole files, top-3 chunks + file BM25), tier budgets, nested greedy packer, skeletons; aggressive tier takes chunks without callees (`CLOSURE_HOPS`) |
118
+ | `tokencut/trimming/render.py`, `variants.py` | rendering; `generate_variants` |
119
+ | `api/main.py` | demo API: `POST /context/variants` |
120
+ | `tokencut/gateway/adapters/` | Anthropic / OpenAI / Gemini request ↔ Segments |
121
+ | `tokencut/gateway/trimmer.py` | on-arrival, cache-safe trimming; edit safety (files the user named / edited stay whole) |
122
+ | `tokencut/gateway/codeview.py` | code trimming inside tool results (kept lines verbatim incl. line numbers; `[TokenCut: lines a-b of f.py omitted …]`) |
123
+ | `tokencut/gateway/textcut.py` | command / search output trimming |
124
+ | `tokencut/gateway/history.py`, `session.py` | cache-event trimming, per-conversation memory, tool pruning |
125
+ | `tokencut/gateway/usage.py` | billed usage (incl. cache read / write) from SSE / JSON responses |
126
+ | `tokencut/gateway/records.py`, `app.py` | analysis, logs, stats, modes, settings |
127
+ | `tokencut/gateway/note.py` | fixed system note on TokenCut's markers (`TOKENCUT_SYSTEM_NOTE`) |
128
+ | `tokencut/gateway/task_router.py`, `brief.py`, `prompt_router.py` | task briefs: machine-text rules + ONNX router, six brief texts (`TOKENCUT_PROMPT_BRIEF`) |
129
+ | `tokencut/gateway/routing.py` | model routing: GPT (and optionally Claude) models to LiteLLM |
130
+ | `tokencut/cli.py`, `launch.py`, `wizard.py`, `userconfig.py`, `litellm_proxy.py` | the installed `tokencut` command: setup, config, hidden LiteLLM, launcher, status / logs (section 5a) |
131
+ | `eval/loader.py`, `metrics.py`, `run_eval.py` | SWE-bench Lite loader (HF datasets-server), gold chunks, Iteration 1 eval |
132
+ | `eval/benchmark.py`, `embeddings.py`, `prepare.py` | head-to-head benchmark vs baselines; task sets in `eval/tasks_iter2.json` |
133
+ | `tests/` | 381 tests (all must pass) |
134
+
135
+ ### Gateway settings (environment variables)
136
+ | Variable | Default | Meaning |
137
+ |---|---|---|
138
+ | `TOKENCUT_MODE` | `shadow` | `shadow` = forward unchanged, report possible savings; `trim` = send trimmed request |
139
+ | `TOKENCUT_TIER` | `moderate` | `aggressive` / `moderate` / `conservative` |
140
+ | `TOKENCUT_MIN_TRIM_TOKENS` | `2000` | smaller tool results are never trimmed (use `300` on the tiny playground repo) |
141
+ | `TOKENCUT_CACHE_EVENTS` | `1` | `0` disables history re-trimming at cache events |
142
+ | `TOKENCUT_CACHE_TTL` | `300` | idle seconds after which the cache counts as expired (requests marked `"ttl": "1h"` use 3600 — Claude Code appears to use this) |
143
+ | `TOKENCUT_PRUNE_TOOLS` | off | `1` = at cache events keep only tools used so far + core tools |
144
+ | `TOKENCUT_CORE_TOOLS` | Claude Code / Gemini CLI basics | comma-separated tools always kept when pruning |
145
+ | `TOKENCUT_TASK_MASK` | **on** (P1) | at a new user task, mask earlier tasks' large outputs if the cache rewrite pays off (R10); `0` = off |
146
+ | `TOKENCUT_TASK_MASK_TOKENS` | `300` | smallest output masked at a new task |
147
+ | `TOKENCUT_CCR` | off | `1` = shortened outputs carry a `tc_…` reference; gateway offers and answers `tokencut_retrieve` (P1) |
148
+ | `TOKENCUT_LAZY_TOOLS` | off | `1` = only core / used tools sent + catalog tool `tokencut_enable_tools` (P1) |
149
+ | `TOKENCUT_SYSTEM_NOTE` | off | `1` = fixed ~125-token note on TokenCut's markers at the end of the system prompt (agent requests, trim mode only; prompt enhancement, not yet A/B tested) |
150
+ | `TOKENCUT_PROMPT_BRIEF` | off | `1` = task brief appended to confidently classified typed user messages (rules + ONNX router; agent requests, trim mode only; stateless repeat rule; fail open) |
151
+ | `TOKENCUT_BRIEF_THRESHOLD` | `0.87` | router probability needed for a brief |
152
+ | `TOKENCUT_ROUTER_DIR` | — | local router model folder (4 files, not in git) |
153
+ | `TOKENCUT_ROUTER_REPO` | `humataj/tokencut-router` | public HF repo (no login), downloaded once to `~/.cache/tokencut/router` if no local folder |
154
+ | `TOKENCUT_GPT_UPSTREAM` | — | LiteLLM URL; GPT models (`TOKENCUT_GPT_MODELS` prefixes) are routed there |
155
+ | `TOKENCUT_GPT_KEY` | — | LiteLLM master key |
156
+ | `TOKENCUT_CLAUDE_VIA_LITELLM` | off | `1` = Claude models through LiteLLM too (the deployment default) |
157
+ | `TOKENCUT_INDEX_DIR` | `~/.cache/tokencut/repoindex` | repo index cache (P1, `python -m tokencut.repoindex`) |
158
+ | `TOKENCUT_KEY` | — | if set, proxy requests must send it in `x-tokencut-key` (set by `tokencut` from `~/.tokencut/tokencut_key`; never forwarded) |
159
+ | `TOKENCUT_CAPTURE` | off | `1` = save full request bodies locally (private data!) |
160
+ | `TOKENCUT_LOG_DIR` | `logs/gateway` | numbers-only JSONL logs |
161
+ | `TOKENCUT_{ANTHROPIC,OPENAI,GEMINI}_UPSTREAM` | official URLs | other compatible endpoints |
162
+ | `TOKENCUT_HOST` / `TOKENCUT_PORT` | `127.0.0.1` / `8787` | listen address |
163
+
164
+ Agent wiring: Claude Code `ANTHROPIC_BASE_URL=http://127.0.0.1:8787/anthropic`; OpenAI SDKs /
165
+ Cline base URL `http://127.0.0.1:8787/openai/v1`; Gemini base URL `http://127.0.0.1:8787/gemini`.
166
+
167
+ ---
168
+
169
+ ## 4. Results so far (what has actually been measured)
170
+
171
+ ### Offline benchmark (`eval/results/benchmark_test.md`, 81 held-out SWE-bench Lite tasks, never tuned on)
172
+ Metric: share of the chunks edited by the real fix that survive as full code. Equal token budgets.
173
+
174
+ **Pure trimming** (all methods shrink the same neutral 32k context; share of *available* gold kept):
175
+
176
+ | Budget | TokenCut | BM25 | Embeddings | Head truncation | Random |
177
+ |---|---|---|---|---|---|
178
+ | 15% | 0.68 | 0.65 | 0.72 | 0.15 | 0.12 |
179
+ | 35% | 0.86 | 0.80 | 0.90 | 0.47 | 0.36 |
180
+ | 60% | 0.90 | 0.93 | 0.96 | 0.69 | 0.63 |
181
+
182
+ TokenCut ≫ truncation / random (p < 0.001); level with BM25 and embeddings (no significant
183
+ difference). **Whole-repo selection:** embeddings lead (0.43 / 0.57 / 0.62 vs TokenCut 0.36 /
184
+ 0.49 / 0.58); TokenCut's whole-file 32k pool limits file recall (~62%). On the 40 dev tasks
185
+ TokenCut looked better than on test — part of that was tuning; report test numbers.
186
+
187
+ ### Gateway, offline
188
+ On-arrival trimming of a request with 4 real source files + a long pytest log: **70% smaller**,
189
+ **~85 ms** added (first request ~330 ms warm-up). Cache-event re-trim of a synthetic 30-turn
190
+ session: 71–85% smaller event request, ~30 ms (synthetic best case).
191
+
192
+ ### Gateway, live (Claude Code on the tiny playground repo, Haiku 5.5 ↔ Sonnet 5.5)
193
+ | Session | Result |
194
+ |---|---|
195
+ | Shadow mode | Tool definitions 66–80% of request tokens; 96% of input tokens are cache reads; file reads often arrive through `cat` (Bash) |
196
+ | Trim mode, on-arrival only | 22% of tool output removed, ~2% of request tokens; all tasks correct; cache intact |
197
+ | + cache-event trimming (model switch) | 6.4% of request tokens saved; post-switch summary complete and accurate |
198
+ | + tool pruning | **71% of request tokens saved** (34k tokens of tool definitions removed per request after the switch); all tasks correct; Claude used `curl` when the pruned web-fetch tool was missing |
199
+
200
+ These live sessions are **short and on a tiny repo**.
201
+
202
+ ### Task success on real bugs (step D, RESULTS_REPORT §7, 29 sympy SWE-bench tasks, Claude Code + Haiku)
203
+ - Core trimming (§7): resolved 19/29 with TokenCut vs 18/29 without; cost about −7 % per
204
+ request, −14 % per task (not statistically certain); 90 % of the saving came from R10.
205
+ - P1 features (§8: + lazy tools, CCR, repo index): cost −25 % (CI excludes zero) but resolved
206
+ 18/29 vs 22/29 (p = 0.125); one of the 4 losses was a TokenCut bug (fixed), 2 passed on a
207
+ re-run, 1 (22840) is still unexplained. Lazy tools stay opt-in. Later work
208
+ (prompt hints, packing fix, prompt router, task briefs): RESULTS_REPORT §10–§11.
209
+
210
+ ---
211
+
212
+ ## 5. Setup and commands
213
+
214
+ ### Install (Python 3.11)
215
+ ```bash
216
+ python3 -m venv .venv && source .venv/bin/activate
217
+ pip install -r requirements.txt
218
+ pytest -q # must be 381+ passed
219
+ ruff check . # must be clean
220
+ ```
221
+ The machine may have no `python` command; use `python3` or the venv. The first token count
222
+ downloads a small tiktoken file (needs internet once).
223
+
224
+ ### 5a. As an installed package (branch `feat/cli-install`)
225
+ ```bash
226
+ pip install -e ".[dev]" # editable install with test tools
227
+ tokencut setup # Claude Code check, ChatGPT question, hidden LiteLLM, router download
228
+ tokencut # connector + gateway + Claude Code (same as `tokencut code`)
229
+ tokencut status | logs [gateway|connector] | gateway | --version
230
+ ```
231
+ Users install with `install.sh` / `install.ps1` (uv + `uv tool install tokencut-proxy`; PyPI name
232
+ `tokencut` is taken). Code: `tokencut/cli.py`, `wizard.py` (setup), `userconfig.py`
233
+ (`~/.tokencut/config.json`, keys, env), `litellm_proxy.py` (pinned LiteLLM venv, generated
234
+ config), `launch.py`. `TOKENCUT_HOME` moves `~/.tokencut` (tests). `TOKENCUT_KEY` (gateway):
235
+ proxy requests need `x-tokencut-key`; the launcher passes it via `ANTHROPIC_CUSTOM_HEADERS`.
236
+ `pyproject.toml` holds version ranges (version from the git tag, setuptools-scm);
237
+ `requirements.txt` keeps the exact versions the team tests with. Release: push `vX.Y.Z` →
238
+ `.github/workflows/release.yml` (tests + package matrix → PyPI trusted publishing → GitHub
239
+ Release with wheel, sdist, `install.sh`, `install.ps1`; users fetch
240
+ `releases/latest/download/install.sh`, which needs a public repository). Keep `LITELLM_VERSION` equal to `deploy/litellm/requirements.txt` (tested).
241
+
242
+ ### Offline benchmark
243
+ Local data is NOT in git (gitignored, several GB): `swebench_repos/`, `cache/`
244
+ (parsed chunks, embeddings). Rebuild once:
245
+ ```bash
246
+ python -m eval.prepare --set dev # 40 tuning tasks (requests, flask, pytest, django)
247
+ python -m eval.prepare --set test # 81 held-out tasks (10 repos) — clones with --filter=blob:none
248
+ python -m eval.benchmark --set test --methods tokencut,bm25,random,head,bm25_files # minutes
249
+ python -m eval.benchmark --set test --methods tokencut,tokencut_hybrid,bm25,embed,random,head,bm25_files # hours on a laptop CPU (embeddings), cached after
250
+ python -m eval.benchmark --merge a.csv b.csv --set test --out eval/results # combine runs
251
+ python -m eval.run_eval # Iteration 1 metrics on the 40 dev tasks
252
+ ```
253
+ Results: `eval/results/benchmark_<set>.md` / `.csv`. **Never tune parameters on the test
254
+ set.** If you must tune, use dev, then confirm on a fresh held-out set (≈150 unused SWE-bench
255
+ Lite tasks remain; build it like `eval/loader.select_test_ids`).
256
+
257
+ ### Live gateway
258
+ ```bash
259
+ # terminal 1
260
+ TOKENCUT_MODE=trim TOKENCUT_MIN_TRIM_TOKENS=300 python -m tokencut.gateway
261
+ # terminal 2 (never on an important repo — use a throwaway copy)
262
+ cp -r fixtures/sample_repo ~/tokencut-playground && cd ~/tokencut-playground && git init -q && git add -A && git commit -qm start
263
+ ANTHROPIC_BASE_URL=http://127.0.0.1:8787/anthropic claude
264
+ ```
265
+ Stats: `http://127.0.0.1:8787/tokencut/stats` (`trimming.saved_by`, `trimming.cache_events`,
266
+ `billed_usage`). Per-request log lines: `logs/gateway/requests-<date>.jsonl`, e.g.
267
+ ```bash
268
+ grep -o '"model": "[^"]*"\|"cache_event": [^,]*' logs/gateway/requests-*.jsonl | paste - - | tail -25
269
+ grep -o '"how": "[a-z_]*", "tool": "[^"]*", "before": [0-9]*, "after": [0-9]*' logs/gateway/requests-*.jsonl | tail
270
+ ```
271
+ Restart the gateway between test runs (it keeps per-conversation memory in RAM).
272
+
273
+ ---
274
+
275
+ ## 6. The testing & evaluation work program (done before the mid evaluation; kept as reference)
276
+
277
+ Work through these in order. Record every run (date, machine, settings, agent, model, repo,
278
+ task script) in the report.
279
+
280
+ ### A. Reproduce (do first)
281
+ 1. `pytest -q`, `ruff check .` — note results.
282
+ 2. `python -m eval.prepare --set test`, then the no-embedding benchmark on test; compare with
283
+ `eval/results/benchmark_test.md`. Differences must be explained (versions, data).
284
+
285
+ ### B. Offline analysis (no API cost)
286
+ 1. **Per-repo breakdown** of the benchmark (django, sympy, matplotlib, …): where does TokenCut
287
+ win or lose, and why (look at individual tasks).
288
+ 2. **Skeleton value**: report `chunk_recall_with_skeletons` vs `chunk_recall` for TokenCut.
289
+ 3. **Latency**: time `trim_request` / `trim_with_cache_events` on realistic request bodies of
290
+ increasing size (10k, 50k, 100k, 200k tokens); report median and p95.
291
+ 4. **Failure analysis**: list the held-out tasks where gold code is lost at 35% and categorise
292
+ the cause (mention missing, large function, wrong file, module-level edit, …).
293
+ 5. Optional: re-run the embedding baselines if time allows (they are slow; cached after).
294
+
295
+ ### C. Live test matrix (real agent, throwaway playground repo, and ideally one larger real repo copy)
296
+ Use **the same scripted task list** for every configuration so results are comparable
297
+ (e.g. 8–12 tasks: run tests, read 2–3 files, grep, add a function with tests, fix a failing
298
+ test, summarise the session, …). For each configuration record `/tokencut/stats`, the log
299
+ lines above, task correctness, and any agent confusion (re-reads, mentions of TokenCut notes,
300
+ missing tools, errors).
301
+
302
+ | # | Configuration | Purpose |
303
+ |---|---|---|
304
+ | C1 | **No TokenCut** (agent talks to the provider directly) — or `TOKENCUT_MODE=shadow` | baseline cost (shadow = same numbers, nothing changed) |
305
+ | C2 | `trim`, `moderate`, cache events off (`TOKENCUT_CACHE_EVENTS=0`) | on-arrival trimming alone |
306
+ | C3 | `trim`, `moderate`, cache events on, switch model once mid-session | history re-trim at a model switch |
307
+ | C4 | C3 + `TOKENCUT_PRUNE_TOOLS=1` | tool pruning |
308
+ | C5 | C3 with `aggressive` and with `conservative` | tier comparison |
309
+ | C6 | long session (40+ turns) to reach `context_full`, and a >1 h idle gap for `cache_expired` | other cache events |
310
+ | C7 | another agent if available (Gemini CLI via `/gemini`, Cline / OpenAI-compatible via `/openai/v1`) | multi-provider check |
311
+
312
+ Key questions for each: **tokens saved (%), billed input per request, cache read / write
313
+ share, latency added, tasks correct (yes / no), extra turns or re-reads caused by trimming.**
314
+
315
+ ### D. Task-success check (needs the user's go-ahead and an API budget — ask first)
316
+ Run a small set of real SWE-bench tasks with an agent through the gateway, with and without
317
+ trimming, and compare resolve rate and cost. Do not spend money without explicit approval.
318
+
319
+ ### E. Report
320
+ Fill `docs/RESULTS_REPORT.md` (section 8). Lead with the honest bottom line, then tables.
321
+
322
+ ---
323
+
324
+ ## 7. Conventions
325
+
326
+ - Branch per piece of work: `test/<topic>` or `docs/<topic>`; small PRs; one teammate reviews.
327
+ - Before every commit: `ruff check .` and `pytest -q` (CI runs both). Ruff 0.16 enables extra
328
+ default rules (e.g. blind `except Exception` must log, unused unpacked variables, `datetime.UTC`).
329
+ - Contracts are law: modules talk through `tokencut/contracts.py` classes only.
330
+ - Deterministic output: sort, break ties by chunk id, never iterate a `set` to build output.
331
+ - Count tokens only with `tokencut.tokens.count_tokens` (tiktoken `o200k_base` proxy; it
332
+ undercounts Claude by ~1.5× — use billed usage for money).
333
+ - Record any change of behaviour or tuning in the `docs/DECISIONS.md` change log.
334
+ - PR descriptions end with an AI-assistance note.
335
+ - Tests that fake a provider must return **streamed** bodies (`ChunkedBody` / `reply_json` in
336
+ `tests/test_gateway_app.py`); httpx2's `MockTransport` pre-reads plain byte bodies.
337
+ - The machine used so far could not `git push` (no credentials): commit locally and give the
338
+ user the push command and a PR description.
339
+
340
+ ---
341
+
342
+ ## 8. Report template — `docs/RESULTS_REPORT.md`
343
+
344
+ ```markdown
345
+ # TokenCut — Results Report (<date>)
346
+
347
+ ## Bottom line (5 lines, plain words)
348
+ - How much TokenCut saves (offline / live), whether agents still work, how it compares, main weakness.
349
+
350
+ ## Setup
351
+ Machine, Python, commit hash, agent + version, models, repos, settings per run.
352
+
353
+ ## 1. Offline benchmark
354
+ Tables (trim / retrieve, 15/35/60%), CIs, significance, per-repo breakdown, failure categories.
355
+
356
+ ## 2. Latency
357
+ Per request size: median / p95 for on-arrival and cache-event trimming.
358
+
359
+ ## 3. Live sessions
360
+ Per configuration C1–C7: tokens saved, billed input per request, cache read/write share,
361
+ events, tasks correct, re-reads / confusion, notes.
362
+
363
+ ## 4. Comparison with research (SWE-Pruner, SWEzze, LongCodeZip, Pichay, Complexity Trap, Anthropic context editing)
364
+ What they report vs what we measured; where we are better / worse / not comparable.
365
+
366
+ ## 5. Limitations and risks
367
+ ## 6. Recommendations (do NOT implement — for the team to decide)
368
+ ```
369
+
370
+ ---
371
+
372
+ ## 9. Known limitations / open issues
373
+
374
+ 1. Task-success evidence is one run per task on one repo (sympy) and one model (Haiku): 19/29 vs 18/29 (core), 18/29 vs 22/29 (P1).
375
+ 2. Held-out benchmark: level with BM25 / embeddings, not clearly ahead; whole-repo file recall ~62%.
376
+ 3. On small projects most tokens are Claude Code's tool definitions; without tool pruning the saving is small (~2–6%).
377
+ 4. Code trimming is Python-only (logs / search work for any language).
378
+ 5. Per-conversation memory is in RAM: a gateway restart costs one cache rewrite (never a broken request).
379
+ 6. Tool pruning can block a tool with no substitute until the next cache event (hence opt-in).
380
+ 7. Conversation identity = system prompt + first user message; Claude Code's automatic
381
+ first-message text may make two sessions in the same folder look alike (seen once; a model
382
+ switch now only counts if the new model's cache is cold, which limits the impact).
383
+ 8. Live playground results are short sessions on a tiny repo; step D adds 29 real sympy tasks.
384
+ 10. Prompt enhancement (system note, task briefs) is not yet A/B tested for task success or tokens.
385
+ 9. Re-request protection works per file, not per line range.
386
+
387
+ ## 10. Glossary
388
+ - **Chunk** — one module / class / function / method of a Python file.
389
+ - **Tier** — aggressive (15%), moderate (35%), conservative (60%), full (100%); nested.
390
+ - **Gold chunk** — a chunk edited by the real SWE-bench fix.
391
+ - **Prompt cache** — provider feature: a repeated prompt prefix is billed at ~0.1×; writing it costs ~1.25×.
392
+ - **Cache event** — model switch to a cold model, cache TTL expired, or context nearly full.
393
+ - **On-arrival trimming** — trimming a tool result the first time it appears; identical afterwards.
394
+ - **Shadow mode** — measure only, forward unchanged.
395
+ - **ODE** — the planned bandit that chooses model (and effort) per task; in scope, not built.