xithead 1.5.0 → 1.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (662) hide show
  1. package/bin/xithead.js +486 -226
  2. package/{npm/index.js → index.js} +2 -2
  3. package/package.json +46 -46
  4. package/.actrc +0 -14
  5. package/.actrc.local.example +0 -15
  6. package/.changelog.md +0 -2
  7. package/.claude/agents/academic-anthropologist.md +0 -125
  8. package/.claude/agents/academic-geographer.md +0 -127
  9. package/.claude/agents/academic-historian.md +0 -123
  10. package/.claude/agents/academic-narratologist.md +0 -118
  11. package/.claude/agents/academic-psychologist.md +0 -118
  12. package/.claude/agents/academic-statistician.md +0 -144
  13. package/.claude/agents/accounts-payable-agent.md +0 -205
  14. package/.claude/agents/agentic-identity-trust.md +0 -394
  15. package/.claude/agents/agents-orchestrator.md +0 -367
  16. package/.claude/agents/automation-governance-architect.md +0 -216
  17. package/.claude/agents/blender-addon-engineer.md +0 -243
  18. package/.claude/agents/business-strategist.md +0 -488
  19. package/.claude/agents/change-management-consultant.md +0 -497
  20. package/.claude/agents/chief-financial-officer.md +0 -388
  21. package/.claude/agents/corporate-training-designer.md +0 -192
  22. package/.claude/agents/customer-service.md +0 -398
  23. package/.claude/agents/customer-success-manager.md +0 -460
  24. package/.claude/agents/data-consolidation-agent.md +0 -60
  25. package/.claude/agents/data-privacy-officer.md +0 -412
  26. package/.claude/agents/design-brand-guardian.md +0 -322
  27. package/.claude/agents/design-image-prompt-engineer.md +0 -236
  28. package/.claude/agents/design-inclusive-visuals-specialist.md +0 -73
  29. package/.claude/agents/design-persona-walkthrough.md +0 -272
  30. package/.claude/agents/design-ui-designer.md +0 -386
  31. package/.claude/agents/design-ui-finish-gate-reviewer.md +0 -217
  32. package/.claude/agents/design-ux-architect.md +0 -482
  33. package/.claude/agents/design-ux-researcher.md +0 -329
  34. package/.claude/agents/design-visual-storyteller.md +0 -149
  35. package/.claude/agents/design-whimsy-injector.md +0 -460
  36. package/.claude/agents/economy-designer.md +0 -156
  37. package/.claude/agents/engineering-ai-data-remediation-engineer.md +0 -211
  38. package/.claude/agents/engineering-ai-engineer.md +0 -146
  39. package/.claude/agents/engineering-api-platform-engineer.md +0 -181
  40. package/.claude/agents/engineering-ats-validator-architect.md +0 -383
  41. package/.claude/agents/engineering-autonomous-optimization-architect.md +0 -121
  42. package/.claude/agents/engineering-backend-architect.md +0 -255
  43. package/.claude/agents/engineering-china-network-engineer.md +0 -252
  44. package/.claude/agents/engineering-cms-developer.md +0 -536
  45. package/.claude/agents/engineering-code-reviewer.md +0 -76
  46. package/.claude/agents/engineering-codebase-onboarding-engineer.md +0 -173
  47. package/.claude/agents/engineering-data-engineer.md +0 -349
  48. package/.claude/agents/engineering-data-visualization-engineer.md +0 -151
  49. package/.claude/agents/engineering-database-optimizer.md +0 -188
  50. package/.claude/agents/engineering-database-reliability-engineer.md +0 -201
  51. package/.claude/agents/engineering-desktop-app-engineer.md +0 -209
  52. package/.claude/agents/engineering-developer-tooling-engineer.md +0 -153
  53. package/.claude/agents/engineering-devops-automator.md +0 -385
  54. package/.claude/agents/engineering-drupal-performance.md +0 -347
  55. package/.claude/agents/engineering-drupal-shopping-cart.md +0 -360
  56. package/.claude/agents/engineering-electrical-engineer.md +0 -575
  57. package/.claude/agents/engineering-email-intelligence-engineer.md +0 -390
  58. package/.claude/agents/engineering-embedded-firmware-engineer.md +0 -196
  59. package/.claude/agents/engineering-feishu-integration-developer.md +0 -606
  60. package/.claude/agents/engineering-filament-optimization-specialist.md +0 -283
  61. package/.claude/agents/engineering-finops-engineer.md +0 -164
  62. package/.claude/agents/engineering-frontend-developer.md +0 -257
  63. package/.claude/agents/engineering-gaussdb-expert.md +0 -334
  64. package/.claude/agents/engineering-git-workflow-master.md +0 -98
  65. package/.claude/agents/engineering-i18n-engineer.md +0 -214
  66. package/.claude/agents/engineering-identity-access-engineer.md +0 -217
  67. package/.claude/agents/engineering-incident-response-commander.md +0 -444
  68. package/.claude/agents/engineering-iot-fleet-engineer.md +0 -148
  69. package/.claude/agents/engineering-it-service-manager.md +0 -561
  70. package/.claude/agents/engineering-knowledge-graph-engineer.md +0 -372
  71. package/.claude/agents/engineering-llm-post-training-engineer.md +0 -166
  72. package/.claude/agents/engineering-minimal-change-engineer.md +0 -207
  73. package/.claude/agents/engineering-mobile-app-builder.md +0 -493
  74. package/.claude/agents/engineering-mobile-release-engineer.md +0 -164
  75. package/.claude/agents/engineering-multi-agent-systems-architect.md +0 -600
  76. package/.claude/agents/engineering-network-engineer.md +0 -245
  77. package/.claude/agents/engineering-orgscript-engineer.md +0 -113
  78. package/.claude/agents/engineering-payments-billing-engineer.md +0 -228
  79. package/.claude/agents/engineering-pdf-engine-architect.md +0 -696
  80. package/.claude/agents/engineering-platform-engineer.md +0 -270
  81. package/.claude/agents/engineering-privacy-engineer.md +0 -152
  82. package/.claude/agents/engineering-prompt-engineer.md +0 -209
  83. package/.claude/agents/engineering-rag-pipeline-engineer.md +0 -450
  84. package/.claude/agents/engineering-rapid-prototyper.md +0 -466
  85. package/.claude/agents/engineering-realtime-collaboration-engineer.md +0 -200
  86. package/.claude/agents/engineering-rust-refactoring-specialist.md +0 -313
  87. package/.claude/agents/engineering-search-relevance-engineer.md +0 -237
  88. package/.claude/agents/engineering-section-508-specialist.md +0 -339
  89. package/.claude/agents/engineering-senior-developer.md +0 -176
  90. package/.claude/agents/engineering-servicenow-developer-mentor.md +0 -180
  91. package/.claude/agents/engineering-software-architect.md +0 -112
  92. package/.claude/agents/engineering-solidity-smart-contract-engineer.md +0 -522
  93. package/.claude/agents/engineering-sre.md +0 -90
  94. package/.claude/agents/engineering-technical-writer.md +0 -393
  95. package/.claude/agents/engineering-universal-document-compiler.md +0 -370
  96. package/.claude/agents/engineering-uswds-developer.md +0 -340
  97. package/.claude/agents/engineering-video-streaming-engineer.md +0 -151
  98. package/.claude/agents/engineering-voice-ai-integration-engineer.md +0 -584
  99. package/.claude/agents/engineering-webassembly-engineer.md +0 -153
  100. package/.claude/agents/engineering-wechat-mini-program-developer.md +0 -356
  101. package/.claude/agents/engineering-wordpress-performance.md +0 -346
  102. package/.claude/agents/engineering-wordpress-shopping-cart.md +0 -346
  103. package/.claude/agents/esg-sustainability-officer.md +0 -396
  104. package/.claude/agents/finance-bookkeeper-controller.md +0 -260
  105. package/.claude/agents/finance-financial-analyst.md +0 -234
  106. package/.claude/agents/finance-fpa-analyst.md +0 -263
  107. package/.claude/agents/finance-investment-researcher.md +0 -272
  108. package/.claude/agents/finance-tax-strategist.md +0 -239
  109. package/.claude/agents/game-audio-engineer.md +0 -264
  110. package/.claude/agents/game-designer.md +0 -167
  111. package/.claude/agents/gis-3d-scene-developer.md +0 -111
  112. package/.claude/agents/gis-analyst.md +0 -91
  113. package/.claude/agents/gis-bim-specialist.md +0 -108
  114. package/.claude/agents/gis-cartography-designer.md +0 -150
  115. package/.claude/agents/gis-drone-reality-mapping.md +0 -120
  116. package/.claude/agents/gis-geoai-ml-engineer.md +0 -105
  117. package/.claude/agents/gis-geoprocessing-specialist.md +0 -97
  118. package/.claude/agents/gis-qa-engineer.md +0 -133
  119. package/.claude/agents/gis-solution-engineer.md +0 -101
  120. package/.claude/agents/gis-spatial-data-engineer.md +0 -97
  121. package/.claude/agents/gis-spatial-data-scientist.md +0 -111
  122. package/.claude/agents/gis-technical-consultant.md +0 -86
  123. package/.claude/agents/gis-web-gis-developer.md +0 -108
  124. package/.claude/agents/godot-gameplay-scripter.md +0 -338
  125. package/.claude/agents/godot-multiplayer-engineer.md +0 -301
  126. package/.claude/agents/godot-shader-developer.md +0 -267
  127. package/.claude/agents/government-digital-presales-consultant.md +0 -363
  128. package/.claude/agents/grant-writer.md +0 -511
  129. package/.claude/agents/healthcare-aging-parent-care-companion.md +0 -414
  130. package/.claude/agents/healthcare-clinical-evidence-agent.md +0 -231
  131. package/.claude/agents/healthcare-customer-service.md +0 -389
  132. package/.claude/agents/healthcare-innovation-strategist.md +0 -433
  133. package/.claude/agents/healthcare-marketing-compliance.md +0 -395
  134. package/.claude/agents/healthcare-sovereign-health-systems-agent.md +0 -312
  135. package/.claude/agents/hospitality-guest-services.md +0 -603
  136. package/.claude/agents/hr-onboarding.md +0 -451
  137. package/.claude/agents/identity-graph-operator.md +0 -303
  138. package/.claude/agents/language-translator.md +0 -264
  139. package/.claude/agents/legal-billing-time-tracking.md +0 -569
  140. package/.claude/agents/legal-client-intake.md +0 -492
  141. package/.claude/agents/legal-document-review.md +0 -454
  142. package/.claude/agents/level-designer.md +0 -208
  143. package/.claude/agents/loan-officer-assistant.md +0 -555
  144. package/.claude/agents/lsp-index-engineer.md +0 -320
  145. package/.claude/agents/ma-integration-manager.md +0 -427
  146. package/.claude/agents/macos-spatial-metal-engineer.md +0 -337
  147. package/.claude/agents/marketing-aeo-foundations.md +0 -264
  148. package/.claude/agents/marketing-agentic-search-optimizer.md +0 -279
  149. package/.claude/agents/marketing-ai-citation-strategist.md +0 -172
  150. package/.claude/agents/marketing-app-store-optimizer.md +0 -321
  151. package/.claude/agents/marketing-baidu-seo-specialist.md +0 -226
  152. package/.claude/agents/marketing-bilibili-content-strategist.md +0 -199
  153. package/.claude/agents/marketing-book-co-author.md +0 -110
  154. package/.claude/agents/marketing-carousel-growth-engine.md +0 -204
  155. package/.claude/agents/marketing-china-ecommerce-operator.md +0 -283
  156. package/.claude/agents/marketing-china-market-localization-strategist.md +0 -283
  157. package/.claude/agents/marketing-content-creator.md +0 -54
  158. package/.claude/agents/marketing-cross-border-ecommerce.md +0 -259
  159. package/.claude/agents/marketing-developer-community-builder.md +0 -285
  160. package/.claude/agents/marketing-douyin-strategist.md +0 -149
  161. package/.claude/agents/marketing-email-strategist.md +0 -249
  162. package/.claude/agents/marketing-global-podcast-strategist.md +0 -206
  163. package/.claude/agents/marketing-growth-hacker.md +0 -54
  164. package/.claude/agents/marketing-instagram-curator.md +0 -113
  165. package/.claude/agents/marketing-kuaishou-strategist.md +0 -223
  166. package/.claude/agents/marketing-linkedin-content-creator.md +0 -214
  167. package/.claude/agents/marketing-livestream-commerce-coach.md +0 -305
  168. package/.claude/agents/marketing-multi-platform-publisher.md +0 -217
  169. package/.claude/agents/marketing-podcast-strategist.md +0 -277
  170. package/.claude/agents/marketing-pr-communications-manager.md +0 -473
  171. package/.claude/agents/marketing-private-domain-operator.md +0 -308
  172. package/.claude/agents/marketing-reddit-community-builder.md +0 -123
  173. package/.claude/agents/marketing-seo-specialist.md +0 -370
  174. package/.claude/agents/marketing-short-video-editing-coach.md +0 -412
  175. package/.claude/agents/marketing-social-media-strategist.md +0 -125
  176. package/.claude/agents/marketing-tiktok-strategist.md +0 -125
  177. package/.claude/agents/marketing-twitter-engager.md +0 -126
  178. package/.claude/agents/marketing-video-optimization-specialist.md +0 -119
  179. package/.claude/agents/marketing-wechat-official-account.md +0 -145
  180. package/.claude/agents/marketing-weibo-strategist.md +0 -240
  181. package/.claude/agents/marketing-x-twitter-intelligence-analyst.md +0 -161
  182. package/.claude/agents/marketing-xiaohongshu-specialist.md +0 -138
  183. package/.claude/agents/marketing-zhihu-strategist.md +0 -162
  184. package/.claude/agents/medical-billing-coding-specialist.md +0 -491
  185. package/.claude/agents/narrative-designer.md +0 -243
  186. package/.claude/agents/operations-manager.md +0 -399
  187. package/.claude/agents/organizational-psychologist.md +0 -391
  188. package/.claude/agents/paid-media-auditor.md +0 -71
  189. package/.claude/agents/paid-media-creative-strategist.md +0 -71
  190. package/.claude/agents/paid-media-paid-social-strategist.md +0 -71
  191. package/.claude/agents/paid-media-ppc-strategist.md +0 -71
  192. package/.claude/agents/paid-media-programmatic-buyer.md +0 -71
  193. package/.claude/agents/paid-media-search-query-analyst.md +0 -71
  194. package/.claude/agents/paid-media-tracking-specialist.md +0 -71
  195. package/.claude/agents/personal-growth-mentor.md +0 -159
  196. package/.claude/agents/product-behavioral-nudge-engine.md +0 -83
  197. package/.claude/agents/product-dx-engineer.md +0 -332
  198. package/.claude/agents/product-feedback-synthesizer.md +0 -119
  199. package/.claude/agents/product-manager.md +0 -469
  200. package/.claude/agents/product-sprint-prioritizer.md +0 -154
  201. package/.claude/agents/product-trend-researcher.md +0 -159
  202. package/.claude/agents/project-management-experiment-tracker.md +0 -198
  203. package/.claude/agents/project-management-jira-workflow-steward.md +0 -233
  204. package/.claude/agents/project-management-meeting-notes-specialist.md +0 -95
  205. package/.claude/agents/project-management-project-shepherd.md +0 -194
  206. package/.claude/agents/project-management-studio-operations.md +0 -200
  207. package/.claude/agents/project-management-studio-producer.md +0 -203
  208. package/.claude/agents/project-manager-senior.md +0 -135
  209. package/.claude/agents/real-estate-buyer-seller.md +0 -596
  210. package/.claude/agents/recruitment-specialist.md +0 -515
  211. package/.claude/agents/report-distribution-agent.md +0 -65
  212. package/.claude/agents/research-synthesist.md +0 -136
  213. package/.claude/agents/resume-tailor.md +0 -230
  214. package/.claude/agents/retail-customer-returns.md +0 -566
  215. package/.claude/agents/roblox-avatar-creator.md +0 -297
  216. package/.claude/agents/roblox-experience-designer.md +0 -306
  217. package/.claude/agents/roblox-systems-scripter.md +0 -332
  218. package/.claude/agents/sales-account-strategist.md +0 -227
  219. package/.claude/agents/sales-coach.md +0 -271
  220. package/.claude/agents/sales-data-extraction-agent.md +0 -67
  221. package/.claude/agents/sales-deal-strategist.md +0 -180
  222. package/.claude/agents/sales-discovery-coach.md +0 -225
  223. package/.claude/agents/sales-engineer.md +0 -182
  224. package/.claude/agents/sales-offer-lead-gen-strategist.md +0 -257
  225. package/.claude/agents/sales-outbound-strategist.md +0 -201
  226. package/.claude/agents/sales-outreach.md +0 -425
  227. package/.claude/agents/sales-pipeline-analyst.md +0 -267
  228. package/.claude/agents/sales-proposal-strategist.md +0 -217
  229. package/.claude/agents/security-ai-generated-code-auditor.md +0 -207
  230. package/.claude/agents/security-appsec-engineer.md +0 -536
  231. package/.claude/agents/security-architect.md +0 -307
  232. package/.claude/agents/security-blockchain-security-auditor.md +0 -482
  233. package/.claude/agents/security-cloud-security-architect.md +0 -576
  234. package/.claude/agents/security-compliance-auditor.md +0 -158
  235. package/.claude/agents/security-incident-responder.md +0 -450
  236. package/.claude/agents/security-penetration-tester.md +0 -399
  237. package/.claude/agents/security-secrets-credential-engineer.md +0 -189
  238. package/.claude/agents/security-senior-secops.md +0 -760
  239. package/.claude/agents/security-threat-detection-engineer.md +0 -553
  240. package/.claude/agents/security-threat-intelligence-analyst.md +0 -661
  241. package/.claude/agents/specialized-chief-of-staff.md +0 -279
  242. package/.claude/agents/specialized-civil-engineer.md +0 -356
  243. package/.claude/agents/specialized-codebase-archaeologist.md +0 -341
  244. package/.claude/agents/specialized-cultural-intelligence-strategist.md +0 -88
  245. package/.claude/agents/specialized-developer-advocate.md +0 -317
  246. package/.claude/agents/specialized-document-generator.md +0 -55
  247. package/.claude/agents/specialized-fedramp-rmf-compliance.md +0 -378
  248. package/.claude/agents/specialized-focus-music-architect.md +0 -175
  249. package/.claude/agents/specialized-french-consulting-market.md +0 -194
  250. package/.claude/agents/specialized-korean-business-navigator.md +0 -216
  251. package/.claude/agents/specialized-master-plan-architect.md +0 -157
  252. package/.claude/agents/specialized-mcp-builder.md +0 -260
  253. package/.claude/agents/specialized-model-qa.md +0 -528
  254. package/.claude/agents/specialized-pricing-analyst.md +0 -243
  255. package/.claude/agents/specialized-salesforce-architect.md +0 -182
  256. package/.claude/agents/specialized-strategy-duel-agent.md +0 -130
  257. package/.claude/agents/specialized-workflow-architect.md +0 -597
  258. package/.claude/agents/study-abroad-advisor.md +0 -282
  259. package/.claude/agents/supply-chain-strategist.md +0 -591
  260. package/.claude/agents/support-analytics-reporter.md +0 -387
  261. package/.claude/agents/support-executive-summary-generator.md +0 -212
  262. package/.claude/agents/support-finance-tracker.md +0 -480
  263. package/.claude/agents/support-infrastructure-maintainer.md +0 -628
  264. package/.claude/agents/support-legal-compliance-checker.md +0 -591
  265. package/.claude/agents/support-support-responder.md +0 -607
  266. package/.claude/agents/technical-artist.md +0 -233
  267. package/.claude/agents/terminal-integration-specialist.md +0 -70
  268. package/.claude/agents/testing-accessibility-auditor.md +0 -327
  269. package/.claude/agents/testing-api-tester.md +0 -334
  270. package/.claude/agents/testing-evidence-collector.md +0 -210
  271. package/.claude/agents/testing-performance-benchmarker.md +0 -283
  272. package/.claude/agents/testing-reality-checker.md +0 -249
  273. package/.claude/agents/testing-test-automation-engineer.md +0 -196
  274. package/.claude/agents/testing-test-results-analyzer.md +0 -330
  275. package/.claude/agents/testing-tool-evaluator.md +0 -416
  276. package/.claude/agents/testing-workflow-optimizer.md +0 -463
  277. package/.claude/agents/unity-architect.md +0 -271
  278. package/.claude/agents/unity-editor-tool-developer.md +0 -310
  279. package/.claude/agents/unity-multiplayer-engineer.md +0 -323
  280. package/.claude/agents/unity-shader-graph-artist.md +0 -269
  281. package/.claude/agents/unreal-multiplayer-architect.md +0 -364
  282. package/.claude/agents/unreal-systems-engineer.md +0 -322
  283. package/.claude/agents/unreal-technical-artist.md +0 -256
  284. package/.claude/agents/unreal-world-builder.md +0 -273
  285. package/.claude/agents/visionos-spatial-engineer.md +0 -54
  286. package/.claude/agents/xr-cockpit-interaction-specialist.md +0 -32
  287. package/.claude/agents/xr-immersive-developer.md +0 -32
  288. package/.claude/agents/xr-interface-architect.md +0 -32
  289. package/.claude/agents/zk-steward.md +0 -211
  290. package/.claude-plugin/marketplace.json +0 -30
  291. package/.commitlintrc.json +0 -26
  292. package/.devcontainer/Dockerfile +0 -24
  293. package/.devcontainer/devcontainer.json +0 -70
  294. package/.devcontainer/docker-compose.memory.yml +0 -42
  295. package/.devcontainer/memory-stack/devcontainer.json +0 -84
  296. package/.devcontainer/post-create.sh +0 -83
  297. package/.dockerignore +0 -23
  298. package/.env.act.example +0 -15
  299. package/.env.example +0 -171
  300. package/.eslintrc.json +0 -16
  301. package/.git-blame-ignore-revs +0 -10
  302. package/.gitattributes +0 -2
  303. package/.gitguardian.yaml +0 -50
  304. package/.githooks/pre-commit +0 -6
  305. package/.pre-commit-config.yaml +0 -30
  306. package/.release-please-config-xithead.json +0 -22
  307. package/.release-please-config.json +0 -44
  308. package/.release-please-manifest-xithead.json +0 -3
  309. package/.release-please-manifest.json +0 -3
  310. package/CODE_OF_CONDUCT.md +0 -133
  311. package/CONTRIBUTING.md +0 -133
  312. package/Cargo.lock +0 -5557
  313. package/Cargo.toml +0 -121
  314. package/Dockerfile +0 -38
  315. package/Dockerfile.proxy +0 -48
  316. package/ENTERPRISE.md +0 -5
  317. package/FEATURES_ADVANCED.md +0 -311
  318. package/Headroom-2.gif +0 -0
  319. package/HeadroomDemo-Fast.gif +0 -0
  320. package/LICENSE +0 -190
  321. package/MANIFEST.in +0 -4
  322. package/MODERNIZATION_GUIDE.md +0 -358
  323. package/Makefile +0 -140
  324. package/NOTICE +0 -43
  325. package/NPM_TOKEN_SETUP.md +0 -81
  326. package/PR.md +0 -170
  327. package/PUBLISH.md +0 -77
  328. package/README.md +0 -108
  329. package/README_FRONTEND.md +0 -196
  330. package/REALIGNMENT/00-overview.md +0 -53
  331. package/REALIGNMENT/01-bug-list.md +0 -421
  332. package/REALIGNMENT/02-architecture.md +0 -322
  333. package/REALIGNMENT/03-phase-A-lockdown.md +0 -420
  334. package/REALIGNMENT/04-phase-B-live-zone.md +0 -470
  335. package/REALIGNMENT/05-phase-C-rust-proxy.md +0 -329
  336. package/REALIGNMENT/06-phase-D-bedrock-vertex.md +0 -222
  337. package/REALIGNMENT/07-phase-E-cache-stabilization.md +0 -338
  338. package/REALIGNMENT/08-phase-F-auth-mode.md +0 -254
  339. package/REALIGNMENT/09-phase-G-rtk-observability.md +0 -194
  340. package/REALIGNMENT/10-phase-H-python-retirement.md +0 -214
  341. package/REALIGNMENT/11-phase-I-test-infra.md +0 -385
  342. package/REALIGNMENT/12-decisions-needed.md +0 -196
  343. package/REALIGNMENT/INDEX.md +0 -82
  344. package/REFACTORING_CAMPAIGN.md +0 -277
  345. package/RESTRUCTURE_PLAN.md +0 -100
  346. package/RUST_DEV.md +0 -361
  347. package/SECURITY.md +0 -65
  348. package/SETUP_AUTO_PUBLISH.md +0 -164
  349. package/TESTING-copilot-subscription.md +0 -153
  350. package/app/api/config/route.ts +0 -22
  351. package/app/api/export/route.ts +0 -83
  352. package/app/api/models/route.ts +0 -12
  353. package/app/api/stats/route.ts +0 -46
  354. package/app/dashboard/page.tsx +0 -445
  355. package/app/globals.css +0 -35
  356. package/app/installation/page.tsx +0 -293
  357. package/app/layout.tsx +0 -35
  358. package/app/master-design-system.css +0 -2076
  359. package/app/master-design-system.source.css +0 -8
  360. package/app/page.tsx +0 -187
  361. package/app/rechtliches/agb/page.tsx +0 -165
  362. package/app/rechtliches/datenschutz/page.tsx +0 -128
  363. package/app/rechtliches/impressum/page.tsx +0 -122
  364. package/app/rechtliches/widerruf/page.tsx +0 -185
  365. package/codecov.yml +0 -34
  366. package/components/layout/AppShell.tsx +0 -25
  367. package/components/layout/Footer.tsx +0 -120
  368. package/components/layout/Header.tsx +0 -145
  369. package/components/marketing/LegalHighlights.tsx +0 -45
  370. package/components/marketing/LegalPageShell.tsx +0 -49
  371. package/components/marketing/LegalPanel.tsx +0 -41
  372. package/components/marketing/index.ts +0 -3
  373. package/crates/headroom-core/Cargo.toml +0 -165
  374. package/crates/headroom-core/benches/auth_mode.rs +0 -74
  375. package/crates/headroom-core/benches/ccr_store.rs +0 -224
  376. package/crates/headroom-core/benches/tokenizer.rs +0 -52
  377. package/crates/headroom-core/config/pipeline.toml +0 -151
  378. package/crates/headroom-core/proptest-regressions/transforms/tag_protector.txt +0 -9
  379. package/crates/headroom-core/src/auth_mode.rs +0 -250
  380. package/crates/headroom-core/src/cache_control.rs +0 -414
  381. package/crates/headroom-core/src/ccr/backends/in_memory.rs +0 -317
  382. package/crates/headroom-core/src/ccr/backends/mod.rs +0 -152
  383. package/crates/headroom-core/src/ccr/backends/redis.rs +0 -146
  384. package/crates/headroom-core/src/ccr/backends/sqlite.rs +0 -205
  385. package/crates/headroom-core/src/ccr/mod.rs +0 -119
  386. package/crates/headroom-core/src/compression_policy.rs +0 -505
  387. package/crates/headroom-core/src/lib.rs +0 -33
  388. package/crates/headroom-core/src/relevance/base.rs +0 -139
  389. package/crates/headroom-core/src/relevance/bm25.rs +0 -385
  390. package/crates/headroom-core/src/relevance/embedding.rs +0 -354
  391. package/crates/headroom-core/src/relevance/hybrid.rs +0 -364
  392. package/crates/headroom-core/src/relevance/mod.rs +0 -64
  393. package/crates/headroom-core/src/signals/keyword_detector.rs +0 -433
  394. package/crates/headroom-core/src/signals/line_importance.rs +0 -84
  395. package/crates/headroom-core/src/signals/mod.rs +0 -61
  396. package/crates/headroom-core/src/signals/tiered.rs +0 -141
  397. package/crates/headroom-core/src/tokenizer/estimator.rs +0 -152
  398. package/crates/headroom-core/src/tokenizer/hf_impl.rs +0 -314
  399. package/crates/headroom-core/src/tokenizer/mod.rs +0 -56
  400. package/crates/headroom-core/src/tokenizer/registry.rs +0 -342
  401. package/crates/headroom-core/src/tokenizer/tiktoken_impl.rs +0 -265
  402. package/crates/headroom-core/src/transforms/adaptive_sizer.rs +0 -610
  403. package/crates/headroom-core/src/transforms/anchor_selector.rs +0 -1189
  404. package/crates/headroom-core/src/transforms/content_detector.rs +0 -769
  405. package/crates/headroom-core/src/transforms/detection.rs +0 -222
  406. package/crates/headroom-core/src/transforms/diff_compressor.rs +0 -1685
  407. package/crates/headroom-core/src/transforms/live_zone.rs +0 -2967
  408. package/crates/headroom-core/src/transforms/log_compressor.rs +0 -1295
  409. package/crates/headroom-core/src/transforms/magika_detector.rs +0 -395
  410. package/crates/headroom-core/src/transforms/mod.rs +0 -64
  411. package/crates/headroom-core/src/transforms/pipeline/config.rs +0 -331
  412. package/crates/headroom-core/src/transforms/pipeline/mod.rs +0 -103
  413. package/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rs +0 -524
  414. package/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs +0 -279
  415. package/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs +0 -400
  416. package/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rs +0 -284
  417. package/crates/headroom-core/src/transforms/pipeline/offloads/mod.rs +0 -34
  418. package/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rs +0 -331
  419. package/crates/headroom-core/src/transforms/pipeline/orchestrator.rs +0 -848
  420. package/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rs +0 -175
  421. package/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rs +0 -545
  422. package/crates/headroom-core/src/transforms/pipeline/reformats/mod.rs +0 -12
  423. package/crates/headroom-core/src/transforms/pipeline/traits.rs +0 -338
  424. package/crates/headroom-core/src/transforms/recommendations.rs +0 -329
  425. package/crates/headroom-core/src/transforms/safety.rs +0 -215
  426. package/crates/headroom-core/src/transforms/search_compressor.rs +0 -902
  427. package/crates/headroom-core/src/transforms/smart_crusher/analyzer.rs +0 -1262
  428. package/crates/headroom-core/src/transforms/smart_crusher/anchors.rs +0 -407
  429. package/crates/headroom-core/src/transforms/smart_crusher/builder.rs +0 -263
  430. package/crates/headroom-core/src/transforms/smart_crusher/classifier.rs +0 -214
  431. package/crates/headroom-core/src/transforms/smart_crusher/compaction/classifier.rs +0 -283
  432. package/crates/headroom-core/src/transforms/smart_crusher/compaction/compactor.rs +0 -733
  433. package/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rs +0 -961
  434. package/crates/headroom-core/src/transforms/smart_crusher/compaction/ir.rs +0 -252
  435. package/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs +0 -127
  436. package/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs +0 -397
  437. package/crates/headroom-core/src/transforms/smart_crusher/config.rs +0 -173
  438. package/crates/headroom-core/src/transforms/smart_crusher/constraints.rs +0 -159
  439. package/crates/headroom-core/src/transforms/smart_crusher/crusher.rs +0 -1665
  440. package/crates/headroom-core/src/transforms/smart_crusher/crushers.rs +0 -865
  441. package/crates/headroom-core/src/transforms/smart_crusher/error_keywords.rs +0 -77
  442. package/crates/headroom-core/src/transforms/smart_crusher/field_detect.rs +0 -393
  443. package/crates/headroom-core/src/transforms/smart_crusher/hashing.rs +0 -72
  444. package/crates/headroom-core/src/transforms/smart_crusher/mod.rs +0 -81
  445. package/crates/headroom-core/src/transforms/smart_crusher/observer.rs +0 -66
  446. package/crates/headroom-core/src/transforms/smart_crusher/orchestration.rs +0 -459
  447. package/crates/headroom-core/src/transforms/smart_crusher/outliers.rs +0 -507
  448. package/crates/headroom-core/src/transforms/smart_crusher/planning.rs +0 -904
  449. package/crates/headroom-core/src/transforms/smart_crusher/statistics.rs +0 -510
  450. package/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs +0 -287
  451. package/crates/headroom-core/src/transforms/smart_crusher/traits.rs +0 -203
  452. package/crates/headroom-core/src/transforms/smart_crusher/types.rs +0 -263
  453. package/crates/headroom-core/src/transforms/tag_protector.rs +0 -1272
  454. package/crates/headroom-core/src/transforms/unidiff_detector.rs +0 -232
  455. package/crates/headroom-parity/Cargo.toml +0 -26
  456. package/crates/headroom-parity/examples/diff_fixture.rs +0 -66
  457. package/crates/headroom-parity/src/bin/parity_run.rs +0 -78
  458. package/crates/headroom-parity/src/lib.rs +0 -613
  459. package/crates/headroom-proxy/Cargo.toml +0 -119
  460. package/crates/headroom-proxy/data/model_prices_and_context_window.json +0 -39767
  461. package/crates/headroom-proxy/src/bedrock/auth_mode_layer.rs +0 -192
  462. package/crates/headroom-proxy/src/bedrock/envelope.rs +0 -239
  463. package/crates/headroom-proxy/src/bedrock/eventstream.rs +0 -769
  464. package/crates/headroom-proxy/src/bedrock/eventstream_to_sse.rs +0 -459
  465. package/crates/headroom-proxy/src/bedrock/invoke.rs +0 -735
  466. package/crates/headroom-proxy/src/bedrock/invoke_streaming.rs +0 -1098
  467. package/crates/headroom-proxy/src/bedrock/mod.rs +0 -66
  468. package/crates/headroom-proxy/src/bedrock/sigv4.rs +0 -324
  469. package/crates/headroom-proxy/src/bedrock/vendor.rs +0 -82
  470. package/crates/headroom-proxy/src/cache_stabilization/anthropic_cache_control.rs +0 -698
  471. package/crates/headroom-proxy/src/cache_stabilization/drift_detector.rs +0 -683
  472. package/crates/headroom-proxy/src/cache_stabilization/mod.rs +0 -61
  473. package/crates/headroom-proxy/src/cache_stabilization/openai_cache_key.rs +0 -600
  474. package/crates/headroom-proxy/src/cache_stabilization/tool_def_normalize.rs +0 -514
  475. package/crates/headroom-proxy/src/cache_stabilization/volatile_detector.rs +0 -720
  476. package/crates/headroom-proxy/src/compression/anthropic.rs +0 -92
  477. package/crates/headroom-proxy/src/compression/live_zone_anthropic.rs +0 -1311
  478. package/crates/headroom-proxy/src/compression/live_zone_openai.rs +0 -660
  479. package/crates/headroom-proxy/src/compression/live_zone_responses.rs +0 -678
  480. package/crates/headroom-proxy/src/compression/mod.rs +0 -127
  481. package/crates/headroom-proxy/src/compression/model_limits.rs +0 -211
  482. package/crates/headroom-proxy/src/config.rs +0 -646
  483. package/crates/headroom-proxy/src/error.rs +0 -69
  484. package/crates/headroom-proxy/src/handlers/chat_completions.rs +0 -100
  485. package/crates/headroom-proxy/src/handlers/conversations.rs +0 -241
  486. package/crates/headroom-proxy/src/handlers/mod.rs +0 -13
  487. package/crates/headroom-proxy/src/handlers/responses.rs +0 -256
  488. package/crates/headroom-proxy/src/headers.rs +0 -288
  489. package/crates/headroom-proxy/src/health.rs +0 -41
  490. package/crates/headroom-proxy/src/lib.rs +0 -21
  491. package/crates/headroom-proxy/src/main.rs +0 -141
  492. package/crates/headroom-proxy/src/observability/cache_hit_rate.rs +0 -340
  493. package/crates/headroom-proxy/src/observability/compression_ratio.rs +0 -158
  494. package/crates/headroom-proxy/src/observability/metric_names.rs +0 -174
  495. package/crates/headroom-proxy/src/observability/mod.rs +0 -68
  496. package/crates/headroom-proxy/src/observability/prometheus.rs +0 -394
  497. package/crates/headroom-proxy/src/observability/proxy_metrics.rs +0 -368
  498. package/crates/headroom-proxy/src/proxy.rs +0 -1619
  499. package/crates/headroom-proxy/src/responses_items.rs +0 -565
  500. package/crates/headroom-proxy/src/sse/anthropic.rs +0 -415
  501. package/crates/headroom-proxy/src/sse/framing.rs +0 -366
  502. package/crates/headroom-proxy/src/sse/mod.rs +0 -40
  503. package/crates/headroom-proxy/src/sse/openai_chat.rs +0 -231
  504. package/crates/headroom-proxy/src/sse/openai_responses.rs +0 -458
  505. package/crates/headroom-proxy/src/vertex/adc.rs +0 -315
  506. package/crates/headroom-proxy/src/vertex/envelope.rs +0 -167
  507. package/crates/headroom-proxy/src/vertex/mod.rs +0 -277
  508. package/crates/headroom-proxy/src/vertex/raw_predict.rs +0 -490
  509. package/crates/headroom-proxy/src/vertex/stream_raw_predict.rs +0 -49
  510. package/crates/headroom-proxy/src/websocket.rs +0 -246
  511. package/crates/headroom-py/Cargo.toml +0 -37
  512. package/crates/headroom-py/build.rs +0 -77
  513. package/crates/headroom-py/glibc_compat.c +0 -170
  514. package/crates/headroom-py/src/lib.rs +0 -1655
  515. package/deny.toml +0 -32
  516. package/design-system/base.css +0 -127
  517. package/design-system/components.css +0 -1943
  518. package/design-system/fonts.css +0 -17
  519. package/design-system/tailwind-preset.js +0 -95
  520. package/design-system/tokens.css +0 -48
  521. package/design-system/utilities.css +0 -53
  522. package/docker/differential-network-capture/Dockerfile.runner +0 -14
  523. package/docker/differential-network-capture/docker-compose.yml +0 -142
  524. package/docker/differential-network-capture/run-claude-lane.sh +0 -24
  525. package/docker/docker-compose.native.yml +0 -44
  526. package/docker-bake.hcl +0 -88
  527. package/docker-compose.yml +0 -80
  528. package/e2e/_lib/make_shim.ps1 +0 -24
  529. package/e2e/_lib/make_shim.sh +0 -28
  530. package/e2e/docker-native-install.sh +0 -120
  531. package/e2e/init/Dockerfile +0 -79
  532. package/e2e/wrap/Dockerfile +0 -115
  533. package/examples/07-context-compression.ipynb +0 -279
  534. package/examples/deployment/macos-launchagent/com.headroom.proxy.plist.template +0 -65
  535. package/examples/deployment/macos-launchagent/install.sh +0 -227
  536. package/examples/deployment/macos-launchagent/shell-integration.sh +0 -74
  537. package/examples/deployment/macos-launchagent/uninstall.sh +0 -145
  538. package/headroom-savings.png +0 -0
  539. package/headroom_learn.gif +0 -0
  540. package/lib/config/index.ts +0 -12
  541. package/lib/hooks/index.ts +0 -4
  542. package/lib/hooks/use-mounted.ts +0 -13
  543. package/lib/hooks/use-session.ts +0 -34
  544. package/lib/hooks/use-settings.ts +0 -49
  545. package/lib/utils/index.ts +0 -1
  546. package/lib/utils/settings.ts +0 -162
  547. package/lib/utils/utils.ts +0 -6
  548. package/llms.txt +0 -67
  549. package/middleware.ts +0 -26
  550. package/mkdocs.yml +0 -136
  551. package/next-env.d.ts +0 -7
  552. package/next.config.ts +0 -12
  553. package/npm/bin/xithead.js +0 -366
  554. package/npm/bin/xtream-head.js +0 -366
  555. package/npm/package.json +0 -55
  556. package/plugins/headroom-agent-hooks/.claude-plugin/plugin.json +0 -17
  557. package/plugins/headroom-agent-hooks/hooks/hooks.json +0 -29
  558. package/plugins/headroom-oauth2/LICENSE +0 -3
  559. package/plugins/headroom-oauth2/SPEC.md +0 -115
  560. package/plugins/headroom-oauth2/pyproject.toml +0 -48
  561. package/plugins/hermes/headroom_retrieve/plugin.yaml +0 -6
  562. package/plugins/openclaw/hook-shim/HOOK.md +0 -21
  563. package/plugins/openclaw/hook-shim/handler.js +0 -3
  564. package/plugins/openclaw/openclaw.plugin.json +0 -98
  565. package/plugins/openclaw/package-lock.json +0 -2692
  566. package/plugins/openclaw/package.json +0 -54
  567. package/plugins/openclaw/prepare-dist.mjs +0 -56
  568. package/plugins/openclaw/src/convert.ts +0 -418
  569. package/plugins/openclaw/src/engine.ts +0 -284
  570. package/plugins/openclaw/src/gateway-config.ts +0 -143
  571. package/plugins/openclaw/src/index.ts +0 -11
  572. package/plugins/openclaw/src/plugin/index.ts +0 -98
  573. package/plugins/openclaw/src/proxy-manager.ts +0 -477
  574. package/plugins/openclaw/src/tools/headroom-retrieve.ts +0 -76
  575. package/plugins/openclaw/test/convert.test.ts +0 -96
  576. package/plugins/openclaw/test/engine-normalization.test.ts +0 -21
  577. package/plugins/openclaw/test/engine.test.ts +0 -101
  578. package/plugins/openclaw/test/gateway-config.test.ts +0 -278
  579. package/plugins/openclaw/test/plugin-runtime-routing.test.ts +0 -141
  580. package/plugins/openclaw/test/proxy-manager.test.ts +0 -343
  581. package/plugins/openclaw/tsup.config.ts +0 -9
  582. package/postcss.config.mjs +0 -7
  583. package/pyproject.toml +0 -438
  584. package/rust-toolchain.toml +0 -16
  585. package/sdk/typescript/examples/basic-compress.ts +0 -64
  586. package/sdk/typescript/examples/ccr-retrieve.ts +0 -80
  587. package/sdk/typescript/examples/hooks-custom-compression.ts +0 -117
  588. package/sdk/typescript/examples/middleware-composition.ts +0 -59
  589. package/sdk/typescript/examples/multi-provider.ts +0 -53
  590. package/sdk/typescript/examples/openai-anthropic-adapters.ts +0 -73
  591. package/sdk/typescript/examples/shared-context-multi-agent.ts +0 -74
  592. package/sdk/typescript/examples/simulation-dry-run.ts +0 -80
  593. package/sdk/typescript/examples/streaming-chat.ts +0 -71
  594. package/sdk/typescript/examples/structured-output.ts +0 -99
  595. package/sdk/typescript/examples/tool-calling-agent.ts +0 -86
  596. package/sdk/typescript/examples/with-headroom-vercel.ts +0 -57
  597. package/sdk/typescript/package-lock.json +0 -3437
  598. package/sdk/typescript/package.json +0 -108
  599. package/sdk/typescript/src/adapters/anthropic.ts +0 -218
  600. package/sdk/typescript/src/adapters/gemini.ts +0 -83
  601. package/sdk/typescript/src/adapters/openai.ts +0 -77
  602. package/sdk/typescript/src/adapters/vercel-ai.ts +0 -131
  603. package/sdk/typescript/src/client.ts +0 -632
  604. package/sdk/typescript/src/compress.ts +0 -87
  605. package/sdk/typescript/src/errors.ts +0 -114
  606. package/sdk/typescript/src/hooks.ts +0 -113
  607. package/sdk/typescript/src/index.ts +0 -140
  608. package/sdk/typescript/src/paths.ts +0 -272
  609. package/sdk/typescript/src/shared-context.ts +0 -188
  610. package/sdk/typescript/src/simulate.ts +0 -52
  611. package/sdk/typescript/src/types/config.ts +0 -183
  612. package/sdk/typescript/src/types/models.ts +0 -314
  613. package/sdk/typescript/src/types.ts +0 -139
  614. package/sdk/typescript/src/utils/case.ts +0 -34
  615. package/sdk/typescript/src/utils/format.ts +0 -480
  616. package/sdk/typescript/src/utils/stream.ts +0 -50
  617. package/sdk/typescript/test/adapters/anthropic.test.ts +0 -217
  618. package/sdk/typescript/test/adapters/gemini.test.ts +0 -131
  619. package/sdk/typescript/test/adapters/openai.test.ts +0 -174
  620. package/sdk/typescript/test/adapters/vercel-ai.test.ts +0 -306
  621. package/sdk/typescript/test/client-expanded.test.ts +0 -333
  622. package/sdk/typescript/test/client.test.ts +0 -266
  623. package/sdk/typescript/test/compress.test.ts +0 -118
  624. package/sdk/typescript/test/config.test.ts +0 -275
  625. package/sdk/typescript/test/errors.test.ts +0 -169
  626. package/sdk/typescript/test/format-integration.test.ts +0 -252
  627. package/sdk/typescript/test/hooks.test.ts +0 -265
  628. package/sdk/typescript/test/integration.test.ts +0 -249
  629. package/sdk/typescript/test/mdx-examples.test.ts +0 -253
  630. package/sdk/typescript/test/models.test.ts +0 -318
  631. package/sdk/typescript/test/parity.test.ts +0 -337
  632. package/sdk/typescript/test/paths.test.ts +0 -447
  633. package/sdk/typescript/test/shared-context.test.ts +0 -245
  634. package/sdk/typescript/test/simulate.test.ts +0 -84
  635. package/sdk/typescript/test/types.test.ts +0 -244
  636. package/sdk/typescript/test/utils/case.test.ts +0 -129
  637. package/sdk/typescript/test/utils/detect-format.test.ts +0 -133
  638. package/sdk/typescript/test/utils/format.test.ts +0 -434
  639. package/sdk/typescript/test/vercel-ai-e2e.test.ts +0 -179
  640. package/sdk/typescript/tsup.config.ts +0 -17
  641. package/tailwind.master.config.cjs +0 -47
  642. package/types/api.ts +0 -12
  643. package/types/config.ts +0 -9
  644. package/types/index.ts +0 -2
  645. package/uv.lock +0 -6951
  646. package/xithead.egg-info/PKG-INFO +0 -22
  647. package/xithead.egg-info/SOURCES.txt +0 -16
  648. package/xithead.egg-info/dependency_links.txt +0 -1
  649. package/xithead.egg-info/entry_points.txt +0 -2
  650. package/xithead.egg-info/requires.txt +0 -10
  651. package/xithead.egg-info/top_level.txt +0 -2
  652. package/xithead.ps1 +0 -136
  653. package/xithead.sh +0 -129
  654. package/xithead_dist/pyproject.toml +0 -39
  655. package/xithead_dist/xithead.egg-info/PKG-INFO +0 -22
  656. package/xithead_dist/xithead.egg-info/SOURCES.txt +0 -16
  657. package/xithead_dist/xithead.egg-info/dependency_links.txt +0 -1
  658. package/xithead_dist/xithead.egg-info/entry_points.txt +0 -3
  659. package/xithead_dist/xithead.egg-info/requires.txt +0 -10
  660. package/xithead_dist/xithead.egg-info/top_level.txt +0 -2
  661. /package/{npm/index.d.ts → index.d.ts} +0 -0
  662. /package/{npm/postinstall.js → postinstall.js} +0 -0
@@ -1,1665 +0,0 @@
1
- //! `SmartCrusher` struct — top-level entry point for compression.
2
- //!
3
- //! Owns the `config`, `anchor_selector`, `scorer`, and `analyzer`
4
- //! singletons that every per-message call needs. Constructed once
5
- //! per process; the struct is `Send + Sync` so it can sit behind an
6
- //! `Arc` in a multi-threaded proxy.
7
- //!
8
- //! This module ports three Python entry points:
9
- //!
10
- //! - `_execute_plan` (line 3617) → `SmartCrusher::execute_plan`
11
- //! - `_crush_array` (line 2400) → `SmartCrusher::crush_array`
12
- //! - `_crush_mixed_array` (line 2914) → `SmartCrusher::crush_mixed_array`
13
- //!
14
- //! # Stubs that match Python's "everything-disabled" path
15
- //!
16
- //! Python's `_crush_array` calls into TOIN (cross-user pattern
17
- //! learning), feedback (per-tool compression hints), CCR (compress-
18
- //! cache-retrieve store), and telemetry. All four are large separate
19
- //! systems with their own state. For the like-for-like port at Stage
20
- //! 3c.1, we mirror Python's behavior **when those subsystems are
21
- //! disabled**:
22
- //!
23
- //! - **TOIN**: never produces a recommendation, never overrides
24
- //! `effective_max_items`, never injects preserve_fields/strategy/level.
25
- //! - **Feedback**: never produces hints; default `effective_max_items`.
26
- //! - **CCR**: `enabled=false`; result has `ccr_hash = None`.
27
- //! - **Telemetry**: no-op.
28
- //! - **`_compress_text_within_items`**: pass-through (returns input
29
- //! unchanged) since text compression has its own port pipeline.
30
- //! - **`summarize_dropped_items`**: empty string.
31
- //!
32
- //! Parity fixtures will be recorded with all four disabled on the
33
- //! Python side, locking byte-equal output. The TOIN/CCR/feedback
34
- //! integration ports happen later (Stage 3c.2 follow-ups).
35
-
36
- use std::sync::Arc;
37
-
38
- use serde_json::Value;
39
-
40
- use super::analyzer::SmartAnalyzer;
41
- use super::builder::SmartCrusherBuilder;
42
- use super::classifier::{classify_array, ArrayType};
43
- use super::compaction::{
44
- classify_cell, emit_opaque_ccr_marker, try_parse_json_container, CellClass, ClassifyConfig,
45
- CompactConfig, Compaction, CompactionStage,
46
- };
47
- use super::config::SmartCrusherConfig;
48
- use super::crushers::{compute_k_split, crush_number_array, crush_object, crush_string_array};
49
- use super::planning::SmartCrusherPlanner;
50
- use super::traits::{Constraint, CrushEvent, Observer};
51
- use super::types::{CompressionPlan, CompressionStrategy, CrushResult};
52
- use crate::ccr::CcrStore;
53
- use crate::relevance::RelevanceScorer;
54
- use crate::transforms::adaptive_sizer::compute_optimal_k;
55
- use crate::transforms::anchor_selector::AnchorSelector;
56
-
57
- /// Return type for `crush_array`.
58
- ///
59
- /// Two operating paths feed the same result type:
60
- ///
61
- /// - **Lossless path** — input compacted to a smaller inline form
62
- /// (e.g. CSV+schema). Nothing dropped; `compacted` is populated;
63
- /// `ccr_hash` is `None` (no retrieval needed because everything is
64
- /// already in the prompt).
65
- /// - **Lossy path** — input compressed by row-dropping. `items` holds
66
- /// the kept subset; `ccr_hash` is `Some(hash)` so the runtime can
67
- /// cache the **full original** keyed by that hash and serve it back
68
- /// to the LLM via a retrieval tool call. **No data is lost** —
69
- /// "lossy" here means "compressed view inline; full payload cached
70
- /// for tool retrieval," matching Python's CCR-Dropped semantics.
71
- ///
72
- /// The runtime (PyO3 bridge / proxy server) owns the cache; this crate
73
- /// computes the hash and emits a marker so the prompt knows where to
74
- /// look.
75
- pub struct CrushArrayResult {
76
- /// Kept items. For the lossless path this is the full original
77
- /// (nothing was dropped). For the lossy path this is the surviving
78
- /// subset; the rest is retrievable via `ccr_hash`.
79
- pub items: Vec<Value>,
80
- /// Strategy debug string. One of:
81
- /// - `"none:adaptive_at_limit"` / `"skip:<reason>"` — passthrough
82
- /// - `"lossless:table"` / `"lossless:buckets"` — lossless wins
83
- /// - `"smart_sample"` / `"top_n"` / `"cluster"` / `"time_series"` —
84
- /// lossy path with row-dropping.
85
- pub strategy_info: String,
86
- /// 12-char SHA-256 hex prefix of the **full original input**.
87
- /// Populated when the lossy path dropped rows; the runtime is
88
- /// expected to cache the original items keyed by this hash so a
89
- /// retrieval tool can serve them back. `None` when nothing was
90
- /// dropped (lossless path or below adaptive_k boundary).
91
- pub ccr_hash: Option<String>,
92
- /// Marker text inserted into the prompt to advertise the CCR
93
- /// pointer (e.g. `<<ccr:abc123def456 42_rows_offloaded>>`). Empty
94
- /// when `ccr_hash` is `None`.
95
- pub dropped_summary: String,
96
- /// Rendered bytes from the compaction stage when the **lossless
97
- /// path** won. `None` for the lossy path or when compaction wasn't
98
- /// configured.
99
- pub compacted: Option<String>,
100
- /// Top-level [`Compaction`] variant tag — `"table"`, `"buckets"`,
101
- /// `"ccr"`. Mirrors `compacted` — populated only when lossless won.
102
- pub compaction_kind: Option<&'static str>,
103
- }
104
-
105
- /// Top-level SmartCrusher.
106
- ///
107
- /// Three pluggable extensions (Stage 3c.2 PR1):
108
- /// - `scorer` — relevance scoring (`HybridScorer` by default).
109
- /// - `constraints` — must-keep predicates (`KeepErrorsConstraint` +
110
- /// `KeepStructuralOutliersConstraint` by default).
111
- /// - `observers` — decision-stream telemetry (`TracingObserver` by
112
- /// default).
113
- ///
114
- /// Compose via [`SmartCrusherBuilder`]; or call `SmartCrusher::new()`
115
- /// for the OSS default composition.
116
- pub struct SmartCrusher {
117
- pub config: SmartCrusherConfig,
118
- pub anchor_selector: AnchorSelector,
119
- pub scorer: Box<dyn RelevanceScorer + Send + Sync>,
120
- pub analyzer: SmartAnalyzer,
121
- pub constraints: Vec<Box<dyn Constraint>>,
122
- pub observers: Vec<Box<dyn Observer>>,
123
- /// Optional lossless-first compaction stage (Stage 3c.2 PR2). When
124
- /// set, `crush_array` runs compaction up front and short-circuits
125
- /// the lossy path on success. When `None` (default OSS), parity
126
- /// with the pre-PR2 lossy-only pipeline is preserved exactly.
127
- pub compaction: Option<CompactionStage>,
128
- /// Optional CCR store. When set, the lossy path stashes the **full
129
- /// original** array into the store keyed by `ccr_hash` before
130
- /// returning — the runtime can then serve dropped rows back via
131
- /// retrieval tool calls. When `None`, hashes are still emitted but
132
- /// nothing is stored (legacy / parity mode).
133
- ///
134
- /// `Arc` so callers can keep their own handle to the same store
135
- /// (e.g. the proxy server holds it for retrieval lookups while
136
- /// SmartCrusher writes through it).
137
- pub ccr_store: Option<Arc<dyn CcrStore>>,
138
- }
139
-
140
- impl SmartCrusher {
141
- /// Construct with the OSS default composition: scorer + constraints +
142
- /// observer + **lossless-first compaction stage**. Calling
143
- /// `crush_array` runs the dispatch:
144
- ///
145
- /// 1. Try the lossless compactor.
146
- /// 2. If savings ratio ≥ `config.lossless_min_savings_ratio`
147
- /// (default `0.30`), ship lossless — `compacted` populated,
148
- /// `ccr_hash = None`, nothing dropped.
149
- /// 3. Otherwise fall through to the lossy path — drop rows,
150
- /// populate `ccr_hash` with a hash of the full original so the
151
- /// runtime can cache the payload for tool retrieval.
152
- ///
153
- /// **No data is ever lost.** The lossy path moves dropped rows to
154
- /// CCR cache, not to nowhere — same semantics as Python's
155
- /// SmartCrusher with CCR enabled.
156
- pub fn new(config: SmartCrusherConfig) -> Self {
157
- // Carry the compaction heuristics from the crusher config into
158
- // the compaction stage; everything not exposed on
159
- // SmartCrusherConfig keeps its CompactConfig default.
160
- let compact_cfg = CompactConfig {
161
- core_field_fraction: config.compaction_core_field_fraction,
162
- heterogeneous_core_ratio: config.compaction_heterogeneous_core_ratio,
163
- max_flatten_inner_keys: config.compaction_max_flatten_inner_keys,
164
- min_buckets: config.compaction_min_buckets,
165
- max_buckets: config.compaction_max_buckets,
166
- ..CompactConfig::default()
167
- };
168
- SmartCrusherBuilder::new(config)
169
- .with_default_oss_setup()
170
- .with_compaction(CompactionStage::csv_schema(compact_cfg))
171
- .with_default_ccr_store()
172
- .build()
173
- }
174
-
175
- /// Construct WITHOUT the compaction stage. Pre-PR4 behavior:
176
- /// `crush_array` skips the lossless attempt and runs the lossy
177
- /// path directly (still with CCR-Dropped retrieval markers from
178
- /// PR4). Used by:
179
- ///
180
- /// - The 17 legacy parity fixtures (recorded against the
181
- /// lossy-only path; using this constructor preserves byte-equal
182
- /// coverage).
183
- /// - Callers who explicitly don't want lossless attempts (e.g.
184
- /// workloads where the compactor's overhead isn't worth the
185
- /// modest tabular wins).
186
- pub fn without_compaction(config: SmartCrusherConfig) -> Self {
187
- SmartCrusherBuilder::new(config)
188
- .with_default_oss_setup()
189
- .with_default_ccr_store()
190
- .build()
191
- }
192
-
193
- /// Construct like [`SmartCrusher::new`] but with the compaction
194
- /// stage's formatter chosen by name (`"csv-schema"`, `"json"`,
195
- /// `"markdown-kv"`). `None` for unknown names — callers own the
196
- /// fallback/error policy. `"csv-schema"` is equivalent to `new`.
197
- pub fn with_compaction_format(config: SmartCrusherConfig, format_name: &str) -> Option<Self> {
198
- let stage = CompactionStage::from_format_name(format_name)?;
199
- Some(
200
- SmartCrusherBuilder::new(config)
201
- .with_default_oss_setup()
202
- .with_compaction(stage)
203
- .with_default_ccr_store()
204
- .build(),
205
- )
206
- }
207
-
208
- /// Begin a builder chain for custom composition. The Enterprise
209
- /// entry point: swap the scorer, add business-rule constraints,
210
- /// attach an audit observer.
211
- pub fn builder(config: SmartCrusherConfig) -> SmartCrusherBuilder {
212
- SmartCrusherBuilder::new(config)
213
- }
214
-
215
- /// Construct with a custom scorer (legacy convenience). Equivalent
216
- /// to `SmartCrusher::builder(config).with_scorer(scorer).with_default_oss_setup().build()`
217
- /// minus the default scorer override; preserved for backward
218
- /// compatibility with pre-PR1 callers.
219
- pub fn with_scorer(
220
- config: SmartCrusherConfig,
221
- scorer: Box<dyn RelevanceScorer + Send + Sync>,
222
- ) -> Self {
223
- SmartCrusherBuilder::new(config)
224
- .with_scorer(scorer)
225
- .add_default_oss_constraints()
226
- .build()
227
- }
228
-
229
- /// Construct directly from owned parts. Used by
230
- /// [`SmartCrusherBuilder::build`] — not part of the public stable
231
- /// API. Prefer the builder.
232
- #[doc(hidden)]
233
- #[allow(clippy::too_many_arguments)]
234
- pub fn from_parts(
235
- config: SmartCrusherConfig,
236
- anchor_selector: AnchorSelector,
237
- scorer: Box<dyn RelevanceScorer + Send + Sync>,
238
- analyzer: SmartAnalyzer,
239
- constraints: Vec<Box<dyn Constraint>>,
240
- observers: Vec<Box<dyn Observer>>,
241
- compaction: Option<CompactionStage>,
242
- ccr_store: Option<Arc<dyn CcrStore>>,
243
- ) -> Self {
244
- SmartCrusher {
245
- config,
246
- anchor_selector,
247
- scorer,
248
- analyzer,
249
- constraints,
250
- observers,
251
- compaction,
252
- ccr_store,
253
- }
254
- }
255
-
256
- /// Handle to the CCR store, if configured. Used by the runtime
257
- /// (proxy server, PyO3 bridge) to look up originals when retrieval
258
- /// tool calls fire.
259
- pub fn ccr_store(&self) -> Option<&Arc<dyn CcrStore>> {
260
- self.ccr_store.as_ref()
261
- }
262
-
263
- fn planner(&self) -> SmartCrusherPlanner<'_> {
264
- SmartCrusherPlanner::new(
265
- &self.config,
266
- &self.anchor_selector,
267
- &*self.scorer,
268
- &self.analyzer,
269
- &self.constraints,
270
- )
271
- }
272
-
273
- /// Execute a `CompressionPlan` against `items`, returning the
274
- /// kept-items list in original-array order. Mirrors Python's
275
- /// `_execute_plan` (line 3617-3633).
276
- ///
277
- /// Schema-preserving by default: each kept item is cloned unchanged.
278
- /// No summary objects, generated fields, or wrapper metadata.
279
- ///
280
- /// When `factor_out_constants` is enabled (default off), fields the
281
- /// analyzer found constant across ALL items are stripped from each
282
- /// kept object and emitted once in a leading
283
- /// `{"_constant_fields": {...}}` sentinel — same output-shape
284
- /// convention as the `_ccr_dropped` sentinel. Stripping is
285
- /// defensive: a key is only removed from an item when its value
286
- /// equals the recorded constant, so a drifted item keeps its own
287
- /// value. The CCR store always holds the full unfactored original.
288
- pub fn execute_plan(&self, plan: &CompressionPlan, items: &[Value]) -> Vec<Value> {
289
- let mut indices = plan.keep_indices.clone();
290
- indices.sort_unstable();
291
- let mut kept: Vec<Value> = indices
292
- .into_iter()
293
- .filter(|&idx| idx < items.len())
294
- .map(|idx| items[idx].clone())
295
- .collect();
296
-
297
- if self.config.factor_out_constants && !plan.constant_fields.is_empty() && kept.len() >= 2 {
298
- let mut any_stripped = false;
299
- for item in kept.iter_mut() {
300
- if let Value::Object(map) = item {
301
- for (key, constant) in &plan.constant_fields {
302
- if map.get(key) == Some(constant) {
303
- map.remove(key);
304
- any_stripped = true;
305
- }
306
- }
307
- }
308
- }
309
- if any_stripped {
310
- let mut sentinel = serde_json::Map::new();
311
- sentinel.insert(
312
- "_constant_fields".to_string(),
313
- Value::Object(plan.constant_fields.clone().into_iter().collect()),
314
- );
315
- kept.insert(0, Value::Object(sentinel));
316
- }
317
- }
318
-
319
- kept
320
- }
321
-
322
- /// Top-level entry point. Mirrors Python `SmartCrusher.crush`
323
- /// (line 1581-1603) — used by `ContentRouter` when routing JSON
324
- /// arrays.
325
- ///
326
- /// Parses `content` as JSON, recursively processes it (compressing
327
- /// arrays at every depth via the appropriate per-type crusher),
328
- /// then re-serializes with Python-compatible formatting (`, ` and
329
- /// `: ` separators, ASCII-escaped non-ASCII).
330
- ///
331
- /// Returns a `CrushResult` with:
332
- /// - `compressed`: the re-serialized JSON.
333
- /// - `original`: the input string (unmodified).
334
- /// - `was_modified`: whether `compressed` differs from `content`'s
335
- /// trimmed form.
336
- /// - `strategy`: combined strategy info from all crushed arrays
337
- /// (or `"passthrough"`).
338
- pub fn crush(&self, content: &str, query: &str, bias: f64) -> CrushResult {
339
- let start = std::time::Instant::now();
340
- let (compressed, was_modified, info) = self.smart_crush_content(content, query, bias);
341
- let strategy = if info.is_empty() {
342
- "passthrough".to_string()
343
- } else {
344
- info
345
- };
346
-
347
- // Fire one event per top-level crush. Cheap when no observers
348
- // are configured (`for o in &[]` is a single null-pointer
349
- // check); cheap when only `TracingObserver` is configured if
350
- // the subscriber filters `debug` out (the default in
351
- // production). Custom observers — audit logs, Loop training
352
- // stream, metrics — pay whatever they pay.
353
- if !self.observers.is_empty() {
354
- let event = CrushEvent {
355
- strategy: strategy.clone(),
356
- input_bytes: content.len(),
357
- output_bytes: compressed.len(),
358
- elapsed_ns: start.elapsed().as_nanos() as u64,
359
- was_modified,
360
- };
361
- for observer in &self.observers {
362
- observer.on_event(&event);
363
- }
364
- }
365
-
366
- CrushResult {
367
- compressed,
368
- original: content.to_string(),
369
- was_modified,
370
- strategy,
371
- }
372
- }
373
-
374
- /// `SmartCrusher._smart_crush_content` (Python line 2243-2301).
375
- /// JSON-parse, recursively process, re-serialize. CCR marker
376
- /// injection is stubbed (CCR is disabled in this stage).
377
- ///
378
- /// Returns `(crushed_content, was_modified, info)`.
379
- pub fn smart_crush_content(
380
- &self,
381
- content: &str,
382
- query_context: &str,
383
- bias: f64,
384
- ) -> (String, bool, String) {
385
- // Parse — non-JSON content passes through unchanged.
386
- let Ok(parsed) = serde_json::from_str::<Value>(content) else {
387
- return (content.to_string(), false, String::new());
388
- };
389
-
390
- let (crushed, info) = self.process_value(&parsed, 0, query_context, bias);
391
-
392
- // Re-serialize with Python `safe_json_dumps` formatting:
393
- // compact `(",", ":")` separators + `ensure_ascii=False`,
394
- // preserving object-key insertion order. Matches the Python
395
- // SmartCrusher output bytes the proxy writes.
396
- let result = crate::transforms::anchor_selector::python_safe_json_dumps(&crushed);
397
- let was_modified = result != content.trim();
398
- (result, was_modified, info)
399
- }
400
-
401
- /// Maximum recursion depth for nested JSON. Mirrors Python's
402
- /// `_MAX_PROCESS_DEPTH = 50`. Beyond this, values are returned as-is.
403
- const MAX_PROCESS_DEPTH: usize = 50;
404
-
405
- /// Recursively process a value, crushing arrays where appropriate.
406
- /// Mirrors Python `_process_value` (line 2307-2398).
407
- ///
408
- /// Returns `(processed_value, info_string)`. CCR markers are
409
- /// stubbed (Python's tuple has a third element for them — Rust's
410
- /// version omits since we never produce markers in this stage).
411
- pub fn process_value(
412
- &self,
413
- value: &Value,
414
- depth: usize,
415
- query_context: &str,
416
- bias: f64,
417
- ) -> (Value, String) {
418
- if depth >= Self::MAX_PROCESS_DEPTH {
419
- return (value.clone(), String::new());
420
- }
421
-
422
- let mut info_parts: Vec<String> = Vec::new();
423
-
424
- match value {
425
- Value::Array(arr) => {
426
- let n = arr.len();
427
- if n >= self.config.min_items_to_analyze {
428
- let arr_type = classify_array(arr);
429
- match arr_type {
430
- ArrayType::DictArray => {
431
- let result = self.crush_array(arr, query_context, bias);
432
- // Lossless path won → substitute the array
433
- // with the compacted string in place. This
434
- // makes the lossless win visible to the
435
- // public `crush()` API: the output JSON
436
- // has a string where the array used to be.
437
- // The wrapping JSON structure is preserved.
438
- if let Some(rendered) = result.compacted {
439
- info_parts.push(format!(
440
- "{}({}->len={})",
441
- result.strategy_info,
442
- n,
443
- rendered.len()
444
- ));
445
- return (Value::String(rendered), info_parts.join(","));
446
- }
447
- info_parts.push(format!(
448
- "{}({}->{})",
449
- result.strategy_info,
450
- n,
451
- result.items.len()
452
- ));
453
- // Lossy path with rows dropped → append a
454
- // CCR-Dropped sentinel object as the last
455
- // element of the kept-items array. This is
456
- // the **only** place the LLM sees the
457
- // `<<ccr:HASH ...>>` pointer in the prompt.
458
- // Without this, the store has the data but
459
- // no model can ever ask for it.
460
- //
461
- // Sentinel shape: `{"_ccr_dropped":
462
- // "<<ccr:HASH N_rows_offloaded>>"}` —
463
- // preserves "array-of-objects" shape so
464
- // downstream consumers iterating with
465
- // `x.get(...)` keep working; the well-known
466
- // `_ccr_dropped` key signals metadata
467
- // unambiguously.
468
- let mut items = result.items;
469
- if !result.dropped_summary.is_empty() {
470
- let mut sentinel = serde_json::Map::new();
471
- sentinel.insert(
472
- "_ccr_dropped".to_string(),
473
- Value::String(result.dropped_summary),
474
- );
475
- items.push(Value::Object(sentinel));
476
- }
477
- return (Value::Array(items), info_parts.join(","));
478
- }
479
- ArrayType::StringArray => {
480
- let strs: Vec<&str> = arr.iter().filter_map(|v| v.as_str()).collect();
481
- let (crushed, strategy) = crush_string_array(&strs, &self.config, bias);
482
- info_parts.push(format!("{}({}->{})", strategy, n, crushed.len()));
483
- let crushed_values: Vec<Value> =
484
- crushed.into_iter().map(Value::String).collect();
485
- return (Value::Array(crushed_values), info_parts.join(","));
486
- }
487
- ArrayType::NumberArray => {
488
- let (crushed, strategy) = crush_number_array(arr, &self.config, bias);
489
- info_parts.push(format!("{}({}->{})", strategy, n, crushed.len()));
490
- return (Value::Array(crushed), info_parts.join(","));
491
- }
492
- ArrayType::MixedArray => {
493
- let (crushed, strategy) =
494
- self.crush_mixed_array(arr, query_context, bias);
495
- info_parts.push(format!("{}({}->{})", strategy, n, crushed.len()));
496
- return (Value::Array(crushed), info_parts.join(","));
497
- }
498
- // NestedArray, BoolArray, Empty → fall through
499
- // to recursive descent.
500
- _ => {}
501
- }
502
- }
503
-
504
- // Below threshold or not crushable → recurse into items.
505
- let mut processed: Vec<Value> = Vec::with_capacity(n);
506
- for item in arr {
507
- let (p_item, p_info) = self.process_value(item, depth + 1, query_context, bias);
508
- processed.push(p_item);
509
- if !p_info.is_empty() {
510
- info_parts.push(p_info);
511
- }
512
- }
513
- (Value::Array(processed), info_parts.join(","))
514
- }
515
- Value::Object(map) => {
516
- // First pass: recurse into values to compress nested arrays.
517
- let mut processed = serde_json::Map::new();
518
- for (k, v) in map {
519
- let (p_val, p_info) = self.process_value(v, depth + 1, query_context, bias);
520
- processed.insert(k.clone(), p_val);
521
- if !p_info.is_empty() {
522
- info_parts.push(p_info);
523
- }
524
- }
525
-
526
- // Second pass: if the object itself has many keys,
527
- // compress at the key level.
528
- if processed.len() >= self.config.min_items_to_analyze {
529
- let (crushed_dict, strategy) = crush_object(&processed, &self.config, bias);
530
- if strategy != "object:passthrough" {
531
- info_parts.push(strategy);
532
- return (Value::Object(crushed_dict), info_parts.join(","));
533
- }
534
- }
535
-
536
- (Value::Object(processed), info_parts.join(","))
537
- }
538
- // Strings: walker-equivalent handling. Delegates to
539
- // `process_string` which parses stringified-JSON containers
540
- // (recursing through `process_value`) and CCR-substitutes
541
- // opaque blobs (with store-write so retrieval works).
542
- Value::String(s) => self.process_string(s, depth, query_context, bias),
543
- // Other scalars — passthrough.
544
- _ => (value.clone(), String::new()),
545
- }
546
- }
547
-
548
- /// Walker-equivalent string handling. Mirrors `walker::walk_string`
549
- /// in `compaction/walker.rs` but lives on `SmartCrusher` so the
550
- /// public `crush()` path picks it up.
551
- ///
552
- /// Two cases:
553
- /// 1. **Stringified-JSON.** Strings that parse to a JSON object or
554
- /// array → recurse via `process_value`, then re-emit the result
555
- /// as a compact JSON string. The wrapping string is preserved
556
- /// (so the parent JSON shape stays a string-typed field), but
557
- /// its contents are processed end-to-end.
558
- /// 2. **Opaque blobs.** Strings classified as
559
- /// [`CellClass::Opaque`] (long base64 / HTML / long-text) →
560
- /// substitute with a `<<ccr:HASH,KIND,SIZE>>` marker. Same
561
- /// format as `compaction::walker::format_ccr_marker` so
562
- /// downstream consumers can pattern-match markers regardless
563
- /// of which path emitted them.
564
- fn process_string(
565
- &self,
566
- s: &str,
567
- depth: usize,
568
- query_context: &str,
569
- bias: f64,
570
- ) -> (Value, String) {
571
- // 1. Stringified-JSON: parse, recurse, re-render.
572
- if let Some(parsed) = try_parse_json_container(s) {
573
- let (processed, sub_info) = self.process_value(&parsed, depth + 1, query_context, bias);
574
- // If recursion produced something different, re-emit.
575
- // Special case: if the recursion returned a `Value::String`
576
- // (lossless compaction substituted the array with a
577
- // rendered CSV+schema string), use that string directly.
578
- // Re-encoding it as JSON would produce a quoted string
579
- // literal — double-encoded — which is not what callers
580
- // expect in the wrapping field.
581
- if processed != parsed {
582
- let rendered = match &processed {
583
- Value::String(rendered_str) => rendered_str.clone(),
584
- _ => serde_json::to_string(&processed).unwrap_or_else(|_| s.to_string()),
585
- };
586
- let info = if sub_info.is_empty() {
587
- "string_json".to_string()
588
- } else {
589
- format!("string_json[{sub_info}]")
590
- };
591
- return (Value::String(rendered), info);
592
- }
593
- }
594
-
595
- // 2. Opaque blob: substitute with CCR marker AND stash the
596
- // original in the store (PR8) so retrieval works. Hash + format
597
- // identical to walker.rs via the shared helper — zero drift.
598
- let cfg = ClassifyConfig::default();
599
- if let CellClass::Opaque(kind) = classify_cell(&Value::String(s.to_string()), &cfg) {
600
- let marker = emit_opaque_ccr_marker(s, &kind, self.ccr_store.as_ref());
601
- let kind_label = opaque_kind_label(&kind);
602
- return (Value::String(marker), format!("string_ccr:{kind_label}"));
603
- }
604
-
605
- // 3. Plain string — passthrough.
606
- (Value::String(s.to_string()), String::new())
607
- }
608
-
609
- /// Compress an array of dict items.
610
- ///
611
- /// Direct port of `_crush_array` (Python line 2400-2687) with the
612
- /// optional subsystems (TOIN / CCR / feedback / telemetry) wired
613
- /// in their disabled-by-default behavior. See module-level docs
614
- /// for the rationale.
615
- ///
616
- /// # Pipeline
617
- ///
618
- /// 1. Compute `item_strings` once (used as input to adaptive
619
- /// sizing and downstream relevance scoring).
620
- /// 2. `compute_optimal_k` → `adaptive_k`.
621
- /// 3. If `n <= adaptive_k`, return passthrough.
622
- /// 4. `analyzer.analyze_array(items)` → `analysis`.
623
- /// 5. If `analysis.recommended_strategy == Skip`, return passthrough
624
- /// with a `skip:<reason>` strategy string.
625
- /// 6. `planner.create_plan(analysis, items, query_context, ...)`.
626
- /// 7. `execute_plan(plan, items)` → result.
627
- /// 8. Strategy info = `analysis.recommended_strategy.as_str()`.
628
- pub fn crush_array(&self, items: &[Value], query_context: &str, bias: f64) -> CrushArrayResult {
629
- let item_strings: Vec<String> = items
630
- .iter()
631
- .map(|i| serde_json::to_string(i).unwrap_or_default())
632
- .collect();
633
- let item_str_refs: Vec<&str> = item_strings.iter().map(|s| s.as_str()).collect();
634
-
635
- let max_k = if self.config.max_items_after_crush > 0 {
636
- Some(self.config.max_items_after_crush)
637
- } else {
638
- None
639
- };
640
- let adaptive_k = compute_optimal_k(&item_str_refs, bias, 3, max_k);
641
-
642
- // Tier-1 boundary: array already small enough — passthrough,
643
- // nothing to compact, nothing to drop.
644
- if items.len() <= adaptive_k {
645
- return CrushArrayResult {
646
- items: items.to_vec(),
647
- strategy_info: "none:adaptive_at_limit".to_string(),
648
- ccr_hash: None,
649
- dropped_summary: String::new(),
650
- compacted: None,
651
- compaction_kind: None,
652
- };
653
- }
654
-
655
- // ── Lossless-first attempt ──
656
- //
657
- // Run the compaction stage if present, then check the savings
658
- // ratio against `config.lossless_min_savings_ratio`. If the
659
- // lossless rendering shrinks the input by at least that much,
660
- // ship it — nothing dropped, no CCR retrieval needed.
661
- // Otherwise fall through to the lossy path.
662
- if let Some(stage) = &self.compaction {
663
- let (c, rendered) = stage.run(items);
664
- if c.was_compacted() {
665
- let input_bytes = estimate_array_bytes(&item_strings);
666
- let savings_ratio = if input_bytes > 0 {
667
- 1.0 - (rendered.len() as f64 / input_bytes as f64)
668
- } else {
669
- 0.0
670
- };
671
- if savings_ratio >= self.config.lossless_min_savings_ratio {
672
- let kind = compaction_kind_str(&c);
673
- return CrushArrayResult {
674
- items: items.to_vec(), // nothing dropped
675
- strategy_info: format!("lossless:{kind}"),
676
- ccr_hash: None,
677
- dropped_summary: String::new(),
678
- compacted: Some(rendered),
679
- compaction_kind: Some(kind),
680
- };
681
- }
682
- }
683
- }
684
-
685
- // ── Lossy path: compress inline + cache full original via CCR ──
686
- //
687
- // The runtime caller (PyO3 bridge / proxy server) is expected
688
- // to stash the full input keyed by `ccr_hash` so a retrieval
689
- // tool can serve dropped rows back to the LLM on demand.
690
- // **No data is lost** — "lossy" here means "compressed view
691
- // inline; full payload retrievable via CCR cache."
692
-
693
- let effective_max_items = adaptive_k;
694
- let analysis = self.analyzer.analyze_array(items);
695
-
696
- // Crushability gate: not safe to crush → passthrough, no CCR.
697
- if analysis.recommended_strategy == CompressionStrategy::Skip {
698
- let reason = match &analysis.crushability {
699
- Some(c) => format!("skip:{}", c.reason),
700
- None => String::new(),
701
- };
702
- return CrushArrayResult {
703
- items: items.to_vec(),
704
- strategy_info: reason,
705
- ccr_hash: None,
706
- dropped_summary: String::new(),
707
- compacted: None,
708
- compaction_kind: None,
709
- };
710
- }
711
-
712
- let plan = self.planner().create_plan(
713
- &analysis,
714
- items,
715
- query_context,
716
- None, // preserve_fields (TOIN — stubbed)
717
- Some(effective_max_items),
718
- Some(&item_strings),
719
- );
720
- let result = self.execute_plan(&plan, items);
721
-
722
- // Emit CCR-Dropped marker iff rows were actually dropped AND
723
- // the marker gate is on. **The marker is the cornerstone of
724
- // CCR's no-data-loss guarantee:** we hash the full original,
725
- // stash it in the configured store, and emit a marker pointing
726
- // at that hash. The runtime later serves the original back via
727
- // retrieval tool calls.
728
- //
729
- // When `enable_ccr_marker` is false (Python shim's path for
730
- // `ccr_config.enabled=False` or `inject_retrieval_marker=False`)
731
- // we keep the row drops (compression is still requested) but
732
- // skip the marker text and the store write — there's no point
733
- // storing a payload that nothing in the prompt can reference.
734
- let dropped_count = items.len().saturating_sub(result.len());
735
- let (ccr_hash, dropped_summary) = if dropped_count > 0 && self.config.enable_ccr_marker {
736
- // Serialize the original array exactly ONCE. The hash is
737
- // taken over those bytes, and (if a store is configured) the
738
- // same bytes get stored — eliminating a redundant tree clone
739
- // (`items.to_vec()`) and a redundant `serde_json::to_string`
740
- // pass that the previous version did per dropped array.
741
- let canonical = canonical_array_json(items);
742
- let h = hash_canonical(&canonical);
743
- let marker = format!("<<ccr:{h} {dropped_count}_rows_offloaded>>");
744
- if let Some(store) = &self.ccr_store {
745
- store.put(&h, &canonical);
746
- }
747
- (Some(h), marker)
748
- } else {
749
- (None, String::new())
750
- };
751
-
752
- CrushArrayResult {
753
- items: result,
754
- strategy_info: analysis.recommended_strategy.as_str().to_string(),
755
- ccr_hash,
756
- dropped_summary,
757
- compacted: None,
758
- compaction_kind: None,
759
- }
760
- }
761
-
762
- /// Compress a mixed-type array by grouping items by type and
763
- /// compressing each group with the appropriate handler.
764
- ///
765
- /// Direct port of `_crush_mixed_array` (Python line 2914-3013).
766
- ///
767
- /// Strategy:
768
- /// 1. Group by type (dict / str / number / list / null / bool / other).
769
- /// 2. For groups with >= `min_items_to_analyze` items: apply the
770
- /// type-specific compressor.
771
- /// 3. For small groups: keep all items.
772
- /// 4. Reassemble in original order.
773
- ///
774
- /// Returns `(crushed_items, strategy_string)`.
775
- pub fn crush_mixed_array(
776
- &self,
777
- items: &[Value],
778
- query_context: &str,
779
- bias: f64,
780
- ) -> (Vec<Value>, String) {
781
- let n = items.len();
782
- if n <= 8 {
783
- return (items.to_vec(), "mixed:passthrough".to_string());
784
- }
785
-
786
- // Group by type, tracking original indices.
787
- let mut groups: GroupBuckets = GroupBuckets::default();
788
- for (i, item) in items.iter().enumerate() {
789
- groups.push(group_key(item), i, item.clone());
790
- }
791
-
792
- let mut keep_indices: std::collections::BTreeSet<usize> = std::collections::BTreeSet::new();
793
- let mut strategy_parts: Vec<String> = Vec::new();
794
-
795
- for (type_key, indices, values) in groups.into_iter() {
796
- // Small groups: keep all items.
797
- if values.len() < self.config.min_items_to_analyze {
798
- keep_indices.extend(&indices);
799
- continue;
800
- }
801
-
802
- match type_key {
803
- "dict" => {
804
- let CrushArrayResult { items: crushed, .. } =
805
- self.crush_array(&values, query_context, bias);
806
- // Find which original indices survived by matching
807
- // canonical-JSON serialization. Mirrors Python's
808
- // `json.dumps(c, sort_keys=True, default=str)`-keyed
809
- // set match.
810
- let crushed_keys: std::collections::HashSet<String> =
811
- crushed.iter().map(canonical_json_for_match).collect();
812
- for (i, idx) in indices.iter().enumerate() {
813
- if crushed_keys.contains(&canonical_json_for_match(&values[i])) {
814
- keep_indices.insert(*idx);
815
- }
816
- }
817
- strategy_parts.push(format!("dict:{}->{}", values.len(), crushed.len()));
818
- }
819
- "str" => {
820
- let strs: Vec<&str> = values.iter().filter_map(|v| v.as_str()).collect();
821
- let (crushed, _) = crush_string_array(&strs, &self.config, bias);
822
- let crushed_set: std::collections::HashSet<&str> =
823
- crushed.iter().map(|s| s.as_str()).collect();
824
- for (i, idx) in indices.iter().enumerate() {
825
- if let Some(s) = values[i].as_str() {
826
- if crushed_set.contains(s) {
827
- keep_indices.insert(*idx);
828
- }
829
- }
830
- }
831
- strategy_parts.push(format!("str:{}->{}", values.len(), crushed.len()));
832
- }
833
- "number" => {
834
- // Python: just adaptive sampling + outlier detection
835
- // (no summary prefix). Keeps first/last by index
836
- // and items >variance_threshold σ from mean.
837
- let item_strings: Vec<String> = values.iter().map(|v| v.to_string()).collect();
838
- let item_refs: Vec<&str> = item_strings.iter().map(|s| s.as_str()).collect();
839
- let (_kt, kf, kl, _) = compute_k_split(&item_refs, &self.config, bias);
840
-
841
- let kf = kf.min(values.len());
842
- let kl = kl.min(values.len().saturating_sub(kf));
843
- let first_idx: Vec<usize> = indices.iter().take(kf).copied().collect();
844
- let last_idx: Vec<usize> =
845
- indices.iter().rev().take(kl).copied().collect::<Vec<_>>();
846
- keep_indices.extend(&first_idx);
847
- keep_indices.extend(&last_idx);
848
-
849
- // Outliers via finite-only stats.
850
- let finite: Vec<f64> = values
851
- .iter()
852
- .filter_map(|v| v.as_f64().filter(|f| f.is_finite()))
853
- .collect();
854
- if finite.len() > 1 {
855
- if let Some(mean_v) = super::stats_math::mean(&finite) {
856
- if let Some(std_v) = super::stats_math::sample_stdev(&finite) {
857
- if std_v > 0.0 {
858
- let threshold = self.config.variance_threshold * std_v;
859
- for (i, val) in values.iter().enumerate() {
860
- if let Some(num) = val.as_f64().filter(|f| f.is_finite()) {
861
- if (num - mean_v).abs() > threshold {
862
- keep_indices.insert(indices[i]);
863
- }
864
- }
865
- }
866
- }
867
- }
868
- }
869
- }
870
- strategy_parts.push(format!("num:{}", values.len()));
871
- }
872
- _ => {
873
- // list / bool / none / other → keep all items.
874
- keep_indices.extend(&indices);
875
- }
876
- }
877
- }
878
-
879
- // Reassemble in original order.
880
- let result: Vec<Value> = keep_indices.iter().map(|&i| items[i].clone()).collect();
881
- let strategy = format!(
882
- "mixed:adaptive({}->{},{})",
883
- n,
884
- result.len(),
885
- strategy_parts.join(",")
886
- );
887
- (result, strategy)
888
- }
889
- }
890
-
891
- // ---------- helpers ----------
892
-
893
- /// Group key that mirrors Python's `_crush_mixed_array` switch on
894
- /// `isinstance`. Note the bool-before-number ordering: in Python, bool
895
- /// is a subclass of int, but JSON treats them as distinct types, so we
896
- /// don't have the Python ordering hazard.
897
- fn group_key(item: &Value) -> &'static str {
898
- match item {
899
- Value::Object(_) => "dict",
900
- Value::String(_) => "str",
901
- Value::Bool(_) => "bool",
902
- Value::Number(_) => "number",
903
- Value::Array(_) => "list",
904
- Value::Null => "none",
905
- }
906
- }
907
-
908
- /// Group buckets keyed by the type-string. Preserves first-occurrence
909
- /// order across keys so dict/str/number/list/none/bool always come out
910
- /// in the same order — matters because `keep_indices` is built
911
- /// incrementally and Python iterates `groups.items()` (insertion order
912
- /// in 3.7+).
913
- #[derive(Default)]
914
- struct GroupBuckets {
915
- entries: Vec<(&'static str, Vec<usize>, Vec<Value>)>,
916
- index_of: std::collections::HashMap<&'static str, usize>,
917
- }
918
-
919
- impl GroupBuckets {
920
- fn push(&mut self, key: &'static str, idx: usize, value: Value) {
921
- match self.index_of.get(key).copied() {
922
- Some(i) => {
923
- self.entries[i].1.push(idx);
924
- self.entries[i].2.push(value);
925
- }
926
- None => {
927
- self.index_of.insert(key, self.entries.len());
928
- self.entries.push((key, vec![idx], vec![value]));
929
- }
930
- }
931
- }
932
- }
933
-
934
- impl IntoIterator for GroupBuckets {
935
- type Item = (&'static str, Vec<usize>, Vec<Value>);
936
- type IntoIter = std::vec::IntoIter<Self::Item>;
937
- fn into_iter(self) -> Self::IntoIter {
938
- self.entries.into_iter()
939
- }
940
- }
941
-
942
- /// Serialize a `Value` for membership comparison. Mirrors Python's
943
- /// `json.dumps(c, sort_keys=True, default=str)` used by
944
- /// `_crush_mixed_array` to match crushed dict items back to their
945
- /// original indices. The `default=str` fallback only matters for
946
- /// non-JSON-serializable Python values; in serde_json land everything
947
- /// is already JSON-native, so plain canonical JSON suffices.
948
- fn canonical_json_for_match(value: &Value) -> String {
949
- crate::transforms::anchor_selector::python_json_dumps_sort_keys(value)
950
- }
951
-
952
- /// Maps a `Compaction` to a stable kind tag exposed via `CrushArrayResult`.
953
- fn compaction_kind_str(c: &Compaction) -> &'static str {
954
- match c {
955
- Compaction::Table { .. } => "table",
956
- Compaction::Buckets { .. } => "buckets",
957
- Compaction::OpaqueRef { .. } => "ccr",
958
- Compaction::Untouched(_) => "untouched",
959
- }
960
- }
961
-
962
- /// Approximate byte size of `[v0, v1, ...]` JSON serialization, given
963
- /// each item's already-serialized form. Adds 2 for outer brackets and
964
- /// 1 per inter-item comma. Used by the lossless savings-ratio check.
965
- fn estimate_array_bytes(item_strings: &[String]) -> usize {
966
- let payload: usize = item_strings.iter().map(|s| s.len()).sum();
967
- let separators = item_strings.len().saturating_sub(1);
968
- payload + separators + 2
969
- }
970
-
971
- /// Serialize `[v0, v1, ...]` once into the canonical JSON form used by
972
- /// the CCR retrieval contract. `serde_json` writes a slice of `Value` as
973
- /// the same bytes it would write for `Value::Array(items.to_vec())`, so
974
- /// we skip the array-wrapper allocation and the deep tree clone it
975
- /// requires. Used by both the hash (input) and the store payload (write).
976
- fn canonical_array_json(items: &[Value]) -> String {
977
- serde_json::to_string(items).unwrap_or_default()
978
- }
979
-
980
- /// 12-char SHA-256 hex prefix of an already-serialized canonical JSON
981
- /// string. Caller is responsible for producing the canonical form via
982
- /// [`canonical_array_json`] (or another byte-equal serializer) — the
983
- /// hash is over the bytes, so a stable serializer is the contract.
984
- fn hash_canonical(canonical: &str) -> String {
985
- use sha2::{Digest, Sha256};
986
- let mut h = Sha256::new();
987
- h.update(canonical.as_bytes());
988
- h.finalize()
989
- .iter()
990
- .take(6)
991
- .map(|b| format!("{b:02x}"))
992
- .collect()
993
- }
994
-
995
- // `hash_array_for_ccr` (a test-only `canonical_array_json + hash_canonical`
996
- // convenience) lived here previously but had no callers — clippy flagged
997
- // it as dead code. Reintroduce as a test fixture if a future test wants
998
- // the one-liner; production callsites inline both steps so the canonical
999
- // bytes can be reused for the store payload.
1000
-
1001
- // ─── PR5 walker-integration helpers (string handling) ──────────────────────
1002
- //
1003
- // Parse-as-JSON-container, marker formatting, and humanize-bytes used to
1004
- // live here as locals. PR8 extracted them into `compaction::walker` so
1005
- // `walker.rs` and `process_value` share one canonical implementation —
1006
- // killing the drift risk where the two paths could format markers
1007
- // differently. `process_string` now calls `try_parse_json_container` and
1008
- // `emit_opaque_ccr_marker` directly. Only `opaque_kind_label` survives
1009
- // here because `process_string`'s `string_ccr:<kind>` strategy-info
1010
- // label is local to this module's debug-string convention.
1011
-
1012
- fn opaque_kind_label(kind: &super::compaction::OpaqueKind) -> &str {
1013
- use super::compaction::OpaqueKind;
1014
- match kind {
1015
- OpaqueKind::Base64Blob => "base64",
1016
- OpaqueKind::LongString => "string",
1017
- OpaqueKind::HtmlChunk => "html",
1018
- OpaqueKind::Other(s) => s.as_str(),
1019
- }
1020
- }
1021
-
1022
- #[cfg(test)]
1023
- mod tests {
1024
- use super::*;
1025
- use serde_json::json;
1026
-
1027
- fn crusher() -> SmartCrusher {
1028
- SmartCrusher::new(SmartCrusherConfig::default())
1029
- }
1030
-
1031
- // ---------- execute_plan ----------
1032
-
1033
- #[test]
1034
- fn execute_plan_empty_indices_returns_empty() {
1035
- let c = crusher();
1036
- let plan = CompressionPlan::default();
1037
- let items: Vec<Value> = (0..5).map(|i| json!({"id": i})).collect();
1038
- let result = c.execute_plan(&plan, &items);
1039
- assert!(result.is_empty());
1040
- }
1041
-
1042
- #[test]
1043
- fn execute_plan_returns_items_in_sorted_index_order() {
1044
- let c = crusher();
1045
- let items: Vec<Value> = (0..10).map(|i| json!({"id": i})).collect();
1046
- let plan = CompressionPlan {
1047
- keep_indices: vec![5, 2, 8, 0],
1048
- ..CompressionPlan::default()
1049
- };
1050
- let result = c.execute_plan(&plan, &items);
1051
- assert_eq!(result.len(), 4);
1052
- assert_eq!(result[0]["id"], 0);
1053
- assert_eq!(result[1]["id"], 2);
1054
- assert_eq!(result[2]["id"], 5);
1055
- assert_eq!(result[3]["id"], 8);
1056
- }
1057
-
1058
- #[test]
1059
- fn execute_plan_skips_out_of_bounds() {
1060
- let c = crusher();
1061
- let items: Vec<Value> = (0..3).map(|i| json!({"id": i})).collect();
1062
- let plan = CompressionPlan {
1063
- keep_indices: vec![0, 5, 2],
1064
- ..CompressionPlan::default()
1065
- };
1066
- let result = c.execute_plan(&plan, &items);
1067
- assert_eq!(result.len(), 2);
1068
- }
1069
-
1070
- #[test]
1071
- fn execute_plan_factors_constants_when_enabled() {
1072
- let cfg = SmartCrusherConfig {
1073
- factor_out_constants: true,
1074
- ..Default::default()
1075
- };
1076
- let c = SmartCrusher::new(cfg);
1077
- let items: Vec<Value> = (0..4)
1078
- .map(|i| json!({"id": i, "region": "us-west-2", "status": "ok"}))
1079
- .collect();
1080
- let mut constant_fields = std::collections::BTreeMap::new();
1081
- constant_fields.insert("region".to_string(), json!("us-west-2"));
1082
- constant_fields.insert("status".to_string(), json!("ok"));
1083
- let plan = CompressionPlan {
1084
- keep_indices: vec![0, 1, 2],
1085
- constant_fields,
1086
- ..CompressionPlan::default()
1087
- };
1088
- let result = c.execute_plan(&plan, &items);
1089
- // Sentinel first, then 3 slim items.
1090
- assert_eq!(result.len(), 4);
1091
- assert_eq!(result[0]["_constant_fields"]["region"], "us-west-2");
1092
- assert_eq!(result[0]["_constant_fields"]["status"], "ok");
1093
- for item in &result[1..] {
1094
- assert!(item.get("region").is_none());
1095
- assert!(item.get("status").is_none());
1096
- assert!(item.get("id").is_some());
1097
- }
1098
- }
1099
-
1100
- #[test]
1101
- fn execute_plan_keeps_drifted_values_when_factoring() {
1102
- // Defensive strip: an item whose value differs from the recorded
1103
- // constant keeps its own value.
1104
- let cfg = SmartCrusherConfig {
1105
- factor_out_constants: true,
1106
- ..Default::default()
1107
- };
1108
- let c = SmartCrusher::new(cfg);
1109
- let items = vec![
1110
- json!({"id": 0, "status": "ok"}),
1111
- json!({"id": 1, "status": "FAILED"}),
1112
- ];
1113
- let mut constant_fields = std::collections::BTreeMap::new();
1114
- constant_fields.insert("status".to_string(), json!("ok"));
1115
- let plan = CompressionPlan {
1116
- keep_indices: vec![0, 1],
1117
- constant_fields,
1118
- ..CompressionPlan::default()
1119
- };
1120
- let result = c.execute_plan(&plan, &items);
1121
- assert_eq!(result.len(), 3);
1122
- assert!(result[1].get("status").is_none()); // matched → stripped
1123
- assert_eq!(result[2]["status"], "FAILED"); // drifted → kept
1124
- }
1125
-
1126
- #[test]
1127
- fn execute_plan_default_off_leaves_items_unchanged() {
1128
- // factor_out_constants defaults to false: schema preserved even
1129
- // when the plan carries constant_fields.
1130
- let c = crusher();
1131
- let items: Vec<Value> = (0..3).map(|i| json!({"id": i, "k": "v"})).collect();
1132
- let mut constant_fields = std::collections::BTreeMap::new();
1133
- constant_fields.insert("k".to_string(), json!("v"));
1134
- let plan = CompressionPlan {
1135
- keep_indices: vec![0, 1, 2],
1136
- constant_fields,
1137
- ..CompressionPlan::default()
1138
- };
1139
- let result = c.execute_plan(&plan, &items);
1140
- assert_eq!(result.len(), 3);
1141
- assert_eq!(result[0]["k"], "v");
1142
- }
1143
-
1144
- // ---------- crush_array ----------
1145
-
1146
- #[test]
1147
- fn crush_array_passthrough_when_below_adaptive_k() {
1148
- let c = crusher();
1149
- let items: Vec<Value> = (0..3).map(|i| json!({"id": i})).collect();
1150
- let result = c.crush_array(&items, "", 1.0);
1151
- assert_eq!(result.items.len(), 3);
1152
- assert_eq!(result.strategy_info, "none:adaptive_at_limit");
1153
- assert!(result.ccr_hash.is_none());
1154
- }
1155
-
1156
- #[test]
1157
- fn crush_array_skip_path_returns_original_items() {
1158
- // 30 unique dict items with ID-like fields → analyzer should
1159
- // detect "unique_entities_no_signal" and SKIP. Use the
1160
- // no-compaction constructor so we exercise the lossy/skip
1161
- // gate; the lossless path would otherwise short-circuit
1162
- // because uniform-tabular input is the lossless sweet spot.
1163
- let c = SmartCrusher::without_compaction(SmartCrusherConfig::default());
1164
- let items: Vec<Value> = (0..30)
1165
- .map(|i| json!({"id": i, "name": format!("user_{}", i)}))
1166
- .collect();
1167
- let result = c.crush_array(&items, "", 1.0);
1168
- // skip path returns the original items unchanged.
1169
- assert_eq!(result.items.len(), 30);
1170
- assert!(
1171
- result.strategy_info.starts_with("skip:"),
1172
- "expected skip:..., got {}",
1173
- result.strategy_info
1174
- );
1175
- }
1176
-
1177
- #[test]
1178
- fn crush_array_low_uniqueness_compresses() {
1179
- // 30 items with status=ok across all → low_uniqueness path
1180
- // (crushable, smart_sample strategy).
1181
- let c = crusher();
1182
- let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();
1183
- let result = c.crush_array(&items, "", 1.0);
1184
- assert!(result.items.len() <= 30, "should not exceed original count");
1185
- }
1186
-
1187
- #[test]
1188
- fn crush_array_keeps_error_items() {
1189
- let c = crusher();
1190
- let mut items: Vec<Value> = (0..30).map(|i| json!({"id": i, "status": "ok"})).collect();
1191
- items.push(json!({"id": 30, "status": "error", "msg": "FATAL"}));
1192
- let result = c.crush_array(&items, "", 1.0);
1193
- // Whatever path is taken, the error item should survive.
1194
- assert!(
1195
- result
1196
- .items
1197
- .iter()
1198
- .any(|item| { item.get("status").and_then(|v| v.as_str()) == Some("error") }),
1199
- "error item must survive crush_array"
1200
- );
1201
- }
1202
-
1203
- // ---------- crush_mixed_array ----------
1204
-
1205
- #[test]
1206
- fn crush_mixed_passthrough_at_threshold() {
1207
- let c = crusher();
1208
- let items: Vec<Value> = vec![
1209
- json!(1),
1210
- json!("two"),
1211
- json!({"k": "v"}),
1212
- json!([1, 2]),
1213
- json!(null),
1214
- json!(true),
1215
- json!(3),
1216
- json!("four"),
1217
- ];
1218
- let (result, strat) = c.crush_mixed_array(&items, "", 1.0);
1219
- assert_eq!(result.len(), 8);
1220
- assert_eq!(strat, "mixed:passthrough");
1221
- }
1222
-
1223
- #[test]
1224
- fn crush_mixed_groups_and_compresses_dicts() {
1225
- let c = crusher();
1226
- // 25 dicts (large group → gets crushed) + 5 strings (small group → all kept).
1227
- let mut items: Vec<Value> = (0..25).map(|i| json!({"id": i, "status": "ok"})).collect();
1228
- for i in 0..5 {
1229
- items.push(json!(format!("string_{}", i)));
1230
- }
1231
- let (result, strat) = c.crush_mixed_array(&items, "", 1.0);
1232
- assert!(strat.starts_with("mixed:adaptive("));
1233
- // The 5 strings (small group) all survive.
1234
- let str_count = result.iter().filter(|v| v.is_string()).count();
1235
- assert_eq!(str_count, 5);
1236
- }
1237
-
1238
- #[test]
1239
- fn crush_mixed_keeps_lists_and_nulls_unchanged() {
1240
- let c = crusher();
1241
- let mut items: Vec<Value> = vec![json!([1, 2]); 6];
1242
- items.extend(vec![json!(null); 6]);
1243
- items.extend(vec![json!({"k": 1}); 10]);
1244
- let (result, _strat) = c.crush_mixed_array(&items, "", 1.0);
1245
- // Lists and nulls (not dict/str/number) → fall through to "keep all".
1246
- let list_count = result.iter().filter(|v| v.is_array()).count();
1247
- let null_count = result.iter().filter(|v| v.is_null()).count();
1248
- assert_eq!(list_count, 6);
1249
- assert_eq!(null_count, 6);
1250
- }
1251
-
1252
- #[test]
1253
- fn crusher_construction_default() {
1254
- let c = SmartCrusher::new(SmartCrusherConfig::default());
1255
- assert_eq!(c.config.max_items_after_crush, 15);
1256
- }
1257
-
1258
- // ---------- top-level crush ----------
1259
-
1260
- #[test]
1261
- fn crush_non_json_passes_through_unchanged() {
1262
- let c = crusher();
1263
- let result = c.crush("not json at all", "", 1.0);
1264
- assert!(!result.was_modified);
1265
- assert_eq!(result.compressed, "not json at all");
1266
- assert_eq!(result.strategy, "passthrough");
1267
- }
1268
-
1269
- #[test]
1270
- fn crush_scalar_json_passes_through() {
1271
- let c = crusher();
1272
- let result = c.crush("42", "", 1.0);
1273
- // A scalar is not crushable; should round-trip unchanged.
1274
- assert_eq!(result.compressed, "42");
1275
- assert!(!result.was_modified);
1276
- }
1277
-
1278
- #[test]
1279
- fn crush_small_array_passes_through() {
1280
- let c = crusher();
1281
- // Compact-form input matches the compact serializer output, so
1282
- // the array is not "modified" even though it round-trips
1283
- // through parse → serialize. (The spaced form `[1, 2, 3]`
1284
- // would mark `was_modified=true` because the compact
1285
- // serializer rewrites it to `[1,2,3]`.)
1286
- let result = c.crush(r#"[1,2,3]"#, "", 1.0);
1287
- // Below min_items_to_analyze=5 → no crushing of the structure.
1288
- assert!(!result.was_modified);
1289
- assert_eq!(result.compressed, "[1,2,3]");
1290
- }
1291
-
1292
- #[test]
1293
- fn crush_dict_array_crushes_when_low_uniqueness() {
1294
- // The public `crush()` API serializes back to JSON; the
1295
- // lossless-path output (a compacted string) is exposed via
1296
- // `crush_array().compacted` rather than being substituted into
1297
- // the JSON re-serialization. So we exercise the lossy path
1298
- // here via `without_compaction()` to validate the original
1299
- // intent: low-uniqueness dicts compress via row-dropping.
1300
- let c = SmartCrusher::without_compaction(SmartCrusherConfig::default());
1301
- let mut input = String::from("[");
1302
- for i in 0..30 {
1303
- if i > 0 {
1304
- input.push(',');
1305
- }
1306
- input.push_str(r#"{"status":"ok"}"#);
1307
- }
1308
- input.push(']');
1309
- let result = c.crush(&input, "", 1.0);
1310
- assert!(
1311
- result.was_modified,
1312
- "30 identical dicts should compress (low_uniqueness_safe_to_sample)"
1313
- );
1314
- assert_ne!(result.strategy, "passthrough");
1315
- }
1316
-
1317
- #[test]
1318
- fn crush_serializes_with_python_safe_format() {
1319
- let c = crusher();
1320
- // SmartCrusher uses Python's `safe_json_dumps`: compact
1321
- // separators `(",", ":")` + `ensure_ascii=False`, preserving
1322
- // object-key insertion order. A spaced input round-trips to
1323
- // the compact form.
1324
- let input = r#"{"a": 1, "b": 2, "c": 3}"#;
1325
- let result = c.crush(input, "", 1.0);
1326
- assert_eq!(
1327
- result.compressed, r#"{"a":1,"b":2,"c":3}"#,
1328
- "safe_json_dumps emits compact `,` / `:` separators"
1329
- );
1330
- }
1331
-
1332
- #[test]
1333
- fn crush_recurses_into_nested_arrays() {
1334
- let c = crusher();
1335
- // Top-level dict with a nested array of 30 identical items.
1336
- // The inner array should compress (low_uniqueness path).
1337
- let mut inner = String::from("[");
1338
- for i in 0..30 {
1339
- if i > 0 {
1340
- inner.push(',');
1341
- }
1342
- inner.push_str(r#"{"status":"ok"}"#);
1343
- }
1344
- inner.push(']');
1345
- let input = format!(r#"{{"data": {}}}"#, inner);
1346
- let result = c.crush(&input, "", 1.0);
1347
- assert!(
1348
- result.was_modified,
1349
- "nested compressible array must be crushed even inside a wrapper object"
1350
- );
1351
- }
1352
-
1353
- #[test]
1354
- fn crusher_with_custom_scorer() {
1355
- use crate::relevance::BM25Scorer;
1356
- let c = SmartCrusher::with_scorer(
1357
- SmartCrusherConfig::default(),
1358
- Box::new(BM25Scorer::default()),
1359
- );
1360
- // Sanity: crushing still works with a swapped scorer.
1361
- let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();
1362
- let result = c.crush_array(&items, "anything", 1.0);
1363
- assert!(result.items.len() <= 30);
1364
- }
1365
-
1366
- // ---------- Stage 3c.2 PR4: lossless-first default with threshold + CCR-Dropped ----------
1367
-
1368
- #[test]
1369
- fn without_compaction_yields_none_compacted_field() {
1370
- // The opt-out constructor preserves pre-PR4 lossy-only path.
1371
- // No lossless attempt → compacted/compaction_kind always None.
1372
- let c = SmartCrusher::without_compaction(SmartCrusherConfig::default());
1373
- let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();
1374
- let result = c.crush_array(&items, "", 1.0);
1375
- assert!(result.compacted.is_none());
1376
- assert!(result.compaction_kind.is_none());
1377
- }
1378
-
1379
- #[test]
1380
- fn lossless_wins_when_savings_above_threshold() {
1381
- // 50 uniform tabular dicts → CSV+schema compaction shrinks
1382
- // the input dramatically (well above the 0.30 default).
1383
- // Default `SmartCrusher::new()` should pick lossless.
1384
- let c = crusher();
1385
- let items: Vec<Value> = (0..50)
1386
- .map(|i| json!({"id": i, "name": format!("u_{i}"), "status": "ok"}))
1387
- .collect();
1388
- let result = c.crush_array(&items, "", 1.0);
1389
- let compacted = result.compacted.expect("compacted should be set");
1390
- assert!(compacted.starts_with("[50]{"), "got: {compacted}");
1391
- assert_eq!(result.compaction_kind, Some("table"));
1392
- assert!(
1393
- result.strategy_info.starts_with("lossless:table"),
1394
- "got: {}",
1395
- result.strategy_info
1396
- );
1397
- // Lossless = nothing dropped → no CCR retrieval needed.
1398
- assert!(result.ccr_hash.is_none());
1399
- // items preserved (full original).
1400
- assert_eq!(result.items.len(), 50);
1401
- }
1402
-
1403
- #[test]
1404
- fn lossy_falls_through_when_savings_below_threshold() {
1405
- // Force the threshold high enough that even tabular savings
1406
- // can't satisfy it → lossy path runs → CCR-Dropped fires.
1407
- // Use low-uniqueness items so the analyzer is willing to
1408
- // crush (unique id+name per row would trip the
1409
- // "unique_entities_no_signal" skip gate instead).
1410
- let cfg = SmartCrusherConfig {
1411
- lossless_min_savings_ratio: 0.99,
1412
- ..Default::default()
1413
- };
1414
- let c = SmartCrusher::new(cfg);
1415
- let items: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();
1416
- let result = c.crush_array(&items, "", 1.0);
1417
- // Lossless declined → no compacted output.
1418
- assert!(result.compacted.is_none());
1419
- // Lossy ran → rows dropped.
1420
- assert!(
1421
- result.items.len() < 50,
1422
- "expected lossy drop, got {} items",
1423
- result.items.len()
1424
- );
1425
- // CCR hash populated for retrieval.
1426
- let h = result.ccr_hash.expect("ccr_hash populated on drop");
1427
- assert_eq!(h.len(), 12);
1428
- // Marker visible in dropped_summary.
1429
- assert!(
1430
- result.dropped_summary.contains(&format!("<<ccr:{h}")),
1431
- "got: {}",
1432
- result.dropped_summary
1433
- );
1434
- assert!(result.dropped_summary.contains("rows_offloaded"));
1435
- }
1436
-
1437
- #[test]
1438
- fn ccr_hash_is_deterministic() {
1439
- // Same input → same hash, so the runtime cache key is stable.
1440
- let cfg = SmartCrusherConfig {
1441
- lossless_min_savings_ratio: 0.99, // force lossy path
1442
- ..Default::default()
1443
- };
1444
- let c = SmartCrusher::new(cfg);
1445
- let items: Vec<Value> = (0..30).map(|i| json!({"id": i, "tag": "ok"})).collect();
1446
- let r1 = c.crush_array(&items, "", 1.0);
1447
- let r2 = c.crush_array(&items, "", 1.0);
1448
- assert_eq!(r1.ccr_hash, r2.ccr_hash);
1449
- assert!(r1.ccr_hash.is_some());
1450
- }
1451
-
1452
- #[test]
1453
- fn ccr_hash_changes_with_input() {
1454
- let cfg = SmartCrusherConfig {
1455
- lossless_min_savings_ratio: 0.99,
1456
- ..Default::default()
1457
- };
1458
- let c = SmartCrusher::new(cfg);
1459
- let a: Vec<Value> = (0..30).map(|i| json!({"id": i})).collect();
1460
- let b: Vec<Value> = (100..130).map(|i| json!({"id": i})).collect();
1461
- let ra = c.crush_array(&a, "", 1.0);
1462
- let rb = c.crush_array(&b, "", 1.0);
1463
- assert_ne!(ra.ccr_hash, rb.ccr_hash);
1464
- }
1465
-
1466
- #[test]
1467
- fn lossy_without_compaction_still_emits_ccr_hash() {
1468
- // The CCR-Dropped restoration applies regardless of whether
1469
- // lossless was attempted — without_compaction also gets the
1470
- // ccr_hash on row drops.
1471
- let c = SmartCrusher::without_compaction(SmartCrusherConfig::default());
1472
- let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();
1473
- let result = c.crush_array(&items, "", 1.0);
1474
- if result.items.len() < items.len() {
1475
- assert!(result.ccr_hash.is_some());
1476
- assert!(!result.dropped_summary.is_empty());
1477
- }
1478
- }
1479
-
1480
- #[test]
1481
- fn passthrough_paths_do_not_emit_ccr_hash() {
1482
- // Tier-1 boundary (items.len() <= adaptive_k): nothing
1483
- // dropped, no CCR. Skip path: same.
1484
- let c = crusher();
1485
- let small: Vec<Value> = (0..3).map(|i| json!({"id": i})).collect();
1486
- let r = c.crush_array(&small, "", 1.0);
1487
- assert!(r.ccr_hash.is_none());
1488
- assert_eq!(r.dropped_summary, "");
1489
- }
1490
-
1491
- #[test]
1492
- fn compaction_skips_non_object_array() {
1493
- // Compactor returns Untouched for non-object arrays → no
1494
- // compacted field populated, no kind tag.
1495
- let c = SmartCrusherBuilder::new(SmartCrusherConfig::default())
1496
- .with_default_oss_setup()
1497
- .with_default_compaction()
1498
- .build();
1499
- let items: Vec<Value> = (0..30).map(|i| json!(i)).collect();
1500
- let result = c.crush_array(&items, "", 1.0);
1501
- assert!(result.compacted.is_none());
1502
- assert!(result.compaction_kind.is_none());
1503
- }
1504
-
1505
- // ---------- Stage 3c.2 PR5: walker-integration in process_value ----------
1506
-
1507
- #[test]
1508
- fn process_string_short_string_passthrough() {
1509
- let c = SmartCrusher::new(SmartCrusherConfig::default());
1510
- let (out, info) = c.process_value(&json!("hello world"), 0, "", 1.0);
1511
- assert_eq!(out, json!("hello world"));
1512
- assert!(info.is_empty());
1513
- }
1514
-
1515
- #[test]
1516
- fn process_string_stringified_json_array_recurses() {
1517
- // A string-typed field whose value is a JSON-encoded array of
1518
- // dicts. process_value should parse it, recurse, and return
1519
- // the processed JSON re-rendered as a string.
1520
- let c = SmartCrusher::new(SmartCrusherConfig::default());
1521
- let big_array_json = serde_json::to_string(
1522
- &(0..50)
1523
- .map(|i| json!({"id": i, "level": "info", "msg": "ok"}))
1524
- .collect::<Vec<_>>(),
1525
- )
1526
- .unwrap();
1527
- let doc = json!({"payload": big_array_json.clone()});
1528
- let (out, info) = c.process_value(&doc, 0, "", 1.0);
1529
- // payload still a string-typed field — we preserved the
1530
- // wrapping shape — but its content was processed.
1531
- let payload = out.pointer("/payload").and_then(|v| v.as_str()).unwrap();
1532
- // Either compressed or unchanged; if compressed, info reflects.
1533
- // For 50 items with low-uniqueness, compression should fire.
1534
- // The strategy info should mention string_json processing.
1535
- assert!(
1536
- info.contains("string_json") || payload != big_array_json,
1537
- "expected processing trace; info={info}, len before={}, after={}",
1538
- big_array_json.len(),
1539
- payload.len(),
1540
- );
1541
- }
1542
-
1543
- #[test]
1544
- fn process_string_opaque_blob_becomes_ccr_marker() {
1545
- let c = SmartCrusher::new(SmartCrusherConfig::default());
1546
- let big_b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".repeat(8);
1547
- let doc = json!({"id": 1, "blob": big_b64});
1548
- let (out, _info) = c.process_value(&doc, 0, "", 1.0);
1549
- let blob = out.pointer("/blob").and_then(|v| v.as_str()).unwrap();
1550
- assert!(blob.starts_with("<<ccr:"), "got: {blob}");
1551
- assert!(blob.contains(",base64,"));
1552
- }
1553
-
1554
- #[test]
1555
- fn process_string_top_level_string_processed() {
1556
- // crush() takes a string; if it doesn't parse as JSON, today's
1557
- // behavior returns it unchanged. But if it's a stringified
1558
- // JSON object/array, it should now get processed.
1559
- let c = SmartCrusher::new(SmartCrusherConfig::default());
1560
- // Non-JSON top-level string — passthrough.
1561
- let plain = "just some plain text";
1562
- let result = c.crush(plain, "", 1.0);
1563
- assert_eq!(result.compressed, plain);
1564
- }
1565
-
1566
- #[test]
1567
- fn process_string_does_not_alter_short_quoted_strings() {
1568
- // Strings that look JSON-like but are short shouldn't be
1569
- // CCR-substituted.
1570
- let c = SmartCrusher::new(SmartCrusherConfig::default());
1571
- let doc = json!({"msg": "{this looks like json but isnt}"});
1572
- let (out, _) = c.process_value(&doc, 0, "", 1.0);
1573
- assert_eq!(out, doc);
1574
- }
1575
-
1576
- #[test]
1577
- fn process_string_helper_parses_only_containers() {
1578
- assert!(try_parse_json_container("{\"a\":1}").is_some());
1579
- assert!(try_parse_json_container("[1,2,3]").is_some());
1580
- assert!(try_parse_json_container("123").is_none()); // bare scalar
1581
- assert!(try_parse_json_container("\"hello\"").is_none()); // bare string
1582
- assert!(try_parse_json_container("not json").is_none());
1583
- assert!(try_parse_json_container("{malformed").is_none());
1584
- }
1585
-
1586
- // ---------- enable_ccr_marker gate (PR #301 re-land) ----------
1587
-
1588
- #[test]
1589
- fn enable_ccr_marker_false_suppresses_marker_and_store() {
1590
- // The Rust-side gate. Compression still runs (rows drop) but
1591
- // the result carries no marker text, no hash, and the CCR
1592
- // store does NOT grow — there's no point storing what nothing
1593
- // in the prompt can reference.
1594
- use crate::ccr::InMemoryCcrStore;
1595
- use crate::transforms::smart_crusher::SmartCrusherBuilder;
1596
- use std::sync::Arc;
1597
-
1598
- let store: Arc<dyn CcrStore> = Arc::new(InMemoryCcrStore::new());
1599
- let cfg = SmartCrusherConfig {
1600
- lossless_min_savings_ratio: 0.99, // force lossy path
1601
- enable_ccr_marker: false,
1602
- ..SmartCrusherConfig::default()
1603
- };
1604
- let c = SmartCrusherBuilder::new(cfg)
1605
- .with_ccr_store(Arc::clone(&store))
1606
- .build();
1607
- let items: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();
1608
-
1609
- let store_len_before = store.len();
1610
- let result = c.crush_array(&items, "", 1.0);
1611
- let store_len_after = store.len();
1612
-
1613
- // Rows were dropped (we built 50, kept fewer).
1614
- assert!(result.items.len() < items.len(), "lossy path didn't fire");
1615
- // Gate held: no marker, no hash.
1616
- assert!(result.ccr_hash.is_none(), "ccr_hash should be None");
1617
- assert!(
1618
- result.dropped_summary.is_empty(),
1619
- "dropped_summary should be empty, got: {:?}",
1620
- result.dropped_summary
1621
- );
1622
- // Store did NOT grow.
1623
- assert_eq!(
1624
- store_len_after, store_len_before,
1625
- "ccr_store grew despite enable_ccr_marker=false"
1626
- );
1627
- }
1628
-
1629
- #[test]
1630
- fn enable_ccr_marker_true_is_default_behavior() {
1631
- // Default config still emits markers + stores when rows drop.
1632
- // Sanity: the gate is opt-out, not opt-in.
1633
- use crate::ccr::InMemoryCcrStore;
1634
- use crate::transforms::smart_crusher::SmartCrusherBuilder;
1635
- use std::sync::Arc;
1636
-
1637
- let store: Arc<dyn CcrStore> = Arc::new(InMemoryCcrStore::new());
1638
- let cfg = SmartCrusherConfig {
1639
- lossless_min_savings_ratio: 0.99, // force lossy path
1640
- ..SmartCrusherConfig::default()
1641
- };
1642
- // Default: enable_ccr_marker = true.
1643
- assert!(cfg.enable_ccr_marker);
1644
- let c = SmartCrusherBuilder::new(cfg)
1645
- .with_ccr_store(Arc::clone(&store))
1646
- .build();
1647
- let items: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();
1648
-
1649
- let store_len_before = store.len();
1650
- let result = c.crush_array(&items, "", 1.0);
1651
- let store_len_after = store.len();
1652
-
1653
- assert!(result.items.len() < items.len(), "lossy path didn't fire");
1654
- assert!(result.ccr_hash.is_some(), "default should produce a hash");
1655
- assert!(
1656
- result.dropped_summary.contains("<<ccr:"),
1657
- "default should produce a marker: {:?}",
1658
- result.dropped_summary
1659
- );
1660
- assert!(
1661
- store_len_after > store_len_before,
1662
- "default should write to ccr_store"
1663
- );
1664
- }
1665
- }