xithead 1.5.0 → 1.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (649) hide show
  1. package/design-system/README.md +80 -0
  2. package/package.json +20 -1
  3. package/public/apple-touch-icon.png +0 -0
  4. package/public/logo-transparent.png +0 -0
  5. package/public/xithead.ps1 +937 -0
  6. package/scripts/README.md +77 -0
  7. package/scripts/audit_wheel_glibc_symbols.py +217 -0
  8. package/scripts/build-master-design-system.cjs +33 -0
  9. package/scripts/build_rust_extension.sh +58 -0
  10. package/scripts/changelog-gen.py +203 -0
  11. package/scripts/eval_output_shaper.py +235 -0
  12. package/scripts/export_kompress_v2_onnx.py +267 -0
  13. package/scripts/fixtures/anthropic_replay_body.json +133 -0
  14. package/scripts/fixtures/codex_response_create_frame.json +52 -0
  15. package/scripts/install-git-hooks.sh +104 -0
  16. package/scripts/install.ps1 +1809 -0
  17. package/scripts/install.sh +1673 -0
  18. package/scripts/pr-governance.py +297 -0
  19. package/scripts/record_fixtures.py +59 -0
  20. package/scripts/refresh_model_limits.sh +48 -0
  21. package/scripts/replay_codex_ws_load.py +445 -0
  22. package/scripts/repro_codex_replay.py +699 -0
  23. package/scripts/smoke_issue_327.py +233 -0
  24. package/scripts/sync-plugin-versions.py +116 -0
  25. package/scripts/tests/test_changelog_gen.py +302 -0
  26. package/scripts/tests/test_pr_governance.py +163 -0
  27. package/scripts/tests/test_pr_health_labels.py +94 -0
  28. package/scripts/tests/test_pr_health_workflow.py +14 -0
  29. package/scripts/tests/test_sync_plugin_versions.py +131 -0
  30. package/scripts/tests/test_version_sync.py +291 -0
  31. package/scripts/validate-workflows.sh +39 -0
  32. package/scripts/verify-versions.py +106 -0
  33. package/scripts/version-sync.py +207 -0
  34. package/tailwind.config.js +31 -0
  35. package/tsconfig.json +84 -0
  36. package/.actrc +0 -14
  37. package/.actrc.local.example +0 -15
  38. package/.changelog.md +0 -2
  39. package/.claude/agents/academic-anthropologist.md +0 -125
  40. package/.claude/agents/academic-geographer.md +0 -127
  41. package/.claude/agents/academic-historian.md +0 -123
  42. package/.claude/agents/academic-narratologist.md +0 -118
  43. package/.claude/agents/academic-psychologist.md +0 -118
  44. package/.claude/agents/academic-statistician.md +0 -144
  45. package/.claude/agents/accounts-payable-agent.md +0 -205
  46. package/.claude/agents/agentic-identity-trust.md +0 -394
  47. package/.claude/agents/agents-orchestrator.md +0 -367
  48. package/.claude/agents/automation-governance-architect.md +0 -216
  49. package/.claude/agents/blender-addon-engineer.md +0 -243
  50. package/.claude/agents/business-strategist.md +0 -488
  51. package/.claude/agents/change-management-consultant.md +0 -497
  52. package/.claude/agents/chief-financial-officer.md +0 -388
  53. package/.claude/agents/corporate-training-designer.md +0 -192
  54. package/.claude/agents/customer-service.md +0 -398
  55. package/.claude/agents/customer-success-manager.md +0 -460
  56. package/.claude/agents/data-consolidation-agent.md +0 -60
  57. package/.claude/agents/data-privacy-officer.md +0 -412
  58. package/.claude/agents/design-brand-guardian.md +0 -322
  59. package/.claude/agents/design-image-prompt-engineer.md +0 -236
  60. package/.claude/agents/design-inclusive-visuals-specialist.md +0 -73
  61. package/.claude/agents/design-persona-walkthrough.md +0 -272
  62. package/.claude/agents/design-ui-designer.md +0 -386
  63. package/.claude/agents/design-ui-finish-gate-reviewer.md +0 -217
  64. package/.claude/agents/design-ux-architect.md +0 -482
  65. package/.claude/agents/design-ux-researcher.md +0 -329
  66. package/.claude/agents/design-visual-storyteller.md +0 -149
  67. package/.claude/agents/design-whimsy-injector.md +0 -460
  68. package/.claude/agents/economy-designer.md +0 -156
  69. package/.claude/agents/engineering-ai-data-remediation-engineer.md +0 -211
  70. package/.claude/agents/engineering-ai-engineer.md +0 -146
  71. package/.claude/agents/engineering-api-platform-engineer.md +0 -181
  72. package/.claude/agents/engineering-ats-validator-architect.md +0 -383
  73. package/.claude/agents/engineering-autonomous-optimization-architect.md +0 -121
  74. package/.claude/agents/engineering-backend-architect.md +0 -255
  75. package/.claude/agents/engineering-china-network-engineer.md +0 -252
  76. package/.claude/agents/engineering-cms-developer.md +0 -536
  77. package/.claude/agents/engineering-code-reviewer.md +0 -76
  78. package/.claude/agents/engineering-codebase-onboarding-engineer.md +0 -173
  79. package/.claude/agents/engineering-data-engineer.md +0 -349
  80. package/.claude/agents/engineering-data-visualization-engineer.md +0 -151
  81. package/.claude/agents/engineering-database-optimizer.md +0 -188
  82. package/.claude/agents/engineering-database-reliability-engineer.md +0 -201
  83. package/.claude/agents/engineering-desktop-app-engineer.md +0 -209
  84. package/.claude/agents/engineering-developer-tooling-engineer.md +0 -153
  85. package/.claude/agents/engineering-devops-automator.md +0 -385
  86. package/.claude/agents/engineering-drupal-performance.md +0 -347
  87. package/.claude/agents/engineering-drupal-shopping-cart.md +0 -360
  88. package/.claude/agents/engineering-electrical-engineer.md +0 -575
  89. package/.claude/agents/engineering-email-intelligence-engineer.md +0 -390
  90. package/.claude/agents/engineering-embedded-firmware-engineer.md +0 -196
  91. package/.claude/agents/engineering-feishu-integration-developer.md +0 -606
  92. package/.claude/agents/engineering-filament-optimization-specialist.md +0 -283
  93. package/.claude/agents/engineering-finops-engineer.md +0 -164
  94. package/.claude/agents/engineering-frontend-developer.md +0 -257
  95. package/.claude/agents/engineering-gaussdb-expert.md +0 -334
  96. package/.claude/agents/engineering-git-workflow-master.md +0 -98
  97. package/.claude/agents/engineering-i18n-engineer.md +0 -214
  98. package/.claude/agents/engineering-identity-access-engineer.md +0 -217
  99. package/.claude/agents/engineering-incident-response-commander.md +0 -444
  100. package/.claude/agents/engineering-iot-fleet-engineer.md +0 -148
  101. package/.claude/agents/engineering-it-service-manager.md +0 -561
  102. package/.claude/agents/engineering-knowledge-graph-engineer.md +0 -372
  103. package/.claude/agents/engineering-llm-post-training-engineer.md +0 -166
  104. package/.claude/agents/engineering-minimal-change-engineer.md +0 -207
  105. package/.claude/agents/engineering-mobile-app-builder.md +0 -493
  106. package/.claude/agents/engineering-mobile-release-engineer.md +0 -164
  107. package/.claude/agents/engineering-multi-agent-systems-architect.md +0 -600
  108. package/.claude/agents/engineering-network-engineer.md +0 -245
  109. package/.claude/agents/engineering-orgscript-engineer.md +0 -113
  110. package/.claude/agents/engineering-payments-billing-engineer.md +0 -228
  111. package/.claude/agents/engineering-pdf-engine-architect.md +0 -696
  112. package/.claude/agents/engineering-platform-engineer.md +0 -270
  113. package/.claude/agents/engineering-privacy-engineer.md +0 -152
  114. package/.claude/agents/engineering-prompt-engineer.md +0 -209
  115. package/.claude/agents/engineering-rag-pipeline-engineer.md +0 -450
  116. package/.claude/agents/engineering-rapid-prototyper.md +0 -466
  117. package/.claude/agents/engineering-realtime-collaboration-engineer.md +0 -200
  118. package/.claude/agents/engineering-rust-refactoring-specialist.md +0 -313
  119. package/.claude/agents/engineering-search-relevance-engineer.md +0 -237
  120. package/.claude/agents/engineering-section-508-specialist.md +0 -339
  121. package/.claude/agents/engineering-senior-developer.md +0 -176
  122. package/.claude/agents/engineering-servicenow-developer-mentor.md +0 -180
  123. package/.claude/agents/engineering-software-architect.md +0 -112
  124. package/.claude/agents/engineering-solidity-smart-contract-engineer.md +0 -522
  125. package/.claude/agents/engineering-sre.md +0 -90
  126. package/.claude/agents/engineering-technical-writer.md +0 -393
  127. package/.claude/agents/engineering-universal-document-compiler.md +0 -370
  128. package/.claude/agents/engineering-uswds-developer.md +0 -340
  129. package/.claude/agents/engineering-video-streaming-engineer.md +0 -151
  130. package/.claude/agents/engineering-voice-ai-integration-engineer.md +0 -584
  131. package/.claude/agents/engineering-webassembly-engineer.md +0 -153
  132. package/.claude/agents/engineering-wechat-mini-program-developer.md +0 -356
  133. package/.claude/agents/engineering-wordpress-performance.md +0 -346
  134. package/.claude/agents/engineering-wordpress-shopping-cart.md +0 -346
  135. package/.claude/agents/esg-sustainability-officer.md +0 -396
  136. package/.claude/agents/finance-bookkeeper-controller.md +0 -260
  137. package/.claude/agents/finance-financial-analyst.md +0 -234
  138. package/.claude/agents/finance-fpa-analyst.md +0 -263
  139. package/.claude/agents/finance-investment-researcher.md +0 -272
  140. package/.claude/agents/finance-tax-strategist.md +0 -239
  141. package/.claude/agents/game-audio-engineer.md +0 -264
  142. package/.claude/agents/game-designer.md +0 -167
  143. package/.claude/agents/gis-3d-scene-developer.md +0 -111
  144. package/.claude/agents/gis-analyst.md +0 -91
  145. package/.claude/agents/gis-bim-specialist.md +0 -108
  146. package/.claude/agents/gis-cartography-designer.md +0 -150
  147. package/.claude/agents/gis-drone-reality-mapping.md +0 -120
  148. package/.claude/agents/gis-geoai-ml-engineer.md +0 -105
  149. package/.claude/agents/gis-geoprocessing-specialist.md +0 -97
  150. package/.claude/agents/gis-qa-engineer.md +0 -133
  151. package/.claude/agents/gis-solution-engineer.md +0 -101
  152. package/.claude/agents/gis-spatial-data-engineer.md +0 -97
  153. package/.claude/agents/gis-spatial-data-scientist.md +0 -111
  154. package/.claude/agents/gis-technical-consultant.md +0 -86
  155. package/.claude/agents/gis-web-gis-developer.md +0 -108
  156. package/.claude/agents/godot-gameplay-scripter.md +0 -338
  157. package/.claude/agents/godot-multiplayer-engineer.md +0 -301
  158. package/.claude/agents/godot-shader-developer.md +0 -267
  159. package/.claude/agents/government-digital-presales-consultant.md +0 -363
  160. package/.claude/agents/grant-writer.md +0 -511
  161. package/.claude/agents/healthcare-aging-parent-care-companion.md +0 -414
  162. package/.claude/agents/healthcare-clinical-evidence-agent.md +0 -231
  163. package/.claude/agents/healthcare-customer-service.md +0 -389
  164. package/.claude/agents/healthcare-innovation-strategist.md +0 -433
  165. package/.claude/agents/healthcare-marketing-compliance.md +0 -395
  166. package/.claude/agents/healthcare-sovereign-health-systems-agent.md +0 -312
  167. package/.claude/agents/hospitality-guest-services.md +0 -603
  168. package/.claude/agents/hr-onboarding.md +0 -451
  169. package/.claude/agents/identity-graph-operator.md +0 -303
  170. package/.claude/agents/language-translator.md +0 -264
  171. package/.claude/agents/legal-billing-time-tracking.md +0 -569
  172. package/.claude/agents/legal-client-intake.md +0 -492
  173. package/.claude/agents/legal-document-review.md +0 -454
  174. package/.claude/agents/level-designer.md +0 -208
  175. package/.claude/agents/loan-officer-assistant.md +0 -555
  176. package/.claude/agents/lsp-index-engineer.md +0 -320
  177. package/.claude/agents/ma-integration-manager.md +0 -427
  178. package/.claude/agents/macos-spatial-metal-engineer.md +0 -337
  179. package/.claude/agents/marketing-aeo-foundations.md +0 -264
  180. package/.claude/agents/marketing-agentic-search-optimizer.md +0 -279
  181. package/.claude/agents/marketing-ai-citation-strategist.md +0 -172
  182. package/.claude/agents/marketing-app-store-optimizer.md +0 -321
  183. package/.claude/agents/marketing-baidu-seo-specialist.md +0 -226
  184. package/.claude/agents/marketing-bilibili-content-strategist.md +0 -199
  185. package/.claude/agents/marketing-book-co-author.md +0 -110
  186. package/.claude/agents/marketing-carousel-growth-engine.md +0 -204
  187. package/.claude/agents/marketing-china-ecommerce-operator.md +0 -283
  188. package/.claude/agents/marketing-china-market-localization-strategist.md +0 -283
  189. package/.claude/agents/marketing-content-creator.md +0 -54
  190. package/.claude/agents/marketing-cross-border-ecommerce.md +0 -259
  191. package/.claude/agents/marketing-developer-community-builder.md +0 -285
  192. package/.claude/agents/marketing-douyin-strategist.md +0 -149
  193. package/.claude/agents/marketing-email-strategist.md +0 -249
  194. package/.claude/agents/marketing-global-podcast-strategist.md +0 -206
  195. package/.claude/agents/marketing-growth-hacker.md +0 -54
  196. package/.claude/agents/marketing-instagram-curator.md +0 -113
  197. package/.claude/agents/marketing-kuaishou-strategist.md +0 -223
  198. package/.claude/agents/marketing-linkedin-content-creator.md +0 -214
  199. package/.claude/agents/marketing-livestream-commerce-coach.md +0 -305
  200. package/.claude/agents/marketing-multi-platform-publisher.md +0 -217
  201. package/.claude/agents/marketing-podcast-strategist.md +0 -277
  202. package/.claude/agents/marketing-pr-communications-manager.md +0 -473
  203. package/.claude/agents/marketing-private-domain-operator.md +0 -308
  204. package/.claude/agents/marketing-reddit-community-builder.md +0 -123
  205. package/.claude/agents/marketing-seo-specialist.md +0 -370
  206. package/.claude/agents/marketing-short-video-editing-coach.md +0 -412
  207. package/.claude/agents/marketing-social-media-strategist.md +0 -125
  208. package/.claude/agents/marketing-tiktok-strategist.md +0 -125
  209. package/.claude/agents/marketing-twitter-engager.md +0 -126
  210. package/.claude/agents/marketing-video-optimization-specialist.md +0 -119
  211. package/.claude/agents/marketing-wechat-official-account.md +0 -145
  212. package/.claude/agents/marketing-weibo-strategist.md +0 -240
  213. package/.claude/agents/marketing-x-twitter-intelligence-analyst.md +0 -161
  214. package/.claude/agents/marketing-xiaohongshu-specialist.md +0 -138
  215. package/.claude/agents/marketing-zhihu-strategist.md +0 -162
  216. package/.claude/agents/medical-billing-coding-specialist.md +0 -491
  217. package/.claude/agents/narrative-designer.md +0 -243
  218. package/.claude/agents/operations-manager.md +0 -399
  219. package/.claude/agents/organizational-psychologist.md +0 -391
  220. package/.claude/agents/paid-media-auditor.md +0 -71
  221. package/.claude/agents/paid-media-creative-strategist.md +0 -71
  222. package/.claude/agents/paid-media-paid-social-strategist.md +0 -71
  223. package/.claude/agents/paid-media-ppc-strategist.md +0 -71
  224. package/.claude/agents/paid-media-programmatic-buyer.md +0 -71
  225. package/.claude/agents/paid-media-search-query-analyst.md +0 -71
  226. package/.claude/agents/paid-media-tracking-specialist.md +0 -71
  227. package/.claude/agents/personal-growth-mentor.md +0 -159
  228. package/.claude/agents/product-behavioral-nudge-engine.md +0 -83
  229. package/.claude/agents/product-dx-engineer.md +0 -332
  230. package/.claude/agents/product-feedback-synthesizer.md +0 -119
  231. package/.claude/agents/product-manager.md +0 -469
  232. package/.claude/agents/product-sprint-prioritizer.md +0 -154
  233. package/.claude/agents/product-trend-researcher.md +0 -159
  234. package/.claude/agents/project-management-experiment-tracker.md +0 -198
  235. package/.claude/agents/project-management-jira-workflow-steward.md +0 -233
  236. package/.claude/agents/project-management-meeting-notes-specialist.md +0 -95
  237. package/.claude/agents/project-management-project-shepherd.md +0 -194
  238. package/.claude/agents/project-management-studio-operations.md +0 -200
  239. package/.claude/agents/project-management-studio-producer.md +0 -203
  240. package/.claude/agents/project-manager-senior.md +0 -135
  241. package/.claude/agents/real-estate-buyer-seller.md +0 -596
  242. package/.claude/agents/recruitment-specialist.md +0 -515
  243. package/.claude/agents/report-distribution-agent.md +0 -65
  244. package/.claude/agents/research-synthesist.md +0 -136
  245. package/.claude/agents/resume-tailor.md +0 -230
  246. package/.claude/agents/retail-customer-returns.md +0 -566
  247. package/.claude/agents/roblox-avatar-creator.md +0 -297
  248. package/.claude/agents/roblox-experience-designer.md +0 -306
  249. package/.claude/agents/roblox-systems-scripter.md +0 -332
  250. package/.claude/agents/sales-account-strategist.md +0 -227
  251. package/.claude/agents/sales-coach.md +0 -271
  252. package/.claude/agents/sales-data-extraction-agent.md +0 -67
  253. package/.claude/agents/sales-deal-strategist.md +0 -180
  254. package/.claude/agents/sales-discovery-coach.md +0 -225
  255. package/.claude/agents/sales-engineer.md +0 -182
  256. package/.claude/agents/sales-offer-lead-gen-strategist.md +0 -257
  257. package/.claude/agents/sales-outbound-strategist.md +0 -201
  258. package/.claude/agents/sales-outreach.md +0 -425
  259. package/.claude/agents/sales-pipeline-analyst.md +0 -267
  260. package/.claude/agents/sales-proposal-strategist.md +0 -217
  261. package/.claude/agents/security-ai-generated-code-auditor.md +0 -207
  262. package/.claude/agents/security-appsec-engineer.md +0 -536
  263. package/.claude/agents/security-architect.md +0 -307
  264. package/.claude/agents/security-blockchain-security-auditor.md +0 -482
  265. package/.claude/agents/security-cloud-security-architect.md +0 -576
  266. package/.claude/agents/security-compliance-auditor.md +0 -158
  267. package/.claude/agents/security-incident-responder.md +0 -450
  268. package/.claude/agents/security-penetration-tester.md +0 -399
  269. package/.claude/agents/security-secrets-credential-engineer.md +0 -189
  270. package/.claude/agents/security-senior-secops.md +0 -760
  271. package/.claude/agents/security-threat-detection-engineer.md +0 -553
  272. package/.claude/agents/security-threat-intelligence-analyst.md +0 -661
  273. package/.claude/agents/specialized-chief-of-staff.md +0 -279
  274. package/.claude/agents/specialized-civil-engineer.md +0 -356
  275. package/.claude/agents/specialized-codebase-archaeologist.md +0 -341
  276. package/.claude/agents/specialized-cultural-intelligence-strategist.md +0 -88
  277. package/.claude/agents/specialized-developer-advocate.md +0 -317
  278. package/.claude/agents/specialized-document-generator.md +0 -55
  279. package/.claude/agents/specialized-fedramp-rmf-compliance.md +0 -378
  280. package/.claude/agents/specialized-focus-music-architect.md +0 -175
  281. package/.claude/agents/specialized-french-consulting-market.md +0 -194
  282. package/.claude/agents/specialized-korean-business-navigator.md +0 -216
  283. package/.claude/agents/specialized-master-plan-architect.md +0 -157
  284. package/.claude/agents/specialized-mcp-builder.md +0 -260
  285. package/.claude/agents/specialized-model-qa.md +0 -528
  286. package/.claude/agents/specialized-pricing-analyst.md +0 -243
  287. package/.claude/agents/specialized-salesforce-architect.md +0 -182
  288. package/.claude/agents/specialized-strategy-duel-agent.md +0 -130
  289. package/.claude/agents/specialized-workflow-architect.md +0 -597
  290. package/.claude/agents/study-abroad-advisor.md +0 -282
  291. package/.claude/agents/supply-chain-strategist.md +0 -591
  292. package/.claude/agents/support-analytics-reporter.md +0 -387
  293. package/.claude/agents/support-executive-summary-generator.md +0 -212
  294. package/.claude/agents/support-finance-tracker.md +0 -480
  295. package/.claude/agents/support-infrastructure-maintainer.md +0 -628
  296. package/.claude/agents/support-legal-compliance-checker.md +0 -591
  297. package/.claude/agents/support-support-responder.md +0 -607
  298. package/.claude/agents/technical-artist.md +0 -233
  299. package/.claude/agents/terminal-integration-specialist.md +0 -70
  300. package/.claude/agents/testing-accessibility-auditor.md +0 -327
  301. package/.claude/agents/testing-api-tester.md +0 -334
  302. package/.claude/agents/testing-evidence-collector.md +0 -210
  303. package/.claude/agents/testing-performance-benchmarker.md +0 -283
  304. package/.claude/agents/testing-reality-checker.md +0 -249
  305. package/.claude/agents/testing-test-automation-engineer.md +0 -196
  306. package/.claude/agents/testing-test-results-analyzer.md +0 -330
  307. package/.claude/agents/testing-tool-evaluator.md +0 -416
  308. package/.claude/agents/testing-workflow-optimizer.md +0 -463
  309. package/.claude/agents/unity-architect.md +0 -271
  310. package/.claude/agents/unity-editor-tool-developer.md +0 -310
  311. package/.claude/agents/unity-multiplayer-engineer.md +0 -323
  312. package/.claude/agents/unity-shader-graph-artist.md +0 -269
  313. package/.claude/agents/unreal-multiplayer-architect.md +0 -364
  314. package/.claude/agents/unreal-systems-engineer.md +0 -322
  315. package/.claude/agents/unreal-technical-artist.md +0 -256
  316. package/.claude/agents/unreal-world-builder.md +0 -273
  317. package/.claude/agents/visionos-spatial-engineer.md +0 -54
  318. package/.claude/agents/xr-cockpit-interaction-specialist.md +0 -32
  319. package/.claude/agents/xr-immersive-developer.md +0 -32
  320. package/.claude/agents/xr-interface-architect.md +0 -32
  321. package/.claude/agents/zk-steward.md +0 -211
  322. package/.claude-plugin/marketplace.json +0 -30
  323. package/.commitlintrc.json +0 -26
  324. package/.devcontainer/Dockerfile +0 -24
  325. package/.devcontainer/devcontainer.json +0 -70
  326. package/.devcontainer/docker-compose.memory.yml +0 -42
  327. package/.devcontainer/memory-stack/devcontainer.json +0 -84
  328. package/.devcontainer/post-create.sh +0 -83
  329. package/.dockerignore +0 -23
  330. package/.env.act.example +0 -15
  331. package/.env.example +0 -171
  332. package/.eslintrc.json +0 -16
  333. package/.git-blame-ignore-revs +0 -10
  334. package/.gitattributes +0 -2
  335. package/.gitguardian.yaml +0 -50
  336. package/.githooks/pre-commit +0 -6
  337. package/.pre-commit-config.yaml +0 -30
  338. package/.release-please-config-xithead.json +0 -22
  339. package/.release-please-config.json +0 -44
  340. package/.release-please-manifest-xithead.json +0 -3
  341. package/.release-please-manifest.json +0 -3
  342. package/CODE_OF_CONDUCT.md +0 -133
  343. package/CONTRIBUTING.md +0 -133
  344. package/Cargo.lock +0 -5557
  345. package/Cargo.toml +0 -121
  346. package/Dockerfile +0 -38
  347. package/Dockerfile.proxy +0 -48
  348. package/ENTERPRISE.md +0 -5
  349. package/FEATURES_ADVANCED.md +0 -311
  350. package/Headroom-2.gif +0 -0
  351. package/HeadroomDemo-Fast.gif +0 -0
  352. package/MANIFEST.in +0 -4
  353. package/MODERNIZATION_GUIDE.md +0 -358
  354. package/Makefile +0 -140
  355. package/NOTICE +0 -43
  356. package/NPM_TOKEN_SETUP.md +0 -81
  357. package/PR.md +0 -170
  358. package/PUBLISH.md +0 -77
  359. package/README_FRONTEND.md +0 -196
  360. package/REALIGNMENT/00-overview.md +0 -53
  361. package/REALIGNMENT/01-bug-list.md +0 -421
  362. package/REALIGNMENT/02-architecture.md +0 -322
  363. package/REALIGNMENT/03-phase-A-lockdown.md +0 -420
  364. package/REALIGNMENT/04-phase-B-live-zone.md +0 -470
  365. package/REALIGNMENT/05-phase-C-rust-proxy.md +0 -329
  366. package/REALIGNMENT/06-phase-D-bedrock-vertex.md +0 -222
  367. package/REALIGNMENT/07-phase-E-cache-stabilization.md +0 -338
  368. package/REALIGNMENT/08-phase-F-auth-mode.md +0 -254
  369. package/REALIGNMENT/09-phase-G-rtk-observability.md +0 -194
  370. package/REALIGNMENT/10-phase-H-python-retirement.md +0 -214
  371. package/REALIGNMENT/11-phase-I-test-infra.md +0 -385
  372. package/REALIGNMENT/12-decisions-needed.md +0 -196
  373. package/REALIGNMENT/INDEX.md +0 -82
  374. package/REFACTORING_CAMPAIGN.md +0 -277
  375. package/RESTRUCTURE_PLAN.md +0 -100
  376. package/RUST_DEV.md +0 -361
  377. package/SECURITY.md +0 -65
  378. package/SETUP_AUTO_PUBLISH.md +0 -164
  379. package/TESTING-copilot-subscription.md +0 -153
  380. package/codecov.yml +0 -34
  381. package/crates/headroom-core/Cargo.toml +0 -165
  382. package/crates/headroom-core/benches/auth_mode.rs +0 -74
  383. package/crates/headroom-core/benches/ccr_store.rs +0 -224
  384. package/crates/headroom-core/benches/tokenizer.rs +0 -52
  385. package/crates/headroom-core/config/pipeline.toml +0 -151
  386. package/crates/headroom-core/proptest-regressions/transforms/tag_protector.txt +0 -9
  387. package/crates/headroom-core/src/auth_mode.rs +0 -250
  388. package/crates/headroom-core/src/cache_control.rs +0 -414
  389. package/crates/headroom-core/src/ccr/backends/in_memory.rs +0 -317
  390. package/crates/headroom-core/src/ccr/backends/mod.rs +0 -152
  391. package/crates/headroom-core/src/ccr/backends/redis.rs +0 -146
  392. package/crates/headroom-core/src/ccr/backends/sqlite.rs +0 -205
  393. package/crates/headroom-core/src/ccr/mod.rs +0 -119
  394. package/crates/headroom-core/src/compression_policy.rs +0 -505
  395. package/crates/headroom-core/src/lib.rs +0 -33
  396. package/crates/headroom-core/src/relevance/base.rs +0 -139
  397. package/crates/headroom-core/src/relevance/bm25.rs +0 -385
  398. package/crates/headroom-core/src/relevance/embedding.rs +0 -354
  399. package/crates/headroom-core/src/relevance/hybrid.rs +0 -364
  400. package/crates/headroom-core/src/relevance/mod.rs +0 -64
  401. package/crates/headroom-core/src/signals/keyword_detector.rs +0 -433
  402. package/crates/headroom-core/src/signals/line_importance.rs +0 -84
  403. package/crates/headroom-core/src/signals/mod.rs +0 -61
  404. package/crates/headroom-core/src/signals/tiered.rs +0 -141
  405. package/crates/headroom-core/src/tokenizer/estimator.rs +0 -152
  406. package/crates/headroom-core/src/tokenizer/hf_impl.rs +0 -314
  407. package/crates/headroom-core/src/tokenizer/mod.rs +0 -56
  408. package/crates/headroom-core/src/tokenizer/registry.rs +0 -342
  409. package/crates/headroom-core/src/tokenizer/tiktoken_impl.rs +0 -265
  410. package/crates/headroom-core/src/transforms/adaptive_sizer.rs +0 -610
  411. package/crates/headroom-core/src/transforms/anchor_selector.rs +0 -1189
  412. package/crates/headroom-core/src/transforms/content_detector.rs +0 -769
  413. package/crates/headroom-core/src/transforms/detection.rs +0 -222
  414. package/crates/headroom-core/src/transforms/diff_compressor.rs +0 -1685
  415. package/crates/headroom-core/src/transforms/live_zone.rs +0 -2967
  416. package/crates/headroom-core/src/transforms/log_compressor.rs +0 -1295
  417. package/crates/headroom-core/src/transforms/magika_detector.rs +0 -395
  418. package/crates/headroom-core/src/transforms/mod.rs +0 -64
  419. package/crates/headroom-core/src/transforms/pipeline/config.rs +0 -331
  420. package/crates/headroom-core/src/transforms/pipeline/mod.rs +0 -103
  421. package/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rs +0 -524
  422. package/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs +0 -279
  423. package/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs +0 -400
  424. package/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rs +0 -284
  425. package/crates/headroom-core/src/transforms/pipeline/offloads/mod.rs +0 -34
  426. package/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rs +0 -331
  427. package/crates/headroom-core/src/transforms/pipeline/orchestrator.rs +0 -848
  428. package/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rs +0 -175
  429. package/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rs +0 -545
  430. package/crates/headroom-core/src/transforms/pipeline/reformats/mod.rs +0 -12
  431. package/crates/headroom-core/src/transforms/pipeline/traits.rs +0 -338
  432. package/crates/headroom-core/src/transforms/recommendations.rs +0 -329
  433. package/crates/headroom-core/src/transforms/safety.rs +0 -215
  434. package/crates/headroom-core/src/transforms/search_compressor.rs +0 -902
  435. package/crates/headroom-core/src/transforms/smart_crusher/analyzer.rs +0 -1262
  436. package/crates/headroom-core/src/transforms/smart_crusher/anchors.rs +0 -407
  437. package/crates/headroom-core/src/transforms/smart_crusher/builder.rs +0 -263
  438. package/crates/headroom-core/src/transforms/smart_crusher/classifier.rs +0 -214
  439. package/crates/headroom-core/src/transforms/smart_crusher/compaction/classifier.rs +0 -283
  440. package/crates/headroom-core/src/transforms/smart_crusher/compaction/compactor.rs +0 -733
  441. package/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rs +0 -961
  442. package/crates/headroom-core/src/transforms/smart_crusher/compaction/ir.rs +0 -252
  443. package/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs +0 -127
  444. package/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs +0 -397
  445. package/crates/headroom-core/src/transforms/smart_crusher/config.rs +0 -173
  446. package/crates/headroom-core/src/transforms/smart_crusher/constraints.rs +0 -159
  447. package/crates/headroom-core/src/transforms/smart_crusher/crusher.rs +0 -1665
  448. package/crates/headroom-core/src/transforms/smart_crusher/crushers.rs +0 -865
  449. package/crates/headroom-core/src/transforms/smart_crusher/error_keywords.rs +0 -77
  450. package/crates/headroom-core/src/transforms/smart_crusher/field_detect.rs +0 -393
  451. package/crates/headroom-core/src/transforms/smart_crusher/hashing.rs +0 -72
  452. package/crates/headroom-core/src/transforms/smart_crusher/mod.rs +0 -81
  453. package/crates/headroom-core/src/transforms/smart_crusher/observer.rs +0 -66
  454. package/crates/headroom-core/src/transforms/smart_crusher/orchestration.rs +0 -459
  455. package/crates/headroom-core/src/transforms/smart_crusher/outliers.rs +0 -507
  456. package/crates/headroom-core/src/transforms/smart_crusher/planning.rs +0 -904
  457. package/crates/headroom-core/src/transforms/smart_crusher/statistics.rs +0 -510
  458. package/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs +0 -287
  459. package/crates/headroom-core/src/transforms/smart_crusher/traits.rs +0 -203
  460. package/crates/headroom-core/src/transforms/smart_crusher/types.rs +0 -263
  461. package/crates/headroom-core/src/transforms/tag_protector.rs +0 -1272
  462. package/crates/headroom-core/src/transforms/unidiff_detector.rs +0 -232
  463. package/crates/headroom-parity/Cargo.toml +0 -26
  464. package/crates/headroom-parity/examples/diff_fixture.rs +0 -66
  465. package/crates/headroom-parity/src/bin/parity_run.rs +0 -78
  466. package/crates/headroom-parity/src/lib.rs +0 -613
  467. package/crates/headroom-proxy/Cargo.toml +0 -119
  468. package/crates/headroom-proxy/data/model_prices_and_context_window.json +0 -39767
  469. package/crates/headroom-proxy/src/bedrock/auth_mode_layer.rs +0 -192
  470. package/crates/headroom-proxy/src/bedrock/envelope.rs +0 -239
  471. package/crates/headroom-proxy/src/bedrock/eventstream.rs +0 -769
  472. package/crates/headroom-proxy/src/bedrock/eventstream_to_sse.rs +0 -459
  473. package/crates/headroom-proxy/src/bedrock/invoke.rs +0 -735
  474. package/crates/headroom-proxy/src/bedrock/invoke_streaming.rs +0 -1098
  475. package/crates/headroom-proxy/src/bedrock/mod.rs +0 -66
  476. package/crates/headroom-proxy/src/bedrock/sigv4.rs +0 -324
  477. package/crates/headroom-proxy/src/bedrock/vendor.rs +0 -82
  478. package/crates/headroom-proxy/src/cache_stabilization/anthropic_cache_control.rs +0 -698
  479. package/crates/headroom-proxy/src/cache_stabilization/drift_detector.rs +0 -683
  480. package/crates/headroom-proxy/src/cache_stabilization/mod.rs +0 -61
  481. package/crates/headroom-proxy/src/cache_stabilization/openai_cache_key.rs +0 -600
  482. package/crates/headroom-proxy/src/cache_stabilization/tool_def_normalize.rs +0 -514
  483. package/crates/headroom-proxy/src/cache_stabilization/volatile_detector.rs +0 -720
  484. package/crates/headroom-proxy/src/compression/anthropic.rs +0 -92
  485. package/crates/headroom-proxy/src/compression/live_zone_anthropic.rs +0 -1311
  486. package/crates/headroom-proxy/src/compression/live_zone_openai.rs +0 -660
  487. package/crates/headroom-proxy/src/compression/live_zone_responses.rs +0 -678
  488. package/crates/headroom-proxy/src/compression/mod.rs +0 -127
  489. package/crates/headroom-proxy/src/compression/model_limits.rs +0 -211
  490. package/crates/headroom-proxy/src/config.rs +0 -646
  491. package/crates/headroom-proxy/src/error.rs +0 -69
  492. package/crates/headroom-proxy/src/handlers/chat_completions.rs +0 -100
  493. package/crates/headroom-proxy/src/handlers/conversations.rs +0 -241
  494. package/crates/headroom-proxy/src/handlers/mod.rs +0 -13
  495. package/crates/headroom-proxy/src/handlers/responses.rs +0 -256
  496. package/crates/headroom-proxy/src/headers.rs +0 -288
  497. package/crates/headroom-proxy/src/health.rs +0 -41
  498. package/crates/headroom-proxy/src/lib.rs +0 -21
  499. package/crates/headroom-proxy/src/main.rs +0 -141
  500. package/crates/headroom-proxy/src/observability/cache_hit_rate.rs +0 -340
  501. package/crates/headroom-proxy/src/observability/compression_ratio.rs +0 -158
  502. package/crates/headroom-proxy/src/observability/metric_names.rs +0 -174
  503. package/crates/headroom-proxy/src/observability/mod.rs +0 -68
  504. package/crates/headroom-proxy/src/observability/prometheus.rs +0 -394
  505. package/crates/headroom-proxy/src/observability/proxy_metrics.rs +0 -368
  506. package/crates/headroom-proxy/src/proxy.rs +0 -1619
  507. package/crates/headroom-proxy/src/responses_items.rs +0 -565
  508. package/crates/headroom-proxy/src/sse/anthropic.rs +0 -415
  509. package/crates/headroom-proxy/src/sse/framing.rs +0 -366
  510. package/crates/headroom-proxy/src/sse/mod.rs +0 -40
  511. package/crates/headroom-proxy/src/sse/openai_chat.rs +0 -231
  512. package/crates/headroom-proxy/src/sse/openai_responses.rs +0 -458
  513. package/crates/headroom-proxy/src/vertex/adc.rs +0 -315
  514. package/crates/headroom-proxy/src/vertex/envelope.rs +0 -167
  515. package/crates/headroom-proxy/src/vertex/mod.rs +0 -277
  516. package/crates/headroom-proxy/src/vertex/raw_predict.rs +0 -490
  517. package/crates/headroom-proxy/src/vertex/stream_raw_predict.rs +0 -49
  518. package/crates/headroom-proxy/src/websocket.rs +0 -246
  519. package/crates/headroom-py/Cargo.toml +0 -37
  520. package/crates/headroom-py/build.rs +0 -77
  521. package/crates/headroom-py/glibc_compat.c +0 -170
  522. package/crates/headroom-py/src/lib.rs +0 -1655
  523. package/deny.toml +0 -32
  524. package/docker/differential-network-capture/Dockerfile.runner +0 -14
  525. package/docker/differential-network-capture/docker-compose.yml +0 -142
  526. package/docker/differential-network-capture/run-claude-lane.sh +0 -24
  527. package/docker/docker-compose.native.yml +0 -44
  528. package/docker-bake.hcl +0 -88
  529. package/docker-compose.yml +0 -80
  530. package/e2e/_lib/make_shim.ps1 +0 -24
  531. package/e2e/_lib/make_shim.sh +0 -28
  532. package/e2e/docker-native-install.sh +0 -120
  533. package/e2e/init/Dockerfile +0 -79
  534. package/e2e/wrap/Dockerfile +0 -115
  535. package/examples/07-context-compression.ipynb +0 -279
  536. package/examples/deployment/macos-launchagent/com.headroom.proxy.plist.template +0 -65
  537. package/examples/deployment/macos-launchagent/install.sh +0 -227
  538. package/examples/deployment/macos-launchagent/shell-integration.sh +0 -74
  539. package/examples/deployment/macos-launchagent/uninstall.sh +0 -145
  540. package/headroom-savings.png +0 -0
  541. package/headroom_learn.gif +0 -0
  542. package/llms.txt +0 -67
  543. package/mkdocs.yml +0 -136
  544. package/npm/bin/xithead.js +0 -366
  545. package/npm/bin/xtream-head.js +0 -366
  546. package/npm/index.d.ts +0 -57
  547. package/npm/index.js +0 -139
  548. package/npm/package.json +0 -55
  549. package/npm/postinstall.js +0 -43
  550. package/plugins/headroom-agent-hooks/.claude-plugin/plugin.json +0 -17
  551. package/plugins/headroom-agent-hooks/hooks/hooks.json +0 -29
  552. package/plugins/headroom-oauth2/LICENSE +0 -3
  553. package/plugins/headroom-oauth2/SPEC.md +0 -115
  554. package/plugins/headroom-oauth2/pyproject.toml +0 -48
  555. package/plugins/hermes/headroom_retrieve/plugin.yaml +0 -6
  556. package/plugins/openclaw/hook-shim/HOOK.md +0 -21
  557. package/plugins/openclaw/hook-shim/handler.js +0 -3
  558. package/plugins/openclaw/openclaw.plugin.json +0 -98
  559. package/plugins/openclaw/package-lock.json +0 -2692
  560. package/plugins/openclaw/package.json +0 -54
  561. package/plugins/openclaw/prepare-dist.mjs +0 -56
  562. package/plugins/openclaw/src/convert.ts +0 -418
  563. package/plugins/openclaw/src/engine.ts +0 -284
  564. package/plugins/openclaw/src/gateway-config.ts +0 -143
  565. package/plugins/openclaw/src/index.ts +0 -11
  566. package/plugins/openclaw/src/plugin/index.ts +0 -98
  567. package/plugins/openclaw/src/proxy-manager.ts +0 -477
  568. package/plugins/openclaw/src/tools/headroom-retrieve.ts +0 -76
  569. package/plugins/openclaw/test/convert.test.ts +0 -96
  570. package/plugins/openclaw/test/engine-normalization.test.ts +0 -21
  571. package/plugins/openclaw/test/engine.test.ts +0 -101
  572. package/plugins/openclaw/test/gateway-config.test.ts +0 -278
  573. package/plugins/openclaw/test/plugin-runtime-routing.test.ts +0 -141
  574. package/plugins/openclaw/test/proxy-manager.test.ts +0 -343
  575. package/plugins/openclaw/tsup.config.ts +0 -9
  576. package/pyproject.toml +0 -438
  577. package/rust-toolchain.toml +0 -16
  578. package/sdk/typescript/examples/basic-compress.ts +0 -64
  579. package/sdk/typescript/examples/ccr-retrieve.ts +0 -80
  580. package/sdk/typescript/examples/hooks-custom-compression.ts +0 -117
  581. package/sdk/typescript/examples/middleware-composition.ts +0 -59
  582. package/sdk/typescript/examples/multi-provider.ts +0 -53
  583. package/sdk/typescript/examples/openai-anthropic-adapters.ts +0 -73
  584. package/sdk/typescript/examples/shared-context-multi-agent.ts +0 -74
  585. package/sdk/typescript/examples/simulation-dry-run.ts +0 -80
  586. package/sdk/typescript/examples/streaming-chat.ts +0 -71
  587. package/sdk/typescript/examples/structured-output.ts +0 -99
  588. package/sdk/typescript/examples/tool-calling-agent.ts +0 -86
  589. package/sdk/typescript/examples/with-headroom-vercel.ts +0 -57
  590. package/sdk/typescript/package-lock.json +0 -3437
  591. package/sdk/typescript/package.json +0 -108
  592. package/sdk/typescript/src/adapters/anthropic.ts +0 -218
  593. package/sdk/typescript/src/adapters/gemini.ts +0 -83
  594. package/sdk/typescript/src/adapters/openai.ts +0 -77
  595. package/sdk/typescript/src/adapters/vercel-ai.ts +0 -131
  596. package/sdk/typescript/src/client.ts +0 -632
  597. package/sdk/typescript/src/compress.ts +0 -87
  598. package/sdk/typescript/src/errors.ts +0 -114
  599. package/sdk/typescript/src/hooks.ts +0 -113
  600. package/sdk/typescript/src/index.ts +0 -140
  601. package/sdk/typescript/src/paths.ts +0 -272
  602. package/sdk/typescript/src/shared-context.ts +0 -188
  603. package/sdk/typescript/src/simulate.ts +0 -52
  604. package/sdk/typescript/src/types/config.ts +0 -183
  605. package/sdk/typescript/src/types/models.ts +0 -314
  606. package/sdk/typescript/src/types.ts +0 -139
  607. package/sdk/typescript/src/utils/case.ts +0 -34
  608. package/sdk/typescript/src/utils/format.ts +0 -480
  609. package/sdk/typescript/src/utils/stream.ts +0 -50
  610. package/sdk/typescript/test/adapters/anthropic.test.ts +0 -217
  611. package/sdk/typescript/test/adapters/gemini.test.ts +0 -131
  612. package/sdk/typescript/test/adapters/openai.test.ts +0 -174
  613. package/sdk/typescript/test/adapters/vercel-ai.test.ts +0 -306
  614. package/sdk/typescript/test/client-expanded.test.ts +0 -333
  615. package/sdk/typescript/test/client.test.ts +0 -266
  616. package/sdk/typescript/test/compress.test.ts +0 -118
  617. package/sdk/typescript/test/config.test.ts +0 -275
  618. package/sdk/typescript/test/errors.test.ts +0 -169
  619. package/sdk/typescript/test/format-integration.test.ts +0 -252
  620. package/sdk/typescript/test/hooks.test.ts +0 -265
  621. package/sdk/typescript/test/integration.test.ts +0 -249
  622. package/sdk/typescript/test/mdx-examples.test.ts +0 -253
  623. package/sdk/typescript/test/models.test.ts +0 -318
  624. package/sdk/typescript/test/parity.test.ts +0 -337
  625. package/sdk/typescript/test/paths.test.ts +0 -447
  626. package/sdk/typescript/test/shared-context.test.ts +0 -245
  627. package/sdk/typescript/test/simulate.test.ts +0 -84
  628. package/sdk/typescript/test/types.test.ts +0 -244
  629. package/sdk/typescript/test/utils/case.test.ts +0 -129
  630. package/sdk/typescript/test/utils/detect-format.test.ts +0 -133
  631. package/sdk/typescript/test/utils/format.test.ts +0 -434
  632. package/sdk/typescript/test/vercel-ai-e2e.test.ts +0 -179
  633. package/sdk/typescript/tsup.config.ts +0 -17
  634. package/uv.lock +0 -6951
  635. package/xithead.egg-info/PKG-INFO +0 -22
  636. package/xithead.egg-info/SOURCES.txt +0 -16
  637. package/xithead.egg-info/dependency_links.txt +0 -1
  638. package/xithead.egg-info/entry_points.txt +0 -2
  639. package/xithead.egg-info/requires.txt +0 -10
  640. package/xithead.egg-info/top_level.txt +0 -2
  641. package/xithead.ps1 +0 -136
  642. package/xithead.sh +0 -129
  643. package/xithead_dist/pyproject.toml +0 -39
  644. package/xithead_dist/xithead.egg-info/PKG-INFO +0 -22
  645. package/xithead_dist/xithead.egg-info/SOURCES.txt +0 -16
  646. package/xithead_dist/xithead.egg-info/dependency_links.txt +0 -1
  647. package/xithead_dist/xithead.egg-info/entry_points.txt +0 -3
  648. package/xithead_dist/xithead.egg-info/requires.txt +0 -10
  649. package/xithead_dist/xithead.egg-info/top_level.txt +0 -2
@@ -1,1665 +0,0 @@
1
- //! `SmartCrusher` struct — top-level entry point for compression.
2
- //!
3
- //! Owns the `config`, `anchor_selector`, `scorer`, and `analyzer`
4
- //! singletons that every per-message call needs. Constructed once
5
- //! per process; the struct is `Send + Sync` so it can sit behind an
6
- //! `Arc` in a multi-threaded proxy.
7
- //!
8
- //! This module ports three Python entry points:
9
- //!
10
- //! - `_execute_plan` (line 3617) → `SmartCrusher::execute_plan`
11
- //! - `_crush_array` (line 2400) → `SmartCrusher::crush_array`
12
- //! - `_crush_mixed_array` (line 2914) → `SmartCrusher::crush_mixed_array`
13
- //!
14
- //! # Stubs that match Python's "everything-disabled" path
15
- //!
16
- //! Python's `_crush_array` calls into TOIN (cross-user pattern
17
- //! learning), feedback (per-tool compression hints), CCR (compress-
18
- //! cache-retrieve store), and telemetry. All four are large separate
19
- //! systems with their own state. For the like-for-like port at Stage
20
- //! 3c.1, we mirror Python's behavior **when those subsystems are
21
- //! disabled**:
22
- //!
23
- //! - **TOIN**: never produces a recommendation, never overrides
24
- //! `effective_max_items`, never injects preserve_fields/strategy/level.
25
- //! - **Feedback**: never produces hints; default `effective_max_items`.
26
- //! - **CCR**: `enabled=false`; result has `ccr_hash = None`.
27
- //! - **Telemetry**: no-op.
28
- //! - **`_compress_text_within_items`**: pass-through (returns input
29
- //! unchanged) since text compression has its own port pipeline.
30
- //! - **`summarize_dropped_items`**: empty string.
31
- //!
32
- //! Parity fixtures will be recorded with all four disabled on the
33
- //! Python side, locking byte-equal output. The TOIN/CCR/feedback
34
- //! integration ports happen later (Stage 3c.2 follow-ups).
35
-
36
- use std::sync::Arc;
37
-
38
- use serde_json::Value;
39
-
40
- use super::analyzer::SmartAnalyzer;
41
- use super::builder::SmartCrusherBuilder;
42
- use super::classifier::{classify_array, ArrayType};
43
- use super::compaction::{
44
- classify_cell, emit_opaque_ccr_marker, try_parse_json_container, CellClass, ClassifyConfig,
45
- CompactConfig, Compaction, CompactionStage,
46
- };
47
- use super::config::SmartCrusherConfig;
48
- use super::crushers::{compute_k_split, crush_number_array, crush_object, crush_string_array};
49
- use super::planning::SmartCrusherPlanner;
50
- use super::traits::{Constraint, CrushEvent, Observer};
51
- use super::types::{CompressionPlan, CompressionStrategy, CrushResult};
52
- use crate::ccr::CcrStore;
53
- use crate::relevance::RelevanceScorer;
54
- use crate::transforms::adaptive_sizer::compute_optimal_k;
55
- use crate::transforms::anchor_selector::AnchorSelector;
56
-
57
- /// Return type for `crush_array`.
58
- ///
59
- /// Two operating paths feed the same result type:
60
- ///
61
- /// - **Lossless path** — input compacted to a smaller inline form
62
- /// (e.g. CSV+schema). Nothing dropped; `compacted` is populated;
63
- /// `ccr_hash` is `None` (no retrieval needed because everything is
64
- /// already in the prompt).
65
- /// - **Lossy path** — input compressed by row-dropping. `items` holds
66
- /// the kept subset; `ccr_hash` is `Some(hash)` so the runtime can
67
- /// cache the **full original** keyed by that hash and serve it back
68
- /// to the LLM via a retrieval tool call. **No data is lost** —
69
- /// "lossy" here means "compressed view inline; full payload cached
70
- /// for tool retrieval," matching Python's CCR-Dropped semantics.
71
- ///
72
- /// The runtime (PyO3 bridge / proxy server) owns the cache; this crate
73
- /// computes the hash and emits a marker so the prompt knows where to
74
- /// look.
75
- pub struct CrushArrayResult {
76
- /// Kept items. For the lossless path this is the full original
77
- /// (nothing was dropped). For the lossy path this is the surviving
78
- /// subset; the rest is retrievable via `ccr_hash`.
79
- pub items: Vec<Value>,
80
- /// Strategy debug string. One of:
81
- /// - `"none:adaptive_at_limit"` / `"skip:<reason>"` — passthrough
82
- /// - `"lossless:table"` / `"lossless:buckets"` — lossless wins
83
- /// - `"smart_sample"` / `"top_n"` / `"cluster"` / `"time_series"` —
84
- /// lossy path with row-dropping.
85
- pub strategy_info: String,
86
- /// 12-char SHA-256 hex prefix of the **full original input**.
87
- /// Populated when the lossy path dropped rows; the runtime is
88
- /// expected to cache the original items keyed by this hash so a
89
- /// retrieval tool can serve them back. `None` when nothing was
90
- /// dropped (lossless path or below adaptive_k boundary).
91
- pub ccr_hash: Option<String>,
92
- /// Marker text inserted into the prompt to advertise the CCR
93
- /// pointer (e.g. `<<ccr:abc123def456 42_rows_offloaded>>`). Empty
94
- /// when `ccr_hash` is `None`.
95
- pub dropped_summary: String,
96
- /// Rendered bytes from the compaction stage when the **lossless
97
- /// path** won. `None` for the lossy path or when compaction wasn't
98
- /// configured.
99
- pub compacted: Option<String>,
100
- /// Top-level [`Compaction`] variant tag — `"table"`, `"buckets"`,
101
- /// `"ccr"`. Mirrors `compacted` — populated only when lossless won.
102
- pub compaction_kind: Option<&'static str>,
103
- }
104
-
105
- /// Top-level SmartCrusher.
106
- ///
107
- /// Three pluggable extensions (Stage 3c.2 PR1):
108
- /// - `scorer` — relevance scoring (`HybridScorer` by default).
109
- /// - `constraints` — must-keep predicates (`KeepErrorsConstraint` +
110
- /// `KeepStructuralOutliersConstraint` by default).
111
- /// - `observers` — decision-stream telemetry (`TracingObserver` by
112
- /// default).
113
- ///
114
- /// Compose via [`SmartCrusherBuilder`]; or call `SmartCrusher::new()`
115
- /// for the OSS default composition.
116
- pub struct SmartCrusher {
117
- pub config: SmartCrusherConfig,
118
- pub anchor_selector: AnchorSelector,
119
- pub scorer: Box<dyn RelevanceScorer + Send + Sync>,
120
- pub analyzer: SmartAnalyzer,
121
- pub constraints: Vec<Box<dyn Constraint>>,
122
- pub observers: Vec<Box<dyn Observer>>,
123
- /// Optional lossless-first compaction stage (Stage 3c.2 PR2). When
124
- /// set, `crush_array` runs compaction up front and short-circuits
125
- /// the lossy path on success. When `None` (default OSS), parity
126
- /// with the pre-PR2 lossy-only pipeline is preserved exactly.
127
- pub compaction: Option<CompactionStage>,
128
- /// Optional CCR store. When set, the lossy path stashes the **full
129
- /// original** array into the store keyed by `ccr_hash` before
130
- /// returning — the runtime can then serve dropped rows back via
131
- /// retrieval tool calls. When `None`, hashes are still emitted but
132
- /// nothing is stored (legacy / parity mode).
133
- ///
134
- /// `Arc` so callers can keep their own handle to the same store
135
- /// (e.g. the proxy server holds it for retrieval lookups while
136
- /// SmartCrusher writes through it).
137
- pub ccr_store: Option<Arc<dyn CcrStore>>,
138
- }
139
-
140
- impl SmartCrusher {
141
- /// Construct with the OSS default composition: scorer + constraints +
142
- /// observer + **lossless-first compaction stage**. Calling
143
- /// `crush_array` runs the dispatch:
144
- ///
145
- /// 1. Try the lossless compactor.
146
- /// 2. If savings ratio ≥ `config.lossless_min_savings_ratio`
147
- /// (default `0.30`), ship lossless — `compacted` populated,
148
- /// `ccr_hash = None`, nothing dropped.
149
- /// 3. Otherwise fall through to the lossy path — drop rows,
150
- /// populate `ccr_hash` with a hash of the full original so the
151
- /// runtime can cache the payload for tool retrieval.
152
- ///
153
- /// **No data is ever lost.** The lossy path moves dropped rows to
154
- /// CCR cache, not to nowhere — same semantics as Python's
155
- /// SmartCrusher with CCR enabled.
156
- pub fn new(config: SmartCrusherConfig) -> Self {
157
- // Carry the compaction heuristics from the crusher config into
158
- // the compaction stage; everything not exposed on
159
- // SmartCrusherConfig keeps its CompactConfig default.
160
- let compact_cfg = CompactConfig {
161
- core_field_fraction: config.compaction_core_field_fraction,
162
- heterogeneous_core_ratio: config.compaction_heterogeneous_core_ratio,
163
- max_flatten_inner_keys: config.compaction_max_flatten_inner_keys,
164
- min_buckets: config.compaction_min_buckets,
165
- max_buckets: config.compaction_max_buckets,
166
- ..CompactConfig::default()
167
- };
168
- SmartCrusherBuilder::new(config)
169
- .with_default_oss_setup()
170
- .with_compaction(CompactionStage::csv_schema(compact_cfg))
171
- .with_default_ccr_store()
172
- .build()
173
- }
174
-
175
- /// Construct WITHOUT the compaction stage. Pre-PR4 behavior:
176
- /// `crush_array` skips the lossless attempt and runs the lossy
177
- /// path directly (still with CCR-Dropped retrieval markers from
178
- /// PR4). Used by:
179
- ///
180
- /// - The 17 legacy parity fixtures (recorded against the
181
- /// lossy-only path; using this constructor preserves byte-equal
182
- /// coverage).
183
- /// - Callers who explicitly don't want lossless attempts (e.g.
184
- /// workloads where the compactor's overhead isn't worth the
185
- /// modest tabular wins).
186
- pub fn without_compaction(config: SmartCrusherConfig) -> Self {
187
- SmartCrusherBuilder::new(config)
188
- .with_default_oss_setup()
189
- .with_default_ccr_store()
190
- .build()
191
- }
192
-
193
- /// Construct like [`SmartCrusher::new`] but with the compaction
194
- /// stage's formatter chosen by name (`"csv-schema"`, `"json"`,
195
- /// `"markdown-kv"`). `None` for unknown names — callers own the
196
- /// fallback/error policy. `"csv-schema"` is equivalent to `new`.
197
- pub fn with_compaction_format(config: SmartCrusherConfig, format_name: &str) -> Option<Self> {
198
- let stage = CompactionStage::from_format_name(format_name)?;
199
- Some(
200
- SmartCrusherBuilder::new(config)
201
- .with_default_oss_setup()
202
- .with_compaction(stage)
203
- .with_default_ccr_store()
204
- .build(),
205
- )
206
- }
207
-
208
- /// Begin a builder chain for custom composition. The Enterprise
209
- /// entry point: swap the scorer, add business-rule constraints,
210
- /// attach an audit observer.
211
- pub fn builder(config: SmartCrusherConfig) -> SmartCrusherBuilder {
212
- SmartCrusherBuilder::new(config)
213
- }
214
-
215
- /// Construct with a custom scorer (legacy convenience). Equivalent
216
- /// to `SmartCrusher::builder(config).with_scorer(scorer).with_default_oss_setup().build()`
217
- /// minus the default scorer override; preserved for backward
218
- /// compatibility with pre-PR1 callers.
219
- pub fn with_scorer(
220
- config: SmartCrusherConfig,
221
- scorer: Box<dyn RelevanceScorer + Send + Sync>,
222
- ) -> Self {
223
- SmartCrusherBuilder::new(config)
224
- .with_scorer(scorer)
225
- .add_default_oss_constraints()
226
- .build()
227
- }
228
-
229
- /// Construct directly from owned parts. Used by
230
- /// [`SmartCrusherBuilder::build`] — not part of the public stable
231
- /// API. Prefer the builder.
232
- #[doc(hidden)]
233
- #[allow(clippy::too_many_arguments)]
234
- pub fn from_parts(
235
- config: SmartCrusherConfig,
236
- anchor_selector: AnchorSelector,
237
- scorer: Box<dyn RelevanceScorer + Send + Sync>,
238
- analyzer: SmartAnalyzer,
239
- constraints: Vec<Box<dyn Constraint>>,
240
- observers: Vec<Box<dyn Observer>>,
241
- compaction: Option<CompactionStage>,
242
- ccr_store: Option<Arc<dyn CcrStore>>,
243
- ) -> Self {
244
- SmartCrusher {
245
- config,
246
- anchor_selector,
247
- scorer,
248
- analyzer,
249
- constraints,
250
- observers,
251
- compaction,
252
- ccr_store,
253
- }
254
- }
255
-
256
- /// Handle to the CCR store, if configured. Used by the runtime
257
- /// (proxy server, PyO3 bridge) to look up originals when retrieval
258
- /// tool calls fire.
259
- pub fn ccr_store(&self) -> Option<&Arc<dyn CcrStore>> {
260
- self.ccr_store.as_ref()
261
- }
262
-
263
- fn planner(&self) -> SmartCrusherPlanner<'_> {
264
- SmartCrusherPlanner::new(
265
- &self.config,
266
- &self.anchor_selector,
267
- &*self.scorer,
268
- &self.analyzer,
269
- &self.constraints,
270
- )
271
- }
272
-
273
- /// Execute a `CompressionPlan` against `items`, returning the
274
- /// kept-items list in original-array order. Mirrors Python's
275
- /// `_execute_plan` (line 3617-3633).
276
- ///
277
- /// Schema-preserving by default: each kept item is cloned unchanged.
278
- /// No summary objects, generated fields, or wrapper metadata.
279
- ///
280
- /// When `factor_out_constants` is enabled (default off), fields the
281
- /// analyzer found constant across ALL items are stripped from each
282
- /// kept object and emitted once in a leading
283
- /// `{"_constant_fields": {...}}` sentinel — same output-shape
284
- /// convention as the `_ccr_dropped` sentinel. Stripping is
285
- /// defensive: a key is only removed from an item when its value
286
- /// equals the recorded constant, so a drifted item keeps its own
287
- /// value. The CCR store always holds the full unfactored original.
288
- pub fn execute_plan(&self, plan: &CompressionPlan, items: &[Value]) -> Vec<Value> {
289
- let mut indices = plan.keep_indices.clone();
290
- indices.sort_unstable();
291
- let mut kept: Vec<Value> = indices
292
- .into_iter()
293
- .filter(|&idx| idx < items.len())
294
- .map(|idx| items[idx].clone())
295
- .collect();
296
-
297
- if self.config.factor_out_constants && !plan.constant_fields.is_empty() && kept.len() >= 2 {
298
- let mut any_stripped = false;
299
- for item in kept.iter_mut() {
300
- if let Value::Object(map) = item {
301
- for (key, constant) in &plan.constant_fields {
302
- if map.get(key) == Some(constant) {
303
- map.remove(key);
304
- any_stripped = true;
305
- }
306
- }
307
- }
308
- }
309
- if any_stripped {
310
- let mut sentinel = serde_json::Map::new();
311
- sentinel.insert(
312
- "_constant_fields".to_string(),
313
- Value::Object(plan.constant_fields.clone().into_iter().collect()),
314
- );
315
- kept.insert(0, Value::Object(sentinel));
316
- }
317
- }
318
-
319
- kept
320
- }
321
-
322
- /// Top-level entry point. Mirrors Python `SmartCrusher.crush`
323
- /// (line 1581-1603) — used by `ContentRouter` when routing JSON
324
- /// arrays.
325
- ///
326
- /// Parses `content` as JSON, recursively processes it (compressing
327
- /// arrays at every depth via the appropriate per-type crusher),
328
- /// then re-serializes with Python-compatible formatting (`, ` and
329
- /// `: ` separators, ASCII-escaped non-ASCII).
330
- ///
331
- /// Returns a `CrushResult` with:
332
- /// - `compressed`: the re-serialized JSON.
333
- /// - `original`: the input string (unmodified).
334
- /// - `was_modified`: whether `compressed` differs from `content`'s
335
- /// trimmed form.
336
- /// - `strategy`: combined strategy info from all crushed arrays
337
- /// (or `"passthrough"`).
338
- pub fn crush(&self, content: &str, query: &str, bias: f64) -> CrushResult {
339
- let start = std::time::Instant::now();
340
- let (compressed, was_modified, info) = self.smart_crush_content(content, query, bias);
341
- let strategy = if info.is_empty() {
342
- "passthrough".to_string()
343
- } else {
344
- info
345
- };
346
-
347
- // Fire one event per top-level crush. Cheap when no observers
348
- // are configured (`for o in &[]` is a single null-pointer
349
- // check); cheap when only `TracingObserver` is configured if
350
- // the subscriber filters `debug` out (the default in
351
- // production). Custom observers — audit logs, Loop training
352
- // stream, metrics — pay whatever they pay.
353
- if !self.observers.is_empty() {
354
- let event = CrushEvent {
355
- strategy: strategy.clone(),
356
- input_bytes: content.len(),
357
- output_bytes: compressed.len(),
358
- elapsed_ns: start.elapsed().as_nanos() as u64,
359
- was_modified,
360
- };
361
- for observer in &self.observers {
362
- observer.on_event(&event);
363
- }
364
- }
365
-
366
- CrushResult {
367
- compressed,
368
- original: content.to_string(),
369
- was_modified,
370
- strategy,
371
- }
372
- }
373
-
374
- /// `SmartCrusher._smart_crush_content` (Python line 2243-2301).
375
- /// JSON-parse, recursively process, re-serialize. CCR marker
376
- /// injection is stubbed (CCR is disabled in this stage).
377
- ///
378
- /// Returns `(crushed_content, was_modified, info)`.
379
- pub fn smart_crush_content(
380
- &self,
381
- content: &str,
382
- query_context: &str,
383
- bias: f64,
384
- ) -> (String, bool, String) {
385
- // Parse — non-JSON content passes through unchanged.
386
- let Ok(parsed) = serde_json::from_str::<Value>(content) else {
387
- return (content.to_string(), false, String::new());
388
- };
389
-
390
- let (crushed, info) = self.process_value(&parsed, 0, query_context, bias);
391
-
392
- // Re-serialize with Python `safe_json_dumps` formatting:
393
- // compact `(",", ":")` separators + `ensure_ascii=False`,
394
- // preserving object-key insertion order. Matches the Python
395
- // SmartCrusher output bytes the proxy writes.
396
- let result = crate::transforms::anchor_selector::python_safe_json_dumps(&crushed);
397
- let was_modified = result != content.trim();
398
- (result, was_modified, info)
399
- }
400
-
401
- /// Maximum recursion depth for nested JSON. Mirrors Python's
402
- /// `_MAX_PROCESS_DEPTH = 50`. Beyond this, values are returned as-is.
403
- const MAX_PROCESS_DEPTH: usize = 50;
404
-
405
- /// Recursively process a value, crushing arrays where appropriate.
406
- /// Mirrors Python `_process_value` (line 2307-2398).
407
- ///
408
- /// Returns `(processed_value, info_string)`. CCR markers are
409
- /// stubbed (Python's tuple has a third element for them — Rust's
410
- /// version omits since we never produce markers in this stage).
411
- pub fn process_value(
412
- &self,
413
- value: &Value,
414
- depth: usize,
415
- query_context: &str,
416
- bias: f64,
417
- ) -> (Value, String) {
418
- if depth >= Self::MAX_PROCESS_DEPTH {
419
- return (value.clone(), String::new());
420
- }
421
-
422
- let mut info_parts: Vec<String> = Vec::new();
423
-
424
- match value {
425
- Value::Array(arr) => {
426
- let n = arr.len();
427
- if n >= self.config.min_items_to_analyze {
428
- let arr_type = classify_array(arr);
429
- match arr_type {
430
- ArrayType::DictArray => {
431
- let result = self.crush_array(arr, query_context, bias);
432
- // Lossless path won → substitute the array
433
- // with the compacted string in place. This
434
- // makes the lossless win visible to the
435
- // public `crush()` API: the output JSON
436
- // has a string where the array used to be.
437
- // The wrapping JSON structure is preserved.
438
- if let Some(rendered) = result.compacted {
439
- info_parts.push(format!(
440
- "{}({}->len={})",
441
- result.strategy_info,
442
- n,
443
- rendered.len()
444
- ));
445
- return (Value::String(rendered), info_parts.join(","));
446
- }
447
- info_parts.push(format!(
448
- "{}({}->{})",
449
- result.strategy_info,
450
- n,
451
- result.items.len()
452
- ));
453
- // Lossy path with rows dropped → append a
454
- // CCR-Dropped sentinel object as the last
455
- // element of the kept-items array. This is
456
- // the **only** place the LLM sees the
457
- // `<<ccr:HASH ...>>` pointer in the prompt.
458
- // Without this, the store has the data but
459
- // no model can ever ask for it.
460
- //
461
- // Sentinel shape: `{"_ccr_dropped":
462
- // "<<ccr:HASH N_rows_offloaded>>"}` —
463
- // preserves "array-of-objects" shape so
464
- // downstream consumers iterating with
465
- // `x.get(...)` keep working; the well-known
466
- // `_ccr_dropped` key signals metadata
467
- // unambiguously.
468
- let mut items = result.items;
469
- if !result.dropped_summary.is_empty() {
470
- let mut sentinel = serde_json::Map::new();
471
- sentinel.insert(
472
- "_ccr_dropped".to_string(),
473
- Value::String(result.dropped_summary),
474
- );
475
- items.push(Value::Object(sentinel));
476
- }
477
- return (Value::Array(items), info_parts.join(","));
478
- }
479
- ArrayType::StringArray => {
480
- let strs: Vec<&str> = arr.iter().filter_map(|v| v.as_str()).collect();
481
- let (crushed, strategy) = crush_string_array(&strs, &self.config, bias);
482
- info_parts.push(format!("{}({}->{})", strategy, n, crushed.len()));
483
- let crushed_values: Vec<Value> =
484
- crushed.into_iter().map(Value::String).collect();
485
- return (Value::Array(crushed_values), info_parts.join(","));
486
- }
487
- ArrayType::NumberArray => {
488
- let (crushed, strategy) = crush_number_array(arr, &self.config, bias);
489
- info_parts.push(format!("{}({}->{})", strategy, n, crushed.len()));
490
- return (Value::Array(crushed), info_parts.join(","));
491
- }
492
- ArrayType::MixedArray => {
493
- let (crushed, strategy) =
494
- self.crush_mixed_array(arr, query_context, bias);
495
- info_parts.push(format!("{}({}->{})", strategy, n, crushed.len()));
496
- return (Value::Array(crushed), info_parts.join(","));
497
- }
498
- // NestedArray, BoolArray, Empty → fall through
499
- // to recursive descent.
500
- _ => {}
501
- }
502
- }
503
-
504
- // Below threshold or not crushable → recurse into items.
505
- let mut processed: Vec<Value> = Vec::with_capacity(n);
506
- for item in arr {
507
- let (p_item, p_info) = self.process_value(item, depth + 1, query_context, bias);
508
- processed.push(p_item);
509
- if !p_info.is_empty() {
510
- info_parts.push(p_info);
511
- }
512
- }
513
- (Value::Array(processed), info_parts.join(","))
514
- }
515
- Value::Object(map) => {
516
- // First pass: recurse into values to compress nested arrays.
517
- let mut processed = serde_json::Map::new();
518
- for (k, v) in map {
519
- let (p_val, p_info) = self.process_value(v, depth + 1, query_context, bias);
520
- processed.insert(k.clone(), p_val);
521
- if !p_info.is_empty() {
522
- info_parts.push(p_info);
523
- }
524
- }
525
-
526
- // Second pass: if the object itself has many keys,
527
- // compress at the key level.
528
- if processed.len() >= self.config.min_items_to_analyze {
529
- let (crushed_dict, strategy) = crush_object(&processed, &self.config, bias);
530
- if strategy != "object:passthrough" {
531
- info_parts.push(strategy);
532
- return (Value::Object(crushed_dict), info_parts.join(","));
533
- }
534
- }
535
-
536
- (Value::Object(processed), info_parts.join(","))
537
- }
538
- // Strings: walker-equivalent handling. Delegates to
539
- // `process_string` which parses stringified-JSON containers
540
- // (recursing through `process_value`) and CCR-substitutes
541
- // opaque blobs (with store-write so retrieval works).
542
- Value::String(s) => self.process_string(s, depth, query_context, bias),
543
- // Other scalars — passthrough.
544
- _ => (value.clone(), String::new()),
545
- }
546
- }
547
-
548
- /// Walker-equivalent string handling. Mirrors `walker::walk_string`
549
- /// in `compaction/walker.rs` but lives on `SmartCrusher` so the
550
- /// public `crush()` path picks it up.
551
- ///
552
- /// Two cases:
553
- /// 1. **Stringified-JSON.** Strings that parse to a JSON object or
554
- /// array → recurse via `process_value`, then re-emit the result
555
- /// as a compact JSON string. The wrapping string is preserved
556
- /// (so the parent JSON shape stays a string-typed field), but
557
- /// its contents are processed end-to-end.
558
- /// 2. **Opaque blobs.** Strings classified as
559
- /// [`CellClass::Opaque`] (long base64 / HTML / long-text) →
560
- /// substitute with a `<<ccr:HASH,KIND,SIZE>>` marker. Same
561
- /// format as `compaction::walker::format_ccr_marker` so
562
- /// downstream consumers can pattern-match markers regardless
563
- /// of which path emitted them.
564
- fn process_string(
565
- &self,
566
- s: &str,
567
- depth: usize,
568
- query_context: &str,
569
- bias: f64,
570
- ) -> (Value, String) {
571
- // 1. Stringified-JSON: parse, recurse, re-render.
572
- if let Some(parsed) = try_parse_json_container(s) {
573
- let (processed, sub_info) = self.process_value(&parsed, depth + 1, query_context, bias);
574
- // If recursion produced something different, re-emit.
575
- // Special case: if the recursion returned a `Value::String`
576
- // (lossless compaction substituted the array with a
577
- // rendered CSV+schema string), use that string directly.
578
- // Re-encoding it as JSON would produce a quoted string
579
- // literal — double-encoded — which is not what callers
580
- // expect in the wrapping field.
581
- if processed != parsed {
582
- let rendered = match &processed {
583
- Value::String(rendered_str) => rendered_str.clone(),
584
- _ => serde_json::to_string(&processed).unwrap_or_else(|_| s.to_string()),
585
- };
586
- let info = if sub_info.is_empty() {
587
- "string_json".to_string()
588
- } else {
589
- format!("string_json[{sub_info}]")
590
- };
591
- return (Value::String(rendered), info);
592
- }
593
- }
594
-
595
- // 2. Opaque blob: substitute with CCR marker AND stash the
596
- // original in the store (PR8) so retrieval works. Hash + format
597
- // identical to walker.rs via the shared helper — zero drift.
598
- let cfg = ClassifyConfig::default();
599
- if let CellClass::Opaque(kind) = classify_cell(&Value::String(s.to_string()), &cfg) {
600
- let marker = emit_opaque_ccr_marker(s, &kind, self.ccr_store.as_ref());
601
- let kind_label = opaque_kind_label(&kind);
602
- return (Value::String(marker), format!("string_ccr:{kind_label}"));
603
- }
604
-
605
- // 3. Plain string — passthrough.
606
- (Value::String(s.to_string()), String::new())
607
- }
608
-
609
- /// Compress an array of dict items.
610
- ///
611
- /// Direct port of `_crush_array` (Python line 2400-2687) with the
612
- /// optional subsystems (TOIN / CCR / feedback / telemetry) wired
613
- /// in their disabled-by-default behavior. See module-level docs
614
- /// for the rationale.
615
- ///
616
- /// # Pipeline
617
- ///
618
- /// 1. Compute `item_strings` once (used as input to adaptive
619
- /// sizing and downstream relevance scoring).
620
- /// 2. `compute_optimal_k` → `adaptive_k`.
621
- /// 3. If `n <= adaptive_k`, return passthrough.
622
- /// 4. `analyzer.analyze_array(items)` → `analysis`.
623
- /// 5. If `analysis.recommended_strategy == Skip`, return passthrough
624
- /// with a `skip:<reason>` strategy string.
625
- /// 6. `planner.create_plan(analysis, items, query_context, ...)`.
626
- /// 7. `execute_plan(plan, items)` → result.
627
- /// 8. Strategy info = `analysis.recommended_strategy.as_str()`.
628
- pub fn crush_array(&self, items: &[Value], query_context: &str, bias: f64) -> CrushArrayResult {
629
- let item_strings: Vec<String> = items
630
- .iter()
631
- .map(|i| serde_json::to_string(i).unwrap_or_default())
632
- .collect();
633
- let item_str_refs: Vec<&str> = item_strings.iter().map(|s| s.as_str()).collect();
634
-
635
- let max_k = if self.config.max_items_after_crush > 0 {
636
- Some(self.config.max_items_after_crush)
637
- } else {
638
- None
639
- };
640
- let adaptive_k = compute_optimal_k(&item_str_refs, bias, 3, max_k);
641
-
642
- // Tier-1 boundary: array already small enough — passthrough,
643
- // nothing to compact, nothing to drop.
644
- if items.len() <= adaptive_k {
645
- return CrushArrayResult {
646
- items: items.to_vec(),
647
- strategy_info: "none:adaptive_at_limit".to_string(),
648
- ccr_hash: None,
649
- dropped_summary: String::new(),
650
- compacted: None,
651
- compaction_kind: None,
652
- };
653
- }
654
-
655
- // ── Lossless-first attempt ──
656
- //
657
- // Run the compaction stage if present, then check the savings
658
- // ratio against `config.lossless_min_savings_ratio`. If the
659
- // lossless rendering shrinks the input by at least that much,
660
- // ship it — nothing dropped, no CCR retrieval needed.
661
- // Otherwise fall through to the lossy path.
662
- if let Some(stage) = &self.compaction {
663
- let (c, rendered) = stage.run(items);
664
- if c.was_compacted() {
665
- let input_bytes = estimate_array_bytes(&item_strings);
666
- let savings_ratio = if input_bytes > 0 {
667
- 1.0 - (rendered.len() as f64 / input_bytes as f64)
668
- } else {
669
- 0.0
670
- };
671
- if savings_ratio >= self.config.lossless_min_savings_ratio {
672
- let kind = compaction_kind_str(&c);
673
- return CrushArrayResult {
674
- items: items.to_vec(), // nothing dropped
675
- strategy_info: format!("lossless:{kind}"),
676
- ccr_hash: None,
677
- dropped_summary: String::new(),
678
- compacted: Some(rendered),
679
- compaction_kind: Some(kind),
680
- };
681
- }
682
- }
683
- }
684
-
685
- // ── Lossy path: compress inline + cache full original via CCR ──
686
- //
687
- // The runtime caller (PyO3 bridge / proxy server) is expected
688
- // to stash the full input keyed by `ccr_hash` so a retrieval
689
- // tool can serve dropped rows back to the LLM on demand.
690
- // **No data is lost** — "lossy" here means "compressed view
691
- // inline; full payload retrievable via CCR cache."
692
-
693
- let effective_max_items = adaptive_k;
694
- let analysis = self.analyzer.analyze_array(items);
695
-
696
- // Crushability gate: not safe to crush → passthrough, no CCR.
697
- if analysis.recommended_strategy == CompressionStrategy::Skip {
698
- let reason = match &analysis.crushability {
699
- Some(c) => format!("skip:{}", c.reason),
700
- None => String::new(),
701
- };
702
- return CrushArrayResult {
703
- items: items.to_vec(),
704
- strategy_info: reason,
705
- ccr_hash: None,
706
- dropped_summary: String::new(),
707
- compacted: None,
708
- compaction_kind: None,
709
- };
710
- }
711
-
712
- let plan = self.planner().create_plan(
713
- &analysis,
714
- items,
715
- query_context,
716
- None, // preserve_fields (TOIN — stubbed)
717
- Some(effective_max_items),
718
- Some(&item_strings),
719
- );
720
- let result = self.execute_plan(&plan, items);
721
-
722
- // Emit CCR-Dropped marker iff rows were actually dropped AND
723
- // the marker gate is on. **The marker is the cornerstone of
724
- // CCR's no-data-loss guarantee:** we hash the full original,
725
- // stash it in the configured store, and emit a marker pointing
726
- // at that hash. The runtime later serves the original back via
727
- // retrieval tool calls.
728
- //
729
- // When `enable_ccr_marker` is false (Python shim's path for
730
- // `ccr_config.enabled=False` or `inject_retrieval_marker=False`)
731
- // we keep the row drops (compression is still requested) but
732
- // skip the marker text and the store write — there's no point
733
- // storing a payload that nothing in the prompt can reference.
734
- let dropped_count = items.len().saturating_sub(result.len());
735
- let (ccr_hash, dropped_summary) = if dropped_count > 0 && self.config.enable_ccr_marker {
736
- // Serialize the original array exactly ONCE. The hash is
737
- // taken over those bytes, and (if a store is configured) the
738
- // same bytes get stored — eliminating a redundant tree clone
739
- // (`items.to_vec()`) and a redundant `serde_json::to_string`
740
- // pass that the previous version did per dropped array.
741
- let canonical = canonical_array_json(items);
742
- let h = hash_canonical(&canonical);
743
- let marker = format!("<<ccr:{h} {dropped_count}_rows_offloaded>>");
744
- if let Some(store) = &self.ccr_store {
745
- store.put(&h, &canonical);
746
- }
747
- (Some(h), marker)
748
- } else {
749
- (None, String::new())
750
- };
751
-
752
- CrushArrayResult {
753
- items: result,
754
- strategy_info: analysis.recommended_strategy.as_str().to_string(),
755
- ccr_hash,
756
- dropped_summary,
757
- compacted: None,
758
- compaction_kind: None,
759
- }
760
- }
761
-
762
- /// Compress a mixed-type array by grouping items by type and
763
- /// compressing each group with the appropriate handler.
764
- ///
765
- /// Direct port of `_crush_mixed_array` (Python line 2914-3013).
766
- ///
767
- /// Strategy:
768
- /// 1. Group by type (dict / str / number / list / null / bool / other).
769
- /// 2. For groups with >= `min_items_to_analyze` items: apply the
770
- /// type-specific compressor.
771
- /// 3. For small groups: keep all items.
772
- /// 4. Reassemble in original order.
773
- ///
774
- /// Returns `(crushed_items, strategy_string)`.
775
- pub fn crush_mixed_array(
776
- &self,
777
- items: &[Value],
778
- query_context: &str,
779
- bias: f64,
780
- ) -> (Vec<Value>, String) {
781
- let n = items.len();
782
- if n <= 8 {
783
- return (items.to_vec(), "mixed:passthrough".to_string());
784
- }
785
-
786
- // Group by type, tracking original indices.
787
- let mut groups: GroupBuckets = GroupBuckets::default();
788
- for (i, item) in items.iter().enumerate() {
789
- groups.push(group_key(item), i, item.clone());
790
- }
791
-
792
- let mut keep_indices: std::collections::BTreeSet<usize> = std::collections::BTreeSet::new();
793
- let mut strategy_parts: Vec<String> = Vec::new();
794
-
795
- for (type_key, indices, values) in groups.into_iter() {
796
- // Small groups: keep all items.
797
- if values.len() < self.config.min_items_to_analyze {
798
- keep_indices.extend(&indices);
799
- continue;
800
- }
801
-
802
- match type_key {
803
- "dict" => {
804
- let CrushArrayResult { items: crushed, .. } =
805
- self.crush_array(&values, query_context, bias);
806
- // Find which original indices survived by matching
807
- // canonical-JSON serialization. Mirrors Python's
808
- // `json.dumps(c, sort_keys=True, default=str)`-keyed
809
- // set match.
810
- let crushed_keys: std::collections::HashSet<String> =
811
- crushed.iter().map(canonical_json_for_match).collect();
812
- for (i, idx) in indices.iter().enumerate() {
813
- if crushed_keys.contains(&canonical_json_for_match(&values[i])) {
814
- keep_indices.insert(*idx);
815
- }
816
- }
817
- strategy_parts.push(format!("dict:{}->{}", values.len(), crushed.len()));
818
- }
819
- "str" => {
820
- let strs: Vec<&str> = values.iter().filter_map(|v| v.as_str()).collect();
821
- let (crushed, _) = crush_string_array(&strs, &self.config, bias);
822
- let crushed_set: std::collections::HashSet<&str> =
823
- crushed.iter().map(|s| s.as_str()).collect();
824
- for (i, idx) in indices.iter().enumerate() {
825
- if let Some(s) = values[i].as_str() {
826
- if crushed_set.contains(s) {
827
- keep_indices.insert(*idx);
828
- }
829
- }
830
- }
831
- strategy_parts.push(format!("str:{}->{}", values.len(), crushed.len()));
832
- }
833
- "number" => {
834
- // Python: just adaptive sampling + outlier detection
835
- // (no summary prefix). Keeps first/last by index
836
- // and items >variance_threshold σ from mean.
837
- let item_strings: Vec<String> = values.iter().map(|v| v.to_string()).collect();
838
- let item_refs: Vec<&str> = item_strings.iter().map(|s| s.as_str()).collect();
839
- let (_kt, kf, kl, _) = compute_k_split(&item_refs, &self.config, bias);
840
-
841
- let kf = kf.min(values.len());
842
- let kl = kl.min(values.len().saturating_sub(kf));
843
- let first_idx: Vec<usize> = indices.iter().take(kf).copied().collect();
844
- let last_idx: Vec<usize> =
845
- indices.iter().rev().take(kl).copied().collect::<Vec<_>>();
846
- keep_indices.extend(&first_idx);
847
- keep_indices.extend(&last_idx);
848
-
849
- // Outliers via finite-only stats.
850
- let finite: Vec<f64> = values
851
- .iter()
852
- .filter_map(|v| v.as_f64().filter(|f| f.is_finite()))
853
- .collect();
854
- if finite.len() > 1 {
855
- if let Some(mean_v) = super::stats_math::mean(&finite) {
856
- if let Some(std_v) = super::stats_math::sample_stdev(&finite) {
857
- if std_v > 0.0 {
858
- let threshold = self.config.variance_threshold * std_v;
859
- for (i, val) in values.iter().enumerate() {
860
- if let Some(num) = val.as_f64().filter(|f| f.is_finite()) {
861
- if (num - mean_v).abs() > threshold {
862
- keep_indices.insert(indices[i]);
863
- }
864
- }
865
- }
866
- }
867
- }
868
- }
869
- }
870
- strategy_parts.push(format!("num:{}", values.len()));
871
- }
872
- _ => {
873
- // list / bool / none / other → keep all items.
874
- keep_indices.extend(&indices);
875
- }
876
- }
877
- }
878
-
879
- // Reassemble in original order.
880
- let result: Vec<Value> = keep_indices.iter().map(|&i| items[i].clone()).collect();
881
- let strategy = format!(
882
- "mixed:adaptive({}->{},{})",
883
- n,
884
- result.len(),
885
- strategy_parts.join(",")
886
- );
887
- (result, strategy)
888
- }
889
- }
890
-
891
- // ---------- helpers ----------
892
-
893
- /// Group key that mirrors Python's `_crush_mixed_array` switch on
894
- /// `isinstance`. Note the bool-before-number ordering: in Python, bool
895
- /// is a subclass of int, but JSON treats them as distinct types, so we
896
- /// don't have the Python ordering hazard.
897
- fn group_key(item: &Value) -> &'static str {
898
- match item {
899
- Value::Object(_) => "dict",
900
- Value::String(_) => "str",
901
- Value::Bool(_) => "bool",
902
- Value::Number(_) => "number",
903
- Value::Array(_) => "list",
904
- Value::Null => "none",
905
- }
906
- }
907
-
908
- /// Group buckets keyed by the type-string. Preserves first-occurrence
909
- /// order across keys so dict/str/number/list/none/bool always come out
910
- /// in the same order — matters because `keep_indices` is built
911
- /// incrementally and Python iterates `groups.items()` (insertion order
912
- /// in 3.7+).
913
- #[derive(Default)]
914
- struct GroupBuckets {
915
- entries: Vec<(&'static str, Vec<usize>, Vec<Value>)>,
916
- index_of: std::collections::HashMap<&'static str, usize>,
917
- }
918
-
919
- impl GroupBuckets {
920
- fn push(&mut self, key: &'static str, idx: usize, value: Value) {
921
- match self.index_of.get(key).copied() {
922
- Some(i) => {
923
- self.entries[i].1.push(idx);
924
- self.entries[i].2.push(value);
925
- }
926
- None => {
927
- self.index_of.insert(key, self.entries.len());
928
- self.entries.push((key, vec![idx], vec![value]));
929
- }
930
- }
931
- }
932
- }
933
-
934
- impl IntoIterator for GroupBuckets {
935
- type Item = (&'static str, Vec<usize>, Vec<Value>);
936
- type IntoIter = std::vec::IntoIter<Self::Item>;
937
- fn into_iter(self) -> Self::IntoIter {
938
- self.entries.into_iter()
939
- }
940
- }
941
-
942
- /// Serialize a `Value` for membership comparison. Mirrors Python's
943
- /// `json.dumps(c, sort_keys=True, default=str)` used by
944
- /// `_crush_mixed_array` to match crushed dict items back to their
945
- /// original indices. The `default=str` fallback only matters for
946
- /// non-JSON-serializable Python values; in serde_json land everything
947
- /// is already JSON-native, so plain canonical JSON suffices.
948
- fn canonical_json_for_match(value: &Value) -> String {
949
- crate::transforms::anchor_selector::python_json_dumps_sort_keys(value)
950
- }
951
-
952
- /// Maps a `Compaction` to a stable kind tag exposed via `CrushArrayResult`.
953
- fn compaction_kind_str(c: &Compaction) -> &'static str {
954
- match c {
955
- Compaction::Table { .. } => "table",
956
- Compaction::Buckets { .. } => "buckets",
957
- Compaction::OpaqueRef { .. } => "ccr",
958
- Compaction::Untouched(_) => "untouched",
959
- }
960
- }
961
-
962
- /// Approximate byte size of `[v0, v1, ...]` JSON serialization, given
963
- /// each item's already-serialized form. Adds 2 for outer brackets and
964
- /// 1 per inter-item comma. Used by the lossless savings-ratio check.
965
- fn estimate_array_bytes(item_strings: &[String]) -> usize {
966
- let payload: usize = item_strings.iter().map(|s| s.len()).sum();
967
- let separators = item_strings.len().saturating_sub(1);
968
- payload + separators + 2
969
- }
970
-
971
- /// Serialize `[v0, v1, ...]` once into the canonical JSON form used by
972
- /// the CCR retrieval contract. `serde_json` writes a slice of `Value` as
973
- /// the same bytes it would write for `Value::Array(items.to_vec())`, so
974
- /// we skip the array-wrapper allocation and the deep tree clone it
975
- /// requires. Used by both the hash (input) and the store payload (write).
976
- fn canonical_array_json(items: &[Value]) -> String {
977
- serde_json::to_string(items).unwrap_or_default()
978
- }
979
-
980
- /// 12-char SHA-256 hex prefix of an already-serialized canonical JSON
981
- /// string. Caller is responsible for producing the canonical form via
982
- /// [`canonical_array_json`] (or another byte-equal serializer) — the
983
- /// hash is over the bytes, so a stable serializer is the contract.
984
- fn hash_canonical(canonical: &str) -> String {
985
- use sha2::{Digest, Sha256};
986
- let mut h = Sha256::new();
987
- h.update(canonical.as_bytes());
988
- h.finalize()
989
- .iter()
990
- .take(6)
991
- .map(|b| format!("{b:02x}"))
992
- .collect()
993
- }
994
-
995
- // `hash_array_for_ccr` (a test-only `canonical_array_json + hash_canonical`
996
- // convenience) lived here previously but had no callers — clippy flagged
997
- // it as dead code. Reintroduce as a test fixture if a future test wants
998
- // the one-liner; production callsites inline both steps so the canonical
999
- // bytes can be reused for the store payload.
1000
-
1001
- // ─── PR5 walker-integration helpers (string handling) ──────────────────────
1002
- //
1003
- // Parse-as-JSON-container, marker formatting, and humanize-bytes used to
1004
- // live here as locals. PR8 extracted them into `compaction::walker` so
1005
- // `walker.rs` and `process_value` share one canonical implementation —
1006
- // killing the drift risk where the two paths could format markers
1007
- // differently. `process_string` now calls `try_parse_json_container` and
1008
- // `emit_opaque_ccr_marker` directly. Only `opaque_kind_label` survives
1009
- // here because `process_string`'s `string_ccr:<kind>` strategy-info
1010
- // label is local to this module's debug-string convention.
1011
-
1012
- fn opaque_kind_label(kind: &super::compaction::OpaqueKind) -> &str {
1013
- use super::compaction::OpaqueKind;
1014
- match kind {
1015
- OpaqueKind::Base64Blob => "base64",
1016
- OpaqueKind::LongString => "string",
1017
- OpaqueKind::HtmlChunk => "html",
1018
- OpaqueKind::Other(s) => s.as_str(),
1019
- }
1020
- }
1021
-
1022
- #[cfg(test)]
1023
- mod tests {
1024
- use super::*;
1025
- use serde_json::json;
1026
-
1027
- fn crusher() -> SmartCrusher {
1028
- SmartCrusher::new(SmartCrusherConfig::default())
1029
- }
1030
-
1031
- // ---------- execute_plan ----------
1032
-
1033
- #[test]
1034
- fn execute_plan_empty_indices_returns_empty() {
1035
- let c = crusher();
1036
- let plan = CompressionPlan::default();
1037
- let items: Vec<Value> = (0..5).map(|i| json!({"id": i})).collect();
1038
- let result = c.execute_plan(&plan, &items);
1039
- assert!(result.is_empty());
1040
- }
1041
-
1042
- #[test]
1043
- fn execute_plan_returns_items_in_sorted_index_order() {
1044
- let c = crusher();
1045
- let items: Vec<Value> = (0..10).map(|i| json!({"id": i})).collect();
1046
- let plan = CompressionPlan {
1047
- keep_indices: vec![5, 2, 8, 0],
1048
- ..CompressionPlan::default()
1049
- };
1050
- let result = c.execute_plan(&plan, &items);
1051
- assert_eq!(result.len(), 4);
1052
- assert_eq!(result[0]["id"], 0);
1053
- assert_eq!(result[1]["id"], 2);
1054
- assert_eq!(result[2]["id"], 5);
1055
- assert_eq!(result[3]["id"], 8);
1056
- }
1057
-
1058
- #[test]
1059
- fn execute_plan_skips_out_of_bounds() {
1060
- let c = crusher();
1061
- let items: Vec<Value> = (0..3).map(|i| json!({"id": i})).collect();
1062
- let plan = CompressionPlan {
1063
- keep_indices: vec![0, 5, 2],
1064
- ..CompressionPlan::default()
1065
- };
1066
- let result = c.execute_plan(&plan, &items);
1067
- assert_eq!(result.len(), 2);
1068
- }
1069
-
1070
- #[test]
1071
- fn execute_plan_factors_constants_when_enabled() {
1072
- let cfg = SmartCrusherConfig {
1073
- factor_out_constants: true,
1074
- ..Default::default()
1075
- };
1076
- let c = SmartCrusher::new(cfg);
1077
- let items: Vec<Value> = (0..4)
1078
- .map(|i| json!({"id": i, "region": "us-west-2", "status": "ok"}))
1079
- .collect();
1080
- let mut constant_fields = std::collections::BTreeMap::new();
1081
- constant_fields.insert("region".to_string(), json!("us-west-2"));
1082
- constant_fields.insert("status".to_string(), json!("ok"));
1083
- let plan = CompressionPlan {
1084
- keep_indices: vec![0, 1, 2],
1085
- constant_fields,
1086
- ..CompressionPlan::default()
1087
- };
1088
- let result = c.execute_plan(&plan, &items);
1089
- // Sentinel first, then 3 slim items.
1090
- assert_eq!(result.len(), 4);
1091
- assert_eq!(result[0]["_constant_fields"]["region"], "us-west-2");
1092
- assert_eq!(result[0]["_constant_fields"]["status"], "ok");
1093
- for item in &result[1..] {
1094
- assert!(item.get("region").is_none());
1095
- assert!(item.get("status").is_none());
1096
- assert!(item.get("id").is_some());
1097
- }
1098
- }
1099
-
1100
- #[test]
1101
- fn execute_plan_keeps_drifted_values_when_factoring() {
1102
- // Defensive strip: an item whose value differs from the recorded
1103
- // constant keeps its own value.
1104
- let cfg = SmartCrusherConfig {
1105
- factor_out_constants: true,
1106
- ..Default::default()
1107
- };
1108
- let c = SmartCrusher::new(cfg);
1109
- let items = vec![
1110
- json!({"id": 0, "status": "ok"}),
1111
- json!({"id": 1, "status": "FAILED"}),
1112
- ];
1113
- let mut constant_fields = std::collections::BTreeMap::new();
1114
- constant_fields.insert("status".to_string(), json!("ok"));
1115
- let plan = CompressionPlan {
1116
- keep_indices: vec![0, 1],
1117
- constant_fields,
1118
- ..CompressionPlan::default()
1119
- };
1120
- let result = c.execute_plan(&plan, &items);
1121
- assert_eq!(result.len(), 3);
1122
- assert!(result[1].get("status").is_none()); // matched → stripped
1123
- assert_eq!(result[2]["status"], "FAILED"); // drifted → kept
1124
- }
1125
-
1126
- #[test]
1127
- fn execute_plan_default_off_leaves_items_unchanged() {
1128
- // factor_out_constants defaults to false: schema preserved even
1129
- // when the plan carries constant_fields.
1130
- let c = crusher();
1131
- let items: Vec<Value> = (0..3).map(|i| json!({"id": i, "k": "v"})).collect();
1132
- let mut constant_fields = std::collections::BTreeMap::new();
1133
- constant_fields.insert("k".to_string(), json!("v"));
1134
- let plan = CompressionPlan {
1135
- keep_indices: vec![0, 1, 2],
1136
- constant_fields,
1137
- ..CompressionPlan::default()
1138
- };
1139
- let result = c.execute_plan(&plan, &items);
1140
- assert_eq!(result.len(), 3);
1141
- assert_eq!(result[0]["k"], "v");
1142
- }
1143
-
1144
- // ---------- crush_array ----------
1145
-
1146
- #[test]
1147
- fn crush_array_passthrough_when_below_adaptive_k() {
1148
- let c = crusher();
1149
- let items: Vec<Value> = (0..3).map(|i| json!({"id": i})).collect();
1150
- let result = c.crush_array(&items, "", 1.0);
1151
- assert_eq!(result.items.len(), 3);
1152
- assert_eq!(result.strategy_info, "none:adaptive_at_limit");
1153
- assert!(result.ccr_hash.is_none());
1154
- }
1155
-
1156
- #[test]
1157
- fn crush_array_skip_path_returns_original_items() {
1158
- // 30 unique dict items with ID-like fields → analyzer should
1159
- // detect "unique_entities_no_signal" and SKIP. Use the
1160
- // no-compaction constructor so we exercise the lossy/skip
1161
- // gate; the lossless path would otherwise short-circuit
1162
- // because uniform-tabular input is the lossless sweet spot.
1163
- let c = SmartCrusher::without_compaction(SmartCrusherConfig::default());
1164
- let items: Vec<Value> = (0..30)
1165
- .map(|i| json!({"id": i, "name": format!("user_{}", i)}))
1166
- .collect();
1167
- let result = c.crush_array(&items, "", 1.0);
1168
- // skip path returns the original items unchanged.
1169
- assert_eq!(result.items.len(), 30);
1170
- assert!(
1171
- result.strategy_info.starts_with("skip:"),
1172
- "expected skip:..., got {}",
1173
- result.strategy_info
1174
- );
1175
- }
1176
-
1177
- #[test]
1178
- fn crush_array_low_uniqueness_compresses() {
1179
- // 30 items with status=ok across all → low_uniqueness path
1180
- // (crushable, smart_sample strategy).
1181
- let c = crusher();
1182
- let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();
1183
- let result = c.crush_array(&items, "", 1.0);
1184
- assert!(result.items.len() <= 30, "should not exceed original count");
1185
- }
1186
-
1187
- #[test]
1188
- fn crush_array_keeps_error_items() {
1189
- let c = crusher();
1190
- let mut items: Vec<Value> = (0..30).map(|i| json!({"id": i, "status": "ok"})).collect();
1191
- items.push(json!({"id": 30, "status": "error", "msg": "FATAL"}));
1192
- let result = c.crush_array(&items, "", 1.0);
1193
- // Whatever path is taken, the error item should survive.
1194
- assert!(
1195
- result
1196
- .items
1197
- .iter()
1198
- .any(|item| { item.get("status").and_then(|v| v.as_str()) == Some("error") }),
1199
- "error item must survive crush_array"
1200
- );
1201
- }
1202
-
1203
- // ---------- crush_mixed_array ----------
1204
-
1205
- #[test]
1206
- fn crush_mixed_passthrough_at_threshold() {
1207
- let c = crusher();
1208
- let items: Vec<Value> = vec![
1209
- json!(1),
1210
- json!("two"),
1211
- json!({"k": "v"}),
1212
- json!([1, 2]),
1213
- json!(null),
1214
- json!(true),
1215
- json!(3),
1216
- json!("four"),
1217
- ];
1218
- let (result, strat) = c.crush_mixed_array(&items, "", 1.0);
1219
- assert_eq!(result.len(), 8);
1220
- assert_eq!(strat, "mixed:passthrough");
1221
- }
1222
-
1223
- #[test]
1224
- fn crush_mixed_groups_and_compresses_dicts() {
1225
- let c = crusher();
1226
- // 25 dicts (large group → gets crushed) + 5 strings (small group → all kept).
1227
- let mut items: Vec<Value> = (0..25).map(|i| json!({"id": i, "status": "ok"})).collect();
1228
- for i in 0..5 {
1229
- items.push(json!(format!("string_{}", i)));
1230
- }
1231
- let (result, strat) = c.crush_mixed_array(&items, "", 1.0);
1232
- assert!(strat.starts_with("mixed:adaptive("));
1233
- // The 5 strings (small group) all survive.
1234
- let str_count = result.iter().filter(|v| v.is_string()).count();
1235
- assert_eq!(str_count, 5);
1236
- }
1237
-
1238
- #[test]
1239
- fn crush_mixed_keeps_lists_and_nulls_unchanged() {
1240
- let c = crusher();
1241
- let mut items: Vec<Value> = vec![json!([1, 2]); 6];
1242
- items.extend(vec![json!(null); 6]);
1243
- items.extend(vec![json!({"k": 1}); 10]);
1244
- let (result, _strat) = c.crush_mixed_array(&items, "", 1.0);
1245
- // Lists and nulls (not dict/str/number) → fall through to "keep all".
1246
- let list_count = result.iter().filter(|v| v.is_array()).count();
1247
- let null_count = result.iter().filter(|v| v.is_null()).count();
1248
- assert_eq!(list_count, 6);
1249
- assert_eq!(null_count, 6);
1250
- }
1251
-
1252
- #[test]
1253
- fn crusher_construction_default() {
1254
- let c = SmartCrusher::new(SmartCrusherConfig::default());
1255
- assert_eq!(c.config.max_items_after_crush, 15);
1256
- }
1257
-
1258
- // ---------- top-level crush ----------
1259
-
1260
- #[test]
1261
- fn crush_non_json_passes_through_unchanged() {
1262
- let c = crusher();
1263
- let result = c.crush("not json at all", "", 1.0);
1264
- assert!(!result.was_modified);
1265
- assert_eq!(result.compressed, "not json at all");
1266
- assert_eq!(result.strategy, "passthrough");
1267
- }
1268
-
1269
- #[test]
1270
- fn crush_scalar_json_passes_through() {
1271
- let c = crusher();
1272
- let result = c.crush("42", "", 1.0);
1273
- // A scalar is not crushable; should round-trip unchanged.
1274
- assert_eq!(result.compressed, "42");
1275
- assert!(!result.was_modified);
1276
- }
1277
-
1278
- #[test]
1279
- fn crush_small_array_passes_through() {
1280
- let c = crusher();
1281
- // Compact-form input matches the compact serializer output, so
1282
- // the array is not "modified" even though it round-trips
1283
- // through parse → serialize. (The spaced form `[1, 2, 3]`
1284
- // would mark `was_modified=true` because the compact
1285
- // serializer rewrites it to `[1,2,3]`.)
1286
- let result = c.crush(r#"[1,2,3]"#, "", 1.0);
1287
- // Below min_items_to_analyze=5 → no crushing of the structure.
1288
- assert!(!result.was_modified);
1289
- assert_eq!(result.compressed, "[1,2,3]");
1290
- }
1291
-
1292
- #[test]
1293
- fn crush_dict_array_crushes_when_low_uniqueness() {
1294
- // The public `crush()` API serializes back to JSON; the
1295
- // lossless-path output (a compacted string) is exposed via
1296
- // `crush_array().compacted` rather than being substituted into
1297
- // the JSON re-serialization. So we exercise the lossy path
1298
- // here via `without_compaction()` to validate the original
1299
- // intent: low-uniqueness dicts compress via row-dropping.
1300
- let c = SmartCrusher::without_compaction(SmartCrusherConfig::default());
1301
- let mut input = String::from("[");
1302
- for i in 0..30 {
1303
- if i > 0 {
1304
- input.push(',');
1305
- }
1306
- input.push_str(r#"{"status":"ok"}"#);
1307
- }
1308
- input.push(']');
1309
- let result = c.crush(&input, "", 1.0);
1310
- assert!(
1311
- result.was_modified,
1312
- "30 identical dicts should compress (low_uniqueness_safe_to_sample)"
1313
- );
1314
- assert_ne!(result.strategy, "passthrough");
1315
- }
1316
-
1317
- #[test]
1318
- fn crush_serializes_with_python_safe_format() {
1319
- let c = crusher();
1320
- // SmartCrusher uses Python's `safe_json_dumps`: compact
1321
- // separators `(",", ":")` + `ensure_ascii=False`, preserving
1322
- // object-key insertion order. A spaced input round-trips to
1323
- // the compact form.
1324
- let input = r#"{"a": 1, "b": 2, "c": 3}"#;
1325
- let result = c.crush(input, "", 1.0);
1326
- assert_eq!(
1327
- result.compressed, r#"{"a":1,"b":2,"c":3}"#,
1328
- "safe_json_dumps emits compact `,` / `:` separators"
1329
- );
1330
- }
1331
-
1332
- #[test]
1333
- fn crush_recurses_into_nested_arrays() {
1334
- let c = crusher();
1335
- // Top-level dict with a nested array of 30 identical items.
1336
- // The inner array should compress (low_uniqueness path).
1337
- let mut inner = String::from("[");
1338
- for i in 0..30 {
1339
- if i > 0 {
1340
- inner.push(',');
1341
- }
1342
- inner.push_str(r#"{"status":"ok"}"#);
1343
- }
1344
- inner.push(']');
1345
- let input = format!(r#"{{"data": {}}}"#, inner);
1346
- let result = c.crush(&input, "", 1.0);
1347
- assert!(
1348
- result.was_modified,
1349
- "nested compressible array must be crushed even inside a wrapper object"
1350
- );
1351
- }
1352
-
1353
- #[test]
1354
- fn crusher_with_custom_scorer() {
1355
- use crate::relevance::BM25Scorer;
1356
- let c = SmartCrusher::with_scorer(
1357
- SmartCrusherConfig::default(),
1358
- Box::new(BM25Scorer::default()),
1359
- );
1360
- // Sanity: crushing still works with a swapped scorer.
1361
- let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();
1362
- let result = c.crush_array(&items, "anything", 1.0);
1363
- assert!(result.items.len() <= 30);
1364
- }
1365
-
1366
- // ---------- Stage 3c.2 PR4: lossless-first default with threshold + CCR-Dropped ----------
1367
-
1368
- #[test]
1369
- fn without_compaction_yields_none_compacted_field() {
1370
- // The opt-out constructor preserves pre-PR4 lossy-only path.
1371
- // No lossless attempt → compacted/compaction_kind always None.
1372
- let c = SmartCrusher::without_compaction(SmartCrusherConfig::default());
1373
- let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();
1374
- let result = c.crush_array(&items, "", 1.0);
1375
- assert!(result.compacted.is_none());
1376
- assert!(result.compaction_kind.is_none());
1377
- }
1378
-
1379
- #[test]
1380
- fn lossless_wins_when_savings_above_threshold() {
1381
- // 50 uniform tabular dicts → CSV+schema compaction shrinks
1382
- // the input dramatically (well above the 0.30 default).
1383
- // Default `SmartCrusher::new()` should pick lossless.
1384
- let c = crusher();
1385
- let items: Vec<Value> = (0..50)
1386
- .map(|i| json!({"id": i, "name": format!("u_{i}"), "status": "ok"}))
1387
- .collect();
1388
- let result = c.crush_array(&items, "", 1.0);
1389
- let compacted = result.compacted.expect("compacted should be set");
1390
- assert!(compacted.starts_with("[50]{"), "got: {compacted}");
1391
- assert_eq!(result.compaction_kind, Some("table"));
1392
- assert!(
1393
- result.strategy_info.starts_with("lossless:table"),
1394
- "got: {}",
1395
- result.strategy_info
1396
- );
1397
- // Lossless = nothing dropped → no CCR retrieval needed.
1398
- assert!(result.ccr_hash.is_none());
1399
- // items preserved (full original).
1400
- assert_eq!(result.items.len(), 50);
1401
- }
1402
-
1403
- #[test]
1404
- fn lossy_falls_through_when_savings_below_threshold() {
1405
- // Force the threshold high enough that even tabular savings
1406
- // can't satisfy it → lossy path runs → CCR-Dropped fires.
1407
- // Use low-uniqueness items so the analyzer is willing to
1408
- // crush (unique id+name per row would trip the
1409
- // "unique_entities_no_signal" skip gate instead).
1410
- let cfg = SmartCrusherConfig {
1411
- lossless_min_savings_ratio: 0.99,
1412
- ..Default::default()
1413
- };
1414
- let c = SmartCrusher::new(cfg);
1415
- let items: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();
1416
- let result = c.crush_array(&items, "", 1.0);
1417
- // Lossless declined → no compacted output.
1418
- assert!(result.compacted.is_none());
1419
- // Lossy ran → rows dropped.
1420
- assert!(
1421
- result.items.len() < 50,
1422
- "expected lossy drop, got {} items",
1423
- result.items.len()
1424
- );
1425
- // CCR hash populated for retrieval.
1426
- let h = result.ccr_hash.expect("ccr_hash populated on drop");
1427
- assert_eq!(h.len(), 12);
1428
- // Marker visible in dropped_summary.
1429
- assert!(
1430
- result.dropped_summary.contains(&format!("<<ccr:{h}")),
1431
- "got: {}",
1432
- result.dropped_summary
1433
- );
1434
- assert!(result.dropped_summary.contains("rows_offloaded"));
1435
- }
1436
-
1437
- #[test]
1438
- fn ccr_hash_is_deterministic() {
1439
- // Same input → same hash, so the runtime cache key is stable.
1440
- let cfg = SmartCrusherConfig {
1441
- lossless_min_savings_ratio: 0.99, // force lossy path
1442
- ..Default::default()
1443
- };
1444
- let c = SmartCrusher::new(cfg);
1445
- let items: Vec<Value> = (0..30).map(|i| json!({"id": i, "tag": "ok"})).collect();
1446
- let r1 = c.crush_array(&items, "", 1.0);
1447
- let r2 = c.crush_array(&items, "", 1.0);
1448
- assert_eq!(r1.ccr_hash, r2.ccr_hash);
1449
- assert!(r1.ccr_hash.is_some());
1450
- }
1451
-
1452
- #[test]
1453
- fn ccr_hash_changes_with_input() {
1454
- let cfg = SmartCrusherConfig {
1455
- lossless_min_savings_ratio: 0.99,
1456
- ..Default::default()
1457
- };
1458
- let c = SmartCrusher::new(cfg);
1459
- let a: Vec<Value> = (0..30).map(|i| json!({"id": i})).collect();
1460
- let b: Vec<Value> = (100..130).map(|i| json!({"id": i})).collect();
1461
- let ra = c.crush_array(&a, "", 1.0);
1462
- let rb = c.crush_array(&b, "", 1.0);
1463
- assert_ne!(ra.ccr_hash, rb.ccr_hash);
1464
- }
1465
-
1466
- #[test]
1467
- fn lossy_without_compaction_still_emits_ccr_hash() {
1468
- // The CCR-Dropped restoration applies regardless of whether
1469
- // lossless was attempted — without_compaction also gets the
1470
- // ccr_hash on row drops.
1471
- let c = SmartCrusher::without_compaction(SmartCrusherConfig::default());
1472
- let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();
1473
- let result = c.crush_array(&items, "", 1.0);
1474
- if result.items.len() < items.len() {
1475
- assert!(result.ccr_hash.is_some());
1476
- assert!(!result.dropped_summary.is_empty());
1477
- }
1478
- }
1479
-
1480
- #[test]
1481
- fn passthrough_paths_do_not_emit_ccr_hash() {
1482
- // Tier-1 boundary (items.len() <= adaptive_k): nothing
1483
- // dropped, no CCR. Skip path: same.
1484
- let c = crusher();
1485
- let small: Vec<Value> = (0..3).map(|i| json!({"id": i})).collect();
1486
- let r = c.crush_array(&small, "", 1.0);
1487
- assert!(r.ccr_hash.is_none());
1488
- assert_eq!(r.dropped_summary, "");
1489
- }
1490
-
1491
- #[test]
1492
- fn compaction_skips_non_object_array() {
1493
- // Compactor returns Untouched for non-object arrays → no
1494
- // compacted field populated, no kind tag.
1495
- let c = SmartCrusherBuilder::new(SmartCrusherConfig::default())
1496
- .with_default_oss_setup()
1497
- .with_default_compaction()
1498
- .build();
1499
- let items: Vec<Value> = (0..30).map(|i| json!(i)).collect();
1500
- let result = c.crush_array(&items, "", 1.0);
1501
- assert!(result.compacted.is_none());
1502
- assert!(result.compaction_kind.is_none());
1503
- }
1504
-
1505
- // ---------- Stage 3c.2 PR5: walker-integration in process_value ----------
1506
-
1507
- #[test]
1508
- fn process_string_short_string_passthrough() {
1509
- let c = SmartCrusher::new(SmartCrusherConfig::default());
1510
- let (out, info) = c.process_value(&json!("hello world"), 0, "", 1.0);
1511
- assert_eq!(out, json!("hello world"));
1512
- assert!(info.is_empty());
1513
- }
1514
-
1515
- #[test]
1516
- fn process_string_stringified_json_array_recurses() {
1517
- // A string-typed field whose value is a JSON-encoded array of
1518
- // dicts. process_value should parse it, recurse, and return
1519
- // the processed JSON re-rendered as a string.
1520
- let c = SmartCrusher::new(SmartCrusherConfig::default());
1521
- let big_array_json = serde_json::to_string(
1522
- &(0..50)
1523
- .map(|i| json!({"id": i, "level": "info", "msg": "ok"}))
1524
- .collect::<Vec<_>>(),
1525
- )
1526
- .unwrap();
1527
- let doc = json!({"payload": big_array_json.clone()});
1528
- let (out, info) = c.process_value(&doc, 0, "", 1.0);
1529
- // payload still a string-typed field — we preserved the
1530
- // wrapping shape — but its content was processed.
1531
- let payload = out.pointer("/payload").and_then(|v| v.as_str()).unwrap();
1532
- // Either compressed or unchanged; if compressed, info reflects.
1533
- // For 50 items with low-uniqueness, compression should fire.
1534
- // The strategy info should mention string_json processing.
1535
- assert!(
1536
- info.contains("string_json") || payload != big_array_json,
1537
- "expected processing trace; info={info}, len before={}, after={}",
1538
- big_array_json.len(),
1539
- payload.len(),
1540
- );
1541
- }
1542
-
1543
- #[test]
1544
- fn process_string_opaque_blob_becomes_ccr_marker() {
1545
- let c = SmartCrusher::new(SmartCrusherConfig::default());
1546
- let big_b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".repeat(8);
1547
- let doc = json!({"id": 1, "blob": big_b64});
1548
- let (out, _info) = c.process_value(&doc, 0, "", 1.0);
1549
- let blob = out.pointer("/blob").and_then(|v| v.as_str()).unwrap();
1550
- assert!(blob.starts_with("<<ccr:"), "got: {blob}");
1551
- assert!(blob.contains(",base64,"));
1552
- }
1553
-
1554
- #[test]
1555
- fn process_string_top_level_string_processed() {
1556
- // crush() takes a string; if it doesn't parse as JSON, today's
1557
- // behavior returns it unchanged. But if it's a stringified
1558
- // JSON object/array, it should now get processed.
1559
- let c = SmartCrusher::new(SmartCrusherConfig::default());
1560
- // Non-JSON top-level string — passthrough.
1561
- let plain = "just some plain text";
1562
- let result = c.crush(plain, "", 1.0);
1563
- assert_eq!(result.compressed, plain);
1564
- }
1565
-
1566
- #[test]
1567
- fn process_string_does_not_alter_short_quoted_strings() {
1568
- // Strings that look JSON-like but are short shouldn't be
1569
- // CCR-substituted.
1570
- let c = SmartCrusher::new(SmartCrusherConfig::default());
1571
- let doc = json!({"msg": "{this looks like json but isnt}"});
1572
- let (out, _) = c.process_value(&doc, 0, "", 1.0);
1573
- assert_eq!(out, doc);
1574
- }
1575
-
1576
- #[test]
1577
- fn process_string_helper_parses_only_containers() {
1578
- assert!(try_parse_json_container("{\"a\":1}").is_some());
1579
- assert!(try_parse_json_container("[1,2,3]").is_some());
1580
- assert!(try_parse_json_container("123").is_none()); // bare scalar
1581
- assert!(try_parse_json_container("\"hello\"").is_none()); // bare string
1582
- assert!(try_parse_json_container("not json").is_none());
1583
- assert!(try_parse_json_container("{malformed").is_none());
1584
- }
1585
-
1586
- // ---------- enable_ccr_marker gate (PR #301 re-land) ----------
1587
-
1588
- #[test]
1589
- fn enable_ccr_marker_false_suppresses_marker_and_store() {
1590
- // The Rust-side gate. Compression still runs (rows drop) but
1591
- // the result carries no marker text, no hash, and the CCR
1592
- // store does NOT grow — there's no point storing what nothing
1593
- // in the prompt can reference.
1594
- use crate::ccr::InMemoryCcrStore;
1595
- use crate::transforms::smart_crusher::SmartCrusherBuilder;
1596
- use std::sync::Arc;
1597
-
1598
- let store: Arc<dyn CcrStore> = Arc::new(InMemoryCcrStore::new());
1599
- let cfg = SmartCrusherConfig {
1600
- lossless_min_savings_ratio: 0.99, // force lossy path
1601
- enable_ccr_marker: false,
1602
- ..SmartCrusherConfig::default()
1603
- };
1604
- let c = SmartCrusherBuilder::new(cfg)
1605
- .with_ccr_store(Arc::clone(&store))
1606
- .build();
1607
- let items: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();
1608
-
1609
- let store_len_before = store.len();
1610
- let result = c.crush_array(&items, "", 1.0);
1611
- let store_len_after = store.len();
1612
-
1613
- // Rows were dropped (we built 50, kept fewer).
1614
- assert!(result.items.len() < items.len(), "lossy path didn't fire");
1615
- // Gate held: no marker, no hash.
1616
- assert!(result.ccr_hash.is_none(), "ccr_hash should be None");
1617
- assert!(
1618
- result.dropped_summary.is_empty(),
1619
- "dropped_summary should be empty, got: {:?}",
1620
- result.dropped_summary
1621
- );
1622
- // Store did NOT grow.
1623
- assert_eq!(
1624
- store_len_after, store_len_before,
1625
- "ccr_store grew despite enable_ccr_marker=false"
1626
- );
1627
- }
1628
-
1629
- #[test]
1630
- fn enable_ccr_marker_true_is_default_behavior() {
1631
- // Default config still emits markers + stores when rows drop.
1632
- // Sanity: the gate is opt-out, not opt-in.
1633
- use crate::ccr::InMemoryCcrStore;
1634
- use crate::transforms::smart_crusher::SmartCrusherBuilder;
1635
- use std::sync::Arc;
1636
-
1637
- let store: Arc<dyn CcrStore> = Arc::new(InMemoryCcrStore::new());
1638
- let cfg = SmartCrusherConfig {
1639
- lossless_min_savings_ratio: 0.99, // force lossy path
1640
- ..SmartCrusherConfig::default()
1641
- };
1642
- // Default: enable_ccr_marker = true.
1643
- assert!(cfg.enable_ccr_marker);
1644
- let c = SmartCrusherBuilder::new(cfg)
1645
- .with_ccr_store(Arc::clone(&store))
1646
- .build();
1647
- let items: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();
1648
-
1649
- let store_len_before = store.len();
1650
- let result = c.crush_array(&items, "", 1.0);
1651
- let store_len_after = store.len();
1652
-
1653
- assert!(result.items.len() < items.len(), "lossy path didn't fire");
1654
- assert!(result.ccr_hash.is_some(), "default should produce a hash");
1655
- assert!(
1656
- result.dropped_summary.contains("<<ccr:"),
1657
- "default should produce a marker: {:?}",
1658
- result.dropped_summary
1659
- );
1660
- assert!(
1661
- store_len_after > store_len_before,
1662
- "default should write to ccr_store"
1663
- );
1664
- }
1665
- }