xithead 1.4.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (661) hide show
  1. package/.actrc +14 -0
  2. package/.actrc.local.example +15 -0
  3. package/.changelog.md +2 -0
  4. package/.claude/agents/academic-anthropologist.md +125 -0
  5. package/.claude/agents/academic-geographer.md +127 -0
  6. package/.claude/agents/academic-historian.md +123 -0
  7. package/.claude/agents/academic-narratologist.md +118 -0
  8. package/.claude/agents/academic-psychologist.md +118 -0
  9. package/.claude/agents/academic-statistician.md +144 -0
  10. package/.claude/agents/accounts-payable-agent.md +205 -0
  11. package/.claude/agents/agentic-identity-trust.md +394 -0
  12. package/.claude/agents/agents-orchestrator.md +367 -0
  13. package/.claude/agents/automation-governance-architect.md +216 -0
  14. package/.claude/agents/blender-addon-engineer.md +243 -0
  15. package/.claude/agents/business-strategist.md +488 -0
  16. package/.claude/agents/change-management-consultant.md +497 -0
  17. package/.claude/agents/chief-financial-officer.md +388 -0
  18. package/.claude/agents/corporate-training-designer.md +192 -0
  19. package/.claude/agents/customer-service.md +398 -0
  20. package/.claude/agents/customer-success-manager.md +460 -0
  21. package/.claude/agents/data-consolidation-agent.md +60 -0
  22. package/.claude/agents/data-privacy-officer.md +412 -0
  23. package/.claude/agents/design-brand-guardian.md +322 -0
  24. package/.claude/agents/design-image-prompt-engineer.md +236 -0
  25. package/.claude/agents/design-inclusive-visuals-specialist.md +73 -0
  26. package/.claude/agents/design-persona-walkthrough.md +272 -0
  27. package/.claude/agents/design-ui-designer.md +386 -0
  28. package/.claude/agents/design-ui-finish-gate-reviewer.md +217 -0
  29. package/.claude/agents/design-ux-architect.md +482 -0
  30. package/.claude/agents/design-ux-researcher.md +329 -0
  31. package/.claude/agents/design-visual-storyteller.md +149 -0
  32. package/.claude/agents/design-whimsy-injector.md +460 -0
  33. package/.claude/agents/economy-designer.md +156 -0
  34. package/.claude/agents/engineering-ai-data-remediation-engineer.md +211 -0
  35. package/.claude/agents/engineering-ai-engineer.md +146 -0
  36. package/.claude/agents/engineering-api-platform-engineer.md +181 -0
  37. package/.claude/agents/engineering-ats-validator-architect.md +383 -0
  38. package/.claude/agents/engineering-autonomous-optimization-architect.md +121 -0
  39. package/.claude/agents/engineering-backend-architect.md +255 -0
  40. package/.claude/agents/engineering-china-network-engineer.md +252 -0
  41. package/.claude/agents/engineering-cms-developer.md +536 -0
  42. package/.claude/agents/engineering-code-reviewer.md +76 -0
  43. package/.claude/agents/engineering-codebase-onboarding-engineer.md +173 -0
  44. package/.claude/agents/engineering-data-engineer.md +349 -0
  45. package/.claude/agents/engineering-data-visualization-engineer.md +151 -0
  46. package/.claude/agents/engineering-database-optimizer.md +188 -0
  47. package/.claude/agents/engineering-database-reliability-engineer.md +201 -0
  48. package/.claude/agents/engineering-desktop-app-engineer.md +209 -0
  49. package/.claude/agents/engineering-developer-tooling-engineer.md +153 -0
  50. package/.claude/agents/engineering-devops-automator.md +385 -0
  51. package/.claude/agents/engineering-drupal-performance.md +347 -0
  52. package/.claude/agents/engineering-drupal-shopping-cart.md +360 -0
  53. package/.claude/agents/engineering-electrical-engineer.md +575 -0
  54. package/.claude/agents/engineering-email-intelligence-engineer.md +390 -0
  55. package/.claude/agents/engineering-embedded-firmware-engineer.md +196 -0
  56. package/.claude/agents/engineering-feishu-integration-developer.md +606 -0
  57. package/.claude/agents/engineering-filament-optimization-specialist.md +283 -0
  58. package/.claude/agents/engineering-finops-engineer.md +164 -0
  59. package/.claude/agents/engineering-frontend-developer.md +257 -0
  60. package/.claude/agents/engineering-gaussdb-expert.md +334 -0
  61. package/.claude/agents/engineering-git-workflow-master.md +98 -0
  62. package/.claude/agents/engineering-i18n-engineer.md +214 -0
  63. package/.claude/agents/engineering-identity-access-engineer.md +217 -0
  64. package/.claude/agents/engineering-incident-response-commander.md +444 -0
  65. package/.claude/agents/engineering-iot-fleet-engineer.md +148 -0
  66. package/.claude/agents/engineering-it-service-manager.md +561 -0
  67. package/.claude/agents/engineering-knowledge-graph-engineer.md +372 -0
  68. package/.claude/agents/engineering-llm-post-training-engineer.md +166 -0
  69. package/.claude/agents/engineering-minimal-change-engineer.md +207 -0
  70. package/.claude/agents/engineering-mobile-app-builder.md +493 -0
  71. package/.claude/agents/engineering-mobile-release-engineer.md +164 -0
  72. package/.claude/agents/engineering-multi-agent-systems-architect.md +600 -0
  73. package/.claude/agents/engineering-network-engineer.md +245 -0
  74. package/.claude/agents/engineering-orgscript-engineer.md +113 -0
  75. package/.claude/agents/engineering-payments-billing-engineer.md +228 -0
  76. package/.claude/agents/engineering-pdf-engine-architect.md +696 -0
  77. package/.claude/agents/engineering-platform-engineer.md +270 -0
  78. package/.claude/agents/engineering-privacy-engineer.md +152 -0
  79. package/.claude/agents/engineering-prompt-engineer.md +209 -0
  80. package/.claude/agents/engineering-rag-pipeline-engineer.md +450 -0
  81. package/.claude/agents/engineering-rapid-prototyper.md +466 -0
  82. package/.claude/agents/engineering-realtime-collaboration-engineer.md +200 -0
  83. package/.claude/agents/engineering-rust-refactoring-specialist.md +313 -0
  84. package/.claude/agents/engineering-search-relevance-engineer.md +237 -0
  85. package/.claude/agents/engineering-section-508-specialist.md +339 -0
  86. package/.claude/agents/engineering-senior-developer.md +176 -0
  87. package/.claude/agents/engineering-servicenow-developer-mentor.md +180 -0
  88. package/.claude/agents/engineering-software-architect.md +112 -0
  89. package/.claude/agents/engineering-solidity-smart-contract-engineer.md +522 -0
  90. package/.claude/agents/engineering-sre.md +90 -0
  91. package/.claude/agents/engineering-technical-writer.md +393 -0
  92. package/.claude/agents/engineering-universal-document-compiler.md +370 -0
  93. package/.claude/agents/engineering-uswds-developer.md +340 -0
  94. package/.claude/agents/engineering-video-streaming-engineer.md +151 -0
  95. package/.claude/agents/engineering-voice-ai-integration-engineer.md +584 -0
  96. package/.claude/agents/engineering-webassembly-engineer.md +153 -0
  97. package/.claude/agents/engineering-wechat-mini-program-developer.md +356 -0
  98. package/.claude/agents/engineering-wordpress-performance.md +346 -0
  99. package/.claude/agents/engineering-wordpress-shopping-cart.md +346 -0
  100. package/.claude/agents/esg-sustainability-officer.md +396 -0
  101. package/.claude/agents/finance-bookkeeper-controller.md +260 -0
  102. package/.claude/agents/finance-financial-analyst.md +234 -0
  103. package/.claude/agents/finance-fpa-analyst.md +263 -0
  104. package/.claude/agents/finance-investment-researcher.md +272 -0
  105. package/.claude/agents/finance-tax-strategist.md +239 -0
  106. package/.claude/agents/game-audio-engineer.md +264 -0
  107. package/.claude/agents/game-designer.md +167 -0
  108. package/.claude/agents/gis-3d-scene-developer.md +111 -0
  109. package/.claude/agents/gis-analyst.md +91 -0
  110. package/.claude/agents/gis-bim-specialist.md +108 -0
  111. package/.claude/agents/gis-cartography-designer.md +150 -0
  112. package/.claude/agents/gis-drone-reality-mapping.md +120 -0
  113. package/.claude/agents/gis-geoai-ml-engineer.md +105 -0
  114. package/.claude/agents/gis-geoprocessing-specialist.md +97 -0
  115. package/.claude/agents/gis-qa-engineer.md +133 -0
  116. package/.claude/agents/gis-solution-engineer.md +101 -0
  117. package/.claude/agents/gis-spatial-data-engineer.md +97 -0
  118. package/.claude/agents/gis-spatial-data-scientist.md +111 -0
  119. package/.claude/agents/gis-technical-consultant.md +86 -0
  120. package/.claude/agents/gis-web-gis-developer.md +108 -0
  121. package/.claude/agents/godot-gameplay-scripter.md +338 -0
  122. package/.claude/agents/godot-multiplayer-engineer.md +301 -0
  123. package/.claude/agents/godot-shader-developer.md +267 -0
  124. package/.claude/agents/government-digital-presales-consultant.md +363 -0
  125. package/.claude/agents/grant-writer.md +511 -0
  126. package/.claude/agents/healthcare-aging-parent-care-companion.md +414 -0
  127. package/.claude/agents/healthcare-clinical-evidence-agent.md +231 -0
  128. package/.claude/agents/healthcare-customer-service.md +389 -0
  129. package/.claude/agents/healthcare-innovation-strategist.md +433 -0
  130. package/.claude/agents/healthcare-marketing-compliance.md +395 -0
  131. package/.claude/agents/healthcare-sovereign-health-systems-agent.md +312 -0
  132. package/.claude/agents/hospitality-guest-services.md +603 -0
  133. package/.claude/agents/hr-onboarding.md +451 -0
  134. package/.claude/agents/identity-graph-operator.md +303 -0
  135. package/.claude/agents/language-translator.md +264 -0
  136. package/.claude/agents/legal-billing-time-tracking.md +569 -0
  137. package/.claude/agents/legal-client-intake.md +492 -0
  138. package/.claude/agents/legal-document-review.md +454 -0
  139. package/.claude/agents/level-designer.md +208 -0
  140. package/.claude/agents/loan-officer-assistant.md +555 -0
  141. package/.claude/agents/lsp-index-engineer.md +320 -0
  142. package/.claude/agents/ma-integration-manager.md +427 -0
  143. package/.claude/agents/macos-spatial-metal-engineer.md +337 -0
  144. package/.claude/agents/marketing-aeo-foundations.md +264 -0
  145. package/.claude/agents/marketing-agentic-search-optimizer.md +279 -0
  146. package/.claude/agents/marketing-ai-citation-strategist.md +172 -0
  147. package/.claude/agents/marketing-app-store-optimizer.md +321 -0
  148. package/.claude/agents/marketing-baidu-seo-specialist.md +226 -0
  149. package/.claude/agents/marketing-bilibili-content-strategist.md +199 -0
  150. package/.claude/agents/marketing-book-co-author.md +110 -0
  151. package/.claude/agents/marketing-carousel-growth-engine.md +204 -0
  152. package/.claude/agents/marketing-china-ecommerce-operator.md +283 -0
  153. package/.claude/agents/marketing-china-market-localization-strategist.md +283 -0
  154. package/.claude/agents/marketing-content-creator.md +54 -0
  155. package/.claude/agents/marketing-cross-border-ecommerce.md +259 -0
  156. package/.claude/agents/marketing-developer-community-builder.md +285 -0
  157. package/.claude/agents/marketing-douyin-strategist.md +149 -0
  158. package/.claude/agents/marketing-email-strategist.md +249 -0
  159. package/.claude/agents/marketing-global-podcast-strategist.md +206 -0
  160. package/.claude/agents/marketing-growth-hacker.md +54 -0
  161. package/.claude/agents/marketing-instagram-curator.md +113 -0
  162. package/.claude/agents/marketing-kuaishou-strategist.md +223 -0
  163. package/.claude/agents/marketing-linkedin-content-creator.md +214 -0
  164. package/.claude/agents/marketing-livestream-commerce-coach.md +305 -0
  165. package/.claude/agents/marketing-multi-platform-publisher.md +217 -0
  166. package/.claude/agents/marketing-podcast-strategist.md +277 -0
  167. package/.claude/agents/marketing-pr-communications-manager.md +473 -0
  168. package/.claude/agents/marketing-private-domain-operator.md +308 -0
  169. package/.claude/agents/marketing-reddit-community-builder.md +123 -0
  170. package/.claude/agents/marketing-seo-specialist.md +370 -0
  171. package/.claude/agents/marketing-short-video-editing-coach.md +412 -0
  172. package/.claude/agents/marketing-social-media-strategist.md +125 -0
  173. package/.claude/agents/marketing-tiktok-strategist.md +125 -0
  174. package/.claude/agents/marketing-twitter-engager.md +126 -0
  175. package/.claude/agents/marketing-video-optimization-specialist.md +119 -0
  176. package/.claude/agents/marketing-wechat-official-account.md +145 -0
  177. package/.claude/agents/marketing-weibo-strategist.md +240 -0
  178. package/.claude/agents/marketing-x-twitter-intelligence-analyst.md +161 -0
  179. package/.claude/agents/marketing-xiaohongshu-specialist.md +138 -0
  180. package/.claude/agents/marketing-zhihu-strategist.md +162 -0
  181. package/.claude/agents/medical-billing-coding-specialist.md +491 -0
  182. package/.claude/agents/narrative-designer.md +243 -0
  183. package/.claude/agents/operations-manager.md +399 -0
  184. package/.claude/agents/organizational-psychologist.md +391 -0
  185. package/.claude/agents/paid-media-auditor.md +71 -0
  186. package/.claude/agents/paid-media-creative-strategist.md +71 -0
  187. package/.claude/agents/paid-media-paid-social-strategist.md +71 -0
  188. package/.claude/agents/paid-media-ppc-strategist.md +71 -0
  189. package/.claude/agents/paid-media-programmatic-buyer.md +71 -0
  190. package/.claude/agents/paid-media-search-query-analyst.md +71 -0
  191. package/.claude/agents/paid-media-tracking-specialist.md +71 -0
  192. package/.claude/agents/personal-growth-mentor.md +159 -0
  193. package/.claude/agents/product-behavioral-nudge-engine.md +83 -0
  194. package/.claude/agents/product-dx-engineer.md +332 -0
  195. package/.claude/agents/product-feedback-synthesizer.md +119 -0
  196. package/.claude/agents/product-manager.md +469 -0
  197. package/.claude/agents/product-sprint-prioritizer.md +154 -0
  198. package/.claude/agents/product-trend-researcher.md +159 -0
  199. package/.claude/agents/project-management-experiment-tracker.md +198 -0
  200. package/.claude/agents/project-management-jira-workflow-steward.md +233 -0
  201. package/.claude/agents/project-management-meeting-notes-specialist.md +95 -0
  202. package/.claude/agents/project-management-project-shepherd.md +194 -0
  203. package/.claude/agents/project-management-studio-operations.md +200 -0
  204. package/.claude/agents/project-management-studio-producer.md +203 -0
  205. package/.claude/agents/project-manager-senior.md +135 -0
  206. package/.claude/agents/real-estate-buyer-seller.md +596 -0
  207. package/.claude/agents/recruitment-specialist.md +515 -0
  208. package/.claude/agents/report-distribution-agent.md +65 -0
  209. package/.claude/agents/research-synthesist.md +136 -0
  210. package/.claude/agents/resume-tailor.md +230 -0
  211. package/.claude/agents/retail-customer-returns.md +566 -0
  212. package/.claude/agents/roblox-avatar-creator.md +297 -0
  213. package/.claude/agents/roblox-experience-designer.md +306 -0
  214. package/.claude/agents/roblox-systems-scripter.md +332 -0
  215. package/.claude/agents/sales-account-strategist.md +227 -0
  216. package/.claude/agents/sales-coach.md +271 -0
  217. package/.claude/agents/sales-data-extraction-agent.md +67 -0
  218. package/.claude/agents/sales-deal-strategist.md +180 -0
  219. package/.claude/agents/sales-discovery-coach.md +225 -0
  220. package/.claude/agents/sales-engineer.md +182 -0
  221. package/.claude/agents/sales-offer-lead-gen-strategist.md +257 -0
  222. package/.claude/agents/sales-outbound-strategist.md +201 -0
  223. package/.claude/agents/sales-outreach.md +425 -0
  224. package/.claude/agents/sales-pipeline-analyst.md +267 -0
  225. package/.claude/agents/sales-proposal-strategist.md +217 -0
  226. package/.claude/agents/security-ai-generated-code-auditor.md +207 -0
  227. package/.claude/agents/security-appsec-engineer.md +536 -0
  228. package/.claude/agents/security-architect.md +307 -0
  229. package/.claude/agents/security-blockchain-security-auditor.md +482 -0
  230. package/.claude/agents/security-cloud-security-architect.md +576 -0
  231. package/.claude/agents/security-compliance-auditor.md +158 -0
  232. package/.claude/agents/security-incident-responder.md +450 -0
  233. package/.claude/agents/security-penetration-tester.md +399 -0
  234. package/.claude/agents/security-secrets-credential-engineer.md +189 -0
  235. package/.claude/agents/security-senior-secops.md +760 -0
  236. package/.claude/agents/security-threat-detection-engineer.md +553 -0
  237. package/.claude/agents/security-threat-intelligence-analyst.md +661 -0
  238. package/.claude/agents/specialized-chief-of-staff.md +279 -0
  239. package/.claude/agents/specialized-civil-engineer.md +356 -0
  240. package/.claude/agents/specialized-codebase-archaeologist.md +341 -0
  241. package/.claude/agents/specialized-cultural-intelligence-strategist.md +88 -0
  242. package/.claude/agents/specialized-developer-advocate.md +317 -0
  243. package/.claude/agents/specialized-document-generator.md +55 -0
  244. package/.claude/agents/specialized-fedramp-rmf-compliance.md +378 -0
  245. package/.claude/agents/specialized-focus-music-architect.md +175 -0
  246. package/.claude/agents/specialized-french-consulting-market.md +194 -0
  247. package/.claude/agents/specialized-korean-business-navigator.md +216 -0
  248. package/.claude/agents/specialized-master-plan-architect.md +157 -0
  249. package/.claude/agents/specialized-mcp-builder.md +260 -0
  250. package/.claude/agents/specialized-model-qa.md +528 -0
  251. package/.claude/agents/specialized-pricing-analyst.md +243 -0
  252. package/.claude/agents/specialized-salesforce-architect.md +182 -0
  253. package/.claude/agents/specialized-strategy-duel-agent.md +130 -0
  254. package/.claude/agents/specialized-workflow-architect.md +597 -0
  255. package/.claude/agents/study-abroad-advisor.md +282 -0
  256. package/.claude/agents/supply-chain-strategist.md +591 -0
  257. package/.claude/agents/support-analytics-reporter.md +387 -0
  258. package/.claude/agents/support-executive-summary-generator.md +212 -0
  259. package/.claude/agents/support-finance-tracker.md +480 -0
  260. package/.claude/agents/support-infrastructure-maintainer.md +628 -0
  261. package/.claude/agents/support-legal-compliance-checker.md +591 -0
  262. package/.claude/agents/support-support-responder.md +607 -0
  263. package/.claude/agents/technical-artist.md +233 -0
  264. package/.claude/agents/terminal-integration-specialist.md +70 -0
  265. package/.claude/agents/testing-accessibility-auditor.md +327 -0
  266. package/.claude/agents/testing-api-tester.md +334 -0
  267. package/.claude/agents/testing-evidence-collector.md +210 -0
  268. package/.claude/agents/testing-performance-benchmarker.md +283 -0
  269. package/.claude/agents/testing-reality-checker.md +249 -0
  270. package/.claude/agents/testing-test-automation-engineer.md +196 -0
  271. package/.claude/agents/testing-test-results-analyzer.md +330 -0
  272. package/.claude/agents/testing-tool-evaluator.md +416 -0
  273. package/.claude/agents/testing-workflow-optimizer.md +463 -0
  274. package/.claude/agents/unity-architect.md +271 -0
  275. package/.claude/agents/unity-editor-tool-developer.md +310 -0
  276. package/.claude/agents/unity-multiplayer-engineer.md +323 -0
  277. package/.claude/agents/unity-shader-graph-artist.md +269 -0
  278. package/.claude/agents/unreal-multiplayer-architect.md +364 -0
  279. package/.claude/agents/unreal-systems-engineer.md +322 -0
  280. package/.claude/agents/unreal-technical-artist.md +256 -0
  281. package/.claude/agents/unreal-world-builder.md +273 -0
  282. package/.claude/agents/visionos-spatial-engineer.md +54 -0
  283. package/.claude/agents/xr-cockpit-interaction-specialist.md +32 -0
  284. package/.claude/agents/xr-immersive-developer.md +32 -0
  285. package/.claude/agents/xr-interface-architect.md +32 -0
  286. package/.claude/agents/zk-steward.md +211 -0
  287. package/.claude-plugin/marketplace.json +30 -0
  288. package/.commitlintrc.json +26 -0
  289. package/.devcontainer/Dockerfile +24 -0
  290. package/.devcontainer/devcontainer.json +70 -0
  291. package/.devcontainer/docker-compose.memory.yml +42 -0
  292. package/.devcontainer/memory-stack/devcontainer.json +84 -0
  293. package/.devcontainer/post-create.sh +83 -0
  294. package/.dockerignore +23 -0
  295. package/.env.act.example +15 -0
  296. package/.env.example +171 -0
  297. package/.eslintrc.json +16 -0
  298. package/.git-blame-ignore-revs +10 -0
  299. package/.gitattributes +2 -0
  300. package/.gitguardian.yaml +50 -0
  301. package/.githooks/pre-commit +6 -0
  302. package/.pre-commit-config.yaml +30 -0
  303. package/.release-please-config-xithead.json +22 -0
  304. package/.release-please-config.json +44 -0
  305. package/.release-please-manifest-xithead.json +3 -0
  306. package/.release-please-manifest.json +3 -0
  307. package/CODE_OF_CONDUCT.md +133 -0
  308. package/CONTRIBUTING.md +133 -0
  309. package/Cargo.lock +5557 -0
  310. package/Cargo.toml +121 -0
  311. package/Dockerfile +38 -0
  312. package/Dockerfile.proxy +48 -0
  313. package/ENTERPRISE.md +5 -0
  314. package/FEATURES_ADVANCED.md +311 -0
  315. package/Headroom-2.gif +0 -0
  316. package/HeadroomDemo-Fast.gif +0 -0
  317. package/MANIFEST.in +4 -0
  318. package/MODERNIZATION_GUIDE.md +358 -0
  319. package/Makefile +140 -0
  320. package/NOTICE +43 -0
  321. package/NPM_TOKEN_SETUP.md +81 -0
  322. package/PR.md +170 -0
  323. package/PUBLISH.md +77 -0
  324. package/README.md +108 -72
  325. package/README_FRONTEND.md +196 -0
  326. package/REALIGNMENT/00-overview.md +53 -0
  327. package/REALIGNMENT/01-bug-list.md +421 -0
  328. package/REALIGNMENT/02-architecture.md +322 -0
  329. package/REALIGNMENT/03-phase-A-lockdown.md +420 -0
  330. package/REALIGNMENT/04-phase-B-live-zone.md +470 -0
  331. package/REALIGNMENT/05-phase-C-rust-proxy.md +329 -0
  332. package/REALIGNMENT/06-phase-D-bedrock-vertex.md +222 -0
  333. package/REALIGNMENT/07-phase-E-cache-stabilization.md +338 -0
  334. package/REALIGNMENT/08-phase-F-auth-mode.md +254 -0
  335. package/REALIGNMENT/09-phase-G-rtk-observability.md +194 -0
  336. package/REALIGNMENT/10-phase-H-python-retirement.md +214 -0
  337. package/REALIGNMENT/11-phase-I-test-infra.md +385 -0
  338. package/REALIGNMENT/12-decisions-needed.md +196 -0
  339. package/REALIGNMENT/INDEX.md +82 -0
  340. package/REFACTORING_CAMPAIGN.md +277 -0
  341. package/RESTRUCTURE_PLAN.md +100 -0
  342. package/RUST_DEV.md +361 -0
  343. package/SECURITY.md +65 -0
  344. package/SETUP_AUTO_PUBLISH.md +164 -0
  345. package/TESTING-copilot-subscription.md +153 -0
  346. package/app/api/config/route.ts +22 -0
  347. package/app/api/export/route.ts +83 -0
  348. package/app/api/models/route.ts +12 -0
  349. package/app/api/stats/route.ts +46 -0
  350. package/app/dashboard/page.tsx +445 -0
  351. package/app/globals.css +35 -0
  352. package/app/installation/page.tsx +293 -0
  353. package/app/layout.tsx +35 -0
  354. package/app/master-design-system.css +2076 -0
  355. package/app/master-design-system.source.css +8 -0
  356. package/app/page.tsx +187 -0
  357. package/app/rechtliches/agb/page.tsx +165 -0
  358. package/app/rechtliches/datenschutz/page.tsx +128 -0
  359. package/app/rechtliches/impressum/page.tsx +122 -0
  360. package/app/rechtliches/widerruf/page.tsx +185 -0
  361. package/bin/xithead.js +264 -0
  362. package/codecov.yml +34 -0
  363. package/components/layout/AppShell.tsx +25 -0
  364. package/components/layout/Footer.tsx +120 -0
  365. package/components/layout/Header.tsx +145 -0
  366. package/components/marketing/LegalHighlights.tsx +45 -0
  367. package/components/marketing/LegalPageShell.tsx +49 -0
  368. package/components/marketing/LegalPanel.tsx +41 -0
  369. package/components/marketing/index.ts +3 -0
  370. package/crates/headroom-core/Cargo.toml +165 -0
  371. package/crates/headroom-core/benches/auth_mode.rs +74 -0
  372. package/crates/headroom-core/benches/ccr_store.rs +224 -0
  373. package/crates/headroom-core/benches/tokenizer.rs +52 -0
  374. package/crates/headroom-core/config/pipeline.toml +151 -0
  375. package/crates/headroom-core/proptest-regressions/transforms/tag_protector.txt +9 -0
  376. package/crates/headroom-core/src/auth_mode.rs +250 -0
  377. package/crates/headroom-core/src/cache_control.rs +414 -0
  378. package/crates/headroom-core/src/ccr/backends/in_memory.rs +317 -0
  379. package/crates/headroom-core/src/ccr/backends/mod.rs +152 -0
  380. package/crates/headroom-core/src/ccr/backends/redis.rs +146 -0
  381. package/crates/headroom-core/src/ccr/backends/sqlite.rs +205 -0
  382. package/crates/headroom-core/src/ccr/mod.rs +119 -0
  383. package/crates/headroom-core/src/compression_policy.rs +505 -0
  384. package/crates/headroom-core/src/lib.rs +33 -0
  385. package/crates/headroom-core/src/relevance/base.rs +139 -0
  386. package/crates/headroom-core/src/relevance/bm25.rs +385 -0
  387. package/crates/headroom-core/src/relevance/embedding.rs +354 -0
  388. package/crates/headroom-core/src/relevance/hybrid.rs +364 -0
  389. package/crates/headroom-core/src/relevance/mod.rs +64 -0
  390. package/crates/headroom-core/src/signals/keyword_detector.rs +433 -0
  391. package/crates/headroom-core/src/signals/line_importance.rs +84 -0
  392. package/crates/headroom-core/src/signals/mod.rs +61 -0
  393. package/crates/headroom-core/src/signals/tiered.rs +141 -0
  394. package/crates/headroom-core/src/tokenizer/estimator.rs +152 -0
  395. package/crates/headroom-core/src/tokenizer/hf_impl.rs +314 -0
  396. package/crates/headroom-core/src/tokenizer/mod.rs +56 -0
  397. package/crates/headroom-core/src/tokenizer/registry.rs +342 -0
  398. package/crates/headroom-core/src/tokenizer/tiktoken_impl.rs +265 -0
  399. package/crates/headroom-core/src/transforms/adaptive_sizer.rs +610 -0
  400. package/crates/headroom-core/src/transforms/anchor_selector.rs +1189 -0
  401. package/crates/headroom-core/src/transforms/content_detector.rs +769 -0
  402. package/crates/headroom-core/src/transforms/detection.rs +222 -0
  403. package/crates/headroom-core/src/transforms/diff_compressor.rs +1685 -0
  404. package/crates/headroom-core/src/transforms/live_zone.rs +2967 -0
  405. package/crates/headroom-core/src/transforms/log_compressor.rs +1295 -0
  406. package/crates/headroom-core/src/transforms/magika_detector.rs +395 -0
  407. package/crates/headroom-core/src/transforms/mod.rs +64 -0
  408. package/crates/headroom-core/src/transforms/pipeline/config.rs +331 -0
  409. package/crates/headroom-core/src/transforms/pipeline/mod.rs +103 -0
  410. package/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rs +524 -0
  411. package/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs +279 -0
  412. package/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs +400 -0
  413. package/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rs +284 -0
  414. package/crates/headroom-core/src/transforms/pipeline/offloads/mod.rs +34 -0
  415. package/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rs +331 -0
  416. package/crates/headroom-core/src/transforms/pipeline/orchestrator.rs +848 -0
  417. package/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rs +175 -0
  418. package/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rs +545 -0
  419. package/crates/headroom-core/src/transforms/pipeline/reformats/mod.rs +12 -0
  420. package/crates/headroom-core/src/transforms/pipeline/traits.rs +338 -0
  421. package/crates/headroom-core/src/transforms/recommendations.rs +329 -0
  422. package/crates/headroom-core/src/transforms/safety.rs +215 -0
  423. package/crates/headroom-core/src/transforms/search_compressor.rs +902 -0
  424. package/crates/headroom-core/src/transforms/smart_crusher/analyzer.rs +1262 -0
  425. package/crates/headroom-core/src/transforms/smart_crusher/anchors.rs +407 -0
  426. package/crates/headroom-core/src/transforms/smart_crusher/builder.rs +263 -0
  427. package/crates/headroom-core/src/transforms/smart_crusher/classifier.rs +214 -0
  428. package/crates/headroom-core/src/transforms/smart_crusher/compaction/classifier.rs +283 -0
  429. package/crates/headroom-core/src/transforms/smart_crusher/compaction/compactor.rs +733 -0
  430. package/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rs +961 -0
  431. package/crates/headroom-core/src/transforms/smart_crusher/compaction/ir.rs +252 -0
  432. package/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs +127 -0
  433. package/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs +397 -0
  434. package/crates/headroom-core/src/transforms/smart_crusher/config.rs +173 -0
  435. package/crates/headroom-core/src/transforms/smart_crusher/constraints.rs +159 -0
  436. package/crates/headroom-core/src/transforms/smart_crusher/crusher.rs +1665 -0
  437. package/crates/headroom-core/src/transforms/smart_crusher/crushers.rs +865 -0
  438. package/crates/headroom-core/src/transforms/smart_crusher/error_keywords.rs +77 -0
  439. package/crates/headroom-core/src/transforms/smart_crusher/field_detect.rs +393 -0
  440. package/crates/headroom-core/src/transforms/smart_crusher/hashing.rs +72 -0
  441. package/crates/headroom-core/src/transforms/smart_crusher/mod.rs +81 -0
  442. package/crates/headroom-core/src/transforms/smart_crusher/observer.rs +66 -0
  443. package/crates/headroom-core/src/transforms/smart_crusher/orchestration.rs +459 -0
  444. package/crates/headroom-core/src/transforms/smart_crusher/outliers.rs +507 -0
  445. package/crates/headroom-core/src/transforms/smart_crusher/planning.rs +904 -0
  446. package/crates/headroom-core/src/transforms/smart_crusher/statistics.rs +510 -0
  447. package/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs +287 -0
  448. package/crates/headroom-core/src/transforms/smart_crusher/traits.rs +203 -0
  449. package/crates/headroom-core/src/transforms/smart_crusher/types.rs +263 -0
  450. package/crates/headroom-core/src/transforms/tag_protector.rs +1272 -0
  451. package/crates/headroom-core/src/transforms/unidiff_detector.rs +232 -0
  452. package/crates/headroom-parity/Cargo.toml +26 -0
  453. package/crates/headroom-parity/examples/diff_fixture.rs +66 -0
  454. package/crates/headroom-parity/src/bin/parity_run.rs +78 -0
  455. package/crates/headroom-parity/src/lib.rs +613 -0
  456. package/crates/headroom-proxy/Cargo.toml +119 -0
  457. package/crates/headroom-proxy/data/model_prices_and_context_window.json +39767 -0
  458. package/crates/headroom-proxy/src/bedrock/auth_mode_layer.rs +192 -0
  459. package/crates/headroom-proxy/src/bedrock/envelope.rs +239 -0
  460. package/crates/headroom-proxy/src/bedrock/eventstream.rs +769 -0
  461. package/crates/headroom-proxy/src/bedrock/eventstream_to_sse.rs +459 -0
  462. package/crates/headroom-proxy/src/bedrock/invoke.rs +735 -0
  463. package/crates/headroom-proxy/src/bedrock/invoke_streaming.rs +1098 -0
  464. package/crates/headroom-proxy/src/bedrock/mod.rs +66 -0
  465. package/crates/headroom-proxy/src/bedrock/sigv4.rs +324 -0
  466. package/crates/headroom-proxy/src/bedrock/vendor.rs +82 -0
  467. package/crates/headroom-proxy/src/cache_stabilization/anthropic_cache_control.rs +698 -0
  468. package/crates/headroom-proxy/src/cache_stabilization/drift_detector.rs +683 -0
  469. package/crates/headroom-proxy/src/cache_stabilization/mod.rs +61 -0
  470. package/crates/headroom-proxy/src/cache_stabilization/openai_cache_key.rs +600 -0
  471. package/crates/headroom-proxy/src/cache_stabilization/tool_def_normalize.rs +514 -0
  472. package/crates/headroom-proxy/src/cache_stabilization/volatile_detector.rs +720 -0
  473. package/crates/headroom-proxy/src/compression/anthropic.rs +92 -0
  474. package/crates/headroom-proxy/src/compression/live_zone_anthropic.rs +1311 -0
  475. package/crates/headroom-proxy/src/compression/live_zone_openai.rs +660 -0
  476. package/crates/headroom-proxy/src/compression/live_zone_responses.rs +678 -0
  477. package/crates/headroom-proxy/src/compression/mod.rs +127 -0
  478. package/crates/headroom-proxy/src/compression/model_limits.rs +211 -0
  479. package/crates/headroom-proxy/src/config.rs +646 -0
  480. package/crates/headroom-proxy/src/error.rs +69 -0
  481. package/crates/headroom-proxy/src/handlers/chat_completions.rs +100 -0
  482. package/crates/headroom-proxy/src/handlers/conversations.rs +241 -0
  483. package/crates/headroom-proxy/src/handlers/mod.rs +13 -0
  484. package/crates/headroom-proxy/src/handlers/responses.rs +256 -0
  485. package/crates/headroom-proxy/src/headers.rs +288 -0
  486. package/crates/headroom-proxy/src/health.rs +41 -0
  487. package/crates/headroom-proxy/src/lib.rs +21 -0
  488. package/crates/headroom-proxy/src/main.rs +141 -0
  489. package/crates/headroom-proxy/src/observability/cache_hit_rate.rs +340 -0
  490. package/crates/headroom-proxy/src/observability/compression_ratio.rs +158 -0
  491. package/crates/headroom-proxy/src/observability/metric_names.rs +174 -0
  492. package/crates/headroom-proxy/src/observability/mod.rs +68 -0
  493. package/crates/headroom-proxy/src/observability/prometheus.rs +394 -0
  494. package/crates/headroom-proxy/src/observability/proxy_metrics.rs +368 -0
  495. package/crates/headroom-proxy/src/proxy.rs +1619 -0
  496. package/crates/headroom-proxy/src/responses_items.rs +565 -0
  497. package/crates/headroom-proxy/src/sse/anthropic.rs +415 -0
  498. package/crates/headroom-proxy/src/sse/framing.rs +366 -0
  499. package/crates/headroom-proxy/src/sse/mod.rs +40 -0
  500. package/crates/headroom-proxy/src/sse/openai_chat.rs +231 -0
  501. package/crates/headroom-proxy/src/sse/openai_responses.rs +458 -0
  502. package/crates/headroom-proxy/src/vertex/adc.rs +315 -0
  503. package/crates/headroom-proxy/src/vertex/envelope.rs +167 -0
  504. package/crates/headroom-proxy/src/vertex/mod.rs +277 -0
  505. package/crates/headroom-proxy/src/vertex/raw_predict.rs +490 -0
  506. package/crates/headroom-proxy/src/vertex/stream_raw_predict.rs +49 -0
  507. package/crates/headroom-proxy/src/websocket.rs +246 -0
  508. package/crates/headroom-py/Cargo.toml +37 -0
  509. package/crates/headroom-py/build.rs +77 -0
  510. package/crates/headroom-py/glibc_compat.c +170 -0
  511. package/crates/headroom-py/src/lib.rs +1655 -0
  512. package/deny.toml +32 -0
  513. package/design-system/base.css +127 -0
  514. package/design-system/components.css +1943 -0
  515. package/design-system/fonts.css +17 -0
  516. package/design-system/tailwind-preset.js +95 -0
  517. package/design-system/tokens.css +48 -0
  518. package/design-system/utilities.css +53 -0
  519. package/docker/differential-network-capture/Dockerfile.runner +14 -0
  520. package/docker/differential-network-capture/docker-compose.yml +142 -0
  521. package/docker/differential-network-capture/run-claude-lane.sh +24 -0
  522. package/docker/docker-compose.native.yml +44 -0
  523. package/docker-bake.hcl +88 -0
  524. package/docker-compose.yml +80 -0
  525. package/e2e/_lib/make_shim.ps1 +24 -0
  526. package/e2e/_lib/make_shim.sh +28 -0
  527. package/e2e/docker-native-install.sh +120 -0
  528. package/e2e/init/Dockerfile +79 -0
  529. package/e2e/wrap/Dockerfile +115 -0
  530. package/examples/07-context-compression.ipynb +279 -0
  531. package/examples/deployment/macos-launchagent/com.headroom.proxy.plist.template +65 -0
  532. package/examples/deployment/macos-launchagent/install.sh +227 -0
  533. package/examples/deployment/macos-launchagent/shell-integration.sh +74 -0
  534. package/examples/deployment/macos-launchagent/uninstall.sh +145 -0
  535. package/headroom-savings.png +0 -0
  536. package/headroom_learn.gif +0 -0
  537. package/lib/config/index.ts +12 -0
  538. package/lib/hooks/index.ts +4 -0
  539. package/lib/hooks/use-mounted.ts +13 -0
  540. package/lib/hooks/use-session.ts +34 -0
  541. package/lib/hooks/use-settings.ts +49 -0
  542. package/lib/utils/index.ts +1 -0
  543. package/lib/utils/settings.ts +162 -0
  544. package/lib/utils/utils.ts +6 -0
  545. package/llms.txt +67 -0
  546. package/middleware.ts +26 -0
  547. package/mkdocs.yml +136 -0
  548. package/next-env.d.ts +7 -0
  549. package/next.config.ts +12 -0
  550. package/npm/bin/xithead.js +366 -366
  551. package/npm/bin/xtream-head.js +366 -0
  552. package/npm/index.d.ts +57 -57
  553. package/npm/index.js +139 -139
  554. package/npm/package.json +55 -54
  555. package/npm/postinstall.js +43 -43
  556. package/package.json +58 -36
  557. package/plugins/headroom-agent-hooks/.claude-plugin/plugin.json +17 -0
  558. package/plugins/headroom-agent-hooks/hooks/hooks.json +29 -0
  559. package/plugins/headroom-oauth2/LICENSE +3 -0
  560. package/plugins/headroom-oauth2/SPEC.md +115 -0
  561. package/plugins/headroom-oauth2/pyproject.toml +48 -0
  562. package/plugins/hermes/headroom_retrieve/plugin.yaml +6 -0
  563. package/plugins/openclaw/hook-shim/HOOK.md +21 -0
  564. package/plugins/openclaw/hook-shim/handler.js +3 -0
  565. package/plugins/openclaw/openclaw.plugin.json +98 -0
  566. package/plugins/openclaw/package-lock.json +2692 -0
  567. package/plugins/openclaw/package.json +54 -0
  568. package/plugins/openclaw/prepare-dist.mjs +56 -0
  569. package/plugins/openclaw/src/convert.ts +418 -0
  570. package/plugins/openclaw/src/engine.ts +284 -0
  571. package/plugins/openclaw/src/gateway-config.ts +143 -0
  572. package/plugins/openclaw/src/index.ts +11 -0
  573. package/plugins/openclaw/src/plugin/index.ts +98 -0
  574. package/plugins/openclaw/src/proxy-manager.ts +477 -0
  575. package/plugins/openclaw/src/tools/headroom-retrieve.ts +76 -0
  576. package/plugins/openclaw/test/convert.test.ts +96 -0
  577. package/plugins/openclaw/test/engine-normalization.test.ts +21 -0
  578. package/plugins/openclaw/test/engine.test.ts +101 -0
  579. package/plugins/openclaw/test/gateway-config.test.ts +278 -0
  580. package/plugins/openclaw/test/plugin-runtime-routing.test.ts +141 -0
  581. package/plugins/openclaw/test/proxy-manager.test.ts +343 -0
  582. package/plugins/openclaw/tsup.config.ts +9 -0
  583. package/postcss.config.mjs +7 -0
  584. package/pyproject.toml +438 -0
  585. package/rust-toolchain.toml +16 -0
  586. package/sdk/typescript/examples/basic-compress.ts +64 -0
  587. package/sdk/typescript/examples/ccr-retrieve.ts +80 -0
  588. package/sdk/typescript/examples/hooks-custom-compression.ts +117 -0
  589. package/sdk/typescript/examples/middleware-composition.ts +59 -0
  590. package/sdk/typescript/examples/multi-provider.ts +53 -0
  591. package/sdk/typescript/examples/openai-anthropic-adapters.ts +73 -0
  592. package/sdk/typescript/examples/shared-context-multi-agent.ts +74 -0
  593. package/sdk/typescript/examples/simulation-dry-run.ts +80 -0
  594. package/sdk/typescript/examples/streaming-chat.ts +71 -0
  595. package/sdk/typescript/examples/structured-output.ts +99 -0
  596. package/sdk/typescript/examples/tool-calling-agent.ts +86 -0
  597. package/sdk/typescript/examples/with-headroom-vercel.ts +57 -0
  598. package/sdk/typescript/package-lock.json +3437 -0
  599. package/sdk/typescript/package.json +108 -0
  600. package/sdk/typescript/src/adapters/anthropic.ts +218 -0
  601. package/sdk/typescript/src/adapters/gemini.ts +83 -0
  602. package/sdk/typescript/src/adapters/openai.ts +77 -0
  603. package/sdk/typescript/src/adapters/vercel-ai.ts +131 -0
  604. package/sdk/typescript/src/client.ts +632 -0
  605. package/sdk/typescript/src/compress.ts +87 -0
  606. package/sdk/typescript/src/errors.ts +114 -0
  607. package/sdk/typescript/src/hooks.ts +113 -0
  608. package/sdk/typescript/src/index.ts +140 -0
  609. package/sdk/typescript/src/paths.ts +272 -0
  610. package/sdk/typescript/src/shared-context.ts +188 -0
  611. package/sdk/typescript/src/simulate.ts +52 -0
  612. package/sdk/typescript/src/types/config.ts +183 -0
  613. package/sdk/typescript/src/types/models.ts +314 -0
  614. package/sdk/typescript/src/types.ts +139 -0
  615. package/sdk/typescript/src/utils/case.ts +34 -0
  616. package/sdk/typescript/src/utils/format.ts +480 -0
  617. package/sdk/typescript/src/utils/stream.ts +50 -0
  618. package/sdk/typescript/test/adapters/anthropic.test.ts +217 -0
  619. package/sdk/typescript/test/adapters/gemini.test.ts +131 -0
  620. package/sdk/typescript/test/adapters/openai.test.ts +174 -0
  621. package/sdk/typescript/test/adapters/vercel-ai.test.ts +306 -0
  622. package/sdk/typescript/test/client-expanded.test.ts +333 -0
  623. package/sdk/typescript/test/client.test.ts +266 -0
  624. package/sdk/typescript/test/compress.test.ts +118 -0
  625. package/sdk/typescript/test/config.test.ts +275 -0
  626. package/sdk/typescript/test/errors.test.ts +169 -0
  627. package/sdk/typescript/test/format-integration.test.ts +252 -0
  628. package/sdk/typescript/test/hooks.test.ts +265 -0
  629. package/sdk/typescript/test/integration.test.ts +249 -0
  630. package/sdk/typescript/test/mdx-examples.test.ts +253 -0
  631. package/sdk/typescript/test/models.test.ts +318 -0
  632. package/sdk/typescript/test/parity.test.ts +337 -0
  633. package/sdk/typescript/test/paths.test.ts +447 -0
  634. package/sdk/typescript/test/shared-context.test.ts +245 -0
  635. package/sdk/typescript/test/simulate.test.ts +84 -0
  636. package/sdk/typescript/test/types.test.ts +244 -0
  637. package/sdk/typescript/test/utils/case.test.ts +129 -0
  638. package/sdk/typescript/test/utils/detect-format.test.ts +133 -0
  639. package/sdk/typescript/test/utils/format.test.ts +434 -0
  640. package/sdk/typescript/test/vercel-ai-e2e.test.ts +179 -0
  641. package/sdk/typescript/tsup.config.ts +17 -0
  642. package/tailwind.master.config.cjs +47 -0
  643. package/types/api.ts +12 -0
  644. package/types/config.ts +9 -0
  645. package/types/index.ts +2 -0
  646. package/uv.lock +6951 -0
  647. package/xithead.egg-info/PKG-INFO +22 -0
  648. package/xithead.egg-info/SOURCES.txt +16 -0
  649. package/xithead.egg-info/dependency_links.txt +1 -0
  650. package/xithead.egg-info/entry_points.txt +2 -0
  651. package/xithead.egg-info/requires.txt +10 -0
  652. package/xithead.egg-info/top_level.txt +2 -0
  653. package/xithead.ps1 +136 -0
  654. package/xithead.sh +129 -0
  655. package/xithead_dist/pyproject.toml +39 -0
  656. package/xithead_dist/xithead.egg-info/PKG-INFO +22 -0
  657. package/xithead_dist/xithead.egg-info/SOURCES.txt +16 -0
  658. package/xithead_dist/xithead.egg-info/dependency_links.txt +1 -0
  659. package/xithead_dist/xithead.egg-info/entry_points.txt +3 -0
  660. package/xithead_dist/xithead.egg-info/requires.txt +10 -0
  661. package/xithead_dist/xithead.egg-info/top_level.txt +2 -0
@@ -0,0 +1,1665 @@
1
+ //! `SmartCrusher` struct — top-level entry point for compression.
2
+ //!
3
+ //! Owns the `config`, `anchor_selector`, `scorer`, and `analyzer`
4
+ //! singletons that every per-message call needs. Constructed once
5
+ //! per process; the struct is `Send + Sync` so it can sit behind an
6
+ //! `Arc` in a multi-threaded proxy.
7
+ //!
8
+ //! This module ports three Python entry points:
9
+ //!
10
+ //! - `_execute_plan` (line 3617) → `SmartCrusher::execute_plan`
11
+ //! - `_crush_array` (line 2400) → `SmartCrusher::crush_array`
12
+ //! - `_crush_mixed_array` (line 2914) → `SmartCrusher::crush_mixed_array`
13
+ //!
14
+ //! # Stubs that match Python's "everything-disabled" path
15
+ //!
16
+ //! Python's `_crush_array` calls into TOIN (cross-user pattern
17
+ //! learning), feedback (per-tool compression hints), CCR (compress-
18
+ //! cache-retrieve store), and telemetry. All four are large separate
19
+ //! systems with their own state. For the like-for-like port at Stage
20
+ //! 3c.1, we mirror Python's behavior **when those subsystems are
21
+ //! disabled**:
22
+ //!
23
+ //! - **TOIN**: never produces a recommendation, never overrides
24
+ //! `effective_max_items`, never injects preserve_fields/strategy/level.
25
+ //! - **Feedback**: never produces hints; default `effective_max_items`.
26
+ //! - **CCR**: `enabled=false`; result has `ccr_hash = None`.
27
+ //! - **Telemetry**: no-op.
28
+ //! - **`_compress_text_within_items`**: pass-through (returns input
29
+ //! unchanged) since text compression has its own port pipeline.
30
+ //! - **`summarize_dropped_items`**: empty string.
31
+ //!
32
+ //! Parity fixtures will be recorded with all four disabled on the
33
+ //! Python side, locking byte-equal output. The TOIN/CCR/feedback
34
+ //! integration ports happen later (Stage 3c.2 follow-ups).
35
+
36
+ use std::sync::Arc;
37
+
38
+ use serde_json::Value;
39
+
40
+ use super::analyzer::SmartAnalyzer;
41
+ use super::builder::SmartCrusherBuilder;
42
+ use super::classifier::{classify_array, ArrayType};
43
+ use super::compaction::{
44
+ classify_cell, emit_opaque_ccr_marker, try_parse_json_container, CellClass, ClassifyConfig,
45
+ CompactConfig, Compaction, CompactionStage,
46
+ };
47
+ use super::config::SmartCrusherConfig;
48
+ use super::crushers::{compute_k_split, crush_number_array, crush_object, crush_string_array};
49
+ use super::planning::SmartCrusherPlanner;
50
+ use super::traits::{Constraint, CrushEvent, Observer};
51
+ use super::types::{CompressionPlan, CompressionStrategy, CrushResult};
52
+ use crate::ccr::CcrStore;
53
+ use crate::relevance::RelevanceScorer;
54
+ use crate::transforms::adaptive_sizer::compute_optimal_k;
55
+ use crate::transforms::anchor_selector::AnchorSelector;
56
+
57
+ /// Return type for `crush_array`.
58
+ ///
59
+ /// Two operating paths feed the same result type:
60
+ ///
61
+ /// - **Lossless path** — input compacted to a smaller inline form
62
+ /// (e.g. CSV+schema). Nothing dropped; `compacted` is populated;
63
+ /// `ccr_hash` is `None` (no retrieval needed because everything is
64
+ /// already in the prompt).
65
+ /// - **Lossy path** — input compressed by row-dropping. `items` holds
66
+ /// the kept subset; `ccr_hash` is `Some(hash)` so the runtime can
67
+ /// cache the **full original** keyed by that hash and serve it back
68
+ /// to the LLM via a retrieval tool call. **No data is lost** —
69
+ /// "lossy" here means "compressed view inline; full payload cached
70
+ /// for tool retrieval," matching Python's CCR-Dropped semantics.
71
+ ///
72
+ /// The runtime (PyO3 bridge / proxy server) owns the cache; this crate
73
+ /// computes the hash and emits a marker so the prompt knows where to
74
+ /// look.
75
+ pub struct CrushArrayResult {
76
+ /// Kept items. For the lossless path this is the full original
77
+ /// (nothing was dropped). For the lossy path this is the surviving
78
+ /// subset; the rest is retrievable via `ccr_hash`.
79
+ pub items: Vec<Value>,
80
+ /// Strategy debug string. One of:
81
+ /// - `"none:adaptive_at_limit"` / `"skip:<reason>"` — passthrough
82
+ /// - `"lossless:table"` / `"lossless:buckets"` — lossless wins
83
+ /// - `"smart_sample"` / `"top_n"` / `"cluster"` / `"time_series"` —
84
+ /// lossy path with row-dropping.
85
+ pub strategy_info: String,
86
+ /// 12-char SHA-256 hex prefix of the **full original input**.
87
+ /// Populated when the lossy path dropped rows; the runtime is
88
+ /// expected to cache the original items keyed by this hash so a
89
+ /// retrieval tool can serve them back. `None` when nothing was
90
+ /// dropped (lossless path or below adaptive_k boundary).
91
+ pub ccr_hash: Option<String>,
92
+ /// Marker text inserted into the prompt to advertise the CCR
93
+ /// pointer (e.g. `<<ccr:abc123def456 42_rows_offloaded>>`). Empty
94
+ /// when `ccr_hash` is `None`.
95
+ pub dropped_summary: String,
96
+ /// Rendered bytes from the compaction stage when the **lossless
97
+ /// path** won. `None` for the lossy path or when compaction wasn't
98
+ /// configured.
99
+ pub compacted: Option<String>,
100
+ /// Top-level [`Compaction`] variant tag — `"table"`, `"buckets"`,
101
+ /// `"ccr"`. Mirrors `compacted` — populated only when lossless won.
102
+ pub compaction_kind: Option<&'static str>,
103
+ }
104
+
105
+ /// Top-level SmartCrusher.
106
+ ///
107
+ /// Three pluggable extensions (Stage 3c.2 PR1):
108
+ /// - `scorer` — relevance scoring (`HybridScorer` by default).
109
+ /// - `constraints` — must-keep predicates (`KeepErrorsConstraint` +
110
+ /// `KeepStructuralOutliersConstraint` by default).
111
+ /// - `observers` — decision-stream telemetry (`TracingObserver` by
112
+ /// default).
113
+ ///
114
+ /// Compose via [`SmartCrusherBuilder`]; or call `SmartCrusher::new()`
115
+ /// for the OSS default composition.
116
+ pub struct SmartCrusher {
117
+ pub config: SmartCrusherConfig,
118
+ pub anchor_selector: AnchorSelector,
119
+ pub scorer: Box<dyn RelevanceScorer + Send + Sync>,
120
+ pub analyzer: SmartAnalyzer,
121
+ pub constraints: Vec<Box<dyn Constraint>>,
122
+ pub observers: Vec<Box<dyn Observer>>,
123
+ /// Optional lossless-first compaction stage (Stage 3c.2 PR2). When
124
+ /// set, `crush_array` runs compaction up front and short-circuits
125
+ /// the lossy path on success. When `None` (default OSS), parity
126
+ /// with the pre-PR2 lossy-only pipeline is preserved exactly.
127
+ pub compaction: Option<CompactionStage>,
128
+ /// Optional CCR store. When set, the lossy path stashes the **full
129
+ /// original** array into the store keyed by `ccr_hash` before
130
+ /// returning — the runtime can then serve dropped rows back via
131
+ /// retrieval tool calls. When `None`, hashes are still emitted but
132
+ /// nothing is stored (legacy / parity mode).
133
+ ///
134
+ /// `Arc` so callers can keep their own handle to the same store
135
+ /// (e.g. the proxy server holds it for retrieval lookups while
136
+ /// SmartCrusher writes through it).
137
+ pub ccr_store: Option<Arc<dyn CcrStore>>,
138
+ }
139
+
140
+ impl SmartCrusher {
141
+ /// Construct with the OSS default composition: scorer + constraints +
142
+ /// observer + **lossless-first compaction stage**. Calling
143
+ /// `crush_array` runs the dispatch:
144
+ ///
145
+ /// 1. Try the lossless compactor.
146
+ /// 2. If savings ratio ≥ `config.lossless_min_savings_ratio`
147
+ /// (default `0.30`), ship lossless — `compacted` populated,
148
+ /// `ccr_hash = None`, nothing dropped.
149
+ /// 3. Otherwise fall through to the lossy path — drop rows,
150
+ /// populate `ccr_hash` with a hash of the full original so the
151
+ /// runtime can cache the payload for tool retrieval.
152
+ ///
153
+ /// **No data is ever lost.** The lossy path moves dropped rows to
154
+ /// CCR cache, not to nowhere — same semantics as Python's
155
+ /// SmartCrusher with CCR enabled.
156
+ pub fn new(config: SmartCrusherConfig) -> Self {
157
+ // Carry the compaction heuristics from the crusher config into
158
+ // the compaction stage; everything not exposed on
159
+ // SmartCrusherConfig keeps its CompactConfig default.
160
+ let compact_cfg = CompactConfig {
161
+ core_field_fraction: config.compaction_core_field_fraction,
162
+ heterogeneous_core_ratio: config.compaction_heterogeneous_core_ratio,
163
+ max_flatten_inner_keys: config.compaction_max_flatten_inner_keys,
164
+ min_buckets: config.compaction_min_buckets,
165
+ max_buckets: config.compaction_max_buckets,
166
+ ..CompactConfig::default()
167
+ };
168
+ SmartCrusherBuilder::new(config)
169
+ .with_default_oss_setup()
170
+ .with_compaction(CompactionStage::csv_schema(compact_cfg))
171
+ .with_default_ccr_store()
172
+ .build()
173
+ }
174
+
175
+ /// Construct WITHOUT the compaction stage. Pre-PR4 behavior:
176
+ /// `crush_array` skips the lossless attempt and runs the lossy
177
+ /// path directly (still with CCR-Dropped retrieval markers from
178
+ /// PR4). Used by:
179
+ ///
180
+ /// - The 17 legacy parity fixtures (recorded against the
181
+ /// lossy-only path; using this constructor preserves byte-equal
182
+ /// coverage).
183
+ /// - Callers who explicitly don't want lossless attempts (e.g.
184
+ /// workloads where the compactor's overhead isn't worth the
185
+ /// modest tabular wins).
186
+ pub fn without_compaction(config: SmartCrusherConfig) -> Self {
187
+ SmartCrusherBuilder::new(config)
188
+ .with_default_oss_setup()
189
+ .with_default_ccr_store()
190
+ .build()
191
+ }
192
+
193
+ /// Construct like [`SmartCrusher::new`] but with the compaction
194
+ /// stage's formatter chosen by name (`"csv-schema"`, `"json"`,
195
+ /// `"markdown-kv"`). `None` for unknown names — callers own the
196
+ /// fallback/error policy. `"csv-schema"` is equivalent to `new`.
197
+ pub fn with_compaction_format(config: SmartCrusherConfig, format_name: &str) -> Option<Self> {
198
+ let stage = CompactionStage::from_format_name(format_name)?;
199
+ Some(
200
+ SmartCrusherBuilder::new(config)
201
+ .with_default_oss_setup()
202
+ .with_compaction(stage)
203
+ .with_default_ccr_store()
204
+ .build(),
205
+ )
206
+ }
207
+
208
+ /// Begin a builder chain for custom composition. The Enterprise
209
+ /// entry point: swap the scorer, add business-rule constraints,
210
+ /// attach an audit observer.
211
+ pub fn builder(config: SmartCrusherConfig) -> SmartCrusherBuilder {
212
+ SmartCrusherBuilder::new(config)
213
+ }
214
+
215
+ /// Construct with a custom scorer (legacy convenience). Equivalent
216
+ /// to `SmartCrusher::builder(config).with_scorer(scorer).with_default_oss_setup().build()`
217
+ /// minus the default scorer override; preserved for backward
218
+ /// compatibility with pre-PR1 callers.
219
+ pub fn with_scorer(
220
+ config: SmartCrusherConfig,
221
+ scorer: Box<dyn RelevanceScorer + Send + Sync>,
222
+ ) -> Self {
223
+ SmartCrusherBuilder::new(config)
224
+ .with_scorer(scorer)
225
+ .add_default_oss_constraints()
226
+ .build()
227
+ }
228
+
229
+ /// Construct directly from owned parts. Used by
230
+ /// [`SmartCrusherBuilder::build`] — not part of the public stable
231
+ /// API. Prefer the builder.
232
+ #[doc(hidden)]
233
+ #[allow(clippy::too_many_arguments)]
234
+ pub fn from_parts(
235
+ config: SmartCrusherConfig,
236
+ anchor_selector: AnchorSelector,
237
+ scorer: Box<dyn RelevanceScorer + Send + Sync>,
238
+ analyzer: SmartAnalyzer,
239
+ constraints: Vec<Box<dyn Constraint>>,
240
+ observers: Vec<Box<dyn Observer>>,
241
+ compaction: Option<CompactionStage>,
242
+ ccr_store: Option<Arc<dyn CcrStore>>,
243
+ ) -> Self {
244
+ SmartCrusher {
245
+ config,
246
+ anchor_selector,
247
+ scorer,
248
+ analyzer,
249
+ constraints,
250
+ observers,
251
+ compaction,
252
+ ccr_store,
253
+ }
254
+ }
255
+
256
+ /// Handle to the CCR store, if configured. Used by the runtime
257
+ /// (proxy server, PyO3 bridge) to look up originals when retrieval
258
+ /// tool calls fire.
259
+ pub fn ccr_store(&self) -> Option<&Arc<dyn CcrStore>> {
260
+ self.ccr_store.as_ref()
261
+ }
262
+
263
+ fn planner(&self) -> SmartCrusherPlanner<'_> {
264
+ SmartCrusherPlanner::new(
265
+ &self.config,
266
+ &self.anchor_selector,
267
+ &*self.scorer,
268
+ &self.analyzer,
269
+ &self.constraints,
270
+ )
271
+ }
272
+
273
+ /// Execute a `CompressionPlan` against `items`, returning the
274
+ /// kept-items list in original-array order. Mirrors Python's
275
+ /// `_execute_plan` (line 3617-3633).
276
+ ///
277
+ /// Schema-preserving by default: each kept item is cloned unchanged.
278
+ /// No summary objects, generated fields, or wrapper metadata.
279
+ ///
280
+ /// When `factor_out_constants` is enabled (default off), fields the
281
+ /// analyzer found constant across ALL items are stripped from each
282
+ /// kept object and emitted once in a leading
283
+ /// `{"_constant_fields": {...}}` sentinel — same output-shape
284
+ /// convention as the `_ccr_dropped` sentinel. Stripping is
285
+ /// defensive: a key is only removed from an item when its value
286
+ /// equals the recorded constant, so a drifted item keeps its own
287
+ /// value. The CCR store always holds the full unfactored original.
288
+ pub fn execute_plan(&self, plan: &CompressionPlan, items: &[Value]) -> Vec<Value> {
289
+ let mut indices = plan.keep_indices.clone();
290
+ indices.sort_unstable();
291
+ let mut kept: Vec<Value> = indices
292
+ .into_iter()
293
+ .filter(|&idx| idx < items.len())
294
+ .map(|idx| items[idx].clone())
295
+ .collect();
296
+
297
+ if self.config.factor_out_constants && !plan.constant_fields.is_empty() && kept.len() >= 2 {
298
+ let mut any_stripped = false;
299
+ for item in kept.iter_mut() {
300
+ if let Value::Object(map) = item {
301
+ for (key, constant) in &plan.constant_fields {
302
+ if map.get(key) == Some(constant) {
303
+ map.remove(key);
304
+ any_stripped = true;
305
+ }
306
+ }
307
+ }
308
+ }
309
+ if any_stripped {
310
+ let mut sentinel = serde_json::Map::new();
311
+ sentinel.insert(
312
+ "_constant_fields".to_string(),
313
+ Value::Object(plan.constant_fields.clone().into_iter().collect()),
314
+ );
315
+ kept.insert(0, Value::Object(sentinel));
316
+ }
317
+ }
318
+
319
+ kept
320
+ }
321
+
322
+ /// Top-level entry point. Mirrors Python `SmartCrusher.crush`
323
+ /// (line 1581-1603) — used by `ContentRouter` when routing JSON
324
+ /// arrays.
325
+ ///
326
+ /// Parses `content` as JSON, recursively processes it (compressing
327
+ /// arrays at every depth via the appropriate per-type crusher),
328
+ /// then re-serializes with Python-compatible formatting (`, ` and
329
+ /// `: ` separators, ASCII-escaped non-ASCII).
330
+ ///
331
+ /// Returns a `CrushResult` with:
332
+ /// - `compressed`: the re-serialized JSON.
333
+ /// - `original`: the input string (unmodified).
334
+ /// - `was_modified`: whether `compressed` differs from `content`'s
335
+ /// trimmed form.
336
+ /// - `strategy`: combined strategy info from all crushed arrays
337
+ /// (or `"passthrough"`).
338
+ pub fn crush(&self, content: &str, query: &str, bias: f64) -> CrushResult {
339
+ let start = std::time::Instant::now();
340
+ let (compressed, was_modified, info) = self.smart_crush_content(content, query, bias);
341
+ let strategy = if info.is_empty() {
342
+ "passthrough".to_string()
343
+ } else {
344
+ info
345
+ };
346
+
347
+ // Fire one event per top-level crush. Cheap when no observers
348
+ // are configured (`for o in &[]` is a single null-pointer
349
+ // check); cheap when only `TracingObserver` is configured if
350
+ // the subscriber filters `debug` out (the default in
351
+ // production). Custom observers — audit logs, Loop training
352
+ // stream, metrics — pay whatever they pay.
353
+ if !self.observers.is_empty() {
354
+ let event = CrushEvent {
355
+ strategy: strategy.clone(),
356
+ input_bytes: content.len(),
357
+ output_bytes: compressed.len(),
358
+ elapsed_ns: start.elapsed().as_nanos() as u64,
359
+ was_modified,
360
+ };
361
+ for observer in &self.observers {
362
+ observer.on_event(&event);
363
+ }
364
+ }
365
+
366
+ CrushResult {
367
+ compressed,
368
+ original: content.to_string(),
369
+ was_modified,
370
+ strategy,
371
+ }
372
+ }
373
+
374
+ /// `SmartCrusher._smart_crush_content` (Python line 2243-2301).
375
+ /// JSON-parse, recursively process, re-serialize. CCR marker
376
+ /// injection is stubbed (CCR is disabled in this stage).
377
+ ///
378
+ /// Returns `(crushed_content, was_modified, info)`.
379
+ pub fn smart_crush_content(
380
+ &self,
381
+ content: &str,
382
+ query_context: &str,
383
+ bias: f64,
384
+ ) -> (String, bool, String) {
385
+ // Parse — non-JSON content passes through unchanged.
386
+ let Ok(parsed) = serde_json::from_str::<Value>(content) else {
387
+ return (content.to_string(), false, String::new());
388
+ };
389
+
390
+ let (crushed, info) = self.process_value(&parsed, 0, query_context, bias);
391
+
392
+ // Re-serialize with Python `safe_json_dumps` formatting:
393
+ // compact `(",", ":")` separators + `ensure_ascii=False`,
394
+ // preserving object-key insertion order. Matches the Python
395
+ // SmartCrusher output bytes the proxy writes.
396
+ let result = crate::transforms::anchor_selector::python_safe_json_dumps(&crushed);
397
+ let was_modified = result != content.trim();
398
+ (result, was_modified, info)
399
+ }
400
+
401
+ /// Maximum recursion depth for nested JSON. Mirrors Python's
402
+ /// `_MAX_PROCESS_DEPTH = 50`. Beyond this, values are returned as-is.
403
+ const MAX_PROCESS_DEPTH: usize = 50;
404
+
405
+ /// Recursively process a value, crushing arrays where appropriate.
406
+ /// Mirrors Python `_process_value` (line 2307-2398).
407
+ ///
408
+ /// Returns `(processed_value, info_string)`. CCR markers are
409
+ /// stubbed (Python's tuple has a third element for them — Rust's
410
+ /// version omits since we never produce markers in this stage).
411
+ pub fn process_value(
412
+ &self,
413
+ value: &Value,
414
+ depth: usize,
415
+ query_context: &str,
416
+ bias: f64,
417
+ ) -> (Value, String) {
418
+ if depth >= Self::MAX_PROCESS_DEPTH {
419
+ return (value.clone(), String::new());
420
+ }
421
+
422
+ let mut info_parts: Vec<String> = Vec::new();
423
+
424
+ match value {
425
+ Value::Array(arr) => {
426
+ let n = arr.len();
427
+ if n >= self.config.min_items_to_analyze {
428
+ let arr_type = classify_array(arr);
429
+ match arr_type {
430
+ ArrayType::DictArray => {
431
+ let result = self.crush_array(arr, query_context, bias);
432
+ // Lossless path won → substitute the array
433
+ // with the compacted string in place. This
434
+ // makes the lossless win visible to the
435
+ // public `crush()` API: the output JSON
436
+ // has a string where the array used to be.
437
+ // The wrapping JSON structure is preserved.
438
+ if let Some(rendered) = result.compacted {
439
+ info_parts.push(format!(
440
+ "{}({}->len={})",
441
+ result.strategy_info,
442
+ n,
443
+ rendered.len()
444
+ ));
445
+ return (Value::String(rendered), info_parts.join(","));
446
+ }
447
+ info_parts.push(format!(
448
+ "{}({}->{})",
449
+ result.strategy_info,
450
+ n,
451
+ result.items.len()
452
+ ));
453
+ // Lossy path with rows dropped → append a
454
+ // CCR-Dropped sentinel object as the last
455
+ // element of the kept-items array. This is
456
+ // the **only** place the LLM sees the
457
+ // `<<ccr:HASH ...>>` pointer in the prompt.
458
+ // Without this, the store has the data but
459
+ // no model can ever ask for it.
460
+ //
461
+ // Sentinel shape: `{"_ccr_dropped":
462
+ // "<<ccr:HASH N_rows_offloaded>>"}` —
463
+ // preserves "array-of-objects" shape so
464
+ // downstream consumers iterating with
465
+ // `x.get(...)` keep working; the well-known
466
+ // `_ccr_dropped` key signals metadata
467
+ // unambiguously.
468
+ let mut items = result.items;
469
+ if !result.dropped_summary.is_empty() {
470
+ let mut sentinel = serde_json::Map::new();
471
+ sentinel.insert(
472
+ "_ccr_dropped".to_string(),
473
+ Value::String(result.dropped_summary),
474
+ );
475
+ items.push(Value::Object(sentinel));
476
+ }
477
+ return (Value::Array(items), info_parts.join(","));
478
+ }
479
+ ArrayType::StringArray => {
480
+ let strs: Vec<&str> = arr.iter().filter_map(|v| v.as_str()).collect();
481
+ let (crushed, strategy) = crush_string_array(&strs, &self.config, bias);
482
+ info_parts.push(format!("{}({}->{})", strategy, n, crushed.len()));
483
+ let crushed_values: Vec<Value> =
484
+ crushed.into_iter().map(Value::String).collect();
485
+ return (Value::Array(crushed_values), info_parts.join(","));
486
+ }
487
+ ArrayType::NumberArray => {
488
+ let (crushed, strategy) = crush_number_array(arr, &self.config, bias);
489
+ info_parts.push(format!("{}({}->{})", strategy, n, crushed.len()));
490
+ return (Value::Array(crushed), info_parts.join(","));
491
+ }
492
+ ArrayType::MixedArray => {
493
+ let (crushed, strategy) =
494
+ self.crush_mixed_array(arr, query_context, bias);
495
+ info_parts.push(format!("{}({}->{})", strategy, n, crushed.len()));
496
+ return (Value::Array(crushed), info_parts.join(","));
497
+ }
498
+ // NestedArray, BoolArray, Empty → fall through
499
+ // to recursive descent.
500
+ _ => {}
501
+ }
502
+ }
503
+
504
+ // Below threshold or not crushable → recurse into items.
505
+ let mut processed: Vec<Value> = Vec::with_capacity(n);
506
+ for item in arr {
507
+ let (p_item, p_info) = self.process_value(item, depth + 1, query_context, bias);
508
+ processed.push(p_item);
509
+ if !p_info.is_empty() {
510
+ info_parts.push(p_info);
511
+ }
512
+ }
513
+ (Value::Array(processed), info_parts.join(","))
514
+ }
515
+ Value::Object(map) => {
516
+ // First pass: recurse into values to compress nested arrays.
517
+ let mut processed = serde_json::Map::new();
518
+ for (k, v) in map {
519
+ let (p_val, p_info) = self.process_value(v, depth + 1, query_context, bias);
520
+ processed.insert(k.clone(), p_val);
521
+ if !p_info.is_empty() {
522
+ info_parts.push(p_info);
523
+ }
524
+ }
525
+
526
+ // Second pass: if the object itself has many keys,
527
+ // compress at the key level.
528
+ if processed.len() >= self.config.min_items_to_analyze {
529
+ let (crushed_dict, strategy) = crush_object(&processed, &self.config, bias);
530
+ if strategy != "object:passthrough" {
531
+ info_parts.push(strategy);
532
+ return (Value::Object(crushed_dict), info_parts.join(","));
533
+ }
534
+ }
535
+
536
+ (Value::Object(processed), info_parts.join(","))
537
+ }
538
+ // Strings: walker-equivalent handling. Delegates to
539
+ // `process_string` which parses stringified-JSON containers
540
+ // (recursing through `process_value`) and CCR-substitutes
541
+ // opaque blobs (with store-write so retrieval works).
542
+ Value::String(s) => self.process_string(s, depth, query_context, bias),
543
+ // Other scalars — passthrough.
544
+ _ => (value.clone(), String::new()),
545
+ }
546
+ }
547
+
548
+ /// Walker-equivalent string handling. Mirrors `walker::walk_string`
549
+ /// in `compaction/walker.rs` but lives on `SmartCrusher` so the
550
+ /// public `crush()` path picks it up.
551
+ ///
552
+ /// Two cases:
553
+ /// 1. **Stringified-JSON.** Strings that parse to a JSON object or
554
+ /// array → recurse via `process_value`, then re-emit the result
555
+ /// as a compact JSON string. The wrapping string is preserved
556
+ /// (so the parent JSON shape stays a string-typed field), but
557
+ /// its contents are processed end-to-end.
558
+ /// 2. **Opaque blobs.** Strings classified as
559
+ /// [`CellClass::Opaque`] (long base64 / HTML / long-text) →
560
+ /// substitute with a `<<ccr:HASH,KIND,SIZE>>` marker. Same
561
+ /// format as `compaction::walker::format_ccr_marker` so
562
+ /// downstream consumers can pattern-match markers regardless
563
+ /// of which path emitted them.
564
+ fn process_string(
565
+ &self,
566
+ s: &str,
567
+ depth: usize,
568
+ query_context: &str,
569
+ bias: f64,
570
+ ) -> (Value, String) {
571
+ // 1. Stringified-JSON: parse, recurse, re-render.
572
+ if let Some(parsed) = try_parse_json_container(s) {
573
+ let (processed, sub_info) = self.process_value(&parsed, depth + 1, query_context, bias);
574
+ // If recursion produced something different, re-emit.
575
+ // Special case: if the recursion returned a `Value::String`
576
+ // (lossless compaction substituted the array with a
577
+ // rendered CSV+schema string), use that string directly.
578
+ // Re-encoding it as JSON would produce a quoted string
579
+ // literal — double-encoded — which is not what callers
580
+ // expect in the wrapping field.
581
+ if processed != parsed {
582
+ let rendered = match &processed {
583
+ Value::String(rendered_str) => rendered_str.clone(),
584
+ _ => serde_json::to_string(&processed).unwrap_or_else(|_| s.to_string()),
585
+ };
586
+ let info = if sub_info.is_empty() {
587
+ "string_json".to_string()
588
+ } else {
589
+ format!("string_json[{sub_info}]")
590
+ };
591
+ return (Value::String(rendered), info);
592
+ }
593
+ }
594
+
595
+ // 2. Opaque blob: substitute with CCR marker AND stash the
596
+ // original in the store (PR8) so retrieval works. Hash + format
597
+ // identical to walker.rs via the shared helper — zero drift.
598
+ let cfg = ClassifyConfig::default();
599
+ if let CellClass::Opaque(kind) = classify_cell(&Value::String(s.to_string()), &cfg) {
600
+ let marker = emit_opaque_ccr_marker(s, &kind, self.ccr_store.as_ref());
601
+ let kind_label = opaque_kind_label(&kind);
602
+ return (Value::String(marker), format!("string_ccr:{kind_label}"));
603
+ }
604
+
605
+ // 3. Plain string — passthrough.
606
+ (Value::String(s.to_string()), String::new())
607
+ }
608
+
609
+ /// Compress an array of dict items.
610
+ ///
611
+ /// Direct port of `_crush_array` (Python line 2400-2687) with the
612
+ /// optional subsystems (TOIN / CCR / feedback / telemetry) wired
613
+ /// in their disabled-by-default behavior. See module-level docs
614
+ /// for the rationale.
615
+ ///
616
+ /// # Pipeline
617
+ ///
618
+ /// 1. Compute `item_strings` once (used as input to adaptive
619
+ /// sizing and downstream relevance scoring).
620
+ /// 2. `compute_optimal_k` → `adaptive_k`.
621
+ /// 3. If `n <= adaptive_k`, return passthrough.
622
+ /// 4. `analyzer.analyze_array(items)` → `analysis`.
623
+ /// 5. If `analysis.recommended_strategy == Skip`, return passthrough
624
+ /// with a `skip:<reason>` strategy string.
625
+ /// 6. `planner.create_plan(analysis, items, query_context, ...)`.
626
+ /// 7. `execute_plan(plan, items)` → result.
627
+ /// 8. Strategy info = `analysis.recommended_strategy.as_str()`.
628
+ pub fn crush_array(&self, items: &[Value], query_context: &str, bias: f64) -> CrushArrayResult {
629
+ let item_strings: Vec<String> = items
630
+ .iter()
631
+ .map(|i| serde_json::to_string(i).unwrap_or_default())
632
+ .collect();
633
+ let item_str_refs: Vec<&str> = item_strings.iter().map(|s| s.as_str()).collect();
634
+
635
+ let max_k = if self.config.max_items_after_crush > 0 {
636
+ Some(self.config.max_items_after_crush)
637
+ } else {
638
+ None
639
+ };
640
+ let adaptive_k = compute_optimal_k(&item_str_refs, bias, 3, max_k);
641
+
642
+ // Tier-1 boundary: array already small enough — passthrough,
643
+ // nothing to compact, nothing to drop.
644
+ if items.len() <= adaptive_k {
645
+ return CrushArrayResult {
646
+ items: items.to_vec(),
647
+ strategy_info: "none:adaptive_at_limit".to_string(),
648
+ ccr_hash: None,
649
+ dropped_summary: String::new(),
650
+ compacted: None,
651
+ compaction_kind: None,
652
+ };
653
+ }
654
+
655
+ // ── Lossless-first attempt ──
656
+ //
657
+ // Run the compaction stage if present, then check the savings
658
+ // ratio against `config.lossless_min_savings_ratio`. If the
659
+ // lossless rendering shrinks the input by at least that much,
660
+ // ship it — nothing dropped, no CCR retrieval needed.
661
+ // Otherwise fall through to the lossy path.
662
+ if let Some(stage) = &self.compaction {
663
+ let (c, rendered) = stage.run(items);
664
+ if c.was_compacted() {
665
+ let input_bytes = estimate_array_bytes(&item_strings);
666
+ let savings_ratio = if input_bytes > 0 {
667
+ 1.0 - (rendered.len() as f64 / input_bytes as f64)
668
+ } else {
669
+ 0.0
670
+ };
671
+ if savings_ratio >= self.config.lossless_min_savings_ratio {
672
+ let kind = compaction_kind_str(&c);
673
+ return CrushArrayResult {
674
+ items: items.to_vec(), // nothing dropped
675
+ strategy_info: format!("lossless:{kind}"),
676
+ ccr_hash: None,
677
+ dropped_summary: String::new(),
678
+ compacted: Some(rendered),
679
+ compaction_kind: Some(kind),
680
+ };
681
+ }
682
+ }
683
+ }
684
+
685
+ // ── Lossy path: compress inline + cache full original via CCR ──
686
+ //
687
+ // The runtime caller (PyO3 bridge / proxy server) is expected
688
+ // to stash the full input keyed by `ccr_hash` so a retrieval
689
+ // tool can serve dropped rows back to the LLM on demand.
690
+ // **No data is lost** — "lossy" here means "compressed view
691
+ // inline; full payload retrievable via CCR cache."
692
+
693
+ let effective_max_items = adaptive_k;
694
+ let analysis = self.analyzer.analyze_array(items);
695
+
696
+ // Crushability gate: not safe to crush → passthrough, no CCR.
697
+ if analysis.recommended_strategy == CompressionStrategy::Skip {
698
+ let reason = match &analysis.crushability {
699
+ Some(c) => format!("skip:{}", c.reason),
700
+ None => String::new(),
701
+ };
702
+ return CrushArrayResult {
703
+ items: items.to_vec(),
704
+ strategy_info: reason,
705
+ ccr_hash: None,
706
+ dropped_summary: String::new(),
707
+ compacted: None,
708
+ compaction_kind: None,
709
+ };
710
+ }
711
+
712
+ let plan = self.planner().create_plan(
713
+ &analysis,
714
+ items,
715
+ query_context,
716
+ None, // preserve_fields (TOIN — stubbed)
717
+ Some(effective_max_items),
718
+ Some(&item_strings),
719
+ );
720
+ let result = self.execute_plan(&plan, items);
721
+
722
+ // Emit CCR-Dropped marker iff rows were actually dropped AND
723
+ // the marker gate is on. **The marker is the cornerstone of
724
+ // CCR's no-data-loss guarantee:** we hash the full original,
725
+ // stash it in the configured store, and emit a marker pointing
726
+ // at that hash. The runtime later serves the original back via
727
+ // retrieval tool calls.
728
+ //
729
+ // When `enable_ccr_marker` is false (Python shim's path for
730
+ // `ccr_config.enabled=False` or `inject_retrieval_marker=False`)
731
+ // we keep the row drops (compression is still requested) but
732
+ // skip the marker text and the store write — there's no point
733
+ // storing a payload that nothing in the prompt can reference.
734
+ let dropped_count = items.len().saturating_sub(result.len());
735
+ let (ccr_hash, dropped_summary) = if dropped_count > 0 && self.config.enable_ccr_marker {
736
+ // Serialize the original array exactly ONCE. The hash is
737
+ // taken over those bytes, and (if a store is configured) the
738
+ // same bytes get stored — eliminating a redundant tree clone
739
+ // (`items.to_vec()`) and a redundant `serde_json::to_string`
740
+ // pass that the previous version did per dropped array.
741
+ let canonical = canonical_array_json(items);
742
+ let h = hash_canonical(&canonical);
743
+ let marker = format!("<<ccr:{h} {dropped_count}_rows_offloaded>>");
744
+ if let Some(store) = &self.ccr_store {
745
+ store.put(&h, &canonical);
746
+ }
747
+ (Some(h), marker)
748
+ } else {
749
+ (None, String::new())
750
+ };
751
+
752
+ CrushArrayResult {
753
+ items: result,
754
+ strategy_info: analysis.recommended_strategy.as_str().to_string(),
755
+ ccr_hash,
756
+ dropped_summary,
757
+ compacted: None,
758
+ compaction_kind: None,
759
+ }
760
+ }
761
+
762
+ /// Compress a mixed-type array by grouping items by type and
763
+ /// compressing each group with the appropriate handler.
764
+ ///
765
+ /// Direct port of `_crush_mixed_array` (Python line 2914-3013).
766
+ ///
767
+ /// Strategy:
768
+ /// 1. Group by type (dict / str / number / list / null / bool / other).
769
+ /// 2. For groups with >= `min_items_to_analyze` items: apply the
770
+ /// type-specific compressor.
771
+ /// 3. For small groups: keep all items.
772
+ /// 4. Reassemble in original order.
773
+ ///
774
+ /// Returns `(crushed_items, strategy_string)`.
775
+ pub fn crush_mixed_array(
776
+ &self,
777
+ items: &[Value],
778
+ query_context: &str,
779
+ bias: f64,
780
+ ) -> (Vec<Value>, String) {
781
+ let n = items.len();
782
+ if n <= 8 {
783
+ return (items.to_vec(), "mixed:passthrough".to_string());
784
+ }
785
+
786
+ // Group by type, tracking original indices.
787
+ let mut groups: GroupBuckets = GroupBuckets::default();
788
+ for (i, item) in items.iter().enumerate() {
789
+ groups.push(group_key(item), i, item.clone());
790
+ }
791
+
792
+ let mut keep_indices: std::collections::BTreeSet<usize> = std::collections::BTreeSet::new();
793
+ let mut strategy_parts: Vec<String> = Vec::new();
794
+
795
+ for (type_key, indices, values) in groups.into_iter() {
796
+ // Small groups: keep all items.
797
+ if values.len() < self.config.min_items_to_analyze {
798
+ keep_indices.extend(&indices);
799
+ continue;
800
+ }
801
+
802
+ match type_key {
803
+ "dict" => {
804
+ let CrushArrayResult { items: crushed, .. } =
805
+ self.crush_array(&values, query_context, bias);
806
+ // Find which original indices survived by matching
807
+ // canonical-JSON serialization. Mirrors Python's
808
+ // `json.dumps(c, sort_keys=True, default=str)`-keyed
809
+ // set match.
810
+ let crushed_keys: std::collections::HashSet<String> =
811
+ crushed.iter().map(canonical_json_for_match).collect();
812
+ for (i, idx) in indices.iter().enumerate() {
813
+ if crushed_keys.contains(&canonical_json_for_match(&values[i])) {
814
+ keep_indices.insert(*idx);
815
+ }
816
+ }
817
+ strategy_parts.push(format!("dict:{}->{}", values.len(), crushed.len()));
818
+ }
819
+ "str" => {
820
+ let strs: Vec<&str> = values.iter().filter_map(|v| v.as_str()).collect();
821
+ let (crushed, _) = crush_string_array(&strs, &self.config, bias);
822
+ let crushed_set: std::collections::HashSet<&str> =
823
+ crushed.iter().map(|s| s.as_str()).collect();
824
+ for (i, idx) in indices.iter().enumerate() {
825
+ if let Some(s) = values[i].as_str() {
826
+ if crushed_set.contains(s) {
827
+ keep_indices.insert(*idx);
828
+ }
829
+ }
830
+ }
831
+ strategy_parts.push(format!("str:{}->{}", values.len(), crushed.len()));
832
+ }
833
+ "number" => {
834
+ // Python: just adaptive sampling + outlier detection
835
+ // (no summary prefix). Keeps first/last by index
836
+ // and items >variance_threshold σ from mean.
837
+ let item_strings: Vec<String> = values.iter().map(|v| v.to_string()).collect();
838
+ let item_refs: Vec<&str> = item_strings.iter().map(|s| s.as_str()).collect();
839
+ let (_kt, kf, kl, _) = compute_k_split(&item_refs, &self.config, bias);
840
+
841
+ let kf = kf.min(values.len());
842
+ let kl = kl.min(values.len().saturating_sub(kf));
843
+ let first_idx: Vec<usize> = indices.iter().take(kf).copied().collect();
844
+ let last_idx: Vec<usize> =
845
+ indices.iter().rev().take(kl).copied().collect::<Vec<_>>();
846
+ keep_indices.extend(&first_idx);
847
+ keep_indices.extend(&last_idx);
848
+
849
+ // Outliers via finite-only stats.
850
+ let finite: Vec<f64> = values
851
+ .iter()
852
+ .filter_map(|v| v.as_f64().filter(|f| f.is_finite()))
853
+ .collect();
854
+ if finite.len() > 1 {
855
+ if let Some(mean_v) = super::stats_math::mean(&finite) {
856
+ if let Some(std_v) = super::stats_math::sample_stdev(&finite) {
857
+ if std_v > 0.0 {
858
+ let threshold = self.config.variance_threshold * std_v;
859
+ for (i, val) in values.iter().enumerate() {
860
+ if let Some(num) = val.as_f64().filter(|f| f.is_finite()) {
861
+ if (num - mean_v).abs() > threshold {
862
+ keep_indices.insert(indices[i]);
863
+ }
864
+ }
865
+ }
866
+ }
867
+ }
868
+ }
869
+ }
870
+ strategy_parts.push(format!("num:{}", values.len()));
871
+ }
872
+ _ => {
873
+ // list / bool / none / other → keep all items.
874
+ keep_indices.extend(&indices);
875
+ }
876
+ }
877
+ }
878
+
879
+ // Reassemble in original order.
880
+ let result: Vec<Value> = keep_indices.iter().map(|&i| items[i].clone()).collect();
881
+ let strategy = format!(
882
+ "mixed:adaptive({}->{},{})",
883
+ n,
884
+ result.len(),
885
+ strategy_parts.join(",")
886
+ );
887
+ (result, strategy)
888
+ }
889
+ }
890
+
891
+ // ---------- helpers ----------
892
+
893
+ /// Group key that mirrors Python's `_crush_mixed_array` switch on
894
+ /// `isinstance`. Note the bool-before-number ordering: in Python, bool
895
+ /// is a subclass of int, but JSON treats them as distinct types, so we
896
+ /// don't have the Python ordering hazard.
897
+ fn group_key(item: &Value) -> &'static str {
898
+ match item {
899
+ Value::Object(_) => "dict",
900
+ Value::String(_) => "str",
901
+ Value::Bool(_) => "bool",
902
+ Value::Number(_) => "number",
903
+ Value::Array(_) => "list",
904
+ Value::Null => "none",
905
+ }
906
+ }
907
+
908
+ /// Group buckets keyed by the type-string. Preserves first-occurrence
909
+ /// order across keys so dict/str/number/list/none/bool always come out
910
+ /// in the same order — matters because `keep_indices` is built
911
+ /// incrementally and Python iterates `groups.items()` (insertion order
912
+ /// in 3.7+).
913
+ #[derive(Default)]
914
+ struct GroupBuckets {
915
+ entries: Vec<(&'static str, Vec<usize>, Vec<Value>)>,
916
+ index_of: std::collections::HashMap<&'static str, usize>,
917
+ }
918
+
919
+ impl GroupBuckets {
920
+ fn push(&mut self, key: &'static str, idx: usize, value: Value) {
921
+ match self.index_of.get(key).copied() {
922
+ Some(i) => {
923
+ self.entries[i].1.push(idx);
924
+ self.entries[i].2.push(value);
925
+ }
926
+ None => {
927
+ self.index_of.insert(key, self.entries.len());
928
+ self.entries.push((key, vec![idx], vec![value]));
929
+ }
930
+ }
931
+ }
932
+ }
933
+
934
+ impl IntoIterator for GroupBuckets {
935
+ type Item = (&'static str, Vec<usize>, Vec<Value>);
936
+ type IntoIter = std::vec::IntoIter<Self::Item>;
937
+ fn into_iter(self) -> Self::IntoIter {
938
+ self.entries.into_iter()
939
+ }
940
+ }
941
+
942
+ /// Serialize a `Value` for membership comparison. Mirrors Python's
943
+ /// `json.dumps(c, sort_keys=True, default=str)` used by
944
+ /// `_crush_mixed_array` to match crushed dict items back to their
945
+ /// original indices. The `default=str` fallback only matters for
946
+ /// non-JSON-serializable Python values; in serde_json land everything
947
+ /// is already JSON-native, so plain canonical JSON suffices.
948
+ fn canonical_json_for_match(value: &Value) -> String {
949
+ crate::transforms::anchor_selector::python_json_dumps_sort_keys(value)
950
+ }
951
+
952
+ /// Maps a `Compaction` to a stable kind tag exposed via `CrushArrayResult`.
953
+ fn compaction_kind_str(c: &Compaction) -> &'static str {
954
+ match c {
955
+ Compaction::Table { .. } => "table",
956
+ Compaction::Buckets { .. } => "buckets",
957
+ Compaction::OpaqueRef { .. } => "ccr",
958
+ Compaction::Untouched(_) => "untouched",
959
+ }
960
+ }
961
+
962
+ /// Approximate byte size of `[v0, v1, ...]` JSON serialization, given
963
+ /// each item's already-serialized form. Adds 2 for outer brackets and
964
+ /// 1 per inter-item comma. Used by the lossless savings-ratio check.
965
+ fn estimate_array_bytes(item_strings: &[String]) -> usize {
966
+ let payload: usize = item_strings.iter().map(|s| s.len()).sum();
967
+ let separators = item_strings.len().saturating_sub(1);
968
+ payload + separators + 2
969
+ }
970
+
971
+ /// Serialize `[v0, v1, ...]` once into the canonical JSON form used by
972
+ /// the CCR retrieval contract. `serde_json` writes a slice of `Value` as
973
+ /// the same bytes it would write for `Value::Array(items.to_vec())`, so
974
+ /// we skip the array-wrapper allocation and the deep tree clone it
975
+ /// requires. Used by both the hash (input) and the store payload (write).
976
+ fn canonical_array_json(items: &[Value]) -> String {
977
+ serde_json::to_string(items).unwrap_or_default()
978
+ }
979
+
980
+ /// 12-char SHA-256 hex prefix of an already-serialized canonical JSON
981
+ /// string. Caller is responsible for producing the canonical form via
982
+ /// [`canonical_array_json`] (or another byte-equal serializer) — the
983
+ /// hash is over the bytes, so a stable serializer is the contract.
984
+ fn hash_canonical(canonical: &str) -> String {
985
+ use sha2::{Digest, Sha256};
986
+ let mut h = Sha256::new();
987
+ h.update(canonical.as_bytes());
988
+ h.finalize()
989
+ .iter()
990
+ .take(6)
991
+ .map(|b| format!("{b:02x}"))
992
+ .collect()
993
+ }
994
+
995
+ // `hash_array_for_ccr` (a test-only `canonical_array_json + hash_canonical`
996
+ // convenience) lived here previously but had no callers — clippy flagged
997
+ // it as dead code. Reintroduce as a test fixture if a future test wants
998
+ // the one-liner; production callsites inline both steps so the canonical
999
+ // bytes can be reused for the store payload.
1000
+
1001
+ // ─── PR5 walker-integration helpers (string handling) ──────────────────────
1002
+ //
1003
+ // Parse-as-JSON-container, marker formatting, and humanize-bytes used to
1004
+ // live here as locals. PR8 extracted them into `compaction::walker` so
1005
+ // `walker.rs` and `process_value` share one canonical implementation —
1006
+ // killing the drift risk where the two paths could format markers
1007
+ // differently. `process_string` now calls `try_parse_json_container` and
1008
+ // `emit_opaque_ccr_marker` directly. Only `opaque_kind_label` survives
1009
+ // here because `process_string`'s `string_ccr:<kind>` strategy-info
1010
+ // label is local to this module's debug-string convention.
1011
+
1012
+ fn opaque_kind_label(kind: &super::compaction::OpaqueKind) -> &str {
1013
+ use super::compaction::OpaqueKind;
1014
+ match kind {
1015
+ OpaqueKind::Base64Blob => "base64",
1016
+ OpaqueKind::LongString => "string",
1017
+ OpaqueKind::HtmlChunk => "html",
1018
+ OpaqueKind::Other(s) => s.as_str(),
1019
+ }
1020
+ }
1021
+
1022
+ #[cfg(test)]
1023
+ mod tests {
1024
+ use super::*;
1025
+ use serde_json::json;
1026
+
1027
+ fn crusher() -> SmartCrusher {
1028
+ SmartCrusher::new(SmartCrusherConfig::default())
1029
+ }
1030
+
1031
+ // ---------- execute_plan ----------
1032
+
1033
+ #[test]
1034
+ fn execute_plan_empty_indices_returns_empty() {
1035
+ let c = crusher();
1036
+ let plan = CompressionPlan::default();
1037
+ let items: Vec<Value> = (0..5).map(|i| json!({"id": i})).collect();
1038
+ let result = c.execute_plan(&plan, &items);
1039
+ assert!(result.is_empty());
1040
+ }
1041
+
1042
+ #[test]
1043
+ fn execute_plan_returns_items_in_sorted_index_order() {
1044
+ let c = crusher();
1045
+ let items: Vec<Value> = (0..10).map(|i| json!({"id": i})).collect();
1046
+ let plan = CompressionPlan {
1047
+ keep_indices: vec![5, 2, 8, 0],
1048
+ ..CompressionPlan::default()
1049
+ };
1050
+ let result = c.execute_plan(&plan, &items);
1051
+ assert_eq!(result.len(), 4);
1052
+ assert_eq!(result[0]["id"], 0);
1053
+ assert_eq!(result[1]["id"], 2);
1054
+ assert_eq!(result[2]["id"], 5);
1055
+ assert_eq!(result[3]["id"], 8);
1056
+ }
1057
+
1058
+ #[test]
1059
+ fn execute_plan_skips_out_of_bounds() {
1060
+ let c = crusher();
1061
+ let items: Vec<Value> = (0..3).map(|i| json!({"id": i})).collect();
1062
+ let plan = CompressionPlan {
1063
+ keep_indices: vec![0, 5, 2],
1064
+ ..CompressionPlan::default()
1065
+ };
1066
+ let result = c.execute_plan(&plan, &items);
1067
+ assert_eq!(result.len(), 2);
1068
+ }
1069
+
1070
+ #[test]
1071
+ fn execute_plan_factors_constants_when_enabled() {
1072
+ let cfg = SmartCrusherConfig {
1073
+ factor_out_constants: true,
1074
+ ..Default::default()
1075
+ };
1076
+ let c = SmartCrusher::new(cfg);
1077
+ let items: Vec<Value> = (0..4)
1078
+ .map(|i| json!({"id": i, "region": "us-west-2", "status": "ok"}))
1079
+ .collect();
1080
+ let mut constant_fields = std::collections::BTreeMap::new();
1081
+ constant_fields.insert("region".to_string(), json!("us-west-2"));
1082
+ constant_fields.insert("status".to_string(), json!("ok"));
1083
+ let plan = CompressionPlan {
1084
+ keep_indices: vec![0, 1, 2],
1085
+ constant_fields,
1086
+ ..CompressionPlan::default()
1087
+ };
1088
+ let result = c.execute_plan(&plan, &items);
1089
+ // Sentinel first, then 3 slim items.
1090
+ assert_eq!(result.len(), 4);
1091
+ assert_eq!(result[0]["_constant_fields"]["region"], "us-west-2");
1092
+ assert_eq!(result[0]["_constant_fields"]["status"], "ok");
1093
+ for item in &result[1..] {
1094
+ assert!(item.get("region").is_none());
1095
+ assert!(item.get("status").is_none());
1096
+ assert!(item.get("id").is_some());
1097
+ }
1098
+ }
1099
+
1100
+ #[test]
1101
+ fn execute_plan_keeps_drifted_values_when_factoring() {
1102
+ // Defensive strip: an item whose value differs from the recorded
1103
+ // constant keeps its own value.
1104
+ let cfg = SmartCrusherConfig {
1105
+ factor_out_constants: true,
1106
+ ..Default::default()
1107
+ };
1108
+ let c = SmartCrusher::new(cfg);
1109
+ let items = vec![
1110
+ json!({"id": 0, "status": "ok"}),
1111
+ json!({"id": 1, "status": "FAILED"}),
1112
+ ];
1113
+ let mut constant_fields = std::collections::BTreeMap::new();
1114
+ constant_fields.insert("status".to_string(), json!("ok"));
1115
+ let plan = CompressionPlan {
1116
+ keep_indices: vec![0, 1],
1117
+ constant_fields,
1118
+ ..CompressionPlan::default()
1119
+ };
1120
+ let result = c.execute_plan(&plan, &items);
1121
+ assert_eq!(result.len(), 3);
1122
+ assert!(result[1].get("status").is_none()); // matched → stripped
1123
+ assert_eq!(result[2]["status"], "FAILED"); // drifted → kept
1124
+ }
1125
+
1126
+ #[test]
1127
+ fn execute_plan_default_off_leaves_items_unchanged() {
1128
+ // factor_out_constants defaults to false: schema preserved even
1129
+ // when the plan carries constant_fields.
1130
+ let c = crusher();
1131
+ let items: Vec<Value> = (0..3).map(|i| json!({"id": i, "k": "v"})).collect();
1132
+ let mut constant_fields = std::collections::BTreeMap::new();
1133
+ constant_fields.insert("k".to_string(), json!("v"));
1134
+ let plan = CompressionPlan {
1135
+ keep_indices: vec![0, 1, 2],
1136
+ constant_fields,
1137
+ ..CompressionPlan::default()
1138
+ };
1139
+ let result = c.execute_plan(&plan, &items);
1140
+ assert_eq!(result.len(), 3);
1141
+ assert_eq!(result[0]["k"], "v");
1142
+ }
1143
+
1144
+ // ---------- crush_array ----------
1145
+
1146
+ #[test]
1147
+ fn crush_array_passthrough_when_below_adaptive_k() {
1148
+ let c = crusher();
1149
+ let items: Vec<Value> = (0..3).map(|i| json!({"id": i})).collect();
1150
+ let result = c.crush_array(&items, "", 1.0);
1151
+ assert_eq!(result.items.len(), 3);
1152
+ assert_eq!(result.strategy_info, "none:adaptive_at_limit");
1153
+ assert!(result.ccr_hash.is_none());
1154
+ }
1155
+
1156
+ #[test]
1157
+ fn crush_array_skip_path_returns_original_items() {
1158
+ // 30 unique dict items with ID-like fields → analyzer should
1159
+ // detect "unique_entities_no_signal" and SKIP. Use the
1160
+ // no-compaction constructor so we exercise the lossy/skip
1161
+ // gate; the lossless path would otherwise short-circuit
1162
+ // because uniform-tabular input is the lossless sweet spot.
1163
+ let c = SmartCrusher::without_compaction(SmartCrusherConfig::default());
1164
+ let items: Vec<Value> = (0..30)
1165
+ .map(|i| json!({"id": i, "name": format!("user_{}", i)}))
1166
+ .collect();
1167
+ let result = c.crush_array(&items, "", 1.0);
1168
+ // skip path returns the original items unchanged.
1169
+ assert_eq!(result.items.len(), 30);
1170
+ assert!(
1171
+ result.strategy_info.starts_with("skip:"),
1172
+ "expected skip:..., got {}",
1173
+ result.strategy_info
1174
+ );
1175
+ }
1176
+
1177
+ #[test]
1178
+ fn crush_array_low_uniqueness_compresses() {
1179
+ // 30 items with status=ok across all → low_uniqueness path
1180
+ // (crushable, smart_sample strategy).
1181
+ let c = crusher();
1182
+ let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();
1183
+ let result = c.crush_array(&items, "", 1.0);
1184
+ assert!(result.items.len() <= 30, "should not exceed original count");
1185
+ }
1186
+
1187
+ #[test]
1188
+ fn crush_array_keeps_error_items() {
1189
+ let c = crusher();
1190
+ let mut items: Vec<Value> = (0..30).map(|i| json!({"id": i, "status": "ok"})).collect();
1191
+ items.push(json!({"id": 30, "status": "error", "msg": "FATAL"}));
1192
+ let result = c.crush_array(&items, "", 1.0);
1193
+ // Whatever path is taken, the error item should survive.
1194
+ assert!(
1195
+ result
1196
+ .items
1197
+ .iter()
1198
+ .any(|item| { item.get("status").and_then(|v| v.as_str()) == Some("error") }),
1199
+ "error item must survive crush_array"
1200
+ );
1201
+ }
1202
+
1203
+ // ---------- crush_mixed_array ----------
1204
+
1205
+ #[test]
1206
+ fn crush_mixed_passthrough_at_threshold() {
1207
+ let c = crusher();
1208
+ let items: Vec<Value> = vec![
1209
+ json!(1),
1210
+ json!("two"),
1211
+ json!({"k": "v"}),
1212
+ json!([1, 2]),
1213
+ json!(null),
1214
+ json!(true),
1215
+ json!(3),
1216
+ json!("four"),
1217
+ ];
1218
+ let (result, strat) = c.crush_mixed_array(&items, "", 1.0);
1219
+ assert_eq!(result.len(), 8);
1220
+ assert_eq!(strat, "mixed:passthrough");
1221
+ }
1222
+
1223
+ #[test]
1224
+ fn crush_mixed_groups_and_compresses_dicts() {
1225
+ let c = crusher();
1226
+ // 25 dicts (large group → gets crushed) + 5 strings (small group → all kept).
1227
+ let mut items: Vec<Value> = (0..25).map(|i| json!({"id": i, "status": "ok"})).collect();
1228
+ for i in 0..5 {
1229
+ items.push(json!(format!("string_{}", i)));
1230
+ }
1231
+ let (result, strat) = c.crush_mixed_array(&items, "", 1.0);
1232
+ assert!(strat.starts_with("mixed:adaptive("));
1233
+ // The 5 strings (small group) all survive.
1234
+ let str_count = result.iter().filter(|v| v.is_string()).count();
1235
+ assert_eq!(str_count, 5);
1236
+ }
1237
+
1238
+ #[test]
1239
+ fn crush_mixed_keeps_lists_and_nulls_unchanged() {
1240
+ let c = crusher();
1241
+ let mut items: Vec<Value> = vec![json!([1, 2]); 6];
1242
+ items.extend(vec![json!(null); 6]);
1243
+ items.extend(vec![json!({"k": 1}); 10]);
1244
+ let (result, _strat) = c.crush_mixed_array(&items, "", 1.0);
1245
+ // Lists and nulls (not dict/str/number) → fall through to "keep all".
1246
+ let list_count = result.iter().filter(|v| v.is_array()).count();
1247
+ let null_count = result.iter().filter(|v| v.is_null()).count();
1248
+ assert_eq!(list_count, 6);
1249
+ assert_eq!(null_count, 6);
1250
+ }
1251
+
1252
+ #[test]
1253
+ fn crusher_construction_default() {
1254
+ let c = SmartCrusher::new(SmartCrusherConfig::default());
1255
+ assert_eq!(c.config.max_items_after_crush, 15);
1256
+ }
1257
+
1258
+ // ---------- top-level crush ----------
1259
+
1260
+ #[test]
1261
+ fn crush_non_json_passes_through_unchanged() {
1262
+ let c = crusher();
1263
+ let result = c.crush("not json at all", "", 1.0);
1264
+ assert!(!result.was_modified);
1265
+ assert_eq!(result.compressed, "not json at all");
1266
+ assert_eq!(result.strategy, "passthrough");
1267
+ }
1268
+
1269
+ #[test]
1270
+ fn crush_scalar_json_passes_through() {
1271
+ let c = crusher();
1272
+ let result = c.crush("42", "", 1.0);
1273
+ // A scalar is not crushable; should round-trip unchanged.
1274
+ assert_eq!(result.compressed, "42");
1275
+ assert!(!result.was_modified);
1276
+ }
1277
+
1278
+ #[test]
1279
+ fn crush_small_array_passes_through() {
1280
+ let c = crusher();
1281
+ // Compact-form input matches the compact serializer output, so
1282
+ // the array is not "modified" even though it round-trips
1283
+ // through parse → serialize. (The spaced form `[1, 2, 3]`
1284
+ // would mark `was_modified=true` because the compact
1285
+ // serializer rewrites it to `[1,2,3]`.)
1286
+ let result = c.crush(r#"[1,2,3]"#, "", 1.0);
1287
+ // Below min_items_to_analyze=5 → no crushing of the structure.
1288
+ assert!(!result.was_modified);
1289
+ assert_eq!(result.compressed, "[1,2,3]");
1290
+ }
1291
+
1292
+ #[test]
1293
+ fn crush_dict_array_crushes_when_low_uniqueness() {
1294
+ // The public `crush()` API serializes back to JSON; the
1295
+ // lossless-path output (a compacted string) is exposed via
1296
+ // `crush_array().compacted` rather than being substituted into
1297
+ // the JSON re-serialization. So we exercise the lossy path
1298
+ // here via `without_compaction()` to validate the original
1299
+ // intent: low-uniqueness dicts compress via row-dropping.
1300
+ let c = SmartCrusher::without_compaction(SmartCrusherConfig::default());
1301
+ let mut input = String::from("[");
1302
+ for i in 0..30 {
1303
+ if i > 0 {
1304
+ input.push(',');
1305
+ }
1306
+ input.push_str(r#"{"status":"ok"}"#);
1307
+ }
1308
+ input.push(']');
1309
+ let result = c.crush(&input, "", 1.0);
1310
+ assert!(
1311
+ result.was_modified,
1312
+ "30 identical dicts should compress (low_uniqueness_safe_to_sample)"
1313
+ );
1314
+ assert_ne!(result.strategy, "passthrough");
1315
+ }
1316
+
1317
+ #[test]
1318
+ fn crush_serializes_with_python_safe_format() {
1319
+ let c = crusher();
1320
+ // SmartCrusher uses Python's `safe_json_dumps`: compact
1321
+ // separators `(",", ":")` + `ensure_ascii=False`, preserving
1322
+ // object-key insertion order. A spaced input round-trips to
1323
+ // the compact form.
1324
+ let input = r#"{"a": 1, "b": 2, "c": 3}"#;
1325
+ let result = c.crush(input, "", 1.0);
1326
+ assert_eq!(
1327
+ result.compressed, r#"{"a":1,"b":2,"c":3}"#,
1328
+ "safe_json_dumps emits compact `,` / `:` separators"
1329
+ );
1330
+ }
1331
+
1332
+ #[test]
1333
+ fn crush_recurses_into_nested_arrays() {
1334
+ let c = crusher();
1335
+ // Top-level dict with a nested array of 30 identical items.
1336
+ // The inner array should compress (low_uniqueness path).
1337
+ let mut inner = String::from("[");
1338
+ for i in 0..30 {
1339
+ if i > 0 {
1340
+ inner.push(',');
1341
+ }
1342
+ inner.push_str(r#"{"status":"ok"}"#);
1343
+ }
1344
+ inner.push(']');
1345
+ let input = format!(r#"{{"data": {}}}"#, inner);
1346
+ let result = c.crush(&input, "", 1.0);
1347
+ assert!(
1348
+ result.was_modified,
1349
+ "nested compressible array must be crushed even inside a wrapper object"
1350
+ );
1351
+ }
1352
+
1353
+ #[test]
1354
+ fn crusher_with_custom_scorer() {
1355
+ use crate::relevance::BM25Scorer;
1356
+ let c = SmartCrusher::with_scorer(
1357
+ SmartCrusherConfig::default(),
1358
+ Box::new(BM25Scorer::default()),
1359
+ );
1360
+ // Sanity: crushing still works with a swapped scorer.
1361
+ let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();
1362
+ let result = c.crush_array(&items, "anything", 1.0);
1363
+ assert!(result.items.len() <= 30);
1364
+ }
1365
+
1366
+ // ---------- Stage 3c.2 PR4: lossless-first default with threshold + CCR-Dropped ----------
1367
+
1368
+ #[test]
1369
+ fn without_compaction_yields_none_compacted_field() {
1370
+ // The opt-out constructor preserves pre-PR4 lossy-only path.
1371
+ // No lossless attempt → compacted/compaction_kind always None.
1372
+ let c = SmartCrusher::without_compaction(SmartCrusherConfig::default());
1373
+ let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();
1374
+ let result = c.crush_array(&items, "", 1.0);
1375
+ assert!(result.compacted.is_none());
1376
+ assert!(result.compaction_kind.is_none());
1377
+ }
1378
+
1379
+ #[test]
1380
+ fn lossless_wins_when_savings_above_threshold() {
1381
+ // 50 uniform tabular dicts → CSV+schema compaction shrinks
1382
+ // the input dramatically (well above the 0.30 default).
1383
+ // Default `SmartCrusher::new()` should pick lossless.
1384
+ let c = crusher();
1385
+ let items: Vec<Value> = (0..50)
1386
+ .map(|i| json!({"id": i, "name": format!("u_{i}"), "status": "ok"}))
1387
+ .collect();
1388
+ let result = c.crush_array(&items, "", 1.0);
1389
+ let compacted = result.compacted.expect("compacted should be set");
1390
+ assert!(compacted.starts_with("[50]{"), "got: {compacted}");
1391
+ assert_eq!(result.compaction_kind, Some("table"));
1392
+ assert!(
1393
+ result.strategy_info.starts_with("lossless:table"),
1394
+ "got: {}",
1395
+ result.strategy_info
1396
+ );
1397
+ // Lossless = nothing dropped → no CCR retrieval needed.
1398
+ assert!(result.ccr_hash.is_none());
1399
+ // items preserved (full original).
1400
+ assert_eq!(result.items.len(), 50);
1401
+ }
1402
+
1403
+ #[test]
1404
+ fn lossy_falls_through_when_savings_below_threshold() {
1405
+ // Force the threshold high enough that even tabular savings
1406
+ // can't satisfy it → lossy path runs → CCR-Dropped fires.
1407
+ // Use low-uniqueness items so the analyzer is willing to
1408
+ // crush (unique id+name per row would trip the
1409
+ // "unique_entities_no_signal" skip gate instead).
1410
+ let cfg = SmartCrusherConfig {
1411
+ lossless_min_savings_ratio: 0.99,
1412
+ ..Default::default()
1413
+ };
1414
+ let c = SmartCrusher::new(cfg);
1415
+ let items: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();
1416
+ let result = c.crush_array(&items, "", 1.0);
1417
+ // Lossless declined → no compacted output.
1418
+ assert!(result.compacted.is_none());
1419
+ // Lossy ran → rows dropped.
1420
+ assert!(
1421
+ result.items.len() < 50,
1422
+ "expected lossy drop, got {} items",
1423
+ result.items.len()
1424
+ );
1425
+ // CCR hash populated for retrieval.
1426
+ let h = result.ccr_hash.expect("ccr_hash populated on drop");
1427
+ assert_eq!(h.len(), 12);
1428
+ // Marker visible in dropped_summary.
1429
+ assert!(
1430
+ result.dropped_summary.contains(&format!("<<ccr:{h}")),
1431
+ "got: {}",
1432
+ result.dropped_summary
1433
+ );
1434
+ assert!(result.dropped_summary.contains("rows_offloaded"));
1435
+ }
1436
+
1437
+ #[test]
1438
+ fn ccr_hash_is_deterministic() {
1439
+ // Same input → same hash, so the runtime cache key is stable.
1440
+ let cfg = SmartCrusherConfig {
1441
+ lossless_min_savings_ratio: 0.99, // force lossy path
1442
+ ..Default::default()
1443
+ };
1444
+ let c = SmartCrusher::new(cfg);
1445
+ let items: Vec<Value> = (0..30).map(|i| json!({"id": i, "tag": "ok"})).collect();
1446
+ let r1 = c.crush_array(&items, "", 1.0);
1447
+ let r2 = c.crush_array(&items, "", 1.0);
1448
+ assert_eq!(r1.ccr_hash, r2.ccr_hash);
1449
+ assert!(r1.ccr_hash.is_some());
1450
+ }
1451
+
1452
+ #[test]
1453
+ fn ccr_hash_changes_with_input() {
1454
+ let cfg = SmartCrusherConfig {
1455
+ lossless_min_savings_ratio: 0.99,
1456
+ ..Default::default()
1457
+ };
1458
+ let c = SmartCrusher::new(cfg);
1459
+ let a: Vec<Value> = (0..30).map(|i| json!({"id": i})).collect();
1460
+ let b: Vec<Value> = (100..130).map(|i| json!({"id": i})).collect();
1461
+ let ra = c.crush_array(&a, "", 1.0);
1462
+ let rb = c.crush_array(&b, "", 1.0);
1463
+ assert_ne!(ra.ccr_hash, rb.ccr_hash);
1464
+ }
1465
+
1466
+ #[test]
1467
+ fn lossy_without_compaction_still_emits_ccr_hash() {
1468
+ // The CCR-Dropped restoration applies regardless of whether
1469
+ // lossless was attempted — without_compaction also gets the
1470
+ // ccr_hash on row drops.
1471
+ let c = SmartCrusher::without_compaction(SmartCrusherConfig::default());
1472
+ let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();
1473
+ let result = c.crush_array(&items, "", 1.0);
1474
+ if result.items.len() < items.len() {
1475
+ assert!(result.ccr_hash.is_some());
1476
+ assert!(!result.dropped_summary.is_empty());
1477
+ }
1478
+ }
1479
+
1480
+ #[test]
1481
+ fn passthrough_paths_do_not_emit_ccr_hash() {
1482
+ // Tier-1 boundary (items.len() <= adaptive_k): nothing
1483
+ // dropped, no CCR. Skip path: same.
1484
+ let c = crusher();
1485
+ let small: Vec<Value> = (0..3).map(|i| json!({"id": i})).collect();
1486
+ let r = c.crush_array(&small, "", 1.0);
1487
+ assert!(r.ccr_hash.is_none());
1488
+ assert_eq!(r.dropped_summary, "");
1489
+ }
1490
+
1491
+ #[test]
1492
+ fn compaction_skips_non_object_array() {
1493
+ // Compactor returns Untouched for non-object arrays → no
1494
+ // compacted field populated, no kind tag.
1495
+ let c = SmartCrusherBuilder::new(SmartCrusherConfig::default())
1496
+ .with_default_oss_setup()
1497
+ .with_default_compaction()
1498
+ .build();
1499
+ let items: Vec<Value> = (0..30).map(|i| json!(i)).collect();
1500
+ let result = c.crush_array(&items, "", 1.0);
1501
+ assert!(result.compacted.is_none());
1502
+ assert!(result.compaction_kind.is_none());
1503
+ }
1504
+
1505
+ // ---------- Stage 3c.2 PR5: walker-integration in process_value ----------
1506
+
1507
+ #[test]
1508
+ fn process_string_short_string_passthrough() {
1509
+ let c = SmartCrusher::new(SmartCrusherConfig::default());
1510
+ let (out, info) = c.process_value(&json!("hello world"), 0, "", 1.0);
1511
+ assert_eq!(out, json!("hello world"));
1512
+ assert!(info.is_empty());
1513
+ }
1514
+
1515
+ #[test]
1516
+ fn process_string_stringified_json_array_recurses() {
1517
+ // A string-typed field whose value is a JSON-encoded array of
1518
+ // dicts. process_value should parse it, recurse, and return
1519
+ // the processed JSON re-rendered as a string.
1520
+ let c = SmartCrusher::new(SmartCrusherConfig::default());
1521
+ let big_array_json = serde_json::to_string(
1522
+ &(0..50)
1523
+ .map(|i| json!({"id": i, "level": "info", "msg": "ok"}))
1524
+ .collect::<Vec<_>>(),
1525
+ )
1526
+ .unwrap();
1527
+ let doc = json!({"payload": big_array_json.clone()});
1528
+ let (out, info) = c.process_value(&doc, 0, "", 1.0);
1529
+ // payload still a string-typed field — we preserved the
1530
+ // wrapping shape — but its content was processed.
1531
+ let payload = out.pointer("/payload").and_then(|v| v.as_str()).unwrap();
1532
+ // Either compressed or unchanged; if compressed, info reflects.
1533
+ // For 50 items with low-uniqueness, compression should fire.
1534
+ // The strategy info should mention string_json processing.
1535
+ assert!(
1536
+ info.contains("string_json") || payload != big_array_json,
1537
+ "expected processing trace; info={info}, len before={}, after={}",
1538
+ big_array_json.len(),
1539
+ payload.len(),
1540
+ );
1541
+ }
1542
+
1543
+ #[test]
1544
+ fn process_string_opaque_blob_becomes_ccr_marker() {
1545
+ let c = SmartCrusher::new(SmartCrusherConfig::default());
1546
+ let big_b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".repeat(8);
1547
+ let doc = json!({"id": 1, "blob": big_b64});
1548
+ let (out, _info) = c.process_value(&doc, 0, "", 1.0);
1549
+ let blob = out.pointer("/blob").and_then(|v| v.as_str()).unwrap();
1550
+ assert!(blob.starts_with("<<ccr:"), "got: {blob}");
1551
+ assert!(blob.contains(",base64,"));
1552
+ }
1553
+
1554
+ #[test]
1555
+ fn process_string_top_level_string_processed() {
1556
+ // crush() takes a string; if it doesn't parse as JSON, today's
1557
+ // behavior returns it unchanged. But if it's a stringified
1558
+ // JSON object/array, it should now get processed.
1559
+ let c = SmartCrusher::new(SmartCrusherConfig::default());
1560
+ // Non-JSON top-level string — passthrough.
1561
+ let plain = "just some plain text";
1562
+ let result = c.crush(plain, "", 1.0);
1563
+ assert_eq!(result.compressed, plain);
1564
+ }
1565
+
1566
+ #[test]
1567
+ fn process_string_does_not_alter_short_quoted_strings() {
1568
+ // Strings that look JSON-like but are short shouldn't be
1569
+ // CCR-substituted.
1570
+ let c = SmartCrusher::new(SmartCrusherConfig::default());
1571
+ let doc = json!({"msg": "{this looks like json but isnt}"});
1572
+ let (out, _) = c.process_value(&doc, 0, "", 1.0);
1573
+ assert_eq!(out, doc);
1574
+ }
1575
+
1576
+ #[test]
1577
+ fn process_string_helper_parses_only_containers() {
1578
+ assert!(try_parse_json_container("{\"a\":1}").is_some());
1579
+ assert!(try_parse_json_container("[1,2,3]").is_some());
1580
+ assert!(try_parse_json_container("123").is_none()); // bare scalar
1581
+ assert!(try_parse_json_container("\"hello\"").is_none()); // bare string
1582
+ assert!(try_parse_json_container("not json").is_none());
1583
+ assert!(try_parse_json_container("{malformed").is_none());
1584
+ }
1585
+
1586
+ // ---------- enable_ccr_marker gate (PR #301 re-land) ----------
1587
+
1588
+ #[test]
1589
+ fn enable_ccr_marker_false_suppresses_marker_and_store() {
1590
+ // The Rust-side gate. Compression still runs (rows drop) but
1591
+ // the result carries no marker text, no hash, and the CCR
1592
+ // store does NOT grow — there's no point storing what nothing
1593
+ // in the prompt can reference.
1594
+ use crate::ccr::InMemoryCcrStore;
1595
+ use crate::transforms::smart_crusher::SmartCrusherBuilder;
1596
+ use std::sync::Arc;
1597
+
1598
+ let store: Arc<dyn CcrStore> = Arc::new(InMemoryCcrStore::new());
1599
+ let cfg = SmartCrusherConfig {
1600
+ lossless_min_savings_ratio: 0.99, // force lossy path
1601
+ enable_ccr_marker: false,
1602
+ ..SmartCrusherConfig::default()
1603
+ };
1604
+ let c = SmartCrusherBuilder::new(cfg)
1605
+ .with_ccr_store(Arc::clone(&store))
1606
+ .build();
1607
+ let items: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();
1608
+
1609
+ let store_len_before = store.len();
1610
+ let result = c.crush_array(&items, "", 1.0);
1611
+ let store_len_after = store.len();
1612
+
1613
+ // Rows were dropped (we built 50, kept fewer).
1614
+ assert!(result.items.len() < items.len(), "lossy path didn't fire");
1615
+ // Gate held: no marker, no hash.
1616
+ assert!(result.ccr_hash.is_none(), "ccr_hash should be None");
1617
+ assert!(
1618
+ result.dropped_summary.is_empty(),
1619
+ "dropped_summary should be empty, got: {:?}",
1620
+ result.dropped_summary
1621
+ );
1622
+ // Store did NOT grow.
1623
+ assert_eq!(
1624
+ store_len_after, store_len_before,
1625
+ "ccr_store grew despite enable_ccr_marker=false"
1626
+ );
1627
+ }
1628
+
1629
+ #[test]
1630
+ fn enable_ccr_marker_true_is_default_behavior() {
1631
+ // Default config still emits markers + stores when rows drop.
1632
+ // Sanity: the gate is opt-out, not opt-in.
1633
+ use crate::ccr::InMemoryCcrStore;
1634
+ use crate::transforms::smart_crusher::SmartCrusherBuilder;
1635
+ use std::sync::Arc;
1636
+
1637
+ let store: Arc<dyn CcrStore> = Arc::new(InMemoryCcrStore::new());
1638
+ let cfg = SmartCrusherConfig {
1639
+ lossless_min_savings_ratio: 0.99, // force lossy path
1640
+ ..SmartCrusherConfig::default()
1641
+ };
1642
+ // Default: enable_ccr_marker = true.
1643
+ assert!(cfg.enable_ccr_marker);
1644
+ let c = SmartCrusherBuilder::new(cfg)
1645
+ .with_ccr_store(Arc::clone(&store))
1646
+ .build();
1647
+ let items: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();
1648
+
1649
+ let store_len_before = store.len();
1650
+ let result = c.crush_array(&items, "", 1.0);
1651
+ let store_len_after = store.len();
1652
+
1653
+ assert!(result.items.len() < items.len(), "lossy path didn't fire");
1654
+ assert!(result.ccr_hash.is_some(), "default should produce a hash");
1655
+ assert!(
1656
+ result.dropped_summary.contains("<<ccr:"),
1657
+ "default should produce a marker: {:?}",
1658
+ result.dropped_summary
1659
+ );
1660
+ assert!(
1661
+ store_len_after > store_len_before,
1662
+ "default should write to ccr_store"
1663
+ );
1664
+ }
1665
+ }