xithead 1.4.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (661) hide show
  1. package/.actrc +14 -0
  2. package/.actrc.local.example +15 -0
  3. package/.changelog.md +2 -0
  4. package/.claude/agents/academic-anthropologist.md +125 -0
  5. package/.claude/agents/academic-geographer.md +127 -0
  6. package/.claude/agents/academic-historian.md +123 -0
  7. package/.claude/agents/academic-narratologist.md +118 -0
  8. package/.claude/agents/academic-psychologist.md +118 -0
  9. package/.claude/agents/academic-statistician.md +144 -0
  10. package/.claude/agents/accounts-payable-agent.md +205 -0
  11. package/.claude/agents/agentic-identity-trust.md +394 -0
  12. package/.claude/agents/agents-orchestrator.md +367 -0
  13. package/.claude/agents/automation-governance-architect.md +216 -0
  14. package/.claude/agents/blender-addon-engineer.md +243 -0
  15. package/.claude/agents/business-strategist.md +488 -0
  16. package/.claude/agents/change-management-consultant.md +497 -0
  17. package/.claude/agents/chief-financial-officer.md +388 -0
  18. package/.claude/agents/corporate-training-designer.md +192 -0
  19. package/.claude/agents/customer-service.md +398 -0
  20. package/.claude/agents/customer-success-manager.md +460 -0
  21. package/.claude/agents/data-consolidation-agent.md +60 -0
  22. package/.claude/agents/data-privacy-officer.md +412 -0
  23. package/.claude/agents/design-brand-guardian.md +322 -0
  24. package/.claude/agents/design-image-prompt-engineer.md +236 -0
  25. package/.claude/agents/design-inclusive-visuals-specialist.md +73 -0
  26. package/.claude/agents/design-persona-walkthrough.md +272 -0
  27. package/.claude/agents/design-ui-designer.md +386 -0
  28. package/.claude/agents/design-ui-finish-gate-reviewer.md +217 -0
  29. package/.claude/agents/design-ux-architect.md +482 -0
  30. package/.claude/agents/design-ux-researcher.md +329 -0
  31. package/.claude/agents/design-visual-storyteller.md +149 -0
  32. package/.claude/agents/design-whimsy-injector.md +460 -0
  33. package/.claude/agents/economy-designer.md +156 -0
  34. package/.claude/agents/engineering-ai-data-remediation-engineer.md +211 -0
  35. package/.claude/agents/engineering-ai-engineer.md +146 -0
  36. package/.claude/agents/engineering-api-platform-engineer.md +181 -0
  37. package/.claude/agents/engineering-ats-validator-architect.md +383 -0
  38. package/.claude/agents/engineering-autonomous-optimization-architect.md +121 -0
  39. package/.claude/agents/engineering-backend-architect.md +255 -0
  40. package/.claude/agents/engineering-china-network-engineer.md +252 -0
  41. package/.claude/agents/engineering-cms-developer.md +536 -0
  42. package/.claude/agents/engineering-code-reviewer.md +76 -0
  43. package/.claude/agents/engineering-codebase-onboarding-engineer.md +173 -0
  44. package/.claude/agents/engineering-data-engineer.md +349 -0
  45. package/.claude/agents/engineering-data-visualization-engineer.md +151 -0
  46. package/.claude/agents/engineering-database-optimizer.md +188 -0
  47. package/.claude/agents/engineering-database-reliability-engineer.md +201 -0
  48. package/.claude/agents/engineering-desktop-app-engineer.md +209 -0
  49. package/.claude/agents/engineering-developer-tooling-engineer.md +153 -0
  50. package/.claude/agents/engineering-devops-automator.md +385 -0
  51. package/.claude/agents/engineering-drupal-performance.md +347 -0
  52. package/.claude/agents/engineering-drupal-shopping-cart.md +360 -0
  53. package/.claude/agents/engineering-electrical-engineer.md +575 -0
  54. package/.claude/agents/engineering-email-intelligence-engineer.md +390 -0
  55. package/.claude/agents/engineering-embedded-firmware-engineer.md +196 -0
  56. package/.claude/agents/engineering-feishu-integration-developer.md +606 -0
  57. package/.claude/agents/engineering-filament-optimization-specialist.md +283 -0
  58. package/.claude/agents/engineering-finops-engineer.md +164 -0
  59. package/.claude/agents/engineering-frontend-developer.md +257 -0
  60. package/.claude/agents/engineering-gaussdb-expert.md +334 -0
  61. package/.claude/agents/engineering-git-workflow-master.md +98 -0
  62. package/.claude/agents/engineering-i18n-engineer.md +214 -0
  63. package/.claude/agents/engineering-identity-access-engineer.md +217 -0
  64. package/.claude/agents/engineering-incident-response-commander.md +444 -0
  65. package/.claude/agents/engineering-iot-fleet-engineer.md +148 -0
  66. package/.claude/agents/engineering-it-service-manager.md +561 -0
  67. package/.claude/agents/engineering-knowledge-graph-engineer.md +372 -0
  68. package/.claude/agents/engineering-llm-post-training-engineer.md +166 -0
  69. package/.claude/agents/engineering-minimal-change-engineer.md +207 -0
  70. package/.claude/agents/engineering-mobile-app-builder.md +493 -0
  71. package/.claude/agents/engineering-mobile-release-engineer.md +164 -0
  72. package/.claude/agents/engineering-multi-agent-systems-architect.md +600 -0
  73. package/.claude/agents/engineering-network-engineer.md +245 -0
  74. package/.claude/agents/engineering-orgscript-engineer.md +113 -0
  75. package/.claude/agents/engineering-payments-billing-engineer.md +228 -0
  76. package/.claude/agents/engineering-pdf-engine-architect.md +696 -0
  77. package/.claude/agents/engineering-platform-engineer.md +270 -0
  78. package/.claude/agents/engineering-privacy-engineer.md +152 -0
  79. package/.claude/agents/engineering-prompt-engineer.md +209 -0
  80. package/.claude/agents/engineering-rag-pipeline-engineer.md +450 -0
  81. package/.claude/agents/engineering-rapid-prototyper.md +466 -0
  82. package/.claude/agents/engineering-realtime-collaboration-engineer.md +200 -0
  83. package/.claude/agents/engineering-rust-refactoring-specialist.md +313 -0
  84. package/.claude/agents/engineering-search-relevance-engineer.md +237 -0
  85. package/.claude/agents/engineering-section-508-specialist.md +339 -0
  86. package/.claude/agents/engineering-senior-developer.md +176 -0
  87. package/.claude/agents/engineering-servicenow-developer-mentor.md +180 -0
  88. package/.claude/agents/engineering-software-architect.md +112 -0
  89. package/.claude/agents/engineering-solidity-smart-contract-engineer.md +522 -0
  90. package/.claude/agents/engineering-sre.md +90 -0
  91. package/.claude/agents/engineering-technical-writer.md +393 -0
  92. package/.claude/agents/engineering-universal-document-compiler.md +370 -0
  93. package/.claude/agents/engineering-uswds-developer.md +340 -0
  94. package/.claude/agents/engineering-video-streaming-engineer.md +151 -0
  95. package/.claude/agents/engineering-voice-ai-integration-engineer.md +584 -0
  96. package/.claude/agents/engineering-webassembly-engineer.md +153 -0
  97. package/.claude/agents/engineering-wechat-mini-program-developer.md +356 -0
  98. package/.claude/agents/engineering-wordpress-performance.md +346 -0
  99. package/.claude/agents/engineering-wordpress-shopping-cart.md +346 -0
  100. package/.claude/agents/esg-sustainability-officer.md +396 -0
  101. package/.claude/agents/finance-bookkeeper-controller.md +260 -0
  102. package/.claude/agents/finance-financial-analyst.md +234 -0
  103. package/.claude/agents/finance-fpa-analyst.md +263 -0
  104. package/.claude/agents/finance-investment-researcher.md +272 -0
  105. package/.claude/agents/finance-tax-strategist.md +239 -0
  106. package/.claude/agents/game-audio-engineer.md +264 -0
  107. package/.claude/agents/game-designer.md +167 -0
  108. package/.claude/agents/gis-3d-scene-developer.md +111 -0
  109. package/.claude/agents/gis-analyst.md +91 -0
  110. package/.claude/agents/gis-bim-specialist.md +108 -0
  111. package/.claude/agents/gis-cartography-designer.md +150 -0
  112. package/.claude/agents/gis-drone-reality-mapping.md +120 -0
  113. package/.claude/agents/gis-geoai-ml-engineer.md +105 -0
  114. package/.claude/agents/gis-geoprocessing-specialist.md +97 -0
  115. package/.claude/agents/gis-qa-engineer.md +133 -0
  116. package/.claude/agents/gis-solution-engineer.md +101 -0
  117. package/.claude/agents/gis-spatial-data-engineer.md +97 -0
  118. package/.claude/agents/gis-spatial-data-scientist.md +111 -0
  119. package/.claude/agents/gis-technical-consultant.md +86 -0
  120. package/.claude/agents/gis-web-gis-developer.md +108 -0
  121. package/.claude/agents/godot-gameplay-scripter.md +338 -0
  122. package/.claude/agents/godot-multiplayer-engineer.md +301 -0
  123. package/.claude/agents/godot-shader-developer.md +267 -0
  124. package/.claude/agents/government-digital-presales-consultant.md +363 -0
  125. package/.claude/agents/grant-writer.md +511 -0
  126. package/.claude/agents/healthcare-aging-parent-care-companion.md +414 -0
  127. package/.claude/agents/healthcare-clinical-evidence-agent.md +231 -0
  128. package/.claude/agents/healthcare-customer-service.md +389 -0
  129. package/.claude/agents/healthcare-innovation-strategist.md +433 -0
  130. package/.claude/agents/healthcare-marketing-compliance.md +395 -0
  131. package/.claude/agents/healthcare-sovereign-health-systems-agent.md +312 -0
  132. package/.claude/agents/hospitality-guest-services.md +603 -0
  133. package/.claude/agents/hr-onboarding.md +451 -0
  134. package/.claude/agents/identity-graph-operator.md +303 -0
  135. package/.claude/agents/language-translator.md +264 -0
  136. package/.claude/agents/legal-billing-time-tracking.md +569 -0
  137. package/.claude/agents/legal-client-intake.md +492 -0
  138. package/.claude/agents/legal-document-review.md +454 -0
  139. package/.claude/agents/level-designer.md +208 -0
  140. package/.claude/agents/loan-officer-assistant.md +555 -0
  141. package/.claude/agents/lsp-index-engineer.md +320 -0
  142. package/.claude/agents/ma-integration-manager.md +427 -0
  143. package/.claude/agents/macos-spatial-metal-engineer.md +337 -0
  144. package/.claude/agents/marketing-aeo-foundations.md +264 -0
  145. package/.claude/agents/marketing-agentic-search-optimizer.md +279 -0
  146. package/.claude/agents/marketing-ai-citation-strategist.md +172 -0
  147. package/.claude/agents/marketing-app-store-optimizer.md +321 -0
  148. package/.claude/agents/marketing-baidu-seo-specialist.md +226 -0
  149. package/.claude/agents/marketing-bilibili-content-strategist.md +199 -0
  150. package/.claude/agents/marketing-book-co-author.md +110 -0
  151. package/.claude/agents/marketing-carousel-growth-engine.md +204 -0
  152. package/.claude/agents/marketing-china-ecommerce-operator.md +283 -0
  153. package/.claude/agents/marketing-china-market-localization-strategist.md +283 -0
  154. package/.claude/agents/marketing-content-creator.md +54 -0
  155. package/.claude/agents/marketing-cross-border-ecommerce.md +259 -0
  156. package/.claude/agents/marketing-developer-community-builder.md +285 -0
  157. package/.claude/agents/marketing-douyin-strategist.md +149 -0
  158. package/.claude/agents/marketing-email-strategist.md +249 -0
  159. package/.claude/agents/marketing-global-podcast-strategist.md +206 -0
  160. package/.claude/agents/marketing-growth-hacker.md +54 -0
  161. package/.claude/agents/marketing-instagram-curator.md +113 -0
  162. package/.claude/agents/marketing-kuaishou-strategist.md +223 -0
  163. package/.claude/agents/marketing-linkedin-content-creator.md +214 -0
  164. package/.claude/agents/marketing-livestream-commerce-coach.md +305 -0
  165. package/.claude/agents/marketing-multi-platform-publisher.md +217 -0
  166. package/.claude/agents/marketing-podcast-strategist.md +277 -0
  167. package/.claude/agents/marketing-pr-communications-manager.md +473 -0
  168. package/.claude/agents/marketing-private-domain-operator.md +308 -0
  169. package/.claude/agents/marketing-reddit-community-builder.md +123 -0
  170. package/.claude/agents/marketing-seo-specialist.md +370 -0
  171. package/.claude/agents/marketing-short-video-editing-coach.md +412 -0
  172. package/.claude/agents/marketing-social-media-strategist.md +125 -0
  173. package/.claude/agents/marketing-tiktok-strategist.md +125 -0
  174. package/.claude/agents/marketing-twitter-engager.md +126 -0
  175. package/.claude/agents/marketing-video-optimization-specialist.md +119 -0
  176. package/.claude/agents/marketing-wechat-official-account.md +145 -0
  177. package/.claude/agents/marketing-weibo-strategist.md +240 -0
  178. package/.claude/agents/marketing-x-twitter-intelligence-analyst.md +161 -0
  179. package/.claude/agents/marketing-xiaohongshu-specialist.md +138 -0
  180. package/.claude/agents/marketing-zhihu-strategist.md +162 -0
  181. package/.claude/agents/medical-billing-coding-specialist.md +491 -0
  182. package/.claude/agents/narrative-designer.md +243 -0
  183. package/.claude/agents/operations-manager.md +399 -0
  184. package/.claude/agents/organizational-psychologist.md +391 -0
  185. package/.claude/agents/paid-media-auditor.md +71 -0
  186. package/.claude/agents/paid-media-creative-strategist.md +71 -0
  187. package/.claude/agents/paid-media-paid-social-strategist.md +71 -0
  188. package/.claude/agents/paid-media-ppc-strategist.md +71 -0
  189. package/.claude/agents/paid-media-programmatic-buyer.md +71 -0
  190. package/.claude/agents/paid-media-search-query-analyst.md +71 -0
  191. package/.claude/agents/paid-media-tracking-specialist.md +71 -0
  192. package/.claude/agents/personal-growth-mentor.md +159 -0
  193. package/.claude/agents/product-behavioral-nudge-engine.md +83 -0
  194. package/.claude/agents/product-dx-engineer.md +332 -0
  195. package/.claude/agents/product-feedback-synthesizer.md +119 -0
  196. package/.claude/agents/product-manager.md +469 -0
  197. package/.claude/agents/product-sprint-prioritizer.md +154 -0
  198. package/.claude/agents/product-trend-researcher.md +159 -0
  199. package/.claude/agents/project-management-experiment-tracker.md +198 -0
  200. package/.claude/agents/project-management-jira-workflow-steward.md +233 -0
  201. package/.claude/agents/project-management-meeting-notes-specialist.md +95 -0
  202. package/.claude/agents/project-management-project-shepherd.md +194 -0
  203. package/.claude/agents/project-management-studio-operations.md +200 -0
  204. package/.claude/agents/project-management-studio-producer.md +203 -0
  205. package/.claude/agents/project-manager-senior.md +135 -0
  206. package/.claude/agents/real-estate-buyer-seller.md +596 -0
  207. package/.claude/agents/recruitment-specialist.md +515 -0
  208. package/.claude/agents/report-distribution-agent.md +65 -0
  209. package/.claude/agents/research-synthesist.md +136 -0
  210. package/.claude/agents/resume-tailor.md +230 -0
  211. package/.claude/agents/retail-customer-returns.md +566 -0
  212. package/.claude/agents/roblox-avatar-creator.md +297 -0
  213. package/.claude/agents/roblox-experience-designer.md +306 -0
  214. package/.claude/agents/roblox-systems-scripter.md +332 -0
  215. package/.claude/agents/sales-account-strategist.md +227 -0
  216. package/.claude/agents/sales-coach.md +271 -0
  217. package/.claude/agents/sales-data-extraction-agent.md +67 -0
  218. package/.claude/agents/sales-deal-strategist.md +180 -0
  219. package/.claude/agents/sales-discovery-coach.md +225 -0
  220. package/.claude/agents/sales-engineer.md +182 -0
  221. package/.claude/agents/sales-offer-lead-gen-strategist.md +257 -0
  222. package/.claude/agents/sales-outbound-strategist.md +201 -0
  223. package/.claude/agents/sales-outreach.md +425 -0
  224. package/.claude/agents/sales-pipeline-analyst.md +267 -0
  225. package/.claude/agents/sales-proposal-strategist.md +217 -0
  226. package/.claude/agents/security-ai-generated-code-auditor.md +207 -0
  227. package/.claude/agents/security-appsec-engineer.md +536 -0
  228. package/.claude/agents/security-architect.md +307 -0
  229. package/.claude/agents/security-blockchain-security-auditor.md +482 -0
  230. package/.claude/agents/security-cloud-security-architect.md +576 -0
  231. package/.claude/agents/security-compliance-auditor.md +158 -0
  232. package/.claude/agents/security-incident-responder.md +450 -0
  233. package/.claude/agents/security-penetration-tester.md +399 -0
  234. package/.claude/agents/security-secrets-credential-engineer.md +189 -0
  235. package/.claude/agents/security-senior-secops.md +760 -0
  236. package/.claude/agents/security-threat-detection-engineer.md +553 -0
  237. package/.claude/agents/security-threat-intelligence-analyst.md +661 -0
  238. package/.claude/agents/specialized-chief-of-staff.md +279 -0
  239. package/.claude/agents/specialized-civil-engineer.md +356 -0
  240. package/.claude/agents/specialized-codebase-archaeologist.md +341 -0
  241. package/.claude/agents/specialized-cultural-intelligence-strategist.md +88 -0
  242. package/.claude/agents/specialized-developer-advocate.md +317 -0
  243. package/.claude/agents/specialized-document-generator.md +55 -0
  244. package/.claude/agents/specialized-fedramp-rmf-compliance.md +378 -0
  245. package/.claude/agents/specialized-focus-music-architect.md +175 -0
  246. package/.claude/agents/specialized-french-consulting-market.md +194 -0
  247. package/.claude/agents/specialized-korean-business-navigator.md +216 -0
  248. package/.claude/agents/specialized-master-plan-architect.md +157 -0
  249. package/.claude/agents/specialized-mcp-builder.md +260 -0
  250. package/.claude/agents/specialized-model-qa.md +528 -0
  251. package/.claude/agents/specialized-pricing-analyst.md +243 -0
  252. package/.claude/agents/specialized-salesforce-architect.md +182 -0
  253. package/.claude/agents/specialized-strategy-duel-agent.md +130 -0
  254. package/.claude/agents/specialized-workflow-architect.md +597 -0
  255. package/.claude/agents/study-abroad-advisor.md +282 -0
  256. package/.claude/agents/supply-chain-strategist.md +591 -0
  257. package/.claude/agents/support-analytics-reporter.md +387 -0
  258. package/.claude/agents/support-executive-summary-generator.md +212 -0
  259. package/.claude/agents/support-finance-tracker.md +480 -0
  260. package/.claude/agents/support-infrastructure-maintainer.md +628 -0
  261. package/.claude/agents/support-legal-compliance-checker.md +591 -0
  262. package/.claude/agents/support-support-responder.md +607 -0
  263. package/.claude/agents/technical-artist.md +233 -0
  264. package/.claude/agents/terminal-integration-specialist.md +70 -0
  265. package/.claude/agents/testing-accessibility-auditor.md +327 -0
  266. package/.claude/agents/testing-api-tester.md +334 -0
  267. package/.claude/agents/testing-evidence-collector.md +210 -0
  268. package/.claude/agents/testing-performance-benchmarker.md +283 -0
  269. package/.claude/agents/testing-reality-checker.md +249 -0
  270. package/.claude/agents/testing-test-automation-engineer.md +196 -0
  271. package/.claude/agents/testing-test-results-analyzer.md +330 -0
  272. package/.claude/agents/testing-tool-evaluator.md +416 -0
  273. package/.claude/agents/testing-workflow-optimizer.md +463 -0
  274. package/.claude/agents/unity-architect.md +271 -0
  275. package/.claude/agents/unity-editor-tool-developer.md +310 -0
  276. package/.claude/agents/unity-multiplayer-engineer.md +323 -0
  277. package/.claude/agents/unity-shader-graph-artist.md +269 -0
  278. package/.claude/agents/unreal-multiplayer-architect.md +364 -0
  279. package/.claude/agents/unreal-systems-engineer.md +322 -0
  280. package/.claude/agents/unreal-technical-artist.md +256 -0
  281. package/.claude/agents/unreal-world-builder.md +273 -0
  282. package/.claude/agents/visionos-spatial-engineer.md +54 -0
  283. package/.claude/agents/xr-cockpit-interaction-specialist.md +32 -0
  284. package/.claude/agents/xr-immersive-developer.md +32 -0
  285. package/.claude/agents/xr-interface-architect.md +32 -0
  286. package/.claude/agents/zk-steward.md +211 -0
  287. package/.claude-plugin/marketplace.json +30 -0
  288. package/.commitlintrc.json +26 -0
  289. package/.devcontainer/Dockerfile +24 -0
  290. package/.devcontainer/devcontainer.json +70 -0
  291. package/.devcontainer/docker-compose.memory.yml +42 -0
  292. package/.devcontainer/memory-stack/devcontainer.json +84 -0
  293. package/.devcontainer/post-create.sh +83 -0
  294. package/.dockerignore +23 -0
  295. package/.env.act.example +15 -0
  296. package/.env.example +171 -0
  297. package/.eslintrc.json +16 -0
  298. package/.git-blame-ignore-revs +10 -0
  299. package/.gitattributes +2 -0
  300. package/.gitguardian.yaml +50 -0
  301. package/.githooks/pre-commit +6 -0
  302. package/.pre-commit-config.yaml +30 -0
  303. package/.release-please-config-xithead.json +22 -0
  304. package/.release-please-config.json +44 -0
  305. package/.release-please-manifest-xithead.json +3 -0
  306. package/.release-please-manifest.json +3 -0
  307. package/CODE_OF_CONDUCT.md +133 -0
  308. package/CONTRIBUTING.md +133 -0
  309. package/Cargo.lock +5557 -0
  310. package/Cargo.toml +121 -0
  311. package/Dockerfile +38 -0
  312. package/Dockerfile.proxy +48 -0
  313. package/ENTERPRISE.md +5 -0
  314. package/FEATURES_ADVANCED.md +311 -0
  315. package/Headroom-2.gif +0 -0
  316. package/HeadroomDemo-Fast.gif +0 -0
  317. package/MANIFEST.in +4 -0
  318. package/MODERNIZATION_GUIDE.md +358 -0
  319. package/Makefile +140 -0
  320. package/NOTICE +43 -0
  321. package/NPM_TOKEN_SETUP.md +81 -0
  322. package/PR.md +170 -0
  323. package/PUBLISH.md +77 -0
  324. package/README.md +108 -72
  325. package/README_FRONTEND.md +196 -0
  326. package/REALIGNMENT/00-overview.md +53 -0
  327. package/REALIGNMENT/01-bug-list.md +421 -0
  328. package/REALIGNMENT/02-architecture.md +322 -0
  329. package/REALIGNMENT/03-phase-A-lockdown.md +420 -0
  330. package/REALIGNMENT/04-phase-B-live-zone.md +470 -0
  331. package/REALIGNMENT/05-phase-C-rust-proxy.md +329 -0
  332. package/REALIGNMENT/06-phase-D-bedrock-vertex.md +222 -0
  333. package/REALIGNMENT/07-phase-E-cache-stabilization.md +338 -0
  334. package/REALIGNMENT/08-phase-F-auth-mode.md +254 -0
  335. package/REALIGNMENT/09-phase-G-rtk-observability.md +194 -0
  336. package/REALIGNMENT/10-phase-H-python-retirement.md +214 -0
  337. package/REALIGNMENT/11-phase-I-test-infra.md +385 -0
  338. package/REALIGNMENT/12-decisions-needed.md +196 -0
  339. package/REALIGNMENT/INDEX.md +82 -0
  340. package/REFACTORING_CAMPAIGN.md +277 -0
  341. package/RESTRUCTURE_PLAN.md +100 -0
  342. package/RUST_DEV.md +361 -0
  343. package/SECURITY.md +65 -0
  344. package/SETUP_AUTO_PUBLISH.md +164 -0
  345. package/TESTING-copilot-subscription.md +153 -0
  346. package/app/api/config/route.ts +22 -0
  347. package/app/api/export/route.ts +83 -0
  348. package/app/api/models/route.ts +12 -0
  349. package/app/api/stats/route.ts +46 -0
  350. package/app/dashboard/page.tsx +445 -0
  351. package/app/globals.css +35 -0
  352. package/app/installation/page.tsx +293 -0
  353. package/app/layout.tsx +35 -0
  354. package/app/master-design-system.css +2076 -0
  355. package/app/master-design-system.source.css +8 -0
  356. package/app/page.tsx +187 -0
  357. package/app/rechtliches/agb/page.tsx +165 -0
  358. package/app/rechtliches/datenschutz/page.tsx +128 -0
  359. package/app/rechtliches/impressum/page.tsx +122 -0
  360. package/app/rechtliches/widerruf/page.tsx +185 -0
  361. package/bin/xithead.js +264 -0
  362. package/codecov.yml +34 -0
  363. package/components/layout/AppShell.tsx +25 -0
  364. package/components/layout/Footer.tsx +120 -0
  365. package/components/layout/Header.tsx +145 -0
  366. package/components/marketing/LegalHighlights.tsx +45 -0
  367. package/components/marketing/LegalPageShell.tsx +49 -0
  368. package/components/marketing/LegalPanel.tsx +41 -0
  369. package/components/marketing/index.ts +3 -0
  370. package/crates/headroom-core/Cargo.toml +165 -0
  371. package/crates/headroom-core/benches/auth_mode.rs +74 -0
  372. package/crates/headroom-core/benches/ccr_store.rs +224 -0
  373. package/crates/headroom-core/benches/tokenizer.rs +52 -0
  374. package/crates/headroom-core/config/pipeline.toml +151 -0
  375. package/crates/headroom-core/proptest-regressions/transforms/tag_protector.txt +9 -0
  376. package/crates/headroom-core/src/auth_mode.rs +250 -0
  377. package/crates/headroom-core/src/cache_control.rs +414 -0
  378. package/crates/headroom-core/src/ccr/backends/in_memory.rs +317 -0
  379. package/crates/headroom-core/src/ccr/backends/mod.rs +152 -0
  380. package/crates/headroom-core/src/ccr/backends/redis.rs +146 -0
  381. package/crates/headroom-core/src/ccr/backends/sqlite.rs +205 -0
  382. package/crates/headroom-core/src/ccr/mod.rs +119 -0
  383. package/crates/headroom-core/src/compression_policy.rs +505 -0
  384. package/crates/headroom-core/src/lib.rs +33 -0
  385. package/crates/headroom-core/src/relevance/base.rs +139 -0
  386. package/crates/headroom-core/src/relevance/bm25.rs +385 -0
  387. package/crates/headroom-core/src/relevance/embedding.rs +354 -0
  388. package/crates/headroom-core/src/relevance/hybrid.rs +364 -0
  389. package/crates/headroom-core/src/relevance/mod.rs +64 -0
  390. package/crates/headroom-core/src/signals/keyword_detector.rs +433 -0
  391. package/crates/headroom-core/src/signals/line_importance.rs +84 -0
  392. package/crates/headroom-core/src/signals/mod.rs +61 -0
  393. package/crates/headroom-core/src/signals/tiered.rs +141 -0
  394. package/crates/headroom-core/src/tokenizer/estimator.rs +152 -0
  395. package/crates/headroom-core/src/tokenizer/hf_impl.rs +314 -0
  396. package/crates/headroom-core/src/tokenizer/mod.rs +56 -0
  397. package/crates/headroom-core/src/tokenizer/registry.rs +342 -0
  398. package/crates/headroom-core/src/tokenizer/tiktoken_impl.rs +265 -0
  399. package/crates/headroom-core/src/transforms/adaptive_sizer.rs +610 -0
  400. package/crates/headroom-core/src/transforms/anchor_selector.rs +1189 -0
  401. package/crates/headroom-core/src/transforms/content_detector.rs +769 -0
  402. package/crates/headroom-core/src/transforms/detection.rs +222 -0
  403. package/crates/headroom-core/src/transforms/diff_compressor.rs +1685 -0
  404. package/crates/headroom-core/src/transforms/live_zone.rs +2967 -0
  405. package/crates/headroom-core/src/transforms/log_compressor.rs +1295 -0
  406. package/crates/headroom-core/src/transforms/magika_detector.rs +395 -0
  407. package/crates/headroom-core/src/transforms/mod.rs +64 -0
  408. package/crates/headroom-core/src/transforms/pipeline/config.rs +331 -0
  409. package/crates/headroom-core/src/transforms/pipeline/mod.rs +103 -0
  410. package/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rs +524 -0
  411. package/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs +279 -0
  412. package/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs +400 -0
  413. package/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rs +284 -0
  414. package/crates/headroom-core/src/transforms/pipeline/offloads/mod.rs +34 -0
  415. package/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rs +331 -0
  416. package/crates/headroom-core/src/transforms/pipeline/orchestrator.rs +848 -0
  417. package/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rs +175 -0
  418. package/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rs +545 -0
  419. package/crates/headroom-core/src/transforms/pipeline/reformats/mod.rs +12 -0
  420. package/crates/headroom-core/src/transforms/pipeline/traits.rs +338 -0
  421. package/crates/headroom-core/src/transforms/recommendations.rs +329 -0
  422. package/crates/headroom-core/src/transforms/safety.rs +215 -0
  423. package/crates/headroom-core/src/transforms/search_compressor.rs +902 -0
  424. package/crates/headroom-core/src/transforms/smart_crusher/analyzer.rs +1262 -0
  425. package/crates/headroom-core/src/transforms/smart_crusher/anchors.rs +407 -0
  426. package/crates/headroom-core/src/transforms/smart_crusher/builder.rs +263 -0
  427. package/crates/headroom-core/src/transforms/smart_crusher/classifier.rs +214 -0
  428. package/crates/headroom-core/src/transforms/smart_crusher/compaction/classifier.rs +283 -0
  429. package/crates/headroom-core/src/transforms/smart_crusher/compaction/compactor.rs +733 -0
  430. package/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rs +961 -0
  431. package/crates/headroom-core/src/transforms/smart_crusher/compaction/ir.rs +252 -0
  432. package/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs +127 -0
  433. package/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs +397 -0
  434. package/crates/headroom-core/src/transforms/smart_crusher/config.rs +173 -0
  435. package/crates/headroom-core/src/transforms/smart_crusher/constraints.rs +159 -0
  436. package/crates/headroom-core/src/transforms/smart_crusher/crusher.rs +1665 -0
  437. package/crates/headroom-core/src/transforms/smart_crusher/crushers.rs +865 -0
  438. package/crates/headroom-core/src/transforms/smart_crusher/error_keywords.rs +77 -0
  439. package/crates/headroom-core/src/transforms/smart_crusher/field_detect.rs +393 -0
  440. package/crates/headroom-core/src/transforms/smart_crusher/hashing.rs +72 -0
  441. package/crates/headroom-core/src/transforms/smart_crusher/mod.rs +81 -0
  442. package/crates/headroom-core/src/transforms/smart_crusher/observer.rs +66 -0
  443. package/crates/headroom-core/src/transforms/smart_crusher/orchestration.rs +459 -0
  444. package/crates/headroom-core/src/transforms/smart_crusher/outliers.rs +507 -0
  445. package/crates/headroom-core/src/transforms/smart_crusher/planning.rs +904 -0
  446. package/crates/headroom-core/src/transforms/smart_crusher/statistics.rs +510 -0
  447. package/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs +287 -0
  448. package/crates/headroom-core/src/transforms/smart_crusher/traits.rs +203 -0
  449. package/crates/headroom-core/src/transforms/smart_crusher/types.rs +263 -0
  450. package/crates/headroom-core/src/transforms/tag_protector.rs +1272 -0
  451. package/crates/headroom-core/src/transforms/unidiff_detector.rs +232 -0
  452. package/crates/headroom-parity/Cargo.toml +26 -0
  453. package/crates/headroom-parity/examples/diff_fixture.rs +66 -0
  454. package/crates/headroom-parity/src/bin/parity_run.rs +78 -0
  455. package/crates/headroom-parity/src/lib.rs +613 -0
  456. package/crates/headroom-proxy/Cargo.toml +119 -0
  457. package/crates/headroom-proxy/data/model_prices_and_context_window.json +39767 -0
  458. package/crates/headroom-proxy/src/bedrock/auth_mode_layer.rs +192 -0
  459. package/crates/headroom-proxy/src/bedrock/envelope.rs +239 -0
  460. package/crates/headroom-proxy/src/bedrock/eventstream.rs +769 -0
  461. package/crates/headroom-proxy/src/bedrock/eventstream_to_sse.rs +459 -0
  462. package/crates/headroom-proxy/src/bedrock/invoke.rs +735 -0
  463. package/crates/headroom-proxy/src/bedrock/invoke_streaming.rs +1098 -0
  464. package/crates/headroom-proxy/src/bedrock/mod.rs +66 -0
  465. package/crates/headroom-proxy/src/bedrock/sigv4.rs +324 -0
  466. package/crates/headroom-proxy/src/bedrock/vendor.rs +82 -0
  467. package/crates/headroom-proxy/src/cache_stabilization/anthropic_cache_control.rs +698 -0
  468. package/crates/headroom-proxy/src/cache_stabilization/drift_detector.rs +683 -0
  469. package/crates/headroom-proxy/src/cache_stabilization/mod.rs +61 -0
  470. package/crates/headroom-proxy/src/cache_stabilization/openai_cache_key.rs +600 -0
  471. package/crates/headroom-proxy/src/cache_stabilization/tool_def_normalize.rs +514 -0
  472. package/crates/headroom-proxy/src/cache_stabilization/volatile_detector.rs +720 -0
  473. package/crates/headroom-proxy/src/compression/anthropic.rs +92 -0
  474. package/crates/headroom-proxy/src/compression/live_zone_anthropic.rs +1311 -0
  475. package/crates/headroom-proxy/src/compression/live_zone_openai.rs +660 -0
  476. package/crates/headroom-proxy/src/compression/live_zone_responses.rs +678 -0
  477. package/crates/headroom-proxy/src/compression/mod.rs +127 -0
  478. package/crates/headroom-proxy/src/compression/model_limits.rs +211 -0
  479. package/crates/headroom-proxy/src/config.rs +646 -0
  480. package/crates/headroom-proxy/src/error.rs +69 -0
  481. package/crates/headroom-proxy/src/handlers/chat_completions.rs +100 -0
  482. package/crates/headroom-proxy/src/handlers/conversations.rs +241 -0
  483. package/crates/headroom-proxy/src/handlers/mod.rs +13 -0
  484. package/crates/headroom-proxy/src/handlers/responses.rs +256 -0
  485. package/crates/headroom-proxy/src/headers.rs +288 -0
  486. package/crates/headroom-proxy/src/health.rs +41 -0
  487. package/crates/headroom-proxy/src/lib.rs +21 -0
  488. package/crates/headroom-proxy/src/main.rs +141 -0
  489. package/crates/headroom-proxy/src/observability/cache_hit_rate.rs +340 -0
  490. package/crates/headroom-proxy/src/observability/compression_ratio.rs +158 -0
  491. package/crates/headroom-proxy/src/observability/metric_names.rs +174 -0
  492. package/crates/headroom-proxy/src/observability/mod.rs +68 -0
  493. package/crates/headroom-proxy/src/observability/prometheus.rs +394 -0
  494. package/crates/headroom-proxy/src/observability/proxy_metrics.rs +368 -0
  495. package/crates/headroom-proxy/src/proxy.rs +1619 -0
  496. package/crates/headroom-proxy/src/responses_items.rs +565 -0
  497. package/crates/headroom-proxy/src/sse/anthropic.rs +415 -0
  498. package/crates/headroom-proxy/src/sse/framing.rs +366 -0
  499. package/crates/headroom-proxy/src/sse/mod.rs +40 -0
  500. package/crates/headroom-proxy/src/sse/openai_chat.rs +231 -0
  501. package/crates/headroom-proxy/src/sse/openai_responses.rs +458 -0
  502. package/crates/headroom-proxy/src/vertex/adc.rs +315 -0
  503. package/crates/headroom-proxy/src/vertex/envelope.rs +167 -0
  504. package/crates/headroom-proxy/src/vertex/mod.rs +277 -0
  505. package/crates/headroom-proxy/src/vertex/raw_predict.rs +490 -0
  506. package/crates/headroom-proxy/src/vertex/stream_raw_predict.rs +49 -0
  507. package/crates/headroom-proxy/src/websocket.rs +246 -0
  508. package/crates/headroom-py/Cargo.toml +37 -0
  509. package/crates/headroom-py/build.rs +77 -0
  510. package/crates/headroom-py/glibc_compat.c +170 -0
  511. package/crates/headroom-py/src/lib.rs +1655 -0
  512. package/deny.toml +32 -0
  513. package/design-system/base.css +127 -0
  514. package/design-system/components.css +1943 -0
  515. package/design-system/fonts.css +17 -0
  516. package/design-system/tailwind-preset.js +95 -0
  517. package/design-system/tokens.css +48 -0
  518. package/design-system/utilities.css +53 -0
  519. package/docker/differential-network-capture/Dockerfile.runner +14 -0
  520. package/docker/differential-network-capture/docker-compose.yml +142 -0
  521. package/docker/differential-network-capture/run-claude-lane.sh +24 -0
  522. package/docker/docker-compose.native.yml +44 -0
  523. package/docker-bake.hcl +88 -0
  524. package/docker-compose.yml +80 -0
  525. package/e2e/_lib/make_shim.ps1 +24 -0
  526. package/e2e/_lib/make_shim.sh +28 -0
  527. package/e2e/docker-native-install.sh +120 -0
  528. package/e2e/init/Dockerfile +79 -0
  529. package/e2e/wrap/Dockerfile +115 -0
  530. package/examples/07-context-compression.ipynb +279 -0
  531. package/examples/deployment/macos-launchagent/com.headroom.proxy.plist.template +65 -0
  532. package/examples/deployment/macos-launchagent/install.sh +227 -0
  533. package/examples/deployment/macos-launchagent/shell-integration.sh +74 -0
  534. package/examples/deployment/macos-launchagent/uninstall.sh +145 -0
  535. package/headroom-savings.png +0 -0
  536. package/headroom_learn.gif +0 -0
  537. package/lib/config/index.ts +12 -0
  538. package/lib/hooks/index.ts +4 -0
  539. package/lib/hooks/use-mounted.ts +13 -0
  540. package/lib/hooks/use-session.ts +34 -0
  541. package/lib/hooks/use-settings.ts +49 -0
  542. package/lib/utils/index.ts +1 -0
  543. package/lib/utils/settings.ts +162 -0
  544. package/lib/utils/utils.ts +6 -0
  545. package/llms.txt +67 -0
  546. package/middleware.ts +26 -0
  547. package/mkdocs.yml +136 -0
  548. package/next-env.d.ts +7 -0
  549. package/next.config.ts +12 -0
  550. package/npm/bin/xithead.js +366 -366
  551. package/npm/bin/xtream-head.js +366 -0
  552. package/npm/index.d.ts +57 -57
  553. package/npm/index.js +139 -139
  554. package/npm/package.json +55 -54
  555. package/npm/postinstall.js +43 -43
  556. package/package.json +58 -36
  557. package/plugins/headroom-agent-hooks/.claude-plugin/plugin.json +17 -0
  558. package/plugins/headroom-agent-hooks/hooks/hooks.json +29 -0
  559. package/plugins/headroom-oauth2/LICENSE +3 -0
  560. package/plugins/headroom-oauth2/SPEC.md +115 -0
  561. package/plugins/headroom-oauth2/pyproject.toml +48 -0
  562. package/plugins/hermes/headroom_retrieve/plugin.yaml +6 -0
  563. package/plugins/openclaw/hook-shim/HOOK.md +21 -0
  564. package/plugins/openclaw/hook-shim/handler.js +3 -0
  565. package/plugins/openclaw/openclaw.plugin.json +98 -0
  566. package/plugins/openclaw/package-lock.json +2692 -0
  567. package/plugins/openclaw/package.json +54 -0
  568. package/plugins/openclaw/prepare-dist.mjs +56 -0
  569. package/plugins/openclaw/src/convert.ts +418 -0
  570. package/plugins/openclaw/src/engine.ts +284 -0
  571. package/plugins/openclaw/src/gateway-config.ts +143 -0
  572. package/plugins/openclaw/src/index.ts +11 -0
  573. package/plugins/openclaw/src/plugin/index.ts +98 -0
  574. package/plugins/openclaw/src/proxy-manager.ts +477 -0
  575. package/plugins/openclaw/src/tools/headroom-retrieve.ts +76 -0
  576. package/plugins/openclaw/test/convert.test.ts +96 -0
  577. package/plugins/openclaw/test/engine-normalization.test.ts +21 -0
  578. package/plugins/openclaw/test/engine.test.ts +101 -0
  579. package/plugins/openclaw/test/gateway-config.test.ts +278 -0
  580. package/plugins/openclaw/test/plugin-runtime-routing.test.ts +141 -0
  581. package/plugins/openclaw/test/proxy-manager.test.ts +343 -0
  582. package/plugins/openclaw/tsup.config.ts +9 -0
  583. package/postcss.config.mjs +7 -0
  584. package/pyproject.toml +438 -0
  585. package/rust-toolchain.toml +16 -0
  586. package/sdk/typescript/examples/basic-compress.ts +64 -0
  587. package/sdk/typescript/examples/ccr-retrieve.ts +80 -0
  588. package/sdk/typescript/examples/hooks-custom-compression.ts +117 -0
  589. package/sdk/typescript/examples/middleware-composition.ts +59 -0
  590. package/sdk/typescript/examples/multi-provider.ts +53 -0
  591. package/sdk/typescript/examples/openai-anthropic-adapters.ts +73 -0
  592. package/sdk/typescript/examples/shared-context-multi-agent.ts +74 -0
  593. package/sdk/typescript/examples/simulation-dry-run.ts +80 -0
  594. package/sdk/typescript/examples/streaming-chat.ts +71 -0
  595. package/sdk/typescript/examples/structured-output.ts +99 -0
  596. package/sdk/typescript/examples/tool-calling-agent.ts +86 -0
  597. package/sdk/typescript/examples/with-headroom-vercel.ts +57 -0
  598. package/sdk/typescript/package-lock.json +3437 -0
  599. package/sdk/typescript/package.json +108 -0
  600. package/sdk/typescript/src/adapters/anthropic.ts +218 -0
  601. package/sdk/typescript/src/adapters/gemini.ts +83 -0
  602. package/sdk/typescript/src/adapters/openai.ts +77 -0
  603. package/sdk/typescript/src/adapters/vercel-ai.ts +131 -0
  604. package/sdk/typescript/src/client.ts +632 -0
  605. package/sdk/typescript/src/compress.ts +87 -0
  606. package/sdk/typescript/src/errors.ts +114 -0
  607. package/sdk/typescript/src/hooks.ts +113 -0
  608. package/sdk/typescript/src/index.ts +140 -0
  609. package/sdk/typescript/src/paths.ts +272 -0
  610. package/sdk/typescript/src/shared-context.ts +188 -0
  611. package/sdk/typescript/src/simulate.ts +52 -0
  612. package/sdk/typescript/src/types/config.ts +183 -0
  613. package/sdk/typescript/src/types/models.ts +314 -0
  614. package/sdk/typescript/src/types.ts +139 -0
  615. package/sdk/typescript/src/utils/case.ts +34 -0
  616. package/sdk/typescript/src/utils/format.ts +480 -0
  617. package/sdk/typescript/src/utils/stream.ts +50 -0
  618. package/sdk/typescript/test/adapters/anthropic.test.ts +217 -0
  619. package/sdk/typescript/test/adapters/gemini.test.ts +131 -0
  620. package/sdk/typescript/test/adapters/openai.test.ts +174 -0
  621. package/sdk/typescript/test/adapters/vercel-ai.test.ts +306 -0
  622. package/sdk/typescript/test/client-expanded.test.ts +333 -0
  623. package/sdk/typescript/test/client.test.ts +266 -0
  624. package/sdk/typescript/test/compress.test.ts +118 -0
  625. package/sdk/typescript/test/config.test.ts +275 -0
  626. package/sdk/typescript/test/errors.test.ts +169 -0
  627. package/sdk/typescript/test/format-integration.test.ts +252 -0
  628. package/sdk/typescript/test/hooks.test.ts +265 -0
  629. package/sdk/typescript/test/integration.test.ts +249 -0
  630. package/sdk/typescript/test/mdx-examples.test.ts +253 -0
  631. package/sdk/typescript/test/models.test.ts +318 -0
  632. package/sdk/typescript/test/parity.test.ts +337 -0
  633. package/sdk/typescript/test/paths.test.ts +447 -0
  634. package/sdk/typescript/test/shared-context.test.ts +245 -0
  635. package/sdk/typescript/test/simulate.test.ts +84 -0
  636. package/sdk/typescript/test/types.test.ts +244 -0
  637. package/sdk/typescript/test/utils/case.test.ts +129 -0
  638. package/sdk/typescript/test/utils/detect-format.test.ts +133 -0
  639. package/sdk/typescript/test/utils/format.test.ts +434 -0
  640. package/sdk/typescript/test/vercel-ai-e2e.test.ts +179 -0
  641. package/sdk/typescript/tsup.config.ts +17 -0
  642. package/tailwind.master.config.cjs +47 -0
  643. package/types/api.ts +12 -0
  644. package/types/config.ts +9 -0
  645. package/types/index.ts +2 -0
  646. package/uv.lock +6951 -0
  647. package/xithead.egg-info/PKG-INFO +22 -0
  648. package/xithead.egg-info/SOURCES.txt +16 -0
  649. package/xithead.egg-info/dependency_links.txt +1 -0
  650. package/xithead.egg-info/entry_points.txt +2 -0
  651. package/xithead.egg-info/requires.txt +10 -0
  652. package/xithead.egg-info/top_level.txt +2 -0
  653. package/xithead.ps1 +136 -0
  654. package/xithead.sh +129 -0
  655. package/xithead_dist/pyproject.toml +39 -0
  656. package/xithead_dist/xithead.egg-info/PKG-INFO +22 -0
  657. package/xithead_dist/xithead.egg-info/SOURCES.txt +16 -0
  658. package/xithead_dist/xithead.egg-info/dependency_links.txt +1 -0
  659. package/xithead_dist/xithead.egg-info/entry_points.txt +3 -0
  660. package/xithead_dist/xithead.egg-info/requires.txt +10 -0
  661. package/xithead_dist/xithead.egg-info/top_level.txt +2 -0
@@ -0,0 +1,961 @@
1
+ //! Formatter trait + the built-in implementations.
2
+ //!
3
+ //! [`Formatter`] walks a [`Compaction`] tree and renders bytes. It's the
4
+ //! pluggable seam where users (or Enterprise plugins) choose how the
5
+ //! compacted output looks.
6
+ //!
7
+ //! # Built-ins
8
+ //!
9
+ //! - [`JsonFormatter`] — single-line / pretty JSON. Easy to parse,
10
+ //! wider model familiarity, larger byte size. Default for the
11
+ //! debugging path.
12
+ //! - [`CsvSchemaFormatter`] — `[N]{cols}` row-count-and-shape
13
+ //! declaration + typed column header + CSV-escaped rows. Steals
14
+ //! TOON's most useful idea (the `[N]{cols}` declaration) without
15
+ //! adopting TOON's bespoke escaping rules — every model has seen
16
+ //! millions of CSV examples in training.
17
+ //! - [`MarkdownKvFormatter`] — the same `[N]{cols}` declaration +
18
+ //! one Markdown list item per row with `key: value` lines.
19
+ //! Token-heavier than CSV (field names repeat per row) but
20
+ //! format-comprehension benchmarks favor KV for read-back accuracy.
21
+ //!
22
+ //! # Nested cells
23
+ //!
24
+ //! The formatters handle [`CellValue::Nested`] by recursively
25
+ //! formatting the sub-compaction and embedding the result. The CSV
26
+ //! formatter wraps nested output in CSV-quoted form; the JSON
27
+ //! formatter embeds it as a structured JSON object.
28
+ //!
29
+ //! # Opaque cells
30
+ //!
31
+ //! [`CellValue::OpaqueRef`] renders as a structured marker the model
32
+ //! can recognize: `<<ccr:HASH,KIND,SIZE>>`. This format is fixed across
33
+ //! all built-in formatters so downstream consumers can pattern-match
34
+ //! markers regardless of which formatter produced them.
35
+
36
+ use serde_json::{json, Value};
37
+
38
+ use super::ir::{CellValue, Compaction, OpaqueKind, Row, Schema};
39
+
40
+ /// Format a `Compaction` tree into bytes.
41
+ pub trait Formatter: Send + Sync {
42
+ /// Stable name for telemetry (e.g. `"json"`, `"csv-schema"`).
43
+ fn name(&self) -> &str;
44
+
45
+ /// Render the compaction. Implementations should be deterministic
46
+ /// for stable test parity.
47
+ fn format(&self, c: &Compaction) -> String;
48
+
49
+ /// Cheap byte-size estimate. Default impl renders and measures.
50
+ /// Override for cases where rendering is expensive.
51
+ fn estimate_bytes(&self, c: &Compaction) -> usize {
52
+ self.format(c).len()
53
+ }
54
+ }
55
+
56
+ // ─────────────────────────── JSON formatter ───────────────────────────
57
+
58
+ /// Renders a `Compaction` as structured JSON. Single-line by default
59
+ /// for token-tight output; set `pretty = true` for human inspection.
60
+ #[derive(Debug, Clone, Default)]
61
+ pub struct JsonFormatter {
62
+ pub pretty: bool,
63
+ }
64
+
65
+ impl JsonFormatter {
66
+ pub fn new() -> Self {
67
+ Self::default()
68
+ }
69
+ pub fn pretty(mut self) -> Self {
70
+ self.pretty = true;
71
+ self
72
+ }
73
+ }
74
+
75
+ impl Formatter for JsonFormatter {
76
+ fn name(&self) -> &str {
77
+ "json"
78
+ }
79
+
80
+ fn format(&self, c: &Compaction) -> String {
81
+ let v = compaction_to_json(c);
82
+ if self.pretty {
83
+ serde_json::to_string_pretty(&v).unwrap_or_default()
84
+ } else {
85
+ serde_json::to_string(&v).unwrap_or_default()
86
+ }
87
+ }
88
+ }
89
+
90
+ fn compaction_to_json(c: &Compaction) -> Value {
91
+ match c {
92
+ Compaction::Table {
93
+ schema,
94
+ rows,
95
+ original_count,
96
+ } => json!({
97
+ "_compaction": "table",
98
+ "_schema": schema_to_json(schema),
99
+ "_kept": rows.len(),
100
+ "_total": original_count,
101
+ "_rows": rows.iter().map(row_to_json).collect::<Vec<_>>(),
102
+ }),
103
+ Compaction::Buckets {
104
+ discriminator,
105
+ buckets,
106
+ original_count,
107
+ } => json!({
108
+ "_compaction": "buckets",
109
+ "_discriminator": discriminator,
110
+ "_total": original_count,
111
+ "_buckets": buckets
112
+ .iter()
113
+ .map(|b| json!({
114
+ "_key": b.key.clone(),
115
+ "_schema": schema_to_json(&b.schema),
116
+ "_rows": b.rows.iter().map(row_to_json).collect::<Vec<_>>(),
117
+ }))
118
+ .collect::<Vec<_>>(),
119
+ }),
120
+ Compaction::OpaqueRef {
121
+ ccr_hash,
122
+ byte_size,
123
+ kind,
124
+ } => json!({
125
+ "_compaction": "ccr",
126
+ "_hash": ccr_hash,
127
+ "_size": byte_size,
128
+ "_kind": opaque_kind_str(kind),
129
+ }),
130
+ Compaction::Untouched(v) => v.clone(),
131
+ }
132
+ }
133
+
134
+ fn schema_to_json(s: &Schema) -> Value {
135
+ Value::Array(
136
+ s.fields
137
+ .iter()
138
+ .map(|f| {
139
+ let mut obj = serde_json::Map::new();
140
+ obj.insert("name".into(), Value::String(f.name.clone()));
141
+ obj.insert("type".into(), Value::String(f.type_tag.clone()));
142
+ if f.nullable {
143
+ obj.insert("nullable".into(), Value::Bool(true));
144
+ }
145
+ Value::Object(obj)
146
+ })
147
+ .collect(),
148
+ )
149
+ }
150
+
151
+ fn row_to_json(row: &Row) -> Value {
152
+ Value::Array(row.0.iter().map(cell_to_json).collect())
153
+ }
154
+
155
+ fn cell_to_json(c: &CellValue) -> Value {
156
+ match c {
157
+ CellValue::Scalar(v) => v.clone(),
158
+ CellValue::Missing => Value::Null,
159
+ CellValue::Nested(sub) => compaction_to_json(sub),
160
+ CellValue::OpaqueRef {
161
+ ccr_hash,
162
+ byte_size,
163
+ kind,
164
+ } => json!({
165
+ "_ccr": ccr_hash,
166
+ "_size": byte_size,
167
+ "_kind": opaque_kind_str(kind),
168
+ }),
169
+ }
170
+ }
171
+
172
+ fn opaque_kind_str(k: &OpaqueKind) -> String {
173
+ match k {
174
+ OpaqueKind::Base64Blob => "base64".into(),
175
+ OpaqueKind::LongString => "string".into(),
176
+ OpaqueKind::HtmlChunk => "html".into(),
177
+ OpaqueKind::Other(s) => s.clone(),
178
+ }
179
+ }
180
+
181
+ // ─────────────────────────── CSV+schema formatter ───────────────────────────
182
+
183
+ /// Renders a `Compaction` as `[N]{col1:type1,col2:type2}` declaration +
184
+ /// CSV-escaped rows. Nested cells render as JSON inline; opaque cells
185
+ /// render as `<<ccr:...>>` markers.
186
+ #[derive(Debug, Clone, Default)]
187
+ pub struct CsvSchemaFormatter {
188
+ /// If true, emit a `__total:N` line when rows were dropped under
189
+ /// budget. Costs a few bytes; useful for downstream telemetry.
190
+ pub include_drop_summary: bool,
191
+ }
192
+
193
+ impl CsvSchemaFormatter {
194
+ pub fn new() -> Self {
195
+ Self::default()
196
+ }
197
+ pub fn with_drop_summary(mut self) -> Self {
198
+ self.include_drop_summary = true;
199
+ self
200
+ }
201
+ }
202
+
203
+ impl Formatter for CsvSchemaFormatter {
204
+ fn name(&self) -> &str {
205
+ "csv-schema"
206
+ }
207
+
208
+ fn format(&self, c: &Compaction) -> String {
209
+ let mut out = String::new();
210
+ write_compaction(&mut out, c, self);
211
+ out
212
+ }
213
+ }
214
+
215
+ fn write_compaction(out: &mut String, c: &Compaction, fmt: &CsvSchemaFormatter) {
216
+ match c {
217
+ Compaction::Table {
218
+ schema,
219
+ rows,
220
+ original_count,
221
+ } => {
222
+ write_table(out, schema, rows, *original_count, fmt);
223
+ }
224
+ Compaction::Buckets {
225
+ discriminator,
226
+ buckets,
227
+ original_count,
228
+ } => {
229
+ out.push_str("__buckets:");
230
+ out.push_str(discriminator);
231
+ if fmt.include_drop_summary {
232
+ let kept: usize = buckets.iter().map(|b| b.rows.len()).sum();
233
+ if kept < *original_count {
234
+ out.push_str(&format!(" __dropped:{}", original_count - kept));
235
+ }
236
+ }
237
+ out.push('\n');
238
+ for b in buckets {
239
+ out.push_str(&format!("__key:{}\n", json_scalar_to_csv(&b.key)));
240
+ write_table(out, &b.schema, &b.rows, b.rows.len(), fmt);
241
+ }
242
+ }
243
+ Compaction::OpaqueRef {
244
+ ccr_hash,
245
+ byte_size,
246
+ kind,
247
+ } => {
248
+ out.push_str(&format_ccr_marker(ccr_hash, *byte_size, kind));
249
+ }
250
+ Compaction::Untouched(v) => {
251
+ out.push_str(&serde_json::to_string(v).unwrap_or_default());
252
+ }
253
+ }
254
+ }
255
+
256
+ fn write_table(
257
+ out: &mut String,
258
+ schema: &Schema,
259
+ rows: &[Row],
260
+ original_count: usize,
261
+ fmt: &CsvSchemaFormatter,
262
+ ) {
263
+ // Declaration line: [N]{col:type,col:type,...}
264
+ out.push('[');
265
+ out.push_str(&rows.len().to_string());
266
+ out.push_str("]{");
267
+ let col_decl: Vec<String> = schema
268
+ .fields
269
+ .iter()
270
+ .map(|f| {
271
+ if f.nullable {
272
+ format!("{}:{}?", f.name, f.type_tag)
273
+ } else {
274
+ format!("{}:{}", f.name, f.type_tag)
275
+ }
276
+ })
277
+ .collect();
278
+ out.push_str(&col_decl.join(","));
279
+ out.push('}');
280
+ if fmt.include_drop_summary && rows.len() < original_count {
281
+ out.push_str(&format!(" __dropped:{}", original_count - rows.len()));
282
+ }
283
+ out.push('\n');
284
+
285
+ // Rows.
286
+ for row in rows {
287
+ let cells: Vec<String> = row.0.iter().map(format_cell).collect();
288
+ out.push_str(&cells.join(","));
289
+ out.push('\n');
290
+ }
291
+ }
292
+
293
+ fn format_cell(c: &CellValue) -> String {
294
+ match c {
295
+ CellValue::Missing => String::new(),
296
+ CellValue::Scalar(v) => json_scalar_to_csv(v),
297
+ CellValue::Nested(sub) => {
298
+ // Render nested as compact JSON; CSV-quote because it
299
+ // contains commas and structural chars.
300
+ let nested_fmt = JsonFormatter::new();
301
+ csv_quote(&nested_fmt.format(sub))
302
+ }
303
+ CellValue::OpaqueRef {
304
+ ccr_hash,
305
+ byte_size,
306
+ kind,
307
+ } => format_ccr_marker(ccr_hash, *byte_size, kind),
308
+ }
309
+ }
310
+
311
+ fn format_ccr_marker(hash: &str, byte_size: usize, kind: &OpaqueKind) -> String {
312
+ let kind_str = match kind {
313
+ OpaqueKind::Base64Blob => "base64",
314
+ OpaqueKind::LongString => "string",
315
+ OpaqueKind::HtmlChunk => "html",
316
+ OpaqueKind::Other(s) => s.as_str(),
317
+ };
318
+ format!(
319
+ "<<ccr:{},{},{}>>",
320
+ hash,
321
+ kind_str,
322
+ humanize_bytes(byte_size)
323
+ )
324
+ }
325
+
326
+ fn humanize_bytes(n: usize) -> String {
327
+ if n < 1024 {
328
+ return format!("{n}B");
329
+ }
330
+ let kb = n as f64 / 1024.0;
331
+ if kb < 1024.0 {
332
+ return format!("{kb:.1}KB");
333
+ }
334
+ let mb = kb / 1024.0;
335
+ format!("{mb:.1}MB")
336
+ }
337
+
338
+ fn json_scalar_to_csv(v: &Value) -> String {
339
+ match v {
340
+ Value::Null => String::new(),
341
+ Value::Bool(b) => if *b { "true" } else { "false" }.to_string(),
342
+ Value::Number(n) => n.to_string(),
343
+ Value::String(s) => {
344
+ if needs_csv_quote(s) {
345
+ csv_quote(s)
346
+ } else {
347
+ s.clone()
348
+ }
349
+ }
350
+ // Object/array fall back to JSON-quoted (rare — usually
351
+ // already promoted to Nested by the compactor).
352
+ _ => csv_quote(&serde_json::to_string(v).unwrap_or_default()),
353
+ }
354
+ }
355
+
356
+ fn needs_csv_quote(s: &str) -> bool {
357
+ s.contains(',') || s.contains('"') || s.contains('\n') || s.contains('\r')
358
+ }
359
+
360
+ fn csv_quote(s: &str) -> String {
361
+ let mut out = String::with_capacity(s.len() + 2);
362
+ out.push('"');
363
+ for c in s.chars() {
364
+ if c == '"' {
365
+ out.push('"');
366
+ out.push('"');
367
+ } else {
368
+ out.push(c);
369
+ }
370
+ }
371
+ out.push('"');
372
+ out
373
+ }
374
+
375
+ // ─────────────────────────── Markdown-KV formatter ───────────────────────────
376
+
377
+ /// Renders a `Compaction` as a `[N]{cols}` declaration followed by one
378
+ /// Markdown list item per row, each cell on its own `key: value` line.
379
+ ///
380
+ /// Token-heavier than [`CsvSchemaFormatter`] (field names repeat per
381
+ /// row), but format-comprehension benchmarks show models retrieve
382
+ /// values from Markdown-KV substantially more reliably than from CSV.
383
+ /// Offered as an opt-in trade of tokens for read accuracy.
384
+ ///
385
+ /// Rendering rules:
386
+ /// - Missing cells are omitted entirely (no `key:` line) — sparse rows
387
+ /// cost nothing, unlike CSV's positional empty cells.
388
+ /// - Strings that would be ambiguous on a line (contain newlines,
389
+ /// leading/trailing whitespace, or are empty) render JSON-quoted;
390
+ /// everything else renders raw.
391
+ /// - Nested cells render as compact inline JSON, matching
392
+ /// [`CsvSchemaFormatter`].
393
+ /// - Opaque cells keep the fixed `<<ccr:HASH,KIND,SIZE>>` marker
394
+ /// contract shared by all formatters.
395
+ #[derive(Debug, Clone, Default)]
396
+ pub struct MarkdownKvFormatter {
397
+ /// If true, emit a `__dropped:N` note on the declaration line when
398
+ /// rows were dropped under budget. Mirrors
399
+ /// [`CsvSchemaFormatter::include_drop_summary`].
400
+ pub include_drop_summary: bool,
401
+ }
402
+
403
+ impl MarkdownKvFormatter {
404
+ pub fn new() -> Self {
405
+ Self::default()
406
+ }
407
+ pub fn with_drop_summary(mut self) -> Self {
408
+ self.include_drop_summary = true;
409
+ self
410
+ }
411
+ }
412
+
413
+ impl Formatter for MarkdownKvFormatter {
414
+ fn name(&self) -> &str {
415
+ "markdown-kv"
416
+ }
417
+
418
+ fn format(&self, c: &Compaction) -> String {
419
+ let mut out = String::new();
420
+ write_compaction_kv(&mut out, c, self);
421
+ out
422
+ }
423
+ }
424
+
425
+ fn write_compaction_kv(out: &mut String, c: &Compaction, fmt: &MarkdownKvFormatter) {
426
+ match c {
427
+ Compaction::Table {
428
+ schema,
429
+ rows,
430
+ original_count,
431
+ } => {
432
+ write_kv_table(out, schema, rows, *original_count, fmt);
433
+ }
434
+ Compaction::Buckets {
435
+ discriminator,
436
+ buckets,
437
+ original_count,
438
+ } => {
439
+ out.push_str("__buckets:");
440
+ out.push_str(discriminator);
441
+ if fmt.include_drop_summary {
442
+ let kept: usize = buckets.iter().map(|b| b.rows.len()).sum();
443
+ if kept < *original_count {
444
+ out.push_str(&format!(" __dropped:{}", original_count - kept));
445
+ }
446
+ }
447
+ out.push('\n');
448
+ for b in buckets {
449
+ out.push_str(&format!("__key:{}\n", kv_scalar(&b.key)));
450
+ write_kv_table(out, &b.schema, &b.rows, b.rows.len(), fmt);
451
+ }
452
+ }
453
+ Compaction::OpaqueRef {
454
+ ccr_hash,
455
+ byte_size,
456
+ kind,
457
+ } => {
458
+ out.push_str(&format_ccr_marker(ccr_hash, *byte_size, kind));
459
+ }
460
+ Compaction::Untouched(v) => {
461
+ out.push_str(&serde_json::to_string(v).unwrap_or_default());
462
+ }
463
+ }
464
+ }
465
+
466
+ fn write_kv_table(
467
+ out: &mut String,
468
+ schema: &Schema,
469
+ rows: &[Row],
470
+ original_count: usize,
471
+ fmt: &MarkdownKvFormatter,
472
+ ) {
473
+ // Same declaration line as the CSV formatter: keeps row count and
474
+ // typed shape up front where the model (and telemetry) expect it.
475
+ // Unlike CSV (pre-existing exposure, kept byte-identical), KV quotes
476
+ // pathological field names here so the declaration parses the same
477
+ // way as the row lines below.
478
+ out.push('[');
479
+ out.push_str(&rows.len().to_string());
480
+ out.push_str("]{");
481
+ let col_decl: Vec<String> = schema
482
+ .fields
483
+ .iter()
484
+ .map(|f| {
485
+ let name = kv_field_name(&f.name);
486
+ if f.nullable {
487
+ format!("{}:{}?", name, f.type_tag)
488
+ } else {
489
+ format!("{}:{}", name, f.type_tag)
490
+ }
491
+ })
492
+ .collect();
493
+ out.push_str(&col_decl.join(","));
494
+ out.push('}');
495
+ if fmt.include_drop_summary && rows.len() < original_count {
496
+ out.push_str(&format!(" __dropped:{}", original_count - rows.len()));
497
+ }
498
+ out.push('\n');
499
+
500
+ for row in rows {
501
+ // Compactor invariant: one cell per schema field. zip() would
502
+ // silently drop extras — fail loudly in debug builds instead.
503
+ debug_assert_eq!(row.0.len(), schema.fields.len());
504
+ let mut wrote_first = false;
505
+ for (field, cell) in schema.fields.iter().zip(row.0.iter()) {
506
+ let rendered = match cell {
507
+ CellValue::Missing => continue,
508
+ CellValue::Scalar(v) => kv_scalar(v),
509
+ CellValue::Nested(sub) => JsonFormatter::new().format(sub),
510
+ CellValue::OpaqueRef {
511
+ ccr_hash,
512
+ byte_size,
513
+ kind,
514
+ } => format_ccr_marker(ccr_hash, *byte_size, kind),
515
+ };
516
+ out.push_str(if wrote_first { " " } else { "- " });
517
+ out.push_str(&kv_field_name(&field.name));
518
+ out.push_str(": ");
519
+ out.push_str(&rendered);
520
+ out.push('\n');
521
+ wrote_first = true;
522
+ }
523
+ // All-missing row: keep a bare list item so the rendered row
524
+ // count still matches the declaration.
525
+ if !wrote_first {
526
+ out.push_str("-\n");
527
+ }
528
+ }
529
+ }
530
+
531
+ fn kv_scalar(v: &Value) -> String {
532
+ match v {
533
+ Value::Null => "null".to_string(),
534
+ Value::Bool(b) => if *b { "true" } else { "false" }.to_string(),
535
+ Value::Number(n) => n.to_string(),
536
+ Value::String(s) => {
537
+ if needs_kv_quote(s) {
538
+ serde_json::to_string(s).unwrap_or_default()
539
+ } else {
540
+ s.clone()
541
+ }
542
+ }
543
+ // Object/array fall back to compact JSON (rare — usually
544
+ // already promoted to Nested by the compactor).
545
+ _ => serde_json::to_string(v).unwrap_or_default(),
546
+ }
547
+ }
548
+
549
+ fn needs_kv_quote(s: &str) -> bool {
550
+ s.is_empty()
551
+ || s.contains('\n')
552
+ || s.contains('\r')
553
+ || s.starts_with(char::is_whitespace)
554
+ || s.ends_with(char::is_whitespace)
555
+ }
556
+
557
+ /// Field names are normally bare identifiers, but nothing upstream
558
+ /// enforces that. Quote the pathological ones the same way as values:
559
+ /// an embedded newline would inject fake row lines, and `": "` inside
560
+ /// a key would split the line at the wrong colon on read-back.
561
+ fn kv_field_name(name: &str) -> String {
562
+ if needs_kv_quote(name) || name.contains(": ") {
563
+ serde_json::to_string(name).unwrap_or_default()
564
+ } else {
565
+ name.to_string()
566
+ }
567
+ }
568
+
569
+ #[cfg(test)]
570
+ mod tests {
571
+ use super::*;
572
+ use crate::transforms::smart_crusher::compaction::compactor::compact;
573
+ use crate::transforms::smart_crusher::compaction::compactor::CompactConfig;
574
+ use serde_json::json;
575
+
576
+ fn cfg() -> CompactConfig {
577
+ CompactConfig::default()
578
+ }
579
+
580
+ // ── JsonFormatter ──
581
+
582
+ #[test]
583
+ fn json_formatter_renders_table() {
584
+ let items = vec![
585
+ json!({"id": 1, "name": "alice"}),
586
+ json!({"id": 2, "name": "bob"}),
587
+ ];
588
+ let c = compact(&items, &cfg());
589
+ let out = JsonFormatter::new().format(&c);
590
+ assert!(out.contains("\"_compaction\":\"table\""), "got: {out}");
591
+ assert!(out.contains("\"_kept\":2"));
592
+ assert!(out.contains("alice"));
593
+ }
594
+
595
+ #[test]
596
+ fn json_formatter_renders_untouched_verbatim() {
597
+ let c = Compaction::Untouched(json!({"a": 1, "b": [2, 3]}));
598
+ let out = JsonFormatter::new().format(&c);
599
+ assert_eq!(out, r#"{"a":1,"b":[2,3]}"#);
600
+ }
601
+
602
+ #[test]
603
+ fn json_formatter_renders_opaque_ref_marker() {
604
+ let mut row = Row::new(vec![CellValue::OpaqueRef {
605
+ ccr_hash: "abc123def456".into(),
606
+ byte_size: 2048,
607
+ kind: OpaqueKind::Base64Blob,
608
+ }]);
609
+ let c = Compaction::Table {
610
+ schema: Schema {
611
+ fields: vec![super::super::ir::FieldSpec {
612
+ name: "blob".into(),
613
+ type_tag: "ccr".into(),
614
+ nullable: false,
615
+ }],
616
+ },
617
+ rows: vec![std::mem::replace(&mut row, Row::new(vec![]))],
618
+ original_count: 1,
619
+ };
620
+ let out = JsonFormatter::new().format(&c);
621
+ assert!(out.contains("\"_ccr\":\"abc123def456\""));
622
+ assert!(out.contains("base64"));
623
+ }
624
+
625
+ // ── CsvSchemaFormatter ──
626
+
627
+ #[test]
628
+ fn csv_formatter_pure_tabular() {
629
+ let items = vec![
630
+ json!({"id": 1, "name": "alice", "status": "ok"}),
631
+ json!({"id": 2, "name": "bob", "status": "ok"}),
632
+ json!({"id": 3, "name": "carol", "status": "fail"}),
633
+ ];
634
+ let c = compact(&items, &cfg());
635
+ let out = CsvSchemaFormatter::new().format(&c);
636
+ let lines: Vec<&str> = out.trim_end().lines().collect();
637
+ // First line: declaration with [3]{...}
638
+ assert!(lines[0].starts_with("[3]{"), "got line[0]: {}", lines[0]);
639
+ assert!(lines[0].contains("id:int"));
640
+ assert!(lines[0].contains("name:string"));
641
+ assert!(lines[0].contains("status:string"));
642
+ assert_eq!(lines.len(), 4);
643
+ }
644
+
645
+ #[test]
646
+ fn csv_formatter_quotes_strings_with_commas() {
647
+ let items = vec![
648
+ json!({"id": 1, "name": "alice, the great"}),
649
+ json!({"id": 2, "name": "bob"}),
650
+ ];
651
+ let c = compact(&items, &cfg());
652
+ let out = CsvSchemaFormatter::new().format(&c);
653
+ assert!(out.contains(r#""alice, the great""#));
654
+ }
655
+
656
+ #[test]
657
+ fn csv_formatter_escapes_internal_quotes() {
658
+ let items = vec![
659
+ json!({"id": 1, "msg": "she said \"hi\""}),
660
+ json!({"id": 2, "msg": "ok"}),
661
+ ];
662
+ let c = compact(&items, &cfg());
663
+ let out = CsvSchemaFormatter::new().format(&c);
664
+ assert!(out.contains(r#""she said ""hi""""#));
665
+ }
666
+
667
+ #[test]
668
+ fn csv_formatter_renders_buckets() {
669
+ let items = vec![
670
+ json!({"type": "user", "id": 1, "name": "alice"}),
671
+ json!({"type": "user", "id": 2, "name": "bob"}),
672
+ json!({"type": "order", "id": 99, "total": 50}),
673
+ json!({"type": "order", "id": 100, "total": 75}),
674
+ ];
675
+ let c = compact(&items, &cfg());
676
+ let out = CsvSchemaFormatter::new().format(&c);
677
+ assert!(out.starts_with("__buckets:type"));
678
+ assert!(out.contains("__key:order"));
679
+ assert!(out.contains("__key:user"));
680
+ }
681
+
682
+ #[test]
683
+ fn csv_formatter_emits_ccr_marker() {
684
+ let big = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".repeat(8);
685
+ let items = vec![
686
+ json!({"id": 1, "blob": big.clone()}),
687
+ json!({"id": 2, "blob": big.clone()}),
688
+ ];
689
+ let c = compact(&items, &cfg());
690
+ let out = CsvSchemaFormatter::new().format(&c);
691
+ assert!(out.contains("<<ccr:"), "got: {out}");
692
+ assert!(out.contains(",base64,"));
693
+ }
694
+
695
+ #[test]
696
+ fn csv_formatter_nested_cell_inline_json() {
697
+ let items = vec![
698
+ json!({"event": "batch", "payload": r#"[{"x":1},{"x":2},{"x":3}]"#}),
699
+ json!({"event": "batch", "payload": r#"[{"x":4},{"x":5}]"#}),
700
+ ];
701
+ let c = compact(&items, &cfg());
702
+ let out = CsvSchemaFormatter::new().format(&c);
703
+ // Nested compaction is JSON-rendered then CSV-quoted, so a
704
+ // `_compaction":"table"` substring should appear inside quotes.
705
+ assert!(out.contains("_compaction"), "got: {out}");
706
+ }
707
+
708
+ #[test]
709
+ fn csv_formatter_drop_summary_opt_in() {
710
+ let mut rows = vec![Row::new(vec![CellValue::Scalar(json!(1))])];
711
+ rows.push(Row::new(vec![CellValue::Scalar(json!(2))]));
712
+ let c = Compaction::Table {
713
+ schema: Schema {
714
+ fields: vec![super::super::ir::FieldSpec {
715
+ name: "x".into(),
716
+ type_tag: "int".into(),
717
+ nullable: false,
718
+ }],
719
+ },
720
+ rows,
721
+ original_count: 5, // 3 dropped
722
+ };
723
+ let with_summary = CsvSchemaFormatter::new().with_drop_summary().format(&c);
724
+ assert!(with_summary.contains("__dropped:3"));
725
+ let without = CsvSchemaFormatter::new().format(&c);
726
+ assert!(!without.contains("__dropped"));
727
+ }
728
+
729
+ #[test]
730
+ fn estimate_matches_format_len() {
731
+ let items = vec![json!({"a": 1}), json!({"a": 2})];
732
+ let c = compact(&items, &cfg());
733
+ let f = CsvSchemaFormatter::new();
734
+ assert_eq!(f.estimate_bytes(&c), f.format(&c).len());
735
+ }
736
+
737
+ // ── Cross-formatter property: same input → smaller CSV than JSON ──
738
+ // This is the headline value prop. If it doesn't hold for "obviously
739
+ // tabular" input, the formatter is broken or the fixture is wrong.
740
+
741
+ #[test]
742
+ fn csv_smaller_than_json_for_tabular() {
743
+ let items: Vec<Value> = (0..50)
744
+ .map(|i| {
745
+ json!({
746
+ "id": i,
747
+ "name": format!("user_{i}"),
748
+ "email": format!("user_{i}@example.com"),
749
+ "status": if i % 3 == 0 { "ok" } else { "pending" },
750
+ })
751
+ })
752
+ .collect();
753
+ let c = compact(&items, &cfg());
754
+ let json_out = JsonFormatter::new().format(&c);
755
+ let csv_out = CsvSchemaFormatter::new().format(&c);
756
+ // CSV should beat the structured-JSON formatter (both
757
+ // deduplicate the schema, so the win comes from removing
758
+ // structural punctuation only — modest, but real).
759
+ assert!(
760
+ csv_out.len() < json_out.len(),
761
+ "csv {} bytes vs json {} bytes",
762
+ csv_out.len(),
763
+ json_out.len()
764
+ );
765
+ }
766
+
767
+ #[test]
768
+ fn csv_substantially_smaller_than_raw_json() {
769
+ // The headline value prop: CSV+schema beats naïve JSON
770
+ // serialization of the same array (where every row repeats
771
+ // every field name) by a wide margin.
772
+ let items: Vec<Value> = (0..50)
773
+ .map(|i| {
774
+ json!({
775
+ "id": i,
776
+ "name": format!("user_{i}"),
777
+ "email": format!("user_{i}@example.com"),
778
+ "status": if i % 3 == 0 { "ok" } else { "pending" },
779
+ })
780
+ })
781
+ .collect();
782
+ let c = compact(&items, &cfg());
783
+ let csv_out = CsvSchemaFormatter::new().format(&c);
784
+ let raw_json = serde_json::to_string(&Value::Array(items.clone())).unwrap();
785
+ assert!(
786
+ csv_out.len() * 10 < raw_json.len() * 7,
787
+ "csv {} bytes vs raw json {} bytes — expected >30% reduction",
788
+ csv_out.len(),
789
+ raw_json.len()
790
+ );
791
+ }
792
+
793
+ // ── MarkdownKvFormatter ──
794
+
795
+ #[test]
796
+ fn markdown_kv_renders_table() {
797
+ let items = vec![
798
+ json!({"id": 1, "name": "alice"}),
799
+ json!({"id": 2, "name": "bob"}),
800
+ ];
801
+ let c = compact(&items, &cfg());
802
+ let out = MarkdownKvFormatter::new().format(&c);
803
+ let lines: Vec<&str> = out.trim_end().lines().collect();
804
+ assert!(lines[0].starts_with("[2]{"), "got line[0]: {}", lines[0]);
805
+ assert!(lines[0].contains("id:int"));
806
+ assert!(out.contains("- id: 1\n name: alice\n"), "got: {out}");
807
+ assert!(out.contains("- id: 2\n name: bob\n"), "got: {out}");
808
+ }
809
+
810
+ #[test]
811
+ fn markdown_kv_omits_missing_cells() {
812
+ let items = vec![json!({"id": 1, "note": "has note"}), json!({"id": 2})];
813
+ let c = compact(&items, &cfg());
814
+ let out = MarkdownKvFormatter::new().format(&c);
815
+ assert!(out.contains("note: has note"), "got: {out}");
816
+ // Row 2 has no `note:` line at all.
817
+ let row2 = out.split("- id: 2").nth(1).expect("row 2 present");
818
+ assert!(!row2.contains("note:"), "got row2 tail: {row2}");
819
+ }
820
+
821
+ #[test]
822
+ fn markdown_kv_quotes_ambiguous_strings() {
823
+ let items = vec![
824
+ json!({"id": 1, "msg": "line one\nline two"}),
825
+ json!({"id": 2, "msg": "plain"}),
826
+ ];
827
+ let c = compact(&items, &cfg());
828
+ let out = MarkdownKvFormatter::new().format(&c);
829
+ assert!(
830
+ out.contains(r#"msg: "line one\nline two""#),
831
+ "multiline must be JSON-quoted, got: {out}"
832
+ );
833
+ assert!(out.contains("msg: plain\n"), "got: {out}");
834
+ }
835
+
836
+ #[test]
837
+ fn markdown_kv_quotes_pathological_field_names() {
838
+ // A newline in a key would inject fake row lines; ": " in a key
839
+ // would split read-back at the wrong colon. Both get JSON-quoted
840
+ // in the declaration and in every row line.
841
+ let items = vec![
842
+ json!({"bad\nkey": 1, "note: extra": "x"}),
843
+ json!({"bad\nkey": 2, "note: extra": "y"}),
844
+ ];
845
+ let c = compact(&items, &cfg());
846
+ let out = MarkdownKvFormatter::new().format(&c);
847
+ assert!(!out.contains("bad\nkey"), "raw newline key leaked: {out}");
848
+ assert!(out.contains(r#""bad\nkey""#), "got: {out}");
849
+ assert!(out.contains(r#""note: extra": x"#), "got: {out}");
850
+ let decl = out.lines().next().unwrap();
851
+ assert!(decl.contains(r#""bad\nkey":int"#), "got decl: {decl}");
852
+ }
853
+
854
+ #[test]
855
+ fn markdown_kv_plain_strings_unquoted() {
856
+ let items = vec![
857
+ json!({"id": 1, "name": "alice, the \"great\""}),
858
+ json!({"id": 2, "name": "bob"}),
859
+ ];
860
+ let c = compact(&items, &cfg());
861
+ let out = MarkdownKvFormatter::new().format(&c);
862
+ // Commas and quotes are fine on a KV line — no CSV-style quoting.
863
+ assert!(out.contains(r#"name: alice, the "great""#), "got: {out}");
864
+ }
865
+
866
+ #[test]
867
+ fn markdown_kv_emits_ccr_marker() {
868
+ let big = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".repeat(8);
869
+ let items = vec![
870
+ json!({"id": 1, "blob": big.clone()}),
871
+ json!({"id": 2, "blob": big.clone()}),
872
+ ];
873
+ let c = compact(&items, &cfg());
874
+ let out = MarkdownKvFormatter::new().format(&c);
875
+ assert!(out.contains("<<ccr:"), "got: {out}");
876
+ assert!(out.contains(",base64,"));
877
+ }
878
+
879
+ #[test]
880
+ fn markdown_kv_renders_buckets() {
881
+ let items = vec![
882
+ json!({"type": "user", "id": 1, "name": "alice"}),
883
+ json!({"type": "user", "id": 2, "name": "bob"}),
884
+ json!({"type": "order", "id": 99, "total": 50}),
885
+ json!({"type": "order", "id": 100, "total": 75}),
886
+ ];
887
+ let c = compact(&items, &cfg());
888
+ let out = MarkdownKvFormatter::new().format(&c);
889
+ assert!(out.starts_with("__buckets:type"), "got: {out}");
890
+ assert!(out.contains("__key:order"));
891
+ assert!(out.contains("__key:user"));
892
+ assert!(out.contains("- id:"), "got: {out}");
893
+ }
894
+
895
+ #[test]
896
+ fn markdown_kv_drop_summary_opt_in() {
897
+ let rows = vec![
898
+ Row::new(vec![CellValue::Scalar(json!(1))]),
899
+ Row::new(vec![CellValue::Scalar(json!(2))]),
900
+ ];
901
+ let c = Compaction::Table {
902
+ schema: Schema {
903
+ fields: vec![super::super::ir::FieldSpec {
904
+ name: "x".into(),
905
+ type_tag: "int".into(),
906
+ nullable: false,
907
+ }],
908
+ },
909
+ rows,
910
+ original_count: 5, // 3 dropped
911
+ };
912
+ let with_summary = MarkdownKvFormatter::new().with_drop_summary().format(&c);
913
+ assert!(with_summary.contains("__dropped:3"));
914
+ let without = MarkdownKvFormatter::new().format(&c);
915
+ assert!(!without.contains("__dropped"));
916
+ }
917
+
918
+ #[test]
919
+ fn markdown_kv_nested_cell_inline_json() {
920
+ let items = vec![
921
+ json!({"event": "batch", "payload": r#"[{"x":1},{"x":2},{"x":3}]"#}),
922
+ json!({"event": "batch", "payload": r#"[{"x":4},{"x":5}]"#}),
923
+ ];
924
+ let c = compact(&items, &cfg());
925
+ let out = MarkdownKvFormatter::new().format(&c);
926
+ assert!(out.contains("_compaction"), "got: {out}");
927
+ }
928
+
929
+ #[test]
930
+ fn markdown_kv_estimate_matches_format_len() {
931
+ let items = vec![json!({"a": 1}), json!({"a": 2})];
932
+ let c = compact(&items, &cfg());
933
+ let f = MarkdownKvFormatter::new();
934
+ assert_eq!(f.estimate_bytes(&c), f.format(&c).len());
935
+ }
936
+
937
+ #[test]
938
+ fn markdown_kv_smaller_than_raw_json() {
939
+ // KV repeats field names per row, so it loses to CSV on bytes —
940
+ // but it should still beat naïve JSON (quotes + braces + commas).
941
+ let items: Vec<Value> = (0..50)
942
+ .map(|i| {
943
+ json!({
944
+ "id": i,
945
+ "name": format!("user_{i}"),
946
+ "email": format!("user_{i}@example.com"),
947
+ "status": if i % 3 == 0 { "ok" } else { "pending" },
948
+ })
949
+ })
950
+ .collect();
951
+ let c = compact(&items, &cfg());
952
+ let kv_out = MarkdownKvFormatter::new().format(&c);
953
+ let raw_json = serde_json::to_string(&Value::Array(items.clone())).unwrap();
954
+ assert!(
955
+ kv_out.len() < raw_json.len(),
956
+ "kv {} bytes vs raw json {} bytes",
957
+ kv_out.len(),
958
+ raw_json.len()
959
+ );
960
+ }
961
+ }