xithead 1.2.2 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (659) hide show
  1. package/.actrc +14 -0
  2. package/.actrc.local.example +15 -0
  3. package/.changelog.md +2 -0
  4. package/.claude/agents/academic-anthropologist.md +125 -0
  5. package/.claude/agents/academic-geographer.md +127 -0
  6. package/.claude/agents/academic-historian.md +123 -0
  7. package/.claude/agents/academic-narratologist.md +118 -0
  8. package/.claude/agents/academic-psychologist.md +118 -0
  9. package/.claude/agents/academic-statistician.md +144 -0
  10. package/.claude/agents/accounts-payable-agent.md +205 -0
  11. package/.claude/agents/agentic-identity-trust.md +394 -0
  12. package/.claude/agents/agents-orchestrator.md +367 -0
  13. package/.claude/agents/automation-governance-architect.md +216 -0
  14. package/.claude/agents/blender-addon-engineer.md +243 -0
  15. package/.claude/agents/business-strategist.md +488 -0
  16. package/.claude/agents/change-management-consultant.md +497 -0
  17. package/.claude/agents/chief-financial-officer.md +388 -0
  18. package/.claude/agents/corporate-training-designer.md +192 -0
  19. package/.claude/agents/customer-service.md +398 -0
  20. package/.claude/agents/customer-success-manager.md +460 -0
  21. package/.claude/agents/data-consolidation-agent.md +60 -0
  22. package/.claude/agents/data-privacy-officer.md +412 -0
  23. package/.claude/agents/design-brand-guardian.md +322 -0
  24. package/.claude/agents/design-image-prompt-engineer.md +236 -0
  25. package/.claude/agents/design-inclusive-visuals-specialist.md +73 -0
  26. package/.claude/agents/design-persona-walkthrough.md +272 -0
  27. package/.claude/agents/design-ui-designer.md +386 -0
  28. package/.claude/agents/design-ui-finish-gate-reviewer.md +217 -0
  29. package/.claude/agents/design-ux-architect.md +482 -0
  30. package/.claude/agents/design-ux-researcher.md +329 -0
  31. package/.claude/agents/design-visual-storyteller.md +149 -0
  32. package/.claude/agents/design-whimsy-injector.md +460 -0
  33. package/.claude/agents/economy-designer.md +156 -0
  34. package/.claude/agents/engineering-ai-data-remediation-engineer.md +211 -0
  35. package/.claude/agents/engineering-ai-engineer.md +146 -0
  36. package/.claude/agents/engineering-api-platform-engineer.md +181 -0
  37. package/.claude/agents/engineering-ats-validator-architect.md +383 -0
  38. package/.claude/agents/engineering-autonomous-optimization-architect.md +121 -0
  39. package/.claude/agents/engineering-backend-architect.md +255 -0
  40. package/.claude/agents/engineering-china-network-engineer.md +252 -0
  41. package/.claude/agents/engineering-cms-developer.md +536 -0
  42. package/.claude/agents/engineering-code-reviewer.md +76 -0
  43. package/.claude/agents/engineering-codebase-onboarding-engineer.md +173 -0
  44. package/.claude/agents/engineering-data-engineer.md +349 -0
  45. package/.claude/agents/engineering-data-visualization-engineer.md +151 -0
  46. package/.claude/agents/engineering-database-optimizer.md +188 -0
  47. package/.claude/agents/engineering-database-reliability-engineer.md +201 -0
  48. package/.claude/agents/engineering-desktop-app-engineer.md +209 -0
  49. package/.claude/agents/engineering-developer-tooling-engineer.md +153 -0
  50. package/.claude/agents/engineering-devops-automator.md +385 -0
  51. package/.claude/agents/engineering-drupal-performance.md +347 -0
  52. package/.claude/agents/engineering-drupal-shopping-cart.md +360 -0
  53. package/.claude/agents/engineering-electrical-engineer.md +575 -0
  54. package/.claude/agents/engineering-email-intelligence-engineer.md +390 -0
  55. package/.claude/agents/engineering-embedded-firmware-engineer.md +196 -0
  56. package/.claude/agents/engineering-feishu-integration-developer.md +606 -0
  57. package/.claude/agents/engineering-filament-optimization-specialist.md +283 -0
  58. package/.claude/agents/engineering-finops-engineer.md +164 -0
  59. package/.claude/agents/engineering-frontend-developer.md +257 -0
  60. package/.claude/agents/engineering-gaussdb-expert.md +334 -0
  61. package/.claude/agents/engineering-git-workflow-master.md +98 -0
  62. package/.claude/agents/engineering-i18n-engineer.md +214 -0
  63. package/.claude/agents/engineering-identity-access-engineer.md +217 -0
  64. package/.claude/agents/engineering-incident-response-commander.md +444 -0
  65. package/.claude/agents/engineering-iot-fleet-engineer.md +148 -0
  66. package/.claude/agents/engineering-it-service-manager.md +561 -0
  67. package/.claude/agents/engineering-knowledge-graph-engineer.md +372 -0
  68. package/.claude/agents/engineering-llm-post-training-engineer.md +166 -0
  69. package/.claude/agents/engineering-minimal-change-engineer.md +207 -0
  70. package/.claude/agents/engineering-mobile-app-builder.md +493 -0
  71. package/.claude/agents/engineering-mobile-release-engineer.md +164 -0
  72. package/.claude/agents/engineering-multi-agent-systems-architect.md +600 -0
  73. package/.claude/agents/engineering-network-engineer.md +245 -0
  74. package/.claude/agents/engineering-orgscript-engineer.md +113 -0
  75. package/.claude/agents/engineering-payments-billing-engineer.md +228 -0
  76. package/.claude/agents/engineering-pdf-engine-architect.md +696 -0
  77. package/.claude/agents/engineering-platform-engineer.md +270 -0
  78. package/.claude/agents/engineering-privacy-engineer.md +152 -0
  79. package/.claude/agents/engineering-prompt-engineer.md +209 -0
  80. package/.claude/agents/engineering-rag-pipeline-engineer.md +450 -0
  81. package/.claude/agents/engineering-rapid-prototyper.md +466 -0
  82. package/.claude/agents/engineering-realtime-collaboration-engineer.md +200 -0
  83. package/.claude/agents/engineering-rust-refactoring-specialist.md +313 -0
  84. package/.claude/agents/engineering-search-relevance-engineer.md +237 -0
  85. package/.claude/agents/engineering-section-508-specialist.md +339 -0
  86. package/.claude/agents/engineering-senior-developer.md +176 -0
  87. package/.claude/agents/engineering-servicenow-developer-mentor.md +180 -0
  88. package/.claude/agents/engineering-software-architect.md +112 -0
  89. package/.claude/agents/engineering-solidity-smart-contract-engineer.md +522 -0
  90. package/.claude/agents/engineering-sre.md +90 -0
  91. package/.claude/agents/engineering-technical-writer.md +393 -0
  92. package/.claude/agents/engineering-universal-document-compiler.md +370 -0
  93. package/.claude/agents/engineering-uswds-developer.md +340 -0
  94. package/.claude/agents/engineering-video-streaming-engineer.md +151 -0
  95. package/.claude/agents/engineering-voice-ai-integration-engineer.md +584 -0
  96. package/.claude/agents/engineering-webassembly-engineer.md +153 -0
  97. package/.claude/agents/engineering-wechat-mini-program-developer.md +356 -0
  98. package/.claude/agents/engineering-wordpress-performance.md +346 -0
  99. package/.claude/agents/engineering-wordpress-shopping-cart.md +346 -0
  100. package/.claude/agents/esg-sustainability-officer.md +396 -0
  101. package/.claude/agents/finance-bookkeeper-controller.md +260 -0
  102. package/.claude/agents/finance-financial-analyst.md +234 -0
  103. package/.claude/agents/finance-fpa-analyst.md +263 -0
  104. package/.claude/agents/finance-investment-researcher.md +272 -0
  105. package/.claude/agents/finance-tax-strategist.md +239 -0
  106. package/.claude/agents/game-audio-engineer.md +264 -0
  107. package/.claude/agents/game-designer.md +167 -0
  108. package/.claude/agents/gis-3d-scene-developer.md +111 -0
  109. package/.claude/agents/gis-analyst.md +91 -0
  110. package/.claude/agents/gis-bim-specialist.md +108 -0
  111. package/.claude/agents/gis-cartography-designer.md +150 -0
  112. package/.claude/agents/gis-drone-reality-mapping.md +120 -0
  113. package/.claude/agents/gis-geoai-ml-engineer.md +105 -0
  114. package/.claude/agents/gis-geoprocessing-specialist.md +97 -0
  115. package/.claude/agents/gis-qa-engineer.md +133 -0
  116. package/.claude/agents/gis-solution-engineer.md +101 -0
  117. package/.claude/agents/gis-spatial-data-engineer.md +97 -0
  118. package/.claude/agents/gis-spatial-data-scientist.md +111 -0
  119. package/.claude/agents/gis-technical-consultant.md +86 -0
  120. package/.claude/agents/gis-web-gis-developer.md +108 -0
  121. package/.claude/agents/godot-gameplay-scripter.md +338 -0
  122. package/.claude/agents/godot-multiplayer-engineer.md +301 -0
  123. package/.claude/agents/godot-shader-developer.md +267 -0
  124. package/.claude/agents/government-digital-presales-consultant.md +363 -0
  125. package/.claude/agents/grant-writer.md +511 -0
  126. package/.claude/agents/healthcare-aging-parent-care-companion.md +414 -0
  127. package/.claude/agents/healthcare-clinical-evidence-agent.md +231 -0
  128. package/.claude/agents/healthcare-customer-service.md +389 -0
  129. package/.claude/agents/healthcare-innovation-strategist.md +433 -0
  130. package/.claude/agents/healthcare-marketing-compliance.md +395 -0
  131. package/.claude/agents/healthcare-sovereign-health-systems-agent.md +312 -0
  132. package/.claude/agents/hospitality-guest-services.md +603 -0
  133. package/.claude/agents/hr-onboarding.md +451 -0
  134. package/.claude/agents/identity-graph-operator.md +303 -0
  135. package/.claude/agents/language-translator.md +264 -0
  136. package/.claude/agents/legal-billing-time-tracking.md +569 -0
  137. package/.claude/agents/legal-client-intake.md +492 -0
  138. package/.claude/agents/legal-document-review.md +454 -0
  139. package/.claude/agents/level-designer.md +208 -0
  140. package/.claude/agents/loan-officer-assistant.md +555 -0
  141. package/.claude/agents/lsp-index-engineer.md +320 -0
  142. package/.claude/agents/ma-integration-manager.md +427 -0
  143. package/.claude/agents/macos-spatial-metal-engineer.md +337 -0
  144. package/.claude/agents/marketing-aeo-foundations.md +264 -0
  145. package/.claude/agents/marketing-agentic-search-optimizer.md +279 -0
  146. package/.claude/agents/marketing-ai-citation-strategist.md +172 -0
  147. package/.claude/agents/marketing-app-store-optimizer.md +321 -0
  148. package/.claude/agents/marketing-baidu-seo-specialist.md +226 -0
  149. package/.claude/agents/marketing-bilibili-content-strategist.md +199 -0
  150. package/.claude/agents/marketing-book-co-author.md +110 -0
  151. package/.claude/agents/marketing-carousel-growth-engine.md +204 -0
  152. package/.claude/agents/marketing-china-ecommerce-operator.md +283 -0
  153. package/.claude/agents/marketing-china-market-localization-strategist.md +283 -0
  154. package/.claude/agents/marketing-content-creator.md +54 -0
  155. package/.claude/agents/marketing-cross-border-ecommerce.md +259 -0
  156. package/.claude/agents/marketing-developer-community-builder.md +285 -0
  157. package/.claude/agents/marketing-douyin-strategist.md +149 -0
  158. package/.claude/agents/marketing-email-strategist.md +249 -0
  159. package/.claude/agents/marketing-global-podcast-strategist.md +206 -0
  160. package/.claude/agents/marketing-growth-hacker.md +54 -0
  161. package/.claude/agents/marketing-instagram-curator.md +113 -0
  162. package/.claude/agents/marketing-kuaishou-strategist.md +223 -0
  163. package/.claude/agents/marketing-linkedin-content-creator.md +214 -0
  164. package/.claude/agents/marketing-livestream-commerce-coach.md +305 -0
  165. package/.claude/agents/marketing-multi-platform-publisher.md +217 -0
  166. package/.claude/agents/marketing-podcast-strategist.md +277 -0
  167. package/.claude/agents/marketing-pr-communications-manager.md +473 -0
  168. package/.claude/agents/marketing-private-domain-operator.md +308 -0
  169. package/.claude/agents/marketing-reddit-community-builder.md +123 -0
  170. package/.claude/agents/marketing-seo-specialist.md +370 -0
  171. package/.claude/agents/marketing-short-video-editing-coach.md +412 -0
  172. package/.claude/agents/marketing-social-media-strategist.md +125 -0
  173. package/.claude/agents/marketing-tiktok-strategist.md +125 -0
  174. package/.claude/agents/marketing-twitter-engager.md +126 -0
  175. package/.claude/agents/marketing-video-optimization-specialist.md +119 -0
  176. package/.claude/agents/marketing-wechat-official-account.md +145 -0
  177. package/.claude/agents/marketing-weibo-strategist.md +240 -0
  178. package/.claude/agents/marketing-x-twitter-intelligence-analyst.md +161 -0
  179. package/.claude/agents/marketing-xiaohongshu-specialist.md +138 -0
  180. package/.claude/agents/marketing-zhihu-strategist.md +162 -0
  181. package/.claude/agents/medical-billing-coding-specialist.md +491 -0
  182. package/.claude/agents/narrative-designer.md +243 -0
  183. package/.claude/agents/operations-manager.md +399 -0
  184. package/.claude/agents/organizational-psychologist.md +391 -0
  185. package/.claude/agents/paid-media-auditor.md +71 -0
  186. package/.claude/agents/paid-media-creative-strategist.md +71 -0
  187. package/.claude/agents/paid-media-paid-social-strategist.md +71 -0
  188. package/.claude/agents/paid-media-ppc-strategist.md +71 -0
  189. package/.claude/agents/paid-media-programmatic-buyer.md +71 -0
  190. package/.claude/agents/paid-media-search-query-analyst.md +71 -0
  191. package/.claude/agents/paid-media-tracking-specialist.md +71 -0
  192. package/.claude/agents/personal-growth-mentor.md +159 -0
  193. package/.claude/agents/product-behavioral-nudge-engine.md +83 -0
  194. package/.claude/agents/product-dx-engineer.md +332 -0
  195. package/.claude/agents/product-feedback-synthesizer.md +119 -0
  196. package/.claude/agents/product-manager.md +469 -0
  197. package/.claude/agents/product-sprint-prioritizer.md +154 -0
  198. package/.claude/agents/product-trend-researcher.md +159 -0
  199. package/.claude/agents/project-management-experiment-tracker.md +198 -0
  200. package/.claude/agents/project-management-jira-workflow-steward.md +233 -0
  201. package/.claude/agents/project-management-meeting-notes-specialist.md +95 -0
  202. package/.claude/agents/project-management-project-shepherd.md +194 -0
  203. package/.claude/agents/project-management-studio-operations.md +200 -0
  204. package/.claude/agents/project-management-studio-producer.md +203 -0
  205. package/.claude/agents/project-manager-senior.md +135 -0
  206. package/.claude/agents/real-estate-buyer-seller.md +596 -0
  207. package/.claude/agents/recruitment-specialist.md +515 -0
  208. package/.claude/agents/report-distribution-agent.md +65 -0
  209. package/.claude/agents/research-synthesist.md +136 -0
  210. package/.claude/agents/resume-tailor.md +230 -0
  211. package/.claude/agents/retail-customer-returns.md +566 -0
  212. package/.claude/agents/roblox-avatar-creator.md +297 -0
  213. package/.claude/agents/roblox-experience-designer.md +306 -0
  214. package/.claude/agents/roblox-systems-scripter.md +332 -0
  215. package/.claude/agents/sales-account-strategist.md +227 -0
  216. package/.claude/agents/sales-coach.md +271 -0
  217. package/.claude/agents/sales-data-extraction-agent.md +67 -0
  218. package/.claude/agents/sales-deal-strategist.md +180 -0
  219. package/.claude/agents/sales-discovery-coach.md +225 -0
  220. package/.claude/agents/sales-engineer.md +182 -0
  221. package/.claude/agents/sales-offer-lead-gen-strategist.md +257 -0
  222. package/.claude/agents/sales-outbound-strategist.md +201 -0
  223. package/.claude/agents/sales-outreach.md +425 -0
  224. package/.claude/agents/sales-pipeline-analyst.md +267 -0
  225. package/.claude/agents/sales-proposal-strategist.md +217 -0
  226. package/.claude/agents/security-ai-generated-code-auditor.md +207 -0
  227. package/.claude/agents/security-appsec-engineer.md +536 -0
  228. package/.claude/agents/security-architect.md +307 -0
  229. package/.claude/agents/security-blockchain-security-auditor.md +482 -0
  230. package/.claude/agents/security-cloud-security-architect.md +576 -0
  231. package/.claude/agents/security-compliance-auditor.md +158 -0
  232. package/.claude/agents/security-incident-responder.md +450 -0
  233. package/.claude/agents/security-penetration-tester.md +399 -0
  234. package/.claude/agents/security-secrets-credential-engineer.md +189 -0
  235. package/.claude/agents/security-senior-secops.md +760 -0
  236. package/.claude/agents/security-threat-detection-engineer.md +553 -0
  237. package/.claude/agents/security-threat-intelligence-analyst.md +661 -0
  238. package/.claude/agents/specialized-chief-of-staff.md +279 -0
  239. package/.claude/agents/specialized-civil-engineer.md +356 -0
  240. package/.claude/agents/specialized-codebase-archaeologist.md +341 -0
  241. package/.claude/agents/specialized-cultural-intelligence-strategist.md +88 -0
  242. package/.claude/agents/specialized-developer-advocate.md +317 -0
  243. package/.claude/agents/specialized-document-generator.md +55 -0
  244. package/.claude/agents/specialized-fedramp-rmf-compliance.md +378 -0
  245. package/.claude/agents/specialized-focus-music-architect.md +175 -0
  246. package/.claude/agents/specialized-french-consulting-market.md +194 -0
  247. package/.claude/agents/specialized-korean-business-navigator.md +216 -0
  248. package/.claude/agents/specialized-master-plan-architect.md +157 -0
  249. package/.claude/agents/specialized-mcp-builder.md +260 -0
  250. package/.claude/agents/specialized-model-qa.md +528 -0
  251. package/.claude/agents/specialized-pricing-analyst.md +243 -0
  252. package/.claude/agents/specialized-salesforce-architect.md +182 -0
  253. package/.claude/agents/specialized-strategy-duel-agent.md +130 -0
  254. package/.claude/agents/specialized-workflow-architect.md +597 -0
  255. package/.claude/agents/study-abroad-advisor.md +282 -0
  256. package/.claude/agents/supply-chain-strategist.md +591 -0
  257. package/.claude/agents/support-analytics-reporter.md +387 -0
  258. package/.claude/agents/support-executive-summary-generator.md +212 -0
  259. package/.claude/agents/support-finance-tracker.md +480 -0
  260. package/.claude/agents/support-infrastructure-maintainer.md +628 -0
  261. package/.claude/agents/support-legal-compliance-checker.md +591 -0
  262. package/.claude/agents/support-support-responder.md +607 -0
  263. package/.claude/agents/technical-artist.md +233 -0
  264. package/.claude/agents/terminal-integration-specialist.md +70 -0
  265. package/.claude/agents/testing-accessibility-auditor.md +327 -0
  266. package/.claude/agents/testing-api-tester.md +334 -0
  267. package/.claude/agents/testing-evidence-collector.md +210 -0
  268. package/.claude/agents/testing-performance-benchmarker.md +283 -0
  269. package/.claude/agents/testing-reality-checker.md +249 -0
  270. package/.claude/agents/testing-test-automation-engineer.md +196 -0
  271. package/.claude/agents/testing-test-results-analyzer.md +330 -0
  272. package/.claude/agents/testing-tool-evaluator.md +416 -0
  273. package/.claude/agents/testing-workflow-optimizer.md +463 -0
  274. package/.claude/agents/unity-architect.md +271 -0
  275. package/.claude/agents/unity-editor-tool-developer.md +310 -0
  276. package/.claude/agents/unity-multiplayer-engineer.md +323 -0
  277. package/.claude/agents/unity-shader-graph-artist.md +269 -0
  278. package/.claude/agents/unreal-multiplayer-architect.md +364 -0
  279. package/.claude/agents/unreal-systems-engineer.md +322 -0
  280. package/.claude/agents/unreal-technical-artist.md +256 -0
  281. package/.claude/agents/unreal-world-builder.md +273 -0
  282. package/.claude/agents/visionos-spatial-engineer.md +54 -0
  283. package/.claude/agents/xr-cockpit-interaction-specialist.md +32 -0
  284. package/.claude/agents/xr-immersive-developer.md +32 -0
  285. package/.claude/agents/xr-interface-architect.md +32 -0
  286. package/.claude/agents/zk-steward.md +211 -0
  287. package/.claude-plugin/marketplace.json +30 -0
  288. package/.commitlintrc.json +26 -0
  289. package/.devcontainer/Dockerfile +24 -0
  290. package/.devcontainer/devcontainer.json +70 -0
  291. package/.devcontainer/docker-compose.memory.yml +42 -0
  292. package/.devcontainer/memory-stack/devcontainer.json +84 -0
  293. package/.devcontainer/post-create.sh +83 -0
  294. package/.dockerignore +23 -0
  295. package/.env.act.example +15 -0
  296. package/.env.example +171 -0
  297. package/.eslintrc.json +16 -0
  298. package/.git-blame-ignore-revs +10 -0
  299. package/.gitattributes +2 -0
  300. package/.gitguardian.yaml +50 -0
  301. package/.githooks/pre-commit +6 -0
  302. package/.pre-commit-config.yaml +30 -0
  303. package/.release-please-config-xithead.json +22 -0
  304. package/.release-please-config.json +44 -0
  305. package/.release-please-manifest-xithead.json +3 -0
  306. package/.release-please-manifest.json +3 -0
  307. package/CODE_OF_CONDUCT.md +133 -0
  308. package/CONTRIBUTING.md +133 -0
  309. package/Cargo.lock +5557 -0
  310. package/Cargo.toml +121 -0
  311. package/Dockerfile +38 -0
  312. package/Dockerfile.proxy +48 -0
  313. package/ENTERPRISE.md +5 -0
  314. package/FEATURES_ADVANCED.md +311 -0
  315. package/Headroom-2.gif +0 -0
  316. package/HeadroomDemo-Fast.gif +0 -0
  317. package/MANIFEST.in +4 -0
  318. package/MODERNIZATION_GUIDE.md +358 -0
  319. package/Makefile +140 -0
  320. package/NOTICE +43 -0
  321. package/NPM_TOKEN_SETUP.md +81 -0
  322. package/PR.md +170 -0
  323. package/PUBLISH.md +77 -0
  324. package/README.md +58 -137
  325. package/README_FRONTEND.md +196 -0
  326. package/REALIGNMENT/00-overview.md +53 -0
  327. package/REALIGNMENT/01-bug-list.md +421 -0
  328. package/REALIGNMENT/02-architecture.md +322 -0
  329. package/REALIGNMENT/03-phase-A-lockdown.md +420 -0
  330. package/REALIGNMENT/04-phase-B-live-zone.md +470 -0
  331. package/REALIGNMENT/05-phase-C-rust-proxy.md +329 -0
  332. package/REALIGNMENT/06-phase-D-bedrock-vertex.md +222 -0
  333. package/REALIGNMENT/07-phase-E-cache-stabilization.md +338 -0
  334. package/REALIGNMENT/08-phase-F-auth-mode.md +254 -0
  335. package/REALIGNMENT/09-phase-G-rtk-observability.md +194 -0
  336. package/REALIGNMENT/10-phase-H-python-retirement.md +214 -0
  337. package/REALIGNMENT/11-phase-I-test-infra.md +385 -0
  338. package/REALIGNMENT/12-decisions-needed.md +196 -0
  339. package/REALIGNMENT/INDEX.md +82 -0
  340. package/REFACTORING_CAMPAIGN.md +277 -0
  341. package/RESTRUCTURE_PLAN.md +100 -0
  342. package/RUST_DEV.md +361 -0
  343. package/SECURITY.md +65 -0
  344. package/SETUP_AUTO_PUBLISH.md +164 -0
  345. package/TESTING-copilot-subscription.md +153 -0
  346. package/app/api/config/route.ts +22 -0
  347. package/app/api/export/route.ts +83 -0
  348. package/app/api/models/route.ts +12 -0
  349. package/app/api/stats/route.ts +46 -0
  350. package/app/dashboard/page.tsx +445 -0
  351. package/app/globals.css +35 -0
  352. package/app/installation/page.tsx +293 -0
  353. package/app/layout.tsx +35 -0
  354. package/app/master-design-system.css +2076 -0
  355. package/app/master-design-system.source.css +8 -0
  356. package/app/page.tsx +187 -0
  357. package/app/rechtliches/agb/page.tsx +165 -0
  358. package/app/rechtliches/datenschutz/page.tsx +128 -0
  359. package/app/rechtliches/impressum/page.tsx +122 -0
  360. package/app/rechtliches/widerruf/page.tsx +185 -0
  361. package/bin/xithead.js +264 -0
  362. package/codecov.yml +34 -0
  363. package/components/layout/AppShell.tsx +25 -0
  364. package/components/layout/Footer.tsx +120 -0
  365. package/components/layout/Header.tsx +145 -0
  366. package/components/marketing/LegalHighlights.tsx +45 -0
  367. package/components/marketing/LegalPageShell.tsx +49 -0
  368. package/components/marketing/LegalPanel.tsx +41 -0
  369. package/components/marketing/index.ts +3 -0
  370. package/crates/headroom-core/Cargo.toml +165 -0
  371. package/crates/headroom-core/benches/auth_mode.rs +74 -0
  372. package/crates/headroom-core/benches/ccr_store.rs +224 -0
  373. package/crates/headroom-core/benches/tokenizer.rs +52 -0
  374. package/crates/headroom-core/config/pipeline.toml +151 -0
  375. package/crates/headroom-core/proptest-regressions/transforms/tag_protector.txt +9 -0
  376. package/crates/headroom-core/src/auth_mode.rs +250 -0
  377. package/crates/headroom-core/src/cache_control.rs +414 -0
  378. package/crates/headroom-core/src/ccr/backends/in_memory.rs +317 -0
  379. package/crates/headroom-core/src/ccr/backends/mod.rs +152 -0
  380. package/crates/headroom-core/src/ccr/backends/redis.rs +146 -0
  381. package/crates/headroom-core/src/ccr/backends/sqlite.rs +205 -0
  382. package/crates/headroom-core/src/ccr/mod.rs +119 -0
  383. package/crates/headroom-core/src/compression_policy.rs +505 -0
  384. package/crates/headroom-core/src/lib.rs +33 -0
  385. package/crates/headroom-core/src/relevance/base.rs +139 -0
  386. package/crates/headroom-core/src/relevance/bm25.rs +385 -0
  387. package/crates/headroom-core/src/relevance/embedding.rs +354 -0
  388. package/crates/headroom-core/src/relevance/hybrid.rs +364 -0
  389. package/crates/headroom-core/src/relevance/mod.rs +64 -0
  390. package/crates/headroom-core/src/signals/keyword_detector.rs +433 -0
  391. package/crates/headroom-core/src/signals/line_importance.rs +84 -0
  392. package/crates/headroom-core/src/signals/mod.rs +61 -0
  393. package/crates/headroom-core/src/signals/tiered.rs +141 -0
  394. package/crates/headroom-core/src/tokenizer/estimator.rs +152 -0
  395. package/crates/headroom-core/src/tokenizer/hf_impl.rs +314 -0
  396. package/crates/headroom-core/src/tokenizer/mod.rs +56 -0
  397. package/crates/headroom-core/src/tokenizer/registry.rs +342 -0
  398. package/crates/headroom-core/src/tokenizer/tiktoken_impl.rs +265 -0
  399. package/crates/headroom-core/src/transforms/adaptive_sizer.rs +610 -0
  400. package/crates/headroom-core/src/transforms/anchor_selector.rs +1189 -0
  401. package/crates/headroom-core/src/transforms/content_detector.rs +769 -0
  402. package/crates/headroom-core/src/transforms/detection.rs +222 -0
  403. package/crates/headroom-core/src/transforms/diff_compressor.rs +1685 -0
  404. package/crates/headroom-core/src/transforms/live_zone.rs +2967 -0
  405. package/crates/headroom-core/src/transforms/log_compressor.rs +1295 -0
  406. package/crates/headroom-core/src/transforms/magika_detector.rs +395 -0
  407. package/crates/headroom-core/src/transforms/mod.rs +64 -0
  408. package/crates/headroom-core/src/transforms/pipeline/config.rs +331 -0
  409. package/crates/headroom-core/src/transforms/pipeline/mod.rs +103 -0
  410. package/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rs +524 -0
  411. package/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs +279 -0
  412. package/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs +400 -0
  413. package/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rs +284 -0
  414. package/crates/headroom-core/src/transforms/pipeline/offloads/mod.rs +34 -0
  415. package/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rs +331 -0
  416. package/crates/headroom-core/src/transforms/pipeline/orchestrator.rs +848 -0
  417. package/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rs +175 -0
  418. package/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rs +545 -0
  419. package/crates/headroom-core/src/transforms/pipeline/reformats/mod.rs +12 -0
  420. package/crates/headroom-core/src/transforms/pipeline/traits.rs +338 -0
  421. package/crates/headroom-core/src/transforms/recommendations.rs +329 -0
  422. package/crates/headroom-core/src/transforms/safety.rs +215 -0
  423. package/crates/headroom-core/src/transforms/search_compressor.rs +902 -0
  424. package/crates/headroom-core/src/transforms/smart_crusher/analyzer.rs +1262 -0
  425. package/crates/headroom-core/src/transforms/smart_crusher/anchors.rs +407 -0
  426. package/crates/headroom-core/src/transforms/smart_crusher/builder.rs +263 -0
  427. package/crates/headroom-core/src/transforms/smart_crusher/classifier.rs +214 -0
  428. package/crates/headroom-core/src/transforms/smart_crusher/compaction/classifier.rs +283 -0
  429. package/crates/headroom-core/src/transforms/smart_crusher/compaction/compactor.rs +733 -0
  430. package/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rs +961 -0
  431. package/crates/headroom-core/src/transforms/smart_crusher/compaction/ir.rs +252 -0
  432. package/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs +127 -0
  433. package/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs +397 -0
  434. package/crates/headroom-core/src/transforms/smart_crusher/config.rs +173 -0
  435. package/crates/headroom-core/src/transforms/smart_crusher/constraints.rs +159 -0
  436. package/crates/headroom-core/src/transforms/smart_crusher/crusher.rs +1665 -0
  437. package/crates/headroom-core/src/transforms/smart_crusher/crushers.rs +865 -0
  438. package/crates/headroom-core/src/transforms/smart_crusher/error_keywords.rs +77 -0
  439. package/crates/headroom-core/src/transforms/smart_crusher/field_detect.rs +393 -0
  440. package/crates/headroom-core/src/transforms/smart_crusher/hashing.rs +72 -0
  441. package/crates/headroom-core/src/transforms/smart_crusher/mod.rs +81 -0
  442. package/crates/headroom-core/src/transforms/smart_crusher/observer.rs +66 -0
  443. package/crates/headroom-core/src/transforms/smart_crusher/orchestration.rs +459 -0
  444. package/crates/headroom-core/src/transforms/smart_crusher/outliers.rs +507 -0
  445. package/crates/headroom-core/src/transforms/smart_crusher/planning.rs +904 -0
  446. package/crates/headroom-core/src/transforms/smart_crusher/statistics.rs +510 -0
  447. package/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs +287 -0
  448. package/crates/headroom-core/src/transforms/smart_crusher/traits.rs +203 -0
  449. package/crates/headroom-core/src/transforms/smart_crusher/types.rs +263 -0
  450. package/crates/headroom-core/src/transforms/tag_protector.rs +1272 -0
  451. package/crates/headroom-core/src/transforms/unidiff_detector.rs +232 -0
  452. package/crates/headroom-parity/Cargo.toml +26 -0
  453. package/crates/headroom-parity/examples/diff_fixture.rs +66 -0
  454. package/crates/headroom-parity/src/bin/parity_run.rs +78 -0
  455. package/crates/headroom-parity/src/lib.rs +613 -0
  456. package/crates/headroom-proxy/Cargo.toml +119 -0
  457. package/crates/headroom-proxy/data/model_prices_and_context_window.json +39767 -0
  458. package/crates/headroom-proxy/src/bedrock/auth_mode_layer.rs +192 -0
  459. package/crates/headroom-proxy/src/bedrock/envelope.rs +239 -0
  460. package/crates/headroom-proxy/src/bedrock/eventstream.rs +769 -0
  461. package/crates/headroom-proxy/src/bedrock/eventstream_to_sse.rs +459 -0
  462. package/crates/headroom-proxy/src/bedrock/invoke.rs +735 -0
  463. package/crates/headroom-proxy/src/bedrock/invoke_streaming.rs +1098 -0
  464. package/crates/headroom-proxy/src/bedrock/mod.rs +66 -0
  465. package/crates/headroom-proxy/src/bedrock/sigv4.rs +324 -0
  466. package/crates/headroom-proxy/src/bedrock/vendor.rs +82 -0
  467. package/crates/headroom-proxy/src/cache_stabilization/anthropic_cache_control.rs +698 -0
  468. package/crates/headroom-proxy/src/cache_stabilization/drift_detector.rs +683 -0
  469. package/crates/headroom-proxy/src/cache_stabilization/mod.rs +61 -0
  470. package/crates/headroom-proxy/src/cache_stabilization/openai_cache_key.rs +600 -0
  471. package/crates/headroom-proxy/src/cache_stabilization/tool_def_normalize.rs +514 -0
  472. package/crates/headroom-proxy/src/cache_stabilization/volatile_detector.rs +720 -0
  473. package/crates/headroom-proxy/src/compression/anthropic.rs +92 -0
  474. package/crates/headroom-proxy/src/compression/live_zone_anthropic.rs +1311 -0
  475. package/crates/headroom-proxy/src/compression/live_zone_openai.rs +660 -0
  476. package/crates/headroom-proxy/src/compression/live_zone_responses.rs +678 -0
  477. package/crates/headroom-proxy/src/compression/mod.rs +127 -0
  478. package/crates/headroom-proxy/src/compression/model_limits.rs +211 -0
  479. package/crates/headroom-proxy/src/config.rs +646 -0
  480. package/crates/headroom-proxy/src/error.rs +69 -0
  481. package/crates/headroom-proxy/src/handlers/chat_completions.rs +100 -0
  482. package/crates/headroom-proxy/src/handlers/conversations.rs +241 -0
  483. package/crates/headroom-proxy/src/handlers/mod.rs +13 -0
  484. package/crates/headroom-proxy/src/handlers/responses.rs +256 -0
  485. package/crates/headroom-proxy/src/headers.rs +288 -0
  486. package/crates/headroom-proxy/src/health.rs +41 -0
  487. package/crates/headroom-proxy/src/lib.rs +21 -0
  488. package/crates/headroom-proxy/src/main.rs +141 -0
  489. package/crates/headroom-proxy/src/observability/cache_hit_rate.rs +340 -0
  490. package/crates/headroom-proxy/src/observability/compression_ratio.rs +158 -0
  491. package/crates/headroom-proxy/src/observability/metric_names.rs +174 -0
  492. package/crates/headroom-proxy/src/observability/mod.rs +68 -0
  493. package/crates/headroom-proxy/src/observability/prometheus.rs +394 -0
  494. package/crates/headroom-proxy/src/observability/proxy_metrics.rs +368 -0
  495. package/crates/headroom-proxy/src/proxy.rs +1619 -0
  496. package/crates/headroom-proxy/src/responses_items.rs +565 -0
  497. package/crates/headroom-proxy/src/sse/anthropic.rs +415 -0
  498. package/crates/headroom-proxy/src/sse/framing.rs +366 -0
  499. package/crates/headroom-proxy/src/sse/mod.rs +40 -0
  500. package/crates/headroom-proxy/src/sse/openai_chat.rs +231 -0
  501. package/crates/headroom-proxy/src/sse/openai_responses.rs +458 -0
  502. package/crates/headroom-proxy/src/vertex/adc.rs +315 -0
  503. package/crates/headroom-proxy/src/vertex/envelope.rs +167 -0
  504. package/crates/headroom-proxy/src/vertex/mod.rs +277 -0
  505. package/crates/headroom-proxy/src/vertex/raw_predict.rs +490 -0
  506. package/crates/headroom-proxy/src/vertex/stream_raw_predict.rs +49 -0
  507. package/crates/headroom-proxy/src/websocket.rs +246 -0
  508. package/crates/headroom-py/Cargo.toml +37 -0
  509. package/crates/headroom-py/build.rs +77 -0
  510. package/crates/headroom-py/glibc_compat.c +170 -0
  511. package/crates/headroom-py/src/lib.rs +1655 -0
  512. package/deny.toml +32 -0
  513. package/design-system/base.css +127 -0
  514. package/design-system/components.css +1943 -0
  515. package/design-system/fonts.css +17 -0
  516. package/design-system/tailwind-preset.js +95 -0
  517. package/design-system/tokens.css +48 -0
  518. package/design-system/utilities.css +53 -0
  519. package/docker/differential-network-capture/Dockerfile.runner +14 -0
  520. package/docker/differential-network-capture/docker-compose.yml +142 -0
  521. package/docker/differential-network-capture/run-claude-lane.sh +24 -0
  522. package/docker/docker-compose.native.yml +44 -0
  523. package/docker-bake.hcl +88 -0
  524. package/docker-compose.yml +80 -0
  525. package/e2e/_lib/make_shim.ps1 +24 -0
  526. package/e2e/_lib/make_shim.sh +28 -0
  527. package/e2e/docker-native-install.sh +120 -0
  528. package/e2e/init/Dockerfile +79 -0
  529. package/e2e/wrap/Dockerfile +115 -0
  530. package/examples/07-context-compression.ipynb +279 -0
  531. package/examples/deployment/macos-launchagent/com.headroom.proxy.plist.template +65 -0
  532. package/examples/deployment/macos-launchagent/install.sh +227 -0
  533. package/examples/deployment/macos-launchagent/shell-integration.sh +74 -0
  534. package/examples/deployment/macos-launchagent/uninstall.sh +145 -0
  535. package/headroom-savings.png +0 -0
  536. package/headroom_learn.gif +0 -0
  537. package/lib/config/index.ts +12 -0
  538. package/lib/hooks/index.ts +4 -0
  539. package/lib/hooks/use-mounted.ts +13 -0
  540. package/lib/hooks/use-session.ts +34 -0
  541. package/lib/hooks/use-settings.ts +49 -0
  542. package/lib/utils/index.ts +1 -0
  543. package/lib/utils/settings.ts +162 -0
  544. package/lib/utils/utils.ts +6 -0
  545. package/llms.txt +67 -0
  546. package/middleware.ts +26 -0
  547. package/mkdocs.yml +136 -0
  548. package/next-env.d.ts +7 -0
  549. package/next.config.ts +12 -0
  550. package/npm/bin/xithead.js +2 -2
  551. package/npm/bin/xtream-head.js +366 -0
  552. package/npm/index.js +18 -91
  553. package/npm/package.json +55 -0
  554. package/package.json +47 -42
  555. package/plugins/headroom-agent-hooks/.claude-plugin/plugin.json +17 -0
  556. package/plugins/headroom-agent-hooks/hooks/hooks.json +29 -0
  557. package/plugins/headroom-oauth2/LICENSE +3 -0
  558. package/plugins/headroom-oauth2/SPEC.md +115 -0
  559. package/plugins/headroom-oauth2/pyproject.toml +48 -0
  560. package/plugins/hermes/headroom_retrieve/plugin.yaml +6 -0
  561. package/plugins/openclaw/hook-shim/HOOK.md +21 -0
  562. package/plugins/openclaw/hook-shim/handler.js +3 -0
  563. package/plugins/openclaw/openclaw.plugin.json +98 -0
  564. package/plugins/openclaw/package-lock.json +2692 -0
  565. package/plugins/openclaw/package.json +54 -0
  566. package/plugins/openclaw/prepare-dist.mjs +56 -0
  567. package/plugins/openclaw/src/convert.ts +418 -0
  568. package/plugins/openclaw/src/engine.ts +284 -0
  569. package/plugins/openclaw/src/gateway-config.ts +143 -0
  570. package/plugins/openclaw/src/index.ts +11 -0
  571. package/plugins/openclaw/src/plugin/index.ts +98 -0
  572. package/plugins/openclaw/src/proxy-manager.ts +477 -0
  573. package/plugins/openclaw/src/tools/headroom-retrieve.ts +76 -0
  574. package/plugins/openclaw/test/convert.test.ts +96 -0
  575. package/plugins/openclaw/test/engine-normalization.test.ts +21 -0
  576. package/plugins/openclaw/test/engine.test.ts +101 -0
  577. package/plugins/openclaw/test/gateway-config.test.ts +278 -0
  578. package/plugins/openclaw/test/plugin-runtime-routing.test.ts +141 -0
  579. package/plugins/openclaw/test/proxy-manager.test.ts +343 -0
  580. package/plugins/openclaw/tsup.config.ts +9 -0
  581. package/postcss.config.mjs +7 -0
  582. package/pyproject.toml +438 -0
  583. package/rust-toolchain.toml +16 -0
  584. package/sdk/typescript/examples/basic-compress.ts +64 -0
  585. package/sdk/typescript/examples/ccr-retrieve.ts +80 -0
  586. package/sdk/typescript/examples/hooks-custom-compression.ts +117 -0
  587. package/sdk/typescript/examples/middleware-composition.ts +59 -0
  588. package/sdk/typescript/examples/multi-provider.ts +53 -0
  589. package/sdk/typescript/examples/openai-anthropic-adapters.ts +73 -0
  590. package/sdk/typescript/examples/shared-context-multi-agent.ts +74 -0
  591. package/sdk/typescript/examples/simulation-dry-run.ts +80 -0
  592. package/sdk/typescript/examples/streaming-chat.ts +71 -0
  593. package/sdk/typescript/examples/structured-output.ts +99 -0
  594. package/sdk/typescript/examples/tool-calling-agent.ts +86 -0
  595. package/sdk/typescript/examples/with-headroom-vercel.ts +57 -0
  596. package/sdk/typescript/package-lock.json +3437 -0
  597. package/sdk/typescript/package.json +108 -0
  598. package/sdk/typescript/src/adapters/anthropic.ts +218 -0
  599. package/sdk/typescript/src/adapters/gemini.ts +83 -0
  600. package/sdk/typescript/src/adapters/openai.ts +77 -0
  601. package/sdk/typescript/src/adapters/vercel-ai.ts +131 -0
  602. package/sdk/typescript/src/client.ts +632 -0
  603. package/sdk/typescript/src/compress.ts +87 -0
  604. package/sdk/typescript/src/errors.ts +114 -0
  605. package/sdk/typescript/src/hooks.ts +113 -0
  606. package/sdk/typescript/src/index.ts +140 -0
  607. package/sdk/typescript/src/paths.ts +272 -0
  608. package/sdk/typescript/src/shared-context.ts +188 -0
  609. package/sdk/typescript/src/simulate.ts +52 -0
  610. package/sdk/typescript/src/types/config.ts +183 -0
  611. package/sdk/typescript/src/types/models.ts +314 -0
  612. package/sdk/typescript/src/types.ts +139 -0
  613. package/sdk/typescript/src/utils/case.ts +34 -0
  614. package/sdk/typescript/src/utils/format.ts +480 -0
  615. package/sdk/typescript/src/utils/stream.ts +50 -0
  616. package/sdk/typescript/test/adapters/anthropic.test.ts +217 -0
  617. package/sdk/typescript/test/adapters/gemini.test.ts +131 -0
  618. package/sdk/typescript/test/adapters/openai.test.ts +174 -0
  619. package/sdk/typescript/test/adapters/vercel-ai.test.ts +306 -0
  620. package/sdk/typescript/test/client-expanded.test.ts +333 -0
  621. package/sdk/typescript/test/client.test.ts +266 -0
  622. package/sdk/typescript/test/compress.test.ts +118 -0
  623. package/sdk/typescript/test/config.test.ts +275 -0
  624. package/sdk/typescript/test/errors.test.ts +169 -0
  625. package/sdk/typescript/test/format-integration.test.ts +252 -0
  626. package/sdk/typescript/test/hooks.test.ts +265 -0
  627. package/sdk/typescript/test/integration.test.ts +249 -0
  628. package/sdk/typescript/test/mdx-examples.test.ts +253 -0
  629. package/sdk/typescript/test/models.test.ts +318 -0
  630. package/sdk/typescript/test/parity.test.ts +337 -0
  631. package/sdk/typescript/test/paths.test.ts +447 -0
  632. package/sdk/typescript/test/shared-context.test.ts +245 -0
  633. package/sdk/typescript/test/simulate.test.ts +84 -0
  634. package/sdk/typescript/test/types.test.ts +244 -0
  635. package/sdk/typescript/test/utils/case.test.ts +129 -0
  636. package/sdk/typescript/test/utils/detect-format.test.ts +133 -0
  637. package/sdk/typescript/test/utils/format.test.ts +434 -0
  638. package/sdk/typescript/test/vercel-ai-e2e.test.ts +179 -0
  639. package/sdk/typescript/tsup.config.ts +17 -0
  640. package/tailwind.master.config.cjs +47 -0
  641. package/types/api.ts +12 -0
  642. package/types/config.ts +9 -0
  643. package/types/index.ts +2 -0
  644. package/uv.lock +6951 -0
  645. package/xithead.egg-info/PKG-INFO +22 -0
  646. package/xithead.egg-info/SOURCES.txt +16 -0
  647. package/xithead.egg-info/dependency_links.txt +1 -0
  648. package/xithead.egg-info/entry_points.txt +2 -0
  649. package/xithead.egg-info/requires.txt +10 -0
  650. package/xithead.egg-info/top_level.txt +2 -0
  651. package/xithead.ps1 +136 -0
  652. package/xithead.sh +129 -0
  653. package/xithead_dist/pyproject.toml +39 -0
  654. package/xithead_dist/xithead.egg-info/PKG-INFO +22 -0
  655. package/xithead_dist/xithead.egg-info/SOURCES.txt +16 -0
  656. package/xithead_dist/xithead.egg-info/dependency_links.txt +1 -0
  657. package/xithead_dist/xithead.egg-info/entry_points.txt +3 -0
  658. package/xithead_dist/xithead.egg-info/requires.txt +10 -0
  659. package/xithead_dist/xithead.egg-info/top_level.txt +2 -0
@@ -0,0 +1,141 @@
1
+ //! Composition combinator for layered detectors.
2
+ //!
3
+ //! `Tiered<dyn Trait>` chains an ordered list of detectors. The first
4
+ //! tier whose signal exceeds [`ESCALATE_THRESHOLD`] confidence wins;
5
+ //! lower-confidence tiers are skipped past. If no tier exceeds the
6
+ //! threshold, the highest-confidence signal seen is returned (so the
7
+ //! caller still gets the best guess, with the confidence score
8
+ //! reflecting how unsure the stack is).
9
+ //!
10
+ //! Tiering is *composition*, not inheritance. `KeywordDetector` knows
11
+ //! nothing about a future ML detector; the ML detector knows nothing
12
+ //! about the keyword detector. They both implement the trait and the
13
+ //! `Tiered` wrapper orders them.
14
+
15
+ use super::line_importance::{ImportanceContext, ImportanceSignal, LineImportanceDetector};
16
+
17
+ /// Confidence at which `Tiered` accepts a tier's signal without
18
+ /// consulting later tiers. KeywordDetector emits 0.7, so it wins by
19
+ /// default; an ML tier with calibrated confidence ≥ 0.8 (high-precision
20
+ /// region) would short-circuit the keyword tier.
21
+ pub const ESCALATE_THRESHOLD: f32 = 0.7;
22
+
23
+ pub struct Tiered<T: ?Sized> {
24
+ tiers: Vec<Box<T>>,
25
+ }
26
+
27
+ impl<T: ?Sized> Tiered<T> {
28
+ pub fn new() -> Self {
29
+ Self { tiers: Vec::new() }
30
+ }
31
+
32
+ /// Push a tier onto the stack. Order matters: most-precise first.
33
+ pub fn with(mut self, tier: Box<T>) -> Self {
34
+ self.tiers.push(tier);
35
+ self
36
+ }
37
+
38
+ pub fn len(&self) -> usize {
39
+ self.tiers.len()
40
+ }
41
+
42
+ pub fn is_empty(&self) -> bool {
43
+ self.tiers.is_empty()
44
+ }
45
+ }
46
+
47
+ impl<T: ?Sized> Default for Tiered<T> {
48
+ fn default() -> Self {
49
+ Self::new()
50
+ }
51
+ }
52
+
53
+ impl LineImportanceDetector for Tiered<dyn LineImportanceDetector> {
54
+ fn score(&self, line: &str, ctx: ImportanceContext) -> ImportanceSignal {
55
+ let mut best = ImportanceSignal::neutral();
56
+ for tier in &self.tiers {
57
+ let signal = tier.score(line, ctx);
58
+ if signal.confidence >= ESCALATE_THRESHOLD {
59
+ return signal;
60
+ }
61
+ if signal.confidence > best.confidence {
62
+ best = signal;
63
+ }
64
+ }
65
+ best
66
+ }
67
+ }
68
+
69
+ impl Tiered<dyn LineImportanceDetector> {
70
+ /// Convenience: take an owned detector, box it, coerce it to the
71
+ /// trait object. Keeps callsites free of `as Box<dyn …>` clutter.
72
+ pub fn with_detector<D: LineImportanceDetector + 'static>(self, detector: D) -> Self {
73
+ self.with(Box::new(detector) as Box<dyn LineImportanceDetector>)
74
+ }
75
+ }
76
+
77
+ #[cfg(test)]
78
+ mod tests {
79
+ use super::*;
80
+ use crate::signals::keyword_detector::KeywordDetector;
81
+ use crate::signals::line_importance::ImportanceCategory;
82
+
83
+ /// Synthetic high-confidence detector for testing short-circuit
84
+ /// behavior. Always asserts a specific signal so we can prove
85
+ /// `Tiered` consults it before the keyword tier.
86
+ struct AlwaysFiresHigh;
87
+ impl LineImportanceDetector for AlwaysFiresHigh {
88
+ fn score(&self, _line: &str, _ctx: ImportanceContext) -> ImportanceSignal {
89
+ ImportanceSignal::matched(ImportanceCategory::Security, 0.99, 0.95)
90
+ }
91
+ }
92
+
93
+ /// Synthetic low-confidence detector. Confidence 0.5 is below the
94
+ /// escalate threshold so `Tiered` MUST fall through to the next
95
+ /// tier.
96
+ struct AlwaysFiresLow;
97
+ impl LineImportanceDetector for AlwaysFiresLow {
98
+ fn score(&self, _line: &str, _ctx: ImportanceContext) -> ImportanceSignal {
99
+ ImportanceSignal::matched(ImportanceCategory::Importance, 0.4, 0.5)
100
+ }
101
+ }
102
+
103
+ #[test]
104
+ fn high_confidence_tier_short_circuits() {
105
+ let tiered: Tiered<dyn LineImportanceDetector> = Tiered::new()
106
+ .with_detector(AlwaysFiresHigh)
107
+ .with_detector(KeywordDetector::new());
108
+ let s = tiered.score("ERROR: connection refused", ImportanceContext::Diff);
109
+ // AlwaysFiresHigh asserts Security; if the keyword detector ran
110
+ // it would have asserted Error.
111
+ assert_eq!(s.category, Some(ImportanceCategory::Security));
112
+ }
113
+
114
+ #[test]
115
+ fn low_confidence_tier_falls_through_to_keyword() {
116
+ let tiered: Tiered<dyn LineImportanceDetector> = Tiered::new()
117
+ .with_detector(AlwaysFiresLow)
118
+ .with_detector(KeywordDetector::new());
119
+ let s = tiered.score("ERROR: connection refused", ImportanceContext::Diff);
120
+ assert_eq!(s.category, Some(ImportanceCategory::Error));
121
+ }
122
+
123
+ #[test]
124
+ fn no_tier_matches_returns_best_seen() {
125
+ let tiered: Tiered<dyn LineImportanceDetector> = Tiered::new()
126
+ .with_detector(AlwaysFiresLow)
127
+ .with_detector(KeywordDetector::new());
128
+ let s = tiered.score("the quick brown fox", ImportanceContext::Text);
129
+ // Keyword detector returns neutral (confidence 0.0); AlwaysFiresLow
130
+ // returned Importance @ 0.5 so that wins as best-seen.
131
+ assert_eq!(s.category, Some(ImportanceCategory::Importance));
132
+ assert_eq!(s.confidence, 0.5);
133
+ }
134
+
135
+ #[test]
136
+ fn empty_stack_returns_neutral() {
137
+ let tiered: Tiered<dyn LineImportanceDetector> = Tiered::new();
138
+ let s = tiered.score("anything", ImportanceContext::Text);
139
+ assert!(!s.is_match());
140
+ }
141
+ }
@@ -0,0 +1,152 @@
1
+ //! Character-density estimator. Used as a fallback for any tokenizer family
2
+ //! we haven't wired in yet (Anthropic Claude, Google Gemini, Cohere, …).
3
+ //!
4
+ //! Mirrors `headroom.tokenizers.estimator.EstimatingTokenCounter`. The formula
5
+ //! is `ceil(chars / chars_per_token)`. `chars` is *Unicode scalar count*, not
6
+ //! byte length, to match Python's `len(text)` semantics on str.
7
+
8
+ use super::{Backend, Tokenizer};
9
+
10
+ #[derive(Debug, Clone, Copy)]
11
+ pub struct EstimatingCounter {
12
+ chars_per_token: f64,
13
+ }
14
+
15
+ impl Default for EstimatingCounter {
16
+ fn default() -> Self {
17
+ Self {
18
+ chars_per_token: 4.0,
19
+ }
20
+ }
21
+ }
22
+
23
+ impl EstimatingCounter {
24
+ /// `chars_per_token` must be `> 0.0`. Common calibrations:
25
+ /// - 3.5 — Claude-family (Python uses this in `_create_anthropic`)
26
+ /// - 4.0 — Gemini, Cohere, generic fallback
27
+ pub fn new(chars_per_token: f64) -> Self {
28
+ assert!(
29
+ chars_per_token > 0.0,
30
+ "chars_per_token must be positive, got {chars_per_token}"
31
+ );
32
+ Self { chars_per_token }
33
+ }
34
+
35
+ pub fn chars_per_token(&self) -> f64 {
36
+ self.chars_per_token
37
+ }
38
+ }
39
+
40
+ impl Tokenizer for EstimatingCounter {
41
+ fn count_text(&self, text: &str) -> usize {
42
+ if text.is_empty() {
43
+ return 0;
44
+ }
45
+ // Match Python `EstimatingTokenCounter.count_text`:
46
+ // max(1, int(len(text) / chars_per_token + 0.5))
47
+ // Python `int()` truncates toward zero; for non-negative inputs that's
48
+ // identical to `as usize` saturating-cast semantics in Rust >= 1.45.
49
+ // Adding 0.5 then truncating yields round-half-up. We previously used
50
+ // ceil, which over-counted in the middle of the range (e.g. "aaaaa"
51
+ // at 4.0 cpt returned 2 here vs 1 in Python).
52
+ let chars = text.chars().count() as f64;
53
+ let raw = (chars / self.chars_per_token + 0.5) as usize;
54
+ raw.max(1)
55
+ }
56
+
57
+ fn backend(&self) -> Backend {
58
+ Backend::Estimation
59
+ }
60
+ }
61
+
62
+ #[cfg(test)]
63
+ mod tests {
64
+ use super::*;
65
+
66
+ #[test]
67
+ fn empty_string_is_zero() {
68
+ let est = EstimatingCounter::default();
69
+ assert_eq!(est.count_text(""), 0);
70
+ }
71
+
72
+ /// Expected values cross-checked against Python:
73
+ /// max(1, int(len / chars_per_token + 0.5))
74
+ #[test]
75
+ fn default_is_four_chars_per_token() {
76
+ let est = EstimatingCounter::default();
77
+ // 4 / 4.0 = 1.0 + 0.5 -> int(1.5) -> 1
78
+ assert_eq!(est.count_text("aaaa"), 1);
79
+ // 5 / 4.0 = 1.25 + 0.5 -> int(1.75) -> 1 (Python rounds half-up)
80
+ assert_eq!(est.count_text("aaaaa"), 1);
81
+ // 6 / 4.0 = 1.5 + 0.5 -> int(2.0) -> 2
82
+ assert_eq!(est.count_text("aaaaaa"), 2);
83
+ // 40 / 4.0 = 10.0 + 0.5 -> int(10.5) -> 10
84
+ assert_eq!(est.count_text(&"a".repeat(40)), 10);
85
+ }
86
+
87
+ #[test]
88
+ fn claude_density_matches_python() {
89
+ let est = EstimatingCounter::new(3.5);
90
+ // 35 / 3.5 = 10.0 -> int(10.5) -> 10
91
+ assert_eq!(est.count_text(&"a".repeat(35)), 10);
92
+ // 36 / 3.5 ≈ 10.286 -> int(10.786) -> 10 (NOT 11 — that was ceil)
93
+ assert_eq!(est.count_text(&"a".repeat(36)), 10);
94
+ // 38 / 3.5 ≈ 10.857 -> int(11.357) -> 11
95
+ assert_eq!(est.count_text(&"a".repeat(38)), 11);
96
+ }
97
+
98
+ #[test]
99
+ fn unicode_uses_char_count_not_bytes() {
100
+ let est = EstimatingCounter::default();
101
+ // "héllo" is 5 chars, 6 bytes; 5/4.0 = 1.25 -> int(1.75) -> 1
102
+ assert_eq!(est.count_text("héllo"), 1);
103
+ // 4 emojis = 4 chars; 4/4.0 = 1.0 -> int(1.5) -> 1
104
+ assert_eq!(est.count_text("🦀🦀🦀🦀"), 1);
105
+ }
106
+
107
+ #[test]
108
+ fn min_is_one_for_non_empty_input() {
109
+ let est = EstimatingCounter::default();
110
+ // 1/4.0 = 0.25 + 0.5 -> int(0.75) -> 0; max(1, 0) -> 1
111
+ assert_eq!(est.count_text("a"), 1);
112
+ assert_eq!(est.count_text("ab"), 1);
113
+ // 3/4.0 = 0.75 + 0.5 -> int(1.25) -> 1
114
+ assert_eq!(est.count_text("abc"), 1);
115
+ // 6/4.0 = 1.5 + 0.5 -> int(2.0) -> 2
116
+ assert_eq!(est.count_text("aaaaaa"), 2);
117
+ }
118
+
119
+ #[test]
120
+ fn deterministic() {
121
+ let est = EstimatingCounter::default();
122
+ let s = "the quick brown fox jumps over the lazy dog";
123
+ let a = est.count_text(s);
124
+ let b = est.count_text(s);
125
+ assert_eq!(a, b);
126
+ }
127
+
128
+ #[test]
129
+ fn very_long_input_does_not_overflow() {
130
+ let est = EstimatingCounter::default();
131
+ let s = "a".repeat(1_000_000);
132
+ assert_eq!(est.count_text(&s), 250_000);
133
+ }
134
+
135
+ #[test]
136
+ #[should_panic(expected = "chars_per_token must be positive")]
137
+ fn rejects_zero_density() {
138
+ let _ = EstimatingCounter::new(0.0);
139
+ }
140
+
141
+ #[test]
142
+ #[should_panic(expected = "chars_per_token must be positive")]
143
+ fn rejects_negative_density() {
144
+ let _ = EstimatingCounter::new(-1.0);
145
+ }
146
+
147
+ #[test]
148
+ fn backend_is_estimation() {
149
+ let est = EstimatingCounter::default();
150
+ assert_eq!(est.backend(), Backend::Estimation);
151
+ }
152
+ }
@@ -0,0 +1,314 @@
1
+ //! HuggingFace `tokenizers`-crate adapter implementing [`Tokenizer`].
2
+ //!
3
+ //! Loads a `tokenizer.json` (HuggingFace's serialization format) and counts
4
+ //! tokens via real BPE / Unigram / WordPiece — whatever the file describes.
5
+ //! This closes the gap between OpenAI (tiktoken, byte-equal) and the
6
+ //! Anthropic/Google chars-per-token fallback: every other major family
7
+ //! (Cohere `command-*`, Llama-3.x, Mistral, Qwen, BERT, T5, …) publishes a
8
+ //! `tokenizer.json` on the HuggingFace Hub and the `tokenizers` crate is a
9
+ //! pure-Rust loader, so we don't have to estimate.
10
+ //!
11
+ //! # Sources
12
+ //! - [`HfTokenizer::from_bytes`] — for tokenizers embedded via `include_bytes!`.
13
+ //! - [`HfTokenizer::from_file`] — for a local `tokenizer.json`.
14
+ //! - [`HfTokenizer::from_pretrained`] — pulls `tokenizer.json` from the
15
+ //! HuggingFace Hub via `hf-hub`. First call downloads to
16
+ //! `~/.cache/huggingface/hub`, subsequent calls hit the cache. Blocking;
17
+ //! call from `main()` or `tokio::task::spawn_blocking`. Gated repos
18
+ //! (Llama, Mistral) require an `HF_TOKEN` env var or a token in
19
+ //! `~/.cache/huggingface/token`.
20
+ //!
21
+ //! # What's NOT here
22
+ //! - **No tokenizer.json bundled in the binary.** Bundling Llama / Cohere
23
+ //! tokenizers would add several MB of binary bloat for code paths most users
24
+ //! don't hit. `from_pretrained` lazily downloads instead.
25
+
26
+ use std::path::Path;
27
+ use std::sync::Arc;
28
+
29
+ use thiserror::Error;
30
+ use tokenizers::Tokenizer as HfInner;
31
+
32
+ use super::{Backend, Tokenizer};
33
+
34
+ #[derive(Debug, Error)]
35
+ pub enum HfTokenizerError {
36
+ /// The bytes / file did not parse as a valid HuggingFace `tokenizer.json`,
37
+ /// or the model component referenced an unsupported algorithm.
38
+ #[error("failed to load tokenizer for `{name}`: {source}")]
39
+ Load {
40
+ name: String,
41
+ #[source]
42
+ source: Box<dyn std::error::Error + Send + Sync>,
43
+ },
44
+ /// The HuggingFace Hub fetch failed: network error, 404 on the repo, or
45
+ /// 401 on a gated model without an `HF_TOKEN`.
46
+ #[error("failed to download `{repo}` from HuggingFace Hub: {source}")]
47
+ Hub {
48
+ repo: String,
49
+ #[source]
50
+ source: Box<dyn std::error::Error + Send + Sync>,
51
+ },
52
+ }
53
+
54
+ /// Token counter backed by a HuggingFace `tokenizer.json`.
55
+ ///
56
+ /// Cheap to clone — internally an `Arc<tokenizers::Tokenizer>`. Construct once
57
+ /// at startup, share across handlers.
58
+ #[derive(Clone)]
59
+ pub struct HfTokenizer {
60
+ name: String,
61
+ inner: Arc<HfInner>,
62
+ }
63
+
64
+ impl std::fmt::Debug for HfTokenizer {
65
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
66
+ f.debug_struct("HfTokenizer")
67
+ .field("name", &self.name)
68
+ .finish()
69
+ }
70
+ }
71
+
72
+ impl HfTokenizer {
73
+ /// Build from raw `tokenizer.json` bytes. Use this when the tokenizer is
74
+ /// embedded via `include_bytes!` or fetched from a non-HF source.
75
+ pub fn from_bytes(name: impl Into<String>, bytes: &[u8]) -> Result<Self, HfTokenizerError> {
76
+ let name = name.into();
77
+ let inner = HfInner::from_bytes(bytes).map_err(|e| HfTokenizerError::Load {
78
+ name: name.clone(),
79
+ source: e,
80
+ })?;
81
+ Ok(Self {
82
+ name,
83
+ inner: Arc::new(inner),
84
+ })
85
+ }
86
+
87
+ /// Build from a `tokenizer.json` on disk.
88
+ pub fn from_file(
89
+ name: impl Into<String>,
90
+ path: impl AsRef<Path>,
91
+ ) -> Result<Self, HfTokenizerError> {
92
+ let name = name.into();
93
+ let inner = HfInner::from_file(path.as_ref()).map_err(|e| HfTokenizerError::Load {
94
+ name: name.clone(),
95
+ source: e,
96
+ })?;
97
+ Ok(Self {
98
+ name,
99
+ inner: Arc::new(inner),
100
+ })
101
+ }
102
+
103
+ /// Download (or fetch from cache) `tokenizer.json` for `repo` from the
104
+ /// HuggingFace Hub and load it. `repo` is the canonical Hub identifier,
105
+ /// e.g. `"CohereForAI/c4ai-command-r-v01"` or `"meta-llama/Meta-Llama-3-8B"`.
106
+ ///
107
+ /// Uses the `main` revision. Blocking — calls into `ureq` synchronously.
108
+ /// First successful call writes the file to `~/.cache/huggingface/hub`
109
+ /// (or `$HF_HOME` if set); subsequent calls in the same or later
110
+ /// processes hit the on-disk cache.
111
+ ///
112
+ /// Errors:
113
+ /// - [`HfTokenizerError::Hub`] for download failures (no network, 404,
114
+ /// 401 on a gated model without `HF_TOKEN`).
115
+ /// - [`HfTokenizerError::Load`] if the downloaded bytes don't parse as
116
+ /// a valid `tokenizer.json`. Should not happen for healthy HF repos.
117
+ pub fn from_pretrained(repo: &str) -> Result<Self, HfTokenizerError> {
118
+ let api = hf_hub::api::sync::Api::new().map_err(|e| HfTokenizerError::Hub {
119
+ repo: repo.to_string(),
120
+ source: Box::new(e),
121
+ })?;
122
+ let path = api
123
+ .model(repo.to_string())
124
+ .get("tokenizer.json")
125
+ .map_err(|e| HfTokenizerError::Hub {
126
+ repo: repo.to_string(),
127
+ source: Box::new(e),
128
+ })?;
129
+ // `get` returns the on-disk path; reuse `from_file` to keep the load
130
+ // path identical to user-supplied tokenizer.json files.
131
+ Self::from_file(repo, path)
132
+ }
133
+
134
+ /// The logical name this tokenizer was registered under (e.g.
135
+ /// `"command-r-plus"`). Used in logs and metrics.
136
+ pub fn name(&self) -> &str {
137
+ &self.name
138
+ }
139
+ }
140
+
141
+ impl Tokenizer for HfTokenizer {
142
+ fn count_text(&self, text: &str) -> usize {
143
+ if text.is_empty() {
144
+ return 0;
145
+ }
146
+ // `add_special_tokens=false` matches the spirit of
147
+ // `tiktoken.encode_ordinary`: count *content* tokens, leaving
148
+ // BOS/EOS/CLS/SEP padding to be added (or not) by the upstream API.
149
+ // Different providers add different specials, so counting them here
150
+ // would systematically over-charge. Documented for future readers.
151
+ match self.inner.encode(text, false) {
152
+ Ok(enc) => enc.len(),
153
+ // `encode` only fails for malformed inputs that pass UTF-8 but
154
+ // violate the tokenizer's constraints (e.g. a normalizer that
155
+ // rejects certain code points). We degrade to "0 known tokens"
156
+ // rather than panic — the proxy must keep flowing requests.
157
+ Err(_) => 0,
158
+ }
159
+ }
160
+
161
+ fn backend(&self) -> Backend {
162
+ Backend::HuggingFace
163
+ }
164
+ }
165
+
166
+ #[cfg(test)]
167
+ mod tests {
168
+ use super::*;
169
+
170
+ /// Minimal valid `tokenizer.json`: WordLevel model, Whitespace
171
+ /// pre-tokenizer, three-token vocabulary. Lets us test the API surface
172
+ /// without committing a multi-MB tokenizer fixture or hitting the
173
+ /// HuggingFace Hub.
174
+ const TINY_TOKENIZER_JSON: &str = r#"{
175
+ "version": "1.0",
176
+ "truncation": null,
177
+ "padding": null,
178
+ "added_tokens": [],
179
+ "normalizer": null,
180
+ "pre_tokenizer": {"type": "Whitespace"},
181
+ "post_processor": null,
182
+ "decoder": null,
183
+ "model": {
184
+ "type": "WordLevel",
185
+ "vocab": {"hello": 0, "world": 1, "[UNK]": 2},
186
+ "unk_token": "[UNK]"
187
+ }
188
+ }"#;
189
+
190
+ fn tiny() -> HfTokenizer {
191
+ HfTokenizer::from_bytes("tiny-test", TINY_TOKENIZER_JSON.as_bytes())
192
+ .expect("tiny tokenizer.json parses")
193
+ }
194
+
195
+ #[test]
196
+ fn empty_is_zero() {
197
+ assert_eq!(tiny().count_text(""), 0);
198
+ }
199
+
200
+ #[test]
201
+ fn known_vocab_matches_count() {
202
+ let t = tiny();
203
+ // Each word in the vocab is one token; whitespace splits them.
204
+ assert_eq!(t.count_text("hello"), 1);
205
+ assert_eq!(t.count_text("hello world"), 2);
206
+ assert_eq!(t.count_text("hello world hello"), 3);
207
+ }
208
+
209
+ #[test]
210
+ fn unknown_words_become_unk() {
211
+ // OOV tokens collapse to [UNK] — still 1 per whitespace-split chunk.
212
+ let t = tiny();
213
+ assert_eq!(t.count_text("supercalifragilistic"), 1);
214
+ assert_eq!(t.count_text("foo bar baz"), 3);
215
+ }
216
+
217
+ #[test]
218
+ fn deterministic() {
219
+ let t = tiny();
220
+ let s = "hello world hello world";
221
+ let first = t.count_text(s);
222
+ for _ in 0..100 {
223
+ assert_eq!(t.count_text(s), first);
224
+ }
225
+ }
226
+
227
+ #[test]
228
+ fn unicode_does_not_panic() {
229
+ let t = tiny();
230
+ for s in ["héllo wörld", "你好世界", "🦀 ferris", "\n\t\r"] {
231
+ // We only assert non-panic and a reasonable upper bound; the exact
232
+ // count depends on Whitespace pre-tokenizer behavior, which is
233
+ // tokenizers-crate internal.
234
+ let n = t.count_text(s);
235
+ assert!(n < s.len() * 4 + 10, "absurd count {n} for {s:?}");
236
+ }
237
+ }
238
+
239
+ #[test]
240
+ fn invalid_bytes_returns_error() {
241
+ let r = HfTokenizer::from_bytes("bad", b"not a tokenizer.json");
242
+ assert!(matches!(r, Err(HfTokenizerError::Load { .. })));
243
+ }
244
+
245
+ #[test]
246
+ fn name_round_trips() {
247
+ let t = tiny();
248
+ assert_eq!(t.name(), "tiny-test");
249
+ }
250
+
251
+ #[test]
252
+ fn backend_is_huggingface() {
253
+ assert_eq!(tiny().backend(), Backend::HuggingFace);
254
+ }
255
+
256
+ #[test]
257
+ fn clone_shares_inner() {
258
+ let a = tiny();
259
+ let b = a.clone();
260
+ assert!(Arc::ptr_eq(&a.inner, &b.inner));
261
+ }
262
+
263
+ #[test]
264
+ fn from_file_loads_a_real_file() {
265
+ // Round-trip via a temp file to cover the on-disk constructor.
266
+ use std::io::Write;
267
+ let dir = std::env::temp_dir().join(format!(
268
+ "headroom-hf-test-{}",
269
+ std::time::SystemTime::now()
270
+ .duration_since(std::time::UNIX_EPOCH)
271
+ .unwrap()
272
+ .as_nanos()
273
+ ));
274
+ std::fs::create_dir_all(&dir).unwrap();
275
+ let path = dir.join("tokenizer.json");
276
+ let mut f = std::fs::File::create(&path).unwrap();
277
+ f.write_all(TINY_TOKENIZER_JSON.as_bytes()).unwrap();
278
+ drop(f);
279
+
280
+ let t = HfTokenizer::from_file("from-file", &path).expect("loads");
281
+ assert_eq!(t.count_text("hello world"), 2);
282
+
283
+ let _ = std::fs::remove_dir_all(&dir);
284
+ }
285
+
286
+ /// Network-dependent: hits HuggingFace Hub. Run with
287
+ /// `cargo test -p headroom-core -- --ignored from_pretrained_downloads_real_tokenizer`.
288
+ /// `gpt2` is a small public unauthenticated repo (~1.4 MB tokenizer.json).
289
+ #[test]
290
+ #[ignore = "network-dependent: hits HuggingFace Hub"]
291
+ fn from_pretrained_downloads_real_tokenizer() {
292
+ let t = HfTokenizer::from_pretrained("gpt2").expect("download succeeds");
293
+ // GPT-2 BPE: "hello world" is 2 tokens. Locks in that we got a real
294
+ // BPE tokenizer (not a WhitespaceSplit fixture) from HF.
295
+ assert_eq!(t.count_text("hello world"), 2);
296
+ assert_eq!(t.name(), "gpt2");
297
+ assert_eq!(t.backend(), Backend::HuggingFace);
298
+ }
299
+
300
+ /// Negative path: a malformed repo name fails before any network call
301
+ /// (the Hub URL builder rejects it). No network required to run.
302
+ #[test]
303
+ fn from_pretrained_invalid_repo_returns_hub_error() {
304
+ // Empty repo name — hf-hub rejects this without making a network
305
+ // request. Locks in that we propagate Hub errors as `Hub`, not
306
+ // `Load`, so callers can distinguish "couldn't fetch" from
307
+ // "fetched but malformed".
308
+ let r = HfTokenizer::from_pretrained("");
309
+ assert!(
310
+ matches!(r, Err(HfTokenizerError::Hub { .. })),
311
+ "expected HfTokenizerError::Hub, got {r:?}"
312
+ );
313
+ }
314
+ }
@@ -0,0 +1,56 @@
1
+ //! Token counting for Headroom transforms.
2
+ //!
3
+ //! Mirrors the public surface of the Python `headroom.tokenizers` package:
4
+ //! a `Tokenizer` trait, a tiktoken-backed counter for OpenAI / o-series models
5
+ //! (via the `tiktoken-rs` crate, which uses the same BPE data files as Python's
6
+ //! `tiktoken` and therefore returns byte-identical token IDs), and an estimation
7
+ //! fallback for everything else.
8
+ //!
9
+ //! # Why this exists
10
+ //! Counting tokens currently round-trips into Python's `tiktoken` (itself a
11
+ //! Rust extension under the hood). For Rust transforms running on the proxy
12
+ //! hot path, counting natively avoids the Python-Rust FFI cost and keeps the
13
+ //! Rust binary self-contained.
14
+ //!
15
+ //! # What this is NOT
16
+ //! - Not used by `headroom-proxy` yet (Stage 2 is library-only; no production
17
+ //! wiring).
18
+ //! - Not a real Anthropic Claude tokenizer (Anthropic doesn't publish theirs;
19
+ //! estimation matches what the Python implementation does).
20
+ //! - SentencePiece — Gemini's tokenizer is SP-based but Google doesn't
21
+ //! publish the model. Falls through to estimation.
22
+ //!
23
+ //! # What is here
24
+ //! - [`TiktokenCounter`] — OpenAI / o-series, byte-equal to Python `tiktoken`.
25
+ //! - [`HfTokenizer`] — any model with a public `tokenizer.json` (Cohere
26
+ //! `command-*`, Llama-3.x, Mistral, Qwen, BERT, T5, …). Construct it from
27
+ //! bytes or a file path; register it via [`register_hf`] to make it the
28
+ //! default for a given model-name prefix.
29
+ //! - [`EstimatingCounter`] — last-resort `chars / cpt` fallback for Anthropic
30
+ //! and Gemini, calibrated to match the Python implementation.
31
+
32
+ mod estimator;
33
+ mod hf_impl;
34
+ mod registry;
35
+ mod tiktoken_impl;
36
+
37
+ pub use estimator::EstimatingCounter;
38
+ pub use hf_impl::{HfTokenizer, HfTokenizerError};
39
+ pub use registry::{
40
+ clear_hf_registrations, detect_backend, get_tokenizer, register_hf, try_register_hf, Backend,
41
+ };
42
+ pub use tiktoken_impl::{TiktokenCounter, TiktokenError};
43
+
44
+ /// Counts tokens. Implementations must be thread-safe (`Send + Sync`).
45
+ ///
46
+ /// # Conventions (preserved across all built-in implementations)
47
+ /// - `count_text("")` returns `0`.
48
+ /// - Counts are deterministic for a given input and instance.
49
+ /// - For non-empty input, counts are `>= 1`.
50
+ pub trait Tokenizer: Send + Sync + std::fmt::Debug {
51
+ /// Number of tokens that this tokenizer assigns to `text`.
52
+ fn count_text(&self, text: &str) -> usize;
53
+
54
+ /// Which backend produced the count. Useful for logs and metrics.
55
+ fn backend(&self) -> Backend;
56
+ }