xithead 1.4.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (661) hide show
  1. package/.actrc +14 -0
  2. package/.actrc.local.example +15 -0
  3. package/.changelog.md +2 -0
  4. package/.claude/agents/academic-anthropologist.md +125 -0
  5. package/.claude/agents/academic-geographer.md +127 -0
  6. package/.claude/agents/academic-historian.md +123 -0
  7. package/.claude/agents/academic-narratologist.md +118 -0
  8. package/.claude/agents/academic-psychologist.md +118 -0
  9. package/.claude/agents/academic-statistician.md +144 -0
  10. package/.claude/agents/accounts-payable-agent.md +205 -0
  11. package/.claude/agents/agentic-identity-trust.md +394 -0
  12. package/.claude/agents/agents-orchestrator.md +367 -0
  13. package/.claude/agents/automation-governance-architect.md +216 -0
  14. package/.claude/agents/blender-addon-engineer.md +243 -0
  15. package/.claude/agents/business-strategist.md +488 -0
  16. package/.claude/agents/change-management-consultant.md +497 -0
  17. package/.claude/agents/chief-financial-officer.md +388 -0
  18. package/.claude/agents/corporate-training-designer.md +192 -0
  19. package/.claude/agents/customer-service.md +398 -0
  20. package/.claude/agents/customer-success-manager.md +460 -0
  21. package/.claude/agents/data-consolidation-agent.md +60 -0
  22. package/.claude/agents/data-privacy-officer.md +412 -0
  23. package/.claude/agents/design-brand-guardian.md +322 -0
  24. package/.claude/agents/design-image-prompt-engineer.md +236 -0
  25. package/.claude/agents/design-inclusive-visuals-specialist.md +73 -0
  26. package/.claude/agents/design-persona-walkthrough.md +272 -0
  27. package/.claude/agents/design-ui-designer.md +386 -0
  28. package/.claude/agents/design-ui-finish-gate-reviewer.md +217 -0
  29. package/.claude/agents/design-ux-architect.md +482 -0
  30. package/.claude/agents/design-ux-researcher.md +329 -0
  31. package/.claude/agents/design-visual-storyteller.md +149 -0
  32. package/.claude/agents/design-whimsy-injector.md +460 -0
  33. package/.claude/agents/economy-designer.md +156 -0
  34. package/.claude/agents/engineering-ai-data-remediation-engineer.md +211 -0
  35. package/.claude/agents/engineering-ai-engineer.md +146 -0
  36. package/.claude/agents/engineering-api-platform-engineer.md +181 -0
  37. package/.claude/agents/engineering-ats-validator-architect.md +383 -0
  38. package/.claude/agents/engineering-autonomous-optimization-architect.md +121 -0
  39. package/.claude/agents/engineering-backend-architect.md +255 -0
  40. package/.claude/agents/engineering-china-network-engineer.md +252 -0
  41. package/.claude/agents/engineering-cms-developer.md +536 -0
  42. package/.claude/agents/engineering-code-reviewer.md +76 -0
  43. package/.claude/agents/engineering-codebase-onboarding-engineer.md +173 -0
  44. package/.claude/agents/engineering-data-engineer.md +349 -0
  45. package/.claude/agents/engineering-data-visualization-engineer.md +151 -0
  46. package/.claude/agents/engineering-database-optimizer.md +188 -0
  47. package/.claude/agents/engineering-database-reliability-engineer.md +201 -0
  48. package/.claude/agents/engineering-desktop-app-engineer.md +209 -0
  49. package/.claude/agents/engineering-developer-tooling-engineer.md +153 -0
  50. package/.claude/agents/engineering-devops-automator.md +385 -0
  51. package/.claude/agents/engineering-drupal-performance.md +347 -0
  52. package/.claude/agents/engineering-drupal-shopping-cart.md +360 -0
  53. package/.claude/agents/engineering-electrical-engineer.md +575 -0
  54. package/.claude/agents/engineering-email-intelligence-engineer.md +390 -0
  55. package/.claude/agents/engineering-embedded-firmware-engineer.md +196 -0
  56. package/.claude/agents/engineering-feishu-integration-developer.md +606 -0
  57. package/.claude/agents/engineering-filament-optimization-specialist.md +283 -0
  58. package/.claude/agents/engineering-finops-engineer.md +164 -0
  59. package/.claude/agents/engineering-frontend-developer.md +257 -0
  60. package/.claude/agents/engineering-gaussdb-expert.md +334 -0
  61. package/.claude/agents/engineering-git-workflow-master.md +98 -0
  62. package/.claude/agents/engineering-i18n-engineer.md +214 -0
  63. package/.claude/agents/engineering-identity-access-engineer.md +217 -0
  64. package/.claude/agents/engineering-incident-response-commander.md +444 -0
  65. package/.claude/agents/engineering-iot-fleet-engineer.md +148 -0
  66. package/.claude/agents/engineering-it-service-manager.md +561 -0
  67. package/.claude/agents/engineering-knowledge-graph-engineer.md +372 -0
  68. package/.claude/agents/engineering-llm-post-training-engineer.md +166 -0
  69. package/.claude/agents/engineering-minimal-change-engineer.md +207 -0
  70. package/.claude/agents/engineering-mobile-app-builder.md +493 -0
  71. package/.claude/agents/engineering-mobile-release-engineer.md +164 -0
  72. package/.claude/agents/engineering-multi-agent-systems-architect.md +600 -0
  73. package/.claude/agents/engineering-network-engineer.md +245 -0
  74. package/.claude/agents/engineering-orgscript-engineer.md +113 -0
  75. package/.claude/agents/engineering-payments-billing-engineer.md +228 -0
  76. package/.claude/agents/engineering-pdf-engine-architect.md +696 -0
  77. package/.claude/agents/engineering-platform-engineer.md +270 -0
  78. package/.claude/agents/engineering-privacy-engineer.md +152 -0
  79. package/.claude/agents/engineering-prompt-engineer.md +209 -0
  80. package/.claude/agents/engineering-rag-pipeline-engineer.md +450 -0
  81. package/.claude/agents/engineering-rapid-prototyper.md +466 -0
  82. package/.claude/agents/engineering-realtime-collaboration-engineer.md +200 -0
  83. package/.claude/agents/engineering-rust-refactoring-specialist.md +313 -0
  84. package/.claude/agents/engineering-search-relevance-engineer.md +237 -0
  85. package/.claude/agents/engineering-section-508-specialist.md +339 -0
  86. package/.claude/agents/engineering-senior-developer.md +176 -0
  87. package/.claude/agents/engineering-servicenow-developer-mentor.md +180 -0
  88. package/.claude/agents/engineering-software-architect.md +112 -0
  89. package/.claude/agents/engineering-solidity-smart-contract-engineer.md +522 -0
  90. package/.claude/agents/engineering-sre.md +90 -0
  91. package/.claude/agents/engineering-technical-writer.md +393 -0
  92. package/.claude/agents/engineering-universal-document-compiler.md +370 -0
  93. package/.claude/agents/engineering-uswds-developer.md +340 -0
  94. package/.claude/agents/engineering-video-streaming-engineer.md +151 -0
  95. package/.claude/agents/engineering-voice-ai-integration-engineer.md +584 -0
  96. package/.claude/agents/engineering-webassembly-engineer.md +153 -0
  97. package/.claude/agents/engineering-wechat-mini-program-developer.md +356 -0
  98. package/.claude/agents/engineering-wordpress-performance.md +346 -0
  99. package/.claude/agents/engineering-wordpress-shopping-cart.md +346 -0
  100. package/.claude/agents/esg-sustainability-officer.md +396 -0
  101. package/.claude/agents/finance-bookkeeper-controller.md +260 -0
  102. package/.claude/agents/finance-financial-analyst.md +234 -0
  103. package/.claude/agents/finance-fpa-analyst.md +263 -0
  104. package/.claude/agents/finance-investment-researcher.md +272 -0
  105. package/.claude/agents/finance-tax-strategist.md +239 -0
  106. package/.claude/agents/game-audio-engineer.md +264 -0
  107. package/.claude/agents/game-designer.md +167 -0
  108. package/.claude/agents/gis-3d-scene-developer.md +111 -0
  109. package/.claude/agents/gis-analyst.md +91 -0
  110. package/.claude/agents/gis-bim-specialist.md +108 -0
  111. package/.claude/agents/gis-cartography-designer.md +150 -0
  112. package/.claude/agents/gis-drone-reality-mapping.md +120 -0
  113. package/.claude/agents/gis-geoai-ml-engineer.md +105 -0
  114. package/.claude/agents/gis-geoprocessing-specialist.md +97 -0
  115. package/.claude/agents/gis-qa-engineer.md +133 -0
  116. package/.claude/agents/gis-solution-engineer.md +101 -0
  117. package/.claude/agents/gis-spatial-data-engineer.md +97 -0
  118. package/.claude/agents/gis-spatial-data-scientist.md +111 -0
  119. package/.claude/agents/gis-technical-consultant.md +86 -0
  120. package/.claude/agents/gis-web-gis-developer.md +108 -0
  121. package/.claude/agents/godot-gameplay-scripter.md +338 -0
  122. package/.claude/agents/godot-multiplayer-engineer.md +301 -0
  123. package/.claude/agents/godot-shader-developer.md +267 -0
  124. package/.claude/agents/government-digital-presales-consultant.md +363 -0
  125. package/.claude/agents/grant-writer.md +511 -0
  126. package/.claude/agents/healthcare-aging-parent-care-companion.md +414 -0
  127. package/.claude/agents/healthcare-clinical-evidence-agent.md +231 -0
  128. package/.claude/agents/healthcare-customer-service.md +389 -0
  129. package/.claude/agents/healthcare-innovation-strategist.md +433 -0
  130. package/.claude/agents/healthcare-marketing-compliance.md +395 -0
  131. package/.claude/agents/healthcare-sovereign-health-systems-agent.md +312 -0
  132. package/.claude/agents/hospitality-guest-services.md +603 -0
  133. package/.claude/agents/hr-onboarding.md +451 -0
  134. package/.claude/agents/identity-graph-operator.md +303 -0
  135. package/.claude/agents/language-translator.md +264 -0
  136. package/.claude/agents/legal-billing-time-tracking.md +569 -0
  137. package/.claude/agents/legal-client-intake.md +492 -0
  138. package/.claude/agents/legal-document-review.md +454 -0
  139. package/.claude/agents/level-designer.md +208 -0
  140. package/.claude/agents/loan-officer-assistant.md +555 -0
  141. package/.claude/agents/lsp-index-engineer.md +320 -0
  142. package/.claude/agents/ma-integration-manager.md +427 -0
  143. package/.claude/agents/macos-spatial-metal-engineer.md +337 -0
  144. package/.claude/agents/marketing-aeo-foundations.md +264 -0
  145. package/.claude/agents/marketing-agentic-search-optimizer.md +279 -0
  146. package/.claude/agents/marketing-ai-citation-strategist.md +172 -0
  147. package/.claude/agents/marketing-app-store-optimizer.md +321 -0
  148. package/.claude/agents/marketing-baidu-seo-specialist.md +226 -0
  149. package/.claude/agents/marketing-bilibili-content-strategist.md +199 -0
  150. package/.claude/agents/marketing-book-co-author.md +110 -0
  151. package/.claude/agents/marketing-carousel-growth-engine.md +204 -0
  152. package/.claude/agents/marketing-china-ecommerce-operator.md +283 -0
  153. package/.claude/agents/marketing-china-market-localization-strategist.md +283 -0
  154. package/.claude/agents/marketing-content-creator.md +54 -0
  155. package/.claude/agents/marketing-cross-border-ecommerce.md +259 -0
  156. package/.claude/agents/marketing-developer-community-builder.md +285 -0
  157. package/.claude/agents/marketing-douyin-strategist.md +149 -0
  158. package/.claude/agents/marketing-email-strategist.md +249 -0
  159. package/.claude/agents/marketing-global-podcast-strategist.md +206 -0
  160. package/.claude/agents/marketing-growth-hacker.md +54 -0
  161. package/.claude/agents/marketing-instagram-curator.md +113 -0
  162. package/.claude/agents/marketing-kuaishou-strategist.md +223 -0
  163. package/.claude/agents/marketing-linkedin-content-creator.md +214 -0
  164. package/.claude/agents/marketing-livestream-commerce-coach.md +305 -0
  165. package/.claude/agents/marketing-multi-platform-publisher.md +217 -0
  166. package/.claude/agents/marketing-podcast-strategist.md +277 -0
  167. package/.claude/agents/marketing-pr-communications-manager.md +473 -0
  168. package/.claude/agents/marketing-private-domain-operator.md +308 -0
  169. package/.claude/agents/marketing-reddit-community-builder.md +123 -0
  170. package/.claude/agents/marketing-seo-specialist.md +370 -0
  171. package/.claude/agents/marketing-short-video-editing-coach.md +412 -0
  172. package/.claude/agents/marketing-social-media-strategist.md +125 -0
  173. package/.claude/agents/marketing-tiktok-strategist.md +125 -0
  174. package/.claude/agents/marketing-twitter-engager.md +126 -0
  175. package/.claude/agents/marketing-video-optimization-specialist.md +119 -0
  176. package/.claude/agents/marketing-wechat-official-account.md +145 -0
  177. package/.claude/agents/marketing-weibo-strategist.md +240 -0
  178. package/.claude/agents/marketing-x-twitter-intelligence-analyst.md +161 -0
  179. package/.claude/agents/marketing-xiaohongshu-specialist.md +138 -0
  180. package/.claude/agents/marketing-zhihu-strategist.md +162 -0
  181. package/.claude/agents/medical-billing-coding-specialist.md +491 -0
  182. package/.claude/agents/narrative-designer.md +243 -0
  183. package/.claude/agents/operations-manager.md +399 -0
  184. package/.claude/agents/organizational-psychologist.md +391 -0
  185. package/.claude/agents/paid-media-auditor.md +71 -0
  186. package/.claude/agents/paid-media-creative-strategist.md +71 -0
  187. package/.claude/agents/paid-media-paid-social-strategist.md +71 -0
  188. package/.claude/agents/paid-media-ppc-strategist.md +71 -0
  189. package/.claude/agents/paid-media-programmatic-buyer.md +71 -0
  190. package/.claude/agents/paid-media-search-query-analyst.md +71 -0
  191. package/.claude/agents/paid-media-tracking-specialist.md +71 -0
  192. package/.claude/agents/personal-growth-mentor.md +159 -0
  193. package/.claude/agents/product-behavioral-nudge-engine.md +83 -0
  194. package/.claude/agents/product-dx-engineer.md +332 -0
  195. package/.claude/agents/product-feedback-synthesizer.md +119 -0
  196. package/.claude/agents/product-manager.md +469 -0
  197. package/.claude/agents/product-sprint-prioritizer.md +154 -0
  198. package/.claude/agents/product-trend-researcher.md +159 -0
  199. package/.claude/agents/project-management-experiment-tracker.md +198 -0
  200. package/.claude/agents/project-management-jira-workflow-steward.md +233 -0
  201. package/.claude/agents/project-management-meeting-notes-specialist.md +95 -0
  202. package/.claude/agents/project-management-project-shepherd.md +194 -0
  203. package/.claude/agents/project-management-studio-operations.md +200 -0
  204. package/.claude/agents/project-management-studio-producer.md +203 -0
  205. package/.claude/agents/project-manager-senior.md +135 -0
  206. package/.claude/agents/real-estate-buyer-seller.md +596 -0
  207. package/.claude/agents/recruitment-specialist.md +515 -0
  208. package/.claude/agents/report-distribution-agent.md +65 -0
  209. package/.claude/agents/research-synthesist.md +136 -0
  210. package/.claude/agents/resume-tailor.md +230 -0
  211. package/.claude/agents/retail-customer-returns.md +566 -0
  212. package/.claude/agents/roblox-avatar-creator.md +297 -0
  213. package/.claude/agents/roblox-experience-designer.md +306 -0
  214. package/.claude/agents/roblox-systems-scripter.md +332 -0
  215. package/.claude/agents/sales-account-strategist.md +227 -0
  216. package/.claude/agents/sales-coach.md +271 -0
  217. package/.claude/agents/sales-data-extraction-agent.md +67 -0
  218. package/.claude/agents/sales-deal-strategist.md +180 -0
  219. package/.claude/agents/sales-discovery-coach.md +225 -0
  220. package/.claude/agents/sales-engineer.md +182 -0
  221. package/.claude/agents/sales-offer-lead-gen-strategist.md +257 -0
  222. package/.claude/agents/sales-outbound-strategist.md +201 -0
  223. package/.claude/agents/sales-outreach.md +425 -0
  224. package/.claude/agents/sales-pipeline-analyst.md +267 -0
  225. package/.claude/agents/sales-proposal-strategist.md +217 -0
  226. package/.claude/agents/security-ai-generated-code-auditor.md +207 -0
  227. package/.claude/agents/security-appsec-engineer.md +536 -0
  228. package/.claude/agents/security-architect.md +307 -0
  229. package/.claude/agents/security-blockchain-security-auditor.md +482 -0
  230. package/.claude/agents/security-cloud-security-architect.md +576 -0
  231. package/.claude/agents/security-compliance-auditor.md +158 -0
  232. package/.claude/agents/security-incident-responder.md +450 -0
  233. package/.claude/agents/security-penetration-tester.md +399 -0
  234. package/.claude/agents/security-secrets-credential-engineer.md +189 -0
  235. package/.claude/agents/security-senior-secops.md +760 -0
  236. package/.claude/agents/security-threat-detection-engineer.md +553 -0
  237. package/.claude/agents/security-threat-intelligence-analyst.md +661 -0
  238. package/.claude/agents/specialized-chief-of-staff.md +279 -0
  239. package/.claude/agents/specialized-civil-engineer.md +356 -0
  240. package/.claude/agents/specialized-codebase-archaeologist.md +341 -0
  241. package/.claude/agents/specialized-cultural-intelligence-strategist.md +88 -0
  242. package/.claude/agents/specialized-developer-advocate.md +317 -0
  243. package/.claude/agents/specialized-document-generator.md +55 -0
  244. package/.claude/agents/specialized-fedramp-rmf-compliance.md +378 -0
  245. package/.claude/agents/specialized-focus-music-architect.md +175 -0
  246. package/.claude/agents/specialized-french-consulting-market.md +194 -0
  247. package/.claude/agents/specialized-korean-business-navigator.md +216 -0
  248. package/.claude/agents/specialized-master-plan-architect.md +157 -0
  249. package/.claude/agents/specialized-mcp-builder.md +260 -0
  250. package/.claude/agents/specialized-model-qa.md +528 -0
  251. package/.claude/agents/specialized-pricing-analyst.md +243 -0
  252. package/.claude/agents/specialized-salesforce-architect.md +182 -0
  253. package/.claude/agents/specialized-strategy-duel-agent.md +130 -0
  254. package/.claude/agents/specialized-workflow-architect.md +597 -0
  255. package/.claude/agents/study-abroad-advisor.md +282 -0
  256. package/.claude/agents/supply-chain-strategist.md +591 -0
  257. package/.claude/agents/support-analytics-reporter.md +387 -0
  258. package/.claude/agents/support-executive-summary-generator.md +212 -0
  259. package/.claude/agents/support-finance-tracker.md +480 -0
  260. package/.claude/agents/support-infrastructure-maintainer.md +628 -0
  261. package/.claude/agents/support-legal-compliance-checker.md +591 -0
  262. package/.claude/agents/support-support-responder.md +607 -0
  263. package/.claude/agents/technical-artist.md +233 -0
  264. package/.claude/agents/terminal-integration-specialist.md +70 -0
  265. package/.claude/agents/testing-accessibility-auditor.md +327 -0
  266. package/.claude/agents/testing-api-tester.md +334 -0
  267. package/.claude/agents/testing-evidence-collector.md +210 -0
  268. package/.claude/agents/testing-performance-benchmarker.md +283 -0
  269. package/.claude/agents/testing-reality-checker.md +249 -0
  270. package/.claude/agents/testing-test-automation-engineer.md +196 -0
  271. package/.claude/agents/testing-test-results-analyzer.md +330 -0
  272. package/.claude/agents/testing-tool-evaluator.md +416 -0
  273. package/.claude/agents/testing-workflow-optimizer.md +463 -0
  274. package/.claude/agents/unity-architect.md +271 -0
  275. package/.claude/agents/unity-editor-tool-developer.md +310 -0
  276. package/.claude/agents/unity-multiplayer-engineer.md +323 -0
  277. package/.claude/agents/unity-shader-graph-artist.md +269 -0
  278. package/.claude/agents/unreal-multiplayer-architect.md +364 -0
  279. package/.claude/agents/unreal-systems-engineer.md +322 -0
  280. package/.claude/agents/unreal-technical-artist.md +256 -0
  281. package/.claude/agents/unreal-world-builder.md +273 -0
  282. package/.claude/agents/visionos-spatial-engineer.md +54 -0
  283. package/.claude/agents/xr-cockpit-interaction-specialist.md +32 -0
  284. package/.claude/agents/xr-immersive-developer.md +32 -0
  285. package/.claude/agents/xr-interface-architect.md +32 -0
  286. package/.claude/agents/zk-steward.md +211 -0
  287. package/.claude-plugin/marketplace.json +30 -0
  288. package/.commitlintrc.json +26 -0
  289. package/.devcontainer/Dockerfile +24 -0
  290. package/.devcontainer/devcontainer.json +70 -0
  291. package/.devcontainer/docker-compose.memory.yml +42 -0
  292. package/.devcontainer/memory-stack/devcontainer.json +84 -0
  293. package/.devcontainer/post-create.sh +83 -0
  294. package/.dockerignore +23 -0
  295. package/.env.act.example +15 -0
  296. package/.env.example +171 -0
  297. package/.eslintrc.json +16 -0
  298. package/.git-blame-ignore-revs +10 -0
  299. package/.gitattributes +2 -0
  300. package/.gitguardian.yaml +50 -0
  301. package/.githooks/pre-commit +6 -0
  302. package/.pre-commit-config.yaml +30 -0
  303. package/.release-please-config-xithead.json +22 -0
  304. package/.release-please-config.json +44 -0
  305. package/.release-please-manifest-xithead.json +3 -0
  306. package/.release-please-manifest.json +3 -0
  307. package/CODE_OF_CONDUCT.md +133 -0
  308. package/CONTRIBUTING.md +133 -0
  309. package/Cargo.lock +5557 -0
  310. package/Cargo.toml +121 -0
  311. package/Dockerfile +38 -0
  312. package/Dockerfile.proxy +48 -0
  313. package/ENTERPRISE.md +5 -0
  314. package/FEATURES_ADVANCED.md +311 -0
  315. package/Headroom-2.gif +0 -0
  316. package/HeadroomDemo-Fast.gif +0 -0
  317. package/MANIFEST.in +4 -0
  318. package/MODERNIZATION_GUIDE.md +358 -0
  319. package/Makefile +140 -0
  320. package/NOTICE +43 -0
  321. package/NPM_TOKEN_SETUP.md +81 -0
  322. package/PR.md +170 -0
  323. package/PUBLISH.md +77 -0
  324. package/README.md +108 -72
  325. package/README_FRONTEND.md +196 -0
  326. package/REALIGNMENT/00-overview.md +53 -0
  327. package/REALIGNMENT/01-bug-list.md +421 -0
  328. package/REALIGNMENT/02-architecture.md +322 -0
  329. package/REALIGNMENT/03-phase-A-lockdown.md +420 -0
  330. package/REALIGNMENT/04-phase-B-live-zone.md +470 -0
  331. package/REALIGNMENT/05-phase-C-rust-proxy.md +329 -0
  332. package/REALIGNMENT/06-phase-D-bedrock-vertex.md +222 -0
  333. package/REALIGNMENT/07-phase-E-cache-stabilization.md +338 -0
  334. package/REALIGNMENT/08-phase-F-auth-mode.md +254 -0
  335. package/REALIGNMENT/09-phase-G-rtk-observability.md +194 -0
  336. package/REALIGNMENT/10-phase-H-python-retirement.md +214 -0
  337. package/REALIGNMENT/11-phase-I-test-infra.md +385 -0
  338. package/REALIGNMENT/12-decisions-needed.md +196 -0
  339. package/REALIGNMENT/INDEX.md +82 -0
  340. package/REFACTORING_CAMPAIGN.md +277 -0
  341. package/RESTRUCTURE_PLAN.md +100 -0
  342. package/RUST_DEV.md +361 -0
  343. package/SECURITY.md +65 -0
  344. package/SETUP_AUTO_PUBLISH.md +164 -0
  345. package/TESTING-copilot-subscription.md +153 -0
  346. package/app/api/config/route.ts +22 -0
  347. package/app/api/export/route.ts +83 -0
  348. package/app/api/models/route.ts +12 -0
  349. package/app/api/stats/route.ts +46 -0
  350. package/app/dashboard/page.tsx +445 -0
  351. package/app/globals.css +35 -0
  352. package/app/installation/page.tsx +293 -0
  353. package/app/layout.tsx +35 -0
  354. package/app/master-design-system.css +2076 -0
  355. package/app/master-design-system.source.css +8 -0
  356. package/app/page.tsx +187 -0
  357. package/app/rechtliches/agb/page.tsx +165 -0
  358. package/app/rechtliches/datenschutz/page.tsx +128 -0
  359. package/app/rechtliches/impressum/page.tsx +122 -0
  360. package/app/rechtliches/widerruf/page.tsx +185 -0
  361. package/bin/xithead.js +264 -0
  362. package/codecov.yml +34 -0
  363. package/components/layout/AppShell.tsx +25 -0
  364. package/components/layout/Footer.tsx +120 -0
  365. package/components/layout/Header.tsx +145 -0
  366. package/components/marketing/LegalHighlights.tsx +45 -0
  367. package/components/marketing/LegalPageShell.tsx +49 -0
  368. package/components/marketing/LegalPanel.tsx +41 -0
  369. package/components/marketing/index.ts +3 -0
  370. package/crates/headroom-core/Cargo.toml +165 -0
  371. package/crates/headroom-core/benches/auth_mode.rs +74 -0
  372. package/crates/headroom-core/benches/ccr_store.rs +224 -0
  373. package/crates/headroom-core/benches/tokenizer.rs +52 -0
  374. package/crates/headroom-core/config/pipeline.toml +151 -0
  375. package/crates/headroom-core/proptest-regressions/transforms/tag_protector.txt +9 -0
  376. package/crates/headroom-core/src/auth_mode.rs +250 -0
  377. package/crates/headroom-core/src/cache_control.rs +414 -0
  378. package/crates/headroom-core/src/ccr/backends/in_memory.rs +317 -0
  379. package/crates/headroom-core/src/ccr/backends/mod.rs +152 -0
  380. package/crates/headroom-core/src/ccr/backends/redis.rs +146 -0
  381. package/crates/headroom-core/src/ccr/backends/sqlite.rs +205 -0
  382. package/crates/headroom-core/src/ccr/mod.rs +119 -0
  383. package/crates/headroom-core/src/compression_policy.rs +505 -0
  384. package/crates/headroom-core/src/lib.rs +33 -0
  385. package/crates/headroom-core/src/relevance/base.rs +139 -0
  386. package/crates/headroom-core/src/relevance/bm25.rs +385 -0
  387. package/crates/headroom-core/src/relevance/embedding.rs +354 -0
  388. package/crates/headroom-core/src/relevance/hybrid.rs +364 -0
  389. package/crates/headroom-core/src/relevance/mod.rs +64 -0
  390. package/crates/headroom-core/src/signals/keyword_detector.rs +433 -0
  391. package/crates/headroom-core/src/signals/line_importance.rs +84 -0
  392. package/crates/headroom-core/src/signals/mod.rs +61 -0
  393. package/crates/headroom-core/src/signals/tiered.rs +141 -0
  394. package/crates/headroom-core/src/tokenizer/estimator.rs +152 -0
  395. package/crates/headroom-core/src/tokenizer/hf_impl.rs +314 -0
  396. package/crates/headroom-core/src/tokenizer/mod.rs +56 -0
  397. package/crates/headroom-core/src/tokenizer/registry.rs +342 -0
  398. package/crates/headroom-core/src/tokenizer/tiktoken_impl.rs +265 -0
  399. package/crates/headroom-core/src/transforms/adaptive_sizer.rs +610 -0
  400. package/crates/headroom-core/src/transforms/anchor_selector.rs +1189 -0
  401. package/crates/headroom-core/src/transforms/content_detector.rs +769 -0
  402. package/crates/headroom-core/src/transforms/detection.rs +222 -0
  403. package/crates/headroom-core/src/transforms/diff_compressor.rs +1685 -0
  404. package/crates/headroom-core/src/transforms/live_zone.rs +2967 -0
  405. package/crates/headroom-core/src/transforms/log_compressor.rs +1295 -0
  406. package/crates/headroom-core/src/transforms/magika_detector.rs +395 -0
  407. package/crates/headroom-core/src/transforms/mod.rs +64 -0
  408. package/crates/headroom-core/src/transforms/pipeline/config.rs +331 -0
  409. package/crates/headroom-core/src/transforms/pipeline/mod.rs +103 -0
  410. package/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rs +524 -0
  411. package/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs +279 -0
  412. package/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs +400 -0
  413. package/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rs +284 -0
  414. package/crates/headroom-core/src/transforms/pipeline/offloads/mod.rs +34 -0
  415. package/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rs +331 -0
  416. package/crates/headroom-core/src/transforms/pipeline/orchestrator.rs +848 -0
  417. package/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rs +175 -0
  418. package/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rs +545 -0
  419. package/crates/headroom-core/src/transforms/pipeline/reformats/mod.rs +12 -0
  420. package/crates/headroom-core/src/transforms/pipeline/traits.rs +338 -0
  421. package/crates/headroom-core/src/transforms/recommendations.rs +329 -0
  422. package/crates/headroom-core/src/transforms/safety.rs +215 -0
  423. package/crates/headroom-core/src/transforms/search_compressor.rs +902 -0
  424. package/crates/headroom-core/src/transforms/smart_crusher/analyzer.rs +1262 -0
  425. package/crates/headroom-core/src/transforms/smart_crusher/anchors.rs +407 -0
  426. package/crates/headroom-core/src/transforms/smart_crusher/builder.rs +263 -0
  427. package/crates/headroom-core/src/transforms/smart_crusher/classifier.rs +214 -0
  428. package/crates/headroom-core/src/transforms/smart_crusher/compaction/classifier.rs +283 -0
  429. package/crates/headroom-core/src/transforms/smart_crusher/compaction/compactor.rs +733 -0
  430. package/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rs +961 -0
  431. package/crates/headroom-core/src/transforms/smart_crusher/compaction/ir.rs +252 -0
  432. package/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs +127 -0
  433. package/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs +397 -0
  434. package/crates/headroom-core/src/transforms/smart_crusher/config.rs +173 -0
  435. package/crates/headroom-core/src/transforms/smart_crusher/constraints.rs +159 -0
  436. package/crates/headroom-core/src/transforms/smart_crusher/crusher.rs +1665 -0
  437. package/crates/headroom-core/src/transforms/smart_crusher/crushers.rs +865 -0
  438. package/crates/headroom-core/src/transforms/smart_crusher/error_keywords.rs +77 -0
  439. package/crates/headroom-core/src/transforms/smart_crusher/field_detect.rs +393 -0
  440. package/crates/headroom-core/src/transforms/smart_crusher/hashing.rs +72 -0
  441. package/crates/headroom-core/src/transforms/smart_crusher/mod.rs +81 -0
  442. package/crates/headroom-core/src/transforms/smart_crusher/observer.rs +66 -0
  443. package/crates/headroom-core/src/transforms/smart_crusher/orchestration.rs +459 -0
  444. package/crates/headroom-core/src/transforms/smart_crusher/outliers.rs +507 -0
  445. package/crates/headroom-core/src/transforms/smart_crusher/planning.rs +904 -0
  446. package/crates/headroom-core/src/transforms/smart_crusher/statistics.rs +510 -0
  447. package/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs +287 -0
  448. package/crates/headroom-core/src/transforms/smart_crusher/traits.rs +203 -0
  449. package/crates/headroom-core/src/transforms/smart_crusher/types.rs +263 -0
  450. package/crates/headroom-core/src/transforms/tag_protector.rs +1272 -0
  451. package/crates/headroom-core/src/transforms/unidiff_detector.rs +232 -0
  452. package/crates/headroom-parity/Cargo.toml +26 -0
  453. package/crates/headroom-parity/examples/diff_fixture.rs +66 -0
  454. package/crates/headroom-parity/src/bin/parity_run.rs +78 -0
  455. package/crates/headroom-parity/src/lib.rs +613 -0
  456. package/crates/headroom-proxy/Cargo.toml +119 -0
  457. package/crates/headroom-proxy/data/model_prices_and_context_window.json +39767 -0
  458. package/crates/headroom-proxy/src/bedrock/auth_mode_layer.rs +192 -0
  459. package/crates/headroom-proxy/src/bedrock/envelope.rs +239 -0
  460. package/crates/headroom-proxy/src/bedrock/eventstream.rs +769 -0
  461. package/crates/headroom-proxy/src/bedrock/eventstream_to_sse.rs +459 -0
  462. package/crates/headroom-proxy/src/bedrock/invoke.rs +735 -0
  463. package/crates/headroom-proxy/src/bedrock/invoke_streaming.rs +1098 -0
  464. package/crates/headroom-proxy/src/bedrock/mod.rs +66 -0
  465. package/crates/headroom-proxy/src/bedrock/sigv4.rs +324 -0
  466. package/crates/headroom-proxy/src/bedrock/vendor.rs +82 -0
  467. package/crates/headroom-proxy/src/cache_stabilization/anthropic_cache_control.rs +698 -0
  468. package/crates/headroom-proxy/src/cache_stabilization/drift_detector.rs +683 -0
  469. package/crates/headroom-proxy/src/cache_stabilization/mod.rs +61 -0
  470. package/crates/headroom-proxy/src/cache_stabilization/openai_cache_key.rs +600 -0
  471. package/crates/headroom-proxy/src/cache_stabilization/tool_def_normalize.rs +514 -0
  472. package/crates/headroom-proxy/src/cache_stabilization/volatile_detector.rs +720 -0
  473. package/crates/headroom-proxy/src/compression/anthropic.rs +92 -0
  474. package/crates/headroom-proxy/src/compression/live_zone_anthropic.rs +1311 -0
  475. package/crates/headroom-proxy/src/compression/live_zone_openai.rs +660 -0
  476. package/crates/headroom-proxy/src/compression/live_zone_responses.rs +678 -0
  477. package/crates/headroom-proxy/src/compression/mod.rs +127 -0
  478. package/crates/headroom-proxy/src/compression/model_limits.rs +211 -0
  479. package/crates/headroom-proxy/src/config.rs +646 -0
  480. package/crates/headroom-proxy/src/error.rs +69 -0
  481. package/crates/headroom-proxy/src/handlers/chat_completions.rs +100 -0
  482. package/crates/headroom-proxy/src/handlers/conversations.rs +241 -0
  483. package/crates/headroom-proxy/src/handlers/mod.rs +13 -0
  484. package/crates/headroom-proxy/src/handlers/responses.rs +256 -0
  485. package/crates/headroom-proxy/src/headers.rs +288 -0
  486. package/crates/headroom-proxy/src/health.rs +41 -0
  487. package/crates/headroom-proxy/src/lib.rs +21 -0
  488. package/crates/headroom-proxy/src/main.rs +141 -0
  489. package/crates/headroom-proxy/src/observability/cache_hit_rate.rs +340 -0
  490. package/crates/headroom-proxy/src/observability/compression_ratio.rs +158 -0
  491. package/crates/headroom-proxy/src/observability/metric_names.rs +174 -0
  492. package/crates/headroom-proxy/src/observability/mod.rs +68 -0
  493. package/crates/headroom-proxy/src/observability/prometheus.rs +394 -0
  494. package/crates/headroom-proxy/src/observability/proxy_metrics.rs +368 -0
  495. package/crates/headroom-proxy/src/proxy.rs +1619 -0
  496. package/crates/headroom-proxy/src/responses_items.rs +565 -0
  497. package/crates/headroom-proxy/src/sse/anthropic.rs +415 -0
  498. package/crates/headroom-proxy/src/sse/framing.rs +366 -0
  499. package/crates/headroom-proxy/src/sse/mod.rs +40 -0
  500. package/crates/headroom-proxy/src/sse/openai_chat.rs +231 -0
  501. package/crates/headroom-proxy/src/sse/openai_responses.rs +458 -0
  502. package/crates/headroom-proxy/src/vertex/adc.rs +315 -0
  503. package/crates/headroom-proxy/src/vertex/envelope.rs +167 -0
  504. package/crates/headroom-proxy/src/vertex/mod.rs +277 -0
  505. package/crates/headroom-proxy/src/vertex/raw_predict.rs +490 -0
  506. package/crates/headroom-proxy/src/vertex/stream_raw_predict.rs +49 -0
  507. package/crates/headroom-proxy/src/websocket.rs +246 -0
  508. package/crates/headroom-py/Cargo.toml +37 -0
  509. package/crates/headroom-py/build.rs +77 -0
  510. package/crates/headroom-py/glibc_compat.c +170 -0
  511. package/crates/headroom-py/src/lib.rs +1655 -0
  512. package/deny.toml +32 -0
  513. package/design-system/base.css +127 -0
  514. package/design-system/components.css +1943 -0
  515. package/design-system/fonts.css +17 -0
  516. package/design-system/tailwind-preset.js +95 -0
  517. package/design-system/tokens.css +48 -0
  518. package/design-system/utilities.css +53 -0
  519. package/docker/differential-network-capture/Dockerfile.runner +14 -0
  520. package/docker/differential-network-capture/docker-compose.yml +142 -0
  521. package/docker/differential-network-capture/run-claude-lane.sh +24 -0
  522. package/docker/docker-compose.native.yml +44 -0
  523. package/docker-bake.hcl +88 -0
  524. package/docker-compose.yml +80 -0
  525. package/e2e/_lib/make_shim.ps1 +24 -0
  526. package/e2e/_lib/make_shim.sh +28 -0
  527. package/e2e/docker-native-install.sh +120 -0
  528. package/e2e/init/Dockerfile +79 -0
  529. package/e2e/wrap/Dockerfile +115 -0
  530. package/examples/07-context-compression.ipynb +279 -0
  531. package/examples/deployment/macos-launchagent/com.headroom.proxy.plist.template +65 -0
  532. package/examples/deployment/macos-launchagent/install.sh +227 -0
  533. package/examples/deployment/macos-launchagent/shell-integration.sh +74 -0
  534. package/examples/deployment/macos-launchagent/uninstall.sh +145 -0
  535. package/headroom-savings.png +0 -0
  536. package/headroom_learn.gif +0 -0
  537. package/lib/config/index.ts +12 -0
  538. package/lib/hooks/index.ts +4 -0
  539. package/lib/hooks/use-mounted.ts +13 -0
  540. package/lib/hooks/use-session.ts +34 -0
  541. package/lib/hooks/use-settings.ts +49 -0
  542. package/lib/utils/index.ts +1 -0
  543. package/lib/utils/settings.ts +162 -0
  544. package/lib/utils/utils.ts +6 -0
  545. package/llms.txt +67 -0
  546. package/middleware.ts +26 -0
  547. package/mkdocs.yml +136 -0
  548. package/next-env.d.ts +7 -0
  549. package/next.config.ts +12 -0
  550. package/npm/bin/xithead.js +366 -366
  551. package/npm/bin/xtream-head.js +366 -0
  552. package/npm/index.d.ts +57 -57
  553. package/npm/index.js +139 -139
  554. package/npm/package.json +55 -54
  555. package/npm/postinstall.js +43 -43
  556. package/package.json +58 -36
  557. package/plugins/headroom-agent-hooks/.claude-plugin/plugin.json +17 -0
  558. package/plugins/headroom-agent-hooks/hooks/hooks.json +29 -0
  559. package/plugins/headroom-oauth2/LICENSE +3 -0
  560. package/plugins/headroom-oauth2/SPEC.md +115 -0
  561. package/plugins/headroom-oauth2/pyproject.toml +48 -0
  562. package/plugins/hermes/headroom_retrieve/plugin.yaml +6 -0
  563. package/plugins/openclaw/hook-shim/HOOK.md +21 -0
  564. package/plugins/openclaw/hook-shim/handler.js +3 -0
  565. package/plugins/openclaw/openclaw.plugin.json +98 -0
  566. package/plugins/openclaw/package-lock.json +2692 -0
  567. package/plugins/openclaw/package.json +54 -0
  568. package/plugins/openclaw/prepare-dist.mjs +56 -0
  569. package/plugins/openclaw/src/convert.ts +418 -0
  570. package/plugins/openclaw/src/engine.ts +284 -0
  571. package/plugins/openclaw/src/gateway-config.ts +143 -0
  572. package/plugins/openclaw/src/index.ts +11 -0
  573. package/plugins/openclaw/src/plugin/index.ts +98 -0
  574. package/plugins/openclaw/src/proxy-manager.ts +477 -0
  575. package/plugins/openclaw/src/tools/headroom-retrieve.ts +76 -0
  576. package/plugins/openclaw/test/convert.test.ts +96 -0
  577. package/plugins/openclaw/test/engine-normalization.test.ts +21 -0
  578. package/plugins/openclaw/test/engine.test.ts +101 -0
  579. package/plugins/openclaw/test/gateway-config.test.ts +278 -0
  580. package/plugins/openclaw/test/plugin-runtime-routing.test.ts +141 -0
  581. package/plugins/openclaw/test/proxy-manager.test.ts +343 -0
  582. package/plugins/openclaw/tsup.config.ts +9 -0
  583. package/postcss.config.mjs +7 -0
  584. package/pyproject.toml +438 -0
  585. package/rust-toolchain.toml +16 -0
  586. package/sdk/typescript/examples/basic-compress.ts +64 -0
  587. package/sdk/typescript/examples/ccr-retrieve.ts +80 -0
  588. package/sdk/typescript/examples/hooks-custom-compression.ts +117 -0
  589. package/sdk/typescript/examples/middleware-composition.ts +59 -0
  590. package/sdk/typescript/examples/multi-provider.ts +53 -0
  591. package/sdk/typescript/examples/openai-anthropic-adapters.ts +73 -0
  592. package/sdk/typescript/examples/shared-context-multi-agent.ts +74 -0
  593. package/sdk/typescript/examples/simulation-dry-run.ts +80 -0
  594. package/sdk/typescript/examples/streaming-chat.ts +71 -0
  595. package/sdk/typescript/examples/structured-output.ts +99 -0
  596. package/sdk/typescript/examples/tool-calling-agent.ts +86 -0
  597. package/sdk/typescript/examples/with-headroom-vercel.ts +57 -0
  598. package/sdk/typescript/package-lock.json +3437 -0
  599. package/sdk/typescript/package.json +108 -0
  600. package/sdk/typescript/src/adapters/anthropic.ts +218 -0
  601. package/sdk/typescript/src/adapters/gemini.ts +83 -0
  602. package/sdk/typescript/src/adapters/openai.ts +77 -0
  603. package/sdk/typescript/src/adapters/vercel-ai.ts +131 -0
  604. package/sdk/typescript/src/client.ts +632 -0
  605. package/sdk/typescript/src/compress.ts +87 -0
  606. package/sdk/typescript/src/errors.ts +114 -0
  607. package/sdk/typescript/src/hooks.ts +113 -0
  608. package/sdk/typescript/src/index.ts +140 -0
  609. package/sdk/typescript/src/paths.ts +272 -0
  610. package/sdk/typescript/src/shared-context.ts +188 -0
  611. package/sdk/typescript/src/simulate.ts +52 -0
  612. package/sdk/typescript/src/types/config.ts +183 -0
  613. package/sdk/typescript/src/types/models.ts +314 -0
  614. package/sdk/typescript/src/types.ts +139 -0
  615. package/sdk/typescript/src/utils/case.ts +34 -0
  616. package/sdk/typescript/src/utils/format.ts +480 -0
  617. package/sdk/typescript/src/utils/stream.ts +50 -0
  618. package/sdk/typescript/test/adapters/anthropic.test.ts +217 -0
  619. package/sdk/typescript/test/adapters/gemini.test.ts +131 -0
  620. package/sdk/typescript/test/adapters/openai.test.ts +174 -0
  621. package/sdk/typescript/test/adapters/vercel-ai.test.ts +306 -0
  622. package/sdk/typescript/test/client-expanded.test.ts +333 -0
  623. package/sdk/typescript/test/client.test.ts +266 -0
  624. package/sdk/typescript/test/compress.test.ts +118 -0
  625. package/sdk/typescript/test/config.test.ts +275 -0
  626. package/sdk/typescript/test/errors.test.ts +169 -0
  627. package/sdk/typescript/test/format-integration.test.ts +252 -0
  628. package/sdk/typescript/test/hooks.test.ts +265 -0
  629. package/sdk/typescript/test/integration.test.ts +249 -0
  630. package/sdk/typescript/test/mdx-examples.test.ts +253 -0
  631. package/sdk/typescript/test/models.test.ts +318 -0
  632. package/sdk/typescript/test/parity.test.ts +337 -0
  633. package/sdk/typescript/test/paths.test.ts +447 -0
  634. package/sdk/typescript/test/shared-context.test.ts +245 -0
  635. package/sdk/typescript/test/simulate.test.ts +84 -0
  636. package/sdk/typescript/test/types.test.ts +244 -0
  637. package/sdk/typescript/test/utils/case.test.ts +129 -0
  638. package/sdk/typescript/test/utils/detect-format.test.ts +133 -0
  639. package/sdk/typescript/test/utils/format.test.ts +434 -0
  640. package/sdk/typescript/test/vercel-ai-e2e.test.ts +179 -0
  641. package/sdk/typescript/tsup.config.ts +17 -0
  642. package/tailwind.master.config.cjs +47 -0
  643. package/types/api.ts +12 -0
  644. package/types/config.ts +9 -0
  645. package/types/index.ts +2 -0
  646. package/uv.lock +6951 -0
  647. package/xithead.egg-info/PKG-INFO +22 -0
  648. package/xithead.egg-info/SOURCES.txt +16 -0
  649. package/xithead.egg-info/dependency_links.txt +1 -0
  650. package/xithead.egg-info/entry_points.txt +2 -0
  651. package/xithead.egg-info/requires.txt +10 -0
  652. package/xithead.egg-info/top_level.txt +2 -0
  653. package/xithead.ps1 +136 -0
  654. package/xithead.sh +129 -0
  655. package/xithead_dist/pyproject.toml +39 -0
  656. package/xithead_dist/xithead.egg-info/PKG-INFO +22 -0
  657. package/xithead_dist/xithead.egg-info/SOURCES.txt +16 -0
  658. package/xithead_dist/xithead.egg-info/dependency_links.txt +1 -0
  659. package/xithead_dist/xithead.egg-info/entry_points.txt +3 -0
  660. package/xithead_dist/xithead.egg-info/requires.txt +10 -0
  661. package/xithead_dist/xithead.egg-info/top_level.txt +2 -0
@@ -0,0 +1,342 @@
1
+ //! Model-name → tokenizer dispatch.
2
+ //!
3
+ //! Mirrors `MODEL_PATTERNS` in `headroom/tokenizers/registry.py`. Three
4
+ //! backends in priority order:
5
+ //!
6
+ //! 1. **HuggingFace** — anything the caller has registered via
7
+ //! [`register_hf`] for a given model-name prefix. Real BPE/Unigram/
8
+ //! WordPiece counts. This is opt-in: tokenizer.json files aren't bundled,
9
+ //! so nothing routes here until the embedding application calls
10
+ //! [`register_hf`] at startup. Wins over the rules below when registered.
11
+ //! 2. **Tiktoken** — OpenAI / o-series via `tiktoken-rs`. Byte-identical to
12
+ //! Python `tiktoken`.
13
+ //! 3. **Estimation** — `chars / cpt` fallback for Anthropic Claude (3.5),
14
+ //! Gemini / Cohere / Command without an HF registration (4.0), and
15
+ //! everything else (4.0).
16
+
17
+ use std::collections::HashMap;
18
+ use std::sync::{OnceLock, RwLock};
19
+
20
+ use super::{EstimatingCounter, HfTokenizer, HfTokenizerError, TiktokenCounter, Tokenizer};
21
+
22
+ /// Which family of tokenizer was selected for a model.
23
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
24
+ pub enum Backend {
25
+ /// Real BPE via `tiktoken-rs`. Byte-identical to Python `tiktoken`.
26
+ Tiktoken,
27
+ /// HuggingFace `tokenizers` crate (`tokenizer.json` loaded by caller).
28
+ HuggingFace,
29
+ /// Character-density estimation (chars/token formula).
30
+ Estimation,
31
+ }
32
+
33
+ /// Pick a backend purely from the model name, ignoring runtime registrations.
34
+ ///
35
+ /// Patterns and ordering match `headroom.tokenizers.registry.MODEL_PATTERNS`
36
+ /// for the families this stage supports. Anything outside the OpenAI BPE
37
+ /// family lands in `Estimation` here — even if [`register_hf`] would route
38
+ /// it to `HuggingFace` at runtime. Use [`get_tokenizer`] for the real
39
+ /// dispatch.
40
+ pub fn detect_backend(model: &str) -> Backend {
41
+ let m = model.to_ascii_lowercase();
42
+
43
+ // OpenAI BPE-tokenized families (gpt-3.5/4/4o + o1/o3 reasoning + embeddings + legacy davinci/curie/babbage/ada + code-).
44
+ if m.starts_with("gpt-4o")
45
+ || m.starts_with("gpt-4")
46
+ || m.starts_with("gpt-3.5")
47
+ || m.starts_with("o1")
48
+ || m.starts_with("o3")
49
+ || m.starts_with("text-embedding")
50
+ || m.starts_with("text-davinci")
51
+ || m.starts_with("davinci")
52
+ || m.starts_with("curie")
53
+ || m.starts_with("babbage")
54
+ || m.starts_with("ada")
55
+ || m.starts_with("code-")
56
+ {
57
+ return Backend::Tiktoken;
58
+ }
59
+
60
+ Backend::Estimation
61
+ }
62
+
63
+ /// Return a tokenizer for `model`. Resolution order:
64
+ /// 1. HuggingFace tokenizers registered via [`register_hf`] (longest matching
65
+ /// prefix wins).
66
+ /// 2. Tiktoken for OpenAI / o-series families.
67
+ /// 3. Estimation, with density calibrated per family (Claude → 3.5, Gemini /
68
+ /// Cohere / Command → 4.0, otherwise 4.0).
69
+ pub fn get_tokenizer(model: &str) -> Box<dyn Tokenizer> {
70
+ if let Some(hf) = lookup_hf(model) {
71
+ return Box::new(hf);
72
+ }
73
+ match detect_backend(model) {
74
+ Backend::Tiktoken => match TiktokenCounter::for_model(model) {
75
+ Ok(t) => Box::new(t),
76
+ Err(_) => Box::new(default_estimator_for(model)),
77
+ },
78
+ // Backend::HuggingFace from detect_backend is unreachable — only
79
+ // runtime registrations produce HF, and we already checked above.
80
+ Backend::HuggingFace | Backend::Estimation => Box::new(default_estimator_for(model)),
81
+ }
82
+ }
83
+
84
+ fn default_estimator_for(model: &str) -> EstimatingCounter {
85
+ let m = model.to_ascii_lowercase();
86
+ if m.starts_with("claude-") {
87
+ EstimatingCounter::new(3.5)
88
+ } else if m.starts_with("gemini") || m.starts_with("palm") || m.starts_with("command") {
89
+ EstimatingCounter::new(4.0)
90
+ } else {
91
+ EstimatingCounter::default()
92
+ }
93
+ }
94
+
95
+ // ---- HuggingFace runtime registry --------------------------------------
96
+ //
97
+ // A process-global table mapping a lowercased model-name *prefix* to a loaded
98
+ // `HfTokenizer`. The embedding application owns this — there's no autoloader
99
+ // in core because we don't want to bundle tokenizer.json files (multi-MB) or
100
+ // pull in HF Hub networking here.
101
+
102
+ fn hf_table() -> &'static RwLock<HashMap<String, HfTokenizer>> {
103
+ static TABLE: OnceLock<RwLock<HashMap<String, HfTokenizer>>> = OnceLock::new();
104
+ TABLE.get_or_init(|| RwLock::new(HashMap::new()))
105
+ }
106
+
107
+ /// Register `tokenizer` to handle every model whose lowercased name starts
108
+ /// with `prefix`. Multiple prefixes can coexist; on lookup the longest
109
+ /// matching prefix wins, so registering `"command-r-plus"` overrides a more
110
+ /// general `"command-"` registration for that one model.
111
+ ///
112
+ /// Calling this with the same `prefix` twice replaces the previous tokenizer.
113
+ pub fn register_hf(prefix: impl Into<String>, tokenizer: HfTokenizer) {
114
+ let key = prefix.into().to_ascii_lowercase();
115
+ hf_table()
116
+ .write()
117
+ .expect("hf registry poisoned")
118
+ .insert(key, tokenizer);
119
+ }
120
+
121
+ /// Drop every HuggingFace registration. Intended for tests; production code
122
+ /// should register once at startup.
123
+ pub fn clear_hf_registrations() {
124
+ hf_table().write().expect("hf registry poisoned").clear();
125
+ }
126
+
127
+ /// Convenience: download `tokenizer.json` for `repo` from the HuggingFace
128
+ /// Hub and register it under `prefix`. One-line glue around
129
+ /// [`HfTokenizer::from_pretrained`] + [`register_hf`].
130
+ ///
131
+ /// Useful for proxy startup code that wants real tokenizers for the major
132
+ /// non-OpenAI families. Each call is independent — failure for one model
133
+ /// (e.g. a gated Llama repo without `HF_TOKEN`) does not affect others.
134
+ ///
135
+ /// ```no_run
136
+ /// use headroom_core::tokenizer::try_register_hf;
137
+ /// let _ = try_register_hf("command-", "CohereForAI/c4ai-command-r-v01");
138
+ /// let _ = try_register_hf("mistral-", "mistralai/Mistral-7B-v0.1");
139
+ /// ```
140
+ pub fn try_register_hf(prefix: &str, repo: &str) -> Result<(), HfTokenizerError> {
141
+ let t = HfTokenizer::from_pretrained(repo)?;
142
+ register_hf(prefix, t);
143
+ Ok(())
144
+ }
145
+
146
+ fn lookup_hf(model: &str) -> Option<HfTokenizer> {
147
+ let m = model.to_ascii_lowercase();
148
+ let table = hf_table().read().expect("hf registry poisoned");
149
+ // Longest prefix wins: a `command-r-plus` registration must beat
150
+ // `command-` for that model, regardless of insertion order.
151
+ table
152
+ .iter()
153
+ .filter(|(prefix, _)| m.starts_with(prefix.as_str()))
154
+ .max_by_key(|(prefix, _)| prefix.len())
155
+ .map(|(_, t)| t.clone())
156
+ }
157
+
158
+ #[cfg(test)]
159
+ mod tests {
160
+ use super::*;
161
+
162
+ /// Same minimal tokenizer.json used by hf_impl tests; inlined here so we
163
+ /// can register it without depending on a fixture file.
164
+ const TINY_TOKENIZER_JSON: &str = r#"{
165
+ "version": "1.0",
166
+ "truncation": null,
167
+ "padding": null,
168
+ "added_tokens": [],
169
+ "normalizer": null,
170
+ "pre_tokenizer": {"type": "Whitespace"},
171
+ "post_processor": null,
172
+ "decoder": null,
173
+ "model": {
174
+ "type": "WordLevel",
175
+ "vocab": {"hello": 0, "world": 1, "[UNK]": 2},
176
+ "unk_token": "[UNK]"
177
+ }
178
+ }"#;
179
+
180
+ fn tiny(name: &str) -> HfTokenizer {
181
+ HfTokenizer::from_bytes(name, TINY_TOKENIZER_JSON.as_bytes()).unwrap()
182
+ }
183
+
184
+ /// Tests share a process-global table. `cargo test` runs them in parallel
185
+ /// by default, so without serialization one test's `clear` would wipe
186
+ /// another test's registrations mid-assertion. We serialize the tests
187
+ /// that touch the registry behind a single mutex.
188
+ static REGISTRY_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
189
+
190
+ struct RegistryGuard<'a> {
191
+ _g: std::sync::MutexGuard<'a, ()>,
192
+ }
193
+ impl<'a> RegistryGuard<'a> {
194
+ fn acquire() -> Self {
195
+ // Recover from a poisoned lock — a panic in one test should not
196
+ // break every subsequent test in the file.
197
+ let g = REGISTRY_LOCK.lock().unwrap_or_else(|e| e.into_inner());
198
+ clear_hf_registrations();
199
+ Self { _g: g }
200
+ }
201
+ }
202
+ impl<'a> Drop for RegistryGuard<'a> {
203
+ fn drop(&mut self) {
204
+ clear_hf_registrations();
205
+ }
206
+ }
207
+
208
+ #[test]
209
+ fn openai_models_pick_tiktoken() {
210
+ for m in [
211
+ "gpt-4o",
212
+ "gpt-4o-mini",
213
+ "gpt-4",
214
+ "gpt-4-turbo",
215
+ "gpt-3.5-turbo",
216
+ "o1-preview",
217
+ "o3-mini",
218
+ "text-embedding-3-small",
219
+ "text-davinci-003",
220
+ "davinci",
221
+ "babbage-002",
222
+ "code-davinci-002",
223
+ ] {
224
+ assert_eq!(detect_backend(m), Backend::Tiktoken, "{m}");
225
+ }
226
+ }
227
+
228
+ #[test]
229
+ fn non_openai_models_fall_through_to_estimation() {
230
+ for m in [
231
+ "claude-haiku-4-5-20251001",
232
+ "claude-3-opus",
233
+ "gemini-1.5-pro",
234
+ "command-r-plus",
235
+ "llama-3-70b",
236
+ "mistral-large",
237
+ "qwen-72b",
238
+ "made-up-model-name",
239
+ ] {
240
+ assert_eq!(detect_backend(m), Backend::Estimation, "{m}");
241
+ }
242
+ }
243
+
244
+ #[test]
245
+ fn case_insensitive() {
246
+ assert_eq!(detect_backend("GPT-4o"), Backend::Tiktoken);
247
+ assert_eq!(detect_backend("Claude-haiku"), Backend::Estimation);
248
+ }
249
+
250
+ #[test]
251
+ fn estimator_density_per_family() {
252
+ let _g = RegistryGuard::acquire();
253
+ // Round-trip through the public dispatch and check that the chosen
254
+ // estimator behaves with the right density. We can't introspect the
255
+ // trait object's chars_per_token directly, so we use a known-length
256
+ // string and back-compute.
257
+ let claude = get_tokenizer("claude-3-opus");
258
+ // 3.5 chars/token: 35 chars -> 10 tokens.
259
+ assert_eq!(claude.count_text(&"a".repeat(35)), 10);
260
+
261
+ let gemini = get_tokenizer("gemini-1.5-pro");
262
+ // 4.0 chars/token: 40 chars -> 10 tokens.
263
+ assert_eq!(gemini.count_text(&"a".repeat(40)), 10);
264
+ }
265
+
266
+ #[test]
267
+ fn registered_hf_wins_over_estimator() {
268
+ let _g = RegistryGuard::acquire();
269
+ register_hf("command-", tiny("cohere"));
270
+ let t = get_tokenizer("command-r-plus");
271
+ // Whitespace tokenization → 2 tokens, *not* the chars/4 estimator.
272
+ assert_eq!(t.count_text("hello world"), 2);
273
+ assert_eq!(t.backend(), Backend::HuggingFace);
274
+ }
275
+
276
+ #[test]
277
+ fn registered_hf_does_not_override_tiktoken() {
278
+ // A user accidentally registers an HF tokenizer with the prefix
279
+ // `gpt-` — should the OpenAI route still win? Per docstring: HF
280
+ // always wins when registered. We document this behavior so the
281
+ // test pins it down: registration is a deliberate override.
282
+ let _g = RegistryGuard::acquire();
283
+ register_hf("gpt-4o", tiny("oops"));
284
+ let t = get_tokenizer("gpt-4o-mini");
285
+ assert_eq!(t.backend(), Backend::HuggingFace);
286
+ }
287
+
288
+ #[test]
289
+ fn longest_prefix_wins() {
290
+ let _g = RegistryGuard::acquire();
291
+ register_hf("command-", tiny("general"));
292
+ register_hf("command-r-plus", tiny("specific"));
293
+ // `command-r-plus` should pick the more specific one.
294
+ let t = get_tokenizer("command-r-plus");
295
+ assert_eq!(t.backend(), Backend::HuggingFace);
296
+ // Counting the same input via both should give identical counts since
297
+ // we used the same tokenizer.json — but the *path* through the
298
+ // registry is different. This locks in that the longer prefix is
299
+ // selected, not the shorter one.
300
+ let count_specific = t.count_text("hello world hello");
301
+ assert_eq!(count_specific, 3);
302
+
303
+ // A model only matched by the shorter prefix still works.
304
+ let t2 = get_tokenizer("command-light");
305
+ assert_eq!(t2.backend(), Backend::HuggingFace);
306
+ }
307
+
308
+ #[test]
309
+ fn case_insensitive_registration() {
310
+ let _g = RegistryGuard::acquire();
311
+ register_hf("Command-", tiny("cohere"));
312
+ let t = get_tokenizer("COMMAND-R-PLUS");
313
+ assert_eq!(t.backend(), Backend::HuggingFace);
314
+ }
315
+
316
+ #[test]
317
+ fn clear_resets_state() {
318
+ let _g = RegistryGuard::acquire();
319
+ register_hf("command-", tiny("cohere"));
320
+ clear_hf_registrations();
321
+ let t = get_tokenizer("command-r-plus");
322
+ // Without an HF registration, falls back to estimator (4.0 cpt).
323
+ assert_eq!(t.backend(), Backend::Estimation);
324
+ }
325
+
326
+ #[test]
327
+ fn unrelated_models_still_estimate() {
328
+ let _g = RegistryGuard::acquire();
329
+ register_hf("command-", tiny("cohere"));
330
+ let t = get_tokenizer("claude-3-opus");
331
+ assert_eq!(t.backend(), Backend::Estimation);
332
+ }
333
+
334
+ #[test]
335
+ fn detect_backend_ignores_runtime_registrations() {
336
+ let _g = RegistryGuard::acquire();
337
+ register_hf("command-", tiny("cohere"));
338
+ // detect_backend is a pure function of the model name; runtime
339
+ // registrations don't show up here.
340
+ assert_eq!(detect_backend("command-r-plus"), Backend::Estimation);
341
+ }
342
+ }
@@ -0,0 +1,265 @@
1
+ //! `tiktoken-rs` adapter implementing [`Tokenizer`].
2
+ //!
3
+ //! `tiktoken-rs` and Python `tiktoken` use the same BPE merge tables; for the
4
+ //! same model and same input, this returns byte-identical token IDs and
5
+ //! therefore byte-identical token *counts*. This is what makes the parity
6
+ //! tests "byte-equal" rather than "approximate".
7
+ //!
8
+ //! Initialization (loading the BPE table) is non-trivial. Each encoding is
9
+ //! built lazily on first use and shared via `LazyLock<Arc<CoreBPE>>`, so the
10
+ //! first `for_model` call pays the cost and every subsequent call is cheap.
11
+
12
+ use std::sync::{Arc, LazyLock};
13
+
14
+ use thiserror::Error;
15
+ use tiktoken_rs::CoreBPE;
16
+
17
+ use super::{Backend, Tokenizer};
18
+
19
+ #[derive(Debug, Error)]
20
+ pub enum TiktokenError {
21
+ /// We don't know which encoding `model` should use. The caller can fall
22
+ /// back to estimation; the registry handles that automatically.
23
+ #[error("unknown encoding for model `{0}`")]
24
+ UnknownEncoding(String),
25
+ }
26
+
27
+ /// Lazy-built shared BPE for the four named encodings. Init failure here would
28
+ /// indicate `tiktoken-rs` itself is broken; we treat that as a programmer error
29
+ /// and panic.
30
+ static O200K: LazyLock<Arc<CoreBPE>> =
31
+ LazyLock::new(|| Arc::new(tiktoken_rs::o200k_base().expect("o200k_base init")));
32
+ static CL100K: LazyLock<Arc<CoreBPE>> =
33
+ LazyLock::new(|| Arc::new(tiktoken_rs::cl100k_base().expect("cl100k_base init")));
34
+ static P50K: LazyLock<Arc<CoreBPE>> =
35
+ LazyLock::new(|| Arc::new(tiktoken_rs::p50k_base().expect("p50k_base init")));
36
+ static R50K: LazyLock<Arc<CoreBPE>> =
37
+ LazyLock::new(|| Arc::new(tiktoken_rs::r50k_base().expect("r50k_base init")));
38
+
39
+ /// BPE token counter for OpenAI / o-series models.
40
+ pub struct TiktokenCounter {
41
+ model: String,
42
+ encoding_name: &'static str,
43
+ bpe: Arc<CoreBPE>,
44
+ }
45
+
46
+ impl std::fmt::Debug for TiktokenCounter {
47
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
48
+ f.debug_struct("TiktokenCounter")
49
+ .field("model", &self.model)
50
+ .field("encoding", &self.encoding_name)
51
+ .finish()
52
+ }
53
+ }
54
+
55
+ impl TiktokenCounter {
56
+ /// Build a counter for `model`. Returns `UnknownEncoding` if the model
57
+ /// doesn't fall into any of the supported BPE families.
58
+ pub fn for_model(model: &str) -> Result<Self, TiktokenError> {
59
+ let encoding_name = encoding_for(model)?;
60
+ let bpe = match encoding_name {
61
+ "o200k_base" => O200K.clone(),
62
+ "cl100k_base" => CL100K.clone(),
63
+ "p50k_base" => P50K.clone(),
64
+ "r50k_base" => R50K.clone(),
65
+ // unreachable: encoding_for only returns the four names above.
66
+ _ => return Err(TiktokenError::UnknownEncoding(model.to_string())),
67
+ };
68
+ Ok(Self {
69
+ model: model.to_string(),
70
+ encoding_name,
71
+ bpe,
72
+ })
73
+ }
74
+
75
+ pub fn model(&self) -> &str {
76
+ &self.model
77
+ }
78
+
79
+ pub fn encoding_name(&self) -> &'static str {
80
+ self.encoding_name
81
+ }
82
+ }
83
+
84
+ impl Tokenizer for TiktokenCounter {
85
+ fn count_text(&self, text: &str) -> usize {
86
+ if text.is_empty() {
87
+ // Match Python `TiktokenCounter.count_text`: short-circuit empty.
88
+ return 0;
89
+ }
90
+ // For ORDINARY input (no literal special-token strings) `encode_ordinary`
91
+ // here and `encoding.encode(text)` in Python yield identical token IDs
92
+ // and counts — that's the byte-equality the parity harness verifies.
93
+ //
94
+ // Divergence (rare in practice): if `text` contains a literal
95
+ // `<|endoftext|>` (or any other special-token string), Python's default
96
+ // `encode` raises (because `disallowed_special="all"`) while we treat
97
+ // it as ordinary text. We chose tolerance over panic since proxy users
98
+ // can legitimately send those substrings; document for future readers.
99
+ self.bpe.encode_ordinary(text).len()
100
+ }
101
+
102
+ fn backend(&self) -> Backend {
103
+ Backend::Tiktoken
104
+ }
105
+ }
106
+
107
+ /// Map model → encoding name. Mirrors `MODEL_ENCODINGS` and the prefix
108
+ /// fallbacks in `headroom/tokenizers/tiktoken_counter.py`.
109
+ fn encoding_for(model: &str) -> Result<&'static str, TiktokenError> {
110
+ let m = model.to_ascii_lowercase();
111
+
112
+ // o200k_base: GPT-4o + o1/o3 reasoning families.
113
+ if m.starts_with("gpt-4o") || m.starts_with("o1") || m.starts_with("o3") {
114
+ return Ok("o200k_base");
115
+ }
116
+
117
+ // cl100k_base: GPT-4, GPT-3.5-turbo, embeddings.
118
+ if m.starts_with("gpt-4") || m.starts_with("gpt-3.5") || m.starts_with("text-embedding") {
119
+ return Ok("cl100k_base");
120
+ }
121
+
122
+ // p50k_base: code-* and the davinci-002/003 text-completion line.
123
+ if m.starts_with("code-")
124
+ || m.starts_with("text-davinci-002")
125
+ || m.starts_with("text-davinci-003")
126
+ {
127
+ return Ok("p50k_base");
128
+ }
129
+
130
+ // r50k_base: legacy davinci-001 and earlier completion families.
131
+ if m.starts_with("text-davinci")
132
+ || m.starts_with("davinci")
133
+ || m.starts_with("curie")
134
+ || m.starts_with("babbage")
135
+ || m.starts_with("ada")
136
+ {
137
+ return Ok("r50k_base");
138
+ }
139
+
140
+ Err(TiktokenError::UnknownEncoding(model.to_string()))
141
+ }
142
+
143
+ #[cfg(test)]
144
+ mod tests {
145
+ use super::*;
146
+
147
+ #[test]
148
+ fn empty_string_is_zero() {
149
+ let t = TiktokenCounter::for_model("gpt-4o-mini").unwrap();
150
+ assert_eq!(t.count_text(""), 0);
151
+ }
152
+
153
+ #[test]
154
+ fn nonempty_text_is_at_least_one_token() {
155
+ let t = TiktokenCounter::for_model("gpt-4o-mini").unwrap();
156
+ assert!(t.count_text("a") >= 1);
157
+ }
158
+
159
+ #[test]
160
+ fn known_token_counts_for_o200k() {
161
+ // These constants are the o200k_base BPE token counts produced by
162
+ // both Python `tiktoken.encoding_for_model("gpt-4o-mini")` and
163
+ // `tiktoken-rs::o200k_base()` for the given strings. They lock in
164
+ // byte-equal parity so a future tiktoken-rs upgrade that subtly
165
+ // changes BPE behavior would fail this test.
166
+ let t = TiktokenCounter::for_model("gpt-4o-mini").unwrap();
167
+ assert_eq!(t.count_text("hello"), 1);
168
+ assert_eq!(t.count_text("Hello, world!"), 4);
169
+ assert_eq!(
170
+ t.count_text("the quick brown fox jumps over the lazy dog"),
171
+ 9
172
+ );
173
+ }
174
+
175
+ #[test]
176
+ fn determinism() {
177
+ let t = TiktokenCounter::for_model("gpt-4o").unwrap();
178
+ let s = "Determinism check across many calls.";
179
+ let first = t.count_text(s);
180
+ for _ in 0..1000 {
181
+ assert_eq!(t.count_text(s), first);
182
+ }
183
+ }
184
+
185
+ #[test]
186
+ fn unicode_input_does_not_panic() {
187
+ let t = TiktokenCounter::for_model("gpt-4o-mini").unwrap();
188
+ // Each call should produce a reasonable count (>=1 for non-empty),
189
+ // not panic, and not return absurd values.
190
+ for s in [
191
+ "héllo wörld", // accented
192
+ "你好世界", // CJK
193
+ "مرحبا بالعالم", // Arabic (RTL)
194
+ "🦀 ferris the crab", // emoji
195
+ "\n\t\r\x07", // control chars
196
+ ] {
197
+ let n = t.count_text(s);
198
+ assert!(n >= 1, "{s:?}");
199
+ assert!(n < s.len() * 4 + 10, "absurd count {n} for {s:?}");
200
+ }
201
+ }
202
+
203
+ #[test]
204
+ fn very_long_input() {
205
+ let t = TiktokenCounter::for_model("gpt-4o-mini").unwrap();
206
+ let s = "the quick brown fox ".repeat(50_000); // ~1MB
207
+ let n = t.count_text(&s);
208
+ // 50k repeats * ~5 tokens per repeat = ~250k tokens, sanity bound.
209
+ assert!(n > 100_000 && n < 1_000_000, "n={n}");
210
+ }
211
+
212
+ #[test]
213
+ fn encoding_dispatch() {
214
+ for (model, expected) in [
215
+ ("gpt-4o", "o200k_base"),
216
+ ("gpt-4o-mini", "o200k_base"),
217
+ ("gpt-4o-2024-08-06", "o200k_base"),
218
+ ("o1-preview", "o200k_base"),
219
+ ("o3-mini", "o200k_base"),
220
+ ("gpt-4", "cl100k_base"),
221
+ ("gpt-4-turbo", "cl100k_base"),
222
+ ("gpt-3.5-turbo", "cl100k_base"),
223
+ ("text-embedding-3-small", "cl100k_base"),
224
+ ("code-davinci-002", "p50k_base"),
225
+ ("text-davinci-002", "p50k_base"),
226
+ ("text-davinci-003", "p50k_base"),
227
+ ("text-davinci-001", "r50k_base"),
228
+ ("davinci", "r50k_base"),
229
+ ("curie", "r50k_base"),
230
+ ("babbage", "r50k_base"),
231
+ ("ada", "r50k_base"),
232
+ ] {
233
+ let t = TiktokenCounter::for_model(model)
234
+ .unwrap_or_else(|e| panic!("for_model({model}) failed: {e}"));
235
+ assert_eq!(t.encoding_name(), expected, "{model}");
236
+ }
237
+ }
238
+
239
+ #[test]
240
+ fn unknown_model_returns_error() {
241
+ let r = TiktokenCounter::for_model("claude-3-opus");
242
+ assert!(matches!(r, Err(TiktokenError::UnknownEncoding(_))));
243
+ }
244
+
245
+ #[test]
246
+ fn case_insensitive_dispatch() {
247
+ let t = TiktokenCounter::for_model("GPT-4o-Mini").unwrap();
248
+ assert_eq!(t.encoding_name(), "o200k_base");
249
+ }
250
+
251
+ #[test]
252
+ fn shared_bpe_instances() {
253
+ // Two counters for the same encoding should share the underlying BPE
254
+ // (same Arc), proving the LazyLock cache works.
255
+ let a = TiktokenCounter::for_model("gpt-4o").unwrap();
256
+ let b = TiktokenCounter::for_model("gpt-4o-mini").unwrap();
257
+ assert!(Arc::ptr_eq(&a.bpe, &b.bpe));
258
+ }
259
+
260
+ #[test]
261
+ fn backend_is_tiktoken() {
262
+ let t = TiktokenCounter::for_model("gpt-4o-mini").unwrap();
263
+ assert_eq!(t.backend(), Backend::Tiktoken);
264
+ }
265
+ }