xithead 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.actrc +14 -0
- package/.actrc.local.example +15 -0
- package/.changelog.md +2 -0
- package/.claude/agents/academic-anthropologist.md +125 -0
- package/.claude/agents/academic-geographer.md +127 -0
- package/.claude/agents/academic-historian.md +123 -0
- package/.claude/agents/academic-narratologist.md +118 -0
- package/.claude/agents/academic-psychologist.md +118 -0
- package/.claude/agents/academic-statistician.md +144 -0
- package/.claude/agents/accounts-payable-agent.md +205 -0
- package/.claude/agents/agentic-identity-trust.md +394 -0
- package/.claude/agents/agents-orchestrator.md +367 -0
- package/.claude/agents/automation-governance-architect.md +216 -0
- package/.claude/agents/blender-addon-engineer.md +243 -0
- package/.claude/agents/business-strategist.md +488 -0
- package/.claude/agents/change-management-consultant.md +497 -0
- package/.claude/agents/chief-financial-officer.md +388 -0
- package/.claude/agents/corporate-training-designer.md +192 -0
- package/.claude/agents/customer-service.md +398 -0
- package/.claude/agents/customer-success-manager.md +460 -0
- package/.claude/agents/data-consolidation-agent.md +60 -0
- package/.claude/agents/data-privacy-officer.md +412 -0
- package/.claude/agents/design-brand-guardian.md +322 -0
- package/.claude/agents/design-image-prompt-engineer.md +236 -0
- package/.claude/agents/design-inclusive-visuals-specialist.md +73 -0
- package/.claude/agents/design-persona-walkthrough.md +272 -0
- package/.claude/agents/design-ui-designer.md +386 -0
- package/.claude/agents/design-ui-finish-gate-reviewer.md +217 -0
- package/.claude/agents/design-ux-architect.md +482 -0
- package/.claude/agents/design-ux-researcher.md +329 -0
- package/.claude/agents/design-visual-storyteller.md +149 -0
- package/.claude/agents/design-whimsy-injector.md +460 -0
- package/.claude/agents/economy-designer.md +156 -0
- package/.claude/agents/engineering-ai-data-remediation-engineer.md +211 -0
- package/.claude/agents/engineering-ai-engineer.md +146 -0
- package/.claude/agents/engineering-api-platform-engineer.md +181 -0
- package/.claude/agents/engineering-ats-validator-architect.md +383 -0
- package/.claude/agents/engineering-autonomous-optimization-architect.md +121 -0
- package/.claude/agents/engineering-backend-architect.md +255 -0
- package/.claude/agents/engineering-china-network-engineer.md +252 -0
- package/.claude/agents/engineering-cms-developer.md +536 -0
- package/.claude/agents/engineering-code-reviewer.md +76 -0
- package/.claude/agents/engineering-codebase-onboarding-engineer.md +173 -0
- package/.claude/agents/engineering-data-engineer.md +349 -0
- package/.claude/agents/engineering-data-visualization-engineer.md +151 -0
- package/.claude/agents/engineering-database-optimizer.md +188 -0
- package/.claude/agents/engineering-database-reliability-engineer.md +201 -0
- package/.claude/agents/engineering-desktop-app-engineer.md +209 -0
- package/.claude/agents/engineering-developer-tooling-engineer.md +153 -0
- package/.claude/agents/engineering-devops-automator.md +385 -0
- package/.claude/agents/engineering-drupal-performance.md +347 -0
- package/.claude/agents/engineering-drupal-shopping-cart.md +360 -0
- package/.claude/agents/engineering-electrical-engineer.md +575 -0
- package/.claude/agents/engineering-email-intelligence-engineer.md +390 -0
- package/.claude/agents/engineering-embedded-firmware-engineer.md +196 -0
- package/.claude/agents/engineering-feishu-integration-developer.md +606 -0
- package/.claude/agents/engineering-filament-optimization-specialist.md +283 -0
- package/.claude/agents/engineering-finops-engineer.md +164 -0
- package/.claude/agents/engineering-frontend-developer.md +257 -0
- package/.claude/agents/engineering-gaussdb-expert.md +334 -0
- package/.claude/agents/engineering-git-workflow-master.md +98 -0
- package/.claude/agents/engineering-i18n-engineer.md +214 -0
- package/.claude/agents/engineering-identity-access-engineer.md +217 -0
- package/.claude/agents/engineering-incident-response-commander.md +444 -0
- package/.claude/agents/engineering-iot-fleet-engineer.md +148 -0
- package/.claude/agents/engineering-it-service-manager.md +561 -0
- package/.claude/agents/engineering-knowledge-graph-engineer.md +372 -0
- package/.claude/agents/engineering-llm-post-training-engineer.md +166 -0
- package/.claude/agents/engineering-minimal-change-engineer.md +207 -0
- package/.claude/agents/engineering-mobile-app-builder.md +493 -0
- package/.claude/agents/engineering-mobile-release-engineer.md +164 -0
- package/.claude/agents/engineering-multi-agent-systems-architect.md +600 -0
- package/.claude/agents/engineering-network-engineer.md +245 -0
- package/.claude/agents/engineering-orgscript-engineer.md +113 -0
- package/.claude/agents/engineering-payments-billing-engineer.md +228 -0
- package/.claude/agents/engineering-pdf-engine-architect.md +696 -0
- package/.claude/agents/engineering-platform-engineer.md +270 -0
- package/.claude/agents/engineering-privacy-engineer.md +152 -0
- package/.claude/agents/engineering-prompt-engineer.md +209 -0
- package/.claude/agents/engineering-rag-pipeline-engineer.md +450 -0
- package/.claude/agents/engineering-rapid-prototyper.md +466 -0
- package/.claude/agents/engineering-realtime-collaboration-engineer.md +200 -0
- package/.claude/agents/engineering-rust-refactoring-specialist.md +313 -0
- package/.claude/agents/engineering-search-relevance-engineer.md +237 -0
- package/.claude/agents/engineering-section-508-specialist.md +339 -0
- package/.claude/agents/engineering-senior-developer.md +176 -0
- package/.claude/agents/engineering-servicenow-developer-mentor.md +180 -0
- package/.claude/agents/engineering-software-architect.md +112 -0
- package/.claude/agents/engineering-solidity-smart-contract-engineer.md +522 -0
- package/.claude/agents/engineering-sre.md +90 -0
- package/.claude/agents/engineering-technical-writer.md +393 -0
- package/.claude/agents/engineering-universal-document-compiler.md +370 -0
- package/.claude/agents/engineering-uswds-developer.md +340 -0
- package/.claude/agents/engineering-video-streaming-engineer.md +151 -0
- package/.claude/agents/engineering-voice-ai-integration-engineer.md +584 -0
- package/.claude/agents/engineering-webassembly-engineer.md +153 -0
- package/.claude/agents/engineering-wechat-mini-program-developer.md +356 -0
- package/.claude/agents/engineering-wordpress-performance.md +346 -0
- package/.claude/agents/engineering-wordpress-shopping-cart.md +346 -0
- package/.claude/agents/esg-sustainability-officer.md +396 -0
- package/.claude/agents/finance-bookkeeper-controller.md +260 -0
- package/.claude/agents/finance-financial-analyst.md +234 -0
- package/.claude/agents/finance-fpa-analyst.md +263 -0
- package/.claude/agents/finance-investment-researcher.md +272 -0
- package/.claude/agents/finance-tax-strategist.md +239 -0
- package/.claude/agents/game-audio-engineer.md +264 -0
- package/.claude/agents/game-designer.md +167 -0
- package/.claude/agents/gis-3d-scene-developer.md +111 -0
- package/.claude/agents/gis-analyst.md +91 -0
- package/.claude/agents/gis-bim-specialist.md +108 -0
- package/.claude/agents/gis-cartography-designer.md +150 -0
- package/.claude/agents/gis-drone-reality-mapping.md +120 -0
- package/.claude/agents/gis-geoai-ml-engineer.md +105 -0
- package/.claude/agents/gis-geoprocessing-specialist.md +97 -0
- package/.claude/agents/gis-qa-engineer.md +133 -0
- package/.claude/agents/gis-solution-engineer.md +101 -0
- package/.claude/agents/gis-spatial-data-engineer.md +97 -0
- package/.claude/agents/gis-spatial-data-scientist.md +111 -0
- package/.claude/agents/gis-technical-consultant.md +86 -0
- package/.claude/agents/gis-web-gis-developer.md +108 -0
- package/.claude/agents/godot-gameplay-scripter.md +338 -0
- package/.claude/agents/godot-multiplayer-engineer.md +301 -0
- package/.claude/agents/godot-shader-developer.md +267 -0
- package/.claude/agents/government-digital-presales-consultant.md +363 -0
- package/.claude/agents/grant-writer.md +511 -0
- package/.claude/agents/healthcare-aging-parent-care-companion.md +414 -0
- package/.claude/agents/healthcare-clinical-evidence-agent.md +231 -0
- package/.claude/agents/healthcare-customer-service.md +389 -0
- package/.claude/agents/healthcare-innovation-strategist.md +433 -0
- package/.claude/agents/healthcare-marketing-compliance.md +395 -0
- package/.claude/agents/healthcare-sovereign-health-systems-agent.md +312 -0
- package/.claude/agents/hospitality-guest-services.md +603 -0
- package/.claude/agents/hr-onboarding.md +451 -0
- package/.claude/agents/identity-graph-operator.md +303 -0
- package/.claude/agents/language-translator.md +264 -0
- package/.claude/agents/legal-billing-time-tracking.md +569 -0
- package/.claude/agents/legal-client-intake.md +492 -0
- package/.claude/agents/legal-document-review.md +454 -0
- package/.claude/agents/level-designer.md +208 -0
- package/.claude/agents/loan-officer-assistant.md +555 -0
- package/.claude/agents/lsp-index-engineer.md +320 -0
- package/.claude/agents/ma-integration-manager.md +427 -0
- package/.claude/agents/macos-spatial-metal-engineer.md +337 -0
- package/.claude/agents/marketing-aeo-foundations.md +264 -0
- package/.claude/agents/marketing-agentic-search-optimizer.md +279 -0
- package/.claude/agents/marketing-ai-citation-strategist.md +172 -0
- package/.claude/agents/marketing-app-store-optimizer.md +321 -0
- package/.claude/agents/marketing-baidu-seo-specialist.md +226 -0
- package/.claude/agents/marketing-bilibili-content-strategist.md +199 -0
- package/.claude/agents/marketing-book-co-author.md +110 -0
- package/.claude/agents/marketing-carousel-growth-engine.md +204 -0
- package/.claude/agents/marketing-china-ecommerce-operator.md +283 -0
- package/.claude/agents/marketing-china-market-localization-strategist.md +283 -0
- package/.claude/agents/marketing-content-creator.md +54 -0
- package/.claude/agents/marketing-cross-border-ecommerce.md +259 -0
- package/.claude/agents/marketing-developer-community-builder.md +285 -0
- package/.claude/agents/marketing-douyin-strategist.md +149 -0
- package/.claude/agents/marketing-email-strategist.md +249 -0
- package/.claude/agents/marketing-global-podcast-strategist.md +206 -0
- package/.claude/agents/marketing-growth-hacker.md +54 -0
- package/.claude/agents/marketing-instagram-curator.md +113 -0
- package/.claude/agents/marketing-kuaishou-strategist.md +223 -0
- package/.claude/agents/marketing-linkedin-content-creator.md +214 -0
- package/.claude/agents/marketing-livestream-commerce-coach.md +305 -0
- package/.claude/agents/marketing-multi-platform-publisher.md +217 -0
- package/.claude/agents/marketing-podcast-strategist.md +277 -0
- package/.claude/agents/marketing-pr-communications-manager.md +473 -0
- package/.claude/agents/marketing-private-domain-operator.md +308 -0
- package/.claude/agents/marketing-reddit-community-builder.md +123 -0
- package/.claude/agents/marketing-seo-specialist.md +370 -0
- package/.claude/agents/marketing-short-video-editing-coach.md +412 -0
- package/.claude/agents/marketing-social-media-strategist.md +125 -0
- package/.claude/agents/marketing-tiktok-strategist.md +125 -0
- package/.claude/agents/marketing-twitter-engager.md +126 -0
- package/.claude/agents/marketing-video-optimization-specialist.md +119 -0
- package/.claude/agents/marketing-wechat-official-account.md +145 -0
- package/.claude/agents/marketing-weibo-strategist.md +240 -0
- package/.claude/agents/marketing-x-twitter-intelligence-analyst.md +161 -0
- package/.claude/agents/marketing-xiaohongshu-specialist.md +138 -0
- package/.claude/agents/marketing-zhihu-strategist.md +162 -0
- package/.claude/agents/medical-billing-coding-specialist.md +491 -0
- package/.claude/agents/narrative-designer.md +243 -0
- package/.claude/agents/operations-manager.md +399 -0
- package/.claude/agents/organizational-psychologist.md +391 -0
- package/.claude/agents/paid-media-auditor.md +71 -0
- package/.claude/agents/paid-media-creative-strategist.md +71 -0
- package/.claude/agents/paid-media-paid-social-strategist.md +71 -0
- package/.claude/agents/paid-media-ppc-strategist.md +71 -0
- package/.claude/agents/paid-media-programmatic-buyer.md +71 -0
- package/.claude/agents/paid-media-search-query-analyst.md +71 -0
- package/.claude/agents/paid-media-tracking-specialist.md +71 -0
- package/.claude/agents/personal-growth-mentor.md +159 -0
- package/.claude/agents/product-behavioral-nudge-engine.md +83 -0
- package/.claude/agents/product-dx-engineer.md +332 -0
- package/.claude/agents/product-feedback-synthesizer.md +119 -0
- package/.claude/agents/product-manager.md +469 -0
- package/.claude/agents/product-sprint-prioritizer.md +154 -0
- package/.claude/agents/product-trend-researcher.md +159 -0
- package/.claude/agents/project-management-experiment-tracker.md +198 -0
- package/.claude/agents/project-management-jira-workflow-steward.md +233 -0
- package/.claude/agents/project-management-meeting-notes-specialist.md +95 -0
- package/.claude/agents/project-management-project-shepherd.md +194 -0
- package/.claude/agents/project-management-studio-operations.md +200 -0
- package/.claude/agents/project-management-studio-producer.md +203 -0
- package/.claude/agents/project-manager-senior.md +135 -0
- package/.claude/agents/real-estate-buyer-seller.md +596 -0
- package/.claude/agents/recruitment-specialist.md +515 -0
- package/.claude/agents/report-distribution-agent.md +65 -0
- package/.claude/agents/research-synthesist.md +136 -0
- package/.claude/agents/resume-tailor.md +230 -0
- package/.claude/agents/retail-customer-returns.md +566 -0
- package/.claude/agents/roblox-avatar-creator.md +297 -0
- package/.claude/agents/roblox-experience-designer.md +306 -0
- package/.claude/agents/roblox-systems-scripter.md +332 -0
- package/.claude/agents/sales-account-strategist.md +227 -0
- package/.claude/agents/sales-coach.md +271 -0
- package/.claude/agents/sales-data-extraction-agent.md +67 -0
- package/.claude/agents/sales-deal-strategist.md +180 -0
- package/.claude/agents/sales-discovery-coach.md +225 -0
- package/.claude/agents/sales-engineer.md +182 -0
- package/.claude/agents/sales-offer-lead-gen-strategist.md +257 -0
- package/.claude/agents/sales-outbound-strategist.md +201 -0
- package/.claude/agents/sales-outreach.md +425 -0
- package/.claude/agents/sales-pipeline-analyst.md +267 -0
- package/.claude/agents/sales-proposal-strategist.md +217 -0
- package/.claude/agents/security-ai-generated-code-auditor.md +207 -0
- package/.claude/agents/security-appsec-engineer.md +536 -0
- package/.claude/agents/security-architect.md +307 -0
- package/.claude/agents/security-blockchain-security-auditor.md +482 -0
- package/.claude/agents/security-cloud-security-architect.md +576 -0
- package/.claude/agents/security-compliance-auditor.md +158 -0
- package/.claude/agents/security-incident-responder.md +450 -0
- package/.claude/agents/security-penetration-tester.md +399 -0
- package/.claude/agents/security-secrets-credential-engineer.md +189 -0
- package/.claude/agents/security-senior-secops.md +760 -0
- package/.claude/agents/security-threat-detection-engineer.md +553 -0
- package/.claude/agents/security-threat-intelligence-analyst.md +661 -0
- package/.claude/agents/specialized-chief-of-staff.md +279 -0
- package/.claude/agents/specialized-civil-engineer.md +356 -0
- package/.claude/agents/specialized-codebase-archaeologist.md +341 -0
- package/.claude/agents/specialized-cultural-intelligence-strategist.md +88 -0
- package/.claude/agents/specialized-developer-advocate.md +317 -0
- package/.claude/agents/specialized-document-generator.md +55 -0
- package/.claude/agents/specialized-fedramp-rmf-compliance.md +378 -0
- package/.claude/agents/specialized-focus-music-architect.md +175 -0
- package/.claude/agents/specialized-french-consulting-market.md +194 -0
- package/.claude/agents/specialized-korean-business-navigator.md +216 -0
- package/.claude/agents/specialized-master-plan-architect.md +157 -0
- package/.claude/agents/specialized-mcp-builder.md +260 -0
- package/.claude/agents/specialized-model-qa.md +528 -0
- package/.claude/agents/specialized-pricing-analyst.md +243 -0
- package/.claude/agents/specialized-salesforce-architect.md +182 -0
- package/.claude/agents/specialized-strategy-duel-agent.md +130 -0
- package/.claude/agents/specialized-workflow-architect.md +597 -0
- package/.claude/agents/study-abroad-advisor.md +282 -0
- package/.claude/agents/supply-chain-strategist.md +591 -0
- package/.claude/agents/support-analytics-reporter.md +387 -0
- package/.claude/agents/support-executive-summary-generator.md +212 -0
- package/.claude/agents/support-finance-tracker.md +480 -0
- package/.claude/agents/support-infrastructure-maintainer.md +628 -0
- package/.claude/agents/support-legal-compliance-checker.md +591 -0
- package/.claude/agents/support-support-responder.md +607 -0
- package/.claude/agents/technical-artist.md +233 -0
- package/.claude/agents/terminal-integration-specialist.md +70 -0
- package/.claude/agents/testing-accessibility-auditor.md +327 -0
- package/.claude/agents/testing-api-tester.md +334 -0
- package/.claude/agents/testing-evidence-collector.md +210 -0
- package/.claude/agents/testing-performance-benchmarker.md +283 -0
- package/.claude/agents/testing-reality-checker.md +249 -0
- package/.claude/agents/testing-test-automation-engineer.md +196 -0
- package/.claude/agents/testing-test-results-analyzer.md +330 -0
- package/.claude/agents/testing-tool-evaluator.md +416 -0
- package/.claude/agents/testing-workflow-optimizer.md +463 -0
- package/.claude/agents/unity-architect.md +271 -0
- package/.claude/agents/unity-editor-tool-developer.md +310 -0
- package/.claude/agents/unity-multiplayer-engineer.md +323 -0
- package/.claude/agents/unity-shader-graph-artist.md +269 -0
- package/.claude/agents/unreal-multiplayer-architect.md +364 -0
- package/.claude/agents/unreal-systems-engineer.md +322 -0
- package/.claude/agents/unreal-technical-artist.md +256 -0
- package/.claude/agents/unreal-world-builder.md +273 -0
- package/.claude/agents/visionos-spatial-engineer.md +54 -0
- package/.claude/agents/xr-cockpit-interaction-specialist.md +32 -0
- package/.claude/agents/xr-immersive-developer.md +32 -0
- package/.claude/agents/xr-interface-architect.md +32 -0
- package/.claude/agents/zk-steward.md +211 -0
- package/.claude-plugin/marketplace.json +30 -0
- package/.commitlintrc.json +26 -0
- package/.devcontainer/Dockerfile +24 -0
- package/.devcontainer/devcontainer.json +70 -0
- package/.devcontainer/docker-compose.memory.yml +42 -0
- package/.devcontainer/memory-stack/devcontainer.json +84 -0
- package/.devcontainer/post-create.sh +83 -0
- package/.dockerignore +23 -0
- package/.env.act.example +15 -0
- package/.env.example +171 -0
- package/.eslintrc.json +16 -0
- package/.git-blame-ignore-revs +10 -0
- package/.gitattributes +2 -0
- package/.gitguardian.yaml +50 -0
- package/.githooks/pre-commit +6 -0
- package/.pre-commit-config.yaml +30 -0
- package/.release-please-config-xithead.json +22 -0
- package/.release-please-config.json +44 -0
- package/.release-please-manifest-xithead.json +3 -0
- package/.release-please-manifest.json +3 -0
- package/CODE_OF_CONDUCT.md +133 -0
- package/CONTRIBUTING.md +133 -0
- package/Cargo.lock +5557 -0
- package/Cargo.toml +121 -0
- package/Dockerfile +38 -0
- package/Dockerfile.proxy +48 -0
- package/ENTERPRISE.md +5 -0
- package/FEATURES_ADVANCED.md +311 -0
- package/Headroom-2.gif +0 -0
- package/HeadroomDemo-Fast.gif +0 -0
- package/MANIFEST.in +4 -0
- package/MODERNIZATION_GUIDE.md +358 -0
- package/Makefile +140 -0
- package/NOTICE +43 -0
- package/NPM_TOKEN_SETUP.md +81 -0
- package/PR.md +170 -0
- package/PUBLISH.md +77 -0
- package/README.md +108 -72
- package/README_FRONTEND.md +196 -0
- package/REALIGNMENT/00-overview.md +53 -0
- package/REALIGNMENT/01-bug-list.md +421 -0
- package/REALIGNMENT/02-architecture.md +322 -0
- package/REALIGNMENT/03-phase-A-lockdown.md +420 -0
- package/REALIGNMENT/04-phase-B-live-zone.md +470 -0
- package/REALIGNMENT/05-phase-C-rust-proxy.md +329 -0
- package/REALIGNMENT/06-phase-D-bedrock-vertex.md +222 -0
- package/REALIGNMENT/07-phase-E-cache-stabilization.md +338 -0
- package/REALIGNMENT/08-phase-F-auth-mode.md +254 -0
- package/REALIGNMENT/09-phase-G-rtk-observability.md +194 -0
- package/REALIGNMENT/10-phase-H-python-retirement.md +214 -0
- package/REALIGNMENT/11-phase-I-test-infra.md +385 -0
- package/REALIGNMENT/12-decisions-needed.md +196 -0
- package/REALIGNMENT/INDEX.md +82 -0
- package/REFACTORING_CAMPAIGN.md +277 -0
- package/RESTRUCTURE_PLAN.md +100 -0
- package/RUST_DEV.md +361 -0
- package/SECURITY.md +65 -0
- package/SETUP_AUTO_PUBLISH.md +164 -0
- package/TESTING-copilot-subscription.md +153 -0
- package/app/api/config/route.ts +22 -0
- package/app/api/export/route.ts +83 -0
- package/app/api/models/route.ts +12 -0
- package/app/api/stats/route.ts +46 -0
- package/app/dashboard/page.tsx +445 -0
- package/app/globals.css +35 -0
- package/app/installation/page.tsx +293 -0
- package/app/layout.tsx +35 -0
- package/app/master-design-system.css +2076 -0
- package/app/master-design-system.source.css +8 -0
- package/app/page.tsx +187 -0
- package/app/rechtliches/agb/page.tsx +165 -0
- package/app/rechtliches/datenschutz/page.tsx +128 -0
- package/app/rechtliches/impressum/page.tsx +122 -0
- package/app/rechtliches/widerruf/page.tsx +185 -0
- package/bin/xithead.js +264 -0
- package/codecov.yml +34 -0
- package/components/layout/AppShell.tsx +25 -0
- package/components/layout/Footer.tsx +120 -0
- package/components/layout/Header.tsx +145 -0
- package/components/marketing/LegalHighlights.tsx +45 -0
- package/components/marketing/LegalPageShell.tsx +49 -0
- package/components/marketing/LegalPanel.tsx +41 -0
- package/components/marketing/index.ts +3 -0
- package/crates/headroom-core/Cargo.toml +165 -0
- package/crates/headroom-core/benches/auth_mode.rs +74 -0
- package/crates/headroom-core/benches/ccr_store.rs +224 -0
- package/crates/headroom-core/benches/tokenizer.rs +52 -0
- package/crates/headroom-core/config/pipeline.toml +151 -0
- package/crates/headroom-core/proptest-regressions/transforms/tag_protector.txt +9 -0
- package/crates/headroom-core/src/auth_mode.rs +250 -0
- package/crates/headroom-core/src/cache_control.rs +414 -0
- package/crates/headroom-core/src/ccr/backends/in_memory.rs +317 -0
- package/crates/headroom-core/src/ccr/backends/mod.rs +152 -0
- package/crates/headroom-core/src/ccr/backends/redis.rs +146 -0
- package/crates/headroom-core/src/ccr/backends/sqlite.rs +205 -0
- package/crates/headroom-core/src/ccr/mod.rs +119 -0
- package/crates/headroom-core/src/compression_policy.rs +505 -0
- package/crates/headroom-core/src/lib.rs +33 -0
- package/crates/headroom-core/src/relevance/base.rs +139 -0
- package/crates/headroom-core/src/relevance/bm25.rs +385 -0
- package/crates/headroom-core/src/relevance/embedding.rs +354 -0
- package/crates/headroom-core/src/relevance/hybrid.rs +364 -0
- package/crates/headroom-core/src/relevance/mod.rs +64 -0
- package/crates/headroom-core/src/signals/keyword_detector.rs +433 -0
- package/crates/headroom-core/src/signals/line_importance.rs +84 -0
- package/crates/headroom-core/src/signals/mod.rs +61 -0
- package/crates/headroom-core/src/signals/tiered.rs +141 -0
- package/crates/headroom-core/src/tokenizer/estimator.rs +152 -0
- package/crates/headroom-core/src/tokenizer/hf_impl.rs +314 -0
- package/crates/headroom-core/src/tokenizer/mod.rs +56 -0
- package/crates/headroom-core/src/tokenizer/registry.rs +342 -0
- package/crates/headroom-core/src/tokenizer/tiktoken_impl.rs +265 -0
- package/crates/headroom-core/src/transforms/adaptive_sizer.rs +610 -0
- package/crates/headroom-core/src/transforms/anchor_selector.rs +1189 -0
- package/crates/headroom-core/src/transforms/content_detector.rs +769 -0
- package/crates/headroom-core/src/transforms/detection.rs +222 -0
- package/crates/headroom-core/src/transforms/diff_compressor.rs +1685 -0
- package/crates/headroom-core/src/transforms/live_zone.rs +2967 -0
- package/crates/headroom-core/src/transforms/log_compressor.rs +1295 -0
- package/crates/headroom-core/src/transforms/magika_detector.rs +395 -0
- package/crates/headroom-core/src/transforms/mod.rs +64 -0
- package/crates/headroom-core/src/transforms/pipeline/config.rs +331 -0
- package/crates/headroom-core/src/transforms/pipeline/mod.rs +103 -0
- package/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rs +524 -0
- package/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs +279 -0
- package/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs +400 -0
- package/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rs +284 -0
- package/crates/headroom-core/src/transforms/pipeline/offloads/mod.rs +34 -0
- package/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rs +331 -0
- package/crates/headroom-core/src/transforms/pipeline/orchestrator.rs +848 -0
- package/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rs +175 -0
- package/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rs +545 -0
- package/crates/headroom-core/src/transforms/pipeline/reformats/mod.rs +12 -0
- package/crates/headroom-core/src/transforms/pipeline/traits.rs +338 -0
- package/crates/headroom-core/src/transforms/recommendations.rs +329 -0
- package/crates/headroom-core/src/transforms/safety.rs +215 -0
- package/crates/headroom-core/src/transforms/search_compressor.rs +902 -0
- package/crates/headroom-core/src/transforms/smart_crusher/analyzer.rs +1262 -0
- package/crates/headroom-core/src/transforms/smart_crusher/anchors.rs +407 -0
- package/crates/headroom-core/src/transforms/smart_crusher/builder.rs +263 -0
- package/crates/headroom-core/src/transforms/smart_crusher/classifier.rs +214 -0
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/classifier.rs +283 -0
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/compactor.rs +733 -0
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rs +961 -0
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/ir.rs +252 -0
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs +127 -0
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs +397 -0
- package/crates/headroom-core/src/transforms/smart_crusher/config.rs +173 -0
- package/crates/headroom-core/src/transforms/smart_crusher/constraints.rs +159 -0
- package/crates/headroom-core/src/transforms/smart_crusher/crusher.rs +1665 -0
- package/crates/headroom-core/src/transforms/smart_crusher/crushers.rs +865 -0
- package/crates/headroom-core/src/transforms/smart_crusher/error_keywords.rs +77 -0
- package/crates/headroom-core/src/transforms/smart_crusher/field_detect.rs +393 -0
- package/crates/headroom-core/src/transforms/smart_crusher/hashing.rs +72 -0
- package/crates/headroom-core/src/transforms/smart_crusher/mod.rs +81 -0
- package/crates/headroom-core/src/transforms/smart_crusher/observer.rs +66 -0
- package/crates/headroom-core/src/transforms/smart_crusher/orchestration.rs +459 -0
- package/crates/headroom-core/src/transforms/smart_crusher/outliers.rs +507 -0
- package/crates/headroom-core/src/transforms/smart_crusher/planning.rs +904 -0
- package/crates/headroom-core/src/transforms/smart_crusher/statistics.rs +510 -0
- package/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs +287 -0
- package/crates/headroom-core/src/transforms/smart_crusher/traits.rs +203 -0
- package/crates/headroom-core/src/transforms/smart_crusher/types.rs +263 -0
- package/crates/headroom-core/src/transforms/tag_protector.rs +1272 -0
- package/crates/headroom-core/src/transforms/unidiff_detector.rs +232 -0
- package/crates/headroom-parity/Cargo.toml +26 -0
- package/crates/headroom-parity/examples/diff_fixture.rs +66 -0
- package/crates/headroom-parity/src/bin/parity_run.rs +78 -0
- package/crates/headroom-parity/src/lib.rs +613 -0
- package/crates/headroom-proxy/Cargo.toml +119 -0
- package/crates/headroom-proxy/data/model_prices_and_context_window.json +39767 -0
- package/crates/headroom-proxy/src/bedrock/auth_mode_layer.rs +192 -0
- package/crates/headroom-proxy/src/bedrock/envelope.rs +239 -0
- package/crates/headroom-proxy/src/bedrock/eventstream.rs +769 -0
- package/crates/headroom-proxy/src/bedrock/eventstream_to_sse.rs +459 -0
- package/crates/headroom-proxy/src/bedrock/invoke.rs +735 -0
- package/crates/headroom-proxy/src/bedrock/invoke_streaming.rs +1098 -0
- package/crates/headroom-proxy/src/bedrock/mod.rs +66 -0
- package/crates/headroom-proxy/src/bedrock/sigv4.rs +324 -0
- package/crates/headroom-proxy/src/bedrock/vendor.rs +82 -0
- package/crates/headroom-proxy/src/cache_stabilization/anthropic_cache_control.rs +698 -0
- package/crates/headroom-proxy/src/cache_stabilization/drift_detector.rs +683 -0
- package/crates/headroom-proxy/src/cache_stabilization/mod.rs +61 -0
- package/crates/headroom-proxy/src/cache_stabilization/openai_cache_key.rs +600 -0
- package/crates/headroom-proxy/src/cache_stabilization/tool_def_normalize.rs +514 -0
- package/crates/headroom-proxy/src/cache_stabilization/volatile_detector.rs +720 -0
- package/crates/headroom-proxy/src/compression/anthropic.rs +92 -0
- package/crates/headroom-proxy/src/compression/live_zone_anthropic.rs +1311 -0
- package/crates/headroom-proxy/src/compression/live_zone_openai.rs +660 -0
- package/crates/headroom-proxy/src/compression/live_zone_responses.rs +678 -0
- package/crates/headroom-proxy/src/compression/mod.rs +127 -0
- package/crates/headroom-proxy/src/compression/model_limits.rs +211 -0
- package/crates/headroom-proxy/src/config.rs +646 -0
- package/crates/headroom-proxy/src/error.rs +69 -0
- package/crates/headroom-proxy/src/handlers/chat_completions.rs +100 -0
- package/crates/headroom-proxy/src/handlers/conversations.rs +241 -0
- package/crates/headroom-proxy/src/handlers/mod.rs +13 -0
- package/crates/headroom-proxy/src/handlers/responses.rs +256 -0
- package/crates/headroom-proxy/src/headers.rs +288 -0
- package/crates/headroom-proxy/src/health.rs +41 -0
- package/crates/headroom-proxy/src/lib.rs +21 -0
- package/crates/headroom-proxy/src/main.rs +141 -0
- package/crates/headroom-proxy/src/observability/cache_hit_rate.rs +340 -0
- package/crates/headroom-proxy/src/observability/compression_ratio.rs +158 -0
- package/crates/headroom-proxy/src/observability/metric_names.rs +174 -0
- package/crates/headroom-proxy/src/observability/mod.rs +68 -0
- package/crates/headroom-proxy/src/observability/prometheus.rs +394 -0
- package/crates/headroom-proxy/src/observability/proxy_metrics.rs +368 -0
- package/crates/headroom-proxy/src/proxy.rs +1619 -0
- package/crates/headroom-proxy/src/responses_items.rs +565 -0
- package/crates/headroom-proxy/src/sse/anthropic.rs +415 -0
- package/crates/headroom-proxy/src/sse/framing.rs +366 -0
- package/crates/headroom-proxy/src/sse/mod.rs +40 -0
- package/crates/headroom-proxy/src/sse/openai_chat.rs +231 -0
- package/crates/headroom-proxy/src/sse/openai_responses.rs +458 -0
- package/crates/headroom-proxy/src/vertex/adc.rs +315 -0
- package/crates/headroom-proxy/src/vertex/envelope.rs +167 -0
- package/crates/headroom-proxy/src/vertex/mod.rs +277 -0
- package/crates/headroom-proxy/src/vertex/raw_predict.rs +490 -0
- package/crates/headroom-proxy/src/vertex/stream_raw_predict.rs +49 -0
- package/crates/headroom-proxy/src/websocket.rs +246 -0
- package/crates/headroom-py/Cargo.toml +37 -0
- package/crates/headroom-py/build.rs +77 -0
- package/crates/headroom-py/glibc_compat.c +170 -0
- package/crates/headroom-py/src/lib.rs +1655 -0
- package/deny.toml +32 -0
- package/design-system/base.css +127 -0
- package/design-system/components.css +1943 -0
- package/design-system/fonts.css +17 -0
- package/design-system/tailwind-preset.js +95 -0
- package/design-system/tokens.css +48 -0
- package/design-system/utilities.css +53 -0
- package/docker/differential-network-capture/Dockerfile.runner +14 -0
- package/docker/differential-network-capture/docker-compose.yml +142 -0
- package/docker/differential-network-capture/run-claude-lane.sh +24 -0
- package/docker/docker-compose.native.yml +44 -0
- package/docker-bake.hcl +88 -0
- package/docker-compose.yml +80 -0
- package/e2e/_lib/make_shim.ps1 +24 -0
- package/e2e/_lib/make_shim.sh +28 -0
- package/e2e/docker-native-install.sh +120 -0
- package/e2e/init/Dockerfile +79 -0
- package/e2e/wrap/Dockerfile +115 -0
- package/examples/07-context-compression.ipynb +279 -0
- package/examples/deployment/macos-launchagent/com.headroom.proxy.plist.template +65 -0
- package/examples/deployment/macos-launchagent/install.sh +227 -0
- package/examples/deployment/macos-launchagent/shell-integration.sh +74 -0
- package/examples/deployment/macos-launchagent/uninstall.sh +145 -0
- package/headroom-savings.png +0 -0
- package/headroom_learn.gif +0 -0
- package/lib/config/index.ts +12 -0
- package/lib/hooks/index.ts +4 -0
- package/lib/hooks/use-mounted.ts +13 -0
- package/lib/hooks/use-session.ts +34 -0
- package/lib/hooks/use-settings.ts +49 -0
- package/lib/utils/index.ts +1 -0
- package/lib/utils/settings.ts +162 -0
- package/lib/utils/utils.ts +6 -0
- package/llms.txt +67 -0
- package/middleware.ts +26 -0
- package/mkdocs.yml +136 -0
- package/next-env.d.ts +7 -0
- package/next.config.ts +12 -0
- package/npm/bin/xithead.js +366 -366
- package/npm/bin/xtream-head.js +366 -0
- package/npm/index.d.ts +57 -57
- package/npm/index.js +139 -139
- package/npm/package.json +55 -54
- package/npm/postinstall.js +43 -43
- package/package.json +58 -36
- package/plugins/headroom-agent-hooks/.claude-plugin/plugin.json +17 -0
- package/plugins/headroom-agent-hooks/hooks/hooks.json +29 -0
- package/plugins/headroom-oauth2/LICENSE +3 -0
- package/plugins/headroom-oauth2/SPEC.md +115 -0
- package/plugins/headroom-oauth2/pyproject.toml +48 -0
- package/plugins/hermes/headroom_retrieve/plugin.yaml +6 -0
- package/plugins/openclaw/hook-shim/HOOK.md +21 -0
- package/plugins/openclaw/hook-shim/handler.js +3 -0
- package/plugins/openclaw/openclaw.plugin.json +98 -0
- package/plugins/openclaw/package-lock.json +2692 -0
- package/plugins/openclaw/package.json +54 -0
- package/plugins/openclaw/prepare-dist.mjs +56 -0
- package/plugins/openclaw/src/convert.ts +418 -0
- package/plugins/openclaw/src/engine.ts +284 -0
- package/plugins/openclaw/src/gateway-config.ts +143 -0
- package/plugins/openclaw/src/index.ts +11 -0
- package/plugins/openclaw/src/plugin/index.ts +98 -0
- package/plugins/openclaw/src/proxy-manager.ts +477 -0
- package/plugins/openclaw/src/tools/headroom-retrieve.ts +76 -0
- package/plugins/openclaw/test/convert.test.ts +96 -0
- package/plugins/openclaw/test/engine-normalization.test.ts +21 -0
- package/plugins/openclaw/test/engine.test.ts +101 -0
- package/plugins/openclaw/test/gateway-config.test.ts +278 -0
- package/plugins/openclaw/test/plugin-runtime-routing.test.ts +141 -0
- package/plugins/openclaw/test/proxy-manager.test.ts +343 -0
- package/plugins/openclaw/tsup.config.ts +9 -0
- package/postcss.config.mjs +7 -0
- package/pyproject.toml +438 -0
- package/rust-toolchain.toml +16 -0
- package/sdk/typescript/examples/basic-compress.ts +64 -0
- package/sdk/typescript/examples/ccr-retrieve.ts +80 -0
- package/sdk/typescript/examples/hooks-custom-compression.ts +117 -0
- package/sdk/typescript/examples/middleware-composition.ts +59 -0
- package/sdk/typescript/examples/multi-provider.ts +53 -0
- package/sdk/typescript/examples/openai-anthropic-adapters.ts +73 -0
- package/sdk/typescript/examples/shared-context-multi-agent.ts +74 -0
- package/sdk/typescript/examples/simulation-dry-run.ts +80 -0
- package/sdk/typescript/examples/streaming-chat.ts +71 -0
- package/sdk/typescript/examples/structured-output.ts +99 -0
- package/sdk/typescript/examples/tool-calling-agent.ts +86 -0
- package/sdk/typescript/examples/with-headroom-vercel.ts +57 -0
- package/sdk/typescript/package-lock.json +3437 -0
- package/sdk/typescript/package.json +108 -0
- package/sdk/typescript/src/adapters/anthropic.ts +218 -0
- package/sdk/typescript/src/adapters/gemini.ts +83 -0
- package/sdk/typescript/src/adapters/openai.ts +77 -0
- package/sdk/typescript/src/adapters/vercel-ai.ts +131 -0
- package/sdk/typescript/src/client.ts +632 -0
- package/sdk/typescript/src/compress.ts +87 -0
- package/sdk/typescript/src/errors.ts +114 -0
- package/sdk/typescript/src/hooks.ts +113 -0
- package/sdk/typescript/src/index.ts +140 -0
- package/sdk/typescript/src/paths.ts +272 -0
- package/sdk/typescript/src/shared-context.ts +188 -0
- package/sdk/typescript/src/simulate.ts +52 -0
- package/sdk/typescript/src/types/config.ts +183 -0
- package/sdk/typescript/src/types/models.ts +314 -0
- package/sdk/typescript/src/types.ts +139 -0
- package/sdk/typescript/src/utils/case.ts +34 -0
- package/sdk/typescript/src/utils/format.ts +480 -0
- package/sdk/typescript/src/utils/stream.ts +50 -0
- package/sdk/typescript/test/adapters/anthropic.test.ts +217 -0
- package/sdk/typescript/test/adapters/gemini.test.ts +131 -0
- package/sdk/typescript/test/adapters/openai.test.ts +174 -0
- package/sdk/typescript/test/adapters/vercel-ai.test.ts +306 -0
- package/sdk/typescript/test/client-expanded.test.ts +333 -0
- package/sdk/typescript/test/client.test.ts +266 -0
- package/sdk/typescript/test/compress.test.ts +118 -0
- package/sdk/typescript/test/config.test.ts +275 -0
- package/sdk/typescript/test/errors.test.ts +169 -0
- package/sdk/typescript/test/format-integration.test.ts +252 -0
- package/sdk/typescript/test/hooks.test.ts +265 -0
- package/sdk/typescript/test/integration.test.ts +249 -0
- package/sdk/typescript/test/mdx-examples.test.ts +253 -0
- package/sdk/typescript/test/models.test.ts +318 -0
- package/sdk/typescript/test/parity.test.ts +337 -0
- package/sdk/typescript/test/paths.test.ts +447 -0
- package/sdk/typescript/test/shared-context.test.ts +245 -0
- package/sdk/typescript/test/simulate.test.ts +84 -0
- package/sdk/typescript/test/types.test.ts +244 -0
- package/sdk/typescript/test/utils/case.test.ts +129 -0
- package/sdk/typescript/test/utils/detect-format.test.ts +133 -0
- package/sdk/typescript/test/utils/format.test.ts +434 -0
- package/sdk/typescript/test/vercel-ai-e2e.test.ts +179 -0
- package/sdk/typescript/tsup.config.ts +17 -0
- package/tailwind.master.config.cjs +47 -0
- package/types/api.ts +12 -0
- package/types/config.ts +9 -0
- package/types/index.ts +2 -0
- package/uv.lock +6951 -0
- package/xithead.egg-info/PKG-INFO +22 -0
- package/xithead.egg-info/SOURCES.txt +16 -0
- package/xithead.egg-info/dependency_links.txt +1 -0
- package/xithead.egg-info/entry_points.txt +2 -0
- package/xithead.egg-info/requires.txt +10 -0
- package/xithead.egg-info/top_level.txt +2 -0
- package/xithead.ps1 +136 -0
- package/xithead.sh +129 -0
- package/xithead_dist/pyproject.toml +39 -0
- package/xithead_dist/xithead.egg-info/PKG-INFO +22 -0
- package/xithead_dist/xithead.egg-info/SOURCES.txt +16 -0
- package/xithead_dist/xithead.egg-info/dependency_links.txt +1 -0
- package/xithead_dist/xithead.egg-info/entry_points.txt +3 -0
- package/xithead_dist/xithead.egg-info/requires.txt +10 -0
- package/xithead_dist/xithead.egg-info/top_level.txt +2 -0
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
//! Composition combinator for layered detectors.
|
|
2
|
+
//!
|
|
3
|
+
//! `Tiered<dyn Trait>` chains an ordered list of detectors. The first
|
|
4
|
+
//! tier whose signal exceeds [`ESCALATE_THRESHOLD`] confidence wins;
|
|
5
|
+
//! lower-confidence tiers are skipped past. If no tier exceeds the
|
|
6
|
+
//! threshold, the highest-confidence signal seen is returned (so the
|
|
7
|
+
//! caller still gets the best guess, with the confidence score
|
|
8
|
+
//! reflecting how unsure the stack is).
|
|
9
|
+
//!
|
|
10
|
+
//! Tiering is *composition*, not inheritance. `KeywordDetector` knows
|
|
11
|
+
//! nothing about a future ML detector; the ML detector knows nothing
|
|
12
|
+
//! about the keyword detector. They both implement the trait and the
|
|
13
|
+
//! `Tiered` wrapper orders them.
|
|
14
|
+
|
|
15
|
+
use super::line_importance::{ImportanceContext, ImportanceSignal, LineImportanceDetector};
|
|
16
|
+
|
|
17
|
+
/// Confidence at which `Tiered` accepts a tier's signal without
|
|
18
|
+
/// consulting later tiers. KeywordDetector emits 0.7, so it wins by
|
|
19
|
+
/// default; an ML tier with calibrated confidence ≥ 0.8 (high-precision
|
|
20
|
+
/// region) would short-circuit the keyword tier.
|
|
21
|
+
pub const ESCALATE_THRESHOLD: f32 = 0.7;
|
|
22
|
+
|
|
23
|
+
pub struct Tiered<T: ?Sized> {
|
|
24
|
+
tiers: Vec<Box<T>>,
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
impl<T: ?Sized> Tiered<T> {
|
|
28
|
+
pub fn new() -> Self {
|
|
29
|
+
Self { tiers: Vec::new() }
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/// Push a tier onto the stack. Order matters: most-precise first.
|
|
33
|
+
pub fn with(mut self, tier: Box<T>) -> Self {
|
|
34
|
+
self.tiers.push(tier);
|
|
35
|
+
self
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
pub fn len(&self) -> usize {
|
|
39
|
+
self.tiers.len()
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
pub fn is_empty(&self) -> bool {
|
|
43
|
+
self.tiers.is_empty()
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
impl<T: ?Sized> Default for Tiered<T> {
|
|
48
|
+
fn default() -> Self {
|
|
49
|
+
Self::new()
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
impl LineImportanceDetector for Tiered<dyn LineImportanceDetector> {
|
|
54
|
+
fn score(&self, line: &str, ctx: ImportanceContext) -> ImportanceSignal {
|
|
55
|
+
let mut best = ImportanceSignal::neutral();
|
|
56
|
+
for tier in &self.tiers {
|
|
57
|
+
let signal = tier.score(line, ctx);
|
|
58
|
+
if signal.confidence >= ESCALATE_THRESHOLD {
|
|
59
|
+
return signal;
|
|
60
|
+
}
|
|
61
|
+
if signal.confidence > best.confidence {
|
|
62
|
+
best = signal;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
best
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
impl Tiered<dyn LineImportanceDetector> {
|
|
70
|
+
/// Convenience: take an owned detector, box it, coerce it to the
|
|
71
|
+
/// trait object. Keeps callsites free of `as Box<dyn …>` clutter.
|
|
72
|
+
pub fn with_detector<D: LineImportanceDetector + 'static>(self, detector: D) -> Self {
|
|
73
|
+
self.with(Box::new(detector) as Box<dyn LineImportanceDetector>)
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
#[cfg(test)]
|
|
78
|
+
mod tests {
|
|
79
|
+
use super::*;
|
|
80
|
+
use crate::signals::keyword_detector::KeywordDetector;
|
|
81
|
+
use crate::signals::line_importance::ImportanceCategory;
|
|
82
|
+
|
|
83
|
+
/// Synthetic high-confidence detector for testing short-circuit
|
|
84
|
+
/// behavior. Always asserts a specific signal so we can prove
|
|
85
|
+
/// `Tiered` consults it before the keyword tier.
|
|
86
|
+
struct AlwaysFiresHigh;
|
|
87
|
+
impl LineImportanceDetector for AlwaysFiresHigh {
|
|
88
|
+
fn score(&self, _line: &str, _ctx: ImportanceContext) -> ImportanceSignal {
|
|
89
|
+
ImportanceSignal::matched(ImportanceCategory::Security, 0.99, 0.95)
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/// Synthetic low-confidence detector. Confidence 0.5 is below the
|
|
94
|
+
/// escalate threshold so `Tiered` MUST fall through to the next
|
|
95
|
+
/// tier.
|
|
96
|
+
struct AlwaysFiresLow;
|
|
97
|
+
impl LineImportanceDetector for AlwaysFiresLow {
|
|
98
|
+
fn score(&self, _line: &str, _ctx: ImportanceContext) -> ImportanceSignal {
|
|
99
|
+
ImportanceSignal::matched(ImportanceCategory::Importance, 0.4, 0.5)
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
#[test]
|
|
104
|
+
fn high_confidence_tier_short_circuits() {
|
|
105
|
+
let tiered: Tiered<dyn LineImportanceDetector> = Tiered::new()
|
|
106
|
+
.with_detector(AlwaysFiresHigh)
|
|
107
|
+
.with_detector(KeywordDetector::new());
|
|
108
|
+
let s = tiered.score("ERROR: connection refused", ImportanceContext::Diff);
|
|
109
|
+
// AlwaysFiresHigh asserts Security; if the keyword detector ran
|
|
110
|
+
// it would have asserted Error.
|
|
111
|
+
assert_eq!(s.category, Some(ImportanceCategory::Security));
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
#[test]
|
|
115
|
+
fn low_confidence_tier_falls_through_to_keyword() {
|
|
116
|
+
let tiered: Tiered<dyn LineImportanceDetector> = Tiered::new()
|
|
117
|
+
.with_detector(AlwaysFiresLow)
|
|
118
|
+
.with_detector(KeywordDetector::new());
|
|
119
|
+
let s = tiered.score("ERROR: connection refused", ImportanceContext::Diff);
|
|
120
|
+
assert_eq!(s.category, Some(ImportanceCategory::Error));
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
#[test]
|
|
124
|
+
fn no_tier_matches_returns_best_seen() {
|
|
125
|
+
let tiered: Tiered<dyn LineImportanceDetector> = Tiered::new()
|
|
126
|
+
.with_detector(AlwaysFiresLow)
|
|
127
|
+
.with_detector(KeywordDetector::new());
|
|
128
|
+
let s = tiered.score("the quick brown fox", ImportanceContext::Text);
|
|
129
|
+
// Keyword detector returns neutral (confidence 0.0); AlwaysFiresLow
|
|
130
|
+
// returned Importance @ 0.5 so that wins as best-seen.
|
|
131
|
+
assert_eq!(s.category, Some(ImportanceCategory::Importance));
|
|
132
|
+
assert_eq!(s.confidence, 0.5);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
#[test]
|
|
136
|
+
fn empty_stack_returns_neutral() {
|
|
137
|
+
let tiered: Tiered<dyn LineImportanceDetector> = Tiered::new();
|
|
138
|
+
let s = tiered.score("anything", ImportanceContext::Text);
|
|
139
|
+
assert!(!s.is_match());
|
|
140
|
+
}
|
|
141
|
+
}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
//! Character-density estimator. Used as a fallback for any tokenizer family
|
|
2
|
+
//! we haven't wired in yet (Anthropic Claude, Google Gemini, Cohere, …).
|
|
3
|
+
//!
|
|
4
|
+
//! Mirrors `headroom.tokenizers.estimator.EstimatingTokenCounter`. The formula
|
|
5
|
+
//! is `ceil(chars / chars_per_token)`. `chars` is *Unicode scalar count*, not
|
|
6
|
+
//! byte length, to match Python's `len(text)` semantics on str.
|
|
7
|
+
|
|
8
|
+
use super::{Backend, Tokenizer};
|
|
9
|
+
|
|
10
|
+
#[derive(Debug, Clone, Copy)]
|
|
11
|
+
pub struct EstimatingCounter {
|
|
12
|
+
chars_per_token: f64,
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
impl Default for EstimatingCounter {
|
|
16
|
+
fn default() -> Self {
|
|
17
|
+
Self {
|
|
18
|
+
chars_per_token: 4.0,
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
impl EstimatingCounter {
|
|
24
|
+
/// `chars_per_token` must be `> 0.0`. Common calibrations:
|
|
25
|
+
/// - 3.5 — Claude-family (Python uses this in `_create_anthropic`)
|
|
26
|
+
/// - 4.0 — Gemini, Cohere, generic fallback
|
|
27
|
+
pub fn new(chars_per_token: f64) -> Self {
|
|
28
|
+
assert!(
|
|
29
|
+
chars_per_token > 0.0,
|
|
30
|
+
"chars_per_token must be positive, got {chars_per_token}"
|
|
31
|
+
);
|
|
32
|
+
Self { chars_per_token }
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
pub fn chars_per_token(&self) -> f64 {
|
|
36
|
+
self.chars_per_token
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
impl Tokenizer for EstimatingCounter {
|
|
41
|
+
fn count_text(&self, text: &str) -> usize {
|
|
42
|
+
if text.is_empty() {
|
|
43
|
+
return 0;
|
|
44
|
+
}
|
|
45
|
+
// Match Python `EstimatingTokenCounter.count_text`:
|
|
46
|
+
// max(1, int(len(text) / chars_per_token + 0.5))
|
|
47
|
+
// Python `int()` truncates toward zero; for non-negative inputs that's
|
|
48
|
+
// identical to `as usize` saturating-cast semantics in Rust >= 1.45.
|
|
49
|
+
// Adding 0.5 then truncating yields round-half-up. We previously used
|
|
50
|
+
// ceil, which over-counted in the middle of the range (e.g. "aaaaa"
|
|
51
|
+
// at 4.0 cpt returned 2 here vs 1 in Python).
|
|
52
|
+
let chars = text.chars().count() as f64;
|
|
53
|
+
let raw = (chars / self.chars_per_token + 0.5) as usize;
|
|
54
|
+
raw.max(1)
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
fn backend(&self) -> Backend {
|
|
58
|
+
Backend::Estimation
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
#[cfg(test)]
|
|
63
|
+
mod tests {
|
|
64
|
+
use super::*;
|
|
65
|
+
|
|
66
|
+
#[test]
|
|
67
|
+
fn empty_string_is_zero() {
|
|
68
|
+
let est = EstimatingCounter::default();
|
|
69
|
+
assert_eq!(est.count_text(""), 0);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/// Expected values cross-checked against Python:
|
|
73
|
+
/// max(1, int(len / chars_per_token + 0.5))
|
|
74
|
+
#[test]
|
|
75
|
+
fn default_is_four_chars_per_token() {
|
|
76
|
+
let est = EstimatingCounter::default();
|
|
77
|
+
// 4 / 4.0 = 1.0 + 0.5 -> int(1.5) -> 1
|
|
78
|
+
assert_eq!(est.count_text("aaaa"), 1);
|
|
79
|
+
// 5 / 4.0 = 1.25 + 0.5 -> int(1.75) -> 1 (Python rounds half-up)
|
|
80
|
+
assert_eq!(est.count_text("aaaaa"), 1);
|
|
81
|
+
// 6 / 4.0 = 1.5 + 0.5 -> int(2.0) -> 2
|
|
82
|
+
assert_eq!(est.count_text("aaaaaa"), 2);
|
|
83
|
+
// 40 / 4.0 = 10.0 + 0.5 -> int(10.5) -> 10
|
|
84
|
+
assert_eq!(est.count_text(&"a".repeat(40)), 10);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
#[test]
|
|
88
|
+
fn claude_density_matches_python() {
|
|
89
|
+
let est = EstimatingCounter::new(3.5);
|
|
90
|
+
// 35 / 3.5 = 10.0 -> int(10.5) -> 10
|
|
91
|
+
assert_eq!(est.count_text(&"a".repeat(35)), 10);
|
|
92
|
+
// 36 / 3.5 ≈ 10.286 -> int(10.786) -> 10 (NOT 11 — that was ceil)
|
|
93
|
+
assert_eq!(est.count_text(&"a".repeat(36)), 10);
|
|
94
|
+
// 38 / 3.5 ≈ 10.857 -> int(11.357) -> 11
|
|
95
|
+
assert_eq!(est.count_text(&"a".repeat(38)), 11);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
#[test]
|
|
99
|
+
fn unicode_uses_char_count_not_bytes() {
|
|
100
|
+
let est = EstimatingCounter::default();
|
|
101
|
+
// "héllo" is 5 chars, 6 bytes; 5/4.0 = 1.25 -> int(1.75) -> 1
|
|
102
|
+
assert_eq!(est.count_text("héllo"), 1);
|
|
103
|
+
// 4 emojis = 4 chars; 4/4.0 = 1.0 -> int(1.5) -> 1
|
|
104
|
+
assert_eq!(est.count_text("🦀🦀🦀🦀"), 1);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
#[test]
|
|
108
|
+
fn min_is_one_for_non_empty_input() {
|
|
109
|
+
let est = EstimatingCounter::default();
|
|
110
|
+
// 1/4.0 = 0.25 + 0.5 -> int(0.75) -> 0; max(1, 0) -> 1
|
|
111
|
+
assert_eq!(est.count_text("a"), 1);
|
|
112
|
+
assert_eq!(est.count_text("ab"), 1);
|
|
113
|
+
// 3/4.0 = 0.75 + 0.5 -> int(1.25) -> 1
|
|
114
|
+
assert_eq!(est.count_text("abc"), 1);
|
|
115
|
+
// 6/4.0 = 1.5 + 0.5 -> int(2.0) -> 2
|
|
116
|
+
assert_eq!(est.count_text("aaaaaa"), 2);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
#[test]
|
|
120
|
+
fn deterministic() {
|
|
121
|
+
let est = EstimatingCounter::default();
|
|
122
|
+
let s = "the quick brown fox jumps over the lazy dog";
|
|
123
|
+
let a = est.count_text(s);
|
|
124
|
+
let b = est.count_text(s);
|
|
125
|
+
assert_eq!(a, b);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
#[test]
|
|
129
|
+
fn very_long_input_does_not_overflow() {
|
|
130
|
+
let est = EstimatingCounter::default();
|
|
131
|
+
let s = "a".repeat(1_000_000);
|
|
132
|
+
assert_eq!(est.count_text(&s), 250_000);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
#[test]
|
|
136
|
+
#[should_panic(expected = "chars_per_token must be positive")]
|
|
137
|
+
fn rejects_zero_density() {
|
|
138
|
+
let _ = EstimatingCounter::new(0.0);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
#[test]
|
|
142
|
+
#[should_panic(expected = "chars_per_token must be positive")]
|
|
143
|
+
fn rejects_negative_density() {
|
|
144
|
+
let _ = EstimatingCounter::new(-1.0);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
#[test]
|
|
148
|
+
fn backend_is_estimation() {
|
|
149
|
+
let est = EstimatingCounter::default();
|
|
150
|
+
assert_eq!(est.backend(), Backend::Estimation);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
//! HuggingFace `tokenizers`-crate adapter implementing [`Tokenizer`].
|
|
2
|
+
//!
|
|
3
|
+
//! Loads a `tokenizer.json` (HuggingFace's serialization format) and counts
|
|
4
|
+
//! tokens via real BPE / Unigram / WordPiece — whatever the file describes.
|
|
5
|
+
//! This closes the gap between OpenAI (tiktoken, byte-equal) and the
|
|
6
|
+
//! Anthropic/Google chars-per-token fallback: every other major family
|
|
7
|
+
//! (Cohere `command-*`, Llama-3.x, Mistral, Qwen, BERT, T5, …) publishes a
|
|
8
|
+
//! `tokenizer.json` on the HuggingFace Hub and the `tokenizers` crate is a
|
|
9
|
+
//! pure-Rust loader, so we don't have to estimate.
|
|
10
|
+
//!
|
|
11
|
+
//! # Sources
|
|
12
|
+
//! - [`HfTokenizer::from_bytes`] — for tokenizers embedded via `include_bytes!`.
|
|
13
|
+
//! - [`HfTokenizer::from_file`] — for a local `tokenizer.json`.
|
|
14
|
+
//! - [`HfTokenizer::from_pretrained`] — pulls `tokenizer.json` from the
|
|
15
|
+
//! HuggingFace Hub via `hf-hub`. First call downloads to
|
|
16
|
+
//! `~/.cache/huggingface/hub`, subsequent calls hit the cache. Blocking;
|
|
17
|
+
//! call from `main()` or `tokio::task::spawn_blocking`. Gated repos
|
|
18
|
+
//! (Llama, Mistral) require an `HF_TOKEN` env var or a token in
|
|
19
|
+
//! `~/.cache/huggingface/token`.
|
|
20
|
+
//!
|
|
21
|
+
//! # What's NOT here
|
|
22
|
+
//! - **No tokenizer.json bundled in the binary.** Bundling Llama / Cohere
|
|
23
|
+
//! tokenizers would add several MB of binary bloat for code paths most users
|
|
24
|
+
//! don't hit. `from_pretrained` lazily downloads instead.
|
|
25
|
+
|
|
26
|
+
use std::path::Path;
|
|
27
|
+
use std::sync::Arc;
|
|
28
|
+
|
|
29
|
+
use thiserror::Error;
|
|
30
|
+
use tokenizers::Tokenizer as HfInner;
|
|
31
|
+
|
|
32
|
+
use super::{Backend, Tokenizer};
|
|
33
|
+
|
|
34
|
+
#[derive(Debug, Error)]
|
|
35
|
+
pub enum HfTokenizerError {
|
|
36
|
+
/// The bytes / file did not parse as a valid HuggingFace `tokenizer.json`,
|
|
37
|
+
/// or the model component referenced an unsupported algorithm.
|
|
38
|
+
#[error("failed to load tokenizer for `{name}`: {source}")]
|
|
39
|
+
Load {
|
|
40
|
+
name: String,
|
|
41
|
+
#[source]
|
|
42
|
+
source: Box<dyn std::error::Error + Send + Sync>,
|
|
43
|
+
},
|
|
44
|
+
/// The HuggingFace Hub fetch failed: network error, 404 on the repo, or
|
|
45
|
+
/// 401 on a gated model without an `HF_TOKEN`.
|
|
46
|
+
#[error("failed to download `{repo}` from HuggingFace Hub: {source}")]
|
|
47
|
+
Hub {
|
|
48
|
+
repo: String,
|
|
49
|
+
#[source]
|
|
50
|
+
source: Box<dyn std::error::Error + Send + Sync>,
|
|
51
|
+
},
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/// Token counter backed by a HuggingFace `tokenizer.json`.
|
|
55
|
+
///
|
|
56
|
+
/// Cheap to clone — internally an `Arc<tokenizers::Tokenizer>`. Construct once
|
|
57
|
+
/// at startup, share across handlers.
|
|
58
|
+
#[derive(Clone)]
|
|
59
|
+
pub struct HfTokenizer {
|
|
60
|
+
name: String,
|
|
61
|
+
inner: Arc<HfInner>,
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
impl std::fmt::Debug for HfTokenizer {
|
|
65
|
+
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
66
|
+
f.debug_struct("HfTokenizer")
|
|
67
|
+
.field("name", &self.name)
|
|
68
|
+
.finish()
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
impl HfTokenizer {
|
|
73
|
+
/// Build from raw `tokenizer.json` bytes. Use this when the tokenizer is
|
|
74
|
+
/// embedded via `include_bytes!` or fetched from a non-HF source.
|
|
75
|
+
pub fn from_bytes(name: impl Into<String>, bytes: &[u8]) -> Result<Self, HfTokenizerError> {
|
|
76
|
+
let name = name.into();
|
|
77
|
+
let inner = HfInner::from_bytes(bytes).map_err(|e| HfTokenizerError::Load {
|
|
78
|
+
name: name.clone(),
|
|
79
|
+
source: e,
|
|
80
|
+
})?;
|
|
81
|
+
Ok(Self {
|
|
82
|
+
name,
|
|
83
|
+
inner: Arc::new(inner),
|
|
84
|
+
})
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/// Build from a `tokenizer.json` on disk.
|
|
88
|
+
pub fn from_file(
|
|
89
|
+
name: impl Into<String>,
|
|
90
|
+
path: impl AsRef<Path>,
|
|
91
|
+
) -> Result<Self, HfTokenizerError> {
|
|
92
|
+
let name = name.into();
|
|
93
|
+
let inner = HfInner::from_file(path.as_ref()).map_err(|e| HfTokenizerError::Load {
|
|
94
|
+
name: name.clone(),
|
|
95
|
+
source: e,
|
|
96
|
+
})?;
|
|
97
|
+
Ok(Self {
|
|
98
|
+
name,
|
|
99
|
+
inner: Arc::new(inner),
|
|
100
|
+
})
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/// Download (or fetch from cache) `tokenizer.json` for `repo` from the
|
|
104
|
+
/// HuggingFace Hub and load it. `repo` is the canonical Hub identifier,
|
|
105
|
+
/// e.g. `"CohereForAI/c4ai-command-r-v01"` or `"meta-llama/Meta-Llama-3-8B"`.
|
|
106
|
+
///
|
|
107
|
+
/// Uses the `main` revision. Blocking — calls into `ureq` synchronously.
|
|
108
|
+
/// First successful call writes the file to `~/.cache/huggingface/hub`
|
|
109
|
+
/// (or `$HF_HOME` if set); subsequent calls in the same or later
|
|
110
|
+
/// processes hit the on-disk cache.
|
|
111
|
+
///
|
|
112
|
+
/// Errors:
|
|
113
|
+
/// - [`HfTokenizerError::Hub`] for download failures (no network, 404,
|
|
114
|
+
/// 401 on a gated model without `HF_TOKEN`).
|
|
115
|
+
/// - [`HfTokenizerError::Load`] if the downloaded bytes don't parse as
|
|
116
|
+
/// a valid `tokenizer.json`. Should not happen for healthy HF repos.
|
|
117
|
+
pub fn from_pretrained(repo: &str) -> Result<Self, HfTokenizerError> {
|
|
118
|
+
let api = hf_hub::api::sync::Api::new().map_err(|e| HfTokenizerError::Hub {
|
|
119
|
+
repo: repo.to_string(),
|
|
120
|
+
source: Box::new(e),
|
|
121
|
+
})?;
|
|
122
|
+
let path = api
|
|
123
|
+
.model(repo.to_string())
|
|
124
|
+
.get("tokenizer.json")
|
|
125
|
+
.map_err(|e| HfTokenizerError::Hub {
|
|
126
|
+
repo: repo.to_string(),
|
|
127
|
+
source: Box::new(e),
|
|
128
|
+
})?;
|
|
129
|
+
// `get` returns the on-disk path; reuse `from_file` to keep the load
|
|
130
|
+
// path identical to user-supplied tokenizer.json files.
|
|
131
|
+
Self::from_file(repo, path)
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/// The logical name this tokenizer was registered under (e.g.
|
|
135
|
+
/// `"command-r-plus"`). Used in logs and metrics.
|
|
136
|
+
pub fn name(&self) -> &str {
|
|
137
|
+
&self.name
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
impl Tokenizer for HfTokenizer {
|
|
142
|
+
fn count_text(&self, text: &str) -> usize {
|
|
143
|
+
if text.is_empty() {
|
|
144
|
+
return 0;
|
|
145
|
+
}
|
|
146
|
+
// `add_special_tokens=false` matches the spirit of
|
|
147
|
+
// `tiktoken.encode_ordinary`: count *content* tokens, leaving
|
|
148
|
+
// BOS/EOS/CLS/SEP padding to be added (or not) by the upstream API.
|
|
149
|
+
// Different providers add different specials, so counting them here
|
|
150
|
+
// would systematically over-charge. Documented for future readers.
|
|
151
|
+
match self.inner.encode(text, false) {
|
|
152
|
+
Ok(enc) => enc.len(),
|
|
153
|
+
// `encode` only fails for malformed inputs that pass UTF-8 but
|
|
154
|
+
// violate the tokenizer's constraints (e.g. a normalizer that
|
|
155
|
+
// rejects certain code points). We degrade to "0 known tokens"
|
|
156
|
+
// rather than panic — the proxy must keep flowing requests.
|
|
157
|
+
Err(_) => 0,
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
fn backend(&self) -> Backend {
|
|
162
|
+
Backend::HuggingFace
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
#[cfg(test)]
|
|
167
|
+
mod tests {
|
|
168
|
+
use super::*;
|
|
169
|
+
|
|
170
|
+
/// Minimal valid `tokenizer.json`: WordLevel model, Whitespace
|
|
171
|
+
/// pre-tokenizer, three-token vocabulary. Lets us test the API surface
|
|
172
|
+
/// without committing a multi-MB tokenizer fixture or hitting the
|
|
173
|
+
/// HuggingFace Hub.
|
|
174
|
+
const TINY_TOKENIZER_JSON: &str = r#"{
|
|
175
|
+
"version": "1.0",
|
|
176
|
+
"truncation": null,
|
|
177
|
+
"padding": null,
|
|
178
|
+
"added_tokens": [],
|
|
179
|
+
"normalizer": null,
|
|
180
|
+
"pre_tokenizer": {"type": "Whitespace"},
|
|
181
|
+
"post_processor": null,
|
|
182
|
+
"decoder": null,
|
|
183
|
+
"model": {
|
|
184
|
+
"type": "WordLevel",
|
|
185
|
+
"vocab": {"hello": 0, "world": 1, "[UNK]": 2},
|
|
186
|
+
"unk_token": "[UNK]"
|
|
187
|
+
}
|
|
188
|
+
}"#;
|
|
189
|
+
|
|
190
|
+
fn tiny() -> HfTokenizer {
|
|
191
|
+
HfTokenizer::from_bytes("tiny-test", TINY_TOKENIZER_JSON.as_bytes())
|
|
192
|
+
.expect("tiny tokenizer.json parses")
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
#[test]
|
|
196
|
+
fn empty_is_zero() {
|
|
197
|
+
assert_eq!(tiny().count_text(""), 0);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
#[test]
|
|
201
|
+
fn known_vocab_matches_count() {
|
|
202
|
+
let t = tiny();
|
|
203
|
+
// Each word in the vocab is one token; whitespace splits them.
|
|
204
|
+
assert_eq!(t.count_text("hello"), 1);
|
|
205
|
+
assert_eq!(t.count_text("hello world"), 2);
|
|
206
|
+
assert_eq!(t.count_text("hello world hello"), 3);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
#[test]
|
|
210
|
+
fn unknown_words_become_unk() {
|
|
211
|
+
// OOV tokens collapse to [UNK] — still 1 per whitespace-split chunk.
|
|
212
|
+
let t = tiny();
|
|
213
|
+
assert_eq!(t.count_text("supercalifragilistic"), 1);
|
|
214
|
+
assert_eq!(t.count_text("foo bar baz"), 3);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
#[test]
|
|
218
|
+
fn deterministic() {
|
|
219
|
+
let t = tiny();
|
|
220
|
+
let s = "hello world hello world";
|
|
221
|
+
let first = t.count_text(s);
|
|
222
|
+
for _ in 0..100 {
|
|
223
|
+
assert_eq!(t.count_text(s), first);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
#[test]
|
|
228
|
+
fn unicode_does_not_panic() {
|
|
229
|
+
let t = tiny();
|
|
230
|
+
for s in ["héllo wörld", "你好世界", "🦀 ferris", "\n\t\r"] {
|
|
231
|
+
// We only assert non-panic and a reasonable upper bound; the exact
|
|
232
|
+
// count depends on Whitespace pre-tokenizer behavior, which is
|
|
233
|
+
// tokenizers-crate internal.
|
|
234
|
+
let n = t.count_text(s);
|
|
235
|
+
assert!(n < s.len() * 4 + 10, "absurd count {n} for {s:?}");
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
#[test]
|
|
240
|
+
fn invalid_bytes_returns_error() {
|
|
241
|
+
let r = HfTokenizer::from_bytes("bad", b"not a tokenizer.json");
|
|
242
|
+
assert!(matches!(r, Err(HfTokenizerError::Load { .. })));
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
#[test]
|
|
246
|
+
fn name_round_trips() {
|
|
247
|
+
let t = tiny();
|
|
248
|
+
assert_eq!(t.name(), "tiny-test");
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
#[test]
|
|
252
|
+
fn backend_is_huggingface() {
|
|
253
|
+
assert_eq!(tiny().backend(), Backend::HuggingFace);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
#[test]
|
|
257
|
+
fn clone_shares_inner() {
|
|
258
|
+
let a = tiny();
|
|
259
|
+
let b = a.clone();
|
|
260
|
+
assert!(Arc::ptr_eq(&a.inner, &b.inner));
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
#[test]
|
|
264
|
+
fn from_file_loads_a_real_file() {
|
|
265
|
+
// Round-trip via a temp file to cover the on-disk constructor.
|
|
266
|
+
use std::io::Write;
|
|
267
|
+
let dir = std::env::temp_dir().join(format!(
|
|
268
|
+
"headroom-hf-test-{}",
|
|
269
|
+
std::time::SystemTime::now()
|
|
270
|
+
.duration_since(std::time::UNIX_EPOCH)
|
|
271
|
+
.unwrap()
|
|
272
|
+
.as_nanos()
|
|
273
|
+
));
|
|
274
|
+
std::fs::create_dir_all(&dir).unwrap();
|
|
275
|
+
let path = dir.join("tokenizer.json");
|
|
276
|
+
let mut f = std::fs::File::create(&path).unwrap();
|
|
277
|
+
f.write_all(TINY_TOKENIZER_JSON.as_bytes()).unwrap();
|
|
278
|
+
drop(f);
|
|
279
|
+
|
|
280
|
+
let t = HfTokenizer::from_file("from-file", &path).expect("loads");
|
|
281
|
+
assert_eq!(t.count_text("hello world"), 2);
|
|
282
|
+
|
|
283
|
+
let _ = std::fs::remove_dir_all(&dir);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/// Network-dependent: hits HuggingFace Hub. Run with
|
|
287
|
+
/// `cargo test -p headroom-core -- --ignored from_pretrained_downloads_real_tokenizer`.
|
|
288
|
+
/// `gpt2` is a small public unauthenticated repo (~1.4 MB tokenizer.json).
|
|
289
|
+
#[test]
|
|
290
|
+
#[ignore = "network-dependent: hits HuggingFace Hub"]
|
|
291
|
+
fn from_pretrained_downloads_real_tokenizer() {
|
|
292
|
+
let t = HfTokenizer::from_pretrained("gpt2").expect("download succeeds");
|
|
293
|
+
// GPT-2 BPE: "hello world" is 2 tokens. Locks in that we got a real
|
|
294
|
+
// BPE tokenizer (not a WhitespaceSplit fixture) from HF.
|
|
295
|
+
assert_eq!(t.count_text("hello world"), 2);
|
|
296
|
+
assert_eq!(t.name(), "gpt2");
|
|
297
|
+
assert_eq!(t.backend(), Backend::HuggingFace);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/// Negative path: a malformed repo name fails before any network call
|
|
301
|
+
/// (the Hub URL builder rejects it). No network required to run.
|
|
302
|
+
#[test]
|
|
303
|
+
fn from_pretrained_invalid_repo_returns_hub_error() {
|
|
304
|
+
// Empty repo name — hf-hub rejects this without making a network
|
|
305
|
+
// request. Locks in that we propagate Hub errors as `Hub`, not
|
|
306
|
+
// `Load`, so callers can distinguish "couldn't fetch" from
|
|
307
|
+
// "fetched but malformed".
|
|
308
|
+
let r = HfTokenizer::from_pretrained("");
|
|
309
|
+
assert!(
|
|
310
|
+
matches!(r, Err(HfTokenizerError::Hub { .. })),
|
|
311
|
+
"expected HfTokenizerError::Hub, got {r:?}"
|
|
312
|
+
);
|
|
313
|
+
}
|
|
314
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
//! Token counting for Headroom transforms.
|
|
2
|
+
//!
|
|
3
|
+
//! Mirrors the public surface of the Python `headroom.tokenizers` package:
|
|
4
|
+
//! a `Tokenizer` trait, a tiktoken-backed counter for OpenAI / o-series models
|
|
5
|
+
//! (via the `tiktoken-rs` crate, which uses the same BPE data files as Python's
|
|
6
|
+
//! `tiktoken` and therefore returns byte-identical token IDs), and an estimation
|
|
7
|
+
//! fallback for everything else.
|
|
8
|
+
//!
|
|
9
|
+
//! # Why this exists
|
|
10
|
+
//! Counting tokens currently round-trips into Python's `tiktoken` (itself a
|
|
11
|
+
//! Rust extension under the hood). For Rust transforms running on the proxy
|
|
12
|
+
//! hot path, counting natively avoids the Python-Rust FFI cost and keeps the
|
|
13
|
+
//! Rust binary self-contained.
|
|
14
|
+
//!
|
|
15
|
+
//! # What this is NOT
|
|
16
|
+
//! - Not used by `headroom-proxy` yet (Stage 2 is library-only; no production
|
|
17
|
+
//! wiring).
|
|
18
|
+
//! - Not a real Anthropic Claude tokenizer (Anthropic doesn't publish theirs;
|
|
19
|
+
//! estimation matches what the Python implementation does).
|
|
20
|
+
//! - SentencePiece — Gemini's tokenizer is SP-based but Google doesn't
|
|
21
|
+
//! publish the model. Falls through to estimation.
|
|
22
|
+
//!
|
|
23
|
+
//! # What is here
|
|
24
|
+
//! - [`TiktokenCounter`] — OpenAI / o-series, byte-equal to Python `tiktoken`.
|
|
25
|
+
//! - [`HfTokenizer`] — any model with a public `tokenizer.json` (Cohere
|
|
26
|
+
//! `command-*`, Llama-3.x, Mistral, Qwen, BERT, T5, …). Construct it from
|
|
27
|
+
//! bytes or a file path; register it via [`register_hf`] to make it the
|
|
28
|
+
//! default for a given model-name prefix.
|
|
29
|
+
//! - [`EstimatingCounter`] — last-resort `chars / cpt` fallback for Anthropic
|
|
30
|
+
//! and Gemini, calibrated to match the Python implementation.
|
|
31
|
+
|
|
32
|
+
mod estimator;
|
|
33
|
+
mod hf_impl;
|
|
34
|
+
mod registry;
|
|
35
|
+
mod tiktoken_impl;
|
|
36
|
+
|
|
37
|
+
pub use estimator::EstimatingCounter;
|
|
38
|
+
pub use hf_impl::{HfTokenizer, HfTokenizerError};
|
|
39
|
+
pub use registry::{
|
|
40
|
+
clear_hf_registrations, detect_backend, get_tokenizer, register_hf, try_register_hf, Backend,
|
|
41
|
+
};
|
|
42
|
+
pub use tiktoken_impl::{TiktokenCounter, TiktokenError};
|
|
43
|
+
|
|
44
|
+
/// Counts tokens. Implementations must be thread-safe (`Send + Sync`).
|
|
45
|
+
///
|
|
46
|
+
/// # Conventions (preserved across all built-in implementations)
|
|
47
|
+
/// - `count_text("")` returns `0`.
|
|
48
|
+
/// - Counts are deterministic for a given input and instance.
|
|
49
|
+
/// - For non-empty input, counts are `>= 1`.
|
|
50
|
+
pub trait Tokenizer: Send + Sync + std::fmt::Debug {
|
|
51
|
+
/// Number of tokens that this tokenizer assigns to `text`.
|
|
52
|
+
fn count_text(&self, text: &str) -> usize;
|
|
53
|
+
|
|
54
|
+
/// Which backend produced the count. Useful for logs and metrics.
|
|
55
|
+
fn backend(&self) -> Backend;
|
|
56
|
+
}
|