xithead 1.2.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.actrc +14 -0
- package/.actrc.local.example +15 -0
- package/.changelog.md +2 -0
- package/.claude/agents/academic-anthropologist.md +125 -0
- package/.claude/agents/academic-geographer.md +127 -0
- package/.claude/agents/academic-historian.md +123 -0
- package/.claude/agents/academic-narratologist.md +118 -0
- package/.claude/agents/academic-psychologist.md +118 -0
- package/.claude/agents/academic-statistician.md +144 -0
- package/.claude/agents/accounts-payable-agent.md +205 -0
- package/.claude/agents/agentic-identity-trust.md +394 -0
- package/.claude/agents/agents-orchestrator.md +367 -0
- package/.claude/agents/automation-governance-architect.md +216 -0
- package/.claude/agents/blender-addon-engineer.md +243 -0
- package/.claude/agents/business-strategist.md +488 -0
- package/.claude/agents/change-management-consultant.md +497 -0
- package/.claude/agents/chief-financial-officer.md +388 -0
- package/.claude/agents/corporate-training-designer.md +192 -0
- package/.claude/agents/customer-service.md +398 -0
- package/.claude/agents/customer-success-manager.md +460 -0
- package/.claude/agents/data-consolidation-agent.md +60 -0
- package/.claude/agents/data-privacy-officer.md +412 -0
- package/.claude/agents/design-brand-guardian.md +322 -0
- package/.claude/agents/design-image-prompt-engineer.md +236 -0
- package/.claude/agents/design-inclusive-visuals-specialist.md +73 -0
- package/.claude/agents/design-persona-walkthrough.md +272 -0
- package/.claude/agents/design-ui-designer.md +386 -0
- package/.claude/agents/design-ui-finish-gate-reviewer.md +217 -0
- package/.claude/agents/design-ux-architect.md +482 -0
- package/.claude/agents/design-ux-researcher.md +329 -0
- package/.claude/agents/design-visual-storyteller.md +149 -0
- package/.claude/agents/design-whimsy-injector.md +460 -0
- package/.claude/agents/economy-designer.md +156 -0
- package/.claude/agents/engineering-ai-data-remediation-engineer.md +211 -0
- package/.claude/agents/engineering-ai-engineer.md +146 -0
- package/.claude/agents/engineering-api-platform-engineer.md +181 -0
- package/.claude/agents/engineering-ats-validator-architect.md +383 -0
- package/.claude/agents/engineering-autonomous-optimization-architect.md +121 -0
- package/.claude/agents/engineering-backend-architect.md +255 -0
- package/.claude/agents/engineering-china-network-engineer.md +252 -0
- package/.claude/agents/engineering-cms-developer.md +536 -0
- package/.claude/agents/engineering-code-reviewer.md +76 -0
- package/.claude/agents/engineering-codebase-onboarding-engineer.md +173 -0
- package/.claude/agents/engineering-data-engineer.md +349 -0
- package/.claude/agents/engineering-data-visualization-engineer.md +151 -0
- package/.claude/agents/engineering-database-optimizer.md +188 -0
- package/.claude/agents/engineering-database-reliability-engineer.md +201 -0
- package/.claude/agents/engineering-desktop-app-engineer.md +209 -0
- package/.claude/agents/engineering-developer-tooling-engineer.md +153 -0
- package/.claude/agents/engineering-devops-automator.md +385 -0
- package/.claude/agents/engineering-drupal-performance.md +347 -0
- package/.claude/agents/engineering-drupal-shopping-cart.md +360 -0
- package/.claude/agents/engineering-electrical-engineer.md +575 -0
- package/.claude/agents/engineering-email-intelligence-engineer.md +390 -0
- package/.claude/agents/engineering-embedded-firmware-engineer.md +196 -0
- package/.claude/agents/engineering-feishu-integration-developer.md +606 -0
- package/.claude/agents/engineering-filament-optimization-specialist.md +283 -0
- package/.claude/agents/engineering-finops-engineer.md +164 -0
- package/.claude/agents/engineering-frontend-developer.md +257 -0
- package/.claude/agents/engineering-gaussdb-expert.md +334 -0
- package/.claude/agents/engineering-git-workflow-master.md +98 -0
- package/.claude/agents/engineering-i18n-engineer.md +214 -0
- package/.claude/agents/engineering-identity-access-engineer.md +217 -0
- package/.claude/agents/engineering-incident-response-commander.md +444 -0
- package/.claude/agents/engineering-iot-fleet-engineer.md +148 -0
- package/.claude/agents/engineering-it-service-manager.md +561 -0
- package/.claude/agents/engineering-knowledge-graph-engineer.md +372 -0
- package/.claude/agents/engineering-llm-post-training-engineer.md +166 -0
- package/.claude/agents/engineering-minimal-change-engineer.md +207 -0
- package/.claude/agents/engineering-mobile-app-builder.md +493 -0
- package/.claude/agents/engineering-mobile-release-engineer.md +164 -0
- package/.claude/agents/engineering-multi-agent-systems-architect.md +600 -0
- package/.claude/agents/engineering-network-engineer.md +245 -0
- package/.claude/agents/engineering-orgscript-engineer.md +113 -0
- package/.claude/agents/engineering-payments-billing-engineer.md +228 -0
- package/.claude/agents/engineering-pdf-engine-architect.md +696 -0
- package/.claude/agents/engineering-platform-engineer.md +270 -0
- package/.claude/agents/engineering-privacy-engineer.md +152 -0
- package/.claude/agents/engineering-prompt-engineer.md +209 -0
- package/.claude/agents/engineering-rag-pipeline-engineer.md +450 -0
- package/.claude/agents/engineering-rapid-prototyper.md +466 -0
- package/.claude/agents/engineering-realtime-collaboration-engineer.md +200 -0
- package/.claude/agents/engineering-rust-refactoring-specialist.md +313 -0
- package/.claude/agents/engineering-search-relevance-engineer.md +237 -0
- package/.claude/agents/engineering-section-508-specialist.md +339 -0
- package/.claude/agents/engineering-senior-developer.md +176 -0
- package/.claude/agents/engineering-servicenow-developer-mentor.md +180 -0
- package/.claude/agents/engineering-software-architect.md +112 -0
- package/.claude/agents/engineering-solidity-smart-contract-engineer.md +522 -0
- package/.claude/agents/engineering-sre.md +90 -0
- package/.claude/agents/engineering-technical-writer.md +393 -0
- package/.claude/agents/engineering-universal-document-compiler.md +370 -0
- package/.claude/agents/engineering-uswds-developer.md +340 -0
- package/.claude/agents/engineering-video-streaming-engineer.md +151 -0
- package/.claude/agents/engineering-voice-ai-integration-engineer.md +584 -0
- package/.claude/agents/engineering-webassembly-engineer.md +153 -0
- package/.claude/agents/engineering-wechat-mini-program-developer.md +356 -0
- package/.claude/agents/engineering-wordpress-performance.md +346 -0
- package/.claude/agents/engineering-wordpress-shopping-cart.md +346 -0
- package/.claude/agents/esg-sustainability-officer.md +396 -0
- package/.claude/agents/finance-bookkeeper-controller.md +260 -0
- package/.claude/agents/finance-financial-analyst.md +234 -0
- package/.claude/agents/finance-fpa-analyst.md +263 -0
- package/.claude/agents/finance-investment-researcher.md +272 -0
- package/.claude/agents/finance-tax-strategist.md +239 -0
- package/.claude/agents/game-audio-engineer.md +264 -0
- package/.claude/agents/game-designer.md +167 -0
- package/.claude/agents/gis-3d-scene-developer.md +111 -0
- package/.claude/agents/gis-analyst.md +91 -0
- package/.claude/agents/gis-bim-specialist.md +108 -0
- package/.claude/agents/gis-cartography-designer.md +150 -0
- package/.claude/agents/gis-drone-reality-mapping.md +120 -0
- package/.claude/agents/gis-geoai-ml-engineer.md +105 -0
- package/.claude/agents/gis-geoprocessing-specialist.md +97 -0
- package/.claude/agents/gis-qa-engineer.md +133 -0
- package/.claude/agents/gis-solution-engineer.md +101 -0
- package/.claude/agents/gis-spatial-data-engineer.md +97 -0
- package/.claude/agents/gis-spatial-data-scientist.md +111 -0
- package/.claude/agents/gis-technical-consultant.md +86 -0
- package/.claude/agents/gis-web-gis-developer.md +108 -0
- package/.claude/agents/godot-gameplay-scripter.md +338 -0
- package/.claude/agents/godot-multiplayer-engineer.md +301 -0
- package/.claude/agents/godot-shader-developer.md +267 -0
- package/.claude/agents/government-digital-presales-consultant.md +363 -0
- package/.claude/agents/grant-writer.md +511 -0
- package/.claude/agents/healthcare-aging-parent-care-companion.md +414 -0
- package/.claude/agents/healthcare-clinical-evidence-agent.md +231 -0
- package/.claude/agents/healthcare-customer-service.md +389 -0
- package/.claude/agents/healthcare-innovation-strategist.md +433 -0
- package/.claude/agents/healthcare-marketing-compliance.md +395 -0
- package/.claude/agents/healthcare-sovereign-health-systems-agent.md +312 -0
- package/.claude/agents/hospitality-guest-services.md +603 -0
- package/.claude/agents/hr-onboarding.md +451 -0
- package/.claude/agents/identity-graph-operator.md +303 -0
- package/.claude/agents/language-translator.md +264 -0
- package/.claude/agents/legal-billing-time-tracking.md +569 -0
- package/.claude/agents/legal-client-intake.md +492 -0
- package/.claude/agents/legal-document-review.md +454 -0
- package/.claude/agents/level-designer.md +208 -0
- package/.claude/agents/loan-officer-assistant.md +555 -0
- package/.claude/agents/lsp-index-engineer.md +320 -0
- package/.claude/agents/ma-integration-manager.md +427 -0
- package/.claude/agents/macos-spatial-metal-engineer.md +337 -0
- package/.claude/agents/marketing-aeo-foundations.md +264 -0
- package/.claude/agents/marketing-agentic-search-optimizer.md +279 -0
- package/.claude/agents/marketing-ai-citation-strategist.md +172 -0
- package/.claude/agents/marketing-app-store-optimizer.md +321 -0
- package/.claude/agents/marketing-baidu-seo-specialist.md +226 -0
- package/.claude/agents/marketing-bilibili-content-strategist.md +199 -0
- package/.claude/agents/marketing-book-co-author.md +110 -0
- package/.claude/agents/marketing-carousel-growth-engine.md +204 -0
- package/.claude/agents/marketing-china-ecommerce-operator.md +283 -0
- package/.claude/agents/marketing-china-market-localization-strategist.md +283 -0
- package/.claude/agents/marketing-content-creator.md +54 -0
- package/.claude/agents/marketing-cross-border-ecommerce.md +259 -0
- package/.claude/agents/marketing-developer-community-builder.md +285 -0
- package/.claude/agents/marketing-douyin-strategist.md +149 -0
- package/.claude/agents/marketing-email-strategist.md +249 -0
- package/.claude/agents/marketing-global-podcast-strategist.md +206 -0
- package/.claude/agents/marketing-growth-hacker.md +54 -0
- package/.claude/agents/marketing-instagram-curator.md +113 -0
- package/.claude/agents/marketing-kuaishou-strategist.md +223 -0
- package/.claude/agents/marketing-linkedin-content-creator.md +214 -0
- package/.claude/agents/marketing-livestream-commerce-coach.md +305 -0
- package/.claude/agents/marketing-multi-platform-publisher.md +217 -0
- package/.claude/agents/marketing-podcast-strategist.md +277 -0
- package/.claude/agents/marketing-pr-communications-manager.md +473 -0
- package/.claude/agents/marketing-private-domain-operator.md +308 -0
- package/.claude/agents/marketing-reddit-community-builder.md +123 -0
- package/.claude/agents/marketing-seo-specialist.md +370 -0
- package/.claude/agents/marketing-short-video-editing-coach.md +412 -0
- package/.claude/agents/marketing-social-media-strategist.md +125 -0
- package/.claude/agents/marketing-tiktok-strategist.md +125 -0
- package/.claude/agents/marketing-twitter-engager.md +126 -0
- package/.claude/agents/marketing-video-optimization-specialist.md +119 -0
- package/.claude/agents/marketing-wechat-official-account.md +145 -0
- package/.claude/agents/marketing-weibo-strategist.md +240 -0
- package/.claude/agents/marketing-x-twitter-intelligence-analyst.md +161 -0
- package/.claude/agents/marketing-xiaohongshu-specialist.md +138 -0
- package/.claude/agents/marketing-zhihu-strategist.md +162 -0
- package/.claude/agents/medical-billing-coding-specialist.md +491 -0
- package/.claude/agents/narrative-designer.md +243 -0
- package/.claude/agents/operations-manager.md +399 -0
- package/.claude/agents/organizational-psychologist.md +391 -0
- package/.claude/agents/paid-media-auditor.md +71 -0
- package/.claude/agents/paid-media-creative-strategist.md +71 -0
- package/.claude/agents/paid-media-paid-social-strategist.md +71 -0
- package/.claude/agents/paid-media-ppc-strategist.md +71 -0
- package/.claude/agents/paid-media-programmatic-buyer.md +71 -0
- package/.claude/agents/paid-media-search-query-analyst.md +71 -0
- package/.claude/agents/paid-media-tracking-specialist.md +71 -0
- package/.claude/agents/personal-growth-mentor.md +159 -0
- package/.claude/agents/product-behavioral-nudge-engine.md +83 -0
- package/.claude/agents/product-dx-engineer.md +332 -0
- package/.claude/agents/product-feedback-synthesizer.md +119 -0
- package/.claude/agents/product-manager.md +469 -0
- package/.claude/agents/product-sprint-prioritizer.md +154 -0
- package/.claude/agents/product-trend-researcher.md +159 -0
- package/.claude/agents/project-management-experiment-tracker.md +198 -0
- package/.claude/agents/project-management-jira-workflow-steward.md +233 -0
- package/.claude/agents/project-management-meeting-notes-specialist.md +95 -0
- package/.claude/agents/project-management-project-shepherd.md +194 -0
- package/.claude/agents/project-management-studio-operations.md +200 -0
- package/.claude/agents/project-management-studio-producer.md +203 -0
- package/.claude/agents/project-manager-senior.md +135 -0
- package/.claude/agents/real-estate-buyer-seller.md +596 -0
- package/.claude/agents/recruitment-specialist.md +515 -0
- package/.claude/agents/report-distribution-agent.md +65 -0
- package/.claude/agents/research-synthesist.md +136 -0
- package/.claude/agents/resume-tailor.md +230 -0
- package/.claude/agents/retail-customer-returns.md +566 -0
- package/.claude/agents/roblox-avatar-creator.md +297 -0
- package/.claude/agents/roblox-experience-designer.md +306 -0
- package/.claude/agents/roblox-systems-scripter.md +332 -0
- package/.claude/agents/sales-account-strategist.md +227 -0
- package/.claude/agents/sales-coach.md +271 -0
- package/.claude/agents/sales-data-extraction-agent.md +67 -0
- package/.claude/agents/sales-deal-strategist.md +180 -0
- package/.claude/agents/sales-discovery-coach.md +225 -0
- package/.claude/agents/sales-engineer.md +182 -0
- package/.claude/agents/sales-offer-lead-gen-strategist.md +257 -0
- package/.claude/agents/sales-outbound-strategist.md +201 -0
- package/.claude/agents/sales-outreach.md +425 -0
- package/.claude/agents/sales-pipeline-analyst.md +267 -0
- package/.claude/agents/sales-proposal-strategist.md +217 -0
- package/.claude/agents/security-ai-generated-code-auditor.md +207 -0
- package/.claude/agents/security-appsec-engineer.md +536 -0
- package/.claude/agents/security-architect.md +307 -0
- package/.claude/agents/security-blockchain-security-auditor.md +482 -0
- package/.claude/agents/security-cloud-security-architect.md +576 -0
- package/.claude/agents/security-compliance-auditor.md +158 -0
- package/.claude/agents/security-incident-responder.md +450 -0
- package/.claude/agents/security-penetration-tester.md +399 -0
- package/.claude/agents/security-secrets-credential-engineer.md +189 -0
- package/.claude/agents/security-senior-secops.md +760 -0
- package/.claude/agents/security-threat-detection-engineer.md +553 -0
- package/.claude/agents/security-threat-intelligence-analyst.md +661 -0
- package/.claude/agents/specialized-chief-of-staff.md +279 -0
- package/.claude/agents/specialized-civil-engineer.md +356 -0
- package/.claude/agents/specialized-codebase-archaeologist.md +341 -0
- package/.claude/agents/specialized-cultural-intelligence-strategist.md +88 -0
- package/.claude/agents/specialized-developer-advocate.md +317 -0
- package/.claude/agents/specialized-document-generator.md +55 -0
- package/.claude/agents/specialized-fedramp-rmf-compliance.md +378 -0
- package/.claude/agents/specialized-focus-music-architect.md +175 -0
- package/.claude/agents/specialized-french-consulting-market.md +194 -0
- package/.claude/agents/specialized-korean-business-navigator.md +216 -0
- package/.claude/agents/specialized-master-plan-architect.md +157 -0
- package/.claude/agents/specialized-mcp-builder.md +260 -0
- package/.claude/agents/specialized-model-qa.md +528 -0
- package/.claude/agents/specialized-pricing-analyst.md +243 -0
- package/.claude/agents/specialized-salesforce-architect.md +182 -0
- package/.claude/agents/specialized-strategy-duel-agent.md +130 -0
- package/.claude/agents/specialized-workflow-architect.md +597 -0
- package/.claude/agents/study-abroad-advisor.md +282 -0
- package/.claude/agents/supply-chain-strategist.md +591 -0
- package/.claude/agents/support-analytics-reporter.md +387 -0
- package/.claude/agents/support-executive-summary-generator.md +212 -0
- package/.claude/agents/support-finance-tracker.md +480 -0
- package/.claude/agents/support-infrastructure-maintainer.md +628 -0
- package/.claude/agents/support-legal-compliance-checker.md +591 -0
- package/.claude/agents/support-support-responder.md +607 -0
- package/.claude/agents/technical-artist.md +233 -0
- package/.claude/agents/terminal-integration-specialist.md +70 -0
- package/.claude/agents/testing-accessibility-auditor.md +327 -0
- package/.claude/agents/testing-api-tester.md +334 -0
- package/.claude/agents/testing-evidence-collector.md +210 -0
- package/.claude/agents/testing-performance-benchmarker.md +283 -0
- package/.claude/agents/testing-reality-checker.md +249 -0
- package/.claude/agents/testing-test-automation-engineer.md +196 -0
- package/.claude/agents/testing-test-results-analyzer.md +330 -0
- package/.claude/agents/testing-tool-evaluator.md +416 -0
- package/.claude/agents/testing-workflow-optimizer.md +463 -0
- package/.claude/agents/unity-architect.md +271 -0
- package/.claude/agents/unity-editor-tool-developer.md +310 -0
- package/.claude/agents/unity-multiplayer-engineer.md +323 -0
- package/.claude/agents/unity-shader-graph-artist.md +269 -0
- package/.claude/agents/unreal-multiplayer-architect.md +364 -0
- package/.claude/agents/unreal-systems-engineer.md +322 -0
- package/.claude/agents/unreal-technical-artist.md +256 -0
- package/.claude/agents/unreal-world-builder.md +273 -0
- package/.claude/agents/visionos-spatial-engineer.md +54 -0
- package/.claude/agents/xr-cockpit-interaction-specialist.md +32 -0
- package/.claude/agents/xr-immersive-developer.md +32 -0
- package/.claude/agents/xr-interface-architect.md +32 -0
- package/.claude/agents/zk-steward.md +211 -0
- package/.claude-plugin/marketplace.json +30 -0
- package/.commitlintrc.json +26 -0
- package/.devcontainer/Dockerfile +24 -0
- package/.devcontainer/devcontainer.json +70 -0
- package/.devcontainer/docker-compose.memory.yml +42 -0
- package/.devcontainer/memory-stack/devcontainer.json +84 -0
- package/.devcontainer/post-create.sh +83 -0
- package/.dockerignore +23 -0
- package/.env.act.example +15 -0
- package/.env.example +171 -0
- package/.eslintrc.json +16 -0
- package/.git-blame-ignore-revs +10 -0
- package/.gitattributes +2 -0
- package/.gitguardian.yaml +50 -0
- package/.githooks/pre-commit +6 -0
- package/.pre-commit-config.yaml +30 -0
- package/.release-please-config-xithead.json +22 -0
- package/.release-please-config.json +44 -0
- package/.release-please-manifest-xithead.json +3 -0
- package/.release-please-manifest.json +3 -0
- package/CODE_OF_CONDUCT.md +133 -0
- package/CONTRIBUTING.md +133 -0
- package/Cargo.lock +5557 -0
- package/Cargo.toml +121 -0
- package/Dockerfile +38 -0
- package/Dockerfile.proxy +48 -0
- package/ENTERPRISE.md +5 -0
- package/FEATURES_ADVANCED.md +311 -0
- package/Headroom-2.gif +0 -0
- package/HeadroomDemo-Fast.gif +0 -0
- package/MANIFEST.in +4 -0
- package/MODERNIZATION_GUIDE.md +358 -0
- package/Makefile +140 -0
- package/NOTICE +43 -0
- package/NPM_TOKEN_SETUP.md +81 -0
- package/PR.md +170 -0
- package/PUBLISH.md +77 -0
- package/README.md +58 -137
- package/README_FRONTEND.md +196 -0
- package/REALIGNMENT/00-overview.md +53 -0
- package/REALIGNMENT/01-bug-list.md +421 -0
- package/REALIGNMENT/02-architecture.md +322 -0
- package/REALIGNMENT/03-phase-A-lockdown.md +420 -0
- package/REALIGNMENT/04-phase-B-live-zone.md +470 -0
- package/REALIGNMENT/05-phase-C-rust-proxy.md +329 -0
- package/REALIGNMENT/06-phase-D-bedrock-vertex.md +222 -0
- package/REALIGNMENT/07-phase-E-cache-stabilization.md +338 -0
- package/REALIGNMENT/08-phase-F-auth-mode.md +254 -0
- package/REALIGNMENT/09-phase-G-rtk-observability.md +194 -0
- package/REALIGNMENT/10-phase-H-python-retirement.md +214 -0
- package/REALIGNMENT/11-phase-I-test-infra.md +385 -0
- package/REALIGNMENT/12-decisions-needed.md +196 -0
- package/REALIGNMENT/INDEX.md +82 -0
- package/REFACTORING_CAMPAIGN.md +277 -0
- package/RESTRUCTURE_PLAN.md +100 -0
- package/RUST_DEV.md +361 -0
- package/SECURITY.md +65 -0
- package/SETUP_AUTO_PUBLISH.md +164 -0
- package/TESTING-copilot-subscription.md +153 -0
- package/app/api/config/route.ts +22 -0
- package/app/api/export/route.ts +83 -0
- package/app/api/models/route.ts +12 -0
- package/app/api/stats/route.ts +46 -0
- package/app/dashboard/page.tsx +445 -0
- package/app/globals.css +35 -0
- package/app/installation/page.tsx +293 -0
- package/app/layout.tsx +35 -0
- package/app/master-design-system.css +2076 -0
- package/app/master-design-system.source.css +8 -0
- package/app/page.tsx +187 -0
- package/app/rechtliches/agb/page.tsx +165 -0
- package/app/rechtliches/datenschutz/page.tsx +128 -0
- package/app/rechtliches/impressum/page.tsx +122 -0
- package/app/rechtliches/widerruf/page.tsx +185 -0
- package/bin/xithead.js +264 -0
- package/codecov.yml +34 -0
- package/components/layout/AppShell.tsx +25 -0
- package/components/layout/Footer.tsx +120 -0
- package/components/layout/Header.tsx +145 -0
- package/components/marketing/LegalHighlights.tsx +45 -0
- package/components/marketing/LegalPageShell.tsx +49 -0
- package/components/marketing/LegalPanel.tsx +41 -0
- package/components/marketing/index.ts +3 -0
- package/crates/headroom-core/Cargo.toml +165 -0
- package/crates/headroom-core/benches/auth_mode.rs +74 -0
- package/crates/headroom-core/benches/ccr_store.rs +224 -0
- package/crates/headroom-core/benches/tokenizer.rs +52 -0
- package/crates/headroom-core/config/pipeline.toml +151 -0
- package/crates/headroom-core/proptest-regressions/transforms/tag_protector.txt +9 -0
- package/crates/headroom-core/src/auth_mode.rs +250 -0
- package/crates/headroom-core/src/cache_control.rs +414 -0
- package/crates/headroom-core/src/ccr/backends/in_memory.rs +317 -0
- package/crates/headroom-core/src/ccr/backends/mod.rs +152 -0
- package/crates/headroom-core/src/ccr/backends/redis.rs +146 -0
- package/crates/headroom-core/src/ccr/backends/sqlite.rs +205 -0
- package/crates/headroom-core/src/ccr/mod.rs +119 -0
- package/crates/headroom-core/src/compression_policy.rs +505 -0
- package/crates/headroom-core/src/lib.rs +33 -0
- package/crates/headroom-core/src/relevance/base.rs +139 -0
- package/crates/headroom-core/src/relevance/bm25.rs +385 -0
- package/crates/headroom-core/src/relevance/embedding.rs +354 -0
- package/crates/headroom-core/src/relevance/hybrid.rs +364 -0
- package/crates/headroom-core/src/relevance/mod.rs +64 -0
- package/crates/headroom-core/src/signals/keyword_detector.rs +433 -0
- package/crates/headroom-core/src/signals/line_importance.rs +84 -0
- package/crates/headroom-core/src/signals/mod.rs +61 -0
- package/crates/headroom-core/src/signals/tiered.rs +141 -0
- package/crates/headroom-core/src/tokenizer/estimator.rs +152 -0
- package/crates/headroom-core/src/tokenizer/hf_impl.rs +314 -0
- package/crates/headroom-core/src/tokenizer/mod.rs +56 -0
- package/crates/headroom-core/src/tokenizer/registry.rs +342 -0
- package/crates/headroom-core/src/tokenizer/tiktoken_impl.rs +265 -0
- package/crates/headroom-core/src/transforms/adaptive_sizer.rs +610 -0
- package/crates/headroom-core/src/transforms/anchor_selector.rs +1189 -0
- package/crates/headroom-core/src/transforms/content_detector.rs +769 -0
- package/crates/headroom-core/src/transforms/detection.rs +222 -0
- package/crates/headroom-core/src/transforms/diff_compressor.rs +1685 -0
- package/crates/headroom-core/src/transforms/live_zone.rs +2967 -0
- package/crates/headroom-core/src/transforms/log_compressor.rs +1295 -0
- package/crates/headroom-core/src/transforms/magika_detector.rs +395 -0
- package/crates/headroom-core/src/transforms/mod.rs +64 -0
- package/crates/headroom-core/src/transforms/pipeline/config.rs +331 -0
- package/crates/headroom-core/src/transforms/pipeline/mod.rs +103 -0
- package/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rs +524 -0
- package/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs +279 -0
- package/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs +400 -0
- package/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rs +284 -0
- package/crates/headroom-core/src/transforms/pipeline/offloads/mod.rs +34 -0
- package/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rs +331 -0
- package/crates/headroom-core/src/transforms/pipeline/orchestrator.rs +848 -0
- package/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rs +175 -0
- package/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rs +545 -0
- package/crates/headroom-core/src/transforms/pipeline/reformats/mod.rs +12 -0
- package/crates/headroom-core/src/transforms/pipeline/traits.rs +338 -0
- package/crates/headroom-core/src/transforms/recommendations.rs +329 -0
- package/crates/headroom-core/src/transforms/safety.rs +215 -0
- package/crates/headroom-core/src/transforms/search_compressor.rs +902 -0
- package/crates/headroom-core/src/transforms/smart_crusher/analyzer.rs +1262 -0
- package/crates/headroom-core/src/transforms/smart_crusher/anchors.rs +407 -0
- package/crates/headroom-core/src/transforms/smart_crusher/builder.rs +263 -0
- package/crates/headroom-core/src/transforms/smart_crusher/classifier.rs +214 -0
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/classifier.rs +283 -0
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/compactor.rs +733 -0
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rs +961 -0
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/ir.rs +252 -0
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs +127 -0
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs +397 -0
- package/crates/headroom-core/src/transforms/smart_crusher/config.rs +173 -0
- package/crates/headroom-core/src/transforms/smart_crusher/constraints.rs +159 -0
- package/crates/headroom-core/src/transforms/smart_crusher/crusher.rs +1665 -0
- package/crates/headroom-core/src/transforms/smart_crusher/crushers.rs +865 -0
- package/crates/headroom-core/src/transforms/smart_crusher/error_keywords.rs +77 -0
- package/crates/headroom-core/src/transforms/smart_crusher/field_detect.rs +393 -0
- package/crates/headroom-core/src/transforms/smart_crusher/hashing.rs +72 -0
- package/crates/headroom-core/src/transforms/smart_crusher/mod.rs +81 -0
- package/crates/headroom-core/src/transforms/smart_crusher/observer.rs +66 -0
- package/crates/headroom-core/src/transforms/smart_crusher/orchestration.rs +459 -0
- package/crates/headroom-core/src/transforms/smart_crusher/outliers.rs +507 -0
- package/crates/headroom-core/src/transforms/smart_crusher/planning.rs +904 -0
- package/crates/headroom-core/src/transforms/smart_crusher/statistics.rs +510 -0
- package/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs +287 -0
- package/crates/headroom-core/src/transforms/smart_crusher/traits.rs +203 -0
- package/crates/headroom-core/src/transforms/smart_crusher/types.rs +263 -0
- package/crates/headroom-core/src/transforms/tag_protector.rs +1272 -0
- package/crates/headroom-core/src/transforms/unidiff_detector.rs +232 -0
- package/crates/headroom-parity/Cargo.toml +26 -0
- package/crates/headroom-parity/examples/diff_fixture.rs +66 -0
- package/crates/headroom-parity/src/bin/parity_run.rs +78 -0
- package/crates/headroom-parity/src/lib.rs +613 -0
- package/crates/headroom-proxy/Cargo.toml +119 -0
- package/crates/headroom-proxy/data/model_prices_and_context_window.json +39767 -0
- package/crates/headroom-proxy/src/bedrock/auth_mode_layer.rs +192 -0
- package/crates/headroom-proxy/src/bedrock/envelope.rs +239 -0
- package/crates/headroom-proxy/src/bedrock/eventstream.rs +769 -0
- package/crates/headroom-proxy/src/bedrock/eventstream_to_sse.rs +459 -0
- package/crates/headroom-proxy/src/bedrock/invoke.rs +735 -0
- package/crates/headroom-proxy/src/bedrock/invoke_streaming.rs +1098 -0
- package/crates/headroom-proxy/src/bedrock/mod.rs +66 -0
- package/crates/headroom-proxy/src/bedrock/sigv4.rs +324 -0
- package/crates/headroom-proxy/src/bedrock/vendor.rs +82 -0
- package/crates/headroom-proxy/src/cache_stabilization/anthropic_cache_control.rs +698 -0
- package/crates/headroom-proxy/src/cache_stabilization/drift_detector.rs +683 -0
- package/crates/headroom-proxy/src/cache_stabilization/mod.rs +61 -0
- package/crates/headroom-proxy/src/cache_stabilization/openai_cache_key.rs +600 -0
- package/crates/headroom-proxy/src/cache_stabilization/tool_def_normalize.rs +514 -0
- package/crates/headroom-proxy/src/cache_stabilization/volatile_detector.rs +720 -0
- package/crates/headroom-proxy/src/compression/anthropic.rs +92 -0
- package/crates/headroom-proxy/src/compression/live_zone_anthropic.rs +1311 -0
- package/crates/headroom-proxy/src/compression/live_zone_openai.rs +660 -0
- package/crates/headroom-proxy/src/compression/live_zone_responses.rs +678 -0
- package/crates/headroom-proxy/src/compression/mod.rs +127 -0
- package/crates/headroom-proxy/src/compression/model_limits.rs +211 -0
- package/crates/headroom-proxy/src/config.rs +646 -0
- package/crates/headroom-proxy/src/error.rs +69 -0
- package/crates/headroom-proxy/src/handlers/chat_completions.rs +100 -0
- package/crates/headroom-proxy/src/handlers/conversations.rs +241 -0
- package/crates/headroom-proxy/src/handlers/mod.rs +13 -0
- package/crates/headroom-proxy/src/handlers/responses.rs +256 -0
- package/crates/headroom-proxy/src/headers.rs +288 -0
- package/crates/headroom-proxy/src/health.rs +41 -0
- package/crates/headroom-proxy/src/lib.rs +21 -0
- package/crates/headroom-proxy/src/main.rs +141 -0
- package/crates/headroom-proxy/src/observability/cache_hit_rate.rs +340 -0
- package/crates/headroom-proxy/src/observability/compression_ratio.rs +158 -0
- package/crates/headroom-proxy/src/observability/metric_names.rs +174 -0
- package/crates/headroom-proxy/src/observability/mod.rs +68 -0
- package/crates/headroom-proxy/src/observability/prometheus.rs +394 -0
- package/crates/headroom-proxy/src/observability/proxy_metrics.rs +368 -0
- package/crates/headroom-proxy/src/proxy.rs +1619 -0
- package/crates/headroom-proxy/src/responses_items.rs +565 -0
- package/crates/headroom-proxy/src/sse/anthropic.rs +415 -0
- package/crates/headroom-proxy/src/sse/framing.rs +366 -0
- package/crates/headroom-proxy/src/sse/mod.rs +40 -0
- package/crates/headroom-proxy/src/sse/openai_chat.rs +231 -0
- package/crates/headroom-proxy/src/sse/openai_responses.rs +458 -0
- package/crates/headroom-proxy/src/vertex/adc.rs +315 -0
- package/crates/headroom-proxy/src/vertex/envelope.rs +167 -0
- package/crates/headroom-proxy/src/vertex/mod.rs +277 -0
- package/crates/headroom-proxy/src/vertex/raw_predict.rs +490 -0
- package/crates/headroom-proxy/src/vertex/stream_raw_predict.rs +49 -0
- package/crates/headroom-proxy/src/websocket.rs +246 -0
- package/crates/headroom-py/Cargo.toml +37 -0
- package/crates/headroom-py/build.rs +77 -0
- package/crates/headroom-py/glibc_compat.c +170 -0
- package/crates/headroom-py/src/lib.rs +1655 -0
- package/deny.toml +32 -0
- package/design-system/base.css +127 -0
- package/design-system/components.css +1943 -0
- package/design-system/fonts.css +17 -0
- package/design-system/tailwind-preset.js +95 -0
- package/design-system/tokens.css +48 -0
- package/design-system/utilities.css +53 -0
- package/docker/differential-network-capture/Dockerfile.runner +14 -0
- package/docker/differential-network-capture/docker-compose.yml +142 -0
- package/docker/differential-network-capture/run-claude-lane.sh +24 -0
- package/docker/docker-compose.native.yml +44 -0
- package/docker-bake.hcl +88 -0
- package/docker-compose.yml +80 -0
- package/e2e/_lib/make_shim.ps1 +24 -0
- package/e2e/_lib/make_shim.sh +28 -0
- package/e2e/docker-native-install.sh +120 -0
- package/e2e/init/Dockerfile +79 -0
- package/e2e/wrap/Dockerfile +115 -0
- package/examples/07-context-compression.ipynb +279 -0
- package/examples/deployment/macos-launchagent/com.headroom.proxy.plist.template +65 -0
- package/examples/deployment/macos-launchagent/install.sh +227 -0
- package/examples/deployment/macos-launchagent/shell-integration.sh +74 -0
- package/examples/deployment/macos-launchagent/uninstall.sh +145 -0
- package/headroom-savings.png +0 -0
- package/headroom_learn.gif +0 -0
- package/lib/config/index.ts +12 -0
- package/lib/hooks/index.ts +4 -0
- package/lib/hooks/use-mounted.ts +13 -0
- package/lib/hooks/use-session.ts +34 -0
- package/lib/hooks/use-settings.ts +49 -0
- package/lib/utils/index.ts +1 -0
- package/lib/utils/settings.ts +162 -0
- package/lib/utils/utils.ts +6 -0
- package/llms.txt +67 -0
- package/middleware.ts +26 -0
- package/mkdocs.yml +136 -0
- package/next-env.d.ts +7 -0
- package/next.config.ts +12 -0
- package/npm/bin/xithead.js +2 -2
- package/npm/bin/xtream-head.js +366 -0
- package/npm/index.js +18 -91
- package/npm/package.json +55 -0
- package/package.json +47 -42
- package/plugins/headroom-agent-hooks/.claude-plugin/plugin.json +17 -0
- package/plugins/headroom-agent-hooks/hooks/hooks.json +29 -0
- package/plugins/headroom-oauth2/LICENSE +3 -0
- package/plugins/headroom-oauth2/SPEC.md +115 -0
- package/plugins/headroom-oauth2/pyproject.toml +48 -0
- package/plugins/hermes/headroom_retrieve/plugin.yaml +6 -0
- package/plugins/openclaw/hook-shim/HOOK.md +21 -0
- package/plugins/openclaw/hook-shim/handler.js +3 -0
- package/plugins/openclaw/openclaw.plugin.json +98 -0
- package/plugins/openclaw/package-lock.json +2692 -0
- package/plugins/openclaw/package.json +54 -0
- package/plugins/openclaw/prepare-dist.mjs +56 -0
- package/plugins/openclaw/src/convert.ts +418 -0
- package/plugins/openclaw/src/engine.ts +284 -0
- package/plugins/openclaw/src/gateway-config.ts +143 -0
- package/plugins/openclaw/src/index.ts +11 -0
- package/plugins/openclaw/src/plugin/index.ts +98 -0
- package/plugins/openclaw/src/proxy-manager.ts +477 -0
- package/plugins/openclaw/src/tools/headroom-retrieve.ts +76 -0
- package/plugins/openclaw/test/convert.test.ts +96 -0
- package/plugins/openclaw/test/engine-normalization.test.ts +21 -0
- package/plugins/openclaw/test/engine.test.ts +101 -0
- package/plugins/openclaw/test/gateway-config.test.ts +278 -0
- package/plugins/openclaw/test/plugin-runtime-routing.test.ts +141 -0
- package/plugins/openclaw/test/proxy-manager.test.ts +343 -0
- package/plugins/openclaw/tsup.config.ts +9 -0
- package/postcss.config.mjs +7 -0
- package/pyproject.toml +438 -0
- package/rust-toolchain.toml +16 -0
- package/sdk/typescript/examples/basic-compress.ts +64 -0
- package/sdk/typescript/examples/ccr-retrieve.ts +80 -0
- package/sdk/typescript/examples/hooks-custom-compression.ts +117 -0
- package/sdk/typescript/examples/middleware-composition.ts +59 -0
- package/sdk/typescript/examples/multi-provider.ts +53 -0
- package/sdk/typescript/examples/openai-anthropic-adapters.ts +73 -0
- package/sdk/typescript/examples/shared-context-multi-agent.ts +74 -0
- package/sdk/typescript/examples/simulation-dry-run.ts +80 -0
- package/sdk/typescript/examples/streaming-chat.ts +71 -0
- package/sdk/typescript/examples/structured-output.ts +99 -0
- package/sdk/typescript/examples/tool-calling-agent.ts +86 -0
- package/sdk/typescript/examples/with-headroom-vercel.ts +57 -0
- package/sdk/typescript/package-lock.json +3437 -0
- package/sdk/typescript/package.json +108 -0
- package/sdk/typescript/src/adapters/anthropic.ts +218 -0
- package/sdk/typescript/src/adapters/gemini.ts +83 -0
- package/sdk/typescript/src/adapters/openai.ts +77 -0
- package/sdk/typescript/src/adapters/vercel-ai.ts +131 -0
- package/sdk/typescript/src/client.ts +632 -0
- package/sdk/typescript/src/compress.ts +87 -0
- package/sdk/typescript/src/errors.ts +114 -0
- package/sdk/typescript/src/hooks.ts +113 -0
- package/sdk/typescript/src/index.ts +140 -0
- package/sdk/typescript/src/paths.ts +272 -0
- package/sdk/typescript/src/shared-context.ts +188 -0
- package/sdk/typescript/src/simulate.ts +52 -0
- package/sdk/typescript/src/types/config.ts +183 -0
- package/sdk/typescript/src/types/models.ts +314 -0
- package/sdk/typescript/src/types.ts +139 -0
- package/sdk/typescript/src/utils/case.ts +34 -0
- package/sdk/typescript/src/utils/format.ts +480 -0
- package/sdk/typescript/src/utils/stream.ts +50 -0
- package/sdk/typescript/test/adapters/anthropic.test.ts +217 -0
- package/sdk/typescript/test/adapters/gemini.test.ts +131 -0
- package/sdk/typescript/test/adapters/openai.test.ts +174 -0
- package/sdk/typescript/test/adapters/vercel-ai.test.ts +306 -0
- package/sdk/typescript/test/client-expanded.test.ts +333 -0
- package/sdk/typescript/test/client.test.ts +266 -0
- package/sdk/typescript/test/compress.test.ts +118 -0
- package/sdk/typescript/test/config.test.ts +275 -0
- package/sdk/typescript/test/errors.test.ts +169 -0
- package/sdk/typescript/test/format-integration.test.ts +252 -0
- package/sdk/typescript/test/hooks.test.ts +265 -0
- package/sdk/typescript/test/integration.test.ts +249 -0
- package/sdk/typescript/test/mdx-examples.test.ts +253 -0
- package/sdk/typescript/test/models.test.ts +318 -0
- package/sdk/typescript/test/parity.test.ts +337 -0
- package/sdk/typescript/test/paths.test.ts +447 -0
- package/sdk/typescript/test/shared-context.test.ts +245 -0
- package/sdk/typescript/test/simulate.test.ts +84 -0
- package/sdk/typescript/test/types.test.ts +244 -0
- package/sdk/typescript/test/utils/case.test.ts +129 -0
- package/sdk/typescript/test/utils/detect-format.test.ts +133 -0
- package/sdk/typescript/test/utils/format.test.ts +434 -0
- package/sdk/typescript/test/vercel-ai-e2e.test.ts +179 -0
- package/sdk/typescript/tsup.config.ts +17 -0
- package/tailwind.master.config.cjs +47 -0
- package/types/api.ts +12 -0
- package/types/config.ts +9 -0
- package/types/index.ts +2 -0
- package/uv.lock +6951 -0
- package/xithead.egg-info/PKG-INFO +22 -0
- package/xithead.egg-info/SOURCES.txt +16 -0
- package/xithead.egg-info/dependency_links.txt +1 -0
- package/xithead.egg-info/entry_points.txt +2 -0
- package/xithead.egg-info/requires.txt +10 -0
- package/xithead.egg-info/top_level.txt +2 -0
- package/xithead.ps1 +136 -0
- package/xithead.sh +129 -0
- package/xithead_dist/pyproject.toml +39 -0
- package/xithead_dist/xithead.egg-info/PKG-INFO +22 -0
- package/xithead_dist/xithead.egg-info/SOURCES.txt +16 -0
- package/xithead_dist/xithead.egg-info/dependency_links.txt +1 -0
- package/xithead_dist/xithead.egg-info/entry_points.txt +3 -0
- package/xithead_dist/xithead.egg-info/requires.txt +10 -0
- package/xithead_dist/xithead.egg-info/top_level.txt +2 -0
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
//! `DiffOffload` — wraps [`DiffCompressor`] as an [`OffloadTransform`].
|
|
2
|
+
//!
|
|
3
|
+
//! # Bloat heuristic — context-to-change ratio
|
|
4
|
+
//!
|
|
5
|
+
//! A unified diff is bloaty when most of its lines are context: a
|
|
6
|
+
//! 200-line diff that changes only 3 lines is 197 lines of noise.
|
|
7
|
+
//! The estimator walks the input, counts:
|
|
8
|
+
//!
|
|
9
|
+
//! * `change` lines — start with `+` or `-` (excluding the per-file
|
|
10
|
+
//! `+++`/`---` headers).
|
|
11
|
+
//! * `context` lines — start with ` ` (a single space) inside any
|
|
12
|
+
//! hunk.
|
|
13
|
+
//!
|
|
14
|
+
//! Score = `(context / (context + change) − normal_context_ratio) /
|
|
15
|
+
//! (1 − normal_context_ratio)`, clamped to [0.0, 1.0]. Below
|
|
16
|
+
//! the configured `normal_context_ratio` the diff is dense enough that
|
|
17
|
+
//! offload isn't worth the retrieval cost (score 0); at 100% context
|
|
18
|
+
//! it scores 1.0.
|
|
19
|
+
//!
|
|
20
|
+
//! No regex — pure byte-prefix checks. Cost: O(n) over the input
|
|
21
|
+
//! (single pass, no allocations).
|
|
22
|
+
//!
|
|
23
|
+
//! # CCR persistence
|
|
24
|
+
//!
|
|
25
|
+
//! `DiffCompressor::compress_with_store` writes the original payload to
|
|
26
|
+
//! the orchestrator-supplied store under the same `cache_key` it embeds
|
|
27
|
+
//! in the wire marker. The trait contract is satisfied by the
|
|
28
|
+
//! compressor itself; no double-store hack needed at the wrapper level.
|
|
29
|
+
//! (Earlier revisions of this offload did the post-hoc store from here
|
|
30
|
+
//! because the compressor lacked a store parameter — that's been
|
|
31
|
+
//! upstreamed in the audit-cleanup PR.)
|
|
32
|
+
//!
|
|
33
|
+
//! [`DiffCompressor`]: crate::transforms::diff_compressor::DiffCompressor
|
|
34
|
+
//! [`OffloadTransform`]: crate::transforms::pipeline::traits::OffloadTransform
|
|
35
|
+
|
|
36
|
+
use crate::ccr::CcrStore;
|
|
37
|
+
use crate::transforms::diff_compressor::{DiffCompressor, DiffCompressorConfig};
|
|
38
|
+
use crate::transforms::pipeline::config::DiffBloatConfig;
|
|
39
|
+
use crate::transforms::pipeline::traits::{
|
|
40
|
+
CompressionContext, OffloadOutput, OffloadTransform, TransformError,
|
|
41
|
+
};
|
|
42
|
+
use crate::transforms::ContentType;
|
|
43
|
+
|
|
44
|
+
const NAME: &str = "diff_offload";
|
|
45
|
+
/// Confidence is high — DiffCompressor has 20+ parity fixtures.
|
|
46
|
+
const CONFIDENCE: f32 = 0.85;
|
|
47
|
+
|
|
48
|
+
pub struct DiffOffload {
|
|
49
|
+
compressor: DiffCompressor,
|
|
50
|
+
bloat: DiffBloatConfig,
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
impl DiffOffload {
|
|
54
|
+
pub fn new(bloat: DiffBloatConfig) -> Self {
|
|
55
|
+
Self::with_compressor(DiffCompressor::new(DiffCompressorConfig::default()), bloat)
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
pub fn with_compressor(compressor: DiffCompressor, bloat: DiffBloatConfig) -> Self {
|
|
59
|
+
Self { compressor, bloat }
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
impl OffloadTransform for DiffOffload {
|
|
64
|
+
fn name(&self) -> &'static str {
|
|
65
|
+
NAME
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
fn applies_to(&self) -> &[ContentType] {
|
|
69
|
+
&[ContentType::GitDiff]
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
fn estimate_bloat(&self, content: &str) -> f32 {
|
|
73
|
+
if content.is_empty() {
|
|
74
|
+
return 0.0;
|
|
75
|
+
}
|
|
76
|
+
let mut total_lines = 0usize;
|
|
77
|
+
let mut change = 0usize;
|
|
78
|
+
let mut context = 0usize;
|
|
79
|
+
let mut in_hunk = false;
|
|
80
|
+
|
|
81
|
+
for line in content.lines() {
|
|
82
|
+
total_lines += 1;
|
|
83
|
+
// Hunk header signals "the next ` `/`+`/`-` lines are part
|
|
84
|
+
// of a real hunk." Bare prefix checks — no regex.
|
|
85
|
+
if line.starts_with("@@") {
|
|
86
|
+
in_hunk = true;
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
// File-level headers can carry `+++`/`---` prefixes; those
|
|
90
|
+
// aren't change lines. Reset hunk state on a new file
|
|
91
|
+
// header (begins with `diff --git`).
|
|
92
|
+
if line.starts_with("diff --git") {
|
|
93
|
+
in_hunk = false;
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
if line.starts_with("+++") || line.starts_with("---") {
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
if !in_hunk {
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
match line.as_bytes().first() {
|
|
103
|
+
Some(b'+') | Some(b'-') => change += 1,
|
|
104
|
+
Some(b' ') => context += 1,
|
|
105
|
+
_ => {}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
if total_lines < self.bloat.min_lines {
|
|
110
|
+
return 0.0;
|
|
111
|
+
}
|
|
112
|
+
let denom = (context + change) as f64;
|
|
113
|
+
if denom == 0.0 {
|
|
114
|
+
return 0.0;
|
|
115
|
+
}
|
|
116
|
+
let ratio = context as f64 / denom;
|
|
117
|
+
let normal = self.bloat.normal_context_ratio;
|
|
118
|
+
if ratio <= normal {
|
|
119
|
+
return 0.0;
|
|
120
|
+
}
|
|
121
|
+
// Map (normal, 1.0] → (0, 1] linearly.
|
|
122
|
+
let span = 1.0 - normal;
|
|
123
|
+
if span <= 0.0 {
|
|
124
|
+
// Degenerate config: treat any context as full bloat.
|
|
125
|
+
return 1.0;
|
|
126
|
+
}
|
|
127
|
+
((ratio - normal) / span).clamp(0.0, 1.0) as f32
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
fn apply(
|
|
131
|
+
&self,
|
|
132
|
+
content: &str,
|
|
133
|
+
ctx: &CompressionContext,
|
|
134
|
+
store: &dyn CcrStore,
|
|
135
|
+
) -> Result<OffloadOutput, TransformError> {
|
|
136
|
+
// `compress_with_store` (added in the audit-cleanup PR) writes
|
|
137
|
+
// the original to `store` under the same `cache_key` it embeds
|
|
138
|
+
// in the marker. Earlier versions of this offload had to
|
|
139
|
+
// double-store post-hoc because the compressor lacked a store
|
|
140
|
+
// parameter — that hack is gone.
|
|
141
|
+
let (result, _) = self
|
|
142
|
+
.compressor
|
|
143
|
+
.compress_with_store(content, &ctx.query, Some(store));
|
|
144
|
+
|
|
145
|
+
let Some(key) = result.cache_key else {
|
|
146
|
+
return Err(TransformError::skipped(
|
|
147
|
+
NAME,
|
|
148
|
+
"diff compressor did not emit a cache_key",
|
|
149
|
+
));
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
Ok(OffloadOutput::from_lengths(
|
|
153
|
+
content.len(),
|
|
154
|
+
result.compressed,
|
|
155
|
+
key,
|
|
156
|
+
))
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
fn confidence(&self) -> f32 {
|
|
160
|
+
CONFIDENCE
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
#[cfg(test)]
|
|
165
|
+
mod tests {
|
|
166
|
+
use super::*;
|
|
167
|
+
use crate::ccr::InMemoryCcrStore;
|
|
168
|
+
use crate::transforms::pipeline::config::PipelineConfig;
|
|
169
|
+
|
|
170
|
+
fn default_bloat() -> DiffBloatConfig {
|
|
171
|
+
PipelineConfig::default().bloat.diff
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
fn offload() -> DiffOffload {
|
|
175
|
+
DiffOffload::new(default_bloat())
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
fn build_diff(num_files: usize, context_per_file: usize, changes_per_file: usize) -> String {
|
|
179
|
+
let mut s = String::new();
|
|
180
|
+
for f in 0..num_files {
|
|
181
|
+
s.push_str(&format!(
|
|
182
|
+
"diff --git a/file{f}.txt b/file{f}.txt\n--- a/file{f}.txt\n+++ b/file{f}.txt\n@@ -1,{} +1,{} @@\n",
|
|
183
|
+
context_per_file + changes_per_file,
|
|
184
|
+
context_per_file + changes_per_file
|
|
185
|
+
));
|
|
186
|
+
for _ in 0..context_per_file {
|
|
187
|
+
s.push_str(" context line\n");
|
|
188
|
+
}
|
|
189
|
+
for c in 0..changes_per_file {
|
|
190
|
+
s.push_str(&format!("-removed line {c}\n"));
|
|
191
|
+
s.push_str(&format!("+added line {c}\n"));
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
s
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
#[test]
|
|
198
|
+
fn name_and_applies_to() {
|
|
199
|
+
let o = offload();
|
|
200
|
+
assert_eq!(o.name(), "diff_offload");
|
|
201
|
+
assert_eq!(o.applies_to(), &[ContentType::GitDiff]);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
#[test]
|
|
205
|
+
fn estimate_bloat_empty_input_is_zero() {
|
|
206
|
+
assert_eq!(offload().estimate_bloat(""), 0.0);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
#[test]
|
|
210
|
+
fn estimate_bloat_below_min_lines_is_zero() {
|
|
211
|
+
let small = build_diff(1, 5, 1); // ~13 lines
|
|
212
|
+
assert_eq!(offload().estimate_bloat(&small), 0.0);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
#[test]
|
|
216
|
+
fn estimate_bloat_dense_diff_scores_zero() {
|
|
217
|
+
// Lots of changes, little context — below `normal_context_ratio=0.6`.
|
|
218
|
+
let diff = build_diff(2, 5, 60); // 120 changes, 10 context
|
|
219
|
+
let score = offload().estimate_bloat(&diff);
|
|
220
|
+
assert_eq!(score, 0.0, "dense diff should score zero, got {score}");
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
#[test]
|
|
224
|
+
fn estimate_bloat_context_heavy_diff_scores_high() {
|
|
225
|
+
// Mostly context, few changes — far above 0.6.
|
|
226
|
+
// 200 context lines, 5 changes each side = 200 / (200+10) ≈ 0.95.
|
|
227
|
+
let diff = build_diff(1, 200, 5);
|
|
228
|
+
let score = offload().estimate_bloat(&diff);
|
|
229
|
+
assert!(
|
|
230
|
+
score > 0.7,
|
|
231
|
+
"context-heavy diff should score high, got {score}"
|
|
232
|
+
);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
#[test]
|
|
236
|
+
fn estimate_bloat_at_threshold_scores_zero() {
|
|
237
|
+
// Exactly normal_context_ratio worth of context — maps to 0.
|
|
238
|
+
// Construct: 60 context, 20 changes-per-side (total 40 changes)
|
|
239
|
+
// → context / (context+change) = 60/100 = 0.6 = threshold.
|
|
240
|
+
let diff = build_diff(1, 60, 20);
|
|
241
|
+
// Make sure it clears min_lines.
|
|
242
|
+
let score = offload().estimate_bloat(&diff);
|
|
243
|
+
assert_eq!(score, 0.0, "at threshold should be zero, got {score}");
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
#[test]
|
|
247
|
+
fn estimate_bloat_safe_on_huge_inputs() {
|
|
248
|
+
let diff = build_diff(50, 100, 50); // many files, big hunks
|
|
249
|
+
let _ = offload().estimate_bloat(&diff);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
#[test]
|
|
253
|
+
fn apply_emits_key_and_persists_original() {
|
|
254
|
+
let diff = build_diff(1, 200, 5);
|
|
255
|
+
let store = InMemoryCcrStore::new();
|
|
256
|
+
let r = offload()
|
|
257
|
+
.apply(&diff, &CompressionContext::default(), &store)
|
|
258
|
+
.expect("should compress");
|
|
259
|
+
assert!(!r.cache_key.is_empty());
|
|
260
|
+
// Bug-fix verification: DiffCompressor itself wouldn't have
|
|
261
|
+
// stored, but our wrapper must.
|
|
262
|
+
assert_eq!(store.get(&r.cache_key).as_deref(), Some(diff.as_str()));
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
#[test]
|
|
266
|
+
fn apply_skipped_when_compressor_declines_ccr() {
|
|
267
|
+
// Below DiffCompressor's `min_lines_for_ccr=50`.
|
|
268
|
+
let diff = build_diff(1, 5, 2);
|
|
269
|
+
let store = InMemoryCcrStore::new();
|
|
270
|
+
let err = offload()
|
|
271
|
+
.apply(&diff, &CompressionContext::default(), &store)
|
|
272
|
+
.expect_err("must skip");
|
|
273
|
+
match err {
|
|
274
|
+
TransformError::Skipped { transform, .. } => assert_eq!(transform, "diff_offload"),
|
|
275
|
+
_ => panic!("expected Skipped, got {err:?}"),
|
|
276
|
+
}
|
|
277
|
+
assert_eq!(store.len(), 0);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
@@ -0,0 +1,400 @@
|
|
|
1
|
+
//! `JsonOffload` — wraps [`SmartCrusher`] as an [`OffloadTransform`].
|
|
2
|
+
//!
|
|
3
|
+
//! # What this offload does
|
|
4
|
+
//!
|
|
5
|
+
//! JSON arrays of dicts (rows) are the worst tool-output shape for
|
|
6
|
+
//! token bloat: every row repeats its field names, and many tools
|
|
7
|
+
//! return long arrays with near-duplicate or low-information rows
|
|
8
|
+
//! (audit logs, search index dumps, paginated API results). SmartCrusher
|
|
9
|
+
//! is the existing Rust port that handles this — schema dedup, row
|
|
10
|
+
//! sampling, anchor-aware row selection, lossless tabular compaction —
|
|
11
|
+
//! and emits CCR markers for any rows it drops.
|
|
12
|
+
//!
|
|
13
|
+
//! `JsonOffload` plugs SmartCrusher into the pipeline's
|
|
14
|
+
//! [`OffloadTransform`] contract:
|
|
15
|
+
//!
|
|
16
|
+
//! 1. `estimate_bloat` is a CHEAP byte scan that spots the
|
|
17
|
+
//! array-of-objects shape and counts row separators (`}` followed
|
|
18
|
+
//! by `,` or `]`). No JSON parse — that's reserved for `apply`.
|
|
19
|
+
//! 2. `apply` delegates to `SmartCrusher::crush(content, ctx.query)`
|
|
20
|
+
//! and adds a wrapper-level CCR marker. The orchestrator-supplied
|
|
21
|
+
//! store gets the original payload under the wrapper hash.
|
|
22
|
+
//!
|
|
23
|
+
//! # Why a wrapper-level CCR hash on top of SmartCrusher's internal one
|
|
24
|
+
//!
|
|
25
|
+
//! SmartCrusher mints PER-SUB-ARRAY hashes (`<<ccr:abc 42_rows_offloaded>>`)
|
|
26
|
+
//! that resolve in its **internal** store. The pipeline orchestrator
|
|
27
|
+
//! has its **own** store (passed to `apply` via the trait). To honor
|
|
28
|
+
//! the trait contract — `cache_key` MUST resolve in the orchestrator's
|
|
29
|
+
//! store — `JsonOffload` hashes the WHOLE input and stashes it via the
|
|
30
|
+
//! orchestrator store under that hash, returning that hash as the
|
|
31
|
+
//! `cache_key`.
|
|
32
|
+
//!
|
|
33
|
+
//! For retrieval the LLM uses the wrapper's outer hash (recovers the
|
|
34
|
+
//! whole original JSON). SmartCrusher's per-array markers in the
|
|
35
|
+
//! compressed body are informational hints — not directly resolvable
|
|
36
|
+
//! through the pipeline-level store, but the LLM doesn't need them
|
|
37
|
+
//! when the wrapper hash gives back the full payload.
|
|
38
|
+
//!
|
|
39
|
+
//! # No regex
|
|
40
|
+
//!
|
|
41
|
+
//! Per project convention. Estimator is byte-prefix scan +
|
|
42
|
+
//! `str::matches` over fixed substrings. SmartCrusher itself uses
|
|
43
|
+
//! `serde_json` for parsing, never regex.
|
|
44
|
+
//!
|
|
45
|
+
//! [`SmartCrusher`]: crate::transforms::smart_crusher::SmartCrusher
|
|
46
|
+
//! [`OffloadTransform`]: crate::transforms::pipeline::traits::OffloadTransform
|
|
47
|
+
|
|
48
|
+
use md5::{Digest, Md5};
|
|
49
|
+
|
|
50
|
+
use crate::ccr::CcrStore;
|
|
51
|
+
use crate::transforms::pipeline::config::JsonOffloadConfig;
|
|
52
|
+
use crate::transforms::pipeline::traits::{
|
|
53
|
+
CompressionContext, OffloadOutput, OffloadTransform, TransformError,
|
|
54
|
+
};
|
|
55
|
+
use crate::transforms::smart_crusher::{SmartCrusher, SmartCrusherConfig};
|
|
56
|
+
use crate::transforms::ContentType;
|
|
57
|
+
|
|
58
|
+
const NAME: &str = "json_offload";
|
|
59
|
+
/// SmartCrusher has 50+ parity fixtures and shadow-validated against
|
|
60
|
+
/// Python — high confidence.
|
|
61
|
+
const CONFIDENCE: f32 = 0.85;
|
|
62
|
+
|
|
63
|
+
pub struct JsonOffload {
|
|
64
|
+
crusher: SmartCrusher,
|
|
65
|
+
config: JsonOffloadConfig,
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
impl JsonOffload {
|
|
69
|
+
/// Default constructor — builds SmartCrusher with the OSS default
|
|
70
|
+
/// composition (scorer + constraints + compaction stage + internal
|
|
71
|
+
/// CCR store). The internal store handles SmartCrusher's per-array
|
|
72
|
+
/// markers; the wrapper still emits its own outer marker that
|
|
73
|
+
/// resolves in the orchestrator-supplied store.
|
|
74
|
+
pub fn new(config: JsonOffloadConfig) -> Self {
|
|
75
|
+
Self {
|
|
76
|
+
crusher: SmartCrusher::new(SmartCrusherConfig::default()),
|
|
77
|
+
config,
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/// Custom constructor — used by tests that want a stubbed crusher
|
|
82
|
+
/// or a custom SmartCrusher config.
|
|
83
|
+
pub fn with_crusher(crusher: SmartCrusher, config: JsonOffloadConfig) -> Self {
|
|
84
|
+
Self { crusher, config }
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
impl OffloadTransform for JsonOffload {
|
|
89
|
+
fn name(&self) -> &'static str {
|
|
90
|
+
NAME
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
fn applies_to(&self) -> &[ContentType] {
|
|
94
|
+
&[ContentType::JsonArray]
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
fn estimate_bloat(&self, content: &str) -> f32 {
|
|
98
|
+
if content.is_empty() {
|
|
99
|
+
return 0.0;
|
|
100
|
+
}
|
|
101
|
+
let trimmed = content.trim_start();
|
|
102
|
+
if !trimmed.starts_with('[') {
|
|
103
|
+
return 0.0;
|
|
104
|
+
}
|
|
105
|
+
// Cheap structural scan: count `},{` or `}, {` occurrences as
|
|
106
|
+
// row boundaries. Doesn't parse JSON — that's reserved for
|
|
107
|
+
// `apply`. Conservative: counts only adjacent-row patterns,
|
|
108
|
+
// not the trailing `}` before `]` (so a 5-row array reports 4
|
|
109
|
+
// separators, but for our threshold logic that rounds correctly).
|
|
110
|
+
let separators = count_row_separators(content);
|
|
111
|
+
if separators < self.config.min_array_rows.saturating_sub(1) {
|
|
112
|
+
return 0.0;
|
|
113
|
+
}
|
|
114
|
+
let saturation = self.config.saturation_rows.saturating_sub(1).max(1);
|
|
115
|
+
(separators as f32 / saturation as f32).clamp(0.0, 1.0)
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
fn apply(
|
|
119
|
+
&self,
|
|
120
|
+
content: &str,
|
|
121
|
+
ctx: &CompressionContext,
|
|
122
|
+
store: &dyn CcrStore,
|
|
123
|
+
) -> Result<OffloadOutput, TransformError> {
|
|
124
|
+
let result = self.crusher.crush(content, &ctx.query, 0.0);
|
|
125
|
+
if !result.was_modified {
|
|
126
|
+
return Err(TransformError::skipped(
|
|
127
|
+
NAME,
|
|
128
|
+
"smart crusher returned passthrough",
|
|
129
|
+
));
|
|
130
|
+
}
|
|
131
|
+
if result.compressed.len() >= content.len() {
|
|
132
|
+
return Err(TransformError::skipped(NAME, "no savings after crush"));
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// Wrapper-level CCR: hash the WHOLE original input, stash it
|
|
136
|
+
// through the orchestrator-supplied store, append a marker so
|
|
137
|
+
// the LLM sees a resolvable handle. SmartCrusher's per-array
|
|
138
|
+
// markers in `result.compressed` stay informational — the LLM
|
|
139
|
+
// retrieves the full original via this outer hash.
|
|
140
|
+
let key = md5_hex_24(content);
|
|
141
|
+
store.put(&key, content);
|
|
142
|
+
let mut output = result.compressed;
|
|
143
|
+
output.push_str("\n[json_offload CCR: hash=");
|
|
144
|
+
output.push_str(&key);
|
|
145
|
+
output.push(']');
|
|
146
|
+
|
|
147
|
+
Ok(OffloadOutput::from_lengths(content.len(), output, key))
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
fn confidence(&self) -> f32 {
|
|
151
|
+
CONFIDENCE
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/// Count plausible row-boundary patterns in `content`. Used by the
|
|
156
|
+
/// bloat estimator as a cheap proxy for JSON array length. Patterns
|
|
157
|
+
/// counted (no overlap):
|
|
158
|
+
///
|
|
159
|
+
/// * `},{` — compact JSON, no whitespace.
|
|
160
|
+
/// * `}, {` — pretty-printed JSON with single-space.
|
|
161
|
+
/// * `},\n` — pretty-printed JSON with newline-after-comma.
|
|
162
|
+
///
|
|
163
|
+
/// We don't attempt to be perfect — the estimator just needs an
|
|
164
|
+
/// order-of-magnitude signal. False positives on `},` strings inside
|
|
165
|
+
/// quoted values are tolerated; their rate is low and the eventual
|
|
166
|
+
/// `apply` does the rigorous JSON parse anyway.
|
|
167
|
+
fn count_row_separators(content: &str) -> usize {
|
|
168
|
+
content.matches("},{").count()
|
|
169
|
+
+ content.matches("}, {").count()
|
|
170
|
+
+ content.matches("},\n").count()
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/// MD5 of `content`, hex-encoded, truncated to 24 chars. Matches the
|
|
174
|
+
/// CCR convention used by other offload wrappers.
|
|
175
|
+
fn md5_hex_24(content: &str) -> String {
|
|
176
|
+
let mut h = Md5::new();
|
|
177
|
+
h.update(content.as_bytes());
|
|
178
|
+
let digest = h.finalize();
|
|
179
|
+
let mut hex = String::with_capacity(32);
|
|
180
|
+
for b in digest.iter() {
|
|
181
|
+
hex.push_str(&format!("{:02x}", b));
|
|
182
|
+
}
|
|
183
|
+
hex.truncate(24);
|
|
184
|
+
hex
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
#[cfg(test)]
|
|
188
|
+
mod tests {
|
|
189
|
+
use super::*;
|
|
190
|
+
use crate::ccr::InMemoryCcrStore;
|
|
191
|
+
use crate::transforms::pipeline::config::PipelineConfig;
|
|
192
|
+
|
|
193
|
+
fn cfg() -> JsonOffloadConfig {
|
|
194
|
+
PipelineConfig::default().offload.json
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
fn offload() -> JsonOffload {
|
|
198
|
+
JsonOffload::new(cfg())
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/// Build a JSON array of N similar dicts with id + name + value.
|
|
202
|
+
/// Compact JSON (no extra whitespace) so byte counts are predictable.
|
|
203
|
+
fn build_tabular_array(n: usize) -> String {
|
|
204
|
+
let mut s = String::from("[");
|
|
205
|
+
for i in 0..n {
|
|
206
|
+
if i > 0 {
|
|
207
|
+
s.push(',');
|
|
208
|
+
}
|
|
209
|
+
s.push_str(&format!(
|
|
210
|
+
"{{\"id\":{},\"name\":\"item-{}\",\"value\":{}}}",
|
|
211
|
+
i,
|
|
212
|
+
i,
|
|
213
|
+
i * 100
|
|
214
|
+
));
|
|
215
|
+
}
|
|
216
|
+
s.push(']');
|
|
217
|
+
s
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
#[test]
|
|
221
|
+
fn name_and_applies_to() {
|
|
222
|
+
let o = offload();
|
|
223
|
+
assert_eq!(o.name(), "json_offload");
|
|
224
|
+
assert_eq!(o.applies_to(), &[ContentType::JsonArray]);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
#[test]
|
|
228
|
+
fn estimate_bloat_empty_input_zero() {
|
|
229
|
+
assert_eq!(offload().estimate_bloat(""), 0.0);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
#[test]
|
|
233
|
+
fn estimate_bloat_non_array_input_zero() {
|
|
234
|
+
// Object, not array.
|
|
235
|
+
assert_eq!(offload().estimate_bloat(r#"{"a":1,"b":2}"#), 0.0);
|
|
236
|
+
// Plain text.
|
|
237
|
+
assert_eq!(offload().estimate_bloat("just words here"), 0.0);
|
|
238
|
+
// Number.
|
|
239
|
+
assert_eq!(offload().estimate_bloat("42"), 0.0);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
#[test]
|
|
243
|
+
fn estimate_bloat_below_min_rows_zero() {
|
|
244
|
+
let arr = build_tabular_array(3); // 2 separators, default min_array_rows=5 → need ≥4
|
|
245
|
+
assert_eq!(offload().estimate_bloat(&arr), 0.0);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
#[test]
|
|
249
|
+
fn estimate_bloat_at_saturation_is_one() {
|
|
250
|
+
// 100 rows → 99 separators. saturation_rows=50 → 49 saturates.
|
|
251
|
+
let arr = build_tabular_array(100);
|
|
252
|
+
let score = offload().estimate_bloat(&arr);
|
|
253
|
+
assert!(score >= 0.99, "expected ~1.0 at saturation, got {score}");
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
#[test]
|
|
257
|
+
fn estimate_bloat_scales_linearly_in_middle_range() {
|
|
258
|
+
// 25 rows → 24 separators. saturation=49 → score ≈ 0.49.
|
|
259
|
+
let arr = build_tabular_array(25);
|
|
260
|
+
let score = offload().estimate_bloat(&arr);
|
|
261
|
+
assert!(
|
|
262
|
+
(0.4..=0.6).contains(&score),
|
|
263
|
+
"expected mid-range score, got {score}"
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
#[test]
|
|
268
|
+
fn estimate_bloat_handles_whitespace_after_array_open() {
|
|
269
|
+
// Pretty-printed JSON: `[ {`, then `}, {`, etc.
|
|
270
|
+
let mut s = String::from("[\n");
|
|
271
|
+
for i in 0..30 {
|
|
272
|
+
if i > 0 {
|
|
273
|
+
s.push_str(",\n");
|
|
274
|
+
}
|
|
275
|
+
s.push_str(&format!(
|
|
276
|
+
" {{\"id\":{i},\"name\":\"item-{i}\",\"value\":{}}}",
|
|
277
|
+
i * 100
|
|
278
|
+
));
|
|
279
|
+
}
|
|
280
|
+
s.push_str("\n]");
|
|
281
|
+
let score = offload().estimate_bloat(&s);
|
|
282
|
+
assert!(
|
|
283
|
+
score > 0.4,
|
|
284
|
+
"pretty-printed array should still score, got {score}"
|
|
285
|
+
);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
#[test]
|
|
289
|
+
fn estimate_bloat_handles_huge_input_safely() {
|
|
290
|
+
let arr = build_tabular_array(50_000);
|
|
291
|
+
// Just must not panic; saturation handles this fine.
|
|
292
|
+
let score = offload().estimate_bloat(&arr);
|
|
293
|
+
assert!((0.99..=1.0).contains(&score));
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
#[test]
|
|
297
|
+
fn apply_compresses_large_tabular_array_and_stores_original() {
|
|
298
|
+
// Big enough that SmartCrusher will engage.
|
|
299
|
+
let arr = build_tabular_array(500);
|
|
300
|
+
let store = InMemoryCcrStore::new();
|
|
301
|
+
let r = offload()
|
|
302
|
+
.apply(&arr, &CompressionContext::default(), &store)
|
|
303
|
+
.expect("smart crusher should compress");
|
|
304
|
+
assert!(r.bytes_saved > 0);
|
|
305
|
+
assert!(!r.cache_key.is_empty());
|
|
306
|
+
// Wrapper marker present.
|
|
307
|
+
assert!(r.output.contains("[json_offload CCR: hash="));
|
|
308
|
+
// Original recoverable through the orchestrator's store.
|
|
309
|
+
assert_eq!(store.get(&r.cache_key).as_deref(), Some(arr.as_str()));
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
#[test]
|
|
313
|
+
fn apply_skipped_when_smart_crusher_passes_through() {
|
|
314
|
+
// 2-row array: below SmartCrusher's `min_items_to_analyze` so
|
|
315
|
+
// it returns passthrough — wrapper must surface that as Skipped.
|
|
316
|
+
let arr = build_tabular_array(2);
|
|
317
|
+
let store = InMemoryCcrStore::new();
|
|
318
|
+
let err = offload()
|
|
319
|
+
.apply(&arr, &CompressionContext::default(), &store)
|
|
320
|
+
.expect_err("must skip");
|
|
321
|
+
match err {
|
|
322
|
+
TransformError::Skipped { transform, .. } => assert_eq!(transform, "json_offload"),
|
|
323
|
+
_ => panic!("expected Skipped, got {err:?}"),
|
|
324
|
+
}
|
|
325
|
+
assert_eq!(store.len(), 0);
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
#[test]
|
|
329
|
+
fn apply_skipped_for_non_json_input() {
|
|
330
|
+
let store = InMemoryCcrStore::new();
|
|
331
|
+
let err = offload()
|
|
332
|
+
.apply("not json at all", &CompressionContext::default(), &store)
|
|
333
|
+
.expect_err("must skip non-json");
|
|
334
|
+
match err {
|
|
335
|
+
TransformError::Skipped { .. } => {}
|
|
336
|
+
_ => panic!("expected Skipped, got {err:?}"),
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
#[test]
|
|
341
|
+
fn apply_propagates_query_anchors_into_smart_crusher() {
|
|
342
|
+
// Construct a tabular array where one row has an anchor that
|
|
343
|
+
// matches the query. SmartCrusher should be biased to keep
|
|
344
|
+
// that row. We don't assert exactly what survives — just that
|
|
345
|
+
// it ran with the query in scope.
|
|
346
|
+
let mut s = String::from("[");
|
|
347
|
+
for i in 0..50 {
|
|
348
|
+
if i > 0 {
|
|
349
|
+
s.push(',');
|
|
350
|
+
}
|
|
351
|
+
let name = if i == 17 {
|
|
352
|
+
"needle".to_string()
|
|
353
|
+
} else {
|
|
354
|
+
format!("hay-{i}")
|
|
355
|
+
};
|
|
356
|
+
s.push_str(&format!(
|
|
357
|
+
"{{\"id\":{i},\"name\":\"{name}\",\"score\":{}}}",
|
|
358
|
+
i % 7
|
|
359
|
+
));
|
|
360
|
+
}
|
|
361
|
+
s.push(']');
|
|
362
|
+
let store = InMemoryCcrStore::new();
|
|
363
|
+
let ctx = CompressionContext::with_query("needle");
|
|
364
|
+
let r = offload()
|
|
365
|
+
.apply(&s, &ctx, &store)
|
|
366
|
+
.expect("crusher should run");
|
|
367
|
+
// Output should reference "needle" — SmartCrusher's anchor logic
|
|
368
|
+
// should keep that row.
|
|
369
|
+
assert!(
|
|
370
|
+
r.output.contains("needle"),
|
|
371
|
+
"anchor row should survive crush"
|
|
372
|
+
);
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
#[test]
|
|
376
|
+
fn cache_key_is_stable_across_calls_for_same_input() {
|
|
377
|
+
let arr = build_tabular_array(100);
|
|
378
|
+
let store_a = InMemoryCcrStore::new();
|
|
379
|
+
let store_b = InMemoryCcrStore::new();
|
|
380
|
+
let r_a = offload()
|
|
381
|
+
.apply(&arr, &CompressionContext::default(), &store_a)
|
|
382
|
+
.expect("ok");
|
|
383
|
+
let r_b = offload()
|
|
384
|
+
.apply(&arr, &CompressionContext::default(), &store_b)
|
|
385
|
+
.expect("ok");
|
|
386
|
+
assert_eq!(
|
|
387
|
+
r_a.cache_key, r_b.cache_key,
|
|
388
|
+
"cache_key should be a deterministic hash of input"
|
|
389
|
+
);
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
#[test]
|
|
393
|
+
fn count_row_separators_handles_compact_and_pretty() {
|
|
394
|
+
assert_eq!(count_row_separators(""), 0);
|
|
395
|
+
assert_eq!(count_row_separators("[]"), 0);
|
|
396
|
+
assert_eq!(count_row_separators(r#"[{"a":1}]"#), 0);
|
|
397
|
+
assert_eq!(count_row_separators(r#"[{"a":1},{"a":2}]"#), 1);
|
|
398
|
+
assert_eq!(count_row_separators(r#"[{"a":1}, {"a":2}, {"a":3}]"#), 2);
|
|
399
|
+
}
|
|
400
|
+
}
|