xithead 1.2.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.actrc +14 -0
- package/.actrc.local.example +15 -0
- package/.changelog.md +2 -0
- package/.claude/agents/academic-anthropologist.md +125 -0
- package/.claude/agents/academic-geographer.md +127 -0
- package/.claude/agents/academic-historian.md +123 -0
- package/.claude/agents/academic-narratologist.md +118 -0
- package/.claude/agents/academic-psychologist.md +118 -0
- package/.claude/agents/academic-statistician.md +144 -0
- package/.claude/agents/accounts-payable-agent.md +205 -0
- package/.claude/agents/agentic-identity-trust.md +394 -0
- package/.claude/agents/agents-orchestrator.md +367 -0
- package/.claude/agents/automation-governance-architect.md +216 -0
- package/.claude/agents/blender-addon-engineer.md +243 -0
- package/.claude/agents/business-strategist.md +488 -0
- package/.claude/agents/change-management-consultant.md +497 -0
- package/.claude/agents/chief-financial-officer.md +388 -0
- package/.claude/agents/corporate-training-designer.md +192 -0
- package/.claude/agents/customer-service.md +398 -0
- package/.claude/agents/customer-success-manager.md +460 -0
- package/.claude/agents/data-consolidation-agent.md +60 -0
- package/.claude/agents/data-privacy-officer.md +412 -0
- package/.claude/agents/design-brand-guardian.md +322 -0
- package/.claude/agents/design-image-prompt-engineer.md +236 -0
- package/.claude/agents/design-inclusive-visuals-specialist.md +73 -0
- package/.claude/agents/design-persona-walkthrough.md +272 -0
- package/.claude/agents/design-ui-designer.md +386 -0
- package/.claude/agents/design-ui-finish-gate-reviewer.md +217 -0
- package/.claude/agents/design-ux-architect.md +482 -0
- package/.claude/agents/design-ux-researcher.md +329 -0
- package/.claude/agents/design-visual-storyteller.md +149 -0
- package/.claude/agents/design-whimsy-injector.md +460 -0
- package/.claude/agents/economy-designer.md +156 -0
- package/.claude/agents/engineering-ai-data-remediation-engineer.md +211 -0
- package/.claude/agents/engineering-ai-engineer.md +146 -0
- package/.claude/agents/engineering-api-platform-engineer.md +181 -0
- package/.claude/agents/engineering-ats-validator-architect.md +383 -0
- package/.claude/agents/engineering-autonomous-optimization-architect.md +121 -0
- package/.claude/agents/engineering-backend-architect.md +255 -0
- package/.claude/agents/engineering-china-network-engineer.md +252 -0
- package/.claude/agents/engineering-cms-developer.md +536 -0
- package/.claude/agents/engineering-code-reviewer.md +76 -0
- package/.claude/agents/engineering-codebase-onboarding-engineer.md +173 -0
- package/.claude/agents/engineering-data-engineer.md +349 -0
- package/.claude/agents/engineering-data-visualization-engineer.md +151 -0
- package/.claude/agents/engineering-database-optimizer.md +188 -0
- package/.claude/agents/engineering-database-reliability-engineer.md +201 -0
- package/.claude/agents/engineering-desktop-app-engineer.md +209 -0
- package/.claude/agents/engineering-developer-tooling-engineer.md +153 -0
- package/.claude/agents/engineering-devops-automator.md +385 -0
- package/.claude/agents/engineering-drupal-performance.md +347 -0
- package/.claude/agents/engineering-drupal-shopping-cart.md +360 -0
- package/.claude/agents/engineering-electrical-engineer.md +575 -0
- package/.claude/agents/engineering-email-intelligence-engineer.md +390 -0
- package/.claude/agents/engineering-embedded-firmware-engineer.md +196 -0
- package/.claude/agents/engineering-feishu-integration-developer.md +606 -0
- package/.claude/agents/engineering-filament-optimization-specialist.md +283 -0
- package/.claude/agents/engineering-finops-engineer.md +164 -0
- package/.claude/agents/engineering-frontend-developer.md +257 -0
- package/.claude/agents/engineering-gaussdb-expert.md +334 -0
- package/.claude/agents/engineering-git-workflow-master.md +98 -0
- package/.claude/agents/engineering-i18n-engineer.md +214 -0
- package/.claude/agents/engineering-identity-access-engineer.md +217 -0
- package/.claude/agents/engineering-incident-response-commander.md +444 -0
- package/.claude/agents/engineering-iot-fleet-engineer.md +148 -0
- package/.claude/agents/engineering-it-service-manager.md +561 -0
- package/.claude/agents/engineering-knowledge-graph-engineer.md +372 -0
- package/.claude/agents/engineering-llm-post-training-engineer.md +166 -0
- package/.claude/agents/engineering-minimal-change-engineer.md +207 -0
- package/.claude/agents/engineering-mobile-app-builder.md +493 -0
- package/.claude/agents/engineering-mobile-release-engineer.md +164 -0
- package/.claude/agents/engineering-multi-agent-systems-architect.md +600 -0
- package/.claude/agents/engineering-network-engineer.md +245 -0
- package/.claude/agents/engineering-orgscript-engineer.md +113 -0
- package/.claude/agents/engineering-payments-billing-engineer.md +228 -0
- package/.claude/agents/engineering-pdf-engine-architect.md +696 -0
- package/.claude/agents/engineering-platform-engineer.md +270 -0
- package/.claude/agents/engineering-privacy-engineer.md +152 -0
- package/.claude/agents/engineering-prompt-engineer.md +209 -0
- package/.claude/agents/engineering-rag-pipeline-engineer.md +450 -0
- package/.claude/agents/engineering-rapid-prototyper.md +466 -0
- package/.claude/agents/engineering-realtime-collaboration-engineer.md +200 -0
- package/.claude/agents/engineering-rust-refactoring-specialist.md +313 -0
- package/.claude/agents/engineering-search-relevance-engineer.md +237 -0
- package/.claude/agents/engineering-section-508-specialist.md +339 -0
- package/.claude/agents/engineering-senior-developer.md +176 -0
- package/.claude/agents/engineering-servicenow-developer-mentor.md +180 -0
- package/.claude/agents/engineering-software-architect.md +112 -0
- package/.claude/agents/engineering-solidity-smart-contract-engineer.md +522 -0
- package/.claude/agents/engineering-sre.md +90 -0
- package/.claude/agents/engineering-technical-writer.md +393 -0
- package/.claude/agents/engineering-universal-document-compiler.md +370 -0
- package/.claude/agents/engineering-uswds-developer.md +340 -0
- package/.claude/agents/engineering-video-streaming-engineer.md +151 -0
- package/.claude/agents/engineering-voice-ai-integration-engineer.md +584 -0
- package/.claude/agents/engineering-webassembly-engineer.md +153 -0
- package/.claude/agents/engineering-wechat-mini-program-developer.md +356 -0
- package/.claude/agents/engineering-wordpress-performance.md +346 -0
- package/.claude/agents/engineering-wordpress-shopping-cart.md +346 -0
- package/.claude/agents/esg-sustainability-officer.md +396 -0
- package/.claude/agents/finance-bookkeeper-controller.md +260 -0
- package/.claude/agents/finance-financial-analyst.md +234 -0
- package/.claude/agents/finance-fpa-analyst.md +263 -0
- package/.claude/agents/finance-investment-researcher.md +272 -0
- package/.claude/agents/finance-tax-strategist.md +239 -0
- package/.claude/agents/game-audio-engineer.md +264 -0
- package/.claude/agents/game-designer.md +167 -0
- package/.claude/agents/gis-3d-scene-developer.md +111 -0
- package/.claude/agents/gis-analyst.md +91 -0
- package/.claude/agents/gis-bim-specialist.md +108 -0
- package/.claude/agents/gis-cartography-designer.md +150 -0
- package/.claude/agents/gis-drone-reality-mapping.md +120 -0
- package/.claude/agents/gis-geoai-ml-engineer.md +105 -0
- package/.claude/agents/gis-geoprocessing-specialist.md +97 -0
- package/.claude/agents/gis-qa-engineer.md +133 -0
- package/.claude/agents/gis-solution-engineer.md +101 -0
- package/.claude/agents/gis-spatial-data-engineer.md +97 -0
- package/.claude/agents/gis-spatial-data-scientist.md +111 -0
- package/.claude/agents/gis-technical-consultant.md +86 -0
- package/.claude/agents/gis-web-gis-developer.md +108 -0
- package/.claude/agents/godot-gameplay-scripter.md +338 -0
- package/.claude/agents/godot-multiplayer-engineer.md +301 -0
- package/.claude/agents/godot-shader-developer.md +267 -0
- package/.claude/agents/government-digital-presales-consultant.md +363 -0
- package/.claude/agents/grant-writer.md +511 -0
- package/.claude/agents/healthcare-aging-parent-care-companion.md +414 -0
- package/.claude/agents/healthcare-clinical-evidence-agent.md +231 -0
- package/.claude/agents/healthcare-customer-service.md +389 -0
- package/.claude/agents/healthcare-innovation-strategist.md +433 -0
- package/.claude/agents/healthcare-marketing-compliance.md +395 -0
- package/.claude/agents/healthcare-sovereign-health-systems-agent.md +312 -0
- package/.claude/agents/hospitality-guest-services.md +603 -0
- package/.claude/agents/hr-onboarding.md +451 -0
- package/.claude/agents/identity-graph-operator.md +303 -0
- package/.claude/agents/language-translator.md +264 -0
- package/.claude/agents/legal-billing-time-tracking.md +569 -0
- package/.claude/agents/legal-client-intake.md +492 -0
- package/.claude/agents/legal-document-review.md +454 -0
- package/.claude/agents/level-designer.md +208 -0
- package/.claude/agents/loan-officer-assistant.md +555 -0
- package/.claude/agents/lsp-index-engineer.md +320 -0
- package/.claude/agents/ma-integration-manager.md +427 -0
- package/.claude/agents/macos-spatial-metal-engineer.md +337 -0
- package/.claude/agents/marketing-aeo-foundations.md +264 -0
- package/.claude/agents/marketing-agentic-search-optimizer.md +279 -0
- package/.claude/agents/marketing-ai-citation-strategist.md +172 -0
- package/.claude/agents/marketing-app-store-optimizer.md +321 -0
- package/.claude/agents/marketing-baidu-seo-specialist.md +226 -0
- package/.claude/agents/marketing-bilibili-content-strategist.md +199 -0
- package/.claude/agents/marketing-book-co-author.md +110 -0
- package/.claude/agents/marketing-carousel-growth-engine.md +204 -0
- package/.claude/agents/marketing-china-ecommerce-operator.md +283 -0
- package/.claude/agents/marketing-china-market-localization-strategist.md +283 -0
- package/.claude/agents/marketing-content-creator.md +54 -0
- package/.claude/agents/marketing-cross-border-ecommerce.md +259 -0
- package/.claude/agents/marketing-developer-community-builder.md +285 -0
- package/.claude/agents/marketing-douyin-strategist.md +149 -0
- package/.claude/agents/marketing-email-strategist.md +249 -0
- package/.claude/agents/marketing-global-podcast-strategist.md +206 -0
- package/.claude/agents/marketing-growth-hacker.md +54 -0
- package/.claude/agents/marketing-instagram-curator.md +113 -0
- package/.claude/agents/marketing-kuaishou-strategist.md +223 -0
- package/.claude/agents/marketing-linkedin-content-creator.md +214 -0
- package/.claude/agents/marketing-livestream-commerce-coach.md +305 -0
- package/.claude/agents/marketing-multi-platform-publisher.md +217 -0
- package/.claude/agents/marketing-podcast-strategist.md +277 -0
- package/.claude/agents/marketing-pr-communications-manager.md +473 -0
- package/.claude/agents/marketing-private-domain-operator.md +308 -0
- package/.claude/agents/marketing-reddit-community-builder.md +123 -0
- package/.claude/agents/marketing-seo-specialist.md +370 -0
- package/.claude/agents/marketing-short-video-editing-coach.md +412 -0
- package/.claude/agents/marketing-social-media-strategist.md +125 -0
- package/.claude/agents/marketing-tiktok-strategist.md +125 -0
- package/.claude/agents/marketing-twitter-engager.md +126 -0
- package/.claude/agents/marketing-video-optimization-specialist.md +119 -0
- package/.claude/agents/marketing-wechat-official-account.md +145 -0
- package/.claude/agents/marketing-weibo-strategist.md +240 -0
- package/.claude/agents/marketing-x-twitter-intelligence-analyst.md +161 -0
- package/.claude/agents/marketing-xiaohongshu-specialist.md +138 -0
- package/.claude/agents/marketing-zhihu-strategist.md +162 -0
- package/.claude/agents/medical-billing-coding-specialist.md +491 -0
- package/.claude/agents/narrative-designer.md +243 -0
- package/.claude/agents/operations-manager.md +399 -0
- package/.claude/agents/organizational-psychologist.md +391 -0
- package/.claude/agents/paid-media-auditor.md +71 -0
- package/.claude/agents/paid-media-creative-strategist.md +71 -0
- package/.claude/agents/paid-media-paid-social-strategist.md +71 -0
- package/.claude/agents/paid-media-ppc-strategist.md +71 -0
- package/.claude/agents/paid-media-programmatic-buyer.md +71 -0
- package/.claude/agents/paid-media-search-query-analyst.md +71 -0
- package/.claude/agents/paid-media-tracking-specialist.md +71 -0
- package/.claude/agents/personal-growth-mentor.md +159 -0
- package/.claude/agents/product-behavioral-nudge-engine.md +83 -0
- package/.claude/agents/product-dx-engineer.md +332 -0
- package/.claude/agents/product-feedback-synthesizer.md +119 -0
- package/.claude/agents/product-manager.md +469 -0
- package/.claude/agents/product-sprint-prioritizer.md +154 -0
- package/.claude/agents/product-trend-researcher.md +159 -0
- package/.claude/agents/project-management-experiment-tracker.md +198 -0
- package/.claude/agents/project-management-jira-workflow-steward.md +233 -0
- package/.claude/agents/project-management-meeting-notes-specialist.md +95 -0
- package/.claude/agents/project-management-project-shepherd.md +194 -0
- package/.claude/agents/project-management-studio-operations.md +200 -0
- package/.claude/agents/project-management-studio-producer.md +203 -0
- package/.claude/agents/project-manager-senior.md +135 -0
- package/.claude/agents/real-estate-buyer-seller.md +596 -0
- package/.claude/agents/recruitment-specialist.md +515 -0
- package/.claude/agents/report-distribution-agent.md +65 -0
- package/.claude/agents/research-synthesist.md +136 -0
- package/.claude/agents/resume-tailor.md +230 -0
- package/.claude/agents/retail-customer-returns.md +566 -0
- package/.claude/agents/roblox-avatar-creator.md +297 -0
- package/.claude/agents/roblox-experience-designer.md +306 -0
- package/.claude/agents/roblox-systems-scripter.md +332 -0
- package/.claude/agents/sales-account-strategist.md +227 -0
- package/.claude/agents/sales-coach.md +271 -0
- package/.claude/agents/sales-data-extraction-agent.md +67 -0
- package/.claude/agents/sales-deal-strategist.md +180 -0
- package/.claude/agents/sales-discovery-coach.md +225 -0
- package/.claude/agents/sales-engineer.md +182 -0
- package/.claude/agents/sales-offer-lead-gen-strategist.md +257 -0
- package/.claude/agents/sales-outbound-strategist.md +201 -0
- package/.claude/agents/sales-outreach.md +425 -0
- package/.claude/agents/sales-pipeline-analyst.md +267 -0
- package/.claude/agents/sales-proposal-strategist.md +217 -0
- package/.claude/agents/security-ai-generated-code-auditor.md +207 -0
- package/.claude/agents/security-appsec-engineer.md +536 -0
- package/.claude/agents/security-architect.md +307 -0
- package/.claude/agents/security-blockchain-security-auditor.md +482 -0
- package/.claude/agents/security-cloud-security-architect.md +576 -0
- package/.claude/agents/security-compliance-auditor.md +158 -0
- package/.claude/agents/security-incident-responder.md +450 -0
- package/.claude/agents/security-penetration-tester.md +399 -0
- package/.claude/agents/security-secrets-credential-engineer.md +189 -0
- package/.claude/agents/security-senior-secops.md +760 -0
- package/.claude/agents/security-threat-detection-engineer.md +553 -0
- package/.claude/agents/security-threat-intelligence-analyst.md +661 -0
- package/.claude/agents/specialized-chief-of-staff.md +279 -0
- package/.claude/agents/specialized-civil-engineer.md +356 -0
- package/.claude/agents/specialized-codebase-archaeologist.md +341 -0
- package/.claude/agents/specialized-cultural-intelligence-strategist.md +88 -0
- package/.claude/agents/specialized-developer-advocate.md +317 -0
- package/.claude/agents/specialized-document-generator.md +55 -0
- package/.claude/agents/specialized-fedramp-rmf-compliance.md +378 -0
- package/.claude/agents/specialized-focus-music-architect.md +175 -0
- package/.claude/agents/specialized-french-consulting-market.md +194 -0
- package/.claude/agents/specialized-korean-business-navigator.md +216 -0
- package/.claude/agents/specialized-master-plan-architect.md +157 -0
- package/.claude/agents/specialized-mcp-builder.md +260 -0
- package/.claude/agents/specialized-model-qa.md +528 -0
- package/.claude/agents/specialized-pricing-analyst.md +243 -0
- package/.claude/agents/specialized-salesforce-architect.md +182 -0
- package/.claude/agents/specialized-strategy-duel-agent.md +130 -0
- package/.claude/agents/specialized-workflow-architect.md +597 -0
- package/.claude/agents/study-abroad-advisor.md +282 -0
- package/.claude/agents/supply-chain-strategist.md +591 -0
- package/.claude/agents/support-analytics-reporter.md +387 -0
- package/.claude/agents/support-executive-summary-generator.md +212 -0
- package/.claude/agents/support-finance-tracker.md +480 -0
- package/.claude/agents/support-infrastructure-maintainer.md +628 -0
- package/.claude/agents/support-legal-compliance-checker.md +591 -0
- package/.claude/agents/support-support-responder.md +607 -0
- package/.claude/agents/technical-artist.md +233 -0
- package/.claude/agents/terminal-integration-specialist.md +70 -0
- package/.claude/agents/testing-accessibility-auditor.md +327 -0
- package/.claude/agents/testing-api-tester.md +334 -0
- package/.claude/agents/testing-evidence-collector.md +210 -0
- package/.claude/agents/testing-performance-benchmarker.md +283 -0
- package/.claude/agents/testing-reality-checker.md +249 -0
- package/.claude/agents/testing-test-automation-engineer.md +196 -0
- package/.claude/agents/testing-test-results-analyzer.md +330 -0
- package/.claude/agents/testing-tool-evaluator.md +416 -0
- package/.claude/agents/testing-workflow-optimizer.md +463 -0
- package/.claude/agents/unity-architect.md +271 -0
- package/.claude/agents/unity-editor-tool-developer.md +310 -0
- package/.claude/agents/unity-multiplayer-engineer.md +323 -0
- package/.claude/agents/unity-shader-graph-artist.md +269 -0
- package/.claude/agents/unreal-multiplayer-architect.md +364 -0
- package/.claude/agents/unreal-systems-engineer.md +322 -0
- package/.claude/agents/unreal-technical-artist.md +256 -0
- package/.claude/agents/unreal-world-builder.md +273 -0
- package/.claude/agents/visionos-spatial-engineer.md +54 -0
- package/.claude/agents/xr-cockpit-interaction-specialist.md +32 -0
- package/.claude/agents/xr-immersive-developer.md +32 -0
- package/.claude/agents/xr-interface-architect.md +32 -0
- package/.claude/agents/zk-steward.md +211 -0
- package/.claude-plugin/marketplace.json +30 -0
- package/.commitlintrc.json +26 -0
- package/.devcontainer/Dockerfile +24 -0
- package/.devcontainer/devcontainer.json +70 -0
- package/.devcontainer/docker-compose.memory.yml +42 -0
- package/.devcontainer/memory-stack/devcontainer.json +84 -0
- package/.devcontainer/post-create.sh +83 -0
- package/.dockerignore +23 -0
- package/.env.act.example +15 -0
- package/.env.example +171 -0
- package/.eslintrc.json +16 -0
- package/.git-blame-ignore-revs +10 -0
- package/.gitattributes +2 -0
- package/.gitguardian.yaml +50 -0
- package/.githooks/pre-commit +6 -0
- package/.pre-commit-config.yaml +30 -0
- package/.release-please-config-xithead.json +22 -0
- package/.release-please-config.json +44 -0
- package/.release-please-manifest-xithead.json +3 -0
- package/.release-please-manifest.json +3 -0
- package/CODE_OF_CONDUCT.md +133 -0
- package/CONTRIBUTING.md +133 -0
- package/Cargo.lock +5557 -0
- package/Cargo.toml +121 -0
- package/Dockerfile +38 -0
- package/Dockerfile.proxy +48 -0
- package/ENTERPRISE.md +5 -0
- package/FEATURES_ADVANCED.md +311 -0
- package/Headroom-2.gif +0 -0
- package/HeadroomDemo-Fast.gif +0 -0
- package/MANIFEST.in +4 -0
- package/MODERNIZATION_GUIDE.md +358 -0
- package/Makefile +140 -0
- package/NOTICE +43 -0
- package/NPM_TOKEN_SETUP.md +81 -0
- package/PR.md +170 -0
- package/PUBLISH.md +77 -0
- package/README.md +58 -137
- package/README_FRONTEND.md +196 -0
- package/REALIGNMENT/00-overview.md +53 -0
- package/REALIGNMENT/01-bug-list.md +421 -0
- package/REALIGNMENT/02-architecture.md +322 -0
- package/REALIGNMENT/03-phase-A-lockdown.md +420 -0
- package/REALIGNMENT/04-phase-B-live-zone.md +470 -0
- package/REALIGNMENT/05-phase-C-rust-proxy.md +329 -0
- package/REALIGNMENT/06-phase-D-bedrock-vertex.md +222 -0
- package/REALIGNMENT/07-phase-E-cache-stabilization.md +338 -0
- package/REALIGNMENT/08-phase-F-auth-mode.md +254 -0
- package/REALIGNMENT/09-phase-G-rtk-observability.md +194 -0
- package/REALIGNMENT/10-phase-H-python-retirement.md +214 -0
- package/REALIGNMENT/11-phase-I-test-infra.md +385 -0
- package/REALIGNMENT/12-decisions-needed.md +196 -0
- package/REALIGNMENT/INDEX.md +82 -0
- package/REFACTORING_CAMPAIGN.md +277 -0
- package/RESTRUCTURE_PLAN.md +100 -0
- package/RUST_DEV.md +361 -0
- package/SECURITY.md +65 -0
- package/SETUP_AUTO_PUBLISH.md +164 -0
- package/TESTING-copilot-subscription.md +153 -0
- package/app/api/config/route.ts +22 -0
- package/app/api/export/route.ts +83 -0
- package/app/api/models/route.ts +12 -0
- package/app/api/stats/route.ts +46 -0
- package/app/dashboard/page.tsx +445 -0
- package/app/globals.css +35 -0
- package/app/installation/page.tsx +293 -0
- package/app/layout.tsx +35 -0
- package/app/master-design-system.css +2076 -0
- package/app/master-design-system.source.css +8 -0
- package/app/page.tsx +187 -0
- package/app/rechtliches/agb/page.tsx +165 -0
- package/app/rechtliches/datenschutz/page.tsx +128 -0
- package/app/rechtliches/impressum/page.tsx +122 -0
- package/app/rechtliches/widerruf/page.tsx +185 -0
- package/bin/xithead.js +264 -0
- package/codecov.yml +34 -0
- package/components/layout/AppShell.tsx +25 -0
- package/components/layout/Footer.tsx +120 -0
- package/components/layout/Header.tsx +145 -0
- package/components/marketing/LegalHighlights.tsx +45 -0
- package/components/marketing/LegalPageShell.tsx +49 -0
- package/components/marketing/LegalPanel.tsx +41 -0
- package/components/marketing/index.ts +3 -0
- package/crates/headroom-core/Cargo.toml +165 -0
- package/crates/headroom-core/benches/auth_mode.rs +74 -0
- package/crates/headroom-core/benches/ccr_store.rs +224 -0
- package/crates/headroom-core/benches/tokenizer.rs +52 -0
- package/crates/headroom-core/config/pipeline.toml +151 -0
- package/crates/headroom-core/proptest-regressions/transforms/tag_protector.txt +9 -0
- package/crates/headroom-core/src/auth_mode.rs +250 -0
- package/crates/headroom-core/src/cache_control.rs +414 -0
- package/crates/headroom-core/src/ccr/backends/in_memory.rs +317 -0
- package/crates/headroom-core/src/ccr/backends/mod.rs +152 -0
- package/crates/headroom-core/src/ccr/backends/redis.rs +146 -0
- package/crates/headroom-core/src/ccr/backends/sqlite.rs +205 -0
- package/crates/headroom-core/src/ccr/mod.rs +119 -0
- package/crates/headroom-core/src/compression_policy.rs +505 -0
- package/crates/headroom-core/src/lib.rs +33 -0
- package/crates/headroom-core/src/relevance/base.rs +139 -0
- package/crates/headroom-core/src/relevance/bm25.rs +385 -0
- package/crates/headroom-core/src/relevance/embedding.rs +354 -0
- package/crates/headroom-core/src/relevance/hybrid.rs +364 -0
- package/crates/headroom-core/src/relevance/mod.rs +64 -0
- package/crates/headroom-core/src/signals/keyword_detector.rs +433 -0
- package/crates/headroom-core/src/signals/line_importance.rs +84 -0
- package/crates/headroom-core/src/signals/mod.rs +61 -0
- package/crates/headroom-core/src/signals/tiered.rs +141 -0
- package/crates/headroom-core/src/tokenizer/estimator.rs +152 -0
- package/crates/headroom-core/src/tokenizer/hf_impl.rs +314 -0
- package/crates/headroom-core/src/tokenizer/mod.rs +56 -0
- package/crates/headroom-core/src/tokenizer/registry.rs +342 -0
- package/crates/headroom-core/src/tokenizer/tiktoken_impl.rs +265 -0
- package/crates/headroom-core/src/transforms/adaptive_sizer.rs +610 -0
- package/crates/headroom-core/src/transforms/anchor_selector.rs +1189 -0
- package/crates/headroom-core/src/transforms/content_detector.rs +769 -0
- package/crates/headroom-core/src/transforms/detection.rs +222 -0
- package/crates/headroom-core/src/transforms/diff_compressor.rs +1685 -0
- package/crates/headroom-core/src/transforms/live_zone.rs +2967 -0
- package/crates/headroom-core/src/transforms/log_compressor.rs +1295 -0
- package/crates/headroom-core/src/transforms/magika_detector.rs +395 -0
- package/crates/headroom-core/src/transforms/mod.rs +64 -0
- package/crates/headroom-core/src/transforms/pipeline/config.rs +331 -0
- package/crates/headroom-core/src/transforms/pipeline/mod.rs +103 -0
- package/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rs +524 -0
- package/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs +279 -0
- package/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs +400 -0
- package/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rs +284 -0
- package/crates/headroom-core/src/transforms/pipeline/offloads/mod.rs +34 -0
- package/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rs +331 -0
- package/crates/headroom-core/src/transforms/pipeline/orchestrator.rs +848 -0
- package/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rs +175 -0
- package/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rs +545 -0
- package/crates/headroom-core/src/transforms/pipeline/reformats/mod.rs +12 -0
- package/crates/headroom-core/src/transforms/pipeline/traits.rs +338 -0
- package/crates/headroom-core/src/transforms/recommendations.rs +329 -0
- package/crates/headroom-core/src/transforms/safety.rs +215 -0
- package/crates/headroom-core/src/transforms/search_compressor.rs +902 -0
- package/crates/headroom-core/src/transforms/smart_crusher/analyzer.rs +1262 -0
- package/crates/headroom-core/src/transforms/smart_crusher/anchors.rs +407 -0
- package/crates/headroom-core/src/transforms/smart_crusher/builder.rs +263 -0
- package/crates/headroom-core/src/transforms/smart_crusher/classifier.rs +214 -0
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/classifier.rs +283 -0
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/compactor.rs +733 -0
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rs +961 -0
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/ir.rs +252 -0
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs +127 -0
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs +397 -0
- package/crates/headroom-core/src/transforms/smart_crusher/config.rs +173 -0
- package/crates/headroom-core/src/transforms/smart_crusher/constraints.rs +159 -0
- package/crates/headroom-core/src/transforms/smart_crusher/crusher.rs +1665 -0
- package/crates/headroom-core/src/transforms/smart_crusher/crushers.rs +865 -0
- package/crates/headroom-core/src/transforms/smart_crusher/error_keywords.rs +77 -0
- package/crates/headroom-core/src/transforms/smart_crusher/field_detect.rs +393 -0
- package/crates/headroom-core/src/transforms/smart_crusher/hashing.rs +72 -0
- package/crates/headroom-core/src/transforms/smart_crusher/mod.rs +81 -0
- package/crates/headroom-core/src/transforms/smart_crusher/observer.rs +66 -0
- package/crates/headroom-core/src/transforms/smart_crusher/orchestration.rs +459 -0
- package/crates/headroom-core/src/transforms/smart_crusher/outliers.rs +507 -0
- package/crates/headroom-core/src/transforms/smart_crusher/planning.rs +904 -0
- package/crates/headroom-core/src/transforms/smart_crusher/statistics.rs +510 -0
- package/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs +287 -0
- package/crates/headroom-core/src/transforms/smart_crusher/traits.rs +203 -0
- package/crates/headroom-core/src/transforms/smart_crusher/types.rs +263 -0
- package/crates/headroom-core/src/transforms/tag_protector.rs +1272 -0
- package/crates/headroom-core/src/transforms/unidiff_detector.rs +232 -0
- package/crates/headroom-parity/Cargo.toml +26 -0
- package/crates/headroom-parity/examples/diff_fixture.rs +66 -0
- package/crates/headroom-parity/src/bin/parity_run.rs +78 -0
- package/crates/headroom-parity/src/lib.rs +613 -0
- package/crates/headroom-proxy/Cargo.toml +119 -0
- package/crates/headroom-proxy/data/model_prices_and_context_window.json +39767 -0
- package/crates/headroom-proxy/src/bedrock/auth_mode_layer.rs +192 -0
- package/crates/headroom-proxy/src/bedrock/envelope.rs +239 -0
- package/crates/headroom-proxy/src/bedrock/eventstream.rs +769 -0
- package/crates/headroom-proxy/src/bedrock/eventstream_to_sse.rs +459 -0
- package/crates/headroom-proxy/src/bedrock/invoke.rs +735 -0
- package/crates/headroom-proxy/src/bedrock/invoke_streaming.rs +1098 -0
- package/crates/headroom-proxy/src/bedrock/mod.rs +66 -0
- package/crates/headroom-proxy/src/bedrock/sigv4.rs +324 -0
- package/crates/headroom-proxy/src/bedrock/vendor.rs +82 -0
- package/crates/headroom-proxy/src/cache_stabilization/anthropic_cache_control.rs +698 -0
- package/crates/headroom-proxy/src/cache_stabilization/drift_detector.rs +683 -0
- package/crates/headroom-proxy/src/cache_stabilization/mod.rs +61 -0
- package/crates/headroom-proxy/src/cache_stabilization/openai_cache_key.rs +600 -0
- package/crates/headroom-proxy/src/cache_stabilization/tool_def_normalize.rs +514 -0
- package/crates/headroom-proxy/src/cache_stabilization/volatile_detector.rs +720 -0
- package/crates/headroom-proxy/src/compression/anthropic.rs +92 -0
- package/crates/headroom-proxy/src/compression/live_zone_anthropic.rs +1311 -0
- package/crates/headroom-proxy/src/compression/live_zone_openai.rs +660 -0
- package/crates/headroom-proxy/src/compression/live_zone_responses.rs +678 -0
- package/crates/headroom-proxy/src/compression/mod.rs +127 -0
- package/crates/headroom-proxy/src/compression/model_limits.rs +211 -0
- package/crates/headroom-proxy/src/config.rs +646 -0
- package/crates/headroom-proxy/src/error.rs +69 -0
- package/crates/headroom-proxy/src/handlers/chat_completions.rs +100 -0
- package/crates/headroom-proxy/src/handlers/conversations.rs +241 -0
- package/crates/headroom-proxy/src/handlers/mod.rs +13 -0
- package/crates/headroom-proxy/src/handlers/responses.rs +256 -0
- package/crates/headroom-proxy/src/headers.rs +288 -0
- package/crates/headroom-proxy/src/health.rs +41 -0
- package/crates/headroom-proxy/src/lib.rs +21 -0
- package/crates/headroom-proxy/src/main.rs +141 -0
- package/crates/headroom-proxy/src/observability/cache_hit_rate.rs +340 -0
- package/crates/headroom-proxy/src/observability/compression_ratio.rs +158 -0
- package/crates/headroom-proxy/src/observability/metric_names.rs +174 -0
- package/crates/headroom-proxy/src/observability/mod.rs +68 -0
- package/crates/headroom-proxy/src/observability/prometheus.rs +394 -0
- package/crates/headroom-proxy/src/observability/proxy_metrics.rs +368 -0
- package/crates/headroom-proxy/src/proxy.rs +1619 -0
- package/crates/headroom-proxy/src/responses_items.rs +565 -0
- package/crates/headroom-proxy/src/sse/anthropic.rs +415 -0
- package/crates/headroom-proxy/src/sse/framing.rs +366 -0
- package/crates/headroom-proxy/src/sse/mod.rs +40 -0
- package/crates/headroom-proxy/src/sse/openai_chat.rs +231 -0
- package/crates/headroom-proxy/src/sse/openai_responses.rs +458 -0
- package/crates/headroom-proxy/src/vertex/adc.rs +315 -0
- package/crates/headroom-proxy/src/vertex/envelope.rs +167 -0
- package/crates/headroom-proxy/src/vertex/mod.rs +277 -0
- package/crates/headroom-proxy/src/vertex/raw_predict.rs +490 -0
- package/crates/headroom-proxy/src/vertex/stream_raw_predict.rs +49 -0
- package/crates/headroom-proxy/src/websocket.rs +246 -0
- package/crates/headroom-py/Cargo.toml +37 -0
- package/crates/headroom-py/build.rs +77 -0
- package/crates/headroom-py/glibc_compat.c +170 -0
- package/crates/headroom-py/src/lib.rs +1655 -0
- package/deny.toml +32 -0
- package/design-system/base.css +127 -0
- package/design-system/components.css +1943 -0
- package/design-system/fonts.css +17 -0
- package/design-system/tailwind-preset.js +95 -0
- package/design-system/tokens.css +48 -0
- package/design-system/utilities.css +53 -0
- package/docker/differential-network-capture/Dockerfile.runner +14 -0
- package/docker/differential-network-capture/docker-compose.yml +142 -0
- package/docker/differential-network-capture/run-claude-lane.sh +24 -0
- package/docker/docker-compose.native.yml +44 -0
- package/docker-bake.hcl +88 -0
- package/docker-compose.yml +80 -0
- package/e2e/_lib/make_shim.ps1 +24 -0
- package/e2e/_lib/make_shim.sh +28 -0
- package/e2e/docker-native-install.sh +120 -0
- package/e2e/init/Dockerfile +79 -0
- package/e2e/wrap/Dockerfile +115 -0
- package/examples/07-context-compression.ipynb +279 -0
- package/examples/deployment/macos-launchagent/com.headroom.proxy.plist.template +65 -0
- package/examples/deployment/macos-launchagent/install.sh +227 -0
- package/examples/deployment/macos-launchagent/shell-integration.sh +74 -0
- package/examples/deployment/macos-launchagent/uninstall.sh +145 -0
- package/headroom-savings.png +0 -0
- package/headroom_learn.gif +0 -0
- package/lib/config/index.ts +12 -0
- package/lib/hooks/index.ts +4 -0
- package/lib/hooks/use-mounted.ts +13 -0
- package/lib/hooks/use-session.ts +34 -0
- package/lib/hooks/use-settings.ts +49 -0
- package/lib/utils/index.ts +1 -0
- package/lib/utils/settings.ts +162 -0
- package/lib/utils/utils.ts +6 -0
- package/llms.txt +67 -0
- package/middleware.ts +26 -0
- package/mkdocs.yml +136 -0
- package/next-env.d.ts +7 -0
- package/next.config.ts +12 -0
- package/npm/bin/xithead.js +2 -2
- package/npm/bin/xtream-head.js +366 -0
- package/npm/index.js +18 -91
- package/npm/package.json +55 -0
- package/package.json +47 -42
- package/plugins/headroom-agent-hooks/.claude-plugin/plugin.json +17 -0
- package/plugins/headroom-agent-hooks/hooks/hooks.json +29 -0
- package/plugins/headroom-oauth2/LICENSE +3 -0
- package/plugins/headroom-oauth2/SPEC.md +115 -0
- package/plugins/headroom-oauth2/pyproject.toml +48 -0
- package/plugins/hermes/headroom_retrieve/plugin.yaml +6 -0
- package/plugins/openclaw/hook-shim/HOOK.md +21 -0
- package/plugins/openclaw/hook-shim/handler.js +3 -0
- package/plugins/openclaw/openclaw.plugin.json +98 -0
- package/plugins/openclaw/package-lock.json +2692 -0
- package/plugins/openclaw/package.json +54 -0
- package/plugins/openclaw/prepare-dist.mjs +56 -0
- package/plugins/openclaw/src/convert.ts +418 -0
- package/plugins/openclaw/src/engine.ts +284 -0
- package/plugins/openclaw/src/gateway-config.ts +143 -0
- package/plugins/openclaw/src/index.ts +11 -0
- package/plugins/openclaw/src/plugin/index.ts +98 -0
- package/plugins/openclaw/src/proxy-manager.ts +477 -0
- package/plugins/openclaw/src/tools/headroom-retrieve.ts +76 -0
- package/plugins/openclaw/test/convert.test.ts +96 -0
- package/plugins/openclaw/test/engine-normalization.test.ts +21 -0
- package/plugins/openclaw/test/engine.test.ts +101 -0
- package/plugins/openclaw/test/gateway-config.test.ts +278 -0
- package/plugins/openclaw/test/plugin-runtime-routing.test.ts +141 -0
- package/plugins/openclaw/test/proxy-manager.test.ts +343 -0
- package/plugins/openclaw/tsup.config.ts +9 -0
- package/postcss.config.mjs +7 -0
- package/pyproject.toml +438 -0
- package/rust-toolchain.toml +16 -0
- package/sdk/typescript/examples/basic-compress.ts +64 -0
- package/sdk/typescript/examples/ccr-retrieve.ts +80 -0
- package/sdk/typescript/examples/hooks-custom-compression.ts +117 -0
- package/sdk/typescript/examples/middleware-composition.ts +59 -0
- package/sdk/typescript/examples/multi-provider.ts +53 -0
- package/sdk/typescript/examples/openai-anthropic-adapters.ts +73 -0
- package/sdk/typescript/examples/shared-context-multi-agent.ts +74 -0
- package/sdk/typescript/examples/simulation-dry-run.ts +80 -0
- package/sdk/typescript/examples/streaming-chat.ts +71 -0
- package/sdk/typescript/examples/structured-output.ts +99 -0
- package/sdk/typescript/examples/tool-calling-agent.ts +86 -0
- package/sdk/typescript/examples/with-headroom-vercel.ts +57 -0
- package/sdk/typescript/package-lock.json +3437 -0
- package/sdk/typescript/package.json +108 -0
- package/sdk/typescript/src/adapters/anthropic.ts +218 -0
- package/sdk/typescript/src/adapters/gemini.ts +83 -0
- package/sdk/typescript/src/adapters/openai.ts +77 -0
- package/sdk/typescript/src/adapters/vercel-ai.ts +131 -0
- package/sdk/typescript/src/client.ts +632 -0
- package/sdk/typescript/src/compress.ts +87 -0
- package/sdk/typescript/src/errors.ts +114 -0
- package/sdk/typescript/src/hooks.ts +113 -0
- package/sdk/typescript/src/index.ts +140 -0
- package/sdk/typescript/src/paths.ts +272 -0
- package/sdk/typescript/src/shared-context.ts +188 -0
- package/sdk/typescript/src/simulate.ts +52 -0
- package/sdk/typescript/src/types/config.ts +183 -0
- package/sdk/typescript/src/types/models.ts +314 -0
- package/sdk/typescript/src/types.ts +139 -0
- package/sdk/typescript/src/utils/case.ts +34 -0
- package/sdk/typescript/src/utils/format.ts +480 -0
- package/sdk/typescript/src/utils/stream.ts +50 -0
- package/sdk/typescript/test/adapters/anthropic.test.ts +217 -0
- package/sdk/typescript/test/adapters/gemini.test.ts +131 -0
- package/sdk/typescript/test/adapters/openai.test.ts +174 -0
- package/sdk/typescript/test/adapters/vercel-ai.test.ts +306 -0
- package/sdk/typescript/test/client-expanded.test.ts +333 -0
- package/sdk/typescript/test/client.test.ts +266 -0
- package/sdk/typescript/test/compress.test.ts +118 -0
- package/sdk/typescript/test/config.test.ts +275 -0
- package/sdk/typescript/test/errors.test.ts +169 -0
- package/sdk/typescript/test/format-integration.test.ts +252 -0
- package/sdk/typescript/test/hooks.test.ts +265 -0
- package/sdk/typescript/test/integration.test.ts +249 -0
- package/sdk/typescript/test/mdx-examples.test.ts +253 -0
- package/sdk/typescript/test/models.test.ts +318 -0
- package/sdk/typescript/test/parity.test.ts +337 -0
- package/sdk/typescript/test/paths.test.ts +447 -0
- package/sdk/typescript/test/shared-context.test.ts +245 -0
- package/sdk/typescript/test/simulate.test.ts +84 -0
- package/sdk/typescript/test/types.test.ts +244 -0
- package/sdk/typescript/test/utils/case.test.ts +129 -0
- package/sdk/typescript/test/utils/detect-format.test.ts +133 -0
- package/sdk/typescript/test/utils/format.test.ts +434 -0
- package/sdk/typescript/test/vercel-ai-e2e.test.ts +179 -0
- package/sdk/typescript/tsup.config.ts +17 -0
- package/tailwind.master.config.cjs +47 -0
- package/types/api.ts +12 -0
- package/types/config.ts +9 -0
- package/types/index.ts +2 -0
- package/uv.lock +6951 -0
- package/xithead.egg-info/PKG-INFO +22 -0
- package/xithead.egg-info/SOURCES.txt +16 -0
- package/xithead.egg-info/dependency_links.txt +1 -0
- package/xithead.egg-info/entry_points.txt +2 -0
- package/xithead.egg-info/requires.txt +10 -0
- package/xithead.egg-info/top_level.txt +2 -0
- package/xithead.ps1 +136 -0
- package/xithead.sh +129 -0
- package/xithead_dist/pyproject.toml +39 -0
- package/xithead_dist/xithead.egg-info/PKG-INFO +22 -0
- package/xithead_dist/xithead.egg-info/SOURCES.txt +16 -0
- package/xithead_dist/xithead.egg-info/dependency_links.txt +1 -0
- package/xithead_dist/xithead.egg-info/entry_points.txt +3 -0
- package/xithead_dist/xithead.egg-info/requires.txt +10 -0
- package/xithead_dist/xithead.egg-info/top_level.txt +2 -0
|
@@ -0,0 +1,1665 @@
|
|
|
1
|
+
//! `SmartCrusher` struct — top-level entry point for compression.
|
|
2
|
+
//!
|
|
3
|
+
//! Owns the `config`, `anchor_selector`, `scorer`, and `analyzer`
|
|
4
|
+
//! singletons that every per-message call needs. Constructed once
|
|
5
|
+
//! per process; the struct is `Send + Sync` so it can sit behind an
|
|
6
|
+
//! `Arc` in a multi-threaded proxy.
|
|
7
|
+
//!
|
|
8
|
+
//! This module ports three Python entry points:
|
|
9
|
+
//!
|
|
10
|
+
//! - `_execute_plan` (line 3617) → `SmartCrusher::execute_plan`
|
|
11
|
+
//! - `_crush_array` (line 2400) → `SmartCrusher::crush_array`
|
|
12
|
+
//! - `_crush_mixed_array` (line 2914) → `SmartCrusher::crush_mixed_array`
|
|
13
|
+
//!
|
|
14
|
+
//! # Stubs that match Python's "everything-disabled" path
|
|
15
|
+
//!
|
|
16
|
+
//! Python's `_crush_array` calls into TOIN (cross-user pattern
|
|
17
|
+
//! learning), feedback (per-tool compression hints), CCR (compress-
|
|
18
|
+
//! cache-retrieve store), and telemetry. All four are large separate
|
|
19
|
+
//! systems with their own state. For the like-for-like port at Stage
|
|
20
|
+
//! 3c.1, we mirror Python's behavior **when those subsystems are
|
|
21
|
+
//! disabled**:
|
|
22
|
+
//!
|
|
23
|
+
//! - **TOIN**: never produces a recommendation, never overrides
|
|
24
|
+
//! `effective_max_items`, never injects preserve_fields/strategy/level.
|
|
25
|
+
//! - **Feedback**: never produces hints; default `effective_max_items`.
|
|
26
|
+
//! - **CCR**: `enabled=false`; result has `ccr_hash = None`.
|
|
27
|
+
//! - **Telemetry**: no-op.
|
|
28
|
+
//! - **`_compress_text_within_items`**: pass-through (returns input
|
|
29
|
+
//! unchanged) since text compression has its own port pipeline.
|
|
30
|
+
//! - **`summarize_dropped_items`**: empty string.
|
|
31
|
+
//!
|
|
32
|
+
//! Parity fixtures will be recorded with all four disabled on the
|
|
33
|
+
//! Python side, locking byte-equal output. The TOIN/CCR/feedback
|
|
34
|
+
//! integration ports happen later (Stage 3c.2 follow-ups).
|
|
35
|
+
|
|
36
|
+
use std::sync::Arc;
|
|
37
|
+
|
|
38
|
+
use serde_json::Value;
|
|
39
|
+
|
|
40
|
+
use super::analyzer::SmartAnalyzer;
|
|
41
|
+
use super::builder::SmartCrusherBuilder;
|
|
42
|
+
use super::classifier::{classify_array, ArrayType};
|
|
43
|
+
use super::compaction::{
|
|
44
|
+
classify_cell, emit_opaque_ccr_marker, try_parse_json_container, CellClass, ClassifyConfig,
|
|
45
|
+
CompactConfig, Compaction, CompactionStage,
|
|
46
|
+
};
|
|
47
|
+
use super::config::SmartCrusherConfig;
|
|
48
|
+
use super::crushers::{compute_k_split, crush_number_array, crush_object, crush_string_array};
|
|
49
|
+
use super::planning::SmartCrusherPlanner;
|
|
50
|
+
use super::traits::{Constraint, CrushEvent, Observer};
|
|
51
|
+
use super::types::{CompressionPlan, CompressionStrategy, CrushResult};
|
|
52
|
+
use crate::ccr::CcrStore;
|
|
53
|
+
use crate::relevance::RelevanceScorer;
|
|
54
|
+
use crate::transforms::adaptive_sizer::compute_optimal_k;
|
|
55
|
+
use crate::transforms::anchor_selector::AnchorSelector;
|
|
56
|
+
|
|
57
|
+
/// Return type for `crush_array`.
|
|
58
|
+
///
|
|
59
|
+
/// Two operating paths feed the same result type:
|
|
60
|
+
///
|
|
61
|
+
/// - **Lossless path** — input compacted to a smaller inline form
|
|
62
|
+
/// (e.g. CSV+schema). Nothing dropped; `compacted` is populated;
|
|
63
|
+
/// `ccr_hash` is `None` (no retrieval needed because everything is
|
|
64
|
+
/// already in the prompt).
|
|
65
|
+
/// - **Lossy path** — input compressed by row-dropping. `items` holds
|
|
66
|
+
/// the kept subset; `ccr_hash` is `Some(hash)` so the runtime can
|
|
67
|
+
/// cache the **full original** keyed by that hash and serve it back
|
|
68
|
+
/// to the LLM via a retrieval tool call. **No data is lost** —
|
|
69
|
+
/// "lossy" here means "compressed view inline; full payload cached
|
|
70
|
+
/// for tool retrieval," matching Python's CCR-Dropped semantics.
|
|
71
|
+
///
|
|
72
|
+
/// The runtime (PyO3 bridge / proxy server) owns the cache; this crate
|
|
73
|
+
/// computes the hash and emits a marker so the prompt knows where to
|
|
74
|
+
/// look.
|
|
75
|
+
pub struct CrushArrayResult {
|
|
76
|
+
/// Kept items. For the lossless path this is the full original
|
|
77
|
+
/// (nothing was dropped). For the lossy path this is the surviving
|
|
78
|
+
/// subset; the rest is retrievable via `ccr_hash`.
|
|
79
|
+
pub items: Vec<Value>,
|
|
80
|
+
/// Strategy debug string. One of:
|
|
81
|
+
/// - `"none:adaptive_at_limit"` / `"skip:<reason>"` — passthrough
|
|
82
|
+
/// - `"lossless:table"` / `"lossless:buckets"` — lossless wins
|
|
83
|
+
/// - `"smart_sample"` / `"top_n"` / `"cluster"` / `"time_series"` —
|
|
84
|
+
/// lossy path with row-dropping.
|
|
85
|
+
pub strategy_info: String,
|
|
86
|
+
/// 12-char SHA-256 hex prefix of the **full original input**.
|
|
87
|
+
/// Populated when the lossy path dropped rows; the runtime is
|
|
88
|
+
/// expected to cache the original items keyed by this hash so a
|
|
89
|
+
/// retrieval tool can serve them back. `None` when nothing was
|
|
90
|
+
/// dropped (lossless path or below adaptive_k boundary).
|
|
91
|
+
pub ccr_hash: Option<String>,
|
|
92
|
+
/// Marker text inserted into the prompt to advertise the CCR
|
|
93
|
+
/// pointer (e.g. `<<ccr:abc123def456 42_rows_offloaded>>`). Empty
|
|
94
|
+
/// when `ccr_hash` is `None`.
|
|
95
|
+
pub dropped_summary: String,
|
|
96
|
+
/// Rendered bytes from the compaction stage when the **lossless
|
|
97
|
+
/// path** won. `None` for the lossy path or when compaction wasn't
|
|
98
|
+
/// configured.
|
|
99
|
+
pub compacted: Option<String>,
|
|
100
|
+
/// Top-level [`Compaction`] variant tag — `"table"`, `"buckets"`,
|
|
101
|
+
/// `"ccr"`. Mirrors `compacted` — populated only when lossless won.
|
|
102
|
+
pub compaction_kind: Option<&'static str>,
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/// Top-level SmartCrusher.
|
|
106
|
+
///
|
|
107
|
+
/// Three pluggable extensions (Stage 3c.2 PR1):
|
|
108
|
+
/// - `scorer` — relevance scoring (`HybridScorer` by default).
|
|
109
|
+
/// - `constraints` — must-keep predicates (`KeepErrorsConstraint` +
|
|
110
|
+
/// `KeepStructuralOutliersConstraint` by default).
|
|
111
|
+
/// - `observers` — decision-stream telemetry (`TracingObserver` by
|
|
112
|
+
/// default).
|
|
113
|
+
///
|
|
114
|
+
/// Compose via [`SmartCrusherBuilder`]; or call `SmartCrusher::new()`
|
|
115
|
+
/// for the OSS default composition.
|
|
116
|
+
pub struct SmartCrusher {
|
|
117
|
+
pub config: SmartCrusherConfig,
|
|
118
|
+
pub anchor_selector: AnchorSelector,
|
|
119
|
+
pub scorer: Box<dyn RelevanceScorer + Send + Sync>,
|
|
120
|
+
pub analyzer: SmartAnalyzer,
|
|
121
|
+
pub constraints: Vec<Box<dyn Constraint>>,
|
|
122
|
+
pub observers: Vec<Box<dyn Observer>>,
|
|
123
|
+
/// Optional lossless-first compaction stage (Stage 3c.2 PR2). When
|
|
124
|
+
/// set, `crush_array` runs compaction up front and short-circuits
|
|
125
|
+
/// the lossy path on success. When `None` (default OSS), parity
|
|
126
|
+
/// with the pre-PR2 lossy-only pipeline is preserved exactly.
|
|
127
|
+
pub compaction: Option<CompactionStage>,
|
|
128
|
+
/// Optional CCR store. When set, the lossy path stashes the **full
|
|
129
|
+
/// original** array into the store keyed by `ccr_hash` before
|
|
130
|
+
/// returning — the runtime can then serve dropped rows back via
|
|
131
|
+
/// retrieval tool calls. When `None`, hashes are still emitted but
|
|
132
|
+
/// nothing is stored (legacy / parity mode).
|
|
133
|
+
///
|
|
134
|
+
/// `Arc` so callers can keep their own handle to the same store
|
|
135
|
+
/// (e.g. the proxy server holds it for retrieval lookups while
|
|
136
|
+
/// SmartCrusher writes through it).
|
|
137
|
+
pub ccr_store: Option<Arc<dyn CcrStore>>,
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
impl SmartCrusher {
|
|
141
|
+
/// Construct with the OSS default composition: scorer + constraints +
|
|
142
|
+
/// observer + **lossless-first compaction stage**. Calling
|
|
143
|
+
/// `crush_array` runs the dispatch:
|
|
144
|
+
///
|
|
145
|
+
/// 1. Try the lossless compactor.
|
|
146
|
+
/// 2. If savings ratio ≥ `config.lossless_min_savings_ratio`
|
|
147
|
+
/// (default `0.30`), ship lossless — `compacted` populated,
|
|
148
|
+
/// `ccr_hash = None`, nothing dropped.
|
|
149
|
+
/// 3. Otherwise fall through to the lossy path — drop rows,
|
|
150
|
+
/// populate `ccr_hash` with a hash of the full original so the
|
|
151
|
+
/// runtime can cache the payload for tool retrieval.
|
|
152
|
+
///
|
|
153
|
+
/// **No data is ever lost.** The lossy path moves dropped rows to
|
|
154
|
+
/// CCR cache, not to nowhere — same semantics as Python's
|
|
155
|
+
/// SmartCrusher with CCR enabled.
|
|
156
|
+
pub fn new(config: SmartCrusherConfig) -> Self {
|
|
157
|
+
// Carry the compaction heuristics from the crusher config into
|
|
158
|
+
// the compaction stage; everything not exposed on
|
|
159
|
+
// SmartCrusherConfig keeps its CompactConfig default.
|
|
160
|
+
let compact_cfg = CompactConfig {
|
|
161
|
+
core_field_fraction: config.compaction_core_field_fraction,
|
|
162
|
+
heterogeneous_core_ratio: config.compaction_heterogeneous_core_ratio,
|
|
163
|
+
max_flatten_inner_keys: config.compaction_max_flatten_inner_keys,
|
|
164
|
+
min_buckets: config.compaction_min_buckets,
|
|
165
|
+
max_buckets: config.compaction_max_buckets,
|
|
166
|
+
..CompactConfig::default()
|
|
167
|
+
};
|
|
168
|
+
SmartCrusherBuilder::new(config)
|
|
169
|
+
.with_default_oss_setup()
|
|
170
|
+
.with_compaction(CompactionStage::csv_schema(compact_cfg))
|
|
171
|
+
.with_default_ccr_store()
|
|
172
|
+
.build()
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/// Construct WITHOUT the compaction stage. Pre-PR4 behavior:
|
|
176
|
+
/// `crush_array` skips the lossless attempt and runs the lossy
|
|
177
|
+
/// path directly (still with CCR-Dropped retrieval markers from
|
|
178
|
+
/// PR4). Used by:
|
|
179
|
+
///
|
|
180
|
+
/// - The 17 legacy parity fixtures (recorded against the
|
|
181
|
+
/// lossy-only path; using this constructor preserves byte-equal
|
|
182
|
+
/// coverage).
|
|
183
|
+
/// - Callers who explicitly don't want lossless attempts (e.g.
|
|
184
|
+
/// workloads where the compactor's overhead isn't worth the
|
|
185
|
+
/// modest tabular wins).
|
|
186
|
+
pub fn without_compaction(config: SmartCrusherConfig) -> Self {
|
|
187
|
+
SmartCrusherBuilder::new(config)
|
|
188
|
+
.with_default_oss_setup()
|
|
189
|
+
.with_default_ccr_store()
|
|
190
|
+
.build()
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/// Construct like [`SmartCrusher::new`] but with the compaction
|
|
194
|
+
/// stage's formatter chosen by name (`"csv-schema"`, `"json"`,
|
|
195
|
+
/// `"markdown-kv"`). `None` for unknown names — callers own the
|
|
196
|
+
/// fallback/error policy. `"csv-schema"` is equivalent to `new`.
|
|
197
|
+
pub fn with_compaction_format(config: SmartCrusherConfig, format_name: &str) -> Option<Self> {
|
|
198
|
+
let stage = CompactionStage::from_format_name(format_name)?;
|
|
199
|
+
Some(
|
|
200
|
+
SmartCrusherBuilder::new(config)
|
|
201
|
+
.with_default_oss_setup()
|
|
202
|
+
.with_compaction(stage)
|
|
203
|
+
.with_default_ccr_store()
|
|
204
|
+
.build(),
|
|
205
|
+
)
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/// Begin a builder chain for custom composition. The Enterprise
|
|
209
|
+
/// entry point: swap the scorer, add business-rule constraints,
|
|
210
|
+
/// attach an audit observer.
|
|
211
|
+
pub fn builder(config: SmartCrusherConfig) -> SmartCrusherBuilder {
|
|
212
|
+
SmartCrusherBuilder::new(config)
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/// Construct with a custom scorer (legacy convenience). Equivalent
|
|
216
|
+
/// to `SmartCrusher::builder(config).with_scorer(scorer).with_default_oss_setup().build()`
|
|
217
|
+
/// minus the default scorer override; preserved for backward
|
|
218
|
+
/// compatibility with pre-PR1 callers.
|
|
219
|
+
pub fn with_scorer(
|
|
220
|
+
config: SmartCrusherConfig,
|
|
221
|
+
scorer: Box<dyn RelevanceScorer + Send + Sync>,
|
|
222
|
+
) -> Self {
|
|
223
|
+
SmartCrusherBuilder::new(config)
|
|
224
|
+
.with_scorer(scorer)
|
|
225
|
+
.add_default_oss_constraints()
|
|
226
|
+
.build()
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/// Construct directly from owned parts. Used by
|
|
230
|
+
/// [`SmartCrusherBuilder::build`] — not part of the public stable
|
|
231
|
+
/// API. Prefer the builder.
|
|
232
|
+
#[doc(hidden)]
|
|
233
|
+
#[allow(clippy::too_many_arguments)]
|
|
234
|
+
pub fn from_parts(
|
|
235
|
+
config: SmartCrusherConfig,
|
|
236
|
+
anchor_selector: AnchorSelector,
|
|
237
|
+
scorer: Box<dyn RelevanceScorer + Send + Sync>,
|
|
238
|
+
analyzer: SmartAnalyzer,
|
|
239
|
+
constraints: Vec<Box<dyn Constraint>>,
|
|
240
|
+
observers: Vec<Box<dyn Observer>>,
|
|
241
|
+
compaction: Option<CompactionStage>,
|
|
242
|
+
ccr_store: Option<Arc<dyn CcrStore>>,
|
|
243
|
+
) -> Self {
|
|
244
|
+
SmartCrusher {
|
|
245
|
+
config,
|
|
246
|
+
anchor_selector,
|
|
247
|
+
scorer,
|
|
248
|
+
analyzer,
|
|
249
|
+
constraints,
|
|
250
|
+
observers,
|
|
251
|
+
compaction,
|
|
252
|
+
ccr_store,
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/// Handle to the CCR store, if configured. Used by the runtime
|
|
257
|
+
/// (proxy server, PyO3 bridge) to look up originals when retrieval
|
|
258
|
+
/// tool calls fire.
|
|
259
|
+
pub fn ccr_store(&self) -> Option<&Arc<dyn CcrStore>> {
|
|
260
|
+
self.ccr_store.as_ref()
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
fn planner(&self) -> SmartCrusherPlanner<'_> {
|
|
264
|
+
SmartCrusherPlanner::new(
|
|
265
|
+
&self.config,
|
|
266
|
+
&self.anchor_selector,
|
|
267
|
+
&*self.scorer,
|
|
268
|
+
&self.analyzer,
|
|
269
|
+
&self.constraints,
|
|
270
|
+
)
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/// Execute a `CompressionPlan` against `items`, returning the
|
|
274
|
+
/// kept-items list in original-array order. Mirrors Python's
|
|
275
|
+
/// `_execute_plan` (line 3617-3633).
|
|
276
|
+
///
|
|
277
|
+
/// Schema-preserving by default: each kept item is cloned unchanged.
|
|
278
|
+
/// No summary objects, generated fields, or wrapper metadata.
|
|
279
|
+
///
|
|
280
|
+
/// When `factor_out_constants` is enabled (default off), fields the
|
|
281
|
+
/// analyzer found constant across ALL items are stripped from each
|
|
282
|
+
/// kept object and emitted once in a leading
|
|
283
|
+
/// `{"_constant_fields": {...}}` sentinel — same output-shape
|
|
284
|
+
/// convention as the `_ccr_dropped` sentinel. Stripping is
|
|
285
|
+
/// defensive: a key is only removed from an item when its value
|
|
286
|
+
/// equals the recorded constant, so a drifted item keeps its own
|
|
287
|
+
/// value. The CCR store always holds the full unfactored original.
|
|
288
|
+
pub fn execute_plan(&self, plan: &CompressionPlan, items: &[Value]) -> Vec<Value> {
|
|
289
|
+
let mut indices = plan.keep_indices.clone();
|
|
290
|
+
indices.sort_unstable();
|
|
291
|
+
let mut kept: Vec<Value> = indices
|
|
292
|
+
.into_iter()
|
|
293
|
+
.filter(|&idx| idx < items.len())
|
|
294
|
+
.map(|idx| items[idx].clone())
|
|
295
|
+
.collect();
|
|
296
|
+
|
|
297
|
+
if self.config.factor_out_constants && !plan.constant_fields.is_empty() && kept.len() >= 2 {
|
|
298
|
+
let mut any_stripped = false;
|
|
299
|
+
for item in kept.iter_mut() {
|
|
300
|
+
if let Value::Object(map) = item {
|
|
301
|
+
for (key, constant) in &plan.constant_fields {
|
|
302
|
+
if map.get(key) == Some(constant) {
|
|
303
|
+
map.remove(key);
|
|
304
|
+
any_stripped = true;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
if any_stripped {
|
|
310
|
+
let mut sentinel = serde_json::Map::new();
|
|
311
|
+
sentinel.insert(
|
|
312
|
+
"_constant_fields".to_string(),
|
|
313
|
+
Value::Object(plan.constant_fields.clone().into_iter().collect()),
|
|
314
|
+
);
|
|
315
|
+
kept.insert(0, Value::Object(sentinel));
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
kept
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/// Top-level entry point. Mirrors Python `SmartCrusher.crush`
|
|
323
|
+
/// (line 1581-1603) — used by `ContentRouter` when routing JSON
|
|
324
|
+
/// arrays.
|
|
325
|
+
///
|
|
326
|
+
/// Parses `content` as JSON, recursively processes it (compressing
|
|
327
|
+
/// arrays at every depth via the appropriate per-type crusher),
|
|
328
|
+
/// then re-serializes with Python-compatible formatting (`, ` and
|
|
329
|
+
/// `: ` separators, ASCII-escaped non-ASCII).
|
|
330
|
+
///
|
|
331
|
+
/// Returns a `CrushResult` with:
|
|
332
|
+
/// - `compressed`: the re-serialized JSON.
|
|
333
|
+
/// - `original`: the input string (unmodified).
|
|
334
|
+
/// - `was_modified`: whether `compressed` differs from `content`'s
|
|
335
|
+
/// trimmed form.
|
|
336
|
+
/// - `strategy`: combined strategy info from all crushed arrays
|
|
337
|
+
/// (or `"passthrough"`).
|
|
338
|
+
pub fn crush(&self, content: &str, query: &str, bias: f64) -> CrushResult {
|
|
339
|
+
let start = std::time::Instant::now();
|
|
340
|
+
let (compressed, was_modified, info) = self.smart_crush_content(content, query, bias);
|
|
341
|
+
let strategy = if info.is_empty() {
|
|
342
|
+
"passthrough".to_string()
|
|
343
|
+
} else {
|
|
344
|
+
info
|
|
345
|
+
};
|
|
346
|
+
|
|
347
|
+
// Fire one event per top-level crush. Cheap when no observers
|
|
348
|
+
// are configured (`for o in &[]` is a single null-pointer
|
|
349
|
+
// check); cheap when only `TracingObserver` is configured if
|
|
350
|
+
// the subscriber filters `debug` out (the default in
|
|
351
|
+
// production). Custom observers — audit logs, Loop training
|
|
352
|
+
// stream, metrics — pay whatever they pay.
|
|
353
|
+
if !self.observers.is_empty() {
|
|
354
|
+
let event = CrushEvent {
|
|
355
|
+
strategy: strategy.clone(),
|
|
356
|
+
input_bytes: content.len(),
|
|
357
|
+
output_bytes: compressed.len(),
|
|
358
|
+
elapsed_ns: start.elapsed().as_nanos() as u64,
|
|
359
|
+
was_modified,
|
|
360
|
+
};
|
|
361
|
+
for observer in &self.observers {
|
|
362
|
+
observer.on_event(&event);
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
CrushResult {
|
|
367
|
+
compressed,
|
|
368
|
+
original: content.to_string(),
|
|
369
|
+
was_modified,
|
|
370
|
+
strategy,
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/// `SmartCrusher._smart_crush_content` (Python line 2243-2301).
|
|
375
|
+
/// JSON-parse, recursively process, re-serialize. CCR marker
|
|
376
|
+
/// injection is stubbed (CCR is disabled in this stage).
|
|
377
|
+
///
|
|
378
|
+
/// Returns `(crushed_content, was_modified, info)`.
|
|
379
|
+
pub fn smart_crush_content(
|
|
380
|
+
&self,
|
|
381
|
+
content: &str,
|
|
382
|
+
query_context: &str,
|
|
383
|
+
bias: f64,
|
|
384
|
+
) -> (String, bool, String) {
|
|
385
|
+
// Parse — non-JSON content passes through unchanged.
|
|
386
|
+
let Ok(parsed) = serde_json::from_str::<Value>(content) else {
|
|
387
|
+
return (content.to_string(), false, String::new());
|
|
388
|
+
};
|
|
389
|
+
|
|
390
|
+
let (crushed, info) = self.process_value(&parsed, 0, query_context, bias);
|
|
391
|
+
|
|
392
|
+
// Re-serialize with Python `safe_json_dumps` formatting:
|
|
393
|
+
// compact `(",", ":")` separators + `ensure_ascii=False`,
|
|
394
|
+
// preserving object-key insertion order. Matches the Python
|
|
395
|
+
// SmartCrusher output bytes the proxy writes.
|
|
396
|
+
let result = crate::transforms::anchor_selector::python_safe_json_dumps(&crushed);
|
|
397
|
+
let was_modified = result != content.trim();
|
|
398
|
+
(result, was_modified, info)
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/// Maximum recursion depth for nested JSON. Mirrors Python's
|
|
402
|
+
/// `_MAX_PROCESS_DEPTH = 50`. Beyond this, values are returned as-is.
|
|
403
|
+
const MAX_PROCESS_DEPTH: usize = 50;
|
|
404
|
+
|
|
405
|
+
/// Recursively process a value, crushing arrays where appropriate.
|
|
406
|
+
/// Mirrors Python `_process_value` (line 2307-2398).
|
|
407
|
+
///
|
|
408
|
+
/// Returns `(processed_value, info_string)`. CCR markers are
|
|
409
|
+
/// stubbed (Python's tuple has a third element for them — Rust's
|
|
410
|
+
/// version omits since we never produce markers in this stage).
|
|
411
|
+
pub fn process_value(
|
|
412
|
+
&self,
|
|
413
|
+
value: &Value,
|
|
414
|
+
depth: usize,
|
|
415
|
+
query_context: &str,
|
|
416
|
+
bias: f64,
|
|
417
|
+
) -> (Value, String) {
|
|
418
|
+
if depth >= Self::MAX_PROCESS_DEPTH {
|
|
419
|
+
return (value.clone(), String::new());
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
let mut info_parts: Vec<String> = Vec::new();
|
|
423
|
+
|
|
424
|
+
match value {
|
|
425
|
+
Value::Array(arr) => {
|
|
426
|
+
let n = arr.len();
|
|
427
|
+
if n >= self.config.min_items_to_analyze {
|
|
428
|
+
let arr_type = classify_array(arr);
|
|
429
|
+
match arr_type {
|
|
430
|
+
ArrayType::DictArray => {
|
|
431
|
+
let result = self.crush_array(arr, query_context, bias);
|
|
432
|
+
// Lossless path won → substitute the array
|
|
433
|
+
// with the compacted string in place. This
|
|
434
|
+
// makes the lossless win visible to the
|
|
435
|
+
// public `crush()` API: the output JSON
|
|
436
|
+
// has a string where the array used to be.
|
|
437
|
+
// The wrapping JSON structure is preserved.
|
|
438
|
+
if let Some(rendered) = result.compacted {
|
|
439
|
+
info_parts.push(format!(
|
|
440
|
+
"{}({}->len={})",
|
|
441
|
+
result.strategy_info,
|
|
442
|
+
n,
|
|
443
|
+
rendered.len()
|
|
444
|
+
));
|
|
445
|
+
return (Value::String(rendered), info_parts.join(","));
|
|
446
|
+
}
|
|
447
|
+
info_parts.push(format!(
|
|
448
|
+
"{}({}->{})",
|
|
449
|
+
result.strategy_info,
|
|
450
|
+
n,
|
|
451
|
+
result.items.len()
|
|
452
|
+
));
|
|
453
|
+
// Lossy path with rows dropped → append a
|
|
454
|
+
// CCR-Dropped sentinel object as the last
|
|
455
|
+
// element of the kept-items array. This is
|
|
456
|
+
// the **only** place the LLM sees the
|
|
457
|
+
// `<<ccr:HASH ...>>` pointer in the prompt.
|
|
458
|
+
// Without this, the store has the data but
|
|
459
|
+
// no model can ever ask for it.
|
|
460
|
+
//
|
|
461
|
+
// Sentinel shape: `{"_ccr_dropped":
|
|
462
|
+
// "<<ccr:HASH N_rows_offloaded>>"}` —
|
|
463
|
+
// preserves "array-of-objects" shape so
|
|
464
|
+
// downstream consumers iterating with
|
|
465
|
+
// `x.get(...)` keep working; the well-known
|
|
466
|
+
// `_ccr_dropped` key signals metadata
|
|
467
|
+
// unambiguously.
|
|
468
|
+
let mut items = result.items;
|
|
469
|
+
if !result.dropped_summary.is_empty() {
|
|
470
|
+
let mut sentinel = serde_json::Map::new();
|
|
471
|
+
sentinel.insert(
|
|
472
|
+
"_ccr_dropped".to_string(),
|
|
473
|
+
Value::String(result.dropped_summary),
|
|
474
|
+
);
|
|
475
|
+
items.push(Value::Object(sentinel));
|
|
476
|
+
}
|
|
477
|
+
return (Value::Array(items), info_parts.join(","));
|
|
478
|
+
}
|
|
479
|
+
ArrayType::StringArray => {
|
|
480
|
+
let strs: Vec<&str> = arr.iter().filter_map(|v| v.as_str()).collect();
|
|
481
|
+
let (crushed, strategy) = crush_string_array(&strs, &self.config, bias);
|
|
482
|
+
info_parts.push(format!("{}({}->{})", strategy, n, crushed.len()));
|
|
483
|
+
let crushed_values: Vec<Value> =
|
|
484
|
+
crushed.into_iter().map(Value::String).collect();
|
|
485
|
+
return (Value::Array(crushed_values), info_parts.join(","));
|
|
486
|
+
}
|
|
487
|
+
ArrayType::NumberArray => {
|
|
488
|
+
let (crushed, strategy) = crush_number_array(arr, &self.config, bias);
|
|
489
|
+
info_parts.push(format!("{}({}->{})", strategy, n, crushed.len()));
|
|
490
|
+
return (Value::Array(crushed), info_parts.join(","));
|
|
491
|
+
}
|
|
492
|
+
ArrayType::MixedArray => {
|
|
493
|
+
let (crushed, strategy) =
|
|
494
|
+
self.crush_mixed_array(arr, query_context, bias);
|
|
495
|
+
info_parts.push(format!("{}({}->{})", strategy, n, crushed.len()));
|
|
496
|
+
return (Value::Array(crushed), info_parts.join(","));
|
|
497
|
+
}
|
|
498
|
+
// NestedArray, BoolArray, Empty → fall through
|
|
499
|
+
// to recursive descent.
|
|
500
|
+
_ => {}
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
// Below threshold or not crushable → recurse into items.
|
|
505
|
+
let mut processed: Vec<Value> = Vec::with_capacity(n);
|
|
506
|
+
for item in arr {
|
|
507
|
+
let (p_item, p_info) = self.process_value(item, depth + 1, query_context, bias);
|
|
508
|
+
processed.push(p_item);
|
|
509
|
+
if !p_info.is_empty() {
|
|
510
|
+
info_parts.push(p_info);
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
(Value::Array(processed), info_parts.join(","))
|
|
514
|
+
}
|
|
515
|
+
Value::Object(map) => {
|
|
516
|
+
// First pass: recurse into values to compress nested arrays.
|
|
517
|
+
let mut processed = serde_json::Map::new();
|
|
518
|
+
for (k, v) in map {
|
|
519
|
+
let (p_val, p_info) = self.process_value(v, depth + 1, query_context, bias);
|
|
520
|
+
processed.insert(k.clone(), p_val);
|
|
521
|
+
if !p_info.is_empty() {
|
|
522
|
+
info_parts.push(p_info);
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
// Second pass: if the object itself has many keys,
|
|
527
|
+
// compress at the key level.
|
|
528
|
+
if processed.len() >= self.config.min_items_to_analyze {
|
|
529
|
+
let (crushed_dict, strategy) = crush_object(&processed, &self.config, bias);
|
|
530
|
+
if strategy != "object:passthrough" {
|
|
531
|
+
info_parts.push(strategy);
|
|
532
|
+
return (Value::Object(crushed_dict), info_parts.join(","));
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
(Value::Object(processed), info_parts.join(","))
|
|
537
|
+
}
|
|
538
|
+
// Strings: walker-equivalent handling. Delegates to
|
|
539
|
+
// `process_string` which parses stringified-JSON containers
|
|
540
|
+
// (recursing through `process_value`) and CCR-substitutes
|
|
541
|
+
// opaque blobs (with store-write so retrieval works).
|
|
542
|
+
Value::String(s) => self.process_string(s, depth, query_context, bias),
|
|
543
|
+
// Other scalars — passthrough.
|
|
544
|
+
_ => (value.clone(), String::new()),
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
/// Walker-equivalent string handling. Mirrors `walker::walk_string`
|
|
549
|
+
/// in `compaction/walker.rs` but lives on `SmartCrusher` so the
|
|
550
|
+
/// public `crush()` path picks it up.
|
|
551
|
+
///
|
|
552
|
+
/// Two cases:
|
|
553
|
+
/// 1. **Stringified-JSON.** Strings that parse to a JSON object or
|
|
554
|
+
/// array → recurse via `process_value`, then re-emit the result
|
|
555
|
+
/// as a compact JSON string. The wrapping string is preserved
|
|
556
|
+
/// (so the parent JSON shape stays a string-typed field), but
|
|
557
|
+
/// its contents are processed end-to-end.
|
|
558
|
+
/// 2. **Opaque blobs.** Strings classified as
|
|
559
|
+
/// [`CellClass::Opaque`] (long base64 / HTML / long-text) →
|
|
560
|
+
/// substitute with a `<<ccr:HASH,KIND,SIZE>>` marker. Same
|
|
561
|
+
/// format as `compaction::walker::format_ccr_marker` so
|
|
562
|
+
/// downstream consumers can pattern-match markers regardless
|
|
563
|
+
/// of which path emitted them.
|
|
564
|
+
fn process_string(
|
|
565
|
+
&self,
|
|
566
|
+
s: &str,
|
|
567
|
+
depth: usize,
|
|
568
|
+
query_context: &str,
|
|
569
|
+
bias: f64,
|
|
570
|
+
) -> (Value, String) {
|
|
571
|
+
// 1. Stringified-JSON: parse, recurse, re-render.
|
|
572
|
+
if let Some(parsed) = try_parse_json_container(s) {
|
|
573
|
+
let (processed, sub_info) = self.process_value(&parsed, depth + 1, query_context, bias);
|
|
574
|
+
// If recursion produced something different, re-emit.
|
|
575
|
+
// Special case: if the recursion returned a `Value::String`
|
|
576
|
+
// (lossless compaction substituted the array with a
|
|
577
|
+
// rendered CSV+schema string), use that string directly.
|
|
578
|
+
// Re-encoding it as JSON would produce a quoted string
|
|
579
|
+
// literal — double-encoded — which is not what callers
|
|
580
|
+
// expect in the wrapping field.
|
|
581
|
+
if processed != parsed {
|
|
582
|
+
let rendered = match &processed {
|
|
583
|
+
Value::String(rendered_str) => rendered_str.clone(),
|
|
584
|
+
_ => serde_json::to_string(&processed).unwrap_or_else(|_| s.to_string()),
|
|
585
|
+
};
|
|
586
|
+
let info = if sub_info.is_empty() {
|
|
587
|
+
"string_json".to_string()
|
|
588
|
+
} else {
|
|
589
|
+
format!("string_json[{sub_info}]")
|
|
590
|
+
};
|
|
591
|
+
return (Value::String(rendered), info);
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
// 2. Opaque blob: substitute with CCR marker AND stash the
|
|
596
|
+
// original in the store (PR8) so retrieval works. Hash + format
|
|
597
|
+
// identical to walker.rs via the shared helper — zero drift.
|
|
598
|
+
let cfg = ClassifyConfig::default();
|
|
599
|
+
if let CellClass::Opaque(kind) = classify_cell(&Value::String(s.to_string()), &cfg) {
|
|
600
|
+
let marker = emit_opaque_ccr_marker(s, &kind, self.ccr_store.as_ref());
|
|
601
|
+
let kind_label = opaque_kind_label(&kind);
|
|
602
|
+
return (Value::String(marker), format!("string_ccr:{kind_label}"));
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
// 3. Plain string — passthrough.
|
|
606
|
+
(Value::String(s.to_string()), String::new())
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
/// Compress an array of dict items.
|
|
610
|
+
///
|
|
611
|
+
/// Direct port of `_crush_array` (Python line 2400-2687) with the
|
|
612
|
+
/// optional subsystems (TOIN / CCR / feedback / telemetry) wired
|
|
613
|
+
/// in their disabled-by-default behavior. See module-level docs
|
|
614
|
+
/// for the rationale.
|
|
615
|
+
///
|
|
616
|
+
/// # Pipeline
|
|
617
|
+
///
|
|
618
|
+
/// 1. Compute `item_strings` once (used as input to adaptive
|
|
619
|
+
/// sizing and downstream relevance scoring).
|
|
620
|
+
/// 2. `compute_optimal_k` → `adaptive_k`.
|
|
621
|
+
/// 3. If `n <= adaptive_k`, return passthrough.
|
|
622
|
+
/// 4. `analyzer.analyze_array(items)` → `analysis`.
|
|
623
|
+
/// 5. If `analysis.recommended_strategy == Skip`, return passthrough
|
|
624
|
+
/// with a `skip:<reason>` strategy string.
|
|
625
|
+
/// 6. `planner.create_plan(analysis, items, query_context, ...)`.
|
|
626
|
+
/// 7. `execute_plan(plan, items)` → result.
|
|
627
|
+
/// 8. Strategy info = `analysis.recommended_strategy.as_str()`.
|
|
628
|
+
pub fn crush_array(&self, items: &[Value], query_context: &str, bias: f64) -> CrushArrayResult {
|
|
629
|
+
let item_strings: Vec<String> = items
|
|
630
|
+
.iter()
|
|
631
|
+
.map(|i| serde_json::to_string(i).unwrap_or_default())
|
|
632
|
+
.collect();
|
|
633
|
+
let item_str_refs: Vec<&str> = item_strings.iter().map(|s| s.as_str()).collect();
|
|
634
|
+
|
|
635
|
+
let max_k = if self.config.max_items_after_crush > 0 {
|
|
636
|
+
Some(self.config.max_items_after_crush)
|
|
637
|
+
} else {
|
|
638
|
+
None
|
|
639
|
+
};
|
|
640
|
+
let adaptive_k = compute_optimal_k(&item_str_refs, bias, 3, max_k);
|
|
641
|
+
|
|
642
|
+
// Tier-1 boundary: array already small enough — passthrough,
|
|
643
|
+
// nothing to compact, nothing to drop.
|
|
644
|
+
if items.len() <= adaptive_k {
|
|
645
|
+
return CrushArrayResult {
|
|
646
|
+
items: items.to_vec(),
|
|
647
|
+
strategy_info: "none:adaptive_at_limit".to_string(),
|
|
648
|
+
ccr_hash: None,
|
|
649
|
+
dropped_summary: String::new(),
|
|
650
|
+
compacted: None,
|
|
651
|
+
compaction_kind: None,
|
|
652
|
+
};
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
// ── Lossless-first attempt ──
|
|
656
|
+
//
|
|
657
|
+
// Run the compaction stage if present, then check the savings
|
|
658
|
+
// ratio against `config.lossless_min_savings_ratio`. If the
|
|
659
|
+
// lossless rendering shrinks the input by at least that much,
|
|
660
|
+
// ship it — nothing dropped, no CCR retrieval needed.
|
|
661
|
+
// Otherwise fall through to the lossy path.
|
|
662
|
+
if let Some(stage) = &self.compaction {
|
|
663
|
+
let (c, rendered) = stage.run(items);
|
|
664
|
+
if c.was_compacted() {
|
|
665
|
+
let input_bytes = estimate_array_bytes(&item_strings);
|
|
666
|
+
let savings_ratio = if input_bytes > 0 {
|
|
667
|
+
1.0 - (rendered.len() as f64 / input_bytes as f64)
|
|
668
|
+
} else {
|
|
669
|
+
0.0
|
|
670
|
+
};
|
|
671
|
+
if savings_ratio >= self.config.lossless_min_savings_ratio {
|
|
672
|
+
let kind = compaction_kind_str(&c);
|
|
673
|
+
return CrushArrayResult {
|
|
674
|
+
items: items.to_vec(), // nothing dropped
|
|
675
|
+
strategy_info: format!("lossless:{kind}"),
|
|
676
|
+
ccr_hash: None,
|
|
677
|
+
dropped_summary: String::new(),
|
|
678
|
+
compacted: Some(rendered),
|
|
679
|
+
compaction_kind: Some(kind),
|
|
680
|
+
};
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
// ── Lossy path: compress inline + cache full original via CCR ──
|
|
686
|
+
//
|
|
687
|
+
// The runtime caller (PyO3 bridge / proxy server) is expected
|
|
688
|
+
// to stash the full input keyed by `ccr_hash` so a retrieval
|
|
689
|
+
// tool can serve dropped rows back to the LLM on demand.
|
|
690
|
+
// **No data is lost** — "lossy" here means "compressed view
|
|
691
|
+
// inline; full payload retrievable via CCR cache."
|
|
692
|
+
|
|
693
|
+
let effective_max_items = adaptive_k;
|
|
694
|
+
let analysis = self.analyzer.analyze_array(items);
|
|
695
|
+
|
|
696
|
+
// Crushability gate: not safe to crush → passthrough, no CCR.
|
|
697
|
+
if analysis.recommended_strategy == CompressionStrategy::Skip {
|
|
698
|
+
let reason = match &analysis.crushability {
|
|
699
|
+
Some(c) => format!("skip:{}", c.reason),
|
|
700
|
+
None => String::new(),
|
|
701
|
+
};
|
|
702
|
+
return CrushArrayResult {
|
|
703
|
+
items: items.to_vec(),
|
|
704
|
+
strategy_info: reason,
|
|
705
|
+
ccr_hash: None,
|
|
706
|
+
dropped_summary: String::new(),
|
|
707
|
+
compacted: None,
|
|
708
|
+
compaction_kind: None,
|
|
709
|
+
};
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
let plan = self.planner().create_plan(
|
|
713
|
+
&analysis,
|
|
714
|
+
items,
|
|
715
|
+
query_context,
|
|
716
|
+
None, // preserve_fields (TOIN — stubbed)
|
|
717
|
+
Some(effective_max_items),
|
|
718
|
+
Some(&item_strings),
|
|
719
|
+
);
|
|
720
|
+
let result = self.execute_plan(&plan, items);
|
|
721
|
+
|
|
722
|
+
// Emit CCR-Dropped marker iff rows were actually dropped AND
|
|
723
|
+
// the marker gate is on. **The marker is the cornerstone of
|
|
724
|
+
// CCR's no-data-loss guarantee:** we hash the full original,
|
|
725
|
+
// stash it in the configured store, and emit a marker pointing
|
|
726
|
+
// at that hash. The runtime later serves the original back via
|
|
727
|
+
// retrieval tool calls.
|
|
728
|
+
//
|
|
729
|
+
// When `enable_ccr_marker` is false (Python shim's path for
|
|
730
|
+
// `ccr_config.enabled=False` or `inject_retrieval_marker=False`)
|
|
731
|
+
// we keep the row drops (compression is still requested) but
|
|
732
|
+
// skip the marker text and the store write — there's no point
|
|
733
|
+
// storing a payload that nothing in the prompt can reference.
|
|
734
|
+
let dropped_count = items.len().saturating_sub(result.len());
|
|
735
|
+
let (ccr_hash, dropped_summary) = if dropped_count > 0 && self.config.enable_ccr_marker {
|
|
736
|
+
// Serialize the original array exactly ONCE. The hash is
|
|
737
|
+
// taken over those bytes, and (if a store is configured) the
|
|
738
|
+
// same bytes get stored — eliminating a redundant tree clone
|
|
739
|
+
// (`items.to_vec()`) and a redundant `serde_json::to_string`
|
|
740
|
+
// pass that the previous version did per dropped array.
|
|
741
|
+
let canonical = canonical_array_json(items);
|
|
742
|
+
let h = hash_canonical(&canonical);
|
|
743
|
+
let marker = format!("<<ccr:{h} {dropped_count}_rows_offloaded>>");
|
|
744
|
+
if let Some(store) = &self.ccr_store {
|
|
745
|
+
store.put(&h, &canonical);
|
|
746
|
+
}
|
|
747
|
+
(Some(h), marker)
|
|
748
|
+
} else {
|
|
749
|
+
(None, String::new())
|
|
750
|
+
};
|
|
751
|
+
|
|
752
|
+
CrushArrayResult {
|
|
753
|
+
items: result,
|
|
754
|
+
strategy_info: analysis.recommended_strategy.as_str().to_string(),
|
|
755
|
+
ccr_hash,
|
|
756
|
+
dropped_summary,
|
|
757
|
+
compacted: None,
|
|
758
|
+
compaction_kind: None,
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
/// Compress a mixed-type array by grouping items by type and
|
|
763
|
+
/// compressing each group with the appropriate handler.
|
|
764
|
+
///
|
|
765
|
+
/// Direct port of `_crush_mixed_array` (Python line 2914-3013).
|
|
766
|
+
///
|
|
767
|
+
/// Strategy:
|
|
768
|
+
/// 1. Group by type (dict / str / number / list / null / bool / other).
|
|
769
|
+
/// 2. For groups with >= `min_items_to_analyze` items: apply the
|
|
770
|
+
/// type-specific compressor.
|
|
771
|
+
/// 3. For small groups: keep all items.
|
|
772
|
+
/// 4. Reassemble in original order.
|
|
773
|
+
///
|
|
774
|
+
/// Returns `(crushed_items, strategy_string)`.
|
|
775
|
+
pub fn crush_mixed_array(
|
|
776
|
+
&self,
|
|
777
|
+
items: &[Value],
|
|
778
|
+
query_context: &str,
|
|
779
|
+
bias: f64,
|
|
780
|
+
) -> (Vec<Value>, String) {
|
|
781
|
+
let n = items.len();
|
|
782
|
+
if n <= 8 {
|
|
783
|
+
return (items.to_vec(), "mixed:passthrough".to_string());
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
// Group by type, tracking original indices.
|
|
787
|
+
let mut groups: GroupBuckets = GroupBuckets::default();
|
|
788
|
+
for (i, item) in items.iter().enumerate() {
|
|
789
|
+
groups.push(group_key(item), i, item.clone());
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
let mut keep_indices: std::collections::BTreeSet<usize> = std::collections::BTreeSet::new();
|
|
793
|
+
let mut strategy_parts: Vec<String> = Vec::new();
|
|
794
|
+
|
|
795
|
+
for (type_key, indices, values) in groups.into_iter() {
|
|
796
|
+
// Small groups: keep all items.
|
|
797
|
+
if values.len() < self.config.min_items_to_analyze {
|
|
798
|
+
keep_indices.extend(&indices);
|
|
799
|
+
continue;
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
match type_key {
|
|
803
|
+
"dict" => {
|
|
804
|
+
let CrushArrayResult { items: crushed, .. } =
|
|
805
|
+
self.crush_array(&values, query_context, bias);
|
|
806
|
+
// Find which original indices survived by matching
|
|
807
|
+
// canonical-JSON serialization. Mirrors Python's
|
|
808
|
+
// `json.dumps(c, sort_keys=True, default=str)`-keyed
|
|
809
|
+
// set match.
|
|
810
|
+
let crushed_keys: std::collections::HashSet<String> =
|
|
811
|
+
crushed.iter().map(canonical_json_for_match).collect();
|
|
812
|
+
for (i, idx) in indices.iter().enumerate() {
|
|
813
|
+
if crushed_keys.contains(&canonical_json_for_match(&values[i])) {
|
|
814
|
+
keep_indices.insert(*idx);
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
strategy_parts.push(format!("dict:{}->{}", values.len(), crushed.len()));
|
|
818
|
+
}
|
|
819
|
+
"str" => {
|
|
820
|
+
let strs: Vec<&str> = values.iter().filter_map(|v| v.as_str()).collect();
|
|
821
|
+
let (crushed, _) = crush_string_array(&strs, &self.config, bias);
|
|
822
|
+
let crushed_set: std::collections::HashSet<&str> =
|
|
823
|
+
crushed.iter().map(|s| s.as_str()).collect();
|
|
824
|
+
for (i, idx) in indices.iter().enumerate() {
|
|
825
|
+
if let Some(s) = values[i].as_str() {
|
|
826
|
+
if crushed_set.contains(s) {
|
|
827
|
+
keep_indices.insert(*idx);
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
strategy_parts.push(format!("str:{}->{}", values.len(), crushed.len()));
|
|
832
|
+
}
|
|
833
|
+
"number" => {
|
|
834
|
+
// Python: just adaptive sampling + outlier detection
|
|
835
|
+
// (no summary prefix). Keeps first/last by index
|
|
836
|
+
// and items >variance_threshold σ from mean.
|
|
837
|
+
let item_strings: Vec<String> = values.iter().map(|v| v.to_string()).collect();
|
|
838
|
+
let item_refs: Vec<&str> = item_strings.iter().map(|s| s.as_str()).collect();
|
|
839
|
+
let (_kt, kf, kl, _) = compute_k_split(&item_refs, &self.config, bias);
|
|
840
|
+
|
|
841
|
+
let kf = kf.min(values.len());
|
|
842
|
+
let kl = kl.min(values.len().saturating_sub(kf));
|
|
843
|
+
let first_idx: Vec<usize> = indices.iter().take(kf).copied().collect();
|
|
844
|
+
let last_idx: Vec<usize> =
|
|
845
|
+
indices.iter().rev().take(kl).copied().collect::<Vec<_>>();
|
|
846
|
+
keep_indices.extend(&first_idx);
|
|
847
|
+
keep_indices.extend(&last_idx);
|
|
848
|
+
|
|
849
|
+
// Outliers via finite-only stats.
|
|
850
|
+
let finite: Vec<f64> = values
|
|
851
|
+
.iter()
|
|
852
|
+
.filter_map(|v| v.as_f64().filter(|f| f.is_finite()))
|
|
853
|
+
.collect();
|
|
854
|
+
if finite.len() > 1 {
|
|
855
|
+
if let Some(mean_v) = super::stats_math::mean(&finite) {
|
|
856
|
+
if let Some(std_v) = super::stats_math::sample_stdev(&finite) {
|
|
857
|
+
if std_v > 0.0 {
|
|
858
|
+
let threshold = self.config.variance_threshold * std_v;
|
|
859
|
+
for (i, val) in values.iter().enumerate() {
|
|
860
|
+
if let Some(num) = val.as_f64().filter(|f| f.is_finite()) {
|
|
861
|
+
if (num - mean_v).abs() > threshold {
|
|
862
|
+
keep_indices.insert(indices[i]);
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
strategy_parts.push(format!("num:{}", values.len()));
|
|
871
|
+
}
|
|
872
|
+
_ => {
|
|
873
|
+
// list / bool / none / other → keep all items.
|
|
874
|
+
keep_indices.extend(&indices);
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
// Reassemble in original order.
|
|
880
|
+
let result: Vec<Value> = keep_indices.iter().map(|&i| items[i].clone()).collect();
|
|
881
|
+
let strategy = format!(
|
|
882
|
+
"mixed:adaptive({}->{},{})",
|
|
883
|
+
n,
|
|
884
|
+
result.len(),
|
|
885
|
+
strategy_parts.join(",")
|
|
886
|
+
);
|
|
887
|
+
(result, strategy)
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
// ---------- helpers ----------
|
|
892
|
+
|
|
893
|
+
/// Group key that mirrors Python's `_crush_mixed_array` switch on
|
|
894
|
+
/// `isinstance`. Note the bool-before-number ordering: in Python, bool
|
|
895
|
+
/// is a subclass of int, but JSON treats them as distinct types, so we
|
|
896
|
+
/// don't have the Python ordering hazard.
|
|
897
|
+
fn group_key(item: &Value) -> &'static str {
|
|
898
|
+
match item {
|
|
899
|
+
Value::Object(_) => "dict",
|
|
900
|
+
Value::String(_) => "str",
|
|
901
|
+
Value::Bool(_) => "bool",
|
|
902
|
+
Value::Number(_) => "number",
|
|
903
|
+
Value::Array(_) => "list",
|
|
904
|
+
Value::Null => "none",
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
/// Group buckets keyed by the type-string. Preserves first-occurrence
|
|
909
|
+
/// order across keys so dict/str/number/list/none/bool always come out
|
|
910
|
+
/// in the same order — matters because `keep_indices` is built
|
|
911
|
+
/// incrementally and Python iterates `groups.items()` (insertion order
|
|
912
|
+
/// in 3.7+).
|
|
913
|
+
#[derive(Default)]
|
|
914
|
+
struct GroupBuckets {
|
|
915
|
+
entries: Vec<(&'static str, Vec<usize>, Vec<Value>)>,
|
|
916
|
+
index_of: std::collections::HashMap<&'static str, usize>,
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
impl GroupBuckets {
|
|
920
|
+
fn push(&mut self, key: &'static str, idx: usize, value: Value) {
|
|
921
|
+
match self.index_of.get(key).copied() {
|
|
922
|
+
Some(i) => {
|
|
923
|
+
self.entries[i].1.push(idx);
|
|
924
|
+
self.entries[i].2.push(value);
|
|
925
|
+
}
|
|
926
|
+
None => {
|
|
927
|
+
self.index_of.insert(key, self.entries.len());
|
|
928
|
+
self.entries.push((key, vec![idx], vec![value]));
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
impl IntoIterator for GroupBuckets {
|
|
935
|
+
type Item = (&'static str, Vec<usize>, Vec<Value>);
|
|
936
|
+
type IntoIter = std::vec::IntoIter<Self::Item>;
|
|
937
|
+
fn into_iter(self) -> Self::IntoIter {
|
|
938
|
+
self.entries.into_iter()
|
|
939
|
+
}
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
/// Serialize a `Value` for membership comparison. Mirrors Python's
|
|
943
|
+
/// `json.dumps(c, sort_keys=True, default=str)` used by
|
|
944
|
+
/// `_crush_mixed_array` to match crushed dict items back to their
|
|
945
|
+
/// original indices. The `default=str` fallback only matters for
|
|
946
|
+
/// non-JSON-serializable Python values; in serde_json land everything
|
|
947
|
+
/// is already JSON-native, so plain canonical JSON suffices.
|
|
948
|
+
fn canonical_json_for_match(value: &Value) -> String {
|
|
949
|
+
crate::transforms::anchor_selector::python_json_dumps_sort_keys(value)
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
/// Maps a `Compaction` to a stable kind tag exposed via `CrushArrayResult`.
|
|
953
|
+
fn compaction_kind_str(c: &Compaction) -> &'static str {
|
|
954
|
+
match c {
|
|
955
|
+
Compaction::Table { .. } => "table",
|
|
956
|
+
Compaction::Buckets { .. } => "buckets",
|
|
957
|
+
Compaction::OpaqueRef { .. } => "ccr",
|
|
958
|
+
Compaction::Untouched(_) => "untouched",
|
|
959
|
+
}
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
/// Approximate byte size of `[v0, v1, ...]` JSON serialization, given
|
|
963
|
+
/// each item's already-serialized form. Adds 2 for outer brackets and
|
|
964
|
+
/// 1 per inter-item comma. Used by the lossless savings-ratio check.
|
|
965
|
+
fn estimate_array_bytes(item_strings: &[String]) -> usize {
|
|
966
|
+
let payload: usize = item_strings.iter().map(|s| s.len()).sum();
|
|
967
|
+
let separators = item_strings.len().saturating_sub(1);
|
|
968
|
+
payload + separators + 2
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
/// Serialize `[v0, v1, ...]` once into the canonical JSON form used by
|
|
972
|
+
/// the CCR retrieval contract. `serde_json` writes a slice of `Value` as
|
|
973
|
+
/// the same bytes it would write for `Value::Array(items.to_vec())`, so
|
|
974
|
+
/// we skip the array-wrapper allocation and the deep tree clone it
|
|
975
|
+
/// requires. Used by both the hash (input) and the store payload (write).
|
|
976
|
+
fn canonical_array_json(items: &[Value]) -> String {
|
|
977
|
+
serde_json::to_string(items).unwrap_or_default()
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
/// 12-char SHA-256 hex prefix of an already-serialized canonical JSON
|
|
981
|
+
/// string. Caller is responsible for producing the canonical form via
|
|
982
|
+
/// [`canonical_array_json`] (or another byte-equal serializer) — the
|
|
983
|
+
/// hash is over the bytes, so a stable serializer is the contract.
|
|
984
|
+
fn hash_canonical(canonical: &str) -> String {
|
|
985
|
+
use sha2::{Digest, Sha256};
|
|
986
|
+
let mut h = Sha256::new();
|
|
987
|
+
h.update(canonical.as_bytes());
|
|
988
|
+
h.finalize()
|
|
989
|
+
.iter()
|
|
990
|
+
.take(6)
|
|
991
|
+
.map(|b| format!("{b:02x}"))
|
|
992
|
+
.collect()
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
// `hash_array_for_ccr` (a test-only `canonical_array_json + hash_canonical`
|
|
996
|
+
// convenience) lived here previously but had no callers — clippy flagged
|
|
997
|
+
// it as dead code. Reintroduce as a test fixture if a future test wants
|
|
998
|
+
// the one-liner; production callsites inline both steps so the canonical
|
|
999
|
+
// bytes can be reused for the store payload.
|
|
1000
|
+
|
|
1001
|
+
// ─── PR5 walker-integration helpers (string handling) ──────────────────────
|
|
1002
|
+
//
|
|
1003
|
+
// Parse-as-JSON-container, marker formatting, and humanize-bytes used to
|
|
1004
|
+
// live here as locals. PR8 extracted them into `compaction::walker` so
|
|
1005
|
+
// `walker.rs` and `process_value` share one canonical implementation —
|
|
1006
|
+
// killing the drift risk where the two paths could format markers
|
|
1007
|
+
// differently. `process_string` now calls `try_parse_json_container` and
|
|
1008
|
+
// `emit_opaque_ccr_marker` directly. Only `opaque_kind_label` survives
|
|
1009
|
+
// here because `process_string`'s `string_ccr:<kind>` strategy-info
|
|
1010
|
+
// label is local to this module's debug-string convention.
|
|
1011
|
+
|
|
1012
|
+
fn opaque_kind_label(kind: &super::compaction::OpaqueKind) -> &str {
|
|
1013
|
+
use super::compaction::OpaqueKind;
|
|
1014
|
+
match kind {
|
|
1015
|
+
OpaqueKind::Base64Blob => "base64",
|
|
1016
|
+
OpaqueKind::LongString => "string",
|
|
1017
|
+
OpaqueKind::HtmlChunk => "html",
|
|
1018
|
+
OpaqueKind::Other(s) => s.as_str(),
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
#[cfg(test)]
|
|
1023
|
+
mod tests {
|
|
1024
|
+
use super::*;
|
|
1025
|
+
use serde_json::json;
|
|
1026
|
+
|
|
1027
|
+
fn crusher() -> SmartCrusher {
|
|
1028
|
+
SmartCrusher::new(SmartCrusherConfig::default())
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
// ---------- execute_plan ----------
|
|
1032
|
+
|
|
1033
|
+
#[test]
|
|
1034
|
+
fn execute_plan_empty_indices_returns_empty() {
|
|
1035
|
+
let c = crusher();
|
|
1036
|
+
let plan = CompressionPlan::default();
|
|
1037
|
+
let items: Vec<Value> = (0..5).map(|i| json!({"id": i})).collect();
|
|
1038
|
+
let result = c.execute_plan(&plan, &items);
|
|
1039
|
+
assert!(result.is_empty());
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
#[test]
|
|
1043
|
+
fn execute_plan_returns_items_in_sorted_index_order() {
|
|
1044
|
+
let c = crusher();
|
|
1045
|
+
let items: Vec<Value> = (0..10).map(|i| json!({"id": i})).collect();
|
|
1046
|
+
let plan = CompressionPlan {
|
|
1047
|
+
keep_indices: vec![5, 2, 8, 0],
|
|
1048
|
+
..CompressionPlan::default()
|
|
1049
|
+
};
|
|
1050
|
+
let result = c.execute_plan(&plan, &items);
|
|
1051
|
+
assert_eq!(result.len(), 4);
|
|
1052
|
+
assert_eq!(result[0]["id"], 0);
|
|
1053
|
+
assert_eq!(result[1]["id"], 2);
|
|
1054
|
+
assert_eq!(result[2]["id"], 5);
|
|
1055
|
+
assert_eq!(result[3]["id"], 8);
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
#[test]
|
|
1059
|
+
fn execute_plan_skips_out_of_bounds() {
|
|
1060
|
+
let c = crusher();
|
|
1061
|
+
let items: Vec<Value> = (0..3).map(|i| json!({"id": i})).collect();
|
|
1062
|
+
let plan = CompressionPlan {
|
|
1063
|
+
keep_indices: vec![0, 5, 2],
|
|
1064
|
+
..CompressionPlan::default()
|
|
1065
|
+
};
|
|
1066
|
+
let result = c.execute_plan(&plan, &items);
|
|
1067
|
+
assert_eq!(result.len(), 2);
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
#[test]
|
|
1071
|
+
fn execute_plan_factors_constants_when_enabled() {
|
|
1072
|
+
let cfg = SmartCrusherConfig {
|
|
1073
|
+
factor_out_constants: true,
|
|
1074
|
+
..Default::default()
|
|
1075
|
+
};
|
|
1076
|
+
let c = SmartCrusher::new(cfg);
|
|
1077
|
+
let items: Vec<Value> = (0..4)
|
|
1078
|
+
.map(|i| json!({"id": i, "region": "us-west-2", "status": "ok"}))
|
|
1079
|
+
.collect();
|
|
1080
|
+
let mut constant_fields = std::collections::BTreeMap::new();
|
|
1081
|
+
constant_fields.insert("region".to_string(), json!("us-west-2"));
|
|
1082
|
+
constant_fields.insert("status".to_string(), json!("ok"));
|
|
1083
|
+
let plan = CompressionPlan {
|
|
1084
|
+
keep_indices: vec![0, 1, 2],
|
|
1085
|
+
constant_fields,
|
|
1086
|
+
..CompressionPlan::default()
|
|
1087
|
+
};
|
|
1088
|
+
let result = c.execute_plan(&plan, &items);
|
|
1089
|
+
// Sentinel first, then 3 slim items.
|
|
1090
|
+
assert_eq!(result.len(), 4);
|
|
1091
|
+
assert_eq!(result[0]["_constant_fields"]["region"], "us-west-2");
|
|
1092
|
+
assert_eq!(result[0]["_constant_fields"]["status"], "ok");
|
|
1093
|
+
for item in &result[1..] {
|
|
1094
|
+
assert!(item.get("region").is_none());
|
|
1095
|
+
assert!(item.get("status").is_none());
|
|
1096
|
+
assert!(item.get("id").is_some());
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
#[test]
|
|
1101
|
+
fn execute_plan_keeps_drifted_values_when_factoring() {
|
|
1102
|
+
// Defensive strip: an item whose value differs from the recorded
|
|
1103
|
+
// constant keeps its own value.
|
|
1104
|
+
let cfg = SmartCrusherConfig {
|
|
1105
|
+
factor_out_constants: true,
|
|
1106
|
+
..Default::default()
|
|
1107
|
+
};
|
|
1108
|
+
let c = SmartCrusher::new(cfg);
|
|
1109
|
+
let items = vec![
|
|
1110
|
+
json!({"id": 0, "status": "ok"}),
|
|
1111
|
+
json!({"id": 1, "status": "FAILED"}),
|
|
1112
|
+
];
|
|
1113
|
+
let mut constant_fields = std::collections::BTreeMap::new();
|
|
1114
|
+
constant_fields.insert("status".to_string(), json!("ok"));
|
|
1115
|
+
let plan = CompressionPlan {
|
|
1116
|
+
keep_indices: vec![0, 1],
|
|
1117
|
+
constant_fields,
|
|
1118
|
+
..CompressionPlan::default()
|
|
1119
|
+
};
|
|
1120
|
+
let result = c.execute_plan(&plan, &items);
|
|
1121
|
+
assert_eq!(result.len(), 3);
|
|
1122
|
+
assert!(result[1].get("status").is_none()); // matched → stripped
|
|
1123
|
+
assert_eq!(result[2]["status"], "FAILED"); // drifted → kept
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
#[test]
|
|
1127
|
+
fn execute_plan_default_off_leaves_items_unchanged() {
|
|
1128
|
+
// factor_out_constants defaults to false: schema preserved even
|
|
1129
|
+
// when the plan carries constant_fields.
|
|
1130
|
+
let c = crusher();
|
|
1131
|
+
let items: Vec<Value> = (0..3).map(|i| json!({"id": i, "k": "v"})).collect();
|
|
1132
|
+
let mut constant_fields = std::collections::BTreeMap::new();
|
|
1133
|
+
constant_fields.insert("k".to_string(), json!("v"));
|
|
1134
|
+
let plan = CompressionPlan {
|
|
1135
|
+
keep_indices: vec![0, 1, 2],
|
|
1136
|
+
constant_fields,
|
|
1137
|
+
..CompressionPlan::default()
|
|
1138
|
+
};
|
|
1139
|
+
let result = c.execute_plan(&plan, &items);
|
|
1140
|
+
assert_eq!(result.len(), 3);
|
|
1141
|
+
assert_eq!(result[0]["k"], "v");
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
// ---------- crush_array ----------
|
|
1145
|
+
|
|
1146
|
+
#[test]
|
|
1147
|
+
fn crush_array_passthrough_when_below_adaptive_k() {
|
|
1148
|
+
let c = crusher();
|
|
1149
|
+
let items: Vec<Value> = (0..3).map(|i| json!({"id": i})).collect();
|
|
1150
|
+
let result = c.crush_array(&items, "", 1.0);
|
|
1151
|
+
assert_eq!(result.items.len(), 3);
|
|
1152
|
+
assert_eq!(result.strategy_info, "none:adaptive_at_limit");
|
|
1153
|
+
assert!(result.ccr_hash.is_none());
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
#[test]
|
|
1157
|
+
fn crush_array_skip_path_returns_original_items() {
|
|
1158
|
+
// 30 unique dict items with ID-like fields → analyzer should
|
|
1159
|
+
// detect "unique_entities_no_signal" and SKIP. Use the
|
|
1160
|
+
// no-compaction constructor so we exercise the lossy/skip
|
|
1161
|
+
// gate; the lossless path would otherwise short-circuit
|
|
1162
|
+
// because uniform-tabular input is the lossless sweet spot.
|
|
1163
|
+
let c = SmartCrusher::without_compaction(SmartCrusherConfig::default());
|
|
1164
|
+
let items: Vec<Value> = (0..30)
|
|
1165
|
+
.map(|i| json!({"id": i, "name": format!("user_{}", i)}))
|
|
1166
|
+
.collect();
|
|
1167
|
+
let result = c.crush_array(&items, "", 1.0);
|
|
1168
|
+
// skip path returns the original items unchanged.
|
|
1169
|
+
assert_eq!(result.items.len(), 30);
|
|
1170
|
+
assert!(
|
|
1171
|
+
result.strategy_info.starts_with("skip:"),
|
|
1172
|
+
"expected skip:..., got {}",
|
|
1173
|
+
result.strategy_info
|
|
1174
|
+
);
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
#[test]
|
|
1178
|
+
fn crush_array_low_uniqueness_compresses() {
|
|
1179
|
+
// 30 items with status=ok across all → low_uniqueness path
|
|
1180
|
+
// (crushable, smart_sample strategy).
|
|
1181
|
+
let c = crusher();
|
|
1182
|
+
let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();
|
|
1183
|
+
let result = c.crush_array(&items, "", 1.0);
|
|
1184
|
+
assert!(result.items.len() <= 30, "should not exceed original count");
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1187
|
+
#[test]
|
|
1188
|
+
fn crush_array_keeps_error_items() {
|
|
1189
|
+
let c = crusher();
|
|
1190
|
+
let mut items: Vec<Value> = (0..30).map(|i| json!({"id": i, "status": "ok"})).collect();
|
|
1191
|
+
items.push(json!({"id": 30, "status": "error", "msg": "FATAL"}));
|
|
1192
|
+
let result = c.crush_array(&items, "", 1.0);
|
|
1193
|
+
// Whatever path is taken, the error item should survive.
|
|
1194
|
+
assert!(
|
|
1195
|
+
result
|
|
1196
|
+
.items
|
|
1197
|
+
.iter()
|
|
1198
|
+
.any(|item| { item.get("status").and_then(|v| v.as_str()) == Some("error") }),
|
|
1199
|
+
"error item must survive crush_array"
|
|
1200
|
+
);
|
|
1201
|
+
}
|
|
1202
|
+
|
|
1203
|
+
// ---------- crush_mixed_array ----------
|
|
1204
|
+
|
|
1205
|
+
#[test]
|
|
1206
|
+
fn crush_mixed_passthrough_at_threshold() {
|
|
1207
|
+
let c = crusher();
|
|
1208
|
+
let items: Vec<Value> = vec![
|
|
1209
|
+
json!(1),
|
|
1210
|
+
json!("two"),
|
|
1211
|
+
json!({"k": "v"}),
|
|
1212
|
+
json!([1, 2]),
|
|
1213
|
+
json!(null),
|
|
1214
|
+
json!(true),
|
|
1215
|
+
json!(3),
|
|
1216
|
+
json!("four"),
|
|
1217
|
+
];
|
|
1218
|
+
let (result, strat) = c.crush_mixed_array(&items, "", 1.0);
|
|
1219
|
+
assert_eq!(result.len(), 8);
|
|
1220
|
+
assert_eq!(strat, "mixed:passthrough");
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
#[test]
|
|
1224
|
+
fn crush_mixed_groups_and_compresses_dicts() {
|
|
1225
|
+
let c = crusher();
|
|
1226
|
+
// 25 dicts (large group → gets crushed) + 5 strings (small group → all kept).
|
|
1227
|
+
let mut items: Vec<Value> = (0..25).map(|i| json!({"id": i, "status": "ok"})).collect();
|
|
1228
|
+
for i in 0..5 {
|
|
1229
|
+
items.push(json!(format!("string_{}", i)));
|
|
1230
|
+
}
|
|
1231
|
+
let (result, strat) = c.crush_mixed_array(&items, "", 1.0);
|
|
1232
|
+
assert!(strat.starts_with("mixed:adaptive("));
|
|
1233
|
+
// The 5 strings (small group) all survive.
|
|
1234
|
+
let str_count = result.iter().filter(|v| v.is_string()).count();
|
|
1235
|
+
assert_eq!(str_count, 5);
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
#[test]
|
|
1239
|
+
fn crush_mixed_keeps_lists_and_nulls_unchanged() {
|
|
1240
|
+
let c = crusher();
|
|
1241
|
+
let mut items: Vec<Value> = vec![json!([1, 2]); 6];
|
|
1242
|
+
items.extend(vec![json!(null); 6]);
|
|
1243
|
+
items.extend(vec![json!({"k": 1}); 10]);
|
|
1244
|
+
let (result, _strat) = c.crush_mixed_array(&items, "", 1.0);
|
|
1245
|
+
// Lists and nulls (not dict/str/number) → fall through to "keep all".
|
|
1246
|
+
let list_count = result.iter().filter(|v| v.is_array()).count();
|
|
1247
|
+
let null_count = result.iter().filter(|v| v.is_null()).count();
|
|
1248
|
+
assert_eq!(list_count, 6);
|
|
1249
|
+
assert_eq!(null_count, 6);
|
|
1250
|
+
}
|
|
1251
|
+
|
|
1252
|
+
#[test]
|
|
1253
|
+
fn crusher_construction_default() {
|
|
1254
|
+
let c = SmartCrusher::new(SmartCrusherConfig::default());
|
|
1255
|
+
assert_eq!(c.config.max_items_after_crush, 15);
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
// ---------- top-level crush ----------
|
|
1259
|
+
|
|
1260
|
+
#[test]
|
|
1261
|
+
fn crush_non_json_passes_through_unchanged() {
|
|
1262
|
+
let c = crusher();
|
|
1263
|
+
let result = c.crush("not json at all", "", 1.0);
|
|
1264
|
+
assert!(!result.was_modified);
|
|
1265
|
+
assert_eq!(result.compressed, "not json at all");
|
|
1266
|
+
assert_eq!(result.strategy, "passthrough");
|
|
1267
|
+
}
|
|
1268
|
+
|
|
1269
|
+
#[test]
|
|
1270
|
+
fn crush_scalar_json_passes_through() {
|
|
1271
|
+
let c = crusher();
|
|
1272
|
+
let result = c.crush("42", "", 1.0);
|
|
1273
|
+
// A scalar is not crushable; should round-trip unchanged.
|
|
1274
|
+
assert_eq!(result.compressed, "42");
|
|
1275
|
+
assert!(!result.was_modified);
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1278
|
+
#[test]
|
|
1279
|
+
fn crush_small_array_passes_through() {
|
|
1280
|
+
let c = crusher();
|
|
1281
|
+
// Compact-form input matches the compact serializer output, so
|
|
1282
|
+
// the array is not "modified" even though it round-trips
|
|
1283
|
+
// through parse → serialize. (The spaced form `[1, 2, 3]`
|
|
1284
|
+
// would mark `was_modified=true` because the compact
|
|
1285
|
+
// serializer rewrites it to `[1,2,3]`.)
|
|
1286
|
+
let result = c.crush(r#"[1,2,3]"#, "", 1.0);
|
|
1287
|
+
// Below min_items_to_analyze=5 → no crushing of the structure.
|
|
1288
|
+
assert!(!result.was_modified);
|
|
1289
|
+
assert_eq!(result.compressed, "[1,2,3]");
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
#[test]
|
|
1293
|
+
fn crush_dict_array_crushes_when_low_uniqueness() {
|
|
1294
|
+
// The public `crush()` API serializes back to JSON; the
|
|
1295
|
+
// lossless-path output (a compacted string) is exposed via
|
|
1296
|
+
// `crush_array().compacted` rather than being substituted into
|
|
1297
|
+
// the JSON re-serialization. So we exercise the lossy path
|
|
1298
|
+
// here via `without_compaction()` to validate the original
|
|
1299
|
+
// intent: low-uniqueness dicts compress via row-dropping.
|
|
1300
|
+
let c = SmartCrusher::without_compaction(SmartCrusherConfig::default());
|
|
1301
|
+
let mut input = String::from("[");
|
|
1302
|
+
for i in 0..30 {
|
|
1303
|
+
if i > 0 {
|
|
1304
|
+
input.push(',');
|
|
1305
|
+
}
|
|
1306
|
+
input.push_str(r#"{"status":"ok"}"#);
|
|
1307
|
+
}
|
|
1308
|
+
input.push(']');
|
|
1309
|
+
let result = c.crush(&input, "", 1.0);
|
|
1310
|
+
assert!(
|
|
1311
|
+
result.was_modified,
|
|
1312
|
+
"30 identical dicts should compress (low_uniqueness_safe_to_sample)"
|
|
1313
|
+
);
|
|
1314
|
+
assert_ne!(result.strategy, "passthrough");
|
|
1315
|
+
}
|
|
1316
|
+
|
|
1317
|
+
#[test]
|
|
1318
|
+
fn crush_serializes_with_python_safe_format() {
|
|
1319
|
+
let c = crusher();
|
|
1320
|
+
// SmartCrusher uses Python's `safe_json_dumps`: compact
|
|
1321
|
+
// separators `(",", ":")` + `ensure_ascii=False`, preserving
|
|
1322
|
+
// object-key insertion order. A spaced input round-trips to
|
|
1323
|
+
// the compact form.
|
|
1324
|
+
let input = r#"{"a": 1, "b": 2, "c": 3}"#;
|
|
1325
|
+
let result = c.crush(input, "", 1.0);
|
|
1326
|
+
assert_eq!(
|
|
1327
|
+
result.compressed, r#"{"a":1,"b":2,"c":3}"#,
|
|
1328
|
+
"safe_json_dumps emits compact `,` / `:` separators"
|
|
1329
|
+
);
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1332
|
+
#[test]
|
|
1333
|
+
fn crush_recurses_into_nested_arrays() {
|
|
1334
|
+
let c = crusher();
|
|
1335
|
+
// Top-level dict with a nested array of 30 identical items.
|
|
1336
|
+
// The inner array should compress (low_uniqueness path).
|
|
1337
|
+
let mut inner = String::from("[");
|
|
1338
|
+
for i in 0..30 {
|
|
1339
|
+
if i > 0 {
|
|
1340
|
+
inner.push(',');
|
|
1341
|
+
}
|
|
1342
|
+
inner.push_str(r#"{"status":"ok"}"#);
|
|
1343
|
+
}
|
|
1344
|
+
inner.push(']');
|
|
1345
|
+
let input = format!(r#"{{"data": {}}}"#, inner);
|
|
1346
|
+
let result = c.crush(&input, "", 1.0);
|
|
1347
|
+
assert!(
|
|
1348
|
+
result.was_modified,
|
|
1349
|
+
"nested compressible array must be crushed even inside a wrapper object"
|
|
1350
|
+
);
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
#[test]
|
|
1354
|
+
fn crusher_with_custom_scorer() {
|
|
1355
|
+
use crate::relevance::BM25Scorer;
|
|
1356
|
+
let c = SmartCrusher::with_scorer(
|
|
1357
|
+
SmartCrusherConfig::default(),
|
|
1358
|
+
Box::new(BM25Scorer::default()),
|
|
1359
|
+
);
|
|
1360
|
+
// Sanity: crushing still works with a swapped scorer.
|
|
1361
|
+
let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();
|
|
1362
|
+
let result = c.crush_array(&items, "anything", 1.0);
|
|
1363
|
+
assert!(result.items.len() <= 30);
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
// ---------- Stage 3c.2 PR4: lossless-first default with threshold + CCR-Dropped ----------
|
|
1367
|
+
|
|
1368
|
+
#[test]
|
|
1369
|
+
fn without_compaction_yields_none_compacted_field() {
|
|
1370
|
+
// The opt-out constructor preserves pre-PR4 lossy-only path.
|
|
1371
|
+
// No lossless attempt → compacted/compaction_kind always None.
|
|
1372
|
+
let c = SmartCrusher::without_compaction(SmartCrusherConfig::default());
|
|
1373
|
+
let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();
|
|
1374
|
+
let result = c.crush_array(&items, "", 1.0);
|
|
1375
|
+
assert!(result.compacted.is_none());
|
|
1376
|
+
assert!(result.compaction_kind.is_none());
|
|
1377
|
+
}
|
|
1378
|
+
|
|
1379
|
+
#[test]
|
|
1380
|
+
fn lossless_wins_when_savings_above_threshold() {
|
|
1381
|
+
// 50 uniform tabular dicts → CSV+schema compaction shrinks
|
|
1382
|
+
// the input dramatically (well above the 0.30 default).
|
|
1383
|
+
// Default `SmartCrusher::new()` should pick lossless.
|
|
1384
|
+
let c = crusher();
|
|
1385
|
+
let items: Vec<Value> = (0..50)
|
|
1386
|
+
.map(|i| json!({"id": i, "name": format!("u_{i}"), "status": "ok"}))
|
|
1387
|
+
.collect();
|
|
1388
|
+
let result = c.crush_array(&items, "", 1.0);
|
|
1389
|
+
let compacted = result.compacted.expect("compacted should be set");
|
|
1390
|
+
assert!(compacted.starts_with("[50]{"), "got: {compacted}");
|
|
1391
|
+
assert_eq!(result.compaction_kind, Some("table"));
|
|
1392
|
+
assert!(
|
|
1393
|
+
result.strategy_info.starts_with("lossless:table"),
|
|
1394
|
+
"got: {}",
|
|
1395
|
+
result.strategy_info
|
|
1396
|
+
);
|
|
1397
|
+
// Lossless = nothing dropped → no CCR retrieval needed.
|
|
1398
|
+
assert!(result.ccr_hash.is_none());
|
|
1399
|
+
// items preserved (full original).
|
|
1400
|
+
assert_eq!(result.items.len(), 50);
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1403
|
+
#[test]
|
|
1404
|
+
fn lossy_falls_through_when_savings_below_threshold() {
|
|
1405
|
+
// Force the threshold high enough that even tabular savings
|
|
1406
|
+
// can't satisfy it → lossy path runs → CCR-Dropped fires.
|
|
1407
|
+
// Use low-uniqueness items so the analyzer is willing to
|
|
1408
|
+
// crush (unique id+name per row would trip the
|
|
1409
|
+
// "unique_entities_no_signal" skip gate instead).
|
|
1410
|
+
let cfg = SmartCrusherConfig {
|
|
1411
|
+
lossless_min_savings_ratio: 0.99,
|
|
1412
|
+
..Default::default()
|
|
1413
|
+
};
|
|
1414
|
+
let c = SmartCrusher::new(cfg);
|
|
1415
|
+
let items: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();
|
|
1416
|
+
let result = c.crush_array(&items, "", 1.0);
|
|
1417
|
+
// Lossless declined → no compacted output.
|
|
1418
|
+
assert!(result.compacted.is_none());
|
|
1419
|
+
// Lossy ran → rows dropped.
|
|
1420
|
+
assert!(
|
|
1421
|
+
result.items.len() < 50,
|
|
1422
|
+
"expected lossy drop, got {} items",
|
|
1423
|
+
result.items.len()
|
|
1424
|
+
);
|
|
1425
|
+
// CCR hash populated for retrieval.
|
|
1426
|
+
let h = result.ccr_hash.expect("ccr_hash populated on drop");
|
|
1427
|
+
assert_eq!(h.len(), 12);
|
|
1428
|
+
// Marker visible in dropped_summary.
|
|
1429
|
+
assert!(
|
|
1430
|
+
result.dropped_summary.contains(&format!("<<ccr:{h}")),
|
|
1431
|
+
"got: {}",
|
|
1432
|
+
result.dropped_summary
|
|
1433
|
+
);
|
|
1434
|
+
assert!(result.dropped_summary.contains("rows_offloaded"));
|
|
1435
|
+
}
|
|
1436
|
+
|
|
1437
|
+
#[test]
|
|
1438
|
+
fn ccr_hash_is_deterministic() {
|
|
1439
|
+
// Same input → same hash, so the runtime cache key is stable.
|
|
1440
|
+
let cfg = SmartCrusherConfig {
|
|
1441
|
+
lossless_min_savings_ratio: 0.99, // force lossy path
|
|
1442
|
+
..Default::default()
|
|
1443
|
+
};
|
|
1444
|
+
let c = SmartCrusher::new(cfg);
|
|
1445
|
+
let items: Vec<Value> = (0..30).map(|i| json!({"id": i, "tag": "ok"})).collect();
|
|
1446
|
+
let r1 = c.crush_array(&items, "", 1.0);
|
|
1447
|
+
let r2 = c.crush_array(&items, "", 1.0);
|
|
1448
|
+
assert_eq!(r1.ccr_hash, r2.ccr_hash);
|
|
1449
|
+
assert!(r1.ccr_hash.is_some());
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
#[test]
|
|
1453
|
+
fn ccr_hash_changes_with_input() {
|
|
1454
|
+
let cfg = SmartCrusherConfig {
|
|
1455
|
+
lossless_min_savings_ratio: 0.99,
|
|
1456
|
+
..Default::default()
|
|
1457
|
+
};
|
|
1458
|
+
let c = SmartCrusher::new(cfg);
|
|
1459
|
+
let a: Vec<Value> = (0..30).map(|i| json!({"id": i})).collect();
|
|
1460
|
+
let b: Vec<Value> = (100..130).map(|i| json!({"id": i})).collect();
|
|
1461
|
+
let ra = c.crush_array(&a, "", 1.0);
|
|
1462
|
+
let rb = c.crush_array(&b, "", 1.0);
|
|
1463
|
+
assert_ne!(ra.ccr_hash, rb.ccr_hash);
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
#[test]
|
|
1467
|
+
fn lossy_without_compaction_still_emits_ccr_hash() {
|
|
1468
|
+
// The CCR-Dropped restoration applies regardless of whether
|
|
1469
|
+
// lossless was attempted — without_compaction also gets the
|
|
1470
|
+
// ccr_hash on row drops.
|
|
1471
|
+
let c = SmartCrusher::without_compaction(SmartCrusherConfig::default());
|
|
1472
|
+
let items: Vec<Value> = (0..30).map(|_| json!({"status": "ok"})).collect();
|
|
1473
|
+
let result = c.crush_array(&items, "", 1.0);
|
|
1474
|
+
if result.items.len() < items.len() {
|
|
1475
|
+
assert!(result.ccr_hash.is_some());
|
|
1476
|
+
assert!(!result.dropped_summary.is_empty());
|
|
1477
|
+
}
|
|
1478
|
+
}
|
|
1479
|
+
|
|
1480
|
+
#[test]
|
|
1481
|
+
fn passthrough_paths_do_not_emit_ccr_hash() {
|
|
1482
|
+
// Tier-1 boundary (items.len() <= adaptive_k): nothing
|
|
1483
|
+
// dropped, no CCR. Skip path: same.
|
|
1484
|
+
let c = crusher();
|
|
1485
|
+
let small: Vec<Value> = (0..3).map(|i| json!({"id": i})).collect();
|
|
1486
|
+
let r = c.crush_array(&small, "", 1.0);
|
|
1487
|
+
assert!(r.ccr_hash.is_none());
|
|
1488
|
+
assert_eq!(r.dropped_summary, "");
|
|
1489
|
+
}
|
|
1490
|
+
|
|
1491
|
+
#[test]
|
|
1492
|
+
fn compaction_skips_non_object_array() {
|
|
1493
|
+
// Compactor returns Untouched for non-object arrays → no
|
|
1494
|
+
// compacted field populated, no kind tag.
|
|
1495
|
+
let c = SmartCrusherBuilder::new(SmartCrusherConfig::default())
|
|
1496
|
+
.with_default_oss_setup()
|
|
1497
|
+
.with_default_compaction()
|
|
1498
|
+
.build();
|
|
1499
|
+
let items: Vec<Value> = (0..30).map(|i| json!(i)).collect();
|
|
1500
|
+
let result = c.crush_array(&items, "", 1.0);
|
|
1501
|
+
assert!(result.compacted.is_none());
|
|
1502
|
+
assert!(result.compaction_kind.is_none());
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
// ---------- Stage 3c.2 PR5: walker-integration in process_value ----------
|
|
1506
|
+
|
|
1507
|
+
#[test]
|
|
1508
|
+
fn process_string_short_string_passthrough() {
|
|
1509
|
+
let c = SmartCrusher::new(SmartCrusherConfig::default());
|
|
1510
|
+
let (out, info) = c.process_value(&json!("hello world"), 0, "", 1.0);
|
|
1511
|
+
assert_eq!(out, json!("hello world"));
|
|
1512
|
+
assert!(info.is_empty());
|
|
1513
|
+
}
|
|
1514
|
+
|
|
1515
|
+
#[test]
|
|
1516
|
+
fn process_string_stringified_json_array_recurses() {
|
|
1517
|
+
// A string-typed field whose value is a JSON-encoded array of
|
|
1518
|
+
// dicts. process_value should parse it, recurse, and return
|
|
1519
|
+
// the processed JSON re-rendered as a string.
|
|
1520
|
+
let c = SmartCrusher::new(SmartCrusherConfig::default());
|
|
1521
|
+
let big_array_json = serde_json::to_string(
|
|
1522
|
+
&(0..50)
|
|
1523
|
+
.map(|i| json!({"id": i, "level": "info", "msg": "ok"}))
|
|
1524
|
+
.collect::<Vec<_>>(),
|
|
1525
|
+
)
|
|
1526
|
+
.unwrap();
|
|
1527
|
+
let doc = json!({"payload": big_array_json.clone()});
|
|
1528
|
+
let (out, info) = c.process_value(&doc, 0, "", 1.0);
|
|
1529
|
+
// payload still a string-typed field — we preserved the
|
|
1530
|
+
// wrapping shape — but its content was processed.
|
|
1531
|
+
let payload = out.pointer("/payload").and_then(|v| v.as_str()).unwrap();
|
|
1532
|
+
// Either compressed or unchanged; if compressed, info reflects.
|
|
1533
|
+
// For 50 items with low-uniqueness, compression should fire.
|
|
1534
|
+
// The strategy info should mention string_json processing.
|
|
1535
|
+
assert!(
|
|
1536
|
+
info.contains("string_json") || payload != big_array_json,
|
|
1537
|
+
"expected processing trace; info={info}, len before={}, after={}",
|
|
1538
|
+
big_array_json.len(),
|
|
1539
|
+
payload.len(),
|
|
1540
|
+
);
|
|
1541
|
+
}
|
|
1542
|
+
|
|
1543
|
+
#[test]
|
|
1544
|
+
fn process_string_opaque_blob_becomes_ccr_marker() {
|
|
1545
|
+
let c = SmartCrusher::new(SmartCrusherConfig::default());
|
|
1546
|
+
let big_b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".repeat(8);
|
|
1547
|
+
let doc = json!({"id": 1, "blob": big_b64});
|
|
1548
|
+
let (out, _info) = c.process_value(&doc, 0, "", 1.0);
|
|
1549
|
+
let blob = out.pointer("/blob").and_then(|v| v.as_str()).unwrap();
|
|
1550
|
+
assert!(blob.starts_with("<<ccr:"), "got: {blob}");
|
|
1551
|
+
assert!(blob.contains(",base64,"));
|
|
1552
|
+
}
|
|
1553
|
+
|
|
1554
|
+
#[test]
|
|
1555
|
+
fn process_string_top_level_string_processed() {
|
|
1556
|
+
// crush() takes a string; if it doesn't parse as JSON, today's
|
|
1557
|
+
// behavior returns it unchanged. But if it's a stringified
|
|
1558
|
+
// JSON object/array, it should now get processed.
|
|
1559
|
+
let c = SmartCrusher::new(SmartCrusherConfig::default());
|
|
1560
|
+
// Non-JSON top-level string — passthrough.
|
|
1561
|
+
let plain = "just some plain text";
|
|
1562
|
+
let result = c.crush(plain, "", 1.0);
|
|
1563
|
+
assert_eq!(result.compressed, plain);
|
|
1564
|
+
}
|
|
1565
|
+
|
|
1566
|
+
#[test]
|
|
1567
|
+
fn process_string_does_not_alter_short_quoted_strings() {
|
|
1568
|
+
// Strings that look JSON-like but are short shouldn't be
|
|
1569
|
+
// CCR-substituted.
|
|
1570
|
+
let c = SmartCrusher::new(SmartCrusherConfig::default());
|
|
1571
|
+
let doc = json!({"msg": "{this looks like json but isnt}"});
|
|
1572
|
+
let (out, _) = c.process_value(&doc, 0, "", 1.0);
|
|
1573
|
+
assert_eq!(out, doc);
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
#[test]
|
|
1577
|
+
fn process_string_helper_parses_only_containers() {
|
|
1578
|
+
assert!(try_parse_json_container("{\"a\":1}").is_some());
|
|
1579
|
+
assert!(try_parse_json_container("[1,2,3]").is_some());
|
|
1580
|
+
assert!(try_parse_json_container("123").is_none()); // bare scalar
|
|
1581
|
+
assert!(try_parse_json_container("\"hello\"").is_none()); // bare string
|
|
1582
|
+
assert!(try_parse_json_container("not json").is_none());
|
|
1583
|
+
assert!(try_parse_json_container("{malformed").is_none());
|
|
1584
|
+
}
|
|
1585
|
+
|
|
1586
|
+
// ---------- enable_ccr_marker gate (PR #301 re-land) ----------
|
|
1587
|
+
|
|
1588
|
+
#[test]
|
|
1589
|
+
fn enable_ccr_marker_false_suppresses_marker_and_store() {
|
|
1590
|
+
// The Rust-side gate. Compression still runs (rows drop) but
|
|
1591
|
+
// the result carries no marker text, no hash, and the CCR
|
|
1592
|
+
// store does NOT grow — there's no point storing what nothing
|
|
1593
|
+
// in the prompt can reference.
|
|
1594
|
+
use crate::ccr::InMemoryCcrStore;
|
|
1595
|
+
use crate::transforms::smart_crusher::SmartCrusherBuilder;
|
|
1596
|
+
use std::sync::Arc;
|
|
1597
|
+
|
|
1598
|
+
let store: Arc<dyn CcrStore> = Arc::new(InMemoryCcrStore::new());
|
|
1599
|
+
let cfg = SmartCrusherConfig {
|
|
1600
|
+
lossless_min_savings_ratio: 0.99, // force lossy path
|
|
1601
|
+
enable_ccr_marker: false,
|
|
1602
|
+
..SmartCrusherConfig::default()
|
|
1603
|
+
};
|
|
1604
|
+
let c = SmartCrusherBuilder::new(cfg)
|
|
1605
|
+
.with_ccr_store(Arc::clone(&store))
|
|
1606
|
+
.build();
|
|
1607
|
+
let items: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();
|
|
1608
|
+
|
|
1609
|
+
let store_len_before = store.len();
|
|
1610
|
+
let result = c.crush_array(&items, "", 1.0);
|
|
1611
|
+
let store_len_after = store.len();
|
|
1612
|
+
|
|
1613
|
+
// Rows were dropped (we built 50, kept fewer).
|
|
1614
|
+
assert!(result.items.len() < items.len(), "lossy path didn't fire");
|
|
1615
|
+
// Gate held: no marker, no hash.
|
|
1616
|
+
assert!(result.ccr_hash.is_none(), "ccr_hash should be None");
|
|
1617
|
+
assert!(
|
|
1618
|
+
result.dropped_summary.is_empty(),
|
|
1619
|
+
"dropped_summary should be empty, got: {:?}",
|
|
1620
|
+
result.dropped_summary
|
|
1621
|
+
);
|
|
1622
|
+
// Store did NOT grow.
|
|
1623
|
+
assert_eq!(
|
|
1624
|
+
store_len_after, store_len_before,
|
|
1625
|
+
"ccr_store grew despite enable_ccr_marker=false"
|
|
1626
|
+
);
|
|
1627
|
+
}
|
|
1628
|
+
|
|
1629
|
+
#[test]
|
|
1630
|
+
fn enable_ccr_marker_true_is_default_behavior() {
|
|
1631
|
+
// Default config still emits markers + stores when rows drop.
|
|
1632
|
+
// Sanity: the gate is opt-out, not opt-in.
|
|
1633
|
+
use crate::ccr::InMemoryCcrStore;
|
|
1634
|
+
use crate::transforms::smart_crusher::SmartCrusherBuilder;
|
|
1635
|
+
use std::sync::Arc;
|
|
1636
|
+
|
|
1637
|
+
let store: Arc<dyn CcrStore> = Arc::new(InMemoryCcrStore::new());
|
|
1638
|
+
let cfg = SmartCrusherConfig {
|
|
1639
|
+
lossless_min_savings_ratio: 0.99, // force lossy path
|
|
1640
|
+
..SmartCrusherConfig::default()
|
|
1641
|
+
};
|
|
1642
|
+
// Default: enable_ccr_marker = true.
|
|
1643
|
+
assert!(cfg.enable_ccr_marker);
|
|
1644
|
+
let c = SmartCrusherBuilder::new(cfg)
|
|
1645
|
+
.with_ccr_store(Arc::clone(&store))
|
|
1646
|
+
.build();
|
|
1647
|
+
let items: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();
|
|
1648
|
+
|
|
1649
|
+
let store_len_before = store.len();
|
|
1650
|
+
let result = c.crush_array(&items, "", 1.0);
|
|
1651
|
+
let store_len_after = store.len();
|
|
1652
|
+
|
|
1653
|
+
assert!(result.items.len() < items.len(), "lossy path didn't fire");
|
|
1654
|
+
assert!(result.ccr_hash.is_some(), "default should produce a hash");
|
|
1655
|
+
assert!(
|
|
1656
|
+
result.dropped_summary.contains("<<ccr:"),
|
|
1657
|
+
"default should produce a marker: {:?}",
|
|
1658
|
+
result.dropped_summary
|
|
1659
|
+
);
|
|
1660
|
+
assert!(
|
|
1661
|
+
store_len_after > store_len_before,
|
|
1662
|
+
"default should write to ccr_store"
|
|
1663
|
+
);
|
|
1664
|
+
}
|
|
1665
|
+
}
|