xithead 1.5.0 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/design-system/README.md +80 -0
- package/package.json +20 -1
- package/public/apple-touch-icon.png +0 -0
- package/public/logo-transparent.png +0 -0
- package/public/xithead.ps1 +937 -0
- package/scripts/README.md +77 -0
- package/scripts/audit_wheel_glibc_symbols.py +217 -0
- package/scripts/build-master-design-system.cjs +33 -0
- package/scripts/build_rust_extension.sh +58 -0
- package/scripts/changelog-gen.py +203 -0
- package/scripts/eval_output_shaper.py +235 -0
- package/scripts/export_kompress_v2_onnx.py +267 -0
- package/scripts/fixtures/anthropic_replay_body.json +133 -0
- package/scripts/fixtures/codex_response_create_frame.json +52 -0
- package/scripts/install-git-hooks.sh +104 -0
- package/scripts/install.ps1 +1809 -0
- package/scripts/install.sh +1673 -0
- package/scripts/pr-governance.py +297 -0
- package/scripts/record_fixtures.py +59 -0
- package/scripts/refresh_model_limits.sh +48 -0
- package/scripts/replay_codex_ws_load.py +445 -0
- package/scripts/repro_codex_replay.py +699 -0
- package/scripts/smoke_issue_327.py +233 -0
- package/scripts/sync-plugin-versions.py +116 -0
- package/scripts/tests/test_changelog_gen.py +302 -0
- package/scripts/tests/test_pr_governance.py +163 -0
- package/scripts/tests/test_pr_health_labels.py +94 -0
- package/scripts/tests/test_pr_health_workflow.py +14 -0
- package/scripts/tests/test_sync_plugin_versions.py +131 -0
- package/scripts/tests/test_version_sync.py +291 -0
- package/scripts/validate-workflows.sh +39 -0
- package/scripts/verify-versions.py +106 -0
- package/scripts/version-sync.py +207 -0
- package/tailwind.config.js +31 -0
- package/tsconfig.json +84 -0
- package/.actrc +0 -14
- package/.actrc.local.example +0 -15
- package/.changelog.md +0 -2
- package/.claude/agents/academic-anthropologist.md +0 -125
- package/.claude/agents/academic-geographer.md +0 -127
- package/.claude/agents/academic-historian.md +0 -123
- package/.claude/agents/academic-narratologist.md +0 -118
- package/.claude/agents/academic-psychologist.md +0 -118
- package/.claude/agents/academic-statistician.md +0 -144
- package/.claude/agents/accounts-payable-agent.md +0 -205
- package/.claude/agents/agentic-identity-trust.md +0 -394
- package/.claude/agents/agents-orchestrator.md +0 -367
- package/.claude/agents/automation-governance-architect.md +0 -216
- package/.claude/agents/blender-addon-engineer.md +0 -243
- package/.claude/agents/business-strategist.md +0 -488
- package/.claude/agents/change-management-consultant.md +0 -497
- package/.claude/agents/chief-financial-officer.md +0 -388
- package/.claude/agents/corporate-training-designer.md +0 -192
- package/.claude/agents/customer-service.md +0 -398
- package/.claude/agents/customer-success-manager.md +0 -460
- package/.claude/agents/data-consolidation-agent.md +0 -60
- package/.claude/agents/data-privacy-officer.md +0 -412
- package/.claude/agents/design-brand-guardian.md +0 -322
- package/.claude/agents/design-image-prompt-engineer.md +0 -236
- package/.claude/agents/design-inclusive-visuals-specialist.md +0 -73
- package/.claude/agents/design-persona-walkthrough.md +0 -272
- package/.claude/agents/design-ui-designer.md +0 -386
- package/.claude/agents/design-ui-finish-gate-reviewer.md +0 -217
- package/.claude/agents/design-ux-architect.md +0 -482
- package/.claude/agents/design-ux-researcher.md +0 -329
- package/.claude/agents/design-visual-storyteller.md +0 -149
- package/.claude/agents/design-whimsy-injector.md +0 -460
- package/.claude/agents/economy-designer.md +0 -156
- package/.claude/agents/engineering-ai-data-remediation-engineer.md +0 -211
- package/.claude/agents/engineering-ai-engineer.md +0 -146
- package/.claude/agents/engineering-api-platform-engineer.md +0 -181
- package/.claude/agents/engineering-ats-validator-architect.md +0 -383
- package/.claude/agents/engineering-autonomous-optimization-architect.md +0 -121
- package/.claude/agents/engineering-backend-architect.md +0 -255
- package/.claude/agents/engineering-china-network-engineer.md +0 -252
- package/.claude/agents/engineering-cms-developer.md +0 -536
- package/.claude/agents/engineering-code-reviewer.md +0 -76
- package/.claude/agents/engineering-codebase-onboarding-engineer.md +0 -173
- package/.claude/agents/engineering-data-engineer.md +0 -349
- package/.claude/agents/engineering-data-visualization-engineer.md +0 -151
- package/.claude/agents/engineering-database-optimizer.md +0 -188
- package/.claude/agents/engineering-database-reliability-engineer.md +0 -201
- package/.claude/agents/engineering-desktop-app-engineer.md +0 -209
- package/.claude/agents/engineering-developer-tooling-engineer.md +0 -153
- package/.claude/agents/engineering-devops-automator.md +0 -385
- package/.claude/agents/engineering-drupal-performance.md +0 -347
- package/.claude/agents/engineering-drupal-shopping-cart.md +0 -360
- package/.claude/agents/engineering-electrical-engineer.md +0 -575
- package/.claude/agents/engineering-email-intelligence-engineer.md +0 -390
- package/.claude/agents/engineering-embedded-firmware-engineer.md +0 -196
- package/.claude/agents/engineering-feishu-integration-developer.md +0 -606
- package/.claude/agents/engineering-filament-optimization-specialist.md +0 -283
- package/.claude/agents/engineering-finops-engineer.md +0 -164
- package/.claude/agents/engineering-frontend-developer.md +0 -257
- package/.claude/agents/engineering-gaussdb-expert.md +0 -334
- package/.claude/agents/engineering-git-workflow-master.md +0 -98
- package/.claude/agents/engineering-i18n-engineer.md +0 -214
- package/.claude/agents/engineering-identity-access-engineer.md +0 -217
- package/.claude/agents/engineering-incident-response-commander.md +0 -444
- package/.claude/agents/engineering-iot-fleet-engineer.md +0 -148
- package/.claude/agents/engineering-it-service-manager.md +0 -561
- package/.claude/agents/engineering-knowledge-graph-engineer.md +0 -372
- package/.claude/agents/engineering-llm-post-training-engineer.md +0 -166
- package/.claude/agents/engineering-minimal-change-engineer.md +0 -207
- package/.claude/agents/engineering-mobile-app-builder.md +0 -493
- package/.claude/agents/engineering-mobile-release-engineer.md +0 -164
- package/.claude/agents/engineering-multi-agent-systems-architect.md +0 -600
- package/.claude/agents/engineering-network-engineer.md +0 -245
- package/.claude/agents/engineering-orgscript-engineer.md +0 -113
- package/.claude/agents/engineering-payments-billing-engineer.md +0 -228
- package/.claude/agents/engineering-pdf-engine-architect.md +0 -696
- package/.claude/agents/engineering-platform-engineer.md +0 -270
- package/.claude/agents/engineering-privacy-engineer.md +0 -152
- package/.claude/agents/engineering-prompt-engineer.md +0 -209
- package/.claude/agents/engineering-rag-pipeline-engineer.md +0 -450
- package/.claude/agents/engineering-rapid-prototyper.md +0 -466
- package/.claude/agents/engineering-realtime-collaboration-engineer.md +0 -200
- package/.claude/agents/engineering-rust-refactoring-specialist.md +0 -313
- package/.claude/agents/engineering-search-relevance-engineer.md +0 -237
- package/.claude/agents/engineering-section-508-specialist.md +0 -339
- package/.claude/agents/engineering-senior-developer.md +0 -176
- package/.claude/agents/engineering-servicenow-developer-mentor.md +0 -180
- package/.claude/agents/engineering-software-architect.md +0 -112
- package/.claude/agents/engineering-solidity-smart-contract-engineer.md +0 -522
- package/.claude/agents/engineering-sre.md +0 -90
- package/.claude/agents/engineering-technical-writer.md +0 -393
- package/.claude/agents/engineering-universal-document-compiler.md +0 -370
- package/.claude/agents/engineering-uswds-developer.md +0 -340
- package/.claude/agents/engineering-video-streaming-engineer.md +0 -151
- package/.claude/agents/engineering-voice-ai-integration-engineer.md +0 -584
- package/.claude/agents/engineering-webassembly-engineer.md +0 -153
- package/.claude/agents/engineering-wechat-mini-program-developer.md +0 -356
- package/.claude/agents/engineering-wordpress-performance.md +0 -346
- package/.claude/agents/engineering-wordpress-shopping-cart.md +0 -346
- package/.claude/agents/esg-sustainability-officer.md +0 -396
- package/.claude/agents/finance-bookkeeper-controller.md +0 -260
- package/.claude/agents/finance-financial-analyst.md +0 -234
- package/.claude/agents/finance-fpa-analyst.md +0 -263
- package/.claude/agents/finance-investment-researcher.md +0 -272
- package/.claude/agents/finance-tax-strategist.md +0 -239
- package/.claude/agents/game-audio-engineer.md +0 -264
- package/.claude/agents/game-designer.md +0 -167
- package/.claude/agents/gis-3d-scene-developer.md +0 -111
- package/.claude/agents/gis-analyst.md +0 -91
- package/.claude/agents/gis-bim-specialist.md +0 -108
- package/.claude/agents/gis-cartography-designer.md +0 -150
- package/.claude/agents/gis-drone-reality-mapping.md +0 -120
- package/.claude/agents/gis-geoai-ml-engineer.md +0 -105
- package/.claude/agents/gis-geoprocessing-specialist.md +0 -97
- package/.claude/agents/gis-qa-engineer.md +0 -133
- package/.claude/agents/gis-solution-engineer.md +0 -101
- package/.claude/agents/gis-spatial-data-engineer.md +0 -97
- package/.claude/agents/gis-spatial-data-scientist.md +0 -111
- package/.claude/agents/gis-technical-consultant.md +0 -86
- package/.claude/agents/gis-web-gis-developer.md +0 -108
- package/.claude/agents/godot-gameplay-scripter.md +0 -338
- package/.claude/agents/godot-multiplayer-engineer.md +0 -301
- package/.claude/agents/godot-shader-developer.md +0 -267
- package/.claude/agents/government-digital-presales-consultant.md +0 -363
- package/.claude/agents/grant-writer.md +0 -511
- package/.claude/agents/healthcare-aging-parent-care-companion.md +0 -414
- package/.claude/agents/healthcare-clinical-evidence-agent.md +0 -231
- package/.claude/agents/healthcare-customer-service.md +0 -389
- package/.claude/agents/healthcare-innovation-strategist.md +0 -433
- package/.claude/agents/healthcare-marketing-compliance.md +0 -395
- package/.claude/agents/healthcare-sovereign-health-systems-agent.md +0 -312
- package/.claude/agents/hospitality-guest-services.md +0 -603
- package/.claude/agents/hr-onboarding.md +0 -451
- package/.claude/agents/identity-graph-operator.md +0 -303
- package/.claude/agents/language-translator.md +0 -264
- package/.claude/agents/legal-billing-time-tracking.md +0 -569
- package/.claude/agents/legal-client-intake.md +0 -492
- package/.claude/agents/legal-document-review.md +0 -454
- package/.claude/agents/level-designer.md +0 -208
- package/.claude/agents/loan-officer-assistant.md +0 -555
- package/.claude/agents/lsp-index-engineer.md +0 -320
- package/.claude/agents/ma-integration-manager.md +0 -427
- package/.claude/agents/macos-spatial-metal-engineer.md +0 -337
- package/.claude/agents/marketing-aeo-foundations.md +0 -264
- package/.claude/agents/marketing-agentic-search-optimizer.md +0 -279
- package/.claude/agents/marketing-ai-citation-strategist.md +0 -172
- package/.claude/agents/marketing-app-store-optimizer.md +0 -321
- package/.claude/agents/marketing-baidu-seo-specialist.md +0 -226
- package/.claude/agents/marketing-bilibili-content-strategist.md +0 -199
- package/.claude/agents/marketing-book-co-author.md +0 -110
- package/.claude/agents/marketing-carousel-growth-engine.md +0 -204
- package/.claude/agents/marketing-china-ecommerce-operator.md +0 -283
- package/.claude/agents/marketing-china-market-localization-strategist.md +0 -283
- package/.claude/agents/marketing-content-creator.md +0 -54
- package/.claude/agents/marketing-cross-border-ecommerce.md +0 -259
- package/.claude/agents/marketing-developer-community-builder.md +0 -285
- package/.claude/agents/marketing-douyin-strategist.md +0 -149
- package/.claude/agents/marketing-email-strategist.md +0 -249
- package/.claude/agents/marketing-global-podcast-strategist.md +0 -206
- package/.claude/agents/marketing-growth-hacker.md +0 -54
- package/.claude/agents/marketing-instagram-curator.md +0 -113
- package/.claude/agents/marketing-kuaishou-strategist.md +0 -223
- package/.claude/agents/marketing-linkedin-content-creator.md +0 -214
- package/.claude/agents/marketing-livestream-commerce-coach.md +0 -305
- package/.claude/agents/marketing-multi-platform-publisher.md +0 -217
- package/.claude/agents/marketing-podcast-strategist.md +0 -277
- package/.claude/agents/marketing-pr-communications-manager.md +0 -473
- package/.claude/agents/marketing-private-domain-operator.md +0 -308
- package/.claude/agents/marketing-reddit-community-builder.md +0 -123
- package/.claude/agents/marketing-seo-specialist.md +0 -370
- package/.claude/agents/marketing-short-video-editing-coach.md +0 -412
- package/.claude/agents/marketing-social-media-strategist.md +0 -125
- package/.claude/agents/marketing-tiktok-strategist.md +0 -125
- package/.claude/agents/marketing-twitter-engager.md +0 -126
- package/.claude/agents/marketing-video-optimization-specialist.md +0 -119
- package/.claude/agents/marketing-wechat-official-account.md +0 -145
- package/.claude/agents/marketing-weibo-strategist.md +0 -240
- package/.claude/agents/marketing-x-twitter-intelligence-analyst.md +0 -161
- package/.claude/agents/marketing-xiaohongshu-specialist.md +0 -138
- package/.claude/agents/marketing-zhihu-strategist.md +0 -162
- package/.claude/agents/medical-billing-coding-specialist.md +0 -491
- package/.claude/agents/narrative-designer.md +0 -243
- package/.claude/agents/operations-manager.md +0 -399
- package/.claude/agents/organizational-psychologist.md +0 -391
- package/.claude/agents/paid-media-auditor.md +0 -71
- package/.claude/agents/paid-media-creative-strategist.md +0 -71
- package/.claude/agents/paid-media-paid-social-strategist.md +0 -71
- package/.claude/agents/paid-media-ppc-strategist.md +0 -71
- package/.claude/agents/paid-media-programmatic-buyer.md +0 -71
- package/.claude/agents/paid-media-search-query-analyst.md +0 -71
- package/.claude/agents/paid-media-tracking-specialist.md +0 -71
- package/.claude/agents/personal-growth-mentor.md +0 -159
- package/.claude/agents/product-behavioral-nudge-engine.md +0 -83
- package/.claude/agents/product-dx-engineer.md +0 -332
- package/.claude/agents/product-feedback-synthesizer.md +0 -119
- package/.claude/agents/product-manager.md +0 -469
- package/.claude/agents/product-sprint-prioritizer.md +0 -154
- package/.claude/agents/product-trend-researcher.md +0 -159
- package/.claude/agents/project-management-experiment-tracker.md +0 -198
- package/.claude/agents/project-management-jira-workflow-steward.md +0 -233
- package/.claude/agents/project-management-meeting-notes-specialist.md +0 -95
- package/.claude/agents/project-management-project-shepherd.md +0 -194
- package/.claude/agents/project-management-studio-operations.md +0 -200
- package/.claude/agents/project-management-studio-producer.md +0 -203
- package/.claude/agents/project-manager-senior.md +0 -135
- package/.claude/agents/real-estate-buyer-seller.md +0 -596
- package/.claude/agents/recruitment-specialist.md +0 -515
- package/.claude/agents/report-distribution-agent.md +0 -65
- package/.claude/agents/research-synthesist.md +0 -136
- package/.claude/agents/resume-tailor.md +0 -230
- package/.claude/agents/retail-customer-returns.md +0 -566
- package/.claude/agents/roblox-avatar-creator.md +0 -297
- package/.claude/agents/roblox-experience-designer.md +0 -306
- package/.claude/agents/roblox-systems-scripter.md +0 -332
- package/.claude/agents/sales-account-strategist.md +0 -227
- package/.claude/agents/sales-coach.md +0 -271
- package/.claude/agents/sales-data-extraction-agent.md +0 -67
- package/.claude/agents/sales-deal-strategist.md +0 -180
- package/.claude/agents/sales-discovery-coach.md +0 -225
- package/.claude/agents/sales-engineer.md +0 -182
- package/.claude/agents/sales-offer-lead-gen-strategist.md +0 -257
- package/.claude/agents/sales-outbound-strategist.md +0 -201
- package/.claude/agents/sales-outreach.md +0 -425
- package/.claude/agents/sales-pipeline-analyst.md +0 -267
- package/.claude/agents/sales-proposal-strategist.md +0 -217
- package/.claude/agents/security-ai-generated-code-auditor.md +0 -207
- package/.claude/agents/security-appsec-engineer.md +0 -536
- package/.claude/agents/security-architect.md +0 -307
- package/.claude/agents/security-blockchain-security-auditor.md +0 -482
- package/.claude/agents/security-cloud-security-architect.md +0 -576
- package/.claude/agents/security-compliance-auditor.md +0 -158
- package/.claude/agents/security-incident-responder.md +0 -450
- package/.claude/agents/security-penetration-tester.md +0 -399
- package/.claude/agents/security-secrets-credential-engineer.md +0 -189
- package/.claude/agents/security-senior-secops.md +0 -760
- package/.claude/agents/security-threat-detection-engineer.md +0 -553
- package/.claude/agents/security-threat-intelligence-analyst.md +0 -661
- package/.claude/agents/specialized-chief-of-staff.md +0 -279
- package/.claude/agents/specialized-civil-engineer.md +0 -356
- package/.claude/agents/specialized-codebase-archaeologist.md +0 -341
- package/.claude/agents/specialized-cultural-intelligence-strategist.md +0 -88
- package/.claude/agents/specialized-developer-advocate.md +0 -317
- package/.claude/agents/specialized-document-generator.md +0 -55
- package/.claude/agents/specialized-fedramp-rmf-compliance.md +0 -378
- package/.claude/agents/specialized-focus-music-architect.md +0 -175
- package/.claude/agents/specialized-french-consulting-market.md +0 -194
- package/.claude/agents/specialized-korean-business-navigator.md +0 -216
- package/.claude/agents/specialized-master-plan-architect.md +0 -157
- package/.claude/agents/specialized-mcp-builder.md +0 -260
- package/.claude/agents/specialized-model-qa.md +0 -528
- package/.claude/agents/specialized-pricing-analyst.md +0 -243
- package/.claude/agents/specialized-salesforce-architect.md +0 -182
- package/.claude/agents/specialized-strategy-duel-agent.md +0 -130
- package/.claude/agents/specialized-workflow-architect.md +0 -597
- package/.claude/agents/study-abroad-advisor.md +0 -282
- package/.claude/agents/supply-chain-strategist.md +0 -591
- package/.claude/agents/support-analytics-reporter.md +0 -387
- package/.claude/agents/support-executive-summary-generator.md +0 -212
- package/.claude/agents/support-finance-tracker.md +0 -480
- package/.claude/agents/support-infrastructure-maintainer.md +0 -628
- package/.claude/agents/support-legal-compliance-checker.md +0 -591
- package/.claude/agents/support-support-responder.md +0 -607
- package/.claude/agents/technical-artist.md +0 -233
- package/.claude/agents/terminal-integration-specialist.md +0 -70
- package/.claude/agents/testing-accessibility-auditor.md +0 -327
- package/.claude/agents/testing-api-tester.md +0 -334
- package/.claude/agents/testing-evidence-collector.md +0 -210
- package/.claude/agents/testing-performance-benchmarker.md +0 -283
- package/.claude/agents/testing-reality-checker.md +0 -249
- package/.claude/agents/testing-test-automation-engineer.md +0 -196
- package/.claude/agents/testing-test-results-analyzer.md +0 -330
- package/.claude/agents/testing-tool-evaluator.md +0 -416
- package/.claude/agents/testing-workflow-optimizer.md +0 -463
- package/.claude/agents/unity-architect.md +0 -271
- package/.claude/agents/unity-editor-tool-developer.md +0 -310
- package/.claude/agents/unity-multiplayer-engineer.md +0 -323
- package/.claude/agents/unity-shader-graph-artist.md +0 -269
- package/.claude/agents/unreal-multiplayer-architect.md +0 -364
- package/.claude/agents/unreal-systems-engineer.md +0 -322
- package/.claude/agents/unreal-technical-artist.md +0 -256
- package/.claude/agents/unreal-world-builder.md +0 -273
- package/.claude/agents/visionos-spatial-engineer.md +0 -54
- package/.claude/agents/xr-cockpit-interaction-specialist.md +0 -32
- package/.claude/agents/xr-immersive-developer.md +0 -32
- package/.claude/agents/xr-interface-architect.md +0 -32
- package/.claude/agents/zk-steward.md +0 -211
- package/.claude-plugin/marketplace.json +0 -30
- package/.commitlintrc.json +0 -26
- package/.devcontainer/Dockerfile +0 -24
- package/.devcontainer/devcontainer.json +0 -70
- package/.devcontainer/docker-compose.memory.yml +0 -42
- package/.devcontainer/memory-stack/devcontainer.json +0 -84
- package/.devcontainer/post-create.sh +0 -83
- package/.dockerignore +0 -23
- package/.env.act.example +0 -15
- package/.env.example +0 -171
- package/.eslintrc.json +0 -16
- package/.git-blame-ignore-revs +0 -10
- package/.gitattributes +0 -2
- package/.gitguardian.yaml +0 -50
- package/.githooks/pre-commit +0 -6
- package/.pre-commit-config.yaml +0 -30
- package/.release-please-config-xithead.json +0 -22
- package/.release-please-config.json +0 -44
- package/.release-please-manifest-xithead.json +0 -3
- package/.release-please-manifest.json +0 -3
- package/CODE_OF_CONDUCT.md +0 -133
- package/CONTRIBUTING.md +0 -133
- package/Cargo.lock +0 -5557
- package/Cargo.toml +0 -121
- package/Dockerfile +0 -38
- package/Dockerfile.proxy +0 -48
- package/ENTERPRISE.md +0 -5
- package/FEATURES_ADVANCED.md +0 -311
- package/Headroom-2.gif +0 -0
- package/HeadroomDemo-Fast.gif +0 -0
- package/MANIFEST.in +0 -4
- package/MODERNIZATION_GUIDE.md +0 -358
- package/Makefile +0 -140
- package/NOTICE +0 -43
- package/NPM_TOKEN_SETUP.md +0 -81
- package/PR.md +0 -170
- package/PUBLISH.md +0 -77
- package/README_FRONTEND.md +0 -196
- package/REALIGNMENT/00-overview.md +0 -53
- package/REALIGNMENT/01-bug-list.md +0 -421
- package/REALIGNMENT/02-architecture.md +0 -322
- package/REALIGNMENT/03-phase-A-lockdown.md +0 -420
- package/REALIGNMENT/04-phase-B-live-zone.md +0 -470
- package/REALIGNMENT/05-phase-C-rust-proxy.md +0 -329
- package/REALIGNMENT/06-phase-D-bedrock-vertex.md +0 -222
- package/REALIGNMENT/07-phase-E-cache-stabilization.md +0 -338
- package/REALIGNMENT/08-phase-F-auth-mode.md +0 -254
- package/REALIGNMENT/09-phase-G-rtk-observability.md +0 -194
- package/REALIGNMENT/10-phase-H-python-retirement.md +0 -214
- package/REALIGNMENT/11-phase-I-test-infra.md +0 -385
- package/REALIGNMENT/12-decisions-needed.md +0 -196
- package/REALIGNMENT/INDEX.md +0 -82
- package/REFACTORING_CAMPAIGN.md +0 -277
- package/RESTRUCTURE_PLAN.md +0 -100
- package/RUST_DEV.md +0 -361
- package/SECURITY.md +0 -65
- package/SETUP_AUTO_PUBLISH.md +0 -164
- package/TESTING-copilot-subscription.md +0 -153
- package/codecov.yml +0 -34
- package/crates/headroom-core/Cargo.toml +0 -165
- package/crates/headroom-core/benches/auth_mode.rs +0 -74
- package/crates/headroom-core/benches/ccr_store.rs +0 -224
- package/crates/headroom-core/benches/tokenizer.rs +0 -52
- package/crates/headroom-core/config/pipeline.toml +0 -151
- package/crates/headroom-core/proptest-regressions/transforms/tag_protector.txt +0 -9
- package/crates/headroom-core/src/auth_mode.rs +0 -250
- package/crates/headroom-core/src/cache_control.rs +0 -414
- package/crates/headroom-core/src/ccr/backends/in_memory.rs +0 -317
- package/crates/headroom-core/src/ccr/backends/mod.rs +0 -152
- package/crates/headroom-core/src/ccr/backends/redis.rs +0 -146
- package/crates/headroom-core/src/ccr/backends/sqlite.rs +0 -205
- package/crates/headroom-core/src/ccr/mod.rs +0 -119
- package/crates/headroom-core/src/compression_policy.rs +0 -505
- package/crates/headroom-core/src/lib.rs +0 -33
- package/crates/headroom-core/src/relevance/base.rs +0 -139
- package/crates/headroom-core/src/relevance/bm25.rs +0 -385
- package/crates/headroom-core/src/relevance/embedding.rs +0 -354
- package/crates/headroom-core/src/relevance/hybrid.rs +0 -364
- package/crates/headroom-core/src/relevance/mod.rs +0 -64
- package/crates/headroom-core/src/signals/keyword_detector.rs +0 -433
- package/crates/headroom-core/src/signals/line_importance.rs +0 -84
- package/crates/headroom-core/src/signals/mod.rs +0 -61
- package/crates/headroom-core/src/signals/tiered.rs +0 -141
- package/crates/headroom-core/src/tokenizer/estimator.rs +0 -152
- package/crates/headroom-core/src/tokenizer/hf_impl.rs +0 -314
- package/crates/headroom-core/src/tokenizer/mod.rs +0 -56
- package/crates/headroom-core/src/tokenizer/registry.rs +0 -342
- package/crates/headroom-core/src/tokenizer/tiktoken_impl.rs +0 -265
- package/crates/headroom-core/src/transforms/adaptive_sizer.rs +0 -610
- package/crates/headroom-core/src/transforms/anchor_selector.rs +0 -1189
- package/crates/headroom-core/src/transforms/content_detector.rs +0 -769
- package/crates/headroom-core/src/transforms/detection.rs +0 -222
- package/crates/headroom-core/src/transforms/diff_compressor.rs +0 -1685
- package/crates/headroom-core/src/transforms/live_zone.rs +0 -2967
- package/crates/headroom-core/src/transforms/log_compressor.rs +0 -1295
- package/crates/headroom-core/src/transforms/magika_detector.rs +0 -395
- package/crates/headroom-core/src/transforms/mod.rs +0 -64
- package/crates/headroom-core/src/transforms/pipeline/config.rs +0 -331
- package/crates/headroom-core/src/transforms/pipeline/mod.rs +0 -103
- package/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rs +0 -524
- package/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs +0 -279
- package/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs +0 -400
- package/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rs +0 -284
- package/crates/headroom-core/src/transforms/pipeline/offloads/mod.rs +0 -34
- package/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rs +0 -331
- package/crates/headroom-core/src/transforms/pipeline/orchestrator.rs +0 -848
- package/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rs +0 -175
- package/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rs +0 -545
- package/crates/headroom-core/src/transforms/pipeline/reformats/mod.rs +0 -12
- package/crates/headroom-core/src/transforms/pipeline/traits.rs +0 -338
- package/crates/headroom-core/src/transforms/recommendations.rs +0 -329
- package/crates/headroom-core/src/transforms/safety.rs +0 -215
- package/crates/headroom-core/src/transforms/search_compressor.rs +0 -902
- package/crates/headroom-core/src/transforms/smart_crusher/analyzer.rs +0 -1262
- package/crates/headroom-core/src/transforms/smart_crusher/anchors.rs +0 -407
- package/crates/headroom-core/src/transforms/smart_crusher/builder.rs +0 -263
- package/crates/headroom-core/src/transforms/smart_crusher/classifier.rs +0 -214
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/classifier.rs +0 -283
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/compactor.rs +0 -733
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rs +0 -961
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/ir.rs +0 -252
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs +0 -127
- package/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs +0 -397
- package/crates/headroom-core/src/transforms/smart_crusher/config.rs +0 -173
- package/crates/headroom-core/src/transforms/smart_crusher/constraints.rs +0 -159
- package/crates/headroom-core/src/transforms/smart_crusher/crusher.rs +0 -1665
- package/crates/headroom-core/src/transforms/smart_crusher/crushers.rs +0 -865
- package/crates/headroom-core/src/transforms/smart_crusher/error_keywords.rs +0 -77
- package/crates/headroom-core/src/transforms/smart_crusher/field_detect.rs +0 -393
- package/crates/headroom-core/src/transforms/smart_crusher/hashing.rs +0 -72
- package/crates/headroom-core/src/transforms/smart_crusher/mod.rs +0 -81
- package/crates/headroom-core/src/transforms/smart_crusher/observer.rs +0 -66
- package/crates/headroom-core/src/transforms/smart_crusher/orchestration.rs +0 -459
- package/crates/headroom-core/src/transforms/smart_crusher/outliers.rs +0 -507
- package/crates/headroom-core/src/transforms/smart_crusher/planning.rs +0 -904
- package/crates/headroom-core/src/transforms/smart_crusher/statistics.rs +0 -510
- package/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs +0 -287
- package/crates/headroom-core/src/transforms/smart_crusher/traits.rs +0 -203
- package/crates/headroom-core/src/transforms/smart_crusher/types.rs +0 -263
- package/crates/headroom-core/src/transforms/tag_protector.rs +0 -1272
- package/crates/headroom-core/src/transforms/unidiff_detector.rs +0 -232
- package/crates/headroom-parity/Cargo.toml +0 -26
- package/crates/headroom-parity/examples/diff_fixture.rs +0 -66
- package/crates/headroom-parity/src/bin/parity_run.rs +0 -78
- package/crates/headroom-parity/src/lib.rs +0 -613
- package/crates/headroom-proxy/Cargo.toml +0 -119
- package/crates/headroom-proxy/data/model_prices_and_context_window.json +0 -39767
- package/crates/headroom-proxy/src/bedrock/auth_mode_layer.rs +0 -192
- package/crates/headroom-proxy/src/bedrock/envelope.rs +0 -239
- package/crates/headroom-proxy/src/bedrock/eventstream.rs +0 -769
- package/crates/headroom-proxy/src/bedrock/eventstream_to_sse.rs +0 -459
- package/crates/headroom-proxy/src/bedrock/invoke.rs +0 -735
- package/crates/headroom-proxy/src/bedrock/invoke_streaming.rs +0 -1098
- package/crates/headroom-proxy/src/bedrock/mod.rs +0 -66
- package/crates/headroom-proxy/src/bedrock/sigv4.rs +0 -324
- package/crates/headroom-proxy/src/bedrock/vendor.rs +0 -82
- package/crates/headroom-proxy/src/cache_stabilization/anthropic_cache_control.rs +0 -698
- package/crates/headroom-proxy/src/cache_stabilization/drift_detector.rs +0 -683
- package/crates/headroom-proxy/src/cache_stabilization/mod.rs +0 -61
- package/crates/headroom-proxy/src/cache_stabilization/openai_cache_key.rs +0 -600
- package/crates/headroom-proxy/src/cache_stabilization/tool_def_normalize.rs +0 -514
- package/crates/headroom-proxy/src/cache_stabilization/volatile_detector.rs +0 -720
- package/crates/headroom-proxy/src/compression/anthropic.rs +0 -92
- package/crates/headroom-proxy/src/compression/live_zone_anthropic.rs +0 -1311
- package/crates/headroom-proxy/src/compression/live_zone_openai.rs +0 -660
- package/crates/headroom-proxy/src/compression/live_zone_responses.rs +0 -678
- package/crates/headroom-proxy/src/compression/mod.rs +0 -127
- package/crates/headroom-proxy/src/compression/model_limits.rs +0 -211
- package/crates/headroom-proxy/src/config.rs +0 -646
- package/crates/headroom-proxy/src/error.rs +0 -69
- package/crates/headroom-proxy/src/handlers/chat_completions.rs +0 -100
- package/crates/headroom-proxy/src/handlers/conversations.rs +0 -241
- package/crates/headroom-proxy/src/handlers/mod.rs +0 -13
- package/crates/headroom-proxy/src/handlers/responses.rs +0 -256
- package/crates/headroom-proxy/src/headers.rs +0 -288
- package/crates/headroom-proxy/src/health.rs +0 -41
- package/crates/headroom-proxy/src/lib.rs +0 -21
- package/crates/headroom-proxy/src/main.rs +0 -141
- package/crates/headroom-proxy/src/observability/cache_hit_rate.rs +0 -340
- package/crates/headroom-proxy/src/observability/compression_ratio.rs +0 -158
- package/crates/headroom-proxy/src/observability/metric_names.rs +0 -174
- package/crates/headroom-proxy/src/observability/mod.rs +0 -68
- package/crates/headroom-proxy/src/observability/prometheus.rs +0 -394
- package/crates/headroom-proxy/src/observability/proxy_metrics.rs +0 -368
- package/crates/headroom-proxy/src/proxy.rs +0 -1619
- package/crates/headroom-proxy/src/responses_items.rs +0 -565
- package/crates/headroom-proxy/src/sse/anthropic.rs +0 -415
- package/crates/headroom-proxy/src/sse/framing.rs +0 -366
- package/crates/headroom-proxy/src/sse/mod.rs +0 -40
- package/crates/headroom-proxy/src/sse/openai_chat.rs +0 -231
- package/crates/headroom-proxy/src/sse/openai_responses.rs +0 -458
- package/crates/headroom-proxy/src/vertex/adc.rs +0 -315
- package/crates/headroom-proxy/src/vertex/envelope.rs +0 -167
- package/crates/headroom-proxy/src/vertex/mod.rs +0 -277
- package/crates/headroom-proxy/src/vertex/raw_predict.rs +0 -490
- package/crates/headroom-proxy/src/vertex/stream_raw_predict.rs +0 -49
- package/crates/headroom-proxy/src/websocket.rs +0 -246
- package/crates/headroom-py/Cargo.toml +0 -37
- package/crates/headroom-py/build.rs +0 -77
- package/crates/headroom-py/glibc_compat.c +0 -170
- package/crates/headroom-py/src/lib.rs +0 -1655
- package/deny.toml +0 -32
- package/docker/differential-network-capture/Dockerfile.runner +0 -14
- package/docker/differential-network-capture/docker-compose.yml +0 -142
- package/docker/differential-network-capture/run-claude-lane.sh +0 -24
- package/docker/docker-compose.native.yml +0 -44
- package/docker-bake.hcl +0 -88
- package/docker-compose.yml +0 -80
- package/e2e/_lib/make_shim.ps1 +0 -24
- package/e2e/_lib/make_shim.sh +0 -28
- package/e2e/docker-native-install.sh +0 -120
- package/e2e/init/Dockerfile +0 -79
- package/e2e/wrap/Dockerfile +0 -115
- package/examples/07-context-compression.ipynb +0 -279
- package/examples/deployment/macos-launchagent/com.headroom.proxy.plist.template +0 -65
- package/examples/deployment/macos-launchagent/install.sh +0 -227
- package/examples/deployment/macos-launchagent/shell-integration.sh +0 -74
- package/examples/deployment/macos-launchagent/uninstall.sh +0 -145
- package/headroom-savings.png +0 -0
- package/headroom_learn.gif +0 -0
- package/llms.txt +0 -67
- package/mkdocs.yml +0 -136
- package/npm/bin/xithead.js +0 -366
- package/npm/bin/xtream-head.js +0 -366
- package/npm/index.d.ts +0 -57
- package/npm/index.js +0 -139
- package/npm/package.json +0 -55
- package/npm/postinstall.js +0 -43
- package/plugins/headroom-agent-hooks/.claude-plugin/plugin.json +0 -17
- package/plugins/headroom-agent-hooks/hooks/hooks.json +0 -29
- package/plugins/headroom-oauth2/LICENSE +0 -3
- package/plugins/headroom-oauth2/SPEC.md +0 -115
- package/plugins/headroom-oauth2/pyproject.toml +0 -48
- package/plugins/hermes/headroom_retrieve/plugin.yaml +0 -6
- package/plugins/openclaw/hook-shim/HOOK.md +0 -21
- package/plugins/openclaw/hook-shim/handler.js +0 -3
- package/plugins/openclaw/openclaw.plugin.json +0 -98
- package/plugins/openclaw/package-lock.json +0 -2692
- package/plugins/openclaw/package.json +0 -54
- package/plugins/openclaw/prepare-dist.mjs +0 -56
- package/plugins/openclaw/src/convert.ts +0 -418
- package/plugins/openclaw/src/engine.ts +0 -284
- package/plugins/openclaw/src/gateway-config.ts +0 -143
- package/plugins/openclaw/src/index.ts +0 -11
- package/plugins/openclaw/src/plugin/index.ts +0 -98
- package/plugins/openclaw/src/proxy-manager.ts +0 -477
- package/plugins/openclaw/src/tools/headroom-retrieve.ts +0 -76
- package/plugins/openclaw/test/convert.test.ts +0 -96
- package/plugins/openclaw/test/engine-normalization.test.ts +0 -21
- package/plugins/openclaw/test/engine.test.ts +0 -101
- package/plugins/openclaw/test/gateway-config.test.ts +0 -278
- package/plugins/openclaw/test/plugin-runtime-routing.test.ts +0 -141
- package/plugins/openclaw/test/proxy-manager.test.ts +0 -343
- package/plugins/openclaw/tsup.config.ts +0 -9
- package/pyproject.toml +0 -438
- package/rust-toolchain.toml +0 -16
- package/sdk/typescript/examples/basic-compress.ts +0 -64
- package/sdk/typescript/examples/ccr-retrieve.ts +0 -80
- package/sdk/typescript/examples/hooks-custom-compression.ts +0 -117
- package/sdk/typescript/examples/middleware-composition.ts +0 -59
- package/sdk/typescript/examples/multi-provider.ts +0 -53
- package/sdk/typescript/examples/openai-anthropic-adapters.ts +0 -73
- package/sdk/typescript/examples/shared-context-multi-agent.ts +0 -74
- package/sdk/typescript/examples/simulation-dry-run.ts +0 -80
- package/sdk/typescript/examples/streaming-chat.ts +0 -71
- package/sdk/typescript/examples/structured-output.ts +0 -99
- package/sdk/typescript/examples/tool-calling-agent.ts +0 -86
- package/sdk/typescript/examples/with-headroom-vercel.ts +0 -57
- package/sdk/typescript/package-lock.json +0 -3437
- package/sdk/typescript/package.json +0 -108
- package/sdk/typescript/src/adapters/anthropic.ts +0 -218
- package/sdk/typescript/src/adapters/gemini.ts +0 -83
- package/sdk/typescript/src/adapters/openai.ts +0 -77
- package/sdk/typescript/src/adapters/vercel-ai.ts +0 -131
- package/sdk/typescript/src/client.ts +0 -632
- package/sdk/typescript/src/compress.ts +0 -87
- package/sdk/typescript/src/errors.ts +0 -114
- package/sdk/typescript/src/hooks.ts +0 -113
- package/sdk/typescript/src/index.ts +0 -140
- package/sdk/typescript/src/paths.ts +0 -272
- package/sdk/typescript/src/shared-context.ts +0 -188
- package/sdk/typescript/src/simulate.ts +0 -52
- package/sdk/typescript/src/types/config.ts +0 -183
- package/sdk/typescript/src/types/models.ts +0 -314
- package/sdk/typescript/src/types.ts +0 -139
- package/sdk/typescript/src/utils/case.ts +0 -34
- package/sdk/typescript/src/utils/format.ts +0 -480
- package/sdk/typescript/src/utils/stream.ts +0 -50
- package/sdk/typescript/test/adapters/anthropic.test.ts +0 -217
- package/sdk/typescript/test/adapters/gemini.test.ts +0 -131
- package/sdk/typescript/test/adapters/openai.test.ts +0 -174
- package/sdk/typescript/test/adapters/vercel-ai.test.ts +0 -306
- package/sdk/typescript/test/client-expanded.test.ts +0 -333
- package/sdk/typescript/test/client.test.ts +0 -266
- package/sdk/typescript/test/compress.test.ts +0 -118
- package/sdk/typescript/test/config.test.ts +0 -275
- package/sdk/typescript/test/errors.test.ts +0 -169
- package/sdk/typescript/test/format-integration.test.ts +0 -252
- package/sdk/typescript/test/hooks.test.ts +0 -265
- package/sdk/typescript/test/integration.test.ts +0 -249
- package/sdk/typescript/test/mdx-examples.test.ts +0 -253
- package/sdk/typescript/test/models.test.ts +0 -318
- package/sdk/typescript/test/parity.test.ts +0 -337
- package/sdk/typescript/test/paths.test.ts +0 -447
- package/sdk/typescript/test/shared-context.test.ts +0 -245
- package/sdk/typescript/test/simulate.test.ts +0 -84
- package/sdk/typescript/test/types.test.ts +0 -244
- package/sdk/typescript/test/utils/case.test.ts +0 -129
- package/sdk/typescript/test/utils/detect-format.test.ts +0 -133
- package/sdk/typescript/test/utils/format.test.ts +0 -434
- package/sdk/typescript/test/vercel-ai-e2e.test.ts +0 -179
- package/sdk/typescript/tsup.config.ts +0 -17
- package/uv.lock +0 -6951
- package/xithead.egg-info/PKG-INFO +0 -22
- package/xithead.egg-info/SOURCES.txt +0 -16
- package/xithead.egg-info/dependency_links.txt +0 -1
- package/xithead.egg-info/entry_points.txt +0 -2
- package/xithead.egg-info/requires.txt +0 -10
- package/xithead.egg-info/top_level.txt +0 -2
- package/xithead.ps1 +0 -136
- package/xithead.sh +0 -129
- package/xithead_dist/pyproject.toml +0 -39
- package/xithead_dist/xithead.egg-info/PKG-INFO +0 -22
- package/xithead_dist/xithead.egg-info/SOURCES.txt +0 -16
- package/xithead_dist/xithead.egg-info/dependency_links.txt +0 -1
- package/xithead_dist/xithead.egg-info/entry_points.txt +0 -3
- package/xithead_dist/xithead.egg-info/requires.txt +0 -10
- package/xithead_dist/xithead.egg-info/top_level.txt +0 -2
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
//! Canonical error keyword set for item preservation.
|
|
2
|
-
//!
|
|
3
|
-
//! Direct port of `ERROR_KEYWORDS` from `headroom/transforms/error_detection.py:18-33`.
|
|
4
|
-
//! These are the **FALLBACK** preservation signal when TOIN field
|
|
5
|
-
//! semantics aren't available yet (per the Python module-level
|
|
6
|
-
//! comment). Intentionally broad — better to over-preserve than to
|
|
7
|
-
//! drop a real error item.
|
|
8
|
-
//!
|
|
9
|
-
//! Used by `detect_error_items_for_preservation`. The list is small
|
|
10
|
-
//! enough to keep as a `&[&str]`; if we ever cross ~50 keywords, switch
|
|
11
|
-
//! to a `phf::Set` or pre-built FST for sub-linear lookup.
|
|
12
|
-
|
|
13
|
-
/// 12 error/failure keywords. Order doesn't matter for correctness, but
|
|
14
|
-
/// matches Python's set-literal order so reading both side-by-side is
|
|
15
|
-
/// easier. Lowercase by construction; callers must lowercase the
|
|
16
|
-
/// haystack before substring-matching.
|
|
17
|
-
pub const ERROR_KEYWORDS: &[&str] = &[
|
|
18
|
-
"error",
|
|
19
|
-
"exception",
|
|
20
|
-
"failed",
|
|
21
|
-
"failure",
|
|
22
|
-
"critical",
|
|
23
|
-
"fatal",
|
|
24
|
-
"crash",
|
|
25
|
-
"panic",
|
|
26
|
-
"abort",
|
|
27
|
-
"timeout",
|
|
28
|
-
"denied",
|
|
29
|
-
"rejected",
|
|
30
|
-
];
|
|
31
|
-
|
|
32
|
-
#[cfg(test)]
|
|
33
|
-
mod tests {
|
|
34
|
-
use super::*;
|
|
35
|
-
|
|
36
|
-
#[test]
|
|
37
|
-
fn matches_python_count() {
|
|
38
|
-
// Python `len(ERROR_KEYWORDS) == 12`. If this drifts, the
|
|
39
|
-
// Python set was edited and the Rust list needs a matching
|
|
40
|
-
// update.
|
|
41
|
-
assert_eq!(ERROR_KEYWORDS.len(), 12);
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
#[test]
|
|
45
|
-
fn all_lowercase_invariant() {
|
|
46
|
-
for &kw in ERROR_KEYWORDS {
|
|
47
|
-
assert_eq!(
|
|
48
|
-
kw,
|
|
49
|
-
kw.to_lowercase(),
|
|
50
|
-
"ERROR_KEYWORDS must all be lowercase"
|
|
51
|
-
);
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
#[test]
|
|
56
|
-
fn pinned_membership() {
|
|
57
|
-
// Pin the exact set so accidental edits surface in CI rather
|
|
58
|
-
// than silently changing item-preservation behavior.
|
|
59
|
-
let expected = [
|
|
60
|
-
"error",
|
|
61
|
-
"exception",
|
|
62
|
-
"failed",
|
|
63
|
-
"failure",
|
|
64
|
-
"critical",
|
|
65
|
-
"fatal",
|
|
66
|
-
"crash",
|
|
67
|
-
"panic",
|
|
68
|
-
"abort",
|
|
69
|
-
"timeout",
|
|
70
|
-
"denied",
|
|
71
|
-
"rejected",
|
|
72
|
-
];
|
|
73
|
-
let actual: std::collections::BTreeSet<&str> = ERROR_KEYWORDS.iter().copied().collect();
|
|
74
|
-
let expected: std::collections::BTreeSet<&str> = expected.iter().copied().collect();
|
|
75
|
-
assert_eq!(actual, expected);
|
|
76
|
-
}
|
|
77
|
-
}
|
|
@@ -1,393 +0,0 @@
|
|
|
1
|
-
//! Statistical detectors for ID-like and score-like fields.
|
|
2
|
-
//!
|
|
3
|
-
//! Direct ports of `_detect_id_field_statistically` and
|
|
4
|
-
//! `_detect_score_field_statistically` from `smart_crusher.py:484-603`.
|
|
5
|
-
//!
|
|
6
|
-
//! These run *after* per-field statistics are computed and consume a
|
|
7
|
-
//! `FieldStats` plus the raw values. They're called by the analyzer's
|
|
8
|
-
//! crushability logic to decide whether a field carries a meaningful
|
|
9
|
-
//! ranking signal (score) or is just a unique identifier (ID) that
|
|
10
|
-
//! shouldn't drive compression decisions.
|
|
11
|
-
|
|
12
|
-
use serde_json::Value;
|
|
13
|
-
|
|
14
|
-
use super::statistics::{calculate_string_entropy, detect_sequential_pattern, is_uuid_format};
|
|
15
|
-
use super::types::FieldStats;
|
|
16
|
-
|
|
17
|
-
/// Detect whether a field is an "ID field" — high-uniqueness column
|
|
18
|
-
/// that doesn't carry semantic information.
|
|
19
|
-
///
|
|
20
|
-
/// Direct port of `_detect_id_field_statistically` (Python
|
|
21
|
-
/// `smart_crusher.py:484-530`). Returns `(is_id, confidence)` where
|
|
22
|
-
/// confidence ∈ [0.0, 1.0]. The caller uses this to decide whether the
|
|
23
|
-
/// field is a strong enough signal to drive crushability analysis.
|
|
24
|
-
///
|
|
25
|
-
/// # Detection rules (mirroring Python step-by-step)
|
|
26
|
-
///
|
|
27
|
-
/// 1. Hard gate: `unique_ratio < 0.9` → not an ID field.
|
|
28
|
-
/// 2. String fields:
|
|
29
|
-
/// - >80% of first-20 sample values look like UUIDs → confidence 0.95.
|
|
30
|
-
/// - Average entropy >0.7 AND `unique_ratio > 0.95` → confidence 0.8.
|
|
31
|
-
/// 3. Numeric fields:
|
|
32
|
-
/// - Sequential pattern (via `detect_sequential_pattern`) AND
|
|
33
|
-
/// `unique_ratio > 0.95` → confidence 0.9.
|
|
34
|
-
/// - Has a value range AND `unique_ratio > 0.95` → confidence 0.85.
|
|
35
|
-
/// 4. Catch-all: very high uniqueness (`> 0.98`) → confidence 0.7.
|
|
36
|
-
pub fn detect_id_field_statistically(stats: &FieldStats, values: &[Value]) -> (bool, f64) {
|
|
37
|
-
// Hard gate matching Python line 494.
|
|
38
|
-
if stats.unique_ratio < 0.9 {
|
|
39
|
-
return (false, 0.0);
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
// String-field branches.
|
|
43
|
-
if stats.field_type == "string" {
|
|
44
|
-
// First 20 string-typed values for sampling. Python: `values[:20]`
|
|
45
|
-
// then filters by `isinstance(v, str)` — order-preserving slice
|
|
46
|
-
// before filter, so we mirror that.
|
|
47
|
-
let sample_values: Vec<&str> = values.iter().take(20).filter_map(|v| v.as_str()).collect();
|
|
48
|
-
|
|
49
|
-
if !sample_values.is_empty() {
|
|
50
|
-
let uuid_count = sample_values.iter().filter(|s| is_uuid_format(s)).count();
|
|
51
|
-
// Python: `uuid_count / len(sample_values) > 0.8`.
|
|
52
|
-
if (uuid_count as f64 / sample_values.len() as f64) > 0.8 {
|
|
53
|
-
return (true, 0.95);
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
// Python: average entropy across the sample.
|
|
57
|
-
let avg_entropy = sample_values
|
|
58
|
-
.iter()
|
|
59
|
-
.map(|s| calculate_string_entropy(s))
|
|
60
|
-
.sum::<f64>()
|
|
61
|
-
/ sample_values.len() as f64;
|
|
62
|
-
if avg_entropy > 0.7 && stats.unique_ratio > 0.95 {
|
|
63
|
-
return (true, 0.8);
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
// Numeric-field branches.
|
|
69
|
-
if stats.field_type == "numeric" {
|
|
70
|
-
// Python: passes `values` (full list, may include strings) through
|
|
71
|
-
// `_detect_sequential_pattern` with default `check_order=True`.
|
|
72
|
-
if detect_sequential_pattern(values, true) && stats.unique_ratio > 0.95 {
|
|
73
|
-
return (true, 0.9);
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
// High-uniqueness numeric with non-trivial range — likely an ID
|
|
77
|
-
// even without sequential structure (e.g., random ints in a wide
|
|
78
|
-
// band).
|
|
79
|
-
if let (Some(min_v), Some(max_v)) = (stats.min_val, stats.max_val) {
|
|
80
|
-
let value_range = max_v - min_v;
|
|
81
|
-
if value_range > 0.0 && stats.unique_ratio > 0.95 {
|
|
82
|
-
return (true, 0.85);
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
// Catch-all: very high uniqueness alone is a signal.
|
|
88
|
-
if stats.unique_ratio > 0.98 {
|
|
89
|
-
return (true, 0.7);
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
(false, 0.0)
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
/// Detect whether a field is a "score field" — bounded-range numeric
|
|
96
|
-
/// where higher values mean "more relevant".
|
|
97
|
-
///
|
|
98
|
-
/// Direct port of `_detect_score_field_statistically` (Python
|
|
99
|
-
/// `smart_crusher.py:533-603`). Returns `(is_score, confidence)`.
|
|
100
|
-
///
|
|
101
|
-
/// # Detection rules (mirroring Python)
|
|
102
|
-
///
|
|
103
|
-
/// 1. Field must be numeric AND have both `min_val` and `max_val`.
|
|
104
|
-
/// 2. Range must match a "common score range":
|
|
105
|
-
/// - `[0, 1]` (most common ML score range) → +0.4
|
|
106
|
-
/// - `[0, 10]` → +0.3
|
|
107
|
-
/// - `[0, 100]` → +0.25
|
|
108
|
-
/// - `[-1, 1]` (signed similarity) → +0.35
|
|
109
|
-
/// 3. Must NOT be a sequential pattern (IDs are sequential; scores aren't).
|
|
110
|
-
/// 4. If first-50 values appear sorted descending (>70% of pairs) → +0.3.
|
|
111
|
-
/// 5. If >30% of first-20 are non-integer floats → +0.1.
|
|
112
|
-
/// 6. Returns `(confidence >= 0.4, min(confidence, 0.95))`.
|
|
113
|
-
///
|
|
114
|
-
/// `items` is the list of original-array dict items so we can pull the
|
|
115
|
-
/// field's values in array order for the descending-sort check.
|
|
116
|
-
pub fn detect_score_field_statistically(stats: &FieldStats, items: &[Value]) -> (bool, f64) {
|
|
117
|
-
if stats.field_type != "numeric" {
|
|
118
|
-
return (false, 0.0);
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
let (min_val, max_val) = match (stats.min_val, stats.max_val) {
|
|
122
|
-
(Some(min_v), Some(max_v)) => (min_v, max_v),
|
|
123
|
-
_ => return (false, 0.0),
|
|
124
|
-
};
|
|
125
|
-
|
|
126
|
-
let mut confidence: f64 = 0.0;
|
|
127
|
-
|
|
128
|
-
// Range check (Python lines 555-568). The conditions are arranged
|
|
129
|
-
// exactly as in Python — `if/elif` chain, first match wins.
|
|
130
|
-
let is_bounded = if (0.0..=1.0).contains(&min_val) && (0.0..=1.0).contains(&max_val) {
|
|
131
|
-
confidence += 0.4;
|
|
132
|
-
true
|
|
133
|
-
} else if (0.0..=10.0).contains(&min_val) && (0.0..=10.0).contains(&max_val) {
|
|
134
|
-
confidence += 0.3;
|
|
135
|
-
true
|
|
136
|
-
} else if (0.0..=100.0).contains(&min_val) && (0.0..=100.0).contains(&max_val) {
|
|
137
|
-
confidence += 0.25;
|
|
138
|
-
true
|
|
139
|
-
} else if min_val >= -1.0 && max_val <= 1.0 {
|
|
140
|
-
// Python: `elif -1 <= min_val and max_val <= 1`. Note Python's
|
|
141
|
-
// chained `<=` only on max_val side; min_val is checked
|
|
142
|
-
// separately. Pinned exactly.
|
|
143
|
-
confidence += 0.35;
|
|
144
|
-
true
|
|
145
|
-
} else {
|
|
146
|
-
false
|
|
147
|
-
};
|
|
148
|
-
|
|
149
|
-
if !is_bounded {
|
|
150
|
-
return (false, 0.0);
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
// Pull this field's values from the FIRST 50 items, dict-style.
|
|
154
|
-
// Python: `[item.get(stats.name) for item in items[:50] if stats.name in item]`.
|
|
155
|
-
let sample_values: Vec<&Value> = items
|
|
156
|
-
.iter()
|
|
157
|
-
.take(50)
|
|
158
|
-
.filter_map(|item| item.as_object().and_then(|m| m.get(&stats.name)))
|
|
159
|
-
.collect();
|
|
160
|
-
|
|
161
|
-
// Sequential check — IDs are sequential, scores aren't. We need
|
|
162
|
-
// owned `Value`s for `detect_sequential_pattern`; clone the sample.
|
|
163
|
-
let sample_owned: Vec<Value> = sample_values.iter().map(|v| (*v).clone()).collect();
|
|
164
|
-
if detect_sequential_pattern(&sample_owned, true) {
|
|
165
|
-
return (false, 0.0);
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
// Descending-sort check on the full items list (Python line 580-593).
|
|
169
|
-
// Filter to finite-numeric values, preserving array order.
|
|
170
|
-
let values_in_order: Vec<f64> = items
|
|
171
|
-
.iter()
|
|
172
|
-
.filter_map(|item| item.as_object().and_then(|m| m.get(&stats.name)))
|
|
173
|
-
.filter_map(|v| v.as_f64())
|
|
174
|
-
.filter(|f| f.is_finite())
|
|
175
|
-
.collect();
|
|
176
|
-
|
|
177
|
-
if values_in_order.len() >= 5 {
|
|
178
|
-
let num_pairs = values_in_order.len() - 1;
|
|
179
|
-
let descending_count = values_in_order.windows(2).filter(|w| w[0] >= w[1]).count();
|
|
180
|
-
if num_pairs > 0 && (descending_count as f64 / num_pairs as f64) > 0.7 {
|
|
181
|
-
confidence += 0.3;
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
// Float-fraction check on first 20 ordered values (Python line 597-601).
|
|
186
|
-
// "Not equal to its int-truncation" ≈ has a fractional part.
|
|
187
|
-
let first_20: &[f64] = if values_in_order.len() > 20 {
|
|
188
|
-
&values_in_order[..20]
|
|
189
|
-
} else {
|
|
190
|
-
&values_in_order[..]
|
|
191
|
-
};
|
|
192
|
-
let float_count = first_20
|
|
193
|
-
.iter()
|
|
194
|
-
.filter(|&&v| v.is_finite() && v != v.trunc())
|
|
195
|
-
.count();
|
|
196
|
-
if !first_20.is_empty() && (float_count as f64) > (first_20.len() as f64 * 0.3) {
|
|
197
|
-
confidence += 0.1;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
let is_score = confidence >= 0.4;
|
|
201
|
-
let bounded_confidence = confidence.min(0.95);
|
|
202
|
-
(is_score, bounded_confidence)
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
#[cfg(test)]
|
|
206
|
-
mod tests {
|
|
207
|
-
use super::*;
|
|
208
|
-
use serde_json::json;
|
|
209
|
-
|
|
210
|
-
fn stats(name: &str, field_type: &str, unique_ratio: f64) -> FieldStats {
|
|
211
|
-
FieldStats {
|
|
212
|
-
name: name.to_string(),
|
|
213
|
-
field_type: field_type.to_string(),
|
|
214
|
-
count: 100,
|
|
215
|
-
unique_count: (100.0 * unique_ratio) as usize,
|
|
216
|
-
unique_ratio,
|
|
217
|
-
is_constant: false,
|
|
218
|
-
constant_value: None,
|
|
219
|
-
min_val: None,
|
|
220
|
-
max_val: None,
|
|
221
|
-
mean_val: None,
|
|
222
|
-
variance: None,
|
|
223
|
-
change_points: Vec::new(),
|
|
224
|
-
avg_length: None,
|
|
225
|
-
top_values: Vec::new(),
|
|
226
|
-
}
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
fn stats_with_range(name: &str, min_v: f64, max_v: f64) -> FieldStats {
|
|
230
|
-
let mut s = stats(name, "numeric", 1.0);
|
|
231
|
-
s.min_val = Some(min_v);
|
|
232
|
-
s.max_val = Some(max_v);
|
|
233
|
-
s
|
|
234
|
-
}
|
|
235
|
-
|
|
236
|
-
// ---------- detect_id_field_statistically ----------
|
|
237
|
-
|
|
238
|
-
#[test]
|
|
239
|
-
fn id_field_low_uniqueness_rejected() {
|
|
240
|
-
let s = stats("status", "string", 0.5);
|
|
241
|
-
let values: Vec<Value> = vec![json!("ok"), json!("error"), json!("ok"), json!("ok")];
|
|
242
|
-
assert_eq!(detect_id_field_statistically(&s, &values), (false, 0.0));
|
|
243
|
-
}
|
|
244
|
-
|
|
245
|
-
#[test]
|
|
246
|
-
fn id_field_uuid_strings_high_confidence() {
|
|
247
|
-
let s = stats("uid", "string", 1.0);
|
|
248
|
-
let values: Vec<Value> = (0..20)
|
|
249
|
-
.map(|i| json!(format!("550e8400-e29b-41d4-a716-{:012x}", i)))
|
|
250
|
-
.collect();
|
|
251
|
-
let (is_id, conf) = detect_id_field_statistically(&s, &values);
|
|
252
|
-
assert!(is_id);
|
|
253
|
-
assert_eq!(conf, 0.95);
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
#[test]
|
|
257
|
-
fn id_field_high_entropy_strings() {
|
|
258
|
-
let mut s = stats("uid", "string", 1.0);
|
|
259
|
-
s.unique_ratio = 0.96;
|
|
260
|
-
// 20 random-looking hex-ish strings — high entropy.
|
|
261
|
-
let values: Vec<Value> = (0..20)
|
|
262
|
-
.map(|i| json!(format!("a3f7b2c{:06x}d8e1f4a7", i)))
|
|
263
|
-
.collect();
|
|
264
|
-
let (is_id, conf) = detect_id_field_statistically(&s, &values);
|
|
265
|
-
assert!(is_id);
|
|
266
|
-
assert!((conf - 0.8).abs() < 1e-9);
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
#[test]
|
|
270
|
-
fn id_field_sequential_numeric() {
|
|
271
|
-
let mut s = stats("id", "numeric", 1.0);
|
|
272
|
-
s.unique_ratio = 0.96;
|
|
273
|
-
s.min_val = Some(1.0);
|
|
274
|
-
s.max_val = Some(100.0);
|
|
275
|
-
let values: Vec<Value> = (1..=100).map(|i| json!(i)).collect();
|
|
276
|
-
let (is_id, conf) = detect_id_field_statistically(&s, &values);
|
|
277
|
-
assert!(is_id);
|
|
278
|
-
assert!((conf - 0.9).abs() < 1e-9);
|
|
279
|
-
}
|
|
280
|
-
|
|
281
|
-
#[test]
|
|
282
|
-
fn id_field_high_uniqueness_alone_triggers_catchall() {
|
|
283
|
-
// Numeric field with unique_ratio = 0.99 but no other signal.
|
|
284
|
-
// Falls through to the 0.98 catch-all.
|
|
285
|
-
let mut s = stats("misc", "numeric", 0.99);
|
|
286
|
-
s.min_val = Some(0.0);
|
|
287
|
-
s.max_val = Some(0.0); // zero range → numeric range branch fails
|
|
288
|
-
let values: Vec<Value> = (0..100).map(|_| json!(0)).collect();
|
|
289
|
-
let (is_id, conf) = detect_id_field_statistically(&s, &values);
|
|
290
|
-
assert!(is_id);
|
|
291
|
-
assert!((conf - 0.7).abs() < 1e-9);
|
|
292
|
-
}
|
|
293
|
-
|
|
294
|
-
// ---------- detect_score_field_statistically ----------
|
|
295
|
-
|
|
296
|
-
#[test]
|
|
297
|
-
fn score_field_unit_range_with_descending_sort() {
|
|
298
|
-
// Range [0, 1] (+0.4) + descending sort (+0.3) = 0.7, which
|
|
299
|
-
// exceeds the 0.4 threshold and is below the 0.95 cap.
|
|
300
|
-
let s = stats_with_range("score", 0.0, 1.0);
|
|
301
|
-
let items: Vec<Value> = (0..10)
|
|
302
|
-
.rev()
|
|
303
|
-
.map(|i| json!({"score": (i as f64) / 10.0}))
|
|
304
|
-
.collect();
|
|
305
|
-
let (is_score, conf) = detect_score_field_statistically(&s, &items);
|
|
306
|
-
assert!(is_score);
|
|
307
|
-
assert!(conf >= 0.7);
|
|
308
|
-
assert!(conf <= 0.95);
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
#[test]
|
|
312
|
-
fn score_field_sequential_rejected() {
|
|
313
|
-
// Bounded but sequential — must NOT be a score.
|
|
314
|
-
let s = stats_with_range("score", 1.0, 10.0);
|
|
315
|
-
let items: Vec<Value> = (1..=10).map(|i| json!({"score": i})).collect();
|
|
316
|
-
let (is_score, _) = detect_score_field_statistically(&s, &items);
|
|
317
|
-
assert!(!is_score);
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
#[test]
|
|
321
|
-
fn score_field_unbounded_range_rejected() {
|
|
322
|
-
// Range [0, 1000] doesn't match any of the bounded-range cases.
|
|
323
|
-
let s = stats_with_range("metric", 0.0, 1000.0);
|
|
324
|
-
let items: Vec<Value> = (0..10).map(|i| json!({"metric": i * 100})).collect();
|
|
325
|
-
let (is_score, _) = detect_score_field_statistically(&s, &items);
|
|
326
|
-
assert!(!is_score);
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
#[test]
|
|
330
|
-
fn score_field_signed_similarity_range() {
|
|
331
|
-
// Range [-1, 1] is the cosine-similarity bucket (+0.35).
|
|
332
|
-
let s = stats_with_range("similarity", -0.9, 0.95);
|
|
333
|
-
let items: Vec<Value> = (0..10)
|
|
334
|
-
.rev()
|
|
335
|
-
.map(|i| json!({"similarity": (i as f64) / 10.0 - 0.5}))
|
|
336
|
-
.collect();
|
|
337
|
-
let (is_score, _) = detect_score_field_statistically(&s, &items);
|
|
338
|
-
assert!(is_score);
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
#[test]
|
|
342
|
-
fn score_field_below_threshold_rejected() {
|
|
343
|
-
// Range [0, 100] alone (+0.25) is below the 0.4 threshold.
|
|
344
|
-
// No descending sort, no float fraction → stays below 0.4.
|
|
345
|
-
let s = stats_with_range("metric", 0.0, 100.0);
|
|
346
|
-
// Random unsorted ints — no descending hint.
|
|
347
|
-
let items: Vec<Value> = vec![
|
|
348
|
-
json!({"metric": 50}),
|
|
349
|
-
json!({"metric": 10}),
|
|
350
|
-
json!({"metric": 80}),
|
|
351
|
-
json!({"metric": 20}),
|
|
352
|
-
json!({"metric": 90}),
|
|
353
|
-
];
|
|
354
|
-
let (is_score, _) = detect_score_field_statistically(&s, &items);
|
|
355
|
-
assert!(!is_score);
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
#[test]
|
|
359
|
-
fn score_field_non_numeric_rejected() {
|
|
360
|
-
let s = stats("name", "string", 0.5);
|
|
361
|
-
let items: Vec<Value> = vec![
|
|
362
|
-
json!({"name": "alice"}),
|
|
363
|
-
json!({"name": "bob"}),
|
|
364
|
-
json!({"name": "alice"}),
|
|
365
|
-
];
|
|
366
|
-
let (is_score, _) = detect_score_field_statistically(&s, &items);
|
|
367
|
-
assert!(!is_score);
|
|
368
|
-
}
|
|
369
|
-
|
|
370
|
-
#[test]
|
|
371
|
-
fn score_field_missing_min_max_rejected() {
|
|
372
|
-
let s = stats("score", "numeric", 1.0); // no min/max set
|
|
373
|
-
let items: Vec<Value> = vec![json!({"score": 0.5})];
|
|
374
|
-
let (is_score, _) = detect_score_field_statistically(&s, &items);
|
|
375
|
-
assert!(!is_score);
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
#[test]
|
|
379
|
-
fn score_field_confidence_capped_at_95() {
|
|
380
|
-
// Range [0, 1] (+0.4) + descending sort (+0.3) + float fraction
|
|
381
|
-
// (+0.1) = 0.8. Below cap. To exceed 0.95 we'd need >0.55 from
|
|
382
|
-
// bonuses, which isn't possible with current rules. Pin the cap
|
|
383
|
-
// anyway by constructing the highest-scoring case and confirming
|
|
384
|
-
// it doesn't go above 0.95.
|
|
385
|
-
let s = stats_with_range("score", 0.0, 1.0);
|
|
386
|
-
let items: Vec<Value> = (0..50)
|
|
387
|
-
.rev()
|
|
388
|
-
.map(|i| json!({"score": (i as f64) / 50.0}))
|
|
389
|
-
.collect();
|
|
390
|
-
let (_, conf) = detect_score_field_statistically(&s, &items);
|
|
391
|
-
assert!(conf <= 0.95);
|
|
392
|
-
}
|
|
393
|
-
}
|
|
@@ -1,72 +0,0 @@
|
|
|
1
|
-
//! Field-name hashing for cache keys.
|
|
2
|
-
//!
|
|
3
|
-
//! Direct port of `_hash_field_name` (Python `smart_crusher.py:171-177`).
|
|
4
|
-
//! Used to look up TOIN-anonymized `preserve_fields` — TOIN stores
|
|
5
|
-
//! field names as **SHA-256[:8]** for privacy (per Python doc-comment
|
|
6
|
-
//! at `smart_crusher.py:174-175`), so cache lookups will silently miss
|
|
7
|
-
//! if the truncation length drifts.
|
|
8
|
-
//!
|
|
9
|
-
//! # 16 vs 8 — got it wrong once, now pinned
|
|
10
|
-
//!
|
|
11
|
-
//! The first version of this file used `[:16]` based on a misread of
|
|
12
|
-
//! the Python source. Code review caught the discrepancy: Python uses
|
|
13
|
-
//! `[:8]`. Cache lookups against TOIN's 8-char hashes would have
|
|
14
|
-
//! silently missed every field, defeating the entire `use_feedback_hints`
|
|
15
|
-
//! path. Fixed here; the tests now pin against Python `[:8]` reference
|
|
16
|
-
//! values verified via `python3 -c "...hexdigest()[:8]"`.
|
|
17
|
-
|
|
18
|
-
use sha2::{Digest, Sha256};
|
|
19
|
-
|
|
20
|
-
/// SHA-256 of the UTF-8 bytes, hex-encoded, truncated to **8** chars.
|
|
21
|
-
///
|
|
22
|
-
/// Python equivalent: `hashlib.sha256(field_name.encode()).hexdigest()[:8]`.
|
|
23
|
-
/// Lowercase hex — both Python `hexdigest()` and Rust's `sha2` default
|
|
24
|
-
/// to lowercase, so this is consistent without manual case-coercion.
|
|
25
|
-
pub fn hash_field_name(field_name: &str) -> String {
|
|
26
|
-
let mut hasher = Sha256::new();
|
|
27
|
-
hasher.update(field_name.as_bytes());
|
|
28
|
-
let digest = hasher.finalize();
|
|
29
|
-
// Truncate to first 8 hex chars (4 bytes of digest). MUST match
|
|
30
|
-
// Python's `[:8]` — see module-level note above.
|
|
31
|
-
let hex = format!("{:x}", digest);
|
|
32
|
-
hex[..8].to_string()
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
#[cfg(test)]
|
|
36
|
-
mod tests {
|
|
37
|
-
use super::*;
|
|
38
|
-
|
|
39
|
-
#[test]
|
|
40
|
-
fn matches_python_sha256_truncated_to_8() {
|
|
41
|
-
// Verified against Python: hashlib.sha256(b"customer_id").hexdigest()[:8]
|
|
42
|
-
assert_eq!(hash_field_name("customer_id"), "1e38d67d");
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
#[test]
|
|
46
|
-
fn empty_string() {
|
|
47
|
-
// Verified against Python: hashlib.sha256(b"").hexdigest()[:8]
|
|
48
|
-
assert_eq!(hash_field_name(""), "e3b0c442");
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
#[test]
|
|
52
|
-
fn unicode_field_name() {
|
|
53
|
-
// Verified against Python: hashlib.sha256("café".encode()).hexdigest()[:8]
|
|
54
|
-
// UTF-8 bytes for "café" are 63 61 66 c3 a9 — must encode same way.
|
|
55
|
-
assert_eq!(hash_field_name("café"), "850f7dc4");
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
#[test]
|
|
59
|
-
fn deterministic() {
|
|
60
|
-
// Same input → same output across calls.
|
|
61
|
-
assert_eq!(hash_field_name("test"), hash_field_name("test"));
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
#[test]
|
|
65
|
-
fn output_length_is_8() {
|
|
66
|
-
// Always exactly 8 hex chars regardless of input length.
|
|
67
|
-
// This must match Python's `[:8]`; if you change it, every TOIN
|
|
68
|
-
// preserve-field lookup silently misses.
|
|
69
|
-
assert_eq!(hash_field_name("a").len(), 8);
|
|
70
|
-
assert_eq!(hash_field_name(&"x".repeat(1000)).len(), 8);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
@@ -1,81 +0,0 @@
|
|
|
1
|
-
//! Smart statistical tool output compression — Rust port of
|
|
2
|
-
//! `headroom/transforms/smart_crusher.py`.
|
|
3
|
-
//!
|
|
4
|
-
//! # Stage 3c.1: like-for-like parity port
|
|
5
|
-
//!
|
|
6
|
-
//! This module is a literal Rust port of the Python `SmartCrusher`
|
|
7
|
-
//! implementation. The goal of Stage 3c.1 is **byte-equal output parity** for
|
|
8
|
-
//! every fixture in `tests/parity/fixtures/smart_crusher/`. Architectural
|
|
9
|
-
//! improvements (lossless-first, unified saliency score, structured CCR
|
|
10
|
-
//! markers, token budget) are deferred to Stage 3c.2 and tracked in
|
|
11
|
-
//! `~/Desktop/SmartCrusher-Architecture-Improvements.md`.
|
|
12
|
-
//!
|
|
13
|
-
//! # Bugs fixed in BOTH Python and Rust during 3c.1
|
|
14
|
-
//!
|
|
15
|
-
//! Four defects in the Python source (`headroom/transforms/smart_crusher.py`)
|
|
16
|
-
//! were caught during port review. They're fixed in both languages
|
|
17
|
-
//! simultaneously so the parity fixtures continue to byte-match:
|
|
18
|
-
//!
|
|
19
|
-
//! - **k-split overshoot** (line 2722): `_compute_k_split` keeps 2 items when
|
|
20
|
-
//! `k_total = 1` because `max(1, round(k_total * fraction))` floors both
|
|
21
|
-
//! first and last to 1. Violates `max_items_after_crush`.
|
|
22
|
-
//! - **sequential-pattern false positive** (line 444): `_detect_sequential_pattern`
|
|
23
|
-
//! does `int("001")` and silently loses zero padding. Padded string IDs
|
|
24
|
-
//! misclassified as sequential numeric IDs.
|
|
25
|
-
//! - **rare-status detection short-circuit** (line 674): `_detect_rare_status_values`
|
|
26
|
-
//! exits early at >10 distinct values. Datasets with 50+ error codes lose
|
|
27
|
-
//! rare-error preservation.
|
|
28
|
-
//! - **percentile off-by-one** (line 2844): For `len < 8`, integer-division
|
|
29
|
-
//! percentile indices are off by one. Cosmetic — only affects strategy
|
|
30
|
-
//! debug strings.
|
|
31
|
-
//!
|
|
32
|
-
//! Each fix has a fixture entry in the parity harness and a corresponding
|
|
33
|
-
//! test in `tests/test_transforms/test_smart_crusher_bugs.py`.
|
|
34
|
-
|
|
35
|
-
mod analyzer;
|
|
36
|
-
mod anchors;
|
|
37
|
-
mod builder;
|
|
38
|
-
mod classifier;
|
|
39
|
-
pub mod compaction;
|
|
40
|
-
mod config;
|
|
41
|
-
mod constraints;
|
|
42
|
-
mod crusher;
|
|
43
|
-
mod crushers;
|
|
44
|
-
mod error_keywords;
|
|
45
|
-
mod field_detect;
|
|
46
|
-
mod hashing;
|
|
47
|
-
mod observer;
|
|
48
|
-
mod orchestration;
|
|
49
|
-
mod outliers;
|
|
50
|
-
mod planning;
|
|
51
|
-
mod statistics;
|
|
52
|
-
mod stats_math;
|
|
53
|
-
mod traits;
|
|
54
|
-
mod types;
|
|
55
|
-
|
|
56
|
-
pub use analyzer::SmartAnalyzer;
|
|
57
|
-
pub use anchors::{extract_query_anchors, item_matches_anchors};
|
|
58
|
-
pub use builder::SmartCrusherBuilder;
|
|
59
|
-
pub use classifier::{classify_array, ArrayType};
|
|
60
|
-
pub use config::SmartCrusherConfig;
|
|
61
|
-
pub use constraints::{
|
|
62
|
-
default_oss_constraints, KeepErrorsConstraint, KeepStructuralOutliersConstraint,
|
|
63
|
-
};
|
|
64
|
-
pub use crusher::{CrushArrayResult, SmartCrusher};
|
|
65
|
-
pub use crushers::{compute_k_split, crush_number_array, crush_object, crush_string_array};
|
|
66
|
-
pub use error_keywords::ERROR_KEYWORDS;
|
|
67
|
-
pub use field_detect::{detect_id_field_statistically, detect_score_field_statistically};
|
|
68
|
-
pub use hashing::hash_field_name;
|
|
69
|
-
pub use observer::TracingObserver;
|
|
70
|
-
pub use orchestration::{deduplicate_indices_by_content, fill_remaining_slots, prioritize_indices};
|
|
71
|
-
pub use outliers::{
|
|
72
|
-
detect_error_items_for_preservation, detect_rare_status_values, detect_structural_outliers,
|
|
73
|
-
};
|
|
74
|
-
pub use planning::{item_has_preserve_field_match, map_to_anchor_pattern, SmartCrusherPlanner};
|
|
75
|
-
pub use statistics::{calculate_string_entropy, detect_sequential_pattern, is_uuid_format};
|
|
76
|
-
pub use stats_math::{format_g, mean, median, sample_stdev, sample_variance};
|
|
77
|
-
pub use traits::{Constraint, CrushEvent, Observer, Scorer};
|
|
78
|
-
pub use types::{
|
|
79
|
-
ArrayAnalysis, CompressionPlan, CompressionStrategy, CrushResult, CrushabilityAnalysis,
|
|
80
|
-
FieldStats,
|
|
81
|
-
};
|
|
@@ -1,66 +0,0 @@
|
|
|
1
|
-
//! OSS default `Observer` implementation.
|
|
2
|
-
//!
|
|
3
|
-
//! Ships [`TracingObserver`] which writes each `CrushEvent` to the
|
|
4
|
-
//! `tracing` crate at `debug` level. Subscribers that filter `debug`
|
|
5
|
-
//! out (typical production config) pay nothing — `tracing` stops
|
|
6
|
-
//! evaluation at the level check before constructing the event
|
|
7
|
-
//! fields. Subscribers that retain `debug` get a structured per-crush
|
|
8
|
-
//! event suitable for log analytics.
|
|
9
|
-
//!
|
|
10
|
-
//! Enterprise consumers ship richer observers — `AuditObserver` for
|
|
11
|
-
//! SOC2/HIPAA decision logs, `MetricsObserver` for Datadog/Atlas
|
|
12
|
-
//! gauges, `LoopTrainingObserver` to stream events to Headroom Loop —
|
|
13
|
-
//! all on the same trait.
|
|
14
|
-
|
|
15
|
-
use super::traits::{CrushEvent, Observer};
|
|
16
|
-
|
|
17
|
-
/// Writes each `CrushEvent` to the `tracing` crate at `debug` level.
|
|
18
|
-
/// Zero-cost when the subscriber filters `debug` out.
|
|
19
|
-
#[derive(Debug, Default, Clone, Copy)]
|
|
20
|
-
pub struct TracingObserver;
|
|
21
|
-
|
|
22
|
-
impl Observer for TracingObserver {
|
|
23
|
-
fn name(&self) -> &str {
|
|
24
|
-
"tracing"
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
fn on_event(&self, event: &CrushEvent) {
|
|
28
|
-
// `tracing::debug!` is a macro; the level check happens before
|
|
29
|
-
// the fields are evaluated, so this is essentially free at
|
|
30
|
-
// higher log levels.
|
|
31
|
-
tracing::debug!(
|
|
32
|
-
target: "headroom::smart_crusher",
|
|
33
|
-
strategy = %event.strategy,
|
|
34
|
-
input_bytes = event.input_bytes,
|
|
35
|
-
output_bytes = event.output_bytes,
|
|
36
|
-
elapsed_ns = event.elapsed_ns,
|
|
37
|
-
was_modified = event.was_modified,
|
|
38
|
-
"smart_crusher.crush emitted",
|
|
39
|
-
);
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
#[cfg(test)]
|
|
44
|
-
mod tests {
|
|
45
|
-
use super::*;
|
|
46
|
-
|
|
47
|
-
#[test]
|
|
48
|
-
fn tracing_observer_does_not_panic_on_event() {
|
|
49
|
-
// We can't easily assert what `tracing` did without a
|
|
50
|
-
// subscriber set up — but the call must not panic, and the
|
|
51
|
-
// event fields must be readable.
|
|
52
|
-
let event = CrushEvent {
|
|
53
|
-
strategy: "passthrough".to_string(),
|
|
54
|
-
input_bytes: 100,
|
|
55
|
-
output_bytes: 100,
|
|
56
|
-
elapsed_ns: 0,
|
|
57
|
-
was_modified: false,
|
|
58
|
-
};
|
|
59
|
-
TracingObserver.on_event(&event);
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
#[test]
|
|
63
|
-
fn tracing_observer_name_is_stable() {
|
|
64
|
-
assert_eq!(TracingObserver.name(), "tracing");
|
|
65
|
-
}
|
|
66
|
-
}
|