xithead 1.5.0 → 1.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (649) hide show
  1. package/design-system/README.md +80 -0
  2. package/package.json +20 -1
  3. package/public/apple-touch-icon.png +0 -0
  4. package/public/logo-transparent.png +0 -0
  5. package/public/xithead.ps1 +937 -0
  6. package/scripts/README.md +77 -0
  7. package/scripts/audit_wheel_glibc_symbols.py +217 -0
  8. package/scripts/build-master-design-system.cjs +33 -0
  9. package/scripts/build_rust_extension.sh +58 -0
  10. package/scripts/changelog-gen.py +203 -0
  11. package/scripts/eval_output_shaper.py +235 -0
  12. package/scripts/export_kompress_v2_onnx.py +267 -0
  13. package/scripts/fixtures/anthropic_replay_body.json +133 -0
  14. package/scripts/fixtures/codex_response_create_frame.json +52 -0
  15. package/scripts/install-git-hooks.sh +104 -0
  16. package/scripts/install.ps1 +1809 -0
  17. package/scripts/install.sh +1673 -0
  18. package/scripts/pr-governance.py +297 -0
  19. package/scripts/record_fixtures.py +59 -0
  20. package/scripts/refresh_model_limits.sh +48 -0
  21. package/scripts/replay_codex_ws_load.py +445 -0
  22. package/scripts/repro_codex_replay.py +699 -0
  23. package/scripts/smoke_issue_327.py +233 -0
  24. package/scripts/sync-plugin-versions.py +116 -0
  25. package/scripts/tests/test_changelog_gen.py +302 -0
  26. package/scripts/tests/test_pr_governance.py +163 -0
  27. package/scripts/tests/test_pr_health_labels.py +94 -0
  28. package/scripts/tests/test_pr_health_workflow.py +14 -0
  29. package/scripts/tests/test_sync_plugin_versions.py +131 -0
  30. package/scripts/tests/test_version_sync.py +291 -0
  31. package/scripts/validate-workflows.sh +39 -0
  32. package/scripts/verify-versions.py +106 -0
  33. package/scripts/version-sync.py +207 -0
  34. package/tailwind.config.js +31 -0
  35. package/tsconfig.json +84 -0
  36. package/.actrc +0 -14
  37. package/.actrc.local.example +0 -15
  38. package/.changelog.md +0 -2
  39. package/.claude/agents/academic-anthropologist.md +0 -125
  40. package/.claude/agents/academic-geographer.md +0 -127
  41. package/.claude/agents/academic-historian.md +0 -123
  42. package/.claude/agents/academic-narratologist.md +0 -118
  43. package/.claude/agents/academic-psychologist.md +0 -118
  44. package/.claude/agents/academic-statistician.md +0 -144
  45. package/.claude/agents/accounts-payable-agent.md +0 -205
  46. package/.claude/agents/agentic-identity-trust.md +0 -394
  47. package/.claude/agents/agents-orchestrator.md +0 -367
  48. package/.claude/agents/automation-governance-architect.md +0 -216
  49. package/.claude/agents/blender-addon-engineer.md +0 -243
  50. package/.claude/agents/business-strategist.md +0 -488
  51. package/.claude/agents/change-management-consultant.md +0 -497
  52. package/.claude/agents/chief-financial-officer.md +0 -388
  53. package/.claude/agents/corporate-training-designer.md +0 -192
  54. package/.claude/agents/customer-service.md +0 -398
  55. package/.claude/agents/customer-success-manager.md +0 -460
  56. package/.claude/agents/data-consolidation-agent.md +0 -60
  57. package/.claude/agents/data-privacy-officer.md +0 -412
  58. package/.claude/agents/design-brand-guardian.md +0 -322
  59. package/.claude/agents/design-image-prompt-engineer.md +0 -236
  60. package/.claude/agents/design-inclusive-visuals-specialist.md +0 -73
  61. package/.claude/agents/design-persona-walkthrough.md +0 -272
  62. package/.claude/agents/design-ui-designer.md +0 -386
  63. package/.claude/agents/design-ui-finish-gate-reviewer.md +0 -217
  64. package/.claude/agents/design-ux-architect.md +0 -482
  65. package/.claude/agents/design-ux-researcher.md +0 -329
  66. package/.claude/agents/design-visual-storyteller.md +0 -149
  67. package/.claude/agents/design-whimsy-injector.md +0 -460
  68. package/.claude/agents/economy-designer.md +0 -156
  69. package/.claude/agents/engineering-ai-data-remediation-engineer.md +0 -211
  70. package/.claude/agents/engineering-ai-engineer.md +0 -146
  71. package/.claude/agents/engineering-api-platform-engineer.md +0 -181
  72. package/.claude/agents/engineering-ats-validator-architect.md +0 -383
  73. package/.claude/agents/engineering-autonomous-optimization-architect.md +0 -121
  74. package/.claude/agents/engineering-backend-architect.md +0 -255
  75. package/.claude/agents/engineering-china-network-engineer.md +0 -252
  76. package/.claude/agents/engineering-cms-developer.md +0 -536
  77. package/.claude/agents/engineering-code-reviewer.md +0 -76
  78. package/.claude/agents/engineering-codebase-onboarding-engineer.md +0 -173
  79. package/.claude/agents/engineering-data-engineer.md +0 -349
  80. package/.claude/agents/engineering-data-visualization-engineer.md +0 -151
  81. package/.claude/agents/engineering-database-optimizer.md +0 -188
  82. package/.claude/agents/engineering-database-reliability-engineer.md +0 -201
  83. package/.claude/agents/engineering-desktop-app-engineer.md +0 -209
  84. package/.claude/agents/engineering-developer-tooling-engineer.md +0 -153
  85. package/.claude/agents/engineering-devops-automator.md +0 -385
  86. package/.claude/agents/engineering-drupal-performance.md +0 -347
  87. package/.claude/agents/engineering-drupal-shopping-cart.md +0 -360
  88. package/.claude/agents/engineering-electrical-engineer.md +0 -575
  89. package/.claude/agents/engineering-email-intelligence-engineer.md +0 -390
  90. package/.claude/agents/engineering-embedded-firmware-engineer.md +0 -196
  91. package/.claude/agents/engineering-feishu-integration-developer.md +0 -606
  92. package/.claude/agents/engineering-filament-optimization-specialist.md +0 -283
  93. package/.claude/agents/engineering-finops-engineer.md +0 -164
  94. package/.claude/agents/engineering-frontend-developer.md +0 -257
  95. package/.claude/agents/engineering-gaussdb-expert.md +0 -334
  96. package/.claude/agents/engineering-git-workflow-master.md +0 -98
  97. package/.claude/agents/engineering-i18n-engineer.md +0 -214
  98. package/.claude/agents/engineering-identity-access-engineer.md +0 -217
  99. package/.claude/agents/engineering-incident-response-commander.md +0 -444
  100. package/.claude/agents/engineering-iot-fleet-engineer.md +0 -148
  101. package/.claude/agents/engineering-it-service-manager.md +0 -561
  102. package/.claude/agents/engineering-knowledge-graph-engineer.md +0 -372
  103. package/.claude/agents/engineering-llm-post-training-engineer.md +0 -166
  104. package/.claude/agents/engineering-minimal-change-engineer.md +0 -207
  105. package/.claude/agents/engineering-mobile-app-builder.md +0 -493
  106. package/.claude/agents/engineering-mobile-release-engineer.md +0 -164
  107. package/.claude/agents/engineering-multi-agent-systems-architect.md +0 -600
  108. package/.claude/agents/engineering-network-engineer.md +0 -245
  109. package/.claude/agents/engineering-orgscript-engineer.md +0 -113
  110. package/.claude/agents/engineering-payments-billing-engineer.md +0 -228
  111. package/.claude/agents/engineering-pdf-engine-architect.md +0 -696
  112. package/.claude/agents/engineering-platform-engineer.md +0 -270
  113. package/.claude/agents/engineering-privacy-engineer.md +0 -152
  114. package/.claude/agents/engineering-prompt-engineer.md +0 -209
  115. package/.claude/agents/engineering-rag-pipeline-engineer.md +0 -450
  116. package/.claude/agents/engineering-rapid-prototyper.md +0 -466
  117. package/.claude/agents/engineering-realtime-collaboration-engineer.md +0 -200
  118. package/.claude/agents/engineering-rust-refactoring-specialist.md +0 -313
  119. package/.claude/agents/engineering-search-relevance-engineer.md +0 -237
  120. package/.claude/agents/engineering-section-508-specialist.md +0 -339
  121. package/.claude/agents/engineering-senior-developer.md +0 -176
  122. package/.claude/agents/engineering-servicenow-developer-mentor.md +0 -180
  123. package/.claude/agents/engineering-software-architect.md +0 -112
  124. package/.claude/agents/engineering-solidity-smart-contract-engineer.md +0 -522
  125. package/.claude/agents/engineering-sre.md +0 -90
  126. package/.claude/agents/engineering-technical-writer.md +0 -393
  127. package/.claude/agents/engineering-universal-document-compiler.md +0 -370
  128. package/.claude/agents/engineering-uswds-developer.md +0 -340
  129. package/.claude/agents/engineering-video-streaming-engineer.md +0 -151
  130. package/.claude/agents/engineering-voice-ai-integration-engineer.md +0 -584
  131. package/.claude/agents/engineering-webassembly-engineer.md +0 -153
  132. package/.claude/agents/engineering-wechat-mini-program-developer.md +0 -356
  133. package/.claude/agents/engineering-wordpress-performance.md +0 -346
  134. package/.claude/agents/engineering-wordpress-shopping-cart.md +0 -346
  135. package/.claude/agents/esg-sustainability-officer.md +0 -396
  136. package/.claude/agents/finance-bookkeeper-controller.md +0 -260
  137. package/.claude/agents/finance-financial-analyst.md +0 -234
  138. package/.claude/agents/finance-fpa-analyst.md +0 -263
  139. package/.claude/agents/finance-investment-researcher.md +0 -272
  140. package/.claude/agents/finance-tax-strategist.md +0 -239
  141. package/.claude/agents/game-audio-engineer.md +0 -264
  142. package/.claude/agents/game-designer.md +0 -167
  143. package/.claude/agents/gis-3d-scene-developer.md +0 -111
  144. package/.claude/agents/gis-analyst.md +0 -91
  145. package/.claude/agents/gis-bim-specialist.md +0 -108
  146. package/.claude/agents/gis-cartography-designer.md +0 -150
  147. package/.claude/agents/gis-drone-reality-mapping.md +0 -120
  148. package/.claude/agents/gis-geoai-ml-engineer.md +0 -105
  149. package/.claude/agents/gis-geoprocessing-specialist.md +0 -97
  150. package/.claude/agents/gis-qa-engineer.md +0 -133
  151. package/.claude/agents/gis-solution-engineer.md +0 -101
  152. package/.claude/agents/gis-spatial-data-engineer.md +0 -97
  153. package/.claude/agents/gis-spatial-data-scientist.md +0 -111
  154. package/.claude/agents/gis-technical-consultant.md +0 -86
  155. package/.claude/agents/gis-web-gis-developer.md +0 -108
  156. package/.claude/agents/godot-gameplay-scripter.md +0 -338
  157. package/.claude/agents/godot-multiplayer-engineer.md +0 -301
  158. package/.claude/agents/godot-shader-developer.md +0 -267
  159. package/.claude/agents/government-digital-presales-consultant.md +0 -363
  160. package/.claude/agents/grant-writer.md +0 -511
  161. package/.claude/agents/healthcare-aging-parent-care-companion.md +0 -414
  162. package/.claude/agents/healthcare-clinical-evidence-agent.md +0 -231
  163. package/.claude/agents/healthcare-customer-service.md +0 -389
  164. package/.claude/agents/healthcare-innovation-strategist.md +0 -433
  165. package/.claude/agents/healthcare-marketing-compliance.md +0 -395
  166. package/.claude/agents/healthcare-sovereign-health-systems-agent.md +0 -312
  167. package/.claude/agents/hospitality-guest-services.md +0 -603
  168. package/.claude/agents/hr-onboarding.md +0 -451
  169. package/.claude/agents/identity-graph-operator.md +0 -303
  170. package/.claude/agents/language-translator.md +0 -264
  171. package/.claude/agents/legal-billing-time-tracking.md +0 -569
  172. package/.claude/agents/legal-client-intake.md +0 -492
  173. package/.claude/agents/legal-document-review.md +0 -454
  174. package/.claude/agents/level-designer.md +0 -208
  175. package/.claude/agents/loan-officer-assistant.md +0 -555
  176. package/.claude/agents/lsp-index-engineer.md +0 -320
  177. package/.claude/agents/ma-integration-manager.md +0 -427
  178. package/.claude/agents/macos-spatial-metal-engineer.md +0 -337
  179. package/.claude/agents/marketing-aeo-foundations.md +0 -264
  180. package/.claude/agents/marketing-agentic-search-optimizer.md +0 -279
  181. package/.claude/agents/marketing-ai-citation-strategist.md +0 -172
  182. package/.claude/agents/marketing-app-store-optimizer.md +0 -321
  183. package/.claude/agents/marketing-baidu-seo-specialist.md +0 -226
  184. package/.claude/agents/marketing-bilibili-content-strategist.md +0 -199
  185. package/.claude/agents/marketing-book-co-author.md +0 -110
  186. package/.claude/agents/marketing-carousel-growth-engine.md +0 -204
  187. package/.claude/agents/marketing-china-ecommerce-operator.md +0 -283
  188. package/.claude/agents/marketing-china-market-localization-strategist.md +0 -283
  189. package/.claude/agents/marketing-content-creator.md +0 -54
  190. package/.claude/agents/marketing-cross-border-ecommerce.md +0 -259
  191. package/.claude/agents/marketing-developer-community-builder.md +0 -285
  192. package/.claude/agents/marketing-douyin-strategist.md +0 -149
  193. package/.claude/agents/marketing-email-strategist.md +0 -249
  194. package/.claude/agents/marketing-global-podcast-strategist.md +0 -206
  195. package/.claude/agents/marketing-growth-hacker.md +0 -54
  196. package/.claude/agents/marketing-instagram-curator.md +0 -113
  197. package/.claude/agents/marketing-kuaishou-strategist.md +0 -223
  198. package/.claude/agents/marketing-linkedin-content-creator.md +0 -214
  199. package/.claude/agents/marketing-livestream-commerce-coach.md +0 -305
  200. package/.claude/agents/marketing-multi-platform-publisher.md +0 -217
  201. package/.claude/agents/marketing-podcast-strategist.md +0 -277
  202. package/.claude/agents/marketing-pr-communications-manager.md +0 -473
  203. package/.claude/agents/marketing-private-domain-operator.md +0 -308
  204. package/.claude/agents/marketing-reddit-community-builder.md +0 -123
  205. package/.claude/agents/marketing-seo-specialist.md +0 -370
  206. package/.claude/agents/marketing-short-video-editing-coach.md +0 -412
  207. package/.claude/agents/marketing-social-media-strategist.md +0 -125
  208. package/.claude/agents/marketing-tiktok-strategist.md +0 -125
  209. package/.claude/agents/marketing-twitter-engager.md +0 -126
  210. package/.claude/agents/marketing-video-optimization-specialist.md +0 -119
  211. package/.claude/agents/marketing-wechat-official-account.md +0 -145
  212. package/.claude/agents/marketing-weibo-strategist.md +0 -240
  213. package/.claude/agents/marketing-x-twitter-intelligence-analyst.md +0 -161
  214. package/.claude/agents/marketing-xiaohongshu-specialist.md +0 -138
  215. package/.claude/agents/marketing-zhihu-strategist.md +0 -162
  216. package/.claude/agents/medical-billing-coding-specialist.md +0 -491
  217. package/.claude/agents/narrative-designer.md +0 -243
  218. package/.claude/agents/operations-manager.md +0 -399
  219. package/.claude/agents/organizational-psychologist.md +0 -391
  220. package/.claude/agents/paid-media-auditor.md +0 -71
  221. package/.claude/agents/paid-media-creative-strategist.md +0 -71
  222. package/.claude/agents/paid-media-paid-social-strategist.md +0 -71
  223. package/.claude/agents/paid-media-ppc-strategist.md +0 -71
  224. package/.claude/agents/paid-media-programmatic-buyer.md +0 -71
  225. package/.claude/agents/paid-media-search-query-analyst.md +0 -71
  226. package/.claude/agents/paid-media-tracking-specialist.md +0 -71
  227. package/.claude/agents/personal-growth-mentor.md +0 -159
  228. package/.claude/agents/product-behavioral-nudge-engine.md +0 -83
  229. package/.claude/agents/product-dx-engineer.md +0 -332
  230. package/.claude/agents/product-feedback-synthesizer.md +0 -119
  231. package/.claude/agents/product-manager.md +0 -469
  232. package/.claude/agents/product-sprint-prioritizer.md +0 -154
  233. package/.claude/agents/product-trend-researcher.md +0 -159
  234. package/.claude/agents/project-management-experiment-tracker.md +0 -198
  235. package/.claude/agents/project-management-jira-workflow-steward.md +0 -233
  236. package/.claude/agents/project-management-meeting-notes-specialist.md +0 -95
  237. package/.claude/agents/project-management-project-shepherd.md +0 -194
  238. package/.claude/agents/project-management-studio-operations.md +0 -200
  239. package/.claude/agents/project-management-studio-producer.md +0 -203
  240. package/.claude/agents/project-manager-senior.md +0 -135
  241. package/.claude/agents/real-estate-buyer-seller.md +0 -596
  242. package/.claude/agents/recruitment-specialist.md +0 -515
  243. package/.claude/agents/report-distribution-agent.md +0 -65
  244. package/.claude/agents/research-synthesist.md +0 -136
  245. package/.claude/agents/resume-tailor.md +0 -230
  246. package/.claude/agents/retail-customer-returns.md +0 -566
  247. package/.claude/agents/roblox-avatar-creator.md +0 -297
  248. package/.claude/agents/roblox-experience-designer.md +0 -306
  249. package/.claude/agents/roblox-systems-scripter.md +0 -332
  250. package/.claude/agents/sales-account-strategist.md +0 -227
  251. package/.claude/agents/sales-coach.md +0 -271
  252. package/.claude/agents/sales-data-extraction-agent.md +0 -67
  253. package/.claude/agents/sales-deal-strategist.md +0 -180
  254. package/.claude/agents/sales-discovery-coach.md +0 -225
  255. package/.claude/agents/sales-engineer.md +0 -182
  256. package/.claude/agents/sales-offer-lead-gen-strategist.md +0 -257
  257. package/.claude/agents/sales-outbound-strategist.md +0 -201
  258. package/.claude/agents/sales-outreach.md +0 -425
  259. package/.claude/agents/sales-pipeline-analyst.md +0 -267
  260. package/.claude/agents/sales-proposal-strategist.md +0 -217
  261. package/.claude/agents/security-ai-generated-code-auditor.md +0 -207
  262. package/.claude/agents/security-appsec-engineer.md +0 -536
  263. package/.claude/agents/security-architect.md +0 -307
  264. package/.claude/agents/security-blockchain-security-auditor.md +0 -482
  265. package/.claude/agents/security-cloud-security-architect.md +0 -576
  266. package/.claude/agents/security-compliance-auditor.md +0 -158
  267. package/.claude/agents/security-incident-responder.md +0 -450
  268. package/.claude/agents/security-penetration-tester.md +0 -399
  269. package/.claude/agents/security-secrets-credential-engineer.md +0 -189
  270. package/.claude/agents/security-senior-secops.md +0 -760
  271. package/.claude/agents/security-threat-detection-engineer.md +0 -553
  272. package/.claude/agents/security-threat-intelligence-analyst.md +0 -661
  273. package/.claude/agents/specialized-chief-of-staff.md +0 -279
  274. package/.claude/agents/specialized-civil-engineer.md +0 -356
  275. package/.claude/agents/specialized-codebase-archaeologist.md +0 -341
  276. package/.claude/agents/specialized-cultural-intelligence-strategist.md +0 -88
  277. package/.claude/agents/specialized-developer-advocate.md +0 -317
  278. package/.claude/agents/specialized-document-generator.md +0 -55
  279. package/.claude/agents/specialized-fedramp-rmf-compliance.md +0 -378
  280. package/.claude/agents/specialized-focus-music-architect.md +0 -175
  281. package/.claude/agents/specialized-french-consulting-market.md +0 -194
  282. package/.claude/agents/specialized-korean-business-navigator.md +0 -216
  283. package/.claude/agents/specialized-master-plan-architect.md +0 -157
  284. package/.claude/agents/specialized-mcp-builder.md +0 -260
  285. package/.claude/agents/specialized-model-qa.md +0 -528
  286. package/.claude/agents/specialized-pricing-analyst.md +0 -243
  287. package/.claude/agents/specialized-salesforce-architect.md +0 -182
  288. package/.claude/agents/specialized-strategy-duel-agent.md +0 -130
  289. package/.claude/agents/specialized-workflow-architect.md +0 -597
  290. package/.claude/agents/study-abroad-advisor.md +0 -282
  291. package/.claude/agents/supply-chain-strategist.md +0 -591
  292. package/.claude/agents/support-analytics-reporter.md +0 -387
  293. package/.claude/agents/support-executive-summary-generator.md +0 -212
  294. package/.claude/agents/support-finance-tracker.md +0 -480
  295. package/.claude/agents/support-infrastructure-maintainer.md +0 -628
  296. package/.claude/agents/support-legal-compliance-checker.md +0 -591
  297. package/.claude/agents/support-support-responder.md +0 -607
  298. package/.claude/agents/technical-artist.md +0 -233
  299. package/.claude/agents/terminal-integration-specialist.md +0 -70
  300. package/.claude/agents/testing-accessibility-auditor.md +0 -327
  301. package/.claude/agents/testing-api-tester.md +0 -334
  302. package/.claude/agents/testing-evidence-collector.md +0 -210
  303. package/.claude/agents/testing-performance-benchmarker.md +0 -283
  304. package/.claude/agents/testing-reality-checker.md +0 -249
  305. package/.claude/agents/testing-test-automation-engineer.md +0 -196
  306. package/.claude/agents/testing-test-results-analyzer.md +0 -330
  307. package/.claude/agents/testing-tool-evaluator.md +0 -416
  308. package/.claude/agents/testing-workflow-optimizer.md +0 -463
  309. package/.claude/agents/unity-architect.md +0 -271
  310. package/.claude/agents/unity-editor-tool-developer.md +0 -310
  311. package/.claude/agents/unity-multiplayer-engineer.md +0 -323
  312. package/.claude/agents/unity-shader-graph-artist.md +0 -269
  313. package/.claude/agents/unreal-multiplayer-architect.md +0 -364
  314. package/.claude/agents/unreal-systems-engineer.md +0 -322
  315. package/.claude/agents/unreal-technical-artist.md +0 -256
  316. package/.claude/agents/unreal-world-builder.md +0 -273
  317. package/.claude/agents/visionos-spatial-engineer.md +0 -54
  318. package/.claude/agents/xr-cockpit-interaction-specialist.md +0 -32
  319. package/.claude/agents/xr-immersive-developer.md +0 -32
  320. package/.claude/agents/xr-interface-architect.md +0 -32
  321. package/.claude/agents/zk-steward.md +0 -211
  322. package/.claude-plugin/marketplace.json +0 -30
  323. package/.commitlintrc.json +0 -26
  324. package/.devcontainer/Dockerfile +0 -24
  325. package/.devcontainer/devcontainer.json +0 -70
  326. package/.devcontainer/docker-compose.memory.yml +0 -42
  327. package/.devcontainer/memory-stack/devcontainer.json +0 -84
  328. package/.devcontainer/post-create.sh +0 -83
  329. package/.dockerignore +0 -23
  330. package/.env.act.example +0 -15
  331. package/.env.example +0 -171
  332. package/.eslintrc.json +0 -16
  333. package/.git-blame-ignore-revs +0 -10
  334. package/.gitattributes +0 -2
  335. package/.gitguardian.yaml +0 -50
  336. package/.githooks/pre-commit +0 -6
  337. package/.pre-commit-config.yaml +0 -30
  338. package/.release-please-config-xithead.json +0 -22
  339. package/.release-please-config.json +0 -44
  340. package/.release-please-manifest-xithead.json +0 -3
  341. package/.release-please-manifest.json +0 -3
  342. package/CODE_OF_CONDUCT.md +0 -133
  343. package/CONTRIBUTING.md +0 -133
  344. package/Cargo.lock +0 -5557
  345. package/Cargo.toml +0 -121
  346. package/Dockerfile +0 -38
  347. package/Dockerfile.proxy +0 -48
  348. package/ENTERPRISE.md +0 -5
  349. package/FEATURES_ADVANCED.md +0 -311
  350. package/Headroom-2.gif +0 -0
  351. package/HeadroomDemo-Fast.gif +0 -0
  352. package/MANIFEST.in +0 -4
  353. package/MODERNIZATION_GUIDE.md +0 -358
  354. package/Makefile +0 -140
  355. package/NOTICE +0 -43
  356. package/NPM_TOKEN_SETUP.md +0 -81
  357. package/PR.md +0 -170
  358. package/PUBLISH.md +0 -77
  359. package/README_FRONTEND.md +0 -196
  360. package/REALIGNMENT/00-overview.md +0 -53
  361. package/REALIGNMENT/01-bug-list.md +0 -421
  362. package/REALIGNMENT/02-architecture.md +0 -322
  363. package/REALIGNMENT/03-phase-A-lockdown.md +0 -420
  364. package/REALIGNMENT/04-phase-B-live-zone.md +0 -470
  365. package/REALIGNMENT/05-phase-C-rust-proxy.md +0 -329
  366. package/REALIGNMENT/06-phase-D-bedrock-vertex.md +0 -222
  367. package/REALIGNMENT/07-phase-E-cache-stabilization.md +0 -338
  368. package/REALIGNMENT/08-phase-F-auth-mode.md +0 -254
  369. package/REALIGNMENT/09-phase-G-rtk-observability.md +0 -194
  370. package/REALIGNMENT/10-phase-H-python-retirement.md +0 -214
  371. package/REALIGNMENT/11-phase-I-test-infra.md +0 -385
  372. package/REALIGNMENT/12-decisions-needed.md +0 -196
  373. package/REALIGNMENT/INDEX.md +0 -82
  374. package/REFACTORING_CAMPAIGN.md +0 -277
  375. package/RESTRUCTURE_PLAN.md +0 -100
  376. package/RUST_DEV.md +0 -361
  377. package/SECURITY.md +0 -65
  378. package/SETUP_AUTO_PUBLISH.md +0 -164
  379. package/TESTING-copilot-subscription.md +0 -153
  380. package/codecov.yml +0 -34
  381. package/crates/headroom-core/Cargo.toml +0 -165
  382. package/crates/headroom-core/benches/auth_mode.rs +0 -74
  383. package/crates/headroom-core/benches/ccr_store.rs +0 -224
  384. package/crates/headroom-core/benches/tokenizer.rs +0 -52
  385. package/crates/headroom-core/config/pipeline.toml +0 -151
  386. package/crates/headroom-core/proptest-regressions/transforms/tag_protector.txt +0 -9
  387. package/crates/headroom-core/src/auth_mode.rs +0 -250
  388. package/crates/headroom-core/src/cache_control.rs +0 -414
  389. package/crates/headroom-core/src/ccr/backends/in_memory.rs +0 -317
  390. package/crates/headroom-core/src/ccr/backends/mod.rs +0 -152
  391. package/crates/headroom-core/src/ccr/backends/redis.rs +0 -146
  392. package/crates/headroom-core/src/ccr/backends/sqlite.rs +0 -205
  393. package/crates/headroom-core/src/ccr/mod.rs +0 -119
  394. package/crates/headroom-core/src/compression_policy.rs +0 -505
  395. package/crates/headroom-core/src/lib.rs +0 -33
  396. package/crates/headroom-core/src/relevance/base.rs +0 -139
  397. package/crates/headroom-core/src/relevance/bm25.rs +0 -385
  398. package/crates/headroom-core/src/relevance/embedding.rs +0 -354
  399. package/crates/headroom-core/src/relevance/hybrid.rs +0 -364
  400. package/crates/headroom-core/src/relevance/mod.rs +0 -64
  401. package/crates/headroom-core/src/signals/keyword_detector.rs +0 -433
  402. package/crates/headroom-core/src/signals/line_importance.rs +0 -84
  403. package/crates/headroom-core/src/signals/mod.rs +0 -61
  404. package/crates/headroom-core/src/signals/tiered.rs +0 -141
  405. package/crates/headroom-core/src/tokenizer/estimator.rs +0 -152
  406. package/crates/headroom-core/src/tokenizer/hf_impl.rs +0 -314
  407. package/crates/headroom-core/src/tokenizer/mod.rs +0 -56
  408. package/crates/headroom-core/src/tokenizer/registry.rs +0 -342
  409. package/crates/headroom-core/src/tokenizer/tiktoken_impl.rs +0 -265
  410. package/crates/headroom-core/src/transforms/adaptive_sizer.rs +0 -610
  411. package/crates/headroom-core/src/transforms/anchor_selector.rs +0 -1189
  412. package/crates/headroom-core/src/transforms/content_detector.rs +0 -769
  413. package/crates/headroom-core/src/transforms/detection.rs +0 -222
  414. package/crates/headroom-core/src/transforms/diff_compressor.rs +0 -1685
  415. package/crates/headroom-core/src/transforms/live_zone.rs +0 -2967
  416. package/crates/headroom-core/src/transforms/log_compressor.rs +0 -1295
  417. package/crates/headroom-core/src/transforms/magika_detector.rs +0 -395
  418. package/crates/headroom-core/src/transforms/mod.rs +0 -64
  419. package/crates/headroom-core/src/transforms/pipeline/config.rs +0 -331
  420. package/crates/headroom-core/src/transforms/pipeline/mod.rs +0 -103
  421. package/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rs +0 -524
  422. package/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs +0 -279
  423. package/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs +0 -400
  424. package/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rs +0 -284
  425. package/crates/headroom-core/src/transforms/pipeline/offloads/mod.rs +0 -34
  426. package/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rs +0 -331
  427. package/crates/headroom-core/src/transforms/pipeline/orchestrator.rs +0 -848
  428. package/crates/headroom-core/src/transforms/pipeline/reformats/json_minifier.rs +0 -175
  429. package/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rs +0 -545
  430. package/crates/headroom-core/src/transforms/pipeline/reformats/mod.rs +0 -12
  431. package/crates/headroom-core/src/transforms/pipeline/traits.rs +0 -338
  432. package/crates/headroom-core/src/transforms/recommendations.rs +0 -329
  433. package/crates/headroom-core/src/transforms/safety.rs +0 -215
  434. package/crates/headroom-core/src/transforms/search_compressor.rs +0 -902
  435. package/crates/headroom-core/src/transforms/smart_crusher/analyzer.rs +0 -1262
  436. package/crates/headroom-core/src/transforms/smart_crusher/anchors.rs +0 -407
  437. package/crates/headroom-core/src/transforms/smart_crusher/builder.rs +0 -263
  438. package/crates/headroom-core/src/transforms/smart_crusher/classifier.rs +0 -214
  439. package/crates/headroom-core/src/transforms/smart_crusher/compaction/classifier.rs +0 -283
  440. package/crates/headroom-core/src/transforms/smart_crusher/compaction/compactor.rs +0 -733
  441. package/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rs +0 -961
  442. package/crates/headroom-core/src/transforms/smart_crusher/compaction/ir.rs +0 -252
  443. package/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs +0 -127
  444. package/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs +0 -397
  445. package/crates/headroom-core/src/transforms/smart_crusher/config.rs +0 -173
  446. package/crates/headroom-core/src/transforms/smart_crusher/constraints.rs +0 -159
  447. package/crates/headroom-core/src/transforms/smart_crusher/crusher.rs +0 -1665
  448. package/crates/headroom-core/src/transforms/smart_crusher/crushers.rs +0 -865
  449. package/crates/headroom-core/src/transforms/smart_crusher/error_keywords.rs +0 -77
  450. package/crates/headroom-core/src/transforms/smart_crusher/field_detect.rs +0 -393
  451. package/crates/headroom-core/src/transforms/smart_crusher/hashing.rs +0 -72
  452. package/crates/headroom-core/src/transforms/smart_crusher/mod.rs +0 -81
  453. package/crates/headroom-core/src/transforms/smart_crusher/observer.rs +0 -66
  454. package/crates/headroom-core/src/transforms/smart_crusher/orchestration.rs +0 -459
  455. package/crates/headroom-core/src/transforms/smart_crusher/outliers.rs +0 -507
  456. package/crates/headroom-core/src/transforms/smart_crusher/planning.rs +0 -904
  457. package/crates/headroom-core/src/transforms/smart_crusher/statistics.rs +0 -510
  458. package/crates/headroom-core/src/transforms/smart_crusher/stats_math.rs +0 -287
  459. package/crates/headroom-core/src/transforms/smart_crusher/traits.rs +0 -203
  460. package/crates/headroom-core/src/transforms/smart_crusher/types.rs +0 -263
  461. package/crates/headroom-core/src/transforms/tag_protector.rs +0 -1272
  462. package/crates/headroom-core/src/transforms/unidiff_detector.rs +0 -232
  463. package/crates/headroom-parity/Cargo.toml +0 -26
  464. package/crates/headroom-parity/examples/diff_fixture.rs +0 -66
  465. package/crates/headroom-parity/src/bin/parity_run.rs +0 -78
  466. package/crates/headroom-parity/src/lib.rs +0 -613
  467. package/crates/headroom-proxy/Cargo.toml +0 -119
  468. package/crates/headroom-proxy/data/model_prices_and_context_window.json +0 -39767
  469. package/crates/headroom-proxy/src/bedrock/auth_mode_layer.rs +0 -192
  470. package/crates/headroom-proxy/src/bedrock/envelope.rs +0 -239
  471. package/crates/headroom-proxy/src/bedrock/eventstream.rs +0 -769
  472. package/crates/headroom-proxy/src/bedrock/eventstream_to_sse.rs +0 -459
  473. package/crates/headroom-proxy/src/bedrock/invoke.rs +0 -735
  474. package/crates/headroom-proxy/src/bedrock/invoke_streaming.rs +0 -1098
  475. package/crates/headroom-proxy/src/bedrock/mod.rs +0 -66
  476. package/crates/headroom-proxy/src/bedrock/sigv4.rs +0 -324
  477. package/crates/headroom-proxy/src/bedrock/vendor.rs +0 -82
  478. package/crates/headroom-proxy/src/cache_stabilization/anthropic_cache_control.rs +0 -698
  479. package/crates/headroom-proxy/src/cache_stabilization/drift_detector.rs +0 -683
  480. package/crates/headroom-proxy/src/cache_stabilization/mod.rs +0 -61
  481. package/crates/headroom-proxy/src/cache_stabilization/openai_cache_key.rs +0 -600
  482. package/crates/headroom-proxy/src/cache_stabilization/tool_def_normalize.rs +0 -514
  483. package/crates/headroom-proxy/src/cache_stabilization/volatile_detector.rs +0 -720
  484. package/crates/headroom-proxy/src/compression/anthropic.rs +0 -92
  485. package/crates/headroom-proxy/src/compression/live_zone_anthropic.rs +0 -1311
  486. package/crates/headroom-proxy/src/compression/live_zone_openai.rs +0 -660
  487. package/crates/headroom-proxy/src/compression/live_zone_responses.rs +0 -678
  488. package/crates/headroom-proxy/src/compression/mod.rs +0 -127
  489. package/crates/headroom-proxy/src/compression/model_limits.rs +0 -211
  490. package/crates/headroom-proxy/src/config.rs +0 -646
  491. package/crates/headroom-proxy/src/error.rs +0 -69
  492. package/crates/headroom-proxy/src/handlers/chat_completions.rs +0 -100
  493. package/crates/headroom-proxy/src/handlers/conversations.rs +0 -241
  494. package/crates/headroom-proxy/src/handlers/mod.rs +0 -13
  495. package/crates/headroom-proxy/src/handlers/responses.rs +0 -256
  496. package/crates/headroom-proxy/src/headers.rs +0 -288
  497. package/crates/headroom-proxy/src/health.rs +0 -41
  498. package/crates/headroom-proxy/src/lib.rs +0 -21
  499. package/crates/headroom-proxy/src/main.rs +0 -141
  500. package/crates/headroom-proxy/src/observability/cache_hit_rate.rs +0 -340
  501. package/crates/headroom-proxy/src/observability/compression_ratio.rs +0 -158
  502. package/crates/headroom-proxy/src/observability/metric_names.rs +0 -174
  503. package/crates/headroom-proxy/src/observability/mod.rs +0 -68
  504. package/crates/headroom-proxy/src/observability/prometheus.rs +0 -394
  505. package/crates/headroom-proxy/src/observability/proxy_metrics.rs +0 -368
  506. package/crates/headroom-proxy/src/proxy.rs +0 -1619
  507. package/crates/headroom-proxy/src/responses_items.rs +0 -565
  508. package/crates/headroom-proxy/src/sse/anthropic.rs +0 -415
  509. package/crates/headroom-proxy/src/sse/framing.rs +0 -366
  510. package/crates/headroom-proxy/src/sse/mod.rs +0 -40
  511. package/crates/headroom-proxy/src/sse/openai_chat.rs +0 -231
  512. package/crates/headroom-proxy/src/sse/openai_responses.rs +0 -458
  513. package/crates/headroom-proxy/src/vertex/adc.rs +0 -315
  514. package/crates/headroom-proxy/src/vertex/envelope.rs +0 -167
  515. package/crates/headroom-proxy/src/vertex/mod.rs +0 -277
  516. package/crates/headroom-proxy/src/vertex/raw_predict.rs +0 -490
  517. package/crates/headroom-proxy/src/vertex/stream_raw_predict.rs +0 -49
  518. package/crates/headroom-proxy/src/websocket.rs +0 -246
  519. package/crates/headroom-py/Cargo.toml +0 -37
  520. package/crates/headroom-py/build.rs +0 -77
  521. package/crates/headroom-py/glibc_compat.c +0 -170
  522. package/crates/headroom-py/src/lib.rs +0 -1655
  523. package/deny.toml +0 -32
  524. package/docker/differential-network-capture/Dockerfile.runner +0 -14
  525. package/docker/differential-network-capture/docker-compose.yml +0 -142
  526. package/docker/differential-network-capture/run-claude-lane.sh +0 -24
  527. package/docker/docker-compose.native.yml +0 -44
  528. package/docker-bake.hcl +0 -88
  529. package/docker-compose.yml +0 -80
  530. package/e2e/_lib/make_shim.ps1 +0 -24
  531. package/e2e/_lib/make_shim.sh +0 -28
  532. package/e2e/docker-native-install.sh +0 -120
  533. package/e2e/init/Dockerfile +0 -79
  534. package/e2e/wrap/Dockerfile +0 -115
  535. package/examples/07-context-compression.ipynb +0 -279
  536. package/examples/deployment/macos-launchagent/com.headroom.proxy.plist.template +0 -65
  537. package/examples/deployment/macos-launchagent/install.sh +0 -227
  538. package/examples/deployment/macos-launchagent/shell-integration.sh +0 -74
  539. package/examples/deployment/macos-launchagent/uninstall.sh +0 -145
  540. package/headroom-savings.png +0 -0
  541. package/headroom_learn.gif +0 -0
  542. package/llms.txt +0 -67
  543. package/mkdocs.yml +0 -136
  544. package/npm/bin/xithead.js +0 -366
  545. package/npm/bin/xtream-head.js +0 -366
  546. package/npm/index.d.ts +0 -57
  547. package/npm/index.js +0 -139
  548. package/npm/package.json +0 -55
  549. package/npm/postinstall.js +0 -43
  550. package/plugins/headroom-agent-hooks/.claude-plugin/plugin.json +0 -17
  551. package/plugins/headroom-agent-hooks/hooks/hooks.json +0 -29
  552. package/plugins/headroom-oauth2/LICENSE +0 -3
  553. package/plugins/headroom-oauth2/SPEC.md +0 -115
  554. package/plugins/headroom-oauth2/pyproject.toml +0 -48
  555. package/plugins/hermes/headroom_retrieve/plugin.yaml +0 -6
  556. package/plugins/openclaw/hook-shim/HOOK.md +0 -21
  557. package/plugins/openclaw/hook-shim/handler.js +0 -3
  558. package/plugins/openclaw/openclaw.plugin.json +0 -98
  559. package/plugins/openclaw/package-lock.json +0 -2692
  560. package/plugins/openclaw/package.json +0 -54
  561. package/plugins/openclaw/prepare-dist.mjs +0 -56
  562. package/plugins/openclaw/src/convert.ts +0 -418
  563. package/plugins/openclaw/src/engine.ts +0 -284
  564. package/plugins/openclaw/src/gateway-config.ts +0 -143
  565. package/plugins/openclaw/src/index.ts +0 -11
  566. package/plugins/openclaw/src/plugin/index.ts +0 -98
  567. package/plugins/openclaw/src/proxy-manager.ts +0 -477
  568. package/plugins/openclaw/src/tools/headroom-retrieve.ts +0 -76
  569. package/plugins/openclaw/test/convert.test.ts +0 -96
  570. package/plugins/openclaw/test/engine-normalization.test.ts +0 -21
  571. package/plugins/openclaw/test/engine.test.ts +0 -101
  572. package/plugins/openclaw/test/gateway-config.test.ts +0 -278
  573. package/plugins/openclaw/test/plugin-runtime-routing.test.ts +0 -141
  574. package/plugins/openclaw/test/proxy-manager.test.ts +0 -343
  575. package/plugins/openclaw/tsup.config.ts +0 -9
  576. package/pyproject.toml +0 -438
  577. package/rust-toolchain.toml +0 -16
  578. package/sdk/typescript/examples/basic-compress.ts +0 -64
  579. package/sdk/typescript/examples/ccr-retrieve.ts +0 -80
  580. package/sdk/typescript/examples/hooks-custom-compression.ts +0 -117
  581. package/sdk/typescript/examples/middleware-composition.ts +0 -59
  582. package/sdk/typescript/examples/multi-provider.ts +0 -53
  583. package/sdk/typescript/examples/openai-anthropic-adapters.ts +0 -73
  584. package/sdk/typescript/examples/shared-context-multi-agent.ts +0 -74
  585. package/sdk/typescript/examples/simulation-dry-run.ts +0 -80
  586. package/sdk/typescript/examples/streaming-chat.ts +0 -71
  587. package/sdk/typescript/examples/structured-output.ts +0 -99
  588. package/sdk/typescript/examples/tool-calling-agent.ts +0 -86
  589. package/sdk/typescript/examples/with-headroom-vercel.ts +0 -57
  590. package/sdk/typescript/package-lock.json +0 -3437
  591. package/sdk/typescript/package.json +0 -108
  592. package/sdk/typescript/src/adapters/anthropic.ts +0 -218
  593. package/sdk/typescript/src/adapters/gemini.ts +0 -83
  594. package/sdk/typescript/src/adapters/openai.ts +0 -77
  595. package/sdk/typescript/src/adapters/vercel-ai.ts +0 -131
  596. package/sdk/typescript/src/client.ts +0 -632
  597. package/sdk/typescript/src/compress.ts +0 -87
  598. package/sdk/typescript/src/errors.ts +0 -114
  599. package/sdk/typescript/src/hooks.ts +0 -113
  600. package/sdk/typescript/src/index.ts +0 -140
  601. package/sdk/typescript/src/paths.ts +0 -272
  602. package/sdk/typescript/src/shared-context.ts +0 -188
  603. package/sdk/typescript/src/simulate.ts +0 -52
  604. package/sdk/typescript/src/types/config.ts +0 -183
  605. package/sdk/typescript/src/types/models.ts +0 -314
  606. package/sdk/typescript/src/types.ts +0 -139
  607. package/sdk/typescript/src/utils/case.ts +0 -34
  608. package/sdk/typescript/src/utils/format.ts +0 -480
  609. package/sdk/typescript/src/utils/stream.ts +0 -50
  610. package/sdk/typescript/test/adapters/anthropic.test.ts +0 -217
  611. package/sdk/typescript/test/adapters/gemini.test.ts +0 -131
  612. package/sdk/typescript/test/adapters/openai.test.ts +0 -174
  613. package/sdk/typescript/test/adapters/vercel-ai.test.ts +0 -306
  614. package/sdk/typescript/test/client-expanded.test.ts +0 -333
  615. package/sdk/typescript/test/client.test.ts +0 -266
  616. package/sdk/typescript/test/compress.test.ts +0 -118
  617. package/sdk/typescript/test/config.test.ts +0 -275
  618. package/sdk/typescript/test/errors.test.ts +0 -169
  619. package/sdk/typescript/test/format-integration.test.ts +0 -252
  620. package/sdk/typescript/test/hooks.test.ts +0 -265
  621. package/sdk/typescript/test/integration.test.ts +0 -249
  622. package/sdk/typescript/test/mdx-examples.test.ts +0 -253
  623. package/sdk/typescript/test/models.test.ts +0 -318
  624. package/sdk/typescript/test/parity.test.ts +0 -337
  625. package/sdk/typescript/test/paths.test.ts +0 -447
  626. package/sdk/typescript/test/shared-context.test.ts +0 -245
  627. package/sdk/typescript/test/simulate.test.ts +0 -84
  628. package/sdk/typescript/test/types.test.ts +0 -244
  629. package/sdk/typescript/test/utils/case.test.ts +0 -129
  630. package/sdk/typescript/test/utils/detect-format.test.ts +0 -133
  631. package/sdk/typescript/test/utils/format.test.ts +0 -434
  632. package/sdk/typescript/test/vercel-ai-e2e.test.ts +0 -179
  633. package/sdk/typescript/tsup.config.ts +0 -17
  634. package/uv.lock +0 -6951
  635. package/xithead.egg-info/PKG-INFO +0 -22
  636. package/xithead.egg-info/SOURCES.txt +0 -16
  637. package/xithead.egg-info/dependency_links.txt +0 -1
  638. package/xithead.egg-info/entry_points.txt +0 -2
  639. package/xithead.egg-info/requires.txt +0 -10
  640. package/xithead.egg-info/top_level.txt +0 -2
  641. package/xithead.ps1 +0 -136
  642. package/xithead.sh +0 -129
  643. package/xithead_dist/pyproject.toml +0 -39
  644. package/xithead_dist/xithead.egg-info/PKG-INFO +0 -22
  645. package/xithead_dist/xithead.egg-info/SOURCES.txt +0 -16
  646. package/xithead_dist/xithead.egg-info/dependency_links.txt +0 -1
  647. package/xithead_dist/xithead.egg-info/entry_points.txt +0 -3
  648. package/xithead_dist/xithead.egg-info/requires.txt +0 -10
  649. package/xithead_dist/xithead.egg-info/top_level.txt +0 -2
@@ -1,1272 +0,0 @@
1
- //! Tag protection — keep custom workflow tags out of ML compressors.
2
- //!
3
- //! # Why this exists
4
- //!
5
- //! LLM workflows carry XML-style markers (`<system-reminder>`,
6
- //! `<tool_call>`, `<thinking>`, `<headroom:tool_digest>`, etc.) that
7
- //! downstream code parses as structure. Kompress / LLMLingua sees them
8
- //! as droppable noise and silently strips them, breaking everything
9
- //! that depends on them. ContentRouter calls [`protect_tags`] before
10
- //! every ML-text-compression to swap custom-tag spans for opaque
11
- //! placeholders, runs the compressor on the cleaned body, then calls
12
- //! [`restore_tags`] on the output to splice the originals back in.
13
- //!
14
- //! Standard HTML5 elements (`<div>`, `<p>`, `<span>`, …) are *not*
15
- //! protected — those go through the HTMLExtractor / trafilatura path
16
- //! at a different layer. Anything else is treated as a custom tag.
17
- //!
18
- //! # Algorithm
19
- //!
20
- //! Single-pass tag-stack walker over the input bytes (no regex
21
- //! backtracking, no O(n²) restart loop):
22
- //!
23
- //! 1. Scan forward for `<`. If the next bytes form a valid tag-open
24
- //! (`<name attr=…>` or `<name/>`), classify the tag name.
25
- //! 2. HTML tag → emit verbatim, continue.
26
- //! 3. Custom tag, self-closing → emit a placeholder, record the span.
27
- //! 4. Custom tag, opening → push `(name, start_offset)` onto a stack.
28
- //! 5. `</name>` matching the top of the stack → pop, emit a placeholder
29
- //! for the whole `<name>…</name>` span (when
30
- //! `compress_tagged_content == false`) or emit two placeholders for
31
- //! the markers only (when `compress_tagged_content == true`).
32
- //! 6. Mismatched close (HTML close while a custom tag is on top, or a
33
- //! close with no matching open) → write the close tag verbatim and
34
- //! move on. The walker never attempts to "repair" malformed input.
35
- //!
36
- //! Output is built incrementally with offset-based slicing — never the
37
- //! Python-original's `result.replace(original, placeholder, 1)`, which
38
- //! silently misbehaves when two identical custom-tag blocks appear in
39
- //! the same input (it always replaces the *first* textual occurrence,
40
- //! not the matched one). See `fixed_in_3e4_replace_first` test below.
41
- //!
42
- //! # Bug fixes vs the Python original
43
- //!
44
- //! * **#1: O(n²) on nested custom tags** — the Python loop restarted a
45
- //! full regex scan after every replacement. Rust walks once, in
46
- //! linear time on input length.
47
- //! * **#2: First-occurrence replace bug** — `str.replace(.., .., 1)`
48
- //! replaced the first textual match of the matched block, not the
49
- //! block at the matched offset. Two identical custom-tag blocks in
50
- //! the same input collapsed to one placeholder + a duplicated
51
- //! second block. The Rust walker stitches output by offset.
52
- //! * **#3: Silent 50-iteration cap** — Python had a hard 50-iteration
53
- //! safety limit that quietly truncated tag protection on deeply
54
- //! nested input. The Rust walker's run-time is bounded by input
55
- //! length only.
56
- //! * **#4: Self-closing pass duplicate-replace risk** — Python ran a
57
- //! second loop with the same `replace_first` bug for self-closing
58
- //! tags. Rust handles self-closers in the same single pass.
59
- //! * **#5: Placeholder collision** — if input contains a literal
60
- //! `{{HEADROOM_TAG_…}}` substring, Python silently let the collision
61
- //! stand. We detect that and pick a salted prefix (with a tracing
62
- //! warn) so restoration can't be ambiguous.
63
- //!
64
- //! # Hot path
65
- //!
66
- //! `protect_tags` runs on every ML-compression call from ContentRouter.
67
- //! Most production prompts have 0–10 custom tags so the absolute cost
68
- //! is small either way; the value of the port is correctness (bugs
69
- //! #2, #5) and predictable behavior on adversarial input (bugs #1,
70
- //! #3). The PyO3 bridge releases the GIL during the walk because the
71
- //! algorithm is fully self-contained.
72
-
73
- use std::collections::HashSet;
74
- use std::sync::OnceLock;
75
-
76
- /// HTML5 living-standard element names — the set of tags this module
77
- /// will NEVER protect (they're handled at a different layer; everything
78
- /// else is treated as custom).
79
- ///
80
- /// Generated from
81
- /// <https://html.spec.whatwg.org/multipage/indices.html#elements-3> and
82
- /// matches the Python `KNOWN_HTML_TAGS` frozenset element-for-element
83
- /// so the Rust shim and the Python shim agree.
84
- const HTML5_TAGS: &[&str] = &[
85
- // Main root
86
- "html",
87
- // Document metadata
88
- "base",
89
- "head",
90
- "link",
91
- "meta",
92
- "style",
93
- "title",
94
- // Sectioning root
95
- "body",
96
- // Content sectioning
97
- "address",
98
- "article",
99
- "aside",
100
- "footer",
101
- "h1",
102
- "h2",
103
- "h3",
104
- "h4",
105
- "h5",
106
- "h6",
107
- "header",
108
- "hgroup",
109
- "main",
110
- "nav",
111
- "section",
112
- "search",
113
- // Text content
114
- "blockquote",
115
- "dd",
116
- "div",
117
- "dl",
118
- "dt",
119
- "figcaption",
120
- "figure",
121
- "hr",
122
- "li",
123
- "menu",
124
- "ol",
125
- "p",
126
- "pre",
127
- "ul",
128
- // Inline text semantics
129
- "a",
130
- "abbr",
131
- "b",
132
- "bdi",
133
- "bdo",
134
- "br",
135
- "cite",
136
- "code",
137
- "data",
138
- "dfn",
139
- "em",
140
- "i",
141
- "kbd",
142
- "mark",
143
- "q",
144
- "rp",
145
- "rt",
146
- "ruby",
147
- "s",
148
- "samp",
149
- "small",
150
- "span",
151
- "strong",
152
- "sub",
153
- "sup",
154
- "time",
155
- "u",
156
- "var",
157
- "wbr",
158
- // Image and multimedia
159
- "area",
160
- "audio",
161
- "img",
162
- "map",
163
- "track",
164
- "video",
165
- // Embedded content
166
- "embed",
167
- "iframe",
168
- "object",
169
- "param",
170
- "picture",
171
- "portal",
172
- "source",
173
- // SVG and MathML
174
- "svg",
175
- "math",
176
- // Scripting
177
- "canvas",
178
- "noscript",
179
- "script",
180
- // Demarcating edits
181
- "del",
182
- "ins",
183
- // Table content
184
- "caption",
185
- "col",
186
- "colgroup",
187
- "table",
188
- "tbody",
189
- "td",
190
- "tfoot",
191
- "th",
192
- "thead",
193
- "tr",
194
- // Forms
195
- "button",
196
- "datalist",
197
- "fieldset",
198
- "form",
199
- "input",
200
- "label",
201
- "legend",
202
- "meter",
203
- "optgroup",
204
- "option",
205
- "output",
206
- "progress",
207
- "select",
208
- "textarea",
209
- // Interactive
210
- "details",
211
- "dialog",
212
- "summary",
213
- // Web Components
214
- "slot",
215
- "template",
216
- ];
217
-
218
- fn known_html_tags() -> &'static HashSet<&'static str> {
219
- static SET: OnceLock<HashSet<&'static str>> = OnceLock::new();
220
- SET.get_or_init(|| HTML5_TAGS.iter().copied().collect())
221
- }
222
-
223
- /// Default placeholder prefix. Brace-doubled to look unlike anything a
224
- /// real workflow tag would emit. Falls back to a salted variant if the
225
- /// input itself contains the prefix (see [`pick_placeholder_prefix`]).
226
- const DEFAULT_PREFIX: &str = "{{HEADROOM_TAG_";
227
- const PLACEHOLDER_SUFFIX: &str = "}}";
228
-
229
- /// Sidecar diagnostics — same shape every Rust transform uses.
230
- #[derive(Debug, Default, Clone)]
231
- pub struct ProtectStats {
232
- pub tags_seen: usize,
233
- pub html_tags_skipped: usize,
234
- pub custom_blocks_protected: usize,
235
- pub self_closing_protected: usize,
236
- /// Closes that didn't match any open on the stack (malformed input
237
- /// or HTML interleaving). Emitted verbatim. Non-zero is a smell
238
- /// worth tracking but not necessarily a bug.
239
- pub orphan_closes: usize,
240
- /// True iff the placeholder prefix had to be salted because the
241
- /// input contained a literal `{{HEADROOM_TAG_` substring.
242
- pub placeholder_collision_avoided: bool,
243
- }
244
-
245
- /// Case-insensitive HTML tag check. Lowercases the input lazily so we
246
- /// don't allocate for the common ASCII-lowercase case.
247
- pub fn is_known_html_tag(tag_name: &str) -> bool {
248
- let set = known_html_tags();
249
- if set.contains(tag_name) {
250
- return true;
251
- }
252
- if tag_name.bytes().any(|b| b.is_ascii_uppercase()) {
253
- let lower = tag_name.to_ascii_lowercase();
254
- return set.contains(lower.as_str());
255
- }
256
- false
257
- }
258
-
259
- /// Iterate the canonical HTML tag list. Used by the PyO3 shim to
260
- /// expose `KNOWN_HTML_TAGS` to Python without re-declaring the set.
261
- pub fn known_html_tag_names() -> &'static [&'static str] {
262
- HTML5_TAGS
263
- }
264
-
265
- /// Pick a placeholder prefix that doesn't collide with anything in
266
- /// `text`. We try `{{HEADROOM_TAG_` first; if the input contains it
267
- /// literally we salt with a per-call counter until we miss. The salt
268
- /// is bounded; in practice we never need more than one attempt.
269
- fn pick_placeholder_prefix(text: &str) -> (String, bool) {
270
- if !text.contains(DEFAULT_PREFIX) {
271
- return (DEFAULT_PREFIX.to_string(), false);
272
- }
273
- for salt in 0u32..16 {
274
- let candidate = format!("{{{{HEADROOM_TAG_{salt}_");
275
- if !text.contains(&candidate) {
276
- return (candidate, true);
277
- }
278
- }
279
- // 16 salt attempts collided — fall back to a UUID-shaped marker.
280
- // The OnceLock cache is so two consecutive calls in the same
281
- // process don't pay the formatting cost.
282
- static FALLBACK: OnceLock<String> = OnceLock::new();
283
- let prefix = FALLBACK
284
- .get_or_init(|| "{{HEADROOM_TAG_FALLBACK_a4f1c7e2_".to_string())
285
- .clone();
286
- (prefix, true)
287
- }
288
-
289
- #[derive(Debug)]
290
- struct OpenTag {
291
- /// Lowercase name for case-insensitive close-matching.
292
- name_lower: String,
293
- /// Byte offset of the `<` that opened this tag.
294
- open_start: usize,
295
- }
296
-
297
- /// Outcome of a single `<…>` parse attempt at a given offset.
298
- enum TagParse {
299
- /// Opening tag (`<name attr=…>`). `name_end` is exclusive.
300
- Open {
301
- name_end: usize,
302
- tag_end: usize,
303
- is_self_closing: bool,
304
- },
305
- /// Closing tag (`</name>`).
306
- Close { name_end: usize, tag_end: usize },
307
- /// Not a tag (e.g. `<` followed by whitespace or digit).
308
- NotTag,
309
- }
310
-
311
- /// Parse a `<…>` starting at `start`. Returns the byte offset of the
312
- /// closing `>` (exclusive end of the tag) and the kind. Conservatively
313
- /// rejects malformed shapes — we'd rather emit a `<` verbatim than
314
- /// over-protect on bad input.
315
- fn parse_tag_at(bytes: &[u8], start: usize) -> TagParse {
316
- debug_assert!(bytes[start] == b'<');
317
- let mut i = start + 1;
318
- let n = bytes.len();
319
- if i >= n {
320
- return TagParse::NotTag;
321
- }
322
-
323
- let is_close = bytes[i] == b'/';
324
- if is_close {
325
- i += 1;
326
- }
327
- // After consuming a possible '/' we may be at end-of-input
328
- // (e.g. literal `</` with nothing after). Guard the bounds
329
- // before indexing into `bytes[i]` for the name-start check —
330
- // proptest discovered the OOB on input `</`.
331
- if i >= n {
332
- return TagParse::NotTag;
333
- }
334
- let name_start = i;
335
- if !is_name_start(bytes[i]) {
336
- return TagParse::NotTag;
337
- }
338
- i += 1;
339
- while i < n && is_name_cont(bytes[i]) {
340
- i += 1;
341
- }
342
- let name_end = i;
343
- if name_end == name_start {
344
- return TagParse::NotTag;
345
- }
346
-
347
- if is_close {
348
- // Allow optional whitespace, then `>`.
349
- while i < n && bytes[i].is_ascii_whitespace() {
350
- i += 1;
351
- }
352
- if i >= n || bytes[i] != b'>' {
353
- return TagParse::NotTag;
354
- }
355
- return TagParse::Close {
356
- name_end,
357
- tag_end: i + 1,
358
- };
359
- }
360
-
361
- // Opening tag: skip attributes until `>` (handle `/>` for
362
- // self-closing). Quoted attribute values can contain `>`; a
363
- // single-pass attribute lexer handles the common cases.
364
- let mut self_closing = false;
365
- while i < n {
366
- match bytes[i] {
367
- b'>' => {
368
- return TagParse::Open {
369
- name_end,
370
- tag_end: i + 1,
371
- is_self_closing: self_closing,
372
- };
373
- }
374
- b'/' => {
375
- self_closing = true;
376
- i += 1;
377
- }
378
- b'"' | b'\'' => {
379
- let quote = bytes[i];
380
- i += 1;
381
- while i < n && bytes[i] != quote {
382
- i += 1;
383
- }
384
- if i >= n {
385
- return TagParse::NotTag;
386
- }
387
- i += 1;
388
- self_closing = false;
389
- }
390
- _ => {
391
- if bytes[i].is_ascii_whitespace() {
392
- self_closing = false;
393
- }
394
- i += 1;
395
- }
396
- }
397
- }
398
-
399
- TagParse::NotTag
400
- }
401
-
402
- #[inline]
403
- fn is_name_start(b: u8) -> bool {
404
- b.is_ascii_alphabetic() || b == b'_'
405
- }
406
-
407
- #[inline]
408
- fn is_name_cont(b: u8) -> bool {
409
- b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'.' | b':')
410
- }
411
-
412
- /// A single span that was identified as worth replacing.
413
- ///
414
- /// In block mode every matched custom-tag span (open..=close) becomes
415
- /// one Span and is replaced by a single placeholder; self-closing
416
- /// custom tags become a Span covering just the tag bytes.
417
- ///
418
- /// In marker-only mode each opening custom tag and each closing custom
419
- /// tag becomes its own Span (the body between them is left visible to
420
- /// the compressor).
421
- #[derive(Debug, Clone, Copy)]
422
- struct Span {
423
- start: usize,
424
- end: usize,
425
- kind: SpanKind,
426
- }
427
-
428
- #[derive(Debug, Clone, Copy)]
429
- enum SpanKind {
430
- /// Whole `<custom>…</custom>` block (block mode).
431
- Block,
432
- /// Self-closing `<custom/>` (block mode).
433
- SelfClosing,
434
- /// Opening `<custom>` marker (marker-only mode).
435
- OpenMarker,
436
- /// Closing `</custom>` marker (marker-only mode).
437
- CloseMarker,
438
- }
439
-
440
- /// Protect custom workflow tags from text compression.
441
- ///
442
- /// * `compress_tagged_content = false` (default) — replace each entire
443
- /// `<custom>…</custom>` span (including nested children) with a
444
- /// single placeholder. Self-closing custom tags become a single
445
- /// placeholder. The body between the markers is *not* exposed to
446
- /// compression.
447
- /// * `compress_tagged_content = true` — replace only the tag markers
448
- /// (open and close emitted as separate placeholders) so the
449
- /// compressor can squash content while the tag boundaries survive.
450
- ///
451
- /// Returns `(cleaned, blocks, stats)` where `blocks` is a list of
452
- /// `(placeholder, original)` pairs for [`restore_tags`]. The blocks
453
- /// are listed in left-to-right order of appearance in the input, which
454
- /// keeps the restore step trivial.
455
- pub fn protect_tags(
456
- text: &str,
457
- compress_tagged_content: bool,
458
- ) -> (String, Vec<(String, String)>, ProtectStats) {
459
- let mut stats = ProtectStats::default();
460
- if text.is_empty() || !text.contains('<') {
461
- return (text.to_string(), Vec::new(), stats);
462
- }
463
-
464
- let (prefix, salted) = pick_placeholder_prefix(text);
465
- stats.placeholder_collision_avoided = salted;
466
-
467
- // Phase 1: walk once, classify every tag, build a list of spans
468
- // worth replacing. No output emitted yet — this is purely
469
- // discovery so we can decide which byte ranges to swap.
470
- let spans = identify_spans(text, compress_tagged_content, &mut stats);
471
-
472
- // Phase 2: emit. Walk the input once more, splicing placeholders
473
- // for span bytes and copying everything else verbatim. Because
474
- // `spans` is sorted left-to-right and non-overlapping (block mode
475
- // collapses nested matches into the outermost span; marker mode
476
- // emits open/close markers that are byte-disjoint by construction)
477
- // this is a straightforward scan.
478
- match emit_output(text, &spans, &prefix) {
479
- Some((cleaned, blocks)) => (cleaned, blocks, stats),
480
- // Should be unreachable — `identify_spans` returns spans whose
481
- // bytes are slices of `text`. If we ever fail to splice them
482
- // back, fall back to emitting the original.
483
- None => (text.to_string(), Vec::new(), stats),
484
- }
485
- }
486
-
487
- fn identify_spans(
488
- text: &str,
489
- compress_tagged_content: bool,
490
- stats: &mut ProtectStats,
491
- ) -> Vec<Span> {
492
- let bytes = text.as_bytes();
493
- let n = bytes.len();
494
- let mut spans: Vec<Span> = Vec::new();
495
- let mut stack: Vec<OpenTag> = Vec::new();
496
-
497
- let mut i = 0;
498
- while i < n {
499
- let b = bytes[i];
500
- if b != b'<' {
501
- // Skip ahead to the next `<`. We don't care about non-tag
502
- // bytes for span identification; they'll be copied verbatim
503
- // in phase 2.
504
- i = memchr(b'<', &bytes[i..]).map(|j| i + j).unwrap_or(n);
505
- continue;
506
- }
507
-
508
- match parse_tag_at(bytes, i) {
509
- TagParse::NotTag => {
510
- i += 1;
511
- }
512
- TagParse::Open {
513
- name_end,
514
- tag_end,
515
- is_self_closing,
516
- } => {
517
- stats.tags_seen += 1;
518
- let name = &text[i + 1..name_end];
519
- if is_known_html_tag(name) {
520
- stats.html_tags_skipped += 1;
521
- i = tag_end;
522
- continue;
523
- }
524
- if is_self_closing {
525
- spans.push(Span {
526
- start: i,
527
- end: tag_end,
528
- kind: SpanKind::SelfClosing,
529
- });
530
- stats.self_closing_protected += 1;
531
- i = tag_end;
532
- continue;
533
- }
534
- if compress_tagged_content {
535
- // Marker-only mode: emit the open as its own span
536
- // *and* push the name on the stack so the close
537
- // gets matched and emitted as its own span.
538
- spans.push(Span {
539
- start: i,
540
- end: tag_end,
541
- kind: SpanKind::OpenMarker,
542
- });
543
- }
544
- // Both modes push to the stack so close-matching works.
545
- stack.push(OpenTag {
546
- name_lower: name.to_ascii_lowercase(),
547
- open_start: i,
548
- });
549
- i = tag_end;
550
- }
551
- TagParse::Close { name_end, tag_end } => {
552
- stats.tags_seen += 1;
553
- let close_name = &text[i + 2..name_end];
554
- if is_known_html_tag(close_name) {
555
- stats.html_tags_skipped += 1;
556
- i = tag_end;
557
- continue;
558
- }
559
- let close_name_lower = close_name.to_ascii_lowercase();
560
- let matching = stack
561
- .iter()
562
- .rposition(|open| open.name_lower == close_name_lower);
563
-
564
- match matching {
565
- Some(stack_idx) => {
566
- if compress_tagged_content {
567
- // Pop everything above (orphan opens
568
- // inside the matched span — their open
569
- // markers were already recorded as spans
570
- // and we keep them).
571
- stack.truncate(stack_idx);
572
- let _ = stack.pop();
573
- spans.push(Span {
574
- start: i,
575
- end: tag_end,
576
- kind: SpanKind::CloseMarker,
577
- });
578
- } else {
579
- // Block mode: collapse [open..close] into
580
- // a single span. Drop any inner unmatched
581
- // opens (they're part of this span's body).
582
- // Also DROP any inner spans we already
583
- // recorded that are now subsumed by this
584
- // outer block — that's how nested custom
585
- // tags collapse to one placeholder.
586
- let open_start = stack[stack_idx].open_start;
587
- stack.truncate(stack_idx);
588
- spans.retain(|s| s.start < open_start);
589
- spans.push(Span {
590
- start: open_start,
591
- end: tag_end,
592
- kind: SpanKind::Block,
593
- });
594
- stats.custom_blocks_protected += 1;
595
- }
596
- i = tag_end;
597
- }
598
- None => {
599
- stats.orphan_closes += 1;
600
- i = tag_end;
601
- }
602
- }
603
- }
604
- }
605
- }
606
-
607
- // Stack remnants are orphan opens (no matching close ever arrived).
608
- // We don't protect those — they'll fall through to the compressor
609
- // as raw `<name>` bytes, same as Python's original behavior. In
610
- // block mode their inner self-closing spans we recorded are still
611
- // safe to keep: they were below an unmatched outer open, so they
612
- // were never collapsed. Spans are sorted by start ascending due to
613
- // the monotonic walk; phase 2 expects that.
614
- spans
615
- }
616
-
617
- fn emit_output(
618
- text: &str,
619
- spans: &[Span],
620
- prefix: &str,
621
- ) -> Option<(String, Vec<(String, String)>)> {
622
- let mut out = String::with_capacity(text.len());
623
- let mut blocks: Vec<(String, String)> = Vec::new();
624
- let mut cursor: usize = 0;
625
-
626
- for (counter, span) in (0_u64..).zip(spans.iter()) {
627
- if span.start < cursor {
628
- // Overlap shouldn't happen given how we collapse nested
629
- // spans, but bail loudly if it does — silently producing
630
- // wrong output is worse than failing the test.
631
- return None;
632
- }
633
- out.push_str(&text[cursor..span.start]);
634
- let placeholder = format!("{prefix}{counter}{PLACEHOLDER_SUFFIX}");
635
- let original = &text[span.start..span.end];
636
- blocks.push((placeholder.clone(), original.to_string()));
637
- out.push_str(&placeholder);
638
- cursor = span.end;
639
- let _ = span.kind; // SpanKind is informational only at this layer
640
- }
641
- out.push_str(&text[cursor..]);
642
- Some((out, blocks))
643
- }
644
-
645
- /// Restore protected tag spans after the compressor ran on the
646
- /// cleaned text.
647
- ///
648
- /// # Hotfix-A9 — discard-wrap semantics
649
- ///
650
- /// If a placeholder went missing during compression (the compressor
651
- /// stripped or rewrote it) the wrap is **discarded**: the compressed
652
- /// text flows downstream as-is and the original tag bytes are NOT
653
- /// re-injected anywhere. This is a deliberate behavior change vs the
654
- /// original "append the orphan tag at the trailing edge" fallback,
655
- /// which produced silently malformed XML (an opening tag with no
656
- /// closing tag and no body) on ~350 production requests over 9 days.
657
- ///
658
- /// An ERROR-level `tracing` event with structured fields
659
- /// (`event = tag_protector_placeholder_lost`, `tag_preview`,
660
- /// `compressed_length`, optional `request_id`) is emitted per lost
661
- /// placeholder so operators can alert on the corruption rather than
662
- /// having it disappear into a WARN line. Token validation downstream
663
- /// is responsible for catching cases where the discard regressed the
664
- /// final output vs the original input.
665
- ///
666
- /// Invariants enforced:
667
- /// 1. Symmetry — never emit asymmetric tag counts (the input either
668
- /// survives both opens and closes via successful substitution, or
669
- /// neither survives via discard).
670
- /// 2. No orphan tag injection — `restore_tags` adds bytes only as part
671
- /// of placeholder substitution. No appends, no prepends, no
672
- /// whitespace insertion outside placeholder substitutions.
673
- /// 3. Idempotence on missing placeholders — if every placeholder is
674
- /// absent from `compressed`, the function returns `compressed`
675
- /// byte-for-byte unchanged.
676
- pub fn restore_tags(text: &str, blocks: &[(String, String)]) -> String {
677
- restore_tags_with_request_id(text, blocks, None)
678
- }
679
-
680
- /// Variant of [`restore_tags`] that threads an optional `request_id`
681
- /// into the structured ERROR log emitted on placeholder loss. The
682
- /// PyO3 binding currently calls [`restore_tags`] (no request id);
683
- /// this entry point exists so the proxy layer can wire request
684
- /// context through once it has one available end-to-end.
685
- pub fn restore_tags_with_request_id(
686
- text: &str,
687
- blocks: &[(String, String)],
688
- request_id: Option<&str>,
689
- ) -> String {
690
- if blocks.is_empty() {
691
- return text.to_string();
692
- }
693
-
694
- let mut result = text.to_string();
695
- let mut lost_count: usize = 0;
696
- let compressed_length = text.len();
697
- for (placeholder, original) in blocks {
698
- if result.contains(placeholder.as_str()) {
699
- result = result.replace(placeholder.as_str(), original);
700
- } else {
701
- lost_count += 1;
702
- tag_lost_error(original, compressed_length, request_id);
703
- }
704
- }
705
- // No tail_appends. The compressed text is returned with the
706
- // wraps for the lost placeholders fully discarded — never
707
- // appended back as orphan opens (Hotfix-A9).
708
- let _ = lost_count; // surfaced via the per-event log; aggregate
709
- // counters live on the stats sidecar in
710
- // callers that have one.
711
- result
712
- }
713
-
714
- #[inline(never)]
715
- fn tag_lost_error(original: &str, compressed_length: usize, request_id: Option<&str>) {
716
- let preview: String = original.chars().take(80).collect();
717
- match request_id {
718
- Some(rid) => tracing::error!(
719
- target: "headroom::tag_protector",
720
- event = "tag_protector_placeholder_lost",
721
- tag_preview = %preview,
722
- compressed_length = compressed_length,
723
- request_id = %rid,
724
- action = "discarded_wrap",
725
- "tag placeholder lost during compression — wrap discarded"
726
- ),
727
- None => tracing::error!(
728
- target: "headroom::tag_protector",
729
- event = "tag_protector_placeholder_lost",
730
- tag_preview = %preview,
731
- compressed_length = compressed_length,
732
- action = "discarded_wrap",
733
- "tag placeholder lost during compression — wrap discarded"
734
- ),
735
- }
736
- }
737
-
738
- // ─── Tiny byte-search helper ──────────────────────────────────────────
739
-
740
- #[inline]
741
- fn memchr(needle: u8, haystack: &[u8]) -> Option<usize> {
742
- haystack.iter().position(|&b| b == needle)
743
- }
744
-
745
- #[cfg(test)]
746
- mod tests {
747
- use super::*;
748
-
749
- fn protect(text: &str) -> (String, Vec<(String, String)>) {
750
- let (cleaned, blocks, _stats) = protect_tags(text, false);
751
- (cleaned, blocks)
752
- }
753
-
754
- #[test]
755
- fn passthrough_when_no_angle_bracket() {
756
- let (cleaned, blocks) = protect("Just plain text");
757
- assert_eq!(cleaned, "Just plain text");
758
- assert!(blocks.is_empty());
759
- }
760
-
761
- #[test]
762
- fn html_tags_emitted_verbatim() {
763
- let text = "<div>Some content</div>";
764
- let (cleaned, blocks) = protect(text);
765
- assert_eq!(cleaned, text);
766
- assert!(blocks.is_empty());
767
- }
768
-
769
- #[test]
770
- fn html_tag_check_case_insensitive() {
771
- assert!(is_known_html_tag("DIV"));
772
- assert!(is_known_html_tag("Span"));
773
- assert!(!is_known_html_tag("system-reminder"));
774
- assert!(!is_known_html_tag("EXTREMELY_IMPORTANT"));
775
- }
776
-
777
- #[test]
778
- fn custom_tag_replaced_with_placeholder() {
779
- let text = "Before <system-reminder>Important</system-reminder> After";
780
- let (cleaned, blocks) = protect(text);
781
- assert!(!cleaned.contains("<system-reminder>"));
782
- assert!(!cleaned.contains("Important"));
783
- assert!(cleaned.contains("Before"));
784
- assert!(cleaned.contains("After"));
785
- assert_eq!(blocks.len(), 1);
786
- assert_eq!(blocks[0].1, "<system-reminder>Important</system-reminder>");
787
- }
788
-
789
- #[test]
790
- fn custom_tag_with_attributes() {
791
- let text = r#"<context key="session" type="persistent">user data</context>"#;
792
- let (_cleaned, blocks) = protect(text);
793
- assert_eq!(blocks.len(), 1);
794
- assert!(blocks[0].1.contains(r#"key="session""#));
795
- }
796
-
797
- #[test]
798
- fn self_closing_custom_tag() {
799
- let text = "Text <marker/> more text";
800
- let (_cleaned, blocks) = protect(text);
801
- assert_eq!(blocks.len(), 1);
802
- assert_eq!(blocks[0].1, "<marker/>");
803
- }
804
-
805
- #[test]
806
- fn self_closing_html_tag_not_protected() {
807
- let text = "Text <br/> more <hr/> text";
808
- let (cleaned, blocks) = protect(text);
809
- assert_eq!(cleaned, text);
810
- assert!(blocks.is_empty());
811
- }
812
-
813
- #[test]
814
- fn nested_custom_tags_collapse_to_outer_span() {
815
- let text = "<outer><inner>deep</inner></outer>";
816
- let (cleaned, blocks) = protect(text);
817
- assert!(!cleaned.contains("<outer>"));
818
- assert!(!cleaned.contains("<inner>"));
819
- // Outer span captures inner — single placeholder.
820
- assert_eq!(blocks.len(), 1);
821
- assert_eq!(blocks[0].1, "<outer><inner>deep</inner></outer>");
822
- }
823
-
824
- #[test]
825
- fn mixed_html_and_custom() {
826
- let text = "<div>HTML</div> <system-reminder>Rule</system-reminder> <p>HTML2</p>";
827
- let (cleaned, blocks) = protect(text);
828
- assert!(cleaned.contains("<div>"));
829
- assert!(cleaned.contains("<p>"));
830
- assert!(!cleaned.contains("<system-reminder>"));
831
- assert_eq!(blocks.len(), 1);
832
- }
833
-
834
- #[test]
835
- fn real_workflow_tags() {
836
- let cases = [
837
- "<tool_call>search({query: 'test'})</tool_call>",
838
- "<thinking>Let me analyze this</thinking>",
839
- "<EXTREMELY_IMPORTANT>Never skip validation</EXTREMELY_IMPORTANT>",
840
- "<user-prompt-submit-hook>check perms</user-prompt-submit-hook>",
841
- "<system-reminder>Rules</system-reminder>",
842
- "<result>Success: 42 items</result>",
843
- ];
844
- for tag in cases {
845
- let text = format!("Before {tag} After");
846
- let (_cleaned, blocks) = protect(&text);
847
- assert_eq!(blocks.len(), 1, "failed to protect: {tag}");
848
- assert_eq!(blocks[0].1, tag);
849
- }
850
- }
851
-
852
- #[test]
853
- fn empty_input_returns_empty() {
854
- let (cleaned, blocks) = protect("");
855
- assert!(cleaned.is_empty());
856
- assert!(blocks.is_empty());
857
- }
858
-
859
- #[test]
860
- fn compress_tagged_content_true_emits_marker_placeholders() {
861
- let text = "Before <system-reminder>Compressible content</system-reminder> After";
862
- let (cleaned, blocks, _stats) = protect_tags(text, true);
863
- assert!(!cleaned.contains("<system-reminder>"));
864
- assert!(!cleaned.contains("</system-reminder>"));
865
- assert!(cleaned.contains("Compressible content"));
866
- assert_eq!(blocks.len(), 2);
867
- }
868
-
869
- #[test]
870
- fn restore_basic() {
871
- let original = "Before <system-reminder>Rule</system-reminder> After";
872
- let (cleaned, blocks, _stats) = protect_tags(original, false);
873
- let restored = restore_tags(&cleaned, &blocks);
874
- assert_eq!(restored, original);
875
- }
876
-
877
- #[test]
878
- fn restore_empty_blocks_passthrough() {
879
- assert_eq!(restore_tags("untouched", &[]), "untouched");
880
- }
881
-
882
- #[test]
883
- fn restore_lost_placeholder_discards_wrap() {
884
- // Hotfix-A9: when a placeholder is missing from the compressed
885
- // text, the wrap is DISCARDED — the compressed text is returned
886
- // as-is, with no orphan-tag append. (The original behavior of
887
- // appending the tag at the trailing edge produced silently
888
- // malformed XML in ~350 production requests over 9 days.)
889
- let blocks = vec![(
890
- "{{HEADROOM_TAG_0}}".to_string(),
891
- "<tag>data</tag>".to_string(),
892
- )];
893
- let compressed = "text without placeholder";
894
- let restored = restore_tags(compressed, &blocks);
895
- // Compressed text returned unchanged; original tag NOT injected.
896
- assert_eq!(restored, compressed);
897
- assert!(!restored.contains("<tag>"));
898
- assert!(!restored.contains("</tag>"));
899
- assert!(!restored.contains("<tag>data</tag>"));
900
- }
901
-
902
- #[test]
903
- fn restore_lost_placeholder_idempotent_when_all_missing() {
904
- // Invariant #3: if every placeholder is missing from the
905
- // compressed text, the function returns the compressed text
906
- // byte-for-byte unchanged.
907
- let blocks = vec![
908
- ("{{HEADROOM_TAG_0}}".to_string(), "<a>1</a>".to_string()),
909
- ("{{HEADROOM_TAG_1}}".to_string(), "<b>2</b>".to_string()),
910
- ("{{HEADROOM_TAG_2}}".to_string(), "<c>3</c>".to_string()),
911
- ];
912
- let compressed = "compressor stripped every placeholder";
913
- let restored = restore_tags(compressed, &blocks);
914
- assert_eq!(restored, compressed);
915
- }
916
-
917
- #[test]
918
- fn restore_partial_loss_keeps_present_drops_lost() {
919
- // Mixed case: some placeholders survive, others are lost. The
920
- // surviving ones get substituted; the lost ones are discarded.
921
- // No orphan-tag bytes appear anywhere in the output.
922
- let blocks = vec![
923
- ("{{HEADROOM_TAG_0}}".to_string(), "<a>1</a>".to_string()),
924
- (
925
- "{{HEADROOM_TAG_1}}".to_string(),
926
- "<lost>x</lost>".to_string(),
927
- ),
928
- ];
929
- let compressed = "head {{HEADROOM_TAG_0}} tail";
930
- let restored = restore_tags(compressed, &blocks);
931
- assert_eq!(restored, "head <a>1</a> tail");
932
- assert!(!restored.contains("<lost"));
933
- assert!(!restored.contains("</lost>"));
934
- }
935
-
936
- #[test]
937
- fn restore_roundtrip_preserves_content() {
938
- let original = "Start <system-reminder>Rule 1: validate</system-reminder> middle \
939
- <tool_call>search(q='test')</tool_call> end";
940
- let (cleaned, blocks, _stats) = protect_tags(original, false);
941
- let restored = restore_tags(&cleaned, &blocks);
942
- assert_eq!(restored, original);
943
- }
944
-
945
- // ─── Bug-fix tests (fixed_in_3e4) ─────────────────────────────────
946
-
947
- #[test]
948
- fn fixed_in_3e4_replace_first_does_not_collide_on_duplicate_blocks() {
949
- // Bug #2: Python's `result.replace(original, placeholder, 1)`
950
- // replaces the FIRST textual occurrence of `original`, not
951
- // necessarily the matched offset. Two identical custom-tag
952
- // blocks would collapse to a single placeholder + a stray
953
- // duplicate of the second block in the output.
954
- let text = "<system-reminder>same</system-reminder> middle \
955
- <system-reminder>same</system-reminder>";
956
- let (cleaned, blocks, _stats) = protect_tags(text, false);
957
- // BOTH blocks should be replaced by DIFFERENT placeholders.
958
- assert_eq!(blocks.len(), 2);
959
- assert!(!cleaned.contains("<system-reminder>"));
960
- assert!(!cleaned.contains("</system-reminder>"));
961
- assert_ne!(blocks[0].0, blocks[1].0);
962
- // Roundtrip is exact.
963
- assert_eq!(restore_tags(&cleaned, &blocks), text);
964
- }
965
-
966
- #[test]
967
- fn fixed_in_3e4_handles_50_plus_nested_custom_tags() {
968
- // Bug #3: Python had a hard-coded 50-iteration safety cap that
969
- // silently truncated tag protection on deeply nested input.
970
- // Build 60 nested custom tags and verify all get caught in
971
- // the outermost span.
972
- let depth = 60;
973
- let mut text = String::new();
974
- for _ in 0..depth {
975
- text.push_str("<lvl>");
976
- }
977
- text.push_str("core");
978
- for _ in 0..depth {
979
- text.push_str("</lvl>");
980
- }
981
- let (cleaned, blocks, _stats) = protect_tags(&text, false);
982
- // The outermost span eats everything: one placeholder, no
983
- // residual `<lvl>` markers in the cleaned text.
984
- assert!(!cleaned.contains("<lvl>"));
985
- assert!(!cleaned.contains("</lvl>"));
986
- assert_eq!(blocks.len(), 1);
987
- // Roundtrip exact even at depth=60.
988
- assert_eq!(restore_tags(&cleaned, &blocks), text);
989
- }
990
-
991
- #[test]
992
- fn fixed_in_3e4_self_closing_duplicates_get_distinct_placeholders() {
993
- // Bug #4: same first-occurrence-replace bug for self-closing
994
- // tags. `<marker/>` appearing twice would collapse.
995
- let text = "<marker/> middle <marker/>";
996
- let (cleaned, blocks, _stats) = protect_tags(text, false);
997
- assert_eq!(blocks.len(), 2);
998
- assert_ne!(blocks[0].0, blocks[1].0);
999
- assert!(!cleaned.contains("<marker/>"));
1000
- assert_eq!(restore_tags(&cleaned, &blocks), text);
1001
- }
1002
-
1003
- #[test]
1004
- fn fixed_in_3e4_placeholder_collision_is_avoided() {
1005
- // Bug #5: input contains literal `{{HEADROOM_TAG_…}}`. The
1006
- // walker should pick a salted prefix and report the collision
1007
- // in stats.
1008
- let text = "User wrote {{HEADROOM_TAG_0}} on purpose. \
1009
- <system-reminder>real one</system-reminder>";
1010
- let (_cleaned, blocks, stats) = protect_tags(text, false);
1011
- assert!(stats.placeholder_collision_avoided);
1012
- assert_eq!(blocks.len(), 1);
1013
- // Placeholder used must NOT collide with the user's literal.
1014
- assert_ne!(blocks[0].0, "{{HEADROOM_TAG_0}}");
1015
- }
1016
-
1017
- // ─── Edge-case correctness ────────────────────────────────────────
1018
-
1019
- #[test]
1020
- fn orphan_close_tag_emitted_verbatim() {
1021
- let text = "no opener </ghost> here";
1022
- let (cleaned, blocks, stats) = protect_tags(text, false);
1023
- // Nothing protected; close stays in the cleaned text.
1024
- assert_eq!(blocks.len(), 0);
1025
- assert!(cleaned.contains("</ghost>"));
1026
- assert_eq!(stats.orphan_closes, 1);
1027
- }
1028
-
1029
- #[test]
1030
- fn orphan_open_tag_emitted_verbatim() {
1031
- // An open with no matching close should round-trip exactly —
1032
- // no protection, no data loss.
1033
- let text = "<ghost>dangling content with no close";
1034
- let (cleaned, blocks, _stats) = protect_tags(text, false);
1035
- assert!(blocks.is_empty());
1036
- assert_eq!(cleaned, text);
1037
- }
1038
-
1039
- #[test]
1040
- fn malformed_lone_lt_emitted_verbatim() {
1041
- let text = "if a < b then c";
1042
- let (cleaned, blocks, _stats) = protect_tags(text, false);
1043
- assert_eq!(cleaned, text);
1044
- assert!(blocks.is_empty());
1045
- }
1046
-
1047
- #[test]
1048
- fn truncated_close_marker_does_not_panic() {
1049
- // Hotfix-A9: proptest seed `</` would index past end-of-input
1050
- // in `parse_tag_at`. Pre-fix this panicked with an OOB; the
1051
- // bounds-check now returns NotTag and the function falls
1052
- // through to emitting `</` verbatim.
1053
- for text in ["</", "<", "<a/", "<a", "<a /", "</a"] {
1054
- let (cleaned, blocks, _stats) = protect_tags(text, false);
1055
- assert_eq!(cleaned, text);
1056
- assert!(blocks.is_empty());
1057
- }
1058
- }
1059
-
1060
- #[test]
1061
- fn attribute_with_gt_inside_quotes() {
1062
- let text = r#"<context attr="a > b">payload</context>"#;
1063
- let (cleaned, blocks, _stats) = protect_tags(text, false);
1064
- assert_eq!(blocks.len(), 1);
1065
- assert_eq!(blocks[0].1, text);
1066
- assert!(!cleaned.contains("payload"));
1067
- }
1068
-
1069
- #[test]
1070
- fn html_close_inside_custom_block_does_not_pop_stack() {
1071
- // An HTML close tag while a custom open is on top should not
1072
- // confuse the stack: the HTML close is emitted verbatim, the
1073
- // custom span still closes when its own close arrives.
1074
- let text = "<custom>x</div> y</custom>";
1075
- let (cleaned, blocks, stats) = protect_tags(text, false);
1076
- // The whole `<custom>...</custom>` span wins, including the
1077
- // verbatim `</div>` inside.
1078
- assert_eq!(blocks.len(), 1);
1079
- assert_eq!(blocks[0].1, "<custom>x</div> y</custom>");
1080
- assert!(!cleaned.contains("<custom>"));
1081
- // `</div>` is HTML, not orphan.
1082
- assert_eq!(stats.html_tags_skipped, 1);
1083
- assert_eq!(stats.orphan_closes, 0);
1084
- }
1085
-
1086
- // ─── Hotfix-A9 invariants ────────────────────────────────────────
1087
-
1088
- /// Count `<custom>` style opening tags (excludes self-closers and
1089
- /// excludes the closing-tag `</…>` form). Any `<` that is followed
1090
- /// by an alphabetic name and ends with `>` (without an embedded
1091
- /// `/>`) counts. Only used by the proptest below — keeps the
1092
- /// invariant check independent of the parser under test.
1093
- fn count_open_tags(s: &str) -> usize {
1094
- let bytes = s.as_bytes();
1095
- let mut count = 0_usize;
1096
- let mut i = 0_usize;
1097
- while i < bytes.len() {
1098
- if bytes[i] != b'<' {
1099
- i += 1;
1100
- continue;
1101
- }
1102
- // Skip closing tags `</…>`.
1103
- if i + 1 < bytes.len() && bytes[i + 1] == b'/' {
1104
- i += 1;
1105
- continue;
1106
- }
1107
- // Must be followed by a name-start char to count as a tag.
1108
- if i + 1 >= bytes.len() || !is_name_start(bytes[i + 1]) {
1109
- i += 1;
1110
- continue;
1111
- }
1112
- // Walk to the matching `>`. If we hit `/>` first, this is
1113
- // self-closing and doesn't count as an unbalanced opener.
1114
- let mut j = i + 1;
1115
- let mut self_closing = false;
1116
- while j < bytes.len() && bytes[j] != b'>' {
1117
- if bytes[j] == b'/' {
1118
- self_closing = true;
1119
- }
1120
- j += 1;
1121
- }
1122
- if j >= bytes.len() {
1123
- // No closing `>` — not a tag.
1124
- break;
1125
- }
1126
- if !self_closing {
1127
- count += 1;
1128
- }
1129
- i = j + 1;
1130
- }
1131
- count
1132
- }
1133
-
1134
- fn count_close_tags(s: &str) -> usize {
1135
- let bytes = s.as_bytes();
1136
- let mut count = 0_usize;
1137
- let mut i = 0_usize;
1138
- while i < bytes.len() {
1139
- if bytes[i] != b'<' {
1140
- i += 1;
1141
- continue;
1142
- }
1143
- if i + 1 >= bytes.len() || bytes[i + 1] != b'/' {
1144
- i += 1;
1145
- continue;
1146
- }
1147
- // `</name>` — confirm name-start and find the closing `>`.
1148
- if i + 2 >= bytes.len() || !is_name_start(bytes[i + 2]) {
1149
- i += 1;
1150
- continue;
1151
- }
1152
- let mut j = i + 2;
1153
- while j < bytes.len() && bytes[j] != b'>' {
1154
- j += 1;
1155
- }
1156
- if j >= bytes.len() {
1157
- break;
1158
- }
1159
- count += 1;
1160
- i = j + 1;
1161
- }
1162
- count
1163
- }
1164
-
1165
- proptest::proptest! {
1166
- /// Invariant: `restore_tags` never INTRODUCES tag-count
1167
- /// asymmetry. Concretely: restoring on a compressed text with
1168
- /// any subset of placeholders missing must produce the same
1169
- /// `opens - closes` skew as the cleaned text after stripping
1170
- /// the placeholders. The orphan-append bug fixed by Hotfix-A9
1171
- /// could turn a symmetric `<a>x</a>` into an asymmetric
1172
- /// `compressed-stuff <a>` whenever the placeholder was
1173
- /// dropped — the discard-wrap path makes that impossible
1174
- /// because every protected span is a balanced wrap (or a
1175
- /// self-closer) so dropping it changes opens and closes by
1176
- /// the same amount.
1177
- #[test]
1178
- fn restore_never_introduces_asymmetry(content in "[a-z<>/]{0,200}") {
1179
- let (cleaned, blocks, _stats) = protect_tags(&content, false);
1180
- // Baseline: strip every placeholder from `cleaned`. This
1181
- // is the "lost everything" worst case; the discard-wrap
1182
- // path must produce exactly this output.
1183
- let mut stripped = cleaned.clone();
1184
- for (placeholder, _original) in &blocks {
1185
- stripped = stripped.replace(placeholder.as_str(), "");
1186
- }
1187
- let baseline_skew = count_open_tags(&stripped) as i64
1188
- - count_close_tags(&stripped) as i64;
1189
-
1190
- // With every placeholder lost, restore_tags must return
1191
- // the compressed text with placeholders dropped — which
1192
- // is exactly `stripped`. So asymmetry equals baseline.
1193
- let restored_all_lost = restore_tags(&stripped, &blocks);
1194
- let lost_skew = count_open_tags(&restored_all_lost) as i64
1195
- - count_close_tags(&restored_all_lost) as i64;
1196
- proptest::prop_assert_eq!(
1197
- lost_skew, baseline_skew,
1198
- "discard-wrap path introduced asymmetry: baseline={}, after_restore={}, restored={:?}",
1199
- baseline_skew, lost_skew, restored_all_lost
1200
- );
1201
-
1202
- // With every placeholder PRESENT, restore_tags must round-
1203
- // trip exactly to the original `content`, which by
1204
- // construction has the same skew as `content` itself.
1205
- let restored_full = restore_tags(&cleaned, &blocks);
1206
- let full_skew = count_open_tags(&restored_full) as i64
1207
- - count_close_tags(&restored_full) as i64;
1208
- let content_skew = count_open_tags(&content) as i64
1209
- - count_close_tags(&content) as i64;
1210
- proptest::prop_assert_eq!(
1211
- full_skew, content_skew,
1212
- "full-restore path drifted from input skew: input={}, restored={}",
1213
- content_skew, full_skew
1214
- );
1215
- }
1216
-
1217
- /// Invariant: when every placeholder is stripped before
1218
- /// restore, the function returns the compressed text
1219
- /// byte-for-byte unchanged (no orphan-tag injection, no
1220
- /// whitespace insertion, no prepends/appends).
1221
- #[test]
1222
- fn restore_idempotent_when_all_placeholders_lost(
1223
- content in "[a-z<>/]{0,200}",
1224
- compressed in "[ -~]{0,200}",
1225
- ) {
1226
- let (_cleaned, blocks, _stats) = protect_tags(&content, false);
1227
- // Drop all placeholders by feeding `restore_tags` arbitrary
1228
- // text the compressor "produced". If none of the
1229
- // placeholders happen to appear in `compressed` (the
1230
- // common case for arbitrary strings), the discard-wrap
1231
- // path runs end-to-end.
1232
- let any_placeholder_present = blocks
1233
- .iter()
1234
- .any(|(p, _)| compressed.contains(p.as_str()));
1235
- proptest::prop_assume!(!any_placeholder_present);
1236
- let restored = restore_tags(&compressed, &blocks);
1237
- proptest::prop_assert_eq!(restored, compressed);
1238
- }
1239
-
1240
- /// Invariant: `restore_tags` never adds bytes that weren't
1241
- /// already in `compressed` or part of a substituted placeholder
1242
- /// original. Concretely: the restored length is at most
1243
- /// `compressed.len()` plus the sum of lengths of originals
1244
- /// that actually got substituted; lost-placeholder originals
1245
- /// contribute zero bytes.
1246
- #[test]
1247
- fn restore_no_orphan_byte_injection(
1248
- content in "[a-z<>/]{0,200}",
1249
- ) {
1250
- let (cleaned, blocks, _stats) = protect_tags(&content, false);
1251
- let restored = restore_tags(&cleaned, &blocks);
1252
- // Sum of the byte-lengths of the originals that were
1253
- // actually substituted (placeholder still present in
1254
- // `cleaned`). Lost placeholders contribute zero.
1255
- let substituted_original_bytes: usize = blocks
1256
- .iter()
1257
- .filter(|(p, _)| cleaned.contains(p.as_str()))
1258
- .map(|(p, original)| original.len().saturating_sub(p.len()))
1259
- .sum();
1260
- // Upper bound: cleaned.len() + delta from substitution.
1261
- // (Substitution replaces each placeholder of len p.len()
1262
- // with original of len original.len(); delta per substitution
1263
- // is original.len() - p.len(), summed across substitutions.)
1264
- let upper_bound = cleaned.len() + substituted_original_bytes;
1265
- proptest::prop_assert!(
1266
- restored.len() <= upper_bound,
1267
- "restored too long: restored.len={} upper_bound={} cleaned.len={}",
1268
- restored.len(), upper_bound, cleaned.len()
1269
- );
1270
- }
1271
- }
1272
- }