thumbgate 1.34.3 β 1.37.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/cyberstrike-compare-not-clone/SKILL.md +36 -0
- package/.agents/skills/gitlab-sandbox-allowlist-not-trust/SKILL.md +77 -0
- package/.agents/skills/jit-harness-compare-not-clone/SKILL.md +34 -0
- package/.agents/skills/openui-catalog-compose-honesty/SKILL.md +64 -0
- package/.agents/skills/zvec-grep-compare-not-clone/SKILL.md +34 -0
- package/.claude-plugin/plugin.json +1 -1
- package/.well-known/llms.txt +1 -0
- package/.well-known/mcp/server-card.json +1 -1
- package/CONTRIBUTING.md +95 -0
- package/README.md +195 -632
- package/THIRD_PARTY_NOTICES.md +89 -0
- package/adapters/claude/.mcp.json +2 -2
- package/adapters/forge/forge.yaml +3 -3
- package/adapters/future-agi/.mcp.json +8 -0
- package/adapters/future-agi/FUTURE_AGI.md +23 -0
- package/adapters/future-agi/config.toml +3 -0
- package/adapters/future-agi/future-agi-bridge.js +9 -0
- package/adapters/future-agi/opencode.json +8 -0
- package/adapters/herdr/herdr-plugin.toml +18 -0
- package/adapters/mcp/server-stdio.js +238 -25
- package/adapters/opencode/opencode.json +1 -1
- package/adapters/workos/WORKOS.md +52 -0
- package/bin/cli.js +373 -5
- package/bin/futureagi-bridge +9 -0
- package/config/gate-templates.json +653 -4
- package/config/gates/actor-critic-audit.json +34 -0
- package/config/gates/default.json +21 -2
- package/config/gates/five-walls-governance.json +34 -0
- package/config/gates/future-agi-guardrails.json +34 -0
- package/config/gates/radware-threat-defense-2026.json +61 -0
- package/config/gates/simatree-data-governance.json +33 -0
- package/config/mcp-allowlists.json +4 -0
- package/config/merge-quality-checks.json +10 -1
- package/config/model-candidates.json +382 -24
- package/config/model-tiers.json +18 -0
- package/config/post-deploy-marketing-pages.json +10 -0
- package/config/progressive/01-wire-only.json +11 -0
- package/config/progressive/02-dashboard-empty-ok.json +10 -0
- package/config/progressive/03-one-lesson.json +10 -0
- package/config/progressive/04-warn-fires.json +11 -0
- package/config/progressive/05-strict-optional.json +11 -0
- package/config/progressive/README.md +15 -0
- package/config/schemas/broker-execution-receipt.schema.json +139 -0
- package/config/schemas/provider-execution-attestation-v1.schema.json +58 -0
- package/conformance/provider-attestation/vectors.json +320 -0
- package/docs/specs/provider-execution-attestation-v1.md +69 -0
- package/openapi/openapi.yaml +15 -0
- package/package.json +401 -147
- package/public/about.html +2 -2
- package/public/ai-malpractice-prevention.html +7 -7
- package/public/blog/a-10-dollar-vps-is-not-a-computer.html +143 -0
- package/public/blog/a-receipt-is-not-world-state.html +388 -0
- package/public/blog/git-at-agent-scale.html +374 -0
- package/public/blog/no-llm-in-the-gate.html +133 -0
- package/public/blog.html +80 -0
- package/public/case-studies.html +16 -1
- package/public/compare.html +28 -0
- package/public/diagnostic.html +216 -7
- package/public/docs/connectors.html +39 -0
- package/public/federal.html +2 -2
- package/public/founders.html +639 -0
- package/public/index.html +87 -9
- package/public/install.html +8 -8
- package/public/learn.html +39 -0
- package/public/numbers.html +2 -2
- package/public/peter.html +310 -0
- package/public/platform-partners.html +119 -0
- package/public/pricing.html +24 -3
- package/public/privacy.html +117 -0
- package/public/pro.html +17 -0
- package/public/support.html +62 -0
- package/public/terms.html +130 -0
- package/public/third-party-notices.html +95 -0
- package/public/yt.html +351 -0
- package/scripts/action-receipts.js +133 -3
- package/scripts/adaptive-governance-arena.js +349 -0
- package/scripts/admin-override.js +205 -0
- package/scripts/agent-action-inventory.js +869 -0
- package/scripts/agent-audit-trace.js +42 -2
- package/scripts/agent-egress-policy.js +1117 -0
- package/scripts/agent-memory-lifecycle.js +141 -2
- package/scripts/agent-operations-planner.js +441 -1
- package/scripts/agent-readiness.js +68 -0
- package/scripts/agent-security-central.js +647 -0
- package/scripts/allowlist-bridge-honesty.js +417 -0
- package/scripts/async-job-runner.js +102 -11
- package/scripts/audit-trail.js +212 -0
- package/scripts/auto-promote-gates.js +178 -27
- package/scripts/billing.js +1 -1
- package/scripts/broker-execution-receipts.js +719 -0
- package/scripts/budget-aware-gates-proof.js +423 -0
- package/scripts/claude-feedback-sync.js +29 -3
- package/scripts/claw-harness-production.js +237 -0
- package/scripts/cli-progress.js +111 -0
- package/scripts/cli-schema.js +163 -1
- package/scripts/codex-runbook-flywheel.js +318 -0
- package/scripts/context-footprint.js +186 -0
- package/scripts/contextfs.js +143 -61
- package/scripts/dashboard-limits.js +27 -0
- package/scripts/dashboard.js +279 -9
- package/scripts/deepseek-v4-runtime-guardrails.js +72 -6
- package/scripts/docker-sandbox-planner.js +18 -0
- package/scripts/double-blind-eval-protocol.js +252 -0
- package/scripts/edotenv-rl-gateway.js +259 -0
- package/scripts/ensure-production-search-corpus.js +162 -0
- package/scripts/eval-holdout.js +311 -0
- package/scripts/feedback-aggregate.js +21 -2
- package/scripts/feedback-loop.js +87 -5
- package/scripts/feedback-quality.js +9 -0
- package/scripts/file-ledger-lock.js +4 -1
- package/scripts/financial-control-plane.js +41 -1
- package/scripts/find-dormant-requires.js +118 -0
- package/scripts/fs-utils.js +84 -8
- package/scripts/gate-stats.js +2 -2
- package/scripts/gates-engine.js +859 -58
- package/scripts/generate-case-study-outreach.js +24 -15
- package/scripts/git-at-scale.js +628 -0
- package/scripts/governance-conflict-audit.js +1650 -0
- package/scripts/governance-difficulty-curriculum.js +328 -0
- package/scripts/graphrag-retrieval.js +275 -0
- package/scripts/gurobi-optimizer.js +324 -0
- package/scripts/gurobi_optimizer.py +485 -0
- package/scripts/harness-selector.js +82 -1
- package/scripts/hidden-entry-points.js +284 -0
- package/scripts/human-escalation.js +199 -1
- package/scripts/hybrid-feedback-context.js +152 -19
- package/scripts/intent-governed-execution.js +602 -0
- package/scripts/intervention-policy.js +123 -20
- package/scripts/jit-harness-compose.js +628 -0
- package/scripts/jsonl-watcher.js +10 -0
- package/scripts/lesson-embedding-index.js +95 -12
- package/scripts/lesson-retrieval.js +105 -19
- package/scripts/local-model-profile.js +19 -2
- package/scripts/mailer/resend-mailer.js +1 -1
- package/scripts/matryoshka-embedding.js +235 -0
- package/scripts/mcp-oauth.js +42 -4
- package/scripts/mcp-session-handles.js +1016 -0
- package/scripts/mcp-wiring-doctor.js +314 -0
- package/scripts/memory-firewall.js +115 -2
- package/scripts/memory-scope-readiness.js +299 -0
- package/scripts/memory-vs-rag-route.js +161 -0
- package/scripts/model-tier-router.js +148 -21
- package/scripts/nvidia-specdecode-al-doctor.js +536 -0
- package/scripts/openui-catalog-compose-honesty.js +593 -0
- package/scripts/operational-integrity.js +19 -1
- package/scripts/override-audit.js +213 -0
- package/scripts/package-manager-honesty-doctor.js +458 -0
- package/scripts/pr-manager.js +63 -1
- package/scripts/prove-herdr-adapter.js +52 -0
- package/scripts/prove-memory-pyramid-and-symbolic-canvas.js +95 -0
- package/scripts/prove-workos.js +73 -0
- package/scripts/provider-attestation-conformance.js +192 -0
- package/scripts/provider-receipt-contract.js +136 -0
- package/scripts/qwen38-max-cost-optimizer.js +401 -0
- package/scripts/radware-threat-defense.js +280 -0
- package/scripts/rag-embedding-identity.js +221 -0
- package/scripts/rag-precision-guardrails.js +112 -2
- package/scripts/remote-feedback-capture.js +159 -0
- package/scripts/research-agent-harness.js +256 -0
- package/scripts/rsi-safety-hillclimb.js +200 -0
- package/scripts/rule-sprawl.js +188 -0
- package/scripts/schedule-manager.js +147 -0
- package/scripts/self-heal.js +8 -0
- package/scripts/session-lease.js +415 -0
- package/scripts/simatree-data-governance.js +347 -0
- package/scripts/slo-alert-engine.js +172 -7
- package/scripts/solver-parity.js +539 -0
- package/scripts/stealth-memory-injection-gate.js +333 -0
- package/scripts/switchyard-router.js +366 -0
- package/scripts/telemetry-analytics.js +84 -27
- package/scripts/temporal-decay-weighting.js +138 -0
- package/scripts/test-all.js +165 -0
- package/scripts/token-savings.js +42 -0
- package/scripts/tool-kpi-tracker.js +108 -5
- package/scripts/tool-registry.js +193 -5
- package/scripts/universal-claim-evaluator.js +14 -2
- package/scripts/vector-store.js +279 -9
- package/scripts/workflow-notebook.js +391 -0
- package/scripts/workflow-sentinel.js +111 -12
- package/scripts/workos-production-guard.js +260 -0
- package/scripts/workspace-search-route.js +515 -0
- package/server.json +2 -2
- package/src/agent-identity-boundary.js +76 -0
- package/src/agent-retrieval-cache.js +155 -0
- package/src/alert-noise-ledger.js +502 -0
- package/src/api/server.js +802 -185
- package/src/git-fast-cache.js +220 -0
- package/src/git-wal-sync.js +156 -0
- package/src/hash-anchored-edit.js +82 -0
- package/src/hermes-platform-protocol.js +475 -0
- package/src/hermes-sync-plane.js +241 -0
- package/src/index.js +30 -1
- package/src/iso42001-compliance-guard.js +97 -0
- package/src/latency-budget.js +244 -0
- package/src/mcp-writeguard.js +316 -0
- package/src/miminions-adapter.js +106 -0
- package/src/pipeline-compass.js +104 -0
- package/src/ppl-alert-pipeline.js +284 -0
- package/src/rendezvous-router.js +90 -0
- package/src/security-questionnaire.js +195 -0
|
@@ -37,6 +37,43 @@
|
|
|
37
37
|
"roi": "Raises trust in autonomous runs and reduces manual re-checking.",
|
|
38
38
|
"rollout": "Use for every workflow where proof matters more than speed."
|
|
39
39
|
},
|
|
40
|
+
{
|
|
41
|
+
"id": "require-catalog-compose-only",
|
|
42
|
+
"name": "Require catalog-compose-only agent UI",
|
|
43
|
+
"category": "Agent Honesty",
|
|
44
|
+
"signal": "π",
|
|
45
|
+
"defaultAction": "block",
|
|
46
|
+
"severity": "high",
|
|
47
|
+
"pattern": "(generative\\s*ui|compose\\s*ui|openui|@openuidev).*(invent|arbitrary|eval|new Function|dangerouslySetInnerHTML|install\\s+@openuidev|clone\\s+openui)|component\\s+not\\s+in\\s+catalog|unknown\\s+component",
|
|
48
|
+
"problem": "Blocks agents from inventing UI components outside an allowlisted catalog or running generated code. OpenUI's safe-by-default FORMAT: compose only from your components.",
|
|
49
|
+
"roi": "Stops generative-UI theater and arbitrary-code compose paths. Keeps agent dashboards/widgets inside a known catalog without cloning OpenUI/Thesys.",
|
|
50
|
+
"rollout": "Enable for any agent-composed UI, dashboard widget, or tool-result card. Pair with openui-catalog-compose-honesty --catalog --stream --json."
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
"id": "require-repair-before-compose-claim",
|
|
54
|
+
"name": "Require repair-before-claim on composed UI",
|
|
55
|
+
"category": "Agent Honesty",
|
|
56
|
+
"signal": "π",
|
|
57
|
+
"defaultAction": "block",
|
|
58
|
+
"severity": "high",
|
|
59
|
+
"pattern": "(ui|compose|generative\\s*ui|dashboard\\s*widget).*(ready|done|shipped|fixed|live)|claim.*compose.*(ready|done)",
|
|
60
|
+
"problem": "Blocks claiming composed UI is ready before a line-level validate+repair pass (root present, unknown components dropped, zero arbitrary-code lines).",
|
|
61
|
+
"roi": "Mirrors OpenUI Gateway's repair-before-users-see-it FORMAT on ThumbGate rails: invalid compose never becomes a completion claim.",
|
|
62
|
+
"rollout": "Require openui-catalog-compose-honesty --repair --claim-ready evidence before any compose/UI done claim. Does not install OpenUI Gateway."
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"id": "require-broker-signed-execution-receipt",
|
|
66
|
+
"name": "Require broker-signed execution receipts",
|
|
67
|
+
"category": "Provider Side-Effect Governance",
|
|
68
|
+
"signal": "π",
|
|
69
|
+
"defaultAction": "block",
|
|
70
|
+
"severity": "critical",
|
|
71
|
+
"pattern": "broker[_ -]?signed|provider[_ -]?credential|providerEventId",
|
|
72
|
+
"problem": "Stops agents from treating self-written receipts as proof of provider side effects. Only a credential-holding broker outside the agent trust boundary may sign.",
|
|
73
|
+
"roi": "Closes the 'agent rehashed a receipt' integrity hole on payments, messaging, and credentialed APIs.",
|
|
74
|
+
"rollout": "Set THUMBGATE_BROKER_RECEIPT_MODE=enforce and install broker public keys on the host. Pair with aigate or any external credential broker."
|
|
75
|
+
},
|
|
76
|
+
|
|
40
77
|
{
|
|
41
78
|
"id": "protect-production-sql",
|
|
42
79
|
"name": "Protect production SQL",
|
|
@@ -169,6 +206,42 @@
|
|
|
169
206
|
"roi": "Prevents AI agents from introducing vulnerable packages into the workspace. Complements zero-egress local-first scanning toolings (like OWASP CVE Lite CLI) directly at the terminal level.",
|
|
170
207
|
"rollout": "Enable on every repo where agents can install dependencies or modify lockfiles."
|
|
171
208
|
},
|
|
209
|
+
{
|
|
210
|
+
"id": "require-package-manager-lockfile-ci-parity",
|
|
211
|
+
"name": "Require package-manager lockfile and CI parity",
|
|
212
|
+
"category": "Supply Chain Safety",
|
|
213
|
+
"signal": "π",
|
|
214
|
+
"defaultAction": "block",
|
|
215
|
+
"severity": "high",
|
|
216
|
+
"pattern": "(pnpm-lock|yarn\\.lock|bun\\.lock|package-lock).*(and|with|plus).*(package-lock|pnpm-lock|yarn\\.lock)|packageManager.*(mismatch|disagree)|ci.*(pnpm|yarn|bun).*(npm ci|package-lock)",
|
|
217
|
+
"problem": "Blocks dual lockfiles and CI install commands that disagree with the canonical lockfile manager.",
|
|
218
|
+
"roi": "Stops agents from introducing pnpm/yarn/bun beside npm (or vice versa) and silently breaking reproducible installs after package-manager rewrites like pnpm 12.",
|
|
219
|
+
"rollout": "Enable on every repo with agent install rights. Pair with package-manager-honesty-doctor."
|
|
220
|
+
},
|
|
221
|
+
{
|
|
222
|
+
"id": "checkpoint-package-manager-switch",
|
|
223
|
+
"name": "Checkpoint package-manager switch",
|
|
224
|
+
"category": "Supply Chain Safety",
|
|
225
|
+
"signal": "π",
|
|
226
|
+
"defaultAction": "block",
|
|
227
|
+
"severity": "high",
|
|
228
|
+
"pattern": "(migrate|switch|adopt|move).*(pnpm|yarn|bun|npm)|pnpm self-update|corepack enable.*(pnpm|yarn)|add pnpm-lock",
|
|
229
|
+
"problem": "Requires an explicit migration plan before switching package managers: single lockfile, packageManager pin, rewrite every CI install, delete the old lockfile, and prove CI green.",
|
|
230
|
+
"roi": "pnpm 12-style rewrites look cheap until CI still runs npm ci against a deleted lockfile. Fail closed on silent switches; prefer --ignore-scripts honesty on the current manager.",
|
|
231
|
+
"rollout": "Block agent-proposed manager switches; allow only with package-manager-honesty-doctor --propose-switch evidence attached to the PR."
|
|
232
|
+
},
|
|
233
|
+
{
|
|
234
|
+
"id": "checkpoint-allowlist-bridge-not-trust",
|
|
235
|
+
"name": "Checkpoint allowlisted registry/proxy as a hop, not a trust boundary",
|
|
236
|
+
"category": "Supply Chain Safety",
|
|
237
|
+
"signal": "π",
|
|
238
|
+
"defaultAction": "warn",
|
|
239
|
+
"severity": "high",
|
|
240
|
+
"pattern": "(?=.*(sandbox|allowlist))(?=.*(trusted|safe|secure))(?=.*(npm|pypi|huggingface|package proxy|registry\\.npmjs|verdaccio|nexus)).+",
|
|
241
|
+
"problem": "Blocks treating an allowlisted package registry, proxy, or Hugging Face host as a sandbox trust boundary. GitLab's 2026-09 analysis showed a permitted package proxy became the escape hop.",
|
|
242
|
+
"roi": "Keeps deny-network-egress and agent-egress-policy honest: hops stay hops. Pair with allowlist-bridge-honesty. Does not clone GitLab Duo.",
|
|
243
|
+
"rollout": "Warn when agents claim an allowlisted registry is trusted; require allowlist-bridge-honesty --json evidence before expanding egress allowlists."
|
|
244
|
+
},
|
|
172
245
|
{
|
|
173
246
|
"id": "require-section-tree-before-multimodal-answer",
|
|
174
247
|
"name": "Require section tree before multimodal answers",
|
|
@@ -356,10 +429,10 @@
|
|
|
356
429
|
"signal": "π",
|
|
357
430
|
"defaultAction": "warn",
|
|
358
431
|
"severity": "high",
|
|
359
|
-
"pattern": "(speculative|mtp|eagle).*(accept|rollback|draft).*(low|missing|unstable)",
|
|
360
|
-
"problem": "Requires
|
|
361
|
-
"roi": "Avoids routing traffic to a faster-looking path that collapses acceptance length or hides draft-token correctness failures.",
|
|
362
|
-
"rollout": "Start in warn mode for lab benchmarks; promote to block when speculation is enabled for customer traffic."
|
|
432
|
+
"pattern": "(speculative|mtp|eagle|draft.?length|accept.?length).*(accept|rollback|draft|speedup|al\\b).*(low|missing|unstable|overclaim|above.?theory)",
|
|
433
|
+
"problem": "Requires measured accept length (AL), draft length (D), rollback/coherence evidence, and speedup claims at or below AL/(1+ΟD) before speculative decoding is treated as a production speedup.",
|
|
434
|
+
"roi": "Avoids routing traffic to a faster-looking path that collapses acceptance length, overclaims beyond AL/(1+ΟD), or hides draft-token correctness failures.",
|
|
435
|
+
"rollout": "Start in warn mode for lab benchmarks; promote to block when speculation is enabled for customer traffic. Pair with nvidia-specdecode-al-doctor for AL/D math."
|
|
363
436
|
},
|
|
364
437
|
{
|
|
365
438
|
"id": "require-long-context-kv-offload-capacity-plan",
|
|
@@ -577,6 +650,42 @@
|
|
|
577
650
|
"roi": "Critical for compliance, forensics, and feedback loops. Enables proper capture of agent-specific lessons and prevention rules. Matches industry push (Okta, etc.).",
|
|
578
651
|
"rollout": "Block any claw or autonomous agent action that authenticates as a human user. Require dedicated agent service accounts / identities with scoped permissions."
|
|
579
652
|
},
|
|
653
|
+
{
|
|
654
|
+
"id": "block-workos-spend-cap-violations",
|
|
655
|
+
"name": "Block WorkOS spend-cap violations under $10/mo AuthKit policy",
|
|
656
|
+
"category": "Enterprise Identity & MCP Auth Governance",
|
|
657
|
+
"signal": "π",
|
|
658
|
+
"defaultAction": "block",
|
|
659
|
+
"severity": "critical",
|
|
660
|
+
"pattern": "(workos|authkit).*(custom domain|enterprise sso|saml connection|directory sync|scim|radar|audit logs siem|\\$99|\\$125|staging\\.authkit)",
|
|
661
|
+
"problem": "ThumbGate public auth is WorkOS Production AuthKit under a hard β€$10/mo spend cap. Enabling custom AuthKit domains (~$99/mo), enterprise SSO/SAML connections (~$125/conn), Directory Sync, or driving public traffic to staging AuthKit blows the cap or regresses login.",
|
|
662
|
+
"roi": "Protects conversion (thumbgate.app login) and prevents silent monthly burn. Complements WorkOS MCP Auth: identity stays cheap; runtime gates stay ThumbGate.",
|
|
663
|
+
"rollout": "Run npm run prove:workos. Keep Active SSO connections = 0. Prefer own Google OAuth app under AuthKit free tier. Never buy custom AuthKit domain without written budget rewrite."
|
|
664
|
+
},
|
|
665
|
+
{
|
|
666
|
+
"id": "require-mcp-oauth-scope-for-mutating-tools",
|
|
667
|
+
"name": "Require MCP OAuth scope for mutating remote tools",
|
|
668
|
+
"category": "Enterprise Identity & MCP Auth Governance",
|
|
669
|
+
"signal": "π",
|
|
670
|
+
"defaultAction": "block",
|
|
671
|
+
"severity": "high",
|
|
672
|
+
"pattern": "(mcp|oauth|bearer|access_token).*(missing scope|insufficient_scope|no auth|unauthenticated).*(write|mutate|capture|gate|tool)",
|
|
673
|
+
"problem": "Enterprise MCP Auth (WorkOS AuthKit MCP, RFC 8707 audience binding) maps OAuth scopes to tool privileges. Mutating tools without mcp:write (or equivalent) or with audience mismatch leak or mutate shared state.",
|
|
674
|
+
"roi": "Closes directory-reviewer and multi-tenant MCP holes. Aligns ThumbGate remote MCP with industry MCP OAuth 2.1 patterns while keeping pre-action gates as the enforcement layer after identity.",
|
|
675
|
+
"rollout": "Hosted MCP: OAuth PKCE required for remote connectors. Reviewer keys stay read-only. Operators use mcp:write (implies read/gates/feedback)."
|
|
676
|
+
},
|
|
677
|
+
{
|
|
678
|
+
"id": "review-workos-skill-and-auth-installer-actions",
|
|
679
|
+
"name": "Review WorkOS skill install and auth-route mutation by agents",
|
|
680
|
+
"category": "Enterprise Identity & MCP Auth Governance",
|
|
681
|
+
"signal": "π",
|
|
682
|
+
"defaultAction": "require-approval",
|
|
683
|
+
"severity": "high",
|
|
684
|
+
"pattern": "(workos|authkit|npx workos|workos install|auth callback|sso connection|redirect_uri).*(install|scaffold|write|env|secret|client_id|api_key)",
|
|
685
|
+
"problem": "WorkOS AI CLI installers and agent skills mutate auth routes, env secrets, and SSO configuration. Unreviewed agent edits can reintroduce staging clients, wrong redirect URIs, or paid enterprise SSO connections.",
|
|
686
|
+
"roi": "Prevents auth regressions that drop Google login or point production at staging β a proven conversion killer. Gates dynamic skill installs (claw-style) on identity infrastructure.",
|
|
687
|
+
"rollout": "Require human review for WorkOS/AuthKit scaffolding. After changes run scripts/workos-production-guard.js and prove:workos."
|
|
688
|
+
},
|
|
580
689
|
{
|
|
581
690
|
"id": "gate-claw-file-system-access",
|
|
582
691
|
"name": "Gate claw-style agent file system access",
|
|
@@ -589,6 +698,162 @@
|
|
|
589
698
|
"roi": "Directly supports the hybrid/on-prem reality emphasized in EnterpriseClaw coverage. Prevents broad access from becoming broad exfil or corruption. Ties to ThumbGate's existing path globs and protected files.",
|
|
590
699
|
"rollout": "Use existing protected-paths + new claw-specific rules. Start with read-only for most, explicit approval for writes on sensitive dirs."
|
|
591
700
|
},
|
|
701
|
+
{
|
|
702
|
+
"id": "block-stealth-memory-injection-from-external",
|
|
703
|
+
"name": "Block stealth memory injection from external content",
|
|
704
|
+
"category": "Claw-Style Enterprise Agent Governance",
|
|
705
|
+
"signal": "π",
|
|
706
|
+
"defaultAction": "block",
|
|
707
|
+
"severity": "critical",
|
|
708
|
+
"pattern": "(silently|do not tell|do not announce|stealth memory|memghost|imap|smtp|inbox).*(MEMORY\\.md|AGENTS\\.md|SOUL\\.md|USER\\.md|HEARTBEAT\\.md|long-term memory|preference|remember that)",
|
|
709
|
+
"problem": "Persistent personal agents (OpenClaw, Hermes, NanoClaw) can absorb untrusted email/tool content into durable memory carriers while staying silent in the user-visible reply. MemGhost (WhisperBench, paper 2607.05189) shows 87.5% end-to-end success for one-shot stealth memory injection β delayed sessions then treat poisoned facts/preferences as trusted state.",
|
|
710
|
+
"roi": "High: closes the durable-memory compromise path that ordinary PreToolUse hijack defenses miss. Direct product fit for OpenClaw/Hermes governance buyers and ThumbGate's Infrastructure Firewall narrative. Blocks fact + preference poisoning before it becomes trusted context.",
|
|
711
|
+
"rollout": "Enabled as structural gate in gates-engine (scripts/stealth-memory-injection-gate.js). Allow only with explicit operator override (THUMBGATE_ALLOW_STEALTH_MEMORY_INJECTION=1) or stealthMemoryInjectionApproved=true after human review. Capture blocked attempts with tags stealth-memory-injection,whisperbench,memghost."
|
|
712
|
+
},
|
|
713
|
+
{
|
|
714
|
+
"id": "require-claw-observability-before-production",
|
|
715
|
+
"name": "Require OpenTelemetry observability before claw production rollout",
|
|
716
|
+
"category": "Claw-Style Enterprise Agent Governance",
|
|
717
|
+
"signal": "π",
|
|
718
|
+
"defaultAction": "warn",
|
|
719
|
+
"severity": "high",
|
|
720
|
+
"pattern": "(claw|harness agent|hosted agent|production rollout|deploy agent).*(no telemetry|no traces|no otel|observability|token usage|tool calls)",
|
|
721
|
+
"problem": "A claw that reads files, runs code, and calls tools is a small distributed system. Microsoft Agent Framework's production-readiness guide (2026) makes OpenTelemetry spans for every model turn and tool call, token-usage metrics, and structured logs a launch prerequisite β without them a misfiring skill, looping tool, or 10x cost spike is invisible until a customer reports it.",
|
|
722
|
+
"roi": "High: observability is the cheapest incident-response accelerator for enterprise claw buyers. ThumbGate's tool KPI telemetry and decision logs become the audit surface; this gate converts 'works on my machine' into 'we can see it run'. Maps directly to the AGENTS.md 20-observed-tool-calls production-monitoring floor.",
|
|
723
|
+
"rollout": "Warn by default; escalate to block for regulated verticals (finance/healthcare). Require an OTLP/collector endpoint or ThumbGate decision-log sink configured before any hosted or fleet deployment. Capture evidence with tags claw-observability,otel,production-readiness."
|
|
724
|
+
},
|
|
725
|
+
{
|
|
726
|
+
"id": "require-policy-content-screening",
|
|
727
|
+
"name": "Require prompt/response content-policy screening (Purview-style)",
|
|
728
|
+
"category": "Claw-Style Enterprise Agent Governance",
|
|
729
|
+
"signal": "π",
|
|
730
|
+
"defaultAction": "block",
|
|
731
|
+
"severity": "critical",
|
|
732
|
+
"pattern": "(prompt|response|chat client|model call|agent output).*(credit card|pii|confidential|regulated|policy screen|purview|dls|data loss)",
|
|
733
|
+
"problem": "In regulated settings every prompt must be screened before it reaches the model and every response before it reaches the user, with an audit trail β Microsoft's production claw wraps the chat client in Purview policy middleware for exactly this. Unscreened claw traffic against sensitive documents (finance, healthcare) is a data-loss incident waiting for an audit.",
|
|
734
|
+
"roi": "High: content-policy screening is the compliance story that unlocks enterprise budgets (SOC 2, HIPAA, DLP). ThumbGate's memory-firewall + secret-leak gates already implement the pattern locally; this gate extends it to bidirectional prompt/response screening and positions ThumbGate as the vendor-neutral alternative to tenant-locked E5-only Purview.",
|
|
735
|
+
"rollout": "Block unscreened model calls for regulated workloads; allow only when a screening layer (ThumbGate memory firewall, Purview middleware, or equivalent DLP) is configured on the chat path. Opt-in by environment flag so unregulated dev loops stay fast. Capture with tags content-policy,dlp,purview-style."
|
|
736
|
+
},
|
|
737
|
+
{
|
|
738
|
+
"id": "block-hosted-claw-shell-and-fs",
|
|
739
|
+
"name": "Block shell and container-disk file access on hosted claw agents",
|
|
740
|
+
"category": "Claw-Style Enterprise Agent Governance",
|
|
741
|
+
"signal": "π",
|
|
742
|
+
"defaultAction": "block",
|
|
743
|
+
"severity": "critical",
|
|
744
|
+
"pattern": "(hosted|container|shared tenant|multi tenant|foundry|cloud deploy).*(enable shell|shell access|file access|write container|container disk|arbitrary read)",
|
|
745
|
+
"problem": "Microsoft's hosted claw deliberately sets EnableFileAccess=false and EnableShell=false: in a shared hosted environment, arbitrary container filesystem or shell access is a data-exfiltration, tampering, and persistence vector even behind a deny-list. When file access is genuinely needed the pattern is an external governed file store (e.g., blob-backed AgentFileStore), never the ephemeral container disk.",
|
|
746
|
+
"roi": "High: the single clearest hosted-claw hardening rule from a first-party production guide. ThumbGate enforces it as a deterministic pre-action gate plus proof artifact, so buyers get the Microsoft-recommended lockdown with an evidence packet instead of trusting config drift.",
|
|
747
|
+
"rollout": "Block by default for any hosted/shared-tenant agent build. Explicit, reviewed exceptions only, and file access must route through a governed external store, never container disk. Capture with tags hosted-hardening,shell-off,filestore."
|
|
748
|
+
},
|
|
749
|
+
{
|
|
750
|
+
"id": "require-local-evals-on-every-change",
|
|
751
|
+
"name": "Require local deterministic evals on every claw change",
|
|
752
|
+
"category": "Claw-Style Enterprise Agent Governance",
|
|
753
|
+
"signal": "π",
|
|
754
|
+
"defaultAction": "warn",
|
|
755
|
+
"severity": "high",
|
|
756
|
+
"pattern": "(claw|agent|prompt|tool|skill).*(regression|eval|golden|quality gate|ci check|passed/total)",
|
|
757
|
+
"problem": "The production harness pattern runs local deterministic eval checks (plain functions: passed/total over a fixed query set) on every change, always in CI, with model-graded quality scores optional and gated. Without a local eval ratchet, every prompt/tool/skill edit is a blind change β the tuning loop that keeps a deployed agent honest does not exist.",
|
|
758
|
+
"roi": "High: evals-in-CI is the cheapest regression insurance and directly extends ThumbGate's existing golden-eval discipline (gate-decisions golden, retrieval ranking MRR/recall thresholds, fail-closed RAG gates). Selling point: ThumbGate ships the local-eval layer; hosted model-graded evals stay optional.",
|
|
759
|
+
"rollout": "Warn when a prompt/tool/skill change ships without a local eval run; block merges on eval regression in CI. Keep checks deterministic, fast, free, and runnable offline; require >= 95% pass on golden cases before widening deployment. Capture with tags local-evals,ratchet,production-readiness."
|
|
760
|
+
},
|
|
761
|
+
{
|
|
762
|
+
"id": "require-double-blind-holdout-eval",
|
|
763
|
+
"name": "Require double-blind holdout for benchmark tuning",
|
|
764
|
+
"category": "Eval Integrity",
|
|
765
|
+
"signal": "π",
|
|
766
|
+
"defaultAction": "warn",
|
|
767
|
+
"severity": "high",
|
|
768
|
+
"pattern": "(benchmark|golden set|eval set|leaderboard|model score|sota).*(leak|contamin|overfit|tune against|memoriz|inflate)",
|
|
769
|
+
"problem": "Google DeepMind's first double-blind AI evaluation (2026) responds to research finding benchmark leakage in roughly half of 31 models tested; contamination inflates scores, especially for larger models. Tuning prompts, rules, or models against a fully visible benchmark set teaches to the test β the reported score stops measuring generalization. DeepMind's fix seals both sides (weights and questions) in a confidential enclave; the local equivalent is a hash-sealed holdout slice whose questions never enter the tuning loop.",
|
|
770
|
+
"roi": "High: eval score integrity is the trust product β ThumbGate sells evidence, and inflated benchmark scores are fabricated evidence. Cheap to enforce (deterministic hash partition + sealed manifest, no confidential hardware). Maps to the AGENTS.md production regression gate (>= 6 golden cases, fail-closed) and extends scripts/eval-holdout.js.",
|
|
771
|
+
"rollout": "Partition every mined benchmark deterministically (sha256 of case identity) into public + holdout (default 1/3). Store holdout as hashes and sealed verdicts only β never raw questions. Re-freeze deliberately with a human decision, like eval:baseline; block releases when the holdout seal is broken or holdout verdicts drift unexplained. Capture with tags eval-integrity,holdout,contamination."
|
|
772
|
+
},
|
|
773
|
+
{
|
|
774
|
+
"id": "require-ai-use-case-inventory",
|
|
775
|
+
"name": "Require AI use-case inventory (CMDB for AI)",
|
|
776
|
+
"category": "AI Governance Operating Plan",
|
|
777
|
+
"signal": "π",
|
|
778
|
+
"defaultAction": "warn",
|
|
779
|
+
"severity": "high",
|
|
780
|
+
"pattern": "(deploy|onboard|introduce|roll ?out|new).*(ai|model|agent|llm|automation).*(no inventory|no register|no cmdb|no owner|untracked)",
|
|
781
|
+
"problem": "The episode's core governance finding: incomplete AI-use-case inventories are the root problem. Without a register of every model, vendor/API, agent, data source, tool integration, owner, user group, and business purpose β plus a risk tier β you cannot govern what you cannot enumerate.",
|
|
782
|
+
"roi": "High: the inventory is the prerequisite for every other control (data classification, least privilege, incident routing). A CMDB-for-AI is the cheapest high-leverage deliverable in the first 7 days of any AI governance program.",
|
|
783
|
+
"rollout": "Maintain a machine-readable register (one row per model/agent/integration: vendor, API, data sources, tools, owner, user group, business purpose, risk tier). Require an entry before any new AI capability ships. Warn by default; block for customer-facing or regulated-data use cases. Capture with tags ai-inventory,cmdb,use-case-register."
|
|
784
|
+
},
|
|
785
|
+
{
|
|
786
|
+
"id": "require-data-classification-before-ai",
|
|
787
|
+
"name": "Require data classification before AI workflow",
|
|
788
|
+
"category": "AI Governance Operating Plan",
|
|
789
|
+
"signal": "π",
|
|
790
|
+
"defaultAction": "block",
|
|
791
|
+
"severity": "critical",
|
|
792
|
+
"pattern": "(ai|model|llm|agent|prompt|rag).*(pii|confidential|financial|credential|secret|customer data|internal).*(no classification|unclassified|unapproved model|unmanaged)",
|
|
793
|
+
"problem": "Understand your data first: for each AI workflow, label inputs and outputs (public, internal, confidential, PII, financial, customer, credentials/secrets) and explicitly prohibit sensitive data from flowing into unapproved models or unmanaged browser sessions.",
|
|
794
|
+
"roi": "High: data classification is the single control that prevents the most expensive AI incident class β sensitive-data exfiltration into a third-party model. Cheap to implement, blocks the worst blast radius.",
|
|
795
|
+
"rollout": "Before enabling any AI workflow, classify its inputs/outputs. Block sensitive-class data from unapproved models and unmanaged sessions. Maintain an approved-model policy keyed by data class. Capture with tags data-classification,pii,approved-model."
|
|
796
|
+
},
|
|
797
|
+
{
|
|
798
|
+
"id": "require-ai-threat-model-blast-radius",
|
|
799
|
+
"name": "Require AI threat model + blast-radius assessment",
|
|
800
|
+
"category": "AI Governance Operating Plan",
|
|
801
|
+
"signal": "π",
|
|
802
|
+
"defaultAction": "warn",
|
|
803
|
+
"severity": "high",
|
|
804
|
+
"pattern": "(agent|ai workflow|automation).*(production|money|customer|privileged|deploy).*(no threat model|no blast radius|unassessed)",
|
|
805
|
+
"problem": "For each high-value AI workflow, map inputs, model, prompt construction, RAG sources, tools/actions, output recipients, logs, and human approval points β then ask: if this agent is compromised or wrong, what systems, records, money, customer communications, or privileges can it affect? Require explicit approval before external communications, payments, production changes, deletion, permission changes, or code deployment.",
|
|
806
|
+
"roi": "High: blast-radius assessment turns vague 'AI risk' into a concrete list of gated actions. Every consequential action behind an approval checkpoint is a prevented incident.",
|
|
807
|
+
"rollout": "Produce a threat model per high-value workflow before launch. Wire the blast-radius actions (external comms, payments, prod changes, deletion, permission changes, deploy) to explicit human approval gates. Capture with tags threat-model,blast-radius,approval-gate."
|
|
808
|
+
},
|
|
809
|
+
{
|
|
810
|
+
"id": "require-nhi-least-privilege",
|
|
811
|
+
"name": "Require least privilege for non-human identities",
|
|
812
|
+
"category": "AI Governance Operating Plan",
|
|
813
|
+
"signal": "π",
|
|
814
|
+
"defaultAction": "block",
|
|
815
|
+
"severity": "critical",
|
|
816
|
+
"pattern": "(agent|service account|api key|oauth|ci token|mcp|tool server).*(broad|birthright|admin|overprivileged|all scopes|wildcard)",
|
|
817
|
+
"problem": "Overprivileged non-human identities are a material exposure: inventory every API key, OAuth app, service account, CI token, tool credential, and MCP/tool server; eliminate broad birthright permissions; give each agent only the minimum scopes, data access, and action rights it needs; rotate secrets and prefer short-lived credentials; separate read-only research agents from action-capable agents.",
|
|
818
|
+
"roi": "High: least privilege for machine identities is the cheapest containment for a compromised or prompt-injected agent β it bounds what the agent can do even when the reasoning layer is fully captured.",
|
|
819
|
+
"rollout": "Inventory all non-human identities. Scope each agent to minimum required permissions, rotate long-lived secrets, prefer short-lived credentials, and split read-only from action-capable agents. Block any new broad-scope credential. Capture with tags nhi,least-privilege,credential-hygiene."
|
|
820
|
+
},
|
|
821
|
+
{
|
|
822
|
+
"id": "require-ai-incident-taxonomy-logging",
|
|
823
|
+
"name": "Require AI incident taxonomy + centralized logging",
|
|
824
|
+
"category": "AI Governance Operating Plan",
|
|
825
|
+
"signal": "π",
|
|
826
|
+
"defaultAction": "warn",
|
|
827
|
+
"severity": "high",
|
|
828
|
+
"pattern": "(ai|agent|model|llm).*(incident|failure|hallucination|injection|leak|drift).*(no taxonomy|no logging|untracked|no owner)",
|
|
829
|
+
"problem": "Define reportable AI events (hallucinated high-impact output, unsafe tool call, data leakage, prompt injection, unauthorized retrieval, abnormal cost, output drift) and capture model/version, sanitized prompt metadata, retrieved documents, tool calls, action results, approval identity, latency, cost, and failure outcome. Tie each event type to an owner, severity, containment action, and post-incident review.",
|
|
830
|
+
"roi": "High: an incident taxonomy + centralized log is what turns a single AI failure into a prevented class. Without it, every incident is a novel fire drill; with it, each one becomes a reusable prevention rule.",
|
|
831
|
+
"rollout": "Stand up a fixed AI event taxonomy and centralized log (ThumbGate's decision journal is the local substrate). Assign owner + severity + containment + review per event type. Run the SOC/CSIRT through a prompt-injection tabletop. Capture with tags incident-taxonomy,ml-secops,ai-logging."
|
|
832
|
+
},
|
|
833
|
+
{
|
|
834
|
+
"id": "require-constrained-ai-pilot",
|
|
835
|
+
"name": "Require constrained low-risk AI pilot before autonomous agents",
|
|
836
|
+
"category": "AI Governance Operating Plan",
|
|
837
|
+
"signal": "π",
|
|
838
|
+
"defaultAction": "allow",
|
|
839
|
+
"severity": "medium",
|
|
840
|
+
"pattern": "(start|pilot|begin|first).*(ai|agent|automation).*(retrieval|extraction|classification|summarization)",
|
|
841
|
+
"problem": "Start with retrieval, extraction, classification, or summarization β not an autonomous reasoning agent that can modify production systems. Define one success metric (e.g. 'reduce ticket-summary time by 60% while maintaining >= 95% factual accuracy in a reviewed sample'). The episode warns against jumping straight into complex reasoning without foundational maturity.",
|
|
842
|
+
"roi": "High: a constrained pilot is the cheapest way to prove value, build the eval baseline, and earn the trust needed to expand capability later β without betting the production environment on an unproven autonomous agent.",
|
|
843
|
+
"rollout": "Approve and route teams toward low-risk read/summarize/extract pilots first; require one measurable KPI and a reviewed accuracy sample before expanding to action-capable agents. Capture with tags ai-pilot,constrained,foundational-maturity."
|
|
844
|
+
},
|
|
845
|
+
{
|
|
846
|
+
"id": "require-workflow-notebook-plan-approval",
|
|
847
|
+
"name": "Require reviewed plan before agent execution (workflow notebook)",
|
|
848
|
+
"category": "AI Governance Operating Plan",
|
|
849
|
+
"signal": "π",
|
|
850
|
+
"defaultAction": "warn",
|
|
851
|
+
"severity": "medium",
|
|
852
|
+
"pattern": "(agent|codex|automation).*(repetitive|recurring|runbook|workflow).*(no plan|skip approval|execute without review)",
|
|
853
|
+
"problem": "Automating repetitive work at OpenAI with Codex (2026): the durable pattern is a goal cell, a written plan the human reviews, approval BEFORE execution starts, then captured commands/outputs/decisions/dead-ends and a discoverable index so the next run is cheaper. Skipping the plan-approval boundary lets an agent execute an unreviewed approach.",
|
|
854
|
+
"roi": "High: plan-then-approve is the cheapest structural control that keeps a human deciding when an approach is ready, while still letting the agent do the repetitive execution and record what happened.",
|
|
855
|
+
"rollout": "Use scripts/workflow-notebook.js for repetitive agent workflows: create β plan β approve β record steps/decisions β finish (writes *.index.md for discovery). Warn when an agent begins recording execution on an unapproved notebook. Capture with tags workflow-notebook,plan-approval,decision-capture."
|
|
856
|
+
},
|
|
592
857
|
{
|
|
593
858
|
"id": "block-unauthorized-multi-channel-posts",
|
|
594
859
|
"name": "Block unauthorized multi-channel posts by Hermes Agent",
|
|
@@ -660,6 +925,390 @@
|
|
|
660
925
|
"problem": "Blocks any file edit or write tool call that is not within a specified active/target directory.",
|
|
661
926
|
"roi": "Restricts agent edits to a controlled workspace, preventing scope drift, accidental modification of configuration or source files, and unwanted global refactors during focused task debugging.",
|
|
662
927
|
"rollout": "Enable dynamically on-demand using /freeze to specify the only path(s) the agent is allowed to write to during the session."
|
|
928
|
+
},
|
|
929
|
+
{
|
|
930
|
+
"id": "block-vlt-install-vulnerable-deps",
|
|
931
|
+
"name": "Block vlt install of vulnerable dependencies",
|
|
932
|
+
"category": "JavaScript Package Registry Governance",
|
|
933
|
+
"signal": "π",
|
|
934
|
+
"defaultAction": "block",
|
|
935
|
+
"severity": "critical",
|
|
936
|
+
"pattern": "(vlt\\s+install|vlt\\s+add).*?(--audit\\s+critical|vuln|cve|high-severity|known-vuln|left-pad|event-stream|colors|unsafe|insecure|deprecated|obsolete|legacy|vulnerable|unresolved-cve)",
|
|
937
|
+
"problem": "Blocks vlt install/add of packages flagged with --audit critical or known-vulnerable CVE references (e.g. left-pad, event-stream, colors). vlt's hosted registry provides real-time vulnerability scanning; agents must not bypass the audit pipeline.",
|
|
938
|
+
"roi": "Prevents supply-chain compromise via known-vulnerable dependencies. Direct 1st-order risk: left-pad-style incidents cause cascading production outages. Second-order: agents that install without audit never learn to check. Captured feedback (thumbgate capture --tag supply-chain-security) turns each bypass into a prevention rule.",
|
|
939
|
+
"rollout": "Require --audit critical on all vlt install/add. Block packages with known CVE references. Auto-capture feedback for each bypass attempt."
|
|
940
|
+
},
|
|
941
|
+
{
|
|
942
|
+
"id": "require-review-vlt-registry-override",
|
|
943
|
+
"name": "Require review for vlt registry override to non-approved hosts",
|
|
944
|
+
"category": "JavaScript Package Registry Governance",
|
|
945
|
+
"signal": "π",
|
|
946
|
+
"defaultAction": "block",
|
|
947
|
+
"severity": "critical",
|
|
948
|
+
"pattern": "(vlt\\s+config\\s+set|vlt\\s+registry\\s+add|vlt\\s+set\\s+registry).*(registry|registry-url|self-host|override|redirect|mirror|vsr)",
|
|
949
|
+
"problem": "Blocks vlt registry configuration that redirects dependency resolution to non-approved self-hosted VSR or mirror hosts. Agents must only pull from allowlisted registries (vlt.sh, registry.npmjs.org, approved internal VSR).",
|
|
950
|
+
"roi": "High: Prevents supply-chain attacks via rogue registry configuration. Critical for enterprise on-prem/air-gapped environments where vlt's self-hosted VSR is the approved source. Captured feedback (thumbgate capture --tag registry-override) generates prevention rules for unauthorized redirect patterns.",
|
|
951
|
+
"rollout": "Block all registry-setting commands for non-allowlisted hosts. Warn (human review) for internal VSR. Allowlist vlt.sh, registry.npmjs.org, and approved internal domains."
|
|
952
|
+
},
|
|
953
|
+
{
|
|
954
|
+
"id": "enforce-vlt-workspace-dep-pinning",
|
|
955
|
+
"name": "Enforce pinned dependency versions in vlt workspace installs",
|
|
956
|
+
"category": "JavaScript Package Registry Governance",
|
|
957
|
+
"signal": "π",
|
|
958
|
+
"defaultAction": "block",
|
|
959
|
+
"severity": "high",
|
|
960
|
+
"pattern": "(vlt\\s+add|vlt\\s+install).*?(@\\s*(?:\\^|~|>|<|>=|<=|\\*|latest|next))",
|
|
961
|
+
"problem": "Blocks vlt add/install with unpinned (wildcard, range, latest/next) dependency versions. vlt workspace installs with @*, @^, @~, @>, @<, @>=, @<=, @latest, @next resolve to potentially vulnerable latest versions instead of audited pinned versions.",
|
|
962
|
+
"roi": "Pinned dependencies are the bedrock of reproducible builds and vulnerability scanning. vlt's hosted registry provides lockfiles by default; agents must respect them. Feedback (thumbgate capture --tag dep-pinning) from each bypass generates a rule for the specific unpinned package pattern.",
|
|
963
|
+
"rollout": "Enforce exact version pinning in workspace vlt add/install. Block @* ^ ~ > < >= <= latest next. Allow exact versions like @4.18.2."
|
|
964
|
+
},
|
|
965
|
+
{
|
|
966
|
+
"id": "gate-vlt-package-publishing",
|
|
967
|
+
"name": "Gate vlt package publishing for provenance and scope safety",
|
|
968
|
+
"category": "JavaScript Package Registry Governance",
|
|
969
|
+
"signal": "π",
|
|
970
|
+
"defaultAction": "block",
|
|
971
|
+
"severity": "critical",
|
|
972
|
+
"pattern": "(vlt\\s+publish|vlt\\s+pack|npm\\s+publish).*(no-provenance|unsigned|no-2fa|no-attestation|typosquat|squat|unscoped|no-scope)",
|
|
973
|
+
"problem": "Blocks vlt/npm publish of packages without provenance attestation, 2FA, or proper scope/access controls. vlt's hosted registry supports provenance; agents must not publish unsigned or unscoped packages.",
|
|
974
|
+
"roi": "Prevents typosquatting and supply-chain attacks via unsigned package publishing. Aligns with vlt's security posture and npm provenance ecosystem. Captures high-value feedback for prevention rules.",
|
|
975
|
+
"rollout": "Require provenance attestation and 2FA for all vlt publish operations. Block publishes to scopes matching existing packages (typosquatting detection)."
|
|
976
|
+
},
|
|
977
|
+
{
|
|
978
|
+
"id": "block-vlt-private-registry-bypass",
|
|
979
|
+
"name": "Block vlt private registry bypass via npmrc or env override",
|
|
980
|
+
"category": "JavaScript Package Registry Governance",
|
|
981
|
+
"signal": "π",
|
|
982
|
+
"defaultAction": "block",
|
|
983
|
+
"severity": "critical",
|
|
984
|
+
"pattern": "(\\.npmrc|vlt.*config|NPM_CONFIG_REGISTRY|VLT_REGISTRY).*(https?://(?!(?:registry\\.npmjs\\.org|vlt\\.sh|localhost|127\\.0\\.0\\.1|\\[::1\\]|0\\.0\\.0\\.0)(?:[:/?#]|$))[\\w.-]+)",
|
|
985
|
+
"problem": "Blocks vlt/npm configuration that redirects dependency resolution to unapproved private or external registries via .npmrc, environment variables, or CLI config overrides.",
|
|
986
|
+
"roi": "High: Prevents data exfil and supply-chain compromise via rogue registry configuration. Critical for enterprise on-prem/air-gapped environments where vlt's self-hosted VSR is used.",
|
|
987
|
+
"rollout": "Block all registry overrides to non-allowlisted hosts. Allowlist vlt.sh, registry.npmjs.org, and approved internal VSR domains."
|
|
988
|
+
},
|
|
989
|
+
{
|
|
990
|
+
"id": "validate-context-before-codegen",
|
|
991
|
+
"name": "Validate context freshness and relevance before code generation",
|
|
992
|
+
"category": "AI Engineering Stack Safety",
|
|
993
|
+
"signal": "π",
|
|
994
|
+
"defaultAction": "block",
|
|
995
|
+
"severity": "high",
|
|
996
|
+
"pattern": "(Edit|Write|MultiEdit|StrReplace).*",
|
|
997
|
+
"problem": "Agents frequently generate code without first ensuring AGENTS.md, CLAUDE.md, test infrastructure, and ownership context are fresh and relevant. Per the Hugging Face Context Course: 'an agent is only as good as the context it has.' Results in style violations, wrong test commands, and rework.",
|
|
998
|
+
"roi": "Eliminates rework from stale or missing project context. Direct 1st-order improvement to context-engineering workflows (Hugging Face Context Course Units 1-3). Reduces PR churn and review cycles.",
|
|
999
|
+
"rollout": "Before any Edit/Write/MultiEdit, verify AGENTS.md exists and is fresh (modified within last 7 days). Require explicit context-freshness check before code edits. Capture feedback with 'missing-context' and 'stale-context' tags."
|
|
1000
|
+
},
|
|
1001
|
+
{
|
|
1002
|
+
"id": "require-matryoshka-embedding-dimension",
|
|
1003
|
+
"name": "Require Matryoshka-aware embedding dimension selection",
|
|
1004
|
+
"category": "Embedding Quality Gates",
|
|
1005
|
+
"signal": "π",
|
|
1006
|
+
"defaultAction": "warn",
|
|
1007
|
+
"severity": "medium",
|
|
1008
|
+
"pattern": "(embedding\\s*dimension|output.*dimension|embed.*dim).*(not.*tier|not.*matryoshka|unsupported)",
|
|
1009
|
+
"problem": "Embedding dimensions should align with Matryoshka tiers (256, 512, 768, 1024, 1536, 2048, 3072, 4096, 7680) for progressive disclosure retrieval. Non-tier dimensions waste compute or hurt recall.",
|
|
1010
|
+
"roi": "Reduces token spend and improves retrieval quality by using optimal dimension tiers. Enables cost-effective progressive disclosure: query at 256 dims, refine at 768, full precision at 1536+.",
|
|
1011
|
+
"rollout": "Warn when embedding dimension is not a Matryoshka tier. Recommend nearest tier. Track cost savings and recall metrics in feedback loop."
|
|
1012
|
+
},
|
|
1013
|
+
{
|
|
1014
|
+
"id": "block-embedding-recall-below-threshold",
|
|
1015
|
+
"name": "Block embedding pipeline changes with insufficient recall",
|
|
1016
|
+
"category": "Embedding Quality Gates",
|
|
1017
|
+
"signal": "π",
|
|
1018
|
+
"defaultAction": "block",
|
|
1019
|
+
"severity": "high",
|
|
1020
|
+
"pattern": "(embedding|retrieval).*(recall.*<.*0\\.95|precision.*<.*0\\.15)",
|
|
1021
|
+
"problem": "RAG pipelines require minimum recall@k β₯ 0.95 and precision@k β₯ 0.15. Changes that degrade below these thresholds hide failures from agents. Based on podcast insights: embedding model choice is the #1 underrated RAG decision.",
|
|
1022
|
+
"roi": "Prevents silent RAG pipeline degradation. Ensures every embedding change maintains measurable quality. Aligns with '4 out of 5 teams have metrics but 1 in 5 sees ROI' problem - we catch quality drops before they become failures.",
|
|
1023
|
+
"rollout": "Block any embedding deployment without verified baseline recall β₯ 0.95. Require recall/precision report before threshold changes. Capture feedback with tags 'embedding-quality', 'recall-regression', 'precision-degradation'."
|
|
1024
|
+
},
|
|
1025
|
+
{
|
|
1026
|
+
"id": "require-embedding-baseline-before-tuning",
|
|
1027
|
+
"name": "Require embedding baseline before precision tuning",
|
|
1028
|
+
"category": "RAG Pipeline Safety",
|
|
1029
|
+
"signal": "π",
|
|
1030
|
+
"defaultAction": "block",
|
|
1031
|
+
"severity": "high",
|
|
1032
|
+
"pattern": "(embedding|vector|retriever).*(tune|fine[- ]?tune|threshold|precision).*\\b(?!.*baseline)(?!.*recall[@\\s]*=?\\s*0\\.?9?5?)",
|
|
1033
|
+
"problem": "Per the podcast 'RAG Mistake Almost Every Team Is Making': tuning embedding thresholds without documented baseline causes silent degradation. 80% of teams have metrics but only 20% see ROI - we prevent the 80% from making tuning mistakes.",
|
|
1034
|
+
"roi": "Catches the #1 RAG pipeline mistake early. Ensures embedding improvements are measured against a preserved baseline recall. Directly addresses the 80/20 ROI gap discussed with Pete Johnson (MongoDB Field CTO).",
|
|
1035
|
+
"rollout": "Block any embedding threshold/optimization without 'recall@baseline: X.XX' in context. Encourage hybrid retrieval patterns for efficient progressive disclosure."
|
|
1036
|
+
},
|
|
1037
|
+
{
|
|
1038
|
+
"id": "enable-hybrid-retrieval-gate",
|
|
1039
|
+
"name": "Enable hybrid retrieval for efficient agentic memory",
|
|
1040
|
+
"category": "RAG Pipeline Optimization",
|
|
1041
|
+
"signal": "π",
|
|
1042
|
+
"defaultAction": "allow",
|
|
1043
|
+
"severity": "medium",
|
|
1044
|
+
"pattern": "(hybrid.*retrieval|progressive.*disclosure|matryoshka.*embed|fast.*query.*rerank)",
|
|
1045
|
+
"problem": "Agentic memory (ThumbGate's pre-action enforcement) benefits from hybrid retrieval: query at 256 dims for speed, refine at 768 dims, full precision at 1536+ only when needed. This is 'better agentic memory' per the podcast.",
|
|
1046
|
+
"roi": "Reduces token burn by 40-60% while maintaining recall. Enables real-time prevention without full embedding cost. Aligns with Matryoshka insights for efficient RAG.",
|
|
1047
|
+
"rollout": "Enable by default for memory retrieval. Log dimension-tier performance. Route based on query complexity and result certainty."
|
|
1048
|
+
},
|
|
1049
|
+
{
|
|
1050
|
+
"id": "require-multi-model-routing-for-complex-tasks",
|
|
1051
|
+
"name": "Require multi-model routing for complex agentic tasks",
|
|
1052
|
+
"category": "Hybrid Inference Governance",
|
|
1053
|
+
"signal": "π",
|
|
1054
|
+
"defaultAction": "block",
|
|
1055
|
+
"severity": "high",
|
|
1056
|
+
"pattern": "(complex_task|multi-step|agentic_run).*(single.*model|only.*one.*model).*processing",
|
|
1057
|
+
"problem": "Per NVIDIA Nemotron/NeMo Switchyard: AI agents need more than one model. Complex tasks should route steps to specialized models for accuracy, efficiency, customization, and control - not use a single expensive model for everything.",
|
|
1058
|
+
"roi": "Prevents 'expensive model for everything' anti-pattern. 30B MoE models like Nemotron 3.5 Lightning can evaluate routing algorithms across specialized tasks while keeping costs efficient.",
|
|
1059
|
+
"rollout": "Implement model-step routing matrix. Use lightweight model for intent detection, Qwen/Claude for reasoning, specialized model for coding, etc."
|
|
1060
|
+
},
|
|
1061
|
+
{
|
|
1062
|
+
"id": "checkpoint-model-step-routing-decision",
|
|
1063
|
+
"name": "Checkpoint model step routing decision",
|
|
1064
|
+
"category": "Hybrid Inference Governance",
|
|
1065
|
+
"signal": "π",
|
|
1066
|
+
"defaultAction": "warn",
|
|
1067
|
+
"severity": "high",
|
|
1068
|
+
"pattern": "(model_routing|routing_algorithm|step_route).*(missing|no).*(evidence|evaluation|cost_quality_tradeoff)",
|
|
1069
|
+
"problem": "Multi-model routing decisions must be evaluated against cost, quality, latency, and customization requirements before being used in production agents.",
|
|
1070
|
+
"roi": "Ensures routing algorithms like NeMo Switchyard are measured and produce better outcomes than single-model approaches.",
|
|
1071
|
+
"rollout": "Log routing decisions with metric outcomes. Evaluate routing quality via task success rate and cost savings."
|
|
1072
|
+
},
|
|
1073
|
+
{
|
|
1074
|
+
"id": "recommend-specialized-models-by-task",
|
|
1075
|
+
"name": "Recommend specialized models by task type",
|
|
1076
|
+
"category": "Model Routing Optimization",
|
|
1077
|
+
"signal": "π",
|
|
1078
|
+
"defaultAction": "allow",
|
|
1079
|
+
"severity": "medium",
|
|
1080
|
+
"pattern": "(task_type|coding|reasoning|math|code).*(model_recommendation.*missing|no.*specialized.*model)",
|
|
1081
|
+
"problem": "Different tasks benefit from specialized models: coding (Qwen3.8-Max), math (DeepSeek R1), concise reasoning (Gemini Flash). Per Nemotron: route to the right model per step.",
|
|
1082
|
+
"roi": "Maximizes capability-efficiency tradeoff. Uses cheapest capable model for each task instead of hammering expensive generalists.",
|
|
1083
|
+
"rollout": "Maintain task-to-model routing table. Auto-suggest specialized model based on task characteristics and cost-capabilities matrix."
|
|
1084
|
+
},
|
|
1085
|
+
{
|
|
1086
|
+
"id": "require-routing-evidence-for-cost-savings",
|
|
1087
|
+
"name": "Require routing evidence for cost savings claims",
|
|
1088
|
+
"category": "Cost Efficiency",
|
|
1089
|
+
"signal": "π",
|
|
1090
|
+
"defaultAction": "block",
|
|
1091
|
+
"severity": "high",
|
|
1092
|
+
"pattern": "(cost_savings|multi-model).*(claim|route|switched).*(missing|no).*(evidence|baseline|delta_metrics)",
|
|
1093
|
+
"problem": "Cost savings from multi-model routing must be measured against baseline single-model performance: output quality, latency, and actual USD saved.",
|
|
1094
|
+
"roi": "Prevents routing optimization theater. Ensures multi-model routing delivers real cost savings, not just theory like '30B MoE sounds impressive'.",
|
|
1095
|
+
"rollout": "Capture baseline metrics before enabling multi-model routing. Log actual vs expected savings. Feed to adaptive routing evaluation."
|
|
1096
|
+
},
|
|
1097
|
+
{
|
|
1098
|
+
"id": "route-high-output-tasks-to-qwen-max",
|
|
1099
|
+
"name": "Route high-output tasks to Qwen3.8-Max for cost optimization",
|
|
1100
|
+
"category": "Cost Efficiency",
|
|
1101
|
+
"signal": "π",
|
|
1102
|
+
"defaultAction": "allow",
|
|
1103
|
+
"severity": "high",
|
|
1104
|
+
"pattern": "(agentic_programming|coding_agent|bulk_content|long_reasoning|high_output).*(output_tokens.*>=\\s*100000|estimated_cost.*>=\\s*0.60|claude.*Sonnet)",
|
|
1105
|
+
"problem": "Qwen3.8-Max output pricing ($6/M) is 50-75% cheaper than Claude Sonnet 4.6/5 ($15-$10/M) and Gemini Pro ($10-$15/M). High-output agentic tasks should route to Qwen by default for cost savings.",
|
|
1106
|
+
"roi": "Realizes 50%+ output-token cost savings on bulk automation, long agentic coding runs, and repetitive high-volume tasks. $6/mo Token Plan adds further discount for steady workloads.",
|
|
1107
|
+
"rollout": "Enable cost-routing for all agentic operations. Calculate: output_tokens * $0.000006 (Qwen) vs output_tokens * $0.000010-0.000015 (Claude/Gemini). Auto-route to Qwen when cost_delta > 15%."
|
|
1108
|
+
},
|
|
1109
|
+
{
|
|
1110
|
+
"id": "require-cost-evidence-before-model-upgrade",
|
|
1111
|
+
"name": "Require cost evidence before model upgrade",
|
|
1112
|
+
"category": "Cost Efficiency",
|
|
1113
|
+
"signal": "π",
|
|
1114
|
+
"defaultAction": "block",
|
|
1115
|
+
"severity": "high",
|
|
1116
|
+
"pattern": "(upgrade|migrate|switch).*(model|provider).*(cost|token|budget).*(without|no).*baseline",
|
|
1117
|
+
"problem": "Blocks model upgrades when baseline cost-per-outcome and ROI are not documented and benchmarked first.",
|
|
1118
|
+
"roi": "Prevents expensive model sprawl by requiring measured cost evidence before adopting new providers like Qwen3.8-Max.",
|
|
1119
|
+
"rollout": "Capture token usage, latency, and quality metrics before any model change. Feed to token-plan optimization."
|
|
1120
|
+
},
|
|
1121
|
+
{
|
|
1122
|
+
"id": "block-claude-sonnet-post-priceincrease",
|
|
1123
|
+
"name": "Block or warn on Claude Sonnet post-August 2026 pricing",
|
|
1124
|
+
"category": "Cost Efficiency",
|
|
1125
|
+
"signal": "π",
|
|
1126
|
+
"defaultAction": "warn",
|
|
1127
|
+
"severity": "high",
|
|
1128
|
+
"pattern": "(model_routing|cost_optimization).*(only.*claude|claude.*Sonnet.*only).*(output_tokens.*>=\\s*50000|estimated_cost.*>=\\s*0.50)",
|
|
1129
|
+
"problem": "Claude Sonnet 5 output pricing increases to $15/M on Sept 1, 2026 (from $10/M). Bulk tasks should NOT use Claude-only routing for cost-sensitive workloads.",
|
|
1130
|
+
"roi": "Prevents $6 savings per 1M output tokens by routing high-volume work to Qwen3.8-Max ($6/M) instead of Claude Sonnet ($15/M).",
|
|
1131
|
+
"rollout": "Warn on any high-output task routed to Claude-only. Suggest Qwen3.8-Max for bulk automation. Update after Sept 1 pricing change."
|
|
1132
|
+
},
|
|
1133
|
+
{
|
|
1134
|
+
"id": "log-token-plan-compliance",
|
|
1135
|
+
"name": "Log token plan compliance for budget protection",
|
|
1136
|
+
"category": "Cost Efficiency",
|
|
1137
|
+
"signal": "π",
|
|
1138
|
+
"defaultAction": "allow",
|
|
1139
|
+
"severity": "medium",
|
|
1140
|
+
"pattern": "token_plan_usage.*>=\\s*(90|95)%",
|
|
1141
|
+
"problem": "The $6/mo Token Plan model works best when we observe usage patterns and adjust quotas before hitting limits.",
|
|
1142
|
+
"roi": "Enables proactive budget management and prevents surprise billing spikes for multimodal video generation.",
|
|
1143
|
+
"rollout": "Log when >90% of token plan used. Trigger approval workflows for quota increases or local fallback mode."
|
|
1144
|
+
},
|
|
1145
|
+
{
|
|
1146
|
+
"id": "track-token-cost-per-agentic-operation",
|
|
1147
|
+
"name": "Track token cost per agentic operation",
|
|
1148
|
+
"category": "Cost Efficiency",
|
|
1149
|
+
"signal": "π",
|
|
1150
|
+
"defaultAction": "allow",
|
|
1151
|
+
"severity": "medium",
|
|
1152
|
+
"pattern": "(token_usage_cost|input_tokens|output_tokens|token_cost).*>=\\s*(10000|50000|100000)",
|
|
1153
|
+
"problem": "Observes expensive agent operations and logs cost signals to the feedback loop for cost-optimization routing decisions.",
|
|
1154
|
+
"roi": "Enables data-driven cost optimization by correlating token burn with task outcomes, routing decisions, and ROI metrics.",
|
|
1155
|
+
"rollout": "Enable on all LLM operations. Feed cost data to hybrid-routing decisions and prompt this insight back to agents."
|
|
1156
|
+
},
|
|
1157
|
+
{
|
|
1158
|
+
"id": "require-usage-quota-for-multimodal-operations",
|
|
1159
|
+
"name": "Require usage quota check for multimodal operations",
|
|
1160
|
+
"category": "Cost Efficiency",
|
|
1161
|
+
"signal": "π",
|
|
1162
|
+
"defaultAction": "block",
|
|
1163
|
+
"severity": "high",
|
|
1164
|
+
"pattern": "(video|image|audio).*(generation|creation|edit).*(missing|no).*(quota|budget|cost limit)",
|
|
1165
|
+
"problem": "Blocks expensive multimodal generation when usage quota or cost budget would be exceeded, preventing surprise bills.",
|
|
1166
|
+
"roi": "Prevents cloud egress and expensive model operations from blowing through monthly budgets, especially for Qwen-Wan video models.",
|
|
1167
|
+
"rollout": "Set per-project quotas aligned with the Token Plan model ($0.084-$0.18/s for video). Block or warn when approaching limits."
|
|
1168
|
+
},
|
|
1169
|
+
{
|
|
1170
|
+
"id": "checkpoint-hybrid-cost-routing",
|
|
1171
|
+
"name": "Checkpoint hybrid cost-routing decisions",
|
|
1172
|
+
"category": "Cost Efficiency",
|
|
1173
|
+
"signal": "π",
|
|
1174
|
+
"defaultAction": "warn",
|
|
1175
|
+
"severity": "medium",
|
|
1176
|
+
"pattern": "(hybrid|local-cloud|perplexity.*hybrid).*(sensitive|secret|pii|codebase|customer).*(escalat|cloud|send to server)",
|
|
1177
|
+
"problem": "Requires cost-budget approval before expensive cloud escalation for local-sensitive work, balancing privacy and ROI.",
|
|
1178
|
+
"roi": "Enables hybrid cost savings while preventing accidental expensive cloud bills for sensitive data that should stay local.",
|
|
1179
|
+
"rollout": "Start as warn; capture hybrid-route feedback tags and evolve to block for PII-heavy workloads."
|
|
1180
|
+
},
|
|
1181
|
+
{
|
|
1182
|
+
"id": "gate-qwen-model-studio-egress",
|
|
1183
|
+
"name": "Gate Qwen Model Studio egress",
|
|
1184
|
+
"category": "Qwen Agent Governance",
|
|
1185
|
+
"signal": "π",
|
|
1186
|
+
"defaultAction": "warn",
|
|
1187
|
+
"severity": "medium",
|
|
1188
|
+
"pattern": "dashscope(-intl)?\\.aliyuncs\\.com",
|
|
1189
|
+
"problem": "Audits outbound model service calls to Alibaba Cloud Model Studio endpoints for telemetry & budget compliance.",
|
|
1190
|
+
"roi": "Prevents unmonitored model egress and unexpected API spend.",
|
|
1191
|
+
"rollout": "Enable when using Qwen3.8-Max or Qwen3.7-Plus models in hybrid agent runtimes."
|
|
1192
|
+
},
|
|
1193
|
+
{
|
|
1194
|
+
"id": "block-unverified-qwen-gui-actions",
|
|
1195
|
+
"name": "Block unverified Qwen visual GUI actions",
|
|
1196
|
+
"category": "Qwen Agent Governance",
|
|
1197
|
+
"signal": "π",
|
|
1198
|
+
"defaultAction": "block",
|
|
1199
|
+
"severity": "high",
|
|
1200
|
+
"pattern": "qwen_gui_action_without_verification",
|
|
1201
|
+
"problem": "Prevents Qwen visual-agent models from taking destructive GUI desktop or browser actions without prior evidence check.",
|
|
1202
|
+
"roi": "Safeguards computer-use and visual agent automation from unintended clicks.",
|
|
1203
|
+
"rollout": "Enable for Qwen3-VL-Plus and Qwen3.7-Plus agent workflows."
|
|
1204
|
+
},
|
|
1205
|
+
{
|
|
1206
|
+
"id": "require-provider-interoperability-audit",
|
|
1207
|
+
"name": "Require provider interoperability audit trail",
|
|
1208
|
+
"category": "Hybrid Inference Governance",
|
|
1209
|
+
"signal": "π",
|
|
1210
|
+
"defaultAction": "block",
|
|
1211
|
+
"severity": "high",
|
|
1212
|
+
"pattern": "(provider_switch|model_route|hybrid_route).*(deepseek|kimi|openai|claude|grok).*(no audit|no trace|no routing_log)",
|
|
1213
|
+
"problem": "Hybrid local-cloud routing across multiple providers must leave an audit trail linking input, provider choice, cost, and quality outcome.",
|
|
1214
|
+
"roi": "Enables cost-quality tradeoffs across DeepSeek, Kimi, OpenAI, Claude, Grok, and Qwen - prevents cost-optimized local decisions from being reversed by expensive cloud lookups.",
|
|
1215
|
+
"rollout": "Enable for all hybrid routing decisions. Capture provider, cost, latency, and quality in the audit trail."
|
|
1216
|
+
},
|
|
1217
|
+
{
|
|
1218
|
+
"id": "require-model-studio-readiness-check",
|
|
1219
|
+
"name": "Require Model Studio readiness check before deployment",
|
|
1220
|
+
"category": "Production AI Safety",
|
|
1221
|
+
"signal": "π",
|
|
1222
|
+
"defaultAction": "block",
|
|
1223
|
+
"severity": "critical",
|
|
1224
|
+
"pattern": "(production|deploy|go-live).*(no model.*studio|no readiness|no baseline|no traffic.*test)",
|
|
1225
|
+
"problem": "Your fastest path to production AI (Model Studio) requires baseline traffic tests, error budgets, and rollback plans before rollout.",
|
|
1226
|
+
"roi": "Prevents untested model deployments from reaching production users and damaging trust.",
|
|
1227
|
+
"rollout": "Run traffic shadow tests before any production model switch. Enable for Qwen3.8-Max and other new model deployments."
|
|
1228
|
+
},
|
|
1229
|
+
{
|
|
1230
|
+
"id": "require-multimodal-safety-gate",
|
|
1231
|
+
"name": "Require multimodal safety gate for Qwen-VL and image generation",
|
|
1232
|
+
"category": "Visual Agent Governance",
|
|
1233
|
+
"signal": "π",
|
|
1234
|
+
"defaultAction": "block",
|
|
1235
|
+
"severity": "high",
|
|
1236
|
+
"pattern": "(multimodal|vision.*model|image.*generation|video.*generation).*(no safety gate|no content filter|no guardrail)",
|
|
1237
|
+
"problem": "Qwen-VL-Plus, Qwen-Image, and video generation models need content safety filters to prevent harmful output.",
|
|
1238
|
+
"roi": "Protects brand and users from inappropriate content while enabling powerful creative generation capabilities.",
|
|
1239
|
+
"rollout": "Enable for Qwen3-VL-Plus, Qwen-Image-3.0-Pro, Wan2.7 series. Filter at input and output stages."
|
|
1240
|
+
},
|
|
1241
|
+
{
|
|
1242
|
+
"id": "recommend-qwen-for-bulk-automation",
|
|
1243
|
+
"name": "Recommend Qwen3.8-Max for bulk automation workflows",
|
|
1244
|
+
"category": "Cost Optimization",
|
|
1245
|
+
"signal": "π",
|
|
1246
|
+
"defaultAction": "allow",
|
|
1247
|
+
"severity": "medium",
|
|
1248
|
+
"pattern": "(bulk_operation|mass_generation|repetitive_task|long_running_agent).*(recommended_model.*!=.*qwen)",
|
|
1249
|
+
"problem": "For bulk, repetitive, or long-running agent workflows, Qwen3.8-Max is the optimal default: 30-60% cheaper output than Claude/Sonnet, with 1M context window for complex reasoning.",
|
|
1250
|
+
"roi": "Directs high-volume workloads to the most cost-effective model. Saves $9 per 1M output tokens vs Claude Sonnet 4.6 ($15/M).",
|
|
1251
|
+
"rollout": "Set Qwen3.8-Max as default for bulk operations. Allow explicit override with cost-delta documentation."
|
|
1252
|
+
},
|
|
1253
|
+
{
|
|
1254
|
+
"id": "github-ruleset-migration-audit",
|
|
1255
|
+
"name": "Audit GitHub Ruleset Migration & Bypass Permissions",
|
|
1256
|
+
"category": "GitHub Ruleset Governance",
|
|
1257
|
+
"signal": "π",
|
|
1258
|
+
"defaultAction": "block",
|
|
1259
|
+
"severity": "critical",
|
|
1260
|
+
"pattern": "(ruleset|branch_protection|bypass_actors)\\s+(delete|bypass|disable|override|admin_force)",
|
|
1261
|
+
"problem": "Prevents agents or un-audited processes from disabling GitHub Rulesets or modifying bypass actors on protected repositories during migration.",
|
|
1262
|
+
"roi": "Protects repository rulesets during branch-protection to rulesets migration; keeps zero-bypass policy enforceable.",
|
|
1263
|
+
"rollout": "Enable on all production repos with layered classic protection + repository rulesets."
|
|
1264
|
+
},
|
|
1265
|
+
{
|
|
1266
|
+
"id": "enforce-ruleset-bypass-actor-scoping",
|
|
1267
|
+
"name": "Enforce strict scoping on Ruleset bypass actors",
|
|
1268
|
+
"category": "GitHub Ruleset Governance",
|
|
1269
|
+
"signal": "π",
|
|
1270
|
+
"defaultAction": "block",
|
|
1271
|
+
"severity": "high",
|
|
1272
|
+
"pattern": "(ruleset_bypass|bypass_actor)\\s+(add_user|all_admins|wildcard_actor)",
|
|
1273
|
+
"problem": "Requires fine-grained app/bot scoping on Ruleset bypass permissions instead of granting broad user or admin bypass rights.",
|
|
1274
|
+
"roi": "Blocks owner-token / admin soft-bypass of merge governance; keeps agent autonomy inside protection bounds.",
|
|
1275
|
+
"rollout": "Pair with config/main-branch-ruleset.json zero bypass_actors and npm run rulesets:check."
|
|
1276
|
+
},
|
|
1277
|
+
{
|
|
1278
|
+
"id": "require-hypothesis-verify-before-claim",
|
|
1279
|
+
"name": "Require hypothesis β verify before research claims",
|
|
1280
|
+
"category": "Research Agent Governance",
|
|
1281
|
+
"signal": "π",
|
|
1282
|
+
"defaultAction": "block",
|
|
1283
|
+
"severity": "critical",
|
|
1284
|
+
"pattern": "(claim|progress_claim|research_done).*(without[_ -]?verify|no[_ -]?evidence|skip[_ -]?verify)|CLAIM_WITHOUT_VERIFY",
|
|
1285
|
+
"problem": "Stops research/RSI agents from claiming progress without a completed hypothesisβexperimentβverify cycle (EdotEnv operating pattern, ThumbGate enforcement).",
|
|
1286
|
+
"roi": "Prevents false research progress, bad post-training labels, and 'we improved' theater in multi-step agent loops.",
|
|
1287
|
+
"rollout": "Enable on research harnesses, autoresearch, and any agent that writes experiment claims. Wire scripts/research-agent-harness.js."
|
|
1288
|
+
},
|
|
1289
|
+
{
|
|
1290
|
+
"id": "ratchet-difficulty-after-verified-pass",
|
|
1291
|
+
"name": "Ratchet difficulty after verified pass",
|
|
1292
|
+
"category": "Research Agent Governance",
|
|
1293
|
+
"signal": "π",
|
|
1294
|
+
"defaultAction": "allow",
|
|
1295
|
+
"severity": "medium",
|
|
1296
|
+
"pattern": "(harder[_ -]?next[_ -]?round|difficulty[_ -]?ratchet|curriculum[_ -]?promote)",
|
|
1297
|
+
"problem": "After a full verified research cycle, the next round must get harder so agents cannot farm easy tasks forever (EdotEnv frontier transfer).",
|
|
1298
|
+
"roi": "Keeps evaluation pressure real; surfaces weak gate configs before production RSI-style loops overfit easy cases.",
|
|
1299
|
+
"rollout": "Use scripts/governance-difficulty-curriculum.js and research-agent-harness harderNextRound after CLAIM_ACCEPTED."
|
|
1300
|
+
},
|
|
1301
|
+
{
|
|
1302
|
+
"id": "require-safety-hillclimb-before-rl-post-train",
|
|
1303
|
+
"name": "Require safety hillclimb before RL post-training",
|
|
1304
|
+
"category": "Research Agent Governance",
|
|
1305
|
+
"signal": "π",
|
|
1306
|
+
"defaultAction": "block",
|
|
1307
|
+
"severity": "high",
|
|
1308
|
+
"pattern": "(post[_ -]?train|grpo|gspo|rlhf|rl_post).*(without[_ -]?hillclimb|skip[_ -]?safety|no[_ -]?eval)",
|
|
1309
|
+
"problem": "Blocks RL / GRPO / GSPO post-training launches that lack a safety hillclimb pass and verified research-cycle evidence.",
|
|
1310
|
+
"roi": "Stops reward-hacking agents from being promoted into production after only train-set wins.",
|
|
1311
|
+
"rollout": "Pair with scripts/post-training-governance.js and scripts/rsi-safety-hillclimb.js on any RL training kickoff."
|
|
663
1312
|
}
|
|
664
1313
|
]
|
|
665
1314
|
}
|