thumbgate 1.34.3 → 1.37.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. package/.agents/skills/cyberstrike-compare-not-clone/SKILL.md +36 -0
  2. package/.agents/skills/gitlab-sandbox-allowlist-not-trust/SKILL.md +77 -0
  3. package/.agents/skills/jit-harness-compare-not-clone/SKILL.md +34 -0
  4. package/.agents/skills/openui-catalog-compose-honesty/SKILL.md +64 -0
  5. package/.agents/skills/zvec-grep-compare-not-clone/SKILL.md +34 -0
  6. package/.claude-plugin/plugin.json +1 -1
  7. package/.well-known/llms.txt +1 -0
  8. package/.well-known/mcp/server-card.json +1 -1
  9. package/CONTRIBUTING.md +95 -0
  10. package/README.md +195 -632
  11. package/THIRD_PARTY_NOTICES.md +89 -0
  12. package/adapters/claude/.mcp.json +2 -2
  13. package/adapters/forge/forge.yaml +3 -3
  14. package/adapters/future-agi/.mcp.json +8 -0
  15. package/adapters/future-agi/FUTURE_AGI.md +23 -0
  16. package/adapters/future-agi/config.toml +3 -0
  17. package/adapters/future-agi/future-agi-bridge.js +9 -0
  18. package/adapters/future-agi/opencode.json +8 -0
  19. package/adapters/herdr/herdr-plugin.toml +18 -0
  20. package/adapters/mcp/server-stdio.js +238 -25
  21. package/adapters/opencode/opencode.json +1 -1
  22. package/adapters/workos/WORKOS.md +52 -0
  23. package/bin/cli.js +373 -5
  24. package/bin/futureagi-bridge +9 -0
  25. package/config/gate-templates.json +653 -4
  26. package/config/gates/actor-critic-audit.json +34 -0
  27. package/config/gates/default.json +21 -2
  28. package/config/gates/five-walls-governance.json +34 -0
  29. package/config/gates/future-agi-guardrails.json +34 -0
  30. package/config/gates/radware-threat-defense-2026.json +61 -0
  31. package/config/gates/simatree-data-governance.json +33 -0
  32. package/config/mcp-allowlists.json +4 -0
  33. package/config/merge-quality-checks.json +10 -1
  34. package/config/model-candidates.json +382 -24
  35. package/config/model-tiers.json +18 -0
  36. package/config/post-deploy-marketing-pages.json +10 -0
  37. package/config/progressive/01-wire-only.json +11 -0
  38. package/config/progressive/02-dashboard-empty-ok.json +10 -0
  39. package/config/progressive/03-one-lesson.json +10 -0
  40. package/config/progressive/04-warn-fires.json +11 -0
  41. package/config/progressive/05-strict-optional.json +11 -0
  42. package/config/progressive/README.md +15 -0
  43. package/config/schemas/broker-execution-receipt.schema.json +139 -0
  44. package/config/schemas/provider-execution-attestation-v1.schema.json +58 -0
  45. package/conformance/provider-attestation/vectors.json +320 -0
  46. package/docs/specs/provider-execution-attestation-v1.md +69 -0
  47. package/openapi/openapi.yaml +15 -0
  48. package/package.json +401 -147
  49. package/public/about.html +2 -2
  50. package/public/ai-malpractice-prevention.html +7 -7
  51. package/public/blog/a-10-dollar-vps-is-not-a-computer.html +143 -0
  52. package/public/blog/a-receipt-is-not-world-state.html +388 -0
  53. package/public/blog/git-at-agent-scale.html +374 -0
  54. package/public/blog/no-llm-in-the-gate.html +133 -0
  55. package/public/blog.html +80 -0
  56. package/public/case-studies.html +16 -1
  57. package/public/compare.html +28 -0
  58. package/public/diagnostic.html +216 -7
  59. package/public/docs/connectors.html +39 -0
  60. package/public/federal.html +2 -2
  61. package/public/founders.html +639 -0
  62. package/public/index.html +87 -9
  63. package/public/install.html +8 -8
  64. package/public/learn.html +39 -0
  65. package/public/numbers.html +2 -2
  66. package/public/peter.html +310 -0
  67. package/public/platform-partners.html +119 -0
  68. package/public/pricing.html +24 -3
  69. package/public/privacy.html +117 -0
  70. package/public/pro.html +17 -0
  71. package/public/support.html +62 -0
  72. package/public/terms.html +130 -0
  73. package/public/third-party-notices.html +95 -0
  74. package/public/yt.html +351 -0
  75. package/scripts/action-receipts.js +133 -3
  76. package/scripts/adaptive-governance-arena.js +349 -0
  77. package/scripts/admin-override.js +205 -0
  78. package/scripts/agent-action-inventory.js +869 -0
  79. package/scripts/agent-audit-trace.js +42 -2
  80. package/scripts/agent-egress-policy.js +1117 -0
  81. package/scripts/agent-memory-lifecycle.js +141 -2
  82. package/scripts/agent-operations-planner.js +441 -1
  83. package/scripts/agent-readiness.js +68 -0
  84. package/scripts/agent-security-central.js +647 -0
  85. package/scripts/allowlist-bridge-honesty.js +417 -0
  86. package/scripts/async-job-runner.js +102 -11
  87. package/scripts/audit-trail.js +212 -0
  88. package/scripts/auto-promote-gates.js +178 -27
  89. package/scripts/billing.js +1 -1
  90. package/scripts/broker-execution-receipts.js +719 -0
  91. package/scripts/budget-aware-gates-proof.js +423 -0
  92. package/scripts/claude-feedback-sync.js +29 -3
  93. package/scripts/claw-harness-production.js +237 -0
  94. package/scripts/cli-progress.js +111 -0
  95. package/scripts/cli-schema.js +163 -1
  96. package/scripts/codex-runbook-flywheel.js +318 -0
  97. package/scripts/context-footprint.js +186 -0
  98. package/scripts/contextfs.js +143 -61
  99. package/scripts/dashboard-limits.js +27 -0
  100. package/scripts/dashboard.js +279 -9
  101. package/scripts/deepseek-v4-runtime-guardrails.js +72 -6
  102. package/scripts/docker-sandbox-planner.js +18 -0
  103. package/scripts/double-blind-eval-protocol.js +252 -0
  104. package/scripts/edotenv-rl-gateway.js +259 -0
  105. package/scripts/ensure-production-search-corpus.js +162 -0
  106. package/scripts/eval-holdout.js +311 -0
  107. package/scripts/feedback-aggregate.js +21 -2
  108. package/scripts/feedback-loop.js +87 -5
  109. package/scripts/feedback-quality.js +9 -0
  110. package/scripts/file-ledger-lock.js +4 -1
  111. package/scripts/financial-control-plane.js +41 -1
  112. package/scripts/find-dormant-requires.js +118 -0
  113. package/scripts/fs-utils.js +84 -8
  114. package/scripts/gate-stats.js +2 -2
  115. package/scripts/gates-engine.js +859 -58
  116. package/scripts/generate-case-study-outreach.js +24 -15
  117. package/scripts/git-at-scale.js +628 -0
  118. package/scripts/governance-conflict-audit.js +1650 -0
  119. package/scripts/governance-difficulty-curriculum.js +328 -0
  120. package/scripts/graphrag-retrieval.js +275 -0
  121. package/scripts/gurobi-optimizer.js +324 -0
  122. package/scripts/gurobi_optimizer.py +485 -0
  123. package/scripts/harness-selector.js +82 -1
  124. package/scripts/hidden-entry-points.js +284 -0
  125. package/scripts/human-escalation.js +199 -1
  126. package/scripts/hybrid-feedback-context.js +152 -19
  127. package/scripts/intent-governed-execution.js +602 -0
  128. package/scripts/intervention-policy.js +123 -20
  129. package/scripts/jit-harness-compose.js +628 -0
  130. package/scripts/jsonl-watcher.js +10 -0
  131. package/scripts/lesson-embedding-index.js +95 -12
  132. package/scripts/lesson-retrieval.js +105 -19
  133. package/scripts/local-model-profile.js +19 -2
  134. package/scripts/mailer/resend-mailer.js +1 -1
  135. package/scripts/matryoshka-embedding.js +235 -0
  136. package/scripts/mcp-oauth.js +42 -4
  137. package/scripts/mcp-session-handles.js +1016 -0
  138. package/scripts/mcp-wiring-doctor.js +314 -0
  139. package/scripts/memory-firewall.js +115 -2
  140. package/scripts/memory-scope-readiness.js +299 -0
  141. package/scripts/memory-vs-rag-route.js +161 -0
  142. package/scripts/model-tier-router.js +148 -21
  143. package/scripts/nvidia-specdecode-al-doctor.js +536 -0
  144. package/scripts/openui-catalog-compose-honesty.js +593 -0
  145. package/scripts/operational-integrity.js +19 -1
  146. package/scripts/override-audit.js +213 -0
  147. package/scripts/package-manager-honesty-doctor.js +458 -0
  148. package/scripts/pr-manager.js +63 -1
  149. package/scripts/prove-herdr-adapter.js +52 -0
  150. package/scripts/prove-memory-pyramid-and-symbolic-canvas.js +95 -0
  151. package/scripts/prove-workos.js +73 -0
  152. package/scripts/provider-attestation-conformance.js +192 -0
  153. package/scripts/provider-receipt-contract.js +136 -0
  154. package/scripts/qwen38-max-cost-optimizer.js +401 -0
  155. package/scripts/radware-threat-defense.js +280 -0
  156. package/scripts/rag-embedding-identity.js +221 -0
  157. package/scripts/rag-precision-guardrails.js +112 -2
  158. package/scripts/remote-feedback-capture.js +159 -0
  159. package/scripts/research-agent-harness.js +256 -0
  160. package/scripts/rsi-safety-hillclimb.js +200 -0
  161. package/scripts/rule-sprawl.js +188 -0
  162. package/scripts/schedule-manager.js +147 -0
  163. package/scripts/self-heal.js +8 -0
  164. package/scripts/session-lease.js +415 -0
  165. package/scripts/simatree-data-governance.js +347 -0
  166. package/scripts/slo-alert-engine.js +172 -7
  167. package/scripts/solver-parity.js +539 -0
  168. package/scripts/stealth-memory-injection-gate.js +333 -0
  169. package/scripts/switchyard-router.js +366 -0
  170. package/scripts/telemetry-analytics.js +84 -27
  171. package/scripts/temporal-decay-weighting.js +138 -0
  172. package/scripts/test-all.js +165 -0
  173. package/scripts/token-savings.js +42 -0
  174. package/scripts/tool-kpi-tracker.js +108 -5
  175. package/scripts/tool-registry.js +193 -5
  176. package/scripts/universal-claim-evaluator.js +14 -2
  177. package/scripts/vector-store.js +279 -9
  178. package/scripts/workflow-notebook.js +391 -0
  179. package/scripts/workflow-sentinel.js +111 -12
  180. package/scripts/workos-production-guard.js +260 -0
  181. package/scripts/workspace-search-route.js +515 -0
  182. package/server.json +2 -2
  183. package/src/agent-identity-boundary.js +76 -0
  184. package/src/agent-retrieval-cache.js +155 -0
  185. package/src/alert-noise-ledger.js +502 -0
  186. package/src/api/server.js +802 -185
  187. package/src/git-fast-cache.js +220 -0
  188. package/src/git-wal-sync.js +156 -0
  189. package/src/hash-anchored-edit.js +82 -0
  190. package/src/hermes-platform-protocol.js +475 -0
  191. package/src/hermes-sync-plane.js +241 -0
  192. package/src/index.js +30 -1
  193. package/src/iso42001-compliance-guard.js +97 -0
  194. package/src/latency-budget.js +244 -0
  195. package/src/mcp-writeguard.js +316 -0
  196. package/src/miminions-adapter.js +106 -0
  197. package/src/pipeline-compass.js +104 -0
  198. package/src/ppl-alert-pipeline.js +284 -0
  199. package/src/rendezvous-router.js +90 -0
  200. package/src/security-questionnaire.js +195 -0
@@ -0,0 +1,311 @@
1
+ #!/usr/bin/env node
2
+ 'use strict';
3
+
4
+ /**
5
+ * eval-holdout.js — double-blind holdout seal for the mined gate-decisions benchmark.
6
+ *
7
+ * Problem (Google DeepMind, 2026): benchmark contamination inflates scores — research
8
+ * found signs of leakage in about half of 31 models tested. DeepMind's answer was the
9
+ * first double-blind evaluation (confidential GPU + PySyft): the evaluator never sees
10
+ * the model weights, the provider never sees the questions, and only permitted results
11
+ * leave the enclave.
12
+ *
13
+ * Our local adaptation, no confidential hardware required:
14
+ *
15
+ * Blind 1 (questions hidden from the tuning loop): the golden set is partitioned
16
+ * deterministically (sha256 of the canonical case key) into a PUBLIC slice
17
+ * that development may tune against, and a HOLDOUT slice whose questions are
18
+ * never written down in plaintext. The holdout manifest stores only case
19
+ * hashes and sealed verdicts.
20
+ * Blind 2 (state hidden from the evaluator): holdout replay runs the engine in an
21
+ * isolated sandbox with empty feedback/memory state — the learned corpus
22
+ * cannot leak into holdout verdicts (same discipline as eval-baseline.js).
23
+ *
24
+ * Modes:
25
+ * --freeze derive the partition, replay the holdout slice in a sandbox, and write
26
+ * evals/gate-decisions.holdout.json (caseHash + sealed verdict only —
27
+ * never the raw command). Re-freeze deliberately, like eval:baseline.
28
+ * --check verify the seal WITHOUT replaying anything: partition reproducibility,
29
+ * no case swapped in/out, no raw question text leaked into the manifest,
30
+ * content hash intact. CI-safe; exit 1 on any violation.
31
+ * --verify replay the holdout slice and compare against the sealed verdicts.
32
+ * Deliberately reports only the aggregate (compared/drifted) — the
33
+ * "permitted results" pattern. Per-case detail stays inside the script.
34
+ *
35
+ * The hash scheme is the contract: caseHash = sha256(toolName + '\u0000' + command),
36
+ * hex. A case is holdout when its first 8 hex chars fall under holdoutRatio * 2^32.
37
+ */
38
+
39
+ const fs = require('node:fs');
40
+ const os = require('node:os');
41
+ const path = require('node:path');
42
+ const crypto = require('node:crypto');
43
+ const { execFileSync } = require('node:child_process');
44
+
45
+ const GOLDEN = process.env.THUMBGATE_EVAL_GOLDEN
46
+ || path.join(__dirname, '..', 'evals', 'gate-decisions.golden.jsonl');
47
+ const HOLDOUT_MANIFEST = process.env.THUMBGATE_EVAL_HOLDOUT
48
+ || path.join(__dirname, '..', 'evals', 'gate-decisions.holdout.json');
49
+ const DEFAULT_HOLDOUT_RATIO = 1 / 3;
50
+
51
+ function caseHash(toolName, command) {
52
+ return crypto.createHash('sha256')
53
+ .update(`${toolName}\u0000${command}`)
54
+ .digest('hex');
55
+ }
56
+
57
+ function isHoldout(hash, ratio = DEFAULT_HOLDOUT_RATIO) {
58
+ const bucket = parseInt(hash.slice(0, 8), 16);
59
+ return bucket < Math.floor(ratio * 0x100000000);
60
+ }
61
+
62
+ function loadGolden(goldenPath = GOLDEN) {
63
+ if (!fs.existsSync(goldenPath)) {
64
+ throw new Error(`no golden set at ${goldenPath} — run: npm run eval:mine`);
65
+ }
66
+ return fs.readFileSync(goldenPath, 'utf8')
67
+ .split('\n').filter(Boolean)
68
+ .map((line) => { try { return JSON.parse(line); } catch { return null; } })
69
+ .filter(Boolean);
70
+ }
71
+
72
+ function partition(cases, ratio = DEFAULT_HOLDOUT_RATIO) {
73
+ const holdout = [];
74
+ const publicCases = [];
75
+ for (const c of cases) {
76
+ const hash = caseHash(c.toolName, c.command);
77
+ (isHoldout(hash, ratio) ? holdout : publicCases).push({ ...c, caseHash: hash });
78
+ }
79
+ return { holdout, publicCases };
80
+ }
81
+
82
+ function contentHash(manifest) {
83
+ const body = { ...manifest };
84
+ delete body.contentHash;
85
+ return crypto.createHash('sha256')
86
+ .update(JSON.stringify(body))
87
+ .digest('hex');
88
+ }
89
+
90
+ // --- Sandbox replay (blind 2): isolated engine state, empty learned corpus ---
91
+
92
+ function makeSandbox() {
93
+ const gatesEngine = require('./gates-engine.js');
94
+ const original = {
95
+ STATE_PATH: gatesEngine.STATE_PATH,
96
+ STATS_PATH: gatesEngine.STATS_PATH,
97
+ CONSTRAINTS_PATH: gatesEngine.CONSTRAINTS_PATH,
98
+ SESSION_ACTIONS_PATH: gatesEngine.SESSION_ACTIONS_PATH,
99
+ GOVERNANCE_STATE_PATH: gatesEngine.GOVERNANCE_STATE_PATH,
100
+ };
101
+ const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'tg-holdout-state-'));
102
+ gatesEngine.STATE_PATH = path.join(stateDir, 'gate-state.json');
103
+ gatesEngine.STATS_PATH = path.join(stateDir, 'gate-stats.json');
104
+ gatesEngine.CONSTRAINTS_PATH = path.join(stateDir, 'session-constraints.json');
105
+ gatesEngine.SESSION_ACTIONS_PATH = path.join(stateDir, 'session-actions.json');
106
+ gatesEngine.GOVERNANCE_STATE_PATH = path.join(stateDir, 'governance-state.json');
107
+
108
+ const emptyFeedbackDir = path.join(stateDir, 'empty-feedback');
109
+ fs.mkdirSync(emptyFeedbackDir, { recursive: true });
110
+ const origFeedback = process.env.THUMBGATE_FEEDBACK_DIR;
111
+ const origMemory = process.env.CLAUDE_MEMORY_DIR;
112
+ process.env.THUMBGATE_FEEDBACK_DIR = emptyFeedbackDir;
113
+ process.env.CLAUDE_MEMORY_DIR = emptyFeedbackDir;
114
+
115
+ const repo = fs.mkdtempSync(path.join(os.tmpdir(), 'tg-holdout-repo-'));
116
+ const GIT_BIN = ['/usr/bin/git', '/usr/local/bin/git', '/opt/homebrew/bin/git']
117
+ .find((candidate) => fs.existsSync(candidate));
118
+ if (!GIT_BIN) throw new Error('no git binary at a known absolute path');
119
+ const git = (a) => execFileSync(GIT_BIN, a, {
120
+ cwd: repo,
121
+ stdio: ['ignore', 'ignore', 'ignore'],
122
+ env: { ...process.env, PATH: '/usr/local/bin:/usr/bin:/bin' },
123
+ });
124
+ git(['init']); git(['config', 'user.email', 't@example.com']); git(['config', 'user.name', 't']);
125
+ fs.writeFileSync(path.join(repo, 'seed.txt'), 'seed\n');
126
+ git(['add', 'seed.txt']); git(['commit', '-m', 'init']);
127
+ fs.mkdirSync(path.join(repo, 'src'), { recursive: true });
128
+ fs.writeFileSync(path.join(repo, 'src', 'app.js'), 'code\n');
129
+
130
+ return {
131
+ gatesEngine,
132
+ repo,
133
+ dispose() {
134
+ Object.assign(gatesEngine, original);
135
+ if (origFeedback !== undefined) process.env.THUMBGATE_FEEDBACK_DIR = origFeedback;
136
+ else delete process.env.THUMBGATE_FEEDBACK_DIR;
137
+ if (origMemory !== undefined) process.env.CLAUDE_MEMORY_DIR = origMemory;
138
+ else delete process.env.CLAUDE_MEMORY_DIR;
139
+ fs.rmSync(stateDir, { recursive: true, force: true });
140
+ fs.rmSync(repo, { recursive: true, force: true });
141
+ },
142
+ };
143
+ }
144
+
145
+ async function replayHoldout(holdoutCases) {
146
+ const sandbox = makeSandbox();
147
+ try {
148
+ const verdicts = new Map();
149
+ for (const c of holdoutCases) {
150
+ const v = await sandbox.gatesEngine.evaluateGatesAsync(c.toolName, {
151
+ command: c.command,
152
+ cwd: sandbox.repo,
153
+ });
154
+ verdicts.set(c.caseHash, v ? v.decision : 'none');
155
+ }
156
+ return verdicts;
157
+ } finally {
158
+ sandbox.dispose();
159
+ }
160
+ }
161
+
162
+ // --- Modes ---
163
+
164
+ async function freeze(ratio = DEFAULT_HOLDOUT_RATIO) {
165
+ const cases = loadGolden();
166
+ const { holdout, publicCases } = partition(cases, ratio);
167
+ const verdicts = await replayHoldout(holdout);
168
+
169
+ const manifest = {
170
+ version: 1,
171
+ scheme: 'sha256(toolName + NUL + command); holdout when first 8 hex < ratio * 2^32',
172
+ holdoutRatio: ratio,
173
+ holdoutThreshold: Math.floor(ratio * 0x100000000),
174
+ goldenCases: cases.length,
175
+ holdoutCases: holdout.length,
176
+ publicCases: publicCases.length,
177
+ holdout: holdout.map((c) => ({
178
+ caseHash: c.caseHash,
179
+ gateId: c.expect?.gateId || null,
180
+ sealedVerdict: verdicts.get(c.caseHash),
181
+ })),
182
+ sealedAt: new Date().toISOString(),
183
+ };
184
+ manifest.contentHash = contentHash(manifest);
185
+ fs.writeFileSync(HOLDOUT_MANIFEST, `${JSON.stringify(manifest, null, 2)}\n`);
186
+ process.stdout.write(
187
+ `Holdout sealed: ${holdout.length}/${cases.length} cases held out `
188
+ + `(public ${publicCases.length}) -> ${HOLDOUT_MANIFEST}\n`,
189
+ );
190
+ return 0;
191
+ }
192
+
193
+ function check() {
194
+ if (!fs.existsSync(HOLDOUT_MANIFEST)) {
195
+ process.stderr.write('eval-holdout check: no manifest — run: npm run eval:holdout:freeze\n');
196
+ return 2;
197
+ }
198
+ const raw = fs.readFileSync(HOLDOUT_MANIFEST, 'utf8');
199
+ const manifest = JSON.parse(raw);
200
+ const violations = [];
201
+
202
+ // 1. Content hash intact (manifest not hand-edited after sealing).
203
+ if (manifest.contentHash !== contentHash(manifest)) {
204
+ violations.push('contentHash mismatch — manifest edited after sealing');
205
+ }
206
+
207
+ // 2. Partition reproducibility: the manifest must describe exactly the holdout
208
+ // slice the hash scheme derives from the CURRENT golden set. Catches both
209
+ // golden-set edits that smuggle cases in/out of the holdout and manifest swaps.
210
+ const cases = loadGolden();
211
+ const { holdout: derivedHoldout } = partition(cases, manifest.holdoutRatio ?? DEFAULT_HOLDOUT_RATIO);
212
+ const derived = new Set(derivedHoldout.map((c) => c.caseHash));
213
+ const sealed = new Set((manifest.holdout || []).map((e) => e.caseHash));
214
+ if (manifest.goldenCases !== cases.length) {
215
+ violations.push(`goldenCases ${manifest.goldenCases} != current ${cases.length}`);
216
+ }
217
+ for (const hash of derived) {
218
+ if (!sealed.has(hash)) violations.push(`derived holdout case ${hash.slice(0, 12)}… missing from manifest`);
219
+ }
220
+ for (const hash of sealed) {
221
+ if (!derived.has(hash)) violations.push(`manifest case ${hash.slice(0, 12)}… not in derived holdout`);
222
+ }
223
+ if (sealed.size === 0) violations.push('holdout is empty — nothing is sealed');
224
+
225
+ // 3. Leak guard: the questions must not appear in the manifest. The seal is
226
+ // worthless if raw commands were pasted in alongside their hashes.
227
+ for (const c of cases) {
228
+ const probe = String(c.command || '').slice(0, 40);
229
+ if (probe.length >= 20 && raw.includes(probe)) {
230
+ violations.push(`raw question text leaked into manifest (${probe.slice(0, 20)}…)`);
231
+ break;
232
+ }
233
+ }
234
+
235
+ if (violations.length > 0) {
236
+ process.stderr.write(`eval-holdout check: SEAL BROKEN\n ${violations.join('\n ')}\n`);
237
+ return 1;
238
+ }
239
+ process.stdout.write(
240
+ `Holdout seal intact: ${sealed.size}/${cases.length} cases sealed, no question leakage.\n`,
241
+ );
242
+ return 0;
243
+ }
244
+
245
+ async function verify() {
246
+ if (!fs.existsSync(HOLDOUT_MANIFEST)) {
247
+ process.stderr.write('eval-holdout verify: no manifest — run: npm run eval:holdout:freeze\n');
248
+ return 2;
249
+ }
250
+ const manifest = JSON.parse(fs.readFileSync(HOLDOUT_MANIFEST, 'utf8'));
251
+ const cases = loadGolden();
252
+ const { holdout } = partition(cases, manifest.holdoutRatio ?? DEFAULT_HOLDOUT_RATIO);
253
+ const byHash = new Map(holdout.map((c) => [c.caseHash, c]));
254
+ const verdicts = await replayHoldout(holdout);
255
+
256
+ const sealedCount = (manifest.holdout || []).length;
257
+ let compared = 0;
258
+ let drifted = 0;
259
+ for (const entry of manifest.holdout || []) {
260
+ const c = byHash.get(entry.caseHash);
261
+ if (!c) continue;
262
+ compared += 1;
263
+ if (verdicts.get(entry.caseHash) !== entry.sealedVerdict) drifted += 1;
264
+ }
265
+ if (sealedCount === 0 || compared !== sealedCount) {
266
+ process.stderr.write(
267
+ `Holdout verify: partition drift — compared ${compared} of ${sealedCount} sealed cases. `
268
+ + 'Run: npm run eval:holdout:check\n',
269
+ );
270
+ return 1;
271
+ }
272
+ // Permitted-results pattern: only the aggregate leaves this function. Per-case
273
+ // drift detail is deliberately not printed, so the holdout stays un-memorable.
274
+ process.stdout.write(
275
+ `Holdout verify: compared=${compared} drifted=${drifted} (sealed set)\n`,
276
+ );
277
+ if (drifted > 0) {
278
+ process.stderr.write(
279
+ 'Holdout verdicts moved. If the change is intended, re-freeze deliberately: npm run eval:holdout:freeze\n',
280
+ );
281
+ return 1;
282
+ }
283
+ return 0;
284
+ }
285
+
286
+ async function main() {
287
+ const args = new Set(process.argv.slice(2));
288
+ if (args.has('--freeze')) return freeze();
289
+ if (args.has('--check')) return check();
290
+ if (args.has('--verify')) return verify();
291
+ process.stderr.write('usage: eval-holdout.js --freeze | --check | --verify\n');
292
+ return 2;
293
+ }
294
+
295
+ module.exports = {
296
+ caseHash,
297
+ isHoldout,
298
+ partition,
299
+ contentHash,
300
+ loadGolden,
301
+ freeze,
302
+ check,
303
+ verify,
304
+ GOLDEN,
305
+ HOLDOUT_MANIFEST,
306
+ DEFAULT_HOLDOUT_RATIO,
307
+ };
308
+
309
+ if (require.main?.filename === module.filename) {
310
+ main().then((c) => process.exit(c));
311
+ }
@@ -71,11 +71,15 @@ function immediateChildDirs(parentDir) {
71
71
  function ancestorProjectFeedbackDirs(projectDir, options = {}) {
72
72
  const home = normalizePath(getHomeDir(options));
73
73
  const start = normalizePath(projectDir);
74
+ const tempRoot = normalizePath(os.tmpdir());
74
75
  if (!start) return [];
75
76
 
76
77
  const dirs = [];
77
78
  let cursor = start;
78
79
  while (cursor && cursor !== path.dirname(cursor)) {
80
+ // Temp projects share one OS-level parent. Reading that parent's store leaks
81
+ // feedback across unrelated test runs, npx sessions, and sandboxed agents.
82
+ if (tempRoot && cursor === tempRoot) break;
79
83
  dirs.push(
80
84
  path.join(cursor, '.thumbgate'),
81
85
  path.join(cursor, '.thumbgate-compat'),
@@ -159,7 +163,21 @@ function collectAggregateLogEntries(fileName, options = {}) {
159
163
  for (const dir of stores) {
160
164
  const logPath = path.join(dir, fileName);
161
165
  if (!safeExists(logPath)) continue;
162
- const rows = readJsonl(logPath, { maxLines: 0 }) || [];
166
+ // Never full-scan multi-hundred-MB feedback logs (prod dashboard_data 503).
167
+ // Callers that truly need lifetime history must pass { full: true }.
168
+ const wantFull = options.full === true;
169
+ const maxLines = wantFull
170
+ ? Math.max(0, Number(options.maxLines) || 0)
171
+ : (Number(options.maxLines) > 0 ? Number(options.maxLines) : 20_000);
172
+ const maxBytes = wantFull
173
+ ? Math.max(0, Number(options.maxBytes) || 0)
174
+ : (Number(options.maxBytes) > 0 ? Number(options.maxBytes) : 4 * 1024 * 1024);
175
+ const rows = readJsonl(logPath, {
176
+ maxLines,
177
+ maxBytes,
178
+ tail: !wantFull,
179
+ full: wantFull,
180
+ }) || [];
163
181
  for (let index = 0; index < rows.length; index += 1) {
164
182
  const rawEntry = rows[index];
165
183
  const entry = { ...rawEntry };
@@ -185,7 +203,7 @@ function summarizeFeedbackEntries(entries) {
185
203
  for (const entry of entries) {
186
204
  if (entry.signal === 'positive') totalPositive += 1;
187
205
  if (entry.signal === 'negative') totalNegative += 1;
188
- if (entry.rubric && entry.rubric.weightedScore != null) rubricSamples += 1;
206
+ if (entry.rubric?.weightedScore != null) rubricSamples += 1;
189
207
  }
190
208
 
191
209
  return { totalPositive, totalNegative, rubricSamples };
@@ -275,6 +293,7 @@ function getAggregateStatuslineCachePath(options = {}) {
275
293
  }
276
294
 
277
295
  module.exports = {
296
+ ancestorProjectFeedbackDirs,
278
297
  collectAggregateLogEntries,
279
298
  computeAggregateFeedbackStats,
280
299
  getAggregateStatuslineCachePath,
@@ -46,7 +46,7 @@ const {
46
46
  aggregateFailureDiagnostics,
47
47
  } = require('./failure-diagnostics');
48
48
  const { getEffectiveSetting } = require('./evolution-state');
49
- const { ensureDir } = require('./fs-utils');
49
+ const { ensureDir, readTextTail } = require('./fs-utils');
50
50
  const {
51
51
  buildFeedbackPathsFromDir,
52
52
  getFeedbackPaths: resolveFeedbackPaths,
@@ -569,6 +569,7 @@ function maybeBlockMemoryIngress({ feedbackEvent, memoryRecord = null, summary,
569
569
  firewallResult: decision.firewallResult || null,
570
570
  blockedPatterns: Array.isArray(decision.blockedPatterns) ? decision.blockedPatterns : [],
571
571
  requestedProvider: decision.requestedProvider || null,
572
+ memoryInjectionTelemetry: decision.telemetry || null,
572
573
  },
573
574
  diagnosis: {
574
575
  diagnosed: true,
@@ -597,6 +598,8 @@ function maybeBlockMemoryIngress({ feedbackEvent, memoryRecord = null, summary,
597
598
  mode: decision.mode || null,
598
599
  threatIndicators: decision.threatIndicators || [],
599
600
  degraded: Boolean(decision.degraded),
601
+ provenance: decision.provenance || null,
602
+ telemetry: decision.telemetry || null,
600
603
  },
601
604
  };
602
605
  }
@@ -634,11 +637,22 @@ function getPendingBackgroundSideEffectCount() {
634
637
  return pendingBackgroundSideEffects.size;
635
638
  }
636
639
 
637
- function readJSONL(filePath, { maxLines = 500 } = {}) {
640
+ function readJSONL(filePath, { maxLines = 500, maxBytes = 4 * 1024 * 1024 } = {}) {
638
641
  if (!fs.existsSync(filePath)) return [];
639
- const content = fs.readFileSync(filePath, 'utf8');
642
+ // Always go through readTextTail. A maxLines slice after readFileSync still
643
+ // materializes the whole file and throws on prod-sized JSONL (0x1fffffe8).
644
+ let content = '';
645
+ try {
646
+ content = maxBytes > 0
647
+ ? readTextTail(filePath, maxBytes).text
648
+ : readTextTail(filePath, 0).text;
649
+ } catch {
650
+ return [];
651
+ }
640
652
  const lines = content.split('\n').filter(Boolean);
641
- const tail = maxLines > 0 ? lines.slice(-maxLines) : lines;
653
+ // maxLines <= 0 used to mean "all lines"; keep a hard safety cap for callers.
654
+ const effectiveMax = maxLines > 0 ? maxLines : 20_000;
655
+ const tail = lines.slice(-effectiveMax);
642
656
  const results = [];
643
657
  for (const line of tail) {
644
658
  try { results.push(JSON.parse(line)); } catch { /* skip malformed */ }
@@ -1441,6 +1455,13 @@ function captureFeedback(params) {
1441
1455
  gateAction: params.gateAction || null,
1442
1456
  reviewOrigin: normalizeReviewOrigin(params.reviewOrigin),
1443
1457
  sourceEvent: publicFeedbackSourceMetadata(params.sourceEvent),
1458
+ memorySource: params.memorySource && typeof params.memorySource === 'object'
1459
+ ? {
1460
+ type: params.memorySource.type || null,
1461
+ identifier: params.memorySource.identifier || null,
1462
+ trust: params.memorySource.trust || null,
1463
+ }
1464
+ : null,
1444
1465
  timestamp: now,
1445
1466
  };
1446
1467
 
@@ -1526,6 +1547,61 @@ function captureFeedback(params) {
1526
1547
  };
1527
1548
  }
1528
1549
 
1550
+ // Lifecycle hard blocks for transport transcripts and oversized blobs must
1551
+ // reject promotion before memory-log writes. Secret-like content intentionally
1552
+ // continues to the ingress firewall below, which blocks all raw writes and
1553
+ // emits only redacted diagnostics.
1554
+ try {
1555
+ const { evaluateMemoryPromotion } = require('./agent-memory-lifecycle');
1556
+ const lifecycle = evaluateMemoryPromotion({
1557
+ type: action.memory && action.memory.type,
1558
+ content: [
1559
+ action.memory && action.memory.title,
1560
+ action.memory && action.memory.content,
1561
+ context,
1562
+ whatWentWrong,
1563
+ whatWorked,
1564
+ whatToChange,
1565
+ ].filter(Boolean).join('\n'),
1566
+ source: feedbackEvent.sourceEvent || feedbackEvent.id || 'capture_feedback',
1567
+ outcome: whatWorked || whatWentWrong || action.reason || null,
1568
+ explicitUserSignal: signal === 'positive' || signal === 'negative',
1569
+ allowWorkingPromotion: false,
1570
+ });
1571
+ if (lifecycle && lifecycle.decision !== 'promote') {
1572
+ const hardIssues = (lifecycle.issues || []).filter((issue) =>
1573
+ issue === 'transport_transcript'
1574
+ || issue === 'oversized_blob'
1575
+ );
1576
+ if (hardIssues.length > 0) {
1577
+ summary.rejected += 1;
1578
+ summary.lastUpdated = now;
1579
+ saveSummary(summary);
1580
+ const blockedEvent = {
1581
+ ...feedbackEvent,
1582
+ lifecycleDecision: lifecycle.decision,
1583
+ lifecycleIssues: lifecycle.issues,
1584
+ };
1585
+ appendJSONL(FEEDBACK_LOG_PATH, blockedEvent);
1586
+ emitAnonymousFeedbackPing(signal);
1587
+ return {
1588
+ accepted: true,
1589
+ captured: true,
1590
+ promoted: false,
1591
+ promotionAccepted: false,
1592
+ signalLogged: true,
1593
+ status: 'rejected',
1594
+ reason: `Memory lifecycle blocked promotion: ${hardIssues.join(', ')}`,
1595
+ message: 'Signal logged, but reusable memory was not created (lifecycle policy).',
1596
+ feedbackEvent: blockedEvent,
1597
+ lifecycle,
1598
+ };
1599
+ }
1600
+ }
1601
+ } catch (_lifecycleErr) {
1602
+ /* lifecycle evaluation is fail-open for soft issues only; hard path above is explicit */
1603
+ }
1604
+
1529
1605
  const prepared = prepareForStorage(action.memory);
1530
1606
  if (!prepared.ok) {
1531
1607
  const firewallBlocked = maybeBlockMemoryIngress({ feedbackEvent, summary, now });
@@ -2176,7 +2252,13 @@ function analyzeFeedback(logPath, options = {}) {
2176
2252
  const feedbackDir = path.dirname(resolvedPath);
2177
2253
  const paths = buildFeedbackPathsFromDir(feedbackDir);
2178
2254
  const useSQLite = !options.humanOnly && (!logPath || path.resolve(resolvedPath) === path.resolve(FEEDBACK_LOG_PATH));
2179
- let entries = readJSONL(resolvedPath, { maxLines: 0 });
2255
+ // Prefer caller-supplied entries (dashboard already bounded the read).
2256
+ let entries = Array.isArray(options.entries)
2257
+ ? options.entries.slice()
2258
+ : readJSONL(resolvedPath, {
2259
+ maxLines: Number(options.maxLines) > 0 ? Number(options.maxLines) : 20_000,
2260
+ maxBytes: Number(options.maxBytes) > 0 ? Number(options.maxBytes) : 4 * 1024 * 1024,
2261
+ });
2180
2262
  const rawTotal = entries.length;
2181
2263
  if (options.humanOnly) {
2182
2264
  entries = entries.filter((entry) => normalizeReviewOrigin(entry.reviewOrigin) === 'human');
@@ -114,6 +114,11 @@ function detectFeedbackSignal(value) {
114
114
  /^that failed\b/,
115
115
  /^it failed\b/,
116
116
  /^that was wrong\b/,
117
+ /^that was bad\b/,
118
+ /^this is wrong\b/,
119
+ /^th(?:is|at) (?:response|answer) (?:is|was) (?:inadequate|wrong|bad|incorrect)\b/,
120
+ /^that did(?: not|n t) work\b/,
121
+ /^that does(?: not|n t) work\b/,
117
122
  ];
118
123
  if (exactDown.some((pattern) => pattern.test(normalized))) {
119
124
  return { signal: 'down', confidence: 'exact', match: normalized };
@@ -130,6 +135,10 @@ function detectFeedbackSignal(value) {
130
135
  /^nice work\b/,
131
136
  /^perfect\b/,
132
137
  /^lgtm\b/,
138
+ /^great job\b/,
139
+ /^great work\b/,
140
+ /^well done\b/,
141
+ /^nicely done\b/,
133
142
  ];
134
143
  if (exactUp.some((pattern) => pattern.test(normalized))) {
135
144
  return { signal: 'up', confidence: 'exact', match: normalized };
@@ -36,12 +36,15 @@ function acquireLock(lockPath, options) {
36
36
  nonce: crypto.randomUUID(),
37
37
  acquiredAt: now.toISOString(),
38
38
  };
39
- for (let attempt = 0; attempt < 3; attempt += 1) {
39
+ for (let attempt = 0; attempt < 5; attempt += 1) {
40
40
  try {
41
41
  fs.mkdirSync(lockPath);
42
42
  writeOwner(lockPath, owner);
43
43
  return owner;
44
44
  } catch (error) {
45
+ // ENOENT: another recoverStaleLock removed the lock dir between mkdir and
46
+ // writeOwner — treat as a lost race and retry acquisition.
47
+ if (error.code === 'ENOENT') continue;
45
48
  if (error.code !== 'EEXIST') throw error;
46
49
  if (!recoverStaleLock(lockPath, now, staleMs)) {
47
50
  throw lockError(options, 'ledger is busy; deny and retry only after the active writer finishes');
@@ -40,7 +40,40 @@ const CONTROL_PLANE_TOOLS = new Set([
40
40
  'reserve_purchase_requisition',
41
41
  'settle_purchase_requisition',
42
42
  'reconcile_purchase_ledger',
43
+ 'satisfy_gate',
44
+ 'capture_feedback',
45
+ 'capture_memory_feedback',
46
+ 'record_task_outcome',
47
+ 'record_action_receipt',
48
+ 'get_scope_state',
49
+ 'set_task_scope',
50
+ 'report_product_issue',
51
+ 'open_feedback_session',
52
+ 'append_feedback_context',
53
+ 'finalize_feedback_session',
54
+ 'search_lessons',
55
+ 'retrieve_lessons',
56
+ 'verify_claim',
57
+ 'track_action',
43
58
  ]);
59
+ const REMEDY_TOOLS = new Set([
60
+ 'satisfy_gate',
61
+ 'capture_feedback',
62
+ 'capture_memory_feedback',
63
+ 'record_task_outcome',
64
+ 'diagnose_failure',
65
+ 'set_task_scope',
66
+ 'approve_protected_action',
67
+ 'track_action',
68
+ 'verify_claim',
69
+ 'break_glass_emergency',
70
+ ]);
71
+
72
+ function isRemedyControlTool(toolName) {
73
+ const normalized = String(toolName || '').trim();
74
+ const bare = normalized.replace(/^mcp__.+?__/, '');
75
+ return REMEDY_TOOLS.has(bare) || CONTROL_PLANE_TOOLS.has(bare);
76
+ }
44
77
 
45
78
  // Keep these expressions deliberately small and auditable. The second group
46
79
  // covers provider-native noun-first CLI forms such as `subscriptions create`.
@@ -339,6 +372,7 @@ function advanceFinancialLedgerAnchor(previousHead, event, options = {}) {
339
372
 
340
373
  function detectEconomicAction(toolName, toolInput = {}) {
341
374
  const normalizedToolName = String(toolName || '').trim();
375
+ if (isRemedyControlTool(normalizedToolName)) return false;
342
376
  if ([...CONTROL_PLANE_TOOLS].some((name) => (
343
377
  normalizedToolName === name || normalizedToolName.endsWith(`__${name}`)
344
378
  ))) return false;
@@ -423,7 +457,12 @@ function shellEconomicText(toolName, rawCommand) {
423
457
  .map((segment) => {
424
458
  const clean = segment.trim().replace(/^(?:sudo\s+)?(?:env\s+)?(?:[A-Za-z_]\w*=[^\s]+\s+)*/, '');
425
459
  const executable = /^([\w./-]+)/.exec(clean)?.[1]?.split('/').at(-1)?.toLowerCase();
426
- if (['rg', 'grep', 'git', 'cat', 'head', 'tail', 'less', 'more', 'echo', 'printf'].includes(executable)) return '';
460
+ if (['rg', 'grep', 'git', 'cat', 'head', 'tail', 'less', 'more', 'echo', 'printf'].includes(executable)) {
461
+ return '';
462
+ }
463
+ if (executable === 'gh' && /\b(?:issue|pr)\s+(?:create|view|list|status|edit|comment)\b/i.test(clean)) {
464
+ return '';
465
+ }
427
466
  return clean;
428
467
  })
429
468
  .join(' ');
@@ -1607,6 +1646,7 @@ module.exports = {
1607
1646
  createPurchaseRequisition,
1608
1647
  buildActionAuthorization,
1609
1648
  detectEconomicAction,
1649
+ isRemedyControlTool,
1610
1650
  detectOpaqueScreenMutation,
1611
1651
  evaluateFinancialControl,
1612
1652
  getLedgerHeadPath,